1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 32 #include <asm/apic.h> 33 #include <asm/stacktrace.h> 34 #include <asm/nmi.h> 35 #include <asm/smp.h> 36 #include <asm/alternative.h> 37 #include <asm/mmu_context.h> 38 #include <asm/tlbflush.h> 39 #include <asm/timer.h> 40 #include <asm/desc.h> 41 #include <asm/ldt.h> 42 #include <asm/unwind.h> 43 44 #include "perf_event.h" 45 46 struct x86_pmu x86_pmu __read_mostly; 47 48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 49 .enabled = 1, 50 }; 51 52 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 53 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 54 55 u64 __read_mostly hw_cache_event_ids 56 [PERF_COUNT_HW_CACHE_MAX] 57 [PERF_COUNT_HW_CACHE_OP_MAX] 58 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 59 u64 __read_mostly hw_cache_extra_regs 60 [PERF_COUNT_HW_CACHE_MAX] 61 [PERF_COUNT_HW_CACHE_OP_MAX] 62 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 63 64 /* 65 * Propagate event elapsed time into the generic event. 66 * Can only be executed on the CPU where the event is active. 67 * Returns the delta events processed. 68 */ 69 u64 x86_perf_event_update(struct perf_event *event) 70 { 71 struct hw_perf_event *hwc = &event->hw; 72 int shift = 64 - x86_pmu.cntval_bits; 73 u64 prev_raw_count, new_raw_count; 74 int idx = hwc->idx; 75 u64 delta; 76 77 if (idx == INTEL_PMC_IDX_FIXED_BTS) 78 return 0; 79 80 /* 81 * Careful: an NMI might modify the previous event value. 82 * 83 * Our tactic to handle this is to first atomically read and 84 * exchange a new raw count - then add that new-prev delta 85 * count to the generic event atomically: 86 */ 87 again: 88 prev_raw_count = local64_read(&hwc->prev_count); 89 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 90 91 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 92 new_raw_count) != prev_raw_count) 93 goto again; 94 95 /* 96 * Now we have the new raw value and have updated the prev 97 * timestamp already. We can now calculate the elapsed delta 98 * (event-)time and add that to the generic event. 99 * 100 * Careful, not all hw sign-extends above the physical width 101 * of the count. 102 */ 103 delta = (new_raw_count << shift) - (prev_raw_count << shift); 104 delta >>= shift; 105 106 local64_add(delta, &event->count); 107 local64_sub(delta, &hwc->period_left); 108 109 return new_raw_count; 110 } 111 112 /* 113 * Find and validate any extra registers to set up. 114 */ 115 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 116 { 117 struct hw_perf_event_extra *reg; 118 struct extra_reg *er; 119 120 reg = &event->hw.extra_reg; 121 122 if (!x86_pmu.extra_regs) 123 return 0; 124 125 for (er = x86_pmu.extra_regs; er->msr; er++) { 126 if (er->event != (config & er->config_mask)) 127 continue; 128 if (event->attr.config1 & ~er->valid_mask) 129 return -EINVAL; 130 /* Check if the extra msrs can be safely accessed*/ 131 if (!er->extra_msr_access) 132 return -ENXIO; 133 134 reg->idx = er->idx; 135 reg->config = event->attr.config1; 136 reg->reg = er->msr; 137 break; 138 } 139 return 0; 140 } 141 142 static atomic_t active_events; 143 static atomic_t pmc_refcount; 144 static DEFINE_MUTEX(pmc_reserve_mutex); 145 146 #ifdef CONFIG_X86_LOCAL_APIC 147 148 static bool reserve_pmc_hardware(void) 149 { 150 int i; 151 152 for (i = 0; i < x86_pmu.num_counters; i++) { 153 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 154 goto perfctr_fail; 155 } 156 157 for (i = 0; i < x86_pmu.num_counters; i++) { 158 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 159 goto eventsel_fail; 160 } 161 162 return true; 163 164 eventsel_fail: 165 for (i--; i >= 0; i--) 166 release_evntsel_nmi(x86_pmu_config_addr(i)); 167 168 i = x86_pmu.num_counters; 169 170 perfctr_fail: 171 for (i--; i >= 0; i--) 172 release_perfctr_nmi(x86_pmu_event_addr(i)); 173 174 return false; 175 } 176 177 static void release_pmc_hardware(void) 178 { 179 int i; 180 181 for (i = 0; i < x86_pmu.num_counters; i++) { 182 release_perfctr_nmi(x86_pmu_event_addr(i)); 183 release_evntsel_nmi(x86_pmu_config_addr(i)); 184 } 185 } 186 187 #else 188 189 static bool reserve_pmc_hardware(void) { return true; } 190 static void release_pmc_hardware(void) {} 191 192 #endif 193 194 static bool check_hw_exists(void) 195 { 196 u64 val, val_fail = -1, val_new= ~0; 197 int i, reg, reg_fail = -1, ret = 0; 198 int bios_fail = 0; 199 int reg_safe = -1; 200 201 /* 202 * Check to see if the BIOS enabled any of the counters, if so 203 * complain and bail. 204 */ 205 for (i = 0; i < x86_pmu.num_counters; i++) { 206 reg = x86_pmu_config_addr(i); 207 ret = rdmsrl_safe(reg, &val); 208 if (ret) 209 goto msr_fail; 210 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 211 bios_fail = 1; 212 val_fail = val; 213 reg_fail = reg; 214 } else { 215 reg_safe = i; 216 } 217 } 218 219 if (x86_pmu.num_counters_fixed) { 220 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 221 ret = rdmsrl_safe(reg, &val); 222 if (ret) 223 goto msr_fail; 224 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 225 if (val & (0x03 << i*4)) { 226 bios_fail = 1; 227 val_fail = val; 228 reg_fail = reg; 229 } 230 } 231 } 232 233 /* 234 * If all the counters are enabled, the below test will always 235 * fail. The tools will also become useless in this scenario. 236 * Just fail and disable the hardware counters. 237 */ 238 239 if (reg_safe == -1) { 240 reg = reg_safe; 241 goto msr_fail; 242 } 243 244 /* 245 * Read the current value, change it and read it back to see if it 246 * matches, this is needed to detect certain hardware emulators 247 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 248 */ 249 reg = x86_pmu_event_addr(reg_safe); 250 if (rdmsrl_safe(reg, &val)) 251 goto msr_fail; 252 val ^= 0xffffUL; 253 ret = wrmsrl_safe(reg, val); 254 ret |= rdmsrl_safe(reg, &val_new); 255 if (ret || val != val_new) 256 goto msr_fail; 257 258 /* 259 * We still allow the PMU driver to operate: 260 */ 261 if (bios_fail) { 262 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 263 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 264 reg_fail, val_fail); 265 } 266 267 return true; 268 269 msr_fail: 270 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 271 pr_cont("PMU not available due to virtualization, using software events only.\n"); 272 } else { 273 pr_cont("Broken PMU hardware detected, using software events only.\n"); 274 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 275 reg, val_new); 276 } 277 278 return false; 279 } 280 281 static void hw_perf_event_destroy(struct perf_event *event) 282 { 283 x86_release_hardware(); 284 atomic_dec(&active_events); 285 } 286 287 void hw_perf_lbr_event_destroy(struct perf_event *event) 288 { 289 hw_perf_event_destroy(event); 290 291 /* undo the lbr/bts event accounting */ 292 x86_del_exclusive(x86_lbr_exclusive_lbr); 293 } 294 295 static inline int x86_pmu_initialized(void) 296 { 297 return x86_pmu.handle_irq != NULL; 298 } 299 300 static inline int 301 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 302 { 303 struct perf_event_attr *attr = &event->attr; 304 unsigned int cache_type, cache_op, cache_result; 305 u64 config, val; 306 307 config = attr->config; 308 309 cache_type = (config >> 0) & 0xff; 310 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 311 return -EINVAL; 312 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 313 314 cache_op = (config >> 8) & 0xff; 315 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 316 return -EINVAL; 317 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 318 319 cache_result = (config >> 16) & 0xff; 320 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 321 return -EINVAL; 322 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 323 324 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 325 326 if (val == 0) 327 return -ENOENT; 328 329 if (val == -1) 330 return -EINVAL; 331 332 hwc->config |= val; 333 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 334 return x86_pmu_extra_regs(val, event); 335 } 336 337 int x86_reserve_hardware(void) 338 { 339 int err = 0; 340 341 if (!atomic_inc_not_zero(&pmc_refcount)) { 342 mutex_lock(&pmc_reserve_mutex); 343 if (atomic_read(&pmc_refcount) == 0) { 344 if (!reserve_pmc_hardware()) 345 err = -EBUSY; 346 else 347 reserve_ds_buffers(); 348 } 349 if (!err) 350 atomic_inc(&pmc_refcount); 351 mutex_unlock(&pmc_reserve_mutex); 352 } 353 354 return err; 355 } 356 357 void x86_release_hardware(void) 358 { 359 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 360 release_pmc_hardware(); 361 release_ds_buffers(); 362 mutex_unlock(&pmc_reserve_mutex); 363 } 364 } 365 366 /* 367 * Check if we can create event of a certain type (that no conflicting events 368 * are present). 369 */ 370 int x86_add_exclusive(unsigned int what) 371 { 372 int i; 373 374 /* 375 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 376 * LBR and BTS are still mutually exclusive. 377 */ 378 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 379 goto out; 380 381 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 382 mutex_lock(&pmc_reserve_mutex); 383 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 384 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 385 goto fail_unlock; 386 } 387 atomic_inc(&x86_pmu.lbr_exclusive[what]); 388 mutex_unlock(&pmc_reserve_mutex); 389 } 390 391 out: 392 atomic_inc(&active_events); 393 return 0; 394 395 fail_unlock: 396 mutex_unlock(&pmc_reserve_mutex); 397 return -EBUSY; 398 } 399 400 void x86_del_exclusive(unsigned int what) 401 { 402 atomic_dec(&active_events); 403 404 /* 405 * See the comment in x86_add_exclusive(). 406 */ 407 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 408 return; 409 410 atomic_dec(&x86_pmu.lbr_exclusive[what]); 411 } 412 413 int x86_setup_perfctr(struct perf_event *event) 414 { 415 struct perf_event_attr *attr = &event->attr; 416 struct hw_perf_event *hwc = &event->hw; 417 u64 config; 418 419 if (!is_sampling_event(event)) { 420 hwc->sample_period = x86_pmu.max_period; 421 hwc->last_period = hwc->sample_period; 422 local64_set(&hwc->period_left, hwc->sample_period); 423 } 424 425 if (attr->type == PERF_TYPE_RAW) 426 return x86_pmu_extra_regs(event->attr.config, event); 427 428 if (attr->type == PERF_TYPE_HW_CACHE) 429 return set_ext_hw_attr(hwc, event); 430 431 if (attr->config >= x86_pmu.max_events) 432 return -EINVAL; 433 434 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 435 436 /* 437 * The generic map: 438 */ 439 config = x86_pmu.event_map(attr->config); 440 441 if (config == 0) 442 return -ENOENT; 443 444 if (config == -1LL) 445 return -EINVAL; 446 447 hwc->config |= config; 448 449 return 0; 450 } 451 452 /* 453 * check that branch_sample_type is compatible with 454 * settings needed for precise_ip > 1 which implies 455 * using the LBR to capture ALL taken branches at the 456 * priv levels of the measurement 457 */ 458 static inline int precise_br_compat(struct perf_event *event) 459 { 460 u64 m = event->attr.branch_sample_type; 461 u64 b = 0; 462 463 /* must capture all branches */ 464 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 465 return 0; 466 467 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 468 469 if (!event->attr.exclude_user) 470 b |= PERF_SAMPLE_BRANCH_USER; 471 472 if (!event->attr.exclude_kernel) 473 b |= PERF_SAMPLE_BRANCH_KERNEL; 474 475 /* 476 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 477 */ 478 479 return m == b; 480 } 481 482 int x86_pmu_max_precise(void) 483 { 484 int precise = 0; 485 486 /* Support for constant skid */ 487 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 488 precise++; 489 490 /* Support for IP fixup */ 491 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 492 precise++; 493 494 if (x86_pmu.pebs_prec_dist) 495 precise++; 496 } 497 return precise; 498 } 499 500 int x86_pmu_hw_config(struct perf_event *event) 501 { 502 if (event->attr.precise_ip) { 503 int precise = x86_pmu_max_precise(); 504 505 if (event->attr.precise_ip > precise) 506 return -EOPNOTSUPP; 507 508 /* There's no sense in having PEBS for non sampling events: */ 509 if (!is_sampling_event(event)) 510 return -EINVAL; 511 } 512 /* 513 * check that PEBS LBR correction does not conflict with 514 * whatever the user is asking with attr->branch_sample_type 515 */ 516 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 517 u64 *br_type = &event->attr.branch_sample_type; 518 519 if (has_branch_stack(event)) { 520 if (!precise_br_compat(event)) 521 return -EOPNOTSUPP; 522 523 /* branch_sample_type is compatible */ 524 525 } else { 526 /* 527 * user did not specify branch_sample_type 528 * 529 * For PEBS fixups, we capture all 530 * the branches at the priv level of the 531 * event. 532 */ 533 *br_type = PERF_SAMPLE_BRANCH_ANY; 534 535 if (!event->attr.exclude_user) 536 *br_type |= PERF_SAMPLE_BRANCH_USER; 537 538 if (!event->attr.exclude_kernel) 539 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 540 } 541 } 542 543 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 544 event->attach_state |= PERF_ATTACH_TASK_DATA; 545 546 /* 547 * Generate PMC IRQs: 548 * (keep 'enabled' bit clear for now) 549 */ 550 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 551 552 /* 553 * Count user and OS events unless requested not to 554 */ 555 if (!event->attr.exclude_user) 556 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 557 if (!event->attr.exclude_kernel) 558 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 559 560 if (event->attr.type == PERF_TYPE_RAW) 561 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 562 563 if (event->attr.sample_period && x86_pmu.limit_period) { 564 if (x86_pmu.limit_period(event, event->attr.sample_period) > 565 event->attr.sample_period) 566 return -EINVAL; 567 } 568 569 /* sample_regs_user never support XMM registers */ 570 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) 571 return -EINVAL; 572 /* 573 * Besides the general purpose registers, XMM registers may 574 * be collected in PEBS on some platforms, e.g. Icelake 575 */ 576 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { 577 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 578 return -EINVAL; 579 580 if (!event->attr.precise_ip) 581 return -EINVAL; 582 } 583 584 return x86_setup_perfctr(event); 585 } 586 587 /* 588 * Setup the hardware configuration for a given attr_type 589 */ 590 static int __x86_pmu_event_init(struct perf_event *event) 591 { 592 int err; 593 594 if (!x86_pmu_initialized()) 595 return -ENODEV; 596 597 err = x86_reserve_hardware(); 598 if (err) 599 return err; 600 601 atomic_inc(&active_events); 602 event->destroy = hw_perf_event_destroy; 603 604 event->hw.idx = -1; 605 event->hw.last_cpu = -1; 606 event->hw.last_tag = ~0ULL; 607 608 /* mark unused */ 609 event->hw.extra_reg.idx = EXTRA_REG_NONE; 610 event->hw.branch_reg.idx = EXTRA_REG_NONE; 611 612 return x86_pmu.hw_config(event); 613 } 614 615 void x86_pmu_disable_all(void) 616 { 617 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 618 int idx; 619 620 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 621 u64 val; 622 623 if (!test_bit(idx, cpuc->active_mask)) 624 continue; 625 rdmsrl(x86_pmu_config_addr(idx), val); 626 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 627 continue; 628 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 629 wrmsrl(x86_pmu_config_addr(idx), val); 630 } 631 } 632 633 /* 634 * There may be PMI landing after enabled=0. The PMI hitting could be before or 635 * after disable_all. 636 * 637 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 638 * It will not be re-enabled in the NMI handler again, because enabled=0. After 639 * handling the NMI, disable_all will be called, which will not change the 640 * state either. If PMI hits after disable_all, the PMU is already disabled 641 * before entering NMI handler. The NMI handler will not change the state 642 * either. 643 * 644 * So either situation is harmless. 645 */ 646 static void x86_pmu_disable(struct pmu *pmu) 647 { 648 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 649 650 if (!x86_pmu_initialized()) 651 return; 652 653 if (!cpuc->enabled) 654 return; 655 656 cpuc->n_added = 0; 657 cpuc->enabled = 0; 658 barrier(); 659 660 x86_pmu.disable_all(); 661 } 662 663 void x86_pmu_enable_all(int added) 664 { 665 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 666 int idx; 667 668 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 669 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 670 671 if (!test_bit(idx, cpuc->active_mask)) 672 continue; 673 674 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 675 } 676 } 677 678 static struct pmu pmu; 679 680 static inline int is_x86_event(struct perf_event *event) 681 { 682 return event->pmu == &pmu; 683 } 684 685 struct pmu *x86_get_pmu(void) 686 { 687 return &pmu; 688 } 689 /* 690 * Event scheduler state: 691 * 692 * Assign events iterating over all events and counters, beginning 693 * with events with least weights first. Keep the current iterator 694 * state in struct sched_state. 695 */ 696 struct sched_state { 697 int weight; 698 int event; /* event index */ 699 int counter; /* counter index */ 700 int unassigned; /* number of events to be assigned left */ 701 int nr_gp; /* number of GP counters used */ 702 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 703 }; 704 705 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 706 #define SCHED_STATES_MAX 2 707 708 struct perf_sched { 709 int max_weight; 710 int max_events; 711 int max_gp; 712 int saved_states; 713 struct event_constraint **constraints; 714 struct sched_state state; 715 struct sched_state saved[SCHED_STATES_MAX]; 716 }; 717 718 /* 719 * Initialize interator that runs through all events and counters. 720 */ 721 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 722 int num, int wmin, int wmax, int gpmax) 723 { 724 int idx; 725 726 memset(sched, 0, sizeof(*sched)); 727 sched->max_events = num; 728 sched->max_weight = wmax; 729 sched->max_gp = gpmax; 730 sched->constraints = constraints; 731 732 for (idx = 0; idx < num; idx++) { 733 if (constraints[idx]->weight == wmin) 734 break; 735 } 736 737 sched->state.event = idx; /* start with min weight */ 738 sched->state.weight = wmin; 739 sched->state.unassigned = num; 740 } 741 742 static void perf_sched_save_state(struct perf_sched *sched) 743 { 744 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 745 return; 746 747 sched->saved[sched->saved_states] = sched->state; 748 sched->saved_states++; 749 } 750 751 static bool perf_sched_restore_state(struct perf_sched *sched) 752 { 753 if (!sched->saved_states) 754 return false; 755 756 sched->saved_states--; 757 sched->state = sched->saved[sched->saved_states]; 758 759 /* continue with next counter: */ 760 clear_bit(sched->state.counter++, sched->state.used); 761 762 return true; 763 } 764 765 /* 766 * Select a counter for the current event to schedule. Return true on 767 * success. 768 */ 769 static bool __perf_sched_find_counter(struct perf_sched *sched) 770 { 771 struct event_constraint *c; 772 int idx; 773 774 if (!sched->state.unassigned) 775 return false; 776 777 if (sched->state.event >= sched->max_events) 778 return false; 779 780 c = sched->constraints[sched->state.event]; 781 /* Prefer fixed purpose counters */ 782 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 783 idx = INTEL_PMC_IDX_FIXED; 784 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 785 if (!__test_and_set_bit(idx, sched->state.used)) 786 goto done; 787 } 788 } 789 790 /* Grab the first unused counter starting with idx */ 791 idx = sched->state.counter; 792 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 793 if (!__test_and_set_bit(idx, sched->state.used)) { 794 if (sched->state.nr_gp++ >= sched->max_gp) 795 return false; 796 797 goto done; 798 } 799 } 800 801 return false; 802 803 done: 804 sched->state.counter = idx; 805 806 if (c->overlap) 807 perf_sched_save_state(sched); 808 809 return true; 810 } 811 812 static bool perf_sched_find_counter(struct perf_sched *sched) 813 { 814 while (!__perf_sched_find_counter(sched)) { 815 if (!perf_sched_restore_state(sched)) 816 return false; 817 } 818 819 return true; 820 } 821 822 /* 823 * Go through all unassigned events and find the next one to schedule. 824 * Take events with the least weight first. Return true on success. 825 */ 826 static bool perf_sched_next_event(struct perf_sched *sched) 827 { 828 struct event_constraint *c; 829 830 if (!sched->state.unassigned || !--sched->state.unassigned) 831 return false; 832 833 do { 834 /* next event */ 835 sched->state.event++; 836 if (sched->state.event >= sched->max_events) { 837 /* next weight */ 838 sched->state.event = 0; 839 sched->state.weight++; 840 if (sched->state.weight > sched->max_weight) 841 return false; 842 } 843 c = sched->constraints[sched->state.event]; 844 } while (c->weight != sched->state.weight); 845 846 sched->state.counter = 0; /* start with first counter */ 847 848 return true; 849 } 850 851 /* 852 * Assign a counter for each event. 853 */ 854 int perf_assign_events(struct event_constraint **constraints, int n, 855 int wmin, int wmax, int gpmax, int *assign) 856 { 857 struct perf_sched sched; 858 859 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 860 861 do { 862 if (!perf_sched_find_counter(&sched)) 863 break; /* failed */ 864 if (assign) 865 assign[sched.state.event] = sched.state.counter; 866 } while (perf_sched_next_event(&sched)); 867 868 return sched.state.unassigned; 869 } 870 EXPORT_SYMBOL_GPL(perf_assign_events); 871 872 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 873 { 874 struct event_constraint *c; 875 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 876 struct perf_event *e; 877 int n0, i, wmin, wmax, unsched = 0; 878 struct hw_perf_event *hwc; 879 880 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 881 882 /* 883 * Compute the number of events already present; see x86_pmu_add(), 884 * validate_group() and x86_pmu_commit_txn(). For the former two 885 * cpuc->n_events hasn't been updated yet, while for the latter 886 * cpuc->n_txn contains the number of events added in the current 887 * transaction. 888 */ 889 n0 = cpuc->n_events; 890 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 891 n0 -= cpuc->n_txn; 892 893 if (x86_pmu.start_scheduling) 894 x86_pmu.start_scheduling(cpuc); 895 896 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 897 c = cpuc->event_constraint[i]; 898 899 /* 900 * Previously scheduled events should have a cached constraint, 901 * while new events should not have one. 902 */ 903 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 904 905 /* 906 * Request constraints for new events; or for those events that 907 * have a dynamic constraint -- for those the constraint can 908 * change due to external factors (sibling state, allow_tfa). 909 */ 910 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 911 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 912 cpuc->event_constraint[i] = c; 913 } 914 915 wmin = min(wmin, c->weight); 916 wmax = max(wmax, c->weight); 917 } 918 919 /* 920 * fastpath, try to reuse previous register 921 */ 922 for (i = 0; i < n; i++) { 923 hwc = &cpuc->event_list[i]->hw; 924 c = cpuc->event_constraint[i]; 925 926 /* never assigned */ 927 if (hwc->idx == -1) 928 break; 929 930 /* constraint still honored */ 931 if (!test_bit(hwc->idx, c->idxmsk)) 932 break; 933 934 /* not already used */ 935 if (test_bit(hwc->idx, used_mask)) 936 break; 937 938 __set_bit(hwc->idx, used_mask); 939 if (assign) 940 assign[i] = hwc->idx; 941 } 942 943 /* slow path */ 944 if (i != n) { 945 int gpmax = x86_pmu.num_counters; 946 947 /* 948 * Do not allow scheduling of more than half the available 949 * generic counters. 950 * 951 * This helps avoid counter starvation of sibling thread by 952 * ensuring at most half the counters cannot be in exclusive 953 * mode. There is no designated counters for the limits. Any 954 * N/2 counters can be used. This helps with events with 955 * specific counter constraints. 956 */ 957 if (is_ht_workaround_enabled() && !cpuc->is_fake && 958 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 959 gpmax /= 2; 960 961 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 962 wmax, gpmax, assign); 963 } 964 965 /* 966 * In case of success (unsched = 0), mark events as committed, 967 * so we do not put_constraint() in case new events are added 968 * and fail to be scheduled 969 * 970 * We invoke the lower level commit callback to lock the resource 971 * 972 * We do not need to do all of this in case we are called to 973 * validate an event group (assign == NULL) 974 */ 975 if (!unsched && assign) { 976 for (i = 0; i < n; i++) { 977 e = cpuc->event_list[i]; 978 if (x86_pmu.commit_scheduling) 979 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 980 } 981 } else { 982 for (i = n0; i < n; i++) { 983 e = cpuc->event_list[i]; 984 985 /* 986 * release events that failed scheduling 987 */ 988 if (x86_pmu.put_event_constraints) 989 x86_pmu.put_event_constraints(cpuc, e); 990 991 cpuc->event_constraint[i] = NULL; 992 } 993 } 994 995 if (x86_pmu.stop_scheduling) 996 x86_pmu.stop_scheduling(cpuc); 997 998 return unsched ? -EINVAL : 0; 999 } 1000 1001 /* 1002 * dogrp: true if must collect siblings events (group) 1003 * returns total number of events and error code 1004 */ 1005 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1006 { 1007 struct perf_event *event; 1008 int n, max_count; 1009 1010 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 1011 1012 /* current number of events already accepted */ 1013 n = cpuc->n_events; 1014 if (!cpuc->n_events) 1015 cpuc->pebs_output = 0; 1016 1017 if (!cpuc->is_fake && leader->attr.precise_ip) { 1018 /* 1019 * For PEBS->PT, if !aux_event, the group leader (PT) went 1020 * away, the group was broken down and this singleton event 1021 * can't schedule any more. 1022 */ 1023 if (is_pebs_pt(leader) && !leader->aux_event) 1024 return -EINVAL; 1025 1026 /* 1027 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1028 */ 1029 if (cpuc->pebs_output && 1030 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1031 return -EINVAL; 1032 1033 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1034 } 1035 1036 if (is_x86_event(leader)) { 1037 if (n >= max_count) 1038 return -EINVAL; 1039 cpuc->event_list[n] = leader; 1040 n++; 1041 } 1042 if (!dogrp) 1043 return n; 1044 1045 for_each_sibling_event(event, leader) { 1046 if (!is_x86_event(event) || 1047 event->state <= PERF_EVENT_STATE_OFF) 1048 continue; 1049 1050 if (n >= max_count) 1051 return -EINVAL; 1052 1053 cpuc->event_list[n] = event; 1054 n++; 1055 } 1056 return n; 1057 } 1058 1059 static inline void x86_assign_hw_event(struct perf_event *event, 1060 struct cpu_hw_events *cpuc, int i) 1061 { 1062 struct hw_perf_event *hwc = &event->hw; 1063 1064 hwc->idx = cpuc->assign[i]; 1065 hwc->last_cpu = smp_processor_id(); 1066 hwc->last_tag = ++cpuc->tags[i]; 1067 1068 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1069 hwc->config_base = 0; 1070 hwc->event_base = 0; 1071 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1072 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1073 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1074 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1075 } else { 1076 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1077 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1078 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1079 } 1080 } 1081 1082 /** 1083 * x86_perf_rdpmc_index - Return PMC counter used for event 1084 * @event: the perf_event to which the PMC counter was assigned 1085 * 1086 * The counter assigned to this performance event may change if interrupts 1087 * are enabled. This counter should thus never be used while interrupts are 1088 * enabled. Before this function is used to obtain the assigned counter the 1089 * event should be checked for validity using, for example, 1090 * perf_event_read_local(), within the same interrupt disabled section in 1091 * which this counter is planned to be used. 1092 * 1093 * Return: The index of the performance monitoring counter assigned to 1094 * @perf_event. 1095 */ 1096 int x86_perf_rdpmc_index(struct perf_event *event) 1097 { 1098 lockdep_assert_irqs_disabled(); 1099 1100 return event->hw.event_base_rdpmc; 1101 } 1102 1103 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1104 struct cpu_hw_events *cpuc, 1105 int i) 1106 { 1107 return hwc->idx == cpuc->assign[i] && 1108 hwc->last_cpu == smp_processor_id() && 1109 hwc->last_tag == cpuc->tags[i]; 1110 } 1111 1112 static void x86_pmu_start(struct perf_event *event, int flags); 1113 1114 static void x86_pmu_enable(struct pmu *pmu) 1115 { 1116 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1117 struct perf_event *event; 1118 struct hw_perf_event *hwc; 1119 int i, added = cpuc->n_added; 1120 1121 if (!x86_pmu_initialized()) 1122 return; 1123 1124 if (cpuc->enabled) 1125 return; 1126 1127 if (cpuc->n_added) { 1128 int n_running = cpuc->n_events - cpuc->n_added; 1129 /* 1130 * apply assignment obtained either from 1131 * hw_perf_group_sched_in() or x86_pmu_enable() 1132 * 1133 * step1: save events moving to new counters 1134 */ 1135 for (i = 0; i < n_running; i++) { 1136 event = cpuc->event_list[i]; 1137 hwc = &event->hw; 1138 1139 /* 1140 * we can avoid reprogramming counter if: 1141 * - assigned same counter as last time 1142 * - running on same CPU as last time 1143 * - no other event has used the counter since 1144 */ 1145 if (hwc->idx == -1 || 1146 match_prev_assignment(hwc, cpuc, i)) 1147 continue; 1148 1149 /* 1150 * Ensure we don't accidentally enable a stopped 1151 * counter simply because we rescheduled. 1152 */ 1153 if (hwc->state & PERF_HES_STOPPED) 1154 hwc->state |= PERF_HES_ARCH; 1155 1156 x86_pmu_stop(event, PERF_EF_UPDATE); 1157 } 1158 1159 /* 1160 * step2: reprogram moved events into new counters 1161 */ 1162 for (i = 0; i < cpuc->n_events; i++) { 1163 event = cpuc->event_list[i]; 1164 hwc = &event->hw; 1165 1166 if (!match_prev_assignment(hwc, cpuc, i)) 1167 x86_assign_hw_event(event, cpuc, i); 1168 else if (i < n_running) 1169 continue; 1170 1171 if (hwc->state & PERF_HES_ARCH) 1172 continue; 1173 1174 x86_pmu_start(event, PERF_EF_RELOAD); 1175 } 1176 cpuc->n_added = 0; 1177 perf_events_lapic_init(); 1178 } 1179 1180 cpuc->enabled = 1; 1181 barrier(); 1182 1183 x86_pmu.enable_all(added); 1184 } 1185 1186 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1187 1188 /* 1189 * Set the next IRQ period, based on the hwc->period_left value. 1190 * To be called with the event disabled in hw: 1191 */ 1192 int x86_perf_event_set_period(struct perf_event *event) 1193 { 1194 struct hw_perf_event *hwc = &event->hw; 1195 s64 left = local64_read(&hwc->period_left); 1196 s64 period = hwc->sample_period; 1197 int ret = 0, idx = hwc->idx; 1198 1199 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1200 return 0; 1201 1202 /* 1203 * If we are way outside a reasonable range then just skip forward: 1204 */ 1205 if (unlikely(left <= -period)) { 1206 left = period; 1207 local64_set(&hwc->period_left, left); 1208 hwc->last_period = period; 1209 ret = 1; 1210 } 1211 1212 if (unlikely(left <= 0)) { 1213 left += period; 1214 local64_set(&hwc->period_left, left); 1215 hwc->last_period = period; 1216 ret = 1; 1217 } 1218 /* 1219 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1220 */ 1221 if (unlikely(left < 2)) 1222 left = 2; 1223 1224 if (left > x86_pmu.max_period) 1225 left = x86_pmu.max_period; 1226 1227 if (x86_pmu.limit_period) 1228 left = x86_pmu.limit_period(event, left); 1229 1230 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1231 1232 /* 1233 * The hw event starts counting from this event offset, 1234 * mark it to be able to extra future deltas: 1235 */ 1236 local64_set(&hwc->prev_count, (u64)-left); 1237 1238 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1239 1240 /* 1241 * Due to erratum on certan cpu we need 1242 * a second write to be sure the register 1243 * is updated properly 1244 */ 1245 if (x86_pmu.perfctr_second_write) { 1246 wrmsrl(hwc->event_base, 1247 (u64)(-left) & x86_pmu.cntval_mask); 1248 } 1249 1250 perf_event_update_userpage(event); 1251 1252 return ret; 1253 } 1254 1255 void x86_pmu_enable_event(struct perf_event *event) 1256 { 1257 if (__this_cpu_read(cpu_hw_events.enabled)) 1258 __x86_pmu_enable_event(&event->hw, 1259 ARCH_PERFMON_EVENTSEL_ENABLE); 1260 } 1261 1262 /* 1263 * Add a single event to the PMU. 1264 * 1265 * The event is added to the group of enabled events 1266 * but only if it can be scheduled with existing events. 1267 */ 1268 static int x86_pmu_add(struct perf_event *event, int flags) 1269 { 1270 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1271 struct hw_perf_event *hwc; 1272 int assign[X86_PMC_IDX_MAX]; 1273 int n, n0, ret; 1274 1275 hwc = &event->hw; 1276 1277 n0 = cpuc->n_events; 1278 ret = n = collect_events(cpuc, event, false); 1279 if (ret < 0) 1280 goto out; 1281 1282 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1283 if (!(flags & PERF_EF_START)) 1284 hwc->state |= PERF_HES_ARCH; 1285 1286 /* 1287 * If group events scheduling transaction was started, 1288 * skip the schedulability test here, it will be performed 1289 * at commit time (->commit_txn) as a whole. 1290 * 1291 * If commit fails, we'll call ->del() on all events 1292 * for which ->add() was called. 1293 */ 1294 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1295 goto done_collect; 1296 1297 ret = x86_pmu.schedule_events(cpuc, n, assign); 1298 if (ret) 1299 goto out; 1300 /* 1301 * copy new assignment, now we know it is possible 1302 * will be used by hw_perf_enable() 1303 */ 1304 memcpy(cpuc->assign, assign, n*sizeof(int)); 1305 1306 done_collect: 1307 /* 1308 * Commit the collect_events() state. See x86_pmu_del() and 1309 * x86_pmu_*_txn(). 1310 */ 1311 cpuc->n_events = n; 1312 cpuc->n_added += n - n0; 1313 cpuc->n_txn += n - n0; 1314 1315 if (x86_pmu.add) { 1316 /* 1317 * This is before x86_pmu_enable() will call x86_pmu_start(), 1318 * so we enable LBRs before an event needs them etc.. 1319 */ 1320 x86_pmu.add(event); 1321 } 1322 1323 ret = 0; 1324 out: 1325 return ret; 1326 } 1327 1328 static void x86_pmu_start(struct perf_event *event, int flags) 1329 { 1330 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1331 int idx = event->hw.idx; 1332 1333 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1334 return; 1335 1336 if (WARN_ON_ONCE(idx == -1)) 1337 return; 1338 1339 if (flags & PERF_EF_RELOAD) { 1340 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1341 x86_perf_event_set_period(event); 1342 } 1343 1344 event->hw.state = 0; 1345 1346 cpuc->events[idx] = event; 1347 __set_bit(idx, cpuc->active_mask); 1348 __set_bit(idx, cpuc->running); 1349 x86_pmu.enable(event); 1350 perf_event_update_userpage(event); 1351 } 1352 1353 void perf_event_print_debug(void) 1354 { 1355 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1356 u64 pebs, debugctl; 1357 struct cpu_hw_events *cpuc; 1358 unsigned long flags; 1359 int cpu, idx; 1360 1361 if (!x86_pmu.num_counters) 1362 return; 1363 1364 local_irq_save(flags); 1365 1366 cpu = smp_processor_id(); 1367 cpuc = &per_cpu(cpu_hw_events, cpu); 1368 1369 if (x86_pmu.version >= 2) { 1370 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1371 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1372 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1373 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1374 1375 pr_info("\n"); 1376 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1377 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1378 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1379 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1380 if (x86_pmu.pebs_constraints) { 1381 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1382 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1383 } 1384 if (x86_pmu.lbr_nr) { 1385 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1386 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1387 } 1388 } 1389 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1390 1391 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1392 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1393 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1394 1395 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1396 1397 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1398 cpu, idx, pmc_ctrl); 1399 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1400 cpu, idx, pmc_count); 1401 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1402 cpu, idx, prev_left); 1403 } 1404 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1405 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1406 1407 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1408 cpu, idx, pmc_count); 1409 } 1410 local_irq_restore(flags); 1411 } 1412 1413 void x86_pmu_stop(struct perf_event *event, int flags) 1414 { 1415 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1416 struct hw_perf_event *hwc = &event->hw; 1417 1418 if (test_bit(hwc->idx, cpuc->active_mask)) { 1419 x86_pmu.disable(event); 1420 __clear_bit(hwc->idx, cpuc->active_mask); 1421 cpuc->events[hwc->idx] = NULL; 1422 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1423 hwc->state |= PERF_HES_STOPPED; 1424 } 1425 1426 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1427 /* 1428 * Drain the remaining delta count out of a event 1429 * that we are disabling: 1430 */ 1431 x86_perf_event_update(event); 1432 hwc->state |= PERF_HES_UPTODATE; 1433 } 1434 } 1435 1436 static void x86_pmu_del(struct perf_event *event, int flags) 1437 { 1438 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1439 int i; 1440 1441 /* 1442 * If we're called during a txn, we only need to undo x86_pmu.add. 1443 * The events never got scheduled and ->cancel_txn will truncate 1444 * the event_list. 1445 * 1446 * XXX assumes any ->del() called during a TXN will only be on 1447 * an event added during that same TXN. 1448 */ 1449 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1450 goto do_del; 1451 1452 /* 1453 * Not a TXN, therefore cleanup properly. 1454 */ 1455 x86_pmu_stop(event, PERF_EF_UPDATE); 1456 1457 for (i = 0; i < cpuc->n_events; i++) { 1458 if (event == cpuc->event_list[i]) 1459 break; 1460 } 1461 1462 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1463 return; 1464 1465 /* If we have a newly added event; make sure to decrease n_added. */ 1466 if (i >= cpuc->n_events - cpuc->n_added) 1467 --cpuc->n_added; 1468 1469 if (x86_pmu.put_event_constraints) 1470 x86_pmu.put_event_constraints(cpuc, event); 1471 1472 /* Delete the array entry. */ 1473 while (++i < cpuc->n_events) { 1474 cpuc->event_list[i-1] = cpuc->event_list[i]; 1475 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1476 } 1477 cpuc->event_constraint[i-1] = NULL; 1478 --cpuc->n_events; 1479 1480 perf_event_update_userpage(event); 1481 1482 do_del: 1483 if (x86_pmu.del) { 1484 /* 1485 * This is after x86_pmu_stop(); so we disable LBRs after any 1486 * event can need them etc.. 1487 */ 1488 x86_pmu.del(event); 1489 } 1490 } 1491 1492 int x86_pmu_handle_irq(struct pt_regs *regs) 1493 { 1494 struct perf_sample_data data; 1495 struct cpu_hw_events *cpuc; 1496 struct perf_event *event; 1497 int idx, handled = 0; 1498 u64 val; 1499 1500 cpuc = this_cpu_ptr(&cpu_hw_events); 1501 1502 /* 1503 * Some chipsets need to unmask the LVTPC in a particular spot 1504 * inside the nmi handler. As a result, the unmasking was pushed 1505 * into all the nmi handlers. 1506 * 1507 * This generic handler doesn't seem to have any issues where the 1508 * unmasking occurs so it was left at the top. 1509 */ 1510 apic_write(APIC_LVTPC, APIC_DM_NMI); 1511 1512 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1513 if (!test_bit(idx, cpuc->active_mask)) 1514 continue; 1515 1516 event = cpuc->events[idx]; 1517 1518 val = x86_perf_event_update(event); 1519 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1520 continue; 1521 1522 /* 1523 * event overflow 1524 */ 1525 handled++; 1526 perf_sample_data_init(&data, 0, event->hw.last_period); 1527 1528 if (!x86_perf_event_set_period(event)) 1529 continue; 1530 1531 if (perf_event_overflow(event, &data, regs)) 1532 x86_pmu_stop(event, 0); 1533 } 1534 1535 if (handled) 1536 inc_irq_stat(apic_perf_irqs); 1537 1538 return handled; 1539 } 1540 1541 void perf_events_lapic_init(void) 1542 { 1543 if (!x86_pmu.apic || !x86_pmu_initialized()) 1544 return; 1545 1546 /* 1547 * Always use NMI for PMU 1548 */ 1549 apic_write(APIC_LVTPC, APIC_DM_NMI); 1550 } 1551 1552 static int 1553 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1554 { 1555 u64 start_clock; 1556 u64 finish_clock; 1557 int ret; 1558 1559 /* 1560 * All PMUs/events that share this PMI handler should make sure to 1561 * increment active_events for their events. 1562 */ 1563 if (!atomic_read(&active_events)) 1564 return NMI_DONE; 1565 1566 start_clock = sched_clock(); 1567 ret = x86_pmu.handle_irq(regs); 1568 finish_clock = sched_clock(); 1569 1570 perf_sample_event_took(finish_clock - start_clock); 1571 1572 return ret; 1573 } 1574 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1575 1576 struct event_constraint emptyconstraint; 1577 struct event_constraint unconstrained; 1578 1579 static int x86_pmu_prepare_cpu(unsigned int cpu) 1580 { 1581 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1582 int i; 1583 1584 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1585 cpuc->kfree_on_online[i] = NULL; 1586 if (x86_pmu.cpu_prepare) 1587 return x86_pmu.cpu_prepare(cpu); 1588 return 0; 1589 } 1590 1591 static int x86_pmu_dead_cpu(unsigned int cpu) 1592 { 1593 if (x86_pmu.cpu_dead) 1594 x86_pmu.cpu_dead(cpu); 1595 return 0; 1596 } 1597 1598 static int x86_pmu_online_cpu(unsigned int cpu) 1599 { 1600 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1601 int i; 1602 1603 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1604 kfree(cpuc->kfree_on_online[i]); 1605 cpuc->kfree_on_online[i] = NULL; 1606 } 1607 return 0; 1608 } 1609 1610 static int x86_pmu_starting_cpu(unsigned int cpu) 1611 { 1612 if (x86_pmu.cpu_starting) 1613 x86_pmu.cpu_starting(cpu); 1614 return 0; 1615 } 1616 1617 static int x86_pmu_dying_cpu(unsigned int cpu) 1618 { 1619 if (x86_pmu.cpu_dying) 1620 x86_pmu.cpu_dying(cpu); 1621 return 0; 1622 } 1623 1624 static void __init pmu_check_apic(void) 1625 { 1626 if (boot_cpu_has(X86_FEATURE_APIC)) 1627 return; 1628 1629 x86_pmu.apic = 0; 1630 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1631 pr_info("no hardware sampling interrupt available.\n"); 1632 1633 /* 1634 * If we have a PMU initialized but no APIC 1635 * interrupts, we cannot sample hardware 1636 * events (user-space has to fall back and 1637 * sample via a hrtimer based software event): 1638 */ 1639 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1640 1641 } 1642 1643 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1644 .name = "format", 1645 .attrs = NULL, 1646 }; 1647 1648 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1649 { 1650 struct perf_pmu_events_attr *pmu_attr = 1651 container_of(attr, struct perf_pmu_events_attr, attr); 1652 u64 config = 0; 1653 1654 if (pmu_attr->id < x86_pmu.max_events) 1655 config = x86_pmu.event_map(pmu_attr->id); 1656 1657 /* string trumps id */ 1658 if (pmu_attr->event_str) 1659 return sprintf(page, "%s", pmu_attr->event_str); 1660 1661 return x86_pmu.events_sysfs_show(page, config); 1662 } 1663 EXPORT_SYMBOL_GPL(events_sysfs_show); 1664 1665 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1666 char *page) 1667 { 1668 struct perf_pmu_events_ht_attr *pmu_attr = 1669 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1670 1671 /* 1672 * Report conditional events depending on Hyper-Threading. 1673 * 1674 * This is overly conservative as usually the HT special 1675 * handling is not needed if the other CPU thread is idle. 1676 * 1677 * Note this does not (and cannot) handle the case when thread 1678 * siblings are invisible, for example with virtualization 1679 * if they are owned by some other guest. The user tool 1680 * has to re-read when a thread sibling gets onlined later. 1681 */ 1682 return sprintf(page, "%s", 1683 topology_max_smt_threads() > 1 ? 1684 pmu_attr->event_str_ht : 1685 pmu_attr->event_str_noht); 1686 } 1687 1688 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1689 EVENT_ATTR(instructions, INSTRUCTIONS ); 1690 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1691 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1692 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1693 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1694 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1695 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1696 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1697 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1698 1699 static struct attribute *empty_attrs; 1700 1701 static struct attribute *events_attr[] = { 1702 EVENT_PTR(CPU_CYCLES), 1703 EVENT_PTR(INSTRUCTIONS), 1704 EVENT_PTR(CACHE_REFERENCES), 1705 EVENT_PTR(CACHE_MISSES), 1706 EVENT_PTR(BRANCH_INSTRUCTIONS), 1707 EVENT_PTR(BRANCH_MISSES), 1708 EVENT_PTR(BUS_CYCLES), 1709 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1710 EVENT_PTR(STALLED_CYCLES_BACKEND), 1711 EVENT_PTR(REF_CPU_CYCLES), 1712 NULL, 1713 }; 1714 1715 /* 1716 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1717 * out of events_attr attributes. 1718 */ 1719 static umode_t 1720 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 1721 { 1722 struct perf_pmu_events_attr *pmu_attr; 1723 1724 if (idx >= x86_pmu.max_events) 1725 return 0; 1726 1727 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 1728 /* str trumps id */ 1729 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 1730 } 1731 1732 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1733 .name = "events", 1734 .attrs = events_attr, 1735 .is_visible = is_visible, 1736 }; 1737 1738 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1739 { 1740 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1741 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1742 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1743 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1744 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1745 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1746 ssize_t ret; 1747 1748 /* 1749 * We have whole page size to spend and just little data 1750 * to write, so we can safely use sprintf. 1751 */ 1752 ret = sprintf(page, "event=0x%02llx", event); 1753 1754 if (umask) 1755 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1756 1757 if (edge) 1758 ret += sprintf(page + ret, ",edge"); 1759 1760 if (pc) 1761 ret += sprintf(page + ret, ",pc"); 1762 1763 if (any) 1764 ret += sprintf(page + ret, ",any"); 1765 1766 if (inv) 1767 ret += sprintf(page + ret, ",inv"); 1768 1769 if (cmask) 1770 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1771 1772 ret += sprintf(page + ret, "\n"); 1773 1774 return ret; 1775 } 1776 1777 static struct attribute_group x86_pmu_attr_group; 1778 static struct attribute_group x86_pmu_caps_group; 1779 1780 static int __init init_hw_perf_events(void) 1781 { 1782 struct x86_pmu_quirk *quirk; 1783 int err; 1784 1785 pr_info("Performance Events: "); 1786 1787 switch (boot_cpu_data.x86_vendor) { 1788 case X86_VENDOR_INTEL: 1789 err = intel_pmu_init(); 1790 break; 1791 case X86_VENDOR_AMD: 1792 err = amd_pmu_init(); 1793 break; 1794 case X86_VENDOR_HYGON: 1795 err = amd_pmu_init(); 1796 x86_pmu.name = "HYGON"; 1797 break; 1798 default: 1799 err = -ENOTSUPP; 1800 } 1801 if (err != 0) { 1802 pr_cont("no PMU driver, software events only.\n"); 1803 return 0; 1804 } 1805 1806 pmu_check_apic(); 1807 1808 /* sanity check that the hardware exists or is emulated */ 1809 if (!check_hw_exists()) 1810 return 0; 1811 1812 pr_cont("%s PMU driver.\n", x86_pmu.name); 1813 1814 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1815 1816 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1817 quirk->func(); 1818 1819 if (!x86_pmu.intel_ctrl) 1820 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1821 1822 perf_events_lapic_init(); 1823 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1824 1825 unconstrained = (struct event_constraint) 1826 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1827 0, x86_pmu.num_counters, 0, 0); 1828 1829 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1830 1831 if (!x86_pmu.events_sysfs_show) 1832 x86_pmu_events_group.attrs = &empty_attrs; 1833 1834 pmu.attr_update = x86_pmu.attr_update; 1835 1836 pr_info("... version: %d\n", x86_pmu.version); 1837 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1838 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1839 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1840 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1841 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1842 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1843 1844 /* 1845 * Install callbacks. Core will call them for each online 1846 * cpu. 1847 */ 1848 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1849 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1850 if (err) 1851 return err; 1852 1853 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1854 "perf/x86:starting", x86_pmu_starting_cpu, 1855 x86_pmu_dying_cpu); 1856 if (err) 1857 goto out; 1858 1859 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1860 x86_pmu_online_cpu, NULL); 1861 if (err) 1862 goto out1; 1863 1864 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1865 if (err) 1866 goto out2; 1867 1868 return 0; 1869 1870 out2: 1871 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1872 out1: 1873 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1874 out: 1875 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1876 return err; 1877 } 1878 early_initcall(init_hw_perf_events); 1879 1880 static inline void x86_pmu_read(struct perf_event *event) 1881 { 1882 if (x86_pmu.read) 1883 return x86_pmu.read(event); 1884 x86_perf_event_update(event); 1885 } 1886 1887 /* 1888 * Start group events scheduling transaction 1889 * Set the flag to make pmu::enable() not perform the 1890 * schedulability test, it will be performed at commit time 1891 * 1892 * We only support PERF_PMU_TXN_ADD transactions. Save the 1893 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1894 * transactions. 1895 */ 1896 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1897 { 1898 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1899 1900 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1901 1902 cpuc->txn_flags = txn_flags; 1903 if (txn_flags & ~PERF_PMU_TXN_ADD) 1904 return; 1905 1906 perf_pmu_disable(pmu); 1907 __this_cpu_write(cpu_hw_events.n_txn, 0); 1908 } 1909 1910 /* 1911 * Stop group events scheduling transaction 1912 * Clear the flag and pmu::enable() will perform the 1913 * schedulability test. 1914 */ 1915 static void x86_pmu_cancel_txn(struct pmu *pmu) 1916 { 1917 unsigned int txn_flags; 1918 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1919 1920 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1921 1922 txn_flags = cpuc->txn_flags; 1923 cpuc->txn_flags = 0; 1924 if (txn_flags & ~PERF_PMU_TXN_ADD) 1925 return; 1926 1927 /* 1928 * Truncate collected array by the number of events added in this 1929 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1930 */ 1931 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1932 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1933 perf_pmu_enable(pmu); 1934 } 1935 1936 /* 1937 * Commit group events scheduling transaction 1938 * Perform the group schedulability test as a whole 1939 * Return 0 if success 1940 * 1941 * Does not cancel the transaction on failure; expects the caller to do this. 1942 */ 1943 static int x86_pmu_commit_txn(struct pmu *pmu) 1944 { 1945 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1946 int assign[X86_PMC_IDX_MAX]; 1947 int n, ret; 1948 1949 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1950 1951 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1952 cpuc->txn_flags = 0; 1953 return 0; 1954 } 1955 1956 n = cpuc->n_events; 1957 1958 if (!x86_pmu_initialized()) 1959 return -EAGAIN; 1960 1961 ret = x86_pmu.schedule_events(cpuc, n, assign); 1962 if (ret) 1963 return ret; 1964 1965 /* 1966 * copy new assignment, now we know it is possible 1967 * will be used by hw_perf_enable() 1968 */ 1969 memcpy(cpuc->assign, assign, n*sizeof(int)); 1970 1971 cpuc->txn_flags = 0; 1972 perf_pmu_enable(pmu); 1973 return 0; 1974 } 1975 /* 1976 * a fake_cpuc is used to validate event groups. Due to 1977 * the extra reg logic, we need to also allocate a fake 1978 * per_core and per_cpu structure. Otherwise, group events 1979 * using extra reg may conflict without the kernel being 1980 * able to catch this when the last event gets added to 1981 * the group. 1982 */ 1983 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 1984 { 1985 intel_cpuc_finish(cpuc); 1986 kfree(cpuc); 1987 } 1988 1989 static struct cpu_hw_events *allocate_fake_cpuc(void) 1990 { 1991 struct cpu_hw_events *cpuc; 1992 int cpu = raw_smp_processor_id(); 1993 1994 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 1995 if (!cpuc) 1996 return ERR_PTR(-ENOMEM); 1997 cpuc->is_fake = 1; 1998 1999 if (intel_cpuc_prepare(cpuc, cpu)) 2000 goto error; 2001 2002 return cpuc; 2003 error: 2004 free_fake_cpuc(cpuc); 2005 return ERR_PTR(-ENOMEM); 2006 } 2007 2008 /* 2009 * validate that we can schedule this event 2010 */ 2011 static int validate_event(struct perf_event *event) 2012 { 2013 struct cpu_hw_events *fake_cpuc; 2014 struct event_constraint *c; 2015 int ret = 0; 2016 2017 fake_cpuc = allocate_fake_cpuc(); 2018 if (IS_ERR(fake_cpuc)) 2019 return PTR_ERR(fake_cpuc); 2020 2021 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2022 2023 if (!c || !c->weight) 2024 ret = -EINVAL; 2025 2026 if (x86_pmu.put_event_constraints) 2027 x86_pmu.put_event_constraints(fake_cpuc, event); 2028 2029 free_fake_cpuc(fake_cpuc); 2030 2031 return ret; 2032 } 2033 2034 /* 2035 * validate a single event group 2036 * 2037 * validation include: 2038 * - check events are compatible which each other 2039 * - events do not compete for the same counter 2040 * - number of events <= number of counters 2041 * 2042 * validation ensures the group can be loaded onto the 2043 * PMU if it was the only group available. 2044 */ 2045 static int validate_group(struct perf_event *event) 2046 { 2047 struct perf_event *leader = event->group_leader; 2048 struct cpu_hw_events *fake_cpuc; 2049 int ret = -EINVAL, n; 2050 2051 fake_cpuc = allocate_fake_cpuc(); 2052 if (IS_ERR(fake_cpuc)) 2053 return PTR_ERR(fake_cpuc); 2054 /* 2055 * the event is not yet connected with its 2056 * siblings therefore we must first collect 2057 * existing siblings, then add the new event 2058 * before we can simulate the scheduling 2059 */ 2060 n = collect_events(fake_cpuc, leader, true); 2061 if (n < 0) 2062 goto out; 2063 2064 fake_cpuc->n_events = n; 2065 n = collect_events(fake_cpuc, event, false); 2066 if (n < 0) 2067 goto out; 2068 2069 fake_cpuc->n_events = 0; 2070 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2071 2072 out: 2073 free_fake_cpuc(fake_cpuc); 2074 return ret; 2075 } 2076 2077 static int x86_pmu_event_init(struct perf_event *event) 2078 { 2079 struct pmu *tmp; 2080 int err; 2081 2082 switch (event->attr.type) { 2083 case PERF_TYPE_RAW: 2084 case PERF_TYPE_HARDWARE: 2085 case PERF_TYPE_HW_CACHE: 2086 break; 2087 2088 default: 2089 return -ENOENT; 2090 } 2091 2092 err = __x86_pmu_event_init(event); 2093 if (!err) { 2094 /* 2095 * we temporarily connect event to its pmu 2096 * such that validate_group() can classify 2097 * it as an x86 event using is_x86_event() 2098 */ 2099 tmp = event->pmu; 2100 event->pmu = &pmu; 2101 2102 if (event->group_leader != event) 2103 err = validate_group(event); 2104 else 2105 err = validate_event(event); 2106 2107 event->pmu = tmp; 2108 } 2109 if (err) { 2110 if (event->destroy) 2111 event->destroy(event); 2112 } 2113 2114 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2115 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2116 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2117 2118 return err; 2119 } 2120 2121 static void refresh_pce(void *ignored) 2122 { 2123 load_mm_cr4_irqsoff(this_cpu_read(cpu_tlbstate.loaded_mm)); 2124 } 2125 2126 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2127 { 2128 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2129 return; 2130 2131 /* 2132 * This function relies on not being called concurrently in two 2133 * tasks in the same mm. Otherwise one task could observe 2134 * perf_rdpmc_allowed > 1 and return all the way back to 2135 * userspace with CR4.PCE clear while another task is still 2136 * doing on_each_cpu_mask() to propagate CR4.PCE. 2137 * 2138 * For now, this can't happen because all callers hold mmap_sem 2139 * for write. If this changes, we'll need a different solution. 2140 */ 2141 lockdep_assert_held_write(&mm->mmap_sem); 2142 2143 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2144 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2145 } 2146 2147 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2148 { 2149 2150 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2151 return; 2152 2153 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2154 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2155 } 2156 2157 static int x86_pmu_event_idx(struct perf_event *event) 2158 { 2159 int idx = event->hw.idx; 2160 2161 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2162 return 0; 2163 2164 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2165 idx -= INTEL_PMC_IDX_FIXED; 2166 idx |= 1 << 30; 2167 } 2168 2169 return idx + 1; 2170 } 2171 2172 static ssize_t get_attr_rdpmc(struct device *cdev, 2173 struct device_attribute *attr, 2174 char *buf) 2175 { 2176 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2177 } 2178 2179 static ssize_t set_attr_rdpmc(struct device *cdev, 2180 struct device_attribute *attr, 2181 const char *buf, size_t count) 2182 { 2183 unsigned long val; 2184 ssize_t ret; 2185 2186 ret = kstrtoul(buf, 0, &val); 2187 if (ret) 2188 return ret; 2189 2190 if (val > 2) 2191 return -EINVAL; 2192 2193 if (x86_pmu.attr_rdpmc_broken) 2194 return -ENOTSUPP; 2195 2196 if (val != x86_pmu.attr_rdpmc) { 2197 /* 2198 * Changing into or out of never available or always available, 2199 * aka perf-event-bypassing mode. This path is extremely slow, 2200 * but only root can trigger it, so it's okay. 2201 */ 2202 if (val == 0) 2203 static_branch_inc(&rdpmc_never_available_key); 2204 else if (x86_pmu.attr_rdpmc == 0) 2205 static_branch_dec(&rdpmc_never_available_key); 2206 2207 if (val == 2) 2208 static_branch_inc(&rdpmc_always_available_key); 2209 else if (x86_pmu.attr_rdpmc == 2) 2210 static_branch_dec(&rdpmc_always_available_key); 2211 2212 on_each_cpu(refresh_pce, NULL, 1); 2213 x86_pmu.attr_rdpmc = val; 2214 } 2215 2216 return count; 2217 } 2218 2219 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2220 2221 static struct attribute *x86_pmu_attrs[] = { 2222 &dev_attr_rdpmc.attr, 2223 NULL, 2224 }; 2225 2226 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2227 .attrs = x86_pmu_attrs, 2228 }; 2229 2230 static ssize_t max_precise_show(struct device *cdev, 2231 struct device_attribute *attr, 2232 char *buf) 2233 { 2234 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2235 } 2236 2237 static DEVICE_ATTR_RO(max_precise); 2238 2239 static struct attribute *x86_pmu_caps_attrs[] = { 2240 &dev_attr_max_precise.attr, 2241 NULL 2242 }; 2243 2244 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2245 .name = "caps", 2246 .attrs = x86_pmu_caps_attrs, 2247 }; 2248 2249 static const struct attribute_group *x86_pmu_attr_groups[] = { 2250 &x86_pmu_attr_group, 2251 &x86_pmu_format_group, 2252 &x86_pmu_events_group, 2253 &x86_pmu_caps_group, 2254 NULL, 2255 }; 2256 2257 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2258 { 2259 if (x86_pmu.sched_task) 2260 x86_pmu.sched_task(ctx, sched_in); 2261 } 2262 2263 static void x86_pmu_swap_task_ctx(struct perf_event_context *prev, 2264 struct perf_event_context *next) 2265 { 2266 if (x86_pmu.swap_task_ctx) 2267 x86_pmu.swap_task_ctx(prev, next); 2268 } 2269 2270 void perf_check_microcode(void) 2271 { 2272 if (x86_pmu.check_microcode) 2273 x86_pmu.check_microcode(); 2274 } 2275 2276 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2277 { 2278 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2279 return -EINVAL; 2280 2281 if (value && x86_pmu.limit_period) { 2282 if (x86_pmu.limit_period(event, value) > value) 2283 return -EINVAL; 2284 } 2285 2286 return 0; 2287 } 2288 2289 static int x86_pmu_aux_output_match(struct perf_event *event) 2290 { 2291 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 2292 return 0; 2293 2294 if (x86_pmu.aux_output_match) 2295 return x86_pmu.aux_output_match(event); 2296 2297 return 0; 2298 } 2299 2300 static struct pmu pmu = { 2301 .pmu_enable = x86_pmu_enable, 2302 .pmu_disable = x86_pmu_disable, 2303 2304 .attr_groups = x86_pmu_attr_groups, 2305 2306 .event_init = x86_pmu_event_init, 2307 2308 .event_mapped = x86_pmu_event_mapped, 2309 .event_unmapped = x86_pmu_event_unmapped, 2310 2311 .add = x86_pmu_add, 2312 .del = x86_pmu_del, 2313 .start = x86_pmu_start, 2314 .stop = x86_pmu_stop, 2315 .read = x86_pmu_read, 2316 2317 .start_txn = x86_pmu_start_txn, 2318 .cancel_txn = x86_pmu_cancel_txn, 2319 .commit_txn = x86_pmu_commit_txn, 2320 2321 .event_idx = x86_pmu_event_idx, 2322 .sched_task = x86_pmu_sched_task, 2323 .task_ctx_size = sizeof(struct x86_perf_task_context), 2324 .swap_task_ctx = x86_pmu_swap_task_ctx, 2325 .check_period = x86_pmu_check_period, 2326 2327 .aux_output_match = x86_pmu_aux_output_match, 2328 }; 2329 2330 void arch_perf_update_userpage(struct perf_event *event, 2331 struct perf_event_mmap_page *userpg, u64 now) 2332 { 2333 struct cyc2ns_data data; 2334 u64 offset; 2335 2336 userpg->cap_user_time = 0; 2337 userpg->cap_user_time_zero = 0; 2338 userpg->cap_user_rdpmc = 2339 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2340 userpg->pmc_width = x86_pmu.cntval_bits; 2341 2342 if (!using_native_sched_clock() || !sched_clock_stable()) 2343 return; 2344 2345 cyc2ns_read_begin(&data); 2346 2347 offset = data.cyc2ns_offset + __sched_clock_offset; 2348 2349 /* 2350 * Internal timekeeping for enabled/running/stopped times 2351 * is always in the local_clock domain. 2352 */ 2353 userpg->cap_user_time = 1; 2354 userpg->time_mult = data.cyc2ns_mul; 2355 userpg->time_shift = data.cyc2ns_shift; 2356 userpg->time_offset = offset - now; 2357 2358 /* 2359 * cap_user_time_zero doesn't make sense when we're using a different 2360 * time base for the records. 2361 */ 2362 if (!event->attr.use_clockid) { 2363 userpg->cap_user_time_zero = 1; 2364 userpg->time_zero = offset; 2365 } 2366 2367 cyc2ns_read_end(); 2368 } 2369 2370 /* 2371 * Determine whether the regs were taken from an irq/exception handler rather 2372 * than from perf_arch_fetch_caller_regs(). 2373 */ 2374 static bool perf_hw_regs(struct pt_regs *regs) 2375 { 2376 return regs->flags & X86_EFLAGS_FIXED; 2377 } 2378 2379 void 2380 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2381 { 2382 struct unwind_state state; 2383 unsigned long addr; 2384 2385 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2386 /* TODO: We don't support guest os callchain now */ 2387 return; 2388 } 2389 2390 if (perf_callchain_store(entry, regs->ip)) 2391 return; 2392 2393 if (perf_hw_regs(regs)) 2394 unwind_start(&state, current, regs, NULL); 2395 else 2396 unwind_start(&state, current, NULL, (void *)regs->sp); 2397 2398 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2399 addr = unwind_get_return_address(&state); 2400 if (!addr || perf_callchain_store(entry, addr)) 2401 return; 2402 } 2403 } 2404 2405 static inline int 2406 valid_user_frame(const void __user *fp, unsigned long size) 2407 { 2408 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2409 } 2410 2411 static unsigned long get_segment_base(unsigned int segment) 2412 { 2413 struct desc_struct *desc; 2414 unsigned int idx = segment >> 3; 2415 2416 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2417 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2418 struct ldt_struct *ldt; 2419 2420 /* IRQs are off, so this synchronizes with smp_store_release */ 2421 ldt = READ_ONCE(current->active_mm->context.ldt); 2422 if (!ldt || idx >= ldt->nr_entries) 2423 return 0; 2424 2425 desc = &ldt->entries[idx]; 2426 #else 2427 return 0; 2428 #endif 2429 } else { 2430 if (idx >= GDT_ENTRIES) 2431 return 0; 2432 2433 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2434 } 2435 2436 return get_desc_base(desc); 2437 } 2438 2439 #ifdef CONFIG_IA32_EMULATION 2440 2441 #include <linux/compat.h> 2442 2443 static inline int 2444 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2445 { 2446 /* 32-bit process in 64-bit kernel. */ 2447 unsigned long ss_base, cs_base; 2448 struct stack_frame_ia32 frame; 2449 const void __user *fp; 2450 2451 if (!test_thread_flag(TIF_IA32)) 2452 return 0; 2453 2454 cs_base = get_segment_base(regs->cs); 2455 ss_base = get_segment_base(regs->ss); 2456 2457 fp = compat_ptr(ss_base + regs->bp); 2458 pagefault_disable(); 2459 while (entry->nr < entry->max_stack) { 2460 unsigned long bytes; 2461 frame.next_frame = 0; 2462 frame.return_address = 0; 2463 2464 if (!valid_user_frame(fp, sizeof(frame))) 2465 break; 2466 2467 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2468 if (bytes != 0) 2469 break; 2470 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2471 if (bytes != 0) 2472 break; 2473 2474 perf_callchain_store(entry, cs_base + frame.return_address); 2475 fp = compat_ptr(ss_base + frame.next_frame); 2476 } 2477 pagefault_enable(); 2478 return 1; 2479 } 2480 #else 2481 static inline int 2482 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2483 { 2484 return 0; 2485 } 2486 #endif 2487 2488 void 2489 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2490 { 2491 struct stack_frame frame; 2492 const unsigned long __user *fp; 2493 2494 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2495 /* TODO: We don't support guest os callchain now */ 2496 return; 2497 } 2498 2499 /* 2500 * We don't know what to do with VM86 stacks.. ignore them for now. 2501 */ 2502 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2503 return; 2504 2505 fp = (unsigned long __user *)regs->bp; 2506 2507 perf_callchain_store(entry, regs->ip); 2508 2509 if (!nmi_uaccess_okay()) 2510 return; 2511 2512 if (perf_callchain_user32(regs, entry)) 2513 return; 2514 2515 pagefault_disable(); 2516 while (entry->nr < entry->max_stack) { 2517 unsigned long bytes; 2518 2519 frame.next_frame = NULL; 2520 frame.return_address = 0; 2521 2522 if (!valid_user_frame(fp, sizeof(frame))) 2523 break; 2524 2525 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2526 if (bytes != 0) 2527 break; 2528 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2529 if (bytes != 0) 2530 break; 2531 2532 perf_callchain_store(entry, frame.return_address); 2533 fp = (void __user *)frame.next_frame; 2534 } 2535 pagefault_enable(); 2536 } 2537 2538 /* 2539 * Deal with code segment offsets for the various execution modes: 2540 * 2541 * VM86 - the good olde 16 bit days, where the linear address is 2542 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2543 * 2544 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2545 * to figure out what the 32bit base address is. 2546 * 2547 * X32 - has TIF_X32 set, but is running in x86_64 2548 * 2549 * X86_64 - CS,DS,SS,ES are all zero based. 2550 */ 2551 static unsigned long code_segment_base(struct pt_regs *regs) 2552 { 2553 /* 2554 * For IA32 we look at the GDT/LDT segment base to convert the 2555 * effective IP to a linear address. 2556 */ 2557 2558 #ifdef CONFIG_X86_32 2559 /* 2560 * If we are in VM86 mode, add the segment offset to convert to a 2561 * linear address. 2562 */ 2563 if (regs->flags & X86_VM_MASK) 2564 return 0x10 * regs->cs; 2565 2566 if (user_mode(regs) && regs->cs != __USER_CS) 2567 return get_segment_base(regs->cs); 2568 #else 2569 if (user_mode(regs) && !user_64bit_mode(regs) && 2570 regs->cs != __USER32_CS) 2571 return get_segment_base(regs->cs); 2572 #endif 2573 return 0; 2574 } 2575 2576 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2577 { 2578 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2579 return perf_guest_cbs->get_guest_ip(); 2580 2581 return regs->ip + code_segment_base(regs); 2582 } 2583 2584 unsigned long perf_misc_flags(struct pt_regs *regs) 2585 { 2586 int misc = 0; 2587 2588 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2589 if (perf_guest_cbs->is_user_mode()) 2590 misc |= PERF_RECORD_MISC_GUEST_USER; 2591 else 2592 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2593 } else { 2594 if (user_mode(regs)) 2595 misc |= PERF_RECORD_MISC_USER; 2596 else 2597 misc |= PERF_RECORD_MISC_KERNEL; 2598 } 2599 2600 if (regs->flags & PERF_EFLAGS_EXACT) 2601 misc |= PERF_RECORD_MISC_EXACT_IP; 2602 2603 return misc; 2604 } 2605 2606 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2607 { 2608 cap->version = x86_pmu.version; 2609 cap->num_counters_gp = x86_pmu.num_counters; 2610 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2611 cap->bit_width_gp = x86_pmu.cntval_bits; 2612 cap->bit_width_fixed = x86_pmu.cntval_bits; 2613 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2614 cap->events_mask_len = x86_pmu.events_mask_len; 2615 } 2616 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2617