1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 32 #include <asm/apic.h> 33 #include <asm/stacktrace.h> 34 #include <asm/nmi.h> 35 #include <asm/smp.h> 36 #include <asm/alternative.h> 37 #include <asm/mmu_context.h> 38 #include <asm/tlbflush.h> 39 #include <asm/timer.h> 40 #include <asm/desc.h> 41 #include <asm/ldt.h> 42 #include <asm/unwind.h> 43 44 #include "perf_event.h" 45 46 struct x86_pmu x86_pmu __read_mostly; 47 48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 49 .enabled = 1, 50 }; 51 52 DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key); 53 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 54 55 u64 __read_mostly hw_cache_event_ids 56 [PERF_COUNT_HW_CACHE_MAX] 57 [PERF_COUNT_HW_CACHE_OP_MAX] 58 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 59 u64 __read_mostly hw_cache_extra_regs 60 [PERF_COUNT_HW_CACHE_MAX] 61 [PERF_COUNT_HW_CACHE_OP_MAX] 62 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 63 64 /* 65 * Propagate event elapsed time into the generic event. 66 * Can only be executed on the CPU where the event is active. 67 * Returns the delta events processed. 68 */ 69 u64 x86_perf_event_update(struct perf_event *event) 70 { 71 struct hw_perf_event *hwc = &event->hw; 72 int shift = 64 - x86_pmu.cntval_bits; 73 u64 prev_raw_count, new_raw_count; 74 int idx = hwc->idx; 75 u64 delta; 76 77 if (idx == INTEL_PMC_IDX_FIXED_BTS) 78 return 0; 79 80 /* 81 * Careful: an NMI might modify the previous event value. 82 * 83 * Our tactic to handle this is to first atomically read and 84 * exchange a new raw count - then add that new-prev delta 85 * count to the generic event atomically: 86 */ 87 again: 88 prev_raw_count = local64_read(&hwc->prev_count); 89 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 90 91 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 92 new_raw_count) != prev_raw_count) 93 goto again; 94 95 /* 96 * Now we have the new raw value and have updated the prev 97 * timestamp already. We can now calculate the elapsed delta 98 * (event-)time and add that to the generic event. 99 * 100 * Careful, not all hw sign-extends above the physical width 101 * of the count. 102 */ 103 delta = (new_raw_count << shift) - (prev_raw_count << shift); 104 delta >>= shift; 105 106 local64_add(delta, &event->count); 107 local64_sub(delta, &hwc->period_left); 108 109 return new_raw_count; 110 } 111 112 /* 113 * Find and validate any extra registers to set up. 114 */ 115 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 116 { 117 struct hw_perf_event_extra *reg; 118 struct extra_reg *er; 119 120 reg = &event->hw.extra_reg; 121 122 if (!x86_pmu.extra_regs) 123 return 0; 124 125 for (er = x86_pmu.extra_regs; er->msr; er++) { 126 if (er->event != (config & er->config_mask)) 127 continue; 128 if (event->attr.config1 & ~er->valid_mask) 129 return -EINVAL; 130 /* Check if the extra msrs can be safely accessed*/ 131 if (!er->extra_msr_access) 132 return -ENXIO; 133 134 reg->idx = er->idx; 135 reg->config = event->attr.config1; 136 reg->reg = er->msr; 137 break; 138 } 139 return 0; 140 } 141 142 static atomic_t active_events; 143 static atomic_t pmc_refcount; 144 static DEFINE_MUTEX(pmc_reserve_mutex); 145 146 #ifdef CONFIG_X86_LOCAL_APIC 147 148 static bool reserve_pmc_hardware(void) 149 { 150 int i; 151 152 for (i = 0; i < x86_pmu.num_counters; i++) { 153 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 154 goto perfctr_fail; 155 } 156 157 for (i = 0; i < x86_pmu.num_counters; i++) { 158 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 159 goto eventsel_fail; 160 } 161 162 return true; 163 164 eventsel_fail: 165 for (i--; i >= 0; i--) 166 release_evntsel_nmi(x86_pmu_config_addr(i)); 167 168 i = x86_pmu.num_counters; 169 170 perfctr_fail: 171 for (i--; i >= 0; i--) 172 release_perfctr_nmi(x86_pmu_event_addr(i)); 173 174 return false; 175 } 176 177 static void release_pmc_hardware(void) 178 { 179 int i; 180 181 for (i = 0; i < x86_pmu.num_counters; i++) { 182 release_perfctr_nmi(x86_pmu_event_addr(i)); 183 release_evntsel_nmi(x86_pmu_config_addr(i)); 184 } 185 } 186 187 #else 188 189 static bool reserve_pmc_hardware(void) { return true; } 190 static void release_pmc_hardware(void) {} 191 192 #endif 193 194 static bool check_hw_exists(void) 195 { 196 u64 val, val_fail = -1, val_new= ~0; 197 int i, reg, reg_fail = -1, ret = 0; 198 int bios_fail = 0; 199 int reg_safe = -1; 200 201 /* 202 * Check to see if the BIOS enabled any of the counters, if so 203 * complain and bail. 204 */ 205 for (i = 0; i < x86_pmu.num_counters; i++) { 206 reg = x86_pmu_config_addr(i); 207 ret = rdmsrl_safe(reg, &val); 208 if (ret) 209 goto msr_fail; 210 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 211 bios_fail = 1; 212 val_fail = val; 213 reg_fail = reg; 214 } else { 215 reg_safe = i; 216 } 217 } 218 219 if (x86_pmu.num_counters_fixed) { 220 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 221 ret = rdmsrl_safe(reg, &val); 222 if (ret) 223 goto msr_fail; 224 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 225 if (val & (0x03 << i*4)) { 226 bios_fail = 1; 227 val_fail = val; 228 reg_fail = reg; 229 } 230 } 231 } 232 233 /* 234 * If all the counters are enabled, the below test will always 235 * fail. The tools will also become useless in this scenario. 236 * Just fail and disable the hardware counters. 237 */ 238 239 if (reg_safe == -1) { 240 reg = reg_safe; 241 goto msr_fail; 242 } 243 244 /* 245 * Read the current value, change it and read it back to see if it 246 * matches, this is needed to detect certain hardware emulators 247 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 248 */ 249 reg = x86_pmu_event_addr(reg_safe); 250 if (rdmsrl_safe(reg, &val)) 251 goto msr_fail; 252 val ^= 0xffffUL; 253 ret = wrmsrl_safe(reg, val); 254 ret |= rdmsrl_safe(reg, &val_new); 255 if (ret || val != val_new) 256 goto msr_fail; 257 258 /* 259 * We still allow the PMU driver to operate: 260 */ 261 if (bios_fail) { 262 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 263 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 264 reg_fail, val_fail); 265 } 266 267 return true; 268 269 msr_fail: 270 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 271 pr_cont("PMU not available due to virtualization, using software events only.\n"); 272 } else { 273 pr_cont("Broken PMU hardware detected, using software events only.\n"); 274 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 275 reg, val_new); 276 } 277 278 return false; 279 } 280 281 static void hw_perf_event_destroy(struct perf_event *event) 282 { 283 x86_release_hardware(); 284 atomic_dec(&active_events); 285 } 286 287 void hw_perf_lbr_event_destroy(struct perf_event *event) 288 { 289 hw_perf_event_destroy(event); 290 291 /* undo the lbr/bts event accounting */ 292 x86_del_exclusive(x86_lbr_exclusive_lbr); 293 } 294 295 static inline int x86_pmu_initialized(void) 296 { 297 return x86_pmu.handle_irq != NULL; 298 } 299 300 static inline int 301 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 302 { 303 struct perf_event_attr *attr = &event->attr; 304 unsigned int cache_type, cache_op, cache_result; 305 u64 config, val; 306 307 config = attr->config; 308 309 cache_type = (config >> 0) & 0xff; 310 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 311 return -EINVAL; 312 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 313 314 cache_op = (config >> 8) & 0xff; 315 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 316 return -EINVAL; 317 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 318 319 cache_result = (config >> 16) & 0xff; 320 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 321 return -EINVAL; 322 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 323 324 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 325 326 if (val == 0) 327 return -ENOENT; 328 329 if (val == -1) 330 return -EINVAL; 331 332 hwc->config |= val; 333 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 334 return x86_pmu_extra_regs(val, event); 335 } 336 337 int x86_reserve_hardware(void) 338 { 339 int err = 0; 340 341 if (!atomic_inc_not_zero(&pmc_refcount)) { 342 mutex_lock(&pmc_reserve_mutex); 343 if (atomic_read(&pmc_refcount) == 0) { 344 if (!reserve_pmc_hardware()) 345 err = -EBUSY; 346 else 347 reserve_ds_buffers(); 348 } 349 if (!err) 350 atomic_inc(&pmc_refcount); 351 mutex_unlock(&pmc_reserve_mutex); 352 } 353 354 return err; 355 } 356 357 void x86_release_hardware(void) 358 { 359 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 360 release_pmc_hardware(); 361 release_ds_buffers(); 362 mutex_unlock(&pmc_reserve_mutex); 363 } 364 } 365 366 /* 367 * Check if we can create event of a certain type (that no conflicting events 368 * are present). 369 */ 370 int x86_add_exclusive(unsigned int what) 371 { 372 int i; 373 374 /* 375 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 376 * LBR and BTS are still mutually exclusive. 377 */ 378 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 379 return 0; 380 381 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 382 mutex_lock(&pmc_reserve_mutex); 383 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 384 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 385 goto fail_unlock; 386 } 387 atomic_inc(&x86_pmu.lbr_exclusive[what]); 388 mutex_unlock(&pmc_reserve_mutex); 389 } 390 391 atomic_inc(&active_events); 392 return 0; 393 394 fail_unlock: 395 mutex_unlock(&pmc_reserve_mutex); 396 return -EBUSY; 397 } 398 399 void x86_del_exclusive(unsigned int what) 400 { 401 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 402 return; 403 404 atomic_dec(&x86_pmu.lbr_exclusive[what]); 405 atomic_dec(&active_events); 406 } 407 408 int x86_setup_perfctr(struct perf_event *event) 409 { 410 struct perf_event_attr *attr = &event->attr; 411 struct hw_perf_event *hwc = &event->hw; 412 u64 config; 413 414 if (!is_sampling_event(event)) { 415 hwc->sample_period = x86_pmu.max_period; 416 hwc->last_period = hwc->sample_period; 417 local64_set(&hwc->period_left, hwc->sample_period); 418 } 419 420 if (attr->type == PERF_TYPE_RAW) 421 return x86_pmu_extra_regs(event->attr.config, event); 422 423 if (attr->type == PERF_TYPE_HW_CACHE) 424 return set_ext_hw_attr(hwc, event); 425 426 if (attr->config >= x86_pmu.max_events) 427 return -EINVAL; 428 429 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 430 431 /* 432 * The generic map: 433 */ 434 config = x86_pmu.event_map(attr->config); 435 436 if (config == 0) 437 return -ENOENT; 438 439 if (config == -1LL) 440 return -EINVAL; 441 442 hwc->config |= config; 443 444 return 0; 445 } 446 447 /* 448 * check that branch_sample_type is compatible with 449 * settings needed for precise_ip > 1 which implies 450 * using the LBR to capture ALL taken branches at the 451 * priv levels of the measurement 452 */ 453 static inline int precise_br_compat(struct perf_event *event) 454 { 455 u64 m = event->attr.branch_sample_type; 456 u64 b = 0; 457 458 /* must capture all branches */ 459 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 460 return 0; 461 462 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 463 464 if (!event->attr.exclude_user) 465 b |= PERF_SAMPLE_BRANCH_USER; 466 467 if (!event->attr.exclude_kernel) 468 b |= PERF_SAMPLE_BRANCH_KERNEL; 469 470 /* 471 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 472 */ 473 474 return m == b; 475 } 476 477 int x86_pmu_max_precise(void) 478 { 479 int precise = 0; 480 481 /* Support for constant skid */ 482 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 483 precise++; 484 485 /* Support for IP fixup */ 486 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 487 precise++; 488 489 if (x86_pmu.pebs_prec_dist) 490 precise++; 491 } 492 return precise; 493 } 494 495 int x86_pmu_hw_config(struct perf_event *event) 496 { 497 if (event->attr.precise_ip) { 498 int precise = x86_pmu_max_precise(); 499 500 if (event->attr.precise_ip > precise) 501 return -EOPNOTSUPP; 502 503 /* There's no sense in having PEBS for non sampling events: */ 504 if (!is_sampling_event(event)) 505 return -EINVAL; 506 } 507 /* 508 * check that PEBS LBR correction does not conflict with 509 * whatever the user is asking with attr->branch_sample_type 510 */ 511 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 512 u64 *br_type = &event->attr.branch_sample_type; 513 514 if (has_branch_stack(event)) { 515 if (!precise_br_compat(event)) 516 return -EOPNOTSUPP; 517 518 /* branch_sample_type is compatible */ 519 520 } else { 521 /* 522 * user did not specify branch_sample_type 523 * 524 * For PEBS fixups, we capture all 525 * the branches at the priv level of the 526 * event. 527 */ 528 *br_type = PERF_SAMPLE_BRANCH_ANY; 529 530 if (!event->attr.exclude_user) 531 *br_type |= PERF_SAMPLE_BRANCH_USER; 532 533 if (!event->attr.exclude_kernel) 534 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 535 } 536 } 537 538 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 539 event->attach_state |= PERF_ATTACH_TASK_DATA; 540 541 /* 542 * Generate PMC IRQs: 543 * (keep 'enabled' bit clear for now) 544 */ 545 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 546 547 /* 548 * Count user and OS events unless requested not to 549 */ 550 if (!event->attr.exclude_user) 551 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 552 if (!event->attr.exclude_kernel) 553 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 554 555 if (event->attr.type == PERF_TYPE_RAW) 556 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 557 558 if (event->attr.sample_period && x86_pmu.limit_period) { 559 if (x86_pmu.limit_period(event, event->attr.sample_period) > 560 event->attr.sample_period) 561 return -EINVAL; 562 } 563 564 /* sample_regs_user never support XMM registers */ 565 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) 566 return -EINVAL; 567 /* 568 * Besides the general purpose registers, XMM registers may 569 * be collected in PEBS on some platforms, e.g. Icelake 570 */ 571 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { 572 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) 573 return -EINVAL; 574 575 if (!event->attr.precise_ip) 576 return -EINVAL; 577 } 578 579 return x86_setup_perfctr(event); 580 } 581 582 /* 583 * Setup the hardware configuration for a given attr_type 584 */ 585 static int __x86_pmu_event_init(struct perf_event *event) 586 { 587 int err; 588 589 if (!x86_pmu_initialized()) 590 return -ENODEV; 591 592 err = x86_reserve_hardware(); 593 if (err) 594 return err; 595 596 atomic_inc(&active_events); 597 event->destroy = hw_perf_event_destroy; 598 599 event->hw.idx = -1; 600 event->hw.last_cpu = -1; 601 event->hw.last_tag = ~0ULL; 602 603 /* mark unused */ 604 event->hw.extra_reg.idx = EXTRA_REG_NONE; 605 event->hw.branch_reg.idx = EXTRA_REG_NONE; 606 607 return x86_pmu.hw_config(event); 608 } 609 610 void x86_pmu_disable_all(void) 611 { 612 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 613 int idx; 614 615 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 616 u64 val; 617 618 if (!test_bit(idx, cpuc->active_mask)) 619 continue; 620 rdmsrl(x86_pmu_config_addr(idx), val); 621 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 622 continue; 623 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 624 wrmsrl(x86_pmu_config_addr(idx), val); 625 } 626 } 627 628 /* 629 * There may be PMI landing after enabled=0. The PMI hitting could be before or 630 * after disable_all. 631 * 632 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 633 * It will not be re-enabled in the NMI handler again, because enabled=0. After 634 * handling the NMI, disable_all will be called, which will not change the 635 * state either. If PMI hits after disable_all, the PMU is already disabled 636 * before entering NMI handler. The NMI handler will not change the state 637 * either. 638 * 639 * So either situation is harmless. 640 */ 641 static void x86_pmu_disable(struct pmu *pmu) 642 { 643 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 644 645 if (!x86_pmu_initialized()) 646 return; 647 648 if (!cpuc->enabled) 649 return; 650 651 cpuc->n_added = 0; 652 cpuc->enabled = 0; 653 barrier(); 654 655 x86_pmu.disable_all(); 656 } 657 658 void x86_pmu_enable_all(int added) 659 { 660 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 661 int idx; 662 663 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 664 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 665 666 if (!test_bit(idx, cpuc->active_mask)) 667 continue; 668 669 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 670 } 671 } 672 673 static struct pmu pmu; 674 675 static inline int is_x86_event(struct perf_event *event) 676 { 677 return event->pmu == &pmu; 678 } 679 680 struct pmu *x86_get_pmu(void) 681 { 682 return &pmu; 683 } 684 /* 685 * Event scheduler state: 686 * 687 * Assign events iterating over all events and counters, beginning 688 * with events with least weights first. Keep the current iterator 689 * state in struct sched_state. 690 */ 691 struct sched_state { 692 int weight; 693 int event; /* event index */ 694 int counter; /* counter index */ 695 int unassigned; /* number of events to be assigned left */ 696 int nr_gp; /* number of GP counters used */ 697 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 698 }; 699 700 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 701 #define SCHED_STATES_MAX 2 702 703 struct perf_sched { 704 int max_weight; 705 int max_events; 706 int max_gp; 707 int saved_states; 708 struct event_constraint **constraints; 709 struct sched_state state; 710 struct sched_state saved[SCHED_STATES_MAX]; 711 }; 712 713 /* 714 * Initialize interator that runs through all events and counters. 715 */ 716 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 717 int num, int wmin, int wmax, int gpmax) 718 { 719 int idx; 720 721 memset(sched, 0, sizeof(*sched)); 722 sched->max_events = num; 723 sched->max_weight = wmax; 724 sched->max_gp = gpmax; 725 sched->constraints = constraints; 726 727 for (idx = 0; idx < num; idx++) { 728 if (constraints[idx]->weight == wmin) 729 break; 730 } 731 732 sched->state.event = idx; /* start with min weight */ 733 sched->state.weight = wmin; 734 sched->state.unassigned = num; 735 } 736 737 static void perf_sched_save_state(struct perf_sched *sched) 738 { 739 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 740 return; 741 742 sched->saved[sched->saved_states] = sched->state; 743 sched->saved_states++; 744 } 745 746 static bool perf_sched_restore_state(struct perf_sched *sched) 747 { 748 if (!sched->saved_states) 749 return false; 750 751 sched->saved_states--; 752 sched->state = sched->saved[sched->saved_states]; 753 754 /* continue with next counter: */ 755 clear_bit(sched->state.counter++, sched->state.used); 756 757 return true; 758 } 759 760 /* 761 * Select a counter for the current event to schedule. Return true on 762 * success. 763 */ 764 static bool __perf_sched_find_counter(struct perf_sched *sched) 765 { 766 struct event_constraint *c; 767 int idx; 768 769 if (!sched->state.unassigned) 770 return false; 771 772 if (sched->state.event >= sched->max_events) 773 return false; 774 775 c = sched->constraints[sched->state.event]; 776 /* Prefer fixed purpose counters */ 777 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 778 idx = INTEL_PMC_IDX_FIXED; 779 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 780 if (!__test_and_set_bit(idx, sched->state.used)) 781 goto done; 782 } 783 } 784 785 /* Grab the first unused counter starting with idx */ 786 idx = sched->state.counter; 787 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 788 if (!__test_and_set_bit(idx, sched->state.used)) { 789 if (sched->state.nr_gp++ >= sched->max_gp) 790 return false; 791 792 goto done; 793 } 794 } 795 796 return false; 797 798 done: 799 sched->state.counter = idx; 800 801 if (c->overlap) 802 perf_sched_save_state(sched); 803 804 return true; 805 } 806 807 static bool perf_sched_find_counter(struct perf_sched *sched) 808 { 809 while (!__perf_sched_find_counter(sched)) { 810 if (!perf_sched_restore_state(sched)) 811 return false; 812 } 813 814 return true; 815 } 816 817 /* 818 * Go through all unassigned events and find the next one to schedule. 819 * Take events with the least weight first. Return true on success. 820 */ 821 static bool perf_sched_next_event(struct perf_sched *sched) 822 { 823 struct event_constraint *c; 824 825 if (!sched->state.unassigned || !--sched->state.unassigned) 826 return false; 827 828 do { 829 /* next event */ 830 sched->state.event++; 831 if (sched->state.event >= sched->max_events) { 832 /* next weight */ 833 sched->state.event = 0; 834 sched->state.weight++; 835 if (sched->state.weight > sched->max_weight) 836 return false; 837 } 838 c = sched->constraints[sched->state.event]; 839 } while (c->weight != sched->state.weight); 840 841 sched->state.counter = 0; /* start with first counter */ 842 843 return true; 844 } 845 846 /* 847 * Assign a counter for each event. 848 */ 849 int perf_assign_events(struct event_constraint **constraints, int n, 850 int wmin, int wmax, int gpmax, int *assign) 851 { 852 struct perf_sched sched; 853 854 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 855 856 do { 857 if (!perf_sched_find_counter(&sched)) 858 break; /* failed */ 859 if (assign) 860 assign[sched.state.event] = sched.state.counter; 861 } while (perf_sched_next_event(&sched)); 862 863 return sched.state.unassigned; 864 } 865 EXPORT_SYMBOL_GPL(perf_assign_events); 866 867 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 868 { 869 struct event_constraint *c; 870 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 871 struct perf_event *e; 872 int n0, i, wmin, wmax, unsched = 0; 873 struct hw_perf_event *hwc; 874 875 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 876 877 /* 878 * Compute the number of events already present; see x86_pmu_add(), 879 * validate_group() and x86_pmu_commit_txn(). For the former two 880 * cpuc->n_events hasn't been updated yet, while for the latter 881 * cpuc->n_txn contains the number of events added in the current 882 * transaction. 883 */ 884 n0 = cpuc->n_events; 885 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 886 n0 -= cpuc->n_txn; 887 888 if (x86_pmu.start_scheduling) 889 x86_pmu.start_scheduling(cpuc); 890 891 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 892 c = cpuc->event_constraint[i]; 893 894 /* 895 * Previously scheduled events should have a cached constraint, 896 * while new events should not have one. 897 */ 898 WARN_ON_ONCE((c && i >= n0) || (!c && i < n0)); 899 900 /* 901 * Request constraints for new events; or for those events that 902 * have a dynamic constraint -- for those the constraint can 903 * change due to external factors (sibling state, allow_tfa). 904 */ 905 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { 906 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 907 cpuc->event_constraint[i] = c; 908 } 909 910 wmin = min(wmin, c->weight); 911 wmax = max(wmax, c->weight); 912 } 913 914 /* 915 * fastpath, try to reuse previous register 916 */ 917 for (i = 0; i < n; i++) { 918 hwc = &cpuc->event_list[i]->hw; 919 c = cpuc->event_constraint[i]; 920 921 /* never assigned */ 922 if (hwc->idx == -1) 923 break; 924 925 /* constraint still honored */ 926 if (!test_bit(hwc->idx, c->idxmsk)) 927 break; 928 929 /* not already used */ 930 if (test_bit(hwc->idx, used_mask)) 931 break; 932 933 __set_bit(hwc->idx, used_mask); 934 if (assign) 935 assign[i] = hwc->idx; 936 } 937 938 /* slow path */ 939 if (i != n) { 940 int gpmax = x86_pmu.num_counters; 941 942 /* 943 * Do not allow scheduling of more than half the available 944 * generic counters. 945 * 946 * This helps avoid counter starvation of sibling thread by 947 * ensuring at most half the counters cannot be in exclusive 948 * mode. There is no designated counters for the limits. Any 949 * N/2 counters can be used. This helps with events with 950 * specific counter constraints. 951 */ 952 if (is_ht_workaround_enabled() && !cpuc->is_fake && 953 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 954 gpmax /= 2; 955 956 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 957 wmax, gpmax, assign); 958 } 959 960 /* 961 * In case of success (unsched = 0), mark events as committed, 962 * so we do not put_constraint() in case new events are added 963 * and fail to be scheduled 964 * 965 * We invoke the lower level commit callback to lock the resource 966 * 967 * We do not need to do all of this in case we are called to 968 * validate an event group (assign == NULL) 969 */ 970 if (!unsched && assign) { 971 for (i = 0; i < n; i++) { 972 e = cpuc->event_list[i]; 973 if (x86_pmu.commit_scheduling) 974 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 975 } 976 } else { 977 for (i = n0; i < n; i++) { 978 e = cpuc->event_list[i]; 979 980 /* 981 * release events that failed scheduling 982 */ 983 if (x86_pmu.put_event_constraints) 984 x86_pmu.put_event_constraints(cpuc, e); 985 986 cpuc->event_constraint[i] = NULL; 987 } 988 } 989 990 if (x86_pmu.stop_scheduling) 991 x86_pmu.stop_scheduling(cpuc); 992 993 return unsched ? -EINVAL : 0; 994 } 995 996 /* 997 * dogrp: true if must collect siblings events (group) 998 * returns total number of events and error code 999 */ 1000 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 1001 { 1002 struct perf_event *event; 1003 int n, max_count; 1004 1005 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 1006 1007 /* current number of events already accepted */ 1008 n = cpuc->n_events; 1009 if (!cpuc->n_events) 1010 cpuc->pebs_output = 0; 1011 1012 if (!cpuc->is_fake && leader->attr.precise_ip) { 1013 /* 1014 * For PEBS->PT, if !aux_event, the group leader (PT) went 1015 * away, the group was broken down and this singleton event 1016 * can't schedule any more. 1017 */ 1018 if (is_pebs_pt(leader) && !leader->aux_event) 1019 return -EINVAL; 1020 1021 /* 1022 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS 1023 */ 1024 if (cpuc->pebs_output && 1025 cpuc->pebs_output != is_pebs_pt(leader) + 1) 1026 return -EINVAL; 1027 1028 cpuc->pebs_output = is_pebs_pt(leader) + 1; 1029 } 1030 1031 if (is_x86_event(leader)) { 1032 if (n >= max_count) 1033 return -EINVAL; 1034 cpuc->event_list[n] = leader; 1035 n++; 1036 } 1037 if (!dogrp) 1038 return n; 1039 1040 for_each_sibling_event(event, leader) { 1041 if (!is_x86_event(event) || 1042 event->state <= PERF_EVENT_STATE_OFF) 1043 continue; 1044 1045 if (n >= max_count) 1046 return -EINVAL; 1047 1048 cpuc->event_list[n] = event; 1049 n++; 1050 } 1051 return n; 1052 } 1053 1054 static inline void x86_assign_hw_event(struct perf_event *event, 1055 struct cpu_hw_events *cpuc, int i) 1056 { 1057 struct hw_perf_event *hwc = &event->hw; 1058 1059 hwc->idx = cpuc->assign[i]; 1060 hwc->last_cpu = smp_processor_id(); 1061 hwc->last_tag = ++cpuc->tags[i]; 1062 1063 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1064 hwc->config_base = 0; 1065 hwc->event_base = 0; 1066 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1067 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1068 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1069 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1070 } else { 1071 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1072 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1073 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1074 } 1075 } 1076 1077 /** 1078 * x86_perf_rdpmc_index - Return PMC counter used for event 1079 * @event: the perf_event to which the PMC counter was assigned 1080 * 1081 * The counter assigned to this performance event may change if interrupts 1082 * are enabled. This counter should thus never be used while interrupts are 1083 * enabled. Before this function is used to obtain the assigned counter the 1084 * event should be checked for validity using, for example, 1085 * perf_event_read_local(), within the same interrupt disabled section in 1086 * which this counter is planned to be used. 1087 * 1088 * Return: The index of the performance monitoring counter assigned to 1089 * @perf_event. 1090 */ 1091 int x86_perf_rdpmc_index(struct perf_event *event) 1092 { 1093 lockdep_assert_irqs_disabled(); 1094 1095 return event->hw.event_base_rdpmc; 1096 } 1097 1098 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1099 struct cpu_hw_events *cpuc, 1100 int i) 1101 { 1102 return hwc->idx == cpuc->assign[i] && 1103 hwc->last_cpu == smp_processor_id() && 1104 hwc->last_tag == cpuc->tags[i]; 1105 } 1106 1107 static void x86_pmu_start(struct perf_event *event, int flags); 1108 1109 static void x86_pmu_enable(struct pmu *pmu) 1110 { 1111 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1112 struct perf_event *event; 1113 struct hw_perf_event *hwc; 1114 int i, added = cpuc->n_added; 1115 1116 if (!x86_pmu_initialized()) 1117 return; 1118 1119 if (cpuc->enabled) 1120 return; 1121 1122 if (cpuc->n_added) { 1123 int n_running = cpuc->n_events - cpuc->n_added; 1124 /* 1125 * apply assignment obtained either from 1126 * hw_perf_group_sched_in() or x86_pmu_enable() 1127 * 1128 * step1: save events moving to new counters 1129 */ 1130 for (i = 0; i < n_running; i++) { 1131 event = cpuc->event_list[i]; 1132 hwc = &event->hw; 1133 1134 /* 1135 * we can avoid reprogramming counter if: 1136 * - assigned same counter as last time 1137 * - running on same CPU as last time 1138 * - no other event has used the counter since 1139 */ 1140 if (hwc->idx == -1 || 1141 match_prev_assignment(hwc, cpuc, i)) 1142 continue; 1143 1144 /* 1145 * Ensure we don't accidentally enable a stopped 1146 * counter simply because we rescheduled. 1147 */ 1148 if (hwc->state & PERF_HES_STOPPED) 1149 hwc->state |= PERF_HES_ARCH; 1150 1151 x86_pmu_stop(event, PERF_EF_UPDATE); 1152 } 1153 1154 /* 1155 * step2: reprogram moved events into new counters 1156 */ 1157 for (i = 0; i < cpuc->n_events; i++) { 1158 event = cpuc->event_list[i]; 1159 hwc = &event->hw; 1160 1161 if (!match_prev_assignment(hwc, cpuc, i)) 1162 x86_assign_hw_event(event, cpuc, i); 1163 else if (i < n_running) 1164 continue; 1165 1166 if (hwc->state & PERF_HES_ARCH) 1167 continue; 1168 1169 x86_pmu_start(event, PERF_EF_RELOAD); 1170 } 1171 cpuc->n_added = 0; 1172 perf_events_lapic_init(); 1173 } 1174 1175 cpuc->enabled = 1; 1176 barrier(); 1177 1178 x86_pmu.enable_all(added); 1179 } 1180 1181 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1182 1183 /* 1184 * Set the next IRQ period, based on the hwc->period_left value. 1185 * To be called with the event disabled in hw: 1186 */ 1187 int x86_perf_event_set_period(struct perf_event *event) 1188 { 1189 struct hw_perf_event *hwc = &event->hw; 1190 s64 left = local64_read(&hwc->period_left); 1191 s64 period = hwc->sample_period; 1192 int ret = 0, idx = hwc->idx; 1193 1194 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1195 return 0; 1196 1197 /* 1198 * If we are way outside a reasonable range then just skip forward: 1199 */ 1200 if (unlikely(left <= -period)) { 1201 left = period; 1202 local64_set(&hwc->period_left, left); 1203 hwc->last_period = period; 1204 ret = 1; 1205 } 1206 1207 if (unlikely(left <= 0)) { 1208 left += period; 1209 local64_set(&hwc->period_left, left); 1210 hwc->last_period = period; 1211 ret = 1; 1212 } 1213 /* 1214 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1215 */ 1216 if (unlikely(left < 2)) 1217 left = 2; 1218 1219 if (left > x86_pmu.max_period) 1220 left = x86_pmu.max_period; 1221 1222 if (x86_pmu.limit_period) 1223 left = x86_pmu.limit_period(event, left); 1224 1225 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1226 1227 /* 1228 * The hw event starts counting from this event offset, 1229 * mark it to be able to extra future deltas: 1230 */ 1231 local64_set(&hwc->prev_count, (u64)-left); 1232 1233 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1234 1235 /* 1236 * Due to erratum on certan cpu we need 1237 * a second write to be sure the register 1238 * is updated properly 1239 */ 1240 if (x86_pmu.perfctr_second_write) { 1241 wrmsrl(hwc->event_base, 1242 (u64)(-left) & x86_pmu.cntval_mask); 1243 } 1244 1245 perf_event_update_userpage(event); 1246 1247 return ret; 1248 } 1249 1250 void x86_pmu_enable_event(struct perf_event *event) 1251 { 1252 if (__this_cpu_read(cpu_hw_events.enabled)) 1253 __x86_pmu_enable_event(&event->hw, 1254 ARCH_PERFMON_EVENTSEL_ENABLE); 1255 } 1256 1257 /* 1258 * Add a single event to the PMU. 1259 * 1260 * The event is added to the group of enabled events 1261 * but only if it can be scheduled with existing events. 1262 */ 1263 static int x86_pmu_add(struct perf_event *event, int flags) 1264 { 1265 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1266 struct hw_perf_event *hwc; 1267 int assign[X86_PMC_IDX_MAX]; 1268 int n, n0, ret; 1269 1270 hwc = &event->hw; 1271 1272 n0 = cpuc->n_events; 1273 ret = n = collect_events(cpuc, event, false); 1274 if (ret < 0) 1275 goto out; 1276 1277 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1278 if (!(flags & PERF_EF_START)) 1279 hwc->state |= PERF_HES_ARCH; 1280 1281 /* 1282 * If group events scheduling transaction was started, 1283 * skip the schedulability test here, it will be performed 1284 * at commit time (->commit_txn) as a whole. 1285 * 1286 * If commit fails, we'll call ->del() on all events 1287 * for which ->add() was called. 1288 */ 1289 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1290 goto done_collect; 1291 1292 ret = x86_pmu.schedule_events(cpuc, n, assign); 1293 if (ret) 1294 goto out; 1295 /* 1296 * copy new assignment, now we know it is possible 1297 * will be used by hw_perf_enable() 1298 */ 1299 memcpy(cpuc->assign, assign, n*sizeof(int)); 1300 1301 done_collect: 1302 /* 1303 * Commit the collect_events() state. See x86_pmu_del() and 1304 * x86_pmu_*_txn(). 1305 */ 1306 cpuc->n_events = n; 1307 cpuc->n_added += n - n0; 1308 cpuc->n_txn += n - n0; 1309 1310 if (x86_pmu.add) { 1311 /* 1312 * This is before x86_pmu_enable() will call x86_pmu_start(), 1313 * so we enable LBRs before an event needs them etc.. 1314 */ 1315 x86_pmu.add(event); 1316 } 1317 1318 ret = 0; 1319 out: 1320 return ret; 1321 } 1322 1323 static void x86_pmu_start(struct perf_event *event, int flags) 1324 { 1325 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1326 int idx = event->hw.idx; 1327 1328 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1329 return; 1330 1331 if (WARN_ON_ONCE(idx == -1)) 1332 return; 1333 1334 if (flags & PERF_EF_RELOAD) { 1335 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1336 x86_perf_event_set_period(event); 1337 } 1338 1339 event->hw.state = 0; 1340 1341 cpuc->events[idx] = event; 1342 __set_bit(idx, cpuc->active_mask); 1343 __set_bit(idx, cpuc->running); 1344 x86_pmu.enable(event); 1345 perf_event_update_userpage(event); 1346 } 1347 1348 void perf_event_print_debug(void) 1349 { 1350 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1351 u64 pebs, debugctl; 1352 struct cpu_hw_events *cpuc; 1353 unsigned long flags; 1354 int cpu, idx; 1355 1356 if (!x86_pmu.num_counters) 1357 return; 1358 1359 local_irq_save(flags); 1360 1361 cpu = smp_processor_id(); 1362 cpuc = &per_cpu(cpu_hw_events, cpu); 1363 1364 if (x86_pmu.version >= 2) { 1365 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1366 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1367 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1368 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1369 1370 pr_info("\n"); 1371 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1372 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1373 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1374 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1375 if (x86_pmu.pebs_constraints) { 1376 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1377 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1378 } 1379 if (x86_pmu.lbr_nr) { 1380 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1381 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1382 } 1383 } 1384 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1385 1386 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1387 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1388 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1389 1390 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1391 1392 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1393 cpu, idx, pmc_ctrl); 1394 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1395 cpu, idx, pmc_count); 1396 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1397 cpu, idx, prev_left); 1398 } 1399 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1400 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1401 1402 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1403 cpu, idx, pmc_count); 1404 } 1405 local_irq_restore(flags); 1406 } 1407 1408 void x86_pmu_stop(struct perf_event *event, int flags) 1409 { 1410 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1411 struct hw_perf_event *hwc = &event->hw; 1412 1413 if (test_bit(hwc->idx, cpuc->active_mask)) { 1414 x86_pmu.disable(event); 1415 __clear_bit(hwc->idx, cpuc->active_mask); 1416 cpuc->events[hwc->idx] = NULL; 1417 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1418 hwc->state |= PERF_HES_STOPPED; 1419 } 1420 1421 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1422 /* 1423 * Drain the remaining delta count out of a event 1424 * that we are disabling: 1425 */ 1426 x86_perf_event_update(event); 1427 hwc->state |= PERF_HES_UPTODATE; 1428 } 1429 } 1430 1431 static void x86_pmu_del(struct perf_event *event, int flags) 1432 { 1433 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1434 int i; 1435 1436 /* 1437 * If we're called during a txn, we only need to undo x86_pmu.add. 1438 * The events never got scheduled and ->cancel_txn will truncate 1439 * the event_list. 1440 * 1441 * XXX assumes any ->del() called during a TXN will only be on 1442 * an event added during that same TXN. 1443 */ 1444 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1445 goto do_del; 1446 1447 /* 1448 * Not a TXN, therefore cleanup properly. 1449 */ 1450 x86_pmu_stop(event, PERF_EF_UPDATE); 1451 1452 for (i = 0; i < cpuc->n_events; i++) { 1453 if (event == cpuc->event_list[i]) 1454 break; 1455 } 1456 1457 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1458 return; 1459 1460 /* If we have a newly added event; make sure to decrease n_added. */ 1461 if (i >= cpuc->n_events - cpuc->n_added) 1462 --cpuc->n_added; 1463 1464 if (x86_pmu.put_event_constraints) 1465 x86_pmu.put_event_constraints(cpuc, event); 1466 1467 /* Delete the array entry. */ 1468 while (++i < cpuc->n_events) { 1469 cpuc->event_list[i-1] = cpuc->event_list[i]; 1470 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1471 } 1472 cpuc->event_constraint[i-1] = NULL; 1473 --cpuc->n_events; 1474 1475 perf_event_update_userpage(event); 1476 1477 do_del: 1478 if (x86_pmu.del) { 1479 /* 1480 * This is after x86_pmu_stop(); so we disable LBRs after any 1481 * event can need them etc.. 1482 */ 1483 x86_pmu.del(event); 1484 } 1485 } 1486 1487 int x86_pmu_handle_irq(struct pt_regs *regs) 1488 { 1489 struct perf_sample_data data; 1490 struct cpu_hw_events *cpuc; 1491 struct perf_event *event; 1492 int idx, handled = 0; 1493 u64 val; 1494 1495 cpuc = this_cpu_ptr(&cpu_hw_events); 1496 1497 /* 1498 * Some chipsets need to unmask the LVTPC in a particular spot 1499 * inside the nmi handler. As a result, the unmasking was pushed 1500 * into all the nmi handlers. 1501 * 1502 * This generic handler doesn't seem to have any issues where the 1503 * unmasking occurs so it was left at the top. 1504 */ 1505 apic_write(APIC_LVTPC, APIC_DM_NMI); 1506 1507 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1508 if (!test_bit(idx, cpuc->active_mask)) 1509 continue; 1510 1511 event = cpuc->events[idx]; 1512 1513 val = x86_perf_event_update(event); 1514 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1515 continue; 1516 1517 /* 1518 * event overflow 1519 */ 1520 handled++; 1521 perf_sample_data_init(&data, 0, event->hw.last_period); 1522 1523 if (!x86_perf_event_set_period(event)) 1524 continue; 1525 1526 if (perf_event_overflow(event, &data, regs)) 1527 x86_pmu_stop(event, 0); 1528 } 1529 1530 if (handled) 1531 inc_irq_stat(apic_perf_irqs); 1532 1533 return handled; 1534 } 1535 1536 void perf_events_lapic_init(void) 1537 { 1538 if (!x86_pmu.apic || !x86_pmu_initialized()) 1539 return; 1540 1541 /* 1542 * Always use NMI for PMU 1543 */ 1544 apic_write(APIC_LVTPC, APIC_DM_NMI); 1545 } 1546 1547 static int 1548 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1549 { 1550 u64 start_clock; 1551 u64 finish_clock; 1552 int ret; 1553 1554 /* 1555 * All PMUs/events that share this PMI handler should make sure to 1556 * increment active_events for their events. 1557 */ 1558 if (!atomic_read(&active_events)) 1559 return NMI_DONE; 1560 1561 start_clock = sched_clock(); 1562 ret = x86_pmu.handle_irq(regs); 1563 finish_clock = sched_clock(); 1564 1565 perf_sample_event_took(finish_clock - start_clock); 1566 1567 return ret; 1568 } 1569 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1570 1571 struct event_constraint emptyconstraint; 1572 struct event_constraint unconstrained; 1573 1574 static int x86_pmu_prepare_cpu(unsigned int cpu) 1575 { 1576 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1577 int i; 1578 1579 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1580 cpuc->kfree_on_online[i] = NULL; 1581 if (x86_pmu.cpu_prepare) 1582 return x86_pmu.cpu_prepare(cpu); 1583 return 0; 1584 } 1585 1586 static int x86_pmu_dead_cpu(unsigned int cpu) 1587 { 1588 if (x86_pmu.cpu_dead) 1589 x86_pmu.cpu_dead(cpu); 1590 return 0; 1591 } 1592 1593 static int x86_pmu_online_cpu(unsigned int cpu) 1594 { 1595 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1596 int i; 1597 1598 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1599 kfree(cpuc->kfree_on_online[i]); 1600 cpuc->kfree_on_online[i] = NULL; 1601 } 1602 return 0; 1603 } 1604 1605 static int x86_pmu_starting_cpu(unsigned int cpu) 1606 { 1607 if (x86_pmu.cpu_starting) 1608 x86_pmu.cpu_starting(cpu); 1609 return 0; 1610 } 1611 1612 static int x86_pmu_dying_cpu(unsigned int cpu) 1613 { 1614 if (x86_pmu.cpu_dying) 1615 x86_pmu.cpu_dying(cpu); 1616 return 0; 1617 } 1618 1619 static void __init pmu_check_apic(void) 1620 { 1621 if (boot_cpu_has(X86_FEATURE_APIC)) 1622 return; 1623 1624 x86_pmu.apic = 0; 1625 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1626 pr_info("no hardware sampling interrupt available.\n"); 1627 1628 /* 1629 * If we have a PMU initialized but no APIC 1630 * interrupts, we cannot sample hardware 1631 * events (user-space has to fall back and 1632 * sample via a hrtimer based software event): 1633 */ 1634 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1635 1636 } 1637 1638 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1639 .name = "format", 1640 .attrs = NULL, 1641 }; 1642 1643 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1644 { 1645 struct perf_pmu_events_attr *pmu_attr = \ 1646 container_of(attr, struct perf_pmu_events_attr, attr); 1647 u64 config = x86_pmu.event_map(pmu_attr->id); 1648 1649 /* string trumps id */ 1650 if (pmu_attr->event_str) 1651 return sprintf(page, "%s", pmu_attr->event_str); 1652 1653 return x86_pmu.events_sysfs_show(page, config); 1654 } 1655 EXPORT_SYMBOL_GPL(events_sysfs_show); 1656 1657 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1658 char *page) 1659 { 1660 struct perf_pmu_events_ht_attr *pmu_attr = 1661 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1662 1663 /* 1664 * Report conditional events depending on Hyper-Threading. 1665 * 1666 * This is overly conservative as usually the HT special 1667 * handling is not needed if the other CPU thread is idle. 1668 * 1669 * Note this does not (and cannot) handle the case when thread 1670 * siblings are invisible, for example with virtualization 1671 * if they are owned by some other guest. The user tool 1672 * has to re-read when a thread sibling gets onlined later. 1673 */ 1674 return sprintf(page, "%s", 1675 topology_max_smt_threads() > 1 ? 1676 pmu_attr->event_str_ht : 1677 pmu_attr->event_str_noht); 1678 } 1679 1680 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1681 EVENT_ATTR(instructions, INSTRUCTIONS ); 1682 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1683 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1684 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1685 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1686 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1687 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1688 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1689 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1690 1691 static struct attribute *empty_attrs; 1692 1693 static struct attribute *events_attr[] = { 1694 EVENT_PTR(CPU_CYCLES), 1695 EVENT_PTR(INSTRUCTIONS), 1696 EVENT_PTR(CACHE_REFERENCES), 1697 EVENT_PTR(CACHE_MISSES), 1698 EVENT_PTR(BRANCH_INSTRUCTIONS), 1699 EVENT_PTR(BRANCH_MISSES), 1700 EVENT_PTR(BUS_CYCLES), 1701 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1702 EVENT_PTR(STALLED_CYCLES_BACKEND), 1703 EVENT_PTR(REF_CPU_CYCLES), 1704 NULL, 1705 }; 1706 1707 /* 1708 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1709 * out of events_attr attributes. 1710 */ 1711 static umode_t 1712 is_visible(struct kobject *kobj, struct attribute *attr, int idx) 1713 { 1714 struct perf_pmu_events_attr *pmu_attr; 1715 1716 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); 1717 /* str trumps id */ 1718 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; 1719 } 1720 1721 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1722 .name = "events", 1723 .attrs = events_attr, 1724 .is_visible = is_visible, 1725 }; 1726 1727 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1728 { 1729 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1730 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1731 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1732 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1733 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1734 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1735 ssize_t ret; 1736 1737 /* 1738 * We have whole page size to spend and just little data 1739 * to write, so we can safely use sprintf. 1740 */ 1741 ret = sprintf(page, "event=0x%02llx", event); 1742 1743 if (umask) 1744 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1745 1746 if (edge) 1747 ret += sprintf(page + ret, ",edge"); 1748 1749 if (pc) 1750 ret += sprintf(page + ret, ",pc"); 1751 1752 if (any) 1753 ret += sprintf(page + ret, ",any"); 1754 1755 if (inv) 1756 ret += sprintf(page + ret, ",inv"); 1757 1758 if (cmask) 1759 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1760 1761 ret += sprintf(page + ret, "\n"); 1762 1763 return ret; 1764 } 1765 1766 static struct attribute_group x86_pmu_attr_group; 1767 static struct attribute_group x86_pmu_caps_group; 1768 1769 static int __init init_hw_perf_events(void) 1770 { 1771 struct x86_pmu_quirk *quirk; 1772 int err; 1773 1774 pr_info("Performance Events: "); 1775 1776 switch (boot_cpu_data.x86_vendor) { 1777 case X86_VENDOR_INTEL: 1778 err = intel_pmu_init(); 1779 break; 1780 case X86_VENDOR_AMD: 1781 err = amd_pmu_init(); 1782 break; 1783 case X86_VENDOR_HYGON: 1784 err = amd_pmu_init(); 1785 x86_pmu.name = "HYGON"; 1786 break; 1787 default: 1788 err = -ENOTSUPP; 1789 } 1790 if (err != 0) { 1791 pr_cont("no PMU driver, software events only.\n"); 1792 return 0; 1793 } 1794 1795 pmu_check_apic(); 1796 1797 /* sanity check that the hardware exists or is emulated */ 1798 if (!check_hw_exists()) 1799 return 0; 1800 1801 pr_cont("%s PMU driver.\n", x86_pmu.name); 1802 1803 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1804 1805 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1806 quirk->func(); 1807 1808 if (!x86_pmu.intel_ctrl) 1809 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1810 1811 perf_events_lapic_init(); 1812 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1813 1814 unconstrained = (struct event_constraint) 1815 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1816 0, x86_pmu.num_counters, 0, 0); 1817 1818 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1819 1820 if (!x86_pmu.events_sysfs_show) 1821 x86_pmu_events_group.attrs = &empty_attrs; 1822 1823 pmu.attr_update = x86_pmu.attr_update; 1824 1825 pr_info("... version: %d\n", x86_pmu.version); 1826 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1827 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1828 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1829 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1830 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1831 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1832 1833 /* 1834 * Install callbacks. Core will call them for each online 1835 * cpu. 1836 */ 1837 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1838 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1839 if (err) 1840 return err; 1841 1842 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1843 "perf/x86:starting", x86_pmu_starting_cpu, 1844 x86_pmu_dying_cpu); 1845 if (err) 1846 goto out; 1847 1848 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1849 x86_pmu_online_cpu, NULL); 1850 if (err) 1851 goto out1; 1852 1853 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1854 if (err) 1855 goto out2; 1856 1857 return 0; 1858 1859 out2: 1860 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1861 out1: 1862 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1863 out: 1864 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1865 return err; 1866 } 1867 early_initcall(init_hw_perf_events); 1868 1869 static inline void x86_pmu_read(struct perf_event *event) 1870 { 1871 if (x86_pmu.read) 1872 return x86_pmu.read(event); 1873 x86_perf_event_update(event); 1874 } 1875 1876 /* 1877 * Start group events scheduling transaction 1878 * Set the flag to make pmu::enable() not perform the 1879 * schedulability test, it will be performed at commit time 1880 * 1881 * We only support PERF_PMU_TXN_ADD transactions. Save the 1882 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1883 * transactions. 1884 */ 1885 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1886 { 1887 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1888 1889 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1890 1891 cpuc->txn_flags = txn_flags; 1892 if (txn_flags & ~PERF_PMU_TXN_ADD) 1893 return; 1894 1895 perf_pmu_disable(pmu); 1896 __this_cpu_write(cpu_hw_events.n_txn, 0); 1897 } 1898 1899 /* 1900 * Stop group events scheduling transaction 1901 * Clear the flag and pmu::enable() will perform the 1902 * schedulability test. 1903 */ 1904 static void x86_pmu_cancel_txn(struct pmu *pmu) 1905 { 1906 unsigned int txn_flags; 1907 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1908 1909 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1910 1911 txn_flags = cpuc->txn_flags; 1912 cpuc->txn_flags = 0; 1913 if (txn_flags & ~PERF_PMU_TXN_ADD) 1914 return; 1915 1916 /* 1917 * Truncate collected array by the number of events added in this 1918 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1919 */ 1920 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1921 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1922 perf_pmu_enable(pmu); 1923 } 1924 1925 /* 1926 * Commit group events scheduling transaction 1927 * Perform the group schedulability test as a whole 1928 * Return 0 if success 1929 * 1930 * Does not cancel the transaction on failure; expects the caller to do this. 1931 */ 1932 static int x86_pmu_commit_txn(struct pmu *pmu) 1933 { 1934 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1935 int assign[X86_PMC_IDX_MAX]; 1936 int n, ret; 1937 1938 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1939 1940 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1941 cpuc->txn_flags = 0; 1942 return 0; 1943 } 1944 1945 n = cpuc->n_events; 1946 1947 if (!x86_pmu_initialized()) 1948 return -EAGAIN; 1949 1950 ret = x86_pmu.schedule_events(cpuc, n, assign); 1951 if (ret) 1952 return ret; 1953 1954 /* 1955 * copy new assignment, now we know it is possible 1956 * will be used by hw_perf_enable() 1957 */ 1958 memcpy(cpuc->assign, assign, n*sizeof(int)); 1959 1960 cpuc->txn_flags = 0; 1961 perf_pmu_enable(pmu); 1962 return 0; 1963 } 1964 /* 1965 * a fake_cpuc is used to validate event groups. Due to 1966 * the extra reg logic, we need to also allocate a fake 1967 * per_core and per_cpu structure. Otherwise, group events 1968 * using extra reg may conflict without the kernel being 1969 * able to catch this when the last event gets added to 1970 * the group. 1971 */ 1972 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 1973 { 1974 intel_cpuc_finish(cpuc); 1975 kfree(cpuc); 1976 } 1977 1978 static struct cpu_hw_events *allocate_fake_cpuc(void) 1979 { 1980 struct cpu_hw_events *cpuc; 1981 int cpu = raw_smp_processor_id(); 1982 1983 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 1984 if (!cpuc) 1985 return ERR_PTR(-ENOMEM); 1986 cpuc->is_fake = 1; 1987 1988 if (intel_cpuc_prepare(cpuc, cpu)) 1989 goto error; 1990 1991 return cpuc; 1992 error: 1993 free_fake_cpuc(cpuc); 1994 return ERR_PTR(-ENOMEM); 1995 } 1996 1997 /* 1998 * validate that we can schedule this event 1999 */ 2000 static int validate_event(struct perf_event *event) 2001 { 2002 struct cpu_hw_events *fake_cpuc; 2003 struct event_constraint *c; 2004 int ret = 0; 2005 2006 fake_cpuc = allocate_fake_cpuc(); 2007 if (IS_ERR(fake_cpuc)) 2008 return PTR_ERR(fake_cpuc); 2009 2010 c = x86_pmu.get_event_constraints(fake_cpuc, 0, event); 2011 2012 if (!c || !c->weight) 2013 ret = -EINVAL; 2014 2015 if (x86_pmu.put_event_constraints) 2016 x86_pmu.put_event_constraints(fake_cpuc, event); 2017 2018 free_fake_cpuc(fake_cpuc); 2019 2020 return ret; 2021 } 2022 2023 /* 2024 * validate a single event group 2025 * 2026 * validation include: 2027 * - check events are compatible which each other 2028 * - events do not compete for the same counter 2029 * - number of events <= number of counters 2030 * 2031 * validation ensures the group can be loaded onto the 2032 * PMU if it was the only group available. 2033 */ 2034 static int validate_group(struct perf_event *event) 2035 { 2036 struct perf_event *leader = event->group_leader; 2037 struct cpu_hw_events *fake_cpuc; 2038 int ret = -EINVAL, n; 2039 2040 fake_cpuc = allocate_fake_cpuc(); 2041 if (IS_ERR(fake_cpuc)) 2042 return PTR_ERR(fake_cpuc); 2043 /* 2044 * the event is not yet connected with its 2045 * siblings therefore we must first collect 2046 * existing siblings, then add the new event 2047 * before we can simulate the scheduling 2048 */ 2049 n = collect_events(fake_cpuc, leader, true); 2050 if (n < 0) 2051 goto out; 2052 2053 fake_cpuc->n_events = n; 2054 n = collect_events(fake_cpuc, event, false); 2055 if (n < 0) 2056 goto out; 2057 2058 fake_cpuc->n_events = 0; 2059 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2060 2061 out: 2062 free_fake_cpuc(fake_cpuc); 2063 return ret; 2064 } 2065 2066 static int x86_pmu_event_init(struct perf_event *event) 2067 { 2068 struct pmu *tmp; 2069 int err; 2070 2071 switch (event->attr.type) { 2072 case PERF_TYPE_RAW: 2073 case PERF_TYPE_HARDWARE: 2074 case PERF_TYPE_HW_CACHE: 2075 break; 2076 2077 default: 2078 return -ENOENT; 2079 } 2080 2081 err = __x86_pmu_event_init(event); 2082 if (!err) { 2083 /* 2084 * we temporarily connect event to its pmu 2085 * such that validate_group() can classify 2086 * it as an x86 event using is_x86_event() 2087 */ 2088 tmp = event->pmu; 2089 event->pmu = &pmu; 2090 2091 if (event->group_leader != event) 2092 err = validate_group(event); 2093 else 2094 err = validate_event(event); 2095 2096 event->pmu = tmp; 2097 } 2098 if (err) { 2099 if (event->destroy) 2100 event->destroy(event); 2101 } 2102 2103 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2104 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2105 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2106 2107 return err; 2108 } 2109 2110 static void refresh_pce(void *ignored) 2111 { 2112 load_mm_cr4_irqsoff(this_cpu_read(cpu_tlbstate.loaded_mm)); 2113 } 2114 2115 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2116 { 2117 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2118 return; 2119 2120 /* 2121 * This function relies on not being called concurrently in two 2122 * tasks in the same mm. Otherwise one task could observe 2123 * perf_rdpmc_allowed > 1 and return all the way back to 2124 * userspace with CR4.PCE clear while another task is still 2125 * doing on_each_cpu_mask() to propagate CR4.PCE. 2126 * 2127 * For now, this can't happen because all callers hold mmap_sem 2128 * for write. If this changes, we'll need a different solution. 2129 */ 2130 lockdep_assert_held_write(&mm->mmap_sem); 2131 2132 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2133 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2134 } 2135 2136 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2137 { 2138 2139 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2140 return; 2141 2142 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2143 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2144 } 2145 2146 static int x86_pmu_event_idx(struct perf_event *event) 2147 { 2148 int idx = event->hw.idx; 2149 2150 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2151 return 0; 2152 2153 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2154 idx -= INTEL_PMC_IDX_FIXED; 2155 idx |= 1 << 30; 2156 } 2157 2158 return idx + 1; 2159 } 2160 2161 static ssize_t get_attr_rdpmc(struct device *cdev, 2162 struct device_attribute *attr, 2163 char *buf) 2164 { 2165 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2166 } 2167 2168 static ssize_t set_attr_rdpmc(struct device *cdev, 2169 struct device_attribute *attr, 2170 const char *buf, size_t count) 2171 { 2172 unsigned long val; 2173 ssize_t ret; 2174 2175 ret = kstrtoul(buf, 0, &val); 2176 if (ret) 2177 return ret; 2178 2179 if (val > 2) 2180 return -EINVAL; 2181 2182 if (x86_pmu.attr_rdpmc_broken) 2183 return -ENOTSUPP; 2184 2185 if (val != x86_pmu.attr_rdpmc) { 2186 /* 2187 * Changing into or out of never available or always available, 2188 * aka perf-event-bypassing mode. This path is extremely slow, 2189 * but only root can trigger it, so it's okay. 2190 */ 2191 if (val == 0) 2192 static_branch_inc(&rdpmc_never_available_key); 2193 else if (x86_pmu.attr_rdpmc == 0) 2194 static_branch_dec(&rdpmc_never_available_key); 2195 2196 if (val == 2) 2197 static_branch_inc(&rdpmc_always_available_key); 2198 else if (x86_pmu.attr_rdpmc == 2) 2199 static_branch_dec(&rdpmc_always_available_key); 2200 2201 on_each_cpu(refresh_pce, NULL, 1); 2202 x86_pmu.attr_rdpmc = val; 2203 } 2204 2205 return count; 2206 } 2207 2208 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2209 2210 static struct attribute *x86_pmu_attrs[] = { 2211 &dev_attr_rdpmc.attr, 2212 NULL, 2213 }; 2214 2215 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2216 .attrs = x86_pmu_attrs, 2217 }; 2218 2219 static ssize_t max_precise_show(struct device *cdev, 2220 struct device_attribute *attr, 2221 char *buf) 2222 { 2223 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2224 } 2225 2226 static DEVICE_ATTR_RO(max_precise); 2227 2228 static struct attribute *x86_pmu_caps_attrs[] = { 2229 &dev_attr_max_precise.attr, 2230 NULL 2231 }; 2232 2233 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2234 .name = "caps", 2235 .attrs = x86_pmu_caps_attrs, 2236 }; 2237 2238 static const struct attribute_group *x86_pmu_attr_groups[] = { 2239 &x86_pmu_attr_group, 2240 &x86_pmu_format_group, 2241 &x86_pmu_events_group, 2242 &x86_pmu_caps_group, 2243 NULL, 2244 }; 2245 2246 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2247 { 2248 if (x86_pmu.sched_task) 2249 x86_pmu.sched_task(ctx, sched_in); 2250 } 2251 2252 static void x86_pmu_swap_task_ctx(struct perf_event_context *prev, 2253 struct perf_event_context *next) 2254 { 2255 if (x86_pmu.swap_task_ctx) 2256 x86_pmu.swap_task_ctx(prev, next); 2257 } 2258 2259 void perf_check_microcode(void) 2260 { 2261 if (x86_pmu.check_microcode) 2262 x86_pmu.check_microcode(); 2263 } 2264 2265 static int x86_pmu_check_period(struct perf_event *event, u64 value) 2266 { 2267 if (x86_pmu.check_period && x86_pmu.check_period(event, value)) 2268 return -EINVAL; 2269 2270 if (value && x86_pmu.limit_period) { 2271 if (x86_pmu.limit_period(event, value) > value) 2272 return -EINVAL; 2273 } 2274 2275 return 0; 2276 } 2277 2278 static int x86_pmu_aux_output_match(struct perf_event *event) 2279 { 2280 if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT)) 2281 return 0; 2282 2283 if (x86_pmu.aux_output_match) 2284 return x86_pmu.aux_output_match(event); 2285 2286 return 0; 2287 } 2288 2289 static struct pmu pmu = { 2290 .pmu_enable = x86_pmu_enable, 2291 .pmu_disable = x86_pmu_disable, 2292 2293 .attr_groups = x86_pmu_attr_groups, 2294 2295 .event_init = x86_pmu_event_init, 2296 2297 .event_mapped = x86_pmu_event_mapped, 2298 .event_unmapped = x86_pmu_event_unmapped, 2299 2300 .add = x86_pmu_add, 2301 .del = x86_pmu_del, 2302 .start = x86_pmu_start, 2303 .stop = x86_pmu_stop, 2304 .read = x86_pmu_read, 2305 2306 .start_txn = x86_pmu_start_txn, 2307 .cancel_txn = x86_pmu_cancel_txn, 2308 .commit_txn = x86_pmu_commit_txn, 2309 2310 .event_idx = x86_pmu_event_idx, 2311 .sched_task = x86_pmu_sched_task, 2312 .task_ctx_size = sizeof(struct x86_perf_task_context), 2313 .swap_task_ctx = x86_pmu_swap_task_ctx, 2314 .check_period = x86_pmu_check_period, 2315 2316 .aux_output_match = x86_pmu_aux_output_match, 2317 }; 2318 2319 void arch_perf_update_userpage(struct perf_event *event, 2320 struct perf_event_mmap_page *userpg, u64 now) 2321 { 2322 struct cyc2ns_data data; 2323 u64 offset; 2324 2325 userpg->cap_user_time = 0; 2326 userpg->cap_user_time_zero = 0; 2327 userpg->cap_user_rdpmc = 2328 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2329 userpg->pmc_width = x86_pmu.cntval_bits; 2330 2331 if (!using_native_sched_clock() || !sched_clock_stable()) 2332 return; 2333 2334 cyc2ns_read_begin(&data); 2335 2336 offset = data.cyc2ns_offset + __sched_clock_offset; 2337 2338 /* 2339 * Internal timekeeping for enabled/running/stopped times 2340 * is always in the local_clock domain. 2341 */ 2342 userpg->cap_user_time = 1; 2343 userpg->time_mult = data.cyc2ns_mul; 2344 userpg->time_shift = data.cyc2ns_shift; 2345 userpg->time_offset = offset - now; 2346 2347 /* 2348 * cap_user_time_zero doesn't make sense when we're using a different 2349 * time base for the records. 2350 */ 2351 if (!event->attr.use_clockid) { 2352 userpg->cap_user_time_zero = 1; 2353 userpg->time_zero = offset; 2354 } 2355 2356 cyc2ns_read_end(); 2357 } 2358 2359 /* 2360 * Determine whether the regs were taken from an irq/exception handler rather 2361 * than from perf_arch_fetch_caller_regs(). 2362 */ 2363 static bool perf_hw_regs(struct pt_regs *regs) 2364 { 2365 return regs->flags & X86_EFLAGS_FIXED; 2366 } 2367 2368 void 2369 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2370 { 2371 struct unwind_state state; 2372 unsigned long addr; 2373 2374 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2375 /* TODO: We don't support guest os callchain now */ 2376 return; 2377 } 2378 2379 if (perf_callchain_store(entry, regs->ip)) 2380 return; 2381 2382 if (perf_hw_regs(regs)) 2383 unwind_start(&state, current, regs, NULL); 2384 else 2385 unwind_start(&state, current, NULL, (void *)regs->sp); 2386 2387 for (; !unwind_done(&state); unwind_next_frame(&state)) { 2388 addr = unwind_get_return_address(&state); 2389 if (!addr || perf_callchain_store(entry, addr)) 2390 return; 2391 } 2392 } 2393 2394 static inline int 2395 valid_user_frame(const void __user *fp, unsigned long size) 2396 { 2397 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2398 } 2399 2400 static unsigned long get_segment_base(unsigned int segment) 2401 { 2402 struct desc_struct *desc; 2403 unsigned int idx = segment >> 3; 2404 2405 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2406 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2407 struct ldt_struct *ldt; 2408 2409 /* IRQs are off, so this synchronizes with smp_store_release */ 2410 ldt = READ_ONCE(current->active_mm->context.ldt); 2411 if (!ldt || idx >= ldt->nr_entries) 2412 return 0; 2413 2414 desc = &ldt->entries[idx]; 2415 #else 2416 return 0; 2417 #endif 2418 } else { 2419 if (idx >= GDT_ENTRIES) 2420 return 0; 2421 2422 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2423 } 2424 2425 return get_desc_base(desc); 2426 } 2427 2428 #ifdef CONFIG_IA32_EMULATION 2429 2430 #include <linux/compat.h> 2431 2432 static inline int 2433 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2434 { 2435 /* 32-bit process in 64-bit kernel. */ 2436 unsigned long ss_base, cs_base; 2437 struct stack_frame_ia32 frame; 2438 const void __user *fp; 2439 2440 if (!test_thread_flag(TIF_IA32)) 2441 return 0; 2442 2443 cs_base = get_segment_base(regs->cs); 2444 ss_base = get_segment_base(regs->ss); 2445 2446 fp = compat_ptr(ss_base + regs->bp); 2447 pagefault_disable(); 2448 while (entry->nr < entry->max_stack) { 2449 unsigned long bytes; 2450 frame.next_frame = 0; 2451 frame.return_address = 0; 2452 2453 if (!valid_user_frame(fp, sizeof(frame))) 2454 break; 2455 2456 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2457 if (bytes != 0) 2458 break; 2459 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2460 if (bytes != 0) 2461 break; 2462 2463 perf_callchain_store(entry, cs_base + frame.return_address); 2464 fp = compat_ptr(ss_base + frame.next_frame); 2465 } 2466 pagefault_enable(); 2467 return 1; 2468 } 2469 #else 2470 static inline int 2471 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2472 { 2473 return 0; 2474 } 2475 #endif 2476 2477 void 2478 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2479 { 2480 struct stack_frame frame; 2481 const unsigned long __user *fp; 2482 2483 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2484 /* TODO: We don't support guest os callchain now */ 2485 return; 2486 } 2487 2488 /* 2489 * We don't know what to do with VM86 stacks.. ignore them for now. 2490 */ 2491 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2492 return; 2493 2494 fp = (unsigned long __user *)regs->bp; 2495 2496 perf_callchain_store(entry, regs->ip); 2497 2498 if (!nmi_uaccess_okay()) 2499 return; 2500 2501 if (perf_callchain_user32(regs, entry)) 2502 return; 2503 2504 pagefault_disable(); 2505 while (entry->nr < entry->max_stack) { 2506 unsigned long bytes; 2507 2508 frame.next_frame = NULL; 2509 frame.return_address = 0; 2510 2511 if (!valid_user_frame(fp, sizeof(frame))) 2512 break; 2513 2514 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2515 if (bytes != 0) 2516 break; 2517 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2518 if (bytes != 0) 2519 break; 2520 2521 perf_callchain_store(entry, frame.return_address); 2522 fp = (void __user *)frame.next_frame; 2523 } 2524 pagefault_enable(); 2525 } 2526 2527 /* 2528 * Deal with code segment offsets for the various execution modes: 2529 * 2530 * VM86 - the good olde 16 bit days, where the linear address is 2531 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2532 * 2533 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2534 * to figure out what the 32bit base address is. 2535 * 2536 * X32 - has TIF_X32 set, but is running in x86_64 2537 * 2538 * X86_64 - CS,DS,SS,ES are all zero based. 2539 */ 2540 static unsigned long code_segment_base(struct pt_regs *regs) 2541 { 2542 /* 2543 * For IA32 we look at the GDT/LDT segment base to convert the 2544 * effective IP to a linear address. 2545 */ 2546 2547 #ifdef CONFIG_X86_32 2548 /* 2549 * If we are in VM86 mode, add the segment offset to convert to a 2550 * linear address. 2551 */ 2552 if (regs->flags & X86_VM_MASK) 2553 return 0x10 * regs->cs; 2554 2555 if (user_mode(regs) && regs->cs != __USER_CS) 2556 return get_segment_base(regs->cs); 2557 #else 2558 if (user_mode(regs) && !user_64bit_mode(regs) && 2559 regs->cs != __USER32_CS) 2560 return get_segment_base(regs->cs); 2561 #endif 2562 return 0; 2563 } 2564 2565 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2566 { 2567 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2568 return perf_guest_cbs->get_guest_ip(); 2569 2570 return regs->ip + code_segment_base(regs); 2571 } 2572 2573 unsigned long perf_misc_flags(struct pt_regs *regs) 2574 { 2575 int misc = 0; 2576 2577 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2578 if (perf_guest_cbs->is_user_mode()) 2579 misc |= PERF_RECORD_MISC_GUEST_USER; 2580 else 2581 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2582 } else { 2583 if (user_mode(regs)) 2584 misc |= PERF_RECORD_MISC_USER; 2585 else 2586 misc |= PERF_RECORD_MISC_KERNEL; 2587 } 2588 2589 if (regs->flags & PERF_EFLAGS_EXACT) 2590 misc |= PERF_RECORD_MISC_EXACT_IP; 2591 2592 return misc; 2593 } 2594 2595 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2596 { 2597 cap->version = x86_pmu.version; 2598 cap->num_counters_gp = x86_pmu.num_counters; 2599 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2600 cap->bit_width_gp = x86_pmu.cntval_bits; 2601 cap->bit_width_fixed = x86_pmu.cntval_bits; 2602 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2603 cap->events_mask_len = x86_pmu.events_mask_len; 2604 } 2605 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2606