1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched/mm.h> 24 #include <linux/sched/clock.h> 25 #include <linux/uaccess.h> 26 #include <linux/slab.h> 27 #include <linux/cpu.h> 28 #include <linux/bitops.h> 29 #include <linux/device.h> 30 #include <linux/nospec.h> 31 32 #include <asm/apic.h> 33 #include <asm/stacktrace.h> 34 #include <asm/nmi.h> 35 #include <asm/smp.h> 36 #include <asm/alternative.h> 37 #include <asm/mmu_context.h> 38 #include <asm/tlbflush.h> 39 #include <asm/timer.h> 40 #include <asm/desc.h> 41 #include <asm/ldt.h> 42 #include <asm/unwind.h> 43 44 #include "perf_event.h" 45 46 struct x86_pmu x86_pmu __read_mostly; 47 48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 49 .enabled = 1, 50 }; 51 52 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key); 53 54 u64 __read_mostly hw_cache_event_ids 55 [PERF_COUNT_HW_CACHE_MAX] 56 [PERF_COUNT_HW_CACHE_OP_MAX] 57 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 58 u64 __read_mostly hw_cache_extra_regs 59 [PERF_COUNT_HW_CACHE_MAX] 60 [PERF_COUNT_HW_CACHE_OP_MAX] 61 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 62 63 /* 64 * Propagate event elapsed time into the generic event. 65 * Can only be executed on the CPU where the event is active. 66 * Returns the delta events processed. 67 */ 68 u64 x86_perf_event_update(struct perf_event *event) 69 { 70 struct hw_perf_event *hwc = &event->hw; 71 int shift = 64 - x86_pmu.cntval_bits; 72 u64 prev_raw_count, new_raw_count; 73 int idx = hwc->idx; 74 u64 delta; 75 76 if (idx == INTEL_PMC_IDX_FIXED_BTS) 77 return 0; 78 79 /* 80 * Careful: an NMI might modify the previous event value. 81 * 82 * Our tactic to handle this is to first atomically read and 83 * exchange a new raw count - then add that new-prev delta 84 * count to the generic event atomically: 85 */ 86 again: 87 prev_raw_count = local64_read(&hwc->prev_count); 88 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 89 90 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 91 new_raw_count) != prev_raw_count) 92 goto again; 93 94 /* 95 * Now we have the new raw value and have updated the prev 96 * timestamp already. We can now calculate the elapsed delta 97 * (event-)time and add that to the generic event. 98 * 99 * Careful, not all hw sign-extends above the physical width 100 * of the count. 101 */ 102 delta = (new_raw_count << shift) - (prev_raw_count << shift); 103 delta >>= shift; 104 105 local64_add(delta, &event->count); 106 local64_sub(delta, &hwc->period_left); 107 108 return new_raw_count; 109 } 110 111 /* 112 * Find and validate any extra registers to set up. 113 */ 114 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 115 { 116 struct hw_perf_event_extra *reg; 117 struct extra_reg *er; 118 119 reg = &event->hw.extra_reg; 120 121 if (!x86_pmu.extra_regs) 122 return 0; 123 124 for (er = x86_pmu.extra_regs; er->msr; er++) { 125 if (er->event != (config & er->config_mask)) 126 continue; 127 if (event->attr.config1 & ~er->valid_mask) 128 return -EINVAL; 129 /* Check if the extra msrs can be safely accessed*/ 130 if (!er->extra_msr_access) 131 return -ENXIO; 132 133 reg->idx = er->idx; 134 reg->config = event->attr.config1; 135 reg->reg = er->msr; 136 break; 137 } 138 return 0; 139 } 140 141 static atomic_t active_events; 142 static atomic_t pmc_refcount; 143 static DEFINE_MUTEX(pmc_reserve_mutex); 144 145 #ifdef CONFIG_X86_LOCAL_APIC 146 147 static bool reserve_pmc_hardware(void) 148 { 149 int i; 150 151 for (i = 0; i < x86_pmu.num_counters; i++) { 152 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 153 goto perfctr_fail; 154 } 155 156 for (i = 0; i < x86_pmu.num_counters; i++) { 157 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 158 goto eventsel_fail; 159 } 160 161 return true; 162 163 eventsel_fail: 164 for (i--; i >= 0; i--) 165 release_evntsel_nmi(x86_pmu_config_addr(i)); 166 167 i = x86_pmu.num_counters; 168 169 perfctr_fail: 170 for (i--; i >= 0; i--) 171 release_perfctr_nmi(x86_pmu_event_addr(i)); 172 173 return false; 174 } 175 176 static void release_pmc_hardware(void) 177 { 178 int i; 179 180 for (i = 0; i < x86_pmu.num_counters; i++) { 181 release_perfctr_nmi(x86_pmu_event_addr(i)); 182 release_evntsel_nmi(x86_pmu_config_addr(i)); 183 } 184 } 185 186 #else 187 188 static bool reserve_pmc_hardware(void) { return true; } 189 static void release_pmc_hardware(void) {} 190 191 #endif 192 193 static bool check_hw_exists(void) 194 { 195 u64 val, val_fail = -1, val_new= ~0; 196 int i, reg, reg_fail = -1, ret = 0; 197 int bios_fail = 0; 198 int reg_safe = -1; 199 200 /* 201 * Check to see if the BIOS enabled any of the counters, if so 202 * complain and bail. 203 */ 204 for (i = 0; i < x86_pmu.num_counters; i++) { 205 reg = x86_pmu_config_addr(i); 206 ret = rdmsrl_safe(reg, &val); 207 if (ret) 208 goto msr_fail; 209 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 210 bios_fail = 1; 211 val_fail = val; 212 reg_fail = reg; 213 } else { 214 reg_safe = i; 215 } 216 } 217 218 if (x86_pmu.num_counters_fixed) { 219 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 220 ret = rdmsrl_safe(reg, &val); 221 if (ret) 222 goto msr_fail; 223 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 224 if (val & (0x03 << i*4)) { 225 bios_fail = 1; 226 val_fail = val; 227 reg_fail = reg; 228 } 229 } 230 } 231 232 /* 233 * If all the counters are enabled, the below test will always 234 * fail. The tools will also become useless in this scenario. 235 * Just fail and disable the hardware counters. 236 */ 237 238 if (reg_safe == -1) { 239 reg = reg_safe; 240 goto msr_fail; 241 } 242 243 /* 244 * Read the current value, change it and read it back to see if it 245 * matches, this is needed to detect certain hardware emulators 246 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 247 */ 248 reg = x86_pmu_event_addr(reg_safe); 249 if (rdmsrl_safe(reg, &val)) 250 goto msr_fail; 251 val ^= 0xffffUL; 252 ret = wrmsrl_safe(reg, val); 253 ret |= rdmsrl_safe(reg, &val_new); 254 if (ret || val != val_new) 255 goto msr_fail; 256 257 /* 258 * We still allow the PMU driver to operate: 259 */ 260 if (bios_fail) { 261 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 262 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 263 reg_fail, val_fail); 264 } 265 266 return true; 267 268 msr_fail: 269 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 270 pr_cont("PMU not available due to virtualization, using software events only.\n"); 271 } else { 272 pr_cont("Broken PMU hardware detected, using software events only.\n"); 273 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 274 reg, val_new); 275 } 276 277 return false; 278 } 279 280 static void hw_perf_event_destroy(struct perf_event *event) 281 { 282 x86_release_hardware(); 283 atomic_dec(&active_events); 284 } 285 286 void hw_perf_lbr_event_destroy(struct perf_event *event) 287 { 288 hw_perf_event_destroy(event); 289 290 /* undo the lbr/bts event accounting */ 291 x86_del_exclusive(x86_lbr_exclusive_lbr); 292 } 293 294 static inline int x86_pmu_initialized(void) 295 { 296 return x86_pmu.handle_irq != NULL; 297 } 298 299 static inline int 300 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 301 { 302 struct perf_event_attr *attr = &event->attr; 303 unsigned int cache_type, cache_op, cache_result; 304 u64 config, val; 305 306 config = attr->config; 307 308 cache_type = (config >> 0) & 0xff; 309 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 310 return -EINVAL; 311 cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX); 312 313 cache_op = (config >> 8) & 0xff; 314 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 315 return -EINVAL; 316 cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX); 317 318 cache_result = (config >> 16) & 0xff; 319 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 320 return -EINVAL; 321 cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX); 322 323 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 324 325 if (val == 0) 326 return -ENOENT; 327 328 if (val == -1) 329 return -EINVAL; 330 331 hwc->config |= val; 332 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 333 return x86_pmu_extra_regs(val, event); 334 } 335 336 int x86_reserve_hardware(void) 337 { 338 int err = 0; 339 340 if (!atomic_inc_not_zero(&pmc_refcount)) { 341 mutex_lock(&pmc_reserve_mutex); 342 if (atomic_read(&pmc_refcount) == 0) { 343 if (!reserve_pmc_hardware()) 344 err = -EBUSY; 345 else 346 reserve_ds_buffers(); 347 } 348 if (!err) 349 atomic_inc(&pmc_refcount); 350 mutex_unlock(&pmc_reserve_mutex); 351 } 352 353 return err; 354 } 355 356 void x86_release_hardware(void) 357 { 358 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 359 release_pmc_hardware(); 360 release_ds_buffers(); 361 mutex_unlock(&pmc_reserve_mutex); 362 } 363 } 364 365 /* 366 * Check if we can create event of a certain type (that no conflicting events 367 * are present). 368 */ 369 int x86_add_exclusive(unsigned int what) 370 { 371 int i; 372 373 /* 374 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 375 * LBR and BTS are still mutually exclusive. 376 */ 377 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 378 return 0; 379 380 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 381 mutex_lock(&pmc_reserve_mutex); 382 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 383 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 384 goto fail_unlock; 385 } 386 atomic_inc(&x86_pmu.lbr_exclusive[what]); 387 mutex_unlock(&pmc_reserve_mutex); 388 } 389 390 atomic_inc(&active_events); 391 return 0; 392 393 fail_unlock: 394 mutex_unlock(&pmc_reserve_mutex); 395 return -EBUSY; 396 } 397 398 void x86_del_exclusive(unsigned int what) 399 { 400 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 401 return; 402 403 atomic_dec(&x86_pmu.lbr_exclusive[what]); 404 atomic_dec(&active_events); 405 } 406 407 int x86_setup_perfctr(struct perf_event *event) 408 { 409 struct perf_event_attr *attr = &event->attr; 410 struct hw_perf_event *hwc = &event->hw; 411 u64 config; 412 413 if (!is_sampling_event(event)) { 414 hwc->sample_period = x86_pmu.max_period; 415 hwc->last_period = hwc->sample_period; 416 local64_set(&hwc->period_left, hwc->sample_period); 417 } 418 419 if (attr->type == PERF_TYPE_RAW) 420 return x86_pmu_extra_regs(event->attr.config, event); 421 422 if (attr->type == PERF_TYPE_HW_CACHE) 423 return set_ext_hw_attr(hwc, event); 424 425 if (attr->config >= x86_pmu.max_events) 426 return -EINVAL; 427 428 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); 429 430 /* 431 * The generic map: 432 */ 433 config = x86_pmu.event_map(attr->config); 434 435 if (config == 0) 436 return -ENOENT; 437 438 if (config == -1LL) 439 return -EINVAL; 440 441 /* 442 * Branch tracing: 443 */ 444 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS && 445 !attr->freq && hwc->sample_period == 1) { 446 /* BTS is not supported by this architecture. */ 447 if (!x86_pmu.bts_active) 448 return -EOPNOTSUPP; 449 450 /* BTS is currently only allowed for user-mode. */ 451 if (!attr->exclude_kernel) 452 return -EOPNOTSUPP; 453 454 /* disallow bts if conflicting events are present */ 455 if (x86_add_exclusive(x86_lbr_exclusive_lbr)) 456 return -EBUSY; 457 458 event->destroy = hw_perf_lbr_event_destroy; 459 } 460 461 hwc->config |= config; 462 463 return 0; 464 } 465 466 /* 467 * check that branch_sample_type is compatible with 468 * settings needed for precise_ip > 1 which implies 469 * using the LBR to capture ALL taken branches at the 470 * priv levels of the measurement 471 */ 472 static inline int precise_br_compat(struct perf_event *event) 473 { 474 u64 m = event->attr.branch_sample_type; 475 u64 b = 0; 476 477 /* must capture all branches */ 478 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 479 return 0; 480 481 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 482 483 if (!event->attr.exclude_user) 484 b |= PERF_SAMPLE_BRANCH_USER; 485 486 if (!event->attr.exclude_kernel) 487 b |= PERF_SAMPLE_BRANCH_KERNEL; 488 489 /* 490 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 491 */ 492 493 return m == b; 494 } 495 496 int x86_pmu_max_precise(void) 497 { 498 int precise = 0; 499 500 /* Support for constant skid */ 501 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 502 precise++; 503 504 /* Support for IP fixup */ 505 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 506 precise++; 507 508 if (x86_pmu.pebs_prec_dist) 509 precise++; 510 } 511 return precise; 512 } 513 514 int x86_pmu_hw_config(struct perf_event *event) 515 { 516 if (event->attr.precise_ip) { 517 int precise = x86_pmu_max_precise(); 518 519 if (event->attr.precise_ip > precise) 520 return -EOPNOTSUPP; 521 522 /* There's no sense in having PEBS for non sampling events: */ 523 if (!is_sampling_event(event)) 524 return -EINVAL; 525 } 526 /* 527 * check that PEBS LBR correction does not conflict with 528 * whatever the user is asking with attr->branch_sample_type 529 */ 530 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 531 u64 *br_type = &event->attr.branch_sample_type; 532 533 if (has_branch_stack(event)) { 534 if (!precise_br_compat(event)) 535 return -EOPNOTSUPP; 536 537 /* branch_sample_type is compatible */ 538 539 } else { 540 /* 541 * user did not specify branch_sample_type 542 * 543 * For PEBS fixups, we capture all 544 * the branches at the priv level of the 545 * event. 546 */ 547 *br_type = PERF_SAMPLE_BRANCH_ANY; 548 549 if (!event->attr.exclude_user) 550 *br_type |= PERF_SAMPLE_BRANCH_USER; 551 552 if (!event->attr.exclude_kernel) 553 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 554 } 555 } 556 557 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 558 event->attach_state |= PERF_ATTACH_TASK_DATA; 559 560 /* 561 * Generate PMC IRQs: 562 * (keep 'enabled' bit clear for now) 563 */ 564 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 565 566 /* 567 * Count user and OS events unless requested not to 568 */ 569 if (!event->attr.exclude_user) 570 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 571 if (!event->attr.exclude_kernel) 572 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 573 574 if (event->attr.type == PERF_TYPE_RAW) 575 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 576 577 if (event->attr.sample_period && x86_pmu.limit_period) { 578 if (x86_pmu.limit_period(event, event->attr.sample_period) > 579 event->attr.sample_period) 580 return -EINVAL; 581 } 582 583 return x86_setup_perfctr(event); 584 } 585 586 /* 587 * Setup the hardware configuration for a given attr_type 588 */ 589 static int __x86_pmu_event_init(struct perf_event *event) 590 { 591 int err; 592 593 if (!x86_pmu_initialized()) 594 return -ENODEV; 595 596 err = x86_reserve_hardware(); 597 if (err) 598 return err; 599 600 atomic_inc(&active_events); 601 event->destroy = hw_perf_event_destroy; 602 603 event->hw.idx = -1; 604 event->hw.last_cpu = -1; 605 event->hw.last_tag = ~0ULL; 606 607 /* mark unused */ 608 event->hw.extra_reg.idx = EXTRA_REG_NONE; 609 event->hw.branch_reg.idx = EXTRA_REG_NONE; 610 611 return x86_pmu.hw_config(event); 612 } 613 614 void x86_pmu_disable_all(void) 615 { 616 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 617 int idx; 618 619 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 620 u64 val; 621 622 if (!test_bit(idx, cpuc->active_mask)) 623 continue; 624 rdmsrl(x86_pmu_config_addr(idx), val); 625 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 626 continue; 627 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 628 wrmsrl(x86_pmu_config_addr(idx), val); 629 } 630 } 631 632 /* 633 * There may be PMI landing after enabled=0. The PMI hitting could be before or 634 * after disable_all. 635 * 636 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 637 * It will not be re-enabled in the NMI handler again, because enabled=0. After 638 * handling the NMI, disable_all will be called, which will not change the 639 * state either. If PMI hits after disable_all, the PMU is already disabled 640 * before entering NMI handler. The NMI handler will not change the state 641 * either. 642 * 643 * So either situation is harmless. 644 */ 645 static void x86_pmu_disable(struct pmu *pmu) 646 { 647 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 648 649 if (!x86_pmu_initialized()) 650 return; 651 652 if (!cpuc->enabled) 653 return; 654 655 cpuc->n_added = 0; 656 cpuc->enabled = 0; 657 barrier(); 658 659 x86_pmu.disable_all(); 660 } 661 662 void x86_pmu_enable_all(int added) 663 { 664 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 665 int idx; 666 667 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 668 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 669 670 if (!test_bit(idx, cpuc->active_mask)) 671 continue; 672 673 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 674 } 675 } 676 677 static struct pmu pmu; 678 679 static inline int is_x86_event(struct perf_event *event) 680 { 681 return event->pmu == &pmu; 682 } 683 684 /* 685 * Event scheduler state: 686 * 687 * Assign events iterating over all events and counters, beginning 688 * with events with least weights first. Keep the current iterator 689 * state in struct sched_state. 690 */ 691 struct sched_state { 692 int weight; 693 int event; /* event index */ 694 int counter; /* counter index */ 695 int unassigned; /* number of events to be assigned left */ 696 int nr_gp; /* number of GP counters used */ 697 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 698 }; 699 700 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 701 #define SCHED_STATES_MAX 2 702 703 struct perf_sched { 704 int max_weight; 705 int max_events; 706 int max_gp; 707 int saved_states; 708 struct event_constraint **constraints; 709 struct sched_state state; 710 struct sched_state saved[SCHED_STATES_MAX]; 711 }; 712 713 /* 714 * Initialize interator that runs through all events and counters. 715 */ 716 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 717 int num, int wmin, int wmax, int gpmax) 718 { 719 int idx; 720 721 memset(sched, 0, sizeof(*sched)); 722 sched->max_events = num; 723 sched->max_weight = wmax; 724 sched->max_gp = gpmax; 725 sched->constraints = constraints; 726 727 for (idx = 0; idx < num; idx++) { 728 if (constraints[idx]->weight == wmin) 729 break; 730 } 731 732 sched->state.event = idx; /* start with min weight */ 733 sched->state.weight = wmin; 734 sched->state.unassigned = num; 735 } 736 737 static void perf_sched_save_state(struct perf_sched *sched) 738 { 739 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 740 return; 741 742 sched->saved[sched->saved_states] = sched->state; 743 sched->saved_states++; 744 } 745 746 static bool perf_sched_restore_state(struct perf_sched *sched) 747 { 748 if (!sched->saved_states) 749 return false; 750 751 sched->saved_states--; 752 sched->state = sched->saved[sched->saved_states]; 753 754 /* continue with next counter: */ 755 clear_bit(sched->state.counter++, sched->state.used); 756 757 return true; 758 } 759 760 /* 761 * Select a counter for the current event to schedule. Return true on 762 * success. 763 */ 764 static bool __perf_sched_find_counter(struct perf_sched *sched) 765 { 766 struct event_constraint *c; 767 int idx; 768 769 if (!sched->state.unassigned) 770 return false; 771 772 if (sched->state.event >= sched->max_events) 773 return false; 774 775 c = sched->constraints[sched->state.event]; 776 /* Prefer fixed purpose counters */ 777 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 778 idx = INTEL_PMC_IDX_FIXED; 779 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 780 if (!__test_and_set_bit(idx, sched->state.used)) 781 goto done; 782 } 783 } 784 785 /* Grab the first unused counter starting with idx */ 786 idx = sched->state.counter; 787 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 788 if (!__test_and_set_bit(idx, sched->state.used)) { 789 if (sched->state.nr_gp++ >= sched->max_gp) 790 return false; 791 792 goto done; 793 } 794 } 795 796 return false; 797 798 done: 799 sched->state.counter = idx; 800 801 if (c->overlap) 802 perf_sched_save_state(sched); 803 804 return true; 805 } 806 807 static bool perf_sched_find_counter(struct perf_sched *sched) 808 { 809 while (!__perf_sched_find_counter(sched)) { 810 if (!perf_sched_restore_state(sched)) 811 return false; 812 } 813 814 return true; 815 } 816 817 /* 818 * Go through all unassigned events and find the next one to schedule. 819 * Take events with the least weight first. Return true on success. 820 */ 821 static bool perf_sched_next_event(struct perf_sched *sched) 822 { 823 struct event_constraint *c; 824 825 if (!sched->state.unassigned || !--sched->state.unassigned) 826 return false; 827 828 do { 829 /* next event */ 830 sched->state.event++; 831 if (sched->state.event >= sched->max_events) { 832 /* next weight */ 833 sched->state.event = 0; 834 sched->state.weight++; 835 if (sched->state.weight > sched->max_weight) 836 return false; 837 } 838 c = sched->constraints[sched->state.event]; 839 } while (c->weight != sched->state.weight); 840 841 sched->state.counter = 0; /* start with first counter */ 842 843 return true; 844 } 845 846 /* 847 * Assign a counter for each event. 848 */ 849 int perf_assign_events(struct event_constraint **constraints, int n, 850 int wmin, int wmax, int gpmax, int *assign) 851 { 852 struct perf_sched sched; 853 854 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 855 856 do { 857 if (!perf_sched_find_counter(&sched)) 858 break; /* failed */ 859 if (assign) 860 assign[sched.state.event] = sched.state.counter; 861 } while (perf_sched_next_event(&sched)); 862 863 return sched.state.unassigned; 864 } 865 EXPORT_SYMBOL_GPL(perf_assign_events); 866 867 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 868 { 869 struct event_constraint *c; 870 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 871 struct perf_event *e; 872 int i, wmin, wmax, unsched = 0; 873 struct hw_perf_event *hwc; 874 875 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 876 877 if (x86_pmu.start_scheduling) 878 x86_pmu.start_scheduling(cpuc); 879 880 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 881 cpuc->event_constraint[i] = NULL; 882 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 883 cpuc->event_constraint[i] = c; 884 885 wmin = min(wmin, c->weight); 886 wmax = max(wmax, c->weight); 887 } 888 889 /* 890 * fastpath, try to reuse previous register 891 */ 892 for (i = 0; i < n; i++) { 893 hwc = &cpuc->event_list[i]->hw; 894 c = cpuc->event_constraint[i]; 895 896 /* never assigned */ 897 if (hwc->idx == -1) 898 break; 899 900 /* constraint still honored */ 901 if (!test_bit(hwc->idx, c->idxmsk)) 902 break; 903 904 /* not already used */ 905 if (test_bit(hwc->idx, used_mask)) 906 break; 907 908 __set_bit(hwc->idx, used_mask); 909 if (assign) 910 assign[i] = hwc->idx; 911 } 912 913 /* slow path */ 914 if (i != n) { 915 int gpmax = x86_pmu.num_counters; 916 917 /* 918 * Do not allow scheduling of more than half the available 919 * generic counters. 920 * 921 * This helps avoid counter starvation of sibling thread by 922 * ensuring at most half the counters cannot be in exclusive 923 * mode. There is no designated counters for the limits. Any 924 * N/2 counters can be used. This helps with events with 925 * specific counter constraints. 926 */ 927 if (is_ht_workaround_enabled() && !cpuc->is_fake && 928 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 929 gpmax /= 2; 930 931 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 932 wmax, gpmax, assign); 933 } 934 935 /* 936 * In case of success (unsched = 0), mark events as committed, 937 * so we do not put_constraint() in case new events are added 938 * and fail to be scheduled 939 * 940 * We invoke the lower level commit callback to lock the resource 941 * 942 * We do not need to do all of this in case we are called to 943 * validate an event group (assign == NULL) 944 */ 945 if (!unsched && assign) { 946 for (i = 0; i < n; i++) { 947 e = cpuc->event_list[i]; 948 e->hw.flags |= PERF_X86_EVENT_COMMITTED; 949 if (x86_pmu.commit_scheduling) 950 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 951 } 952 } else { 953 for (i = 0; i < n; i++) { 954 e = cpuc->event_list[i]; 955 /* 956 * do not put_constraint() on comitted events, 957 * because they are good to go 958 */ 959 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED)) 960 continue; 961 962 /* 963 * release events that failed scheduling 964 */ 965 if (x86_pmu.put_event_constraints) 966 x86_pmu.put_event_constraints(cpuc, e); 967 } 968 } 969 970 if (x86_pmu.stop_scheduling) 971 x86_pmu.stop_scheduling(cpuc); 972 973 return unsched ? -EINVAL : 0; 974 } 975 976 /* 977 * dogrp: true if must collect siblings events (group) 978 * returns total number of events and error code 979 */ 980 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 981 { 982 struct perf_event *event; 983 int n, max_count; 984 985 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 986 987 /* current number of events already accepted */ 988 n = cpuc->n_events; 989 990 if (is_x86_event(leader)) { 991 if (n >= max_count) 992 return -EINVAL; 993 cpuc->event_list[n] = leader; 994 n++; 995 } 996 if (!dogrp) 997 return n; 998 999 for_each_sibling_event(event, leader) { 1000 if (!is_x86_event(event) || 1001 event->state <= PERF_EVENT_STATE_OFF) 1002 continue; 1003 1004 if (n >= max_count) 1005 return -EINVAL; 1006 1007 cpuc->event_list[n] = event; 1008 n++; 1009 } 1010 return n; 1011 } 1012 1013 static inline void x86_assign_hw_event(struct perf_event *event, 1014 struct cpu_hw_events *cpuc, int i) 1015 { 1016 struct hw_perf_event *hwc = &event->hw; 1017 1018 hwc->idx = cpuc->assign[i]; 1019 hwc->last_cpu = smp_processor_id(); 1020 hwc->last_tag = ++cpuc->tags[i]; 1021 1022 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1023 hwc->config_base = 0; 1024 hwc->event_base = 0; 1025 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1026 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1027 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1028 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1029 } else { 1030 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1031 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1032 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1033 } 1034 } 1035 1036 /** 1037 * x86_perf_rdpmc_index - Return PMC counter used for event 1038 * @event: the perf_event to which the PMC counter was assigned 1039 * 1040 * The counter assigned to this performance event may change if interrupts 1041 * are enabled. This counter should thus never be used while interrupts are 1042 * enabled. Before this function is used to obtain the assigned counter the 1043 * event should be checked for validity using, for example, 1044 * perf_event_read_local(), within the same interrupt disabled section in 1045 * which this counter is planned to be used. 1046 * 1047 * Return: The index of the performance monitoring counter assigned to 1048 * @perf_event. 1049 */ 1050 int x86_perf_rdpmc_index(struct perf_event *event) 1051 { 1052 lockdep_assert_irqs_disabled(); 1053 1054 return event->hw.event_base_rdpmc; 1055 } 1056 1057 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1058 struct cpu_hw_events *cpuc, 1059 int i) 1060 { 1061 return hwc->idx == cpuc->assign[i] && 1062 hwc->last_cpu == smp_processor_id() && 1063 hwc->last_tag == cpuc->tags[i]; 1064 } 1065 1066 static void x86_pmu_start(struct perf_event *event, int flags); 1067 1068 static void x86_pmu_enable(struct pmu *pmu) 1069 { 1070 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1071 struct perf_event *event; 1072 struct hw_perf_event *hwc; 1073 int i, added = cpuc->n_added; 1074 1075 if (!x86_pmu_initialized()) 1076 return; 1077 1078 if (cpuc->enabled) 1079 return; 1080 1081 if (cpuc->n_added) { 1082 int n_running = cpuc->n_events - cpuc->n_added; 1083 /* 1084 * apply assignment obtained either from 1085 * hw_perf_group_sched_in() or x86_pmu_enable() 1086 * 1087 * step1: save events moving to new counters 1088 */ 1089 for (i = 0; i < n_running; i++) { 1090 event = cpuc->event_list[i]; 1091 hwc = &event->hw; 1092 1093 /* 1094 * we can avoid reprogramming counter if: 1095 * - assigned same counter as last time 1096 * - running on same CPU as last time 1097 * - no other event has used the counter since 1098 */ 1099 if (hwc->idx == -1 || 1100 match_prev_assignment(hwc, cpuc, i)) 1101 continue; 1102 1103 /* 1104 * Ensure we don't accidentally enable a stopped 1105 * counter simply because we rescheduled. 1106 */ 1107 if (hwc->state & PERF_HES_STOPPED) 1108 hwc->state |= PERF_HES_ARCH; 1109 1110 x86_pmu_stop(event, PERF_EF_UPDATE); 1111 } 1112 1113 /* 1114 * step2: reprogram moved events into new counters 1115 */ 1116 for (i = 0; i < cpuc->n_events; i++) { 1117 event = cpuc->event_list[i]; 1118 hwc = &event->hw; 1119 1120 if (!match_prev_assignment(hwc, cpuc, i)) 1121 x86_assign_hw_event(event, cpuc, i); 1122 else if (i < n_running) 1123 continue; 1124 1125 if (hwc->state & PERF_HES_ARCH) 1126 continue; 1127 1128 x86_pmu_start(event, PERF_EF_RELOAD); 1129 } 1130 cpuc->n_added = 0; 1131 perf_events_lapic_init(); 1132 } 1133 1134 cpuc->enabled = 1; 1135 barrier(); 1136 1137 x86_pmu.enable_all(added); 1138 } 1139 1140 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1141 1142 /* 1143 * Set the next IRQ period, based on the hwc->period_left value. 1144 * To be called with the event disabled in hw: 1145 */ 1146 int x86_perf_event_set_period(struct perf_event *event) 1147 { 1148 struct hw_perf_event *hwc = &event->hw; 1149 s64 left = local64_read(&hwc->period_left); 1150 s64 period = hwc->sample_period; 1151 int ret = 0, idx = hwc->idx; 1152 1153 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1154 return 0; 1155 1156 /* 1157 * If we are way outside a reasonable range then just skip forward: 1158 */ 1159 if (unlikely(left <= -period)) { 1160 left = period; 1161 local64_set(&hwc->period_left, left); 1162 hwc->last_period = period; 1163 ret = 1; 1164 } 1165 1166 if (unlikely(left <= 0)) { 1167 left += period; 1168 local64_set(&hwc->period_left, left); 1169 hwc->last_period = period; 1170 ret = 1; 1171 } 1172 /* 1173 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1174 */ 1175 if (unlikely(left < 2)) 1176 left = 2; 1177 1178 if (left > x86_pmu.max_period) 1179 left = x86_pmu.max_period; 1180 1181 if (x86_pmu.limit_period) 1182 left = x86_pmu.limit_period(event, left); 1183 1184 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1185 1186 /* 1187 * The hw event starts counting from this event offset, 1188 * mark it to be able to extra future deltas: 1189 */ 1190 local64_set(&hwc->prev_count, (u64)-left); 1191 1192 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1193 1194 /* 1195 * Due to erratum on certan cpu we need 1196 * a second write to be sure the register 1197 * is updated properly 1198 */ 1199 if (x86_pmu.perfctr_second_write) { 1200 wrmsrl(hwc->event_base, 1201 (u64)(-left) & x86_pmu.cntval_mask); 1202 } 1203 1204 perf_event_update_userpage(event); 1205 1206 return ret; 1207 } 1208 1209 void x86_pmu_enable_event(struct perf_event *event) 1210 { 1211 if (__this_cpu_read(cpu_hw_events.enabled)) 1212 __x86_pmu_enable_event(&event->hw, 1213 ARCH_PERFMON_EVENTSEL_ENABLE); 1214 } 1215 1216 /* 1217 * Add a single event to the PMU. 1218 * 1219 * The event is added to the group of enabled events 1220 * but only if it can be scehduled with existing events. 1221 */ 1222 static int x86_pmu_add(struct perf_event *event, int flags) 1223 { 1224 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1225 struct hw_perf_event *hwc; 1226 int assign[X86_PMC_IDX_MAX]; 1227 int n, n0, ret; 1228 1229 hwc = &event->hw; 1230 1231 n0 = cpuc->n_events; 1232 ret = n = collect_events(cpuc, event, false); 1233 if (ret < 0) 1234 goto out; 1235 1236 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1237 if (!(flags & PERF_EF_START)) 1238 hwc->state |= PERF_HES_ARCH; 1239 1240 /* 1241 * If group events scheduling transaction was started, 1242 * skip the schedulability test here, it will be performed 1243 * at commit time (->commit_txn) as a whole. 1244 * 1245 * If commit fails, we'll call ->del() on all events 1246 * for which ->add() was called. 1247 */ 1248 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1249 goto done_collect; 1250 1251 ret = x86_pmu.schedule_events(cpuc, n, assign); 1252 if (ret) 1253 goto out; 1254 /* 1255 * copy new assignment, now we know it is possible 1256 * will be used by hw_perf_enable() 1257 */ 1258 memcpy(cpuc->assign, assign, n*sizeof(int)); 1259 1260 done_collect: 1261 /* 1262 * Commit the collect_events() state. See x86_pmu_del() and 1263 * x86_pmu_*_txn(). 1264 */ 1265 cpuc->n_events = n; 1266 cpuc->n_added += n - n0; 1267 cpuc->n_txn += n - n0; 1268 1269 if (x86_pmu.add) { 1270 /* 1271 * This is before x86_pmu_enable() will call x86_pmu_start(), 1272 * so we enable LBRs before an event needs them etc.. 1273 */ 1274 x86_pmu.add(event); 1275 } 1276 1277 ret = 0; 1278 out: 1279 return ret; 1280 } 1281 1282 static void x86_pmu_start(struct perf_event *event, int flags) 1283 { 1284 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1285 int idx = event->hw.idx; 1286 1287 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1288 return; 1289 1290 if (WARN_ON_ONCE(idx == -1)) 1291 return; 1292 1293 if (flags & PERF_EF_RELOAD) { 1294 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1295 x86_perf_event_set_period(event); 1296 } 1297 1298 event->hw.state = 0; 1299 1300 cpuc->events[idx] = event; 1301 __set_bit(idx, cpuc->active_mask); 1302 __set_bit(idx, cpuc->running); 1303 x86_pmu.enable(event); 1304 perf_event_update_userpage(event); 1305 } 1306 1307 void perf_event_print_debug(void) 1308 { 1309 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1310 u64 pebs, debugctl; 1311 struct cpu_hw_events *cpuc; 1312 unsigned long flags; 1313 int cpu, idx; 1314 1315 if (!x86_pmu.num_counters) 1316 return; 1317 1318 local_irq_save(flags); 1319 1320 cpu = smp_processor_id(); 1321 cpuc = &per_cpu(cpu_hw_events, cpu); 1322 1323 if (x86_pmu.version >= 2) { 1324 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1325 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1326 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1327 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1328 1329 pr_info("\n"); 1330 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1331 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1332 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1333 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1334 if (x86_pmu.pebs_constraints) { 1335 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1336 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1337 } 1338 if (x86_pmu.lbr_nr) { 1339 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1340 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1341 } 1342 } 1343 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1344 1345 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1346 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1347 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1348 1349 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1350 1351 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1352 cpu, idx, pmc_ctrl); 1353 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1354 cpu, idx, pmc_count); 1355 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1356 cpu, idx, prev_left); 1357 } 1358 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1359 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1360 1361 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1362 cpu, idx, pmc_count); 1363 } 1364 local_irq_restore(flags); 1365 } 1366 1367 void x86_pmu_stop(struct perf_event *event, int flags) 1368 { 1369 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1370 struct hw_perf_event *hwc = &event->hw; 1371 1372 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) { 1373 x86_pmu.disable(event); 1374 cpuc->events[hwc->idx] = NULL; 1375 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1376 hwc->state |= PERF_HES_STOPPED; 1377 } 1378 1379 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1380 /* 1381 * Drain the remaining delta count out of a event 1382 * that we are disabling: 1383 */ 1384 x86_perf_event_update(event); 1385 hwc->state |= PERF_HES_UPTODATE; 1386 } 1387 } 1388 1389 static void x86_pmu_del(struct perf_event *event, int flags) 1390 { 1391 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1392 int i; 1393 1394 /* 1395 * event is descheduled 1396 */ 1397 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; 1398 1399 /* 1400 * If we're called during a txn, we only need to undo x86_pmu.add. 1401 * The events never got scheduled and ->cancel_txn will truncate 1402 * the event_list. 1403 * 1404 * XXX assumes any ->del() called during a TXN will only be on 1405 * an event added during that same TXN. 1406 */ 1407 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1408 goto do_del; 1409 1410 /* 1411 * Not a TXN, therefore cleanup properly. 1412 */ 1413 x86_pmu_stop(event, PERF_EF_UPDATE); 1414 1415 for (i = 0; i < cpuc->n_events; i++) { 1416 if (event == cpuc->event_list[i]) 1417 break; 1418 } 1419 1420 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1421 return; 1422 1423 /* If we have a newly added event; make sure to decrease n_added. */ 1424 if (i >= cpuc->n_events - cpuc->n_added) 1425 --cpuc->n_added; 1426 1427 if (x86_pmu.put_event_constraints) 1428 x86_pmu.put_event_constraints(cpuc, event); 1429 1430 /* Delete the array entry. */ 1431 while (++i < cpuc->n_events) { 1432 cpuc->event_list[i-1] = cpuc->event_list[i]; 1433 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1434 } 1435 --cpuc->n_events; 1436 1437 perf_event_update_userpage(event); 1438 1439 do_del: 1440 if (x86_pmu.del) { 1441 /* 1442 * This is after x86_pmu_stop(); so we disable LBRs after any 1443 * event can need them etc.. 1444 */ 1445 x86_pmu.del(event); 1446 } 1447 } 1448 1449 int x86_pmu_handle_irq(struct pt_regs *regs) 1450 { 1451 struct perf_sample_data data; 1452 struct cpu_hw_events *cpuc; 1453 struct perf_event *event; 1454 int idx, handled = 0; 1455 u64 val; 1456 1457 cpuc = this_cpu_ptr(&cpu_hw_events); 1458 1459 /* 1460 * Some chipsets need to unmask the LVTPC in a particular spot 1461 * inside the nmi handler. As a result, the unmasking was pushed 1462 * into all the nmi handlers. 1463 * 1464 * This generic handler doesn't seem to have any issues where the 1465 * unmasking occurs so it was left at the top. 1466 */ 1467 apic_write(APIC_LVTPC, APIC_DM_NMI); 1468 1469 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1470 if (!test_bit(idx, cpuc->active_mask)) { 1471 /* 1472 * Though we deactivated the counter some cpus 1473 * might still deliver spurious interrupts still 1474 * in flight. Catch them: 1475 */ 1476 if (__test_and_clear_bit(idx, cpuc->running)) 1477 handled++; 1478 continue; 1479 } 1480 1481 event = cpuc->events[idx]; 1482 1483 val = x86_perf_event_update(event); 1484 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1485 continue; 1486 1487 /* 1488 * event overflow 1489 */ 1490 handled++; 1491 perf_sample_data_init(&data, 0, event->hw.last_period); 1492 1493 if (!x86_perf_event_set_period(event)) 1494 continue; 1495 1496 if (perf_event_overflow(event, &data, regs)) 1497 x86_pmu_stop(event, 0); 1498 } 1499 1500 if (handled) 1501 inc_irq_stat(apic_perf_irqs); 1502 1503 return handled; 1504 } 1505 1506 void perf_events_lapic_init(void) 1507 { 1508 if (!x86_pmu.apic || !x86_pmu_initialized()) 1509 return; 1510 1511 /* 1512 * Always use NMI for PMU 1513 */ 1514 apic_write(APIC_LVTPC, APIC_DM_NMI); 1515 } 1516 1517 static int 1518 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1519 { 1520 u64 start_clock; 1521 u64 finish_clock; 1522 int ret; 1523 1524 /* 1525 * All PMUs/events that share this PMI handler should make sure to 1526 * increment active_events for their events. 1527 */ 1528 if (!atomic_read(&active_events)) 1529 return NMI_DONE; 1530 1531 start_clock = sched_clock(); 1532 ret = x86_pmu.handle_irq(regs); 1533 finish_clock = sched_clock(); 1534 1535 perf_sample_event_took(finish_clock - start_clock); 1536 1537 return ret; 1538 } 1539 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1540 1541 struct event_constraint emptyconstraint; 1542 struct event_constraint unconstrained; 1543 1544 static int x86_pmu_prepare_cpu(unsigned int cpu) 1545 { 1546 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1547 int i; 1548 1549 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1550 cpuc->kfree_on_online[i] = NULL; 1551 if (x86_pmu.cpu_prepare) 1552 return x86_pmu.cpu_prepare(cpu); 1553 return 0; 1554 } 1555 1556 static int x86_pmu_dead_cpu(unsigned int cpu) 1557 { 1558 if (x86_pmu.cpu_dead) 1559 x86_pmu.cpu_dead(cpu); 1560 return 0; 1561 } 1562 1563 static int x86_pmu_online_cpu(unsigned int cpu) 1564 { 1565 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1566 int i; 1567 1568 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1569 kfree(cpuc->kfree_on_online[i]); 1570 cpuc->kfree_on_online[i] = NULL; 1571 } 1572 return 0; 1573 } 1574 1575 static int x86_pmu_starting_cpu(unsigned int cpu) 1576 { 1577 if (x86_pmu.cpu_starting) 1578 x86_pmu.cpu_starting(cpu); 1579 return 0; 1580 } 1581 1582 static int x86_pmu_dying_cpu(unsigned int cpu) 1583 { 1584 if (x86_pmu.cpu_dying) 1585 x86_pmu.cpu_dying(cpu); 1586 return 0; 1587 } 1588 1589 static void __init pmu_check_apic(void) 1590 { 1591 if (boot_cpu_has(X86_FEATURE_APIC)) 1592 return; 1593 1594 x86_pmu.apic = 0; 1595 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1596 pr_info("no hardware sampling interrupt available.\n"); 1597 1598 /* 1599 * If we have a PMU initialized but no APIC 1600 * interrupts, we cannot sample hardware 1601 * events (user-space has to fall back and 1602 * sample via a hrtimer based software event): 1603 */ 1604 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1605 1606 } 1607 1608 static struct attribute_group x86_pmu_format_group __ro_after_init = { 1609 .name = "format", 1610 .attrs = NULL, 1611 }; 1612 1613 /* 1614 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1615 * out of events_attr attributes. 1616 */ 1617 static void __init filter_events(struct attribute **attrs) 1618 { 1619 struct device_attribute *d; 1620 struct perf_pmu_events_attr *pmu_attr; 1621 int offset = 0; 1622 int i, j; 1623 1624 for (i = 0; attrs[i]; i++) { 1625 d = (struct device_attribute *)attrs[i]; 1626 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); 1627 /* str trumps id */ 1628 if (pmu_attr->event_str) 1629 continue; 1630 if (x86_pmu.event_map(i + offset)) 1631 continue; 1632 1633 for (j = i; attrs[j]; j++) 1634 attrs[j] = attrs[j + 1]; 1635 1636 /* Check the shifted attr. */ 1637 i--; 1638 1639 /* 1640 * event_map() is index based, the attrs array is organized 1641 * by increasing event index. If we shift the events, then 1642 * we need to compensate for the event_map(), otherwise 1643 * we are looking up the wrong event in the map 1644 */ 1645 offset++; 1646 } 1647 } 1648 1649 /* Merge two pointer arrays */ 1650 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) 1651 { 1652 struct attribute **new; 1653 int j, i; 1654 1655 for (j = 0; a && a[j]; j++) 1656 ; 1657 for (i = 0; b && b[i]; i++) 1658 j++; 1659 j++; 1660 1661 new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL); 1662 if (!new) 1663 return NULL; 1664 1665 j = 0; 1666 for (i = 0; a && a[i]; i++) 1667 new[j++] = a[i]; 1668 for (i = 0; b && b[i]; i++) 1669 new[j++] = b[i]; 1670 new[j] = NULL; 1671 1672 return new; 1673 } 1674 1675 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1676 { 1677 struct perf_pmu_events_attr *pmu_attr = \ 1678 container_of(attr, struct perf_pmu_events_attr, attr); 1679 u64 config = x86_pmu.event_map(pmu_attr->id); 1680 1681 /* string trumps id */ 1682 if (pmu_attr->event_str) 1683 return sprintf(page, "%s", pmu_attr->event_str); 1684 1685 return x86_pmu.events_sysfs_show(page, config); 1686 } 1687 EXPORT_SYMBOL_GPL(events_sysfs_show); 1688 1689 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1690 char *page) 1691 { 1692 struct perf_pmu_events_ht_attr *pmu_attr = 1693 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1694 1695 /* 1696 * Report conditional events depending on Hyper-Threading. 1697 * 1698 * This is overly conservative as usually the HT special 1699 * handling is not needed if the other CPU thread is idle. 1700 * 1701 * Note this does not (and cannot) handle the case when thread 1702 * siblings are invisible, for example with virtualization 1703 * if they are owned by some other guest. The user tool 1704 * has to re-read when a thread sibling gets onlined later. 1705 */ 1706 return sprintf(page, "%s", 1707 topology_max_smt_threads() > 1 ? 1708 pmu_attr->event_str_ht : 1709 pmu_attr->event_str_noht); 1710 } 1711 1712 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1713 EVENT_ATTR(instructions, INSTRUCTIONS ); 1714 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1715 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1716 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1717 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1718 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1719 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1720 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1721 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1722 1723 static struct attribute *empty_attrs; 1724 1725 static struct attribute *events_attr[] = { 1726 EVENT_PTR(CPU_CYCLES), 1727 EVENT_PTR(INSTRUCTIONS), 1728 EVENT_PTR(CACHE_REFERENCES), 1729 EVENT_PTR(CACHE_MISSES), 1730 EVENT_PTR(BRANCH_INSTRUCTIONS), 1731 EVENT_PTR(BRANCH_MISSES), 1732 EVENT_PTR(BUS_CYCLES), 1733 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1734 EVENT_PTR(STALLED_CYCLES_BACKEND), 1735 EVENT_PTR(REF_CPU_CYCLES), 1736 NULL, 1737 }; 1738 1739 static struct attribute_group x86_pmu_events_group __ro_after_init = { 1740 .name = "events", 1741 .attrs = events_attr, 1742 }; 1743 1744 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1745 { 1746 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1747 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1748 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1749 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1750 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1751 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1752 ssize_t ret; 1753 1754 /* 1755 * We have whole page size to spend and just little data 1756 * to write, so we can safely use sprintf. 1757 */ 1758 ret = sprintf(page, "event=0x%02llx", event); 1759 1760 if (umask) 1761 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1762 1763 if (edge) 1764 ret += sprintf(page + ret, ",edge"); 1765 1766 if (pc) 1767 ret += sprintf(page + ret, ",pc"); 1768 1769 if (any) 1770 ret += sprintf(page + ret, ",any"); 1771 1772 if (inv) 1773 ret += sprintf(page + ret, ",inv"); 1774 1775 if (cmask) 1776 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1777 1778 ret += sprintf(page + ret, "\n"); 1779 1780 return ret; 1781 } 1782 1783 static struct attribute_group x86_pmu_attr_group; 1784 static struct attribute_group x86_pmu_caps_group; 1785 1786 static int __init init_hw_perf_events(void) 1787 { 1788 struct x86_pmu_quirk *quirk; 1789 int err; 1790 1791 pr_info("Performance Events: "); 1792 1793 switch (boot_cpu_data.x86_vendor) { 1794 case X86_VENDOR_INTEL: 1795 err = intel_pmu_init(); 1796 break; 1797 case X86_VENDOR_AMD: 1798 err = amd_pmu_init(); 1799 break; 1800 case X86_VENDOR_HYGON: 1801 err = amd_pmu_init(); 1802 x86_pmu.name = "HYGON"; 1803 break; 1804 default: 1805 err = -ENOTSUPP; 1806 } 1807 if (err != 0) { 1808 pr_cont("no PMU driver, software events only.\n"); 1809 return 0; 1810 } 1811 1812 pmu_check_apic(); 1813 1814 /* sanity check that the hardware exists or is emulated */ 1815 if (!check_hw_exists()) 1816 return 0; 1817 1818 pr_cont("%s PMU driver.\n", x86_pmu.name); 1819 1820 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1821 1822 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1823 quirk->func(); 1824 1825 if (!x86_pmu.intel_ctrl) 1826 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1827 1828 perf_events_lapic_init(); 1829 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1830 1831 unconstrained = (struct event_constraint) 1832 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1833 0, x86_pmu.num_counters, 0, 0); 1834 1835 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1836 1837 if (x86_pmu.caps_attrs) { 1838 struct attribute **tmp; 1839 1840 tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs); 1841 if (!WARN_ON(!tmp)) 1842 x86_pmu_caps_group.attrs = tmp; 1843 } 1844 1845 if (x86_pmu.event_attrs) 1846 x86_pmu_events_group.attrs = x86_pmu.event_attrs; 1847 1848 if (!x86_pmu.events_sysfs_show) 1849 x86_pmu_events_group.attrs = &empty_attrs; 1850 else 1851 filter_events(x86_pmu_events_group.attrs); 1852 1853 if (x86_pmu.cpu_events) { 1854 struct attribute **tmp; 1855 1856 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); 1857 if (!WARN_ON(!tmp)) 1858 x86_pmu_events_group.attrs = tmp; 1859 } 1860 1861 if (x86_pmu.attrs) { 1862 struct attribute **tmp; 1863 1864 tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs); 1865 if (!WARN_ON(!tmp)) 1866 x86_pmu_attr_group.attrs = tmp; 1867 } 1868 1869 pr_info("... version: %d\n", x86_pmu.version); 1870 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1871 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1872 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1873 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1874 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1875 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1876 1877 /* 1878 * Install callbacks. Core will call them for each online 1879 * cpu. 1880 */ 1881 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1882 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1883 if (err) 1884 return err; 1885 1886 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1887 "perf/x86:starting", x86_pmu_starting_cpu, 1888 x86_pmu_dying_cpu); 1889 if (err) 1890 goto out; 1891 1892 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1893 x86_pmu_online_cpu, NULL); 1894 if (err) 1895 goto out1; 1896 1897 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1898 if (err) 1899 goto out2; 1900 1901 return 0; 1902 1903 out2: 1904 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1905 out1: 1906 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1907 out: 1908 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1909 return err; 1910 } 1911 early_initcall(init_hw_perf_events); 1912 1913 static inline void x86_pmu_read(struct perf_event *event) 1914 { 1915 if (x86_pmu.read) 1916 return x86_pmu.read(event); 1917 x86_perf_event_update(event); 1918 } 1919 1920 /* 1921 * Start group events scheduling transaction 1922 * Set the flag to make pmu::enable() not perform the 1923 * schedulability test, it will be performed at commit time 1924 * 1925 * We only support PERF_PMU_TXN_ADD transactions. Save the 1926 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1927 * transactions. 1928 */ 1929 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1930 { 1931 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1932 1933 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1934 1935 cpuc->txn_flags = txn_flags; 1936 if (txn_flags & ~PERF_PMU_TXN_ADD) 1937 return; 1938 1939 perf_pmu_disable(pmu); 1940 __this_cpu_write(cpu_hw_events.n_txn, 0); 1941 } 1942 1943 /* 1944 * Stop group events scheduling transaction 1945 * Clear the flag and pmu::enable() will perform the 1946 * schedulability test. 1947 */ 1948 static void x86_pmu_cancel_txn(struct pmu *pmu) 1949 { 1950 unsigned int txn_flags; 1951 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1952 1953 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1954 1955 txn_flags = cpuc->txn_flags; 1956 cpuc->txn_flags = 0; 1957 if (txn_flags & ~PERF_PMU_TXN_ADD) 1958 return; 1959 1960 /* 1961 * Truncate collected array by the number of events added in this 1962 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1963 */ 1964 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1965 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1966 perf_pmu_enable(pmu); 1967 } 1968 1969 /* 1970 * Commit group events scheduling transaction 1971 * Perform the group schedulability test as a whole 1972 * Return 0 if success 1973 * 1974 * Does not cancel the transaction on failure; expects the caller to do this. 1975 */ 1976 static int x86_pmu_commit_txn(struct pmu *pmu) 1977 { 1978 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1979 int assign[X86_PMC_IDX_MAX]; 1980 int n, ret; 1981 1982 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1983 1984 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1985 cpuc->txn_flags = 0; 1986 return 0; 1987 } 1988 1989 n = cpuc->n_events; 1990 1991 if (!x86_pmu_initialized()) 1992 return -EAGAIN; 1993 1994 ret = x86_pmu.schedule_events(cpuc, n, assign); 1995 if (ret) 1996 return ret; 1997 1998 /* 1999 * copy new assignment, now we know it is possible 2000 * will be used by hw_perf_enable() 2001 */ 2002 memcpy(cpuc->assign, assign, n*sizeof(int)); 2003 2004 cpuc->txn_flags = 0; 2005 perf_pmu_enable(pmu); 2006 return 0; 2007 } 2008 /* 2009 * a fake_cpuc is used to validate event groups. Due to 2010 * the extra reg logic, we need to also allocate a fake 2011 * per_core and per_cpu structure. Otherwise, group events 2012 * using extra reg may conflict without the kernel being 2013 * able to catch this when the last event gets added to 2014 * the group. 2015 */ 2016 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 2017 { 2018 kfree(cpuc->shared_regs); 2019 kfree(cpuc); 2020 } 2021 2022 static struct cpu_hw_events *allocate_fake_cpuc(void) 2023 { 2024 struct cpu_hw_events *cpuc; 2025 int cpu = raw_smp_processor_id(); 2026 2027 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 2028 if (!cpuc) 2029 return ERR_PTR(-ENOMEM); 2030 2031 /* only needed, if we have extra_regs */ 2032 if (x86_pmu.extra_regs) { 2033 cpuc->shared_regs = allocate_shared_regs(cpu); 2034 if (!cpuc->shared_regs) 2035 goto error; 2036 } 2037 cpuc->is_fake = 1; 2038 return cpuc; 2039 error: 2040 free_fake_cpuc(cpuc); 2041 return ERR_PTR(-ENOMEM); 2042 } 2043 2044 /* 2045 * validate that we can schedule this event 2046 */ 2047 static int validate_event(struct perf_event *event) 2048 { 2049 struct cpu_hw_events *fake_cpuc; 2050 struct event_constraint *c; 2051 int ret = 0; 2052 2053 fake_cpuc = allocate_fake_cpuc(); 2054 if (IS_ERR(fake_cpuc)) 2055 return PTR_ERR(fake_cpuc); 2056 2057 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event); 2058 2059 if (!c || !c->weight) 2060 ret = -EINVAL; 2061 2062 if (x86_pmu.put_event_constraints) 2063 x86_pmu.put_event_constraints(fake_cpuc, event); 2064 2065 free_fake_cpuc(fake_cpuc); 2066 2067 return ret; 2068 } 2069 2070 /* 2071 * validate a single event group 2072 * 2073 * validation include: 2074 * - check events are compatible which each other 2075 * - events do not compete for the same counter 2076 * - number of events <= number of counters 2077 * 2078 * validation ensures the group can be loaded onto the 2079 * PMU if it was the only group available. 2080 */ 2081 static int validate_group(struct perf_event *event) 2082 { 2083 struct perf_event *leader = event->group_leader; 2084 struct cpu_hw_events *fake_cpuc; 2085 int ret = -EINVAL, n; 2086 2087 fake_cpuc = allocate_fake_cpuc(); 2088 if (IS_ERR(fake_cpuc)) 2089 return PTR_ERR(fake_cpuc); 2090 /* 2091 * the event is not yet connected with its 2092 * siblings therefore we must first collect 2093 * existing siblings, then add the new event 2094 * before we can simulate the scheduling 2095 */ 2096 n = collect_events(fake_cpuc, leader, true); 2097 if (n < 0) 2098 goto out; 2099 2100 fake_cpuc->n_events = n; 2101 n = collect_events(fake_cpuc, event, false); 2102 if (n < 0) 2103 goto out; 2104 2105 fake_cpuc->n_events = n; 2106 2107 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2108 2109 out: 2110 free_fake_cpuc(fake_cpuc); 2111 return ret; 2112 } 2113 2114 static int x86_pmu_event_init(struct perf_event *event) 2115 { 2116 struct pmu *tmp; 2117 int err; 2118 2119 switch (event->attr.type) { 2120 case PERF_TYPE_RAW: 2121 case PERF_TYPE_HARDWARE: 2122 case PERF_TYPE_HW_CACHE: 2123 break; 2124 2125 default: 2126 return -ENOENT; 2127 } 2128 2129 err = __x86_pmu_event_init(event); 2130 if (!err) { 2131 /* 2132 * we temporarily connect event to its pmu 2133 * such that validate_group() can classify 2134 * it as an x86 event using is_x86_event() 2135 */ 2136 tmp = event->pmu; 2137 event->pmu = &pmu; 2138 2139 if (event->group_leader != event) 2140 err = validate_group(event); 2141 else 2142 err = validate_event(event); 2143 2144 event->pmu = tmp; 2145 } 2146 if (err) { 2147 if (event->destroy) 2148 event->destroy(event); 2149 } 2150 2151 if (READ_ONCE(x86_pmu.attr_rdpmc) && 2152 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) 2153 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2154 2155 return err; 2156 } 2157 2158 static void refresh_pce(void *ignored) 2159 { 2160 load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm)); 2161 } 2162 2163 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm) 2164 { 2165 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2166 return; 2167 2168 /* 2169 * This function relies on not being called concurrently in two 2170 * tasks in the same mm. Otherwise one task could observe 2171 * perf_rdpmc_allowed > 1 and return all the way back to 2172 * userspace with CR4.PCE clear while another task is still 2173 * doing on_each_cpu_mask() to propagate CR4.PCE. 2174 * 2175 * For now, this can't happen because all callers hold mmap_sem 2176 * for write. If this changes, we'll need a different solution. 2177 */ 2178 lockdep_assert_held_exclusive(&mm->mmap_sem); 2179 2180 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) 2181 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2182 } 2183 2184 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm) 2185 { 2186 2187 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2188 return; 2189 2190 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) 2191 on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1); 2192 } 2193 2194 static int x86_pmu_event_idx(struct perf_event *event) 2195 { 2196 int idx = event->hw.idx; 2197 2198 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2199 return 0; 2200 2201 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2202 idx -= INTEL_PMC_IDX_FIXED; 2203 idx |= 1 << 30; 2204 } 2205 2206 return idx + 1; 2207 } 2208 2209 static ssize_t get_attr_rdpmc(struct device *cdev, 2210 struct device_attribute *attr, 2211 char *buf) 2212 { 2213 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2214 } 2215 2216 static ssize_t set_attr_rdpmc(struct device *cdev, 2217 struct device_attribute *attr, 2218 const char *buf, size_t count) 2219 { 2220 unsigned long val; 2221 ssize_t ret; 2222 2223 ret = kstrtoul(buf, 0, &val); 2224 if (ret) 2225 return ret; 2226 2227 if (val > 2) 2228 return -EINVAL; 2229 2230 if (x86_pmu.attr_rdpmc_broken) 2231 return -ENOTSUPP; 2232 2233 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) { 2234 /* 2235 * Changing into or out of always available, aka 2236 * perf-event-bypassing mode. This path is extremely slow, 2237 * but only root can trigger it, so it's okay. 2238 */ 2239 if (val == 2) 2240 static_branch_inc(&rdpmc_always_available_key); 2241 else 2242 static_branch_dec(&rdpmc_always_available_key); 2243 on_each_cpu(refresh_pce, NULL, 1); 2244 } 2245 2246 x86_pmu.attr_rdpmc = val; 2247 2248 return count; 2249 } 2250 2251 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2252 2253 static struct attribute *x86_pmu_attrs[] = { 2254 &dev_attr_rdpmc.attr, 2255 NULL, 2256 }; 2257 2258 static struct attribute_group x86_pmu_attr_group __ro_after_init = { 2259 .attrs = x86_pmu_attrs, 2260 }; 2261 2262 static ssize_t max_precise_show(struct device *cdev, 2263 struct device_attribute *attr, 2264 char *buf) 2265 { 2266 return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise()); 2267 } 2268 2269 static DEVICE_ATTR_RO(max_precise); 2270 2271 static struct attribute *x86_pmu_caps_attrs[] = { 2272 &dev_attr_max_precise.attr, 2273 NULL 2274 }; 2275 2276 static struct attribute_group x86_pmu_caps_group __ro_after_init = { 2277 .name = "caps", 2278 .attrs = x86_pmu_caps_attrs, 2279 }; 2280 2281 static const struct attribute_group *x86_pmu_attr_groups[] = { 2282 &x86_pmu_attr_group, 2283 &x86_pmu_format_group, 2284 &x86_pmu_events_group, 2285 &x86_pmu_caps_group, 2286 NULL, 2287 }; 2288 2289 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2290 { 2291 if (x86_pmu.sched_task) 2292 x86_pmu.sched_task(ctx, sched_in); 2293 } 2294 2295 void perf_check_microcode(void) 2296 { 2297 if (x86_pmu.check_microcode) 2298 x86_pmu.check_microcode(); 2299 } 2300 2301 static struct pmu pmu = { 2302 .pmu_enable = x86_pmu_enable, 2303 .pmu_disable = x86_pmu_disable, 2304 2305 .attr_groups = x86_pmu_attr_groups, 2306 2307 .event_init = x86_pmu_event_init, 2308 2309 .event_mapped = x86_pmu_event_mapped, 2310 .event_unmapped = x86_pmu_event_unmapped, 2311 2312 .add = x86_pmu_add, 2313 .del = x86_pmu_del, 2314 .start = x86_pmu_start, 2315 .stop = x86_pmu_stop, 2316 .read = x86_pmu_read, 2317 2318 .start_txn = x86_pmu_start_txn, 2319 .cancel_txn = x86_pmu_cancel_txn, 2320 .commit_txn = x86_pmu_commit_txn, 2321 2322 .event_idx = x86_pmu_event_idx, 2323 .sched_task = x86_pmu_sched_task, 2324 .task_ctx_size = sizeof(struct x86_perf_task_context), 2325 }; 2326 2327 void arch_perf_update_userpage(struct perf_event *event, 2328 struct perf_event_mmap_page *userpg, u64 now) 2329 { 2330 struct cyc2ns_data data; 2331 u64 offset; 2332 2333 userpg->cap_user_time = 0; 2334 userpg->cap_user_time_zero = 0; 2335 userpg->cap_user_rdpmc = 2336 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2337 userpg->pmc_width = x86_pmu.cntval_bits; 2338 2339 if (!using_native_sched_clock() || !sched_clock_stable()) 2340 return; 2341 2342 cyc2ns_read_begin(&data); 2343 2344 offset = data.cyc2ns_offset + __sched_clock_offset; 2345 2346 /* 2347 * Internal timekeeping for enabled/running/stopped times 2348 * is always in the local_clock domain. 2349 */ 2350 userpg->cap_user_time = 1; 2351 userpg->time_mult = data.cyc2ns_mul; 2352 userpg->time_shift = data.cyc2ns_shift; 2353 userpg->time_offset = offset - now; 2354 2355 /* 2356 * cap_user_time_zero doesn't make sense when we're using a different 2357 * time base for the records. 2358 */ 2359 if (!event->attr.use_clockid) { 2360 userpg->cap_user_time_zero = 1; 2361 userpg->time_zero = offset; 2362 } 2363 2364 cyc2ns_read_end(); 2365 } 2366 2367 void 2368 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2369 { 2370 struct unwind_state state; 2371 unsigned long addr; 2372 2373 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2374 /* TODO: We don't support guest os callchain now */ 2375 return; 2376 } 2377 2378 if (perf_callchain_store(entry, regs->ip)) 2379 return; 2380 2381 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state); 2382 unwind_next_frame(&state)) { 2383 addr = unwind_get_return_address(&state); 2384 if (!addr || perf_callchain_store(entry, addr)) 2385 return; 2386 } 2387 } 2388 2389 static inline int 2390 valid_user_frame(const void __user *fp, unsigned long size) 2391 { 2392 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2393 } 2394 2395 static unsigned long get_segment_base(unsigned int segment) 2396 { 2397 struct desc_struct *desc; 2398 unsigned int idx = segment >> 3; 2399 2400 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2401 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2402 struct ldt_struct *ldt; 2403 2404 /* IRQs are off, so this synchronizes with smp_store_release */ 2405 ldt = READ_ONCE(current->active_mm->context.ldt); 2406 if (!ldt || idx >= ldt->nr_entries) 2407 return 0; 2408 2409 desc = &ldt->entries[idx]; 2410 #else 2411 return 0; 2412 #endif 2413 } else { 2414 if (idx >= GDT_ENTRIES) 2415 return 0; 2416 2417 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2418 } 2419 2420 return get_desc_base(desc); 2421 } 2422 2423 #ifdef CONFIG_IA32_EMULATION 2424 2425 #include <linux/compat.h> 2426 2427 static inline int 2428 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2429 { 2430 /* 32-bit process in 64-bit kernel. */ 2431 unsigned long ss_base, cs_base; 2432 struct stack_frame_ia32 frame; 2433 const void __user *fp; 2434 2435 if (!test_thread_flag(TIF_IA32)) 2436 return 0; 2437 2438 cs_base = get_segment_base(regs->cs); 2439 ss_base = get_segment_base(regs->ss); 2440 2441 fp = compat_ptr(ss_base + regs->bp); 2442 pagefault_disable(); 2443 while (entry->nr < entry->max_stack) { 2444 unsigned long bytes; 2445 frame.next_frame = 0; 2446 frame.return_address = 0; 2447 2448 if (!valid_user_frame(fp, sizeof(frame))) 2449 break; 2450 2451 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2452 if (bytes != 0) 2453 break; 2454 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2455 if (bytes != 0) 2456 break; 2457 2458 perf_callchain_store(entry, cs_base + frame.return_address); 2459 fp = compat_ptr(ss_base + frame.next_frame); 2460 } 2461 pagefault_enable(); 2462 return 1; 2463 } 2464 #else 2465 static inline int 2466 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2467 { 2468 return 0; 2469 } 2470 #endif 2471 2472 void 2473 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2474 { 2475 struct stack_frame frame; 2476 const unsigned long __user *fp; 2477 2478 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2479 /* TODO: We don't support guest os callchain now */ 2480 return; 2481 } 2482 2483 /* 2484 * We don't know what to do with VM86 stacks.. ignore them for now. 2485 */ 2486 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2487 return; 2488 2489 fp = (unsigned long __user *)regs->bp; 2490 2491 perf_callchain_store(entry, regs->ip); 2492 2493 if (!nmi_uaccess_okay()) 2494 return; 2495 2496 if (perf_callchain_user32(regs, entry)) 2497 return; 2498 2499 pagefault_disable(); 2500 while (entry->nr < entry->max_stack) { 2501 unsigned long bytes; 2502 2503 frame.next_frame = NULL; 2504 frame.return_address = 0; 2505 2506 if (!valid_user_frame(fp, sizeof(frame))) 2507 break; 2508 2509 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2510 if (bytes != 0) 2511 break; 2512 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2513 if (bytes != 0) 2514 break; 2515 2516 perf_callchain_store(entry, frame.return_address); 2517 fp = (void __user *)frame.next_frame; 2518 } 2519 pagefault_enable(); 2520 } 2521 2522 /* 2523 * Deal with code segment offsets for the various execution modes: 2524 * 2525 * VM86 - the good olde 16 bit days, where the linear address is 2526 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2527 * 2528 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2529 * to figure out what the 32bit base address is. 2530 * 2531 * X32 - has TIF_X32 set, but is running in x86_64 2532 * 2533 * X86_64 - CS,DS,SS,ES are all zero based. 2534 */ 2535 static unsigned long code_segment_base(struct pt_regs *regs) 2536 { 2537 /* 2538 * For IA32 we look at the GDT/LDT segment base to convert the 2539 * effective IP to a linear address. 2540 */ 2541 2542 #ifdef CONFIG_X86_32 2543 /* 2544 * If we are in VM86 mode, add the segment offset to convert to a 2545 * linear address. 2546 */ 2547 if (regs->flags & X86_VM_MASK) 2548 return 0x10 * regs->cs; 2549 2550 if (user_mode(regs) && regs->cs != __USER_CS) 2551 return get_segment_base(regs->cs); 2552 #else 2553 if (user_mode(regs) && !user_64bit_mode(regs) && 2554 regs->cs != __USER32_CS) 2555 return get_segment_base(regs->cs); 2556 #endif 2557 return 0; 2558 } 2559 2560 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2561 { 2562 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2563 return perf_guest_cbs->get_guest_ip(); 2564 2565 return regs->ip + code_segment_base(regs); 2566 } 2567 2568 unsigned long perf_misc_flags(struct pt_regs *regs) 2569 { 2570 int misc = 0; 2571 2572 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2573 if (perf_guest_cbs->is_user_mode()) 2574 misc |= PERF_RECORD_MISC_GUEST_USER; 2575 else 2576 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2577 } else { 2578 if (user_mode(regs)) 2579 misc |= PERF_RECORD_MISC_USER; 2580 else 2581 misc |= PERF_RECORD_MISC_KERNEL; 2582 } 2583 2584 if (regs->flags & PERF_EFLAGS_EXACT) 2585 misc |= PERF_RECORD_MISC_EXACT_IP; 2586 2587 return misc; 2588 } 2589 2590 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2591 { 2592 cap->version = x86_pmu.version; 2593 cap->num_counters_gp = x86_pmu.num_counters; 2594 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2595 cap->bit_width_gp = x86_pmu.cntval_bits; 2596 cap->bit_width_fixed = x86_pmu.cntval_bits; 2597 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2598 cap->events_mask_len = x86_pmu.events_mask_len; 2599 } 2600 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2601