1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched.h> 24 #include <linux/uaccess.h> 25 #include <linux/slab.h> 26 #include <linux/cpu.h> 27 #include <linux/bitops.h> 28 #include <linux/device.h> 29 30 #include <asm/apic.h> 31 #include <asm/stacktrace.h> 32 #include <asm/nmi.h> 33 #include <asm/smp.h> 34 #include <asm/alternative.h> 35 #include <asm/mmu_context.h> 36 #include <asm/tlbflush.h> 37 #include <asm/timer.h> 38 #include <asm/desc.h> 39 #include <asm/ldt.h> 40 #include <asm/unwind.h> 41 42 #include "perf_event.h" 43 44 struct x86_pmu x86_pmu __read_mostly; 45 46 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 47 .enabled = 1, 48 }; 49 50 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE; 51 52 u64 __read_mostly hw_cache_event_ids 53 [PERF_COUNT_HW_CACHE_MAX] 54 [PERF_COUNT_HW_CACHE_OP_MAX] 55 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 56 u64 __read_mostly hw_cache_extra_regs 57 [PERF_COUNT_HW_CACHE_MAX] 58 [PERF_COUNT_HW_CACHE_OP_MAX] 59 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 60 61 /* 62 * Propagate event elapsed time into the generic event. 63 * Can only be executed on the CPU where the event is active. 64 * Returns the delta events processed. 65 */ 66 u64 x86_perf_event_update(struct perf_event *event) 67 { 68 struct hw_perf_event *hwc = &event->hw; 69 int shift = 64 - x86_pmu.cntval_bits; 70 u64 prev_raw_count, new_raw_count; 71 int idx = hwc->idx; 72 u64 delta; 73 74 if (idx == INTEL_PMC_IDX_FIXED_BTS) 75 return 0; 76 77 /* 78 * Careful: an NMI might modify the previous event value. 79 * 80 * Our tactic to handle this is to first atomically read and 81 * exchange a new raw count - then add that new-prev delta 82 * count to the generic event atomically: 83 */ 84 again: 85 prev_raw_count = local64_read(&hwc->prev_count); 86 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 87 88 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 89 new_raw_count) != prev_raw_count) 90 goto again; 91 92 /* 93 * Now we have the new raw value and have updated the prev 94 * timestamp already. We can now calculate the elapsed delta 95 * (event-)time and add that to the generic event. 96 * 97 * Careful, not all hw sign-extends above the physical width 98 * of the count. 99 */ 100 delta = (new_raw_count << shift) - (prev_raw_count << shift); 101 delta >>= shift; 102 103 local64_add(delta, &event->count); 104 local64_sub(delta, &hwc->period_left); 105 106 return new_raw_count; 107 } 108 109 /* 110 * Find and validate any extra registers to set up. 111 */ 112 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 113 { 114 struct hw_perf_event_extra *reg; 115 struct extra_reg *er; 116 117 reg = &event->hw.extra_reg; 118 119 if (!x86_pmu.extra_regs) 120 return 0; 121 122 for (er = x86_pmu.extra_regs; er->msr; er++) { 123 if (er->event != (config & er->config_mask)) 124 continue; 125 if (event->attr.config1 & ~er->valid_mask) 126 return -EINVAL; 127 /* Check if the extra msrs can be safely accessed*/ 128 if (!er->extra_msr_access) 129 return -ENXIO; 130 131 reg->idx = er->idx; 132 reg->config = event->attr.config1; 133 reg->reg = er->msr; 134 break; 135 } 136 return 0; 137 } 138 139 static atomic_t active_events; 140 static atomic_t pmc_refcount; 141 static DEFINE_MUTEX(pmc_reserve_mutex); 142 143 #ifdef CONFIG_X86_LOCAL_APIC 144 145 static bool reserve_pmc_hardware(void) 146 { 147 int i; 148 149 for (i = 0; i < x86_pmu.num_counters; i++) { 150 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 151 goto perfctr_fail; 152 } 153 154 for (i = 0; i < x86_pmu.num_counters; i++) { 155 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 156 goto eventsel_fail; 157 } 158 159 return true; 160 161 eventsel_fail: 162 for (i--; i >= 0; i--) 163 release_evntsel_nmi(x86_pmu_config_addr(i)); 164 165 i = x86_pmu.num_counters; 166 167 perfctr_fail: 168 for (i--; i >= 0; i--) 169 release_perfctr_nmi(x86_pmu_event_addr(i)); 170 171 return false; 172 } 173 174 static void release_pmc_hardware(void) 175 { 176 int i; 177 178 for (i = 0; i < x86_pmu.num_counters; i++) { 179 release_perfctr_nmi(x86_pmu_event_addr(i)); 180 release_evntsel_nmi(x86_pmu_config_addr(i)); 181 } 182 } 183 184 #else 185 186 static bool reserve_pmc_hardware(void) { return true; } 187 static void release_pmc_hardware(void) {} 188 189 #endif 190 191 static bool check_hw_exists(void) 192 { 193 u64 val, val_fail, val_new= ~0; 194 int i, reg, reg_fail, ret = 0; 195 int bios_fail = 0; 196 int reg_safe = -1; 197 198 /* 199 * Check to see if the BIOS enabled any of the counters, if so 200 * complain and bail. 201 */ 202 for (i = 0; i < x86_pmu.num_counters; i++) { 203 reg = x86_pmu_config_addr(i); 204 ret = rdmsrl_safe(reg, &val); 205 if (ret) 206 goto msr_fail; 207 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 208 bios_fail = 1; 209 val_fail = val; 210 reg_fail = reg; 211 } else { 212 reg_safe = i; 213 } 214 } 215 216 if (x86_pmu.num_counters_fixed) { 217 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 218 ret = rdmsrl_safe(reg, &val); 219 if (ret) 220 goto msr_fail; 221 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 222 if (val & (0x03 << i*4)) { 223 bios_fail = 1; 224 val_fail = val; 225 reg_fail = reg; 226 } 227 } 228 } 229 230 /* 231 * If all the counters are enabled, the below test will always 232 * fail. The tools will also become useless in this scenario. 233 * Just fail and disable the hardware counters. 234 */ 235 236 if (reg_safe == -1) { 237 reg = reg_safe; 238 goto msr_fail; 239 } 240 241 /* 242 * Read the current value, change it and read it back to see if it 243 * matches, this is needed to detect certain hardware emulators 244 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 245 */ 246 reg = x86_pmu_event_addr(reg_safe); 247 if (rdmsrl_safe(reg, &val)) 248 goto msr_fail; 249 val ^= 0xffffUL; 250 ret = wrmsrl_safe(reg, val); 251 ret |= rdmsrl_safe(reg, &val_new); 252 if (ret || val != val_new) 253 goto msr_fail; 254 255 /* 256 * We still allow the PMU driver to operate: 257 */ 258 if (bios_fail) { 259 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 260 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 261 reg_fail, val_fail); 262 } 263 264 return true; 265 266 msr_fail: 267 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 268 pr_cont("PMU not available due to virtualization, using software events only.\n"); 269 } else { 270 pr_cont("Broken PMU hardware detected, using software events only.\n"); 271 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 272 reg, val_new); 273 } 274 275 return false; 276 } 277 278 static void hw_perf_event_destroy(struct perf_event *event) 279 { 280 x86_release_hardware(); 281 atomic_dec(&active_events); 282 } 283 284 void hw_perf_lbr_event_destroy(struct perf_event *event) 285 { 286 hw_perf_event_destroy(event); 287 288 /* undo the lbr/bts event accounting */ 289 x86_del_exclusive(x86_lbr_exclusive_lbr); 290 } 291 292 static inline int x86_pmu_initialized(void) 293 { 294 return x86_pmu.handle_irq != NULL; 295 } 296 297 static inline int 298 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 299 { 300 struct perf_event_attr *attr = &event->attr; 301 unsigned int cache_type, cache_op, cache_result; 302 u64 config, val; 303 304 config = attr->config; 305 306 cache_type = (config >> 0) & 0xff; 307 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 308 return -EINVAL; 309 310 cache_op = (config >> 8) & 0xff; 311 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 312 return -EINVAL; 313 314 cache_result = (config >> 16) & 0xff; 315 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 316 return -EINVAL; 317 318 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 319 320 if (val == 0) 321 return -ENOENT; 322 323 if (val == -1) 324 return -EINVAL; 325 326 hwc->config |= val; 327 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 328 return x86_pmu_extra_regs(val, event); 329 } 330 331 int x86_reserve_hardware(void) 332 { 333 int err = 0; 334 335 if (!atomic_inc_not_zero(&pmc_refcount)) { 336 mutex_lock(&pmc_reserve_mutex); 337 if (atomic_read(&pmc_refcount) == 0) { 338 if (!reserve_pmc_hardware()) 339 err = -EBUSY; 340 else 341 reserve_ds_buffers(); 342 } 343 if (!err) 344 atomic_inc(&pmc_refcount); 345 mutex_unlock(&pmc_reserve_mutex); 346 } 347 348 return err; 349 } 350 351 void x86_release_hardware(void) 352 { 353 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 354 release_pmc_hardware(); 355 release_ds_buffers(); 356 mutex_unlock(&pmc_reserve_mutex); 357 } 358 } 359 360 /* 361 * Check if we can create event of a certain type (that no conflicting events 362 * are present). 363 */ 364 int x86_add_exclusive(unsigned int what) 365 { 366 int i; 367 368 /* 369 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS. 370 * LBR and BTS are still mutually exclusive. 371 */ 372 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 373 return 0; 374 375 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 376 mutex_lock(&pmc_reserve_mutex); 377 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 378 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 379 goto fail_unlock; 380 } 381 atomic_inc(&x86_pmu.lbr_exclusive[what]); 382 mutex_unlock(&pmc_reserve_mutex); 383 } 384 385 atomic_inc(&active_events); 386 return 0; 387 388 fail_unlock: 389 mutex_unlock(&pmc_reserve_mutex); 390 return -EBUSY; 391 } 392 393 void x86_del_exclusive(unsigned int what) 394 { 395 if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt) 396 return; 397 398 atomic_dec(&x86_pmu.lbr_exclusive[what]); 399 atomic_dec(&active_events); 400 } 401 402 int x86_setup_perfctr(struct perf_event *event) 403 { 404 struct perf_event_attr *attr = &event->attr; 405 struct hw_perf_event *hwc = &event->hw; 406 u64 config; 407 408 if (!is_sampling_event(event)) { 409 hwc->sample_period = x86_pmu.max_period; 410 hwc->last_period = hwc->sample_period; 411 local64_set(&hwc->period_left, hwc->sample_period); 412 } 413 414 if (attr->type == PERF_TYPE_RAW) 415 return x86_pmu_extra_regs(event->attr.config, event); 416 417 if (attr->type == PERF_TYPE_HW_CACHE) 418 return set_ext_hw_attr(hwc, event); 419 420 if (attr->config >= x86_pmu.max_events) 421 return -EINVAL; 422 423 /* 424 * The generic map: 425 */ 426 config = x86_pmu.event_map(attr->config); 427 428 if (config == 0) 429 return -ENOENT; 430 431 if (config == -1LL) 432 return -EINVAL; 433 434 /* 435 * Branch tracing: 436 */ 437 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS && 438 !attr->freq && hwc->sample_period == 1) { 439 /* BTS is not supported by this architecture. */ 440 if (!x86_pmu.bts_active) 441 return -EOPNOTSUPP; 442 443 /* BTS is currently only allowed for user-mode. */ 444 if (!attr->exclude_kernel) 445 return -EOPNOTSUPP; 446 447 /* disallow bts if conflicting events are present */ 448 if (x86_add_exclusive(x86_lbr_exclusive_lbr)) 449 return -EBUSY; 450 451 event->destroy = hw_perf_lbr_event_destroy; 452 } 453 454 hwc->config |= config; 455 456 return 0; 457 } 458 459 /* 460 * check that branch_sample_type is compatible with 461 * settings needed for precise_ip > 1 which implies 462 * using the LBR to capture ALL taken branches at the 463 * priv levels of the measurement 464 */ 465 static inline int precise_br_compat(struct perf_event *event) 466 { 467 u64 m = event->attr.branch_sample_type; 468 u64 b = 0; 469 470 /* must capture all branches */ 471 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 472 return 0; 473 474 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 475 476 if (!event->attr.exclude_user) 477 b |= PERF_SAMPLE_BRANCH_USER; 478 479 if (!event->attr.exclude_kernel) 480 b |= PERF_SAMPLE_BRANCH_KERNEL; 481 482 /* 483 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 484 */ 485 486 return m == b; 487 } 488 489 int x86_pmu_hw_config(struct perf_event *event) 490 { 491 if (event->attr.precise_ip) { 492 int precise = 0; 493 494 /* Support for constant skid */ 495 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 496 precise++; 497 498 /* Support for IP fixup */ 499 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 500 precise++; 501 502 if (x86_pmu.pebs_prec_dist) 503 precise++; 504 } 505 506 if (event->attr.precise_ip > precise) 507 return -EOPNOTSUPP; 508 } 509 /* 510 * check that PEBS LBR correction does not conflict with 511 * whatever the user is asking with attr->branch_sample_type 512 */ 513 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 514 u64 *br_type = &event->attr.branch_sample_type; 515 516 if (has_branch_stack(event)) { 517 if (!precise_br_compat(event)) 518 return -EOPNOTSUPP; 519 520 /* branch_sample_type is compatible */ 521 522 } else { 523 /* 524 * user did not specify branch_sample_type 525 * 526 * For PEBS fixups, we capture all 527 * the branches at the priv level of the 528 * event. 529 */ 530 *br_type = PERF_SAMPLE_BRANCH_ANY; 531 532 if (!event->attr.exclude_user) 533 *br_type |= PERF_SAMPLE_BRANCH_USER; 534 535 if (!event->attr.exclude_kernel) 536 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 537 } 538 } 539 540 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 541 event->attach_state |= PERF_ATTACH_TASK_DATA; 542 543 /* 544 * Generate PMC IRQs: 545 * (keep 'enabled' bit clear for now) 546 */ 547 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 548 549 /* 550 * Count user and OS events unless requested not to 551 */ 552 if (!event->attr.exclude_user) 553 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 554 if (!event->attr.exclude_kernel) 555 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 556 557 if (event->attr.type == PERF_TYPE_RAW) 558 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 559 560 if (event->attr.sample_period && x86_pmu.limit_period) { 561 if (x86_pmu.limit_period(event, event->attr.sample_period) > 562 event->attr.sample_period) 563 return -EINVAL; 564 } 565 566 return x86_setup_perfctr(event); 567 } 568 569 /* 570 * Setup the hardware configuration for a given attr_type 571 */ 572 static int __x86_pmu_event_init(struct perf_event *event) 573 { 574 int err; 575 576 if (!x86_pmu_initialized()) 577 return -ENODEV; 578 579 err = x86_reserve_hardware(); 580 if (err) 581 return err; 582 583 atomic_inc(&active_events); 584 event->destroy = hw_perf_event_destroy; 585 586 event->hw.idx = -1; 587 event->hw.last_cpu = -1; 588 event->hw.last_tag = ~0ULL; 589 590 /* mark unused */ 591 event->hw.extra_reg.idx = EXTRA_REG_NONE; 592 event->hw.branch_reg.idx = EXTRA_REG_NONE; 593 594 return x86_pmu.hw_config(event); 595 } 596 597 void x86_pmu_disable_all(void) 598 { 599 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 600 int idx; 601 602 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 603 u64 val; 604 605 if (!test_bit(idx, cpuc->active_mask)) 606 continue; 607 rdmsrl(x86_pmu_config_addr(idx), val); 608 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 609 continue; 610 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 611 wrmsrl(x86_pmu_config_addr(idx), val); 612 } 613 } 614 615 /* 616 * There may be PMI landing after enabled=0. The PMI hitting could be before or 617 * after disable_all. 618 * 619 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 620 * It will not be re-enabled in the NMI handler again, because enabled=0. After 621 * handling the NMI, disable_all will be called, which will not change the 622 * state either. If PMI hits after disable_all, the PMU is already disabled 623 * before entering NMI handler. The NMI handler will not change the state 624 * either. 625 * 626 * So either situation is harmless. 627 */ 628 static void x86_pmu_disable(struct pmu *pmu) 629 { 630 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 631 632 if (!x86_pmu_initialized()) 633 return; 634 635 if (!cpuc->enabled) 636 return; 637 638 cpuc->n_added = 0; 639 cpuc->enabled = 0; 640 barrier(); 641 642 x86_pmu.disable_all(); 643 } 644 645 void x86_pmu_enable_all(int added) 646 { 647 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 648 int idx; 649 650 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 651 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 652 653 if (!test_bit(idx, cpuc->active_mask)) 654 continue; 655 656 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 657 } 658 } 659 660 static struct pmu pmu; 661 662 static inline int is_x86_event(struct perf_event *event) 663 { 664 return event->pmu == &pmu; 665 } 666 667 /* 668 * Event scheduler state: 669 * 670 * Assign events iterating over all events and counters, beginning 671 * with events with least weights first. Keep the current iterator 672 * state in struct sched_state. 673 */ 674 struct sched_state { 675 int weight; 676 int event; /* event index */ 677 int counter; /* counter index */ 678 int unassigned; /* number of events to be assigned left */ 679 int nr_gp; /* number of GP counters used */ 680 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 681 }; 682 683 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 684 #define SCHED_STATES_MAX 2 685 686 struct perf_sched { 687 int max_weight; 688 int max_events; 689 int max_gp; 690 int saved_states; 691 struct event_constraint **constraints; 692 struct sched_state state; 693 struct sched_state saved[SCHED_STATES_MAX]; 694 }; 695 696 /* 697 * Initialize interator that runs through all events and counters. 698 */ 699 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 700 int num, int wmin, int wmax, int gpmax) 701 { 702 int idx; 703 704 memset(sched, 0, sizeof(*sched)); 705 sched->max_events = num; 706 sched->max_weight = wmax; 707 sched->max_gp = gpmax; 708 sched->constraints = constraints; 709 710 for (idx = 0; idx < num; idx++) { 711 if (constraints[idx]->weight == wmin) 712 break; 713 } 714 715 sched->state.event = idx; /* start with min weight */ 716 sched->state.weight = wmin; 717 sched->state.unassigned = num; 718 } 719 720 static void perf_sched_save_state(struct perf_sched *sched) 721 { 722 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 723 return; 724 725 sched->saved[sched->saved_states] = sched->state; 726 sched->saved_states++; 727 } 728 729 static bool perf_sched_restore_state(struct perf_sched *sched) 730 { 731 if (!sched->saved_states) 732 return false; 733 734 sched->saved_states--; 735 sched->state = sched->saved[sched->saved_states]; 736 737 /* continue with next counter: */ 738 clear_bit(sched->state.counter++, sched->state.used); 739 740 return true; 741 } 742 743 /* 744 * Select a counter for the current event to schedule. Return true on 745 * success. 746 */ 747 static bool __perf_sched_find_counter(struct perf_sched *sched) 748 { 749 struct event_constraint *c; 750 int idx; 751 752 if (!sched->state.unassigned) 753 return false; 754 755 if (sched->state.event >= sched->max_events) 756 return false; 757 758 c = sched->constraints[sched->state.event]; 759 /* Prefer fixed purpose counters */ 760 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 761 idx = INTEL_PMC_IDX_FIXED; 762 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 763 if (!__test_and_set_bit(idx, sched->state.used)) 764 goto done; 765 } 766 } 767 768 /* Grab the first unused counter starting with idx */ 769 idx = sched->state.counter; 770 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 771 if (!__test_and_set_bit(idx, sched->state.used)) { 772 if (sched->state.nr_gp++ >= sched->max_gp) 773 return false; 774 775 goto done; 776 } 777 } 778 779 return false; 780 781 done: 782 sched->state.counter = idx; 783 784 if (c->overlap) 785 perf_sched_save_state(sched); 786 787 return true; 788 } 789 790 static bool perf_sched_find_counter(struct perf_sched *sched) 791 { 792 while (!__perf_sched_find_counter(sched)) { 793 if (!perf_sched_restore_state(sched)) 794 return false; 795 } 796 797 return true; 798 } 799 800 /* 801 * Go through all unassigned events and find the next one to schedule. 802 * Take events with the least weight first. Return true on success. 803 */ 804 static bool perf_sched_next_event(struct perf_sched *sched) 805 { 806 struct event_constraint *c; 807 808 if (!sched->state.unassigned || !--sched->state.unassigned) 809 return false; 810 811 do { 812 /* next event */ 813 sched->state.event++; 814 if (sched->state.event >= sched->max_events) { 815 /* next weight */ 816 sched->state.event = 0; 817 sched->state.weight++; 818 if (sched->state.weight > sched->max_weight) 819 return false; 820 } 821 c = sched->constraints[sched->state.event]; 822 } while (c->weight != sched->state.weight); 823 824 sched->state.counter = 0; /* start with first counter */ 825 826 return true; 827 } 828 829 /* 830 * Assign a counter for each event. 831 */ 832 int perf_assign_events(struct event_constraint **constraints, int n, 833 int wmin, int wmax, int gpmax, int *assign) 834 { 835 struct perf_sched sched; 836 837 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 838 839 do { 840 if (!perf_sched_find_counter(&sched)) 841 break; /* failed */ 842 if (assign) 843 assign[sched.state.event] = sched.state.counter; 844 } while (perf_sched_next_event(&sched)); 845 846 return sched.state.unassigned; 847 } 848 EXPORT_SYMBOL_GPL(perf_assign_events); 849 850 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 851 { 852 struct event_constraint *c; 853 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 854 struct perf_event *e; 855 int i, wmin, wmax, unsched = 0; 856 struct hw_perf_event *hwc; 857 858 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 859 860 if (x86_pmu.start_scheduling) 861 x86_pmu.start_scheduling(cpuc); 862 863 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 864 cpuc->event_constraint[i] = NULL; 865 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 866 cpuc->event_constraint[i] = c; 867 868 wmin = min(wmin, c->weight); 869 wmax = max(wmax, c->weight); 870 } 871 872 /* 873 * fastpath, try to reuse previous register 874 */ 875 for (i = 0; i < n; i++) { 876 hwc = &cpuc->event_list[i]->hw; 877 c = cpuc->event_constraint[i]; 878 879 /* never assigned */ 880 if (hwc->idx == -1) 881 break; 882 883 /* constraint still honored */ 884 if (!test_bit(hwc->idx, c->idxmsk)) 885 break; 886 887 /* not already used */ 888 if (test_bit(hwc->idx, used_mask)) 889 break; 890 891 __set_bit(hwc->idx, used_mask); 892 if (assign) 893 assign[i] = hwc->idx; 894 } 895 896 /* slow path */ 897 if (i != n) { 898 int gpmax = x86_pmu.num_counters; 899 900 /* 901 * Do not allow scheduling of more than half the available 902 * generic counters. 903 * 904 * This helps avoid counter starvation of sibling thread by 905 * ensuring at most half the counters cannot be in exclusive 906 * mode. There is no designated counters for the limits. Any 907 * N/2 counters can be used. This helps with events with 908 * specific counter constraints. 909 */ 910 if (is_ht_workaround_enabled() && !cpuc->is_fake && 911 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 912 gpmax /= 2; 913 914 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 915 wmax, gpmax, assign); 916 } 917 918 /* 919 * In case of success (unsched = 0), mark events as committed, 920 * so we do not put_constraint() in case new events are added 921 * and fail to be scheduled 922 * 923 * We invoke the lower level commit callback to lock the resource 924 * 925 * We do not need to do all of this in case we are called to 926 * validate an event group (assign == NULL) 927 */ 928 if (!unsched && assign) { 929 for (i = 0; i < n; i++) { 930 e = cpuc->event_list[i]; 931 e->hw.flags |= PERF_X86_EVENT_COMMITTED; 932 if (x86_pmu.commit_scheduling) 933 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 934 } 935 } else { 936 for (i = 0; i < n; i++) { 937 e = cpuc->event_list[i]; 938 /* 939 * do not put_constraint() on comitted events, 940 * because they are good to go 941 */ 942 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED)) 943 continue; 944 945 /* 946 * release events that failed scheduling 947 */ 948 if (x86_pmu.put_event_constraints) 949 x86_pmu.put_event_constraints(cpuc, e); 950 } 951 } 952 953 if (x86_pmu.stop_scheduling) 954 x86_pmu.stop_scheduling(cpuc); 955 956 return unsched ? -EINVAL : 0; 957 } 958 959 /* 960 * dogrp: true if must collect siblings events (group) 961 * returns total number of events and error code 962 */ 963 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 964 { 965 struct perf_event *event; 966 int n, max_count; 967 968 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 969 970 /* current number of events already accepted */ 971 n = cpuc->n_events; 972 973 if (is_x86_event(leader)) { 974 if (n >= max_count) 975 return -EINVAL; 976 cpuc->event_list[n] = leader; 977 n++; 978 } 979 if (!dogrp) 980 return n; 981 982 list_for_each_entry(event, &leader->sibling_list, group_entry) { 983 if (!is_x86_event(event) || 984 event->state <= PERF_EVENT_STATE_OFF) 985 continue; 986 987 if (n >= max_count) 988 return -EINVAL; 989 990 cpuc->event_list[n] = event; 991 n++; 992 } 993 return n; 994 } 995 996 static inline void x86_assign_hw_event(struct perf_event *event, 997 struct cpu_hw_events *cpuc, int i) 998 { 999 struct hw_perf_event *hwc = &event->hw; 1000 1001 hwc->idx = cpuc->assign[i]; 1002 hwc->last_cpu = smp_processor_id(); 1003 hwc->last_tag = ++cpuc->tags[i]; 1004 1005 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1006 hwc->config_base = 0; 1007 hwc->event_base = 0; 1008 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1009 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1010 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1011 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1012 } else { 1013 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1014 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1015 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1016 } 1017 } 1018 1019 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1020 struct cpu_hw_events *cpuc, 1021 int i) 1022 { 1023 return hwc->idx == cpuc->assign[i] && 1024 hwc->last_cpu == smp_processor_id() && 1025 hwc->last_tag == cpuc->tags[i]; 1026 } 1027 1028 static void x86_pmu_start(struct perf_event *event, int flags); 1029 1030 static void x86_pmu_enable(struct pmu *pmu) 1031 { 1032 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1033 struct perf_event *event; 1034 struct hw_perf_event *hwc; 1035 int i, added = cpuc->n_added; 1036 1037 if (!x86_pmu_initialized()) 1038 return; 1039 1040 if (cpuc->enabled) 1041 return; 1042 1043 if (cpuc->n_added) { 1044 int n_running = cpuc->n_events - cpuc->n_added; 1045 /* 1046 * apply assignment obtained either from 1047 * hw_perf_group_sched_in() or x86_pmu_enable() 1048 * 1049 * step1: save events moving to new counters 1050 */ 1051 for (i = 0; i < n_running; i++) { 1052 event = cpuc->event_list[i]; 1053 hwc = &event->hw; 1054 1055 /* 1056 * we can avoid reprogramming counter if: 1057 * - assigned same counter as last time 1058 * - running on same CPU as last time 1059 * - no other event has used the counter since 1060 */ 1061 if (hwc->idx == -1 || 1062 match_prev_assignment(hwc, cpuc, i)) 1063 continue; 1064 1065 /* 1066 * Ensure we don't accidentally enable a stopped 1067 * counter simply because we rescheduled. 1068 */ 1069 if (hwc->state & PERF_HES_STOPPED) 1070 hwc->state |= PERF_HES_ARCH; 1071 1072 x86_pmu_stop(event, PERF_EF_UPDATE); 1073 } 1074 1075 /* 1076 * step2: reprogram moved events into new counters 1077 */ 1078 for (i = 0; i < cpuc->n_events; i++) { 1079 event = cpuc->event_list[i]; 1080 hwc = &event->hw; 1081 1082 if (!match_prev_assignment(hwc, cpuc, i)) 1083 x86_assign_hw_event(event, cpuc, i); 1084 else if (i < n_running) 1085 continue; 1086 1087 if (hwc->state & PERF_HES_ARCH) 1088 continue; 1089 1090 x86_pmu_start(event, PERF_EF_RELOAD); 1091 } 1092 cpuc->n_added = 0; 1093 perf_events_lapic_init(); 1094 } 1095 1096 cpuc->enabled = 1; 1097 barrier(); 1098 1099 x86_pmu.enable_all(added); 1100 } 1101 1102 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1103 1104 /* 1105 * Set the next IRQ period, based on the hwc->period_left value. 1106 * To be called with the event disabled in hw: 1107 */ 1108 int x86_perf_event_set_period(struct perf_event *event) 1109 { 1110 struct hw_perf_event *hwc = &event->hw; 1111 s64 left = local64_read(&hwc->period_left); 1112 s64 period = hwc->sample_period; 1113 int ret = 0, idx = hwc->idx; 1114 1115 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1116 return 0; 1117 1118 /* 1119 * If we are way outside a reasonable range then just skip forward: 1120 */ 1121 if (unlikely(left <= -period)) { 1122 left = period; 1123 local64_set(&hwc->period_left, left); 1124 hwc->last_period = period; 1125 ret = 1; 1126 } 1127 1128 if (unlikely(left <= 0)) { 1129 left += period; 1130 local64_set(&hwc->period_left, left); 1131 hwc->last_period = period; 1132 ret = 1; 1133 } 1134 /* 1135 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1136 */ 1137 if (unlikely(left < 2)) 1138 left = 2; 1139 1140 if (left > x86_pmu.max_period) 1141 left = x86_pmu.max_period; 1142 1143 if (x86_pmu.limit_period) 1144 left = x86_pmu.limit_period(event, left); 1145 1146 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1147 1148 if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) || 1149 local64_read(&hwc->prev_count) != (u64)-left) { 1150 /* 1151 * The hw event starts counting from this event offset, 1152 * mark it to be able to extra future deltas: 1153 */ 1154 local64_set(&hwc->prev_count, (u64)-left); 1155 1156 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1157 } 1158 1159 /* 1160 * Due to erratum on certan cpu we need 1161 * a second write to be sure the register 1162 * is updated properly 1163 */ 1164 if (x86_pmu.perfctr_second_write) { 1165 wrmsrl(hwc->event_base, 1166 (u64)(-left) & x86_pmu.cntval_mask); 1167 } 1168 1169 perf_event_update_userpage(event); 1170 1171 return ret; 1172 } 1173 1174 void x86_pmu_enable_event(struct perf_event *event) 1175 { 1176 if (__this_cpu_read(cpu_hw_events.enabled)) 1177 __x86_pmu_enable_event(&event->hw, 1178 ARCH_PERFMON_EVENTSEL_ENABLE); 1179 } 1180 1181 /* 1182 * Add a single event to the PMU. 1183 * 1184 * The event is added to the group of enabled events 1185 * but only if it can be scehduled with existing events. 1186 */ 1187 static int x86_pmu_add(struct perf_event *event, int flags) 1188 { 1189 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1190 struct hw_perf_event *hwc; 1191 int assign[X86_PMC_IDX_MAX]; 1192 int n, n0, ret; 1193 1194 hwc = &event->hw; 1195 1196 n0 = cpuc->n_events; 1197 ret = n = collect_events(cpuc, event, false); 1198 if (ret < 0) 1199 goto out; 1200 1201 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1202 if (!(flags & PERF_EF_START)) 1203 hwc->state |= PERF_HES_ARCH; 1204 1205 /* 1206 * If group events scheduling transaction was started, 1207 * skip the schedulability test here, it will be performed 1208 * at commit time (->commit_txn) as a whole. 1209 * 1210 * If commit fails, we'll call ->del() on all events 1211 * for which ->add() was called. 1212 */ 1213 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1214 goto done_collect; 1215 1216 ret = x86_pmu.schedule_events(cpuc, n, assign); 1217 if (ret) 1218 goto out; 1219 /* 1220 * copy new assignment, now we know it is possible 1221 * will be used by hw_perf_enable() 1222 */ 1223 memcpy(cpuc->assign, assign, n*sizeof(int)); 1224 1225 done_collect: 1226 /* 1227 * Commit the collect_events() state. See x86_pmu_del() and 1228 * x86_pmu_*_txn(). 1229 */ 1230 cpuc->n_events = n; 1231 cpuc->n_added += n - n0; 1232 cpuc->n_txn += n - n0; 1233 1234 if (x86_pmu.add) { 1235 /* 1236 * This is before x86_pmu_enable() will call x86_pmu_start(), 1237 * so we enable LBRs before an event needs them etc.. 1238 */ 1239 x86_pmu.add(event); 1240 } 1241 1242 ret = 0; 1243 out: 1244 return ret; 1245 } 1246 1247 static void x86_pmu_start(struct perf_event *event, int flags) 1248 { 1249 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1250 int idx = event->hw.idx; 1251 1252 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1253 return; 1254 1255 if (WARN_ON_ONCE(idx == -1)) 1256 return; 1257 1258 if (flags & PERF_EF_RELOAD) { 1259 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1260 x86_perf_event_set_period(event); 1261 } 1262 1263 event->hw.state = 0; 1264 1265 cpuc->events[idx] = event; 1266 __set_bit(idx, cpuc->active_mask); 1267 __set_bit(idx, cpuc->running); 1268 x86_pmu.enable(event); 1269 perf_event_update_userpage(event); 1270 } 1271 1272 void perf_event_print_debug(void) 1273 { 1274 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1275 u64 pebs, debugctl; 1276 struct cpu_hw_events *cpuc; 1277 unsigned long flags; 1278 int cpu, idx; 1279 1280 if (!x86_pmu.num_counters) 1281 return; 1282 1283 local_irq_save(flags); 1284 1285 cpu = smp_processor_id(); 1286 cpuc = &per_cpu(cpu_hw_events, cpu); 1287 1288 if (x86_pmu.version >= 2) { 1289 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1290 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1291 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1292 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1293 1294 pr_info("\n"); 1295 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1296 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1297 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1298 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1299 if (x86_pmu.pebs_constraints) { 1300 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1301 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1302 } 1303 if (x86_pmu.lbr_nr) { 1304 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1305 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1306 } 1307 } 1308 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1309 1310 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1311 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1312 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1313 1314 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1315 1316 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1317 cpu, idx, pmc_ctrl); 1318 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1319 cpu, idx, pmc_count); 1320 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1321 cpu, idx, prev_left); 1322 } 1323 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1324 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1325 1326 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1327 cpu, idx, pmc_count); 1328 } 1329 local_irq_restore(flags); 1330 } 1331 1332 void x86_pmu_stop(struct perf_event *event, int flags) 1333 { 1334 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1335 struct hw_perf_event *hwc = &event->hw; 1336 1337 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) { 1338 x86_pmu.disable(event); 1339 cpuc->events[hwc->idx] = NULL; 1340 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1341 hwc->state |= PERF_HES_STOPPED; 1342 } 1343 1344 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1345 /* 1346 * Drain the remaining delta count out of a event 1347 * that we are disabling: 1348 */ 1349 x86_perf_event_update(event); 1350 hwc->state |= PERF_HES_UPTODATE; 1351 } 1352 } 1353 1354 static void x86_pmu_del(struct perf_event *event, int flags) 1355 { 1356 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1357 int i; 1358 1359 /* 1360 * event is descheduled 1361 */ 1362 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; 1363 1364 /* 1365 * If we're called during a txn, we only need to undo x86_pmu.add. 1366 * The events never got scheduled and ->cancel_txn will truncate 1367 * the event_list. 1368 * 1369 * XXX assumes any ->del() called during a TXN will only be on 1370 * an event added during that same TXN. 1371 */ 1372 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1373 goto do_del; 1374 1375 /* 1376 * Not a TXN, therefore cleanup properly. 1377 */ 1378 x86_pmu_stop(event, PERF_EF_UPDATE); 1379 1380 for (i = 0; i < cpuc->n_events; i++) { 1381 if (event == cpuc->event_list[i]) 1382 break; 1383 } 1384 1385 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1386 return; 1387 1388 /* If we have a newly added event; make sure to decrease n_added. */ 1389 if (i >= cpuc->n_events - cpuc->n_added) 1390 --cpuc->n_added; 1391 1392 if (x86_pmu.put_event_constraints) 1393 x86_pmu.put_event_constraints(cpuc, event); 1394 1395 /* Delete the array entry. */ 1396 while (++i < cpuc->n_events) { 1397 cpuc->event_list[i-1] = cpuc->event_list[i]; 1398 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1399 } 1400 --cpuc->n_events; 1401 1402 perf_event_update_userpage(event); 1403 1404 do_del: 1405 if (x86_pmu.del) { 1406 /* 1407 * This is after x86_pmu_stop(); so we disable LBRs after any 1408 * event can need them etc.. 1409 */ 1410 x86_pmu.del(event); 1411 } 1412 } 1413 1414 int x86_pmu_handle_irq(struct pt_regs *regs) 1415 { 1416 struct perf_sample_data data; 1417 struct cpu_hw_events *cpuc; 1418 struct perf_event *event; 1419 int idx, handled = 0; 1420 u64 val; 1421 1422 cpuc = this_cpu_ptr(&cpu_hw_events); 1423 1424 /* 1425 * Some chipsets need to unmask the LVTPC in a particular spot 1426 * inside the nmi handler. As a result, the unmasking was pushed 1427 * into all the nmi handlers. 1428 * 1429 * This generic handler doesn't seem to have any issues where the 1430 * unmasking occurs so it was left at the top. 1431 */ 1432 apic_write(APIC_LVTPC, APIC_DM_NMI); 1433 1434 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1435 if (!test_bit(idx, cpuc->active_mask)) { 1436 /* 1437 * Though we deactivated the counter some cpus 1438 * might still deliver spurious interrupts still 1439 * in flight. Catch them: 1440 */ 1441 if (__test_and_clear_bit(idx, cpuc->running)) 1442 handled++; 1443 continue; 1444 } 1445 1446 event = cpuc->events[idx]; 1447 1448 val = x86_perf_event_update(event); 1449 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1450 continue; 1451 1452 /* 1453 * event overflow 1454 */ 1455 handled++; 1456 perf_sample_data_init(&data, 0, event->hw.last_period); 1457 1458 if (!x86_perf_event_set_period(event)) 1459 continue; 1460 1461 if (perf_event_overflow(event, &data, regs)) 1462 x86_pmu_stop(event, 0); 1463 } 1464 1465 if (handled) 1466 inc_irq_stat(apic_perf_irqs); 1467 1468 return handled; 1469 } 1470 1471 void perf_events_lapic_init(void) 1472 { 1473 if (!x86_pmu.apic || !x86_pmu_initialized()) 1474 return; 1475 1476 /* 1477 * Always use NMI for PMU 1478 */ 1479 apic_write(APIC_LVTPC, APIC_DM_NMI); 1480 } 1481 1482 static int 1483 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1484 { 1485 u64 start_clock; 1486 u64 finish_clock; 1487 int ret; 1488 1489 /* 1490 * All PMUs/events that share this PMI handler should make sure to 1491 * increment active_events for their events. 1492 */ 1493 if (!atomic_read(&active_events)) 1494 return NMI_DONE; 1495 1496 start_clock = sched_clock(); 1497 ret = x86_pmu.handle_irq(regs); 1498 finish_clock = sched_clock(); 1499 1500 perf_sample_event_took(finish_clock - start_clock); 1501 1502 return ret; 1503 } 1504 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1505 1506 struct event_constraint emptyconstraint; 1507 struct event_constraint unconstrained; 1508 1509 static int x86_pmu_prepare_cpu(unsigned int cpu) 1510 { 1511 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1512 int i; 1513 1514 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1515 cpuc->kfree_on_online[i] = NULL; 1516 if (x86_pmu.cpu_prepare) 1517 return x86_pmu.cpu_prepare(cpu); 1518 return 0; 1519 } 1520 1521 static int x86_pmu_dead_cpu(unsigned int cpu) 1522 { 1523 if (x86_pmu.cpu_dead) 1524 x86_pmu.cpu_dead(cpu); 1525 return 0; 1526 } 1527 1528 static int x86_pmu_online_cpu(unsigned int cpu) 1529 { 1530 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1531 int i; 1532 1533 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1534 kfree(cpuc->kfree_on_online[i]); 1535 cpuc->kfree_on_online[i] = NULL; 1536 } 1537 return 0; 1538 } 1539 1540 static int x86_pmu_starting_cpu(unsigned int cpu) 1541 { 1542 if (x86_pmu.cpu_starting) 1543 x86_pmu.cpu_starting(cpu); 1544 return 0; 1545 } 1546 1547 static int x86_pmu_dying_cpu(unsigned int cpu) 1548 { 1549 if (x86_pmu.cpu_dying) 1550 x86_pmu.cpu_dying(cpu); 1551 return 0; 1552 } 1553 1554 static void __init pmu_check_apic(void) 1555 { 1556 if (boot_cpu_has(X86_FEATURE_APIC)) 1557 return; 1558 1559 x86_pmu.apic = 0; 1560 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1561 pr_info("no hardware sampling interrupt available.\n"); 1562 1563 /* 1564 * If we have a PMU initialized but no APIC 1565 * interrupts, we cannot sample hardware 1566 * events (user-space has to fall back and 1567 * sample via a hrtimer based software event): 1568 */ 1569 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1570 1571 } 1572 1573 static struct attribute_group x86_pmu_format_group = { 1574 .name = "format", 1575 .attrs = NULL, 1576 }; 1577 1578 /* 1579 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1580 * out of events_attr attributes. 1581 */ 1582 static void __init filter_events(struct attribute **attrs) 1583 { 1584 struct device_attribute *d; 1585 struct perf_pmu_events_attr *pmu_attr; 1586 int offset = 0; 1587 int i, j; 1588 1589 for (i = 0; attrs[i]; i++) { 1590 d = (struct device_attribute *)attrs[i]; 1591 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); 1592 /* str trumps id */ 1593 if (pmu_attr->event_str) 1594 continue; 1595 if (x86_pmu.event_map(i + offset)) 1596 continue; 1597 1598 for (j = i; attrs[j]; j++) 1599 attrs[j] = attrs[j + 1]; 1600 1601 /* Check the shifted attr. */ 1602 i--; 1603 1604 /* 1605 * event_map() is index based, the attrs array is organized 1606 * by increasing event index. If we shift the events, then 1607 * we need to compensate for the event_map(), otherwise 1608 * we are looking up the wrong event in the map 1609 */ 1610 offset++; 1611 } 1612 } 1613 1614 /* Merge two pointer arrays */ 1615 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) 1616 { 1617 struct attribute **new; 1618 int j, i; 1619 1620 for (j = 0; a[j]; j++) 1621 ; 1622 for (i = 0; b[i]; i++) 1623 j++; 1624 j++; 1625 1626 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL); 1627 if (!new) 1628 return NULL; 1629 1630 j = 0; 1631 for (i = 0; a[i]; i++) 1632 new[j++] = a[i]; 1633 for (i = 0; b[i]; i++) 1634 new[j++] = b[i]; 1635 new[j] = NULL; 1636 1637 return new; 1638 } 1639 1640 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1641 { 1642 struct perf_pmu_events_attr *pmu_attr = \ 1643 container_of(attr, struct perf_pmu_events_attr, attr); 1644 u64 config = x86_pmu.event_map(pmu_attr->id); 1645 1646 /* string trumps id */ 1647 if (pmu_attr->event_str) 1648 return sprintf(page, "%s", pmu_attr->event_str); 1649 1650 return x86_pmu.events_sysfs_show(page, config); 1651 } 1652 EXPORT_SYMBOL_GPL(events_sysfs_show); 1653 1654 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1655 char *page) 1656 { 1657 struct perf_pmu_events_ht_attr *pmu_attr = 1658 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1659 1660 /* 1661 * Report conditional events depending on Hyper-Threading. 1662 * 1663 * This is overly conservative as usually the HT special 1664 * handling is not needed if the other CPU thread is idle. 1665 * 1666 * Note this does not (and cannot) handle the case when thread 1667 * siblings are invisible, for example with virtualization 1668 * if they are owned by some other guest. The user tool 1669 * has to re-read when a thread sibling gets onlined later. 1670 */ 1671 return sprintf(page, "%s", 1672 topology_max_smt_threads() > 1 ? 1673 pmu_attr->event_str_ht : 1674 pmu_attr->event_str_noht); 1675 } 1676 1677 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1678 EVENT_ATTR(instructions, INSTRUCTIONS ); 1679 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1680 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1681 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1682 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1683 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1684 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1685 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1686 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1687 1688 static struct attribute *empty_attrs; 1689 1690 static struct attribute *events_attr[] = { 1691 EVENT_PTR(CPU_CYCLES), 1692 EVENT_PTR(INSTRUCTIONS), 1693 EVENT_PTR(CACHE_REFERENCES), 1694 EVENT_PTR(CACHE_MISSES), 1695 EVENT_PTR(BRANCH_INSTRUCTIONS), 1696 EVENT_PTR(BRANCH_MISSES), 1697 EVENT_PTR(BUS_CYCLES), 1698 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1699 EVENT_PTR(STALLED_CYCLES_BACKEND), 1700 EVENT_PTR(REF_CPU_CYCLES), 1701 NULL, 1702 }; 1703 1704 static struct attribute_group x86_pmu_events_group = { 1705 .name = "events", 1706 .attrs = events_attr, 1707 }; 1708 1709 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1710 { 1711 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1712 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1713 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1714 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1715 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1716 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1717 ssize_t ret; 1718 1719 /* 1720 * We have whole page size to spend and just little data 1721 * to write, so we can safely use sprintf. 1722 */ 1723 ret = sprintf(page, "event=0x%02llx", event); 1724 1725 if (umask) 1726 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1727 1728 if (edge) 1729 ret += sprintf(page + ret, ",edge"); 1730 1731 if (pc) 1732 ret += sprintf(page + ret, ",pc"); 1733 1734 if (any) 1735 ret += sprintf(page + ret, ",any"); 1736 1737 if (inv) 1738 ret += sprintf(page + ret, ",inv"); 1739 1740 if (cmask) 1741 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1742 1743 ret += sprintf(page + ret, "\n"); 1744 1745 return ret; 1746 } 1747 1748 static int __init init_hw_perf_events(void) 1749 { 1750 struct x86_pmu_quirk *quirk; 1751 int err; 1752 1753 pr_info("Performance Events: "); 1754 1755 switch (boot_cpu_data.x86_vendor) { 1756 case X86_VENDOR_INTEL: 1757 err = intel_pmu_init(); 1758 break; 1759 case X86_VENDOR_AMD: 1760 err = amd_pmu_init(); 1761 break; 1762 default: 1763 err = -ENOTSUPP; 1764 } 1765 if (err != 0) { 1766 pr_cont("no PMU driver, software events only.\n"); 1767 return 0; 1768 } 1769 1770 pmu_check_apic(); 1771 1772 /* sanity check that the hardware exists or is emulated */ 1773 if (!check_hw_exists()) 1774 return 0; 1775 1776 pr_cont("%s PMU driver.\n", x86_pmu.name); 1777 1778 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1779 1780 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1781 quirk->func(); 1782 1783 if (!x86_pmu.intel_ctrl) 1784 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1785 1786 perf_events_lapic_init(); 1787 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1788 1789 unconstrained = (struct event_constraint) 1790 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1791 0, x86_pmu.num_counters, 0, 0); 1792 1793 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1794 1795 if (x86_pmu.event_attrs) 1796 x86_pmu_events_group.attrs = x86_pmu.event_attrs; 1797 1798 if (!x86_pmu.events_sysfs_show) 1799 x86_pmu_events_group.attrs = &empty_attrs; 1800 else 1801 filter_events(x86_pmu_events_group.attrs); 1802 1803 if (x86_pmu.cpu_events) { 1804 struct attribute **tmp; 1805 1806 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); 1807 if (!WARN_ON(!tmp)) 1808 x86_pmu_events_group.attrs = tmp; 1809 } 1810 1811 pr_info("... version: %d\n", x86_pmu.version); 1812 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1813 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1814 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1815 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1816 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1817 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1818 1819 /* 1820 * Install callbacks. Core will call them for each online 1821 * cpu. 1822 */ 1823 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", 1824 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1825 if (err) 1826 return err; 1827 1828 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1829 "perf/x86:starting", x86_pmu_starting_cpu, 1830 x86_pmu_dying_cpu); 1831 if (err) 1832 goto out; 1833 1834 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online", 1835 x86_pmu_online_cpu, NULL); 1836 if (err) 1837 goto out1; 1838 1839 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1840 if (err) 1841 goto out2; 1842 1843 return 0; 1844 1845 out2: 1846 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1847 out1: 1848 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1849 out: 1850 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1851 return err; 1852 } 1853 early_initcall(init_hw_perf_events); 1854 1855 static inline void x86_pmu_read(struct perf_event *event) 1856 { 1857 x86_perf_event_update(event); 1858 } 1859 1860 /* 1861 * Start group events scheduling transaction 1862 * Set the flag to make pmu::enable() not perform the 1863 * schedulability test, it will be performed at commit time 1864 * 1865 * We only support PERF_PMU_TXN_ADD transactions. Save the 1866 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1867 * transactions. 1868 */ 1869 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1870 { 1871 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1872 1873 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1874 1875 cpuc->txn_flags = txn_flags; 1876 if (txn_flags & ~PERF_PMU_TXN_ADD) 1877 return; 1878 1879 perf_pmu_disable(pmu); 1880 __this_cpu_write(cpu_hw_events.n_txn, 0); 1881 } 1882 1883 /* 1884 * Stop group events scheduling transaction 1885 * Clear the flag and pmu::enable() will perform the 1886 * schedulability test. 1887 */ 1888 static void x86_pmu_cancel_txn(struct pmu *pmu) 1889 { 1890 unsigned int txn_flags; 1891 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1892 1893 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1894 1895 txn_flags = cpuc->txn_flags; 1896 cpuc->txn_flags = 0; 1897 if (txn_flags & ~PERF_PMU_TXN_ADD) 1898 return; 1899 1900 /* 1901 * Truncate collected array by the number of events added in this 1902 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1903 */ 1904 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1905 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1906 perf_pmu_enable(pmu); 1907 } 1908 1909 /* 1910 * Commit group events scheduling transaction 1911 * Perform the group schedulability test as a whole 1912 * Return 0 if success 1913 * 1914 * Does not cancel the transaction on failure; expects the caller to do this. 1915 */ 1916 static int x86_pmu_commit_txn(struct pmu *pmu) 1917 { 1918 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1919 int assign[X86_PMC_IDX_MAX]; 1920 int n, ret; 1921 1922 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1923 1924 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1925 cpuc->txn_flags = 0; 1926 return 0; 1927 } 1928 1929 n = cpuc->n_events; 1930 1931 if (!x86_pmu_initialized()) 1932 return -EAGAIN; 1933 1934 ret = x86_pmu.schedule_events(cpuc, n, assign); 1935 if (ret) 1936 return ret; 1937 1938 /* 1939 * copy new assignment, now we know it is possible 1940 * will be used by hw_perf_enable() 1941 */ 1942 memcpy(cpuc->assign, assign, n*sizeof(int)); 1943 1944 cpuc->txn_flags = 0; 1945 perf_pmu_enable(pmu); 1946 return 0; 1947 } 1948 /* 1949 * a fake_cpuc is used to validate event groups. Due to 1950 * the extra reg logic, we need to also allocate a fake 1951 * per_core and per_cpu structure. Otherwise, group events 1952 * using extra reg may conflict without the kernel being 1953 * able to catch this when the last event gets added to 1954 * the group. 1955 */ 1956 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 1957 { 1958 kfree(cpuc->shared_regs); 1959 kfree(cpuc); 1960 } 1961 1962 static struct cpu_hw_events *allocate_fake_cpuc(void) 1963 { 1964 struct cpu_hw_events *cpuc; 1965 int cpu = raw_smp_processor_id(); 1966 1967 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 1968 if (!cpuc) 1969 return ERR_PTR(-ENOMEM); 1970 1971 /* only needed, if we have extra_regs */ 1972 if (x86_pmu.extra_regs) { 1973 cpuc->shared_regs = allocate_shared_regs(cpu); 1974 if (!cpuc->shared_regs) 1975 goto error; 1976 } 1977 cpuc->is_fake = 1; 1978 return cpuc; 1979 error: 1980 free_fake_cpuc(cpuc); 1981 return ERR_PTR(-ENOMEM); 1982 } 1983 1984 /* 1985 * validate that we can schedule this event 1986 */ 1987 static int validate_event(struct perf_event *event) 1988 { 1989 struct cpu_hw_events *fake_cpuc; 1990 struct event_constraint *c; 1991 int ret = 0; 1992 1993 fake_cpuc = allocate_fake_cpuc(); 1994 if (IS_ERR(fake_cpuc)) 1995 return PTR_ERR(fake_cpuc); 1996 1997 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event); 1998 1999 if (!c || !c->weight) 2000 ret = -EINVAL; 2001 2002 if (x86_pmu.put_event_constraints) 2003 x86_pmu.put_event_constraints(fake_cpuc, event); 2004 2005 free_fake_cpuc(fake_cpuc); 2006 2007 return ret; 2008 } 2009 2010 /* 2011 * validate a single event group 2012 * 2013 * validation include: 2014 * - check events are compatible which each other 2015 * - events do not compete for the same counter 2016 * - number of events <= number of counters 2017 * 2018 * validation ensures the group can be loaded onto the 2019 * PMU if it was the only group available. 2020 */ 2021 static int validate_group(struct perf_event *event) 2022 { 2023 struct perf_event *leader = event->group_leader; 2024 struct cpu_hw_events *fake_cpuc; 2025 int ret = -EINVAL, n; 2026 2027 fake_cpuc = allocate_fake_cpuc(); 2028 if (IS_ERR(fake_cpuc)) 2029 return PTR_ERR(fake_cpuc); 2030 /* 2031 * the event is not yet connected with its 2032 * siblings therefore we must first collect 2033 * existing siblings, then add the new event 2034 * before we can simulate the scheduling 2035 */ 2036 n = collect_events(fake_cpuc, leader, true); 2037 if (n < 0) 2038 goto out; 2039 2040 fake_cpuc->n_events = n; 2041 n = collect_events(fake_cpuc, event, false); 2042 if (n < 0) 2043 goto out; 2044 2045 fake_cpuc->n_events = n; 2046 2047 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2048 2049 out: 2050 free_fake_cpuc(fake_cpuc); 2051 return ret; 2052 } 2053 2054 static int x86_pmu_event_init(struct perf_event *event) 2055 { 2056 struct pmu *tmp; 2057 int err; 2058 2059 switch (event->attr.type) { 2060 case PERF_TYPE_RAW: 2061 case PERF_TYPE_HARDWARE: 2062 case PERF_TYPE_HW_CACHE: 2063 break; 2064 2065 default: 2066 return -ENOENT; 2067 } 2068 2069 err = __x86_pmu_event_init(event); 2070 if (!err) { 2071 /* 2072 * we temporarily connect event to its pmu 2073 * such that validate_group() can classify 2074 * it as an x86 event using is_x86_event() 2075 */ 2076 tmp = event->pmu; 2077 event->pmu = &pmu; 2078 2079 if (event->group_leader != event) 2080 err = validate_group(event); 2081 else 2082 err = validate_event(event); 2083 2084 event->pmu = tmp; 2085 } 2086 if (err) { 2087 if (event->destroy) 2088 event->destroy(event); 2089 } 2090 2091 if (ACCESS_ONCE(x86_pmu.attr_rdpmc)) 2092 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2093 2094 return err; 2095 } 2096 2097 static void refresh_pce(void *ignored) 2098 { 2099 if (current->mm) 2100 load_mm_cr4(current->mm); 2101 } 2102 2103 static void x86_pmu_event_mapped(struct perf_event *event) 2104 { 2105 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2106 return; 2107 2108 if (atomic_inc_return(¤t->mm->context.perf_rdpmc_allowed) == 1) 2109 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1); 2110 } 2111 2112 static void x86_pmu_event_unmapped(struct perf_event *event) 2113 { 2114 if (!current->mm) 2115 return; 2116 2117 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2118 return; 2119 2120 if (atomic_dec_and_test(¤t->mm->context.perf_rdpmc_allowed)) 2121 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1); 2122 } 2123 2124 static int x86_pmu_event_idx(struct perf_event *event) 2125 { 2126 int idx = event->hw.idx; 2127 2128 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2129 return 0; 2130 2131 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2132 idx -= INTEL_PMC_IDX_FIXED; 2133 idx |= 1 << 30; 2134 } 2135 2136 return idx + 1; 2137 } 2138 2139 static ssize_t get_attr_rdpmc(struct device *cdev, 2140 struct device_attribute *attr, 2141 char *buf) 2142 { 2143 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2144 } 2145 2146 static ssize_t set_attr_rdpmc(struct device *cdev, 2147 struct device_attribute *attr, 2148 const char *buf, size_t count) 2149 { 2150 unsigned long val; 2151 ssize_t ret; 2152 2153 ret = kstrtoul(buf, 0, &val); 2154 if (ret) 2155 return ret; 2156 2157 if (val > 2) 2158 return -EINVAL; 2159 2160 if (x86_pmu.attr_rdpmc_broken) 2161 return -ENOTSUPP; 2162 2163 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) { 2164 /* 2165 * Changing into or out of always available, aka 2166 * perf-event-bypassing mode. This path is extremely slow, 2167 * but only root can trigger it, so it's okay. 2168 */ 2169 if (val == 2) 2170 static_key_slow_inc(&rdpmc_always_available); 2171 else 2172 static_key_slow_dec(&rdpmc_always_available); 2173 on_each_cpu(refresh_pce, NULL, 1); 2174 } 2175 2176 x86_pmu.attr_rdpmc = val; 2177 2178 return count; 2179 } 2180 2181 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2182 2183 static struct attribute *x86_pmu_attrs[] = { 2184 &dev_attr_rdpmc.attr, 2185 NULL, 2186 }; 2187 2188 static struct attribute_group x86_pmu_attr_group = { 2189 .attrs = x86_pmu_attrs, 2190 }; 2191 2192 static const struct attribute_group *x86_pmu_attr_groups[] = { 2193 &x86_pmu_attr_group, 2194 &x86_pmu_format_group, 2195 &x86_pmu_events_group, 2196 NULL, 2197 }; 2198 2199 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2200 { 2201 if (x86_pmu.sched_task) 2202 x86_pmu.sched_task(ctx, sched_in); 2203 } 2204 2205 void perf_check_microcode(void) 2206 { 2207 if (x86_pmu.check_microcode) 2208 x86_pmu.check_microcode(); 2209 } 2210 EXPORT_SYMBOL_GPL(perf_check_microcode); 2211 2212 static struct pmu pmu = { 2213 .pmu_enable = x86_pmu_enable, 2214 .pmu_disable = x86_pmu_disable, 2215 2216 .attr_groups = x86_pmu_attr_groups, 2217 2218 .event_init = x86_pmu_event_init, 2219 2220 .event_mapped = x86_pmu_event_mapped, 2221 .event_unmapped = x86_pmu_event_unmapped, 2222 2223 .add = x86_pmu_add, 2224 .del = x86_pmu_del, 2225 .start = x86_pmu_start, 2226 .stop = x86_pmu_stop, 2227 .read = x86_pmu_read, 2228 2229 .start_txn = x86_pmu_start_txn, 2230 .cancel_txn = x86_pmu_cancel_txn, 2231 .commit_txn = x86_pmu_commit_txn, 2232 2233 .event_idx = x86_pmu_event_idx, 2234 .sched_task = x86_pmu_sched_task, 2235 .task_ctx_size = sizeof(struct x86_perf_task_context), 2236 }; 2237 2238 void arch_perf_update_userpage(struct perf_event *event, 2239 struct perf_event_mmap_page *userpg, u64 now) 2240 { 2241 struct cyc2ns_data *data; 2242 2243 userpg->cap_user_time = 0; 2244 userpg->cap_user_time_zero = 0; 2245 userpg->cap_user_rdpmc = 2246 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2247 userpg->pmc_width = x86_pmu.cntval_bits; 2248 2249 if (!sched_clock_stable()) 2250 return; 2251 2252 data = cyc2ns_read_begin(); 2253 2254 /* 2255 * Internal timekeeping for enabled/running/stopped times 2256 * is always in the local_clock domain. 2257 */ 2258 userpg->cap_user_time = 1; 2259 userpg->time_mult = data->cyc2ns_mul; 2260 userpg->time_shift = data->cyc2ns_shift; 2261 userpg->time_offset = data->cyc2ns_offset - now; 2262 2263 /* 2264 * cap_user_time_zero doesn't make sense when we're using a different 2265 * time base for the records. 2266 */ 2267 if (!event->attr.use_clockid) { 2268 userpg->cap_user_time_zero = 1; 2269 userpg->time_zero = data->cyc2ns_offset; 2270 } 2271 2272 cyc2ns_read_end(data); 2273 } 2274 2275 void 2276 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2277 { 2278 struct unwind_state state; 2279 unsigned long addr; 2280 2281 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2282 /* TODO: We don't support guest os callchain now */ 2283 return; 2284 } 2285 2286 if (perf_callchain_store(entry, regs->ip)) 2287 return; 2288 2289 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state); 2290 unwind_next_frame(&state)) { 2291 addr = unwind_get_return_address(&state); 2292 if (!addr || perf_callchain_store(entry, addr)) 2293 return; 2294 } 2295 } 2296 2297 static inline int 2298 valid_user_frame(const void __user *fp, unsigned long size) 2299 { 2300 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2301 } 2302 2303 static unsigned long get_segment_base(unsigned int segment) 2304 { 2305 struct desc_struct *desc; 2306 unsigned int idx = segment >> 3; 2307 2308 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2309 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2310 struct ldt_struct *ldt; 2311 2312 if (idx > LDT_ENTRIES) 2313 return 0; 2314 2315 /* IRQs are off, so this synchronizes with smp_store_release */ 2316 ldt = lockless_dereference(current->active_mm->context.ldt); 2317 if (!ldt || idx > ldt->size) 2318 return 0; 2319 2320 desc = &ldt->entries[idx]; 2321 #else 2322 return 0; 2323 #endif 2324 } else { 2325 if (idx > GDT_ENTRIES) 2326 return 0; 2327 2328 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2329 } 2330 2331 return get_desc_base(desc); 2332 } 2333 2334 #ifdef CONFIG_IA32_EMULATION 2335 2336 #include <asm/compat.h> 2337 2338 static inline int 2339 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2340 { 2341 /* 32-bit process in 64-bit kernel. */ 2342 unsigned long ss_base, cs_base; 2343 struct stack_frame_ia32 frame; 2344 const void __user *fp; 2345 2346 if (!test_thread_flag(TIF_IA32)) 2347 return 0; 2348 2349 cs_base = get_segment_base(regs->cs); 2350 ss_base = get_segment_base(regs->ss); 2351 2352 fp = compat_ptr(ss_base + regs->bp); 2353 pagefault_disable(); 2354 while (entry->nr < entry->max_stack) { 2355 unsigned long bytes; 2356 frame.next_frame = 0; 2357 frame.return_address = 0; 2358 2359 if (!valid_user_frame(fp, sizeof(frame))) 2360 break; 2361 2362 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2363 if (bytes != 0) 2364 break; 2365 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2366 if (bytes != 0) 2367 break; 2368 2369 perf_callchain_store(entry, cs_base + frame.return_address); 2370 fp = compat_ptr(ss_base + frame.next_frame); 2371 } 2372 pagefault_enable(); 2373 return 1; 2374 } 2375 #else 2376 static inline int 2377 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2378 { 2379 return 0; 2380 } 2381 #endif 2382 2383 void 2384 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2385 { 2386 struct stack_frame frame; 2387 const unsigned long __user *fp; 2388 2389 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2390 /* TODO: We don't support guest os callchain now */ 2391 return; 2392 } 2393 2394 /* 2395 * We don't know what to do with VM86 stacks.. ignore them for now. 2396 */ 2397 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2398 return; 2399 2400 fp = (unsigned long __user *)regs->bp; 2401 2402 perf_callchain_store(entry, regs->ip); 2403 2404 if (!current->mm) 2405 return; 2406 2407 if (perf_callchain_user32(regs, entry)) 2408 return; 2409 2410 pagefault_disable(); 2411 while (entry->nr < entry->max_stack) { 2412 unsigned long bytes; 2413 2414 frame.next_frame = NULL; 2415 frame.return_address = 0; 2416 2417 if (!valid_user_frame(fp, sizeof(frame))) 2418 break; 2419 2420 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2421 if (bytes != 0) 2422 break; 2423 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2424 if (bytes != 0) 2425 break; 2426 2427 perf_callchain_store(entry, frame.return_address); 2428 fp = (void __user *)frame.next_frame; 2429 } 2430 pagefault_enable(); 2431 } 2432 2433 /* 2434 * Deal with code segment offsets for the various execution modes: 2435 * 2436 * VM86 - the good olde 16 bit days, where the linear address is 2437 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2438 * 2439 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2440 * to figure out what the 32bit base address is. 2441 * 2442 * X32 - has TIF_X32 set, but is running in x86_64 2443 * 2444 * X86_64 - CS,DS,SS,ES are all zero based. 2445 */ 2446 static unsigned long code_segment_base(struct pt_regs *regs) 2447 { 2448 /* 2449 * For IA32 we look at the GDT/LDT segment base to convert the 2450 * effective IP to a linear address. 2451 */ 2452 2453 #ifdef CONFIG_X86_32 2454 /* 2455 * If we are in VM86 mode, add the segment offset to convert to a 2456 * linear address. 2457 */ 2458 if (regs->flags & X86_VM_MASK) 2459 return 0x10 * regs->cs; 2460 2461 if (user_mode(regs) && regs->cs != __USER_CS) 2462 return get_segment_base(regs->cs); 2463 #else 2464 if (user_mode(regs) && !user_64bit_mode(regs) && 2465 regs->cs != __USER32_CS) 2466 return get_segment_base(regs->cs); 2467 #endif 2468 return 0; 2469 } 2470 2471 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2472 { 2473 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2474 return perf_guest_cbs->get_guest_ip(); 2475 2476 return regs->ip + code_segment_base(regs); 2477 } 2478 2479 unsigned long perf_misc_flags(struct pt_regs *regs) 2480 { 2481 int misc = 0; 2482 2483 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2484 if (perf_guest_cbs->is_user_mode()) 2485 misc |= PERF_RECORD_MISC_GUEST_USER; 2486 else 2487 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2488 } else { 2489 if (user_mode(regs)) 2490 misc |= PERF_RECORD_MISC_USER; 2491 else 2492 misc |= PERF_RECORD_MISC_KERNEL; 2493 } 2494 2495 if (regs->flags & PERF_EFLAGS_EXACT) 2496 misc |= PERF_RECORD_MISC_EXACT_IP; 2497 2498 return misc; 2499 } 2500 2501 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2502 { 2503 cap->version = x86_pmu.version; 2504 cap->num_counters_gp = x86_pmu.num_counters; 2505 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2506 cap->bit_width_gp = x86_pmu.cntval_bits; 2507 cap->bit_width_fixed = x86_pmu.cntval_bits; 2508 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2509 cap->events_mask_len = x86_pmu.events_mask_len; 2510 } 2511 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2512