1 /* 2 * Performance events x86 architecture code 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2009 Jaswinder Singh Rajput 7 * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter 8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra 9 * Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com> 10 * Copyright (C) 2009 Google, Inc., Stephane Eranian 11 * 12 * For licencing details see kernel-base/COPYING 13 */ 14 15 #include <linux/perf_event.h> 16 #include <linux/capability.h> 17 #include <linux/notifier.h> 18 #include <linux/hardirq.h> 19 #include <linux/kprobes.h> 20 #include <linux/export.h> 21 #include <linux/init.h> 22 #include <linux/kdebug.h> 23 #include <linux/sched.h> 24 #include <linux/uaccess.h> 25 #include <linux/slab.h> 26 #include <linux/cpu.h> 27 #include <linux/bitops.h> 28 #include <linux/device.h> 29 30 #include <asm/apic.h> 31 #include <asm/stacktrace.h> 32 #include <asm/nmi.h> 33 #include <asm/smp.h> 34 #include <asm/alternative.h> 35 #include <asm/mmu_context.h> 36 #include <asm/tlbflush.h> 37 #include <asm/timer.h> 38 #include <asm/desc.h> 39 #include <asm/ldt.h> 40 #include <asm/unwind.h> 41 42 #include "perf_event.h" 43 44 struct x86_pmu x86_pmu __read_mostly; 45 46 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 47 .enabled = 1, 48 }; 49 50 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE; 51 52 u64 __read_mostly hw_cache_event_ids 53 [PERF_COUNT_HW_CACHE_MAX] 54 [PERF_COUNT_HW_CACHE_OP_MAX] 55 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 56 u64 __read_mostly hw_cache_extra_regs 57 [PERF_COUNT_HW_CACHE_MAX] 58 [PERF_COUNT_HW_CACHE_OP_MAX] 59 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 60 61 /* 62 * Propagate event elapsed time into the generic event. 63 * Can only be executed on the CPU where the event is active. 64 * Returns the delta events processed. 65 */ 66 u64 x86_perf_event_update(struct perf_event *event) 67 { 68 struct hw_perf_event *hwc = &event->hw; 69 int shift = 64 - x86_pmu.cntval_bits; 70 u64 prev_raw_count, new_raw_count; 71 int idx = hwc->idx; 72 s64 delta; 73 74 if (idx == INTEL_PMC_IDX_FIXED_BTS) 75 return 0; 76 77 /* 78 * Careful: an NMI might modify the previous event value. 79 * 80 * Our tactic to handle this is to first atomically read and 81 * exchange a new raw count - then add that new-prev delta 82 * count to the generic event atomically: 83 */ 84 again: 85 prev_raw_count = local64_read(&hwc->prev_count); 86 rdpmcl(hwc->event_base_rdpmc, new_raw_count); 87 88 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 89 new_raw_count) != prev_raw_count) 90 goto again; 91 92 /* 93 * Now we have the new raw value and have updated the prev 94 * timestamp already. We can now calculate the elapsed delta 95 * (event-)time and add that to the generic event. 96 * 97 * Careful, not all hw sign-extends above the physical width 98 * of the count. 99 */ 100 delta = (new_raw_count << shift) - (prev_raw_count << shift); 101 delta >>= shift; 102 103 local64_add(delta, &event->count); 104 local64_sub(delta, &hwc->period_left); 105 106 return new_raw_count; 107 } 108 109 /* 110 * Find and validate any extra registers to set up. 111 */ 112 static int x86_pmu_extra_regs(u64 config, struct perf_event *event) 113 { 114 struct hw_perf_event_extra *reg; 115 struct extra_reg *er; 116 117 reg = &event->hw.extra_reg; 118 119 if (!x86_pmu.extra_regs) 120 return 0; 121 122 for (er = x86_pmu.extra_regs; er->msr; er++) { 123 if (er->event != (config & er->config_mask)) 124 continue; 125 if (event->attr.config1 & ~er->valid_mask) 126 return -EINVAL; 127 /* Check if the extra msrs can be safely accessed*/ 128 if (!er->extra_msr_access) 129 return -ENXIO; 130 131 reg->idx = er->idx; 132 reg->config = event->attr.config1; 133 reg->reg = er->msr; 134 break; 135 } 136 return 0; 137 } 138 139 static atomic_t active_events; 140 static atomic_t pmc_refcount; 141 static DEFINE_MUTEX(pmc_reserve_mutex); 142 143 #ifdef CONFIG_X86_LOCAL_APIC 144 145 static bool reserve_pmc_hardware(void) 146 { 147 int i; 148 149 for (i = 0; i < x86_pmu.num_counters; i++) { 150 if (!reserve_perfctr_nmi(x86_pmu_event_addr(i))) 151 goto perfctr_fail; 152 } 153 154 for (i = 0; i < x86_pmu.num_counters; i++) { 155 if (!reserve_evntsel_nmi(x86_pmu_config_addr(i))) 156 goto eventsel_fail; 157 } 158 159 return true; 160 161 eventsel_fail: 162 for (i--; i >= 0; i--) 163 release_evntsel_nmi(x86_pmu_config_addr(i)); 164 165 i = x86_pmu.num_counters; 166 167 perfctr_fail: 168 for (i--; i >= 0; i--) 169 release_perfctr_nmi(x86_pmu_event_addr(i)); 170 171 return false; 172 } 173 174 static void release_pmc_hardware(void) 175 { 176 int i; 177 178 for (i = 0; i < x86_pmu.num_counters; i++) { 179 release_perfctr_nmi(x86_pmu_event_addr(i)); 180 release_evntsel_nmi(x86_pmu_config_addr(i)); 181 } 182 } 183 184 #else 185 186 static bool reserve_pmc_hardware(void) { return true; } 187 static void release_pmc_hardware(void) {} 188 189 #endif 190 191 static bool check_hw_exists(void) 192 { 193 u64 val, val_fail, val_new= ~0; 194 int i, reg, reg_fail, ret = 0; 195 int bios_fail = 0; 196 int reg_safe = -1; 197 198 /* 199 * Check to see if the BIOS enabled any of the counters, if so 200 * complain and bail. 201 */ 202 for (i = 0; i < x86_pmu.num_counters; i++) { 203 reg = x86_pmu_config_addr(i); 204 ret = rdmsrl_safe(reg, &val); 205 if (ret) 206 goto msr_fail; 207 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) { 208 bios_fail = 1; 209 val_fail = val; 210 reg_fail = reg; 211 } else { 212 reg_safe = i; 213 } 214 } 215 216 if (x86_pmu.num_counters_fixed) { 217 reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 218 ret = rdmsrl_safe(reg, &val); 219 if (ret) 220 goto msr_fail; 221 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 222 if (val & (0x03 << i*4)) { 223 bios_fail = 1; 224 val_fail = val; 225 reg_fail = reg; 226 } 227 } 228 } 229 230 /* 231 * If all the counters are enabled, the below test will always 232 * fail. The tools will also become useless in this scenario. 233 * Just fail and disable the hardware counters. 234 */ 235 236 if (reg_safe == -1) { 237 reg = reg_safe; 238 goto msr_fail; 239 } 240 241 /* 242 * Read the current value, change it and read it back to see if it 243 * matches, this is needed to detect certain hardware emulators 244 * (qemu/kvm) that don't trap on the MSR access and always return 0s. 245 */ 246 reg = x86_pmu_event_addr(reg_safe); 247 if (rdmsrl_safe(reg, &val)) 248 goto msr_fail; 249 val ^= 0xffffUL; 250 ret = wrmsrl_safe(reg, val); 251 ret |= rdmsrl_safe(reg, &val_new); 252 if (ret || val != val_new) 253 goto msr_fail; 254 255 /* 256 * We still allow the PMU driver to operate: 257 */ 258 if (bios_fail) { 259 pr_cont("Broken BIOS detected, complain to your hardware vendor.\n"); 260 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", 261 reg_fail, val_fail); 262 } 263 264 return true; 265 266 msr_fail: 267 if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) { 268 pr_cont("PMU not available due to virtualization, using software events only.\n"); 269 } else { 270 pr_cont("Broken PMU hardware detected, using software events only.\n"); 271 pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n", 272 reg, val_new); 273 } 274 275 return false; 276 } 277 278 static void hw_perf_event_destroy(struct perf_event *event) 279 { 280 x86_release_hardware(); 281 atomic_dec(&active_events); 282 } 283 284 void hw_perf_lbr_event_destroy(struct perf_event *event) 285 { 286 hw_perf_event_destroy(event); 287 288 /* undo the lbr/bts event accounting */ 289 x86_del_exclusive(x86_lbr_exclusive_lbr); 290 } 291 292 static inline int x86_pmu_initialized(void) 293 { 294 return x86_pmu.handle_irq != NULL; 295 } 296 297 static inline int 298 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event) 299 { 300 struct perf_event_attr *attr = &event->attr; 301 unsigned int cache_type, cache_op, cache_result; 302 u64 config, val; 303 304 config = attr->config; 305 306 cache_type = (config >> 0) & 0xff; 307 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 308 return -EINVAL; 309 310 cache_op = (config >> 8) & 0xff; 311 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 312 return -EINVAL; 313 314 cache_result = (config >> 16) & 0xff; 315 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 316 return -EINVAL; 317 318 val = hw_cache_event_ids[cache_type][cache_op][cache_result]; 319 320 if (val == 0) 321 return -ENOENT; 322 323 if (val == -1) 324 return -EINVAL; 325 326 hwc->config |= val; 327 attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result]; 328 return x86_pmu_extra_regs(val, event); 329 } 330 331 int x86_reserve_hardware(void) 332 { 333 int err = 0; 334 335 if (!atomic_inc_not_zero(&pmc_refcount)) { 336 mutex_lock(&pmc_reserve_mutex); 337 if (atomic_read(&pmc_refcount) == 0) { 338 if (!reserve_pmc_hardware()) 339 err = -EBUSY; 340 else 341 reserve_ds_buffers(); 342 } 343 if (!err) 344 atomic_inc(&pmc_refcount); 345 mutex_unlock(&pmc_reserve_mutex); 346 } 347 348 return err; 349 } 350 351 void x86_release_hardware(void) 352 { 353 if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) { 354 release_pmc_hardware(); 355 release_ds_buffers(); 356 mutex_unlock(&pmc_reserve_mutex); 357 } 358 } 359 360 /* 361 * Check if we can create event of a certain type (that no conflicting events 362 * are present). 363 */ 364 int x86_add_exclusive(unsigned int what) 365 { 366 int i; 367 368 if (x86_pmu.lbr_pt_coexist) 369 return 0; 370 371 if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) { 372 mutex_lock(&pmc_reserve_mutex); 373 for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) { 374 if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i])) 375 goto fail_unlock; 376 } 377 atomic_inc(&x86_pmu.lbr_exclusive[what]); 378 mutex_unlock(&pmc_reserve_mutex); 379 } 380 381 atomic_inc(&active_events); 382 return 0; 383 384 fail_unlock: 385 mutex_unlock(&pmc_reserve_mutex); 386 return -EBUSY; 387 } 388 389 void x86_del_exclusive(unsigned int what) 390 { 391 if (x86_pmu.lbr_pt_coexist) 392 return; 393 394 atomic_dec(&x86_pmu.lbr_exclusive[what]); 395 atomic_dec(&active_events); 396 } 397 398 int x86_setup_perfctr(struct perf_event *event) 399 { 400 struct perf_event_attr *attr = &event->attr; 401 struct hw_perf_event *hwc = &event->hw; 402 u64 config; 403 404 if (!is_sampling_event(event)) { 405 hwc->sample_period = x86_pmu.max_period; 406 hwc->last_period = hwc->sample_period; 407 local64_set(&hwc->period_left, hwc->sample_period); 408 } 409 410 if (attr->type == PERF_TYPE_RAW) 411 return x86_pmu_extra_regs(event->attr.config, event); 412 413 if (attr->type == PERF_TYPE_HW_CACHE) 414 return set_ext_hw_attr(hwc, event); 415 416 if (attr->config >= x86_pmu.max_events) 417 return -EINVAL; 418 419 /* 420 * The generic map: 421 */ 422 config = x86_pmu.event_map(attr->config); 423 424 if (config == 0) 425 return -ENOENT; 426 427 if (config == -1LL) 428 return -EINVAL; 429 430 /* 431 * Branch tracing: 432 */ 433 if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS && 434 !attr->freq && hwc->sample_period == 1) { 435 /* BTS is not supported by this architecture. */ 436 if (!x86_pmu.bts_active) 437 return -EOPNOTSUPP; 438 439 /* BTS is currently only allowed for user-mode. */ 440 if (!attr->exclude_kernel) 441 return -EOPNOTSUPP; 442 443 /* disallow bts if conflicting events are present */ 444 if (x86_add_exclusive(x86_lbr_exclusive_lbr)) 445 return -EBUSY; 446 447 event->destroy = hw_perf_lbr_event_destroy; 448 } 449 450 hwc->config |= config; 451 452 return 0; 453 } 454 455 /* 456 * check that branch_sample_type is compatible with 457 * settings needed for precise_ip > 1 which implies 458 * using the LBR to capture ALL taken branches at the 459 * priv levels of the measurement 460 */ 461 static inline int precise_br_compat(struct perf_event *event) 462 { 463 u64 m = event->attr.branch_sample_type; 464 u64 b = 0; 465 466 /* must capture all branches */ 467 if (!(m & PERF_SAMPLE_BRANCH_ANY)) 468 return 0; 469 470 m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER; 471 472 if (!event->attr.exclude_user) 473 b |= PERF_SAMPLE_BRANCH_USER; 474 475 if (!event->attr.exclude_kernel) 476 b |= PERF_SAMPLE_BRANCH_KERNEL; 477 478 /* 479 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86 480 */ 481 482 return m == b; 483 } 484 485 int x86_pmu_hw_config(struct perf_event *event) 486 { 487 if (event->attr.precise_ip) { 488 int precise = 0; 489 490 /* Support for constant skid */ 491 if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) { 492 precise++; 493 494 /* Support for IP fixup */ 495 if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2) 496 precise++; 497 498 if (x86_pmu.pebs_prec_dist) 499 precise++; 500 } 501 502 if (event->attr.precise_ip > precise) 503 return -EOPNOTSUPP; 504 } 505 /* 506 * check that PEBS LBR correction does not conflict with 507 * whatever the user is asking with attr->branch_sample_type 508 */ 509 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { 510 u64 *br_type = &event->attr.branch_sample_type; 511 512 if (has_branch_stack(event)) { 513 if (!precise_br_compat(event)) 514 return -EOPNOTSUPP; 515 516 /* branch_sample_type is compatible */ 517 518 } else { 519 /* 520 * user did not specify branch_sample_type 521 * 522 * For PEBS fixups, we capture all 523 * the branches at the priv level of the 524 * event. 525 */ 526 *br_type = PERF_SAMPLE_BRANCH_ANY; 527 528 if (!event->attr.exclude_user) 529 *br_type |= PERF_SAMPLE_BRANCH_USER; 530 531 if (!event->attr.exclude_kernel) 532 *br_type |= PERF_SAMPLE_BRANCH_KERNEL; 533 } 534 } 535 536 if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK) 537 event->attach_state |= PERF_ATTACH_TASK_DATA; 538 539 /* 540 * Generate PMC IRQs: 541 * (keep 'enabled' bit clear for now) 542 */ 543 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; 544 545 /* 546 * Count user and OS events unless requested not to 547 */ 548 if (!event->attr.exclude_user) 549 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; 550 if (!event->attr.exclude_kernel) 551 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; 552 553 if (event->attr.type == PERF_TYPE_RAW) 554 event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK; 555 556 if (event->attr.sample_period && x86_pmu.limit_period) { 557 if (x86_pmu.limit_period(event, event->attr.sample_period) > 558 event->attr.sample_period) 559 return -EINVAL; 560 } 561 562 return x86_setup_perfctr(event); 563 } 564 565 /* 566 * Setup the hardware configuration for a given attr_type 567 */ 568 static int __x86_pmu_event_init(struct perf_event *event) 569 { 570 int err; 571 572 if (!x86_pmu_initialized()) 573 return -ENODEV; 574 575 err = x86_reserve_hardware(); 576 if (err) 577 return err; 578 579 atomic_inc(&active_events); 580 event->destroy = hw_perf_event_destroy; 581 582 event->hw.idx = -1; 583 event->hw.last_cpu = -1; 584 event->hw.last_tag = ~0ULL; 585 586 /* mark unused */ 587 event->hw.extra_reg.idx = EXTRA_REG_NONE; 588 event->hw.branch_reg.idx = EXTRA_REG_NONE; 589 590 return x86_pmu.hw_config(event); 591 } 592 593 void x86_pmu_disable_all(void) 594 { 595 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 596 int idx; 597 598 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 599 u64 val; 600 601 if (!test_bit(idx, cpuc->active_mask)) 602 continue; 603 rdmsrl(x86_pmu_config_addr(idx), val); 604 if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE)) 605 continue; 606 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE; 607 wrmsrl(x86_pmu_config_addr(idx), val); 608 } 609 } 610 611 /* 612 * There may be PMI landing after enabled=0. The PMI hitting could be before or 613 * after disable_all. 614 * 615 * If PMI hits before disable_all, the PMU will be disabled in the NMI handler. 616 * It will not be re-enabled in the NMI handler again, because enabled=0. After 617 * handling the NMI, disable_all will be called, which will not change the 618 * state either. If PMI hits after disable_all, the PMU is already disabled 619 * before entering NMI handler. The NMI handler will not change the state 620 * either. 621 * 622 * So either situation is harmless. 623 */ 624 static void x86_pmu_disable(struct pmu *pmu) 625 { 626 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 627 628 if (!x86_pmu_initialized()) 629 return; 630 631 if (!cpuc->enabled) 632 return; 633 634 cpuc->n_added = 0; 635 cpuc->enabled = 0; 636 barrier(); 637 638 x86_pmu.disable_all(); 639 } 640 641 void x86_pmu_enable_all(int added) 642 { 643 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 644 int idx; 645 646 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 647 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; 648 649 if (!test_bit(idx, cpuc->active_mask)) 650 continue; 651 652 __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); 653 } 654 } 655 656 static struct pmu pmu; 657 658 static inline int is_x86_event(struct perf_event *event) 659 { 660 return event->pmu == &pmu; 661 } 662 663 /* 664 * Event scheduler state: 665 * 666 * Assign events iterating over all events and counters, beginning 667 * with events with least weights first. Keep the current iterator 668 * state in struct sched_state. 669 */ 670 struct sched_state { 671 int weight; 672 int event; /* event index */ 673 int counter; /* counter index */ 674 int unassigned; /* number of events to be assigned left */ 675 int nr_gp; /* number of GP counters used */ 676 unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 677 }; 678 679 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */ 680 #define SCHED_STATES_MAX 2 681 682 struct perf_sched { 683 int max_weight; 684 int max_events; 685 int max_gp; 686 int saved_states; 687 struct event_constraint **constraints; 688 struct sched_state state; 689 struct sched_state saved[SCHED_STATES_MAX]; 690 }; 691 692 /* 693 * Initialize interator that runs through all events and counters. 694 */ 695 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints, 696 int num, int wmin, int wmax, int gpmax) 697 { 698 int idx; 699 700 memset(sched, 0, sizeof(*sched)); 701 sched->max_events = num; 702 sched->max_weight = wmax; 703 sched->max_gp = gpmax; 704 sched->constraints = constraints; 705 706 for (idx = 0; idx < num; idx++) { 707 if (constraints[idx]->weight == wmin) 708 break; 709 } 710 711 sched->state.event = idx; /* start with min weight */ 712 sched->state.weight = wmin; 713 sched->state.unassigned = num; 714 } 715 716 static void perf_sched_save_state(struct perf_sched *sched) 717 { 718 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) 719 return; 720 721 sched->saved[sched->saved_states] = sched->state; 722 sched->saved_states++; 723 } 724 725 static bool perf_sched_restore_state(struct perf_sched *sched) 726 { 727 if (!sched->saved_states) 728 return false; 729 730 sched->saved_states--; 731 sched->state = sched->saved[sched->saved_states]; 732 733 /* continue with next counter: */ 734 clear_bit(sched->state.counter++, sched->state.used); 735 736 return true; 737 } 738 739 /* 740 * Select a counter for the current event to schedule. Return true on 741 * success. 742 */ 743 static bool __perf_sched_find_counter(struct perf_sched *sched) 744 { 745 struct event_constraint *c; 746 int idx; 747 748 if (!sched->state.unassigned) 749 return false; 750 751 if (sched->state.event >= sched->max_events) 752 return false; 753 754 c = sched->constraints[sched->state.event]; 755 /* Prefer fixed purpose counters */ 756 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { 757 idx = INTEL_PMC_IDX_FIXED; 758 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { 759 if (!__test_and_set_bit(idx, sched->state.used)) 760 goto done; 761 } 762 } 763 764 /* Grab the first unused counter starting with idx */ 765 idx = sched->state.counter; 766 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { 767 if (!__test_and_set_bit(idx, sched->state.used)) { 768 if (sched->state.nr_gp++ >= sched->max_gp) 769 return false; 770 771 goto done; 772 } 773 } 774 775 return false; 776 777 done: 778 sched->state.counter = idx; 779 780 if (c->overlap) 781 perf_sched_save_state(sched); 782 783 return true; 784 } 785 786 static bool perf_sched_find_counter(struct perf_sched *sched) 787 { 788 while (!__perf_sched_find_counter(sched)) { 789 if (!perf_sched_restore_state(sched)) 790 return false; 791 } 792 793 return true; 794 } 795 796 /* 797 * Go through all unassigned events and find the next one to schedule. 798 * Take events with the least weight first. Return true on success. 799 */ 800 static bool perf_sched_next_event(struct perf_sched *sched) 801 { 802 struct event_constraint *c; 803 804 if (!sched->state.unassigned || !--sched->state.unassigned) 805 return false; 806 807 do { 808 /* next event */ 809 sched->state.event++; 810 if (sched->state.event >= sched->max_events) { 811 /* next weight */ 812 sched->state.event = 0; 813 sched->state.weight++; 814 if (sched->state.weight > sched->max_weight) 815 return false; 816 } 817 c = sched->constraints[sched->state.event]; 818 } while (c->weight != sched->state.weight); 819 820 sched->state.counter = 0; /* start with first counter */ 821 822 return true; 823 } 824 825 /* 826 * Assign a counter for each event. 827 */ 828 int perf_assign_events(struct event_constraint **constraints, int n, 829 int wmin, int wmax, int gpmax, int *assign) 830 { 831 struct perf_sched sched; 832 833 perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax); 834 835 do { 836 if (!perf_sched_find_counter(&sched)) 837 break; /* failed */ 838 if (assign) 839 assign[sched.state.event] = sched.state.counter; 840 } while (perf_sched_next_event(&sched)); 841 842 return sched.state.unassigned; 843 } 844 EXPORT_SYMBOL_GPL(perf_assign_events); 845 846 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign) 847 { 848 struct event_constraint *c; 849 unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)]; 850 struct perf_event *e; 851 int i, wmin, wmax, unsched = 0; 852 struct hw_perf_event *hwc; 853 854 bitmap_zero(used_mask, X86_PMC_IDX_MAX); 855 856 if (x86_pmu.start_scheduling) 857 x86_pmu.start_scheduling(cpuc); 858 859 for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) { 860 cpuc->event_constraint[i] = NULL; 861 c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]); 862 cpuc->event_constraint[i] = c; 863 864 wmin = min(wmin, c->weight); 865 wmax = max(wmax, c->weight); 866 } 867 868 /* 869 * fastpath, try to reuse previous register 870 */ 871 for (i = 0; i < n; i++) { 872 hwc = &cpuc->event_list[i]->hw; 873 c = cpuc->event_constraint[i]; 874 875 /* never assigned */ 876 if (hwc->idx == -1) 877 break; 878 879 /* constraint still honored */ 880 if (!test_bit(hwc->idx, c->idxmsk)) 881 break; 882 883 /* not already used */ 884 if (test_bit(hwc->idx, used_mask)) 885 break; 886 887 __set_bit(hwc->idx, used_mask); 888 if (assign) 889 assign[i] = hwc->idx; 890 } 891 892 /* slow path */ 893 if (i != n) { 894 int gpmax = x86_pmu.num_counters; 895 896 /* 897 * Do not allow scheduling of more than half the available 898 * generic counters. 899 * 900 * This helps avoid counter starvation of sibling thread by 901 * ensuring at most half the counters cannot be in exclusive 902 * mode. There is no designated counters for the limits. Any 903 * N/2 counters can be used. This helps with events with 904 * specific counter constraints. 905 */ 906 if (is_ht_workaround_enabled() && !cpuc->is_fake && 907 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) 908 gpmax /= 2; 909 910 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, 911 wmax, gpmax, assign); 912 } 913 914 /* 915 * In case of success (unsched = 0), mark events as committed, 916 * so we do not put_constraint() in case new events are added 917 * and fail to be scheduled 918 * 919 * We invoke the lower level commit callback to lock the resource 920 * 921 * We do not need to do all of this in case we are called to 922 * validate an event group (assign == NULL) 923 */ 924 if (!unsched && assign) { 925 for (i = 0; i < n; i++) { 926 e = cpuc->event_list[i]; 927 e->hw.flags |= PERF_X86_EVENT_COMMITTED; 928 if (x86_pmu.commit_scheduling) 929 x86_pmu.commit_scheduling(cpuc, i, assign[i]); 930 } 931 } else { 932 for (i = 0; i < n; i++) { 933 e = cpuc->event_list[i]; 934 /* 935 * do not put_constraint() on comitted events, 936 * because they are good to go 937 */ 938 if ((e->hw.flags & PERF_X86_EVENT_COMMITTED)) 939 continue; 940 941 /* 942 * release events that failed scheduling 943 */ 944 if (x86_pmu.put_event_constraints) 945 x86_pmu.put_event_constraints(cpuc, e); 946 } 947 } 948 949 if (x86_pmu.stop_scheduling) 950 x86_pmu.stop_scheduling(cpuc); 951 952 return unsched ? -EINVAL : 0; 953 } 954 955 /* 956 * dogrp: true if must collect siblings events (group) 957 * returns total number of events and error code 958 */ 959 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp) 960 { 961 struct perf_event *event; 962 int n, max_count; 963 964 max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed; 965 966 /* current number of events already accepted */ 967 n = cpuc->n_events; 968 969 if (is_x86_event(leader)) { 970 if (n >= max_count) 971 return -EINVAL; 972 cpuc->event_list[n] = leader; 973 n++; 974 } 975 if (!dogrp) 976 return n; 977 978 list_for_each_entry(event, &leader->sibling_list, group_entry) { 979 if (!is_x86_event(event) || 980 event->state <= PERF_EVENT_STATE_OFF) 981 continue; 982 983 if (n >= max_count) 984 return -EINVAL; 985 986 cpuc->event_list[n] = event; 987 n++; 988 } 989 return n; 990 } 991 992 static inline void x86_assign_hw_event(struct perf_event *event, 993 struct cpu_hw_events *cpuc, int i) 994 { 995 struct hw_perf_event *hwc = &event->hw; 996 997 hwc->idx = cpuc->assign[i]; 998 hwc->last_cpu = smp_processor_id(); 999 hwc->last_tag = ++cpuc->tags[i]; 1000 1001 if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) { 1002 hwc->config_base = 0; 1003 hwc->event_base = 0; 1004 } else if (hwc->idx >= INTEL_PMC_IDX_FIXED) { 1005 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; 1006 hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED); 1007 hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30; 1008 } else { 1009 hwc->config_base = x86_pmu_config_addr(hwc->idx); 1010 hwc->event_base = x86_pmu_event_addr(hwc->idx); 1011 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); 1012 } 1013 } 1014 1015 static inline int match_prev_assignment(struct hw_perf_event *hwc, 1016 struct cpu_hw_events *cpuc, 1017 int i) 1018 { 1019 return hwc->idx == cpuc->assign[i] && 1020 hwc->last_cpu == smp_processor_id() && 1021 hwc->last_tag == cpuc->tags[i]; 1022 } 1023 1024 static void x86_pmu_start(struct perf_event *event, int flags); 1025 1026 static void x86_pmu_enable(struct pmu *pmu) 1027 { 1028 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1029 struct perf_event *event; 1030 struct hw_perf_event *hwc; 1031 int i, added = cpuc->n_added; 1032 1033 if (!x86_pmu_initialized()) 1034 return; 1035 1036 if (cpuc->enabled) 1037 return; 1038 1039 if (cpuc->n_added) { 1040 int n_running = cpuc->n_events - cpuc->n_added; 1041 /* 1042 * apply assignment obtained either from 1043 * hw_perf_group_sched_in() or x86_pmu_enable() 1044 * 1045 * step1: save events moving to new counters 1046 */ 1047 for (i = 0; i < n_running; i++) { 1048 event = cpuc->event_list[i]; 1049 hwc = &event->hw; 1050 1051 /* 1052 * we can avoid reprogramming counter if: 1053 * - assigned same counter as last time 1054 * - running on same CPU as last time 1055 * - no other event has used the counter since 1056 */ 1057 if (hwc->idx == -1 || 1058 match_prev_assignment(hwc, cpuc, i)) 1059 continue; 1060 1061 /* 1062 * Ensure we don't accidentally enable a stopped 1063 * counter simply because we rescheduled. 1064 */ 1065 if (hwc->state & PERF_HES_STOPPED) 1066 hwc->state |= PERF_HES_ARCH; 1067 1068 x86_pmu_stop(event, PERF_EF_UPDATE); 1069 } 1070 1071 /* 1072 * step2: reprogram moved events into new counters 1073 */ 1074 for (i = 0; i < cpuc->n_events; i++) { 1075 event = cpuc->event_list[i]; 1076 hwc = &event->hw; 1077 1078 if (!match_prev_assignment(hwc, cpuc, i)) 1079 x86_assign_hw_event(event, cpuc, i); 1080 else if (i < n_running) 1081 continue; 1082 1083 if (hwc->state & PERF_HES_ARCH) 1084 continue; 1085 1086 x86_pmu_start(event, PERF_EF_RELOAD); 1087 } 1088 cpuc->n_added = 0; 1089 perf_events_lapic_init(); 1090 } 1091 1092 cpuc->enabled = 1; 1093 barrier(); 1094 1095 x86_pmu.enable_all(added); 1096 } 1097 1098 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left); 1099 1100 /* 1101 * Set the next IRQ period, based on the hwc->period_left value. 1102 * To be called with the event disabled in hw: 1103 */ 1104 int x86_perf_event_set_period(struct perf_event *event) 1105 { 1106 struct hw_perf_event *hwc = &event->hw; 1107 s64 left = local64_read(&hwc->period_left); 1108 s64 period = hwc->sample_period; 1109 int ret = 0, idx = hwc->idx; 1110 1111 if (idx == INTEL_PMC_IDX_FIXED_BTS) 1112 return 0; 1113 1114 /* 1115 * If we are way outside a reasonable range then just skip forward: 1116 */ 1117 if (unlikely(left <= -period)) { 1118 left = period; 1119 local64_set(&hwc->period_left, left); 1120 hwc->last_period = period; 1121 ret = 1; 1122 } 1123 1124 if (unlikely(left <= 0)) { 1125 left += period; 1126 local64_set(&hwc->period_left, left); 1127 hwc->last_period = period; 1128 ret = 1; 1129 } 1130 /* 1131 * Quirk: certain CPUs dont like it if just 1 hw_event is left: 1132 */ 1133 if (unlikely(left < 2)) 1134 left = 2; 1135 1136 if (left > x86_pmu.max_period) 1137 left = x86_pmu.max_period; 1138 1139 if (x86_pmu.limit_period) 1140 left = x86_pmu.limit_period(event, left); 1141 1142 per_cpu(pmc_prev_left[idx], smp_processor_id()) = left; 1143 1144 if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) || 1145 local64_read(&hwc->prev_count) != (u64)-left) { 1146 /* 1147 * The hw event starts counting from this event offset, 1148 * mark it to be able to extra future deltas: 1149 */ 1150 local64_set(&hwc->prev_count, (u64)-left); 1151 1152 wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); 1153 } 1154 1155 /* 1156 * Due to erratum on certan cpu we need 1157 * a second write to be sure the register 1158 * is updated properly 1159 */ 1160 if (x86_pmu.perfctr_second_write) { 1161 wrmsrl(hwc->event_base, 1162 (u64)(-left) & x86_pmu.cntval_mask); 1163 } 1164 1165 perf_event_update_userpage(event); 1166 1167 return ret; 1168 } 1169 1170 void x86_pmu_enable_event(struct perf_event *event) 1171 { 1172 if (__this_cpu_read(cpu_hw_events.enabled)) 1173 __x86_pmu_enable_event(&event->hw, 1174 ARCH_PERFMON_EVENTSEL_ENABLE); 1175 } 1176 1177 /* 1178 * Add a single event to the PMU. 1179 * 1180 * The event is added to the group of enabled events 1181 * but only if it can be scehduled with existing events. 1182 */ 1183 static int x86_pmu_add(struct perf_event *event, int flags) 1184 { 1185 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1186 struct hw_perf_event *hwc; 1187 int assign[X86_PMC_IDX_MAX]; 1188 int n, n0, ret; 1189 1190 hwc = &event->hw; 1191 1192 n0 = cpuc->n_events; 1193 ret = n = collect_events(cpuc, event, false); 1194 if (ret < 0) 1195 goto out; 1196 1197 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1198 if (!(flags & PERF_EF_START)) 1199 hwc->state |= PERF_HES_ARCH; 1200 1201 /* 1202 * If group events scheduling transaction was started, 1203 * skip the schedulability test here, it will be performed 1204 * at commit time (->commit_txn) as a whole. 1205 * 1206 * If commit fails, we'll call ->del() on all events 1207 * for which ->add() was called. 1208 */ 1209 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1210 goto done_collect; 1211 1212 ret = x86_pmu.schedule_events(cpuc, n, assign); 1213 if (ret) 1214 goto out; 1215 /* 1216 * copy new assignment, now we know it is possible 1217 * will be used by hw_perf_enable() 1218 */ 1219 memcpy(cpuc->assign, assign, n*sizeof(int)); 1220 1221 done_collect: 1222 /* 1223 * Commit the collect_events() state. See x86_pmu_del() and 1224 * x86_pmu_*_txn(). 1225 */ 1226 cpuc->n_events = n; 1227 cpuc->n_added += n - n0; 1228 cpuc->n_txn += n - n0; 1229 1230 if (x86_pmu.add) { 1231 /* 1232 * This is before x86_pmu_enable() will call x86_pmu_start(), 1233 * so we enable LBRs before an event needs them etc.. 1234 */ 1235 x86_pmu.add(event); 1236 } 1237 1238 ret = 0; 1239 out: 1240 return ret; 1241 } 1242 1243 static void x86_pmu_start(struct perf_event *event, int flags) 1244 { 1245 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1246 int idx = event->hw.idx; 1247 1248 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 1249 return; 1250 1251 if (WARN_ON_ONCE(idx == -1)) 1252 return; 1253 1254 if (flags & PERF_EF_RELOAD) { 1255 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); 1256 x86_perf_event_set_period(event); 1257 } 1258 1259 event->hw.state = 0; 1260 1261 cpuc->events[idx] = event; 1262 __set_bit(idx, cpuc->active_mask); 1263 __set_bit(idx, cpuc->running); 1264 x86_pmu.enable(event); 1265 perf_event_update_userpage(event); 1266 } 1267 1268 void perf_event_print_debug(void) 1269 { 1270 u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed; 1271 u64 pebs, debugctl; 1272 struct cpu_hw_events *cpuc; 1273 unsigned long flags; 1274 int cpu, idx; 1275 1276 if (!x86_pmu.num_counters) 1277 return; 1278 1279 local_irq_save(flags); 1280 1281 cpu = smp_processor_id(); 1282 cpuc = &per_cpu(cpu_hw_events, cpu); 1283 1284 if (x86_pmu.version >= 2) { 1285 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl); 1286 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status); 1287 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow); 1288 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed); 1289 1290 pr_info("\n"); 1291 pr_info("CPU#%d: ctrl: %016llx\n", cpu, ctrl); 1292 pr_info("CPU#%d: status: %016llx\n", cpu, status); 1293 pr_info("CPU#%d: overflow: %016llx\n", cpu, overflow); 1294 pr_info("CPU#%d: fixed: %016llx\n", cpu, fixed); 1295 if (x86_pmu.pebs_constraints) { 1296 rdmsrl(MSR_IA32_PEBS_ENABLE, pebs); 1297 pr_info("CPU#%d: pebs: %016llx\n", cpu, pebs); 1298 } 1299 if (x86_pmu.lbr_nr) { 1300 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl); 1301 pr_info("CPU#%d: debugctl: %016llx\n", cpu, debugctl); 1302 } 1303 } 1304 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); 1305 1306 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1307 rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl); 1308 rdmsrl(x86_pmu_event_addr(idx), pmc_count); 1309 1310 prev_left = per_cpu(pmc_prev_left[idx], cpu); 1311 1312 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", 1313 cpu, idx, pmc_ctrl); 1314 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", 1315 cpu, idx, pmc_count); 1316 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", 1317 cpu, idx, prev_left); 1318 } 1319 for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) { 1320 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count); 1321 1322 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", 1323 cpu, idx, pmc_count); 1324 } 1325 local_irq_restore(flags); 1326 } 1327 1328 void x86_pmu_stop(struct perf_event *event, int flags) 1329 { 1330 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1331 struct hw_perf_event *hwc = &event->hw; 1332 1333 if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) { 1334 x86_pmu.disable(event); 1335 cpuc->events[hwc->idx] = NULL; 1336 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1337 hwc->state |= PERF_HES_STOPPED; 1338 } 1339 1340 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1341 /* 1342 * Drain the remaining delta count out of a event 1343 * that we are disabling: 1344 */ 1345 x86_perf_event_update(event); 1346 hwc->state |= PERF_HES_UPTODATE; 1347 } 1348 } 1349 1350 static void x86_pmu_del(struct perf_event *event, int flags) 1351 { 1352 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1353 int i; 1354 1355 /* 1356 * event is descheduled 1357 */ 1358 event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; 1359 1360 /* 1361 * If we're called during a txn, we only need to undo x86_pmu.add. 1362 * The events never got scheduled and ->cancel_txn will truncate 1363 * the event_list. 1364 * 1365 * XXX assumes any ->del() called during a TXN will only be on 1366 * an event added during that same TXN. 1367 */ 1368 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) 1369 goto do_del; 1370 1371 /* 1372 * Not a TXN, therefore cleanup properly. 1373 */ 1374 x86_pmu_stop(event, PERF_EF_UPDATE); 1375 1376 for (i = 0; i < cpuc->n_events; i++) { 1377 if (event == cpuc->event_list[i]) 1378 break; 1379 } 1380 1381 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ 1382 return; 1383 1384 /* If we have a newly added event; make sure to decrease n_added. */ 1385 if (i >= cpuc->n_events - cpuc->n_added) 1386 --cpuc->n_added; 1387 1388 if (x86_pmu.put_event_constraints) 1389 x86_pmu.put_event_constraints(cpuc, event); 1390 1391 /* Delete the array entry. */ 1392 while (++i < cpuc->n_events) { 1393 cpuc->event_list[i-1] = cpuc->event_list[i]; 1394 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; 1395 } 1396 --cpuc->n_events; 1397 1398 perf_event_update_userpage(event); 1399 1400 do_del: 1401 if (x86_pmu.del) { 1402 /* 1403 * This is after x86_pmu_stop(); so we disable LBRs after any 1404 * event can need them etc.. 1405 */ 1406 x86_pmu.del(event); 1407 } 1408 } 1409 1410 int x86_pmu_handle_irq(struct pt_regs *regs) 1411 { 1412 struct perf_sample_data data; 1413 struct cpu_hw_events *cpuc; 1414 struct perf_event *event; 1415 int idx, handled = 0; 1416 u64 val; 1417 1418 cpuc = this_cpu_ptr(&cpu_hw_events); 1419 1420 /* 1421 * Some chipsets need to unmask the LVTPC in a particular spot 1422 * inside the nmi handler. As a result, the unmasking was pushed 1423 * into all the nmi handlers. 1424 * 1425 * This generic handler doesn't seem to have any issues where the 1426 * unmasking occurs so it was left at the top. 1427 */ 1428 apic_write(APIC_LVTPC, APIC_DM_NMI); 1429 1430 for (idx = 0; idx < x86_pmu.num_counters; idx++) { 1431 if (!test_bit(idx, cpuc->active_mask)) { 1432 /* 1433 * Though we deactivated the counter some cpus 1434 * might still deliver spurious interrupts still 1435 * in flight. Catch them: 1436 */ 1437 if (__test_and_clear_bit(idx, cpuc->running)) 1438 handled++; 1439 continue; 1440 } 1441 1442 event = cpuc->events[idx]; 1443 1444 val = x86_perf_event_update(event); 1445 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) 1446 continue; 1447 1448 /* 1449 * event overflow 1450 */ 1451 handled++; 1452 perf_sample_data_init(&data, 0, event->hw.last_period); 1453 1454 if (!x86_perf_event_set_period(event)) 1455 continue; 1456 1457 if (perf_event_overflow(event, &data, regs)) 1458 x86_pmu_stop(event, 0); 1459 } 1460 1461 if (handled) 1462 inc_irq_stat(apic_perf_irqs); 1463 1464 return handled; 1465 } 1466 1467 void perf_events_lapic_init(void) 1468 { 1469 if (!x86_pmu.apic || !x86_pmu_initialized()) 1470 return; 1471 1472 /* 1473 * Always use NMI for PMU 1474 */ 1475 apic_write(APIC_LVTPC, APIC_DM_NMI); 1476 } 1477 1478 static int 1479 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1480 { 1481 u64 start_clock; 1482 u64 finish_clock; 1483 int ret; 1484 1485 /* 1486 * All PMUs/events that share this PMI handler should make sure to 1487 * increment active_events for their events. 1488 */ 1489 if (!atomic_read(&active_events)) 1490 return NMI_DONE; 1491 1492 start_clock = sched_clock(); 1493 ret = x86_pmu.handle_irq(regs); 1494 finish_clock = sched_clock(); 1495 1496 perf_sample_event_took(finish_clock - start_clock); 1497 1498 return ret; 1499 } 1500 NOKPROBE_SYMBOL(perf_event_nmi_handler); 1501 1502 struct event_constraint emptyconstraint; 1503 struct event_constraint unconstrained; 1504 1505 static int x86_pmu_prepare_cpu(unsigned int cpu) 1506 { 1507 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1508 int i; 1509 1510 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) 1511 cpuc->kfree_on_online[i] = NULL; 1512 if (x86_pmu.cpu_prepare) 1513 return x86_pmu.cpu_prepare(cpu); 1514 return 0; 1515 } 1516 1517 static int x86_pmu_dead_cpu(unsigned int cpu) 1518 { 1519 if (x86_pmu.cpu_dead) 1520 x86_pmu.cpu_dead(cpu); 1521 return 0; 1522 } 1523 1524 static int x86_pmu_online_cpu(unsigned int cpu) 1525 { 1526 struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); 1527 int i; 1528 1529 for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) { 1530 kfree(cpuc->kfree_on_online[i]); 1531 cpuc->kfree_on_online[i] = NULL; 1532 } 1533 return 0; 1534 } 1535 1536 static int x86_pmu_starting_cpu(unsigned int cpu) 1537 { 1538 if (x86_pmu.cpu_starting) 1539 x86_pmu.cpu_starting(cpu); 1540 return 0; 1541 } 1542 1543 static int x86_pmu_dying_cpu(unsigned int cpu) 1544 { 1545 if (x86_pmu.cpu_dying) 1546 x86_pmu.cpu_dying(cpu); 1547 return 0; 1548 } 1549 1550 static void __init pmu_check_apic(void) 1551 { 1552 if (boot_cpu_has(X86_FEATURE_APIC)) 1553 return; 1554 1555 x86_pmu.apic = 0; 1556 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1557 pr_info("no hardware sampling interrupt available.\n"); 1558 1559 /* 1560 * If we have a PMU initialized but no APIC 1561 * interrupts, we cannot sample hardware 1562 * events (user-space has to fall back and 1563 * sample via a hrtimer based software event): 1564 */ 1565 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; 1566 1567 } 1568 1569 static struct attribute_group x86_pmu_format_group = { 1570 .name = "format", 1571 .attrs = NULL, 1572 }; 1573 1574 /* 1575 * Remove all undefined events (x86_pmu.event_map(id) == 0) 1576 * out of events_attr attributes. 1577 */ 1578 static void __init filter_events(struct attribute **attrs) 1579 { 1580 struct device_attribute *d; 1581 struct perf_pmu_events_attr *pmu_attr; 1582 int offset = 0; 1583 int i, j; 1584 1585 for (i = 0; attrs[i]; i++) { 1586 d = (struct device_attribute *)attrs[i]; 1587 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr); 1588 /* str trumps id */ 1589 if (pmu_attr->event_str) 1590 continue; 1591 if (x86_pmu.event_map(i + offset)) 1592 continue; 1593 1594 for (j = i; attrs[j]; j++) 1595 attrs[j] = attrs[j + 1]; 1596 1597 /* Check the shifted attr. */ 1598 i--; 1599 1600 /* 1601 * event_map() is index based, the attrs array is organized 1602 * by increasing event index. If we shift the events, then 1603 * we need to compensate for the event_map(), otherwise 1604 * we are looking up the wrong event in the map 1605 */ 1606 offset++; 1607 } 1608 } 1609 1610 /* Merge two pointer arrays */ 1611 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b) 1612 { 1613 struct attribute **new; 1614 int j, i; 1615 1616 for (j = 0; a[j]; j++) 1617 ; 1618 for (i = 0; b[i]; i++) 1619 j++; 1620 j++; 1621 1622 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL); 1623 if (!new) 1624 return NULL; 1625 1626 j = 0; 1627 for (i = 0; a[i]; i++) 1628 new[j++] = a[i]; 1629 for (i = 0; b[i]; i++) 1630 new[j++] = b[i]; 1631 new[j] = NULL; 1632 1633 return new; 1634 } 1635 1636 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page) 1637 { 1638 struct perf_pmu_events_attr *pmu_attr = \ 1639 container_of(attr, struct perf_pmu_events_attr, attr); 1640 u64 config = x86_pmu.event_map(pmu_attr->id); 1641 1642 /* string trumps id */ 1643 if (pmu_attr->event_str) 1644 return sprintf(page, "%s", pmu_attr->event_str); 1645 1646 return x86_pmu.events_sysfs_show(page, config); 1647 } 1648 EXPORT_SYMBOL_GPL(events_sysfs_show); 1649 1650 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr, 1651 char *page) 1652 { 1653 struct perf_pmu_events_ht_attr *pmu_attr = 1654 container_of(attr, struct perf_pmu_events_ht_attr, attr); 1655 1656 /* 1657 * Report conditional events depending on Hyper-Threading. 1658 * 1659 * This is overly conservative as usually the HT special 1660 * handling is not needed if the other CPU thread is idle. 1661 * 1662 * Note this does not (and cannot) handle the case when thread 1663 * siblings are invisible, for example with virtualization 1664 * if they are owned by some other guest. The user tool 1665 * has to re-read when a thread sibling gets onlined later. 1666 */ 1667 return sprintf(page, "%s", 1668 topology_max_smt_threads() > 1 ? 1669 pmu_attr->event_str_ht : 1670 pmu_attr->event_str_noht); 1671 } 1672 1673 EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1674 EVENT_ATTR(instructions, INSTRUCTIONS ); 1675 EVENT_ATTR(cache-references, CACHE_REFERENCES ); 1676 EVENT_ATTR(cache-misses, CACHE_MISSES ); 1677 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS ); 1678 EVENT_ATTR(branch-misses, BRANCH_MISSES ); 1679 EVENT_ATTR(bus-cycles, BUS_CYCLES ); 1680 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND ); 1681 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND ); 1682 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES ); 1683 1684 static struct attribute *empty_attrs; 1685 1686 static struct attribute *events_attr[] = { 1687 EVENT_PTR(CPU_CYCLES), 1688 EVENT_PTR(INSTRUCTIONS), 1689 EVENT_PTR(CACHE_REFERENCES), 1690 EVENT_PTR(CACHE_MISSES), 1691 EVENT_PTR(BRANCH_INSTRUCTIONS), 1692 EVENT_PTR(BRANCH_MISSES), 1693 EVENT_PTR(BUS_CYCLES), 1694 EVENT_PTR(STALLED_CYCLES_FRONTEND), 1695 EVENT_PTR(STALLED_CYCLES_BACKEND), 1696 EVENT_PTR(REF_CPU_CYCLES), 1697 NULL, 1698 }; 1699 1700 static struct attribute_group x86_pmu_events_group = { 1701 .name = "events", 1702 .attrs = events_attr, 1703 }; 1704 1705 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event) 1706 { 1707 u64 umask = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8; 1708 u64 cmask = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24; 1709 bool edge = (config & ARCH_PERFMON_EVENTSEL_EDGE); 1710 bool pc = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL); 1711 bool any = (config & ARCH_PERFMON_EVENTSEL_ANY); 1712 bool inv = (config & ARCH_PERFMON_EVENTSEL_INV); 1713 ssize_t ret; 1714 1715 /* 1716 * We have whole page size to spend and just little data 1717 * to write, so we can safely use sprintf. 1718 */ 1719 ret = sprintf(page, "event=0x%02llx", event); 1720 1721 if (umask) 1722 ret += sprintf(page + ret, ",umask=0x%02llx", umask); 1723 1724 if (edge) 1725 ret += sprintf(page + ret, ",edge"); 1726 1727 if (pc) 1728 ret += sprintf(page + ret, ",pc"); 1729 1730 if (any) 1731 ret += sprintf(page + ret, ",any"); 1732 1733 if (inv) 1734 ret += sprintf(page + ret, ",inv"); 1735 1736 if (cmask) 1737 ret += sprintf(page + ret, ",cmask=0x%02llx", cmask); 1738 1739 ret += sprintf(page + ret, "\n"); 1740 1741 return ret; 1742 } 1743 1744 static int __init init_hw_perf_events(void) 1745 { 1746 struct x86_pmu_quirk *quirk; 1747 int err; 1748 1749 pr_info("Performance Events: "); 1750 1751 switch (boot_cpu_data.x86_vendor) { 1752 case X86_VENDOR_INTEL: 1753 err = intel_pmu_init(); 1754 break; 1755 case X86_VENDOR_AMD: 1756 err = amd_pmu_init(); 1757 break; 1758 default: 1759 err = -ENOTSUPP; 1760 } 1761 if (err != 0) { 1762 pr_cont("no PMU driver, software events only.\n"); 1763 return 0; 1764 } 1765 1766 pmu_check_apic(); 1767 1768 /* sanity check that the hardware exists or is emulated */ 1769 if (!check_hw_exists()) 1770 return 0; 1771 1772 pr_cont("%s PMU driver.\n", x86_pmu.name); 1773 1774 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1775 1776 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) 1777 quirk->func(); 1778 1779 if (!x86_pmu.intel_ctrl) 1780 x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1; 1781 1782 perf_events_lapic_init(); 1783 register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI"); 1784 1785 unconstrained = (struct event_constraint) 1786 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1787 0, x86_pmu.num_counters, 0, 0); 1788 1789 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1790 1791 if (x86_pmu.event_attrs) 1792 x86_pmu_events_group.attrs = x86_pmu.event_attrs; 1793 1794 if (!x86_pmu.events_sysfs_show) 1795 x86_pmu_events_group.attrs = &empty_attrs; 1796 else 1797 filter_events(x86_pmu_events_group.attrs); 1798 1799 if (x86_pmu.cpu_events) { 1800 struct attribute **tmp; 1801 1802 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events); 1803 if (!WARN_ON(!tmp)) 1804 x86_pmu_events_group.attrs = tmp; 1805 } 1806 1807 pr_info("... version: %d\n", x86_pmu.version); 1808 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1809 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1810 pr_info("... value mask: %016Lx\n", x86_pmu.cntval_mask); 1811 pr_info("... max period: %016Lx\n", x86_pmu.max_period); 1812 pr_info("... fixed-purpose events: %d\n", x86_pmu.num_counters_fixed); 1813 pr_info("... event mask: %016Lx\n", x86_pmu.intel_ctrl); 1814 1815 /* 1816 * Install callbacks. Core will call them for each online 1817 * cpu. 1818 */ 1819 err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "PERF_X86_PREPARE", 1820 x86_pmu_prepare_cpu, x86_pmu_dead_cpu); 1821 if (err) 1822 return err; 1823 1824 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, 1825 "AP_PERF_X86_STARTING", x86_pmu_starting_cpu, 1826 x86_pmu_dying_cpu); 1827 if (err) 1828 goto out; 1829 1830 err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "AP_PERF_X86_ONLINE", 1831 x86_pmu_online_cpu, NULL); 1832 if (err) 1833 goto out1; 1834 1835 err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 1836 if (err) 1837 goto out2; 1838 1839 return 0; 1840 1841 out2: 1842 cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE); 1843 out1: 1844 cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); 1845 out: 1846 cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); 1847 return err; 1848 } 1849 early_initcall(init_hw_perf_events); 1850 1851 static inline void x86_pmu_read(struct perf_event *event) 1852 { 1853 x86_perf_event_update(event); 1854 } 1855 1856 /* 1857 * Start group events scheduling transaction 1858 * Set the flag to make pmu::enable() not perform the 1859 * schedulability test, it will be performed at commit time 1860 * 1861 * We only support PERF_PMU_TXN_ADD transactions. Save the 1862 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 1863 * transactions. 1864 */ 1865 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 1866 { 1867 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1868 1869 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ 1870 1871 cpuc->txn_flags = txn_flags; 1872 if (txn_flags & ~PERF_PMU_TXN_ADD) 1873 return; 1874 1875 perf_pmu_disable(pmu); 1876 __this_cpu_write(cpu_hw_events.n_txn, 0); 1877 } 1878 1879 /* 1880 * Stop group events scheduling transaction 1881 * Clear the flag and pmu::enable() will perform the 1882 * schedulability test. 1883 */ 1884 static void x86_pmu_cancel_txn(struct pmu *pmu) 1885 { 1886 unsigned int txn_flags; 1887 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1888 1889 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1890 1891 txn_flags = cpuc->txn_flags; 1892 cpuc->txn_flags = 0; 1893 if (txn_flags & ~PERF_PMU_TXN_ADD) 1894 return; 1895 1896 /* 1897 * Truncate collected array by the number of events added in this 1898 * transaction. See x86_pmu_add() and x86_pmu_*_txn(). 1899 */ 1900 __this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn)); 1901 __this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn)); 1902 perf_pmu_enable(pmu); 1903 } 1904 1905 /* 1906 * Commit group events scheduling transaction 1907 * Perform the group schedulability test as a whole 1908 * Return 0 if success 1909 * 1910 * Does not cancel the transaction on failure; expects the caller to do this. 1911 */ 1912 static int x86_pmu_commit_txn(struct pmu *pmu) 1913 { 1914 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 1915 int assign[X86_PMC_IDX_MAX]; 1916 int n, ret; 1917 1918 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ 1919 1920 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { 1921 cpuc->txn_flags = 0; 1922 return 0; 1923 } 1924 1925 n = cpuc->n_events; 1926 1927 if (!x86_pmu_initialized()) 1928 return -EAGAIN; 1929 1930 ret = x86_pmu.schedule_events(cpuc, n, assign); 1931 if (ret) 1932 return ret; 1933 1934 /* 1935 * copy new assignment, now we know it is possible 1936 * will be used by hw_perf_enable() 1937 */ 1938 memcpy(cpuc->assign, assign, n*sizeof(int)); 1939 1940 cpuc->txn_flags = 0; 1941 perf_pmu_enable(pmu); 1942 return 0; 1943 } 1944 /* 1945 * a fake_cpuc is used to validate event groups. Due to 1946 * the extra reg logic, we need to also allocate a fake 1947 * per_core and per_cpu structure. Otherwise, group events 1948 * using extra reg may conflict without the kernel being 1949 * able to catch this when the last event gets added to 1950 * the group. 1951 */ 1952 static void free_fake_cpuc(struct cpu_hw_events *cpuc) 1953 { 1954 kfree(cpuc->shared_regs); 1955 kfree(cpuc); 1956 } 1957 1958 static struct cpu_hw_events *allocate_fake_cpuc(void) 1959 { 1960 struct cpu_hw_events *cpuc; 1961 int cpu = raw_smp_processor_id(); 1962 1963 cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL); 1964 if (!cpuc) 1965 return ERR_PTR(-ENOMEM); 1966 1967 /* only needed, if we have extra_regs */ 1968 if (x86_pmu.extra_regs) { 1969 cpuc->shared_regs = allocate_shared_regs(cpu); 1970 if (!cpuc->shared_regs) 1971 goto error; 1972 } 1973 cpuc->is_fake = 1; 1974 return cpuc; 1975 error: 1976 free_fake_cpuc(cpuc); 1977 return ERR_PTR(-ENOMEM); 1978 } 1979 1980 /* 1981 * validate that we can schedule this event 1982 */ 1983 static int validate_event(struct perf_event *event) 1984 { 1985 struct cpu_hw_events *fake_cpuc; 1986 struct event_constraint *c; 1987 int ret = 0; 1988 1989 fake_cpuc = allocate_fake_cpuc(); 1990 if (IS_ERR(fake_cpuc)) 1991 return PTR_ERR(fake_cpuc); 1992 1993 c = x86_pmu.get_event_constraints(fake_cpuc, -1, event); 1994 1995 if (!c || !c->weight) 1996 ret = -EINVAL; 1997 1998 if (x86_pmu.put_event_constraints) 1999 x86_pmu.put_event_constraints(fake_cpuc, event); 2000 2001 free_fake_cpuc(fake_cpuc); 2002 2003 return ret; 2004 } 2005 2006 /* 2007 * validate a single event group 2008 * 2009 * validation include: 2010 * - check events are compatible which each other 2011 * - events do not compete for the same counter 2012 * - number of events <= number of counters 2013 * 2014 * validation ensures the group can be loaded onto the 2015 * PMU if it was the only group available. 2016 */ 2017 static int validate_group(struct perf_event *event) 2018 { 2019 struct perf_event *leader = event->group_leader; 2020 struct cpu_hw_events *fake_cpuc; 2021 int ret = -EINVAL, n; 2022 2023 fake_cpuc = allocate_fake_cpuc(); 2024 if (IS_ERR(fake_cpuc)) 2025 return PTR_ERR(fake_cpuc); 2026 /* 2027 * the event is not yet connected with its 2028 * siblings therefore we must first collect 2029 * existing siblings, then add the new event 2030 * before we can simulate the scheduling 2031 */ 2032 n = collect_events(fake_cpuc, leader, true); 2033 if (n < 0) 2034 goto out; 2035 2036 fake_cpuc->n_events = n; 2037 n = collect_events(fake_cpuc, event, false); 2038 if (n < 0) 2039 goto out; 2040 2041 fake_cpuc->n_events = n; 2042 2043 ret = x86_pmu.schedule_events(fake_cpuc, n, NULL); 2044 2045 out: 2046 free_fake_cpuc(fake_cpuc); 2047 return ret; 2048 } 2049 2050 static int x86_pmu_event_init(struct perf_event *event) 2051 { 2052 struct pmu *tmp; 2053 int err; 2054 2055 switch (event->attr.type) { 2056 case PERF_TYPE_RAW: 2057 case PERF_TYPE_HARDWARE: 2058 case PERF_TYPE_HW_CACHE: 2059 break; 2060 2061 default: 2062 return -ENOENT; 2063 } 2064 2065 err = __x86_pmu_event_init(event); 2066 if (!err) { 2067 /* 2068 * we temporarily connect event to its pmu 2069 * such that validate_group() can classify 2070 * it as an x86 event using is_x86_event() 2071 */ 2072 tmp = event->pmu; 2073 event->pmu = &pmu; 2074 2075 if (event->group_leader != event) 2076 err = validate_group(event); 2077 else 2078 err = validate_event(event); 2079 2080 event->pmu = tmp; 2081 } 2082 if (err) { 2083 if (event->destroy) 2084 event->destroy(event); 2085 } 2086 2087 if (ACCESS_ONCE(x86_pmu.attr_rdpmc)) 2088 event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED; 2089 2090 return err; 2091 } 2092 2093 static void refresh_pce(void *ignored) 2094 { 2095 if (current->mm) 2096 load_mm_cr4(current->mm); 2097 } 2098 2099 static void x86_pmu_event_mapped(struct perf_event *event) 2100 { 2101 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2102 return; 2103 2104 if (atomic_inc_return(¤t->mm->context.perf_rdpmc_allowed) == 1) 2105 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1); 2106 } 2107 2108 static void x86_pmu_event_unmapped(struct perf_event *event) 2109 { 2110 if (!current->mm) 2111 return; 2112 2113 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2114 return; 2115 2116 if (atomic_dec_and_test(¤t->mm->context.perf_rdpmc_allowed)) 2117 on_each_cpu_mask(mm_cpumask(current->mm), refresh_pce, NULL, 1); 2118 } 2119 2120 static int x86_pmu_event_idx(struct perf_event *event) 2121 { 2122 int idx = event->hw.idx; 2123 2124 if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED)) 2125 return 0; 2126 2127 if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) { 2128 idx -= INTEL_PMC_IDX_FIXED; 2129 idx |= 1 << 30; 2130 } 2131 2132 return idx + 1; 2133 } 2134 2135 static ssize_t get_attr_rdpmc(struct device *cdev, 2136 struct device_attribute *attr, 2137 char *buf) 2138 { 2139 return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc); 2140 } 2141 2142 static ssize_t set_attr_rdpmc(struct device *cdev, 2143 struct device_attribute *attr, 2144 const char *buf, size_t count) 2145 { 2146 unsigned long val; 2147 ssize_t ret; 2148 2149 ret = kstrtoul(buf, 0, &val); 2150 if (ret) 2151 return ret; 2152 2153 if (val > 2) 2154 return -EINVAL; 2155 2156 if (x86_pmu.attr_rdpmc_broken) 2157 return -ENOTSUPP; 2158 2159 if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) { 2160 /* 2161 * Changing into or out of always available, aka 2162 * perf-event-bypassing mode. This path is extremely slow, 2163 * but only root can trigger it, so it's okay. 2164 */ 2165 if (val == 2) 2166 static_key_slow_inc(&rdpmc_always_available); 2167 else 2168 static_key_slow_dec(&rdpmc_always_available); 2169 on_each_cpu(refresh_pce, NULL, 1); 2170 } 2171 2172 x86_pmu.attr_rdpmc = val; 2173 2174 return count; 2175 } 2176 2177 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc); 2178 2179 static struct attribute *x86_pmu_attrs[] = { 2180 &dev_attr_rdpmc.attr, 2181 NULL, 2182 }; 2183 2184 static struct attribute_group x86_pmu_attr_group = { 2185 .attrs = x86_pmu_attrs, 2186 }; 2187 2188 static const struct attribute_group *x86_pmu_attr_groups[] = { 2189 &x86_pmu_attr_group, 2190 &x86_pmu_format_group, 2191 &x86_pmu_events_group, 2192 NULL, 2193 }; 2194 2195 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) 2196 { 2197 if (x86_pmu.sched_task) 2198 x86_pmu.sched_task(ctx, sched_in); 2199 } 2200 2201 void perf_check_microcode(void) 2202 { 2203 if (x86_pmu.check_microcode) 2204 x86_pmu.check_microcode(); 2205 } 2206 EXPORT_SYMBOL_GPL(perf_check_microcode); 2207 2208 static struct pmu pmu = { 2209 .pmu_enable = x86_pmu_enable, 2210 .pmu_disable = x86_pmu_disable, 2211 2212 .attr_groups = x86_pmu_attr_groups, 2213 2214 .event_init = x86_pmu_event_init, 2215 2216 .event_mapped = x86_pmu_event_mapped, 2217 .event_unmapped = x86_pmu_event_unmapped, 2218 2219 .add = x86_pmu_add, 2220 .del = x86_pmu_del, 2221 .start = x86_pmu_start, 2222 .stop = x86_pmu_stop, 2223 .read = x86_pmu_read, 2224 2225 .start_txn = x86_pmu_start_txn, 2226 .cancel_txn = x86_pmu_cancel_txn, 2227 .commit_txn = x86_pmu_commit_txn, 2228 2229 .event_idx = x86_pmu_event_idx, 2230 .sched_task = x86_pmu_sched_task, 2231 .task_ctx_size = sizeof(struct x86_perf_task_context), 2232 }; 2233 2234 void arch_perf_update_userpage(struct perf_event *event, 2235 struct perf_event_mmap_page *userpg, u64 now) 2236 { 2237 struct cyc2ns_data *data; 2238 2239 userpg->cap_user_time = 0; 2240 userpg->cap_user_time_zero = 0; 2241 userpg->cap_user_rdpmc = 2242 !!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED); 2243 userpg->pmc_width = x86_pmu.cntval_bits; 2244 2245 if (!sched_clock_stable()) 2246 return; 2247 2248 data = cyc2ns_read_begin(); 2249 2250 /* 2251 * Internal timekeeping for enabled/running/stopped times 2252 * is always in the local_clock domain. 2253 */ 2254 userpg->cap_user_time = 1; 2255 userpg->time_mult = data->cyc2ns_mul; 2256 userpg->time_shift = data->cyc2ns_shift; 2257 userpg->time_offset = data->cyc2ns_offset - now; 2258 2259 /* 2260 * cap_user_time_zero doesn't make sense when we're using a different 2261 * time base for the records. 2262 */ 2263 if (!event->attr.use_clockid) { 2264 userpg->cap_user_time_zero = 1; 2265 userpg->time_zero = data->cyc2ns_offset; 2266 } 2267 2268 cyc2ns_read_end(data); 2269 } 2270 2271 void 2272 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2273 { 2274 struct unwind_state state; 2275 unsigned long addr; 2276 2277 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2278 /* TODO: We don't support guest os callchain now */ 2279 return; 2280 } 2281 2282 if (perf_callchain_store(entry, regs->ip)) 2283 return; 2284 2285 for (unwind_start(&state, current, regs, NULL); !unwind_done(&state); 2286 unwind_next_frame(&state)) { 2287 addr = unwind_get_return_address(&state); 2288 if (!addr || perf_callchain_store(entry, addr)) 2289 return; 2290 } 2291 } 2292 2293 static inline int 2294 valid_user_frame(const void __user *fp, unsigned long size) 2295 { 2296 return (__range_not_ok(fp, size, TASK_SIZE) == 0); 2297 } 2298 2299 static unsigned long get_segment_base(unsigned int segment) 2300 { 2301 struct desc_struct *desc; 2302 int idx = segment >> 3; 2303 2304 if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) { 2305 #ifdef CONFIG_MODIFY_LDT_SYSCALL 2306 struct ldt_struct *ldt; 2307 2308 if (idx > LDT_ENTRIES) 2309 return 0; 2310 2311 /* IRQs are off, so this synchronizes with smp_store_release */ 2312 ldt = lockless_dereference(current->active_mm->context.ldt); 2313 if (!ldt || idx > ldt->size) 2314 return 0; 2315 2316 desc = &ldt->entries[idx]; 2317 #else 2318 return 0; 2319 #endif 2320 } else { 2321 if (idx > GDT_ENTRIES) 2322 return 0; 2323 2324 desc = raw_cpu_ptr(gdt_page.gdt) + idx; 2325 } 2326 2327 return get_desc_base(desc); 2328 } 2329 2330 #ifdef CONFIG_IA32_EMULATION 2331 2332 #include <asm/compat.h> 2333 2334 static inline int 2335 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2336 { 2337 /* 32-bit process in 64-bit kernel. */ 2338 unsigned long ss_base, cs_base; 2339 struct stack_frame_ia32 frame; 2340 const void __user *fp; 2341 2342 if (!test_thread_flag(TIF_IA32)) 2343 return 0; 2344 2345 cs_base = get_segment_base(regs->cs); 2346 ss_base = get_segment_base(regs->ss); 2347 2348 fp = compat_ptr(ss_base + regs->bp); 2349 pagefault_disable(); 2350 while (entry->nr < entry->max_stack) { 2351 unsigned long bytes; 2352 frame.next_frame = 0; 2353 frame.return_address = 0; 2354 2355 if (!access_ok(VERIFY_READ, fp, 8)) 2356 break; 2357 2358 bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4); 2359 if (bytes != 0) 2360 break; 2361 bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4); 2362 if (bytes != 0) 2363 break; 2364 2365 if (!valid_user_frame(fp, sizeof(frame))) 2366 break; 2367 2368 perf_callchain_store(entry, cs_base + frame.return_address); 2369 fp = compat_ptr(ss_base + frame.next_frame); 2370 } 2371 pagefault_enable(); 2372 return 1; 2373 } 2374 #else 2375 static inline int 2376 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry) 2377 { 2378 return 0; 2379 } 2380 #endif 2381 2382 void 2383 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) 2384 { 2385 struct stack_frame frame; 2386 const unsigned long __user *fp; 2387 2388 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2389 /* TODO: We don't support guest os callchain now */ 2390 return; 2391 } 2392 2393 /* 2394 * We don't know what to do with VM86 stacks.. ignore them for now. 2395 */ 2396 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) 2397 return; 2398 2399 fp = (unsigned long __user *)regs->bp; 2400 2401 perf_callchain_store(entry, regs->ip); 2402 2403 if (!current->mm) 2404 return; 2405 2406 if (perf_callchain_user32(regs, entry)) 2407 return; 2408 2409 pagefault_disable(); 2410 while (entry->nr < entry->max_stack) { 2411 unsigned long bytes; 2412 2413 frame.next_frame = NULL; 2414 frame.return_address = 0; 2415 2416 if (!access_ok(VERIFY_READ, fp, sizeof(*fp) * 2)) 2417 break; 2418 2419 bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp)); 2420 if (bytes != 0) 2421 break; 2422 bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp)); 2423 if (bytes != 0) 2424 break; 2425 2426 if (!valid_user_frame(fp, sizeof(frame))) 2427 break; 2428 2429 perf_callchain_store(entry, frame.return_address); 2430 fp = (void __user *)frame.next_frame; 2431 } 2432 pagefault_enable(); 2433 } 2434 2435 /* 2436 * Deal with code segment offsets for the various execution modes: 2437 * 2438 * VM86 - the good olde 16 bit days, where the linear address is 2439 * 20 bits and we use regs->ip + 0x10 * regs->cs. 2440 * 2441 * IA32 - Where we need to look at GDT/LDT segment descriptor tables 2442 * to figure out what the 32bit base address is. 2443 * 2444 * X32 - has TIF_X32 set, but is running in x86_64 2445 * 2446 * X86_64 - CS,DS,SS,ES are all zero based. 2447 */ 2448 static unsigned long code_segment_base(struct pt_regs *regs) 2449 { 2450 /* 2451 * For IA32 we look at the GDT/LDT segment base to convert the 2452 * effective IP to a linear address. 2453 */ 2454 2455 #ifdef CONFIG_X86_32 2456 /* 2457 * If we are in VM86 mode, add the segment offset to convert to a 2458 * linear address. 2459 */ 2460 if (regs->flags & X86_VM_MASK) 2461 return 0x10 * regs->cs; 2462 2463 if (user_mode(regs) && regs->cs != __USER_CS) 2464 return get_segment_base(regs->cs); 2465 #else 2466 if (user_mode(regs) && !user_64bit_mode(regs) && 2467 regs->cs != __USER32_CS) 2468 return get_segment_base(regs->cs); 2469 #endif 2470 return 0; 2471 } 2472 2473 unsigned long perf_instruction_pointer(struct pt_regs *regs) 2474 { 2475 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) 2476 return perf_guest_cbs->get_guest_ip(); 2477 2478 return regs->ip + code_segment_base(regs); 2479 } 2480 2481 unsigned long perf_misc_flags(struct pt_regs *regs) 2482 { 2483 int misc = 0; 2484 2485 if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) { 2486 if (perf_guest_cbs->is_user_mode()) 2487 misc |= PERF_RECORD_MISC_GUEST_USER; 2488 else 2489 misc |= PERF_RECORD_MISC_GUEST_KERNEL; 2490 } else { 2491 if (user_mode(regs)) 2492 misc |= PERF_RECORD_MISC_USER; 2493 else 2494 misc |= PERF_RECORD_MISC_KERNEL; 2495 } 2496 2497 if (regs->flags & PERF_EFLAGS_EXACT) 2498 misc |= PERF_RECORD_MISC_EXACT_IP; 2499 2500 return misc; 2501 } 2502 2503 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) 2504 { 2505 cap->version = x86_pmu.version; 2506 cap->num_counters_gp = x86_pmu.num_counters; 2507 cap->num_counters_fixed = x86_pmu.num_counters_fixed; 2508 cap->bit_width_gp = x86_pmu.cntval_bits; 2509 cap->bit_width_fixed = x86_pmu.cntval_bits; 2510 cap->events_mask = (unsigned int)x86_pmu.events_maskl; 2511 cap->events_mask_len = x86_pmu.events_mask_len; 2512 } 2513 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); 2514