xref: /openbmc/linux/arch/x86/events/core.c (revision 8ee90c5c)
1 /*
2  * Performance events x86 architecture code
3  *
4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
6  *  Copyright (C) 2009 Jaswinder Singh Rajput
7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
11  *
12  *  For licencing details see kernel-base/COPYING
13  */
14 
15 #include <linux/perf_event.h>
16 #include <linux/capability.h>
17 #include <linux/notifier.h>
18 #include <linux/hardirq.h>
19 #include <linux/kprobes.h>
20 #include <linux/export.h>
21 #include <linux/init.h>
22 #include <linux/kdebug.h>
23 #include <linux/sched/mm.h>
24 #include <linux/sched/clock.h>
25 #include <linux/uaccess.h>
26 #include <linux/slab.h>
27 #include <linux/cpu.h>
28 #include <linux/bitops.h>
29 #include <linux/device.h>
30 
31 #include <asm/apic.h>
32 #include <asm/stacktrace.h>
33 #include <asm/nmi.h>
34 #include <asm/smp.h>
35 #include <asm/alternative.h>
36 #include <asm/mmu_context.h>
37 #include <asm/tlbflush.h>
38 #include <asm/timer.h>
39 #include <asm/desc.h>
40 #include <asm/ldt.h>
41 #include <asm/unwind.h>
42 
43 #include "perf_event.h"
44 
45 struct x86_pmu x86_pmu __read_mostly;
46 
47 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
48 	.enabled = 1,
49 };
50 
51 struct static_key rdpmc_always_available = STATIC_KEY_INIT_FALSE;
52 
53 u64 __read_mostly hw_cache_event_ids
54 				[PERF_COUNT_HW_CACHE_MAX]
55 				[PERF_COUNT_HW_CACHE_OP_MAX]
56 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
57 u64 __read_mostly hw_cache_extra_regs
58 				[PERF_COUNT_HW_CACHE_MAX]
59 				[PERF_COUNT_HW_CACHE_OP_MAX]
60 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
61 
62 /*
63  * Propagate event elapsed time into the generic event.
64  * Can only be executed on the CPU where the event is active.
65  * Returns the delta events processed.
66  */
67 u64 x86_perf_event_update(struct perf_event *event)
68 {
69 	struct hw_perf_event *hwc = &event->hw;
70 	int shift = 64 - x86_pmu.cntval_bits;
71 	u64 prev_raw_count, new_raw_count;
72 	int idx = hwc->idx;
73 	u64 delta;
74 
75 	if (idx == INTEL_PMC_IDX_FIXED_BTS)
76 		return 0;
77 
78 	/*
79 	 * Careful: an NMI might modify the previous event value.
80 	 *
81 	 * Our tactic to handle this is to first atomically read and
82 	 * exchange a new raw count - then add that new-prev delta
83 	 * count to the generic event atomically:
84 	 */
85 again:
86 	prev_raw_count = local64_read(&hwc->prev_count);
87 	rdpmcl(hwc->event_base_rdpmc, new_raw_count);
88 
89 	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
90 					new_raw_count) != prev_raw_count)
91 		goto again;
92 
93 	/*
94 	 * Now we have the new raw value and have updated the prev
95 	 * timestamp already. We can now calculate the elapsed delta
96 	 * (event-)time and add that to the generic event.
97 	 *
98 	 * Careful, not all hw sign-extends above the physical width
99 	 * of the count.
100 	 */
101 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
102 	delta >>= shift;
103 
104 	local64_add(delta, &event->count);
105 	local64_sub(delta, &hwc->period_left);
106 
107 	return new_raw_count;
108 }
109 
110 /*
111  * Find and validate any extra registers to set up.
112  */
113 static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
114 {
115 	struct hw_perf_event_extra *reg;
116 	struct extra_reg *er;
117 
118 	reg = &event->hw.extra_reg;
119 
120 	if (!x86_pmu.extra_regs)
121 		return 0;
122 
123 	for (er = x86_pmu.extra_regs; er->msr; er++) {
124 		if (er->event != (config & er->config_mask))
125 			continue;
126 		if (event->attr.config1 & ~er->valid_mask)
127 			return -EINVAL;
128 		/* Check if the extra msrs can be safely accessed*/
129 		if (!er->extra_msr_access)
130 			return -ENXIO;
131 
132 		reg->idx = er->idx;
133 		reg->config = event->attr.config1;
134 		reg->reg = er->msr;
135 		break;
136 	}
137 	return 0;
138 }
139 
140 static atomic_t active_events;
141 static atomic_t pmc_refcount;
142 static DEFINE_MUTEX(pmc_reserve_mutex);
143 
144 #ifdef CONFIG_X86_LOCAL_APIC
145 
146 static bool reserve_pmc_hardware(void)
147 {
148 	int i;
149 
150 	for (i = 0; i < x86_pmu.num_counters; i++) {
151 		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
152 			goto perfctr_fail;
153 	}
154 
155 	for (i = 0; i < x86_pmu.num_counters; i++) {
156 		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
157 			goto eventsel_fail;
158 	}
159 
160 	return true;
161 
162 eventsel_fail:
163 	for (i--; i >= 0; i--)
164 		release_evntsel_nmi(x86_pmu_config_addr(i));
165 
166 	i = x86_pmu.num_counters;
167 
168 perfctr_fail:
169 	for (i--; i >= 0; i--)
170 		release_perfctr_nmi(x86_pmu_event_addr(i));
171 
172 	return false;
173 }
174 
175 static void release_pmc_hardware(void)
176 {
177 	int i;
178 
179 	for (i = 0; i < x86_pmu.num_counters; i++) {
180 		release_perfctr_nmi(x86_pmu_event_addr(i));
181 		release_evntsel_nmi(x86_pmu_config_addr(i));
182 	}
183 }
184 
185 #else
186 
187 static bool reserve_pmc_hardware(void) { return true; }
188 static void release_pmc_hardware(void) {}
189 
190 #endif
191 
192 static bool check_hw_exists(void)
193 {
194 	u64 val, val_fail = -1, val_new= ~0;
195 	int i, reg, reg_fail = -1, ret = 0;
196 	int bios_fail = 0;
197 	int reg_safe = -1;
198 
199 	/*
200 	 * Check to see if the BIOS enabled any of the counters, if so
201 	 * complain and bail.
202 	 */
203 	for (i = 0; i < x86_pmu.num_counters; i++) {
204 		reg = x86_pmu_config_addr(i);
205 		ret = rdmsrl_safe(reg, &val);
206 		if (ret)
207 			goto msr_fail;
208 		if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
209 			bios_fail = 1;
210 			val_fail = val;
211 			reg_fail = reg;
212 		} else {
213 			reg_safe = i;
214 		}
215 	}
216 
217 	if (x86_pmu.num_counters_fixed) {
218 		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
219 		ret = rdmsrl_safe(reg, &val);
220 		if (ret)
221 			goto msr_fail;
222 		for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
223 			if (val & (0x03 << i*4)) {
224 				bios_fail = 1;
225 				val_fail = val;
226 				reg_fail = reg;
227 			}
228 		}
229 	}
230 
231 	/*
232 	 * If all the counters are enabled, the below test will always
233 	 * fail.  The tools will also become useless in this scenario.
234 	 * Just fail and disable the hardware counters.
235 	 */
236 
237 	if (reg_safe == -1) {
238 		reg = reg_safe;
239 		goto msr_fail;
240 	}
241 
242 	/*
243 	 * Read the current value, change it and read it back to see if it
244 	 * matches, this is needed to detect certain hardware emulators
245 	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
246 	 */
247 	reg = x86_pmu_event_addr(reg_safe);
248 	if (rdmsrl_safe(reg, &val))
249 		goto msr_fail;
250 	val ^= 0xffffUL;
251 	ret = wrmsrl_safe(reg, val);
252 	ret |= rdmsrl_safe(reg, &val_new);
253 	if (ret || val != val_new)
254 		goto msr_fail;
255 
256 	/*
257 	 * We still allow the PMU driver to operate:
258 	 */
259 	if (bios_fail) {
260 		pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
261 		pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
262 			      reg_fail, val_fail);
263 	}
264 
265 	return true;
266 
267 msr_fail:
268 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
269 		pr_cont("PMU not available due to virtualization, using software events only.\n");
270 	} else {
271 		pr_cont("Broken PMU hardware detected, using software events only.\n");
272 		pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
273 		       reg, val_new);
274 	}
275 
276 	return false;
277 }
278 
279 static void hw_perf_event_destroy(struct perf_event *event)
280 {
281 	x86_release_hardware();
282 	atomic_dec(&active_events);
283 }
284 
285 void hw_perf_lbr_event_destroy(struct perf_event *event)
286 {
287 	hw_perf_event_destroy(event);
288 
289 	/* undo the lbr/bts event accounting */
290 	x86_del_exclusive(x86_lbr_exclusive_lbr);
291 }
292 
293 static inline int x86_pmu_initialized(void)
294 {
295 	return x86_pmu.handle_irq != NULL;
296 }
297 
298 static inline int
299 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
300 {
301 	struct perf_event_attr *attr = &event->attr;
302 	unsigned int cache_type, cache_op, cache_result;
303 	u64 config, val;
304 
305 	config = attr->config;
306 
307 	cache_type = (config >>  0) & 0xff;
308 	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
309 		return -EINVAL;
310 
311 	cache_op = (config >>  8) & 0xff;
312 	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
313 		return -EINVAL;
314 
315 	cache_result = (config >> 16) & 0xff;
316 	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
317 		return -EINVAL;
318 
319 	val = hw_cache_event_ids[cache_type][cache_op][cache_result];
320 
321 	if (val == 0)
322 		return -ENOENT;
323 
324 	if (val == -1)
325 		return -EINVAL;
326 
327 	hwc->config |= val;
328 	attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
329 	return x86_pmu_extra_regs(val, event);
330 }
331 
332 int x86_reserve_hardware(void)
333 {
334 	int err = 0;
335 
336 	if (!atomic_inc_not_zero(&pmc_refcount)) {
337 		mutex_lock(&pmc_reserve_mutex);
338 		if (atomic_read(&pmc_refcount) == 0) {
339 			if (!reserve_pmc_hardware())
340 				err = -EBUSY;
341 			else
342 				reserve_ds_buffers();
343 		}
344 		if (!err)
345 			atomic_inc(&pmc_refcount);
346 		mutex_unlock(&pmc_reserve_mutex);
347 	}
348 
349 	return err;
350 }
351 
352 void x86_release_hardware(void)
353 {
354 	if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
355 		release_pmc_hardware();
356 		release_ds_buffers();
357 		mutex_unlock(&pmc_reserve_mutex);
358 	}
359 }
360 
361 /*
362  * Check if we can create event of a certain type (that no conflicting events
363  * are present).
364  */
365 int x86_add_exclusive(unsigned int what)
366 {
367 	int i;
368 
369 	/*
370 	 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
371 	 * LBR and BTS are still mutually exclusive.
372 	 */
373 	if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
374 		return 0;
375 
376 	if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
377 		mutex_lock(&pmc_reserve_mutex);
378 		for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
379 			if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
380 				goto fail_unlock;
381 		}
382 		atomic_inc(&x86_pmu.lbr_exclusive[what]);
383 		mutex_unlock(&pmc_reserve_mutex);
384 	}
385 
386 	atomic_inc(&active_events);
387 	return 0;
388 
389 fail_unlock:
390 	mutex_unlock(&pmc_reserve_mutex);
391 	return -EBUSY;
392 }
393 
394 void x86_del_exclusive(unsigned int what)
395 {
396 	if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
397 		return;
398 
399 	atomic_dec(&x86_pmu.lbr_exclusive[what]);
400 	atomic_dec(&active_events);
401 }
402 
403 int x86_setup_perfctr(struct perf_event *event)
404 {
405 	struct perf_event_attr *attr = &event->attr;
406 	struct hw_perf_event *hwc = &event->hw;
407 	u64 config;
408 
409 	if (!is_sampling_event(event)) {
410 		hwc->sample_period = x86_pmu.max_period;
411 		hwc->last_period = hwc->sample_period;
412 		local64_set(&hwc->period_left, hwc->sample_period);
413 	}
414 
415 	if (attr->type == PERF_TYPE_RAW)
416 		return x86_pmu_extra_regs(event->attr.config, event);
417 
418 	if (attr->type == PERF_TYPE_HW_CACHE)
419 		return set_ext_hw_attr(hwc, event);
420 
421 	if (attr->config >= x86_pmu.max_events)
422 		return -EINVAL;
423 
424 	/*
425 	 * The generic map:
426 	 */
427 	config = x86_pmu.event_map(attr->config);
428 
429 	if (config == 0)
430 		return -ENOENT;
431 
432 	if (config == -1LL)
433 		return -EINVAL;
434 
435 	/*
436 	 * Branch tracing:
437 	 */
438 	if (attr->config == PERF_COUNT_HW_BRANCH_INSTRUCTIONS &&
439 	    !attr->freq && hwc->sample_period == 1) {
440 		/* BTS is not supported by this architecture. */
441 		if (!x86_pmu.bts_active)
442 			return -EOPNOTSUPP;
443 
444 		/* BTS is currently only allowed for user-mode. */
445 		if (!attr->exclude_kernel)
446 			return -EOPNOTSUPP;
447 
448 		/* disallow bts if conflicting events are present */
449 		if (x86_add_exclusive(x86_lbr_exclusive_lbr))
450 			return -EBUSY;
451 
452 		event->destroy = hw_perf_lbr_event_destroy;
453 	}
454 
455 	hwc->config |= config;
456 
457 	return 0;
458 }
459 
460 /*
461  * check that branch_sample_type is compatible with
462  * settings needed for precise_ip > 1 which implies
463  * using the LBR to capture ALL taken branches at the
464  * priv levels of the measurement
465  */
466 static inline int precise_br_compat(struct perf_event *event)
467 {
468 	u64 m = event->attr.branch_sample_type;
469 	u64 b = 0;
470 
471 	/* must capture all branches */
472 	if (!(m & PERF_SAMPLE_BRANCH_ANY))
473 		return 0;
474 
475 	m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
476 
477 	if (!event->attr.exclude_user)
478 		b |= PERF_SAMPLE_BRANCH_USER;
479 
480 	if (!event->attr.exclude_kernel)
481 		b |= PERF_SAMPLE_BRANCH_KERNEL;
482 
483 	/*
484 	 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
485 	 */
486 
487 	return m == b;
488 }
489 
490 int x86_pmu_max_precise(void)
491 {
492 	int precise = 0;
493 
494 	/* Support for constant skid */
495 	if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
496 		precise++;
497 
498 		/* Support for IP fixup */
499 		if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
500 			precise++;
501 
502 		if (x86_pmu.pebs_prec_dist)
503 			precise++;
504 	}
505 	return precise;
506 }
507 
508 int x86_pmu_hw_config(struct perf_event *event)
509 {
510 	if (event->attr.precise_ip) {
511 		int precise = x86_pmu_max_precise();
512 
513 		if (event->attr.precise_ip > precise)
514 			return -EOPNOTSUPP;
515 
516 		/* There's no sense in having PEBS for non sampling events: */
517 		if (!is_sampling_event(event))
518 			return -EINVAL;
519 	}
520 	/*
521 	 * check that PEBS LBR correction does not conflict with
522 	 * whatever the user is asking with attr->branch_sample_type
523 	 */
524 	if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
525 		u64 *br_type = &event->attr.branch_sample_type;
526 
527 		if (has_branch_stack(event)) {
528 			if (!precise_br_compat(event))
529 				return -EOPNOTSUPP;
530 
531 			/* branch_sample_type is compatible */
532 
533 		} else {
534 			/*
535 			 * user did not specify  branch_sample_type
536 			 *
537 			 * For PEBS fixups, we capture all
538 			 * the branches at the priv level of the
539 			 * event.
540 			 */
541 			*br_type = PERF_SAMPLE_BRANCH_ANY;
542 
543 			if (!event->attr.exclude_user)
544 				*br_type |= PERF_SAMPLE_BRANCH_USER;
545 
546 			if (!event->attr.exclude_kernel)
547 				*br_type |= PERF_SAMPLE_BRANCH_KERNEL;
548 		}
549 	}
550 
551 	if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
552 		event->attach_state |= PERF_ATTACH_TASK_DATA;
553 
554 	/*
555 	 * Generate PMC IRQs:
556 	 * (keep 'enabled' bit clear for now)
557 	 */
558 	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
559 
560 	/*
561 	 * Count user and OS events unless requested not to
562 	 */
563 	if (!event->attr.exclude_user)
564 		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
565 	if (!event->attr.exclude_kernel)
566 		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
567 
568 	if (event->attr.type == PERF_TYPE_RAW)
569 		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
570 
571 	if (event->attr.sample_period && x86_pmu.limit_period) {
572 		if (x86_pmu.limit_period(event, event->attr.sample_period) >
573 				event->attr.sample_period)
574 			return -EINVAL;
575 	}
576 
577 	return x86_setup_perfctr(event);
578 }
579 
580 /*
581  * Setup the hardware configuration for a given attr_type
582  */
583 static int __x86_pmu_event_init(struct perf_event *event)
584 {
585 	int err;
586 
587 	if (!x86_pmu_initialized())
588 		return -ENODEV;
589 
590 	err = x86_reserve_hardware();
591 	if (err)
592 		return err;
593 
594 	atomic_inc(&active_events);
595 	event->destroy = hw_perf_event_destroy;
596 
597 	event->hw.idx = -1;
598 	event->hw.last_cpu = -1;
599 	event->hw.last_tag = ~0ULL;
600 
601 	/* mark unused */
602 	event->hw.extra_reg.idx = EXTRA_REG_NONE;
603 	event->hw.branch_reg.idx = EXTRA_REG_NONE;
604 
605 	return x86_pmu.hw_config(event);
606 }
607 
608 void x86_pmu_disable_all(void)
609 {
610 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
611 	int idx;
612 
613 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
614 		u64 val;
615 
616 		if (!test_bit(idx, cpuc->active_mask))
617 			continue;
618 		rdmsrl(x86_pmu_config_addr(idx), val);
619 		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
620 			continue;
621 		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
622 		wrmsrl(x86_pmu_config_addr(idx), val);
623 	}
624 }
625 
626 /*
627  * There may be PMI landing after enabled=0. The PMI hitting could be before or
628  * after disable_all.
629  *
630  * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
631  * It will not be re-enabled in the NMI handler again, because enabled=0. After
632  * handling the NMI, disable_all will be called, which will not change the
633  * state either. If PMI hits after disable_all, the PMU is already disabled
634  * before entering NMI handler. The NMI handler will not change the state
635  * either.
636  *
637  * So either situation is harmless.
638  */
639 static void x86_pmu_disable(struct pmu *pmu)
640 {
641 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
642 
643 	if (!x86_pmu_initialized())
644 		return;
645 
646 	if (!cpuc->enabled)
647 		return;
648 
649 	cpuc->n_added = 0;
650 	cpuc->enabled = 0;
651 	barrier();
652 
653 	x86_pmu.disable_all();
654 }
655 
656 void x86_pmu_enable_all(int added)
657 {
658 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
659 	int idx;
660 
661 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
662 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
663 
664 		if (!test_bit(idx, cpuc->active_mask))
665 			continue;
666 
667 		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
668 	}
669 }
670 
671 static struct pmu pmu;
672 
673 static inline int is_x86_event(struct perf_event *event)
674 {
675 	return event->pmu == &pmu;
676 }
677 
678 /*
679  * Event scheduler state:
680  *
681  * Assign events iterating over all events and counters, beginning
682  * with events with least weights first. Keep the current iterator
683  * state in struct sched_state.
684  */
685 struct sched_state {
686 	int	weight;
687 	int	event;		/* event index */
688 	int	counter;	/* counter index */
689 	int	unassigned;	/* number of events to be assigned left */
690 	int	nr_gp;		/* number of GP counters used */
691 	unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
692 };
693 
694 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
695 #define	SCHED_STATES_MAX	2
696 
697 struct perf_sched {
698 	int			max_weight;
699 	int			max_events;
700 	int			max_gp;
701 	int			saved_states;
702 	struct event_constraint	**constraints;
703 	struct sched_state	state;
704 	struct sched_state	saved[SCHED_STATES_MAX];
705 };
706 
707 /*
708  * Initialize interator that runs through all events and counters.
709  */
710 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
711 			    int num, int wmin, int wmax, int gpmax)
712 {
713 	int idx;
714 
715 	memset(sched, 0, sizeof(*sched));
716 	sched->max_events	= num;
717 	sched->max_weight	= wmax;
718 	sched->max_gp		= gpmax;
719 	sched->constraints	= constraints;
720 
721 	for (idx = 0; idx < num; idx++) {
722 		if (constraints[idx]->weight == wmin)
723 			break;
724 	}
725 
726 	sched->state.event	= idx;		/* start with min weight */
727 	sched->state.weight	= wmin;
728 	sched->state.unassigned	= num;
729 }
730 
731 static void perf_sched_save_state(struct perf_sched *sched)
732 {
733 	if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
734 		return;
735 
736 	sched->saved[sched->saved_states] = sched->state;
737 	sched->saved_states++;
738 }
739 
740 static bool perf_sched_restore_state(struct perf_sched *sched)
741 {
742 	if (!sched->saved_states)
743 		return false;
744 
745 	sched->saved_states--;
746 	sched->state = sched->saved[sched->saved_states];
747 
748 	/* continue with next counter: */
749 	clear_bit(sched->state.counter++, sched->state.used);
750 
751 	return true;
752 }
753 
754 /*
755  * Select a counter for the current event to schedule. Return true on
756  * success.
757  */
758 static bool __perf_sched_find_counter(struct perf_sched *sched)
759 {
760 	struct event_constraint *c;
761 	int idx;
762 
763 	if (!sched->state.unassigned)
764 		return false;
765 
766 	if (sched->state.event >= sched->max_events)
767 		return false;
768 
769 	c = sched->constraints[sched->state.event];
770 	/* Prefer fixed purpose counters */
771 	if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
772 		idx = INTEL_PMC_IDX_FIXED;
773 		for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
774 			if (!__test_and_set_bit(idx, sched->state.used))
775 				goto done;
776 		}
777 	}
778 
779 	/* Grab the first unused counter starting with idx */
780 	idx = sched->state.counter;
781 	for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
782 		if (!__test_and_set_bit(idx, sched->state.used)) {
783 			if (sched->state.nr_gp++ >= sched->max_gp)
784 				return false;
785 
786 			goto done;
787 		}
788 	}
789 
790 	return false;
791 
792 done:
793 	sched->state.counter = idx;
794 
795 	if (c->overlap)
796 		perf_sched_save_state(sched);
797 
798 	return true;
799 }
800 
801 static bool perf_sched_find_counter(struct perf_sched *sched)
802 {
803 	while (!__perf_sched_find_counter(sched)) {
804 		if (!perf_sched_restore_state(sched))
805 			return false;
806 	}
807 
808 	return true;
809 }
810 
811 /*
812  * Go through all unassigned events and find the next one to schedule.
813  * Take events with the least weight first. Return true on success.
814  */
815 static bool perf_sched_next_event(struct perf_sched *sched)
816 {
817 	struct event_constraint *c;
818 
819 	if (!sched->state.unassigned || !--sched->state.unassigned)
820 		return false;
821 
822 	do {
823 		/* next event */
824 		sched->state.event++;
825 		if (sched->state.event >= sched->max_events) {
826 			/* next weight */
827 			sched->state.event = 0;
828 			sched->state.weight++;
829 			if (sched->state.weight > sched->max_weight)
830 				return false;
831 		}
832 		c = sched->constraints[sched->state.event];
833 	} while (c->weight != sched->state.weight);
834 
835 	sched->state.counter = 0;	/* start with first counter */
836 
837 	return true;
838 }
839 
840 /*
841  * Assign a counter for each event.
842  */
843 int perf_assign_events(struct event_constraint **constraints, int n,
844 			int wmin, int wmax, int gpmax, int *assign)
845 {
846 	struct perf_sched sched;
847 
848 	perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
849 
850 	do {
851 		if (!perf_sched_find_counter(&sched))
852 			break;	/* failed */
853 		if (assign)
854 			assign[sched.state.event] = sched.state.counter;
855 	} while (perf_sched_next_event(&sched));
856 
857 	return sched.state.unassigned;
858 }
859 EXPORT_SYMBOL_GPL(perf_assign_events);
860 
861 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
862 {
863 	struct event_constraint *c;
864 	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
865 	struct perf_event *e;
866 	int i, wmin, wmax, unsched = 0;
867 	struct hw_perf_event *hwc;
868 
869 	bitmap_zero(used_mask, X86_PMC_IDX_MAX);
870 
871 	if (x86_pmu.start_scheduling)
872 		x86_pmu.start_scheduling(cpuc);
873 
874 	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
875 		cpuc->event_constraint[i] = NULL;
876 		c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
877 		cpuc->event_constraint[i] = c;
878 
879 		wmin = min(wmin, c->weight);
880 		wmax = max(wmax, c->weight);
881 	}
882 
883 	/*
884 	 * fastpath, try to reuse previous register
885 	 */
886 	for (i = 0; i < n; i++) {
887 		hwc = &cpuc->event_list[i]->hw;
888 		c = cpuc->event_constraint[i];
889 
890 		/* never assigned */
891 		if (hwc->idx == -1)
892 			break;
893 
894 		/* constraint still honored */
895 		if (!test_bit(hwc->idx, c->idxmsk))
896 			break;
897 
898 		/* not already used */
899 		if (test_bit(hwc->idx, used_mask))
900 			break;
901 
902 		__set_bit(hwc->idx, used_mask);
903 		if (assign)
904 			assign[i] = hwc->idx;
905 	}
906 
907 	/* slow path */
908 	if (i != n) {
909 		int gpmax = x86_pmu.num_counters;
910 
911 		/*
912 		 * Do not allow scheduling of more than half the available
913 		 * generic counters.
914 		 *
915 		 * This helps avoid counter starvation of sibling thread by
916 		 * ensuring at most half the counters cannot be in exclusive
917 		 * mode. There is no designated counters for the limits. Any
918 		 * N/2 counters can be used. This helps with events with
919 		 * specific counter constraints.
920 		 */
921 		if (is_ht_workaround_enabled() && !cpuc->is_fake &&
922 		    READ_ONCE(cpuc->excl_cntrs->exclusive_present))
923 			gpmax /= 2;
924 
925 		unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
926 					     wmax, gpmax, assign);
927 	}
928 
929 	/*
930 	 * In case of success (unsched = 0), mark events as committed,
931 	 * so we do not put_constraint() in case new events are added
932 	 * and fail to be scheduled
933 	 *
934 	 * We invoke the lower level commit callback to lock the resource
935 	 *
936 	 * We do not need to do all of this in case we are called to
937 	 * validate an event group (assign == NULL)
938 	 */
939 	if (!unsched && assign) {
940 		for (i = 0; i < n; i++) {
941 			e = cpuc->event_list[i];
942 			e->hw.flags |= PERF_X86_EVENT_COMMITTED;
943 			if (x86_pmu.commit_scheduling)
944 				x86_pmu.commit_scheduling(cpuc, i, assign[i]);
945 		}
946 	} else {
947 		for (i = 0; i < n; i++) {
948 			e = cpuc->event_list[i];
949 			/*
950 			 * do not put_constraint() on comitted events,
951 			 * because they are good to go
952 			 */
953 			if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
954 				continue;
955 
956 			/*
957 			 * release events that failed scheduling
958 			 */
959 			if (x86_pmu.put_event_constraints)
960 				x86_pmu.put_event_constraints(cpuc, e);
961 		}
962 	}
963 
964 	if (x86_pmu.stop_scheduling)
965 		x86_pmu.stop_scheduling(cpuc);
966 
967 	return unsched ? -EINVAL : 0;
968 }
969 
970 /*
971  * dogrp: true if must collect siblings events (group)
972  * returns total number of events and error code
973  */
974 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
975 {
976 	struct perf_event *event;
977 	int n, max_count;
978 
979 	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
980 
981 	/* current number of events already accepted */
982 	n = cpuc->n_events;
983 
984 	if (is_x86_event(leader)) {
985 		if (n >= max_count)
986 			return -EINVAL;
987 		cpuc->event_list[n] = leader;
988 		n++;
989 	}
990 	if (!dogrp)
991 		return n;
992 
993 	list_for_each_entry(event, &leader->sibling_list, group_entry) {
994 		if (!is_x86_event(event) ||
995 		    event->state <= PERF_EVENT_STATE_OFF)
996 			continue;
997 
998 		if (n >= max_count)
999 			return -EINVAL;
1000 
1001 		cpuc->event_list[n] = event;
1002 		n++;
1003 	}
1004 	return n;
1005 }
1006 
1007 static inline void x86_assign_hw_event(struct perf_event *event,
1008 				struct cpu_hw_events *cpuc, int i)
1009 {
1010 	struct hw_perf_event *hwc = &event->hw;
1011 
1012 	hwc->idx = cpuc->assign[i];
1013 	hwc->last_cpu = smp_processor_id();
1014 	hwc->last_tag = ++cpuc->tags[i];
1015 
1016 	if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1017 		hwc->config_base = 0;
1018 		hwc->event_base	= 0;
1019 	} else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1020 		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1021 		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1022 		hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1023 	} else {
1024 		hwc->config_base = x86_pmu_config_addr(hwc->idx);
1025 		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
1026 		hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1027 	}
1028 }
1029 
1030 static inline int match_prev_assignment(struct hw_perf_event *hwc,
1031 					struct cpu_hw_events *cpuc,
1032 					int i)
1033 {
1034 	return hwc->idx == cpuc->assign[i] &&
1035 		hwc->last_cpu == smp_processor_id() &&
1036 		hwc->last_tag == cpuc->tags[i];
1037 }
1038 
1039 static void x86_pmu_start(struct perf_event *event, int flags);
1040 
1041 static void x86_pmu_enable(struct pmu *pmu)
1042 {
1043 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1044 	struct perf_event *event;
1045 	struct hw_perf_event *hwc;
1046 	int i, added = cpuc->n_added;
1047 
1048 	if (!x86_pmu_initialized())
1049 		return;
1050 
1051 	if (cpuc->enabled)
1052 		return;
1053 
1054 	if (cpuc->n_added) {
1055 		int n_running = cpuc->n_events - cpuc->n_added;
1056 		/*
1057 		 * apply assignment obtained either from
1058 		 * hw_perf_group_sched_in() or x86_pmu_enable()
1059 		 *
1060 		 * step1: save events moving to new counters
1061 		 */
1062 		for (i = 0; i < n_running; i++) {
1063 			event = cpuc->event_list[i];
1064 			hwc = &event->hw;
1065 
1066 			/*
1067 			 * we can avoid reprogramming counter if:
1068 			 * - assigned same counter as last time
1069 			 * - running on same CPU as last time
1070 			 * - no other event has used the counter since
1071 			 */
1072 			if (hwc->idx == -1 ||
1073 			    match_prev_assignment(hwc, cpuc, i))
1074 				continue;
1075 
1076 			/*
1077 			 * Ensure we don't accidentally enable a stopped
1078 			 * counter simply because we rescheduled.
1079 			 */
1080 			if (hwc->state & PERF_HES_STOPPED)
1081 				hwc->state |= PERF_HES_ARCH;
1082 
1083 			x86_pmu_stop(event, PERF_EF_UPDATE);
1084 		}
1085 
1086 		/*
1087 		 * step2: reprogram moved events into new counters
1088 		 */
1089 		for (i = 0; i < cpuc->n_events; i++) {
1090 			event = cpuc->event_list[i];
1091 			hwc = &event->hw;
1092 
1093 			if (!match_prev_assignment(hwc, cpuc, i))
1094 				x86_assign_hw_event(event, cpuc, i);
1095 			else if (i < n_running)
1096 				continue;
1097 
1098 			if (hwc->state & PERF_HES_ARCH)
1099 				continue;
1100 
1101 			x86_pmu_start(event, PERF_EF_RELOAD);
1102 		}
1103 		cpuc->n_added = 0;
1104 		perf_events_lapic_init();
1105 	}
1106 
1107 	cpuc->enabled = 1;
1108 	barrier();
1109 
1110 	x86_pmu.enable_all(added);
1111 }
1112 
1113 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
1114 
1115 /*
1116  * Set the next IRQ period, based on the hwc->period_left value.
1117  * To be called with the event disabled in hw:
1118  */
1119 int x86_perf_event_set_period(struct perf_event *event)
1120 {
1121 	struct hw_perf_event *hwc = &event->hw;
1122 	s64 left = local64_read(&hwc->period_left);
1123 	s64 period = hwc->sample_period;
1124 	int ret = 0, idx = hwc->idx;
1125 
1126 	if (idx == INTEL_PMC_IDX_FIXED_BTS)
1127 		return 0;
1128 
1129 	/*
1130 	 * If we are way outside a reasonable range then just skip forward:
1131 	 */
1132 	if (unlikely(left <= -period)) {
1133 		left = period;
1134 		local64_set(&hwc->period_left, left);
1135 		hwc->last_period = period;
1136 		ret = 1;
1137 	}
1138 
1139 	if (unlikely(left <= 0)) {
1140 		left += period;
1141 		local64_set(&hwc->period_left, left);
1142 		hwc->last_period = period;
1143 		ret = 1;
1144 	}
1145 	/*
1146 	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
1147 	 */
1148 	if (unlikely(left < 2))
1149 		left = 2;
1150 
1151 	if (left > x86_pmu.max_period)
1152 		left = x86_pmu.max_period;
1153 
1154 	if (x86_pmu.limit_period)
1155 		left = x86_pmu.limit_period(event, left);
1156 
1157 	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
1158 
1159 	if (!(hwc->flags & PERF_X86_EVENT_AUTO_RELOAD) ||
1160 	    local64_read(&hwc->prev_count) != (u64)-left) {
1161 		/*
1162 		 * The hw event starts counting from this event offset,
1163 		 * mark it to be able to extra future deltas:
1164 		 */
1165 		local64_set(&hwc->prev_count, (u64)-left);
1166 
1167 		wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
1168 	}
1169 
1170 	/*
1171 	 * Due to erratum on certan cpu we need
1172 	 * a second write to be sure the register
1173 	 * is updated properly
1174 	 */
1175 	if (x86_pmu.perfctr_second_write) {
1176 		wrmsrl(hwc->event_base,
1177 			(u64)(-left) & x86_pmu.cntval_mask);
1178 	}
1179 
1180 	perf_event_update_userpage(event);
1181 
1182 	return ret;
1183 }
1184 
1185 void x86_pmu_enable_event(struct perf_event *event)
1186 {
1187 	if (__this_cpu_read(cpu_hw_events.enabled))
1188 		__x86_pmu_enable_event(&event->hw,
1189 				       ARCH_PERFMON_EVENTSEL_ENABLE);
1190 }
1191 
1192 /*
1193  * Add a single event to the PMU.
1194  *
1195  * The event is added to the group of enabled events
1196  * but only if it can be scehduled with existing events.
1197  */
1198 static int x86_pmu_add(struct perf_event *event, int flags)
1199 {
1200 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1201 	struct hw_perf_event *hwc;
1202 	int assign[X86_PMC_IDX_MAX];
1203 	int n, n0, ret;
1204 
1205 	hwc = &event->hw;
1206 
1207 	n0 = cpuc->n_events;
1208 	ret = n = collect_events(cpuc, event, false);
1209 	if (ret < 0)
1210 		goto out;
1211 
1212 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
1213 	if (!(flags & PERF_EF_START))
1214 		hwc->state |= PERF_HES_ARCH;
1215 
1216 	/*
1217 	 * If group events scheduling transaction was started,
1218 	 * skip the schedulability test here, it will be performed
1219 	 * at commit time (->commit_txn) as a whole.
1220 	 *
1221 	 * If commit fails, we'll call ->del() on all events
1222 	 * for which ->add() was called.
1223 	 */
1224 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1225 		goto done_collect;
1226 
1227 	ret = x86_pmu.schedule_events(cpuc, n, assign);
1228 	if (ret)
1229 		goto out;
1230 	/*
1231 	 * copy new assignment, now we know it is possible
1232 	 * will be used by hw_perf_enable()
1233 	 */
1234 	memcpy(cpuc->assign, assign, n*sizeof(int));
1235 
1236 done_collect:
1237 	/*
1238 	 * Commit the collect_events() state. See x86_pmu_del() and
1239 	 * x86_pmu_*_txn().
1240 	 */
1241 	cpuc->n_events = n;
1242 	cpuc->n_added += n - n0;
1243 	cpuc->n_txn += n - n0;
1244 
1245 	if (x86_pmu.add) {
1246 		/*
1247 		 * This is before x86_pmu_enable() will call x86_pmu_start(),
1248 		 * so we enable LBRs before an event needs them etc..
1249 		 */
1250 		x86_pmu.add(event);
1251 	}
1252 
1253 	ret = 0;
1254 out:
1255 	return ret;
1256 }
1257 
1258 static void x86_pmu_start(struct perf_event *event, int flags)
1259 {
1260 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1261 	int idx = event->hw.idx;
1262 
1263 	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
1264 		return;
1265 
1266 	if (WARN_ON_ONCE(idx == -1))
1267 		return;
1268 
1269 	if (flags & PERF_EF_RELOAD) {
1270 		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
1271 		x86_perf_event_set_period(event);
1272 	}
1273 
1274 	event->hw.state = 0;
1275 
1276 	cpuc->events[idx] = event;
1277 	__set_bit(idx, cpuc->active_mask);
1278 	__set_bit(idx, cpuc->running);
1279 	x86_pmu.enable(event);
1280 	perf_event_update_userpage(event);
1281 }
1282 
1283 void perf_event_print_debug(void)
1284 {
1285 	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
1286 	u64 pebs, debugctl;
1287 	struct cpu_hw_events *cpuc;
1288 	unsigned long flags;
1289 	int cpu, idx;
1290 
1291 	if (!x86_pmu.num_counters)
1292 		return;
1293 
1294 	local_irq_save(flags);
1295 
1296 	cpu = smp_processor_id();
1297 	cpuc = &per_cpu(cpu_hw_events, cpu);
1298 
1299 	if (x86_pmu.version >= 2) {
1300 		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
1301 		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
1302 		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
1303 		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
1304 
1305 		pr_info("\n");
1306 		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
1307 		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
1308 		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
1309 		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
1310 		if (x86_pmu.pebs_constraints) {
1311 			rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
1312 			pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
1313 		}
1314 		if (x86_pmu.lbr_nr) {
1315 			rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1316 			pr_info("CPU#%d: debugctl:   %016llx\n", cpu, debugctl);
1317 		}
1318 	}
1319 	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
1320 
1321 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1322 		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
1323 		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
1324 
1325 		prev_left = per_cpu(pmc_prev_left[idx], cpu);
1326 
1327 		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
1328 			cpu, idx, pmc_ctrl);
1329 		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
1330 			cpu, idx, pmc_count);
1331 		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
1332 			cpu, idx, prev_left);
1333 	}
1334 	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
1335 		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
1336 
1337 		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
1338 			cpu, idx, pmc_count);
1339 	}
1340 	local_irq_restore(flags);
1341 }
1342 
1343 void x86_pmu_stop(struct perf_event *event, int flags)
1344 {
1345 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1346 	struct hw_perf_event *hwc = &event->hw;
1347 
1348 	if (__test_and_clear_bit(hwc->idx, cpuc->active_mask)) {
1349 		x86_pmu.disable(event);
1350 		cpuc->events[hwc->idx] = NULL;
1351 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1352 		hwc->state |= PERF_HES_STOPPED;
1353 	}
1354 
1355 	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
1356 		/*
1357 		 * Drain the remaining delta count out of a event
1358 		 * that we are disabling:
1359 		 */
1360 		x86_perf_event_update(event);
1361 		hwc->state |= PERF_HES_UPTODATE;
1362 	}
1363 }
1364 
1365 static void x86_pmu_del(struct perf_event *event, int flags)
1366 {
1367 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1368 	int i;
1369 
1370 	/*
1371 	 * event is descheduled
1372 	 */
1373 	event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
1374 
1375 	/*
1376 	 * If we're called during a txn, we only need to undo x86_pmu.add.
1377 	 * The events never got scheduled and ->cancel_txn will truncate
1378 	 * the event_list.
1379 	 *
1380 	 * XXX assumes any ->del() called during a TXN will only be on
1381 	 * an event added during that same TXN.
1382 	 */
1383 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
1384 		goto do_del;
1385 
1386 	/*
1387 	 * Not a TXN, therefore cleanup properly.
1388 	 */
1389 	x86_pmu_stop(event, PERF_EF_UPDATE);
1390 
1391 	for (i = 0; i < cpuc->n_events; i++) {
1392 		if (event == cpuc->event_list[i])
1393 			break;
1394 	}
1395 
1396 	if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
1397 		return;
1398 
1399 	/* If we have a newly added event; make sure to decrease n_added. */
1400 	if (i >= cpuc->n_events - cpuc->n_added)
1401 		--cpuc->n_added;
1402 
1403 	if (x86_pmu.put_event_constraints)
1404 		x86_pmu.put_event_constraints(cpuc, event);
1405 
1406 	/* Delete the array entry. */
1407 	while (++i < cpuc->n_events) {
1408 		cpuc->event_list[i-1] = cpuc->event_list[i];
1409 		cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
1410 	}
1411 	--cpuc->n_events;
1412 
1413 	perf_event_update_userpage(event);
1414 
1415 do_del:
1416 	if (x86_pmu.del) {
1417 		/*
1418 		 * This is after x86_pmu_stop(); so we disable LBRs after any
1419 		 * event can need them etc..
1420 		 */
1421 		x86_pmu.del(event);
1422 	}
1423 }
1424 
1425 int x86_pmu_handle_irq(struct pt_regs *regs)
1426 {
1427 	struct perf_sample_data data;
1428 	struct cpu_hw_events *cpuc;
1429 	struct perf_event *event;
1430 	int idx, handled = 0;
1431 	u64 val;
1432 
1433 	cpuc = this_cpu_ptr(&cpu_hw_events);
1434 
1435 	/*
1436 	 * Some chipsets need to unmask the LVTPC in a particular spot
1437 	 * inside the nmi handler.  As a result, the unmasking was pushed
1438 	 * into all the nmi handlers.
1439 	 *
1440 	 * This generic handler doesn't seem to have any issues where the
1441 	 * unmasking occurs so it was left at the top.
1442 	 */
1443 	apic_write(APIC_LVTPC, APIC_DM_NMI);
1444 
1445 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
1446 		if (!test_bit(idx, cpuc->active_mask)) {
1447 			/*
1448 			 * Though we deactivated the counter some cpus
1449 			 * might still deliver spurious interrupts still
1450 			 * in flight. Catch them:
1451 			 */
1452 			if (__test_and_clear_bit(idx, cpuc->running))
1453 				handled++;
1454 			continue;
1455 		}
1456 
1457 		event = cpuc->events[idx];
1458 
1459 		val = x86_perf_event_update(event);
1460 		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
1461 			continue;
1462 
1463 		/*
1464 		 * event overflow
1465 		 */
1466 		handled++;
1467 		perf_sample_data_init(&data, 0, event->hw.last_period);
1468 
1469 		if (!x86_perf_event_set_period(event))
1470 			continue;
1471 
1472 		if (perf_event_overflow(event, &data, regs))
1473 			x86_pmu_stop(event, 0);
1474 	}
1475 
1476 	if (handled)
1477 		inc_irq_stat(apic_perf_irqs);
1478 
1479 	return handled;
1480 }
1481 
1482 void perf_events_lapic_init(void)
1483 {
1484 	if (!x86_pmu.apic || !x86_pmu_initialized())
1485 		return;
1486 
1487 	/*
1488 	 * Always use NMI for PMU
1489 	 */
1490 	apic_write(APIC_LVTPC, APIC_DM_NMI);
1491 }
1492 
1493 static int
1494 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1495 {
1496 	u64 start_clock;
1497 	u64 finish_clock;
1498 	int ret;
1499 
1500 	/*
1501 	 * All PMUs/events that share this PMI handler should make sure to
1502 	 * increment active_events for their events.
1503 	 */
1504 	if (!atomic_read(&active_events))
1505 		return NMI_DONE;
1506 
1507 	start_clock = sched_clock();
1508 	ret = x86_pmu.handle_irq(regs);
1509 	finish_clock = sched_clock();
1510 
1511 	perf_sample_event_took(finish_clock - start_clock);
1512 
1513 	return ret;
1514 }
1515 NOKPROBE_SYMBOL(perf_event_nmi_handler);
1516 
1517 struct event_constraint emptyconstraint;
1518 struct event_constraint unconstrained;
1519 
1520 static int x86_pmu_prepare_cpu(unsigned int cpu)
1521 {
1522 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1523 	int i;
1524 
1525 	for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
1526 		cpuc->kfree_on_online[i] = NULL;
1527 	if (x86_pmu.cpu_prepare)
1528 		return x86_pmu.cpu_prepare(cpu);
1529 	return 0;
1530 }
1531 
1532 static int x86_pmu_dead_cpu(unsigned int cpu)
1533 {
1534 	if (x86_pmu.cpu_dead)
1535 		x86_pmu.cpu_dead(cpu);
1536 	return 0;
1537 }
1538 
1539 static int x86_pmu_online_cpu(unsigned int cpu)
1540 {
1541 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
1542 	int i;
1543 
1544 	for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
1545 		kfree(cpuc->kfree_on_online[i]);
1546 		cpuc->kfree_on_online[i] = NULL;
1547 	}
1548 	return 0;
1549 }
1550 
1551 static int x86_pmu_starting_cpu(unsigned int cpu)
1552 {
1553 	if (x86_pmu.cpu_starting)
1554 		x86_pmu.cpu_starting(cpu);
1555 	return 0;
1556 }
1557 
1558 static int x86_pmu_dying_cpu(unsigned int cpu)
1559 {
1560 	if (x86_pmu.cpu_dying)
1561 		x86_pmu.cpu_dying(cpu);
1562 	return 0;
1563 }
1564 
1565 static void __init pmu_check_apic(void)
1566 {
1567 	if (boot_cpu_has(X86_FEATURE_APIC))
1568 		return;
1569 
1570 	x86_pmu.apic = 0;
1571 	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1572 	pr_info("no hardware sampling interrupt available.\n");
1573 
1574 	/*
1575 	 * If we have a PMU initialized but no APIC
1576 	 * interrupts, we cannot sample hardware
1577 	 * events (user-space has to fall back and
1578 	 * sample via a hrtimer based software event):
1579 	 */
1580 	pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1581 
1582 }
1583 
1584 static struct attribute_group x86_pmu_format_group = {
1585 	.name = "format",
1586 	.attrs = NULL,
1587 };
1588 
1589 /*
1590  * Remove all undefined events (x86_pmu.event_map(id) == 0)
1591  * out of events_attr attributes.
1592  */
1593 static void __init filter_events(struct attribute **attrs)
1594 {
1595 	struct device_attribute *d;
1596 	struct perf_pmu_events_attr *pmu_attr;
1597 	int offset = 0;
1598 	int i, j;
1599 
1600 	for (i = 0; attrs[i]; i++) {
1601 		d = (struct device_attribute *)attrs[i];
1602 		pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1603 		/* str trumps id */
1604 		if (pmu_attr->event_str)
1605 			continue;
1606 		if (x86_pmu.event_map(i + offset))
1607 			continue;
1608 
1609 		for (j = i; attrs[j]; j++)
1610 			attrs[j] = attrs[j + 1];
1611 
1612 		/* Check the shifted attr. */
1613 		i--;
1614 
1615 		/*
1616 		 * event_map() is index based, the attrs array is organized
1617 		 * by increasing event index. If we shift the events, then
1618 		 * we need to compensate for the event_map(), otherwise
1619 		 * we are looking up the wrong event in the map
1620 		 */
1621 		offset++;
1622 	}
1623 }
1624 
1625 /* Merge two pointer arrays */
1626 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1627 {
1628 	struct attribute **new;
1629 	int j, i;
1630 
1631 	for (j = 0; a[j]; j++)
1632 		;
1633 	for (i = 0; b[i]; i++)
1634 		j++;
1635 	j++;
1636 
1637 	new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
1638 	if (!new)
1639 		return NULL;
1640 
1641 	j = 0;
1642 	for (i = 0; a[i]; i++)
1643 		new[j++] = a[i];
1644 	for (i = 0; b[i]; i++)
1645 		new[j++] = b[i];
1646 	new[j] = NULL;
1647 
1648 	return new;
1649 }
1650 
1651 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
1652 {
1653 	struct perf_pmu_events_attr *pmu_attr = \
1654 		container_of(attr, struct perf_pmu_events_attr, attr);
1655 	u64 config = x86_pmu.event_map(pmu_attr->id);
1656 
1657 	/* string trumps id */
1658 	if (pmu_attr->event_str)
1659 		return sprintf(page, "%s", pmu_attr->event_str);
1660 
1661 	return x86_pmu.events_sysfs_show(page, config);
1662 }
1663 EXPORT_SYMBOL_GPL(events_sysfs_show);
1664 
1665 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
1666 			  char *page)
1667 {
1668 	struct perf_pmu_events_ht_attr *pmu_attr =
1669 		container_of(attr, struct perf_pmu_events_ht_attr, attr);
1670 
1671 	/*
1672 	 * Report conditional events depending on Hyper-Threading.
1673 	 *
1674 	 * This is overly conservative as usually the HT special
1675 	 * handling is not needed if the other CPU thread is idle.
1676 	 *
1677 	 * Note this does not (and cannot) handle the case when thread
1678 	 * siblings are invisible, for example with virtualization
1679 	 * if they are owned by some other guest.  The user tool
1680 	 * has to re-read when a thread sibling gets onlined later.
1681 	 */
1682 	return sprintf(page, "%s",
1683 			topology_max_smt_threads() > 1 ?
1684 			pmu_attr->event_str_ht :
1685 			pmu_attr->event_str_noht);
1686 }
1687 
1688 EVENT_ATTR(cpu-cycles,			CPU_CYCLES		);
1689 EVENT_ATTR(instructions,		INSTRUCTIONS		);
1690 EVENT_ATTR(cache-references,		CACHE_REFERENCES	);
1691 EVENT_ATTR(cache-misses, 		CACHE_MISSES		);
1692 EVENT_ATTR(branch-instructions,		BRANCH_INSTRUCTIONS	);
1693 EVENT_ATTR(branch-misses,		BRANCH_MISSES		);
1694 EVENT_ATTR(bus-cycles,			BUS_CYCLES		);
1695 EVENT_ATTR(stalled-cycles-frontend,	STALLED_CYCLES_FRONTEND	);
1696 EVENT_ATTR(stalled-cycles-backend,	STALLED_CYCLES_BACKEND	);
1697 EVENT_ATTR(ref-cycles,			REF_CPU_CYCLES		);
1698 
1699 static struct attribute *empty_attrs;
1700 
1701 static struct attribute *events_attr[] = {
1702 	EVENT_PTR(CPU_CYCLES),
1703 	EVENT_PTR(INSTRUCTIONS),
1704 	EVENT_PTR(CACHE_REFERENCES),
1705 	EVENT_PTR(CACHE_MISSES),
1706 	EVENT_PTR(BRANCH_INSTRUCTIONS),
1707 	EVENT_PTR(BRANCH_MISSES),
1708 	EVENT_PTR(BUS_CYCLES),
1709 	EVENT_PTR(STALLED_CYCLES_FRONTEND),
1710 	EVENT_PTR(STALLED_CYCLES_BACKEND),
1711 	EVENT_PTR(REF_CPU_CYCLES),
1712 	NULL,
1713 };
1714 
1715 static struct attribute_group x86_pmu_events_group = {
1716 	.name = "events",
1717 	.attrs = events_attr,
1718 };
1719 
1720 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
1721 {
1722 	u64 umask  = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
1723 	u64 cmask  = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
1724 	bool edge  = (config & ARCH_PERFMON_EVENTSEL_EDGE);
1725 	bool pc    = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
1726 	bool any   = (config & ARCH_PERFMON_EVENTSEL_ANY);
1727 	bool inv   = (config & ARCH_PERFMON_EVENTSEL_INV);
1728 	ssize_t ret;
1729 
1730 	/*
1731 	* We have whole page size to spend and just little data
1732 	* to write, so we can safely use sprintf.
1733 	*/
1734 	ret = sprintf(page, "event=0x%02llx", event);
1735 
1736 	if (umask)
1737 		ret += sprintf(page + ret, ",umask=0x%02llx", umask);
1738 
1739 	if (edge)
1740 		ret += sprintf(page + ret, ",edge");
1741 
1742 	if (pc)
1743 		ret += sprintf(page + ret, ",pc");
1744 
1745 	if (any)
1746 		ret += sprintf(page + ret, ",any");
1747 
1748 	if (inv)
1749 		ret += sprintf(page + ret, ",inv");
1750 
1751 	if (cmask)
1752 		ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
1753 
1754 	ret += sprintf(page + ret, "\n");
1755 
1756 	return ret;
1757 }
1758 
1759 static struct attribute_group x86_pmu_attr_group;
1760 static struct attribute_group x86_pmu_caps_group;
1761 
1762 static int __init init_hw_perf_events(void)
1763 {
1764 	struct x86_pmu_quirk *quirk;
1765 	int err;
1766 
1767 	pr_info("Performance Events: ");
1768 
1769 	switch (boot_cpu_data.x86_vendor) {
1770 	case X86_VENDOR_INTEL:
1771 		err = intel_pmu_init();
1772 		break;
1773 	case X86_VENDOR_AMD:
1774 		err = amd_pmu_init();
1775 		break;
1776 	default:
1777 		err = -ENOTSUPP;
1778 	}
1779 	if (err != 0) {
1780 		pr_cont("no PMU driver, software events only.\n");
1781 		return 0;
1782 	}
1783 
1784 	pmu_check_apic();
1785 
1786 	/* sanity check that the hardware exists or is emulated */
1787 	if (!check_hw_exists())
1788 		return 0;
1789 
1790 	pr_cont("%s PMU driver.\n", x86_pmu.name);
1791 
1792 	x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1793 
1794 	for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
1795 		quirk->func();
1796 
1797 	if (!x86_pmu.intel_ctrl)
1798 		x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
1799 
1800 	perf_events_lapic_init();
1801 	register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
1802 
1803 	unconstrained = (struct event_constraint)
1804 		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1805 				   0, x86_pmu.num_counters, 0, 0);
1806 
1807 	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1808 
1809 	if (x86_pmu.caps_attrs) {
1810 		struct attribute **tmp;
1811 
1812 		tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
1813 		if (!WARN_ON(!tmp))
1814 			x86_pmu_caps_group.attrs = tmp;
1815 	}
1816 
1817 	if (x86_pmu.event_attrs)
1818 		x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1819 
1820 	if (!x86_pmu.events_sysfs_show)
1821 		x86_pmu_events_group.attrs = &empty_attrs;
1822 	else
1823 		filter_events(x86_pmu_events_group.attrs);
1824 
1825 	if (x86_pmu.cpu_events) {
1826 		struct attribute **tmp;
1827 
1828 		tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1829 		if (!WARN_ON(!tmp))
1830 			x86_pmu_events_group.attrs = tmp;
1831 	}
1832 
1833 	if (x86_pmu.attrs) {
1834 		struct attribute **tmp;
1835 
1836 		tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
1837 		if (!WARN_ON(!tmp))
1838 			x86_pmu_attr_group.attrs = tmp;
1839 	}
1840 
1841 	pr_info("... version:                %d\n",     x86_pmu.version);
1842 	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
1843 	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
1844 	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
1845 	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
1846 	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
1847 	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
1848 
1849 	/*
1850 	 * Install callbacks. Core will call them for each online
1851 	 * cpu.
1852 	 */
1853 	err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
1854 				x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
1855 	if (err)
1856 		return err;
1857 
1858 	err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
1859 				"perf/x86:starting", x86_pmu_starting_cpu,
1860 				x86_pmu_dying_cpu);
1861 	if (err)
1862 		goto out;
1863 
1864 	err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
1865 				x86_pmu_online_cpu, NULL);
1866 	if (err)
1867 		goto out1;
1868 
1869 	err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
1870 	if (err)
1871 		goto out2;
1872 
1873 	return 0;
1874 
1875 out2:
1876 	cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
1877 out1:
1878 	cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
1879 out:
1880 	cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
1881 	return err;
1882 }
1883 early_initcall(init_hw_perf_events);
1884 
1885 static inline void x86_pmu_read(struct perf_event *event)
1886 {
1887 	x86_perf_event_update(event);
1888 }
1889 
1890 /*
1891  * Start group events scheduling transaction
1892  * Set the flag to make pmu::enable() not perform the
1893  * schedulability test, it will be performed at commit time
1894  *
1895  * We only support PERF_PMU_TXN_ADD transactions. Save the
1896  * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
1897  * transactions.
1898  */
1899 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
1900 {
1901 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1902 
1903 	WARN_ON_ONCE(cpuc->txn_flags);		/* txn already in flight */
1904 
1905 	cpuc->txn_flags = txn_flags;
1906 	if (txn_flags & ~PERF_PMU_TXN_ADD)
1907 		return;
1908 
1909 	perf_pmu_disable(pmu);
1910 	__this_cpu_write(cpu_hw_events.n_txn, 0);
1911 }
1912 
1913 /*
1914  * Stop group events scheduling transaction
1915  * Clear the flag and pmu::enable() will perform the
1916  * schedulability test.
1917  */
1918 static void x86_pmu_cancel_txn(struct pmu *pmu)
1919 {
1920 	unsigned int txn_flags;
1921 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1922 
1923 	WARN_ON_ONCE(!cpuc->txn_flags);	/* no txn in flight */
1924 
1925 	txn_flags = cpuc->txn_flags;
1926 	cpuc->txn_flags = 0;
1927 	if (txn_flags & ~PERF_PMU_TXN_ADD)
1928 		return;
1929 
1930 	/*
1931 	 * Truncate collected array by the number of events added in this
1932 	 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
1933 	 */
1934 	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
1935 	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
1936 	perf_pmu_enable(pmu);
1937 }
1938 
1939 /*
1940  * Commit group events scheduling transaction
1941  * Perform the group schedulability test as a whole
1942  * Return 0 if success
1943  *
1944  * Does not cancel the transaction on failure; expects the caller to do this.
1945  */
1946 static int x86_pmu_commit_txn(struct pmu *pmu)
1947 {
1948 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1949 	int assign[X86_PMC_IDX_MAX];
1950 	int n, ret;
1951 
1952 	WARN_ON_ONCE(!cpuc->txn_flags);	/* no txn in flight */
1953 
1954 	if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
1955 		cpuc->txn_flags = 0;
1956 		return 0;
1957 	}
1958 
1959 	n = cpuc->n_events;
1960 
1961 	if (!x86_pmu_initialized())
1962 		return -EAGAIN;
1963 
1964 	ret = x86_pmu.schedule_events(cpuc, n, assign);
1965 	if (ret)
1966 		return ret;
1967 
1968 	/*
1969 	 * copy new assignment, now we know it is possible
1970 	 * will be used by hw_perf_enable()
1971 	 */
1972 	memcpy(cpuc->assign, assign, n*sizeof(int));
1973 
1974 	cpuc->txn_flags = 0;
1975 	perf_pmu_enable(pmu);
1976 	return 0;
1977 }
1978 /*
1979  * a fake_cpuc is used to validate event groups. Due to
1980  * the extra reg logic, we need to also allocate a fake
1981  * per_core and per_cpu structure. Otherwise, group events
1982  * using extra reg may conflict without the kernel being
1983  * able to catch this when the last event gets added to
1984  * the group.
1985  */
1986 static void free_fake_cpuc(struct cpu_hw_events *cpuc)
1987 {
1988 	kfree(cpuc->shared_regs);
1989 	kfree(cpuc);
1990 }
1991 
1992 static struct cpu_hw_events *allocate_fake_cpuc(void)
1993 {
1994 	struct cpu_hw_events *cpuc;
1995 	int cpu = raw_smp_processor_id();
1996 
1997 	cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
1998 	if (!cpuc)
1999 		return ERR_PTR(-ENOMEM);
2000 
2001 	/* only needed, if we have extra_regs */
2002 	if (x86_pmu.extra_regs) {
2003 		cpuc->shared_regs = allocate_shared_regs(cpu);
2004 		if (!cpuc->shared_regs)
2005 			goto error;
2006 	}
2007 	cpuc->is_fake = 1;
2008 	return cpuc;
2009 error:
2010 	free_fake_cpuc(cpuc);
2011 	return ERR_PTR(-ENOMEM);
2012 }
2013 
2014 /*
2015  * validate that we can schedule this event
2016  */
2017 static int validate_event(struct perf_event *event)
2018 {
2019 	struct cpu_hw_events *fake_cpuc;
2020 	struct event_constraint *c;
2021 	int ret = 0;
2022 
2023 	fake_cpuc = allocate_fake_cpuc();
2024 	if (IS_ERR(fake_cpuc))
2025 		return PTR_ERR(fake_cpuc);
2026 
2027 	c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
2028 
2029 	if (!c || !c->weight)
2030 		ret = -EINVAL;
2031 
2032 	if (x86_pmu.put_event_constraints)
2033 		x86_pmu.put_event_constraints(fake_cpuc, event);
2034 
2035 	free_fake_cpuc(fake_cpuc);
2036 
2037 	return ret;
2038 }
2039 
2040 /*
2041  * validate a single event group
2042  *
2043  * validation include:
2044  *	- check events are compatible which each other
2045  *	- events do not compete for the same counter
2046  *	- number of events <= number of counters
2047  *
2048  * validation ensures the group can be loaded onto the
2049  * PMU if it was the only group available.
2050  */
2051 static int validate_group(struct perf_event *event)
2052 {
2053 	struct perf_event *leader = event->group_leader;
2054 	struct cpu_hw_events *fake_cpuc;
2055 	int ret = -EINVAL, n;
2056 
2057 	fake_cpuc = allocate_fake_cpuc();
2058 	if (IS_ERR(fake_cpuc))
2059 		return PTR_ERR(fake_cpuc);
2060 	/*
2061 	 * the event is not yet connected with its
2062 	 * siblings therefore we must first collect
2063 	 * existing siblings, then add the new event
2064 	 * before we can simulate the scheduling
2065 	 */
2066 	n = collect_events(fake_cpuc, leader, true);
2067 	if (n < 0)
2068 		goto out;
2069 
2070 	fake_cpuc->n_events = n;
2071 	n = collect_events(fake_cpuc, event, false);
2072 	if (n < 0)
2073 		goto out;
2074 
2075 	fake_cpuc->n_events = n;
2076 
2077 	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
2078 
2079 out:
2080 	free_fake_cpuc(fake_cpuc);
2081 	return ret;
2082 }
2083 
2084 static int x86_pmu_event_init(struct perf_event *event)
2085 {
2086 	struct pmu *tmp;
2087 	int err;
2088 
2089 	switch (event->attr.type) {
2090 	case PERF_TYPE_RAW:
2091 	case PERF_TYPE_HARDWARE:
2092 	case PERF_TYPE_HW_CACHE:
2093 		break;
2094 
2095 	default:
2096 		return -ENOENT;
2097 	}
2098 
2099 	err = __x86_pmu_event_init(event);
2100 	if (!err) {
2101 		/*
2102 		 * we temporarily connect event to its pmu
2103 		 * such that validate_group() can classify
2104 		 * it as an x86 event using is_x86_event()
2105 		 */
2106 		tmp = event->pmu;
2107 		event->pmu = &pmu;
2108 
2109 		if (event->group_leader != event)
2110 			err = validate_group(event);
2111 		else
2112 			err = validate_event(event);
2113 
2114 		event->pmu = tmp;
2115 	}
2116 	if (err) {
2117 		if (event->destroy)
2118 			event->destroy(event);
2119 	}
2120 
2121 	if (ACCESS_ONCE(x86_pmu.attr_rdpmc))
2122 		event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
2123 
2124 	return err;
2125 }
2126 
2127 static void refresh_pce(void *ignored)
2128 {
2129 	load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
2130 }
2131 
2132 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
2133 {
2134 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2135 		return;
2136 
2137 	/*
2138 	 * This function relies on not being called concurrently in two
2139 	 * tasks in the same mm.  Otherwise one task could observe
2140 	 * perf_rdpmc_allowed > 1 and return all the way back to
2141 	 * userspace with CR4.PCE clear while another task is still
2142 	 * doing on_each_cpu_mask() to propagate CR4.PCE.
2143 	 *
2144 	 * For now, this can't happen because all callers hold mmap_sem
2145 	 * for write.  If this changes, we'll need a different solution.
2146 	 */
2147 	lockdep_assert_held_exclusive(&mm->mmap_sem);
2148 
2149 	if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2150 		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2151 }
2152 
2153 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
2154 {
2155 
2156 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2157 		return;
2158 
2159 	if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2160 		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2161 }
2162 
2163 static int x86_pmu_event_idx(struct perf_event *event)
2164 {
2165 	int idx = event->hw.idx;
2166 
2167 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2168 		return 0;
2169 
2170 	if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2171 		idx -= INTEL_PMC_IDX_FIXED;
2172 		idx |= 1 << 30;
2173 	}
2174 
2175 	return idx + 1;
2176 }
2177 
2178 static ssize_t get_attr_rdpmc(struct device *cdev,
2179 			      struct device_attribute *attr,
2180 			      char *buf)
2181 {
2182 	return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
2183 }
2184 
2185 static ssize_t set_attr_rdpmc(struct device *cdev,
2186 			      struct device_attribute *attr,
2187 			      const char *buf, size_t count)
2188 {
2189 	unsigned long val;
2190 	ssize_t ret;
2191 
2192 	ret = kstrtoul(buf, 0, &val);
2193 	if (ret)
2194 		return ret;
2195 
2196 	if (val > 2)
2197 		return -EINVAL;
2198 
2199 	if (x86_pmu.attr_rdpmc_broken)
2200 		return -ENOTSUPP;
2201 
2202 	if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2203 		/*
2204 		 * Changing into or out of always available, aka
2205 		 * perf-event-bypassing mode.  This path is extremely slow,
2206 		 * but only root can trigger it, so it's okay.
2207 		 */
2208 		if (val == 2)
2209 			static_key_slow_inc(&rdpmc_always_available);
2210 		else
2211 			static_key_slow_dec(&rdpmc_always_available);
2212 		on_each_cpu(refresh_pce, NULL, 1);
2213 	}
2214 
2215 	x86_pmu.attr_rdpmc = val;
2216 
2217 	return count;
2218 }
2219 
2220 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
2221 
2222 static struct attribute *x86_pmu_attrs[] = {
2223 	&dev_attr_rdpmc.attr,
2224 	NULL,
2225 };
2226 
2227 static struct attribute_group x86_pmu_attr_group = {
2228 	.attrs = x86_pmu_attrs,
2229 };
2230 
2231 static ssize_t max_precise_show(struct device *cdev,
2232 				  struct device_attribute *attr,
2233 				  char *buf)
2234 {
2235 	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
2236 }
2237 
2238 static DEVICE_ATTR_RO(max_precise);
2239 
2240 static struct attribute *x86_pmu_caps_attrs[] = {
2241 	&dev_attr_max_precise.attr,
2242 	NULL
2243 };
2244 
2245 static struct attribute_group x86_pmu_caps_group = {
2246 	.name = "caps",
2247 	.attrs = x86_pmu_caps_attrs,
2248 };
2249 
2250 static const struct attribute_group *x86_pmu_attr_groups[] = {
2251 	&x86_pmu_attr_group,
2252 	&x86_pmu_format_group,
2253 	&x86_pmu_events_group,
2254 	&x86_pmu_caps_group,
2255 	NULL,
2256 };
2257 
2258 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
2259 {
2260 	if (x86_pmu.sched_task)
2261 		x86_pmu.sched_task(ctx, sched_in);
2262 }
2263 
2264 void perf_check_microcode(void)
2265 {
2266 	if (x86_pmu.check_microcode)
2267 		x86_pmu.check_microcode();
2268 }
2269 
2270 static struct pmu pmu = {
2271 	.pmu_enable		= x86_pmu_enable,
2272 	.pmu_disable		= x86_pmu_disable,
2273 
2274 	.attr_groups		= x86_pmu_attr_groups,
2275 
2276 	.event_init		= x86_pmu_event_init,
2277 
2278 	.event_mapped		= x86_pmu_event_mapped,
2279 	.event_unmapped		= x86_pmu_event_unmapped,
2280 
2281 	.add			= x86_pmu_add,
2282 	.del			= x86_pmu_del,
2283 	.start			= x86_pmu_start,
2284 	.stop			= x86_pmu_stop,
2285 	.read			= x86_pmu_read,
2286 
2287 	.start_txn		= x86_pmu_start_txn,
2288 	.cancel_txn		= x86_pmu_cancel_txn,
2289 	.commit_txn		= x86_pmu_commit_txn,
2290 
2291 	.event_idx		= x86_pmu_event_idx,
2292 	.sched_task		= x86_pmu_sched_task,
2293 	.task_ctx_size          = sizeof(struct x86_perf_task_context),
2294 };
2295 
2296 void arch_perf_update_userpage(struct perf_event *event,
2297 			       struct perf_event_mmap_page *userpg, u64 now)
2298 {
2299 	struct cyc2ns_data data;
2300 	u64 offset;
2301 
2302 	userpg->cap_user_time = 0;
2303 	userpg->cap_user_time_zero = 0;
2304 	userpg->cap_user_rdpmc =
2305 		!!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
2306 	userpg->pmc_width = x86_pmu.cntval_bits;
2307 
2308 	if (!using_native_sched_clock() || !sched_clock_stable())
2309 		return;
2310 
2311 	cyc2ns_read_begin(&data);
2312 
2313 	offset = data.cyc2ns_offset + __sched_clock_offset;
2314 
2315 	/*
2316 	 * Internal timekeeping for enabled/running/stopped times
2317 	 * is always in the local_clock domain.
2318 	 */
2319 	userpg->cap_user_time = 1;
2320 	userpg->time_mult = data.cyc2ns_mul;
2321 	userpg->time_shift = data.cyc2ns_shift;
2322 	userpg->time_offset = offset - now;
2323 
2324 	/*
2325 	 * cap_user_time_zero doesn't make sense when we're using a different
2326 	 * time base for the records.
2327 	 */
2328 	if (!event->attr.use_clockid) {
2329 		userpg->cap_user_time_zero = 1;
2330 		userpg->time_zero = offset;
2331 	}
2332 
2333 	cyc2ns_read_end();
2334 }
2335 
2336 void
2337 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2338 {
2339 	struct unwind_state state;
2340 	unsigned long addr;
2341 
2342 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2343 		/* TODO: We don't support guest os callchain now */
2344 		return;
2345 	}
2346 
2347 	if (perf_callchain_store(entry, regs->ip))
2348 		return;
2349 
2350 	for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
2351 	     unwind_next_frame(&state)) {
2352 		addr = unwind_get_return_address(&state);
2353 		if (!addr || perf_callchain_store(entry, addr))
2354 			return;
2355 	}
2356 }
2357 
2358 static inline int
2359 valid_user_frame(const void __user *fp, unsigned long size)
2360 {
2361 	return (__range_not_ok(fp, size, TASK_SIZE) == 0);
2362 }
2363 
2364 static unsigned long get_segment_base(unsigned int segment)
2365 {
2366 	struct desc_struct *desc;
2367 	unsigned int idx = segment >> 3;
2368 
2369 	if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
2370 #ifdef CONFIG_MODIFY_LDT_SYSCALL
2371 		struct ldt_struct *ldt;
2372 
2373 		/* IRQs are off, so this synchronizes with smp_store_release */
2374 		ldt = lockless_dereference(current->active_mm->context.ldt);
2375 		if (!ldt || idx >= ldt->nr_entries)
2376 			return 0;
2377 
2378 		desc = &ldt->entries[idx];
2379 #else
2380 		return 0;
2381 #endif
2382 	} else {
2383 		if (idx >= GDT_ENTRIES)
2384 			return 0;
2385 
2386 		desc = raw_cpu_ptr(gdt_page.gdt) + idx;
2387 	}
2388 
2389 	return get_desc_base(desc);
2390 }
2391 
2392 #ifdef CONFIG_IA32_EMULATION
2393 
2394 #include <asm/compat.h>
2395 
2396 static inline int
2397 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2398 {
2399 	/* 32-bit process in 64-bit kernel. */
2400 	unsigned long ss_base, cs_base;
2401 	struct stack_frame_ia32 frame;
2402 	const void __user *fp;
2403 
2404 	if (!test_thread_flag(TIF_IA32))
2405 		return 0;
2406 
2407 	cs_base = get_segment_base(regs->cs);
2408 	ss_base = get_segment_base(regs->ss);
2409 
2410 	fp = compat_ptr(ss_base + regs->bp);
2411 	pagefault_disable();
2412 	while (entry->nr < entry->max_stack) {
2413 		unsigned long bytes;
2414 		frame.next_frame     = 0;
2415 		frame.return_address = 0;
2416 
2417 		if (!valid_user_frame(fp, sizeof(frame)))
2418 			break;
2419 
2420 		bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2421 		if (bytes != 0)
2422 			break;
2423 		bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2424 		if (bytes != 0)
2425 			break;
2426 
2427 		perf_callchain_store(entry, cs_base + frame.return_address);
2428 		fp = compat_ptr(ss_base + frame.next_frame);
2429 	}
2430 	pagefault_enable();
2431 	return 1;
2432 }
2433 #else
2434 static inline int
2435 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
2436 {
2437     return 0;
2438 }
2439 #endif
2440 
2441 void
2442 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
2443 {
2444 	struct stack_frame frame;
2445 	const unsigned long __user *fp;
2446 
2447 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2448 		/* TODO: We don't support guest os callchain now */
2449 		return;
2450 	}
2451 
2452 	/*
2453 	 * We don't know what to do with VM86 stacks.. ignore them for now.
2454 	 */
2455 	if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
2456 		return;
2457 
2458 	fp = (unsigned long __user *)regs->bp;
2459 
2460 	perf_callchain_store(entry, regs->ip);
2461 
2462 	if (!current->mm)
2463 		return;
2464 
2465 	if (perf_callchain_user32(regs, entry))
2466 		return;
2467 
2468 	pagefault_disable();
2469 	while (entry->nr < entry->max_stack) {
2470 		unsigned long bytes;
2471 
2472 		frame.next_frame	     = NULL;
2473 		frame.return_address = 0;
2474 
2475 		if (!valid_user_frame(fp, sizeof(frame)))
2476 			break;
2477 
2478 		bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2479 		if (bytes != 0)
2480 			break;
2481 		bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2482 		if (bytes != 0)
2483 			break;
2484 
2485 		perf_callchain_store(entry, frame.return_address);
2486 		fp = (void __user *)frame.next_frame;
2487 	}
2488 	pagefault_enable();
2489 }
2490 
2491 /*
2492  * Deal with code segment offsets for the various execution modes:
2493  *
2494  *   VM86 - the good olde 16 bit days, where the linear address is
2495  *          20 bits and we use regs->ip + 0x10 * regs->cs.
2496  *
2497  *   IA32 - Where we need to look at GDT/LDT segment descriptor tables
2498  *          to figure out what the 32bit base address is.
2499  *
2500  *    X32 - has TIF_X32 set, but is running in x86_64
2501  *
2502  * X86_64 - CS,DS,SS,ES are all zero based.
2503  */
2504 static unsigned long code_segment_base(struct pt_regs *regs)
2505 {
2506 	/*
2507 	 * For IA32 we look at the GDT/LDT segment base to convert the
2508 	 * effective IP to a linear address.
2509 	 */
2510 
2511 #ifdef CONFIG_X86_32
2512 	/*
2513 	 * If we are in VM86 mode, add the segment offset to convert to a
2514 	 * linear address.
2515 	 */
2516 	if (regs->flags & X86_VM_MASK)
2517 		return 0x10 * regs->cs;
2518 
2519 	if (user_mode(regs) && regs->cs != __USER_CS)
2520 		return get_segment_base(regs->cs);
2521 #else
2522 	if (user_mode(regs) && !user_64bit_mode(regs) &&
2523 	    regs->cs != __USER32_CS)
2524 		return get_segment_base(regs->cs);
2525 #endif
2526 	return 0;
2527 }
2528 
2529 unsigned long perf_instruction_pointer(struct pt_regs *regs)
2530 {
2531 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2532 		return perf_guest_cbs->get_guest_ip();
2533 
2534 	return regs->ip + code_segment_base(regs);
2535 }
2536 
2537 unsigned long perf_misc_flags(struct pt_regs *regs)
2538 {
2539 	int misc = 0;
2540 
2541 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2542 		if (perf_guest_cbs->is_user_mode())
2543 			misc |= PERF_RECORD_MISC_GUEST_USER;
2544 		else
2545 			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
2546 	} else {
2547 		if (user_mode(regs))
2548 			misc |= PERF_RECORD_MISC_USER;
2549 		else
2550 			misc |= PERF_RECORD_MISC_KERNEL;
2551 	}
2552 
2553 	if (regs->flags & PERF_EFLAGS_EXACT)
2554 		misc |= PERF_RECORD_MISC_EXACT_IP;
2555 
2556 	return misc;
2557 }
2558 
2559 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
2560 {
2561 	cap->version		= x86_pmu.version;
2562 	cap->num_counters_gp	= x86_pmu.num_counters;
2563 	cap->num_counters_fixed	= x86_pmu.num_counters_fixed;
2564 	cap->bit_width_gp	= x86_pmu.cntval_bits;
2565 	cap->bit_width_fixed	= x86_pmu.cntval_bits;
2566 	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;
2567 	cap->events_mask_len	= x86_pmu.events_mask_len;
2568 }
2569 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);
2570