xref: /openbmc/linux/arch/x86/events/intel/uncore.c (revision 2d96b44f)
1 #include <linux/module.h>
2 
3 #include <asm/cpu_device_id.h>
4 #include <asm/intel-family.h>
5 #include "uncore.h"
6 
7 static struct intel_uncore_type *empty_uncore[] = { NULL, };
8 struct intel_uncore_type **uncore_msr_uncores = empty_uncore;
9 struct intel_uncore_type **uncore_pci_uncores = empty_uncore;
10 
11 static bool pcidrv_registered;
12 struct pci_driver *uncore_pci_driver;
13 /* pci bus to socket mapping */
14 DEFINE_RAW_SPINLOCK(pci2phy_map_lock);
15 struct list_head pci2phy_map_head = LIST_HEAD_INIT(pci2phy_map_head);
16 struct pci_extra_dev *uncore_extra_pci_dev;
17 static int max_packages;
18 
19 /* mask of cpus that collect uncore events */
20 static cpumask_t uncore_cpu_mask;
21 
22 /* constraint for the fixed counter */
23 static struct event_constraint uncore_constraint_fixed =
24 	EVENT_CONSTRAINT(~0ULL, 1 << UNCORE_PMC_IDX_FIXED, ~0ULL);
25 struct event_constraint uncore_constraint_empty =
26 	EVENT_CONSTRAINT(0, 0, 0);
27 
28 MODULE_LICENSE("GPL");
29 
30 static int uncore_pcibus_to_physid(struct pci_bus *bus)
31 {
32 	struct pci2phy_map *map;
33 	int phys_id = -1;
34 
35 	raw_spin_lock(&pci2phy_map_lock);
36 	list_for_each_entry(map, &pci2phy_map_head, list) {
37 		if (map->segment == pci_domain_nr(bus)) {
38 			phys_id = map->pbus_to_physid[bus->number];
39 			break;
40 		}
41 	}
42 	raw_spin_unlock(&pci2phy_map_lock);
43 
44 	return phys_id;
45 }
46 
47 static void uncore_free_pcibus_map(void)
48 {
49 	struct pci2phy_map *map, *tmp;
50 
51 	list_for_each_entry_safe(map, tmp, &pci2phy_map_head, list) {
52 		list_del(&map->list);
53 		kfree(map);
54 	}
55 }
56 
57 struct pci2phy_map *__find_pci2phy_map(int segment)
58 {
59 	struct pci2phy_map *map, *alloc = NULL;
60 	int i;
61 
62 	lockdep_assert_held(&pci2phy_map_lock);
63 
64 lookup:
65 	list_for_each_entry(map, &pci2phy_map_head, list) {
66 		if (map->segment == segment)
67 			goto end;
68 	}
69 
70 	if (!alloc) {
71 		raw_spin_unlock(&pci2phy_map_lock);
72 		alloc = kmalloc(sizeof(struct pci2phy_map), GFP_KERNEL);
73 		raw_spin_lock(&pci2phy_map_lock);
74 
75 		if (!alloc)
76 			return NULL;
77 
78 		goto lookup;
79 	}
80 
81 	map = alloc;
82 	alloc = NULL;
83 	map->segment = segment;
84 	for (i = 0; i < 256; i++)
85 		map->pbus_to_physid[i] = -1;
86 	list_add_tail(&map->list, &pci2phy_map_head);
87 
88 end:
89 	kfree(alloc);
90 	return map;
91 }
92 
93 ssize_t uncore_event_show(struct kobject *kobj,
94 			  struct kobj_attribute *attr, char *buf)
95 {
96 	struct uncore_event_desc *event =
97 		container_of(attr, struct uncore_event_desc, attr);
98 	return sprintf(buf, "%s", event->config);
99 }
100 
101 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
102 {
103 	return pmu->boxes[topology_logical_package_id(cpu)];
104 }
105 
106 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event)
107 {
108 	u64 count;
109 
110 	rdmsrl(event->hw.event_base, count);
111 
112 	return count;
113 }
114 
115 /*
116  * generic get constraint function for shared match/mask registers.
117  */
118 struct event_constraint *
119 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
120 {
121 	struct intel_uncore_extra_reg *er;
122 	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
123 	struct hw_perf_event_extra *reg2 = &event->hw.branch_reg;
124 	unsigned long flags;
125 	bool ok = false;
126 
127 	/*
128 	 * reg->alloc can be set due to existing state, so for fake box we
129 	 * need to ignore this, otherwise we might fail to allocate proper
130 	 * fake state for this extra reg constraint.
131 	 */
132 	if (reg1->idx == EXTRA_REG_NONE ||
133 	    (!uncore_box_is_fake(box) && reg1->alloc))
134 		return NULL;
135 
136 	er = &box->shared_regs[reg1->idx];
137 	raw_spin_lock_irqsave(&er->lock, flags);
138 	if (!atomic_read(&er->ref) ||
139 	    (er->config1 == reg1->config && er->config2 == reg2->config)) {
140 		atomic_inc(&er->ref);
141 		er->config1 = reg1->config;
142 		er->config2 = reg2->config;
143 		ok = true;
144 	}
145 	raw_spin_unlock_irqrestore(&er->lock, flags);
146 
147 	if (ok) {
148 		if (!uncore_box_is_fake(box))
149 			reg1->alloc = 1;
150 		return NULL;
151 	}
152 
153 	return &uncore_constraint_empty;
154 }
155 
156 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
157 {
158 	struct intel_uncore_extra_reg *er;
159 	struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
160 
161 	/*
162 	 * Only put constraint if extra reg was actually allocated. Also
163 	 * takes care of event which do not use an extra shared reg.
164 	 *
165 	 * Also, if this is a fake box we shouldn't touch any event state
166 	 * (reg->alloc) and we don't care about leaving inconsistent box
167 	 * state either since it will be thrown out.
168 	 */
169 	if (uncore_box_is_fake(box) || !reg1->alloc)
170 		return;
171 
172 	er = &box->shared_regs[reg1->idx];
173 	atomic_dec(&er->ref);
174 	reg1->alloc = 0;
175 }
176 
177 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
178 {
179 	struct intel_uncore_extra_reg *er;
180 	unsigned long flags;
181 	u64 config;
182 
183 	er = &box->shared_regs[idx];
184 
185 	raw_spin_lock_irqsave(&er->lock, flags);
186 	config = er->config;
187 	raw_spin_unlock_irqrestore(&er->lock, flags);
188 
189 	return config;
190 }
191 
192 static void uncore_assign_hw_event(struct intel_uncore_box *box,
193 				   struct perf_event *event, int idx)
194 {
195 	struct hw_perf_event *hwc = &event->hw;
196 
197 	hwc->idx = idx;
198 	hwc->last_tag = ++box->tags[idx];
199 
200 	if (hwc->idx == UNCORE_PMC_IDX_FIXED) {
201 		hwc->event_base = uncore_fixed_ctr(box);
202 		hwc->config_base = uncore_fixed_ctl(box);
203 		return;
204 	}
205 
206 	hwc->config_base = uncore_event_ctl(box, hwc->idx);
207 	hwc->event_base  = uncore_perf_ctr(box, hwc->idx);
208 }
209 
210 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event)
211 {
212 	u64 prev_count, new_count, delta;
213 	int shift;
214 
215 	if (event->hw.idx >= UNCORE_PMC_IDX_FIXED)
216 		shift = 64 - uncore_fixed_ctr_bits(box);
217 	else
218 		shift = 64 - uncore_perf_ctr_bits(box);
219 
220 	/* the hrtimer might modify the previous event value */
221 again:
222 	prev_count = local64_read(&event->hw.prev_count);
223 	new_count = uncore_read_counter(box, event);
224 	if (local64_xchg(&event->hw.prev_count, new_count) != prev_count)
225 		goto again;
226 
227 	delta = (new_count << shift) - (prev_count << shift);
228 	delta >>= shift;
229 
230 	local64_add(delta, &event->count);
231 }
232 
233 /*
234  * The overflow interrupt is unavailable for SandyBridge-EP, is broken
235  * for SandyBridge. So we use hrtimer to periodically poll the counter
236  * to avoid overflow.
237  */
238 static enum hrtimer_restart uncore_pmu_hrtimer(struct hrtimer *hrtimer)
239 {
240 	struct intel_uncore_box *box;
241 	struct perf_event *event;
242 	unsigned long flags;
243 	int bit;
244 
245 	box = container_of(hrtimer, struct intel_uncore_box, hrtimer);
246 	if (!box->n_active || box->cpu != smp_processor_id())
247 		return HRTIMER_NORESTART;
248 	/*
249 	 * disable local interrupt to prevent uncore_pmu_event_start/stop
250 	 * to interrupt the update process
251 	 */
252 	local_irq_save(flags);
253 
254 	/*
255 	 * handle boxes with an active event list as opposed to active
256 	 * counters
257 	 */
258 	list_for_each_entry(event, &box->active_list, active_entry) {
259 		uncore_perf_event_update(box, event);
260 	}
261 
262 	for_each_set_bit(bit, box->active_mask, UNCORE_PMC_IDX_MAX)
263 		uncore_perf_event_update(box, box->events[bit]);
264 
265 	local_irq_restore(flags);
266 
267 	hrtimer_forward_now(hrtimer, ns_to_ktime(box->hrtimer_duration));
268 	return HRTIMER_RESTART;
269 }
270 
271 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box)
272 {
273 	hrtimer_start(&box->hrtimer, ns_to_ktime(box->hrtimer_duration),
274 		      HRTIMER_MODE_REL_PINNED);
275 }
276 
277 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box)
278 {
279 	hrtimer_cancel(&box->hrtimer);
280 }
281 
282 static void uncore_pmu_init_hrtimer(struct intel_uncore_box *box)
283 {
284 	hrtimer_init(&box->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
285 	box->hrtimer.function = uncore_pmu_hrtimer;
286 }
287 
288 static struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type,
289 						 int node)
290 {
291 	int i, size, numshared = type->num_shared_regs ;
292 	struct intel_uncore_box *box;
293 
294 	size = sizeof(*box) + numshared * sizeof(struct intel_uncore_extra_reg);
295 
296 	box = kzalloc_node(size, GFP_KERNEL, node);
297 	if (!box)
298 		return NULL;
299 
300 	for (i = 0; i < numshared; i++)
301 		raw_spin_lock_init(&box->shared_regs[i].lock);
302 
303 	uncore_pmu_init_hrtimer(box);
304 	box->cpu = -1;
305 	box->pci_phys_id = -1;
306 	box->pkgid = -1;
307 
308 	/* set default hrtimer timeout */
309 	box->hrtimer_duration = UNCORE_PMU_HRTIMER_INTERVAL;
310 
311 	INIT_LIST_HEAD(&box->active_list);
312 
313 	return box;
314 }
315 
316 /*
317  * Using uncore_pmu_event_init pmu event_init callback
318  * as a detection point for uncore events.
319  */
320 static int uncore_pmu_event_init(struct perf_event *event);
321 
322 static bool is_uncore_event(struct perf_event *event)
323 {
324 	return event->pmu->event_init == uncore_pmu_event_init;
325 }
326 
327 static int
328 uncore_collect_events(struct intel_uncore_box *box, struct perf_event *leader,
329 		      bool dogrp)
330 {
331 	struct perf_event *event;
332 	int n, max_count;
333 
334 	max_count = box->pmu->type->num_counters;
335 	if (box->pmu->type->fixed_ctl)
336 		max_count++;
337 
338 	if (box->n_events >= max_count)
339 		return -EINVAL;
340 
341 	n = box->n_events;
342 
343 	if (is_uncore_event(leader)) {
344 		box->event_list[n] = leader;
345 		n++;
346 	}
347 
348 	if (!dogrp)
349 		return n;
350 
351 	list_for_each_entry(event, &leader->sibling_list, group_entry) {
352 		if (!is_uncore_event(event) ||
353 		    event->state <= PERF_EVENT_STATE_OFF)
354 			continue;
355 
356 		if (n >= max_count)
357 			return -EINVAL;
358 
359 		box->event_list[n] = event;
360 		n++;
361 	}
362 	return n;
363 }
364 
365 static struct event_constraint *
366 uncore_get_event_constraint(struct intel_uncore_box *box, struct perf_event *event)
367 {
368 	struct intel_uncore_type *type = box->pmu->type;
369 	struct event_constraint *c;
370 
371 	if (type->ops->get_constraint) {
372 		c = type->ops->get_constraint(box, event);
373 		if (c)
374 			return c;
375 	}
376 
377 	if (event->attr.config == UNCORE_FIXED_EVENT)
378 		return &uncore_constraint_fixed;
379 
380 	if (type->constraints) {
381 		for_each_event_constraint(c, type->constraints) {
382 			if ((event->hw.config & c->cmask) == c->code)
383 				return c;
384 		}
385 	}
386 
387 	return &type->unconstrainted;
388 }
389 
390 static void uncore_put_event_constraint(struct intel_uncore_box *box,
391 					struct perf_event *event)
392 {
393 	if (box->pmu->type->ops->put_constraint)
394 		box->pmu->type->ops->put_constraint(box, event);
395 }
396 
397 static int uncore_assign_events(struct intel_uncore_box *box, int assign[], int n)
398 {
399 	unsigned long used_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
400 	struct event_constraint *c;
401 	int i, wmin, wmax, ret = 0;
402 	struct hw_perf_event *hwc;
403 
404 	bitmap_zero(used_mask, UNCORE_PMC_IDX_MAX);
405 
406 	for (i = 0, wmin = UNCORE_PMC_IDX_MAX, wmax = 0; i < n; i++) {
407 		c = uncore_get_event_constraint(box, box->event_list[i]);
408 		box->event_constraint[i] = c;
409 		wmin = min(wmin, c->weight);
410 		wmax = max(wmax, c->weight);
411 	}
412 
413 	/* fastpath, try to reuse previous register */
414 	for (i = 0; i < n; i++) {
415 		hwc = &box->event_list[i]->hw;
416 		c = box->event_constraint[i];
417 
418 		/* never assigned */
419 		if (hwc->idx == -1)
420 			break;
421 
422 		/* constraint still honored */
423 		if (!test_bit(hwc->idx, c->idxmsk))
424 			break;
425 
426 		/* not already used */
427 		if (test_bit(hwc->idx, used_mask))
428 			break;
429 
430 		__set_bit(hwc->idx, used_mask);
431 		if (assign)
432 			assign[i] = hwc->idx;
433 	}
434 	/* slow path */
435 	if (i != n)
436 		ret = perf_assign_events(box->event_constraint, n,
437 					 wmin, wmax, n, assign);
438 
439 	if (!assign || ret) {
440 		for (i = 0; i < n; i++)
441 			uncore_put_event_constraint(box, box->event_list[i]);
442 	}
443 	return ret ? -EINVAL : 0;
444 }
445 
446 static void uncore_pmu_event_start(struct perf_event *event, int flags)
447 {
448 	struct intel_uncore_box *box = uncore_event_to_box(event);
449 	int idx = event->hw.idx;
450 
451 	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
452 		return;
453 
454 	if (WARN_ON_ONCE(idx == -1 || idx >= UNCORE_PMC_IDX_MAX))
455 		return;
456 
457 	event->hw.state = 0;
458 	box->events[idx] = event;
459 	box->n_active++;
460 	__set_bit(idx, box->active_mask);
461 
462 	local64_set(&event->hw.prev_count, uncore_read_counter(box, event));
463 	uncore_enable_event(box, event);
464 
465 	if (box->n_active == 1) {
466 		uncore_enable_box(box);
467 		uncore_pmu_start_hrtimer(box);
468 	}
469 }
470 
471 static void uncore_pmu_event_stop(struct perf_event *event, int flags)
472 {
473 	struct intel_uncore_box *box = uncore_event_to_box(event);
474 	struct hw_perf_event *hwc = &event->hw;
475 
476 	if (__test_and_clear_bit(hwc->idx, box->active_mask)) {
477 		uncore_disable_event(box, event);
478 		box->n_active--;
479 		box->events[hwc->idx] = NULL;
480 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
481 		hwc->state |= PERF_HES_STOPPED;
482 
483 		if (box->n_active == 0) {
484 			uncore_disable_box(box);
485 			uncore_pmu_cancel_hrtimer(box);
486 		}
487 	}
488 
489 	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
490 		/*
491 		 * Drain the remaining delta count out of a event
492 		 * that we are disabling:
493 		 */
494 		uncore_perf_event_update(box, event);
495 		hwc->state |= PERF_HES_UPTODATE;
496 	}
497 }
498 
499 static int uncore_pmu_event_add(struct perf_event *event, int flags)
500 {
501 	struct intel_uncore_box *box = uncore_event_to_box(event);
502 	struct hw_perf_event *hwc = &event->hw;
503 	int assign[UNCORE_PMC_IDX_MAX];
504 	int i, n, ret;
505 
506 	if (!box)
507 		return -ENODEV;
508 
509 	ret = n = uncore_collect_events(box, event, false);
510 	if (ret < 0)
511 		return ret;
512 
513 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
514 	if (!(flags & PERF_EF_START))
515 		hwc->state |= PERF_HES_ARCH;
516 
517 	ret = uncore_assign_events(box, assign, n);
518 	if (ret)
519 		return ret;
520 
521 	/* save events moving to new counters */
522 	for (i = 0; i < box->n_events; i++) {
523 		event = box->event_list[i];
524 		hwc = &event->hw;
525 
526 		if (hwc->idx == assign[i] &&
527 			hwc->last_tag == box->tags[assign[i]])
528 			continue;
529 		/*
530 		 * Ensure we don't accidentally enable a stopped
531 		 * counter simply because we rescheduled.
532 		 */
533 		if (hwc->state & PERF_HES_STOPPED)
534 			hwc->state |= PERF_HES_ARCH;
535 
536 		uncore_pmu_event_stop(event, PERF_EF_UPDATE);
537 	}
538 
539 	/* reprogram moved events into new counters */
540 	for (i = 0; i < n; i++) {
541 		event = box->event_list[i];
542 		hwc = &event->hw;
543 
544 		if (hwc->idx != assign[i] ||
545 			hwc->last_tag != box->tags[assign[i]])
546 			uncore_assign_hw_event(box, event, assign[i]);
547 		else if (i < box->n_events)
548 			continue;
549 
550 		if (hwc->state & PERF_HES_ARCH)
551 			continue;
552 
553 		uncore_pmu_event_start(event, 0);
554 	}
555 	box->n_events = n;
556 
557 	return 0;
558 }
559 
560 static void uncore_pmu_event_del(struct perf_event *event, int flags)
561 {
562 	struct intel_uncore_box *box = uncore_event_to_box(event);
563 	int i;
564 
565 	uncore_pmu_event_stop(event, PERF_EF_UPDATE);
566 
567 	for (i = 0; i < box->n_events; i++) {
568 		if (event == box->event_list[i]) {
569 			uncore_put_event_constraint(box, event);
570 
571 			for (++i; i < box->n_events; i++)
572 				box->event_list[i - 1] = box->event_list[i];
573 
574 			--box->n_events;
575 			break;
576 		}
577 	}
578 
579 	event->hw.idx = -1;
580 	event->hw.last_tag = ~0ULL;
581 }
582 
583 void uncore_pmu_event_read(struct perf_event *event)
584 {
585 	struct intel_uncore_box *box = uncore_event_to_box(event);
586 	uncore_perf_event_update(box, event);
587 }
588 
589 /*
590  * validation ensures the group can be loaded onto the
591  * PMU if it was the only group available.
592  */
593 static int uncore_validate_group(struct intel_uncore_pmu *pmu,
594 				struct perf_event *event)
595 {
596 	struct perf_event *leader = event->group_leader;
597 	struct intel_uncore_box *fake_box;
598 	int ret = -EINVAL, n;
599 
600 	fake_box = uncore_alloc_box(pmu->type, NUMA_NO_NODE);
601 	if (!fake_box)
602 		return -ENOMEM;
603 
604 	fake_box->pmu = pmu;
605 	/*
606 	 * the event is not yet connected with its
607 	 * siblings therefore we must first collect
608 	 * existing siblings, then add the new event
609 	 * before we can simulate the scheduling
610 	 */
611 	n = uncore_collect_events(fake_box, leader, true);
612 	if (n < 0)
613 		goto out;
614 
615 	fake_box->n_events = n;
616 	n = uncore_collect_events(fake_box, event, false);
617 	if (n < 0)
618 		goto out;
619 
620 	fake_box->n_events = n;
621 
622 	ret = uncore_assign_events(fake_box, NULL, n);
623 out:
624 	kfree(fake_box);
625 	return ret;
626 }
627 
628 static int uncore_pmu_event_init(struct perf_event *event)
629 {
630 	struct intel_uncore_pmu *pmu;
631 	struct intel_uncore_box *box;
632 	struct hw_perf_event *hwc = &event->hw;
633 	int ret;
634 
635 	if (event->attr.type != event->pmu->type)
636 		return -ENOENT;
637 
638 	pmu = uncore_event_to_pmu(event);
639 	/* no device found for this pmu */
640 	if (pmu->func_id < 0)
641 		return -ENOENT;
642 
643 	/*
644 	 * Uncore PMU does measure at all privilege level all the time.
645 	 * So it doesn't make sense to specify any exclude bits.
646 	 */
647 	if (event->attr.exclude_user || event->attr.exclude_kernel ||
648 			event->attr.exclude_hv || event->attr.exclude_idle)
649 		return -EINVAL;
650 
651 	/* Sampling not supported yet */
652 	if (hwc->sample_period)
653 		return -EINVAL;
654 
655 	/*
656 	 * Place all uncore events for a particular physical package
657 	 * onto a single cpu
658 	 */
659 	if (event->cpu < 0)
660 		return -EINVAL;
661 	box = uncore_pmu_to_box(pmu, event->cpu);
662 	if (!box || box->cpu < 0)
663 		return -EINVAL;
664 	event->cpu = box->cpu;
665 	event->pmu_private = box;
666 
667 	event->event_caps |= PERF_EV_CAP_READ_ACTIVE_PKG;
668 
669 	event->hw.idx = -1;
670 	event->hw.last_tag = ~0ULL;
671 	event->hw.extra_reg.idx = EXTRA_REG_NONE;
672 	event->hw.branch_reg.idx = EXTRA_REG_NONE;
673 
674 	if (event->attr.config == UNCORE_FIXED_EVENT) {
675 		/* no fixed counter */
676 		if (!pmu->type->fixed_ctl)
677 			return -EINVAL;
678 		/*
679 		 * if there is only one fixed counter, only the first pmu
680 		 * can access the fixed counter
681 		 */
682 		if (pmu->type->single_fixed && pmu->pmu_idx > 0)
683 			return -EINVAL;
684 
685 		/* fixed counters have event field hardcoded to zero */
686 		hwc->config = 0ULL;
687 	} else {
688 		hwc->config = event->attr.config &
689 			      (pmu->type->event_mask | ((u64)pmu->type->event_mask_ext << 32));
690 		if (pmu->type->ops->hw_config) {
691 			ret = pmu->type->ops->hw_config(box, event);
692 			if (ret)
693 				return ret;
694 		}
695 	}
696 
697 	if (event->group_leader != event)
698 		ret = uncore_validate_group(pmu, event);
699 	else
700 		ret = 0;
701 
702 	return ret;
703 }
704 
705 static ssize_t uncore_get_attr_cpumask(struct device *dev,
706 				struct device_attribute *attr, char *buf)
707 {
708 	return cpumap_print_to_pagebuf(true, buf, &uncore_cpu_mask);
709 }
710 
711 static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
712 
713 static struct attribute *uncore_pmu_attrs[] = {
714 	&dev_attr_cpumask.attr,
715 	NULL,
716 };
717 
718 static struct attribute_group uncore_pmu_attr_group = {
719 	.attrs = uncore_pmu_attrs,
720 };
721 
722 static int uncore_pmu_register(struct intel_uncore_pmu *pmu)
723 {
724 	int ret;
725 
726 	if (!pmu->type->pmu) {
727 		pmu->pmu = (struct pmu) {
728 			.attr_groups	= pmu->type->attr_groups,
729 			.task_ctx_nr	= perf_invalid_context,
730 			.event_init	= uncore_pmu_event_init,
731 			.add		= uncore_pmu_event_add,
732 			.del		= uncore_pmu_event_del,
733 			.start		= uncore_pmu_event_start,
734 			.stop		= uncore_pmu_event_stop,
735 			.read		= uncore_pmu_event_read,
736 		};
737 	} else {
738 		pmu->pmu = *pmu->type->pmu;
739 		pmu->pmu.attr_groups = pmu->type->attr_groups;
740 	}
741 
742 	if (pmu->type->num_boxes == 1) {
743 		if (strlen(pmu->type->name) > 0)
744 			sprintf(pmu->name, "uncore_%s", pmu->type->name);
745 		else
746 			sprintf(pmu->name, "uncore");
747 	} else {
748 		sprintf(pmu->name, "uncore_%s_%d", pmu->type->name,
749 			pmu->pmu_idx);
750 	}
751 
752 	ret = perf_pmu_register(&pmu->pmu, pmu->name, -1);
753 	if (!ret)
754 		pmu->registered = true;
755 	return ret;
756 }
757 
758 static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu)
759 {
760 	if (!pmu->registered)
761 		return;
762 	perf_pmu_unregister(&pmu->pmu);
763 	pmu->registered = false;
764 }
765 
766 static void __uncore_exit_boxes(struct intel_uncore_type *type, int cpu)
767 {
768 	struct intel_uncore_pmu *pmu = type->pmus;
769 	struct intel_uncore_box *box;
770 	int i, pkg;
771 
772 	if (pmu) {
773 		pkg = topology_physical_package_id(cpu);
774 		for (i = 0; i < type->num_boxes; i++, pmu++) {
775 			box = pmu->boxes[pkg];
776 			if (box)
777 				uncore_box_exit(box);
778 		}
779 	}
780 }
781 
782 static void uncore_exit_boxes(void *dummy)
783 {
784 	struct intel_uncore_type **types;
785 
786 	for (types = uncore_msr_uncores; *types; types++)
787 		__uncore_exit_boxes(*types++, smp_processor_id());
788 }
789 
790 static void uncore_free_boxes(struct intel_uncore_pmu *pmu)
791 {
792 	int pkg;
793 
794 	for (pkg = 0; pkg < max_packages; pkg++)
795 		kfree(pmu->boxes[pkg]);
796 	kfree(pmu->boxes);
797 }
798 
799 static void uncore_type_exit(struct intel_uncore_type *type)
800 {
801 	struct intel_uncore_pmu *pmu = type->pmus;
802 	int i;
803 
804 	if (pmu) {
805 		for (i = 0; i < type->num_boxes; i++, pmu++) {
806 			uncore_pmu_unregister(pmu);
807 			uncore_free_boxes(pmu);
808 		}
809 		kfree(type->pmus);
810 		type->pmus = NULL;
811 	}
812 	kfree(type->events_group);
813 	type->events_group = NULL;
814 }
815 
816 static void uncore_types_exit(struct intel_uncore_type **types)
817 {
818 	for (; *types; types++)
819 		uncore_type_exit(*types);
820 }
821 
822 static int __init uncore_type_init(struct intel_uncore_type *type, bool setid)
823 {
824 	struct intel_uncore_pmu *pmus;
825 	struct attribute_group *attr_group;
826 	struct attribute **attrs;
827 	size_t size;
828 	int i, j;
829 
830 	pmus = kzalloc(sizeof(*pmus) * type->num_boxes, GFP_KERNEL);
831 	if (!pmus)
832 		return -ENOMEM;
833 
834 	size = max_packages * sizeof(struct intel_uncore_box *);
835 
836 	for (i = 0; i < type->num_boxes; i++) {
837 		pmus[i].func_id	= setid ? i : -1;
838 		pmus[i].pmu_idx	= i;
839 		pmus[i].type	= type;
840 		pmus[i].boxes	= kzalloc(size, GFP_KERNEL);
841 		if (!pmus[i].boxes)
842 			return -ENOMEM;
843 	}
844 
845 	type->pmus = pmus;
846 	type->unconstrainted = (struct event_constraint)
847 		__EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
848 				0, type->num_counters, 0, 0);
849 
850 	if (type->event_descs) {
851 		for (i = 0; type->event_descs[i].attr.attr.name; i++);
852 
853 		attr_group = kzalloc(sizeof(struct attribute *) * (i + 1) +
854 					sizeof(*attr_group), GFP_KERNEL);
855 		if (!attr_group)
856 			return -ENOMEM;
857 
858 		attrs = (struct attribute **)(attr_group + 1);
859 		attr_group->name = "events";
860 		attr_group->attrs = attrs;
861 
862 		for (j = 0; j < i; j++)
863 			attrs[j] = &type->event_descs[j].attr.attr;
864 
865 		type->events_group = attr_group;
866 	}
867 
868 	type->pmu_group = &uncore_pmu_attr_group;
869 	return 0;
870 }
871 
872 static int __init
873 uncore_types_init(struct intel_uncore_type **types, bool setid)
874 {
875 	int ret;
876 
877 	for (; *types; types++) {
878 		ret = uncore_type_init(*types, setid);
879 		if (ret)
880 			return ret;
881 	}
882 	return 0;
883 }
884 
885 /*
886  * add a pci uncore device
887  */
888 static int uncore_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
889 {
890 	struct intel_uncore_type *type;
891 	struct intel_uncore_pmu *pmu = NULL;
892 	struct intel_uncore_box *box;
893 	int phys_id, pkg, ret;
894 
895 	phys_id = uncore_pcibus_to_physid(pdev->bus);
896 	if (phys_id < 0)
897 		return -ENODEV;
898 
899 	pkg = topology_phys_to_logical_pkg(phys_id);
900 	if (pkg < 0)
901 		return -EINVAL;
902 
903 	if (UNCORE_PCI_DEV_TYPE(id->driver_data) == UNCORE_EXTRA_PCI_DEV) {
904 		int idx = UNCORE_PCI_DEV_IDX(id->driver_data);
905 
906 		uncore_extra_pci_dev[pkg].dev[idx] = pdev;
907 		pci_set_drvdata(pdev, NULL);
908 		return 0;
909 	}
910 
911 	type = uncore_pci_uncores[UNCORE_PCI_DEV_TYPE(id->driver_data)];
912 
913 	/*
914 	 * Some platforms, e.g.  Knights Landing, use a common PCI device ID
915 	 * for multiple instances of an uncore PMU device type. We should check
916 	 * PCI slot and func to indicate the uncore box.
917 	 */
918 	if (id->driver_data & ~0xffff) {
919 		struct pci_driver *pci_drv = pdev->driver;
920 		const struct pci_device_id *ids = pci_drv->id_table;
921 		unsigned int devfn;
922 
923 		while (ids && ids->vendor) {
924 			if ((ids->vendor == pdev->vendor) &&
925 			    (ids->device == pdev->device)) {
926 				devfn = PCI_DEVFN(UNCORE_PCI_DEV_DEV(ids->driver_data),
927 						  UNCORE_PCI_DEV_FUNC(ids->driver_data));
928 				if (devfn == pdev->devfn) {
929 					pmu = &type->pmus[UNCORE_PCI_DEV_IDX(ids->driver_data)];
930 					break;
931 				}
932 			}
933 			ids++;
934 		}
935 		if (pmu == NULL)
936 			return -ENODEV;
937 	} else {
938 		/*
939 		 * for performance monitoring unit with multiple boxes,
940 		 * each box has a different function id.
941 		 */
942 		pmu = &type->pmus[UNCORE_PCI_DEV_IDX(id->driver_data)];
943 	}
944 
945 	if (WARN_ON_ONCE(pmu->boxes[pkg] != NULL))
946 		return -EINVAL;
947 
948 	box = uncore_alloc_box(type, NUMA_NO_NODE);
949 	if (!box)
950 		return -ENOMEM;
951 
952 	if (pmu->func_id < 0)
953 		pmu->func_id = pdev->devfn;
954 	else
955 		WARN_ON_ONCE(pmu->func_id != pdev->devfn);
956 
957 	atomic_inc(&box->refcnt);
958 	box->pci_phys_id = phys_id;
959 	box->pkgid = pkg;
960 	box->pci_dev = pdev;
961 	box->pmu = pmu;
962 	uncore_box_init(box);
963 	pci_set_drvdata(pdev, box);
964 
965 	pmu->boxes[pkg] = box;
966 	if (atomic_inc_return(&pmu->activeboxes) > 1)
967 		return 0;
968 
969 	/* First active box registers the pmu */
970 	ret = uncore_pmu_register(pmu);
971 	if (ret) {
972 		pci_set_drvdata(pdev, NULL);
973 		pmu->boxes[pkg] = NULL;
974 		uncore_box_exit(box);
975 		kfree(box);
976 	}
977 	return ret;
978 }
979 
980 static void uncore_pci_remove(struct pci_dev *pdev)
981 {
982 	struct intel_uncore_box *box;
983 	struct intel_uncore_pmu *pmu;
984 	int i, phys_id, pkg;
985 
986 	phys_id = uncore_pcibus_to_physid(pdev->bus);
987 	pkg = topology_phys_to_logical_pkg(phys_id);
988 
989 	box = pci_get_drvdata(pdev);
990 	if (!box) {
991 		for (i = 0; i < UNCORE_EXTRA_PCI_DEV_MAX; i++) {
992 			if (uncore_extra_pci_dev[pkg].dev[i] == pdev) {
993 				uncore_extra_pci_dev[pkg].dev[i] = NULL;
994 				break;
995 			}
996 		}
997 		WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX);
998 		return;
999 	}
1000 
1001 	pmu = box->pmu;
1002 	if (WARN_ON_ONCE(phys_id != box->pci_phys_id))
1003 		return;
1004 
1005 	pci_set_drvdata(pdev, NULL);
1006 	pmu->boxes[pkg] = NULL;
1007 	if (atomic_dec_return(&pmu->activeboxes) == 0)
1008 		uncore_pmu_unregister(pmu);
1009 	uncore_box_exit(box);
1010 	kfree(box);
1011 }
1012 
1013 static int __init uncore_pci_init(void)
1014 {
1015 	size_t size;
1016 	int ret;
1017 
1018 	size = max_packages * sizeof(struct pci_extra_dev);
1019 	uncore_extra_pci_dev = kzalloc(size, GFP_KERNEL);
1020 	if (!uncore_extra_pci_dev) {
1021 		ret = -ENOMEM;
1022 		goto err;
1023 	}
1024 
1025 	ret = uncore_types_init(uncore_pci_uncores, false);
1026 	if (ret)
1027 		goto errtype;
1028 
1029 	uncore_pci_driver->probe = uncore_pci_probe;
1030 	uncore_pci_driver->remove = uncore_pci_remove;
1031 
1032 	ret = pci_register_driver(uncore_pci_driver);
1033 	if (ret)
1034 		goto errtype;
1035 
1036 	pcidrv_registered = true;
1037 	return 0;
1038 
1039 errtype:
1040 	uncore_types_exit(uncore_pci_uncores);
1041 	kfree(uncore_extra_pci_dev);
1042 	uncore_extra_pci_dev = NULL;
1043 	uncore_free_pcibus_map();
1044 err:
1045 	uncore_pci_uncores = empty_uncore;
1046 	return ret;
1047 }
1048 
1049 static void uncore_pci_exit(void)
1050 {
1051 	if (pcidrv_registered) {
1052 		pcidrv_registered = false;
1053 		pci_unregister_driver(uncore_pci_driver);
1054 		uncore_types_exit(uncore_pci_uncores);
1055 		kfree(uncore_extra_pci_dev);
1056 		uncore_free_pcibus_map();
1057 	}
1058 }
1059 
1060 static int uncore_cpu_dying(unsigned int cpu)
1061 {
1062 	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1063 	struct intel_uncore_pmu *pmu;
1064 	struct intel_uncore_box *box;
1065 	int i, pkg;
1066 
1067 	pkg = topology_logical_package_id(cpu);
1068 	for (; *types; types++) {
1069 		type = *types;
1070 		pmu = type->pmus;
1071 		for (i = 0; i < type->num_boxes; i++, pmu++) {
1072 			box = pmu->boxes[pkg];
1073 			if (box && atomic_dec_return(&box->refcnt) == 0)
1074 				uncore_box_exit(box);
1075 		}
1076 	}
1077 	return 0;
1078 }
1079 
1080 static int first_init;
1081 
1082 static int uncore_cpu_starting(unsigned int cpu)
1083 {
1084 	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1085 	struct intel_uncore_pmu *pmu;
1086 	struct intel_uncore_box *box;
1087 	int i, pkg, ncpus = 1;
1088 
1089 	if (first_init) {
1090 		/*
1091 		 * On init we get the number of online cpus in the package
1092 		 * and set refcount for all of them.
1093 		 */
1094 		ncpus = cpumask_weight(topology_core_cpumask(cpu));
1095 	}
1096 
1097 	pkg = topology_logical_package_id(cpu);
1098 	for (; *types; types++) {
1099 		type = *types;
1100 		pmu = type->pmus;
1101 		for (i = 0; i < type->num_boxes; i++, pmu++) {
1102 			box = pmu->boxes[pkg];
1103 			if (!box)
1104 				continue;
1105 			/* The first cpu on a package activates the box */
1106 			if (atomic_add_return(ncpus, &box->refcnt) == ncpus)
1107 				uncore_box_init(box);
1108 		}
1109 	}
1110 
1111 	return 0;
1112 }
1113 
1114 static int uncore_cpu_prepare(unsigned int cpu)
1115 {
1116 	struct intel_uncore_type *type, **types = uncore_msr_uncores;
1117 	struct intel_uncore_pmu *pmu;
1118 	struct intel_uncore_box *box;
1119 	int i, pkg;
1120 
1121 	pkg = topology_logical_package_id(cpu);
1122 	for (; *types; types++) {
1123 		type = *types;
1124 		pmu = type->pmus;
1125 		for (i = 0; i < type->num_boxes; i++, pmu++) {
1126 			if (pmu->boxes[pkg])
1127 				continue;
1128 			/* First cpu of a package allocates the box */
1129 			box = uncore_alloc_box(type, cpu_to_node(cpu));
1130 			if (!box)
1131 				return -ENOMEM;
1132 			box->pmu = pmu;
1133 			box->pkgid = pkg;
1134 			pmu->boxes[pkg] = box;
1135 		}
1136 	}
1137 	return 0;
1138 }
1139 
1140 static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu,
1141 				   int new_cpu)
1142 {
1143 	struct intel_uncore_pmu *pmu = type->pmus;
1144 	struct intel_uncore_box *box;
1145 	int i, pkg;
1146 
1147 	pkg = topology_logical_package_id(old_cpu < 0 ? new_cpu : old_cpu);
1148 	for (i = 0; i < type->num_boxes; i++, pmu++) {
1149 		box = pmu->boxes[pkg];
1150 		if (!box)
1151 			continue;
1152 
1153 		if (old_cpu < 0) {
1154 			WARN_ON_ONCE(box->cpu != -1);
1155 			box->cpu = new_cpu;
1156 			continue;
1157 		}
1158 
1159 		WARN_ON_ONCE(box->cpu != old_cpu);
1160 		box->cpu = -1;
1161 		if (new_cpu < 0)
1162 			continue;
1163 
1164 		uncore_pmu_cancel_hrtimer(box);
1165 		perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu);
1166 		box->cpu = new_cpu;
1167 	}
1168 }
1169 
1170 static void uncore_change_context(struct intel_uncore_type **uncores,
1171 				  int old_cpu, int new_cpu)
1172 {
1173 	for (; *uncores; uncores++)
1174 		uncore_change_type_ctx(*uncores, old_cpu, new_cpu);
1175 }
1176 
1177 static int uncore_event_cpu_offline(unsigned int cpu)
1178 {
1179 	int target;
1180 
1181 	/* Check if exiting cpu is used for collecting uncore events */
1182 	if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
1183 		return 0;
1184 
1185 	/* Find a new cpu to collect uncore events */
1186 	target = cpumask_any_but(topology_core_cpumask(cpu), cpu);
1187 
1188 	/* Migrate uncore events to the new target */
1189 	if (target < nr_cpu_ids)
1190 		cpumask_set_cpu(target, &uncore_cpu_mask);
1191 	else
1192 		target = -1;
1193 
1194 	uncore_change_context(uncore_msr_uncores, cpu, target);
1195 	uncore_change_context(uncore_pci_uncores, cpu, target);
1196 	return 0;
1197 }
1198 
1199 static int uncore_event_cpu_online(unsigned int cpu)
1200 {
1201 	int target;
1202 
1203 	/*
1204 	 * Check if there is an online cpu in the package
1205 	 * which collects uncore events already.
1206 	 */
1207 	target = cpumask_any_and(&uncore_cpu_mask, topology_core_cpumask(cpu));
1208 	if (target < nr_cpu_ids)
1209 		return 0;
1210 
1211 	cpumask_set_cpu(cpu, &uncore_cpu_mask);
1212 
1213 	uncore_change_context(uncore_msr_uncores, -1, cpu);
1214 	uncore_change_context(uncore_pci_uncores, -1, cpu);
1215 	return 0;
1216 }
1217 
1218 static int __init type_pmu_register(struct intel_uncore_type *type)
1219 {
1220 	int i, ret;
1221 
1222 	for (i = 0; i < type->num_boxes; i++) {
1223 		ret = uncore_pmu_register(&type->pmus[i]);
1224 		if (ret)
1225 			return ret;
1226 	}
1227 	return 0;
1228 }
1229 
1230 static int __init uncore_msr_pmus_register(void)
1231 {
1232 	struct intel_uncore_type **types = uncore_msr_uncores;
1233 	int ret;
1234 
1235 	for (; *types; types++) {
1236 		ret = type_pmu_register(*types);
1237 		if (ret)
1238 			return ret;
1239 	}
1240 	return 0;
1241 }
1242 
1243 static int __init uncore_cpu_init(void)
1244 {
1245 	int ret;
1246 
1247 	ret = uncore_types_init(uncore_msr_uncores, true);
1248 	if (ret)
1249 		goto err;
1250 
1251 	ret = uncore_msr_pmus_register();
1252 	if (ret)
1253 		goto err;
1254 	return 0;
1255 err:
1256 	uncore_types_exit(uncore_msr_uncores);
1257 	uncore_msr_uncores = empty_uncore;
1258 	return ret;
1259 }
1260 
1261 #define X86_UNCORE_MODEL_MATCH(model, init)	\
1262 	{ X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&init }
1263 
1264 struct intel_uncore_init_fun {
1265 	void	(*cpu_init)(void);
1266 	int	(*pci_init)(void);
1267 };
1268 
1269 static const struct intel_uncore_init_fun nhm_uncore_init __initconst = {
1270 	.cpu_init = nhm_uncore_cpu_init,
1271 };
1272 
1273 static const struct intel_uncore_init_fun snb_uncore_init __initconst = {
1274 	.cpu_init = snb_uncore_cpu_init,
1275 	.pci_init = snb_uncore_pci_init,
1276 };
1277 
1278 static const struct intel_uncore_init_fun ivb_uncore_init __initconst = {
1279 	.cpu_init = snb_uncore_cpu_init,
1280 	.pci_init = ivb_uncore_pci_init,
1281 };
1282 
1283 static const struct intel_uncore_init_fun hsw_uncore_init __initconst = {
1284 	.cpu_init = snb_uncore_cpu_init,
1285 	.pci_init = hsw_uncore_pci_init,
1286 };
1287 
1288 static const struct intel_uncore_init_fun bdw_uncore_init __initconst = {
1289 	.cpu_init = snb_uncore_cpu_init,
1290 	.pci_init = bdw_uncore_pci_init,
1291 };
1292 
1293 static const struct intel_uncore_init_fun snbep_uncore_init __initconst = {
1294 	.cpu_init = snbep_uncore_cpu_init,
1295 	.pci_init = snbep_uncore_pci_init,
1296 };
1297 
1298 static const struct intel_uncore_init_fun nhmex_uncore_init __initconst = {
1299 	.cpu_init = nhmex_uncore_cpu_init,
1300 };
1301 
1302 static const struct intel_uncore_init_fun ivbep_uncore_init __initconst = {
1303 	.cpu_init = ivbep_uncore_cpu_init,
1304 	.pci_init = ivbep_uncore_pci_init,
1305 };
1306 
1307 static const struct intel_uncore_init_fun hswep_uncore_init __initconst = {
1308 	.cpu_init = hswep_uncore_cpu_init,
1309 	.pci_init = hswep_uncore_pci_init,
1310 };
1311 
1312 static const struct intel_uncore_init_fun bdx_uncore_init __initconst = {
1313 	.cpu_init = bdx_uncore_cpu_init,
1314 	.pci_init = bdx_uncore_pci_init,
1315 };
1316 
1317 static const struct intel_uncore_init_fun knl_uncore_init __initconst = {
1318 	.cpu_init = knl_uncore_cpu_init,
1319 	.pci_init = knl_uncore_pci_init,
1320 };
1321 
1322 static const struct intel_uncore_init_fun skl_uncore_init __initconst = {
1323 	.cpu_init = skl_uncore_cpu_init,
1324 	.pci_init = skl_uncore_pci_init,
1325 };
1326 
1327 static const struct intel_uncore_init_fun skx_uncore_init __initconst = {
1328 	.cpu_init = skx_uncore_cpu_init,
1329 	.pci_init = skx_uncore_pci_init,
1330 };
1331 
1332 static const struct x86_cpu_id intel_uncore_match[] __initconst = {
1333 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EP,	  nhm_uncore_init),
1334 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM,	  nhm_uncore_init),
1335 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE,	  nhm_uncore_init),
1336 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EP,	  nhm_uncore_init),
1337 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE,	  snb_uncore_init),
1338 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE,	  ivb_uncore_init),
1339 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_CORE,	  hsw_uncore_init),
1340 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_ULT,	  hsw_uncore_init),
1341 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_GT3E,	  hsw_uncore_init),
1342 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_CORE, bdw_uncore_init),
1343 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_GT3E, bdw_uncore_init),
1344 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SANDYBRIDGE_X,  snbep_uncore_init),
1345 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_NEHALEM_EX,	  nhmex_uncore_init),
1346 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_WESTMERE_EX,	  nhmex_uncore_init),
1347 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_IVYBRIDGE_X,	  ivbep_uncore_init),
1348 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_HASWELL_X,	  hswep_uncore_init),
1349 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_X,	  bdx_uncore_init),
1350 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_BROADWELL_XEON_D, bdx_uncore_init),
1351 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_XEON_PHI_KNL,	  knl_uncore_init),
1352 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_DESKTOP,skl_uncore_init),
1353 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_MOBILE, skl_uncore_init),
1354 	X86_UNCORE_MODEL_MATCH(INTEL_FAM6_SKYLAKE_X,      skx_uncore_init),
1355 	{},
1356 };
1357 
1358 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_match);
1359 
1360 static int __init intel_uncore_init(void)
1361 {
1362 	const struct x86_cpu_id *id;
1363 	struct intel_uncore_init_fun *uncore_init;
1364 	int pret = 0, cret = 0, ret;
1365 
1366 	id = x86_match_cpu(intel_uncore_match);
1367 	if (!id)
1368 		return -ENODEV;
1369 
1370 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
1371 		return -ENODEV;
1372 
1373 	max_packages = topology_max_packages();
1374 
1375 	uncore_init = (struct intel_uncore_init_fun *)id->driver_data;
1376 	if (uncore_init->pci_init) {
1377 		pret = uncore_init->pci_init();
1378 		if (!pret)
1379 			pret = uncore_pci_init();
1380 	}
1381 
1382 	if (uncore_init->cpu_init) {
1383 		uncore_init->cpu_init();
1384 		cret = uncore_cpu_init();
1385 	}
1386 
1387 	if (cret && pret)
1388 		return -ENODEV;
1389 
1390 	/*
1391 	 * Install callbacks. Core will call them for each online cpu.
1392 	 *
1393 	 * The first online cpu of each package allocates and takes
1394 	 * the refcounts for all other online cpus in that package.
1395 	 * If msrs are not enabled no allocation is required and
1396 	 * uncore_cpu_prepare() is not called for each online cpu.
1397 	 */
1398 	if (!cret) {
1399 	       ret = cpuhp_setup_state(CPUHP_PERF_X86_UNCORE_PREP,
1400 					"PERF_X86_UNCORE_PREP",
1401 					uncore_cpu_prepare, NULL);
1402 		if (ret)
1403 			goto err;
1404 	} else {
1405 		cpuhp_setup_state_nocalls(CPUHP_PERF_X86_UNCORE_PREP,
1406 					  "PERF_X86_UNCORE_PREP",
1407 					  uncore_cpu_prepare, NULL);
1408 	}
1409 	first_init = 1;
1410 	cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_STARTING,
1411 			  "AP_PERF_X86_UNCORE_STARTING",
1412 			  uncore_cpu_starting, uncore_cpu_dying);
1413 	first_init = 0;
1414 	cpuhp_setup_state(CPUHP_AP_PERF_X86_UNCORE_ONLINE,
1415 			  "AP_PERF_X86_UNCORE_ONLINE",
1416 			  uncore_event_cpu_online, uncore_event_cpu_offline);
1417 	return 0;
1418 
1419 err:
1420 	/* Undo box->init_box() */
1421 	on_each_cpu_mask(&uncore_cpu_mask, uncore_exit_boxes, NULL, 1);
1422 	uncore_types_exit(uncore_msr_uncores);
1423 	uncore_pci_exit();
1424 	return ret;
1425 }
1426 module_init(intel_uncore_init);
1427 
1428 static void __exit intel_uncore_exit(void)
1429 {
1430 	cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_UNCORE_ONLINE);
1431 	cpuhp_remove_state_nocalls(CPUHP_AP_PERF_X86_UNCORE_STARTING);
1432 	cpuhp_remove_state_nocalls(CPUHP_PERF_X86_UNCORE_PREP);
1433 	uncore_types_exit(uncore_msr_uncores);
1434 	uncore_pci_exit();
1435 }
1436 module_exit(intel_uncore_exit);
1437