1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * 5 * Author: Jacob Shin <jacob.shin@amd.com> 6 */ 7 8 #include <linux/perf_event.h> 9 #include <linux/percpu.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/init.h> 13 #include <linux/cpu.h> 14 #include <linux/cpumask.h> 15 #include <linux/cpufeature.h> 16 #include <linux/smp.h> 17 18 #include <asm/perf_event.h> 19 #include <asm/msr.h> 20 21 #define NUM_COUNTERS_NB 4 22 #define NUM_COUNTERS_L2 4 23 #define NUM_COUNTERS_L3 6 24 #define MAX_COUNTERS 6 25 26 #define RDPMC_BASE_NB 6 27 #define RDPMC_BASE_LLC 10 28 29 #define COUNTER_SHIFT 16 30 31 #undef pr_fmt 32 #define pr_fmt(fmt) "amd_uncore: " fmt 33 34 static int num_counters_llc; 35 static int num_counters_nb; 36 static bool l3_mask; 37 38 static HLIST_HEAD(uncore_unused_list); 39 40 struct amd_uncore { 41 int id; 42 int refcnt; 43 int cpu; 44 int num_counters; 45 int rdpmc_base; 46 u32 msr_base; 47 cpumask_t *active_mask; 48 struct pmu *pmu; 49 struct perf_event *events[MAX_COUNTERS]; 50 struct hlist_node node; 51 }; 52 53 static struct amd_uncore * __percpu *amd_uncore_nb; 54 static struct amd_uncore * __percpu *amd_uncore_llc; 55 56 static struct pmu amd_nb_pmu; 57 static struct pmu amd_llc_pmu; 58 59 static cpumask_t amd_nb_active_mask; 60 static cpumask_t amd_llc_active_mask; 61 62 static bool is_nb_event(struct perf_event *event) 63 { 64 return event->pmu->type == amd_nb_pmu.type; 65 } 66 67 static bool is_llc_event(struct perf_event *event) 68 { 69 return event->pmu->type == amd_llc_pmu.type; 70 } 71 72 static struct amd_uncore *event_to_amd_uncore(struct perf_event *event) 73 { 74 if (is_nb_event(event) && amd_uncore_nb) 75 return *per_cpu_ptr(amd_uncore_nb, event->cpu); 76 else if (is_llc_event(event) && amd_uncore_llc) 77 return *per_cpu_ptr(amd_uncore_llc, event->cpu); 78 79 return NULL; 80 } 81 82 static void amd_uncore_read(struct perf_event *event) 83 { 84 struct hw_perf_event *hwc = &event->hw; 85 u64 prev, new; 86 s64 delta; 87 88 /* 89 * since we do not enable counter overflow interrupts, 90 * we do not have to worry about prev_count changing on us 91 */ 92 93 prev = local64_read(&hwc->prev_count); 94 rdpmcl(hwc->event_base_rdpmc, new); 95 local64_set(&hwc->prev_count, new); 96 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); 97 delta >>= COUNTER_SHIFT; 98 local64_add(delta, &event->count); 99 } 100 101 static void amd_uncore_start(struct perf_event *event, int flags) 102 { 103 struct hw_perf_event *hwc = &event->hw; 104 105 if (flags & PERF_EF_RELOAD) 106 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); 107 108 hwc->state = 0; 109 wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); 110 perf_event_update_userpage(event); 111 } 112 113 static void amd_uncore_stop(struct perf_event *event, int flags) 114 { 115 struct hw_perf_event *hwc = &event->hw; 116 117 wrmsrl(hwc->config_base, hwc->config); 118 hwc->state |= PERF_HES_STOPPED; 119 120 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 121 amd_uncore_read(event); 122 hwc->state |= PERF_HES_UPTODATE; 123 } 124 } 125 126 static int amd_uncore_add(struct perf_event *event, int flags) 127 { 128 int i; 129 struct amd_uncore *uncore = event_to_amd_uncore(event); 130 struct hw_perf_event *hwc = &event->hw; 131 132 /* are we already assigned? */ 133 if (hwc->idx != -1 && uncore->events[hwc->idx] == event) 134 goto out; 135 136 for (i = 0; i < uncore->num_counters; i++) { 137 if (uncore->events[i] == event) { 138 hwc->idx = i; 139 goto out; 140 } 141 } 142 143 /* if not, take the first available counter */ 144 hwc->idx = -1; 145 for (i = 0; i < uncore->num_counters; i++) { 146 if (cmpxchg(&uncore->events[i], NULL, event) == NULL) { 147 hwc->idx = i; 148 break; 149 } 150 } 151 152 out: 153 if (hwc->idx == -1) 154 return -EBUSY; 155 156 hwc->config_base = uncore->msr_base + (2 * hwc->idx); 157 hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); 158 hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; 159 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 160 161 if (flags & PERF_EF_START) 162 amd_uncore_start(event, PERF_EF_RELOAD); 163 164 return 0; 165 } 166 167 static void amd_uncore_del(struct perf_event *event, int flags) 168 { 169 int i; 170 struct amd_uncore *uncore = event_to_amd_uncore(event); 171 struct hw_perf_event *hwc = &event->hw; 172 173 amd_uncore_stop(event, PERF_EF_UPDATE); 174 175 for (i = 0; i < uncore->num_counters; i++) { 176 if (cmpxchg(&uncore->events[i], event, NULL) == event) 177 break; 178 } 179 180 hwc->idx = -1; 181 } 182 183 /* 184 * Return a full thread and slice mask unless user 185 * has provided them 186 */ 187 static u64 l3_thread_slice_mask(u64 config) 188 { 189 if (boot_cpu_data.x86 <= 0x18) 190 return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) | 191 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK); 192 193 /* 194 * If the user doesn't specify a threadmask, they're not trying to 195 * count core 0, so we enable all cores & threads. 196 * We'll also assume that they want to count slice 0 if they specify 197 * a threadmask and leave sliceid and enallslices unpopulated. 198 */ 199 if (!(config & AMD64_L3_F19H_THREAD_MASK)) 200 return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES | 201 AMD64_L3_EN_ALL_CORES; 202 203 return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK | 204 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES | 205 AMD64_L3_COREID_MASK); 206 } 207 208 static int amd_uncore_event_init(struct perf_event *event) 209 { 210 struct amd_uncore *uncore; 211 struct hw_perf_event *hwc = &event->hw; 212 213 if (event->attr.type != event->pmu->type) 214 return -ENOENT; 215 216 /* 217 * NB and Last level cache counters (MSRs) are shared across all cores 218 * that share the same NB / Last level cache. On family 16h and below, 219 * Interrupts can be directed to a single target core, however, event 220 * counts generated by processes running on other cores cannot be masked 221 * out. So we do not support sampling and per-thread events via 222 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts: 223 */ 224 hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB; 225 hwc->idx = -1; 226 227 if (event->cpu < 0) 228 return -EINVAL; 229 230 /* 231 * SliceMask and ThreadMask need to be set for certain L3 events. 232 * For other events, the two fields do not affect the count. 233 */ 234 if (l3_mask && is_llc_event(event)) 235 hwc->config |= l3_thread_slice_mask(event->attr.config); 236 237 uncore = event_to_amd_uncore(event); 238 if (!uncore) 239 return -ENODEV; 240 241 /* 242 * since request can come in to any of the shared cores, we will remap 243 * to a single common cpu. 244 */ 245 event->cpu = uncore->cpu; 246 247 return 0; 248 } 249 250 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, 251 struct device_attribute *attr, 252 char *buf) 253 { 254 cpumask_t *active_mask; 255 struct pmu *pmu = dev_get_drvdata(dev); 256 257 if (pmu->type == amd_nb_pmu.type) 258 active_mask = &amd_nb_active_mask; 259 else if (pmu->type == amd_llc_pmu.type) 260 active_mask = &amd_llc_active_mask; 261 else 262 return 0; 263 264 return cpumap_print_to_pagebuf(true, buf, active_mask); 265 } 266 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL); 267 268 static struct attribute *amd_uncore_attrs[] = { 269 &dev_attr_cpumask.attr, 270 NULL, 271 }; 272 273 static struct attribute_group amd_uncore_attr_group = { 274 .attrs = amd_uncore_attrs, 275 }; 276 277 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ 278 static ssize_t __uncore_##_var##_show(struct device *dev, \ 279 struct device_attribute *attr, \ 280 char *page) \ 281 { \ 282 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 283 return sprintf(page, _format "\n"); \ 284 } \ 285 static struct device_attribute format_attr_##_var = \ 286 __ATTR(_name, 0444, __uncore_##_var##_show, NULL) 287 288 DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); 289 DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */ 290 DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */ 291 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 292 DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */ 293 DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */ 294 DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */ 295 DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */ 296 DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */ 297 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ 298 DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ 299 300 static struct attribute *amd_uncore_df_format_attr[] = { 301 &format_attr_event12.attr, /* event14 if F17h+ */ 302 &format_attr_umask.attr, 303 NULL, 304 }; 305 306 static struct attribute *amd_uncore_l3_format_attr[] = { 307 &format_attr_event12.attr, /* event8 if F17h+ */ 308 &format_attr_umask.attr, 309 NULL, /* slicemask if F17h, coreid if F19h */ 310 NULL, /* threadmask8 if F17h, enallslices if F19h */ 311 NULL, /* enallcores if F19h */ 312 NULL, /* sliceid if F19h */ 313 NULL, /* threadmask2 if F19h */ 314 NULL, 315 }; 316 317 static struct attribute_group amd_uncore_df_format_group = { 318 .name = "format", 319 .attrs = amd_uncore_df_format_attr, 320 }; 321 322 static struct attribute_group amd_uncore_l3_format_group = { 323 .name = "format", 324 .attrs = amd_uncore_l3_format_attr, 325 }; 326 327 static const struct attribute_group *amd_uncore_df_attr_groups[] = { 328 &amd_uncore_attr_group, 329 &amd_uncore_df_format_group, 330 NULL, 331 }; 332 333 static const struct attribute_group *amd_uncore_l3_attr_groups[] = { 334 &amd_uncore_attr_group, 335 &amd_uncore_l3_format_group, 336 NULL, 337 }; 338 339 static struct pmu amd_nb_pmu = { 340 .task_ctx_nr = perf_invalid_context, 341 .attr_groups = amd_uncore_df_attr_groups, 342 .name = "amd_nb", 343 .event_init = amd_uncore_event_init, 344 .add = amd_uncore_add, 345 .del = amd_uncore_del, 346 .start = amd_uncore_start, 347 .stop = amd_uncore_stop, 348 .read = amd_uncore_read, 349 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 350 .module = THIS_MODULE, 351 }; 352 353 static struct pmu amd_llc_pmu = { 354 .task_ctx_nr = perf_invalid_context, 355 .attr_groups = amd_uncore_l3_attr_groups, 356 .name = "amd_l2", 357 .event_init = amd_uncore_event_init, 358 .add = amd_uncore_add, 359 .del = amd_uncore_del, 360 .start = amd_uncore_start, 361 .stop = amd_uncore_stop, 362 .read = amd_uncore_read, 363 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 364 .module = THIS_MODULE, 365 }; 366 367 static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) 368 { 369 return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, 370 cpu_to_node(cpu)); 371 } 372 373 static int amd_uncore_cpu_up_prepare(unsigned int cpu) 374 { 375 struct amd_uncore *uncore_nb = NULL, *uncore_llc; 376 377 if (amd_uncore_nb) { 378 uncore_nb = amd_uncore_alloc(cpu); 379 if (!uncore_nb) 380 goto fail; 381 uncore_nb->cpu = cpu; 382 uncore_nb->num_counters = num_counters_nb; 383 uncore_nb->rdpmc_base = RDPMC_BASE_NB; 384 uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL; 385 uncore_nb->active_mask = &amd_nb_active_mask; 386 uncore_nb->pmu = &amd_nb_pmu; 387 uncore_nb->id = -1; 388 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb; 389 } 390 391 if (amd_uncore_llc) { 392 uncore_llc = amd_uncore_alloc(cpu); 393 if (!uncore_llc) 394 goto fail; 395 uncore_llc->cpu = cpu; 396 uncore_llc->num_counters = num_counters_llc; 397 uncore_llc->rdpmc_base = RDPMC_BASE_LLC; 398 uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL; 399 uncore_llc->active_mask = &amd_llc_active_mask; 400 uncore_llc->pmu = &amd_llc_pmu; 401 uncore_llc->id = -1; 402 *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc; 403 } 404 405 return 0; 406 407 fail: 408 if (amd_uncore_nb) 409 *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; 410 kfree(uncore_nb); 411 return -ENOMEM; 412 } 413 414 static struct amd_uncore * 415 amd_uncore_find_online_sibling(struct amd_uncore *this, 416 struct amd_uncore * __percpu *uncores) 417 { 418 unsigned int cpu; 419 struct amd_uncore *that; 420 421 for_each_online_cpu(cpu) { 422 that = *per_cpu_ptr(uncores, cpu); 423 424 if (!that) 425 continue; 426 427 if (this == that) 428 continue; 429 430 if (this->id == that->id) { 431 hlist_add_head(&this->node, &uncore_unused_list); 432 this = that; 433 break; 434 } 435 } 436 437 this->refcnt++; 438 return this; 439 } 440 441 static int amd_uncore_cpu_starting(unsigned int cpu) 442 { 443 unsigned int eax, ebx, ecx, edx; 444 struct amd_uncore *uncore; 445 446 if (amd_uncore_nb) { 447 uncore = *per_cpu_ptr(amd_uncore_nb, cpu); 448 cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); 449 uncore->id = ecx & 0xff; 450 451 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb); 452 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore; 453 } 454 455 if (amd_uncore_llc) { 456 uncore = *per_cpu_ptr(amd_uncore_llc, cpu); 457 uncore->id = get_llc_id(cpu); 458 459 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc); 460 *per_cpu_ptr(amd_uncore_llc, cpu) = uncore; 461 } 462 463 return 0; 464 } 465 466 static void uncore_clean_online(void) 467 { 468 struct amd_uncore *uncore; 469 struct hlist_node *n; 470 471 hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { 472 hlist_del(&uncore->node); 473 kfree(uncore); 474 } 475 } 476 477 static void uncore_online(unsigned int cpu, 478 struct amd_uncore * __percpu *uncores) 479 { 480 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 481 482 uncore_clean_online(); 483 484 if (cpu == uncore->cpu) 485 cpumask_set_cpu(cpu, uncore->active_mask); 486 } 487 488 static int amd_uncore_cpu_online(unsigned int cpu) 489 { 490 if (amd_uncore_nb) 491 uncore_online(cpu, amd_uncore_nb); 492 493 if (amd_uncore_llc) 494 uncore_online(cpu, amd_uncore_llc); 495 496 return 0; 497 } 498 499 static void uncore_down_prepare(unsigned int cpu, 500 struct amd_uncore * __percpu *uncores) 501 { 502 unsigned int i; 503 struct amd_uncore *this = *per_cpu_ptr(uncores, cpu); 504 505 if (this->cpu != cpu) 506 return; 507 508 /* this cpu is going down, migrate to a shared sibling if possible */ 509 for_each_online_cpu(i) { 510 struct amd_uncore *that = *per_cpu_ptr(uncores, i); 511 512 if (cpu == i) 513 continue; 514 515 if (this == that) { 516 perf_pmu_migrate_context(this->pmu, cpu, i); 517 cpumask_clear_cpu(cpu, that->active_mask); 518 cpumask_set_cpu(i, that->active_mask); 519 that->cpu = i; 520 break; 521 } 522 } 523 } 524 525 static int amd_uncore_cpu_down_prepare(unsigned int cpu) 526 { 527 if (amd_uncore_nb) 528 uncore_down_prepare(cpu, amd_uncore_nb); 529 530 if (amd_uncore_llc) 531 uncore_down_prepare(cpu, amd_uncore_llc); 532 533 return 0; 534 } 535 536 static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) 537 { 538 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 539 540 if (cpu == uncore->cpu) 541 cpumask_clear_cpu(cpu, uncore->active_mask); 542 543 if (!--uncore->refcnt) 544 kfree(uncore); 545 *per_cpu_ptr(uncores, cpu) = NULL; 546 } 547 548 static int amd_uncore_cpu_dead(unsigned int cpu) 549 { 550 if (amd_uncore_nb) 551 uncore_dead(cpu, amd_uncore_nb); 552 553 if (amd_uncore_llc) 554 uncore_dead(cpu, amd_uncore_llc); 555 556 return 0; 557 } 558 559 static int __init amd_uncore_init(void) 560 { 561 struct attribute **df_attr = amd_uncore_df_format_attr; 562 struct attribute **l3_attr = amd_uncore_l3_format_attr; 563 int ret = -ENODEV; 564 565 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 566 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 567 return -ENODEV; 568 569 if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) 570 return -ENODEV; 571 572 num_counters_nb = NUM_COUNTERS_NB; 573 num_counters_llc = NUM_COUNTERS_L2; 574 if (boot_cpu_data.x86 >= 0x17) { 575 /* 576 * For F17h and above, the Northbridge counters are 577 * repurposed as Data Fabric counters. Also, L3 578 * counters are supported too. The PMUs are exported 579 * based on family as either L2 or L3 and NB or DF. 580 */ 581 num_counters_llc = NUM_COUNTERS_L3; 582 amd_nb_pmu.name = "amd_df"; 583 amd_llc_pmu.name = "amd_l3"; 584 l3_mask = true; 585 } 586 587 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { 588 if (boot_cpu_data.x86 >= 0x17) 589 *df_attr = &format_attr_event14.attr; 590 591 amd_uncore_nb = alloc_percpu(struct amd_uncore *); 592 if (!amd_uncore_nb) { 593 ret = -ENOMEM; 594 goto fail_nb; 595 } 596 ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1); 597 if (ret) 598 goto fail_nb; 599 600 pr_info("%d %s %s counters detected\n", num_counters_nb, 601 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", 602 amd_nb_pmu.name); 603 604 ret = 0; 605 } 606 607 if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { 608 if (boot_cpu_data.x86 >= 0x19) { 609 *l3_attr++ = &format_attr_event8.attr; 610 *l3_attr++ = &format_attr_umask.attr; 611 *l3_attr++ = &format_attr_coreid.attr; 612 *l3_attr++ = &format_attr_enallslices.attr; 613 *l3_attr++ = &format_attr_enallcores.attr; 614 *l3_attr++ = &format_attr_sliceid.attr; 615 *l3_attr++ = &format_attr_threadmask2.attr; 616 } else if (boot_cpu_data.x86 >= 0x17) { 617 *l3_attr++ = &format_attr_event8.attr; 618 *l3_attr++ = &format_attr_umask.attr; 619 *l3_attr++ = &format_attr_slicemask.attr; 620 *l3_attr++ = &format_attr_threadmask8.attr; 621 } 622 623 amd_uncore_llc = alloc_percpu(struct amd_uncore *); 624 if (!amd_uncore_llc) { 625 ret = -ENOMEM; 626 goto fail_llc; 627 } 628 ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1); 629 if (ret) 630 goto fail_llc; 631 632 pr_info("%d %s %s counters detected\n", num_counters_llc, 633 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", 634 amd_llc_pmu.name); 635 ret = 0; 636 } 637 638 /* 639 * Install callbacks. Core will call them for each online cpu. 640 */ 641 if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP, 642 "perf/x86/amd/uncore:prepare", 643 amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead)) 644 goto fail_llc; 645 646 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING, 647 "perf/x86/amd/uncore:starting", 648 amd_uncore_cpu_starting, NULL)) 649 goto fail_prep; 650 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE, 651 "perf/x86/amd/uncore:online", 652 amd_uncore_cpu_online, 653 amd_uncore_cpu_down_prepare)) 654 goto fail_start; 655 return 0; 656 657 fail_start: 658 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 659 fail_prep: 660 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 661 fail_llc: 662 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) 663 perf_pmu_unregister(&amd_nb_pmu); 664 free_percpu(amd_uncore_llc); 665 fail_nb: 666 free_percpu(amd_uncore_nb); 667 668 return ret; 669 } 670 671 static void __exit amd_uncore_exit(void) 672 { 673 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE); 674 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 675 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 676 677 if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { 678 perf_pmu_unregister(&amd_llc_pmu); 679 free_percpu(amd_uncore_llc); 680 amd_uncore_llc = NULL; 681 } 682 683 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { 684 perf_pmu_unregister(&amd_nb_pmu); 685 free_percpu(amd_uncore_nb); 686 amd_uncore_nb = NULL; 687 } 688 } 689 690 module_init(amd_uncore_init); 691 module_exit(amd_uncore_exit); 692 693 MODULE_DESCRIPTION("AMD Uncore Driver"); 694 MODULE_LICENSE("GPL v2"); 695