1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013 Advanced Micro Devices, Inc. 4 * 5 * Author: Jacob Shin <jacob.shin@amd.com> 6 */ 7 8 #include <linux/perf_event.h> 9 #include <linux/percpu.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/init.h> 13 #include <linux/cpu.h> 14 #include <linux/cpumask.h> 15 16 #include <asm/cpufeature.h> 17 #include <asm/perf_event.h> 18 #include <asm/msr.h> 19 #include <asm/smp.h> 20 21 #define NUM_COUNTERS_NB 4 22 #define NUM_COUNTERS_L2 4 23 #define NUM_COUNTERS_L3 6 24 #define MAX_COUNTERS 6 25 26 #define RDPMC_BASE_NB 6 27 #define RDPMC_BASE_LLC 10 28 29 #define COUNTER_SHIFT 16 30 31 #undef pr_fmt 32 #define pr_fmt(fmt) "amd_uncore: " fmt 33 34 static int num_counters_llc; 35 static int num_counters_nb; 36 static bool l3_mask; 37 38 static HLIST_HEAD(uncore_unused_list); 39 40 struct amd_uncore { 41 int id; 42 int refcnt; 43 int cpu; 44 int num_counters; 45 int rdpmc_base; 46 u32 msr_base; 47 cpumask_t *active_mask; 48 struct pmu *pmu; 49 struct perf_event *events[MAX_COUNTERS]; 50 struct hlist_node node; 51 }; 52 53 static struct amd_uncore * __percpu *amd_uncore_nb; 54 static struct amd_uncore * __percpu *amd_uncore_llc; 55 56 static struct pmu amd_nb_pmu; 57 static struct pmu amd_llc_pmu; 58 59 static cpumask_t amd_nb_active_mask; 60 static cpumask_t amd_llc_active_mask; 61 62 static bool is_nb_event(struct perf_event *event) 63 { 64 return event->pmu->type == amd_nb_pmu.type; 65 } 66 67 static bool is_llc_event(struct perf_event *event) 68 { 69 return event->pmu->type == amd_llc_pmu.type; 70 } 71 72 static struct amd_uncore *event_to_amd_uncore(struct perf_event *event) 73 { 74 if (is_nb_event(event) && amd_uncore_nb) 75 return *per_cpu_ptr(amd_uncore_nb, event->cpu); 76 else if (is_llc_event(event) && amd_uncore_llc) 77 return *per_cpu_ptr(amd_uncore_llc, event->cpu); 78 79 return NULL; 80 } 81 82 static void amd_uncore_read(struct perf_event *event) 83 { 84 struct hw_perf_event *hwc = &event->hw; 85 u64 prev, new; 86 s64 delta; 87 88 /* 89 * since we do not enable counter overflow interrupts, 90 * we do not have to worry about prev_count changing on us 91 */ 92 93 prev = local64_read(&hwc->prev_count); 94 rdpmcl(hwc->event_base_rdpmc, new); 95 local64_set(&hwc->prev_count, new); 96 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); 97 delta >>= COUNTER_SHIFT; 98 local64_add(delta, &event->count); 99 } 100 101 static void amd_uncore_start(struct perf_event *event, int flags) 102 { 103 struct hw_perf_event *hwc = &event->hw; 104 105 if (flags & PERF_EF_RELOAD) 106 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); 107 108 hwc->state = 0; 109 wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); 110 perf_event_update_userpage(event); 111 } 112 113 static void amd_uncore_stop(struct perf_event *event, int flags) 114 { 115 struct hw_perf_event *hwc = &event->hw; 116 117 wrmsrl(hwc->config_base, hwc->config); 118 hwc->state |= PERF_HES_STOPPED; 119 120 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 121 amd_uncore_read(event); 122 hwc->state |= PERF_HES_UPTODATE; 123 } 124 } 125 126 static int amd_uncore_add(struct perf_event *event, int flags) 127 { 128 int i; 129 struct amd_uncore *uncore = event_to_amd_uncore(event); 130 struct hw_perf_event *hwc = &event->hw; 131 132 /* are we already assigned? */ 133 if (hwc->idx != -1 && uncore->events[hwc->idx] == event) 134 goto out; 135 136 for (i = 0; i < uncore->num_counters; i++) { 137 if (uncore->events[i] == event) { 138 hwc->idx = i; 139 goto out; 140 } 141 } 142 143 /* if not, take the first available counter */ 144 hwc->idx = -1; 145 for (i = 0; i < uncore->num_counters; i++) { 146 if (cmpxchg(&uncore->events[i], NULL, event) == NULL) { 147 hwc->idx = i; 148 break; 149 } 150 } 151 152 out: 153 if (hwc->idx == -1) 154 return -EBUSY; 155 156 hwc->config_base = uncore->msr_base + (2 * hwc->idx); 157 hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); 158 hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; 159 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 160 161 if (flags & PERF_EF_START) 162 amd_uncore_start(event, PERF_EF_RELOAD); 163 164 return 0; 165 } 166 167 static void amd_uncore_del(struct perf_event *event, int flags) 168 { 169 int i; 170 struct amd_uncore *uncore = event_to_amd_uncore(event); 171 struct hw_perf_event *hwc = &event->hw; 172 173 amd_uncore_stop(event, PERF_EF_UPDATE); 174 175 for (i = 0; i < uncore->num_counters; i++) { 176 if (cmpxchg(&uncore->events[i], event, NULL) == event) 177 break; 178 } 179 180 hwc->idx = -1; 181 } 182 183 /* 184 * Return a full thread and slice mask unless user 185 * has provided them 186 */ 187 static u64 l3_thread_slice_mask(u64 config) 188 { 189 if (boot_cpu_data.x86 <= 0x18) 190 return ((config & AMD64_L3_SLICE_MASK) ? : AMD64_L3_SLICE_MASK) | 191 ((config & AMD64_L3_THREAD_MASK) ? : AMD64_L3_THREAD_MASK); 192 193 /* 194 * If the user doesn't specify a threadmask, they're not trying to 195 * count core 0, so we enable all cores & threads. 196 * We'll also assume that they want to count slice 0 if they specify 197 * a threadmask and leave sliceid and enallslices unpopulated. 198 */ 199 if (!(config & AMD64_L3_F19H_THREAD_MASK)) 200 return AMD64_L3_F19H_THREAD_MASK | AMD64_L3_EN_ALL_SLICES | 201 AMD64_L3_EN_ALL_CORES; 202 203 return config & (AMD64_L3_F19H_THREAD_MASK | AMD64_L3_SLICEID_MASK | 204 AMD64_L3_EN_ALL_CORES | AMD64_L3_EN_ALL_SLICES | 205 AMD64_L3_COREID_MASK); 206 } 207 208 static int amd_uncore_event_init(struct perf_event *event) 209 { 210 struct amd_uncore *uncore; 211 struct hw_perf_event *hwc = &event->hw; 212 213 if (event->attr.type != event->pmu->type) 214 return -ENOENT; 215 216 /* 217 * NB and Last level cache counters (MSRs) are shared across all cores 218 * that share the same NB / Last level cache. On family 16h and below, 219 * Interrupts can be directed to a single target core, however, event 220 * counts generated by processes running on other cores cannot be masked 221 * out. So we do not support sampling and per-thread events via 222 * CAP_NO_INTERRUPT, and we do not enable counter overflow interrupts: 223 */ 224 hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB; 225 hwc->idx = -1; 226 227 if (event->cpu < 0) 228 return -EINVAL; 229 230 /* 231 * SliceMask and ThreadMask need to be set for certain L3 events. 232 * For other events, the two fields do not affect the count. 233 */ 234 if (l3_mask && is_llc_event(event)) 235 hwc->config |= l3_thread_slice_mask(event->attr.config); 236 237 uncore = event_to_amd_uncore(event); 238 if (!uncore) 239 return -ENODEV; 240 241 /* 242 * since request can come in to any of the shared cores, we will remap 243 * to a single common cpu. 244 */ 245 event->cpu = uncore->cpu; 246 247 return 0; 248 } 249 250 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, 251 struct device_attribute *attr, 252 char *buf) 253 { 254 cpumask_t *active_mask; 255 struct pmu *pmu = dev_get_drvdata(dev); 256 257 if (pmu->type == amd_nb_pmu.type) 258 active_mask = &amd_nb_active_mask; 259 else if (pmu->type == amd_llc_pmu.type) 260 active_mask = &amd_llc_active_mask; 261 else 262 return 0; 263 264 return cpumap_print_to_pagebuf(true, buf, active_mask); 265 } 266 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL); 267 268 static struct attribute *amd_uncore_attrs[] = { 269 &dev_attr_cpumask.attr, 270 NULL, 271 }; 272 273 static struct attribute_group amd_uncore_attr_group = { 274 .attrs = amd_uncore_attrs, 275 }; 276 277 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format) \ 278 static ssize_t __uncore_##_var##_show(struct kobject *kobj, \ 279 struct kobj_attribute *attr, \ 280 char *page) \ 281 { \ 282 BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE); \ 283 return sprintf(page, _format "\n"); \ 284 } \ 285 static struct kobj_attribute format_attr_##_var = \ 286 __ATTR(_name, 0444, __uncore_##_var##_show, NULL) 287 288 DEFINE_UNCORE_FORMAT_ATTR(event12, event, "config:0-7,32-35"); 289 DEFINE_UNCORE_FORMAT_ATTR(event14, event, "config:0-7,32-35,59-60"); /* F17h+ DF */ 290 DEFINE_UNCORE_FORMAT_ATTR(event8, event, "config:0-7"); /* F17h+ L3 */ 291 DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 292 DEFINE_UNCORE_FORMAT_ATTR(coreid, coreid, "config:42-44"); /* F19h L3 */ 293 DEFINE_UNCORE_FORMAT_ATTR(slicemask, slicemask, "config:48-51"); /* F17h L3 */ 294 DEFINE_UNCORE_FORMAT_ATTR(threadmask8, threadmask, "config:56-63"); /* F17h L3 */ 295 DEFINE_UNCORE_FORMAT_ATTR(threadmask2, threadmask, "config:56-57"); /* F19h L3 */ 296 DEFINE_UNCORE_FORMAT_ATTR(enallslices, enallslices, "config:46"); /* F19h L3 */ 297 DEFINE_UNCORE_FORMAT_ATTR(enallcores, enallcores, "config:47"); /* F19h L3 */ 298 DEFINE_UNCORE_FORMAT_ATTR(sliceid, sliceid, "config:48-50"); /* F19h L3 */ 299 300 static struct attribute *amd_uncore_df_format_attr[] = { 301 &format_attr_event12.attr, /* event14 if F17h+ */ 302 &format_attr_umask.attr, 303 NULL, 304 }; 305 306 static struct attribute *amd_uncore_l3_format_attr[] = { 307 &format_attr_event12.attr, /* event8 if F17h+ */ 308 &format_attr_umask.attr, 309 NULL, /* slicemask if F17h, coreid if F19h */ 310 NULL, /* threadmask8 if F17h, enallslices if F19h */ 311 NULL, /* enallcores if F19h */ 312 NULL, /* sliceid if F19h */ 313 NULL, /* threadmask2 if F19h */ 314 NULL, 315 }; 316 317 static struct attribute_group amd_uncore_df_format_group = { 318 .name = "format", 319 .attrs = amd_uncore_df_format_attr, 320 }; 321 322 static struct attribute_group amd_uncore_l3_format_group = { 323 .name = "format", 324 .attrs = amd_uncore_l3_format_attr, 325 }; 326 327 static const struct attribute_group *amd_uncore_df_attr_groups[] = { 328 &amd_uncore_attr_group, 329 &amd_uncore_df_format_group, 330 NULL, 331 }; 332 333 static const struct attribute_group *amd_uncore_l3_attr_groups[] = { 334 &amd_uncore_attr_group, 335 &amd_uncore_l3_format_group, 336 NULL, 337 }; 338 339 static struct pmu amd_nb_pmu = { 340 .task_ctx_nr = perf_invalid_context, 341 .attr_groups = amd_uncore_df_attr_groups, 342 .name = "amd_nb", 343 .event_init = amd_uncore_event_init, 344 .add = amd_uncore_add, 345 .del = amd_uncore_del, 346 .start = amd_uncore_start, 347 .stop = amd_uncore_stop, 348 .read = amd_uncore_read, 349 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 350 }; 351 352 static struct pmu amd_llc_pmu = { 353 .task_ctx_nr = perf_invalid_context, 354 .attr_groups = amd_uncore_l3_attr_groups, 355 .name = "amd_l2", 356 .event_init = amd_uncore_event_init, 357 .add = amd_uncore_add, 358 .del = amd_uncore_del, 359 .start = amd_uncore_start, 360 .stop = amd_uncore_stop, 361 .read = amd_uncore_read, 362 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, 363 }; 364 365 static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) 366 { 367 return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, 368 cpu_to_node(cpu)); 369 } 370 371 static int amd_uncore_cpu_up_prepare(unsigned int cpu) 372 { 373 struct amd_uncore *uncore_nb = NULL, *uncore_llc; 374 375 if (amd_uncore_nb) { 376 uncore_nb = amd_uncore_alloc(cpu); 377 if (!uncore_nb) 378 goto fail; 379 uncore_nb->cpu = cpu; 380 uncore_nb->num_counters = num_counters_nb; 381 uncore_nb->rdpmc_base = RDPMC_BASE_NB; 382 uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL; 383 uncore_nb->active_mask = &amd_nb_active_mask; 384 uncore_nb->pmu = &amd_nb_pmu; 385 uncore_nb->id = -1; 386 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb; 387 } 388 389 if (amd_uncore_llc) { 390 uncore_llc = amd_uncore_alloc(cpu); 391 if (!uncore_llc) 392 goto fail; 393 uncore_llc->cpu = cpu; 394 uncore_llc->num_counters = num_counters_llc; 395 uncore_llc->rdpmc_base = RDPMC_BASE_LLC; 396 uncore_llc->msr_base = MSR_F16H_L2I_PERF_CTL; 397 uncore_llc->active_mask = &amd_llc_active_mask; 398 uncore_llc->pmu = &amd_llc_pmu; 399 uncore_llc->id = -1; 400 *per_cpu_ptr(amd_uncore_llc, cpu) = uncore_llc; 401 } 402 403 return 0; 404 405 fail: 406 if (amd_uncore_nb) 407 *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; 408 kfree(uncore_nb); 409 return -ENOMEM; 410 } 411 412 static struct amd_uncore * 413 amd_uncore_find_online_sibling(struct amd_uncore *this, 414 struct amd_uncore * __percpu *uncores) 415 { 416 unsigned int cpu; 417 struct amd_uncore *that; 418 419 for_each_online_cpu(cpu) { 420 that = *per_cpu_ptr(uncores, cpu); 421 422 if (!that) 423 continue; 424 425 if (this == that) 426 continue; 427 428 if (this->id == that->id) { 429 hlist_add_head(&this->node, &uncore_unused_list); 430 this = that; 431 break; 432 } 433 } 434 435 this->refcnt++; 436 return this; 437 } 438 439 static int amd_uncore_cpu_starting(unsigned int cpu) 440 { 441 unsigned int eax, ebx, ecx, edx; 442 struct amd_uncore *uncore; 443 444 if (amd_uncore_nb) { 445 uncore = *per_cpu_ptr(amd_uncore_nb, cpu); 446 cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); 447 uncore->id = ecx & 0xff; 448 449 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb); 450 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore; 451 } 452 453 if (amd_uncore_llc) { 454 uncore = *per_cpu_ptr(amd_uncore_llc, cpu); 455 uncore->id = per_cpu(cpu_llc_id, cpu); 456 457 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_llc); 458 *per_cpu_ptr(amd_uncore_llc, cpu) = uncore; 459 } 460 461 return 0; 462 } 463 464 static void uncore_clean_online(void) 465 { 466 struct amd_uncore *uncore; 467 struct hlist_node *n; 468 469 hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { 470 hlist_del(&uncore->node); 471 kfree(uncore); 472 } 473 } 474 475 static void uncore_online(unsigned int cpu, 476 struct amd_uncore * __percpu *uncores) 477 { 478 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 479 480 uncore_clean_online(); 481 482 if (cpu == uncore->cpu) 483 cpumask_set_cpu(cpu, uncore->active_mask); 484 } 485 486 static int amd_uncore_cpu_online(unsigned int cpu) 487 { 488 if (amd_uncore_nb) 489 uncore_online(cpu, amd_uncore_nb); 490 491 if (amd_uncore_llc) 492 uncore_online(cpu, amd_uncore_llc); 493 494 return 0; 495 } 496 497 static void uncore_down_prepare(unsigned int cpu, 498 struct amd_uncore * __percpu *uncores) 499 { 500 unsigned int i; 501 struct amd_uncore *this = *per_cpu_ptr(uncores, cpu); 502 503 if (this->cpu != cpu) 504 return; 505 506 /* this cpu is going down, migrate to a shared sibling if possible */ 507 for_each_online_cpu(i) { 508 struct amd_uncore *that = *per_cpu_ptr(uncores, i); 509 510 if (cpu == i) 511 continue; 512 513 if (this == that) { 514 perf_pmu_migrate_context(this->pmu, cpu, i); 515 cpumask_clear_cpu(cpu, that->active_mask); 516 cpumask_set_cpu(i, that->active_mask); 517 that->cpu = i; 518 break; 519 } 520 } 521 } 522 523 static int amd_uncore_cpu_down_prepare(unsigned int cpu) 524 { 525 if (amd_uncore_nb) 526 uncore_down_prepare(cpu, amd_uncore_nb); 527 528 if (amd_uncore_llc) 529 uncore_down_prepare(cpu, amd_uncore_llc); 530 531 return 0; 532 } 533 534 static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) 535 { 536 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 537 538 if (cpu == uncore->cpu) 539 cpumask_clear_cpu(cpu, uncore->active_mask); 540 541 if (!--uncore->refcnt) 542 kfree(uncore); 543 *per_cpu_ptr(uncores, cpu) = NULL; 544 } 545 546 static int amd_uncore_cpu_dead(unsigned int cpu) 547 { 548 if (amd_uncore_nb) 549 uncore_dead(cpu, amd_uncore_nb); 550 551 if (amd_uncore_llc) 552 uncore_dead(cpu, amd_uncore_llc); 553 554 return 0; 555 } 556 557 static int __init amd_uncore_init(void) 558 { 559 struct attribute **df_attr = amd_uncore_df_format_attr; 560 struct attribute **l3_attr = amd_uncore_l3_format_attr; 561 int ret = -ENODEV; 562 563 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD && 564 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON) 565 return -ENODEV; 566 567 if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) 568 return -ENODEV; 569 570 num_counters_nb = NUM_COUNTERS_NB; 571 num_counters_llc = NUM_COUNTERS_L2; 572 if (boot_cpu_data.x86 >= 0x17) { 573 /* 574 * For F17h and above, the Northbridge counters are 575 * repurposed as Data Fabric counters. Also, L3 576 * counters are supported too. The PMUs are exported 577 * based on family as either L2 or L3 and NB or DF. 578 */ 579 num_counters_llc = NUM_COUNTERS_L3; 580 amd_nb_pmu.name = "amd_df"; 581 amd_llc_pmu.name = "amd_l3"; 582 l3_mask = true; 583 } 584 585 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { 586 if (boot_cpu_data.x86 >= 0x17) 587 *df_attr = &format_attr_event14.attr; 588 589 amd_uncore_nb = alloc_percpu(struct amd_uncore *); 590 if (!amd_uncore_nb) { 591 ret = -ENOMEM; 592 goto fail_nb; 593 } 594 ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1); 595 if (ret) 596 goto fail_nb; 597 598 pr_info("%d %s %s counters detected\n", num_counters_nb, 599 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", 600 amd_nb_pmu.name); 601 602 ret = 0; 603 } 604 605 if (boot_cpu_has(X86_FEATURE_PERFCTR_LLC)) { 606 if (boot_cpu_data.x86 >= 0x19) { 607 *l3_attr++ = &format_attr_event8.attr; 608 *l3_attr++ = &format_attr_umask.attr; 609 *l3_attr++ = &format_attr_coreid.attr; 610 *l3_attr++ = &format_attr_enallslices.attr; 611 *l3_attr++ = &format_attr_enallcores.attr; 612 *l3_attr++ = &format_attr_sliceid.attr; 613 *l3_attr++ = &format_attr_threadmask2.attr; 614 } else if (boot_cpu_data.x86 >= 0x17) { 615 *l3_attr++ = &format_attr_event8.attr; 616 *l3_attr++ = &format_attr_umask.attr; 617 *l3_attr++ = &format_attr_slicemask.attr; 618 *l3_attr++ = &format_attr_threadmask8.attr; 619 } 620 621 amd_uncore_llc = alloc_percpu(struct amd_uncore *); 622 if (!amd_uncore_llc) { 623 ret = -ENOMEM; 624 goto fail_llc; 625 } 626 ret = perf_pmu_register(&amd_llc_pmu, amd_llc_pmu.name, -1); 627 if (ret) 628 goto fail_llc; 629 630 pr_info("%d %s %s counters detected\n", num_counters_llc, 631 boot_cpu_data.x86_vendor == X86_VENDOR_HYGON ? "HYGON" : "", 632 amd_llc_pmu.name); 633 ret = 0; 634 } 635 636 /* 637 * Install callbacks. Core will call them for each online cpu. 638 */ 639 if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP, 640 "perf/x86/amd/uncore:prepare", 641 amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead)) 642 goto fail_llc; 643 644 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING, 645 "perf/x86/amd/uncore:starting", 646 amd_uncore_cpu_starting, NULL)) 647 goto fail_prep; 648 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE, 649 "perf/x86/amd/uncore:online", 650 amd_uncore_cpu_online, 651 amd_uncore_cpu_down_prepare)) 652 goto fail_start; 653 return 0; 654 655 fail_start: 656 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 657 fail_prep: 658 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 659 fail_llc: 660 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) 661 perf_pmu_unregister(&amd_nb_pmu); 662 if (amd_uncore_llc) 663 free_percpu(amd_uncore_llc); 664 fail_nb: 665 if (amd_uncore_nb) 666 free_percpu(amd_uncore_nb); 667 668 return ret; 669 } 670 device_initcall(amd_uncore_init); 671