1 /* 2 * Copyright (C) 2013 Advanced Micro Devices, Inc. 3 * 4 * Author: Jacob Shin <jacob.shin@amd.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/perf_event.h> 12 #include <linux/percpu.h> 13 #include <linux/types.h> 14 #include <linux/slab.h> 15 #include <linux/init.h> 16 #include <linux/cpu.h> 17 #include <linux/cpumask.h> 18 19 #include <asm/cpufeature.h> 20 #include <asm/perf_event.h> 21 #include <asm/msr.h> 22 23 #define NUM_COUNTERS_NB 4 24 #define NUM_COUNTERS_L2 4 25 #define MAX_COUNTERS NUM_COUNTERS_NB 26 27 #define RDPMC_BASE_NB 6 28 #define RDPMC_BASE_L2 10 29 30 #define COUNTER_SHIFT 16 31 32 static HLIST_HEAD(uncore_unused_list); 33 34 struct amd_uncore { 35 int id; 36 int refcnt; 37 int cpu; 38 int num_counters; 39 int rdpmc_base; 40 u32 msr_base; 41 cpumask_t *active_mask; 42 struct pmu *pmu; 43 struct perf_event *events[MAX_COUNTERS]; 44 struct hlist_node node; 45 }; 46 47 static struct amd_uncore * __percpu *amd_uncore_nb; 48 static struct amd_uncore * __percpu *amd_uncore_l2; 49 50 static struct pmu amd_nb_pmu; 51 static struct pmu amd_l2_pmu; 52 53 static cpumask_t amd_nb_active_mask; 54 static cpumask_t amd_l2_active_mask; 55 56 static bool is_nb_event(struct perf_event *event) 57 { 58 return event->pmu->type == amd_nb_pmu.type; 59 } 60 61 static bool is_l2_event(struct perf_event *event) 62 { 63 return event->pmu->type == amd_l2_pmu.type; 64 } 65 66 static struct amd_uncore *event_to_amd_uncore(struct perf_event *event) 67 { 68 if (is_nb_event(event) && amd_uncore_nb) 69 return *per_cpu_ptr(amd_uncore_nb, event->cpu); 70 else if (is_l2_event(event) && amd_uncore_l2) 71 return *per_cpu_ptr(amd_uncore_l2, event->cpu); 72 73 return NULL; 74 } 75 76 static void amd_uncore_read(struct perf_event *event) 77 { 78 struct hw_perf_event *hwc = &event->hw; 79 u64 prev, new; 80 s64 delta; 81 82 /* 83 * since we do not enable counter overflow interrupts, 84 * we do not have to worry about prev_count changing on us 85 */ 86 87 prev = local64_read(&hwc->prev_count); 88 rdpmcl(hwc->event_base_rdpmc, new); 89 local64_set(&hwc->prev_count, new); 90 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); 91 delta >>= COUNTER_SHIFT; 92 local64_add(delta, &event->count); 93 } 94 95 static void amd_uncore_start(struct perf_event *event, int flags) 96 { 97 struct hw_perf_event *hwc = &event->hw; 98 99 if (flags & PERF_EF_RELOAD) 100 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count)); 101 102 hwc->state = 0; 103 wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE)); 104 perf_event_update_userpage(event); 105 } 106 107 static void amd_uncore_stop(struct perf_event *event, int flags) 108 { 109 struct hw_perf_event *hwc = &event->hw; 110 111 wrmsrl(hwc->config_base, hwc->config); 112 hwc->state |= PERF_HES_STOPPED; 113 114 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 115 amd_uncore_read(event); 116 hwc->state |= PERF_HES_UPTODATE; 117 } 118 } 119 120 static int amd_uncore_add(struct perf_event *event, int flags) 121 { 122 int i; 123 struct amd_uncore *uncore = event_to_amd_uncore(event); 124 struct hw_perf_event *hwc = &event->hw; 125 126 /* are we already assigned? */ 127 if (hwc->idx != -1 && uncore->events[hwc->idx] == event) 128 goto out; 129 130 for (i = 0; i < uncore->num_counters; i++) { 131 if (uncore->events[i] == event) { 132 hwc->idx = i; 133 goto out; 134 } 135 } 136 137 /* if not, take the first available counter */ 138 hwc->idx = -1; 139 for (i = 0; i < uncore->num_counters; i++) { 140 if (cmpxchg(&uncore->events[i], NULL, event) == NULL) { 141 hwc->idx = i; 142 break; 143 } 144 } 145 146 out: 147 if (hwc->idx == -1) 148 return -EBUSY; 149 150 hwc->config_base = uncore->msr_base + (2 * hwc->idx); 151 hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx); 152 hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx; 153 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 154 155 if (flags & PERF_EF_START) 156 amd_uncore_start(event, PERF_EF_RELOAD); 157 158 return 0; 159 } 160 161 static void amd_uncore_del(struct perf_event *event, int flags) 162 { 163 int i; 164 struct amd_uncore *uncore = event_to_amd_uncore(event); 165 struct hw_perf_event *hwc = &event->hw; 166 167 amd_uncore_stop(event, PERF_EF_UPDATE); 168 169 for (i = 0; i < uncore->num_counters; i++) { 170 if (cmpxchg(&uncore->events[i], event, NULL) == event) 171 break; 172 } 173 174 hwc->idx = -1; 175 } 176 177 static int amd_uncore_event_init(struct perf_event *event) 178 { 179 struct amd_uncore *uncore; 180 struct hw_perf_event *hwc = &event->hw; 181 182 if (event->attr.type != event->pmu->type) 183 return -ENOENT; 184 185 /* 186 * NB and L2 counters (MSRs) are shared across all cores that share the 187 * same NB / L2 cache. Interrupts can be directed to a single target 188 * core, however, event counts generated by processes running on other 189 * cores cannot be masked out. So we do not support sampling and 190 * per-thread events. 191 */ 192 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 193 return -EINVAL; 194 195 /* NB and L2 counters do not have usr/os/guest/host bits */ 196 if (event->attr.exclude_user || event->attr.exclude_kernel || 197 event->attr.exclude_host || event->attr.exclude_guest) 198 return -EINVAL; 199 200 /* and we do not enable counter overflow interrupts */ 201 hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB; 202 hwc->idx = -1; 203 204 if (event->cpu < 0) 205 return -EINVAL; 206 207 uncore = event_to_amd_uncore(event); 208 if (!uncore) 209 return -ENODEV; 210 211 /* 212 * since request can come in to any of the shared cores, we will remap 213 * to a single common cpu. 214 */ 215 event->cpu = uncore->cpu; 216 217 return 0; 218 } 219 220 static ssize_t amd_uncore_attr_show_cpumask(struct device *dev, 221 struct device_attribute *attr, 222 char *buf) 223 { 224 cpumask_t *active_mask; 225 struct pmu *pmu = dev_get_drvdata(dev); 226 227 if (pmu->type == amd_nb_pmu.type) 228 active_mask = &amd_nb_active_mask; 229 else if (pmu->type == amd_l2_pmu.type) 230 active_mask = &amd_l2_active_mask; 231 else 232 return 0; 233 234 return cpumap_print_to_pagebuf(true, buf, active_mask); 235 } 236 static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL); 237 238 static struct attribute *amd_uncore_attrs[] = { 239 &dev_attr_cpumask.attr, 240 NULL, 241 }; 242 243 static struct attribute_group amd_uncore_attr_group = { 244 .attrs = amd_uncore_attrs, 245 }; 246 247 PMU_FORMAT_ATTR(event, "config:0-7,32-35"); 248 PMU_FORMAT_ATTR(umask, "config:8-15"); 249 250 static struct attribute *amd_uncore_format_attr[] = { 251 &format_attr_event.attr, 252 &format_attr_umask.attr, 253 NULL, 254 }; 255 256 static struct attribute_group amd_uncore_format_group = { 257 .name = "format", 258 .attrs = amd_uncore_format_attr, 259 }; 260 261 static const struct attribute_group *amd_uncore_attr_groups[] = { 262 &amd_uncore_attr_group, 263 &amd_uncore_format_group, 264 NULL, 265 }; 266 267 static struct pmu amd_nb_pmu = { 268 .task_ctx_nr = perf_invalid_context, 269 .attr_groups = amd_uncore_attr_groups, 270 .name = "amd_nb", 271 .event_init = amd_uncore_event_init, 272 .add = amd_uncore_add, 273 .del = amd_uncore_del, 274 .start = amd_uncore_start, 275 .stop = amd_uncore_stop, 276 .read = amd_uncore_read, 277 }; 278 279 static struct pmu amd_l2_pmu = { 280 .task_ctx_nr = perf_invalid_context, 281 .attr_groups = amd_uncore_attr_groups, 282 .name = "amd_l2", 283 .event_init = amd_uncore_event_init, 284 .add = amd_uncore_add, 285 .del = amd_uncore_del, 286 .start = amd_uncore_start, 287 .stop = amd_uncore_stop, 288 .read = amd_uncore_read, 289 }; 290 291 static struct amd_uncore *amd_uncore_alloc(unsigned int cpu) 292 { 293 return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL, 294 cpu_to_node(cpu)); 295 } 296 297 static int amd_uncore_cpu_up_prepare(unsigned int cpu) 298 { 299 struct amd_uncore *uncore_nb = NULL, *uncore_l2; 300 301 if (amd_uncore_nb) { 302 uncore_nb = amd_uncore_alloc(cpu); 303 if (!uncore_nb) 304 goto fail; 305 uncore_nb->cpu = cpu; 306 uncore_nb->num_counters = NUM_COUNTERS_NB; 307 uncore_nb->rdpmc_base = RDPMC_BASE_NB; 308 uncore_nb->msr_base = MSR_F15H_NB_PERF_CTL; 309 uncore_nb->active_mask = &amd_nb_active_mask; 310 uncore_nb->pmu = &amd_nb_pmu; 311 uncore_nb->id = -1; 312 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore_nb; 313 } 314 315 if (amd_uncore_l2) { 316 uncore_l2 = amd_uncore_alloc(cpu); 317 if (!uncore_l2) 318 goto fail; 319 uncore_l2->cpu = cpu; 320 uncore_l2->num_counters = NUM_COUNTERS_L2; 321 uncore_l2->rdpmc_base = RDPMC_BASE_L2; 322 uncore_l2->msr_base = MSR_F16H_L2I_PERF_CTL; 323 uncore_l2->active_mask = &amd_l2_active_mask; 324 uncore_l2->pmu = &amd_l2_pmu; 325 uncore_l2->id = -1; 326 *per_cpu_ptr(amd_uncore_l2, cpu) = uncore_l2; 327 } 328 329 return 0; 330 331 fail: 332 if (amd_uncore_nb) 333 *per_cpu_ptr(amd_uncore_nb, cpu) = NULL; 334 kfree(uncore_nb); 335 return -ENOMEM; 336 } 337 338 static struct amd_uncore * 339 amd_uncore_find_online_sibling(struct amd_uncore *this, 340 struct amd_uncore * __percpu *uncores) 341 { 342 unsigned int cpu; 343 struct amd_uncore *that; 344 345 for_each_online_cpu(cpu) { 346 that = *per_cpu_ptr(uncores, cpu); 347 348 if (!that) 349 continue; 350 351 if (this == that) 352 continue; 353 354 if (this->id == that->id) { 355 hlist_add_head(&this->node, &uncore_unused_list); 356 this = that; 357 break; 358 } 359 } 360 361 this->refcnt++; 362 return this; 363 } 364 365 static int amd_uncore_cpu_starting(unsigned int cpu) 366 { 367 unsigned int eax, ebx, ecx, edx; 368 struct amd_uncore *uncore; 369 370 if (amd_uncore_nb) { 371 uncore = *per_cpu_ptr(amd_uncore_nb, cpu); 372 cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); 373 uncore->id = ecx & 0xff; 374 375 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb); 376 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore; 377 } 378 379 if (amd_uncore_l2) { 380 unsigned int apicid = cpu_data(cpu).apicid; 381 unsigned int nshared; 382 383 uncore = *per_cpu_ptr(amd_uncore_l2, cpu); 384 cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx); 385 nshared = ((eax >> 14) & 0xfff) + 1; 386 uncore->id = apicid - (apicid % nshared); 387 388 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2); 389 *per_cpu_ptr(amd_uncore_l2, cpu) = uncore; 390 } 391 392 return 0; 393 } 394 395 static void uncore_clean_online(void) 396 { 397 struct amd_uncore *uncore; 398 struct hlist_node *n; 399 400 hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { 401 hlist_del(&uncore->node); 402 kfree(uncore); 403 } 404 } 405 406 static void uncore_online(unsigned int cpu, 407 struct amd_uncore * __percpu *uncores) 408 { 409 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 410 411 uncore_clean_online(); 412 413 if (cpu == uncore->cpu) 414 cpumask_set_cpu(cpu, uncore->active_mask); 415 } 416 417 static int amd_uncore_cpu_online(unsigned int cpu) 418 { 419 if (amd_uncore_nb) 420 uncore_online(cpu, amd_uncore_nb); 421 422 if (amd_uncore_l2) 423 uncore_online(cpu, amd_uncore_l2); 424 425 return 0; 426 } 427 428 static void uncore_down_prepare(unsigned int cpu, 429 struct amd_uncore * __percpu *uncores) 430 { 431 unsigned int i; 432 struct amd_uncore *this = *per_cpu_ptr(uncores, cpu); 433 434 if (this->cpu != cpu) 435 return; 436 437 /* this cpu is going down, migrate to a shared sibling if possible */ 438 for_each_online_cpu(i) { 439 struct amd_uncore *that = *per_cpu_ptr(uncores, i); 440 441 if (cpu == i) 442 continue; 443 444 if (this == that) { 445 perf_pmu_migrate_context(this->pmu, cpu, i); 446 cpumask_clear_cpu(cpu, that->active_mask); 447 cpumask_set_cpu(i, that->active_mask); 448 that->cpu = i; 449 break; 450 } 451 } 452 } 453 454 static int amd_uncore_cpu_down_prepare(unsigned int cpu) 455 { 456 if (amd_uncore_nb) 457 uncore_down_prepare(cpu, amd_uncore_nb); 458 459 if (amd_uncore_l2) 460 uncore_down_prepare(cpu, amd_uncore_l2); 461 462 return 0; 463 } 464 465 static void uncore_dead(unsigned int cpu, struct amd_uncore * __percpu *uncores) 466 { 467 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu); 468 469 if (cpu == uncore->cpu) 470 cpumask_clear_cpu(cpu, uncore->active_mask); 471 472 if (!--uncore->refcnt) 473 kfree(uncore); 474 *per_cpu_ptr(uncores, cpu) = NULL; 475 } 476 477 static int amd_uncore_cpu_dead(unsigned int cpu) 478 { 479 if (amd_uncore_nb) 480 uncore_dead(cpu, amd_uncore_nb); 481 482 if (amd_uncore_l2) 483 uncore_dead(cpu, amd_uncore_l2); 484 485 return 0; 486 } 487 488 static int __init amd_uncore_init(void) 489 { 490 int ret = -ENODEV; 491 492 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD) 493 goto fail_nodev; 494 495 if (!boot_cpu_has(X86_FEATURE_TOPOEXT)) 496 goto fail_nodev; 497 498 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) { 499 amd_uncore_nb = alloc_percpu(struct amd_uncore *); 500 if (!amd_uncore_nb) { 501 ret = -ENOMEM; 502 goto fail_nb; 503 } 504 ret = perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1); 505 if (ret) 506 goto fail_nb; 507 508 pr_info("perf: AMD NB counters detected\n"); 509 ret = 0; 510 } 511 512 if (boot_cpu_has(X86_FEATURE_PERFCTR_L2)) { 513 amd_uncore_l2 = alloc_percpu(struct amd_uncore *); 514 if (!amd_uncore_l2) { 515 ret = -ENOMEM; 516 goto fail_l2; 517 } 518 ret = perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1); 519 if (ret) 520 goto fail_l2; 521 522 pr_info("perf: AMD L2I counters detected\n"); 523 ret = 0; 524 } 525 526 /* 527 * Install callbacks. Core will call them for each online cpu. 528 */ 529 if (cpuhp_setup_state(CPUHP_PERF_X86_AMD_UNCORE_PREP, 530 "perf/x86/amd/uncore:prepare", 531 amd_uncore_cpu_up_prepare, amd_uncore_cpu_dead)) 532 goto fail_l2; 533 534 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING, 535 "perf/x86/amd/uncore:starting", 536 amd_uncore_cpu_starting, NULL)) 537 goto fail_prep; 538 if (cpuhp_setup_state(CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE, 539 "perf/x86/amd/uncore:online", 540 amd_uncore_cpu_online, 541 amd_uncore_cpu_down_prepare)) 542 goto fail_start; 543 return 0; 544 545 fail_start: 546 cpuhp_remove_state(CPUHP_AP_PERF_X86_AMD_UNCORE_STARTING); 547 fail_prep: 548 cpuhp_remove_state(CPUHP_PERF_X86_AMD_UNCORE_PREP); 549 fail_l2: 550 if (boot_cpu_has(X86_FEATURE_PERFCTR_NB)) 551 perf_pmu_unregister(&amd_nb_pmu); 552 if (amd_uncore_l2) 553 free_percpu(amd_uncore_l2); 554 fail_nb: 555 if (amd_uncore_nb) 556 free_percpu(amd_uncore_nb); 557 558 fail_nodev: 559 return ret; 560 } 561 device_initcall(amd_uncore_init); 562