1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved. 3 */ 4 #include <linux/acpi.h> 5 #include <linux/bitops.h> 6 #include <linux/bug.h> 7 #include <linux/cpuhotplug.h> 8 #include <linux/cpumask.h> 9 #include <linux/device.h> 10 #include <linux/errno.h> 11 #include <linux/interrupt.h> 12 #include <linux/irq.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/percpu.h> 16 #include <linux/perf_event.h> 17 #include <linux/platform_device.h> 18 #include <linux/smp.h> 19 #include <linux/spinlock.h> 20 #include <linux/sysfs.h> 21 #include <linux/types.h> 22 23 #include <asm/barrier.h> 24 #include <asm/local64.h> 25 #include <asm/sysreg.h> 26 #include <soc/qcom/kryo-l2-accessors.h> 27 28 #define MAX_L2_CTRS 9 29 30 #define L2PMCR_NUM_EV_SHIFT 11 31 #define L2PMCR_NUM_EV_MASK 0x1F 32 33 #define L2PMCR 0x400 34 #define L2PMCNTENCLR 0x403 35 #define L2PMCNTENSET 0x404 36 #define L2PMINTENCLR 0x405 37 #define L2PMINTENSET 0x406 38 #define L2PMOVSCLR 0x407 39 #define L2PMOVSSET 0x408 40 #define L2PMCCNTCR 0x409 41 #define L2PMCCNTR 0x40A 42 #define L2PMCCNTSR 0x40C 43 #define L2PMRESR 0x410 44 #define IA_L2PMXEVCNTCR_BASE 0x420 45 #define IA_L2PMXEVCNTR_BASE 0x421 46 #define IA_L2PMXEVFILTER_BASE 0x423 47 #define IA_L2PMXEVTYPER_BASE 0x424 48 49 #define IA_L2_REG_OFFSET 0x10 50 51 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000 52 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004 53 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003 54 55 #define L2EVTYPER_REG_SHIFT 3 56 57 #define L2PMRESR_GROUP_BITS 8 58 #define L2PMRESR_GROUP_MASK GENMASK(7, 0) 59 60 #define L2CYCLE_CTR_BIT 31 61 #define L2CYCLE_CTR_RAW_CODE 0xFE 62 63 #define L2PMCR_RESET_ALL 0x6 64 #define L2PMCR_COUNTERS_ENABLE 0x1 65 #define L2PMCR_COUNTERS_DISABLE 0x0 66 67 #define L2PMRESR_EN BIT_ULL(63) 68 69 #define L2_EVT_MASK 0x00000FFF 70 #define L2_EVT_CODE_MASK 0x00000FF0 71 #define L2_EVT_GRP_MASK 0x0000000F 72 #define L2_EVT_CODE_SHIFT 4 73 #define L2_EVT_GRP_SHIFT 0 74 75 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT) 76 #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT) 77 78 #define L2_EVT_GROUP_MAX 7 79 80 #define L2_COUNTER_RELOAD BIT_ULL(31) 81 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63) 82 83 84 #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE) 85 86 /* 87 * Events 88 */ 89 #define L2_EVENT_CYCLES 0xfe 90 #define L2_EVENT_DCACHE_OPS 0x400 91 #define L2_EVENT_ICACHE_OPS 0x401 92 #define L2_EVENT_TLBI 0x402 93 #define L2_EVENT_BARRIERS 0x403 94 #define L2_EVENT_TOTAL_READS 0x405 95 #define L2_EVENT_TOTAL_WRITES 0x406 96 #define L2_EVENT_TOTAL_REQUESTS 0x407 97 #define L2_EVENT_LDREX 0x420 98 #define L2_EVENT_STREX 0x421 99 #define L2_EVENT_CLREX 0x422 100 101 102 103 struct cluster_pmu; 104 105 /* 106 * Aggregate PMU. Implements the core pmu functions and manages 107 * the hardware PMUs. 108 */ 109 struct l2cache_pmu { 110 struct hlist_node node; 111 u32 num_pmus; 112 struct pmu pmu; 113 int num_counters; 114 cpumask_t cpumask; 115 struct platform_device *pdev; 116 struct cluster_pmu * __percpu *pmu_cluster; 117 struct list_head clusters; 118 }; 119 120 /* 121 * The cache is made up of one or more clusters, each cluster has its own PMU. 122 * Each cluster is associated with one or more CPUs. 123 * This structure represents one of the hardware PMUs. 124 * 125 * Events can be envisioned as a 2-dimensional array. Each column represents 126 * a group of events. There are 8 groups. Only one entry from each 127 * group can be in use at a time. 128 * 129 * Events are specified as 0xCCG, where CC is 2 hex digits specifying 130 * the code (array row) and G specifies the group (column). 131 * 132 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE 133 * which is outside the above scheme. 134 */ 135 struct cluster_pmu { 136 struct list_head next; 137 struct perf_event *events[MAX_L2_CTRS]; 138 struct l2cache_pmu *l2cache_pmu; 139 DECLARE_BITMAP(used_counters, MAX_L2_CTRS); 140 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1); 141 int irq; 142 int cluster_id; 143 /* The CPU that is used for collecting events on this cluster */ 144 int on_cpu; 145 /* All the CPUs associated with this cluster */ 146 cpumask_t cluster_cpus; 147 spinlock_t pmu_lock; 148 }; 149 150 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu)) 151 152 static u32 l2_cycle_ctr_idx; 153 static u32 l2_counter_present_mask; 154 155 static inline u32 idx_to_reg_bit(u32 idx) 156 { 157 if (idx == l2_cycle_ctr_idx) 158 return BIT(L2CYCLE_CTR_BIT); 159 160 return BIT(idx); 161 } 162 163 static inline struct cluster_pmu *get_cluster_pmu( 164 struct l2cache_pmu *l2cache_pmu, int cpu) 165 { 166 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu); 167 } 168 169 static void cluster_pmu_reset(void) 170 { 171 /* Reset all counters */ 172 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_RESET_ALL); 173 kryo_l2_set_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask); 174 kryo_l2_set_indirect_reg(L2PMINTENCLR, l2_counter_present_mask); 175 kryo_l2_set_indirect_reg(L2PMOVSCLR, l2_counter_present_mask); 176 } 177 178 static inline void cluster_pmu_enable(void) 179 { 180 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE); 181 } 182 183 static inline void cluster_pmu_disable(void) 184 { 185 kryo_l2_set_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE); 186 } 187 188 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value) 189 { 190 if (idx == l2_cycle_ctr_idx) 191 kryo_l2_set_indirect_reg(L2PMCCNTR, value); 192 else 193 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value); 194 } 195 196 static inline u64 cluster_pmu_counter_get_value(u32 idx) 197 { 198 u64 value; 199 200 if (idx == l2_cycle_ctr_idx) 201 value = kryo_l2_get_indirect_reg(L2PMCCNTR); 202 else 203 value = kryo_l2_get_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx)); 204 205 return value; 206 } 207 208 static inline void cluster_pmu_counter_enable(u32 idx) 209 { 210 kryo_l2_set_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx)); 211 } 212 213 static inline void cluster_pmu_counter_disable(u32 idx) 214 { 215 kryo_l2_set_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx)); 216 } 217 218 static inline void cluster_pmu_counter_enable_interrupt(u32 idx) 219 { 220 kryo_l2_set_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx)); 221 } 222 223 static inline void cluster_pmu_counter_disable_interrupt(u32 idx) 224 { 225 kryo_l2_set_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx)); 226 } 227 228 static inline void cluster_pmu_set_evccntcr(u32 val) 229 { 230 kryo_l2_set_indirect_reg(L2PMCCNTCR, val); 231 } 232 233 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val) 234 { 235 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val); 236 } 237 238 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val) 239 { 240 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val); 241 } 242 243 static void cluster_pmu_set_resr(struct cluster_pmu *cluster, 244 u32 event_group, u32 event_cc) 245 { 246 u64 field; 247 u64 resr_val; 248 u32 shift; 249 unsigned long flags; 250 251 shift = L2PMRESR_GROUP_BITS * event_group; 252 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift); 253 254 spin_lock_irqsave(&cluster->pmu_lock, flags); 255 256 resr_val = kryo_l2_get_indirect_reg(L2PMRESR); 257 resr_val &= ~(L2PMRESR_GROUP_MASK << shift); 258 resr_val |= field; 259 resr_val |= L2PMRESR_EN; 260 kryo_l2_set_indirect_reg(L2PMRESR, resr_val); 261 262 spin_unlock_irqrestore(&cluster->pmu_lock, flags); 263 } 264 265 /* 266 * Hardware allows filtering of events based on the originating 267 * CPU. Turn this off by setting filter bits to allow events from 268 * all CPUS, subunits and ID independent events in this cluster. 269 */ 270 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr) 271 { 272 u32 val = L2PMXEVFILTER_SUFILTER_ALL | 273 L2PMXEVFILTER_ORGFILTER_IDINDEP | 274 L2PMXEVFILTER_ORGFILTER_ALL; 275 276 kryo_l2_set_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val); 277 } 278 279 static inline u32 cluster_pmu_getreset_ovsr(void) 280 { 281 u32 result = kryo_l2_get_indirect_reg(L2PMOVSSET); 282 283 kryo_l2_set_indirect_reg(L2PMOVSCLR, result); 284 return result; 285 } 286 287 static inline bool cluster_pmu_has_overflowed(u32 ovsr) 288 { 289 return !!(ovsr & l2_counter_present_mask); 290 } 291 292 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx) 293 { 294 return !!(ovsr & idx_to_reg_bit(idx)); 295 } 296 297 static void l2_cache_event_update(struct perf_event *event) 298 { 299 struct hw_perf_event *hwc = &event->hw; 300 u64 delta, prev, now; 301 u32 idx = hwc->idx; 302 303 do { 304 prev = local64_read(&hwc->prev_count); 305 now = cluster_pmu_counter_get_value(idx); 306 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 307 308 /* 309 * The cycle counter is 64-bit, but all other counters are 310 * 32-bit, and we must handle 32-bit overflow explicitly. 311 */ 312 delta = now - prev; 313 if (idx != l2_cycle_ctr_idx) 314 delta &= 0xffffffff; 315 316 local64_add(delta, &event->count); 317 } 318 319 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster, 320 struct hw_perf_event *hwc) 321 { 322 u32 idx = hwc->idx; 323 u64 new; 324 325 /* 326 * We limit the max period to half the max counter value so 327 * that even in the case of extreme interrupt latency the 328 * counter will (hopefully) not wrap past its initial value. 329 */ 330 if (idx == l2_cycle_ctr_idx) 331 new = L2_CYCLE_COUNTER_RELOAD; 332 else 333 new = L2_COUNTER_RELOAD; 334 335 local64_set(&hwc->prev_count, new); 336 cluster_pmu_counter_set_value(idx, new); 337 } 338 339 static int l2_cache_get_event_idx(struct cluster_pmu *cluster, 340 struct perf_event *event) 341 { 342 struct hw_perf_event *hwc = &event->hw; 343 int idx; 344 int num_ctrs = cluster->l2cache_pmu->num_counters - 1; 345 unsigned int group; 346 347 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 348 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters)) 349 return -EAGAIN; 350 351 return l2_cycle_ctr_idx; 352 } 353 354 idx = find_first_zero_bit(cluster->used_counters, num_ctrs); 355 if (idx == num_ctrs) 356 /* The counters are all in use. */ 357 return -EAGAIN; 358 359 /* 360 * Check for column exclusion: event column already in use by another 361 * event. This is for events which are not in the same group. 362 * Conflicting events in the same group are detected in event_init. 363 */ 364 group = L2_EVT_GROUP(hwc->config_base); 365 if (test_bit(group, cluster->used_groups)) 366 return -EAGAIN; 367 368 set_bit(idx, cluster->used_counters); 369 set_bit(group, cluster->used_groups); 370 371 return idx; 372 } 373 374 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster, 375 struct perf_event *event) 376 { 377 struct hw_perf_event *hwc = &event->hw; 378 int idx = hwc->idx; 379 380 clear_bit(idx, cluster->used_counters); 381 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE) 382 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups); 383 } 384 385 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data) 386 { 387 struct cluster_pmu *cluster = data; 388 int num_counters = cluster->l2cache_pmu->num_counters; 389 u32 ovsr; 390 int idx; 391 392 ovsr = cluster_pmu_getreset_ovsr(); 393 if (!cluster_pmu_has_overflowed(ovsr)) 394 return IRQ_NONE; 395 396 for_each_set_bit(idx, cluster->used_counters, num_counters) { 397 struct perf_event *event = cluster->events[idx]; 398 struct hw_perf_event *hwc; 399 400 if (WARN_ON_ONCE(!event)) 401 continue; 402 403 if (!cluster_pmu_counter_has_overflowed(ovsr, idx)) 404 continue; 405 406 l2_cache_event_update(event); 407 hwc = &event->hw; 408 409 l2_cache_cluster_set_period(cluster, hwc); 410 } 411 412 return IRQ_HANDLED; 413 } 414 415 /* 416 * Implementation of abstract pmu functionality required by 417 * the core perf events code. 418 */ 419 420 static void l2_cache_pmu_enable(struct pmu *pmu) 421 { 422 /* 423 * Although there is only one PMU (per socket) controlling multiple 424 * physical PMUs (per cluster), because we do not support per-task mode 425 * each event is associated with a CPU. Each event has pmu_enable 426 * called on its CPU, so here it is only necessary to enable the 427 * counters for the current CPU. 428 */ 429 430 cluster_pmu_enable(); 431 } 432 433 static void l2_cache_pmu_disable(struct pmu *pmu) 434 { 435 cluster_pmu_disable(); 436 } 437 438 static int l2_cache_event_init(struct perf_event *event) 439 { 440 struct hw_perf_event *hwc = &event->hw; 441 struct cluster_pmu *cluster; 442 struct perf_event *sibling; 443 struct l2cache_pmu *l2cache_pmu; 444 445 if (event->attr.type != event->pmu->type) 446 return -ENOENT; 447 448 l2cache_pmu = to_l2cache_pmu(event->pmu); 449 450 if (hwc->sample_period) { 451 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 452 "Sampling not supported\n"); 453 return -EOPNOTSUPP; 454 } 455 456 if (event->cpu < 0) { 457 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 458 "Per-task mode not supported\n"); 459 return -EOPNOTSUPP; 460 } 461 462 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) || 463 ((event->attr.config & ~L2_EVT_MASK) != 0)) && 464 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) { 465 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 466 "Invalid config %llx\n", 467 event->attr.config); 468 return -EINVAL; 469 } 470 471 /* Don't allow groups with mixed PMUs, except for s/w events */ 472 if (event->group_leader->pmu != event->pmu && 473 !is_software_event(event->group_leader)) { 474 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 475 "Can't create mixed PMU group\n"); 476 return -EINVAL; 477 } 478 479 for_each_sibling_event(sibling, event->group_leader) { 480 if (sibling->pmu != event->pmu && 481 !is_software_event(sibling)) { 482 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 483 "Can't create mixed PMU group\n"); 484 return -EINVAL; 485 } 486 } 487 488 cluster = get_cluster_pmu(l2cache_pmu, event->cpu); 489 if (!cluster) { 490 /* CPU has not been initialised */ 491 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 492 "CPU%d not associated with L2 cluster\n", event->cpu); 493 return -EINVAL; 494 } 495 496 /* Ensure all events in a group are on the same cpu */ 497 if ((event->group_leader != event) && 498 (cluster->on_cpu != event->group_leader->cpu)) { 499 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 500 "Can't create group on CPUs %d and %d", 501 event->cpu, event->group_leader->cpu); 502 return -EINVAL; 503 } 504 505 if ((event != event->group_leader) && 506 !is_software_event(event->group_leader) && 507 (L2_EVT_GROUP(event->group_leader->attr.config) == 508 L2_EVT_GROUP(event->attr.config))) { 509 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 510 "Column exclusion: conflicting events %llx %llx\n", 511 event->group_leader->attr.config, 512 event->attr.config); 513 return -EINVAL; 514 } 515 516 for_each_sibling_event(sibling, event->group_leader) { 517 if ((sibling != event) && 518 !is_software_event(sibling) && 519 (L2_EVT_GROUP(sibling->attr.config) == 520 L2_EVT_GROUP(event->attr.config))) { 521 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 522 "Column exclusion: conflicting events %llx %llx\n", 523 sibling->attr.config, 524 event->attr.config); 525 return -EINVAL; 526 } 527 } 528 529 hwc->idx = -1; 530 hwc->config_base = event->attr.config; 531 532 /* 533 * Ensure all events are on the same cpu so all events are in the 534 * same cpu context, to avoid races on pmu_enable etc. 535 */ 536 event->cpu = cluster->on_cpu; 537 538 return 0; 539 } 540 541 static void l2_cache_event_start(struct perf_event *event, int flags) 542 { 543 struct cluster_pmu *cluster; 544 struct hw_perf_event *hwc = &event->hw; 545 int idx = hwc->idx; 546 u32 config; 547 u32 event_cc, event_group; 548 549 hwc->state = 0; 550 551 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 552 553 l2_cache_cluster_set_period(cluster, hwc); 554 555 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 556 cluster_pmu_set_evccntcr(0); 557 } else { 558 config = hwc->config_base; 559 event_cc = L2_EVT_CODE(config); 560 event_group = L2_EVT_GROUP(config); 561 562 cluster_pmu_set_evcntcr(idx, 0); 563 cluster_pmu_set_evtyper(idx, event_group); 564 cluster_pmu_set_resr(cluster, event_group, event_cc); 565 cluster_pmu_set_evfilter_sys_mode(idx); 566 } 567 568 cluster_pmu_counter_enable_interrupt(idx); 569 cluster_pmu_counter_enable(idx); 570 } 571 572 static void l2_cache_event_stop(struct perf_event *event, int flags) 573 { 574 struct hw_perf_event *hwc = &event->hw; 575 int idx = hwc->idx; 576 577 if (hwc->state & PERF_HES_STOPPED) 578 return; 579 580 cluster_pmu_counter_disable_interrupt(idx); 581 cluster_pmu_counter_disable(idx); 582 583 if (flags & PERF_EF_UPDATE) 584 l2_cache_event_update(event); 585 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 586 } 587 588 static int l2_cache_event_add(struct perf_event *event, int flags) 589 { 590 struct hw_perf_event *hwc = &event->hw; 591 int idx; 592 int err = 0; 593 struct cluster_pmu *cluster; 594 595 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 596 597 idx = l2_cache_get_event_idx(cluster, event); 598 if (idx < 0) 599 return idx; 600 601 hwc->idx = idx; 602 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 603 cluster->events[idx] = event; 604 local64_set(&hwc->prev_count, 0); 605 606 if (flags & PERF_EF_START) 607 l2_cache_event_start(event, flags); 608 609 /* Propagate changes to the userspace mapping. */ 610 perf_event_update_userpage(event); 611 612 return err; 613 } 614 615 static void l2_cache_event_del(struct perf_event *event, int flags) 616 { 617 struct hw_perf_event *hwc = &event->hw; 618 struct cluster_pmu *cluster; 619 int idx = hwc->idx; 620 621 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 622 623 l2_cache_event_stop(event, flags | PERF_EF_UPDATE); 624 cluster->events[idx] = NULL; 625 l2_cache_clear_event_idx(cluster, event); 626 627 perf_event_update_userpage(event); 628 } 629 630 static void l2_cache_event_read(struct perf_event *event) 631 { 632 l2_cache_event_update(event); 633 } 634 635 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev, 636 struct device_attribute *attr, 637 char *buf) 638 { 639 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev)); 640 641 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask); 642 } 643 644 static struct device_attribute l2_cache_pmu_cpumask_attr = 645 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL); 646 647 static struct attribute *l2_cache_pmu_cpumask_attrs[] = { 648 &l2_cache_pmu_cpumask_attr.attr, 649 NULL, 650 }; 651 652 static const struct attribute_group l2_cache_pmu_cpumask_group = { 653 .attrs = l2_cache_pmu_cpumask_attrs, 654 }; 655 656 /* CCG format for perf RAW codes. */ 657 PMU_FORMAT_ATTR(l2_code, "config:4-11"); 658 PMU_FORMAT_ATTR(l2_group, "config:0-3"); 659 PMU_FORMAT_ATTR(event, "config:0-11"); 660 661 static struct attribute *l2_cache_pmu_formats[] = { 662 &format_attr_l2_code.attr, 663 &format_attr_l2_group.attr, 664 &format_attr_event.attr, 665 NULL, 666 }; 667 668 static const struct attribute_group l2_cache_pmu_format_group = { 669 .name = "format", 670 .attrs = l2_cache_pmu_formats, 671 }; 672 673 static ssize_t l2cache_pmu_event_show(struct device *dev, 674 struct device_attribute *attr, char *page) 675 { 676 struct perf_pmu_events_attr *pmu_attr; 677 678 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 679 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 680 } 681 682 #define L2CACHE_EVENT_ATTR(_name, _id) \ 683 PMU_EVENT_ATTR_ID(_name, l2cache_pmu_event_show, _id) 684 685 static struct attribute *l2_cache_pmu_events[] = { 686 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES), 687 L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS), 688 L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS), 689 L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI), 690 L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS), 691 L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS), 692 L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES), 693 L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS), 694 L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX), 695 L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX), 696 L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX), 697 NULL 698 }; 699 700 static const struct attribute_group l2_cache_pmu_events_group = { 701 .name = "events", 702 .attrs = l2_cache_pmu_events, 703 }; 704 705 static const struct attribute_group *l2_cache_pmu_attr_grps[] = { 706 &l2_cache_pmu_format_group, 707 &l2_cache_pmu_cpumask_group, 708 &l2_cache_pmu_events_group, 709 NULL, 710 }; 711 712 /* 713 * Generic device handlers 714 */ 715 716 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = { 717 { "QCOM8130", }, 718 { } 719 }; 720 721 static int get_num_counters(void) 722 { 723 int val; 724 725 val = kryo_l2_get_indirect_reg(L2PMCR); 726 727 /* 728 * Read number of counters from L2PMCR and add 1 729 * for the cycle counter. 730 */ 731 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1; 732 } 733 734 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster( 735 struct l2cache_pmu *l2cache_pmu, int cpu) 736 { 737 u64 mpidr; 738 int cpu_cluster_id; 739 struct cluster_pmu *cluster; 740 741 /* 742 * This assumes that the cluster_id is in MPIDR[aff1] for 743 * single-threaded cores, and MPIDR[aff2] for multi-threaded 744 * cores. This logic will have to be updated if this changes. 745 */ 746 mpidr = read_cpuid_mpidr(); 747 if (mpidr & MPIDR_MT_BITMASK) 748 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); 749 else 750 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); 751 752 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) { 753 if (cluster->cluster_id != cpu_cluster_id) 754 continue; 755 756 dev_info(&l2cache_pmu->pdev->dev, 757 "CPU%d associated with cluster %d\n", cpu, 758 cluster->cluster_id); 759 cpumask_set_cpu(cpu, &cluster->cluster_cpus); 760 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster; 761 return cluster; 762 } 763 764 return NULL; 765 } 766 767 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 768 { 769 struct cluster_pmu *cluster; 770 struct l2cache_pmu *l2cache_pmu; 771 772 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 773 cluster = get_cluster_pmu(l2cache_pmu, cpu); 774 if (!cluster) { 775 /* First time this CPU has come online */ 776 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu); 777 if (!cluster) { 778 /* Only if broken firmware doesn't list every cluster */ 779 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu); 780 return 0; 781 } 782 } 783 784 /* If another CPU is managing this cluster, we're done */ 785 if (cluster->on_cpu != -1) 786 return 0; 787 788 /* 789 * All CPUs on this cluster were down, use this one. 790 * Reset to put it into sane state. 791 */ 792 cluster->on_cpu = cpu; 793 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask); 794 cluster_pmu_reset(); 795 796 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu))); 797 enable_irq(cluster->irq); 798 799 return 0; 800 } 801 802 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 803 { 804 struct cluster_pmu *cluster; 805 struct l2cache_pmu *l2cache_pmu; 806 cpumask_t cluster_online_cpus; 807 unsigned int target; 808 809 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 810 cluster = get_cluster_pmu(l2cache_pmu, cpu); 811 if (!cluster) 812 return 0; 813 814 /* If this CPU is not managing the cluster, we're done */ 815 if (cluster->on_cpu != cpu) 816 return 0; 817 818 /* Give up ownership of cluster */ 819 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask); 820 cluster->on_cpu = -1; 821 822 /* Any other CPU for this cluster which is still online */ 823 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus, 824 cpu_online_mask); 825 target = cpumask_any_but(&cluster_online_cpus, cpu); 826 if (target >= nr_cpu_ids) { 827 disable_irq(cluster->irq); 828 return 0; 829 } 830 831 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target); 832 cluster->on_cpu = target; 833 cpumask_set_cpu(target, &l2cache_pmu->cpumask); 834 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target))); 835 836 return 0; 837 } 838 839 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) 840 { 841 struct platform_device *pdev = to_platform_device(dev->parent); 842 struct platform_device *sdev = to_platform_device(dev); 843 struct l2cache_pmu *l2cache_pmu = data; 844 struct cluster_pmu *cluster; 845 u64 fw_cluster_id; 846 int err; 847 int irq; 848 849 err = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &fw_cluster_id); 850 if (err) { 851 dev_err(&pdev->dev, "unable to read ACPI uid\n"); 852 return err; 853 } 854 855 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL); 856 if (!cluster) 857 return -ENOMEM; 858 859 INIT_LIST_HEAD(&cluster->next); 860 list_add(&cluster->next, &l2cache_pmu->clusters); 861 cluster->cluster_id = fw_cluster_id; 862 863 irq = platform_get_irq(sdev, 0); 864 if (irq < 0) 865 return irq; 866 cluster->irq = irq; 867 868 cluster->l2cache_pmu = l2cache_pmu; 869 cluster->on_cpu = -1; 870 871 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, 872 IRQF_NOBALANCING | IRQF_NO_THREAD | 873 IRQF_NO_AUTOEN, 874 "l2-cache-pmu", cluster); 875 if (err) { 876 dev_err(&pdev->dev, 877 "Unable to request IRQ%d for L2 PMU counters\n", irq); 878 return err; 879 } 880 881 dev_info(&pdev->dev, 882 "Registered L2 cache PMU cluster %lld\n", fw_cluster_id); 883 884 spin_lock_init(&cluster->pmu_lock); 885 886 l2cache_pmu->num_pmus++; 887 888 return 0; 889 } 890 891 static int l2_cache_pmu_probe(struct platform_device *pdev) 892 { 893 int err; 894 struct l2cache_pmu *l2cache_pmu; 895 896 l2cache_pmu = 897 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL); 898 if (!l2cache_pmu) 899 return -ENOMEM; 900 901 INIT_LIST_HEAD(&l2cache_pmu->clusters); 902 903 platform_set_drvdata(pdev, l2cache_pmu); 904 l2cache_pmu->pmu = (struct pmu) { 905 /* suffix is instance id for future use with multiple sockets */ 906 .name = "l2cache_0", 907 .task_ctx_nr = perf_invalid_context, 908 .pmu_enable = l2_cache_pmu_enable, 909 .pmu_disable = l2_cache_pmu_disable, 910 .event_init = l2_cache_event_init, 911 .add = l2_cache_event_add, 912 .del = l2_cache_event_del, 913 .start = l2_cache_event_start, 914 .stop = l2_cache_event_stop, 915 .read = l2_cache_event_read, 916 .attr_groups = l2_cache_pmu_attr_grps, 917 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 918 }; 919 920 l2cache_pmu->num_counters = get_num_counters(); 921 l2cache_pmu->pdev = pdev; 922 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev, 923 struct cluster_pmu *); 924 if (!l2cache_pmu->pmu_cluster) 925 return -ENOMEM; 926 927 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1; 928 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) | 929 BIT(L2CYCLE_CTR_BIT); 930 931 cpumask_clear(&l2cache_pmu->cpumask); 932 933 /* Read cluster info and initialize each cluster */ 934 err = device_for_each_child(&pdev->dev, l2cache_pmu, 935 l2_cache_pmu_probe_cluster); 936 if (err) 937 return err; 938 939 if (l2cache_pmu->num_pmus == 0) { 940 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n"); 941 return -ENODEV; 942 } 943 944 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 945 &l2cache_pmu->node); 946 if (err) { 947 dev_err(&pdev->dev, "Error %d registering hotplug", err); 948 return err; 949 } 950 951 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1); 952 if (err) { 953 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err); 954 goto out_unregister; 955 } 956 957 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n", 958 l2cache_pmu->num_pmus); 959 960 return err; 961 962 out_unregister: 963 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 964 &l2cache_pmu->node); 965 return err; 966 } 967 968 static int l2_cache_pmu_remove(struct platform_device *pdev) 969 { 970 struct l2cache_pmu *l2cache_pmu = 971 to_l2cache_pmu(platform_get_drvdata(pdev)); 972 973 perf_pmu_unregister(&l2cache_pmu->pmu); 974 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 975 &l2cache_pmu->node); 976 return 0; 977 } 978 979 static struct platform_driver l2_cache_pmu_driver = { 980 .driver = { 981 .name = "qcom-l2cache-pmu", 982 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match), 983 .suppress_bind_attrs = true, 984 }, 985 .probe = l2_cache_pmu_probe, 986 .remove = l2_cache_pmu_remove, 987 }; 988 989 static int __init register_l2_cache_pmu_driver(void) 990 { 991 int err; 992 993 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 994 "AP_PERF_ARM_QCOM_L2_ONLINE", 995 l2cache_pmu_online_cpu, 996 l2cache_pmu_offline_cpu); 997 if (err) 998 return err; 999 1000 return platform_driver_register(&l2_cache_pmu_driver); 1001 } 1002 device_initcall(register_l2_cache_pmu_driver); 1003