1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2015-2017 The Linux Foundation. All rights reserved. 3 */ 4 #include <linux/acpi.h> 5 #include <linux/bitops.h> 6 #include <linux/bug.h> 7 #include <linux/cpuhotplug.h> 8 #include <linux/cpumask.h> 9 #include <linux/device.h> 10 #include <linux/errno.h> 11 #include <linux/interrupt.h> 12 #include <linux/irq.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/percpu.h> 16 #include <linux/perf_event.h> 17 #include <linux/platform_device.h> 18 #include <linux/smp.h> 19 #include <linux/spinlock.h> 20 #include <linux/sysfs.h> 21 #include <linux/types.h> 22 23 #include <asm/barrier.h> 24 #include <asm/local64.h> 25 #include <asm/sysreg.h> 26 27 #define MAX_L2_CTRS 9 28 29 #define L2PMCR_NUM_EV_SHIFT 11 30 #define L2PMCR_NUM_EV_MASK 0x1F 31 32 #define L2PMCR 0x400 33 #define L2PMCNTENCLR 0x403 34 #define L2PMCNTENSET 0x404 35 #define L2PMINTENCLR 0x405 36 #define L2PMINTENSET 0x406 37 #define L2PMOVSCLR 0x407 38 #define L2PMOVSSET 0x408 39 #define L2PMCCNTCR 0x409 40 #define L2PMCCNTR 0x40A 41 #define L2PMCCNTSR 0x40C 42 #define L2PMRESR 0x410 43 #define IA_L2PMXEVCNTCR_BASE 0x420 44 #define IA_L2PMXEVCNTR_BASE 0x421 45 #define IA_L2PMXEVFILTER_BASE 0x423 46 #define IA_L2PMXEVTYPER_BASE 0x424 47 48 #define IA_L2_REG_OFFSET 0x10 49 50 #define L2PMXEVFILTER_SUFILTER_ALL 0x000E0000 51 #define L2PMXEVFILTER_ORGFILTER_IDINDEP 0x00000004 52 #define L2PMXEVFILTER_ORGFILTER_ALL 0x00000003 53 54 #define L2EVTYPER_REG_SHIFT 3 55 56 #define L2PMRESR_GROUP_BITS 8 57 #define L2PMRESR_GROUP_MASK GENMASK(7, 0) 58 59 #define L2CYCLE_CTR_BIT 31 60 #define L2CYCLE_CTR_RAW_CODE 0xFE 61 62 #define L2PMCR_RESET_ALL 0x6 63 #define L2PMCR_COUNTERS_ENABLE 0x1 64 #define L2PMCR_COUNTERS_DISABLE 0x0 65 66 #define L2PMRESR_EN BIT_ULL(63) 67 68 #define L2_EVT_MASK 0x00000FFF 69 #define L2_EVT_CODE_MASK 0x00000FF0 70 #define L2_EVT_GRP_MASK 0x0000000F 71 #define L2_EVT_CODE_SHIFT 4 72 #define L2_EVT_GRP_SHIFT 0 73 74 #define L2_EVT_CODE(event) (((event) & L2_EVT_CODE_MASK) >> L2_EVT_CODE_SHIFT) 75 #define L2_EVT_GROUP(event) (((event) & L2_EVT_GRP_MASK) >> L2_EVT_GRP_SHIFT) 76 77 #define L2_EVT_GROUP_MAX 7 78 79 #define L2_COUNTER_RELOAD BIT_ULL(31) 80 #define L2_CYCLE_COUNTER_RELOAD BIT_ULL(63) 81 82 #define L2CPUSRSELR_EL1 sys_reg(3, 3, 15, 0, 6) 83 #define L2CPUSRDR_EL1 sys_reg(3, 3, 15, 0, 7) 84 85 #define reg_idx(reg, i) (((i) * IA_L2_REG_OFFSET) + reg##_BASE) 86 87 /* 88 * Events 89 */ 90 #define L2_EVENT_CYCLES 0xfe 91 #define L2_EVENT_DCACHE_OPS 0x400 92 #define L2_EVENT_ICACHE_OPS 0x401 93 #define L2_EVENT_TLBI 0x402 94 #define L2_EVENT_BARRIERS 0x403 95 #define L2_EVENT_TOTAL_READS 0x405 96 #define L2_EVENT_TOTAL_WRITES 0x406 97 #define L2_EVENT_TOTAL_REQUESTS 0x407 98 #define L2_EVENT_LDREX 0x420 99 #define L2_EVENT_STREX 0x421 100 #define L2_EVENT_CLREX 0x422 101 102 static DEFINE_RAW_SPINLOCK(l2_access_lock); 103 104 /** 105 * set_l2_indirect_reg: write value to an L2 register 106 * @reg: Address of L2 register. 107 * @value: Value to be written to register. 108 * 109 * Use architecturally required barriers for ordering between system register 110 * accesses 111 */ 112 static void set_l2_indirect_reg(u64 reg, u64 val) 113 { 114 unsigned long flags; 115 116 raw_spin_lock_irqsave(&l2_access_lock, flags); 117 write_sysreg_s(reg, L2CPUSRSELR_EL1); 118 isb(); 119 write_sysreg_s(val, L2CPUSRDR_EL1); 120 isb(); 121 raw_spin_unlock_irqrestore(&l2_access_lock, flags); 122 } 123 124 /** 125 * get_l2_indirect_reg: read an L2 register value 126 * @reg: Address of L2 register. 127 * 128 * Use architecturally required barriers for ordering between system register 129 * accesses 130 */ 131 static u64 get_l2_indirect_reg(u64 reg) 132 { 133 u64 val; 134 unsigned long flags; 135 136 raw_spin_lock_irqsave(&l2_access_lock, flags); 137 write_sysreg_s(reg, L2CPUSRSELR_EL1); 138 isb(); 139 val = read_sysreg_s(L2CPUSRDR_EL1); 140 raw_spin_unlock_irqrestore(&l2_access_lock, flags); 141 142 return val; 143 } 144 145 struct cluster_pmu; 146 147 /* 148 * Aggregate PMU. Implements the core pmu functions and manages 149 * the hardware PMUs. 150 */ 151 struct l2cache_pmu { 152 struct hlist_node node; 153 u32 num_pmus; 154 struct pmu pmu; 155 int num_counters; 156 cpumask_t cpumask; 157 struct platform_device *pdev; 158 struct cluster_pmu * __percpu *pmu_cluster; 159 struct list_head clusters; 160 }; 161 162 /* 163 * The cache is made up of one or more clusters, each cluster has its own PMU. 164 * Each cluster is associated with one or more CPUs. 165 * This structure represents one of the hardware PMUs. 166 * 167 * Events can be envisioned as a 2-dimensional array. Each column represents 168 * a group of events. There are 8 groups. Only one entry from each 169 * group can be in use at a time. 170 * 171 * Events are specified as 0xCCG, where CC is 2 hex digits specifying 172 * the code (array row) and G specifies the group (column). 173 * 174 * In addition there is a cycle counter event specified by L2CYCLE_CTR_RAW_CODE 175 * which is outside the above scheme. 176 */ 177 struct cluster_pmu { 178 struct list_head next; 179 struct perf_event *events[MAX_L2_CTRS]; 180 struct l2cache_pmu *l2cache_pmu; 181 DECLARE_BITMAP(used_counters, MAX_L2_CTRS); 182 DECLARE_BITMAP(used_groups, L2_EVT_GROUP_MAX + 1); 183 int irq; 184 int cluster_id; 185 /* The CPU that is used for collecting events on this cluster */ 186 int on_cpu; 187 /* All the CPUs associated with this cluster */ 188 cpumask_t cluster_cpus; 189 spinlock_t pmu_lock; 190 }; 191 192 #define to_l2cache_pmu(p) (container_of(p, struct l2cache_pmu, pmu)) 193 194 static u32 l2_cycle_ctr_idx; 195 static u32 l2_counter_present_mask; 196 197 static inline u32 idx_to_reg_bit(u32 idx) 198 { 199 if (idx == l2_cycle_ctr_idx) 200 return BIT(L2CYCLE_CTR_BIT); 201 202 return BIT(idx); 203 } 204 205 static inline struct cluster_pmu *get_cluster_pmu( 206 struct l2cache_pmu *l2cache_pmu, int cpu) 207 { 208 return *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu); 209 } 210 211 static void cluster_pmu_reset(void) 212 { 213 /* Reset all counters */ 214 set_l2_indirect_reg(L2PMCR, L2PMCR_RESET_ALL); 215 set_l2_indirect_reg(L2PMCNTENCLR, l2_counter_present_mask); 216 set_l2_indirect_reg(L2PMINTENCLR, l2_counter_present_mask); 217 set_l2_indirect_reg(L2PMOVSCLR, l2_counter_present_mask); 218 } 219 220 static inline void cluster_pmu_enable(void) 221 { 222 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_ENABLE); 223 } 224 225 static inline void cluster_pmu_disable(void) 226 { 227 set_l2_indirect_reg(L2PMCR, L2PMCR_COUNTERS_DISABLE); 228 } 229 230 static inline void cluster_pmu_counter_set_value(u32 idx, u64 value) 231 { 232 if (idx == l2_cycle_ctr_idx) 233 set_l2_indirect_reg(L2PMCCNTR, value); 234 else 235 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx), value); 236 } 237 238 static inline u64 cluster_pmu_counter_get_value(u32 idx) 239 { 240 u64 value; 241 242 if (idx == l2_cycle_ctr_idx) 243 value = get_l2_indirect_reg(L2PMCCNTR); 244 else 245 value = get_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTR, idx)); 246 247 return value; 248 } 249 250 static inline void cluster_pmu_counter_enable(u32 idx) 251 { 252 set_l2_indirect_reg(L2PMCNTENSET, idx_to_reg_bit(idx)); 253 } 254 255 static inline void cluster_pmu_counter_disable(u32 idx) 256 { 257 set_l2_indirect_reg(L2PMCNTENCLR, idx_to_reg_bit(idx)); 258 } 259 260 static inline void cluster_pmu_counter_enable_interrupt(u32 idx) 261 { 262 set_l2_indirect_reg(L2PMINTENSET, idx_to_reg_bit(idx)); 263 } 264 265 static inline void cluster_pmu_counter_disable_interrupt(u32 idx) 266 { 267 set_l2_indirect_reg(L2PMINTENCLR, idx_to_reg_bit(idx)); 268 } 269 270 static inline void cluster_pmu_set_evccntcr(u32 val) 271 { 272 set_l2_indirect_reg(L2PMCCNTCR, val); 273 } 274 275 static inline void cluster_pmu_set_evcntcr(u32 ctr, u32 val) 276 { 277 set_l2_indirect_reg(reg_idx(IA_L2PMXEVCNTCR, ctr), val); 278 } 279 280 static inline void cluster_pmu_set_evtyper(u32 ctr, u32 val) 281 { 282 set_l2_indirect_reg(reg_idx(IA_L2PMXEVTYPER, ctr), val); 283 } 284 285 static void cluster_pmu_set_resr(struct cluster_pmu *cluster, 286 u32 event_group, u32 event_cc) 287 { 288 u64 field; 289 u64 resr_val; 290 u32 shift; 291 unsigned long flags; 292 293 shift = L2PMRESR_GROUP_BITS * event_group; 294 field = ((u64)(event_cc & L2PMRESR_GROUP_MASK) << shift); 295 296 spin_lock_irqsave(&cluster->pmu_lock, flags); 297 298 resr_val = get_l2_indirect_reg(L2PMRESR); 299 resr_val &= ~(L2PMRESR_GROUP_MASK << shift); 300 resr_val |= field; 301 resr_val |= L2PMRESR_EN; 302 set_l2_indirect_reg(L2PMRESR, resr_val); 303 304 spin_unlock_irqrestore(&cluster->pmu_lock, flags); 305 } 306 307 /* 308 * Hardware allows filtering of events based on the originating 309 * CPU. Turn this off by setting filter bits to allow events from 310 * all CPUS, subunits and ID independent events in this cluster. 311 */ 312 static inline void cluster_pmu_set_evfilter_sys_mode(u32 ctr) 313 { 314 u32 val = L2PMXEVFILTER_SUFILTER_ALL | 315 L2PMXEVFILTER_ORGFILTER_IDINDEP | 316 L2PMXEVFILTER_ORGFILTER_ALL; 317 318 set_l2_indirect_reg(reg_idx(IA_L2PMXEVFILTER, ctr), val); 319 } 320 321 static inline u32 cluster_pmu_getreset_ovsr(void) 322 { 323 u32 result = get_l2_indirect_reg(L2PMOVSSET); 324 325 set_l2_indirect_reg(L2PMOVSCLR, result); 326 return result; 327 } 328 329 static inline bool cluster_pmu_has_overflowed(u32 ovsr) 330 { 331 return !!(ovsr & l2_counter_present_mask); 332 } 333 334 static inline bool cluster_pmu_counter_has_overflowed(u32 ovsr, u32 idx) 335 { 336 return !!(ovsr & idx_to_reg_bit(idx)); 337 } 338 339 static void l2_cache_event_update(struct perf_event *event) 340 { 341 struct hw_perf_event *hwc = &event->hw; 342 u64 delta, prev, now; 343 u32 idx = hwc->idx; 344 345 do { 346 prev = local64_read(&hwc->prev_count); 347 now = cluster_pmu_counter_get_value(idx); 348 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 349 350 /* 351 * The cycle counter is 64-bit, but all other counters are 352 * 32-bit, and we must handle 32-bit overflow explicitly. 353 */ 354 delta = now - prev; 355 if (idx != l2_cycle_ctr_idx) 356 delta &= 0xffffffff; 357 358 local64_add(delta, &event->count); 359 } 360 361 static void l2_cache_cluster_set_period(struct cluster_pmu *cluster, 362 struct hw_perf_event *hwc) 363 { 364 u32 idx = hwc->idx; 365 u64 new; 366 367 /* 368 * We limit the max period to half the max counter value so 369 * that even in the case of extreme interrupt latency the 370 * counter will (hopefully) not wrap past its initial value. 371 */ 372 if (idx == l2_cycle_ctr_idx) 373 new = L2_CYCLE_COUNTER_RELOAD; 374 else 375 new = L2_COUNTER_RELOAD; 376 377 local64_set(&hwc->prev_count, new); 378 cluster_pmu_counter_set_value(idx, new); 379 } 380 381 static int l2_cache_get_event_idx(struct cluster_pmu *cluster, 382 struct perf_event *event) 383 { 384 struct hw_perf_event *hwc = &event->hw; 385 int idx; 386 int num_ctrs = cluster->l2cache_pmu->num_counters - 1; 387 unsigned int group; 388 389 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 390 if (test_and_set_bit(l2_cycle_ctr_idx, cluster->used_counters)) 391 return -EAGAIN; 392 393 return l2_cycle_ctr_idx; 394 } 395 396 idx = find_first_zero_bit(cluster->used_counters, num_ctrs); 397 if (idx == num_ctrs) 398 /* The counters are all in use. */ 399 return -EAGAIN; 400 401 /* 402 * Check for column exclusion: event column already in use by another 403 * event. This is for events which are not in the same group. 404 * Conflicting events in the same group are detected in event_init. 405 */ 406 group = L2_EVT_GROUP(hwc->config_base); 407 if (test_bit(group, cluster->used_groups)) 408 return -EAGAIN; 409 410 set_bit(idx, cluster->used_counters); 411 set_bit(group, cluster->used_groups); 412 413 return idx; 414 } 415 416 static void l2_cache_clear_event_idx(struct cluster_pmu *cluster, 417 struct perf_event *event) 418 { 419 struct hw_perf_event *hwc = &event->hw; 420 int idx = hwc->idx; 421 422 clear_bit(idx, cluster->used_counters); 423 if (hwc->config_base != L2CYCLE_CTR_RAW_CODE) 424 clear_bit(L2_EVT_GROUP(hwc->config_base), cluster->used_groups); 425 } 426 427 static irqreturn_t l2_cache_handle_irq(int irq_num, void *data) 428 { 429 struct cluster_pmu *cluster = data; 430 int num_counters = cluster->l2cache_pmu->num_counters; 431 u32 ovsr; 432 int idx; 433 434 ovsr = cluster_pmu_getreset_ovsr(); 435 if (!cluster_pmu_has_overflowed(ovsr)) 436 return IRQ_NONE; 437 438 for_each_set_bit(idx, cluster->used_counters, num_counters) { 439 struct perf_event *event = cluster->events[idx]; 440 struct hw_perf_event *hwc; 441 442 if (WARN_ON_ONCE(!event)) 443 continue; 444 445 if (!cluster_pmu_counter_has_overflowed(ovsr, idx)) 446 continue; 447 448 l2_cache_event_update(event); 449 hwc = &event->hw; 450 451 l2_cache_cluster_set_period(cluster, hwc); 452 } 453 454 return IRQ_HANDLED; 455 } 456 457 /* 458 * Implementation of abstract pmu functionality required by 459 * the core perf events code. 460 */ 461 462 static void l2_cache_pmu_enable(struct pmu *pmu) 463 { 464 /* 465 * Although there is only one PMU (per socket) controlling multiple 466 * physical PMUs (per cluster), because we do not support per-task mode 467 * each event is associated with a CPU. Each event has pmu_enable 468 * called on its CPU, so here it is only necessary to enable the 469 * counters for the current CPU. 470 */ 471 472 cluster_pmu_enable(); 473 } 474 475 static void l2_cache_pmu_disable(struct pmu *pmu) 476 { 477 cluster_pmu_disable(); 478 } 479 480 static int l2_cache_event_init(struct perf_event *event) 481 { 482 struct hw_perf_event *hwc = &event->hw; 483 struct cluster_pmu *cluster; 484 struct perf_event *sibling; 485 struct l2cache_pmu *l2cache_pmu; 486 487 if (event->attr.type != event->pmu->type) 488 return -ENOENT; 489 490 l2cache_pmu = to_l2cache_pmu(event->pmu); 491 492 if (hwc->sample_period) { 493 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 494 "Sampling not supported\n"); 495 return -EOPNOTSUPP; 496 } 497 498 if (event->cpu < 0) { 499 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 500 "Per-task mode not supported\n"); 501 return -EOPNOTSUPP; 502 } 503 504 if (((L2_EVT_GROUP(event->attr.config) > L2_EVT_GROUP_MAX) || 505 ((event->attr.config & ~L2_EVT_MASK) != 0)) && 506 (event->attr.config != L2CYCLE_CTR_RAW_CODE)) { 507 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 508 "Invalid config %llx\n", 509 event->attr.config); 510 return -EINVAL; 511 } 512 513 /* Don't allow groups with mixed PMUs, except for s/w events */ 514 if (event->group_leader->pmu != event->pmu && 515 !is_software_event(event->group_leader)) { 516 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 517 "Can't create mixed PMU group\n"); 518 return -EINVAL; 519 } 520 521 for_each_sibling_event(sibling, event->group_leader) { 522 if (sibling->pmu != event->pmu && 523 !is_software_event(sibling)) { 524 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 525 "Can't create mixed PMU group\n"); 526 return -EINVAL; 527 } 528 } 529 530 cluster = get_cluster_pmu(l2cache_pmu, event->cpu); 531 if (!cluster) { 532 /* CPU has not been initialised */ 533 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 534 "CPU%d not associated with L2 cluster\n", event->cpu); 535 return -EINVAL; 536 } 537 538 /* Ensure all events in a group are on the same cpu */ 539 if ((event->group_leader != event) && 540 (cluster->on_cpu != event->group_leader->cpu)) { 541 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 542 "Can't create group on CPUs %d and %d", 543 event->cpu, event->group_leader->cpu); 544 return -EINVAL; 545 } 546 547 if ((event != event->group_leader) && 548 !is_software_event(event->group_leader) && 549 (L2_EVT_GROUP(event->group_leader->attr.config) == 550 L2_EVT_GROUP(event->attr.config))) { 551 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 552 "Column exclusion: conflicting events %llx %llx\n", 553 event->group_leader->attr.config, 554 event->attr.config); 555 return -EINVAL; 556 } 557 558 for_each_sibling_event(sibling, event->group_leader) { 559 if ((sibling != event) && 560 !is_software_event(sibling) && 561 (L2_EVT_GROUP(sibling->attr.config) == 562 L2_EVT_GROUP(event->attr.config))) { 563 dev_dbg_ratelimited(&l2cache_pmu->pdev->dev, 564 "Column exclusion: conflicting events %llx %llx\n", 565 sibling->attr.config, 566 event->attr.config); 567 return -EINVAL; 568 } 569 } 570 571 hwc->idx = -1; 572 hwc->config_base = event->attr.config; 573 574 /* 575 * Ensure all events are on the same cpu so all events are in the 576 * same cpu context, to avoid races on pmu_enable etc. 577 */ 578 event->cpu = cluster->on_cpu; 579 580 return 0; 581 } 582 583 static void l2_cache_event_start(struct perf_event *event, int flags) 584 { 585 struct cluster_pmu *cluster; 586 struct hw_perf_event *hwc = &event->hw; 587 int idx = hwc->idx; 588 u32 config; 589 u32 event_cc, event_group; 590 591 hwc->state = 0; 592 593 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 594 595 l2_cache_cluster_set_period(cluster, hwc); 596 597 if (hwc->config_base == L2CYCLE_CTR_RAW_CODE) { 598 cluster_pmu_set_evccntcr(0); 599 } else { 600 config = hwc->config_base; 601 event_cc = L2_EVT_CODE(config); 602 event_group = L2_EVT_GROUP(config); 603 604 cluster_pmu_set_evcntcr(idx, 0); 605 cluster_pmu_set_evtyper(idx, event_group); 606 cluster_pmu_set_resr(cluster, event_group, event_cc); 607 cluster_pmu_set_evfilter_sys_mode(idx); 608 } 609 610 cluster_pmu_counter_enable_interrupt(idx); 611 cluster_pmu_counter_enable(idx); 612 } 613 614 static void l2_cache_event_stop(struct perf_event *event, int flags) 615 { 616 struct hw_perf_event *hwc = &event->hw; 617 int idx = hwc->idx; 618 619 if (hwc->state & PERF_HES_STOPPED) 620 return; 621 622 cluster_pmu_counter_disable_interrupt(idx); 623 cluster_pmu_counter_disable(idx); 624 625 if (flags & PERF_EF_UPDATE) 626 l2_cache_event_update(event); 627 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 628 } 629 630 static int l2_cache_event_add(struct perf_event *event, int flags) 631 { 632 struct hw_perf_event *hwc = &event->hw; 633 int idx; 634 int err = 0; 635 struct cluster_pmu *cluster; 636 637 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 638 639 idx = l2_cache_get_event_idx(cluster, event); 640 if (idx < 0) 641 return idx; 642 643 hwc->idx = idx; 644 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 645 cluster->events[idx] = event; 646 local64_set(&hwc->prev_count, 0); 647 648 if (flags & PERF_EF_START) 649 l2_cache_event_start(event, flags); 650 651 /* Propagate changes to the userspace mapping. */ 652 perf_event_update_userpage(event); 653 654 return err; 655 } 656 657 static void l2_cache_event_del(struct perf_event *event, int flags) 658 { 659 struct hw_perf_event *hwc = &event->hw; 660 struct cluster_pmu *cluster; 661 int idx = hwc->idx; 662 663 cluster = get_cluster_pmu(to_l2cache_pmu(event->pmu), event->cpu); 664 665 l2_cache_event_stop(event, flags | PERF_EF_UPDATE); 666 cluster->events[idx] = NULL; 667 l2_cache_clear_event_idx(cluster, event); 668 669 perf_event_update_userpage(event); 670 } 671 672 static void l2_cache_event_read(struct perf_event *event) 673 { 674 l2_cache_event_update(event); 675 } 676 677 static ssize_t l2_cache_pmu_cpumask_show(struct device *dev, 678 struct device_attribute *attr, 679 char *buf) 680 { 681 struct l2cache_pmu *l2cache_pmu = to_l2cache_pmu(dev_get_drvdata(dev)); 682 683 return cpumap_print_to_pagebuf(true, buf, &l2cache_pmu->cpumask); 684 } 685 686 static struct device_attribute l2_cache_pmu_cpumask_attr = 687 __ATTR(cpumask, S_IRUGO, l2_cache_pmu_cpumask_show, NULL); 688 689 static struct attribute *l2_cache_pmu_cpumask_attrs[] = { 690 &l2_cache_pmu_cpumask_attr.attr, 691 NULL, 692 }; 693 694 static struct attribute_group l2_cache_pmu_cpumask_group = { 695 .attrs = l2_cache_pmu_cpumask_attrs, 696 }; 697 698 /* CCG format for perf RAW codes. */ 699 PMU_FORMAT_ATTR(l2_code, "config:4-11"); 700 PMU_FORMAT_ATTR(l2_group, "config:0-3"); 701 PMU_FORMAT_ATTR(event, "config:0-11"); 702 703 static struct attribute *l2_cache_pmu_formats[] = { 704 &format_attr_l2_code.attr, 705 &format_attr_l2_group.attr, 706 &format_attr_event.attr, 707 NULL, 708 }; 709 710 static struct attribute_group l2_cache_pmu_format_group = { 711 .name = "format", 712 .attrs = l2_cache_pmu_formats, 713 }; 714 715 static ssize_t l2cache_pmu_event_show(struct device *dev, 716 struct device_attribute *attr, char *page) 717 { 718 struct perf_pmu_events_attr *pmu_attr; 719 720 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 721 return sprintf(page, "event=0x%02llx\n", pmu_attr->id); 722 } 723 724 #define L2CACHE_EVENT_ATTR(_name, _id) \ 725 (&((struct perf_pmu_events_attr[]) { \ 726 { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \ 727 .id = _id, } \ 728 })[0].attr.attr) 729 730 static struct attribute *l2_cache_pmu_events[] = { 731 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES), 732 L2CACHE_EVENT_ATTR(dcache-ops, L2_EVENT_DCACHE_OPS), 733 L2CACHE_EVENT_ATTR(icache-ops, L2_EVENT_ICACHE_OPS), 734 L2CACHE_EVENT_ATTR(tlbi, L2_EVENT_TLBI), 735 L2CACHE_EVENT_ATTR(barriers, L2_EVENT_BARRIERS), 736 L2CACHE_EVENT_ATTR(total-reads, L2_EVENT_TOTAL_READS), 737 L2CACHE_EVENT_ATTR(total-writes, L2_EVENT_TOTAL_WRITES), 738 L2CACHE_EVENT_ATTR(total-requests, L2_EVENT_TOTAL_REQUESTS), 739 L2CACHE_EVENT_ATTR(ldrex, L2_EVENT_LDREX), 740 L2CACHE_EVENT_ATTR(strex, L2_EVENT_STREX), 741 L2CACHE_EVENT_ATTR(clrex, L2_EVENT_CLREX), 742 NULL 743 }; 744 745 static struct attribute_group l2_cache_pmu_events_group = { 746 .name = "events", 747 .attrs = l2_cache_pmu_events, 748 }; 749 750 static const struct attribute_group *l2_cache_pmu_attr_grps[] = { 751 &l2_cache_pmu_format_group, 752 &l2_cache_pmu_cpumask_group, 753 &l2_cache_pmu_events_group, 754 NULL, 755 }; 756 757 /* 758 * Generic device handlers 759 */ 760 761 static const struct acpi_device_id l2_cache_pmu_acpi_match[] = { 762 { "QCOM8130", }, 763 { } 764 }; 765 766 static int get_num_counters(void) 767 { 768 int val; 769 770 val = get_l2_indirect_reg(L2PMCR); 771 772 /* 773 * Read number of counters from L2PMCR and add 1 774 * for the cycle counter. 775 */ 776 return ((val >> L2PMCR_NUM_EV_SHIFT) & L2PMCR_NUM_EV_MASK) + 1; 777 } 778 779 static struct cluster_pmu *l2_cache_associate_cpu_with_cluster( 780 struct l2cache_pmu *l2cache_pmu, int cpu) 781 { 782 u64 mpidr; 783 int cpu_cluster_id; 784 struct cluster_pmu *cluster = NULL; 785 786 /* 787 * This assumes that the cluster_id is in MPIDR[aff1] for 788 * single-threaded cores, and MPIDR[aff2] for multi-threaded 789 * cores. This logic will have to be updated if this changes. 790 */ 791 mpidr = read_cpuid_mpidr(); 792 if (mpidr & MPIDR_MT_BITMASK) 793 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); 794 else 795 cpu_cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); 796 797 list_for_each_entry(cluster, &l2cache_pmu->clusters, next) { 798 if (cluster->cluster_id != cpu_cluster_id) 799 continue; 800 801 dev_info(&l2cache_pmu->pdev->dev, 802 "CPU%d associated with cluster %d\n", cpu, 803 cluster->cluster_id); 804 cpumask_set_cpu(cpu, &cluster->cluster_cpus); 805 *per_cpu_ptr(l2cache_pmu->pmu_cluster, cpu) = cluster; 806 break; 807 } 808 809 return cluster; 810 } 811 812 static int l2cache_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 813 { 814 struct cluster_pmu *cluster; 815 struct l2cache_pmu *l2cache_pmu; 816 817 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 818 cluster = get_cluster_pmu(l2cache_pmu, cpu); 819 if (!cluster) { 820 /* First time this CPU has come online */ 821 cluster = l2_cache_associate_cpu_with_cluster(l2cache_pmu, cpu); 822 if (!cluster) { 823 /* Only if broken firmware doesn't list every cluster */ 824 WARN_ONCE(1, "No L2 cache cluster for CPU%d\n", cpu); 825 return 0; 826 } 827 } 828 829 /* If another CPU is managing this cluster, we're done */ 830 if (cluster->on_cpu != -1) 831 return 0; 832 833 /* 834 * All CPUs on this cluster were down, use this one. 835 * Reset to put it into sane state. 836 */ 837 cluster->on_cpu = cpu; 838 cpumask_set_cpu(cpu, &l2cache_pmu->cpumask); 839 cluster_pmu_reset(); 840 841 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(cpu))); 842 enable_irq(cluster->irq); 843 844 return 0; 845 } 846 847 static int l2cache_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 848 { 849 struct cluster_pmu *cluster; 850 struct l2cache_pmu *l2cache_pmu; 851 cpumask_t cluster_online_cpus; 852 unsigned int target; 853 854 l2cache_pmu = hlist_entry_safe(node, struct l2cache_pmu, node); 855 cluster = get_cluster_pmu(l2cache_pmu, cpu); 856 if (!cluster) 857 return 0; 858 859 /* If this CPU is not managing the cluster, we're done */ 860 if (cluster->on_cpu != cpu) 861 return 0; 862 863 /* Give up ownership of cluster */ 864 cpumask_clear_cpu(cpu, &l2cache_pmu->cpumask); 865 cluster->on_cpu = -1; 866 867 /* Any other CPU for this cluster which is still online */ 868 cpumask_and(&cluster_online_cpus, &cluster->cluster_cpus, 869 cpu_online_mask); 870 target = cpumask_any_but(&cluster_online_cpus, cpu); 871 if (target >= nr_cpu_ids) { 872 disable_irq(cluster->irq); 873 return 0; 874 } 875 876 perf_pmu_migrate_context(&l2cache_pmu->pmu, cpu, target); 877 cluster->on_cpu = target; 878 cpumask_set_cpu(target, &l2cache_pmu->cpumask); 879 WARN_ON(irq_set_affinity(cluster->irq, cpumask_of(target))); 880 881 return 0; 882 } 883 884 static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) 885 { 886 struct platform_device *pdev = to_platform_device(dev->parent); 887 struct platform_device *sdev = to_platform_device(dev); 888 struct l2cache_pmu *l2cache_pmu = data; 889 struct cluster_pmu *cluster; 890 struct acpi_device *device; 891 unsigned long fw_cluster_id; 892 int err; 893 int irq; 894 895 if (acpi_bus_get_device(ACPI_HANDLE(dev), &device)) 896 return -ENODEV; 897 898 if (kstrtoul(device->pnp.unique_id, 10, &fw_cluster_id) < 0) { 899 dev_err(&pdev->dev, "unable to read ACPI uid\n"); 900 return -ENODEV; 901 } 902 903 cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL); 904 if (!cluster) 905 return -ENOMEM; 906 907 INIT_LIST_HEAD(&cluster->next); 908 list_add(&cluster->next, &l2cache_pmu->clusters); 909 cluster->cluster_id = fw_cluster_id; 910 911 irq = platform_get_irq(sdev, 0); 912 if (irq < 0) 913 return irq; 914 irq_set_status_flags(irq, IRQ_NOAUTOEN); 915 cluster->irq = irq; 916 917 cluster->l2cache_pmu = l2cache_pmu; 918 cluster->on_cpu = -1; 919 920 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, 921 IRQF_NOBALANCING | IRQF_NO_THREAD, 922 "l2-cache-pmu", cluster); 923 if (err) { 924 dev_err(&pdev->dev, 925 "Unable to request IRQ%d for L2 PMU counters\n", irq); 926 return err; 927 } 928 929 dev_info(&pdev->dev, 930 "Registered L2 cache PMU cluster %ld\n", fw_cluster_id); 931 932 spin_lock_init(&cluster->pmu_lock); 933 934 l2cache_pmu->num_pmus++; 935 936 return 0; 937 } 938 939 static int l2_cache_pmu_probe(struct platform_device *pdev) 940 { 941 int err; 942 struct l2cache_pmu *l2cache_pmu; 943 944 l2cache_pmu = 945 devm_kzalloc(&pdev->dev, sizeof(*l2cache_pmu), GFP_KERNEL); 946 if (!l2cache_pmu) 947 return -ENOMEM; 948 949 INIT_LIST_HEAD(&l2cache_pmu->clusters); 950 951 platform_set_drvdata(pdev, l2cache_pmu); 952 l2cache_pmu->pmu = (struct pmu) { 953 /* suffix is instance id for future use with multiple sockets */ 954 .name = "l2cache_0", 955 .task_ctx_nr = perf_invalid_context, 956 .pmu_enable = l2_cache_pmu_enable, 957 .pmu_disable = l2_cache_pmu_disable, 958 .event_init = l2_cache_event_init, 959 .add = l2_cache_event_add, 960 .del = l2_cache_event_del, 961 .start = l2_cache_event_start, 962 .stop = l2_cache_event_stop, 963 .read = l2_cache_event_read, 964 .attr_groups = l2_cache_pmu_attr_grps, 965 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 966 }; 967 968 l2cache_pmu->num_counters = get_num_counters(); 969 l2cache_pmu->pdev = pdev; 970 l2cache_pmu->pmu_cluster = devm_alloc_percpu(&pdev->dev, 971 struct cluster_pmu *); 972 if (!l2cache_pmu->pmu_cluster) 973 return -ENOMEM; 974 975 l2_cycle_ctr_idx = l2cache_pmu->num_counters - 1; 976 l2_counter_present_mask = GENMASK(l2cache_pmu->num_counters - 2, 0) | 977 BIT(L2CYCLE_CTR_BIT); 978 979 cpumask_clear(&l2cache_pmu->cpumask); 980 981 /* Read cluster info and initialize each cluster */ 982 err = device_for_each_child(&pdev->dev, l2cache_pmu, 983 l2_cache_pmu_probe_cluster); 984 if (err) 985 return err; 986 987 if (l2cache_pmu->num_pmus == 0) { 988 dev_err(&pdev->dev, "No hardware L2 cache PMUs found\n"); 989 return -ENODEV; 990 } 991 992 err = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 993 &l2cache_pmu->node); 994 if (err) { 995 dev_err(&pdev->dev, "Error %d registering hotplug", err); 996 return err; 997 } 998 999 err = perf_pmu_register(&l2cache_pmu->pmu, l2cache_pmu->pmu.name, -1); 1000 if (err) { 1001 dev_err(&pdev->dev, "Error %d registering L2 cache PMU\n", err); 1002 goto out_unregister; 1003 } 1004 1005 dev_info(&pdev->dev, "Registered L2 cache PMU using %d HW PMUs\n", 1006 l2cache_pmu->num_pmus); 1007 1008 return err; 1009 1010 out_unregister: 1011 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 1012 &l2cache_pmu->node); 1013 return err; 1014 } 1015 1016 static int l2_cache_pmu_remove(struct platform_device *pdev) 1017 { 1018 struct l2cache_pmu *l2cache_pmu = 1019 to_l2cache_pmu(platform_get_drvdata(pdev)); 1020 1021 perf_pmu_unregister(&l2cache_pmu->pmu); 1022 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 1023 &l2cache_pmu->node); 1024 return 0; 1025 } 1026 1027 static struct platform_driver l2_cache_pmu_driver = { 1028 .driver = { 1029 .name = "qcom-l2cache-pmu", 1030 .acpi_match_table = ACPI_PTR(l2_cache_pmu_acpi_match), 1031 }, 1032 .probe = l2_cache_pmu_probe, 1033 .remove = l2_cache_pmu_remove, 1034 }; 1035 1036 static int __init register_l2_cache_pmu_driver(void) 1037 { 1038 int err; 1039 1040 err = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_QCOM_L2_ONLINE, 1041 "AP_PERF_ARM_QCOM_L2_ONLINE", 1042 l2cache_pmu_online_cpu, 1043 l2cache_pmu_offline_cpu); 1044 if (err) 1045 return err; 1046 1047 return platform_driver_register(&l2_cache_pmu_driver); 1048 } 1049 device_initcall(register_l2_cache_pmu_driver); 1050