1 /* 2 * L220/L310 cache controller support 3 * 4 * Copyright (C) 2016 ARM Limited 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 #include <linux/errno.h> 20 #include <linux/hrtimer.h> 21 #include <linux/io.h> 22 #include <linux/list.h> 23 #include <linux/perf_event.h> 24 #include <linux/printk.h> 25 #include <linux/slab.h> 26 #include <linux/types.h> 27 28 #include <asm/hardware/cache-l2x0.h> 29 30 #define PMU_NR_COUNTERS 2 31 32 static void __iomem *l2x0_base; 33 static struct pmu *l2x0_pmu; 34 static cpumask_t pmu_cpu; 35 36 static const char *l2x0_name; 37 38 static ktime_t l2x0_pmu_poll_period; 39 static struct hrtimer l2x0_pmu_hrtimer; 40 41 /* 42 * The L220/PL310 PMU has two equivalent counters, Counter1 and Counter0. 43 * Registers controlling these are laid out in pairs, in descending order, i.e. 44 * the register for Counter1 comes first, followed by the register for 45 * Counter0. 46 * We ensure that idx 0 -> Counter0, and idx1 -> Counter1. 47 */ 48 static struct perf_event *events[PMU_NR_COUNTERS]; 49 50 /* Find an unused counter */ 51 static int l2x0_pmu_find_idx(void) 52 { 53 int i; 54 55 for (i = 0; i < PMU_NR_COUNTERS; i++) { 56 if (!events[i]) 57 return i; 58 } 59 60 return -1; 61 } 62 63 /* How many counters are allocated? */ 64 static int l2x0_pmu_num_active_counters(void) 65 { 66 int i, cnt = 0; 67 68 for (i = 0; i < PMU_NR_COUNTERS; i++) { 69 if (events[i]) 70 cnt++; 71 } 72 73 return cnt; 74 } 75 76 static void l2x0_pmu_counter_config_write(int idx, u32 val) 77 { 78 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_CFG - 4 * idx); 79 } 80 81 static u32 l2x0_pmu_counter_read(int idx) 82 { 83 return readl_relaxed(l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx); 84 } 85 86 static void l2x0_pmu_counter_write(int idx, u32 val) 87 { 88 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT0_VAL - 4 * idx); 89 } 90 91 static void __l2x0_pmu_enable(void) 92 { 93 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL); 94 val |= L2X0_EVENT_CNT_CTRL_ENABLE; 95 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL); 96 } 97 98 static void __l2x0_pmu_disable(void) 99 { 100 u32 val = readl_relaxed(l2x0_base + L2X0_EVENT_CNT_CTRL); 101 val &= ~L2X0_EVENT_CNT_CTRL_ENABLE; 102 writel_relaxed(val, l2x0_base + L2X0_EVENT_CNT_CTRL); 103 } 104 105 static void l2x0_pmu_enable(struct pmu *pmu) 106 { 107 if (l2x0_pmu_num_active_counters() == 0) 108 return; 109 110 __l2x0_pmu_enable(); 111 } 112 113 static void l2x0_pmu_disable(struct pmu *pmu) 114 { 115 if (l2x0_pmu_num_active_counters() == 0) 116 return; 117 118 __l2x0_pmu_disable(); 119 } 120 121 static void warn_if_saturated(u32 count) 122 { 123 if (count != 0xffffffff) 124 return; 125 126 pr_warn_ratelimited("L2X0 counter saturated. Poll period too long\n"); 127 } 128 129 static void l2x0_pmu_event_read(struct perf_event *event) 130 { 131 struct hw_perf_event *hw = &event->hw; 132 u64 prev_count, new_count, mask; 133 134 do { 135 prev_count = local64_read(&hw->prev_count); 136 new_count = l2x0_pmu_counter_read(hw->idx); 137 } while (local64_xchg(&hw->prev_count, new_count) != prev_count); 138 139 mask = GENMASK_ULL(31, 0); 140 local64_add((new_count - prev_count) & mask, &event->count); 141 142 warn_if_saturated(new_count); 143 } 144 145 static void l2x0_pmu_event_configure(struct perf_event *event) 146 { 147 struct hw_perf_event *hw = &event->hw; 148 149 /* 150 * The L2X0 counters saturate at 0xffffffff rather than wrapping, so we 151 * will *always* lose some number of events when a counter saturates, 152 * and have no way of detecting how many were lost. 153 * 154 * To minimize the impact of this, we try to maximize the period by 155 * always starting counters at zero. To ensure that group ratios are 156 * representative, we poll periodically to avoid counters saturating. 157 * See l2x0_pmu_poll(). 158 */ 159 local64_set(&hw->prev_count, 0); 160 l2x0_pmu_counter_write(hw->idx, 0); 161 } 162 163 static enum hrtimer_restart l2x0_pmu_poll(struct hrtimer *hrtimer) 164 { 165 unsigned long flags; 166 int i; 167 168 local_irq_save(flags); 169 __l2x0_pmu_disable(); 170 171 for (i = 0; i < PMU_NR_COUNTERS; i++) { 172 struct perf_event *event = events[i]; 173 174 if (!event) 175 continue; 176 177 l2x0_pmu_event_read(event); 178 l2x0_pmu_event_configure(event); 179 } 180 181 __l2x0_pmu_enable(); 182 local_irq_restore(flags); 183 184 hrtimer_forward_now(hrtimer, l2x0_pmu_poll_period); 185 return HRTIMER_RESTART; 186 } 187 188 189 static void __l2x0_pmu_event_enable(int idx, u32 event) 190 { 191 u32 val; 192 193 val = event << L2X0_EVENT_CNT_CFG_SRC_SHIFT; 194 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED; 195 l2x0_pmu_counter_config_write(idx, val); 196 } 197 198 static void l2x0_pmu_event_start(struct perf_event *event, int flags) 199 { 200 struct hw_perf_event *hw = &event->hw; 201 202 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) 203 return; 204 205 if (flags & PERF_EF_RELOAD) { 206 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE)); 207 l2x0_pmu_event_configure(event); 208 } 209 210 hw->state = 0; 211 212 __l2x0_pmu_event_enable(hw->idx, hw->config_base); 213 } 214 215 static void __l2x0_pmu_event_disable(int idx) 216 { 217 u32 val; 218 219 val = L2X0_EVENT_CNT_CFG_SRC_DISABLED << L2X0_EVENT_CNT_CFG_SRC_SHIFT; 220 val |= L2X0_EVENT_CNT_CFG_INT_DISABLED; 221 l2x0_pmu_counter_config_write(idx, val); 222 } 223 224 static void l2x0_pmu_event_stop(struct perf_event *event, int flags) 225 { 226 struct hw_perf_event *hw = &event->hw; 227 228 if (WARN_ON_ONCE(event->hw.state & PERF_HES_STOPPED)) 229 return; 230 231 __l2x0_pmu_event_disable(hw->idx); 232 233 hw->state |= PERF_HES_STOPPED; 234 235 if (flags & PERF_EF_UPDATE) { 236 l2x0_pmu_event_read(event); 237 hw->state |= PERF_HES_UPTODATE; 238 } 239 } 240 241 static int l2x0_pmu_event_add(struct perf_event *event, int flags) 242 { 243 struct hw_perf_event *hw = &event->hw; 244 int idx = l2x0_pmu_find_idx(); 245 246 if (idx == -1) 247 return -EAGAIN; 248 249 /* 250 * Pin the timer, so that the overflows are handled by the chosen 251 * event->cpu (this is the same one as presented in "cpumask" 252 * attribute). 253 */ 254 if (l2x0_pmu_num_active_counters() == 0) 255 hrtimer_start(&l2x0_pmu_hrtimer, l2x0_pmu_poll_period, 256 HRTIMER_MODE_REL_PINNED); 257 258 events[idx] = event; 259 hw->idx = idx; 260 261 l2x0_pmu_event_configure(event); 262 263 hw->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 264 265 if (flags & PERF_EF_START) 266 l2x0_pmu_event_start(event, 0); 267 268 return 0; 269 } 270 271 static void l2x0_pmu_event_del(struct perf_event *event, int flags) 272 { 273 struct hw_perf_event *hw = &event->hw; 274 275 l2x0_pmu_event_stop(event, PERF_EF_UPDATE); 276 277 events[hw->idx] = NULL; 278 hw->idx = -1; 279 280 if (l2x0_pmu_num_active_counters() == 0) 281 hrtimer_cancel(&l2x0_pmu_hrtimer); 282 } 283 284 static bool l2x0_pmu_group_is_valid(struct perf_event *event) 285 { 286 struct pmu *pmu = event->pmu; 287 struct perf_event *leader = event->group_leader; 288 struct perf_event *sibling; 289 int num_hw = 0; 290 291 if (leader->pmu == pmu) 292 num_hw++; 293 else if (!is_software_event(leader)) 294 return false; 295 296 for_each_sibling_event(sibling, leader) { 297 if (sibling->pmu == pmu) 298 num_hw++; 299 else if (!is_software_event(sibling)) 300 return false; 301 } 302 303 return num_hw <= PMU_NR_COUNTERS; 304 } 305 306 static int l2x0_pmu_event_init(struct perf_event *event) 307 { 308 struct hw_perf_event *hw = &event->hw; 309 310 if (event->attr.type != l2x0_pmu->type) 311 return -ENOENT; 312 313 if (is_sampling_event(event) || 314 event->attach_state & PERF_ATTACH_TASK) 315 return -EINVAL; 316 317 if (event->cpu < 0) 318 return -EINVAL; 319 320 if (event->attr.config & ~L2X0_EVENT_CNT_CFG_SRC_MASK) 321 return -EINVAL; 322 323 hw->config_base = event->attr.config; 324 325 if (!l2x0_pmu_group_is_valid(event)) 326 return -EINVAL; 327 328 event->cpu = cpumask_first(&pmu_cpu); 329 330 return 0; 331 } 332 333 struct l2x0_event_attribute { 334 struct device_attribute attr; 335 unsigned int config; 336 bool pl310_only; 337 }; 338 339 #define L2X0_EVENT_ATTR(_name, _config, _pl310_only) \ 340 (&((struct l2x0_event_attribute[]) {{ \ 341 .attr = __ATTR(_name, S_IRUGO, l2x0_pmu_event_show, NULL), \ 342 .config = _config, \ 343 .pl310_only = _pl310_only, \ 344 }})[0].attr.attr) 345 346 #define L220_PLUS_EVENT_ATTR(_name, _config) \ 347 L2X0_EVENT_ATTR(_name, _config, false) 348 349 #define PL310_EVENT_ATTR(_name, _config) \ 350 L2X0_EVENT_ATTR(_name, _config, true) 351 352 static ssize_t l2x0_pmu_event_show(struct device *dev, 353 struct device_attribute *attr, char *buf) 354 { 355 struct l2x0_event_attribute *lattr; 356 357 lattr = container_of(attr, typeof(*lattr), attr); 358 return snprintf(buf, PAGE_SIZE, "config=0x%x\n", lattr->config); 359 } 360 361 static umode_t l2x0_pmu_event_attr_is_visible(struct kobject *kobj, 362 struct attribute *attr, 363 int unused) 364 { 365 struct device *dev = kobj_to_dev(kobj); 366 struct pmu *pmu = dev_get_drvdata(dev); 367 struct l2x0_event_attribute *lattr; 368 369 lattr = container_of(attr, typeof(*lattr), attr.attr); 370 371 if (!lattr->pl310_only || strcmp("l2c_310", pmu->name) == 0) 372 return attr->mode; 373 374 return 0; 375 } 376 377 static struct attribute *l2x0_pmu_event_attrs[] = { 378 L220_PLUS_EVENT_ATTR(co, 0x1), 379 L220_PLUS_EVENT_ATTR(drhit, 0x2), 380 L220_PLUS_EVENT_ATTR(drreq, 0x3), 381 L220_PLUS_EVENT_ATTR(dwhit, 0x4), 382 L220_PLUS_EVENT_ATTR(dwreq, 0x5), 383 L220_PLUS_EVENT_ATTR(dwtreq, 0x6), 384 L220_PLUS_EVENT_ATTR(irhit, 0x7), 385 L220_PLUS_EVENT_ATTR(irreq, 0x8), 386 L220_PLUS_EVENT_ATTR(wa, 0x9), 387 PL310_EVENT_ATTR(ipfalloc, 0xa), 388 PL310_EVENT_ATTR(epfhit, 0xb), 389 PL310_EVENT_ATTR(epfalloc, 0xc), 390 PL310_EVENT_ATTR(srrcvd, 0xd), 391 PL310_EVENT_ATTR(srconf, 0xe), 392 PL310_EVENT_ATTR(epfrcvd, 0xf), 393 NULL 394 }; 395 396 static struct attribute_group l2x0_pmu_event_attrs_group = { 397 .name = "events", 398 .attrs = l2x0_pmu_event_attrs, 399 .is_visible = l2x0_pmu_event_attr_is_visible, 400 }; 401 402 static ssize_t l2x0_pmu_cpumask_show(struct device *dev, 403 struct device_attribute *attr, char *buf) 404 { 405 return cpumap_print_to_pagebuf(true, buf, &pmu_cpu); 406 } 407 408 static struct device_attribute l2x0_pmu_cpumask_attr = 409 __ATTR(cpumask, S_IRUGO, l2x0_pmu_cpumask_show, NULL); 410 411 static struct attribute *l2x0_pmu_cpumask_attrs[] = { 412 &l2x0_pmu_cpumask_attr.attr, 413 NULL, 414 }; 415 416 static struct attribute_group l2x0_pmu_cpumask_attr_group = { 417 .attrs = l2x0_pmu_cpumask_attrs, 418 }; 419 420 static const struct attribute_group *l2x0_pmu_attr_groups[] = { 421 &l2x0_pmu_event_attrs_group, 422 &l2x0_pmu_cpumask_attr_group, 423 NULL, 424 }; 425 426 static void l2x0_pmu_reset(void) 427 { 428 int i; 429 430 __l2x0_pmu_disable(); 431 432 for (i = 0; i < PMU_NR_COUNTERS; i++) 433 __l2x0_pmu_event_disable(i); 434 } 435 436 static int l2x0_pmu_offline_cpu(unsigned int cpu) 437 { 438 unsigned int target; 439 440 if (!cpumask_test_and_clear_cpu(cpu, &pmu_cpu)) 441 return 0; 442 443 target = cpumask_any_but(cpu_online_mask, cpu); 444 if (target >= nr_cpu_ids) 445 return 0; 446 447 perf_pmu_migrate_context(l2x0_pmu, cpu, target); 448 cpumask_set_cpu(target, &pmu_cpu); 449 450 return 0; 451 } 452 453 void l2x0_pmu_suspend(void) 454 { 455 int i; 456 457 if (!l2x0_pmu) 458 return; 459 460 l2x0_pmu_disable(l2x0_pmu); 461 462 for (i = 0; i < PMU_NR_COUNTERS; i++) { 463 if (events[i]) 464 l2x0_pmu_event_stop(events[i], PERF_EF_UPDATE); 465 } 466 467 } 468 469 void l2x0_pmu_resume(void) 470 { 471 int i; 472 473 if (!l2x0_pmu) 474 return; 475 476 l2x0_pmu_reset(); 477 478 for (i = 0; i < PMU_NR_COUNTERS; i++) { 479 if (events[i]) 480 l2x0_pmu_event_start(events[i], PERF_EF_RELOAD); 481 } 482 483 l2x0_pmu_enable(l2x0_pmu); 484 } 485 486 void __init l2x0_pmu_register(void __iomem *base, u32 part) 487 { 488 /* 489 * Determine whether we support the PMU, and choose the name for sysfs. 490 * This is also used by l2x0_pmu_event_attr_is_visible to determine 491 * which events to display, as the PL310 PMU supports a superset of 492 * L220 events. 493 * 494 * The L210 PMU has a different programmer's interface, and is not 495 * supported by this driver. 496 * 497 * We must defer registering the PMU until the perf subsystem is up and 498 * running, so just stash the name and base, and leave that to another 499 * initcall. 500 */ 501 switch (part & L2X0_CACHE_ID_PART_MASK) { 502 case L2X0_CACHE_ID_PART_L220: 503 l2x0_name = "l2c_220"; 504 break; 505 case L2X0_CACHE_ID_PART_L310: 506 l2x0_name = "l2c_310"; 507 break; 508 default: 509 return; 510 } 511 512 l2x0_base = base; 513 } 514 515 static __init int l2x0_pmu_init(void) 516 { 517 int ret; 518 519 if (!l2x0_base) 520 return 0; 521 522 l2x0_pmu = kzalloc(sizeof(*l2x0_pmu), GFP_KERNEL); 523 if (!l2x0_pmu) { 524 pr_warn("Unable to allocate L2x0 PMU\n"); 525 return -ENOMEM; 526 } 527 528 *l2x0_pmu = (struct pmu) { 529 .task_ctx_nr = perf_invalid_context, 530 .pmu_enable = l2x0_pmu_enable, 531 .pmu_disable = l2x0_pmu_disable, 532 .read = l2x0_pmu_event_read, 533 .start = l2x0_pmu_event_start, 534 .stop = l2x0_pmu_event_stop, 535 .add = l2x0_pmu_event_add, 536 .del = l2x0_pmu_event_del, 537 .event_init = l2x0_pmu_event_init, 538 .attr_groups = l2x0_pmu_attr_groups, 539 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 540 }; 541 542 l2x0_pmu_reset(); 543 544 /* 545 * We always use a hrtimer rather than an interrupt. 546 * See comments in l2x0_pmu_event_configure and l2x0_pmu_poll. 547 * 548 * Polling once a second allows the counters to fill up to 1/128th on a 549 * quad-core test chip with cores clocked at 400MHz. Hopefully this 550 * leaves sufficient headroom to avoid overflow on production silicon 551 * at higher frequencies. 552 */ 553 l2x0_pmu_poll_period = ms_to_ktime(1000); 554 hrtimer_init(&l2x0_pmu_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 555 l2x0_pmu_hrtimer.function = l2x0_pmu_poll; 556 557 cpumask_set_cpu(0, &pmu_cpu); 558 ret = cpuhp_setup_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE, 559 "perf/arm/l2x0:online", NULL, 560 l2x0_pmu_offline_cpu); 561 if (ret) 562 goto out_pmu; 563 564 ret = perf_pmu_register(l2x0_pmu, l2x0_name, -1); 565 if (ret) 566 goto out_cpuhp; 567 568 return 0; 569 570 out_cpuhp: 571 cpuhp_remove_state_nocalls(CPUHP_AP_PERF_ARM_L2X0_ONLINE); 572 out_pmu: 573 kfree(l2x0_pmu); 574 l2x0_pmu = NULL; 575 return ret; 576 } 577 device_initcall(l2x0_pmu_init); 578