1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Perf support for the Statistical Profiling Extension, introduced as 4 * part of ARMv8.2. 5 * 6 * Copyright (C) 2016 ARM Limited 7 * 8 * Author: Will Deacon <will.deacon@arm.com> 9 */ 10 11 #define PMUNAME "arm_spe" 12 #define DRVNAME PMUNAME "_pmu" 13 #define pr_fmt(fmt) DRVNAME ": " fmt 14 15 #include <linux/bitops.h> 16 #include <linux/bug.h> 17 #include <linux/capability.h> 18 #include <linux/cpuhotplug.h> 19 #include <linux/cpumask.h> 20 #include <linux/device.h> 21 #include <linux/errno.h> 22 #include <linux/interrupt.h> 23 #include <linux/irq.h> 24 #include <linux/kernel.h> 25 #include <linux/list.h> 26 #include <linux/module.h> 27 #include <linux/of_address.h> 28 #include <linux/of_device.h> 29 #include <linux/perf_event.h> 30 #include <linux/perf/arm_pmu.h> 31 #include <linux/platform_device.h> 32 #include <linux/printk.h> 33 #include <linux/slab.h> 34 #include <linux/smp.h> 35 #include <linux/vmalloc.h> 36 37 #include <asm/barrier.h> 38 #include <asm/cpufeature.h> 39 #include <asm/mmu.h> 40 #include <asm/sysreg.h> 41 42 /* 43 * Cache if the event is allowed to trace Context information. 44 * This allows us to perform the check, i.e, perfmon_capable(), 45 * in the context of the event owner, once, during the event_init(). 46 */ 47 #define SPE_PMU_HW_FLAGS_CX BIT(0) 48 49 static void set_spe_event_has_cx(struct perf_event *event) 50 { 51 if (IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR) && perfmon_capable()) 52 event->hw.flags |= SPE_PMU_HW_FLAGS_CX; 53 } 54 55 static bool get_spe_event_has_cx(struct perf_event *event) 56 { 57 return !!(event->hw.flags & SPE_PMU_HW_FLAGS_CX); 58 } 59 60 #define ARM_SPE_BUF_PAD_BYTE 0 61 62 struct arm_spe_pmu_buf { 63 int nr_pages; 64 bool snapshot; 65 void *base; 66 }; 67 68 struct arm_spe_pmu { 69 struct pmu pmu; 70 struct platform_device *pdev; 71 cpumask_t supported_cpus; 72 struct hlist_node hotplug_node; 73 74 int irq; /* PPI */ 75 u16 pmsver; 76 u16 min_period; 77 u16 counter_sz; 78 79 #define SPE_PMU_FEAT_FILT_EVT (1UL << 0) 80 #define SPE_PMU_FEAT_FILT_TYP (1UL << 1) 81 #define SPE_PMU_FEAT_FILT_LAT (1UL << 2) 82 #define SPE_PMU_FEAT_ARCH_INST (1UL << 3) 83 #define SPE_PMU_FEAT_LDS (1UL << 4) 84 #define SPE_PMU_FEAT_ERND (1UL << 5) 85 #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) 86 u64 features; 87 88 u16 max_record_sz; 89 u16 align; 90 struct perf_output_handle __percpu *handle; 91 }; 92 93 #define to_spe_pmu(p) (container_of(p, struct arm_spe_pmu, pmu)) 94 95 /* Convert a free-running index from perf into an SPE buffer offset */ 96 #define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) 97 98 /* Keep track of our dynamic hotplug state */ 99 static enum cpuhp_state arm_spe_pmu_online; 100 101 enum arm_spe_pmu_buf_fault_action { 102 SPE_PMU_BUF_FAULT_ACT_SPURIOUS, 103 SPE_PMU_BUF_FAULT_ACT_FATAL, 104 SPE_PMU_BUF_FAULT_ACT_OK, 105 }; 106 107 /* This sysfs gunk was really good fun to write. */ 108 enum arm_spe_pmu_capabilities { 109 SPE_PMU_CAP_ARCH_INST = 0, 110 SPE_PMU_CAP_ERND, 111 SPE_PMU_CAP_FEAT_MAX, 112 SPE_PMU_CAP_CNT_SZ = SPE_PMU_CAP_FEAT_MAX, 113 SPE_PMU_CAP_MIN_IVAL, 114 }; 115 116 static int arm_spe_pmu_feat_caps[SPE_PMU_CAP_FEAT_MAX] = { 117 [SPE_PMU_CAP_ARCH_INST] = SPE_PMU_FEAT_ARCH_INST, 118 [SPE_PMU_CAP_ERND] = SPE_PMU_FEAT_ERND, 119 }; 120 121 static u32 arm_spe_pmu_cap_get(struct arm_spe_pmu *spe_pmu, int cap) 122 { 123 if (cap < SPE_PMU_CAP_FEAT_MAX) 124 return !!(spe_pmu->features & arm_spe_pmu_feat_caps[cap]); 125 126 switch (cap) { 127 case SPE_PMU_CAP_CNT_SZ: 128 return spe_pmu->counter_sz; 129 case SPE_PMU_CAP_MIN_IVAL: 130 return spe_pmu->min_period; 131 default: 132 WARN(1, "unknown cap %d\n", cap); 133 } 134 135 return 0; 136 } 137 138 static ssize_t arm_spe_pmu_cap_show(struct device *dev, 139 struct device_attribute *attr, 140 char *buf) 141 { 142 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 143 struct dev_ext_attribute *ea = 144 container_of(attr, struct dev_ext_attribute, attr); 145 int cap = (long)ea->var; 146 147 return sysfs_emit(buf, "%u\n", arm_spe_pmu_cap_get(spe_pmu, cap)); 148 } 149 150 #define SPE_EXT_ATTR_ENTRY(_name, _func, _var) \ 151 &((struct dev_ext_attribute[]) { \ 152 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_var } \ 153 })[0].attr.attr 154 155 #define SPE_CAP_EXT_ATTR_ENTRY(_name, _var) \ 156 SPE_EXT_ATTR_ENTRY(_name, arm_spe_pmu_cap_show, _var) 157 158 static struct attribute *arm_spe_pmu_cap_attr[] = { 159 SPE_CAP_EXT_ATTR_ENTRY(arch_inst, SPE_PMU_CAP_ARCH_INST), 160 SPE_CAP_EXT_ATTR_ENTRY(ernd, SPE_PMU_CAP_ERND), 161 SPE_CAP_EXT_ATTR_ENTRY(count_size, SPE_PMU_CAP_CNT_SZ), 162 SPE_CAP_EXT_ATTR_ENTRY(min_interval, SPE_PMU_CAP_MIN_IVAL), 163 NULL, 164 }; 165 166 static const struct attribute_group arm_spe_pmu_cap_group = { 167 .name = "caps", 168 .attrs = arm_spe_pmu_cap_attr, 169 }; 170 171 /* User ABI */ 172 #define ATTR_CFG_FLD_ts_enable_CFG config /* PMSCR_EL1.TS */ 173 #define ATTR_CFG_FLD_ts_enable_LO 0 174 #define ATTR_CFG_FLD_ts_enable_HI 0 175 #define ATTR_CFG_FLD_pa_enable_CFG config /* PMSCR_EL1.PA */ 176 #define ATTR_CFG_FLD_pa_enable_LO 1 177 #define ATTR_CFG_FLD_pa_enable_HI 1 178 #define ATTR_CFG_FLD_pct_enable_CFG config /* PMSCR_EL1.PCT */ 179 #define ATTR_CFG_FLD_pct_enable_LO 2 180 #define ATTR_CFG_FLD_pct_enable_HI 2 181 #define ATTR_CFG_FLD_jitter_CFG config /* PMSIRR_EL1.RND */ 182 #define ATTR_CFG_FLD_jitter_LO 16 183 #define ATTR_CFG_FLD_jitter_HI 16 184 #define ATTR_CFG_FLD_branch_filter_CFG config /* PMSFCR_EL1.B */ 185 #define ATTR_CFG_FLD_branch_filter_LO 32 186 #define ATTR_CFG_FLD_branch_filter_HI 32 187 #define ATTR_CFG_FLD_load_filter_CFG config /* PMSFCR_EL1.LD */ 188 #define ATTR_CFG_FLD_load_filter_LO 33 189 #define ATTR_CFG_FLD_load_filter_HI 33 190 #define ATTR_CFG_FLD_store_filter_CFG config /* PMSFCR_EL1.ST */ 191 #define ATTR_CFG_FLD_store_filter_LO 34 192 #define ATTR_CFG_FLD_store_filter_HI 34 193 194 #define ATTR_CFG_FLD_event_filter_CFG config1 /* PMSEVFR_EL1 */ 195 #define ATTR_CFG_FLD_event_filter_LO 0 196 #define ATTR_CFG_FLD_event_filter_HI 63 197 198 #define ATTR_CFG_FLD_min_latency_CFG config2 /* PMSLATFR_EL1.MINLAT */ 199 #define ATTR_CFG_FLD_min_latency_LO 0 200 #define ATTR_CFG_FLD_min_latency_HI 11 201 202 /* Why does everything I do descend into this? */ 203 #define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \ 204 (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi 205 206 #define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \ 207 __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) 208 209 #define GEN_PMU_FORMAT_ATTR(name) \ 210 PMU_FORMAT_ATTR(name, \ 211 _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \ 212 ATTR_CFG_FLD_##name##_LO, \ 213 ATTR_CFG_FLD_##name##_HI)) 214 215 #define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \ 216 ((((attr)->cfg) >> lo) & GENMASK(hi - lo, 0)) 217 218 #define ATTR_CFG_GET_FLD(attr, name) \ 219 _ATTR_CFG_GET_FLD(attr, \ 220 ATTR_CFG_FLD_##name##_CFG, \ 221 ATTR_CFG_FLD_##name##_LO, \ 222 ATTR_CFG_FLD_##name##_HI) 223 224 GEN_PMU_FORMAT_ATTR(ts_enable); 225 GEN_PMU_FORMAT_ATTR(pa_enable); 226 GEN_PMU_FORMAT_ATTR(pct_enable); 227 GEN_PMU_FORMAT_ATTR(jitter); 228 GEN_PMU_FORMAT_ATTR(branch_filter); 229 GEN_PMU_FORMAT_ATTR(load_filter); 230 GEN_PMU_FORMAT_ATTR(store_filter); 231 GEN_PMU_FORMAT_ATTR(event_filter); 232 GEN_PMU_FORMAT_ATTR(min_latency); 233 234 static struct attribute *arm_spe_pmu_formats_attr[] = { 235 &format_attr_ts_enable.attr, 236 &format_attr_pa_enable.attr, 237 &format_attr_pct_enable.attr, 238 &format_attr_jitter.attr, 239 &format_attr_branch_filter.attr, 240 &format_attr_load_filter.attr, 241 &format_attr_store_filter.attr, 242 &format_attr_event_filter.attr, 243 &format_attr_min_latency.attr, 244 NULL, 245 }; 246 247 static const struct attribute_group arm_spe_pmu_format_group = { 248 .name = "format", 249 .attrs = arm_spe_pmu_formats_attr, 250 }; 251 252 static ssize_t cpumask_show(struct device *dev, 253 struct device_attribute *attr, char *buf) 254 { 255 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); 256 257 return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); 258 } 259 static DEVICE_ATTR_RO(cpumask); 260 261 static struct attribute *arm_spe_pmu_attrs[] = { 262 &dev_attr_cpumask.attr, 263 NULL, 264 }; 265 266 static const struct attribute_group arm_spe_pmu_group = { 267 .attrs = arm_spe_pmu_attrs, 268 }; 269 270 static const struct attribute_group *arm_spe_pmu_attr_groups[] = { 271 &arm_spe_pmu_group, 272 &arm_spe_pmu_cap_group, 273 &arm_spe_pmu_format_group, 274 NULL, 275 }; 276 277 /* Convert between user ABI and register values */ 278 static u64 arm_spe_event_to_pmscr(struct perf_event *event) 279 { 280 struct perf_event_attr *attr = &event->attr; 281 u64 reg = 0; 282 283 reg |= ATTR_CFG_GET_FLD(attr, ts_enable) << SYS_PMSCR_EL1_TS_SHIFT; 284 reg |= ATTR_CFG_GET_FLD(attr, pa_enable) << SYS_PMSCR_EL1_PA_SHIFT; 285 reg |= ATTR_CFG_GET_FLD(attr, pct_enable) << SYS_PMSCR_EL1_PCT_SHIFT; 286 287 if (!attr->exclude_user) 288 reg |= BIT(SYS_PMSCR_EL1_E0SPE_SHIFT); 289 290 if (!attr->exclude_kernel) 291 reg |= BIT(SYS_PMSCR_EL1_E1SPE_SHIFT); 292 293 if (get_spe_event_has_cx(event)) 294 reg |= BIT(SYS_PMSCR_EL1_CX_SHIFT); 295 296 return reg; 297 } 298 299 static void arm_spe_event_sanitise_period(struct perf_event *event) 300 { 301 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 302 u64 period = event->hw.sample_period; 303 u64 max_period = SYS_PMSIRR_EL1_INTERVAL_MASK 304 << SYS_PMSIRR_EL1_INTERVAL_SHIFT; 305 306 if (period < spe_pmu->min_period) 307 period = spe_pmu->min_period; 308 else if (period > max_period) 309 period = max_period; 310 else 311 period &= max_period; 312 313 event->hw.sample_period = period; 314 } 315 316 static u64 arm_spe_event_to_pmsirr(struct perf_event *event) 317 { 318 struct perf_event_attr *attr = &event->attr; 319 u64 reg = 0; 320 321 arm_spe_event_sanitise_period(event); 322 323 reg |= ATTR_CFG_GET_FLD(attr, jitter) << SYS_PMSIRR_EL1_RND_SHIFT; 324 reg |= event->hw.sample_period; 325 326 return reg; 327 } 328 329 static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) 330 { 331 struct perf_event_attr *attr = &event->attr; 332 u64 reg = 0; 333 334 reg |= ATTR_CFG_GET_FLD(attr, load_filter) << SYS_PMSFCR_EL1_LD_SHIFT; 335 reg |= ATTR_CFG_GET_FLD(attr, store_filter) << SYS_PMSFCR_EL1_ST_SHIFT; 336 reg |= ATTR_CFG_GET_FLD(attr, branch_filter) << SYS_PMSFCR_EL1_B_SHIFT; 337 338 if (reg) 339 reg |= BIT(SYS_PMSFCR_EL1_FT_SHIFT); 340 341 if (ATTR_CFG_GET_FLD(attr, event_filter)) 342 reg |= BIT(SYS_PMSFCR_EL1_FE_SHIFT); 343 344 if (ATTR_CFG_GET_FLD(attr, min_latency)) 345 reg |= BIT(SYS_PMSFCR_EL1_FL_SHIFT); 346 347 return reg; 348 } 349 350 static u64 arm_spe_event_to_pmsevfr(struct perf_event *event) 351 { 352 struct perf_event_attr *attr = &event->attr; 353 return ATTR_CFG_GET_FLD(attr, event_filter); 354 } 355 356 static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) 357 { 358 struct perf_event_attr *attr = &event->attr; 359 return ATTR_CFG_GET_FLD(attr, min_latency) 360 << SYS_PMSLATFR_EL1_MINLAT_SHIFT; 361 } 362 363 static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) 364 { 365 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 366 u64 head = PERF_IDX2OFF(handle->head, buf); 367 368 memset(buf->base + head, ARM_SPE_BUF_PAD_BYTE, len); 369 if (!buf->snapshot) 370 perf_aux_output_skip(handle, len); 371 } 372 373 static u64 arm_spe_pmu_next_snapshot_off(struct perf_output_handle *handle) 374 { 375 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 376 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 377 u64 head = PERF_IDX2OFF(handle->head, buf); 378 u64 limit = buf->nr_pages * PAGE_SIZE; 379 380 /* 381 * The trace format isn't parseable in reverse, so clamp 382 * the limit to half of the buffer size in snapshot mode 383 * so that the worst case is half a buffer of records, as 384 * opposed to a single record. 385 */ 386 if (head < limit >> 1) 387 limit >>= 1; 388 389 /* 390 * If we're within max_record_sz of the limit, we must 391 * pad, move the head index and recompute the limit. 392 */ 393 if (limit - head < spe_pmu->max_record_sz) { 394 arm_spe_pmu_pad_buf(handle, limit - head); 395 handle->head = PERF_IDX2OFF(limit, buf); 396 limit = ((buf->nr_pages * PAGE_SIZE) >> 1) + handle->head; 397 } 398 399 return limit; 400 } 401 402 static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) 403 { 404 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 405 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 406 const u64 bufsize = buf->nr_pages * PAGE_SIZE; 407 u64 limit = bufsize; 408 u64 head, tail, wakeup; 409 410 /* 411 * The head can be misaligned for two reasons: 412 * 413 * 1. The hardware left PMBPTR pointing to the first byte after 414 * a record when generating a buffer management event. 415 * 416 * 2. We used perf_aux_output_skip to consume handle->size bytes 417 * and CIRC_SPACE was used to compute the size, which always 418 * leaves one entry free. 419 * 420 * Deal with this by padding to the next alignment boundary and 421 * moving the head index. If we run out of buffer space, we'll 422 * reduce handle->size to zero and end up reporting truncation. 423 */ 424 head = PERF_IDX2OFF(handle->head, buf); 425 if (!IS_ALIGNED(head, spe_pmu->align)) { 426 unsigned long delta = roundup(head, spe_pmu->align) - head; 427 428 delta = min(delta, handle->size); 429 arm_spe_pmu_pad_buf(handle, delta); 430 head = PERF_IDX2OFF(handle->head, buf); 431 } 432 433 /* If we've run out of free space, then nothing more to do */ 434 if (!handle->size) 435 goto no_space; 436 437 /* Compute the tail and wakeup indices now that we've aligned head */ 438 tail = PERF_IDX2OFF(handle->head + handle->size, buf); 439 wakeup = PERF_IDX2OFF(handle->wakeup, buf); 440 441 /* 442 * Avoid clobbering unconsumed data. We know we have space, so 443 * if we see head == tail we know that the buffer is empty. If 444 * head > tail, then there's nothing to clobber prior to 445 * wrapping. 446 */ 447 if (head < tail) 448 limit = round_down(tail, PAGE_SIZE); 449 450 /* 451 * Wakeup may be arbitrarily far into the future. If it's not in 452 * the current generation, either we'll wrap before hitting it, 453 * or it's in the past and has been handled already. 454 * 455 * If there's a wakeup before we wrap, arrange to be woken up by 456 * the page boundary following it. Keep the tail boundary if 457 * that's lower. 458 */ 459 if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) 460 limit = min(limit, round_up(wakeup, PAGE_SIZE)); 461 462 if (limit > head) 463 return limit; 464 465 arm_spe_pmu_pad_buf(handle, handle->size); 466 no_space: 467 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); 468 perf_aux_output_end(handle, 0); 469 return 0; 470 } 471 472 static u64 arm_spe_pmu_next_off(struct perf_output_handle *handle) 473 { 474 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 475 struct arm_spe_pmu *spe_pmu = to_spe_pmu(handle->event->pmu); 476 u64 limit = __arm_spe_pmu_next_off(handle); 477 u64 head = PERF_IDX2OFF(handle->head, buf); 478 479 /* 480 * If the head has come too close to the end of the buffer, 481 * then pad to the end and recompute the limit. 482 */ 483 if (limit && (limit - head < spe_pmu->max_record_sz)) { 484 arm_spe_pmu_pad_buf(handle, limit - head); 485 limit = __arm_spe_pmu_next_off(handle); 486 } 487 488 return limit; 489 } 490 491 static void arm_spe_perf_aux_output_begin(struct perf_output_handle *handle, 492 struct perf_event *event) 493 { 494 u64 base, limit; 495 struct arm_spe_pmu_buf *buf; 496 497 /* Start a new aux session */ 498 buf = perf_aux_output_begin(handle, event); 499 if (!buf) { 500 event->hw.state |= PERF_HES_STOPPED; 501 /* 502 * We still need to clear the limit pointer, since the 503 * profiler might only be disabled by virtue of a fault. 504 */ 505 limit = 0; 506 goto out_write_limit; 507 } 508 509 limit = buf->snapshot ? arm_spe_pmu_next_snapshot_off(handle) 510 : arm_spe_pmu_next_off(handle); 511 if (limit) 512 limit |= BIT(SYS_PMBLIMITR_EL1_E_SHIFT); 513 514 limit += (u64)buf->base; 515 base = (u64)buf->base + PERF_IDX2OFF(handle->head, buf); 516 write_sysreg_s(base, SYS_PMBPTR_EL1); 517 518 out_write_limit: 519 write_sysreg_s(limit, SYS_PMBLIMITR_EL1); 520 } 521 522 static void arm_spe_perf_aux_output_end(struct perf_output_handle *handle) 523 { 524 struct arm_spe_pmu_buf *buf = perf_get_aux(handle); 525 u64 offset, size; 526 527 offset = read_sysreg_s(SYS_PMBPTR_EL1) - (u64)buf->base; 528 size = offset - PERF_IDX2OFF(handle->head, buf); 529 530 if (buf->snapshot) 531 handle->head = offset; 532 533 perf_aux_output_end(handle, size); 534 } 535 536 static void arm_spe_pmu_disable_and_drain_local(void) 537 { 538 /* Disable profiling at EL0 and EL1 */ 539 write_sysreg_s(0, SYS_PMSCR_EL1); 540 isb(); 541 542 /* Drain any buffered data */ 543 psb_csync(); 544 dsb(nsh); 545 546 /* Disable the profiling buffer */ 547 write_sysreg_s(0, SYS_PMBLIMITR_EL1); 548 isb(); 549 } 550 551 /* IRQ handling */ 552 static enum arm_spe_pmu_buf_fault_action 553 arm_spe_pmu_buf_get_fault_act(struct perf_output_handle *handle) 554 { 555 const char *err_str; 556 u64 pmbsr; 557 enum arm_spe_pmu_buf_fault_action ret; 558 559 /* 560 * Ensure new profiling data is visible to the CPU and any external 561 * aborts have been resolved. 562 */ 563 psb_csync(); 564 dsb(nsh); 565 566 /* Ensure hardware updates to PMBPTR_EL1 are visible */ 567 isb(); 568 569 /* Service required? */ 570 pmbsr = read_sysreg_s(SYS_PMBSR_EL1); 571 if (!(pmbsr & BIT(SYS_PMBSR_EL1_S_SHIFT))) 572 return SPE_PMU_BUF_FAULT_ACT_SPURIOUS; 573 574 /* 575 * If we've lost data, disable profiling and also set the PARTIAL 576 * flag to indicate that the last record is corrupted. 577 */ 578 if (pmbsr & BIT(SYS_PMBSR_EL1_DL_SHIFT)) 579 perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED | 580 PERF_AUX_FLAG_PARTIAL); 581 582 /* Report collisions to userspace so that it can up the period */ 583 if (pmbsr & BIT(SYS_PMBSR_EL1_COLL_SHIFT)) 584 perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); 585 586 /* We only expect buffer management events */ 587 switch (pmbsr & (SYS_PMBSR_EL1_EC_MASK << SYS_PMBSR_EL1_EC_SHIFT)) { 588 case SYS_PMBSR_EL1_EC_BUF: 589 /* Handled below */ 590 break; 591 case SYS_PMBSR_EL1_EC_FAULT_S1: 592 case SYS_PMBSR_EL1_EC_FAULT_S2: 593 err_str = "Unexpected buffer fault"; 594 goto out_err; 595 default: 596 err_str = "Unknown error code"; 597 goto out_err; 598 } 599 600 /* Buffer management event */ 601 switch (pmbsr & 602 (SYS_PMBSR_EL1_BUF_BSC_MASK << SYS_PMBSR_EL1_BUF_BSC_SHIFT)) { 603 case SYS_PMBSR_EL1_BUF_BSC_FULL: 604 ret = SPE_PMU_BUF_FAULT_ACT_OK; 605 goto out_stop; 606 default: 607 err_str = "Unknown buffer status code"; 608 } 609 610 out_err: 611 pr_err_ratelimited("%s on CPU %d [PMBSR=0x%016llx, PMBPTR=0x%016llx, PMBLIMITR=0x%016llx]\n", 612 err_str, smp_processor_id(), pmbsr, 613 read_sysreg_s(SYS_PMBPTR_EL1), 614 read_sysreg_s(SYS_PMBLIMITR_EL1)); 615 ret = SPE_PMU_BUF_FAULT_ACT_FATAL; 616 617 out_stop: 618 arm_spe_perf_aux_output_end(handle); 619 return ret; 620 } 621 622 static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev) 623 { 624 struct perf_output_handle *handle = dev; 625 struct perf_event *event = handle->event; 626 enum arm_spe_pmu_buf_fault_action act; 627 628 if (!perf_get_aux(handle)) 629 return IRQ_NONE; 630 631 act = arm_spe_pmu_buf_get_fault_act(handle); 632 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 633 return IRQ_NONE; 634 635 /* 636 * Ensure perf callbacks have completed, which may disable the 637 * profiling buffer in response to a TRUNCATION flag. 638 */ 639 irq_work_run(); 640 641 switch (act) { 642 case SPE_PMU_BUF_FAULT_ACT_FATAL: 643 /* 644 * If a fatal exception occurred then leaving the profiling 645 * buffer enabled is a recipe waiting to happen. Since 646 * fatal faults don't always imply truncation, make sure 647 * that the profiling buffer is disabled explicitly before 648 * clearing the syndrome register. 649 */ 650 arm_spe_pmu_disable_and_drain_local(); 651 break; 652 case SPE_PMU_BUF_FAULT_ACT_OK: 653 /* 654 * We handled the fault (the buffer was full), so resume 655 * profiling as long as we didn't detect truncation. 656 * PMBPTR might be misaligned, but we'll burn that bridge 657 * when we get to it. 658 */ 659 if (!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)) { 660 arm_spe_perf_aux_output_begin(handle, event); 661 isb(); 662 } 663 break; 664 case SPE_PMU_BUF_FAULT_ACT_SPURIOUS: 665 /* We've seen you before, but GCC has the memory of a sieve. */ 666 break; 667 } 668 669 /* The buffer pointers are now sane, so resume profiling. */ 670 write_sysreg_s(0, SYS_PMBSR_EL1); 671 return IRQ_HANDLED; 672 } 673 674 static u64 arm_spe_pmsevfr_res0(u16 pmsver) 675 { 676 switch (pmsver) { 677 case ID_AA64DFR0_PMSVER_8_2: 678 return SYS_PMSEVFR_EL1_RES0_8_2; 679 case ID_AA64DFR0_PMSVER_8_3: 680 /* Return the highest version we support in default */ 681 default: 682 return SYS_PMSEVFR_EL1_RES0_8_3; 683 } 684 } 685 686 /* Perf callbacks */ 687 static int arm_spe_pmu_event_init(struct perf_event *event) 688 { 689 u64 reg; 690 struct perf_event_attr *attr = &event->attr; 691 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 692 693 /* This is, of course, deeply driver-specific */ 694 if (attr->type != event->pmu->type) 695 return -ENOENT; 696 697 if (event->cpu >= 0 && 698 !cpumask_test_cpu(event->cpu, &spe_pmu->supported_cpus)) 699 return -ENOENT; 700 701 if (arm_spe_event_to_pmsevfr(event) & arm_spe_pmsevfr_res0(spe_pmu->pmsver)) 702 return -EOPNOTSUPP; 703 704 if (attr->exclude_idle) 705 return -EOPNOTSUPP; 706 707 /* 708 * Feedback-directed frequency throttling doesn't work when we 709 * have a buffer of samples. We'd need to manually count the 710 * samples in the buffer when it fills up and adjust the event 711 * count to reflect that. Instead, just force the user to specify 712 * a sample period. 713 */ 714 if (attr->freq) 715 return -EINVAL; 716 717 reg = arm_spe_event_to_pmsfcr(event); 718 if ((reg & BIT(SYS_PMSFCR_EL1_FE_SHIFT)) && 719 !(spe_pmu->features & SPE_PMU_FEAT_FILT_EVT)) 720 return -EOPNOTSUPP; 721 722 if ((reg & BIT(SYS_PMSFCR_EL1_FT_SHIFT)) && 723 !(spe_pmu->features & SPE_PMU_FEAT_FILT_TYP)) 724 return -EOPNOTSUPP; 725 726 if ((reg & BIT(SYS_PMSFCR_EL1_FL_SHIFT)) && 727 !(spe_pmu->features & SPE_PMU_FEAT_FILT_LAT)) 728 return -EOPNOTSUPP; 729 730 set_spe_event_has_cx(event); 731 reg = arm_spe_event_to_pmscr(event); 732 if (!perfmon_capable() && 733 (reg & (BIT(SYS_PMSCR_EL1_PA_SHIFT) | 734 BIT(SYS_PMSCR_EL1_PCT_SHIFT)))) 735 return -EACCES; 736 737 return 0; 738 } 739 740 static void arm_spe_pmu_start(struct perf_event *event, int flags) 741 { 742 u64 reg; 743 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 744 struct hw_perf_event *hwc = &event->hw; 745 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 746 747 hwc->state = 0; 748 arm_spe_perf_aux_output_begin(handle, event); 749 if (hwc->state) 750 return; 751 752 reg = arm_spe_event_to_pmsfcr(event); 753 write_sysreg_s(reg, SYS_PMSFCR_EL1); 754 755 reg = arm_spe_event_to_pmsevfr(event); 756 write_sysreg_s(reg, SYS_PMSEVFR_EL1); 757 758 reg = arm_spe_event_to_pmslatfr(event); 759 write_sysreg_s(reg, SYS_PMSLATFR_EL1); 760 761 if (flags & PERF_EF_RELOAD) { 762 reg = arm_spe_event_to_pmsirr(event); 763 write_sysreg_s(reg, SYS_PMSIRR_EL1); 764 isb(); 765 reg = local64_read(&hwc->period_left); 766 write_sysreg_s(reg, SYS_PMSICR_EL1); 767 } 768 769 reg = arm_spe_event_to_pmscr(event); 770 isb(); 771 write_sysreg_s(reg, SYS_PMSCR_EL1); 772 } 773 774 static void arm_spe_pmu_stop(struct perf_event *event, int flags) 775 { 776 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 777 struct hw_perf_event *hwc = &event->hw; 778 struct perf_output_handle *handle = this_cpu_ptr(spe_pmu->handle); 779 780 /* If we're already stopped, then nothing to do */ 781 if (hwc->state & PERF_HES_STOPPED) 782 return; 783 784 /* Stop all trace generation */ 785 arm_spe_pmu_disable_and_drain_local(); 786 787 if (flags & PERF_EF_UPDATE) { 788 /* 789 * If there's a fault pending then ensure we contain it 790 * to this buffer, since we might be on the context-switch 791 * path. 792 */ 793 if (perf_get_aux(handle)) { 794 enum arm_spe_pmu_buf_fault_action act; 795 796 act = arm_spe_pmu_buf_get_fault_act(handle); 797 if (act == SPE_PMU_BUF_FAULT_ACT_SPURIOUS) 798 arm_spe_perf_aux_output_end(handle); 799 else 800 write_sysreg_s(0, SYS_PMBSR_EL1); 801 } 802 803 /* 804 * This may also contain ECOUNT, but nobody else should 805 * be looking at period_left, since we forbid frequency 806 * based sampling. 807 */ 808 local64_set(&hwc->period_left, read_sysreg_s(SYS_PMSICR_EL1)); 809 hwc->state |= PERF_HES_UPTODATE; 810 } 811 812 hwc->state |= PERF_HES_STOPPED; 813 } 814 815 static int arm_spe_pmu_add(struct perf_event *event, int flags) 816 { 817 int ret = 0; 818 struct arm_spe_pmu *spe_pmu = to_spe_pmu(event->pmu); 819 struct hw_perf_event *hwc = &event->hw; 820 int cpu = event->cpu == -1 ? smp_processor_id() : event->cpu; 821 822 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 823 return -ENOENT; 824 825 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 826 827 if (flags & PERF_EF_START) { 828 arm_spe_pmu_start(event, PERF_EF_RELOAD); 829 if (hwc->state & PERF_HES_STOPPED) 830 ret = -EINVAL; 831 } 832 833 return ret; 834 } 835 836 static void arm_spe_pmu_del(struct perf_event *event, int flags) 837 { 838 arm_spe_pmu_stop(event, PERF_EF_UPDATE); 839 } 840 841 static void arm_spe_pmu_read(struct perf_event *event) 842 { 843 } 844 845 static void *arm_spe_pmu_setup_aux(struct perf_event *event, void **pages, 846 int nr_pages, bool snapshot) 847 { 848 int i, cpu = event->cpu; 849 struct page **pglist; 850 struct arm_spe_pmu_buf *buf; 851 852 /* We need at least two pages for this to work. */ 853 if (nr_pages < 2) 854 return NULL; 855 856 /* 857 * We require an even number of pages for snapshot mode, so that 858 * we can effectively treat the buffer as consisting of two equal 859 * parts and give userspace a fighting chance of getting some 860 * useful data out of it. 861 */ 862 if (snapshot && (nr_pages & 1)) 863 return NULL; 864 865 if (cpu == -1) 866 cpu = raw_smp_processor_id(); 867 868 buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, cpu_to_node(cpu)); 869 if (!buf) 870 return NULL; 871 872 pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); 873 if (!pglist) 874 goto out_free_buf; 875 876 for (i = 0; i < nr_pages; ++i) 877 pglist[i] = virt_to_page(pages[i]); 878 879 buf->base = vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); 880 if (!buf->base) 881 goto out_free_pglist; 882 883 buf->nr_pages = nr_pages; 884 buf->snapshot = snapshot; 885 886 kfree(pglist); 887 return buf; 888 889 out_free_pglist: 890 kfree(pglist); 891 out_free_buf: 892 kfree(buf); 893 return NULL; 894 } 895 896 static void arm_spe_pmu_free_aux(void *aux) 897 { 898 struct arm_spe_pmu_buf *buf = aux; 899 900 vunmap(buf->base); 901 kfree(buf); 902 } 903 904 /* Initialisation and teardown functions */ 905 static int arm_spe_pmu_perf_init(struct arm_spe_pmu *spe_pmu) 906 { 907 static atomic_t pmu_idx = ATOMIC_INIT(-1); 908 909 int idx; 910 char *name; 911 struct device *dev = &spe_pmu->pdev->dev; 912 913 spe_pmu->pmu = (struct pmu) { 914 .module = THIS_MODULE, 915 .capabilities = PERF_PMU_CAP_EXCLUSIVE | PERF_PMU_CAP_ITRACE, 916 .attr_groups = arm_spe_pmu_attr_groups, 917 /* 918 * We hitch a ride on the software context here, so that 919 * we can support per-task profiling (which is not possible 920 * with the invalid context as it doesn't get sched callbacks). 921 * This requires that userspace either uses a dummy event for 922 * perf_event_open, since the aux buffer is not setup until 923 * a subsequent mmap, or creates the profiling event in a 924 * disabled state and explicitly PERF_EVENT_IOC_ENABLEs it 925 * once the buffer has been created. 926 */ 927 .task_ctx_nr = perf_sw_context, 928 .event_init = arm_spe_pmu_event_init, 929 .add = arm_spe_pmu_add, 930 .del = arm_spe_pmu_del, 931 .start = arm_spe_pmu_start, 932 .stop = arm_spe_pmu_stop, 933 .read = arm_spe_pmu_read, 934 .setup_aux = arm_spe_pmu_setup_aux, 935 .free_aux = arm_spe_pmu_free_aux, 936 }; 937 938 idx = atomic_inc_return(&pmu_idx); 939 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", PMUNAME, idx); 940 if (!name) { 941 dev_err(dev, "failed to allocate name for pmu %d\n", idx); 942 return -ENOMEM; 943 } 944 945 return perf_pmu_register(&spe_pmu->pmu, name, -1); 946 } 947 948 static void arm_spe_pmu_perf_destroy(struct arm_spe_pmu *spe_pmu) 949 { 950 perf_pmu_unregister(&spe_pmu->pmu); 951 } 952 953 static void __arm_spe_pmu_dev_probe(void *info) 954 { 955 int fld; 956 u64 reg; 957 struct arm_spe_pmu *spe_pmu = info; 958 struct device *dev = &spe_pmu->pdev->dev; 959 960 fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1), 961 ID_AA64DFR0_PMSVER_SHIFT); 962 if (!fld) { 963 dev_err(dev, 964 "unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n", 965 fld, smp_processor_id()); 966 return; 967 } 968 spe_pmu->pmsver = (u16)fld; 969 970 /* Read PMBIDR first to determine whether or not we have access */ 971 reg = read_sysreg_s(SYS_PMBIDR_EL1); 972 if (reg & BIT(SYS_PMBIDR_EL1_P_SHIFT)) { 973 dev_err(dev, 974 "profiling buffer owned by higher exception level\n"); 975 return; 976 } 977 978 /* Minimum alignment. If it's out-of-range, then fail the probe */ 979 fld = reg >> SYS_PMBIDR_EL1_ALIGN_SHIFT & SYS_PMBIDR_EL1_ALIGN_MASK; 980 spe_pmu->align = 1 << fld; 981 if (spe_pmu->align > SZ_2K) { 982 dev_err(dev, "unsupported PMBIDR.Align [%d] on CPU %d\n", 983 fld, smp_processor_id()); 984 return; 985 } 986 987 /* It's now safe to read PMSIDR and figure out what we've got */ 988 reg = read_sysreg_s(SYS_PMSIDR_EL1); 989 if (reg & BIT(SYS_PMSIDR_EL1_FE_SHIFT)) 990 spe_pmu->features |= SPE_PMU_FEAT_FILT_EVT; 991 992 if (reg & BIT(SYS_PMSIDR_EL1_FT_SHIFT)) 993 spe_pmu->features |= SPE_PMU_FEAT_FILT_TYP; 994 995 if (reg & BIT(SYS_PMSIDR_EL1_FL_SHIFT)) 996 spe_pmu->features |= SPE_PMU_FEAT_FILT_LAT; 997 998 if (reg & BIT(SYS_PMSIDR_EL1_ARCHINST_SHIFT)) 999 spe_pmu->features |= SPE_PMU_FEAT_ARCH_INST; 1000 1001 if (reg & BIT(SYS_PMSIDR_EL1_LDS_SHIFT)) 1002 spe_pmu->features |= SPE_PMU_FEAT_LDS; 1003 1004 if (reg & BIT(SYS_PMSIDR_EL1_ERND_SHIFT)) 1005 spe_pmu->features |= SPE_PMU_FEAT_ERND; 1006 1007 /* This field has a spaced out encoding, so just use a look-up */ 1008 fld = reg >> SYS_PMSIDR_EL1_INTERVAL_SHIFT & SYS_PMSIDR_EL1_INTERVAL_MASK; 1009 switch (fld) { 1010 case 0: 1011 spe_pmu->min_period = 256; 1012 break; 1013 case 2: 1014 spe_pmu->min_period = 512; 1015 break; 1016 case 3: 1017 spe_pmu->min_period = 768; 1018 break; 1019 case 4: 1020 spe_pmu->min_period = 1024; 1021 break; 1022 case 5: 1023 spe_pmu->min_period = 1536; 1024 break; 1025 case 6: 1026 spe_pmu->min_period = 2048; 1027 break; 1028 case 7: 1029 spe_pmu->min_period = 3072; 1030 break; 1031 default: 1032 dev_warn(dev, "unknown PMSIDR_EL1.Interval [%d]; assuming 8\n", 1033 fld); 1034 fallthrough; 1035 case 8: 1036 spe_pmu->min_period = 4096; 1037 } 1038 1039 /* Maximum record size. If it's out-of-range, then fail the probe */ 1040 fld = reg >> SYS_PMSIDR_EL1_MAXSIZE_SHIFT & SYS_PMSIDR_EL1_MAXSIZE_MASK; 1041 spe_pmu->max_record_sz = 1 << fld; 1042 if (spe_pmu->max_record_sz > SZ_2K || spe_pmu->max_record_sz < 16) { 1043 dev_err(dev, "unsupported PMSIDR_EL1.MaxSize [%d] on CPU %d\n", 1044 fld, smp_processor_id()); 1045 return; 1046 } 1047 1048 fld = reg >> SYS_PMSIDR_EL1_COUNTSIZE_SHIFT & SYS_PMSIDR_EL1_COUNTSIZE_MASK; 1049 switch (fld) { 1050 default: 1051 dev_warn(dev, "unknown PMSIDR_EL1.CountSize [%d]; assuming 2\n", 1052 fld); 1053 fallthrough; 1054 case 2: 1055 spe_pmu->counter_sz = 12; 1056 break; 1057 case 3: 1058 spe_pmu->counter_sz = 16; 1059 } 1060 1061 dev_info(dev, 1062 "probed for CPUs %*pbl [max_record_sz %u, align %u, features 0x%llx]\n", 1063 cpumask_pr_args(&spe_pmu->supported_cpus), 1064 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); 1065 1066 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; 1067 } 1068 1069 static void __arm_spe_pmu_reset_local(void) 1070 { 1071 /* 1072 * This is probably overkill, as we have no idea where we're 1073 * draining any buffered data to... 1074 */ 1075 arm_spe_pmu_disable_and_drain_local(); 1076 1077 /* Reset the buffer base pointer */ 1078 write_sysreg_s(0, SYS_PMBPTR_EL1); 1079 isb(); 1080 1081 /* Clear any pending management interrupts */ 1082 write_sysreg_s(0, SYS_PMBSR_EL1); 1083 isb(); 1084 } 1085 1086 static void __arm_spe_pmu_setup_one(void *info) 1087 { 1088 struct arm_spe_pmu *spe_pmu = info; 1089 1090 __arm_spe_pmu_reset_local(); 1091 enable_percpu_irq(spe_pmu->irq, IRQ_TYPE_NONE); 1092 } 1093 1094 static void __arm_spe_pmu_stop_one(void *info) 1095 { 1096 struct arm_spe_pmu *spe_pmu = info; 1097 1098 disable_percpu_irq(spe_pmu->irq); 1099 __arm_spe_pmu_reset_local(); 1100 } 1101 1102 static int arm_spe_pmu_cpu_startup(unsigned int cpu, struct hlist_node *node) 1103 { 1104 struct arm_spe_pmu *spe_pmu; 1105 1106 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1107 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1108 return 0; 1109 1110 __arm_spe_pmu_setup_one(spe_pmu); 1111 return 0; 1112 } 1113 1114 static int arm_spe_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) 1115 { 1116 struct arm_spe_pmu *spe_pmu; 1117 1118 spe_pmu = hlist_entry_safe(node, struct arm_spe_pmu, hotplug_node); 1119 if (!cpumask_test_cpu(cpu, &spe_pmu->supported_cpus)) 1120 return 0; 1121 1122 __arm_spe_pmu_stop_one(spe_pmu); 1123 return 0; 1124 } 1125 1126 static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) 1127 { 1128 int ret; 1129 cpumask_t *mask = &spe_pmu->supported_cpus; 1130 1131 /* Make sure we probe the hardware on a relevant CPU */ 1132 ret = smp_call_function_any(mask, __arm_spe_pmu_dev_probe, spe_pmu, 1); 1133 if (ret || !(spe_pmu->features & SPE_PMU_FEAT_DEV_PROBED)) 1134 return -ENXIO; 1135 1136 /* Request our PPIs (note that the IRQ is still disabled) */ 1137 ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME, 1138 spe_pmu->handle); 1139 if (ret) 1140 return ret; 1141 1142 /* 1143 * Register our hotplug notifier now so we don't miss any events. 1144 * This will enable the IRQ for any supported CPUs that are already 1145 * up. 1146 */ 1147 ret = cpuhp_state_add_instance(arm_spe_pmu_online, 1148 &spe_pmu->hotplug_node); 1149 if (ret) 1150 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1151 1152 return ret; 1153 } 1154 1155 static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) 1156 { 1157 cpuhp_state_remove_instance(arm_spe_pmu_online, &spe_pmu->hotplug_node); 1158 free_percpu_irq(spe_pmu->irq, spe_pmu->handle); 1159 } 1160 1161 /* Driver and device probing */ 1162 static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) 1163 { 1164 struct platform_device *pdev = spe_pmu->pdev; 1165 int irq = platform_get_irq(pdev, 0); 1166 1167 if (irq < 0) 1168 return -ENXIO; 1169 1170 if (!irq_is_percpu(irq)) { 1171 dev_err(&pdev->dev, "expected PPI but got SPI (%d)\n", irq); 1172 return -EINVAL; 1173 } 1174 1175 if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) { 1176 dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq); 1177 return -EINVAL; 1178 } 1179 1180 spe_pmu->irq = irq; 1181 return 0; 1182 } 1183 1184 static const struct of_device_id arm_spe_pmu_of_match[] = { 1185 { .compatible = "arm,statistical-profiling-extension-v1", .data = (void *)1 }, 1186 { /* Sentinel */ }, 1187 }; 1188 MODULE_DEVICE_TABLE(of, arm_spe_pmu_of_match); 1189 1190 static const struct platform_device_id arm_spe_match[] = { 1191 { ARMV8_SPE_PDEV_NAME, 0}, 1192 { } 1193 }; 1194 MODULE_DEVICE_TABLE(platform, arm_spe_match); 1195 1196 static int arm_spe_pmu_device_probe(struct platform_device *pdev) 1197 { 1198 int ret; 1199 struct arm_spe_pmu *spe_pmu; 1200 struct device *dev = &pdev->dev; 1201 1202 /* 1203 * If kernelspace is unmapped when running at EL0, then the SPE 1204 * buffer will fault and prematurely terminate the AUX session. 1205 */ 1206 if (arm64_kernel_unmapped_at_el0()) { 1207 dev_warn_once(dev, "profiling buffer inaccessible. Try passing \"kpti=off\" on the kernel command line\n"); 1208 return -EPERM; 1209 } 1210 1211 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); 1212 if (!spe_pmu) 1213 return -ENOMEM; 1214 1215 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); 1216 if (!spe_pmu->handle) 1217 return -ENOMEM; 1218 1219 spe_pmu->pdev = pdev; 1220 platform_set_drvdata(pdev, spe_pmu); 1221 1222 ret = arm_spe_pmu_irq_probe(spe_pmu); 1223 if (ret) 1224 goto out_free_handle; 1225 1226 ret = arm_spe_pmu_dev_init(spe_pmu); 1227 if (ret) 1228 goto out_free_handle; 1229 1230 ret = arm_spe_pmu_perf_init(spe_pmu); 1231 if (ret) 1232 goto out_teardown_dev; 1233 1234 return 0; 1235 1236 out_teardown_dev: 1237 arm_spe_pmu_dev_teardown(spe_pmu); 1238 out_free_handle: 1239 free_percpu(spe_pmu->handle); 1240 return ret; 1241 } 1242 1243 static int arm_spe_pmu_device_remove(struct platform_device *pdev) 1244 { 1245 struct arm_spe_pmu *spe_pmu = platform_get_drvdata(pdev); 1246 1247 arm_spe_pmu_perf_destroy(spe_pmu); 1248 arm_spe_pmu_dev_teardown(spe_pmu); 1249 free_percpu(spe_pmu->handle); 1250 return 0; 1251 } 1252 1253 static struct platform_driver arm_spe_pmu_driver = { 1254 .id_table = arm_spe_match, 1255 .driver = { 1256 .name = DRVNAME, 1257 .of_match_table = of_match_ptr(arm_spe_pmu_of_match), 1258 .suppress_bind_attrs = true, 1259 }, 1260 .probe = arm_spe_pmu_device_probe, 1261 .remove = arm_spe_pmu_device_remove, 1262 }; 1263 1264 static int __init arm_spe_pmu_init(void) 1265 { 1266 int ret; 1267 1268 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, 1269 arm_spe_pmu_cpu_startup, 1270 arm_spe_pmu_cpu_teardown); 1271 if (ret < 0) 1272 return ret; 1273 arm_spe_pmu_online = ret; 1274 1275 ret = platform_driver_register(&arm_spe_pmu_driver); 1276 if (ret) 1277 cpuhp_remove_multi_state(arm_spe_pmu_online); 1278 1279 return ret; 1280 } 1281 1282 static void __exit arm_spe_pmu_exit(void) 1283 { 1284 platform_driver_unregister(&arm_spe_pmu_driver); 1285 cpuhp_remove_multi_state(arm_spe_pmu_online); 1286 } 1287 1288 module_init(arm_spe_pmu_init); 1289 module_exit(arm_spe_pmu_exit); 1290 1291 MODULE_DESCRIPTION("Perf driver for the ARMv8.2 Statistical Profiling Extension"); 1292 MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); 1293 MODULE_LICENSE("GPL v2"); 1294