1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * ARM DMC-620 memory controller PMU driver 4 * 5 * Copyright (C) 2020 Ampere Computing LLC. 6 */ 7 8 #define DMC620_PMUNAME "arm_dmc620" 9 #define DMC620_DRVNAME DMC620_PMUNAME "_pmu" 10 #define pr_fmt(fmt) DMC620_DRVNAME ": " fmt 11 12 #include <linux/acpi.h> 13 #include <linux/bitfield.h> 14 #include <linux/bitops.h> 15 #include <linux/cpuhotplug.h> 16 #include <linux/cpumask.h> 17 #include <linux/device.h> 18 #include <linux/errno.h> 19 #include <linux/interrupt.h> 20 #include <linux/irq.h> 21 #include <linux/kernel.h> 22 #include <linux/list.h> 23 #include <linux/module.h> 24 #include <linux/mutex.h> 25 #include <linux/perf_event.h> 26 #include <linux/platform_device.h> 27 #include <linux/printk.h> 28 #include <linux/rculist.h> 29 #include <linux/refcount.h> 30 31 #define DMC620_PA_SHIFT 12 32 #define DMC620_CNT_INIT 0x80000000 33 #define DMC620_CNT_MAX_PERIOD 0xffffffff 34 #define DMC620_PMU_CLKDIV2_MAX_COUNTERS 8 35 #define DMC620_PMU_CLK_MAX_COUNTERS 2 36 #define DMC620_PMU_MAX_COUNTERS \ 37 (DMC620_PMU_CLKDIV2_MAX_COUNTERS + DMC620_PMU_CLK_MAX_COUNTERS) 38 39 /* 40 * The PMU registers start at 0xA00 in the DMC-620 memory map, and these 41 * offsets are relative to that base. 42 * 43 * Each counter has a group of control/value registers, and the 44 * DMC620_PMU_COUNTERn offsets are within a counter group. 45 * 46 * The counter registers groups start at 0xA10. 47 */ 48 #define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2 0x8 49 #define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK \ 50 (DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1) 51 #define DMC620_PMU_OVERFLOW_STATUS_CLK 0xC 52 #define DMC620_PMU_OVERFLOW_STATUS_CLK_MASK \ 53 (DMC620_PMU_CLK_MAX_COUNTERS - 1) 54 #define DMC620_PMU_COUNTERS_BASE 0x10 55 #define DMC620_PMU_COUNTERn_MASK_31_00 0x0 56 #define DMC620_PMU_COUNTERn_MASK_63_32 0x4 57 #define DMC620_PMU_COUNTERn_MATCH_31_00 0x8 58 #define DMC620_PMU_COUNTERn_MATCH_63_32 0xC 59 #define DMC620_PMU_COUNTERn_CONTROL 0x10 60 #define DMC620_PMU_COUNTERn_CONTROL_ENABLE BIT(0) 61 #define DMC620_PMU_COUNTERn_CONTROL_INVERT BIT(1) 62 #define DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX GENMASK(6, 2) 63 #define DMC620_PMU_COUNTERn_CONTROL_INCR_MUX GENMASK(8, 7) 64 #define DMC620_PMU_COUNTERn_VALUE 0x20 65 /* Offset of the registers for a given counter, relative to 0xA00 */ 66 #define DMC620_PMU_COUNTERn_OFFSET(n) \ 67 (DMC620_PMU_COUNTERS_BASE + 0x28 * (n)) 68 69 static LIST_HEAD(dmc620_pmu_irqs); 70 static DEFINE_MUTEX(dmc620_pmu_irqs_lock); 71 72 struct dmc620_pmu_irq { 73 struct hlist_node node; 74 struct list_head pmus_node; 75 struct list_head irqs_node; 76 refcount_t refcount; 77 unsigned int irq_num; 78 unsigned int cpu; 79 }; 80 81 struct dmc620_pmu { 82 struct pmu pmu; 83 84 void __iomem *base; 85 struct dmc620_pmu_irq *irq; 86 struct list_head pmus_node; 87 88 /* 89 * We put all clkdiv2 and clk counters to a same array. 90 * The first DMC620_PMU_CLKDIV2_MAX_COUNTERS bits belong to 91 * clkdiv2 counters, the last DMC620_PMU_CLK_MAX_COUNTERS 92 * belong to clk counters. 93 */ 94 DECLARE_BITMAP(used_mask, DMC620_PMU_MAX_COUNTERS); 95 struct perf_event *events[DMC620_PMU_MAX_COUNTERS]; 96 }; 97 98 #define to_dmc620_pmu(p) (container_of(p, struct dmc620_pmu, pmu)) 99 100 static int cpuhp_state_num; 101 102 struct dmc620_pmu_event_attr { 103 struct device_attribute attr; 104 u8 clkdiv2; 105 u8 eventid; 106 }; 107 108 static ssize_t 109 dmc620_pmu_event_show(struct device *dev, 110 struct device_attribute *attr, char *page) 111 { 112 struct dmc620_pmu_event_attr *eattr; 113 114 eattr = container_of(attr, typeof(*eattr), attr); 115 116 return sysfs_emit(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2); 117 } 118 119 #define DMC620_PMU_EVENT_ATTR(_name, _eventid, _clkdiv2) \ 120 (&((struct dmc620_pmu_event_attr[]) {{ \ 121 .attr = __ATTR(_name, 0444, dmc620_pmu_event_show, NULL), \ 122 .clkdiv2 = _clkdiv2, \ 123 .eventid = _eventid, \ 124 }})[0].attr.attr) 125 126 static struct attribute *dmc620_pmu_events_attrs[] = { 127 /* clkdiv2 events list */ 128 DMC620_PMU_EVENT_ATTR(clkdiv2_cycle_count, 0x0, 1), 129 DMC620_PMU_EVENT_ATTR(clkdiv2_allocate, 0x1, 1), 130 DMC620_PMU_EVENT_ATTR(clkdiv2_queue_depth, 0x2, 1), 131 DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_wr_data, 0x3, 1), 132 DMC620_PMU_EVENT_ATTR(clkdiv2_read_backlog, 0x4, 1), 133 DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_mi, 0x5, 1), 134 DMC620_PMU_EVENT_ATTR(clkdiv2_hazard_resolution, 0x6, 1), 135 DMC620_PMU_EVENT_ATTR(clkdiv2_enqueue, 0x7, 1), 136 DMC620_PMU_EVENT_ATTR(clkdiv2_arbitrate, 0x8, 1), 137 DMC620_PMU_EVENT_ATTR(clkdiv2_lrank_turnaround_activate, 0x9, 1), 138 DMC620_PMU_EVENT_ATTR(clkdiv2_prank_turnaround_activate, 0xa, 1), 139 DMC620_PMU_EVENT_ATTR(clkdiv2_read_depth, 0xb, 1), 140 DMC620_PMU_EVENT_ATTR(clkdiv2_write_depth, 0xc, 1), 141 DMC620_PMU_EVENT_ATTR(clkdiv2_highigh_qos_depth, 0xd, 1), 142 DMC620_PMU_EVENT_ATTR(clkdiv2_high_qos_depth, 0xe, 1), 143 DMC620_PMU_EVENT_ATTR(clkdiv2_medium_qos_depth, 0xf, 1), 144 DMC620_PMU_EVENT_ATTR(clkdiv2_low_qos_depth, 0x10, 1), 145 DMC620_PMU_EVENT_ATTR(clkdiv2_activate, 0x11, 1), 146 DMC620_PMU_EVENT_ATTR(clkdiv2_rdwr, 0x12, 1), 147 DMC620_PMU_EVENT_ATTR(clkdiv2_refresh, 0x13, 1), 148 DMC620_PMU_EVENT_ATTR(clkdiv2_training_request, 0x14, 1), 149 DMC620_PMU_EVENT_ATTR(clkdiv2_t_mac_tracker, 0x15, 1), 150 DMC620_PMU_EVENT_ATTR(clkdiv2_bk_fsm_tracker, 0x16, 1), 151 DMC620_PMU_EVENT_ATTR(clkdiv2_bk_open_tracker, 0x17, 1), 152 DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_pwr_down, 0x18, 1), 153 DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_sref, 0x19, 1), 154 155 /* clk events list */ 156 DMC620_PMU_EVENT_ATTR(clk_cycle_count, 0x0, 0), 157 DMC620_PMU_EVENT_ATTR(clk_request, 0x1, 0), 158 DMC620_PMU_EVENT_ATTR(clk_upload_stall, 0x2, 0), 159 NULL, 160 }; 161 162 static const struct attribute_group dmc620_pmu_events_attr_group = { 163 .name = "events", 164 .attrs = dmc620_pmu_events_attrs, 165 }; 166 167 /* User ABI */ 168 #define ATTR_CFG_FLD_mask_CFG config 169 #define ATTR_CFG_FLD_mask_LO 0 170 #define ATTR_CFG_FLD_mask_HI 44 171 #define ATTR_CFG_FLD_match_CFG config1 172 #define ATTR_CFG_FLD_match_LO 0 173 #define ATTR_CFG_FLD_match_HI 44 174 #define ATTR_CFG_FLD_invert_CFG config2 175 #define ATTR_CFG_FLD_invert_LO 0 176 #define ATTR_CFG_FLD_invert_HI 0 177 #define ATTR_CFG_FLD_incr_CFG config2 178 #define ATTR_CFG_FLD_incr_LO 1 179 #define ATTR_CFG_FLD_incr_HI 2 180 #define ATTR_CFG_FLD_event_CFG config2 181 #define ATTR_CFG_FLD_event_LO 3 182 #define ATTR_CFG_FLD_event_HI 8 183 #define ATTR_CFG_FLD_clkdiv2_CFG config2 184 #define ATTR_CFG_FLD_clkdiv2_LO 9 185 #define ATTR_CFG_FLD_clkdiv2_HI 9 186 187 #define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \ 188 (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi 189 190 #define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \ 191 __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) 192 193 #define GEN_PMU_FORMAT_ATTR(name) \ 194 PMU_FORMAT_ATTR(name, \ 195 _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \ 196 ATTR_CFG_FLD_##name##_LO, \ 197 ATTR_CFG_FLD_##name##_HI)) 198 199 #define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \ 200 ((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0)) 201 202 #define ATTR_CFG_GET_FLD(attr, name) \ 203 _ATTR_CFG_GET_FLD(attr, \ 204 ATTR_CFG_FLD_##name##_CFG, \ 205 ATTR_CFG_FLD_##name##_LO, \ 206 ATTR_CFG_FLD_##name##_HI) 207 208 GEN_PMU_FORMAT_ATTR(mask); 209 GEN_PMU_FORMAT_ATTR(match); 210 GEN_PMU_FORMAT_ATTR(invert); 211 GEN_PMU_FORMAT_ATTR(incr); 212 GEN_PMU_FORMAT_ATTR(event); 213 GEN_PMU_FORMAT_ATTR(clkdiv2); 214 215 static struct attribute *dmc620_pmu_formats_attrs[] = { 216 &format_attr_mask.attr, 217 &format_attr_match.attr, 218 &format_attr_invert.attr, 219 &format_attr_incr.attr, 220 &format_attr_event.attr, 221 &format_attr_clkdiv2.attr, 222 NULL, 223 }; 224 225 static const struct attribute_group dmc620_pmu_format_attr_group = { 226 .name = "format", 227 .attrs = dmc620_pmu_formats_attrs, 228 }; 229 230 static ssize_t dmc620_pmu_cpumask_show(struct device *dev, 231 struct device_attribute *attr, char *buf) 232 { 233 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(dev_get_drvdata(dev)); 234 235 return cpumap_print_to_pagebuf(true, buf, 236 cpumask_of(dmc620_pmu->irq->cpu)); 237 } 238 239 static struct device_attribute dmc620_pmu_cpumask_attr = 240 __ATTR(cpumask, 0444, dmc620_pmu_cpumask_show, NULL); 241 242 static struct attribute *dmc620_pmu_cpumask_attrs[] = { 243 &dmc620_pmu_cpumask_attr.attr, 244 NULL, 245 }; 246 247 static const struct attribute_group dmc620_pmu_cpumask_attr_group = { 248 .attrs = dmc620_pmu_cpumask_attrs, 249 }; 250 251 static const struct attribute_group *dmc620_pmu_attr_groups[] = { 252 &dmc620_pmu_events_attr_group, 253 &dmc620_pmu_format_attr_group, 254 &dmc620_pmu_cpumask_attr_group, 255 NULL, 256 }; 257 258 static inline 259 u32 dmc620_pmu_creg_read(struct dmc620_pmu *dmc620_pmu, 260 unsigned int idx, unsigned int reg) 261 { 262 return readl(dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg); 263 } 264 265 static inline 266 void dmc620_pmu_creg_write(struct dmc620_pmu *dmc620_pmu, 267 unsigned int idx, unsigned int reg, u32 val) 268 { 269 writel(val, dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg); 270 } 271 272 static 273 unsigned int dmc620_event_to_counter_control(struct perf_event *event) 274 { 275 struct perf_event_attr *attr = &event->attr; 276 unsigned int reg = 0; 277 278 reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INVERT, 279 ATTR_CFG_GET_FLD(attr, invert)); 280 reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX, 281 ATTR_CFG_GET_FLD(attr, event)); 282 reg |= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INCR_MUX, 283 ATTR_CFG_GET_FLD(attr, incr)); 284 285 return reg; 286 } 287 288 static int dmc620_get_event_idx(struct perf_event *event) 289 { 290 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 291 int idx, start_idx, end_idx; 292 293 if (ATTR_CFG_GET_FLD(&event->attr, clkdiv2)) { 294 start_idx = 0; 295 end_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS; 296 } else { 297 start_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS; 298 end_idx = DMC620_PMU_MAX_COUNTERS; 299 } 300 301 for (idx = start_idx; idx < end_idx; ++idx) { 302 if (!test_and_set_bit(idx, dmc620_pmu->used_mask)) 303 return idx; 304 } 305 306 /* The counters are all in use. */ 307 return -EAGAIN; 308 } 309 310 static inline 311 u64 dmc620_pmu_read_counter(struct perf_event *event) 312 { 313 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 314 315 return dmc620_pmu_creg_read(dmc620_pmu, 316 event->hw.idx, DMC620_PMU_COUNTERn_VALUE); 317 } 318 319 static void dmc620_pmu_event_update(struct perf_event *event) 320 { 321 struct hw_perf_event *hwc = &event->hw; 322 u64 delta, prev_count, new_count; 323 324 do { 325 /* We may also be called from the irq handler */ 326 prev_count = local64_read(&hwc->prev_count); 327 new_count = dmc620_pmu_read_counter(event); 328 } while (local64_cmpxchg(&hwc->prev_count, 329 prev_count, new_count) != prev_count); 330 delta = (new_count - prev_count) & DMC620_CNT_MAX_PERIOD; 331 local64_add(delta, &event->count); 332 } 333 334 static void dmc620_pmu_event_set_period(struct perf_event *event) 335 { 336 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 337 338 local64_set(&event->hw.prev_count, DMC620_CNT_INIT); 339 dmc620_pmu_creg_write(dmc620_pmu, 340 event->hw.idx, DMC620_PMU_COUNTERn_VALUE, DMC620_CNT_INIT); 341 } 342 343 static void dmc620_pmu_enable_counter(struct perf_event *event) 344 { 345 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 346 u32 reg; 347 348 reg = dmc620_event_to_counter_control(event) | DMC620_PMU_COUNTERn_CONTROL_ENABLE; 349 dmc620_pmu_creg_write(dmc620_pmu, 350 event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, reg); 351 } 352 353 static void dmc620_pmu_disable_counter(struct perf_event *event) 354 { 355 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 356 357 dmc620_pmu_creg_write(dmc620_pmu, 358 event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, 0); 359 } 360 361 static irqreturn_t dmc620_pmu_handle_irq(int irq_num, void *data) 362 { 363 struct dmc620_pmu_irq *irq = data; 364 struct dmc620_pmu *dmc620_pmu; 365 irqreturn_t ret = IRQ_NONE; 366 367 rcu_read_lock(); 368 list_for_each_entry_rcu(dmc620_pmu, &irq->pmus_node, pmus_node) { 369 unsigned long status; 370 struct perf_event *event; 371 unsigned int idx; 372 373 /* 374 * HW doesn't provide a control to atomically disable all counters. 375 * To prevent race condition (overflow happens while clearing status register), 376 * disable all events before continuing 377 */ 378 for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) { 379 event = dmc620_pmu->events[idx]; 380 if (!event) 381 continue; 382 dmc620_pmu_disable_counter(event); 383 } 384 385 status = readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2); 386 status |= (readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK) << 387 DMC620_PMU_CLKDIV2_MAX_COUNTERS); 388 if (status) { 389 for_each_set_bit(idx, &status, 390 DMC620_PMU_MAX_COUNTERS) { 391 event = dmc620_pmu->events[idx]; 392 if (WARN_ON_ONCE(!event)) 393 continue; 394 dmc620_pmu_event_update(event); 395 dmc620_pmu_event_set_period(event); 396 } 397 398 if (status & DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK) 399 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2); 400 401 if ((status >> DMC620_PMU_CLKDIV2_MAX_COUNTERS) & 402 DMC620_PMU_OVERFLOW_STATUS_CLK_MASK) 403 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK); 404 } 405 406 for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) { 407 event = dmc620_pmu->events[idx]; 408 if (!event) 409 continue; 410 if (!(event->hw.state & PERF_HES_STOPPED)) 411 dmc620_pmu_enable_counter(event); 412 } 413 414 ret = IRQ_HANDLED; 415 } 416 rcu_read_unlock(); 417 418 return ret; 419 } 420 421 static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num) 422 { 423 struct dmc620_pmu_irq *irq; 424 int ret; 425 426 list_for_each_entry(irq, &dmc620_pmu_irqs, irqs_node) 427 if (irq->irq_num == irq_num && refcount_inc_not_zero(&irq->refcount)) 428 return irq; 429 430 irq = kzalloc(sizeof(*irq), GFP_KERNEL); 431 if (!irq) 432 return ERR_PTR(-ENOMEM); 433 434 INIT_LIST_HEAD(&irq->pmus_node); 435 436 /* Pick one CPU to be the preferred one to use */ 437 irq->cpu = raw_smp_processor_id(); 438 refcount_set(&irq->refcount, 1); 439 440 ret = request_irq(irq_num, dmc620_pmu_handle_irq, 441 IRQF_NOBALANCING | IRQF_NO_THREAD, 442 "dmc620-pmu", irq); 443 if (ret) 444 goto out_free_aff; 445 446 ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu)); 447 if (ret) 448 goto out_free_irq; 449 450 ret = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &irq->node); 451 if (ret) 452 goto out_free_irq; 453 454 irq->irq_num = irq_num; 455 list_add(&irq->irqs_node, &dmc620_pmu_irqs); 456 457 return irq; 458 459 out_free_irq: 460 free_irq(irq_num, irq); 461 out_free_aff: 462 kfree(irq); 463 return ERR_PTR(ret); 464 } 465 466 static int dmc620_pmu_get_irq(struct dmc620_pmu *dmc620_pmu, int irq_num) 467 { 468 struct dmc620_pmu_irq *irq; 469 470 mutex_lock(&dmc620_pmu_irqs_lock); 471 irq = __dmc620_pmu_get_irq(irq_num); 472 mutex_unlock(&dmc620_pmu_irqs_lock); 473 474 if (IS_ERR(irq)) 475 return PTR_ERR(irq); 476 477 dmc620_pmu->irq = irq; 478 mutex_lock(&dmc620_pmu_irqs_lock); 479 list_add_rcu(&dmc620_pmu->pmus_node, &irq->pmus_node); 480 mutex_unlock(&dmc620_pmu_irqs_lock); 481 482 return 0; 483 } 484 485 static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu) 486 { 487 struct dmc620_pmu_irq *irq = dmc620_pmu->irq; 488 489 mutex_lock(&dmc620_pmu_irqs_lock); 490 list_del_rcu(&dmc620_pmu->pmus_node); 491 492 if (!refcount_dec_and_test(&irq->refcount)) { 493 mutex_unlock(&dmc620_pmu_irqs_lock); 494 return; 495 } 496 497 list_del(&irq->irqs_node); 498 mutex_unlock(&dmc620_pmu_irqs_lock); 499 500 free_irq(irq->irq_num, irq); 501 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node); 502 kfree(irq); 503 } 504 505 static int dmc620_pmu_event_init(struct perf_event *event) 506 { 507 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 508 struct hw_perf_event *hwc = &event->hw; 509 struct perf_event *sibling; 510 511 if (event->attr.type != event->pmu->type) 512 return -ENOENT; 513 514 /* 515 * DMC 620 PMUs are shared across all cpus and cannot 516 * support task bound and sampling events. 517 */ 518 if (is_sampling_event(event) || 519 event->attach_state & PERF_ATTACH_TASK) { 520 dev_dbg(dmc620_pmu->pmu.dev, 521 "Can't support per-task counters\n"); 522 return -EOPNOTSUPP; 523 } 524 525 /* 526 * Many perf core operations (eg. events rotation) operate on a 527 * single CPU context. This is obvious for CPU PMUs, where one 528 * expects the same sets of events being observed on all CPUs, 529 * but can lead to issues for off-core PMUs, where each 530 * event could be theoretically assigned to a different CPU. To 531 * mitigate this, we enforce CPU assignment to one, selected 532 * processor. 533 */ 534 event->cpu = dmc620_pmu->irq->cpu; 535 if (event->cpu < 0) 536 return -EINVAL; 537 538 /* 539 * We can't atomically disable all HW counters so only one event allowed, 540 * although software events are acceptable. 541 */ 542 if (event->group_leader != event && 543 !is_software_event(event->group_leader)) 544 return -EINVAL; 545 546 for_each_sibling_event(sibling, event->group_leader) { 547 if (sibling != event && 548 !is_software_event(sibling)) 549 return -EINVAL; 550 } 551 552 hwc->idx = -1; 553 return 0; 554 } 555 556 static void dmc620_pmu_read(struct perf_event *event) 557 { 558 dmc620_pmu_event_update(event); 559 } 560 561 static void dmc620_pmu_start(struct perf_event *event, int flags) 562 { 563 event->hw.state = 0; 564 dmc620_pmu_event_set_period(event); 565 dmc620_pmu_enable_counter(event); 566 } 567 568 static void dmc620_pmu_stop(struct perf_event *event, int flags) 569 { 570 if (event->hw.state & PERF_HES_STOPPED) 571 return; 572 573 dmc620_pmu_disable_counter(event); 574 dmc620_pmu_event_update(event); 575 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 576 } 577 578 static int dmc620_pmu_add(struct perf_event *event, int flags) 579 { 580 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 581 struct perf_event_attr *attr = &event->attr; 582 struct hw_perf_event *hwc = &event->hw; 583 int idx; 584 u64 reg; 585 586 idx = dmc620_get_event_idx(event); 587 if (idx < 0) 588 return idx; 589 590 hwc->idx = idx; 591 dmc620_pmu->events[idx] = event; 592 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 593 594 reg = ATTR_CFG_GET_FLD(attr, mask); 595 dmc620_pmu_creg_write(dmc620_pmu, 596 idx, DMC620_PMU_COUNTERn_MASK_31_00, lower_32_bits(reg)); 597 dmc620_pmu_creg_write(dmc620_pmu, 598 idx, DMC620_PMU_COUNTERn_MASK_63_32, upper_32_bits(reg)); 599 600 reg = ATTR_CFG_GET_FLD(attr, match); 601 dmc620_pmu_creg_write(dmc620_pmu, 602 idx, DMC620_PMU_COUNTERn_MATCH_31_00, lower_32_bits(reg)); 603 dmc620_pmu_creg_write(dmc620_pmu, 604 idx, DMC620_PMU_COUNTERn_MATCH_63_32, upper_32_bits(reg)); 605 606 if (flags & PERF_EF_START) 607 dmc620_pmu_start(event, PERF_EF_RELOAD); 608 609 perf_event_update_userpage(event); 610 return 0; 611 } 612 613 static void dmc620_pmu_del(struct perf_event *event, int flags) 614 { 615 struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu); 616 struct hw_perf_event *hwc = &event->hw; 617 int idx = hwc->idx; 618 619 dmc620_pmu_stop(event, PERF_EF_UPDATE); 620 dmc620_pmu->events[idx] = NULL; 621 clear_bit(idx, dmc620_pmu->used_mask); 622 perf_event_update_userpage(event); 623 } 624 625 static int dmc620_pmu_cpu_teardown(unsigned int cpu, 626 struct hlist_node *node) 627 { 628 struct dmc620_pmu_irq *irq; 629 struct dmc620_pmu *dmc620_pmu; 630 unsigned int target; 631 632 irq = hlist_entry_safe(node, struct dmc620_pmu_irq, node); 633 if (cpu != irq->cpu) 634 return 0; 635 636 target = cpumask_any_but(cpu_online_mask, cpu); 637 if (target >= nr_cpu_ids) 638 return 0; 639 640 /* We're only reading, but this isn't the place to be involving RCU */ 641 mutex_lock(&dmc620_pmu_irqs_lock); 642 list_for_each_entry(dmc620_pmu, &irq->pmus_node, pmus_node) 643 perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target); 644 mutex_unlock(&dmc620_pmu_irqs_lock); 645 646 WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target))); 647 irq->cpu = target; 648 649 return 0; 650 } 651 652 static int dmc620_pmu_device_probe(struct platform_device *pdev) 653 { 654 struct dmc620_pmu *dmc620_pmu; 655 struct resource *res; 656 char *name; 657 int irq_num; 658 int i, ret; 659 660 dmc620_pmu = devm_kzalloc(&pdev->dev, 661 sizeof(struct dmc620_pmu), GFP_KERNEL); 662 if (!dmc620_pmu) 663 return -ENOMEM; 664 665 platform_set_drvdata(pdev, dmc620_pmu); 666 667 dmc620_pmu->pmu = (struct pmu) { 668 .module = THIS_MODULE, 669 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 670 .task_ctx_nr = perf_invalid_context, 671 .event_init = dmc620_pmu_event_init, 672 .add = dmc620_pmu_add, 673 .del = dmc620_pmu_del, 674 .start = dmc620_pmu_start, 675 .stop = dmc620_pmu_stop, 676 .read = dmc620_pmu_read, 677 .attr_groups = dmc620_pmu_attr_groups, 678 }; 679 680 dmc620_pmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 681 if (IS_ERR(dmc620_pmu->base)) 682 return PTR_ERR(dmc620_pmu->base); 683 684 /* Make sure device is reset before enabling interrupt */ 685 for (i = 0; i < DMC620_PMU_MAX_COUNTERS; i++) 686 dmc620_pmu_creg_write(dmc620_pmu, i, DMC620_PMU_COUNTERn_CONTROL, 0); 687 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2); 688 writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK); 689 690 irq_num = platform_get_irq(pdev, 0); 691 if (irq_num < 0) 692 return irq_num; 693 694 ret = dmc620_pmu_get_irq(dmc620_pmu, irq_num); 695 if (ret) 696 return ret; 697 698 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, 699 "%s_%llx", DMC620_PMUNAME, 700 (u64)(res->start >> DMC620_PA_SHIFT)); 701 if (!name) { 702 dev_err(&pdev->dev, 703 "Create name failed, PMU @%pa\n", &res->start); 704 ret = -ENOMEM; 705 goto out_teardown_dev; 706 } 707 708 ret = perf_pmu_register(&dmc620_pmu->pmu, name, -1); 709 if (ret) 710 goto out_teardown_dev; 711 712 return 0; 713 714 out_teardown_dev: 715 dmc620_pmu_put_irq(dmc620_pmu); 716 synchronize_rcu(); 717 return ret; 718 } 719 720 static int dmc620_pmu_device_remove(struct platform_device *pdev) 721 { 722 struct dmc620_pmu *dmc620_pmu = platform_get_drvdata(pdev); 723 724 dmc620_pmu_put_irq(dmc620_pmu); 725 726 /* perf will synchronise RCU before devres can free dmc620_pmu */ 727 perf_pmu_unregister(&dmc620_pmu->pmu); 728 729 return 0; 730 } 731 732 static const struct acpi_device_id dmc620_acpi_match[] = { 733 { "ARMHD620", 0}, 734 {}, 735 }; 736 MODULE_DEVICE_TABLE(acpi, dmc620_acpi_match); 737 static struct platform_driver dmc620_pmu_driver = { 738 .driver = { 739 .name = DMC620_DRVNAME, 740 .acpi_match_table = dmc620_acpi_match, 741 .suppress_bind_attrs = true, 742 }, 743 .probe = dmc620_pmu_device_probe, 744 .remove = dmc620_pmu_device_remove, 745 }; 746 747 static int __init dmc620_pmu_init(void) 748 { 749 int ret; 750 751 cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 752 DMC620_DRVNAME, 753 NULL, 754 dmc620_pmu_cpu_teardown); 755 if (cpuhp_state_num < 0) 756 return cpuhp_state_num; 757 758 ret = platform_driver_register(&dmc620_pmu_driver); 759 if (ret) 760 cpuhp_remove_multi_state(cpuhp_state_num); 761 762 return ret; 763 } 764 765 static void __exit dmc620_pmu_exit(void) 766 { 767 platform_driver_unregister(&dmc620_pmu_driver); 768 cpuhp_remove_multi_state(cpuhp_state_num); 769 } 770 771 module_init(dmc620_pmu_init); 772 module_exit(dmc620_pmu_exit); 773 774 MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller"); 775 MODULE_AUTHOR("Tuan Phan <tuanphan@os.amperecomputing.com"); 776 MODULE_LICENSE("GPL v2"); 777