1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Alibaba DDR Sub-System Driveway PMU driver 4 * 5 * Copyright (C) 2022 Alibaba Inc 6 */ 7 8 #define ALI_DRW_PMUNAME "ali_drw" 9 #define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu" 10 #define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt 11 12 #include <linux/acpi.h> 13 #include <linux/bitfield.h> 14 #include <linux/bitmap.h> 15 #include <linux/bitops.h> 16 #include <linux/cpuhotplug.h> 17 #include <linux/cpumask.h> 18 #include <linux/device.h> 19 #include <linux/errno.h> 20 #include <linux/interrupt.h> 21 #include <linux/irq.h> 22 #include <linux/kernel.h> 23 #include <linux/list.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/perf_event.h> 27 #include <linux/platform_device.h> 28 #include <linux/printk.h> 29 #include <linux/rculist.h> 30 #include <linux/refcount.h> 31 32 33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16 34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19 35 36 #define ALI_DRW_PMU_PA_SHIFT 12 37 #define ALI_DRW_PMU_CNT_INIT 0x00000000 38 #define ALI_DRW_CNT_MAX_PERIOD 0xffffffff 39 #define ALI_DRW_PMU_CYCLE_EVT_ID 0x80 40 41 #define ALI_DRW_PMU_CNT_CTRL 0xC00 42 #define ALI_DRW_PMU_CNT_RST BIT(2) 43 #define ALI_DRW_PMU_CNT_STOP BIT(1) 44 #define ALI_DRW_PMU_CNT_START BIT(0) 45 46 #define ALI_DRW_PMU_CNT_STATE 0xC04 47 #define ALI_DRW_PMU_TEST_CTRL 0xC08 48 #define ALI_DRW_PMU_CNT_PRELOAD 0xC0C 49 50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0) 51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0) 52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10 53 #define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14 54 55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */ 56 #define ALI_DRW_PMU_EVENT_SEL0 0xC68 57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/ 58 #define ALI_DRW_PMU_EVENT_SELn(n) \ 59 (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4) 60 #define ALI_DRW_PMCOM_CNT_EN BIT(7) 61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0) 62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \ 63 (8 * (n % 4)) 64 65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */ 66 #define ALI_DRW_PMU_COMMON_COUNTER0 0xC78 67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \ 68 (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n)) 69 70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8 71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC 72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0 73 #define ALI_DRW_PMU_OV_INTR_CLR 0xCC4 74 #define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8 75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8) 76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0) 77 #define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0) 78 79 static int ali_drw_cpuhp_state_num; 80 81 static LIST_HEAD(ali_drw_pmu_irqs); 82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock); 83 84 struct ali_drw_pmu_irq { 85 struct hlist_node node; 86 struct list_head irqs_node; 87 struct list_head pmus_node; 88 int irq_num; 89 int cpu; 90 refcount_t refcount; 91 }; 92 93 struct ali_drw_pmu { 94 void __iomem *cfg_base; 95 struct device *dev; 96 97 struct list_head pmus_node; 98 struct ali_drw_pmu_irq *irq; 99 int irq_num; 100 int cpu; 101 DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS); 102 struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; 103 int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; 104 105 struct pmu pmu; 106 }; 107 108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu)) 109 110 #define DRW_CONFIG_EVENTID GENMASK(7, 0) 111 #define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config) 112 113 static ssize_t ali_drw_pmu_format_show(struct device *dev, 114 struct device_attribute *attr, char *buf) 115 { 116 struct dev_ext_attribute *eattr; 117 118 eattr = container_of(attr, struct dev_ext_attribute, attr); 119 120 return sprintf(buf, "%s\n", (char *)eattr->var); 121 } 122 123 /* 124 * PMU event attributes 125 */ 126 static ssize_t ali_drw_pmu_event_show(struct device *dev, 127 struct device_attribute *attr, char *page) 128 { 129 struct dev_ext_attribute *eattr; 130 131 eattr = container_of(attr, struct dev_ext_attribute, attr); 132 133 return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var); 134 } 135 136 #define ALI_DRW_PMU_ATTR(_name, _func, _config) \ 137 (&((struct dev_ext_attribute[]) { \ 138 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ 139 })[0].attr.attr) 140 141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \ 142 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config) 143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \ 144 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config) 145 146 static struct attribute *ali_drw_pmu_events_attrs[] = { 147 ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0), 148 ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1), 149 ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2), 150 ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3), 151 ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4), 152 ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7), 153 ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8), 154 ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9), 155 ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA), 156 ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB), 157 ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC), 158 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD), 159 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE), 160 ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF), 161 ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10), 162 ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11), 163 ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12), 164 ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13), 165 ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14), 166 ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15), 167 ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16), 168 ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17), 169 ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18), 170 ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19), 171 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A), 172 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B), 173 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C), 174 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D), 175 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E), 176 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F), 177 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20), 178 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21), 179 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26), 180 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27), 181 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28), 182 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29), 183 ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A), 184 ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B), 185 ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D), 186 ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E), 187 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30), 188 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31), 189 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34), 190 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35), 191 ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36), 192 ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37), 193 ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38), 194 ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39), 195 ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A), 196 ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B), 197 ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C), 198 ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D), 199 ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80), 200 NULL, 201 }; 202 203 static struct attribute_group ali_drw_pmu_events_attr_group = { 204 .name = "events", 205 .attrs = ali_drw_pmu_events_attrs, 206 }; 207 208 static struct attribute *ali_drw_pmu_format_attr[] = { 209 ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"), 210 NULL, 211 }; 212 213 static const struct attribute_group ali_drw_pmu_format_group = { 214 .name = "format", 215 .attrs = ali_drw_pmu_format_attr, 216 }; 217 218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev, 219 struct device_attribute *attr, 220 char *buf) 221 { 222 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev)); 223 224 return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu)); 225 } 226 227 static struct device_attribute ali_drw_pmu_cpumask_attr = 228 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL); 229 230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = { 231 &ali_drw_pmu_cpumask_attr.attr, 232 NULL, 233 }; 234 235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = { 236 .attrs = ali_drw_pmu_cpumask_attrs, 237 }; 238 239 static ssize_t ali_drw_pmu_identifier_show(struct device *dev, 240 struct device_attribute *attr, 241 char *page) 242 { 243 return sysfs_emit(page, "%s\n", "ali_drw_pmu"); 244 } 245 246 static umode_t ali_drw_pmu_identifier_attr_visible(struct kobject *kobj, 247 struct attribute *attr, int n) 248 { 249 return attr->mode; 250 } 251 252 static struct device_attribute ali_drw_pmu_identifier_attr = 253 __ATTR(identifier, 0444, ali_drw_pmu_identifier_show, NULL); 254 255 static struct attribute *ali_drw_pmu_identifier_attrs[] = { 256 &ali_drw_pmu_identifier_attr.attr, 257 NULL 258 }; 259 260 static const struct attribute_group ali_drw_pmu_identifier_attr_group = { 261 .attrs = ali_drw_pmu_identifier_attrs, 262 .is_visible = ali_drw_pmu_identifier_attr_visible 263 }; 264 265 static const struct attribute_group *ali_drw_pmu_attr_groups[] = { 266 &ali_drw_pmu_events_attr_group, 267 &ali_drw_pmu_cpumask_attr_group, 268 &ali_drw_pmu_format_group, 269 &ali_drw_pmu_identifier_attr_group, 270 NULL, 271 }; 272 273 /* find a counter for event, then in add func, hw.idx will equal to counter */ 274 static int ali_drw_get_counter_idx(struct perf_event *event) 275 { 276 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 277 int idx; 278 279 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) { 280 if (!test_and_set_bit(idx, drw_pmu->used_mask)) 281 return idx; 282 } 283 284 /* The counters are all in use. */ 285 return -EBUSY; 286 } 287 288 static u64 ali_drw_pmu_read_counter(struct perf_event *event) 289 { 290 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 291 u64 cycle_high, cycle_low; 292 293 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { 294 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH); 295 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK; 296 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW); 297 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK; 298 return (cycle_high << 32 | cycle_low); 299 } 300 301 return readl(drw_pmu->cfg_base + 302 ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx)); 303 } 304 305 static void ali_drw_pmu_event_update(struct perf_event *event) 306 { 307 struct hw_perf_event *hwc = &event->hw; 308 u64 delta, prev, now; 309 310 do { 311 prev = local64_read(&hwc->prev_count); 312 now = ali_drw_pmu_read_counter(event); 313 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 314 315 /* handle overflow. */ 316 delta = now - prev; 317 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) 318 delta &= ALI_DRW_PMU_OV_INTR_MASK; 319 else 320 delta &= ALI_DRW_CNT_MAX_PERIOD; 321 local64_add(delta, &event->count); 322 } 323 324 static void ali_drw_pmu_event_set_period(struct perf_event *event) 325 { 326 u64 pre_val; 327 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 328 329 /* set a preload counter for test purpose */ 330 writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx, 331 drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); 332 333 /* set conunter initial value */ 334 pre_val = ALI_DRW_PMU_CNT_INIT; 335 writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); 336 local64_set(&event->hw.prev_count, pre_val); 337 338 /* set sel mode to zero to start test */ 339 writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); 340 } 341 342 static void ali_drw_pmu_enable_counter(struct perf_event *event) 343 { 344 u32 val, subval, reg, shift; 345 int counter = event->hw.idx; 346 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 347 348 reg = ALI_DRW_PMU_EVENT_SELn(counter); 349 val = readl(drw_pmu->cfg_base + reg); 350 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) | 351 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]); 352 353 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); 354 val &= ~(GENMASK(7, 0) << shift); 355 val |= subval << shift; 356 357 writel(val, drw_pmu->cfg_base + reg); 358 } 359 360 static void ali_drw_pmu_disable_counter(struct perf_event *event) 361 { 362 u32 val, reg, subval, shift; 363 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 364 int counter = event->hw.idx; 365 366 reg = ALI_DRW_PMU_EVENT_SELn(counter); 367 val = readl(drw_pmu->cfg_base + reg); 368 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) | 369 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0); 370 371 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); 372 val &= ~(GENMASK(7, 0) << shift); 373 val |= subval << shift; 374 375 writel(val, drw_pmu->cfg_base + reg); 376 } 377 378 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data) 379 { 380 struct ali_drw_pmu_irq *irq = data; 381 struct ali_drw_pmu *drw_pmu; 382 irqreturn_t ret = IRQ_NONE; 383 384 rcu_read_lock(); 385 list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) { 386 unsigned long status, clr_status; 387 struct perf_event *event; 388 unsigned int idx; 389 390 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { 391 event = drw_pmu->events[idx]; 392 if (!event) 393 continue; 394 ali_drw_pmu_disable_counter(event); 395 } 396 397 /* common counter intr status */ 398 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS); 399 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status); 400 if (status) { 401 for_each_set_bit(idx, &status, 402 ALI_DRW_PMU_COMMON_MAX_COUNTERS) { 403 event = drw_pmu->events[idx]; 404 if (WARN_ON_ONCE(!event)) 405 continue; 406 ali_drw_pmu_event_update(event); 407 ali_drw_pmu_event_set_period(event); 408 } 409 410 /* clear common counter intr status */ 411 clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1); 412 writel(clr_status, 413 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); 414 } 415 416 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { 417 event = drw_pmu->events[idx]; 418 if (!event) 419 continue; 420 if (!(event->hw.state & PERF_HES_STOPPED)) 421 ali_drw_pmu_enable_counter(event); 422 } 423 if (status) 424 ret = IRQ_HANDLED; 425 } 426 rcu_read_unlock(); 427 return ret; 428 } 429 430 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device 431 *pdev, int irq_num) 432 { 433 int ret; 434 struct ali_drw_pmu_irq *irq; 435 436 list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) { 437 if (irq->irq_num == irq_num 438 && refcount_inc_not_zero(&irq->refcount)) 439 return irq; 440 } 441 442 irq = kzalloc(sizeof(*irq), GFP_KERNEL); 443 if (!irq) 444 return ERR_PTR(-ENOMEM); 445 446 INIT_LIST_HEAD(&irq->pmus_node); 447 448 /* Pick one CPU to be the preferred one to use */ 449 irq->cpu = smp_processor_id(); 450 refcount_set(&irq->refcount, 1); 451 452 /* 453 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same 454 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers 455 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not 456 * share with other component. 457 */ 458 ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, 459 IRQF_SHARED, dev_name(&pdev->dev), irq); 460 if (ret < 0) { 461 dev_err(&pdev->dev, 462 "Fail to request IRQ:%d ret:%d\n", irq_num, ret); 463 goto out_free; 464 } 465 466 ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); 467 if (ret) 468 goto out_free; 469 470 ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num, 471 &irq->node); 472 if (ret) 473 goto out_free; 474 475 irq->irq_num = irq_num; 476 list_add(&irq->irqs_node, &ali_drw_pmu_irqs); 477 478 return irq; 479 480 out_free: 481 kfree(irq); 482 return ERR_PTR(ret); 483 } 484 485 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu, 486 struct platform_device *pdev) 487 { 488 int irq_num; 489 struct ali_drw_pmu_irq *irq; 490 491 /* Read and init IRQ */ 492 irq_num = platform_get_irq(pdev, 0); 493 if (irq_num < 0) 494 return irq_num; 495 496 mutex_lock(&ali_drw_pmu_irqs_lock); 497 irq = __ali_drw_pmu_init_irq(pdev, irq_num); 498 mutex_unlock(&ali_drw_pmu_irqs_lock); 499 500 if (IS_ERR(irq)) 501 return PTR_ERR(irq); 502 503 drw_pmu->irq = irq; 504 505 mutex_lock(&ali_drw_pmu_irqs_lock); 506 list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node); 507 mutex_unlock(&ali_drw_pmu_irqs_lock); 508 509 return 0; 510 } 511 512 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu) 513 { 514 struct ali_drw_pmu_irq *irq = drw_pmu->irq; 515 516 mutex_lock(&ali_drw_pmu_irqs_lock); 517 list_del_rcu(&drw_pmu->pmus_node); 518 519 if (!refcount_dec_and_test(&irq->refcount)) { 520 mutex_unlock(&ali_drw_pmu_irqs_lock); 521 return; 522 } 523 524 list_del(&irq->irqs_node); 525 mutex_unlock(&ali_drw_pmu_irqs_lock); 526 527 WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); 528 cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num, 529 &irq->node); 530 kfree(irq); 531 } 532 533 static int ali_drw_pmu_event_init(struct perf_event *event) 534 { 535 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 536 struct hw_perf_event *hwc = &event->hw; 537 struct perf_event *sibling; 538 struct device *dev = drw_pmu->pmu.dev; 539 540 if (event->attr.type != event->pmu->type) 541 return -ENOENT; 542 543 if (is_sampling_event(event)) { 544 dev_err(dev, "Sampling not supported!\n"); 545 return -EOPNOTSUPP; 546 } 547 548 if (event->attach_state & PERF_ATTACH_TASK) { 549 dev_err(dev, "Per-task counter cannot allocate!\n"); 550 return -EOPNOTSUPP; 551 } 552 553 event->cpu = drw_pmu->cpu; 554 if (event->cpu < 0) { 555 dev_err(dev, "Per-task mode not supported!\n"); 556 return -EOPNOTSUPP; 557 } 558 559 if (event->group_leader != event && 560 !is_software_event(event->group_leader)) { 561 dev_err(dev, "driveway only allow one event!\n"); 562 return -EINVAL; 563 } 564 565 for_each_sibling_event(sibling, event->group_leader) { 566 if (sibling != event && !is_software_event(sibling)) { 567 dev_err(dev, "driveway event not allowed!\n"); 568 return -EINVAL; 569 } 570 } 571 572 /* reset all the pmu counters */ 573 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 574 575 hwc->idx = -1; 576 577 return 0; 578 } 579 580 static void ali_drw_pmu_start(struct perf_event *event, int flags) 581 { 582 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 583 584 event->hw.state = 0; 585 586 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { 587 writel(ALI_DRW_PMU_CNT_START, 588 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 589 return; 590 } 591 592 ali_drw_pmu_event_set_period(event); 593 if (flags & PERF_EF_RELOAD) { 594 unsigned long prev_raw_count = 595 local64_read(&event->hw.prev_count); 596 writel(prev_raw_count, 597 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); 598 } 599 600 ali_drw_pmu_enable_counter(event); 601 602 writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 603 } 604 605 static void ali_drw_pmu_stop(struct perf_event *event, int flags) 606 { 607 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 608 609 if (event->hw.state & PERF_HES_STOPPED) 610 return; 611 612 if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID) 613 ali_drw_pmu_disable_counter(event); 614 615 writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 616 617 ali_drw_pmu_event_update(event); 618 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 619 } 620 621 static int ali_drw_pmu_add(struct perf_event *event, int flags) 622 { 623 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 624 struct hw_perf_event *hwc = &event->hw; 625 int idx = -1; 626 int evtid; 627 628 evtid = GET_DRW_EVENTID(event); 629 630 if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) { 631 idx = ali_drw_get_counter_idx(event); 632 if (idx < 0) 633 return idx; 634 drw_pmu->events[idx] = event; 635 drw_pmu->evtids[idx] = evtid; 636 } 637 hwc->idx = idx; 638 639 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 640 641 if (flags & PERF_EF_START) 642 ali_drw_pmu_start(event, PERF_EF_RELOAD); 643 644 /* Propagate our changes to the userspace mapping. */ 645 perf_event_update_userpage(event); 646 647 return 0; 648 } 649 650 static void ali_drw_pmu_del(struct perf_event *event, int flags) 651 { 652 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 653 struct hw_perf_event *hwc = &event->hw; 654 int idx = hwc->idx; 655 656 ali_drw_pmu_stop(event, PERF_EF_UPDATE); 657 658 if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) { 659 drw_pmu->events[idx] = NULL; 660 drw_pmu->evtids[idx] = 0; 661 clear_bit(idx, drw_pmu->used_mask); 662 } 663 664 perf_event_update_userpage(event); 665 } 666 667 static void ali_drw_pmu_read(struct perf_event *event) 668 { 669 ali_drw_pmu_event_update(event); 670 } 671 672 static int ali_drw_pmu_probe(struct platform_device *pdev) 673 { 674 struct ali_drw_pmu *drw_pmu; 675 struct resource *res; 676 char *name; 677 int ret; 678 679 drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL); 680 if (!drw_pmu) 681 return -ENOMEM; 682 683 drw_pmu->dev = &pdev->dev; 684 platform_set_drvdata(pdev, drw_pmu); 685 686 drw_pmu->cfg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 687 if (IS_ERR(drw_pmu->cfg_base)) 688 return PTR_ERR(drw_pmu->cfg_base); 689 690 name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx", 691 (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT)); 692 if (!name) 693 return -ENOMEM; 694 695 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 696 697 /* enable the generation of interrupt by all common counters */ 698 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 699 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL); 700 701 /* clearing interrupt status */ 702 writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); 703 704 drw_pmu->cpu = smp_processor_id(); 705 706 ret = ali_drw_pmu_init_irq(drw_pmu, pdev); 707 if (ret) 708 return ret; 709 710 drw_pmu->pmu = (struct pmu) { 711 .module = THIS_MODULE, 712 .task_ctx_nr = perf_invalid_context, 713 .event_init = ali_drw_pmu_event_init, 714 .add = ali_drw_pmu_add, 715 .del = ali_drw_pmu_del, 716 .start = ali_drw_pmu_start, 717 .stop = ali_drw_pmu_stop, 718 .read = ali_drw_pmu_read, 719 .attr_groups = ali_drw_pmu_attr_groups, 720 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 721 }; 722 723 ret = perf_pmu_register(&drw_pmu->pmu, name, -1); 724 if (ret) { 725 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n"); 726 ali_drw_pmu_uninit_irq(drw_pmu); 727 } 728 729 return ret; 730 } 731 732 static int ali_drw_pmu_remove(struct platform_device *pdev) 733 { 734 struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); 735 736 /* disable the generation of interrupt by all common counters */ 737 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 738 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL); 739 740 ali_drw_pmu_uninit_irq(drw_pmu); 741 perf_pmu_unregister(&drw_pmu->pmu); 742 743 return 0; 744 } 745 746 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 747 { 748 struct ali_drw_pmu_irq *irq; 749 struct ali_drw_pmu *drw_pmu; 750 unsigned int target; 751 int ret; 752 cpumask_t node_online_cpus; 753 754 irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); 755 if (cpu != irq->cpu) 756 return 0; 757 758 ret = cpumask_and(&node_online_cpus, 759 cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); 760 if (ret) 761 target = cpumask_any_but(&node_online_cpus, cpu); 762 else 763 target = cpumask_any_but(cpu_online_mask, cpu); 764 765 if (target >= nr_cpu_ids) 766 return 0; 767 768 /* We're only reading, but this isn't the place to be involving RCU */ 769 mutex_lock(&ali_drw_pmu_irqs_lock); 770 list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node) 771 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target); 772 mutex_unlock(&ali_drw_pmu_irqs_lock); 773 774 WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); 775 irq->cpu = target; 776 777 return 0; 778 } 779 780 /* 781 * Due to historical reasons, the HID used in the production environment is 782 * ARMHD700, so we leave ARMHD700 as Compatible ID. 783 */ 784 static const struct acpi_device_id ali_drw_acpi_match[] = { 785 {"BABA5000", 0}, 786 {"ARMHD700", 0}, 787 {} 788 }; 789 790 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match); 791 792 static struct platform_driver ali_drw_pmu_driver = { 793 .driver = { 794 .name = "ali_drw_pmu", 795 .acpi_match_table = ali_drw_acpi_match, 796 }, 797 .probe = ali_drw_pmu_probe, 798 .remove = ali_drw_pmu_remove, 799 }; 800 801 static int __init ali_drw_pmu_init(void) 802 { 803 int ret; 804 805 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 806 "ali_drw_pmu:online", 807 NULL, ali_drw_pmu_offline_cpu); 808 809 if (ret < 0) { 810 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n", 811 ret); 812 return ret; 813 } 814 ali_drw_cpuhp_state_num = ret; 815 816 ret = platform_driver_register(&ali_drw_pmu_driver); 817 if (ret) 818 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); 819 820 return ret; 821 } 822 823 static void __exit ali_drw_pmu_exit(void) 824 { 825 platform_driver_unregister(&ali_drw_pmu_driver); 826 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); 827 } 828 829 module_init(ali_drw_pmu_init); 830 module_exit(ali_drw_pmu_exit); 831 832 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>"); 833 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>"); 834 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 835 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver"); 836 MODULE_LICENSE("GPL v2"); 837