1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022 Amlogic, Inc. All rights reserved. 4 */ 5 6 #include <linux/bitfield.h> 7 #include <linux/init.h> 8 #include <linux/irqreturn.h> 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/of_device.h> 13 #include <linux/of_irq.h> 14 #include <linux/perf_event.h> 15 #include <linux/platform_device.h> 16 #include <linux/printk.h> 17 #include <linux/sysfs.h> 18 #include <linux/types.h> 19 20 #include <soc/amlogic/meson_ddr_pmu.h> 21 22 struct ddr_pmu { 23 struct pmu pmu; 24 struct dmc_info info; 25 struct dmc_counter counters; /* save counters from hw */ 26 bool pmu_enabled; 27 struct device *dev; 28 char *name; 29 struct hlist_node node; 30 enum cpuhp_state cpuhp_state; 31 int cpu; /* for cpu hotplug */ 32 }; 33 34 #define DDR_PERF_DEV_NAME "meson_ddr_bw" 35 #define MAX_AXI_PORTS_OF_CHANNEL 4 /* A DMC channel can monitor max 4 axi ports */ 36 37 #define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu) 38 #define dmc_info_to_pmu(p) container_of(p, struct ddr_pmu, info) 39 40 static void dmc_pmu_enable(struct ddr_pmu *pmu) 41 { 42 if (!pmu->pmu_enabled) 43 pmu->info.hw_info->enable(&pmu->info); 44 45 pmu->pmu_enabled = true; 46 } 47 48 static void dmc_pmu_disable(struct ddr_pmu *pmu) 49 { 50 if (pmu->pmu_enabled) 51 pmu->info.hw_info->disable(&pmu->info); 52 53 pmu->pmu_enabled = false; 54 } 55 56 static void meson_ddr_set_axi_filter(struct perf_event *event, u8 axi_id) 57 { 58 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); 59 int chann; 60 61 if (event->attr.config > ALL_CHAN_COUNTER_ID && 62 event->attr.config < COUNTER_MAX_ID) { 63 chann = event->attr.config - CHAN1_COUNTER_ID; 64 65 pmu->info.hw_info->set_axi_filter(&pmu->info, axi_id, chann); 66 } 67 } 68 69 static void ddr_cnt_addition(struct dmc_counter *sum, 70 struct dmc_counter *add1, 71 struct dmc_counter *add2, 72 int chann_nr) 73 { 74 int i; 75 u64 cnt1, cnt2; 76 77 sum->all_cnt = add1->all_cnt + add2->all_cnt; 78 sum->all_req = add1->all_req + add2->all_req; 79 for (i = 0; i < chann_nr; i++) { 80 cnt1 = add1->channel_cnt[i]; 81 cnt2 = add2->channel_cnt[i]; 82 83 sum->channel_cnt[i] = cnt1 + cnt2; 84 } 85 } 86 87 static void meson_ddr_perf_event_update(struct perf_event *event) 88 { 89 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); 90 u64 new_raw_count = 0; 91 struct dmc_counter dc = {0}, sum_dc = {0}; 92 int idx; 93 int chann_nr = pmu->info.hw_info->chann_nr; 94 95 /* get the remain counters in register. */ 96 pmu->info.hw_info->get_counters(&pmu->info, &dc); 97 98 ddr_cnt_addition(&sum_dc, &pmu->counters, &dc, chann_nr); 99 100 switch (event->attr.config) { 101 case ALL_CHAN_COUNTER_ID: 102 new_raw_count = sum_dc.all_cnt; 103 break; 104 case CHAN1_COUNTER_ID: 105 case CHAN2_COUNTER_ID: 106 case CHAN3_COUNTER_ID: 107 case CHAN4_COUNTER_ID: 108 case CHAN5_COUNTER_ID: 109 case CHAN6_COUNTER_ID: 110 case CHAN7_COUNTER_ID: 111 case CHAN8_COUNTER_ID: 112 idx = event->attr.config - CHAN1_COUNTER_ID; 113 new_raw_count = sum_dc.channel_cnt[idx]; 114 break; 115 } 116 117 local64_set(&event->count, new_raw_count); 118 } 119 120 static int meson_ddr_perf_event_init(struct perf_event *event) 121 { 122 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); 123 u64 config1 = event->attr.config1; 124 u64 config2 = event->attr.config2; 125 126 if (event->attr.type != event->pmu->type) 127 return -ENOENT; 128 129 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 130 return -EOPNOTSUPP; 131 132 if (event->cpu < 0) 133 return -EOPNOTSUPP; 134 135 /* check if the number of parameters is too much */ 136 if (event->attr.config != ALL_CHAN_COUNTER_ID && 137 hweight64(config1) + hweight64(config2) > MAX_AXI_PORTS_OF_CHANNEL) 138 return -EOPNOTSUPP; 139 140 event->cpu = pmu->cpu; 141 142 return 0; 143 } 144 145 static void meson_ddr_perf_event_start(struct perf_event *event, int flags) 146 { 147 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); 148 149 memset(&pmu->counters, 0, sizeof(pmu->counters)); 150 dmc_pmu_enable(pmu); 151 } 152 153 static int meson_ddr_perf_event_add(struct perf_event *event, int flags) 154 { 155 u64 config1 = event->attr.config1; 156 u64 config2 = event->attr.config2; 157 int i; 158 159 for_each_set_bit(i, 160 (const unsigned long *)&config1, 161 BITS_PER_TYPE(config1)) 162 meson_ddr_set_axi_filter(event, i); 163 164 for_each_set_bit(i, 165 (const unsigned long *)&config2, 166 BITS_PER_TYPE(config2)) 167 meson_ddr_set_axi_filter(event, i + 64); 168 169 if (flags & PERF_EF_START) 170 meson_ddr_perf_event_start(event, flags); 171 172 return 0; 173 } 174 175 static void meson_ddr_perf_event_stop(struct perf_event *event, int flags) 176 { 177 struct ddr_pmu *pmu = to_ddr_pmu(event->pmu); 178 179 if (flags & PERF_EF_UPDATE) 180 meson_ddr_perf_event_update(event); 181 182 dmc_pmu_disable(pmu); 183 } 184 185 static void meson_ddr_perf_event_del(struct perf_event *event, int flags) 186 { 187 meson_ddr_perf_event_stop(event, PERF_EF_UPDATE); 188 } 189 190 static ssize_t meson_ddr_perf_cpumask_show(struct device *dev, 191 struct device_attribute *attr, 192 char *buf) 193 { 194 struct ddr_pmu *pmu = dev_get_drvdata(dev); 195 196 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu)); 197 } 198 199 static struct device_attribute meson_ddr_perf_cpumask_attr = 200 __ATTR(cpumask, 0444, meson_ddr_perf_cpumask_show, NULL); 201 202 static struct attribute *meson_ddr_perf_cpumask_attrs[] = { 203 &meson_ddr_perf_cpumask_attr.attr, 204 NULL, 205 }; 206 207 static const struct attribute_group ddr_perf_cpumask_attr_group = { 208 .attrs = meson_ddr_perf_cpumask_attrs, 209 }; 210 211 static ssize_t 212 pmu_event_show(struct device *dev, struct device_attribute *attr, 213 char *page) 214 { 215 struct perf_pmu_events_attr *pmu_attr; 216 217 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 218 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 219 } 220 221 static ssize_t 222 event_show_unit(struct device *dev, struct device_attribute *attr, 223 char *page) 224 { 225 return sysfs_emit(page, "MB\n"); 226 } 227 228 static ssize_t 229 event_show_scale(struct device *dev, struct device_attribute *attr, 230 char *page) 231 { 232 /* one count = 16byte = 1.52587890625e-05 MB */ 233 return sysfs_emit(page, "1.52587890625e-05\n"); 234 } 235 236 #define AML_DDR_PMU_EVENT_ATTR(_name, _id) \ 237 { \ 238 .attr = __ATTR(_name, 0444, pmu_event_show, NULL), \ 239 .id = _id, \ 240 } 241 242 #define AML_DDR_PMU_EVENT_UNIT_ATTR(_name) \ 243 __ATTR(_name.unit, 0444, event_show_unit, NULL) 244 245 #define AML_DDR_PMU_EVENT_SCALE_ATTR(_name) \ 246 __ATTR(_name.scale, 0444, event_show_scale, NULL) 247 248 static struct device_attribute event_unit_attrs[] = { 249 AML_DDR_PMU_EVENT_UNIT_ATTR(total_rw_bytes), 250 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_1_rw_bytes), 251 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_2_rw_bytes), 252 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_3_rw_bytes), 253 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_4_rw_bytes), 254 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_5_rw_bytes), 255 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_6_rw_bytes), 256 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_7_rw_bytes), 257 AML_DDR_PMU_EVENT_UNIT_ATTR(chan_8_rw_bytes), 258 }; 259 260 static struct device_attribute event_scale_attrs[] = { 261 AML_DDR_PMU_EVENT_SCALE_ATTR(total_rw_bytes), 262 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_1_rw_bytes), 263 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_2_rw_bytes), 264 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_3_rw_bytes), 265 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_4_rw_bytes), 266 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_5_rw_bytes), 267 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_6_rw_bytes), 268 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_7_rw_bytes), 269 AML_DDR_PMU_EVENT_SCALE_ATTR(chan_8_rw_bytes), 270 }; 271 272 static struct perf_pmu_events_attr event_attrs[] = { 273 AML_DDR_PMU_EVENT_ATTR(total_rw_bytes, ALL_CHAN_COUNTER_ID), 274 AML_DDR_PMU_EVENT_ATTR(chan_1_rw_bytes, CHAN1_COUNTER_ID), 275 AML_DDR_PMU_EVENT_ATTR(chan_2_rw_bytes, CHAN2_COUNTER_ID), 276 AML_DDR_PMU_EVENT_ATTR(chan_3_rw_bytes, CHAN3_COUNTER_ID), 277 AML_DDR_PMU_EVENT_ATTR(chan_4_rw_bytes, CHAN4_COUNTER_ID), 278 AML_DDR_PMU_EVENT_ATTR(chan_5_rw_bytes, CHAN5_COUNTER_ID), 279 AML_DDR_PMU_EVENT_ATTR(chan_6_rw_bytes, CHAN6_COUNTER_ID), 280 AML_DDR_PMU_EVENT_ATTR(chan_7_rw_bytes, CHAN7_COUNTER_ID), 281 AML_DDR_PMU_EVENT_ATTR(chan_8_rw_bytes, CHAN8_COUNTER_ID), 282 }; 283 284 /* three attrs are combined an event */ 285 static struct attribute *ddr_perf_events_attrs[COUNTER_MAX_ID * 3]; 286 287 static struct attribute_group ddr_perf_events_attr_group = { 288 .name = "events", 289 .attrs = ddr_perf_events_attrs, 290 }; 291 292 static umode_t meson_ddr_perf_format_attr_visible(struct kobject *kobj, 293 struct attribute *attr, 294 int n) 295 { 296 struct pmu *pmu = dev_get_drvdata(kobj_to_dev(kobj)); 297 struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu); 298 const u64 *capability = ddr_pmu->info.hw_info->capability; 299 struct device_attribute *dev_attr; 300 int id; 301 char value[20]; // config1:xxx, 20 is enough 302 303 dev_attr = container_of(attr, struct device_attribute, attr); 304 dev_attr->show(NULL, NULL, value); 305 306 if (sscanf(value, "config1:%d", &id) == 1) 307 return capability[0] & (1ULL << id) ? attr->mode : 0; 308 309 if (sscanf(value, "config2:%d", &id) == 1) 310 return capability[1] & (1ULL << id) ? attr->mode : 0; 311 312 return attr->mode; 313 } 314 315 static struct attribute_group ddr_perf_format_attr_group = { 316 .name = "format", 317 .is_visible = meson_ddr_perf_format_attr_visible, 318 }; 319 320 static ssize_t meson_ddr_perf_identifier_show(struct device *dev, 321 struct device_attribute *attr, 322 char *page) 323 { 324 struct ddr_pmu *pmu = dev_get_drvdata(dev); 325 326 return sysfs_emit(page, "%s\n", pmu->name); 327 } 328 329 static struct device_attribute meson_ddr_perf_identifier_attr = 330 __ATTR(identifier, 0444, meson_ddr_perf_identifier_show, NULL); 331 332 static struct attribute *meson_ddr_perf_identifier_attrs[] = { 333 &meson_ddr_perf_identifier_attr.attr, 334 NULL, 335 }; 336 337 static const struct attribute_group ddr_perf_identifier_attr_group = { 338 .attrs = meson_ddr_perf_identifier_attrs, 339 }; 340 341 static const struct attribute_group *attr_groups[] = { 342 &ddr_perf_events_attr_group, 343 &ddr_perf_format_attr_group, 344 &ddr_perf_cpumask_attr_group, 345 &ddr_perf_identifier_attr_group, 346 NULL, 347 }; 348 349 static irqreturn_t dmc_irq_handler(int irq, void *dev_id) 350 { 351 struct dmc_info *info = dev_id; 352 struct ddr_pmu *pmu; 353 struct dmc_counter counters, *sum_cnter; 354 int i; 355 356 pmu = dmc_info_to_pmu(info); 357 358 if (info->hw_info->irq_handler(info, &counters) != 0) 359 goto out; 360 361 sum_cnter = &pmu->counters; 362 sum_cnter->all_cnt += counters.all_cnt; 363 sum_cnter->all_req += counters.all_req; 364 365 for (i = 0; i < pmu->info.hw_info->chann_nr; i++) 366 sum_cnter->channel_cnt[i] += counters.channel_cnt[i]; 367 368 if (pmu->pmu_enabled) 369 /* 370 * the timer interrupt only supprt 371 * one shot mode, we have to re-enable 372 * it in ISR to support continue mode. 373 */ 374 info->hw_info->enable(info); 375 376 dev_dbg(pmu->dev, "counts: %llu %llu %llu, %llu, %llu, %llu\t\t" 377 "sum: %llu %llu %llu, %llu, %llu, %llu\n", 378 counters.all_req, 379 counters.all_cnt, 380 counters.channel_cnt[0], 381 counters.channel_cnt[1], 382 counters.channel_cnt[2], 383 counters.channel_cnt[3], 384 385 pmu->counters.all_req, 386 pmu->counters.all_cnt, 387 pmu->counters.channel_cnt[0], 388 pmu->counters.channel_cnt[1], 389 pmu->counters.channel_cnt[2], 390 pmu->counters.channel_cnt[3]); 391 out: 392 return IRQ_HANDLED; 393 } 394 395 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) 396 { 397 struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node); 398 int target; 399 400 if (cpu != pmu->cpu) 401 return 0; 402 403 target = cpumask_any_but(cpu_online_mask, cpu); 404 if (target >= nr_cpu_ids) 405 return 0; 406 407 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 408 pmu->cpu = target; 409 410 WARN_ON(irq_set_affinity(pmu->info.irq_num, cpumask_of(pmu->cpu))); 411 412 return 0; 413 } 414 415 static void fill_event_attr(struct ddr_pmu *pmu) 416 { 417 int i, j, k; 418 struct attribute **dst = ddr_perf_events_attrs; 419 420 j = 0; 421 k = 0; 422 423 /* fill ALL_CHAN_COUNTER_ID event */ 424 dst[j++] = &event_attrs[k].attr.attr; 425 dst[j++] = &event_unit_attrs[k].attr; 426 dst[j++] = &event_scale_attrs[k].attr; 427 428 k++; 429 430 /* fill each channel event */ 431 for (i = 0; i < pmu->info.hw_info->chann_nr; i++, k++) { 432 dst[j++] = &event_attrs[k].attr.attr; 433 dst[j++] = &event_unit_attrs[k].attr; 434 dst[j++] = &event_scale_attrs[k].attr; 435 } 436 437 dst[j] = NULL; /* mark end */ 438 } 439 440 static void fmt_attr_fill(struct attribute **fmt_attr) 441 { 442 ddr_perf_format_attr_group.attrs = fmt_attr; 443 } 444 445 static int ddr_pmu_parse_dt(struct platform_device *pdev, 446 struct dmc_info *info) 447 { 448 void __iomem *base; 449 int i, ret; 450 451 info->hw_info = of_device_get_match_data(&pdev->dev); 452 453 for (i = 0; i < info->hw_info->dmc_nr; i++) { 454 /* resource 0 for ddr register base */ 455 base = devm_platform_ioremap_resource(pdev, i); 456 if (IS_ERR(base)) 457 return PTR_ERR(base); 458 459 info->ddr_reg[i] = base; 460 } 461 462 /* resource i for pll register base */ 463 base = devm_platform_ioremap_resource(pdev, i); 464 if (IS_ERR(base)) 465 return PTR_ERR(base); 466 467 info->pll_reg = base; 468 469 ret = platform_get_irq(pdev, 0); 470 if (ret < 0) 471 return ret; 472 473 info->irq_num = ret; 474 475 ret = devm_request_irq(&pdev->dev, info->irq_num, dmc_irq_handler, 476 IRQF_NOBALANCING, dev_name(&pdev->dev), 477 (void *)info); 478 if (ret < 0) 479 return ret; 480 481 return 0; 482 } 483 484 int meson_ddr_pmu_create(struct platform_device *pdev) 485 { 486 int ret; 487 char *name; 488 struct ddr_pmu *pmu; 489 490 pmu = devm_kzalloc(&pdev->dev, sizeof(struct ddr_pmu), GFP_KERNEL); 491 if (!pmu) 492 return -ENOMEM; 493 494 *pmu = (struct ddr_pmu) { 495 .pmu = { 496 .module = THIS_MODULE, 497 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 498 .task_ctx_nr = perf_invalid_context, 499 .attr_groups = attr_groups, 500 .event_init = meson_ddr_perf_event_init, 501 .add = meson_ddr_perf_event_add, 502 .del = meson_ddr_perf_event_del, 503 .start = meson_ddr_perf_event_start, 504 .stop = meson_ddr_perf_event_stop, 505 .read = meson_ddr_perf_event_update, 506 }, 507 }; 508 509 ret = ddr_pmu_parse_dt(pdev, &pmu->info); 510 if (ret < 0) 511 return ret; 512 513 fmt_attr_fill(pmu->info.hw_info->fmt_attr); 514 515 pmu->cpu = smp_processor_id(); 516 517 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME); 518 if (!name) 519 return -ENOMEM; 520 521 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, name, NULL, 522 ddr_perf_offline_cpu); 523 if (ret < 0) 524 return ret; 525 526 pmu->cpuhp_state = ret; 527 528 /* Register the pmu instance for cpu hotplug */ 529 ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node); 530 if (ret) 531 goto cpuhp_instance_err; 532 533 fill_event_attr(pmu); 534 535 ret = perf_pmu_register(&pmu->pmu, name, -1); 536 if (ret) 537 goto pmu_register_err; 538 539 pmu->name = name; 540 pmu->dev = &pdev->dev; 541 pmu->pmu_enabled = false; 542 543 platform_set_drvdata(pdev, pmu); 544 545 return 0; 546 547 pmu_register_err: 548 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node); 549 550 cpuhp_instance_err: 551 cpuhp_remove_state(pmu->cpuhp_state); 552 553 return ret; 554 } 555 556 int meson_ddr_pmu_remove(struct platform_device *pdev) 557 { 558 struct ddr_pmu *pmu = platform_get_drvdata(pdev); 559 560 perf_pmu_unregister(&pmu->pmu); 561 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node); 562 cpuhp_remove_state(pmu->cpuhp_state); 563 564 return 0; 565 } 566