1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * HiSilicon SoC Hardware event counters support 4 * 5 * Copyright (C) 2017 HiSilicon Limited 6 * Author: Anurup M <anurup.m@huawei.com> 7 * Shaokun Zhang <zhangshaokun@hisilicon.com> 8 * 9 * This code is based on the uncore PMUs like arm-cci and arm-ccn. 10 */ 11 #include <linux/bitmap.h> 12 #include <linux/bitops.h> 13 #include <linux/bug.h> 14 #include <linux/err.h> 15 #include <linux/errno.h> 16 #include <linux/interrupt.h> 17 18 #include <asm/cputype.h> 19 #include <asm/local64.h> 20 21 #include "hisi_uncore_pmu.h" 22 23 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff) 24 #define HISI_MAX_PERIOD(nr) (GENMASK_ULL((nr) - 1, 0)) 25 26 /* 27 * PMU format attributes 28 */ 29 ssize_t hisi_format_sysfs_show(struct device *dev, 30 struct device_attribute *attr, char *buf) 31 { 32 struct dev_ext_attribute *eattr; 33 34 eattr = container_of(attr, struct dev_ext_attribute, attr); 35 36 return sysfs_emit(buf, "%s\n", (char *)eattr->var); 37 } 38 EXPORT_SYMBOL_GPL(hisi_format_sysfs_show); 39 40 /* 41 * PMU event attributes 42 */ 43 ssize_t hisi_event_sysfs_show(struct device *dev, 44 struct device_attribute *attr, char *page) 45 { 46 struct dev_ext_attribute *eattr; 47 48 eattr = container_of(attr, struct dev_ext_attribute, attr); 49 50 return sysfs_emit(page, "config=0x%lx\n", (unsigned long)eattr->var); 51 } 52 EXPORT_SYMBOL_GPL(hisi_event_sysfs_show); 53 54 /* 55 * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show 56 */ 57 ssize_t hisi_cpumask_sysfs_show(struct device *dev, 58 struct device_attribute *attr, char *buf) 59 { 60 struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev)); 61 62 return sysfs_emit(buf, "%d\n", hisi_pmu->on_cpu); 63 } 64 EXPORT_SYMBOL_GPL(hisi_cpumask_sysfs_show); 65 66 static bool hisi_validate_event_group(struct perf_event *event) 67 { 68 struct perf_event *sibling, *leader = event->group_leader; 69 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 70 /* Include count for the event */ 71 int counters = 1; 72 73 if (!is_software_event(leader)) { 74 /* 75 * We must NOT create groups containing mixed PMUs, although 76 * software events are acceptable 77 */ 78 if (leader->pmu != event->pmu) 79 return false; 80 81 /* Increment counter for the leader */ 82 if (leader != event) 83 counters++; 84 } 85 86 for_each_sibling_event(sibling, event->group_leader) { 87 if (is_software_event(sibling)) 88 continue; 89 if (sibling->pmu != event->pmu) 90 return false; 91 /* Increment counter for each sibling */ 92 counters++; 93 } 94 95 /* The group can not count events more than the counters in the HW */ 96 return counters <= hisi_pmu->num_counters; 97 } 98 99 int hisi_uncore_pmu_get_event_idx(struct perf_event *event) 100 { 101 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 102 unsigned long *used_mask = hisi_pmu->pmu_events.used_mask; 103 u32 num_counters = hisi_pmu->num_counters; 104 int idx; 105 106 idx = find_first_zero_bit(used_mask, num_counters); 107 if (idx == num_counters) 108 return -EAGAIN; 109 110 set_bit(idx, used_mask); 111 112 return idx; 113 } 114 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_get_event_idx); 115 116 ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev, 117 struct device_attribute *attr, 118 char *page) 119 { 120 struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev)); 121 122 return sysfs_emit(page, "0x%08x\n", hisi_pmu->identifier); 123 } 124 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_identifier_attr_show); 125 126 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx) 127 { 128 clear_bit(idx, hisi_pmu->pmu_events.used_mask); 129 } 130 131 static irqreturn_t hisi_uncore_pmu_isr(int irq, void *data) 132 { 133 struct hisi_pmu *hisi_pmu = data; 134 struct perf_event *event; 135 unsigned long overflown; 136 int idx; 137 138 overflown = hisi_pmu->ops->get_int_status(hisi_pmu); 139 if (!overflown) 140 return IRQ_NONE; 141 142 /* 143 * Find the counter index which overflowed if the bit was set 144 * and handle it. 145 */ 146 for_each_set_bit(idx, &overflown, hisi_pmu->num_counters) { 147 /* Write 1 to clear the IRQ status flag */ 148 hisi_pmu->ops->clear_int_status(hisi_pmu, idx); 149 /* Get the corresponding event struct */ 150 event = hisi_pmu->pmu_events.hw_events[idx]; 151 if (!event) 152 continue; 153 154 hisi_uncore_pmu_event_update(event); 155 hisi_uncore_pmu_set_event_period(event); 156 } 157 158 return IRQ_HANDLED; 159 } 160 161 int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu, 162 struct platform_device *pdev) 163 { 164 int irq, ret; 165 166 irq = platform_get_irq(pdev, 0); 167 if (irq < 0) 168 return irq; 169 170 ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr, 171 IRQF_NOBALANCING | IRQF_NO_THREAD, 172 dev_name(&pdev->dev), hisi_pmu); 173 if (ret < 0) { 174 dev_err(&pdev->dev, 175 "Fail to request IRQ: %d ret: %d.\n", irq, ret); 176 return ret; 177 } 178 179 hisi_pmu->irq = irq; 180 181 return 0; 182 } 183 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_init_irq); 184 185 int hisi_uncore_pmu_event_init(struct perf_event *event) 186 { 187 struct hw_perf_event *hwc = &event->hw; 188 struct hisi_pmu *hisi_pmu; 189 190 if (event->attr.type != event->pmu->type) 191 return -ENOENT; 192 193 /* 194 * We do not support sampling as the counters are all 195 * shared by all CPU cores in a CPU die(SCCL). Also we 196 * do not support attach to a task(per-process mode) 197 */ 198 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 199 return -EOPNOTSUPP; 200 201 /* 202 * The uncore counters not specific to any CPU, so cannot 203 * support per-task 204 */ 205 if (event->cpu < 0) 206 return -EINVAL; 207 208 /* 209 * Validate if the events in group does not exceed the 210 * available counters in hardware. 211 */ 212 if (!hisi_validate_event_group(event)) 213 return -EINVAL; 214 215 hisi_pmu = to_hisi_pmu(event->pmu); 216 if (event->attr.config > hisi_pmu->check_event) 217 return -EINVAL; 218 219 if (hisi_pmu->on_cpu == -1) 220 return -EINVAL; 221 /* 222 * We don't assign an index until we actually place the event onto 223 * hardware. Use -1 to signify that we haven't decided where to put it 224 * yet. 225 */ 226 hwc->idx = -1; 227 hwc->config_base = event->attr.config; 228 229 /* Enforce to use the same CPU for all events in this PMU */ 230 event->cpu = hisi_pmu->on_cpu; 231 232 return 0; 233 } 234 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_init); 235 236 /* 237 * Set the counter to count the event that we're interested in, 238 * and enable interrupt and counter. 239 */ 240 static void hisi_uncore_pmu_enable_event(struct perf_event *event) 241 { 242 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 243 struct hw_perf_event *hwc = &event->hw; 244 245 hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx, 246 HISI_GET_EVENTID(event)); 247 248 if (hisi_pmu->ops->enable_filter) 249 hisi_pmu->ops->enable_filter(event); 250 251 hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc); 252 hisi_pmu->ops->enable_counter(hisi_pmu, hwc); 253 } 254 255 /* 256 * Disable counter and interrupt. 257 */ 258 static void hisi_uncore_pmu_disable_event(struct perf_event *event) 259 { 260 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 261 struct hw_perf_event *hwc = &event->hw; 262 263 hisi_pmu->ops->disable_counter(hisi_pmu, hwc); 264 hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc); 265 266 if (hisi_pmu->ops->disable_filter) 267 hisi_pmu->ops->disable_filter(event); 268 } 269 270 void hisi_uncore_pmu_set_event_period(struct perf_event *event) 271 { 272 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 273 struct hw_perf_event *hwc = &event->hw; 274 275 /* 276 * The HiSilicon PMU counters support 32 bits or 48 bits, depending on 277 * the PMU. We reduce it to 2^(counter_bits - 1) to account for the 278 * extreme interrupt latency. So we could hopefully handle the overflow 279 * interrupt before another 2^(counter_bits - 1) events occur and the 280 * counter overtakes its previous value. 281 */ 282 u64 val = BIT_ULL(hisi_pmu->counter_bits - 1); 283 284 local64_set(&hwc->prev_count, val); 285 /* Write start value to the hardware event counter */ 286 hisi_pmu->ops->write_counter(hisi_pmu, hwc, val); 287 } 288 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_set_event_period); 289 290 void hisi_uncore_pmu_event_update(struct perf_event *event) 291 { 292 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 293 struct hw_perf_event *hwc = &event->hw; 294 u64 delta, prev_raw_count, new_raw_count; 295 296 do { 297 /* Read the count from the counter register */ 298 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc); 299 prev_raw_count = local64_read(&hwc->prev_count); 300 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 301 new_raw_count) != prev_raw_count); 302 /* 303 * compute the delta 304 */ 305 delta = (new_raw_count - prev_raw_count) & 306 HISI_MAX_PERIOD(hisi_pmu->counter_bits); 307 local64_add(delta, &event->count); 308 } 309 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_event_update); 310 311 void hisi_uncore_pmu_start(struct perf_event *event, int flags) 312 { 313 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 314 struct hw_perf_event *hwc = &event->hw; 315 316 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 317 return; 318 319 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 320 hwc->state = 0; 321 hisi_uncore_pmu_set_event_period(event); 322 323 if (flags & PERF_EF_RELOAD) { 324 u64 prev_raw_count = local64_read(&hwc->prev_count); 325 326 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count); 327 } 328 329 hisi_uncore_pmu_enable_event(event); 330 perf_event_update_userpage(event); 331 } 332 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_start); 333 334 void hisi_uncore_pmu_stop(struct perf_event *event, int flags) 335 { 336 struct hw_perf_event *hwc = &event->hw; 337 338 hisi_uncore_pmu_disable_event(event); 339 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 340 hwc->state |= PERF_HES_STOPPED; 341 342 if (hwc->state & PERF_HES_UPTODATE) 343 return; 344 345 /* Read hardware counter and update the perf counter statistics */ 346 hisi_uncore_pmu_event_update(event); 347 hwc->state |= PERF_HES_UPTODATE; 348 } 349 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_stop); 350 351 int hisi_uncore_pmu_add(struct perf_event *event, int flags) 352 { 353 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 354 struct hw_perf_event *hwc = &event->hw; 355 int idx; 356 357 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 358 359 /* Get an available counter index for counting */ 360 idx = hisi_pmu->ops->get_event_idx(event); 361 if (idx < 0) 362 return idx; 363 364 event->hw.idx = idx; 365 hisi_pmu->pmu_events.hw_events[idx] = event; 366 367 if (flags & PERF_EF_START) 368 hisi_uncore_pmu_start(event, PERF_EF_RELOAD); 369 370 return 0; 371 } 372 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_add); 373 374 void hisi_uncore_pmu_del(struct perf_event *event, int flags) 375 { 376 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 377 struct hw_perf_event *hwc = &event->hw; 378 379 hisi_uncore_pmu_stop(event, PERF_EF_UPDATE); 380 hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx); 381 perf_event_update_userpage(event); 382 hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL; 383 } 384 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_del); 385 386 void hisi_uncore_pmu_read(struct perf_event *event) 387 { 388 /* Read hardware counter and update the perf counter statistics */ 389 hisi_uncore_pmu_event_update(event); 390 } 391 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_read); 392 393 void hisi_uncore_pmu_enable(struct pmu *pmu) 394 { 395 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu); 396 bool enabled = !bitmap_empty(hisi_pmu->pmu_events.used_mask, 397 hisi_pmu->num_counters); 398 399 if (!enabled) 400 return; 401 402 hisi_pmu->ops->start_counters(hisi_pmu); 403 } 404 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_enable); 405 406 void hisi_uncore_pmu_disable(struct pmu *pmu) 407 { 408 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu); 409 410 hisi_pmu->ops->stop_counters(hisi_pmu); 411 } 412 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_disable); 413 414 415 /* 416 * The Super CPU Cluster (SCCL) and CPU Cluster (CCL) IDs can be 417 * determined from the MPIDR_EL1, but the encoding varies by CPU: 418 * 419 * - For MT variants of TSV110: 420 * SCCL is Aff2[7:3], CCL is Aff2[2:0] 421 * 422 * - For other MT parts: 423 * SCCL is Aff3[7:0], CCL is Aff2[7:0] 424 * 425 * - For non-MT parts: 426 * SCCL is Aff2[7:0], CCL is Aff1[7:0] 427 */ 428 static void hisi_read_sccl_and_ccl_id(int *scclp, int *cclp) 429 { 430 u64 mpidr = read_cpuid_mpidr(); 431 int aff3 = MPIDR_AFFINITY_LEVEL(mpidr, 3); 432 int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2); 433 int aff1 = MPIDR_AFFINITY_LEVEL(mpidr, 1); 434 bool mt = mpidr & MPIDR_MT_BITMASK; 435 int sccl, ccl; 436 437 if (mt && read_cpuid_part_number() == HISI_CPU_PART_TSV110) { 438 sccl = aff2 >> 3; 439 ccl = aff2 & 0x7; 440 } else if (mt) { 441 sccl = aff3; 442 ccl = aff2; 443 } else { 444 sccl = aff2; 445 ccl = aff1; 446 } 447 448 if (scclp) 449 *scclp = sccl; 450 if (cclp) 451 *cclp = ccl; 452 } 453 454 /* 455 * Check whether the CPU is associated with this uncore PMU 456 */ 457 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu) 458 { 459 int sccl_id, ccl_id; 460 461 /* If SCCL_ID is -1, the PMU is in a SICL and has no CPU affinity */ 462 if (hisi_pmu->sccl_id == -1) 463 return true; 464 465 if (hisi_pmu->ccl_id == -1) { 466 /* If CCL_ID is -1, the PMU only shares the same SCCL */ 467 hisi_read_sccl_and_ccl_id(&sccl_id, NULL); 468 469 return sccl_id == hisi_pmu->sccl_id; 470 } 471 472 hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id); 473 474 return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id; 475 } 476 477 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 478 { 479 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu, 480 node); 481 482 if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu)) 483 return 0; 484 485 cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus); 486 487 /* If another CPU is already managing this PMU, simply return. */ 488 if (hisi_pmu->on_cpu != -1) 489 return 0; 490 491 /* Use this CPU in cpumask for event counting */ 492 hisi_pmu->on_cpu = cpu; 493 494 /* Overflow interrupt also should use the same CPU */ 495 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); 496 497 return 0; 498 } 499 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_online_cpu); 500 501 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 502 { 503 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu, 504 node); 505 cpumask_t pmu_online_cpus; 506 unsigned int target; 507 508 if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus)) 509 return 0; 510 511 /* Nothing to do if this CPU doesn't own the PMU */ 512 if (hisi_pmu->on_cpu != cpu) 513 return 0; 514 515 /* Give up ownership of the PMU */ 516 hisi_pmu->on_cpu = -1; 517 518 /* Choose a new CPU to migrate ownership of the PMU to */ 519 cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus, 520 cpu_online_mask); 521 target = cpumask_any_but(&pmu_online_cpus, cpu); 522 if (target >= nr_cpu_ids) 523 return 0; 524 525 perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target); 526 /* Use this CPU for event counting */ 527 hisi_pmu->on_cpu = target; 528 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); 529 530 return 0; 531 } 532 EXPORT_SYMBOL_GPL(hisi_uncore_pmu_offline_cpu); 533 534 void hisi_pmu_init(struct hisi_pmu *hisi_pmu, const char *name, 535 struct module *module) 536 { 537 struct pmu *pmu = &hisi_pmu->pmu; 538 539 pmu->name = name; 540 pmu->module = module; 541 pmu->task_ctx_nr = perf_invalid_context; 542 pmu->event_init = hisi_uncore_pmu_event_init; 543 pmu->pmu_enable = hisi_uncore_pmu_enable; 544 pmu->pmu_disable = hisi_uncore_pmu_disable; 545 pmu->add = hisi_uncore_pmu_add; 546 pmu->del = hisi_uncore_pmu_del; 547 pmu->start = hisi_uncore_pmu_start; 548 pmu->stop = hisi_uncore_pmu_stop; 549 pmu->read = hisi_uncore_pmu_read; 550 pmu->attr_groups = hisi_pmu->pmu_events.attr_groups; 551 pmu->capabilities = PERF_PMU_CAP_NO_EXCLUDE; 552 } 553 EXPORT_SYMBOL_GPL(hisi_pmu_init); 554 555 MODULE_LICENSE("GPL v2"); 556