1 /* 2 * HiSilicon SoC Hardware event counters support 3 * 4 * Copyright (C) 2017 Hisilicon Limited 5 * Author: Anurup M <anurup.m@huawei.com> 6 * Shaokun Zhang <zhangshaokun@hisilicon.com> 7 * 8 * This code is based on the uncore PMUs like arm-cci and arm-ccn. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 */ 14 #include <linux/bitmap.h> 15 #include <linux/bitops.h> 16 #include <linux/bug.h> 17 #include <linux/err.h> 18 #include <linux/errno.h> 19 #include <linux/interrupt.h> 20 21 #include <asm/local64.h> 22 23 #include "hisi_uncore_pmu.h" 24 25 #define HISI_GET_EVENTID(ev) (ev->hw.config_base & 0xff) 26 #define HISI_MAX_PERIOD(nr) (BIT_ULL(nr) - 1) 27 28 /* 29 * PMU format attributes 30 */ 31 ssize_t hisi_format_sysfs_show(struct device *dev, 32 struct device_attribute *attr, char *buf) 33 { 34 struct dev_ext_attribute *eattr; 35 36 eattr = container_of(attr, struct dev_ext_attribute, attr); 37 38 return sprintf(buf, "%s\n", (char *)eattr->var); 39 } 40 41 /* 42 * PMU event attributes 43 */ 44 ssize_t hisi_event_sysfs_show(struct device *dev, 45 struct device_attribute *attr, char *page) 46 { 47 struct dev_ext_attribute *eattr; 48 49 eattr = container_of(attr, struct dev_ext_attribute, attr); 50 51 return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var); 52 } 53 54 /* 55 * sysfs cpumask attributes. For uncore PMU, we only have a single CPU to show 56 */ 57 ssize_t hisi_cpumask_sysfs_show(struct device *dev, 58 struct device_attribute *attr, char *buf) 59 { 60 struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev)); 61 62 return sprintf(buf, "%d\n", hisi_pmu->on_cpu); 63 } 64 65 static bool hisi_validate_event_group(struct perf_event *event) 66 { 67 struct perf_event *sibling, *leader = event->group_leader; 68 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 69 /* Include count for the event */ 70 int counters = 1; 71 72 if (!is_software_event(leader)) { 73 /* 74 * We must NOT create groups containing mixed PMUs, although 75 * software events are acceptable 76 */ 77 if (leader->pmu != event->pmu) 78 return false; 79 80 /* Increment counter for the leader */ 81 if (leader != event) 82 counters++; 83 } 84 85 for_each_sibling_event(sibling, event->group_leader) { 86 if (is_software_event(sibling)) 87 continue; 88 if (sibling->pmu != event->pmu) 89 return false; 90 /* Increment counter for each sibling */ 91 counters++; 92 } 93 94 /* The group can not count events more than the counters in the HW */ 95 return counters <= hisi_pmu->num_counters; 96 } 97 98 int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx) 99 { 100 return idx >= 0 && idx < hisi_pmu->num_counters; 101 } 102 103 int hisi_uncore_pmu_get_event_idx(struct perf_event *event) 104 { 105 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 106 unsigned long *used_mask = hisi_pmu->pmu_events.used_mask; 107 u32 num_counters = hisi_pmu->num_counters; 108 int idx; 109 110 idx = find_first_zero_bit(used_mask, num_counters); 111 if (idx == num_counters) 112 return -EAGAIN; 113 114 set_bit(idx, used_mask); 115 116 return idx; 117 } 118 119 static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx) 120 { 121 if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) { 122 dev_err(hisi_pmu->dev, "Unsupported event index:%d!\n", idx); 123 return; 124 } 125 126 clear_bit(idx, hisi_pmu->pmu_events.used_mask); 127 } 128 129 int hisi_uncore_pmu_event_init(struct perf_event *event) 130 { 131 struct hw_perf_event *hwc = &event->hw; 132 struct hisi_pmu *hisi_pmu; 133 134 if (event->attr.type != event->pmu->type) 135 return -ENOENT; 136 137 /* 138 * We do not support sampling as the counters are all 139 * shared by all CPU cores in a CPU die(SCCL). Also we 140 * do not support attach to a task(per-process mode) 141 */ 142 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 143 return -EOPNOTSUPP; 144 145 /* 146 * The uncore counters not specific to any CPU, so cannot 147 * support per-task 148 */ 149 if (event->cpu < 0) 150 return -EINVAL; 151 152 /* 153 * Validate if the events in group does not exceed the 154 * available counters in hardware. 155 */ 156 if (!hisi_validate_event_group(event)) 157 return -EINVAL; 158 159 hisi_pmu = to_hisi_pmu(event->pmu); 160 if (event->attr.config > hisi_pmu->check_event) 161 return -EINVAL; 162 163 if (hisi_pmu->on_cpu == -1) 164 return -EINVAL; 165 /* 166 * We don't assign an index until we actually place the event onto 167 * hardware. Use -1 to signify that we haven't decided where to put it 168 * yet. 169 */ 170 hwc->idx = -1; 171 hwc->config_base = event->attr.config; 172 173 /* Enforce to use the same CPU for all events in this PMU */ 174 event->cpu = hisi_pmu->on_cpu; 175 176 return 0; 177 } 178 179 /* 180 * Set the counter to count the event that we're interested in, 181 * and enable interrupt and counter. 182 */ 183 static void hisi_uncore_pmu_enable_event(struct perf_event *event) 184 { 185 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 186 struct hw_perf_event *hwc = &event->hw; 187 188 hisi_pmu->ops->write_evtype(hisi_pmu, hwc->idx, 189 HISI_GET_EVENTID(event)); 190 191 hisi_pmu->ops->enable_counter_int(hisi_pmu, hwc); 192 hisi_pmu->ops->enable_counter(hisi_pmu, hwc); 193 } 194 195 /* 196 * Disable counter and interrupt. 197 */ 198 static void hisi_uncore_pmu_disable_event(struct perf_event *event) 199 { 200 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 201 struct hw_perf_event *hwc = &event->hw; 202 203 hisi_pmu->ops->disable_counter(hisi_pmu, hwc); 204 hisi_pmu->ops->disable_counter_int(hisi_pmu, hwc); 205 } 206 207 void hisi_uncore_pmu_set_event_period(struct perf_event *event) 208 { 209 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 210 struct hw_perf_event *hwc = &event->hw; 211 212 /* 213 * The HiSilicon PMU counters support 32 bits or 48 bits, depending on 214 * the PMU. We reduce it to 2^(counter_bits - 1) to account for the 215 * extreme interrupt latency. So we could hopefully handle the overflow 216 * interrupt before another 2^(counter_bits - 1) events occur and the 217 * counter overtakes its previous value. 218 */ 219 u64 val = BIT_ULL(hisi_pmu->counter_bits - 1); 220 221 local64_set(&hwc->prev_count, val); 222 /* Write start value to the hardware event counter */ 223 hisi_pmu->ops->write_counter(hisi_pmu, hwc, val); 224 } 225 226 void hisi_uncore_pmu_event_update(struct perf_event *event) 227 { 228 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 229 struct hw_perf_event *hwc = &event->hw; 230 u64 delta, prev_raw_count, new_raw_count; 231 232 do { 233 /* Read the count from the counter register */ 234 new_raw_count = hisi_pmu->ops->read_counter(hisi_pmu, hwc); 235 prev_raw_count = local64_read(&hwc->prev_count); 236 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 237 new_raw_count) != prev_raw_count); 238 /* 239 * compute the delta 240 */ 241 delta = (new_raw_count - prev_raw_count) & 242 HISI_MAX_PERIOD(hisi_pmu->counter_bits); 243 local64_add(delta, &event->count); 244 } 245 246 void hisi_uncore_pmu_start(struct perf_event *event, int flags) 247 { 248 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 249 struct hw_perf_event *hwc = &event->hw; 250 251 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 252 return; 253 254 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 255 hwc->state = 0; 256 hisi_uncore_pmu_set_event_period(event); 257 258 if (flags & PERF_EF_RELOAD) { 259 u64 prev_raw_count = local64_read(&hwc->prev_count); 260 261 hisi_pmu->ops->write_counter(hisi_pmu, hwc, prev_raw_count); 262 } 263 264 hisi_uncore_pmu_enable_event(event); 265 perf_event_update_userpage(event); 266 } 267 268 void hisi_uncore_pmu_stop(struct perf_event *event, int flags) 269 { 270 struct hw_perf_event *hwc = &event->hw; 271 272 hisi_uncore_pmu_disable_event(event); 273 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 274 hwc->state |= PERF_HES_STOPPED; 275 276 if (hwc->state & PERF_HES_UPTODATE) 277 return; 278 279 /* Read hardware counter and update the perf counter statistics */ 280 hisi_uncore_pmu_event_update(event); 281 hwc->state |= PERF_HES_UPTODATE; 282 } 283 284 int hisi_uncore_pmu_add(struct perf_event *event, int flags) 285 { 286 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 287 struct hw_perf_event *hwc = &event->hw; 288 int idx; 289 290 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 291 292 /* Get an available counter index for counting */ 293 idx = hisi_pmu->ops->get_event_idx(event); 294 if (idx < 0) 295 return idx; 296 297 event->hw.idx = idx; 298 hisi_pmu->pmu_events.hw_events[idx] = event; 299 300 if (flags & PERF_EF_START) 301 hisi_uncore_pmu_start(event, PERF_EF_RELOAD); 302 303 return 0; 304 } 305 306 void hisi_uncore_pmu_del(struct perf_event *event, int flags) 307 { 308 struct hisi_pmu *hisi_pmu = to_hisi_pmu(event->pmu); 309 struct hw_perf_event *hwc = &event->hw; 310 311 hisi_uncore_pmu_stop(event, PERF_EF_UPDATE); 312 hisi_uncore_pmu_clear_event_idx(hisi_pmu, hwc->idx); 313 perf_event_update_userpage(event); 314 hisi_pmu->pmu_events.hw_events[hwc->idx] = NULL; 315 } 316 317 void hisi_uncore_pmu_read(struct perf_event *event) 318 { 319 /* Read hardware counter and update the perf counter statistics */ 320 hisi_uncore_pmu_event_update(event); 321 } 322 323 void hisi_uncore_pmu_enable(struct pmu *pmu) 324 { 325 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu); 326 int enabled = bitmap_weight(hisi_pmu->pmu_events.used_mask, 327 hisi_pmu->num_counters); 328 329 if (!enabled) 330 return; 331 332 hisi_pmu->ops->start_counters(hisi_pmu); 333 } 334 335 void hisi_uncore_pmu_disable(struct pmu *pmu) 336 { 337 struct hisi_pmu *hisi_pmu = to_hisi_pmu(pmu); 338 339 hisi_pmu->ops->stop_counters(hisi_pmu); 340 } 341 342 /* 343 * Read Super CPU cluster and CPU cluster ID from MPIDR_EL1. 344 * If multi-threading is supported, CCL_ID is the low 3-bits in MPIDR[Aff2] 345 * and SCCL_ID is the upper 5-bits of Aff2 field; if not, SCCL_ID 346 * is in MPIDR[Aff2] and CCL_ID is in MPIDR[Aff1]. 347 */ 348 static void hisi_read_sccl_and_ccl_id(int *sccl_id, int *ccl_id) 349 { 350 u64 mpidr = read_cpuid_mpidr(); 351 352 if (mpidr & MPIDR_MT_BITMASK) { 353 int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2); 354 355 if (sccl_id) 356 *sccl_id = aff2 >> 3; 357 if (ccl_id) 358 *ccl_id = aff2 & 0x7; 359 } else { 360 if (sccl_id) 361 *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); 362 if (ccl_id) 363 *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); 364 } 365 } 366 367 /* 368 * Check whether the CPU is associated with this uncore PMU 369 */ 370 static bool hisi_pmu_cpu_is_associated_pmu(struct hisi_pmu *hisi_pmu) 371 { 372 int sccl_id, ccl_id; 373 374 if (hisi_pmu->ccl_id == -1) { 375 /* If CCL_ID is -1, the PMU only shares the same SCCL */ 376 hisi_read_sccl_and_ccl_id(&sccl_id, NULL); 377 378 return sccl_id == hisi_pmu->sccl_id; 379 } 380 381 hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id); 382 383 return sccl_id == hisi_pmu->sccl_id && ccl_id == hisi_pmu->ccl_id; 384 } 385 386 int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 387 { 388 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu, 389 node); 390 391 if (!hisi_pmu_cpu_is_associated_pmu(hisi_pmu)) 392 return 0; 393 394 cpumask_set_cpu(cpu, &hisi_pmu->associated_cpus); 395 396 /* If another CPU is already managing this PMU, simply return. */ 397 if (hisi_pmu->on_cpu != -1) 398 return 0; 399 400 /* Use this CPU in cpumask for event counting */ 401 hisi_pmu->on_cpu = cpu; 402 403 /* Overflow interrupt also should use the same CPU */ 404 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); 405 406 return 0; 407 } 408 409 int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 410 { 411 struct hisi_pmu *hisi_pmu = hlist_entry_safe(node, struct hisi_pmu, 412 node); 413 cpumask_t pmu_online_cpus; 414 unsigned int target; 415 416 if (!cpumask_test_and_clear_cpu(cpu, &hisi_pmu->associated_cpus)) 417 return 0; 418 419 /* Nothing to do if this CPU doesn't own the PMU */ 420 if (hisi_pmu->on_cpu != cpu) 421 return 0; 422 423 /* Give up ownership of the PMU */ 424 hisi_pmu->on_cpu = -1; 425 426 /* Choose a new CPU to migrate ownership of the PMU to */ 427 cpumask_and(&pmu_online_cpus, &hisi_pmu->associated_cpus, 428 cpu_online_mask); 429 target = cpumask_any_but(&pmu_online_cpus, cpu); 430 if (target >= nr_cpu_ids) 431 return 0; 432 433 perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target); 434 /* Use this CPU for event counting */ 435 hisi_pmu->on_cpu = target; 436 WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); 437 438 return 0; 439 } 440