1212188a5SHendrik Brueckner /* 2212188a5SHendrik Brueckner * Performance event support for s390x - CPU-measurement Counter Facility 3212188a5SHendrik Brueckner * 4*db17160dSHendrik Brueckner * Copyright IBM Corp. 2012, 2017 5212188a5SHendrik Brueckner * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 6212188a5SHendrik Brueckner * 7212188a5SHendrik Brueckner * This program is free software; you can redistribute it and/or modify 8212188a5SHendrik Brueckner * it under the terms of the GNU General Public License (version 2 only) 9212188a5SHendrik Brueckner * as published by the Free Software Foundation. 10212188a5SHendrik Brueckner */ 11212188a5SHendrik Brueckner #define KMSG_COMPONENT "cpum_cf" 12212188a5SHendrik Brueckner #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 13212188a5SHendrik Brueckner 14212188a5SHendrik Brueckner #include <linux/kernel.h> 15212188a5SHendrik Brueckner #include <linux/kernel_stat.h> 16212188a5SHendrik Brueckner #include <linux/perf_event.h> 17212188a5SHendrik Brueckner #include <linux/percpu.h> 18212188a5SHendrik Brueckner #include <linux/notifier.h> 19212188a5SHendrik Brueckner #include <linux/init.h> 20212188a5SHendrik Brueckner #include <linux/export.h> 211e3cab2fSHeiko Carstens #include <asm/ctl_reg.h> 22212188a5SHendrik Brueckner #include <asm/irq.h> 23212188a5SHendrik Brueckner #include <asm/cpu_mf.h> 24212188a5SHendrik Brueckner 25212188a5SHendrik Brueckner /* CPU-measurement counter facility supports these CPU counter sets: 26212188a5SHendrik Brueckner * For CPU counter sets: 27212188a5SHendrik Brueckner * Basic counter set: 0-31 28212188a5SHendrik Brueckner * Problem-state counter set: 32-63 29212188a5SHendrik Brueckner * Crypto-activity counter set: 64-127 30212188a5SHendrik Brueckner * Extented counter set: 128-159 31212188a5SHendrik Brueckner */ 32212188a5SHendrik Brueckner enum cpumf_ctr_set { 33212188a5SHendrik Brueckner /* CPU counter sets */ 34212188a5SHendrik Brueckner CPUMF_CTR_SET_BASIC = 0, 35212188a5SHendrik Brueckner CPUMF_CTR_SET_USER = 1, 36212188a5SHendrik Brueckner CPUMF_CTR_SET_CRYPTO = 2, 37212188a5SHendrik Brueckner CPUMF_CTR_SET_EXT = 3, 38212188a5SHendrik Brueckner 39212188a5SHendrik Brueckner /* Maximum number of counter sets */ 40212188a5SHendrik Brueckner CPUMF_CTR_SET_MAX, 41212188a5SHendrik Brueckner }; 42212188a5SHendrik Brueckner 43212188a5SHendrik Brueckner #define CPUMF_LCCTL_ENABLE_SHIFT 16 44212188a5SHendrik Brueckner #define CPUMF_LCCTL_ACTCTL_SHIFT 0 45212188a5SHendrik Brueckner static const u64 cpumf_state_ctl[CPUMF_CTR_SET_MAX] = { 46212188a5SHendrik Brueckner [CPUMF_CTR_SET_BASIC] = 0x02, 47212188a5SHendrik Brueckner [CPUMF_CTR_SET_USER] = 0x04, 48212188a5SHendrik Brueckner [CPUMF_CTR_SET_CRYPTO] = 0x08, 49212188a5SHendrik Brueckner [CPUMF_CTR_SET_EXT] = 0x01, 50212188a5SHendrik Brueckner }; 51212188a5SHendrik Brueckner 52212188a5SHendrik Brueckner static void ctr_set_enable(u64 *state, int ctr_set) 53212188a5SHendrik Brueckner { 54212188a5SHendrik Brueckner *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT; 55212188a5SHendrik Brueckner } 56212188a5SHendrik Brueckner static void ctr_set_disable(u64 *state, int ctr_set) 57212188a5SHendrik Brueckner { 58212188a5SHendrik Brueckner *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT); 59212188a5SHendrik Brueckner } 60212188a5SHendrik Brueckner static void ctr_set_start(u64 *state, int ctr_set) 61212188a5SHendrik Brueckner { 62212188a5SHendrik Brueckner *state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT; 63212188a5SHendrik Brueckner } 64212188a5SHendrik Brueckner static void ctr_set_stop(u64 *state, int ctr_set) 65212188a5SHendrik Brueckner { 66212188a5SHendrik Brueckner *state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT); 67212188a5SHendrik Brueckner } 68212188a5SHendrik Brueckner 69212188a5SHendrik Brueckner /* Local CPUMF event structure */ 70212188a5SHendrik Brueckner struct cpu_hw_events { 71212188a5SHendrik Brueckner struct cpumf_ctr_info info; 72212188a5SHendrik Brueckner atomic_t ctr_set[CPUMF_CTR_SET_MAX]; 73212188a5SHendrik Brueckner u64 state, tx_state; 74212188a5SHendrik Brueckner unsigned int flags; 75fbbe0701SSukadev Bhattiprolu unsigned int txn_flags; 76212188a5SHendrik Brueckner }; 77212188a5SHendrik Brueckner static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 78212188a5SHendrik Brueckner .ctr_set = { 79212188a5SHendrik Brueckner [CPUMF_CTR_SET_BASIC] = ATOMIC_INIT(0), 80212188a5SHendrik Brueckner [CPUMF_CTR_SET_USER] = ATOMIC_INIT(0), 81212188a5SHendrik Brueckner [CPUMF_CTR_SET_CRYPTO] = ATOMIC_INIT(0), 82212188a5SHendrik Brueckner [CPUMF_CTR_SET_EXT] = ATOMIC_INIT(0), 83212188a5SHendrik Brueckner }, 84212188a5SHendrik Brueckner .state = 0, 85212188a5SHendrik Brueckner .flags = 0, 86fbbe0701SSukadev Bhattiprolu .txn_flags = 0, 87212188a5SHendrik Brueckner }; 88212188a5SHendrik Brueckner 89212188a5SHendrik Brueckner static int get_counter_set(u64 event) 90212188a5SHendrik Brueckner { 91212188a5SHendrik Brueckner int set = -1; 92212188a5SHendrik Brueckner 93212188a5SHendrik Brueckner if (event < 32) 94212188a5SHendrik Brueckner set = CPUMF_CTR_SET_BASIC; 95212188a5SHendrik Brueckner else if (event < 64) 96212188a5SHendrik Brueckner set = CPUMF_CTR_SET_USER; 97212188a5SHendrik Brueckner else if (event < 128) 98212188a5SHendrik Brueckner set = CPUMF_CTR_SET_CRYPTO; 99f47586b2SHendrik Brueckner else if (event < 256) 100212188a5SHendrik Brueckner set = CPUMF_CTR_SET_EXT; 101212188a5SHendrik Brueckner 102212188a5SHendrik Brueckner return set; 103212188a5SHendrik Brueckner } 104212188a5SHendrik Brueckner 105212188a5SHendrik Brueckner static int validate_ctr_version(const struct hw_perf_event *hwc) 106212188a5SHendrik Brueckner { 107212188a5SHendrik Brueckner struct cpu_hw_events *cpuhw; 108212188a5SHendrik Brueckner int err = 0; 109212188a5SHendrik Brueckner 110212188a5SHendrik Brueckner cpuhw = &get_cpu_var(cpu_hw_events); 111212188a5SHendrik Brueckner 112212188a5SHendrik Brueckner /* check required version for counter sets */ 113212188a5SHendrik Brueckner switch (hwc->config_base) { 114212188a5SHendrik Brueckner case CPUMF_CTR_SET_BASIC: 115212188a5SHendrik Brueckner case CPUMF_CTR_SET_USER: 116212188a5SHendrik Brueckner if (cpuhw->info.cfvn < 1) 117212188a5SHendrik Brueckner err = -EOPNOTSUPP; 118212188a5SHendrik Brueckner break; 119212188a5SHendrik Brueckner case CPUMF_CTR_SET_CRYPTO: 120212188a5SHendrik Brueckner case CPUMF_CTR_SET_EXT: 121212188a5SHendrik Brueckner if (cpuhw->info.csvn < 1) 122212188a5SHendrik Brueckner err = -EOPNOTSUPP; 123f47586b2SHendrik Brueckner if ((cpuhw->info.csvn == 1 && hwc->config > 159) || 124f47586b2SHendrik Brueckner (cpuhw->info.csvn == 2 && hwc->config > 175) || 125f47586b2SHendrik Brueckner (cpuhw->info.csvn > 2 && hwc->config > 255)) 126f47586b2SHendrik Brueckner err = -EOPNOTSUPP; 127212188a5SHendrik Brueckner break; 128212188a5SHendrik Brueckner } 129212188a5SHendrik Brueckner 130212188a5SHendrik Brueckner put_cpu_var(cpu_hw_events); 131212188a5SHendrik Brueckner return err; 132212188a5SHendrik Brueckner } 133212188a5SHendrik Brueckner 134212188a5SHendrik Brueckner static int validate_ctr_auth(const struct hw_perf_event *hwc) 135212188a5SHendrik Brueckner { 136212188a5SHendrik Brueckner struct cpu_hw_events *cpuhw; 137212188a5SHendrik Brueckner u64 ctrs_state; 138212188a5SHendrik Brueckner int err = 0; 139212188a5SHendrik Brueckner 140212188a5SHendrik Brueckner cpuhw = &get_cpu_var(cpu_hw_events); 141212188a5SHendrik Brueckner 14258f8e9daSHendrik Brueckner /* Check authorization for cpu counter sets. 14358f8e9daSHendrik Brueckner * If the particular CPU counter set is not authorized, 14458f8e9daSHendrik Brueckner * return with -ENOENT in order to fall back to other 14558f8e9daSHendrik Brueckner * PMUs that might suffice the event request. 14658f8e9daSHendrik Brueckner */ 147212188a5SHendrik Brueckner ctrs_state = cpumf_state_ctl[hwc->config_base]; 148212188a5SHendrik Brueckner if (!(ctrs_state & cpuhw->info.auth_ctl)) 14958f8e9daSHendrik Brueckner err = -ENOENT; 150212188a5SHendrik Brueckner 151212188a5SHendrik Brueckner put_cpu_var(cpu_hw_events); 152212188a5SHendrik Brueckner return err; 153212188a5SHendrik Brueckner } 154212188a5SHendrik Brueckner 155212188a5SHendrik Brueckner /* 156212188a5SHendrik Brueckner * Change the CPUMF state to active. 157212188a5SHendrik Brueckner * Enable and activate the CPU-counter sets according 158212188a5SHendrik Brueckner * to the per-cpu control state. 159212188a5SHendrik Brueckner */ 160212188a5SHendrik Brueckner static void cpumf_pmu_enable(struct pmu *pmu) 161212188a5SHendrik Brueckner { 162eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 163212188a5SHendrik Brueckner int err; 164212188a5SHendrik Brueckner 165212188a5SHendrik Brueckner if (cpuhw->flags & PMU_F_ENABLED) 166212188a5SHendrik Brueckner return; 167212188a5SHendrik Brueckner 168212188a5SHendrik Brueckner err = lcctl(cpuhw->state); 169212188a5SHendrik Brueckner if (err) { 170212188a5SHendrik Brueckner pr_err("Enabling the performance measuring unit " 171af0ee94eSHeiko Carstens "failed with rc=%x\n", err); 172212188a5SHendrik Brueckner return; 173212188a5SHendrik Brueckner } 174212188a5SHendrik Brueckner 175212188a5SHendrik Brueckner cpuhw->flags |= PMU_F_ENABLED; 176212188a5SHendrik Brueckner } 177212188a5SHendrik Brueckner 178212188a5SHendrik Brueckner /* 179212188a5SHendrik Brueckner * Change the CPUMF state to inactive. 180212188a5SHendrik Brueckner * Disable and enable (inactive) the CPU-counter sets according 181212188a5SHendrik Brueckner * to the per-cpu control state. 182212188a5SHendrik Brueckner */ 183212188a5SHendrik Brueckner static void cpumf_pmu_disable(struct pmu *pmu) 184212188a5SHendrik Brueckner { 185eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 186212188a5SHendrik Brueckner int err; 187212188a5SHendrik Brueckner u64 inactive; 188212188a5SHendrik Brueckner 189212188a5SHendrik Brueckner if (!(cpuhw->flags & PMU_F_ENABLED)) 190212188a5SHendrik Brueckner return; 191212188a5SHendrik Brueckner 192212188a5SHendrik Brueckner inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); 193212188a5SHendrik Brueckner err = lcctl(inactive); 194212188a5SHendrik Brueckner if (err) { 195212188a5SHendrik Brueckner pr_err("Disabling the performance measuring unit " 196af0ee94eSHeiko Carstens "failed with rc=%x\n", err); 197212188a5SHendrik Brueckner return; 198212188a5SHendrik Brueckner } 199212188a5SHendrik Brueckner 200212188a5SHendrik Brueckner cpuhw->flags &= ~PMU_F_ENABLED; 201212188a5SHendrik Brueckner } 202212188a5SHendrik Brueckner 203212188a5SHendrik Brueckner 204212188a5SHendrik Brueckner /* Number of perf events counting hardware events */ 205212188a5SHendrik Brueckner static atomic_t num_events = ATOMIC_INIT(0); 206212188a5SHendrik Brueckner /* Used to avoid races in calling reserve/release_cpumf_hardware */ 207212188a5SHendrik Brueckner static DEFINE_MUTEX(pmc_reserve_mutex); 208212188a5SHendrik Brueckner 209212188a5SHendrik Brueckner /* CPU-measurement alerts for the counter facility */ 210212188a5SHendrik Brueckner static void cpumf_measurement_alert(struct ext_code ext_code, 211212188a5SHendrik Brueckner unsigned int alert, unsigned long unused) 212212188a5SHendrik Brueckner { 213212188a5SHendrik Brueckner struct cpu_hw_events *cpuhw; 214212188a5SHendrik Brueckner 215212188a5SHendrik Brueckner if (!(alert & CPU_MF_INT_CF_MASK)) 216212188a5SHendrik Brueckner return; 217212188a5SHendrik Brueckner 218420f42ecSHeiko Carstens inc_irq_stat(IRQEXT_CMC); 219eb7e7d76SChristoph Lameter cpuhw = this_cpu_ptr(&cpu_hw_events); 220212188a5SHendrik Brueckner 221212188a5SHendrik Brueckner /* Measurement alerts are shared and might happen when the PMU 222212188a5SHendrik Brueckner * is not reserved. Ignore these alerts in this case. */ 223212188a5SHendrik Brueckner if (!(cpuhw->flags & PMU_F_RESERVED)) 224212188a5SHendrik Brueckner return; 225212188a5SHendrik Brueckner 226212188a5SHendrik Brueckner /* counter authorization change alert */ 227212188a5SHendrik Brueckner if (alert & CPU_MF_INT_CF_CACA) 228212188a5SHendrik Brueckner qctri(&cpuhw->info); 229212188a5SHendrik Brueckner 230212188a5SHendrik Brueckner /* loss of counter data alert */ 231212188a5SHendrik Brueckner if (alert & CPU_MF_INT_CF_LCDA) 232212188a5SHendrik Brueckner pr_err("CPU[%i] Counter data was lost\n", smp_processor_id()); 233212188a5SHendrik Brueckner } 234212188a5SHendrik Brueckner 235212188a5SHendrik Brueckner #define PMC_INIT 0 236212188a5SHendrik Brueckner #define PMC_RELEASE 1 237212188a5SHendrik Brueckner static void setup_pmc_cpu(void *flags) 238212188a5SHendrik Brueckner { 239eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 240212188a5SHendrik Brueckner 241212188a5SHendrik Brueckner switch (*((int *) flags)) { 242212188a5SHendrik Brueckner case PMC_INIT: 243212188a5SHendrik Brueckner memset(&cpuhw->info, 0, sizeof(cpuhw->info)); 244212188a5SHendrik Brueckner qctri(&cpuhw->info); 245212188a5SHendrik Brueckner cpuhw->flags |= PMU_F_RESERVED; 246212188a5SHendrik Brueckner break; 247212188a5SHendrik Brueckner 248212188a5SHendrik Brueckner case PMC_RELEASE: 249212188a5SHendrik Brueckner cpuhw->flags &= ~PMU_F_RESERVED; 250212188a5SHendrik Brueckner break; 251212188a5SHendrik Brueckner } 252212188a5SHendrik Brueckner 253212188a5SHendrik Brueckner /* Disable CPU counter sets */ 254212188a5SHendrik Brueckner lcctl(0); 255212188a5SHendrik Brueckner } 256212188a5SHendrik Brueckner 257212188a5SHendrik Brueckner /* Initialize the CPU-measurement facility */ 258212188a5SHendrik Brueckner static int reserve_pmc_hardware(void) 259212188a5SHendrik Brueckner { 260212188a5SHendrik Brueckner int flags = PMC_INIT; 261212188a5SHendrik Brueckner 262212188a5SHendrik Brueckner on_each_cpu(setup_pmc_cpu, &flags, 1); 26382003c3eSHeiko Carstens irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); 264212188a5SHendrik Brueckner 265212188a5SHendrik Brueckner return 0; 266212188a5SHendrik Brueckner } 267212188a5SHendrik Brueckner 268212188a5SHendrik Brueckner /* Release the CPU-measurement facility */ 269212188a5SHendrik Brueckner static void release_pmc_hardware(void) 270212188a5SHendrik Brueckner { 271212188a5SHendrik Brueckner int flags = PMC_RELEASE; 272212188a5SHendrik Brueckner 273212188a5SHendrik Brueckner on_each_cpu(setup_pmc_cpu, &flags, 1); 27482003c3eSHeiko Carstens irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); 275212188a5SHendrik Brueckner } 276212188a5SHendrik Brueckner 277212188a5SHendrik Brueckner /* Release the PMU if event is the last perf event */ 278212188a5SHendrik Brueckner static void hw_perf_event_destroy(struct perf_event *event) 279212188a5SHendrik Brueckner { 280212188a5SHendrik Brueckner if (!atomic_add_unless(&num_events, -1, 1)) { 281212188a5SHendrik Brueckner mutex_lock(&pmc_reserve_mutex); 282212188a5SHendrik Brueckner if (atomic_dec_return(&num_events) == 0) 283212188a5SHendrik Brueckner release_pmc_hardware(); 284212188a5SHendrik Brueckner mutex_unlock(&pmc_reserve_mutex); 285212188a5SHendrik Brueckner } 286212188a5SHendrik Brueckner } 287212188a5SHendrik Brueckner 288212188a5SHendrik Brueckner /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */ 289212188a5SHendrik Brueckner static const int cpumf_generic_events_basic[] = { 290212188a5SHendrik Brueckner [PERF_COUNT_HW_CPU_CYCLES] = 0, 291212188a5SHendrik Brueckner [PERF_COUNT_HW_INSTRUCTIONS] = 1, 292212188a5SHendrik Brueckner [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 293212188a5SHendrik Brueckner [PERF_COUNT_HW_CACHE_MISSES] = -1, 294212188a5SHendrik Brueckner [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 295212188a5SHendrik Brueckner [PERF_COUNT_HW_BRANCH_MISSES] = -1, 296212188a5SHendrik Brueckner [PERF_COUNT_HW_BUS_CYCLES] = -1, 297212188a5SHendrik Brueckner }; 298212188a5SHendrik Brueckner /* CPUMF <-> perf event mappings for userspace (problem-state set) */ 299212188a5SHendrik Brueckner static const int cpumf_generic_events_user[] = { 300212188a5SHendrik Brueckner [PERF_COUNT_HW_CPU_CYCLES] = 32, 301212188a5SHendrik Brueckner [PERF_COUNT_HW_INSTRUCTIONS] = 33, 302212188a5SHendrik Brueckner [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 303212188a5SHendrik Brueckner [PERF_COUNT_HW_CACHE_MISSES] = -1, 304212188a5SHendrik Brueckner [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 305212188a5SHendrik Brueckner [PERF_COUNT_HW_BRANCH_MISSES] = -1, 306212188a5SHendrik Brueckner [PERF_COUNT_HW_BUS_CYCLES] = -1, 307212188a5SHendrik Brueckner }; 308212188a5SHendrik Brueckner 309212188a5SHendrik Brueckner static int __hw_perf_event_init(struct perf_event *event) 310212188a5SHendrik Brueckner { 311212188a5SHendrik Brueckner struct perf_event_attr *attr = &event->attr; 312212188a5SHendrik Brueckner struct hw_perf_event *hwc = &event->hw; 313212188a5SHendrik Brueckner int err; 314212188a5SHendrik Brueckner u64 ev; 315212188a5SHendrik Brueckner 316212188a5SHendrik Brueckner switch (attr->type) { 317212188a5SHendrik Brueckner case PERF_TYPE_RAW: 318212188a5SHendrik Brueckner /* Raw events are used to access counters directly, 319212188a5SHendrik Brueckner * hence do not permit excludes */ 320212188a5SHendrik Brueckner if (attr->exclude_kernel || attr->exclude_user || 321212188a5SHendrik Brueckner attr->exclude_hv) 322212188a5SHendrik Brueckner return -EOPNOTSUPP; 323212188a5SHendrik Brueckner ev = attr->config; 324212188a5SHendrik Brueckner break; 325212188a5SHendrik Brueckner 326212188a5SHendrik Brueckner case PERF_TYPE_HARDWARE: 327212188a5SHendrik Brueckner ev = attr->config; 328212188a5SHendrik Brueckner /* Count user space (problem-state) only */ 329212188a5SHendrik Brueckner if (!attr->exclude_user && attr->exclude_kernel) { 330212188a5SHendrik Brueckner if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) 331212188a5SHendrik Brueckner return -EOPNOTSUPP; 332212188a5SHendrik Brueckner ev = cpumf_generic_events_user[ev]; 333212188a5SHendrik Brueckner 334212188a5SHendrik Brueckner /* No support for kernel space counters only */ 335212188a5SHendrik Brueckner } else if (!attr->exclude_kernel && attr->exclude_user) { 336212188a5SHendrik Brueckner return -EOPNOTSUPP; 337212188a5SHendrik Brueckner 338212188a5SHendrik Brueckner /* Count user and kernel space */ 339212188a5SHendrik Brueckner } else { 340212188a5SHendrik Brueckner if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) 341212188a5SHendrik Brueckner return -EOPNOTSUPP; 342212188a5SHendrik Brueckner ev = cpumf_generic_events_basic[ev]; 343212188a5SHendrik Brueckner } 344212188a5SHendrik Brueckner break; 345212188a5SHendrik Brueckner 346212188a5SHendrik Brueckner default: 347212188a5SHendrik Brueckner return -ENOENT; 348212188a5SHendrik Brueckner } 349212188a5SHendrik Brueckner 350212188a5SHendrik Brueckner if (ev == -1) 351212188a5SHendrik Brueckner return -ENOENT; 352212188a5SHendrik Brueckner 35320ba46daSHendrik Brueckner if (ev > PERF_CPUM_CF_MAX_CTR) 354212188a5SHendrik Brueckner return -EINVAL; 355212188a5SHendrik Brueckner 356212188a5SHendrik Brueckner /* Use the hardware perf event structure to store the counter number 357212188a5SHendrik Brueckner * in 'config' member and the counter set to which the counter belongs 358212188a5SHendrik Brueckner * in the 'config_base'. The counter set (config_base) is then used 359212188a5SHendrik Brueckner * to enable/disable the counters. 360212188a5SHendrik Brueckner */ 361212188a5SHendrik Brueckner hwc->config = ev; 362212188a5SHendrik Brueckner hwc->config_base = get_counter_set(ev); 363212188a5SHendrik Brueckner 364212188a5SHendrik Brueckner /* Initialize for using the CPU-measurement counter facility */ 365212188a5SHendrik Brueckner if (!atomic_inc_not_zero(&num_events)) { 366212188a5SHendrik Brueckner mutex_lock(&pmc_reserve_mutex); 367212188a5SHendrik Brueckner if (atomic_read(&num_events) == 0 && reserve_pmc_hardware()) 368212188a5SHendrik Brueckner err = -EBUSY; 369212188a5SHendrik Brueckner else 370212188a5SHendrik Brueckner atomic_inc(&num_events); 371212188a5SHendrik Brueckner mutex_unlock(&pmc_reserve_mutex); 372212188a5SHendrik Brueckner } 373212188a5SHendrik Brueckner event->destroy = hw_perf_event_destroy; 374212188a5SHendrik Brueckner 375212188a5SHendrik Brueckner /* Finally, validate version and authorization of the counter set */ 376212188a5SHendrik Brueckner err = validate_ctr_auth(hwc); 377212188a5SHendrik Brueckner if (!err) 378212188a5SHendrik Brueckner err = validate_ctr_version(hwc); 379212188a5SHendrik Brueckner 380212188a5SHendrik Brueckner return err; 381212188a5SHendrik Brueckner } 382212188a5SHendrik Brueckner 383212188a5SHendrik Brueckner static int cpumf_pmu_event_init(struct perf_event *event) 384212188a5SHendrik Brueckner { 385212188a5SHendrik Brueckner int err; 386212188a5SHendrik Brueckner 387212188a5SHendrik Brueckner switch (event->attr.type) { 388212188a5SHendrik Brueckner case PERF_TYPE_HARDWARE: 389212188a5SHendrik Brueckner case PERF_TYPE_HW_CACHE: 390212188a5SHendrik Brueckner case PERF_TYPE_RAW: 391212188a5SHendrik Brueckner err = __hw_perf_event_init(event); 392212188a5SHendrik Brueckner break; 393212188a5SHendrik Brueckner default: 394212188a5SHendrik Brueckner return -ENOENT; 395212188a5SHendrik Brueckner } 396212188a5SHendrik Brueckner 397212188a5SHendrik Brueckner if (unlikely(err) && event->destroy) 398212188a5SHendrik Brueckner event->destroy(event); 399212188a5SHendrik Brueckner 400212188a5SHendrik Brueckner return err; 401212188a5SHendrik Brueckner } 402212188a5SHendrik Brueckner 403212188a5SHendrik Brueckner static int hw_perf_event_reset(struct perf_event *event) 404212188a5SHendrik Brueckner { 405212188a5SHendrik Brueckner u64 prev, new; 406212188a5SHendrik Brueckner int err; 407212188a5SHendrik Brueckner 408212188a5SHendrik Brueckner do { 409212188a5SHendrik Brueckner prev = local64_read(&event->hw.prev_count); 410212188a5SHendrik Brueckner err = ecctr(event->hw.config, &new); 411212188a5SHendrik Brueckner if (err) { 412212188a5SHendrik Brueckner if (err != 3) 413212188a5SHendrik Brueckner break; 414212188a5SHendrik Brueckner /* The counter is not (yet) available. This 415212188a5SHendrik Brueckner * might happen if the counter set to which 416212188a5SHendrik Brueckner * this counter belongs is in the disabled 417212188a5SHendrik Brueckner * state. 418212188a5SHendrik Brueckner */ 419212188a5SHendrik Brueckner new = 0; 420212188a5SHendrik Brueckner } 421212188a5SHendrik Brueckner } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); 422212188a5SHendrik Brueckner 423212188a5SHendrik Brueckner return err; 424212188a5SHendrik Brueckner } 425212188a5SHendrik Brueckner 426212188a5SHendrik Brueckner static int hw_perf_event_update(struct perf_event *event) 427212188a5SHendrik Brueckner { 428212188a5SHendrik Brueckner u64 prev, new, delta; 429212188a5SHendrik Brueckner int err; 430212188a5SHendrik Brueckner 431212188a5SHendrik Brueckner do { 432212188a5SHendrik Brueckner prev = local64_read(&event->hw.prev_count); 433212188a5SHendrik Brueckner err = ecctr(event->hw.config, &new); 434212188a5SHendrik Brueckner if (err) 435212188a5SHendrik Brueckner goto out; 436212188a5SHendrik Brueckner } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); 437212188a5SHendrik Brueckner 438212188a5SHendrik Brueckner delta = (prev <= new) ? new - prev 439212188a5SHendrik Brueckner : (-1ULL - prev) + new + 1; /* overflow */ 440212188a5SHendrik Brueckner local64_add(delta, &event->count); 441212188a5SHendrik Brueckner out: 442212188a5SHendrik Brueckner return err; 443212188a5SHendrik Brueckner } 444212188a5SHendrik Brueckner 445212188a5SHendrik Brueckner static void cpumf_pmu_read(struct perf_event *event) 446212188a5SHendrik Brueckner { 447212188a5SHendrik Brueckner if (event->hw.state & PERF_HES_STOPPED) 448212188a5SHendrik Brueckner return; 449212188a5SHendrik Brueckner 450212188a5SHendrik Brueckner hw_perf_event_update(event); 451212188a5SHendrik Brueckner } 452212188a5SHendrik Brueckner 453212188a5SHendrik Brueckner static void cpumf_pmu_start(struct perf_event *event, int flags) 454212188a5SHendrik Brueckner { 455eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 456212188a5SHendrik Brueckner struct hw_perf_event *hwc = &event->hw; 457212188a5SHendrik Brueckner 458212188a5SHendrik Brueckner if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 459212188a5SHendrik Brueckner return; 460212188a5SHendrik Brueckner 461212188a5SHendrik Brueckner if (WARN_ON_ONCE(hwc->config == -1)) 462212188a5SHendrik Brueckner return; 463212188a5SHendrik Brueckner 464212188a5SHendrik Brueckner if (flags & PERF_EF_RELOAD) 465212188a5SHendrik Brueckner WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 466212188a5SHendrik Brueckner 467212188a5SHendrik Brueckner hwc->state = 0; 468212188a5SHendrik Brueckner 469212188a5SHendrik Brueckner /* (Re-)enable and activate the counter set */ 470212188a5SHendrik Brueckner ctr_set_enable(&cpuhw->state, hwc->config_base); 471212188a5SHendrik Brueckner ctr_set_start(&cpuhw->state, hwc->config_base); 472212188a5SHendrik Brueckner 473212188a5SHendrik Brueckner /* The counter set to which this counter belongs can be already active. 474212188a5SHendrik Brueckner * Because all counters in a set are active, the event->hw.prev_count 475212188a5SHendrik Brueckner * needs to be synchronized. At this point, the counter set can be in 476212188a5SHendrik Brueckner * the inactive or disabled state. 477212188a5SHendrik Brueckner */ 478212188a5SHendrik Brueckner hw_perf_event_reset(event); 479212188a5SHendrik Brueckner 480212188a5SHendrik Brueckner /* increment refcount for this counter set */ 481212188a5SHendrik Brueckner atomic_inc(&cpuhw->ctr_set[hwc->config_base]); 482212188a5SHendrik Brueckner } 483212188a5SHendrik Brueckner 484212188a5SHendrik Brueckner static void cpumf_pmu_stop(struct perf_event *event, int flags) 485212188a5SHendrik Brueckner { 486eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 487212188a5SHendrik Brueckner struct hw_perf_event *hwc = &event->hw; 488212188a5SHendrik Brueckner 489212188a5SHendrik Brueckner if (!(hwc->state & PERF_HES_STOPPED)) { 490212188a5SHendrik Brueckner /* Decrement reference count for this counter set and if this 491212188a5SHendrik Brueckner * is the last used counter in the set, clear activation 492212188a5SHendrik Brueckner * control and set the counter set state to inactive. 493212188a5SHendrik Brueckner */ 494212188a5SHendrik Brueckner if (!atomic_dec_return(&cpuhw->ctr_set[hwc->config_base])) 495212188a5SHendrik Brueckner ctr_set_stop(&cpuhw->state, hwc->config_base); 496212188a5SHendrik Brueckner event->hw.state |= PERF_HES_STOPPED; 497212188a5SHendrik Brueckner } 498212188a5SHendrik Brueckner 499212188a5SHendrik Brueckner if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 500212188a5SHendrik Brueckner hw_perf_event_update(event); 501212188a5SHendrik Brueckner event->hw.state |= PERF_HES_UPTODATE; 502212188a5SHendrik Brueckner } 503212188a5SHendrik Brueckner } 504212188a5SHendrik Brueckner 505212188a5SHendrik Brueckner static int cpumf_pmu_add(struct perf_event *event, int flags) 506212188a5SHendrik Brueckner { 507eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 508212188a5SHendrik Brueckner 509212188a5SHendrik Brueckner /* Check authorization for the counter set to which this 510212188a5SHendrik Brueckner * counter belongs. 511212188a5SHendrik Brueckner * For group events transaction, the authorization check is 512212188a5SHendrik Brueckner * done in cpumf_pmu_commit_txn(). 513212188a5SHendrik Brueckner */ 5148f3e5684SSukadev Bhattiprolu if (!(cpuhw->txn_flags & PERF_PMU_TXN_ADD)) 515212188a5SHendrik Brueckner if (validate_ctr_auth(&event->hw)) 51658f8e9daSHendrik Brueckner return -ENOENT; 517212188a5SHendrik Brueckner 518212188a5SHendrik Brueckner ctr_set_enable(&cpuhw->state, event->hw.config_base); 519212188a5SHendrik Brueckner event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 520212188a5SHendrik Brueckner 521212188a5SHendrik Brueckner if (flags & PERF_EF_START) 522212188a5SHendrik Brueckner cpumf_pmu_start(event, PERF_EF_RELOAD); 523212188a5SHendrik Brueckner 524212188a5SHendrik Brueckner perf_event_update_userpage(event); 525212188a5SHendrik Brueckner 526212188a5SHendrik Brueckner return 0; 527212188a5SHendrik Brueckner } 528212188a5SHendrik Brueckner 529212188a5SHendrik Brueckner static void cpumf_pmu_del(struct perf_event *event, int flags) 530212188a5SHendrik Brueckner { 531eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 532212188a5SHendrik Brueckner 533212188a5SHendrik Brueckner cpumf_pmu_stop(event, PERF_EF_UPDATE); 534212188a5SHendrik Brueckner 535212188a5SHendrik Brueckner /* Check if any counter in the counter set is still used. If not used, 536212188a5SHendrik Brueckner * change the counter set to the disabled state. This also clears the 537212188a5SHendrik Brueckner * content of all counters in the set. 538212188a5SHendrik Brueckner * 539212188a5SHendrik Brueckner * When a new perf event has been added but not yet started, this can 540212188a5SHendrik Brueckner * clear enable control and resets all counters in a set. Therefore, 541212188a5SHendrik Brueckner * cpumf_pmu_start() always has to reenable a counter set. 542212188a5SHendrik Brueckner */ 543212188a5SHendrik Brueckner if (!atomic_read(&cpuhw->ctr_set[event->hw.config_base])) 544212188a5SHendrik Brueckner ctr_set_disable(&cpuhw->state, event->hw.config_base); 545212188a5SHendrik Brueckner 546212188a5SHendrik Brueckner perf_event_update_userpage(event); 547212188a5SHendrik Brueckner } 548212188a5SHendrik Brueckner 549212188a5SHendrik Brueckner /* 550212188a5SHendrik Brueckner * Start group events scheduling transaction. 551212188a5SHendrik Brueckner * Set flags to perform a single test at commit time. 552fbbe0701SSukadev Bhattiprolu * 553fbbe0701SSukadev Bhattiprolu * We only support PERF_PMU_TXN_ADD transactions. Save the 554fbbe0701SSukadev Bhattiprolu * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD 555fbbe0701SSukadev Bhattiprolu * transactions. 556212188a5SHendrik Brueckner */ 557fbbe0701SSukadev Bhattiprolu static void cpumf_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) 558212188a5SHendrik Brueckner { 559eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 560212188a5SHendrik Brueckner 561fbbe0701SSukadev Bhattiprolu WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ 562fbbe0701SSukadev Bhattiprolu 563fbbe0701SSukadev Bhattiprolu cpuhw->txn_flags = txn_flags; 564fbbe0701SSukadev Bhattiprolu if (txn_flags & ~PERF_PMU_TXN_ADD) 565fbbe0701SSukadev Bhattiprolu return; 566fbbe0701SSukadev Bhattiprolu 567212188a5SHendrik Brueckner perf_pmu_disable(pmu); 568212188a5SHendrik Brueckner cpuhw->tx_state = cpuhw->state; 569212188a5SHendrik Brueckner } 570212188a5SHendrik Brueckner 571212188a5SHendrik Brueckner /* 572212188a5SHendrik Brueckner * Stop and cancel a group events scheduling tranctions. 573212188a5SHendrik Brueckner * Assumes cpumf_pmu_del() is called for each successful added 574212188a5SHendrik Brueckner * cpumf_pmu_add() during the transaction. 575212188a5SHendrik Brueckner */ 576212188a5SHendrik Brueckner static void cpumf_pmu_cancel_txn(struct pmu *pmu) 577212188a5SHendrik Brueckner { 578fbbe0701SSukadev Bhattiprolu unsigned int txn_flags; 579eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 580212188a5SHendrik Brueckner 581fbbe0701SSukadev Bhattiprolu WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ 582fbbe0701SSukadev Bhattiprolu 583fbbe0701SSukadev Bhattiprolu txn_flags = cpuhw->txn_flags; 584fbbe0701SSukadev Bhattiprolu cpuhw->txn_flags = 0; 585fbbe0701SSukadev Bhattiprolu if (txn_flags & ~PERF_PMU_TXN_ADD) 586fbbe0701SSukadev Bhattiprolu return; 587fbbe0701SSukadev Bhattiprolu 588212188a5SHendrik Brueckner WARN_ON(cpuhw->tx_state != cpuhw->state); 589212188a5SHendrik Brueckner 590212188a5SHendrik Brueckner perf_pmu_enable(pmu); 591212188a5SHendrik Brueckner } 592212188a5SHendrik Brueckner 593212188a5SHendrik Brueckner /* 594212188a5SHendrik Brueckner * Commit the group events scheduling transaction. On success, the 595212188a5SHendrik Brueckner * transaction is closed. On error, the transaction is kept open 596212188a5SHendrik Brueckner * until cpumf_pmu_cancel_txn() is called. 597212188a5SHendrik Brueckner */ 598212188a5SHendrik Brueckner static int cpumf_pmu_commit_txn(struct pmu *pmu) 599212188a5SHendrik Brueckner { 600eb7e7d76SChristoph Lameter struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); 601212188a5SHendrik Brueckner u64 state; 602212188a5SHendrik Brueckner 603fbbe0701SSukadev Bhattiprolu WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ 604fbbe0701SSukadev Bhattiprolu 605fbbe0701SSukadev Bhattiprolu if (cpuhw->txn_flags & ~PERF_PMU_TXN_ADD) { 606fbbe0701SSukadev Bhattiprolu cpuhw->txn_flags = 0; 607fbbe0701SSukadev Bhattiprolu return 0; 608fbbe0701SSukadev Bhattiprolu } 609fbbe0701SSukadev Bhattiprolu 610212188a5SHendrik Brueckner /* check if the updated state can be scheduled */ 611212188a5SHendrik Brueckner state = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); 612212188a5SHendrik Brueckner state >>= CPUMF_LCCTL_ENABLE_SHIFT; 613212188a5SHendrik Brueckner if ((state & cpuhw->info.auth_ctl) != state) 61458f8e9daSHendrik Brueckner return -ENOENT; 615212188a5SHendrik Brueckner 616fbbe0701SSukadev Bhattiprolu cpuhw->txn_flags = 0; 617212188a5SHendrik Brueckner perf_pmu_enable(pmu); 618212188a5SHendrik Brueckner return 0; 619212188a5SHendrik Brueckner } 620212188a5SHendrik Brueckner 621212188a5SHendrik Brueckner /* Performance monitoring unit for s390x */ 622212188a5SHendrik Brueckner static struct pmu cpumf_pmu = { 6239254e70cSHendrik Brueckner .task_ctx_nr = perf_sw_context, 6249254e70cSHendrik Brueckner .capabilities = PERF_PMU_CAP_NO_INTERRUPT, 625212188a5SHendrik Brueckner .pmu_enable = cpumf_pmu_enable, 626212188a5SHendrik Brueckner .pmu_disable = cpumf_pmu_disable, 627212188a5SHendrik Brueckner .event_init = cpumf_pmu_event_init, 628212188a5SHendrik Brueckner .add = cpumf_pmu_add, 629212188a5SHendrik Brueckner .del = cpumf_pmu_del, 630212188a5SHendrik Brueckner .start = cpumf_pmu_start, 631212188a5SHendrik Brueckner .stop = cpumf_pmu_stop, 632212188a5SHendrik Brueckner .read = cpumf_pmu_read, 633212188a5SHendrik Brueckner .start_txn = cpumf_pmu_start_txn, 634212188a5SHendrik Brueckner .commit_txn = cpumf_pmu_commit_txn, 635212188a5SHendrik Brueckner .cancel_txn = cpumf_pmu_cancel_txn, 636212188a5SHendrik Brueckner }; 637212188a5SHendrik Brueckner 6384f0f8217SThomas Gleixner static int cpumf_pmf_setup(unsigned int cpu, int flags) 639212188a5SHendrik Brueckner { 6405bc73539SAnna-Maria Gleixner local_irq_disable(); 6415bc73539SAnna-Maria Gleixner setup_pmc_cpu(&flags); 6425bc73539SAnna-Maria Gleixner local_irq_enable(); 6434f0f8217SThomas Gleixner return 0; 644212188a5SHendrik Brueckner } 645212188a5SHendrik Brueckner 6464f0f8217SThomas Gleixner static int s390_pmu_online_cpu(unsigned int cpu) 6474f0f8217SThomas Gleixner { 6484f0f8217SThomas Gleixner return cpumf_pmf_setup(cpu, PMC_INIT); 6494f0f8217SThomas Gleixner } 6504f0f8217SThomas Gleixner 6514f0f8217SThomas Gleixner static int s390_pmu_offline_cpu(unsigned int cpu) 6524f0f8217SThomas Gleixner { 6534f0f8217SThomas Gleixner return cpumf_pmf_setup(cpu, PMC_RELEASE); 654212188a5SHendrik Brueckner } 655212188a5SHendrik Brueckner 656212188a5SHendrik Brueckner static int __init cpumf_pmu_init(void) 657212188a5SHendrik Brueckner { 658212188a5SHendrik Brueckner int rc; 659212188a5SHendrik Brueckner 660212188a5SHendrik Brueckner if (!cpum_cf_avail()) 661212188a5SHendrik Brueckner return -ENODEV; 662212188a5SHendrik Brueckner 663212188a5SHendrik Brueckner /* clear bit 15 of cr0 to unauthorize problem-state to 664212188a5SHendrik Brueckner * extract measurement counters */ 665212188a5SHendrik Brueckner ctl_clear_bit(0, 48); 666212188a5SHendrik Brueckner 667212188a5SHendrik Brueckner /* register handler for measurement-alert interruptions */ 6681dad093bSThomas Huth rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, 6691dad093bSThomas Huth cpumf_measurement_alert); 670212188a5SHendrik Brueckner if (rc) { 671212188a5SHendrik Brueckner pr_err("Registering for CPU-measurement alerts " 672212188a5SHendrik Brueckner "failed with rc=%i\n", rc); 6734f0f8217SThomas Gleixner return rc; 674212188a5SHendrik Brueckner } 675212188a5SHendrik Brueckner 676c7168325SHendrik Brueckner cpumf_pmu.attr_groups = cpumf_cf_event_group(); 677212188a5SHendrik Brueckner rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW); 678212188a5SHendrik Brueckner if (rc) { 679212188a5SHendrik Brueckner pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc); 6801dad093bSThomas Huth unregister_external_irq(EXT_IRQ_MEASURE_ALERT, 6811dad093bSThomas Huth cpumf_measurement_alert); 682212188a5SHendrik Brueckner return rc; 683212188a5SHendrik Brueckner } 6844f0f8217SThomas Gleixner return cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, 68573c1b41eSThomas Gleixner "perf/s390/cf:online", 6864f0f8217SThomas Gleixner s390_pmu_online_cpu, s390_pmu_offline_cpu); 6874f0f8217SThomas Gleixner } 688212188a5SHendrik Brueckner early_initcall(cpumf_pmu_init); 689