1*5d7107c7SJonathan Cameron // SPDX-License-Identifier: GPL-2.0-only 2*5d7107c7SJonathan Cameron 3*5d7107c7SJonathan Cameron /* 4*5d7107c7SJonathan Cameron * Copyright(c) 2023 Huawei 5*5d7107c7SJonathan Cameron * 6*5d7107c7SJonathan Cameron * The CXL 3.0 specification includes a standard Performance Monitoring Unit, 7*5d7107c7SJonathan Cameron * called the CXL PMU, or CPMU. In order to allow a high degree of 8*5d7107c7SJonathan Cameron * implementation flexibility the specification provides a wide range of 9*5d7107c7SJonathan Cameron * options all of which are self describing. 10*5d7107c7SJonathan Cameron * 11*5d7107c7SJonathan Cameron * Details in CXL rev 3.0 section 8.2.7 CPMU Register Interface 12*5d7107c7SJonathan Cameron */ 13*5d7107c7SJonathan Cameron 14*5d7107c7SJonathan Cameron #include <linux/io-64-nonatomic-lo-hi.h> 15*5d7107c7SJonathan Cameron #include <linux/perf_event.h> 16*5d7107c7SJonathan Cameron #include <linux/bitops.h> 17*5d7107c7SJonathan Cameron #include <linux/device.h> 18*5d7107c7SJonathan Cameron #include <linux/bits.h> 19*5d7107c7SJonathan Cameron #include <linux/list.h> 20*5d7107c7SJonathan Cameron #include <linux/bug.h> 21*5d7107c7SJonathan Cameron #include <linux/pci.h> 22*5d7107c7SJonathan Cameron 23*5d7107c7SJonathan Cameron #include "../cxl/cxlpci.h" 24*5d7107c7SJonathan Cameron #include "../cxl/cxl.h" 25*5d7107c7SJonathan Cameron #include "../cxl/pmu.h" 26*5d7107c7SJonathan Cameron 27*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_REG 0x0 28*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_NUM_COUNTERS_MSK GENMASK_ULL(4, 0) 29*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_COUNTER_WIDTH_MSK GENMASK_ULL(15, 8) 30*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK GENMASK_ULL(24, 20) 31*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_FILTERS_SUP_MSK GENMASK_ULL(39, 32) 32*5d7107c7SJonathan Cameron #define CXL_PMU_FILTER_HDM BIT(0) 33*5d7107c7SJonathan Cameron #define CXL_PMU_FILTER_CHAN_RANK_BANK BIT(1) 34*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_MSI_N_MSK GENMASK_ULL(47, 44) 35*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN BIT_ULL(48) 36*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_FREEZE BIT_ULL(49) 37*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_INT BIT_ULL(50) 38*5d7107c7SJonathan Cameron #define CXL_PMU_CAP_VERSION_MSK GENMASK_ULL(63, 60) 39*5d7107c7SJonathan Cameron 40*5d7107c7SJonathan Cameron #define CXL_PMU_OVERFLOW_REG 0x10 41*5d7107c7SJonathan Cameron #define CXL_PMU_FREEZE_REG 0x18 42*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_CAP_REG(n) (0x100 + 8 * (n)) 43*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK GENMASK_ULL(31, 0) 44*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_CAP_GROUP_ID_MSK GENMASK_ULL(47, 32) 45*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_CAP_VENDOR_ID_MSK GENMASK_ULL(63, 48) 46*5d7107c7SJonathan Cameron 47*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_REG(n) (0x200 + 8 * (n)) 48*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_TYPE_MSK GENMASK_ULL(1, 0) 49*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_TYPE_FREE_RUN 0 50*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN 1 51*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE 2 52*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_ENABLE BIT_ULL(8) 53*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW BIT_ULL(9) 54*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW BIT_ULL(10) 55*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_EDGE BIT_ULL(11) 56*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_INVERT BIT_ULL(12) 57*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_THRESHOLD_MSK GENMASK_ULL(23, 16) 58*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_EVENTS_MSK GENMASK_ULL(55, 24) 59*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK GENMASK_ULL(63, 59) 60*5d7107c7SJonathan Cameron 61*5d7107c7SJonathan Cameron #define CXL_PMU_FILTER_CFG_REG(n, f) (0x400 + 4 * ((f) + (n) * 8)) 62*5d7107c7SJonathan Cameron #define CXL_PMU_FILTER_CFG_VALUE_MSK GENMASK(15, 0) 63*5d7107c7SJonathan Cameron 64*5d7107c7SJonathan Cameron #define CXL_PMU_COUNTER_REG(n) (0xc00 + 8 * (n)) 65*5d7107c7SJonathan Cameron 66*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 13-5 Events under CXL Vendor ID */ 67*5d7107c7SJonathan Cameron #define CXL_PMU_GID_CLOCK_TICKS 0x00 68*5d7107c7SJonathan Cameron #define CXL_PMU_GID_D2H_REQ 0x0010 69*5d7107c7SJonathan Cameron #define CXL_PMU_GID_D2H_RSP 0x0011 70*5d7107c7SJonathan Cameron #define CXL_PMU_GID_H2D_REQ 0x0012 71*5d7107c7SJonathan Cameron #define CXL_PMU_GID_H2D_RSP 0x0013 72*5d7107c7SJonathan Cameron #define CXL_PMU_GID_CACHE_DATA 0x0014 73*5d7107c7SJonathan Cameron #define CXL_PMU_GID_M2S_REQ 0x0020 74*5d7107c7SJonathan Cameron #define CXL_PMU_GID_M2S_RWD 0x0021 75*5d7107c7SJonathan Cameron #define CXL_PMU_GID_M2S_BIRSP 0x0022 76*5d7107c7SJonathan Cameron #define CXL_PMU_GID_S2M_BISNP 0x0023 77*5d7107c7SJonathan Cameron #define CXL_PMU_GID_S2M_NDR 0x0024 78*5d7107c7SJonathan Cameron #define CXL_PMU_GID_S2M_DRS 0x0025 79*5d7107c7SJonathan Cameron #define CXL_PMU_GID_DDR 0x8000 80*5d7107c7SJonathan Cameron 81*5d7107c7SJonathan Cameron static int cxl_pmu_cpuhp_state_num; 82*5d7107c7SJonathan Cameron 83*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap { 84*5d7107c7SJonathan Cameron u16 vid; 85*5d7107c7SJonathan Cameron u16 gid; 86*5d7107c7SJonathan Cameron u32 msk; 87*5d7107c7SJonathan Cameron union { 88*5d7107c7SJonathan Cameron int counter_idx; /* fixed counters */ 89*5d7107c7SJonathan Cameron int event_idx; /* configurable counters */ 90*5d7107c7SJonathan Cameron }; 91*5d7107c7SJonathan Cameron struct list_head node; 92*5d7107c7SJonathan Cameron }; 93*5d7107c7SJonathan Cameron 94*5d7107c7SJonathan Cameron #define CXL_PMU_MAX_COUNTERS 64 95*5d7107c7SJonathan Cameron struct cxl_pmu_info { 96*5d7107c7SJonathan Cameron struct pmu pmu; 97*5d7107c7SJonathan Cameron void __iomem *base; 98*5d7107c7SJonathan Cameron struct perf_event **hw_events; 99*5d7107c7SJonathan Cameron struct list_head event_caps_configurable; 100*5d7107c7SJonathan Cameron struct list_head event_caps_fixed; 101*5d7107c7SJonathan Cameron DECLARE_BITMAP(used_counter_bm, CXL_PMU_MAX_COUNTERS); 102*5d7107c7SJonathan Cameron DECLARE_BITMAP(conf_counter_bm, CXL_PMU_MAX_COUNTERS); 103*5d7107c7SJonathan Cameron u16 counter_width; 104*5d7107c7SJonathan Cameron u8 num_counters; 105*5d7107c7SJonathan Cameron u8 num_event_capabilities; 106*5d7107c7SJonathan Cameron int on_cpu; 107*5d7107c7SJonathan Cameron struct hlist_node node; 108*5d7107c7SJonathan Cameron bool filter_hdm; 109*5d7107c7SJonathan Cameron int irq; 110*5d7107c7SJonathan Cameron }; 111*5d7107c7SJonathan Cameron 112*5d7107c7SJonathan Cameron #define pmu_to_cxl_pmu_info(_pmu) container_of(_pmu, struct cxl_pmu_info, pmu) 113*5d7107c7SJonathan Cameron 114*5d7107c7SJonathan Cameron /* 115*5d7107c7SJonathan Cameron * All CPMU counters are discoverable via the Event Capabilities Registers. 116*5d7107c7SJonathan Cameron * Each Event Capability register contains a a VID / GroupID. 117*5d7107c7SJonathan Cameron * A counter may then count any combination (by summing) of events in 118*5d7107c7SJonathan Cameron * that group which are in the Supported Events Bitmask. 119*5d7107c7SJonathan Cameron * However, there are some complexities to the scheme. 120*5d7107c7SJonathan Cameron * - Fixed function counters refer to an Event Capabilities register. 121*5d7107c7SJonathan Cameron * That event capability register is not then used for Configurable 122*5d7107c7SJonathan Cameron * counters. 123*5d7107c7SJonathan Cameron */ 124*5d7107c7SJonathan Cameron static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) 125*5d7107c7SJonathan Cameron { 126*5d7107c7SJonathan Cameron unsigned long fixed_counter_event_cap_bm = 0; 127*5d7107c7SJonathan Cameron void __iomem *base = info->base; 128*5d7107c7SJonathan Cameron bool freeze_for_enable; 129*5d7107c7SJonathan Cameron u64 val, eval; 130*5d7107c7SJonathan Cameron int i; 131*5d7107c7SJonathan Cameron 132*5d7107c7SJonathan Cameron val = readq(base + CXL_PMU_CAP_REG); 133*5d7107c7SJonathan Cameron freeze_for_enable = FIELD_GET(CXL_PMU_CAP_WRITEABLE_WHEN_FROZEN, val) && 134*5d7107c7SJonathan Cameron FIELD_GET(CXL_PMU_CAP_FREEZE, val); 135*5d7107c7SJonathan Cameron if (!freeze_for_enable) { 136*5d7107c7SJonathan Cameron dev_err(dev, "Counters not writable while frozen\n"); 137*5d7107c7SJonathan Cameron return -ENODEV; 138*5d7107c7SJonathan Cameron } 139*5d7107c7SJonathan Cameron 140*5d7107c7SJonathan Cameron info->num_counters = FIELD_GET(CXL_PMU_CAP_NUM_COUNTERS_MSK, val) + 1; 141*5d7107c7SJonathan Cameron info->counter_width = FIELD_GET(CXL_PMU_CAP_COUNTER_WIDTH_MSK, val); 142*5d7107c7SJonathan Cameron info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; 143*5d7107c7SJonathan Cameron 144*5d7107c7SJonathan Cameron info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; 145*5d7107c7SJonathan Cameron if (FIELD_GET(CXL_PMU_CAP_INT, val)) 146*5d7107c7SJonathan Cameron info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); 147*5d7107c7SJonathan Cameron else 148*5d7107c7SJonathan Cameron info->irq = -1; 149*5d7107c7SJonathan Cameron 150*5d7107c7SJonathan Cameron /* First handle fixed function counters; note if configurable counters found */ 151*5d7107c7SJonathan Cameron for (i = 0; i < info->num_counters; i++) { 152*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap *pmu_ev; 153*5d7107c7SJonathan Cameron u32 events_msk; 154*5d7107c7SJonathan Cameron u8 group_idx; 155*5d7107c7SJonathan Cameron 156*5d7107c7SJonathan Cameron val = readq(base + CXL_PMU_COUNTER_CFG_REG(i)); 157*5d7107c7SJonathan Cameron 158*5d7107c7SJonathan Cameron if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) == 159*5d7107c7SJonathan Cameron CXL_PMU_COUNTER_CFG_TYPE_CONFIGURABLE) { 160*5d7107c7SJonathan Cameron set_bit(i, info->conf_counter_bm); 161*5d7107c7SJonathan Cameron } 162*5d7107c7SJonathan Cameron 163*5d7107c7SJonathan Cameron if (FIELD_GET(CXL_PMU_COUNTER_CFG_TYPE_MSK, val) != 164*5d7107c7SJonathan Cameron CXL_PMU_COUNTER_CFG_TYPE_FIXED_FUN) 165*5d7107c7SJonathan Cameron continue; 166*5d7107c7SJonathan Cameron 167*5d7107c7SJonathan Cameron /* In this case we know which fields are const */ 168*5d7107c7SJonathan Cameron group_idx = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, val); 169*5d7107c7SJonathan Cameron events_msk = FIELD_GET(CXL_PMU_COUNTER_CFG_EVENTS_MSK, val); 170*5d7107c7SJonathan Cameron eval = readq(base + CXL_PMU_EVENT_CAP_REG(group_idx)); 171*5d7107c7SJonathan Cameron pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 172*5d7107c7SJonathan Cameron if (!pmu_ev) 173*5d7107c7SJonathan Cameron return -ENOMEM; 174*5d7107c7SJonathan Cameron 175*5d7107c7SJonathan Cameron pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 176*5d7107c7SJonathan Cameron pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 177*5d7107c7SJonathan Cameron /* For a fixed purpose counter use the events mask from the counter CFG */ 178*5d7107c7SJonathan Cameron pmu_ev->msk = events_msk; 179*5d7107c7SJonathan Cameron pmu_ev->counter_idx = i; 180*5d7107c7SJonathan Cameron /* This list add is never unwound as all entries deleted on remove */ 181*5d7107c7SJonathan Cameron list_add(&pmu_ev->node, &info->event_caps_fixed); 182*5d7107c7SJonathan Cameron /* 183*5d7107c7SJonathan Cameron * Configurable counters must not use an Event Capability registers that 184*5d7107c7SJonathan Cameron * is in use for a Fixed counter 185*5d7107c7SJonathan Cameron */ 186*5d7107c7SJonathan Cameron set_bit(group_idx, &fixed_counter_event_cap_bm); 187*5d7107c7SJonathan Cameron } 188*5d7107c7SJonathan Cameron 189*5d7107c7SJonathan Cameron if (!bitmap_empty(info->conf_counter_bm, CXL_PMU_MAX_COUNTERS)) { 190*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap *pmu_ev; 191*5d7107c7SJonathan Cameron int j; 192*5d7107c7SJonathan Cameron /* Walk event capabilities unused by fixed counters */ 193*5d7107c7SJonathan Cameron for_each_clear_bit(j, &fixed_counter_event_cap_bm, 194*5d7107c7SJonathan Cameron info->num_event_capabilities) { 195*5d7107c7SJonathan Cameron pmu_ev = devm_kzalloc(dev, sizeof(*pmu_ev), GFP_KERNEL); 196*5d7107c7SJonathan Cameron if (!pmu_ev) 197*5d7107c7SJonathan Cameron return -ENOMEM; 198*5d7107c7SJonathan Cameron 199*5d7107c7SJonathan Cameron eval = readq(base + CXL_PMU_EVENT_CAP_REG(j)); 200*5d7107c7SJonathan Cameron pmu_ev->vid = FIELD_GET(CXL_PMU_EVENT_CAP_VENDOR_ID_MSK, eval); 201*5d7107c7SJonathan Cameron pmu_ev->gid = FIELD_GET(CXL_PMU_EVENT_CAP_GROUP_ID_MSK, eval); 202*5d7107c7SJonathan Cameron pmu_ev->msk = FIELD_GET(CXL_PMU_EVENT_CAP_SUPPORTED_EVENTS_MSK, eval); 203*5d7107c7SJonathan Cameron pmu_ev->event_idx = j; 204*5d7107c7SJonathan Cameron list_add(&pmu_ev->node, &info->event_caps_configurable); 205*5d7107c7SJonathan Cameron } 206*5d7107c7SJonathan Cameron } 207*5d7107c7SJonathan Cameron 208*5d7107c7SJonathan Cameron return 0; 209*5d7107c7SJonathan Cameron } 210*5d7107c7SJonathan Cameron 211*5d7107c7SJonathan Cameron static ssize_t cxl_pmu_format_sysfs_show(struct device *dev, 212*5d7107c7SJonathan Cameron struct device_attribute *attr, char *buf) 213*5d7107c7SJonathan Cameron { 214*5d7107c7SJonathan Cameron struct dev_ext_attribute *eattr; 215*5d7107c7SJonathan Cameron 216*5d7107c7SJonathan Cameron eattr = container_of(attr, struct dev_ext_attribute, attr); 217*5d7107c7SJonathan Cameron 218*5d7107c7SJonathan Cameron return sysfs_emit(buf, "%s\n", (char *)eattr->var); 219*5d7107c7SJonathan Cameron } 220*5d7107c7SJonathan Cameron 221*5d7107c7SJonathan Cameron #define CXL_PMU_FORMAT_ATTR(_name, _format)\ 222*5d7107c7SJonathan Cameron (&((struct dev_ext_attribute[]) { \ 223*5d7107c7SJonathan Cameron { \ 224*5d7107c7SJonathan Cameron .attr = __ATTR(_name, 0444, \ 225*5d7107c7SJonathan Cameron cxl_pmu_format_sysfs_show, NULL), \ 226*5d7107c7SJonathan Cameron .var = (void *)_format \ 227*5d7107c7SJonathan Cameron } \ 228*5d7107c7SJonathan Cameron })[0].attr.attr) 229*5d7107c7SJonathan Cameron 230*5d7107c7SJonathan Cameron enum { 231*5d7107c7SJonathan Cameron cxl_pmu_mask_attr, 232*5d7107c7SJonathan Cameron cxl_pmu_gid_attr, 233*5d7107c7SJonathan Cameron cxl_pmu_vid_attr, 234*5d7107c7SJonathan Cameron cxl_pmu_threshold_attr, 235*5d7107c7SJonathan Cameron cxl_pmu_invert_attr, 236*5d7107c7SJonathan Cameron cxl_pmu_edge_attr, 237*5d7107c7SJonathan Cameron cxl_pmu_hdm_filter_en_attr, 238*5d7107c7SJonathan Cameron cxl_pmu_hdm_attr, 239*5d7107c7SJonathan Cameron }; 240*5d7107c7SJonathan Cameron 241*5d7107c7SJonathan Cameron static struct attribute *cxl_pmu_format_attr[] = { 242*5d7107c7SJonathan Cameron [cxl_pmu_mask_attr] = CXL_PMU_FORMAT_ATTR(mask, "config:0-31"), 243*5d7107c7SJonathan Cameron [cxl_pmu_gid_attr] = CXL_PMU_FORMAT_ATTR(gid, "config:32-47"), 244*5d7107c7SJonathan Cameron [cxl_pmu_vid_attr] = CXL_PMU_FORMAT_ATTR(vid, "config:48-63"), 245*5d7107c7SJonathan Cameron [cxl_pmu_threshold_attr] = CXL_PMU_FORMAT_ATTR(threshold, "config1:0-15"), 246*5d7107c7SJonathan Cameron [cxl_pmu_invert_attr] = CXL_PMU_FORMAT_ATTR(invert, "config1:16"), 247*5d7107c7SJonathan Cameron [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), 248*5d7107c7SJonathan Cameron [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), 249*5d7107c7SJonathan Cameron [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), 250*5d7107c7SJonathan Cameron NULL 251*5d7107c7SJonathan Cameron }; 252*5d7107c7SJonathan Cameron 253*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG_MASK_MSK GENMASK_ULL(31, 0) 254*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG_GID_MSK GENMASK_ULL(47, 32) 255*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG_VID_MSK GENMASK_ULL(63, 48) 256*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK GENMASK_ULL(15, 0) 257*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) 258*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) 259*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) 260*5d7107c7SJonathan Cameron #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) 261*5d7107c7SJonathan Cameron 262*5d7107c7SJonathan Cameron static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, 263*5d7107c7SJonathan Cameron struct attribute *attr, int a) 264*5d7107c7SJonathan Cameron { 265*5d7107c7SJonathan Cameron struct device *dev = kobj_to_dev(kobj); 266*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = dev_get_drvdata(dev); 267*5d7107c7SJonathan Cameron 268*5d7107c7SJonathan Cameron /* 269*5d7107c7SJonathan Cameron * Filter capability at the CPMU level, so hide the attributes if the particular 270*5d7107c7SJonathan Cameron * filter is not supported. 271*5d7107c7SJonathan Cameron */ 272*5d7107c7SJonathan Cameron if (!info->filter_hdm && 273*5d7107c7SJonathan Cameron (attr == cxl_pmu_format_attr[cxl_pmu_hdm_filter_en_attr] || 274*5d7107c7SJonathan Cameron attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) 275*5d7107c7SJonathan Cameron return 0; 276*5d7107c7SJonathan Cameron 277*5d7107c7SJonathan Cameron return attr->mode; 278*5d7107c7SJonathan Cameron } 279*5d7107c7SJonathan Cameron 280*5d7107c7SJonathan Cameron static const struct attribute_group cxl_pmu_format_group = { 281*5d7107c7SJonathan Cameron .name = "format", 282*5d7107c7SJonathan Cameron .attrs = cxl_pmu_format_attr, 283*5d7107c7SJonathan Cameron .is_visible = cxl_pmu_format_is_visible, 284*5d7107c7SJonathan Cameron }; 285*5d7107c7SJonathan Cameron 286*5d7107c7SJonathan Cameron static u32 cxl_pmu_config_get_mask(struct perf_event *event) 287*5d7107c7SJonathan Cameron { 288*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, event->attr.config); 289*5d7107c7SJonathan Cameron } 290*5d7107c7SJonathan Cameron 291*5d7107c7SJonathan Cameron static u16 cxl_pmu_config_get_gid(struct perf_event *event) 292*5d7107c7SJonathan Cameron { 293*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, event->attr.config); 294*5d7107c7SJonathan Cameron } 295*5d7107c7SJonathan Cameron 296*5d7107c7SJonathan Cameron static u16 cxl_pmu_config_get_vid(struct perf_event *event) 297*5d7107c7SJonathan Cameron { 298*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, event->attr.config); 299*5d7107c7SJonathan Cameron } 300*5d7107c7SJonathan Cameron 301*5d7107c7SJonathan Cameron static u8 cxl_pmu_config1_get_threshold(struct perf_event *event) 302*5d7107c7SJonathan Cameron { 303*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG1_THRESHOLD_MSK, event->attr.config1); 304*5d7107c7SJonathan Cameron } 305*5d7107c7SJonathan Cameron 306*5d7107c7SJonathan Cameron static bool cxl_pmu_config1_get_invert(struct perf_event *event) 307*5d7107c7SJonathan Cameron { 308*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG1_INVERT_MSK, event->attr.config1); 309*5d7107c7SJonathan Cameron } 310*5d7107c7SJonathan Cameron 311*5d7107c7SJonathan Cameron static bool cxl_pmu_config1_get_edge(struct perf_event *event) 312*5d7107c7SJonathan Cameron { 313*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG1_EDGE_MSK, event->attr.config1); 314*5d7107c7SJonathan Cameron } 315*5d7107c7SJonathan Cameron 316*5d7107c7SJonathan Cameron /* 317*5d7107c7SJonathan Cameron * CPMU specification allows for 8 filters, each with a 16 bit value... 318*5d7107c7SJonathan Cameron * So we need to find 8x16bits to store it in. 319*5d7107c7SJonathan Cameron * As the value used for disable is 0xffff, a separate enable switch 320*5d7107c7SJonathan Cameron * is needed. 321*5d7107c7SJonathan Cameron */ 322*5d7107c7SJonathan Cameron 323*5d7107c7SJonathan Cameron static bool cxl_pmu_config1_hdm_filter_en(struct perf_event *event) 324*5d7107c7SJonathan Cameron { 325*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK, event->attr.config1); 326*5d7107c7SJonathan Cameron } 327*5d7107c7SJonathan Cameron 328*5d7107c7SJonathan Cameron static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) 329*5d7107c7SJonathan Cameron { 330*5d7107c7SJonathan Cameron return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); 331*5d7107c7SJonathan Cameron } 332*5d7107c7SJonathan Cameron 333*5d7107c7SJonathan Cameron static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, 334*5d7107c7SJonathan Cameron struct device_attribute *attr, char *buf) 335*5d7107c7SJonathan Cameron { 336*5d7107c7SJonathan Cameron struct perf_pmu_events_attr *pmu_attr = 337*5d7107c7SJonathan Cameron container_of(attr, struct perf_pmu_events_attr, attr); 338*5d7107c7SJonathan Cameron 339*5d7107c7SJonathan Cameron return sysfs_emit(buf, "config=%#llx\n", pmu_attr->id); 340*5d7107c7SJonathan Cameron } 341*5d7107c7SJonathan Cameron 342*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_ATTR(_name, _vid, _gid, _msk) \ 343*5d7107c7SJonathan Cameron PMU_EVENT_ATTR_ID(_name, cxl_pmu_event_sysfs_show, \ 344*5d7107c7SJonathan Cameron ((u64)(_vid) << 48) | ((u64)(_gid) << 32) | (u64)(_msk)) 345*5d7107c7SJonathan Cameron 346*5d7107c7SJonathan Cameron /* For CXL spec defined events */ 347*5d7107c7SJonathan Cameron #define CXL_PMU_EVENT_CXL_ATTR(_name, _gid, _msk) \ 348*5d7107c7SJonathan Cameron CXL_PMU_EVENT_ATTR(_name, PCI_DVSEC_VENDOR_ID_CXL, _gid, _msk) 349*5d7107c7SJonathan Cameron 350*5d7107c7SJonathan Cameron static struct attribute *cxl_pmu_event_attrs[] = { 351*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(clock_ticks, CXL_PMU_GID_CLOCK_TICKS, BIT(0)), 352*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-17 - Device to Host Requests */ 353*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdcurr, CXL_PMU_GID_D2H_REQ, BIT(1)), 354*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdown, CXL_PMU_GID_D2H_REQ, BIT(2)), 355*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdshared, CXL_PMU_GID_D2H_REQ, BIT(3)), 356*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdany, CXL_PMU_GID_D2H_REQ, BIT(4)), 357*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_rdownnodata, CXL_PMU_GID_D2H_REQ, BIT(5)), 358*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_itomwr, CXL_PMU_GID_D2H_REQ, BIT(6)), 359*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrcurr, CXL_PMU_GID_D2H_REQ, BIT(7)), 360*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_clflush, CXL_PMU_GID_D2H_REQ, BIT(8)), 361*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevict, CXL_PMU_GID_D2H_REQ, BIT(9)), 362*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_dirtyevict, CXL_PMU_GID_D2H_REQ, BIT(10)), 363*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_cleanevictnodata, CXL_PMU_GID_D2H_REQ, BIT(11)), 364*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinv, CXL_PMU_GID_D2H_REQ, BIT(12)), 365*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_wowrinvf, CXL_PMU_GID_D2H_REQ, BIT(13)), 366*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_wrinv, CXL_PMU_GID_D2H_REQ, BIT(14)), 367*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_req_cacheflushed, CXL_PMU_GID_D2H_REQ, BIT(16)), 368*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-20 - D2H Repsonse Encodings */ 369*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihiti, CXL_PMU_GID_D2H_RSP, BIT(4)), 370*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvhitv, CXL_PMU_GID_D2H_RSP, BIT(6)), 371*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspihitse, CXL_PMU_GID_D2H_RSP, BIT(5)), 372*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspshitse, CXL_PMU_GID_D2H_RSP, BIT(1)), 373*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspsfwdm, CXL_PMU_GID_D2H_RSP, BIT(7)), 374*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspifwdm, CXL_PMU_GID_D2H_RSP, BIT(15)), 375*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(d2h_rsp_rspvfwdv, CXL_PMU_GID_D2H_RSP, BIT(22)), 376*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-21 - CXL.cache - Mapping of H2D Requests to D2H Responses */ 377*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpdata, CXL_PMU_GID_H2D_REQ, BIT(1)), 378*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpinv, CXL_PMU_GID_H2D_REQ, BIT(2)), 379*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_req_snpcur, CXL_PMU_GID_H2D_REQ, BIT(3)), 380*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-22 - H2D Response Opcode Encodings */ 381*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_writepull, CXL_PMU_GID_H2D_RSP, BIT(1)), 382*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_go, CXL_PMU_GID_H2D_RSP, BIT(4)), 383*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepull, CXL_PMU_GID_H2D_RSP, BIT(5)), 384*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_extcmp, CXL_PMU_GID_H2D_RSP, BIT(6)), 385*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_gowritepulldrop, CXL_PMU_GID_H2D_RSP, BIT(8)), 386*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_fastgowritepull, CXL_PMU_GID_H2D_RSP, BIT(13)), 387*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(h2d_rsp_goerrwritepull, CXL_PMU_GID_H2D_RSP, BIT(15)), 388*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 13-5 directly lists these */ 389*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(cachedata_d2h_data, CXL_PMU_GID_CACHE_DATA, BIT(0)), 390*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(cachedata_h2d_data, CXL_PMU_GID_CACHE_DATA, BIT(1)), 391*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-29 M2S Req Memory Opcodes */ 392*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminv, CXL_PMU_GID_M2S_REQ, BIT(0)), 393*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrd, CXL_PMU_GID_M2S_REQ, BIT(1)), 394*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddata, CXL_PMU_GID_M2S_REQ, BIT(2)), 395*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdfwd, CXL_PMU_GID_M2S_REQ, BIT(3)), 396*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), 397*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), 398*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), 399*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), 400*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ 401*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), 402*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), 403*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), 404*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ 405*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), 406*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), 407*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_e, CXL_PMU_GID_M2S_BIRSP, BIT(2)), 408*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_iblk, CXL_PMU_GID_M2S_BIRSP, BIT(4)), 409*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_sblk, CXL_PMU_GID_M2S_BIRSP, BIT(5)), 410*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_eblk, CXL_PMU_GID_M2S_BIRSP, BIT(6)), 411*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-40 S2M BISnp Opcodes */ 412*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_cur, CXL_PMU_GID_S2M_BISNP, BIT(0)), 413*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_data, CXL_PMU_GID_S2M_BISNP, BIT(1)), 414*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_inv, CXL_PMU_GID_S2M_BISNP, BIT(2)), 415*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), 416*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), 417*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), 418*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-43 S2M NDR Opcopdes */ 419*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), 420*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), 421*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), 422*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(3)), 423*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ 424*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), 425*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), 426*5d7107c7SJonathan Cameron /* CXL rev 3.0 Table 13-5 directly lists these */ 427*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), 428*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), 429*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_casrd, CXL_PMU_GID_DDR, BIT(2)), 430*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_caswr, CXL_PMU_GID_DDR, BIT(3)), 431*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), 432*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), 433*5d7107c7SJonathan Cameron CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), 434*5d7107c7SJonathan Cameron NULL 435*5d7107c7SJonathan Cameron }; 436*5d7107c7SJonathan Cameron 437*5d7107c7SJonathan Cameron static struct cxl_pmu_ev_cap *cxl_pmu_find_fixed_counter_ev_cap(struct cxl_pmu_info *info, 438*5d7107c7SJonathan Cameron int vid, int gid, int msk) 439*5d7107c7SJonathan Cameron { 440*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap *pmu_ev; 441*5d7107c7SJonathan Cameron 442*5d7107c7SJonathan Cameron list_for_each_entry(pmu_ev, &info->event_caps_fixed, node) { 443*5d7107c7SJonathan Cameron if (vid != pmu_ev->vid || gid != pmu_ev->gid) 444*5d7107c7SJonathan Cameron continue; 445*5d7107c7SJonathan Cameron 446*5d7107c7SJonathan Cameron /* Precise match for fixed counter */ 447*5d7107c7SJonathan Cameron if (msk == pmu_ev->msk) 448*5d7107c7SJonathan Cameron return pmu_ev; 449*5d7107c7SJonathan Cameron } 450*5d7107c7SJonathan Cameron 451*5d7107c7SJonathan Cameron return ERR_PTR(-EINVAL); 452*5d7107c7SJonathan Cameron } 453*5d7107c7SJonathan Cameron 454*5d7107c7SJonathan Cameron static struct cxl_pmu_ev_cap *cxl_pmu_find_config_counter_ev_cap(struct cxl_pmu_info *info, 455*5d7107c7SJonathan Cameron int vid, int gid, int msk) 456*5d7107c7SJonathan Cameron { 457*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap *pmu_ev; 458*5d7107c7SJonathan Cameron 459*5d7107c7SJonathan Cameron list_for_each_entry(pmu_ev, &info->event_caps_configurable, node) { 460*5d7107c7SJonathan Cameron if (vid != pmu_ev->vid || gid != pmu_ev->gid) 461*5d7107c7SJonathan Cameron continue; 462*5d7107c7SJonathan Cameron 463*5d7107c7SJonathan Cameron /* Request mask must be subset of supported */ 464*5d7107c7SJonathan Cameron if (msk & ~pmu_ev->msk) 465*5d7107c7SJonathan Cameron continue; 466*5d7107c7SJonathan Cameron 467*5d7107c7SJonathan Cameron return pmu_ev; 468*5d7107c7SJonathan Cameron } 469*5d7107c7SJonathan Cameron 470*5d7107c7SJonathan Cameron return ERR_PTR(-EINVAL); 471*5d7107c7SJonathan Cameron } 472*5d7107c7SJonathan Cameron 473*5d7107c7SJonathan Cameron static umode_t cxl_pmu_event_is_visible(struct kobject *kobj, struct attribute *attr, int a) 474*5d7107c7SJonathan Cameron { 475*5d7107c7SJonathan Cameron struct device_attribute *dev_attr = container_of(attr, struct device_attribute, attr); 476*5d7107c7SJonathan Cameron struct perf_pmu_events_attr *pmu_attr = 477*5d7107c7SJonathan Cameron container_of(dev_attr, struct perf_pmu_events_attr, attr); 478*5d7107c7SJonathan Cameron struct device *dev = kobj_to_dev(kobj); 479*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = dev_get_drvdata(dev); 480*5d7107c7SJonathan Cameron int vid = FIELD_GET(CXL_PMU_ATTR_CONFIG_VID_MSK, pmu_attr->id); 481*5d7107c7SJonathan Cameron int gid = FIELD_GET(CXL_PMU_ATTR_CONFIG_GID_MSK, pmu_attr->id); 482*5d7107c7SJonathan Cameron int msk = FIELD_GET(CXL_PMU_ATTR_CONFIG_MASK_MSK, pmu_attr->id); 483*5d7107c7SJonathan Cameron 484*5d7107c7SJonathan Cameron if (!IS_ERR(cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, msk))) 485*5d7107c7SJonathan Cameron return attr->mode; 486*5d7107c7SJonathan Cameron 487*5d7107c7SJonathan Cameron if (!IS_ERR(cxl_pmu_find_config_counter_ev_cap(info, vid, gid, msk))) 488*5d7107c7SJonathan Cameron return attr->mode; 489*5d7107c7SJonathan Cameron 490*5d7107c7SJonathan Cameron return 0; 491*5d7107c7SJonathan Cameron } 492*5d7107c7SJonathan Cameron 493*5d7107c7SJonathan Cameron static const struct attribute_group cxl_pmu_events = { 494*5d7107c7SJonathan Cameron .name = "events", 495*5d7107c7SJonathan Cameron .attrs = cxl_pmu_event_attrs, 496*5d7107c7SJonathan Cameron .is_visible = cxl_pmu_event_is_visible, 497*5d7107c7SJonathan Cameron }; 498*5d7107c7SJonathan Cameron 499*5d7107c7SJonathan Cameron static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 500*5d7107c7SJonathan Cameron char *buf) 501*5d7107c7SJonathan Cameron { 502*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = dev_get_drvdata(dev); 503*5d7107c7SJonathan Cameron 504*5d7107c7SJonathan Cameron return cpumap_print_to_pagebuf(true, buf, cpumask_of(info->on_cpu)); 505*5d7107c7SJonathan Cameron } 506*5d7107c7SJonathan Cameron static DEVICE_ATTR_RO(cpumask); 507*5d7107c7SJonathan Cameron 508*5d7107c7SJonathan Cameron static struct attribute *cxl_pmu_cpumask_attrs[] = { 509*5d7107c7SJonathan Cameron &dev_attr_cpumask.attr, 510*5d7107c7SJonathan Cameron NULL 511*5d7107c7SJonathan Cameron }; 512*5d7107c7SJonathan Cameron 513*5d7107c7SJonathan Cameron static const struct attribute_group cxl_pmu_cpumask_group = { 514*5d7107c7SJonathan Cameron .attrs = cxl_pmu_cpumask_attrs, 515*5d7107c7SJonathan Cameron }; 516*5d7107c7SJonathan Cameron 517*5d7107c7SJonathan Cameron static const struct attribute_group *cxl_pmu_attr_groups[] = { 518*5d7107c7SJonathan Cameron &cxl_pmu_events, 519*5d7107c7SJonathan Cameron &cxl_pmu_format_group, 520*5d7107c7SJonathan Cameron &cxl_pmu_cpumask_group, 521*5d7107c7SJonathan Cameron NULL 522*5d7107c7SJonathan Cameron }; 523*5d7107c7SJonathan Cameron 524*5d7107c7SJonathan Cameron /* If counter_idx == NULL, don't try to allocate a counter. */ 525*5d7107c7SJonathan Cameron static int cxl_pmu_get_event_idx(struct perf_event *event, int *counter_idx, 526*5d7107c7SJonathan Cameron int *event_idx) 527*5d7107c7SJonathan Cameron { 528*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 529*5d7107c7SJonathan Cameron DECLARE_BITMAP(configurable_and_free, CXL_PMU_MAX_COUNTERS); 530*5d7107c7SJonathan Cameron struct cxl_pmu_ev_cap *pmu_ev; 531*5d7107c7SJonathan Cameron u32 mask; 532*5d7107c7SJonathan Cameron u16 gid, vid; 533*5d7107c7SJonathan Cameron int i; 534*5d7107c7SJonathan Cameron 535*5d7107c7SJonathan Cameron vid = cxl_pmu_config_get_vid(event); 536*5d7107c7SJonathan Cameron gid = cxl_pmu_config_get_gid(event); 537*5d7107c7SJonathan Cameron mask = cxl_pmu_config_get_mask(event); 538*5d7107c7SJonathan Cameron 539*5d7107c7SJonathan Cameron pmu_ev = cxl_pmu_find_fixed_counter_ev_cap(info, vid, gid, mask); 540*5d7107c7SJonathan Cameron if (!IS_ERR(pmu_ev)) { 541*5d7107c7SJonathan Cameron if (!counter_idx) 542*5d7107c7SJonathan Cameron return 0; 543*5d7107c7SJonathan Cameron if (!test_bit(pmu_ev->counter_idx, info->used_counter_bm)) { 544*5d7107c7SJonathan Cameron *counter_idx = pmu_ev->counter_idx; 545*5d7107c7SJonathan Cameron return 0; 546*5d7107c7SJonathan Cameron } 547*5d7107c7SJonathan Cameron /* Fixed counter is in use, but maybe a configurable one? */ 548*5d7107c7SJonathan Cameron } 549*5d7107c7SJonathan Cameron 550*5d7107c7SJonathan Cameron pmu_ev = cxl_pmu_find_config_counter_ev_cap(info, vid, gid, mask); 551*5d7107c7SJonathan Cameron if (!IS_ERR(pmu_ev)) { 552*5d7107c7SJonathan Cameron if (!counter_idx) 553*5d7107c7SJonathan Cameron return 0; 554*5d7107c7SJonathan Cameron 555*5d7107c7SJonathan Cameron bitmap_andnot(configurable_and_free, info->conf_counter_bm, 556*5d7107c7SJonathan Cameron info->used_counter_bm, CXL_PMU_MAX_COUNTERS); 557*5d7107c7SJonathan Cameron 558*5d7107c7SJonathan Cameron i = find_first_bit(configurable_and_free, CXL_PMU_MAX_COUNTERS); 559*5d7107c7SJonathan Cameron if (i == CXL_PMU_MAX_COUNTERS) 560*5d7107c7SJonathan Cameron return -EINVAL; 561*5d7107c7SJonathan Cameron 562*5d7107c7SJonathan Cameron *counter_idx = i; 563*5d7107c7SJonathan Cameron return 0; 564*5d7107c7SJonathan Cameron } 565*5d7107c7SJonathan Cameron 566*5d7107c7SJonathan Cameron return -EINVAL; 567*5d7107c7SJonathan Cameron } 568*5d7107c7SJonathan Cameron 569*5d7107c7SJonathan Cameron static int cxl_pmu_event_init(struct perf_event *event) 570*5d7107c7SJonathan Cameron { 571*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 572*5d7107c7SJonathan Cameron int rc; 573*5d7107c7SJonathan Cameron 574*5d7107c7SJonathan Cameron /* Top level type sanity check - is this a Hardware Event being requested */ 575*5d7107c7SJonathan Cameron if (event->attr.type != event->pmu->type) 576*5d7107c7SJonathan Cameron return -ENOENT; 577*5d7107c7SJonathan Cameron 578*5d7107c7SJonathan Cameron if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 579*5d7107c7SJonathan Cameron return -EOPNOTSUPP; 580*5d7107c7SJonathan Cameron /* TODO: Validation of any filter */ 581*5d7107c7SJonathan Cameron 582*5d7107c7SJonathan Cameron /* 583*5d7107c7SJonathan Cameron * Verify that it is possible to count what was requested. Either must 584*5d7107c7SJonathan Cameron * be a fixed counter that is a precise match or a configurable counter 585*5d7107c7SJonathan Cameron * where this is a subset. 586*5d7107c7SJonathan Cameron */ 587*5d7107c7SJonathan Cameron rc = cxl_pmu_get_event_idx(event, NULL, NULL); 588*5d7107c7SJonathan Cameron if (rc < 0) 589*5d7107c7SJonathan Cameron return rc; 590*5d7107c7SJonathan Cameron 591*5d7107c7SJonathan Cameron event->cpu = info->on_cpu; 592*5d7107c7SJonathan Cameron 593*5d7107c7SJonathan Cameron return 0; 594*5d7107c7SJonathan Cameron } 595*5d7107c7SJonathan Cameron 596*5d7107c7SJonathan Cameron static void cxl_pmu_enable(struct pmu *pmu) 597*5d7107c7SJonathan Cameron { 598*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 599*5d7107c7SJonathan Cameron void __iomem *base = info->base; 600*5d7107c7SJonathan Cameron 601*5d7107c7SJonathan Cameron /* Can assume frozen at this stage */ 602*5d7107c7SJonathan Cameron writeq(0, base + CXL_PMU_FREEZE_REG); 603*5d7107c7SJonathan Cameron } 604*5d7107c7SJonathan Cameron 605*5d7107c7SJonathan Cameron static void cxl_pmu_disable(struct pmu *pmu) 606*5d7107c7SJonathan Cameron { 607*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(pmu); 608*5d7107c7SJonathan Cameron void __iomem *base = info->base; 609*5d7107c7SJonathan Cameron 610*5d7107c7SJonathan Cameron /* 611*5d7107c7SJonathan Cameron * Whilst bits above number of counters are RsvdZ 612*5d7107c7SJonathan Cameron * they are unlikely to be repurposed given 613*5d7107c7SJonathan Cameron * number of counters is allowed to be 64 leaving 614*5d7107c7SJonathan Cameron * no reserved bits. Hence this is only slightly 615*5d7107c7SJonathan Cameron * naughty. 616*5d7107c7SJonathan Cameron */ 617*5d7107c7SJonathan Cameron writeq(GENMASK_ULL(63, 0), base + CXL_PMU_FREEZE_REG); 618*5d7107c7SJonathan Cameron } 619*5d7107c7SJonathan Cameron 620*5d7107c7SJonathan Cameron static void cxl_pmu_event_start(struct perf_event *event, int flags) 621*5d7107c7SJonathan Cameron { 622*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 623*5d7107c7SJonathan Cameron struct hw_perf_event *hwc = &event->hw; 624*5d7107c7SJonathan Cameron void __iomem *base = info->base; 625*5d7107c7SJonathan Cameron u64 cfg; 626*5d7107c7SJonathan Cameron 627*5d7107c7SJonathan Cameron /* 628*5d7107c7SJonathan Cameron * All paths to here should either set these flags directly or 629*5d7107c7SJonathan Cameron * call cxl_pmu_event_stop() which will ensure the correct state. 630*5d7107c7SJonathan Cameron */ 631*5d7107c7SJonathan Cameron if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 632*5d7107c7SJonathan Cameron return; 633*5d7107c7SJonathan Cameron 634*5d7107c7SJonathan Cameron WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 635*5d7107c7SJonathan Cameron hwc->state = 0; 636*5d7107c7SJonathan Cameron 637*5d7107c7SJonathan Cameron /* 638*5d7107c7SJonathan Cameron * Currently only hdm filter control is implemnted, this code will 639*5d7107c7SJonathan Cameron * want generalizing when more filters are added. 640*5d7107c7SJonathan Cameron */ 641*5d7107c7SJonathan Cameron if (info->filter_hdm) { 642*5d7107c7SJonathan Cameron if (cxl_pmu_config1_hdm_filter_en(event)) 643*5d7107c7SJonathan Cameron cfg = cxl_pmu_config2_get_hdm_decoder(event); 644*5d7107c7SJonathan Cameron else 645*5d7107c7SJonathan Cameron cfg = GENMASK(15, 0); /* No filtering if 0xFFFF_FFFF */ 646*5d7107c7SJonathan Cameron writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); 647*5d7107c7SJonathan Cameron } 648*5d7107c7SJonathan Cameron 649*5d7107c7SJonathan Cameron cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 650*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1); 651*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1); 652*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1); 653*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EDGE, 654*5d7107c7SJonathan Cameron cxl_pmu_config1_get_edge(event) ? 1 : 0); 655*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INVERT, 656*5d7107c7SJonathan Cameron cxl_pmu_config1_get_invert(event) ? 1 : 0); 657*5d7107c7SJonathan Cameron 658*5d7107c7SJonathan Cameron /* Fixed purpose counters have next two fields RO */ 659*5d7107c7SJonathan Cameron if (test_bit(hwc->idx, info->conf_counter_bm)) { 660*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENT_GRP_ID_IDX_MSK, 661*5d7107c7SJonathan Cameron hwc->event_base); 662*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_EVENTS_MSK, 663*5d7107c7SJonathan Cameron cxl_pmu_config_get_mask(event)); 664*5d7107c7SJonathan Cameron } 665*5d7107c7SJonathan Cameron cfg &= ~CXL_PMU_COUNTER_CFG_THRESHOLD_MSK; 666*5d7107c7SJonathan Cameron /* 667*5d7107c7SJonathan Cameron * For events that generate only 1 count per clock the CXL 3.0 spec 668*5d7107c7SJonathan Cameron * states the threshold shall be set to 1 but if set to 0 it will 669*5d7107c7SJonathan Cameron * count the raw value anwyay? 670*5d7107c7SJonathan Cameron * There is no definition of what events will count multiple per cycle 671*5d7107c7SJonathan Cameron * and hence to which non 1 values of threshold can apply. 672*5d7107c7SJonathan Cameron * (CXL 3.0 8.2.7.2.1 Counter Configuration - threshold field definition) 673*5d7107c7SJonathan Cameron */ 674*5d7107c7SJonathan Cameron cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_THRESHOLD_MSK, 675*5d7107c7SJonathan Cameron cxl_pmu_config1_get_threshold(event)); 676*5d7107c7SJonathan Cameron writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 677*5d7107c7SJonathan Cameron 678*5d7107c7SJonathan Cameron local64_set(&hwc->prev_count, 0); 679*5d7107c7SJonathan Cameron writeq(0, base + CXL_PMU_COUNTER_REG(hwc->idx)); 680*5d7107c7SJonathan Cameron 681*5d7107c7SJonathan Cameron perf_event_update_userpage(event); 682*5d7107c7SJonathan Cameron } 683*5d7107c7SJonathan Cameron 684*5d7107c7SJonathan Cameron static u64 cxl_pmu_read_counter(struct perf_event *event) 685*5d7107c7SJonathan Cameron { 686*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 687*5d7107c7SJonathan Cameron void __iomem *base = info->base; 688*5d7107c7SJonathan Cameron 689*5d7107c7SJonathan Cameron return readq(base + CXL_PMU_COUNTER_REG(event->hw.idx)); 690*5d7107c7SJonathan Cameron } 691*5d7107c7SJonathan Cameron 692*5d7107c7SJonathan Cameron static void __cxl_pmu_read(struct perf_event *event, bool overflow) 693*5d7107c7SJonathan Cameron { 694*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 695*5d7107c7SJonathan Cameron struct hw_perf_event *hwc = &event->hw; 696*5d7107c7SJonathan Cameron u64 new_cnt, prev_cnt, delta; 697*5d7107c7SJonathan Cameron 698*5d7107c7SJonathan Cameron do { 699*5d7107c7SJonathan Cameron prev_cnt = local64_read(&hwc->prev_count); 700*5d7107c7SJonathan Cameron new_cnt = cxl_pmu_read_counter(event); 701*5d7107c7SJonathan Cameron } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != prev_cnt); 702*5d7107c7SJonathan Cameron 703*5d7107c7SJonathan Cameron /* 704*5d7107c7SJonathan Cameron * If we know an overflow occur then take that into account. 705*5d7107c7SJonathan Cameron * Note counter is not reset as that would lose events 706*5d7107c7SJonathan Cameron */ 707*5d7107c7SJonathan Cameron delta = (new_cnt - prev_cnt) & GENMASK_ULL(info->counter_width - 1, 0); 708*5d7107c7SJonathan Cameron if (overflow && delta < GENMASK_ULL(info->counter_width - 1, 0)) 709*5d7107c7SJonathan Cameron delta += (1UL << info->counter_width); 710*5d7107c7SJonathan Cameron 711*5d7107c7SJonathan Cameron local64_add(delta, &event->count); 712*5d7107c7SJonathan Cameron } 713*5d7107c7SJonathan Cameron 714*5d7107c7SJonathan Cameron static void cxl_pmu_read(struct perf_event *event) 715*5d7107c7SJonathan Cameron { 716*5d7107c7SJonathan Cameron __cxl_pmu_read(event, false); 717*5d7107c7SJonathan Cameron } 718*5d7107c7SJonathan Cameron 719*5d7107c7SJonathan Cameron static void cxl_pmu_event_stop(struct perf_event *event, int flags) 720*5d7107c7SJonathan Cameron { 721*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 722*5d7107c7SJonathan Cameron void __iomem *base = info->base; 723*5d7107c7SJonathan Cameron struct hw_perf_event *hwc = &event->hw; 724*5d7107c7SJonathan Cameron u64 cfg; 725*5d7107c7SJonathan Cameron 726*5d7107c7SJonathan Cameron cxl_pmu_read(event); 727*5d7107c7SJonathan Cameron WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 728*5d7107c7SJonathan Cameron hwc->state |= PERF_HES_STOPPED; 729*5d7107c7SJonathan Cameron 730*5d7107c7SJonathan Cameron cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 731*5d7107c7SJonathan Cameron cfg &= ~(FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1) | 732*5d7107c7SJonathan Cameron FIELD_PREP(CXL_PMU_COUNTER_CFG_ENABLE, 1)); 733*5d7107c7SJonathan Cameron writeq(cfg, base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); 734*5d7107c7SJonathan Cameron 735*5d7107c7SJonathan Cameron hwc->state |= PERF_HES_UPTODATE; 736*5d7107c7SJonathan Cameron } 737*5d7107c7SJonathan Cameron 738*5d7107c7SJonathan Cameron static int cxl_pmu_event_add(struct perf_event *event, int flags) 739*5d7107c7SJonathan Cameron { 740*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 741*5d7107c7SJonathan Cameron struct hw_perf_event *hwc = &event->hw; 742*5d7107c7SJonathan Cameron int idx, rc; 743*5d7107c7SJonathan Cameron int event_idx = 0; 744*5d7107c7SJonathan Cameron 745*5d7107c7SJonathan Cameron hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 746*5d7107c7SJonathan Cameron 747*5d7107c7SJonathan Cameron rc = cxl_pmu_get_event_idx(event, &idx, &event_idx); 748*5d7107c7SJonathan Cameron if (rc < 0) 749*5d7107c7SJonathan Cameron return rc; 750*5d7107c7SJonathan Cameron 751*5d7107c7SJonathan Cameron hwc->idx = idx; 752*5d7107c7SJonathan Cameron 753*5d7107c7SJonathan Cameron /* Only set for configurable counters */ 754*5d7107c7SJonathan Cameron hwc->event_base = event_idx; 755*5d7107c7SJonathan Cameron info->hw_events[idx] = event; 756*5d7107c7SJonathan Cameron set_bit(idx, info->used_counter_bm); 757*5d7107c7SJonathan Cameron 758*5d7107c7SJonathan Cameron if (flags & PERF_EF_START) 759*5d7107c7SJonathan Cameron cxl_pmu_event_start(event, PERF_EF_RELOAD); 760*5d7107c7SJonathan Cameron 761*5d7107c7SJonathan Cameron return 0; 762*5d7107c7SJonathan Cameron } 763*5d7107c7SJonathan Cameron 764*5d7107c7SJonathan Cameron static void cxl_pmu_event_del(struct perf_event *event, int flags) 765*5d7107c7SJonathan Cameron { 766*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = pmu_to_cxl_pmu_info(event->pmu); 767*5d7107c7SJonathan Cameron struct hw_perf_event *hwc = &event->hw; 768*5d7107c7SJonathan Cameron 769*5d7107c7SJonathan Cameron cxl_pmu_event_stop(event, PERF_EF_UPDATE); 770*5d7107c7SJonathan Cameron clear_bit(hwc->idx, info->used_counter_bm); 771*5d7107c7SJonathan Cameron info->hw_events[hwc->idx] = NULL; 772*5d7107c7SJonathan Cameron perf_event_update_userpage(event); 773*5d7107c7SJonathan Cameron } 774*5d7107c7SJonathan Cameron 775*5d7107c7SJonathan Cameron static irqreturn_t cxl_pmu_irq(int irq, void *data) 776*5d7107c7SJonathan Cameron { 777*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = data; 778*5d7107c7SJonathan Cameron void __iomem *base = info->base; 779*5d7107c7SJonathan Cameron u64 overflowed; 780*5d7107c7SJonathan Cameron DECLARE_BITMAP(overflowedbm, 64); 781*5d7107c7SJonathan Cameron int i; 782*5d7107c7SJonathan Cameron 783*5d7107c7SJonathan Cameron overflowed = readq(base + CXL_PMU_OVERFLOW_REG); 784*5d7107c7SJonathan Cameron 785*5d7107c7SJonathan Cameron /* Interrupt may be shared, so maybe it isn't ours */ 786*5d7107c7SJonathan Cameron if (!overflowed) 787*5d7107c7SJonathan Cameron return IRQ_NONE; 788*5d7107c7SJonathan Cameron 789*5d7107c7SJonathan Cameron bitmap_from_arr64(overflowedbm, &overflowed, 64); 790*5d7107c7SJonathan Cameron for_each_set_bit(i, overflowedbm, info->num_counters) { 791*5d7107c7SJonathan Cameron struct perf_event *event = info->hw_events[i]; 792*5d7107c7SJonathan Cameron 793*5d7107c7SJonathan Cameron if (!event) { 794*5d7107c7SJonathan Cameron dev_dbg(info->pmu.dev, 795*5d7107c7SJonathan Cameron "overflow but on non enabled counter %d\n", i); 796*5d7107c7SJonathan Cameron continue; 797*5d7107c7SJonathan Cameron } 798*5d7107c7SJonathan Cameron 799*5d7107c7SJonathan Cameron __cxl_pmu_read(event, true); 800*5d7107c7SJonathan Cameron } 801*5d7107c7SJonathan Cameron 802*5d7107c7SJonathan Cameron writeq(overflowed, base + CXL_PMU_OVERFLOW_REG); 803*5d7107c7SJonathan Cameron 804*5d7107c7SJonathan Cameron return IRQ_HANDLED; 805*5d7107c7SJonathan Cameron } 806*5d7107c7SJonathan Cameron 807*5d7107c7SJonathan Cameron static void cxl_pmu_perf_unregister(void *_info) 808*5d7107c7SJonathan Cameron { 809*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = _info; 810*5d7107c7SJonathan Cameron 811*5d7107c7SJonathan Cameron perf_pmu_unregister(&info->pmu); 812*5d7107c7SJonathan Cameron } 813*5d7107c7SJonathan Cameron 814*5d7107c7SJonathan Cameron static void cxl_pmu_cpuhp_remove(void *_info) 815*5d7107c7SJonathan Cameron { 816*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = _info; 817*5d7107c7SJonathan Cameron 818*5d7107c7SJonathan Cameron cpuhp_state_remove_instance_nocalls(cxl_pmu_cpuhp_state_num, &info->node); 819*5d7107c7SJonathan Cameron } 820*5d7107c7SJonathan Cameron 821*5d7107c7SJonathan Cameron static int cxl_pmu_probe(struct device *dev) 822*5d7107c7SJonathan Cameron { 823*5d7107c7SJonathan Cameron struct cxl_pmu *pmu = to_cxl_pmu(dev); 824*5d7107c7SJonathan Cameron struct pci_dev *pdev = to_pci_dev(dev->parent); 825*5d7107c7SJonathan Cameron struct cxl_pmu_info *info; 826*5d7107c7SJonathan Cameron char *irq_name; 827*5d7107c7SJonathan Cameron char *dev_name; 828*5d7107c7SJonathan Cameron int rc, irq; 829*5d7107c7SJonathan Cameron 830*5d7107c7SJonathan Cameron info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); 831*5d7107c7SJonathan Cameron if (!info) 832*5d7107c7SJonathan Cameron return -ENOMEM; 833*5d7107c7SJonathan Cameron 834*5d7107c7SJonathan Cameron dev_set_drvdata(dev, info); 835*5d7107c7SJonathan Cameron INIT_LIST_HEAD(&info->event_caps_fixed); 836*5d7107c7SJonathan Cameron INIT_LIST_HEAD(&info->event_caps_configurable); 837*5d7107c7SJonathan Cameron 838*5d7107c7SJonathan Cameron info->base = pmu->base; 839*5d7107c7SJonathan Cameron 840*5d7107c7SJonathan Cameron info->on_cpu = -1; 841*5d7107c7SJonathan Cameron rc = cxl_pmu_parse_caps(dev, info); 842*5d7107c7SJonathan Cameron if (rc) 843*5d7107c7SJonathan Cameron return rc; 844*5d7107c7SJonathan Cameron 845*5d7107c7SJonathan Cameron info->hw_events = devm_kcalloc(dev, sizeof(*info->hw_events), 846*5d7107c7SJonathan Cameron info->num_counters, GFP_KERNEL); 847*5d7107c7SJonathan Cameron if (!info->hw_events) 848*5d7107c7SJonathan Cameron return -ENOMEM; 849*5d7107c7SJonathan Cameron 850*5d7107c7SJonathan Cameron switch (pmu->type) { 851*5d7107c7SJonathan Cameron case CXL_PMU_MEMDEV: 852*5d7107c7SJonathan Cameron dev_name = devm_kasprintf(dev, GFP_KERNEL, "cxl_pmu_mem%d.%d", 853*5d7107c7SJonathan Cameron pmu->assoc_id, pmu->index); 854*5d7107c7SJonathan Cameron break; 855*5d7107c7SJonathan Cameron } 856*5d7107c7SJonathan Cameron if (!dev_name) 857*5d7107c7SJonathan Cameron return -ENOMEM; 858*5d7107c7SJonathan Cameron 859*5d7107c7SJonathan Cameron info->pmu = (struct pmu) { 860*5d7107c7SJonathan Cameron .name = dev_name, 861*5d7107c7SJonathan Cameron .parent = dev, 862*5d7107c7SJonathan Cameron .module = THIS_MODULE, 863*5d7107c7SJonathan Cameron .event_init = cxl_pmu_event_init, 864*5d7107c7SJonathan Cameron .pmu_enable = cxl_pmu_enable, 865*5d7107c7SJonathan Cameron .pmu_disable = cxl_pmu_disable, 866*5d7107c7SJonathan Cameron .add = cxl_pmu_event_add, 867*5d7107c7SJonathan Cameron .del = cxl_pmu_event_del, 868*5d7107c7SJonathan Cameron .start = cxl_pmu_event_start, 869*5d7107c7SJonathan Cameron .stop = cxl_pmu_event_stop, 870*5d7107c7SJonathan Cameron .read = cxl_pmu_read, 871*5d7107c7SJonathan Cameron .task_ctx_nr = perf_invalid_context, 872*5d7107c7SJonathan Cameron .attr_groups = cxl_pmu_attr_groups, 873*5d7107c7SJonathan Cameron .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 874*5d7107c7SJonathan Cameron }; 875*5d7107c7SJonathan Cameron 876*5d7107c7SJonathan Cameron if (info->irq <= 0) 877*5d7107c7SJonathan Cameron return -EINVAL; 878*5d7107c7SJonathan Cameron 879*5d7107c7SJonathan Cameron rc = pci_irq_vector(pdev, info->irq); 880*5d7107c7SJonathan Cameron if (rc < 0) 881*5d7107c7SJonathan Cameron return rc; 882*5d7107c7SJonathan Cameron irq = rc; 883*5d7107c7SJonathan Cameron 884*5d7107c7SJonathan Cameron irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_overflow\n", dev_name); 885*5d7107c7SJonathan Cameron if (!irq_name) 886*5d7107c7SJonathan Cameron return -ENOMEM; 887*5d7107c7SJonathan Cameron 888*5d7107c7SJonathan Cameron rc = devm_request_irq(dev, irq, cxl_pmu_irq, IRQF_SHARED | IRQF_ONESHOT, 889*5d7107c7SJonathan Cameron irq_name, info); 890*5d7107c7SJonathan Cameron if (rc) 891*5d7107c7SJonathan Cameron return rc; 892*5d7107c7SJonathan Cameron info->irq = irq; 893*5d7107c7SJonathan Cameron 894*5d7107c7SJonathan Cameron rc = cpuhp_state_add_instance(cxl_pmu_cpuhp_state_num, &info->node); 895*5d7107c7SJonathan Cameron if (rc) 896*5d7107c7SJonathan Cameron return rc; 897*5d7107c7SJonathan Cameron 898*5d7107c7SJonathan Cameron rc = devm_add_action_or_reset(dev, cxl_pmu_cpuhp_remove, info); 899*5d7107c7SJonathan Cameron if (rc) 900*5d7107c7SJonathan Cameron return rc; 901*5d7107c7SJonathan Cameron 902*5d7107c7SJonathan Cameron rc = perf_pmu_register(&info->pmu, info->pmu.name, -1); 903*5d7107c7SJonathan Cameron if (rc) 904*5d7107c7SJonathan Cameron return rc; 905*5d7107c7SJonathan Cameron 906*5d7107c7SJonathan Cameron rc = devm_add_action_or_reset(dev, cxl_pmu_perf_unregister, info); 907*5d7107c7SJonathan Cameron if (rc) 908*5d7107c7SJonathan Cameron return rc; 909*5d7107c7SJonathan Cameron 910*5d7107c7SJonathan Cameron return 0; 911*5d7107c7SJonathan Cameron } 912*5d7107c7SJonathan Cameron 913*5d7107c7SJonathan Cameron static struct cxl_driver cxl_pmu_driver = { 914*5d7107c7SJonathan Cameron .name = "cxl_pmu", 915*5d7107c7SJonathan Cameron .probe = cxl_pmu_probe, 916*5d7107c7SJonathan Cameron .id = CXL_DEVICE_PMU, 917*5d7107c7SJonathan Cameron }; 918*5d7107c7SJonathan Cameron 919*5d7107c7SJonathan Cameron static int cxl_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 920*5d7107c7SJonathan Cameron { 921*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 922*5d7107c7SJonathan Cameron 923*5d7107c7SJonathan Cameron if (info->on_cpu != -1) 924*5d7107c7SJonathan Cameron return 0; 925*5d7107c7SJonathan Cameron 926*5d7107c7SJonathan Cameron info->on_cpu = cpu; 927*5d7107c7SJonathan Cameron /* 928*5d7107c7SJonathan Cameron * CPU HP lock is held so we should be guaranteed that the CPU hasn't yet 929*5d7107c7SJonathan Cameron * gone away again. 930*5d7107c7SJonathan Cameron */ 931*5d7107c7SJonathan Cameron WARN_ON(irq_set_affinity(info->irq, cpumask_of(cpu))); 932*5d7107c7SJonathan Cameron 933*5d7107c7SJonathan Cameron return 0; 934*5d7107c7SJonathan Cameron } 935*5d7107c7SJonathan Cameron 936*5d7107c7SJonathan Cameron static int cxl_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 937*5d7107c7SJonathan Cameron { 938*5d7107c7SJonathan Cameron struct cxl_pmu_info *info = hlist_entry_safe(node, struct cxl_pmu_info, node); 939*5d7107c7SJonathan Cameron unsigned int target; 940*5d7107c7SJonathan Cameron 941*5d7107c7SJonathan Cameron if (info->on_cpu != cpu) 942*5d7107c7SJonathan Cameron return 0; 943*5d7107c7SJonathan Cameron 944*5d7107c7SJonathan Cameron info->on_cpu = -1; 945*5d7107c7SJonathan Cameron target = cpumask_any_but(cpu_online_mask, cpu); 946*5d7107c7SJonathan Cameron if (target >= nr_cpu_ids) { 947*5d7107c7SJonathan Cameron dev_err(info->pmu.dev, "Unable to find a suitable CPU\n"); 948*5d7107c7SJonathan Cameron return 0; 949*5d7107c7SJonathan Cameron } 950*5d7107c7SJonathan Cameron 951*5d7107c7SJonathan Cameron perf_pmu_migrate_context(&info->pmu, cpu, target); 952*5d7107c7SJonathan Cameron info->on_cpu = target; 953*5d7107c7SJonathan Cameron /* 954*5d7107c7SJonathan Cameron * CPU HP lock is held so we should be guaranteed that this CPU hasn't yet 955*5d7107c7SJonathan Cameron * gone away. 956*5d7107c7SJonathan Cameron */ 957*5d7107c7SJonathan Cameron WARN_ON(irq_set_affinity(info->irq, cpumask_of(target))); 958*5d7107c7SJonathan Cameron 959*5d7107c7SJonathan Cameron return 0; 960*5d7107c7SJonathan Cameron } 961*5d7107c7SJonathan Cameron 962*5d7107c7SJonathan Cameron static __init int cxl_pmu_init(void) 963*5d7107c7SJonathan Cameron { 964*5d7107c7SJonathan Cameron int rc; 965*5d7107c7SJonathan Cameron 966*5d7107c7SJonathan Cameron rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 967*5d7107c7SJonathan Cameron "AP_PERF_CXL_PMU_ONLINE", 968*5d7107c7SJonathan Cameron cxl_pmu_online_cpu, cxl_pmu_offline_cpu); 969*5d7107c7SJonathan Cameron if (rc < 0) 970*5d7107c7SJonathan Cameron return rc; 971*5d7107c7SJonathan Cameron cxl_pmu_cpuhp_state_num = rc; 972*5d7107c7SJonathan Cameron 973*5d7107c7SJonathan Cameron rc = cxl_driver_register(&cxl_pmu_driver); 974*5d7107c7SJonathan Cameron if (rc) 975*5d7107c7SJonathan Cameron cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 976*5d7107c7SJonathan Cameron 977*5d7107c7SJonathan Cameron return rc; 978*5d7107c7SJonathan Cameron } 979*5d7107c7SJonathan Cameron 980*5d7107c7SJonathan Cameron static __exit void cxl_pmu_exit(void) 981*5d7107c7SJonathan Cameron { 982*5d7107c7SJonathan Cameron cxl_driver_unregister(&cxl_pmu_driver); 983*5d7107c7SJonathan Cameron cpuhp_remove_multi_state(cxl_pmu_cpuhp_state_num); 984*5d7107c7SJonathan Cameron } 985*5d7107c7SJonathan Cameron 986*5d7107c7SJonathan Cameron MODULE_LICENSE("GPL"); 987*5d7107c7SJonathan Cameron MODULE_IMPORT_NS(CXL); 988*5d7107c7SJonathan Cameron module_init(cxl_pmu_init); 989*5d7107c7SJonathan Cameron module_exit(cxl_pmu_exit); 990*5d7107c7SJonathan Cameron MODULE_ALIAS_CXL(CXL_DEVICE_PMU); 991