1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * This driver adds support for HNS3 PMU iEP device. Related perf events are 4 * bandwidth, latency, packet rate, interrupt rate etc. 5 * 6 * Copyright (C) 2022 HiSilicon Limited 7 */ 8 #include <linux/bitfield.h> 9 #include <linux/bitmap.h> 10 #include <linux/bug.h> 11 #include <linux/cpuhotplug.h> 12 #include <linux/cpumask.h> 13 #include <linux/delay.h> 14 #include <linux/device.h> 15 #include <linux/err.h> 16 #include <linux/interrupt.h> 17 #include <linux/iopoll.h> 18 #include <linux/io-64-nonatomic-hi-lo.h> 19 #include <linux/irq.h> 20 #include <linux/kernel.h> 21 #include <linux/list.h> 22 #include <linux/module.h> 23 #include <linux/pci.h> 24 #include <linux/pci-epf.h> 25 #include <linux/perf_event.h> 26 #include <linux/smp.h> 27 28 /* registers offset address */ 29 #define HNS3_PMU_REG_GLOBAL_CTRL 0x0000 30 #define HNS3_PMU_REG_CLOCK_FREQ 0x0020 31 #define HNS3_PMU_REG_BDF 0x0fe0 32 #define HNS3_PMU_REG_VERSION 0x0fe4 33 #define HNS3_PMU_REG_DEVICE_ID 0x0fe8 34 35 #define HNS3_PMU_REG_EVENT_OFFSET 0x1000 36 #define HNS3_PMU_REG_EVENT_SIZE 0x1000 37 #define HNS3_PMU_REG_EVENT_CTRL_LOW 0x00 38 #define HNS3_PMU_REG_EVENT_CTRL_HIGH 0x04 39 #define HNS3_PMU_REG_EVENT_INTR_STATUS 0x08 40 #define HNS3_PMU_REG_EVENT_INTR_MASK 0x0c 41 #define HNS3_PMU_REG_EVENT_COUNTER 0x10 42 #define HNS3_PMU_REG_EVENT_EXT_COUNTER 0x18 43 #define HNS3_PMU_REG_EVENT_QID_CTRL 0x28 44 #define HNS3_PMU_REG_EVENT_QID_PARA 0x2c 45 46 #define HNS3_PMU_FILTER_SUPPORT_GLOBAL BIT(0) 47 #define HNS3_PMU_FILTER_SUPPORT_PORT BIT(1) 48 #define HNS3_PMU_FILTER_SUPPORT_PORT_TC BIT(2) 49 #define HNS3_PMU_FILTER_SUPPORT_FUNC BIT(3) 50 #define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE BIT(4) 51 #define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR BIT(5) 52 53 #define HNS3_PMU_FILTER_ALL_TC 0xf 54 #define HNS3_PMU_FILTER_ALL_QUEUE 0xffff 55 56 #define HNS3_PMU_CTRL_SUBEVENT_S 4 57 #define HNS3_PMU_CTRL_FILTER_MODE_S 24 58 59 #define HNS3_PMU_GLOBAL_START BIT(0) 60 61 #define HNS3_PMU_EVENT_STATUS_RESET BIT(11) 62 #define HNS3_PMU_EVENT_EN BIT(12) 63 #define HNS3_PMU_EVENT_OVERFLOW_RESTART BIT(15) 64 65 #define HNS3_PMU_QID_PARA_FUNC_S 0 66 #define HNS3_PMU_QID_PARA_QUEUE_S 16 67 68 #define HNS3_PMU_QID_CTRL_REQ_ENABLE BIT(0) 69 #define HNS3_PMU_QID_CTRL_DONE BIT(1) 70 #define HNS3_PMU_QID_CTRL_MISS BIT(2) 71 72 #define HNS3_PMU_INTR_MASK_OVERFLOW BIT(1) 73 74 #define HNS3_PMU_MAX_HW_EVENTS 8 75 76 /* 77 * Each hardware event contains two registers (counter and ext_counter) for 78 * bandwidth, packet rate, latency and interrupt rate. These two registers will 79 * be triggered to run at the same when a hardware event is enabled. The meaning 80 * of counter and ext_counter of different event type are different, their 81 * meaning show as follow: 82 * 83 * +----------------+------------------+---------------+ 84 * | event type | counter | ext_counter | 85 * +----------------+------------------+---------------+ 86 * | bandwidth | byte number | cycle number | 87 * +----------------+------------------+---------------+ 88 * | packet rate | packet number | cycle number | 89 * +----------------+------------------+---------------+ 90 * | latency | cycle number | packet number | 91 * +----------------+------------------+---------------+ 92 * | interrupt rate | interrupt number | cycle number | 93 * +----------------+------------------+---------------+ 94 * 95 * The cycle number indicates increment of counter of hardware timer, the 96 * frequency of hardware timer can be read from hw_clk_freq file. 97 * 98 * Performance of each hardware event is calculated by: counter / ext_counter. 99 * 100 * Since processing of data is preferred to be done in userspace, we expose 101 * ext_counter as a separate event for userspace and use bit 16 to indicate it. 102 * For example, event 0x00001 and 0x10001 are actually one event for hardware 103 * because bit 0-15 are same. If the bit 16 of one event is 0 means to read 104 * counter register, otherwise means to read ext_counter register. 105 */ 106 /* bandwidth events */ 107 #define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM 0x00001 108 #define HNS3_PMU_EVT_BW_SSU_EGU_TIME 0x10001 109 #define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM 0x00002 110 #define HNS3_PMU_EVT_BW_SSU_RPU_TIME 0x10002 111 #define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM 0x00003 112 #define HNS3_PMU_EVT_BW_SSU_ROCE_TIME 0x10003 113 #define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM 0x00004 114 #define HNS3_PMU_EVT_BW_ROCE_SSU_TIME 0x10004 115 #define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM 0x00005 116 #define HNS3_PMU_EVT_BW_TPU_SSU_TIME 0x10005 117 #define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM 0x00006 118 #define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME 0x10006 119 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM 0x00008 120 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME 0x10008 121 #define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM 0x00009 122 #define HNS3_PMU_EVT_BW_WR_FBD_TIME 0x10009 123 #define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM 0x0000a 124 #define HNS3_PMU_EVT_BW_WR_EBD_TIME 0x1000a 125 #define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM 0x0000b 126 #define HNS3_PMU_EVT_BW_RD_FBD_TIME 0x1000b 127 #define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM 0x0000c 128 #define HNS3_PMU_EVT_BW_RD_EBD_TIME 0x1000c 129 #define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM 0x0000d 130 #define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME 0x1000d 131 #define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM 0x0000e 132 #define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME 0x1000e 133 #define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM 0x0000f 134 #define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME 0x1000f 135 #define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM 0x00010 136 #define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME 0x10010 137 138 /* packet rate events */ 139 #define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM 0x00100 140 #define HNS3_PMU_EVT_PPS_IGU_SSU_TIME 0x10100 141 #define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM 0x00101 142 #define HNS3_PMU_EVT_PPS_SSU_EGU_TIME 0x10101 143 #define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM 0x00102 144 #define HNS3_PMU_EVT_PPS_SSU_RPU_TIME 0x10102 145 #define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM 0x00103 146 #define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME 0x10103 147 #define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM 0x00104 148 #define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME 0x10104 149 #define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM 0x00105 150 #define HNS3_PMU_EVT_PPS_TPU_SSU_TIME 0x10105 151 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM 0x00106 152 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME 0x10106 153 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM 0x00107 154 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME 0x10107 155 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM 0x00108 156 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME 0x10108 157 #define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM 0x00109 158 #define HNS3_PMU_EVT_PPS_WR_FBD_TIME 0x10109 159 #define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM 0x0010a 160 #define HNS3_PMU_EVT_PPS_WR_EBD_TIME 0x1010a 161 #define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM 0x0010b 162 #define HNS3_PMU_EVT_PPS_RD_FBD_TIME 0x1010b 163 #define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM 0x0010c 164 #define HNS3_PMU_EVT_PPS_RD_EBD_TIME 0x1010c 165 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM 0x0010d 166 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME 0x1010d 167 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM 0x0010e 168 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME 0x1010e 169 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM 0x0010f 170 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME 0x1010f 171 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM 0x00110 172 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME 0x10110 173 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM 0x00111 174 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME 0x10111 175 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM 0x00112 176 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME 0x10112 177 178 /* latency events */ 179 #define HNS3_PMU_EVT_DLY_TX_PUSH_TIME 0x00202 180 #define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM 0x10202 181 #define HNS3_PMU_EVT_DLY_TX_TIME 0x00204 182 #define HNS3_PMU_EVT_DLY_TX_PACKET_NUM 0x10204 183 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME 0x00206 184 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM 0x10206 185 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME 0x00207 186 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM 0x10207 187 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME 0x00208 188 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM 0x10208 189 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME 0x00209 190 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM 0x10209 191 #define HNS3_PMU_EVT_DLY_RPU_TIME 0x0020e 192 #define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM 0x1020e 193 #define HNS3_PMU_EVT_DLY_TPU_TIME 0x0020f 194 #define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM 0x1020f 195 #define HNS3_PMU_EVT_DLY_RPE_TIME 0x00210 196 #define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM 0x10210 197 #define HNS3_PMU_EVT_DLY_TPE_TIME 0x00211 198 #define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM 0x10211 199 #define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME 0x00212 200 #define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM 0x10212 201 #define HNS3_PMU_EVT_DLY_WR_FBD_TIME 0x00213 202 #define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM 0x10213 203 #define HNS3_PMU_EVT_DLY_WR_EBD_TIME 0x00214 204 #define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM 0x10214 205 #define HNS3_PMU_EVT_DLY_RD_FBD_TIME 0x00215 206 #define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM 0x10215 207 #define HNS3_PMU_EVT_DLY_RD_EBD_TIME 0x00216 208 #define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM 0x10216 209 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME 0x00217 210 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM 0x10217 211 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME 0x00218 212 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM 0x10218 213 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME 0x00219 214 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM 0x10219 215 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME 0x0021a 216 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM 0x1021a 217 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME 0x0021c 218 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM 0x1021c 219 220 /* interrupt rate events */ 221 #define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM 0x00300 222 #define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME 0x10300 223 224 /* filter mode supported by each bandwidth event */ 225 #define HNS3_PMU_FILTER_BW_SSU_EGU 0x07 226 #define HNS3_PMU_FILTER_BW_SSU_RPU 0x1f 227 #define HNS3_PMU_FILTER_BW_SSU_ROCE 0x0f 228 #define HNS3_PMU_FILTER_BW_ROCE_SSU 0x0f 229 #define HNS3_PMU_FILTER_BW_TPU_SSU 0x1f 230 #define HNS3_PMU_FILTER_BW_RPU_RCBRX 0x11 231 #define HNS3_PMU_FILTER_BW_RCBTX_TXSCH 0x11 232 #define HNS3_PMU_FILTER_BW_WR_FBD 0x1b 233 #define HNS3_PMU_FILTER_BW_WR_EBD 0x11 234 #define HNS3_PMU_FILTER_BW_RD_FBD 0x01 235 #define HNS3_PMU_FILTER_BW_RD_EBD 0x1b 236 #define HNS3_PMU_FILTER_BW_RD_PAY_M0 0x01 237 #define HNS3_PMU_FILTER_BW_RD_PAY_M1 0x01 238 #define HNS3_PMU_FILTER_BW_WR_PAY_M0 0x01 239 #define HNS3_PMU_FILTER_BW_WR_PAY_M1 0x01 240 241 /* filter mode supported by each packet rate event */ 242 #define HNS3_PMU_FILTER_PPS_IGU_SSU 0x07 243 #define HNS3_PMU_FILTER_PPS_SSU_EGU 0x07 244 #define HNS3_PMU_FILTER_PPS_SSU_RPU 0x1f 245 #define HNS3_PMU_FILTER_PPS_SSU_ROCE 0x0f 246 #define HNS3_PMU_FILTER_PPS_ROCE_SSU 0x0f 247 #define HNS3_PMU_FILTER_PPS_TPU_SSU 0x1f 248 #define HNS3_PMU_FILTER_PPS_RPU_RCBRX 0x11 249 #define HNS3_PMU_FILTER_PPS_RCBTX_TPU 0x1f 250 #define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH 0x11 251 #define HNS3_PMU_FILTER_PPS_WR_FBD 0x1b 252 #define HNS3_PMU_FILTER_PPS_WR_EBD 0x11 253 #define HNS3_PMU_FILTER_PPS_RD_FBD 0x01 254 #define HNS3_PMU_FILTER_PPS_RD_EBD 0x1b 255 #define HNS3_PMU_FILTER_PPS_RD_PAY_M0 0x01 256 #define HNS3_PMU_FILTER_PPS_RD_PAY_M1 0x01 257 #define HNS3_PMU_FILTER_PPS_WR_PAY_M0 0x01 258 #define HNS3_PMU_FILTER_PPS_WR_PAY_M1 0x01 259 #define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE 0x01 260 #define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE 0x01 261 262 /* filter mode supported by each latency event */ 263 #define HNS3_PMU_FILTER_DLY_TX_PUSH 0x01 264 #define HNS3_PMU_FILTER_DLY_TX 0x01 265 #define HNS3_PMU_FILTER_DLY_SSU_TX_NIC 0x07 266 #define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE 0x07 267 #define HNS3_PMU_FILTER_DLY_SSU_RX_NIC 0x07 268 #define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE 0x07 269 #define HNS3_PMU_FILTER_DLY_RPU 0x11 270 #define HNS3_PMU_FILTER_DLY_TPU 0x1f 271 #define HNS3_PMU_FILTER_DLY_RPE 0x01 272 #define HNS3_PMU_FILTER_DLY_TPE 0x0b 273 #define HNS3_PMU_FILTER_DLY_TPE_PUSH 0x1b 274 #define HNS3_PMU_FILTER_DLY_WR_FBD 0x1b 275 #define HNS3_PMU_FILTER_DLY_WR_EBD 0x11 276 #define HNS3_PMU_FILTER_DLY_RD_FBD 0x01 277 #define HNS3_PMU_FILTER_DLY_RD_EBD 0x1b 278 #define HNS3_PMU_FILTER_DLY_RD_PAY_M0 0x01 279 #define HNS3_PMU_FILTER_DLY_RD_PAY_M1 0x01 280 #define HNS3_PMU_FILTER_DLY_WR_PAY_M0 0x01 281 #define HNS3_PMU_FILTER_DLY_WR_PAY_M1 0x01 282 #define HNS3_PMU_FILTER_DLY_MSIX_WRITE 0x01 283 284 /* filter mode supported by each interrupt rate event */ 285 #define HNS3_PMU_FILTER_INTR_MSIX_NIC 0x01 286 287 enum hns3_pmu_hw_filter_mode { 288 HNS3_PMU_HW_FILTER_GLOBAL, 289 HNS3_PMU_HW_FILTER_PORT, 290 HNS3_PMU_HW_FILTER_PORT_TC, 291 HNS3_PMU_HW_FILTER_FUNC, 292 HNS3_PMU_HW_FILTER_FUNC_QUEUE, 293 HNS3_PMU_HW_FILTER_FUNC_INTR, 294 }; 295 296 struct hns3_pmu_event_attr { 297 u32 event; 298 u16 filter_support; 299 }; 300 301 struct hns3_pmu { 302 struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS]; 303 struct hlist_node node; 304 struct pci_dev *pdev; 305 struct pmu pmu; 306 void __iomem *base; 307 int irq; 308 int on_cpu; 309 u32 identifier; 310 u32 hw_clk_freq; /* hardware clock frequency of PMU */ 311 /* maximum and minimum bdf allowed by PMU */ 312 u16 bdf_min; 313 u16 bdf_max; 314 }; 315 316 #define to_hns3_pmu(p) (container_of((p), struct hns3_pmu, pmu)) 317 318 #define GET_PCI_DEVFN(bdf) ((bdf) & 0xff) 319 320 #define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff) 321 #define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07)) 322 #define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func)) 323 324 #define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end) \ 325 static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \ 326 { \ 327 return FIELD_GET(GENMASK_ULL(_end, _start), \ 328 event->attr._config); \ 329 } 330 331 HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7); 332 HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15); 333 HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16); 334 HNS3_PMU_FILTER_ATTR(port, config1, 0, 3); 335 HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7); 336 HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23); 337 HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39); 338 HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51); 339 HNS3_PMU_FILTER_ATTR(global, config1, 52, 52); 340 341 #define HNS3_BW_EVT_BYTE_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 342 HNS3_PMU_EVT_BW_##_name##_BYTE_NUM, \ 343 HNS3_PMU_FILTER_BW_##_name}) 344 #define HNS3_BW_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 345 HNS3_PMU_EVT_BW_##_name##_TIME, \ 346 HNS3_PMU_FILTER_BW_##_name}) 347 #define HNS3_PPS_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 348 HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM, \ 349 HNS3_PMU_FILTER_PPS_##_name}) 350 #define HNS3_PPS_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 351 HNS3_PMU_EVT_PPS_##_name##_TIME, \ 352 HNS3_PMU_FILTER_PPS_##_name}) 353 #define HNS3_DLY_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 354 HNS3_PMU_EVT_DLY_##_name##_TIME, \ 355 HNS3_PMU_FILTER_DLY_##_name}) 356 #define HNS3_DLY_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 357 HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM, \ 358 HNS3_PMU_FILTER_DLY_##_name}) 359 #define HNS3_INTR_EVT_INTR_NUM(_name) (&(struct hns3_pmu_event_attr) {\ 360 HNS3_PMU_EVT_PPS_##_name##_INTR_NUM, \ 361 HNS3_PMU_FILTER_INTR_##_name}) 362 #define HNS3_INTR_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\ 363 HNS3_PMU_EVT_PPS_##_name##_TIME, \ 364 HNS3_PMU_FILTER_INTR_##_name}) 365 366 static ssize_t hns3_pmu_format_show(struct device *dev, 367 struct device_attribute *attr, char *buf) 368 { 369 struct dev_ext_attribute *eattr; 370 371 eattr = container_of(attr, struct dev_ext_attribute, attr); 372 373 return sysfs_emit(buf, "%s\n", (char *)eattr->var); 374 } 375 376 static ssize_t hns3_pmu_event_show(struct device *dev, 377 struct device_attribute *attr, char *buf) 378 { 379 struct hns3_pmu_event_attr *event; 380 struct dev_ext_attribute *eattr; 381 382 eattr = container_of(attr, struct dev_ext_attribute, attr); 383 event = eattr->var; 384 385 return sysfs_emit(buf, "config=0x%x\n", event->event); 386 } 387 388 static ssize_t hns3_pmu_filter_mode_show(struct device *dev, 389 struct device_attribute *attr, 390 char *buf) 391 { 392 struct hns3_pmu_event_attr *event; 393 struct dev_ext_attribute *eattr; 394 int len; 395 396 eattr = container_of(attr, struct dev_ext_attribute, attr); 397 event = eattr->var; 398 399 len = sysfs_emit_at(buf, 0, "filter mode supported: "); 400 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL) 401 len += sysfs_emit_at(buf, len, "global "); 402 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT) 403 len += sysfs_emit_at(buf, len, "port "); 404 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC) 405 len += sysfs_emit_at(buf, len, "port-tc "); 406 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC) 407 len += sysfs_emit_at(buf, len, "func "); 408 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE) 409 len += sysfs_emit_at(buf, len, "func-queue "); 410 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR) 411 len += sysfs_emit_at(buf, len, "func-intr "); 412 413 len += sysfs_emit_at(buf, len, "\n"); 414 415 return len; 416 } 417 418 #define HNS3_PMU_ATTR(_name, _func, _config) \ 419 (&((struct dev_ext_attribute[]) { \ 420 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ 421 })[0].attr.attr) 422 423 #define HNS3_PMU_FORMAT_ATTR(_name, _format) \ 424 HNS3_PMU_ATTR(_name, hns3_pmu_format_show, (void *)_format) 425 #define HNS3_PMU_EVENT_ATTR(_name, _event) \ 426 HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event) 427 #define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \ 428 HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event) 429 430 #define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \ 431 HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \ 432 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro)) 433 #define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \ 434 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \ 435 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro)) 436 #define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \ 437 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \ 438 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro)) 439 #define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \ 440 HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \ 441 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro)) 442 443 #define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \ 444 HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \ 445 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro)) 446 #define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \ 447 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \ 448 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro)) 449 #define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \ 450 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \ 451 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro)) 452 #define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \ 453 HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \ 454 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro)) 455 456 static u8 hns3_pmu_hw_filter_modes[] = { 457 HNS3_PMU_HW_FILTER_GLOBAL, 458 HNS3_PMU_HW_FILTER_PORT, 459 HNS3_PMU_HW_FILTER_PORT_TC, 460 HNS3_PMU_HW_FILTER_FUNC, 461 HNS3_PMU_HW_FILTER_FUNC_QUEUE, 462 HNS3_PMU_HW_FILTER_FUNC_INTR, 463 }; 464 465 #define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \ 466 ((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)]) 467 468 static ssize_t identifier_show(struct device *dev, 469 struct device_attribute *attr, char *buf) 470 { 471 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 472 473 return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier); 474 } 475 static DEVICE_ATTR_RO(identifier); 476 477 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 478 char *buf) 479 { 480 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 481 482 return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu); 483 } 484 static DEVICE_ATTR_RO(cpumask); 485 486 static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr, 487 char *buf) 488 { 489 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 490 u16 bdf = hns3_pmu->bdf_min; 491 492 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf), 493 PCI_SLOT(bdf), PCI_FUNC(bdf)); 494 } 495 static DEVICE_ATTR_RO(bdf_min); 496 497 static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr, 498 char *buf) 499 { 500 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 501 u16 bdf = hns3_pmu->bdf_max; 502 503 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf), 504 PCI_SLOT(bdf), PCI_FUNC(bdf)); 505 } 506 static DEVICE_ATTR_RO(bdf_max); 507 508 static ssize_t hw_clk_freq_show(struct device *dev, 509 struct device_attribute *attr, char *buf) 510 { 511 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev)); 512 513 return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq); 514 } 515 static DEVICE_ATTR_RO(hw_clk_freq); 516 517 static struct attribute *hns3_pmu_events_attr[] = { 518 /* bandwidth events */ 519 HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU), 520 HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU), 521 HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE), 522 HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU), 523 HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU), 524 HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX), 525 HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH), 526 HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD), 527 HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD), 528 HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD), 529 HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD), 530 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0), 531 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1), 532 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0), 533 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1), 534 535 /* packet rate events */ 536 HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU), 537 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU), 538 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU), 539 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE), 540 HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU), 541 HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU), 542 HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX), 543 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU), 544 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH), 545 HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD), 546 HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD), 547 HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD), 548 HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD), 549 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0), 550 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1), 551 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0), 552 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1), 553 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE), 554 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE), 555 556 /* latency events */ 557 HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH), 558 HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX), 559 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC), 560 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE), 561 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC), 562 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE), 563 HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU), 564 HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU), 565 HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE), 566 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE), 567 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH), 568 HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD), 569 HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD), 570 HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD), 571 HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD), 572 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0), 573 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1), 574 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0), 575 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1), 576 HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE), 577 578 /* interrupt rate events */ 579 HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC), 580 581 NULL 582 }; 583 584 static struct attribute *hns3_pmu_filter_mode_attr[] = { 585 /* bandwidth events */ 586 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU), 587 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU), 588 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE), 589 HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU), 590 HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU), 591 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX), 592 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH), 593 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD), 594 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD), 595 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD), 596 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD), 597 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0), 598 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1), 599 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0), 600 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1), 601 602 /* packet rate events */ 603 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU), 604 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU), 605 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU), 606 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE), 607 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU), 608 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU), 609 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX), 610 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU), 611 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH), 612 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD), 613 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD), 614 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD), 615 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD), 616 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0), 617 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1), 618 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0), 619 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1), 620 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE), 621 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE), 622 623 /* latency events */ 624 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH), 625 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX), 626 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC), 627 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE), 628 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC), 629 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE), 630 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU), 631 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU), 632 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE), 633 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE), 634 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH), 635 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD), 636 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD), 637 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD), 638 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD), 639 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0), 640 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1), 641 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0), 642 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1), 643 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE), 644 645 /* interrupt rate events */ 646 HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC), 647 648 NULL 649 }; 650 651 static struct attribute_group hns3_pmu_events_group = { 652 .name = "events", 653 .attrs = hns3_pmu_events_attr, 654 }; 655 656 static struct attribute_group hns3_pmu_filter_mode_group = { 657 .name = "filtermode", 658 .attrs = hns3_pmu_filter_mode_attr, 659 }; 660 661 static struct attribute *hns3_pmu_format_attr[] = { 662 HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"), 663 HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"), 664 HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"), 665 HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"), 666 HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"), 667 HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"), 668 HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"), 669 HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"), 670 HNS3_PMU_FORMAT_ATTR(global, "config1:52"), 671 NULL 672 }; 673 674 static struct attribute_group hns3_pmu_format_group = { 675 .name = "format", 676 .attrs = hns3_pmu_format_attr, 677 }; 678 679 static struct attribute *hns3_pmu_cpumask_attrs[] = { 680 &dev_attr_cpumask.attr, 681 NULL 682 }; 683 684 static struct attribute_group hns3_pmu_cpumask_attr_group = { 685 .attrs = hns3_pmu_cpumask_attrs, 686 }; 687 688 static struct attribute *hns3_pmu_identifier_attrs[] = { 689 &dev_attr_identifier.attr, 690 NULL 691 }; 692 693 static struct attribute_group hns3_pmu_identifier_attr_group = { 694 .attrs = hns3_pmu_identifier_attrs, 695 }; 696 697 static struct attribute *hns3_pmu_bdf_range_attrs[] = { 698 &dev_attr_bdf_min.attr, 699 &dev_attr_bdf_max.attr, 700 NULL 701 }; 702 703 static struct attribute_group hns3_pmu_bdf_range_attr_group = { 704 .attrs = hns3_pmu_bdf_range_attrs, 705 }; 706 707 static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = { 708 &dev_attr_hw_clk_freq.attr, 709 NULL 710 }; 711 712 static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = { 713 .attrs = hns3_pmu_hw_clk_freq_attrs, 714 }; 715 716 static const struct attribute_group *hns3_pmu_attr_groups[] = { 717 &hns3_pmu_events_group, 718 &hns3_pmu_filter_mode_group, 719 &hns3_pmu_format_group, 720 &hns3_pmu_cpumask_attr_group, 721 &hns3_pmu_identifier_attr_group, 722 &hns3_pmu_bdf_range_attr_group, 723 &hns3_pmu_hw_clk_freq_attr_group, 724 NULL 725 }; 726 727 static u32 hns3_pmu_get_event(struct perf_event *event) 728 { 729 return hns3_pmu_get_ext_counter_used(event) << 16 | 730 hns3_pmu_get_event_type(event) << 8 | 731 hns3_pmu_get_subevent(event); 732 } 733 734 static u32 hns3_pmu_get_real_event(struct perf_event *event) 735 { 736 return hns3_pmu_get_event_type(event) << 8 | 737 hns3_pmu_get_subevent(event); 738 } 739 740 static u32 hns3_pmu_get_offset(u32 offset, u32 idx) 741 { 742 return offset + HNS3_PMU_REG_EVENT_OFFSET + 743 HNS3_PMU_REG_EVENT_SIZE * idx; 744 } 745 746 static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx) 747 { 748 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 749 750 return readl(hns3_pmu->base + offset); 751 } 752 753 static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx, 754 u32 val) 755 { 756 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 757 758 writel(val, hns3_pmu->base + offset); 759 } 760 761 static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx) 762 { 763 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 764 765 return readq(hns3_pmu->base + offset); 766 } 767 768 static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx, 769 u64 val) 770 { 771 u32 offset = hns3_pmu_get_offset(reg_offset, idx); 772 773 writeq(val, hns3_pmu->base + offset); 774 } 775 776 static bool hns3_pmu_cmp_event(struct perf_event *target, 777 struct perf_event *event) 778 { 779 return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event); 780 } 781 782 static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu, 783 struct perf_event *event) 784 { 785 struct perf_event *sibling; 786 int hw_event_used = 0; 787 int idx; 788 789 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 790 sibling = hns3_pmu->hw_events[idx]; 791 if (!sibling) 792 continue; 793 794 hw_event_used++; 795 796 if (!hns3_pmu_cmp_event(sibling, event)) 797 continue; 798 799 /* Related events is used in group */ 800 if (sibling->group_leader == event->group_leader) 801 return idx; 802 } 803 804 /* No related event and all hardware events are used up */ 805 if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS) 806 return -EBUSY; 807 808 /* No related event and there is extra hardware events can be use */ 809 return -ENOENT; 810 } 811 812 static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu) 813 { 814 int idx; 815 816 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 817 if (!hns3_pmu->hw_events[idx]) 818 return idx; 819 } 820 821 return -EBUSY; 822 } 823 824 static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf) 825 { 826 struct pci_dev *pdev; 827 828 if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) { 829 pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf); 830 return false; 831 } 832 833 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus), 834 PCI_BUS_NUM(bdf), 835 GET_PCI_DEVFN(bdf)); 836 if (!pdev) { 837 pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf); 838 return false; 839 } 840 841 pci_dev_put(pdev); 842 return true; 843 } 844 845 static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf, 846 u16 queue) 847 { 848 u32 val; 849 850 val = GET_PCI_DEVFN(bdf); 851 val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S; 852 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val); 853 } 854 855 static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx) 856 { 857 bool queue_id_valid = false; 858 u32 reg_qid_ctrl, val; 859 int err; 860 861 /* enable queue id request */ 862 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 863 HNS3_PMU_QID_CTRL_REQ_ENABLE); 864 865 reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx); 866 err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val, 867 val & HNS3_PMU_QID_CTRL_DONE, 1, 1000); 868 if (err == -ETIMEDOUT) { 869 pci_err(hns3_pmu->pdev, "QID request timeout!\n"); 870 goto out; 871 } 872 873 queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS); 874 875 out: 876 /* disable qid request and clear status */ 877 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0); 878 879 return queue_id_valid; 880 } 881 882 static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf, 883 u16 queue) 884 { 885 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue); 886 887 return hns3_pmu_qid_req_start(hns3_pmu, idx); 888 } 889 890 static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event) 891 { 892 struct hns3_pmu_event_attr *pmu_event; 893 struct dev_ext_attribute *eattr; 894 struct device_attribute *dattr; 895 struct attribute *attr; 896 u32 i; 897 898 for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) { 899 attr = hns3_pmu_events_attr[i]; 900 dattr = container_of(attr, struct device_attribute, attr); 901 eattr = container_of(dattr, struct dev_ext_attribute, attr); 902 pmu_event = eattr->var; 903 904 if (event == pmu_event->event) 905 return pmu_event; 906 } 907 908 return NULL; 909 } 910 911 static int hns3_pmu_set_func_mode(struct perf_event *event, 912 struct hns3_pmu *hns3_pmu) 913 { 914 struct hw_perf_event *hwc = &event->hw; 915 u16 bdf = hns3_pmu_get_bdf(event); 916 917 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf)) 918 return -ENOENT; 919 920 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC); 921 922 return 0; 923 } 924 925 static int hns3_pmu_set_func_queue_mode(struct perf_event *event, 926 struct hns3_pmu *hns3_pmu) 927 { 928 u16 queue_id = hns3_pmu_get_queue(event); 929 struct hw_perf_event *hwc = &event->hw; 930 u16 bdf = hns3_pmu_get_bdf(event); 931 932 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf)) 933 return -ENOENT; 934 935 if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) { 936 pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id); 937 return -ENOENT; 938 } 939 940 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE); 941 942 return 0; 943 } 944 945 static bool 946 hns3_pmu_is_enabled_global_mode(struct perf_event *event, 947 struct hns3_pmu_event_attr *pmu_event) 948 { 949 u8 global = hns3_pmu_get_global(event); 950 951 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)) 952 return false; 953 954 return global; 955 } 956 957 static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event, 958 struct hns3_pmu_event_attr *pmu_event) 959 { 960 u16 queue_id = hns3_pmu_get_queue(event); 961 u16 bdf = hns3_pmu_get_bdf(event); 962 963 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)) 964 return false; 965 else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE) 966 return false; 967 968 return bdf; 969 } 970 971 static bool 972 hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event, 973 struct hns3_pmu_event_attr *pmu_event) 974 { 975 u16 queue_id = hns3_pmu_get_queue(event); 976 u16 bdf = hns3_pmu_get_bdf(event); 977 978 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)) 979 return false; 980 else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE) 981 return false; 982 983 return bdf; 984 } 985 986 static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event, 987 struct hns3_pmu_event_attr *pmu_event) 988 { 989 u8 tc_id = hns3_pmu_get_tc(event); 990 991 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)) 992 return false; 993 994 return tc_id == HNS3_PMU_FILTER_ALL_TC; 995 } 996 997 static bool 998 hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event, 999 struct hns3_pmu_event_attr *pmu_event) 1000 { 1001 u8 tc_id = hns3_pmu_get_tc(event); 1002 1003 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)) 1004 return false; 1005 1006 return tc_id != HNS3_PMU_FILTER_ALL_TC; 1007 } 1008 1009 static bool 1010 hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event, 1011 struct hns3_pmu *hns3_pmu, 1012 struct hns3_pmu_event_attr *pmu_event) 1013 { 1014 u16 bdf = hns3_pmu_get_bdf(event); 1015 1016 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)) 1017 return false; 1018 1019 return hns3_pmu_valid_bdf(hns3_pmu, bdf); 1020 } 1021 1022 static int hns3_pmu_select_filter_mode(struct perf_event *event, 1023 struct hns3_pmu *hns3_pmu) 1024 { 1025 u32 event_id = hns3_pmu_get_event(event); 1026 struct hw_perf_event *hwc = &event->hw; 1027 struct hns3_pmu_event_attr *pmu_event; 1028 1029 pmu_event = hns3_pmu_get_pmu_event(event_id); 1030 if (!pmu_event) { 1031 pci_err(hns3_pmu->pdev, "Invalid pmu event\n"); 1032 return -ENOENT; 1033 } 1034 1035 if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) { 1036 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL); 1037 return 0; 1038 } 1039 1040 if (hns3_pmu_is_enabled_func_mode(event, pmu_event)) 1041 return hns3_pmu_set_func_mode(event, hns3_pmu); 1042 1043 if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event)) 1044 return hns3_pmu_set_func_queue_mode(event, hns3_pmu); 1045 1046 if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) { 1047 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT); 1048 return 0; 1049 } 1050 1051 if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) { 1052 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC); 1053 return 0; 1054 } 1055 1056 if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) { 1057 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR); 1058 return 0; 1059 } 1060 1061 return -ENOENT; 1062 } 1063 1064 static bool hns3_pmu_validate_event_group(struct perf_event *event) 1065 { 1066 struct perf_event *sibling, *leader = event->group_leader; 1067 struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS]; 1068 int counters = 1; 1069 int num; 1070 1071 event_group[0] = leader; 1072 if (!is_software_event(leader)) { 1073 if (leader->pmu != event->pmu) 1074 return false; 1075 1076 if (leader != event && !hns3_pmu_cmp_event(leader, event)) 1077 event_group[counters++] = event; 1078 } 1079 1080 for_each_sibling_event(sibling, event->group_leader) { 1081 if (is_software_event(sibling)) 1082 continue; 1083 1084 if (sibling->pmu != event->pmu) 1085 return false; 1086 1087 for (num = 0; num < counters; num++) { 1088 /* 1089 * If we find a related event, then it's a valid group 1090 * since we don't need to allocate a new counter for it. 1091 */ 1092 if (hns3_pmu_cmp_event(event_group[num], sibling)) 1093 break; 1094 } 1095 1096 /* 1097 * Otherwise it's a new event but if there's no available counter, 1098 * fail the check since we cannot schedule all the events in 1099 * the group simultaneously. 1100 */ 1101 if (num == HNS3_PMU_MAX_HW_EVENTS) 1102 return false; 1103 1104 if (num == counters) 1105 event_group[counters++] = sibling; 1106 } 1107 1108 return true; 1109 } 1110 1111 static u32 hns3_pmu_get_filter_condition(struct perf_event *event) 1112 { 1113 struct hw_perf_event *hwc = &event->hw; 1114 u16 intr_id = hns3_pmu_get_intr(event); 1115 u8 port_id = hns3_pmu_get_port(event); 1116 u16 bdf = hns3_pmu_get_bdf(event); 1117 u8 tc_id = hns3_pmu_get_tc(event); 1118 u8 filter_mode; 1119 1120 filter_mode = *(u8 *)hwc->addr_filters; 1121 switch (filter_mode) { 1122 case HNS3_PMU_HW_FILTER_PORT: 1123 return FILTER_CONDITION_PORT(port_id); 1124 case HNS3_PMU_HW_FILTER_PORT_TC: 1125 return FILTER_CONDITION_PORT_TC(port_id, tc_id); 1126 case HNS3_PMU_HW_FILTER_FUNC: 1127 case HNS3_PMU_HW_FILTER_FUNC_QUEUE: 1128 return GET_PCI_DEVFN(bdf); 1129 case HNS3_PMU_HW_FILTER_FUNC_INTR: 1130 return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id); 1131 default: 1132 break; 1133 } 1134 1135 return 0; 1136 } 1137 1138 static void hns3_pmu_config_filter(struct perf_event *event) 1139 { 1140 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1141 u8 event_type = hns3_pmu_get_event_type(event); 1142 u8 subevent_id = hns3_pmu_get_subevent(event); 1143 u16 queue_id = hns3_pmu_get_queue(event); 1144 struct hw_perf_event *hwc = &event->hw; 1145 u8 filter_mode = *(u8 *)hwc->addr_filters; 1146 u16 bdf = hns3_pmu_get_bdf(event); 1147 u32 idx = hwc->idx; 1148 u32 val; 1149 1150 val = event_type; 1151 val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S; 1152 val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S; 1153 val |= HNS3_PMU_EVENT_OVERFLOW_RESTART; 1154 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1155 1156 val = hns3_pmu_get_filter_condition(event); 1157 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val); 1158 1159 if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE) 1160 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id); 1161 } 1162 1163 static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu, 1164 struct hw_perf_event *hwc) 1165 { 1166 u32 idx = hwc->idx; 1167 u32 val; 1168 1169 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1170 val |= HNS3_PMU_EVENT_EN; 1171 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1172 } 1173 1174 static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu, 1175 struct hw_perf_event *hwc) 1176 { 1177 u32 idx = hwc->idx; 1178 u32 val; 1179 1180 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1181 val &= ~HNS3_PMU_EVENT_EN; 1182 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1183 } 1184 1185 static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu, 1186 struct hw_perf_event *hwc) 1187 { 1188 u32 idx = hwc->idx; 1189 u32 val; 1190 1191 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx); 1192 val &= ~HNS3_PMU_INTR_MASK_OVERFLOW; 1193 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val); 1194 } 1195 1196 static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu, 1197 struct hw_perf_event *hwc) 1198 { 1199 u32 idx = hwc->idx; 1200 u32 val; 1201 1202 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx); 1203 val |= HNS3_PMU_INTR_MASK_OVERFLOW; 1204 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val); 1205 } 1206 1207 static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx) 1208 { 1209 u32 val; 1210 1211 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1212 val |= HNS3_PMU_EVENT_STATUS_RESET; 1213 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1214 1215 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx); 1216 val &= ~HNS3_PMU_EVENT_STATUS_RESET; 1217 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val); 1218 } 1219 1220 static u64 hns3_pmu_read_counter(struct perf_event *event) 1221 { 1222 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1223 1224 return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx); 1225 } 1226 1227 static void hns3_pmu_write_counter(struct perf_event *event, u64 value) 1228 { 1229 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1230 u32 idx = event->hw.idx; 1231 1232 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value); 1233 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value); 1234 } 1235 1236 static void hns3_pmu_init_counter(struct perf_event *event) 1237 { 1238 struct hw_perf_event *hwc = &event->hw; 1239 1240 local64_set(&hwc->prev_count, 0); 1241 hns3_pmu_write_counter(event, 0); 1242 } 1243 1244 static int hns3_pmu_event_init(struct perf_event *event) 1245 { 1246 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1247 struct hw_perf_event *hwc = &event->hw; 1248 int idx; 1249 int ret; 1250 1251 if (event->attr.type != event->pmu->type) 1252 return -ENOENT; 1253 1254 /* Sampling is not supported */ 1255 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1256 return -EOPNOTSUPP; 1257 1258 event->cpu = hns3_pmu->on_cpu; 1259 1260 idx = hns3_pmu_get_event_idx(hns3_pmu); 1261 if (idx < 0) { 1262 pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n", 1263 HNS3_PMU_MAX_HW_EVENTS); 1264 return -EBUSY; 1265 } 1266 1267 hwc->idx = idx; 1268 1269 ret = hns3_pmu_select_filter_mode(event, hns3_pmu); 1270 if (ret) { 1271 pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret); 1272 return ret; 1273 } 1274 1275 if (!hns3_pmu_validate_event_group(event)) { 1276 pci_err(hns3_pmu->pdev, "Invalid event group.\n"); 1277 return -EINVAL; 1278 } 1279 1280 if (hns3_pmu_get_ext_counter_used(event)) 1281 hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER; 1282 else 1283 hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER; 1284 1285 return 0; 1286 } 1287 1288 static void hns3_pmu_read(struct perf_event *event) 1289 { 1290 struct hw_perf_event *hwc = &event->hw; 1291 u64 new_cnt, prev_cnt, delta; 1292 1293 do { 1294 prev_cnt = local64_read(&hwc->prev_count); 1295 new_cnt = hns3_pmu_read_counter(event); 1296 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) != 1297 prev_cnt); 1298 1299 delta = new_cnt - prev_cnt; 1300 local64_add(delta, &event->count); 1301 } 1302 1303 static void hns3_pmu_start(struct perf_event *event, int flags) 1304 { 1305 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1306 struct hw_perf_event *hwc = &event->hw; 1307 1308 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 1309 return; 1310 1311 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 1312 hwc->state = 0; 1313 1314 hns3_pmu_config_filter(event); 1315 hns3_pmu_init_counter(event); 1316 hns3_pmu_enable_intr(hns3_pmu, hwc); 1317 hns3_pmu_enable_counter(hns3_pmu, hwc); 1318 1319 perf_event_update_userpage(event); 1320 } 1321 1322 static void hns3_pmu_stop(struct perf_event *event, int flags) 1323 { 1324 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1325 struct hw_perf_event *hwc = &event->hw; 1326 1327 hns3_pmu_disable_counter(hns3_pmu, hwc); 1328 hns3_pmu_disable_intr(hns3_pmu, hwc); 1329 1330 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 1331 hwc->state |= PERF_HES_STOPPED; 1332 1333 if (hwc->state & PERF_HES_UPTODATE) 1334 return; 1335 1336 /* Read hardware counter and update the perf counter statistics */ 1337 hns3_pmu_read(event); 1338 hwc->state |= PERF_HES_UPTODATE; 1339 } 1340 1341 static int hns3_pmu_add(struct perf_event *event, int flags) 1342 { 1343 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1344 struct hw_perf_event *hwc = &event->hw; 1345 int idx; 1346 1347 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1348 1349 /* Check all working events to find a related event. */ 1350 idx = hns3_pmu_find_related_event_idx(hns3_pmu, event); 1351 if (idx < 0 && idx != -ENOENT) 1352 return idx; 1353 1354 /* Current event shares an enabled hardware event with related event */ 1355 if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) { 1356 hwc->idx = idx; 1357 goto start_count; 1358 } 1359 1360 idx = hns3_pmu_get_event_idx(hns3_pmu); 1361 if (idx < 0) 1362 return idx; 1363 1364 hwc->idx = idx; 1365 hns3_pmu->hw_events[idx] = event; 1366 1367 start_count: 1368 if (flags & PERF_EF_START) 1369 hns3_pmu_start(event, PERF_EF_RELOAD); 1370 1371 return 0; 1372 } 1373 1374 static void hns3_pmu_del(struct perf_event *event, int flags) 1375 { 1376 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu); 1377 struct hw_perf_event *hwc = &event->hw; 1378 1379 hns3_pmu_stop(event, PERF_EF_UPDATE); 1380 hns3_pmu->hw_events[hwc->idx] = NULL; 1381 perf_event_update_userpage(event); 1382 } 1383 1384 static void hns3_pmu_enable(struct pmu *pmu) 1385 { 1386 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu); 1387 u32 val; 1388 1389 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1390 val |= HNS3_PMU_GLOBAL_START; 1391 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1392 } 1393 1394 static void hns3_pmu_disable(struct pmu *pmu) 1395 { 1396 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu); 1397 u32 val; 1398 1399 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1400 val &= ~HNS3_PMU_GLOBAL_START; 1401 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL); 1402 } 1403 1404 static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu) 1405 { 1406 u16 device_id; 1407 char *name; 1408 u32 val; 1409 1410 hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2]; 1411 if (!hns3_pmu->base) { 1412 pci_err(pdev, "ioremap failed\n"); 1413 return -ENOMEM; 1414 } 1415 1416 hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ); 1417 1418 val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF); 1419 hns3_pmu->bdf_min = val & 0xffff; 1420 hns3_pmu->bdf_max = val >> 16; 1421 1422 val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID); 1423 device_id = val & 0xffff; 1424 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id); 1425 if (!name) 1426 return -ENOMEM; 1427 1428 hns3_pmu->pdev = pdev; 1429 hns3_pmu->on_cpu = -1; 1430 hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION); 1431 hns3_pmu->pmu = (struct pmu) { 1432 .name = name, 1433 .module = THIS_MODULE, 1434 .event_init = hns3_pmu_event_init, 1435 .pmu_enable = hns3_pmu_enable, 1436 .pmu_disable = hns3_pmu_disable, 1437 .add = hns3_pmu_add, 1438 .del = hns3_pmu_del, 1439 .start = hns3_pmu_start, 1440 .stop = hns3_pmu_stop, 1441 .read = hns3_pmu_read, 1442 .task_ctx_nr = perf_invalid_context, 1443 .attr_groups = hns3_pmu_attr_groups, 1444 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1445 }; 1446 1447 return 0; 1448 } 1449 1450 static irqreturn_t hns3_pmu_irq(int irq, void *data) 1451 { 1452 struct hns3_pmu *hns3_pmu = data; 1453 u32 intr_status, idx; 1454 1455 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) { 1456 intr_status = hns3_pmu_readl(hns3_pmu, 1457 HNS3_PMU_REG_EVENT_INTR_STATUS, 1458 idx); 1459 1460 /* 1461 * As each counter will restart from 0 when it is overflowed, 1462 * extra processing is no need, just clear interrupt status. 1463 */ 1464 if (intr_status) 1465 hns3_pmu_clear_intr_status(hns3_pmu, idx); 1466 } 1467 1468 return IRQ_HANDLED; 1469 } 1470 1471 static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 1472 { 1473 struct hns3_pmu *hns3_pmu; 1474 1475 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node); 1476 if (!hns3_pmu) 1477 return -ENODEV; 1478 1479 if (hns3_pmu->on_cpu == -1) { 1480 hns3_pmu->on_cpu = cpu; 1481 irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu)); 1482 } 1483 1484 return 0; 1485 } 1486 1487 static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1488 { 1489 struct hns3_pmu *hns3_pmu; 1490 unsigned int target; 1491 1492 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node); 1493 if (!hns3_pmu) 1494 return -ENODEV; 1495 1496 /* Nothing to do if this CPU doesn't own the PMU */ 1497 if (hns3_pmu->on_cpu != cpu) 1498 return 0; 1499 1500 /* Choose a new CPU from all online cpus */ 1501 target = cpumask_any_but(cpu_online_mask, cpu); 1502 if (target >= nr_cpu_ids) 1503 return 0; 1504 1505 perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target); 1506 hns3_pmu->on_cpu = target; 1507 irq_set_affinity(hns3_pmu->irq, cpumask_of(target)); 1508 1509 return 0; 1510 } 1511 1512 static void hns3_pmu_free_irq(void *data) 1513 { 1514 struct pci_dev *pdev = data; 1515 1516 pci_free_irq_vectors(pdev); 1517 } 1518 1519 static int hns3_pmu_irq_register(struct pci_dev *pdev, 1520 struct hns3_pmu *hns3_pmu) 1521 { 1522 int irq, ret; 1523 1524 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); 1525 if (ret < 0) { 1526 pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret); 1527 return ret; 1528 } 1529 1530 ret = devm_add_action_or_reset(&pdev->dev, hns3_pmu_free_irq, pdev); 1531 if (ret) { 1532 pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret); 1533 return ret; 1534 } 1535 1536 irq = pci_irq_vector(pdev, 0); 1537 ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0, 1538 hns3_pmu->pmu.name, hns3_pmu); 1539 if (ret) { 1540 pci_err(pdev, "failed to register irq, ret = %d.\n", ret); 1541 return ret; 1542 } 1543 1544 hns3_pmu->irq = irq; 1545 1546 return 0; 1547 } 1548 1549 static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu) 1550 { 1551 int ret; 1552 1553 ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu); 1554 if (ret) 1555 return ret; 1556 1557 ret = hns3_pmu_irq_register(pdev, hns3_pmu); 1558 if (ret) 1559 return ret; 1560 1561 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1562 &hns3_pmu->node); 1563 if (ret) { 1564 pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret); 1565 return ret; 1566 } 1567 1568 ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1); 1569 if (ret) { 1570 pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret); 1571 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1572 &hns3_pmu->node); 1573 } 1574 1575 return ret; 1576 } 1577 1578 static void hns3_pmu_uninit_pmu(struct pci_dev *pdev) 1579 { 1580 struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev); 1581 1582 perf_pmu_unregister(&hns3_pmu->pmu); 1583 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1584 &hns3_pmu->node); 1585 } 1586 1587 static int hns3_pmu_init_dev(struct pci_dev *pdev) 1588 { 1589 int ret; 1590 1591 ret = pcim_enable_device(pdev); 1592 if (ret) { 1593 pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret); 1594 return ret; 1595 } 1596 1597 ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu"); 1598 if (ret < 0) { 1599 pci_err(pdev, "failed to request pci region, ret = %d.\n", ret); 1600 return ret; 1601 } 1602 1603 pci_set_master(pdev); 1604 1605 return 0; 1606 } 1607 1608 static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1609 { 1610 struct hns3_pmu *hns3_pmu; 1611 int ret; 1612 1613 hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL); 1614 if (!hns3_pmu) 1615 return -ENOMEM; 1616 1617 ret = hns3_pmu_init_dev(pdev); 1618 if (ret) 1619 return ret; 1620 1621 ret = hns3_pmu_init_pmu(pdev, hns3_pmu); 1622 if (ret) { 1623 pci_clear_master(pdev); 1624 return ret; 1625 } 1626 1627 pci_set_drvdata(pdev, hns3_pmu); 1628 1629 return ret; 1630 } 1631 1632 static void hns3_pmu_remove(struct pci_dev *pdev) 1633 { 1634 hns3_pmu_uninit_pmu(pdev); 1635 pci_clear_master(pdev); 1636 pci_set_drvdata(pdev, NULL); 1637 } 1638 1639 static const struct pci_device_id hns3_pmu_ids[] = { 1640 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) }, 1641 { 0, } 1642 }; 1643 MODULE_DEVICE_TABLE(pci, hns3_pmu_ids); 1644 1645 static struct pci_driver hns3_pmu_driver = { 1646 .name = "hns3_pmu", 1647 .id_table = hns3_pmu_ids, 1648 .probe = hns3_pmu_probe, 1649 .remove = hns3_pmu_remove, 1650 }; 1651 1652 static int __init hns3_pmu_module_init(void) 1653 { 1654 int ret; 1655 1656 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, 1657 "AP_PERF_ARM_HNS3_PMU_ONLINE", 1658 hns3_pmu_online_cpu, 1659 hns3_pmu_offline_cpu); 1660 if (ret) { 1661 pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret); 1662 return ret; 1663 } 1664 1665 ret = pci_register_driver(&hns3_pmu_driver); 1666 if (ret) { 1667 pr_err("failed to register pci driver, ret = %d.\n", ret); 1668 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE); 1669 } 1670 1671 return ret; 1672 } 1673 module_init(hns3_pmu_module_init); 1674 1675 static void __exit hns3_pmu_module_exit(void) 1676 { 1677 pci_unregister_driver(&hns3_pmu_driver); 1678 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE); 1679 } 1680 module_exit(hns3_pmu_module_exit); 1681 1682 MODULE_DESCRIPTION("HNS3 PMU driver"); 1683 MODULE_LICENSE("GPL v2"); 1684