1 // SPDX-License-Identifier: GPL-2.0-only OR Linux-OpenIB
2 /*
3  * Mellanox BlueField Performance Monitoring Counters driver
4  *
5  * This driver provides a sysfs interface for monitoring
6  * performance statistics in BlueField SoC.
7  *
8  * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
9  */
10 
11 #include <linux/acpi.h>
12 #include <linux/arm-smccc.h>
13 #include <linux/bitfield.h>
14 #include <linux/errno.h>
15 #include <linux/hwmon.h>
16 #include <linux/platform_device.h>
17 #include <linux/string.h>
18 #include <uapi/linux/psci.h>
19 
20 #define MLXBF_PMC_WRITE_REG_32 0x82000009
21 #define MLXBF_PMC_READ_REG_32 0x8200000A
22 #define MLXBF_PMC_WRITE_REG_64 0x8200000B
23 #define MLXBF_PMC_READ_REG_64 0x8200000C
24 #define MLXBF_PMC_SIP_SVC_UID 0x8200ff01
25 #define MLXBF_PMC_SIP_SVC_VERSION 0x8200ff03
26 #define MLXBF_PMC_SVC_REQ_MAJOR 0
27 #define MLXBF_PMC_SVC_MIN_MINOR 3
28 
29 #define MLXBF_PMC_SMCCC_ACCESS_VIOLATION -4
30 
31 #define MLXBF_PMC_EVENT_SET_BF1 0
32 #define MLXBF_PMC_EVENT_SET_BF2 1
33 #define MLXBF_PMC_EVENT_INFO_LEN 100
34 
35 #define MLXBF_PMC_MAX_BLOCKS 30
36 #define MLXBF_PMC_MAX_ATTRS 30
37 #define MLXBF_PMC_INFO_SZ 4
38 #define MLXBF_PMC_REG_SIZE 8
39 #define MLXBF_PMC_L3C_REG_SIZE 4
40 
41 #define MLXBF_PMC_TYPE_COUNTER 1
42 #define MLXBF_PMC_TYPE_REGISTER 0
43 
44 #define MLXBF_PMC_PERFCTL 0
45 #define MLXBF_PMC_PERFEVT 1
46 #define MLXBF_PMC_PERFACC0 4
47 
48 #define MLXBF_PMC_PERFMON_CONFIG_WR_R_B BIT(0)
49 #define MLXBF_PMC_PERFMON_CONFIG_STROBE BIT(1)
50 #define MLXBF_PMC_PERFMON_CONFIG_ADDR GENMASK_ULL(4, 2)
51 #define MLXBF_PMC_PERFMON_CONFIG_WDATA GENMASK_ULL(60, 5)
52 
53 #define MLXBF_PMC_PERFCTL_FM0 GENMASK_ULL(18, 16)
54 #define MLXBF_PMC_PERFCTL_MS0 GENMASK_ULL(21, 20)
55 #define MLXBF_PMC_PERFCTL_ACCM0 GENMASK_ULL(26, 24)
56 #define MLXBF_PMC_PERFCTL_AD0 BIT(27)
57 #define MLXBF_PMC_PERFCTL_ETRIG0 GENMASK_ULL(29, 28)
58 #define MLXBF_PMC_PERFCTL_EB0 BIT(30)
59 #define MLXBF_PMC_PERFCTL_EN0 BIT(31)
60 
61 #define MLXBF_PMC_PERFEVT_EVTSEL GENMASK_ULL(31, 24)
62 
63 #define MLXBF_PMC_L3C_PERF_CNT_CFG 0x0
64 #define MLXBF_PMC_L3C_PERF_CNT_SEL 0x10
65 #define MLXBF_PMC_L3C_PERF_CNT_SEL_1 0x14
66 #define MLXBF_PMC_L3C_PERF_CNT_LOW 0x40
67 #define MLXBF_PMC_L3C_PERF_CNT_HIGH 0x60
68 
69 #define MLXBF_PMC_L3C_PERF_CNT_CFG_EN BIT(0)
70 #define MLXBF_PMC_L3C_PERF_CNT_CFG_RST BIT(1)
71 #define MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_0 GENMASK(5, 0)
72 #define MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_1 GENMASK(13, 8)
73 #define MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_2 GENMASK(21, 16)
74 #define MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_3 GENMASK(29, 24)
75 
76 #define MLXBF_PMC_L3C_PERF_CNT_SEL_1_CNT_4 GENMASK(5, 0)
77 
78 #define MLXBF_PMC_L3C_PERF_CNT_LOW_VAL GENMASK(31, 0)
79 #define MLXBF_PMC_L3C_PERF_CNT_HIGH_VAL GENMASK(24, 0)
80 
81 /**
82  * struct mlxbf_pmc_attribute - Structure to hold attribute and block info
83  * for each sysfs entry
84  * @dev_attr: Device attribute struct
85  * @index: index to identify counter number within a block
86  * @nr: block number to which the sysfs belongs
87  */
88 struct mlxbf_pmc_attribute {
89 	struct device_attribute dev_attr;
90 	int index;
91 	int nr;
92 };
93 
94 /**
95  * struct mlxbf_pmc_block_info - Structure to hold info for each HW block
96  *
97  * @mmio_base: The VA at which the PMC block is mapped
98  * @blk_size: Size of each mapped region
99  * @counters: Number of counters in the block
100  * @type: Type of counters in the block
101  * @attr_counter: Attributes for "counter" sysfs files
102  * @attr_event: Attributes for "event" sysfs files
103  * @attr_event_list: Attributes for "event_list" sysfs files
104  * @attr_enable: Attributes for "enable" sysfs files
105  * @block_attr: All attributes needed for the block
106  * @block_attr_grp: Attribute group for the block
107  */
108 struct mlxbf_pmc_block_info {
109 	void __iomem *mmio_base;
110 	size_t blk_size;
111 	size_t counters;
112 	int type;
113 	struct mlxbf_pmc_attribute *attr_counter;
114 	struct mlxbf_pmc_attribute *attr_event;
115 	struct mlxbf_pmc_attribute attr_event_list;
116 	struct mlxbf_pmc_attribute attr_enable;
117 	struct attribute *block_attr[MLXBF_PMC_MAX_ATTRS];
118 	struct attribute_group block_attr_grp;
119 };
120 
121 /**
122  * struct mlxbf_pmc_context - Structure to hold PMC context info
123  *
124  * @pdev: The kernel structure representing the device
125  * @total_blocks: Total number of blocks
126  * @tile_count: Number of tiles in the system
127  * @hwmon_dev: Hwmon device for bfperf
128  * @block_name: Block name
129  * @block:  Block info
130  * @groups:  Attribute groups from each block
131  * @svc_sreg_support: Whether SMCs are used to access performance registers
132  * @sreg_tbl_perf: Secure register access table number
133  * @event_set: Event set to use
134  */
135 struct mlxbf_pmc_context {
136 	struct platform_device *pdev;
137 	uint32_t total_blocks;
138 	uint32_t tile_count;
139 	struct device *hwmon_dev;
140 	const char *block_name[MLXBF_PMC_MAX_BLOCKS];
141 	struct mlxbf_pmc_block_info block[MLXBF_PMC_MAX_BLOCKS];
142 	const struct attribute_group *groups[MLXBF_PMC_MAX_BLOCKS];
143 	bool svc_sreg_support;
144 	uint32_t sreg_tbl_perf;
145 	unsigned int event_set;
146 };
147 
148 /**
149  * struct mlxbf_pmc_events - Structure to hold supported events for each block
150  * @evt_num: Event number used to program counters
151  * @evt_name: Name of the event
152  */
153 struct mlxbf_pmc_events {
154 	int evt_num;
155 	char *evt_name;
156 };
157 
158 static const struct mlxbf_pmc_events mlxbf_pmc_pcie_events[] = {
159 	{ 0x0, "IN_P_PKT_CNT" },
160 	{ 0x10, "IN_NP_PKT_CNT" },
161 	{ 0x18, "IN_C_PKT_CNT" },
162 	{ 0x20, "OUT_P_PKT_CNT" },
163 	{ 0x28, "OUT_NP_PKT_CNT" },
164 	{ 0x30, "OUT_C_PKT_CNT" },
165 	{ 0x38, "IN_P_BYTE_CNT" },
166 	{ 0x40, "IN_NP_BYTE_CNT" },
167 	{ 0x48, "IN_C_BYTE_CNT" },
168 	{ 0x50, "OUT_P_BYTE_CNT" },
169 	{ 0x58, "OUT_NP_BYTE_CNT" },
170 	{ 0x60, "OUT_C_BYTE_CNT" },
171 };
172 
173 static const struct mlxbf_pmc_events mlxbf_pmc_smgen_events[] = {
174 	{ 0x0, "AW_REQ" },
175 	{ 0x1, "AW_BEATS" },
176 	{ 0x2, "AW_TRANS" },
177 	{ 0x3, "AW_RESP" },
178 	{ 0x4, "AW_STL" },
179 	{ 0x5, "AW_LAT" },
180 	{ 0x6, "AW_REQ_TBU" },
181 	{ 0x8, "AR_REQ" },
182 	{ 0x9, "AR_BEATS" },
183 	{ 0xa, "AR_TRANS" },
184 	{ 0xb, "AR_STL" },
185 	{ 0xc, "AR_LAT" },
186 	{ 0xd, "AR_REQ_TBU" },
187 	{ 0xe, "TBU_MISS" },
188 	{ 0xf, "TX_DAT_AF" },
189 	{ 0x10, "RX_DAT_AF" },
190 	{ 0x11, "RETRYQ_CRED" },
191 };
192 
193 static const struct mlxbf_pmc_events mlxbf_pmc_trio_events_1[] = {
194 	{ 0xa0, "TPIO_DATA_BEAT" },
195 	{ 0xa1, "TDMA_DATA_BEAT" },
196 	{ 0xa2, "MAP_DATA_BEAT" },
197 	{ 0xa3, "TXMSG_DATA_BEAT" },
198 	{ 0xa4, "TPIO_DATA_PACKET" },
199 	{ 0xa5, "TDMA_DATA_PACKET" },
200 	{ 0xa6, "MAP_DATA_PACKET" },
201 	{ 0xa7, "TXMSG_DATA_PACKET" },
202 	{ 0xa8, "TDMA_RT_AF" },
203 	{ 0xa9, "TDMA_PBUF_MAC_AF" },
204 	{ 0xaa, "TRIO_MAP_WRQ_BUF_EMPTY" },
205 	{ 0xab, "TRIO_MAP_CPL_BUF_EMPTY" },
206 	{ 0xac, "TRIO_MAP_RDQ0_BUF_EMPTY" },
207 	{ 0xad, "TRIO_MAP_RDQ1_BUF_EMPTY" },
208 	{ 0xae, "TRIO_MAP_RDQ2_BUF_EMPTY" },
209 	{ 0xaf, "TRIO_MAP_RDQ3_BUF_EMPTY" },
210 	{ 0xb0, "TRIO_MAP_RDQ4_BUF_EMPTY" },
211 	{ 0xb1, "TRIO_MAP_RDQ5_BUF_EMPTY" },
212 	{ 0xb2, "TRIO_MAP_RDQ6_BUF_EMPTY" },
213 	{ 0xb3, "TRIO_MAP_RDQ7_BUF_EMPTY" },
214 };
215 
216 static const struct mlxbf_pmc_events mlxbf_pmc_trio_events_2[] = {
217 	{ 0xa0, "TPIO_DATA_BEAT" },
218 	{ 0xa1, "TDMA_DATA_BEAT" },
219 	{ 0xa2, "MAP_DATA_BEAT" },
220 	{ 0xa3, "TXMSG_DATA_BEAT" },
221 	{ 0xa4, "TPIO_DATA_PACKET" },
222 	{ 0xa5, "TDMA_DATA_PACKET" },
223 	{ 0xa6, "MAP_DATA_PACKET" },
224 	{ 0xa7, "TXMSG_DATA_PACKET" },
225 	{ 0xa8, "TDMA_RT_AF" },
226 	{ 0xa9, "TDMA_PBUF_MAC_AF" },
227 	{ 0xaa, "TRIO_MAP_WRQ_BUF_EMPTY" },
228 	{ 0xab, "TRIO_MAP_CPL_BUF_EMPTY" },
229 	{ 0xac, "TRIO_MAP_RDQ0_BUF_EMPTY" },
230 	{ 0xad, "TRIO_MAP_RDQ1_BUF_EMPTY" },
231 	{ 0xae, "TRIO_MAP_RDQ2_BUF_EMPTY" },
232 	{ 0xaf, "TRIO_MAP_RDQ3_BUF_EMPTY" },
233 	{ 0xb0, "TRIO_MAP_RDQ4_BUF_EMPTY" },
234 	{ 0xb1, "TRIO_MAP_RDQ5_BUF_EMPTY" },
235 	{ 0xb2, "TRIO_MAP_RDQ6_BUF_EMPTY" },
236 	{ 0xb3, "TRIO_MAP_RDQ7_BUF_EMPTY" },
237 	{ 0xb4, "TRIO_RING_TX_FLIT_CH0" },
238 	{ 0xb5, "TRIO_RING_TX_FLIT_CH1" },
239 	{ 0xb6, "TRIO_RING_TX_FLIT_CH2" },
240 	{ 0xb7, "TRIO_RING_TX_FLIT_CH3" },
241 	{ 0xb8, "TRIO_RING_TX_FLIT_CH4" },
242 	{ 0xb9, "TRIO_RING_RX_FLIT_CH0" },
243 	{ 0xba, "TRIO_RING_RX_FLIT_CH1" },
244 	{ 0xbb, "TRIO_RING_RX_FLIT_CH2" },
245 	{ 0xbc, "TRIO_RING_RX_FLIT_CH3" },
246 };
247 
248 static const struct mlxbf_pmc_events mlxbf_pmc_ecc_events[] = {
249 	{ 0x100, "ECC_SINGLE_ERROR_CNT" },
250 	{ 0x104, "ECC_DOUBLE_ERROR_CNT" },
251 	{ 0x114, "SERR_INJ" },
252 	{ 0x118, "DERR_INJ" },
253 	{ 0x124, "ECC_SINGLE_ERROR_0" },
254 	{ 0x164, "ECC_DOUBLE_ERROR_0" },
255 	{ 0x340, "DRAM_ECC_COUNT" },
256 	{ 0x344, "DRAM_ECC_INJECT" },
257 	{ 0x348, "DRAM_ECC_ERROR" },
258 };
259 
260 static const struct mlxbf_pmc_events mlxbf_pmc_mss_events[] = {
261 	{ 0xc0, "RXREQ_MSS" },
262 	{ 0xc1, "RXDAT_MSS" },
263 	{ 0xc2, "TXRSP_MSS" },
264 	{ 0xc3, "TXDAT_MSS" },
265 };
266 
267 static const struct mlxbf_pmc_events mlxbf_pmc_hnf_events[] = {
268 	{ 0x45, "HNF_REQUESTS" },
269 	{ 0x46, "HNF_REJECTS" },
270 	{ 0x47, "ALL_BUSY" },
271 	{ 0x48, "MAF_BUSY" },
272 	{ 0x49, "MAF_REQUESTS" },
273 	{ 0x4a, "RNF_REQUESTS" },
274 	{ 0x4b, "REQUEST_TYPE" },
275 	{ 0x4c, "MEMORY_READS" },
276 	{ 0x4d, "MEMORY_WRITES" },
277 	{ 0x4e, "VICTIM_WRITE" },
278 	{ 0x4f, "POC_FULL" },
279 	{ 0x50, "POC_FAIL" },
280 	{ 0x51, "POC_SUCCESS" },
281 	{ 0x52, "POC_WRITES" },
282 	{ 0x53, "POC_READS" },
283 	{ 0x54, "FORWARD" },
284 	{ 0x55, "RXREQ_HNF" },
285 	{ 0x56, "RXRSP_HNF" },
286 	{ 0x57, "RXDAT_HNF" },
287 	{ 0x58, "TXREQ_HNF" },
288 	{ 0x59, "TXRSP_HNF" },
289 	{ 0x5a, "TXDAT_HNF" },
290 	{ 0x5b, "TXSNP_HNF" },
291 	{ 0x5c, "INDEX_MATCH" },
292 	{ 0x5d, "A72_ACCESS" },
293 	{ 0x5e, "IO_ACCESS" },
294 	{ 0x5f, "TSO_WRITE" },
295 	{ 0x60, "TSO_CONFLICT" },
296 	{ 0x61, "DIR_HIT" },
297 	{ 0x62, "HNF_ACCEPTS" },
298 	{ 0x63, "REQ_BUF_EMPTY" },
299 	{ 0x64, "REQ_BUF_IDLE_MAF" },
300 	{ 0x65, "TSO_NOARB" },
301 	{ 0x66, "TSO_NOARB_CYCLES" },
302 	{ 0x67, "MSS_NO_CREDIT" },
303 	{ 0x68, "TXDAT_NO_LCRD" },
304 	{ 0x69, "TXSNP_NO_LCRD" },
305 	{ 0x6a, "TXRSP_NO_LCRD" },
306 	{ 0x6b, "TXREQ_NO_LCRD" },
307 	{ 0x6c, "TSO_CL_MATCH" },
308 	{ 0x6d, "MEMORY_READS_BYPASS" },
309 	{ 0x6e, "TSO_NOARB_TIMEOUT" },
310 	{ 0x6f, "ALLOCATE" },
311 	{ 0x70, "VICTIM" },
312 	{ 0x71, "A72_WRITE" },
313 	{ 0x72, "A72_READ" },
314 	{ 0x73, "IO_WRITE" },
315 	{ 0x74, "IO_READ" },
316 	{ 0x75, "TSO_REJECT" },
317 	{ 0x80, "TXREQ_RN" },
318 	{ 0x81, "TXRSP_RN" },
319 	{ 0x82, "TXDAT_RN" },
320 	{ 0x83, "RXSNP_RN" },
321 	{ 0x84, "RXRSP_RN" },
322 	{ 0x85, "RXDAT_RN" },
323 };
324 
325 static const struct mlxbf_pmc_events mlxbf_pmc_hnfnet_events[] = {
326 	{ 0x12, "CDN_REQ" },
327 	{ 0x13, "DDN_REQ" },
328 	{ 0x14, "NDN_REQ" },
329 	{ 0x15, "CDN_DIAG_N_OUT_OF_CRED" },
330 	{ 0x16, "CDN_DIAG_S_OUT_OF_CRED" },
331 	{ 0x17, "CDN_DIAG_E_OUT_OF_CRED" },
332 	{ 0x18, "CDN_DIAG_W_OUT_OF_CRED" },
333 	{ 0x19, "CDN_DIAG_C_OUT_OF_CRED" },
334 	{ 0x1a, "CDN_DIAG_N_EGRESS" },
335 	{ 0x1b, "CDN_DIAG_S_EGRESS" },
336 	{ 0x1c, "CDN_DIAG_E_EGRESS" },
337 	{ 0x1d, "CDN_DIAG_W_EGRESS" },
338 	{ 0x1e, "CDN_DIAG_C_EGRESS" },
339 	{ 0x1f, "CDN_DIAG_N_INGRESS" },
340 	{ 0x20, "CDN_DIAG_S_INGRESS" },
341 	{ 0x21, "CDN_DIAG_E_INGRESS" },
342 	{ 0x22, "CDN_DIAG_W_INGRESS" },
343 	{ 0x23, "CDN_DIAG_C_INGRESS" },
344 	{ 0x24, "CDN_DIAG_CORE_SENT" },
345 	{ 0x25, "DDN_DIAG_N_OUT_OF_CRED" },
346 	{ 0x26, "DDN_DIAG_S_OUT_OF_CRED" },
347 	{ 0x27, "DDN_DIAG_E_OUT_OF_CRED" },
348 	{ 0x28, "DDN_DIAG_W_OUT_OF_CRED" },
349 	{ 0x29, "DDN_DIAG_C_OUT_OF_CRED" },
350 	{ 0x2a, "DDN_DIAG_N_EGRESS" },
351 	{ 0x2b, "DDN_DIAG_S_EGRESS" },
352 	{ 0x2c, "DDN_DIAG_E_EGRESS" },
353 	{ 0x2d, "DDN_DIAG_W_EGRESS" },
354 	{ 0x2e, "DDN_DIAG_C_EGRESS" },
355 	{ 0x2f, "DDN_DIAG_N_INGRESS" },
356 	{ 0x30, "DDN_DIAG_S_INGRESS" },
357 	{ 0x31, "DDN_DIAG_E_INGRESS" },
358 	{ 0x32, "DDN_DIAG_W_INGRESS" },
359 	{ 0x33, "DDN_DIAG_C_INGRESS" },
360 	{ 0x34, "DDN_DIAG_CORE_SENT" },
361 	{ 0x35, "NDN_DIAG_N_OUT_OF_CRED" },
362 	{ 0x36, "NDN_DIAG_S_OUT_OF_CRED" },
363 	{ 0x37, "NDN_DIAG_E_OUT_OF_CRED" },
364 	{ 0x38, "NDN_DIAG_W_OUT_OF_CRED" },
365 	{ 0x39, "NDN_DIAG_C_OUT_OF_CRED" },
366 	{ 0x3a, "NDN_DIAG_N_EGRESS" },
367 	{ 0x3b, "NDN_DIAG_S_EGRESS" },
368 	{ 0x3c, "NDN_DIAG_E_EGRESS" },
369 	{ 0x3d, "NDN_DIAG_W_EGRESS" },
370 	{ 0x3e, "NDN_DIAG_C_EGRESS" },
371 	{ 0x3f, "NDN_DIAG_N_INGRESS" },
372 	{ 0x40, "NDN_DIAG_S_INGRESS" },
373 	{ 0x41, "NDN_DIAG_E_INGRESS" },
374 	{ 0x42, "NDN_DIAG_W_INGRESS" },
375 	{ 0x43, "NDN_DIAG_C_INGRESS" },
376 	{ 0x44, "NDN_DIAG_CORE_SENT" },
377 };
378 
379 static const struct mlxbf_pmc_events mlxbf_pmc_l3c_events[] = {
380 	{ 0x00, "DISABLE" },
381 	{ 0x01, "CYCLES" },
382 	{ 0x02, "TOTAL_RD_REQ_IN" },
383 	{ 0x03, "TOTAL_WR_REQ_IN" },
384 	{ 0x04, "TOTAL_WR_DBID_ACK" },
385 	{ 0x05, "TOTAL_WR_DATA_IN" },
386 	{ 0x06, "TOTAL_WR_COMP" },
387 	{ 0x07, "TOTAL_RD_DATA_OUT" },
388 	{ 0x08, "TOTAL_CDN_REQ_IN_BANK0" },
389 	{ 0x09, "TOTAL_CDN_REQ_IN_BANK1" },
390 	{ 0x0a, "TOTAL_DDN_REQ_IN_BANK0" },
391 	{ 0x0b, "TOTAL_DDN_REQ_IN_BANK1" },
392 	{ 0x0c, "TOTAL_EMEM_RD_RES_IN_BANK0" },
393 	{ 0x0d, "TOTAL_EMEM_RD_RES_IN_BANK1" },
394 	{ 0x0e, "TOTAL_CACHE_RD_RES_IN_BANK0" },
395 	{ 0x0f, "TOTAL_CACHE_RD_RES_IN_BANK1" },
396 	{ 0x10, "TOTAL_EMEM_RD_REQ_BANK0" },
397 	{ 0x11, "TOTAL_EMEM_RD_REQ_BANK1" },
398 	{ 0x12, "TOTAL_EMEM_WR_REQ_BANK0" },
399 	{ 0x13, "TOTAL_EMEM_WR_REQ_BANK1" },
400 	{ 0x14, "TOTAL_RD_REQ_OUT" },
401 	{ 0x15, "TOTAL_WR_REQ_OUT" },
402 	{ 0x16, "TOTAL_RD_RES_IN" },
403 	{ 0x17, "HITS_BANK0" },
404 	{ 0x18, "HITS_BANK1" },
405 	{ 0x19, "MISSES_BANK0" },
406 	{ 0x1a, "MISSES_BANK1" },
407 	{ 0x1b, "ALLOCATIONS_BANK0" },
408 	{ 0x1c, "ALLOCATIONS_BANK1" },
409 	{ 0x1d, "EVICTIONS_BANK0" },
410 	{ 0x1e, "EVICTIONS_BANK1" },
411 	{ 0x1f, "DBID_REJECT" },
412 	{ 0x20, "WRDB_REJECT_BANK0" },
413 	{ 0x21, "WRDB_REJECT_BANK1" },
414 	{ 0x22, "CMDQ_REJECT_BANK0" },
415 	{ 0x23, "CMDQ_REJECT_BANK1" },
416 	{ 0x24, "COB_REJECT_BANK0" },
417 	{ 0x25, "COB_REJECT_BANK1" },
418 	{ 0x26, "TRB_REJECT_BANK0" },
419 	{ 0x27, "TRB_REJECT_BANK1" },
420 	{ 0x28, "TAG_REJECT_BANK0" },
421 	{ 0x29, "TAG_REJECT_BANK1" },
422 	{ 0x2a, "ANY_REJECT_BANK0" },
423 	{ 0x2b, "ANY_REJECT_BANK1" },
424 };
425 
426 static struct mlxbf_pmc_context *pmc;
427 
428 /* UUID used to probe ATF service. */
429 static const char *mlxbf_pmc_svc_uuid_str = "89c036b4-e7d7-11e6-8797-001aca00bfc4";
430 
431 /* Calls an SMC to access a performance register */
432 static int mlxbf_pmc_secure_read(void __iomem *addr, uint32_t command,
433 				 uint64_t *result)
434 {
435 	struct arm_smccc_res res;
436 	int status, err = 0;
437 
438 	arm_smccc_smc(command, pmc->sreg_tbl_perf, (uintptr_t)addr, 0, 0, 0, 0,
439 		      0, &res);
440 
441 	status = res.a0;
442 
443 	switch (status) {
444 	case PSCI_RET_NOT_SUPPORTED:
445 		err = -EINVAL;
446 		break;
447 	case MLXBF_PMC_SMCCC_ACCESS_VIOLATION:
448 		err = -EACCES;
449 		break;
450 	default:
451 		*result = res.a1;
452 		break;
453 	}
454 
455 	return err;
456 }
457 
458 /* Read from a performance counter */
459 static int mlxbf_pmc_read(void __iomem *addr, uint32_t command,
460 			  uint64_t *result)
461 {
462 	if (pmc->svc_sreg_support)
463 		return mlxbf_pmc_secure_read(addr, command, result);
464 
465 	if (command == MLXBF_PMC_READ_REG_32)
466 		*result = readl(addr);
467 	else
468 		*result = readq(addr);
469 
470 	return 0;
471 }
472 
473 /* Convenience function for 32-bit reads */
474 static int mlxbf_pmc_readl(void __iomem *addr, uint32_t *result)
475 {
476 	uint64_t read_out;
477 	int status;
478 
479 	status = mlxbf_pmc_read(addr, MLXBF_PMC_READ_REG_32, &read_out);
480 	if (status)
481 		return status;
482 	*result = (uint32_t)read_out;
483 
484 	return 0;
485 }
486 
487 /* Calls an SMC to access a performance register */
488 static int mlxbf_pmc_secure_write(void __iomem *addr, uint32_t command,
489 				  uint64_t value)
490 {
491 	struct arm_smccc_res res;
492 	int status, err = 0;
493 
494 	arm_smccc_smc(command, pmc->sreg_tbl_perf, value, (uintptr_t)addr, 0, 0,
495 		      0, 0, &res);
496 
497 	status = res.a0;
498 
499 	switch (status) {
500 	case PSCI_RET_NOT_SUPPORTED:
501 		err = -EINVAL;
502 		break;
503 	case MLXBF_PMC_SMCCC_ACCESS_VIOLATION:
504 		err = -EACCES;
505 		break;
506 	}
507 
508 	return err;
509 }
510 
511 /* Write to a performance counter */
512 static int mlxbf_pmc_write(void __iomem *addr, int command, uint64_t value)
513 {
514 	if (pmc->svc_sreg_support)
515 		return mlxbf_pmc_secure_write(addr, command, value);
516 
517 	if (command == MLXBF_PMC_WRITE_REG_32)
518 		writel(value, addr);
519 	else
520 		writeq(value, addr);
521 
522 	return 0;
523 }
524 
525 /* Check if the register offset is within the mapped region for the block */
526 static bool mlxbf_pmc_valid_range(int blk_num, uint32_t offset)
527 {
528 	if ((offset >= 0) && !(offset % MLXBF_PMC_REG_SIZE) &&
529 	    (offset + MLXBF_PMC_REG_SIZE <= pmc->block[blk_num].blk_size))
530 		return true; /* inside the mapped PMC space */
531 
532 	return false;
533 }
534 
535 /* Get the event list corresponding to a certain block */
536 static const struct mlxbf_pmc_events *mlxbf_pmc_event_list(const char *blk,
537 							   int *size)
538 {
539 	const struct mlxbf_pmc_events *events;
540 
541 	if (strstr(blk, "tilenet")) {
542 		events = mlxbf_pmc_hnfnet_events;
543 		*size = ARRAY_SIZE(mlxbf_pmc_hnfnet_events);
544 	} else if (strstr(blk, "tile")) {
545 		events = mlxbf_pmc_hnf_events;
546 		*size = ARRAY_SIZE(mlxbf_pmc_hnf_events);
547 	} else if (strstr(blk, "triogen")) {
548 		events = mlxbf_pmc_smgen_events;
549 		*size = ARRAY_SIZE(mlxbf_pmc_smgen_events);
550 	} else if (strstr(blk, "trio")) {
551 		switch (pmc->event_set) {
552 		case MLXBF_PMC_EVENT_SET_BF1:
553 			events = mlxbf_pmc_trio_events_1;
554 			*size = ARRAY_SIZE(mlxbf_pmc_trio_events_1);
555 			break;
556 		case MLXBF_PMC_EVENT_SET_BF2:
557 			events = mlxbf_pmc_trio_events_2;
558 			*size = ARRAY_SIZE(mlxbf_pmc_trio_events_2);
559 			break;
560 		default:
561 			events = NULL;
562 			*size = 0;
563 			break;
564 		}
565 	} else if (strstr(blk, "mss")) {
566 		events = mlxbf_pmc_mss_events;
567 		*size = ARRAY_SIZE(mlxbf_pmc_mss_events);
568 	} else if (strstr(blk, "ecc")) {
569 		events = mlxbf_pmc_ecc_events;
570 		*size = ARRAY_SIZE(mlxbf_pmc_ecc_events);
571 	} else if (strstr(blk, "pcie")) {
572 		events = mlxbf_pmc_pcie_events;
573 		*size = ARRAY_SIZE(mlxbf_pmc_pcie_events);
574 	} else if (strstr(blk, "l3cache")) {
575 		events = mlxbf_pmc_l3c_events;
576 		*size = ARRAY_SIZE(mlxbf_pmc_l3c_events);
577 	} else if (strstr(blk, "gic")) {
578 		events = mlxbf_pmc_smgen_events;
579 		*size = ARRAY_SIZE(mlxbf_pmc_smgen_events);
580 	} else if (strstr(blk, "smmu")) {
581 		events = mlxbf_pmc_smgen_events;
582 		*size = ARRAY_SIZE(mlxbf_pmc_smgen_events);
583 	} else {
584 		events = NULL;
585 		*size = 0;
586 	}
587 
588 	return events;
589 }
590 
591 /* Get the event number given the name */
592 static int mlxbf_pmc_get_event_num(const char *blk, const char *evt)
593 {
594 	const struct mlxbf_pmc_events *events;
595 	int i, size;
596 
597 	events = mlxbf_pmc_event_list(blk, &size);
598 	if (!events)
599 		return -EINVAL;
600 
601 	for (i = 0; i < size; ++i) {
602 		if (!strcmp(evt, events[i].evt_name))
603 			return events[i].evt_num;
604 	}
605 
606 	return -ENODEV;
607 }
608 
609 /* Get the event number given the name */
610 static char *mlxbf_pmc_get_event_name(const char *blk, int evt)
611 {
612 	const struct mlxbf_pmc_events *events;
613 	int i, size;
614 
615 	events = mlxbf_pmc_event_list(blk, &size);
616 	if (!events)
617 		return NULL;
618 
619 	for (i = 0; i < size; ++i) {
620 		if (evt == events[i].evt_num)
621 			return events[i].evt_name;
622 	}
623 
624 	return NULL;
625 }
626 
627 /* Method to enable/disable/reset l3cache counters */
628 static int mlxbf_pmc_config_l3_counters(int blk_num, bool enable, bool reset)
629 {
630 	uint32_t perfcnt_cfg = 0;
631 
632 	if (enable)
633 		perfcnt_cfg |= MLXBF_PMC_L3C_PERF_CNT_CFG_EN;
634 	if (reset)
635 		perfcnt_cfg |= MLXBF_PMC_L3C_PERF_CNT_CFG_RST;
636 
637 	return mlxbf_pmc_write(pmc->block[blk_num].mmio_base +
638 				       MLXBF_PMC_L3C_PERF_CNT_CFG,
639 			       MLXBF_PMC_WRITE_REG_32, perfcnt_cfg);
640 }
641 
642 /* Method to handle l3cache counter programming */
643 static int mlxbf_pmc_program_l3_counter(int blk_num, uint32_t cnt_num,
644 					uint32_t evt)
645 {
646 	uint32_t perfcnt_sel_1 = 0;
647 	uint32_t perfcnt_sel = 0;
648 	uint32_t *wordaddr;
649 	void __iomem *pmcaddr;
650 	int ret;
651 
652 	/* Disable all counters before programming them */
653 	if (mlxbf_pmc_config_l3_counters(blk_num, false, false))
654 		return -EINVAL;
655 
656 	/* Select appropriate register information */
657 	switch (cnt_num) {
658 	case 0 ... 3:
659 		pmcaddr = pmc->block[blk_num].mmio_base +
660 			  MLXBF_PMC_L3C_PERF_CNT_SEL;
661 		wordaddr = &perfcnt_sel;
662 		break;
663 	case 4:
664 		pmcaddr = pmc->block[blk_num].mmio_base +
665 			  MLXBF_PMC_L3C_PERF_CNT_SEL_1;
666 		wordaddr = &perfcnt_sel_1;
667 		break;
668 	default:
669 		return -EINVAL;
670 	}
671 
672 	ret = mlxbf_pmc_readl(pmcaddr, wordaddr);
673 	if (ret)
674 		return ret;
675 
676 	switch (cnt_num) {
677 	case 0:
678 		perfcnt_sel &= ~MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_0;
679 		perfcnt_sel |= FIELD_PREP(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_0,
680 					  evt);
681 		break;
682 	case 1:
683 		perfcnt_sel &= ~MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_1;
684 		perfcnt_sel |= FIELD_PREP(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_1,
685 					  evt);
686 		break;
687 	case 2:
688 		perfcnt_sel &= ~MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_2;
689 		perfcnt_sel |= FIELD_PREP(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_2,
690 					  evt);
691 		break;
692 	case 3:
693 		perfcnt_sel &= ~MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_3;
694 		perfcnt_sel |= FIELD_PREP(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_3,
695 					  evt);
696 		break;
697 	case 4:
698 		perfcnt_sel_1 &= ~MLXBF_PMC_L3C_PERF_CNT_SEL_1_CNT_4;
699 		perfcnt_sel_1 |= FIELD_PREP(MLXBF_PMC_L3C_PERF_CNT_SEL_1_CNT_4,
700 					    evt);
701 		break;
702 	default:
703 		return -EINVAL;
704 	}
705 
706 	return mlxbf_pmc_write(pmcaddr, MLXBF_PMC_WRITE_REG_32, *wordaddr);
707 }
708 
709 /* Method to program a counter to monitor an event */
710 static int mlxbf_pmc_program_counter(int blk_num, uint32_t cnt_num,
711 				     uint32_t evt, bool is_l3)
712 {
713 	uint64_t perfctl, perfevt, perfmon_cfg;
714 
715 	if (cnt_num >= pmc->block[blk_num].counters)
716 		return -ENODEV;
717 
718 	if (is_l3)
719 		return mlxbf_pmc_program_l3_counter(blk_num, cnt_num, evt);
720 
721 	/* Configure the counter */
722 	perfctl = FIELD_PREP(MLXBF_PMC_PERFCTL_EN0, 1);
723 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_EB0, 0);
724 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_ETRIG0, 1);
725 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_AD0, 0);
726 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_ACCM0, 0);
727 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_MS0, 0);
728 	perfctl |= FIELD_PREP(MLXBF_PMC_PERFCTL_FM0, 0);
729 
730 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WDATA, perfctl);
731 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
732 				  MLXBF_PMC_PERFCTL);
733 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
734 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 1);
735 
736 	if (mlxbf_pmc_write(pmc->block[blk_num].mmio_base +
737 				    cnt_num * MLXBF_PMC_REG_SIZE,
738 			    MLXBF_PMC_WRITE_REG_64, perfmon_cfg))
739 		return -EFAULT;
740 
741 	/* Select the event */
742 	perfevt = FIELD_PREP(MLXBF_PMC_PERFEVT_EVTSEL, evt);
743 
744 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WDATA, perfevt);
745 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
746 				  MLXBF_PMC_PERFEVT);
747 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
748 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 1);
749 
750 	if (mlxbf_pmc_write(pmc->block[blk_num].mmio_base +
751 				    cnt_num * MLXBF_PMC_REG_SIZE,
752 			    MLXBF_PMC_WRITE_REG_64, perfmon_cfg))
753 		return -EFAULT;
754 
755 	/* Clear the accumulator */
756 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
757 				 MLXBF_PMC_PERFACC0);
758 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
759 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 1);
760 
761 	if (mlxbf_pmc_write(pmc->block[blk_num].mmio_base +
762 				    cnt_num * MLXBF_PMC_REG_SIZE,
763 			    MLXBF_PMC_WRITE_REG_64, perfmon_cfg))
764 		return -EFAULT;
765 
766 	return 0;
767 }
768 
769 /* Method to handle l3 counter reads */
770 static int mlxbf_pmc_read_l3_counter(int blk_num, uint32_t cnt_num,
771 				     uint64_t *result)
772 {
773 	uint32_t perfcnt_low = 0, perfcnt_high = 0;
774 	uint64_t value;
775 	int status = 0;
776 
777 	status = mlxbf_pmc_readl(pmc->block[blk_num].mmio_base +
778 					 MLXBF_PMC_L3C_PERF_CNT_LOW +
779 					 cnt_num * MLXBF_PMC_L3C_REG_SIZE,
780 				 &perfcnt_low);
781 
782 	if (status)
783 		return status;
784 
785 	status = mlxbf_pmc_readl(pmc->block[blk_num].mmio_base +
786 					 MLXBF_PMC_L3C_PERF_CNT_HIGH +
787 					 cnt_num * MLXBF_PMC_L3C_REG_SIZE,
788 				 &perfcnt_high);
789 
790 	if (status)
791 		return status;
792 
793 	value = perfcnt_high;
794 	value = value << 32;
795 	value |= perfcnt_low;
796 	*result = value;
797 
798 	return 0;
799 }
800 
801 /* Method to read the counter value */
802 static int mlxbf_pmc_read_counter(int blk_num, uint32_t cnt_num, bool is_l3,
803 				  uint64_t *result)
804 {
805 	uint32_t perfcfg_offset, perfval_offset;
806 	uint64_t perfmon_cfg;
807 	int status;
808 
809 	if (cnt_num >= pmc->block[blk_num].counters)
810 		return -EINVAL;
811 
812 	if (is_l3)
813 		return mlxbf_pmc_read_l3_counter(blk_num, cnt_num, result);
814 
815 	perfcfg_offset = cnt_num * MLXBF_PMC_REG_SIZE;
816 	perfval_offset = perfcfg_offset +
817 			 pmc->block[blk_num].counters * MLXBF_PMC_REG_SIZE;
818 
819 	/* Set counter in "read" mode */
820 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
821 				 MLXBF_PMC_PERFACC0);
822 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
823 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 0);
824 
825 	status = mlxbf_pmc_write(pmc->block[blk_num].mmio_base + perfcfg_offset,
826 				 MLXBF_PMC_WRITE_REG_64, perfmon_cfg);
827 
828 	if (status)
829 		return status;
830 
831 	/* Get the counter value */
832 	return mlxbf_pmc_read(pmc->block[blk_num].mmio_base + perfval_offset,
833 			      MLXBF_PMC_READ_REG_64, result);
834 }
835 
836 /* Method to read L3 block event */
837 static int mlxbf_pmc_read_l3_event(int blk_num, uint32_t cnt_num,
838 				   uint64_t *result)
839 {
840 	uint32_t perfcnt_sel = 0, perfcnt_sel_1 = 0;
841 	uint32_t *wordaddr;
842 	void __iomem *pmcaddr;
843 	uint64_t evt;
844 
845 	/* Select appropriate register information */
846 	switch (cnt_num) {
847 	case 0 ... 3:
848 		pmcaddr = pmc->block[blk_num].mmio_base +
849 			  MLXBF_PMC_L3C_PERF_CNT_SEL;
850 		wordaddr = &perfcnt_sel;
851 		break;
852 	case 4:
853 		pmcaddr = pmc->block[blk_num].mmio_base +
854 			  MLXBF_PMC_L3C_PERF_CNT_SEL_1;
855 		wordaddr = &perfcnt_sel_1;
856 		break;
857 	default:
858 		return -EINVAL;
859 	}
860 
861 	if (mlxbf_pmc_readl(pmcaddr, wordaddr))
862 		return -EINVAL;
863 
864 	/* Read from appropriate register field for the counter */
865 	switch (cnt_num) {
866 	case 0:
867 		evt = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_0, perfcnt_sel);
868 		break;
869 	case 1:
870 		evt = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_1, perfcnt_sel);
871 		break;
872 	case 2:
873 		evt = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_2, perfcnt_sel);
874 		break;
875 	case 3:
876 		evt = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_SEL_CNT_3, perfcnt_sel);
877 		break;
878 	case 4:
879 		evt = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_SEL_1_CNT_4,
880 				perfcnt_sel_1);
881 		break;
882 	default:
883 		return -EINVAL;
884 	}
885 	*result = evt;
886 
887 	return 0;
888 }
889 
890 /* Method to find the event currently being monitored by a counter */
891 static int mlxbf_pmc_read_event(int blk_num, uint32_t cnt_num, bool is_l3,
892 				uint64_t *result)
893 {
894 	uint32_t perfcfg_offset, perfval_offset;
895 	uint64_t perfmon_cfg, perfevt, perfctl;
896 
897 	if (cnt_num >= pmc->block[blk_num].counters)
898 		return -EINVAL;
899 
900 	if (is_l3)
901 		return mlxbf_pmc_read_l3_event(blk_num, cnt_num, result);
902 
903 	perfcfg_offset = cnt_num * MLXBF_PMC_REG_SIZE;
904 	perfval_offset = perfcfg_offset +
905 			 pmc->block[blk_num].counters * MLXBF_PMC_REG_SIZE;
906 
907 	/* Set counter in "read" mode */
908 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
909 				 MLXBF_PMC_PERFCTL);
910 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
911 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 0);
912 
913 	if (mlxbf_pmc_write(pmc->block[blk_num].mmio_base + perfcfg_offset,
914 			    MLXBF_PMC_WRITE_REG_64, perfmon_cfg))
915 		return -EFAULT;
916 
917 	/* Check if the counter is enabled */
918 
919 	if (mlxbf_pmc_read(pmc->block[blk_num].mmio_base + perfval_offset,
920 			   MLXBF_PMC_READ_REG_64, &perfctl))
921 		return -EFAULT;
922 
923 	if (!FIELD_GET(MLXBF_PMC_PERFCTL_EN0, perfctl))
924 		return -EINVAL;
925 
926 	/* Set counter in "read" mode */
927 	perfmon_cfg = FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_ADDR,
928 				 MLXBF_PMC_PERFEVT);
929 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_STROBE, 1);
930 	perfmon_cfg |= FIELD_PREP(MLXBF_PMC_PERFMON_CONFIG_WR_R_B, 0);
931 
932 	if (mlxbf_pmc_write(pmc->block[blk_num].mmio_base + perfcfg_offset,
933 			    MLXBF_PMC_WRITE_REG_64, perfmon_cfg))
934 		return -EFAULT;
935 
936 	/* Get the event number */
937 	if (mlxbf_pmc_read(pmc->block[blk_num].mmio_base + perfval_offset,
938 			   MLXBF_PMC_READ_REG_64, &perfevt))
939 		return -EFAULT;
940 
941 	*result = FIELD_GET(MLXBF_PMC_PERFEVT_EVTSEL, perfevt);
942 
943 	return 0;
944 }
945 
946 /* Method to read a register */
947 static int mlxbf_pmc_read_reg(int blk_num, uint32_t offset, uint64_t *result)
948 {
949 	uint32_t ecc_out;
950 
951 	if (strstr(pmc->block_name[blk_num], "ecc")) {
952 		if (mlxbf_pmc_readl(pmc->block[blk_num].mmio_base + offset,
953 				    &ecc_out))
954 			return -EFAULT;
955 
956 		*result = ecc_out;
957 		return 0;
958 	}
959 
960 	if (mlxbf_pmc_valid_range(blk_num, offset))
961 		return mlxbf_pmc_read(pmc->block[blk_num].mmio_base + offset,
962 				      MLXBF_PMC_READ_REG_64, result);
963 
964 	return -EINVAL;
965 }
966 
967 /* Method to write to a register */
968 static int mlxbf_pmc_write_reg(int blk_num, uint32_t offset, uint64_t data)
969 {
970 	if (strstr(pmc->block_name[blk_num], "ecc")) {
971 		return mlxbf_pmc_write(pmc->block[blk_num].mmio_base + offset,
972 				       MLXBF_PMC_WRITE_REG_32, data);
973 	}
974 
975 	if (mlxbf_pmc_valid_range(blk_num, offset))
976 		return mlxbf_pmc_write(pmc->block[blk_num].mmio_base + offset,
977 				       MLXBF_PMC_WRITE_REG_64, data);
978 
979 	return -EINVAL;
980 }
981 
982 /* Show function for "counter" sysfs files */
983 static ssize_t mlxbf_pmc_counter_show(struct device *dev,
984 				      struct device_attribute *attr, char *buf)
985 {
986 	struct mlxbf_pmc_attribute *attr_counter = container_of(
987 		attr, struct mlxbf_pmc_attribute, dev_attr);
988 	int blk_num, cnt_num, offset;
989 	bool is_l3 = false;
990 	uint64_t value;
991 
992 	blk_num = attr_counter->nr;
993 	cnt_num = attr_counter->index;
994 
995 	if (strstr(pmc->block_name[blk_num], "l3cache"))
996 		is_l3 = true;
997 
998 	if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_COUNTER) {
999 		if (mlxbf_pmc_read_counter(blk_num, cnt_num, is_l3, &value))
1000 			return -EINVAL;
1001 	} else if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_REGISTER) {
1002 		offset = mlxbf_pmc_get_event_num(pmc->block_name[blk_num],
1003 						 attr->attr.name);
1004 		if (offset < 0)
1005 			return -EINVAL;
1006 		if (mlxbf_pmc_read_reg(blk_num, offset, &value))
1007 			return -EINVAL;
1008 	} else
1009 		return -EINVAL;
1010 
1011 	return sprintf(buf, "0x%llx\n", value);
1012 }
1013 
1014 /* Store function for "counter" sysfs files */
1015 static ssize_t mlxbf_pmc_counter_store(struct device *dev,
1016 				       struct device_attribute *attr,
1017 				       const char *buf, size_t count)
1018 {
1019 	struct mlxbf_pmc_attribute *attr_counter = container_of(
1020 		attr, struct mlxbf_pmc_attribute, dev_attr);
1021 	int blk_num, cnt_num, offset, err, data;
1022 	bool is_l3 = false;
1023 	uint64_t evt_num;
1024 
1025 	blk_num = attr_counter->nr;
1026 	cnt_num = attr_counter->index;
1027 
1028 	err = kstrtoint(buf, 0, &data);
1029 	if (err < 0)
1030 		return err;
1031 
1032 	/* Allow non-zero writes only to the ecc regs */
1033 	if (!(strstr(pmc->block_name[blk_num], "ecc")) && data)
1034 		return -EINVAL;
1035 
1036 	/* Do not allow writes to the L3C regs */
1037 	if (strstr(pmc->block_name[blk_num], "l3cache"))
1038 		return -EINVAL;
1039 
1040 	if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_COUNTER) {
1041 		err = mlxbf_pmc_read_event(blk_num, cnt_num, is_l3, &evt_num);
1042 		if (err)
1043 			return err;
1044 		err = mlxbf_pmc_program_counter(blk_num, cnt_num, evt_num,
1045 						is_l3);
1046 		if (err)
1047 			return err;
1048 	} else if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_REGISTER) {
1049 		offset = mlxbf_pmc_get_event_num(pmc->block_name[blk_num],
1050 						 attr->attr.name);
1051 		if (offset < 0)
1052 			return -EINVAL;
1053 		err = mlxbf_pmc_write_reg(blk_num, offset, data);
1054 		if (err)
1055 			return err;
1056 	} else
1057 		return -EINVAL;
1058 
1059 	return count;
1060 }
1061 
1062 /* Show function for "event" sysfs files */
1063 static ssize_t mlxbf_pmc_event_show(struct device *dev,
1064 				    struct device_attribute *attr, char *buf)
1065 {
1066 	struct mlxbf_pmc_attribute *attr_event = container_of(
1067 		attr, struct mlxbf_pmc_attribute, dev_attr);
1068 	int blk_num, cnt_num, err;
1069 	bool is_l3 = false;
1070 	uint64_t evt_num;
1071 	char *evt_name;
1072 
1073 	blk_num = attr_event->nr;
1074 	cnt_num = attr_event->index;
1075 
1076 	if (strstr(pmc->block_name[blk_num], "l3cache"))
1077 		is_l3 = true;
1078 
1079 	err = mlxbf_pmc_read_event(blk_num, cnt_num, is_l3, &evt_num);
1080 	if (err)
1081 		return sprintf(buf, "No event being monitored\n");
1082 
1083 	evt_name = mlxbf_pmc_get_event_name(pmc->block_name[blk_num], evt_num);
1084 	if (!evt_name)
1085 		return -EINVAL;
1086 
1087 	return sprintf(buf, "0x%llx: %s\n", evt_num, evt_name);
1088 }
1089 
1090 /* Store function for "event" sysfs files */
1091 static ssize_t mlxbf_pmc_event_store(struct device *dev,
1092 				     struct device_attribute *attr,
1093 				     const char *buf, size_t count)
1094 {
1095 	struct mlxbf_pmc_attribute *attr_event = container_of(
1096 		attr, struct mlxbf_pmc_attribute, dev_attr);
1097 	int blk_num, cnt_num, evt_num, err;
1098 	bool is_l3 = false;
1099 
1100 	blk_num = attr_event->nr;
1101 	cnt_num = attr_event->index;
1102 
1103 	if (isalpha(buf[0])) {
1104 		evt_num = mlxbf_pmc_get_event_num(pmc->block_name[blk_num],
1105 						  buf);
1106 		if (evt_num < 0)
1107 			return -EINVAL;
1108 	} else {
1109 		err = kstrtoint(buf, 0, &evt_num);
1110 		if (err < 0)
1111 			return err;
1112 	}
1113 
1114 	if (strstr(pmc->block_name[blk_num], "l3cache"))
1115 		is_l3 = true;
1116 
1117 	err = mlxbf_pmc_program_counter(blk_num, cnt_num, evt_num, is_l3);
1118 	if (err)
1119 		return err;
1120 
1121 	return count;
1122 }
1123 
1124 /* Show function for "event_list" sysfs files */
1125 static ssize_t mlxbf_pmc_event_list_show(struct device *dev,
1126 					 struct device_attribute *attr,
1127 					 char *buf)
1128 {
1129 	struct mlxbf_pmc_attribute *attr_event_list = container_of(
1130 		attr, struct mlxbf_pmc_attribute, dev_attr);
1131 	int blk_num, i, size, len = 0, ret = 0;
1132 	const struct mlxbf_pmc_events *events;
1133 	char e_info[MLXBF_PMC_EVENT_INFO_LEN];
1134 
1135 	blk_num = attr_event_list->nr;
1136 
1137 	events = mlxbf_pmc_event_list(pmc->block_name[blk_num], &size);
1138 	if (!events)
1139 		return -EINVAL;
1140 
1141 	for (i = 0, buf[0] = '\0'; i < size; ++i) {
1142 		len += sprintf(e_info, "0x%x: %s\n", events[i].evt_num,
1143 			       events[i].evt_name);
1144 		if (len > PAGE_SIZE)
1145 			break;
1146 		strcat(buf, e_info);
1147 		ret = len;
1148 	}
1149 
1150 	return ret;
1151 }
1152 
1153 /* Show function for "enable" sysfs files - only for l3cache */
1154 static ssize_t mlxbf_pmc_enable_show(struct device *dev,
1155 				     struct device_attribute *attr, char *buf)
1156 {
1157 	struct mlxbf_pmc_attribute *attr_enable = container_of(
1158 		attr, struct mlxbf_pmc_attribute, dev_attr);
1159 	uint32_t perfcnt_cfg;
1160 	int blk_num, value;
1161 
1162 	blk_num = attr_enable->nr;
1163 
1164 	if (mlxbf_pmc_readl(pmc->block[blk_num].mmio_base +
1165 				    MLXBF_PMC_L3C_PERF_CNT_CFG,
1166 			    &perfcnt_cfg))
1167 		return -EINVAL;
1168 
1169 	value = FIELD_GET(MLXBF_PMC_L3C_PERF_CNT_CFG_EN, perfcnt_cfg);
1170 
1171 	return sprintf(buf, "%d\n", value);
1172 }
1173 
1174 /* Store function for "enable" sysfs files - only for l3cache */
1175 static ssize_t mlxbf_pmc_enable_store(struct device *dev,
1176 				      struct device_attribute *attr,
1177 				      const char *buf, size_t count)
1178 {
1179 	struct mlxbf_pmc_attribute *attr_enable = container_of(
1180 		attr, struct mlxbf_pmc_attribute, dev_attr);
1181 	int err, en, blk_num;
1182 
1183 	blk_num = attr_enable->nr;
1184 
1185 	err = kstrtoint(buf, 0, &en);
1186 	if (err < 0)
1187 		return err;
1188 
1189 	if (!en) {
1190 		err = mlxbf_pmc_config_l3_counters(blk_num, false, false);
1191 		if (err)
1192 			return err;
1193 	} else if (en == 1) {
1194 		err = mlxbf_pmc_config_l3_counters(blk_num, false, true);
1195 		if (err)
1196 			return err;
1197 		err = mlxbf_pmc_config_l3_counters(blk_num, true, false);
1198 		if (err)
1199 			return err;
1200 	} else
1201 		return -EINVAL;
1202 
1203 	return count;
1204 }
1205 
1206 /* Populate attributes for blocks with counters to monitor performance */
1207 static int mlxbf_pmc_init_perftype_counter(struct device *dev, int blk_num)
1208 {
1209 	struct mlxbf_pmc_attribute *attr;
1210 	int i = 0, j = 0;
1211 
1212 	/* "event_list" sysfs to list events supported by the block */
1213 	attr = &pmc->block[blk_num].attr_event_list;
1214 	attr->dev_attr.attr.mode = 0444;
1215 	attr->dev_attr.show = mlxbf_pmc_event_list_show;
1216 	attr->nr = blk_num;
1217 	attr->dev_attr.attr.name = devm_kasprintf(dev, GFP_KERNEL, "event_list");
1218 	pmc->block[blk_num].block_attr[i] = &attr->dev_attr.attr;
1219 	attr = NULL;
1220 
1221 	/* "enable" sysfs to start/stop the counters. Only in L3C blocks */
1222 	if (strstr(pmc->block_name[blk_num], "l3cache")) {
1223 		attr = &pmc->block[blk_num].attr_enable;
1224 		attr->dev_attr.attr.mode = 0644;
1225 		attr->dev_attr.show = mlxbf_pmc_enable_show;
1226 		attr->dev_attr.store = mlxbf_pmc_enable_store;
1227 		attr->nr = blk_num;
1228 		attr->dev_attr.attr.name = devm_kasprintf(dev, GFP_KERNEL,
1229 							  "enable");
1230 		pmc->block[blk_num].block_attr[++i] = &attr->dev_attr.attr;
1231 		attr = NULL;
1232 	}
1233 
1234 	pmc->block[blk_num].attr_counter = devm_kcalloc(
1235 		dev, pmc->block[blk_num].counters,
1236 		sizeof(struct mlxbf_pmc_attribute), GFP_KERNEL);
1237 	if (!pmc->block[blk_num].attr_counter)
1238 		return -ENOMEM;
1239 
1240 	pmc->block[blk_num].attr_event = devm_kcalloc(
1241 		dev, pmc->block[blk_num].counters,
1242 		sizeof(struct mlxbf_pmc_attribute), GFP_KERNEL);
1243 	if (!pmc->block[blk_num].attr_event)
1244 		return -ENOMEM;
1245 
1246 	/* "eventX" and "counterX" sysfs to program and read counter values */
1247 	for (j = 0; j < pmc->block[blk_num].counters; ++j) {
1248 		attr = &pmc->block[blk_num].attr_counter[j];
1249 		attr->dev_attr.attr.mode = 0644;
1250 		attr->dev_attr.show = mlxbf_pmc_counter_show;
1251 		attr->dev_attr.store = mlxbf_pmc_counter_store;
1252 		attr->index = j;
1253 		attr->nr = blk_num;
1254 		attr->dev_attr.attr.name = devm_kasprintf(dev, GFP_KERNEL,
1255 							  "counter%d", j);
1256 		pmc->block[blk_num].block_attr[++i] = &attr->dev_attr.attr;
1257 		attr = NULL;
1258 
1259 		attr = &pmc->block[blk_num].attr_event[j];
1260 		attr->dev_attr.attr.mode = 0644;
1261 		attr->dev_attr.show = mlxbf_pmc_event_show;
1262 		attr->dev_attr.store = mlxbf_pmc_event_store;
1263 		attr->index = j;
1264 		attr->nr = blk_num;
1265 		attr->dev_attr.attr.name = devm_kasprintf(dev, GFP_KERNEL,
1266 							  "event%d", j);
1267 		pmc->block[blk_num].block_attr[++i] = &attr->dev_attr.attr;
1268 		attr = NULL;
1269 	}
1270 
1271 	return 0;
1272 }
1273 
1274 /* Populate attributes for blocks with registers to monitor performance */
1275 static int mlxbf_pmc_init_perftype_reg(struct device *dev, int blk_num)
1276 {
1277 	struct mlxbf_pmc_attribute *attr;
1278 	const struct mlxbf_pmc_events *events;
1279 	int i = 0, j = 0;
1280 
1281 	events = mlxbf_pmc_event_list(pmc->block_name[blk_num], &j);
1282 	if (!events)
1283 		return -EINVAL;
1284 
1285 	pmc->block[blk_num].attr_event = devm_kcalloc(
1286 		dev, j, sizeof(struct mlxbf_pmc_attribute), GFP_KERNEL);
1287 	if (!pmc->block[blk_num].attr_event)
1288 		return -ENOMEM;
1289 
1290 	while (j > 0) {
1291 		--j;
1292 		attr = &pmc->block[blk_num].attr_event[j];
1293 		attr->dev_attr.attr.mode = 0644;
1294 		attr->dev_attr.show = mlxbf_pmc_counter_show;
1295 		attr->dev_attr.store = mlxbf_pmc_counter_store;
1296 		attr->nr = blk_num;
1297 		attr->dev_attr.attr.name = devm_kasprintf(dev, GFP_KERNEL,
1298 							  events[j].evt_name);
1299 		pmc->block[blk_num].block_attr[i] = &attr->dev_attr.attr;
1300 		attr = NULL;
1301 		i++;
1302 	}
1303 
1304 	return 0;
1305 }
1306 
1307 /* Helper to create the bfperf sysfs sub-directories and files */
1308 static int mlxbf_pmc_create_groups(struct device *dev, int blk_num)
1309 {
1310 	int err;
1311 
1312 	/* Populate attributes based on counter type */
1313 	if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_COUNTER)
1314 		err = mlxbf_pmc_init_perftype_counter(dev, blk_num);
1315 	else if (pmc->block[blk_num].type == MLXBF_PMC_TYPE_REGISTER)
1316 		err = mlxbf_pmc_init_perftype_reg(dev, blk_num);
1317 	else
1318 		err = -EINVAL;
1319 
1320 	if (err)
1321 		return err;
1322 
1323 	/* Add a new attribute_group for the block */
1324 	pmc->block[blk_num].block_attr_grp.attrs = pmc->block[blk_num].block_attr;
1325 	pmc->block[blk_num].block_attr_grp.name = devm_kasprintf(
1326 		dev, GFP_KERNEL, pmc->block_name[blk_num]);
1327 	pmc->groups[blk_num] = &pmc->block[blk_num].block_attr_grp;
1328 
1329 	return 0;
1330 }
1331 
1332 static bool mlxbf_pmc_guid_match(const guid_t *guid,
1333 				 const struct arm_smccc_res *res)
1334 {
1335 	guid_t id = GUID_INIT(res->a0, res->a1, res->a1 >> 16, res->a2,
1336 			      res->a2 >> 8, res->a2 >> 16, res->a2 >> 24,
1337 			      res->a3, res->a3 >> 8, res->a3 >> 16,
1338 			      res->a3 >> 24);
1339 
1340 	return guid_equal(guid, &id);
1341 }
1342 
1343 /* Helper to map the Performance Counters from the varios blocks */
1344 static int mlxbf_pmc_map_counters(struct device *dev)
1345 {
1346 	uint64_t info[MLXBF_PMC_INFO_SZ];
1347 	int i, tile_num, ret;
1348 
1349 	for (i = 0; i < pmc->total_blocks; ++i) {
1350 		if (strstr(pmc->block_name[i], "tile")) {
1351 			if (sscanf(pmc->block_name[i], "tile%d", &tile_num) != 1)
1352 				return -EINVAL;
1353 
1354 			if (tile_num >= pmc->tile_count)
1355 				continue;
1356 		}
1357 		ret = device_property_read_u64_array(dev, pmc->block_name[i],
1358 						     info, MLXBF_PMC_INFO_SZ);
1359 		if (ret)
1360 			return ret;
1361 
1362 		/*
1363 		 * Do not remap if the proper SMC calls are supported,
1364 		 * since the SMC calls expect physical addresses.
1365 		 */
1366 		if (pmc->svc_sreg_support)
1367 			pmc->block[i].mmio_base = (void __iomem *)info[0];
1368 		else
1369 			pmc->block[i].mmio_base =
1370 				devm_ioremap(dev, info[0], info[1]);
1371 
1372 		pmc->block[i].blk_size = info[1];
1373 		pmc->block[i].counters = info[2];
1374 		pmc->block[i].type = info[3];
1375 
1376 		if (!pmc->block[i].mmio_base)
1377 			return -ENOMEM;
1378 
1379 		ret = mlxbf_pmc_create_groups(dev, i);
1380 		if (ret)
1381 			return ret;
1382 	}
1383 
1384 	return 0;
1385 }
1386 
1387 static int mlxbf_pmc_probe(struct platform_device *pdev)
1388 {
1389 	struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
1390 	const char *hid = acpi_device_hid(acpi_dev);
1391 	struct device *dev = &pdev->dev;
1392 	struct arm_smccc_res res;
1393 	guid_t guid;
1394 	int ret;
1395 
1396 	/* Ensure we have the UUID we expect for this service. */
1397 	arm_smccc_smc(MLXBF_PMC_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res);
1398 	guid_parse(mlxbf_pmc_svc_uuid_str, &guid);
1399 	if (!mlxbf_pmc_guid_match(&guid, &res))
1400 		return -ENODEV;
1401 
1402 	pmc = devm_kzalloc(dev, sizeof(struct mlxbf_pmc_context), GFP_KERNEL);
1403 	if (!pmc)
1404 		return -ENOMEM;
1405 
1406 	/*
1407 	 * ACPI indicates whether we use SMCs to access registers or not.
1408 	 * If sreg_tbl_perf is not present, just assume we're not using SMCs.
1409 	 */
1410 	ret = device_property_read_u32(dev, "sec_reg_block",
1411 				       &pmc->sreg_tbl_perf);
1412 	if (ret) {
1413 		pmc->svc_sreg_support = false;
1414 	} else {
1415 		/*
1416 		 * Check service version to see if we actually do support the
1417 		 * needed SMCs. If we have the calls we need, mark support for
1418 		 * them in the pmc struct.
1419 		 */
1420 		arm_smccc_smc(MLXBF_PMC_SIP_SVC_VERSION, 0, 0, 0, 0, 0, 0, 0,
1421 			      &res);
1422 		if (res.a0 == MLXBF_PMC_SVC_REQ_MAJOR &&
1423 		    res.a1 >= MLXBF_PMC_SVC_MIN_MINOR)
1424 			pmc->svc_sreg_support = true;
1425 		else
1426 			return -EINVAL;
1427 	}
1428 
1429 	if (!strcmp(hid, "MLNXBFD0"))
1430 		pmc->event_set = MLXBF_PMC_EVENT_SET_BF1;
1431 	else if (!strcmp(hid, "MLNXBFD1"))
1432 		pmc->event_set = MLXBF_PMC_EVENT_SET_BF2;
1433 	else
1434 		return -ENODEV;
1435 
1436 	ret = device_property_read_u32(dev, "block_num", &pmc->total_blocks);
1437 	if (ret)
1438 		return ret;
1439 
1440 	ret = device_property_read_string_array(dev, "block_name",
1441 						pmc->block_name,
1442 						pmc->total_blocks);
1443 	if (ret != pmc->total_blocks)
1444 		return -EFAULT;
1445 
1446 	ret = device_property_read_u32(dev, "tile_num", &pmc->tile_count);
1447 	if (ret)
1448 		return ret;
1449 
1450 	pmc->pdev = pdev;
1451 
1452 	ret = mlxbf_pmc_map_counters(dev);
1453 	if (ret)
1454 		return ret;
1455 
1456 	pmc->hwmon_dev = devm_hwmon_device_register_with_groups(
1457 		dev, "bfperf", pmc, pmc->groups);
1458 	platform_set_drvdata(pdev, pmc);
1459 
1460 	return 0;
1461 }
1462 
1463 static const struct acpi_device_id mlxbf_pmc_acpi_ids[] = { { "MLNXBFD0", 0 },
1464 							    { "MLNXBFD1", 0 },
1465 							    {}, };
1466 
1467 MODULE_DEVICE_TABLE(acpi, mlxbf_pmc_acpi_ids);
1468 static struct platform_driver pmc_driver = {
1469 	.driver = { .name = "mlxbf-pmc",
1470 		    .acpi_match_table = ACPI_PTR(mlxbf_pmc_acpi_ids), },
1471 	.probe = mlxbf_pmc_probe,
1472 };
1473 
1474 module_platform_driver(pmc_driver);
1475 
1476 MODULE_AUTHOR("Shravan Kumar Ramani <sramani@mellanox.com>");
1477 MODULE_DESCRIPTION("Mellanox PMC driver");
1478 MODULE_LICENSE("Dual BSD/GPL");
1479