1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4  */
5 
6 #include "mlx5_ib.h"
7 #include <linux/mlx5/eswitch.h>
8 #include <linux/mlx5/vport.h>
9 #include "counters.h"
10 #include "ib_rep.h"
11 #include "qp.h"
12 
13 struct mlx5_ib_counter {
14 	const char *name;
15 	size_t offset;
16 	u32 type;
17 };
18 
19 #define INIT_Q_COUNTER(_name)		\
20 	{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
21 
22 #define INIT_VPORT_Q_COUNTER(_name)		\
23 	{ .name = "vport_" #_name, .offset =	\
24 		MLX5_BYTE_OFF(query_q_counter_out, _name)}
25 
26 static const struct mlx5_ib_counter basic_q_cnts[] = {
27 	INIT_Q_COUNTER(rx_write_requests),
28 	INIT_Q_COUNTER(rx_read_requests),
29 	INIT_Q_COUNTER(rx_atomic_requests),
30 	INIT_Q_COUNTER(rx_dct_connect),
31 	INIT_Q_COUNTER(out_of_buffer),
32 };
33 
34 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
35 	INIT_Q_COUNTER(out_of_sequence),
36 };
37 
38 static const struct mlx5_ib_counter retrans_q_cnts[] = {
39 	INIT_Q_COUNTER(duplicate_request),
40 	INIT_Q_COUNTER(rnr_nak_retry_err),
41 	INIT_Q_COUNTER(packet_seq_err),
42 	INIT_Q_COUNTER(implied_nak_seq_err),
43 	INIT_Q_COUNTER(local_ack_timeout_err),
44 };
45 
46 static const struct mlx5_ib_counter vport_basic_q_cnts[] = {
47 	INIT_VPORT_Q_COUNTER(rx_write_requests),
48 	INIT_VPORT_Q_COUNTER(rx_read_requests),
49 	INIT_VPORT_Q_COUNTER(rx_atomic_requests),
50 	INIT_VPORT_Q_COUNTER(rx_dct_connect),
51 	INIT_VPORT_Q_COUNTER(out_of_buffer),
52 };
53 
54 static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {
55 	INIT_VPORT_Q_COUNTER(out_of_sequence),
56 };
57 
58 static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {
59 	INIT_VPORT_Q_COUNTER(duplicate_request),
60 	INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),
61 	INIT_VPORT_Q_COUNTER(packet_seq_err),
62 	INIT_VPORT_Q_COUNTER(implied_nak_seq_err),
63 	INIT_VPORT_Q_COUNTER(local_ack_timeout_err),
64 };
65 
66 #define INIT_CONG_COUNTER(_name)		\
67 	{ .name = #_name, .offset =	\
68 		MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
69 
70 static const struct mlx5_ib_counter cong_cnts[] = {
71 	INIT_CONG_COUNTER(rp_cnp_ignored),
72 	INIT_CONG_COUNTER(rp_cnp_handled),
73 	INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
74 	INIT_CONG_COUNTER(np_cnp_sent),
75 };
76 
77 static const struct mlx5_ib_counter extended_err_cnts[] = {
78 	INIT_Q_COUNTER(resp_local_length_error),
79 	INIT_Q_COUNTER(resp_cqe_error),
80 	INIT_Q_COUNTER(req_cqe_error),
81 	INIT_Q_COUNTER(req_remote_invalid_request),
82 	INIT_Q_COUNTER(req_remote_access_errors),
83 	INIT_Q_COUNTER(resp_remote_access_errors),
84 	INIT_Q_COUNTER(resp_cqe_flush_error),
85 	INIT_Q_COUNTER(req_cqe_flush_error),
86 };
87 
88 static const struct mlx5_ib_counter roce_accl_cnts[] = {
89 	INIT_Q_COUNTER(roce_adp_retrans),
90 	INIT_Q_COUNTER(roce_adp_retrans_to),
91 	INIT_Q_COUNTER(roce_slow_restart),
92 	INIT_Q_COUNTER(roce_slow_restart_cnps),
93 	INIT_Q_COUNTER(roce_slow_restart_trans),
94 };
95 
96 static const struct mlx5_ib_counter vport_extended_err_cnts[] = {
97 	INIT_VPORT_Q_COUNTER(resp_local_length_error),
98 	INIT_VPORT_Q_COUNTER(resp_cqe_error),
99 	INIT_VPORT_Q_COUNTER(req_cqe_error),
100 	INIT_VPORT_Q_COUNTER(req_remote_invalid_request),
101 	INIT_VPORT_Q_COUNTER(req_remote_access_errors),
102 	INIT_VPORT_Q_COUNTER(resp_remote_access_errors),
103 	INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),
104 	INIT_VPORT_Q_COUNTER(req_cqe_flush_error),
105 };
106 
107 static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {
108 	INIT_VPORT_Q_COUNTER(roce_adp_retrans),
109 	INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),
110 	INIT_VPORT_Q_COUNTER(roce_slow_restart),
111 	INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),
112 	INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),
113 };
114 
115 #define INIT_EXT_PPCNT_COUNTER(_name)		\
116 	{ .name = #_name, .offset =	\
117 	MLX5_BYTE_OFF(ppcnt_reg, \
118 		      counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
119 
120 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
121 	INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
122 };
123 
124 #define INIT_OP_COUNTER(_name, _type)		\
125 	{ .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
126 
127 static const struct mlx5_ib_counter basic_op_cnts[] = {
128 	INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
129 };
130 
131 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
132 	INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
133 };
134 
135 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
136 	INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
137 };
138 
mlx5_ib_read_counters(struct ib_counters * counters,struct ib_counters_read_attr * read_attr,struct uverbs_attr_bundle * attrs)139 static int mlx5_ib_read_counters(struct ib_counters *counters,
140 				 struct ib_counters_read_attr *read_attr,
141 				 struct uverbs_attr_bundle *attrs)
142 {
143 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
144 	struct mlx5_read_counters_attr mread_attr = {};
145 	struct mlx5_ib_flow_counters_desc *desc;
146 	int ret, i;
147 
148 	mutex_lock(&mcounters->mcntrs_mutex);
149 	if (mcounters->cntrs_max_index > read_attr->ncounters) {
150 		ret = -EINVAL;
151 		goto err_bound;
152 	}
153 
154 	mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
155 				 GFP_KERNEL);
156 	if (!mread_attr.out) {
157 		ret = -ENOMEM;
158 		goto err_bound;
159 	}
160 
161 	mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
162 	mread_attr.flags = read_attr->flags;
163 	ret = mcounters->read_counters(counters->device, &mread_attr);
164 	if (ret)
165 		goto err_read;
166 
167 	/* do the pass over the counters data array to assign according to the
168 	 * descriptions and indexing pairs
169 	 */
170 	desc = mcounters->counters_data;
171 	for (i = 0; i < mcounters->ncounters; i++)
172 		read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
173 
174 err_read:
175 	kfree(mread_attr.out);
176 err_bound:
177 	mutex_unlock(&mcounters->mcntrs_mutex);
178 	return ret;
179 }
180 
mlx5_ib_destroy_counters(struct ib_counters * counters)181 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
182 {
183 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
184 
185 	mlx5_ib_counters_clear_description(counters);
186 	if (mcounters->hw_cntrs_hndl)
187 		mlx5_fc_destroy(to_mdev(counters->device)->mdev,
188 				mcounters->hw_cntrs_hndl);
189 	return 0;
190 }
191 
mlx5_ib_create_counters(struct ib_counters * counters,struct uverbs_attr_bundle * attrs)192 static int mlx5_ib_create_counters(struct ib_counters *counters,
193 				   struct uverbs_attr_bundle *attrs)
194 {
195 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
196 
197 	mutex_init(&mcounters->mcntrs_mutex);
198 	return 0;
199 }
200 
vport_qcounters_supported(struct mlx5_ib_dev * dev)201 static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)
202 {
203 	return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) &&
204 	       MLX5_CAP_GEN(dev->mdev, q_counter_aggregation);
205 }
206 
get_counters(struct mlx5_ib_dev * dev,u32 port_num)207 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
208 						   u32 port_num)
209 {
210 	if ((is_mdev_switchdev_mode(dev->mdev) &&
211 	     !vport_qcounters_supported(dev)) || !port_num)
212 		return &dev->port[0].cnts;
213 
214 	return is_mdev_switchdev_mode(dev->mdev) ?
215 	       &dev->port[1].cnts : &dev->port[port_num - 1].cnts;
216 }
217 
218 /**
219  * mlx5_ib_get_counters_id - Returns counters id to use for device+port
220  * @dev:	Pointer to mlx5 IB device
221  * @port_num:	Zero based port number
222  *
223  * mlx5_ib_get_counters_id() Returns counters set id to use for given
224  * device port combination in switchdev and non switchdev mode of the
225  * parent device.
226  */
mlx5_ib_get_counters_id(struct mlx5_ib_dev * dev,u32 port_num)227 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
228 {
229 	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);
230 
231 	return cnts->set_id;
232 }
233 
do_alloc_stats(const struct mlx5_ib_counters * cnts)234 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
235 {
236 	struct rdma_hw_stats *stats;
237 	u32 num_hw_counters;
238 	int i;
239 
240 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
241 			  cnts->num_ext_ppcnt_counters;
242 	stats = rdma_alloc_hw_stats_struct(cnts->descs,
243 					   num_hw_counters +
244 					   cnts->num_op_counters,
245 					   RDMA_HW_STATS_DEFAULT_LIFESPAN);
246 	if (!stats)
247 		return NULL;
248 
249 	for (i = 0; i < cnts->num_op_counters; i++)
250 		set_bit(num_hw_counters + i, stats->is_disabled);
251 
252 	return stats;
253 }
254 
255 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_device_stats(struct ib_device * ibdev)256 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
257 {
258 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
259 	const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
260 
261 	return do_alloc_stats(cnts);
262 }
263 
264 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_port_stats(struct ib_device * ibdev,u32 port_num)265 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
266 {
267 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
268 	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
269 
270 	return do_alloc_stats(cnts);
271 }
272 
mlx5_ib_query_q_counters(struct mlx5_core_dev * mdev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats,u16 set_id)273 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
274 				    const struct mlx5_ib_counters *cnts,
275 				    struct rdma_hw_stats *stats,
276 				    u16 set_id)
277 {
278 	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
279 	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
280 	__be32 val;
281 	int ret, i;
282 
283 	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
284 	MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
285 	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
286 	if (ret)
287 		return ret;
288 
289 	for (i = 0; i < cnts->num_q_counters; i++) {
290 		val = *(__be32 *)((void *)out + cnts->offsets[i]);
291 		stats->value[i] = (u64)be32_to_cpu(val);
292 	}
293 
294 	return 0;
295 }
296 
mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev * dev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)297 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
298 					    const struct mlx5_ib_counters *cnts,
299 					    struct rdma_hw_stats *stats)
300 {
301 	int offset = cnts->num_q_counters + cnts->num_cong_counters;
302 	u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
303 	int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
304 	int ret, i;
305 	void *out;
306 
307 	out = kvzalloc(sz, GFP_KERNEL);
308 	if (!out)
309 		return -ENOMEM;
310 
311 	MLX5_SET(ppcnt_reg, in, local_port, 1);
312 	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
313 	ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
314 				   0, 0);
315 	if (ret)
316 		goto free;
317 
318 	for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
319 		stats->value[i + offset] =
320 			be64_to_cpup((__be64 *)(out +
321 				    cnts->offsets[i + offset]));
322 free:
323 	kvfree(out);
324 	return ret;
325 }
326 
mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev * dev,u32 port_num,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)327 static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,
328 					  u32 port_num,
329 					  const struct mlx5_ib_counters *cnts,
330 					  struct rdma_hw_stats *stats)
331 
332 {
333 	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
334 	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
335 	struct mlx5_core_dev *mdev;
336 	__be32 val;
337 	int ret, i;
338 
339 	if (!dev->port[port_num].rep ||
340 	    dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK)
341 		return 0;
342 
343 	mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw);
344 	if (!mdev)
345 		return -EOPNOTSUPP;
346 
347 	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
348 	MLX5_SET(query_q_counter_in, in, other_vport, 1);
349 	MLX5_SET(query_q_counter_in, in, vport_number,
350 		 dev->port[port_num].rep->vport);
351 	MLX5_SET(query_q_counter_in, in, aggregate, 1);
352 	ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
353 	if (ret)
354 		return ret;
355 
356 	for (i = 0; i < cnts->num_q_counters; i++) {
357 		val = *(__be32 *)((void *)out + cnts->offsets[i]);
358 		stats->value[i] = (u64)be32_to_cpu(val);
359 	}
360 
361 	return 0;
362 }
363 
do_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)364 static int do_get_hw_stats(struct ib_device *ibdev,
365 			   struct rdma_hw_stats *stats,
366 			   u32 port_num, int index)
367 {
368 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
369 	const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
370 	struct mlx5_core_dev *mdev;
371 	int ret, num_counters;
372 
373 	if (!stats)
374 		return -EINVAL;
375 
376 	num_counters = cnts->num_q_counters +
377 		       cnts->num_cong_counters +
378 		       cnts->num_ext_ppcnt_counters;
379 
380 	if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0)
381 		ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,
382 						     stats);
383 	else
384 		ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats,
385 					       cnts->set_id);
386 	if (ret)
387 		return ret;
388 
389 	/* We don't expose device counters over Vports */
390 	if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0)
391 		goto done;
392 
393 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
394 		ret =  mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
395 		if (ret)
396 			return ret;
397 	}
398 
399 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
400 		if (!port_num)
401 			port_num = 1;
402 		mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);
403 		if (!mdev) {
404 			/* If port is not affiliated yet, its in down state
405 			 * which doesn't have any counters yet, so it would be
406 			 * zero. So no need to read from the HCA.
407 			 */
408 			goto done;
409 		}
410 		ret = mlx5_lag_query_cong_counters(dev->mdev,
411 						   stats->value +
412 						   cnts->num_q_counters,
413 						   cnts->num_cong_counters,
414 						   cnts->offsets +
415 						   cnts->num_q_counters);
416 
417 		mlx5_ib_put_native_port_mdev(dev, port_num);
418 		if (ret)
419 			return ret;
420 	}
421 
422 done:
423 	return num_counters;
424 }
425 
do_get_op_stat(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)426 static int do_get_op_stat(struct ib_device *ibdev,
427 			  struct rdma_hw_stats *stats,
428 			  u32 port_num, int index)
429 {
430 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
431 	const struct mlx5_ib_counters *cnts;
432 	const struct mlx5_ib_op_fc *opfcs;
433 	u64 packets = 0, bytes;
434 	u32 type;
435 	int ret;
436 
437 	cnts = get_counters(dev, port_num);
438 
439 	opfcs = cnts->opfcs;
440 	type = *(u32 *)cnts->descs[index].priv;
441 	if (type >= MLX5_IB_OPCOUNTER_MAX)
442 		return -EINVAL;
443 
444 	if (!opfcs[type].fc)
445 		goto out;
446 
447 	ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
448 			    &packets, &bytes);
449 	if (ret)
450 		return ret;
451 
452 out:
453 	stats->value[index] = packets;
454 	return index;
455 }
456 
do_get_op_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num)457 static int do_get_op_stats(struct ib_device *ibdev,
458 			   struct rdma_hw_stats *stats,
459 			   u32 port_num)
460 {
461 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
462 	const struct mlx5_ib_counters *cnts;
463 	int index, ret, num_hw_counters;
464 
465 	cnts = get_counters(dev, port_num);
466 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
467 			  cnts->num_ext_ppcnt_counters;
468 	for (index = num_hw_counters;
469 	     index < (num_hw_counters + cnts->num_op_counters); index++) {
470 		ret = do_get_op_stat(ibdev, stats, port_num, index);
471 		if (ret != index)
472 			return ret;
473 	}
474 
475 	return cnts->num_op_counters;
476 }
477 
mlx5_ib_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)478 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
479 				struct rdma_hw_stats *stats,
480 				u32 port_num, int index)
481 {
482 	int num_counters, num_hw_counters, num_op_counters;
483 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
484 	const struct mlx5_ib_counters *cnts;
485 
486 	cnts = get_counters(dev, port_num);
487 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
488 		cnts->num_ext_ppcnt_counters;
489 	num_counters = num_hw_counters + cnts->num_op_counters;
490 
491 	if (index < 0 || index > num_counters)
492 		return -EINVAL;
493 	else if (index > 0 && index < num_hw_counters)
494 		return do_get_hw_stats(ibdev, stats, port_num, index);
495 	else if (index >= num_hw_counters && index < num_counters)
496 		return do_get_op_stat(ibdev, stats, port_num, index);
497 
498 	num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
499 	if (num_hw_counters < 0)
500 		return num_hw_counters;
501 
502 	num_op_counters = do_get_op_stats(ibdev, stats, port_num);
503 	if (num_op_counters < 0)
504 		return num_op_counters;
505 
506 	return num_hw_counters + num_op_counters;
507 }
508 
509 static struct rdma_hw_stats *
mlx5_ib_counter_alloc_stats(struct rdma_counter * counter)510 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
511 {
512 	struct mlx5_ib_dev *dev = to_mdev(counter->device);
513 	const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
514 
515 	return do_alloc_stats(cnts);
516 }
517 
mlx5_ib_counter_update_stats(struct rdma_counter * counter)518 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
519 {
520 	struct mlx5_ib_dev *dev = to_mdev(counter->device);
521 	const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
522 
523 	return mlx5_ib_query_q_counters(dev->mdev, cnts,
524 					counter->stats, counter->id);
525 }
526 
mlx5_ib_counter_dealloc(struct rdma_counter * counter)527 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
528 {
529 	struct mlx5_ib_dev *dev = to_mdev(counter->device);
530 	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
531 
532 	if (!counter->id)
533 		return 0;
534 
535 	MLX5_SET(dealloc_q_counter_in, in, opcode,
536 		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
537 	MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
538 	return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
539 }
540 
mlx5_ib_counter_bind_qp(struct rdma_counter * counter,struct ib_qp * qp)541 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
542 				   struct ib_qp *qp)
543 {
544 	struct mlx5_ib_dev *dev = to_mdev(qp->device);
545 	int err;
546 
547 	if (!counter->id) {
548 		u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
549 		u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
550 
551 		MLX5_SET(alloc_q_counter_in, in, opcode,
552 			 MLX5_CMD_OP_ALLOC_Q_COUNTER);
553 		MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
554 		err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
555 		if (err)
556 			return err;
557 		counter->id =
558 			MLX5_GET(alloc_q_counter_out, out, counter_set_id);
559 	}
560 
561 	err = mlx5_ib_qp_set_counter(qp, counter);
562 	if (err)
563 		goto fail_set_counter;
564 
565 	return 0;
566 
567 fail_set_counter:
568 	mlx5_ib_counter_dealloc(counter);
569 	counter->id = 0;
570 
571 	return err;
572 }
573 
mlx5_ib_counter_unbind_qp(struct ib_qp * qp)574 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
575 {
576 	return mlx5_ib_qp_set_counter(qp, NULL);
577 }
578 
mlx5_ib_fill_counters(struct mlx5_ib_dev * dev,struct rdma_stat_desc * descs,size_t * offsets,u32 port_num)579 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
580 				  struct rdma_stat_desc *descs, size_t *offsets,
581 				  u32 port_num)
582 {
583 	bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
584 			port_num != MLX5_VPORT_PF;
585 	const struct mlx5_ib_counter *names;
586 	int j = 0, i, size;
587 
588 	names = is_vport ? vport_basic_q_cnts : basic_q_cnts;
589 	size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
590 			  ARRAY_SIZE(basic_q_cnts);
591 	for (i = 0; i < size; i++, j++) {
592 		descs[j].name = names[i].name;
593 		offsets[j] = names[i].offset;
594 	}
595 
596 	names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;
597 	size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
598 			  ARRAY_SIZE(out_of_seq_q_cnts);
599 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
600 		for (i = 0; i < size; i++, j++) {
601 			descs[j].name = names[i].name;
602 			offsets[j] = names[i].offset;
603 		}
604 	}
605 
606 	names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;
607 	size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
608 			  ARRAY_SIZE(retrans_q_cnts);
609 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
610 		for (i = 0; i < size; i++, j++) {
611 			descs[j].name = names[i].name;
612 			offsets[j] = names[i].offset;
613 		}
614 	}
615 
616 	names = is_vport ? vport_extended_err_cnts : extended_err_cnts;
617 	size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
618 			  ARRAY_SIZE(extended_err_cnts);
619 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
620 		for (i = 0; i < size; i++, j++) {
621 			descs[j].name = names[i].name;
622 			offsets[j] = names[i].offset;
623 		}
624 	}
625 
626 	names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;
627 	size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
628 			  ARRAY_SIZE(roce_accl_cnts);
629 	if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
630 		for (i = 0; i < size; i++, j++) {
631 			descs[j].name = names[i].name;
632 			offsets[j] = names[i].offset;
633 		}
634 	}
635 
636 	if (is_vport)
637 		return;
638 
639 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
640 		for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
641 			descs[j].name = cong_cnts[i].name;
642 			offsets[j] = cong_cnts[i].offset;
643 		}
644 	}
645 
646 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
647 		for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
648 			descs[j].name = ext_ppcnt_cnts[i].name;
649 			offsets[j] = ext_ppcnt_cnts[i].offset;
650 		}
651 	}
652 
653 	for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
654 		descs[j].name = basic_op_cnts[i].name;
655 		descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
656 		descs[j].priv = &basic_op_cnts[i].type;
657 	}
658 
659 	if (MLX5_CAP_FLOWTABLE(dev->mdev,
660 			       ft_field_support_2_nic_receive_rdma.bth_opcode)) {
661 		for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
662 			descs[j].name = rdmarx_cnp_op_cnts[i].name;
663 			descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
664 			descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
665 		}
666 	}
667 
668 	if (MLX5_CAP_FLOWTABLE(dev->mdev,
669 			       ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
670 		for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
671 			descs[j].name = rdmatx_cnp_op_cnts[i].name;
672 			descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
673 			descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
674 		}
675 	}
676 }
677 
678 
__mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev,struct mlx5_ib_counters * cnts,u32 port_num)679 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
680 				    struct mlx5_ib_counters *cnts, u32 port_num)
681 {
682 	bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
683 			port_num != MLX5_VPORT_PF;
684 	u32 num_counters, num_op_counters = 0, size;
685 
686 	size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
687 			  ARRAY_SIZE(basic_q_cnts);
688 	num_counters = size;
689 
690 	size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
691 			  ARRAY_SIZE(out_of_seq_q_cnts);
692 	if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
693 		num_counters += size;
694 
695 	size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
696 			  ARRAY_SIZE(retrans_q_cnts);
697 	if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
698 		num_counters += size;
699 
700 	size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
701 			  ARRAY_SIZE(extended_err_cnts);
702 	if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
703 		num_counters += size;
704 
705 	size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
706 			  ARRAY_SIZE(roce_accl_cnts);
707 	if (MLX5_CAP_GEN(dev->mdev, roce_accl))
708 		num_counters += size;
709 
710 	cnts->num_q_counters = num_counters;
711 
712 	if (is_vport)
713 		goto skip_non_qcounters;
714 
715 	if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
716 		cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
717 		num_counters += ARRAY_SIZE(cong_cnts);
718 	}
719 	if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
720 		cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
721 		num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
722 	}
723 
724 	num_op_counters = ARRAY_SIZE(basic_op_cnts);
725 
726 	if (MLX5_CAP_FLOWTABLE(dev->mdev,
727 			       ft_field_support_2_nic_receive_rdma.bth_opcode))
728 		num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
729 
730 	if (MLX5_CAP_FLOWTABLE(dev->mdev,
731 			       ft_field_support_2_nic_transmit_rdma.bth_opcode))
732 		num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
733 
734 skip_non_qcounters:
735 	cnts->num_op_counters = num_op_counters;
736 	num_counters += num_op_counters;
737 	cnts->descs = kcalloc(num_counters,
738 			      sizeof(struct rdma_stat_desc), GFP_KERNEL);
739 	if (!cnts->descs)
740 		return -ENOMEM;
741 
742 	cnts->offsets = kcalloc(num_counters,
743 				sizeof(*cnts->offsets), GFP_KERNEL);
744 	if (!cnts->offsets)
745 		goto err;
746 
747 	return 0;
748 
749 err:
750 	kfree(cnts->descs);
751 	cnts->descs = NULL;
752 	return -ENOMEM;
753 }
754 
mlx5_ib_dealloc_counters(struct mlx5_ib_dev * dev)755 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
756 {
757 	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
758 	int num_cnt_ports = dev->num_ports;
759 	int i, j;
760 
761 	if (is_mdev_switchdev_mode(dev->mdev))
762 		num_cnt_ports = min(2, num_cnt_ports);
763 
764 	MLX5_SET(dealloc_q_counter_in, in, opcode,
765 		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
766 
767 	for (i = 0; i < num_cnt_ports; i++) {
768 		if (dev->port[i].cnts.set_id) {
769 			MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
770 				 dev->port[i].cnts.set_id);
771 			mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
772 		}
773 		kfree(dev->port[i].cnts.descs);
774 		kfree(dev->port[i].cnts.offsets);
775 
776 		for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
777 			if (!dev->port[i].cnts.opfcs[j].fc)
778 				continue;
779 
780 			if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
781 				mlx5_ib_fs_remove_op_fc(dev,
782 					&dev->port[i].cnts.opfcs[j], j);
783 			mlx5_fc_destroy(dev->mdev,
784 					dev->port[i].cnts.opfcs[j].fc);
785 			dev->port[i].cnts.opfcs[j].fc = NULL;
786 		}
787 	}
788 }
789 
mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev)790 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
791 {
792 	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
793 	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
794 	int num_cnt_ports = dev->num_ports;
795 	int err = 0;
796 	int i;
797 	bool is_shared;
798 
799 	MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
800 	is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
801 
802 	/*
803 	 * In switchdev we need to allocate two ports, one that is used for
804 	 * the device Q_counters and it is essentially the real Q_counters of
805 	 * this device, while the other is used as a helper for PF to be able to
806 	 * query all other vports.
807 	 */
808 	if (is_mdev_switchdev_mode(dev->mdev))
809 		num_cnt_ports = min(2, num_cnt_ports);
810 
811 	for (i = 0; i < num_cnt_ports; i++) {
812 		err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i);
813 		if (err)
814 			goto err_alloc;
815 
816 		mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
817 				      dev->port[i].cnts.offsets, i);
818 
819 		MLX5_SET(alloc_q_counter_in, in, uid,
820 			 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
821 
822 		err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
823 		if (err) {
824 			mlx5_ib_warn(dev,
825 				     "couldn't allocate queue counter for port %d, err %d\n",
826 				     i + 1, err);
827 			goto err_alloc;
828 		}
829 
830 		dev->port[i].cnts.set_id =
831 			MLX5_GET(alloc_q_counter_out, out, counter_set_id);
832 	}
833 	return 0;
834 
835 err_alloc:
836 	mlx5_ib_dealloc_counters(dev);
837 	return err;
838 }
839 
read_flow_counters(struct ib_device * ibdev,struct mlx5_read_counters_attr * read_attr)840 static int read_flow_counters(struct ib_device *ibdev,
841 			      struct mlx5_read_counters_attr *read_attr)
842 {
843 	struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
844 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
845 
846 	return mlx5_fc_query(dev->mdev, fc,
847 			     &read_attr->out[IB_COUNTER_PACKETS],
848 			     &read_attr->out[IB_COUNTER_BYTES]);
849 }
850 
851 /* flow counters currently expose two counters packets and bytes */
852 #define FLOW_COUNTERS_NUM 2
counters_set_description(struct ib_counters * counters,enum mlx5_ib_counters_type counters_type,struct mlx5_ib_flow_counters_desc * desc_data,u32 ncounters)853 static int counters_set_description(
854 	struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
855 	struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
856 {
857 	struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
858 	u32 cntrs_max_index = 0;
859 	int i;
860 
861 	if (counters_type != MLX5_IB_COUNTERS_FLOW)
862 		return -EINVAL;
863 
864 	/* init the fields for the object */
865 	mcounters->type = counters_type;
866 	mcounters->read_counters = read_flow_counters;
867 	mcounters->counters_num = FLOW_COUNTERS_NUM;
868 	mcounters->ncounters = ncounters;
869 	/* each counter entry have both description and index pair */
870 	for (i = 0; i < ncounters; i++) {
871 		if (desc_data[i].description > IB_COUNTER_BYTES)
872 			return -EINVAL;
873 
874 		if (cntrs_max_index <= desc_data[i].index)
875 			cntrs_max_index = desc_data[i].index + 1;
876 	}
877 
878 	mutex_lock(&mcounters->mcntrs_mutex);
879 	mcounters->counters_data = desc_data;
880 	mcounters->cntrs_max_index = cntrs_max_index;
881 	mutex_unlock(&mcounters->mcntrs_mutex);
882 
883 	return 0;
884 }
885 
886 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
mlx5_ib_flow_counters_set_data(struct ib_counters * ibcounters,struct mlx5_ib_create_flow * ucmd)887 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
888 				   struct mlx5_ib_create_flow *ucmd)
889 {
890 	struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
891 	struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
892 	struct mlx5_ib_flow_counters_desc *desc_data = NULL;
893 	bool hw_hndl = false;
894 	int ret = 0;
895 
896 	if (ucmd && ucmd->ncounters_data != 0) {
897 		cntrs_data = ucmd->data;
898 		if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
899 			return -EINVAL;
900 
901 		desc_data = kcalloc(cntrs_data->ncounters,
902 				    sizeof(*desc_data),
903 				    GFP_KERNEL);
904 		if (!desc_data)
905 			return  -ENOMEM;
906 
907 		if (copy_from_user(desc_data,
908 				   u64_to_user_ptr(cntrs_data->counters_data),
909 				   sizeof(*desc_data) * cntrs_data->ncounters)) {
910 			ret = -EFAULT;
911 			goto free;
912 		}
913 	}
914 
915 	if (!mcounters->hw_cntrs_hndl) {
916 		mcounters->hw_cntrs_hndl = mlx5_fc_create(
917 			to_mdev(ibcounters->device)->mdev, false);
918 		if (IS_ERR(mcounters->hw_cntrs_hndl)) {
919 			ret = PTR_ERR(mcounters->hw_cntrs_hndl);
920 			goto free;
921 		}
922 		hw_hndl = true;
923 	}
924 
925 	if (desc_data) {
926 		/* counters already bound to at least one flow */
927 		if (mcounters->cntrs_max_index) {
928 			ret = -EINVAL;
929 			goto free_hndl;
930 		}
931 
932 		ret = counters_set_description(ibcounters,
933 					       MLX5_IB_COUNTERS_FLOW,
934 					       desc_data,
935 					       cntrs_data->ncounters);
936 		if (ret)
937 			goto free_hndl;
938 
939 	} else if (!mcounters->cntrs_max_index) {
940 		/* counters not bound yet, must have udata passed */
941 		ret = -EINVAL;
942 		goto free_hndl;
943 	}
944 
945 	return 0;
946 
947 free_hndl:
948 	if (hw_hndl) {
949 		mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
950 				mcounters->hw_cntrs_hndl);
951 		mcounters->hw_cntrs_hndl = NULL;
952 	}
953 free:
954 	kfree(desc_data);
955 	return ret;
956 }
957 
mlx5_ib_counters_clear_description(struct ib_counters * counters)958 void mlx5_ib_counters_clear_description(struct ib_counters *counters)
959 {
960 	struct mlx5_ib_mcounters *mcounters;
961 
962 	if (!counters || atomic_read(&counters->usecnt) != 1)
963 		return;
964 
965 	mcounters = to_mcounters(counters);
966 
967 	mutex_lock(&mcounters->mcntrs_mutex);
968 	kfree(mcounters->counters_data);
969 	mcounters->counters_data = NULL;
970 	mcounters->cntrs_max_index = 0;
971 	mutex_unlock(&mcounters->mcntrs_mutex);
972 }
973 
mlx5_ib_modify_stat(struct ib_device * device,u32 port,unsigned int index,bool enable)974 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
975 			       unsigned int index, bool enable)
976 {
977 	struct mlx5_ib_dev *dev = to_mdev(device);
978 	struct mlx5_ib_counters *cnts;
979 	struct mlx5_ib_op_fc *opfc;
980 	u32 num_hw_counters, type;
981 	int ret;
982 
983 	cnts = &dev->port[port - 1].cnts;
984 	num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
985 		cnts->num_ext_ppcnt_counters;
986 	if (index < num_hw_counters ||
987 	    index >= (num_hw_counters + cnts->num_op_counters))
988 		return -EINVAL;
989 
990 	if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
991 		return -EINVAL;
992 
993 	type = *(u32 *)cnts->descs[index].priv;
994 	if (type >= MLX5_IB_OPCOUNTER_MAX)
995 		return -EINVAL;
996 
997 	opfc = &cnts->opfcs[type];
998 
999 	if (enable) {
1000 		if (opfc->fc)
1001 			return -EEXIST;
1002 
1003 		opfc->fc = mlx5_fc_create(dev->mdev, false);
1004 		if (IS_ERR(opfc->fc))
1005 			return PTR_ERR(opfc->fc);
1006 
1007 		ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
1008 		if (ret) {
1009 			mlx5_fc_destroy(dev->mdev, opfc->fc);
1010 			opfc->fc = NULL;
1011 		}
1012 		return ret;
1013 	}
1014 
1015 	if (!opfc->fc)
1016 		return -EINVAL;
1017 
1018 	mlx5_ib_fs_remove_op_fc(dev, opfc, type);
1019 	mlx5_fc_destroy(dev->mdev, opfc->fc);
1020 	opfc->fc = NULL;
1021 	return 0;
1022 }
1023 
1024 static const struct ib_device_ops hw_stats_ops = {
1025 	.alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1026 	.get_hw_stats = mlx5_ib_get_hw_stats,
1027 	.counter_bind_qp = mlx5_ib_counter_bind_qp,
1028 	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1029 	.counter_dealloc = mlx5_ib_counter_dealloc,
1030 	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1031 	.counter_update_stats = mlx5_ib_counter_update_stats,
1032 	.modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ?
1033 			  mlx5_ib_modify_stat : NULL,
1034 };
1035 
1036 static const struct ib_device_ops hw_switchdev_vport_op = {
1037 	.alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1038 };
1039 
1040 static const struct ib_device_ops hw_switchdev_stats_ops = {
1041 	.alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
1042 	.get_hw_stats = mlx5_ib_get_hw_stats,
1043 	.counter_bind_qp = mlx5_ib_counter_bind_qp,
1044 	.counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1045 	.counter_dealloc = mlx5_ib_counter_dealloc,
1046 	.counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1047 	.counter_update_stats = mlx5_ib_counter_update_stats,
1048 };
1049 
1050 static const struct ib_device_ops counters_ops = {
1051 	.create_counters = mlx5_ib_create_counters,
1052 	.destroy_counters = mlx5_ib_destroy_counters,
1053 	.read_counters = mlx5_ib_read_counters,
1054 
1055 	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
1056 };
1057 
mlx5_ib_counters_init(struct mlx5_ib_dev * dev)1058 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
1059 {
1060 	ib_set_device_ops(&dev->ib_dev, &counters_ops);
1061 
1062 	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1063 		return 0;
1064 
1065 	if (is_mdev_switchdev_mode(dev->mdev)) {
1066 		ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
1067 		if (vport_qcounters_supported(dev))
1068 			ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op);
1069 	} else
1070 		ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
1071 	return mlx5_ib_alloc_counters(dev);
1072 }
1073 
mlx5_ib_counters_cleanup(struct mlx5_ib_dev * dev)1074 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
1075 {
1076 	if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1077 		return;
1078 
1079 	mlx5_ib_dealloc_counters(dev);
1080 }
1081