1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include "mlx5_ib.h"
7 #include <linux/mlx5/eswitch.h>
8 #include <linux/mlx5/vport.h>
9 #include "counters.h"
10 #include "ib_rep.h"
11 #include "qp.h"
12
13 struct mlx5_ib_counter {
14 const char *name;
15 size_t offset;
16 u32 type;
17 };
18
19 #define INIT_Q_COUNTER(_name) \
20 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
21
22 #define INIT_VPORT_Q_COUNTER(_name) \
23 { .name = "vport_" #_name, .offset = \
24 MLX5_BYTE_OFF(query_q_counter_out, _name)}
25
26 static const struct mlx5_ib_counter basic_q_cnts[] = {
27 INIT_Q_COUNTER(rx_write_requests),
28 INIT_Q_COUNTER(rx_read_requests),
29 INIT_Q_COUNTER(rx_atomic_requests),
30 INIT_Q_COUNTER(rx_dct_connect),
31 INIT_Q_COUNTER(out_of_buffer),
32 };
33
34 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
35 INIT_Q_COUNTER(out_of_sequence),
36 };
37
38 static const struct mlx5_ib_counter retrans_q_cnts[] = {
39 INIT_Q_COUNTER(duplicate_request),
40 INIT_Q_COUNTER(rnr_nak_retry_err),
41 INIT_Q_COUNTER(packet_seq_err),
42 INIT_Q_COUNTER(implied_nak_seq_err),
43 INIT_Q_COUNTER(local_ack_timeout_err),
44 };
45
46 static const struct mlx5_ib_counter vport_basic_q_cnts[] = {
47 INIT_VPORT_Q_COUNTER(rx_write_requests),
48 INIT_VPORT_Q_COUNTER(rx_read_requests),
49 INIT_VPORT_Q_COUNTER(rx_atomic_requests),
50 INIT_VPORT_Q_COUNTER(rx_dct_connect),
51 INIT_VPORT_Q_COUNTER(out_of_buffer),
52 };
53
54 static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {
55 INIT_VPORT_Q_COUNTER(out_of_sequence),
56 };
57
58 static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {
59 INIT_VPORT_Q_COUNTER(duplicate_request),
60 INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),
61 INIT_VPORT_Q_COUNTER(packet_seq_err),
62 INIT_VPORT_Q_COUNTER(implied_nak_seq_err),
63 INIT_VPORT_Q_COUNTER(local_ack_timeout_err),
64 };
65
66 #define INIT_CONG_COUNTER(_name) \
67 { .name = #_name, .offset = \
68 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
69
70 static const struct mlx5_ib_counter cong_cnts[] = {
71 INIT_CONG_COUNTER(rp_cnp_ignored),
72 INIT_CONG_COUNTER(rp_cnp_handled),
73 INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
74 INIT_CONG_COUNTER(np_cnp_sent),
75 };
76
77 static const struct mlx5_ib_counter extended_err_cnts[] = {
78 INIT_Q_COUNTER(resp_local_length_error),
79 INIT_Q_COUNTER(resp_cqe_error),
80 INIT_Q_COUNTER(req_cqe_error),
81 INIT_Q_COUNTER(req_remote_invalid_request),
82 INIT_Q_COUNTER(req_remote_access_errors),
83 INIT_Q_COUNTER(resp_remote_access_errors),
84 INIT_Q_COUNTER(resp_cqe_flush_error),
85 INIT_Q_COUNTER(req_cqe_flush_error),
86 };
87
88 static const struct mlx5_ib_counter roce_accl_cnts[] = {
89 INIT_Q_COUNTER(roce_adp_retrans),
90 INIT_Q_COUNTER(roce_adp_retrans_to),
91 INIT_Q_COUNTER(roce_slow_restart),
92 INIT_Q_COUNTER(roce_slow_restart_cnps),
93 INIT_Q_COUNTER(roce_slow_restart_trans),
94 };
95
96 static const struct mlx5_ib_counter vport_extended_err_cnts[] = {
97 INIT_VPORT_Q_COUNTER(resp_local_length_error),
98 INIT_VPORT_Q_COUNTER(resp_cqe_error),
99 INIT_VPORT_Q_COUNTER(req_cqe_error),
100 INIT_VPORT_Q_COUNTER(req_remote_invalid_request),
101 INIT_VPORT_Q_COUNTER(req_remote_access_errors),
102 INIT_VPORT_Q_COUNTER(resp_remote_access_errors),
103 INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),
104 INIT_VPORT_Q_COUNTER(req_cqe_flush_error),
105 };
106
107 static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {
108 INIT_VPORT_Q_COUNTER(roce_adp_retrans),
109 INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),
110 INIT_VPORT_Q_COUNTER(roce_slow_restart),
111 INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),
112 INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),
113 };
114
115 #define INIT_EXT_PPCNT_COUNTER(_name) \
116 { .name = #_name, .offset = \
117 MLX5_BYTE_OFF(ppcnt_reg, \
118 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
119
120 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
121 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
122 };
123
124 #define INIT_OP_COUNTER(_name, _type) \
125 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
126
127 static const struct mlx5_ib_counter basic_op_cnts[] = {
128 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
129 };
130
131 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
132 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
133 };
134
135 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
136 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
137 };
138
mlx5_ib_read_counters(struct ib_counters * counters,struct ib_counters_read_attr * read_attr,struct uverbs_attr_bundle * attrs)139 static int mlx5_ib_read_counters(struct ib_counters *counters,
140 struct ib_counters_read_attr *read_attr,
141 struct uverbs_attr_bundle *attrs)
142 {
143 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
144 struct mlx5_read_counters_attr mread_attr = {};
145 struct mlx5_ib_flow_counters_desc *desc;
146 int ret, i;
147
148 mutex_lock(&mcounters->mcntrs_mutex);
149 if (mcounters->cntrs_max_index > read_attr->ncounters) {
150 ret = -EINVAL;
151 goto err_bound;
152 }
153
154 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
155 GFP_KERNEL);
156 if (!mread_attr.out) {
157 ret = -ENOMEM;
158 goto err_bound;
159 }
160
161 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
162 mread_attr.flags = read_attr->flags;
163 ret = mcounters->read_counters(counters->device, &mread_attr);
164 if (ret)
165 goto err_read;
166
167 /* do the pass over the counters data array to assign according to the
168 * descriptions and indexing pairs
169 */
170 desc = mcounters->counters_data;
171 for (i = 0; i < mcounters->ncounters; i++)
172 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
173
174 err_read:
175 kfree(mread_attr.out);
176 err_bound:
177 mutex_unlock(&mcounters->mcntrs_mutex);
178 return ret;
179 }
180
mlx5_ib_destroy_counters(struct ib_counters * counters)181 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
182 {
183 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
184
185 mlx5_ib_counters_clear_description(counters);
186 if (mcounters->hw_cntrs_hndl)
187 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
188 mcounters->hw_cntrs_hndl);
189 return 0;
190 }
191
mlx5_ib_create_counters(struct ib_counters * counters,struct uverbs_attr_bundle * attrs)192 static int mlx5_ib_create_counters(struct ib_counters *counters,
193 struct uverbs_attr_bundle *attrs)
194 {
195 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
196
197 mutex_init(&mcounters->mcntrs_mutex);
198 return 0;
199 }
200
vport_qcounters_supported(struct mlx5_ib_dev * dev)201 static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)
202 {
203 return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) &&
204 MLX5_CAP_GEN(dev->mdev, q_counter_aggregation);
205 }
206
get_counters(struct mlx5_ib_dev * dev,u32 port_num)207 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
208 u32 port_num)
209 {
210 if ((is_mdev_switchdev_mode(dev->mdev) &&
211 !vport_qcounters_supported(dev)) || !port_num)
212 return &dev->port[0].cnts;
213
214 return is_mdev_switchdev_mode(dev->mdev) ?
215 &dev->port[1].cnts : &dev->port[port_num - 1].cnts;
216 }
217
218 /**
219 * mlx5_ib_get_counters_id - Returns counters id to use for device+port
220 * @dev: Pointer to mlx5 IB device
221 * @port_num: Zero based port number
222 *
223 * mlx5_ib_get_counters_id() Returns counters set id to use for given
224 * device port combination in switchdev and non switchdev mode of the
225 * parent device.
226 */
mlx5_ib_get_counters_id(struct mlx5_ib_dev * dev,u32 port_num)227 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
228 {
229 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);
230
231 return cnts->set_id;
232 }
233
do_alloc_stats(const struct mlx5_ib_counters * cnts)234 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
235 {
236 struct rdma_hw_stats *stats;
237 u32 num_hw_counters;
238 int i;
239
240 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
241 cnts->num_ext_ppcnt_counters;
242 stats = rdma_alloc_hw_stats_struct(cnts->descs,
243 num_hw_counters +
244 cnts->num_op_counters,
245 RDMA_HW_STATS_DEFAULT_LIFESPAN);
246 if (!stats)
247 return NULL;
248
249 for (i = 0; i < cnts->num_op_counters; i++)
250 set_bit(num_hw_counters + i, stats->is_disabled);
251
252 return stats;
253 }
254
255 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_device_stats(struct ib_device * ibdev)256 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
257 {
258 struct mlx5_ib_dev *dev = to_mdev(ibdev);
259 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
260
261 return do_alloc_stats(cnts);
262 }
263
264 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_port_stats(struct ib_device * ibdev,u32 port_num)265 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
266 {
267 struct mlx5_ib_dev *dev = to_mdev(ibdev);
268 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
269
270 return do_alloc_stats(cnts);
271 }
272
mlx5_ib_query_q_counters(struct mlx5_core_dev * mdev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats,u16 set_id)273 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
274 const struct mlx5_ib_counters *cnts,
275 struct rdma_hw_stats *stats,
276 u16 set_id)
277 {
278 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
279 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
280 __be32 val;
281 int ret, i;
282
283 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
284 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
285 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
286 if (ret)
287 return ret;
288
289 for (i = 0; i < cnts->num_q_counters; i++) {
290 val = *(__be32 *)((void *)out + cnts->offsets[i]);
291 stats->value[i] = (u64)be32_to_cpu(val);
292 }
293
294 return 0;
295 }
296
mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev * dev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)297 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
298 const struct mlx5_ib_counters *cnts,
299 struct rdma_hw_stats *stats)
300 {
301 int offset = cnts->num_q_counters + cnts->num_cong_counters;
302 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
303 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
304 int ret, i;
305 void *out;
306
307 out = kvzalloc(sz, GFP_KERNEL);
308 if (!out)
309 return -ENOMEM;
310
311 MLX5_SET(ppcnt_reg, in, local_port, 1);
312 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
313 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
314 0, 0);
315 if (ret)
316 goto free;
317
318 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
319 stats->value[i + offset] =
320 be64_to_cpup((__be64 *)(out +
321 cnts->offsets[i + offset]));
322 free:
323 kvfree(out);
324 return ret;
325 }
326
mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev * dev,u32 port_num,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)327 static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,
328 u32 port_num,
329 const struct mlx5_ib_counters *cnts,
330 struct rdma_hw_stats *stats)
331
332 {
333 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
334 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
335 struct mlx5_core_dev *mdev;
336 __be32 val;
337 int ret, i;
338
339 if (!dev->port[port_num].rep ||
340 dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK)
341 return 0;
342
343 mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw);
344 if (!mdev)
345 return -EOPNOTSUPP;
346
347 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
348 MLX5_SET(query_q_counter_in, in, other_vport, 1);
349 MLX5_SET(query_q_counter_in, in, vport_number,
350 dev->port[port_num].rep->vport);
351 MLX5_SET(query_q_counter_in, in, aggregate, 1);
352 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
353 if (ret)
354 return ret;
355
356 for (i = 0; i < cnts->num_q_counters; i++) {
357 val = *(__be32 *)((void *)out + cnts->offsets[i]);
358 stats->value[i] = (u64)be32_to_cpu(val);
359 }
360
361 return 0;
362 }
363
do_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)364 static int do_get_hw_stats(struct ib_device *ibdev,
365 struct rdma_hw_stats *stats,
366 u32 port_num, int index)
367 {
368 struct mlx5_ib_dev *dev = to_mdev(ibdev);
369 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
370 struct mlx5_core_dev *mdev;
371 int ret, num_counters;
372
373 if (!stats)
374 return -EINVAL;
375
376 num_counters = cnts->num_q_counters +
377 cnts->num_cong_counters +
378 cnts->num_ext_ppcnt_counters;
379
380 if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0)
381 ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,
382 stats);
383 else
384 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats,
385 cnts->set_id);
386 if (ret)
387 return ret;
388
389 /* We don't expose device counters over Vports */
390 if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0)
391 goto done;
392
393 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
394 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
395 if (ret)
396 return ret;
397 }
398
399 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
400 if (!port_num)
401 port_num = 1;
402 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);
403 if (!mdev) {
404 /* If port is not affiliated yet, its in down state
405 * which doesn't have any counters yet, so it would be
406 * zero. So no need to read from the HCA.
407 */
408 goto done;
409 }
410 ret = mlx5_lag_query_cong_counters(dev->mdev,
411 stats->value +
412 cnts->num_q_counters,
413 cnts->num_cong_counters,
414 cnts->offsets +
415 cnts->num_q_counters);
416
417 mlx5_ib_put_native_port_mdev(dev, port_num);
418 if (ret)
419 return ret;
420 }
421
422 done:
423 return num_counters;
424 }
425
do_get_op_stat(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)426 static int do_get_op_stat(struct ib_device *ibdev,
427 struct rdma_hw_stats *stats,
428 u32 port_num, int index)
429 {
430 struct mlx5_ib_dev *dev = to_mdev(ibdev);
431 const struct mlx5_ib_counters *cnts;
432 const struct mlx5_ib_op_fc *opfcs;
433 u64 packets = 0, bytes;
434 u32 type;
435 int ret;
436
437 cnts = get_counters(dev, port_num);
438
439 opfcs = cnts->opfcs;
440 type = *(u32 *)cnts->descs[index].priv;
441 if (type >= MLX5_IB_OPCOUNTER_MAX)
442 return -EINVAL;
443
444 if (!opfcs[type].fc)
445 goto out;
446
447 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
448 &packets, &bytes);
449 if (ret)
450 return ret;
451
452 out:
453 stats->value[index] = packets;
454 return index;
455 }
456
do_get_op_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num)457 static int do_get_op_stats(struct ib_device *ibdev,
458 struct rdma_hw_stats *stats,
459 u32 port_num)
460 {
461 struct mlx5_ib_dev *dev = to_mdev(ibdev);
462 const struct mlx5_ib_counters *cnts;
463 int index, ret, num_hw_counters;
464
465 cnts = get_counters(dev, port_num);
466 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
467 cnts->num_ext_ppcnt_counters;
468 for (index = num_hw_counters;
469 index < (num_hw_counters + cnts->num_op_counters); index++) {
470 ret = do_get_op_stat(ibdev, stats, port_num, index);
471 if (ret != index)
472 return ret;
473 }
474
475 return cnts->num_op_counters;
476 }
477
mlx5_ib_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)478 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
479 struct rdma_hw_stats *stats,
480 u32 port_num, int index)
481 {
482 int num_counters, num_hw_counters, num_op_counters;
483 struct mlx5_ib_dev *dev = to_mdev(ibdev);
484 const struct mlx5_ib_counters *cnts;
485
486 cnts = get_counters(dev, port_num);
487 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
488 cnts->num_ext_ppcnt_counters;
489 num_counters = num_hw_counters + cnts->num_op_counters;
490
491 if (index < 0 || index > num_counters)
492 return -EINVAL;
493 else if (index > 0 && index < num_hw_counters)
494 return do_get_hw_stats(ibdev, stats, port_num, index);
495 else if (index >= num_hw_counters && index < num_counters)
496 return do_get_op_stat(ibdev, stats, port_num, index);
497
498 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
499 if (num_hw_counters < 0)
500 return num_hw_counters;
501
502 num_op_counters = do_get_op_stats(ibdev, stats, port_num);
503 if (num_op_counters < 0)
504 return num_op_counters;
505
506 return num_hw_counters + num_op_counters;
507 }
508
509 static struct rdma_hw_stats *
mlx5_ib_counter_alloc_stats(struct rdma_counter * counter)510 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
511 {
512 struct mlx5_ib_dev *dev = to_mdev(counter->device);
513 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
514
515 return do_alloc_stats(cnts);
516 }
517
mlx5_ib_counter_update_stats(struct rdma_counter * counter)518 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
519 {
520 struct mlx5_ib_dev *dev = to_mdev(counter->device);
521 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
522
523 return mlx5_ib_query_q_counters(dev->mdev, cnts,
524 counter->stats, counter->id);
525 }
526
mlx5_ib_counter_dealloc(struct rdma_counter * counter)527 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
528 {
529 struct mlx5_ib_dev *dev = to_mdev(counter->device);
530 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
531
532 if (!counter->id)
533 return 0;
534
535 MLX5_SET(dealloc_q_counter_in, in, opcode,
536 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
537 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
538 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
539 }
540
mlx5_ib_counter_bind_qp(struct rdma_counter * counter,struct ib_qp * qp)541 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
542 struct ib_qp *qp)
543 {
544 struct mlx5_ib_dev *dev = to_mdev(qp->device);
545 bool new = false;
546 int err;
547
548 if (!counter->id) {
549 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
550 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
551
552 MLX5_SET(alloc_q_counter_in, in, opcode,
553 MLX5_CMD_OP_ALLOC_Q_COUNTER);
554 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
555 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
556 if (err)
557 return err;
558 counter->id =
559 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
560 new = true;
561 }
562
563 err = mlx5_ib_qp_set_counter(qp, counter);
564 if (err)
565 goto fail_set_counter;
566
567 return 0;
568
569 fail_set_counter:
570 if (new) {
571 mlx5_ib_counter_dealloc(counter);
572 counter->id = 0;
573 }
574
575 return err;
576 }
577
mlx5_ib_counter_unbind_qp(struct ib_qp * qp)578 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp)
579 {
580 return mlx5_ib_qp_set_counter(qp, NULL);
581 }
582
mlx5_ib_fill_counters(struct mlx5_ib_dev * dev,struct rdma_stat_desc * descs,size_t * offsets,u32 port_num)583 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
584 struct rdma_stat_desc *descs, size_t *offsets,
585 u32 port_num)
586 {
587 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
588 port_num != MLX5_VPORT_PF;
589 const struct mlx5_ib_counter *names;
590 int j = 0, i, size;
591
592 names = is_vport ? vport_basic_q_cnts : basic_q_cnts;
593 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
594 ARRAY_SIZE(basic_q_cnts);
595 for (i = 0; i < size; i++, j++) {
596 descs[j].name = names[i].name;
597 offsets[j] = names[i].offset;
598 }
599
600 names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;
601 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
602 ARRAY_SIZE(out_of_seq_q_cnts);
603 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
604 for (i = 0; i < size; i++, j++) {
605 descs[j].name = names[i].name;
606 offsets[j] = names[i].offset;
607 }
608 }
609
610 names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;
611 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
612 ARRAY_SIZE(retrans_q_cnts);
613 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
614 for (i = 0; i < size; i++, j++) {
615 descs[j].name = names[i].name;
616 offsets[j] = names[i].offset;
617 }
618 }
619
620 names = is_vport ? vport_extended_err_cnts : extended_err_cnts;
621 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
622 ARRAY_SIZE(extended_err_cnts);
623 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
624 for (i = 0; i < size; i++, j++) {
625 descs[j].name = names[i].name;
626 offsets[j] = names[i].offset;
627 }
628 }
629
630 names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;
631 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
632 ARRAY_SIZE(roce_accl_cnts);
633 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
634 for (i = 0; i < size; i++, j++) {
635 descs[j].name = names[i].name;
636 offsets[j] = names[i].offset;
637 }
638 }
639
640 if (is_vport)
641 return;
642
643 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
644 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
645 descs[j].name = cong_cnts[i].name;
646 offsets[j] = cong_cnts[i].offset;
647 }
648 }
649
650 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
651 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
652 descs[j].name = ext_ppcnt_cnts[i].name;
653 offsets[j] = ext_ppcnt_cnts[i].offset;
654 }
655 }
656
657 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
658 descs[j].name = basic_op_cnts[i].name;
659 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
660 descs[j].priv = &basic_op_cnts[i].type;
661 }
662
663 if (MLX5_CAP_FLOWTABLE(dev->mdev,
664 ft_field_support_2_nic_receive_rdma.bth_opcode)) {
665 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
666 descs[j].name = rdmarx_cnp_op_cnts[i].name;
667 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
668 descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
669 }
670 }
671
672 if (MLX5_CAP_FLOWTABLE(dev->mdev,
673 ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
674 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
675 descs[j].name = rdmatx_cnp_op_cnts[i].name;
676 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
677 descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
678 }
679 }
680 }
681
682
__mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev,struct mlx5_ib_counters * cnts,u32 port_num)683 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
684 struct mlx5_ib_counters *cnts, u32 port_num)
685 {
686 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
687 port_num != MLX5_VPORT_PF;
688 u32 num_counters, num_op_counters = 0, size;
689
690 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
691 ARRAY_SIZE(basic_q_cnts);
692 num_counters = size;
693
694 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
695 ARRAY_SIZE(out_of_seq_q_cnts);
696 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
697 num_counters += size;
698
699 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
700 ARRAY_SIZE(retrans_q_cnts);
701 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
702 num_counters += size;
703
704 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
705 ARRAY_SIZE(extended_err_cnts);
706 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
707 num_counters += size;
708
709 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
710 ARRAY_SIZE(roce_accl_cnts);
711 if (MLX5_CAP_GEN(dev->mdev, roce_accl))
712 num_counters += size;
713
714 cnts->num_q_counters = num_counters;
715
716 if (is_vport)
717 goto skip_non_qcounters;
718
719 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
720 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
721 num_counters += ARRAY_SIZE(cong_cnts);
722 }
723 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
724 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
725 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
726 }
727
728 num_op_counters = ARRAY_SIZE(basic_op_cnts);
729
730 if (MLX5_CAP_FLOWTABLE(dev->mdev,
731 ft_field_support_2_nic_receive_rdma.bth_opcode))
732 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
733
734 if (MLX5_CAP_FLOWTABLE(dev->mdev,
735 ft_field_support_2_nic_transmit_rdma.bth_opcode))
736 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
737
738 skip_non_qcounters:
739 cnts->num_op_counters = num_op_counters;
740 num_counters += num_op_counters;
741 cnts->descs = kcalloc(num_counters,
742 sizeof(struct rdma_stat_desc), GFP_KERNEL);
743 if (!cnts->descs)
744 return -ENOMEM;
745
746 cnts->offsets = kcalloc(num_counters,
747 sizeof(*cnts->offsets), GFP_KERNEL);
748 if (!cnts->offsets)
749 goto err;
750
751 return 0;
752
753 err:
754 kfree(cnts->descs);
755 cnts->descs = NULL;
756 return -ENOMEM;
757 }
758
mlx5_ib_dealloc_counters(struct mlx5_ib_dev * dev)759 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
760 {
761 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
762 int num_cnt_ports = dev->num_ports;
763 int i, j;
764
765 if (is_mdev_switchdev_mode(dev->mdev))
766 num_cnt_ports = min(2, num_cnt_ports);
767
768 MLX5_SET(dealloc_q_counter_in, in, opcode,
769 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
770
771 for (i = 0; i < num_cnt_ports; i++) {
772 if (dev->port[i].cnts.set_id) {
773 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
774 dev->port[i].cnts.set_id);
775 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
776 }
777 kfree(dev->port[i].cnts.descs);
778 kfree(dev->port[i].cnts.offsets);
779
780 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
781 if (!dev->port[i].cnts.opfcs[j].fc)
782 continue;
783
784 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
785 mlx5_ib_fs_remove_op_fc(dev,
786 &dev->port[i].cnts.opfcs[j], j);
787 mlx5_fc_destroy(dev->mdev,
788 dev->port[i].cnts.opfcs[j].fc);
789 dev->port[i].cnts.opfcs[j].fc = NULL;
790 }
791 }
792 }
793
mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev)794 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
795 {
796 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
797 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
798 int num_cnt_ports = dev->num_ports;
799 int err = 0;
800 int i;
801 bool is_shared;
802
803 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
804 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
805
806 /*
807 * In switchdev we need to allocate two ports, one that is used for
808 * the device Q_counters and it is essentially the real Q_counters of
809 * this device, while the other is used as a helper for PF to be able to
810 * query all other vports.
811 */
812 if (is_mdev_switchdev_mode(dev->mdev))
813 num_cnt_ports = min(2, num_cnt_ports);
814
815 for (i = 0; i < num_cnt_ports; i++) {
816 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i);
817 if (err)
818 goto err_alloc;
819
820 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
821 dev->port[i].cnts.offsets, i);
822
823 MLX5_SET(alloc_q_counter_in, in, uid,
824 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
825
826 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
827 if (err) {
828 mlx5_ib_warn(dev,
829 "couldn't allocate queue counter for port %d, err %d\n",
830 i + 1, err);
831 goto err_alloc;
832 }
833
834 dev->port[i].cnts.set_id =
835 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
836 }
837 return 0;
838
839 err_alloc:
840 mlx5_ib_dealloc_counters(dev);
841 return err;
842 }
843
read_flow_counters(struct ib_device * ibdev,struct mlx5_read_counters_attr * read_attr)844 static int read_flow_counters(struct ib_device *ibdev,
845 struct mlx5_read_counters_attr *read_attr)
846 {
847 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
848 struct mlx5_ib_dev *dev = to_mdev(ibdev);
849
850 return mlx5_fc_query(dev->mdev, fc,
851 &read_attr->out[IB_COUNTER_PACKETS],
852 &read_attr->out[IB_COUNTER_BYTES]);
853 }
854
855 /* flow counters currently expose two counters packets and bytes */
856 #define FLOW_COUNTERS_NUM 2
counters_set_description(struct ib_counters * counters,enum mlx5_ib_counters_type counters_type,struct mlx5_ib_flow_counters_desc * desc_data,u32 ncounters)857 static int counters_set_description(
858 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
859 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
860 {
861 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
862 u32 cntrs_max_index = 0;
863 int i;
864
865 if (counters_type != MLX5_IB_COUNTERS_FLOW)
866 return -EINVAL;
867
868 /* init the fields for the object */
869 mcounters->type = counters_type;
870 mcounters->read_counters = read_flow_counters;
871 mcounters->counters_num = FLOW_COUNTERS_NUM;
872 mcounters->ncounters = ncounters;
873 /* each counter entry have both description and index pair */
874 for (i = 0; i < ncounters; i++) {
875 if (desc_data[i].description > IB_COUNTER_BYTES)
876 return -EINVAL;
877
878 if (cntrs_max_index <= desc_data[i].index)
879 cntrs_max_index = desc_data[i].index + 1;
880 }
881
882 mutex_lock(&mcounters->mcntrs_mutex);
883 mcounters->counters_data = desc_data;
884 mcounters->cntrs_max_index = cntrs_max_index;
885 mutex_unlock(&mcounters->mcntrs_mutex);
886
887 return 0;
888 }
889
890 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
mlx5_ib_flow_counters_set_data(struct ib_counters * ibcounters,struct mlx5_ib_create_flow * ucmd)891 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
892 struct mlx5_ib_create_flow *ucmd)
893 {
894 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
895 struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
896 struct mlx5_ib_flow_counters_desc *desc_data = NULL;
897 bool hw_hndl = false;
898 int ret = 0;
899
900 if (ucmd && ucmd->ncounters_data != 0) {
901 cntrs_data = ucmd->data;
902 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
903 return -EINVAL;
904
905 desc_data = kcalloc(cntrs_data->ncounters,
906 sizeof(*desc_data),
907 GFP_KERNEL);
908 if (!desc_data)
909 return -ENOMEM;
910
911 if (copy_from_user(desc_data,
912 u64_to_user_ptr(cntrs_data->counters_data),
913 sizeof(*desc_data) * cntrs_data->ncounters)) {
914 ret = -EFAULT;
915 goto free;
916 }
917 }
918
919 if (!mcounters->hw_cntrs_hndl) {
920 mcounters->hw_cntrs_hndl = mlx5_fc_create(
921 to_mdev(ibcounters->device)->mdev, false);
922 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
923 ret = PTR_ERR(mcounters->hw_cntrs_hndl);
924 goto free;
925 }
926 hw_hndl = true;
927 }
928
929 if (desc_data) {
930 /* counters already bound to at least one flow */
931 if (mcounters->cntrs_max_index) {
932 ret = -EINVAL;
933 goto free_hndl;
934 }
935
936 ret = counters_set_description(ibcounters,
937 MLX5_IB_COUNTERS_FLOW,
938 desc_data,
939 cntrs_data->ncounters);
940 if (ret)
941 goto free_hndl;
942
943 } else if (!mcounters->cntrs_max_index) {
944 /* counters not bound yet, must have udata passed */
945 ret = -EINVAL;
946 goto free_hndl;
947 }
948
949 return 0;
950
951 free_hndl:
952 if (hw_hndl) {
953 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
954 mcounters->hw_cntrs_hndl);
955 mcounters->hw_cntrs_hndl = NULL;
956 }
957 free:
958 kfree(desc_data);
959 return ret;
960 }
961
mlx5_ib_counters_clear_description(struct ib_counters * counters)962 void mlx5_ib_counters_clear_description(struct ib_counters *counters)
963 {
964 struct mlx5_ib_mcounters *mcounters;
965
966 if (!counters || atomic_read(&counters->usecnt) != 1)
967 return;
968
969 mcounters = to_mcounters(counters);
970
971 mutex_lock(&mcounters->mcntrs_mutex);
972 kfree(mcounters->counters_data);
973 mcounters->counters_data = NULL;
974 mcounters->cntrs_max_index = 0;
975 mutex_unlock(&mcounters->mcntrs_mutex);
976 }
977
mlx5_ib_modify_stat(struct ib_device * device,u32 port,unsigned int index,bool enable)978 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
979 unsigned int index, bool enable)
980 {
981 struct mlx5_ib_dev *dev = to_mdev(device);
982 struct mlx5_ib_counters *cnts;
983 struct mlx5_ib_op_fc *opfc;
984 u32 num_hw_counters, type;
985 int ret;
986
987 cnts = &dev->port[port - 1].cnts;
988 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
989 cnts->num_ext_ppcnt_counters;
990 if (index < num_hw_counters ||
991 index >= (num_hw_counters + cnts->num_op_counters))
992 return -EINVAL;
993
994 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
995 return -EINVAL;
996
997 type = *(u32 *)cnts->descs[index].priv;
998 if (type >= MLX5_IB_OPCOUNTER_MAX)
999 return -EINVAL;
1000
1001 opfc = &cnts->opfcs[type];
1002
1003 if (enable) {
1004 if (opfc->fc)
1005 return -EEXIST;
1006
1007 opfc->fc = mlx5_fc_create(dev->mdev, false);
1008 if (IS_ERR(opfc->fc))
1009 return PTR_ERR(opfc->fc);
1010
1011 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
1012 if (ret) {
1013 mlx5_fc_destroy(dev->mdev, opfc->fc);
1014 opfc->fc = NULL;
1015 }
1016 return ret;
1017 }
1018
1019 if (!opfc->fc)
1020 return -EINVAL;
1021
1022 mlx5_ib_fs_remove_op_fc(dev, opfc, type);
1023 mlx5_fc_destroy(dev->mdev, opfc->fc);
1024 opfc->fc = NULL;
1025 return 0;
1026 }
1027
1028 static const struct ib_device_ops hw_stats_ops = {
1029 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1030 .get_hw_stats = mlx5_ib_get_hw_stats,
1031 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1032 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1033 .counter_dealloc = mlx5_ib_counter_dealloc,
1034 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1035 .counter_update_stats = mlx5_ib_counter_update_stats,
1036 .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ?
1037 mlx5_ib_modify_stat : NULL,
1038 };
1039
1040 static const struct ib_device_ops hw_switchdev_vport_op = {
1041 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1042 };
1043
1044 static const struct ib_device_ops hw_switchdev_stats_ops = {
1045 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
1046 .get_hw_stats = mlx5_ib_get_hw_stats,
1047 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1048 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1049 .counter_dealloc = mlx5_ib_counter_dealloc,
1050 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1051 .counter_update_stats = mlx5_ib_counter_update_stats,
1052 };
1053
1054 static const struct ib_device_ops counters_ops = {
1055 .create_counters = mlx5_ib_create_counters,
1056 .destroy_counters = mlx5_ib_destroy_counters,
1057 .read_counters = mlx5_ib_read_counters,
1058
1059 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
1060 };
1061
mlx5_ib_counters_init(struct mlx5_ib_dev * dev)1062 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
1063 {
1064 ib_set_device_ops(&dev->ib_dev, &counters_ops);
1065
1066 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1067 return 0;
1068
1069 if (is_mdev_switchdev_mode(dev->mdev)) {
1070 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
1071 if (vport_qcounters_supported(dev))
1072 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op);
1073 } else
1074 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
1075 return mlx5_ib_alloc_counters(dev);
1076 }
1077
mlx5_ib_counters_cleanup(struct mlx5_ib_dev * dev)1078 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
1079 {
1080 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1081 return;
1082
1083 mlx5_ib_dealloc_counters(dev);
1084 }
1085