1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include "mlx5_ib.h" 7 #include <linux/mlx5/eswitch.h> 8 #include "counters.h" 9 #include "ib_rep.h" 10 #include "qp.h" 11 12 struct mlx5_ib_counter { 13 const char *name; 14 size_t offset; 15 u32 type; 16 }; 17 18 #define INIT_Q_COUNTER(_name) \ 19 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} 20 21 static const struct mlx5_ib_counter basic_q_cnts[] = { 22 INIT_Q_COUNTER(rx_write_requests), 23 INIT_Q_COUNTER(rx_read_requests), 24 INIT_Q_COUNTER(rx_atomic_requests), 25 INIT_Q_COUNTER(out_of_buffer), 26 }; 27 28 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { 29 INIT_Q_COUNTER(out_of_sequence), 30 }; 31 32 static const struct mlx5_ib_counter retrans_q_cnts[] = { 33 INIT_Q_COUNTER(duplicate_request), 34 INIT_Q_COUNTER(rnr_nak_retry_err), 35 INIT_Q_COUNTER(packet_seq_err), 36 INIT_Q_COUNTER(implied_nak_seq_err), 37 INIT_Q_COUNTER(local_ack_timeout_err), 38 }; 39 40 #define INIT_CONG_COUNTER(_name) \ 41 { .name = #_name, .offset = \ 42 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} 43 44 static const struct mlx5_ib_counter cong_cnts[] = { 45 INIT_CONG_COUNTER(rp_cnp_ignored), 46 INIT_CONG_COUNTER(rp_cnp_handled), 47 INIT_CONG_COUNTER(np_ecn_marked_roce_packets), 48 INIT_CONG_COUNTER(np_cnp_sent), 49 }; 50 51 static const struct mlx5_ib_counter extended_err_cnts[] = { 52 INIT_Q_COUNTER(resp_local_length_error), 53 INIT_Q_COUNTER(resp_cqe_error), 54 INIT_Q_COUNTER(req_cqe_error), 55 INIT_Q_COUNTER(req_remote_invalid_request), 56 INIT_Q_COUNTER(req_remote_access_errors), 57 INIT_Q_COUNTER(resp_remote_access_errors), 58 INIT_Q_COUNTER(resp_cqe_flush_error), 59 INIT_Q_COUNTER(req_cqe_flush_error), 60 }; 61 62 static const struct mlx5_ib_counter roce_accl_cnts[] = { 63 INIT_Q_COUNTER(roce_adp_retrans), 64 INIT_Q_COUNTER(roce_adp_retrans_to), 65 INIT_Q_COUNTER(roce_slow_restart), 66 INIT_Q_COUNTER(roce_slow_restart_cnps), 67 INIT_Q_COUNTER(roce_slow_restart_trans), 68 }; 69 70 #define INIT_EXT_PPCNT_COUNTER(_name) \ 71 { .name = #_name, .offset = \ 72 MLX5_BYTE_OFF(ppcnt_reg, \ 73 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} 74 75 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { 76 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), 77 }; 78 79 #define INIT_OP_COUNTER(_name, _type) \ 80 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type} 81 82 static const struct mlx5_ib_counter basic_op_cnts[] = { 83 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS), 84 }; 85 86 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = { 87 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS), 88 }; 89 90 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = { 91 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS), 92 }; 93 94 static int mlx5_ib_read_counters(struct ib_counters *counters, 95 struct ib_counters_read_attr *read_attr, 96 struct uverbs_attr_bundle *attrs) 97 { 98 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 99 struct mlx5_read_counters_attr mread_attr = {}; 100 struct mlx5_ib_flow_counters_desc *desc; 101 int ret, i; 102 103 mutex_lock(&mcounters->mcntrs_mutex); 104 if (mcounters->cntrs_max_index > read_attr->ncounters) { 105 ret = -EINVAL; 106 goto err_bound; 107 } 108 109 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), 110 GFP_KERNEL); 111 if (!mread_attr.out) { 112 ret = -ENOMEM; 113 goto err_bound; 114 } 115 116 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; 117 mread_attr.flags = read_attr->flags; 118 ret = mcounters->read_counters(counters->device, &mread_attr); 119 if (ret) 120 goto err_read; 121 122 /* do the pass over the counters data array to assign according to the 123 * descriptions and indexing pairs 124 */ 125 desc = mcounters->counters_data; 126 for (i = 0; i < mcounters->ncounters; i++) 127 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; 128 129 err_read: 130 kfree(mread_attr.out); 131 err_bound: 132 mutex_unlock(&mcounters->mcntrs_mutex); 133 return ret; 134 } 135 136 static int mlx5_ib_destroy_counters(struct ib_counters *counters) 137 { 138 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 139 140 mlx5_ib_counters_clear_description(counters); 141 if (mcounters->hw_cntrs_hndl) 142 mlx5_fc_destroy(to_mdev(counters->device)->mdev, 143 mcounters->hw_cntrs_hndl); 144 return 0; 145 } 146 147 static int mlx5_ib_create_counters(struct ib_counters *counters, 148 struct uverbs_attr_bundle *attrs) 149 { 150 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 151 152 mutex_init(&mcounters->mcntrs_mutex); 153 return 0; 154 } 155 156 157 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, 158 u32 port_num) 159 { 160 return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : 161 &dev->port[port_num].cnts; 162 } 163 164 /** 165 * mlx5_ib_get_counters_id - Returns counters id to use for device+port 166 * @dev: Pointer to mlx5 IB device 167 * @port_num: Zero based port number 168 * 169 * mlx5_ib_get_counters_id() Returns counters set id to use for given 170 * device port combination in switchdev and non switchdev mode of the 171 * parent device. 172 */ 173 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num) 174 { 175 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 176 177 return cnts->set_id; 178 } 179 180 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts) 181 { 182 struct rdma_hw_stats *stats; 183 u32 num_hw_counters; 184 int i; 185 186 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 187 cnts->num_ext_ppcnt_counters; 188 stats = rdma_alloc_hw_stats_struct(cnts->descs, 189 num_hw_counters + 190 cnts->num_op_counters, 191 RDMA_HW_STATS_DEFAULT_LIFESPAN); 192 if (!stats) 193 return NULL; 194 195 for (i = 0; i < cnts->num_op_counters; i++) 196 set_bit(num_hw_counters + i, stats->is_disabled); 197 198 return stats; 199 } 200 201 static struct rdma_hw_stats * 202 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev) 203 { 204 struct mlx5_ib_dev *dev = to_mdev(ibdev); 205 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts; 206 207 return do_alloc_stats(cnts); 208 } 209 210 static struct rdma_hw_stats * 211 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num) 212 { 213 struct mlx5_ib_dev *dev = to_mdev(ibdev); 214 const struct mlx5_ib_counters *cnts = &dev->port[port_num - 1].cnts; 215 216 return do_alloc_stats(cnts); 217 } 218 219 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, 220 const struct mlx5_ib_counters *cnts, 221 struct rdma_hw_stats *stats, 222 u16 set_id) 223 { 224 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 225 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 226 __be32 val; 227 int ret, i; 228 229 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 230 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); 231 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 232 if (ret) 233 return ret; 234 235 for (i = 0; i < cnts->num_q_counters; i++) { 236 val = *(__be32 *)((void *)out + cnts->offsets[i]); 237 stats->value[i] = (u64)be32_to_cpu(val); 238 } 239 240 return 0; 241 } 242 243 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, 244 const struct mlx5_ib_counters *cnts, 245 struct rdma_hw_stats *stats) 246 { 247 int offset = cnts->num_q_counters + cnts->num_cong_counters; 248 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; 249 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); 250 int ret, i; 251 void *out; 252 253 out = kvzalloc(sz, GFP_KERNEL); 254 if (!out) 255 return -ENOMEM; 256 257 MLX5_SET(ppcnt_reg, in, local_port, 1); 258 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); 259 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, 260 0, 0); 261 if (ret) 262 goto free; 263 264 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) 265 stats->value[i + offset] = 266 be64_to_cpup((__be64 *)(out + 267 cnts->offsets[i + offset])); 268 free: 269 kvfree(out); 270 return ret; 271 } 272 273 static int do_get_hw_stats(struct ib_device *ibdev, 274 struct rdma_hw_stats *stats, 275 u32 port_num, int index) 276 { 277 struct mlx5_ib_dev *dev = to_mdev(ibdev); 278 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); 279 struct mlx5_core_dev *mdev; 280 int ret, num_counters; 281 u32 mdev_port_num; 282 283 if (!stats) 284 return -EINVAL; 285 286 num_counters = cnts->num_q_counters + 287 cnts->num_cong_counters + 288 cnts->num_ext_ppcnt_counters; 289 290 /* q_counters are per IB device, query the master mdev */ 291 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id); 292 if (ret) 293 return ret; 294 295 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 296 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); 297 if (ret) 298 return ret; 299 } 300 301 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 302 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, 303 &mdev_port_num); 304 if (!mdev) { 305 /* If port is not affiliated yet, its in down state 306 * which doesn't have any counters yet, so it would be 307 * zero. So no need to read from the HCA. 308 */ 309 goto done; 310 } 311 ret = mlx5_lag_query_cong_counters(dev->mdev, 312 stats->value + 313 cnts->num_q_counters, 314 cnts->num_cong_counters, 315 cnts->offsets + 316 cnts->num_q_counters); 317 318 mlx5_ib_put_native_port_mdev(dev, port_num); 319 if (ret) 320 return ret; 321 } 322 323 done: 324 return num_counters; 325 } 326 327 static int do_get_op_stat(struct ib_device *ibdev, 328 struct rdma_hw_stats *stats, 329 u32 port_num, int index) 330 { 331 struct mlx5_ib_dev *dev = to_mdev(ibdev); 332 const struct mlx5_ib_counters *cnts; 333 const struct mlx5_ib_op_fc *opfcs; 334 u64 packets = 0, bytes; 335 u32 type; 336 int ret; 337 338 cnts = get_counters(dev, port_num - 1); 339 opfcs = cnts->opfcs; 340 type = *(u32 *)cnts->descs[index].priv; 341 if (type >= MLX5_IB_OPCOUNTER_MAX) 342 return -EINVAL; 343 344 if (!opfcs[type].fc) 345 goto out; 346 347 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc, 348 &packets, &bytes); 349 if (ret) 350 return ret; 351 352 out: 353 stats->value[index] = packets; 354 return index; 355 } 356 357 static int do_get_op_stats(struct ib_device *ibdev, 358 struct rdma_hw_stats *stats, 359 u32 port_num) 360 { 361 struct mlx5_ib_dev *dev = to_mdev(ibdev); 362 const struct mlx5_ib_counters *cnts; 363 int index, ret, num_hw_counters; 364 365 cnts = get_counters(dev, port_num - 1); 366 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 367 cnts->num_ext_ppcnt_counters; 368 for (index = num_hw_counters; 369 index < (num_hw_counters + cnts->num_op_counters); index++) { 370 ret = do_get_op_stat(ibdev, stats, port_num, index); 371 if (ret != index) 372 return ret; 373 } 374 375 return cnts->num_op_counters; 376 } 377 378 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, 379 struct rdma_hw_stats *stats, 380 u32 port_num, int index) 381 { 382 int num_counters, num_hw_counters, num_op_counters; 383 struct mlx5_ib_dev *dev = to_mdev(ibdev); 384 const struct mlx5_ib_counters *cnts; 385 386 cnts = get_counters(dev, port_num - 1); 387 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 388 cnts->num_ext_ppcnt_counters; 389 num_counters = num_hw_counters + cnts->num_op_counters; 390 391 if (index < 0 || index > num_counters) 392 return -EINVAL; 393 else if (index > 0 && index < num_hw_counters) 394 return do_get_hw_stats(ibdev, stats, port_num, index); 395 else if (index >= num_hw_counters && index < num_counters) 396 return do_get_op_stat(ibdev, stats, port_num, index); 397 398 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index); 399 if (num_hw_counters < 0) 400 return num_hw_counters; 401 402 num_op_counters = do_get_op_stats(ibdev, stats, port_num); 403 if (num_op_counters < 0) 404 return num_op_counters; 405 406 return num_hw_counters + num_op_counters; 407 } 408 409 static struct rdma_hw_stats * 410 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) 411 { 412 struct mlx5_ib_dev *dev = to_mdev(counter->device); 413 const struct mlx5_ib_counters *cnts = 414 get_counters(dev, counter->port - 1); 415 416 return do_alloc_stats(cnts); 417 } 418 419 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) 420 { 421 struct mlx5_ib_dev *dev = to_mdev(counter->device); 422 const struct mlx5_ib_counters *cnts = 423 get_counters(dev, counter->port - 1); 424 425 return mlx5_ib_query_q_counters(dev->mdev, cnts, 426 counter->stats, counter->id); 427 } 428 429 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) 430 { 431 struct mlx5_ib_dev *dev = to_mdev(counter->device); 432 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 433 434 if (!counter->id) 435 return 0; 436 437 MLX5_SET(dealloc_q_counter_in, in, opcode, 438 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 439 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); 440 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 441 } 442 443 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, 444 struct ib_qp *qp) 445 { 446 struct mlx5_ib_dev *dev = to_mdev(qp->device); 447 int err; 448 449 if (!counter->id) { 450 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 451 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 452 453 MLX5_SET(alloc_q_counter_in, in, opcode, 454 MLX5_CMD_OP_ALLOC_Q_COUNTER); 455 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); 456 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 457 if (err) 458 return err; 459 counter->id = 460 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 461 } 462 463 err = mlx5_ib_qp_set_counter(qp, counter); 464 if (err) 465 goto fail_set_counter; 466 467 return 0; 468 469 fail_set_counter: 470 mlx5_ib_counter_dealloc(counter); 471 counter->id = 0; 472 473 return err; 474 } 475 476 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) 477 { 478 return mlx5_ib_qp_set_counter(qp, NULL); 479 } 480 481 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, 482 struct rdma_stat_desc *descs, size_t *offsets) 483 { 484 int i; 485 int j = 0; 486 487 for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) { 488 descs[j].name = basic_q_cnts[i].name; 489 offsets[j] = basic_q_cnts[i].offset; 490 } 491 492 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { 493 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) { 494 descs[j].name = out_of_seq_q_cnts[i].name; 495 offsets[j] = out_of_seq_q_cnts[i].offset; 496 } 497 } 498 499 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 500 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) { 501 descs[j].name = retrans_q_cnts[i].name; 502 offsets[j] = retrans_q_cnts[i].offset; 503 } 504 } 505 506 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { 507 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) { 508 descs[j].name = extended_err_cnts[i].name; 509 offsets[j] = extended_err_cnts[i].offset; 510 } 511 } 512 513 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { 514 for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { 515 descs[j].name = roce_accl_cnts[i].name; 516 offsets[j] = roce_accl_cnts[i].offset; 517 } 518 } 519 520 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 521 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { 522 descs[j].name = cong_cnts[i].name; 523 offsets[j] = cong_cnts[i].offset; 524 } 525 } 526 527 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 528 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { 529 descs[j].name = ext_ppcnt_cnts[i].name; 530 offsets[j] = ext_ppcnt_cnts[i].offset; 531 } 532 } 533 534 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) { 535 descs[j].name = basic_op_cnts[i].name; 536 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 537 descs[j].priv = &basic_op_cnts[i].type; 538 } 539 540 if (MLX5_CAP_FLOWTABLE(dev->mdev, 541 ft_field_support_2_nic_receive_rdma.bth_opcode)) { 542 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) { 543 descs[j].name = rdmarx_cnp_op_cnts[i].name; 544 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 545 descs[j].priv = &rdmarx_cnp_op_cnts[i].type; 546 } 547 } 548 549 if (MLX5_CAP_FLOWTABLE(dev->mdev, 550 ft_field_support_2_nic_transmit_rdma.bth_opcode)) { 551 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) { 552 descs[j].name = rdmatx_cnp_op_cnts[i].name; 553 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 554 descs[j].priv = &rdmatx_cnp_op_cnts[i].type; 555 } 556 } 557 } 558 559 560 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, 561 struct mlx5_ib_counters *cnts) 562 { 563 u32 num_counters, num_op_counters; 564 565 num_counters = ARRAY_SIZE(basic_q_cnts); 566 567 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) 568 num_counters += ARRAY_SIZE(out_of_seq_q_cnts); 569 570 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 571 num_counters += ARRAY_SIZE(retrans_q_cnts); 572 573 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) 574 num_counters += ARRAY_SIZE(extended_err_cnts); 575 576 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) 577 num_counters += ARRAY_SIZE(roce_accl_cnts); 578 579 cnts->num_q_counters = num_counters; 580 581 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 582 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); 583 num_counters += ARRAY_SIZE(cong_cnts); 584 } 585 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 586 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); 587 num_counters += ARRAY_SIZE(ext_ppcnt_cnts); 588 } 589 590 num_op_counters = ARRAY_SIZE(basic_op_cnts); 591 592 if (MLX5_CAP_FLOWTABLE(dev->mdev, 593 ft_field_support_2_nic_receive_rdma.bth_opcode)) 594 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts); 595 596 if (MLX5_CAP_FLOWTABLE(dev->mdev, 597 ft_field_support_2_nic_transmit_rdma.bth_opcode)) 598 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts); 599 600 cnts->num_op_counters = num_op_counters; 601 num_counters += num_op_counters; 602 cnts->descs = kcalloc(num_counters, 603 sizeof(struct rdma_stat_desc), GFP_KERNEL); 604 if (!cnts->descs) 605 return -ENOMEM; 606 607 cnts->offsets = kcalloc(num_counters, 608 sizeof(*cnts->offsets), GFP_KERNEL); 609 if (!cnts->offsets) 610 goto err; 611 612 return 0; 613 614 err: 615 kfree(cnts->descs); 616 cnts->descs = NULL; 617 return -ENOMEM; 618 } 619 620 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) 621 { 622 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 623 int num_cnt_ports; 624 int i, j; 625 626 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 627 628 MLX5_SET(dealloc_q_counter_in, in, opcode, 629 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 630 631 for (i = 0; i < num_cnt_ports; i++) { 632 if (dev->port[i].cnts.set_id) { 633 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 634 dev->port[i].cnts.set_id); 635 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 636 } 637 kfree(dev->port[i].cnts.descs); 638 kfree(dev->port[i].cnts.offsets); 639 640 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) { 641 if (!dev->port[i].cnts.opfcs[j].fc) 642 continue; 643 644 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) 645 mlx5_ib_fs_remove_op_fc(dev, 646 &dev->port[i].cnts.opfcs[j], j); 647 mlx5_fc_destroy(dev->mdev, 648 dev->port[i].cnts.opfcs[j].fc); 649 dev->port[i].cnts.opfcs[j].fc = NULL; 650 } 651 } 652 } 653 654 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) 655 { 656 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 657 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 658 int num_cnt_ports; 659 int err = 0; 660 int i; 661 bool is_shared; 662 663 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 664 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; 665 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 666 667 for (i = 0; i < num_cnt_ports; i++) { 668 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts); 669 if (err) 670 goto err_alloc; 671 672 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs, 673 dev->port[i].cnts.offsets); 674 675 MLX5_SET(alloc_q_counter_in, in, uid, 676 is_shared ? MLX5_SHARED_RESOURCE_UID : 0); 677 678 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 679 if (err) { 680 mlx5_ib_warn(dev, 681 "couldn't allocate queue counter for port %d, err %d\n", 682 i + 1, err); 683 goto err_alloc; 684 } 685 686 dev->port[i].cnts.set_id = 687 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 688 } 689 return 0; 690 691 err_alloc: 692 mlx5_ib_dealloc_counters(dev); 693 return err; 694 } 695 696 static int read_flow_counters(struct ib_device *ibdev, 697 struct mlx5_read_counters_attr *read_attr) 698 { 699 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; 700 struct mlx5_ib_dev *dev = to_mdev(ibdev); 701 702 return mlx5_fc_query(dev->mdev, fc, 703 &read_attr->out[IB_COUNTER_PACKETS], 704 &read_attr->out[IB_COUNTER_BYTES]); 705 } 706 707 /* flow counters currently expose two counters packets and bytes */ 708 #define FLOW_COUNTERS_NUM 2 709 static int counters_set_description( 710 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, 711 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) 712 { 713 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 714 u32 cntrs_max_index = 0; 715 int i; 716 717 if (counters_type != MLX5_IB_COUNTERS_FLOW) 718 return -EINVAL; 719 720 /* init the fields for the object */ 721 mcounters->type = counters_type; 722 mcounters->read_counters = read_flow_counters; 723 mcounters->counters_num = FLOW_COUNTERS_NUM; 724 mcounters->ncounters = ncounters; 725 /* each counter entry have both description and index pair */ 726 for (i = 0; i < ncounters; i++) { 727 if (desc_data[i].description > IB_COUNTER_BYTES) 728 return -EINVAL; 729 730 if (cntrs_max_index <= desc_data[i].index) 731 cntrs_max_index = desc_data[i].index + 1; 732 } 733 734 mutex_lock(&mcounters->mcntrs_mutex); 735 mcounters->counters_data = desc_data; 736 mcounters->cntrs_max_index = cntrs_max_index; 737 mutex_unlock(&mcounters->mcntrs_mutex); 738 739 return 0; 740 } 741 742 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) 743 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, 744 struct mlx5_ib_create_flow *ucmd) 745 { 746 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); 747 struct mlx5_ib_flow_counters_data *cntrs_data = NULL; 748 struct mlx5_ib_flow_counters_desc *desc_data = NULL; 749 bool hw_hndl = false; 750 int ret = 0; 751 752 if (ucmd && ucmd->ncounters_data != 0) { 753 cntrs_data = ucmd->data; 754 if (cntrs_data->ncounters > MAX_COUNTERS_NUM) 755 return -EINVAL; 756 757 desc_data = kcalloc(cntrs_data->ncounters, 758 sizeof(*desc_data), 759 GFP_KERNEL); 760 if (!desc_data) 761 return -ENOMEM; 762 763 if (copy_from_user(desc_data, 764 u64_to_user_ptr(cntrs_data->counters_data), 765 sizeof(*desc_data) * cntrs_data->ncounters)) { 766 ret = -EFAULT; 767 goto free; 768 } 769 } 770 771 if (!mcounters->hw_cntrs_hndl) { 772 mcounters->hw_cntrs_hndl = mlx5_fc_create( 773 to_mdev(ibcounters->device)->mdev, false); 774 if (IS_ERR(mcounters->hw_cntrs_hndl)) { 775 ret = PTR_ERR(mcounters->hw_cntrs_hndl); 776 goto free; 777 } 778 hw_hndl = true; 779 } 780 781 if (desc_data) { 782 /* counters already bound to at least one flow */ 783 if (mcounters->cntrs_max_index) { 784 ret = -EINVAL; 785 goto free_hndl; 786 } 787 788 ret = counters_set_description(ibcounters, 789 MLX5_IB_COUNTERS_FLOW, 790 desc_data, 791 cntrs_data->ncounters); 792 if (ret) 793 goto free_hndl; 794 795 } else if (!mcounters->cntrs_max_index) { 796 /* counters not bound yet, must have udata passed */ 797 ret = -EINVAL; 798 goto free_hndl; 799 } 800 801 return 0; 802 803 free_hndl: 804 if (hw_hndl) { 805 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, 806 mcounters->hw_cntrs_hndl); 807 mcounters->hw_cntrs_hndl = NULL; 808 } 809 free: 810 kfree(desc_data); 811 return ret; 812 } 813 814 void mlx5_ib_counters_clear_description(struct ib_counters *counters) 815 { 816 struct mlx5_ib_mcounters *mcounters; 817 818 if (!counters || atomic_read(&counters->usecnt) != 1) 819 return; 820 821 mcounters = to_mcounters(counters); 822 823 mutex_lock(&mcounters->mcntrs_mutex); 824 kfree(mcounters->counters_data); 825 mcounters->counters_data = NULL; 826 mcounters->cntrs_max_index = 0; 827 mutex_unlock(&mcounters->mcntrs_mutex); 828 } 829 830 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port, 831 unsigned int index, bool enable) 832 { 833 struct mlx5_ib_dev *dev = to_mdev(device); 834 struct mlx5_ib_counters *cnts; 835 struct mlx5_ib_op_fc *opfc; 836 u32 num_hw_counters, type; 837 int ret; 838 839 cnts = &dev->port[port - 1].cnts; 840 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 841 cnts->num_ext_ppcnt_counters; 842 if (index < num_hw_counters || 843 index >= (num_hw_counters + cnts->num_op_counters)) 844 return -EINVAL; 845 846 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 847 return -EINVAL; 848 849 type = *(u32 *)cnts->descs[index].priv; 850 if (type >= MLX5_IB_OPCOUNTER_MAX) 851 return -EINVAL; 852 853 opfc = &cnts->opfcs[type]; 854 855 if (enable) { 856 if (opfc->fc) 857 return -EEXIST; 858 859 opfc->fc = mlx5_fc_create(dev->mdev, false); 860 if (IS_ERR(opfc->fc)) 861 return PTR_ERR(opfc->fc); 862 863 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type); 864 if (ret) { 865 mlx5_fc_destroy(dev->mdev, opfc->fc); 866 opfc->fc = NULL; 867 } 868 return ret; 869 } 870 871 if (!opfc->fc) 872 return -EINVAL; 873 874 mlx5_ib_fs_remove_op_fc(dev, opfc, type); 875 mlx5_fc_destroy(dev->mdev, opfc->fc); 876 opfc->fc = NULL; 877 return 0; 878 } 879 880 static const struct ib_device_ops hw_stats_ops = { 881 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats, 882 .get_hw_stats = mlx5_ib_get_hw_stats, 883 .counter_bind_qp = mlx5_ib_counter_bind_qp, 884 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 885 .counter_dealloc = mlx5_ib_counter_dealloc, 886 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 887 .counter_update_stats = mlx5_ib_counter_update_stats, 888 .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ? 889 mlx5_ib_modify_stat : NULL, 890 }; 891 892 static const struct ib_device_ops hw_switchdev_stats_ops = { 893 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats, 894 .get_hw_stats = mlx5_ib_get_hw_stats, 895 .counter_bind_qp = mlx5_ib_counter_bind_qp, 896 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 897 .counter_dealloc = mlx5_ib_counter_dealloc, 898 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 899 .counter_update_stats = mlx5_ib_counter_update_stats, 900 }; 901 902 static const struct ib_device_ops counters_ops = { 903 .create_counters = mlx5_ib_create_counters, 904 .destroy_counters = mlx5_ib_destroy_counters, 905 .read_counters = mlx5_ib_read_counters, 906 907 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), 908 }; 909 910 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) 911 { 912 ib_set_device_ops(&dev->ib_dev, &counters_ops); 913 914 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 915 return 0; 916 917 if (is_mdev_switchdev_mode(dev->mdev)) 918 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops); 919 else 920 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); 921 return mlx5_ib_alloc_counters(dev); 922 } 923 924 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) 925 { 926 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 927 return; 928 929 mlx5_ib_dealloc_counters(dev); 930 } 931