1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include "mlx5_ib.h" 7 #include <linux/mlx5/eswitch.h> 8 #include "counters.h" 9 #include "ib_rep.h" 10 #include "qp.h" 11 12 struct mlx5_ib_counter { 13 const char *name; 14 size_t offset; 15 }; 16 17 #define INIT_Q_COUNTER(_name) \ 18 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} 19 20 static const struct mlx5_ib_counter basic_q_cnts[] = { 21 INIT_Q_COUNTER(rx_write_requests), 22 INIT_Q_COUNTER(rx_read_requests), 23 INIT_Q_COUNTER(rx_atomic_requests), 24 INIT_Q_COUNTER(out_of_buffer), 25 }; 26 27 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { 28 INIT_Q_COUNTER(out_of_sequence), 29 }; 30 31 static const struct mlx5_ib_counter retrans_q_cnts[] = { 32 INIT_Q_COUNTER(duplicate_request), 33 INIT_Q_COUNTER(rnr_nak_retry_err), 34 INIT_Q_COUNTER(packet_seq_err), 35 INIT_Q_COUNTER(implied_nak_seq_err), 36 INIT_Q_COUNTER(local_ack_timeout_err), 37 }; 38 39 #define INIT_CONG_COUNTER(_name) \ 40 { .name = #_name, .offset = \ 41 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} 42 43 static const struct mlx5_ib_counter cong_cnts[] = { 44 INIT_CONG_COUNTER(rp_cnp_ignored), 45 INIT_CONG_COUNTER(rp_cnp_handled), 46 INIT_CONG_COUNTER(np_ecn_marked_roce_packets), 47 INIT_CONG_COUNTER(np_cnp_sent), 48 }; 49 50 static const struct mlx5_ib_counter extended_err_cnts[] = { 51 INIT_Q_COUNTER(resp_local_length_error), 52 INIT_Q_COUNTER(resp_cqe_error), 53 INIT_Q_COUNTER(req_cqe_error), 54 INIT_Q_COUNTER(req_remote_invalid_request), 55 INIT_Q_COUNTER(req_remote_access_errors), 56 INIT_Q_COUNTER(resp_remote_access_errors), 57 INIT_Q_COUNTER(resp_cqe_flush_error), 58 INIT_Q_COUNTER(req_cqe_flush_error), 59 }; 60 61 static const struct mlx5_ib_counter roce_accl_cnts[] = { 62 INIT_Q_COUNTER(roce_adp_retrans), 63 INIT_Q_COUNTER(roce_adp_retrans_to), 64 INIT_Q_COUNTER(roce_slow_restart), 65 INIT_Q_COUNTER(roce_slow_restart_cnps), 66 INIT_Q_COUNTER(roce_slow_restart_trans), 67 }; 68 69 #define INIT_EXT_PPCNT_COUNTER(_name) \ 70 { .name = #_name, .offset = \ 71 MLX5_BYTE_OFF(ppcnt_reg, \ 72 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} 73 74 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { 75 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), 76 }; 77 78 static int mlx5_ib_read_counters(struct ib_counters *counters, 79 struct ib_counters_read_attr *read_attr, 80 struct uverbs_attr_bundle *attrs) 81 { 82 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 83 struct mlx5_read_counters_attr mread_attr = {}; 84 struct mlx5_ib_flow_counters_desc *desc; 85 int ret, i; 86 87 mutex_lock(&mcounters->mcntrs_mutex); 88 if (mcounters->cntrs_max_index > read_attr->ncounters) { 89 ret = -EINVAL; 90 goto err_bound; 91 } 92 93 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), 94 GFP_KERNEL); 95 if (!mread_attr.out) { 96 ret = -ENOMEM; 97 goto err_bound; 98 } 99 100 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; 101 mread_attr.flags = read_attr->flags; 102 ret = mcounters->read_counters(counters->device, &mread_attr); 103 if (ret) 104 goto err_read; 105 106 /* do the pass over the counters data array to assign according to the 107 * descriptions and indexing pairs 108 */ 109 desc = mcounters->counters_data; 110 for (i = 0; i < mcounters->ncounters; i++) 111 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; 112 113 err_read: 114 kfree(mread_attr.out); 115 err_bound: 116 mutex_unlock(&mcounters->mcntrs_mutex); 117 return ret; 118 } 119 120 static int mlx5_ib_destroy_counters(struct ib_counters *counters) 121 { 122 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 123 124 mlx5_ib_counters_clear_description(counters); 125 if (mcounters->hw_cntrs_hndl) 126 mlx5_fc_destroy(to_mdev(counters->device)->mdev, 127 mcounters->hw_cntrs_hndl); 128 return 0; 129 } 130 131 static int mlx5_ib_create_counters(struct ib_counters *counters, 132 struct uverbs_attr_bundle *attrs) 133 { 134 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 135 136 mutex_init(&mcounters->mcntrs_mutex); 137 return 0; 138 } 139 140 141 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, 142 u8 port_num) 143 { 144 return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : 145 &dev->port[port_num].cnts; 146 } 147 148 /** 149 * mlx5_ib_get_counters_id - Returns counters id to use for device+port 150 * @dev: Pointer to mlx5 IB device 151 * @port_num: Zero based port number 152 * 153 * mlx5_ib_get_counters_id() Returns counters set id to use for given 154 * device port combination in switchdev and non switchdev mode of the 155 * parent device. 156 */ 157 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) 158 { 159 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 160 161 return cnts->set_id; 162 } 163 164 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, 165 u8 port_num) 166 { 167 struct mlx5_ib_dev *dev = to_mdev(ibdev); 168 const struct mlx5_ib_counters *cnts; 169 bool is_switchdev = is_mdev_switchdev_mode(dev->mdev); 170 171 if ((is_switchdev && port_num) || (!is_switchdev && !port_num)) 172 return NULL; 173 174 cnts = get_counters(dev, port_num - 1); 175 176 return rdma_alloc_hw_stats_struct(cnts->names, 177 cnts->num_q_counters + 178 cnts->num_cong_counters + 179 cnts->num_ext_ppcnt_counters, 180 RDMA_HW_STATS_DEFAULT_LIFESPAN); 181 } 182 183 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, 184 const struct mlx5_ib_counters *cnts, 185 struct rdma_hw_stats *stats, 186 u16 set_id) 187 { 188 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 189 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 190 __be32 val; 191 int ret, i; 192 193 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 194 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); 195 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 196 if (ret) 197 return ret; 198 199 for (i = 0; i < cnts->num_q_counters; i++) { 200 val = *(__be32 *)((void *)out + cnts->offsets[i]); 201 stats->value[i] = (u64)be32_to_cpu(val); 202 } 203 204 return 0; 205 } 206 207 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, 208 const struct mlx5_ib_counters *cnts, 209 struct rdma_hw_stats *stats) 210 { 211 int offset = cnts->num_q_counters + cnts->num_cong_counters; 212 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; 213 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); 214 int ret, i; 215 void *out; 216 217 out = kvzalloc(sz, GFP_KERNEL); 218 if (!out) 219 return -ENOMEM; 220 221 MLX5_SET(ppcnt_reg, in, local_port, 1); 222 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); 223 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, 224 0, 0); 225 if (ret) 226 goto free; 227 228 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) 229 stats->value[i + offset] = 230 be64_to_cpup((__be64 *)(out + 231 cnts->offsets[i + offset])); 232 free: 233 kvfree(out); 234 return ret; 235 } 236 237 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, 238 struct rdma_hw_stats *stats, 239 u8 port_num, int index) 240 { 241 struct mlx5_ib_dev *dev = to_mdev(ibdev); 242 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); 243 struct mlx5_core_dev *mdev; 244 int ret, num_counters; 245 u8 mdev_port_num; 246 247 if (!stats) 248 return -EINVAL; 249 250 num_counters = cnts->num_q_counters + 251 cnts->num_cong_counters + 252 cnts->num_ext_ppcnt_counters; 253 254 /* q_counters are per IB device, query the master mdev */ 255 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id); 256 if (ret) 257 return ret; 258 259 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 260 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); 261 if (ret) 262 return ret; 263 } 264 265 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 266 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, 267 &mdev_port_num); 268 if (!mdev) { 269 /* If port is not affiliated yet, its in down state 270 * which doesn't have any counters yet, so it would be 271 * zero. So no need to read from the HCA. 272 */ 273 goto done; 274 } 275 ret = mlx5_lag_query_cong_counters(dev->mdev, 276 stats->value + 277 cnts->num_q_counters, 278 cnts->num_cong_counters, 279 cnts->offsets + 280 cnts->num_q_counters); 281 282 mlx5_ib_put_native_port_mdev(dev, port_num); 283 if (ret) 284 return ret; 285 } 286 287 done: 288 return num_counters; 289 } 290 291 static struct rdma_hw_stats * 292 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) 293 { 294 struct mlx5_ib_dev *dev = to_mdev(counter->device); 295 const struct mlx5_ib_counters *cnts = 296 get_counters(dev, counter->port - 1); 297 298 return rdma_alloc_hw_stats_struct(cnts->names, 299 cnts->num_q_counters + 300 cnts->num_cong_counters + 301 cnts->num_ext_ppcnt_counters, 302 RDMA_HW_STATS_DEFAULT_LIFESPAN); 303 } 304 305 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) 306 { 307 struct mlx5_ib_dev *dev = to_mdev(counter->device); 308 const struct mlx5_ib_counters *cnts = 309 get_counters(dev, counter->port - 1); 310 311 return mlx5_ib_query_q_counters(dev->mdev, cnts, 312 counter->stats, counter->id); 313 } 314 315 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) 316 { 317 struct mlx5_ib_dev *dev = to_mdev(counter->device); 318 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 319 320 if (!counter->id) 321 return 0; 322 323 MLX5_SET(dealloc_q_counter_in, in, opcode, 324 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 325 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); 326 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 327 } 328 329 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, 330 struct ib_qp *qp) 331 { 332 struct mlx5_ib_dev *dev = to_mdev(qp->device); 333 int err; 334 335 if (!counter->id) { 336 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 337 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 338 339 MLX5_SET(alloc_q_counter_in, in, opcode, 340 MLX5_CMD_OP_ALLOC_Q_COUNTER); 341 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); 342 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 343 if (err) 344 return err; 345 counter->id = 346 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 347 } 348 349 err = mlx5_ib_qp_set_counter(qp, counter); 350 if (err) 351 goto fail_set_counter; 352 353 return 0; 354 355 fail_set_counter: 356 mlx5_ib_counter_dealloc(counter); 357 counter->id = 0; 358 359 return err; 360 } 361 362 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) 363 { 364 return mlx5_ib_qp_set_counter(qp, NULL); 365 } 366 367 368 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, 369 const char **names, 370 size_t *offsets) 371 { 372 int i; 373 int j = 0; 374 375 for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) { 376 names[j] = basic_q_cnts[i].name; 377 offsets[j] = basic_q_cnts[i].offset; 378 } 379 380 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { 381 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) { 382 names[j] = out_of_seq_q_cnts[i].name; 383 offsets[j] = out_of_seq_q_cnts[i].offset; 384 } 385 } 386 387 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 388 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) { 389 names[j] = retrans_q_cnts[i].name; 390 offsets[j] = retrans_q_cnts[i].offset; 391 } 392 } 393 394 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { 395 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) { 396 names[j] = extended_err_cnts[i].name; 397 offsets[j] = extended_err_cnts[i].offset; 398 } 399 } 400 401 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { 402 for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { 403 names[j] = roce_accl_cnts[i].name; 404 offsets[j] = roce_accl_cnts[i].offset; 405 } 406 } 407 408 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 409 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { 410 names[j] = cong_cnts[i].name; 411 offsets[j] = cong_cnts[i].offset; 412 } 413 } 414 415 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 416 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { 417 names[j] = ext_ppcnt_cnts[i].name; 418 offsets[j] = ext_ppcnt_cnts[i].offset; 419 } 420 } 421 } 422 423 424 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, 425 struct mlx5_ib_counters *cnts) 426 { 427 u32 num_counters; 428 429 num_counters = ARRAY_SIZE(basic_q_cnts); 430 431 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) 432 num_counters += ARRAY_SIZE(out_of_seq_q_cnts); 433 434 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 435 num_counters += ARRAY_SIZE(retrans_q_cnts); 436 437 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) 438 num_counters += ARRAY_SIZE(extended_err_cnts); 439 440 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) 441 num_counters += ARRAY_SIZE(roce_accl_cnts); 442 443 cnts->num_q_counters = num_counters; 444 445 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 446 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); 447 num_counters += ARRAY_SIZE(cong_cnts); 448 } 449 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 450 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); 451 num_counters += ARRAY_SIZE(ext_ppcnt_cnts); 452 } 453 cnts->names = kcalloc(num_counters, sizeof(*cnts->names), GFP_KERNEL); 454 if (!cnts->names) 455 return -ENOMEM; 456 457 cnts->offsets = kcalloc(num_counters, 458 sizeof(*cnts->offsets), GFP_KERNEL); 459 if (!cnts->offsets) 460 goto err_names; 461 462 return 0; 463 464 err_names: 465 kfree(cnts->names); 466 cnts->names = NULL; 467 return -ENOMEM; 468 } 469 470 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) 471 { 472 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 473 int num_cnt_ports; 474 int i; 475 476 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 477 478 MLX5_SET(dealloc_q_counter_in, in, opcode, 479 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 480 481 for (i = 0; i < num_cnt_ports; i++) { 482 if (dev->port[i].cnts.set_id) { 483 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 484 dev->port[i].cnts.set_id); 485 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 486 } 487 kfree(dev->port[i].cnts.names); 488 kfree(dev->port[i].cnts.offsets); 489 } 490 } 491 492 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) 493 { 494 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 495 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 496 int num_cnt_ports; 497 int err = 0; 498 int i; 499 bool is_shared; 500 501 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 502 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; 503 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 504 505 for (i = 0; i < num_cnt_ports; i++) { 506 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts); 507 if (err) 508 goto err_alloc; 509 510 mlx5_ib_fill_counters(dev, dev->port[i].cnts.names, 511 dev->port[i].cnts.offsets); 512 513 MLX5_SET(alloc_q_counter_in, in, uid, 514 is_shared ? MLX5_SHARED_RESOURCE_UID : 0); 515 516 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 517 if (err) { 518 mlx5_ib_warn(dev, 519 "couldn't allocate queue counter for port %d, err %d\n", 520 i + 1, err); 521 goto err_alloc; 522 } 523 524 dev->port[i].cnts.set_id = 525 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 526 } 527 return 0; 528 529 err_alloc: 530 mlx5_ib_dealloc_counters(dev); 531 return err; 532 } 533 534 static int read_flow_counters(struct ib_device *ibdev, 535 struct mlx5_read_counters_attr *read_attr) 536 { 537 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; 538 struct mlx5_ib_dev *dev = to_mdev(ibdev); 539 540 return mlx5_fc_query(dev->mdev, fc, 541 &read_attr->out[IB_COUNTER_PACKETS], 542 &read_attr->out[IB_COUNTER_BYTES]); 543 } 544 545 /* flow counters currently expose two counters packets and bytes */ 546 #define FLOW_COUNTERS_NUM 2 547 static int counters_set_description( 548 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, 549 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) 550 { 551 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 552 u32 cntrs_max_index = 0; 553 int i; 554 555 if (counters_type != MLX5_IB_COUNTERS_FLOW) 556 return -EINVAL; 557 558 /* init the fields for the object */ 559 mcounters->type = counters_type; 560 mcounters->read_counters = read_flow_counters; 561 mcounters->counters_num = FLOW_COUNTERS_NUM; 562 mcounters->ncounters = ncounters; 563 /* each counter entry have both description and index pair */ 564 for (i = 0; i < ncounters; i++) { 565 if (desc_data[i].description > IB_COUNTER_BYTES) 566 return -EINVAL; 567 568 if (cntrs_max_index <= desc_data[i].index) 569 cntrs_max_index = desc_data[i].index + 1; 570 } 571 572 mutex_lock(&mcounters->mcntrs_mutex); 573 mcounters->counters_data = desc_data; 574 mcounters->cntrs_max_index = cntrs_max_index; 575 mutex_unlock(&mcounters->mcntrs_mutex); 576 577 return 0; 578 } 579 580 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) 581 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, 582 struct mlx5_ib_create_flow *ucmd) 583 { 584 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); 585 struct mlx5_ib_flow_counters_data *cntrs_data = NULL; 586 struct mlx5_ib_flow_counters_desc *desc_data = NULL; 587 bool hw_hndl = false; 588 int ret = 0; 589 590 if (ucmd && ucmd->ncounters_data != 0) { 591 cntrs_data = ucmd->data; 592 if (cntrs_data->ncounters > MAX_COUNTERS_NUM) 593 return -EINVAL; 594 595 desc_data = kcalloc(cntrs_data->ncounters, 596 sizeof(*desc_data), 597 GFP_KERNEL); 598 if (!desc_data) 599 return -ENOMEM; 600 601 if (copy_from_user(desc_data, 602 u64_to_user_ptr(cntrs_data->counters_data), 603 sizeof(*desc_data) * cntrs_data->ncounters)) { 604 ret = -EFAULT; 605 goto free; 606 } 607 } 608 609 if (!mcounters->hw_cntrs_hndl) { 610 mcounters->hw_cntrs_hndl = mlx5_fc_create( 611 to_mdev(ibcounters->device)->mdev, false); 612 if (IS_ERR(mcounters->hw_cntrs_hndl)) { 613 ret = PTR_ERR(mcounters->hw_cntrs_hndl); 614 goto free; 615 } 616 hw_hndl = true; 617 } 618 619 if (desc_data) { 620 /* counters already bound to at least one flow */ 621 if (mcounters->cntrs_max_index) { 622 ret = -EINVAL; 623 goto free_hndl; 624 } 625 626 ret = counters_set_description(ibcounters, 627 MLX5_IB_COUNTERS_FLOW, 628 desc_data, 629 cntrs_data->ncounters); 630 if (ret) 631 goto free_hndl; 632 633 } else if (!mcounters->cntrs_max_index) { 634 /* counters not bound yet, must have udata passed */ 635 ret = -EINVAL; 636 goto free_hndl; 637 } 638 639 return 0; 640 641 free_hndl: 642 if (hw_hndl) { 643 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, 644 mcounters->hw_cntrs_hndl); 645 mcounters->hw_cntrs_hndl = NULL; 646 } 647 free: 648 kfree(desc_data); 649 return ret; 650 } 651 652 void mlx5_ib_counters_clear_description(struct ib_counters *counters) 653 { 654 struct mlx5_ib_mcounters *mcounters; 655 656 if (!counters || atomic_read(&counters->usecnt) != 1) 657 return; 658 659 mcounters = to_mcounters(counters); 660 661 mutex_lock(&mcounters->mcntrs_mutex); 662 kfree(mcounters->counters_data); 663 mcounters->counters_data = NULL; 664 mcounters->cntrs_max_index = 0; 665 mutex_unlock(&mcounters->mcntrs_mutex); 666 } 667 668 static const struct ib_device_ops hw_stats_ops = { 669 .alloc_hw_stats = mlx5_ib_alloc_hw_stats, 670 .get_hw_stats = mlx5_ib_get_hw_stats, 671 .counter_bind_qp = mlx5_ib_counter_bind_qp, 672 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 673 .counter_dealloc = mlx5_ib_counter_dealloc, 674 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 675 .counter_update_stats = mlx5_ib_counter_update_stats, 676 }; 677 678 static const struct ib_device_ops counters_ops = { 679 .create_counters = mlx5_ib_create_counters, 680 .destroy_counters = mlx5_ib_destroy_counters, 681 .read_counters = mlx5_ib_read_counters, 682 683 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), 684 }; 685 686 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) 687 { 688 ib_set_device_ops(&dev->ib_dev, &counters_ops); 689 690 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 691 return 0; 692 693 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); 694 return mlx5_ib_alloc_counters(dev); 695 } 696 697 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) 698 { 699 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 700 return; 701 702 mlx5_ib_dealloc_counters(dev); 703 } 704