1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include "mlx5_ib.h" 7 #include <linux/mlx5/eswitch.h> 8 #include "counters.h" 9 #include "ib_rep.h" 10 #include "qp.h" 11 12 struct mlx5_ib_counter { 13 const char *name; 14 size_t offset; 15 }; 16 17 #define INIT_Q_COUNTER(_name) \ 18 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} 19 20 static const struct mlx5_ib_counter basic_q_cnts[] = { 21 INIT_Q_COUNTER(rx_write_requests), 22 INIT_Q_COUNTER(rx_read_requests), 23 INIT_Q_COUNTER(rx_atomic_requests), 24 INIT_Q_COUNTER(out_of_buffer), 25 }; 26 27 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { 28 INIT_Q_COUNTER(out_of_sequence), 29 }; 30 31 static const struct mlx5_ib_counter retrans_q_cnts[] = { 32 INIT_Q_COUNTER(duplicate_request), 33 INIT_Q_COUNTER(rnr_nak_retry_err), 34 INIT_Q_COUNTER(packet_seq_err), 35 INIT_Q_COUNTER(implied_nak_seq_err), 36 INIT_Q_COUNTER(local_ack_timeout_err), 37 }; 38 39 #define INIT_CONG_COUNTER(_name) \ 40 { .name = #_name, .offset = \ 41 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} 42 43 static const struct mlx5_ib_counter cong_cnts[] = { 44 INIT_CONG_COUNTER(rp_cnp_ignored), 45 INIT_CONG_COUNTER(rp_cnp_handled), 46 INIT_CONG_COUNTER(np_ecn_marked_roce_packets), 47 INIT_CONG_COUNTER(np_cnp_sent), 48 }; 49 50 static const struct mlx5_ib_counter extended_err_cnts[] = { 51 INIT_Q_COUNTER(resp_local_length_error), 52 INIT_Q_COUNTER(resp_cqe_error), 53 INIT_Q_COUNTER(req_cqe_error), 54 INIT_Q_COUNTER(req_remote_invalid_request), 55 INIT_Q_COUNTER(req_remote_access_errors), 56 INIT_Q_COUNTER(resp_remote_access_errors), 57 INIT_Q_COUNTER(resp_cqe_flush_error), 58 INIT_Q_COUNTER(req_cqe_flush_error), 59 }; 60 61 static const struct mlx5_ib_counter roce_accl_cnts[] = { 62 INIT_Q_COUNTER(roce_adp_retrans), 63 INIT_Q_COUNTER(roce_adp_retrans_to), 64 INIT_Q_COUNTER(roce_slow_restart), 65 INIT_Q_COUNTER(roce_slow_restart_cnps), 66 INIT_Q_COUNTER(roce_slow_restart_trans), 67 }; 68 69 #define INIT_EXT_PPCNT_COUNTER(_name) \ 70 { .name = #_name, .offset = \ 71 MLX5_BYTE_OFF(ppcnt_reg, \ 72 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} 73 74 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { 75 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), 76 }; 77 78 static int mlx5_ib_read_counters(struct ib_counters *counters, 79 struct ib_counters_read_attr *read_attr, 80 struct uverbs_attr_bundle *attrs) 81 { 82 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 83 struct mlx5_read_counters_attr mread_attr = {}; 84 struct mlx5_ib_flow_counters_desc *desc; 85 int ret, i; 86 87 mutex_lock(&mcounters->mcntrs_mutex); 88 if (mcounters->cntrs_max_index > read_attr->ncounters) { 89 ret = -EINVAL; 90 goto err_bound; 91 } 92 93 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), 94 GFP_KERNEL); 95 if (!mread_attr.out) { 96 ret = -ENOMEM; 97 goto err_bound; 98 } 99 100 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; 101 mread_attr.flags = read_attr->flags; 102 ret = mcounters->read_counters(counters->device, &mread_attr); 103 if (ret) 104 goto err_read; 105 106 /* do the pass over the counters data array to assign according to the 107 * descriptions and indexing pairs 108 */ 109 desc = mcounters->counters_data; 110 for (i = 0; i < mcounters->ncounters; i++) 111 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; 112 113 err_read: 114 kfree(mread_attr.out); 115 err_bound: 116 mutex_unlock(&mcounters->mcntrs_mutex); 117 return ret; 118 } 119 120 static void mlx5_ib_destroy_counters(struct ib_counters *counters) 121 { 122 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 123 124 mlx5_ib_counters_clear_description(counters); 125 if (mcounters->hw_cntrs_hndl) 126 mlx5_fc_destroy(to_mdev(counters->device)->mdev, 127 mcounters->hw_cntrs_hndl); 128 } 129 130 static int mlx5_ib_create_counters(struct ib_counters *counters, 131 struct uverbs_attr_bundle *attrs) 132 { 133 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 134 135 mutex_init(&mcounters->mcntrs_mutex); 136 return 0; 137 } 138 139 140 static bool is_mdev_switchdev_mode(const struct mlx5_core_dev *mdev) 141 { 142 return MLX5_ESWITCH_MANAGER(mdev) && 143 mlx5_ib_eswitch_mode(mdev->priv.eswitch) == 144 MLX5_ESWITCH_OFFLOADS; 145 } 146 147 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, 148 u8 port_num) 149 { 150 return is_mdev_switchdev_mode(dev->mdev) ? &dev->port[0].cnts : 151 &dev->port[port_num].cnts; 152 } 153 154 /** 155 * mlx5_ib_get_counters_id - Returns counters id to use for device+port 156 * @dev: Pointer to mlx5 IB device 157 * @port_num: Zero based port number 158 * 159 * mlx5_ib_get_counters_id() Returns counters set id to use for given 160 * device port combination in switchdev and non switchdev mode of the 161 * parent device. 162 */ 163 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u8 port_num) 164 { 165 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 166 167 return cnts->set_id; 168 } 169 170 static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, 171 u8 port_num) 172 { 173 struct mlx5_ib_dev *dev = to_mdev(ibdev); 174 const struct mlx5_ib_counters *cnts; 175 bool is_switchdev = is_mdev_switchdev_mode(dev->mdev); 176 177 if ((is_switchdev && port_num) || (!is_switchdev && !port_num)) 178 return NULL; 179 180 cnts = get_counters(dev, port_num - 1); 181 182 return rdma_alloc_hw_stats_struct(cnts->names, 183 cnts->num_q_counters + 184 cnts->num_cong_counters + 185 cnts->num_ext_ppcnt_counters, 186 RDMA_HW_STATS_DEFAULT_LIFESPAN); 187 } 188 189 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, 190 const struct mlx5_ib_counters *cnts, 191 struct rdma_hw_stats *stats, 192 u16 set_id) 193 { 194 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 195 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 196 __be32 val; 197 int ret, i; 198 199 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 200 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); 201 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 202 if (ret) 203 return ret; 204 205 for (i = 0; i < cnts->num_q_counters; i++) { 206 val = *(__be32 *)((void *)out + cnts->offsets[i]); 207 stats->value[i] = (u64)be32_to_cpu(val); 208 } 209 210 return 0; 211 } 212 213 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, 214 const struct mlx5_ib_counters *cnts, 215 struct rdma_hw_stats *stats) 216 { 217 int offset = cnts->num_q_counters + cnts->num_cong_counters; 218 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; 219 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); 220 int ret, i; 221 void *out; 222 223 out = kvzalloc(sz, GFP_KERNEL); 224 if (!out) 225 return -ENOMEM; 226 227 MLX5_SET(ppcnt_reg, in, local_port, 1); 228 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); 229 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, 230 0, 0); 231 if (ret) 232 goto free; 233 234 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) 235 stats->value[i + offset] = 236 be64_to_cpup((__be64 *)(out + 237 cnts->offsets[i + offset])); 238 free: 239 kvfree(out); 240 return ret; 241 } 242 243 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, 244 struct rdma_hw_stats *stats, 245 u8 port_num, int index) 246 { 247 struct mlx5_ib_dev *dev = to_mdev(ibdev); 248 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num - 1); 249 struct mlx5_core_dev *mdev; 250 int ret, num_counters; 251 u8 mdev_port_num; 252 253 if (!stats) 254 return -EINVAL; 255 256 num_counters = cnts->num_q_counters + 257 cnts->num_cong_counters + 258 cnts->num_ext_ppcnt_counters; 259 260 /* q_counters are per IB device, query the master mdev */ 261 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, cnts->set_id); 262 if (ret) 263 return ret; 264 265 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 266 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); 267 if (ret) 268 return ret; 269 } 270 271 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 272 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, 273 &mdev_port_num); 274 if (!mdev) { 275 /* If port is not affiliated yet, its in down state 276 * which doesn't have any counters yet, so it would be 277 * zero. So no need to read from the HCA. 278 */ 279 goto done; 280 } 281 ret = mlx5_lag_query_cong_counters(dev->mdev, 282 stats->value + 283 cnts->num_q_counters, 284 cnts->num_cong_counters, 285 cnts->offsets + 286 cnts->num_q_counters); 287 288 mlx5_ib_put_native_port_mdev(dev, port_num); 289 if (ret) 290 return ret; 291 } 292 293 done: 294 return num_counters; 295 } 296 297 static struct rdma_hw_stats * 298 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) 299 { 300 struct mlx5_ib_dev *dev = to_mdev(counter->device); 301 const struct mlx5_ib_counters *cnts = 302 get_counters(dev, counter->port - 1); 303 304 return rdma_alloc_hw_stats_struct(cnts->names, 305 cnts->num_q_counters + 306 cnts->num_cong_counters + 307 cnts->num_ext_ppcnt_counters, 308 RDMA_HW_STATS_DEFAULT_LIFESPAN); 309 } 310 311 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) 312 { 313 struct mlx5_ib_dev *dev = to_mdev(counter->device); 314 const struct mlx5_ib_counters *cnts = 315 get_counters(dev, counter->port - 1); 316 317 return mlx5_ib_query_q_counters(dev->mdev, cnts, 318 counter->stats, counter->id); 319 } 320 321 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) 322 { 323 struct mlx5_ib_dev *dev = to_mdev(counter->device); 324 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 325 326 if (!counter->id) 327 return 0; 328 329 MLX5_SET(dealloc_q_counter_in, in, opcode, 330 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 331 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); 332 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 333 } 334 335 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, 336 struct ib_qp *qp) 337 { 338 struct mlx5_ib_dev *dev = to_mdev(qp->device); 339 int err; 340 341 if (!counter->id) { 342 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 343 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 344 345 MLX5_SET(alloc_q_counter_in, in, opcode, 346 MLX5_CMD_OP_ALLOC_Q_COUNTER); 347 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); 348 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 349 if (err) 350 return err; 351 counter->id = 352 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 353 } 354 355 err = mlx5_ib_qp_set_counter(qp, counter); 356 if (err) 357 goto fail_set_counter; 358 359 return 0; 360 361 fail_set_counter: 362 mlx5_ib_counter_dealloc(counter); 363 counter->id = 0; 364 365 return err; 366 } 367 368 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) 369 { 370 return mlx5_ib_qp_set_counter(qp, NULL); 371 } 372 373 374 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, 375 const char **names, 376 size_t *offsets) 377 { 378 int i; 379 int j = 0; 380 381 for (i = 0; i < ARRAY_SIZE(basic_q_cnts); i++, j++) { 382 names[j] = basic_q_cnts[i].name; 383 offsets[j] = basic_q_cnts[i].offset; 384 } 385 386 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { 387 for (i = 0; i < ARRAY_SIZE(out_of_seq_q_cnts); i++, j++) { 388 names[j] = out_of_seq_q_cnts[i].name; 389 offsets[j] = out_of_seq_q_cnts[i].offset; 390 } 391 } 392 393 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 394 for (i = 0; i < ARRAY_SIZE(retrans_q_cnts); i++, j++) { 395 names[j] = retrans_q_cnts[i].name; 396 offsets[j] = retrans_q_cnts[i].offset; 397 } 398 } 399 400 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { 401 for (i = 0; i < ARRAY_SIZE(extended_err_cnts); i++, j++) { 402 names[j] = extended_err_cnts[i].name; 403 offsets[j] = extended_err_cnts[i].offset; 404 } 405 } 406 407 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { 408 for (i = 0; i < ARRAY_SIZE(roce_accl_cnts); i++, j++) { 409 names[j] = roce_accl_cnts[i].name; 410 offsets[j] = roce_accl_cnts[i].offset; 411 } 412 } 413 414 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 415 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { 416 names[j] = cong_cnts[i].name; 417 offsets[j] = cong_cnts[i].offset; 418 } 419 } 420 421 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 422 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { 423 names[j] = ext_ppcnt_cnts[i].name; 424 offsets[j] = ext_ppcnt_cnts[i].offset; 425 } 426 } 427 } 428 429 430 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, 431 struct mlx5_ib_counters *cnts) 432 { 433 u32 num_counters; 434 435 num_counters = ARRAY_SIZE(basic_q_cnts); 436 437 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) 438 num_counters += ARRAY_SIZE(out_of_seq_q_cnts); 439 440 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 441 num_counters += ARRAY_SIZE(retrans_q_cnts); 442 443 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) 444 num_counters += ARRAY_SIZE(extended_err_cnts); 445 446 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) 447 num_counters += ARRAY_SIZE(roce_accl_cnts); 448 449 cnts->num_q_counters = num_counters; 450 451 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 452 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); 453 num_counters += ARRAY_SIZE(cong_cnts); 454 } 455 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 456 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); 457 num_counters += ARRAY_SIZE(ext_ppcnt_cnts); 458 } 459 cnts->names = kcalloc(num_counters, sizeof(cnts->names), GFP_KERNEL); 460 if (!cnts->names) 461 return -ENOMEM; 462 463 cnts->offsets = kcalloc(num_counters, 464 sizeof(cnts->offsets), GFP_KERNEL); 465 if (!cnts->offsets) 466 goto err_names; 467 468 return 0; 469 470 err_names: 471 kfree(cnts->names); 472 cnts->names = NULL; 473 return -ENOMEM; 474 } 475 476 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) 477 { 478 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 479 int num_cnt_ports; 480 int i; 481 482 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 483 484 MLX5_SET(dealloc_q_counter_in, in, opcode, 485 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 486 487 for (i = 0; i < num_cnt_ports; i++) { 488 if (dev->port[i].cnts.set_id) { 489 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 490 dev->port[i].cnts.set_id); 491 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 492 } 493 kfree(dev->port[i].cnts.names); 494 kfree(dev->port[i].cnts.offsets); 495 } 496 } 497 498 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) 499 { 500 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 501 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 502 int num_cnt_ports; 503 int err = 0; 504 int i; 505 bool is_shared; 506 507 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 508 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; 509 num_cnt_ports = is_mdev_switchdev_mode(dev->mdev) ? 1 : dev->num_ports; 510 511 for (i = 0; i < num_cnt_ports; i++) { 512 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts); 513 if (err) 514 goto err_alloc; 515 516 mlx5_ib_fill_counters(dev, dev->port[i].cnts.names, 517 dev->port[i].cnts.offsets); 518 519 MLX5_SET(alloc_q_counter_in, in, uid, 520 is_shared ? MLX5_SHARED_RESOURCE_UID : 0); 521 522 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 523 if (err) { 524 mlx5_ib_warn(dev, 525 "couldn't allocate queue counter for port %d, err %d\n", 526 i + 1, err); 527 goto err_alloc; 528 } 529 530 dev->port[i].cnts.set_id = 531 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 532 } 533 return 0; 534 535 err_alloc: 536 mlx5_ib_dealloc_counters(dev); 537 return err; 538 } 539 540 static int read_flow_counters(struct ib_device *ibdev, 541 struct mlx5_read_counters_attr *read_attr) 542 { 543 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; 544 struct mlx5_ib_dev *dev = to_mdev(ibdev); 545 546 return mlx5_fc_query(dev->mdev, fc, 547 &read_attr->out[IB_COUNTER_PACKETS], 548 &read_attr->out[IB_COUNTER_BYTES]); 549 } 550 551 /* flow counters currently expose two counters packets and bytes */ 552 #define FLOW_COUNTERS_NUM 2 553 static int counters_set_description( 554 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, 555 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) 556 { 557 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 558 u32 cntrs_max_index = 0; 559 int i; 560 561 if (counters_type != MLX5_IB_COUNTERS_FLOW) 562 return -EINVAL; 563 564 /* init the fields for the object */ 565 mcounters->type = counters_type; 566 mcounters->read_counters = read_flow_counters; 567 mcounters->counters_num = FLOW_COUNTERS_NUM; 568 mcounters->ncounters = ncounters; 569 /* each counter entry have both description and index pair */ 570 for (i = 0; i < ncounters; i++) { 571 if (desc_data[i].description > IB_COUNTER_BYTES) 572 return -EINVAL; 573 574 if (cntrs_max_index <= desc_data[i].index) 575 cntrs_max_index = desc_data[i].index + 1; 576 } 577 578 mutex_lock(&mcounters->mcntrs_mutex); 579 mcounters->counters_data = desc_data; 580 mcounters->cntrs_max_index = cntrs_max_index; 581 mutex_unlock(&mcounters->mcntrs_mutex); 582 583 return 0; 584 } 585 586 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) 587 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, 588 struct mlx5_ib_create_flow *ucmd) 589 { 590 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); 591 struct mlx5_ib_flow_counters_data *cntrs_data = NULL; 592 struct mlx5_ib_flow_counters_desc *desc_data = NULL; 593 bool hw_hndl = false; 594 int ret = 0; 595 596 if (ucmd && ucmd->ncounters_data != 0) { 597 cntrs_data = ucmd->data; 598 if (cntrs_data->ncounters > MAX_COUNTERS_NUM) 599 return -EINVAL; 600 601 desc_data = kcalloc(cntrs_data->ncounters, 602 sizeof(*desc_data), 603 GFP_KERNEL); 604 if (!desc_data) 605 return -ENOMEM; 606 607 if (copy_from_user(desc_data, 608 u64_to_user_ptr(cntrs_data->counters_data), 609 sizeof(*desc_data) * cntrs_data->ncounters)) { 610 ret = -EFAULT; 611 goto free; 612 } 613 } 614 615 if (!mcounters->hw_cntrs_hndl) { 616 mcounters->hw_cntrs_hndl = mlx5_fc_create( 617 to_mdev(ibcounters->device)->mdev, false); 618 if (IS_ERR(mcounters->hw_cntrs_hndl)) { 619 ret = PTR_ERR(mcounters->hw_cntrs_hndl); 620 goto free; 621 } 622 hw_hndl = true; 623 } 624 625 if (desc_data) { 626 /* counters already bound to at least one flow */ 627 if (mcounters->cntrs_max_index) { 628 ret = -EINVAL; 629 goto free_hndl; 630 } 631 632 ret = counters_set_description(ibcounters, 633 MLX5_IB_COUNTERS_FLOW, 634 desc_data, 635 cntrs_data->ncounters); 636 if (ret) 637 goto free_hndl; 638 639 } else if (!mcounters->cntrs_max_index) { 640 /* counters not bound yet, must have udata passed */ 641 ret = -EINVAL; 642 goto free_hndl; 643 } 644 645 return 0; 646 647 free_hndl: 648 if (hw_hndl) { 649 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, 650 mcounters->hw_cntrs_hndl); 651 mcounters->hw_cntrs_hndl = NULL; 652 } 653 free: 654 kfree(desc_data); 655 return ret; 656 } 657 658 void mlx5_ib_counters_clear_description(struct ib_counters *counters) 659 { 660 struct mlx5_ib_mcounters *mcounters; 661 662 if (!counters || atomic_read(&counters->usecnt) != 1) 663 return; 664 665 mcounters = to_mcounters(counters); 666 667 mutex_lock(&mcounters->mcntrs_mutex); 668 kfree(mcounters->counters_data); 669 mcounters->counters_data = NULL; 670 mcounters->cntrs_max_index = 0; 671 mutex_unlock(&mcounters->mcntrs_mutex); 672 } 673 674 static const struct ib_device_ops hw_stats_ops = { 675 .alloc_hw_stats = mlx5_ib_alloc_hw_stats, 676 .get_hw_stats = mlx5_ib_get_hw_stats, 677 .counter_bind_qp = mlx5_ib_counter_bind_qp, 678 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 679 .counter_dealloc = mlx5_ib_counter_dealloc, 680 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 681 .counter_update_stats = mlx5_ib_counter_update_stats, 682 }; 683 684 static const struct ib_device_ops counters_ops = { 685 .create_counters = mlx5_ib_create_counters, 686 .destroy_counters = mlx5_ib_destroy_counters, 687 .read_counters = mlx5_ib_read_counters, 688 689 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), 690 }; 691 692 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) 693 { 694 ib_set_device_ops(&dev->ib_dev, &counters_ops); 695 696 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 697 return 0; 698 699 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); 700 return mlx5_ib_alloc_counters(dev); 701 } 702 703 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) 704 { 705 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 706 return; 707 708 mlx5_ib_dealloc_counters(dev); 709 } 710