1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "rx_res.h" 5 #include "channels.h" 6 #include "params.h" 7 8 #define MLX5E_MAX_NUM_RSS 16 9 10 struct mlx5e_rx_res { 11 struct mlx5_core_dev *mdev; 12 enum mlx5e_rx_res_features features; 13 unsigned int max_nch; 14 u32 drop_rqn; 15 16 struct mlx5e_packet_merge_param pkt_merge_param; 17 struct rw_semaphore pkt_merge_param_sem; 18 19 struct mlx5e_rss *rss[MLX5E_MAX_NUM_RSS]; 20 bool rss_active; 21 u32 rss_rqns[MLX5E_INDIR_RQT_SIZE]; 22 unsigned int rss_nch; 23 24 struct { 25 struct mlx5e_rqt direct_rqt; 26 struct mlx5e_tir direct_tir; 27 } *channels; 28 29 struct { 30 struct mlx5e_rqt rqt; 31 struct mlx5e_tir tir; 32 } ptp; 33 }; 34 35 /* API for rx_res_rss_* */ 36 37 static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, 38 unsigned int init_nch) 39 { 40 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 41 struct mlx5e_rss *rss; 42 int err; 43 44 if (WARN_ON(res->rss[0])) 45 return -EINVAL; 46 47 rss = mlx5e_rss_alloc(); 48 if (!rss) 49 return -ENOMEM; 50 51 err = mlx5e_rss_init(rss, res->mdev, inner_ft_support, res->drop_rqn, 52 &res->pkt_merge_param); 53 if (err) 54 goto err_rss_free; 55 56 mlx5e_rss_set_indir_uniform(rss, init_nch); 57 58 res->rss[0] = rss; 59 60 return 0; 61 62 err_rss_free: 63 mlx5e_rss_free(rss); 64 return err; 65 } 66 67 int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 *rss_idx, unsigned int init_nch) 68 { 69 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 70 struct mlx5e_rss *rss; 71 int err, i; 72 73 for (i = 1; i < MLX5E_MAX_NUM_RSS; i++) 74 if (!res->rss[i]) 75 break; 76 77 if (i == MLX5E_MAX_NUM_RSS) 78 return -ENOSPC; 79 80 rss = mlx5e_rss_alloc(); 81 if (!rss) 82 return -ENOMEM; 83 84 err = mlx5e_rss_init_no_tirs(rss, res->mdev, inner_ft_support, res->drop_rqn); 85 if (err) 86 goto err_rss_free; 87 88 mlx5e_rss_set_indir_uniform(rss, init_nch); 89 if (res->rss_active) 90 mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); 91 92 res->rss[i] = rss; 93 *rss_idx = i; 94 95 return 0; 96 97 err_rss_free: 98 mlx5e_rss_free(rss); 99 return err; 100 } 101 102 static int __mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx) 103 { 104 struct mlx5e_rss *rss = res->rss[rss_idx]; 105 int err; 106 107 err = mlx5e_rss_cleanup(rss); 108 if (err) 109 return err; 110 111 mlx5e_rss_free(rss); 112 res->rss[rss_idx] = NULL; 113 114 return 0; 115 } 116 117 int mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx) 118 { 119 struct mlx5e_rss *rss; 120 121 if (rss_idx >= MLX5E_MAX_NUM_RSS) 122 return -EINVAL; 123 124 rss = res->rss[rss_idx]; 125 if (!rss) 126 return -EINVAL; 127 128 return __mlx5e_rx_res_rss_destroy(res, rss_idx); 129 } 130 131 static void mlx5e_rx_res_rss_destroy_all(struct mlx5e_rx_res *res) 132 { 133 int i; 134 135 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 136 struct mlx5e_rss *rss = res->rss[i]; 137 int err; 138 139 if (!rss) 140 continue; 141 142 err = __mlx5e_rx_res_rss_destroy(res, i); 143 if (err) { 144 unsigned int refcount; 145 146 refcount = mlx5e_rss_refcnt_read(rss); 147 mlx5_core_warn(res->mdev, 148 "Failed to destroy RSS context %d, refcount = %u, err = %d\n", 149 i, refcount, err); 150 } 151 } 152 } 153 154 static void mlx5e_rx_res_rss_enable(struct mlx5e_rx_res *res) 155 { 156 int i; 157 158 res->rss_active = true; 159 160 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 161 struct mlx5e_rss *rss = res->rss[i]; 162 163 if (!rss) 164 continue; 165 mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); 166 } 167 } 168 169 static void mlx5e_rx_res_rss_disable(struct mlx5e_rx_res *res) 170 { 171 int i; 172 173 res->rss_active = false; 174 175 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 176 struct mlx5e_rss *rss = res->rss[i]; 177 178 if (!rss) 179 continue; 180 mlx5e_rss_disable(rss); 181 } 182 } 183 184 /* Updates the indirection table SW shadow, does not update the HW resources yet */ 185 void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch) 186 { 187 WARN_ON_ONCE(res->rss_active); 188 mlx5e_rss_set_indir_uniform(res->rss[0], nch); 189 } 190 191 int mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, 192 u32 *indir, u8 *key, u8 *hfunc) 193 { 194 struct mlx5e_rss *rss; 195 196 if (rss_idx >= MLX5E_MAX_NUM_RSS) 197 return -EINVAL; 198 199 rss = res->rss[rss_idx]; 200 if (!rss) 201 return -ENOENT; 202 203 return mlx5e_rss_get_rxfh(rss, indir, key, hfunc); 204 } 205 206 int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, 207 const u32 *indir, const u8 *key, const u8 *hfunc) 208 { 209 struct mlx5e_rss *rss; 210 211 if (rss_idx >= MLX5E_MAX_NUM_RSS) 212 return -EINVAL; 213 214 rss = res->rss[rss_idx]; 215 if (!rss) 216 return -ENOENT; 217 218 return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, res->rss_nch); 219 } 220 221 u8 mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) 222 { 223 struct mlx5e_rss *rss = res->rss[0]; 224 225 return mlx5e_rss_get_hash_fields(rss, tt); 226 } 227 228 int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt, 229 u8 rx_hash_fields) 230 { 231 struct mlx5e_rss *rss = res->rss[0]; 232 233 return mlx5e_rss_set_hash_fields(rss, tt, rx_hash_fields); 234 } 235 236 int mlx5e_rx_res_rss_cnt(struct mlx5e_rx_res *res) 237 { 238 int i, cnt; 239 240 cnt = 0; 241 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) 242 if (res->rss[i]) 243 cnt++; 244 245 return cnt; 246 } 247 248 int mlx5e_rx_res_rss_index(struct mlx5e_rx_res *res, struct mlx5e_rss *rss) 249 { 250 int i; 251 252 if (!rss) 253 return -EINVAL; 254 255 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) 256 if (rss == res->rss[i]) 257 return i; 258 259 return -ENOENT; 260 } 261 262 struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx) 263 { 264 if (rss_idx >= MLX5E_MAX_NUM_RSS) 265 return NULL; 266 267 return res->rss[rss_idx]; 268 } 269 270 /* End of API rx_res_rss_* */ 271 272 struct mlx5e_rx_res *mlx5e_rx_res_alloc(void) 273 { 274 return kvzalloc(sizeof(struct mlx5e_rx_res), GFP_KERNEL); 275 } 276 277 static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res) 278 { 279 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 280 struct mlx5e_tir_builder *builder; 281 int err = 0; 282 int ix; 283 284 builder = mlx5e_tir_builder_alloc(false); 285 if (!builder) 286 return -ENOMEM; 287 288 res->channels = kvcalloc(res->max_nch, sizeof(*res->channels), GFP_KERNEL); 289 if (!res->channels) { 290 err = -ENOMEM; 291 goto out; 292 } 293 294 for (ix = 0; ix < res->max_nch; ix++) { 295 err = mlx5e_rqt_init_direct(&res->channels[ix].direct_rqt, 296 res->mdev, false, res->drop_rqn); 297 if (err) { 298 mlx5_core_warn(res->mdev, "Failed to create a direct RQT: err = %d, ix = %u\n", 299 err, ix); 300 goto err_destroy_direct_rqts; 301 } 302 } 303 304 for (ix = 0; ix < res->max_nch; ix++) { 305 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, 306 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 307 inner_ft_support); 308 mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param); 309 mlx5e_tir_builder_build_direct(builder); 310 311 err = mlx5e_tir_init(&res->channels[ix].direct_tir, builder, res->mdev, true); 312 if (err) { 313 mlx5_core_warn(res->mdev, "Failed to create a direct TIR: err = %d, ix = %u\n", 314 err, ix); 315 goto err_destroy_direct_tirs; 316 } 317 318 mlx5e_tir_builder_clear(builder); 319 } 320 321 goto out; 322 323 err_destroy_direct_tirs: 324 while (--ix >= 0) 325 mlx5e_tir_destroy(&res->channels[ix].direct_tir); 326 327 ix = res->max_nch; 328 err_destroy_direct_rqts: 329 while (--ix >= 0) 330 mlx5e_rqt_destroy(&res->channels[ix].direct_rqt); 331 332 kvfree(res->channels); 333 334 out: 335 mlx5e_tir_builder_free(builder); 336 337 return err; 338 } 339 340 static int mlx5e_rx_res_ptp_init(struct mlx5e_rx_res *res) 341 { 342 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 343 struct mlx5e_tir_builder *builder; 344 int err; 345 346 builder = mlx5e_tir_builder_alloc(false); 347 if (!builder) 348 return -ENOMEM; 349 350 err = mlx5e_rqt_init_direct(&res->ptp.rqt, res->mdev, false, res->drop_rqn); 351 if (err) 352 goto out; 353 354 /* Separated from the channels RQs, does not share pkt_merge state with them */ 355 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, 356 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 357 inner_ft_support); 358 mlx5e_tir_builder_build_direct(builder); 359 360 err = mlx5e_tir_init(&res->ptp.tir, builder, res->mdev, true); 361 if (err) 362 goto err_destroy_ptp_rqt; 363 364 goto out; 365 366 err_destroy_ptp_rqt: 367 mlx5e_rqt_destroy(&res->ptp.rqt); 368 369 out: 370 mlx5e_tir_builder_free(builder); 371 return err; 372 } 373 374 static void mlx5e_rx_res_channels_destroy(struct mlx5e_rx_res *res) 375 { 376 unsigned int ix; 377 378 for (ix = 0; ix < res->max_nch; ix++) { 379 mlx5e_tir_destroy(&res->channels[ix].direct_tir); 380 mlx5e_rqt_destroy(&res->channels[ix].direct_rqt); 381 } 382 383 kvfree(res->channels); 384 } 385 386 static void mlx5e_rx_res_ptp_destroy(struct mlx5e_rx_res *res) 387 { 388 mlx5e_tir_destroy(&res->ptp.tir); 389 mlx5e_rqt_destroy(&res->ptp.rqt); 390 } 391 392 int mlx5e_rx_res_init(struct mlx5e_rx_res *res, struct mlx5_core_dev *mdev, 393 enum mlx5e_rx_res_features features, unsigned int max_nch, 394 u32 drop_rqn, const struct mlx5e_packet_merge_param *init_pkt_merge_param, 395 unsigned int init_nch) 396 { 397 int err; 398 399 res->mdev = mdev; 400 res->features = features; 401 res->max_nch = max_nch; 402 res->drop_rqn = drop_rqn; 403 404 res->pkt_merge_param = *init_pkt_merge_param; 405 init_rwsem(&res->pkt_merge_param_sem); 406 407 err = mlx5e_rx_res_rss_init_def(res, init_nch); 408 if (err) 409 goto err_out; 410 411 err = mlx5e_rx_res_channels_init(res); 412 if (err) 413 goto err_rss_destroy; 414 415 err = mlx5e_rx_res_ptp_init(res); 416 if (err) 417 goto err_channels_destroy; 418 419 return 0; 420 421 err_channels_destroy: 422 mlx5e_rx_res_channels_destroy(res); 423 err_rss_destroy: 424 __mlx5e_rx_res_rss_destroy(res, 0); 425 err_out: 426 return err; 427 } 428 429 void mlx5e_rx_res_destroy(struct mlx5e_rx_res *res) 430 { 431 mlx5e_rx_res_ptp_destroy(res); 432 mlx5e_rx_res_channels_destroy(res); 433 mlx5e_rx_res_rss_destroy_all(res); 434 } 435 436 void mlx5e_rx_res_free(struct mlx5e_rx_res *res) 437 { 438 kvfree(res); 439 } 440 441 u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix) 442 { 443 return mlx5e_tir_get_tirn(&res->channels[ix].direct_tir); 444 } 445 446 u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) 447 { 448 struct mlx5e_rss *rss = res->rss[0]; 449 450 return mlx5e_rss_get_tirn(rss, tt, false); 451 } 452 453 u32 mlx5e_rx_res_get_tirn_rss_inner(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) 454 { 455 struct mlx5e_rss *rss = res->rss[0]; 456 457 return mlx5e_rss_get_tirn(rss, tt, true); 458 } 459 460 u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res) 461 { 462 WARN_ON(!(res->features & MLX5E_RX_RES_FEATURE_PTP)); 463 return mlx5e_tir_get_tirn(&res->ptp.tir); 464 } 465 466 static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int ix) 467 { 468 return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt); 469 } 470 471 static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, 472 struct mlx5e_channels *chs, 473 unsigned int ix) 474 { 475 u32 rqn = res->rss_rqns[ix]; 476 int err; 477 478 err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); 479 if (err) 480 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", 481 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 482 rqn, ix, err); 483 } 484 485 static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, 486 unsigned int ix) 487 { 488 int err; 489 490 err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); 491 if (err) 492 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", 493 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 494 res->drop_rqn, ix, err); 495 } 496 497 void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs) 498 { 499 unsigned int nch, ix; 500 int err; 501 502 nch = mlx5e_channels_get_num(chs); 503 504 for (ix = 0; ix < chs->num; ix++) { 505 if (mlx5e_channels_is_xsk(chs, ix)) 506 mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); 507 else 508 mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); 509 } 510 res->rss_nch = chs->num; 511 512 mlx5e_rx_res_rss_enable(res); 513 514 for (ix = 0; ix < nch; ix++) 515 mlx5e_rx_res_channel_activate_direct(res, chs, ix); 516 for (ix = nch; ix < res->max_nch; ix++) 517 mlx5e_rx_res_channel_deactivate_direct(res, ix); 518 519 if (res->features & MLX5E_RX_RES_FEATURE_PTP) { 520 u32 rqn; 521 522 if (!mlx5e_channels_get_ptp_rqn(chs, &rqn)) 523 rqn = res->drop_rqn; 524 525 err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn); 526 if (err) 527 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (PTP): err = %d\n", 528 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 529 rqn, err); 530 } 531 } 532 533 void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) 534 { 535 unsigned int ix; 536 int err; 537 538 mlx5e_rx_res_rss_disable(res); 539 540 for (ix = 0; ix < res->max_nch; ix++) 541 mlx5e_rx_res_channel_deactivate_direct(res, ix); 542 543 if (res->features & MLX5E_RX_RES_FEATURE_PTP) { 544 err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); 545 if (err) 546 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (PTP): err = %d\n", 547 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 548 res->drop_rqn, err); 549 } 550 } 551 552 void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, 553 unsigned int ix, bool xsk) 554 { 555 if (xsk) 556 mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); 557 else 558 mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); 559 560 mlx5e_rx_res_rss_enable(res); 561 562 mlx5e_rx_res_channel_activate_direct(res, chs, ix); 563 } 564 565 int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res, 566 struct mlx5e_packet_merge_param *pkt_merge_param) 567 { 568 struct mlx5e_tir_builder *builder; 569 int err, final_err; 570 unsigned int ix; 571 572 builder = mlx5e_tir_builder_alloc(true); 573 if (!builder) 574 return -ENOMEM; 575 576 down_write(&res->pkt_merge_param_sem); 577 res->pkt_merge_param = *pkt_merge_param; 578 579 mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param); 580 581 final_err = 0; 582 583 for (ix = 0; ix < MLX5E_MAX_NUM_RSS; ix++) { 584 struct mlx5e_rss *rss = res->rss[ix]; 585 586 if (!rss) 587 continue; 588 589 err = mlx5e_rss_packet_merge_set_param(rss, pkt_merge_param); 590 if (err) 591 final_err = final_err ? : err; 592 } 593 594 for (ix = 0; ix < res->max_nch; ix++) { 595 err = mlx5e_tir_modify(&res->channels[ix].direct_tir, builder); 596 if (err) { 597 mlx5_core_warn(res->mdev, "Failed to update packet merge state of direct TIR %#x for channel %u: err = %d\n", 598 mlx5e_tir_get_tirn(&res->channels[ix].direct_tir), ix, err); 599 if (!final_err) 600 final_err = err; 601 } 602 } 603 604 up_write(&res->pkt_merge_param_sem); 605 mlx5e_tir_builder_free(builder); 606 return final_err; 607 } 608 609 struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *res) 610 { 611 return mlx5e_rss_get_hash(res->rss[0]); 612 } 613 614 int mlx5e_rx_res_tls_tir_create(struct mlx5e_rx_res *res, unsigned int rxq, 615 struct mlx5e_tir *tir) 616 { 617 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 618 struct mlx5e_tir_builder *builder; 619 u32 rqtn; 620 int err; 621 622 builder = mlx5e_tir_builder_alloc(false); 623 if (!builder) 624 return -ENOMEM; 625 626 rqtn = mlx5e_rx_res_get_rqtn_direct(res, rxq); 627 628 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, rqtn, 629 inner_ft_support); 630 mlx5e_tir_builder_build_direct(builder); 631 mlx5e_tir_builder_build_tls(builder); 632 down_read(&res->pkt_merge_param_sem); 633 mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param); 634 err = mlx5e_tir_init(tir, builder, res->mdev, false); 635 up_read(&res->pkt_merge_param_sem); 636 637 mlx5e_tir_builder_free(builder); 638 639 return err; 640 } 641