1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. 3 4 #include "rss.h" 5 6 #define mlx5e_rss_warn(__dev, format, ...) \ 7 dev_warn((__dev)->device, "%s:%d:(pid %d): " format, \ 8 __func__, __LINE__, current->pid, \ 9 ##__VA_ARGS__) 10 11 static const struct mlx5e_rss_params_traffic_type rss_default_config[MLX5E_NUM_INDIR_TIRS] = { 12 [MLX5_TT_IPV4_TCP] = { 13 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 14 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 15 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 16 }, 17 [MLX5_TT_IPV6_TCP] = { 18 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 19 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 20 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 21 }, 22 [MLX5_TT_IPV4_UDP] = { 23 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 24 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 25 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 26 }, 27 [MLX5_TT_IPV6_UDP] = { 28 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 29 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 30 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 31 }, 32 [MLX5_TT_IPV4_IPSEC_AH] = { 33 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 34 .l4_prot_type = 0, 35 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 36 }, 37 [MLX5_TT_IPV6_IPSEC_AH] = { 38 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 39 .l4_prot_type = 0, 40 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 41 }, 42 [MLX5_TT_IPV4_IPSEC_ESP] = { 43 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 44 .l4_prot_type = 0, 45 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 46 }, 47 [MLX5_TT_IPV6_IPSEC_ESP] = { 48 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 49 .l4_prot_type = 0, 50 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 51 }, 52 [MLX5_TT_IPV4] = { 53 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 54 .l4_prot_type = 0, 55 .rx_hash_fields = MLX5_HASH_IP, 56 }, 57 [MLX5_TT_IPV6] = { 58 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 59 .l4_prot_type = 0, 60 .rx_hash_fields = MLX5_HASH_IP, 61 }, 62 }; 63 64 struct mlx5e_rss_params_traffic_type 65 mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt) 66 { 67 return rss_default_config[tt]; 68 } 69 70 struct mlx5e_rss { 71 struct mlx5e_rss_params_hash hash; 72 struct mlx5e_rss_params_indir indir; 73 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS]; 74 struct mlx5e_tir tir[MLX5E_NUM_INDIR_TIRS]; 75 struct mlx5e_tir inner_tir[MLX5E_NUM_INDIR_TIRS]; 76 struct mlx5e_rqt rqt; 77 struct mlx5_core_dev *mdev; 78 u32 drop_rqn; 79 bool inner_ft_support; 80 bool enabled; 81 }; 82 83 struct mlx5e_rss *mlx5e_rss_alloc(void) 84 { 85 return kvzalloc(sizeof(struct mlx5e_rss), GFP_KERNEL); 86 } 87 88 void mlx5e_rss_free(struct mlx5e_rss *rss) 89 { 90 kvfree(rss); 91 } 92 93 static void mlx5e_rss_params_init(struct mlx5e_rss *rss) 94 { 95 enum mlx5_traffic_types tt; 96 97 rss->hash.hfunc = ETH_RSS_HASH_TOP; 98 netdev_rss_key_fill(rss->hash.toeplitz_hash_key, 99 sizeof(rss->hash.toeplitz_hash_key)); 100 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 101 rss->rx_hash_fields[tt] = 102 mlx5e_rss_get_default_tt_config(tt).rx_hash_fields; 103 } 104 105 static struct mlx5e_rss_params_traffic_type 106 mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt) 107 { 108 struct mlx5e_rss_params_traffic_type rss_tt; 109 110 rss_tt = mlx5e_rss_get_default_tt_config(tt); 111 rss_tt.rx_hash_fields = rss->rx_hash_fields[tt]; 112 return rss_tt; 113 } 114 115 static int mlx5e_rss_create_tir(struct mlx5e_rss *rss, 116 enum mlx5_traffic_types tt, 117 const struct mlx5e_lro_param *init_lro_param, 118 bool inner) 119 { 120 struct mlx5e_rss_params_traffic_type rss_tt; 121 struct mlx5e_tir_builder *builder; 122 struct mlx5e_tir *tir; 123 u32 rqtn; 124 int err; 125 126 if (inner && !rss->inner_ft_support) { 127 mlx5e_rss_warn(rss->mdev, 128 "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n", 129 tt); 130 return -EINVAL; 131 } 132 133 tir = inner ? &rss->inner_tir[tt] : &rss->tir[tt]; 134 135 builder = mlx5e_tir_builder_alloc(false); 136 if (!builder) 137 return -ENOMEM; 138 139 rqtn = mlx5e_rqt_get_rqtn(&rss->rqt); 140 mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn, 141 rqtn, rss->inner_ft_support); 142 mlx5e_tir_builder_build_lro(builder, init_lro_param); 143 rss_tt = mlx5e_rss_get_tt_config(rss, tt); 144 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner); 145 146 err = mlx5e_tir_init(tir, builder, rss->mdev, true); 147 mlx5e_tir_builder_free(builder); 148 if (err) 149 mlx5e_rss_warn(rss->mdev, "Failed to create %sindirect TIR: err = %d, tt = %d\n", 150 inner ? "inner " : "", err, tt); 151 return err; 152 } 153 154 static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, 155 bool inner) 156 { 157 struct mlx5e_tir *tir; 158 159 tir = inner ? &rss->inner_tir[tt] : &rss->tir[tt]; 160 mlx5e_tir_destroy(tir); 161 } 162 163 static int mlx5e_rss_create_tirs(struct mlx5e_rss *rss, 164 const struct mlx5e_lro_param *init_lro_param, 165 bool inner) 166 { 167 enum mlx5_traffic_types tt, max_tt; 168 int err; 169 170 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 171 err = mlx5e_rss_create_tir(rss, tt, init_lro_param, inner); 172 if (err) 173 goto err_destroy_tirs; 174 } 175 176 return 0; 177 178 err_destroy_tirs: 179 max_tt = tt; 180 for (tt = 0; tt < max_tt; tt++) 181 mlx5e_rss_destroy_tir(rss, tt, inner); 182 return err; 183 } 184 185 static void mlx5e_rss_destroy_tirs(struct mlx5e_rss *rss, bool inner) 186 { 187 enum mlx5_traffic_types tt; 188 189 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 190 mlx5e_rss_destroy_tir(rss, tt, inner); 191 } 192 193 static int mlx5e_rss_update_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, 194 bool inner) 195 { 196 struct mlx5e_rss_params_traffic_type rss_tt; 197 struct mlx5e_tir_builder *builder; 198 struct mlx5e_tir *tir; 199 int err; 200 201 tir = inner ? &rss->inner_tir[tt] : &rss->tir[tt]; 202 203 builder = mlx5e_tir_builder_alloc(true); 204 if (!builder) 205 return -ENOMEM; 206 207 rss_tt = mlx5e_rss_get_tt_config(rss, tt); 208 209 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner); 210 err = mlx5e_tir_modify(tir, builder); 211 212 mlx5e_tir_builder_free(builder); 213 return err; 214 } 215 216 static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss) 217 { 218 enum mlx5_traffic_types tt; 219 int err, retval; 220 221 retval = 0; 222 223 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 224 err = mlx5e_rss_update_tir(rss, tt, false); 225 if (err) { 226 retval = retval ? : err; 227 mlx5e_rss_warn(rss->mdev, 228 "Failed to update RSS hash of indirect TIR for traffic type %d: err = %d\n", 229 tt, err); 230 } 231 232 if (!rss->inner_ft_support) 233 continue; 234 235 err = mlx5e_rss_update_tir(rss, tt, true); 236 if (err) { 237 retval = retval ? : err; 238 mlx5e_rss_warn(rss->mdev, 239 "Failed to update RSS hash of inner indirect TIR for traffic type %d: err = %d\n", 240 tt, err); 241 } 242 } 243 return retval; 244 } 245 246 int mlx5e_rss_init(struct mlx5e_rss *rss, struct mlx5_core_dev *mdev, 247 bool inner_ft_support, u32 drop_rqn, 248 const struct mlx5e_lro_param *init_lro_param) 249 { 250 int err; 251 252 rss->mdev = mdev; 253 rss->inner_ft_support = inner_ft_support; 254 rss->drop_rqn = drop_rqn; 255 256 mlx5e_rss_params_init(rss); 257 258 err = mlx5e_rqt_init_direct(&rss->rqt, mdev, true, drop_rqn); 259 if (err) 260 goto err_out; 261 262 err = mlx5e_rss_create_tirs(rss, init_lro_param, false); 263 if (err) 264 goto err_destroy_rqt; 265 266 if (inner_ft_support) { 267 err = mlx5e_rss_create_tirs(rss, init_lro_param, true); 268 if (err) 269 goto err_destroy_tirs; 270 } 271 272 return 0; 273 274 err_destroy_tirs: 275 mlx5e_rss_destroy_tirs(rss, false); 276 err_destroy_rqt: 277 mlx5e_rqt_destroy(&rss->rqt); 278 err_out: 279 return err; 280 } 281 282 void mlx5e_rss_cleanup(struct mlx5e_rss *rss) 283 { 284 mlx5e_rss_destroy_tirs(rss, false); 285 286 if (rss->inner_ft_support) 287 mlx5e_rss_destroy_tirs(rss, true); 288 289 mlx5e_rqt_destroy(&rss->rqt); 290 } 291 292 u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, 293 bool inner) 294 { 295 struct mlx5e_tir *tir; 296 297 WARN_ON(inner && !rss->inner_ft_support); 298 tir = inner ? &rss->inner_tir[tt] : &rss->tir[tt]; 299 300 return mlx5e_tir_get_tirn(tir); 301 } 302 303 static void mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) 304 { 305 int err; 306 307 err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, num_rqns, rss->hash.hfunc, &rss->indir); 308 if (err) 309 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n", 310 mlx5e_rqt_get_rqtn(&rss->rqt), err); 311 } 312 313 void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, unsigned int num_rqns) 314 { 315 rss->enabled = true; 316 mlx5e_rss_apply(rss, rqns, num_rqns); 317 } 318 319 void mlx5e_rss_disable(struct mlx5e_rss *rss) 320 { 321 int err; 322 323 rss->enabled = false; 324 err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->drop_rqn); 325 if (err) 326 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n", 327 mlx5e_rqt_get_rqtn(&rss->rqt), rss->drop_rqn, err); 328 } 329 330 int mlx5e_rss_lro_set_param(struct mlx5e_rss *rss, struct mlx5e_lro_param *lro_param) 331 { 332 struct mlx5e_tir_builder *builder; 333 enum mlx5_traffic_types tt; 334 int err, final_err; 335 336 builder = mlx5e_tir_builder_alloc(true); 337 if (!builder) 338 return -ENOMEM; 339 340 mlx5e_tir_builder_build_lro(builder, lro_param); 341 342 final_err = 0; 343 344 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 345 err = mlx5e_tir_modify(&rss->tir[tt], builder); 346 if (err) { 347 mlx5e_rss_warn(rss->mdev, "Failed to update LRO state of indirect TIR %#x for traffic type %d: err = %d\n", 348 mlx5e_tir_get_tirn(&rss->tir[tt]), tt, err); 349 if (!final_err) 350 final_err = err; 351 } 352 353 if (!rss->inner_ft_support) 354 continue; 355 356 err = mlx5e_tir_modify(&rss->inner_tir[tt], builder); 357 if (err) { 358 mlx5e_rss_warn(rss->mdev, "Failed to update LRO state of inner indirect TIR %#x for traffic type %d: err = %d\n", 359 mlx5e_tir_get_tirn(&rss->inner_tir[tt]), tt, err); 360 if (!final_err) 361 final_err = err; 362 } 363 } 364 365 mlx5e_tir_builder_free(builder); 366 return final_err; 367 } 368 369 int mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc) 370 { 371 unsigned int i; 372 373 if (indir) 374 for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) 375 indir[i] = rss->indir.table[i]; 376 377 if (key) 378 memcpy(key, rss->hash.toeplitz_hash_key, 379 sizeof(rss->hash.toeplitz_hash_key)); 380 381 if (hfunc) 382 *hfunc = rss->hash.hfunc; 383 384 return 0; 385 } 386 387 int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir, 388 const u8 *key, const u8 *hfunc, 389 u32 *rqns, unsigned int num_rqns) 390 { 391 bool changed_indir = false; 392 bool changed_hash = false; 393 394 if (hfunc && *hfunc != rss->hash.hfunc) { 395 switch (*hfunc) { 396 case ETH_RSS_HASH_XOR: 397 case ETH_RSS_HASH_TOP: 398 break; 399 default: 400 return -EINVAL; 401 } 402 changed_hash = true; 403 changed_indir = true; 404 rss->hash.hfunc = *hfunc; 405 } 406 407 if (key) { 408 if (rss->hash.hfunc == ETH_RSS_HASH_TOP) 409 changed_hash = true; 410 memcpy(rss->hash.toeplitz_hash_key, key, 411 sizeof(rss->hash.toeplitz_hash_key)); 412 } 413 414 if (indir) { 415 unsigned int i; 416 417 changed_indir = true; 418 419 for (i = 0; i < MLX5E_INDIR_RQT_SIZE; i++) 420 rss->indir.table[i] = indir[i]; 421 } 422 423 if (changed_indir && rss->enabled) 424 mlx5e_rss_apply(rss, rqns, num_rqns); 425 426 if (changed_hash) 427 mlx5e_rss_update_tirs(rss); 428 429 return 0; 430 } 431 432 struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss) 433 { 434 return rss->hash; 435 } 436 437 u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt) 438 { 439 return rss->rx_hash_fields[tt]; 440 } 441 442 int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, 443 u8 rx_hash_fields) 444 { 445 u8 old_rx_hash_fields; 446 int err; 447 448 old_rx_hash_fields = rss->rx_hash_fields[tt]; 449 450 if (old_rx_hash_fields == rx_hash_fields) 451 return 0; 452 453 rss->rx_hash_fields[tt] = rx_hash_fields; 454 455 err = mlx5e_rss_update_tir(rss, tt, false); 456 if (err) { 457 rss->rx_hash_fields[tt] = old_rx_hash_fields; 458 mlx5e_rss_warn(rss->mdev, 459 "Failed to update RSS hash fields of indirect TIR for traffic type %d: err = %d\n", 460 tt, err); 461 return err; 462 } 463 464 if (!(rss->inner_ft_support)) 465 return 0; 466 467 err = mlx5e_rss_update_tir(rss, tt, true); 468 if (err) { 469 /* Partial update happened. Try to revert - it may fail too, but 470 * there is nothing more we can do. 471 */ 472 rss->rx_hash_fields[tt] = old_rx_hash_fields; 473 mlx5e_rss_warn(rss->mdev, 474 "Failed to update RSS hash fields of inner indirect TIR for traffic type %d: err = %d\n", 475 tt, err); 476 if (mlx5e_rss_update_tir(rss, tt, false)) 477 mlx5e_rss_warn(rss->mdev, 478 "Partial update of RSS hash fields happened: failed to revert indirect TIR for traffic type %d to the old values\n", 479 tt); 480 } 481 482 return err; 483 } 484 485 void mlx5e_rss_set_indir_uniform(struct mlx5e_rss *rss, unsigned int nch) 486 { 487 mlx5e_rss_params_indir_init_uniform(&rss->indir, nch); 488 } 489