1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ 3 4 #include <net/macsec.h> 5 #include <linux/mlx5/qp.h> 6 #include <linux/if_vlan.h> 7 #include <linux/mlx5/fs_helpers.h> 8 #include <linux/mlx5/macsec.h> 9 #include "fs_core.h" 10 #include "lib/macsec_fs.h" 11 #include "mlx5_core.h" 12 13 /* MACsec TX flow steering */ 14 #define CRYPTO_NUM_MAXSEC_FTE BIT(15) 15 #define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1 16 17 #define TX_CRYPTO_TABLE_LEVEL 0 18 #define TX_CRYPTO_TABLE_NUM_GROUPS 3 19 #define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1 20 #define TX_CRYPTO_TABLE_SA_GROUP_SIZE \ 21 (CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \ 22 CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE)) 23 #define TX_CHECK_TABLE_LEVEL 1 24 #define TX_CHECK_TABLE_NUM_FTE 2 25 #define RX_CRYPTO_TABLE_LEVEL 0 26 #define RX_CHECK_TABLE_LEVEL 1 27 #define RX_ROCE_TABLE_LEVEL 2 28 #define RX_CHECK_TABLE_NUM_FTE 3 29 #define RX_ROCE_TABLE_NUM_FTE 2 30 #define RX_CRYPTO_TABLE_NUM_GROUPS 3 31 #define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \ 32 ((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2) 33 #define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \ 34 (CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE) 35 #define RX_NUM_OF_RULES_PER_SA 2 36 37 #define RDMA_RX_ROCE_IP_TABLE_LEVEL 0 38 #define RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL 1 39 40 #define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */ 41 #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23 42 #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8 43 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5 44 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) 45 #define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8 46 #define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN) 47 48 /* MACsec RX flow steering */ 49 #define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E 50 51 /* MACsec fs_id handling for steering */ 52 #define macsec_fs_set_tx_fs_id(fs_id) (MLX5_ETH_WQE_FT_META_MACSEC | (fs_id) << 2) 53 #define macsec_fs_set_rx_fs_id(fs_id) ((fs_id) | BIT(30)) 54 55 struct mlx5_sectag_header { 56 __be16 ethertype; 57 u8 tci_an; 58 u8 sl; 59 u32 pn; 60 u8 sci[MACSEC_SCI_LEN]; /* optional */ 61 } __packed; 62 63 struct mlx5_roce_macsec_tx_rule { 64 u32 fs_id; 65 u16 gid_idx; 66 struct list_head entry; 67 struct mlx5_flow_handle *rule; 68 struct mlx5_modify_hdr *meta_modhdr; 69 }; 70 71 struct mlx5_macsec_tx_rule { 72 struct mlx5_flow_handle *rule; 73 struct mlx5_pkt_reformat *pkt_reformat; 74 u32 fs_id; 75 }; 76 77 struct mlx5_macsec_flow_table { 78 int num_groups; 79 struct mlx5_flow_table *t; 80 struct mlx5_flow_group **g; 81 }; 82 83 struct mlx5_macsec_tables { 84 struct mlx5_macsec_flow_table ft_crypto; 85 struct mlx5_flow_handle *crypto_miss_rule; 86 87 struct mlx5_flow_table *ft_check; 88 struct mlx5_flow_group *ft_check_group; 89 struct mlx5_fc *check_miss_rule_counter; 90 struct mlx5_flow_handle *check_miss_rule; 91 struct mlx5_fc *check_rule_counter; 92 93 u32 refcnt; 94 }; 95 96 struct mlx5_fs_id { 97 u32 id; 98 refcount_t refcnt; 99 sci_t sci; 100 struct rhash_head hash; 101 }; 102 103 struct mlx5_macsec_device { 104 struct list_head macsec_devices_list_entry; 105 void *macdev; 106 struct xarray tx_id_xa; 107 struct xarray rx_id_xa; 108 }; 109 110 struct mlx5_macsec_tx { 111 struct mlx5_flow_handle *crypto_mke_rule; 112 struct mlx5_flow_handle *check_rule; 113 114 struct ida tx_halloc; 115 116 struct mlx5_macsec_tables tables; 117 118 struct mlx5_flow_table *ft_rdma_tx; 119 }; 120 121 struct mlx5_roce_macsec_rx_rule { 122 u32 fs_id; 123 u16 gid_idx; 124 struct mlx5_flow_handle *op; 125 struct mlx5_flow_handle *ip; 126 struct list_head entry; 127 }; 128 129 struct mlx5_macsec_rx_rule { 130 struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA]; 131 struct mlx5_modify_hdr *meta_modhdr; 132 }; 133 134 struct mlx5_macsec_miss { 135 struct mlx5_flow_group *g; 136 struct mlx5_flow_handle *rule; 137 }; 138 139 struct mlx5_macsec_rx_roce { 140 /* Flow table/rules in NIC domain, to check if it's a RoCE packet */ 141 struct mlx5_flow_group *g; 142 struct mlx5_flow_table *ft; 143 struct mlx5_flow_handle *rule; 144 struct mlx5_modify_hdr *copy_modify_hdr; 145 struct mlx5_macsec_miss nic_miss; 146 147 /* Flow table/rule in RDMA domain, to check dgid */ 148 struct mlx5_flow_table *ft_ip_check; 149 struct mlx5_flow_table *ft_macsec_op_check; 150 struct mlx5_macsec_miss miss; 151 }; 152 153 struct mlx5_macsec_rx { 154 struct mlx5_flow_handle *check_rule[2]; 155 struct mlx5_pkt_reformat *check_rule_pkt_reformat[2]; 156 157 struct mlx5_macsec_tables tables; 158 struct mlx5_macsec_rx_roce roce; 159 }; 160 161 union mlx5_macsec_rule { 162 struct mlx5_macsec_tx_rule tx_rule; 163 struct mlx5_macsec_rx_rule rx_rule; 164 }; 165 166 static const struct rhashtable_params rhash_sci = { 167 .key_len = sizeof_field(struct mlx5_fs_id, sci), 168 .key_offset = offsetof(struct mlx5_fs_id, sci), 169 .head_offset = offsetof(struct mlx5_fs_id, hash), 170 .automatic_shrinking = true, 171 .min_size = 1, 172 }; 173 174 static const struct rhashtable_params rhash_fs_id = { 175 .key_len = sizeof_field(struct mlx5_fs_id, id), 176 .key_offset = offsetof(struct mlx5_fs_id, id), 177 .head_offset = offsetof(struct mlx5_fs_id, hash), 178 .automatic_shrinking = true, 179 .min_size = 1, 180 }; 181 182 struct mlx5_macsec_fs { 183 struct mlx5_core_dev *mdev; 184 struct mlx5_macsec_tx *tx_fs; 185 struct mlx5_macsec_rx *rx_fs; 186 187 /* Stats manage */ 188 struct mlx5_macsec_stats stats; 189 190 /* Tx sci -> fs id mapping handling */ 191 struct rhashtable sci_hash; /* sci -> mlx5_fs_id */ 192 193 /* RX fs_id -> mlx5_fs_id mapping handling */ 194 struct rhashtable fs_id_hash; /* fs_id -> mlx5_fs_id */ 195 196 /* TX & RX fs_id lists per macsec device */ 197 struct list_head macsec_devices_list; 198 }; 199 200 static void macsec_fs_destroy_groups(struct mlx5_macsec_flow_table *ft) 201 { 202 int i; 203 204 for (i = ft->num_groups - 1; i >= 0; i--) { 205 if (!IS_ERR_OR_NULL(ft->g[i])) 206 mlx5_destroy_flow_group(ft->g[i]); 207 ft->g[i] = NULL; 208 } 209 ft->num_groups = 0; 210 } 211 212 static void macsec_fs_destroy_flow_table(struct mlx5_macsec_flow_table *ft) 213 { 214 macsec_fs_destroy_groups(ft); 215 kfree(ft->g); 216 mlx5_destroy_flow_table(ft->t); 217 ft->t = NULL; 218 } 219 220 static void macsec_fs_tx_destroy(struct mlx5_macsec_fs *macsec_fs) 221 { 222 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 223 struct mlx5_macsec_tables *tx_tables; 224 225 if (mlx5_is_macsec_roce_supported(macsec_fs->mdev)) 226 mlx5_destroy_flow_table(tx_fs->ft_rdma_tx); 227 228 tx_tables = &tx_fs->tables; 229 230 /* Tx check table */ 231 if (tx_fs->check_rule) { 232 mlx5_del_flow_rules(tx_fs->check_rule); 233 tx_fs->check_rule = NULL; 234 } 235 236 if (tx_tables->check_miss_rule) { 237 mlx5_del_flow_rules(tx_tables->check_miss_rule); 238 tx_tables->check_miss_rule = NULL; 239 } 240 241 if (tx_tables->ft_check_group) { 242 mlx5_destroy_flow_group(tx_tables->ft_check_group); 243 tx_tables->ft_check_group = NULL; 244 } 245 246 if (tx_tables->ft_check) { 247 mlx5_destroy_flow_table(tx_tables->ft_check); 248 tx_tables->ft_check = NULL; 249 } 250 251 /* Tx crypto table */ 252 if (tx_fs->crypto_mke_rule) { 253 mlx5_del_flow_rules(tx_fs->crypto_mke_rule); 254 tx_fs->crypto_mke_rule = NULL; 255 } 256 257 if (tx_tables->crypto_miss_rule) { 258 mlx5_del_flow_rules(tx_tables->crypto_miss_rule); 259 tx_tables->crypto_miss_rule = NULL; 260 } 261 262 macsec_fs_destroy_flow_table(&tx_tables->ft_crypto); 263 } 264 265 static int macsec_fs_tx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft) 266 { 267 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 268 int mclen = MLX5_ST_SZ_BYTES(fte_match_param); 269 int ix = 0; 270 u32 *in; 271 int err; 272 u8 *mc; 273 274 ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 275 if (!ft->g) 276 return -ENOMEM; 277 in = kvzalloc(inlen, GFP_KERNEL); 278 279 if (!in) { 280 kfree(ft->g); 281 ft->g = NULL; 282 return -ENOMEM; 283 } 284 285 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 286 287 /* Flow Group for MKE match */ 288 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 289 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 290 291 MLX5_SET_CFG(in, start_flow_index, ix); 292 ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE; 293 MLX5_SET_CFG(in, end_flow_index, ix - 1); 294 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 295 if (IS_ERR(ft->g[ft->num_groups])) 296 goto err; 297 ft->num_groups++; 298 299 /* Flow Group for SA rules */ 300 memset(in, 0, inlen); 301 memset(mc, 0, mclen); 302 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2); 303 MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a, 304 MLX5_ETH_WQE_FT_META_MACSEC_MASK); 305 306 MLX5_SET_CFG(in, start_flow_index, ix); 307 ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE; 308 MLX5_SET_CFG(in, end_flow_index, ix - 1); 309 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 310 if (IS_ERR(ft->g[ft->num_groups])) 311 goto err; 312 ft->num_groups++; 313 314 /* Flow Group for l2 traps */ 315 memset(in, 0, inlen); 316 memset(mc, 0, mclen); 317 MLX5_SET_CFG(in, start_flow_index, ix); 318 ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; 319 MLX5_SET_CFG(in, end_flow_index, ix - 1); 320 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 321 if (IS_ERR(ft->g[ft->num_groups])) 322 goto err; 323 ft->num_groups++; 324 325 kvfree(in); 326 return 0; 327 328 err: 329 err = PTR_ERR(ft->g[ft->num_groups]); 330 ft->g[ft->num_groups] = NULL; 331 kvfree(in); 332 333 return err; 334 } 335 336 static struct mlx5_flow_table 337 *macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags, 338 int level, int max_fte) 339 { 340 struct mlx5_flow_table_attr ft_attr = {}; 341 struct mlx5_flow_table *fdb = NULL; 342 343 /* reserve entry for the match all miss group and rule */ 344 ft_attr.autogroup.num_reserved_entries = 1; 345 ft_attr.autogroup.max_num_groups = 1; 346 ft_attr.prio = 0; 347 ft_attr.flags = flags; 348 ft_attr.level = level; 349 ft_attr.max_fte = max_fte; 350 351 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 352 353 return fdb; 354 } 355 356 enum { 357 RDMA_TX_MACSEC_LEVEL = 0, 358 }; 359 360 static int macsec_fs_tx_roce_create(struct mlx5_macsec_fs *macsec_fs) 361 { 362 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 363 struct mlx5_core_dev *mdev = macsec_fs->mdev; 364 struct mlx5_flow_namespace *ns; 365 struct mlx5_flow_table *ft; 366 int err; 367 368 if (!mlx5_is_macsec_roce_supported(mdev)) { 369 mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n"); 370 return 0; 371 } 372 373 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC); 374 if (!ns) 375 return -ENOMEM; 376 377 /* Tx RoCE crypto table */ 378 ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_TX_MACSEC_LEVEL, CRYPTO_NUM_MAXSEC_FTE); 379 if (IS_ERR(ft)) { 380 err = PTR_ERR(ft); 381 mlx5_core_err(mdev, "Failed to create MACsec RoCE Tx crypto table err(%d)\n", err); 382 return err; 383 } 384 tx_fs->ft_rdma_tx = ft; 385 386 return 0; 387 } 388 389 static int macsec_fs_tx_create(struct mlx5_macsec_fs *macsec_fs) 390 { 391 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 392 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 393 struct mlx5_core_dev *mdev = macsec_fs->mdev; 394 struct mlx5_flow_table_attr ft_attr = {}; 395 struct mlx5_flow_destination dest = {}; 396 struct mlx5_macsec_tables *tx_tables; 397 struct mlx5_flow_act flow_act = {}; 398 struct mlx5_macsec_flow_table *ft_crypto; 399 struct mlx5_flow_table *flow_table; 400 struct mlx5_flow_group *flow_group; 401 struct mlx5_flow_namespace *ns; 402 struct mlx5_flow_handle *rule; 403 struct mlx5_flow_spec *spec; 404 u32 *flow_group_in; 405 int err; 406 407 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); 408 if (!ns) 409 return -ENOMEM; 410 411 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 412 if (!spec) 413 return -ENOMEM; 414 415 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 416 if (!flow_group_in) { 417 err = -ENOMEM; 418 goto out_spec; 419 } 420 421 tx_tables = &tx_fs->tables; 422 ft_crypto = &tx_tables->ft_crypto; 423 424 /* Tx crypto table */ 425 ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 426 ft_attr.level = TX_CRYPTO_TABLE_LEVEL; 427 ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; 428 429 flow_table = mlx5_create_flow_table(ns, &ft_attr); 430 if (IS_ERR(flow_table)) { 431 err = PTR_ERR(flow_table); 432 mlx5_core_err(mdev, "Failed to create MACsec Tx crypto table err(%d)\n", err); 433 goto out_flow_group; 434 } 435 ft_crypto->t = flow_table; 436 437 /* Tx crypto table groups */ 438 err = macsec_fs_tx_create_crypto_table_groups(ft_crypto); 439 if (err) { 440 mlx5_core_err(mdev, 441 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", 442 err); 443 goto err; 444 } 445 446 /* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */ 447 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 448 449 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); 450 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE); 451 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; 452 453 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, NULL, 0); 454 if (IS_ERR(rule)) { 455 err = PTR_ERR(rule); 456 mlx5_core_err(mdev, "Failed to add MACsec TX MKE rule, err=%d\n", err); 457 goto err; 458 } 459 tx_fs->crypto_mke_rule = rule; 460 461 /* Tx crypto table Default miss rule */ 462 memset(&flow_act, 0, sizeof(flow_act)); 463 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; 464 rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); 465 if (IS_ERR(rule)) { 466 err = PTR_ERR(rule); 467 mlx5_core_err(mdev, "Failed to add MACsec Tx table default miss rule %d\n", err); 468 goto err; 469 } 470 tx_tables->crypto_miss_rule = rule; 471 472 /* Tx check table */ 473 flow_table = macsec_fs_auto_group_table_create(ns, 0, TX_CHECK_TABLE_LEVEL, 474 TX_CHECK_TABLE_NUM_FTE); 475 if (IS_ERR(flow_table)) { 476 err = PTR_ERR(flow_table); 477 mlx5_core_err(mdev, "Fail to create MACsec TX check table, err(%d)\n", err); 478 goto err; 479 } 480 tx_tables->ft_check = flow_table; 481 482 /* Tx check table Default miss group/rule */ 483 memset(flow_group_in, 0, inlen); 484 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); 485 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); 486 flow_group = mlx5_create_flow_group(tx_tables->ft_check, flow_group_in); 487 if (IS_ERR(flow_group)) { 488 err = PTR_ERR(flow_group); 489 mlx5_core_err(mdev, 490 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", 491 err); 492 goto err; 493 } 494 tx_tables->ft_check_group = flow_group; 495 496 /* Tx check table default drop rule */ 497 memset(&dest, 0, sizeof(struct mlx5_flow_destination)); 498 memset(&flow_act, 0, sizeof(flow_act)); 499 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 500 dest.counter_id = mlx5_fc_id(tx_tables->check_miss_rule_counter); 501 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 502 rule = mlx5_add_flow_rules(tx_tables->ft_check, NULL, &flow_act, &dest, 1); 503 if (IS_ERR(rule)) { 504 err = PTR_ERR(rule); 505 mlx5_core_err(mdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err); 506 goto err; 507 } 508 tx_tables->check_miss_rule = rule; 509 510 /* Tx check table rule */ 511 memset(spec, 0, sizeof(struct mlx5_flow_spec)); 512 memset(&dest, 0, sizeof(struct mlx5_flow_destination)); 513 memset(&flow_act, 0, sizeof(flow_act)); 514 515 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); 516 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); 517 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 518 519 flow_act.flags = FLOW_ACT_NO_APPEND; 520 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT; 521 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 522 dest.counter_id = mlx5_fc_id(tx_tables->check_rule_counter); 523 rule = mlx5_add_flow_rules(tx_tables->ft_check, spec, &flow_act, &dest, 1); 524 if (IS_ERR(rule)) { 525 err = PTR_ERR(rule); 526 mlx5_core_err(mdev, "Failed to add MACsec check rule, err=%d\n", err); 527 goto err; 528 } 529 tx_fs->check_rule = rule; 530 531 err = macsec_fs_tx_roce_create(macsec_fs); 532 if (err) 533 goto err; 534 535 kvfree(flow_group_in); 536 kvfree(spec); 537 return 0; 538 539 err: 540 macsec_fs_tx_destroy(macsec_fs); 541 out_flow_group: 542 kvfree(flow_group_in); 543 out_spec: 544 kvfree(spec); 545 return err; 546 } 547 548 static int macsec_fs_tx_ft_get(struct mlx5_macsec_fs *macsec_fs) 549 { 550 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 551 struct mlx5_macsec_tables *tx_tables; 552 int err = 0; 553 554 tx_tables = &tx_fs->tables; 555 if (tx_tables->refcnt) 556 goto out; 557 558 err = macsec_fs_tx_create(macsec_fs); 559 if (err) 560 return err; 561 562 out: 563 tx_tables->refcnt++; 564 return err; 565 } 566 567 static void macsec_fs_tx_ft_put(struct mlx5_macsec_fs *macsec_fs) 568 { 569 struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; 570 571 if (--tx_tables->refcnt) 572 return; 573 574 macsec_fs_tx_destroy(macsec_fs); 575 } 576 577 static int macsec_fs_tx_setup_fte(struct mlx5_macsec_fs *macsec_fs, 578 struct mlx5_flow_spec *spec, 579 struct mlx5_flow_act *flow_act, 580 u32 macsec_obj_id, 581 u32 *fs_id) 582 { 583 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 584 int err = 0; 585 u32 id; 586 587 err = ida_alloc_range(&tx_fs->tx_halloc, 1, 588 MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES, 589 GFP_KERNEL); 590 if (err < 0) 591 return err; 592 593 id = err; 594 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 595 596 /* Metadata match */ 597 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, 598 MLX5_ETH_WQE_FT_META_MACSEC_MASK); 599 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, 600 macsec_fs_set_tx_fs_id(id)); 601 602 *fs_id = id; 603 flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; 604 flow_act->crypto.obj_id = macsec_obj_id; 605 606 mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id); 607 return 0; 608 } 609 610 static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx, 611 char *reformatbf, 612 size_t *reformat_size) 613 { 614 const struct macsec_secy *secy = ctx->secy; 615 bool sci_present = macsec_send_sci(secy); 616 struct mlx5_sectag_header sectag = {}; 617 const struct macsec_tx_sc *tx_sc; 618 619 tx_sc = &secy->tx_sc; 620 sectag.ethertype = htons(ETH_P_MACSEC); 621 622 if (sci_present) { 623 sectag.tci_an |= MACSEC_TCI_SC; 624 memcpy(§ag.sci, &secy->sci, 625 sizeof(sectag.sci)); 626 } else { 627 if (tx_sc->end_station) 628 sectag.tci_an |= MACSEC_TCI_ES; 629 if (tx_sc->scb) 630 sectag.tci_an |= MACSEC_TCI_SCB; 631 } 632 633 /* With GCM, C/E clear for !encrypt, both set for encrypt */ 634 if (tx_sc->encrypt) 635 sectag.tci_an |= MACSEC_TCI_CONFID; 636 else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) 637 sectag.tci_an |= MACSEC_TCI_C; 638 639 sectag.tci_an |= tx_sc->encoding_sa; 640 641 *reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0); 642 643 memcpy(reformatbf, §ag, *reformat_size); 644 } 645 646 static bool macsec_fs_is_macsec_device_empty(struct mlx5_macsec_device *macsec_device) 647 { 648 if (xa_empty(&macsec_device->tx_id_xa) && 649 xa_empty(&macsec_device->rx_id_xa)) 650 return true; 651 652 return false; 653 } 654 655 static void macsec_fs_id_del(struct list_head *macsec_devices_list, u32 fs_id, 656 void *macdev, struct rhashtable *hash_table, bool is_tx) 657 { 658 const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id; 659 struct mlx5_macsec_device *iter, *macsec_device = NULL; 660 struct mlx5_fs_id *fs_id_found; 661 struct xarray *fs_id_xa; 662 663 list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) { 664 if (iter->macdev == macdev) { 665 macsec_device = iter; 666 break; 667 } 668 } 669 WARN_ON(!macsec_device); 670 671 fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa : 672 &macsec_device->rx_id_xa; 673 xa_lock(fs_id_xa); 674 fs_id_found = xa_load(fs_id_xa, fs_id); 675 WARN_ON(!fs_id_found); 676 677 if (!refcount_dec_and_test(&fs_id_found->refcnt)) { 678 xa_unlock(fs_id_xa); 679 return; 680 } 681 682 if (fs_id_found->id) { 683 /* Make sure ongoing datapath readers sees a valid SA */ 684 rhashtable_remove_fast(hash_table, &fs_id_found->hash, *rhash); 685 fs_id_found->id = 0; 686 } 687 xa_unlock(fs_id_xa); 688 689 xa_erase(fs_id_xa, fs_id); 690 691 kfree(fs_id_found); 692 693 if (macsec_fs_is_macsec_device_empty(macsec_device)) { 694 list_del(&macsec_device->macsec_devices_list_entry); 695 kfree(macsec_device); 696 } 697 } 698 699 static int macsec_fs_id_add(struct list_head *macsec_devices_list, u32 fs_id, 700 void *macdev, struct rhashtable *hash_table, sci_t sci, 701 bool is_tx) 702 { 703 const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id; 704 struct mlx5_macsec_device *iter, *macsec_device = NULL; 705 struct mlx5_fs_id *fs_id_iter; 706 struct xarray *fs_id_xa; 707 int err; 708 709 if (!is_tx) { 710 rcu_read_lock(); 711 fs_id_iter = rhashtable_lookup(hash_table, &fs_id, rhash_fs_id); 712 if (fs_id_iter) { 713 refcount_inc(&fs_id_iter->refcnt); 714 rcu_read_unlock(); 715 return 0; 716 } 717 rcu_read_unlock(); 718 } 719 720 fs_id_iter = kzalloc(sizeof(*fs_id_iter), GFP_KERNEL); 721 if (!fs_id_iter) 722 return -ENOMEM; 723 724 list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) { 725 if (iter->macdev == macdev) { 726 macsec_device = iter; 727 break; 728 } 729 } 730 731 if (!macsec_device) { /* first time adding a SA to that device */ 732 macsec_device = kzalloc(sizeof(*macsec_device), GFP_KERNEL); 733 if (!macsec_device) { 734 err = -ENOMEM; 735 goto err_alloc_dev; 736 } 737 macsec_device->macdev = macdev; 738 xa_init(&macsec_device->tx_id_xa); 739 xa_init(&macsec_device->rx_id_xa); 740 list_add(&macsec_device->macsec_devices_list_entry, macsec_devices_list); 741 } 742 743 fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa : 744 &macsec_device->rx_id_xa; 745 fs_id_iter->id = fs_id; 746 refcount_set(&fs_id_iter->refcnt, 1); 747 fs_id_iter->sci = sci; 748 err = xa_err(xa_store(fs_id_xa, fs_id, fs_id_iter, GFP_KERNEL)); 749 if (err) 750 goto err_store_id; 751 752 err = rhashtable_insert_fast(hash_table, &fs_id_iter->hash, *rhash); 753 if (err) 754 goto err_hash_insert; 755 756 return 0; 757 758 err_hash_insert: 759 xa_erase(fs_id_xa, fs_id); 760 err_store_id: 761 if (macsec_fs_is_macsec_device_empty(macsec_device)) { 762 list_del(&macsec_device->macsec_devices_list_entry); 763 kfree(macsec_device); 764 } 765 err_alloc_dev: 766 kfree(fs_id_iter); 767 return err; 768 } 769 770 static void macsec_fs_tx_del_rule(struct mlx5_macsec_fs *macsec_fs, 771 struct mlx5_macsec_tx_rule *tx_rule, 772 void *macdev) 773 { 774 macsec_fs_id_del(&macsec_fs->macsec_devices_list, tx_rule->fs_id, macdev, 775 &macsec_fs->sci_hash, true); 776 777 if (tx_rule->rule) { 778 mlx5_del_flow_rules(tx_rule->rule); 779 tx_rule->rule = NULL; 780 } 781 782 if (tx_rule->pkt_reformat) { 783 mlx5_packet_reformat_dealloc(macsec_fs->mdev, tx_rule->pkt_reformat); 784 tx_rule->pkt_reformat = NULL; 785 } 786 787 if (tx_rule->fs_id) { 788 ida_free(&macsec_fs->tx_fs->tx_halloc, tx_rule->fs_id); 789 tx_rule->fs_id = 0; 790 } 791 792 kfree(tx_rule); 793 794 macsec_fs_tx_ft_put(macsec_fs); 795 } 796 797 #define MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES 1 798 799 static union mlx5_macsec_rule * 800 macsec_fs_tx_add_rule(struct mlx5_macsec_fs *macsec_fs, 801 const struct macsec_context *macsec_ctx, 802 struct mlx5_macsec_rule_attrs *attrs, u32 *fs_id) 803 { 804 char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN]; 805 struct mlx5_pkt_reformat_params reformat_params = {}; 806 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 807 struct mlx5_core_dev *mdev = macsec_fs->mdev; 808 union mlx5_macsec_rule *macsec_rule = NULL; 809 struct mlx5_flow_destination dest = {}; 810 struct mlx5_macsec_tables *tx_tables; 811 struct mlx5_macsec_tx_rule *tx_rule; 812 struct mlx5_flow_act flow_act = {}; 813 struct mlx5_flow_handle *rule; 814 struct mlx5_flow_spec *spec; 815 size_t reformat_size; 816 int err = 0; 817 818 tx_tables = &tx_fs->tables; 819 820 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 821 if (!spec) 822 return NULL; 823 824 err = macsec_fs_tx_ft_get(macsec_fs); 825 if (err) 826 goto out_spec; 827 828 macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); 829 if (!macsec_rule) { 830 macsec_fs_tx_ft_put(macsec_fs); 831 goto out_spec; 832 } 833 834 tx_rule = &macsec_rule->tx_rule; 835 836 /* Tx crypto table crypto rule */ 837 macsec_fs_tx_create_sectag_header(macsec_ctx, reformatbf, &reformat_size); 838 839 reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC; 840 reformat_params.size = reformat_size; 841 reformat_params.data = reformatbf; 842 843 if (is_vlan_dev(macsec_ctx->netdev)) 844 reformat_params.param_0 = MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES; 845 846 flow_act.pkt_reformat = mlx5_packet_reformat_alloc(mdev, 847 &reformat_params, 848 MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); 849 if (IS_ERR(flow_act.pkt_reformat)) { 850 err = PTR_ERR(flow_act.pkt_reformat); 851 mlx5_core_err(mdev, "Failed to allocate MACsec Tx reformat context err=%d\n", err); 852 goto err; 853 } 854 tx_rule->pkt_reformat = flow_act.pkt_reformat; 855 856 err = macsec_fs_tx_setup_fte(macsec_fs, spec, &flow_act, attrs->macsec_obj_id, fs_id); 857 if (err) { 858 mlx5_core_err(mdev, 859 "Failed to add packet reformat for MACsec TX crypto rule, err=%d\n", 860 err); 861 goto err; 862 } 863 864 tx_rule->fs_id = *fs_id; 865 866 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 867 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 868 MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 869 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 870 dest.ft = tx_tables->ft_check; 871 rule = mlx5_add_flow_rules(tx_tables->ft_crypto.t, spec, &flow_act, &dest, 1); 872 if (IS_ERR(rule)) { 873 err = PTR_ERR(rule); 874 mlx5_core_err(mdev, "Failed to add MACsec TX crypto rule, err=%d\n", err); 875 goto err; 876 } 877 tx_rule->rule = rule; 878 879 err = macsec_fs_id_add(&macsec_fs->macsec_devices_list, *fs_id, macsec_ctx->secy->netdev, 880 &macsec_fs->sci_hash, attrs->sci, true); 881 if (err) { 882 mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err); 883 goto err; 884 } 885 886 goto out_spec; 887 888 err: 889 macsec_fs_tx_del_rule(macsec_fs, tx_rule, macsec_ctx->secy->netdev); 890 macsec_rule = NULL; 891 out_spec: 892 kvfree(spec); 893 894 return macsec_rule; 895 } 896 897 static void macsec_fs_tx_cleanup(struct mlx5_macsec_fs *macsec_fs) 898 { 899 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 900 struct mlx5_core_dev *mdev = macsec_fs->mdev; 901 struct mlx5_macsec_tables *tx_tables; 902 903 if (!tx_fs) 904 return; 905 906 tx_tables = &tx_fs->tables; 907 if (tx_tables->refcnt) { 908 mlx5_core_err(mdev, 909 "Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n", 910 tx_tables->refcnt); 911 return; 912 } 913 914 ida_destroy(&tx_fs->tx_halloc); 915 916 if (tx_tables->check_miss_rule_counter) { 917 mlx5_fc_destroy(mdev, tx_tables->check_miss_rule_counter); 918 tx_tables->check_miss_rule_counter = NULL; 919 } 920 921 if (tx_tables->check_rule_counter) { 922 mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); 923 tx_tables->check_rule_counter = NULL; 924 } 925 926 kfree(tx_fs); 927 macsec_fs->tx_fs = NULL; 928 } 929 930 static int macsec_fs_tx_init(struct mlx5_macsec_fs *macsec_fs) 931 { 932 struct mlx5_core_dev *mdev = macsec_fs->mdev; 933 struct mlx5_macsec_tables *tx_tables; 934 struct mlx5_macsec_tx *tx_fs; 935 struct mlx5_fc *flow_counter; 936 int err; 937 938 tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL); 939 if (!tx_fs) 940 return -ENOMEM; 941 942 tx_tables = &tx_fs->tables; 943 944 flow_counter = mlx5_fc_create(mdev, false); 945 if (IS_ERR(flow_counter)) { 946 err = PTR_ERR(flow_counter); 947 mlx5_core_err(mdev, 948 "Failed to create MACsec Tx encrypt flow counter, err(%d)\n", 949 err); 950 goto err_encrypt_counter; 951 } 952 tx_tables->check_rule_counter = flow_counter; 953 954 flow_counter = mlx5_fc_create(mdev, false); 955 if (IS_ERR(flow_counter)) { 956 err = PTR_ERR(flow_counter); 957 mlx5_core_err(mdev, 958 "Failed to create MACsec Tx drop flow counter, err(%d)\n", 959 err); 960 goto err_drop_counter; 961 } 962 tx_tables->check_miss_rule_counter = flow_counter; 963 964 ida_init(&tx_fs->tx_halloc); 965 INIT_LIST_HEAD(&macsec_fs->macsec_devices_list); 966 967 macsec_fs->tx_fs = tx_fs; 968 969 return 0; 970 971 err_drop_counter: 972 mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); 973 tx_tables->check_rule_counter = NULL; 974 975 err_encrypt_counter: 976 kfree(tx_fs); 977 macsec_fs->tx_fs = NULL; 978 979 return err; 980 } 981 982 static void macsec_fs_rx_roce_miss_destroy(struct mlx5_macsec_miss *miss) 983 { 984 mlx5_del_flow_rules(miss->rule); 985 mlx5_destroy_flow_group(miss->g); 986 } 987 988 static void macsec_fs_rdma_rx_destroy(struct mlx5_macsec_rx_roce *roce, struct mlx5_core_dev *mdev) 989 { 990 if (!mlx5_is_macsec_roce_supported(mdev)) 991 return; 992 993 mlx5_del_flow_rules(roce->nic_miss.rule); 994 mlx5_del_flow_rules(roce->rule); 995 mlx5_modify_header_dealloc(mdev, roce->copy_modify_hdr); 996 mlx5_destroy_flow_group(roce->nic_miss.g); 997 mlx5_destroy_flow_group(roce->g); 998 mlx5_destroy_flow_table(roce->ft); 999 1000 macsec_fs_rx_roce_miss_destroy(&roce->miss); 1001 mlx5_destroy_flow_table(roce->ft_macsec_op_check); 1002 mlx5_destroy_flow_table(roce->ft_ip_check); 1003 } 1004 1005 static void macsec_fs_rx_destroy(struct mlx5_macsec_fs *macsec_fs) 1006 { 1007 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1008 struct mlx5_macsec_tables *rx_tables; 1009 int i; 1010 1011 /* Rx check table */ 1012 for (i = 1; i >= 0; --i) { 1013 if (rx_fs->check_rule[i]) { 1014 mlx5_del_flow_rules(rx_fs->check_rule[i]); 1015 rx_fs->check_rule[i] = NULL; 1016 } 1017 1018 if (rx_fs->check_rule_pkt_reformat[i]) { 1019 mlx5_packet_reformat_dealloc(macsec_fs->mdev, 1020 rx_fs->check_rule_pkt_reformat[i]); 1021 rx_fs->check_rule_pkt_reformat[i] = NULL; 1022 } 1023 } 1024 1025 rx_tables = &rx_fs->tables; 1026 1027 if (rx_tables->check_miss_rule) { 1028 mlx5_del_flow_rules(rx_tables->check_miss_rule); 1029 rx_tables->check_miss_rule = NULL; 1030 } 1031 1032 if (rx_tables->ft_check_group) { 1033 mlx5_destroy_flow_group(rx_tables->ft_check_group); 1034 rx_tables->ft_check_group = NULL; 1035 } 1036 1037 if (rx_tables->ft_check) { 1038 mlx5_destroy_flow_table(rx_tables->ft_check); 1039 rx_tables->ft_check = NULL; 1040 } 1041 1042 /* Rx crypto table */ 1043 if (rx_tables->crypto_miss_rule) { 1044 mlx5_del_flow_rules(rx_tables->crypto_miss_rule); 1045 rx_tables->crypto_miss_rule = NULL; 1046 } 1047 1048 macsec_fs_destroy_flow_table(&rx_tables->ft_crypto); 1049 1050 macsec_fs_rdma_rx_destroy(&macsec_fs->rx_fs->roce, macsec_fs->mdev); 1051 } 1052 1053 static int macsec_fs_rx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft) 1054 { 1055 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1056 int mclen = MLX5_ST_SZ_BYTES(fte_match_param); 1057 int ix = 0; 1058 u32 *in; 1059 int err; 1060 u8 *mc; 1061 1062 ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); 1063 if (!ft->g) 1064 return -ENOMEM; 1065 1066 in = kvzalloc(inlen, GFP_KERNEL); 1067 if (!in) { 1068 kfree(ft->g); 1069 return -ENOMEM; 1070 } 1071 1072 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1073 1074 /* Flow group for SA rule with SCI */ 1075 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | 1076 MLX5_MATCH_MISC_PARAMETERS_5); 1077 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1078 1079 MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, 1080 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << 1081 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1082 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2); 1083 MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3); 1084 1085 MLX5_SET_CFG(in, start_flow_index, ix); 1086 ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE; 1087 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1088 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1089 if (IS_ERR(ft->g[ft->num_groups])) 1090 goto err; 1091 ft->num_groups++; 1092 1093 /* Flow group for SA rule without SCI */ 1094 memset(in, 0, inlen); 1095 memset(mc, 0, mclen); 1096 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | 1097 MLX5_MATCH_MISC_PARAMETERS_5); 1098 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16); 1099 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0); 1100 MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); 1101 1102 MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, 1103 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1104 1105 MLX5_SET_CFG(in, start_flow_index, ix); 1106 ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE; 1107 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1108 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1109 if (IS_ERR(ft->g[ft->num_groups])) 1110 goto err; 1111 ft->num_groups++; 1112 1113 /* Flow Group for l2 traps */ 1114 memset(in, 0, inlen); 1115 memset(mc, 0, mclen); 1116 MLX5_SET_CFG(in, start_flow_index, ix); 1117 ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; 1118 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1119 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); 1120 if (IS_ERR(ft->g[ft->num_groups])) 1121 goto err; 1122 ft->num_groups++; 1123 1124 kvfree(in); 1125 return 0; 1126 1127 err: 1128 err = PTR_ERR(ft->g[ft->num_groups]); 1129 ft->g[ft->num_groups] = NULL; 1130 kvfree(in); 1131 1132 return err; 1133 } 1134 1135 static int macsec_fs_rx_create_check_decap_rule(struct mlx5_macsec_fs *macsec_fs, 1136 struct mlx5_flow_destination *dest, 1137 struct mlx5_flow_act *flow_act, 1138 struct mlx5_flow_spec *spec, 1139 int reformat_param_size) 1140 { 1141 int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1; 1142 u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI]; 1143 struct mlx5_pkt_reformat_params reformat_params = {}; 1144 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1145 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1146 struct mlx5_flow_destination roce_dest[2]; 1147 struct mlx5_macsec_tables *rx_tables; 1148 struct mlx5_flow_handle *rule; 1149 int err = 0, dstn = 0; 1150 1151 rx_tables = &rx_fs->tables; 1152 1153 /* Rx check table decap 16B rule */ 1154 memset(dest, 0, sizeof(*dest)); 1155 memset(flow_act, 0, sizeof(*flow_act)); 1156 memset(spec, 0, sizeof(*spec)); 1157 1158 reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC; 1159 reformat_params.size = reformat_param_size; 1160 reformat_params.data = mlx5_reformat_buf; 1161 flow_act->pkt_reformat = mlx5_packet_reformat_alloc(mdev, 1162 &reformat_params, 1163 MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); 1164 if (IS_ERR(flow_act->pkt_reformat)) { 1165 err = PTR_ERR(flow_act->pkt_reformat); 1166 mlx5_core_err(mdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err); 1167 return err; 1168 } 1169 rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat; 1170 1171 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1172 /* MACsec syndrome match */ 1173 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome); 1174 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0); 1175 /* ASO return reg syndrome match */ 1176 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); 1177 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); 1178 1179 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; 1180 /* Sectag TCI SC present bit*/ 1181 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, 1182 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1183 1184 if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) 1185 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, 1186 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << 1187 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1188 1189 flow_act->flags = FLOW_ACT_NO_APPEND; 1190 1191 if (rx_fs->roce.ft) { 1192 flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1193 roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1194 roce_dest[dstn].ft = rx_fs->roce.ft; 1195 dstn++; 1196 } else { 1197 flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1198 } 1199 1200 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | 1201 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1202 roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1203 roce_dest[dstn].counter_id = mlx5_fc_id(rx_tables->check_rule_counter); 1204 rule = mlx5_add_flow_rules(rx_tables->ft_check, spec, flow_act, roce_dest, dstn + 1); 1205 1206 if (IS_ERR(rule)) { 1207 err = PTR_ERR(rule); 1208 mlx5_core_err(mdev, "Failed to add MACsec Rx check rule, err=%d\n", err); 1209 return err; 1210 } 1211 1212 rx_fs->check_rule[rule_index] = rule; 1213 1214 return 0; 1215 } 1216 1217 static int macsec_fs_rx_roce_miss_create(struct mlx5_core_dev *mdev, 1218 struct mlx5_macsec_rx_roce *roce) 1219 { 1220 struct mlx5_flow_act flow_act = {}; 1221 struct mlx5_flow_group *flow_group; 1222 struct mlx5_flow_handle *rule; 1223 u32 *flow_group_in; 1224 int err; 1225 1226 flow_group_in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 1227 if (!flow_group_in) 1228 return -ENOMEM; 1229 1230 /* IP check ft has no miss rule since we use default miss action which is go to next PRIO */ 1231 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1232 roce->ft_macsec_op_check->max_fte - 1); 1233 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1234 roce->ft_macsec_op_check->max_fte - 1); 1235 flow_group = mlx5_create_flow_group(roce->ft_macsec_op_check, flow_group_in); 1236 if (IS_ERR(flow_group)) { 1237 err = PTR_ERR(flow_group); 1238 mlx5_core_err(mdev, 1239 "Failed to create miss flow group for MACsec RoCE operation check table err(%d)\n", 1240 err); 1241 goto err_macsec_op_miss_group; 1242 } 1243 roce->miss.g = flow_group; 1244 1245 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 1246 rule = mlx5_add_flow_rules(roce->ft_macsec_op_check, NULL, &flow_act, NULL, 0); 1247 if (IS_ERR(rule)) { 1248 err = PTR_ERR(rule); 1249 mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE operation check table err(%d)\n", 1250 err); 1251 goto err_macsec_op_rule; 1252 } 1253 roce->miss.rule = rule; 1254 1255 kvfree(flow_group_in); 1256 return 0; 1257 1258 err_macsec_op_rule: 1259 mlx5_destroy_flow_group(roce->miss.g); 1260 err_macsec_op_miss_group: 1261 kvfree(flow_group_in); 1262 return err; 1263 } 1264 1265 #define MLX5_RX_ROCE_GROUP_SIZE BIT(0) 1266 1267 static int macsec_fs_rx_roce_jump_to_rdma_groups_create(struct mlx5_core_dev *mdev, 1268 struct mlx5_macsec_rx_roce *roce) 1269 { 1270 struct mlx5_flow_group *g; 1271 void *outer_headers_c; 1272 int ix = 0; 1273 u32 *in; 1274 int err; 1275 u8 *mc; 1276 1277 in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); 1278 if (!in) 1279 return -ENOMEM; 1280 1281 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); 1282 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); 1283 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); 1284 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); 1285 1286 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); 1287 MLX5_SET_CFG(in, start_flow_index, ix); 1288 ix += MLX5_RX_ROCE_GROUP_SIZE; 1289 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1290 g = mlx5_create_flow_group(roce->ft, in); 1291 if (IS_ERR(g)) { 1292 err = PTR_ERR(g); 1293 mlx5_core_err(mdev, "Failed to create main flow group for MACsec RoCE NIC UDP table err(%d)\n", 1294 err); 1295 goto err_udp_group; 1296 } 1297 roce->g = g; 1298 1299 memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); 1300 MLX5_SET_CFG(in, start_flow_index, ix); 1301 ix += MLX5_RX_ROCE_GROUP_SIZE; 1302 MLX5_SET_CFG(in, end_flow_index, ix - 1); 1303 g = mlx5_create_flow_group(roce->ft, in); 1304 if (IS_ERR(g)) { 1305 err = PTR_ERR(g); 1306 mlx5_core_err(mdev, "Failed to create miss flow group for MACsec RoCE NIC UDP table err(%d)\n", 1307 err); 1308 goto err_udp_miss_group; 1309 } 1310 roce->nic_miss.g = g; 1311 1312 kvfree(in); 1313 return 0; 1314 1315 err_udp_miss_group: 1316 mlx5_destroy_flow_group(roce->g); 1317 err_udp_group: 1318 kvfree(in); 1319 return err; 1320 } 1321 1322 static int macsec_fs_rx_roce_jump_to_rdma_rules_create(struct mlx5_macsec_fs *macsec_fs, 1323 struct mlx5_macsec_rx_roce *roce) 1324 { 1325 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1326 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1327 struct mlx5_flow_destination dst = {}; 1328 struct mlx5_modify_hdr *modify_hdr; 1329 MLX5_DECLARE_FLOW_ACT(flow_act); 1330 struct mlx5_flow_handle *rule; 1331 struct mlx5_flow_spec *spec; 1332 int err; 1333 1334 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1335 if (!spec) 1336 return -ENOMEM; 1337 1338 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1339 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 1340 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); 1341 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); 1342 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, ROCE_V2_UDP_DPORT); 1343 1344 MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); 1345 MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1346 MLX5_SET(copy_action_in, action, src_offset, 0); 1347 MLX5_SET(copy_action_in, action, length, 32); 1348 MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_5); 1349 MLX5_SET(copy_action_in, action, dst_offset, 0); 1350 1351 modify_hdr = mlx5_modify_header_alloc(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, 1352 1, action); 1353 1354 if (IS_ERR(modify_hdr)) { 1355 err = PTR_ERR(modify_hdr); 1356 mlx5_core_err(mdev, 1357 "Failed to alloc macsec copy modify_header_id err(%d)\n", err); 1358 goto err_alloc_hdr; 1359 } 1360 1361 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1362 flow_act.modify_hdr = modify_hdr; 1363 dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 1364 dst.ft = roce->ft_ip_check; 1365 rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); 1366 if (IS_ERR(rule)) { 1367 err = PTR_ERR(rule); 1368 mlx5_core_err(mdev, "Failed to add rule to MACsec RoCE NIC UDP table err(%d)\n", 1369 err); 1370 goto err_add_rule; 1371 } 1372 roce->rule = rule; 1373 roce->copy_modify_hdr = modify_hdr; 1374 1375 memset(&flow_act, 0, sizeof(flow_act)); 1376 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1377 rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, NULL, 0); 1378 if (IS_ERR(rule)) { 1379 err = PTR_ERR(rule); 1380 mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE NIC UDP table err(%d)\n", 1381 err); 1382 goto err_add_rule2; 1383 } 1384 roce->nic_miss.rule = rule; 1385 1386 kvfree(spec); 1387 return 0; 1388 1389 err_add_rule2: 1390 mlx5_del_flow_rules(roce->rule); 1391 err_add_rule: 1392 mlx5_modify_header_dealloc(macsec_fs->mdev, modify_hdr); 1393 err_alloc_hdr: 1394 kvfree(spec); 1395 return err; 1396 } 1397 1398 static int macsec_fs_rx_roce_jump_to_rdma_create(struct mlx5_macsec_fs *macsec_fs, 1399 struct mlx5_macsec_rx_roce *roce) 1400 { 1401 int err; 1402 1403 err = macsec_fs_rx_roce_jump_to_rdma_groups_create(macsec_fs->mdev, roce); 1404 if (err) 1405 return err; 1406 1407 err = macsec_fs_rx_roce_jump_to_rdma_rules_create(macsec_fs, roce); 1408 if (err) 1409 goto err; 1410 1411 return 0; 1412 err: 1413 mlx5_destroy_flow_group(roce->nic_miss.g); 1414 mlx5_destroy_flow_group(roce->g); 1415 return err; 1416 } 1417 1418 static int macsec_fs_rx_roce_create(struct mlx5_macsec_fs *macsec_fs) 1419 { 1420 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1421 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1422 struct mlx5_flow_table_attr ft_attr = {}; 1423 struct mlx5_flow_namespace *ns; 1424 struct mlx5_flow_table *ft; 1425 int err = 0; 1426 1427 if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) { 1428 mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n"); 1429 return 0; 1430 } 1431 1432 ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_MACSEC); 1433 if (!ns) 1434 return -ENOMEM; 1435 1436 ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_RX_ROCE_IP_TABLE_LEVEL, 1437 CRYPTO_NUM_MAXSEC_FTE); 1438 if (IS_ERR(ft)) { 1439 err = PTR_ERR(ft); 1440 mlx5_core_err(mdev, 1441 "Failed to create MACsec IP check RoCE table err(%d)\n", err); 1442 return err; 1443 } 1444 rx_fs->roce.ft_ip_check = ft; 1445 1446 ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL, 1447 CRYPTO_NUM_MAXSEC_FTE); 1448 if (IS_ERR(ft)) { 1449 err = PTR_ERR(ft); 1450 mlx5_core_err(mdev, 1451 "Failed to create MACsec operation check RoCE table err(%d)\n", 1452 err); 1453 goto err_macsec_op; 1454 } 1455 rx_fs->roce.ft_macsec_op_check = ft; 1456 1457 err = macsec_fs_rx_roce_miss_create(mdev, &rx_fs->roce); 1458 if (err) 1459 goto err_miss_create; 1460 1461 ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); 1462 if (!ns) { 1463 err = -EOPNOTSUPP; 1464 goto err_ns; 1465 } 1466 1467 ft_attr.level = RX_ROCE_TABLE_LEVEL; 1468 ft_attr.max_fte = RX_ROCE_TABLE_NUM_FTE; 1469 ft = mlx5_create_flow_table(ns, &ft_attr); 1470 if (IS_ERR(ft)) { 1471 err = PTR_ERR(ft); 1472 mlx5_core_err(mdev, 1473 "Failed to create MACsec jump to RX RoCE, NIC table err(%d)\n", err); 1474 goto err_ns; 1475 } 1476 rx_fs->roce.ft = ft; 1477 1478 err = macsec_fs_rx_roce_jump_to_rdma_create(macsec_fs, &rx_fs->roce); 1479 if (err) 1480 goto err_udp_ft; 1481 1482 return 0; 1483 1484 err_udp_ft: 1485 mlx5_destroy_flow_table(rx_fs->roce.ft); 1486 err_ns: 1487 macsec_fs_rx_roce_miss_destroy(&rx_fs->roce.miss); 1488 err_miss_create: 1489 mlx5_destroy_flow_table(rx_fs->roce.ft_macsec_op_check); 1490 err_macsec_op: 1491 mlx5_destroy_flow_table(rx_fs->roce.ft_ip_check); 1492 return err; 1493 } 1494 1495 static int macsec_fs_rx_create(struct mlx5_macsec_fs *macsec_fs) 1496 { 1497 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1498 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1499 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1500 struct mlx5_macsec_flow_table *ft_crypto; 1501 struct mlx5_flow_table_attr ft_attr = {}; 1502 struct mlx5_flow_destination dest = {}; 1503 struct mlx5_macsec_tables *rx_tables; 1504 struct mlx5_flow_table *flow_table; 1505 struct mlx5_flow_group *flow_group; 1506 struct mlx5_flow_act flow_act = {}; 1507 struct mlx5_flow_namespace *ns; 1508 struct mlx5_flow_handle *rule; 1509 struct mlx5_flow_spec *spec; 1510 u32 *flow_group_in; 1511 int err; 1512 1513 ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); 1514 if (!ns) 1515 return -ENOMEM; 1516 1517 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1518 if (!spec) 1519 return -ENOMEM; 1520 1521 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1522 if (!flow_group_in) { 1523 err = -ENOMEM; 1524 goto free_spec; 1525 } 1526 1527 rx_tables = &rx_fs->tables; 1528 ft_crypto = &rx_tables->ft_crypto; 1529 1530 err = macsec_fs_rx_roce_create(macsec_fs); 1531 if (err) 1532 goto out_flow_group; 1533 1534 /* Rx crypto table */ 1535 ft_attr.level = RX_CRYPTO_TABLE_LEVEL; 1536 ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; 1537 1538 flow_table = mlx5_create_flow_table(ns, &ft_attr); 1539 if (IS_ERR(flow_table)) { 1540 err = PTR_ERR(flow_table); 1541 mlx5_core_err(mdev, "Failed to create MACsec Rx crypto table err(%d)\n", err); 1542 goto err; 1543 } 1544 ft_crypto->t = flow_table; 1545 1546 /* Rx crypto table groups */ 1547 err = macsec_fs_rx_create_crypto_table_groups(ft_crypto); 1548 if (err) { 1549 mlx5_core_err(mdev, 1550 "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", 1551 err); 1552 goto err; 1553 } 1554 1555 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; 1556 rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); 1557 if (IS_ERR(rule)) { 1558 err = PTR_ERR(rule); 1559 mlx5_core_err(mdev, 1560 "Failed to add MACsec Rx crypto table default miss rule %d\n", 1561 err); 1562 goto err; 1563 } 1564 rx_tables->crypto_miss_rule = rule; 1565 1566 /* Rx check table */ 1567 flow_table = macsec_fs_auto_group_table_create(ns, 1568 MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT, 1569 RX_CHECK_TABLE_LEVEL, 1570 RX_CHECK_TABLE_NUM_FTE); 1571 if (IS_ERR(flow_table)) { 1572 err = PTR_ERR(flow_table); 1573 mlx5_core_err(mdev, "Fail to create MACsec RX check table, err(%d)\n", err); 1574 goto err; 1575 } 1576 rx_tables->ft_check = flow_table; 1577 1578 /* Rx check table Default miss group/rule */ 1579 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); 1580 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); 1581 flow_group = mlx5_create_flow_group(rx_tables->ft_check, flow_group_in); 1582 if (IS_ERR(flow_group)) { 1583 err = PTR_ERR(flow_group); 1584 mlx5_core_err(mdev, 1585 "Failed to create default flow group for MACsec Rx check table err(%d)\n", 1586 err); 1587 goto err; 1588 } 1589 rx_tables->ft_check_group = flow_group; 1590 1591 /* Rx check table default drop rule */ 1592 memset(&flow_act, 0, sizeof(flow_act)); 1593 1594 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1595 dest.counter_id = mlx5_fc_id(rx_tables->check_miss_rule_counter); 1596 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 1597 rule = mlx5_add_flow_rules(rx_tables->ft_check, NULL, &flow_act, &dest, 1); 1598 if (IS_ERR(rule)) { 1599 err = PTR_ERR(rule); 1600 mlx5_core_err(mdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err); 1601 goto err; 1602 } 1603 rx_tables->check_miss_rule = rule; 1604 1605 /* Rx check table decap rules */ 1606 err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, 1607 MLX5_SECTAG_HEADER_SIZE_WITH_SCI); 1608 if (err) 1609 goto err; 1610 1611 err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, 1612 MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI); 1613 if (err) 1614 goto err; 1615 1616 goto out_flow_group; 1617 1618 err: 1619 macsec_fs_rx_destroy(macsec_fs); 1620 out_flow_group: 1621 kvfree(flow_group_in); 1622 free_spec: 1623 kvfree(spec); 1624 return err; 1625 } 1626 1627 static int macsec_fs_rx_ft_get(struct mlx5_macsec_fs *macsec_fs) 1628 { 1629 struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; 1630 int err = 0; 1631 1632 if (rx_tables->refcnt) 1633 goto out; 1634 1635 err = macsec_fs_rx_create(macsec_fs); 1636 if (err) 1637 return err; 1638 1639 out: 1640 rx_tables->refcnt++; 1641 return err; 1642 } 1643 1644 static void macsec_fs_rx_ft_put(struct mlx5_macsec_fs *macsec_fs) 1645 { 1646 struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; 1647 1648 if (--rx_tables->refcnt) 1649 return; 1650 1651 macsec_fs_rx_destroy(macsec_fs); 1652 } 1653 1654 static void macsec_fs_rx_del_rule(struct mlx5_macsec_fs *macsec_fs, 1655 struct mlx5_macsec_rx_rule *rx_rule, 1656 void *macdev, u32 fs_id) 1657 { 1658 int i; 1659 1660 macsec_fs_id_del(&macsec_fs->macsec_devices_list, fs_id, macdev, 1661 &macsec_fs->fs_id_hash, false); 1662 1663 for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) { 1664 if (rx_rule->rule[i]) { 1665 mlx5_del_flow_rules(rx_rule->rule[i]); 1666 rx_rule->rule[i] = NULL; 1667 } 1668 } 1669 1670 if (rx_rule->meta_modhdr) { 1671 mlx5_modify_header_dealloc(macsec_fs->mdev, rx_rule->meta_modhdr); 1672 rx_rule->meta_modhdr = NULL; 1673 } 1674 1675 kfree(rx_rule); 1676 1677 macsec_fs_rx_ft_put(macsec_fs); 1678 } 1679 1680 static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec, 1681 struct mlx5_flow_act *flow_act, 1682 struct mlx5_macsec_rule_attrs *attrs, 1683 bool sci_present) 1684 { 1685 u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num; 1686 struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto; 1687 __be32 *sci_p = (__be32 *)(&attrs->sci); 1688 1689 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1690 1691 /* MACsec ethertype */ 1692 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); 1693 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC); 1694 1695 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; 1696 1697 /* Sectag AN + TCI SC present bit*/ 1698 MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, 1699 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1700 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, 1701 tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); 1702 1703 if (sci_present) { 1704 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1705 misc_parameters_5.macsec_tag_2); 1706 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2, 1707 be32_to_cpu(sci_p[0])); 1708 1709 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1710 misc_parameters_5.macsec_tag_3); 1711 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3, 1712 be32_to_cpu(sci_p[1])); 1713 } else { 1714 /* When SCI isn't present in the Sectag, need to match the source */ 1715 /* MAC address only if the SCI contains the default MACsec PORT */ 1716 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16); 1717 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0); 1718 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16), 1719 sci_p, ETH_ALEN); 1720 } 1721 1722 crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; 1723 crypto_params->obj_id = attrs->macsec_obj_id; 1724 } 1725 1726 static union mlx5_macsec_rule * 1727 macsec_fs_rx_add_rule(struct mlx5_macsec_fs *macsec_fs, 1728 const struct macsec_context *macsec_ctx, 1729 struct mlx5_macsec_rule_attrs *attrs, 1730 u32 fs_id) 1731 { 1732 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1733 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1734 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1735 union mlx5_macsec_rule *macsec_rule = NULL; 1736 struct mlx5_modify_hdr *modify_hdr = NULL; 1737 struct mlx5_macsec_flow_table *ft_crypto; 1738 struct mlx5_flow_destination dest = {}; 1739 struct mlx5_macsec_tables *rx_tables; 1740 struct mlx5_macsec_rx_rule *rx_rule; 1741 struct mlx5_flow_act flow_act = {}; 1742 struct mlx5_flow_handle *rule; 1743 struct mlx5_flow_spec *spec; 1744 int err = 0; 1745 1746 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1747 if (!spec) 1748 return NULL; 1749 1750 err = macsec_fs_rx_ft_get(macsec_fs); 1751 if (err) 1752 goto out_spec; 1753 1754 macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); 1755 if (!macsec_rule) { 1756 macsec_fs_rx_ft_put(macsec_fs); 1757 goto out_spec; 1758 } 1759 1760 rx_rule = &macsec_rule->rx_rule; 1761 rx_tables = &rx_fs->tables; 1762 ft_crypto = &rx_tables->ft_crypto; 1763 1764 /* Set bit[31 - 30] macsec marker - 0x01 */ 1765 /* Set bit[15-0] fs id */ 1766 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 1767 MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1768 MLX5_SET(set_action_in, action, data, macsec_fs_set_rx_fs_id(fs_id)); 1769 MLX5_SET(set_action_in, action, offset, 0); 1770 MLX5_SET(set_action_in, action, length, 32); 1771 1772 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, 1773 1, action); 1774 if (IS_ERR(modify_hdr)) { 1775 err = PTR_ERR(modify_hdr); 1776 mlx5_core_err(mdev, "Fail to alloc MACsec set modify_header_id err=%d\n", err); 1777 modify_hdr = NULL; 1778 goto err; 1779 } 1780 rx_rule->meta_modhdr = modify_hdr; 1781 1782 /* Rx crypto table with SCI rule */ 1783 macsec_fs_rx_setup_fte(spec, &flow_act, attrs, true); 1784 1785 flow_act.modify_hdr = modify_hdr; 1786 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1787 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 1788 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1789 1790 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1791 dest.ft = rx_tables->ft_check; 1792 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); 1793 if (IS_ERR(rule)) { 1794 err = PTR_ERR(rule); 1795 mlx5_core_err(mdev, 1796 "Failed to add SA with SCI rule to Rx crypto rule, err=%d\n", 1797 err); 1798 goto err; 1799 } 1800 rx_rule->rule[0] = rule; 1801 1802 /* Rx crypto table without SCI rule */ 1803 if ((cpu_to_be64((__force u64)attrs->sci) & 0xFFFF) == ntohs(MACSEC_PORT_ES)) { 1804 memset(spec, 0, sizeof(struct mlx5_flow_spec)); 1805 memset(&dest, 0, sizeof(struct mlx5_flow_destination)); 1806 memset(&flow_act, 0, sizeof(flow_act)); 1807 1808 macsec_fs_rx_setup_fte(spec, &flow_act, attrs, false); 1809 1810 flow_act.modify_hdr = modify_hdr; 1811 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1812 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 1813 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1814 1815 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1816 dest.ft = rx_tables->ft_check; 1817 rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); 1818 if (IS_ERR(rule)) { 1819 err = PTR_ERR(rule); 1820 mlx5_core_err(mdev, 1821 "Failed to add SA without SCI rule to Rx crypto rule, err=%d\n", 1822 err); 1823 goto err; 1824 } 1825 rx_rule->rule[1] = rule; 1826 } 1827 1828 err = macsec_fs_id_add(&macsec_fs->macsec_devices_list, fs_id, macsec_ctx->secy->netdev, 1829 &macsec_fs->fs_id_hash, attrs->sci, false); 1830 if (err) { 1831 mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err); 1832 goto err; 1833 } 1834 1835 kvfree(spec); 1836 return macsec_rule; 1837 1838 err: 1839 macsec_fs_rx_del_rule(macsec_fs, rx_rule, macsec_ctx->secy->netdev, fs_id); 1840 macsec_rule = NULL; 1841 out_spec: 1842 kvfree(spec); 1843 return macsec_rule; 1844 } 1845 1846 static int macsec_fs_rx_init(struct mlx5_macsec_fs *macsec_fs) 1847 { 1848 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1849 struct mlx5_macsec_tables *rx_tables; 1850 struct mlx5_macsec_rx *rx_fs; 1851 struct mlx5_fc *flow_counter; 1852 int err; 1853 1854 rx_fs = kzalloc(sizeof(*rx_fs), GFP_KERNEL); 1855 if (!rx_fs) 1856 return -ENOMEM; 1857 1858 flow_counter = mlx5_fc_create(mdev, false); 1859 if (IS_ERR(flow_counter)) { 1860 err = PTR_ERR(flow_counter); 1861 mlx5_core_err(mdev, 1862 "Failed to create MACsec Rx encrypt flow counter, err(%d)\n", 1863 err); 1864 goto err_encrypt_counter; 1865 } 1866 1867 rx_tables = &rx_fs->tables; 1868 rx_tables->check_rule_counter = flow_counter; 1869 1870 flow_counter = mlx5_fc_create(mdev, false); 1871 if (IS_ERR(flow_counter)) { 1872 err = PTR_ERR(flow_counter); 1873 mlx5_core_err(mdev, 1874 "Failed to create MACsec Rx drop flow counter, err(%d)\n", 1875 err); 1876 goto err_drop_counter; 1877 } 1878 rx_tables->check_miss_rule_counter = flow_counter; 1879 1880 macsec_fs->rx_fs = rx_fs; 1881 1882 return 0; 1883 1884 err_drop_counter: 1885 mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); 1886 rx_tables->check_rule_counter = NULL; 1887 1888 err_encrypt_counter: 1889 kfree(rx_fs); 1890 macsec_fs->rx_fs = NULL; 1891 1892 return err; 1893 } 1894 1895 static void macsec_fs_rx_cleanup(struct mlx5_macsec_fs *macsec_fs) 1896 { 1897 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 1898 struct mlx5_core_dev *mdev = macsec_fs->mdev; 1899 struct mlx5_macsec_tables *rx_tables; 1900 1901 if (!rx_fs) 1902 return; 1903 1904 rx_tables = &rx_fs->tables; 1905 1906 if (rx_tables->refcnt) { 1907 mlx5_core_err(mdev, 1908 "Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n", 1909 rx_tables->refcnt); 1910 return; 1911 } 1912 1913 if (rx_tables->check_miss_rule_counter) { 1914 mlx5_fc_destroy(mdev, rx_tables->check_miss_rule_counter); 1915 rx_tables->check_miss_rule_counter = NULL; 1916 } 1917 1918 if (rx_tables->check_rule_counter) { 1919 mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); 1920 rx_tables->check_rule_counter = NULL; 1921 } 1922 1923 kfree(rx_fs); 1924 macsec_fs->rx_fs = NULL; 1925 } 1926 1927 static void set_ipaddr_spec_v4(struct sockaddr_in *in, struct mlx5_flow_spec *spec, bool is_dst_ip) 1928 { 1929 MLX5_SET(fte_match_param, spec->match_value, 1930 outer_headers.ip_version, MLX5_FS_IPV4_VERSION); 1931 1932 if (is_dst_ip) { 1933 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1934 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 1935 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1936 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 1937 &in->sin_addr.s_addr, 4); 1938 } else { 1939 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1940 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 1941 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1942 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), 1943 &in->sin_addr.s_addr, 4); 1944 } 1945 } 1946 1947 static void set_ipaddr_spec_v6(struct sockaddr_in6 *in6, struct mlx5_flow_spec *spec, 1948 bool is_dst_ip) 1949 { 1950 MLX5_SET(fte_match_param, spec->match_value, 1951 outer_headers.ip_version, MLX5_FS_IPV6_VERSION); 1952 1953 if (is_dst_ip) { 1954 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1955 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 1956 0xff, 16); 1957 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1958 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 1959 &in6->sin6_addr, 16); 1960 } else { 1961 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1962 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 1963 0xff, 16); 1964 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 1965 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 1966 &in6->sin6_addr, 16); 1967 } 1968 } 1969 1970 static void set_ipaddr_spec(const struct sockaddr *addr, 1971 struct mlx5_flow_spec *spec, bool is_dst_ip) 1972 { 1973 struct sockaddr_in6 *in6; 1974 1975 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1976 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 1977 outer_headers.ip_version); 1978 1979 if (addr->sa_family == AF_INET) { 1980 struct sockaddr_in *in = (struct sockaddr_in *)addr; 1981 1982 set_ipaddr_spec_v4(in, spec, is_dst_ip); 1983 return; 1984 } 1985 1986 in6 = (struct sockaddr_in6 *)addr; 1987 set_ipaddr_spec_v6(in6, spec, is_dst_ip); 1988 } 1989 1990 static void macsec_fs_del_roce_rule_rx(struct mlx5_roce_macsec_rx_rule *rx_rule) 1991 { 1992 mlx5_del_flow_rules(rx_rule->op); 1993 mlx5_del_flow_rules(rx_rule->ip); 1994 list_del(&rx_rule->entry); 1995 kfree(rx_rule); 1996 } 1997 1998 static void macsec_fs_del_roce_rules_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, 1999 struct list_head *rx_rules_list) 2000 { 2001 struct mlx5_roce_macsec_rx_rule *rx_rule, *next; 2002 2003 if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) 2004 return; 2005 2006 list_for_each_entry_safe(rx_rule, next, rx_rules_list, entry) { 2007 if (rx_rule->fs_id == fs_id) 2008 macsec_fs_del_roce_rule_rx(rx_rule); 2009 } 2010 } 2011 2012 static void macsec_fs_del_roce_rule_tx(struct mlx5_core_dev *mdev, 2013 struct mlx5_roce_macsec_tx_rule *tx_rule) 2014 { 2015 mlx5_del_flow_rules(tx_rule->rule); 2016 mlx5_modify_header_dealloc(mdev, tx_rule->meta_modhdr); 2017 list_del(&tx_rule->entry); 2018 kfree(tx_rule); 2019 } 2020 2021 static void macsec_fs_del_roce_rules_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, 2022 struct list_head *tx_rules_list) 2023 { 2024 struct mlx5_roce_macsec_tx_rule *tx_rule, *next; 2025 2026 if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) 2027 return; 2028 2029 list_for_each_entry_safe(tx_rule, next, tx_rules_list, entry) { 2030 if (tx_rule->fs_id == fs_id) 2031 macsec_fs_del_roce_rule_tx(macsec_fs->mdev, tx_rule); 2032 } 2033 } 2034 2035 void mlx5_macsec_fs_get_stats_fill(struct mlx5_macsec_fs *macsec_fs, void *macsec_stats) 2036 { 2037 struct mlx5_macsec_stats *stats = (struct mlx5_macsec_stats *)macsec_stats; 2038 struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; 2039 struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; 2040 struct mlx5_core_dev *mdev = macsec_fs->mdev; 2041 2042 if (tx_tables->check_rule_counter) 2043 mlx5_fc_query(mdev, tx_tables->check_rule_counter, 2044 &stats->macsec_tx_pkts, &stats->macsec_tx_bytes); 2045 2046 if (tx_tables->check_miss_rule_counter) 2047 mlx5_fc_query(mdev, tx_tables->check_miss_rule_counter, 2048 &stats->macsec_tx_pkts_drop, &stats->macsec_tx_bytes_drop); 2049 2050 if (rx_tables->check_rule_counter) 2051 mlx5_fc_query(mdev, rx_tables->check_rule_counter, 2052 &stats->macsec_rx_pkts, &stats->macsec_rx_bytes); 2053 2054 if (rx_tables->check_miss_rule_counter) 2055 mlx5_fc_query(mdev, rx_tables->check_miss_rule_counter, 2056 &stats->macsec_rx_pkts_drop, &stats->macsec_rx_bytes_drop); 2057 } 2058 2059 struct mlx5_macsec_stats *mlx5_macsec_fs_get_stats(struct mlx5_macsec_fs *macsec_fs) 2060 { 2061 if (!macsec_fs) 2062 return NULL; 2063 2064 return &macsec_fs->stats; 2065 } 2066 2067 u32 mlx5_macsec_fs_get_fs_id_from_hashtable(struct mlx5_macsec_fs *macsec_fs, sci_t *sci) 2068 { 2069 struct mlx5_fs_id *mlx5_fs_id; 2070 u32 fs_id = 0; 2071 2072 rcu_read_lock(); 2073 mlx5_fs_id = rhashtable_lookup(&macsec_fs->sci_hash, sci, rhash_sci); 2074 if (mlx5_fs_id) 2075 fs_id = mlx5_fs_id->id; 2076 rcu_read_unlock(); 2077 2078 return fs_id; 2079 } 2080 2081 union mlx5_macsec_rule * 2082 mlx5_macsec_fs_add_rule(struct mlx5_macsec_fs *macsec_fs, 2083 const struct macsec_context *macsec_ctx, 2084 struct mlx5_macsec_rule_attrs *attrs, 2085 u32 *sa_fs_id) 2086 { 2087 struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs, 2088 .macdev = macsec_ctx->secy->netdev, 2089 .is_tx = 2090 (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) 2091 }; 2092 union mlx5_macsec_rule *macsec_rule; 2093 u32 tx_new_fs_id; 2094 2095 macsec_rule = (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? 2096 macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, &tx_new_fs_id) : 2097 macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id); 2098 2099 data.fs_id = (data.is_tx) ? tx_new_fs_id : *sa_fs_id; 2100 if (macsec_rule) 2101 blocking_notifier_call_chain(&macsec_fs->mdev->macsec_nh, 2102 MLX5_DRIVER_EVENT_MACSEC_SA_ADDED, 2103 &data); 2104 2105 return macsec_rule; 2106 } 2107 2108 void mlx5_macsec_fs_del_rule(struct mlx5_macsec_fs *macsec_fs, 2109 union mlx5_macsec_rule *macsec_rule, 2110 int action, void *macdev, u32 sa_fs_id) 2111 { 2112 struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs, 2113 .macdev = macdev, 2114 .is_tx = (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) 2115 }; 2116 2117 data.fs_id = (data.is_tx) ? macsec_rule->tx_rule.fs_id : sa_fs_id; 2118 blocking_notifier_call_chain(&macsec_fs->mdev->macsec_nh, 2119 MLX5_DRIVER_EVENT_MACSEC_SA_DELETED, 2120 &data); 2121 2122 (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? 2123 macsec_fs_tx_del_rule(macsec_fs, &macsec_rule->tx_rule, macdev) : 2124 macsec_fs_rx_del_rule(macsec_fs, &macsec_rule->rx_rule, macdev, sa_fs_id); 2125 } 2126 2127 static int mlx5_macsec_fs_add_roce_rule_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx, 2128 const struct sockaddr *addr, 2129 struct list_head *rx_rules_list) 2130 { 2131 struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; 2132 struct mlx5_roce_macsec_rx_rule *rx_rule; 2133 struct mlx5_flow_destination dest = {}; 2134 struct mlx5_flow_act flow_act = {}; 2135 struct mlx5_flow_handle *new_rule; 2136 struct mlx5_flow_spec *spec; 2137 int err = 0; 2138 2139 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2140 if (!spec) 2141 return -ENOMEM; 2142 2143 rx_rule = kzalloc(sizeof(*rx_rule), GFP_KERNEL); 2144 if (!rx_rule) { 2145 err = -ENOMEM; 2146 goto out; 2147 } 2148 2149 set_ipaddr_spec(addr, spec, true); 2150 2151 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2152 dest.ft = rx_fs->roce.ft_macsec_op_check; 2153 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 2154 new_rule = mlx5_add_flow_rules(rx_fs->roce.ft_ip_check, spec, &flow_act, 2155 &dest, 1); 2156 if (IS_ERR(new_rule)) { 2157 err = PTR_ERR(new_rule); 2158 goto ip_rule_err; 2159 } 2160 rx_rule->ip = new_rule; 2161 2162 memset(&flow_act, 0, sizeof(flow_act)); 2163 memset(spec, 0, sizeof(*spec)); 2164 2165 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 2166 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_5); 2167 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_5, 2168 macsec_fs_set_rx_fs_id(fs_id)); 2169 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; 2170 new_rule = mlx5_add_flow_rules(rx_fs->roce.ft_macsec_op_check, spec, &flow_act, 2171 NULL, 0); 2172 if (IS_ERR(new_rule)) { 2173 err = PTR_ERR(new_rule); 2174 goto op_rule_err; 2175 } 2176 rx_rule->op = new_rule; 2177 rx_rule->gid_idx = gid_idx; 2178 rx_rule->fs_id = fs_id; 2179 list_add_tail(&rx_rule->entry, rx_rules_list); 2180 2181 goto out; 2182 2183 op_rule_err: 2184 mlx5_del_flow_rules(rx_rule->ip); 2185 rx_rule->ip = NULL; 2186 ip_rule_err: 2187 kfree(rx_rule); 2188 out: 2189 kvfree(spec); 2190 return err; 2191 } 2192 2193 static int mlx5_macsec_fs_add_roce_rule_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx, 2194 const struct sockaddr *addr, 2195 struct list_head *tx_rules_list) 2196 { 2197 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2198 struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; 2199 struct mlx5_core_dev *mdev = macsec_fs->mdev; 2200 struct mlx5_modify_hdr *modify_hdr = NULL; 2201 struct mlx5_roce_macsec_tx_rule *tx_rule; 2202 struct mlx5_flow_destination dest = {}; 2203 struct mlx5_flow_act flow_act = {}; 2204 struct mlx5_flow_handle *new_rule; 2205 struct mlx5_flow_spec *spec; 2206 int err = 0; 2207 2208 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2209 if (!spec) 2210 return -ENOMEM; 2211 2212 tx_rule = kzalloc(sizeof(*tx_rule), GFP_KERNEL); 2213 if (!tx_rule) { 2214 err = -ENOMEM; 2215 goto out; 2216 } 2217 2218 set_ipaddr_spec(addr, spec, false); 2219 2220 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 2221 MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_A); 2222 MLX5_SET(set_action_in, action, data, macsec_fs_set_tx_fs_id(fs_id)); 2223 MLX5_SET(set_action_in, action, offset, 0); 2224 MLX5_SET(set_action_in, action, length, 32); 2225 2226 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC, 2227 1, action); 2228 if (IS_ERR(modify_hdr)) { 2229 err = PTR_ERR(modify_hdr); 2230 mlx5_core_err(mdev, "Fail to alloc ROCE MACsec set modify_header_id err=%d\n", 2231 err); 2232 modify_hdr = NULL; 2233 goto modify_hdr_err; 2234 } 2235 tx_rule->meta_modhdr = modify_hdr; 2236 2237 flow_act.modify_hdr = modify_hdr; 2238 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 2239 2240 dest.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; 2241 dest.ft = tx_fs->tables.ft_crypto.t; 2242 new_rule = mlx5_add_flow_rules(tx_fs->ft_rdma_tx, spec, &flow_act, &dest, 1); 2243 if (IS_ERR(new_rule)) { 2244 err = PTR_ERR(new_rule); 2245 mlx5_core_err(mdev, "Failed to add ROCE TX rule, err=%d\n", err); 2246 goto rule_err; 2247 } 2248 tx_rule->rule = new_rule; 2249 tx_rule->gid_idx = gid_idx; 2250 tx_rule->fs_id = fs_id; 2251 list_add_tail(&tx_rule->entry, tx_rules_list); 2252 2253 goto out; 2254 2255 rule_err: 2256 mlx5_modify_header_dealloc(mdev, tx_rule->meta_modhdr); 2257 modify_hdr_err: 2258 kfree(tx_rule); 2259 out: 2260 kvfree(spec); 2261 return err; 2262 } 2263 2264 void mlx5_macsec_del_roce_rule(u16 gid_idx, struct mlx5_macsec_fs *macsec_fs, 2265 struct list_head *tx_rules_list, struct list_head *rx_rules_list) 2266 { 2267 struct mlx5_roce_macsec_rx_rule *rx_rule, *next_rx; 2268 struct mlx5_roce_macsec_tx_rule *tx_rule, *next_tx; 2269 2270 list_for_each_entry_safe(tx_rule, next_tx, tx_rules_list, entry) { 2271 if (tx_rule->gid_idx == gid_idx) 2272 macsec_fs_del_roce_rule_tx(macsec_fs->mdev, tx_rule); 2273 } 2274 2275 list_for_each_entry_safe(rx_rule, next_rx, rx_rules_list, entry) { 2276 if (rx_rule->gid_idx == gid_idx) 2277 macsec_fs_del_roce_rule_rx(rx_rule); 2278 } 2279 } 2280 EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_rule); 2281 2282 int mlx5_macsec_add_roce_rule(void *macdev, const struct sockaddr *addr, u16 gid_idx, 2283 struct list_head *tx_rules_list, struct list_head *rx_rules_list, 2284 struct mlx5_macsec_fs *macsec_fs) 2285 { 2286 struct mlx5_macsec_device *iter, *macsec_device = NULL; 2287 struct mlx5_core_dev *mdev = macsec_fs->mdev; 2288 struct mlx5_fs_id *fs_id_iter; 2289 unsigned long index = 0; 2290 int err; 2291 2292 list_for_each_entry(iter, &macsec_fs->macsec_devices_list, macsec_devices_list_entry) { 2293 if (iter->macdev == macdev) { 2294 macsec_device = iter; 2295 break; 2296 } 2297 } 2298 2299 if (!macsec_device) 2300 return 0; 2301 2302 xa_for_each(&macsec_device->tx_id_xa, index, fs_id_iter) { 2303 err = mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id_iter->id, gid_idx, addr, 2304 tx_rules_list); 2305 if (err) { 2306 mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n"); 2307 goto out; 2308 } 2309 } 2310 2311 index = 0; 2312 xa_for_each(&macsec_device->rx_id_xa, index, fs_id_iter) { 2313 err = mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id_iter->id, gid_idx, addr, 2314 rx_rules_list); 2315 if (err) { 2316 mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n"); 2317 goto out; 2318 } 2319 } 2320 2321 return 0; 2322 out: 2323 mlx5_macsec_del_roce_rule(gid_idx, macsec_fs, tx_rules_list, rx_rules_list); 2324 return err; 2325 } 2326 EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_rule); 2327 2328 void mlx5_macsec_add_roce_sa_rules(u32 fs_id, const struct sockaddr *addr, u16 gid_idx, 2329 struct list_head *tx_rules_list, 2330 struct list_head *rx_rules_list, 2331 struct mlx5_macsec_fs *macsec_fs, bool is_tx) 2332 { 2333 (is_tx) ? 2334 mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id, gid_idx, addr, 2335 tx_rules_list) : 2336 mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id, gid_idx, addr, 2337 rx_rules_list); 2338 } 2339 EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_sa_rules); 2340 2341 void mlx5_macsec_del_roce_sa_rules(u32 fs_id, struct mlx5_macsec_fs *macsec_fs, 2342 struct list_head *tx_rules_list, 2343 struct list_head *rx_rules_list, bool is_tx) 2344 { 2345 (is_tx) ? 2346 macsec_fs_del_roce_rules_tx(macsec_fs, fs_id, tx_rules_list) : 2347 macsec_fs_del_roce_rules_rx(macsec_fs, fs_id, rx_rules_list); 2348 } 2349 EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_sa_rules); 2350 2351 void mlx5_macsec_fs_cleanup(struct mlx5_macsec_fs *macsec_fs) 2352 { 2353 macsec_fs_rx_cleanup(macsec_fs); 2354 macsec_fs_tx_cleanup(macsec_fs); 2355 rhashtable_destroy(&macsec_fs->fs_id_hash); 2356 rhashtable_destroy(&macsec_fs->sci_hash); 2357 kfree(macsec_fs); 2358 } 2359 2360 struct mlx5_macsec_fs * 2361 mlx5_macsec_fs_init(struct mlx5_core_dev *mdev) 2362 { 2363 struct mlx5_macsec_fs *macsec_fs; 2364 int err; 2365 2366 macsec_fs = kzalloc(sizeof(*macsec_fs), GFP_KERNEL); 2367 if (!macsec_fs) 2368 return NULL; 2369 2370 macsec_fs->mdev = mdev; 2371 2372 err = rhashtable_init(&macsec_fs->sci_hash, &rhash_sci); 2373 if (err) { 2374 mlx5_core_err(mdev, "MACsec offload: Failed to init SCI hash table, err=%d\n", 2375 err); 2376 goto err_hash; 2377 } 2378 2379 err = rhashtable_init(&macsec_fs->fs_id_hash, &rhash_fs_id); 2380 if (err) { 2381 mlx5_core_err(mdev, "MACsec offload: Failed to init FS_ID hash table, err=%d\n", 2382 err); 2383 goto sci_hash_cleanup; 2384 } 2385 2386 err = macsec_fs_tx_init(macsec_fs); 2387 if (err) { 2388 mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); 2389 goto fs_id_hash_cleanup; 2390 } 2391 2392 err = macsec_fs_rx_init(macsec_fs); 2393 if (err) { 2394 mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); 2395 goto tx_cleanup; 2396 } 2397 2398 BLOCKING_INIT_NOTIFIER_HEAD(&mdev->macsec_nh); 2399 2400 return macsec_fs; 2401 2402 tx_cleanup: 2403 macsec_fs_tx_cleanup(macsec_fs); 2404 fs_id_hash_cleanup: 2405 rhashtable_destroy(&macsec_fs->fs_id_hash); 2406 sci_hash_cleanup: 2407 rhashtable_destroy(&macsec_fs->sci_hash); 2408 err_hash: 2409 kfree(macsec_fs); 2410 return NULL; 2411 } 2412