1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4 #include <linux/netdevice.h> 5 #include "en.h" 6 #include "en/fs.h" 7 #include "eswitch.h" 8 #include "ipsec.h" 9 #include "fs_core.h" 10 #include "lib/ipsec_fs_roce.h" 11 #include "lib/fs_chains.h" 12 13 #define NUM_IPSEC_FTE BIT(15) 14 #define MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE 16 15 #define IPSEC_TUNNEL_DEFAULT_TTL 0x40 16 17 struct mlx5e_ipsec_fc { 18 struct mlx5_fc *cnt; 19 struct mlx5_fc *drop; 20 }; 21 22 struct mlx5e_ipsec_ft { 23 struct mutex mutex; /* Protect changes to this struct */ 24 struct mlx5_flow_table *pol; 25 struct mlx5_flow_table *sa; 26 struct mlx5_flow_table *status; 27 u32 refcnt; 28 }; 29 30 struct mlx5e_ipsec_miss { 31 struct mlx5_flow_group *group; 32 struct mlx5_flow_handle *rule; 33 }; 34 35 struct mlx5e_ipsec_rx { 36 struct mlx5e_ipsec_ft ft; 37 struct mlx5e_ipsec_miss pol; 38 struct mlx5e_ipsec_miss sa; 39 struct mlx5e_ipsec_rule status; 40 struct mlx5e_ipsec_fc *fc; 41 struct mlx5_fs_chains *chains; 42 u8 allow_tunnel_mode : 1; 43 }; 44 45 struct mlx5e_ipsec_tx { 46 struct mlx5e_ipsec_ft ft; 47 struct mlx5e_ipsec_miss pol; 48 struct mlx5e_ipsec_rule status; 49 struct mlx5_flow_namespace *ns; 50 struct mlx5e_ipsec_fc *fc; 51 struct mlx5_fs_chains *chains; 52 u8 allow_tunnel_mode : 1; 53 }; 54 55 /* IPsec RX flow steering */ 56 static enum mlx5_traffic_types family2tt(u32 family) 57 { 58 if (family == AF_INET) 59 return MLX5_TT_IPV4_IPSEC_ESP; 60 return MLX5_TT_IPV6_IPSEC_ESP; 61 } 62 63 static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family) 64 { 65 if (family == AF_INET) 66 return ipsec->rx_ipv4; 67 68 return ipsec->rx_ipv6; 69 } 70 71 static struct mlx5_fs_chains * 72 ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, 73 enum mlx5_flow_namespace_type ns, int base_prio, 74 int base_level, struct mlx5_flow_table **root_ft) 75 { 76 struct mlx5_chains_attr attr = {}; 77 struct mlx5_fs_chains *chains; 78 struct mlx5_flow_table *ft; 79 int err; 80 81 attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED | 82 MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 83 attr.max_grp_num = 2; 84 attr.default_ft = miss_ft; 85 attr.ns = ns; 86 attr.fs_base_prio = base_prio; 87 attr.fs_base_level = base_level; 88 chains = mlx5_chains_create(mdev, &attr); 89 if (IS_ERR(chains)) 90 return chains; 91 92 /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */ 93 ft = mlx5_chains_get_table(chains, 0, 1, 0); 94 if (IS_ERR(ft)) { 95 err = PTR_ERR(ft); 96 goto err_chains_get; 97 } 98 99 *root_ft = ft; 100 return chains; 101 102 err_chains_get: 103 mlx5_chains_destroy(chains); 104 return ERR_PTR(err); 105 } 106 107 static void ipsec_chains_destroy(struct mlx5_fs_chains *chains) 108 { 109 mlx5_chains_put_table(chains, 0, 1, 0); 110 mlx5_chains_destroy(chains); 111 } 112 113 static struct mlx5_flow_table * 114 ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio) 115 { 116 return mlx5_chains_get_table(chains, 0, prio + 1, 0); 117 } 118 119 static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio) 120 { 121 mlx5_chains_put_table(chains, 0, prio + 1, 0); 122 } 123 124 static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, 125 int level, int prio, 126 int max_num_groups, u32 flags) 127 { 128 struct mlx5_flow_table_attr ft_attr = {}; 129 130 ft_attr.autogroup.num_reserved_entries = 1; 131 ft_attr.autogroup.max_num_groups = max_num_groups; 132 ft_attr.max_fte = NUM_IPSEC_FTE; 133 ft_attr.level = level; 134 ft_attr.prio = prio; 135 ft_attr.flags = flags; 136 137 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 138 } 139 140 static int ipsec_status_rule(struct mlx5_core_dev *mdev, 141 struct mlx5e_ipsec_rx *rx, 142 struct mlx5_flow_destination *dest) 143 { 144 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 145 struct mlx5_flow_act flow_act = {}; 146 struct mlx5_modify_hdr *modify_hdr; 147 struct mlx5_flow_handle *fte; 148 struct mlx5_flow_spec *spec; 149 int err; 150 151 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 152 if (!spec) 153 return -ENOMEM; 154 155 /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */ 156 MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); 157 MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME); 158 MLX5_SET(copy_action_in, action, src_offset, 0); 159 MLX5_SET(copy_action_in, action, length, 7); 160 MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 161 MLX5_SET(copy_action_in, action, dst_offset, 24); 162 163 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 164 1, action); 165 166 if (IS_ERR(modify_hdr)) { 167 err = PTR_ERR(modify_hdr); 168 mlx5_core_err(mdev, 169 "fail to alloc ipsec copy modify_header_id err=%d\n", err); 170 goto out_spec; 171 } 172 173 /* create fte */ 174 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 175 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 176 MLX5_FLOW_CONTEXT_ACTION_COUNT; 177 flow_act.modify_hdr = modify_hdr; 178 fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 179 if (IS_ERR(fte)) { 180 err = PTR_ERR(fte); 181 mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); 182 goto out; 183 } 184 185 kvfree(spec); 186 rx->status.rule = fte; 187 rx->status.modify_hdr = modify_hdr; 188 return 0; 189 190 out: 191 mlx5_modify_header_dealloc(mdev, modify_hdr); 192 out_spec: 193 kvfree(spec); 194 return err; 195 } 196 197 static int ipsec_miss_create(struct mlx5_core_dev *mdev, 198 struct mlx5_flow_table *ft, 199 struct mlx5e_ipsec_miss *miss, 200 struct mlx5_flow_destination *dest) 201 { 202 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 203 MLX5_DECLARE_FLOW_ACT(flow_act); 204 struct mlx5_flow_spec *spec; 205 u32 *flow_group_in; 206 int err = 0; 207 208 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 209 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 210 if (!flow_group_in || !spec) { 211 err = -ENOMEM; 212 goto out; 213 } 214 215 /* Create miss_group */ 216 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 217 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 218 miss->group = mlx5_create_flow_group(ft, flow_group_in); 219 if (IS_ERR(miss->group)) { 220 err = PTR_ERR(miss->group); 221 mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", 222 err); 223 goto out; 224 } 225 226 /* Create miss rule */ 227 miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); 228 if (IS_ERR(miss->rule)) { 229 mlx5_destroy_flow_group(miss->group); 230 err = PTR_ERR(miss->rule); 231 mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", 232 err); 233 goto out; 234 } 235 out: 236 kvfree(flow_group_in); 237 kvfree(spec); 238 return err; 239 } 240 241 static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 242 struct mlx5e_ipsec_rx *rx, u32 family) 243 { 244 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 245 246 /* disconnect */ 247 mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); 248 249 if (rx->chains) { 250 ipsec_chains_destroy(rx->chains); 251 } else { 252 mlx5_del_flow_rules(rx->pol.rule); 253 mlx5_destroy_flow_group(rx->pol.group); 254 mlx5_destroy_flow_table(rx->ft.pol); 255 } 256 257 mlx5_del_flow_rules(rx->sa.rule); 258 mlx5_destroy_flow_group(rx->sa.group); 259 mlx5_destroy_flow_table(rx->ft.sa); 260 if (rx->allow_tunnel_mode) 261 mlx5_eswitch_unblock_encap(mdev); 262 mlx5_del_flow_rules(rx->status.rule); 263 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 264 mlx5_destroy_flow_table(rx->ft.status); 265 266 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family); 267 } 268 269 static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 270 struct mlx5e_ipsec_rx *rx, u32 family) 271 { 272 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); 273 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 274 struct mlx5_flow_destination default_dest; 275 struct mlx5_flow_destination dest[2]; 276 struct mlx5_flow_table *ft; 277 u32 flags = 0; 278 int err; 279 280 default_dest = mlx5_ttc_get_default_dest(ttc, family2tt(family)); 281 err = mlx5_ipsec_fs_roce_rx_create(mdev, ipsec->roce, ns, &default_dest, 282 family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, 283 MLX5E_NIC_PRIO); 284 if (err) 285 return err; 286 287 ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, 288 MLX5E_NIC_PRIO, 1, 0); 289 if (IS_ERR(ft)) { 290 err = PTR_ERR(ft); 291 goto err_fs_ft_status; 292 } 293 294 rx->ft.status = ft; 295 296 ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); 297 if (ft) { 298 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 299 dest[0].ft = ft; 300 } else { 301 dest[0] = default_dest; 302 } 303 304 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 305 dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); 306 err = ipsec_status_rule(mdev, rx, dest); 307 if (err) 308 goto err_add; 309 310 /* Create FT */ 311 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 312 rx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); 313 if (rx->allow_tunnel_mode) 314 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 315 ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, 2, 316 flags); 317 if (IS_ERR(ft)) { 318 err = PTR_ERR(ft); 319 goto err_fs_ft; 320 } 321 rx->ft.sa = ft; 322 323 err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, dest); 324 if (err) 325 goto err_fs; 326 327 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 328 rx->chains = ipsec_chains_create(mdev, rx->ft.sa, 329 MLX5_FLOW_NAMESPACE_KERNEL, 330 MLX5E_NIC_PRIO, 331 MLX5E_ACCEL_FS_POL_FT_LEVEL, 332 &rx->ft.pol); 333 if (IS_ERR(rx->chains)) { 334 err = PTR_ERR(rx->chains); 335 goto err_pol_ft; 336 } 337 338 goto connect; 339 } 340 341 ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, 342 2, 0); 343 if (IS_ERR(ft)) { 344 err = PTR_ERR(ft); 345 goto err_pol_ft; 346 } 347 rx->ft.pol = ft; 348 memset(dest, 0x00, 2 * sizeof(*dest)); 349 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 350 dest[0].ft = rx->ft.sa; 351 err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, dest); 352 if (err) 353 goto err_pol_miss; 354 355 connect: 356 /* connect */ 357 memset(dest, 0x00, sizeof(*dest)); 358 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 359 dest[0].ft = rx->ft.pol; 360 mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest[0]); 361 return 0; 362 363 err_pol_miss: 364 mlx5_destroy_flow_table(rx->ft.pol); 365 err_pol_ft: 366 mlx5_del_flow_rules(rx->sa.rule); 367 mlx5_destroy_flow_group(rx->sa.group); 368 err_fs: 369 mlx5_destroy_flow_table(rx->ft.sa); 370 err_fs_ft: 371 if (rx->allow_tunnel_mode) 372 mlx5_eswitch_unblock_encap(mdev); 373 mlx5_del_flow_rules(rx->status.rule); 374 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 375 err_add: 376 mlx5_destroy_flow_table(rx->ft.status); 377 err_fs_ft_status: 378 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family); 379 return err; 380 } 381 382 static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 383 struct mlx5e_ipsec_rx *rx, u32 family) 384 { 385 int err; 386 387 if (rx->ft.refcnt) 388 goto skip; 389 390 err = rx_create(mdev, ipsec, rx, family); 391 if (err) 392 return err; 393 394 skip: 395 rx->ft.refcnt++; 396 return 0; 397 } 398 399 static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, 400 u32 family) 401 { 402 if (--rx->ft.refcnt) 403 return; 404 405 rx_destroy(ipsec->mdev, ipsec, rx, family); 406 } 407 408 static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, 409 struct mlx5e_ipsec *ipsec, u32 family) 410 { 411 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); 412 int err; 413 414 mutex_lock(&rx->ft.mutex); 415 err = rx_get(mdev, ipsec, rx, family); 416 mutex_unlock(&rx->ft.mutex); 417 if (err) 418 return ERR_PTR(err); 419 420 return rx; 421 } 422 423 static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev, 424 struct mlx5e_ipsec *ipsec, 425 u32 family, u32 prio) 426 { 427 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); 428 struct mlx5_flow_table *ft; 429 int err; 430 431 mutex_lock(&rx->ft.mutex); 432 err = rx_get(mdev, ipsec, rx, family); 433 if (err) 434 goto err_get; 435 436 ft = rx->chains ? ipsec_chains_get_table(rx->chains, prio) : rx->ft.pol; 437 if (IS_ERR(ft)) { 438 err = PTR_ERR(ft); 439 goto err_get_ft; 440 } 441 442 mutex_unlock(&rx->ft.mutex); 443 return ft; 444 445 err_get_ft: 446 rx_put(ipsec, rx, family); 447 err_get: 448 mutex_unlock(&rx->ft.mutex); 449 return ERR_PTR(err); 450 } 451 452 static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family) 453 { 454 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); 455 456 mutex_lock(&rx->ft.mutex); 457 rx_put(ipsec, rx, family); 458 mutex_unlock(&rx->ft.mutex); 459 } 460 461 static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio) 462 { 463 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family); 464 465 mutex_lock(&rx->ft.mutex); 466 if (rx->chains) 467 ipsec_chains_put_table(rx->chains, prio); 468 469 rx_put(ipsec, rx, family); 470 mutex_unlock(&rx->ft.mutex); 471 } 472 473 static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 474 { 475 struct mlx5_flow_destination dest = {}; 476 struct mlx5_flow_act flow_act = {}; 477 struct mlx5_flow_handle *fte; 478 struct mlx5_flow_spec *spec; 479 int err; 480 481 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 482 if (!spec) 483 return -ENOMEM; 484 485 /* create fte */ 486 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | 487 MLX5_FLOW_CONTEXT_ACTION_COUNT; 488 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 489 dest.counter_id = mlx5_fc_id(tx->fc->cnt); 490 fte = mlx5_add_flow_rules(tx->ft.status, spec, &flow_act, &dest, 1); 491 if (IS_ERR(fte)) { 492 err = PTR_ERR(fte); 493 mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err); 494 goto err_rule; 495 } 496 497 kvfree(spec); 498 tx->status.rule = fte; 499 return 0; 500 501 err_rule: 502 kvfree(spec); 503 return err; 504 } 505 506 /* IPsec TX flow steering */ 507 static void tx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, 508 struct mlx5_ipsec_fs *roce) 509 { 510 mlx5_ipsec_fs_roce_tx_destroy(roce); 511 if (tx->chains) { 512 ipsec_chains_destroy(tx->chains); 513 } else { 514 mlx5_del_flow_rules(tx->pol.rule); 515 mlx5_destroy_flow_group(tx->pol.group); 516 mlx5_destroy_flow_table(tx->ft.pol); 517 } 518 519 mlx5_destroy_flow_table(tx->ft.sa); 520 if (tx->allow_tunnel_mode) 521 mlx5_eswitch_unblock_encap(mdev); 522 mlx5_del_flow_rules(tx->status.rule); 523 mlx5_destroy_flow_table(tx->ft.status); 524 } 525 526 static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx, 527 struct mlx5_ipsec_fs *roce) 528 { 529 struct mlx5_flow_destination dest = {}; 530 struct mlx5_flow_table *ft; 531 u32 flags = 0; 532 int err; 533 534 ft = ipsec_ft_create(tx->ns, 2, 0, 1, 0); 535 if (IS_ERR(ft)) 536 return PTR_ERR(ft); 537 tx->ft.status = ft; 538 539 err = ipsec_counter_rule_tx(mdev, tx); 540 if (err) 541 goto err_status_rule; 542 543 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 544 tx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); 545 if (tx->allow_tunnel_mode) 546 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 547 ft = ipsec_ft_create(tx->ns, 1, 0, 4, flags); 548 if (IS_ERR(ft)) { 549 err = PTR_ERR(ft); 550 goto err_sa_ft; 551 } 552 tx->ft.sa = ft; 553 554 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 555 tx->chains = ipsec_chains_create( 556 mdev, tx->ft.sa, MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, 0, 0, 557 &tx->ft.pol); 558 if (IS_ERR(tx->chains)) { 559 err = PTR_ERR(tx->chains); 560 goto err_pol_ft; 561 } 562 563 goto connect_roce; 564 } 565 566 ft = ipsec_ft_create(tx->ns, 0, 0, 2, 0); 567 if (IS_ERR(ft)) { 568 err = PTR_ERR(ft); 569 goto err_pol_ft; 570 } 571 tx->ft.pol = ft; 572 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 573 dest.ft = tx->ft.sa; 574 err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); 575 if (err) { 576 mlx5_destroy_flow_table(tx->ft.pol); 577 goto err_pol_ft; 578 } 579 580 connect_roce: 581 err = mlx5_ipsec_fs_roce_tx_create(mdev, roce, tx->ft.pol); 582 if (err) 583 goto err_roce; 584 return 0; 585 586 err_roce: 587 if (tx->chains) { 588 ipsec_chains_destroy(tx->chains); 589 } else { 590 mlx5_del_flow_rules(tx->pol.rule); 591 mlx5_destroy_flow_group(tx->pol.group); 592 mlx5_destroy_flow_table(tx->ft.pol); 593 } 594 err_pol_ft: 595 mlx5_destroy_flow_table(tx->ft.sa); 596 err_sa_ft: 597 if (tx->allow_tunnel_mode) 598 mlx5_eswitch_unblock_encap(mdev); 599 mlx5_del_flow_rules(tx->status.rule); 600 err_status_rule: 601 mlx5_destroy_flow_table(tx->ft.status); 602 return err; 603 } 604 605 static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 606 struct mlx5e_ipsec_tx *tx) 607 { 608 int err; 609 610 if (tx->ft.refcnt) 611 goto skip; 612 613 err = tx_create(mdev, tx, ipsec->roce); 614 if (err) 615 return err; 616 617 skip: 618 tx->ft.refcnt++; 619 return 0; 620 } 621 622 static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) 623 { 624 if (--tx->ft.refcnt) 625 return; 626 627 tx_destroy(ipsec->mdev, tx, ipsec->roce); 628 } 629 630 static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, 631 struct mlx5e_ipsec *ipsec, 632 u32 prio) 633 { 634 struct mlx5e_ipsec_tx *tx = ipsec->tx; 635 struct mlx5_flow_table *ft; 636 int err; 637 638 mutex_lock(&tx->ft.mutex); 639 err = tx_get(mdev, ipsec, tx); 640 if (err) 641 goto err_get; 642 643 ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol; 644 if (IS_ERR(ft)) { 645 err = PTR_ERR(ft); 646 goto err_get_ft; 647 } 648 649 mutex_unlock(&tx->ft.mutex); 650 return ft; 651 652 err_get_ft: 653 tx_put(ipsec, tx); 654 err_get: 655 mutex_unlock(&tx->ft.mutex); 656 return ERR_PTR(err); 657 } 658 659 static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, 660 struct mlx5e_ipsec *ipsec) 661 { 662 struct mlx5e_ipsec_tx *tx = ipsec->tx; 663 int err; 664 665 mutex_lock(&tx->ft.mutex); 666 err = tx_get(mdev, ipsec, tx); 667 mutex_unlock(&tx->ft.mutex); 668 if (err) 669 return ERR_PTR(err); 670 671 return tx; 672 } 673 674 static void tx_ft_put(struct mlx5e_ipsec *ipsec) 675 { 676 struct mlx5e_ipsec_tx *tx = ipsec->tx; 677 678 mutex_lock(&tx->ft.mutex); 679 tx_put(ipsec, tx); 680 mutex_unlock(&tx->ft.mutex); 681 } 682 683 static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio) 684 { 685 struct mlx5e_ipsec_tx *tx = ipsec->tx; 686 687 mutex_lock(&tx->ft.mutex); 688 if (tx->chains) 689 ipsec_chains_put_table(tx->chains, prio); 690 691 tx_put(ipsec, tx); 692 mutex_unlock(&tx->ft.mutex); 693 } 694 695 static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, 696 __be32 *daddr) 697 { 698 if (!*saddr && !*daddr) 699 return; 700 701 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 702 703 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 704 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 705 706 if (*saddr) { 707 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 708 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); 709 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 710 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 711 } 712 713 if (*daddr) { 714 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 715 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); 716 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 717 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 718 } 719 } 720 721 static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, 722 __be32 *daddr) 723 { 724 if (addr6_all_zero(saddr) && addr6_all_zero(daddr)) 725 return; 726 727 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 728 729 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 730 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 731 732 if (!addr6_all_zero(saddr)) { 733 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 734 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); 735 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 736 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); 737 } 738 739 if (!addr6_all_zero(daddr)) { 740 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 741 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); 742 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 743 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); 744 } 745 } 746 747 static void setup_fte_esp(struct mlx5_flow_spec *spec) 748 { 749 /* ESP header */ 750 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 751 752 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 753 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); 754 } 755 756 static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi) 757 { 758 /* SPI number */ 759 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 760 761 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); 762 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); 763 } 764 765 static void setup_fte_no_frags(struct mlx5_flow_spec *spec) 766 { 767 /* Non fragmented */ 768 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 769 770 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); 771 MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); 772 } 773 774 static void setup_fte_reg_a(struct mlx5_flow_spec *spec) 775 { 776 /* Add IPsec indicator in metadata_reg_a */ 777 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 778 779 MLX5_SET(fte_match_param, spec->match_criteria, 780 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 781 MLX5_SET(fte_match_param, spec->match_value, 782 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 783 } 784 785 static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) 786 { 787 /* Pass policy check before choosing this SA */ 788 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 789 790 MLX5_SET(fte_match_param, spec->match_criteria, 791 misc_parameters_2.metadata_reg_c_0, reqid); 792 MLX5_SET(fte_match_param, spec->match_value, 793 misc_parameters_2.metadata_reg_c_0, reqid); 794 } 795 796 static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) 797 { 798 if (upspec->proto != IPPROTO_UDP) 799 return; 800 801 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 802 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); 803 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); 804 if (upspec->dport) { 805 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, 806 upspec->dport_mask); 807 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, upspec->dport); 808 } 809 810 if (upspec->sport) { 811 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, 812 upspec->sport_mask); 813 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, upspec->sport); 814 } 815 } 816 817 static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, 818 struct mlx5_flow_act *flow_act) 819 { 820 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 821 enum mlx5_flow_namespace_type ns_type; 822 struct mlx5_modify_hdr *modify_hdr; 823 824 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 825 switch (dir) { 826 case XFRM_DEV_OFFLOAD_IN: 827 MLX5_SET(set_action_in, action, field, 828 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 829 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 830 break; 831 case XFRM_DEV_OFFLOAD_OUT: 832 MLX5_SET(set_action_in, action, field, 833 MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); 834 ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 835 break; 836 default: 837 return -EINVAL; 838 } 839 840 MLX5_SET(set_action_in, action, data, val); 841 MLX5_SET(set_action_in, action, offset, 0); 842 MLX5_SET(set_action_in, action, length, 32); 843 844 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); 845 if (IS_ERR(modify_hdr)) { 846 mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", 847 PTR_ERR(modify_hdr)); 848 return PTR_ERR(modify_hdr); 849 } 850 851 flow_act->modify_hdr = modify_hdr; 852 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 853 return 0; 854 } 855 856 static int 857 setup_pkt_tunnel_reformat(struct mlx5_core_dev *mdev, 858 struct mlx5_accel_esp_xfrm_attrs *attrs, 859 struct mlx5_pkt_reformat_params *reformat_params) 860 { 861 struct ip_esp_hdr *esp_hdr; 862 struct ipv6hdr *ipv6hdr; 863 struct ethhdr *eth_hdr; 864 struct iphdr *iphdr; 865 char *reformatbf; 866 size_t bfflen; 867 void *hdr; 868 869 bfflen = sizeof(*eth_hdr); 870 871 if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) { 872 bfflen += sizeof(*esp_hdr) + 8; 873 874 switch (attrs->family) { 875 case AF_INET: 876 bfflen += sizeof(*iphdr); 877 break; 878 case AF_INET6: 879 bfflen += sizeof(*ipv6hdr); 880 break; 881 default: 882 return -EINVAL; 883 } 884 } 885 886 reformatbf = kzalloc(bfflen, GFP_KERNEL); 887 if (!reformatbf) 888 return -ENOMEM; 889 890 eth_hdr = (struct ethhdr *)reformatbf; 891 switch (attrs->family) { 892 case AF_INET: 893 eth_hdr->h_proto = htons(ETH_P_IP); 894 break; 895 case AF_INET6: 896 eth_hdr->h_proto = htons(ETH_P_IPV6); 897 break; 898 default: 899 goto free_reformatbf; 900 } 901 902 ether_addr_copy(eth_hdr->h_dest, attrs->dmac); 903 ether_addr_copy(eth_hdr->h_source, attrs->smac); 904 905 switch (attrs->dir) { 906 case XFRM_DEV_OFFLOAD_IN: 907 reformat_params->type = MLX5_REFORMAT_TYPE_L3_ESP_TUNNEL_TO_L2; 908 break; 909 case XFRM_DEV_OFFLOAD_OUT: 910 reformat_params->type = MLX5_REFORMAT_TYPE_L2_TO_L3_ESP_TUNNEL; 911 reformat_params->param_0 = attrs->authsize; 912 913 hdr = reformatbf + sizeof(*eth_hdr); 914 switch (attrs->family) { 915 case AF_INET: 916 iphdr = (struct iphdr *)hdr; 917 memcpy(&iphdr->saddr, &attrs->saddr.a4, 4); 918 memcpy(&iphdr->daddr, &attrs->daddr.a4, 4); 919 iphdr->version = 4; 920 iphdr->ihl = 5; 921 iphdr->ttl = IPSEC_TUNNEL_DEFAULT_TTL; 922 iphdr->protocol = IPPROTO_ESP; 923 hdr += sizeof(*iphdr); 924 break; 925 case AF_INET6: 926 ipv6hdr = (struct ipv6hdr *)hdr; 927 memcpy(&ipv6hdr->saddr, &attrs->saddr.a6, 16); 928 memcpy(&ipv6hdr->daddr, &attrs->daddr.a6, 16); 929 ipv6hdr->nexthdr = IPPROTO_ESP; 930 ipv6hdr->version = 6; 931 ipv6hdr->hop_limit = IPSEC_TUNNEL_DEFAULT_TTL; 932 hdr += sizeof(*ipv6hdr); 933 break; 934 default: 935 goto free_reformatbf; 936 } 937 938 esp_hdr = (struct ip_esp_hdr *)hdr; 939 esp_hdr->spi = htonl(attrs->spi); 940 break; 941 default: 942 goto free_reformatbf; 943 } 944 945 reformat_params->size = bfflen; 946 reformat_params->data = reformatbf; 947 return 0; 948 949 free_reformatbf: 950 kfree(reformatbf); 951 return -EINVAL; 952 } 953 954 static int 955 setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, 956 struct mlx5_pkt_reformat_params *reformat_params) 957 { 958 u8 *reformatbf; 959 __be32 spi; 960 961 switch (attrs->dir) { 962 case XFRM_DEV_OFFLOAD_IN: 963 reformat_params->type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; 964 break; 965 case XFRM_DEV_OFFLOAD_OUT: 966 if (attrs->family == AF_INET) 967 reformat_params->type = 968 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; 969 else 970 reformat_params->type = 971 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; 972 973 reformatbf = kzalloc(MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE, 974 GFP_KERNEL); 975 if (!reformatbf) 976 return -ENOMEM; 977 978 /* convert to network format */ 979 spi = htonl(attrs->spi); 980 memcpy(reformatbf, &spi, sizeof(spi)); 981 982 reformat_params->param_0 = attrs->authsize; 983 reformat_params->size = 984 MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE; 985 reformat_params->data = reformatbf; 986 break; 987 default: 988 return -EINVAL; 989 } 990 991 return 0; 992 } 993 994 static int setup_pkt_reformat(struct mlx5_core_dev *mdev, 995 struct mlx5_accel_esp_xfrm_attrs *attrs, 996 struct mlx5_flow_act *flow_act) 997 { 998 struct mlx5_pkt_reformat_params reformat_params = {}; 999 struct mlx5_pkt_reformat *pkt_reformat; 1000 enum mlx5_flow_namespace_type ns_type; 1001 int ret; 1002 1003 switch (attrs->dir) { 1004 case XFRM_DEV_OFFLOAD_IN: 1005 ns_type = MLX5_FLOW_NAMESPACE_KERNEL; 1006 break; 1007 case XFRM_DEV_OFFLOAD_OUT: 1008 ns_type = MLX5_FLOW_NAMESPACE_EGRESS; 1009 break; 1010 default: 1011 return -EINVAL; 1012 } 1013 1014 switch (attrs->mode) { 1015 case XFRM_MODE_TRANSPORT: 1016 ret = setup_pkt_transport_reformat(attrs, &reformat_params); 1017 break; 1018 case XFRM_MODE_TUNNEL: 1019 ret = setup_pkt_tunnel_reformat(mdev, attrs, &reformat_params); 1020 break; 1021 default: 1022 ret = -EINVAL; 1023 } 1024 1025 if (ret) 1026 return ret; 1027 1028 pkt_reformat = 1029 mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); 1030 kfree(reformat_params.data); 1031 if (IS_ERR(pkt_reformat)) 1032 return PTR_ERR(pkt_reformat); 1033 1034 flow_act->pkt_reformat = pkt_reformat; 1035 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 1036 return 0; 1037 } 1038 1039 static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1040 { 1041 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1042 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1043 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1044 struct mlx5_flow_destination dest[2]; 1045 struct mlx5_flow_act flow_act = {}; 1046 struct mlx5_flow_handle *rule; 1047 struct mlx5_flow_spec *spec; 1048 struct mlx5e_ipsec_rx *rx; 1049 struct mlx5_fc *counter; 1050 int err; 1051 1052 rx = rx_ft_get(mdev, ipsec, attrs->family); 1053 if (IS_ERR(rx)) 1054 return PTR_ERR(rx); 1055 1056 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1057 if (!spec) { 1058 err = -ENOMEM; 1059 goto err_alloc; 1060 } 1061 1062 if (attrs->family == AF_INET) 1063 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1064 else 1065 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1066 1067 setup_fte_spi(spec, attrs->spi); 1068 setup_fte_esp(spec); 1069 setup_fte_no_frags(spec); 1070 1071 err = setup_modify_header(mdev, sa_entry->ipsec_obj_id | BIT(31), 1072 XFRM_DEV_OFFLOAD_IN, &flow_act); 1073 if (err) 1074 goto err_mod_header; 1075 1076 switch (attrs->type) { 1077 case XFRM_DEV_OFFLOAD_PACKET: 1078 err = setup_pkt_reformat(mdev, attrs, &flow_act); 1079 if (err) 1080 goto err_pkt_reformat; 1081 break; 1082 default: 1083 break; 1084 } 1085 1086 counter = mlx5_fc_create(mdev, true); 1087 if (IS_ERR(counter)) { 1088 err = PTR_ERR(counter); 1089 goto err_add_cnt; 1090 } 1091 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1092 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1093 flow_act.flags |= FLOW_ACT_NO_APPEND; 1094 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 1095 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1096 if (attrs->drop) 1097 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1098 else 1099 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1100 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1101 dest[0].ft = rx->ft.status; 1102 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1103 dest[1].counter_id = mlx5_fc_id(counter); 1104 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 1105 if (IS_ERR(rule)) { 1106 err = PTR_ERR(rule); 1107 mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); 1108 goto err_add_flow; 1109 } 1110 kvfree(spec); 1111 1112 sa_entry->ipsec_rule.rule = rule; 1113 sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1114 sa_entry->ipsec_rule.fc = counter; 1115 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 1116 return 0; 1117 1118 err_add_flow: 1119 mlx5_fc_destroy(mdev, counter); 1120 err_add_cnt: 1121 if (flow_act.pkt_reformat) 1122 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1123 err_pkt_reformat: 1124 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1125 err_mod_header: 1126 kvfree(spec); 1127 err_alloc: 1128 rx_ft_put(ipsec, attrs->family); 1129 return err; 1130 } 1131 1132 static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1133 { 1134 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1135 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1136 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1137 struct mlx5_flow_destination dest[2]; 1138 struct mlx5_flow_act flow_act = {}; 1139 struct mlx5_flow_handle *rule; 1140 struct mlx5_flow_spec *spec; 1141 struct mlx5e_ipsec_tx *tx; 1142 struct mlx5_fc *counter; 1143 int err; 1144 1145 tx = tx_ft_get(mdev, ipsec); 1146 if (IS_ERR(tx)) 1147 return PTR_ERR(tx); 1148 1149 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1150 if (!spec) { 1151 err = -ENOMEM; 1152 goto err_alloc; 1153 } 1154 1155 if (attrs->family == AF_INET) 1156 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1157 else 1158 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1159 1160 setup_fte_no_frags(spec); 1161 setup_fte_upper_proto_match(spec, &attrs->upspec); 1162 1163 switch (attrs->type) { 1164 case XFRM_DEV_OFFLOAD_CRYPTO: 1165 setup_fte_spi(spec, attrs->spi); 1166 setup_fte_esp(spec); 1167 setup_fte_reg_a(spec); 1168 break; 1169 case XFRM_DEV_OFFLOAD_PACKET: 1170 if (attrs->reqid) 1171 setup_fte_reg_c0(spec, attrs->reqid); 1172 err = setup_pkt_reformat(mdev, attrs, &flow_act); 1173 if (err) 1174 goto err_pkt_reformat; 1175 break; 1176 default: 1177 break; 1178 } 1179 1180 counter = mlx5_fc_create(mdev, true); 1181 if (IS_ERR(counter)) { 1182 err = PTR_ERR(counter); 1183 goto err_add_cnt; 1184 } 1185 1186 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1187 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1188 flow_act.flags |= FLOW_ACT_NO_APPEND; 1189 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 1190 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1191 if (attrs->drop) 1192 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1193 else 1194 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1195 1196 dest[0].ft = tx->ft.status; 1197 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1198 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1199 dest[1].counter_id = mlx5_fc_id(counter); 1200 rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, dest, 2); 1201 if (IS_ERR(rule)) { 1202 err = PTR_ERR(rule); 1203 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1204 goto err_add_flow; 1205 } 1206 1207 kvfree(spec); 1208 sa_entry->ipsec_rule.rule = rule; 1209 sa_entry->ipsec_rule.fc = counter; 1210 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 1211 return 0; 1212 1213 err_add_flow: 1214 mlx5_fc_destroy(mdev, counter); 1215 err_add_cnt: 1216 if (flow_act.pkt_reformat) 1217 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1218 err_pkt_reformat: 1219 kvfree(spec); 1220 err_alloc: 1221 tx_ft_put(ipsec); 1222 return err; 1223 } 1224 1225 static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1226 { 1227 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1228 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1229 struct mlx5e_ipsec_tx *tx = pol_entry->ipsec->tx; 1230 struct mlx5_flow_destination dest[2] = {}; 1231 struct mlx5_flow_act flow_act = {}; 1232 struct mlx5_flow_handle *rule; 1233 struct mlx5_flow_spec *spec; 1234 struct mlx5_flow_table *ft; 1235 int err, dstn = 0; 1236 1237 ft = tx_ft_get_policy(mdev, pol_entry->ipsec, attrs->prio); 1238 if (IS_ERR(ft)) 1239 return PTR_ERR(ft); 1240 1241 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1242 if (!spec) { 1243 err = -ENOMEM; 1244 goto err_alloc; 1245 } 1246 1247 if (attrs->family == AF_INET) 1248 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1249 else 1250 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1251 1252 setup_fte_no_frags(spec); 1253 setup_fte_upper_proto_match(spec, &attrs->upspec); 1254 1255 switch (attrs->action) { 1256 case XFRM_POLICY_ALLOW: 1257 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1258 if (!attrs->reqid) 1259 break; 1260 1261 err = setup_modify_header(mdev, attrs->reqid, 1262 XFRM_DEV_OFFLOAD_OUT, &flow_act); 1263 if (err) 1264 goto err_mod_header; 1265 break; 1266 case XFRM_POLICY_BLOCK: 1267 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 1268 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1269 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1270 dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); 1271 dstn++; 1272 break; 1273 default: 1274 WARN_ON(true); 1275 err = -EINVAL; 1276 goto err_mod_header; 1277 } 1278 1279 flow_act.flags |= FLOW_ACT_NO_APPEND; 1280 dest[dstn].ft = tx->ft.sa; 1281 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1282 dstn++; 1283 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1284 if (IS_ERR(rule)) { 1285 err = PTR_ERR(rule); 1286 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1287 goto err_action; 1288 } 1289 1290 kvfree(spec); 1291 pol_entry->ipsec_rule.rule = rule; 1292 pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1293 return 0; 1294 1295 err_action: 1296 if (flow_act.modify_hdr) 1297 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1298 err_mod_header: 1299 kvfree(spec); 1300 err_alloc: 1301 tx_ft_put_policy(pol_entry->ipsec, attrs->prio); 1302 return err; 1303 } 1304 1305 static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1306 { 1307 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1308 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1309 struct mlx5_flow_destination dest[2]; 1310 struct mlx5_flow_act flow_act = {}; 1311 struct mlx5_flow_handle *rule; 1312 struct mlx5_flow_spec *spec; 1313 struct mlx5_flow_table *ft; 1314 struct mlx5e_ipsec_rx *rx; 1315 int err, dstn = 0; 1316 1317 ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio); 1318 if (IS_ERR(ft)) 1319 return PTR_ERR(ft); 1320 1321 rx = ipsec_rx(pol_entry->ipsec, attrs->family); 1322 1323 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1324 if (!spec) { 1325 err = -ENOMEM; 1326 goto err_alloc; 1327 } 1328 1329 if (attrs->family == AF_INET) 1330 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1331 else 1332 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1333 1334 setup_fte_no_frags(spec); 1335 1336 switch (attrs->action) { 1337 case XFRM_POLICY_ALLOW: 1338 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1339 break; 1340 case XFRM_POLICY_BLOCK: 1341 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 1342 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1343 dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); 1344 dstn++; 1345 break; 1346 default: 1347 WARN_ON(true); 1348 err = -EINVAL; 1349 goto err_action; 1350 } 1351 1352 flow_act.flags |= FLOW_ACT_NO_APPEND; 1353 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1354 dest[dstn].ft = rx->ft.sa; 1355 dstn++; 1356 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1357 if (IS_ERR(rule)) { 1358 err = PTR_ERR(rule); 1359 mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); 1360 goto err_action; 1361 } 1362 1363 kvfree(spec); 1364 pol_entry->ipsec_rule.rule = rule; 1365 return 0; 1366 1367 err_action: 1368 kvfree(spec); 1369 err_alloc: 1370 rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio); 1371 return err; 1372 } 1373 1374 static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) 1375 { 1376 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1377 struct mlx5_core_dev *mdev = ipsec->mdev; 1378 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1379 1380 mlx5_fc_destroy(mdev, tx->fc->drop); 1381 mlx5_fc_destroy(mdev, tx->fc->cnt); 1382 kfree(tx->fc); 1383 mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); 1384 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1385 kfree(rx_ipv4->fc); 1386 } 1387 1388 static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) 1389 { 1390 struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; 1391 struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; 1392 struct mlx5_core_dev *mdev = ipsec->mdev; 1393 struct mlx5e_ipsec_tx *tx = ipsec->tx; 1394 struct mlx5e_ipsec_fc *fc; 1395 struct mlx5_fc *counter; 1396 int err; 1397 1398 fc = kzalloc(sizeof(*rx_ipv4->fc), GFP_KERNEL); 1399 if (!fc) 1400 return -ENOMEM; 1401 1402 /* Both IPv4 and IPv6 point to same flow counters struct. */ 1403 rx_ipv4->fc = fc; 1404 rx_ipv6->fc = fc; 1405 counter = mlx5_fc_create(mdev, false); 1406 if (IS_ERR(counter)) { 1407 err = PTR_ERR(counter); 1408 goto err_rx_cnt; 1409 } 1410 1411 fc->cnt = counter; 1412 counter = mlx5_fc_create(mdev, false); 1413 if (IS_ERR(counter)) { 1414 err = PTR_ERR(counter); 1415 goto err_rx_drop; 1416 } 1417 1418 fc->drop = counter; 1419 fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); 1420 if (!fc) { 1421 err = -ENOMEM; 1422 goto err_tx_fc; 1423 } 1424 1425 tx->fc = fc; 1426 counter = mlx5_fc_create(mdev, false); 1427 if (IS_ERR(counter)) { 1428 err = PTR_ERR(counter); 1429 goto err_tx_cnt; 1430 } 1431 1432 fc->cnt = counter; 1433 counter = mlx5_fc_create(mdev, false); 1434 if (IS_ERR(counter)) { 1435 err = PTR_ERR(counter); 1436 goto err_tx_drop; 1437 } 1438 1439 fc->drop = counter; 1440 return 0; 1441 1442 err_tx_drop: 1443 mlx5_fc_destroy(mdev, tx->fc->cnt); 1444 err_tx_cnt: 1445 kfree(tx->fc); 1446 err_tx_fc: 1447 mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); 1448 err_rx_drop: 1449 mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); 1450 err_rx_cnt: 1451 kfree(rx_ipv4->fc); 1452 return err; 1453 } 1454 1455 void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) 1456 { 1457 struct mlx5_core_dev *mdev = priv->mdev; 1458 struct mlx5e_ipsec *ipsec = priv->ipsec; 1459 struct mlx5e_ipsec_hw_stats *stats; 1460 struct mlx5e_ipsec_fc *fc; 1461 1462 stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; 1463 1464 stats->ipsec_rx_pkts = 0; 1465 stats->ipsec_rx_bytes = 0; 1466 stats->ipsec_rx_drop_pkts = 0; 1467 stats->ipsec_rx_drop_bytes = 0; 1468 stats->ipsec_tx_pkts = 0; 1469 stats->ipsec_tx_bytes = 0; 1470 stats->ipsec_tx_drop_pkts = 0; 1471 stats->ipsec_tx_drop_bytes = 0; 1472 1473 fc = ipsec->rx_ipv4->fc; 1474 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); 1475 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, 1476 &stats->ipsec_rx_drop_bytes); 1477 1478 fc = ipsec->tx->fc; 1479 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); 1480 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, 1481 &stats->ipsec_tx_drop_bytes); 1482 } 1483 1484 int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1485 { 1486 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 1487 return tx_add_rule(sa_entry); 1488 1489 return rx_add_rule(sa_entry); 1490 } 1491 1492 void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1493 { 1494 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1495 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1496 1497 mlx5_del_flow_rules(ipsec_rule->rule); 1498 mlx5_fc_destroy(mdev, ipsec_rule->fc); 1499 if (ipsec_rule->pkt_reformat) 1500 mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); 1501 1502 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { 1503 tx_ft_put(sa_entry->ipsec); 1504 return; 1505 } 1506 1507 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1508 rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family); 1509 } 1510 1511 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1512 { 1513 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 1514 return tx_add_policy(pol_entry); 1515 1516 return rx_add_policy(pol_entry); 1517 } 1518 1519 void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1520 { 1521 struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; 1522 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1523 1524 mlx5_del_flow_rules(ipsec_rule->rule); 1525 1526 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { 1527 rx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.family, 1528 pol_entry->attrs.prio); 1529 return; 1530 } 1531 1532 if (ipsec_rule->modify_hdr) 1533 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1534 1535 tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio); 1536 } 1537 1538 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 1539 { 1540 if (!ipsec->tx) 1541 return; 1542 1543 if (mlx5_ipsec_device_caps(ipsec->mdev) & MLX5_IPSEC_CAP_ROCE) 1544 mlx5_ipsec_fs_roce_cleanup(ipsec->roce); 1545 1546 ipsec_fs_destroy_counters(ipsec); 1547 mutex_destroy(&ipsec->tx->ft.mutex); 1548 WARN_ON(ipsec->tx->ft.refcnt); 1549 kfree(ipsec->tx); 1550 1551 mutex_destroy(&ipsec->rx_ipv4->ft.mutex); 1552 WARN_ON(ipsec->rx_ipv4->ft.refcnt); 1553 kfree(ipsec->rx_ipv4); 1554 1555 mutex_destroy(&ipsec->rx_ipv6->ft.mutex); 1556 WARN_ON(ipsec->rx_ipv6->ft.refcnt); 1557 kfree(ipsec->rx_ipv6); 1558 } 1559 1560 int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) 1561 { 1562 struct mlx5_core_dev *mdev = ipsec->mdev; 1563 struct mlx5_flow_namespace *ns; 1564 int err = -ENOMEM; 1565 1566 ns = mlx5_get_flow_namespace(ipsec->mdev, 1567 MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 1568 if (!ns) 1569 return -EOPNOTSUPP; 1570 1571 ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); 1572 if (!ipsec->tx) 1573 return -ENOMEM; 1574 1575 ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); 1576 if (!ipsec->rx_ipv4) 1577 goto err_rx_ipv4; 1578 1579 ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); 1580 if (!ipsec->rx_ipv6) 1581 goto err_rx_ipv6; 1582 1583 err = ipsec_fs_init_counters(ipsec); 1584 if (err) 1585 goto err_counters; 1586 1587 mutex_init(&ipsec->tx->ft.mutex); 1588 mutex_init(&ipsec->rx_ipv4->ft.mutex); 1589 mutex_init(&ipsec->rx_ipv6->ft.mutex); 1590 ipsec->tx->ns = ns; 1591 1592 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) 1593 ipsec->roce = mlx5_ipsec_fs_roce_init(mdev); 1594 1595 return 0; 1596 1597 err_counters: 1598 kfree(ipsec->rx_ipv6); 1599 err_rx_ipv6: 1600 kfree(ipsec->rx_ipv4); 1601 err_rx_ipv4: 1602 kfree(ipsec->tx); 1603 return err; 1604 } 1605 1606 void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) 1607 { 1608 struct mlx5e_ipsec_sa_entry sa_entry_shadow = {}; 1609 int err; 1610 1611 memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry)); 1612 memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule)); 1613 1614 err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow); 1615 if (err) 1616 return; 1617 1618 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 1619 memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry)); 1620 } 1621 1622 bool mlx5e_ipsec_fs_tunnel_enabled(struct mlx5e_ipsec_sa_entry *sa_entry) 1623 { 1624 struct mlx5e_ipsec_rx *rx = 1625 ipsec_rx(sa_entry->ipsec, sa_entry->attrs.family); 1626 struct mlx5e_ipsec_tx *tx = sa_entry->ipsec->tx; 1627 1628 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 1629 return tx->allow_tunnel_mode; 1630 1631 return rx->allow_tunnel_mode; 1632 } 1633