1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include <net/netfilter/nf_conntrack.h> 5 #include <net/netfilter/nf_conntrack_core.h> 6 #include <net/netfilter/nf_conntrack_zones.h> 7 #include <net/netfilter/nf_conntrack_labels.h> 8 #include <net/netfilter/nf_conntrack_helper.h> 9 #include <net/netfilter/nf_conntrack_acct.h> 10 #include <uapi/linux/tc_act/tc_pedit.h> 11 #include <net/tc_act/tc_ct.h> 12 #include <net/flow_offload.h> 13 #include <net/netfilter/nf_flow_table.h> 14 #include <linux/workqueue.h> 15 #include <linux/refcount.h> 16 #include <linux/xarray.h> 17 #include <linux/if_macvlan.h> 18 #include <linux/debugfs.h> 19 20 #include "lib/fs_chains.h" 21 #include "en/tc_ct.h" 22 #include "en/tc/ct_fs.h" 23 #include "en/tc_priv.h" 24 #include "en/mod_hdr.h" 25 #include "en/mapping.h" 26 #include "en/tc/post_act.h" 27 #include "en.h" 28 #include "en_tc.h" 29 #include "en_rep.h" 30 #include "fs_core.h" 31 32 #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1) 33 #define MLX5_CT_STATE_TRK_BIT BIT(2) 34 #define MLX5_CT_STATE_NAT_BIT BIT(3) 35 #define MLX5_CT_STATE_REPLY_BIT BIT(4) 36 #define MLX5_CT_STATE_RELATED_BIT BIT(5) 37 #define MLX5_CT_STATE_INVALID_BIT BIT(6) 38 #define MLX5_CT_STATE_NEW_BIT BIT(7) 39 40 #define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG) 41 #define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG) 42 43 /* Statically allocate modify actions for 44 * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10. 45 * This will be increased dynamically if needed (for the ipv6 snat + dnat). 46 */ 47 #define MLX5_CT_MIN_MOD_ACTS 10 48 49 #define ct_dbg(fmt, args...)\ 50 netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args) 51 52 struct mlx5_tc_ct_debugfs { 53 struct { 54 atomic_t offloaded; 55 atomic_t rx_dropped; 56 } stats; 57 58 struct dentry *root; 59 }; 60 61 struct mlx5_tc_ct_priv { 62 struct mlx5_core_dev *dev; 63 struct mlx5e_priv *priv; 64 const struct net_device *netdev; 65 struct mod_hdr_tbl *mod_hdr_tbl; 66 struct xarray tuple_ids; 67 struct rhashtable zone_ht; 68 struct rhashtable ct_tuples_ht; 69 struct rhashtable ct_tuples_nat_ht; 70 struct mlx5_flow_table *ct; 71 struct mlx5_flow_table *ct_nat; 72 struct mlx5e_post_act *post_act; 73 struct mutex control_lock; /* guards parallel adds/dels */ 74 struct mapping_ctx *zone_mapping; 75 struct mapping_ctx *labels_mapping; 76 enum mlx5_flow_namespace_type ns_type; 77 struct mlx5_fs_chains *chains; 78 struct mlx5_ct_fs *fs; 79 struct mlx5_ct_fs_ops *fs_ops; 80 spinlock_t ht_lock; /* protects ft entries */ 81 struct workqueue_struct *wq; 82 83 struct mlx5_tc_ct_debugfs debugfs; 84 }; 85 86 struct mlx5_ct_zone_rule { 87 struct mlx5_ct_fs_rule *rule; 88 struct mlx5e_mod_hdr_handle *mh; 89 struct mlx5_flow_attr *attr; 90 bool nat; 91 }; 92 93 struct mlx5_tc_ct_pre { 94 struct mlx5_flow_table *ft; 95 struct mlx5_flow_group *flow_grp; 96 struct mlx5_flow_group *miss_grp; 97 struct mlx5_flow_handle *flow_rule; 98 struct mlx5_flow_handle *miss_rule; 99 struct mlx5_modify_hdr *modify_hdr; 100 }; 101 102 struct mlx5_ct_ft { 103 struct rhash_head node; 104 u16 zone; 105 u32 zone_restore_id; 106 refcount_t refcount; 107 struct nf_flowtable *nf_ft; 108 struct mlx5_tc_ct_priv *ct_priv; 109 struct rhashtable ct_entries_ht; 110 struct mlx5_tc_ct_pre pre_ct; 111 struct mlx5_tc_ct_pre pre_ct_nat; 112 }; 113 114 struct mlx5_ct_tuple { 115 u16 addr_type; 116 __be16 n_proto; 117 u8 ip_proto; 118 struct { 119 union { 120 __be32 src_v4; 121 struct in6_addr src_v6; 122 }; 123 union { 124 __be32 dst_v4; 125 struct in6_addr dst_v6; 126 }; 127 } ip; 128 struct { 129 __be16 src; 130 __be16 dst; 131 } port; 132 133 u16 zone; 134 }; 135 136 struct mlx5_ct_counter { 137 struct mlx5_fc *counter; 138 refcount_t refcount; 139 bool is_shared; 140 }; 141 142 enum { 143 MLX5_CT_ENTRY_FLAG_VALID, 144 }; 145 146 struct mlx5_ct_entry { 147 struct rhash_head node; 148 struct rhash_head tuple_node; 149 struct rhash_head tuple_nat_node; 150 struct mlx5_ct_counter *counter; 151 unsigned long cookie; 152 unsigned long restore_cookie; 153 struct mlx5_ct_tuple tuple; 154 struct mlx5_ct_tuple tuple_nat; 155 struct mlx5_ct_zone_rule zone_rules[2]; 156 157 struct mlx5_tc_ct_priv *ct_priv; 158 struct work_struct work; 159 160 refcount_t refcnt; 161 unsigned long flags; 162 }; 163 164 static void 165 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, 166 struct mlx5_flow_attr *attr, 167 struct mlx5e_mod_hdr_handle *mh); 168 169 static const struct rhashtable_params cts_ht_params = { 170 .head_offset = offsetof(struct mlx5_ct_entry, node), 171 .key_offset = offsetof(struct mlx5_ct_entry, cookie), 172 .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie), 173 .automatic_shrinking = true, 174 .min_size = 16 * 1024, 175 }; 176 177 static const struct rhashtable_params zone_params = { 178 .head_offset = offsetof(struct mlx5_ct_ft, node), 179 .key_offset = offsetof(struct mlx5_ct_ft, zone), 180 .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone), 181 .automatic_shrinking = true, 182 }; 183 184 static const struct rhashtable_params tuples_ht_params = { 185 .head_offset = offsetof(struct mlx5_ct_entry, tuple_node), 186 .key_offset = offsetof(struct mlx5_ct_entry, tuple), 187 .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple), 188 .automatic_shrinking = true, 189 .min_size = 16 * 1024, 190 }; 191 192 static const struct rhashtable_params tuples_nat_ht_params = { 193 .head_offset = offsetof(struct mlx5_ct_entry, tuple_nat_node), 194 .key_offset = offsetof(struct mlx5_ct_entry, tuple_nat), 195 .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple_nat), 196 .automatic_shrinking = true, 197 .min_size = 16 * 1024, 198 }; 199 200 static bool 201 mlx5_tc_ct_entry_has_nat(struct mlx5_ct_entry *entry) 202 { 203 return !!(entry->tuple_nat_node.next); 204 } 205 206 static int 207 mlx5_get_label_mapping(struct mlx5_tc_ct_priv *ct_priv, 208 u32 *labels, u32 *id) 209 { 210 if (!memchr_inv(labels, 0, sizeof(u32) * 4)) { 211 *id = 0; 212 return 0; 213 } 214 215 if (mapping_add(ct_priv->labels_mapping, labels, id)) 216 return -EOPNOTSUPP; 217 218 return 0; 219 } 220 221 static void 222 mlx5_put_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 id) 223 { 224 if (id) 225 mapping_remove(ct_priv->labels_mapping, id); 226 } 227 228 static int 229 mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule) 230 { 231 struct flow_match_control control; 232 struct flow_match_basic basic; 233 234 flow_rule_match_basic(rule, &basic); 235 flow_rule_match_control(rule, &control); 236 237 tuple->n_proto = basic.key->n_proto; 238 tuple->ip_proto = basic.key->ip_proto; 239 tuple->addr_type = control.key->addr_type; 240 241 if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 242 struct flow_match_ipv4_addrs match; 243 244 flow_rule_match_ipv4_addrs(rule, &match); 245 tuple->ip.src_v4 = match.key->src; 246 tuple->ip.dst_v4 = match.key->dst; 247 } else if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 248 struct flow_match_ipv6_addrs match; 249 250 flow_rule_match_ipv6_addrs(rule, &match); 251 tuple->ip.src_v6 = match.key->src; 252 tuple->ip.dst_v6 = match.key->dst; 253 } else { 254 return -EOPNOTSUPP; 255 } 256 257 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 258 struct flow_match_ports match; 259 260 flow_rule_match_ports(rule, &match); 261 switch (tuple->ip_proto) { 262 case IPPROTO_TCP: 263 case IPPROTO_UDP: 264 tuple->port.src = match.key->src; 265 tuple->port.dst = match.key->dst; 266 break; 267 default: 268 return -EOPNOTSUPP; 269 } 270 } else { 271 if (tuple->ip_proto != IPPROTO_GRE) 272 return -EOPNOTSUPP; 273 } 274 275 return 0; 276 } 277 278 static int 279 mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple, 280 struct flow_rule *rule) 281 { 282 struct flow_action *flow_action = &rule->action; 283 struct flow_action_entry *act; 284 u32 offset, val, ip6_offset; 285 int i; 286 287 flow_action_for_each(i, act, flow_action) { 288 if (act->id != FLOW_ACTION_MANGLE) 289 continue; 290 291 offset = act->mangle.offset; 292 val = act->mangle.val; 293 switch (act->mangle.htype) { 294 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 295 if (offset == offsetof(struct iphdr, saddr)) 296 tuple->ip.src_v4 = cpu_to_be32(val); 297 else if (offset == offsetof(struct iphdr, daddr)) 298 tuple->ip.dst_v4 = cpu_to_be32(val); 299 else 300 return -EOPNOTSUPP; 301 break; 302 303 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 304 ip6_offset = (offset - offsetof(struct ipv6hdr, saddr)); 305 ip6_offset /= 4; 306 if (ip6_offset < 4) 307 tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val); 308 else if (ip6_offset < 8) 309 tuple->ip.dst_v6.s6_addr32[ip6_offset - 4] = cpu_to_be32(val); 310 else 311 return -EOPNOTSUPP; 312 break; 313 314 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 315 if (offset == offsetof(struct tcphdr, source)) 316 tuple->port.src = cpu_to_be16(val); 317 else if (offset == offsetof(struct tcphdr, dest)) 318 tuple->port.dst = cpu_to_be16(val); 319 else 320 return -EOPNOTSUPP; 321 break; 322 323 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 324 if (offset == offsetof(struct udphdr, source)) 325 tuple->port.src = cpu_to_be16(val); 326 else if (offset == offsetof(struct udphdr, dest)) 327 tuple->port.dst = cpu_to_be16(val); 328 else 329 return -EOPNOTSUPP; 330 break; 331 332 default: 333 return -EOPNOTSUPP; 334 } 335 } 336 337 return 0; 338 } 339 340 static int 341 mlx5_tc_ct_get_flow_source_match(struct mlx5_tc_ct_priv *ct_priv, 342 struct net_device *ndev) 343 { 344 struct mlx5e_priv *other_priv = netdev_priv(ndev); 345 struct mlx5_core_dev *mdev = ct_priv->dev; 346 bool vf_rep, uplink_rep; 347 348 vf_rep = mlx5e_eswitch_vf_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev); 349 uplink_rep = mlx5e_eswitch_uplink_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev); 350 351 if (vf_rep) 352 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 353 if (uplink_rep) 354 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 355 if (is_vlan_dev(ndev)) 356 return mlx5_tc_ct_get_flow_source_match(ct_priv, vlan_dev_real_dev(ndev)); 357 if (netif_is_macvlan(ndev)) 358 return mlx5_tc_ct_get_flow_source_match(ct_priv, macvlan_dev_real_dev(ndev)); 359 if (mlx5e_get_tc_tun(ndev) || netif_is_lag_master(ndev)) 360 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 361 362 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT; 363 } 364 365 static int 366 mlx5_tc_ct_set_tuple_match(struct mlx5_tc_ct_priv *ct_priv, 367 struct mlx5_flow_spec *spec, 368 struct flow_rule *rule) 369 { 370 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 371 outer_headers); 372 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 373 outer_headers); 374 u16 addr_type = 0; 375 u8 ip_proto = 0; 376 377 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 378 struct flow_match_basic match; 379 380 flow_rule_match_basic(rule, &match); 381 382 mlx5e_tc_set_ethertype(ct_priv->dev, &match, true, headers_c, headers_v); 383 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol, 384 match.mask->ip_proto); 385 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, 386 match.key->ip_proto); 387 388 ip_proto = match.key->ip_proto; 389 } 390 391 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 392 struct flow_match_control match; 393 394 flow_rule_match_control(rule, &match); 395 addr_type = match.key->addr_type; 396 } 397 398 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 399 struct flow_match_ipv4_addrs match; 400 401 flow_rule_match_ipv4_addrs(rule, &match); 402 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 403 src_ipv4_src_ipv6.ipv4_layout.ipv4), 404 &match.mask->src, sizeof(match.mask->src)); 405 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 406 src_ipv4_src_ipv6.ipv4_layout.ipv4), 407 &match.key->src, sizeof(match.key->src)); 408 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 409 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 410 &match.mask->dst, sizeof(match.mask->dst)); 411 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 412 dst_ipv4_dst_ipv6.ipv4_layout.ipv4), 413 &match.key->dst, sizeof(match.key->dst)); 414 } 415 416 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 417 struct flow_match_ipv6_addrs match; 418 419 flow_rule_match_ipv6_addrs(rule, &match); 420 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 421 src_ipv4_src_ipv6.ipv6_layout.ipv6), 422 &match.mask->src, sizeof(match.mask->src)); 423 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 424 src_ipv4_src_ipv6.ipv6_layout.ipv6), 425 &match.key->src, sizeof(match.key->src)); 426 427 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c, 428 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 429 &match.mask->dst, sizeof(match.mask->dst)); 430 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v, 431 dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 432 &match.key->dst, sizeof(match.key->dst)); 433 } 434 435 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 436 struct flow_match_ports match; 437 438 flow_rule_match_ports(rule, &match); 439 switch (ip_proto) { 440 case IPPROTO_TCP: 441 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 442 tcp_sport, ntohs(match.mask->src)); 443 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 444 tcp_sport, ntohs(match.key->src)); 445 446 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 447 tcp_dport, ntohs(match.mask->dst)); 448 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 449 tcp_dport, ntohs(match.key->dst)); 450 break; 451 452 case IPPROTO_UDP: 453 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 454 udp_sport, ntohs(match.mask->src)); 455 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 456 udp_sport, ntohs(match.key->src)); 457 458 MLX5_SET(fte_match_set_lyr_2_4, headers_c, 459 udp_dport, ntohs(match.mask->dst)); 460 MLX5_SET(fte_match_set_lyr_2_4, headers_v, 461 udp_dport, ntohs(match.key->dst)); 462 break; 463 default: 464 break; 465 } 466 } 467 468 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) { 469 struct flow_match_tcp match; 470 471 flow_rule_match_tcp(rule, &match); 472 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags, 473 ntohs(match.mask->flags)); 474 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags, 475 ntohs(match.key->flags)); 476 } 477 478 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) { 479 struct flow_match_meta match; 480 481 flow_rule_match_meta(rule, &match); 482 483 if (match.key->ingress_ifindex & match.mask->ingress_ifindex) { 484 struct net_device *dev; 485 486 dev = dev_get_by_index(&init_net, match.key->ingress_ifindex); 487 if (dev && MLX5_CAP_ESW_FLOWTABLE(ct_priv->dev, flow_source)) 488 spec->flow_context.flow_source = 489 mlx5_tc_ct_get_flow_source_match(ct_priv, dev); 490 491 dev_put(dev); 492 } 493 } 494 495 return 0; 496 } 497 498 static void 499 mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry) 500 { 501 if (entry->counter->is_shared && 502 !refcount_dec_and_test(&entry->counter->refcount)) 503 return; 504 505 mlx5_fc_destroy(ct_priv->dev, entry->counter->counter); 506 kfree(entry->counter); 507 } 508 509 static void 510 mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv, 511 struct mlx5_ct_entry *entry, 512 bool nat) 513 { 514 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat]; 515 struct mlx5_flow_attr *attr = zone_rule->attr; 516 517 ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone); 518 519 ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule); 520 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh); 521 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); 522 kfree(attr); 523 } 524 525 static void 526 mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv, 527 struct mlx5_ct_entry *entry) 528 { 529 mlx5_tc_ct_entry_del_rule(ct_priv, entry, true); 530 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false); 531 532 atomic_dec(&ct_priv->debugfs.stats.offloaded); 533 } 534 535 static struct flow_action_entry * 536 mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule) 537 { 538 struct flow_action *flow_action = &flow_rule->action; 539 struct flow_action_entry *act; 540 int i; 541 542 flow_action_for_each(i, act, flow_action) { 543 if (act->id == FLOW_ACTION_CT_METADATA) 544 return act; 545 } 546 547 return NULL; 548 } 549 550 static int 551 mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv, 552 struct mlx5e_tc_mod_hdr_acts *mod_acts, 553 u8 ct_state, 554 u32 mark, 555 u32 labels_id, 556 u8 zone_restore_id) 557 { 558 enum mlx5_flow_namespace_type ns = ct_priv->ns_type; 559 struct mlx5_core_dev *dev = ct_priv->dev; 560 int err; 561 562 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns, 563 CTSTATE_TO_REG, ct_state); 564 if (err) 565 return err; 566 567 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns, 568 MARK_TO_REG, mark); 569 if (err) 570 return err; 571 572 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns, 573 LABELS_TO_REG, labels_id); 574 if (err) 575 return err; 576 577 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns, 578 ZONE_RESTORE_TO_REG, zone_restore_id); 579 if (err) 580 return err; 581 582 /* Make another copy of zone id in reg_b for 583 * NIC rx flows since we don't copy reg_c1 to 584 * reg_b upon miss. 585 */ 586 if (ns != MLX5_FLOW_NAMESPACE_FDB) { 587 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns, 588 NIC_ZONE_RESTORE_TO_REG, zone_restore_id); 589 if (err) 590 return err; 591 } 592 return 0; 593 } 594 595 static int 596 mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act, 597 char *modact) 598 { 599 u32 offset = act->mangle.offset, field; 600 601 switch (act->mangle.htype) { 602 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 603 MLX5_SET(set_action_in, modact, length, 0); 604 if (offset == offsetof(struct iphdr, saddr)) 605 field = MLX5_ACTION_IN_FIELD_OUT_SIPV4; 606 else if (offset == offsetof(struct iphdr, daddr)) 607 field = MLX5_ACTION_IN_FIELD_OUT_DIPV4; 608 else 609 return -EOPNOTSUPP; 610 break; 611 612 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 613 MLX5_SET(set_action_in, modact, length, 0); 614 if (offset == offsetof(struct ipv6hdr, saddr) + 12) 615 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0; 616 else if (offset == offsetof(struct ipv6hdr, saddr) + 8) 617 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32; 618 else if (offset == offsetof(struct ipv6hdr, saddr) + 4) 619 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64; 620 else if (offset == offsetof(struct ipv6hdr, saddr)) 621 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96; 622 else if (offset == offsetof(struct ipv6hdr, daddr) + 12) 623 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0; 624 else if (offset == offsetof(struct ipv6hdr, daddr) + 8) 625 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32; 626 else if (offset == offsetof(struct ipv6hdr, daddr) + 4) 627 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64; 628 else if (offset == offsetof(struct ipv6hdr, daddr)) 629 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96; 630 else 631 return -EOPNOTSUPP; 632 break; 633 634 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 635 MLX5_SET(set_action_in, modact, length, 16); 636 if (offset == offsetof(struct tcphdr, source)) 637 field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT; 638 else if (offset == offsetof(struct tcphdr, dest)) 639 field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT; 640 else 641 return -EOPNOTSUPP; 642 break; 643 644 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 645 MLX5_SET(set_action_in, modact, length, 16); 646 if (offset == offsetof(struct udphdr, source)) 647 field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT; 648 else if (offset == offsetof(struct udphdr, dest)) 649 field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT; 650 else 651 return -EOPNOTSUPP; 652 break; 653 654 default: 655 return -EOPNOTSUPP; 656 } 657 658 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET); 659 MLX5_SET(set_action_in, modact, offset, 0); 660 MLX5_SET(set_action_in, modact, field, field); 661 MLX5_SET(set_action_in, modact, data, act->mangle.val); 662 663 return 0; 664 } 665 666 static int 667 mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv, 668 struct flow_rule *flow_rule, 669 struct mlx5e_tc_mod_hdr_acts *mod_acts) 670 { 671 struct flow_action *flow_action = &flow_rule->action; 672 struct mlx5_core_dev *mdev = ct_priv->dev; 673 struct flow_action_entry *act; 674 char *modact; 675 int err, i; 676 677 flow_action_for_each(i, act, flow_action) { 678 switch (act->id) { 679 case FLOW_ACTION_MANGLE: { 680 modact = mlx5e_mod_hdr_alloc(mdev, ct_priv->ns_type, mod_acts); 681 if (IS_ERR(modact)) 682 return PTR_ERR(modact); 683 684 err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact); 685 if (err) 686 return err; 687 688 mod_acts->num_actions++; 689 } 690 break; 691 692 case FLOW_ACTION_CT_METADATA: 693 /* Handled earlier */ 694 continue; 695 default: 696 return -EOPNOTSUPP; 697 } 698 } 699 700 return 0; 701 } 702 703 static int 704 mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, 705 struct mlx5_flow_attr *attr, 706 struct flow_rule *flow_rule, 707 struct mlx5e_mod_hdr_handle **mh, 708 u8 zone_restore_id, bool nat_table, bool has_nat) 709 { 710 DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS); 711 DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr); 712 struct flow_action_entry *meta; 713 enum ip_conntrack_info ctinfo; 714 u16 ct_state = 0; 715 int err; 716 717 meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule); 718 if (!meta) 719 return -EOPNOTSUPP; 720 ctinfo = meta->ct_metadata.cookie & NFCT_INFOMASK; 721 722 err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels, 723 &attr->ct_attr.ct_labels_id); 724 if (err) 725 return -EOPNOTSUPP; 726 if (nat_table) { 727 if (has_nat) { 728 err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule, &mod_acts); 729 if (err) 730 goto err_mapping; 731 } 732 733 ct_state |= MLX5_CT_STATE_NAT_BIT; 734 } 735 736 ct_state |= MLX5_CT_STATE_TRK_BIT; 737 ct_state |= ctinfo == IP_CT_NEW ? MLX5_CT_STATE_NEW_BIT : MLX5_CT_STATE_ESTABLISHED_BIT; 738 ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT; 739 err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts, 740 ct_state, 741 meta->ct_metadata.mark, 742 attr->ct_attr.ct_labels_id, 743 zone_restore_id); 744 if (err) 745 goto err_mapping; 746 747 if (nat_table && has_nat) { 748 attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type, 749 mod_acts.num_actions, 750 mod_acts.actions); 751 if (IS_ERR(attr->modify_hdr)) { 752 err = PTR_ERR(attr->modify_hdr); 753 goto err_mapping; 754 } 755 756 *mh = NULL; 757 } else { 758 *mh = mlx5e_mod_hdr_attach(ct_priv->dev, 759 ct_priv->mod_hdr_tbl, 760 ct_priv->ns_type, 761 &mod_acts); 762 if (IS_ERR(*mh)) { 763 err = PTR_ERR(*mh); 764 goto err_mapping; 765 } 766 attr->modify_hdr = mlx5e_mod_hdr_get(*mh); 767 } 768 769 mlx5e_mod_hdr_dealloc(&mod_acts); 770 return 0; 771 772 err_mapping: 773 mlx5e_mod_hdr_dealloc(&mod_acts); 774 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); 775 return err; 776 } 777 778 static void 779 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, 780 struct mlx5_flow_attr *attr, 781 struct mlx5e_mod_hdr_handle *mh) 782 { 783 if (mh) 784 mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh); 785 else 786 mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr); 787 } 788 789 static int 790 mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv, 791 struct flow_rule *flow_rule, 792 struct mlx5_ct_entry *entry, 793 bool nat, u8 zone_restore_id) 794 { 795 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat]; 796 struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev); 797 struct mlx5_flow_spec *spec = NULL; 798 struct mlx5_flow_attr *attr; 799 int err; 800 801 zone_rule->nat = nat; 802 803 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 804 if (!spec) 805 return -ENOMEM; 806 807 attr = mlx5_alloc_flow_attr(ct_priv->ns_type); 808 if (!attr) { 809 err = -ENOMEM; 810 goto err_attr; 811 } 812 813 err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, 814 &zone_rule->mh, 815 zone_restore_id, 816 nat, 817 mlx5_tc_ct_entry_has_nat(entry)); 818 if (err) { 819 ct_dbg("Failed to create ct entry mod hdr"); 820 goto err_mod_hdr; 821 } 822 823 attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 824 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 825 MLX5_FLOW_CONTEXT_ACTION_COUNT; 826 attr->dest_chain = 0; 827 attr->dest_ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act); 828 attr->ft = nat ? ct_priv->ct_nat : ct_priv->ct; 829 if (entry->tuple.ip_proto == IPPROTO_TCP || 830 entry->tuple.ip_proto == IPPROTO_UDP) 831 attr->outer_match_level = MLX5_MATCH_L4; 832 else 833 attr->outer_match_level = MLX5_MATCH_L3; 834 attr->counter = entry->counter->counter; 835 attr->flags |= MLX5_ATTR_FLAG_NO_IN_PORT; 836 if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB) 837 attr->esw_attr->in_mdev = priv->mdev; 838 839 mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule); 840 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK); 841 842 zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule); 843 if (IS_ERR(zone_rule->rule)) { 844 err = PTR_ERR(zone_rule->rule); 845 ct_dbg("Failed to add ct entry rule, nat: %d", nat); 846 goto err_rule; 847 } 848 849 zone_rule->attr = attr; 850 851 kvfree(spec); 852 ct_dbg("Offloaded ct entry rule in zone %d", entry->tuple.zone); 853 854 return 0; 855 856 err_rule: 857 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh); 858 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); 859 err_mod_hdr: 860 kfree(attr); 861 err_attr: 862 kvfree(spec); 863 return err; 864 } 865 866 static int 867 mlx5_tc_ct_entry_replace_rule(struct mlx5_tc_ct_priv *ct_priv, 868 struct flow_rule *flow_rule, 869 struct mlx5_ct_entry *entry, 870 bool nat, u8 zone_restore_id) 871 { 872 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat]; 873 struct mlx5_flow_attr *attr = zone_rule->attr, *old_attr; 874 struct mlx5e_mod_hdr_handle *mh; 875 struct mlx5_ct_fs_rule *rule; 876 struct mlx5_flow_spec *spec; 877 int err; 878 879 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 880 if (!spec) 881 return -ENOMEM; 882 883 old_attr = mlx5_alloc_flow_attr(ct_priv->ns_type); 884 if (!old_attr) { 885 err = -ENOMEM; 886 goto err_attr; 887 } 888 *old_attr = *attr; 889 890 err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, &mh, zone_restore_id, 891 nat, mlx5_tc_ct_entry_has_nat(entry)); 892 if (err) { 893 ct_dbg("Failed to create ct entry mod hdr"); 894 goto err_mod_hdr; 895 } 896 897 mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule); 898 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK); 899 900 rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule); 901 if (IS_ERR(rule)) { 902 err = PTR_ERR(rule); 903 ct_dbg("Failed to add replacement ct entry rule, nat: %d", nat); 904 goto err_rule; 905 } 906 907 ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule); 908 zone_rule->rule = rule; 909 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, old_attr, zone_rule->mh); 910 zone_rule->mh = mh; 911 mlx5_put_label_mapping(ct_priv, old_attr->ct_attr.ct_labels_id); 912 913 kfree(old_attr); 914 kvfree(spec); 915 ct_dbg("Replaced ct entry rule in zone %d", entry->tuple.zone); 916 917 return 0; 918 919 err_rule: 920 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, mh); 921 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); 922 err_mod_hdr: 923 kfree(old_attr); 924 err_attr: 925 kvfree(spec); 926 return err; 927 } 928 929 static bool 930 mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry) 931 { 932 return test_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags); 933 } 934 935 static struct mlx5_ct_entry * 936 mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple *tuple) 937 { 938 struct mlx5_ct_entry *entry; 939 940 entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_ht, tuple, 941 tuples_ht_params); 942 if (entry && mlx5_tc_ct_entry_valid(entry) && 943 refcount_inc_not_zero(&entry->refcnt)) { 944 return entry; 945 } else if (!entry) { 946 entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_nat_ht, 947 tuple, tuples_nat_ht_params); 948 if (entry && mlx5_tc_ct_entry_valid(entry) && 949 refcount_inc_not_zero(&entry->refcnt)) 950 return entry; 951 } 952 953 return entry ? ERR_PTR(-EINVAL) : NULL; 954 } 955 956 static void mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry *entry) 957 { 958 struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv; 959 960 rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht, 961 &entry->tuple_nat_node, 962 tuples_nat_ht_params); 963 rhashtable_remove_fast(&ct_priv->ct_tuples_ht, &entry->tuple_node, 964 tuples_ht_params); 965 } 966 967 static void mlx5_tc_ct_entry_del(struct mlx5_ct_entry *entry) 968 { 969 struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv; 970 971 mlx5_tc_ct_entry_del_rules(ct_priv, entry); 972 973 spin_lock_bh(&ct_priv->ht_lock); 974 mlx5_tc_ct_entry_remove_from_tuples(entry); 975 spin_unlock_bh(&ct_priv->ht_lock); 976 977 mlx5_tc_ct_counter_put(ct_priv, entry); 978 kfree(entry); 979 } 980 981 static void 982 mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry) 983 { 984 if (!refcount_dec_and_test(&entry->refcnt)) 985 return; 986 987 mlx5_tc_ct_entry_del(entry); 988 } 989 990 static void mlx5_tc_ct_entry_del_work(struct work_struct *work) 991 { 992 struct mlx5_ct_entry *entry = container_of(work, struct mlx5_ct_entry, work); 993 994 mlx5_tc_ct_entry_del(entry); 995 } 996 997 static void 998 __mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry) 999 { 1000 if (!refcount_dec_and_test(&entry->refcnt)) 1001 return; 1002 1003 INIT_WORK(&entry->work, mlx5_tc_ct_entry_del_work); 1004 queue_work(entry->ct_priv->wq, &entry->work); 1005 } 1006 1007 static struct mlx5_ct_counter * 1008 mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv) 1009 { 1010 struct mlx5_ct_counter *counter; 1011 int ret; 1012 1013 counter = kzalloc(sizeof(*counter), GFP_KERNEL); 1014 if (!counter) 1015 return ERR_PTR(-ENOMEM); 1016 1017 counter->is_shared = false; 1018 counter->counter = mlx5_fc_create_ex(ct_priv->dev, true); 1019 if (IS_ERR(counter->counter)) { 1020 ct_dbg("Failed to create counter for ct entry"); 1021 ret = PTR_ERR(counter->counter); 1022 kfree(counter); 1023 return ERR_PTR(ret); 1024 } 1025 1026 return counter; 1027 } 1028 1029 static struct mlx5_ct_counter * 1030 mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, 1031 struct mlx5_ct_entry *entry) 1032 { 1033 struct mlx5_ct_tuple rev_tuple = entry->tuple; 1034 struct mlx5_ct_counter *shared_counter; 1035 struct mlx5_ct_entry *rev_entry; 1036 1037 /* get the reversed tuple */ 1038 swap(rev_tuple.port.src, rev_tuple.port.dst); 1039 1040 if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 1041 __be32 tmp_addr = rev_tuple.ip.src_v4; 1042 1043 rev_tuple.ip.src_v4 = rev_tuple.ip.dst_v4; 1044 rev_tuple.ip.dst_v4 = tmp_addr; 1045 } else if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 1046 struct in6_addr tmp_addr = rev_tuple.ip.src_v6; 1047 1048 rev_tuple.ip.src_v6 = rev_tuple.ip.dst_v6; 1049 rev_tuple.ip.dst_v6 = tmp_addr; 1050 } else { 1051 return ERR_PTR(-EOPNOTSUPP); 1052 } 1053 1054 /* Use the same counter as the reverse direction */ 1055 spin_lock_bh(&ct_priv->ht_lock); 1056 rev_entry = mlx5_tc_ct_entry_get(ct_priv, &rev_tuple); 1057 1058 if (IS_ERR(rev_entry)) { 1059 spin_unlock_bh(&ct_priv->ht_lock); 1060 goto create_counter; 1061 } 1062 1063 if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) { 1064 ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry); 1065 shared_counter = rev_entry->counter; 1066 spin_unlock_bh(&ct_priv->ht_lock); 1067 1068 mlx5_tc_ct_entry_put(rev_entry); 1069 return shared_counter; 1070 } 1071 1072 spin_unlock_bh(&ct_priv->ht_lock); 1073 1074 create_counter: 1075 1076 shared_counter = mlx5_tc_ct_counter_create(ct_priv); 1077 if (IS_ERR(shared_counter)) 1078 return shared_counter; 1079 1080 shared_counter->is_shared = true; 1081 refcount_set(&shared_counter->refcount, 1); 1082 return shared_counter; 1083 } 1084 1085 static int 1086 mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv, 1087 struct flow_rule *flow_rule, 1088 struct mlx5_ct_entry *entry, 1089 u8 zone_restore_id) 1090 { 1091 int err; 1092 1093 if (nf_ct_acct_enabled(dev_net(ct_priv->netdev))) 1094 entry->counter = mlx5_tc_ct_counter_create(ct_priv); 1095 else 1096 entry->counter = mlx5_tc_ct_shared_counter_get(ct_priv, entry); 1097 1098 if (IS_ERR(entry->counter)) { 1099 err = PTR_ERR(entry->counter); 1100 return err; 1101 } 1102 1103 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false, 1104 zone_restore_id); 1105 if (err) 1106 goto err_orig; 1107 1108 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true, 1109 zone_restore_id); 1110 if (err) 1111 goto err_nat; 1112 1113 atomic_inc(&ct_priv->debugfs.stats.offloaded); 1114 return 0; 1115 1116 err_nat: 1117 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false); 1118 err_orig: 1119 mlx5_tc_ct_counter_put(ct_priv, entry); 1120 return err; 1121 } 1122 1123 static int 1124 mlx5_tc_ct_entry_replace_rules(struct mlx5_tc_ct_priv *ct_priv, 1125 struct flow_rule *flow_rule, 1126 struct mlx5_ct_entry *entry, 1127 u8 zone_restore_id) 1128 { 1129 int err; 1130 1131 err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, false, 1132 zone_restore_id); 1133 if (err) 1134 return err; 1135 1136 err = mlx5_tc_ct_entry_replace_rule(ct_priv, flow_rule, entry, true, 1137 zone_restore_id); 1138 if (err) 1139 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false); 1140 return err; 1141 } 1142 1143 static int 1144 mlx5_tc_ct_block_flow_offload_replace(struct mlx5_ct_ft *ft, struct flow_rule *flow_rule, 1145 struct mlx5_ct_entry *entry, unsigned long cookie) 1146 { 1147 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; 1148 int err; 1149 1150 err = mlx5_tc_ct_entry_replace_rules(ct_priv, flow_rule, entry, ft->zone_restore_id); 1151 if (!err) 1152 return 0; 1153 1154 /* If failed to update the entry, then look it up again under ht_lock 1155 * protection and properly delete it. 1156 */ 1157 spin_lock_bh(&ct_priv->ht_lock); 1158 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); 1159 if (entry) { 1160 rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params); 1161 spin_unlock_bh(&ct_priv->ht_lock); 1162 mlx5_tc_ct_entry_put(entry); 1163 } else { 1164 spin_unlock_bh(&ct_priv->ht_lock); 1165 } 1166 return err; 1167 } 1168 1169 static int 1170 mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft, 1171 struct flow_cls_offload *flow) 1172 { 1173 struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow); 1174 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; 1175 struct flow_action_entry *meta_action; 1176 unsigned long cookie = flow->cookie; 1177 struct mlx5_ct_entry *entry; 1178 int err; 1179 1180 meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule); 1181 if (!meta_action) 1182 return -EOPNOTSUPP; 1183 1184 spin_lock_bh(&ct_priv->ht_lock); 1185 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); 1186 if (entry && refcount_inc_not_zero(&entry->refcnt)) { 1187 if (entry->restore_cookie == meta_action->ct_metadata.cookie) { 1188 spin_unlock_bh(&ct_priv->ht_lock); 1189 mlx5_tc_ct_entry_put(entry); 1190 return -EEXIST; 1191 } 1192 entry->restore_cookie = meta_action->ct_metadata.cookie; 1193 spin_unlock_bh(&ct_priv->ht_lock); 1194 1195 err = mlx5_tc_ct_block_flow_offload_replace(ft, flow_rule, entry, cookie); 1196 mlx5_tc_ct_entry_put(entry); 1197 return err; 1198 } 1199 spin_unlock_bh(&ct_priv->ht_lock); 1200 1201 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1202 if (!entry) 1203 return -ENOMEM; 1204 1205 entry->tuple.zone = ft->zone; 1206 entry->cookie = flow->cookie; 1207 entry->restore_cookie = meta_action->ct_metadata.cookie; 1208 refcount_set(&entry->refcnt, 2); 1209 entry->ct_priv = ct_priv; 1210 1211 err = mlx5_tc_ct_rule_to_tuple(&entry->tuple, flow_rule); 1212 if (err) 1213 goto err_set; 1214 1215 memcpy(&entry->tuple_nat, &entry->tuple, sizeof(entry->tuple)); 1216 err = mlx5_tc_ct_rule_to_tuple_nat(&entry->tuple_nat, flow_rule); 1217 if (err) 1218 goto err_set; 1219 1220 spin_lock_bh(&ct_priv->ht_lock); 1221 1222 err = rhashtable_lookup_insert_fast(&ft->ct_entries_ht, &entry->node, 1223 cts_ht_params); 1224 if (err) 1225 goto err_entries; 1226 1227 err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_ht, 1228 &entry->tuple_node, 1229 tuples_ht_params); 1230 if (err) 1231 goto err_tuple; 1232 1233 if (memcmp(&entry->tuple, &entry->tuple_nat, sizeof(entry->tuple))) { 1234 err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_nat_ht, 1235 &entry->tuple_nat_node, 1236 tuples_nat_ht_params); 1237 if (err) 1238 goto err_tuple_nat; 1239 } 1240 spin_unlock_bh(&ct_priv->ht_lock); 1241 1242 err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry, 1243 ft->zone_restore_id); 1244 if (err) 1245 goto err_rules; 1246 1247 set_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags); 1248 mlx5_tc_ct_entry_put(entry); /* this function reference */ 1249 1250 return 0; 1251 1252 err_rules: 1253 spin_lock_bh(&ct_priv->ht_lock); 1254 if (mlx5_tc_ct_entry_has_nat(entry)) 1255 rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht, 1256 &entry->tuple_nat_node, tuples_nat_ht_params); 1257 err_tuple_nat: 1258 rhashtable_remove_fast(&ct_priv->ct_tuples_ht, 1259 &entry->tuple_node, 1260 tuples_ht_params); 1261 err_tuple: 1262 rhashtable_remove_fast(&ft->ct_entries_ht, 1263 &entry->node, 1264 cts_ht_params); 1265 err_entries: 1266 spin_unlock_bh(&ct_priv->ht_lock); 1267 err_set: 1268 kfree(entry); 1269 if (err != -EEXIST) 1270 netdev_warn(ct_priv->netdev, "Failed to offload ct entry, err: %d\n", err); 1271 return err; 1272 } 1273 1274 static int 1275 mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft, 1276 struct flow_cls_offload *flow) 1277 { 1278 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; 1279 unsigned long cookie = flow->cookie; 1280 struct mlx5_ct_entry *entry; 1281 1282 spin_lock_bh(&ct_priv->ht_lock); 1283 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); 1284 if (!entry) { 1285 spin_unlock_bh(&ct_priv->ht_lock); 1286 return -ENOENT; 1287 } 1288 1289 if (!mlx5_tc_ct_entry_valid(entry)) { 1290 spin_unlock_bh(&ct_priv->ht_lock); 1291 return -EINVAL; 1292 } 1293 1294 rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params); 1295 spin_unlock_bh(&ct_priv->ht_lock); 1296 1297 mlx5_tc_ct_entry_put(entry); 1298 1299 return 0; 1300 } 1301 1302 static int 1303 mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft, 1304 struct flow_cls_offload *f) 1305 { 1306 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv; 1307 unsigned long cookie = f->cookie; 1308 struct mlx5_ct_entry *entry; 1309 u64 lastuse, packets, bytes; 1310 1311 spin_lock_bh(&ct_priv->ht_lock); 1312 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params); 1313 if (!entry) { 1314 spin_unlock_bh(&ct_priv->ht_lock); 1315 return -ENOENT; 1316 } 1317 1318 if (!mlx5_tc_ct_entry_valid(entry) || !refcount_inc_not_zero(&entry->refcnt)) { 1319 spin_unlock_bh(&ct_priv->ht_lock); 1320 return -EINVAL; 1321 } 1322 1323 spin_unlock_bh(&ct_priv->ht_lock); 1324 1325 mlx5_fc_query_cached(entry->counter->counter, &bytes, &packets, &lastuse); 1326 flow_stats_update(&f->stats, bytes, packets, 0, lastuse, 1327 FLOW_ACTION_HW_STATS_DELAYED); 1328 1329 mlx5_tc_ct_entry_put(entry); 1330 return 0; 1331 } 1332 1333 static int 1334 mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data, 1335 void *cb_priv) 1336 { 1337 struct flow_cls_offload *f = type_data; 1338 struct mlx5_ct_ft *ft = cb_priv; 1339 1340 if (type != TC_SETUP_CLSFLOWER) 1341 return -EOPNOTSUPP; 1342 1343 switch (f->command) { 1344 case FLOW_CLS_REPLACE: 1345 return mlx5_tc_ct_block_flow_offload_add(ft, f); 1346 case FLOW_CLS_DESTROY: 1347 return mlx5_tc_ct_block_flow_offload_del(ft, f); 1348 case FLOW_CLS_STATS: 1349 return mlx5_tc_ct_block_flow_offload_stats(ft, f); 1350 default: 1351 break; 1352 } 1353 1354 return -EOPNOTSUPP; 1355 } 1356 1357 static bool 1358 mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple, 1359 u16 zone) 1360 { 1361 struct flow_keys flow_keys; 1362 1363 skb_reset_network_header(skb); 1364 skb_flow_dissect_flow_keys(skb, &flow_keys, FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP); 1365 1366 tuple->zone = zone; 1367 1368 if (flow_keys.basic.ip_proto != IPPROTO_TCP && 1369 flow_keys.basic.ip_proto != IPPROTO_UDP && 1370 flow_keys.basic.ip_proto != IPPROTO_GRE) 1371 return false; 1372 1373 if (flow_keys.basic.ip_proto == IPPROTO_TCP || 1374 flow_keys.basic.ip_proto == IPPROTO_UDP) { 1375 tuple->port.src = flow_keys.ports.src; 1376 tuple->port.dst = flow_keys.ports.dst; 1377 } 1378 tuple->n_proto = flow_keys.basic.n_proto; 1379 tuple->ip_proto = flow_keys.basic.ip_proto; 1380 1381 switch (flow_keys.basic.n_proto) { 1382 case htons(ETH_P_IP): 1383 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1384 tuple->ip.src_v4 = flow_keys.addrs.v4addrs.src; 1385 tuple->ip.dst_v4 = flow_keys.addrs.v4addrs.dst; 1386 break; 1387 1388 case htons(ETH_P_IPV6): 1389 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1390 tuple->ip.src_v6 = flow_keys.addrs.v6addrs.src; 1391 tuple->ip.dst_v6 = flow_keys.addrs.v6addrs.dst; 1392 break; 1393 default: 1394 goto out; 1395 } 1396 1397 return true; 1398 1399 out: 1400 return false; 1401 } 1402 1403 int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec) 1404 { 1405 u32 ctstate = 0, ctstate_mask = 0; 1406 1407 mlx5e_tc_match_to_reg_get_match(spec, CTSTATE_TO_REG, 1408 &ctstate, &ctstate_mask); 1409 1410 if ((ctstate & ctstate_mask) == MLX5_CT_STATE_TRK_BIT) 1411 return -EOPNOTSUPP; 1412 1413 ctstate_mask |= MLX5_CT_STATE_TRK_BIT; 1414 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, 1415 ctstate, ctstate_mask); 1416 1417 return 0; 1418 } 1419 1420 void mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv *priv, struct mlx5_ct_attr *ct_attr) 1421 { 1422 if (!priv || !ct_attr->ct_labels_id) 1423 return; 1424 1425 mlx5_put_label_mapping(priv, ct_attr->ct_labels_id); 1426 } 1427 1428 int 1429 mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv, 1430 struct mlx5_flow_spec *spec, 1431 struct flow_cls_offload *f, 1432 struct mlx5_ct_attr *ct_attr, 1433 struct netlink_ext_ack *extack) 1434 { 1435 bool trk, est, untrk, unnew, unest, new, rpl, unrpl, rel, unrel, inv, uninv; 1436 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 1437 struct flow_dissector_key_ct *mask, *key; 1438 u32 ctstate = 0, ctstate_mask = 0; 1439 u16 ct_state_on, ct_state_off; 1440 u16 ct_state, ct_state_mask; 1441 struct flow_match_ct match; 1442 u32 ct_labels[4]; 1443 1444 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) 1445 return 0; 1446 1447 if (!priv) { 1448 NL_SET_ERR_MSG_MOD(extack, 1449 "offload of ct matching isn't available"); 1450 return -EOPNOTSUPP; 1451 } 1452 1453 flow_rule_match_ct(rule, &match); 1454 1455 key = match.key; 1456 mask = match.mask; 1457 1458 ct_state = key->ct_state; 1459 ct_state_mask = mask->ct_state; 1460 1461 if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 1462 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED | 1463 TCA_FLOWER_KEY_CT_FLAGS_NEW | 1464 TCA_FLOWER_KEY_CT_FLAGS_REPLY | 1465 TCA_FLOWER_KEY_CT_FLAGS_RELATED | 1466 TCA_FLOWER_KEY_CT_FLAGS_INVALID)) { 1467 NL_SET_ERR_MSG_MOD(extack, 1468 "only ct_state trk, est, new and rpl are supported for offload"); 1469 return -EOPNOTSUPP; 1470 } 1471 1472 ct_state_on = ct_state & ct_state_mask; 1473 ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask; 1474 trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED; 1475 new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW; 1476 est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED; 1477 rpl = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_REPLY; 1478 rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED; 1479 inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID; 1480 untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED; 1481 unnew = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_NEW; 1482 unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED; 1483 unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY; 1484 unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED; 1485 uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID; 1486 1487 ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0; 1488 ctstate |= new ? MLX5_CT_STATE_NEW_BIT : 0; 1489 ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0; 1490 ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0; 1491 ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0; 1492 ctstate_mask |= (unnew || new) ? MLX5_CT_STATE_NEW_BIT : 0; 1493 ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0; 1494 ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0; 1495 ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0; 1496 ctstate_mask |= uninv ? MLX5_CT_STATE_INVALID_BIT : 0; 1497 1498 if (rel) { 1499 NL_SET_ERR_MSG_MOD(extack, 1500 "matching on ct_state +rel isn't supported"); 1501 return -EOPNOTSUPP; 1502 } 1503 1504 if (inv) { 1505 NL_SET_ERR_MSG_MOD(extack, 1506 "matching on ct_state +inv isn't supported"); 1507 return -EOPNOTSUPP; 1508 } 1509 1510 if (mask->ct_zone) 1511 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, 1512 key->ct_zone, MLX5_CT_ZONE_MASK); 1513 if (ctstate_mask) 1514 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, 1515 ctstate, ctstate_mask); 1516 if (mask->ct_mark) 1517 mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG, 1518 key->ct_mark, mask->ct_mark); 1519 if (mask->ct_labels[0] || mask->ct_labels[1] || mask->ct_labels[2] || 1520 mask->ct_labels[3]) { 1521 ct_labels[0] = key->ct_labels[0] & mask->ct_labels[0]; 1522 ct_labels[1] = key->ct_labels[1] & mask->ct_labels[1]; 1523 ct_labels[2] = key->ct_labels[2] & mask->ct_labels[2]; 1524 ct_labels[3] = key->ct_labels[3] & mask->ct_labels[3]; 1525 if (mlx5_get_label_mapping(priv, ct_labels, &ct_attr->ct_labels_id)) 1526 return -EOPNOTSUPP; 1527 mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, ct_attr->ct_labels_id, 1528 MLX5_CT_LABELS_MASK); 1529 } 1530 1531 return 0; 1532 } 1533 1534 int 1535 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv, 1536 struct mlx5_flow_attr *attr, 1537 const struct flow_action_entry *act, 1538 struct netlink_ext_ack *extack) 1539 { 1540 if (!priv) { 1541 NL_SET_ERR_MSG_MOD(extack, 1542 "offload of ct action isn't available"); 1543 return -EOPNOTSUPP; 1544 } 1545 1546 attr->ct_attr.ct_action |= act->ct.action; /* So we can have clear + ct */ 1547 attr->ct_attr.zone = act->ct.zone; 1548 attr->ct_attr.nf_ft = act->ct.flow_table; 1549 attr->ct_attr.act_miss_cookie = act->miss_cookie; 1550 1551 return 0; 1552 } 1553 1554 static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft, 1555 struct mlx5_tc_ct_pre *pre_ct, 1556 bool nat) 1557 { 1558 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv; 1559 struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {}; 1560 struct mlx5_core_dev *dev = ct_priv->dev; 1561 struct mlx5_flow_table *ft = pre_ct->ft; 1562 struct mlx5_flow_destination dest = {}; 1563 struct mlx5_flow_act flow_act = {}; 1564 struct mlx5_modify_hdr *mod_hdr; 1565 struct mlx5_flow_handle *rule; 1566 struct mlx5_flow_spec *spec; 1567 u32 ctstate; 1568 u16 zone; 1569 int err; 1570 1571 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1572 if (!spec) 1573 return -ENOMEM; 1574 1575 zone = ct_ft->zone & MLX5_CT_ZONE_MASK; 1576 err = mlx5e_tc_match_to_reg_set(dev, &pre_mod_acts, ct_priv->ns_type, 1577 ZONE_TO_REG, zone); 1578 if (err) { 1579 ct_dbg("Failed to set zone register mapping"); 1580 goto err_mapping; 1581 } 1582 1583 mod_hdr = mlx5_modify_header_alloc(dev, ct_priv->ns_type, 1584 pre_mod_acts.num_actions, 1585 pre_mod_acts.actions); 1586 1587 if (IS_ERR(mod_hdr)) { 1588 err = PTR_ERR(mod_hdr); 1589 ct_dbg("Failed to create pre ct mod hdr"); 1590 goto err_mapping; 1591 } 1592 pre_ct->modify_hdr = mod_hdr; 1593 1594 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1595 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1596 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 1597 flow_act.modify_hdr = mod_hdr; 1598 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1599 1600 /* add flow rule */ 1601 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, 1602 zone, MLX5_CT_ZONE_MASK); 1603 ctstate = MLX5_CT_STATE_TRK_BIT; 1604 if (nat) 1605 ctstate |= MLX5_CT_STATE_NAT_BIT; 1606 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, ctstate, ctstate); 1607 1608 dest.ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act); 1609 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1610 if (IS_ERR(rule)) { 1611 err = PTR_ERR(rule); 1612 ct_dbg("Failed to add pre ct flow rule zone %d", zone); 1613 goto err_flow_rule; 1614 } 1615 pre_ct->flow_rule = rule; 1616 1617 /* add miss rule */ 1618 dest.ft = nat ? ct_priv->ct_nat : ct_priv->ct; 1619 rule = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1); 1620 if (IS_ERR(rule)) { 1621 err = PTR_ERR(rule); 1622 ct_dbg("Failed to add pre ct miss rule zone %d", zone); 1623 goto err_miss_rule; 1624 } 1625 pre_ct->miss_rule = rule; 1626 1627 mlx5e_mod_hdr_dealloc(&pre_mod_acts); 1628 kvfree(spec); 1629 return 0; 1630 1631 err_miss_rule: 1632 mlx5_del_flow_rules(pre_ct->flow_rule); 1633 err_flow_rule: 1634 mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr); 1635 err_mapping: 1636 mlx5e_mod_hdr_dealloc(&pre_mod_acts); 1637 kvfree(spec); 1638 return err; 1639 } 1640 1641 static void 1642 tc_ct_pre_ct_del_rules(struct mlx5_ct_ft *ct_ft, 1643 struct mlx5_tc_ct_pre *pre_ct) 1644 { 1645 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv; 1646 struct mlx5_core_dev *dev = ct_priv->dev; 1647 1648 mlx5_del_flow_rules(pre_ct->flow_rule); 1649 mlx5_del_flow_rules(pre_ct->miss_rule); 1650 mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr); 1651 } 1652 1653 static int 1654 mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft *ct_ft, 1655 struct mlx5_tc_ct_pre *pre_ct, 1656 bool nat) 1657 { 1658 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1659 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv; 1660 struct mlx5_core_dev *dev = ct_priv->dev; 1661 struct mlx5_flow_table_attr ft_attr = {}; 1662 struct mlx5_flow_namespace *ns; 1663 struct mlx5_flow_table *ft; 1664 struct mlx5_flow_group *g; 1665 u32 metadata_reg_c_2_mask; 1666 u32 *flow_group_in; 1667 void *misc; 1668 int err; 1669 1670 ns = mlx5_get_flow_namespace(dev, ct_priv->ns_type); 1671 if (!ns) { 1672 err = -EOPNOTSUPP; 1673 ct_dbg("Failed to get flow namespace"); 1674 return err; 1675 } 1676 1677 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1678 if (!flow_group_in) 1679 return -ENOMEM; 1680 1681 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; 1682 ft_attr.prio = ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB ? 1683 FDB_TC_OFFLOAD : MLX5E_TC_PRIO; 1684 ft_attr.max_fte = 2; 1685 ft_attr.level = 1; 1686 ft = mlx5_create_flow_table(ns, &ft_attr); 1687 if (IS_ERR(ft)) { 1688 err = PTR_ERR(ft); 1689 ct_dbg("Failed to create pre ct table"); 1690 goto out_free; 1691 } 1692 pre_ct->ft = ft; 1693 1694 /* create flow group */ 1695 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1696 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0); 1697 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1698 MLX5_MATCH_MISC_PARAMETERS_2); 1699 1700 misc = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1701 match_criteria.misc_parameters_2); 1702 1703 metadata_reg_c_2_mask = MLX5_CT_ZONE_MASK; 1704 metadata_reg_c_2_mask |= (MLX5_CT_STATE_TRK_BIT << 16); 1705 if (nat) 1706 metadata_reg_c_2_mask |= (MLX5_CT_STATE_NAT_BIT << 16); 1707 1708 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_2, 1709 metadata_reg_c_2_mask); 1710 1711 g = mlx5_create_flow_group(ft, flow_group_in); 1712 if (IS_ERR(g)) { 1713 err = PTR_ERR(g); 1714 ct_dbg("Failed to create pre ct group"); 1715 goto err_flow_grp; 1716 } 1717 pre_ct->flow_grp = g; 1718 1719 /* create miss group */ 1720 memset(flow_group_in, 0, inlen); 1721 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1); 1722 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); 1723 g = mlx5_create_flow_group(ft, flow_group_in); 1724 if (IS_ERR(g)) { 1725 err = PTR_ERR(g); 1726 ct_dbg("Failed to create pre ct miss group"); 1727 goto err_miss_grp; 1728 } 1729 pre_ct->miss_grp = g; 1730 1731 err = tc_ct_pre_ct_add_rules(ct_ft, pre_ct, nat); 1732 if (err) 1733 goto err_add_rules; 1734 1735 kvfree(flow_group_in); 1736 return 0; 1737 1738 err_add_rules: 1739 mlx5_destroy_flow_group(pre_ct->miss_grp); 1740 err_miss_grp: 1741 mlx5_destroy_flow_group(pre_ct->flow_grp); 1742 err_flow_grp: 1743 mlx5_destroy_flow_table(ft); 1744 out_free: 1745 kvfree(flow_group_in); 1746 return err; 1747 } 1748 1749 static void 1750 mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft *ct_ft, 1751 struct mlx5_tc_ct_pre *pre_ct) 1752 { 1753 tc_ct_pre_ct_del_rules(ct_ft, pre_ct); 1754 mlx5_destroy_flow_group(pre_ct->miss_grp); 1755 mlx5_destroy_flow_group(pre_ct->flow_grp); 1756 mlx5_destroy_flow_table(pre_ct->ft); 1757 } 1758 1759 static int 1760 mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft *ft) 1761 { 1762 int err; 1763 1764 err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct, false); 1765 if (err) 1766 return err; 1767 1768 err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct_nat, true); 1769 if (err) 1770 goto err_pre_ct_nat; 1771 1772 return 0; 1773 1774 err_pre_ct_nat: 1775 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct); 1776 return err; 1777 } 1778 1779 static void 1780 mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft *ft) 1781 { 1782 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct_nat); 1783 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct); 1784 } 1785 1786 /* To avoid false lock dependency warning set the ct_entries_ht lock 1787 * class different than the lock class of the ht being used when deleting 1788 * last flow from a group and then deleting a group, we get into del_sw_flow_group() 1789 * which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but 1790 * it's different than the ht->mutex here. 1791 */ 1792 static struct lock_class_key ct_entries_ht_lock_key; 1793 1794 static struct mlx5_ct_ft * 1795 mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone, 1796 struct nf_flowtable *nf_ft) 1797 { 1798 struct mlx5_ct_ft *ft; 1799 int err; 1800 1801 ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params); 1802 if (ft) { 1803 refcount_inc(&ft->refcount); 1804 return ft; 1805 } 1806 1807 ft = kzalloc(sizeof(*ft), GFP_KERNEL); 1808 if (!ft) 1809 return ERR_PTR(-ENOMEM); 1810 1811 err = mapping_add(ct_priv->zone_mapping, &zone, &ft->zone_restore_id); 1812 if (err) 1813 goto err_mapping; 1814 1815 ft->zone = zone; 1816 ft->nf_ft = nf_ft; 1817 ft->ct_priv = ct_priv; 1818 refcount_set(&ft->refcount, 1); 1819 1820 err = mlx5_tc_ct_alloc_pre_ct_tables(ft); 1821 if (err) 1822 goto err_alloc_pre_ct; 1823 1824 err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params); 1825 if (err) 1826 goto err_init; 1827 1828 lockdep_set_class(&ft->ct_entries_ht.mutex, &ct_entries_ht_lock_key); 1829 1830 err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node, 1831 zone_params); 1832 if (err) 1833 goto err_insert; 1834 1835 err = nf_flow_table_offload_add_cb(ft->nf_ft, 1836 mlx5_tc_ct_block_flow_offload, ft); 1837 if (err) 1838 goto err_add_cb; 1839 1840 return ft; 1841 1842 err_add_cb: 1843 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params); 1844 err_insert: 1845 rhashtable_destroy(&ft->ct_entries_ht); 1846 err_init: 1847 mlx5_tc_ct_free_pre_ct_tables(ft); 1848 err_alloc_pre_ct: 1849 mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id); 1850 err_mapping: 1851 kfree(ft); 1852 return ERR_PTR(err); 1853 } 1854 1855 static void 1856 mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg) 1857 { 1858 struct mlx5_ct_entry *entry = ptr; 1859 1860 mlx5_tc_ct_entry_put(entry); 1861 } 1862 1863 static void 1864 mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft) 1865 { 1866 if (!refcount_dec_and_test(&ft->refcount)) 1867 return; 1868 1869 flush_workqueue(ct_priv->wq); 1870 nf_flow_table_offload_del_cb(ft->nf_ft, 1871 mlx5_tc_ct_block_flow_offload, ft); 1872 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params); 1873 rhashtable_free_and_destroy(&ft->ct_entries_ht, 1874 mlx5_tc_ct_flush_ft_entry, 1875 ct_priv); 1876 mlx5_tc_ct_free_pre_ct_tables(ft); 1877 mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id); 1878 kfree(ft); 1879 } 1880 1881 /* We translate the tc filter with CT action to the following HW model: 1882 * 1883 * +-----------------------+ 1884 * + rule (either original + 1885 * + or post_act rule) + 1886 * +-----------------------+ 1887 * | set act_miss_cookie mapping 1888 * | set fte_id 1889 * | set tunnel_id 1890 * | rest of actions before the CT action (for this orig/post_act rule) 1891 * | 1892 * +-------------+ 1893 * | Chain 0 | 1894 * | optimization| 1895 * | v 1896 * | +---------------------+ 1897 * | + pre_ct/pre_ct_nat + if matches +----------------------+ 1898 * | + zone+nat match +---------------->+ post_act (see below) + 1899 * | +---------------------+ set zone +----------------------+ 1900 * | | 1901 * +-------------+ set zone 1902 * | 1903 * v 1904 * +--------------------+ 1905 * + CT (nat or no nat) + 1906 * + tuple + zone match + 1907 * +--------------------+ 1908 * | set mark 1909 * | set labels_id 1910 * | set established 1911 * | set zone_restore 1912 * | do nat (if needed) 1913 * v 1914 * +--------------+ 1915 * + post_act + rest of parsed filter's actions 1916 * + fte_id match +------------------------> 1917 * +--------------+ 1918 * 1919 */ 1920 static int 1921 __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, 1922 struct mlx5_flow_attr *attr) 1923 { 1924 bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT; 1925 struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev); 1926 int act_miss_mapping = 0, err; 1927 struct mlx5_ct_ft *ft; 1928 u16 zone; 1929 1930 /* Register for CT established events */ 1931 ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone, 1932 attr->ct_attr.nf_ft); 1933 if (IS_ERR(ft)) { 1934 err = PTR_ERR(ft); 1935 ct_dbg("Failed to register to ft callback"); 1936 goto err_ft; 1937 } 1938 attr->ct_attr.ft = ft; 1939 1940 err = mlx5e_tc_action_miss_mapping_get(ct_priv->priv, attr, attr->ct_attr.act_miss_cookie, 1941 &act_miss_mapping); 1942 if (err) { 1943 ct_dbg("Failed to get register mapping for act miss"); 1944 goto err_get_act_miss; 1945 } 1946 1947 err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts, 1948 ct_priv->ns_type, MAPPED_OBJ_TO_REG, act_miss_mapping); 1949 if (err) { 1950 ct_dbg("Failed to set act miss register mapping"); 1951 goto err_mapping; 1952 } 1953 1954 /* Chain 0 sets the zone and jumps to ct table 1955 * Other chains jump to pre_ct table to align with act_ct cached logic 1956 */ 1957 if (!attr->chain) { 1958 zone = ft->zone & MLX5_CT_ZONE_MASK; 1959 err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts, 1960 ct_priv->ns_type, ZONE_TO_REG, zone); 1961 if (err) { 1962 ct_dbg("Failed to set zone register mapping"); 1963 goto err_mapping; 1964 } 1965 1966 attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct; 1967 } else { 1968 attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft; 1969 } 1970 1971 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1972 attr->ct_attr.act_miss_mapping = act_miss_mapping; 1973 1974 return 0; 1975 1976 err_mapping: 1977 mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, act_miss_mapping); 1978 err_get_act_miss: 1979 mlx5_tc_ct_del_ft_cb(ct_priv, ft); 1980 err_ft: 1981 netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err); 1982 return err; 1983 } 1984 1985 int 1986 mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr) 1987 { 1988 int err; 1989 1990 if (!priv) 1991 return -EOPNOTSUPP; 1992 1993 if (attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR) { 1994 err = mlx5_tc_ct_entry_set_registers(priv, &attr->parse_attr->mod_hdr_acts, 1995 0, 0, 0, 0); 1996 if (err) 1997 return err; 1998 1999 attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 2000 } 2001 2002 if (!attr->ct_attr.nf_ft) /* means only ct clear action, and not ct_clear,ct() */ 2003 return 0; 2004 2005 mutex_lock(&priv->control_lock); 2006 err = __mlx5_tc_ct_flow_offload(priv, attr); 2007 mutex_unlock(&priv->control_lock); 2008 2009 return err; 2010 } 2011 2012 static void 2013 __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv, 2014 struct mlx5_flow_attr *attr) 2015 { 2016 mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, attr->ct_attr.act_miss_mapping); 2017 mlx5_tc_ct_del_ft_cb(ct_priv, attr->ct_attr.ft); 2018 } 2019 2020 void 2021 mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv, 2022 struct mlx5_flow_attr *attr) 2023 { 2024 if (!attr->ct_attr.nf_ft) /* means only ct clear action, and not ct_clear,ct() */ 2025 return; 2026 2027 mutex_lock(&priv->control_lock); 2028 __mlx5_tc_ct_delete_flow(priv, attr); 2029 mutex_unlock(&priv->control_lock); 2030 } 2031 2032 static int 2033 mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv) 2034 { 2035 struct mlx5_flow_table *post_ct = mlx5e_tc_post_act_get_ft(ct_priv->post_act); 2036 struct mlx5_ct_fs_ops *fs_ops = mlx5_ct_fs_dmfs_ops_get(); 2037 int err; 2038 2039 if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB && 2040 ct_priv->dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS) { 2041 ct_dbg("Using SMFS ct flow steering provider"); 2042 fs_ops = mlx5_ct_fs_smfs_ops_get(); 2043 } 2044 2045 ct_priv->fs = kzalloc(sizeof(*ct_priv->fs) + fs_ops->priv_size, GFP_KERNEL); 2046 if (!ct_priv->fs) 2047 return -ENOMEM; 2048 2049 ct_priv->fs->netdev = ct_priv->netdev; 2050 ct_priv->fs->dev = ct_priv->dev; 2051 ct_priv->fs_ops = fs_ops; 2052 2053 err = ct_priv->fs_ops->init(ct_priv->fs, ct_priv->ct, ct_priv->ct_nat, post_ct); 2054 if (err) 2055 goto err_init; 2056 2057 return 0; 2058 2059 err_init: 2060 kfree(ct_priv->fs); 2061 return err; 2062 } 2063 2064 static int 2065 mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw, 2066 const char **err_msg) 2067 { 2068 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) { 2069 /* vlan workaround should be avoided for multi chain rules. 2070 * This is just a sanity check as pop vlan action should 2071 * be supported by any FW that supports ignore_flow_level 2072 */ 2073 2074 *err_msg = "firmware vlan actions support is missing"; 2075 return -EOPNOTSUPP; 2076 } 2077 2078 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, 2079 fdb_modify_header_fwd_to_table)) { 2080 /* CT always writes to registers which are mod header actions. 2081 * Therefore, mod header and goto is required 2082 */ 2083 2084 *err_msg = "firmware fwd and modify support is missing"; 2085 return -EOPNOTSUPP; 2086 } 2087 2088 if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 2089 *err_msg = "register loopback isn't supported"; 2090 return -EOPNOTSUPP; 2091 } 2092 2093 return 0; 2094 } 2095 2096 static int 2097 mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv, 2098 enum mlx5_flow_namespace_type ns_type, 2099 struct mlx5e_post_act *post_act) 2100 { 2101 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 2102 const char *err_msg = NULL; 2103 int err = 0; 2104 2105 if (IS_ERR_OR_NULL(post_act)) { 2106 /* Ignore_flow_level support isn't supported by default for VFs and so post_act 2107 * won't be supported. Skip showing error msg. 2108 */ 2109 if (priv->mdev->coredev_type == MLX5_COREDEV_PF) 2110 err_msg = "post action is missing"; 2111 err = -EOPNOTSUPP; 2112 goto out_err; 2113 } 2114 2115 if (ns_type == MLX5_FLOW_NAMESPACE_FDB) 2116 err = mlx5_tc_ct_init_check_esw_support(esw, &err_msg); 2117 2118 out_err: 2119 if (err && err_msg) 2120 netdev_dbg(priv->netdev, "tc ct offload not supported, %s\n", err_msg); 2121 return err; 2122 } 2123 2124 static void 2125 mlx5_ct_tc_create_dbgfs(struct mlx5_tc_ct_priv *ct_priv) 2126 { 2127 struct mlx5_tc_ct_debugfs *ct_dbgfs = &ct_priv->debugfs; 2128 2129 ct_dbgfs->root = debugfs_create_dir("ct", mlx5_debugfs_get_dev_root(ct_priv->dev)); 2130 debugfs_create_atomic_t("offloaded", 0400, ct_dbgfs->root, 2131 &ct_dbgfs->stats.offloaded); 2132 debugfs_create_atomic_t("rx_dropped", 0400, ct_dbgfs->root, 2133 &ct_dbgfs->stats.rx_dropped); 2134 } 2135 2136 static void 2137 mlx5_ct_tc_remove_dbgfs(struct mlx5_tc_ct_priv *ct_priv) 2138 { 2139 debugfs_remove_recursive(ct_priv->debugfs.root); 2140 } 2141 2142 #define INIT_ERR_PREFIX "tc ct offload init failed" 2143 2144 struct mlx5_tc_ct_priv * 2145 mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, 2146 struct mod_hdr_tbl *mod_hdr, 2147 enum mlx5_flow_namespace_type ns_type, 2148 struct mlx5e_post_act *post_act) 2149 { 2150 struct mlx5_tc_ct_priv *ct_priv; 2151 struct mlx5_core_dev *dev; 2152 u64 mapping_id; 2153 int err; 2154 2155 dev = priv->mdev; 2156 err = mlx5_tc_ct_init_check_support(priv, ns_type, post_act); 2157 if (err) 2158 goto err_support; 2159 2160 ct_priv = kzalloc(sizeof(*ct_priv), GFP_KERNEL); 2161 if (!ct_priv) 2162 goto err_alloc; 2163 2164 mapping_id = mlx5_query_nic_system_image_guid(dev); 2165 2166 ct_priv->zone_mapping = mapping_create_for_id(mapping_id, MAPPING_TYPE_ZONE, 2167 sizeof(u16), 0, true); 2168 if (IS_ERR(ct_priv->zone_mapping)) { 2169 err = PTR_ERR(ct_priv->zone_mapping); 2170 goto err_mapping_zone; 2171 } 2172 2173 ct_priv->labels_mapping = mapping_create_for_id(mapping_id, MAPPING_TYPE_LABELS, 2174 sizeof(u32) * 4, 0, true); 2175 if (IS_ERR(ct_priv->labels_mapping)) { 2176 err = PTR_ERR(ct_priv->labels_mapping); 2177 goto err_mapping_labels; 2178 } 2179 2180 spin_lock_init(&ct_priv->ht_lock); 2181 ct_priv->priv = priv; 2182 ct_priv->ns_type = ns_type; 2183 ct_priv->chains = chains; 2184 ct_priv->netdev = priv->netdev; 2185 ct_priv->dev = priv->mdev; 2186 ct_priv->mod_hdr_tbl = mod_hdr; 2187 ct_priv->ct = mlx5_chains_create_global_table(chains); 2188 if (IS_ERR(ct_priv->ct)) { 2189 err = PTR_ERR(ct_priv->ct); 2190 mlx5_core_warn(dev, 2191 "%s, failed to create ct table err: %d\n", 2192 INIT_ERR_PREFIX, err); 2193 goto err_ct_tbl; 2194 } 2195 2196 ct_priv->ct_nat = mlx5_chains_create_global_table(chains); 2197 if (IS_ERR(ct_priv->ct_nat)) { 2198 err = PTR_ERR(ct_priv->ct_nat); 2199 mlx5_core_warn(dev, 2200 "%s, failed to create ct nat table err: %d\n", 2201 INIT_ERR_PREFIX, err); 2202 goto err_ct_nat_tbl; 2203 } 2204 2205 ct_priv->post_act = post_act; 2206 mutex_init(&ct_priv->control_lock); 2207 if (rhashtable_init(&ct_priv->zone_ht, &zone_params)) 2208 goto err_ct_zone_ht; 2209 if (rhashtable_init(&ct_priv->ct_tuples_ht, &tuples_ht_params)) 2210 goto err_ct_tuples_ht; 2211 if (rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params)) 2212 goto err_ct_tuples_nat_ht; 2213 2214 ct_priv->wq = alloc_ordered_workqueue("mlx5e_ct_priv_wq", 0); 2215 if (!ct_priv->wq) { 2216 err = -ENOMEM; 2217 goto err_wq; 2218 } 2219 2220 err = mlx5_tc_ct_fs_init(ct_priv); 2221 if (err) 2222 goto err_init_fs; 2223 2224 mlx5_ct_tc_create_dbgfs(ct_priv); 2225 return ct_priv; 2226 2227 err_init_fs: 2228 destroy_workqueue(ct_priv->wq); 2229 err_wq: 2230 rhashtable_destroy(&ct_priv->ct_tuples_nat_ht); 2231 err_ct_tuples_nat_ht: 2232 rhashtable_destroy(&ct_priv->ct_tuples_ht); 2233 err_ct_tuples_ht: 2234 rhashtable_destroy(&ct_priv->zone_ht); 2235 err_ct_zone_ht: 2236 mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat); 2237 err_ct_nat_tbl: 2238 mlx5_chains_destroy_global_table(chains, ct_priv->ct); 2239 err_ct_tbl: 2240 mapping_destroy(ct_priv->labels_mapping); 2241 err_mapping_labels: 2242 mapping_destroy(ct_priv->zone_mapping); 2243 err_mapping_zone: 2244 kfree(ct_priv); 2245 err_alloc: 2246 err_support: 2247 2248 return NULL; 2249 } 2250 2251 void 2252 mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv) 2253 { 2254 struct mlx5_fs_chains *chains; 2255 2256 if (!ct_priv) 2257 return; 2258 2259 destroy_workqueue(ct_priv->wq); 2260 mlx5_ct_tc_remove_dbgfs(ct_priv); 2261 chains = ct_priv->chains; 2262 2263 ct_priv->fs_ops->destroy(ct_priv->fs); 2264 kfree(ct_priv->fs); 2265 2266 mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat); 2267 mlx5_chains_destroy_global_table(chains, ct_priv->ct); 2268 mapping_destroy(ct_priv->zone_mapping); 2269 mapping_destroy(ct_priv->labels_mapping); 2270 2271 rhashtable_destroy(&ct_priv->ct_tuples_ht); 2272 rhashtable_destroy(&ct_priv->ct_tuples_nat_ht); 2273 rhashtable_destroy(&ct_priv->zone_ht); 2274 mutex_destroy(&ct_priv->control_lock); 2275 kfree(ct_priv); 2276 } 2277 2278 bool 2279 mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv, 2280 struct sk_buff *skb, u8 zone_restore_id) 2281 { 2282 struct mlx5_ct_tuple tuple = {}; 2283 struct mlx5_ct_entry *entry; 2284 u16 zone; 2285 2286 if (!ct_priv || !zone_restore_id) 2287 return true; 2288 2289 if (mapping_find(ct_priv->zone_mapping, zone_restore_id, &zone)) 2290 goto out_inc_drop; 2291 2292 if (!mlx5_tc_ct_skb_to_tuple(skb, &tuple, zone)) 2293 goto out_inc_drop; 2294 2295 spin_lock(&ct_priv->ht_lock); 2296 2297 entry = mlx5_tc_ct_entry_get(ct_priv, &tuple); 2298 if (!entry) { 2299 spin_unlock(&ct_priv->ht_lock); 2300 goto out_inc_drop; 2301 } 2302 2303 if (IS_ERR(entry)) { 2304 spin_unlock(&ct_priv->ht_lock); 2305 goto out_inc_drop; 2306 } 2307 spin_unlock(&ct_priv->ht_lock); 2308 2309 tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie); 2310 __mlx5_tc_ct_entry_put(entry); 2311 2312 return true; 2313 2314 out_inc_drop: 2315 atomic_inc(&ct_priv->debugfs.stats.rx_dropped); 2316 return false; 2317 } 2318