1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2021 Corigine, Inc. */ 3 4 #include "conntrack.h" 5 6 const struct rhashtable_params nfp_tc_ct_merge_params = { 7 .head_offset = offsetof(struct nfp_fl_ct_tc_merge, 8 hash_node), 9 .key_len = sizeof(unsigned long) * 2, 10 .key_offset = offsetof(struct nfp_fl_ct_tc_merge, cookie), 11 .automatic_shrinking = true, 12 }; 13 14 const struct rhashtable_params nfp_nft_ct_merge_params = { 15 .head_offset = offsetof(struct nfp_fl_nft_tc_merge, 16 hash_node), 17 .key_len = sizeof(unsigned long) * 3, 18 .key_offset = offsetof(struct nfp_fl_nft_tc_merge, cookie), 19 .automatic_shrinking = true, 20 }; 21 22 static struct flow_action_entry *get_flow_act(struct flow_rule *rule, 23 enum flow_action_id act_id); 24 25 /** 26 * get_hashentry() - Wrapper around hashtable lookup. 27 * @ht: hashtable where entry could be found 28 * @key: key to lookup 29 * @params: hashtable params 30 * @size: size of entry to allocate if not in table 31 * 32 * Returns an entry from a hashtable. If entry does not exist 33 * yet allocate the memory for it and return the new entry. 34 */ 35 static void *get_hashentry(struct rhashtable *ht, void *key, 36 const struct rhashtable_params params, size_t size) 37 { 38 void *result; 39 40 result = rhashtable_lookup_fast(ht, key, params); 41 42 if (result) 43 return result; 44 45 result = kzalloc(size, GFP_KERNEL); 46 if (!result) 47 return ERR_PTR(-ENOMEM); 48 49 return result; 50 } 51 52 bool is_pre_ct_flow(struct flow_cls_offload *flow) 53 { 54 struct flow_action_entry *act; 55 int i; 56 57 flow_action_for_each(i, act, &flow->rule->action) { 58 if (act->id == FLOW_ACTION_CT && !act->ct.action) 59 return true; 60 } 61 return false; 62 } 63 64 bool is_post_ct_flow(struct flow_cls_offload *flow) 65 { 66 struct flow_rule *rule = flow_cls_offload_flow_rule(flow); 67 struct flow_dissector *dissector = rule->match.dissector; 68 struct flow_match_ct ct; 69 70 if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) { 71 flow_rule_match_ct(rule, &ct); 72 if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) 73 return true; 74 } 75 return false; 76 } 77 78 static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1, 79 struct nfp_fl_ct_flow_entry *entry2) 80 { 81 unsigned int ovlp_keys = entry1->rule->match.dissector->used_keys & 82 entry2->rule->match.dissector->used_keys; 83 bool out; 84 85 /* check the overlapped fields one by one, the unmasked part 86 * should not conflict with each other. 87 */ 88 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) { 89 struct flow_match_control match1, match2; 90 91 flow_rule_match_control(entry1->rule, &match1); 92 flow_rule_match_control(entry2->rule, &match2); 93 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 94 if (out) 95 goto check_failed; 96 } 97 98 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) { 99 struct flow_match_basic match1, match2; 100 101 flow_rule_match_basic(entry1->rule, &match1); 102 flow_rule_match_basic(entry2->rule, &match2); 103 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 104 if (out) 105 goto check_failed; 106 } 107 108 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 109 struct flow_match_ipv4_addrs match1, match2; 110 111 flow_rule_match_ipv4_addrs(entry1->rule, &match1); 112 flow_rule_match_ipv4_addrs(entry2->rule, &match2); 113 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 114 if (out) 115 goto check_failed; 116 } 117 118 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 119 struct flow_match_ipv6_addrs match1, match2; 120 121 flow_rule_match_ipv6_addrs(entry1->rule, &match1); 122 flow_rule_match_ipv6_addrs(entry2->rule, &match2); 123 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 124 if (out) 125 goto check_failed; 126 } 127 128 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) { 129 struct flow_match_ports match1, match2; 130 131 flow_rule_match_ports(entry1->rule, &match1); 132 flow_rule_match_ports(entry2->rule, &match2); 133 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 134 if (out) 135 goto check_failed; 136 } 137 138 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 139 struct flow_match_eth_addrs match1, match2; 140 141 flow_rule_match_eth_addrs(entry1->rule, &match1); 142 flow_rule_match_eth_addrs(entry2->rule, &match2); 143 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 144 if (out) 145 goto check_failed; 146 } 147 148 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_VLAN)) { 149 struct flow_match_vlan match1, match2; 150 151 flow_rule_match_vlan(entry1->rule, &match1); 152 flow_rule_match_vlan(entry2->rule, &match2); 153 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 154 if (out) 155 goto check_failed; 156 } 157 158 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_MPLS)) { 159 struct flow_match_mpls match1, match2; 160 161 flow_rule_match_mpls(entry1->rule, &match1); 162 flow_rule_match_mpls(entry2->rule, &match2); 163 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 164 if (out) 165 goto check_failed; 166 } 167 168 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_TCP)) { 169 struct flow_match_tcp match1, match2; 170 171 flow_rule_match_tcp(entry1->rule, &match1); 172 flow_rule_match_tcp(entry2->rule, &match2); 173 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 174 if (out) 175 goto check_failed; 176 } 177 178 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IP)) { 179 struct flow_match_ip match1, match2; 180 181 flow_rule_match_ip(entry1->rule, &match1); 182 flow_rule_match_ip(entry2->rule, &match2); 183 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 184 if (out) 185 goto check_failed; 186 } 187 188 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_KEYID)) { 189 struct flow_match_enc_keyid match1, match2; 190 191 flow_rule_match_enc_keyid(entry1->rule, &match1); 192 flow_rule_match_enc_keyid(entry2->rule, &match2); 193 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 194 if (out) 195 goto check_failed; 196 } 197 198 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) { 199 struct flow_match_ipv4_addrs match1, match2; 200 201 flow_rule_match_enc_ipv4_addrs(entry1->rule, &match1); 202 flow_rule_match_enc_ipv4_addrs(entry2->rule, &match2); 203 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 204 if (out) 205 goto check_failed; 206 } 207 208 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) { 209 struct flow_match_ipv6_addrs match1, match2; 210 211 flow_rule_match_enc_ipv6_addrs(entry1->rule, &match1); 212 flow_rule_match_enc_ipv6_addrs(entry2->rule, &match2); 213 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 214 if (out) 215 goto check_failed; 216 } 217 218 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL)) { 219 struct flow_match_control match1, match2; 220 221 flow_rule_match_enc_control(entry1->rule, &match1); 222 flow_rule_match_enc_control(entry2->rule, &match2); 223 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 224 if (out) 225 goto check_failed; 226 } 227 228 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_IP)) { 229 struct flow_match_ip match1, match2; 230 231 flow_rule_match_enc_ip(entry1->rule, &match1); 232 flow_rule_match_enc_ip(entry2->rule, &match2); 233 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 234 if (out) 235 goto check_failed; 236 } 237 238 if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_ENC_OPTS)) { 239 struct flow_match_enc_opts match1, match2; 240 241 flow_rule_match_enc_opts(entry1->rule, &match1); 242 flow_rule_match_enc_opts(entry2->rule, &match2); 243 COMPARE_UNMASKED_FIELDS(match1, match2, &out); 244 if (out) 245 goto check_failed; 246 } 247 248 return 0; 249 250 check_failed: 251 return -EINVAL; 252 } 253 254 static int nfp_ct_check_mangle_merge(struct flow_action_entry *a_in, 255 struct flow_rule *rule) 256 { 257 enum flow_action_mangle_base htype = a_in->mangle.htype; 258 u32 offset = a_in->mangle.offset; 259 260 switch (htype) { 261 case FLOW_ACT_MANGLE_HDR_TYPE_ETH: 262 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) 263 return -EOPNOTSUPP; 264 break; 265 case FLOW_ACT_MANGLE_HDR_TYPE_IP4: 266 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { 267 struct flow_match_ip match; 268 269 flow_rule_match_ip(rule, &match); 270 if (offset == offsetof(struct iphdr, ttl) && 271 match.mask->ttl) 272 return -EOPNOTSUPP; 273 if (offset == round_down(offsetof(struct iphdr, tos), 4) && 274 match.mask->tos) 275 return -EOPNOTSUPP; 276 } 277 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) { 278 struct flow_match_ipv4_addrs match; 279 280 flow_rule_match_ipv4_addrs(rule, &match); 281 if (offset == offsetof(struct iphdr, saddr) && 282 match.mask->src) 283 return -EOPNOTSUPP; 284 if (offset == offsetof(struct iphdr, daddr) && 285 match.mask->dst) 286 return -EOPNOTSUPP; 287 } 288 break; 289 case FLOW_ACT_MANGLE_HDR_TYPE_IP6: 290 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) { 291 struct flow_match_ip match; 292 293 flow_rule_match_ip(rule, &match); 294 if (offset == round_down(offsetof(struct ipv6hdr, hop_limit), 4) && 295 match.mask->ttl) 296 return -EOPNOTSUPP; 297 /* for ipv6, tos and flow_lbl are in the same word */ 298 if (offset == round_down(offsetof(struct ipv6hdr, flow_lbl), 4) && 299 match.mask->tos) 300 return -EOPNOTSUPP; 301 } 302 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IPV6_ADDRS)) { 303 struct flow_match_ipv6_addrs match; 304 305 flow_rule_match_ipv6_addrs(rule, &match); 306 if (offset >= offsetof(struct ipv6hdr, saddr) && 307 offset < offsetof(struct ipv6hdr, daddr) && 308 memchr_inv(&match.mask->src, 0, sizeof(match.mask->src))) 309 return -EOPNOTSUPP; 310 if (offset >= offsetof(struct ipv6hdr, daddr) && 311 offset < sizeof(struct ipv6hdr) && 312 memchr_inv(&match.mask->dst, 0, sizeof(match.mask->dst))) 313 return -EOPNOTSUPP; 314 } 315 break; 316 case FLOW_ACT_MANGLE_HDR_TYPE_TCP: 317 case FLOW_ACT_MANGLE_HDR_TYPE_UDP: 318 /* currently only can modify ports */ 319 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) 320 return -EOPNOTSUPP; 321 break; 322 default: 323 break; 324 } 325 return 0; 326 } 327 328 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry, 329 struct nfp_fl_ct_flow_entry *post_ct_entry, 330 struct nfp_fl_ct_flow_entry *nft_entry) 331 { 332 struct flow_action_entry *act; 333 int err, i; 334 335 /* Check for pre_ct->action conflicts */ 336 flow_action_for_each(i, act, &pre_ct_entry->rule->action) { 337 switch (act->id) { 338 case FLOW_ACTION_MANGLE: 339 err = nfp_ct_check_mangle_merge(act, nft_entry->rule); 340 if (err) 341 return err; 342 err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule); 343 if (err) 344 return err; 345 break; 346 case FLOW_ACTION_VLAN_PUSH: 347 case FLOW_ACTION_VLAN_POP: 348 case FLOW_ACTION_VLAN_MANGLE: 349 case FLOW_ACTION_MPLS_PUSH: 350 case FLOW_ACTION_MPLS_POP: 351 case FLOW_ACTION_MPLS_MANGLE: 352 return -EOPNOTSUPP; 353 default: 354 break; 355 } 356 } 357 358 /* Check for nft->action conflicts */ 359 flow_action_for_each(i, act, &nft_entry->rule->action) { 360 switch (act->id) { 361 case FLOW_ACTION_MANGLE: 362 err = nfp_ct_check_mangle_merge(act, post_ct_entry->rule); 363 if (err) 364 return err; 365 break; 366 case FLOW_ACTION_VLAN_PUSH: 367 case FLOW_ACTION_VLAN_POP: 368 case FLOW_ACTION_VLAN_MANGLE: 369 case FLOW_ACTION_MPLS_PUSH: 370 case FLOW_ACTION_MPLS_POP: 371 case FLOW_ACTION_MPLS_MANGLE: 372 return -EOPNOTSUPP; 373 default: 374 break; 375 } 376 } 377 return 0; 378 } 379 380 static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry, 381 struct nfp_fl_ct_flow_entry *nft_entry) 382 { 383 struct flow_dissector *dissector = post_ct_entry->rule->match.dissector; 384 struct flow_action_entry *ct_met; 385 struct flow_match_ct ct; 386 int i; 387 388 ct_met = get_flow_act(nft_entry->rule, FLOW_ACTION_CT_METADATA); 389 if (ct_met && (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))) { 390 u32 *act_lbl; 391 392 act_lbl = ct_met->ct_metadata.labels; 393 flow_rule_match_ct(post_ct_entry->rule, &ct); 394 for (i = 0; i < 4; i++) { 395 if ((ct.key->ct_labels[i] & ct.mask->ct_labels[i]) ^ 396 (act_lbl[i] & ct.mask->ct_labels[i])) 397 return -EINVAL; 398 } 399 400 if ((ct.key->ct_mark & ct.mask->ct_mark) ^ 401 (ct_met->ct_metadata.mark & ct.mask->ct_mark)) 402 return -EINVAL; 403 404 return 0; 405 } 406 407 return -EINVAL; 408 } 409 410 static int nfp_fl_ct_add_offload(struct nfp_fl_nft_tc_merge *m_entry) 411 { 412 return 0; 413 } 414 415 static int nfp_fl_ct_del_offload(struct nfp_app *app, unsigned long cookie, 416 struct net_device *netdev) 417 { 418 return 0; 419 } 420 421 static int nfp_ct_do_nft_merge(struct nfp_fl_ct_zone_entry *zt, 422 struct nfp_fl_ct_flow_entry *nft_entry, 423 struct nfp_fl_ct_tc_merge *tc_m_entry) 424 { 425 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; 426 struct nfp_fl_nft_tc_merge *nft_m_entry; 427 unsigned long new_cookie[3]; 428 int err; 429 430 pre_ct_entry = tc_m_entry->pre_ct_parent; 431 post_ct_entry = tc_m_entry->post_ct_parent; 432 433 err = nfp_ct_merge_act_check(pre_ct_entry, post_ct_entry, nft_entry); 434 if (err) 435 return err; 436 437 /* Check that the two tc flows are also compatible with 438 * the nft entry. No need to check the pre_ct and post_ct 439 * entries as that was already done during pre_merge. 440 * The nft entry does not have a netdev or chain populated, so 441 * skip this check. 442 */ 443 err = nfp_ct_merge_check(pre_ct_entry, nft_entry); 444 if (err) 445 return err; 446 err = nfp_ct_merge_check(post_ct_entry, nft_entry); 447 if (err) 448 return err; 449 err = nfp_ct_check_meta(post_ct_entry, nft_entry); 450 if (err) 451 return err; 452 453 /* Combine tc_merge and nft cookies for this cookie. */ 454 new_cookie[0] = tc_m_entry->cookie[0]; 455 new_cookie[1] = tc_m_entry->cookie[1]; 456 new_cookie[2] = nft_entry->cookie; 457 nft_m_entry = get_hashentry(&zt->nft_merge_tb, 458 &new_cookie, 459 nfp_nft_ct_merge_params, 460 sizeof(*nft_m_entry)); 461 462 if (IS_ERR(nft_m_entry)) 463 return PTR_ERR(nft_m_entry); 464 465 /* nft_m_entry already present, not merging again */ 466 if (!memcmp(&new_cookie, nft_m_entry->cookie, sizeof(new_cookie))) 467 return 0; 468 469 memcpy(&nft_m_entry->cookie, &new_cookie, sizeof(new_cookie)); 470 nft_m_entry->zt = zt; 471 nft_m_entry->tc_m_parent = tc_m_entry; 472 nft_m_entry->nft_parent = nft_entry; 473 nft_m_entry->tc_flower_cookie = 0; 474 /* Copy the netdev from one the pre_ct entry. When the tc_m_entry was created 475 * it only combined them if the netdevs were the same, so can use any of them. 476 */ 477 nft_m_entry->netdev = pre_ct_entry->netdev; 478 479 /* Add this entry to the tc_m_list and nft_flow lists */ 480 list_add(&nft_m_entry->tc_merge_list, &tc_m_entry->children); 481 list_add(&nft_m_entry->nft_flow_list, &nft_entry->children); 482 483 /* Generate offload structure and send to nfp */ 484 err = nfp_fl_ct_add_offload(nft_m_entry); 485 if (err) 486 goto err_nft_ct_offload; 487 488 err = rhashtable_insert_fast(&zt->nft_merge_tb, &nft_m_entry->hash_node, 489 nfp_nft_ct_merge_params); 490 if (err) 491 goto err_nft_ct_merge_insert; 492 493 zt->nft_merge_count++; 494 495 return err; 496 497 err_nft_ct_merge_insert: 498 nfp_fl_ct_del_offload(zt->priv->app, nft_m_entry->tc_flower_cookie, 499 nft_m_entry->netdev); 500 err_nft_ct_offload: 501 list_del(&nft_m_entry->tc_merge_list); 502 list_del(&nft_m_entry->nft_flow_list); 503 kfree(nft_m_entry); 504 return err; 505 } 506 507 static int nfp_ct_do_tc_merge(struct nfp_fl_ct_zone_entry *zt, 508 struct nfp_fl_ct_flow_entry *ct_entry1, 509 struct nfp_fl_ct_flow_entry *ct_entry2) 510 { 511 struct nfp_fl_ct_flow_entry *post_ct_entry, *pre_ct_entry; 512 struct nfp_fl_ct_flow_entry *nft_entry, *nft_tmp; 513 struct nfp_fl_ct_tc_merge *m_entry; 514 unsigned long new_cookie[2]; 515 int err; 516 517 if (ct_entry1->type == CT_TYPE_PRE_CT) { 518 pre_ct_entry = ct_entry1; 519 post_ct_entry = ct_entry2; 520 } else { 521 post_ct_entry = ct_entry1; 522 pre_ct_entry = ct_entry2; 523 } 524 525 if (post_ct_entry->netdev != pre_ct_entry->netdev) 526 return -EINVAL; 527 /* Checks that the chain_index of the filter matches the 528 * chain_index of the GOTO action. 529 */ 530 if (post_ct_entry->chain_index != pre_ct_entry->chain_index) 531 return -EINVAL; 532 533 err = nfp_ct_merge_check(post_ct_entry, pre_ct_entry); 534 if (err) 535 return err; 536 537 new_cookie[0] = pre_ct_entry->cookie; 538 new_cookie[1] = post_ct_entry->cookie; 539 m_entry = get_hashentry(&zt->tc_merge_tb, &new_cookie, 540 nfp_tc_ct_merge_params, sizeof(*m_entry)); 541 if (IS_ERR(m_entry)) 542 return PTR_ERR(m_entry); 543 544 /* m_entry already present, not merging again */ 545 if (!memcmp(&new_cookie, m_entry->cookie, sizeof(new_cookie))) 546 return 0; 547 548 memcpy(&m_entry->cookie, &new_cookie, sizeof(new_cookie)); 549 m_entry->zt = zt; 550 m_entry->post_ct_parent = post_ct_entry; 551 m_entry->pre_ct_parent = pre_ct_entry; 552 553 /* Add this entry to the pre_ct and post_ct lists */ 554 list_add(&m_entry->post_ct_list, &post_ct_entry->children); 555 list_add(&m_entry->pre_ct_list, &pre_ct_entry->children); 556 INIT_LIST_HEAD(&m_entry->children); 557 558 err = rhashtable_insert_fast(&zt->tc_merge_tb, &m_entry->hash_node, 559 nfp_tc_ct_merge_params); 560 if (err) 561 goto err_ct_tc_merge_insert; 562 zt->tc_merge_count++; 563 564 /* Merge with existing nft flows */ 565 list_for_each_entry_safe(nft_entry, nft_tmp, &zt->nft_flows_list, 566 list_node) { 567 nfp_ct_do_nft_merge(zt, nft_entry, m_entry); 568 } 569 570 return 0; 571 572 err_ct_tc_merge_insert: 573 list_del(&m_entry->post_ct_list); 574 list_del(&m_entry->pre_ct_list); 575 kfree(m_entry); 576 return err; 577 } 578 579 static struct 580 nfp_fl_ct_zone_entry *get_nfp_zone_entry(struct nfp_flower_priv *priv, 581 u16 zone, bool wildcarded) 582 { 583 struct nfp_fl_ct_zone_entry *zt; 584 int err; 585 586 if (wildcarded && priv->ct_zone_wc) 587 return priv->ct_zone_wc; 588 589 if (!wildcarded) { 590 zt = get_hashentry(&priv->ct_zone_table, &zone, 591 nfp_zone_table_params, sizeof(*zt)); 592 593 /* If priv is set this is an existing entry, just return it */ 594 if (IS_ERR(zt) || zt->priv) 595 return zt; 596 } else { 597 zt = kzalloc(sizeof(*zt), GFP_KERNEL); 598 if (!zt) 599 return ERR_PTR(-ENOMEM); 600 } 601 602 zt->zone = zone; 603 zt->priv = priv; 604 zt->nft = NULL; 605 606 /* init the various hash tables and lists*/ 607 INIT_LIST_HEAD(&zt->pre_ct_list); 608 INIT_LIST_HEAD(&zt->post_ct_list); 609 INIT_LIST_HEAD(&zt->nft_flows_list); 610 611 err = rhashtable_init(&zt->tc_merge_tb, &nfp_tc_ct_merge_params); 612 if (err) 613 goto err_tc_merge_tb_init; 614 615 err = rhashtable_init(&zt->nft_merge_tb, &nfp_nft_ct_merge_params); 616 if (err) 617 goto err_nft_merge_tb_init; 618 619 if (wildcarded) { 620 priv->ct_zone_wc = zt; 621 } else { 622 err = rhashtable_insert_fast(&priv->ct_zone_table, 623 &zt->hash_node, 624 nfp_zone_table_params); 625 if (err) 626 goto err_zone_insert; 627 } 628 629 return zt; 630 631 err_zone_insert: 632 rhashtable_destroy(&zt->nft_merge_tb); 633 err_nft_merge_tb_init: 634 rhashtable_destroy(&zt->tc_merge_tb); 635 err_tc_merge_tb_init: 636 kfree(zt); 637 return ERR_PTR(err); 638 } 639 640 static struct 641 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt, 642 struct net_device *netdev, 643 struct flow_cls_offload *flow, 644 bool is_nft, struct netlink_ext_ack *extack) 645 { 646 struct nf_flow_match *nft_match = NULL; 647 struct nfp_fl_ct_flow_entry *entry; 648 struct nfp_fl_ct_map_entry *map; 649 struct flow_action_entry *act; 650 int err, i; 651 652 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 653 if (!entry) 654 return ERR_PTR(-ENOMEM); 655 656 entry->rule = flow_rule_alloc(flow->rule->action.num_entries); 657 if (!entry->rule) { 658 err = -ENOMEM; 659 goto err_pre_ct_rule; 660 } 661 662 /* nft flows gets destroyed after callback return, so need 663 * to do a full copy instead of just a reference. 664 */ 665 if (is_nft) { 666 nft_match = kzalloc(sizeof(*nft_match), GFP_KERNEL); 667 if (!nft_match) { 668 err = -ENOMEM; 669 goto err_pre_ct_act; 670 } 671 memcpy(&nft_match->dissector, flow->rule->match.dissector, 672 sizeof(nft_match->dissector)); 673 memcpy(&nft_match->mask, flow->rule->match.mask, 674 sizeof(nft_match->mask)); 675 memcpy(&nft_match->key, flow->rule->match.key, 676 sizeof(nft_match->key)); 677 entry->rule->match.dissector = &nft_match->dissector; 678 entry->rule->match.mask = &nft_match->mask; 679 entry->rule->match.key = &nft_match->key; 680 } else { 681 entry->rule->match.dissector = flow->rule->match.dissector; 682 entry->rule->match.mask = flow->rule->match.mask; 683 entry->rule->match.key = flow->rule->match.key; 684 } 685 686 entry->zt = zt; 687 entry->netdev = netdev; 688 entry->cookie = flow->cookie; 689 entry->chain_index = flow->common.chain_index; 690 entry->tun_offset = NFP_FL_CT_NO_TUN; 691 692 /* Copy over action data. Unfortunately we do not get a handle to the 693 * original tcf_action data, and the flow objects gets destroyed, so we 694 * cannot just save a pointer to this either, so need to copy over the 695 * data unfortunately. 696 */ 697 entry->rule->action.num_entries = flow->rule->action.num_entries; 698 flow_action_for_each(i, act, &flow->rule->action) { 699 struct flow_action_entry *new_act; 700 701 new_act = &entry->rule->action.entries[i]; 702 memcpy(new_act, act, sizeof(struct flow_action_entry)); 703 /* Entunnel is a special case, need to allocate and copy 704 * tunnel info. 705 */ 706 if (act->id == FLOW_ACTION_TUNNEL_ENCAP) { 707 struct ip_tunnel_info *tun = act->tunnel; 708 size_t tun_size = sizeof(*tun) + tun->options_len; 709 710 new_act->tunnel = kmemdup(tun, tun_size, GFP_ATOMIC); 711 if (!new_act->tunnel) { 712 err = -ENOMEM; 713 goto err_pre_ct_tun_cp; 714 } 715 entry->tun_offset = i; 716 } 717 } 718 719 INIT_LIST_HEAD(&entry->children); 720 721 /* Now add a ct map entry to flower-priv */ 722 map = get_hashentry(&zt->priv->ct_map_table, &flow->cookie, 723 nfp_ct_map_params, sizeof(*map)); 724 if (IS_ERR(map)) { 725 NL_SET_ERR_MSG_MOD(extack, 726 "offload error: ct map entry creation failed"); 727 err = -ENOMEM; 728 goto err_ct_flow_insert; 729 } 730 map->cookie = flow->cookie; 731 map->ct_entry = entry; 732 err = rhashtable_insert_fast(&zt->priv->ct_map_table, 733 &map->hash_node, 734 nfp_ct_map_params); 735 if (err) { 736 NL_SET_ERR_MSG_MOD(extack, 737 "offload error: ct map entry table add failed"); 738 goto err_map_insert; 739 } 740 741 return entry; 742 743 err_map_insert: 744 kfree(map); 745 err_ct_flow_insert: 746 if (entry->tun_offset != NFP_FL_CT_NO_TUN) 747 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); 748 err_pre_ct_tun_cp: 749 kfree(nft_match); 750 err_pre_ct_act: 751 kfree(entry->rule); 752 err_pre_ct_rule: 753 kfree(entry); 754 return ERR_PTR(err); 755 } 756 757 static void cleanup_nft_merge_entry(struct nfp_fl_nft_tc_merge *m_entry) 758 { 759 struct nfp_fl_ct_zone_entry *zt; 760 int err; 761 762 zt = m_entry->zt; 763 764 /* Flow is in HW, need to delete */ 765 if (m_entry->tc_flower_cookie) { 766 err = nfp_fl_ct_del_offload(zt->priv->app, m_entry->tc_flower_cookie, 767 m_entry->netdev); 768 if (err) 769 return; 770 } 771 772 WARN_ON_ONCE(rhashtable_remove_fast(&zt->nft_merge_tb, 773 &m_entry->hash_node, 774 nfp_nft_ct_merge_params)); 775 zt->nft_merge_count--; 776 list_del(&m_entry->tc_merge_list); 777 list_del(&m_entry->nft_flow_list); 778 779 kfree(m_entry); 780 } 781 782 static void nfp_free_nft_merge_children(void *entry, bool is_nft_flow) 783 { 784 struct nfp_fl_nft_tc_merge *m_entry, *tmp; 785 786 /* These post entries are parts of two lists, one is a list of nft_entries 787 * and the other is of from a list of tc_merge structures. Iterate 788 * through the relevant list and cleanup the entries. 789 */ 790 791 if (is_nft_flow) { 792 /* Need to iterate through list of nft_flow entries*/ 793 struct nfp_fl_ct_flow_entry *ct_entry = entry; 794 795 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, 796 nft_flow_list) { 797 cleanup_nft_merge_entry(m_entry); 798 } 799 } else { 800 /* Need to iterate through list of tc_merged_flow entries*/ 801 struct nfp_fl_ct_tc_merge *ct_entry = entry; 802 803 list_for_each_entry_safe(m_entry, tmp, &ct_entry->children, 804 tc_merge_list) { 805 cleanup_nft_merge_entry(m_entry); 806 } 807 } 808 } 809 810 static void nfp_del_tc_merge_entry(struct nfp_fl_ct_tc_merge *m_ent) 811 { 812 struct nfp_fl_ct_zone_entry *zt; 813 int err; 814 815 zt = m_ent->zt; 816 err = rhashtable_remove_fast(&zt->tc_merge_tb, 817 &m_ent->hash_node, 818 nfp_tc_ct_merge_params); 819 if (err) 820 pr_warn("WARNING: could not remove merge_entry from hashtable\n"); 821 zt->tc_merge_count--; 822 list_del(&m_ent->post_ct_list); 823 list_del(&m_ent->pre_ct_list); 824 825 if (!list_empty(&m_ent->children)) 826 nfp_free_nft_merge_children(m_ent, false); 827 kfree(m_ent); 828 } 829 830 static void nfp_free_tc_merge_children(struct nfp_fl_ct_flow_entry *entry) 831 { 832 struct nfp_fl_ct_tc_merge *m_ent, *tmp; 833 834 switch (entry->type) { 835 case CT_TYPE_PRE_CT: 836 list_for_each_entry_safe(m_ent, tmp, &entry->children, pre_ct_list) { 837 nfp_del_tc_merge_entry(m_ent); 838 } 839 break; 840 case CT_TYPE_POST_CT: 841 list_for_each_entry_safe(m_ent, tmp, &entry->children, post_ct_list) { 842 nfp_del_tc_merge_entry(m_ent); 843 } 844 break; 845 default: 846 break; 847 } 848 } 849 850 void nfp_fl_ct_clean_flow_entry(struct nfp_fl_ct_flow_entry *entry) 851 { 852 list_del(&entry->list_node); 853 854 if (!list_empty(&entry->children)) { 855 if (entry->type == CT_TYPE_NFT) 856 nfp_free_nft_merge_children(entry, true); 857 else 858 nfp_free_tc_merge_children(entry); 859 } 860 861 if (entry->tun_offset != NFP_FL_CT_NO_TUN) 862 kfree(entry->rule->action.entries[entry->tun_offset].tunnel); 863 864 if (entry->type == CT_TYPE_NFT) { 865 struct nf_flow_match *nft_match; 866 867 nft_match = container_of(entry->rule->match.dissector, 868 struct nf_flow_match, dissector); 869 kfree(nft_match); 870 } 871 872 kfree(entry->rule); 873 kfree(entry); 874 } 875 876 static struct flow_action_entry *get_flow_act(struct flow_rule *rule, 877 enum flow_action_id act_id) 878 { 879 struct flow_action_entry *act = NULL; 880 int i; 881 882 flow_action_for_each(i, act, &rule->action) { 883 if (act->id == act_id) 884 return act; 885 } 886 return NULL; 887 } 888 889 static void 890 nfp_ct_merge_tc_entries(struct nfp_fl_ct_flow_entry *ct_entry1, 891 struct nfp_fl_ct_zone_entry *zt_src, 892 struct nfp_fl_ct_zone_entry *zt_dst) 893 { 894 struct nfp_fl_ct_flow_entry *ct_entry2, *ct_tmp; 895 struct list_head *ct_list; 896 897 if (ct_entry1->type == CT_TYPE_PRE_CT) 898 ct_list = &zt_src->post_ct_list; 899 else if (ct_entry1->type == CT_TYPE_POST_CT) 900 ct_list = &zt_src->pre_ct_list; 901 else 902 return; 903 904 list_for_each_entry_safe(ct_entry2, ct_tmp, ct_list, 905 list_node) { 906 nfp_ct_do_tc_merge(zt_dst, ct_entry2, ct_entry1); 907 } 908 } 909 910 static void 911 nfp_ct_merge_nft_with_tc(struct nfp_fl_ct_flow_entry *nft_entry, 912 struct nfp_fl_ct_zone_entry *zt) 913 { 914 struct nfp_fl_ct_tc_merge *tc_merge_entry; 915 struct rhashtable_iter iter; 916 917 rhashtable_walk_enter(&zt->tc_merge_tb, &iter); 918 rhashtable_walk_start(&iter); 919 while ((tc_merge_entry = rhashtable_walk_next(&iter)) != NULL) { 920 if (IS_ERR(tc_merge_entry)) 921 continue; 922 rhashtable_walk_stop(&iter); 923 nfp_ct_do_nft_merge(zt, nft_entry, tc_merge_entry); 924 rhashtable_walk_start(&iter); 925 } 926 rhashtable_walk_stop(&iter); 927 rhashtable_walk_exit(&iter); 928 } 929 930 int nfp_fl_ct_handle_pre_ct(struct nfp_flower_priv *priv, 931 struct net_device *netdev, 932 struct flow_cls_offload *flow, 933 struct netlink_ext_ack *extack) 934 { 935 struct flow_action_entry *ct_act, *ct_goto; 936 struct nfp_fl_ct_flow_entry *ct_entry; 937 struct nfp_fl_ct_zone_entry *zt; 938 int err; 939 940 ct_act = get_flow_act(flow->rule, FLOW_ACTION_CT); 941 if (!ct_act) { 942 NL_SET_ERR_MSG_MOD(extack, 943 "unsupported offload: Conntrack action empty in conntrack offload"); 944 return -EOPNOTSUPP; 945 } 946 947 ct_goto = get_flow_act(flow->rule, FLOW_ACTION_GOTO); 948 if (!ct_goto) { 949 NL_SET_ERR_MSG_MOD(extack, 950 "unsupported offload: Conntrack requires ACTION_GOTO"); 951 return -EOPNOTSUPP; 952 } 953 954 zt = get_nfp_zone_entry(priv, ct_act->ct.zone, false); 955 if (IS_ERR(zt)) { 956 NL_SET_ERR_MSG_MOD(extack, 957 "offload error: Could not create zone table entry"); 958 return PTR_ERR(zt); 959 } 960 961 if (!zt->nft) { 962 zt->nft = ct_act->ct.flow_table; 963 err = nf_flow_table_offload_add_cb(zt->nft, nfp_fl_ct_handle_nft_flow, zt); 964 if (err) { 965 NL_SET_ERR_MSG_MOD(extack, 966 "offload error: Could not register nft_callback"); 967 return err; 968 } 969 } 970 971 /* Add entry to pre_ct_list */ 972 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); 973 if (IS_ERR(ct_entry)) 974 return PTR_ERR(ct_entry); 975 ct_entry->type = CT_TYPE_PRE_CT; 976 ct_entry->chain_index = ct_goto->chain_index; 977 list_add(&ct_entry->list_node, &zt->pre_ct_list); 978 zt->pre_ct_count++; 979 980 nfp_ct_merge_tc_entries(ct_entry, zt, zt); 981 982 /* Need to check and merge with tables in the wc_zone as well */ 983 if (priv->ct_zone_wc) 984 nfp_ct_merge_tc_entries(ct_entry, priv->ct_zone_wc, zt); 985 986 return 0; 987 } 988 989 int nfp_fl_ct_handle_post_ct(struct nfp_flower_priv *priv, 990 struct net_device *netdev, 991 struct flow_cls_offload *flow, 992 struct netlink_ext_ack *extack) 993 { 994 struct flow_rule *rule = flow_cls_offload_flow_rule(flow); 995 struct nfp_fl_ct_flow_entry *ct_entry; 996 struct nfp_fl_ct_zone_entry *zt; 997 bool wildcarded = false; 998 struct flow_match_ct ct; 999 1000 flow_rule_match_ct(rule, &ct); 1001 if (!ct.mask->ct_zone) { 1002 wildcarded = true; 1003 } else if (ct.mask->ct_zone != U16_MAX) { 1004 NL_SET_ERR_MSG_MOD(extack, 1005 "unsupported offload: partially wildcarded ct_zone is not supported"); 1006 return -EOPNOTSUPP; 1007 } 1008 1009 zt = get_nfp_zone_entry(priv, ct.key->ct_zone, wildcarded); 1010 if (IS_ERR(zt)) { 1011 NL_SET_ERR_MSG_MOD(extack, 1012 "offload error: Could not create zone table entry"); 1013 return PTR_ERR(zt); 1014 } 1015 1016 /* Add entry to post_ct_list */ 1017 ct_entry = nfp_fl_ct_add_flow(zt, netdev, flow, false, extack); 1018 if (IS_ERR(ct_entry)) 1019 return PTR_ERR(ct_entry); 1020 1021 ct_entry->type = CT_TYPE_POST_CT; 1022 ct_entry->chain_index = flow->common.chain_index; 1023 list_add(&ct_entry->list_node, &zt->post_ct_list); 1024 zt->post_ct_count++; 1025 1026 if (wildcarded) { 1027 /* Iterate through all zone tables if not empty, look for merges with 1028 * pre_ct entries and merge them. 1029 */ 1030 struct rhashtable_iter iter; 1031 struct nfp_fl_ct_zone_entry *zone_table; 1032 1033 rhashtable_walk_enter(&priv->ct_zone_table, &iter); 1034 rhashtable_walk_start(&iter); 1035 while ((zone_table = rhashtable_walk_next(&iter)) != NULL) { 1036 if (IS_ERR(zone_table)) 1037 continue; 1038 rhashtable_walk_stop(&iter); 1039 nfp_ct_merge_tc_entries(ct_entry, zone_table, zone_table); 1040 rhashtable_walk_start(&iter); 1041 } 1042 rhashtable_walk_stop(&iter); 1043 rhashtable_walk_exit(&iter); 1044 } else { 1045 nfp_ct_merge_tc_entries(ct_entry, zt, zt); 1046 } 1047 1048 return 0; 1049 } 1050 1051 static int 1052 nfp_fl_ct_offload_nft_flow(struct nfp_fl_ct_zone_entry *zt, struct flow_cls_offload *flow) 1053 { 1054 struct nfp_fl_ct_map_entry *ct_map_ent; 1055 struct nfp_fl_ct_flow_entry *ct_entry; 1056 struct netlink_ext_ack *extack = NULL; 1057 1058 ASSERT_RTNL(); 1059 1060 extack = flow->common.extack; 1061 switch (flow->command) { 1062 case FLOW_CLS_REPLACE: 1063 /* Netfilter can request offload multiple times for the same 1064 * flow - protect against adding duplicates. 1065 */ 1066 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, 1067 nfp_ct_map_params); 1068 if (!ct_map_ent) { 1069 ct_entry = nfp_fl_ct_add_flow(zt, NULL, flow, true, extack); 1070 if (IS_ERR(ct_entry)) 1071 return PTR_ERR(ct_entry); 1072 ct_entry->type = CT_TYPE_NFT; 1073 list_add(&ct_entry->list_node, &zt->nft_flows_list); 1074 zt->nft_flows_count++; 1075 nfp_ct_merge_nft_with_tc(ct_entry, zt); 1076 } 1077 return 0; 1078 case FLOW_CLS_DESTROY: 1079 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, &flow->cookie, 1080 nfp_ct_map_params); 1081 return nfp_fl_ct_del_flow(ct_map_ent); 1082 case FLOW_CLS_STATS: 1083 return 0; 1084 default: 1085 break; 1086 } 1087 return -EINVAL; 1088 } 1089 1090 int nfp_fl_ct_handle_nft_flow(enum tc_setup_type type, void *type_data, void *cb_priv) 1091 { 1092 struct flow_cls_offload *flow = type_data; 1093 struct nfp_fl_ct_zone_entry *zt = cb_priv; 1094 int err = -EOPNOTSUPP; 1095 1096 switch (type) { 1097 case TC_SETUP_CLSFLOWER: 1098 rtnl_lock(); 1099 err = nfp_fl_ct_offload_nft_flow(zt, flow); 1100 rtnl_unlock(); 1101 break; 1102 default: 1103 return -EOPNOTSUPP; 1104 } 1105 return err; 1106 } 1107 1108 static void 1109 nfp_fl_ct_clean_nft_entries(struct nfp_fl_ct_zone_entry *zt) 1110 { 1111 struct nfp_fl_ct_flow_entry *nft_entry, *ct_tmp; 1112 struct nfp_fl_ct_map_entry *ct_map_ent; 1113 1114 list_for_each_entry_safe(nft_entry, ct_tmp, &zt->nft_flows_list, 1115 list_node) { 1116 ct_map_ent = rhashtable_lookup_fast(&zt->priv->ct_map_table, 1117 &nft_entry->cookie, 1118 nfp_ct_map_params); 1119 nfp_fl_ct_del_flow(ct_map_ent); 1120 } 1121 } 1122 1123 int nfp_fl_ct_del_flow(struct nfp_fl_ct_map_entry *ct_map_ent) 1124 { 1125 struct nfp_fl_ct_flow_entry *ct_entry; 1126 struct nfp_fl_ct_zone_entry *zt; 1127 struct rhashtable *m_table; 1128 1129 if (!ct_map_ent) 1130 return -ENOENT; 1131 1132 zt = ct_map_ent->ct_entry->zt; 1133 ct_entry = ct_map_ent->ct_entry; 1134 m_table = &zt->priv->ct_map_table; 1135 1136 switch (ct_entry->type) { 1137 case CT_TYPE_PRE_CT: 1138 zt->pre_ct_count--; 1139 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 1140 nfp_ct_map_params); 1141 nfp_fl_ct_clean_flow_entry(ct_entry); 1142 kfree(ct_map_ent); 1143 1144 if (!zt->pre_ct_count) { 1145 zt->nft = NULL; 1146 nfp_fl_ct_clean_nft_entries(zt); 1147 } 1148 break; 1149 case CT_TYPE_POST_CT: 1150 zt->post_ct_count--; 1151 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 1152 nfp_ct_map_params); 1153 nfp_fl_ct_clean_flow_entry(ct_entry); 1154 kfree(ct_map_ent); 1155 break; 1156 case CT_TYPE_NFT: 1157 zt->nft_flows_count--; 1158 rhashtable_remove_fast(m_table, &ct_map_ent->hash_node, 1159 nfp_ct_map_params); 1160 nfp_fl_ct_clean_flow_entry(ct_map_ent->ct_entry); 1161 kfree(ct_map_ent); 1162 break; 1163 default: 1164 break; 1165 } 1166 1167 return 0; 1168 } 1169