1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <linux/hash.h> 5 #include <linux/hashtable.h> 6 #include <linux/jhash.h> 7 #include <linux/math64.h> 8 #include <linux/vmalloc.h> 9 #include <net/pkt_cls.h> 10 11 #include "cmsg.h" 12 #include "conntrack.h" 13 #include "main.h" 14 #include "../nfp_app.h" 15 16 struct nfp_mask_id_table { 17 struct hlist_node link; 18 u32 hash_key; 19 u32 ref_cnt; 20 u8 mask_id; 21 }; 22 23 struct nfp_fl_flow_table_cmp_arg { 24 struct net_device *netdev; 25 unsigned long cookie; 26 }; 27 28 struct nfp_fl_stats_ctx_to_flow { 29 struct rhash_head ht_node; 30 u32 stats_cxt; 31 struct nfp_fl_payload *flow; 32 }; 33 34 static const struct rhashtable_params stats_ctx_table_params = { 35 .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), 36 .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), 37 .key_len = sizeof(u32), 38 }; 39 40 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) 41 { 42 struct nfp_flower_priv *priv = app->priv; 43 struct circ_buf *ring; 44 45 ring = &priv->stats_ids.free_list; 46 /* Check if buffer is full. */ 47 if (!CIRC_SPACE(ring->head, ring->tail, 48 priv->stats_ring_size * NFP_FL_STATS_ELEM_RS - 49 NFP_FL_STATS_ELEM_RS + 1)) 50 return -ENOBUFS; 51 52 memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS); 53 ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) % 54 (priv->stats_ring_size * NFP_FL_STATS_ELEM_RS); 55 56 return 0; 57 } 58 59 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) 60 { 61 struct nfp_flower_priv *priv = app->priv; 62 u32 freed_stats_id, temp_stats_id; 63 struct circ_buf *ring; 64 65 ring = &priv->stats_ids.free_list; 66 freed_stats_id = priv->stats_ring_size; 67 /* Check for unallocated entries first. */ 68 if (priv->stats_ids.init_unalloc > 0) { 69 *stats_context_id = 70 FIELD_PREP(NFP_FL_STAT_ID_STAT, 71 priv->stats_ids.init_unalloc - 1) | 72 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, 73 priv->active_mem_unit); 74 75 if (++priv->active_mem_unit == priv->total_mem_units) { 76 priv->stats_ids.init_unalloc--; 77 priv->active_mem_unit = 0; 78 } 79 80 return 0; 81 } 82 83 /* Check if buffer is empty. */ 84 if (ring->head == ring->tail) { 85 *stats_context_id = freed_stats_id; 86 return -ENOENT; 87 } 88 89 memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS); 90 *stats_context_id = temp_stats_id; 91 memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS); 92 ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) % 93 (priv->stats_ring_size * NFP_FL_STATS_ELEM_RS); 94 95 return 0; 96 } 97 98 /* Must be called with either RTNL or rcu_read_lock */ 99 struct nfp_fl_payload * 100 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, 101 struct net_device *netdev) 102 { 103 struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; 104 struct nfp_flower_priv *priv = app->priv; 105 106 flower_cmp_arg.netdev = netdev; 107 flower_cmp_arg.cookie = tc_flower_cookie; 108 109 return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, 110 nfp_flower_table_params); 111 } 112 113 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) 114 { 115 unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); 116 struct nfp_flower_priv *priv = app->priv; 117 struct nfp_fl_stats_frame *stats; 118 unsigned char *msg; 119 u32 ctx_id; 120 int i; 121 122 msg = nfp_flower_cmsg_get_data(skb); 123 124 spin_lock(&priv->stats_lock); 125 for (i = 0; i < msg_len / sizeof(*stats); i++) { 126 stats = (struct nfp_fl_stats_frame *)msg + i; 127 ctx_id = be32_to_cpu(stats->stats_con_id); 128 priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count); 129 priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count); 130 priv->stats[ctx_id].used = jiffies; 131 } 132 spin_unlock(&priv->stats_lock); 133 } 134 135 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) 136 { 137 struct nfp_flower_priv *priv = app->priv; 138 struct circ_buf *ring; 139 140 ring = &priv->mask_ids.mask_id_free_list; 141 /* Checking if buffer is full. */ 142 if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) 143 return -ENOBUFS; 144 145 memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS); 146 ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) % 147 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 148 149 priv->mask_ids.last_used[mask_id] = ktime_get(); 150 151 return 0; 152 } 153 154 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) 155 { 156 struct nfp_flower_priv *priv = app->priv; 157 ktime_t reuse_timeout; 158 struct circ_buf *ring; 159 u8 temp_id, freed_id; 160 161 ring = &priv->mask_ids.mask_id_free_list; 162 freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; 163 /* Checking for unallocated entries first. */ 164 if (priv->mask_ids.init_unallocated > 0) { 165 *mask_id = priv->mask_ids.init_unallocated; 166 priv->mask_ids.init_unallocated--; 167 return 0; 168 } 169 170 /* Checking if buffer is empty. */ 171 if (ring->head == ring->tail) 172 goto err_not_found; 173 174 memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS); 175 *mask_id = temp_id; 176 177 reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id], 178 NFP_FL_MASK_REUSE_TIME_NS); 179 180 if (ktime_before(ktime_get(), reuse_timeout)) 181 goto err_not_found; 182 183 memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS); 184 ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) % 185 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 186 187 return 0; 188 189 err_not_found: 190 *mask_id = freed_id; 191 return -ENOENT; 192 } 193 194 static int 195 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 196 { 197 struct nfp_flower_priv *priv = app->priv; 198 struct nfp_mask_id_table *mask_entry; 199 unsigned long hash_key; 200 u8 mask_id; 201 202 if (nfp_mask_alloc(app, &mask_id)) 203 return -ENOENT; 204 205 mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL); 206 if (!mask_entry) { 207 nfp_release_mask_id(app, mask_id); 208 return -ENOMEM; 209 } 210 211 INIT_HLIST_NODE(&mask_entry->link); 212 mask_entry->mask_id = mask_id; 213 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 214 mask_entry->hash_key = hash_key; 215 mask_entry->ref_cnt = 1; 216 hash_add(priv->mask_table, &mask_entry->link, hash_key); 217 218 return mask_id; 219 } 220 221 static struct nfp_mask_id_table * 222 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 223 { 224 struct nfp_flower_priv *priv = app->priv; 225 struct nfp_mask_id_table *mask_entry; 226 unsigned long hash_key; 227 228 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 229 230 hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) 231 if (mask_entry->hash_key == hash_key) 232 return mask_entry; 233 234 return NULL; 235 } 236 237 static int 238 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 239 { 240 struct nfp_mask_id_table *mask_entry; 241 242 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 243 if (!mask_entry) 244 return -ENOENT; 245 246 mask_entry->ref_cnt++; 247 248 /* Casting u8 to int for later use. */ 249 return mask_entry->mask_id; 250 } 251 252 static bool 253 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, 254 u8 *meta_flags, u8 *mask_id) 255 { 256 int id; 257 258 id = nfp_find_in_mask_table(app, mask_data, mask_len); 259 if (id < 0) { 260 id = nfp_add_mask_table(app, mask_data, mask_len); 261 if (id < 0) 262 return false; 263 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 264 } 265 *mask_id = id; 266 267 return true; 268 } 269 270 static bool 271 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, 272 u8 *meta_flags, u8 *mask_id) 273 { 274 struct nfp_mask_id_table *mask_entry; 275 276 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 277 if (!mask_entry) 278 return false; 279 280 *mask_id = mask_entry->mask_id; 281 mask_entry->ref_cnt--; 282 if (!mask_entry->ref_cnt) { 283 hash_del(&mask_entry->link); 284 nfp_release_mask_id(app, *mask_id); 285 kfree(mask_entry); 286 if (meta_flags) 287 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 288 } 289 290 return true; 291 } 292 293 int nfp_compile_flow_metadata(struct nfp_app *app, 294 struct flow_cls_offload *flow, 295 struct nfp_fl_payload *nfp_flow, 296 struct net_device *netdev, 297 struct netlink_ext_ack *extack) 298 { 299 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 300 struct nfp_flower_priv *priv = app->priv; 301 struct nfp_fl_payload *check_entry; 302 u8 new_mask_id; 303 u32 stats_cxt; 304 int err; 305 306 err = nfp_get_stats_entry(app, &stats_cxt); 307 if (err) { 308 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate new stats context"); 309 return err; 310 } 311 312 nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); 313 nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie); 314 nfp_flow->ingress_dev = netdev; 315 316 ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL); 317 if (!ctx_entry) { 318 err = -ENOMEM; 319 goto err_release_stats; 320 } 321 322 ctx_entry->stats_cxt = stats_cxt; 323 ctx_entry->flow = nfp_flow; 324 325 if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node, 326 stats_ctx_table_params)) { 327 err = -ENOMEM; 328 goto err_free_ctx_entry; 329 } 330 331 /* Do net allocate a mask-id for pre_tun_rules. These flows are used to 332 * configure the pre_tun table and are never actually send to the 333 * firmware as an add-flow message. This causes the mask-id allocation 334 * on the firmware to get out of sync if allocated here. 335 */ 336 new_mask_id = 0; 337 if (!nfp_flow->pre_tun_rule.dev && 338 !nfp_check_mask_add(app, nfp_flow->mask_data, 339 nfp_flow->meta.mask_len, 340 &nfp_flow->meta.flags, &new_mask_id)) { 341 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id"); 342 if (nfp_release_stats_entry(app, stats_cxt)) { 343 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release stats context"); 344 err = -EINVAL; 345 goto err_remove_rhash; 346 } 347 err = -ENOENT; 348 goto err_remove_rhash; 349 } 350 351 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 352 priv->flower_version++; 353 354 /* Update flow payload with mask ids. */ 355 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 356 priv->stats[stats_cxt].pkts = 0; 357 priv->stats[stats_cxt].bytes = 0; 358 priv->stats[stats_cxt].used = jiffies; 359 360 check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev); 361 if (check_entry) { 362 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry"); 363 if (nfp_release_stats_entry(app, stats_cxt)) { 364 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release stats context"); 365 err = -EINVAL; 366 goto err_remove_mask; 367 } 368 369 if (!nfp_flow->pre_tun_rule.dev && 370 !nfp_check_mask_remove(app, nfp_flow->mask_data, 371 nfp_flow->meta.mask_len, 372 NULL, &new_mask_id)) { 373 NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release mask id"); 374 err = -EINVAL; 375 goto err_remove_mask; 376 } 377 378 err = -EEXIST; 379 goto err_remove_mask; 380 } 381 382 return 0; 383 384 err_remove_mask: 385 if (!nfp_flow->pre_tun_rule.dev) 386 nfp_check_mask_remove(app, nfp_flow->mask_data, 387 nfp_flow->meta.mask_len, 388 NULL, &new_mask_id); 389 err_remove_rhash: 390 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 391 &ctx_entry->ht_node, 392 stats_ctx_table_params)); 393 err_free_ctx_entry: 394 kfree(ctx_entry); 395 err_release_stats: 396 nfp_release_stats_entry(app, stats_cxt); 397 398 return err; 399 } 400 401 void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, 402 struct nfp_fl_payload *nfp_flow) 403 { 404 nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; 405 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 406 priv->flower_version++; 407 } 408 409 int nfp_modify_flow_metadata(struct nfp_app *app, 410 struct nfp_fl_payload *nfp_flow) 411 { 412 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 413 struct nfp_flower_priv *priv = app->priv; 414 u8 new_mask_id = 0; 415 u32 temp_ctx_id; 416 417 __nfp_modify_flow_metadata(priv, nfp_flow); 418 419 if (!nfp_flow->pre_tun_rule.dev) 420 nfp_check_mask_remove(app, nfp_flow->mask_data, 421 nfp_flow->meta.mask_len, &nfp_flow->meta.flags, 422 &new_mask_id); 423 424 /* Update flow payload with mask ids. */ 425 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 426 427 /* Release the stats ctx id and ctx to flow table entry. */ 428 temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); 429 430 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id, 431 stats_ctx_table_params); 432 if (!ctx_entry) 433 return -ENOENT; 434 435 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 436 &ctx_entry->ht_node, 437 stats_ctx_table_params)); 438 kfree(ctx_entry); 439 440 return nfp_release_stats_entry(app, temp_ctx_id); 441 } 442 443 struct nfp_fl_payload * 444 nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) 445 { 446 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 447 struct nfp_flower_priv *priv = app->priv; 448 449 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id, 450 stats_ctx_table_params); 451 if (!ctx_entry) 452 return NULL; 453 454 return ctx_entry->flow; 455 } 456 457 static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, 458 const void *obj) 459 { 460 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; 461 const struct nfp_fl_payload *flow_entry = obj; 462 463 if (flow_entry->ingress_dev == cmp_arg->netdev) 464 return flow_entry->tc_flower_cookie != cmp_arg->cookie; 465 466 return 1; 467 } 468 469 static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) 470 { 471 const struct nfp_fl_payload *flower_entry = data; 472 473 return jhash2((u32 *)&flower_entry->tc_flower_cookie, 474 sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), 475 seed); 476 } 477 478 static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) 479 { 480 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; 481 482 return jhash2((u32 *)&cmp_arg->cookie, 483 sizeof(cmp_arg->cookie) / sizeof(u32), seed); 484 } 485 486 const struct rhashtable_params nfp_flower_table_params = { 487 .head_offset = offsetof(struct nfp_fl_payload, fl_node), 488 .hashfn = nfp_fl_key_hashfn, 489 .obj_cmpfn = nfp_fl_obj_cmpfn, 490 .obj_hashfn = nfp_fl_obj_hashfn, 491 .automatic_shrinking = true, 492 }; 493 494 const struct rhashtable_params merge_table_params = { 495 .key_offset = offsetof(struct nfp_merge_info, parent_ctx), 496 .head_offset = offsetof(struct nfp_merge_info, ht_node), 497 .key_len = sizeof(u64), 498 }; 499 500 const struct rhashtable_params nfp_zone_table_params = { 501 .head_offset = offsetof(struct nfp_fl_ct_zone_entry, hash_node), 502 .key_len = sizeof(u16), 503 .key_offset = offsetof(struct nfp_fl_ct_zone_entry, zone), 504 .automatic_shrinking = false, 505 }; 506 507 const struct rhashtable_params nfp_ct_map_params = { 508 .head_offset = offsetof(struct nfp_fl_ct_map_entry, hash_node), 509 .key_len = sizeof(unsigned long), 510 .key_offset = offsetof(struct nfp_fl_ct_map_entry, cookie), 511 .automatic_shrinking = true, 512 }; 513 514 int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, 515 unsigned int host_num_mems) 516 { 517 struct nfp_flower_priv *priv = app->priv; 518 int err, stats_size; 519 520 hash_init(priv->mask_table); 521 522 err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params); 523 if (err) 524 return err; 525 526 err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params); 527 if (err) 528 goto err_free_flow_table; 529 530 err = rhashtable_init(&priv->merge_table, &merge_table_params); 531 if (err) 532 goto err_free_stats_ctx_table; 533 534 err = rhashtable_init(&priv->ct_zone_table, &nfp_zone_table_params); 535 if (err) 536 goto err_free_merge_table; 537 538 err = rhashtable_init(&priv->ct_map_table, &nfp_ct_map_params); 539 if (err) 540 goto err_free_ct_zone_table; 541 542 get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); 543 544 /* Init ring buffer and unallocated mask_ids. */ 545 priv->mask_ids.mask_id_free_list.buf = 546 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 547 NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); 548 if (!priv->mask_ids.mask_id_free_list.buf) 549 goto err_free_ct_map_table; 550 551 priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; 552 553 /* Init timestamps for mask id*/ 554 priv->mask_ids.last_used = 555 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 556 sizeof(*priv->mask_ids.last_used), GFP_KERNEL); 557 if (!priv->mask_ids.last_used) 558 goto err_free_mask_id; 559 560 /* Init ring buffer and unallocated stats_ids. */ 561 priv->stats_ids.free_list.buf = 562 vmalloc(array_size(NFP_FL_STATS_ELEM_RS, 563 priv->stats_ring_size)); 564 if (!priv->stats_ids.free_list.buf) 565 goto err_free_last_used; 566 567 priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems); 568 569 stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) | 570 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1); 571 priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats), 572 GFP_KERNEL); 573 if (!priv->stats) 574 goto err_free_ring_buf; 575 576 spin_lock_init(&priv->stats_lock); 577 578 return 0; 579 580 err_free_ring_buf: 581 vfree(priv->stats_ids.free_list.buf); 582 err_free_last_used: 583 kfree(priv->mask_ids.last_used); 584 err_free_mask_id: 585 kfree(priv->mask_ids.mask_id_free_list.buf); 586 err_free_ct_map_table: 587 rhashtable_destroy(&priv->ct_map_table); 588 err_free_ct_zone_table: 589 rhashtable_destroy(&priv->ct_zone_table); 590 err_free_merge_table: 591 rhashtable_destroy(&priv->merge_table); 592 err_free_stats_ctx_table: 593 rhashtable_destroy(&priv->stats_ctx_table); 594 err_free_flow_table: 595 rhashtable_destroy(&priv->flow_table); 596 return -ENOMEM; 597 } 598 599 static void nfp_zone_table_entry_destroy(struct nfp_fl_ct_zone_entry *zt) 600 { 601 if (!zt) 602 return; 603 604 if (!list_empty(&zt->pre_ct_list)) { 605 struct rhashtable *m_table = &zt->priv->ct_map_table; 606 struct nfp_fl_ct_flow_entry *entry, *tmp; 607 struct nfp_fl_ct_map_entry *map; 608 609 WARN_ONCE(1, "pre_ct_list not empty as expected, cleaning up\n"); 610 list_for_each_entry_safe(entry, tmp, &zt->pre_ct_list, 611 list_node) { 612 map = rhashtable_lookup_fast(m_table, 613 &entry->cookie, 614 nfp_ct_map_params); 615 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 616 &map->hash_node, 617 nfp_ct_map_params)); 618 nfp_fl_ct_clean_flow_entry(entry); 619 kfree(map); 620 } 621 } 622 623 if (!list_empty(&zt->post_ct_list)) { 624 struct rhashtable *m_table = &zt->priv->ct_map_table; 625 struct nfp_fl_ct_flow_entry *entry, *tmp; 626 struct nfp_fl_ct_map_entry *map; 627 628 WARN_ONCE(1, "post_ct_list not empty as expected, cleaning up\n"); 629 list_for_each_entry_safe(entry, tmp, &zt->post_ct_list, 630 list_node) { 631 map = rhashtable_lookup_fast(m_table, 632 &entry->cookie, 633 nfp_ct_map_params); 634 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 635 &map->hash_node, 636 nfp_ct_map_params)); 637 nfp_fl_ct_clean_flow_entry(entry); 638 kfree(map); 639 } 640 } 641 642 if (zt->nft) { 643 nf_flow_table_offload_del_cb(zt->nft, 644 nfp_fl_ct_handle_nft_flow, 645 zt); 646 zt->nft = NULL; 647 } 648 649 if (!list_empty(&zt->nft_flows_list)) { 650 struct rhashtable *m_table = &zt->priv->ct_map_table; 651 struct nfp_fl_ct_flow_entry *entry, *tmp; 652 struct nfp_fl_ct_map_entry *map; 653 654 WARN_ONCE(1, "nft_flows_list not empty as expected, cleaning up\n"); 655 list_for_each_entry_safe(entry, tmp, &zt->nft_flows_list, 656 list_node) { 657 map = rhashtable_lookup_fast(m_table, 658 &entry->cookie, 659 nfp_ct_map_params); 660 WARN_ON_ONCE(rhashtable_remove_fast(m_table, 661 &map->hash_node, 662 nfp_ct_map_params)); 663 nfp_fl_ct_clean_flow_entry(entry); 664 kfree(map); 665 } 666 } 667 668 rhashtable_free_and_destroy(&zt->tc_merge_tb, 669 nfp_check_rhashtable_empty, NULL); 670 rhashtable_free_and_destroy(&zt->nft_merge_tb, 671 nfp_check_rhashtable_empty, NULL); 672 673 kfree(zt); 674 } 675 676 static void nfp_free_zone_table_entry(void *ptr, void *arg) 677 { 678 struct nfp_fl_ct_zone_entry *zt = ptr; 679 680 nfp_zone_table_entry_destroy(zt); 681 } 682 683 static void nfp_free_map_table_entry(void *ptr, void *arg) 684 { 685 struct nfp_fl_ct_map_entry *map = ptr; 686 687 if (!map) 688 return; 689 690 kfree(map); 691 } 692 693 void nfp_flower_metadata_cleanup(struct nfp_app *app) 694 { 695 struct nfp_flower_priv *priv = app->priv; 696 697 if (!priv) 698 return; 699 700 rhashtable_free_and_destroy(&priv->flow_table, 701 nfp_check_rhashtable_empty, NULL); 702 rhashtable_free_and_destroy(&priv->stats_ctx_table, 703 nfp_check_rhashtable_empty, NULL); 704 rhashtable_free_and_destroy(&priv->merge_table, 705 nfp_check_rhashtable_empty, NULL); 706 rhashtable_free_and_destroy(&priv->ct_zone_table, 707 nfp_free_zone_table_entry, NULL); 708 nfp_zone_table_entry_destroy(priv->ct_zone_wc); 709 710 rhashtable_free_and_destroy(&priv->ct_map_table, 711 nfp_free_map_table_entry, NULL); 712 kvfree(priv->stats); 713 kfree(priv->mask_ids.mask_id_free_list.buf); 714 kfree(priv->mask_ids.last_used); 715 vfree(priv->stats_ids.free_list.buf); 716 } 717