1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <linux/hash.h> 5 #include <linux/hashtable.h> 6 #include <linux/jhash.h> 7 #include <linux/math64.h> 8 #include <linux/vmalloc.h> 9 #include <net/pkt_cls.h> 10 11 #include "cmsg.h" 12 #include "main.h" 13 #include "../nfp_app.h" 14 15 struct nfp_mask_id_table { 16 struct hlist_node link; 17 u32 hash_key; 18 u32 ref_cnt; 19 u8 mask_id; 20 }; 21 22 struct nfp_fl_flow_table_cmp_arg { 23 struct net_device *netdev; 24 unsigned long cookie; 25 }; 26 27 struct nfp_fl_stats_ctx_to_flow { 28 struct rhash_head ht_node; 29 u32 stats_cxt; 30 struct nfp_fl_payload *flow; 31 }; 32 33 static const struct rhashtable_params stats_ctx_table_params = { 34 .key_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, stats_cxt), 35 .head_offset = offsetof(struct nfp_fl_stats_ctx_to_flow, ht_node), 36 .key_len = sizeof(u32), 37 }; 38 39 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) 40 { 41 struct nfp_flower_priv *priv = app->priv; 42 struct circ_buf *ring; 43 44 ring = &priv->stats_ids.free_list; 45 /* Check if buffer is full. */ 46 if (!CIRC_SPACE(ring->head, ring->tail, 47 priv->stats_ring_size * NFP_FL_STATS_ELEM_RS - 48 NFP_FL_STATS_ELEM_RS + 1)) 49 return -ENOBUFS; 50 51 memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS); 52 ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) % 53 (priv->stats_ring_size * NFP_FL_STATS_ELEM_RS); 54 55 return 0; 56 } 57 58 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) 59 { 60 struct nfp_flower_priv *priv = app->priv; 61 u32 freed_stats_id, temp_stats_id; 62 struct circ_buf *ring; 63 64 ring = &priv->stats_ids.free_list; 65 freed_stats_id = priv->stats_ring_size; 66 /* Check for unallocated entries first. */ 67 if (priv->stats_ids.init_unalloc > 0) { 68 if (priv->active_mem_unit == priv->total_mem_units) { 69 priv->stats_ids.init_unalloc--; 70 priv->active_mem_unit = 0; 71 } 72 73 *stats_context_id = 74 FIELD_PREP(NFP_FL_STAT_ID_STAT, 75 priv->stats_ids.init_unalloc - 1) | 76 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, 77 priv->active_mem_unit); 78 priv->active_mem_unit++; 79 return 0; 80 } 81 82 /* Check if buffer is empty. */ 83 if (ring->head == ring->tail) { 84 *stats_context_id = freed_stats_id; 85 return -ENOENT; 86 } 87 88 memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS); 89 *stats_context_id = temp_stats_id; 90 memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS); 91 ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) % 92 (priv->stats_ring_size * NFP_FL_STATS_ELEM_RS); 93 94 return 0; 95 } 96 97 /* Must be called with either RTNL or rcu_read_lock */ 98 struct nfp_fl_payload * 99 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, 100 struct net_device *netdev) 101 { 102 struct nfp_fl_flow_table_cmp_arg flower_cmp_arg; 103 struct nfp_flower_priv *priv = app->priv; 104 105 flower_cmp_arg.netdev = netdev; 106 flower_cmp_arg.cookie = tc_flower_cookie; 107 108 return rhashtable_lookup_fast(&priv->flow_table, &flower_cmp_arg, 109 nfp_flower_table_params); 110 } 111 112 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) 113 { 114 unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); 115 struct nfp_flower_priv *priv = app->priv; 116 struct nfp_fl_stats_frame *stats; 117 unsigned char *msg; 118 u32 ctx_id; 119 int i; 120 121 msg = nfp_flower_cmsg_get_data(skb); 122 123 spin_lock(&priv->stats_lock); 124 for (i = 0; i < msg_len / sizeof(*stats); i++) { 125 stats = (struct nfp_fl_stats_frame *)msg + i; 126 ctx_id = be32_to_cpu(stats->stats_con_id); 127 priv->stats[ctx_id].pkts += be32_to_cpu(stats->pkt_count); 128 priv->stats[ctx_id].bytes += be64_to_cpu(stats->byte_count); 129 priv->stats[ctx_id].used = jiffies; 130 } 131 spin_unlock(&priv->stats_lock); 132 } 133 134 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) 135 { 136 struct nfp_flower_priv *priv = app->priv; 137 struct circ_buf *ring; 138 139 ring = &priv->mask_ids.mask_id_free_list; 140 /* Checking if buffer is full. */ 141 if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) 142 return -ENOBUFS; 143 144 memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS); 145 ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) % 146 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 147 148 priv->mask_ids.last_used[mask_id] = ktime_get(); 149 150 return 0; 151 } 152 153 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) 154 { 155 struct nfp_flower_priv *priv = app->priv; 156 ktime_t reuse_timeout; 157 struct circ_buf *ring; 158 u8 temp_id, freed_id; 159 160 ring = &priv->mask_ids.mask_id_free_list; 161 freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; 162 /* Checking for unallocated entries first. */ 163 if (priv->mask_ids.init_unallocated > 0) { 164 *mask_id = priv->mask_ids.init_unallocated; 165 priv->mask_ids.init_unallocated--; 166 return 0; 167 } 168 169 /* Checking if buffer is empty. */ 170 if (ring->head == ring->tail) 171 goto err_not_found; 172 173 memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS); 174 *mask_id = temp_id; 175 176 reuse_timeout = ktime_add_ns(priv->mask_ids.last_used[*mask_id], 177 NFP_FL_MASK_REUSE_TIME_NS); 178 179 if (ktime_before(ktime_get(), reuse_timeout)) 180 goto err_not_found; 181 182 memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS); 183 ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) % 184 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 185 186 return 0; 187 188 err_not_found: 189 *mask_id = freed_id; 190 return -ENOENT; 191 } 192 193 static int 194 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 195 { 196 struct nfp_flower_priv *priv = app->priv; 197 struct nfp_mask_id_table *mask_entry; 198 unsigned long hash_key; 199 u8 mask_id; 200 201 if (nfp_mask_alloc(app, &mask_id)) 202 return -ENOENT; 203 204 mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL); 205 if (!mask_entry) { 206 nfp_release_mask_id(app, mask_id); 207 return -ENOMEM; 208 } 209 210 INIT_HLIST_NODE(&mask_entry->link); 211 mask_entry->mask_id = mask_id; 212 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 213 mask_entry->hash_key = hash_key; 214 mask_entry->ref_cnt = 1; 215 hash_add(priv->mask_table, &mask_entry->link, hash_key); 216 217 return mask_id; 218 } 219 220 static struct nfp_mask_id_table * 221 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 222 { 223 struct nfp_flower_priv *priv = app->priv; 224 struct nfp_mask_id_table *mask_entry; 225 unsigned long hash_key; 226 227 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 228 229 hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) 230 if (mask_entry->hash_key == hash_key) 231 return mask_entry; 232 233 return NULL; 234 } 235 236 static int 237 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 238 { 239 struct nfp_mask_id_table *mask_entry; 240 241 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 242 if (!mask_entry) 243 return -ENOENT; 244 245 mask_entry->ref_cnt++; 246 247 /* Casting u8 to int for later use. */ 248 return mask_entry->mask_id; 249 } 250 251 static bool 252 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, 253 u8 *meta_flags, u8 *mask_id) 254 { 255 int id; 256 257 id = nfp_find_in_mask_table(app, mask_data, mask_len); 258 if (id < 0) { 259 id = nfp_add_mask_table(app, mask_data, mask_len); 260 if (id < 0) 261 return false; 262 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 263 } 264 *mask_id = id; 265 266 return true; 267 } 268 269 static bool 270 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, 271 u8 *meta_flags, u8 *mask_id) 272 { 273 struct nfp_mask_id_table *mask_entry; 274 275 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 276 if (!mask_entry) 277 return false; 278 279 *mask_id = mask_entry->mask_id; 280 mask_entry->ref_cnt--; 281 if (!mask_entry->ref_cnt) { 282 hash_del(&mask_entry->link); 283 nfp_release_mask_id(app, *mask_id); 284 kfree(mask_entry); 285 if (meta_flags) 286 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 287 } 288 289 return true; 290 } 291 292 int nfp_compile_flow_metadata(struct nfp_app *app, 293 struct tc_cls_flower_offload *flow, 294 struct nfp_fl_payload *nfp_flow, 295 struct net_device *netdev) 296 { 297 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 298 struct nfp_flower_priv *priv = app->priv; 299 struct nfp_fl_payload *check_entry; 300 u8 new_mask_id; 301 u32 stats_cxt; 302 int err; 303 304 err = nfp_get_stats_entry(app, &stats_cxt); 305 if (err) 306 return err; 307 308 nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); 309 nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie); 310 nfp_flow->ingress_dev = netdev; 311 312 ctx_entry = kzalloc(sizeof(*ctx_entry), GFP_KERNEL); 313 if (!ctx_entry) { 314 err = -ENOMEM; 315 goto err_release_stats; 316 } 317 318 ctx_entry->stats_cxt = stats_cxt; 319 ctx_entry->flow = nfp_flow; 320 321 if (rhashtable_insert_fast(&priv->stats_ctx_table, &ctx_entry->ht_node, 322 stats_ctx_table_params)) { 323 err = -ENOMEM; 324 goto err_free_ctx_entry; 325 } 326 327 new_mask_id = 0; 328 if (!nfp_check_mask_add(app, nfp_flow->mask_data, 329 nfp_flow->meta.mask_len, 330 &nfp_flow->meta.flags, &new_mask_id)) { 331 err = -ENOENT; 332 goto err_remove_rhash; 333 } 334 335 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 336 priv->flower_version++; 337 338 /* Update flow payload with mask ids. */ 339 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 340 priv->stats[stats_cxt].pkts = 0; 341 priv->stats[stats_cxt].bytes = 0; 342 priv->stats[stats_cxt].used = jiffies; 343 344 check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev); 345 if (check_entry) { 346 err = -EEXIST; 347 goto err_remove_mask; 348 } 349 350 return 0; 351 352 err_remove_mask: 353 nfp_check_mask_remove(app, nfp_flow->mask_data, nfp_flow->meta.mask_len, 354 NULL, &new_mask_id); 355 err_remove_rhash: 356 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 357 &ctx_entry->ht_node, 358 stats_ctx_table_params)); 359 err_free_ctx_entry: 360 kfree(ctx_entry); 361 err_release_stats: 362 nfp_release_stats_entry(app, stats_cxt); 363 364 return err; 365 } 366 367 void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv, 368 struct nfp_fl_payload *nfp_flow) 369 { 370 nfp_flow->meta.flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; 371 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 372 priv->flower_version++; 373 } 374 375 int nfp_modify_flow_metadata(struct nfp_app *app, 376 struct nfp_fl_payload *nfp_flow) 377 { 378 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 379 struct nfp_flower_priv *priv = app->priv; 380 u8 new_mask_id = 0; 381 u32 temp_ctx_id; 382 383 __nfp_modify_flow_metadata(priv, nfp_flow); 384 385 nfp_check_mask_remove(app, nfp_flow->mask_data, 386 nfp_flow->meta.mask_len, &nfp_flow->meta.flags, 387 &new_mask_id); 388 389 /* Update flow payload with mask ids. */ 390 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 391 392 /* Release the stats ctx id and ctx to flow table entry. */ 393 temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); 394 395 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &temp_ctx_id, 396 stats_ctx_table_params); 397 if (!ctx_entry) 398 return -ENOENT; 399 400 WARN_ON_ONCE(rhashtable_remove_fast(&priv->stats_ctx_table, 401 &ctx_entry->ht_node, 402 stats_ctx_table_params)); 403 kfree(ctx_entry); 404 405 return nfp_release_stats_entry(app, temp_ctx_id); 406 } 407 408 struct nfp_fl_payload * 409 nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id) 410 { 411 struct nfp_fl_stats_ctx_to_flow *ctx_entry; 412 struct nfp_flower_priv *priv = app->priv; 413 414 ctx_entry = rhashtable_lookup_fast(&priv->stats_ctx_table, &ctx_id, 415 stats_ctx_table_params); 416 if (!ctx_entry) 417 return NULL; 418 419 return ctx_entry->flow; 420 } 421 422 static int nfp_fl_obj_cmpfn(struct rhashtable_compare_arg *arg, 423 const void *obj) 424 { 425 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = arg->key; 426 const struct nfp_fl_payload *flow_entry = obj; 427 428 if (flow_entry->ingress_dev == cmp_arg->netdev) 429 return flow_entry->tc_flower_cookie != cmp_arg->cookie; 430 431 return 1; 432 } 433 434 static u32 nfp_fl_obj_hashfn(const void *data, u32 len, u32 seed) 435 { 436 const struct nfp_fl_payload *flower_entry = data; 437 438 return jhash2((u32 *)&flower_entry->tc_flower_cookie, 439 sizeof(flower_entry->tc_flower_cookie) / sizeof(u32), 440 seed); 441 } 442 443 static u32 nfp_fl_key_hashfn(const void *data, u32 len, u32 seed) 444 { 445 const struct nfp_fl_flow_table_cmp_arg *cmp_arg = data; 446 447 return jhash2((u32 *)&cmp_arg->cookie, 448 sizeof(cmp_arg->cookie) / sizeof(u32), seed); 449 } 450 451 const struct rhashtable_params nfp_flower_table_params = { 452 .head_offset = offsetof(struct nfp_fl_payload, fl_node), 453 .hashfn = nfp_fl_key_hashfn, 454 .obj_cmpfn = nfp_fl_obj_cmpfn, 455 .obj_hashfn = nfp_fl_obj_hashfn, 456 .automatic_shrinking = true, 457 }; 458 459 int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count, 460 unsigned int host_num_mems) 461 { 462 struct nfp_flower_priv *priv = app->priv; 463 int err, stats_size; 464 465 hash_init(priv->mask_table); 466 467 err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params); 468 if (err) 469 return err; 470 471 err = rhashtable_init(&priv->stats_ctx_table, &stats_ctx_table_params); 472 if (err) 473 goto err_free_flow_table; 474 475 get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); 476 477 /* Init ring buffer and unallocated mask_ids. */ 478 priv->mask_ids.mask_id_free_list.buf = 479 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 480 NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); 481 if (!priv->mask_ids.mask_id_free_list.buf) 482 goto err_free_stats_ctx_table; 483 484 priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; 485 486 /* Init timestamps for mask id*/ 487 priv->mask_ids.last_used = 488 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 489 sizeof(*priv->mask_ids.last_used), GFP_KERNEL); 490 if (!priv->mask_ids.last_used) 491 goto err_free_mask_id; 492 493 /* Init ring buffer and unallocated stats_ids. */ 494 priv->stats_ids.free_list.buf = 495 vmalloc(array_size(NFP_FL_STATS_ELEM_RS, 496 priv->stats_ring_size)); 497 if (!priv->stats_ids.free_list.buf) 498 goto err_free_last_used; 499 500 priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems); 501 502 stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) | 503 FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1); 504 priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats), 505 GFP_KERNEL); 506 if (!priv->stats) 507 goto err_free_ring_buf; 508 509 spin_lock_init(&priv->stats_lock); 510 511 return 0; 512 513 err_free_ring_buf: 514 vfree(priv->stats_ids.free_list.buf); 515 err_free_last_used: 516 kfree(priv->mask_ids.last_used); 517 err_free_mask_id: 518 kfree(priv->mask_ids.mask_id_free_list.buf); 519 err_free_stats_ctx_table: 520 rhashtable_destroy(&priv->stats_ctx_table); 521 err_free_flow_table: 522 rhashtable_destroy(&priv->flow_table); 523 return -ENOMEM; 524 } 525 526 void nfp_flower_metadata_cleanup(struct nfp_app *app) 527 { 528 struct nfp_flower_priv *priv = app->priv; 529 530 if (!priv) 531 return; 532 533 rhashtable_free_and_destroy(&priv->flow_table, 534 nfp_check_rhashtable_empty, NULL); 535 rhashtable_free_and_destroy(&priv->stats_ctx_table, 536 nfp_check_rhashtable_empty, NULL); 537 kvfree(priv->stats); 538 kfree(priv->mask_ids.mask_id_free_list.buf); 539 kfree(priv->mask_ids.last_used); 540 vfree(priv->stats_ids.free_list.buf); 541 } 542