1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/hash.h> 35 #include <linux/hashtable.h> 36 #include <linux/jhash.h> 37 #include <linux/vmalloc.h> 38 #include <net/pkt_cls.h> 39 40 #include "cmsg.h" 41 #include "main.h" 42 #include "../nfp_app.h" 43 44 struct nfp_mask_id_table { 45 struct hlist_node link; 46 u32 hash_key; 47 u32 ref_cnt; 48 u8 mask_id; 49 }; 50 51 static int nfp_release_stats_entry(struct nfp_app *app, u32 stats_context_id) 52 { 53 struct nfp_flower_priv *priv = app->priv; 54 struct circ_buf *ring; 55 56 ring = &priv->stats_ids.free_list; 57 /* Check if buffer is full. */ 58 if (!CIRC_SPACE(ring->head, ring->tail, NFP_FL_STATS_ENTRY_RS * 59 NFP_FL_STATS_ELEM_RS - 60 NFP_FL_STATS_ELEM_RS + 1)) 61 return -ENOBUFS; 62 63 memcpy(&ring->buf[ring->head], &stats_context_id, NFP_FL_STATS_ELEM_RS); 64 ring->head = (ring->head + NFP_FL_STATS_ELEM_RS) % 65 (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS); 66 67 return 0; 68 } 69 70 static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) 71 { 72 struct nfp_flower_priv *priv = app->priv; 73 u32 freed_stats_id, temp_stats_id; 74 struct circ_buf *ring; 75 76 ring = &priv->stats_ids.free_list; 77 freed_stats_id = NFP_FL_STATS_ENTRY_RS; 78 /* Check for unallocated entries first. */ 79 if (priv->stats_ids.init_unalloc > 0) { 80 *stats_context_id = priv->stats_ids.init_unalloc - 1; 81 priv->stats_ids.init_unalloc--; 82 return 0; 83 } 84 85 /* Check if buffer is empty. */ 86 if (ring->head == ring->tail) { 87 *stats_context_id = freed_stats_id; 88 return -ENOENT; 89 } 90 91 memcpy(&temp_stats_id, &ring->buf[ring->tail], NFP_FL_STATS_ELEM_RS); 92 *stats_context_id = temp_stats_id; 93 memcpy(&ring->buf[ring->tail], &freed_stats_id, NFP_FL_STATS_ELEM_RS); 94 ring->tail = (ring->tail + NFP_FL_STATS_ELEM_RS) % 95 (NFP_FL_STATS_ENTRY_RS * NFP_FL_STATS_ELEM_RS); 96 97 return 0; 98 } 99 100 /* Must be called with either RTNL or rcu_read_lock */ 101 struct nfp_fl_payload * 102 nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, 103 struct net_device *netdev, __be32 host_ctx) 104 { 105 struct nfp_flower_priv *priv = app->priv; 106 struct nfp_fl_payload *flower_entry; 107 108 hash_for_each_possible_rcu(priv->flow_table, flower_entry, link, 109 tc_flower_cookie) 110 if (flower_entry->tc_flower_cookie == tc_flower_cookie && 111 (!netdev || flower_entry->ingress_dev == netdev) && 112 (host_ctx == NFP_FL_STATS_CTX_DONT_CARE || 113 flower_entry->meta.host_ctx_id == host_ctx)) 114 return flower_entry; 115 116 return NULL; 117 } 118 119 static void 120 nfp_flower_update_stats(struct nfp_app *app, struct nfp_fl_stats_frame *stats) 121 { 122 struct nfp_fl_payload *nfp_flow; 123 unsigned long flower_cookie; 124 125 flower_cookie = be64_to_cpu(stats->stats_cookie); 126 127 rcu_read_lock(); 128 nfp_flow = nfp_flower_search_fl_table(app, flower_cookie, NULL, 129 stats->stats_con_id); 130 if (!nfp_flow) 131 goto exit_rcu_unlock; 132 133 spin_lock(&nfp_flow->lock); 134 nfp_flow->stats.pkts += be32_to_cpu(stats->pkt_count); 135 nfp_flow->stats.bytes += be64_to_cpu(stats->byte_count); 136 nfp_flow->stats.used = jiffies; 137 spin_unlock(&nfp_flow->lock); 138 139 exit_rcu_unlock: 140 rcu_read_unlock(); 141 } 142 143 void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb) 144 { 145 unsigned int msg_len = nfp_flower_cmsg_get_data_len(skb); 146 struct nfp_fl_stats_frame *stats_frame; 147 unsigned char *msg; 148 int i; 149 150 msg = nfp_flower_cmsg_get_data(skb); 151 152 stats_frame = (struct nfp_fl_stats_frame *)msg; 153 for (i = 0; i < msg_len / sizeof(*stats_frame); i++) 154 nfp_flower_update_stats(app, stats_frame + i); 155 } 156 157 static int nfp_release_mask_id(struct nfp_app *app, u8 mask_id) 158 { 159 struct nfp_flower_priv *priv = app->priv; 160 struct circ_buf *ring; 161 struct timespec64 now; 162 163 ring = &priv->mask_ids.mask_id_free_list; 164 /* Checking if buffer is full. */ 165 if (CIRC_SPACE(ring->head, ring->tail, NFP_FLOWER_MASK_ENTRY_RS) == 0) 166 return -ENOBUFS; 167 168 memcpy(&ring->buf[ring->head], &mask_id, NFP_FLOWER_MASK_ELEMENT_RS); 169 ring->head = (ring->head + NFP_FLOWER_MASK_ELEMENT_RS) % 170 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 171 172 getnstimeofday64(&now); 173 priv->mask_ids.last_used[mask_id] = now; 174 175 return 0; 176 } 177 178 static int nfp_mask_alloc(struct nfp_app *app, u8 *mask_id) 179 { 180 struct nfp_flower_priv *priv = app->priv; 181 struct timespec64 delta, now; 182 struct circ_buf *ring; 183 u8 temp_id, freed_id; 184 185 ring = &priv->mask_ids.mask_id_free_list; 186 freed_id = NFP_FLOWER_MASK_ENTRY_RS - 1; 187 /* Checking for unallocated entries first. */ 188 if (priv->mask_ids.init_unallocated > 0) { 189 *mask_id = priv->mask_ids.init_unallocated; 190 priv->mask_ids.init_unallocated--; 191 return 0; 192 } 193 194 /* Checking if buffer is empty. */ 195 if (ring->head == ring->tail) 196 goto err_not_found; 197 198 memcpy(&temp_id, &ring->buf[ring->tail], NFP_FLOWER_MASK_ELEMENT_RS); 199 *mask_id = temp_id; 200 201 getnstimeofday64(&now); 202 delta = timespec64_sub(now, priv->mask_ids.last_used[*mask_id]); 203 204 if (timespec64_to_ns(&delta) < NFP_FL_MASK_REUSE_TIME_NS) 205 goto err_not_found; 206 207 memcpy(&ring->buf[ring->tail], &freed_id, NFP_FLOWER_MASK_ELEMENT_RS); 208 ring->tail = (ring->tail + NFP_FLOWER_MASK_ELEMENT_RS) % 209 (NFP_FLOWER_MASK_ENTRY_RS * NFP_FLOWER_MASK_ELEMENT_RS); 210 211 return 0; 212 213 err_not_found: 214 *mask_id = freed_id; 215 return -ENOENT; 216 } 217 218 static int 219 nfp_add_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 220 { 221 struct nfp_flower_priv *priv = app->priv; 222 struct nfp_mask_id_table *mask_entry; 223 unsigned long hash_key; 224 u8 mask_id; 225 226 if (nfp_mask_alloc(app, &mask_id)) 227 return -ENOENT; 228 229 mask_entry = kmalloc(sizeof(*mask_entry), GFP_KERNEL); 230 if (!mask_entry) { 231 nfp_release_mask_id(app, mask_id); 232 return -ENOMEM; 233 } 234 235 INIT_HLIST_NODE(&mask_entry->link); 236 mask_entry->mask_id = mask_id; 237 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 238 mask_entry->hash_key = hash_key; 239 mask_entry->ref_cnt = 1; 240 hash_add(priv->mask_table, &mask_entry->link, hash_key); 241 242 return mask_id; 243 } 244 245 static struct nfp_mask_id_table * 246 nfp_search_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 247 { 248 struct nfp_flower_priv *priv = app->priv; 249 struct nfp_mask_id_table *mask_entry; 250 unsigned long hash_key; 251 252 hash_key = jhash(mask_data, mask_len, priv->mask_id_seed); 253 254 hash_for_each_possible(priv->mask_table, mask_entry, link, hash_key) 255 if (mask_entry->hash_key == hash_key) 256 return mask_entry; 257 258 return NULL; 259 } 260 261 static int 262 nfp_find_in_mask_table(struct nfp_app *app, char *mask_data, u32 mask_len) 263 { 264 struct nfp_mask_id_table *mask_entry; 265 266 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 267 if (!mask_entry) 268 return -ENOENT; 269 270 mask_entry->ref_cnt++; 271 272 /* Casting u8 to int for later use. */ 273 return mask_entry->mask_id; 274 } 275 276 static bool 277 nfp_check_mask_add(struct nfp_app *app, char *mask_data, u32 mask_len, 278 u8 *meta_flags, u8 *mask_id) 279 { 280 int id; 281 282 id = nfp_find_in_mask_table(app, mask_data, mask_len); 283 if (id < 0) { 284 id = nfp_add_mask_table(app, mask_data, mask_len); 285 if (id < 0) 286 return false; 287 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 288 } 289 *mask_id = id; 290 291 return true; 292 } 293 294 static bool 295 nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, 296 u8 *meta_flags, u8 *mask_id) 297 { 298 struct nfp_mask_id_table *mask_entry; 299 300 mask_entry = nfp_search_mask_table(app, mask_data, mask_len); 301 if (!mask_entry) 302 return false; 303 304 if (meta_flags) 305 *meta_flags &= ~NFP_FL_META_FLAG_MANAGE_MASK; 306 307 *mask_id = mask_entry->mask_id; 308 mask_entry->ref_cnt--; 309 if (!mask_entry->ref_cnt) { 310 hash_del(&mask_entry->link); 311 nfp_release_mask_id(app, *mask_id); 312 kfree(mask_entry); 313 if (meta_flags) 314 *meta_flags |= NFP_FL_META_FLAG_MANAGE_MASK; 315 } 316 317 return true; 318 } 319 320 int nfp_compile_flow_metadata(struct nfp_app *app, 321 struct tc_cls_flower_offload *flow, 322 struct nfp_fl_payload *nfp_flow, 323 struct net_device *netdev) 324 { 325 struct nfp_flower_priv *priv = app->priv; 326 struct nfp_fl_payload *check_entry; 327 u8 new_mask_id; 328 u32 stats_cxt; 329 330 if (nfp_get_stats_entry(app, &stats_cxt)) 331 return -ENOENT; 332 333 nfp_flow->meta.host_ctx_id = cpu_to_be32(stats_cxt); 334 nfp_flow->meta.host_cookie = cpu_to_be64(flow->cookie); 335 336 new_mask_id = 0; 337 if (!nfp_check_mask_add(app, nfp_flow->mask_data, 338 nfp_flow->meta.mask_len, 339 &nfp_flow->meta.flags, &new_mask_id)) { 340 if (nfp_release_stats_entry(app, stats_cxt)) 341 return -EINVAL; 342 return -ENOENT; 343 } 344 345 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 346 priv->flower_version++; 347 348 /* Update flow payload with mask ids. */ 349 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 350 nfp_flow->stats.pkts = 0; 351 nfp_flow->stats.bytes = 0; 352 nfp_flow->stats.used = jiffies; 353 354 check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev, 355 NFP_FL_STATS_CTX_DONT_CARE); 356 if (check_entry) { 357 if (nfp_release_stats_entry(app, stats_cxt)) 358 return -EINVAL; 359 360 if (!nfp_check_mask_remove(app, nfp_flow->mask_data, 361 nfp_flow->meta.mask_len, 362 NULL, &new_mask_id)) 363 return -EINVAL; 364 365 return -EEXIST; 366 } 367 368 return 0; 369 } 370 371 int nfp_modify_flow_metadata(struct nfp_app *app, 372 struct nfp_fl_payload *nfp_flow) 373 { 374 struct nfp_flower_priv *priv = app->priv; 375 u8 new_mask_id = 0; 376 u32 temp_ctx_id; 377 378 nfp_check_mask_remove(app, nfp_flow->mask_data, 379 nfp_flow->meta.mask_len, &nfp_flow->meta.flags, 380 &new_mask_id); 381 382 nfp_flow->meta.flow_version = cpu_to_be64(priv->flower_version); 383 priv->flower_version++; 384 385 /* Update flow payload with mask ids. */ 386 nfp_flow->unmasked_data[NFP_FL_MASK_ID_LOCATION] = new_mask_id; 387 388 /* Release the stats ctx id. */ 389 temp_ctx_id = be32_to_cpu(nfp_flow->meta.host_ctx_id); 390 391 return nfp_release_stats_entry(app, temp_ctx_id); 392 } 393 394 int nfp_flower_metadata_init(struct nfp_app *app) 395 { 396 struct nfp_flower_priv *priv = app->priv; 397 398 hash_init(priv->mask_table); 399 hash_init(priv->flow_table); 400 get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed)); 401 402 /* Init ring buffer and unallocated mask_ids. */ 403 priv->mask_ids.mask_id_free_list.buf = 404 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 405 NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL); 406 if (!priv->mask_ids.mask_id_free_list.buf) 407 return -ENOMEM; 408 409 priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1; 410 411 /* Init timestamps for mask id*/ 412 priv->mask_ids.last_used = 413 kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS, 414 sizeof(*priv->mask_ids.last_used), GFP_KERNEL); 415 if (!priv->mask_ids.last_used) 416 goto err_free_mask_id; 417 418 /* Init ring buffer and unallocated stats_ids. */ 419 priv->stats_ids.free_list.buf = 420 vmalloc(array_size(NFP_FL_STATS_ELEM_RS, 421 NFP_FL_STATS_ENTRY_RS)); 422 if (!priv->stats_ids.free_list.buf) 423 goto err_free_last_used; 424 425 priv->stats_ids.init_unalloc = NFP_FL_REPEATED_HASH_MAX; 426 427 return 0; 428 429 err_free_last_used: 430 kfree(priv->mask_ids.last_used); 431 err_free_mask_id: 432 kfree(priv->mask_ids.mask_id_free_list.buf); 433 return -ENOMEM; 434 } 435 436 void nfp_flower_metadata_cleanup(struct nfp_app *app) 437 { 438 struct nfp_flower_priv *priv = app->priv; 439 440 if (!priv) 441 return; 442 443 kfree(priv->mask_ids.mask_id_free_list.buf); 444 kfree(priv->mask_ids.last_used); 445 vfree(priv->stats_ids.free_list.buf); 446 } 447