1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/bpf.h> 35 #include <linux/bitops.h> 36 #include <linux/bug.h> 37 #include <linux/jiffies.h> 38 #include <linux/skbuff.h> 39 #include <linux/wait.h> 40 41 #include "../nfp_app.h" 42 #include "../nfp_net.h" 43 #include "fw.h" 44 #include "main.h" 45 46 #define cmsg_warn(bpf, msg...) nn_dp_warn(&(bpf)->app->ctrl->dp, msg) 47 48 #define NFP_BPF_TAG_ALLOC_SPAN (U16_MAX / 4) 49 50 static bool nfp_bpf_all_tags_busy(struct nfp_app_bpf *bpf) 51 { 52 u16 used_tags; 53 54 used_tags = bpf->tag_alloc_next - bpf->tag_alloc_last; 55 56 return used_tags > NFP_BPF_TAG_ALLOC_SPAN; 57 } 58 59 static int nfp_bpf_alloc_tag(struct nfp_app_bpf *bpf) 60 { 61 /* All FW communication for BPF is request-reply. To make sure we 62 * don't reuse the message ID too early after timeout - limit the 63 * number of requests in flight. 64 */ 65 if (nfp_bpf_all_tags_busy(bpf)) { 66 cmsg_warn(bpf, "all FW request contexts busy!\n"); 67 return -EAGAIN; 68 } 69 70 WARN_ON(__test_and_set_bit(bpf->tag_alloc_next, bpf->tag_allocator)); 71 return bpf->tag_alloc_next++; 72 } 73 74 static void nfp_bpf_free_tag(struct nfp_app_bpf *bpf, u16 tag) 75 { 76 WARN_ON(!__test_and_clear_bit(tag, bpf->tag_allocator)); 77 78 while (!test_bit(bpf->tag_alloc_last, bpf->tag_allocator) && 79 bpf->tag_alloc_last != bpf->tag_alloc_next) 80 bpf->tag_alloc_last++; 81 } 82 83 static struct sk_buff * 84 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size) 85 { 86 struct sk_buff *skb; 87 88 skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL); 89 skb_put(skb, size); 90 91 return skb; 92 } 93 94 static struct sk_buff * 95 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n) 96 { 97 unsigned int size; 98 99 size = sizeof(struct cmsg_req_map_op); 100 size += sizeof(struct cmsg_key_value_pair) * n; 101 102 return nfp_bpf_cmsg_alloc(bpf, size); 103 } 104 105 static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb) 106 { 107 struct cmsg_hdr *hdr; 108 109 hdr = (struct cmsg_hdr *)skb->data; 110 111 return be16_to_cpu(hdr->tag); 112 } 113 114 static struct sk_buff *__nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag) 115 { 116 unsigned int msg_tag; 117 struct sk_buff *skb; 118 119 skb_queue_walk(&bpf->cmsg_replies, skb) { 120 msg_tag = nfp_bpf_cmsg_get_tag(skb); 121 if (msg_tag == tag) { 122 nfp_bpf_free_tag(bpf, tag); 123 __skb_unlink(skb, &bpf->cmsg_replies); 124 return skb; 125 } 126 } 127 128 return NULL; 129 } 130 131 static struct sk_buff *nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag) 132 { 133 struct sk_buff *skb; 134 135 nfp_ctrl_lock(bpf->app->ctrl); 136 skb = __nfp_bpf_reply(bpf, tag); 137 nfp_ctrl_unlock(bpf->app->ctrl); 138 139 return skb; 140 } 141 142 static struct sk_buff *nfp_bpf_reply_drop_tag(struct nfp_app_bpf *bpf, u16 tag) 143 { 144 struct sk_buff *skb; 145 146 nfp_ctrl_lock(bpf->app->ctrl); 147 skb = __nfp_bpf_reply(bpf, tag); 148 if (!skb) 149 nfp_bpf_free_tag(bpf, tag); 150 nfp_ctrl_unlock(bpf->app->ctrl); 151 152 return skb; 153 } 154 155 static struct sk_buff * 156 nfp_bpf_cmsg_wait_reply(struct nfp_app_bpf *bpf, enum nfp_bpf_cmsg_type type, 157 int tag) 158 { 159 struct sk_buff *skb; 160 int i, err; 161 162 for (i = 0; i < 50; i++) { 163 udelay(4); 164 skb = nfp_bpf_reply(bpf, tag); 165 if (skb) 166 return skb; 167 } 168 169 err = wait_event_interruptible_timeout(bpf->cmsg_wq, 170 skb = nfp_bpf_reply(bpf, tag), 171 msecs_to_jiffies(5000)); 172 /* We didn't get a response - try last time and atomically drop 173 * the tag even if no response is matched. 174 */ 175 if (!skb) 176 skb = nfp_bpf_reply_drop_tag(bpf, tag); 177 if (err < 0) { 178 cmsg_warn(bpf, "%s waiting for response to 0x%02x: %d\n", 179 err == ERESTARTSYS ? "interrupted" : "error", 180 type, err); 181 return ERR_PTR(err); 182 } 183 if (!skb) { 184 cmsg_warn(bpf, "timeout waiting for response to 0x%02x\n", 185 type); 186 return ERR_PTR(-ETIMEDOUT); 187 } 188 189 return skb; 190 } 191 192 static struct sk_buff * 193 nfp_bpf_cmsg_communicate(struct nfp_app_bpf *bpf, struct sk_buff *skb, 194 enum nfp_bpf_cmsg_type type, unsigned int reply_size) 195 { 196 struct cmsg_hdr *hdr; 197 int tag; 198 199 nfp_ctrl_lock(bpf->app->ctrl); 200 tag = nfp_bpf_alloc_tag(bpf); 201 if (tag < 0) { 202 nfp_ctrl_unlock(bpf->app->ctrl); 203 dev_kfree_skb_any(skb); 204 return ERR_PTR(tag); 205 } 206 207 hdr = (void *)skb->data; 208 hdr->ver = CMSG_MAP_ABI_VERSION; 209 hdr->type = type; 210 hdr->tag = cpu_to_be16(tag); 211 212 __nfp_app_ctrl_tx(bpf->app, skb); 213 214 nfp_ctrl_unlock(bpf->app->ctrl); 215 216 skb = nfp_bpf_cmsg_wait_reply(bpf, type, tag); 217 if (IS_ERR(skb)) 218 return skb; 219 220 hdr = (struct cmsg_hdr *)skb->data; 221 /* 0 reply_size means caller will do the validation */ 222 if (reply_size && skb->len != reply_size) { 223 cmsg_warn(bpf, "cmsg drop - wrong size %d != %d!\n", 224 skb->len, reply_size); 225 goto err_free; 226 } 227 if (hdr->type != __CMSG_REPLY(type)) { 228 cmsg_warn(bpf, "cmsg drop - wrong type 0x%02x != 0x%02lx!\n", 229 hdr->type, __CMSG_REPLY(type)); 230 goto err_free; 231 } 232 233 return skb; 234 err_free: 235 dev_kfree_skb_any(skb); 236 return ERR_PTR(-EIO); 237 } 238 239 static int 240 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf, 241 struct cmsg_reply_map_simple *reply) 242 { 243 static const int res_table[] = { 244 [CMSG_RC_SUCCESS] = 0, 245 [CMSG_RC_ERR_MAP_FD] = -EBADFD, 246 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT, 247 [CMSG_RC_ERR_MAP_ERR] = -EINVAL, 248 [CMSG_RC_ERR_MAP_PARSE] = -EIO, 249 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST, 250 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM, 251 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG, 252 }; 253 u32 rc; 254 255 rc = be32_to_cpu(reply->rc); 256 if (rc >= ARRAY_SIZE(res_table)) { 257 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc); 258 return -EIO; 259 } 260 261 return res_table[rc]; 262 } 263 264 long long int 265 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map) 266 { 267 struct cmsg_reply_map_alloc_tbl *reply; 268 struct cmsg_req_map_alloc_tbl *req; 269 struct sk_buff *skb; 270 u32 tid; 271 int err; 272 273 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req)); 274 if (!skb) 275 return -ENOMEM; 276 277 req = (void *)skb->data; 278 req->key_size = cpu_to_be32(map->key_size); 279 req->value_size = cpu_to_be32(map->value_size); 280 req->max_entries = cpu_to_be32(map->max_entries); 281 req->map_type = cpu_to_be32(map->map_type); 282 req->map_flags = 0; 283 284 skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_ALLOC, 285 sizeof(*reply)); 286 if (IS_ERR(skb)) 287 return PTR_ERR(skb); 288 289 reply = (void *)skb->data; 290 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 291 if (err) 292 goto err_free; 293 294 tid = be32_to_cpu(reply->tid); 295 dev_consume_skb_any(skb); 296 297 return tid; 298 err_free: 299 dev_kfree_skb_any(skb); 300 return err; 301 } 302 303 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map) 304 { 305 struct cmsg_reply_map_free_tbl *reply; 306 struct cmsg_req_map_free_tbl *req; 307 struct sk_buff *skb; 308 int err; 309 310 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req)); 311 if (!skb) { 312 cmsg_warn(bpf, "leaking map - failed to allocate msg\n"); 313 return; 314 } 315 316 req = (void *)skb->data; 317 req->tid = cpu_to_be32(nfp_map->tid); 318 319 skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_FREE, 320 sizeof(*reply)); 321 if (IS_ERR(skb)) { 322 cmsg_warn(bpf, "leaking map - I/O error\n"); 323 return; 324 } 325 326 reply = (void *)skb->data; 327 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 328 if (err) 329 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err); 330 331 dev_consume_skb_any(skb); 332 } 333 334 static int 335 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap, 336 enum nfp_bpf_cmsg_type op, 337 u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value) 338 { 339 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 340 struct nfp_app_bpf *bpf = nfp_map->bpf; 341 struct bpf_map *map = &offmap->map; 342 struct cmsg_reply_map_op *reply; 343 struct cmsg_req_map_op *req; 344 struct sk_buff *skb; 345 int err; 346 347 /* FW messages have no space for more than 32 bits of flags */ 348 if (flags >> 32) 349 return -EOPNOTSUPP; 350 351 skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1); 352 if (!skb) 353 return -ENOMEM; 354 355 req = (void *)skb->data; 356 req->tid = cpu_to_be32(nfp_map->tid); 357 req->count = cpu_to_be32(1); 358 req->flags = cpu_to_be32(flags); 359 360 /* Copy inputs */ 361 if (key) 362 memcpy(&req->elem[0].key, key, map->key_size); 363 if (value) 364 memcpy(&req->elem[0].value, value, map->value_size); 365 366 skb = nfp_bpf_cmsg_communicate(bpf, skb, op, 367 sizeof(*reply) + sizeof(*reply->elem)); 368 if (IS_ERR(skb)) 369 return PTR_ERR(skb); 370 371 reply = (void *)skb->data; 372 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 373 if (err) 374 goto err_free; 375 376 /* Copy outputs */ 377 if (out_key) 378 memcpy(out_key, &reply->elem[0].key, map->key_size); 379 if (out_value) 380 memcpy(out_value, &reply->elem[0].value, map->value_size); 381 382 dev_consume_skb_any(skb); 383 384 return 0; 385 err_free: 386 dev_kfree_skb_any(skb); 387 return err; 388 } 389 390 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap, 391 void *key, void *value, u64 flags) 392 { 393 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_UPDATE, 394 key, value, flags, NULL, NULL); 395 } 396 397 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key) 398 { 399 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_DELETE, 400 key, NULL, 0, NULL, NULL); 401 } 402 403 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap, 404 void *key, void *value) 405 { 406 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_LOOKUP, 407 key, NULL, 0, NULL, value); 408 } 409 410 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap, 411 void *next_key) 412 { 413 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETFIRST, 414 NULL, NULL, 0, next_key, NULL); 415 } 416 417 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap, 418 void *key, void *next_key) 419 { 420 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETNEXT, 421 key, NULL, 0, next_key, NULL); 422 } 423 424 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb) 425 { 426 struct nfp_app_bpf *bpf = app->priv; 427 unsigned int tag; 428 429 if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) { 430 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len); 431 goto err_free; 432 } 433 434 nfp_ctrl_lock(bpf->app->ctrl); 435 436 tag = nfp_bpf_cmsg_get_tag(skb); 437 if (unlikely(!test_bit(tag, bpf->tag_allocator))) { 438 cmsg_warn(bpf, "cmsg drop - no one is waiting for tag %u!\n", 439 tag); 440 goto err_unlock; 441 } 442 443 __skb_queue_tail(&bpf->cmsg_replies, skb); 444 wake_up_interruptible_all(&bpf->cmsg_wq); 445 446 nfp_ctrl_unlock(bpf->app->ctrl); 447 448 return; 449 err_unlock: 450 nfp_ctrl_unlock(bpf->app->ctrl); 451 err_free: 452 dev_kfree_skb_any(skb); 453 } 454