1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/io.h> 6 #include <linux/skbuff.h> 7 8 #include "ccm.h" 9 #include "nfp_net.h" 10 11 /* CCM messages via the mailbox. CMSGs get wrapped into simple TLVs 12 * and copied into the mailbox. Multiple messages can be copied to 13 * form a batch. Threads come in with CMSG formed in an skb, then 14 * enqueue that skb onto the request queue. If threads skb is first 15 * in queue this thread will handle the mailbox operation. It copies 16 * up to 64 messages into the mailbox (making sure that both requests 17 * and replies will fit. After FW is done processing the batch it 18 * copies the data out and wakes waiting threads. 19 * If a thread is waiting it either gets its the message completed 20 * (response is copied into the same skb as the request, overwriting 21 * it), or becomes the first in queue. 22 * Completions and next-to-run are signaled via the control buffer 23 * to limit potential cache line bounces. 24 */ 25 26 #define NFP_CCM_MBOX_BATCH_LIMIT 64 27 #define NFP_CCM_TIMEOUT (NFP_NET_POLL_TIMEOUT * 1000) 28 #define NFP_CCM_MAX_QLEN 1024 29 30 enum nfp_net_mbox_cmsg_state { 31 NFP_NET_MBOX_CMSG_STATE_QUEUED, 32 NFP_NET_MBOX_CMSG_STATE_NEXT, 33 NFP_NET_MBOX_CMSG_STATE_BUSY, 34 NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND, 35 NFP_NET_MBOX_CMSG_STATE_DONE, 36 }; 37 38 /** 39 * struct nfp_ccm_mbox_cmsg_cb - CCM mailbox specific info 40 * @state: processing state (/stage) of the message 41 * @err: error encountered during processing if any 42 * @max_len: max(request_len, reply_len) 43 * @exp_reply: expected reply length (0 means don't validate) 44 * @posted: the message was posted and nobody waits for the reply 45 */ 46 struct nfp_ccm_mbox_cmsg_cb { 47 enum nfp_net_mbox_cmsg_state state; 48 int err; 49 unsigned int max_len; 50 unsigned int exp_reply; 51 bool posted; 52 }; 53 54 static u32 nfp_ccm_mbox_max_msg(struct nfp_net *nn) 55 { 56 return round_down(nn->tlv_caps.mbox_len, 4) - 57 NFP_NET_CFG_MBOX_SIMPLE_VAL - /* common mbox command header */ 58 4 * 2; /* Msg TLV plus End TLV headers */ 59 } 60 61 static void 62 nfp_ccm_mbox_msg_init(struct sk_buff *skb, unsigned int exp_reply, int max_len) 63 { 64 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 65 66 cb->state = NFP_NET_MBOX_CMSG_STATE_QUEUED; 67 cb->err = 0; 68 cb->max_len = max_len; 69 cb->exp_reply = exp_reply; 70 cb->posted = false; 71 } 72 73 static int nfp_ccm_mbox_maxlen(const struct sk_buff *skb) 74 { 75 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 76 77 return cb->max_len; 78 } 79 80 static bool nfp_ccm_mbox_done(struct sk_buff *skb) 81 { 82 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 83 84 return cb->state == NFP_NET_MBOX_CMSG_STATE_DONE; 85 } 86 87 static bool nfp_ccm_mbox_in_progress(struct sk_buff *skb) 88 { 89 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 90 91 return cb->state != NFP_NET_MBOX_CMSG_STATE_QUEUED && 92 cb->state != NFP_NET_MBOX_CMSG_STATE_NEXT; 93 } 94 95 static void nfp_ccm_mbox_set_busy(struct sk_buff *skb) 96 { 97 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 98 99 cb->state = NFP_NET_MBOX_CMSG_STATE_BUSY; 100 } 101 102 static bool nfp_ccm_mbox_is_posted(struct sk_buff *skb) 103 { 104 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 105 106 return cb->posted; 107 } 108 109 static void nfp_ccm_mbox_mark_posted(struct sk_buff *skb) 110 { 111 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 112 113 cb->posted = true; 114 } 115 116 static bool nfp_ccm_mbox_is_first(struct nfp_net *nn, struct sk_buff *skb) 117 { 118 return skb_queue_is_first(&nn->mbox_cmsg.queue, skb); 119 } 120 121 static bool nfp_ccm_mbox_should_run(struct nfp_net *nn, struct sk_buff *skb) 122 { 123 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 124 125 return cb->state == NFP_NET_MBOX_CMSG_STATE_NEXT; 126 } 127 128 static void nfp_ccm_mbox_mark_next_runner(struct nfp_net *nn) 129 { 130 struct nfp_ccm_mbox_cmsg_cb *cb; 131 struct sk_buff *skb; 132 133 skb = skb_peek(&nn->mbox_cmsg.queue); 134 if (!skb) 135 return; 136 137 cb = (void *)skb->cb; 138 cb->state = NFP_NET_MBOX_CMSG_STATE_NEXT; 139 if (cb->posted) 140 queue_work(nn->mbox_cmsg.workq, &nn->mbox_cmsg.runq_work); 141 } 142 143 static void 144 nfp_ccm_mbox_write_tlv(struct nfp_net *nn, u32 off, u32 type, u32 len) 145 { 146 nn_writel(nn, off, 147 FIELD_PREP(NFP_NET_MBOX_TLV_TYPE, type) | 148 FIELD_PREP(NFP_NET_MBOX_TLV_LEN, len)); 149 } 150 151 static void nfp_ccm_mbox_copy_in(struct nfp_net *nn, struct sk_buff *last) 152 { 153 struct sk_buff *skb; 154 int reserve, i, cnt; 155 __be32 *data; 156 u32 off, len; 157 158 off = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL; 159 skb = __skb_peek(&nn->mbox_cmsg.queue); 160 while (true) { 161 nfp_ccm_mbox_write_tlv(nn, off, NFP_NET_MBOX_TLV_TYPE_MSG, 162 skb->len); 163 off += 4; 164 165 /* Write data word by word, skb->data should be aligned */ 166 data = (__be32 *)skb->data; 167 cnt = skb->len / 4; 168 for (i = 0 ; i < cnt; i++) { 169 nn_writel(nn, off, be32_to_cpu(data[i])); 170 off += 4; 171 } 172 if (skb->len & 3) { 173 __be32 tmp = 0; 174 175 memcpy(&tmp, &data[i], skb->len & 3); 176 nn_writel(nn, off, be32_to_cpu(tmp)); 177 off += 4; 178 } 179 180 /* Reserve space if reply is bigger */ 181 len = round_up(skb->len, 4); 182 reserve = nfp_ccm_mbox_maxlen(skb) - len; 183 if (reserve > 0) { 184 nfp_ccm_mbox_write_tlv(nn, off, 185 NFP_NET_MBOX_TLV_TYPE_RESV, 186 reserve); 187 off += 4 + reserve; 188 } 189 190 if (skb == last) 191 break; 192 skb = skb_queue_next(&nn->mbox_cmsg.queue, skb); 193 } 194 195 nfp_ccm_mbox_write_tlv(nn, off, NFP_NET_MBOX_TLV_TYPE_END, 0); 196 } 197 198 static struct sk_buff * 199 nfp_ccm_mbox_find_req(struct nfp_net *nn, __be16 tag, struct sk_buff *last) 200 { 201 struct sk_buff *skb; 202 203 skb = __skb_peek(&nn->mbox_cmsg.queue); 204 while (true) { 205 if (__nfp_ccm_get_tag(skb) == tag) 206 return skb; 207 208 if (skb == last) 209 return NULL; 210 skb = skb_queue_next(&nn->mbox_cmsg.queue, skb); 211 } 212 } 213 214 static void nfp_ccm_mbox_copy_out(struct nfp_net *nn, struct sk_buff *last) 215 { 216 struct nfp_ccm_mbox_cmsg_cb *cb; 217 u8 __iomem *data, *end; 218 struct sk_buff *skb; 219 220 data = nn->dp.ctrl_bar + nn->tlv_caps.mbox_off + 221 NFP_NET_CFG_MBOX_SIMPLE_VAL; 222 end = data + nn->tlv_caps.mbox_len; 223 224 while (true) { 225 unsigned int length, offset, type; 226 struct nfp_ccm_hdr hdr; 227 u32 tlv_hdr; 228 229 tlv_hdr = readl(data); 230 type = FIELD_GET(NFP_NET_MBOX_TLV_TYPE, tlv_hdr); 231 length = FIELD_GET(NFP_NET_MBOX_TLV_LEN, tlv_hdr); 232 offset = data - nn->dp.ctrl_bar; 233 234 /* Advance past the header */ 235 data += 4; 236 237 if (data + length > end) { 238 nn_dp_warn(&nn->dp, "mailbox oversized TLV type:%d offset:%u len:%u\n", 239 type, offset, length); 240 break; 241 } 242 243 if (type == NFP_NET_MBOX_TLV_TYPE_END) 244 break; 245 if (type == NFP_NET_MBOX_TLV_TYPE_RESV) 246 goto next_tlv; 247 if (type != NFP_NET_MBOX_TLV_TYPE_MSG && 248 type != NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) { 249 nn_dp_warn(&nn->dp, "mailbox unknown TLV type:%d offset:%u len:%u\n", 250 type, offset, length); 251 break; 252 } 253 254 if (length < 4) { 255 nn_dp_warn(&nn->dp, "mailbox msg too short to contain header TLV type:%d offset:%u len:%u\n", 256 type, offset, length); 257 break; 258 } 259 260 hdr.raw = cpu_to_be32(readl(data)); 261 262 skb = nfp_ccm_mbox_find_req(nn, hdr.tag, last); 263 if (!skb) { 264 nn_dp_warn(&nn->dp, "mailbox request not found:%u\n", 265 be16_to_cpu(hdr.tag)); 266 break; 267 } 268 cb = (void *)skb->cb; 269 270 if (type == NFP_NET_MBOX_TLV_TYPE_MSG_NOSUP) { 271 nn_dp_warn(&nn->dp, 272 "mailbox msg not supported type:%d\n", 273 nfp_ccm_get_type(skb)); 274 cb->err = -EIO; 275 goto next_tlv; 276 } 277 278 if (hdr.type != __NFP_CCM_REPLY(nfp_ccm_get_type(skb))) { 279 nn_dp_warn(&nn->dp, "mailbox msg reply wrong type:%u expected:%lu\n", 280 hdr.type, 281 __NFP_CCM_REPLY(nfp_ccm_get_type(skb))); 282 cb->err = -EIO; 283 goto next_tlv; 284 } 285 if (cb->exp_reply && length != cb->exp_reply) { 286 nn_dp_warn(&nn->dp, "mailbox msg reply wrong size type:%u expected:%u have:%u\n", 287 hdr.type, length, cb->exp_reply); 288 cb->err = -EIO; 289 goto next_tlv; 290 } 291 if (length > cb->max_len) { 292 nn_dp_warn(&nn->dp, "mailbox msg oversized reply type:%u max:%u have:%u\n", 293 hdr.type, cb->max_len, length); 294 cb->err = -EIO; 295 goto next_tlv; 296 } 297 298 if (!cb->posted) { 299 __be32 *skb_data; 300 int i, cnt; 301 302 if (length <= skb->len) 303 __skb_trim(skb, length); 304 else 305 skb_put(skb, length - skb->len); 306 307 /* We overcopy here slightly, but that's okay, 308 * the skb is large enough, and the garbage will 309 * be ignored (beyond skb->len). 310 */ 311 skb_data = (__be32 *)skb->data; 312 memcpy(skb_data, &hdr, 4); 313 314 cnt = DIV_ROUND_UP(length, 4); 315 for (i = 1 ; i < cnt; i++) 316 skb_data[i] = cpu_to_be32(readl(data + i * 4)); 317 } 318 319 cb->state = NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND; 320 next_tlv: 321 data += round_up(length, 4); 322 if (data + 4 > end) { 323 nn_dp_warn(&nn->dp, 324 "reached end of MBOX without END TLV\n"); 325 break; 326 } 327 } 328 329 smp_wmb(); /* order the skb->data vs. cb->state */ 330 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 331 do { 332 skb = __skb_dequeue(&nn->mbox_cmsg.queue); 333 cb = (void *)skb->cb; 334 335 if (cb->state != NFP_NET_MBOX_CMSG_STATE_REPLY_FOUND) { 336 cb->err = -ENOENT; 337 smp_wmb(); /* order the cb->err vs. cb->state */ 338 } 339 cb->state = NFP_NET_MBOX_CMSG_STATE_DONE; 340 341 if (cb->posted) { 342 if (cb->err) 343 nn_dp_warn(&nn->dp, 344 "mailbox posted msg failed type:%u err:%d\n", 345 nfp_ccm_get_type(skb), cb->err); 346 dev_consume_skb_any(skb); 347 } 348 } while (skb != last); 349 350 nfp_ccm_mbox_mark_next_runner(nn); 351 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 352 } 353 354 static void 355 nfp_ccm_mbox_mark_all_err(struct nfp_net *nn, struct sk_buff *last, int err) 356 { 357 struct nfp_ccm_mbox_cmsg_cb *cb; 358 struct sk_buff *skb; 359 360 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 361 do { 362 skb = __skb_dequeue(&nn->mbox_cmsg.queue); 363 cb = (void *)skb->cb; 364 365 cb->err = err; 366 smp_wmb(); /* order the cb->err vs. cb->state */ 367 cb->state = NFP_NET_MBOX_CMSG_STATE_DONE; 368 } while (skb != last); 369 370 nfp_ccm_mbox_mark_next_runner(nn); 371 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 372 } 373 374 static void nfp_ccm_mbox_run_queue_unlock(struct nfp_net *nn) 375 __releases(&nn->mbox_cmsg.queue.lock) 376 { 377 int space = nn->tlv_caps.mbox_len - NFP_NET_CFG_MBOX_SIMPLE_VAL; 378 struct sk_buff *skb, *last; 379 int cnt, err; 380 381 space -= 4; /* for End TLV */ 382 383 /* First skb must fit, because it's ours and we checked it fits */ 384 cnt = 1; 385 last = skb = __skb_peek(&nn->mbox_cmsg.queue); 386 space -= 4 + nfp_ccm_mbox_maxlen(skb); 387 388 while (!skb_queue_is_last(&nn->mbox_cmsg.queue, last)) { 389 skb = skb_queue_next(&nn->mbox_cmsg.queue, last); 390 space -= 4 + nfp_ccm_mbox_maxlen(skb); 391 if (space < 0) 392 break; 393 last = skb; 394 nfp_ccm_mbox_set_busy(skb); 395 cnt++; 396 if (cnt == NFP_CCM_MBOX_BATCH_LIMIT) 397 break; 398 } 399 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 400 401 /* Now we own all skb's marked in progress, new requests may arrive 402 * at the end of the queue. 403 */ 404 405 nn_ctrl_bar_lock(nn); 406 407 nfp_ccm_mbox_copy_in(nn, last); 408 409 err = nfp_net_mbox_reconfig(nn, NFP_NET_CFG_MBOX_CMD_TLV_CMSG); 410 if (!err) 411 nfp_ccm_mbox_copy_out(nn, last); 412 else 413 nfp_ccm_mbox_mark_all_err(nn, last, -EIO); 414 415 nn_ctrl_bar_unlock(nn); 416 417 wake_up_all(&nn->mbox_cmsg.wq); 418 } 419 420 static int nfp_ccm_mbox_skb_return(struct sk_buff *skb) 421 { 422 struct nfp_ccm_mbox_cmsg_cb *cb = (void *)skb->cb; 423 424 if (cb->err) 425 dev_kfree_skb_any(skb); 426 return cb->err; 427 } 428 429 /* If wait timed out but the command is already in progress we have 430 * to wait until it finishes. Runners has ownership of the skbs marked 431 * as busy. 432 */ 433 static int 434 nfp_ccm_mbox_unlink_unlock(struct nfp_net *nn, struct sk_buff *skb, 435 enum nfp_ccm_type type) 436 __releases(&nn->mbox_cmsg.queue.lock) 437 { 438 bool was_first; 439 440 if (nfp_ccm_mbox_in_progress(skb)) { 441 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 442 443 wait_event(nn->mbox_cmsg.wq, nfp_ccm_mbox_done(skb)); 444 smp_rmb(); /* pairs with smp_wmb() after data is written */ 445 return nfp_ccm_mbox_skb_return(skb); 446 } 447 448 was_first = nfp_ccm_mbox_should_run(nn, skb); 449 __skb_unlink(skb, &nn->mbox_cmsg.queue); 450 if (was_first) 451 nfp_ccm_mbox_mark_next_runner(nn); 452 453 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 454 455 if (was_first) 456 wake_up_all(&nn->mbox_cmsg.wq); 457 458 nn_dp_warn(&nn->dp, "time out waiting for mbox response to 0x%02x\n", 459 type); 460 return -ETIMEDOUT; 461 } 462 463 static int 464 nfp_ccm_mbox_msg_prepare(struct nfp_net *nn, struct sk_buff *skb, 465 enum nfp_ccm_type type, 466 unsigned int reply_size, unsigned int max_reply_size, 467 gfp_t flags) 468 { 469 const unsigned int mbox_max = nfp_ccm_mbox_max_msg(nn); 470 unsigned int max_len; 471 ssize_t undersize; 472 int err; 473 474 if (unlikely(!(nn->tlv_caps.mbox_cmsg_types & BIT(type)))) { 475 nn_dp_warn(&nn->dp, 476 "message type %d not supported by mailbox\n", type); 477 return -EINVAL; 478 } 479 480 /* If the reply size is unknown assume it will take the entire 481 * mailbox, the callers should do their best for this to never 482 * happen. 483 */ 484 if (!max_reply_size) 485 max_reply_size = mbox_max; 486 max_reply_size = round_up(max_reply_size, 4); 487 488 /* Make sure we can fit the entire reply into the skb, 489 * and that we don't have to slow down the mbox handler 490 * with allocations. 491 */ 492 undersize = max_reply_size - (skb_end_pointer(skb) - skb->data); 493 if (undersize > 0) { 494 err = pskb_expand_head(skb, 0, undersize, flags); 495 if (err) { 496 nn_dp_warn(&nn->dp, 497 "can't allocate reply buffer for mailbox\n"); 498 return err; 499 } 500 } 501 502 /* Make sure that request and response both fit into the mailbox */ 503 max_len = max(max_reply_size, round_up(skb->len, 4)); 504 if (max_len > mbox_max) { 505 nn_dp_warn(&nn->dp, 506 "message too big for tha mailbox: %u/%u vs %u\n", 507 skb->len, max_reply_size, mbox_max); 508 return -EMSGSIZE; 509 } 510 511 nfp_ccm_mbox_msg_init(skb, reply_size, max_len); 512 513 return 0; 514 } 515 516 static int 517 nfp_ccm_mbox_msg_enqueue(struct nfp_net *nn, struct sk_buff *skb, 518 enum nfp_ccm_type type, bool critical) 519 { 520 struct nfp_ccm_hdr *hdr; 521 522 assert_spin_locked(&nn->mbox_cmsg.queue.lock); 523 524 if (!critical && nn->mbox_cmsg.queue.qlen >= NFP_CCM_MAX_QLEN) { 525 nn_dp_warn(&nn->dp, "mailbox request queue too long\n"); 526 return -EBUSY; 527 } 528 529 hdr = (void *)skb->data; 530 hdr->ver = NFP_CCM_ABI_VERSION; 531 hdr->type = type; 532 hdr->tag = cpu_to_be16(nn->mbox_cmsg.tag++); 533 534 __skb_queue_tail(&nn->mbox_cmsg.queue, skb); 535 536 return 0; 537 } 538 539 int __nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb, 540 enum nfp_ccm_type type, 541 unsigned int reply_size, 542 unsigned int max_reply_size, bool critical) 543 { 544 int err; 545 546 err = nfp_ccm_mbox_msg_prepare(nn, skb, type, reply_size, 547 max_reply_size, GFP_KERNEL); 548 if (err) 549 goto err_free_skb; 550 551 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 552 553 err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, critical); 554 if (err) 555 goto err_unlock; 556 557 /* First in queue takes the mailbox lock and processes the batch */ 558 if (!nfp_ccm_mbox_is_first(nn, skb)) { 559 bool to; 560 561 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 562 563 to = !wait_event_timeout(nn->mbox_cmsg.wq, 564 nfp_ccm_mbox_done(skb) || 565 nfp_ccm_mbox_should_run(nn, skb), 566 msecs_to_jiffies(NFP_CCM_TIMEOUT)); 567 568 /* fast path for those completed by another thread */ 569 if (nfp_ccm_mbox_done(skb)) { 570 smp_rmb(); /* pairs with wmb after data is written */ 571 return nfp_ccm_mbox_skb_return(skb); 572 } 573 574 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 575 576 if (!nfp_ccm_mbox_is_first(nn, skb)) { 577 WARN_ON(!to); 578 579 err = nfp_ccm_mbox_unlink_unlock(nn, skb, type); 580 if (err) 581 goto err_free_skb; 582 return 0; 583 } 584 } 585 586 /* run queue expects the lock held */ 587 nfp_ccm_mbox_run_queue_unlock(nn); 588 return nfp_ccm_mbox_skb_return(skb); 589 590 err_unlock: 591 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 592 err_free_skb: 593 dev_kfree_skb_any(skb); 594 return err; 595 } 596 597 int nfp_ccm_mbox_communicate(struct nfp_net *nn, struct sk_buff *skb, 598 enum nfp_ccm_type type, 599 unsigned int reply_size, 600 unsigned int max_reply_size) 601 { 602 return __nfp_ccm_mbox_communicate(nn, skb, type, reply_size, 603 max_reply_size, false); 604 } 605 606 static void nfp_ccm_mbox_post_runq_work(struct work_struct *work) 607 { 608 struct sk_buff *skb; 609 struct nfp_net *nn; 610 611 nn = container_of(work, struct nfp_net, mbox_cmsg.runq_work); 612 613 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 614 615 skb = __skb_peek(&nn->mbox_cmsg.queue); 616 if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb) || 617 !nfp_ccm_mbox_should_run(nn, skb))) { 618 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 619 return; 620 } 621 622 nfp_ccm_mbox_run_queue_unlock(nn); 623 } 624 625 static void nfp_ccm_mbox_post_wait_work(struct work_struct *work) 626 { 627 struct sk_buff *skb; 628 struct nfp_net *nn; 629 int err; 630 631 nn = container_of(work, struct nfp_net, mbox_cmsg.wait_work); 632 633 skb = skb_peek(&nn->mbox_cmsg.queue); 634 if (WARN_ON(!skb || !nfp_ccm_mbox_is_posted(skb))) 635 /* Should never happen so it's unclear what to do here.. */ 636 goto exit_unlock_wake; 637 638 err = nfp_net_mbox_reconfig_wait_posted(nn); 639 if (!err) 640 nfp_ccm_mbox_copy_out(nn, skb); 641 else 642 nfp_ccm_mbox_mark_all_err(nn, skb, -EIO); 643 exit_unlock_wake: 644 nn_ctrl_bar_unlock(nn); 645 wake_up_all(&nn->mbox_cmsg.wq); 646 } 647 648 int nfp_ccm_mbox_post(struct nfp_net *nn, struct sk_buff *skb, 649 enum nfp_ccm_type type, unsigned int max_reply_size) 650 { 651 int err; 652 653 err = nfp_ccm_mbox_msg_prepare(nn, skb, type, 0, max_reply_size, 654 GFP_ATOMIC); 655 if (err) 656 goto err_free_skb; 657 658 nfp_ccm_mbox_mark_posted(skb); 659 660 spin_lock_bh(&nn->mbox_cmsg.queue.lock); 661 662 err = nfp_ccm_mbox_msg_enqueue(nn, skb, type, false); 663 if (err) 664 goto err_unlock; 665 666 if (nfp_ccm_mbox_is_first(nn, skb)) { 667 if (nn_ctrl_bar_trylock(nn)) { 668 nfp_ccm_mbox_copy_in(nn, skb); 669 nfp_net_mbox_reconfig_post(nn, 670 NFP_NET_CFG_MBOX_CMD_TLV_CMSG); 671 queue_work(nn->mbox_cmsg.workq, 672 &nn->mbox_cmsg.wait_work); 673 } else { 674 nfp_ccm_mbox_mark_next_runner(nn); 675 } 676 } 677 678 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 679 680 return 0; 681 682 err_unlock: 683 spin_unlock_bh(&nn->mbox_cmsg.queue.lock); 684 err_free_skb: 685 dev_kfree_skb_any(skb); 686 return err; 687 } 688 689 struct sk_buff * 690 nfp_ccm_mbox_msg_alloc(struct nfp_net *nn, unsigned int req_size, 691 unsigned int reply_size, gfp_t flags) 692 { 693 unsigned int max_size; 694 struct sk_buff *skb; 695 696 if (!reply_size) 697 max_size = nfp_ccm_mbox_max_msg(nn); 698 else 699 max_size = max(req_size, reply_size); 700 max_size = round_up(max_size, 4); 701 702 skb = alloc_skb(max_size, flags); 703 if (!skb) 704 return NULL; 705 706 skb_put(skb, req_size); 707 708 return skb; 709 } 710 711 bool nfp_ccm_mbox_fits(struct nfp_net *nn, unsigned int size) 712 { 713 return nfp_ccm_mbox_max_msg(nn) >= size; 714 } 715 716 int nfp_ccm_mbox_init(struct nfp_net *nn) 717 { 718 return 0; 719 } 720 721 void nfp_ccm_mbox_clean(struct nfp_net *nn) 722 { 723 drain_workqueue(nn->mbox_cmsg.workq); 724 } 725 726 int nfp_ccm_mbox_alloc(struct nfp_net *nn) 727 { 728 skb_queue_head_init(&nn->mbox_cmsg.queue); 729 init_waitqueue_head(&nn->mbox_cmsg.wq); 730 INIT_WORK(&nn->mbox_cmsg.wait_work, nfp_ccm_mbox_post_wait_work); 731 INIT_WORK(&nn->mbox_cmsg.runq_work, nfp_ccm_mbox_post_runq_work); 732 733 nn->mbox_cmsg.workq = alloc_workqueue("nfp-ccm-mbox", WQ_UNBOUND, 0); 734 if (!nn->mbox_cmsg.workq) 735 return -ENOMEM; 736 return 0; 737 } 738 739 void nfp_ccm_mbox_free(struct nfp_net *nn) 740 { 741 destroy_workqueue(nn->mbox_cmsg.workq); 742 WARN_ON(!skb_queue_empty(&nn->mbox_cmsg.queue)); 743 } 744