1 #include <linux/kernel.h> 2 #include <linux/errno.h> 3 #include <linux/file.h> 4 #include <linux/slab.h> 5 #include <linux/net.h> 6 #include <linux/io_uring.h> 7 8 #include "io_uring.h" 9 #include "notif.h" 10 #include "rsrc.h" 11 12 static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked) 13 { 14 struct io_notif_data *nd = io_notif_to_data(notif); 15 struct io_ring_ctx *ctx = notif->ctx; 16 17 if (nd->account_pages && ctx->user) { 18 __io_unaccount_mem(ctx->user, nd->account_pages); 19 nd->account_pages = 0; 20 } 21 io_req_task_complete(notif, locked); 22 } 23 24 static inline void io_notif_complete(struct io_kiocb *notif) 25 __must_hold(¬if->ctx->uring_lock) 26 { 27 bool locked = true; 28 29 __io_notif_complete_tw(notif, &locked); 30 } 31 32 static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, 33 struct ubuf_info *uarg, 34 bool success) 35 { 36 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); 37 struct io_kiocb *notif = cmd_to_io_kiocb(nd); 38 39 if (refcount_dec_and_test(&uarg->refcnt)) { 40 notif->io_task_work.func = __io_notif_complete_tw; 41 io_req_task_work_add(notif); 42 } 43 } 44 45 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx) 46 __must_hold(&ctx->uring_lock) 47 { 48 struct io_kiocb *notif; 49 struct io_notif_data *nd; 50 51 if (unlikely(!io_alloc_req_refill(ctx))) 52 return NULL; 53 notif = io_alloc_req(ctx); 54 notif->opcode = IORING_OP_NOP; 55 notif->flags = 0; 56 notif->file = NULL; 57 notif->task = current; 58 io_get_task_refs(1); 59 notif->rsrc_node = NULL; 60 io_req_set_rsrc_node(notif, ctx, 0); 61 62 nd = io_notif_to_data(notif); 63 nd->account_pages = 0; 64 nd->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; 65 nd->uarg.callback = io_uring_tx_zerocopy_callback; 66 refcount_set(&nd->uarg.refcnt, 1); 67 return notif; 68 } 69 70 void io_notif_flush(struct io_kiocb *notif) 71 __must_hold(&slot->notif->ctx->uring_lock) 72 { 73 struct io_notif_data *nd = io_notif_to_data(notif); 74 75 /* drop slot's master ref */ 76 if (refcount_dec_and_test(&nd->uarg.refcnt)) { 77 notif->io_task_work.func = __io_notif_complete_tw; 78 io_req_task_work_add(notif); 79 } 80 } 81