1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/net.h> 4 #include <linux/uio.h> 5 #include <net/sock.h> 6 #include <linux/nospec.h> 7 8 #include "rsrc.h" 9 10 #define IO_NOTIF_SPLICE_BATCH 32 11 #define IORING_MAX_NOTIF_SLOTS (1U << 10) 12 13 struct io_notif_data { 14 struct file *file; 15 struct ubuf_info uarg; 16 unsigned long account_pages; 17 }; 18 19 struct io_notif_slot { 20 /* 21 * Current/active notifier. A slot holds only one active notifier at a 22 * time and keeps one reference to it. Flush releases the reference and 23 * lazily replaces it with a new notifier. 24 */ 25 struct io_kiocb *notif; 26 27 /* 28 * Default ->user_data for this slot notifiers CQEs 29 */ 30 u64 tag; 31 /* 32 * Notifiers of a slot live in generations, we create a new notifier 33 * only after flushing the previous one. Track the sequential number 34 * for all notifiers and copy it into notifiers's cqe->cflags 35 */ 36 u32 seq; 37 }; 38 39 int io_notif_register(struct io_ring_ctx *ctx, 40 void __user *arg, unsigned int size); 41 int io_notif_unregister(struct io_ring_ctx *ctx); 42 43 void io_notif_slot_flush(struct io_notif_slot *slot); 44 struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, 45 struct io_notif_slot *slot); 46 47 static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) 48 { 49 return io_kiocb_to_cmd(notif, struct io_notif_data); 50 } 51 52 static inline struct io_kiocb *io_get_notif(struct io_ring_ctx *ctx, 53 struct io_notif_slot *slot) 54 { 55 if (!slot->notif) 56 slot->notif = io_alloc_notif(ctx, slot); 57 return slot->notif; 58 } 59 60 static inline struct io_notif_slot *io_get_notif_slot(struct io_ring_ctx *ctx, 61 unsigned idx) 62 __must_hold(&ctx->uring_lock) 63 { 64 if (idx >= ctx->nr_notif_slots) 65 return NULL; 66 idx = array_index_nospec(idx, ctx->nr_notif_slots); 67 return &ctx->notif_slots[idx]; 68 } 69 70 static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot, 71 unsigned int issue_flags) 72 { 73 io_notif_slot_flush(slot); 74 } 75 76 static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) 77 { 78 struct io_ring_ctx *ctx = notif->ctx; 79 struct io_notif_data *nd = io_notif_to_data(notif); 80 unsigned nr_pages = (len >> PAGE_SHIFT) + 2; 81 int ret; 82 83 if (ctx->user) { 84 ret = __io_account_mem(ctx->user, nr_pages); 85 if (ret) 86 return ret; 87 nd->account_pages += nr_pages; 88 } 89 return 0; 90 } 91