1eb42cebbSPavel Begunkov // SPDX-License-Identifier: GPL-2.0 2eb42cebbSPavel Begunkov 3eb42cebbSPavel Begunkov #include <linux/net.h> 4eb42cebbSPavel Begunkov #include <linux/uio.h> 5eb42cebbSPavel Begunkov #include <net/sock.h> 6eb42cebbSPavel Begunkov #include <linux/nospec.h> 7eb42cebbSPavel Begunkov 86a9ce66fSPavel Begunkov #include "rsrc.h" 96a9ce66fSPavel Begunkov 10*519760dfSPavel Begunkov #define IO_NOTIF_UBUF_FLAGS (SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN) 11eb4a299bSPavel Begunkov #define IO_NOTIF_SPLICE_BATCH 32 12eb4a299bSPavel Begunkov 1314b146b6SPavel Begunkov struct io_notif_data { 1414b146b6SPavel Begunkov struct file *file; 15eb42cebbSPavel Begunkov struct ubuf_info uarg; 166a9ce66fSPavel Begunkov unsigned long account_pages; 17e307e669SStefan Metzmacher bool zc_report; 18e307e669SStefan Metzmacher bool zc_used; 19e307e669SStefan Metzmacher bool zc_copied; 20eb42cebbSPavel Begunkov }; 21eb42cebbSPavel Begunkov 22b48c312bSPavel Begunkov struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx); 2340725d1bSPavel Begunkov void io_notif_set_extended(struct io_kiocb *notif); 24eb42cebbSPavel Begunkov io_notif_to_data(struct io_kiocb * notif)2514b146b6SPavel Begunkovstatic inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) 2614b146b6SPavel Begunkov { 27f2ccb5aeSStefan Metzmacher return io_kiocb_to_cmd(notif, struct io_notif_data); 2814b146b6SPavel Begunkov } 2914b146b6SPavel Begunkov io_notif_flush(struct io_kiocb * notif)30bedd20bcSPavel Begunkovstatic inline void io_notif_flush(struct io_kiocb *notif) 31bedd20bcSPavel Begunkov __must_hold(¬if->ctx->uring_lock) 32bedd20bcSPavel Begunkov { 33bedd20bcSPavel Begunkov struct io_notif_data *nd = io_notif_to_data(notif); 34bedd20bcSPavel Begunkov 35bedd20bcSPavel Begunkov /* drop slot's master ref */ 36bedd20bcSPavel Begunkov if (refcount_dec_and_test(&nd->uarg.refcnt)) 378751d154SPavel Begunkov __io_req_task_work_add(notif, IOU_F_TWQ_LAZY_WAKE); 38bedd20bcSPavel Begunkov } 39bedd20bcSPavel Begunkov io_notif_account_mem(struct io_kiocb * notif,unsigned len)4014b146b6SPavel Begunkovstatic inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len) 416a9ce66fSPavel Begunkov { 426a9ce66fSPavel Begunkov struct io_ring_ctx *ctx = notif->ctx; 4314b146b6SPavel Begunkov struct io_notif_data *nd = io_notif_to_data(notif); 446a9ce66fSPavel Begunkov unsigned nr_pages = (len >> PAGE_SHIFT) + 2; 456a9ce66fSPavel Begunkov int ret; 466a9ce66fSPavel Begunkov 476a9ce66fSPavel Begunkov if (ctx->user) { 486a9ce66fSPavel Begunkov ret = __io_account_mem(ctx->user, nr_pages); 496a9ce66fSPavel Begunkov if (ret) 506a9ce66fSPavel Begunkov return ret; 5114b146b6SPavel Begunkov nd->account_pages += nr_pages; 526a9ce66fSPavel Begunkov } 536a9ce66fSPavel Begunkov return 0; 546a9ce66fSPavel Begunkov } 55