1f9ead18cSJens Axboe // SPDX-License-Identifier: GPL-2.0 2f9ead18cSJens Axboe #include <linux/kernel.h> 3f9ead18cSJens Axboe #include <linux/errno.h> 4f9ead18cSJens Axboe #include <linux/file.h> 5f9ead18cSJens Axboe #include <linux/slab.h> 6f9ead18cSJens Axboe #include <linux/net.h> 7f9ead18cSJens Axboe #include <linux/compat.h> 8f9ead18cSJens Axboe #include <net/compat.h> 9f9ead18cSJens Axboe #include <linux/io_uring.h> 10f9ead18cSJens Axboe 11f9ead18cSJens Axboe #include <uapi/linux/io_uring.h> 12f9ead18cSJens Axboe 13f9ead18cSJens Axboe #include "io_uring.h" 143b77495aSJens Axboe #include "kbuf.h" 1543e0bbbdSJens Axboe #include "alloc_cache.h" 16f9ead18cSJens Axboe #include "net.h" 1706a5464bSPavel Begunkov #include "notif.h" 1810c7d33eSPavel Begunkov #include "rsrc.h" 19f9ead18cSJens Axboe 20f9ead18cSJens Axboe #if defined(CONFIG_NET) 21f9ead18cSJens Axboe struct io_shutdown { 22f9ead18cSJens Axboe struct file *file; 23f9ead18cSJens Axboe int how; 24f9ead18cSJens Axboe }; 25f9ead18cSJens Axboe 26f9ead18cSJens Axboe struct io_accept { 27f9ead18cSJens Axboe struct file *file; 28f9ead18cSJens Axboe struct sockaddr __user *addr; 29f9ead18cSJens Axboe int __user *addr_len; 30f9ead18cSJens Axboe int flags; 31f9ead18cSJens Axboe u32 file_slot; 32f9ead18cSJens Axboe unsigned long nofile; 33f9ead18cSJens Axboe }; 34f9ead18cSJens Axboe 35f9ead18cSJens Axboe struct io_socket { 36f9ead18cSJens Axboe struct file *file; 37f9ead18cSJens Axboe int domain; 38f9ead18cSJens Axboe int type; 39f9ead18cSJens Axboe int protocol; 40f9ead18cSJens Axboe int flags; 41f9ead18cSJens Axboe u32 file_slot; 42f9ead18cSJens Axboe unsigned long nofile; 43f9ead18cSJens Axboe }; 44f9ead18cSJens Axboe 45f9ead18cSJens Axboe struct io_connect { 46f9ead18cSJens Axboe struct file *file; 47f9ead18cSJens Axboe struct sockaddr __user *addr; 48f9ead18cSJens Axboe int addr_len; 49f9ead18cSJens Axboe }; 50f9ead18cSJens Axboe 51f9ead18cSJens Axboe struct io_sr_msg { 52f9ead18cSJens Axboe struct file *file; 53f9ead18cSJens Axboe union { 54f9ead18cSJens Axboe struct compat_msghdr __user *umsg_compat; 55f9ead18cSJens Axboe struct user_msghdr __user *umsg; 56f9ead18cSJens Axboe void __user *buf; 57f9ead18cSJens Axboe }; 580b048557SPavel Begunkov unsigned len; 590b048557SPavel Begunkov unsigned done_io; 60293402e5SPavel Begunkov unsigned msg_flags; 610b048557SPavel Begunkov u16 flags; 62516e82f0SPavel Begunkov /* initialised and used only by !msg send variants */ 630b048557SPavel Begunkov u16 addr_len; 64092aeedbSPavel Begunkov void __user *addr; 65516e82f0SPavel Begunkov /* used only for send zerocopy */ 66b48c312bSPavel Begunkov struct io_kiocb *notif; 6706a5464bSPavel Begunkov }; 6806a5464bSPavel Begunkov 69f9ead18cSJens Axboe #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) 70f9ead18cSJens Axboe 71f9ead18cSJens Axboe int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 72f9ead18cSJens Axboe { 73f2ccb5aeSStefan Metzmacher struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 74f9ead18cSJens Axboe 75f9ead18cSJens Axboe if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 76f9ead18cSJens Axboe sqe->buf_index || sqe->splice_fd_in)) 77f9ead18cSJens Axboe return -EINVAL; 78f9ead18cSJens Axboe 79f9ead18cSJens Axboe shutdown->how = READ_ONCE(sqe->len); 80f9ead18cSJens Axboe return 0; 81f9ead18cSJens Axboe } 82f9ead18cSJens Axboe 83f9ead18cSJens Axboe int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 84f9ead18cSJens Axboe { 85f2ccb5aeSStefan Metzmacher struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 86f9ead18cSJens Axboe struct socket *sock; 87f9ead18cSJens Axboe int ret; 88f9ead18cSJens Axboe 89f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 90f9ead18cSJens Axboe return -EAGAIN; 91f9ead18cSJens Axboe 92f9ead18cSJens Axboe sock = sock_from_file(req->file); 93f9ead18cSJens Axboe if (unlikely(!sock)) 94f9ead18cSJens Axboe return -ENOTSOCK; 95f9ead18cSJens Axboe 96f9ead18cSJens Axboe ret = __sys_shutdown_sock(sock, shutdown->how); 97f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 98f9ead18cSJens Axboe return IOU_OK; 99f9ead18cSJens Axboe } 100f9ead18cSJens Axboe 101f9ead18cSJens Axboe static bool io_net_retry(struct socket *sock, int flags) 102f9ead18cSJens Axboe { 103f9ead18cSJens Axboe if (!(flags & MSG_WAITALL)) 104f9ead18cSJens Axboe return false; 105f9ead18cSJens Axboe return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 106f9ead18cSJens Axboe } 107f9ead18cSJens Axboe 10843e0bbbdSJens Axboe static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 10943e0bbbdSJens Axboe { 11043e0bbbdSJens Axboe struct io_async_msghdr *hdr = req->async_data; 11143e0bbbdSJens Axboe 11206360426SPavel Begunkov if (!req_has_async_data(req) || issue_flags & IO_URING_F_UNLOCKED) 11343e0bbbdSJens Axboe return; 11443e0bbbdSJens Axboe 11543e0bbbdSJens Axboe /* Let normal cleanup path reap it if we fail adding to the cache */ 11643e0bbbdSJens Axboe if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) { 11743e0bbbdSJens Axboe req->async_data = NULL; 11843e0bbbdSJens Axboe req->flags &= ~REQ_F_ASYNC_DATA; 11943e0bbbdSJens Axboe } 12043e0bbbdSJens Axboe } 12143e0bbbdSJens Axboe 122858c293eSPavel Begunkov static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req, 12343e0bbbdSJens Axboe unsigned int issue_flags) 12443e0bbbdSJens Axboe { 12543e0bbbdSJens Axboe struct io_ring_ctx *ctx = req->ctx; 12643e0bbbdSJens Axboe struct io_cache_entry *entry; 1274c17a496SPavel Begunkov struct io_async_msghdr *hdr; 12843e0bbbdSJens Axboe 12943e0bbbdSJens Axboe if (!(issue_flags & IO_URING_F_UNLOCKED) && 13043e0bbbdSJens Axboe (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { 13143e0bbbdSJens Axboe hdr = container_of(entry, struct io_async_msghdr, cache); 1324c17a496SPavel Begunkov hdr->free_iov = NULL; 13343e0bbbdSJens Axboe req->flags |= REQ_F_ASYNC_DATA; 13443e0bbbdSJens Axboe req->async_data = hdr; 13543e0bbbdSJens Axboe return hdr; 13643e0bbbdSJens Axboe } 13743e0bbbdSJens Axboe 1384c17a496SPavel Begunkov if (!io_alloc_async_data(req)) { 1394c17a496SPavel Begunkov hdr = req->async_data; 1404c17a496SPavel Begunkov hdr->free_iov = NULL; 1414c17a496SPavel Begunkov return hdr; 1424c17a496SPavel Begunkov } 14343e0bbbdSJens Axboe return NULL; 14443e0bbbdSJens Axboe } 14543e0bbbdSJens Axboe 146858c293eSPavel Begunkov static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req) 147858c293eSPavel Begunkov { 148858c293eSPavel Begunkov /* ->prep_async is always called from the submission context */ 149858c293eSPavel Begunkov return io_msg_alloc_async(req, 0); 150858c293eSPavel Begunkov } 151858c293eSPavel Begunkov 152f9ead18cSJens Axboe static int io_setup_async_msg(struct io_kiocb *req, 15343e0bbbdSJens Axboe struct io_async_msghdr *kmsg, 15443e0bbbdSJens Axboe unsigned int issue_flags) 155f9ead18cSJens Axboe { 1563f743e9bSPavel Begunkov struct io_async_msghdr *async_msg; 157f9ead18cSJens Axboe 1583f743e9bSPavel Begunkov if (req_has_async_data(req)) 159f9ead18cSJens Axboe return -EAGAIN; 160858c293eSPavel Begunkov async_msg = io_msg_alloc_async(req, issue_flags); 16143e0bbbdSJens Axboe if (!async_msg) { 162f9ead18cSJens Axboe kfree(kmsg->free_iov); 163f9ead18cSJens Axboe return -ENOMEM; 164f9ead18cSJens Axboe } 165f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 166f9ead18cSJens Axboe memcpy(async_msg, kmsg, sizeof(*kmsg)); 1676f10ae8aSPavel Begunkov if (async_msg->msg.msg_name) 168f9ead18cSJens Axboe async_msg->msg.msg_name = &async_msg->addr; 169f9ead18cSJens Axboe /* if were using fast_iov, set it to the new one */ 1703e4cb6ebSStefan Metzmacher if (!kmsg->free_iov) { 1713e4cb6ebSStefan Metzmacher size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; 1723e4cb6ebSStefan Metzmacher async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; 1733e4cb6ebSStefan Metzmacher } 174f9ead18cSJens Axboe 175f9ead18cSJens Axboe return -EAGAIN; 176f9ead18cSJens Axboe } 177f9ead18cSJens Axboe 178f9ead18cSJens Axboe static int io_sendmsg_copy_hdr(struct io_kiocb *req, 179f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 180f9ead18cSJens Axboe { 181f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 182f9ead18cSJens Axboe 183f9ead18cSJens Axboe iomsg->msg.msg_name = &iomsg->addr; 184f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 185f9ead18cSJens Axboe return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 186f9ead18cSJens Axboe &iomsg->free_iov); 187f9ead18cSJens Axboe } 188f9ead18cSJens Axboe 189516e82f0SPavel Begunkov int io_send_prep_async(struct io_kiocb *req) 190581711c4SPavel Begunkov { 191ac9e5784SPavel Begunkov struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 192581711c4SPavel Begunkov struct io_async_msghdr *io; 193581711c4SPavel Begunkov int ret; 194581711c4SPavel Begunkov 195581711c4SPavel Begunkov if (!zc->addr || req_has_async_data(req)) 196581711c4SPavel Begunkov return 0; 1976bf8ad25SPavel Begunkov io = io_msg_alloc_async_prep(req); 1986bf8ad25SPavel Begunkov if (!io) 199581711c4SPavel Begunkov return -ENOMEM; 200581711c4SPavel Begunkov ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); 201581711c4SPavel Begunkov return ret; 202581711c4SPavel Begunkov } 203581711c4SPavel Begunkov 204581711c4SPavel Begunkov static int io_setup_async_addr(struct io_kiocb *req, 2056ae61b7aSPavel Begunkov struct sockaddr_storage *addr_storage, 206581711c4SPavel Begunkov unsigned int issue_flags) 207581711c4SPavel Begunkov { 2086ae61b7aSPavel Begunkov struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 209581711c4SPavel Begunkov struct io_async_msghdr *io; 210581711c4SPavel Begunkov 2116ae61b7aSPavel Begunkov if (!sr->addr || req_has_async_data(req)) 212581711c4SPavel Begunkov return -EAGAIN; 2136bf8ad25SPavel Begunkov io = io_msg_alloc_async(req, issue_flags); 2146bf8ad25SPavel Begunkov if (!io) 215581711c4SPavel Begunkov return -ENOMEM; 2166ae61b7aSPavel Begunkov memcpy(&io->addr, addr_storage, sizeof(io->addr)); 217581711c4SPavel Begunkov return -EAGAIN; 218581711c4SPavel Begunkov } 219581711c4SPavel Begunkov 220f9ead18cSJens Axboe int io_sendmsg_prep_async(struct io_kiocb *req) 221f9ead18cSJens Axboe { 222f9ead18cSJens Axboe int ret; 223f9ead18cSJens Axboe 224858c293eSPavel Begunkov if (!io_msg_alloc_async_prep(req)) 225858c293eSPavel Begunkov return -ENOMEM; 226f9ead18cSJens Axboe ret = io_sendmsg_copy_hdr(req, req->async_data); 227f9ead18cSJens Axboe if (!ret) 228f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 229f9ead18cSJens Axboe return ret; 230f9ead18cSJens Axboe } 231f9ead18cSJens Axboe 232f9ead18cSJens Axboe void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 233f9ead18cSJens Axboe { 234f9ead18cSJens Axboe struct io_async_msghdr *io = req->async_data; 235f9ead18cSJens Axboe 236f9ead18cSJens Axboe kfree(io->free_iov); 237f9ead18cSJens Axboe } 238f9ead18cSJens Axboe 239f9ead18cSJens Axboe int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 240f9ead18cSJens Axboe { 241f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 242f9ead18cSJens Axboe 243516e82f0SPavel Begunkov if (req->opcode == IORING_OP_SEND) { 244516e82f0SPavel Begunkov if (READ_ONCE(sqe->__pad3[0])) 245f9ead18cSJens Axboe return -EINVAL; 246516e82f0SPavel Begunkov sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 247516e82f0SPavel Begunkov sr->addr_len = READ_ONCE(sqe->addr_len); 248516e82f0SPavel Begunkov } else if (sqe->addr2 || sqe->file_index) { 249516e82f0SPavel Begunkov return -EINVAL; 250516e82f0SPavel Begunkov } 251f9ead18cSJens Axboe 252f9ead18cSJens Axboe sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 253f9ead18cSJens Axboe sr->len = READ_ONCE(sqe->len); 254f9ead18cSJens Axboe sr->flags = READ_ONCE(sqe->ioprio); 255f9ead18cSJens Axboe if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 256f9ead18cSJens Axboe return -EINVAL; 257f9ead18cSJens Axboe sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 258f9ead18cSJens Axboe if (sr->msg_flags & MSG_DONTWAIT) 259f9ead18cSJens Axboe req->flags |= REQ_F_NOWAIT; 260f9ead18cSJens Axboe 261f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 262f9ead18cSJens Axboe if (req->ctx->compat) 263f9ead18cSJens Axboe sr->msg_flags |= MSG_CMSG_COMPAT; 264f9ead18cSJens Axboe #endif 265f9ead18cSJens Axboe sr->done_io = 0; 266f9ead18cSJens Axboe return 0; 267f9ead18cSJens Axboe } 268f9ead18cSJens Axboe 269f9ead18cSJens Axboe int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 270f9ead18cSJens Axboe { 271f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 272f9ead18cSJens Axboe struct io_async_msghdr iomsg, *kmsg; 273f9ead18cSJens Axboe struct socket *sock; 274f9ead18cSJens Axboe unsigned flags; 275f9ead18cSJens Axboe int min_ret = 0; 276f9ead18cSJens Axboe int ret; 277f9ead18cSJens Axboe 278f9ead18cSJens Axboe sock = sock_from_file(req->file); 279f9ead18cSJens Axboe if (unlikely(!sock)) 280f9ead18cSJens Axboe return -ENOTSOCK; 281f9ead18cSJens Axboe 282f9ead18cSJens Axboe if (req_has_async_data(req)) { 283f9ead18cSJens Axboe kmsg = req->async_data; 284f9ead18cSJens Axboe } else { 285f9ead18cSJens Axboe ret = io_sendmsg_copy_hdr(req, &iomsg); 286f9ead18cSJens Axboe if (ret) 287f9ead18cSJens Axboe return ret; 288f9ead18cSJens Axboe kmsg = &iomsg; 289f9ead18cSJens Axboe } 290f9ead18cSJens Axboe 291f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 292f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 29343e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 294f9ead18cSJens Axboe 295f9ead18cSJens Axboe flags = sr->msg_flags; 296f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 297f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 298f9ead18cSJens Axboe if (flags & MSG_WAITALL) 299f9ead18cSJens Axboe min_ret = iov_iter_count(&kmsg->msg.msg_iter); 300f9ead18cSJens Axboe 301f9ead18cSJens Axboe ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 302f9ead18cSJens Axboe 303f9ead18cSJens Axboe if (ret < min_ret) { 304f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 30543e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 306f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 307f9ead18cSJens Axboe sr->done_io += ret; 308f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 30943e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 310f9ead18cSJens Axboe } 31195eafc74SPavel Begunkov if (ret == -ERESTARTSYS) 31295eafc74SPavel Begunkov ret = -EINTR; 313f9ead18cSJens Axboe req_set_fail(req); 314f9ead18cSJens Axboe } 315f9ead18cSJens Axboe /* fast path, check for non-NULL to avoid function call */ 316f9ead18cSJens Axboe if (kmsg->free_iov) 317f9ead18cSJens Axboe kfree(kmsg->free_iov); 318f9ead18cSJens Axboe req->flags &= ~REQ_F_NEED_CLEANUP; 31943e0bbbdSJens Axboe io_netmsg_recycle(req, issue_flags); 320f9ead18cSJens Axboe if (ret >= 0) 321f9ead18cSJens Axboe ret += sr->done_io; 322f9ead18cSJens Axboe else if (sr->done_io) 323f9ead18cSJens Axboe ret = sr->done_io; 324f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 325f9ead18cSJens Axboe return IOU_OK; 326f9ead18cSJens Axboe } 327f9ead18cSJens Axboe 328f9ead18cSJens Axboe int io_send(struct io_kiocb *req, unsigned int issue_flags) 329f9ead18cSJens Axboe { 330516e82f0SPavel Begunkov struct sockaddr_storage __address; 331f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 332f9ead18cSJens Axboe struct msghdr msg; 333f9ead18cSJens Axboe struct iovec iov; 334f9ead18cSJens Axboe struct socket *sock; 335f9ead18cSJens Axboe unsigned flags; 336f9ead18cSJens Axboe int min_ret = 0; 337f9ead18cSJens Axboe int ret; 338f9ead18cSJens Axboe 33904360d3eSPavel Begunkov msg.msg_name = NULL; 34004360d3eSPavel Begunkov msg.msg_control = NULL; 34104360d3eSPavel Begunkov msg.msg_controllen = 0; 34204360d3eSPavel Begunkov msg.msg_namelen = 0; 34304360d3eSPavel Begunkov msg.msg_ubuf = NULL; 34404360d3eSPavel Begunkov 345516e82f0SPavel Begunkov if (sr->addr) { 346516e82f0SPavel Begunkov if (req_has_async_data(req)) { 347516e82f0SPavel Begunkov struct io_async_msghdr *io = req->async_data; 348516e82f0SPavel Begunkov 349516e82f0SPavel Begunkov msg.msg_name = &io->addr; 350516e82f0SPavel Begunkov } else { 351516e82f0SPavel Begunkov ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address); 352516e82f0SPavel Begunkov if (unlikely(ret < 0)) 353516e82f0SPavel Begunkov return ret; 354516e82f0SPavel Begunkov msg.msg_name = (struct sockaddr *)&__address; 355516e82f0SPavel Begunkov } 356516e82f0SPavel Begunkov msg.msg_namelen = sr->addr_len; 357516e82f0SPavel Begunkov } 358516e82f0SPavel Begunkov 359f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 360f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 361516e82f0SPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 362f9ead18cSJens Axboe 363f9ead18cSJens Axboe sock = sock_from_file(req->file); 364f9ead18cSJens Axboe if (unlikely(!sock)) 365f9ead18cSJens Axboe return -ENOTSOCK; 366f9ead18cSJens Axboe 367f9ead18cSJens Axboe ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); 368f9ead18cSJens Axboe if (unlikely(ret)) 369f9ead18cSJens Axboe return ret; 370f9ead18cSJens Axboe 371f9ead18cSJens Axboe flags = sr->msg_flags; 372f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 373f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 374f9ead18cSJens Axboe if (flags & MSG_WAITALL) 375f9ead18cSJens Axboe min_ret = iov_iter_count(&msg.msg_iter); 376f9ead18cSJens Axboe 377f9ead18cSJens Axboe msg.msg_flags = flags; 378f9ead18cSJens Axboe ret = sock_sendmsg(sock, &msg); 379f9ead18cSJens Axboe if (ret < min_ret) { 380f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 381516e82f0SPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 382516e82f0SPavel Begunkov 383f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 384f9ead18cSJens Axboe sr->len -= ret; 385f9ead18cSJens Axboe sr->buf += ret; 386f9ead18cSJens Axboe sr->done_io += ret; 387f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 388516e82f0SPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 389f9ead18cSJens Axboe } 39095eafc74SPavel Begunkov if (ret == -ERESTARTSYS) 39195eafc74SPavel Begunkov ret = -EINTR; 392f9ead18cSJens Axboe req_set_fail(req); 393f9ead18cSJens Axboe } 394f9ead18cSJens Axboe if (ret >= 0) 395f9ead18cSJens Axboe ret += sr->done_io; 396f9ead18cSJens Axboe else if (sr->done_io) 397f9ead18cSJens Axboe ret = sr->done_io; 398f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 399f9ead18cSJens Axboe return IOU_OK; 400f9ead18cSJens Axboe } 401f9ead18cSJens Axboe 4029bb66906SDylan Yudaken static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) 4039bb66906SDylan Yudaken { 4049b0fc3c0SDylan Yudaken int hdr; 4059bb66906SDylan Yudaken 4069b0fc3c0SDylan Yudaken if (iomsg->namelen < 0) 4079bb66906SDylan Yudaken return true; 4089b0fc3c0SDylan Yudaken if (check_add_overflow((int)sizeof(struct io_uring_recvmsg_out), 4099b0fc3c0SDylan Yudaken iomsg->namelen, &hdr)) 4109bb66906SDylan Yudaken return true; 4119b0fc3c0SDylan Yudaken if (check_add_overflow(hdr, (int)iomsg->controllen, &hdr)) 4129bb66906SDylan Yudaken return true; 4139bb66906SDylan Yudaken 4149bb66906SDylan Yudaken return false; 4159bb66906SDylan Yudaken } 4169bb66906SDylan Yudaken 417f9ead18cSJens Axboe static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 418f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 419f9ead18cSJens Axboe { 420f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 4217fa875b8SDylan Yudaken struct user_msghdr msg; 422f9ead18cSJens Axboe int ret; 423f9ead18cSJens Axboe 4247fa875b8SDylan Yudaken if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg))) 4257fa875b8SDylan Yudaken return -EFAULT; 4267fa875b8SDylan Yudaken 4277fa875b8SDylan Yudaken ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 428f9ead18cSJens Axboe if (ret) 429f9ead18cSJens Axboe return ret; 430f9ead18cSJens Axboe 431f9ead18cSJens Axboe if (req->flags & REQ_F_BUFFER_SELECT) { 4327fa875b8SDylan Yudaken if (msg.msg_iovlen == 0) { 4335702196eSDylan Yudaken sr->len = iomsg->fast_iov[0].iov_len = 0; 4345702196eSDylan Yudaken iomsg->fast_iov[0].iov_base = NULL; 4355702196eSDylan Yudaken iomsg->free_iov = NULL; 4367fa875b8SDylan Yudaken } else if (msg.msg_iovlen > 1) { 437f9ead18cSJens Axboe return -EINVAL; 4385702196eSDylan Yudaken } else { 4397fa875b8SDylan Yudaken if (copy_from_user(iomsg->fast_iov, msg.msg_iov, sizeof(*msg.msg_iov))) 440f9ead18cSJens Axboe return -EFAULT; 441f9ead18cSJens Axboe sr->len = iomsg->fast_iov[0].iov_len; 442f9ead18cSJens Axboe iomsg->free_iov = NULL; 4435702196eSDylan Yudaken } 4449bb66906SDylan Yudaken 4459bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 4469bb66906SDylan Yudaken iomsg->namelen = msg.msg_namelen; 4479bb66906SDylan Yudaken iomsg->controllen = msg.msg_controllen; 4489bb66906SDylan Yudaken if (io_recvmsg_multishot_overflow(iomsg)) 4499bb66906SDylan Yudaken return -EOVERFLOW; 4509bb66906SDylan Yudaken } 451f9ead18cSJens Axboe } else { 452f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 4537fa875b8SDylan Yudaken ret = __import_iovec(READ, msg.msg_iov, msg.msg_iovlen, UIO_FASTIOV, 454f9ead18cSJens Axboe &iomsg->free_iov, &iomsg->msg.msg_iter, 455f9ead18cSJens Axboe false); 456f9ead18cSJens Axboe if (ret > 0) 457f9ead18cSJens Axboe ret = 0; 458f9ead18cSJens Axboe } 459f9ead18cSJens Axboe 460f9ead18cSJens Axboe return ret; 461f9ead18cSJens Axboe } 462f9ead18cSJens Axboe 463f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 464f9ead18cSJens Axboe static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 465f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 466f9ead18cSJens Axboe { 467f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 46872c531f8SDylan Yudaken struct compat_msghdr msg; 469f9ead18cSJens Axboe struct compat_iovec __user *uiov; 470f9ead18cSJens Axboe int ret; 471f9ead18cSJens Axboe 47272c531f8SDylan Yudaken if (copy_from_user(&msg, sr->umsg_compat, sizeof(msg))) 47372c531f8SDylan Yudaken return -EFAULT; 47472c531f8SDylan Yudaken 4754f6a94d3SJens Axboe ret = __get_compat_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 476f9ead18cSJens Axboe if (ret) 477f9ead18cSJens Axboe return ret; 478f9ead18cSJens Axboe 47972c531f8SDylan Yudaken uiov = compat_ptr(msg.msg_iov); 480f9ead18cSJens Axboe if (req->flags & REQ_F_BUFFER_SELECT) { 481f9ead18cSJens Axboe compat_ssize_t clen; 482f9ead18cSJens Axboe 48372c531f8SDylan Yudaken if (msg.msg_iovlen == 0) { 4846d2f75a0SDylan Yudaken sr->len = 0; 48572c531f8SDylan Yudaken } else if (msg.msg_iovlen > 1) { 486f9ead18cSJens Axboe return -EINVAL; 4876d2f75a0SDylan Yudaken } else { 488f9ead18cSJens Axboe if (!access_ok(uiov, sizeof(*uiov))) 489f9ead18cSJens Axboe return -EFAULT; 490f9ead18cSJens Axboe if (__get_user(clen, &uiov->iov_len)) 491f9ead18cSJens Axboe return -EFAULT; 492f9ead18cSJens Axboe if (clen < 0) 493f9ead18cSJens Axboe return -EINVAL; 494f9ead18cSJens Axboe sr->len = clen; 4956d2f75a0SDylan Yudaken } 4969bb66906SDylan Yudaken 4979bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 4989bb66906SDylan Yudaken iomsg->namelen = msg.msg_namelen; 4999bb66906SDylan Yudaken iomsg->controllen = msg.msg_controllen; 5009bb66906SDylan Yudaken if (io_recvmsg_multishot_overflow(iomsg)) 5019bb66906SDylan Yudaken return -EOVERFLOW; 5029bb66906SDylan Yudaken } 503f9ead18cSJens Axboe } else { 504f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 50572c531f8SDylan Yudaken ret = __import_iovec(READ, (struct iovec __user *)uiov, msg.msg_iovlen, 506f9ead18cSJens Axboe UIO_FASTIOV, &iomsg->free_iov, 507f9ead18cSJens Axboe &iomsg->msg.msg_iter, true); 508f9ead18cSJens Axboe if (ret < 0) 509f9ead18cSJens Axboe return ret; 510f9ead18cSJens Axboe } 511f9ead18cSJens Axboe 512f9ead18cSJens Axboe return 0; 513f9ead18cSJens Axboe } 514f9ead18cSJens Axboe #endif 515f9ead18cSJens Axboe 516f9ead18cSJens Axboe static int io_recvmsg_copy_hdr(struct io_kiocb *req, 517f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 518f9ead18cSJens Axboe { 519f9ead18cSJens Axboe iomsg->msg.msg_name = &iomsg->addr; 520f9ead18cSJens Axboe 521f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 522f9ead18cSJens Axboe if (req->ctx->compat) 523f9ead18cSJens Axboe return __io_compat_recvmsg_copy_hdr(req, iomsg); 524f9ead18cSJens Axboe #endif 525f9ead18cSJens Axboe 526f9ead18cSJens Axboe return __io_recvmsg_copy_hdr(req, iomsg); 527f9ead18cSJens Axboe } 528f9ead18cSJens Axboe 529f9ead18cSJens Axboe int io_recvmsg_prep_async(struct io_kiocb *req) 530f9ead18cSJens Axboe { 531f9ead18cSJens Axboe int ret; 532f9ead18cSJens Axboe 533858c293eSPavel Begunkov if (!io_msg_alloc_async_prep(req)) 534858c293eSPavel Begunkov return -ENOMEM; 535f9ead18cSJens Axboe ret = io_recvmsg_copy_hdr(req, req->async_data); 536f9ead18cSJens Axboe if (!ret) 537f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 538f9ead18cSJens Axboe return ret; 539f9ead18cSJens Axboe } 540f9ead18cSJens Axboe 541b3fdea6eSDylan Yudaken #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT) 542b3fdea6eSDylan Yudaken 543f9ead18cSJens Axboe int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 544f9ead18cSJens Axboe { 545f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 546f9ead18cSJens Axboe 547f9ead18cSJens Axboe if (unlikely(sqe->file_index || sqe->addr2)) 548f9ead18cSJens Axboe return -EINVAL; 549f9ead18cSJens Axboe 550f9ead18cSJens Axboe sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 551f9ead18cSJens Axboe sr->len = READ_ONCE(sqe->len); 552f9ead18cSJens Axboe sr->flags = READ_ONCE(sqe->ioprio); 553b3fdea6eSDylan Yudaken if (sr->flags & ~(RECVMSG_FLAGS)) 554f9ead18cSJens Axboe return -EINVAL; 555f9ead18cSJens Axboe sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 556f9ead18cSJens Axboe if (sr->msg_flags & MSG_DONTWAIT) 557f9ead18cSJens Axboe req->flags |= REQ_F_NOWAIT; 558f9ead18cSJens Axboe if (sr->msg_flags & MSG_ERRQUEUE) 559f9ead18cSJens Axboe req->flags |= REQ_F_CLEAR_POLLIN; 560b3fdea6eSDylan Yudaken if (sr->flags & IORING_RECV_MULTISHOT) { 561b3fdea6eSDylan Yudaken if (!(req->flags & REQ_F_BUFFER_SELECT)) 562b3fdea6eSDylan Yudaken return -EINVAL; 563b3fdea6eSDylan Yudaken if (sr->msg_flags & MSG_WAITALL) 564b3fdea6eSDylan Yudaken return -EINVAL; 565b3fdea6eSDylan Yudaken if (req->opcode == IORING_OP_RECV && sr->len) 566b3fdea6eSDylan Yudaken return -EINVAL; 567b3fdea6eSDylan Yudaken req->flags |= REQ_F_APOLL_MULTISHOT; 568b3fdea6eSDylan Yudaken } 569f9ead18cSJens Axboe 570f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 571f9ead18cSJens Axboe if (req->ctx->compat) 572f9ead18cSJens Axboe sr->msg_flags |= MSG_CMSG_COMPAT; 573f9ead18cSJens Axboe #endif 574f9ead18cSJens Axboe sr->done_io = 0; 575f9ead18cSJens Axboe return 0; 576f9ead18cSJens Axboe } 577f9ead18cSJens Axboe 578b3fdea6eSDylan Yudaken static inline void io_recv_prep_retry(struct io_kiocb *req) 579b3fdea6eSDylan Yudaken { 580f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 581b3fdea6eSDylan Yudaken 582b3fdea6eSDylan Yudaken sr->done_io = 0; 583b3fdea6eSDylan Yudaken sr->len = 0; /* get from the provided buffer */ 584b3fdea6eSDylan Yudaken } 585b3fdea6eSDylan Yudaken 586b3fdea6eSDylan Yudaken /* 5879bb66906SDylan Yudaken * Finishes io_recv and io_recvmsg. 588b3fdea6eSDylan Yudaken * 589b3fdea6eSDylan Yudaken * Returns true if it is actually finished, or false if it should run 590b3fdea6eSDylan Yudaken * again (for multishot). 591b3fdea6eSDylan Yudaken */ 5929bb66906SDylan Yudaken static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 5939bb66906SDylan Yudaken unsigned int cflags, bool mshot_finished) 594b3fdea6eSDylan Yudaken { 595b3fdea6eSDylan Yudaken if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 596b3fdea6eSDylan Yudaken io_req_set_res(req, *ret, cflags); 597b3fdea6eSDylan Yudaken *ret = IOU_OK; 598b3fdea6eSDylan Yudaken return true; 599b3fdea6eSDylan Yudaken } 600b3fdea6eSDylan Yudaken 6019bb66906SDylan Yudaken if (!mshot_finished) { 602b3fdea6eSDylan Yudaken if (io_post_aux_cqe(req->ctx, req->cqe.user_data, *ret, 603b3fdea6eSDylan Yudaken cflags | IORING_CQE_F_MORE, false)) { 604b3fdea6eSDylan Yudaken io_recv_prep_retry(req); 605b3fdea6eSDylan Yudaken return false; 606b3fdea6eSDylan Yudaken } 607b3fdea6eSDylan Yudaken /* 608b3fdea6eSDylan Yudaken * Otherwise stop multishot but use the current result. 609b3fdea6eSDylan Yudaken * Probably will end up going into overflow, but this means 610b3fdea6eSDylan Yudaken * we cannot trust the ordering anymore 611b3fdea6eSDylan Yudaken */ 612b3fdea6eSDylan Yudaken } 613b3fdea6eSDylan Yudaken 614b3fdea6eSDylan Yudaken io_req_set_res(req, *ret, cflags); 615b3fdea6eSDylan Yudaken 616b3fdea6eSDylan Yudaken if (req->flags & REQ_F_POLLED) 617b3fdea6eSDylan Yudaken *ret = IOU_STOP_MULTISHOT; 618e2df2ccbSDylan Yudaken else 619e2df2ccbSDylan Yudaken *ret = IOU_OK; 620b3fdea6eSDylan Yudaken return true; 621b3fdea6eSDylan Yudaken } 622b3fdea6eSDylan Yudaken 6239bb66906SDylan Yudaken static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 6249bb66906SDylan Yudaken struct io_sr_msg *sr, void __user **buf, 6259bb66906SDylan Yudaken size_t *len) 6269bb66906SDylan Yudaken { 6279bb66906SDylan Yudaken unsigned long ubuf = (unsigned long) *buf; 6289bb66906SDylan Yudaken unsigned long hdr; 6299bb66906SDylan Yudaken 6309bb66906SDylan Yudaken hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 6319bb66906SDylan Yudaken kmsg->controllen; 6329bb66906SDylan Yudaken if (*len < hdr) 6339bb66906SDylan Yudaken return -EFAULT; 6349bb66906SDylan Yudaken 6359bb66906SDylan Yudaken if (kmsg->controllen) { 6369bb66906SDylan Yudaken unsigned long control = ubuf + hdr - kmsg->controllen; 6379bb66906SDylan Yudaken 638d1f6222cSDylan Yudaken kmsg->msg.msg_control_user = (void __user *) control; 6399bb66906SDylan Yudaken kmsg->msg.msg_controllen = kmsg->controllen; 6409bb66906SDylan Yudaken } 6419bb66906SDylan Yudaken 6429bb66906SDylan Yudaken sr->buf = *buf; /* stash for later copy */ 643d1f6222cSDylan Yudaken *buf = (void __user *) (ubuf + hdr); 6449bb66906SDylan Yudaken kmsg->payloadlen = *len = *len - hdr; 6459bb66906SDylan Yudaken return 0; 6469bb66906SDylan Yudaken } 6479bb66906SDylan Yudaken 6489bb66906SDylan Yudaken struct io_recvmsg_multishot_hdr { 6499bb66906SDylan Yudaken struct io_uring_recvmsg_out msg; 6509bb66906SDylan Yudaken struct sockaddr_storage addr; 6519bb66906SDylan Yudaken }; 6529bb66906SDylan Yudaken 6539bb66906SDylan Yudaken static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 6549bb66906SDylan Yudaken struct io_async_msghdr *kmsg, 6559bb66906SDylan Yudaken unsigned int flags, bool *finished) 6569bb66906SDylan Yudaken { 6579bb66906SDylan Yudaken int err; 6589bb66906SDylan Yudaken int copy_len; 6599bb66906SDylan Yudaken struct io_recvmsg_multishot_hdr hdr; 6609bb66906SDylan Yudaken 6619bb66906SDylan Yudaken if (kmsg->namelen) 6629bb66906SDylan Yudaken kmsg->msg.msg_name = &hdr.addr; 6639bb66906SDylan Yudaken kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 6649bb66906SDylan Yudaken kmsg->msg.msg_namelen = 0; 6659bb66906SDylan Yudaken 6669bb66906SDylan Yudaken if (sock->file->f_flags & O_NONBLOCK) 6679bb66906SDylan Yudaken flags |= MSG_DONTWAIT; 6689bb66906SDylan Yudaken 6699bb66906SDylan Yudaken err = sock_recvmsg(sock, &kmsg->msg, flags); 6709bb66906SDylan Yudaken *finished = err <= 0; 6719bb66906SDylan Yudaken if (err < 0) 6729bb66906SDylan Yudaken return err; 6739bb66906SDylan Yudaken 6749bb66906SDylan Yudaken hdr.msg = (struct io_uring_recvmsg_out) { 6759bb66906SDylan Yudaken .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 6769bb66906SDylan Yudaken .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 6779bb66906SDylan Yudaken }; 6789bb66906SDylan Yudaken 6799bb66906SDylan Yudaken hdr.msg.payloadlen = err; 6809bb66906SDylan Yudaken if (err > kmsg->payloadlen) 6819bb66906SDylan Yudaken err = kmsg->payloadlen; 6829bb66906SDylan Yudaken 6839bb66906SDylan Yudaken copy_len = sizeof(struct io_uring_recvmsg_out); 6849bb66906SDylan Yudaken if (kmsg->msg.msg_namelen > kmsg->namelen) 6859bb66906SDylan Yudaken copy_len += kmsg->namelen; 6869bb66906SDylan Yudaken else 6879bb66906SDylan Yudaken copy_len += kmsg->msg.msg_namelen; 6889bb66906SDylan Yudaken 6899bb66906SDylan Yudaken /* 6909bb66906SDylan Yudaken * "fromlen shall refer to the value before truncation.." 6919bb66906SDylan Yudaken * 1003.1g 6929bb66906SDylan Yudaken */ 6939bb66906SDylan Yudaken hdr.msg.namelen = kmsg->msg.msg_namelen; 6949bb66906SDylan Yudaken 6959bb66906SDylan Yudaken /* ensure that there is no gap between hdr and sockaddr_storage */ 6969bb66906SDylan Yudaken BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 6979bb66906SDylan Yudaken sizeof(struct io_uring_recvmsg_out)); 6989bb66906SDylan Yudaken if (copy_to_user(io->buf, &hdr, copy_len)) { 6999bb66906SDylan Yudaken *finished = true; 7009bb66906SDylan Yudaken return -EFAULT; 7019bb66906SDylan Yudaken } 7029bb66906SDylan Yudaken 7039bb66906SDylan Yudaken return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 7049bb66906SDylan Yudaken kmsg->controllen + err; 7059bb66906SDylan Yudaken } 7069bb66906SDylan Yudaken 707f9ead18cSJens Axboe int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 708f9ead18cSJens Axboe { 709f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 710f9ead18cSJens Axboe struct io_async_msghdr iomsg, *kmsg; 711f9ead18cSJens Axboe struct socket *sock; 712f9ead18cSJens Axboe unsigned int cflags; 713f9ead18cSJens Axboe unsigned flags; 714f9ead18cSJens Axboe int ret, min_ret = 0; 715f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 7169bb66906SDylan Yudaken bool mshot_finished = true; 717f9ead18cSJens Axboe 718f9ead18cSJens Axboe sock = sock_from_file(req->file); 719f9ead18cSJens Axboe if (unlikely(!sock)) 720f9ead18cSJens Axboe return -ENOTSOCK; 721f9ead18cSJens Axboe 722f9ead18cSJens Axboe if (req_has_async_data(req)) { 723f9ead18cSJens Axboe kmsg = req->async_data; 724f9ead18cSJens Axboe } else { 725f9ead18cSJens Axboe ret = io_recvmsg_copy_hdr(req, &iomsg); 726f9ead18cSJens Axboe if (ret) 727f9ead18cSJens Axboe return ret; 728f9ead18cSJens Axboe kmsg = &iomsg; 729f9ead18cSJens Axboe } 730f9ead18cSJens Axboe 731f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 732f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 73343e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 734f9ead18cSJens Axboe 7359bb66906SDylan Yudaken retry_multishot: 736f9ead18cSJens Axboe if (io_do_buffer_select(req)) { 737f9ead18cSJens Axboe void __user *buf; 7389bb66906SDylan Yudaken size_t len = sr->len; 739f9ead18cSJens Axboe 7409bb66906SDylan Yudaken buf = io_buffer_select(req, &len, issue_flags); 741f9ead18cSJens Axboe if (!buf) 742f9ead18cSJens Axboe return -ENOBUFS; 7439bb66906SDylan Yudaken 7449bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 7459bb66906SDylan Yudaken ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); 7469bb66906SDylan Yudaken if (ret) { 7479bb66906SDylan Yudaken io_kbuf_recycle(req, issue_flags); 7489bb66906SDylan Yudaken return ret; 7499bb66906SDylan Yudaken } 7509bb66906SDylan Yudaken } 7519bb66906SDylan Yudaken 752f9ead18cSJens Axboe kmsg->fast_iov[0].iov_base = buf; 7539bb66906SDylan Yudaken kmsg->fast_iov[0].iov_len = len; 754f9ead18cSJens Axboe iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, 7559bb66906SDylan Yudaken len); 756f9ead18cSJens Axboe } 757f9ead18cSJens Axboe 758f9ead18cSJens Axboe flags = sr->msg_flags; 759f9ead18cSJens Axboe if (force_nonblock) 760f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 761f9ead18cSJens Axboe if (flags & MSG_WAITALL) 762f9ead18cSJens Axboe min_ret = iov_iter_count(&kmsg->msg.msg_iter); 763f9ead18cSJens Axboe 764f9ead18cSJens Axboe kmsg->msg.msg_get_inq = 1; 7659bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) 7669bb66906SDylan Yudaken ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 7679bb66906SDylan Yudaken &mshot_finished); 7689bb66906SDylan Yudaken else 7699bb66906SDylan Yudaken ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 7709bb66906SDylan Yudaken kmsg->uaddr, flags); 7719bb66906SDylan Yudaken 772f9ead18cSJens Axboe if (ret < min_ret) { 7739bb66906SDylan Yudaken if (ret == -EAGAIN && force_nonblock) { 7749bb66906SDylan Yudaken ret = io_setup_async_msg(req, kmsg, issue_flags); 7759bb66906SDylan Yudaken if (ret == -EAGAIN && (req->flags & IO_APOLL_MULTI_POLLED) == 7769bb66906SDylan Yudaken IO_APOLL_MULTI_POLLED) { 7779bb66906SDylan Yudaken io_kbuf_recycle(req, issue_flags); 7789bb66906SDylan Yudaken return IOU_ISSUE_SKIP_COMPLETE; 7799bb66906SDylan Yudaken } 7809bb66906SDylan Yudaken return ret; 7819bb66906SDylan Yudaken } 782f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 783f9ead18cSJens Axboe sr->done_io += ret; 784f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 78543e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 786f9ead18cSJens Axboe } 78795eafc74SPavel Begunkov if (ret == -ERESTARTSYS) 78895eafc74SPavel Begunkov ret = -EINTR; 789f9ead18cSJens Axboe req_set_fail(req); 790f9ead18cSJens Axboe } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 791f9ead18cSJens Axboe req_set_fail(req); 792f9ead18cSJens Axboe } 793f9ead18cSJens Axboe 794d4e097daSDylan Yudaken if (ret > 0) 795f9ead18cSJens Axboe ret += sr->done_io; 796f9ead18cSJens Axboe else if (sr->done_io) 797f9ead18cSJens Axboe ret = sr->done_io; 798d4e097daSDylan Yudaken else 799d4e097daSDylan Yudaken io_kbuf_recycle(req, issue_flags); 800d4e097daSDylan Yudaken 801f9ead18cSJens Axboe cflags = io_put_kbuf(req, issue_flags); 802f9ead18cSJens Axboe if (kmsg->msg.msg_inq) 803f9ead18cSJens Axboe cflags |= IORING_CQE_F_SOCK_NONEMPTY; 804b3fdea6eSDylan Yudaken 8059bb66906SDylan Yudaken if (!io_recv_finish(req, &ret, cflags, mshot_finished)) 8069bb66906SDylan Yudaken goto retry_multishot; 8079bb66906SDylan Yudaken 8089bb66906SDylan Yudaken if (mshot_finished) { 8099bb66906SDylan Yudaken io_netmsg_recycle(req, issue_flags); 8109bb66906SDylan Yudaken /* fast path, check for non-NULL to avoid function call */ 8119bb66906SDylan Yudaken if (kmsg->free_iov) 8129bb66906SDylan Yudaken kfree(kmsg->free_iov); 8139bb66906SDylan Yudaken req->flags &= ~REQ_F_NEED_CLEANUP; 8149bb66906SDylan Yudaken } 8159bb66906SDylan Yudaken 8169bb66906SDylan Yudaken return ret; 817f9ead18cSJens Axboe } 818f9ead18cSJens Axboe 819f9ead18cSJens Axboe int io_recv(struct io_kiocb *req, unsigned int issue_flags) 820f9ead18cSJens Axboe { 821f2ccb5aeSStefan Metzmacher struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 822f9ead18cSJens Axboe struct msghdr msg; 823f9ead18cSJens Axboe struct socket *sock; 824f9ead18cSJens Axboe struct iovec iov; 825f9ead18cSJens Axboe unsigned int cflags; 826f9ead18cSJens Axboe unsigned flags; 827f9ead18cSJens Axboe int ret, min_ret = 0; 828f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 829b3fdea6eSDylan Yudaken size_t len = sr->len; 830f9ead18cSJens Axboe 831f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 832f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 833f9ead18cSJens Axboe return -EAGAIN; 834f9ead18cSJens Axboe 835f9ead18cSJens Axboe sock = sock_from_file(req->file); 836f9ead18cSJens Axboe if (unlikely(!sock)) 837f9ead18cSJens Axboe return -ENOTSOCK; 838f9ead18cSJens Axboe 839b3fdea6eSDylan Yudaken retry_multishot: 840f9ead18cSJens Axboe if (io_do_buffer_select(req)) { 841f9ead18cSJens Axboe void __user *buf; 842f9ead18cSJens Axboe 843b3fdea6eSDylan Yudaken buf = io_buffer_select(req, &len, issue_flags); 844f9ead18cSJens Axboe if (!buf) 845f9ead18cSJens Axboe return -ENOBUFS; 846f9ead18cSJens Axboe sr->buf = buf; 847f9ead18cSJens Axboe } 848f9ead18cSJens Axboe 849b3fdea6eSDylan Yudaken ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter); 850f9ead18cSJens Axboe if (unlikely(ret)) 851f9ead18cSJens Axboe goto out_free; 852f9ead18cSJens Axboe 853f9ead18cSJens Axboe msg.msg_name = NULL; 854f9ead18cSJens Axboe msg.msg_namelen = 0; 855f9ead18cSJens Axboe msg.msg_control = NULL; 856f9ead18cSJens Axboe msg.msg_get_inq = 1; 857f9ead18cSJens Axboe msg.msg_flags = 0; 858f9ead18cSJens Axboe msg.msg_controllen = 0; 859f9ead18cSJens Axboe msg.msg_iocb = NULL; 860e02b6651SPavel Begunkov msg.msg_ubuf = NULL; 861f9ead18cSJens Axboe 862f9ead18cSJens Axboe flags = sr->msg_flags; 863f9ead18cSJens Axboe if (force_nonblock) 864f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 865f9ead18cSJens Axboe if (flags & MSG_WAITALL) 866f9ead18cSJens Axboe min_ret = iov_iter_count(&msg.msg_iter); 867f9ead18cSJens Axboe 868f9ead18cSJens Axboe ret = sock_recvmsg(sock, &msg, flags); 869f9ead18cSJens Axboe if (ret < min_ret) { 870b3fdea6eSDylan Yudaken if (ret == -EAGAIN && force_nonblock) { 871b3fdea6eSDylan Yudaken if ((req->flags & IO_APOLL_MULTI_POLLED) == IO_APOLL_MULTI_POLLED) { 872b3fdea6eSDylan Yudaken io_kbuf_recycle(req, issue_flags); 873b3fdea6eSDylan Yudaken return IOU_ISSUE_SKIP_COMPLETE; 874b3fdea6eSDylan Yudaken } 875b3fdea6eSDylan Yudaken 876f9ead18cSJens Axboe return -EAGAIN; 877b3fdea6eSDylan Yudaken } 878f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 879f9ead18cSJens Axboe sr->len -= ret; 880f9ead18cSJens Axboe sr->buf += ret; 881f9ead18cSJens Axboe sr->done_io += ret; 882f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 883f9ead18cSJens Axboe return -EAGAIN; 884f9ead18cSJens Axboe } 88595eafc74SPavel Begunkov if (ret == -ERESTARTSYS) 88695eafc74SPavel Begunkov ret = -EINTR; 887f9ead18cSJens Axboe req_set_fail(req); 888f9ead18cSJens Axboe } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 889f9ead18cSJens Axboe out_free: 890f9ead18cSJens Axboe req_set_fail(req); 891f9ead18cSJens Axboe } 892f9ead18cSJens Axboe 893d4e097daSDylan Yudaken if (ret > 0) 894f9ead18cSJens Axboe ret += sr->done_io; 895f9ead18cSJens Axboe else if (sr->done_io) 896f9ead18cSJens Axboe ret = sr->done_io; 897d4e097daSDylan Yudaken else 898d4e097daSDylan Yudaken io_kbuf_recycle(req, issue_flags); 899d4e097daSDylan Yudaken 900f9ead18cSJens Axboe cflags = io_put_kbuf(req, issue_flags); 901f9ead18cSJens Axboe if (msg.msg_inq) 902f9ead18cSJens Axboe cflags |= IORING_CQE_F_SOCK_NONEMPTY; 903b3fdea6eSDylan Yudaken 9049bb66906SDylan Yudaken if (!io_recv_finish(req, &ret, cflags, ret <= 0)) 905b3fdea6eSDylan Yudaken goto retry_multishot; 906b3fdea6eSDylan Yudaken 907b3fdea6eSDylan Yudaken return ret; 908f9ead18cSJens Axboe } 909f9ead18cSJens Axboe 910b0e9b551SPavel Begunkov void io_send_zc_cleanup(struct io_kiocb *req) 911b48c312bSPavel Begunkov { 912ac9e5784SPavel Begunkov struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 913493108d9SPavel Begunkov struct io_async_msghdr *io; 914b48c312bSPavel Begunkov 915493108d9SPavel Begunkov if (req_has_async_data(req)) { 916493108d9SPavel Begunkov io = req->async_data; 9174c17a496SPavel Begunkov /* might be ->fast_iov if *msg_copy_hdr failed */ 9184c17a496SPavel Begunkov if (io->free_iov != io->fast_iov) 919493108d9SPavel Begunkov kfree(io->free_iov); 920493108d9SPavel Begunkov } 921a75155faSPavel Begunkov if (zc->notif) { 922b48c312bSPavel Begunkov io_notif_flush(zc->notif); 923b48c312bSPavel Begunkov zc->notif = NULL; 924b48c312bSPavel Begunkov } 925a75155faSPavel Begunkov } 926b48c312bSPavel Begunkov 927b0e9b551SPavel Begunkov int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 92806a5464bSPavel Begunkov { 929ac9e5784SPavel Begunkov struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 93010c7d33eSPavel Begunkov struct io_ring_ctx *ctx = req->ctx; 931b48c312bSPavel Begunkov struct io_kiocb *notif; 93206a5464bSPavel Begunkov 933493108d9SPavel Begunkov if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))) 934b48c312bSPavel Begunkov return -EINVAL; 935b48c312bSPavel Begunkov /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 936b48c312bSPavel Begunkov if (req->flags & REQ_F_CQE_SKIP) 93706a5464bSPavel Begunkov return -EINVAL; 93806a5464bSPavel Begunkov 93906a5464bSPavel Begunkov zc->flags = READ_ONCE(sqe->ioprio); 94063809137SPavel Begunkov if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | 94157f33224SPavel Begunkov IORING_RECVSEND_FIXED_BUF)) 94206a5464bSPavel Begunkov return -EINVAL; 943b48c312bSPavel Begunkov notif = zc->notif = io_alloc_notif(ctx); 944b48c312bSPavel Begunkov if (!notif) 945b48c312bSPavel Begunkov return -ENOMEM; 946b48c312bSPavel Begunkov notif->cqe.user_data = req->cqe.user_data; 947b48c312bSPavel Begunkov notif->cqe.res = 0; 948b48c312bSPavel Begunkov notif->cqe.flags = IORING_CQE_F_NOTIF; 949b48c312bSPavel Begunkov req->flags |= REQ_F_NEED_CLEANUP; 950e3366e02SPavel Begunkov if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 951e3366e02SPavel Begunkov unsigned idx = READ_ONCE(sqe->buf_index); 952e3366e02SPavel Begunkov 953e3366e02SPavel Begunkov if (unlikely(idx >= ctx->nr_user_bufs)) 954e3366e02SPavel Begunkov return -EFAULT; 955e3366e02SPavel Begunkov idx = array_index_nospec(idx, ctx->nr_user_bufs); 956e3366e02SPavel Begunkov req->imu = READ_ONCE(ctx->user_bufs[idx]); 957e3366e02SPavel Begunkov io_req_set_rsrc_node(notif, ctx, 0); 958e3366e02SPavel Begunkov } 95906a5464bSPavel Begunkov 960493108d9SPavel Begunkov if (req->opcode == IORING_OP_SEND_ZC) { 961493108d9SPavel Begunkov if (READ_ONCE(sqe->__pad3[0])) 962493108d9SPavel Begunkov return -EINVAL; 963493108d9SPavel Begunkov zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 964493108d9SPavel Begunkov zc->addr_len = READ_ONCE(sqe->addr_len); 965493108d9SPavel Begunkov } else { 966493108d9SPavel Begunkov if (unlikely(sqe->addr2 || sqe->file_index)) 967493108d9SPavel Begunkov return -EINVAL; 968493108d9SPavel Begunkov if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) 969493108d9SPavel Begunkov return -EINVAL; 970493108d9SPavel Begunkov } 971493108d9SPavel Begunkov 97206a5464bSPavel Begunkov zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 97306a5464bSPavel Begunkov zc->len = READ_ONCE(sqe->len); 97406a5464bSPavel Begunkov zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 97506a5464bSPavel Begunkov if (zc->msg_flags & MSG_DONTWAIT) 97606a5464bSPavel Begunkov req->flags |= REQ_F_NOWAIT; 977092aeedbSPavel Begunkov 9784a933e62SPavel Begunkov zc->done_io = 0; 979092aeedbSPavel Begunkov 98006a5464bSPavel Begunkov #ifdef CONFIG_COMPAT 98106a5464bSPavel Begunkov if (req->ctx->compat) 98206a5464bSPavel Begunkov zc->msg_flags |= MSG_CMSG_COMPAT; 98306a5464bSPavel Begunkov #endif 98406a5464bSPavel Begunkov return 0; 98506a5464bSPavel Begunkov } 98606a5464bSPavel Begunkov 987cd9021e8SPavel Begunkov static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb, 988cd9021e8SPavel Begunkov struct iov_iter *from, size_t length) 989cd9021e8SPavel Begunkov { 990cd9021e8SPavel Begunkov skb_zcopy_downgrade_managed(skb); 991cd9021e8SPavel Begunkov return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 992cd9021e8SPavel Begunkov } 993cd9021e8SPavel Begunkov 9943ff1a0d3SPavel Begunkov static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, 9953ff1a0d3SPavel Begunkov struct iov_iter *from, size_t length) 9963ff1a0d3SPavel Begunkov { 9973ff1a0d3SPavel Begunkov struct skb_shared_info *shinfo = skb_shinfo(skb); 9983ff1a0d3SPavel Begunkov int frag = shinfo->nr_frags; 9993ff1a0d3SPavel Begunkov int ret = 0; 10003ff1a0d3SPavel Begunkov struct bvec_iter bi; 10013ff1a0d3SPavel Begunkov ssize_t copied = 0; 10023ff1a0d3SPavel Begunkov unsigned long truesize = 0; 10033ff1a0d3SPavel Begunkov 1004cd9021e8SPavel Begunkov if (!frag) 10053ff1a0d3SPavel Begunkov shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1006cd9021e8SPavel Begunkov else if (unlikely(!skb_zcopy_managed(skb))) 10073ff1a0d3SPavel Begunkov return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 10083ff1a0d3SPavel Begunkov 10093ff1a0d3SPavel Begunkov bi.bi_size = min(from->count, length); 10103ff1a0d3SPavel Begunkov bi.bi_bvec_done = from->iov_offset; 10113ff1a0d3SPavel Begunkov bi.bi_idx = 0; 10123ff1a0d3SPavel Begunkov 10133ff1a0d3SPavel Begunkov while (bi.bi_size && frag < MAX_SKB_FRAGS) { 10143ff1a0d3SPavel Begunkov struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 10153ff1a0d3SPavel Begunkov 10163ff1a0d3SPavel Begunkov copied += v.bv_len; 10173ff1a0d3SPavel Begunkov truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 10183ff1a0d3SPavel Begunkov __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 10193ff1a0d3SPavel Begunkov v.bv_offset, v.bv_len); 10203ff1a0d3SPavel Begunkov bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 10213ff1a0d3SPavel Begunkov } 10223ff1a0d3SPavel Begunkov if (bi.bi_size) 10233ff1a0d3SPavel Begunkov ret = -EMSGSIZE; 10243ff1a0d3SPavel Begunkov 10253ff1a0d3SPavel Begunkov shinfo->nr_frags = frag; 10263ff1a0d3SPavel Begunkov from->bvec += bi.bi_idx; 10273ff1a0d3SPavel Begunkov from->nr_segs -= bi.bi_idx; 1028dfb58b17SPavel Begunkov from->count -= copied; 10293ff1a0d3SPavel Begunkov from->iov_offset = bi.bi_bvec_done; 10303ff1a0d3SPavel Begunkov 10313ff1a0d3SPavel Begunkov skb->data_len += copied; 10323ff1a0d3SPavel Begunkov skb->len += copied; 10333ff1a0d3SPavel Begunkov skb->truesize += truesize; 10343ff1a0d3SPavel Begunkov 10353ff1a0d3SPavel Begunkov if (sk && sk->sk_type == SOCK_STREAM) { 10363ff1a0d3SPavel Begunkov sk_wmem_queued_add(sk, truesize); 10373ff1a0d3SPavel Begunkov if (!skb_zcopy_pure(skb)) 10383ff1a0d3SPavel Begunkov sk_mem_charge(sk, truesize); 10393ff1a0d3SPavel Begunkov } else { 10403ff1a0d3SPavel Begunkov refcount_add(truesize, &skb->sk->sk_wmem_alloc); 10413ff1a0d3SPavel Begunkov } 10423ff1a0d3SPavel Begunkov return ret; 10433ff1a0d3SPavel Begunkov } 10443ff1a0d3SPavel Begunkov 1045b0e9b551SPavel Begunkov int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) 104606a5464bSPavel Begunkov { 10476ae61b7aSPavel Begunkov struct sockaddr_storage __address; 1048ac9e5784SPavel Begunkov struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 104906a5464bSPavel Begunkov struct msghdr msg; 105006a5464bSPavel Begunkov struct iovec iov; 105106a5464bSPavel Begunkov struct socket *sock; 10526ae91ac9SPavel Begunkov unsigned msg_flags; 105306a5464bSPavel Begunkov int ret, min_ret = 0; 105406a5464bSPavel Begunkov 105506a5464bSPavel Begunkov sock = sock_from_file(req->file); 105606a5464bSPavel Begunkov if (unlikely(!sock)) 105706a5464bSPavel Begunkov return -ENOTSOCK; 105806a5464bSPavel Begunkov 105906a5464bSPavel Begunkov msg.msg_name = NULL; 106006a5464bSPavel Begunkov msg.msg_control = NULL; 106106a5464bSPavel Begunkov msg.msg_controllen = 0; 106206a5464bSPavel Begunkov msg.msg_namelen = 0; 106306a5464bSPavel Begunkov 106486dc8f23SPavel Begunkov if (zc->addr) { 1065581711c4SPavel Begunkov if (req_has_async_data(req)) { 1066581711c4SPavel Begunkov struct io_async_msghdr *io = req->async_data; 1067581711c4SPavel Begunkov 10686ae61b7aSPavel Begunkov msg.msg_name = &io->addr; 1069581711c4SPavel Begunkov } else { 1070581711c4SPavel Begunkov ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); 107186dc8f23SPavel Begunkov if (unlikely(ret < 0)) 107286dc8f23SPavel Begunkov return ret; 1073581711c4SPavel Begunkov msg.msg_name = (struct sockaddr *)&__address; 1074581711c4SPavel Begunkov } 107586dc8f23SPavel Begunkov msg.msg_namelen = zc->addr_len; 107686dc8f23SPavel Begunkov } 107786dc8f23SPavel Begunkov 10783c840053SPavel Begunkov if (!(req->flags & REQ_F_POLLED) && 10793c840053SPavel Begunkov (zc->flags & IORING_RECVSEND_POLL_FIRST)) 10806ae61b7aSPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 10813c840053SPavel Begunkov 108210c7d33eSPavel Begunkov if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 108310c7d33eSPavel Begunkov ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, 108410c7d33eSPavel Begunkov (u64)(uintptr_t)zc->buf, zc->len); 108510c7d33eSPavel Begunkov if (unlikely(ret)) 108610c7d33eSPavel Begunkov return ret; 1087cd9021e8SPavel Begunkov msg.sg_from_iter = io_sg_from_iter; 108810c7d33eSPavel Begunkov } else { 108910c7d33eSPavel Begunkov ret = import_single_range(WRITE, zc->buf, zc->len, &iov, 109010c7d33eSPavel Begunkov &msg.msg_iter); 109106a5464bSPavel Begunkov if (unlikely(ret)) 109206a5464bSPavel Begunkov return ret; 1093b48c312bSPavel Begunkov ret = io_notif_account_mem(zc->notif, zc->len); 10942e32ba56SPavel Begunkov if (unlikely(ret)) 10952e32ba56SPavel Begunkov return ret; 1096cd9021e8SPavel Begunkov msg.sg_from_iter = io_sg_from_iter_iovec; 109710c7d33eSPavel Begunkov } 109806a5464bSPavel Begunkov 109906a5464bSPavel Begunkov msg_flags = zc->msg_flags | MSG_ZEROCOPY; 110006a5464bSPavel Begunkov if (issue_flags & IO_URING_F_NONBLOCK) 110106a5464bSPavel Begunkov msg_flags |= MSG_DONTWAIT; 110206a5464bSPavel Begunkov if (msg_flags & MSG_WAITALL) 110306a5464bSPavel Begunkov min_ret = iov_iter_count(&msg.msg_iter); 110406a5464bSPavel Begunkov 110506a5464bSPavel Begunkov msg.msg_flags = msg_flags; 1106b48c312bSPavel Begunkov msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; 110706a5464bSPavel Begunkov ret = sock_sendmsg(sock, &msg); 110806a5464bSPavel Begunkov 110906a5464bSPavel Begunkov if (unlikely(ret < min_ret)) { 111006a5464bSPavel Begunkov if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 11116ae61b7aSPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 1112581711c4SPavel Begunkov 11134a933e62SPavel Begunkov if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { 11144a933e62SPavel Begunkov zc->len -= ret; 11154a933e62SPavel Begunkov zc->buf += ret; 11164a933e62SPavel Begunkov zc->done_io += ret; 11174a933e62SPavel Begunkov req->flags |= REQ_F_PARTIAL_IO; 11186ae61b7aSPavel Begunkov return io_setup_async_addr(req, &__address, issue_flags); 11194a933e62SPavel Begunkov } 11204a933e62SPavel Begunkov if (ret == -ERESTARTSYS) 11214a933e62SPavel Begunkov ret = -EINTR; 11225a848b7cSPavel Begunkov req_set_fail(req); 112306a5464bSPavel Begunkov } 112406a5464bSPavel Begunkov 11254a933e62SPavel Begunkov if (ret >= 0) 11264a933e62SPavel Begunkov ret += zc->done_io; 11274a933e62SPavel Begunkov else if (zc->done_io) 11284a933e62SPavel Begunkov ret = zc->done_io; 1129b48c312bSPavel Begunkov 1130*108893ddSPavel Begunkov /* 1131*108893ddSPavel Begunkov * If we're in io-wq we can't rely on tw ordering guarantees, defer 1132*108893ddSPavel Begunkov * flushing notif to io_send_zc_cleanup() 1133*108893ddSPavel Begunkov */ 1134*108893ddSPavel Begunkov if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1135b48c312bSPavel Begunkov io_notif_flush(zc->notif); 1136b48c312bSPavel Begunkov req->flags &= ~REQ_F_NEED_CLEANUP; 1137*108893ddSPavel Begunkov } 11386ae91ac9SPavel Begunkov io_req_set_res(req, ret, IORING_CQE_F_MORE); 113906a5464bSPavel Begunkov return IOU_OK; 114006a5464bSPavel Begunkov } 114106a5464bSPavel Begunkov 1142493108d9SPavel Begunkov int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1143493108d9SPavel Begunkov { 1144493108d9SPavel Begunkov struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1145493108d9SPavel Begunkov struct io_async_msghdr iomsg, *kmsg; 1146493108d9SPavel Begunkov struct socket *sock; 11476ae91ac9SPavel Begunkov unsigned flags; 1148493108d9SPavel Begunkov int ret, min_ret = 0; 1149493108d9SPavel Begunkov 1150493108d9SPavel Begunkov sock = sock_from_file(req->file); 1151493108d9SPavel Begunkov if (unlikely(!sock)) 1152493108d9SPavel Begunkov return -ENOTSOCK; 1153493108d9SPavel Begunkov 1154493108d9SPavel Begunkov if (req_has_async_data(req)) { 1155493108d9SPavel Begunkov kmsg = req->async_data; 1156493108d9SPavel Begunkov } else { 1157493108d9SPavel Begunkov ret = io_sendmsg_copy_hdr(req, &iomsg); 1158493108d9SPavel Begunkov if (ret) 1159493108d9SPavel Begunkov return ret; 1160493108d9SPavel Begunkov kmsg = &iomsg; 1161493108d9SPavel Begunkov } 1162493108d9SPavel Begunkov 1163493108d9SPavel Begunkov if (!(req->flags & REQ_F_POLLED) && 1164493108d9SPavel Begunkov (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1165493108d9SPavel Begunkov return io_setup_async_msg(req, kmsg, issue_flags); 1166493108d9SPavel Begunkov 1167493108d9SPavel Begunkov flags = sr->msg_flags | MSG_ZEROCOPY; 1168493108d9SPavel Begunkov if (issue_flags & IO_URING_F_NONBLOCK) 1169493108d9SPavel Begunkov flags |= MSG_DONTWAIT; 1170493108d9SPavel Begunkov if (flags & MSG_WAITALL) 1171493108d9SPavel Begunkov min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1172493108d9SPavel Begunkov 1173493108d9SPavel Begunkov kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1174493108d9SPavel Begunkov kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1175493108d9SPavel Begunkov ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 1176493108d9SPavel Begunkov 1177493108d9SPavel Begunkov if (unlikely(ret < min_ret)) { 1178493108d9SPavel Begunkov if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1179493108d9SPavel Begunkov return io_setup_async_msg(req, kmsg, issue_flags); 1180493108d9SPavel Begunkov 1181493108d9SPavel Begunkov if (ret > 0 && io_net_retry(sock, flags)) { 1182493108d9SPavel Begunkov sr->done_io += ret; 1183493108d9SPavel Begunkov req->flags |= REQ_F_PARTIAL_IO; 1184493108d9SPavel Begunkov return io_setup_async_msg(req, kmsg, issue_flags); 1185493108d9SPavel Begunkov } 1186493108d9SPavel Begunkov if (ret == -ERESTARTSYS) 1187493108d9SPavel Begunkov ret = -EINTR; 1188493108d9SPavel Begunkov req_set_fail(req); 1189493108d9SPavel Begunkov } 1190493108d9SPavel Begunkov /* fast path, check for non-NULL to avoid function call */ 1191*108893ddSPavel Begunkov if (kmsg->free_iov) { 1192493108d9SPavel Begunkov kfree(kmsg->free_iov); 1193*108893ddSPavel Begunkov kmsg->free_iov = NULL; 1194*108893ddSPavel Begunkov } 1195493108d9SPavel Begunkov 1196493108d9SPavel Begunkov io_netmsg_recycle(req, issue_flags); 1197493108d9SPavel Begunkov if (ret >= 0) 1198493108d9SPavel Begunkov ret += sr->done_io; 1199493108d9SPavel Begunkov else if (sr->done_io) 1200493108d9SPavel Begunkov ret = sr->done_io; 1201493108d9SPavel Begunkov 1202*108893ddSPavel Begunkov /* 1203*108893ddSPavel Begunkov * If we're in io-wq we can't rely on tw ordering guarantees, defer 1204*108893ddSPavel Begunkov * flushing notif to io_send_zc_cleanup() 1205*108893ddSPavel Begunkov */ 1206*108893ddSPavel Begunkov if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1207493108d9SPavel Begunkov io_notif_flush(sr->notif); 1208493108d9SPavel Begunkov req->flags &= ~REQ_F_NEED_CLEANUP; 1209*108893ddSPavel Begunkov } 12106ae91ac9SPavel Begunkov io_req_set_res(req, ret, IORING_CQE_F_MORE); 1211493108d9SPavel Begunkov return IOU_OK; 1212493108d9SPavel Begunkov } 1213493108d9SPavel Begunkov 12147e6b638eSPavel Begunkov void io_sendrecv_fail(struct io_kiocb *req) 12157e6b638eSPavel Begunkov { 12167e6b638eSPavel Begunkov struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 12177e6b638eSPavel Begunkov 12187e6b638eSPavel Begunkov if (req->flags & REQ_F_PARTIAL_IO) 12196ae91ac9SPavel Begunkov req->cqe.res = sr->done_io; 12206ae91ac9SPavel Begunkov 1221c4c0009eSPavel Begunkov if ((req->flags & REQ_F_NEED_CLEANUP) && 12226ae91ac9SPavel Begunkov (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 12236ae91ac9SPavel Begunkov req->cqe.flags |= IORING_CQE_F_MORE; 12245693bcceSPavel Begunkov } 12255693bcceSPavel Begunkov 1226f9ead18cSJens Axboe int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1227f9ead18cSJens Axboe { 1228f2ccb5aeSStefan Metzmacher struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1229f9ead18cSJens Axboe unsigned flags; 1230f9ead18cSJens Axboe 1231f9ead18cSJens Axboe if (sqe->len || sqe->buf_index) 1232f9ead18cSJens Axboe return -EINVAL; 1233f9ead18cSJens Axboe 1234f9ead18cSJens Axboe accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1235f9ead18cSJens Axboe accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1236f9ead18cSJens Axboe accept->flags = READ_ONCE(sqe->accept_flags); 1237f9ead18cSJens Axboe accept->nofile = rlimit(RLIMIT_NOFILE); 1238f9ead18cSJens Axboe flags = READ_ONCE(sqe->ioprio); 1239f9ead18cSJens Axboe if (flags & ~IORING_ACCEPT_MULTISHOT) 1240f9ead18cSJens Axboe return -EINVAL; 1241f9ead18cSJens Axboe 1242f9ead18cSJens Axboe accept->file_slot = READ_ONCE(sqe->file_index); 1243f9ead18cSJens Axboe if (accept->file_slot) { 1244f9ead18cSJens Axboe if (accept->flags & SOCK_CLOEXEC) 1245f9ead18cSJens Axboe return -EINVAL; 1246f9ead18cSJens Axboe if (flags & IORING_ACCEPT_MULTISHOT && 1247f9ead18cSJens Axboe accept->file_slot != IORING_FILE_INDEX_ALLOC) 1248f9ead18cSJens Axboe return -EINVAL; 1249f9ead18cSJens Axboe } 1250f9ead18cSJens Axboe if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1251f9ead18cSJens Axboe return -EINVAL; 1252f9ead18cSJens Axboe if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1253f9ead18cSJens Axboe accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1254f9ead18cSJens Axboe if (flags & IORING_ACCEPT_MULTISHOT) 1255f9ead18cSJens Axboe req->flags |= REQ_F_APOLL_MULTISHOT; 1256f9ead18cSJens Axboe return 0; 1257f9ead18cSJens Axboe } 1258f9ead18cSJens Axboe 1259f9ead18cSJens Axboe int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1260f9ead18cSJens Axboe { 1261f9ead18cSJens Axboe struct io_ring_ctx *ctx = req->ctx; 1262f2ccb5aeSStefan Metzmacher struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1263f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1264f9ead18cSJens Axboe unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 1265f9ead18cSJens Axboe bool fixed = !!accept->file_slot; 1266f9ead18cSJens Axboe struct file *file; 1267f9ead18cSJens Axboe int ret, fd; 1268f9ead18cSJens Axboe 1269f9ead18cSJens Axboe retry: 1270f9ead18cSJens Axboe if (!fixed) { 1271f9ead18cSJens Axboe fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1272f9ead18cSJens Axboe if (unlikely(fd < 0)) 1273f9ead18cSJens Axboe return fd; 1274f9ead18cSJens Axboe } 1275f9ead18cSJens Axboe file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 1276f9ead18cSJens Axboe accept->flags); 1277f9ead18cSJens Axboe if (IS_ERR(file)) { 1278f9ead18cSJens Axboe if (!fixed) 1279f9ead18cSJens Axboe put_unused_fd(fd); 1280f9ead18cSJens Axboe ret = PTR_ERR(file); 1281f9ead18cSJens Axboe if (ret == -EAGAIN && force_nonblock) { 1282f9ead18cSJens Axboe /* 1283f9ead18cSJens Axboe * if it's multishot and polled, we don't need to 1284f9ead18cSJens Axboe * return EAGAIN to arm the poll infra since it 1285f9ead18cSJens Axboe * has already been done 1286f9ead18cSJens Axboe */ 1287f9ead18cSJens Axboe if ((req->flags & IO_APOLL_MULTI_POLLED) == 1288f9ead18cSJens Axboe IO_APOLL_MULTI_POLLED) 1289f9ead18cSJens Axboe ret = IOU_ISSUE_SKIP_COMPLETE; 1290f9ead18cSJens Axboe return ret; 1291f9ead18cSJens Axboe } 1292f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1293f9ead18cSJens Axboe ret = -EINTR; 1294f9ead18cSJens Axboe req_set_fail(req); 1295f9ead18cSJens Axboe } else if (!fixed) { 1296f9ead18cSJens Axboe fd_install(fd, file); 1297f9ead18cSJens Axboe ret = fd; 1298f9ead18cSJens Axboe } else { 1299f9ead18cSJens Axboe ret = io_fixed_fd_install(req, issue_flags, file, 1300f9ead18cSJens Axboe accept->file_slot); 1301f9ead18cSJens Axboe } 1302f9ead18cSJens Axboe 1303f9ead18cSJens Axboe if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 1304f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1305f9ead18cSJens Axboe return IOU_OK; 1306f9ead18cSJens Axboe } 1307f9ead18cSJens Axboe 1308cbd25748SDylan Yudaken if (ret >= 0 && 1309cbd25748SDylan Yudaken io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE, false)) 1310d245bca6SPavel Begunkov goto retry; 1311cbd25748SDylan Yudaken 1312cbd25748SDylan Yudaken io_req_set_res(req, ret, 0); 1313cbd25748SDylan Yudaken if (req->flags & REQ_F_POLLED) 1314cbd25748SDylan Yudaken return IOU_STOP_MULTISHOT; 1315cbd25748SDylan Yudaken return IOU_OK; 1316f9ead18cSJens Axboe } 1317f9ead18cSJens Axboe 1318f9ead18cSJens Axboe int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1319f9ead18cSJens Axboe { 1320f2ccb5aeSStefan Metzmacher struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1321f9ead18cSJens Axboe 1322f9ead18cSJens Axboe if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1323f9ead18cSJens Axboe return -EINVAL; 1324f9ead18cSJens Axboe 1325f9ead18cSJens Axboe sock->domain = READ_ONCE(sqe->fd); 1326f9ead18cSJens Axboe sock->type = READ_ONCE(sqe->off); 1327f9ead18cSJens Axboe sock->protocol = READ_ONCE(sqe->len); 1328f9ead18cSJens Axboe sock->file_slot = READ_ONCE(sqe->file_index); 1329f9ead18cSJens Axboe sock->nofile = rlimit(RLIMIT_NOFILE); 1330f9ead18cSJens Axboe 1331f9ead18cSJens Axboe sock->flags = sock->type & ~SOCK_TYPE_MASK; 1332f9ead18cSJens Axboe if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1333f9ead18cSJens Axboe return -EINVAL; 1334f9ead18cSJens Axboe if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1335f9ead18cSJens Axboe return -EINVAL; 1336f9ead18cSJens Axboe return 0; 1337f9ead18cSJens Axboe } 1338f9ead18cSJens Axboe 1339f9ead18cSJens Axboe int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1340f9ead18cSJens Axboe { 1341f2ccb5aeSStefan Metzmacher struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1342f9ead18cSJens Axboe bool fixed = !!sock->file_slot; 1343f9ead18cSJens Axboe struct file *file; 1344f9ead18cSJens Axboe int ret, fd; 1345f9ead18cSJens Axboe 1346f9ead18cSJens Axboe if (!fixed) { 1347f9ead18cSJens Axboe fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1348f9ead18cSJens Axboe if (unlikely(fd < 0)) 1349f9ead18cSJens Axboe return fd; 1350f9ead18cSJens Axboe } 1351f9ead18cSJens Axboe file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1352f9ead18cSJens Axboe if (IS_ERR(file)) { 1353f9ead18cSJens Axboe if (!fixed) 1354f9ead18cSJens Axboe put_unused_fd(fd); 1355f9ead18cSJens Axboe ret = PTR_ERR(file); 1356f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1357f9ead18cSJens Axboe return -EAGAIN; 1358f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1359f9ead18cSJens Axboe ret = -EINTR; 1360f9ead18cSJens Axboe req_set_fail(req); 1361f9ead18cSJens Axboe } else if (!fixed) { 1362f9ead18cSJens Axboe fd_install(fd, file); 1363f9ead18cSJens Axboe ret = fd; 1364f9ead18cSJens Axboe } else { 1365f9ead18cSJens Axboe ret = io_fixed_fd_install(req, issue_flags, file, 1366f9ead18cSJens Axboe sock->file_slot); 1367f9ead18cSJens Axboe } 1368f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1369f9ead18cSJens Axboe return IOU_OK; 1370f9ead18cSJens Axboe } 1371f9ead18cSJens Axboe 1372f9ead18cSJens Axboe int io_connect_prep_async(struct io_kiocb *req) 1373f9ead18cSJens Axboe { 1374f9ead18cSJens Axboe struct io_async_connect *io = req->async_data; 1375f2ccb5aeSStefan Metzmacher struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1376f9ead18cSJens Axboe 1377f9ead18cSJens Axboe return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 1378f9ead18cSJens Axboe } 1379f9ead18cSJens Axboe 1380f9ead18cSJens Axboe int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1381f9ead18cSJens Axboe { 1382f2ccb5aeSStefan Metzmacher struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1383f9ead18cSJens Axboe 1384f9ead18cSJens Axboe if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1385f9ead18cSJens Axboe return -EINVAL; 1386f9ead18cSJens Axboe 1387f9ead18cSJens Axboe conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1388f9ead18cSJens Axboe conn->addr_len = READ_ONCE(sqe->addr2); 1389f9ead18cSJens Axboe return 0; 1390f9ead18cSJens Axboe } 1391f9ead18cSJens Axboe 1392f9ead18cSJens Axboe int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1393f9ead18cSJens Axboe { 1394f2ccb5aeSStefan Metzmacher struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1395f9ead18cSJens Axboe struct io_async_connect __io, *io; 1396f9ead18cSJens Axboe unsigned file_flags; 1397f9ead18cSJens Axboe int ret; 1398f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1399f9ead18cSJens Axboe 1400f9ead18cSJens Axboe if (req_has_async_data(req)) { 1401f9ead18cSJens Axboe io = req->async_data; 1402f9ead18cSJens Axboe } else { 1403f9ead18cSJens Axboe ret = move_addr_to_kernel(connect->addr, 1404f9ead18cSJens Axboe connect->addr_len, 1405f9ead18cSJens Axboe &__io.address); 1406f9ead18cSJens Axboe if (ret) 1407f9ead18cSJens Axboe goto out; 1408f9ead18cSJens Axboe io = &__io; 1409f9ead18cSJens Axboe } 1410f9ead18cSJens Axboe 1411f9ead18cSJens Axboe file_flags = force_nonblock ? O_NONBLOCK : 0; 1412f9ead18cSJens Axboe 1413f9ead18cSJens Axboe ret = __sys_connect_file(req->file, &io->address, 1414f9ead18cSJens Axboe connect->addr_len, file_flags); 1415f9ead18cSJens Axboe if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { 1416f9ead18cSJens Axboe if (req_has_async_data(req)) 1417f9ead18cSJens Axboe return -EAGAIN; 1418f9ead18cSJens Axboe if (io_alloc_async_data(req)) { 1419f9ead18cSJens Axboe ret = -ENOMEM; 1420f9ead18cSJens Axboe goto out; 1421f9ead18cSJens Axboe } 1422f9ead18cSJens Axboe memcpy(req->async_data, &__io, sizeof(__io)); 1423f9ead18cSJens Axboe return -EAGAIN; 1424f9ead18cSJens Axboe } 1425f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1426f9ead18cSJens Axboe ret = -EINTR; 1427f9ead18cSJens Axboe out: 1428f9ead18cSJens Axboe if (ret < 0) 1429f9ead18cSJens Axboe req_set_fail(req); 1430f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1431f9ead18cSJens Axboe return IOU_OK; 1432f9ead18cSJens Axboe } 143343e0bbbdSJens Axboe 143443e0bbbdSJens Axboe void io_netmsg_cache_free(struct io_cache_entry *entry) 143543e0bbbdSJens Axboe { 143643e0bbbdSJens Axboe kfree(container_of(entry, struct io_async_msghdr, cache)); 143743e0bbbdSJens Axboe } 1438f9ead18cSJens Axboe #endif 1439