1f9ead18cSJens Axboe // SPDX-License-Identifier: GPL-2.0 2f9ead18cSJens Axboe #include <linux/kernel.h> 3f9ead18cSJens Axboe #include <linux/errno.h> 4f9ead18cSJens Axboe #include <linux/file.h> 5f9ead18cSJens Axboe #include <linux/slab.h> 6f9ead18cSJens Axboe #include <linux/net.h> 7f9ead18cSJens Axboe #include <linux/compat.h> 8f9ead18cSJens Axboe #include <net/compat.h> 9f9ead18cSJens Axboe #include <linux/io_uring.h> 10f9ead18cSJens Axboe 11f9ead18cSJens Axboe #include <uapi/linux/io_uring.h> 12f9ead18cSJens Axboe 13f9ead18cSJens Axboe #include "io_uring.h" 143b77495aSJens Axboe #include "kbuf.h" 1543e0bbbdSJens Axboe #include "alloc_cache.h" 16f9ead18cSJens Axboe #include "net.h" 1706a5464bSPavel Begunkov #include "notif.h" 1810c7d33eSPavel Begunkov #include "rsrc.h" 19f9ead18cSJens Axboe 20f9ead18cSJens Axboe #if defined(CONFIG_NET) 21f9ead18cSJens Axboe struct io_shutdown { 22f9ead18cSJens Axboe struct file *file; 23f9ead18cSJens Axboe int how; 24f9ead18cSJens Axboe }; 25f9ead18cSJens Axboe 26f9ead18cSJens Axboe struct io_accept { 27f9ead18cSJens Axboe struct file *file; 28f9ead18cSJens Axboe struct sockaddr __user *addr; 29f9ead18cSJens Axboe int __user *addr_len; 30f9ead18cSJens Axboe int flags; 31f9ead18cSJens Axboe u32 file_slot; 32f9ead18cSJens Axboe unsigned long nofile; 33f9ead18cSJens Axboe }; 34f9ead18cSJens Axboe 35f9ead18cSJens Axboe struct io_socket { 36f9ead18cSJens Axboe struct file *file; 37f9ead18cSJens Axboe int domain; 38f9ead18cSJens Axboe int type; 39f9ead18cSJens Axboe int protocol; 40f9ead18cSJens Axboe int flags; 41f9ead18cSJens Axboe u32 file_slot; 42f9ead18cSJens Axboe unsigned long nofile; 43f9ead18cSJens Axboe }; 44f9ead18cSJens Axboe 45f9ead18cSJens Axboe struct io_connect { 46f9ead18cSJens Axboe struct file *file; 47f9ead18cSJens Axboe struct sockaddr __user *addr; 48f9ead18cSJens Axboe int addr_len; 49f9ead18cSJens Axboe }; 50f9ead18cSJens Axboe 51f9ead18cSJens Axboe struct io_sr_msg { 52f9ead18cSJens Axboe struct file *file; 53f9ead18cSJens Axboe union { 54f9ead18cSJens Axboe struct compat_msghdr __user *umsg_compat; 55f9ead18cSJens Axboe struct user_msghdr __user *umsg; 56f9ead18cSJens Axboe void __user *buf; 57f9ead18cSJens Axboe }; 58f9ead18cSJens Axboe int msg_flags; 59f9ead18cSJens Axboe size_t len; 60f9ead18cSJens Axboe size_t done_io; 61f9ead18cSJens Axboe unsigned int flags; 62f9ead18cSJens Axboe }; 63f9ead18cSJens Axboe 6406a5464bSPavel Begunkov struct io_sendzc { 6506a5464bSPavel Begunkov struct file *file; 6606a5464bSPavel Begunkov void __user *buf; 6706a5464bSPavel Begunkov size_t len; 6806a5464bSPavel Begunkov u16 slot_idx; 6906a5464bSPavel Begunkov unsigned msg_flags; 7006a5464bSPavel Begunkov unsigned flags; 71092aeedbSPavel Begunkov unsigned addr_len; 72092aeedbSPavel Begunkov void __user *addr; 7306a5464bSPavel Begunkov }; 7406a5464bSPavel Begunkov 75f9ead18cSJens Axboe #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) 76f9ead18cSJens Axboe 77f9ead18cSJens Axboe int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 78f9ead18cSJens Axboe { 79f9ead18cSJens Axboe struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 80f9ead18cSJens Axboe 81f9ead18cSJens Axboe if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 82f9ead18cSJens Axboe sqe->buf_index || sqe->splice_fd_in)) 83f9ead18cSJens Axboe return -EINVAL; 84f9ead18cSJens Axboe 85f9ead18cSJens Axboe shutdown->how = READ_ONCE(sqe->len); 86f9ead18cSJens Axboe return 0; 87f9ead18cSJens Axboe } 88f9ead18cSJens Axboe 89f9ead18cSJens Axboe int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 90f9ead18cSJens Axboe { 91f9ead18cSJens Axboe struct io_shutdown *shutdown = io_kiocb_to_cmd(req); 92f9ead18cSJens Axboe struct socket *sock; 93f9ead18cSJens Axboe int ret; 94f9ead18cSJens Axboe 95f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 96f9ead18cSJens Axboe return -EAGAIN; 97f9ead18cSJens Axboe 98f9ead18cSJens Axboe sock = sock_from_file(req->file); 99f9ead18cSJens Axboe if (unlikely(!sock)) 100f9ead18cSJens Axboe return -ENOTSOCK; 101f9ead18cSJens Axboe 102f9ead18cSJens Axboe ret = __sys_shutdown_sock(sock, shutdown->how); 103f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 104f9ead18cSJens Axboe return IOU_OK; 105f9ead18cSJens Axboe } 106f9ead18cSJens Axboe 107f9ead18cSJens Axboe static bool io_net_retry(struct socket *sock, int flags) 108f9ead18cSJens Axboe { 109f9ead18cSJens Axboe if (!(flags & MSG_WAITALL)) 110f9ead18cSJens Axboe return false; 111f9ead18cSJens Axboe return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 112f9ead18cSJens Axboe } 113f9ead18cSJens Axboe 11443e0bbbdSJens Axboe static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 11543e0bbbdSJens Axboe { 11643e0bbbdSJens Axboe struct io_async_msghdr *hdr = req->async_data; 11743e0bbbdSJens Axboe 11843e0bbbdSJens Axboe if (!hdr || issue_flags & IO_URING_F_UNLOCKED) 11943e0bbbdSJens Axboe return; 12043e0bbbdSJens Axboe 12143e0bbbdSJens Axboe /* Let normal cleanup path reap it if we fail adding to the cache */ 12243e0bbbdSJens Axboe if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) { 12343e0bbbdSJens Axboe req->async_data = NULL; 12443e0bbbdSJens Axboe req->flags &= ~REQ_F_ASYNC_DATA; 12543e0bbbdSJens Axboe } 12643e0bbbdSJens Axboe } 12743e0bbbdSJens Axboe 12843e0bbbdSJens Axboe static struct io_async_msghdr *io_recvmsg_alloc_async(struct io_kiocb *req, 12943e0bbbdSJens Axboe unsigned int issue_flags) 13043e0bbbdSJens Axboe { 13143e0bbbdSJens Axboe struct io_ring_ctx *ctx = req->ctx; 13243e0bbbdSJens Axboe struct io_cache_entry *entry; 13343e0bbbdSJens Axboe 13443e0bbbdSJens Axboe if (!(issue_flags & IO_URING_F_UNLOCKED) && 13543e0bbbdSJens Axboe (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { 13643e0bbbdSJens Axboe struct io_async_msghdr *hdr; 13743e0bbbdSJens Axboe 13843e0bbbdSJens Axboe hdr = container_of(entry, struct io_async_msghdr, cache); 13943e0bbbdSJens Axboe req->flags |= REQ_F_ASYNC_DATA; 14043e0bbbdSJens Axboe req->async_data = hdr; 14143e0bbbdSJens Axboe return hdr; 14243e0bbbdSJens Axboe } 14343e0bbbdSJens Axboe 14443e0bbbdSJens Axboe if (!io_alloc_async_data(req)) 14543e0bbbdSJens Axboe return req->async_data; 14643e0bbbdSJens Axboe 14743e0bbbdSJens Axboe return NULL; 14843e0bbbdSJens Axboe } 14943e0bbbdSJens Axboe 150f9ead18cSJens Axboe static int io_setup_async_msg(struct io_kiocb *req, 15143e0bbbdSJens Axboe struct io_async_msghdr *kmsg, 15243e0bbbdSJens Axboe unsigned int issue_flags) 153f9ead18cSJens Axboe { 154f9ead18cSJens Axboe struct io_async_msghdr *async_msg = req->async_data; 155f9ead18cSJens Axboe 156f9ead18cSJens Axboe if (async_msg) 157f9ead18cSJens Axboe return -EAGAIN; 15843e0bbbdSJens Axboe async_msg = io_recvmsg_alloc_async(req, issue_flags); 15943e0bbbdSJens Axboe if (!async_msg) { 160f9ead18cSJens Axboe kfree(kmsg->free_iov); 161f9ead18cSJens Axboe return -ENOMEM; 162f9ead18cSJens Axboe } 163f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 164f9ead18cSJens Axboe memcpy(async_msg, kmsg, sizeof(*kmsg)); 165f9ead18cSJens Axboe async_msg->msg.msg_name = &async_msg->addr; 166f9ead18cSJens Axboe /* if were using fast_iov, set it to the new one */ 167f9ead18cSJens Axboe if (!async_msg->free_iov) 168f9ead18cSJens Axboe async_msg->msg.msg_iter.iov = async_msg->fast_iov; 169f9ead18cSJens Axboe 170f9ead18cSJens Axboe return -EAGAIN; 171f9ead18cSJens Axboe } 172f9ead18cSJens Axboe 173f9ead18cSJens Axboe static int io_sendmsg_copy_hdr(struct io_kiocb *req, 174f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 175f9ead18cSJens Axboe { 176f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 177f9ead18cSJens Axboe 178f9ead18cSJens Axboe iomsg->msg.msg_name = &iomsg->addr; 179f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 180f9ead18cSJens Axboe return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 181f9ead18cSJens Axboe &iomsg->free_iov); 182f9ead18cSJens Axboe } 183f9ead18cSJens Axboe 184f9ead18cSJens Axboe int io_sendmsg_prep_async(struct io_kiocb *req) 185f9ead18cSJens Axboe { 186f9ead18cSJens Axboe int ret; 187f9ead18cSJens Axboe 188f9ead18cSJens Axboe ret = io_sendmsg_copy_hdr(req, req->async_data); 189f9ead18cSJens Axboe if (!ret) 190f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 191f9ead18cSJens Axboe return ret; 192f9ead18cSJens Axboe } 193f9ead18cSJens Axboe 194f9ead18cSJens Axboe void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 195f9ead18cSJens Axboe { 196f9ead18cSJens Axboe struct io_async_msghdr *io = req->async_data; 197f9ead18cSJens Axboe 198f9ead18cSJens Axboe kfree(io->free_iov); 199f9ead18cSJens Axboe } 200f9ead18cSJens Axboe 201f9ead18cSJens Axboe int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 202f9ead18cSJens Axboe { 203f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 204f9ead18cSJens Axboe 205f9ead18cSJens Axboe if (unlikely(sqe->file_index || sqe->addr2)) 206f9ead18cSJens Axboe return -EINVAL; 207f9ead18cSJens Axboe 208f9ead18cSJens Axboe sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 209f9ead18cSJens Axboe sr->len = READ_ONCE(sqe->len); 210f9ead18cSJens Axboe sr->flags = READ_ONCE(sqe->ioprio); 211f9ead18cSJens Axboe if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 212f9ead18cSJens Axboe return -EINVAL; 213f9ead18cSJens Axboe sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 214f9ead18cSJens Axboe if (sr->msg_flags & MSG_DONTWAIT) 215f9ead18cSJens Axboe req->flags |= REQ_F_NOWAIT; 216f9ead18cSJens Axboe 217f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 218f9ead18cSJens Axboe if (req->ctx->compat) 219f9ead18cSJens Axboe sr->msg_flags |= MSG_CMSG_COMPAT; 220f9ead18cSJens Axboe #endif 221f9ead18cSJens Axboe sr->done_io = 0; 222f9ead18cSJens Axboe return 0; 223f9ead18cSJens Axboe } 224f9ead18cSJens Axboe 225f9ead18cSJens Axboe int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 226f9ead18cSJens Axboe { 227f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 228f9ead18cSJens Axboe struct io_async_msghdr iomsg, *kmsg; 229f9ead18cSJens Axboe struct socket *sock; 230f9ead18cSJens Axboe unsigned flags; 231f9ead18cSJens Axboe int min_ret = 0; 232f9ead18cSJens Axboe int ret; 233f9ead18cSJens Axboe 234f9ead18cSJens Axboe sock = sock_from_file(req->file); 235f9ead18cSJens Axboe if (unlikely(!sock)) 236f9ead18cSJens Axboe return -ENOTSOCK; 237f9ead18cSJens Axboe 238f9ead18cSJens Axboe if (req_has_async_data(req)) { 239f9ead18cSJens Axboe kmsg = req->async_data; 240f9ead18cSJens Axboe } else { 241f9ead18cSJens Axboe ret = io_sendmsg_copy_hdr(req, &iomsg); 242f9ead18cSJens Axboe if (ret) 243f9ead18cSJens Axboe return ret; 244f9ead18cSJens Axboe kmsg = &iomsg; 245f9ead18cSJens Axboe } 246f9ead18cSJens Axboe 247f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 248f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 24943e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 250f9ead18cSJens Axboe 251f9ead18cSJens Axboe flags = sr->msg_flags; 252f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 253f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 254f9ead18cSJens Axboe if (flags & MSG_WAITALL) 255f9ead18cSJens Axboe min_ret = iov_iter_count(&kmsg->msg.msg_iter); 256f9ead18cSJens Axboe 257f9ead18cSJens Axboe ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 258f9ead18cSJens Axboe 259f9ead18cSJens Axboe if (ret < min_ret) { 260f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 26143e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 262f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 263f9ead18cSJens Axboe ret = -EINTR; 264f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 265f9ead18cSJens Axboe sr->done_io += ret; 266f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 26743e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 268f9ead18cSJens Axboe } 269f9ead18cSJens Axboe req_set_fail(req); 270f9ead18cSJens Axboe } 271f9ead18cSJens Axboe /* fast path, check for non-NULL to avoid function call */ 272f9ead18cSJens Axboe if (kmsg->free_iov) 273f9ead18cSJens Axboe kfree(kmsg->free_iov); 274f9ead18cSJens Axboe req->flags &= ~REQ_F_NEED_CLEANUP; 27543e0bbbdSJens Axboe io_netmsg_recycle(req, issue_flags); 276f9ead18cSJens Axboe if (ret >= 0) 277f9ead18cSJens Axboe ret += sr->done_io; 278f9ead18cSJens Axboe else if (sr->done_io) 279f9ead18cSJens Axboe ret = sr->done_io; 280f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 281f9ead18cSJens Axboe return IOU_OK; 282f9ead18cSJens Axboe } 283f9ead18cSJens Axboe 284f9ead18cSJens Axboe int io_send(struct io_kiocb *req, unsigned int issue_flags) 285f9ead18cSJens Axboe { 286f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 287f9ead18cSJens Axboe struct msghdr msg; 288f9ead18cSJens Axboe struct iovec iov; 289f9ead18cSJens Axboe struct socket *sock; 290f9ead18cSJens Axboe unsigned flags; 291f9ead18cSJens Axboe int min_ret = 0; 292f9ead18cSJens Axboe int ret; 293f9ead18cSJens Axboe 294f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 295f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 296f9ead18cSJens Axboe return -EAGAIN; 297f9ead18cSJens Axboe 298f9ead18cSJens Axboe sock = sock_from_file(req->file); 299f9ead18cSJens Axboe if (unlikely(!sock)) 300f9ead18cSJens Axboe return -ENOTSOCK; 301f9ead18cSJens Axboe 302f9ead18cSJens Axboe ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); 303f9ead18cSJens Axboe if (unlikely(ret)) 304f9ead18cSJens Axboe return ret; 305f9ead18cSJens Axboe 306f9ead18cSJens Axboe msg.msg_name = NULL; 307f9ead18cSJens Axboe msg.msg_control = NULL; 308f9ead18cSJens Axboe msg.msg_controllen = 0; 309f9ead18cSJens Axboe msg.msg_namelen = 0; 310e02b6651SPavel Begunkov msg.msg_ubuf = NULL; 311f9ead18cSJens Axboe 312f9ead18cSJens Axboe flags = sr->msg_flags; 313f9ead18cSJens Axboe if (issue_flags & IO_URING_F_NONBLOCK) 314f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 315f9ead18cSJens Axboe if (flags & MSG_WAITALL) 316f9ead18cSJens Axboe min_ret = iov_iter_count(&msg.msg_iter); 317f9ead18cSJens Axboe 318f9ead18cSJens Axboe msg.msg_flags = flags; 319f9ead18cSJens Axboe ret = sock_sendmsg(sock, &msg); 320f9ead18cSJens Axboe if (ret < min_ret) { 321f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 322f9ead18cSJens Axboe return -EAGAIN; 323f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 324f9ead18cSJens Axboe ret = -EINTR; 325f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 326f9ead18cSJens Axboe sr->len -= ret; 327f9ead18cSJens Axboe sr->buf += ret; 328f9ead18cSJens Axboe sr->done_io += ret; 329f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 330f9ead18cSJens Axboe return -EAGAIN; 331f9ead18cSJens Axboe } 332f9ead18cSJens Axboe req_set_fail(req); 333f9ead18cSJens Axboe } 334f9ead18cSJens Axboe if (ret >= 0) 335f9ead18cSJens Axboe ret += sr->done_io; 336f9ead18cSJens Axboe else if (sr->done_io) 337f9ead18cSJens Axboe ret = sr->done_io; 338f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 339f9ead18cSJens Axboe return IOU_OK; 340f9ead18cSJens Axboe } 341f9ead18cSJens Axboe 3429bb66906SDylan Yudaken static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) 3439bb66906SDylan Yudaken { 3449b0fc3c0SDylan Yudaken int hdr; 3459bb66906SDylan Yudaken 3469b0fc3c0SDylan Yudaken if (iomsg->namelen < 0) 3479bb66906SDylan Yudaken return true; 3489b0fc3c0SDylan Yudaken if (check_add_overflow((int)sizeof(struct io_uring_recvmsg_out), 3499b0fc3c0SDylan Yudaken iomsg->namelen, &hdr)) 3509bb66906SDylan Yudaken return true; 3519b0fc3c0SDylan Yudaken if (check_add_overflow(hdr, (int)iomsg->controllen, &hdr)) 3529bb66906SDylan Yudaken return true; 3539bb66906SDylan Yudaken 3549bb66906SDylan Yudaken return false; 3559bb66906SDylan Yudaken } 3569bb66906SDylan Yudaken 357f9ead18cSJens Axboe static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 358f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 359f9ead18cSJens Axboe { 360f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 3617fa875b8SDylan Yudaken struct user_msghdr msg; 362f9ead18cSJens Axboe int ret; 363f9ead18cSJens Axboe 3647fa875b8SDylan Yudaken if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg))) 3657fa875b8SDylan Yudaken return -EFAULT; 3667fa875b8SDylan Yudaken 3677fa875b8SDylan Yudaken ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 368f9ead18cSJens Axboe if (ret) 369f9ead18cSJens Axboe return ret; 370f9ead18cSJens Axboe 371f9ead18cSJens Axboe if (req->flags & REQ_F_BUFFER_SELECT) { 3727fa875b8SDylan Yudaken if (msg.msg_iovlen == 0) { 3735702196eSDylan Yudaken sr->len = iomsg->fast_iov[0].iov_len = 0; 3745702196eSDylan Yudaken iomsg->fast_iov[0].iov_base = NULL; 3755702196eSDylan Yudaken iomsg->free_iov = NULL; 3767fa875b8SDylan Yudaken } else if (msg.msg_iovlen > 1) { 377f9ead18cSJens Axboe return -EINVAL; 3785702196eSDylan Yudaken } else { 3797fa875b8SDylan Yudaken if (copy_from_user(iomsg->fast_iov, msg.msg_iov, sizeof(*msg.msg_iov))) 380f9ead18cSJens Axboe return -EFAULT; 381f9ead18cSJens Axboe sr->len = iomsg->fast_iov[0].iov_len; 382f9ead18cSJens Axboe iomsg->free_iov = NULL; 3835702196eSDylan Yudaken } 3849bb66906SDylan Yudaken 3859bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 3869bb66906SDylan Yudaken iomsg->namelen = msg.msg_namelen; 3879bb66906SDylan Yudaken iomsg->controllen = msg.msg_controllen; 3889bb66906SDylan Yudaken if (io_recvmsg_multishot_overflow(iomsg)) 3899bb66906SDylan Yudaken return -EOVERFLOW; 3909bb66906SDylan Yudaken } 391f9ead18cSJens Axboe } else { 392f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 3937fa875b8SDylan Yudaken ret = __import_iovec(READ, msg.msg_iov, msg.msg_iovlen, UIO_FASTIOV, 394f9ead18cSJens Axboe &iomsg->free_iov, &iomsg->msg.msg_iter, 395f9ead18cSJens Axboe false); 396f9ead18cSJens Axboe if (ret > 0) 397f9ead18cSJens Axboe ret = 0; 398f9ead18cSJens Axboe } 399f9ead18cSJens Axboe 400f9ead18cSJens Axboe return ret; 401f9ead18cSJens Axboe } 402f9ead18cSJens Axboe 403f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 404f9ead18cSJens Axboe static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 405f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 406f9ead18cSJens Axboe { 407f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 40872c531f8SDylan Yudaken struct compat_msghdr msg; 409f9ead18cSJens Axboe struct compat_iovec __user *uiov; 410f9ead18cSJens Axboe int ret; 411f9ead18cSJens Axboe 41272c531f8SDylan Yudaken if (copy_from_user(&msg, sr->umsg_compat, sizeof(msg))) 41372c531f8SDylan Yudaken return -EFAULT; 41472c531f8SDylan Yudaken 4154f6a94d3SJens Axboe ret = __get_compat_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 416f9ead18cSJens Axboe if (ret) 417f9ead18cSJens Axboe return ret; 418f9ead18cSJens Axboe 41972c531f8SDylan Yudaken uiov = compat_ptr(msg.msg_iov); 420f9ead18cSJens Axboe if (req->flags & REQ_F_BUFFER_SELECT) { 421f9ead18cSJens Axboe compat_ssize_t clen; 422f9ead18cSJens Axboe 42372c531f8SDylan Yudaken if (msg.msg_iovlen == 0) { 4246d2f75a0SDylan Yudaken sr->len = 0; 4256d2f75a0SDylan Yudaken iomsg->free_iov = NULL; 42672c531f8SDylan Yudaken } else if (msg.msg_iovlen > 1) { 427f9ead18cSJens Axboe return -EINVAL; 4286d2f75a0SDylan Yudaken } else { 429f9ead18cSJens Axboe if (!access_ok(uiov, sizeof(*uiov))) 430f9ead18cSJens Axboe return -EFAULT; 431f9ead18cSJens Axboe if (__get_user(clen, &uiov->iov_len)) 432f9ead18cSJens Axboe return -EFAULT; 433f9ead18cSJens Axboe if (clen < 0) 434f9ead18cSJens Axboe return -EINVAL; 435f9ead18cSJens Axboe sr->len = clen; 436f9ead18cSJens Axboe iomsg->free_iov = NULL; 4376d2f75a0SDylan Yudaken } 4389bb66906SDylan Yudaken 4399bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 4409bb66906SDylan Yudaken iomsg->namelen = msg.msg_namelen; 4419bb66906SDylan Yudaken iomsg->controllen = msg.msg_controllen; 4429bb66906SDylan Yudaken if (io_recvmsg_multishot_overflow(iomsg)) 4439bb66906SDylan Yudaken return -EOVERFLOW; 4449bb66906SDylan Yudaken } 445f9ead18cSJens Axboe } else { 446f9ead18cSJens Axboe iomsg->free_iov = iomsg->fast_iov; 44772c531f8SDylan Yudaken ret = __import_iovec(READ, (struct iovec __user *)uiov, msg.msg_iovlen, 448f9ead18cSJens Axboe UIO_FASTIOV, &iomsg->free_iov, 449f9ead18cSJens Axboe &iomsg->msg.msg_iter, true); 450f9ead18cSJens Axboe if (ret < 0) 451f9ead18cSJens Axboe return ret; 452f9ead18cSJens Axboe } 453f9ead18cSJens Axboe 454f9ead18cSJens Axboe return 0; 455f9ead18cSJens Axboe } 456f9ead18cSJens Axboe #endif 457f9ead18cSJens Axboe 458f9ead18cSJens Axboe static int io_recvmsg_copy_hdr(struct io_kiocb *req, 459f9ead18cSJens Axboe struct io_async_msghdr *iomsg) 460f9ead18cSJens Axboe { 461f9ead18cSJens Axboe iomsg->msg.msg_name = &iomsg->addr; 462f9ead18cSJens Axboe 463f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 464f9ead18cSJens Axboe if (req->ctx->compat) 465f9ead18cSJens Axboe return __io_compat_recvmsg_copy_hdr(req, iomsg); 466f9ead18cSJens Axboe #endif 467f9ead18cSJens Axboe 468f9ead18cSJens Axboe return __io_recvmsg_copy_hdr(req, iomsg); 469f9ead18cSJens Axboe } 470f9ead18cSJens Axboe 471f9ead18cSJens Axboe int io_recvmsg_prep_async(struct io_kiocb *req) 472f9ead18cSJens Axboe { 473f9ead18cSJens Axboe int ret; 474f9ead18cSJens Axboe 475f9ead18cSJens Axboe ret = io_recvmsg_copy_hdr(req, req->async_data); 476f9ead18cSJens Axboe if (!ret) 477f9ead18cSJens Axboe req->flags |= REQ_F_NEED_CLEANUP; 478f9ead18cSJens Axboe return ret; 479f9ead18cSJens Axboe } 480f9ead18cSJens Axboe 481b3fdea6eSDylan Yudaken #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT) 482b3fdea6eSDylan Yudaken 483f9ead18cSJens Axboe int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 484f9ead18cSJens Axboe { 485f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 486f9ead18cSJens Axboe 487f9ead18cSJens Axboe if (unlikely(sqe->file_index || sqe->addr2)) 488f9ead18cSJens Axboe return -EINVAL; 489f9ead18cSJens Axboe 490f9ead18cSJens Axboe sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 491f9ead18cSJens Axboe sr->len = READ_ONCE(sqe->len); 492f9ead18cSJens Axboe sr->flags = READ_ONCE(sqe->ioprio); 493b3fdea6eSDylan Yudaken if (sr->flags & ~(RECVMSG_FLAGS)) 494f9ead18cSJens Axboe return -EINVAL; 495f9ead18cSJens Axboe sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 496f9ead18cSJens Axboe if (sr->msg_flags & MSG_DONTWAIT) 497f9ead18cSJens Axboe req->flags |= REQ_F_NOWAIT; 498f9ead18cSJens Axboe if (sr->msg_flags & MSG_ERRQUEUE) 499f9ead18cSJens Axboe req->flags |= REQ_F_CLEAR_POLLIN; 500b3fdea6eSDylan Yudaken if (sr->flags & IORING_RECV_MULTISHOT) { 501b3fdea6eSDylan Yudaken if (!(req->flags & REQ_F_BUFFER_SELECT)) 502b3fdea6eSDylan Yudaken return -EINVAL; 503b3fdea6eSDylan Yudaken if (sr->msg_flags & MSG_WAITALL) 504b3fdea6eSDylan Yudaken return -EINVAL; 505b3fdea6eSDylan Yudaken if (req->opcode == IORING_OP_RECV && sr->len) 506b3fdea6eSDylan Yudaken return -EINVAL; 507b3fdea6eSDylan Yudaken req->flags |= REQ_F_APOLL_MULTISHOT; 508b3fdea6eSDylan Yudaken } 509f9ead18cSJens Axboe 510f9ead18cSJens Axboe #ifdef CONFIG_COMPAT 511f9ead18cSJens Axboe if (req->ctx->compat) 512f9ead18cSJens Axboe sr->msg_flags |= MSG_CMSG_COMPAT; 513f9ead18cSJens Axboe #endif 514f9ead18cSJens Axboe sr->done_io = 0; 515f9ead18cSJens Axboe return 0; 516f9ead18cSJens Axboe } 517f9ead18cSJens Axboe 518b3fdea6eSDylan Yudaken static inline void io_recv_prep_retry(struct io_kiocb *req) 519b3fdea6eSDylan Yudaken { 520b3fdea6eSDylan Yudaken struct io_sr_msg *sr = io_kiocb_to_cmd(req); 521b3fdea6eSDylan Yudaken 522b3fdea6eSDylan Yudaken sr->done_io = 0; 523b3fdea6eSDylan Yudaken sr->len = 0; /* get from the provided buffer */ 524b3fdea6eSDylan Yudaken } 525b3fdea6eSDylan Yudaken 526b3fdea6eSDylan Yudaken /* 5279bb66906SDylan Yudaken * Finishes io_recv and io_recvmsg. 528b3fdea6eSDylan Yudaken * 529b3fdea6eSDylan Yudaken * Returns true if it is actually finished, or false if it should run 530b3fdea6eSDylan Yudaken * again (for multishot). 531b3fdea6eSDylan Yudaken */ 5329bb66906SDylan Yudaken static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 5339bb66906SDylan Yudaken unsigned int cflags, bool mshot_finished) 534b3fdea6eSDylan Yudaken { 535b3fdea6eSDylan Yudaken if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 536b3fdea6eSDylan Yudaken io_req_set_res(req, *ret, cflags); 537b3fdea6eSDylan Yudaken *ret = IOU_OK; 538b3fdea6eSDylan Yudaken return true; 539b3fdea6eSDylan Yudaken } 540b3fdea6eSDylan Yudaken 5419bb66906SDylan Yudaken if (!mshot_finished) { 542b3fdea6eSDylan Yudaken if (io_post_aux_cqe(req->ctx, req->cqe.user_data, *ret, 543b3fdea6eSDylan Yudaken cflags | IORING_CQE_F_MORE, false)) { 544b3fdea6eSDylan Yudaken io_recv_prep_retry(req); 545b3fdea6eSDylan Yudaken return false; 546b3fdea6eSDylan Yudaken } 547b3fdea6eSDylan Yudaken /* 548b3fdea6eSDylan Yudaken * Otherwise stop multishot but use the current result. 549b3fdea6eSDylan Yudaken * Probably will end up going into overflow, but this means 550b3fdea6eSDylan Yudaken * we cannot trust the ordering anymore 551b3fdea6eSDylan Yudaken */ 552b3fdea6eSDylan Yudaken } 553b3fdea6eSDylan Yudaken 554b3fdea6eSDylan Yudaken io_req_set_res(req, *ret, cflags); 555b3fdea6eSDylan Yudaken 556b3fdea6eSDylan Yudaken if (req->flags & REQ_F_POLLED) 557b3fdea6eSDylan Yudaken *ret = IOU_STOP_MULTISHOT; 558e2df2ccbSDylan Yudaken else 559e2df2ccbSDylan Yudaken *ret = IOU_OK; 560b3fdea6eSDylan Yudaken return true; 561b3fdea6eSDylan Yudaken } 562b3fdea6eSDylan Yudaken 5639bb66906SDylan Yudaken static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 5649bb66906SDylan Yudaken struct io_sr_msg *sr, void __user **buf, 5659bb66906SDylan Yudaken size_t *len) 5669bb66906SDylan Yudaken { 5679bb66906SDylan Yudaken unsigned long ubuf = (unsigned long) *buf; 5689bb66906SDylan Yudaken unsigned long hdr; 5699bb66906SDylan Yudaken 5709bb66906SDylan Yudaken hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 5719bb66906SDylan Yudaken kmsg->controllen; 5729bb66906SDylan Yudaken if (*len < hdr) 5739bb66906SDylan Yudaken return -EFAULT; 5749bb66906SDylan Yudaken 5759bb66906SDylan Yudaken if (kmsg->controllen) { 5769bb66906SDylan Yudaken unsigned long control = ubuf + hdr - kmsg->controllen; 5779bb66906SDylan Yudaken 5789bb66906SDylan Yudaken kmsg->msg.msg_control_user = (void *) control; 5799bb66906SDylan Yudaken kmsg->msg.msg_controllen = kmsg->controllen; 5809bb66906SDylan Yudaken } 5819bb66906SDylan Yudaken 5829bb66906SDylan Yudaken sr->buf = *buf; /* stash for later copy */ 5839bb66906SDylan Yudaken *buf = (void *) (ubuf + hdr); 5849bb66906SDylan Yudaken kmsg->payloadlen = *len = *len - hdr; 5859bb66906SDylan Yudaken return 0; 5869bb66906SDylan Yudaken } 5879bb66906SDylan Yudaken 5889bb66906SDylan Yudaken struct io_recvmsg_multishot_hdr { 5899bb66906SDylan Yudaken struct io_uring_recvmsg_out msg; 5909bb66906SDylan Yudaken struct sockaddr_storage addr; 5919bb66906SDylan Yudaken }; 5929bb66906SDylan Yudaken 5939bb66906SDylan Yudaken static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 5949bb66906SDylan Yudaken struct io_async_msghdr *kmsg, 5959bb66906SDylan Yudaken unsigned int flags, bool *finished) 5969bb66906SDylan Yudaken { 5979bb66906SDylan Yudaken int err; 5989bb66906SDylan Yudaken int copy_len; 5999bb66906SDylan Yudaken struct io_recvmsg_multishot_hdr hdr; 6009bb66906SDylan Yudaken 6019bb66906SDylan Yudaken if (kmsg->namelen) 6029bb66906SDylan Yudaken kmsg->msg.msg_name = &hdr.addr; 6039bb66906SDylan Yudaken kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 6049bb66906SDylan Yudaken kmsg->msg.msg_namelen = 0; 6059bb66906SDylan Yudaken 6069bb66906SDylan Yudaken if (sock->file->f_flags & O_NONBLOCK) 6079bb66906SDylan Yudaken flags |= MSG_DONTWAIT; 6089bb66906SDylan Yudaken 6099bb66906SDylan Yudaken err = sock_recvmsg(sock, &kmsg->msg, flags); 6109bb66906SDylan Yudaken *finished = err <= 0; 6119bb66906SDylan Yudaken if (err < 0) 6129bb66906SDylan Yudaken return err; 6139bb66906SDylan Yudaken 6149bb66906SDylan Yudaken hdr.msg = (struct io_uring_recvmsg_out) { 6159bb66906SDylan Yudaken .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 6169bb66906SDylan Yudaken .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 6179bb66906SDylan Yudaken }; 6189bb66906SDylan Yudaken 6199bb66906SDylan Yudaken hdr.msg.payloadlen = err; 6209bb66906SDylan Yudaken if (err > kmsg->payloadlen) 6219bb66906SDylan Yudaken err = kmsg->payloadlen; 6229bb66906SDylan Yudaken 6239bb66906SDylan Yudaken copy_len = sizeof(struct io_uring_recvmsg_out); 6249bb66906SDylan Yudaken if (kmsg->msg.msg_namelen > kmsg->namelen) 6259bb66906SDylan Yudaken copy_len += kmsg->namelen; 6269bb66906SDylan Yudaken else 6279bb66906SDylan Yudaken copy_len += kmsg->msg.msg_namelen; 6289bb66906SDylan Yudaken 6299bb66906SDylan Yudaken /* 6309bb66906SDylan Yudaken * "fromlen shall refer to the value before truncation.." 6319bb66906SDylan Yudaken * 1003.1g 6329bb66906SDylan Yudaken */ 6339bb66906SDylan Yudaken hdr.msg.namelen = kmsg->msg.msg_namelen; 6349bb66906SDylan Yudaken 6359bb66906SDylan Yudaken /* ensure that there is no gap between hdr and sockaddr_storage */ 6369bb66906SDylan Yudaken BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 6379bb66906SDylan Yudaken sizeof(struct io_uring_recvmsg_out)); 6389bb66906SDylan Yudaken if (copy_to_user(io->buf, &hdr, copy_len)) { 6399bb66906SDylan Yudaken *finished = true; 6409bb66906SDylan Yudaken return -EFAULT; 6419bb66906SDylan Yudaken } 6429bb66906SDylan Yudaken 6439bb66906SDylan Yudaken return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 6449bb66906SDylan Yudaken kmsg->controllen + err; 6459bb66906SDylan Yudaken } 6469bb66906SDylan Yudaken 647f9ead18cSJens Axboe int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 648f9ead18cSJens Axboe { 649f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 650f9ead18cSJens Axboe struct io_async_msghdr iomsg, *kmsg; 651f9ead18cSJens Axboe struct socket *sock; 652f9ead18cSJens Axboe unsigned int cflags; 653f9ead18cSJens Axboe unsigned flags; 654f9ead18cSJens Axboe int ret, min_ret = 0; 655f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 6569bb66906SDylan Yudaken bool mshot_finished = true; 657f9ead18cSJens Axboe 658f9ead18cSJens Axboe sock = sock_from_file(req->file); 659f9ead18cSJens Axboe if (unlikely(!sock)) 660f9ead18cSJens Axboe return -ENOTSOCK; 661f9ead18cSJens Axboe 662f9ead18cSJens Axboe if (req_has_async_data(req)) { 663f9ead18cSJens Axboe kmsg = req->async_data; 664f9ead18cSJens Axboe } else { 665f9ead18cSJens Axboe ret = io_recvmsg_copy_hdr(req, &iomsg); 666f9ead18cSJens Axboe if (ret) 667f9ead18cSJens Axboe return ret; 668f9ead18cSJens Axboe kmsg = &iomsg; 669f9ead18cSJens Axboe } 670f9ead18cSJens Axboe 671f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 672f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 67343e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 674f9ead18cSJens Axboe 6759bb66906SDylan Yudaken retry_multishot: 676f9ead18cSJens Axboe if (io_do_buffer_select(req)) { 677f9ead18cSJens Axboe void __user *buf; 6789bb66906SDylan Yudaken size_t len = sr->len; 679f9ead18cSJens Axboe 6809bb66906SDylan Yudaken buf = io_buffer_select(req, &len, issue_flags); 681f9ead18cSJens Axboe if (!buf) 682f9ead18cSJens Axboe return -ENOBUFS; 6839bb66906SDylan Yudaken 6849bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) { 6859bb66906SDylan Yudaken ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); 6869bb66906SDylan Yudaken if (ret) { 6879bb66906SDylan Yudaken io_kbuf_recycle(req, issue_flags); 6889bb66906SDylan Yudaken return ret; 6899bb66906SDylan Yudaken } 6909bb66906SDylan Yudaken } 6919bb66906SDylan Yudaken 692f9ead18cSJens Axboe kmsg->fast_iov[0].iov_base = buf; 6939bb66906SDylan Yudaken kmsg->fast_iov[0].iov_len = len; 694f9ead18cSJens Axboe iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, 6959bb66906SDylan Yudaken len); 696f9ead18cSJens Axboe } 697f9ead18cSJens Axboe 698f9ead18cSJens Axboe flags = sr->msg_flags; 699f9ead18cSJens Axboe if (force_nonblock) 700f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 701f9ead18cSJens Axboe if (flags & MSG_WAITALL) 702f9ead18cSJens Axboe min_ret = iov_iter_count(&kmsg->msg.msg_iter); 703f9ead18cSJens Axboe 704f9ead18cSJens Axboe kmsg->msg.msg_get_inq = 1; 7059bb66906SDylan Yudaken if (req->flags & REQ_F_APOLL_MULTISHOT) 7069bb66906SDylan Yudaken ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 7079bb66906SDylan Yudaken &mshot_finished); 7089bb66906SDylan Yudaken else 7099bb66906SDylan Yudaken ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 7109bb66906SDylan Yudaken kmsg->uaddr, flags); 7119bb66906SDylan Yudaken 712f9ead18cSJens Axboe if (ret < min_ret) { 7139bb66906SDylan Yudaken if (ret == -EAGAIN && force_nonblock) { 7149bb66906SDylan Yudaken ret = io_setup_async_msg(req, kmsg, issue_flags); 7159bb66906SDylan Yudaken if (ret == -EAGAIN && (req->flags & IO_APOLL_MULTI_POLLED) == 7169bb66906SDylan Yudaken IO_APOLL_MULTI_POLLED) { 7179bb66906SDylan Yudaken io_kbuf_recycle(req, issue_flags); 7189bb66906SDylan Yudaken return IOU_ISSUE_SKIP_COMPLETE; 7199bb66906SDylan Yudaken } 7209bb66906SDylan Yudaken return ret; 7219bb66906SDylan Yudaken } 722f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 723f9ead18cSJens Axboe ret = -EINTR; 724f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 725f9ead18cSJens Axboe sr->done_io += ret; 726f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 72743e0bbbdSJens Axboe return io_setup_async_msg(req, kmsg, issue_flags); 728f9ead18cSJens Axboe } 729f9ead18cSJens Axboe req_set_fail(req); 730f9ead18cSJens Axboe } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 731f9ead18cSJens Axboe req_set_fail(req); 732f9ead18cSJens Axboe } 733f9ead18cSJens Axboe 734d4e097daSDylan Yudaken if (ret > 0) 735f9ead18cSJens Axboe ret += sr->done_io; 736f9ead18cSJens Axboe else if (sr->done_io) 737f9ead18cSJens Axboe ret = sr->done_io; 738d4e097daSDylan Yudaken else 739d4e097daSDylan Yudaken io_kbuf_recycle(req, issue_flags); 740d4e097daSDylan Yudaken 741f9ead18cSJens Axboe cflags = io_put_kbuf(req, issue_flags); 742f9ead18cSJens Axboe if (kmsg->msg.msg_inq) 743f9ead18cSJens Axboe cflags |= IORING_CQE_F_SOCK_NONEMPTY; 744b3fdea6eSDylan Yudaken 7459bb66906SDylan Yudaken if (!io_recv_finish(req, &ret, cflags, mshot_finished)) 7469bb66906SDylan Yudaken goto retry_multishot; 7479bb66906SDylan Yudaken 7489bb66906SDylan Yudaken if (mshot_finished) { 7499bb66906SDylan Yudaken io_netmsg_recycle(req, issue_flags); 7509bb66906SDylan Yudaken /* fast path, check for non-NULL to avoid function call */ 7519bb66906SDylan Yudaken if (kmsg->free_iov) 7529bb66906SDylan Yudaken kfree(kmsg->free_iov); 7539bb66906SDylan Yudaken req->flags &= ~REQ_F_NEED_CLEANUP; 7549bb66906SDylan Yudaken } 7559bb66906SDylan Yudaken 7569bb66906SDylan Yudaken return ret; 757f9ead18cSJens Axboe } 758f9ead18cSJens Axboe 759f9ead18cSJens Axboe int io_recv(struct io_kiocb *req, unsigned int issue_flags) 760f9ead18cSJens Axboe { 761f9ead18cSJens Axboe struct io_sr_msg *sr = io_kiocb_to_cmd(req); 762f9ead18cSJens Axboe struct msghdr msg; 763f9ead18cSJens Axboe struct socket *sock; 764f9ead18cSJens Axboe struct iovec iov; 765f9ead18cSJens Axboe unsigned int cflags; 766f9ead18cSJens Axboe unsigned flags; 767f9ead18cSJens Axboe int ret, min_ret = 0; 768f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 769b3fdea6eSDylan Yudaken size_t len = sr->len; 770f9ead18cSJens Axboe 771f9ead18cSJens Axboe if (!(req->flags & REQ_F_POLLED) && 772f9ead18cSJens Axboe (sr->flags & IORING_RECVSEND_POLL_FIRST)) 773f9ead18cSJens Axboe return -EAGAIN; 774f9ead18cSJens Axboe 775f9ead18cSJens Axboe sock = sock_from_file(req->file); 776f9ead18cSJens Axboe if (unlikely(!sock)) 777f9ead18cSJens Axboe return -ENOTSOCK; 778f9ead18cSJens Axboe 779b3fdea6eSDylan Yudaken retry_multishot: 780f9ead18cSJens Axboe if (io_do_buffer_select(req)) { 781f9ead18cSJens Axboe void __user *buf; 782f9ead18cSJens Axboe 783b3fdea6eSDylan Yudaken buf = io_buffer_select(req, &len, issue_flags); 784f9ead18cSJens Axboe if (!buf) 785f9ead18cSJens Axboe return -ENOBUFS; 786f9ead18cSJens Axboe sr->buf = buf; 787f9ead18cSJens Axboe } 788f9ead18cSJens Axboe 789b3fdea6eSDylan Yudaken ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter); 790f9ead18cSJens Axboe if (unlikely(ret)) 791f9ead18cSJens Axboe goto out_free; 792f9ead18cSJens Axboe 793f9ead18cSJens Axboe msg.msg_name = NULL; 794f9ead18cSJens Axboe msg.msg_namelen = 0; 795f9ead18cSJens Axboe msg.msg_control = NULL; 796f9ead18cSJens Axboe msg.msg_get_inq = 1; 797f9ead18cSJens Axboe msg.msg_flags = 0; 798f9ead18cSJens Axboe msg.msg_controllen = 0; 799f9ead18cSJens Axboe msg.msg_iocb = NULL; 800e02b6651SPavel Begunkov msg.msg_ubuf = NULL; 801f9ead18cSJens Axboe 802f9ead18cSJens Axboe flags = sr->msg_flags; 803f9ead18cSJens Axboe if (force_nonblock) 804f9ead18cSJens Axboe flags |= MSG_DONTWAIT; 805f9ead18cSJens Axboe if (flags & MSG_WAITALL) 806f9ead18cSJens Axboe min_ret = iov_iter_count(&msg.msg_iter); 807f9ead18cSJens Axboe 808f9ead18cSJens Axboe ret = sock_recvmsg(sock, &msg, flags); 809f9ead18cSJens Axboe if (ret < min_ret) { 810b3fdea6eSDylan Yudaken if (ret == -EAGAIN && force_nonblock) { 811b3fdea6eSDylan Yudaken if ((req->flags & IO_APOLL_MULTI_POLLED) == IO_APOLL_MULTI_POLLED) { 812b3fdea6eSDylan Yudaken io_kbuf_recycle(req, issue_flags); 813b3fdea6eSDylan Yudaken return IOU_ISSUE_SKIP_COMPLETE; 814b3fdea6eSDylan Yudaken } 815b3fdea6eSDylan Yudaken 816f9ead18cSJens Axboe return -EAGAIN; 817b3fdea6eSDylan Yudaken } 818f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 819f9ead18cSJens Axboe ret = -EINTR; 820f9ead18cSJens Axboe if (ret > 0 && io_net_retry(sock, flags)) { 821f9ead18cSJens Axboe sr->len -= ret; 822f9ead18cSJens Axboe sr->buf += ret; 823f9ead18cSJens Axboe sr->done_io += ret; 824f9ead18cSJens Axboe req->flags |= REQ_F_PARTIAL_IO; 825f9ead18cSJens Axboe return -EAGAIN; 826f9ead18cSJens Axboe } 827f9ead18cSJens Axboe req_set_fail(req); 828f9ead18cSJens Axboe } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 829f9ead18cSJens Axboe out_free: 830f9ead18cSJens Axboe req_set_fail(req); 831f9ead18cSJens Axboe } 832f9ead18cSJens Axboe 833d4e097daSDylan Yudaken if (ret > 0) 834f9ead18cSJens Axboe ret += sr->done_io; 835f9ead18cSJens Axboe else if (sr->done_io) 836f9ead18cSJens Axboe ret = sr->done_io; 837d4e097daSDylan Yudaken else 838d4e097daSDylan Yudaken io_kbuf_recycle(req, issue_flags); 839d4e097daSDylan Yudaken 840f9ead18cSJens Axboe cflags = io_put_kbuf(req, issue_flags); 841f9ead18cSJens Axboe if (msg.msg_inq) 842f9ead18cSJens Axboe cflags |= IORING_CQE_F_SOCK_NONEMPTY; 843b3fdea6eSDylan Yudaken 8449bb66906SDylan Yudaken if (!io_recv_finish(req, &ret, cflags, ret <= 0)) 845b3fdea6eSDylan Yudaken goto retry_multishot; 846b3fdea6eSDylan Yudaken 847b3fdea6eSDylan Yudaken return ret; 848f9ead18cSJens Axboe } 849f9ead18cSJens Axboe 85006a5464bSPavel Begunkov int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 85106a5464bSPavel Begunkov { 85206a5464bSPavel Begunkov struct io_sendzc *zc = io_kiocb_to_cmd(req); 85310c7d33eSPavel Begunkov struct io_ring_ctx *ctx = req->ctx; 85406a5464bSPavel Begunkov 855092aeedbSPavel Begunkov if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)) 85606a5464bSPavel Begunkov return -EINVAL; 85706a5464bSPavel Begunkov 85806a5464bSPavel Begunkov zc->flags = READ_ONCE(sqe->ioprio); 85963809137SPavel Begunkov if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | 86063809137SPavel Begunkov IORING_RECVSEND_FIXED_BUF | IORING_RECVSEND_NOTIF_FLUSH)) 86106a5464bSPavel Begunkov return -EINVAL; 86210c7d33eSPavel Begunkov if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 86310c7d33eSPavel Begunkov unsigned idx = READ_ONCE(sqe->buf_index); 86410c7d33eSPavel Begunkov 86510c7d33eSPavel Begunkov if (unlikely(idx >= ctx->nr_user_bufs)) 86610c7d33eSPavel Begunkov return -EFAULT; 86710c7d33eSPavel Begunkov idx = array_index_nospec(idx, ctx->nr_user_bufs); 86810c7d33eSPavel Begunkov req->imu = READ_ONCE(ctx->user_bufs[idx]); 86910c7d33eSPavel Begunkov io_req_set_rsrc_node(req, ctx, 0); 87010c7d33eSPavel Begunkov } 87106a5464bSPavel Begunkov 87206a5464bSPavel Begunkov zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 87306a5464bSPavel Begunkov zc->len = READ_ONCE(sqe->len); 87406a5464bSPavel Begunkov zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 87506a5464bSPavel Begunkov zc->slot_idx = READ_ONCE(sqe->notification_idx); 87606a5464bSPavel Begunkov if (zc->msg_flags & MSG_DONTWAIT) 87706a5464bSPavel Begunkov req->flags |= REQ_F_NOWAIT; 878092aeedbSPavel Begunkov 879092aeedbSPavel Begunkov zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 880092aeedbSPavel Begunkov zc->addr_len = READ_ONCE(sqe->addr_len); 881092aeedbSPavel Begunkov 88206a5464bSPavel Begunkov #ifdef CONFIG_COMPAT 88306a5464bSPavel Begunkov if (req->ctx->compat) 88406a5464bSPavel Begunkov zc->msg_flags |= MSG_CMSG_COMPAT; 88506a5464bSPavel Begunkov #endif 88606a5464bSPavel Begunkov return 0; 88706a5464bSPavel Begunkov } 88806a5464bSPavel Begunkov 8893ff1a0d3SPavel Begunkov static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, 8903ff1a0d3SPavel Begunkov struct iov_iter *from, size_t length) 8913ff1a0d3SPavel Begunkov { 8923ff1a0d3SPavel Begunkov struct skb_shared_info *shinfo = skb_shinfo(skb); 8933ff1a0d3SPavel Begunkov int frag = shinfo->nr_frags; 8943ff1a0d3SPavel Begunkov int ret = 0; 8953ff1a0d3SPavel Begunkov struct bvec_iter bi; 8963ff1a0d3SPavel Begunkov ssize_t copied = 0; 8973ff1a0d3SPavel Begunkov unsigned long truesize = 0; 8983ff1a0d3SPavel Begunkov 8993ff1a0d3SPavel Begunkov if (!shinfo->nr_frags) 9003ff1a0d3SPavel Begunkov shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 9013ff1a0d3SPavel Begunkov 9023ff1a0d3SPavel Begunkov if (!skb_zcopy_managed(skb) || !iov_iter_is_bvec(from)) { 9033ff1a0d3SPavel Begunkov skb_zcopy_downgrade_managed(skb); 9043ff1a0d3SPavel Begunkov return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 9053ff1a0d3SPavel Begunkov } 9063ff1a0d3SPavel Begunkov 9073ff1a0d3SPavel Begunkov bi.bi_size = min(from->count, length); 9083ff1a0d3SPavel Begunkov bi.bi_bvec_done = from->iov_offset; 9093ff1a0d3SPavel Begunkov bi.bi_idx = 0; 9103ff1a0d3SPavel Begunkov 9113ff1a0d3SPavel Begunkov while (bi.bi_size && frag < MAX_SKB_FRAGS) { 9123ff1a0d3SPavel Begunkov struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 9133ff1a0d3SPavel Begunkov 9143ff1a0d3SPavel Begunkov copied += v.bv_len; 9153ff1a0d3SPavel Begunkov truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 9163ff1a0d3SPavel Begunkov __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 9173ff1a0d3SPavel Begunkov v.bv_offset, v.bv_len); 9183ff1a0d3SPavel Begunkov bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 9193ff1a0d3SPavel Begunkov } 9203ff1a0d3SPavel Begunkov if (bi.bi_size) 9213ff1a0d3SPavel Begunkov ret = -EMSGSIZE; 9223ff1a0d3SPavel Begunkov 9233ff1a0d3SPavel Begunkov shinfo->nr_frags = frag; 9243ff1a0d3SPavel Begunkov from->bvec += bi.bi_idx; 9253ff1a0d3SPavel Begunkov from->nr_segs -= bi.bi_idx; 9263ff1a0d3SPavel Begunkov from->count = bi.bi_size; 9273ff1a0d3SPavel Begunkov from->iov_offset = bi.bi_bvec_done; 9283ff1a0d3SPavel Begunkov 9293ff1a0d3SPavel Begunkov skb->data_len += copied; 9303ff1a0d3SPavel Begunkov skb->len += copied; 9313ff1a0d3SPavel Begunkov skb->truesize += truesize; 9323ff1a0d3SPavel Begunkov 9333ff1a0d3SPavel Begunkov if (sk && sk->sk_type == SOCK_STREAM) { 9343ff1a0d3SPavel Begunkov sk_wmem_queued_add(sk, truesize); 9353ff1a0d3SPavel Begunkov if (!skb_zcopy_pure(skb)) 9363ff1a0d3SPavel Begunkov sk_mem_charge(sk, truesize); 9373ff1a0d3SPavel Begunkov } else { 9383ff1a0d3SPavel Begunkov refcount_add(truesize, &skb->sk->sk_wmem_alloc); 9393ff1a0d3SPavel Begunkov } 9403ff1a0d3SPavel Begunkov return ret; 9413ff1a0d3SPavel Begunkov } 9423ff1a0d3SPavel Begunkov 94306a5464bSPavel Begunkov int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) 94406a5464bSPavel Begunkov { 945092aeedbSPavel Begunkov struct sockaddr_storage address; 94606a5464bSPavel Begunkov struct io_ring_ctx *ctx = req->ctx; 94706a5464bSPavel Begunkov struct io_sendzc *zc = io_kiocb_to_cmd(req); 94806a5464bSPavel Begunkov struct io_notif_slot *notif_slot; 94906a5464bSPavel Begunkov struct io_notif *notif; 95006a5464bSPavel Begunkov struct msghdr msg; 95106a5464bSPavel Begunkov struct iovec iov; 95206a5464bSPavel Begunkov struct socket *sock; 95306a5464bSPavel Begunkov unsigned msg_flags; 95406a5464bSPavel Begunkov int ret, min_ret = 0; 95506a5464bSPavel Begunkov 95606a5464bSPavel Begunkov if (!(req->flags & REQ_F_POLLED) && 95706a5464bSPavel Begunkov (zc->flags & IORING_RECVSEND_POLL_FIRST)) 95806a5464bSPavel Begunkov return -EAGAIN; 95906a5464bSPavel Begunkov 96006a5464bSPavel Begunkov if (issue_flags & IO_URING_F_UNLOCKED) 96106a5464bSPavel Begunkov return -EAGAIN; 96206a5464bSPavel Begunkov sock = sock_from_file(req->file); 96306a5464bSPavel Begunkov if (unlikely(!sock)) 96406a5464bSPavel Begunkov return -ENOTSOCK; 96506a5464bSPavel Begunkov 96606a5464bSPavel Begunkov notif_slot = io_get_notif_slot(ctx, zc->slot_idx); 96706a5464bSPavel Begunkov if (!notif_slot) 96806a5464bSPavel Begunkov return -EINVAL; 96906a5464bSPavel Begunkov notif = io_get_notif(ctx, notif_slot); 97006a5464bSPavel Begunkov if (!notif) 97106a5464bSPavel Begunkov return -ENOMEM; 97206a5464bSPavel Begunkov 97306a5464bSPavel Begunkov msg.msg_name = NULL; 97406a5464bSPavel Begunkov msg.msg_control = NULL; 97506a5464bSPavel Begunkov msg.msg_controllen = 0; 97606a5464bSPavel Begunkov msg.msg_namelen = 0; 97706a5464bSPavel Begunkov 97810c7d33eSPavel Begunkov if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 97910c7d33eSPavel Begunkov ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, 98010c7d33eSPavel Begunkov (u64)(uintptr_t)zc->buf, zc->len); 98110c7d33eSPavel Begunkov if (unlikely(ret)) 98210c7d33eSPavel Begunkov return ret; 98310c7d33eSPavel Begunkov } else { 98410c7d33eSPavel Begunkov ret = import_single_range(WRITE, zc->buf, zc->len, &iov, 98510c7d33eSPavel Begunkov &msg.msg_iter); 98606a5464bSPavel Begunkov if (unlikely(ret)) 98706a5464bSPavel Begunkov return ret; 988*2e32ba56SPavel Begunkov ret = mm_account_pinned_pages(¬if->uarg.mmp, zc->len); 989*2e32ba56SPavel Begunkov if (unlikely(ret)) 990*2e32ba56SPavel Begunkov return ret; 99110c7d33eSPavel Begunkov } 99206a5464bSPavel Begunkov 993092aeedbSPavel Begunkov if (zc->addr) { 994092aeedbSPavel Begunkov ret = move_addr_to_kernel(zc->addr, zc->addr_len, &address); 995092aeedbSPavel Begunkov if (unlikely(ret < 0)) 996092aeedbSPavel Begunkov return ret; 997092aeedbSPavel Begunkov msg.msg_name = (struct sockaddr *)&address; 998092aeedbSPavel Begunkov msg.msg_namelen = zc->addr_len; 999092aeedbSPavel Begunkov } 1000092aeedbSPavel Begunkov 100106a5464bSPavel Begunkov msg_flags = zc->msg_flags | MSG_ZEROCOPY; 100206a5464bSPavel Begunkov if (issue_flags & IO_URING_F_NONBLOCK) 100306a5464bSPavel Begunkov msg_flags |= MSG_DONTWAIT; 100406a5464bSPavel Begunkov if (msg_flags & MSG_WAITALL) 100506a5464bSPavel Begunkov min_ret = iov_iter_count(&msg.msg_iter); 100606a5464bSPavel Begunkov 100706a5464bSPavel Begunkov msg.msg_flags = msg_flags; 100806a5464bSPavel Begunkov msg.msg_ubuf = ¬if->uarg; 10093ff1a0d3SPavel Begunkov msg.sg_from_iter = io_sg_from_iter; 101006a5464bSPavel Begunkov ret = sock_sendmsg(sock, &msg); 101106a5464bSPavel Begunkov 101206a5464bSPavel Begunkov if (unlikely(ret < min_ret)) { 101306a5464bSPavel Begunkov if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 101406a5464bSPavel Begunkov return -EAGAIN; 101506a5464bSPavel Begunkov return ret == -ERESTARTSYS ? -EINTR : ret; 101606a5464bSPavel Begunkov } 101706a5464bSPavel Begunkov 101863809137SPavel Begunkov if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) 101963809137SPavel Begunkov io_notif_slot_flush_submit(notif_slot, 0); 102006a5464bSPavel Begunkov io_req_set_res(req, ret, 0); 102106a5464bSPavel Begunkov return IOU_OK; 102206a5464bSPavel Begunkov } 102306a5464bSPavel Begunkov 1024f9ead18cSJens Axboe int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1025f9ead18cSJens Axboe { 1026f9ead18cSJens Axboe struct io_accept *accept = io_kiocb_to_cmd(req); 1027f9ead18cSJens Axboe unsigned flags; 1028f9ead18cSJens Axboe 1029f9ead18cSJens Axboe if (sqe->len || sqe->buf_index) 1030f9ead18cSJens Axboe return -EINVAL; 1031f9ead18cSJens Axboe 1032f9ead18cSJens Axboe accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1033f9ead18cSJens Axboe accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1034f9ead18cSJens Axboe accept->flags = READ_ONCE(sqe->accept_flags); 1035f9ead18cSJens Axboe accept->nofile = rlimit(RLIMIT_NOFILE); 1036f9ead18cSJens Axboe flags = READ_ONCE(sqe->ioprio); 1037f9ead18cSJens Axboe if (flags & ~IORING_ACCEPT_MULTISHOT) 1038f9ead18cSJens Axboe return -EINVAL; 1039f9ead18cSJens Axboe 1040f9ead18cSJens Axboe accept->file_slot = READ_ONCE(sqe->file_index); 1041f9ead18cSJens Axboe if (accept->file_slot) { 1042f9ead18cSJens Axboe if (accept->flags & SOCK_CLOEXEC) 1043f9ead18cSJens Axboe return -EINVAL; 1044f9ead18cSJens Axboe if (flags & IORING_ACCEPT_MULTISHOT && 1045f9ead18cSJens Axboe accept->file_slot != IORING_FILE_INDEX_ALLOC) 1046f9ead18cSJens Axboe return -EINVAL; 1047f9ead18cSJens Axboe } 1048f9ead18cSJens Axboe if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1049f9ead18cSJens Axboe return -EINVAL; 1050f9ead18cSJens Axboe if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1051f9ead18cSJens Axboe accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1052f9ead18cSJens Axboe if (flags & IORING_ACCEPT_MULTISHOT) 1053f9ead18cSJens Axboe req->flags |= REQ_F_APOLL_MULTISHOT; 1054f9ead18cSJens Axboe return 0; 1055f9ead18cSJens Axboe } 1056f9ead18cSJens Axboe 1057f9ead18cSJens Axboe int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1058f9ead18cSJens Axboe { 1059f9ead18cSJens Axboe struct io_ring_ctx *ctx = req->ctx; 1060f9ead18cSJens Axboe struct io_accept *accept = io_kiocb_to_cmd(req); 1061f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1062f9ead18cSJens Axboe unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 1063f9ead18cSJens Axboe bool fixed = !!accept->file_slot; 1064f9ead18cSJens Axboe struct file *file; 1065f9ead18cSJens Axboe int ret, fd; 1066f9ead18cSJens Axboe 1067f9ead18cSJens Axboe retry: 1068f9ead18cSJens Axboe if (!fixed) { 1069f9ead18cSJens Axboe fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1070f9ead18cSJens Axboe if (unlikely(fd < 0)) 1071f9ead18cSJens Axboe return fd; 1072f9ead18cSJens Axboe } 1073f9ead18cSJens Axboe file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 1074f9ead18cSJens Axboe accept->flags); 1075f9ead18cSJens Axboe if (IS_ERR(file)) { 1076f9ead18cSJens Axboe if (!fixed) 1077f9ead18cSJens Axboe put_unused_fd(fd); 1078f9ead18cSJens Axboe ret = PTR_ERR(file); 1079f9ead18cSJens Axboe if (ret == -EAGAIN && force_nonblock) { 1080f9ead18cSJens Axboe /* 1081f9ead18cSJens Axboe * if it's multishot and polled, we don't need to 1082f9ead18cSJens Axboe * return EAGAIN to arm the poll infra since it 1083f9ead18cSJens Axboe * has already been done 1084f9ead18cSJens Axboe */ 1085f9ead18cSJens Axboe if ((req->flags & IO_APOLL_MULTI_POLLED) == 1086f9ead18cSJens Axboe IO_APOLL_MULTI_POLLED) 1087f9ead18cSJens Axboe ret = IOU_ISSUE_SKIP_COMPLETE; 1088f9ead18cSJens Axboe return ret; 1089f9ead18cSJens Axboe } 1090f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1091f9ead18cSJens Axboe ret = -EINTR; 1092f9ead18cSJens Axboe req_set_fail(req); 1093f9ead18cSJens Axboe } else if (!fixed) { 1094f9ead18cSJens Axboe fd_install(fd, file); 1095f9ead18cSJens Axboe ret = fd; 1096f9ead18cSJens Axboe } else { 1097f9ead18cSJens Axboe ret = io_fixed_fd_install(req, issue_flags, file, 1098f9ead18cSJens Axboe accept->file_slot); 1099f9ead18cSJens Axboe } 1100f9ead18cSJens Axboe 1101f9ead18cSJens Axboe if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 1102f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1103f9ead18cSJens Axboe return IOU_OK; 1104f9ead18cSJens Axboe } 1105f9ead18cSJens Axboe 1106cbd25748SDylan Yudaken if (ret >= 0 && 1107cbd25748SDylan Yudaken io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE, false)) 1108d245bca6SPavel Begunkov goto retry; 1109cbd25748SDylan Yudaken 1110cbd25748SDylan Yudaken io_req_set_res(req, ret, 0); 1111cbd25748SDylan Yudaken if (req->flags & REQ_F_POLLED) 1112cbd25748SDylan Yudaken return IOU_STOP_MULTISHOT; 1113cbd25748SDylan Yudaken return IOU_OK; 1114f9ead18cSJens Axboe } 1115f9ead18cSJens Axboe 1116f9ead18cSJens Axboe int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1117f9ead18cSJens Axboe { 1118f9ead18cSJens Axboe struct io_socket *sock = io_kiocb_to_cmd(req); 1119f9ead18cSJens Axboe 1120f9ead18cSJens Axboe if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1121f9ead18cSJens Axboe return -EINVAL; 1122f9ead18cSJens Axboe 1123f9ead18cSJens Axboe sock->domain = READ_ONCE(sqe->fd); 1124f9ead18cSJens Axboe sock->type = READ_ONCE(sqe->off); 1125f9ead18cSJens Axboe sock->protocol = READ_ONCE(sqe->len); 1126f9ead18cSJens Axboe sock->file_slot = READ_ONCE(sqe->file_index); 1127f9ead18cSJens Axboe sock->nofile = rlimit(RLIMIT_NOFILE); 1128f9ead18cSJens Axboe 1129f9ead18cSJens Axboe sock->flags = sock->type & ~SOCK_TYPE_MASK; 1130f9ead18cSJens Axboe if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1131f9ead18cSJens Axboe return -EINVAL; 1132f9ead18cSJens Axboe if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1133f9ead18cSJens Axboe return -EINVAL; 1134f9ead18cSJens Axboe return 0; 1135f9ead18cSJens Axboe } 1136f9ead18cSJens Axboe 1137f9ead18cSJens Axboe int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1138f9ead18cSJens Axboe { 1139f9ead18cSJens Axboe struct io_socket *sock = io_kiocb_to_cmd(req); 1140f9ead18cSJens Axboe bool fixed = !!sock->file_slot; 1141f9ead18cSJens Axboe struct file *file; 1142f9ead18cSJens Axboe int ret, fd; 1143f9ead18cSJens Axboe 1144f9ead18cSJens Axboe if (!fixed) { 1145f9ead18cSJens Axboe fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1146f9ead18cSJens Axboe if (unlikely(fd < 0)) 1147f9ead18cSJens Axboe return fd; 1148f9ead18cSJens Axboe } 1149f9ead18cSJens Axboe file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1150f9ead18cSJens Axboe if (IS_ERR(file)) { 1151f9ead18cSJens Axboe if (!fixed) 1152f9ead18cSJens Axboe put_unused_fd(fd); 1153f9ead18cSJens Axboe ret = PTR_ERR(file); 1154f9ead18cSJens Axboe if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1155f9ead18cSJens Axboe return -EAGAIN; 1156f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1157f9ead18cSJens Axboe ret = -EINTR; 1158f9ead18cSJens Axboe req_set_fail(req); 1159f9ead18cSJens Axboe } else if (!fixed) { 1160f9ead18cSJens Axboe fd_install(fd, file); 1161f9ead18cSJens Axboe ret = fd; 1162f9ead18cSJens Axboe } else { 1163f9ead18cSJens Axboe ret = io_fixed_fd_install(req, issue_flags, file, 1164f9ead18cSJens Axboe sock->file_slot); 1165f9ead18cSJens Axboe } 1166f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1167f9ead18cSJens Axboe return IOU_OK; 1168f9ead18cSJens Axboe } 1169f9ead18cSJens Axboe 1170f9ead18cSJens Axboe int io_connect_prep_async(struct io_kiocb *req) 1171f9ead18cSJens Axboe { 1172f9ead18cSJens Axboe struct io_async_connect *io = req->async_data; 1173f9ead18cSJens Axboe struct io_connect *conn = io_kiocb_to_cmd(req); 1174f9ead18cSJens Axboe 1175f9ead18cSJens Axboe return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 1176f9ead18cSJens Axboe } 1177f9ead18cSJens Axboe 1178f9ead18cSJens Axboe int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1179f9ead18cSJens Axboe { 1180f9ead18cSJens Axboe struct io_connect *conn = io_kiocb_to_cmd(req); 1181f9ead18cSJens Axboe 1182f9ead18cSJens Axboe if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1183f9ead18cSJens Axboe return -EINVAL; 1184f9ead18cSJens Axboe 1185f9ead18cSJens Axboe conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1186f9ead18cSJens Axboe conn->addr_len = READ_ONCE(sqe->addr2); 1187f9ead18cSJens Axboe return 0; 1188f9ead18cSJens Axboe } 1189f9ead18cSJens Axboe 1190f9ead18cSJens Axboe int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1191f9ead18cSJens Axboe { 1192f9ead18cSJens Axboe struct io_connect *connect = io_kiocb_to_cmd(req); 1193f9ead18cSJens Axboe struct io_async_connect __io, *io; 1194f9ead18cSJens Axboe unsigned file_flags; 1195f9ead18cSJens Axboe int ret; 1196f9ead18cSJens Axboe bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1197f9ead18cSJens Axboe 1198f9ead18cSJens Axboe if (req_has_async_data(req)) { 1199f9ead18cSJens Axboe io = req->async_data; 1200f9ead18cSJens Axboe } else { 1201f9ead18cSJens Axboe ret = move_addr_to_kernel(connect->addr, 1202f9ead18cSJens Axboe connect->addr_len, 1203f9ead18cSJens Axboe &__io.address); 1204f9ead18cSJens Axboe if (ret) 1205f9ead18cSJens Axboe goto out; 1206f9ead18cSJens Axboe io = &__io; 1207f9ead18cSJens Axboe } 1208f9ead18cSJens Axboe 1209f9ead18cSJens Axboe file_flags = force_nonblock ? O_NONBLOCK : 0; 1210f9ead18cSJens Axboe 1211f9ead18cSJens Axboe ret = __sys_connect_file(req->file, &io->address, 1212f9ead18cSJens Axboe connect->addr_len, file_flags); 1213f9ead18cSJens Axboe if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { 1214f9ead18cSJens Axboe if (req_has_async_data(req)) 1215f9ead18cSJens Axboe return -EAGAIN; 1216f9ead18cSJens Axboe if (io_alloc_async_data(req)) { 1217f9ead18cSJens Axboe ret = -ENOMEM; 1218f9ead18cSJens Axboe goto out; 1219f9ead18cSJens Axboe } 1220f9ead18cSJens Axboe memcpy(req->async_data, &__io, sizeof(__io)); 1221f9ead18cSJens Axboe return -EAGAIN; 1222f9ead18cSJens Axboe } 1223f9ead18cSJens Axboe if (ret == -ERESTARTSYS) 1224f9ead18cSJens Axboe ret = -EINTR; 1225f9ead18cSJens Axboe out: 1226f9ead18cSJens Axboe if (ret < 0) 1227f9ead18cSJens Axboe req_set_fail(req); 1228f9ead18cSJens Axboe io_req_set_res(req, ret, 0); 1229f9ead18cSJens Axboe return IOU_OK; 1230f9ead18cSJens Axboe } 123143e0bbbdSJens Axboe 123243e0bbbdSJens Axboe void io_netmsg_cache_free(struct io_cache_entry *entry) 123343e0bbbdSJens Axboe { 123443e0bbbdSJens Axboe kfree(container_of(entry, struct io_async_msghdr, cache)); 123543e0bbbdSJens Axboe } 1236f9ead18cSJens Axboe #endif 1237