1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/compat.h> 8 #include <net/compat.h> 9 #include <linux/io_uring.h> 10 11 #include <uapi/linux/io_uring.h> 12 13 #include "io_uring.h" 14 #include "kbuf.h" 15 #include "alloc_cache.h" 16 #include "net.h" 17 #include "notif.h" 18 #include "rsrc.h" 19 20 #if defined(CONFIG_NET) 21 struct io_shutdown { 22 struct file *file; 23 int how; 24 }; 25 26 struct io_accept { 27 struct file *file; 28 struct sockaddr __user *addr; 29 int __user *addr_len; 30 int flags; 31 u32 file_slot; 32 unsigned long nofile; 33 }; 34 35 struct io_socket { 36 struct file *file; 37 int domain; 38 int type; 39 int protocol; 40 int flags; 41 u32 file_slot; 42 unsigned long nofile; 43 }; 44 45 struct io_connect { 46 struct file *file; 47 struct sockaddr __user *addr; 48 int addr_len; 49 bool in_progress; 50 bool seen_econnaborted; 51 }; 52 53 struct io_sr_msg { 54 struct file *file; 55 union { 56 struct compat_msghdr __user *umsg_compat; 57 struct user_msghdr __user *umsg; 58 void __user *buf; 59 }; 60 unsigned len; 61 unsigned done_io; 62 unsigned msg_flags; 63 u16 flags; 64 /* initialised and used only by !msg send variants */ 65 u16 addr_len; 66 u16 buf_group; 67 void __user *addr; 68 void __user *msg_control; 69 /* used only for send zerocopy */ 70 struct io_kiocb *notif; 71 }; 72 73 static inline bool io_check_multishot(struct io_kiocb *req, 74 unsigned int issue_flags) 75 { 76 /* 77 * When ->locked_cq is set we only allow to post CQEs from the original 78 * task context. Usual request completions will be handled in other 79 * generic paths but multipoll may decide to post extra cqes. 80 */ 81 return !(issue_flags & IO_URING_F_IOWQ) || 82 !(issue_flags & IO_URING_F_MULTISHOT) || 83 !req->ctx->task_complete; 84 } 85 86 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 87 { 88 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 89 90 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 91 sqe->buf_index || sqe->splice_fd_in)) 92 return -EINVAL; 93 94 shutdown->how = READ_ONCE(sqe->len); 95 req->flags |= REQ_F_FORCE_ASYNC; 96 return 0; 97 } 98 99 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 100 { 101 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 102 struct socket *sock; 103 int ret; 104 105 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); 106 107 sock = sock_from_file(req->file); 108 if (unlikely(!sock)) 109 return -ENOTSOCK; 110 111 ret = __sys_shutdown_sock(sock, shutdown->how); 112 io_req_set_res(req, ret, 0); 113 return IOU_OK; 114 } 115 116 static bool io_net_retry(struct socket *sock, int flags) 117 { 118 if (!(flags & MSG_WAITALL)) 119 return false; 120 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 121 } 122 123 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 124 { 125 struct io_async_msghdr *hdr = req->async_data; 126 127 if (!req_has_async_data(req) || issue_flags & IO_URING_F_UNLOCKED) 128 return; 129 130 /* Let normal cleanup path reap it if we fail adding to the cache */ 131 if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) { 132 req->async_data = NULL; 133 req->flags &= ~REQ_F_ASYNC_DATA; 134 } 135 } 136 137 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req, 138 unsigned int issue_flags) 139 { 140 struct io_ring_ctx *ctx = req->ctx; 141 struct io_cache_entry *entry; 142 struct io_async_msghdr *hdr; 143 144 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 145 entry = io_alloc_cache_get(&ctx->netmsg_cache); 146 if (entry) { 147 hdr = container_of(entry, struct io_async_msghdr, cache); 148 hdr->free_iov = NULL; 149 req->flags |= REQ_F_ASYNC_DATA; 150 req->async_data = hdr; 151 return hdr; 152 } 153 } 154 155 if (!io_alloc_async_data(req)) { 156 hdr = req->async_data; 157 hdr->free_iov = NULL; 158 return hdr; 159 } 160 return NULL; 161 } 162 163 static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req) 164 { 165 /* ->prep_async is always called from the submission context */ 166 return io_msg_alloc_async(req, 0); 167 } 168 169 static int io_setup_async_msg(struct io_kiocb *req, 170 struct io_async_msghdr *kmsg, 171 unsigned int issue_flags) 172 { 173 struct io_async_msghdr *async_msg; 174 175 if (req_has_async_data(req)) 176 return -EAGAIN; 177 async_msg = io_msg_alloc_async(req, issue_flags); 178 if (!async_msg) { 179 kfree(kmsg->free_iov); 180 return -ENOMEM; 181 } 182 req->flags |= REQ_F_NEED_CLEANUP; 183 memcpy(async_msg, kmsg, sizeof(*kmsg)); 184 if (async_msg->msg.msg_name) 185 async_msg->msg.msg_name = &async_msg->addr; 186 /* if were using fast_iov, set it to the new one */ 187 if (iter_is_iovec(&kmsg->msg.msg_iter) && !kmsg->free_iov) { 188 size_t fast_idx = iter_iov(&kmsg->msg.msg_iter) - kmsg->fast_iov; 189 async_msg->msg.msg_iter.__iov = &async_msg->fast_iov[fast_idx]; 190 } 191 192 return -EAGAIN; 193 } 194 195 static int io_sendmsg_copy_hdr(struct io_kiocb *req, 196 struct io_async_msghdr *iomsg) 197 { 198 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 199 int ret; 200 201 iomsg->msg.msg_name = &iomsg->addr; 202 iomsg->free_iov = iomsg->fast_iov; 203 ret = sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 204 &iomsg->free_iov); 205 /* save msg_control as sys_sendmsg() overwrites it */ 206 sr->msg_control = iomsg->msg.msg_control_user; 207 return ret; 208 } 209 210 int io_send_prep_async(struct io_kiocb *req) 211 { 212 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 213 struct io_async_msghdr *io; 214 int ret; 215 216 if (!zc->addr || req_has_async_data(req)) 217 return 0; 218 io = io_msg_alloc_async_prep(req); 219 if (!io) 220 return -ENOMEM; 221 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); 222 return ret; 223 } 224 225 static int io_setup_async_addr(struct io_kiocb *req, 226 struct sockaddr_storage *addr_storage, 227 unsigned int issue_flags) 228 { 229 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 230 struct io_async_msghdr *io; 231 232 if (!sr->addr || req_has_async_data(req)) 233 return -EAGAIN; 234 io = io_msg_alloc_async(req, issue_flags); 235 if (!io) 236 return -ENOMEM; 237 memcpy(&io->addr, addr_storage, sizeof(io->addr)); 238 return -EAGAIN; 239 } 240 241 int io_sendmsg_prep_async(struct io_kiocb *req) 242 { 243 int ret; 244 245 if (!io_msg_alloc_async_prep(req)) 246 return -ENOMEM; 247 ret = io_sendmsg_copy_hdr(req, req->async_data); 248 if (!ret) 249 req->flags |= REQ_F_NEED_CLEANUP; 250 return ret; 251 } 252 253 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 254 { 255 struct io_async_msghdr *io = req->async_data; 256 257 kfree(io->free_iov); 258 } 259 260 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 261 { 262 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 263 264 if (req->opcode == IORING_OP_SEND) { 265 if (READ_ONCE(sqe->__pad3[0])) 266 return -EINVAL; 267 sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 268 sr->addr_len = READ_ONCE(sqe->addr_len); 269 } else if (sqe->addr2 || sqe->file_index) { 270 return -EINVAL; 271 } 272 273 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 274 sr->len = READ_ONCE(sqe->len); 275 sr->flags = READ_ONCE(sqe->ioprio); 276 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 277 return -EINVAL; 278 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 279 if (sr->msg_flags & MSG_DONTWAIT) 280 req->flags |= REQ_F_NOWAIT; 281 282 #ifdef CONFIG_COMPAT 283 if (req->ctx->compat) 284 sr->msg_flags |= MSG_CMSG_COMPAT; 285 #endif 286 sr->done_io = 0; 287 return 0; 288 } 289 290 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 291 { 292 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 293 struct io_async_msghdr iomsg, *kmsg; 294 struct socket *sock; 295 unsigned flags; 296 int min_ret = 0; 297 int ret; 298 299 sock = sock_from_file(req->file); 300 if (unlikely(!sock)) 301 return -ENOTSOCK; 302 303 if (req_has_async_data(req)) { 304 kmsg = req->async_data; 305 kmsg->msg.msg_control_user = sr->msg_control; 306 } else { 307 ret = io_sendmsg_copy_hdr(req, &iomsg); 308 if (ret) 309 return ret; 310 kmsg = &iomsg; 311 } 312 313 if (!(req->flags & REQ_F_POLLED) && 314 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 315 return io_setup_async_msg(req, kmsg, issue_flags); 316 317 flags = sr->msg_flags; 318 if (issue_flags & IO_URING_F_NONBLOCK) 319 flags |= MSG_DONTWAIT; 320 if (flags & MSG_WAITALL) 321 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 322 323 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 324 325 if (ret < min_ret) { 326 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 327 return io_setup_async_msg(req, kmsg, issue_flags); 328 if (ret > 0 && io_net_retry(sock, flags)) { 329 kmsg->msg.msg_controllen = 0; 330 kmsg->msg.msg_control = NULL; 331 sr->done_io += ret; 332 req->flags |= REQ_F_PARTIAL_IO; 333 return io_setup_async_msg(req, kmsg, issue_flags); 334 } 335 if (ret == -ERESTARTSYS) 336 ret = -EINTR; 337 req_set_fail(req); 338 } 339 /* fast path, check for non-NULL to avoid function call */ 340 if (kmsg->free_iov) 341 kfree(kmsg->free_iov); 342 req->flags &= ~REQ_F_NEED_CLEANUP; 343 io_netmsg_recycle(req, issue_flags); 344 if (ret >= 0) 345 ret += sr->done_io; 346 else if (sr->done_io) 347 ret = sr->done_io; 348 io_req_set_res(req, ret, 0); 349 return IOU_OK; 350 } 351 352 int io_send(struct io_kiocb *req, unsigned int issue_flags) 353 { 354 struct sockaddr_storage __address; 355 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 356 struct msghdr msg; 357 struct socket *sock; 358 unsigned flags; 359 int min_ret = 0; 360 int ret; 361 362 msg.msg_name = NULL; 363 msg.msg_control = NULL; 364 msg.msg_controllen = 0; 365 msg.msg_namelen = 0; 366 msg.msg_ubuf = NULL; 367 368 if (sr->addr) { 369 if (req_has_async_data(req)) { 370 struct io_async_msghdr *io = req->async_data; 371 372 msg.msg_name = &io->addr; 373 } else { 374 ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address); 375 if (unlikely(ret < 0)) 376 return ret; 377 msg.msg_name = (struct sockaddr *)&__address; 378 } 379 msg.msg_namelen = sr->addr_len; 380 } 381 382 if (!(req->flags & REQ_F_POLLED) && 383 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 384 return io_setup_async_addr(req, &__address, issue_flags); 385 386 sock = sock_from_file(req->file); 387 if (unlikely(!sock)) 388 return -ENOTSOCK; 389 390 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, &msg.msg_iter); 391 if (unlikely(ret)) 392 return ret; 393 394 flags = sr->msg_flags; 395 if (issue_flags & IO_URING_F_NONBLOCK) 396 flags |= MSG_DONTWAIT; 397 if (flags & MSG_WAITALL) 398 min_ret = iov_iter_count(&msg.msg_iter); 399 400 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 401 msg.msg_flags = flags; 402 ret = sock_sendmsg(sock, &msg); 403 if (ret < min_ret) { 404 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 405 return io_setup_async_addr(req, &__address, issue_flags); 406 407 if (ret > 0 && io_net_retry(sock, flags)) { 408 sr->len -= ret; 409 sr->buf += ret; 410 sr->done_io += ret; 411 req->flags |= REQ_F_PARTIAL_IO; 412 return io_setup_async_addr(req, &__address, issue_flags); 413 } 414 if (ret == -ERESTARTSYS) 415 ret = -EINTR; 416 req_set_fail(req); 417 } 418 if (ret >= 0) 419 ret += sr->done_io; 420 else if (sr->done_io) 421 ret = sr->done_io; 422 io_req_set_res(req, ret, 0); 423 return IOU_OK; 424 } 425 426 static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) 427 { 428 int hdr; 429 430 if (iomsg->namelen < 0) 431 return true; 432 if (check_add_overflow((int)sizeof(struct io_uring_recvmsg_out), 433 iomsg->namelen, &hdr)) 434 return true; 435 if (check_add_overflow(hdr, (int)iomsg->controllen, &hdr)) 436 return true; 437 438 return false; 439 } 440 441 static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 442 struct io_async_msghdr *iomsg) 443 { 444 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 445 struct user_msghdr msg; 446 int ret; 447 448 if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg))) 449 return -EFAULT; 450 451 ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 452 if (ret) 453 return ret; 454 455 if (req->flags & REQ_F_BUFFER_SELECT) { 456 if (msg.msg_iovlen == 0) { 457 sr->len = iomsg->fast_iov[0].iov_len = 0; 458 iomsg->fast_iov[0].iov_base = NULL; 459 iomsg->free_iov = NULL; 460 } else if (msg.msg_iovlen > 1) { 461 return -EINVAL; 462 } else { 463 if (copy_from_user(iomsg->fast_iov, msg.msg_iov, sizeof(*msg.msg_iov))) 464 return -EFAULT; 465 sr->len = iomsg->fast_iov[0].iov_len; 466 iomsg->free_iov = NULL; 467 } 468 469 if (req->flags & REQ_F_APOLL_MULTISHOT) { 470 iomsg->namelen = msg.msg_namelen; 471 iomsg->controllen = msg.msg_controllen; 472 if (io_recvmsg_multishot_overflow(iomsg)) 473 return -EOVERFLOW; 474 } 475 } else { 476 iomsg->free_iov = iomsg->fast_iov; 477 ret = __import_iovec(ITER_DEST, msg.msg_iov, msg.msg_iovlen, UIO_FASTIOV, 478 &iomsg->free_iov, &iomsg->msg.msg_iter, 479 false); 480 if (ret > 0) 481 ret = 0; 482 } 483 484 return ret; 485 } 486 487 #ifdef CONFIG_COMPAT 488 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 489 struct io_async_msghdr *iomsg) 490 { 491 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 492 struct compat_msghdr msg; 493 struct compat_iovec __user *uiov; 494 int ret; 495 496 if (copy_from_user(&msg, sr->umsg_compat, sizeof(msg))) 497 return -EFAULT; 498 499 ret = __get_compat_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 500 if (ret) 501 return ret; 502 503 uiov = compat_ptr(msg.msg_iov); 504 if (req->flags & REQ_F_BUFFER_SELECT) { 505 compat_ssize_t clen; 506 507 iomsg->free_iov = NULL; 508 if (msg.msg_iovlen == 0) { 509 sr->len = 0; 510 } else if (msg.msg_iovlen > 1) { 511 return -EINVAL; 512 } else { 513 if (!access_ok(uiov, sizeof(*uiov))) 514 return -EFAULT; 515 if (__get_user(clen, &uiov->iov_len)) 516 return -EFAULT; 517 if (clen < 0) 518 return -EINVAL; 519 sr->len = clen; 520 } 521 522 if (req->flags & REQ_F_APOLL_MULTISHOT) { 523 iomsg->namelen = msg.msg_namelen; 524 iomsg->controllen = msg.msg_controllen; 525 if (io_recvmsg_multishot_overflow(iomsg)) 526 return -EOVERFLOW; 527 } 528 } else { 529 iomsg->free_iov = iomsg->fast_iov; 530 ret = __import_iovec(ITER_DEST, (struct iovec __user *)uiov, msg.msg_iovlen, 531 UIO_FASTIOV, &iomsg->free_iov, 532 &iomsg->msg.msg_iter, true); 533 if (ret < 0) 534 return ret; 535 } 536 537 return 0; 538 } 539 #endif 540 541 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 542 struct io_async_msghdr *iomsg) 543 { 544 iomsg->msg.msg_name = &iomsg->addr; 545 546 #ifdef CONFIG_COMPAT 547 if (req->ctx->compat) 548 return __io_compat_recvmsg_copy_hdr(req, iomsg); 549 #endif 550 551 return __io_recvmsg_copy_hdr(req, iomsg); 552 } 553 554 int io_recvmsg_prep_async(struct io_kiocb *req) 555 { 556 int ret; 557 558 if (!io_msg_alloc_async_prep(req)) 559 return -ENOMEM; 560 ret = io_recvmsg_copy_hdr(req, req->async_data); 561 if (!ret) 562 req->flags |= REQ_F_NEED_CLEANUP; 563 return ret; 564 } 565 566 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT) 567 568 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 569 { 570 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 571 572 if (unlikely(sqe->file_index || sqe->addr2)) 573 return -EINVAL; 574 575 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 576 sr->len = READ_ONCE(sqe->len); 577 sr->flags = READ_ONCE(sqe->ioprio); 578 if (sr->flags & ~(RECVMSG_FLAGS)) 579 return -EINVAL; 580 sr->msg_flags = READ_ONCE(sqe->msg_flags); 581 if (sr->msg_flags & MSG_DONTWAIT) 582 req->flags |= REQ_F_NOWAIT; 583 if (sr->msg_flags & MSG_ERRQUEUE) 584 req->flags |= REQ_F_CLEAR_POLLIN; 585 if (sr->flags & IORING_RECV_MULTISHOT) { 586 if (!(req->flags & REQ_F_BUFFER_SELECT)) 587 return -EINVAL; 588 if (sr->msg_flags & MSG_WAITALL) 589 return -EINVAL; 590 if (req->opcode == IORING_OP_RECV && sr->len) 591 return -EINVAL; 592 req->flags |= REQ_F_APOLL_MULTISHOT; 593 /* 594 * Store the buffer group for this multishot receive separately, 595 * as if we end up doing an io-wq based issue that selects a 596 * buffer, it has to be committed immediately and that will 597 * clear ->buf_list. This means we lose the link to the buffer 598 * list, and the eventual buffer put on completion then cannot 599 * restore it. 600 */ 601 sr->buf_group = req->buf_index; 602 } 603 604 #ifdef CONFIG_COMPAT 605 if (req->ctx->compat) 606 sr->msg_flags |= MSG_CMSG_COMPAT; 607 #endif 608 sr->done_io = 0; 609 return 0; 610 } 611 612 static inline void io_recv_prep_retry(struct io_kiocb *req) 613 { 614 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 615 616 sr->done_io = 0; 617 sr->len = 0; /* get from the provided buffer */ 618 req->buf_index = sr->buf_group; 619 } 620 621 /* 622 * Finishes io_recv and io_recvmsg. 623 * 624 * Returns true if it is actually finished, or false if it should run 625 * again (for multishot). 626 */ 627 static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 628 unsigned int cflags, bool mshot_finished, 629 unsigned issue_flags) 630 { 631 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 632 io_req_set_res(req, *ret, cflags); 633 *ret = IOU_OK; 634 return true; 635 } 636 637 if (!mshot_finished) { 638 if (io_aux_cqe(req->ctx, issue_flags & IO_URING_F_COMPLETE_DEFER, 639 req->cqe.user_data, *ret, cflags | IORING_CQE_F_MORE, true)) { 640 io_recv_prep_retry(req); 641 return false; 642 } 643 /* Otherwise stop multishot but use the current result. */ 644 } 645 646 io_req_set_res(req, *ret, cflags); 647 648 if (issue_flags & IO_URING_F_MULTISHOT) 649 *ret = IOU_STOP_MULTISHOT; 650 else 651 *ret = IOU_OK; 652 return true; 653 } 654 655 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 656 struct io_sr_msg *sr, void __user **buf, 657 size_t *len) 658 { 659 unsigned long ubuf = (unsigned long) *buf; 660 unsigned long hdr; 661 662 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 663 kmsg->controllen; 664 if (*len < hdr) 665 return -EFAULT; 666 667 if (kmsg->controllen) { 668 unsigned long control = ubuf + hdr - kmsg->controllen; 669 670 kmsg->msg.msg_control_user = (void __user *) control; 671 kmsg->msg.msg_controllen = kmsg->controllen; 672 } 673 674 sr->buf = *buf; /* stash for later copy */ 675 *buf = (void __user *) (ubuf + hdr); 676 kmsg->payloadlen = *len = *len - hdr; 677 return 0; 678 } 679 680 struct io_recvmsg_multishot_hdr { 681 struct io_uring_recvmsg_out msg; 682 struct sockaddr_storage addr; 683 }; 684 685 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 686 struct io_async_msghdr *kmsg, 687 unsigned int flags, bool *finished) 688 { 689 int err; 690 int copy_len; 691 struct io_recvmsg_multishot_hdr hdr; 692 693 if (kmsg->namelen) 694 kmsg->msg.msg_name = &hdr.addr; 695 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 696 kmsg->msg.msg_namelen = 0; 697 698 if (sock->file->f_flags & O_NONBLOCK) 699 flags |= MSG_DONTWAIT; 700 701 err = sock_recvmsg(sock, &kmsg->msg, flags); 702 *finished = err <= 0; 703 if (err < 0) 704 return err; 705 706 hdr.msg = (struct io_uring_recvmsg_out) { 707 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 708 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 709 }; 710 711 hdr.msg.payloadlen = err; 712 if (err > kmsg->payloadlen) 713 err = kmsg->payloadlen; 714 715 copy_len = sizeof(struct io_uring_recvmsg_out); 716 if (kmsg->msg.msg_namelen > kmsg->namelen) 717 copy_len += kmsg->namelen; 718 else 719 copy_len += kmsg->msg.msg_namelen; 720 721 /* 722 * "fromlen shall refer to the value before truncation.." 723 * 1003.1g 724 */ 725 hdr.msg.namelen = kmsg->msg.msg_namelen; 726 727 /* ensure that there is no gap between hdr and sockaddr_storage */ 728 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 729 sizeof(struct io_uring_recvmsg_out)); 730 if (copy_to_user(io->buf, &hdr, copy_len)) { 731 *finished = true; 732 return -EFAULT; 733 } 734 735 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 736 kmsg->controllen + err; 737 } 738 739 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 740 { 741 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 742 struct io_async_msghdr iomsg, *kmsg; 743 struct socket *sock; 744 unsigned int cflags; 745 unsigned flags; 746 int ret, min_ret = 0; 747 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 748 bool mshot_finished = true; 749 750 sock = sock_from_file(req->file); 751 if (unlikely(!sock)) 752 return -ENOTSOCK; 753 754 if (req_has_async_data(req)) { 755 kmsg = req->async_data; 756 } else { 757 ret = io_recvmsg_copy_hdr(req, &iomsg); 758 if (ret) 759 return ret; 760 kmsg = &iomsg; 761 } 762 763 if (!(req->flags & REQ_F_POLLED) && 764 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 765 return io_setup_async_msg(req, kmsg, issue_flags); 766 767 if (!io_check_multishot(req, issue_flags)) 768 return io_setup_async_msg(req, kmsg, issue_flags); 769 770 retry_multishot: 771 if (io_do_buffer_select(req)) { 772 void __user *buf; 773 size_t len = sr->len; 774 775 buf = io_buffer_select(req, &len, issue_flags); 776 if (!buf) 777 return -ENOBUFS; 778 779 if (req->flags & REQ_F_APOLL_MULTISHOT) { 780 ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); 781 if (ret) { 782 io_kbuf_recycle(req, issue_flags); 783 return ret; 784 } 785 } 786 787 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len); 788 } 789 790 flags = sr->msg_flags; 791 if (force_nonblock) 792 flags |= MSG_DONTWAIT; 793 794 kmsg->msg.msg_get_inq = 1; 795 if (req->flags & REQ_F_APOLL_MULTISHOT) { 796 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 797 &mshot_finished); 798 } else { 799 /* disable partial retry for recvmsg with cmsg attached */ 800 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) 801 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 802 803 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 804 kmsg->uaddr, flags); 805 } 806 807 if (ret < min_ret) { 808 if (ret == -EAGAIN && force_nonblock) { 809 ret = io_setup_async_msg(req, kmsg, issue_flags); 810 if (ret == -EAGAIN && (issue_flags & IO_URING_F_MULTISHOT)) { 811 io_kbuf_recycle(req, issue_flags); 812 return IOU_ISSUE_SKIP_COMPLETE; 813 } 814 return ret; 815 } 816 if (ret > 0 && io_net_retry(sock, flags)) { 817 sr->done_io += ret; 818 req->flags |= REQ_F_PARTIAL_IO; 819 return io_setup_async_msg(req, kmsg, issue_flags); 820 } 821 if (ret == -ERESTARTSYS) 822 ret = -EINTR; 823 req_set_fail(req); 824 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 825 req_set_fail(req); 826 } 827 828 if (ret > 0) 829 ret += sr->done_io; 830 else if (sr->done_io) 831 ret = sr->done_io; 832 else 833 io_kbuf_recycle(req, issue_flags); 834 835 cflags = io_put_kbuf(req, issue_flags); 836 if (kmsg->msg.msg_inq) 837 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 838 839 if (!io_recv_finish(req, &ret, cflags, mshot_finished, issue_flags)) 840 goto retry_multishot; 841 842 if (mshot_finished) { 843 /* fast path, check for non-NULL to avoid function call */ 844 if (kmsg->free_iov) 845 kfree(kmsg->free_iov); 846 io_netmsg_recycle(req, issue_flags); 847 req->flags &= ~REQ_F_NEED_CLEANUP; 848 } 849 850 return ret; 851 } 852 853 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 854 { 855 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 856 struct msghdr msg; 857 struct socket *sock; 858 unsigned int cflags; 859 unsigned flags; 860 int ret, min_ret = 0; 861 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 862 size_t len = sr->len; 863 864 if (!(req->flags & REQ_F_POLLED) && 865 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 866 return -EAGAIN; 867 868 if (!io_check_multishot(req, issue_flags)) 869 return -EAGAIN; 870 871 sock = sock_from_file(req->file); 872 if (unlikely(!sock)) 873 return -ENOTSOCK; 874 875 retry_multishot: 876 if (io_do_buffer_select(req)) { 877 void __user *buf; 878 879 buf = io_buffer_select(req, &len, issue_flags); 880 if (!buf) 881 return -ENOBUFS; 882 sr->buf = buf; 883 } 884 885 ret = import_ubuf(ITER_DEST, sr->buf, len, &msg.msg_iter); 886 if (unlikely(ret)) 887 goto out_free; 888 889 msg.msg_name = NULL; 890 msg.msg_namelen = 0; 891 msg.msg_control = NULL; 892 msg.msg_get_inq = 1; 893 msg.msg_flags = 0; 894 msg.msg_controllen = 0; 895 msg.msg_iocb = NULL; 896 msg.msg_ubuf = NULL; 897 898 flags = sr->msg_flags; 899 if (force_nonblock) 900 flags |= MSG_DONTWAIT; 901 if (flags & MSG_WAITALL) 902 min_ret = iov_iter_count(&msg.msg_iter); 903 904 ret = sock_recvmsg(sock, &msg, flags); 905 if (ret < min_ret) { 906 if (ret == -EAGAIN && force_nonblock) { 907 if (issue_flags & IO_URING_F_MULTISHOT) { 908 io_kbuf_recycle(req, issue_flags); 909 return IOU_ISSUE_SKIP_COMPLETE; 910 } 911 912 return -EAGAIN; 913 } 914 if (ret > 0 && io_net_retry(sock, flags)) { 915 sr->len -= ret; 916 sr->buf += ret; 917 sr->done_io += ret; 918 req->flags |= REQ_F_PARTIAL_IO; 919 return -EAGAIN; 920 } 921 if (ret == -ERESTARTSYS) 922 ret = -EINTR; 923 req_set_fail(req); 924 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 925 out_free: 926 req_set_fail(req); 927 } 928 929 if (ret > 0) 930 ret += sr->done_io; 931 else if (sr->done_io) 932 ret = sr->done_io; 933 else 934 io_kbuf_recycle(req, issue_flags); 935 936 cflags = io_put_kbuf(req, issue_flags); 937 if (msg.msg_inq) 938 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 939 940 if (!io_recv_finish(req, &ret, cflags, ret <= 0, issue_flags)) 941 goto retry_multishot; 942 943 return ret; 944 } 945 946 void io_send_zc_cleanup(struct io_kiocb *req) 947 { 948 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 949 struct io_async_msghdr *io; 950 951 if (req_has_async_data(req)) { 952 io = req->async_data; 953 /* might be ->fast_iov if *msg_copy_hdr failed */ 954 if (io->free_iov != io->fast_iov) 955 kfree(io->free_iov); 956 } 957 if (zc->notif) { 958 io_notif_flush(zc->notif); 959 zc->notif = NULL; 960 } 961 } 962 963 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) 964 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE) 965 966 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 967 { 968 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 969 struct io_ring_ctx *ctx = req->ctx; 970 struct io_kiocb *notif; 971 972 if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))) 973 return -EINVAL; 974 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 975 if (req->flags & REQ_F_CQE_SKIP) 976 return -EINVAL; 977 978 notif = zc->notif = io_alloc_notif(ctx); 979 if (!notif) 980 return -ENOMEM; 981 notif->cqe.user_data = req->cqe.user_data; 982 notif->cqe.res = 0; 983 notif->cqe.flags = IORING_CQE_F_NOTIF; 984 req->flags |= REQ_F_NEED_CLEANUP; 985 986 zc->flags = READ_ONCE(sqe->ioprio); 987 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) { 988 if (zc->flags & ~IO_ZC_FLAGS_VALID) 989 return -EINVAL; 990 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) { 991 io_notif_set_extended(notif); 992 io_notif_to_data(notif)->zc_report = true; 993 } 994 } 995 996 if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 997 unsigned idx = READ_ONCE(sqe->buf_index); 998 999 if (unlikely(idx >= ctx->nr_user_bufs)) 1000 return -EFAULT; 1001 idx = array_index_nospec(idx, ctx->nr_user_bufs); 1002 req->imu = READ_ONCE(ctx->user_bufs[idx]); 1003 io_req_set_rsrc_node(notif, ctx, 0); 1004 } 1005 1006 if (req->opcode == IORING_OP_SEND_ZC) { 1007 if (READ_ONCE(sqe->__pad3[0])) 1008 return -EINVAL; 1009 zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1010 zc->addr_len = READ_ONCE(sqe->addr_len); 1011 } else { 1012 if (unlikely(sqe->addr2 || sqe->file_index)) 1013 return -EINVAL; 1014 if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) 1015 return -EINVAL; 1016 } 1017 1018 zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1019 zc->len = READ_ONCE(sqe->len); 1020 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 1021 if (zc->msg_flags & MSG_DONTWAIT) 1022 req->flags |= REQ_F_NOWAIT; 1023 1024 zc->done_io = 0; 1025 1026 #ifdef CONFIG_COMPAT 1027 if (req->ctx->compat) 1028 zc->msg_flags |= MSG_CMSG_COMPAT; 1029 #endif 1030 return 0; 1031 } 1032 1033 static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb, 1034 struct iov_iter *from, size_t length) 1035 { 1036 skb_zcopy_downgrade_managed(skb); 1037 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 1038 } 1039 1040 static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, 1041 struct iov_iter *from, size_t length) 1042 { 1043 struct skb_shared_info *shinfo = skb_shinfo(skb); 1044 int frag = shinfo->nr_frags; 1045 int ret = 0; 1046 struct bvec_iter bi; 1047 ssize_t copied = 0; 1048 unsigned long truesize = 0; 1049 1050 if (!frag) 1051 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1052 else if (unlikely(!skb_zcopy_managed(skb))) 1053 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 1054 1055 bi.bi_size = min(from->count, length); 1056 bi.bi_bvec_done = from->iov_offset; 1057 bi.bi_idx = 0; 1058 1059 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1060 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1061 1062 copied += v.bv_len; 1063 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1064 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1065 v.bv_offset, v.bv_len); 1066 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1067 } 1068 if (bi.bi_size) 1069 ret = -EMSGSIZE; 1070 1071 shinfo->nr_frags = frag; 1072 from->bvec += bi.bi_idx; 1073 from->nr_segs -= bi.bi_idx; 1074 from->count -= copied; 1075 from->iov_offset = bi.bi_bvec_done; 1076 1077 skb->data_len += copied; 1078 skb->len += copied; 1079 skb->truesize += truesize; 1080 1081 if (sk && sk->sk_type == SOCK_STREAM) { 1082 sk_wmem_queued_add(sk, truesize); 1083 if (!skb_zcopy_pure(skb)) 1084 sk_mem_charge(sk, truesize); 1085 } else { 1086 refcount_add(truesize, &skb->sk->sk_wmem_alloc); 1087 } 1088 return ret; 1089 } 1090 1091 int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) 1092 { 1093 struct sockaddr_storage __address; 1094 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1095 struct msghdr msg; 1096 struct socket *sock; 1097 unsigned msg_flags; 1098 int ret, min_ret = 0; 1099 1100 sock = sock_from_file(req->file); 1101 if (unlikely(!sock)) 1102 return -ENOTSOCK; 1103 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1104 return -EOPNOTSUPP; 1105 1106 msg.msg_name = NULL; 1107 msg.msg_control = NULL; 1108 msg.msg_controllen = 0; 1109 msg.msg_namelen = 0; 1110 1111 if (zc->addr) { 1112 if (req_has_async_data(req)) { 1113 struct io_async_msghdr *io = req->async_data; 1114 1115 msg.msg_name = &io->addr; 1116 } else { 1117 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); 1118 if (unlikely(ret < 0)) 1119 return ret; 1120 msg.msg_name = (struct sockaddr *)&__address; 1121 } 1122 msg.msg_namelen = zc->addr_len; 1123 } 1124 1125 if (!(req->flags & REQ_F_POLLED) && 1126 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1127 return io_setup_async_addr(req, &__address, issue_flags); 1128 1129 if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 1130 ret = io_import_fixed(ITER_SOURCE, &msg.msg_iter, req->imu, 1131 (u64)(uintptr_t)zc->buf, zc->len); 1132 if (unlikely(ret)) 1133 return ret; 1134 msg.sg_from_iter = io_sg_from_iter; 1135 } else { 1136 io_notif_set_extended(zc->notif); 1137 ret = import_ubuf(ITER_SOURCE, zc->buf, zc->len, &msg.msg_iter); 1138 if (unlikely(ret)) 1139 return ret; 1140 ret = io_notif_account_mem(zc->notif, zc->len); 1141 if (unlikely(ret)) 1142 return ret; 1143 msg.sg_from_iter = io_sg_from_iter_iovec; 1144 } 1145 1146 msg_flags = zc->msg_flags | MSG_ZEROCOPY; 1147 if (issue_flags & IO_URING_F_NONBLOCK) 1148 msg_flags |= MSG_DONTWAIT; 1149 if (msg_flags & MSG_WAITALL) 1150 min_ret = iov_iter_count(&msg.msg_iter); 1151 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 1152 1153 msg.msg_flags = msg_flags; 1154 msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; 1155 ret = sock_sendmsg(sock, &msg); 1156 1157 if (unlikely(ret < min_ret)) { 1158 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1159 return io_setup_async_addr(req, &__address, issue_flags); 1160 1161 if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { 1162 zc->len -= ret; 1163 zc->buf += ret; 1164 zc->done_io += ret; 1165 req->flags |= REQ_F_PARTIAL_IO; 1166 return io_setup_async_addr(req, &__address, issue_flags); 1167 } 1168 if (ret == -ERESTARTSYS) 1169 ret = -EINTR; 1170 req_set_fail(req); 1171 } 1172 1173 if (ret >= 0) 1174 ret += zc->done_io; 1175 else if (zc->done_io) 1176 ret = zc->done_io; 1177 1178 /* 1179 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1180 * flushing notif to io_send_zc_cleanup() 1181 */ 1182 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1183 io_notif_flush(zc->notif); 1184 req->flags &= ~REQ_F_NEED_CLEANUP; 1185 } 1186 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1187 return IOU_OK; 1188 } 1189 1190 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1191 { 1192 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1193 struct io_async_msghdr iomsg, *kmsg; 1194 struct socket *sock; 1195 unsigned flags; 1196 int ret, min_ret = 0; 1197 1198 io_notif_set_extended(sr->notif); 1199 1200 sock = sock_from_file(req->file); 1201 if (unlikely(!sock)) 1202 return -ENOTSOCK; 1203 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1204 return -EOPNOTSUPP; 1205 1206 if (req_has_async_data(req)) { 1207 kmsg = req->async_data; 1208 } else { 1209 ret = io_sendmsg_copy_hdr(req, &iomsg); 1210 if (ret) 1211 return ret; 1212 kmsg = &iomsg; 1213 } 1214 1215 if (!(req->flags & REQ_F_POLLED) && 1216 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1217 return io_setup_async_msg(req, kmsg, issue_flags); 1218 1219 flags = sr->msg_flags | MSG_ZEROCOPY; 1220 if (issue_flags & IO_URING_F_NONBLOCK) 1221 flags |= MSG_DONTWAIT; 1222 if (flags & MSG_WAITALL) 1223 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1224 1225 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1226 kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1227 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 1228 1229 if (unlikely(ret < min_ret)) { 1230 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1231 return io_setup_async_msg(req, kmsg, issue_flags); 1232 1233 if (ret > 0 && io_net_retry(sock, flags)) { 1234 sr->done_io += ret; 1235 req->flags |= REQ_F_PARTIAL_IO; 1236 return io_setup_async_msg(req, kmsg, issue_flags); 1237 } 1238 if (ret == -ERESTARTSYS) 1239 ret = -EINTR; 1240 req_set_fail(req); 1241 } 1242 /* fast path, check for non-NULL to avoid function call */ 1243 if (kmsg->free_iov) { 1244 kfree(kmsg->free_iov); 1245 kmsg->free_iov = NULL; 1246 } 1247 1248 io_netmsg_recycle(req, issue_flags); 1249 if (ret >= 0) 1250 ret += sr->done_io; 1251 else if (sr->done_io) 1252 ret = sr->done_io; 1253 1254 /* 1255 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1256 * flushing notif to io_send_zc_cleanup() 1257 */ 1258 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1259 io_notif_flush(sr->notif); 1260 req->flags &= ~REQ_F_NEED_CLEANUP; 1261 } 1262 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1263 return IOU_OK; 1264 } 1265 1266 void io_sendrecv_fail(struct io_kiocb *req) 1267 { 1268 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1269 1270 if (req->flags & REQ_F_PARTIAL_IO) 1271 req->cqe.res = sr->done_io; 1272 1273 if ((req->flags & REQ_F_NEED_CLEANUP) && 1274 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1275 req->cqe.flags |= IORING_CQE_F_MORE; 1276 } 1277 1278 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1279 { 1280 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1281 unsigned flags; 1282 1283 if (sqe->len || sqe->buf_index) 1284 return -EINVAL; 1285 1286 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1287 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1288 accept->flags = READ_ONCE(sqe->accept_flags); 1289 accept->nofile = rlimit(RLIMIT_NOFILE); 1290 flags = READ_ONCE(sqe->ioprio); 1291 if (flags & ~IORING_ACCEPT_MULTISHOT) 1292 return -EINVAL; 1293 1294 accept->file_slot = READ_ONCE(sqe->file_index); 1295 if (accept->file_slot) { 1296 if (accept->flags & SOCK_CLOEXEC) 1297 return -EINVAL; 1298 if (flags & IORING_ACCEPT_MULTISHOT && 1299 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1300 return -EINVAL; 1301 } 1302 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1303 return -EINVAL; 1304 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1305 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1306 if (flags & IORING_ACCEPT_MULTISHOT) 1307 req->flags |= REQ_F_APOLL_MULTISHOT; 1308 return 0; 1309 } 1310 1311 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1312 { 1313 struct io_ring_ctx *ctx = req->ctx; 1314 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1315 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1316 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 1317 bool fixed = !!accept->file_slot; 1318 struct file *file; 1319 int ret, fd; 1320 1321 if (!io_check_multishot(req, issue_flags)) 1322 return -EAGAIN; 1323 retry: 1324 if (!fixed) { 1325 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1326 if (unlikely(fd < 0)) 1327 return fd; 1328 } 1329 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 1330 accept->flags); 1331 if (IS_ERR(file)) { 1332 if (!fixed) 1333 put_unused_fd(fd); 1334 ret = PTR_ERR(file); 1335 if (ret == -EAGAIN && force_nonblock) { 1336 /* 1337 * if it's multishot and polled, we don't need to 1338 * return EAGAIN to arm the poll infra since it 1339 * has already been done 1340 */ 1341 if (issue_flags & IO_URING_F_MULTISHOT) 1342 ret = IOU_ISSUE_SKIP_COMPLETE; 1343 return ret; 1344 } 1345 if (ret == -ERESTARTSYS) 1346 ret = -EINTR; 1347 req_set_fail(req); 1348 } else if (!fixed) { 1349 fd_install(fd, file); 1350 ret = fd; 1351 } else { 1352 ret = io_fixed_fd_install(req, issue_flags, file, 1353 accept->file_slot); 1354 } 1355 1356 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 1357 io_req_set_res(req, ret, 0); 1358 return IOU_OK; 1359 } 1360 1361 if (ret < 0) 1362 return ret; 1363 if (io_aux_cqe(ctx, issue_flags & IO_URING_F_COMPLETE_DEFER, 1364 req->cqe.user_data, ret, IORING_CQE_F_MORE, true)) 1365 goto retry; 1366 1367 return -ECANCELED; 1368 } 1369 1370 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1371 { 1372 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1373 1374 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1375 return -EINVAL; 1376 1377 sock->domain = READ_ONCE(sqe->fd); 1378 sock->type = READ_ONCE(sqe->off); 1379 sock->protocol = READ_ONCE(sqe->len); 1380 sock->file_slot = READ_ONCE(sqe->file_index); 1381 sock->nofile = rlimit(RLIMIT_NOFILE); 1382 1383 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1384 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1385 return -EINVAL; 1386 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1387 return -EINVAL; 1388 return 0; 1389 } 1390 1391 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1392 { 1393 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1394 bool fixed = !!sock->file_slot; 1395 struct file *file; 1396 int ret, fd; 1397 1398 if (!fixed) { 1399 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1400 if (unlikely(fd < 0)) 1401 return fd; 1402 } 1403 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1404 if (IS_ERR(file)) { 1405 if (!fixed) 1406 put_unused_fd(fd); 1407 ret = PTR_ERR(file); 1408 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1409 return -EAGAIN; 1410 if (ret == -ERESTARTSYS) 1411 ret = -EINTR; 1412 req_set_fail(req); 1413 } else if (!fixed) { 1414 fd_install(fd, file); 1415 ret = fd; 1416 } else { 1417 ret = io_fixed_fd_install(req, issue_flags, file, 1418 sock->file_slot); 1419 } 1420 io_req_set_res(req, ret, 0); 1421 return IOU_OK; 1422 } 1423 1424 int io_connect_prep_async(struct io_kiocb *req) 1425 { 1426 struct io_async_connect *io = req->async_data; 1427 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1428 1429 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 1430 } 1431 1432 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1433 { 1434 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1435 1436 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1437 return -EINVAL; 1438 1439 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1440 conn->addr_len = READ_ONCE(sqe->addr2); 1441 conn->in_progress = conn->seen_econnaborted = false; 1442 return 0; 1443 } 1444 1445 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1446 { 1447 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1448 struct io_async_connect __io, *io; 1449 unsigned file_flags; 1450 int ret; 1451 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1452 1453 if (connect->in_progress) { 1454 struct socket *socket; 1455 1456 ret = -ENOTSOCK; 1457 socket = sock_from_file(req->file); 1458 if (socket) 1459 ret = sock_error(socket->sk); 1460 goto out; 1461 } 1462 1463 if (req_has_async_data(req)) { 1464 io = req->async_data; 1465 } else { 1466 ret = move_addr_to_kernel(connect->addr, 1467 connect->addr_len, 1468 &__io.address); 1469 if (ret) 1470 goto out; 1471 io = &__io; 1472 } 1473 1474 file_flags = force_nonblock ? O_NONBLOCK : 0; 1475 1476 ret = __sys_connect_file(req->file, &io->address, 1477 connect->addr_len, file_flags); 1478 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) 1479 && force_nonblock) { 1480 if (ret == -EINPROGRESS) { 1481 connect->in_progress = true; 1482 return -EAGAIN; 1483 } 1484 if (ret == -ECONNABORTED) { 1485 if (connect->seen_econnaborted) 1486 goto out; 1487 connect->seen_econnaborted = true; 1488 } 1489 if (req_has_async_data(req)) 1490 return -EAGAIN; 1491 if (io_alloc_async_data(req)) { 1492 ret = -ENOMEM; 1493 goto out; 1494 } 1495 memcpy(req->async_data, &__io, sizeof(__io)); 1496 return -EAGAIN; 1497 } 1498 if (ret == -ERESTARTSYS) 1499 ret = -EINTR; 1500 out: 1501 if (ret < 0) 1502 req_set_fail(req); 1503 io_req_set_res(req, ret, 0); 1504 return IOU_OK; 1505 } 1506 1507 void io_netmsg_cache_free(struct io_cache_entry *entry) 1508 { 1509 kfree(container_of(entry, struct io_async_msghdr, cache)); 1510 } 1511 #endif 1512