Lines Matching +full:supports +full:- +full:cqe
1 // SPDX-License-Identifier: GPL-2.0
6 #include <linux/blk-mq.h>
33 return req->flags & REQ_F_SUPPORT_NOWAIT; in io_file_supports_nowait()
42 uiov = u64_to_user_ptr(rw->addr); in io_iov_compat_buffer_select_prep()
44 return -EFAULT; in io_iov_compat_buffer_select_prep()
45 if (__get_user(clen, &uiov->iov_len)) in io_iov_compat_buffer_select_prep()
46 return -EFAULT; in io_iov_compat_buffer_select_prep()
48 return -EINVAL; in io_iov_compat_buffer_select_prep()
50 rw->len = clen; in io_iov_compat_buffer_select_prep()
61 if (rw->len != 1) in io_iov_buffer_select_prep()
62 return -EINVAL; in io_iov_buffer_select_prep()
65 if (req->ctx->compat) in io_iov_buffer_select_prep()
69 uiov = u64_to_user_ptr(rw->addr); in io_iov_buffer_select_prep()
71 return -EFAULT; in io_iov_buffer_select_prep()
72 rw->len = iov.iov_len; in io_iov_buffer_select_prep()
82 rw->kiocb.ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
83 /* used for fixed read/write too - just read unconditionally */ in io_prep_rw()
84 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
86 if (req->opcode == IORING_OP_READ_FIXED || in io_prep_rw()
87 req->opcode == IORING_OP_WRITE_FIXED) { in io_prep_rw()
88 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw()
91 if (unlikely(req->buf_index >= ctx->nr_user_bufs)) in io_prep_rw()
92 return -EFAULT; in io_prep_rw()
93 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); in io_prep_rw()
94 req->imu = ctx->user_bufs[index]; in io_prep_rw()
98 ioprio = READ_ONCE(sqe->ioprio); in io_prep_rw()
104 rw->kiocb.ki_ioprio = ioprio; in io_prep_rw()
106 rw->kiocb.ki_ioprio = get_current_ioprio(); in io_prep_rw()
108 rw->kiocb.dio_complete = NULL; in io_prep_rw()
110 rw->addr = READ_ONCE(sqe->addr); in io_prep_rw()
111 rw->len = READ_ONCE(sqe->len); in io_prep_rw()
112 rw->flags = READ_ONCE(sqe->rw_flags); in io_prep_rw()
114 /* Have to do this validation here, as this is in io_read() rw->len might in io_prep_rw()
117 if (req->opcode == IORING_OP_READV && req->flags & REQ_F_BUFFER_SELECT) { in io_prep_rw()
128 struct io_async_rw *io = req->async_data; in io_readv_writev_cleanup()
130 kfree(io->free_iovec); in io_readv_writev_cleanup()
136 case -EIOCBQUEUED: in io_rw_done()
138 case -ERESTARTSYS: in io_rw_done()
139 case -ERESTARTNOINTR: in io_rw_done()
140 case -ERESTARTNOHAND: in io_rw_done()
141 case -ERESTART_RESTARTBLOCK: in io_rw_done()
147 ret = -EINTR; in io_rw_done()
150 kiocb->ki_complete(kiocb, ret); in io_rw_done()
158 if (rw->kiocb.ki_pos != -1) in io_kiocb_update_pos()
159 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
161 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
162 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
163 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
164 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
167 rw->kiocb.ki_pos = 0; in io_kiocb_update_pos()
174 struct io_async_rw *io = req->async_data; in io_resubmit_prep()
178 iov_iter_restore(&io->s.iter, &io->s.iter_state); in io_resubmit_prep()
184 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
185 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
189 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
190 !(ctx->flags & IORING_SETUP_IOPOLL))) in io_rw_should_reissue()
195 * -EAGAIN. in io_rw_should_reissue()
197 if (percpu_ref_is_dying(&ctx->refs)) in io_rw_should_reissue()
200 * Play it safe and assume not safe to re-import and reissue if we're in io_rw_should_reissue()
203 if (!same_thread_group(req->task, current) || !in_task()) in io_rw_should_reissue()
220 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
223 kiocb_end_write(&rw->kiocb); in io_req_end_write()
235 if (rw->kiocb.ki_flags & IOCB_WRITE) { in io_req_io_end()
237 fsnotify_modify(req->file); in io_req_io_end()
239 fsnotify_access(req->file); in io_req_io_end()
245 if (unlikely(res != req->cqe.res)) { in __io_complete_rw_common()
246 if ((res == -EAGAIN || res == -EOPNOTSUPP) && in __io_complete_rw_common()
253 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in __io_complete_rw_common()
257 req->cqe.res = res; in __io_complete_rw_common()
264 struct io_async_rw *io = req->async_data; in io_fixup_rw_res()
267 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
269 res = io->bytes_done; in io_fixup_rw_res()
271 res += io->bytes_done; in io_fixup_rw_res()
279 struct kiocb *kiocb = &rw->kiocb; in io_req_rw_complete()
281 if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) { in io_req_rw_complete()
282 long res = kiocb->dio_complete(rw->kiocb.private); in io_req_rw_complete()
289 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { in io_req_rw_complete()
290 unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; in io_req_rw_complete()
292 req->cqe.flags |= io_put_kbuf(req, issue_flags); in io_req_rw_complete()
302 if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) { in io_complete_rw()
307 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
316 if (kiocb->ki_flags & IOCB_WRITE) in io_complete_rw_iopoll()
318 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
319 if (res == -EAGAIN && io_rw_should_reissue(req)) { in io_complete_rw_iopoll()
320 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in io_complete_rw_iopoll()
323 req->cqe.res = res; in io_complete_rw_iopoll()
326 /* order with io_iopoll_complete() checking ->iopoll_completed */ in io_complete_rw_iopoll()
327 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
336 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
337 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
338 if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { in kiocb_done()
350 io_rw_done(&rw->kiocb, ret); in kiocb_done()
353 if (req->flags & REQ_F_REISSUE) { in kiocb_done()
354 req->flags &= ~REQ_F_REISSUE; in kiocb_done()
356 return -EAGAIN; in kiocb_done()
368 struct iov_iter *iter = &s->iter; in __io_import_iovec()
369 u8 opcode = req->opcode; in __io_import_iovec()
376 ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len); in __io_import_iovec()
382 buf = u64_to_user_ptr(rw->addr); in __io_import_iovec()
383 sqe_len = rw->len; in __io_import_iovec()
386 (req->flags & REQ_F_BUFFER_SELECT)) { in __io_import_iovec()
390 return ERR_PTR(-ENOBUFS); in __io_import_iovec()
391 rw->addr = (unsigned long) buf; in __io_import_iovec()
392 rw->len = sqe_len; in __io_import_iovec()
401 iovec = s->fast_iov; in __io_import_iovec()
403 req->ctx->compat); in __io_import_iovec()
417 iov_iter_save_state(&s->iter, &s->iter_state); in io_import_iovec()
423 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; in io_kiocb_ppos()
427 * For files that don't have ->read_iter() and ->write_iter(), handle them
428 * by looping over ->read() or ->write() manually.
432 struct kiocb *kiocb = &rw->kiocb; in loop_rw_iter()
433 struct file *file = kiocb->ki_filp; in loop_rw_iter()
439 * support non-blocking either. For the latter, this just causes in loop_rw_iter()
442 if (kiocb->ki_flags & IOCB_HIPRI) in loop_rw_iter()
443 return -EOPNOTSUPP; in loop_rw_iter()
444 if ((kiocb->ki_flags & IOCB_NOWAIT) && in loop_rw_iter()
445 !(kiocb->ki_filp->f_flags & O_NONBLOCK)) in loop_rw_iter()
446 return -EAGAIN; in loop_rw_iter()
456 addr = iter->ubuf + iter->iov_offset; in loop_rw_iter()
462 addr = u64_to_user_ptr(rw->addr); in loop_rw_iter()
463 len = rw->len; in loop_rw_iter()
467 nr = file->f_op->read(file, addr, len, ppos); in loop_rw_iter()
469 nr = file->f_op->write(file, addr, len, ppos); in loop_rw_iter()
480 rw->addr += nr; in loop_rw_iter()
481 rw->len -= nr; in loop_rw_iter()
482 if (!rw->len) in loop_rw_iter()
495 struct io_async_rw *io = req->async_data; in io_req_map_rw()
497 memcpy(&io->s.iter, iter, sizeof(*iter)); in io_req_map_rw()
498 io->free_iovec = iovec; in io_req_map_rw()
499 io->bytes_done = 0; in io_req_map_rw()
506 io->s.iter.__iov = io->s.fast_iov; in io_req_map_rw()
507 if (iter->__iov != fast_iov) { in io_req_map_rw()
508 iov_off = iter_iov(iter) - fast_iov; in io_req_map_rw()
509 io->s.iter.__iov += iov_off; in io_req_map_rw()
511 if (io->s.fast_iov != fast_iov) in io_req_map_rw()
512 memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off, in io_req_map_rw()
513 sizeof(struct iovec) * iter->nr_segs); in io_req_map_rw()
515 req->flags |= REQ_F_NEED_CLEANUP; in io_req_map_rw()
522 if (!force && !io_cold_defs[req->opcode].prep_async) in io_setup_async_rw()
529 return -ENOMEM; in io_setup_async_rw()
532 io_req_map_rw(req, iovec, s->fast_iov, &s->iter); in io_setup_async_rw()
533 iorw = req->async_data; in io_setup_async_rw()
535 iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); in io_setup_async_rw()
542 struct io_async_rw *iorw = req->async_data; in io_rw_prep_async()
546 iorw->bytes_done = 0; in io_rw_prep_async()
547 iorw->free_iovec = NULL; in io_rw_prep_async()
549 /* submission path, ->uring_lock should already be taken */ in io_rw_prep_async()
550 ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); in io_rw_prep_async()
555 iorw->free_iovec = iov; in io_rw_prep_async()
556 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_prep_async()
586 struct io_kiocb *req = wait->private; in io_async_buf_func()
595 rw->kiocb.ki_flags &= ~IOCB_WAITQ; in io_async_buf_func()
596 list_del_init(&wait->entry); in io_async_buf_func()
615 struct io_async_rw *io = req->async_data; in io_rw_should_retry()
616 struct wait_page_queue *wait = &io->wpq; in io_rw_should_retry()
618 struct kiocb *kiocb = &rw->kiocb; in io_rw_should_retry()
620 /* never retry for NOWAIT, we just complete with -EAGAIN */ in io_rw_should_retry()
621 if (req->flags & REQ_F_NOWAIT) in io_rw_should_retry()
625 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) in io_rw_should_retry()
632 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) in io_rw_should_retry()
635 wait->wait.func = io_async_buf_func; in io_rw_should_retry()
636 wait->wait.private = req; in io_rw_should_retry()
637 wait->wait.flags = 0; in io_rw_should_retry()
638 INIT_LIST_HEAD(&wait->wait.entry); in io_rw_should_retry()
639 kiocb->ki_flags |= IOCB_WAITQ; in io_rw_should_retry()
640 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_rw_should_retry()
641 kiocb->ki_waitq = wait; in io_rw_should_retry()
647 struct file *file = rw->kiocb.ki_filp; in io_iter_do_read()
649 if (likely(file->f_op->read_iter)) in io_iter_do_read()
650 return call_read_iter(file, &rw->kiocb, iter); in io_iter_do_read()
651 else if (file->f_op->read) in io_iter_do_read()
654 return -EINVAL; in io_iter_do_read()
659 return req->flags & REQ_F_ISREG || in need_complete_io()
660 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
666 struct kiocb *kiocb = &rw->kiocb; in io_rw_init_file()
667 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
668 struct file *file = req->file; in io_rw_init_file()
671 if (unlikely(!file || !(file->f_mode & mode))) in io_rw_init_file()
672 return -EBADF; in io_rw_init_file()
674 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
675 req->flags |= io_file_get_flags(file); in io_rw_init_file()
677 kiocb->ki_flags = file->f_iocb_flags; in io_rw_init_file()
678 ret = kiocb_set_rw_flags(kiocb, rw->flags); in io_rw_init_file()
681 kiocb->ki_flags |= IOCB_ALLOC_CACHE; in io_rw_init_file()
685 * supports async. Otherwise it's impossible to use O_NONBLOCK files in io_rw_init_file()
688 if ((kiocb->ki_flags & IOCB_NOWAIT) || in io_rw_init_file()
689 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) in io_rw_init_file()
690 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
692 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_rw_init_file()
693 if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) in io_rw_init_file()
694 return -EOPNOTSUPP; in io_rw_init_file()
696 kiocb->private = NULL; in io_rw_init_file()
697 kiocb->ki_flags |= IOCB_HIPRI; in io_rw_init_file()
698 kiocb->ki_complete = io_complete_rw_iopoll; in io_rw_init_file()
699 req->iopoll_completed = 0; in io_rw_init_file()
701 if (kiocb->ki_flags & IOCB_HIPRI) in io_rw_init_file()
702 return -EINVAL; in io_rw_init_file()
703 kiocb->ki_complete = io_complete_rw; in io_rw_init_file()
714 struct kiocb *kiocb = &rw->kiocb; in __io_read()
725 io = req->async_data; in __io_read()
726 s = &io->s; in __io_read()
729 * Safe and required to re-import if we're using provided in __io_read()
743 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
751 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
757 return ret ?: -EAGAIN; in __io_read()
759 kiocb->ki_flags |= IOCB_NOWAIT; in __io_read()
761 /* Ensure we clear previously set non-block flag */ in __io_read()
762 kiocb->ki_flags &= ~IOCB_NOWAIT; in __io_read()
767 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in __io_read()
773 ret = io_iter_do_read(rw, &s->iter); in __io_read()
776 * Some file systems like to return -EOPNOTSUPP for an IOCB_NOWAIT in __io_read()
777 * issue, even though they should be returning -EAGAIN. To be safe, in __io_read()
780 if (ret == -EOPNOTSUPP && force_nonblock) in __io_read()
781 ret = -EAGAIN; in __io_read()
783 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { in __io_read()
784 req->flags &= ~REQ_F_REISSUE; in __io_read()
786 if (req->opcode == IORING_OP_READ && file_can_poll(req->file)) in __io_read()
787 return -EAGAIN; in __io_read()
788 /* IOPOLL retry should happen for io-wq threads */ in __io_read()
789 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in __io_read()
792 if (req->flags & REQ_F_NOWAIT) in __io_read()
795 } else if (ret == -EIOCBQUEUED) { in __io_read()
799 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in __io_read()
800 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) { in __io_read()
810 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
819 io = req->async_data; in __io_read()
820 s = &io->s; in __io_read()
832 iov_iter_advance(&s->iter, ret); in __io_read()
833 if (!iov_iter_count(&s->iter)) in __io_read()
835 io->bytes_done += ret; in __io_read()
836 iov_iter_save_state(&s->iter, &s->iter_state); in __io_read()
840 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
841 return -EAGAIN; in __io_read()
844 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
847 * we get -EIOCBQUEUED, then we'll get a notification when the in __io_read()
851 ret = io_iter_do_read(rw, &s->iter); in __io_read()
852 if (ret == -EIOCBQUEUED) in __io_read()
855 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
856 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
881 if (!(req->flags & REQ_F_ISREG)) in io_kiocb_start_write()
883 if (!(kiocb->ki_flags & IOCB_NOWAIT)) { in io_kiocb_start_write()
888 inode = file_inode(kiocb->ki_filp); in io_kiocb_start_write()
889 ret = sb_start_write_trylock(inode->i_sb); in io_kiocb_start_write()
891 __sb_writers_release(inode->i_sb, SB_FREEZE_WRITE); in io_kiocb_start_write()
900 struct kiocb *kiocb = &rw->kiocb; in io_write()
910 struct io_async_rw *io = req->async_data; in io_write()
912 s = &io->s; in io_write()
913 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
921 req->cqe.res = iov_iter_count(&s->iter); in io_write()
928 /* File path supports NOWAIT for non-direct_IO only for block devices. */ in io_write()
929 if (!(kiocb->ki_flags & IOCB_DIRECT) && in io_write()
930 !(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) && in io_write()
931 (req->flags & REQ_F_ISREG)) in io_write()
934 kiocb->ki_flags |= IOCB_NOWAIT; in io_write()
936 /* Ensure we clear previously set non-block flag */ in io_write()
937 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_write()
942 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
949 return -EAGAIN; in io_write()
950 kiocb->ki_flags |= IOCB_WRITE; in io_write()
952 if (likely(req->file->f_op->write_iter)) in io_write()
953 ret2 = call_write_iter(req->file, kiocb, &s->iter); in io_write()
954 else if (req->file->f_op->write) in io_write()
955 ret2 = loop_rw_iter(WRITE, rw, &s->iter); in io_write()
957 ret2 = -EINVAL; in io_write()
959 if (req->flags & REQ_F_REISSUE) { in io_write()
960 req->flags &= ~REQ_F_REISSUE; in io_write()
961 ret2 = -EAGAIN; in io_write()
965 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just in io_write()
968 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) in io_write()
969 ret2 = -EAGAIN; in io_write()
971 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
973 if (!force_nonblock || ret2 != -EAGAIN) { in io_write()
974 /* IOPOLL retry should happen for io-wq threads */ in io_write()
975 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
978 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
981 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
982 req->cqe.res, ret2); in io_write()
989 iov_iter_save_state(&s->iter, &s->iter_state); in io_write()
992 io = req->async_data; in io_write()
994 io->bytes_done += ret2; in io_write()
996 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
998 return ret ? ret : -EAGAIN; in io_write()
1004 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
1007 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
1009 return -EAGAIN; in io_write()
1023 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
1024 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1038 if (ctx->poll_multi_queue || force_nonspin) in io_do_iopoll()
1041 wq_list_for_each(pos, start, &ctx->iopoll_list) { in io_do_iopoll()
1043 struct file *file = req->file; in io_do_iopoll()
1051 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1054 if (req->opcode == IORING_OP_URING_CMD) { in io_do_iopoll()
1058 ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, in io_do_iopoll()
1063 ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); in io_do_iopoll()
1072 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1085 /* order with io_complete_rw_iopoll(), e.g. ->result updates */ in io_do_iopoll()
1086 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1089 req->cqe.flags = io_put_kbuf(req, 0); in io_do_iopoll()
1094 pos = start ? start->next : ctx->iopoll_list.first; in io_do_iopoll()
1095 wq_list_cut(&ctx->iopoll_list, prev, start); in io_do_iopoll()
1097 if (WARN_ON_ONCE(!wq_list_empty(&ctx->submit_state.compl_reqs))) in io_do_iopoll()
1099 ctx->submit_state.compl_reqs.first = pos; in io_do_iopoll()