Lines Matching full:req

38 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()  argument
40 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_is_timeout_noseq()
41 struct io_timeout_data *data = req->async_data; in io_is_timeout_noseq()
46 static inline void io_put_req(struct io_kiocb *req) in io_put_req() argument
48 if (req_ref_put_and_test(req)) { in io_put_req()
49 io_queue_next(req); in io_put_req()
50 io_free_req(req); in io_put_req()
68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts) in io_timeout_complete() argument
70 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_timeout_complete()
71 struct io_timeout_data *data = req->async_data; in io_timeout_complete()
72 struct io_ring_ctx *ctx = req->ctx; in io_timeout_complete()
76 filled = io_fill_cqe_req_aux(req, ts->locked, -ETIME, in io_timeout_complete()
89 io_req_task_complete(req, ts); in io_timeout_complete()
92 static bool io_kill_timeout(struct io_kiocb *req, int status) in io_kill_timeout() argument
93 __must_hold(&req->ctx->timeout_lock) in io_kill_timeout()
95 struct io_timeout_data *io = req->async_data; in io_kill_timeout()
98 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_kill_timeout()
101 req_set_fail(req); in io_kill_timeout()
102 atomic_set(&req->ctx->cq_timeouts, in io_kill_timeout()
103 atomic_read(&req->ctx->cq_timeouts) + 1); in io_kill_timeout()
105 io_req_queue_tw_complete(req, status); in io_kill_timeout()
120 struct io_kiocb *req = cmd_to_io_kiocb(timeout); in io_flush_timeouts() local
123 if (io_is_timeout_noseq(req)) in io_flush_timeouts()
138 io_kill_timeout(req, 0); in io_flush_timeouts()
160 static void io_fail_links(struct io_kiocb *req) in io_fail_links() argument
161 __must_hold(&req->ctx->completion_lock) in io_fail_links()
163 struct io_kiocb *link = req->link; in io_fail_links()
164 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES; in io_fail_links()
174 trace_io_uring_fail_link(req, link); in io_fail_links()
178 link = req->link; in io_fail_links()
181 req->link = NULL; in io_fail_links()
184 static inline void io_remove_next_linked(struct io_kiocb *req) in io_remove_next_linked() argument
186 struct io_kiocb *nxt = req->link; in io_remove_next_linked()
188 req->link = nxt->link; in io_remove_next_linked()
192 void io_disarm_next(struct io_kiocb *req) in io_disarm_next() argument
193 __must_hold(&req->ctx->completion_lock) in io_disarm_next()
197 if (req->flags & REQ_F_ARM_LTIMEOUT) { in io_disarm_next()
198 link = req->link; in io_disarm_next()
199 req->flags &= ~REQ_F_ARM_LTIMEOUT; in io_disarm_next()
201 io_remove_next_linked(req); in io_disarm_next()
204 } else if (req->flags & REQ_F_LINK_TIMEOUT) { in io_disarm_next()
205 struct io_ring_ctx *ctx = req->ctx; in io_disarm_next()
208 link = io_disarm_linked_timeout(req); in io_disarm_next()
213 if (unlikely((req->flags & REQ_F_FAIL) && in io_disarm_next()
214 !(req->flags & REQ_F_HARDLINK))) in io_disarm_next()
215 io_fail_links(req); in io_disarm_next()
218 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, in __io_disarm_linked_timeout() argument
220 __must_hold(&req->ctx->completion_lock) in __io_disarm_linked_timeout()
221 __must_hold(&req->ctx->timeout_lock) in __io_disarm_linked_timeout()
226 io_remove_next_linked(req); in __io_disarm_linked_timeout()
240 struct io_kiocb *req = data->req; in io_timeout_fn() local
241 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_timeout_fn()
242 struct io_ring_ctx *ctx = req->ctx; in io_timeout_fn()
247 atomic_set(&req->ctx->cq_timeouts, in io_timeout_fn()
248 atomic_read(&req->ctx->cq_timeouts) + 1); in io_timeout_fn()
252 req_set_fail(req); in io_timeout_fn()
254 io_req_set_res(req, -ETIME, 0); in io_timeout_fn()
255 req->io_task_work.func = io_timeout_complete; in io_timeout_fn()
256 io_req_task_work_add(req); in io_timeout_fn()
266 struct io_kiocb *req = NULL; in io_timeout_extract() local
272 req = tmp; in io_timeout_extract()
276 if (!req) in io_timeout_extract()
279 io = req->async_data; in io_timeout_extract()
282 timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_timeout_extract()
284 return req; in io_timeout_extract()
290 struct io_kiocb *req; in io_timeout_cancel() local
293 req = io_timeout_extract(ctx, cd); in io_timeout_cancel()
296 if (IS_ERR(req)) in io_timeout_cancel()
297 return PTR_ERR(req); in io_timeout_cancel()
298 io_req_task_queue_fail(req, -ECANCELED); in io_timeout_cancel()
302 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts) in io_req_task_link_timeout() argument
305 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_req_task_link_timeout()
310 if (!(req->task->flags & PF_EXITING)) { in io_req_task_link_timeout()
312 .ctx = req->ctx, in io_req_task_link_timeout()
316 ret = io_try_cancel(req->task->io_uring, &cd, issue_flags); in io_req_task_link_timeout()
318 io_req_set_res(req, ret ?: -ETIME, 0); in io_req_task_link_timeout()
319 io_req_task_complete(req, ts); in io_req_task_link_timeout()
322 io_req_set_res(req, -ETIME, 0); in io_req_task_link_timeout()
323 io_req_task_complete(req, ts); in io_req_task_link_timeout()
331 struct io_kiocb *prev, *req = data->req; in io_link_timeout_fn() local
332 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_link_timeout_fn()
333 struct io_ring_ctx *ctx = req->ctx; in io_link_timeout_fn()
353 req->io_task_work.func = io_req_task_link_timeout; in io_link_timeout_fn()
354 io_req_task_work_add(req); in io_link_timeout_fn()
380 struct io_kiocb *req = NULL; in io_linked_timeout_update() local
386 req = tmp; in io_linked_timeout_update()
390 if (!req) in io_linked_timeout_update()
393 io = req->async_data; in io_linked_timeout_update()
407 struct io_kiocb *req = io_timeout_extract(ctx, &cd); in io_timeout_update() local
408 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_timeout_update()
411 if (IS_ERR(req)) in io_timeout_update()
412 return PTR_ERR(req); in io_timeout_update()
415 data = req->async_data; in io_timeout_update()
425 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_timeout_remove_prep() argument
427 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); in io_timeout_remove_prep()
429 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) in io_timeout_remove_prep()
465 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) in io_timeout_remove() argument
467 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); in io_timeout_remove()
468 struct io_ring_ctx *ctx = req->ctx; in io_timeout_remove()
489 req_set_fail(req); in io_timeout_remove()
490 io_req_set_res(req, ret, 0); in io_timeout_remove()
494 static int __io_timeout_prep(struct io_kiocb *req, in __io_timeout_prep() argument
498 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in __io_timeout_prep()
521 if (unlikely(off && !req->ctx->off_timeout_used)) in __io_timeout_prep()
522 req->ctx->off_timeout_used = true; in __io_timeout_prep()
531 if (WARN_ON_ONCE(req_has_async_data(req))) in __io_timeout_prep()
533 if (io_alloc_async_data(req)) in __io_timeout_prep()
536 data = req->async_data; in __io_timeout_prep()
537 data->req = req; in __io_timeout_prep()
551 struct io_submit_link *link = &req->ctx->submit_state.link; in __io_timeout_prep()
563 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_timeout_prep() argument
565 return __io_timeout_prep(req, sqe, false); in io_timeout_prep()
568 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) in io_link_timeout_prep() argument
570 return __io_timeout_prep(req, sqe, true); in io_link_timeout_prep()
573 int io_timeout(struct io_kiocb *req, unsigned int issue_flags) in io_timeout() argument
575 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_timeout()
576 struct io_ring_ctx *ctx = req->ctx; in io_timeout()
577 struct io_timeout_data *data = req->async_data; in io_timeout()
588 if (io_is_timeout_noseq(req)) { in io_timeout()
624 void io_queue_linked_timeout(struct io_kiocb *req) in io_queue_linked_timeout() argument
626 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); in io_queue_linked_timeout()
627 struct io_ring_ctx *ctx = req->ctx; in io_queue_linked_timeout()
635 struct io_timeout_data *data = req->async_data; in io_queue_linked_timeout()
644 io_put_req(req); in io_queue_linked_timeout()
651 struct io_kiocb *req; in io_match_task() local
658 io_for_each_link(req, head) { in io_match_task()
659 if (req->flags & REQ_F_INFLIGHT) in io_match_task()
679 struct io_kiocb *req = cmd_to_io_kiocb(timeout); in io_kill_timeouts() local
681 if (io_match_task(req, tsk, cancel_all) && in io_kill_timeouts()
682 io_kill_timeout(req, -ECANCELED)) in io_kill_timeouts()