38 static inline bool io_is_timeout_noseq(struct io_kiocb *req)  in io_is_timeout_noseq()  argument
40 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_is_timeout_noseq()
41 	struct io_timeout_data *data = req->async_data;  in io_is_timeout_noseq()
46 static inline void io_put_req(struct io_kiocb *req)  in io_put_req()  argument
48 	if (req_ref_put_and_test(req)) {  in io_put_req()
49 		io_queue_next(req);  in io_put_req()
50 		io_free_req(req);  in io_put_req()
68 static void io_timeout_complete(struct io_kiocb *req, struct io_tw_state *ts)  in io_timeout_complete()  argument
70 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_complete()
71 	struct io_timeout_data *data = req->async_data;  in io_timeout_complete()
72 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_complete()
76 		filled = io_fill_cqe_req_aux(req, ts->locked, -ETIME,  in io_timeout_complete()
89 	io_req_task_complete(req, ts);  in io_timeout_complete()
92 static bool io_kill_timeout(struct io_kiocb *req, int status)  in io_kill_timeout()  argument
93 	__must_hold(&req->ctx->timeout_lock)  in io_kill_timeout()
95 	struct io_timeout_data *io = req->async_data;  in io_kill_timeout()
98 		struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_kill_timeout()
101 			req_set_fail(req);  in io_kill_timeout()
102 		atomic_set(&req->ctx->cq_timeouts,  in io_kill_timeout()
103 			atomic_read(&req->ctx->cq_timeouts) + 1);  in io_kill_timeout()
105 		io_req_queue_tw_complete(req, status);  in io_kill_timeout()
120 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_flush_timeouts()  local
123 		if (io_is_timeout_noseq(req))  in io_flush_timeouts()
138 		io_kill_timeout(req, 0);  in io_flush_timeouts()
160 static void io_fail_links(struct io_kiocb *req)  in io_fail_links()  argument
161 	__must_hold(&req->ctx->completion_lock)  in io_fail_links()
163 	struct io_kiocb *link = req->link;  in io_fail_links()
164 	bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;  in io_fail_links()
174 		trace_io_uring_fail_link(req, link);  in io_fail_links()
178 	link = req->link;  in io_fail_links()
181 	req->link = NULL;  in io_fail_links()
184 static inline void io_remove_next_linked(struct io_kiocb *req)  in io_remove_next_linked()  argument
186 	struct io_kiocb *nxt = req->link;  in io_remove_next_linked()
188 	req->link = nxt->link;  in io_remove_next_linked()
192 void io_disarm_next(struct io_kiocb *req)  in io_disarm_next()  argument
193 	__must_hold(&req->ctx->completion_lock)  in io_disarm_next()
197 	if (req->flags & REQ_F_ARM_LTIMEOUT) {  in io_disarm_next()
198 		link = req->link;  in io_disarm_next()
199 		req->flags &= ~REQ_F_ARM_LTIMEOUT;  in io_disarm_next()
201 			io_remove_next_linked(req);  in io_disarm_next()
204 	} else if (req->flags & REQ_F_LINK_TIMEOUT) {  in io_disarm_next()
205 		struct io_ring_ctx *ctx = req->ctx;  in io_disarm_next()
208 		link = io_disarm_linked_timeout(req);  in io_disarm_next()
213 	if (unlikely((req->flags & REQ_F_FAIL) &&  in io_disarm_next()
214 		     !(req->flags & REQ_F_HARDLINK)))  in io_disarm_next()
215 		io_fail_links(req);  in io_disarm_next()
218 struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,  in __io_disarm_linked_timeout()  argument
220 	__must_hold(&req->ctx->completion_lock)  in __io_disarm_linked_timeout()
221 	__must_hold(&req->ctx->timeout_lock)  in __io_disarm_linked_timeout()
226 	io_remove_next_linked(req);  in __io_disarm_linked_timeout()
240 	struct io_kiocb *req = data->req;  in io_timeout_fn()  local
241 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_fn()
242 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_fn()
247 	atomic_set(&req->ctx->cq_timeouts,  in io_timeout_fn()
248 		atomic_read(&req->ctx->cq_timeouts) + 1);  in io_timeout_fn()
252 		req_set_fail(req);  in io_timeout_fn()
254 	io_req_set_res(req, -ETIME, 0);  in io_timeout_fn()
255 	req->io_task_work.func = io_timeout_complete;  in io_timeout_fn()
256 	io_req_task_work_add(req);  in io_timeout_fn()
266 	struct io_kiocb *req = NULL;  in io_timeout_extract()  local
272 			req = tmp;  in io_timeout_extract()
276 	if (!req)  in io_timeout_extract()
279 	io = req->async_data;  in io_timeout_extract()
282 	timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_extract()
284 	return req;  in io_timeout_extract()
290 	struct io_kiocb *req;  in io_timeout_cancel()  local
293 	req = io_timeout_extract(ctx, cd);  in io_timeout_cancel()
296 	if (IS_ERR(req))  in io_timeout_cancel()
297 		return PTR_ERR(req);  in io_timeout_cancel()
298 	io_req_task_queue_fail(req, -ECANCELED);  in io_timeout_cancel()
302 static void io_req_task_link_timeout(struct io_kiocb *req, struct io_tw_state *ts)  in io_req_task_link_timeout()  argument
305 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_req_task_link_timeout()
310 		if (!(req->task->flags & PF_EXITING)) {  in io_req_task_link_timeout()
312 				.ctx		= req->ctx,  in io_req_task_link_timeout()
316 			ret = io_try_cancel(req->task->io_uring, &cd, issue_flags);  in io_req_task_link_timeout()
318 		io_req_set_res(req, ret ?: -ETIME, 0);  in io_req_task_link_timeout()
319 		io_req_task_complete(req, ts);  in io_req_task_link_timeout()
322 		io_req_set_res(req, -ETIME, 0);  in io_req_task_link_timeout()
323 		io_req_task_complete(req, ts);  in io_req_task_link_timeout()
331 	struct io_kiocb *prev, *req = data->req;  in io_link_timeout_fn()  local
332 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_link_timeout_fn()
333 	struct io_ring_ctx *ctx = req->ctx;  in io_link_timeout_fn()
353 	req->io_task_work.func = io_req_task_link_timeout;  in io_link_timeout_fn()
354 	io_req_task_work_add(req);  in io_link_timeout_fn()
380 	struct io_kiocb *req = NULL;  in io_linked_timeout_update()  local
386 			req = tmp;  in io_linked_timeout_update()
390 	if (!req)  in io_linked_timeout_update()
393 	io = req->async_data;  in io_linked_timeout_update()
407 	struct io_kiocb *req = io_timeout_extract(ctx, &cd);  in io_timeout_update()  local
408 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout_update()
411 	if (IS_ERR(req))  in io_timeout_update()
412 		return PTR_ERR(req);  in io_timeout_update()
415 	data = req->async_data;  in io_timeout_update()
425 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_remove_prep()  argument
427 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove_prep()
429 	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))  in io_timeout_remove_prep()
465 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout_remove()  argument
467 	struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);  in io_timeout_remove()
468 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout_remove()
489 		req_set_fail(req);  in io_timeout_remove()
490 	io_req_set_res(req, ret, 0);  in io_timeout_remove()
494 static int __io_timeout_prep(struct io_kiocb *req,  in __io_timeout_prep()  argument
498 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in __io_timeout_prep()
521 	if (unlikely(off && !req->ctx->off_timeout_used))  in __io_timeout_prep()
522 		req->ctx->off_timeout_used = true;  in __io_timeout_prep()
531 	if (WARN_ON_ONCE(req_has_async_data(req)))  in __io_timeout_prep()
533 	if (io_alloc_async_data(req))  in __io_timeout_prep()
536 	data = req->async_data;  in __io_timeout_prep()
537 	data->req = req;  in __io_timeout_prep()
551 		struct io_submit_link *link = &req->ctx->submit_state.link;  in __io_timeout_prep()
563 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_timeout_prep()  argument
565 	return __io_timeout_prep(req, sqe, false);  in io_timeout_prep()
568 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)  in io_link_timeout_prep()  argument
570 	return __io_timeout_prep(req, sqe, true);  in io_link_timeout_prep()
573 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)  in io_timeout()  argument
575 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_timeout()
576 	struct io_ring_ctx *ctx = req->ctx;  in io_timeout()
577 	struct io_timeout_data *data = req->async_data;  in io_timeout()
588 	if (io_is_timeout_noseq(req)) {  in io_timeout()
624 void io_queue_linked_timeout(struct io_kiocb *req)  in io_queue_linked_timeout()  argument
626 	struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);  in io_queue_linked_timeout()
627 	struct io_ring_ctx *ctx = req->ctx;  in io_queue_linked_timeout()
635 		struct io_timeout_data *data = req->async_data;  in io_queue_linked_timeout()
644 	io_put_req(req);  in io_queue_linked_timeout()
651 	struct io_kiocb *req;  in io_match_task()  local
658 	io_for_each_link(req, head) {  in io_match_task()
659 		if (req->flags & REQ_F_INFLIGHT)  in io_match_task()
679 		struct io_kiocb *req = cmd_to_io_kiocb(timeout);  in io_kill_timeouts()  local
681 		if (io_match_task(req, tsk, cancel_all) &&  in io_kill_timeouts()
682 		    io_kill_timeout(req, -ECANCELED))  in io_kill_timeouts()