1ed29b0b4SJens Axboe // SPDX-License-Identifier: GPL-2.0
2ed29b0b4SJens Axboe /*
3ed29b0b4SJens Axboe * Shared application/kernel submission and completion ring pairs, for
4ed29b0b4SJens Axboe * supporting fast/efficient IO.
5ed29b0b4SJens Axboe *
6ed29b0b4SJens Axboe * A note on the read/write ordering memory barriers that are matched between
7ed29b0b4SJens Axboe * the application and kernel side.
8ed29b0b4SJens Axboe *
9ed29b0b4SJens Axboe * After the application reads the CQ ring tail, it must use an
10ed29b0b4SJens Axboe * appropriate smp_rmb() to pair with the smp_wmb() the kernel uses
11ed29b0b4SJens Axboe * before writing the tail (using smp_load_acquire to read the tail will
12ed29b0b4SJens Axboe * do). It also needs a smp_mb() before updating CQ head (ordering the
13ed29b0b4SJens Axboe * entry load(s) with the head store), pairing with an implicit barrier
14ed29b0b4SJens Axboe * through a control-dependency in io_get_cqe (smp_store_release to
15ed29b0b4SJens Axboe * store head will do). Failure to do so could lead to reading invalid
16ed29b0b4SJens Axboe * CQ entries.
17ed29b0b4SJens Axboe *
18ed29b0b4SJens Axboe * Likewise, the application must use an appropriate smp_wmb() before
19ed29b0b4SJens Axboe * writing the SQ tail (ordering SQ entry stores with the tail store),
20ed29b0b4SJens Axboe * which pairs with smp_load_acquire in io_get_sqring (smp_store_release
21ed29b0b4SJens Axboe * to store the tail will do). And it needs a barrier ordering the SQ
22ed29b0b4SJens Axboe * head load before writing new SQ entries (smp_load_acquire to read
23ed29b0b4SJens Axboe * head will do).
24ed29b0b4SJens Axboe *
25ed29b0b4SJens Axboe * When using the SQ poll thread (IORING_SETUP_SQPOLL), the application
26ed29b0b4SJens Axboe * needs to check the SQ flags for IORING_SQ_NEED_WAKEUP *after*
27ed29b0b4SJens Axboe * updating the SQ tail; a full memory barrier smp_mb() is needed
28ed29b0b4SJens Axboe * between.
29ed29b0b4SJens Axboe *
30ed29b0b4SJens Axboe * Also see the examples in the liburing library:
31ed29b0b4SJens Axboe *
32ed29b0b4SJens Axboe * git://git.kernel.dk/liburing
33ed29b0b4SJens Axboe *
34ed29b0b4SJens Axboe * io_uring also uses READ/WRITE_ONCE() for _any_ store or load that happens
35ed29b0b4SJens Axboe * from data shared between the kernel and application. This is done both
36ed29b0b4SJens Axboe * for ordering purposes, but also to ensure that once a value is loaded from
37ed29b0b4SJens Axboe * data that the application could potentially modify, it remains stable.
38ed29b0b4SJens Axboe *
39ed29b0b4SJens Axboe * Copyright (C) 2018-2019 Jens Axboe
40ed29b0b4SJens Axboe * Copyright (c) 2018-2019 Christoph Hellwig
41ed29b0b4SJens Axboe */
42ed29b0b4SJens Axboe #include <linux/kernel.h>
43ed29b0b4SJens Axboe #include <linux/init.h>
44ed29b0b4SJens Axboe #include <linux/errno.h>
45ed29b0b4SJens Axboe #include <linux/syscalls.h>
46ed29b0b4SJens Axboe #include <net/compat.h>
47ed29b0b4SJens Axboe #include <linux/refcount.h>
48ed29b0b4SJens Axboe #include <linux/uio.h>
49ed29b0b4SJens Axboe #include <linux/bits.h>
50ed29b0b4SJens Axboe
51ed29b0b4SJens Axboe #include <linux/sched/signal.h>
52ed29b0b4SJens Axboe #include <linux/fs.h>
53ed29b0b4SJens Axboe #include <linux/file.h>
54ed29b0b4SJens Axboe #include <linux/fdtable.h>
55ed29b0b4SJens Axboe #include <linux/mm.h>
56ed29b0b4SJens Axboe #include <linux/mman.h>
57ed29b0b4SJens Axboe #include <linux/percpu.h>
58ed29b0b4SJens Axboe #include <linux/slab.h>
59ed29b0b4SJens Axboe #include <linux/bvec.h>
60ed29b0b4SJens Axboe #include <linux/net.h>
61ed29b0b4SJens Axboe #include <net/sock.h>
62ed29b0b4SJens Axboe #include <net/af_unix.h>
63ed29b0b4SJens Axboe #include <linux/anon_inodes.h>
64ed29b0b4SJens Axboe #include <linux/sched/mm.h>
65ed29b0b4SJens Axboe #include <linux/uaccess.h>
66ed29b0b4SJens Axboe #include <linux/nospec.h>
67ed29b0b4SJens Axboe #include <linux/highmem.h>
68ed29b0b4SJens Axboe #include <linux/fsnotify.h>
69ed29b0b4SJens Axboe #include <linux/fadvise.h>
70ed29b0b4SJens Axboe #include <linux/task_work.h>
71ed29b0b4SJens Axboe #include <linux/io_uring.h>
72ed29b0b4SJens Axboe #include <linux/audit.h>
73ed29b0b4SJens Axboe #include <linux/security.h>
74d808459bSHelge Deller #include <asm/shmparam.h>
75ed29b0b4SJens Axboe
76ed29b0b4SJens Axboe #define CREATE_TRACE_POINTS
77ed29b0b4SJens Axboe #include <trace/events/io_uring.h>
78ed29b0b4SJens Axboe
79ed29b0b4SJens Axboe #include <uapi/linux/io_uring.h>
80ed29b0b4SJens Axboe
81ed29b0b4SJens Axboe #include "io-wq.h"
82ed29b0b4SJens Axboe
83de23077eSJens Axboe #include "io_uring.h"
84329061d3SJens Axboe #include "opdef.h"
85e418bbc9SJens Axboe #include "refs.h"
86c9f06aa7SJens Axboe #include "tctx.h"
8717437f31SJens Axboe #include "sqpoll.h"
88a4ad4f74SJens Axboe #include "fdinfo.h"
893b77495aSJens Axboe #include "kbuf.h"
9073572984SJens Axboe #include "rsrc.h"
9138513c46SHao Xu #include "cancel.h"
9243e0bbbdSJens Axboe #include "net.h"
93eb42cebbSPavel Begunkov #include "notif.h"
94e27f928eSJens Axboe
9559915143SJens Axboe #include "timeout.h"
96329061d3SJens Axboe #include "poll.h"
97c92fcfc2SJens Axboe #include "rw.h"
989b797a37SJens Axboe #include "alloc_cache.h"
995e2a18d9SJens Axboe
100ed29b0b4SJens Axboe #define IORING_MAX_ENTRIES 32768
101ed29b0b4SJens Axboe #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
102ed29b0b4SJens Axboe
103ed29b0b4SJens Axboe #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
104ed29b0b4SJens Axboe IORING_REGISTER_LAST + IORING_OP_LAST)
105ed29b0b4SJens Axboe
106ed29b0b4SJens Axboe #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
107ed29b0b4SJens Axboe IOSQE_IO_HARDLINK | IOSQE_ASYNC)
108ed29b0b4SJens Axboe
109ed29b0b4SJens Axboe #define SQE_VALID_FLAGS (SQE_COMMON_FLAGS | IOSQE_BUFFER_SELECT | \
110ed29b0b4SJens Axboe IOSQE_IO_DRAIN | IOSQE_CQE_SKIP_SUCCESS)
111ed29b0b4SJens Axboe
112ed29b0b4SJens Axboe #define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
113ed29b0b4SJens Axboe REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \
114ed29b0b4SJens Axboe REQ_F_ASYNC_DATA)
115ed29b0b4SJens Axboe
116ed29b0b4SJens Axboe #define IO_REQ_CLEAN_SLOW_FLAGS (REQ_F_REFCOUNT | REQ_F_LINK | REQ_F_HARDLINK |\
117ed29b0b4SJens Axboe IO_REQ_CLEAN_FLAGS)
118ed29b0b4SJens Axboe
119ed29b0b4SJens Axboe #define IO_TCTX_REFS_CACHE_NR (1U << 10)
120ed29b0b4SJens Axboe
121ed29b0b4SJens Axboe #define IO_COMPL_BATCH 32
122ed29b0b4SJens Axboe #define IO_REQ_ALLOC_BATCH 8
123ed29b0b4SJens Axboe
124ed29b0b4SJens Axboe enum {
125ed29b0b4SJens Axboe IO_CHECK_CQ_OVERFLOW_BIT,
126ed29b0b4SJens Axboe IO_CHECK_CQ_DROPPED_BIT,
127ed29b0b4SJens Axboe };
128ed29b0b4SJens Axboe
12921a091b9SDylan Yudaken enum {
13021a091b9SDylan Yudaken IO_EVENTFD_OP_SIGNAL_BIT,
13121a091b9SDylan Yudaken IO_EVENTFD_OP_FREE_BIT,
13221a091b9SDylan Yudaken };
13321a091b9SDylan Yudaken
134ed29b0b4SJens Axboe struct io_defer_entry {
135ed29b0b4SJens Axboe struct list_head list;
136ed29b0b4SJens Axboe struct io_kiocb *req;
137ed29b0b4SJens Axboe u32 seq;
138ed29b0b4SJens Axboe };
139ed29b0b4SJens Axboe
140ed29b0b4SJens Axboe /* requests with any of those set should undergo io_disarm_next() */
141ed29b0b4SJens Axboe #define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
142ed29b0b4SJens Axboe #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
143ed29b0b4SJens Axboe
144affa87dbSPavel Begunkov static bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
145ed29b0b4SJens Axboe struct task_struct *task,
146ed29b0b4SJens Axboe bool cancel_all);
147ed29b0b4SJens Axboe
148ed29b0b4SJens Axboe static void io_queue_sqe(struct io_kiocb *req);
149ed29b0b4SJens Axboe
150c1755c25SBreno Leitao struct kmem_cache *req_cachep;
1516b9d49bcSJens Axboe static struct workqueue_struct *iou_wq __ro_after_init;
152ed29b0b4SJens Axboe
15376d3ccecSMatteo Rizzo static int __read_mostly sysctl_io_uring_disabled;
15476d3ccecSMatteo Rizzo static int __read_mostly sysctl_io_uring_group = -1;
15576d3ccecSMatteo Rizzo
15676d3ccecSMatteo Rizzo #ifdef CONFIG_SYSCTL
15776d3ccecSMatteo Rizzo static struct ctl_table kernel_io_uring_disabled_table[] = {
15876d3ccecSMatteo Rizzo {
15976d3ccecSMatteo Rizzo .procname = "io_uring_disabled",
16076d3ccecSMatteo Rizzo .data = &sysctl_io_uring_disabled,
16176d3ccecSMatteo Rizzo .maxlen = sizeof(sysctl_io_uring_disabled),
16276d3ccecSMatteo Rizzo .mode = 0644,
16376d3ccecSMatteo Rizzo .proc_handler = proc_dointvec_minmax,
16476d3ccecSMatteo Rizzo .extra1 = SYSCTL_ZERO,
16576d3ccecSMatteo Rizzo .extra2 = SYSCTL_TWO,
16676d3ccecSMatteo Rizzo },
16776d3ccecSMatteo Rizzo {
16876d3ccecSMatteo Rizzo .procname = "io_uring_group",
16976d3ccecSMatteo Rizzo .data = &sysctl_io_uring_group,
17076d3ccecSMatteo Rizzo .maxlen = sizeof(gid_t),
17176d3ccecSMatteo Rizzo .mode = 0644,
17276d3ccecSMatteo Rizzo .proc_handler = proc_dointvec,
17376d3ccecSMatteo Rizzo },
17476d3ccecSMatteo Rizzo {},
17576d3ccecSMatteo Rizzo };
17676d3ccecSMatteo Rizzo #endif
17776d3ccecSMatteo Rizzo
io_submit_flush_completions(struct io_ring_ctx * ctx)178ed29b0b4SJens Axboe static inline void io_submit_flush_completions(struct io_ring_ctx *ctx)
179ed29b0b4SJens Axboe {
180931147ddSDylan Yudaken if (!wq_list_empty(&ctx->submit_state.compl_reqs) ||
181931147ddSDylan Yudaken ctx->submit_state.cqes_count)
182ed29b0b4SJens Axboe __io_submit_flush_completions(ctx);
183ed29b0b4SJens Axboe }
184ed29b0b4SJens Axboe
__io_cqring_events(struct io_ring_ctx * ctx)185faf88ddeSPavel Begunkov static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
186faf88ddeSPavel Begunkov {
187faf88ddeSPavel Begunkov return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
188faf88ddeSPavel Begunkov }
189faf88ddeSPavel Begunkov
__io_cqring_events_user(struct io_ring_ctx * ctx)1900fc8c2acSDylan Yudaken static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx)
1910fc8c2acSDylan Yudaken {
1920fc8c2acSDylan Yudaken return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
1930fc8c2acSDylan Yudaken }
1940fc8c2acSDylan Yudaken
io_match_linked(struct io_kiocb * head)195ed29b0b4SJens Axboe static bool io_match_linked(struct io_kiocb *head)
196ed29b0b4SJens Axboe {
197ed29b0b4SJens Axboe struct io_kiocb *req;
198ed29b0b4SJens Axboe
199ed29b0b4SJens Axboe io_for_each_link(req, head) {
200ed29b0b4SJens Axboe if (req->flags & REQ_F_INFLIGHT)
201ed29b0b4SJens Axboe return true;
202ed29b0b4SJens Axboe }
203ed29b0b4SJens Axboe return false;
204ed29b0b4SJens Axboe }
205ed29b0b4SJens Axboe
206ed29b0b4SJens Axboe /*
207ed29b0b4SJens Axboe * As io_match_task() but protected against racing with linked timeouts.
208ed29b0b4SJens Axboe * User must not hold timeout_lock.
209ed29b0b4SJens Axboe */
io_match_task_safe(struct io_kiocb * head,struct task_struct * task,bool cancel_all)210329061d3SJens Axboe bool io_match_task_safe(struct io_kiocb *head, struct task_struct *task,
211ed29b0b4SJens Axboe bool cancel_all)
212ed29b0b4SJens Axboe {
213ed29b0b4SJens Axboe bool matched;
214ed29b0b4SJens Axboe
215ed29b0b4SJens Axboe if (task && head->task != task)
216ed29b0b4SJens Axboe return false;
217ed29b0b4SJens Axboe if (cancel_all)
218ed29b0b4SJens Axboe return true;
219ed29b0b4SJens Axboe
220ed29b0b4SJens Axboe if (head->flags & REQ_F_LINK_TIMEOUT) {
221ed29b0b4SJens Axboe struct io_ring_ctx *ctx = head->ctx;
222ed29b0b4SJens Axboe
223ed29b0b4SJens Axboe /* protect against races with linked timeouts */
224ed29b0b4SJens Axboe spin_lock_irq(&ctx->timeout_lock);
225ed29b0b4SJens Axboe matched = io_match_linked(head);
226ed29b0b4SJens Axboe spin_unlock_irq(&ctx->timeout_lock);
227ed29b0b4SJens Axboe } else {
228ed29b0b4SJens Axboe matched = io_match_linked(head);
229ed29b0b4SJens Axboe }
230ed29b0b4SJens Axboe return matched;
231ed29b0b4SJens Axboe }
232ed29b0b4SJens Axboe
req_fail_link_node(struct io_kiocb * req,int res)233ed29b0b4SJens Axboe static inline void req_fail_link_node(struct io_kiocb *req, int res)
234ed29b0b4SJens Axboe {
235ed29b0b4SJens Axboe req_set_fail(req);
23697b388d7SJens Axboe io_req_set_res(req, res, 0);
237ed29b0b4SJens Axboe }
238ed29b0b4SJens Axboe
io_req_add_to_cache(struct io_kiocb * req,struct io_ring_ctx * ctx)239ed29b0b4SJens Axboe static inline void io_req_add_to_cache(struct io_kiocb *req, struct io_ring_ctx *ctx)
240ed29b0b4SJens Axboe {
241ed29b0b4SJens Axboe wq_stack_add_head(&req->comp_list, &ctx->submit_state.free_list);
242ed29b0b4SJens Axboe }
243ed29b0b4SJens Axboe
io_ring_ctx_ref_free(struct percpu_ref * ref)244ed29b0b4SJens Axboe static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref)
245ed29b0b4SJens Axboe {
246ed29b0b4SJens Axboe struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
247ed29b0b4SJens Axboe
248ed29b0b4SJens Axboe complete(&ctx->ref_comp);
249ed29b0b4SJens Axboe }
250ed29b0b4SJens Axboe
io_fallback_req_func(struct work_struct * work)251ed29b0b4SJens Axboe static __cold void io_fallback_req_func(struct work_struct *work)
252ed29b0b4SJens Axboe {
253ed29b0b4SJens Axboe struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
254ed29b0b4SJens Axboe fallback_work.work);
255ed29b0b4SJens Axboe struct llist_node *node = llist_del_all(&ctx->fallback_llist);
256ed29b0b4SJens Axboe struct io_kiocb *req, *tmp;
257a282967cSPavel Begunkov struct io_tw_state ts = { .locked = true, };
258ed29b0b4SJens Axboe
25930df2901SPavel Begunkov percpu_ref_get(&ctx->refs);
26031f084b7SPavel Begunkov mutex_lock(&ctx->uring_lock);
2613218e5d3SPavel Begunkov llist_for_each_entry_safe(req, tmp, node, io_task_work.node)
262a282967cSPavel Begunkov req->io_task_work.func(req, &ts);
263a282967cSPavel Begunkov if (WARN_ON_ONCE(!ts.locked))
26431f084b7SPavel Begunkov return;
265ed29b0b4SJens Axboe io_submit_flush_completions(ctx);
266ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
26730df2901SPavel Begunkov percpu_ref_put(&ctx->refs);
268ed29b0b4SJens Axboe }
269ed29b0b4SJens Axboe
io_alloc_hash_table(struct io_hash_table * table,unsigned bits)270e6f89be6SPavel Begunkov static int io_alloc_hash_table(struct io_hash_table *table, unsigned bits)
271e6f89be6SPavel Begunkov {
272e6f89be6SPavel Begunkov unsigned hash_buckets = 1U << bits;
273e6f89be6SPavel Begunkov size_t hash_size = hash_buckets * sizeof(table->hbs[0]);
274e6f89be6SPavel Begunkov
275e6f89be6SPavel Begunkov table->hbs = kmalloc(hash_size, GFP_KERNEL);
276e6f89be6SPavel Begunkov if (!table->hbs)
277e6f89be6SPavel Begunkov return -ENOMEM;
278e6f89be6SPavel Begunkov
279e6f89be6SPavel Begunkov table->hash_bits = bits;
280e6f89be6SPavel Begunkov init_hash_table(table, hash_buckets);
281e6f89be6SPavel Begunkov return 0;
282e6f89be6SPavel Begunkov }
283e6f89be6SPavel Begunkov
io_ring_ctx_alloc(struct io_uring_params * p)284ed29b0b4SJens Axboe static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
285ed29b0b4SJens Axboe {
286ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
287ed29b0b4SJens Axboe int hash_bits;
288ed29b0b4SJens Axboe
289ed29b0b4SJens Axboe ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
290ed29b0b4SJens Axboe if (!ctx)
291ed29b0b4SJens Axboe return NULL;
292ed29b0b4SJens Axboe
293ed29b0b4SJens Axboe xa_init(&ctx->io_bl_xa);
294ed29b0b4SJens Axboe
295ed29b0b4SJens Axboe /*
296ed29b0b4SJens Axboe * Use 5 bits less than the max cq entries, that should give us around
2974a07723fSPavel Begunkov * 32 entries per hash list if totally full and uniformly spread, but
2984a07723fSPavel Begunkov * don't keep too many buckets to not overconsume memory.
299ed29b0b4SJens Axboe */
3004a07723fSPavel Begunkov hash_bits = ilog2(p->cq_entries) - 5;
3014a07723fSPavel Begunkov hash_bits = clamp(hash_bits, 1, 8);
302e6f89be6SPavel Begunkov if (io_alloc_hash_table(&ctx->cancel_table, hash_bits))
303ed29b0b4SJens Axboe goto err;
3049ca9fb24SPavel Begunkov if (io_alloc_hash_table(&ctx->cancel_table_locked, hash_bits))
3059ca9fb24SPavel Begunkov goto err;
306ed29b0b4SJens Axboe if (percpu_ref_init(&ctx->refs, io_ring_ctx_ref_free,
30748904229SMichal Koutný 0, GFP_KERNEL))
308ed29b0b4SJens Axboe goto err;
309ed29b0b4SJens Axboe
310ed29b0b4SJens Axboe ctx->flags = p->flags;
311ed29b0b4SJens Axboe init_waitqueue_head(&ctx->sqo_sq_wait);
312ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->sqd_list);
313ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->cq_overflow_list);
314ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->io_buffers_cache);
3157138ebbeSJens Axboe INIT_HLIST_HEAD(&ctx->io_buf_list);
31669bbc6adSPavel Begunkov io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,
31769bbc6adSPavel Begunkov sizeof(struct io_rsrc_node));
31869bbc6adSPavel Begunkov io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,
31969bbc6adSPavel Begunkov sizeof(struct async_poll));
32069bbc6adSPavel Begunkov io_alloc_cache_init(&ctx->netmsg_cache, IO_ALLOC_CACHE_MAX,
32169bbc6adSPavel Begunkov sizeof(struct io_async_msghdr));
322ed29b0b4SJens Axboe init_completion(&ctx->ref_comp);
323ed29b0b4SJens Axboe xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
324ed29b0b4SJens Axboe mutex_init(&ctx->uring_lock);
325ed29b0b4SJens Axboe init_waitqueue_head(&ctx->cq_wait);
3267b235dd8SPavel Begunkov init_waitqueue_head(&ctx->poll_wq);
3274ea15b56SPavel Begunkov init_waitqueue_head(&ctx->rsrc_quiesce_wq);
328ed29b0b4SJens Axboe spin_lock_init(&ctx->completion_lock);
329ed29b0b4SJens Axboe spin_lock_init(&ctx->timeout_lock);
330ed29b0b4SJens Axboe INIT_WQ_LIST(&ctx->iopoll_list);
331ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->io_buffers_pages);
332ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->io_buffers_comp);
333ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->defer_list);
334ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->timeout_list);
335ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->ltimeout_list);
336ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->rsrc_ref_list);
337c0e0d6baSDylan Yudaken init_llist_head(&ctx->work_llist);
338ed29b0b4SJens Axboe INIT_LIST_HEAD(&ctx->tctx_list);
339ed29b0b4SJens Axboe ctx->submit_state.free_list.next = NULL;
340ed29b0b4SJens Axboe INIT_WQ_LIST(&ctx->locked_free_list);
341ed29b0b4SJens Axboe INIT_DELAYED_WORK(&ctx->fallback_work, io_fallback_req_func);
342ed29b0b4SJens Axboe INIT_WQ_LIST(&ctx->submit_state.compl_reqs);
343ed29b0b4SJens Axboe return ctx;
344ed29b0b4SJens Axboe err:
345e6f89be6SPavel Begunkov kfree(ctx->cancel_table.hbs);
3469ca9fb24SPavel Begunkov kfree(ctx->cancel_table_locked.hbs);
347ed29b0b4SJens Axboe xa_destroy(&ctx->io_bl_xa);
348ed29b0b4SJens Axboe kfree(ctx);
349ed29b0b4SJens Axboe return NULL;
350ed29b0b4SJens Axboe }
351ed29b0b4SJens Axboe
io_account_cq_overflow(struct io_ring_ctx * ctx)352ed29b0b4SJens Axboe static void io_account_cq_overflow(struct io_ring_ctx *ctx)
353ed29b0b4SJens Axboe {
354ed29b0b4SJens Axboe struct io_rings *r = ctx->rings;
355ed29b0b4SJens Axboe
356ed29b0b4SJens Axboe WRITE_ONCE(r->cq_overflow, READ_ONCE(r->cq_overflow) + 1);
357ed29b0b4SJens Axboe ctx->cq_extra--;
358ed29b0b4SJens Axboe }
359ed29b0b4SJens Axboe
req_need_defer(struct io_kiocb * req,u32 seq)360ed29b0b4SJens Axboe static bool req_need_defer(struct io_kiocb *req, u32 seq)
361ed29b0b4SJens Axboe {
362ed29b0b4SJens Axboe if (unlikely(req->flags & REQ_F_IO_DRAIN)) {
363ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
364ed29b0b4SJens Axboe
365ed29b0b4SJens Axboe return seq + READ_ONCE(ctx->cq_extra) != ctx->cached_cq_tail;
366ed29b0b4SJens Axboe }
367ed29b0b4SJens Axboe
368ed29b0b4SJens Axboe return false;
369ed29b0b4SJens Axboe }
370ed29b0b4SJens Axboe
io_clean_op(struct io_kiocb * req)3715a754deaSPavel Begunkov static void io_clean_op(struct io_kiocb *req)
3725a754deaSPavel Begunkov {
3735a754deaSPavel Begunkov if (req->flags & REQ_F_BUFFER_SELECTED) {
3745a754deaSPavel Begunkov spin_lock(&req->ctx->completion_lock);
3755a754deaSPavel Begunkov io_put_kbuf_comp(req);
3765a754deaSPavel Begunkov spin_unlock(&req->ctx->completion_lock);
3775a754deaSPavel Begunkov }
3785a754deaSPavel Begunkov
3795a754deaSPavel Begunkov if (req->flags & REQ_F_NEED_CLEANUP) {
3805a754deaSPavel Begunkov const struct io_cold_def *def = &io_cold_defs[req->opcode];
3815a754deaSPavel Begunkov
3825a754deaSPavel Begunkov if (def->cleanup)
3835a754deaSPavel Begunkov def->cleanup(req);
3845a754deaSPavel Begunkov }
3855a754deaSPavel Begunkov if ((req->flags & REQ_F_POLLED) && req->apoll) {
3865a754deaSPavel Begunkov kfree(req->apoll->double_poll);
3875a754deaSPavel Begunkov kfree(req->apoll);
3885a754deaSPavel Begunkov req->apoll = NULL;
3895a754deaSPavel Begunkov }
3905a754deaSPavel Begunkov if (req->flags & REQ_F_INFLIGHT) {
3915a754deaSPavel Begunkov struct io_uring_task *tctx = req->task->io_uring;
3925a754deaSPavel Begunkov
3935a754deaSPavel Begunkov atomic_dec(&tctx->inflight_tracked);
3945a754deaSPavel Begunkov }
3955a754deaSPavel Begunkov if (req->flags & REQ_F_CREDS)
3965a754deaSPavel Begunkov put_cred(req->creds);
3975a754deaSPavel Begunkov if (req->flags & REQ_F_ASYNC_DATA) {
3985a754deaSPavel Begunkov kfree(req->async_data);
3995a754deaSPavel Begunkov req->async_data = NULL;
4005a754deaSPavel Begunkov }
4015a754deaSPavel Begunkov req->flags &= ~IO_REQ_CLEAN_FLAGS;
4025a754deaSPavel Begunkov }
4035a754deaSPavel Begunkov
io_req_track_inflight(struct io_kiocb * req)404ed29b0b4SJens Axboe static inline void io_req_track_inflight(struct io_kiocb *req)
405ed29b0b4SJens Axboe {
406ed29b0b4SJens Axboe if (!(req->flags & REQ_F_INFLIGHT)) {
407ed29b0b4SJens Axboe req->flags |= REQ_F_INFLIGHT;
408ed29b0b4SJens Axboe atomic_inc(&req->task->io_uring->inflight_tracked);
409ed29b0b4SJens Axboe }
410ed29b0b4SJens Axboe }
411ed29b0b4SJens Axboe
__io_prep_linked_timeout(struct io_kiocb * req)412ed29b0b4SJens Axboe static struct io_kiocb *__io_prep_linked_timeout(struct io_kiocb *req)
413ed29b0b4SJens Axboe {
414ed29b0b4SJens Axboe if (WARN_ON_ONCE(!req->link))
415ed29b0b4SJens Axboe return NULL;
416ed29b0b4SJens Axboe
417ed29b0b4SJens Axboe req->flags &= ~REQ_F_ARM_LTIMEOUT;
418ed29b0b4SJens Axboe req->flags |= REQ_F_LINK_TIMEOUT;
419ed29b0b4SJens Axboe
420ed29b0b4SJens Axboe /* linked timeouts should have two refs once prep'ed */
421ed29b0b4SJens Axboe io_req_set_refcount(req);
422ed29b0b4SJens Axboe __io_req_set_refcount(req->link, 2);
423ed29b0b4SJens Axboe return req->link;
424ed29b0b4SJens Axboe }
425ed29b0b4SJens Axboe
io_prep_linked_timeout(struct io_kiocb * req)426ed29b0b4SJens Axboe static inline struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
427ed29b0b4SJens Axboe {
428ed29b0b4SJens Axboe if (likely(!(req->flags & REQ_F_ARM_LTIMEOUT)))
429ed29b0b4SJens Axboe return NULL;
430ed29b0b4SJens Axboe return __io_prep_linked_timeout(req);
431ed29b0b4SJens Axboe }
432ed29b0b4SJens Axboe
__io_arm_ltimeout(struct io_kiocb * req)433ed29b0b4SJens Axboe static noinline void __io_arm_ltimeout(struct io_kiocb *req)
434ed29b0b4SJens Axboe {
435ed29b0b4SJens Axboe io_queue_linked_timeout(__io_prep_linked_timeout(req));
436ed29b0b4SJens Axboe }
437ed29b0b4SJens Axboe
io_arm_ltimeout(struct io_kiocb * req)438ed29b0b4SJens Axboe static inline void io_arm_ltimeout(struct io_kiocb *req)
439ed29b0b4SJens Axboe {
440ed29b0b4SJens Axboe if (unlikely(req->flags & REQ_F_ARM_LTIMEOUT))
441ed29b0b4SJens Axboe __io_arm_ltimeout(req);
442ed29b0b4SJens Axboe }
443ed29b0b4SJens Axboe
io_prep_async_work(struct io_kiocb * req)444ed29b0b4SJens Axboe static void io_prep_async_work(struct io_kiocb *req)
445ed29b0b4SJens Axboe {
446a7dd2782SBreno Leitao const struct io_issue_def *def = &io_issue_defs[req->opcode];
447ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
448ed29b0b4SJens Axboe
449ed29b0b4SJens Axboe if (!(req->flags & REQ_F_CREDS)) {
450ed29b0b4SJens Axboe req->flags |= REQ_F_CREDS;
451ed29b0b4SJens Axboe req->creds = get_current_cred();
452ed29b0b4SJens Axboe }
453ed29b0b4SJens Axboe
454ed29b0b4SJens Axboe req->work.list.next = NULL;
455ed29b0b4SJens Axboe req->work.flags = 0;
456ed29b0b4SJens Axboe req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
457ed29b0b4SJens Axboe if (req->flags & REQ_F_FORCE_ASYNC)
458ed29b0b4SJens Axboe req->work.flags |= IO_WQ_WORK_CONCURRENT;
459ed29b0b4SJens Axboe
4603beed235SChristoph Hellwig if (req->file && !(req->flags & REQ_F_FIXED_FILE))
4618487f083SChristoph Hellwig req->flags |= io_file_get_flags(req->file);
462f6b543fdSJens Axboe
4638b1df11fSPavel Begunkov if (req->file && (req->flags & REQ_F_ISREG)) {
464d4755e15SJens Axboe bool should_hash = def->hash_reg_file;
465d4755e15SJens Axboe
466d4755e15SJens Axboe /* don't serialize this request if the fs doesn't need it */
467d4755e15SJens Axboe if (should_hash && (req->file->f_flags & O_DIRECT) &&
468d4755e15SJens Axboe (req->file->f_mode & FMODE_DIO_PARALLEL_WRITE))
469d4755e15SJens Axboe should_hash = false;
470d4755e15SJens Axboe if (should_hash || (ctx->flags & IORING_SETUP_IOPOLL))
471ed29b0b4SJens Axboe io_wq_hash_work(&req->work, file_inode(req->file));
472ed29b0b4SJens Axboe } else if (!req->file || !S_ISBLK(file_inode(req->file)->i_mode)) {
473ed29b0b4SJens Axboe if (def->unbound_nonreg_file)
474ed29b0b4SJens Axboe req->work.flags |= IO_WQ_WORK_UNBOUND;
475ed29b0b4SJens Axboe }
476ed29b0b4SJens Axboe }
477ed29b0b4SJens Axboe
io_prep_async_link(struct io_kiocb * req)478ed29b0b4SJens Axboe static void io_prep_async_link(struct io_kiocb *req)
479ed29b0b4SJens Axboe {
480ed29b0b4SJens Axboe struct io_kiocb *cur;
481ed29b0b4SJens Axboe
482ed29b0b4SJens Axboe if (req->flags & REQ_F_LINK_TIMEOUT) {
483ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
484ed29b0b4SJens Axboe
485ed29b0b4SJens Axboe spin_lock_irq(&ctx->timeout_lock);
486ed29b0b4SJens Axboe io_for_each_link(cur, req)
487ed29b0b4SJens Axboe io_prep_async_work(cur);
488ed29b0b4SJens Axboe spin_unlock_irq(&ctx->timeout_lock);
489ed29b0b4SJens Axboe } else {
490ed29b0b4SJens Axboe io_for_each_link(cur, req)
491ed29b0b4SJens Axboe io_prep_async_work(cur);
492ed29b0b4SJens Axboe }
493ed29b0b4SJens Axboe }
494ed29b0b4SJens Axboe
io_queue_iowq(struct io_kiocb * req,struct io_tw_state * ts_dont_use)495a282967cSPavel Begunkov void io_queue_iowq(struct io_kiocb *req, struct io_tw_state *ts_dont_use)
496ed29b0b4SJens Axboe {
497ed29b0b4SJens Axboe struct io_kiocb *link = io_prep_linked_timeout(req);
498ed29b0b4SJens Axboe struct io_uring_task *tctx = req->task->io_uring;
499ed29b0b4SJens Axboe
500ed29b0b4SJens Axboe BUG_ON(!tctx);
501ed29b0b4SJens Axboe BUG_ON(!tctx->io_wq);
502ed29b0b4SJens Axboe
503ed29b0b4SJens Axboe /* init ->work of the whole link before punting */
504ed29b0b4SJens Axboe io_prep_async_link(req);
505ed29b0b4SJens Axboe
506ed29b0b4SJens Axboe /*
507ed29b0b4SJens Axboe * Not expected to happen, but if we do have a bug where this _can_
508ed29b0b4SJens Axboe * happen, catch it here and ensure the request is marked as
509ed29b0b4SJens Axboe * canceled. That will make io-wq go through the usual work cancel
510ed29b0b4SJens Axboe * procedure rather than attempt to run this request (or create a new
511ed29b0b4SJens Axboe * worker for it).
512ed29b0b4SJens Axboe */
513ed29b0b4SJens Axboe if (WARN_ON_ONCE(!same_thread_group(req->task, current)))
514ed29b0b4SJens Axboe req->work.flags |= IO_WQ_WORK_CANCEL;
515ed29b0b4SJens Axboe
51648863ffdSPavel Begunkov trace_io_uring_queue_async_work(req, io_wq_is_hashed(&req->work));
517ed29b0b4SJens Axboe io_wq_enqueue(tctx->io_wq, &req->work);
518ed29b0b4SJens Axboe if (link)
519ed29b0b4SJens Axboe io_queue_linked_timeout(link);
520ed29b0b4SJens Axboe }
521ed29b0b4SJens Axboe
io_queue_deferred(struct io_ring_ctx * ctx)522ed29b0b4SJens Axboe static __cold void io_queue_deferred(struct io_ring_ctx *ctx)
523ed29b0b4SJens Axboe {
524ed29b0b4SJens Axboe while (!list_empty(&ctx->defer_list)) {
525ed29b0b4SJens Axboe struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
526ed29b0b4SJens Axboe struct io_defer_entry, list);
527ed29b0b4SJens Axboe
528ed29b0b4SJens Axboe if (req_need_defer(de->req, de->seq))
529ed29b0b4SJens Axboe break;
530ed29b0b4SJens Axboe list_del_init(&de->list);
531ed29b0b4SJens Axboe io_req_task_queue(de->req);
532ed29b0b4SJens Axboe kfree(de);
533ed29b0b4SJens Axboe }
534ed29b0b4SJens Axboe }
535ed29b0b4SJens Axboe
53621a091b9SDylan Yudaken
io_eventfd_ops(struct rcu_head * rcu)53721a091b9SDylan Yudaken static void io_eventfd_ops(struct rcu_head *rcu)
538d8e9214fSDylan Yudaken {
539d8e9214fSDylan Yudaken struct io_ev_fd *ev_fd = container_of(rcu, struct io_ev_fd, rcu);
54021a091b9SDylan Yudaken int ops = atomic_xchg(&ev_fd->ops, 0);
541d8e9214fSDylan Yudaken
54221a091b9SDylan Yudaken if (ops & BIT(IO_EVENTFD_OP_SIGNAL_BIT))
54344648532SJens Axboe eventfd_signal_mask(ev_fd->cq_ev_fd, 1, EPOLL_URING_WAKE);
54421a091b9SDylan Yudaken
54521a091b9SDylan Yudaken /* IO_EVENTFD_OP_FREE_BIT may not be set here depending on callback
54621a091b9SDylan Yudaken * ordering in a race but if references are 0 we know we have to free
54721a091b9SDylan Yudaken * it regardless.
54821a091b9SDylan Yudaken */
54921a091b9SDylan Yudaken if (atomic_dec_and_test(&ev_fd->refs)) {
550d8e9214fSDylan Yudaken eventfd_ctx_put(ev_fd->cq_ev_fd);
551d8e9214fSDylan Yudaken kfree(ev_fd);
552d8e9214fSDylan Yudaken }
55321a091b9SDylan Yudaken }
554d8e9214fSDylan Yudaken
io_eventfd_signal(struct io_ring_ctx * ctx)555ed29b0b4SJens Axboe static void io_eventfd_signal(struct io_ring_ctx *ctx)
556ed29b0b4SJens Axboe {
55721a091b9SDylan Yudaken struct io_ev_fd *ev_fd = NULL;
558ed29b0b4SJens Axboe
559ed29b0b4SJens Axboe rcu_read_lock();
560ed29b0b4SJens Axboe /*
561ed29b0b4SJens Axboe * rcu_dereference ctx->io_ev_fd once and use it for both for checking
562ed29b0b4SJens Axboe * and eventfd_signal
563ed29b0b4SJens Axboe */
564ed29b0b4SJens Axboe ev_fd = rcu_dereference(ctx->io_ev_fd);
565ed29b0b4SJens Axboe
566ed29b0b4SJens Axboe /*
567ed29b0b4SJens Axboe * Check again if ev_fd exists incase an io_eventfd_unregister call
568ed29b0b4SJens Axboe * completed between the NULL check of ctx->io_ev_fd at the start of
569ed29b0b4SJens Axboe * the function and rcu_read_lock.
570ed29b0b4SJens Axboe */
571ed29b0b4SJens Axboe if (unlikely(!ev_fd))
572ed29b0b4SJens Axboe goto out;
573ed29b0b4SJens Axboe if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
574ed29b0b4SJens Axboe goto out;
57521a091b9SDylan Yudaken if (ev_fd->eventfd_async && !io_wq_current_is_worker())
57621a091b9SDylan Yudaken goto out;
577ed29b0b4SJens Axboe
57821a091b9SDylan Yudaken if (likely(eventfd_signal_allowed())) {
57944648532SJens Axboe eventfd_signal_mask(ev_fd->cq_ev_fd, 1, EPOLL_URING_WAKE);
58021a091b9SDylan Yudaken } else {
58121a091b9SDylan Yudaken atomic_inc(&ev_fd->refs);
58221a091b9SDylan Yudaken if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_SIGNAL_BIT), &ev_fd->ops))
58344a84da4SDylan Yudaken call_rcu_hurry(&ev_fd->rcu, io_eventfd_ops);
58421a091b9SDylan Yudaken else
58521a091b9SDylan Yudaken atomic_dec(&ev_fd->refs);
58621a091b9SDylan Yudaken }
58721a091b9SDylan Yudaken
588ed29b0b4SJens Axboe out:
589ed29b0b4SJens Axboe rcu_read_unlock();
590ed29b0b4SJens Axboe }
591ed29b0b4SJens Axboe
io_eventfd_flush_signal(struct io_ring_ctx * ctx)59221a091b9SDylan Yudaken static void io_eventfd_flush_signal(struct io_ring_ctx *ctx)
59321a091b9SDylan Yudaken {
59421a091b9SDylan Yudaken bool skip;
59521a091b9SDylan Yudaken
59621a091b9SDylan Yudaken spin_lock(&ctx->completion_lock);
59721a091b9SDylan Yudaken
59821a091b9SDylan Yudaken /*
59921a091b9SDylan Yudaken * Eventfd should only get triggered when at least one event has been
60021a091b9SDylan Yudaken * posted. Some applications rely on the eventfd notification count
60121a091b9SDylan Yudaken * only changing IFF a new CQE has been added to the CQ ring. There's
60221a091b9SDylan Yudaken * no depedency on 1:1 relationship between how many times this
60321a091b9SDylan Yudaken * function is called (and hence the eventfd count) and number of CQEs
60421a091b9SDylan Yudaken * posted to the CQ ring.
60521a091b9SDylan Yudaken */
60621a091b9SDylan Yudaken skip = ctx->cached_cq_tail == ctx->evfd_last_cq_tail;
60721a091b9SDylan Yudaken ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
60821a091b9SDylan Yudaken spin_unlock(&ctx->completion_lock);
60921a091b9SDylan Yudaken if (skip)
61021a091b9SDylan Yudaken return;
61121a091b9SDylan Yudaken
61221a091b9SDylan Yudaken io_eventfd_signal(ctx);
61321a091b9SDylan Yudaken }
61421a091b9SDylan Yudaken
__io_commit_cqring_flush(struct io_ring_ctx * ctx)615a830ffd2SPavel Begunkov void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
616a830ffd2SPavel Begunkov {
617bca39f39SPavel Begunkov if (ctx->poll_activated)
618bca39f39SPavel Begunkov io_poll_wq_wake(ctx);
619a830ffd2SPavel Begunkov if (ctx->off_timeout_used)
620a830ffd2SPavel Begunkov io_flush_timeouts(ctx);
621e5f30f6fSPavel Begunkov if (ctx->drain_active) {
622e5f30f6fSPavel Begunkov spin_lock(&ctx->completion_lock);
623a830ffd2SPavel Begunkov io_queue_deferred(ctx);
624a830ffd2SPavel Begunkov spin_unlock(&ctx->completion_lock);
625a830ffd2SPavel Begunkov }
626a830ffd2SPavel Begunkov if (ctx->has_evfd)
62721a091b9SDylan Yudaken io_eventfd_flush_signal(ctx);
628a830ffd2SPavel Begunkov }
629a830ffd2SPavel Begunkov
__io_cq_lock(struct io_ring_ctx * ctx)630f66f7342SPavel Begunkov static inline void __io_cq_lock(struct io_ring_ctx *ctx)
631f66f7342SPavel Begunkov {
632ec26c225SPavel Begunkov if (!ctx->lockless_cq)
633f66f7342SPavel Begunkov spin_lock(&ctx->completion_lock);
634f66f7342SPavel Begunkov }
635f66f7342SPavel Begunkov
io_cq_lock(struct io_ring_ctx * ctx)6366971253fSPavel Begunkov static inline void io_cq_lock(struct io_ring_ctx *ctx)
6376971253fSPavel Begunkov __acquires(ctx->completion_lock)
6386971253fSPavel Begunkov {
6396971253fSPavel Begunkov spin_lock(&ctx->completion_lock);
6406971253fSPavel Begunkov }
6416971253fSPavel Begunkov
__io_cq_unlock_post(struct io_ring_ctx * ctx)642f66f7342SPavel Begunkov static inline void __io_cq_unlock_post(struct io_ring_ctx *ctx)
64325399321SPavel Begunkov {
64425399321SPavel Begunkov io_commit_cqring(ctx);
64554927bafSPavel Begunkov if (!ctx->task_complete) {
646ec26c225SPavel Begunkov if (!ctx->lockless_cq)
647ff126177SPavel Begunkov spin_unlock(&ctx->completion_lock);
648ec26c225SPavel Begunkov /* IOPOLL rings only need to wake up if it's also SQPOLL */
649ec26c225SPavel Begunkov if (!ctx->syscall_iopoll)
6506e7248adSPavel Begunkov io_cqring_wake(ctx);
6513181e22fSPavel Begunkov }
65254927bafSPavel Begunkov io_commit_cqring_flush(ctx);
6533181e22fSPavel Begunkov }
6543181e22fSPavel Begunkov
io_cq_unlock_post(struct io_ring_ctx * ctx)6550fdb9a19SPavel Begunkov static void io_cq_unlock_post(struct io_ring_ctx *ctx)
6565d772916SPavel Begunkov __releases(ctx->completion_lock)
65725399321SPavel Begunkov {
658f66f7342SPavel Begunkov io_commit_cqring(ctx);
659f66f7342SPavel Begunkov spin_unlock(&ctx->completion_lock);
660f66f7342SPavel Begunkov io_cqring_wake(ctx);
66154927bafSPavel Begunkov io_commit_cqring_flush(ctx);
66225399321SPavel Begunkov }
66325399321SPavel Begunkov
664ed29b0b4SJens Axboe /* Returns true if there are no backlogged entries after the flush */
io_cqring_overflow_kill(struct io_ring_ctx * ctx)665a85381d8SPavel Begunkov static void io_cqring_overflow_kill(struct io_ring_ctx *ctx)
666ed29b0b4SJens Axboe {
667a85381d8SPavel Begunkov struct io_overflow_cqe *ocqe;
668a85381d8SPavel Begunkov LIST_HEAD(list);
669a85381d8SPavel Begunkov
670f432b76bSPavel Begunkov spin_lock(&ctx->completion_lock);
671a85381d8SPavel Begunkov list_splice_init(&ctx->cq_overflow_list, &list);
672a85381d8SPavel Begunkov clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
673f432b76bSPavel Begunkov spin_unlock(&ctx->completion_lock);
674a85381d8SPavel Begunkov
675a85381d8SPavel Begunkov while (!list_empty(&list)) {
676a85381d8SPavel Begunkov ocqe = list_first_entry(&list, struct io_overflow_cqe, list);
677a85381d8SPavel Begunkov list_del(&ocqe->list);
678a85381d8SPavel Begunkov kfree(ocqe);
679a85381d8SPavel Begunkov }
680a85381d8SPavel Begunkov }
681a85381d8SPavel Begunkov
__io_cqring_overflow_flush(struct io_ring_ctx * ctx)6821b346e4aSPavel Begunkov static void __io_cqring_overflow_flush(struct io_ring_ctx *ctx)
683ed29b0b4SJens Axboe {
684ed29b0b4SJens Axboe size_t cqe_size = sizeof(struct io_uring_cqe);
685ed29b0b4SJens Axboe
686a85381d8SPavel Begunkov if (__io_cqring_events(ctx) == ctx->cq_entries)
6871b346e4aSPavel Begunkov return;
688ed29b0b4SJens Axboe
689ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_CQE32)
690ed29b0b4SJens Axboe cqe_size <<= 1;
691ed29b0b4SJens Axboe
69225399321SPavel Begunkov io_cq_lock(ctx);
693ed29b0b4SJens Axboe while (!list_empty(&ctx->cq_overflow_list)) {
69459fbc409SPavel Begunkov struct io_uring_cqe *cqe;
695ed29b0b4SJens Axboe struct io_overflow_cqe *ocqe;
696ed29b0b4SJens Axboe
69759fbc409SPavel Begunkov if (!io_get_cqe_overflow(ctx, &cqe, true))
698ed29b0b4SJens Axboe break;
699ed29b0b4SJens Axboe ocqe = list_first_entry(&ctx->cq_overflow_list,
700ed29b0b4SJens Axboe struct io_overflow_cqe, list);
701ed29b0b4SJens Axboe memcpy(cqe, &ocqe->cqe, cqe_size);
702ed29b0b4SJens Axboe list_del(&ocqe->list);
703ed29b0b4SJens Axboe kfree(ocqe);
704ed29b0b4SJens Axboe }
705ed29b0b4SJens Axboe
7061b346e4aSPavel Begunkov if (list_empty(&ctx->cq_overflow_list)) {
707ed29b0b4SJens Axboe clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
708ed29b0b4SJens Axboe atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
709ed29b0b4SJens Axboe }
71025399321SPavel Begunkov io_cq_unlock_post(ctx);
711ed29b0b4SJens Axboe }
712ed29b0b4SJens Axboe
io_cqring_do_overflow_flush(struct io_ring_ctx * ctx)71352ea806aSJens Axboe static void io_cqring_do_overflow_flush(struct io_ring_ctx *ctx)
714ed29b0b4SJens Axboe {
715ed29b0b4SJens Axboe /* iopoll syncs against uring_lock, not completion_lock */
716ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_IOPOLL)
717ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
7181b346e4aSPavel Begunkov __io_cqring_overflow_flush(ctx);
719ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_IOPOLL)
720ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
721ed29b0b4SJens Axboe }
72252ea806aSJens Axboe
io_cqring_overflow_flush(struct io_ring_ctx * ctx)72352ea806aSJens Axboe static void io_cqring_overflow_flush(struct io_ring_ctx *ctx)
72452ea806aSJens Axboe {
72552ea806aSJens Axboe if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
72652ea806aSJens Axboe io_cqring_do_overflow_flush(ctx);
727ed29b0b4SJens Axboe }
728ed29b0b4SJens Axboe
7295afa4650SPavel Begunkov /* can be called by any task */
io_put_task_remote(struct task_struct * task)7302fdd6fb5SPavel Begunkov static void io_put_task_remote(struct task_struct *task)
731ed29b0b4SJens Axboe {
732ed29b0b4SJens Axboe struct io_uring_task *tctx = task->io_uring;
733ed29b0b4SJens Axboe
7342fdd6fb5SPavel Begunkov percpu_counter_sub(&tctx->inflight, 1);
7358d664282SJens Axboe if (unlikely(atomic_read(&tctx->in_cancel)))
736ed29b0b4SJens Axboe wake_up(&tctx->wait);
7372fdd6fb5SPavel Begunkov put_task_struct(task);
738ed29b0b4SJens Axboe }
739ed29b0b4SJens Axboe
7405afa4650SPavel Begunkov /* used by a task to put its own references */
io_put_task_local(struct task_struct * task)7412fdd6fb5SPavel Begunkov static void io_put_task_local(struct task_struct *task)
7425afa4650SPavel Begunkov {
7432fdd6fb5SPavel Begunkov task->io_uring->cached_refs++;
7445afa4650SPavel Begunkov }
7455afa4650SPavel Begunkov
74689800a2dSPavel Begunkov /* must to be called somewhat shortly after putting a request */
io_put_task(struct task_struct * task)7472fdd6fb5SPavel Begunkov static inline void io_put_task(struct task_struct *task)
74889800a2dSPavel Begunkov {
74989800a2dSPavel Begunkov if (likely(task == current))
7502fdd6fb5SPavel Begunkov io_put_task_local(task);
75189800a2dSPavel Begunkov else
7522fdd6fb5SPavel Begunkov io_put_task_remote(task);
75389800a2dSPavel Begunkov }
75489800a2dSPavel Begunkov
io_task_refs_refill(struct io_uring_task * tctx)75563809137SPavel Begunkov void io_task_refs_refill(struct io_uring_task *tctx)
756ed29b0b4SJens Axboe {
757ed29b0b4SJens Axboe unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
758ed29b0b4SJens Axboe
759ed29b0b4SJens Axboe percpu_counter_add(&tctx->inflight, refill);
760ed29b0b4SJens Axboe refcount_add(refill, ¤t->usage);
761ed29b0b4SJens Axboe tctx->cached_refs += refill;
762ed29b0b4SJens Axboe }
763ed29b0b4SJens Axboe
io_uring_drop_tctx_refs(struct task_struct * task)764ed29b0b4SJens Axboe static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
765ed29b0b4SJens Axboe {
766ed29b0b4SJens Axboe struct io_uring_task *tctx = task->io_uring;
767ed29b0b4SJens Axboe unsigned int refs = tctx->cached_refs;
768ed29b0b4SJens Axboe
769ed29b0b4SJens Axboe if (refs) {
770ed29b0b4SJens Axboe tctx->cached_refs = 0;
771ed29b0b4SJens Axboe percpu_counter_sub(&tctx->inflight, refs);
772ed29b0b4SJens Axboe put_task_struct_many(task, refs);
773ed29b0b4SJens Axboe }
774ed29b0b4SJens Axboe }
775ed29b0b4SJens Axboe
io_cqring_event_overflow(struct io_ring_ctx * ctx,u64 user_data,s32 res,u32 cflags,u64 extra1,u64 extra2)77668494a65SPavel Begunkov static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
77768494a65SPavel Begunkov s32 res, u32 cflags, u64 extra1, u64 extra2)
778ed29b0b4SJens Axboe {
779ed29b0b4SJens Axboe struct io_overflow_cqe *ocqe;
780ed29b0b4SJens Axboe size_t ocq_size = sizeof(struct io_overflow_cqe);
781ed29b0b4SJens Axboe bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
782ed29b0b4SJens Axboe
783f26cc959SPavel Begunkov lockdep_assert_held(&ctx->completion_lock);
784f26cc959SPavel Begunkov
785ed29b0b4SJens Axboe if (is_cqe32)
786ed29b0b4SJens Axboe ocq_size += sizeof(struct io_uring_cqe);
787ed29b0b4SJens Axboe
788ed29b0b4SJens Axboe ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
789ed29b0b4SJens Axboe trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
790ed29b0b4SJens Axboe if (!ocqe) {
791ed29b0b4SJens Axboe /*
792ed29b0b4SJens Axboe * If we're in ring overflow flush mode, or in task cancel mode,
793ed29b0b4SJens Axboe * or cannot allocate an overflow entry, then we need to drop it
794ed29b0b4SJens Axboe * on the floor.
795ed29b0b4SJens Axboe */
796ed29b0b4SJens Axboe io_account_cq_overflow(ctx);
797ed29b0b4SJens Axboe set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
798ed29b0b4SJens Axboe return false;
799ed29b0b4SJens Axboe }
800ed29b0b4SJens Axboe if (list_empty(&ctx->cq_overflow_list)) {
801ed29b0b4SJens Axboe set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
802ed29b0b4SJens Axboe atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
803ed29b0b4SJens Axboe
804ed29b0b4SJens Axboe }
805ed29b0b4SJens Axboe ocqe->cqe.user_data = user_data;
806ed29b0b4SJens Axboe ocqe->cqe.res = res;
807ed29b0b4SJens Axboe ocqe->cqe.flags = cflags;
808ed29b0b4SJens Axboe if (is_cqe32) {
809ed29b0b4SJens Axboe ocqe->cqe.big_cqe[0] = extra1;
810ed29b0b4SJens Axboe ocqe->cqe.big_cqe[1] = extra2;
811ed29b0b4SJens Axboe }
812ed29b0b4SJens Axboe list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
813ed29b0b4SJens Axboe return true;
814ed29b0b4SJens Axboe }
815ed29b0b4SJens Axboe
io_req_cqe_overflow(struct io_kiocb * req)816056695bfSPavel Begunkov void io_req_cqe_overflow(struct io_kiocb *req)
81768494a65SPavel Begunkov {
818056695bfSPavel Begunkov io_cqring_event_overflow(req->ctx, req->cqe.user_data,
81968494a65SPavel Begunkov req->cqe.res, req->cqe.flags,
820b24c5d75SPavel Begunkov req->big_cqe.extra1, req->big_cqe.extra2);
821b24c5d75SPavel Begunkov memset(&req->big_cqe, 0, sizeof(req->big_cqe));
82268494a65SPavel Begunkov }
82368494a65SPavel Begunkov
824faf88ddeSPavel Begunkov /*
825faf88ddeSPavel Begunkov * writes to the cq entry need to come after reading head; the
826faf88ddeSPavel Begunkov * control dependency is enough as we're using WRITE_ONCE to
827faf88ddeSPavel Begunkov * fill the cq entry
828faf88ddeSPavel Begunkov */
io_cqe_cache_refill(struct io_ring_ctx * ctx,bool overflow)82920d6b633SPavel Begunkov bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow)
830faf88ddeSPavel Begunkov {
831faf88ddeSPavel Begunkov struct io_rings *rings = ctx->rings;
832faf88ddeSPavel Begunkov unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
833faf88ddeSPavel Begunkov unsigned int free, queued, len;
834faf88ddeSPavel Begunkov
835aa1df3a3SPavel Begunkov /*
836aa1df3a3SPavel Begunkov * Posting into the CQ when there are pending overflowed CQEs may break
837aa1df3a3SPavel Begunkov * ordering guarantees, which will affect links, F_MORE users and more.
838aa1df3a3SPavel Begunkov * Force overflow the completion.
839aa1df3a3SPavel Begunkov */
840aa1df3a3SPavel Begunkov if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
84120d6b633SPavel Begunkov return false;
842faf88ddeSPavel Begunkov
843faf88ddeSPavel Begunkov /* userspace may cheat modifying the tail, be safe and do min */
844faf88ddeSPavel Begunkov queued = min(__io_cqring_events(ctx), ctx->cq_entries);
845faf88ddeSPavel Begunkov free = ctx->cq_entries - queued;
846faf88ddeSPavel Begunkov /* we need a contiguous range, limit based on the current array offset */
847faf88ddeSPavel Begunkov len = min(free, ctx->cq_entries - off);
848faf88ddeSPavel Begunkov if (!len)
84920d6b633SPavel Begunkov return false;
850faf88ddeSPavel Begunkov
851b3659a65SPavel Begunkov if (ctx->flags & IORING_SETUP_CQE32) {
852b3659a65SPavel Begunkov off <<= 1;
853b3659a65SPavel Begunkov len <<= 1;
854b3659a65SPavel Begunkov }
855b3659a65SPavel Begunkov
856faf88ddeSPavel Begunkov ctx->cqe_cached = &rings->cqes[off];
857faf88ddeSPavel Begunkov ctx->cqe_sentinel = ctx->cqe_cached + len;
85820d6b633SPavel Begunkov return true;
859faf88ddeSPavel Begunkov }
860faf88ddeSPavel Begunkov
io_fill_cqe_aux(struct io_ring_ctx * ctx,u64 user_data,s32 res,u32 cflags)861f66f7342SPavel Begunkov static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
862f66f7342SPavel Begunkov u32 cflags)
863ed29b0b4SJens Axboe {
864ed29b0b4SJens Axboe struct io_uring_cqe *cqe;
865ed29b0b4SJens Axboe
866ed29b0b4SJens Axboe ctx->cq_extra++;
867ed29b0b4SJens Axboe
868ed29b0b4SJens Axboe /*
869ed29b0b4SJens Axboe * If we can't get a cq entry, userspace overflowed the
870ed29b0b4SJens Axboe * submission (by quite a lot). Increment the overflow count in
871ed29b0b4SJens Axboe * the ring.
872ed29b0b4SJens Axboe */
87359fbc409SPavel Begunkov if (likely(io_get_cqe(ctx, &cqe))) {
874e0486f3fSDylan Yudaken trace_io_uring_complete(ctx, NULL, user_data, res, cflags, 0, 0);
875e0486f3fSDylan Yudaken
876ed29b0b4SJens Axboe WRITE_ONCE(cqe->user_data, user_data);
877ed29b0b4SJens Axboe WRITE_ONCE(cqe->res, res);
878ed29b0b4SJens Axboe WRITE_ONCE(cqe->flags, cflags);
879ed29b0b4SJens Axboe
880ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_CQE32) {
881ed29b0b4SJens Axboe WRITE_ONCE(cqe->big_cqe[0], 0);
882ed29b0b4SJens Axboe WRITE_ONCE(cqe->big_cqe[1], 0);
883ed29b0b4SJens Axboe }
884ed29b0b4SJens Axboe return true;
885ed29b0b4SJens Axboe }
88652120f0fSDylan Yudaken return false;
887ed29b0b4SJens Axboe }
888ed29b0b4SJens Axboe
__io_flush_post_cqes(struct io_ring_ctx * ctx)889931147ddSDylan Yudaken static void __io_flush_post_cqes(struct io_ring_ctx *ctx)
890931147ddSDylan Yudaken __must_hold(&ctx->uring_lock)
891931147ddSDylan Yudaken {
892931147ddSDylan Yudaken struct io_submit_state *state = &ctx->submit_state;
893931147ddSDylan Yudaken unsigned int i;
894931147ddSDylan Yudaken
895931147ddSDylan Yudaken lockdep_assert_held(&ctx->uring_lock);
896931147ddSDylan Yudaken for (i = 0; i < state->cqes_count; i++) {
8970aa7aa5fSPavel Begunkov struct io_uring_cqe *cqe = &ctx->completion_cqes[i];
898931147ddSDylan Yudaken
899f66f7342SPavel Begunkov if (!io_fill_cqe_aux(ctx, cqe->user_data, cqe->res, cqe->flags)) {
90027122c07SPavel Begunkov if (ctx->lockless_cq) {
901f66f7342SPavel Begunkov spin_lock(&ctx->completion_lock);
902f66f7342SPavel Begunkov io_cqring_event_overflow(ctx, cqe->user_data,
903f66f7342SPavel Begunkov cqe->res, cqe->flags, 0, 0);
904f66f7342SPavel Begunkov spin_unlock(&ctx->completion_lock);
905f66f7342SPavel Begunkov } else {
906f66f7342SPavel Begunkov io_cqring_event_overflow(ctx, cqe->user_data,
907f66f7342SPavel Begunkov cqe->res, cqe->flags, 0, 0);
908f66f7342SPavel Begunkov }
909f66f7342SPavel Begunkov }
910931147ddSDylan Yudaken }
911931147ddSDylan Yudaken state->cqes_count = 0;
912931147ddSDylan Yudaken }
913931147ddSDylan Yudaken
__io_post_aux_cqe(struct io_ring_ctx * ctx,u64 user_data,s32 res,u32 cflags,bool allow_overflow)914b529c96aSDylan Yudaken static bool __io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags,
91552120f0fSDylan Yudaken bool allow_overflow)
916d245bca6SPavel Begunkov {
917d245bca6SPavel Begunkov bool filled;
918d245bca6SPavel Begunkov
91925399321SPavel Begunkov io_cq_lock(ctx);
920f66f7342SPavel Begunkov filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
921f66f7342SPavel Begunkov if (!filled && allow_overflow)
922f66f7342SPavel Begunkov filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
923f66f7342SPavel Begunkov
92425399321SPavel Begunkov io_cq_unlock_post(ctx);
925d245bca6SPavel Begunkov return filled;
926d245bca6SPavel Begunkov }
927d245bca6SPavel Begunkov
io_post_aux_cqe(struct io_ring_ctx * ctx,u64 user_data,s32 res,u32 cflags)928b529c96aSDylan Yudaken bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
929ed29b0b4SJens Axboe {
930b529c96aSDylan Yudaken return __io_post_aux_cqe(ctx, user_data, res, cflags, true);
931b529c96aSDylan Yudaken }
932b529c96aSDylan Yudaken
933b6b2bb58SPavel Begunkov /*
934b6b2bb58SPavel Begunkov * A helper for multishot requests posting additional CQEs.
935b6b2bb58SPavel Begunkov * Should only be used from a task_work including IO_URING_F_MULTISHOT.
936b6b2bb58SPavel Begunkov */
io_fill_cqe_req_aux(struct io_kiocb * req,bool defer,s32 res,u32 cflags)937b6b2bb58SPavel Begunkov bool io_fill_cqe_req_aux(struct io_kiocb *req, bool defer, s32 res, u32 cflags)
9389b8c5475SDylan Yudaken {
939d86eaed1SJens Axboe struct io_ring_ctx *ctx = req->ctx;
940d86eaed1SJens Axboe u64 user_data = req->cqe.user_data;
9419b8c5475SDylan Yudaken struct io_uring_cqe *cqe;
9429b8c5475SDylan Yudaken
9439b8c5475SDylan Yudaken if (!defer)
944b6b2bb58SPavel Begunkov return __io_post_aux_cqe(ctx, user_data, res, cflags, false);
9459b8c5475SDylan Yudaken
9469b8c5475SDylan Yudaken lockdep_assert_held(&ctx->uring_lock);
9479b8c5475SDylan Yudaken
9480aa7aa5fSPavel Begunkov if (ctx->submit_state.cqes_count == ARRAY_SIZE(ctx->completion_cqes)) {
949f66f7342SPavel Begunkov __io_cq_lock(ctx);
9509b8c5475SDylan Yudaken __io_flush_post_cqes(ctx);
9519b8c5475SDylan Yudaken /* no need to flush - flush is deferred */
952f66f7342SPavel Begunkov __io_cq_unlock_post(ctx);
9539b8c5475SDylan Yudaken }
9549b8c5475SDylan Yudaken
9559b8c5475SDylan Yudaken /* For defered completions this is not as strict as it is otherwise,
9569b8c5475SDylan Yudaken * however it's main job is to prevent unbounded posted completions,
9579b8c5475SDylan Yudaken * and in that it works just as well.
9589b8c5475SDylan Yudaken */
959b6b2bb58SPavel Begunkov if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq))
9609b8c5475SDylan Yudaken return false;
9619b8c5475SDylan Yudaken
9620aa7aa5fSPavel Begunkov cqe = &ctx->completion_cqes[ctx->submit_state.cqes_count++];
9639b8c5475SDylan Yudaken cqe->user_data = user_data;
9649b8c5475SDylan Yudaken cqe->res = res;
9659b8c5475SDylan Yudaken cqe->flags = cflags;
9669b8c5475SDylan Yudaken return true;
9679b8c5475SDylan Yudaken }
9689b8c5475SDylan Yudaken
__io_req_complete_post(struct io_kiocb * req,unsigned issue_flags)969ef8ae64fSPavel Begunkov static void __io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
970ed29b0b4SJens Axboe {
971fa18fa22SPavel Begunkov struct io_ring_ctx *ctx = req->ctx;
9722ad4c6d0SPavel Begunkov struct io_rsrc_node *rsrc_node = NULL;
973fa18fa22SPavel Begunkov
974fa18fa22SPavel Begunkov io_cq_lock(ctx);
97500b0db56SPavel Begunkov if (!(req->flags & REQ_F_CQE_SKIP)) {
97600b0db56SPavel Begunkov if (!io_fill_cqe_req(ctx, req))
97700b0db56SPavel Begunkov io_req_cqe_overflow(req);
97800b0db56SPavel Begunkov }
979fa18fa22SPavel Begunkov
980ed29b0b4SJens Axboe /*
981ed29b0b4SJens Axboe * If we're the last reference to this request, add to our locked
982ed29b0b4SJens Axboe * free_list cache.
983ed29b0b4SJens Axboe */
984ed29b0b4SJens Axboe if (req_ref_put_and_test(req)) {
985ed29b0b4SJens Axboe if (req->flags & IO_REQ_LINK_FLAGS) {
986ed29b0b4SJens Axboe if (req->flags & IO_DISARM_MASK)
987ed29b0b4SJens Axboe io_disarm_next(req);
988ed29b0b4SJens Axboe if (req->link) {
989ed29b0b4SJens Axboe io_req_task_queue(req->link);
990ed29b0b4SJens Axboe req->link = NULL;
991ed29b0b4SJens Axboe }
992ed29b0b4SJens Axboe }
99368a2cc1bSPavel Begunkov io_put_kbuf_comp(req);
9943b7a612fSPavel Begunkov if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
9953b7a612fSPavel Begunkov io_clean_op(req);
99617bc2837SJens Axboe io_put_file(req);
9973b7a612fSPavel Begunkov
9982ad4c6d0SPavel Begunkov rsrc_node = req->rsrc_node;
999ed29b0b4SJens Axboe /*
1000ed29b0b4SJens Axboe * Selected buffer deallocation in io_clean_op() assumes that
1001ed29b0b4SJens Axboe * we don't hold ->completion_lock. Clean them here to avoid
1002ed29b0b4SJens Axboe * deadlocks.
1003ed29b0b4SJens Axboe */
10042fdd6fb5SPavel Begunkov io_put_task_remote(req->task);
1005ed29b0b4SJens Axboe wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
1006ed29b0b4SJens Axboe ctx->locked_free_nr++;
1007ed29b0b4SJens Axboe }
100825399321SPavel Begunkov io_cq_unlock_post(ctx);
10092ad4c6d0SPavel Begunkov
1010ef8ae64fSPavel Begunkov if (rsrc_node) {
1011ef8ae64fSPavel Begunkov io_ring_submit_lock(ctx, issue_flags);
10121f2c8f61SPavel Begunkov io_put_rsrc_node(ctx, rsrc_node);
1013ef8ae64fSPavel Begunkov io_ring_submit_unlock(ctx, issue_flags);
1014ef8ae64fSPavel Begunkov }
1015ed29b0b4SJens Axboe }
1016ed29b0b4SJens Axboe
io_req_complete_post(struct io_kiocb * req,unsigned issue_flags)10171bec951cSPavel Begunkov void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
1018ed29b0b4SJens Axboe {
1019860e1c7fSMing Lei if (req->ctx->task_complete && req->ctx->submitter_task != current) {
1020e6aeb272SPavel Begunkov req->io_task_work.func = io_req_task_complete;
1021e6aeb272SPavel Begunkov io_req_task_work_add(req);
1022e6aeb272SPavel Begunkov } else if (!(issue_flags & IO_URING_F_UNLOCKED) ||
10231bec951cSPavel Begunkov !(req->ctx->flags & IORING_SETUP_IOPOLL)) {
1024ef8ae64fSPavel Begunkov __io_req_complete_post(req, issue_flags);
10251bec951cSPavel Begunkov } else {
10261bec951cSPavel Begunkov struct io_ring_ctx *ctx = req->ctx;
10271bec951cSPavel Begunkov
10281bec951cSPavel Begunkov mutex_lock(&ctx->uring_lock);
1029ef8ae64fSPavel Begunkov __io_req_complete_post(req, issue_flags & ~IO_URING_F_UNLOCKED);
10301bec951cSPavel Begunkov mutex_unlock(&ctx->uring_lock);
10311bec951cSPavel Begunkov }
1032ed29b0b4SJens Axboe }
1033ed29b0b4SJens Axboe
io_req_defer_failed(struct io_kiocb * req,s32 res)1034973fc83fSDylan Yudaken void io_req_defer_failed(struct io_kiocb *req, s32 res)
1035e276ae34SPavel Begunkov __must_hold(&ctx->uring_lock)
1036ed29b0b4SJens Axboe {
1037f30bd4d0SBreno Leitao const struct io_cold_def *def = &io_cold_defs[req->opcode];
1038a47b255eSPavel Begunkov
1039e276ae34SPavel Begunkov lockdep_assert_held(&req->ctx->uring_lock);
1040e276ae34SPavel Begunkov
1041ed29b0b4SJens Axboe req_set_fail(req);
104297b388d7SJens Axboe io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED));
1043a47b255eSPavel Begunkov if (def->fail)
1044a47b255eSPavel Begunkov def->fail(req);
1045973fc83fSDylan Yudaken io_req_complete_defer(req);
1046ed29b0b4SJens Axboe }
1047ed29b0b4SJens Axboe
1048ed29b0b4SJens Axboe /*
1049ed29b0b4SJens Axboe * Don't initialise the fields below on every allocation, but do that in
1050ed29b0b4SJens Axboe * advance and keep them valid across allocations.
1051ed29b0b4SJens Axboe */
io_preinit_req(struct io_kiocb * req,struct io_ring_ctx * ctx)1052ed29b0b4SJens Axboe static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1053ed29b0b4SJens Axboe {
1054ed29b0b4SJens Axboe req->ctx = ctx;
1055ed29b0b4SJens Axboe req->link = NULL;
1056ed29b0b4SJens Axboe req->async_data = NULL;
1057ed29b0b4SJens Axboe /* not necessary, but safer to zero */
105831d3ba92SPavel Begunkov memset(&req->cqe, 0, sizeof(req->cqe));
1059b24c5d75SPavel Begunkov memset(&req->big_cqe, 0, sizeof(req->big_cqe));
1060ed29b0b4SJens Axboe }
1061ed29b0b4SJens Axboe
io_flush_cached_locked_reqs(struct io_ring_ctx * ctx,struct io_submit_state * state)1062ed29b0b4SJens Axboe static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
1063ed29b0b4SJens Axboe struct io_submit_state *state)
1064ed29b0b4SJens Axboe {
1065ed29b0b4SJens Axboe spin_lock(&ctx->completion_lock);
1066ed29b0b4SJens Axboe wq_list_splice(&ctx->locked_free_list, &state->free_list);
1067ed29b0b4SJens Axboe ctx->locked_free_nr = 0;
1068ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
1069ed29b0b4SJens Axboe }
1070ed29b0b4SJens Axboe
1071ed29b0b4SJens Axboe /*
1072ed29b0b4SJens Axboe * A request might get retired back into the request caches even before opcode
1073ed29b0b4SJens Axboe * handlers and io_issue_sqe() are done with it, e.g. inline completion path.
1074ed29b0b4SJens Axboe * Because of that, io_alloc_req() should be called only under ->uring_lock
1075ed29b0b4SJens Axboe * and with extra caution to not get a request that is still worked on.
1076ed29b0b4SJens Axboe */
__io_alloc_req_refill(struct io_ring_ctx * ctx)1077bd1a3783SPavel Begunkov __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
1078ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
1079ed29b0b4SJens Axboe {
1080ed29b0b4SJens Axboe gfp_t gfp = GFP_KERNEL | __GFP_NOWARN;
1081ed29b0b4SJens Axboe void *reqs[IO_REQ_ALLOC_BATCH];
1082ed29b0b4SJens Axboe int ret, i;
1083ed29b0b4SJens Axboe
1084ed29b0b4SJens Axboe /*
1085ed29b0b4SJens Axboe * If we have more than a batch's worth of requests in our IRQ side
1086ed29b0b4SJens Axboe * locked cache, grab the lock and move them over to our submission
1087ed29b0b4SJens Axboe * side cache.
1088ed29b0b4SJens Axboe */
1089ed29b0b4SJens Axboe if (data_race(ctx->locked_free_nr) > IO_COMPL_BATCH) {
1090ed29b0b4SJens Axboe io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
1091ed29b0b4SJens Axboe if (!io_req_cache_empty(ctx))
1092ed29b0b4SJens Axboe return true;
1093ed29b0b4SJens Axboe }
1094ed29b0b4SJens Axboe
1095ed29b0b4SJens Axboe ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
1096ed29b0b4SJens Axboe
1097ed29b0b4SJens Axboe /*
1098ed29b0b4SJens Axboe * Bulk alloc is all-or-nothing. If we fail to get a batch,
1099ed29b0b4SJens Axboe * retry single alloc to be on the safe side.
1100ed29b0b4SJens Axboe */
1101ed29b0b4SJens Axboe if (unlikely(ret <= 0)) {
1102ed29b0b4SJens Axboe reqs[0] = kmem_cache_alloc(req_cachep, gfp);
1103ed29b0b4SJens Axboe if (!reqs[0])
1104ed29b0b4SJens Axboe return false;
1105ed29b0b4SJens Axboe ret = 1;
1106ed29b0b4SJens Axboe }
1107ed29b0b4SJens Axboe
1108ed29b0b4SJens Axboe percpu_ref_get_many(&ctx->refs, ret);
1109ed29b0b4SJens Axboe for (i = 0; i < ret; i++) {
1110ed29b0b4SJens Axboe struct io_kiocb *req = reqs[i];
1111ed29b0b4SJens Axboe
1112ed29b0b4SJens Axboe io_preinit_req(req, ctx);
1113ed29b0b4SJens Axboe io_req_add_to_cache(req, ctx);
1114ed29b0b4SJens Axboe }
1115ed29b0b4SJens Axboe return true;
1116ed29b0b4SJens Axboe }
1117ed29b0b4SJens Axboe
io_free_req(struct io_kiocb * req)111803adabe8SPavel Begunkov __cold void io_free_req(struct io_kiocb *req)
111903adabe8SPavel Begunkov {
11206ec9afc7SPavel Begunkov /* refs were already put, restore them for io_req_task_complete() */
11216ec9afc7SPavel Begunkov req->flags &= ~REQ_F_REFCOUNT;
11226ec9afc7SPavel Begunkov /* we only want to free it, don't post CQEs */
11236ec9afc7SPavel Begunkov req->flags |= REQ_F_CQE_SKIP;
11246ec9afc7SPavel Begunkov req->io_task_work.func = io_req_task_complete;
112503adabe8SPavel Begunkov io_req_task_work_add(req);
112603adabe8SPavel Begunkov }
112703adabe8SPavel Begunkov
__io_req_find_next_prep(struct io_kiocb * req)1128ed29b0b4SJens Axboe static void __io_req_find_next_prep(struct io_kiocb *req)
1129ed29b0b4SJens Axboe {
1130ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
1131ed29b0b4SJens Axboe
11326971253fSPavel Begunkov spin_lock(&ctx->completion_lock);
1133305bef98SPavel Begunkov io_disarm_next(req);
11346971253fSPavel Begunkov spin_unlock(&ctx->completion_lock);
1135ed29b0b4SJens Axboe }
1136ed29b0b4SJens Axboe
io_req_find_next(struct io_kiocb * req)1137ed29b0b4SJens Axboe static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
1138ed29b0b4SJens Axboe {
1139ed29b0b4SJens Axboe struct io_kiocb *nxt;
1140ed29b0b4SJens Axboe
1141ed29b0b4SJens Axboe /*
1142ed29b0b4SJens Axboe * If LINK is set, we have dependent requests in this chain. If we
1143ed29b0b4SJens Axboe * didn't fail this request, queue the first one up, moving any other
1144ed29b0b4SJens Axboe * dependencies to the next request. In case of failure, fail the rest
1145ed29b0b4SJens Axboe * of the chain.
1146ed29b0b4SJens Axboe */
1147ed29b0b4SJens Axboe if (unlikely(req->flags & IO_DISARM_MASK))
1148ed29b0b4SJens Axboe __io_req_find_next_prep(req);
1149ed29b0b4SJens Axboe nxt = req->link;
1150ed29b0b4SJens Axboe req->link = NULL;
1151ed29b0b4SJens Axboe return nxt;
1152ed29b0b4SJens Axboe }
1153ed29b0b4SJens Axboe
ctx_flush_and_put(struct io_ring_ctx * ctx,struct io_tw_state * ts)1154a282967cSPavel Begunkov static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)
1155ed29b0b4SJens Axboe {
1156ed29b0b4SJens Axboe if (!ctx)
1157ed29b0b4SJens Axboe return;
1158ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1159ed29b0b4SJens Axboe atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1160a282967cSPavel Begunkov if (ts->locked) {
1161ed29b0b4SJens Axboe io_submit_flush_completions(ctx);
1162ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
1163a282967cSPavel Begunkov ts->locked = false;
1164ed29b0b4SJens Axboe }
1165ed29b0b4SJens Axboe percpu_ref_put(&ctx->refs);
1166ed29b0b4SJens Axboe }
1167ed29b0b4SJens Axboe
handle_tw_list(struct llist_node * node,struct io_ring_ctx ** ctx,struct io_tw_state * ts)1168c6dd763cSDylan Yudaken static unsigned int handle_tw_list(struct llist_node *node,
1169a282967cSPavel Begunkov struct io_ring_ctx **ctx,
11708c0a0ae8SJens Axboe struct io_tw_state *ts)
1171ed29b0b4SJens Axboe {
1172c6dd763cSDylan Yudaken unsigned int count = 0;
1173c6dd763cSDylan Yudaken
11748c0a0ae8SJens Axboe do {
1175f88262e6SDylan Yudaken struct llist_node *next = node->next;
1176ed29b0b4SJens Axboe struct io_kiocb *req = container_of(node, struct io_kiocb,
1177ed29b0b4SJens Axboe io_task_work.node);
1178ed29b0b4SJens Axboe
1179ed29b0b4SJens Axboe prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1180ed29b0b4SJens Axboe
1181ed29b0b4SJens Axboe if (req->ctx != *ctx) {
1182a282967cSPavel Begunkov ctx_flush_and_put(*ctx, ts);
1183ed29b0b4SJens Axboe *ctx = req->ctx;
1184ed29b0b4SJens Axboe /* if not contended, grab and improve batching */
1185a282967cSPavel Begunkov ts->locked = mutex_trylock(&(*ctx)->uring_lock);
1186ed29b0b4SJens Axboe percpu_ref_get(&(*ctx)->refs);
118713bfa6f1SPavel Begunkov }
1188c92fcfc2SJens Axboe INDIRECT_CALL_2(req->io_task_work.func,
1189c92fcfc2SJens Axboe io_poll_task_func, io_req_rw_complete,
1190c92fcfc2SJens Axboe req, ts);
1191ed29b0b4SJens Axboe node = next;
1192c6dd763cSDylan Yudaken count++;
1193f5868008SJens Axboe if (unlikely(need_resched())) {
1194a282967cSPavel Begunkov ctx_flush_and_put(*ctx, ts);
1195f5868008SJens Axboe *ctx = NULL;
1196f5868008SJens Axboe cond_resched();
1197f5868008SJens Axboe }
11988c0a0ae8SJens Axboe } while (node);
1199c6dd763cSDylan Yudaken
1200c6dd763cSDylan Yudaken return count;
1201ed29b0b4SJens Axboe }
1202ed29b0b4SJens Axboe
1203923d1592SDylan Yudaken /**
1204923d1592SDylan Yudaken * io_llist_xchg - swap all entries in a lock-less list
1205923d1592SDylan Yudaken * @head: the head of lock-less list to delete all entries
1206923d1592SDylan Yudaken * @new: new entry as the head of the list
1207923d1592SDylan Yudaken *
1208923d1592SDylan Yudaken * If list is empty, return NULL, otherwise, return the pointer to the first entry.
1209923d1592SDylan Yudaken * The order of entries returned is from the newest to the oldest added one.
1210923d1592SDylan Yudaken */
io_llist_xchg(struct llist_head * head,struct llist_node * new)1211923d1592SDylan Yudaken static inline struct llist_node *io_llist_xchg(struct llist_head *head,
1212923d1592SDylan Yudaken struct llist_node *new)
1213923d1592SDylan Yudaken {
1214923d1592SDylan Yudaken return xchg(&head->first, new);
1215923d1592SDylan Yudaken }
1216923d1592SDylan Yudaken
io_fallback_tw(struct io_uring_task * tctx,bool sync)1217dfbe5561SJens Axboe static __cold void io_fallback_tw(struct io_uring_task *tctx, bool sync)
121810e1c0d5SJens Axboe {
121910e1c0d5SJens Axboe struct llist_node *node = llist_del_all(&tctx->task_list);
1220dfbe5561SJens Axboe struct io_ring_ctx *last_ctx = NULL;
122110e1c0d5SJens Axboe struct io_kiocb *req;
122210e1c0d5SJens Axboe
122310e1c0d5SJens Axboe while (node) {
122410e1c0d5SJens Axboe req = container_of(node, struct io_kiocb, io_task_work.node);
122510e1c0d5SJens Axboe node = node->next;
1226dfbe5561SJens Axboe if (sync && last_ctx != req->ctx) {
1227dfbe5561SJens Axboe if (last_ctx) {
1228dfbe5561SJens Axboe flush_delayed_work(&last_ctx->fallback_work);
1229dfbe5561SJens Axboe percpu_ref_put(&last_ctx->refs);
1230dfbe5561SJens Axboe }
1231dfbe5561SJens Axboe last_ctx = req->ctx;
1232dfbe5561SJens Axboe percpu_ref_get(&last_ctx->refs);
1233dfbe5561SJens Axboe }
123410e1c0d5SJens Axboe if (llist_add(&req->io_task_work.node,
123510e1c0d5SJens Axboe &req->ctx->fallback_llist))
123610e1c0d5SJens Axboe schedule_delayed_work(&req->ctx->fallback_work, 1);
123710e1c0d5SJens Axboe }
1238dfbe5561SJens Axboe
1239dfbe5561SJens Axboe if (last_ctx) {
1240dfbe5561SJens Axboe flush_delayed_work(&last_ctx->fallback_work);
1241dfbe5561SJens Axboe percpu_ref_put(&last_ctx->refs);
1242dfbe5561SJens Axboe }
124310e1c0d5SJens Axboe }
124410e1c0d5SJens Axboe
tctx_task_work(struct callback_head * cb)1245c9f06aa7SJens Axboe void tctx_task_work(struct callback_head *cb)
1246ed29b0b4SJens Axboe {
1247a282967cSPavel Begunkov struct io_tw_state ts = {};
1248ed29b0b4SJens Axboe struct io_ring_ctx *ctx = NULL;
1249ed29b0b4SJens Axboe struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
1250ed29b0b4SJens Axboe task_work);
125177e443abSPavel Begunkov struct llist_node *node;
1252cb6bf7f2SPavel Begunkov unsigned int count = 0;
1253ed29b0b4SJens Axboe
125477e443abSPavel Begunkov if (unlikely(current->flags & PF_EXITING)) {
1255dfbe5561SJens Axboe io_fallback_tw(tctx, true);
125677e443abSPavel Begunkov return;
125777e443abSPavel Begunkov }
125877e443abSPavel Begunkov
12598c0a0ae8SJens Axboe node = llist_del_all(&tctx->task_list);
12608c0a0ae8SJens Axboe if (node)
12618c0a0ae8SJens Axboe count = handle_tw_list(node, &ctx, &ts);
1262ed29b0b4SJens Axboe
1263a282967cSPavel Begunkov ctx_flush_and_put(ctx, &ts);
1264ed29b0b4SJens Axboe
12658d664282SJens Axboe /* relaxed read is enough as only the task itself sets ->in_cancel */
12668d664282SJens Axboe if (unlikely(atomic_read(&tctx->in_cancel)))
1267ed29b0b4SJens Axboe io_uring_drop_tctx_refs(current);
1268c6dd763cSDylan Yudaken
12698c0a0ae8SJens Axboe trace_io_uring_task_work_run(tctx, count, 1);
1270ed29b0b4SJens Axboe }
1271ed29b0b4SJens Axboe
io_req_local_work_add(struct io_kiocb * req,unsigned flags)127291c7884aSPavel Begunkov static inline void io_req_local_work_add(struct io_kiocb *req, unsigned flags)
1273c0e0d6baSDylan Yudaken {
1274c0e0d6baSDylan Yudaken struct io_ring_ctx *ctx = req->ctx;
12758751d154SPavel Begunkov unsigned nr_wait, nr_tw, nr_tw_prev;
127651509400SPavel Begunkov struct llist_node *first;
1277c0e0d6baSDylan Yudaken
12788751d154SPavel Begunkov if (req->flags & (REQ_F_LINK | REQ_F_HARDLINK))
12798751d154SPavel Begunkov flags &= ~IOU_F_TWQ_LAZY_WAKE;
12809ffa13ffSPavel Begunkov
128151509400SPavel Begunkov first = READ_ONCE(ctx->work_llist.first);
128251509400SPavel Begunkov do {
12838751d154SPavel Begunkov nr_tw_prev = 0;
12848751d154SPavel Begunkov if (first) {
12858751d154SPavel Begunkov struct io_kiocb *first_req = container_of(first,
12868751d154SPavel Begunkov struct io_kiocb,
12878751d154SPavel Begunkov io_task_work.node);
12888751d154SPavel Begunkov /*
12898751d154SPavel Begunkov * Might be executed at any moment, rely on
12908751d154SPavel Begunkov * SLAB_TYPESAFE_BY_RCU to keep it alive.
12918751d154SPavel Begunkov */
12928751d154SPavel Begunkov nr_tw_prev = READ_ONCE(first_req->nr_tw);
1293c0e0d6baSDylan Yudaken }
12948751d154SPavel Begunkov nr_tw = nr_tw_prev + 1;
12958751d154SPavel Begunkov /* Large enough to fail the nr_wait comparison below */
12968751d154SPavel Begunkov if (!(flags & IOU_F_TWQ_LAZY_WAKE))
12978836df02SPavel Begunkov nr_tw = INT_MAX;
1298c0e0d6baSDylan Yudaken
12998751d154SPavel Begunkov req->nr_tw = nr_tw;
130051509400SPavel Begunkov req->io_task_work.node.next = first;
130151509400SPavel Begunkov } while (!try_cmpxchg(&ctx->work_llist.first, &first,
130251509400SPavel Begunkov &req->io_task_work.node));
130351509400SPavel Begunkov
13048751d154SPavel Begunkov if (!first) {
1305c0e0d6baSDylan Yudaken if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1306c0e0d6baSDylan Yudaken atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
130721a091b9SDylan Yudaken if (ctx->has_evfd)
130821a091b9SDylan Yudaken io_eventfd_signal(ctx);
1309c0e0d6baSDylan Yudaken }
1310c0e0d6baSDylan Yudaken
13118751d154SPavel Begunkov nr_wait = atomic_read(&ctx->cq_wait_nr);
13128751d154SPavel Begunkov /* no one is waiting */
13138751d154SPavel Begunkov if (!nr_wait)
13148751d154SPavel Begunkov return;
13158751d154SPavel Begunkov /* either not enough or the previous add has already woken it up */
13168751d154SPavel Begunkov if (nr_wait > nr_tw || nr_tw_prev >= nr_wait)
13178751d154SPavel Begunkov return;
13188751d154SPavel Begunkov /* pairs with set_current_state() in io_cqring_wait() */
13198751d154SPavel Begunkov smp_mb__after_atomic();
1320e52d2e58SPavel Begunkov wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);
1321c0e0d6baSDylan Yudaken }
1322c0e0d6baSDylan Yudaken
io_req_normal_work_add(struct io_kiocb * req)132391c7884aSPavel Begunkov static void io_req_normal_work_add(struct io_kiocb *req)
1324ed29b0b4SJens Axboe {
1325c34398a8SDylan Yudaken struct io_uring_task *tctx = req->task->io_uring;
1326ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
1327ed29b0b4SJens Axboe
1328ed29b0b4SJens Axboe /* task_work already pending, we're done */
132932d91f05SDylan Yudaken if (!llist_add(&req->io_task_work.node, &tctx->task_list))
1330ed29b0b4SJens Axboe return;
1331ed29b0b4SJens Axboe
1332ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1333ed29b0b4SJens Axboe atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1334ed29b0b4SJens Axboe
1335ed29b0b4SJens Axboe if (likely(!task_work_add(req->task, &tctx->task_work, ctx->notify_method)))
1336ed29b0b4SJens Axboe return;
1337ed29b0b4SJens Axboe
1338dfbe5561SJens Axboe io_fallback_tw(tctx, false);
1339c0e0d6baSDylan Yudaken }
1340c0e0d6baSDylan Yudaken
__io_req_task_work_add(struct io_kiocb * req,unsigned flags)134191c7884aSPavel Begunkov void __io_req_task_work_add(struct io_kiocb *req, unsigned flags)
134291c7884aSPavel Begunkov {
134391c7884aSPavel Begunkov if (req->ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
134491c7884aSPavel Begunkov rcu_read_lock();
134591c7884aSPavel Begunkov io_req_local_work_add(req, flags);
134691c7884aSPavel Begunkov rcu_read_unlock();
134791c7884aSPavel Begunkov } else {
134891c7884aSPavel Begunkov io_req_normal_work_add(req);
134991c7884aSPavel Begunkov }
135091c7884aSPavel Begunkov }
135191c7884aSPavel Begunkov
io_move_task_work_from_local(struct io_ring_ctx * ctx)1352c0e0d6baSDylan Yudaken static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
1353c0e0d6baSDylan Yudaken {
1354c0e0d6baSDylan Yudaken struct llist_node *node;
1355c0e0d6baSDylan Yudaken
1356c0e0d6baSDylan Yudaken node = llist_del_all(&ctx->work_llist);
1357c0e0d6baSDylan Yudaken while (node) {
1358c0e0d6baSDylan Yudaken struct io_kiocb *req = container_of(node, struct io_kiocb,
1359c0e0d6baSDylan Yudaken io_task_work.node);
1360c0e0d6baSDylan Yudaken
1361c0e0d6baSDylan Yudaken node = node->next;
136291c7884aSPavel Begunkov io_req_normal_work_add(req);
1363c0e0d6baSDylan Yudaken }
1364c0e0d6baSDylan Yudaken }
1365c0e0d6baSDylan Yudaken
io_run_local_work_continue(struct io_ring_ctx * ctx,int events,int min_events)13667b8fa7a0SJens Axboe static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
13677b8fa7a0SJens Axboe int min_events)
13687b8fa7a0SJens Axboe {
13697b8fa7a0SJens Axboe if (llist_empty(&ctx->work_llist))
13707b8fa7a0SJens Axboe return false;
13717b8fa7a0SJens Axboe if (events < min_events)
13727b8fa7a0SJens Axboe return true;
13737b8fa7a0SJens Axboe if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
13747b8fa7a0SJens Axboe atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
13757b8fa7a0SJens Axboe return false;
13767b8fa7a0SJens Axboe }
13777b8fa7a0SJens Axboe
__io_run_local_work(struct io_ring_ctx * ctx,struct io_tw_state * ts,int min_events)13787b8fa7a0SJens Axboe static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
13797b8fa7a0SJens Axboe int min_events)
1380c0e0d6baSDylan Yudaken {
1381c0e0d6baSDylan Yudaken struct llist_node *node;
1382c3f4d39eSPavel Begunkov unsigned int loops = 0;
1383140102aeSPavel Begunkov int ret = 0;
1384c0e0d6baSDylan Yudaken
1385140102aeSPavel Begunkov if (WARN_ON_ONCE(ctx->submitter_task != current))
1386c0e0d6baSDylan Yudaken return -EEXIST;
1387c3f4d39eSPavel Begunkov if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
1388c3f4d39eSPavel Begunkov atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1389c0e0d6baSDylan Yudaken again:
13903af0356cSJens Axboe /*
13913af0356cSJens Axboe * llists are in reverse order, flip it back the right way before
13923af0356cSJens Axboe * running the pending items.
13933af0356cSJens Axboe */
13943af0356cSJens Axboe node = llist_reverse_order(io_llist_xchg(&ctx->work_llist, NULL));
1395c3f4d39eSPavel Begunkov while (node) {
1396c0e0d6baSDylan Yudaken struct llist_node *next = node->next;
1397c0e0d6baSDylan Yudaken struct io_kiocb *req = container_of(node, struct io_kiocb,
1398c0e0d6baSDylan Yudaken io_task_work.node);
1399c0e0d6baSDylan Yudaken prefetch(container_of(next, struct io_kiocb, io_task_work.node));
1400c92fcfc2SJens Axboe INDIRECT_CALL_2(req->io_task_work.func,
1401c92fcfc2SJens Axboe io_poll_task_func, io_req_rw_complete,
1402c92fcfc2SJens Axboe req, ts);
1403c0e0d6baSDylan Yudaken ret++;
1404c0e0d6baSDylan Yudaken node = next;
1405c0e0d6baSDylan Yudaken }
1406f75d5036SDylan Yudaken loops++;
1407c0e0d6baSDylan Yudaken
14087b8fa7a0SJens Axboe if (io_run_local_work_continue(ctx, ret, min_events))
1409c3f4d39eSPavel Begunkov goto again;
1410a282967cSPavel Begunkov if (ts->locked) {
1411c0e0d6baSDylan Yudaken io_submit_flush_completions(ctx);
14127b8fa7a0SJens Axboe if (io_run_local_work_continue(ctx, ret, min_events))
1413b0b7a7d2SPavel Begunkov goto again;
1414b0b7a7d2SPavel Begunkov }
14157b8fa7a0SJens Axboe
1416f75d5036SDylan Yudaken trace_io_uring_local_work_run(ctx, ret, loops);
1417c0e0d6baSDylan Yudaken return ret;
14188ac5d85aSJens Axboe }
14198ac5d85aSJens Axboe
io_run_local_work_locked(struct io_ring_ctx * ctx,int min_events)14207b8fa7a0SJens Axboe static inline int io_run_local_work_locked(struct io_ring_ctx *ctx,
14217b8fa7a0SJens Axboe int min_events)
1422360173abSPavel Begunkov {
1423a282967cSPavel Begunkov struct io_tw_state ts = { .locked = true, };
1424360173abSPavel Begunkov int ret;
1425360173abSPavel Begunkov
1426360173abSPavel Begunkov if (llist_empty(&ctx->work_llist))
1427360173abSPavel Begunkov return 0;
1428360173abSPavel Begunkov
14297b8fa7a0SJens Axboe ret = __io_run_local_work(ctx, &ts, min_events);
1430360173abSPavel Begunkov /* shouldn't happen! */
1431a282967cSPavel Begunkov if (WARN_ON_ONCE(!ts.locked))
1432360173abSPavel Begunkov mutex_lock(&ctx->uring_lock);
1433360173abSPavel Begunkov return ret;
1434360173abSPavel Begunkov }
1435360173abSPavel Begunkov
io_run_local_work(struct io_ring_ctx * ctx,int min_events)14367b8fa7a0SJens Axboe static int io_run_local_work(struct io_ring_ctx *ctx, int min_events)
14378ac5d85aSJens Axboe {
1438a282967cSPavel Begunkov struct io_tw_state ts = {};
14398ac5d85aSJens Axboe int ret;
14408ac5d85aSJens Axboe
1441a282967cSPavel Begunkov ts.locked = mutex_trylock(&ctx->uring_lock);
14427b8fa7a0SJens Axboe ret = __io_run_local_work(ctx, &ts, min_events);
1443a282967cSPavel Begunkov if (ts.locked)
14448ac5d85aSJens Axboe mutex_unlock(&ctx->uring_lock);
14458ac5d85aSJens Axboe
14468ac5d85aSJens Axboe return ret;
1447c0e0d6baSDylan Yudaken }
1448c0e0d6baSDylan Yudaken
io_req_task_cancel(struct io_kiocb * req,struct io_tw_state * ts)1449a282967cSPavel Begunkov static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)
1450ed29b0b4SJens Axboe {
1451a282967cSPavel Begunkov io_tw_lock(req->ctx, ts);
1452973fc83fSDylan Yudaken io_req_defer_failed(req, req->cqe.res);
1453ed29b0b4SJens Axboe }
1454ed29b0b4SJens Axboe
io_req_task_submit(struct io_kiocb * req,struct io_tw_state * ts)1455a282967cSPavel Begunkov void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)
1456ed29b0b4SJens Axboe {
1457a282967cSPavel Begunkov io_tw_lock(req->ctx, ts);
1458ed29b0b4SJens Axboe /* req->task == current here, checking PF_EXITING is safe */
14596bb30855SDylan Yudaken if (unlikely(req->task->flags & PF_EXITING))
1460973fc83fSDylan Yudaken io_req_defer_failed(req, -EFAULT);
14616bb30855SDylan Yudaken else if (req->flags & REQ_F_FORCE_ASYNC)
1462a282967cSPavel Begunkov io_queue_iowq(req, ts);
14636bb30855SDylan Yudaken else
14646bb30855SDylan Yudaken io_queue_sqe(req);
1465ed29b0b4SJens Axboe }
1466ed29b0b4SJens Axboe
io_req_task_queue_fail(struct io_kiocb * req,int ret)146759915143SJens Axboe void io_req_task_queue_fail(struct io_kiocb *req, int ret)
1468ed29b0b4SJens Axboe {
146997b388d7SJens Axboe io_req_set_res(req, ret, 0);
1470ed29b0b4SJens Axboe req->io_task_work.func = io_req_task_cancel;
1471ed29b0b4SJens Axboe io_req_task_work_add(req);
1472ed29b0b4SJens Axboe }
1473ed29b0b4SJens Axboe
io_req_task_queue(struct io_kiocb * req)1474f3b44f92SJens Axboe void io_req_task_queue(struct io_kiocb *req)
1475ed29b0b4SJens Axboe {
1476ed29b0b4SJens Axboe req->io_task_work.func = io_req_task_submit;
1477ed29b0b4SJens Axboe io_req_task_work_add(req);
1478ed29b0b4SJens Axboe }
1479ed29b0b4SJens Axboe
io_queue_next(struct io_kiocb * req)148059915143SJens Axboe void io_queue_next(struct io_kiocb *req)
1481ed29b0b4SJens Axboe {
1482ed29b0b4SJens Axboe struct io_kiocb *nxt = io_req_find_next(req);
1483ed29b0b4SJens Axboe
1484ed29b0b4SJens Axboe if (nxt)
1485ed29b0b4SJens Axboe io_req_task_queue(nxt);
1486ed29b0b4SJens Axboe }
1487ed29b0b4SJens Axboe
io_free_batch_list(struct io_ring_ctx * ctx,struct io_wq_work_node * node)1488ec26c225SPavel Begunkov static void io_free_batch_list(struct io_ring_ctx *ctx,
1489ec26c225SPavel Begunkov struct io_wq_work_node *node)
1490ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
1491ed29b0b4SJens Axboe {
1492ed29b0b4SJens Axboe do {
1493ed29b0b4SJens Axboe struct io_kiocb *req = container_of(node, struct io_kiocb,
1494ed29b0b4SJens Axboe comp_list);
1495ed29b0b4SJens Axboe
1496ed29b0b4SJens Axboe if (unlikely(req->flags & IO_REQ_CLEAN_SLOW_FLAGS)) {
1497ed29b0b4SJens Axboe if (req->flags & REQ_F_REFCOUNT) {
1498ed29b0b4SJens Axboe node = req->comp_list.next;
1499ed29b0b4SJens Axboe if (!req_ref_put_and_test(req))
1500ed29b0b4SJens Axboe continue;
1501ed29b0b4SJens Axboe }
1502ed29b0b4SJens Axboe if ((req->flags & REQ_F_POLLED) && req->apoll) {
1503ed29b0b4SJens Axboe struct async_poll *apoll = req->apoll;
1504ed29b0b4SJens Axboe
1505ed29b0b4SJens Axboe if (apoll->double_poll)
1506ed29b0b4SJens Axboe kfree(apoll->double_poll);
15079731bc98SJens Axboe if (!io_alloc_cache_put(&ctx->apoll_cache, &apoll->cache))
15089731bc98SJens Axboe kfree(apoll);
1509ed29b0b4SJens Axboe req->flags &= ~REQ_F_POLLED;
1510ed29b0b4SJens Axboe }
1511ed29b0b4SJens Axboe if (req->flags & IO_REQ_LINK_FLAGS)
1512ed29b0b4SJens Axboe io_queue_next(req);
1513ed29b0b4SJens Axboe if (unlikely(req->flags & IO_REQ_CLEAN_FLAGS))
1514ed29b0b4SJens Axboe io_clean_op(req);
1515ed29b0b4SJens Axboe }
151617bc2837SJens Axboe io_put_file(req);
1517ed29b0b4SJens Axboe
1518ed29b0b4SJens Axboe io_req_put_rsrc_locked(req, ctx);
1519ed29b0b4SJens Axboe
15202fdd6fb5SPavel Begunkov io_put_task(req->task);
1521ed29b0b4SJens Axboe node = req->comp_list.next;
1522ed29b0b4SJens Axboe io_req_add_to_cache(req, ctx);
1523ed29b0b4SJens Axboe } while (node);
1524ed29b0b4SJens Axboe }
1525ed29b0b4SJens Axboe
__io_submit_flush_completions(struct io_ring_ctx * ctx)1526ec26c225SPavel Begunkov void __io_submit_flush_completions(struct io_ring_ctx *ctx)
1527ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
1528ed29b0b4SJens Axboe {
1529ed29b0b4SJens Axboe struct io_submit_state *state = &ctx->submit_state;
1530fa780334SJens Axboe struct io_wq_work_node *node;
1531ed29b0b4SJens Axboe
1532f66f7342SPavel Begunkov __io_cq_lock(ctx);
1533931147ddSDylan Yudaken /* must come first to preserve CQE ordering in failure cases */
1534931147ddSDylan Yudaken if (state->cqes_count)
1535931147ddSDylan Yudaken __io_flush_post_cqes(ctx);
1536fa780334SJens Axboe __wq_list_for_each(node, &state->compl_reqs) {
1537ed29b0b4SJens Axboe struct io_kiocb *req = container_of(node, struct io_kiocb,
1538ed29b0b4SJens Axboe comp_list);
1539ed29b0b4SJens Axboe
1540f66f7342SPavel Begunkov if (!(req->flags & REQ_F_CQE_SKIP) &&
154100b0db56SPavel Begunkov unlikely(!io_fill_cqe_req(ctx, req))) {
154227122c07SPavel Begunkov if (ctx->lockless_cq) {
1543f66f7342SPavel Begunkov spin_lock(&ctx->completion_lock);
1544f66f7342SPavel Begunkov io_req_cqe_overflow(req);
1545f66f7342SPavel Begunkov spin_unlock(&ctx->completion_lock);
1546f66f7342SPavel Begunkov } else {
1547f66f7342SPavel Begunkov io_req_cqe_overflow(req);
1548ed29b0b4SJens Axboe }
1549f66f7342SPavel Begunkov }
1550f66f7342SPavel Begunkov }
1551c98c81a4SPavel Begunkov __io_cq_unlock_post(ctx);
1552ed29b0b4SJens Axboe
1553931147ddSDylan Yudaken if (!wq_list_empty(&ctx->submit_state.compl_reqs)) {
1554ed29b0b4SJens Axboe io_free_batch_list(ctx, state->compl_reqs.first);
1555ed29b0b4SJens Axboe INIT_WQ_LIST(&state->compl_reqs);
1556ed29b0b4SJens Axboe }
1557931147ddSDylan Yudaken }
1558ed29b0b4SJens Axboe
io_cqring_events(struct io_ring_ctx * ctx)1559ed29b0b4SJens Axboe static unsigned io_cqring_events(struct io_ring_ctx *ctx)
1560ed29b0b4SJens Axboe {
1561ed29b0b4SJens Axboe /* See comment at the top of this file */
1562ed29b0b4SJens Axboe smp_rmb();
1563ed29b0b4SJens Axboe return __io_cqring_events(ctx);
1564ed29b0b4SJens Axboe }
1565ed29b0b4SJens Axboe
1566ed29b0b4SJens Axboe /*
1567ed29b0b4SJens Axboe * We can't just wait for polled events to come to us, we have to actively
1568ed29b0b4SJens Axboe * find and complete them.
1569ed29b0b4SJens Axboe */
io_iopoll_try_reap_events(struct io_ring_ctx * ctx)1570ed29b0b4SJens Axboe static __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
1571ed29b0b4SJens Axboe {
1572ed29b0b4SJens Axboe if (!(ctx->flags & IORING_SETUP_IOPOLL))
1573ed29b0b4SJens Axboe return;
1574ed29b0b4SJens Axboe
1575ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
1576ed29b0b4SJens Axboe while (!wq_list_empty(&ctx->iopoll_list)) {
1577ed29b0b4SJens Axboe /* let it sleep and repeat later if can't complete a request */
1578ed29b0b4SJens Axboe if (io_do_iopoll(ctx, true) == 0)
1579ed29b0b4SJens Axboe break;
1580ed29b0b4SJens Axboe /*
1581ed29b0b4SJens Axboe * Ensure we allow local-to-the-cpu processing to take place,
1582ed29b0b4SJens Axboe * in this case we need to ensure that we reap all events.
1583ed29b0b4SJens Axboe * Also let task_work, etc. to progress by releasing the mutex
1584ed29b0b4SJens Axboe */
1585ed29b0b4SJens Axboe if (need_resched()) {
1586ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
1587ed29b0b4SJens Axboe cond_resched();
1588ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
1589ed29b0b4SJens Axboe }
1590ed29b0b4SJens Axboe }
1591ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
1592ed29b0b4SJens Axboe }
1593ed29b0b4SJens Axboe
io_iopoll_check(struct io_ring_ctx * ctx,long min)1594ed29b0b4SJens Axboe static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
1595ed29b0b4SJens Axboe {
1596ed29b0b4SJens Axboe unsigned int nr_events = 0;
1597ed29b0b4SJens Axboe unsigned long check_cq;
1598ed29b0b4SJens Axboe
159976de6749SPavel Begunkov if (!io_allowed_run_tw(ctx))
160076de6749SPavel Begunkov return -EEXIST;
160176de6749SPavel Begunkov
16023a08576bSPavel Begunkov check_cq = READ_ONCE(ctx->check_cq);
16033a08576bSPavel Begunkov if (unlikely(check_cq)) {
16043a08576bSPavel Begunkov if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
1605a85381d8SPavel Begunkov __io_cqring_overflow_flush(ctx);
16063a08576bSPavel Begunkov /*
16073a08576bSPavel Begunkov * Similarly do not spin if we have not informed the user of any
16083a08576bSPavel Begunkov * dropped CQE.
16093a08576bSPavel Begunkov */
16103a08576bSPavel Begunkov if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT))
16113a08576bSPavel Begunkov return -EBADR;
16123a08576bSPavel Begunkov }
1613ed29b0b4SJens Axboe /*
1614ed29b0b4SJens Axboe * Don't enter poll loop if we already have events pending.
1615ed29b0b4SJens Axboe * If we do, we can potentially be spinning for commands that
1616ed29b0b4SJens Axboe * already triggered a CQE (eg in error).
1617ed29b0b4SJens Axboe */
1618ed29b0b4SJens Axboe if (io_cqring_events(ctx))
1619ed29b0b4SJens Axboe return 0;
1620ed29b0b4SJens Axboe
1621ed29b0b4SJens Axboe do {
16229e4bef2bSJens Axboe int ret = 0;
16239e4bef2bSJens Axboe
1624ed29b0b4SJens Axboe /*
1625ed29b0b4SJens Axboe * If a submit got punted to a workqueue, we can have the
1626ed29b0b4SJens Axboe * application entering polling for a command before it gets
1627ed29b0b4SJens Axboe * issued. That app will hold the uring_lock for the duration
1628ed29b0b4SJens Axboe * of the poll right here, so we need to take a breather every
1629ed29b0b4SJens Axboe * now and then to ensure that the issue has a chance to add
1630ed29b0b4SJens Axboe * the poll to the issued list. Otherwise we can spin here
1631ed29b0b4SJens Axboe * forever, while the workqueue is stuck trying to acquire the
1632ed29b0b4SJens Axboe * very same mutex.
1633ed29b0b4SJens Axboe */
1634dac6a0eaSJens Axboe if (wq_list_empty(&ctx->iopoll_list) ||
1635dac6a0eaSJens Axboe io_task_work_pending(ctx)) {
1636ed29b0b4SJens Axboe u32 tail = ctx->cached_cq_tail;
1637ed29b0b4SJens Axboe
16387b8fa7a0SJens Axboe (void) io_run_local_work_locked(ctx, min);
16391f8d5bbeSPavel Begunkov
16401f8d5bbeSPavel Begunkov if (task_work_pending(current) ||
16411f8d5bbeSPavel Begunkov wq_list_empty(&ctx->iopoll_list)) {
1642ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
1643ed29b0b4SJens Axboe io_run_task_work();
1644ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
16451f8d5bbeSPavel Begunkov }
1646ed29b0b4SJens Axboe /* some requests don't go through iopoll_list */
1647ed29b0b4SJens Axboe if (tail != ctx->cached_cq_tail ||
1648ed29b0b4SJens Axboe wq_list_empty(&ctx->iopoll_list))
1649ed29b0b4SJens Axboe break;
1650ed29b0b4SJens Axboe }
1651ed29b0b4SJens Axboe ret = io_do_iopoll(ctx, !min);
16529e4bef2bSJens Axboe if (unlikely(ret < 0))
1653ed29b0b4SJens Axboe return ret;
1654dc314886SPavel Begunkov
1655dc314886SPavel Begunkov if (task_sigpending(current))
1656dc314886SPavel Begunkov return -EINTR;
16579e4bef2bSJens Axboe if (need_resched())
16589e4bef2bSJens Axboe break;
1659ed29b0b4SJens Axboe
16609e4bef2bSJens Axboe nr_events += ret;
16619e4bef2bSJens Axboe } while (nr_events < min);
16629e4bef2bSJens Axboe
16639e4bef2bSJens Axboe return 0;
1664ed29b0b4SJens Axboe }
16657012c815SPavel Begunkov
io_req_task_complete(struct io_kiocb * req,struct io_tw_state * ts)1666a282967cSPavel Begunkov void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
1667ed29b0b4SJens Axboe {
1668a282967cSPavel Begunkov if (ts->locked)
16699da070b1SPavel Begunkov io_req_complete_defer(req);
16707012c815SPavel Begunkov else
167127f35fe9SDylan Yudaken io_req_complete_post(req, IO_URING_F_UNLOCKED);
1672ed29b0b4SJens Axboe }
1673ed29b0b4SJens Axboe
1674ed29b0b4SJens Axboe /*
1675ed29b0b4SJens Axboe * After the iocb has been issued, it's safe to be found on the poll list.
1676ed29b0b4SJens Axboe * Adding the kiocb to the list AFTER submission ensures that we don't
1677ed29b0b4SJens Axboe * find it from a io_do_iopoll() thread before the issuer is done
1678ed29b0b4SJens Axboe * accessing the kiocb cookie.
1679ed29b0b4SJens Axboe */
io_iopoll_req_issued(struct io_kiocb * req,unsigned int issue_flags)1680ed29b0b4SJens Axboe static void io_iopoll_req_issued(struct io_kiocb *req, unsigned int issue_flags)
1681ed29b0b4SJens Axboe {
1682ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
1683ed29b0b4SJens Axboe const bool needs_lock = issue_flags & IO_URING_F_UNLOCKED;
1684ed29b0b4SJens Axboe
1685ed29b0b4SJens Axboe /* workqueue context doesn't hold uring_lock, grab it now */
1686ed29b0b4SJens Axboe if (unlikely(needs_lock))
1687ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
1688ed29b0b4SJens Axboe
1689ed29b0b4SJens Axboe /*
1690ed29b0b4SJens Axboe * Track whether we have multiple files in our lists. This will impact
1691ed29b0b4SJens Axboe * how we do polling eventually, not spinning if we're on potentially
1692ed29b0b4SJens Axboe * different devices.
1693ed29b0b4SJens Axboe */
1694ed29b0b4SJens Axboe if (wq_list_empty(&ctx->iopoll_list)) {
1695ed29b0b4SJens Axboe ctx->poll_multi_queue = false;
1696ed29b0b4SJens Axboe } else if (!ctx->poll_multi_queue) {
1697ed29b0b4SJens Axboe struct io_kiocb *list_req;
1698ed29b0b4SJens Axboe
1699ed29b0b4SJens Axboe list_req = container_of(ctx->iopoll_list.first, struct io_kiocb,
1700ed29b0b4SJens Axboe comp_list);
1701ed29b0b4SJens Axboe if (list_req->file != req->file)
1702ed29b0b4SJens Axboe ctx->poll_multi_queue = true;
1703ed29b0b4SJens Axboe }
1704ed29b0b4SJens Axboe
1705ed29b0b4SJens Axboe /*
1706ed29b0b4SJens Axboe * For fast devices, IO may have already completed. If it has, add
1707ed29b0b4SJens Axboe * it to the front so we find it first.
1708ed29b0b4SJens Axboe */
1709ed29b0b4SJens Axboe if (READ_ONCE(req->iopoll_completed))
1710ed29b0b4SJens Axboe wq_list_add_head(&req->comp_list, &ctx->iopoll_list);
1711ed29b0b4SJens Axboe else
1712ed29b0b4SJens Axboe wq_list_add_tail(&req->comp_list, &ctx->iopoll_list);
1713ed29b0b4SJens Axboe
1714ed29b0b4SJens Axboe if (unlikely(needs_lock)) {
1715ed29b0b4SJens Axboe /*
1716ed29b0b4SJens Axboe * If IORING_SETUP_SQPOLL is enabled, sqes are either handle
1717ed29b0b4SJens Axboe * in sq thread task context or in io worker task context. If
1718ed29b0b4SJens Axboe * current task context is sq thread, we don't need to check
1719ed29b0b4SJens Axboe * whether should wake up sq thread.
1720ed29b0b4SJens Axboe */
1721ed29b0b4SJens Axboe if ((ctx->flags & IORING_SETUP_SQPOLL) &&
1722ed29b0b4SJens Axboe wq_has_sleeper(&ctx->sq_data->wait))
1723ed29b0b4SJens Axboe wake_up(&ctx->sq_data->wait);
1724ed29b0b4SJens Axboe
1725ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
1726ed29b0b4SJens Axboe }
1727ed29b0b4SJens Axboe }
1728ed29b0b4SJens Axboe
io_file_get_flags(struct file * file)1729a4ad4f74SJens Axboe unsigned int io_file_get_flags(struct file *file)
1730ed29b0b4SJens Axboe {
1731ed29b0b4SJens Axboe unsigned int res = 0;
1732ed29b0b4SJens Axboe
173353cfd5ceSChristoph Hellwig if (S_ISREG(file_inode(file)->i_mode))
17348487f083SChristoph Hellwig res |= REQ_F_ISREG;
1735b9a6c945SChristoph Hellwig if ((file->f_flags & O_NONBLOCK) || (file->f_mode & FMODE_NOWAIT))
17368487f083SChristoph Hellwig res |= REQ_F_SUPPORT_NOWAIT;
1737ed29b0b4SJens Axboe return res;
1738ed29b0b4SJens Axboe }
1739ed29b0b4SJens Axboe
io_alloc_async_data(struct io_kiocb * req)174099f15d8dSJens Axboe bool io_alloc_async_data(struct io_kiocb *req)
1741ed29b0b4SJens Axboe {
1742f30bd4d0SBreno Leitao WARN_ON_ONCE(!io_cold_defs[req->opcode].async_size);
1743f30bd4d0SBreno Leitao req->async_data = kmalloc(io_cold_defs[req->opcode].async_size, GFP_KERNEL);
1744ed29b0b4SJens Axboe if (req->async_data) {
1745ed29b0b4SJens Axboe req->flags |= REQ_F_ASYNC_DATA;
1746ed29b0b4SJens Axboe return false;
1747ed29b0b4SJens Axboe }
1748ed29b0b4SJens Axboe return true;
1749ed29b0b4SJens Axboe }
1750ed29b0b4SJens Axboe
io_req_prep_async(struct io_kiocb * req)1751f3b44f92SJens Axboe int io_req_prep_async(struct io_kiocb *req)
1752ed29b0b4SJens Axboe {
1753f30bd4d0SBreno Leitao const struct io_cold_def *cdef = &io_cold_defs[req->opcode];
1754a7dd2782SBreno Leitao const struct io_issue_def *def = &io_issue_defs[req->opcode];
1755ed29b0b4SJens Axboe
1756ed29b0b4SJens Axboe /* assign early for deferred execution for non-fixed file */
175754aa7f23SJoseph Qi if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE) && !req->file)
1758ed29b0b4SJens Axboe req->file = io_file_get_normal(req, req->cqe.fd);
1759f30bd4d0SBreno Leitao if (!cdef->prep_async)
1760ed29b0b4SJens Axboe return 0;
1761ed29b0b4SJens Axboe if (WARN_ON_ONCE(req_has_async_data(req)))
1762ed29b0b4SJens Axboe return -EFAULT;
1763f30bd4d0SBreno Leitao if (!def->manual_alloc) {
1764ed29b0b4SJens Axboe if (io_alloc_async_data(req))
1765ed29b0b4SJens Axboe return -EAGAIN;
176659169439SPavel Begunkov }
1767f30bd4d0SBreno Leitao return cdef->prep_async(req);
1768ed29b0b4SJens Axboe }
1769ed29b0b4SJens Axboe
io_get_sequence(struct io_kiocb * req)1770ed29b0b4SJens Axboe static u32 io_get_sequence(struct io_kiocb *req)
1771ed29b0b4SJens Axboe {
1772ed29b0b4SJens Axboe u32 seq = req->ctx->cached_sq_head;
1773ed29b0b4SJens Axboe struct io_kiocb *cur;
1774ed29b0b4SJens Axboe
1775ed29b0b4SJens Axboe /* need original cached_sq_head, but it was increased for each req */
1776ed29b0b4SJens Axboe io_for_each_link(cur, req)
1777ed29b0b4SJens Axboe seq--;
1778ed29b0b4SJens Axboe return seq;
1779ed29b0b4SJens Axboe }
1780ed29b0b4SJens Axboe
io_drain_req(struct io_kiocb * req)1781ed29b0b4SJens Axboe static __cold void io_drain_req(struct io_kiocb *req)
1782e276ae34SPavel Begunkov __must_hold(&ctx->uring_lock)
1783ed29b0b4SJens Axboe {
1784ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
1785ed29b0b4SJens Axboe struct io_defer_entry *de;
1786ed29b0b4SJens Axboe int ret;
1787ed29b0b4SJens Axboe u32 seq = io_get_sequence(req);
1788ed29b0b4SJens Axboe
1789ed29b0b4SJens Axboe /* Still need defer if there is pending req in defer list. */
1790ed29b0b4SJens Axboe spin_lock(&ctx->completion_lock);
1791ed29b0b4SJens Axboe if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) {
1792ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
1793ed29b0b4SJens Axboe queue:
1794ed29b0b4SJens Axboe ctx->drain_active = false;
1795ed29b0b4SJens Axboe io_req_task_queue(req);
1796ed29b0b4SJens Axboe return;
1797ed29b0b4SJens Axboe }
1798ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
1799ed29b0b4SJens Axboe
1800ed29b0b4SJens Axboe io_prep_async_link(req);
1801ed29b0b4SJens Axboe de = kmalloc(sizeof(*de), GFP_KERNEL);
1802ed29b0b4SJens Axboe if (!de) {
1803ed29b0b4SJens Axboe ret = -ENOMEM;
1804ef5c600aSDylan Yudaken io_req_defer_failed(req, ret);
1805ef5c600aSDylan Yudaken return;
1806ed29b0b4SJens Axboe }
1807ed29b0b4SJens Axboe
1808ed29b0b4SJens Axboe spin_lock(&ctx->completion_lock);
1809ed29b0b4SJens Axboe if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
1810ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
1811ed29b0b4SJens Axboe kfree(de);
1812ed29b0b4SJens Axboe goto queue;
1813ed29b0b4SJens Axboe }
1814ed29b0b4SJens Axboe
181548863ffdSPavel Begunkov trace_io_uring_defer(req);
1816ed29b0b4SJens Axboe de->req = req;
1817ed29b0b4SJens Axboe de->seq = seq;
1818ed29b0b4SJens Axboe list_add_tail(&de->list, &ctx->defer_list);
1819ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
1820ed29b0b4SJens Axboe }
1821ed29b0b4SJens Axboe
io_assign_file(struct io_kiocb * req,const struct io_issue_def * def,unsigned int issue_flags)1822f4992544SJens Axboe static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def,
1823f4992544SJens Axboe unsigned int issue_flags)
1824ed29b0b4SJens Axboe {
1825f4992544SJens Axboe if (req->file || !def->needs_file)
1826ed29b0b4SJens Axboe return true;
1827ed29b0b4SJens Axboe
1828ed29b0b4SJens Axboe if (req->flags & REQ_F_FIXED_FILE)
1829ed29b0b4SJens Axboe req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);
1830ed29b0b4SJens Axboe else
1831ed29b0b4SJens Axboe req->file = io_file_get_normal(req, req->cqe.fd);
1832ed29b0b4SJens Axboe
1833ed29b0b4SJens Axboe return !!req->file;
1834ed29b0b4SJens Axboe }
1835ed29b0b4SJens Axboe
io_issue_sqe(struct io_kiocb * req,unsigned int issue_flags)1836ed29b0b4SJens Axboe static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
1837ed29b0b4SJens Axboe {
1838a7dd2782SBreno Leitao const struct io_issue_def *def = &io_issue_defs[req->opcode];
1839ed29b0b4SJens Axboe const struct cred *creds = NULL;
1840ed29b0b4SJens Axboe int ret;
1841ed29b0b4SJens Axboe
1842f4992544SJens Axboe if (unlikely(!io_assign_file(req, def, issue_flags)))
1843ed29b0b4SJens Axboe return -EBADF;
1844ed29b0b4SJens Axboe
1845ed29b0b4SJens Axboe if (unlikely((req->flags & REQ_F_CREDS) && req->creds != current_cred()))
1846ed29b0b4SJens Axboe creds = override_creds(req->creds);
1847ed29b0b4SJens Axboe
1848ed29b0b4SJens Axboe if (!def->audit_skip)
1849ed29b0b4SJens Axboe audit_uring_entry(req->opcode);
1850ed29b0b4SJens Axboe
1851ed29b0b4SJens Axboe ret = def->issue(req, issue_flags);
1852ed29b0b4SJens Axboe
1853ed29b0b4SJens Axboe if (!def->audit_skip)
1854ed29b0b4SJens Axboe audit_uring_exit(!ret, ret);
1855ed29b0b4SJens Axboe
1856ed29b0b4SJens Axboe if (creds)
1857ed29b0b4SJens Axboe revert_creds(creds);
185897b388d7SJens Axboe
185975d7b3aeSPavel Begunkov if (ret == IOU_OK) {
186075d7b3aeSPavel Begunkov if (issue_flags & IO_URING_F_COMPLETE_DEFER)
18619da070b1SPavel Begunkov io_req_complete_defer(req);
186275d7b3aeSPavel Begunkov else
18631bec951cSPavel Begunkov io_req_complete_post(req, issue_flags);
18642c487fbfSPavel Begunkov
18652c487fbfSPavel Begunkov return 0;
18662c487fbfSPavel Begunkov }
18672c487fbfSPavel Begunkov
18682c487fbfSPavel Begunkov if (ret != IOU_ISSUE_SKIP_COMPLETE)
1869ed29b0b4SJens Axboe return ret;
187097b388d7SJens Axboe
1871ed29b0b4SJens Axboe /* If the op doesn't have a file, we're not polling for it */
1872ef0ec1adSPavel Begunkov if ((req->ctx->flags & IORING_SETUP_IOPOLL) && def->iopoll_queue)
1873ed29b0b4SJens Axboe io_iopoll_req_issued(req, issue_flags);
1874ed29b0b4SJens Axboe
1875ed29b0b4SJens Axboe return 0;
1876ed29b0b4SJens Axboe }
1877ed29b0b4SJens Axboe
io_poll_issue(struct io_kiocb * req,struct io_tw_state * ts)1878a282967cSPavel Begunkov int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
1879329061d3SJens Axboe {
1880a282967cSPavel Begunkov io_tw_lock(req->ctx, ts);
18819a692451SDylan Yudaken return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
18829a692451SDylan Yudaken IO_URING_F_COMPLETE_DEFER);
1883329061d3SJens Axboe }
1884329061d3SJens Axboe
io_wq_free_work(struct io_wq_work * work)1885c9f06aa7SJens Axboe struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
1886ed29b0b4SJens Axboe {
1887ed29b0b4SJens Axboe struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1888247f97a5SPavel Begunkov struct io_kiocb *nxt = NULL;
1889ed29b0b4SJens Axboe
1890247f97a5SPavel Begunkov if (req_ref_put_and_test(req)) {
1891247f97a5SPavel Begunkov if (req->flags & IO_REQ_LINK_FLAGS)
1892247f97a5SPavel Begunkov nxt = io_req_find_next(req);
1893247f97a5SPavel Begunkov io_free_req(req);
1894247f97a5SPavel Begunkov }
1895247f97a5SPavel Begunkov return nxt ? &nxt->work : NULL;
1896ed29b0b4SJens Axboe }
1897ed29b0b4SJens Axboe
io_wq_submit_work(struct io_wq_work * work)1898c9f06aa7SJens Axboe void io_wq_submit_work(struct io_wq_work *work)
1899ed29b0b4SJens Axboe {
1900ed29b0b4SJens Axboe struct io_kiocb *req = container_of(work, struct io_kiocb, work);
1901a7dd2782SBreno Leitao const struct io_issue_def *def = &io_issue_defs[req->opcode];
1902e6aeb272SPavel Begunkov unsigned int issue_flags = IO_URING_F_UNLOCKED | IO_URING_F_IOWQ;
1903ed29b0b4SJens Axboe bool needs_poll = false;
1904ed29b0b4SJens Axboe int ret = 0, err = -ECANCELED;
1905ed29b0b4SJens Axboe
190623a6c9acSLin Ma /* one will be dropped by ->io_wq_free_work() after returning to io-wq */
1907ed29b0b4SJens Axboe if (!(req->flags & REQ_F_REFCOUNT))
1908ed29b0b4SJens Axboe __io_req_set_refcount(req, 2);
1909ed29b0b4SJens Axboe else
1910ed29b0b4SJens Axboe req_ref_get(req);
1911ed29b0b4SJens Axboe
1912ed29b0b4SJens Axboe io_arm_ltimeout(req);
1913ed29b0b4SJens Axboe
1914ed29b0b4SJens Axboe /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
1915ed29b0b4SJens Axboe if (work->flags & IO_WQ_WORK_CANCEL) {
1916ed29b0b4SJens Axboe fail:
1917ed29b0b4SJens Axboe io_req_task_queue_fail(req, err);
1918ed29b0b4SJens Axboe return;
1919ed29b0b4SJens Axboe }
1920f4992544SJens Axboe if (!io_assign_file(req, def, issue_flags)) {
1921ed29b0b4SJens Axboe err = -EBADF;
1922ed29b0b4SJens Axboe work->flags |= IO_WQ_WORK_CANCEL;
1923ed29b0b4SJens Axboe goto fail;
1924ed29b0b4SJens Axboe }
1925ed29b0b4SJens Axboe
1926ed29b0b4SJens Axboe if (req->flags & REQ_F_FORCE_ASYNC) {
1927ed29b0b4SJens Axboe bool opcode_poll = def->pollin || def->pollout;
1928ed29b0b4SJens Axboe
1929ed29b0b4SJens Axboe if (opcode_poll && file_can_poll(req->file)) {
1930ed29b0b4SJens Axboe needs_poll = true;
1931ed29b0b4SJens Axboe issue_flags |= IO_URING_F_NONBLOCK;
1932ed29b0b4SJens Axboe }
1933ed29b0b4SJens Axboe }
1934ed29b0b4SJens Axboe
1935ed29b0b4SJens Axboe do {
1936ed29b0b4SJens Axboe ret = io_issue_sqe(req, issue_flags);
1937ed29b0b4SJens Axboe if (ret != -EAGAIN)
1938ed29b0b4SJens Axboe break;
1939a9be2022SJens Axboe
1940a9be2022SJens Axboe /*
1941a9be2022SJens Axboe * If REQ_F_NOWAIT is set, then don't wait or retry with
1942a9be2022SJens Axboe * poll. -EAGAIN is final for that case.
1943a9be2022SJens Axboe */
1944a9be2022SJens Axboe if (req->flags & REQ_F_NOWAIT)
1945a9be2022SJens Axboe break;
1946a9be2022SJens Axboe
1947ed29b0b4SJens Axboe /*
1948ed29b0b4SJens Axboe * We can get EAGAIN for iopolled IO even though we're
1949ed29b0b4SJens Axboe * forcing a sync submission from here, since we can't
1950ed29b0b4SJens Axboe * wait for request slots on the block side.
1951ed29b0b4SJens Axboe */
1952ed29b0b4SJens Axboe if (!needs_poll) {
1953ed29b0b4SJens Axboe if (!(req->ctx->flags & IORING_SETUP_IOPOLL))
1954ed29b0b4SJens Axboe break;
195545500dc4SPavel Begunkov if (io_wq_worker_stopped())
195645500dc4SPavel Begunkov break;
1957ed29b0b4SJens Axboe cond_resched();
1958ed29b0b4SJens Axboe continue;
1959ed29b0b4SJens Axboe }
1960ed29b0b4SJens Axboe
1961ed29b0b4SJens Axboe if (io_arm_poll_handler(req, issue_flags) == IO_APOLL_OK)
1962ed29b0b4SJens Axboe return;
1963ed29b0b4SJens Axboe /* aborted or ready, in either case retry blocking */
1964ed29b0b4SJens Axboe needs_poll = false;
1965ed29b0b4SJens Axboe issue_flags &= ~IO_URING_F_NONBLOCK;
1966ed29b0b4SJens Axboe } while (1);
1967ed29b0b4SJens Axboe
1968ed29b0b4SJens Axboe /* avoid locking problems by failing it from a clean context */
196997b388d7SJens Axboe if (ret < 0)
1970ed29b0b4SJens Axboe io_req_task_queue_fail(req, ret);
1971ed29b0b4SJens Axboe }
1972ed29b0b4SJens Axboe
io_file_get_fixed(struct io_kiocb * req,int fd,unsigned int issue_flags)1973531113bbSJens Axboe inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
1974ed29b0b4SJens Axboe unsigned int issue_flags)
1975ed29b0b4SJens Axboe {
1976ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
19774bfb0c9aSChristoph Hellwig struct io_fixed_file *slot;
1978ed29b0b4SJens Axboe struct file *file = NULL;
1979ed29b0b4SJens Axboe
1980ed29b0b4SJens Axboe io_ring_submit_lock(ctx, issue_flags);
1981ed29b0b4SJens Axboe
1982ed29b0b4SJens Axboe if (unlikely((unsigned int)fd >= ctx->nr_user_files))
1983ed29b0b4SJens Axboe goto out;
1984ed29b0b4SJens Axboe fd = array_index_nospec(fd, ctx->nr_user_files);
19854bfb0c9aSChristoph Hellwig slot = io_fixed_file_slot(&ctx->file_table, fd);
19864bfb0c9aSChristoph Hellwig file = io_slot_file(slot);
19874bfb0c9aSChristoph Hellwig req->flags |= io_slot_flags(slot);
1988ed29b0b4SJens Axboe io_req_set_rsrc_node(req, ctx, 0);
1989ed29b0b4SJens Axboe out:
1990ed29b0b4SJens Axboe io_ring_submit_unlock(ctx, issue_flags);
1991ed29b0b4SJens Axboe return file;
1992ed29b0b4SJens Axboe }
1993ed29b0b4SJens Axboe
io_file_get_normal(struct io_kiocb * req,int fd)1994531113bbSJens Axboe struct file *io_file_get_normal(struct io_kiocb *req, int fd)
1995ed29b0b4SJens Axboe {
1996ed29b0b4SJens Axboe struct file *file = fget(fd);
1997ed29b0b4SJens Axboe
199848863ffdSPavel Begunkov trace_io_uring_file_get(req, fd);
1999ed29b0b4SJens Axboe
2000ed29b0b4SJens Axboe /* we don't allow fixed io_uring files */
2001e5550a14SJens Axboe if (file && io_is_uring_fops(file))
2002ed29b0b4SJens Axboe io_req_track_inflight(req);
2003ed29b0b4SJens Axboe return file;
2004ed29b0b4SJens Axboe }
2005ed29b0b4SJens Axboe
io_queue_async(struct io_kiocb * req,int ret)2006ed29b0b4SJens Axboe static void io_queue_async(struct io_kiocb *req, int ret)
2007ed29b0b4SJens Axboe __must_hold(&req->ctx->uring_lock)
2008ed29b0b4SJens Axboe {
2009ed29b0b4SJens Axboe struct io_kiocb *linked_timeout;
2010ed29b0b4SJens Axboe
2011ed29b0b4SJens Axboe if (ret != -EAGAIN || (req->flags & REQ_F_NOWAIT)) {
2012973fc83fSDylan Yudaken io_req_defer_failed(req, ret);
2013ed29b0b4SJens Axboe return;
2014ed29b0b4SJens Axboe }
2015ed29b0b4SJens Axboe
2016ed29b0b4SJens Axboe linked_timeout = io_prep_linked_timeout(req);
2017ed29b0b4SJens Axboe
2018ed29b0b4SJens Axboe switch (io_arm_poll_handler(req, 0)) {
2019ed29b0b4SJens Axboe case IO_APOLL_READY:
2020336d28a8SPavel Begunkov io_kbuf_recycle(req, 0);
2021ed29b0b4SJens Axboe io_req_task_queue(req);
2022ed29b0b4SJens Axboe break;
2023ed29b0b4SJens Axboe case IO_APOLL_ABORTED:
2024ed29b0b4SJens Axboe io_kbuf_recycle(req, 0);
2025ed29b0b4SJens Axboe io_queue_iowq(req, NULL);
2026ed29b0b4SJens Axboe break;
2027ed29b0b4SJens Axboe case IO_APOLL_OK:
2028ed29b0b4SJens Axboe break;
2029ed29b0b4SJens Axboe }
2030ed29b0b4SJens Axboe
2031ed29b0b4SJens Axboe if (linked_timeout)
2032ed29b0b4SJens Axboe io_queue_linked_timeout(linked_timeout);
2033ed29b0b4SJens Axboe }
2034ed29b0b4SJens Axboe
io_queue_sqe(struct io_kiocb * req)2035ed29b0b4SJens Axboe static inline void io_queue_sqe(struct io_kiocb *req)
2036ed29b0b4SJens Axboe __must_hold(&req->ctx->uring_lock)
2037ed29b0b4SJens Axboe {
2038ed29b0b4SJens Axboe int ret;
2039ed29b0b4SJens Axboe
2040ed29b0b4SJens Axboe ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
2041ed29b0b4SJens Axboe
2042ed29b0b4SJens Axboe /*
2043ed29b0b4SJens Axboe * We async punt it if the file wasn't marked NOWAIT, or if the file
2044ed29b0b4SJens Axboe * doesn't support non-blocking read/write attempts
2045ed29b0b4SJens Axboe */
2046ed29b0b4SJens Axboe if (likely(!ret))
2047ed29b0b4SJens Axboe io_arm_ltimeout(req);
2048ed29b0b4SJens Axboe else
2049ed29b0b4SJens Axboe io_queue_async(req, ret);
2050ed29b0b4SJens Axboe }
2051ed29b0b4SJens Axboe
io_queue_sqe_fallback(struct io_kiocb * req)2052ed29b0b4SJens Axboe static void io_queue_sqe_fallback(struct io_kiocb *req)
2053ed29b0b4SJens Axboe __must_hold(&req->ctx->uring_lock)
2054ed29b0b4SJens Axboe {
2055ed29b0b4SJens Axboe if (unlikely(req->flags & REQ_F_FAIL)) {
2056ed29b0b4SJens Axboe /*
2057ed29b0b4SJens Axboe * We don't submit, fail them all, for that replace hardlinks
2058ed29b0b4SJens Axboe * with normal links. Extra REQ_F_LINK is tolerated.
2059ed29b0b4SJens Axboe */
2060ed29b0b4SJens Axboe req->flags &= ~REQ_F_HARDLINK;
2061ed29b0b4SJens Axboe req->flags |= REQ_F_LINK;
2062973fc83fSDylan Yudaken io_req_defer_failed(req, req->cqe.res);
2063ed29b0b4SJens Axboe } else {
2064ed29b0b4SJens Axboe int ret = io_req_prep_async(req);
2065ed29b0b4SJens Axboe
2066ef5c600aSDylan Yudaken if (unlikely(ret)) {
2067973fc83fSDylan Yudaken io_req_defer_failed(req, ret);
2068ef5c600aSDylan Yudaken return;
2069ef5c600aSDylan Yudaken }
2070ef5c600aSDylan Yudaken
2071ef5c600aSDylan Yudaken if (unlikely(req->ctx->drain_active))
2072ef5c600aSDylan Yudaken io_drain_req(req);
2073ed29b0b4SJens Axboe else
2074ed29b0b4SJens Axboe io_queue_iowq(req, NULL);
2075ed29b0b4SJens Axboe }
2076ed29b0b4SJens Axboe }
2077ed29b0b4SJens Axboe
2078ed29b0b4SJens Axboe /*
2079ed29b0b4SJens Axboe * Check SQE restrictions (opcode and flags).
2080ed29b0b4SJens Axboe *
2081ed29b0b4SJens Axboe * Returns 'true' if SQE is allowed, 'false' otherwise.
2082ed29b0b4SJens Axboe */
io_check_restriction(struct io_ring_ctx * ctx,struct io_kiocb * req,unsigned int sqe_flags)2083ed29b0b4SJens Axboe static inline bool io_check_restriction(struct io_ring_ctx *ctx,
2084ed29b0b4SJens Axboe struct io_kiocb *req,
2085ed29b0b4SJens Axboe unsigned int sqe_flags)
2086ed29b0b4SJens Axboe {
2087ed29b0b4SJens Axboe if (!test_bit(req->opcode, ctx->restrictions.sqe_op))
2088ed29b0b4SJens Axboe return false;
2089ed29b0b4SJens Axboe
2090ed29b0b4SJens Axboe if ((sqe_flags & ctx->restrictions.sqe_flags_required) !=
2091ed29b0b4SJens Axboe ctx->restrictions.sqe_flags_required)
2092ed29b0b4SJens Axboe return false;
2093ed29b0b4SJens Axboe
2094ed29b0b4SJens Axboe if (sqe_flags & ~(ctx->restrictions.sqe_flags_allowed |
2095ed29b0b4SJens Axboe ctx->restrictions.sqe_flags_required))
2096ed29b0b4SJens Axboe return false;
2097ed29b0b4SJens Axboe
2098ed29b0b4SJens Axboe return true;
2099ed29b0b4SJens Axboe }
2100ed29b0b4SJens Axboe
io_init_req_drain(struct io_kiocb * req)2101ed29b0b4SJens Axboe static void io_init_req_drain(struct io_kiocb *req)
2102ed29b0b4SJens Axboe {
2103ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
2104ed29b0b4SJens Axboe struct io_kiocb *head = ctx->submit_state.link.head;
2105ed29b0b4SJens Axboe
2106ed29b0b4SJens Axboe ctx->drain_active = true;
2107ed29b0b4SJens Axboe if (head) {
2108ed29b0b4SJens Axboe /*
2109ed29b0b4SJens Axboe * If we need to drain a request in the middle of a link, drain
2110ed29b0b4SJens Axboe * the head request and the next request/link after the current
2111ed29b0b4SJens Axboe * link. Considering sequential execution of links,
2112ed29b0b4SJens Axboe * REQ_F_IO_DRAIN will be maintained for every request of our
2113ed29b0b4SJens Axboe * link.
2114ed29b0b4SJens Axboe */
2115ed29b0b4SJens Axboe head->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
2116ed29b0b4SJens Axboe ctx->drain_next = true;
2117ed29b0b4SJens Axboe }
2118ed29b0b4SJens Axboe }
2119ed29b0b4SJens Axboe
io_init_fail_req(struct io_kiocb * req,int err)212021162ad2SJens Axboe static __cold int io_init_fail_req(struct io_kiocb *req, int err)
212121162ad2SJens Axboe {
212221162ad2SJens Axboe /* ensure per-opcode data is cleared if we fail before prep */
212321162ad2SJens Axboe memset(&req->cmd.data, 0, sizeof(req->cmd.data));
212421162ad2SJens Axboe return err;
212521162ad2SJens Axboe }
212621162ad2SJens Axboe
io_init_req(struct io_ring_ctx * ctx,struct io_kiocb * req,const struct io_uring_sqe * sqe)2127ed29b0b4SJens Axboe static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
2128ed29b0b4SJens Axboe const struct io_uring_sqe *sqe)
2129ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
2130ed29b0b4SJens Axboe {
2131a7dd2782SBreno Leitao const struct io_issue_def *def;
2132ed29b0b4SJens Axboe unsigned int sqe_flags;
2133ed29b0b4SJens Axboe int personality;
2134ed29b0b4SJens Axboe u8 opcode;
2135ed29b0b4SJens Axboe
2136ed29b0b4SJens Axboe /* req is partially pre-initialised, see io_preinit_req() */
2137ed29b0b4SJens Axboe req->opcode = opcode = READ_ONCE(sqe->opcode);
2138ed29b0b4SJens Axboe /* same numerical values with corresponding REQ_F_*, safe to copy */
2139ed29b0b4SJens Axboe req->flags = sqe_flags = READ_ONCE(sqe->flags);
2140ed29b0b4SJens Axboe req->cqe.user_data = READ_ONCE(sqe->user_data);
2141ed29b0b4SJens Axboe req->file = NULL;
2142ed29b0b4SJens Axboe req->rsrc_node = NULL;
2143ed29b0b4SJens Axboe req->task = current;
2144ed29b0b4SJens Axboe
2145ed29b0b4SJens Axboe if (unlikely(opcode >= IORING_OP_LAST)) {
2146ed29b0b4SJens Axboe req->opcode = 0;
214721162ad2SJens Axboe return io_init_fail_req(req, -EINVAL);
2148ed29b0b4SJens Axboe }
2149a7dd2782SBreno Leitao def = &io_issue_defs[opcode];
2150ed29b0b4SJens Axboe if (unlikely(sqe_flags & ~SQE_COMMON_FLAGS)) {
2151ed29b0b4SJens Axboe /* enforce forwards compatibility on users */
2152ed29b0b4SJens Axboe if (sqe_flags & ~SQE_VALID_FLAGS)
215321162ad2SJens Axboe return io_init_fail_req(req, -EINVAL);
2154ed29b0b4SJens Axboe if (sqe_flags & IOSQE_BUFFER_SELECT) {
2155ed29b0b4SJens Axboe if (!def->buffer_select)
215621162ad2SJens Axboe return io_init_fail_req(req, -EOPNOTSUPP);
2157ed29b0b4SJens Axboe req->buf_index = READ_ONCE(sqe->buf_group);
2158ed29b0b4SJens Axboe }
2159ed29b0b4SJens Axboe if (sqe_flags & IOSQE_CQE_SKIP_SUCCESS)
2160ed29b0b4SJens Axboe ctx->drain_disabled = true;
2161ed29b0b4SJens Axboe if (sqe_flags & IOSQE_IO_DRAIN) {
2162ed29b0b4SJens Axboe if (ctx->drain_disabled)
216321162ad2SJens Axboe return io_init_fail_req(req, -EOPNOTSUPP);
2164ed29b0b4SJens Axboe io_init_req_drain(req);
2165ed29b0b4SJens Axboe }
2166ed29b0b4SJens Axboe }
2167ed29b0b4SJens Axboe if (unlikely(ctx->restricted || ctx->drain_active || ctx->drain_next)) {
2168ed29b0b4SJens Axboe if (ctx->restricted && !io_check_restriction(ctx, req, sqe_flags))
216921162ad2SJens Axboe return io_init_fail_req(req, -EACCES);
2170ed29b0b4SJens Axboe /* knock it to the slow queue path, will be drained there */
2171ed29b0b4SJens Axboe if (ctx->drain_active)
2172ed29b0b4SJens Axboe req->flags |= REQ_F_FORCE_ASYNC;
2173ed29b0b4SJens Axboe /* if there is no link, we're at "next" request and need to drain */
2174ed29b0b4SJens Axboe if (unlikely(ctx->drain_next) && !ctx->submit_state.link.head) {
2175ed29b0b4SJens Axboe ctx->drain_next = false;
2176ed29b0b4SJens Axboe ctx->drain_active = true;
2177ed29b0b4SJens Axboe req->flags |= REQ_F_IO_DRAIN | REQ_F_FORCE_ASYNC;
2178ed29b0b4SJens Axboe }
2179ed29b0b4SJens Axboe }
2180ed29b0b4SJens Axboe
2181ed29b0b4SJens Axboe if (!def->ioprio && sqe->ioprio)
218221162ad2SJens Axboe return io_init_fail_req(req, -EINVAL);
2183ed29b0b4SJens Axboe if (!def->iopoll && (ctx->flags & IORING_SETUP_IOPOLL))
218421162ad2SJens Axboe return io_init_fail_req(req, -EINVAL);
2185ed29b0b4SJens Axboe
2186ed29b0b4SJens Axboe if (def->needs_file) {
2187ed29b0b4SJens Axboe struct io_submit_state *state = &ctx->submit_state;
2188ed29b0b4SJens Axboe
2189ed29b0b4SJens Axboe req->cqe.fd = READ_ONCE(sqe->fd);
2190ed29b0b4SJens Axboe
2191ed29b0b4SJens Axboe /*
2192ed29b0b4SJens Axboe * Plug now if we have more than 2 IO left after this, and the
2193ed29b0b4SJens Axboe * target is potentially a read/write to block based storage.
2194ed29b0b4SJens Axboe */
2195ed29b0b4SJens Axboe if (state->need_plug && def->plug) {
2196ed29b0b4SJens Axboe state->plug_started = true;
2197ed29b0b4SJens Axboe state->need_plug = false;
2198ed29b0b4SJens Axboe blk_start_plug_nr_ios(&state->plug, state->submit_nr);
2199ed29b0b4SJens Axboe }
2200ed29b0b4SJens Axboe }
2201ed29b0b4SJens Axboe
2202ed29b0b4SJens Axboe personality = READ_ONCE(sqe->personality);
2203ed29b0b4SJens Axboe if (personality) {
2204ed29b0b4SJens Axboe int ret;
2205ed29b0b4SJens Axboe
2206ed29b0b4SJens Axboe req->creds = xa_load(&ctx->personalities, personality);
2207ed29b0b4SJens Axboe if (!req->creds)
220821162ad2SJens Axboe return io_init_fail_req(req, -EINVAL);
2209ed29b0b4SJens Axboe get_cred(req->creds);
2210ed29b0b4SJens Axboe ret = security_uring_override_creds(req->creds);
2211ed29b0b4SJens Axboe if (ret) {
2212ed29b0b4SJens Axboe put_cred(req->creds);
221321162ad2SJens Axboe return io_init_fail_req(req, ret);
2214ed29b0b4SJens Axboe }
2215ed29b0b4SJens Axboe req->flags |= REQ_F_CREDS;
2216ed29b0b4SJens Axboe }
2217ed29b0b4SJens Axboe
2218ed29b0b4SJens Axboe return def->prep(req, sqe);
2219ed29b0b4SJens Axboe }
2220ed29b0b4SJens Axboe
io_submit_fail_init(const struct io_uring_sqe * sqe,struct io_kiocb * req,int ret)2221ed29b0b4SJens Axboe static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
2222ed29b0b4SJens Axboe struct io_kiocb *req, int ret)
2223ed29b0b4SJens Axboe {
2224ed29b0b4SJens Axboe struct io_ring_ctx *ctx = req->ctx;
2225ed29b0b4SJens Axboe struct io_submit_link *link = &ctx->submit_state.link;
2226ed29b0b4SJens Axboe struct io_kiocb *head = link->head;
2227ed29b0b4SJens Axboe
222848863ffdSPavel Begunkov trace_io_uring_req_failed(sqe, req, ret);
2229ed29b0b4SJens Axboe
2230ed29b0b4SJens Axboe /*
2231ed29b0b4SJens Axboe * Avoid breaking links in the middle as it renders links with SQPOLL
2232ed29b0b4SJens Axboe * unusable. Instead of failing eagerly, continue assembling the link if
2233ed29b0b4SJens Axboe * applicable and mark the head with REQ_F_FAIL. The link flushing code
2234ed29b0b4SJens Axboe * should find the flag and handle the rest.
2235ed29b0b4SJens Axboe */
2236ed29b0b4SJens Axboe req_fail_link_node(req, ret);
2237ed29b0b4SJens Axboe if (head && !(head->flags & REQ_F_FAIL))
2238ed29b0b4SJens Axboe req_fail_link_node(head, -ECANCELED);
2239ed29b0b4SJens Axboe
2240ed29b0b4SJens Axboe if (!(req->flags & IO_REQ_LINK_FLAGS)) {
2241ed29b0b4SJens Axboe if (head) {
2242ed29b0b4SJens Axboe link->last->link = req;
2243ed29b0b4SJens Axboe link->head = NULL;
2244ed29b0b4SJens Axboe req = head;
2245ed29b0b4SJens Axboe }
2246ed29b0b4SJens Axboe io_queue_sqe_fallback(req);
2247ed29b0b4SJens Axboe return ret;
2248ed29b0b4SJens Axboe }
2249ed29b0b4SJens Axboe
2250ed29b0b4SJens Axboe if (head)
2251ed29b0b4SJens Axboe link->last->link = req;
2252ed29b0b4SJens Axboe else
2253ed29b0b4SJens Axboe link->head = req;
2254ed29b0b4SJens Axboe link->last = req;
2255ed29b0b4SJens Axboe return 0;
2256ed29b0b4SJens Axboe }
2257ed29b0b4SJens Axboe
io_submit_sqe(struct io_ring_ctx * ctx,struct io_kiocb * req,const struct io_uring_sqe * sqe)2258ed29b0b4SJens Axboe static inline int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
2259ed29b0b4SJens Axboe const struct io_uring_sqe *sqe)
2260ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
2261ed29b0b4SJens Axboe {
2262ed29b0b4SJens Axboe struct io_submit_link *link = &ctx->submit_state.link;
2263ed29b0b4SJens Axboe int ret;
2264ed29b0b4SJens Axboe
2265ed29b0b4SJens Axboe ret = io_init_req(ctx, req, sqe);
2266ed29b0b4SJens Axboe if (unlikely(ret))
2267ed29b0b4SJens Axboe return io_submit_fail_init(sqe, req, ret);
2268ed29b0b4SJens Axboe
22692ad57931SJens Axboe trace_io_uring_submit_req(req);
2270ed29b0b4SJens Axboe
2271ed29b0b4SJens Axboe /*
2272ed29b0b4SJens Axboe * If we already have a head request, queue this one for async
2273ed29b0b4SJens Axboe * submittal once the head completes. If we don't have a head but
2274ed29b0b4SJens Axboe * IOSQE_IO_LINK is set in the sqe, start a new head. This one will be
2275ed29b0b4SJens Axboe * submitted sync once the chain is complete. If none of those
2276ed29b0b4SJens Axboe * conditions are true (normal request), then just queue it.
2277ed29b0b4SJens Axboe */
2278ed29b0b4SJens Axboe if (unlikely(link->head)) {
2279ed29b0b4SJens Axboe ret = io_req_prep_async(req);
2280ed29b0b4SJens Axboe if (unlikely(ret))
2281ed29b0b4SJens Axboe return io_submit_fail_init(sqe, req, ret);
2282ed29b0b4SJens Axboe
228348863ffdSPavel Begunkov trace_io_uring_link(req, link->head);
2284ed29b0b4SJens Axboe link->last->link = req;
2285ed29b0b4SJens Axboe link->last = req;
2286ed29b0b4SJens Axboe
2287ed29b0b4SJens Axboe if (req->flags & IO_REQ_LINK_FLAGS)
2288ed29b0b4SJens Axboe return 0;
2289ed29b0b4SJens Axboe /* last request of the link, flush it */
2290ed29b0b4SJens Axboe req = link->head;
2291ed29b0b4SJens Axboe link->head = NULL;
2292ed29b0b4SJens Axboe if (req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL))
2293ed29b0b4SJens Axboe goto fallback;
2294ed29b0b4SJens Axboe
2295ed29b0b4SJens Axboe } else if (unlikely(req->flags & (IO_REQ_LINK_FLAGS |
2296ed29b0b4SJens Axboe REQ_F_FORCE_ASYNC | REQ_F_FAIL))) {
2297ed29b0b4SJens Axboe if (req->flags & IO_REQ_LINK_FLAGS) {
2298ed29b0b4SJens Axboe link->head = req;
2299ed29b0b4SJens Axboe link->last = req;
2300ed29b0b4SJens Axboe } else {
2301ed29b0b4SJens Axboe fallback:
2302ed29b0b4SJens Axboe io_queue_sqe_fallback(req);
2303ed29b0b4SJens Axboe }
2304ed29b0b4SJens Axboe return 0;
2305ed29b0b4SJens Axboe }
2306ed29b0b4SJens Axboe
2307ed29b0b4SJens Axboe io_queue_sqe(req);
2308ed29b0b4SJens Axboe return 0;
2309ed29b0b4SJens Axboe }
2310ed29b0b4SJens Axboe
2311ed29b0b4SJens Axboe /*
2312ed29b0b4SJens Axboe * Batched submission is done, ensure local IO is flushed out.
2313ed29b0b4SJens Axboe */
io_submit_state_end(struct io_ring_ctx * ctx)2314ed29b0b4SJens Axboe static void io_submit_state_end(struct io_ring_ctx *ctx)
2315ed29b0b4SJens Axboe {
2316ed29b0b4SJens Axboe struct io_submit_state *state = &ctx->submit_state;
2317ed29b0b4SJens Axboe
2318ed29b0b4SJens Axboe if (unlikely(state->link.head))
2319ed29b0b4SJens Axboe io_queue_sqe_fallback(state->link.head);
2320ed29b0b4SJens Axboe /* flush only after queuing links as they can generate completions */
2321ed29b0b4SJens Axboe io_submit_flush_completions(ctx);
2322ed29b0b4SJens Axboe if (state->plug_started)
2323ed29b0b4SJens Axboe blk_finish_plug(&state->plug);
2324ed29b0b4SJens Axboe }
2325ed29b0b4SJens Axboe
2326ed29b0b4SJens Axboe /*
2327ed29b0b4SJens Axboe * Start submission side cache.
2328ed29b0b4SJens Axboe */
io_submit_state_start(struct io_submit_state * state,unsigned int max_ios)2329ed29b0b4SJens Axboe static void io_submit_state_start(struct io_submit_state *state,
2330ed29b0b4SJens Axboe unsigned int max_ios)
2331ed29b0b4SJens Axboe {
2332ed29b0b4SJens Axboe state->plug_started = false;
2333ed29b0b4SJens Axboe state->need_plug = max_ios > 2;
2334ed29b0b4SJens Axboe state->submit_nr = max_ios;
2335ed29b0b4SJens Axboe /* set only head, no need to init link_last in advance */
2336ed29b0b4SJens Axboe state->link.head = NULL;
2337ed29b0b4SJens Axboe }
2338ed29b0b4SJens Axboe
io_commit_sqring(struct io_ring_ctx * ctx)2339ed29b0b4SJens Axboe static void io_commit_sqring(struct io_ring_ctx *ctx)
2340ed29b0b4SJens Axboe {
2341ed29b0b4SJens Axboe struct io_rings *rings = ctx->rings;
2342ed29b0b4SJens Axboe
2343ed29b0b4SJens Axboe /*
2344ed29b0b4SJens Axboe * Ensure any loads from the SQEs are done at this point,
2345ed29b0b4SJens Axboe * since once we write the new head, the application could
2346ed29b0b4SJens Axboe * write new data to them.
2347ed29b0b4SJens Axboe */
2348ed29b0b4SJens Axboe smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2349ed29b0b4SJens Axboe }
2350ed29b0b4SJens Axboe
2351ed29b0b4SJens Axboe /*
2352ed29b0b4SJens Axboe * Fetch an sqe, if one is available. Note this returns a pointer to memory
2353ed29b0b4SJens Axboe * that is mapped by userspace. This means that care needs to be taken to
2354ed29b0b4SJens Axboe * ensure that reads are stable, as we cannot rely on userspace always
2355ed29b0b4SJens Axboe * being a good citizen. If members of the sqe are validated and then later
2356ed29b0b4SJens Axboe * used, it's important that those reads are done through READ_ONCE() to
2357ed29b0b4SJens Axboe * prevent a re-load down the line.
2358ed29b0b4SJens Axboe */
io_get_sqe(struct io_ring_ctx * ctx,const struct io_uring_sqe ** sqe)2359b5083dfaSPavel Begunkov static bool io_get_sqe(struct io_ring_ctx *ctx, const struct io_uring_sqe **sqe)
2360ed29b0b4SJens Axboe {
23612af89abdSPavel Begunkov unsigned mask = ctx->sq_entries - 1;
23622af89abdSPavel Begunkov unsigned head = ctx->cached_sq_head++ & mask;
23632af89abdSPavel Begunkov
23642af89abdSPavel Begunkov if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) {
23652af89abdSPavel Begunkov head = READ_ONCE(ctx->sq_array[head]);
23662af89abdSPavel Begunkov if (unlikely(head >= ctx->sq_entries)) {
23672af89abdSPavel Begunkov /* drop invalid entries */
23682af89abdSPavel Begunkov spin_lock(&ctx->completion_lock);
23692af89abdSPavel Begunkov ctx->cq_extra--;
23702af89abdSPavel Begunkov spin_unlock(&ctx->completion_lock);
23712af89abdSPavel Begunkov WRITE_ONCE(ctx->rings->sq_dropped,
23722af89abdSPavel Begunkov READ_ONCE(ctx->rings->sq_dropped) + 1);
23732af89abdSPavel Begunkov return false;
23742af89abdSPavel Begunkov }
23752af89abdSPavel Begunkov }
2376ed29b0b4SJens Axboe
2377ed29b0b4SJens Axboe /*
2378ed29b0b4SJens Axboe * The cached sq head (or cq tail) serves two purposes:
2379ed29b0b4SJens Axboe *
2380ed29b0b4SJens Axboe * 1) allows us to batch the cost of updating the user visible
2381ed29b0b4SJens Axboe * head updates.
2382ed29b0b4SJens Axboe * 2) allows the kernel side to track the head on its own, even
2383ed29b0b4SJens Axboe * though the application is the one updating it.
2384ed29b0b4SJens Axboe */
23852af89abdSPavel Begunkov
2386ed29b0b4SJens Axboe /* double index for 128-byte SQEs, twice as long */
2387ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_SQE128)
2388ed29b0b4SJens Axboe head <<= 1;
2389b5083dfaSPavel Begunkov *sqe = &ctx->sq_sqes[head];
2390b5083dfaSPavel Begunkov return true;
2391ed29b0b4SJens Axboe }
2392ed29b0b4SJens Axboe
io_submit_sqes(struct io_ring_ctx * ctx,unsigned int nr)239317437f31SJens Axboe int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
2394ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
2395ed29b0b4SJens Axboe {
2396ed29b0b4SJens Axboe unsigned int entries = io_sqring_entries(ctx);
2397ed29b0b4SJens Axboe unsigned int left;
2398ed29b0b4SJens Axboe int ret;
2399ed29b0b4SJens Axboe
2400ed29b0b4SJens Axboe if (unlikely(!entries))
2401ed29b0b4SJens Axboe return 0;
2402ed29b0b4SJens Axboe /* make sure SQ entry isn't read before tail */
2403e3ef728fSJens Axboe ret = left = min(nr, entries);
2404ed29b0b4SJens Axboe io_get_task_refs(left);
2405ed29b0b4SJens Axboe io_submit_state_start(&ctx->submit_state, left);
2406ed29b0b4SJens Axboe
2407ed29b0b4SJens Axboe do {
2408ed29b0b4SJens Axboe const struct io_uring_sqe *sqe;
2409ed29b0b4SJens Axboe struct io_kiocb *req;
2410ed29b0b4SJens Axboe
2411c8576f3eSPavel Begunkov if (unlikely(!io_alloc_req(ctx, &req)))
2412ed29b0b4SJens Axboe break;
2413b5083dfaSPavel Begunkov if (unlikely(!io_get_sqe(ctx, &sqe))) {
2414ed29b0b4SJens Axboe io_req_add_to_cache(req, ctx);
2415ed29b0b4SJens Axboe break;
2416ed29b0b4SJens Axboe }
2417ed29b0b4SJens Axboe
2418ed29b0b4SJens Axboe /*
2419ed29b0b4SJens Axboe * Continue submitting even for sqe failure if the
2420ed29b0b4SJens Axboe * ring was setup with IORING_SETUP_SUBMIT_ALL
2421ed29b0b4SJens Axboe */
2422ed29b0b4SJens Axboe if (unlikely(io_submit_sqe(ctx, req, sqe)) &&
2423ed29b0b4SJens Axboe !(ctx->flags & IORING_SETUP_SUBMIT_ALL)) {
2424ed29b0b4SJens Axboe left--;
2425ed29b0b4SJens Axboe break;
2426ed29b0b4SJens Axboe }
2427ed29b0b4SJens Axboe } while (--left);
2428ed29b0b4SJens Axboe
2429ed29b0b4SJens Axboe if (unlikely(left)) {
2430ed29b0b4SJens Axboe ret -= left;
2431ed29b0b4SJens Axboe /* try again if it submitted nothing and can't allocate a req */
2432ed29b0b4SJens Axboe if (!ret && io_req_cache_empty(ctx))
2433ed29b0b4SJens Axboe ret = -EAGAIN;
2434ed29b0b4SJens Axboe current->io_uring->cached_refs += left;
2435ed29b0b4SJens Axboe }
2436ed29b0b4SJens Axboe
2437ed29b0b4SJens Axboe io_submit_state_end(ctx);
2438ed29b0b4SJens Axboe /* Commit SQ ring head once we've consumed and submitted all SQEs */
2439ed29b0b4SJens Axboe io_commit_sqring(ctx);
2440ed29b0b4SJens Axboe return ret;
2441ed29b0b4SJens Axboe }
2442ed29b0b4SJens Axboe
2443ed29b0b4SJens Axboe struct io_wait_queue {
2444ed29b0b4SJens Axboe struct wait_queue_entry wq;
2445ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
2446ed29b0b4SJens Axboe unsigned cq_tail;
2447ed29b0b4SJens Axboe unsigned nr_timeouts;
2448d33a39e5SPavel Begunkov ktime_t timeout;
2449ed29b0b4SJens Axboe };
2450ed29b0b4SJens Axboe
io_has_work(struct io_ring_ctx * ctx)2451b4c98d59SDylan Yudaken static inline bool io_has_work(struct io_ring_ctx *ctx)
2452b4c98d59SDylan Yudaken {
2453c0e0d6baSDylan Yudaken return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) ||
2454490c00ebSPavel Begunkov !llist_empty(&ctx->work_llist);
2455b4c98d59SDylan Yudaken }
2456b4c98d59SDylan Yudaken
io_should_wake(struct io_wait_queue * iowq)2457ed29b0b4SJens Axboe static inline bool io_should_wake(struct io_wait_queue *iowq)
2458ed29b0b4SJens Axboe {
2459ed29b0b4SJens Axboe struct io_ring_ctx *ctx = iowq->ctx;
24600fc8c2acSDylan Yudaken int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;
2461ed29b0b4SJens Axboe
2462ed29b0b4SJens Axboe /*
2463ed29b0b4SJens Axboe * Wake up if we have enough events, or if a timeout occurred since we
2464ed29b0b4SJens Axboe * started waiting. For timeouts, we always want to return to userspace,
2465ed29b0b4SJens Axboe * regardless of event count.
2466ed29b0b4SJens Axboe */
2467ed29b0b4SJens Axboe return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts;
2468ed29b0b4SJens Axboe }
2469ed29b0b4SJens Axboe
io_wake_function(struct wait_queue_entry * curr,unsigned int mode,int wake_flags,void * key)2470ed29b0b4SJens Axboe static int io_wake_function(struct wait_queue_entry *curr, unsigned int mode,
2471ed29b0b4SJens Axboe int wake_flags, void *key)
2472ed29b0b4SJens Axboe {
2473bd550173SPavel Begunkov struct io_wait_queue *iowq = container_of(curr, struct io_wait_queue, wq);
2474ed29b0b4SJens Axboe
2475ed29b0b4SJens Axboe /*
2476ed29b0b4SJens Axboe * Cannot safely flush overflowed CQEs from here, ensure we wake up
2477ed29b0b4SJens Axboe * the task, and the next invocation will do it.
2478ed29b0b4SJens Axboe */
2479bd550173SPavel Begunkov if (io_should_wake(iowq) || io_has_work(iowq->ctx))
2480ed29b0b4SJens Axboe return autoremove_wake_function(curr, mode, wake_flags, key);
2481ed29b0b4SJens Axboe return -1;
2482ed29b0b4SJens Axboe }
2483ed29b0b4SJens Axboe
io_run_task_work_sig(struct io_ring_ctx * ctx)2484c0e0d6baSDylan Yudaken int io_run_task_work_sig(struct io_ring_ctx *ctx)
2485ed29b0b4SJens Axboe {
24861414d629SPavel Begunkov if (!llist_empty(&ctx->work_llist)) {
24872f413956SPavel Begunkov __set_current_state(TASK_RUNNING);
24887b8fa7a0SJens Axboe if (io_run_local_work(ctx, INT_MAX) > 0)
2489d246c759SPavel Begunkov return 0;
24901414d629SPavel Begunkov }
24911414d629SPavel Begunkov if (io_run_task_work() > 0)
2492d246c759SPavel Begunkov return 0;
2493ed29b0b4SJens Axboe if (task_sigpending(current))
2494ed29b0b4SJens Axboe return -EINTR;
2495ed29b0b4SJens Axboe return 0;
2496ed29b0b4SJens Axboe }
2497ed29b0b4SJens Axboe
current_pending_io(void)24987b72d661SJens Axboe static bool current_pending_io(void)
24997b72d661SJens Axboe {
25007b72d661SJens Axboe struct io_uring_task *tctx = current->io_uring;
25017b72d661SJens Axboe
25027b72d661SJens Axboe if (!tctx)
25037b72d661SJens Axboe return false;
25047b72d661SJens Axboe return percpu_counter_read_positive(&tctx->inflight);
25057b72d661SJens Axboe }
25067b72d661SJens Axboe
2507ed29b0b4SJens Axboe /* when returns >0, the caller should retry */
io_cqring_wait_schedule(struct io_ring_ctx * ctx,struct io_wait_queue * iowq)2508ed29b0b4SJens Axboe static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
2509d33a39e5SPavel Begunkov struct io_wait_queue *iowq)
2510ed29b0b4SJens Axboe {
2511e627c433SJens Axboe int ret;
25128a796565SAndres Freund
25133fcf19d5SPavel Begunkov if (unlikely(READ_ONCE(ctx->check_cq)))
25143fcf19d5SPavel Begunkov return 1;
2515846072f1SPavel Begunkov if (unlikely(!llist_empty(&ctx->work_llist)))
2516846072f1SPavel Begunkov return 1;
2517846072f1SPavel Begunkov if (unlikely(test_thread_flag(TIF_NOTIFY_SIGNAL)))
2518846072f1SPavel Begunkov return 1;
2519846072f1SPavel Begunkov if (unlikely(task_sigpending(current)))
2520846072f1SPavel Begunkov return -EINTR;
2521846072f1SPavel Begunkov if (unlikely(io_should_wake(iowq)))
2522846072f1SPavel Begunkov return 0;
25238a796565SAndres Freund
25248a796565SAndres Freund /*
25257b72d661SJens Axboe * Mark us as being in io_wait if we have pending requests, so cpufreq
25267b72d661SJens Axboe * can take into account that the task is waiting for IO - turns out
25277b72d661SJens Axboe * to be important for low QD IO.
25288a796565SAndres Freund */
25297b72d661SJens Axboe if (current_pending_io())
25307b72d661SJens Axboe current->in_iowait = 1;
25318a796565SAndres Freund ret = 0;
2532d33a39e5SPavel Begunkov if (iowq->timeout == KTIME_MAX)
253346ae7eefSPavel Begunkov schedule();
2534d33a39e5SPavel Begunkov else if (!schedule_hrtimeout(&iowq->timeout, HRTIMER_MODE_ABS))
25358a796565SAndres Freund ret = -ETIME;
2536e627c433SJens Axboe current->in_iowait = 0;
25378a796565SAndres Freund return ret;
2538ed29b0b4SJens Axboe }
2539ed29b0b4SJens Axboe
2540ed29b0b4SJens Axboe /*
2541ed29b0b4SJens Axboe * Wait until events become available, if we don't already have some. The
2542ed29b0b4SJens Axboe * application must reap them itself, as they reside on the shared cq ring.
2543ed29b0b4SJens Axboe */
io_cqring_wait(struct io_ring_ctx * ctx,int min_events,const sigset_t __user * sig,size_t sigsz,struct __kernel_timespec __user * uts)2544ed29b0b4SJens Axboe static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
2545ed29b0b4SJens Axboe const sigset_t __user *sig, size_t sigsz,
2546ed29b0b4SJens Axboe struct __kernel_timespec __user *uts)
2547ed29b0b4SJens Axboe {
2548ed29b0b4SJens Axboe struct io_wait_queue iowq;
2549ed29b0b4SJens Axboe struct io_rings *rings = ctx->rings;
2550ed29b0b4SJens Axboe int ret;
2551ed29b0b4SJens Axboe
255276de6749SPavel Begunkov if (!io_allowed_run_tw(ctx))
255376de6749SPavel Begunkov return -EEXIST;
2554140102aeSPavel Begunkov if (!llist_empty(&ctx->work_llist))
25557b8fa7a0SJens Axboe io_run_local_work(ctx, min_events);
2556f36ba6cfSPavel Begunkov io_run_task_work();
2557ed29b0b4SJens Axboe io_cqring_overflow_flush(ctx);
25580fc8c2acSDylan Yudaken /* if user messes with these they will just get an early return */
25590fc8c2acSDylan Yudaken if (__io_cqring_events_user(ctx) >= min_events)
2560ed29b0b4SJens Axboe return 0;
2561ed29b0b4SJens Axboe
2562ed29b0b4SJens Axboe init_waitqueue_func_entry(&iowq.wq, io_wake_function);
2563ed29b0b4SJens Axboe iowq.wq.private = current;
2564ed29b0b4SJens Axboe INIT_LIST_HEAD(&iowq.wq.entry);
2565ed29b0b4SJens Axboe iowq.ctx = ctx;
2566ed29b0b4SJens Axboe iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
2567ed29b0b4SJens Axboe iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
2568d33a39e5SPavel Begunkov iowq.timeout = KTIME_MAX;
2569d33a39e5SPavel Begunkov
2570d33a39e5SPavel Begunkov if (uts) {
2571d33a39e5SPavel Begunkov struct timespec64 ts;
2572d33a39e5SPavel Begunkov
2573d33a39e5SPavel Begunkov if (get_timespec64(&ts, uts))
2574d33a39e5SPavel Begunkov return -EFAULT;
2575d33a39e5SPavel Begunkov iowq.timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
2576d33a39e5SPavel Begunkov }
2577ed29b0b4SJens Axboe
257863fb4af8SAlexey Izbyshev if (sig) {
257963fb4af8SAlexey Izbyshev #ifdef CONFIG_COMPAT
258063fb4af8SAlexey Izbyshev if (in_compat_syscall())
258163fb4af8SAlexey Izbyshev ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
258263fb4af8SAlexey Izbyshev sigsz);
258363fb4af8SAlexey Izbyshev else
258463fb4af8SAlexey Izbyshev #endif
258563fb4af8SAlexey Izbyshev ret = set_user_sigmask(sig, sigsz);
258663fb4af8SAlexey Izbyshev
258763fb4af8SAlexey Izbyshev if (ret)
258863fb4af8SAlexey Izbyshev return ret;
258963fb4af8SAlexey Izbyshev }
259063fb4af8SAlexey Izbyshev
2591ed29b0b4SJens Axboe trace_io_uring_cqring_wait(ctx, min_events);
2592ed29b0b4SJens Axboe do {
25937b8fa7a0SJens Axboe int nr_wait = (int) iowq.cq_tail - READ_ONCE(ctx->rings->cq.tail);
25943fcf19d5SPavel Begunkov unsigned long check_cq;
25953fcf19d5SPavel Begunkov
2596130bd686SPavel Begunkov if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
25978751d154SPavel Begunkov atomic_set(&ctx->cq_wait_nr, nr_wait);
2598130bd686SPavel Begunkov set_current_state(TASK_INTERRUPTIBLE);
2599130bd686SPavel Begunkov } else {
2600ed29b0b4SJens Axboe prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
2601ed29b0b4SJens Axboe TASK_INTERRUPTIBLE);
2602130bd686SPavel Begunkov }
2603130bd686SPavel Begunkov
2604d33a39e5SPavel Begunkov ret = io_cqring_wait_schedule(ctx, &iowq);
2605130bd686SPavel Begunkov __set_current_state(TASK_RUNNING);
26068751d154SPavel Begunkov atomic_set(&ctx->cq_wait_nr, 0);
2607d80c0f00SPavel Begunkov
2608846072f1SPavel Begunkov /*
2609846072f1SPavel Begunkov * Run task_work after scheduling and before io_should_wake().
2610846072f1SPavel Begunkov * If we got woken because of task_work being processed, run it
2611846072f1SPavel Begunkov * now rather than let the caller do another wait loop.
2612846072f1SPavel Begunkov */
2613846072f1SPavel Begunkov io_run_task_work();
2614846072f1SPavel Begunkov if (!llist_empty(&ctx->work_llist))
26157b8fa7a0SJens Axboe io_run_local_work(ctx, nr_wait);
26163fcf19d5SPavel Begunkov
26170241f4c2SJens Axboe /*
26180241f4c2SJens Axboe * Non-local task_work will be run on exit to userspace, but
26190241f4c2SJens Axboe * if we're using DEFER_TASKRUN, then we could have waited
26200241f4c2SJens Axboe * with a timeout for a number of requests. If the timeout
26210241f4c2SJens Axboe * hits, we could have some requests ready to process. Ensure
26220241f4c2SJens Axboe * this break is _after_ we have run task_work, to avoid
26230241f4c2SJens Axboe * deferring running potentially pending requests until the
26240241f4c2SJens Axboe * next time we wait for events.
26250241f4c2SJens Axboe */
26260241f4c2SJens Axboe if (ret < 0)
26270241f4c2SJens Axboe break;
26280241f4c2SJens Axboe
26293fcf19d5SPavel Begunkov check_cq = READ_ONCE(ctx->check_cq);
26303fcf19d5SPavel Begunkov if (unlikely(check_cq)) {
26313fcf19d5SPavel Begunkov /* let the caller flush overflows, retry */
2632326a9e48SPavel Begunkov if (check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
26333fcf19d5SPavel Begunkov io_cqring_do_overflow_flush(ctx);
26343fcf19d5SPavel Begunkov if (check_cq & BIT(IO_CHECK_CQ_DROPPED_BIT)) {
26353fcf19d5SPavel Begunkov ret = -EBADR;
26363fcf19d5SPavel Begunkov break;
26373fcf19d5SPavel Begunkov }
26383fcf19d5SPavel Begunkov }
26393fcf19d5SPavel Begunkov
2640846072f1SPavel Begunkov if (io_should_wake(&iowq)) {
2641846072f1SPavel Begunkov ret = 0;
264235d90f95SJens Axboe break;
2643846072f1SPavel Begunkov }
2644ed29b0b4SJens Axboe cond_resched();
2645846072f1SPavel Begunkov } while (1);
2646ed29b0b4SJens Axboe
2647130bd686SPavel Begunkov if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
2648ed29b0b4SJens Axboe finish_wait(&ctx->cq_wait, &iowq.wq);
2649ed29b0b4SJens Axboe restore_saved_sigmask_unless(ret == -EINTR);
2650ed29b0b4SJens Axboe
2651ed29b0b4SJens Axboe return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2652ed29b0b4SJens Axboe }
2653ed29b0b4SJens Axboe
io_mem_free(void * ptr)26544ca5f54aSJens Axboe void io_mem_free(void *ptr)
2655ed29b0b4SJens Axboe {
2656ed29b0b4SJens Axboe if (!ptr)
2657ed29b0b4SJens Axboe return;
2658ed29b0b4SJens Axboe
265999a9e0b8SMatthew Wilcox (Oracle) folio_put(virt_to_folio(ptr));
2660ed29b0b4SJens Axboe }
2661ed29b0b4SJens Axboe
io_pages_free(struct page *** pages,int npages)266203d89a2dSJens Axboe static void io_pages_free(struct page ***pages, int npages)
266303d89a2dSJens Axboe {
266403d89a2dSJens Axboe struct page **page_array;
266503d89a2dSJens Axboe int i;
266603d89a2dSJens Axboe
266703d89a2dSJens Axboe if (!pages)
266803d89a2dSJens Axboe return;
26698b51a395SJens Axboe
267003d89a2dSJens Axboe page_array = *pages;
26718b51a395SJens Axboe if (!page_array)
26728b51a395SJens Axboe return;
26738b51a395SJens Axboe
267403d89a2dSJens Axboe for (i = 0; i < npages; i++)
267503d89a2dSJens Axboe unpin_user_page(page_array[i]);
267603d89a2dSJens Axboe kvfree(page_array);
267703d89a2dSJens Axboe *pages = NULL;
267803d89a2dSJens Axboe }
267903d89a2dSJens Axboe
__io_uaddr_map(struct page *** pages,unsigned short * npages,unsigned long uaddr,size_t size)268003d89a2dSJens Axboe static void *__io_uaddr_map(struct page ***pages, unsigned short *npages,
268103d89a2dSJens Axboe unsigned long uaddr, size_t size)
268203d89a2dSJens Axboe {
268303d89a2dSJens Axboe struct page **page_array;
268403d89a2dSJens Axboe unsigned int nr_pages;
26854be625baSJens Axboe void *page_addr;
26860b6f39c1SGabriel Krisman Bertazi int ret, i, pinned;
268703d89a2dSJens Axboe
268803d89a2dSJens Axboe *npages = 0;
268903d89a2dSJens Axboe
269003d89a2dSJens Axboe if (uaddr & (PAGE_SIZE - 1) || !size)
269103d89a2dSJens Axboe return ERR_PTR(-EINVAL);
269203d89a2dSJens Axboe
269303d89a2dSJens Axboe nr_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
269403d89a2dSJens Axboe if (nr_pages > USHRT_MAX)
269503d89a2dSJens Axboe return ERR_PTR(-EINVAL);
269603d89a2dSJens Axboe page_array = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL);
269703d89a2dSJens Axboe if (!page_array)
269803d89a2dSJens Axboe return ERR_PTR(-ENOMEM);
269903d89a2dSJens Axboe
27000b6f39c1SGabriel Krisman Bertazi
27010b6f39c1SGabriel Krisman Bertazi pinned = pin_user_pages_fast(uaddr, nr_pages, FOLL_WRITE | FOLL_LONGTERM,
270203d89a2dSJens Axboe page_array);
27030b6f39c1SGabriel Krisman Bertazi if (pinned != nr_pages) {
27040b6f39c1SGabriel Krisman Bertazi ret = (pinned < 0) ? pinned : -EFAULT;
27050b6f39c1SGabriel Krisman Bertazi goto free_pages;
270603d89a2dSJens Axboe }
27074be625baSJens Axboe
27084be625baSJens Axboe page_addr = page_address(page_array[0]);
27094be625baSJens Axboe for (i = 0; i < nr_pages; i++) {
27104be625baSJens Axboe ret = -EINVAL;
27114be625baSJens Axboe
271203d89a2dSJens Axboe /*
27134be625baSJens Axboe * Can't support mapping user allocated ring memory on 32-bit
27144be625baSJens Axboe * archs where it could potentially reside in highmem. Just
27154be625baSJens Axboe * fail those with -EINVAL, just like we did on kernels that
27164be625baSJens Axboe * didn't support this feature.
271703d89a2dSJens Axboe */
27184be625baSJens Axboe if (PageHighMem(page_array[i]))
27190b6f39c1SGabriel Krisman Bertazi goto free_pages;
2720223ef474SJens Axboe
2721223ef474SJens Axboe /*
27224be625baSJens Axboe * No support for discontig pages for now, should either be a
27234be625baSJens Axboe * single normal page, or a huge page. Later on we can add
27244be625baSJens Axboe * support for remapping discontig pages, for now we will
27254be625baSJens Axboe * just fail them with EINVAL.
2726223ef474SJens Axboe */
27274be625baSJens Axboe if (page_address(page_array[i]) != page_addr)
27280b6f39c1SGabriel Krisman Bertazi goto free_pages;
27294be625baSJens Axboe page_addr += PAGE_SIZE;
2730223ef474SJens Axboe }
2731223ef474SJens Axboe
273203d89a2dSJens Axboe *pages = page_array;
273303d89a2dSJens Axboe *npages = nr_pages;
273403d89a2dSJens Axboe return page_to_virt(page_array[0]);
27350b6f39c1SGabriel Krisman Bertazi
27360b6f39c1SGabriel Krisman Bertazi free_pages:
27370b6f39c1SGabriel Krisman Bertazi io_pages_free(&page_array, pinned > 0 ? pinned : 0);
27380b6f39c1SGabriel Krisman Bertazi return ERR_PTR(ret);
273903d89a2dSJens Axboe }
274003d89a2dSJens Axboe
io_rings_map(struct io_ring_ctx * ctx,unsigned long uaddr,size_t size)274103d89a2dSJens Axboe static void *io_rings_map(struct io_ring_ctx *ctx, unsigned long uaddr,
274203d89a2dSJens Axboe size_t size)
274303d89a2dSJens Axboe {
274403d89a2dSJens Axboe return __io_uaddr_map(&ctx->ring_pages, &ctx->n_ring_pages, uaddr,
274503d89a2dSJens Axboe size);
274603d89a2dSJens Axboe }
274703d89a2dSJens Axboe
io_sqes_map(struct io_ring_ctx * ctx,unsigned long uaddr,size_t size)274803d89a2dSJens Axboe static void *io_sqes_map(struct io_ring_ctx *ctx, unsigned long uaddr,
274903d89a2dSJens Axboe size_t size)
275003d89a2dSJens Axboe {
275103d89a2dSJens Axboe return __io_uaddr_map(&ctx->sqe_pages, &ctx->n_sqe_pages, uaddr,
275203d89a2dSJens Axboe size);
275303d89a2dSJens Axboe }
275403d89a2dSJens Axboe
io_rings_free(struct io_ring_ctx * ctx)27559c189eeeSJens Axboe static void io_rings_free(struct io_ring_ctx *ctx)
27569c189eeeSJens Axboe {
275703d89a2dSJens Axboe if (!(ctx->flags & IORING_SETUP_NO_MMAP)) {
27589c189eeeSJens Axboe io_mem_free(ctx->rings);
27599c189eeeSJens Axboe io_mem_free(ctx->sq_sqes);
276003d89a2dSJens Axboe } else {
276103d89a2dSJens Axboe io_pages_free(&ctx->ring_pages, ctx->n_ring_pages);
27628b51a395SJens Axboe ctx->n_ring_pages = 0;
276303d89a2dSJens Axboe io_pages_free(&ctx->sqe_pages, ctx->n_sqe_pages);
27648b51a395SJens Axboe ctx->n_sqe_pages = 0;
276503d89a2dSJens Axboe }
27661241052eSPavel Begunkov
27671241052eSPavel Begunkov ctx->rings = NULL;
27681241052eSPavel Begunkov ctx->sq_sqes = NULL;
27699c189eeeSJens Axboe }
27709c189eeeSJens Axboe
io_mem_alloc(size_t size)27714ca5f54aSJens Axboe void *io_mem_alloc(size_t size)
2772ed29b0b4SJens Axboe {
2773ed29b0b4SJens Axboe gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
2774e27cef86SJens Axboe void *ret;
2775ed29b0b4SJens Axboe
2776e27cef86SJens Axboe ret = (void *) __get_free_pages(gfp, get_order(size));
2777e27cef86SJens Axboe if (ret)
2778e27cef86SJens Axboe return ret;
2779e27cef86SJens Axboe return ERR_PTR(-ENOMEM);
2780ed29b0b4SJens Axboe }
2781ed29b0b4SJens Axboe
rings_size(struct io_ring_ctx * ctx,unsigned int sq_entries,unsigned int cq_entries,size_t * sq_offset)2782ed29b0b4SJens Axboe static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries,
2783ed29b0b4SJens Axboe unsigned int cq_entries, size_t *sq_offset)
2784ed29b0b4SJens Axboe {
2785ed29b0b4SJens Axboe struct io_rings *rings;
2786ed29b0b4SJens Axboe size_t off, sq_array_size;
2787ed29b0b4SJens Axboe
2788ed29b0b4SJens Axboe off = struct_size(rings, cqes, cq_entries);
2789ed29b0b4SJens Axboe if (off == SIZE_MAX)
2790ed29b0b4SJens Axboe return SIZE_MAX;
2791ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_CQE32) {
2792ed29b0b4SJens Axboe if (check_shl_overflow(off, 1, &off))
2793ed29b0b4SJens Axboe return SIZE_MAX;
2794ed29b0b4SJens Axboe }
2795ed29b0b4SJens Axboe
2796ed29b0b4SJens Axboe #ifdef CONFIG_SMP
2797ed29b0b4SJens Axboe off = ALIGN(off, SMP_CACHE_BYTES);
2798ed29b0b4SJens Axboe if (off == 0)
2799ed29b0b4SJens Axboe return SIZE_MAX;
2800ed29b0b4SJens Axboe #endif
2801ed29b0b4SJens Axboe
28022af89abdSPavel Begunkov if (ctx->flags & IORING_SETUP_NO_SQARRAY) {
28032af89abdSPavel Begunkov if (sq_offset)
28042af89abdSPavel Begunkov *sq_offset = SIZE_MAX;
28052af89abdSPavel Begunkov return off;
28062af89abdSPavel Begunkov }
28072af89abdSPavel Begunkov
2808ed29b0b4SJens Axboe if (sq_offset)
2809ed29b0b4SJens Axboe *sq_offset = off;
2810ed29b0b4SJens Axboe
2811ed29b0b4SJens Axboe sq_array_size = array_size(sizeof(u32), sq_entries);
2812ed29b0b4SJens Axboe if (sq_array_size == SIZE_MAX)
2813ed29b0b4SJens Axboe return SIZE_MAX;
2814ed29b0b4SJens Axboe
2815ed29b0b4SJens Axboe if (check_add_overflow(off, sq_array_size, &off))
2816ed29b0b4SJens Axboe return SIZE_MAX;
2817ed29b0b4SJens Axboe
2818ed29b0b4SJens Axboe return off;
2819ed29b0b4SJens Axboe }
2820ed29b0b4SJens Axboe
io_eventfd_register(struct io_ring_ctx * ctx,void __user * arg,unsigned int eventfd_async)2821ed29b0b4SJens Axboe static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg,
2822ed29b0b4SJens Axboe unsigned int eventfd_async)
2823ed29b0b4SJens Axboe {
2824ed29b0b4SJens Axboe struct io_ev_fd *ev_fd;
2825ed29b0b4SJens Axboe __s32 __user *fds = arg;
2826ed29b0b4SJens Axboe int fd;
2827ed29b0b4SJens Axboe
2828ed29b0b4SJens Axboe ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
2829ed29b0b4SJens Axboe lockdep_is_held(&ctx->uring_lock));
2830ed29b0b4SJens Axboe if (ev_fd)
2831ed29b0b4SJens Axboe return -EBUSY;
2832ed29b0b4SJens Axboe
2833ed29b0b4SJens Axboe if (copy_from_user(&fd, fds, sizeof(*fds)))
2834ed29b0b4SJens Axboe return -EFAULT;
2835ed29b0b4SJens Axboe
2836ed29b0b4SJens Axboe ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL);
2837ed29b0b4SJens Axboe if (!ev_fd)
2838ed29b0b4SJens Axboe return -ENOMEM;
2839ed29b0b4SJens Axboe
2840ed29b0b4SJens Axboe ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd);
2841ed29b0b4SJens Axboe if (IS_ERR(ev_fd->cq_ev_fd)) {
2842ed29b0b4SJens Axboe int ret = PTR_ERR(ev_fd->cq_ev_fd);
2843ed29b0b4SJens Axboe kfree(ev_fd);
2844ed29b0b4SJens Axboe return ret;
2845ed29b0b4SJens Axboe }
2846305bef98SPavel Begunkov
2847305bef98SPavel Begunkov spin_lock(&ctx->completion_lock);
2848305bef98SPavel Begunkov ctx->evfd_last_cq_tail = ctx->cached_cq_tail;
2849305bef98SPavel Begunkov spin_unlock(&ctx->completion_lock);
2850305bef98SPavel Begunkov
2851ed29b0b4SJens Axboe ev_fd->eventfd_async = eventfd_async;
2852ed29b0b4SJens Axboe ctx->has_evfd = true;
2853ed29b0b4SJens Axboe rcu_assign_pointer(ctx->io_ev_fd, ev_fd);
285421a091b9SDylan Yudaken atomic_set(&ev_fd->refs, 1);
285521a091b9SDylan Yudaken atomic_set(&ev_fd->ops, 0);
2856ed29b0b4SJens Axboe return 0;
2857ed29b0b4SJens Axboe }
2858ed29b0b4SJens Axboe
io_eventfd_unregister(struct io_ring_ctx * ctx)2859ed29b0b4SJens Axboe static int io_eventfd_unregister(struct io_ring_ctx *ctx)
2860ed29b0b4SJens Axboe {
2861ed29b0b4SJens Axboe struct io_ev_fd *ev_fd;
2862ed29b0b4SJens Axboe
2863ed29b0b4SJens Axboe ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
2864ed29b0b4SJens Axboe lockdep_is_held(&ctx->uring_lock));
2865ed29b0b4SJens Axboe if (ev_fd) {
2866ed29b0b4SJens Axboe ctx->has_evfd = false;
2867ed29b0b4SJens Axboe rcu_assign_pointer(ctx->io_ev_fd, NULL);
286821a091b9SDylan Yudaken if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops))
286921a091b9SDylan Yudaken call_rcu(&ev_fd->rcu, io_eventfd_ops);
2870ed29b0b4SJens Axboe return 0;
2871ed29b0b4SJens Axboe }
2872ed29b0b4SJens Axboe
2873ed29b0b4SJens Axboe return -ENXIO;
2874ed29b0b4SJens Axboe }
2875ed29b0b4SJens Axboe
io_req_caches_free(struct io_ring_ctx * ctx)2876ed29b0b4SJens Axboe static void io_req_caches_free(struct io_ring_ctx *ctx)
2877ed29b0b4SJens Axboe {
2878c8576f3eSPavel Begunkov struct io_kiocb *req;
2879ed29b0b4SJens Axboe int nr = 0;
2880ed29b0b4SJens Axboe
2881ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
288234f0bc42SPavel Begunkov io_flush_cached_locked_reqs(ctx, &ctx->submit_state);
2883ed29b0b4SJens Axboe
2884ed29b0b4SJens Axboe while (!io_req_cache_empty(ctx)) {
2885c8576f3eSPavel Begunkov req = io_extract_req(ctx);
2886ed29b0b4SJens Axboe kmem_cache_free(req_cachep, req);
2887ed29b0b4SJens Axboe nr++;
2888ed29b0b4SJens Axboe }
2889ed29b0b4SJens Axboe if (nr)
2890ed29b0b4SJens Axboe percpu_ref_put_many(&ctx->refs, nr);
2891ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
2892ed29b0b4SJens Axboe }
2893ed29b0b4SJens Axboe
io_rsrc_node_cache_free(struct io_cache_entry * entry)28949eae8655SPavel Begunkov static void io_rsrc_node_cache_free(struct io_cache_entry *entry)
28959eae8655SPavel Begunkov {
28969eae8655SPavel Begunkov kfree(container_of(entry, struct io_rsrc_node, cache));
28979eae8655SPavel Begunkov }
28989eae8655SPavel Begunkov
io_ring_ctx_free(struct io_ring_ctx * ctx)2899ed29b0b4SJens Axboe static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
2900ed29b0b4SJens Axboe {
2901ed29b0b4SJens Axboe io_sq_thread_finish(ctx);
2902ed29b0b4SJens Axboe /* __io_rsrc_put_work() may need uring_lock to progress, wait w/o it */
29030b222eebSPavel Begunkov if (WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list)))
29040b222eebSPavel Begunkov return;
2905ed29b0b4SJens Axboe
2906ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
2907ed29b0b4SJens Axboe if (ctx->buf_data)
2908ed29b0b4SJens Axboe __io_sqe_buffers_unregister(ctx);
2909ed29b0b4SJens Axboe if (ctx->file_data)
2910ed29b0b4SJens Axboe __io_sqe_files_unregister(ctx);
2911a85381d8SPavel Begunkov io_cqring_overflow_kill(ctx);
2912ed29b0b4SJens Axboe io_eventfd_unregister(ctx);
29139b797a37SJens Axboe io_alloc_cache_free(&ctx->apoll_cache, io_apoll_cache_free);
291443e0bbbdSJens Axboe io_alloc_cache_free(&ctx->netmsg_cache, io_netmsg_cache_free);
2915ed29b0b4SJens Axboe io_destroy_buffers(ctx);
2916b4a72c05SWojciech Lukowicz mutex_unlock(&ctx->uring_lock);
2917ed29b0b4SJens Axboe if (ctx->sq_creds)
2918ed29b0b4SJens Axboe put_cred(ctx->sq_creds);
291997bbdc06SPavel Begunkov if (ctx->submitter_task)
292097bbdc06SPavel Begunkov put_task_struct(ctx->submitter_task);
2921ed29b0b4SJens Axboe
2922ed29b0b4SJens Axboe /* there are no registered resources left, nobody uses it */
2923ed29b0b4SJens Axboe if (ctx->rsrc_node)
29249eae8655SPavel Begunkov io_rsrc_node_destroy(ctx, ctx->rsrc_node);
2925ed29b0b4SJens Axboe
2926ed29b0b4SJens Axboe WARN_ON_ONCE(!list_empty(&ctx->rsrc_ref_list));
2927ed29b0b4SJens Axboe WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
2928ed29b0b4SJens Axboe
29299eae8655SPavel Begunkov io_alloc_cache_free(&ctx->rsrc_node_cache, io_rsrc_node_cache_free);
293042b6419dSPavel Begunkov if (ctx->mm_account) {
293142b6419dSPavel Begunkov mmdrop(ctx->mm_account);
293242b6419dSPavel Begunkov ctx->mm_account = NULL;
293342b6419dSPavel Begunkov }
29349c189eeeSJens Axboe io_rings_free(ctx);
29357138ebbeSJens Axboe io_kbuf_mmap_list_free(ctx);
2936ed29b0b4SJens Axboe
2937ed29b0b4SJens Axboe percpu_ref_exit(&ctx->refs);
2938ed29b0b4SJens Axboe free_uid(ctx->user);
2939ed29b0b4SJens Axboe io_req_caches_free(ctx);
2940ed29b0b4SJens Axboe if (ctx->hash_map)
2941ed29b0b4SJens Axboe io_wq_put_hash(ctx->hash_map);
2942e6f89be6SPavel Begunkov kfree(ctx->cancel_table.hbs);
29439ca9fb24SPavel Begunkov kfree(ctx->cancel_table_locked.hbs);
2944ed29b0b4SJens Axboe xa_destroy(&ctx->io_bl_xa);
2945ed29b0b4SJens Axboe kfree(ctx);
2946ed29b0b4SJens Axboe }
2947ed29b0b4SJens Axboe
io_activate_pollwq_cb(struct callback_head * cb)2948bca39f39SPavel Begunkov static __cold void io_activate_pollwq_cb(struct callback_head *cb)
2949bca39f39SPavel Begunkov {
2950bca39f39SPavel Begunkov struct io_ring_ctx *ctx = container_of(cb, struct io_ring_ctx,
2951bca39f39SPavel Begunkov poll_wq_task_work);
2952bca39f39SPavel Begunkov
2953bca39f39SPavel Begunkov mutex_lock(&ctx->uring_lock);
2954bca39f39SPavel Begunkov ctx->poll_activated = true;
2955bca39f39SPavel Begunkov mutex_unlock(&ctx->uring_lock);
2956bca39f39SPavel Begunkov
2957bca39f39SPavel Begunkov /*
2958bca39f39SPavel Begunkov * Wake ups for some events between start of polling and activation
2959bca39f39SPavel Begunkov * might've been lost due to loose synchronisation.
2960bca39f39SPavel Begunkov */
2961bca39f39SPavel Begunkov wake_up_all(&ctx->poll_wq);
2962bca39f39SPavel Begunkov percpu_ref_put(&ctx->refs);
2963bca39f39SPavel Begunkov }
2964bca39f39SPavel Begunkov
io_activate_pollwq(struct io_ring_ctx * ctx)2965bca39f39SPavel Begunkov static __cold void io_activate_pollwq(struct io_ring_ctx *ctx)
2966bca39f39SPavel Begunkov {
2967bca39f39SPavel Begunkov spin_lock(&ctx->completion_lock);
2968bca39f39SPavel Begunkov /* already activated or in progress */
2969bca39f39SPavel Begunkov if (ctx->poll_activated || ctx->poll_wq_task_work.func)
2970bca39f39SPavel Begunkov goto out;
2971bca39f39SPavel Begunkov if (WARN_ON_ONCE(!ctx->task_complete))
2972bca39f39SPavel Begunkov goto out;
2973bca39f39SPavel Begunkov if (!ctx->submitter_task)
2974bca39f39SPavel Begunkov goto out;
2975bca39f39SPavel Begunkov /*
2976bca39f39SPavel Begunkov * with ->submitter_task only the submitter task completes requests, we
2977bca39f39SPavel Begunkov * only need to sync with it, which is done by injecting a tw
2978bca39f39SPavel Begunkov */
2979bca39f39SPavel Begunkov init_task_work(&ctx->poll_wq_task_work, io_activate_pollwq_cb);
2980bca39f39SPavel Begunkov percpu_ref_get(&ctx->refs);
2981bca39f39SPavel Begunkov if (task_work_add(ctx->submitter_task, &ctx->poll_wq_task_work, TWA_SIGNAL))
2982bca39f39SPavel Begunkov percpu_ref_put(&ctx->refs);
2983bca39f39SPavel Begunkov out:
2984bca39f39SPavel Begunkov spin_unlock(&ctx->completion_lock);
2985bca39f39SPavel Begunkov }
2986bca39f39SPavel Begunkov
io_uring_poll(struct file * file,poll_table * wait)2987ed29b0b4SJens Axboe static __poll_t io_uring_poll(struct file *file, poll_table *wait)
2988ed29b0b4SJens Axboe {
2989ed29b0b4SJens Axboe struct io_ring_ctx *ctx = file->private_data;
2990ed29b0b4SJens Axboe __poll_t mask = 0;
2991ed29b0b4SJens Axboe
2992bca39f39SPavel Begunkov if (unlikely(!ctx->poll_activated))
2993bca39f39SPavel Begunkov io_activate_pollwq(ctx);
2994bca39f39SPavel Begunkov
29957b235dd8SPavel Begunkov poll_wait(file, &ctx->poll_wq, wait);
2996ed29b0b4SJens Axboe /*
2997ed29b0b4SJens Axboe * synchronizes with barrier from wq_has_sleeper call in
2998ed29b0b4SJens Axboe * io_commit_cqring
2999ed29b0b4SJens Axboe */
3000ed29b0b4SJens Axboe smp_rmb();
3001ed29b0b4SJens Axboe if (!io_sqring_full(ctx))
3002ed29b0b4SJens Axboe mask |= EPOLLOUT | EPOLLWRNORM;
3003ed29b0b4SJens Axboe
3004ed29b0b4SJens Axboe /*
3005ed29b0b4SJens Axboe * Don't flush cqring overflow list here, just do a simple check.
3006ed29b0b4SJens Axboe * Otherwise there could possible be ABBA deadlock:
3007ed29b0b4SJens Axboe * CPU0 CPU1
3008ed29b0b4SJens Axboe * ---- ----
3009ed29b0b4SJens Axboe * lock(&ctx->uring_lock);
3010ed29b0b4SJens Axboe * lock(&ep->mtx);
3011ed29b0b4SJens Axboe * lock(&ctx->uring_lock);
3012ed29b0b4SJens Axboe * lock(&ep->mtx);
3013ed29b0b4SJens Axboe *
3014ed29b0b4SJens Axboe * Users may get EPOLLIN meanwhile seeing nothing in cqring, this
301510d8bc35SDylan Yudaken * pushes them to do the flush.
3016ed29b0b4SJens Axboe */
3017b4c98d59SDylan Yudaken
3018c10bb646SPavel Begunkov if (__io_cqring_events_user(ctx) || io_has_work(ctx))
3019ed29b0b4SJens Axboe mask |= EPOLLIN | EPOLLRDNORM;
3020ed29b0b4SJens Axboe
3021ed29b0b4SJens Axboe return mask;
3022ed29b0b4SJens Axboe }
3023ed29b0b4SJens Axboe
io_unregister_personality(struct io_ring_ctx * ctx,unsigned id)3024ed29b0b4SJens Axboe static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
3025ed29b0b4SJens Axboe {
3026ed29b0b4SJens Axboe const struct cred *creds;
3027ed29b0b4SJens Axboe
3028ed29b0b4SJens Axboe creds = xa_erase(&ctx->personalities, id);
3029ed29b0b4SJens Axboe if (creds) {
3030ed29b0b4SJens Axboe put_cred(creds);
3031ed29b0b4SJens Axboe return 0;
3032ed29b0b4SJens Axboe }
3033ed29b0b4SJens Axboe
3034ed29b0b4SJens Axboe return -EINVAL;
3035ed29b0b4SJens Axboe }
3036ed29b0b4SJens Axboe
3037ed29b0b4SJens Axboe struct io_tctx_exit {
3038ed29b0b4SJens Axboe struct callback_head task_work;
3039ed29b0b4SJens Axboe struct completion completion;
3040ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
3041ed29b0b4SJens Axboe };
3042ed29b0b4SJens Axboe
io_tctx_exit_cb(struct callback_head * cb)3043ed29b0b4SJens Axboe static __cold void io_tctx_exit_cb(struct callback_head *cb)
3044ed29b0b4SJens Axboe {
3045ed29b0b4SJens Axboe struct io_uring_task *tctx = current->io_uring;
3046ed29b0b4SJens Axboe struct io_tctx_exit *work;
3047ed29b0b4SJens Axboe
3048ed29b0b4SJens Axboe work = container_of(cb, struct io_tctx_exit, task_work);
3049ed29b0b4SJens Axboe /*
30508d664282SJens Axboe * When @in_cancel, we're in cancellation and it's racy to remove the
3051ed29b0b4SJens Axboe * node. It'll be removed by the end of cancellation, just ignore it.
3052998b30c3SHarshit Mogalapalli * tctx can be NULL if the queueing of this task_work raced with
3053998b30c3SHarshit Mogalapalli * work cancelation off the exec path.
3054ed29b0b4SJens Axboe */
30558d664282SJens Axboe if (tctx && !atomic_read(&tctx->in_cancel))
3056ed29b0b4SJens Axboe io_uring_del_tctx_node((unsigned long)work->ctx);
3057ed29b0b4SJens Axboe complete(&work->completion);
3058ed29b0b4SJens Axboe }
3059ed29b0b4SJens Axboe
io_cancel_ctx_cb(struct io_wq_work * work,void * data)3060ed29b0b4SJens Axboe static __cold bool io_cancel_ctx_cb(struct io_wq_work *work, void *data)
3061ed29b0b4SJens Axboe {
3062ed29b0b4SJens Axboe struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3063ed29b0b4SJens Axboe
3064ed29b0b4SJens Axboe return req->ctx == data;
3065ed29b0b4SJens Axboe }
3066ed29b0b4SJens Axboe
io_ring_exit_work(struct work_struct * work)3067ed29b0b4SJens Axboe static __cold void io_ring_exit_work(struct work_struct *work)
3068ed29b0b4SJens Axboe {
3069ed29b0b4SJens Axboe struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
3070ed29b0b4SJens Axboe unsigned long timeout = jiffies + HZ * 60 * 5;
3071ed29b0b4SJens Axboe unsigned long interval = HZ / 20;
3072ed29b0b4SJens Axboe struct io_tctx_exit exit;
3073ed29b0b4SJens Axboe struct io_tctx_node *node;
3074ed29b0b4SJens Axboe int ret;
3075ed29b0b4SJens Axboe
3076ed29b0b4SJens Axboe /*
3077ed29b0b4SJens Axboe * If we're doing polled IO and end up having requests being
3078ed29b0b4SJens Axboe * submitted async (out-of-line), then completions can come in while
3079ed29b0b4SJens Axboe * we're waiting for refs to drop. We need to reap these manually,
3080ed29b0b4SJens Axboe * as nobody else will be looking for them.
3081ed29b0b4SJens Axboe */
3082ed29b0b4SJens Axboe do {
3083a85381d8SPavel Begunkov if (test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq)) {
3084a85381d8SPavel Begunkov mutex_lock(&ctx->uring_lock);
3085a85381d8SPavel Begunkov io_cqring_overflow_kill(ctx);
3086a85381d8SPavel Begunkov mutex_unlock(&ctx->uring_lock);
3087a85381d8SPavel Begunkov }
3088a85381d8SPavel Begunkov
3089c0e0d6baSDylan Yudaken if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
3090c0e0d6baSDylan Yudaken io_move_task_work_from_local(ctx);
3091c0e0d6baSDylan Yudaken
3092affa87dbSPavel Begunkov while (io_uring_try_cancel_requests(ctx, NULL, true))
3093affa87dbSPavel Begunkov cond_resched();
3094affa87dbSPavel Begunkov
3095ed29b0b4SJens Axboe if (ctx->sq_data) {
3096ed29b0b4SJens Axboe struct io_sq_data *sqd = ctx->sq_data;
3097ed29b0b4SJens Axboe struct task_struct *tsk;
3098ed29b0b4SJens Axboe
3099ed29b0b4SJens Axboe io_sq_thread_park(sqd);
3100ed29b0b4SJens Axboe tsk = sqd->thread;
3101ed29b0b4SJens Axboe if (tsk && tsk->io_uring && tsk->io_uring->io_wq)
3102ed29b0b4SJens Axboe io_wq_cancel_cb(tsk->io_uring->io_wq,
3103ed29b0b4SJens Axboe io_cancel_ctx_cb, ctx, true);
3104ed29b0b4SJens Axboe io_sq_thread_unpark(sqd);
3105ed29b0b4SJens Axboe }
3106ed29b0b4SJens Axboe
3107ed29b0b4SJens Axboe io_req_caches_free(ctx);
3108ed29b0b4SJens Axboe
3109ed29b0b4SJens Axboe if (WARN_ON_ONCE(time_after(jiffies, timeout))) {
3110ed29b0b4SJens Axboe /* there is little hope left, don't run it too often */
3111ed29b0b4SJens Axboe interval = HZ * 60;
3112ed29b0b4SJens Axboe }
31134826c594SJens Axboe /*
31144826c594SJens Axboe * This is really an uninterruptible wait, as it has to be
31154826c594SJens Axboe * complete. But it's also run from a kworker, which doesn't
31164826c594SJens Axboe * take signals, so it's fine to make it interruptible. This
31174826c594SJens Axboe * avoids scenarios where we knowingly can wait much longer
31184826c594SJens Axboe * on completions, for example if someone does a SIGSTOP on
31194826c594SJens Axboe * a task that needs to finish task_work to make this loop
31204826c594SJens Axboe * complete. That's a synthetic situation that should not
31214826c594SJens Axboe * cause a stuck task backtrace, and hence a potential panic
31224826c594SJens Axboe * on stuck tasks if that is enabled.
31234826c594SJens Axboe */
31244826c594SJens Axboe } while (!wait_for_completion_interruptible_timeout(&ctx->ref_comp, interval));
3125ed29b0b4SJens Axboe
3126ed29b0b4SJens Axboe init_completion(&exit.completion);
3127ed29b0b4SJens Axboe init_task_work(&exit.task_work, io_tctx_exit_cb);
3128ed29b0b4SJens Axboe exit.ctx = ctx;
312930df2901SPavel Begunkov
3130ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3131ed29b0b4SJens Axboe while (!list_empty(&ctx->tctx_list)) {
3132ed29b0b4SJens Axboe WARN_ON_ONCE(time_after(jiffies, timeout));
3133ed29b0b4SJens Axboe
3134ed29b0b4SJens Axboe node = list_first_entry(&ctx->tctx_list, struct io_tctx_node,
3135ed29b0b4SJens Axboe ctx_node);
3136ed29b0b4SJens Axboe /* don't spin on a single task if cancellation failed */
3137ed29b0b4SJens Axboe list_rotate_left(&ctx->tctx_list);
3138ed29b0b4SJens Axboe ret = task_work_add(node->task, &exit.task_work, TWA_SIGNAL);
3139ed29b0b4SJens Axboe if (WARN_ON_ONCE(ret))
3140ed29b0b4SJens Axboe continue;
3141ed29b0b4SJens Axboe
3142ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
31434826c594SJens Axboe /*
31444826c594SJens Axboe * See comment above for
31454826c594SJens Axboe * wait_for_completion_interruptible_timeout() on why this
31464826c594SJens Axboe * wait is marked as interruptible.
31474826c594SJens Axboe */
31484826c594SJens Axboe wait_for_completion_interruptible(&exit.completion);
3149ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3150ed29b0b4SJens Axboe }
3151ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3152ed29b0b4SJens Axboe spin_lock(&ctx->completion_lock);
3153ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
3154ed29b0b4SJens Axboe
3155d73a572dSPavel Begunkov /* pairs with RCU read section in io_req_local_work_add() */
3156d73a572dSPavel Begunkov if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
3157d73a572dSPavel Begunkov synchronize_rcu();
3158d73a572dSPavel Begunkov
3159ed29b0b4SJens Axboe io_ring_ctx_free(ctx);
3160ed29b0b4SJens Axboe }
3161ed29b0b4SJens Axboe
io_ring_ctx_wait_and_kill(struct io_ring_ctx * ctx)3162ed29b0b4SJens Axboe static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
3163ed29b0b4SJens Axboe {
3164ed29b0b4SJens Axboe unsigned long index;
3165ed29b0b4SJens Axboe struct creds *creds;
3166ed29b0b4SJens Axboe
3167ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3168ed29b0b4SJens Axboe percpu_ref_kill(&ctx->refs);
3169ed29b0b4SJens Axboe xa_for_each(&ctx->personalities, index, creds)
3170ed29b0b4SJens Axboe io_unregister_personality(ctx, index);
31719ca9fb24SPavel Begunkov if (ctx->rings)
31729ca9fb24SPavel Begunkov io_poll_remove_all(ctx, NULL, true);
3173ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3174ed29b0b4SJens Axboe
317502bac94bSPavel Begunkov /*
317602bac94bSPavel Begunkov * If we failed setting up the ctx, we might not have any rings
317702bac94bSPavel Begunkov * and therefore did not submit any requests
317802bac94bSPavel Begunkov */
317902bac94bSPavel Begunkov if (ctx->rings)
3180ed29b0b4SJens Axboe io_kill_timeouts(ctx, NULL, true);
3181ed29b0b4SJens Axboe
3182dfbe5561SJens Axboe flush_delayed_work(&ctx->fallback_work);
3183dfbe5561SJens Axboe
3184ed29b0b4SJens Axboe INIT_WORK(&ctx->exit_work, io_ring_exit_work);
3185ed29b0b4SJens Axboe /*
3186ed29b0b4SJens Axboe * Use system_unbound_wq to avoid spawning tons of event kworkers
3187ed29b0b4SJens Axboe * if we're exiting a ton of rings at the same time. It just adds
3188ed29b0b4SJens Axboe * noise and overhead, there's no discernable change in runtime
3189ed29b0b4SJens Axboe * over using system_wq.
3190ed29b0b4SJens Axboe */
31916b9d49bcSJens Axboe queue_work(iou_wq, &ctx->exit_work);
3192ed29b0b4SJens Axboe }
3193ed29b0b4SJens Axboe
io_uring_release(struct inode * inode,struct file * file)3194ed29b0b4SJens Axboe static int io_uring_release(struct inode *inode, struct file *file)
3195ed29b0b4SJens Axboe {
3196ed29b0b4SJens Axboe struct io_ring_ctx *ctx = file->private_data;
3197ed29b0b4SJens Axboe
3198ed29b0b4SJens Axboe file->private_data = NULL;
3199ed29b0b4SJens Axboe io_ring_ctx_wait_and_kill(ctx);
3200ed29b0b4SJens Axboe return 0;
3201ed29b0b4SJens Axboe }
3202ed29b0b4SJens Axboe
3203ed29b0b4SJens Axboe struct io_task_cancel {
3204ed29b0b4SJens Axboe struct task_struct *task;
3205ed29b0b4SJens Axboe bool all;
3206ed29b0b4SJens Axboe };
3207ed29b0b4SJens Axboe
io_cancel_task_cb(struct io_wq_work * work,void * data)3208ed29b0b4SJens Axboe static bool io_cancel_task_cb(struct io_wq_work *work, void *data)
3209ed29b0b4SJens Axboe {
3210ed29b0b4SJens Axboe struct io_kiocb *req = container_of(work, struct io_kiocb, work);
3211ed29b0b4SJens Axboe struct io_task_cancel *cancel = data;
3212ed29b0b4SJens Axboe
3213ed29b0b4SJens Axboe return io_match_task_safe(req, cancel->task, cancel->all);
3214ed29b0b4SJens Axboe }
3215ed29b0b4SJens Axboe
io_cancel_defer_files(struct io_ring_ctx * ctx,struct task_struct * task,bool cancel_all)3216ed29b0b4SJens Axboe static __cold bool io_cancel_defer_files(struct io_ring_ctx *ctx,
3217ed29b0b4SJens Axboe struct task_struct *task,
3218ed29b0b4SJens Axboe bool cancel_all)
3219ed29b0b4SJens Axboe {
3220ed29b0b4SJens Axboe struct io_defer_entry *de;
3221ed29b0b4SJens Axboe LIST_HEAD(list);
3222ed29b0b4SJens Axboe
3223ed29b0b4SJens Axboe spin_lock(&ctx->completion_lock);
3224ed29b0b4SJens Axboe list_for_each_entry_reverse(de, &ctx->defer_list, list) {
3225ed29b0b4SJens Axboe if (io_match_task_safe(de->req, task, cancel_all)) {
3226ed29b0b4SJens Axboe list_cut_position(&list, &ctx->defer_list, &de->list);
3227ed29b0b4SJens Axboe break;
3228ed29b0b4SJens Axboe }
3229ed29b0b4SJens Axboe }
3230ed29b0b4SJens Axboe spin_unlock(&ctx->completion_lock);
3231ed29b0b4SJens Axboe if (list_empty(&list))
3232ed29b0b4SJens Axboe return false;
3233ed29b0b4SJens Axboe
3234ed29b0b4SJens Axboe while (!list_empty(&list)) {
3235ed29b0b4SJens Axboe de = list_first_entry(&list, struct io_defer_entry, list);
3236ed29b0b4SJens Axboe list_del_init(&de->list);
3237e276ae34SPavel Begunkov io_req_task_queue_fail(de->req, -ECANCELED);
3238ed29b0b4SJens Axboe kfree(de);
3239ed29b0b4SJens Axboe }
3240ed29b0b4SJens Axboe return true;
3241ed29b0b4SJens Axboe }
3242ed29b0b4SJens Axboe
io_uring_try_cancel_iowq(struct io_ring_ctx * ctx)3243ed29b0b4SJens Axboe static __cold bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
3244ed29b0b4SJens Axboe {
3245ed29b0b4SJens Axboe struct io_tctx_node *node;
3246ed29b0b4SJens Axboe enum io_wq_cancel cret;
3247ed29b0b4SJens Axboe bool ret = false;
3248ed29b0b4SJens Axboe
3249ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3250ed29b0b4SJens Axboe list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
3251ed29b0b4SJens Axboe struct io_uring_task *tctx = node->task->io_uring;
3252ed29b0b4SJens Axboe
3253ed29b0b4SJens Axboe /*
3254ed29b0b4SJens Axboe * io_wq will stay alive while we hold uring_lock, because it's
3255ed29b0b4SJens Axboe * killed after ctx nodes, which requires to take the lock.
3256ed29b0b4SJens Axboe */
3257ed29b0b4SJens Axboe if (!tctx || !tctx->io_wq)
3258ed29b0b4SJens Axboe continue;
3259ed29b0b4SJens Axboe cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
3260ed29b0b4SJens Axboe ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
3261ed29b0b4SJens Axboe }
3262ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3263ed29b0b4SJens Axboe
3264ed29b0b4SJens Axboe return ret;
3265ed29b0b4SJens Axboe }
3266ed29b0b4SJens Axboe
io_uring_try_cancel_requests(struct io_ring_ctx * ctx,struct task_struct * task,bool cancel_all)3267affa87dbSPavel Begunkov static __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
3268ed29b0b4SJens Axboe struct task_struct *task,
3269ed29b0b4SJens Axboe bool cancel_all)
3270ed29b0b4SJens Axboe {
3271ed29b0b4SJens Axboe struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
3272ed29b0b4SJens Axboe struct io_uring_task *tctx = task ? task->io_uring : NULL;
3273affa87dbSPavel Begunkov enum io_wq_cancel cret;
3274affa87dbSPavel Begunkov bool ret = false;
3275ed29b0b4SJens Axboe
3276360cd42cSPavel Begunkov /* set it so io_req_local_work_add() would wake us up */
3277360cd42cSPavel Begunkov if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
3278360cd42cSPavel Begunkov atomic_set(&ctx->cq_wait_nr, 1);
3279360cd42cSPavel Begunkov smp_mb();
3280360cd42cSPavel Begunkov }
3281360cd42cSPavel Begunkov
3282ed29b0b4SJens Axboe /* failed during ring init, it couldn't have issued any requests */
3283ed29b0b4SJens Axboe if (!ctx->rings)
3284affa87dbSPavel Begunkov return false;
3285ed29b0b4SJens Axboe
3286ed29b0b4SJens Axboe if (!task) {
3287ed29b0b4SJens Axboe ret |= io_uring_try_cancel_iowq(ctx);
3288ed29b0b4SJens Axboe } else if (tctx && tctx->io_wq) {
3289ed29b0b4SJens Axboe /*
3290ed29b0b4SJens Axboe * Cancels requests of all rings, not only @ctx, but
3291ed29b0b4SJens Axboe * it's fine as the task is in exit/exec.
3292ed29b0b4SJens Axboe */
3293ed29b0b4SJens Axboe cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
3294ed29b0b4SJens Axboe &cancel, true);
3295ed29b0b4SJens Axboe ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
3296ed29b0b4SJens Axboe }
3297ed29b0b4SJens Axboe
3298ed29b0b4SJens Axboe /* SQPOLL thread does its own polling */
3299ed29b0b4SJens Axboe if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
3300ed29b0b4SJens Axboe (ctx->sq_data && ctx->sq_data->thread == current)) {
3301ed29b0b4SJens Axboe while (!wq_list_empty(&ctx->iopoll_list)) {
3302ed29b0b4SJens Axboe io_iopoll_try_reap_events(ctx);
3303ed29b0b4SJens Axboe ret = true;
3304fcc926bbSJens Axboe cond_resched();
3305ed29b0b4SJens Axboe }
3306ed29b0b4SJens Axboe }
3307ed29b0b4SJens Axboe
3308140102aeSPavel Begunkov if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
3309140102aeSPavel Begunkov io_allowed_defer_tw_run(ctx))
33107b8fa7a0SJens Axboe ret |= io_run_local_work(ctx, INT_MAX) > 0;
3311ed29b0b4SJens Axboe ret |= io_cancel_defer_files(ctx, task, cancel_all);
33129ca9fb24SPavel Begunkov mutex_lock(&ctx->uring_lock);
3313ed29b0b4SJens Axboe ret |= io_poll_remove_all(ctx, task, cancel_all);
33149ca9fb24SPavel Begunkov mutex_unlock(&ctx->uring_lock);
3315ed29b0b4SJens Axboe ret |= io_kill_timeouts(ctx, task, cancel_all);
3316ed29b0b4SJens Axboe if (task)
3317c0e0d6baSDylan Yudaken ret |= io_run_task_work() > 0;
3318affa87dbSPavel Begunkov return ret;
3319ed29b0b4SJens Axboe }
3320ed29b0b4SJens Axboe
tctx_inflight(struct io_uring_task * tctx,bool tracked)3321ed29b0b4SJens Axboe static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
3322ed29b0b4SJens Axboe {
3323ed29b0b4SJens Axboe if (tracked)
3324ed29b0b4SJens Axboe return atomic_read(&tctx->inflight_tracked);
3325ed29b0b4SJens Axboe return percpu_counter_sum(&tctx->inflight);
3326ed29b0b4SJens Axboe }
3327ed29b0b4SJens Axboe
3328ed29b0b4SJens Axboe /*
3329ed29b0b4SJens Axboe * Find any io_uring ctx that this task has registered or done IO on, and cancel
3330ed29b0b4SJens Axboe * requests. @sqd should be not-null IFF it's an SQPOLL thread cancellation.
3331ed29b0b4SJens Axboe */
io_uring_cancel_generic(bool cancel_all,struct io_sq_data * sqd)333217437f31SJens Axboe __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd)
3333ed29b0b4SJens Axboe {
3334ed29b0b4SJens Axboe struct io_uring_task *tctx = current->io_uring;
3335ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
3336360cd42cSPavel Begunkov struct io_tctx_node *node;
3337360cd42cSPavel Begunkov unsigned long index;
3338ed29b0b4SJens Axboe s64 inflight;
3339ed29b0b4SJens Axboe DEFINE_WAIT(wait);
3340ed29b0b4SJens Axboe
3341ed29b0b4SJens Axboe WARN_ON_ONCE(sqd && sqd->thread != current);
3342ed29b0b4SJens Axboe
3343ed29b0b4SJens Axboe if (!current->io_uring)
3344ed29b0b4SJens Axboe return;
3345ed29b0b4SJens Axboe if (tctx->io_wq)
3346ed29b0b4SJens Axboe io_wq_exit_start(tctx->io_wq);
3347ed29b0b4SJens Axboe
33488d664282SJens Axboe atomic_inc(&tctx->in_cancel);
3349ed29b0b4SJens Axboe do {
3350affa87dbSPavel Begunkov bool loop = false;
3351affa87dbSPavel Begunkov
3352ed29b0b4SJens Axboe io_uring_drop_tctx_refs(current);
3353ed29b0b4SJens Axboe /* read completions before cancelations */
3354ed29b0b4SJens Axboe inflight = tctx_inflight(tctx, !cancel_all);
3355ed29b0b4SJens Axboe if (!inflight)
3356ed29b0b4SJens Axboe break;
3357ed29b0b4SJens Axboe
3358ed29b0b4SJens Axboe if (!sqd) {
3359ed29b0b4SJens Axboe xa_for_each(&tctx->xa, index, node) {
3360ed29b0b4SJens Axboe /* sqpoll task will cancel all its requests */
3361ed29b0b4SJens Axboe if (node->ctx->sq_data)
3362ed29b0b4SJens Axboe continue;
3363affa87dbSPavel Begunkov loop |= io_uring_try_cancel_requests(node->ctx,
3364affa87dbSPavel Begunkov current, cancel_all);
3365ed29b0b4SJens Axboe }
3366ed29b0b4SJens Axboe } else {
3367ed29b0b4SJens Axboe list_for_each_entry(ctx, &sqd->ctx_list, sqd_list)
3368affa87dbSPavel Begunkov loop |= io_uring_try_cancel_requests(ctx,
3369affa87dbSPavel Begunkov current,
3370ed29b0b4SJens Axboe cancel_all);
3371ed29b0b4SJens Axboe }
3372ed29b0b4SJens Axboe
3373affa87dbSPavel Begunkov if (loop) {
3374affa87dbSPavel Begunkov cond_resched();
3375affa87dbSPavel Begunkov continue;
3376affa87dbSPavel Begunkov }
3377affa87dbSPavel Begunkov
3378ed29b0b4SJens Axboe prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
3379ed29b0b4SJens Axboe io_run_task_work();
3380ed29b0b4SJens Axboe io_uring_drop_tctx_refs(current);
3381360cd42cSPavel Begunkov xa_for_each(&tctx->xa, index, node) {
3382360cd42cSPavel Begunkov if (!llist_empty(&node->ctx->work_llist)) {
3383360cd42cSPavel Begunkov WARN_ON_ONCE(node->ctx->submitter_task &&
3384360cd42cSPavel Begunkov node->ctx->submitter_task != current);
3385360cd42cSPavel Begunkov goto end_wait;
3386360cd42cSPavel Begunkov }
3387360cd42cSPavel Begunkov }
3388ed29b0b4SJens Axboe /*
3389ed29b0b4SJens Axboe * If we've seen completions, retry without waiting. This
3390ed29b0b4SJens Axboe * avoids a race where a completion comes in before we did
3391ed29b0b4SJens Axboe * prepare_to_wait().
3392ed29b0b4SJens Axboe */
3393ed29b0b4SJens Axboe if (inflight == tctx_inflight(tctx, !cancel_all))
3394ed29b0b4SJens Axboe schedule();
3395360cd42cSPavel Begunkov end_wait:
3396ed29b0b4SJens Axboe finish_wait(&tctx->wait, &wait);
3397ed29b0b4SJens Axboe } while (1);
3398ed29b0b4SJens Axboe
3399ed29b0b4SJens Axboe io_uring_clean_tctx(tctx);
3400ed29b0b4SJens Axboe if (cancel_all) {
3401ed29b0b4SJens Axboe /*
3402ed29b0b4SJens Axboe * We shouldn't run task_works after cancel, so just leave
34038d664282SJens Axboe * ->in_cancel set for normal exit.
3404ed29b0b4SJens Axboe */
34058d664282SJens Axboe atomic_dec(&tctx->in_cancel);
3406ed29b0b4SJens Axboe /* for exec all current's requests should be gone, kill tctx */
3407ed29b0b4SJens Axboe __io_uring_free(current);
3408ed29b0b4SJens Axboe }
3409ed29b0b4SJens Axboe }
3410ed29b0b4SJens Axboe
__io_uring_cancel(bool cancel_all)3411ed29b0b4SJens Axboe void __io_uring_cancel(bool cancel_all)
3412ed29b0b4SJens Axboe {
3413ed29b0b4SJens Axboe io_uring_cancel_generic(cancel_all, NULL);
3414ed29b0b4SJens Axboe }
3415ed29b0b4SJens Axboe
io_uring_validate_mmap_request(struct file * file,loff_t pgoff,size_t sz)3416ed29b0b4SJens Axboe static void *io_uring_validate_mmap_request(struct file *file,
3417ed29b0b4SJens Axboe loff_t pgoff, size_t sz)
3418ed29b0b4SJens Axboe {
3419ed29b0b4SJens Axboe struct io_ring_ctx *ctx = file->private_data;
3420ed29b0b4SJens Axboe loff_t offset = pgoff << PAGE_SHIFT;
3421ed29b0b4SJens Axboe struct page *page;
3422ed29b0b4SJens Axboe void *ptr;
3423ed29b0b4SJens Axboe
3424c56e022cSJens Axboe switch (offset & IORING_OFF_MMAP_MASK) {
3425ed29b0b4SJens Axboe case IORING_OFF_SQ_RING:
3426ed29b0b4SJens Axboe case IORING_OFF_CQ_RING:
3427b797b7f8SJens Axboe /* Don't allow mmap if the ring was setup without it */
3428b797b7f8SJens Axboe if (ctx->flags & IORING_SETUP_NO_MMAP)
3429b797b7f8SJens Axboe return ERR_PTR(-EINVAL);
3430ed29b0b4SJens Axboe ptr = ctx->rings;
3431ed29b0b4SJens Axboe break;
3432ed29b0b4SJens Axboe case IORING_OFF_SQES:
3433b797b7f8SJens Axboe /* Don't allow mmap if the ring was setup without it */
3434b797b7f8SJens Axboe if (ctx->flags & IORING_SETUP_NO_MMAP)
3435b797b7f8SJens Axboe return ERR_PTR(-EINVAL);
3436ed29b0b4SJens Axboe ptr = ctx->sq_sqes;
3437ed29b0b4SJens Axboe break;
3438c56e022cSJens Axboe case IORING_OFF_PBUF_RING: {
343965938e81SJens Axboe struct io_buffer_list *bl;
3440c56e022cSJens Axboe unsigned int bgid;
3441c56e022cSJens Axboe
3442c56e022cSJens Axboe bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
344365938e81SJens Axboe bl = io_pbuf_get_bl(ctx, bgid);
344465938e81SJens Axboe if (IS_ERR(bl))
344565938e81SJens Axboe return bl;
344665938e81SJens Axboe ptr = bl->buf_ring;
344765938e81SJens Axboe io_put_bl(ctx, bl);
3448c56e022cSJens Axboe break;
3449c56e022cSJens Axboe }
3450ed29b0b4SJens Axboe default:
3451ed29b0b4SJens Axboe return ERR_PTR(-EINVAL);
3452ed29b0b4SJens Axboe }
3453ed29b0b4SJens Axboe
3454ed29b0b4SJens Axboe page = virt_to_head_page(ptr);
3455ed29b0b4SJens Axboe if (sz > page_size(page))
3456ed29b0b4SJens Axboe return ERR_PTR(-EINVAL);
3457ed29b0b4SJens Axboe
3458ed29b0b4SJens Axboe return ptr;
3459ed29b0b4SJens Axboe }
3460ed29b0b4SJens Axboe
3461ed29b0b4SJens Axboe #ifdef CONFIG_MMU
3462ed29b0b4SJens Axboe
io_uring_mmap(struct file * file,struct vm_area_struct * vma)3463ed29b0b4SJens Axboe static __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
3464ed29b0b4SJens Axboe {
3465ed29b0b4SJens Axboe size_t sz = vma->vm_end - vma->vm_start;
3466ed29b0b4SJens Axboe unsigned long pfn;
3467ed29b0b4SJens Axboe void *ptr;
3468ed29b0b4SJens Axboe
3469ed29b0b4SJens Axboe ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
3470ed29b0b4SJens Axboe if (IS_ERR(ptr))
3471ed29b0b4SJens Axboe return PTR_ERR(ptr);
3472ed29b0b4SJens Axboe
3473ed29b0b4SJens Axboe pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
3474ed29b0b4SJens Axboe return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
3475ed29b0b4SJens Axboe }
3476ed29b0b4SJens Axboe
io_uring_mmu_get_unmapped_area(struct file * filp,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)3477d808459bSHelge Deller static unsigned long io_uring_mmu_get_unmapped_area(struct file *filp,
3478d808459bSHelge Deller unsigned long addr, unsigned long len,
3479d808459bSHelge Deller unsigned long pgoff, unsigned long flags)
3480d808459bSHelge Deller {
3481d808459bSHelge Deller void *ptr;
3482d808459bSHelge Deller
3483d808459bSHelge Deller /*
3484d808459bSHelge Deller * Do not allow to map to user-provided address to avoid breaking the
3485d808459bSHelge Deller * aliasing rules. Userspace is not able to guess the offset address of
3486d808459bSHelge Deller * kernel kmalloc()ed memory area.
3487d808459bSHelge Deller */
3488d808459bSHelge Deller if (addr)
3489d808459bSHelge Deller return -EINVAL;
3490d808459bSHelge Deller
3491d808459bSHelge Deller ptr = io_uring_validate_mmap_request(filp, pgoff, len);
3492d808459bSHelge Deller if (IS_ERR(ptr))
3493d808459bSHelge Deller return -ENOMEM;
3494d808459bSHelge Deller
3495d808459bSHelge Deller /*
349632832a40SHelge Deller * Some architectures have strong cache aliasing requirements.
349732832a40SHelge Deller * For such architectures we need a coherent mapping which aliases
349832832a40SHelge Deller * kernel memory *and* userspace memory. To achieve that:
349932832a40SHelge Deller * - use a NULL file pointer to reference physical memory, and
350032832a40SHelge Deller * - use the kernel virtual address of the shared io_uring context
350132832a40SHelge Deller * (instead of the userspace-provided address, which has to be 0UL
350232832a40SHelge Deller * anyway).
350356675f8bSHelge Deller * - use the same pgoff which the get_unmapped_area() uses to
350456675f8bSHelge Deller * calculate the page colouring.
350532832a40SHelge Deller * For architectures without such aliasing requirements, the
350632832a40SHelge Deller * architecture will return any suitable mapping because addr is 0.
3507d808459bSHelge Deller */
350832832a40SHelge Deller filp = NULL;
350932832a40SHelge Deller flags |= MAP_SHARED;
351032832a40SHelge Deller pgoff = 0; /* has been translated to ptr above */
351132832a40SHelge Deller #ifdef SHM_COLOUR
351232832a40SHelge Deller addr = (uintptr_t) ptr;
351356675f8bSHelge Deller pgoff = addr >> PAGE_SHIFT;
351432832a40SHelge Deller #else
351532832a40SHelge Deller addr = 0UL;
351632832a40SHelge Deller #endif
351732832a40SHelge Deller return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
3518d808459bSHelge Deller }
3519d808459bSHelge Deller
3520ed29b0b4SJens Axboe #else /* !CONFIG_MMU */
3521ed29b0b4SJens Axboe
io_uring_mmap(struct file * file,struct vm_area_struct * vma)3522ed29b0b4SJens Axboe static int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
3523ed29b0b4SJens Axboe {
3524fc4f4be9SDavid Hildenbrand return is_nommu_shared_mapping(vma->vm_flags) ? 0 : -EINVAL;
3525ed29b0b4SJens Axboe }
3526ed29b0b4SJens Axboe
io_uring_nommu_mmap_capabilities(struct file * file)3527ed29b0b4SJens Axboe static unsigned int io_uring_nommu_mmap_capabilities(struct file *file)
3528ed29b0b4SJens Axboe {
3529ed29b0b4SJens Axboe return NOMMU_MAP_DIRECT | NOMMU_MAP_READ | NOMMU_MAP_WRITE;
3530ed29b0b4SJens Axboe }
3531ed29b0b4SJens Axboe
io_uring_nommu_get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)3532ed29b0b4SJens Axboe static unsigned long io_uring_nommu_get_unmapped_area(struct file *file,
3533ed29b0b4SJens Axboe unsigned long addr, unsigned long len,
3534ed29b0b4SJens Axboe unsigned long pgoff, unsigned long flags)
3535ed29b0b4SJens Axboe {
3536ed29b0b4SJens Axboe void *ptr;
3537ed29b0b4SJens Axboe
3538ed29b0b4SJens Axboe ptr = io_uring_validate_mmap_request(file, pgoff, len);
3539ed29b0b4SJens Axboe if (IS_ERR(ptr))
3540ed29b0b4SJens Axboe return PTR_ERR(ptr);
3541ed29b0b4SJens Axboe
3542ed29b0b4SJens Axboe return (unsigned long) ptr;
3543ed29b0b4SJens Axboe }
3544ed29b0b4SJens Axboe
3545ed29b0b4SJens Axboe #endif /* !CONFIG_MMU */
3546ed29b0b4SJens Axboe
io_validate_ext_arg(unsigned flags,const void __user * argp,size_t argsz)3547ed29b0b4SJens Axboe static int io_validate_ext_arg(unsigned flags, const void __user *argp, size_t argsz)
3548ed29b0b4SJens Axboe {
3549ed29b0b4SJens Axboe if (flags & IORING_ENTER_EXT_ARG) {
3550ed29b0b4SJens Axboe struct io_uring_getevents_arg arg;
3551ed29b0b4SJens Axboe
3552ed29b0b4SJens Axboe if (argsz != sizeof(arg))
3553ed29b0b4SJens Axboe return -EINVAL;
3554ed29b0b4SJens Axboe if (copy_from_user(&arg, argp, sizeof(arg)))
3555ed29b0b4SJens Axboe return -EFAULT;
3556ed29b0b4SJens Axboe }
3557ed29b0b4SJens Axboe return 0;
3558ed29b0b4SJens Axboe }
3559ed29b0b4SJens Axboe
io_get_ext_arg(unsigned flags,const void __user * argp,size_t * argsz,struct __kernel_timespec __user ** ts,const sigset_t __user ** sig)3560ed29b0b4SJens Axboe static int io_get_ext_arg(unsigned flags, const void __user *argp, size_t *argsz,
3561ed29b0b4SJens Axboe struct __kernel_timespec __user **ts,
3562ed29b0b4SJens Axboe const sigset_t __user **sig)
3563ed29b0b4SJens Axboe {
3564ed29b0b4SJens Axboe struct io_uring_getevents_arg arg;
3565ed29b0b4SJens Axboe
3566ed29b0b4SJens Axboe /*
3567ed29b0b4SJens Axboe * If EXT_ARG isn't set, then we have no timespec and the argp pointer
3568ed29b0b4SJens Axboe * is just a pointer to the sigset_t.
3569ed29b0b4SJens Axboe */
3570ed29b0b4SJens Axboe if (!(flags & IORING_ENTER_EXT_ARG)) {
3571ed29b0b4SJens Axboe *sig = (const sigset_t __user *) argp;
3572ed29b0b4SJens Axboe *ts = NULL;
3573ed29b0b4SJens Axboe return 0;
3574ed29b0b4SJens Axboe }
3575ed29b0b4SJens Axboe
3576ed29b0b4SJens Axboe /*
3577ed29b0b4SJens Axboe * EXT_ARG is set - ensure we agree on the size of it and copy in our
3578ed29b0b4SJens Axboe * timespec and sigset_t pointers if good.
3579ed29b0b4SJens Axboe */
3580ed29b0b4SJens Axboe if (*argsz != sizeof(arg))
3581ed29b0b4SJens Axboe return -EINVAL;
3582ed29b0b4SJens Axboe if (copy_from_user(&arg, argp, sizeof(arg)))
3583ed29b0b4SJens Axboe return -EFAULT;
3584ed29b0b4SJens Axboe if (arg.pad)
3585ed29b0b4SJens Axboe return -EINVAL;
3586ed29b0b4SJens Axboe *sig = u64_to_user_ptr(arg.sigmask);
3587ed29b0b4SJens Axboe *argsz = arg.sigmask_sz;
3588ed29b0b4SJens Axboe *ts = u64_to_user_ptr(arg.ts);
3589ed29b0b4SJens Axboe return 0;
3590ed29b0b4SJens Axboe }
3591ed29b0b4SJens Axboe
SYSCALL_DEFINE6(io_uring_enter,unsigned int,fd,u32,to_submit,u32,min_complete,u32,flags,const void __user *,argp,size_t,argsz)3592ed29b0b4SJens Axboe SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
3593ed29b0b4SJens Axboe u32, min_complete, u32, flags, const void __user *, argp,
3594ed29b0b4SJens Axboe size_t, argsz)
3595ed29b0b4SJens Axboe {
3596ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
35970c7df8c2SJens Axboe struct file *file;
3598ed29b0b4SJens Axboe long ret;
3599ed29b0b4SJens Axboe
3600ed29b0b4SJens Axboe if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
3601ed29b0b4SJens Axboe IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG |
3602ed29b0b4SJens Axboe IORING_ENTER_REGISTERED_RING)))
3603ed29b0b4SJens Axboe return -EINVAL;
3604ed29b0b4SJens Axboe
3605ed29b0b4SJens Axboe /*
3606ed29b0b4SJens Axboe * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
3607ed29b0b4SJens Axboe * need only dereference our task private array to find it.
3608ed29b0b4SJens Axboe */
3609ed29b0b4SJens Axboe if (flags & IORING_ENTER_REGISTERED_RING) {
3610ed29b0b4SJens Axboe struct io_uring_task *tctx = current->io_uring;
3611ed29b0b4SJens Axboe
36123273c440SPavel Begunkov if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
3613ed29b0b4SJens Axboe return -EINVAL;
3614ed29b0b4SJens Axboe fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
36150c7df8c2SJens Axboe file = tctx->registered_rings[fd];
36160c7df8c2SJens Axboe if (unlikely(!file))
3617ed29b0b4SJens Axboe return -EBADF;
36183273c440SPavel Begunkov } else {
36190c7df8c2SJens Axboe file = fget(fd);
36200c7df8c2SJens Axboe if (unlikely(!file))
36213273c440SPavel Begunkov return -EBADF;
3622ed29b0b4SJens Axboe ret = -EOPNOTSUPP;
36230c7df8c2SJens Axboe if (unlikely(!io_is_uring_fops(file)))
3624fbb8bb02SPavel Begunkov goto out;
36253273c440SPavel Begunkov }
3626ed29b0b4SJens Axboe
36270c7df8c2SJens Axboe ctx = file->private_data;
3628ed29b0b4SJens Axboe ret = -EBADFD;
3629ed29b0b4SJens Axboe if (unlikely(ctx->flags & IORING_SETUP_R_DISABLED))
3630ed29b0b4SJens Axboe goto out;
3631ed29b0b4SJens Axboe
3632ed29b0b4SJens Axboe /*
3633ed29b0b4SJens Axboe * For SQ polling, the thread will do all submissions and completions.
3634ed29b0b4SJens Axboe * Just return the requested submit count, and wake the thread if
3635ed29b0b4SJens Axboe * we were asked to.
3636ed29b0b4SJens Axboe */
3637ed29b0b4SJens Axboe ret = 0;
3638ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) {
3639ed29b0b4SJens Axboe io_cqring_overflow_flush(ctx);
3640ed29b0b4SJens Axboe
3641ed29b0b4SJens Axboe if (unlikely(ctx->sq_data->thread == NULL)) {
3642ed29b0b4SJens Axboe ret = -EOWNERDEAD;
3643ed29b0b4SJens Axboe goto out;
3644ed29b0b4SJens Axboe }
3645ed29b0b4SJens Axboe if (flags & IORING_ENTER_SQ_WAKEUP)
3646ed29b0b4SJens Axboe wake_up(&ctx->sq_data->wait);
364788b80534SQuanfa Fu if (flags & IORING_ENTER_SQ_WAIT)
364888b80534SQuanfa Fu io_sqpoll_wait_sq(ctx);
364988b80534SQuanfa Fu
3650ed29b0b4SJens Axboe ret = to_submit;
3651ed29b0b4SJens Axboe } else if (to_submit) {
3652ed29b0b4SJens Axboe ret = io_uring_add_tctx_node(ctx);
3653ed29b0b4SJens Axboe if (unlikely(ret))
3654ed29b0b4SJens Axboe goto out;
3655ed29b0b4SJens Axboe
3656ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3657ed29b0b4SJens Axboe ret = io_submit_sqes(ctx, to_submit);
3658ed29b0b4SJens Axboe if (ret != to_submit) {
3659ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3660ed29b0b4SJens Axboe goto out;
3661ed29b0b4SJens Axboe }
366244f87745SPavel Begunkov if (flags & IORING_ENTER_GETEVENTS) {
366344f87745SPavel Begunkov if (ctx->syscall_iopoll)
3664ed29b0b4SJens Axboe goto iopoll_locked;
366544f87745SPavel Begunkov /*
366644f87745SPavel Begunkov * Ignore errors, we'll soon call io_cqring_wait() and
366744f87745SPavel Begunkov * it should handle ownership problems if any.
366844f87745SPavel Begunkov */
366944f87745SPavel Begunkov if (ctx->flags & IORING_SETUP_DEFER_TASKRUN)
36707b8fa7a0SJens Axboe (void)io_run_local_work_locked(ctx, min_complete);
367144f87745SPavel Begunkov }
3672ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3673ed29b0b4SJens Axboe }
3674c0e0d6baSDylan Yudaken
3675ed29b0b4SJens Axboe if (flags & IORING_ENTER_GETEVENTS) {
3676ed29b0b4SJens Axboe int ret2;
3677c0e0d6baSDylan Yudaken
3678ed29b0b4SJens Axboe if (ctx->syscall_iopoll) {
3679ed29b0b4SJens Axboe /*
3680ed29b0b4SJens Axboe * We disallow the app entering submit/complete with
3681ed29b0b4SJens Axboe * polling, but we still need to lock the ring to
3682ed29b0b4SJens Axboe * prevent racing with polled issue that got punted to
3683ed29b0b4SJens Axboe * a workqueue.
3684ed29b0b4SJens Axboe */
3685ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
3686ed29b0b4SJens Axboe iopoll_locked:
3687ed29b0b4SJens Axboe ret2 = io_validate_ext_arg(flags, argp, argsz);
3688ed29b0b4SJens Axboe if (likely(!ret2)) {
3689ed29b0b4SJens Axboe min_complete = min(min_complete,
3690ed29b0b4SJens Axboe ctx->cq_entries);
3691ed29b0b4SJens Axboe ret2 = io_iopoll_check(ctx, min_complete);
3692ed29b0b4SJens Axboe }
3693ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
3694ed29b0b4SJens Axboe } else {
3695ed29b0b4SJens Axboe const sigset_t __user *sig;
3696ed29b0b4SJens Axboe struct __kernel_timespec __user *ts;
3697ed29b0b4SJens Axboe
3698ed29b0b4SJens Axboe ret2 = io_get_ext_arg(flags, argp, &argsz, &ts, &sig);
3699ed29b0b4SJens Axboe if (likely(!ret2)) {
3700ed29b0b4SJens Axboe min_complete = min(min_complete,
3701ed29b0b4SJens Axboe ctx->cq_entries);
3702ed29b0b4SJens Axboe ret2 = io_cqring_wait(ctx, min_complete, sig,
3703ed29b0b4SJens Axboe argsz, ts);
3704ed29b0b4SJens Axboe }
3705ed29b0b4SJens Axboe }
3706ed29b0b4SJens Axboe
3707ed29b0b4SJens Axboe if (!ret) {
3708ed29b0b4SJens Axboe ret = ret2;
3709ed29b0b4SJens Axboe
3710ed29b0b4SJens Axboe /*
3711ed29b0b4SJens Axboe * EBADR indicates that one or more CQE were dropped.
3712ed29b0b4SJens Axboe * Once the user has been informed we can clear the bit
3713ed29b0b4SJens Axboe * as they are obviously ok with those drops.
3714ed29b0b4SJens Axboe */
3715ed29b0b4SJens Axboe if (unlikely(ret2 == -EBADR))
3716ed29b0b4SJens Axboe clear_bit(IO_CHECK_CQ_DROPPED_BIT,
3717ed29b0b4SJens Axboe &ctx->check_cq);
3718ed29b0b4SJens Axboe }
3719ed29b0b4SJens Axboe }
3720ed29b0b4SJens Axboe out:
37210c7df8c2SJens Axboe if (!(flags & IORING_ENTER_REGISTERED_RING))
37220c7df8c2SJens Axboe fput(file);
3723ed29b0b4SJens Axboe return ret;
3724ed29b0b4SJens Axboe }
3725ed29b0b4SJens Axboe
3726ed29b0b4SJens Axboe static const struct file_operations io_uring_fops = {
3727ed29b0b4SJens Axboe .release = io_uring_release,
3728ed29b0b4SJens Axboe .mmap = io_uring_mmap,
3729ed29b0b4SJens Axboe #ifndef CONFIG_MMU
3730ed29b0b4SJens Axboe .get_unmapped_area = io_uring_nommu_get_unmapped_area,
3731ed29b0b4SJens Axboe .mmap_capabilities = io_uring_nommu_mmap_capabilities,
3732d808459bSHelge Deller #else
3733d808459bSHelge Deller .get_unmapped_area = io_uring_mmu_get_unmapped_area,
3734ed29b0b4SJens Axboe #endif
3735ed29b0b4SJens Axboe .poll = io_uring_poll,
3736ed29b0b4SJens Axboe #ifdef CONFIG_PROC_FS
3737ed29b0b4SJens Axboe .show_fdinfo = io_uring_show_fdinfo,
3738ed29b0b4SJens Axboe #endif
3739ed29b0b4SJens Axboe };
3740ed29b0b4SJens Axboe
io_is_uring_fops(struct file * file)374192ac8beaSJens Axboe bool io_is_uring_fops(struct file *file)
374292ac8beaSJens Axboe {
374392ac8beaSJens Axboe return file->f_op == &io_uring_fops;
374492ac8beaSJens Axboe }
374592ac8beaSJens Axboe
io_allocate_scq_urings(struct io_ring_ctx * ctx,struct io_uring_params * p)3746ed29b0b4SJens Axboe static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
3747ed29b0b4SJens Axboe struct io_uring_params *p)
3748ed29b0b4SJens Axboe {
3749ed29b0b4SJens Axboe struct io_rings *rings;
3750ed29b0b4SJens Axboe size_t size, sq_array_offset;
3751e27cef86SJens Axboe void *ptr;
3752ed29b0b4SJens Axboe
3753ed29b0b4SJens Axboe /* make sure these are sane, as we already accounted them */
3754ed29b0b4SJens Axboe ctx->sq_entries = p->sq_entries;
3755ed29b0b4SJens Axboe ctx->cq_entries = p->cq_entries;
3756ed29b0b4SJens Axboe
3757ed29b0b4SJens Axboe size = rings_size(ctx, p->sq_entries, p->cq_entries, &sq_array_offset);
3758ed29b0b4SJens Axboe if (size == SIZE_MAX)
3759ed29b0b4SJens Axboe return -EOVERFLOW;
3760ed29b0b4SJens Axboe
376103d89a2dSJens Axboe if (!(ctx->flags & IORING_SETUP_NO_MMAP))
3762ed29b0b4SJens Axboe rings = io_mem_alloc(size);
376303d89a2dSJens Axboe else
376403d89a2dSJens Axboe rings = io_rings_map(ctx, p->cq_off.user_addr, size);
376503d89a2dSJens Axboe
3766e27cef86SJens Axboe if (IS_ERR(rings))
3767e27cef86SJens Axboe return PTR_ERR(rings);
3768ed29b0b4SJens Axboe
3769ed29b0b4SJens Axboe ctx->rings = rings;
37702af89abdSPavel Begunkov if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
3771ed29b0b4SJens Axboe ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
3772ed29b0b4SJens Axboe rings->sq_ring_mask = p->sq_entries - 1;
3773ed29b0b4SJens Axboe rings->cq_ring_mask = p->cq_entries - 1;
3774ed29b0b4SJens Axboe rings->sq_ring_entries = p->sq_entries;
3775ed29b0b4SJens Axboe rings->cq_ring_entries = p->cq_entries;
3776ed29b0b4SJens Axboe
3777ed29b0b4SJens Axboe if (p->flags & IORING_SETUP_SQE128)
3778ed29b0b4SJens Axboe size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
3779ed29b0b4SJens Axboe else
3780ed29b0b4SJens Axboe size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
3781ed29b0b4SJens Axboe if (size == SIZE_MAX) {
37829c189eeeSJens Axboe io_rings_free(ctx);
3783ed29b0b4SJens Axboe return -EOVERFLOW;
3784ed29b0b4SJens Axboe }
3785ed29b0b4SJens Axboe
378603d89a2dSJens Axboe if (!(ctx->flags & IORING_SETUP_NO_MMAP))
3787e27cef86SJens Axboe ptr = io_mem_alloc(size);
378803d89a2dSJens Axboe else
378903d89a2dSJens Axboe ptr = io_sqes_map(ctx, p->sq_off.user_addr, size);
379003d89a2dSJens Axboe
3791e27cef86SJens Axboe if (IS_ERR(ptr)) {
37929c189eeeSJens Axboe io_rings_free(ctx);
3793e27cef86SJens Axboe return PTR_ERR(ptr);
3794ed29b0b4SJens Axboe }
3795ed29b0b4SJens Axboe
3796e27cef86SJens Axboe ctx->sq_sqes = ptr;
3797ed29b0b4SJens Axboe return 0;
3798ed29b0b4SJens Axboe }
3799ed29b0b4SJens Axboe
io_uring_install_fd(struct file * file)38006e76ac59SJosh Triplett static int io_uring_install_fd(struct file *file)
3801ed29b0b4SJens Axboe {
38026e76ac59SJosh Triplett int fd;
3803ed29b0b4SJens Axboe
3804ed29b0b4SJens Axboe fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
3805ed29b0b4SJens Axboe if (fd < 0)
3806ed29b0b4SJens Axboe return fd;
3807ed29b0b4SJens Axboe fd_install(fd, file);
3808ed29b0b4SJens Axboe return fd;
3809ed29b0b4SJens Axboe }
3810ed29b0b4SJens Axboe
3811ed29b0b4SJens Axboe /*
3812ed29b0b4SJens Axboe * Allocate an anonymous fd, this is what constitutes the application
3813ed29b0b4SJens Axboe * visible backing of an io_uring instance. The application mmaps this
38146fc19b3dSJens Axboe * fd to gain access to the SQ/CQ ring details.
3815ed29b0b4SJens Axboe */
io_uring_get_file(struct io_ring_ctx * ctx)3816ed29b0b4SJens Axboe static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
3817ed29b0b4SJens Axboe {
38186fc19b3dSJens Axboe return anon_inode_getfile_secure("[io_uring]", &io_uring_fops, ctx,
3819ed29b0b4SJens Axboe O_RDWR | O_CLOEXEC, NULL);
3820ed29b0b4SJens Axboe }
3821ed29b0b4SJens Axboe
io_uring_create(unsigned entries,struct io_uring_params * p,struct io_uring_params __user * params)3822ed29b0b4SJens Axboe static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
3823ed29b0b4SJens Axboe struct io_uring_params __user *params)
3824ed29b0b4SJens Axboe {
3825ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
38266e76ac59SJosh Triplett struct io_uring_task *tctx;
3827ed29b0b4SJens Axboe struct file *file;
3828ed29b0b4SJens Axboe int ret;
3829ed29b0b4SJens Axboe
3830ed29b0b4SJens Axboe if (!entries)
3831ed29b0b4SJens Axboe return -EINVAL;
3832ed29b0b4SJens Axboe if (entries > IORING_MAX_ENTRIES) {
3833ed29b0b4SJens Axboe if (!(p->flags & IORING_SETUP_CLAMP))
3834ed29b0b4SJens Axboe return -EINVAL;
3835ed29b0b4SJens Axboe entries = IORING_MAX_ENTRIES;
3836ed29b0b4SJens Axboe }
3837ed29b0b4SJens Axboe
38386e76ac59SJosh Triplett if ((p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
38396e76ac59SJosh Triplett && !(p->flags & IORING_SETUP_NO_MMAP))
38406e76ac59SJosh Triplett return -EINVAL;
38416e76ac59SJosh Triplett
3842ed29b0b4SJens Axboe /*
3843ed29b0b4SJens Axboe * Use twice as many entries for the CQ ring. It's possible for the
3844ed29b0b4SJens Axboe * application to drive a higher depth than the size of the SQ ring,
3845ed29b0b4SJens Axboe * since the sqes are only used at submission time. This allows for
3846ed29b0b4SJens Axboe * some flexibility in overcommitting a bit. If the application has
3847ed29b0b4SJens Axboe * set IORING_SETUP_CQSIZE, it will have passed in the desired number
3848ed29b0b4SJens Axboe * of CQ ring entries manually.
3849ed29b0b4SJens Axboe */
3850ed29b0b4SJens Axboe p->sq_entries = roundup_pow_of_two(entries);
3851ed29b0b4SJens Axboe if (p->flags & IORING_SETUP_CQSIZE) {
3852ed29b0b4SJens Axboe /*
3853ed29b0b4SJens Axboe * If IORING_SETUP_CQSIZE is set, we do the same roundup
3854ed29b0b4SJens Axboe * to a power-of-two, if it isn't already. We do NOT impose
3855ed29b0b4SJens Axboe * any cq vs sq ring sizing.
3856ed29b0b4SJens Axboe */
3857ed29b0b4SJens Axboe if (!p->cq_entries)
3858ed29b0b4SJens Axboe return -EINVAL;
3859ed29b0b4SJens Axboe if (p->cq_entries > IORING_MAX_CQ_ENTRIES) {
3860ed29b0b4SJens Axboe if (!(p->flags & IORING_SETUP_CLAMP))
3861ed29b0b4SJens Axboe return -EINVAL;
3862ed29b0b4SJens Axboe p->cq_entries = IORING_MAX_CQ_ENTRIES;
3863ed29b0b4SJens Axboe }
3864ed29b0b4SJens Axboe p->cq_entries = roundup_pow_of_two(p->cq_entries);
3865ed29b0b4SJens Axboe if (p->cq_entries < p->sq_entries)
3866ed29b0b4SJens Axboe return -EINVAL;
3867ed29b0b4SJens Axboe } else {
3868ed29b0b4SJens Axboe p->cq_entries = 2 * p->sq_entries;
3869ed29b0b4SJens Axboe }
3870ed29b0b4SJens Axboe
3871ed29b0b4SJens Axboe ctx = io_ring_ctx_alloc(p);
3872ed29b0b4SJens Axboe if (!ctx)
3873ed29b0b4SJens Axboe return -ENOMEM;
3874ed29b0b4SJens Axboe
3875e6aeb272SPavel Begunkov if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) &&
3876e6aeb272SPavel Begunkov !(ctx->flags & IORING_SETUP_IOPOLL) &&
3877e6aeb272SPavel Begunkov !(ctx->flags & IORING_SETUP_SQPOLL))
3878e6aeb272SPavel Begunkov ctx->task_complete = true;
3879e6aeb272SPavel Begunkov
3880ec26c225SPavel Begunkov if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL))
3881ec26c225SPavel Begunkov ctx->lockless_cq = true;
3882ec26c225SPavel Begunkov
3883ed29b0b4SJens Axboe /*
3884bca39f39SPavel Begunkov * lazy poll_wq activation relies on ->task_complete for synchronisation
3885bca39f39SPavel Begunkov * purposes, see io_activate_pollwq()
3886bca39f39SPavel Begunkov */
3887bca39f39SPavel Begunkov if (!ctx->task_complete)
3888bca39f39SPavel Begunkov ctx->poll_activated = true;
3889bca39f39SPavel Begunkov
3890bca39f39SPavel Begunkov /*
3891ed29b0b4SJens Axboe * When SETUP_IOPOLL and SETUP_SQPOLL are both enabled, user
3892ed29b0b4SJens Axboe * space applications don't need to do io completion events
3893ed29b0b4SJens Axboe * polling again, they can rely on io_sq_thread to do polling
3894ed29b0b4SJens Axboe * work, which can reduce cpu usage and uring_lock contention.
3895ed29b0b4SJens Axboe */
3896ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_IOPOLL &&
3897ed29b0b4SJens Axboe !(ctx->flags & IORING_SETUP_SQPOLL))
3898ed29b0b4SJens Axboe ctx->syscall_iopoll = 1;
3899ed29b0b4SJens Axboe
3900ed29b0b4SJens Axboe ctx->compat = in_compat_syscall();
39016adc2272SOndrej Mosnacek if (!ns_capable_noaudit(&init_user_ns, CAP_IPC_LOCK))
3902ed29b0b4SJens Axboe ctx->user = get_uid(current_user());
3903ed29b0b4SJens Axboe
3904ed29b0b4SJens Axboe /*
3905ed29b0b4SJens Axboe * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
3906ed29b0b4SJens Axboe * COOP_TASKRUN is set, then IPIs are never needed by the app.
3907ed29b0b4SJens Axboe */
3908ed29b0b4SJens Axboe ret = -EINVAL;
3909ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) {
3910ed29b0b4SJens Axboe /* IPI related flags don't make sense with SQPOLL */
3911ed29b0b4SJens Axboe if (ctx->flags & (IORING_SETUP_COOP_TASKRUN |
3912c0e0d6baSDylan Yudaken IORING_SETUP_TASKRUN_FLAG |
3913c0e0d6baSDylan Yudaken IORING_SETUP_DEFER_TASKRUN))
3914ed29b0b4SJens Axboe goto err;
3915ed29b0b4SJens Axboe ctx->notify_method = TWA_SIGNAL_NO_IPI;
3916ed29b0b4SJens Axboe } else if (ctx->flags & IORING_SETUP_COOP_TASKRUN) {
3917ed29b0b4SJens Axboe ctx->notify_method = TWA_SIGNAL_NO_IPI;
3918ed29b0b4SJens Axboe } else {
3919c0e0d6baSDylan Yudaken if (ctx->flags & IORING_SETUP_TASKRUN_FLAG &&
3920c0e0d6baSDylan Yudaken !(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
3921ed29b0b4SJens Axboe goto err;
3922ed29b0b4SJens Axboe ctx->notify_method = TWA_SIGNAL;
3923ed29b0b4SJens Axboe }
3924ed29b0b4SJens Axboe
3925ed29b0b4SJens Axboe /*
3926c0e0d6baSDylan Yudaken * For DEFER_TASKRUN we require the completion task to be the same as the
3927c0e0d6baSDylan Yudaken * submission task. This implies that there is only one submitter, so enforce
3928c0e0d6baSDylan Yudaken * that.
3929c0e0d6baSDylan Yudaken */
3930c0e0d6baSDylan Yudaken if (ctx->flags & IORING_SETUP_DEFER_TASKRUN &&
3931c0e0d6baSDylan Yudaken !(ctx->flags & IORING_SETUP_SINGLE_ISSUER)) {
3932c0e0d6baSDylan Yudaken goto err;
3933c0e0d6baSDylan Yudaken }
3934c0e0d6baSDylan Yudaken
3935c0e0d6baSDylan Yudaken /*
3936ed29b0b4SJens Axboe * This is just grabbed for accounting purposes. When a process exits,
3937ed29b0b4SJens Axboe * the mm is exited and dropped before the files, hence we need to hang
3938ed29b0b4SJens Axboe * on to this mm purely for the purposes of being able to unaccount
3939ed29b0b4SJens Axboe * memory (locked/pinned vm). It's not used for anything else.
3940ed29b0b4SJens Axboe */
3941ed29b0b4SJens Axboe mmgrab(current->mm);
3942ed29b0b4SJens Axboe ctx->mm_account = current->mm;
3943ed29b0b4SJens Axboe
3944ed29b0b4SJens Axboe ret = io_allocate_scq_urings(ctx, p);
3945ed29b0b4SJens Axboe if (ret)
3946ed29b0b4SJens Axboe goto err;
3947ed29b0b4SJens Axboe
3948ed29b0b4SJens Axboe ret = io_sq_offload_create(ctx, p);
3949ed29b0b4SJens Axboe if (ret)
3950ed29b0b4SJens Axboe goto err;
39512933ae6eSPavel Begunkov
39522933ae6eSPavel Begunkov ret = io_rsrc_init(ctx);
3953ed29b0b4SJens Axboe if (ret)
3954ed29b0b4SJens Axboe goto err;
3955ed29b0b4SJens Axboe
3956ed29b0b4SJens Axboe p->sq_off.head = offsetof(struct io_rings, sq.head);
3957ed29b0b4SJens Axboe p->sq_off.tail = offsetof(struct io_rings, sq.tail);
3958ed29b0b4SJens Axboe p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
3959ed29b0b4SJens Axboe p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
3960ed29b0b4SJens Axboe p->sq_off.flags = offsetof(struct io_rings, sq_flags);
3961ed29b0b4SJens Axboe p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
39622af89abdSPavel Begunkov if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
3963ed29b0b4SJens Axboe p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
39649b1b58caSJens Axboe p->sq_off.resv1 = 0;
396503d89a2dSJens Axboe if (!(ctx->flags & IORING_SETUP_NO_MMAP))
396603d89a2dSJens Axboe p->sq_off.user_addr = 0;
3967ed29b0b4SJens Axboe
3968ed29b0b4SJens Axboe p->cq_off.head = offsetof(struct io_rings, cq.head);
3969ed29b0b4SJens Axboe p->cq_off.tail = offsetof(struct io_rings, cq.tail);
3970ed29b0b4SJens Axboe p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
3971ed29b0b4SJens Axboe p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
3972ed29b0b4SJens Axboe p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
3973ed29b0b4SJens Axboe p->cq_off.cqes = offsetof(struct io_rings, cqes);
3974ed29b0b4SJens Axboe p->cq_off.flags = offsetof(struct io_rings, cq_flags);
39759b1b58caSJens Axboe p->cq_off.resv1 = 0;
397603d89a2dSJens Axboe if (!(ctx->flags & IORING_SETUP_NO_MMAP))
397703d89a2dSJens Axboe p->cq_off.user_addr = 0;
3978ed29b0b4SJens Axboe
3979ed29b0b4SJens Axboe p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP |
3980ed29b0b4SJens Axboe IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS |
3981ed29b0b4SJens Axboe IORING_FEAT_CUR_PERSONALITY | IORING_FEAT_FAST_POLL |
3982ed29b0b4SJens Axboe IORING_FEAT_POLL_32BITS | IORING_FEAT_SQPOLL_NONFIXED |
3983ed29b0b4SJens Axboe IORING_FEAT_EXT_ARG | IORING_FEAT_NATIVE_WORKERS |
3984ed29b0b4SJens Axboe IORING_FEAT_RSRC_TAGS | IORING_FEAT_CQE_SKIP |
39857d3fd88dSJosh Triplett IORING_FEAT_LINKED_FILE | IORING_FEAT_REG_REG_RING;
3986ed29b0b4SJens Axboe
3987ed29b0b4SJens Axboe if (copy_to_user(params, p, sizeof(*p))) {
3988ed29b0b4SJens Axboe ret = -EFAULT;
3989ed29b0b4SJens Axboe goto err;
3990ed29b0b4SJens Axboe }
3991ed29b0b4SJens Axboe
39927cae596bSDylan Yudaken if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
39937cae596bSDylan Yudaken && !(ctx->flags & IORING_SETUP_R_DISABLED))
39948579538cSPavel Begunkov WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
39957cae596bSDylan Yudaken
3996ed29b0b4SJens Axboe file = io_uring_get_file(ctx);
3997ed29b0b4SJens Axboe if (IS_ERR(file)) {
3998ed29b0b4SJens Axboe ret = PTR_ERR(file);
3999ed29b0b4SJens Axboe goto err;
4000ed29b0b4SJens Axboe }
4001ed29b0b4SJens Axboe
40026e76ac59SJosh Triplett ret = __io_uring_add_tctx_node(ctx);
40036e76ac59SJosh Triplett if (ret)
40046e76ac59SJosh Triplett goto err_fput;
40056e76ac59SJosh Triplett tctx = current->io_uring;
40066e76ac59SJosh Triplett
4007ed29b0b4SJens Axboe /*
4008ed29b0b4SJens Axboe * Install ring fd as the very last thing, so we don't risk someone
4009ed29b0b4SJens Axboe * having closed it before we finish setup
4010ed29b0b4SJens Axboe */
40116e76ac59SJosh Triplett if (p->flags & IORING_SETUP_REGISTERED_FD_ONLY)
40126e76ac59SJosh Triplett ret = io_ring_add_registered_file(tctx, file, 0, IO_RINGFD_REG_MAX);
40136e76ac59SJosh Triplett else
40146e76ac59SJosh Triplett ret = io_uring_install_fd(file);
40156e76ac59SJosh Triplett if (ret < 0)
40166e76ac59SJosh Triplett goto err_fput;
4017ed29b0b4SJens Axboe
4018ed29b0b4SJens Axboe trace_io_uring_create(ret, ctx, p->sq_entries, p->cq_entries, p->flags);
4019ed29b0b4SJens Axboe return ret;
4020ed29b0b4SJens Axboe err:
4021ed29b0b4SJens Axboe io_ring_ctx_wait_and_kill(ctx);
4022ed29b0b4SJens Axboe return ret;
40236e76ac59SJosh Triplett err_fput:
40246e76ac59SJosh Triplett fput(file);
40256e76ac59SJosh Triplett return ret;
4026ed29b0b4SJens Axboe }
4027ed29b0b4SJens Axboe
4028ed29b0b4SJens Axboe /*
4029ed29b0b4SJens Axboe * Sets up an aio uring context, and returns the fd. Applications asks for a
4030ed29b0b4SJens Axboe * ring size, we return the actual sq/cq ring sizes (among other things) in the
4031ed29b0b4SJens Axboe * params structure passed in.
4032ed29b0b4SJens Axboe */
io_uring_setup(u32 entries,struct io_uring_params __user * params)4033ed29b0b4SJens Axboe static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
4034ed29b0b4SJens Axboe {
4035ed29b0b4SJens Axboe struct io_uring_params p;
4036ed29b0b4SJens Axboe int i;
4037ed29b0b4SJens Axboe
4038ed29b0b4SJens Axboe if (copy_from_user(&p, params, sizeof(p)))
4039ed29b0b4SJens Axboe return -EFAULT;
4040ed29b0b4SJens Axboe for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
4041ed29b0b4SJens Axboe if (p.resv[i])
4042ed29b0b4SJens Axboe return -EINVAL;
4043ed29b0b4SJens Axboe }
4044ed29b0b4SJens Axboe
4045ed29b0b4SJens Axboe if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
4046ed29b0b4SJens Axboe IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
4047ed29b0b4SJens Axboe IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ |
4048ed29b0b4SJens Axboe IORING_SETUP_R_DISABLED | IORING_SETUP_SUBMIT_ALL |
4049ed29b0b4SJens Axboe IORING_SETUP_COOP_TASKRUN | IORING_SETUP_TASKRUN_FLAG |
405097bbdc06SPavel Begunkov IORING_SETUP_SQE128 | IORING_SETUP_CQE32 |
405103d89a2dSJens Axboe IORING_SETUP_SINGLE_ISSUER | IORING_SETUP_DEFER_TASKRUN |
40522af89abdSPavel Begunkov IORING_SETUP_NO_MMAP | IORING_SETUP_REGISTERED_FD_ONLY |
40532af89abdSPavel Begunkov IORING_SETUP_NO_SQARRAY))
4054ed29b0b4SJens Axboe return -EINVAL;
4055ed29b0b4SJens Axboe
4056ed29b0b4SJens Axboe return io_uring_create(entries, &p, params);
4057ed29b0b4SJens Axboe }
4058ed29b0b4SJens Axboe
io_uring_allowed(void)405976d3ccecSMatteo Rizzo static inline bool io_uring_allowed(void)
406076d3ccecSMatteo Rizzo {
406176d3ccecSMatteo Rizzo int disabled = READ_ONCE(sysctl_io_uring_disabled);
406276d3ccecSMatteo Rizzo kgid_t io_uring_group;
406376d3ccecSMatteo Rizzo
406476d3ccecSMatteo Rizzo if (disabled == 2)
406576d3ccecSMatteo Rizzo return false;
406676d3ccecSMatteo Rizzo
406776d3ccecSMatteo Rizzo if (disabled == 0 || capable(CAP_SYS_ADMIN))
406876d3ccecSMatteo Rizzo return true;
406976d3ccecSMatteo Rizzo
407076d3ccecSMatteo Rizzo io_uring_group = make_kgid(&init_user_ns, sysctl_io_uring_group);
407176d3ccecSMatteo Rizzo if (!gid_valid(io_uring_group))
407276d3ccecSMatteo Rizzo return false;
407376d3ccecSMatteo Rizzo
407476d3ccecSMatteo Rizzo return in_group_p(io_uring_group);
407576d3ccecSMatteo Rizzo }
407676d3ccecSMatteo Rizzo
SYSCALL_DEFINE2(io_uring_setup,u32,entries,struct io_uring_params __user *,params)4077ed29b0b4SJens Axboe SYSCALL_DEFINE2(io_uring_setup, u32, entries,
4078ed29b0b4SJens Axboe struct io_uring_params __user *, params)
4079ed29b0b4SJens Axboe {
408076d3ccecSMatteo Rizzo if (!io_uring_allowed())
408176d3ccecSMatteo Rizzo return -EPERM;
408276d3ccecSMatteo Rizzo
4083ed29b0b4SJens Axboe return io_uring_setup(entries, params);
4084ed29b0b4SJens Axboe }
4085ed29b0b4SJens Axboe
io_probe(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_args)4086ed29b0b4SJens Axboe static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
4087ed29b0b4SJens Axboe unsigned nr_args)
4088ed29b0b4SJens Axboe {
4089ed29b0b4SJens Axboe struct io_uring_probe *p;
4090ed29b0b4SJens Axboe size_t size;
4091ed29b0b4SJens Axboe int i, ret;
4092ed29b0b4SJens Axboe
4093ed29b0b4SJens Axboe size = struct_size(p, ops, nr_args);
4094ed29b0b4SJens Axboe if (size == SIZE_MAX)
4095ed29b0b4SJens Axboe return -EOVERFLOW;
4096ed29b0b4SJens Axboe p = kzalloc(size, GFP_KERNEL);
4097ed29b0b4SJens Axboe if (!p)
4098ed29b0b4SJens Axboe return -ENOMEM;
4099ed29b0b4SJens Axboe
4100ed29b0b4SJens Axboe ret = -EFAULT;
4101ed29b0b4SJens Axboe if (copy_from_user(p, arg, size))
4102ed29b0b4SJens Axboe goto out;
4103ed29b0b4SJens Axboe ret = -EINVAL;
4104ed29b0b4SJens Axboe if (memchr_inv(p, 0, size))
4105ed29b0b4SJens Axboe goto out;
4106ed29b0b4SJens Axboe
4107ed29b0b4SJens Axboe p->last_op = IORING_OP_LAST - 1;
4108ed29b0b4SJens Axboe if (nr_args > IORING_OP_LAST)
4109ed29b0b4SJens Axboe nr_args = IORING_OP_LAST;
4110ed29b0b4SJens Axboe
4111ed29b0b4SJens Axboe for (i = 0; i < nr_args; i++) {
4112ed29b0b4SJens Axboe p->ops[i].op = i;
4113a7dd2782SBreno Leitao if (!io_issue_defs[i].not_supported)
4114ed29b0b4SJens Axboe p->ops[i].flags = IO_URING_OP_SUPPORTED;
4115ed29b0b4SJens Axboe }
4116ed29b0b4SJens Axboe p->ops_len = i;
4117ed29b0b4SJens Axboe
4118ed29b0b4SJens Axboe ret = 0;
4119ed29b0b4SJens Axboe if (copy_to_user(arg, p, size))
4120ed29b0b4SJens Axboe ret = -EFAULT;
4121ed29b0b4SJens Axboe out:
4122ed29b0b4SJens Axboe kfree(p);
4123ed29b0b4SJens Axboe return ret;
4124ed29b0b4SJens Axboe }
4125ed29b0b4SJens Axboe
io_register_personality(struct io_ring_ctx * ctx)4126ed29b0b4SJens Axboe static int io_register_personality(struct io_ring_ctx *ctx)
4127ed29b0b4SJens Axboe {
4128ed29b0b4SJens Axboe const struct cred *creds;
4129ed29b0b4SJens Axboe u32 id;
4130ed29b0b4SJens Axboe int ret;
4131ed29b0b4SJens Axboe
4132ed29b0b4SJens Axboe creds = get_current_cred();
4133ed29b0b4SJens Axboe
4134ed29b0b4SJens Axboe ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
4135ed29b0b4SJens Axboe XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
4136ed29b0b4SJens Axboe if (ret < 0) {
4137ed29b0b4SJens Axboe put_cred(creds);
4138ed29b0b4SJens Axboe return ret;
4139ed29b0b4SJens Axboe }
4140ed29b0b4SJens Axboe return id;
4141ed29b0b4SJens Axboe }
4142ed29b0b4SJens Axboe
io_register_restrictions(struct io_ring_ctx * ctx,void __user * arg,unsigned int nr_args)4143ed29b0b4SJens Axboe static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
4144ed29b0b4SJens Axboe void __user *arg, unsigned int nr_args)
4145ed29b0b4SJens Axboe {
4146ed29b0b4SJens Axboe struct io_uring_restriction *res;
4147ed29b0b4SJens Axboe size_t size;
4148ed29b0b4SJens Axboe int i, ret;
4149ed29b0b4SJens Axboe
4150ed29b0b4SJens Axboe /* Restrictions allowed only if rings started disabled */
4151ed29b0b4SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED))
4152ed29b0b4SJens Axboe return -EBADFD;
4153ed29b0b4SJens Axboe
4154ed29b0b4SJens Axboe /* We allow only a single restrictions registration */
4155ed29b0b4SJens Axboe if (ctx->restrictions.registered)
4156ed29b0b4SJens Axboe return -EBUSY;
4157ed29b0b4SJens Axboe
4158ed29b0b4SJens Axboe if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
4159ed29b0b4SJens Axboe return -EINVAL;
4160ed29b0b4SJens Axboe
4161ed29b0b4SJens Axboe size = array_size(nr_args, sizeof(*res));
4162ed29b0b4SJens Axboe if (size == SIZE_MAX)
4163ed29b0b4SJens Axboe return -EOVERFLOW;
4164ed29b0b4SJens Axboe
4165ed29b0b4SJens Axboe res = memdup_user(arg, size);
4166ed29b0b4SJens Axboe if (IS_ERR(res))
4167ed29b0b4SJens Axboe return PTR_ERR(res);
4168ed29b0b4SJens Axboe
4169ed29b0b4SJens Axboe ret = 0;
4170ed29b0b4SJens Axboe
4171ed29b0b4SJens Axboe for (i = 0; i < nr_args; i++) {
4172ed29b0b4SJens Axboe switch (res[i].opcode) {
4173ed29b0b4SJens Axboe case IORING_RESTRICTION_REGISTER_OP:
4174ed29b0b4SJens Axboe if (res[i].register_op >= IORING_REGISTER_LAST) {
4175ed29b0b4SJens Axboe ret = -EINVAL;
4176ed29b0b4SJens Axboe goto out;
4177ed29b0b4SJens Axboe }
4178ed29b0b4SJens Axboe
4179ed29b0b4SJens Axboe __set_bit(res[i].register_op,
4180ed29b0b4SJens Axboe ctx->restrictions.register_op);
4181ed29b0b4SJens Axboe break;
4182ed29b0b4SJens Axboe case IORING_RESTRICTION_SQE_OP:
4183ed29b0b4SJens Axboe if (res[i].sqe_op >= IORING_OP_LAST) {
4184ed29b0b4SJens Axboe ret = -EINVAL;
4185ed29b0b4SJens Axboe goto out;
4186ed29b0b4SJens Axboe }
4187ed29b0b4SJens Axboe
4188ed29b0b4SJens Axboe __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
4189ed29b0b4SJens Axboe break;
4190ed29b0b4SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
4191ed29b0b4SJens Axboe ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
4192ed29b0b4SJens Axboe break;
4193ed29b0b4SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
4194ed29b0b4SJens Axboe ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
4195ed29b0b4SJens Axboe break;
4196ed29b0b4SJens Axboe default:
4197ed29b0b4SJens Axboe ret = -EINVAL;
4198ed29b0b4SJens Axboe goto out;
4199ed29b0b4SJens Axboe }
4200ed29b0b4SJens Axboe }
4201ed29b0b4SJens Axboe
4202ed29b0b4SJens Axboe out:
4203ed29b0b4SJens Axboe /* Reset all restrictions if an error happened */
4204ed29b0b4SJens Axboe if (ret != 0)
4205ed29b0b4SJens Axboe memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
4206ed29b0b4SJens Axboe else
4207ed29b0b4SJens Axboe ctx->restrictions.registered = true;
4208ed29b0b4SJens Axboe
4209ed29b0b4SJens Axboe kfree(res);
4210ed29b0b4SJens Axboe return ret;
4211ed29b0b4SJens Axboe }
4212ed29b0b4SJens Axboe
io_register_enable_rings(struct io_ring_ctx * ctx)4213ed29b0b4SJens Axboe static int io_register_enable_rings(struct io_ring_ctx *ctx)
4214ed29b0b4SJens Axboe {
4215ed29b0b4SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED))
4216ed29b0b4SJens Axboe return -EBADFD;
4217ed29b0b4SJens Axboe
4218bca39f39SPavel Begunkov if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {
42198579538cSPavel Begunkov WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
4220bca39f39SPavel Begunkov /*
4221bca39f39SPavel Begunkov * Lazy activation attempts would fail if it was polled before
4222bca39f39SPavel Begunkov * submitter_task is set.
4223bca39f39SPavel Begunkov */
4224bca39f39SPavel Begunkov if (wq_has_sleeper(&ctx->poll_wq))
4225bca39f39SPavel Begunkov io_activate_pollwq(ctx);
4226bca39f39SPavel Begunkov }
42277cae596bSDylan Yudaken
4228ed29b0b4SJens Axboe if (ctx->restrictions.registered)
4229ed29b0b4SJens Axboe ctx->restricted = 1;
4230ed29b0b4SJens Axboe
4231ed29b0b4SJens Axboe ctx->flags &= ~IORING_SETUP_R_DISABLED;
4232ed29b0b4SJens Axboe if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
4233ed29b0b4SJens Axboe wake_up(&ctx->sq_data->wait);
4234ed29b0b4SJens Axboe return 0;
4235ed29b0b4SJens Axboe }
4236ed29b0b4SJens Axboe
__io_register_iowq_aff(struct io_ring_ctx * ctx,cpumask_var_t new_mask)4237ebdfefc0SJens Axboe static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,
4238ebdfefc0SJens Axboe cpumask_var_t new_mask)
4239ebdfefc0SJens Axboe {
4240ebdfefc0SJens Axboe int ret;
4241ebdfefc0SJens Axboe
4242ebdfefc0SJens Axboe if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
4243ebdfefc0SJens Axboe ret = io_wq_cpu_affinity(current->io_uring, new_mask);
4244ebdfefc0SJens Axboe } else {
4245ebdfefc0SJens Axboe mutex_unlock(&ctx->uring_lock);
4246ebdfefc0SJens Axboe ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
4247ebdfefc0SJens Axboe mutex_lock(&ctx->uring_lock);
4248ebdfefc0SJens Axboe }
4249ebdfefc0SJens Axboe
4250ebdfefc0SJens Axboe return ret;
4251ebdfefc0SJens Axboe }
4252ebdfefc0SJens Axboe
io_register_iowq_aff(struct io_ring_ctx * ctx,void __user * arg,unsigned len)4253ed29b0b4SJens Axboe static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
4254ed29b0b4SJens Axboe void __user *arg, unsigned len)
4255ed29b0b4SJens Axboe {
4256ed29b0b4SJens Axboe cpumask_var_t new_mask;
4257ed29b0b4SJens Axboe int ret;
4258ed29b0b4SJens Axboe
4259ed29b0b4SJens Axboe if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
4260ed29b0b4SJens Axboe return -ENOMEM;
4261ed29b0b4SJens Axboe
4262ed29b0b4SJens Axboe cpumask_clear(new_mask);
4263ed29b0b4SJens Axboe if (len > cpumask_size())
4264ed29b0b4SJens Axboe len = cpumask_size();
4265ed29b0b4SJens Axboe
4266ed29b0b4SJens Axboe if (in_compat_syscall()) {
4267ed29b0b4SJens Axboe ret = compat_get_bitmap(cpumask_bits(new_mask),
4268ed29b0b4SJens Axboe (const compat_ulong_t __user *)arg,
4269ed29b0b4SJens Axboe len * 8 /* CHAR_BIT */);
4270ed29b0b4SJens Axboe } else {
4271ed29b0b4SJens Axboe ret = copy_from_user(new_mask, arg, len);
4272ed29b0b4SJens Axboe }
4273ed29b0b4SJens Axboe
4274ed29b0b4SJens Axboe if (ret) {
4275ed29b0b4SJens Axboe free_cpumask_var(new_mask);
4276ed29b0b4SJens Axboe return -EFAULT;
4277ed29b0b4SJens Axboe }
4278ed29b0b4SJens Axboe
4279ebdfefc0SJens Axboe ret = __io_register_iowq_aff(ctx, new_mask);
4280ed29b0b4SJens Axboe free_cpumask_var(new_mask);
4281ed29b0b4SJens Axboe return ret;
4282ed29b0b4SJens Axboe }
4283ed29b0b4SJens Axboe
io_unregister_iowq_aff(struct io_ring_ctx * ctx)4284ed29b0b4SJens Axboe static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
4285ed29b0b4SJens Axboe {
4286ebdfefc0SJens Axboe return __io_register_iowq_aff(ctx, NULL);
4287ed29b0b4SJens Axboe }
4288ed29b0b4SJens Axboe
io_register_iowq_max_workers(struct io_ring_ctx * ctx,void __user * arg)4289ed29b0b4SJens Axboe static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
4290ed29b0b4SJens Axboe void __user *arg)
4291ed29b0b4SJens Axboe __must_hold(&ctx->uring_lock)
4292ed29b0b4SJens Axboe {
4293ed29b0b4SJens Axboe struct io_tctx_node *node;
4294ed29b0b4SJens Axboe struct io_uring_task *tctx = NULL;
4295ed29b0b4SJens Axboe struct io_sq_data *sqd = NULL;
4296ed29b0b4SJens Axboe __u32 new_count[2];
4297ed29b0b4SJens Axboe int i, ret;
4298ed29b0b4SJens Axboe
4299ed29b0b4SJens Axboe if (copy_from_user(new_count, arg, sizeof(new_count)))
4300ed29b0b4SJens Axboe return -EFAULT;
4301ed29b0b4SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
4302ed29b0b4SJens Axboe if (new_count[i] > INT_MAX)
4303ed29b0b4SJens Axboe return -EINVAL;
4304ed29b0b4SJens Axboe
4305ed29b0b4SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) {
4306ed29b0b4SJens Axboe sqd = ctx->sq_data;
4307ed29b0b4SJens Axboe if (sqd) {
4308ed29b0b4SJens Axboe /*
4309ed29b0b4SJens Axboe * Observe the correct sqd->lock -> ctx->uring_lock
4310ed29b0b4SJens Axboe * ordering. Fine to drop uring_lock here, we hold
4311ed29b0b4SJens Axboe * a ref to the ctx.
4312ed29b0b4SJens Axboe */
4313ed29b0b4SJens Axboe refcount_inc(&sqd->refs);
4314ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
4315ed29b0b4SJens Axboe mutex_lock(&sqd->lock);
4316ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
4317ed29b0b4SJens Axboe if (sqd->thread)
4318ed29b0b4SJens Axboe tctx = sqd->thread->io_uring;
4319ed29b0b4SJens Axboe }
4320ed29b0b4SJens Axboe } else {
4321ed29b0b4SJens Axboe tctx = current->io_uring;
4322ed29b0b4SJens Axboe }
4323ed29b0b4SJens Axboe
4324ed29b0b4SJens Axboe BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
4325ed29b0b4SJens Axboe
4326ed29b0b4SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
4327ed29b0b4SJens Axboe if (new_count[i])
4328ed29b0b4SJens Axboe ctx->iowq_limits[i] = new_count[i];
4329ed29b0b4SJens Axboe ctx->iowq_limits_set = true;
4330ed29b0b4SJens Axboe
4331ed29b0b4SJens Axboe if (tctx && tctx->io_wq) {
4332ed29b0b4SJens Axboe ret = io_wq_max_workers(tctx->io_wq, new_count);
4333ed29b0b4SJens Axboe if (ret)
4334ed29b0b4SJens Axboe goto err;
4335ed29b0b4SJens Axboe } else {
4336ed29b0b4SJens Axboe memset(new_count, 0, sizeof(new_count));
4337ed29b0b4SJens Axboe }
4338ed29b0b4SJens Axboe
4339ed29b0b4SJens Axboe if (sqd) {
4340ed29b0b4SJens Axboe mutex_unlock(&sqd->lock);
4341ed29b0b4SJens Axboe io_put_sq_data(sqd);
4342ed29b0b4SJens Axboe }
4343ed29b0b4SJens Axboe
4344ed29b0b4SJens Axboe if (copy_to_user(arg, new_count, sizeof(new_count)))
4345ed29b0b4SJens Axboe return -EFAULT;
4346ed29b0b4SJens Axboe
4347ed29b0b4SJens Axboe /* that's it for SQPOLL, only the SQPOLL task creates requests */
4348ed29b0b4SJens Axboe if (sqd)
4349ed29b0b4SJens Axboe return 0;
4350ed29b0b4SJens Axboe
4351ed29b0b4SJens Axboe /* now propagate the restriction to all registered users */
4352ed29b0b4SJens Axboe list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
4353ed29b0b4SJens Axboe struct io_uring_task *tctx = node->task->io_uring;
4354ed29b0b4SJens Axboe
4355ed29b0b4SJens Axboe if (WARN_ON_ONCE(!tctx->io_wq))
4356ed29b0b4SJens Axboe continue;
4357ed29b0b4SJens Axboe
4358ed29b0b4SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
4359ed29b0b4SJens Axboe new_count[i] = ctx->iowq_limits[i];
4360ed29b0b4SJens Axboe /* ignore errors, it always returns zero anyway */
4361ed29b0b4SJens Axboe (void)io_wq_max_workers(tctx->io_wq, new_count);
4362ed29b0b4SJens Axboe }
4363ed29b0b4SJens Axboe return 0;
4364ed29b0b4SJens Axboe err:
4365ed29b0b4SJens Axboe if (sqd) {
4366ed29b0b4SJens Axboe mutex_unlock(&sqd->lock);
4367ed29b0b4SJens Axboe io_put_sq_data(sqd);
4368ed29b0b4SJens Axboe }
4369ed29b0b4SJens Axboe return ret;
4370ed29b0b4SJens Axboe }
4371ed29b0b4SJens Axboe
__io_uring_register(struct io_ring_ctx * ctx,unsigned opcode,void __user * arg,unsigned nr_args)4372ed29b0b4SJens Axboe static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
4373ed29b0b4SJens Axboe void __user *arg, unsigned nr_args)
4374ed29b0b4SJens Axboe __releases(ctx->uring_lock)
4375ed29b0b4SJens Axboe __acquires(ctx->uring_lock)
4376ed29b0b4SJens Axboe {
4377ed29b0b4SJens Axboe int ret;
4378ed29b0b4SJens Axboe
4379ed29b0b4SJens Axboe /*
4380fbb8bb02SPavel Begunkov * We don't quiesce the refs for register anymore and so it can't be
4381fbb8bb02SPavel Begunkov * dying as we're holding a file ref here.
4382ed29b0b4SJens Axboe */
4383fbb8bb02SPavel Begunkov if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
4384ed29b0b4SJens Axboe return -ENXIO;
4385ed29b0b4SJens Axboe
4386d7cce96cSPavel Begunkov if (ctx->submitter_task && ctx->submitter_task != current)
4387d7cce96cSPavel Begunkov return -EEXIST;
4388d7cce96cSPavel Begunkov
4389ed29b0b4SJens Axboe if (ctx->restricted) {
4390ed29b0b4SJens Axboe opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
4391ed29b0b4SJens Axboe if (!test_bit(opcode, ctx->restrictions.register_op))
4392ed29b0b4SJens Axboe return -EACCES;
4393ed29b0b4SJens Axboe }
4394ed29b0b4SJens Axboe
4395ed29b0b4SJens Axboe switch (opcode) {
4396ed29b0b4SJens Axboe case IORING_REGISTER_BUFFERS:
4397ed29b0b4SJens Axboe ret = -EFAULT;
4398ed29b0b4SJens Axboe if (!arg)
4399ed29b0b4SJens Axboe break;
4400ed29b0b4SJens Axboe ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
4401ed29b0b4SJens Axboe break;
4402ed29b0b4SJens Axboe case IORING_UNREGISTER_BUFFERS:
4403ed29b0b4SJens Axboe ret = -EINVAL;
4404ed29b0b4SJens Axboe if (arg || nr_args)
4405ed29b0b4SJens Axboe break;
4406ed29b0b4SJens Axboe ret = io_sqe_buffers_unregister(ctx);
4407ed29b0b4SJens Axboe break;
4408ed29b0b4SJens Axboe case IORING_REGISTER_FILES:
4409ed29b0b4SJens Axboe ret = -EFAULT;
4410ed29b0b4SJens Axboe if (!arg)
4411ed29b0b4SJens Axboe break;
4412ed29b0b4SJens Axboe ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
4413ed29b0b4SJens Axboe break;
4414ed29b0b4SJens Axboe case IORING_UNREGISTER_FILES:
4415ed29b0b4SJens Axboe ret = -EINVAL;
4416ed29b0b4SJens Axboe if (arg || nr_args)
4417ed29b0b4SJens Axboe break;
4418ed29b0b4SJens Axboe ret = io_sqe_files_unregister(ctx);
4419ed29b0b4SJens Axboe break;
4420ed29b0b4SJens Axboe case IORING_REGISTER_FILES_UPDATE:
4421ed29b0b4SJens Axboe ret = io_register_files_update(ctx, arg, nr_args);
4422ed29b0b4SJens Axboe break;
4423ed29b0b4SJens Axboe case IORING_REGISTER_EVENTFD:
4424ed29b0b4SJens Axboe ret = -EINVAL;
4425ed29b0b4SJens Axboe if (nr_args != 1)
4426ed29b0b4SJens Axboe break;
4427ed29b0b4SJens Axboe ret = io_eventfd_register(ctx, arg, 0);
4428ed29b0b4SJens Axboe break;
4429ed29b0b4SJens Axboe case IORING_REGISTER_EVENTFD_ASYNC:
4430ed29b0b4SJens Axboe ret = -EINVAL;
4431ed29b0b4SJens Axboe if (nr_args != 1)
4432ed29b0b4SJens Axboe break;
4433ed29b0b4SJens Axboe ret = io_eventfd_register(ctx, arg, 1);
4434ed29b0b4SJens Axboe break;
4435ed29b0b4SJens Axboe case IORING_UNREGISTER_EVENTFD:
4436ed29b0b4SJens Axboe ret = -EINVAL;
4437ed29b0b4SJens Axboe if (arg || nr_args)
4438ed29b0b4SJens Axboe break;
4439ed29b0b4SJens Axboe ret = io_eventfd_unregister(ctx);
4440ed29b0b4SJens Axboe break;
4441ed29b0b4SJens Axboe case IORING_REGISTER_PROBE:
4442ed29b0b4SJens Axboe ret = -EINVAL;
4443ed29b0b4SJens Axboe if (!arg || nr_args > 256)
4444ed29b0b4SJens Axboe break;
4445ed29b0b4SJens Axboe ret = io_probe(ctx, arg, nr_args);
4446ed29b0b4SJens Axboe break;
4447ed29b0b4SJens Axboe case IORING_REGISTER_PERSONALITY:
4448ed29b0b4SJens Axboe ret = -EINVAL;
4449ed29b0b4SJens Axboe if (arg || nr_args)
4450ed29b0b4SJens Axboe break;
4451ed29b0b4SJens Axboe ret = io_register_personality(ctx);
4452ed29b0b4SJens Axboe break;
4453ed29b0b4SJens Axboe case IORING_UNREGISTER_PERSONALITY:
4454ed29b0b4SJens Axboe ret = -EINVAL;
4455ed29b0b4SJens Axboe if (arg)
4456ed29b0b4SJens Axboe break;
4457ed29b0b4SJens Axboe ret = io_unregister_personality(ctx, nr_args);
4458ed29b0b4SJens Axboe break;
4459ed29b0b4SJens Axboe case IORING_REGISTER_ENABLE_RINGS:
4460ed29b0b4SJens Axboe ret = -EINVAL;
4461ed29b0b4SJens Axboe if (arg || nr_args)
4462ed29b0b4SJens Axboe break;
4463ed29b0b4SJens Axboe ret = io_register_enable_rings(ctx);
4464ed29b0b4SJens Axboe break;
4465ed29b0b4SJens Axboe case IORING_REGISTER_RESTRICTIONS:
4466ed29b0b4SJens Axboe ret = io_register_restrictions(ctx, arg, nr_args);
4467ed29b0b4SJens Axboe break;
4468ed29b0b4SJens Axboe case IORING_REGISTER_FILES2:
4469ed29b0b4SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
4470ed29b0b4SJens Axboe break;
4471ed29b0b4SJens Axboe case IORING_REGISTER_FILES_UPDATE2:
4472ed29b0b4SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args,
4473ed29b0b4SJens Axboe IORING_RSRC_FILE);
4474ed29b0b4SJens Axboe break;
4475ed29b0b4SJens Axboe case IORING_REGISTER_BUFFERS2:
4476ed29b0b4SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
4477ed29b0b4SJens Axboe break;
4478ed29b0b4SJens Axboe case IORING_REGISTER_BUFFERS_UPDATE:
4479ed29b0b4SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args,
4480ed29b0b4SJens Axboe IORING_RSRC_BUFFER);
4481ed29b0b4SJens Axboe break;
4482ed29b0b4SJens Axboe case IORING_REGISTER_IOWQ_AFF:
4483ed29b0b4SJens Axboe ret = -EINVAL;
4484ed29b0b4SJens Axboe if (!arg || !nr_args)
4485ed29b0b4SJens Axboe break;
4486ed29b0b4SJens Axboe ret = io_register_iowq_aff(ctx, arg, nr_args);
4487ed29b0b4SJens Axboe break;
4488ed29b0b4SJens Axboe case IORING_UNREGISTER_IOWQ_AFF:
4489ed29b0b4SJens Axboe ret = -EINVAL;
4490ed29b0b4SJens Axboe if (arg || nr_args)
4491ed29b0b4SJens Axboe break;
4492ed29b0b4SJens Axboe ret = io_unregister_iowq_aff(ctx);
4493ed29b0b4SJens Axboe break;
4494ed29b0b4SJens Axboe case IORING_REGISTER_IOWQ_MAX_WORKERS:
4495ed29b0b4SJens Axboe ret = -EINVAL;
4496ed29b0b4SJens Axboe if (!arg || nr_args != 2)
4497ed29b0b4SJens Axboe break;
4498ed29b0b4SJens Axboe ret = io_register_iowq_max_workers(ctx, arg);
4499ed29b0b4SJens Axboe break;
4500ed29b0b4SJens Axboe case IORING_REGISTER_RING_FDS:
4501ed29b0b4SJens Axboe ret = io_ringfd_register(ctx, arg, nr_args);
4502ed29b0b4SJens Axboe break;
4503ed29b0b4SJens Axboe case IORING_UNREGISTER_RING_FDS:
4504ed29b0b4SJens Axboe ret = io_ringfd_unregister(ctx, arg, nr_args);
4505ed29b0b4SJens Axboe break;
4506ed29b0b4SJens Axboe case IORING_REGISTER_PBUF_RING:
4507ed29b0b4SJens Axboe ret = -EINVAL;
4508ed29b0b4SJens Axboe if (!arg || nr_args != 1)
4509ed29b0b4SJens Axboe break;
4510ed29b0b4SJens Axboe ret = io_register_pbuf_ring(ctx, arg);
4511ed29b0b4SJens Axboe break;
4512ed29b0b4SJens Axboe case IORING_UNREGISTER_PBUF_RING:
4513ed29b0b4SJens Axboe ret = -EINVAL;
4514ed29b0b4SJens Axboe if (!arg || nr_args != 1)
4515ed29b0b4SJens Axboe break;
4516ed29b0b4SJens Axboe ret = io_unregister_pbuf_ring(ctx, arg);
4517ed29b0b4SJens Axboe break;
451878a861b9SJens Axboe case IORING_REGISTER_SYNC_CANCEL:
451978a861b9SJens Axboe ret = -EINVAL;
452078a861b9SJens Axboe if (!arg || nr_args != 1)
452178a861b9SJens Axboe break;
452278a861b9SJens Axboe ret = io_sync_cancel(ctx, arg);
452378a861b9SJens Axboe break;
45246e73dffbSPavel Begunkov case IORING_REGISTER_FILE_ALLOC_RANGE:
45256e73dffbSPavel Begunkov ret = -EINVAL;
45266e73dffbSPavel Begunkov if (!arg || nr_args)
45276e73dffbSPavel Begunkov break;
45286e73dffbSPavel Begunkov ret = io_register_file_alloc_range(ctx, arg);
45296e73dffbSPavel Begunkov break;
4530ed29b0b4SJens Axboe default:
4531ed29b0b4SJens Axboe ret = -EINVAL;
4532ed29b0b4SJens Axboe break;
4533ed29b0b4SJens Axboe }
4534ed29b0b4SJens Axboe
4535ed29b0b4SJens Axboe return ret;
4536ed29b0b4SJens Axboe }
4537ed29b0b4SJens Axboe
SYSCALL_DEFINE4(io_uring_register,unsigned int,fd,unsigned int,opcode,void __user *,arg,unsigned int,nr_args)4538ed29b0b4SJens Axboe SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
4539ed29b0b4SJens Axboe void __user *, arg, unsigned int, nr_args)
4540ed29b0b4SJens Axboe {
4541ed29b0b4SJens Axboe struct io_ring_ctx *ctx;
4542ed29b0b4SJens Axboe long ret = -EBADF;
45430c7df8c2SJens Axboe struct file *file;
45447d3fd88dSJosh Triplett bool use_registered_ring;
45457d3fd88dSJosh Triplett
45467d3fd88dSJosh Triplett use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);
45477d3fd88dSJosh Triplett opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
4548ed29b0b4SJens Axboe
454934319084SJens Axboe if (opcode >= IORING_REGISTER_LAST)
455034319084SJens Axboe return -EINVAL;
455134319084SJens Axboe
45527d3fd88dSJosh Triplett if (use_registered_ring) {
45537d3fd88dSJosh Triplett /*
45547d3fd88dSJosh Triplett * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
45557d3fd88dSJosh Triplett * need only dereference our task private array to find it.
45567d3fd88dSJosh Triplett */
45577d3fd88dSJosh Triplett struct io_uring_task *tctx = current->io_uring;
4558ed29b0b4SJens Axboe
45597d3fd88dSJosh Triplett if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
45607d3fd88dSJosh Triplett return -EINVAL;
45617d3fd88dSJosh Triplett fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
45620c7df8c2SJens Axboe file = tctx->registered_rings[fd];
45630c7df8c2SJens Axboe if (unlikely(!file))
45647d3fd88dSJosh Triplett return -EBADF;
45657d3fd88dSJosh Triplett } else {
45660c7df8c2SJens Axboe file = fget(fd);
45670c7df8c2SJens Axboe if (unlikely(!file))
45687d3fd88dSJosh Triplett return -EBADF;
4569ed29b0b4SJens Axboe ret = -EOPNOTSUPP;
45700c7df8c2SJens Axboe if (!io_is_uring_fops(file))
4571ed29b0b4SJens Axboe goto out_fput;
45727d3fd88dSJosh Triplett }
4573ed29b0b4SJens Axboe
45740c7df8c2SJens Axboe ctx = file->private_data;
4575ed29b0b4SJens Axboe
4576ed29b0b4SJens Axboe mutex_lock(&ctx->uring_lock);
4577ed29b0b4SJens Axboe ret = __io_uring_register(ctx, opcode, arg, nr_args);
4578ed29b0b4SJens Axboe mutex_unlock(&ctx->uring_lock);
4579ed29b0b4SJens Axboe trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
4580ed29b0b4SJens Axboe out_fput:
45810c7df8c2SJens Axboe if (!use_registered_ring)
45820c7df8c2SJens Axboe fput(file);
4583ed29b0b4SJens Axboe return ret;
4584ed29b0b4SJens Axboe }
4585ed29b0b4SJens Axboe
io_uring_init(void)4586ed29b0b4SJens Axboe static int __init io_uring_init(void)
4587ed29b0b4SJens Axboe {
45889c71d39aSStefan Metzmacher #define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \
4589ed29b0b4SJens Axboe BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
45909c71d39aSStefan Metzmacher BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \
4591ed29b0b4SJens Axboe } while (0)
4592ed29b0b4SJens Axboe
4593ed29b0b4SJens Axboe #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
45949c71d39aSStefan Metzmacher __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, sizeof(etype), ename)
45959c71d39aSStefan Metzmacher #define BUILD_BUG_SQE_ELEM_SIZE(eoffset, esize, ename) \
45969c71d39aSStefan Metzmacher __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, esize, ename)
4597ed29b0b4SJens Axboe BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
4598ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(0, __u8, opcode);
4599ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(1, __u8, flags);
4600ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(2, __u16, ioprio);
4601ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(4, __s32, fd);
4602ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(8, __u64, off);
4603ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(8, __u64, addr2);
46049c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(8, __u32, cmd_op);
46059c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(12, __u32, __pad1);
4606ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(16, __u64, addr);
4607ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in);
4608ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(24, __u32, len);
4609ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __kernel_rwf_t, rw_flags);
4610ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, /* compat */ int, rw_flags);
4611ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
4612ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, fsync_flags);
4613ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, /* compat */ __u16, poll_events);
4614ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, poll32_events);
4615ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, sync_range_flags);
4616ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, msg_flags);
4617ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, timeout_flags);
4618ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, accept_flags);
4619ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, cancel_flags);
4620ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, open_flags);
4621ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, statx_flags);
4622ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice);
4623ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(28, __u32, splice_flags);
46249c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(28, __u32, rename_flags);
46259c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(28, __u32, unlink_flags);
46269c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(28, __u32, hardlink_flags);
46279c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(28, __u32, xattr_flags);
46289c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(28, __u32, msg_ring_flags);
4629ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(32, __u64, user_data);
4630ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(40, __u16, buf_index);
4631ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(40, __u16, buf_group);
4632ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(42, __u16, personality);
4633ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in);
4634ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(44, __u32, file_index);
4635b48c312bSPavel Begunkov BUILD_BUG_SQE_ELEM(44, __u16, addr_len);
4636b48c312bSPavel Begunkov BUILD_BUG_SQE_ELEM(46, __u16, __pad3[0]);
4637ed29b0b4SJens Axboe BUILD_BUG_SQE_ELEM(48, __u64, addr3);
46389c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd);
46399c71d39aSStefan Metzmacher BUILD_BUG_SQE_ELEM(56, __u64, __pad2);
4640ed29b0b4SJens Axboe
4641ed29b0b4SJens Axboe BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
4642ed29b0b4SJens Axboe sizeof(struct io_uring_rsrc_update));
4643ed29b0b4SJens Axboe BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
4644ed29b0b4SJens Axboe sizeof(struct io_uring_rsrc_update2));
4645ed29b0b4SJens Axboe
4646ed29b0b4SJens Axboe /* ->buf_index is u16 */
4647ed29b0b4SJens Axboe BUILD_BUG_ON(offsetof(struct io_uring_buf_ring, bufs) != 0);
4648ed29b0b4SJens Axboe BUILD_BUG_ON(offsetof(struct io_uring_buf, resv) !=
4649ed29b0b4SJens Axboe offsetof(struct io_uring_buf_ring, tail));
4650ed29b0b4SJens Axboe
4651ed29b0b4SJens Axboe /* should fit into one byte */
4652ed29b0b4SJens Axboe BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));
4653ed29b0b4SJens Axboe BUILD_BUG_ON(SQE_COMMON_FLAGS >= (1 << 8));
4654ed29b0b4SJens Axboe BUILD_BUG_ON((SQE_VALID_FLAGS | SQE_COMMON_FLAGS) != SQE_VALID_FLAGS);
4655ed29b0b4SJens Axboe
4656ed29b0b4SJens Axboe BUILD_BUG_ON(__REQ_F_LAST_BIT > 8 * sizeof(int));
4657ed29b0b4SJens Axboe
4658ed29b0b4SJens Axboe BUILD_BUG_ON(sizeof(atomic_t) != sizeof(u32));
4659ed29b0b4SJens Axboe
4660d9b57aa3SJens Axboe io_uring_optable_init();
4661ed29b0b4SJens Axboe
4662b97f96e2SJens Axboe /*
4663b97f96e2SJens Axboe * Allow user copy in the per-command field, which starts after the
4664b97f96e2SJens Axboe * file in io_kiocb and until the opcode field. The openat2 handling
4665b97f96e2SJens Axboe * requires copying in user memory into the io_kiocb object in that
4666b97f96e2SJens Axboe * range, and HARDENED_USERCOPY will complain if we haven't
4667b97f96e2SJens Axboe * correctly annotated this range.
4668b97f96e2SJens Axboe */
4669b97f96e2SJens Axboe req_cachep = kmem_cache_create_usercopy("io_kiocb",
4670b97f96e2SJens Axboe sizeof(struct io_kiocb), 0,
4671b97f96e2SJens Axboe SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4672b97f96e2SJens Axboe SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU,
4673b97f96e2SJens Axboe offsetof(struct io_kiocb, cmd.data),
4674b97f96e2SJens Axboe sizeof_field(struct io_kiocb, cmd.data), NULL);
4675b97f96e2SJens Axboe
46766b9d49bcSJens Axboe iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
46776b9d49bcSJens Axboe
467876d3ccecSMatteo Rizzo #ifdef CONFIG_SYSCTL
467976d3ccecSMatteo Rizzo register_sysctl_init("kernel", kernel_io_uring_disabled_table);
468076d3ccecSMatteo Rizzo #endif
468176d3ccecSMatteo Rizzo
4682ed29b0b4SJens Axboe return 0;
4683ed29b0b4SJens Axboe };
4684ed29b0b4SJens Axboe __initcall(io_uring_init);
4685