xref: /openbmc/linux/io_uring/poll.c (revision ecfb9f40)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/fs.h>
5 #include <linux/file.h>
6 #include <linux/mm.h>
7 #include <linux/slab.h>
8 #include <linux/poll.h>
9 #include <linux/hashtable.h>
10 #include <linux/io_uring.h>
11 
12 #include <trace/events/io_uring.h>
13 
14 #include <uapi/linux/io_uring.h>
15 
16 #include "io_uring.h"
17 #include "refs.h"
18 #include "opdef.h"
19 #include "kbuf.h"
20 #include "poll.h"
21 #include "cancel.h"
22 
23 struct io_poll_update {
24 	struct file			*file;
25 	u64				old_user_data;
26 	u64				new_user_data;
27 	__poll_t			events;
28 	bool				update_events;
29 	bool				update_user_data;
30 };
31 
32 struct io_poll_table {
33 	struct poll_table_struct pt;
34 	struct io_kiocb *req;
35 	int nr_entries;
36 	int error;
37 	bool owning;
38 	/* output value, set only if arm poll returns >0 */
39 	__poll_t result_mask;
40 };
41 
42 #define IO_POLL_CANCEL_FLAG	BIT(31)
43 #define IO_POLL_RETRY_FLAG	BIT(30)
44 #define IO_POLL_REF_MASK	GENMASK(29, 0)
45 
46 /*
47  * We usually have 1-2 refs taken, 128 is more than enough and we want to
48  * maximise the margin between this amount and the moment when it overflows.
49  */
50 #define IO_POLL_REF_BIAS	128
51 
52 #define IO_WQE_F_DOUBLE		1
53 
54 static inline struct io_kiocb *wqe_to_req(struct wait_queue_entry *wqe)
55 {
56 	unsigned long priv = (unsigned long)wqe->private;
57 
58 	return (struct io_kiocb *)(priv & ~IO_WQE_F_DOUBLE);
59 }
60 
61 static inline bool wqe_is_double(struct wait_queue_entry *wqe)
62 {
63 	unsigned long priv = (unsigned long)wqe->private;
64 
65 	return priv & IO_WQE_F_DOUBLE;
66 }
67 
68 static bool io_poll_get_ownership_slowpath(struct io_kiocb *req)
69 {
70 	int v;
71 
72 	/*
73 	 * poll_refs are already elevated and we don't have much hope for
74 	 * grabbing the ownership. Instead of incrementing set a retry flag
75 	 * to notify the loop that there might have been some change.
76 	 */
77 	v = atomic_fetch_or(IO_POLL_RETRY_FLAG, &req->poll_refs);
78 	if (v & IO_POLL_REF_MASK)
79 		return false;
80 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
81 }
82 
83 /*
84  * If refs part of ->poll_refs (see IO_POLL_REF_MASK) is 0, it's free. We can
85  * bump it and acquire ownership. It's disallowed to modify requests while not
86  * owning it, that prevents from races for enqueueing task_work's and b/w
87  * arming poll and wakeups.
88  */
89 static inline bool io_poll_get_ownership(struct io_kiocb *req)
90 {
91 	if (unlikely(atomic_read(&req->poll_refs) >= IO_POLL_REF_BIAS))
92 		return io_poll_get_ownership_slowpath(req);
93 	return !(atomic_fetch_inc(&req->poll_refs) & IO_POLL_REF_MASK);
94 }
95 
96 static void io_poll_mark_cancelled(struct io_kiocb *req)
97 {
98 	atomic_or(IO_POLL_CANCEL_FLAG, &req->poll_refs);
99 }
100 
101 static struct io_poll *io_poll_get_double(struct io_kiocb *req)
102 {
103 	/* pure poll stashes this in ->async_data, poll driven retry elsewhere */
104 	if (req->opcode == IORING_OP_POLL_ADD)
105 		return req->async_data;
106 	return req->apoll->double_poll;
107 }
108 
109 static struct io_poll *io_poll_get_single(struct io_kiocb *req)
110 {
111 	if (req->opcode == IORING_OP_POLL_ADD)
112 		return io_kiocb_to_cmd(req, struct io_poll);
113 	return &req->apoll->poll;
114 }
115 
116 static void io_poll_req_insert(struct io_kiocb *req)
117 {
118 	struct io_hash_table *table = &req->ctx->cancel_table;
119 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);
120 	struct io_hash_bucket *hb = &table->hbs[index];
121 
122 	spin_lock(&hb->lock);
123 	hlist_add_head(&req->hash_node, &hb->list);
124 	spin_unlock(&hb->lock);
125 }
126 
127 static void io_poll_req_delete(struct io_kiocb *req, struct io_ring_ctx *ctx)
128 {
129 	struct io_hash_table *table = &req->ctx->cancel_table;
130 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);
131 	spinlock_t *lock = &table->hbs[index].lock;
132 
133 	spin_lock(lock);
134 	hash_del(&req->hash_node);
135 	spin_unlock(lock);
136 }
137 
138 static void io_poll_req_insert_locked(struct io_kiocb *req)
139 {
140 	struct io_hash_table *table = &req->ctx->cancel_table_locked;
141 	u32 index = hash_long(req->cqe.user_data, table->hash_bits);
142 
143 	lockdep_assert_held(&req->ctx->uring_lock);
144 
145 	hlist_add_head(&req->hash_node, &table->hbs[index].list);
146 }
147 
148 static void io_poll_tw_hash_eject(struct io_kiocb *req, bool *locked)
149 {
150 	struct io_ring_ctx *ctx = req->ctx;
151 
152 	if (req->flags & REQ_F_HASH_LOCKED) {
153 		/*
154 		 * ->cancel_table_locked is protected by ->uring_lock in
155 		 * contrast to per bucket spinlocks. Likely, tctx_task_work()
156 		 * already grabbed the mutex for us, but there is a chance it
157 		 * failed.
158 		 */
159 		io_tw_lock(ctx, locked);
160 		hash_del(&req->hash_node);
161 		req->flags &= ~REQ_F_HASH_LOCKED;
162 	} else {
163 		io_poll_req_delete(req, ctx);
164 	}
165 }
166 
167 static void io_init_poll_iocb(struct io_poll *poll, __poll_t events,
168 			      wait_queue_func_t wake_func)
169 {
170 	poll->head = NULL;
171 #define IO_POLL_UNMASK	(EPOLLERR|EPOLLHUP|EPOLLNVAL|EPOLLRDHUP)
172 	/* mask in events that we always want/need */
173 	poll->events = events | IO_POLL_UNMASK;
174 	INIT_LIST_HEAD(&poll->wait.entry);
175 	init_waitqueue_func_entry(&poll->wait, wake_func);
176 }
177 
178 static inline void io_poll_remove_entry(struct io_poll *poll)
179 {
180 	struct wait_queue_head *head = smp_load_acquire(&poll->head);
181 
182 	if (head) {
183 		spin_lock_irq(&head->lock);
184 		list_del_init(&poll->wait.entry);
185 		poll->head = NULL;
186 		spin_unlock_irq(&head->lock);
187 	}
188 }
189 
190 static void io_poll_remove_entries(struct io_kiocb *req)
191 {
192 	/*
193 	 * Nothing to do if neither of those flags are set. Avoid dipping
194 	 * into the poll/apoll/double cachelines if we can.
195 	 */
196 	if (!(req->flags & (REQ_F_SINGLE_POLL | REQ_F_DOUBLE_POLL)))
197 		return;
198 
199 	/*
200 	 * While we hold the waitqueue lock and the waitqueue is nonempty,
201 	 * wake_up_pollfree() will wait for us.  However, taking the waitqueue
202 	 * lock in the first place can race with the waitqueue being freed.
203 	 *
204 	 * We solve this as eventpoll does: by taking advantage of the fact that
205 	 * all users of wake_up_pollfree() will RCU-delay the actual free.  If
206 	 * we enter rcu_read_lock() and see that the pointer to the queue is
207 	 * non-NULL, we can then lock it without the memory being freed out from
208 	 * under us.
209 	 *
210 	 * Keep holding rcu_read_lock() as long as we hold the queue lock, in
211 	 * case the caller deletes the entry from the queue, leaving it empty.
212 	 * In that case, only RCU prevents the queue memory from being freed.
213 	 */
214 	rcu_read_lock();
215 	if (req->flags & REQ_F_SINGLE_POLL)
216 		io_poll_remove_entry(io_poll_get_single(req));
217 	if (req->flags & REQ_F_DOUBLE_POLL)
218 		io_poll_remove_entry(io_poll_get_double(req));
219 	rcu_read_unlock();
220 }
221 
222 enum {
223 	IOU_POLL_DONE = 0,
224 	IOU_POLL_NO_ACTION = 1,
225 	IOU_POLL_REMOVE_POLL_USE_RES = 2,
226 };
227 
228 /*
229  * All poll tw should go through this. Checks for poll events, manages
230  * references, does rewait, etc.
231  *
232  * Returns a negative error on failure. IOU_POLL_NO_ACTION when no action require,
233  * which is either spurious wakeup or multishot CQE is served.
234  * IOU_POLL_DONE when it's done with the request, then the mask is stored in req->cqe.res.
235  * IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot poll and that the result
236  * is stored in req->cqe.
237  */
238 static int io_poll_check_events(struct io_kiocb *req, bool *locked)
239 {
240 	int v, ret;
241 
242 	/* req->task == current here, checking PF_EXITING is safe */
243 	if (unlikely(req->task->flags & PF_EXITING))
244 		return -ECANCELED;
245 
246 	do {
247 		v = atomic_read(&req->poll_refs);
248 
249 		if (unlikely(v != 1)) {
250 			/* tw should be the owner and so have some refs */
251 			if (WARN_ON_ONCE(!(v & IO_POLL_REF_MASK)))
252 				return IOU_POLL_NO_ACTION;
253 			if (v & IO_POLL_CANCEL_FLAG)
254 				return -ECANCELED;
255 			/*
256 			 * cqe.res contains only events of the first wake up
257 			 * and all others are to be lost. Redo vfs_poll() to get
258 			 * up to date state.
259 			 */
260 			if ((v & IO_POLL_REF_MASK) != 1)
261 				req->cqe.res = 0;
262 
263 			if (v & IO_POLL_RETRY_FLAG) {
264 				req->cqe.res = 0;
265 				/*
266 				 * We won't find new events that came in between
267 				 * vfs_poll and the ref put unless we clear the
268 				 * flag in advance.
269 				 */
270 				atomic_andnot(IO_POLL_RETRY_FLAG, &req->poll_refs);
271 				v &= ~IO_POLL_RETRY_FLAG;
272 			}
273 		}
274 
275 		/* the mask was stashed in __io_poll_execute */
276 		if (!req->cqe.res) {
277 			struct poll_table_struct pt = { ._key = req->apoll_events };
278 			req->cqe.res = vfs_poll(req->file, &pt) & req->apoll_events;
279 		}
280 
281 		if ((unlikely(!req->cqe.res)))
282 			continue;
283 		if (req->apoll_events & EPOLLONESHOT)
284 			return IOU_POLL_DONE;
285 
286 		/* multishot, just fill a CQE and proceed */
287 		if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
288 			__poll_t mask = mangle_poll(req->cqe.res &
289 						    req->apoll_events);
290 
291 			if (!io_aux_cqe(req->ctx, *locked, req->cqe.user_data,
292 					mask, IORING_CQE_F_MORE, false)) {
293 				io_req_set_res(req, mask, 0);
294 				return IOU_POLL_REMOVE_POLL_USE_RES;
295 			}
296 		} else {
297 			ret = io_poll_issue(req, locked);
298 			if (ret == IOU_STOP_MULTISHOT)
299 				return IOU_POLL_REMOVE_POLL_USE_RES;
300 			if (ret < 0)
301 				return ret;
302 		}
303 
304 		/* force the next iteration to vfs_poll() */
305 		req->cqe.res = 0;
306 
307 		/*
308 		 * Release all references, retry if someone tried to restart
309 		 * task_work while we were executing it.
310 		 */
311 	} while (atomic_sub_return(v & IO_POLL_REF_MASK, &req->poll_refs) &
312 					IO_POLL_REF_MASK);
313 
314 	return IOU_POLL_NO_ACTION;
315 }
316 
317 static void io_poll_task_func(struct io_kiocb *req, bool *locked)
318 {
319 	int ret;
320 
321 	ret = io_poll_check_events(req, locked);
322 	if (ret == IOU_POLL_NO_ACTION)
323 		return;
324 	io_poll_remove_entries(req);
325 	io_poll_tw_hash_eject(req, locked);
326 
327 	if (req->opcode == IORING_OP_POLL_ADD) {
328 		if (ret == IOU_POLL_DONE) {
329 			struct io_poll *poll;
330 
331 			poll = io_kiocb_to_cmd(req, struct io_poll);
332 			req->cqe.res = mangle_poll(req->cqe.res & poll->events);
333 		} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
334 			req->cqe.res = ret;
335 			req_set_fail(req);
336 		}
337 
338 		io_req_set_res(req, req->cqe.res, 0);
339 		io_req_task_complete(req, locked);
340 	} else {
341 		io_tw_lock(req->ctx, locked);
342 
343 		if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
344 			io_req_task_complete(req, locked);
345 		else if (ret == IOU_POLL_DONE)
346 			io_req_task_submit(req, locked);
347 		else
348 			io_req_defer_failed(req, ret);
349 	}
350 }
351 
352 static void __io_poll_execute(struct io_kiocb *req, int mask)
353 {
354 	io_req_set_res(req, mask, 0);
355 	req->io_task_work.func = io_poll_task_func;
356 
357 	trace_io_uring_task_add(req, mask);
358 	io_req_task_work_add(req);
359 }
360 
361 static inline void io_poll_execute(struct io_kiocb *req, int res)
362 {
363 	if (io_poll_get_ownership(req))
364 		__io_poll_execute(req, res);
365 }
366 
367 static void io_poll_cancel_req(struct io_kiocb *req)
368 {
369 	io_poll_mark_cancelled(req);
370 	/* kick tw, which should complete the request */
371 	io_poll_execute(req, 0);
372 }
373 
374 #define IO_ASYNC_POLL_COMMON	(EPOLLONESHOT | EPOLLPRI)
375 
376 static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll)
377 {
378 	io_poll_mark_cancelled(req);
379 	/* we have to kick tw in case it's not already */
380 	io_poll_execute(req, 0);
381 
382 	/*
383 	 * If the waitqueue is being freed early but someone is already
384 	 * holds ownership over it, we have to tear down the request as
385 	 * best we can. That means immediately removing the request from
386 	 * its waitqueue and preventing all further accesses to the
387 	 * waitqueue via the request.
388 	 */
389 	list_del_init(&poll->wait.entry);
390 
391 	/*
392 	 * Careful: this *must* be the last step, since as soon
393 	 * as req->head is NULL'ed out, the request can be
394 	 * completed and freed, since aio_poll_complete_work()
395 	 * will no longer need to take the waitqueue lock.
396 	 */
397 	smp_store_release(&poll->head, NULL);
398 	return 1;
399 }
400 
401 static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
402 			void *key)
403 {
404 	struct io_kiocb *req = wqe_to_req(wait);
405 	struct io_poll *poll = container_of(wait, struct io_poll, wait);
406 	__poll_t mask = key_to_poll(key);
407 
408 	if (unlikely(mask & POLLFREE))
409 		return io_pollfree_wake(req, poll);
410 
411 	/* for instances that support it check for an event match first */
412 	if (mask && !(mask & (poll->events & ~IO_ASYNC_POLL_COMMON)))
413 		return 0;
414 
415 	if (io_poll_get_ownership(req)) {
416 		/*
417 		 * If we trigger a multishot poll off our own wakeup path,
418 		 * disable multishot as there is a circular dependency between
419 		 * CQ posting and triggering the event.
420 		 */
421 		if (mask & EPOLL_URING_WAKE)
422 			poll->events |= EPOLLONESHOT;
423 
424 		/* optional, saves extra locking for removal in tw handler */
425 		if (mask && poll->events & EPOLLONESHOT) {
426 			list_del_init(&poll->wait.entry);
427 			poll->head = NULL;
428 			if (wqe_is_double(wait))
429 				req->flags &= ~REQ_F_DOUBLE_POLL;
430 			else
431 				req->flags &= ~REQ_F_SINGLE_POLL;
432 		}
433 		__io_poll_execute(req, mask);
434 	}
435 	return 1;
436 }
437 
438 /* fails only when polling is already completing by the first entry */
439 static bool io_poll_double_prepare(struct io_kiocb *req)
440 {
441 	struct wait_queue_head *head;
442 	struct io_poll *poll = io_poll_get_single(req);
443 
444 	/* head is RCU protected, see io_poll_remove_entries() comments */
445 	rcu_read_lock();
446 	head = smp_load_acquire(&poll->head);
447 	/*
448 	 * poll arm might not hold ownership and so race for req->flags with
449 	 * io_poll_wake(). There is only one poll entry queued, serialise with
450 	 * it by taking its head lock. As we're still arming the tw hanlder
451 	 * is not going to be run, so there are no races with it.
452 	 */
453 	if (head) {
454 		spin_lock_irq(&head->lock);
455 		req->flags |= REQ_F_DOUBLE_POLL;
456 		if (req->opcode == IORING_OP_POLL_ADD)
457 			req->flags |= REQ_F_ASYNC_DATA;
458 		spin_unlock_irq(&head->lock);
459 	}
460 	rcu_read_unlock();
461 	return !!head;
462 }
463 
464 static void __io_queue_proc(struct io_poll *poll, struct io_poll_table *pt,
465 			    struct wait_queue_head *head,
466 			    struct io_poll **poll_ptr)
467 {
468 	struct io_kiocb *req = pt->req;
469 	unsigned long wqe_private = (unsigned long) req;
470 
471 	/*
472 	 * The file being polled uses multiple waitqueues for poll handling
473 	 * (e.g. one for read, one for write). Setup a separate io_poll
474 	 * if this happens.
475 	 */
476 	if (unlikely(pt->nr_entries)) {
477 		struct io_poll *first = poll;
478 
479 		/* double add on the same waitqueue head, ignore */
480 		if (first->head == head)
481 			return;
482 		/* already have a 2nd entry, fail a third attempt */
483 		if (*poll_ptr) {
484 			if ((*poll_ptr)->head == head)
485 				return;
486 			pt->error = -EINVAL;
487 			return;
488 		}
489 
490 		poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
491 		if (!poll) {
492 			pt->error = -ENOMEM;
493 			return;
494 		}
495 
496 		/* mark as double wq entry */
497 		wqe_private |= IO_WQE_F_DOUBLE;
498 		io_init_poll_iocb(poll, first->events, first->wait.func);
499 		if (!io_poll_double_prepare(req)) {
500 			/* the request is completing, just back off */
501 			kfree(poll);
502 			return;
503 		}
504 		*poll_ptr = poll;
505 	} else {
506 		/* fine to modify, there is no poll queued to race with us */
507 		req->flags |= REQ_F_SINGLE_POLL;
508 	}
509 
510 	pt->nr_entries++;
511 	poll->head = head;
512 	poll->wait.private = (void *) wqe_private;
513 
514 	if (poll->events & EPOLLEXCLUSIVE)
515 		add_wait_queue_exclusive(head, &poll->wait);
516 	else
517 		add_wait_queue(head, &poll->wait);
518 }
519 
520 static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head,
521 			       struct poll_table_struct *p)
522 {
523 	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
524 	struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll);
525 
526 	__io_queue_proc(poll, pt, head,
527 			(struct io_poll **) &pt->req->async_data);
528 }
529 
530 static bool io_poll_can_finish_inline(struct io_kiocb *req,
531 				      struct io_poll_table *pt)
532 {
533 	return pt->owning || io_poll_get_ownership(req);
534 }
535 
536 /*
537  * Returns 0 when it's handed over for polling. The caller owns the requests if
538  * it returns non-zero, but otherwise should not touch it. Negative values
539  * contain an error code. When the result is >0, the polling has completed
540  * inline and ipt.result_mask is set to the mask.
541  */
542 static int __io_arm_poll_handler(struct io_kiocb *req,
543 				 struct io_poll *poll,
544 				 struct io_poll_table *ipt, __poll_t mask,
545 				 unsigned issue_flags)
546 {
547 	struct io_ring_ctx *ctx = req->ctx;
548 
549 	INIT_HLIST_NODE(&req->hash_node);
550 	req->work.cancel_seq = atomic_read(&ctx->cancel_seq);
551 	io_init_poll_iocb(poll, mask, io_poll_wake);
552 	poll->file = req->file;
553 	req->apoll_events = poll->events;
554 
555 	ipt->pt._key = mask;
556 	ipt->req = req;
557 	ipt->error = 0;
558 	ipt->nr_entries = 0;
559 	/*
560 	 * Polling is either completed here or via task_work, so if we're in the
561 	 * task context we're naturally serialised with tw by merit of running
562 	 * the same task. When it's io-wq, take the ownership to prevent tw
563 	 * from running. However, when we're in the task context, skip taking
564 	 * it as an optimisation.
565 	 *
566 	 * Note: even though the request won't be completed/freed, without
567 	 * ownership we still can race with io_poll_wake().
568 	 * io_poll_can_finish_inline() tries to deal with that.
569 	 */
570 	ipt->owning = issue_flags & IO_URING_F_UNLOCKED;
571 	atomic_set(&req->poll_refs, (int)ipt->owning);
572 
573 	/* io-wq doesn't hold uring_lock */
574 	if (issue_flags & IO_URING_F_UNLOCKED)
575 		req->flags &= ~REQ_F_HASH_LOCKED;
576 
577 	mask = vfs_poll(req->file, &ipt->pt) & poll->events;
578 
579 	if (unlikely(ipt->error || !ipt->nr_entries)) {
580 		io_poll_remove_entries(req);
581 
582 		if (!io_poll_can_finish_inline(req, ipt)) {
583 			io_poll_mark_cancelled(req);
584 			return 0;
585 		} else if (mask && (poll->events & EPOLLET)) {
586 			ipt->result_mask = mask;
587 			return 1;
588 		}
589 		return ipt->error ?: -EINVAL;
590 	}
591 
592 	if (mask &&
593 	   ((poll->events & (EPOLLET|EPOLLONESHOT)) == (EPOLLET|EPOLLONESHOT))) {
594 		if (!io_poll_can_finish_inline(req, ipt))
595 			return 0;
596 		io_poll_remove_entries(req);
597 		ipt->result_mask = mask;
598 		/* no one else has access to the req, forget about the ref */
599 		return 1;
600 	}
601 
602 	if (req->flags & REQ_F_HASH_LOCKED)
603 		io_poll_req_insert_locked(req);
604 	else
605 		io_poll_req_insert(req);
606 
607 	if (mask && (poll->events & EPOLLET) &&
608 	    io_poll_can_finish_inline(req, ipt)) {
609 		__io_poll_execute(req, mask);
610 		return 0;
611 	}
612 
613 	if (ipt->owning) {
614 		/*
615 		 * Try to release ownership. If we see a change of state, e.g.
616 		 * poll was waken up, queue up a tw, it'll deal with it.
617 		 */
618 		if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1)
619 			__io_poll_execute(req, 0);
620 	}
621 	return 0;
622 }
623 
624 static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
625 			       struct poll_table_struct *p)
626 {
627 	struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
628 	struct async_poll *apoll = pt->req->apoll;
629 
630 	__io_queue_proc(&apoll->poll, pt, head, &apoll->double_poll);
631 }
632 
633 static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req,
634 					     unsigned issue_flags)
635 {
636 	struct io_ring_ctx *ctx = req->ctx;
637 	struct io_cache_entry *entry;
638 	struct async_poll *apoll;
639 
640 	if (req->flags & REQ_F_POLLED) {
641 		apoll = req->apoll;
642 		kfree(apoll->double_poll);
643 	} else if (!(issue_flags & IO_URING_F_UNLOCKED)) {
644 		entry = io_alloc_cache_get(&ctx->apoll_cache);
645 		if (entry == NULL)
646 			goto alloc_apoll;
647 		apoll = container_of(entry, struct async_poll, cache);
648 	} else {
649 alloc_apoll:
650 		apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
651 		if (unlikely(!apoll))
652 			return NULL;
653 	}
654 	apoll->double_poll = NULL;
655 	req->apoll = apoll;
656 	return apoll;
657 }
658 
659 int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags)
660 {
661 	const struct io_op_def *def = &io_op_defs[req->opcode];
662 	struct async_poll *apoll;
663 	struct io_poll_table ipt;
664 	__poll_t mask = POLLPRI | POLLERR | EPOLLET;
665 	int ret;
666 
667 	/*
668 	 * apoll requests already grab the mutex to complete in the tw handler,
669 	 * so removal from the mutex-backed hash is free, use it by default.
670 	 */
671 	req->flags |= REQ_F_HASH_LOCKED;
672 
673 	if (!def->pollin && !def->pollout)
674 		return IO_APOLL_ABORTED;
675 	if (!file_can_poll(req->file))
676 		return IO_APOLL_ABORTED;
677 	if ((req->flags & (REQ_F_POLLED|REQ_F_PARTIAL_IO)) == REQ_F_POLLED)
678 		return IO_APOLL_ABORTED;
679 	if (!(req->flags & REQ_F_APOLL_MULTISHOT))
680 		mask |= EPOLLONESHOT;
681 
682 	if (def->pollin) {
683 		mask |= EPOLLIN | EPOLLRDNORM;
684 
685 		/* If reading from MSG_ERRQUEUE using recvmsg, ignore POLLIN */
686 		if (req->flags & REQ_F_CLEAR_POLLIN)
687 			mask &= ~EPOLLIN;
688 	} else {
689 		mask |= EPOLLOUT | EPOLLWRNORM;
690 	}
691 	if (def->poll_exclusive)
692 		mask |= EPOLLEXCLUSIVE;
693 
694 	apoll = io_req_alloc_apoll(req, issue_flags);
695 	if (!apoll)
696 		return IO_APOLL_ABORTED;
697 	req->flags |= REQ_F_POLLED;
698 	ipt.pt._qproc = io_async_queue_proc;
699 
700 	io_kbuf_recycle(req, issue_flags);
701 
702 	ret = __io_arm_poll_handler(req, &apoll->poll, &ipt, mask, issue_flags);
703 	if (ret)
704 		return ret > 0 ? IO_APOLL_READY : IO_APOLL_ABORTED;
705 	trace_io_uring_poll_arm(req, mask, apoll->poll.events);
706 	return IO_APOLL_OK;
707 }
708 
709 static __cold bool io_poll_remove_all_table(struct task_struct *tsk,
710 					    struct io_hash_table *table,
711 					    bool cancel_all)
712 {
713 	unsigned nr_buckets = 1U << table->hash_bits;
714 	struct hlist_node *tmp;
715 	struct io_kiocb *req;
716 	bool found = false;
717 	int i;
718 
719 	for (i = 0; i < nr_buckets; i++) {
720 		struct io_hash_bucket *hb = &table->hbs[i];
721 
722 		spin_lock(&hb->lock);
723 		hlist_for_each_entry_safe(req, tmp, &hb->list, hash_node) {
724 			if (io_match_task_safe(req, tsk, cancel_all)) {
725 				hlist_del_init(&req->hash_node);
726 				io_poll_cancel_req(req);
727 				found = true;
728 			}
729 		}
730 		spin_unlock(&hb->lock);
731 	}
732 	return found;
733 }
734 
735 /*
736  * Returns true if we found and killed one or more poll requests
737  */
738 __cold bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
739 			       bool cancel_all)
740 	__must_hold(&ctx->uring_lock)
741 {
742 	bool ret;
743 
744 	ret = io_poll_remove_all_table(tsk, &ctx->cancel_table, cancel_all);
745 	ret |= io_poll_remove_all_table(tsk, &ctx->cancel_table_locked, cancel_all);
746 	return ret;
747 }
748 
749 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, bool poll_only,
750 				     struct io_cancel_data *cd,
751 				     struct io_hash_table *table,
752 				     struct io_hash_bucket **out_bucket)
753 {
754 	struct io_kiocb *req;
755 	u32 index = hash_long(cd->data, table->hash_bits);
756 	struct io_hash_bucket *hb = &table->hbs[index];
757 
758 	*out_bucket = NULL;
759 
760 	spin_lock(&hb->lock);
761 	hlist_for_each_entry(req, &hb->list, hash_node) {
762 		if (cd->data != req->cqe.user_data)
763 			continue;
764 		if (poll_only && req->opcode != IORING_OP_POLL_ADD)
765 			continue;
766 		if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
767 			if (cd->seq == req->work.cancel_seq)
768 				continue;
769 			req->work.cancel_seq = cd->seq;
770 		}
771 		*out_bucket = hb;
772 		return req;
773 	}
774 	spin_unlock(&hb->lock);
775 	return NULL;
776 }
777 
778 static struct io_kiocb *io_poll_file_find(struct io_ring_ctx *ctx,
779 					  struct io_cancel_data *cd,
780 					  struct io_hash_table *table,
781 					  struct io_hash_bucket **out_bucket)
782 {
783 	unsigned nr_buckets = 1U << table->hash_bits;
784 	struct io_kiocb *req;
785 	int i;
786 
787 	*out_bucket = NULL;
788 
789 	for (i = 0; i < nr_buckets; i++) {
790 		struct io_hash_bucket *hb = &table->hbs[i];
791 
792 		spin_lock(&hb->lock);
793 		hlist_for_each_entry(req, &hb->list, hash_node) {
794 			if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
795 			    req->file != cd->file)
796 				continue;
797 			if (cd->seq == req->work.cancel_seq)
798 				continue;
799 			req->work.cancel_seq = cd->seq;
800 			*out_bucket = hb;
801 			return req;
802 		}
803 		spin_unlock(&hb->lock);
804 	}
805 	return NULL;
806 }
807 
808 static int io_poll_disarm(struct io_kiocb *req)
809 {
810 	if (!req)
811 		return -ENOENT;
812 	if (!io_poll_get_ownership(req))
813 		return -EALREADY;
814 	io_poll_remove_entries(req);
815 	hash_del(&req->hash_node);
816 	return 0;
817 }
818 
819 static int __io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
820 			    struct io_hash_table *table)
821 {
822 	struct io_hash_bucket *bucket;
823 	struct io_kiocb *req;
824 
825 	if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_ANY))
826 		req = io_poll_file_find(ctx, cd, table, &bucket);
827 	else
828 		req = io_poll_find(ctx, false, cd, table, &bucket);
829 
830 	if (req)
831 		io_poll_cancel_req(req);
832 	if (bucket)
833 		spin_unlock(&bucket->lock);
834 	return req ? 0 : -ENOENT;
835 }
836 
837 int io_poll_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd,
838 		   unsigned issue_flags)
839 {
840 	int ret;
841 
842 	ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table);
843 	if (ret != -ENOENT)
844 		return ret;
845 
846 	io_ring_submit_lock(ctx, issue_flags);
847 	ret = __io_poll_cancel(ctx, cd, &ctx->cancel_table_locked);
848 	io_ring_submit_unlock(ctx, issue_flags);
849 	return ret;
850 }
851 
852 static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe,
853 				     unsigned int flags)
854 {
855 	u32 events;
856 
857 	events = READ_ONCE(sqe->poll32_events);
858 #ifdef __BIG_ENDIAN
859 	events = swahw32(events);
860 #endif
861 	if (!(flags & IORING_POLL_ADD_MULTI))
862 		events |= EPOLLONESHOT;
863 	if (!(flags & IORING_POLL_ADD_LEVEL))
864 		events |= EPOLLET;
865 	return demangle_poll(events) |
866 		(events & (EPOLLEXCLUSIVE|EPOLLONESHOT|EPOLLET));
867 }
868 
869 int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
870 {
871 	struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update);
872 	u32 flags;
873 
874 	if (sqe->buf_index || sqe->splice_fd_in)
875 		return -EINVAL;
876 	flags = READ_ONCE(sqe->len);
877 	if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
878 		      IORING_POLL_ADD_MULTI))
879 		return -EINVAL;
880 	/* meaningless without update */
881 	if (flags == IORING_POLL_ADD_MULTI)
882 		return -EINVAL;
883 
884 	upd->old_user_data = READ_ONCE(sqe->addr);
885 	upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
886 	upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;
887 
888 	upd->new_user_data = READ_ONCE(sqe->off);
889 	if (!upd->update_user_data && upd->new_user_data)
890 		return -EINVAL;
891 	if (upd->update_events)
892 		upd->events = io_poll_parse_events(sqe, flags);
893 	else if (sqe->poll32_events)
894 		return -EINVAL;
895 
896 	return 0;
897 }
898 
899 int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
900 {
901 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
902 	u32 flags;
903 
904 	if (sqe->buf_index || sqe->off || sqe->addr)
905 		return -EINVAL;
906 	flags = READ_ONCE(sqe->len);
907 	if (flags & ~IORING_POLL_ADD_MULTI)
908 		return -EINVAL;
909 	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))
910 		return -EINVAL;
911 
912 	poll->events = io_poll_parse_events(sqe, flags);
913 	return 0;
914 }
915 
916 int io_poll_add(struct io_kiocb *req, unsigned int issue_flags)
917 {
918 	struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll);
919 	struct io_poll_table ipt;
920 	int ret;
921 
922 	ipt.pt._qproc = io_poll_queue_proc;
923 
924 	/*
925 	 * If sqpoll or single issuer, there is no contention for ->uring_lock
926 	 * and we'll end up holding it in tw handlers anyway.
927 	 */
928 	if (req->ctx->flags & (IORING_SETUP_SQPOLL|IORING_SETUP_SINGLE_ISSUER))
929 		req->flags |= REQ_F_HASH_LOCKED;
930 
931 	ret = __io_arm_poll_handler(req, poll, &ipt, poll->events, issue_flags);
932 	if (ret > 0) {
933 		io_req_set_res(req, ipt.result_mask, 0);
934 		return IOU_OK;
935 	}
936 	return ret ?: IOU_ISSUE_SKIP_COMPLETE;
937 }
938 
939 int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags)
940 {
941 	struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update);
942 	struct io_cancel_data cd = { .data = poll_update->old_user_data, };
943 	struct io_ring_ctx *ctx = req->ctx;
944 	struct io_hash_bucket *bucket;
945 	struct io_kiocb *preq;
946 	int ret2, ret = 0;
947 	bool locked;
948 
949 	preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table, &bucket);
950 	ret2 = io_poll_disarm(preq);
951 	if (bucket)
952 		spin_unlock(&bucket->lock);
953 	if (!ret2)
954 		goto found;
955 	if (ret2 != -ENOENT) {
956 		ret = ret2;
957 		goto out;
958 	}
959 
960 	io_ring_submit_lock(ctx, issue_flags);
961 	preq = io_poll_find(ctx, true, &cd, &ctx->cancel_table_locked, &bucket);
962 	ret2 = io_poll_disarm(preq);
963 	if (bucket)
964 		spin_unlock(&bucket->lock);
965 	io_ring_submit_unlock(ctx, issue_flags);
966 	if (ret2) {
967 		ret = ret2;
968 		goto out;
969 	}
970 
971 found:
972 	if (WARN_ON_ONCE(preq->opcode != IORING_OP_POLL_ADD)) {
973 		ret = -EFAULT;
974 		goto out;
975 	}
976 
977 	if (poll_update->update_events || poll_update->update_user_data) {
978 		/* only mask one event flags, keep behavior flags */
979 		if (poll_update->update_events) {
980 			struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll);
981 
982 			poll->events &= ~0xffff;
983 			poll->events |= poll_update->events & 0xffff;
984 			poll->events |= IO_POLL_UNMASK;
985 		}
986 		if (poll_update->update_user_data)
987 			preq->cqe.user_data = poll_update->new_user_data;
988 
989 		ret2 = io_poll_add(preq, issue_flags);
990 		/* successfully updated, don't complete poll request */
991 		if (!ret2 || ret2 == -EIOCBQUEUED)
992 			goto out;
993 	}
994 
995 	req_set_fail(preq);
996 	io_req_set_res(preq, -ECANCELED, 0);
997 	locked = !(issue_flags & IO_URING_F_UNLOCKED);
998 	io_req_task_complete(preq, &locked);
999 out:
1000 	if (ret < 0) {
1001 		req_set_fail(req);
1002 		return ret;
1003 	}
1004 	/* complete update request, we're done with it */
1005 	io_req_set_res(req, ret, 0);
1006 	return IOU_OK;
1007 }
1008 
1009 void io_apoll_cache_free(struct io_cache_entry *entry)
1010 {
1011 	kfree(container_of(entry, struct async_poll, cache));
1012 }
1013