xref: /openbmc/linux/fs/userfaultfd.c (revision e620a1e0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  fs/userfaultfd.c
4  *
5  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
6  *  Copyright (C) 2008-2009 Red Hat, Inc.
7  *  Copyright (C) 2015  Red Hat, Inc.
8  *
9  *  Some part derived from fs/eventfd.c (anon inode setup) and
10  *  mm/ksm.c (mm hashing).
11  */
12 
13 #include <linux/list.h>
14 #include <linux/hashtable.h>
15 #include <linux/sched/signal.h>
16 #include <linux/sched/mm.h>
17 #include <linux/mm.h>
18 #include <linux/poll.h>
19 #include <linux/slab.h>
20 #include <linux/seq_file.h>
21 #include <linux/file.h>
22 #include <linux/bug.h>
23 #include <linux/anon_inodes.h>
24 #include <linux/syscalls.h>
25 #include <linux/userfaultfd_k.h>
26 #include <linux/mempolicy.h>
27 #include <linux/ioctl.h>
28 #include <linux/security.h>
29 #include <linux/hugetlb.h>
30 
31 int sysctl_unprivileged_userfaultfd __read_mostly = 1;
32 
33 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
34 
35 enum userfaultfd_state {
36 	UFFD_STATE_WAIT_API,
37 	UFFD_STATE_RUNNING,
38 };
39 
40 /*
41  * Start with fault_pending_wqh and fault_wqh so they're more likely
42  * to be in the same cacheline.
43  *
44  * Locking order:
45  *	fd_wqh.lock
46  *		fault_pending_wqh.lock
47  *			fault_wqh.lock
48  *		event_wqh.lock
49  *
50  * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
51  * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
52  * also taken in IRQ context.
53  */
54 struct userfaultfd_ctx {
55 	/* waitqueue head for the pending (i.e. not read) userfaults */
56 	wait_queue_head_t fault_pending_wqh;
57 	/* waitqueue head for the userfaults */
58 	wait_queue_head_t fault_wqh;
59 	/* waitqueue head for the pseudo fd to wakeup poll/read */
60 	wait_queue_head_t fd_wqh;
61 	/* waitqueue head for events */
62 	wait_queue_head_t event_wqh;
63 	/* a refile sequence protected by fault_pending_wqh lock */
64 	struct seqcount refile_seq;
65 	/* pseudo fd refcounting */
66 	refcount_t refcount;
67 	/* userfaultfd syscall flags */
68 	unsigned int flags;
69 	/* features requested from the userspace */
70 	unsigned int features;
71 	/* state machine */
72 	enum userfaultfd_state state;
73 	/* released */
74 	bool released;
75 	/* memory mappings are changing because of non-cooperative event */
76 	bool mmap_changing;
77 	/* mm with one ore more vmas attached to this userfaultfd_ctx */
78 	struct mm_struct *mm;
79 };
80 
81 struct userfaultfd_fork_ctx {
82 	struct userfaultfd_ctx *orig;
83 	struct userfaultfd_ctx *new;
84 	struct list_head list;
85 };
86 
87 struct userfaultfd_unmap_ctx {
88 	struct userfaultfd_ctx *ctx;
89 	unsigned long start;
90 	unsigned long end;
91 	struct list_head list;
92 };
93 
94 struct userfaultfd_wait_queue {
95 	struct uffd_msg msg;
96 	wait_queue_entry_t wq;
97 	struct userfaultfd_ctx *ctx;
98 	bool waken;
99 };
100 
101 struct userfaultfd_wake_range {
102 	unsigned long start;
103 	unsigned long len;
104 };
105 
106 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
107 				     int wake_flags, void *key)
108 {
109 	struct userfaultfd_wake_range *range = key;
110 	int ret;
111 	struct userfaultfd_wait_queue *uwq;
112 	unsigned long start, len;
113 
114 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
115 	ret = 0;
116 	/* len == 0 means wake all */
117 	start = range->start;
118 	len = range->len;
119 	if (len && (start > uwq->msg.arg.pagefault.address ||
120 		    start + len <= uwq->msg.arg.pagefault.address))
121 		goto out;
122 	WRITE_ONCE(uwq->waken, true);
123 	/*
124 	 * The Program-Order guarantees provided by the scheduler
125 	 * ensure uwq->waken is visible before the task is woken.
126 	 */
127 	ret = wake_up_state(wq->private, mode);
128 	if (ret) {
129 		/*
130 		 * Wake only once, autoremove behavior.
131 		 *
132 		 * After the effect of list_del_init is visible to the other
133 		 * CPUs, the waitqueue may disappear from under us, see the
134 		 * !list_empty_careful() in handle_userfault().
135 		 *
136 		 * try_to_wake_up() has an implicit smp_mb(), and the
137 		 * wq->private is read before calling the extern function
138 		 * "wake_up_state" (which in turns calls try_to_wake_up).
139 		 */
140 		list_del_init(&wq->entry);
141 	}
142 out:
143 	return ret;
144 }
145 
146 /**
147  * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
148  * context.
149  * @ctx: [in] Pointer to the userfaultfd context.
150  */
151 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
152 {
153 	refcount_inc(&ctx->refcount);
154 }
155 
156 /**
157  * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
158  * context.
159  * @ctx: [in] Pointer to userfaultfd context.
160  *
161  * The userfaultfd context reference must have been previously acquired either
162  * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
163  */
164 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
165 {
166 	if (refcount_dec_and_test(&ctx->refcount)) {
167 		VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
168 		VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
169 		VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
170 		VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
171 		VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
172 		VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
173 		VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
174 		VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
175 		mmdrop(ctx->mm);
176 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
177 	}
178 }
179 
180 static inline void msg_init(struct uffd_msg *msg)
181 {
182 	BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
183 	/*
184 	 * Must use memset to zero out the paddings or kernel data is
185 	 * leaked to userland.
186 	 */
187 	memset(msg, 0, sizeof(struct uffd_msg));
188 }
189 
190 static inline struct uffd_msg userfault_msg(unsigned long address,
191 					    unsigned int flags,
192 					    unsigned long reason,
193 					    unsigned int features)
194 {
195 	struct uffd_msg msg;
196 	msg_init(&msg);
197 	msg.event = UFFD_EVENT_PAGEFAULT;
198 	msg.arg.pagefault.address = address;
199 	if (flags & FAULT_FLAG_WRITE)
200 		/*
201 		 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
202 		 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE
203 		 * was not set in a UFFD_EVENT_PAGEFAULT, it means it
204 		 * was a read fault, otherwise if set it means it's
205 		 * a write fault.
206 		 */
207 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
208 	if (reason & VM_UFFD_WP)
209 		/*
210 		 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
211 		 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was
212 		 * not set in a UFFD_EVENT_PAGEFAULT, it means it was
213 		 * a missing fault, otherwise if set it means it's a
214 		 * write protect fault.
215 		 */
216 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
217 	if (features & UFFD_FEATURE_THREAD_ID)
218 		msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
219 	return msg;
220 }
221 
222 #ifdef CONFIG_HUGETLB_PAGE
223 /*
224  * Same functionality as userfaultfd_must_wait below with modifications for
225  * hugepmd ranges.
226  */
227 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
228 					 struct vm_area_struct *vma,
229 					 unsigned long address,
230 					 unsigned long flags,
231 					 unsigned long reason)
232 {
233 	struct mm_struct *mm = ctx->mm;
234 	pte_t *ptep, pte;
235 	bool ret = true;
236 
237 	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
238 
239 	ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
240 
241 	if (!ptep)
242 		goto out;
243 
244 	ret = false;
245 	pte = huge_ptep_get(ptep);
246 
247 	/*
248 	 * Lockless access: we're in a wait_event so it's ok if it
249 	 * changes under us.
250 	 */
251 	if (huge_pte_none(pte))
252 		ret = true;
253 	if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
254 		ret = true;
255 out:
256 	return ret;
257 }
258 #else
259 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
260 					 struct vm_area_struct *vma,
261 					 unsigned long address,
262 					 unsigned long flags,
263 					 unsigned long reason)
264 {
265 	return false;	/* should never get here */
266 }
267 #endif /* CONFIG_HUGETLB_PAGE */
268 
269 /*
270  * Verify the pagetables are still not ok after having reigstered into
271  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
272  * userfault that has already been resolved, if userfaultfd_read and
273  * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
274  * threads.
275  */
276 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
277 					 unsigned long address,
278 					 unsigned long flags,
279 					 unsigned long reason)
280 {
281 	struct mm_struct *mm = ctx->mm;
282 	pgd_t *pgd;
283 	p4d_t *p4d;
284 	pud_t *pud;
285 	pmd_t *pmd, _pmd;
286 	pte_t *pte;
287 	bool ret = true;
288 
289 	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
290 
291 	pgd = pgd_offset(mm, address);
292 	if (!pgd_present(*pgd))
293 		goto out;
294 	p4d = p4d_offset(pgd, address);
295 	if (!p4d_present(*p4d))
296 		goto out;
297 	pud = pud_offset(p4d, address);
298 	if (!pud_present(*pud))
299 		goto out;
300 	pmd = pmd_offset(pud, address);
301 	/*
302 	 * READ_ONCE must function as a barrier with narrower scope
303 	 * and it must be equivalent to:
304 	 *	_pmd = *pmd; barrier();
305 	 *
306 	 * This is to deal with the instability (as in
307 	 * pmd_trans_unstable) of the pmd.
308 	 */
309 	_pmd = READ_ONCE(*pmd);
310 	if (pmd_none(_pmd))
311 		goto out;
312 
313 	ret = false;
314 	if (!pmd_present(_pmd))
315 		goto out;
316 
317 	if (pmd_trans_huge(_pmd))
318 		goto out;
319 
320 	/*
321 	 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
322 	 * and use the standard pte_offset_map() instead of parsing _pmd.
323 	 */
324 	pte = pte_offset_map(pmd, address);
325 	/*
326 	 * Lockless access: we're in a wait_event so it's ok if it
327 	 * changes under us.
328 	 */
329 	if (pte_none(*pte))
330 		ret = true;
331 	pte_unmap(pte);
332 
333 out:
334 	return ret;
335 }
336 
337 /*
338  * The locking rules involved in returning VM_FAULT_RETRY depending on
339  * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
340  * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
341  * recommendation in __lock_page_or_retry is not an understatement.
342  *
343  * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
344  * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
345  * not set.
346  *
347  * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
348  * set, VM_FAULT_RETRY can still be returned if and only if there are
349  * fatal_signal_pending()s, and the mmap_sem must be released before
350  * returning it.
351  */
352 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
353 {
354 	struct mm_struct *mm = vmf->vma->vm_mm;
355 	struct userfaultfd_ctx *ctx;
356 	struct userfaultfd_wait_queue uwq;
357 	vm_fault_t ret = VM_FAULT_SIGBUS;
358 	bool must_wait, return_to_userland;
359 	long blocking_state;
360 
361 	/*
362 	 * We don't do userfault handling for the final child pid update.
363 	 *
364 	 * We also don't do userfault handling during
365 	 * coredumping. hugetlbfs has the special
366 	 * follow_hugetlb_page() to skip missing pages in the
367 	 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
368 	 * the no_page_table() helper in follow_page_mask(), but the
369 	 * shmem_vm_ops->fault method is invoked even during
370 	 * coredumping without mmap_sem and it ends up here.
371 	 */
372 	if (current->flags & (PF_EXITING|PF_DUMPCORE))
373 		goto out;
374 
375 	/*
376 	 * Coredumping runs without mmap_sem so we can only check that
377 	 * the mmap_sem is held, if PF_DUMPCORE was not set.
378 	 */
379 	WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
380 
381 	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
382 	if (!ctx)
383 		goto out;
384 
385 	BUG_ON(ctx->mm != mm);
386 
387 	VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP));
388 	VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP));
389 
390 	if (ctx->features & UFFD_FEATURE_SIGBUS)
391 		goto out;
392 
393 	/*
394 	 * If it's already released don't get it. This avoids to loop
395 	 * in __get_user_pages if userfaultfd_release waits on the
396 	 * caller of handle_userfault to release the mmap_sem.
397 	 */
398 	if (unlikely(READ_ONCE(ctx->released))) {
399 		/*
400 		 * Don't return VM_FAULT_SIGBUS in this case, so a non
401 		 * cooperative manager can close the uffd after the
402 		 * last UFFDIO_COPY, without risking to trigger an
403 		 * involuntary SIGBUS if the process was starting the
404 		 * userfaultfd while the userfaultfd was still armed
405 		 * (but after the last UFFDIO_COPY). If the uffd
406 		 * wasn't already closed when the userfault reached
407 		 * this point, that would normally be solved by
408 		 * userfaultfd_must_wait returning 'false'.
409 		 *
410 		 * If we were to return VM_FAULT_SIGBUS here, the non
411 		 * cooperative manager would be instead forced to
412 		 * always call UFFDIO_UNREGISTER before it can safely
413 		 * close the uffd.
414 		 */
415 		ret = VM_FAULT_NOPAGE;
416 		goto out;
417 	}
418 
419 	/*
420 	 * Check that we can return VM_FAULT_RETRY.
421 	 *
422 	 * NOTE: it should become possible to return VM_FAULT_RETRY
423 	 * even if FAULT_FLAG_TRIED is set without leading to gup()
424 	 * -EBUSY failures, if the userfaultfd is to be extended for
425 	 * VM_UFFD_WP tracking and we intend to arm the userfault
426 	 * without first stopping userland access to the memory. For
427 	 * VM_UFFD_MISSING userfaults this is enough for now.
428 	 */
429 	if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
430 		/*
431 		 * Validate the invariant that nowait must allow retry
432 		 * to be sure not to return SIGBUS erroneously on
433 		 * nowait invocations.
434 		 */
435 		BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
436 #ifdef CONFIG_DEBUG_VM
437 		if (printk_ratelimit()) {
438 			printk(KERN_WARNING
439 			       "FAULT_FLAG_ALLOW_RETRY missing %x\n",
440 			       vmf->flags);
441 			dump_stack();
442 		}
443 #endif
444 		goto out;
445 	}
446 
447 	/*
448 	 * Handle nowait, not much to do other than tell it to retry
449 	 * and wait.
450 	 */
451 	ret = VM_FAULT_RETRY;
452 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
453 		goto out;
454 
455 	/* take the reference before dropping the mmap_sem */
456 	userfaultfd_ctx_get(ctx);
457 
458 	init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
459 	uwq.wq.private = current;
460 	uwq.msg = userfault_msg(vmf->address, vmf->flags, reason,
461 			ctx->features);
462 	uwq.ctx = ctx;
463 	uwq.waken = false;
464 
465 	return_to_userland =
466 		(vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
467 		(FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
468 	blocking_state = return_to_userland ? TASK_INTERRUPTIBLE :
469 			 TASK_KILLABLE;
470 
471 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
472 	/*
473 	 * After the __add_wait_queue the uwq is visible to userland
474 	 * through poll/read().
475 	 */
476 	__add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
477 	/*
478 	 * The smp_mb() after __set_current_state prevents the reads
479 	 * following the spin_unlock to happen before the list_add in
480 	 * __add_wait_queue.
481 	 */
482 	set_current_state(blocking_state);
483 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
484 
485 	if (!is_vm_hugetlb_page(vmf->vma))
486 		must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
487 						  reason);
488 	else
489 		must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
490 						       vmf->address,
491 						       vmf->flags, reason);
492 	up_read(&mm->mmap_sem);
493 
494 	if (likely(must_wait && !READ_ONCE(ctx->released) &&
495 		   (return_to_userland ? !signal_pending(current) :
496 		    !fatal_signal_pending(current)))) {
497 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
498 		schedule();
499 		ret |= VM_FAULT_MAJOR;
500 
501 		/*
502 		 * False wakeups can orginate even from rwsem before
503 		 * up_read() however userfaults will wait either for a
504 		 * targeted wakeup on the specific uwq waitqueue from
505 		 * wake_userfault() or for signals or for uffd
506 		 * release.
507 		 */
508 		while (!READ_ONCE(uwq.waken)) {
509 			/*
510 			 * This needs the full smp_store_mb()
511 			 * guarantee as the state write must be
512 			 * visible to other CPUs before reading
513 			 * uwq.waken from other CPUs.
514 			 */
515 			set_current_state(blocking_state);
516 			if (READ_ONCE(uwq.waken) ||
517 			    READ_ONCE(ctx->released) ||
518 			    (return_to_userland ? signal_pending(current) :
519 			     fatal_signal_pending(current)))
520 				break;
521 			schedule();
522 		}
523 	}
524 
525 	__set_current_state(TASK_RUNNING);
526 
527 	if (return_to_userland) {
528 		if (signal_pending(current) &&
529 		    !fatal_signal_pending(current)) {
530 			/*
531 			 * If we got a SIGSTOP or SIGCONT and this is
532 			 * a normal userland page fault, just let
533 			 * userland return so the signal will be
534 			 * handled and gdb debugging works.  The page
535 			 * fault code immediately after we return from
536 			 * this function is going to release the
537 			 * mmap_sem and it's not depending on it
538 			 * (unlike gup would if we were not to return
539 			 * VM_FAULT_RETRY).
540 			 *
541 			 * If a fatal signal is pending we still take
542 			 * the streamlined VM_FAULT_RETRY failure path
543 			 * and there's no need to retake the mmap_sem
544 			 * in such case.
545 			 */
546 			down_read(&mm->mmap_sem);
547 			ret = VM_FAULT_NOPAGE;
548 		}
549 	}
550 
551 	/*
552 	 * Here we race with the list_del; list_add in
553 	 * userfaultfd_ctx_read(), however because we don't ever run
554 	 * list_del_init() to refile across the two lists, the prev
555 	 * and next pointers will never point to self. list_add also
556 	 * would never let any of the two pointers to point to
557 	 * self. So list_empty_careful won't risk to see both pointers
558 	 * pointing to self at any time during the list refile. The
559 	 * only case where list_del_init() is called is the full
560 	 * removal in the wake function and there we don't re-list_add
561 	 * and it's fine not to block on the spinlock. The uwq on this
562 	 * kernel stack can be released after the list_del_init.
563 	 */
564 	if (!list_empty_careful(&uwq.wq.entry)) {
565 		spin_lock_irq(&ctx->fault_pending_wqh.lock);
566 		/*
567 		 * No need of list_del_init(), the uwq on the stack
568 		 * will be freed shortly anyway.
569 		 */
570 		list_del(&uwq.wq.entry);
571 		spin_unlock_irq(&ctx->fault_pending_wqh.lock);
572 	}
573 
574 	/*
575 	 * ctx may go away after this if the userfault pseudo fd is
576 	 * already released.
577 	 */
578 	userfaultfd_ctx_put(ctx);
579 
580 out:
581 	return ret;
582 }
583 
584 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
585 					      struct userfaultfd_wait_queue *ewq)
586 {
587 	struct userfaultfd_ctx *release_new_ctx;
588 
589 	if (WARN_ON_ONCE(current->flags & PF_EXITING))
590 		goto out;
591 
592 	ewq->ctx = ctx;
593 	init_waitqueue_entry(&ewq->wq, current);
594 	release_new_ctx = NULL;
595 
596 	spin_lock_irq(&ctx->event_wqh.lock);
597 	/*
598 	 * After the __add_wait_queue the uwq is visible to userland
599 	 * through poll/read().
600 	 */
601 	__add_wait_queue(&ctx->event_wqh, &ewq->wq);
602 	for (;;) {
603 		set_current_state(TASK_KILLABLE);
604 		if (ewq->msg.event == 0)
605 			break;
606 		if (READ_ONCE(ctx->released) ||
607 		    fatal_signal_pending(current)) {
608 			/*
609 			 * &ewq->wq may be queued in fork_event, but
610 			 * __remove_wait_queue ignores the head
611 			 * parameter. It would be a problem if it
612 			 * didn't.
613 			 */
614 			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
615 			if (ewq->msg.event == UFFD_EVENT_FORK) {
616 				struct userfaultfd_ctx *new;
617 
618 				new = (struct userfaultfd_ctx *)
619 					(unsigned long)
620 					ewq->msg.arg.reserved.reserved1;
621 				release_new_ctx = new;
622 			}
623 			break;
624 		}
625 
626 		spin_unlock_irq(&ctx->event_wqh.lock);
627 
628 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
629 		schedule();
630 
631 		spin_lock_irq(&ctx->event_wqh.lock);
632 	}
633 	__set_current_state(TASK_RUNNING);
634 	spin_unlock_irq(&ctx->event_wqh.lock);
635 
636 	if (release_new_ctx) {
637 		struct vm_area_struct *vma;
638 		struct mm_struct *mm = release_new_ctx->mm;
639 
640 		/* the various vma->vm_userfaultfd_ctx still points to it */
641 		down_write(&mm->mmap_sem);
642 		/* no task can run (and in turn coredump) yet */
643 		VM_WARN_ON(!mmget_still_valid(mm));
644 		for (vma = mm->mmap; vma; vma = vma->vm_next)
645 			if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
646 				vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
647 				vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
648 			}
649 		up_write(&mm->mmap_sem);
650 
651 		userfaultfd_ctx_put(release_new_ctx);
652 	}
653 
654 	/*
655 	 * ctx may go away after this if the userfault pseudo fd is
656 	 * already released.
657 	 */
658 out:
659 	WRITE_ONCE(ctx->mmap_changing, false);
660 	userfaultfd_ctx_put(ctx);
661 }
662 
663 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
664 				       struct userfaultfd_wait_queue *ewq)
665 {
666 	ewq->msg.event = 0;
667 	wake_up_locked(&ctx->event_wqh);
668 	__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
669 }
670 
671 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
672 {
673 	struct userfaultfd_ctx *ctx = NULL, *octx;
674 	struct userfaultfd_fork_ctx *fctx;
675 
676 	octx = vma->vm_userfaultfd_ctx.ctx;
677 	if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
678 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
679 		vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
680 		return 0;
681 	}
682 
683 	list_for_each_entry(fctx, fcs, list)
684 		if (fctx->orig == octx) {
685 			ctx = fctx->new;
686 			break;
687 		}
688 
689 	if (!ctx) {
690 		fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
691 		if (!fctx)
692 			return -ENOMEM;
693 
694 		ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
695 		if (!ctx) {
696 			kfree(fctx);
697 			return -ENOMEM;
698 		}
699 
700 		refcount_set(&ctx->refcount, 1);
701 		ctx->flags = octx->flags;
702 		ctx->state = UFFD_STATE_RUNNING;
703 		ctx->features = octx->features;
704 		ctx->released = false;
705 		ctx->mmap_changing = false;
706 		ctx->mm = vma->vm_mm;
707 		mmgrab(ctx->mm);
708 
709 		userfaultfd_ctx_get(octx);
710 		WRITE_ONCE(octx->mmap_changing, true);
711 		fctx->orig = octx;
712 		fctx->new = ctx;
713 		list_add_tail(&fctx->list, fcs);
714 	}
715 
716 	vma->vm_userfaultfd_ctx.ctx = ctx;
717 	return 0;
718 }
719 
720 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
721 {
722 	struct userfaultfd_ctx *ctx = fctx->orig;
723 	struct userfaultfd_wait_queue ewq;
724 
725 	msg_init(&ewq.msg);
726 
727 	ewq.msg.event = UFFD_EVENT_FORK;
728 	ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
729 
730 	userfaultfd_event_wait_completion(ctx, &ewq);
731 }
732 
733 void dup_userfaultfd_complete(struct list_head *fcs)
734 {
735 	struct userfaultfd_fork_ctx *fctx, *n;
736 
737 	list_for_each_entry_safe(fctx, n, fcs, list) {
738 		dup_fctx(fctx);
739 		list_del(&fctx->list);
740 		kfree(fctx);
741 	}
742 }
743 
744 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
745 			     struct vm_userfaultfd_ctx *vm_ctx)
746 {
747 	struct userfaultfd_ctx *ctx;
748 
749 	ctx = vma->vm_userfaultfd_ctx.ctx;
750 
751 	if (!ctx)
752 		return;
753 
754 	if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
755 		vm_ctx->ctx = ctx;
756 		userfaultfd_ctx_get(ctx);
757 		WRITE_ONCE(ctx->mmap_changing, true);
758 	} else {
759 		/* Drop uffd context if remap feature not enabled */
760 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
761 		vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
762 	}
763 }
764 
765 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
766 				 unsigned long from, unsigned long to,
767 				 unsigned long len)
768 {
769 	struct userfaultfd_ctx *ctx = vm_ctx->ctx;
770 	struct userfaultfd_wait_queue ewq;
771 
772 	if (!ctx)
773 		return;
774 
775 	if (to & ~PAGE_MASK) {
776 		userfaultfd_ctx_put(ctx);
777 		return;
778 	}
779 
780 	msg_init(&ewq.msg);
781 
782 	ewq.msg.event = UFFD_EVENT_REMAP;
783 	ewq.msg.arg.remap.from = from;
784 	ewq.msg.arg.remap.to = to;
785 	ewq.msg.arg.remap.len = len;
786 
787 	userfaultfd_event_wait_completion(ctx, &ewq);
788 }
789 
790 bool userfaultfd_remove(struct vm_area_struct *vma,
791 			unsigned long start, unsigned long end)
792 {
793 	struct mm_struct *mm = vma->vm_mm;
794 	struct userfaultfd_ctx *ctx;
795 	struct userfaultfd_wait_queue ewq;
796 
797 	ctx = vma->vm_userfaultfd_ctx.ctx;
798 	if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
799 		return true;
800 
801 	userfaultfd_ctx_get(ctx);
802 	WRITE_ONCE(ctx->mmap_changing, true);
803 	up_read(&mm->mmap_sem);
804 
805 	msg_init(&ewq.msg);
806 
807 	ewq.msg.event = UFFD_EVENT_REMOVE;
808 	ewq.msg.arg.remove.start = start;
809 	ewq.msg.arg.remove.end = end;
810 
811 	userfaultfd_event_wait_completion(ctx, &ewq);
812 
813 	return false;
814 }
815 
816 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
817 			  unsigned long start, unsigned long end)
818 {
819 	struct userfaultfd_unmap_ctx *unmap_ctx;
820 
821 	list_for_each_entry(unmap_ctx, unmaps, list)
822 		if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
823 		    unmap_ctx->end == end)
824 			return true;
825 
826 	return false;
827 }
828 
829 int userfaultfd_unmap_prep(struct vm_area_struct *vma,
830 			   unsigned long start, unsigned long end,
831 			   struct list_head *unmaps)
832 {
833 	for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
834 		struct userfaultfd_unmap_ctx *unmap_ctx;
835 		struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
836 
837 		if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
838 		    has_unmap_ctx(ctx, unmaps, start, end))
839 			continue;
840 
841 		unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
842 		if (!unmap_ctx)
843 			return -ENOMEM;
844 
845 		userfaultfd_ctx_get(ctx);
846 		WRITE_ONCE(ctx->mmap_changing, true);
847 		unmap_ctx->ctx = ctx;
848 		unmap_ctx->start = start;
849 		unmap_ctx->end = end;
850 		list_add_tail(&unmap_ctx->list, unmaps);
851 	}
852 
853 	return 0;
854 }
855 
856 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
857 {
858 	struct userfaultfd_unmap_ctx *ctx, *n;
859 	struct userfaultfd_wait_queue ewq;
860 
861 	list_for_each_entry_safe(ctx, n, uf, list) {
862 		msg_init(&ewq.msg);
863 
864 		ewq.msg.event = UFFD_EVENT_UNMAP;
865 		ewq.msg.arg.remove.start = ctx->start;
866 		ewq.msg.arg.remove.end = ctx->end;
867 
868 		userfaultfd_event_wait_completion(ctx->ctx, &ewq);
869 
870 		list_del(&ctx->list);
871 		kfree(ctx);
872 	}
873 }
874 
875 static int userfaultfd_release(struct inode *inode, struct file *file)
876 {
877 	struct userfaultfd_ctx *ctx = file->private_data;
878 	struct mm_struct *mm = ctx->mm;
879 	struct vm_area_struct *vma, *prev;
880 	/* len == 0 means wake all */
881 	struct userfaultfd_wake_range range = { .len = 0, };
882 	unsigned long new_flags;
883 	bool still_valid;
884 
885 	WRITE_ONCE(ctx->released, true);
886 
887 	if (!mmget_not_zero(mm))
888 		goto wakeup;
889 
890 	/*
891 	 * Flush page faults out of all CPUs. NOTE: all page faults
892 	 * must be retried without returning VM_FAULT_SIGBUS if
893 	 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
894 	 * changes while handle_userfault released the mmap_sem. So
895 	 * it's critical that released is set to true (above), before
896 	 * taking the mmap_sem for writing.
897 	 */
898 	down_write(&mm->mmap_sem);
899 	still_valid = mmget_still_valid(mm);
900 	prev = NULL;
901 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
902 		cond_resched();
903 		BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
904 		       !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
905 		if (vma->vm_userfaultfd_ctx.ctx != ctx) {
906 			prev = vma;
907 			continue;
908 		}
909 		new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
910 		if (still_valid) {
911 			prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
912 					 new_flags, vma->anon_vma,
913 					 vma->vm_file, vma->vm_pgoff,
914 					 vma_policy(vma),
915 					 NULL_VM_UFFD_CTX);
916 			if (prev)
917 				vma = prev;
918 			else
919 				prev = vma;
920 		}
921 		vma->vm_flags = new_flags;
922 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
923 	}
924 	up_write(&mm->mmap_sem);
925 	mmput(mm);
926 wakeup:
927 	/*
928 	 * After no new page faults can wait on this fault_*wqh, flush
929 	 * the last page faults that may have been already waiting on
930 	 * the fault_*wqh.
931 	 */
932 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
933 	__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
934 	__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
935 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
936 
937 	/* Flush pending events that may still wait on event_wqh */
938 	wake_up_all(&ctx->event_wqh);
939 
940 	wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
941 	userfaultfd_ctx_put(ctx);
942 	return 0;
943 }
944 
945 /* fault_pending_wqh.lock must be hold by the caller */
946 static inline struct userfaultfd_wait_queue *find_userfault_in(
947 		wait_queue_head_t *wqh)
948 {
949 	wait_queue_entry_t *wq;
950 	struct userfaultfd_wait_queue *uwq;
951 
952 	lockdep_assert_held(&wqh->lock);
953 
954 	uwq = NULL;
955 	if (!waitqueue_active(wqh))
956 		goto out;
957 	/* walk in reverse to provide FIFO behavior to read userfaults */
958 	wq = list_last_entry(&wqh->head, typeof(*wq), entry);
959 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
960 out:
961 	return uwq;
962 }
963 
964 static inline struct userfaultfd_wait_queue *find_userfault(
965 		struct userfaultfd_ctx *ctx)
966 {
967 	return find_userfault_in(&ctx->fault_pending_wqh);
968 }
969 
970 static inline struct userfaultfd_wait_queue *find_userfault_evt(
971 		struct userfaultfd_ctx *ctx)
972 {
973 	return find_userfault_in(&ctx->event_wqh);
974 }
975 
976 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
977 {
978 	struct userfaultfd_ctx *ctx = file->private_data;
979 	__poll_t ret;
980 
981 	poll_wait(file, &ctx->fd_wqh, wait);
982 
983 	switch (ctx->state) {
984 	case UFFD_STATE_WAIT_API:
985 		return EPOLLERR;
986 	case UFFD_STATE_RUNNING:
987 		/*
988 		 * poll() never guarantees that read won't block.
989 		 * userfaults can be waken before they're read().
990 		 */
991 		if (unlikely(!(file->f_flags & O_NONBLOCK)))
992 			return EPOLLERR;
993 		/*
994 		 * lockless access to see if there are pending faults
995 		 * __pollwait last action is the add_wait_queue but
996 		 * the spin_unlock would allow the waitqueue_active to
997 		 * pass above the actual list_add inside
998 		 * add_wait_queue critical section. So use a full
999 		 * memory barrier to serialize the list_add write of
1000 		 * add_wait_queue() with the waitqueue_active read
1001 		 * below.
1002 		 */
1003 		ret = 0;
1004 		smp_mb();
1005 		if (waitqueue_active(&ctx->fault_pending_wqh))
1006 			ret = EPOLLIN;
1007 		else if (waitqueue_active(&ctx->event_wqh))
1008 			ret = EPOLLIN;
1009 
1010 		return ret;
1011 	default:
1012 		WARN_ON_ONCE(1);
1013 		return EPOLLERR;
1014 	}
1015 }
1016 
1017 static const struct file_operations userfaultfd_fops;
1018 
1019 static int resolve_userfault_fork(struct userfaultfd_ctx *ctx,
1020 				  struct userfaultfd_ctx *new,
1021 				  struct uffd_msg *msg)
1022 {
1023 	int fd;
1024 
1025 	fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, new,
1026 			      O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS));
1027 	if (fd < 0)
1028 		return fd;
1029 
1030 	msg->arg.reserved.reserved1 = 0;
1031 	msg->arg.fork.ufd = fd;
1032 	return 0;
1033 }
1034 
1035 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1036 				    struct uffd_msg *msg)
1037 {
1038 	ssize_t ret;
1039 	DECLARE_WAITQUEUE(wait, current);
1040 	struct userfaultfd_wait_queue *uwq;
1041 	/*
1042 	 * Handling fork event requires sleeping operations, so
1043 	 * we drop the event_wqh lock, then do these ops, then
1044 	 * lock it back and wake up the waiter. While the lock is
1045 	 * dropped the ewq may go away so we keep track of it
1046 	 * carefully.
1047 	 */
1048 	LIST_HEAD(fork_event);
1049 	struct userfaultfd_ctx *fork_nctx = NULL;
1050 
1051 	/* always take the fd_wqh lock before the fault_pending_wqh lock */
1052 	spin_lock_irq(&ctx->fd_wqh.lock);
1053 	__add_wait_queue(&ctx->fd_wqh, &wait);
1054 	for (;;) {
1055 		set_current_state(TASK_INTERRUPTIBLE);
1056 		spin_lock(&ctx->fault_pending_wqh.lock);
1057 		uwq = find_userfault(ctx);
1058 		if (uwq) {
1059 			/*
1060 			 * Use a seqcount to repeat the lockless check
1061 			 * in wake_userfault() to avoid missing
1062 			 * wakeups because during the refile both
1063 			 * waitqueue could become empty if this is the
1064 			 * only userfault.
1065 			 */
1066 			write_seqcount_begin(&ctx->refile_seq);
1067 
1068 			/*
1069 			 * The fault_pending_wqh.lock prevents the uwq
1070 			 * to disappear from under us.
1071 			 *
1072 			 * Refile this userfault from
1073 			 * fault_pending_wqh to fault_wqh, it's not
1074 			 * pending anymore after we read it.
1075 			 *
1076 			 * Use list_del() by hand (as
1077 			 * userfaultfd_wake_function also uses
1078 			 * list_del_init() by hand) to be sure nobody
1079 			 * changes __remove_wait_queue() to use
1080 			 * list_del_init() in turn breaking the
1081 			 * !list_empty_careful() check in
1082 			 * handle_userfault(). The uwq->wq.head list
1083 			 * must never be empty at any time during the
1084 			 * refile, or the waitqueue could disappear
1085 			 * from under us. The "wait_queue_head_t"
1086 			 * parameter of __remove_wait_queue() is unused
1087 			 * anyway.
1088 			 */
1089 			list_del(&uwq->wq.entry);
1090 			add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1091 
1092 			write_seqcount_end(&ctx->refile_seq);
1093 
1094 			/* careful to always initialize msg if ret == 0 */
1095 			*msg = uwq->msg;
1096 			spin_unlock(&ctx->fault_pending_wqh.lock);
1097 			ret = 0;
1098 			break;
1099 		}
1100 		spin_unlock(&ctx->fault_pending_wqh.lock);
1101 
1102 		spin_lock(&ctx->event_wqh.lock);
1103 		uwq = find_userfault_evt(ctx);
1104 		if (uwq) {
1105 			*msg = uwq->msg;
1106 
1107 			if (uwq->msg.event == UFFD_EVENT_FORK) {
1108 				fork_nctx = (struct userfaultfd_ctx *)
1109 					(unsigned long)
1110 					uwq->msg.arg.reserved.reserved1;
1111 				list_move(&uwq->wq.entry, &fork_event);
1112 				/*
1113 				 * fork_nctx can be freed as soon as
1114 				 * we drop the lock, unless we take a
1115 				 * reference on it.
1116 				 */
1117 				userfaultfd_ctx_get(fork_nctx);
1118 				spin_unlock(&ctx->event_wqh.lock);
1119 				ret = 0;
1120 				break;
1121 			}
1122 
1123 			userfaultfd_event_complete(ctx, uwq);
1124 			spin_unlock(&ctx->event_wqh.lock);
1125 			ret = 0;
1126 			break;
1127 		}
1128 		spin_unlock(&ctx->event_wqh.lock);
1129 
1130 		if (signal_pending(current)) {
1131 			ret = -ERESTARTSYS;
1132 			break;
1133 		}
1134 		if (no_wait) {
1135 			ret = -EAGAIN;
1136 			break;
1137 		}
1138 		spin_unlock_irq(&ctx->fd_wqh.lock);
1139 		schedule();
1140 		spin_lock_irq(&ctx->fd_wqh.lock);
1141 	}
1142 	__remove_wait_queue(&ctx->fd_wqh, &wait);
1143 	__set_current_state(TASK_RUNNING);
1144 	spin_unlock_irq(&ctx->fd_wqh.lock);
1145 
1146 	if (!ret && msg->event == UFFD_EVENT_FORK) {
1147 		ret = resolve_userfault_fork(ctx, fork_nctx, msg);
1148 		spin_lock_irq(&ctx->event_wqh.lock);
1149 		if (!list_empty(&fork_event)) {
1150 			/*
1151 			 * The fork thread didn't abort, so we can
1152 			 * drop the temporary refcount.
1153 			 */
1154 			userfaultfd_ctx_put(fork_nctx);
1155 
1156 			uwq = list_first_entry(&fork_event,
1157 					       typeof(*uwq),
1158 					       wq.entry);
1159 			/*
1160 			 * If fork_event list wasn't empty and in turn
1161 			 * the event wasn't already released by fork
1162 			 * (the event is allocated on fork kernel
1163 			 * stack), put the event back to its place in
1164 			 * the event_wq. fork_event head will be freed
1165 			 * as soon as we return so the event cannot
1166 			 * stay queued there no matter the current
1167 			 * "ret" value.
1168 			 */
1169 			list_del(&uwq->wq.entry);
1170 			__add_wait_queue(&ctx->event_wqh, &uwq->wq);
1171 
1172 			/*
1173 			 * Leave the event in the waitqueue and report
1174 			 * error to userland if we failed to resolve
1175 			 * the userfault fork.
1176 			 */
1177 			if (likely(!ret))
1178 				userfaultfd_event_complete(ctx, uwq);
1179 		} else {
1180 			/*
1181 			 * Here the fork thread aborted and the
1182 			 * refcount from the fork thread on fork_nctx
1183 			 * has already been released. We still hold
1184 			 * the reference we took before releasing the
1185 			 * lock above. If resolve_userfault_fork
1186 			 * failed we've to drop it because the
1187 			 * fork_nctx has to be freed in such case. If
1188 			 * it succeeded we'll hold it because the new
1189 			 * uffd references it.
1190 			 */
1191 			if (ret)
1192 				userfaultfd_ctx_put(fork_nctx);
1193 		}
1194 		spin_unlock_irq(&ctx->event_wqh.lock);
1195 	}
1196 
1197 	return ret;
1198 }
1199 
1200 static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1201 				size_t count, loff_t *ppos)
1202 {
1203 	struct userfaultfd_ctx *ctx = file->private_data;
1204 	ssize_t _ret, ret = 0;
1205 	struct uffd_msg msg;
1206 	int no_wait = file->f_flags & O_NONBLOCK;
1207 
1208 	if (ctx->state == UFFD_STATE_WAIT_API)
1209 		return -EINVAL;
1210 
1211 	for (;;) {
1212 		if (count < sizeof(msg))
1213 			return ret ? ret : -EINVAL;
1214 		_ret = userfaultfd_ctx_read(ctx, no_wait, &msg);
1215 		if (_ret < 0)
1216 			return ret ? ret : _ret;
1217 		if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1218 			return ret ? ret : -EFAULT;
1219 		ret += sizeof(msg);
1220 		buf += sizeof(msg);
1221 		count -= sizeof(msg);
1222 		/*
1223 		 * Allow to read more than one fault at time but only
1224 		 * block if waiting for the very first one.
1225 		 */
1226 		no_wait = O_NONBLOCK;
1227 	}
1228 }
1229 
1230 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1231 			     struct userfaultfd_wake_range *range)
1232 {
1233 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
1234 	/* wake all in the range and autoremove */
1235 	if (waitqueue_active(&ctx->fault_pending_wqh))
1236 		__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1237 				     range);
1238 	if (waitqueue_active(&ctx->fault_wqh))
1239 		__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1240 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1241 }
1242 
1243 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1244 					   struct userfaultfd_wake_range *range)
1245 {
1246 	unsigned seq;
1247 	bool need_wakeup;
1248 
1249 	/*
1250 	 * To be sure waitqueue_active() is not reordered by the CPU
1251 	 * before the pagetable update, use an explicit SMP memory
1252 	 * barrier here. PT lock release or up_read(mmap_sem) still
1253 	 * have release semantics that can allow the
1254 	 * waitqueue_active() to be reordered before the pte update.
1255 	 */
1256 	smp_mb();
1257 
1258 	/*
1259 	 * Use waitqueue_active because it's very frequent to
1260 	 * change the address space atomically even if there are no
1261 	 * userfaults yet. So we take the spinlock only when we're
1262 	 * sure we've userfaults to wake.
1263 	 */
1264 	do {
1265 		seq = read_seqcount_begin(&ctx->refile_seq);
1266 		need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1267 			waitqueue_active(&ctx->fault_wqh);
1268 		cond_resched();
1269 	} while (read_seqcount_retry(&ctx->refile_seq, seq));
1270 	if (need_wakeup)
1271 		__wake_userfault(ctx, range);
1272 }
1273 
1274 static __always_inline int validate_range(struct mm_struct *mm,
1275 					  __u64 *start, __u64 len)
1276 {
1277 	__u64 task_size = mm->task_size;
1278 
1279 	*start = untagged_addr(*start);
1280 
1281 	if (*start & ~PAGE_MASK)
1282 		return -EINVAL;
1283 	if (len & ~PAGE_MASK)
1284 		return -EINVAL;
1285 	if (!len)
1286 		return -EINVAL;
1287 	if (*start < mmap_min_addr)
1288 		return -EINVAL;
1289 	if (*start >= task_size)
1290 		return -EINVAL;
1291 	if (len > task_size - *start)
1292 		return -EINVAL;
1293 	return 0;
1294 }
1295 
1296 static inline bool vma_can_userfault(struct vm_area_struct *vma)
1297 {
1298 	return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
1299 		vma_is_shmem(vma);
1300 }
1301 
1302 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1303 				unsigned long arg)
1304 {
1305 	struct mm_struct *mm = ctx->mm;
1306 	struct vm_area_struct *vma, *prev, *cur;
1307 	int ret;
1308 	struct uffdio_register uffdio_register;
1309 	struct uffdio_register __user *user_uffdio_register;
1310 	unsigned long vm_flags, new_flags;
1311 	bool found;
1312 	bool basic_ioctls;
1313 	unsigned long start, end, vma_end;
1314 
1315 	user_uffdio_register = (struct uffdio_register __user *) arg;
1316 
1317 	ret = -EFAULT;
1318 	if (copy_from_user(&uffdio_register, user_uffdio_register,
1319 			   sizeof(uffdio_register)-sizeof(__u64)))
1320 		goto out;
1321 
1322 	ret = -EINVAL;
1323 	if (!uffdio_register.mode)
1324 		goto out;
1325 	if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING|
1326 				     UFFDIO_REGISTER_MODE_WP))
1327 		goto out;
1328 	vm_flags = 0;
1329 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1330 		vm_flags |= VM_UFFD_MISSING;
1331 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1332 		vm_flags |= VM_UFFD_WP;
1333 		/*
1334 		 * FIXME: remove the below error constraint by
1335 		 * implementing the wprotect tracking mode.
1336 		 */
1337 		ret = -EINVAL;
1338 		goto out;
1339 	}
1340 
1341 	ret = validate_range(mm, &uffdio_register.range.start,
1342 			     uffdio_register.range.len);
1343 	if (ret)
1344 		goto out;
1345 
1346 	start = uffdio_register.range.start;
1347 	end = start + uffdio_register.range.len;
1348 
1349 	ret = -ENOMEM;
1350 	if (!mmget_not_zero(mm))
1351 		goto out;
1352 
1353 	down_write(&mm->mmap_sem);
1354 	if (!mmget_still_valid(mm))
1355 		goto out_unlock;
1356 	vma = find_vma_prev(mm, start, &prev);
1357 	if (!vma)
1358 		goto out_unlock;
1359 
1360 	/* check that there's at least one vma in the range */
1361 	ret = -EINVAL;
1362 	if (vma->vm_start >= end)
1363 		goto out_unlock;
1364 
1365 	/*
1366 	 * If the first vma contains huge pages, make sure start address
1367 	 * is aligned to huge page size.
1368 	 */
1369 	if (is_vm_hugetlb_page(vma)) {
1370 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1371 
1372 		if (start & (vma_hpagesize - 1))
1373 			goto out_unlock;
1374 	}
1375 
1376 	/*
1377 	 * Search for not compatible vmas.
1378 	 */
1379 	found = false;
1380 	basic_ioctls = false;
1381 	for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1382 		cond_resched();
1383 
1384 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1385 		       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1386 
1387 		/* check not compatible vmas */
1388 		ret = -EINVAL;
1389 		if (!vma_can_userfault(cur))
1390 			goto out_unlock;
1391 
1392 		/*
1393 		 * UFFDIO_COPY will fill file holes even without
1394 		 * PROT_WRITE. This check enforces that if this is a
1395 		 * MAP_SHARED, the process has write permission to the backing
1396 		 * file. If VM_MAYWRITE is set it also enforces that on a
1397 		 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1398 		 * F_WRITE_SEAL can be taken until the vma is destroyed.
1399 		 */
1400 		ret = -EPERM;
1401 		if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1402 			goto out_unlock;
1403 
1404 		/*
1405 		 * If this vma contains ending address, and huge pages
1406 		 * check alignment.
1407 		 */
1408 		if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1409 		    end > cur->vm_start) {
1410 			unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1411 
1412 			ret = -EINVAL;
1413 
1414 			if (end & (vma_hpagesize - 1))
1415 				goto out_unlock;
1416 		}
1417 
1418 		/*
1419 		 * Check that this vma isn't already owned by a
1420 		 * different userfaultfd. We can't allow more than one
1421 		 * userfaultfd to own a single vma simultaneously or we
1422 		 * wouldn't know which one to deliver the userfaults to.
1423 		 */
1424 		ret = -EBUSY;
1425 		if (cur->vm_userfaultfd_ctx.ctx &&
1426 		    cur->vm_userfaultfd_ctx.ctx != ctx)
1427 			goto out_unlock;
1428 
1429 		/*
1430 		 * Note vmas containing huge pages
1431 		 */
1432 		if (is_vm_hugetlb_page(cur))
1433 			basic_ioctls = true;
1434 
1435 		found = true;
1436 	}
1437 	BUG_ON(!found);
1438 
1439 	if (vma->vm_start < start)
1440 		prev = vma;
1441 
1442 	ret = 0;
1443 	do {
1444 		cond_resched();
1445 
1446 		BUG_ON(!vma_can_userfault(vma));
1447 		BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1448 		       vma->vm_userfaultfd_ctx.ctx != ctx);
1449 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1450 
1451 		/*
1452 		 * Nothing to do: this vma is already registered into this
1453 		 * userfaultfd and with the right tracking mode too.
1454 		 */
1455 		if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1456 		    (vma->vm_flags & vm_flags) == vm_flags)
1457 			goto skip;
1458 
1459 		if (vma->vm_start > start)
1460 			start = vma->vm_start;
1461 		vma_end = min(end, vma->vm_end);
1462 
1463 		new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
1464 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1465 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1466 				 vma_policy(vma),
1467 				 ((struct vm_userfaultfd_ctx){ ctx }));
1468 		if (prev) {
1469 			vma = prev;
1470 			goto next;
1471 		}
1472 		if (vma->vm_start < start) {
1473 			ret = split_vma(mm, vma, start, 1);
1474 			if (ret)
1475 				break;
1476 		}
1477 		if (vma->vm_end > end) {
1478 			ret = split_vma(mm, vma, end, 0);
1479 			if (ret)
1480 				break;
1481 		}
1482 	next:
1483 		/*
1484 		 * In the vma_merge() successful mprotect-like case 8:
1485 		 * the next vma was merged into the current one and
1486 		 * the current one has not been updated yet.
1487 		 */
1488 		vma->vm_flags = new_flags;
1489 		vma->vm_userfaultfd_ctx.ctx = ctx;
1490 
1491 	skip:
1492 		prev = vma;
1493 		start = vma->vm_end;
1494 		vma = vma->vm_next;
1495 	} while (vma && vma->vm_start < end);
1496 out_unlock:
1497 	up_write(&mm->mmap_sem);
1498 	mmput(mm);
1499 	if (!ret) {
1500 		/*
1501 		 * Now that we scanned all vmas we can already tell
1502 		 * userland which ioctls methods are guaranteed to
1503 		 * succeed on this range.
1504 		 */
1505 		if (put_user(basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1506 			     UFFD_API_RANGE_IOCTLS,
1507 			     &user_uffdio_register->ioctls))
1508 			ret = -EFAULT;
1509 	}
1510 out:
1511 	return ret;
1512 }
1513 
1514 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1515 				  unsigned long arg)
1516 {
1517 	struct mm_struct *mm = ctx->mm;
1518 	struct vm_area_struct *vma, *prev, *cur;
1519 	int ret;
1520 	struct uffdio_range uffdio_unregister;
1521 	unsigned long new_flags;
1522 	bool found;
1523 	unsigned long start, end, vma_end;
1524 	const void __user *buf = (void __user *)arg;
1525 
1526 	ret = -EFAULT;
1527 	if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1528 		goto out;
1529 
1530 	ret = validate_range(mm, &uffdio_unregister.start,
1531 			     uffdio_unregister.len);
1532 	if (ret)
1533 		goto out;
1534 
1535 	start = uffdio_unregister.start;
1536 	end = start + uffdio_unregister.len;
1537 
1538 	ret = -ENOMEM;
1539 	if (!mmget_not_zero(mm))
1540 		goto out;
1541 
1542 	down_write(&mm->mmap_sem);
1543 	if (!mmget_still_valid(mm))
1544 		goto out_unlock;
1545 	vma = find_vma_prev(mm, start, &prev);
1546 	if (!vma)
1547 		goto out_unlock;
1548 
1549 	/* check that there's at least one vma in the range */
1550 	ret = -EINVAL;
1551 	if (vma->vm_start >= end)
1552 		goto out_unlock;
1553 
1554 	/*
1555 	 * If the first vma contains huge pages, make sure start address
1556 	 * is aligned to huge page size.
1557 	 */
1558 	if (is_vm_hugetlb_page(vma)) {
1559 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1560 
1561 		if (start & (vma_hpagesize - 1))
1562 			goto out_unlock;
1563 	}
1564 
1565 	/*
1566 	 * Search for not compatible vmas.
1567 	 */
1568 	found = false;
1569 	ret = -EINVAL;
1570 	for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1571 		cond_resched();
1572 
1573 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1574 		       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1575 
1576 		/*
1577 		 * Check not compatible vmas, not strictly required
1578 		 * here as not compatible vmas cannot have an
1579 		 * userfaultfd_ctx registered on them, but this
1580 		 * provides for more strict behavior to notice
1581 		 * unregistration errors.
1582 		 */
1583 		if (!vma_can_userfault(cur))
1584 			goto out_unlock;
1585 
1586 		found = true;
1587 	}
1588 	BUG_ON(!found);
1589 
1590 	if (vma->vm_start < start)
1591 		prev = vma;
1592 
1593 	ret = 0;
1594 	do {
1595 		cond_resched();
1596 
1597 		BUG_ON(!vma_can_userfault(vma));
1598 
1599 		/*
1600 		 * Nothing to do: this vma is already registered into this
1601 		 * userfaultfd and with the right tracking mode too.
1602 		 */
1603 		if (!vma->vm_userfaultfd_ctx.ctx)
1604 			goto skip;
1605 
1606 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1607 
1608 		if (vma->vm_start > start)
1609 			start = vma->vm_start;
1610 		vma_end = min(end, vma->vm_end);
1611 
1612 		if (userfaultfd_missing(vma)) {
1613 			/*
1614 			 * Wake any concurrent pending userfault while
1615 			 * we unregister, so they will not hang
1616 			 * permanently and it avoids userland to call
1617 			 * UFFDIO_WAKE explicitly.
1618 			 */
1619 			struct userfaultfd_wake_range range;
1620 			range.start = start;
1621 			range.len = vma_end - start;
1622 			wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1623 		}
1624 
1625 		new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
1626 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1627 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1628 				 vma_policy(vma),
1629 				 NULL_VM_UFFD_CTX);
1630 		if (prev) {
1631 			vma = prev;
1632 			goto next;
1633 		}
1634 		if (vma->vm_start < start) {
1635 			ret = split_vma(mm, vma, start, 1);
1636 			if (ret)
1637 				break;
1638 		}
1639 		if (vma->vm_end > end) {
1640 			ret = split_vma(mm, vma, end, 0);
1641 			if (ret)
1642 				break;
1643 		}
1644 	next:
1645 		/*
1646 		 * In the vma_merge() successful mprotect-like case 8:
1647 		 * the next vma was merged into the current one and
1648 		 * the current one has not been updated yet.
1649 		 */
1650 		vma->vm_flags = new_flags;
1651 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1652 
1653 	skip:
1654 		prev = vma;
1655 		start = vma->vm_end;
1656 		vma = vma->vm_next;
1657 	} while (vma && vma->vm_start < end);
1658 out_unlock:
1659 	up_write(&mm->mmap_sem);
1660 	mmput(mm);
1661 out:
1662 	return ret;
1663 }
1664 
1665 /*
1666  * userfaultfd_wake may be used in combination with the
1667  * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1668  */
1669 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1670 			    unsigned long arg)
1671 {
1672 	int ret;
1673 	struct uffdio_range uffdio_wake;
1674 	struct userfaultfd_wake_range range;
1675 	const void __user *buf = (void __user *)arg;
1676 
1677 	ret = -EFAULT;
1678 	if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1679 		goto out;
1680 
1681 	ret = validate_range(ctx->mm, &uffdio_wake.start, uffdio_wake.len);
1682 	if (ret)
1683 		goto out;
1684 
1685 	range.start = uffdio_wake.start;
1686 	range.len = uffdio_wake.len;
1687 
1688 	/*
1689 	 * len == 0 means wake all and we don't want to wake all here,
1690 	 * so check it again to be sure.
1691 	 */
1692 	VM_BUG_ON(!range.len);
1693 
1694 	wake_userfault(ctx, &range);
1695 	ret = 0;
1696 
1697 out:
1698 	return ret;
1699 }
1700 
1701 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1702 			    unsigned long arg)
1703 {
1704 	__s64 ret;
1705 	struct uffdio_copy uffdio_copy;
1706 	struct uffdio_copy __user *user_uffdio_copy;
1707 	struct userfaultfd_wake_range range;
1708 
1709 	user_uffdio_copy = (struct uffdio_copy __user *) arg;
1710 
1711 	ret = -EAGAIN;
1712 	if (READ_ONCE(ctx->mmap_changing))
1713 		goto out;
1714 
1715 	ret = -EFAULT;
1716 	if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1717 			   /* don't copy "copy" last field */
1718 			   sizeof(uffdio_copy)-sizeof(__s64)))
1719 		goto out;
1720 
1721 	ret = validate_range(ctx->mm, &uffdio_copy.dst, uffdio_copy.len);
1722 	if (ret)
1723 		goto out;
1724 	/*
1725 	 * double check for wraparound just in case. copy_from_user()
1726 	 * will later check uffdio_copy.src + uffdio_copy.len to fit
1727 	 * in the userland range.
1728 	 */
1729 	ret = -EINVAL;
1730 	if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1731 		goto out;
1732 	if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE)
1733 		goto out;
1734 	if (mmget_not_zero(ctx->mm)) {
1735 		ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1736 				   uffdio_copy.len, &ctx->mmap_changing);
1737 		mmput(ctx->mm);
1738 	} else {
1739 		return -ESRCH;
1740 	}
1741 	if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1742 		return -EFAULT;
1743 	if (ret < 0)
1744 		goto out;
1745 	BUG_ON(!ret);
1746 	/* len == 0 would wake all */
1747 	range.len = ret;
1748 	if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1749 		range.start = uffdio_copy.dst;
1750 		wake_userfault(ctx, &range);
1751 	}
1752 	ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1753 out:
1754 	return ret;
1755 }
1756 
1757 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1758 				unsigned long arg)
1759 {
1760 	__s64 ret;
1761 	struct uffdio_zeropage uffdio_zeropage;
1762 	struct uffdio_zeropage __user *user_uffdio_zeropage;
1763 	struct userfaultfd_wake_range range;
1764 
1765 	user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1766 
1767 	ret = -EAGAIN;
1768 	if (READ_ONCE(ctx->mmap_changing))
1769 		goto out;
1770 
1771 	ret = -EFAULT;
1772 	if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1773 			   /* don't copy "zeropage" last field */
1774 			   sizeof(uffdio_zeropage)-sizeof(__s64)))
1775 		goto out;
1776 
1777 	ret = validate_range(ctx->mm, &uffdio_zeropage.range.start,
1778 			     uffdio_zeropage.range.len);
1779 	if (ret)
1780 		goto out;
1781 	ret = -EINVAL;
1782 	if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1783 		goto out;
1784 
1785 	if (mmget_not_zero(ctx->mm)) {
1786 		ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
1787 				     uffdio_zeropage.range.len,
1788 				     &ctx->mmap_changing);
1789 		mmput(ctx->mm);
1790 	} else {
1791 		return -ESRCH;
1792 	}
1793 	if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1794 		return -EFAULT;
1795 	if (ret < 0)
1796 		goto out;
1797 	/* len == 0 would wake all */
1798 	BUG_ON(!ret);
1799 	range.len = ret;
1800 	if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1801 		range.start = uffdio_zeropage.range.start;
1802 		wake_userfault(ctx, &range);
1803 	}
1804 	ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1805 out:
1806 	return ret;
1807 }
1808 
1809 static inline unsigned int uffd_ctx_features(__u64 user_features)
1810 {
1811 	/*
1812 	 * For the current set of features the bits just coincide
1813 	 */
1814 	return (unsigned int)user_features;
1815 }
1816 
1817 /*
1818  * userland asks for a certain API version and we return which bits
1819  * and ioctl commands are implemented in this kernel for such API
1820  * version or -EINVAL if unknown.
1821  */
1822 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1823 			   unsigned long arg)
1824 {
1825 	struct uffdio_api uffdio_api;
1826 	void __user *buf = (void __user *)arg;
1827 	int ret;
1828 	__u64 features;
1829 
1830 	ret = -EINVAL;
1831 	if (ctx->state != UFFD_STATE_WAIT_API)
1832 		goto out;
1833 	ret = -EFAULT;
1834 	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1835 		goto out;
1836 	features = uffdio_api.features;
1837 	if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
1838 		memset(&uffdio_api, 0, sizeof(uffdio_api));
1839 		if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1840 			goto out;
1841 		ret = -EINVAL;
1842 		goto out;
1843 	}
1844 	/* report all available features and ioctls to userland */
1845 	uffdio_api.features = UFFD_API_FEATURES;
1846 	uffdio_api.ioctls = UFFD_API_IOCTLS;
1847 	ret = -EFAULT;
1848 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1849 		goto out;
1850 	ctx->state = UFFD_STATE_RUNNING;
1851 	/* only enable the requested features for this uffd context */
1852 	ctx->features = uffd_ctx_features(features);
1853 	ret = 0;
1854 out:
1855 	return ret;
1856 }
1857 
1858 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1859 			      unsigned long arg)
1860 {
1861 	int ret = -EINVAL;
1862 	struct userfaultfd_ctx *ctx = file->private_data;
1863 
1864 	if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API)
1865 		return -EINVAL;
1866 
1867 	switch(cmd) {
1868 	case UFFDIO_API:
1869 		ret = userfaultfd_api(ctx, arg);
1870 		break;
1871 	case UFFDIO_REGISTER:
1872 		ret = userfaultfd_register(ctx, arg);
1873 		break;
1874 	case UFFDIO_UNREGISTER:
1875 		ret = userfaultfd_unregister(ctx, arg);
1876 		break;
1877 	case UFFDIO_WAKE:
1878 		ret = userfaultfd_wake(ctx, arg);
1879 		break;
1880 	case UFFDIO_COPY:
1881 		ret = userfaultfd_copy(ctx, arg);
1882 		break;
1883 	case UFFDIO_ZEROPAGE:
1884 		ret = userfaultfd_zeropage(ctx, arg);
1885 		break;
1886 	}
1887 	return ret;
1888 }
1889 
1890 #ifdef CONFIG_PROC_FS
1891 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
1892 {
1893 	struct userfaultfd_ctx *ctx = f->private_data;
1894 	wait_queue_entry_t *wq;
1895 	unsigned long pending = 0, total = 0;
1896 
1897 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
1898 	list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
1899 		pending++;
1900 		total++;
1901 	}
1902 	list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
1903 		total++;
1904 	}
1905 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1906 
1907 	/*
1908 	 * If more protocols will be added, there will be all shown
1909 	 * separated by a space. Like this:
1910 	 *	protocols: aa:... bb:...
1911 	 */
1912 	seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
1913 		   pending, total, UFFD_API, ctx->features,
1914 		   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
1915 }
1916 #endif
1917 
1918 static const struct file_operations userfaultfd_fops = {
1919 #ifdef CONFIG_PROC_FS
1920 	.show_fdinfo	= userfaultfd_show_fdinfo,
1921 #endif
1922 	.release	= userfaultfd_release,
1923 	.poll		= userfaultfd_poll,
1924 	.read		= userfaultfd_read,
1925 	.unlocked_ioctl = userfaultfd_ioctl,
1926 	.compat_ioctl	= userfaultfd_ioctl,
1927 	.llseek		= noop_llseek,
1928 };
1929 
1930 static void init_once_userfaultfd_ctx(void *mem)
1931 {
1932 	struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
1933 
1934 	init_waitqueue_head(&ctx->fault_pending_wqh);
1935 	init_waitqueue_head(&ctx->fault_wqh);
1936 	init_waitqueue_head(&ctx->event_wqh);
1937 	init_waitqueue_head(&ctx->fd_wqh);
1938 	seqcount_init(&ctx->refile_seq);
1939 }
1940 
1941 SYSCALL_DEFINE1(userfaultfd, int, flags)
1942 {
1943 	struct userfaultfd_ctx *ctx;
1944 	int fd;
1945 
1946 	if (!sysctl_unprivileged_userfaultfd && !capable(CAP_SYS_PTRACE))
1947 		return -EPERM;
1948 
1949 	BUG_ON(!current->mm);
1950 
1951 	/* Check the UFFD_* constants for consistency.  */
1952 	BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
1953 	BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
1954 
1955 	if (flags & ~UFFD_SHARED_FCNTL_FLAGS)
1956 		return -EINVAL;
1957 
1958 	ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
1959 	if (!ctx)
1960 		return -ENOMEM;
1961 
1962 	refcount_set(&ctx->refcount, 1);
1963 	ctx->flags = flags;
1964 	ctx->features = 0;
1965 	ctx->state = UFFD_STATE_WAIT_API;
1966 	ctx->released = false;
1967 	ctx->mmap_changing = false;
1968 	ctx->mm = current->mm;
1969 	/* prevent the mm struct to be freed */
1970 	mmgrab(ctx->mm);
1971 
1972 	fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, ctx,
1973 			      O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
1974 	if (fd < 0) {
1975 		mmdrop(ctx->mm);
1976 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
1977 	}
1978 	return fd;
1979 }
1980 
1981 static int __init userfaultfd_init(void)
1982 {
1983 	userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
1984 						sizeof(struct userfaultfd_ctx),
1985 						0,
1986 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1987 						init_once_userfaultfd_ctx);
1988 	return 0;
1989 }
1990 __initcall(userfaultfd_init);
1991