xref: /openbmc/linux/fs/userfaultfd.c (revision 1504b6f9)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  fs/userfaultfd.c
4  *
5  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
6  *  Copyright (C) 2008-2009 Red Hat, Inc.
7  *  Copyright (C) 2015  Red Hat, Inc.
8  *
9  *  Some part derived from fs/eventfd.c (anon inode setup) and
10  *  mm/ksm.c (mm hashing).
11  */
12 
13 #include <linux/list.h>
14 #include <linux/hashtable.h>
15 #include <linux/sched/signal.h>
16 #include <linux/sched/mm.h>
17 #include <linux/mm.h>
18 #include <linux/mm_inline.h>
19 #include <linux/mmu_notifier.h>
20 #include <linux/poll.h>
21 #include <linux/slab.h>
22 #include <linux/seq_file.h>
23 #include <linux/file.h>
24 #include <linux/bug.h>
25 #include <linux/anon_inodes.h>
26 #include <linux/syscalls.h>
27 #include <linux/userfaultfd_k.h>
28 #include <linux/mempolicy.h>
29 #include <linux/ioctl.h>
30 #include <linux/security.h>
31 #include <linux/hugetlb.h>
32 #include <linux/swapops.h>
33 #include <linux/miscdevice.h>
34 
35 int sysctl_unprivileged_userfaultfd __read_mostly;
36 
37 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
38 
39 /*
40  * Start with fault_pending_wqh and fault_wqh so they're more likely
41  * to be in the same cacheline.
42  *
43  * Locking order:
44  *	fd_wqh.lock
45  *		fault_pending_wqh.lock
46  *			fault_wqh.lock
47  *		event_wqh.lock
48  *
49  * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
50  * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
51  * also taken in IRQ context.
52  */
53 struct userfaultfd_ctx {
54 	/* waitqueue head for the pending (i.e. not read) userfaults */
55 	wait_queue_head_t fault_pending_wqh;
56 	/* waitqueue head for the userfaults */
57 	wait_queue_head_t fault_wqh;
58 	/* waitqueue head for the pseudo fd to wakeup poll/read */
59 	wait_queue_head_t fd_wqh;
60 	/* waitqueue head for events */
61 	wait_queue_head_t event_wqh;
62 	/* a refile sequence protected by fault_pending_wqh lock */
63 	seqcount_spinlock_t refile_seq;
64 	/* pseudo fd refcounting */
65 	refcount_t refcount;
66 	/* userfaultfd syscall flags */
67 	unsigned int flags;
68 	/* features requested from the userspace */
69 	unsigned int features;
70 	/* released */
71 	bool released;
72 	/* memory mappings are changing because of non-cooperative event */
73 	atomic_t mmap_changing;
74 	/* mm with one ore more vmas attached to this userfaultfd_ctx */
75 	struct mm_struct *mm;
76 };
77 
78 struct userfaultfd_fork_ctx {
79 	struct userfaultfd_ctx *orig;
80 	struct userfaultfd_ctx *new;
81 	struct list_head list;
82 };
83 
84 struct userfaultfd_unmap_ctx {
85 	struct userfaultfd_ctx *ctx;
86 	unsigned long start;
87 	unsigned long end;
88 	struct list_head list;
89 };
90 
91 struct userfaultfd_wait_queue {
92 	struct uffd_msg msg;
93 	wait_queue_entry_t wq;
94 	struct userfaultfd_ctx *ctx;
95 	bool waken;
96 };
97 
98 struct userfaultfd_wake_range {
99 	unsigned long start;
100 	unsigned long len;
101 };
102 
103 /* internal indication that UFFD_API ioctl was successfully executed */
104 #define UFFD_FEATURE_INITIALIZED		(1u << 31)
105 
106 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
107 {
108 	return ctx->features & UFFD_FEATURE_INITIALIZED;
109 }
110 
111 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
112 				     int wake_flags, void *key)
113 {
114 	struct userfaultfd_wake_range *range = key;
115 	int ret;
116 	struct userfaultfd_wait_queue *uwq;
117 	unsigned long start, len;
118 
119 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
120 	ret = 0;
121 	/* len == 0 means wake all */
122 	start = range->start;
123 	len = range->len;
124 	if (len && (start > uwq->msg.arg.pagefault.address ||
125 		    start + len <= uwq->msg.arg.pagefault.address))
126 		goto out;
127 	WRITE_ONCE(uwq->waken, true);
128 	/*
129 	 * The Program-Order guarantees provided by the scheduler
130 	 * ensure uwq->waken is visible before the task is woken.
131 	 */
132 	ret = wake_up_state(wq->private, mode);
133 	if (ret) {
134 		/*
135 		 * Wake only once, autoremove behavior.
136 		 *
137 		 * After the effect of list_del_init is visible to the other
138 		 * CPUs, the waitqueue may disappear from under us, see the
139 		 * !list_empty_careful() in handle_userfault().
140 		 *
141 		 * try_to_wake_up() has an implicit smp_mb(), and the
142 		 * wq->private is read before calling the extern function
143 		 * "wake_up_state" (which in turns calls try_to_wake_up).
144 		 */
145 		list_del_init(&wq->entry);
146 	}
147 out:
148 	return ret;
149 }
150 
151 /**
152  * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
153  * context.
154  * @ctx: [in] Pointer to the userfaultfd context.
155  */
156 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
157 {
158 	refcount_inc(&ctx->refcount);
159 }
160 
161 /**
162  * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
163  * context.
164  * @ctx: [in] Pointer to userfaultfd context.
165  *
166  * The userfaultfd context reference must have been previously acquired either
167  * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
168  */
169 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
170 {
171 	if (refcount_dec_and_test(&ctx->refcount)) {
172 		VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
173 		VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
174 		VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
175 		VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
176 		VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
177 		VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
178 		VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
179 		VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
180 		mmdrop(ctx->mm);
181 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
182 	}
183 }
184 
185 static inline void msg_init(struct uffd_msg *msg)
186 {
187 	BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
188 	/*
189 	 * Must use memset to zero out the paddings or kernel data is
190 	 * leaked to userland.
191 	 */
192 	memset(msg, 0, sizeof(struct uffd_msg));
193 }
194 
195 static inline struct uffd_msg userfault_msg(unsigned long address,
196 					    unsigned long real_address,
197 					    unsigned int flags,
198 					    unsigned long reason,
199 					    unsigned int features)
200 {
201 	struct uffd_msg msg;
202 
203 	msg_init(&msg);
204 	msg.event = UFFD_EVENT_PAGEFAULT;
205 
206 	msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?
207 				    real_address : address;
208 
209 	/*
210 	 * These flags indicate why the userfault occurred:
211 	 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
212 	 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
213 	 * - Neither of these flags being set indicates a MISSING fault.
214 	 *
215 	 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
216 	 * fault. Otherwise, it was a read fault.
217 	 */
218 	if (flags & FAULT_FLAG_WRITE)
219 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
220 	if (reason & VM_UFFD_WP)
221 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
222 	if (reason & VM_UFFD_MINOR)
223 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
224 	if (features & UFFD_FEATURE_THREAD_ID)
225 		msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
226 	return msg;
227 }
228 
229 #ifdef CONFIG_HUGETLB_PAGE
230 /*
231  * Same functionality as userfaultfd_must_wait below with modifications for
232  * hugepmd ranges.
233  */
234 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
235 					 struct vm_area_struct *vma,
236 					 unsigned long address,
237 					 unsigned long flags,
238 					 unsigned long reason)
239 {
240 	struct mm_struct *mm = ctx->mm;
241 	pte_t *ptep, pte;
242 	bool ret = true;
243 
244 	mmap_assert_locked(mm);
245 
246 	ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
247 
248 	if (!ptep)
249 		goto out;
250 
251 	ret = false;
252 	pte = huge_ptep_get(ptep);
253 
254 	/*
255 	 * Lockless access: we're in a wait_event so it's ok if it
256 	 * changes under us.  PTE markers should be handled the same as none
257 	 * ptes here.
258 	 */
259 	if (huge_pte_none_mostly(pte))
260 		ret = true;
261 	if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
262 		ret = true;
263 out:
264 	return ret;
265 }
266 #else
267 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
268 					 struct vm_area_struct *vma,
269 					 unsigned long address,
270 					 unsigned long flags,
271 					 unsigned long reason)
272 {
273 	return false;	/* should never get here */
274 }
275 #endif /* CONFIG_HUGETLB_PAGE */
276 
277 /*
278  * Verify the pagetables are still not ok after having reigstered into
279  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
280  * userfault that has already been resolved, if userfaultfd_read and
281  * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
282  * threads.
283  */
284 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
285 					 unsigned long address,
286 					 unsigned long flags,
287 					 unsigned long reason)
288 {
289 	struct mm_struct *mm = ctx->mm;
290 	pgd_t *pgd;
291 	p4d_t *p4d;
292 	pud_t *pud;
293 	pmd_t *pmd, _pmd;
294 	pte_t *pte;
295 	bool ret = true;
296 
297 	mmap_assert_locked(mm);
298 
299 	pgd = pgd_offset(mm, address);
300 	if (!pgd_present(*pgd))
301 		goto out;
302 	p4d = p4d_offset(pgd, address);
303 	if (!p4d_present(*p4d))
304 		goto out;
305 	pud = pud_offset(p4d, address);
306 	if (!pud_present(*pud))
307 		goto out;
308 	pmd = pmd_offset(pud, address);
309 	/*
310 	 * READ_ONCE must function as a barrier with narrower scope
311 	 * and it must be equivalent to:
312 	 *	_pmd = *pmd; barrier();
313 	 *
314 	 * This is to deal with the instability (as in
315 	 * pmd_trans_unstable) of the pmd.
316 	 */
317 	_pmd = READ_ONCE(*pmd);
318 	if (pmd_none(_pmd))
319 		goto out;
320 
321 	ret = false;
322 	if (!pmd_present(_pmd))
323 		goto out;
324 
325 	if (pmd_trans_huge(_pmd)) {
326 		if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
327 			ret = true;
328 		goto out;
329 	}
330 
331 	/*
332 	 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
333 	 * and use the standard pte_offset_map() instead of parsing _pmd.
334 	 */
335 	pte = pte_offset_map(pmd, address);
336 	/*
337 	 * Lockless access: we're in a wait_event so it's ok if it
338 	 * changes under us.  PTE markers should be handled the same as none
339 	 * ptes here.
340 	 */
341 	if (pte_none_mostly(*pte))
342 		ret = true;
343 	if (!pte_write(*pte) && (reason & VM_UFFD_WP))
344 		ret = true;
345 	pte_unmap(pte);
346 
347 out:
348 	return ret;
349 }
350 
351 static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
352 {
353 	if (flags & FAULT_FLAG_INTERRUPTIBLE)
354 		return TASK_INTERRUPTIBLE;
355 
356 	if (flags & FAULT_FLAG_KILLABLE)
357 		return TASK_KILLABLE;
358 
359 	return TASK_UNINTERRUPTIBLE;
360 }
361 
362 /*
363  * The locking rules involved in returning VM_FAULT_RETRY depending on
364  * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
365  * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
366  * recommendation in __lock_page_or_retry is not an understatement.
367  *
368  * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
369  * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
370  * not set.
371  *
372  * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
373  * set, VM_FAULT_RETRY can still be returned if and only if there are
374  * fatal_signal_pending()s, and the mmap_lock must be released before
375  * returning it.
376  */
377 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
378 {
379 	struct mm_struct *mm = vmf->vma->vm_mm;
380 	struct userfaultfd_ctx *ctx;
381 	struct userfaultfd_wait_queue uwq;
382 	vm_fault_t ret = VM_FAULT_SIGBUS;
383 	bool must_wait;
384 	unsigned int blocking_state;
385 
386 	/*
387 	 * We don't do userfault handling for the final child pid update.
388 	 *
389 	 * We also don't do userfault handling during
390 	 * coredumping. hugetlbfs has the special
391 	 * follow_hugetlb_page() to skip missing pages in the
392 	 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
393 	 * the no_page_table() helper in follow_page_mask(), but the
394 	 * shmem_vm_ops->fault method is invoked even during
395 	 * coredumping without mmap_lock and it ends up here.
396 	 */
397 	if (current->flags & (PF_EXITING|PF_DUMPCORE))
398 		goto out;
399 
400 	/*
401 	 * Coredumping runs without mmap_lock so we can only check that
402 	 * the mmap_lock is held, if PF_DUMPCORE was not set.
403 	 */
404 	mmap_assert_locked(mm);
405 
406 	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
407 	if (!ctx)
408 		goto out;
409 
410 	BUG_ON(ctx->mm != mm);
411 
412 	/* Any unrecognized flag is a bug. */
413 	VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
414 	/* 0 or > 1 flags set is a bug; we expect exactly 1. */
415 	VM_BUG_ON(!reason || (reason & (reason - 1)));
416 
417 	if (ctx->features & UFFD_FEATURE_SIGBUS)
418 		goto out;
419 	if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
420 		goto out;
421 
422 	/*
423 	 * If it's already released don't get it. This avoids to loop
424 	 * in __get_user_pages if userfaultfd_release waits on the
425 	 * caller of handle_userfault to release the mmap_lock.
426 	 */
427 	if (unlikely(READ_ONCE(ctx->released))) {
428 		/*
429 		 * Don't return VM_FAULT_SIGBUS in this case, so a non
430 		 * cooperative manager can close the uffd after the
431 		 * last UFFDIO_COPY, without risking to trigger an
432 		 * involuntary SIGBUS if the process was starting the
433 		 * userfaultfd while the userfaultfd was still armed
434 		 * (but after the last UFFDIO_COPY). If the uffd
435 		 * wasn't already closed when the userfault reached
436 		 * this point, that would normally be solved by
437 		 * userfaultfd_must_wait returning 'false'.
438 		 *
439 		 * If we were to return VM_FAULT_SIGBUS here, the non
440 		 * cooperative manager would be instead forced to
441 		 * always call UFFDIO_UNREGISTER before it can safely
442 		 * close the uffd.
443 		 */
444 		ret = VM_FAULT_NOPAGE;
445 		goto out;
446 	}
447 
448 	/*
449 	 * Check that we can return VM_FAULT_RETRY.
450 	 *
451 	 * NOTE: it should become possible to return VM_FAULT_RETRY
452 	 * even if FAULT_FLAG_TRIED is set without leading to gup()
453 	 * -EBUSY failures, if the userfaultfd is to be extended for
454 	 * VM_UFFD_WP tracking and we intend to arm the userfault
455 	 * without first stopping userland access to the memory. For
456 	 * VM_UFFD_MISSING userfaults this is enough for now.
457 	 */
458 	if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
459 		/*
460 		 * Validate the invariant that nowait must allow retry
461 		 * to be sure not to return SIGBUS erroneously on
462 		 * nowait invocations.
463 		 */
464 		BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
465 #ifdef CONFIG_DEBUG_VM
466 		if (printk_ratelimit()) {
467 			printk(KERN_WARNING
468 			       "FAULT_FLAG_ALLOW_RETRY missing %x\n",
469 			       vmf->flags);
470 			dump_stack();
471 		}
472 #endif
473 		goto out;
474 	}
475 
476 	/*
477 	 * Handle nowait, not much to do other than tell it to retry
478 	 * and wait.
479 	 */
480 	ret = VM_FAULT_RETRY;
481 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
482 		goto out;
483 
484 	/* take the reference before dropping the mmap_lock */
485 	userfaultfd_ctx_get(ctx);
486 
487 	init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
488 	uwq.wq.private = current;
489 	uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,
490 				reason, ctx->features);
491 	uwq.ctx = ctx;
492 	uwq.waken = false;
493 
494 	blocking_state = userfaultfd_get_blocking_state(vmf->flags);
495 
496 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
497 	/*
498 	 * After the __add_wait_queue the uwq is visible to userland
499 	 * through poll/read().
500 	 */
501 	__add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
502 	/*
503 	 * The smp_mb() after __set_current_state prevents the reads
504 	 * following the spin_unlock to happen before the list_add in
505 	 * __add_wait_queue.
506 	 */
507 	set_current_state(blocking_state);
508 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
509 
510 	if (!is_vm_hugetlb_page(vmf->vma))
511 		must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
512 						  reason);
513 	else
514 		must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
515 						       vmf->address,
516 						       vmf->flags, reason);
517 	mmap_read_unlock(mm);
518 
519 	if (likely(must_wait && !READ_ONCE(ctx->released))) {
520 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
521 		schedule();
522 	}
523 
524 	__set_current_state(TASK_RUNNING);
525 
526 	/*
527 	 * Here we race with the list_del; list_add in
528 	 * userfaultfd_ctx_read(), however because we don't ever run
529 	 * list_del_init() to refile across the two lists, the prev
530 	 * and next pointers will never point to self. list_add also
531 	 * would never let any of the two pointers to point to
532 	 * self. So list_empty_careful won't risk to see both pointers
533 	 * pointing to self at any time during the list refile. The
534 	 * only case where list_del_init() is called is the full
535 	 * removal in the wake function and there we don't re-list_add
536 	 * and it's fine not to block on the spinlock. The uwq on this
537 	 * kernel stack can be released after the list_del_init.
538 	 */
539 	if (!list_empty_careful(&uwq.wq.entry)) {
540 		spin_lock_irq(&ctx->fault_pending_wqh.lock);
541 		/*
542 		 * No need of list_del_init(), the uwq on the stack
543 		 * will be freed shortly anyway.
544 		 */
545 		list_del(&uwq.wq.entry);
546 		spin_unlock_irq(&ctx->fault_pending_wqh.lock);
547 	}
548 
549 	/*
550 	 * ctx may go away after this if the userfault pseudo fd is
551 	 * already released.
552 	 */
553 	userfaultfd_ctx_put(ctx);
554 
555 out:
556 	return ret;
557 }
558 
559 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
560 					      struct userfaultfd_wait_queue *ewq)
561 {
562 	struct userfaultfd_ctx *release_new_ctx;
563 
564 	if (WARN_ON_ONCE(current->flags & PF_EXITING))
565 		goto out;
566 
567 	ewq->ctx = ctx;
568 	init_waitqueue_entry(&ewq->wq, current);
569 	release_new_ctx = NULL;
570 
571 	spin_lock_irq(&ctx->event_wqh.lock);
572 	/*
573 	 * After the __add_wait_queue the uwq is visible to userland
574 	 * through poll/read().
575 	 */
576 	__add_wait_queue(&ctx->event_wqh, &ewq->wq);
577 	for (;;) {
578 		set_current_state(TASK_KILLABLE);
579 		if (ewq->msg.event == 0)
580 			break;
581 		if (READ_ONCE(ctx->released) ||
582 		    fatal_signal_pending(current)) {
583 			/*
584 			 * &ewq->wq may be queued in fork_event, but
585 			 * __remove_wait_queue ignores the head
586 			 * parameter. It would be a problem if it
587 			 * didn't.
588 			 */
589 			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
590 			if (ewq->msg.event == UFFD_EVENT_FORK) {
591 				struct userfaultfd_ctx *new;
592 
593 				new = (struct userfaultfd_ctx *)
594 					(unsigned long)
595 					ewq->msg.arg.reserved.reserved1;
596 				release_new_ctx = new;
597 			}
598 			break;
599 		}
600 
601 		spin_unlock_irq(&ctx->event_wqh.lock);
602 
603 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
604 		schedule();
605 
606 		spin_lock_irq(&ctx->event_wqh.lock);
607 	}
608 	__set_current_state(TASK_RUNNING);
609 	spin_unlock_irq(&ctx->event_wqh.lock);
610 
611 	if (release_new_ctx) {
612 		struct vm_area_struct *vma;
613 		struct mm_struct *mm = release_new_ctx->mm;
614 		VMA_ITERATOR(vmi, mm, 0);
615 
616 		/* the various vma->vm_userfaultfd_ctx still points to it */
617 		mmap_write_lock(mm);
618 		for_each_vma(vmi, vma) {
619 			if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
620 				vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
621 				vma->vm_flags &= ~__VM_UFFD_FLAGS;
622 			}
623 		}
624 		mmap_write_unlock(mm);
625 
626 		userfaultfd_ctx_put(release_new_ctx);
627 	}
628 
629 	/*
630 	 * ctx may go away after this if the userfault pseudo fd is
631 	 * already released.
632 	 */
633 out:
634 	atomic_dec(&ctx->mmap_changing);
635 	VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);
636 	userfaultfd_ctx_put(ctx);
637 }
638 
639 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
640 				       struct userfaultfd_wait_queue *ewq)
641 {
642 	ewq->msg.event = 0;
643 	wake_up_locked(&ctx->event_wqh);
644 	__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
645 }
646 
647 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
648 {
649 	struct userfaultfd_ctx *ctx = NULL, *octx;
650 	struct userfaultfd_fork_ctx *fctx;
651 
652 	octx = vma->vm_userfaultfd_ctx.ctx;
653 	if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
654 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
655 		vma->vm_flags &= ~__VM_UFFD_FLAGS;
656 		return 0;
657 	}
658 
659 	list_for_each_entry(fctx, fcs, list)
660 		if (fctx->orig == octx) {
661 			ctx = fctx->new;
662 			break;
663 		}
664 
665 	if (!ctx) {
666 		fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
667 		if (!fctx)
668 			return -ENOMEM;
669 
670 		ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
671 		if (!ctx) {
672 			kfree(fctx);
673 			return -ENOMEM;
674 		}
675 
676 		refcount_set(&ctx->refcount, 1);
677 		ctx->flags = octx->flags;
678 		ctx->features = octx->features;
679 		ctx->released = false;
680 		atomic_set(&ctx->mmap_changing, 0);
681 		ctx->mm = vma->vm_mm;
682 		mmgrab(ctx->mm);
683 
684 		userfaultfd_ctx_get(octx);
685 		atomic_inc(&octx->mmap_changing);
686 		fctx->orig = octx;
687 		fctx->new = ctx;
688 		list_add_tail(&fctx->list, fcs);
689 	}
690 
691 	vma->vm_userfaultfd_ctx.ctx = ctx;
692 	return 0;
693 }
694 
695 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
696 {
697 	struct userfaultfd_ctx *ctx = fctx->orig;
698 	struct userfaultfd_wait_queue ewq;
699 
700 	msg_init(&ewq.msg);
701 
702 	ewq.msg.event = UFFD_EVENT_FORK;
703 	ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
704 
705 	userfaultfd_event_wait_completion(ctx, &ewq);
706 }
707 
708 void dup_userfaultfd_complete(struct list_head *fcs)
709 {
710 	struct userfaultfd_fork_ctx *fctx, *n;
711 
712 	list_for_each_entry_safe(fctx, n, fcs, list) {
713 		dup_fctx(fctx);
714 		list_del(&fctx->list);
715 		kfree(fctx);
716 	}
717 }
718 
719 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
720 			     struct vm_userfaultfd_ctx *vm_ctx)
721 {
722 	struct userfaultfd_ctx *ctx;
723 
724 	ctx = vma->vm_userfaultfd_ctx.ctx;
725 
726 	if (!ctx)
727 		return;
728 
729 	if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
730 		vm_ctx->ctx = ctx;
731 		userfaultfd_ctx_get(ctx);
732 		atomic_inc(&ctx->mmap_changing);
733 	} else {
734 		/* Drop uffd context if remap feature not enabled */
735 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
736 		vma->vm_flags &= ~__VM_UFFD_FLAGS;
737 	}
738 }
739 
740 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
741 				 unsigned long from, unsigned long to,
742 				 unsigned long len)
743 {
744 	struct userfaultfd_ctx *ctx = vm_ctx->ctx;
745 	struct userfaultfd_wait_queue ewq;
746 
747 	if (!ctx)
748 		return;
749 
750 	if (to & ~PAGE_MASK) {
751 		userfaultfd_ctx_put(ctx);
752 		return;
753 	}
754 
755 	msg_init(&ewq.msg);
756 
757 	ewq.msg.event = UFFD_EVENT_REMAP;
758 	ewq.msg.arg.remap.from = from;
759 	ewq.msg.arg.remap.to = to;
760 	ewq.msg.arg.remap.len = len;
761 
762 	userfaultfd_event_wait_completion(ctx, &ewq);
763 }
764 
765 bool userfaultfd_remove(struct vm_area_struct *vma,
766 			unsigned long start, unsigned long end)
767 {
768 	struct mm_struct *mm = vma->vm_mm;
769 	struct userfaultfd_ctx *ctx;
770 	struct userfaultfd_wait_queue ewq;
771 
772 	ctx = vma->vm_userfaultfd_ctx.ctx;
773 	if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
774 		return true;
775 
776 	userfaultfd_ctx_get(ctx);
777 	atomic_inc(&ctx->mmap_changing);
778 	mmap_read_unlock(mm);
779 
780 	msg_init(&ewq.msg);
781 
782 	ewq.msg.event = UFFD_EVENT_REMOVE;
783 	ewq.msg.arg.remove.start = start;
784 	ewq.msg.arg.remove.end = end;
785 
786 	userfaultfd_event_wait_completion(ctx, &ewq);
787 
788 	return false;
789 }
790 
791 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
792 			  unsigned long start, unsigned long end)
793 {
794 	struct userfaultfd_unmap_ctx *unmap_ctx;
795 
796 	list_for_each_entry(unmap_ctx, unmaps, list)
797 		if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
798 		    unmap_ctx->end == end)
799 			return true;
800 
801 	return false;
802 }
803 
804 int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
805 			   unsigned long end, struct list_head *unmaps)
806 {
807 	VMA_ITERATOR(vmi, mm, start);
808 	struct vm_area_struct *vma;
809 
810 	for_each_vma_range(vmi, vma, end) {
811 		struct userfaultfd_unmap_ctx *unmap_ctx;
812 		struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
813 
814 		if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
815 		    has_unmap_ctx(ctx, unmaps, start, end))
816 			continue;
817 
818 		unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
819 		if (!unmap_ctx)
820 			return -ENOMEM;
821 
822 		userfaultfd_ctx_get(ctx);
823 		atomic_inc(&ctx->mmap_changing);
824 		unmap_ctx->ctx = ctx;
825 		unmap_ctx->start = start;
826 		unmap_ctx->end = end;
827 		list_add_tail(&unmap_ctx->list, unmaps);
828 	}
829 
830 	return 0;
831 }
832 
833 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
834 {
835 	struct userfaultfd_unmap_ctx *ctx, *n;
836 	struct userfaultfd_wait_queue ewq;
837 
838 	list_for_each_entry_safe(ctx, n, uf, list) {
839 		msg_init(&ewq.msg);
840 
841 		ewq.msg.event = UFFD_EVENT_UNMAP;
842 		ewq.msg.arg.remove.start = ctx->start;
843 		ewq.msg.arg.remove.end = ctx->end;
844 
845 		userfaultfd_event_wait_completion(ctx->ctx, &ewq);
846 
847 		list_del(&ctx->list);
848 		kfree(ctx);
849 	}
850 }
851 
852 static int userfaultfd_release(struct inode *inode, struct file *file)
853 {
854 	struct userfaultfd_ctx *ctx = file->private_data;
855 	struct mm_struct *mm = ctx->mm;
856 	struct vm_area_struct *vma, *prev;
857 	/* len == 0 means wake all */
858 	struct userfaultfd_wake_range range = { .len = 0, };
859 	unsigned long new_flags;
860 	MA_STATE(mas, &mm->mm_mt, 0, 0);
861 
862 	WRITE_ONCE(ctx->released, true);
863 
864 	if (!mmget_not_zero(mm))
865 		goto wakeup;
866 
867 	/*
868 	 * Flush page faults out of all CPUs. NOTE: all page faults
869 	 * must be retried without returning VM_FAULT_SIGBUS if
870 	 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
871 	 * changes while handle_userfault released the mmap_lock. So
872 	 * it's critical that released is set to true (above), before
873 	 * taking the mmap_lock for writing.
874 	 */
875 	mmap_write_lock(mm);
876 	prev = NULL;
877 	mas_for_each(&mas, vma, ULONG_MAX) {
878 		cond_resched();
879 		BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
880 		       !!(vma->vm_flags & __VM_UFFD_FLAGS));
881 		if (vma->vm_userfaultfd_ctx.ctx != ctx) {
882 			prev = vma;
883 			continue;
884 		}
885 		new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
886 		prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
887 				 new_flags, vma->anon_vma,
888 				 vma->vm_file, vma->vm_pgoff,
889 				 vma_policy(vma),
890 				 NULL_VM_UFFD_CTX, anon_vma_name(vma));
891 		if (prev) {
892 			mas_pause(&mas);
893 			vma = prev;
894 		} else {
895 			prev = vma;
896 		}
897 
898 		vma->vm_flags = new_flags;
899 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
900 	}
901 	mmap_write_unlock(mm);
902 	mmput(mm);
903 wakeup:
904 	/*
905 	 * After no new page faults can wait on this fault_*wqh, flush
906 	 * the last page faults that may have been already waiting on
907 	 * the fault_*wqh.
908 	 */
909 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
910 	__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
911 	__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
912 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
913 
914 	/* Flush pending events that may still wait on event_wqh */
915 	wake_up_all(&ctx->event_wqh);
916 
917 	wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
918 	userfaultfd_ctx_put(ctx);
919 	return 0;
920 }
921 
922 /* fault_pending_wqh.lock must be hold by the caller */
923 static inline struct userfaultfd_wait_queue *find_userfault_in(
924 		wait_queue_head_t *wqh)
925 {
926 	wait_queue_entry_t *wq;
927 	struct userfaultfd_wait_queue *uwq;
928 
929 	lockdep_assert_held(&wqh->lock);
930 
931 	uwq = NULL;
932 	if (!waitqueue_active(wqh))
933 		goto out;
934 	/* walk in reverse to provide FIFO behavior to read userfaults */
935 	wq = list_last_entry(&wqh->head, typeof(*wq), entry);
936 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
937 out:
938 	return uwq;
939 }
940 
941 static inline struct userfaultfd_wait_queue *find_userfault(
942 		struct userfaultfd_ctx *ctx)
943 {
944 	return find_userfault_in(&ctx->fault_pending_wqh);
945 }
946 
947 static inline struct userfaultfd_wait_queue *find_userfault_evt(
948 		struct userfaultfd_ctx *ctx)
949 {
950 	return find_userfault_in(&ctx->event_wqh);
951 }
952 
953 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
954 {
955 	struct userfaultfd_ctx *ctx = file->private_data;
956 	__poll_t ret;
957 
958 	poll_wait(file, &ctx->fd_wqh, wait);
959 
960 	if (!userfaultfd_is_initialized(ctx))
961 		return EPOLLERR;
962 
963 	/*
964 	 * poll() never guarantees that read won't block.
965 	 * userfaults can be waken before they're read().
966 	 */
967 	if (unlikely(!(file->f_flags & O_NONBLOCK)))
968 		return EPOLLERR;
969 	/*
970 	 * lockless access to see if there are pending faults
971 	 * __pollwait last action is the add_wait_queue but
972 	 * the spin_unlock would allow the waitqueue_active to
973 	 * pass above the actual list_add inside
974 	 * add_wait_queue critical section. So use a full
975 	 * memory barrier to serialize the list_add write of
976 	 * add_wait_queue() with the waitqueue_active read
977 	 * below.
978 	 */
979 	ret = 0;
980 	smp_mb();
981 	if (waitqueue_active(&ctx->fault_pending_wqh))
982 		ret = EPOLLIN;
983 	else if (waitqueue_active(&ctx->event_wqh))
984 		ret = EPOLLIN;
985 
986 	return ret;
987 }
988 
989 static const struct file_operations userfaultfd_fops;
990 
991 static int resolve_userfault_fork(struct userfaultfd_ctx *new,
992 				  struct inode *inode,
993 				  struct uffd_msg *msg)
994 {
995 	int fd;
996 
997 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
998 			O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
999 	if (fd < 0)
1000 		return fd;
1001 
1002 	msg->arg.reserved.reserved1 = 0;
1003 	msg->arg.fork.ufd = fd;
1004 	return 0;
1005 }
1006 
1007 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1008 				    struct uffd_msg *msg, struct inode *inode)
1009 {
1010 	ssize_t ret;
1011 	DECLARE_WAITQUEUE(wait, current);
1012 	struct userfaultfd_wait_queue *uwq;
1013 	/*
1014 	 * Handling fork event requires sleeping operations, so
1015 	 * we drop the event_wqh lock, then do these ops, then
1016 	 * lock it back and wake up the waiter. While the lock is
1017 	 * dropped the ewq may go away so we keep track of it
1018 	 * carefully.
1019 	 */
1020 	LIST_HEAD(fork_event);
1021 	struct userfaultfd_ctx *fork_nctx = NULL;
1022 
1023 	/* always take the fd_wqh lock before the fault_pending_wqh lock */
1024 	spin_lock_irq(&ctx->fd_wqh.lock);
1025 	__add_wait_queue(&ctx->fd_wqh, &wait);
1026 	for (;;) {
1027 		set_current_state(TASK_INTERRUPTIBLE);
1028 		spin_lock(&ctx->fault_pending_wqh.lock);
1029 		uwq = find_userfault(ctx);
1030 		if (uwq) {
1031 			/*
1032 			 * Use a seqcount to repeat the lockless check
1033 			 * in wake_userfault() to avoid missing
1034 			 * wakeups because during the refile both
1035 			 * waitqueue could become empty if this is the
1036 			 * only userfault.
1037 			 */
1038 			write_seqcount_begin(&ctx->refile_seq);
1039 
1040 			/*
1041 			 * The fault_pending_wqh.lock prevents the uwq
1042 			 * to disappear from under us.
1043 			 *
1044 			 * Refile this userfault from
1045 			 * fault_pending_wqh to fault_wqh, it's not
1046 			 * pending anymore after we read it.
1047 			 *
1048 			 * Use list_del() by hand (as
1049 			 * userfaultfd_wake_function also uses
1050 			 * list_del_init() by hand) to be sure nobody
1051 			 * changes __remove_wait_queue() to use
1052 			 * list_del_init() in turn breaking the
1053 			 * !list_empty_careful() check in
1054 			 * handle_userfault(). The uwq->wq.head list
1055 			 * must never be empty at any time during the
1056 			 * refile, or the waitqueue could disappear
1057 			 * from under us. The "wait_queue_head_t"
1058 			 * parameter of __remove_wait_queue() is unused
1059 			 * anyway.
1060 			 */
1061 			list_del(&uwq->wq.entry);
1062 			add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1063 
1064 			write_seqcount_end(&ctx->refile_seq);
1065 
1066 			/* careful to always initialize msg if ret == 0 */
1067 			*msg = uwq->msg;
1068 			spin_unlock(&ctx->fault_pending_wqh.lock);
1069 			ret = 0;
1070 			break;
1071 		}
1072 		spin_unlock(&ctx->fault_pending_wqh.lock);
1073 
1074 		spin_lock(&ctx->event_wqh.lock);
1075 		uwq = find_userfault_evt(ctx);
1076 		if (uwq) {
1077 			*msg = uwq->msg;
1078 
1079 			if (uwq->msg.event == UFFD_EVENT_FORK) {
1080 				fork_nctx = (struct userfaultfd_ctx *)
1081 					(unsigned long)
1082 					uwq->msg.arg.reserved.reserved1;
1083 				list_move(&uwq->wq.entry, &fork_event);
1084 				/*
1085 				 * fork_nctx can be freed as soon as
1086 				 * we drop the lock, unless we take a
1087 				 * reference on it.
1088 				 */
1089 				userfaultfd_ctx_get(fork_nctx);
1090 				spin_unlock(&ctx->event_wqh.lock);
1091 				ret = 0;
1092 				break;
1093 			}
1094 
1095 			userfaultfd_event_complete(ctx, uwq);
1096 			spin_unlock(&ctx->event_wqh.lock);
1097 			ret = 0;
1098 			break;
1099 		}
1100 		spin_unlock(&ctx->event_wqh.lock);
1101 
1102 		if (signal_pending(current)) {
1103 			ret = -ERESTARTSYS;
1104 			break;
1105 		}
1106 		if (no_wait) {
1107 			ret = -EAGAIN;
1108 			break;
1109 		}
1110 		spin_unlock_irq(&ctx->fd_wqh.lock);
1111 		schedule();
1112 		spin_lock_irq(&ctx->fd_wqh.lock);
1113 	}
1114 	__remove_wait_queue(&ctx->fd_wqh, &wait);
1115 	__set_current_state(TASK_RUNNING);
1116 	spin_unlock_irq(&ctx->fd_wqh.lock);
1117 
1118 	if (!ret && msg->event == UFFD_EVENT_FORK) {
1119 		ret = resolve_userfault_fork(fork_nctx, inode, msg);
1120 		spin_lock_irq(&ctx->event_wqh.lock);
1121 		if (!list_empty(&fork_event)) {
1122 			/*
1123 			 * The fork thread didn't abort, so we can
1124 			 * drop the temporary refcount.
1125 			 */
1126 			userfaultfd_ctx_put(fork_nctx);
1127 
1128 			uwq = list_first_entry(&fork_event,
1129 					       typeof(*uwq),
1130 					       wq.entry);
1131 			/*
1132 			 * If fork_event list wasn't empty and in turn
1133 			 * the event wasn't already released by fork
1134 			 * (the event is allocated on fork kernel
1135 			 * stack), put the event back to its place in
1136 			 * the event_wq. fork_event head will be freed
1137 			 * as soon as we return so the event cannot
1138 			 * stay queued there no matter the current
1139 			 * "ret" value.
1140 			 */
1141 			list_del(&uwq->wq.entry);
1142 			__add_wait_queue(&ctx->event_wqh, &uwq->wq);
1143 
1144 			/*
1145 			 * Leave the event in the waitqueue and report
1146 			 * error to userland if we failed to resolve
1147 			 * the userfault fork.
1148 			 */
1149 			if (likely(!ret))
1150 				userfaultfd_event_complete(ctx, uwq);
1151 		} else {
1152 			/*
1153 			 * Here the fork thread aborted and the
1154 			 * refcount from the fork thread on fork_nctx
1155 			 * has already been released. We still hold
1156 			 * the reference we took before releasing the
1157 			 * lock above. If resolve_userfault_fork
1158 			 * failed we've to drop it because the
1159 			 * fork_nctx has to be freed in such case. If
1160 			 * it succeeded we'll hold it because the new
1161 			 * uffd references it.
1162 			 */
1163 			if (ret)
1164 				userfaultfd_ctx_put(fork_nctx);
1165 		}
1166 		spin_unlock_irq(&ctx->event_wqh.lock);
1167 	}
1168 
1169 	return ret;
1170 }
1171 
1172 static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1173 				size_t count, loff_t *ppos)
1174 {
1175 	struct userfaultfd_ctx *ctx = file->private_data;
1176 	ssize_t _ret, ret = 0;
1177 	struct uffd_msg msg;
1178 	int no_wait = file->f_flags & O_NONBLOCK;
1179 	struct inode *inode = file_inode(file);
1180 
1181 	if (!userfaultfd_is_initialized(ctx))
1182 		return -EINVAL;
1183 
1184 	for (;;) {
1185 		if (count < sizeof(msg))
1186 			return ret ? ret : -EINVAL;
1187 		_ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
1188 		if (_ret < 0)
1189 			return ret ? ret : _ret;
1190 		if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1191 			return ret ? ret : -EFAULT;
1192 		ret += sizeof(msg);
1193 		buf += sizeof(msg);
1194 		count -= sizeof(msg);
1195 		/*
1196 		 * Allow to read more than one fault at time but only
1197 		 * block if waiting for the very first one.
1198 		 */
1199 		no_wait = O_NONBLOCK;
1200 	}
1201 }
1202 
1203 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1204 			     struct userfaultfd_wake_range *range)
1205 {
1206 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
1207 	/* wake all in the range and autoremove */
1208 	if (waitqueue_active(&ctx->fault_pending_wqh))
1209 		__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1210 				     range);
1211 	if (waitqueue_active(&ctx->fault_wqh))
1212 		__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1213 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1214 }
1215 
1216 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1217 					   struct userfaultfd_wake_range *range)
1218 {
1219 	unsigned seq;
1220 	bool need_wakeup;
1221 
1222 	/*
1223 	 * To be sure waitqueue_active() is not reordered by the CPU
1224 	 * before the pagetable update, use an explicit SMP memory
1225 	 * barrier here. PT lock release or mmap_read_unlock(mm) still
1226 	 * have release semantics that can allow the
1227 	 * waitqueue_active() to be reordered before the pte update.
1228 	 */
1229 	smp_mb();
1230 
1231 	/*
1232 	 * Use waitqueue_active because it's very frequent to
1233 	 * change the address space atomically even if there are no
1234 	 * userfaults yet. So we take the spinlock only when we're
1235 	 * sure we've userfaults to wake.
1236 	 */
1237 	do {
1238 		seq = read_seqcount_begin(&ctx->refile_seq);
1239 		need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1240 			waitqueue_active(&ctx->fault_wqh);
1241 		cond_resched();
1242 	} while (read_seqcount_retry(&ctx->refile_seq, seq));
1243 	if (need_wakeup)
1244 		__wake_userfault(ctx, range);
1245 }
1246 
1247 static __always_inline int validate_range(struct mm_struct *mm,
1248 					  __u64 start, __u64 len)
1249 {
1250 	__u64 task_size = mm->task_size;
1251 
1252 	if (start & ~PAGE_MASK)
1253 		return -EINVAL;
1254 	if (len & ~PAGE_MASK)
1255 		return -EINVAL;
1256 	if (!len)
1257 		return -EINVAL;
1258 	if (start < mmap_min_addr)
1259 		return -EINVAL;
1260 	if (start >= task_size)
1261 		return -EINVAL;
1262 	if (len > task_size - start)
1263 		return -EINVAL;
1264 	return 0;
1265 }
1266 
1267 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1268 				unsigned long arg)
1269 {
1270 	struct mm_struct *mm = ctx->mm;
1271 	struct vm_area_struct *vma, *prev, *cur;
1272 	int ret;
1273 	struct uffdio_register uffdio_register;
1274 	struct uffdio_register __user *user_uffdio_register;
1275 	unsigned long vm_flags, new_flags;
1276 	bool found;
1277 	bool basic_ioctls;
1278 	unsigned long start, end, vma_end;
1279 	MA_STATE(mas, &mm->mm_mt, 0, 0);
1280 
1281 	user_uffdio_register = (struct uffdio_register __user *) arg;
1282 
1283 	ret = -EFAULT;
1284 	if (copy_from_user(&uffdio_register, user_uffdio_register,
1285 			   sizeof(uffdio_register)-sizeof(__u64)))
1286 		goto out;
1287 
1288 	ret = -EINVAL;
1289 	if (!uffdio_register.mode)
1290 		goto out;
1291 	if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
1292 		goto out;
1293 	vm_flags = 0;
1294 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1295 		vm_flags |= VM_UFFD_MISSING;
1296 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1297 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1298 		goto out;
1299 #endif
1300 		vm_flags |= VM_UFFD_WP;
1301 	}
1302 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
1303 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1304 		goto out;
1305 #endif
1306 		vm_flags |= VM_UFFD_MINOR;
1307 	}
1308 
1309 	ret = validate_range(mm, uffdio_register.range.start,
1310 			     uffdio_register.range.len);
1311 	if (ret)
1312 		goto out;
1313 
1314 	start = uffdio_register.range.start;
1315 	end = start + uffdio_register.range.len;
1316 
1317 	ret = -ENOMEM;
1318 	if (!mmget_not_zero(mm))
1319 		goto out;
1320 
1321 	mmap_write_lock(mm);
1322 	mas_set(&mas, start);
1323 	vma = mas_find(&mas, ULONG_MAX);
1324 	if (!vma)
1325 		goto out_unlock;
1326 
1327 	/* check that there's at least one vma in the range */
1328 	ret = -EINVAL;
1329 	if (vma->vm_start >= end)
1330 		goto out_unlock;
1331 
1332 	/*
1333 	 * If the first vma contains huge pages, make sure start address
1334 	 * is aligned to huge page size.
1335 	 */
1336 	if (is_vm_hugetlb_page(vma)) {
1337 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1338 
1339 		if (start & (vma_hpagesize - 1))
1340 			goto out_unlock;
1341 	}
1342 
1343 	/*
1344 	 * Search for not compatible vmas.
1345 	 */
1346 	found = false;
1347 	basic_ioctls = false;
1348 	for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
1349 		cond_resched();
1350 
1351 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1352 		       !!(cur->vm_flags & __VM_UFFD_FLAGS));
1353 
1354 		/* check not compatible vmas */
1355 		ret = -EINVAL;
1356 		if (!vma_can_userfault(cur, vm_flags))
1357 			goto out_unlock;
1358 
1359 		/*
1360 		 * UFFDIO_COPY will fill file holes even without
1361 		 * PROT_WRITE. This check enforces that if this is a
1362 		 * MAP_SHARED, the process has write permission to the backing
1363 		 * file. If VM_MAYWRITE is set it also enforces that on a
1364 		 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1365 		 * F_WRITE_SEAL can be taken until the vma is destroyed.
1366 		 */
1367 		ret = -EPERM;
1368 		if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1369 			goto out_unlock;
1370 
1371 		/*
1372 		 * If this vma contains ending address, and huge pages
1373 		 * check alignment.
1374 		 */
1375 		if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1376 		    end > cur->vm_start) {
1377 			unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1378 
1379 			ret = -EINVAL;
1380 
1381 			if (end & (vma_hpagesize - 1))
1382 				goto out_unlock;
1383 		}
1384 		if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
1385 			goto out_unlock;
1386 
1387 		/*
1388 		 * Check that this vma isn't already owned by a
1389 		 * different userfaultfd. We can't allow more than one
1390 		 * userfaultfd to own a single vma simultaneously or we
1391 		 * wouldn't know which one to deliver the userfaults to.
1392 		 */
1393 		ret = -EBUSY;
1394 		if (cur->vm_userfaultfd_ctx.ctx &&
1395 		    cur->vm_userfaultfd_ctx.ctx != ctx)
1396 			goto out_unlock;
1397 
1398 		/*
1399 		 * Note vmas containing huge pages
1400 		 */
1401 		if (is_vm_hugetlb_page(cur))
1402 			basic_ioctls = true;
1403 
1404 		found = true;
1405 	}
1406 	BUG_ON(!found);
1407 
1408 	mas_set(&mas, start);
1409 	prev = mas_prev(&mas, 0);
1410 	if (prev != vma)
1411 		mas_next(&mas, ULONG_MAX);
1412 
1413 	ret = 0;
1414 	do {
1415 		cond_resched();
1416 
1417 		BUG_ON(!vma_can_userfault(vma, vm_flags));
1418 		BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1419 		       vma->vm_userfaultfd_ctx.ctx != ctx);
1420 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1421 
1422 		/*
1423 		 * Nothing to do: this vma is already registered into this
1424 		 * userfaultfd and with the right tracking mode too.
1425 		 */
1426 		if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1427 		    (vma->vm_flags & vm_flags) == vm_flags)
1428 			goto skip;
1429 
1430 		if (vma->vm_start > start)
1431 			start = vma->vm_start;
1432 		vma_end = min(end, vma->vm_end);
1433 
1434 		new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
1435 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1436 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1437 				 vma_policy(vma),
1438 				 ((struct vm_userfaultfd_ctx){ ctx }),
1439 				 anon_vma_name(vma));
1440 		if (prev) {
1441 			/* vma_merge() invalidated the mas */
1442 			mas_pause(&mas);
1443 			vma = prev;
1444 			goto next;
1445 		}
1446 		if (vma->vm_start < start) {
1447 			ret = split_vma(mm, vma, start, 1);
1448 			if (ret)
1449 				break;
1450 			/* split_vma() invalidated the mas */
1451 			mas_pause(&mas);
1452 		}
1453 		if (vma->vm_end > end) {
1454 			ret = split_vma(mm, vma, end, 0);
1455 			if (ret)
1456 				break;
1457 			/* split_vma() invalidated the mas */
1458 			mas_pause(&mas);
1459 		}
1460 	next:
1461 		/*
1462 		 * In the vma_merge() successful mprotect-like case 8:
1463 		 * the next vma was merged into the current one and
1464 		 * the current one has not been updated yet.
1465 		 */
1466 		vma->vm_flags = new_flags;
1467 		vma->vm_userfaultfd_ctx.ctx = ctx;
1468 
1469 		if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
1470 			hugetlb_unshare_all_pmds(vma);
1471 
1472 	skip:
1473 		prev = vma;
1474 		start = vma->vm_end;
1475 		vma = mas_next(&mas, end - 1);
1476 	} while (vma);
1477 out_unlock:
1478 	mmap_write_unlock(mm);
1479 	mmput(mm);
1480 	if (!ret) {
1481 		__u64 ioctls_out;
1482 
1483 		ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1484 		    UFFD_API_RANGE_IOCTLS;
1485 
1486 		/*
1487 		 * Declare the WP ioctl only if the WP mode is
1488 		 * specified and all checks passed with the range
1489 		 */
1490 		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
1491 			ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
1492 
1493 		/* CONTINUE ioctl is only supported for MINOR ranges. */
1494 		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
1495 			ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
1496 
1497 		/*
1498 		 * Now that we scanned all vmas we can already tell
1499 		 * userland which ioctls methods are guaranteed to
1500 		 * succeed on this range.
1501 		 */
1502 		if (put_user(ioctls_out, &user_uffdio_register->ioctls))
1503 			ret = -EFAULT;
1504 	}
1505 out:
1506 	return ret;
1507 }
1508 
1509 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1510 				  unsigned long arg)
1511 {
1512 	struct mm_struct *mm = ctx->mm;
1513 	struct vm_area_struct *vma, *prev, *cur;
1514 	int ret;
1515 	struct uffdio_range uffdio_unregister;
1516 	unsigned long new_flags;
1517 	bool found;
1518 	unsigned long start, end, vma_end;
1519 	const void __user *buf = (void __user *)arg;
1520 	MA_STATE(mas, &mm->mm_mt, 0, 0);
1521 
1522 	ret = -EFAULT;
1523 	if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1524 		goto out;
1525 
1526 	ret = validate_range(mm, uffdio_unregister.start,
1527 			     uffdio_unregister.len);
1528 	if (ret)
1529 		goto out;
1530 
1531 	start = uffdio_unregister.start;
1532 	end = start + uffdio_unregister.len;
1533 
1534 	ret = -ENOMEM;
1535 	if (!mmget_not_zero(mm))
1536 		goto out;
1537 
1538 	mmap_write_lock(mm);
1539 	mas_set(&mas, start);
1540 	vma = mas_find(&mas, ULONG_MAX);
1541 	if (!vma)
1542 		goto out_unlock;
1543 
1544 	/* check that there's at least one vma in the range */
1545 	ret = -EINVAL;
1546 	if (vma->vm_start >= end)
1547 		goto out_unlock;
1548 
1549 	/*
1550 	 * If the first vma contains huge pages, make sure start address
1551 	 * is aligned to huge page size.
1552 	 */
1553 	if (is_vm_hugetlb_page(vma)) {
1554 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1555 
1556 		if (start & (vma_hpagesize - 1))
1557 			goto out_unlock;
1558 	}
1559 
1560 	/*
1561 	 * Search for not compatible vmas.
1562 	 */
1563 	found = false;
1564 	ret = -EINVAL;
1565 	for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
1566 		cond_resched();
1567 
1568 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1569 		       !!(cur->vm_flags & __VM_UFFD_FLAGS));
1570 
1571 		/*
1572 		 * Check not compatible vmas, not strictly required
1573 		 * here as not compatible vmas cannot have an
1574 		 * userfaultfd_ctx registered on them, but this
1575 		 * provides for more strict behavior to notice
1576 		 * unregistration errors.
1577 		 */
1578 		if (!vma_can_userfault(cur, cur->vm_flags))
1579 			goto out_unlock;
1580 
1581 		found = true;
1582 	}
1583 	BUG_ON(!found);
1584 
1585 	mas_set(&mas, start);
1586 	prev = mas_prev(&mas, 0);
1587 	if (prev != vma)
1588 		mas_next(&mas, ULONG_MAX);
1589 
1590 	ret = 0;
1591 	do {
1592 		cond_resched();
1593 
1594 		BUG_ON(!vma_can_userfault(vma, vma->vm_flags));
1595 
1596 		/*
1597 		 * Nothing to do: this vma is already registered into this
1598 		 * userfaultfd and with the right tracking mode too.
1599 		 */
1600 		if (!vma->vm_userfaultfd_ctx.ctx)
1601 			goto skip;
1602 
1603 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1604 
1605 		if (vma->vm_start > start)
1606 			start = vma->vm_start;
1607 		vma_end = min(end, vma->vm_end);
1608 
1609 		if (userfaultfd_missing(vma)) {
1610 			/*
1611 			 * Wake any concurrent pending userfault while
1612 			 * we unregister, so they will not hang
1613 			 * permanently and it avoids userland to call
1614 			 * UFFDIO_WAKE explicitly.
1615 			 */
1616 			struct userfaultfd_wake_range range;
1617 			range.start = start;
1618 			range.len = vma_end - start;
1619 			wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1620 		}
1621 
1622 		/* Reset ptes for the whole vma range if wr-protected */
1623 		if (userfaultfd_wp(vma))
1624 			uffd_wp_range(mm, vma, start, vma_end - start, false);
1625 
1626 		new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
1627 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1628 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1629 				 vma_policy(vma),
1630 				 NULL_VM_UFFD_CTX, anon_vma_name(vma));
1631 		if (prev) {
1632 			vma = prev;
1633 			mas_pause(&mas);
1634 			goto next;
1635 		}
1636 		if (vma->vm_start < start) {
1637 			ret = split_vma(mm, vma, start, 1);
1638 			if (ret)
1639 				break;
1640 			mas_pause(&mas);
1641 		}
1642 		if (vma->vm_end > end) {
1643 			ret = split_vma(mm, vma, end, 0);
1644 			if (ret)
1645 				break;
1646 			mas_pause(&mas);
1647 		}
1648 	next:
1649 		/*
1650 		 * In the vma_merge() successful mprotect-like case 8:
1651 		 * the next vma was merged into the current one and
1652 		 * the current one has not been updated yet.
1653 		 */
1654 		vma->vm_flags = new_flags;
1655 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1656 
1657 	skip:
1658 		prev = vma;
1659 		start = vma->vm_end;
1660 		vma = mas_next(&mas, end - 1);
1661 	} while (vma);
1662 out_unlock:
1663 	mmap_write_unlock(mm);
1664 	mmput(mm);
1665 out:
1666 	return ret;
1667 }
1668 
1669 /*
1670  * userfaultfd_wake may be used in combination with the
1671  * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1672  */
1673 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1674 			    unsigned long arg)
1675 {
1676 	int ret;
1677 	struct uffdio_range uffdio_wake;
1678 	struct userfaultfd_wake_range range;
1679 	const void __user *buf = (void __user *)arg;
1680 
1681 	ret = -EFAULT;
1682 	if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1683 		goto out;
1684 
1685 	ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1686 	if (ret)
1687 		goto out;
1688 
1689 	range.start = uffdio_wake.start;
1690 	range.len = uffdio_wake.len;
1691 
1692 	/*
1693 	 * len == 0 means wake all and we don't want to wake all here,
1694 	 * so check it again to be sure.
1695 	 */
1696 	VM_BUG_ON(!range.len);
1697 
1698 	wake_userfault(ctx, &range);
1699 	ret = 0;
1700 
1701 out:
1702 	return ret;
1703 }
1704 
1705 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1706 			    unsigned long arg)
1707 {
1708 	__s64 ret;
1709 	struct uffdio_copy uffdio_copy;
1710 	struct uffdio_copy __user *user_uffdio_copy;
1711 	struct userfaultfd_wake_range range;
1712 
1713 	user_uffdio_copy = (struct uffdio_copy __user *) arg;
1714 
1715 	ret = -EAGAIN;
1716 	if (atomic_read(&ctx->mmap_changing))
1717 		goto out;
1718 
1719 	ret = -EFAULT;
1720 	if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1721 			   /* don't copy "copy" last field */
1722 			   sizeof(uffdio_copy)-sizeof(__s64)))
1723 		goto out;
1724 
1725 	ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1726 	if (ret)
1727 		goto out;
1728 	/*
1729 	 * double check for wraparound just in case. copy_from_user()
1730 	 * will later check uffdio_copy.src + uffdio_copy.len to fit
1731 	 * in the userland range.
1732 	 */
1733 	ret = -EINVAL;
1734 	if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1735 		goto out;
1736 	if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
1737 		goto out;
1738 	if (mmget_not_zero(ctx->mm)) {
1739 		ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1740 				   uffdio_copy.len, &ctx->mmap_changing,
1741 				   uffdio_copy.mode);
1742 		mmput(ctx->mm);
1743 	} else {
1744 		return -ESRCH;
1745 	}
1746 	if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1747 		return -EFAULT;
1748 	if (ret < 0)
1749 		goto out;
1750 	BUG_ON(!ret);
1751 	/* len == 0 would wake all */
1752 	range.len = ret;
1753 	if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1754 		range.start = uffdio_copy.dst;
1755 		wake_userfault(ctx, &range);
1756 	}
1757 	ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1758 out:
1759 	return ret;
1760 }
1761 
1762 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1763 				unsigned long arg)
1764 {
1765 	__s64 ret;
1766 	struct uffdio_zeropage uffdio_zeropage;
1767 	struct uffdio_zeropage __user *user_uffdio_zeropage;
1768 	struct userfaultfd_wake_range range;
1769 
1770 	user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1771 
1772 	ret = -EAGAIN;
1773 	if (atomic_read(&ctx->mmap_changing))
1774 		goto out;
1775 
1776 	ret = -EFAULT;
1777 	if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1778 			   /* don't copy "zeropage" last field */
1779 			   sizeof(uffdio_zeropage)-sizeof(__s64)))
1780 		goto out;
1781 
1782 	ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1783 			     uffdio_zeropage.range.len);
1784 	if (ret)
1785 		goto out;
1786 	ret = -EINVAL;
1787 	if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1788 		goto out;
1789 
1790 	if (mmget_not_zero(ctx->mm)) {
1791 		ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
1792 				     uffdio_zeropage.range.len,
1793 				     &ctx->mmap_changing);
1794 		mmput(ctx->mm);
1795 	} else {
1796 		return -ESRCH;
1797 	}
1798 	if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1799 		return -EFAULT;
1800 	if (ret < 0)
1801 		goto out;
1802 	/* len == 0 would wake all */
1803 	BUG_ON(!ret);
1804 	range.len = ret;
1805 	if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1806 		range.start = uffdio_zeropage.range.start;
1807 		wake_userfault(ctx, &range);
1808 	}
1809 	ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1810 out:
1811 	return ret;
1812 }
1813 
1814 static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
1815 				    unsigned long arg)
1816 {
1817 	int ret;
1818 	struct uffdio_writeprotect uffdio_wp;
1819 	struct uffdio_writeprotect __user *user_uffdio_wp;
1820 	struct userfaultfd_wake_range range;
1821 	bool mode_wp, mode_dontwake;
1822 
1823 	if (atomic_read(&ctx->mmap_changing))
1824 		return -EAGAIN;
1825 
1826 	user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
1827 
1828 	if (copy_from_user(&uffdio_wp, user_uffdio_wp,
1829 			   sizeof(struct uffdio_writeprotect)))
1830 		return -EFAULT;
1831 
1832 	ret = validate_range(ctx->mm, uffdio_wp.range.start,
1833 			     uffdio_wp.range.len);
1834 	if (ret)
1835 		return ret;
1836 
1837 	if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
1838 			       UFFDIO_WRITEPROTECT_MODE_WP))
1839 		return -EINVAL;
1840 
1841 	mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
1842 	mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
1843 
1844 	if (mode_wp && mode_dontwake)
1845 		return -EINVAL;
1846 
1847 	if (mmget_not_zero(ctx->mm)) {
1848 		ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start,
1849 					  uffdio_wp.range.len, mode_wp,
1850 					  &ctx->mmap_changing);
1851 		mmput(ctx->mm);
1852 	} else {
1853 		return -ESRCH;
1854 	}
1855 
1856 	if (ret)
1857 		return ret;
1858 
1859 	if (!mode_wp && !mode_dontwake) {
1860 		range.start = uffdio_wp.range.start;
1861 		range.len = uffdio_wp.range.len;
1862 		wake_userfault(ctx, &range);
1863 	}
1864 	return ret;
1865 }
1866 
1867 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
1868 {
1869 	__s64 ret;
1870 	struct uffdio_continue uffdio_continue;
1871 	struct uffdio_continue __user *user_uffdio_continue;
1872 	struct userfaultfd_wake_range range;
1873 
1874 	user_uffdio_continue = (struct uffdio_continue __user *)arg;
1875 
1876 	ret = -EAGAIN;
1877 	if (atomic_read(&ctx->mmap_changing))
1878 		goto out;
1879 
1880 	ret = -EFAULT;
1881 	if (copy_from_user(&uffdio_continue, user_uffdio_continue,
1882 			   /* don't copy the output fields */
1883 			   sizeof(uffdio_continue) - (sizeof(__s64))))
1884 		goto out;
1885 
1886 	ret = validate_range(ctx->mm, uffdio_continue.range.start,
1887 			     uffdio_continue.range.len);
1888 	if (ret)
1889 		goto out;
1890 
1891 	ret = -EINVAL;
1892 	/* double check for wraparound just in case. */
1893 	if (uffdio_continue.range.start + uffdio_continue.range.len <=
1894 	    uffdio_continue.range.start) {
1895 		goto out;
1896 	}
1897 	if (uffdio_continue.mode & ~UFFDIO_CONTINUE_MODE_DONTWAKE)
1898 		goto out;
1899 
1900 	if (mmget_not_zero(ctx->mm)) {
1901 		ret = mcopy_continue(ctx->mm, uffdio_continue.range.start,
1902 				     uffdio_continue.range.len,
1903 				     &ctx->mmap_changing);
1904 		mmput(ctx->mm);
1905 	} else {
1906 		return -ESRCH;
1907 	}
1908 
1909 	if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
1910 		return -EFAULT;
1911 	if (ret < 0)
1912 		goto out;
1913 
1914 	/* len == 0 would wake all */
1915 	BUG_ON(!ret);
1916 	range.len = ret;
1917 	if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
1918 		range.start = uffdio_continue.range.start;
1919 		wake_userfault(ctx, &range);
1920 	}
1921 	ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
1922 
1923 out:
1924 	return ret;
1925 }
1926 
1927 static inline unsigned int uffd_ctx_features(__u64 user_features)
1928 {
1929 	/*
1930 	 * For the current set of features the bits just coincide. Set
1931 	 * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
1932 	 */
1933 	return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
1934 }
1935 
1936 /*
1937  * userland asks for a certain API version and we return which bits
1938  * and ioctl commands are implemented in this kernel for such API
1939  * version or -EINVAL if unknown.
1940  */
1941 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1942 			   unsigned long arg)
1943 {
1944 	struct uffdio_api uffdio_api;
1945 	void __user *buf = (void __user *)arg;
1946 	unsigned int ctx_features;
1947 	int ret;
1948 	__u64 features;
1949 
1950 	ret = -EFAULT;
1951 	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1952 		goto out;
1953 	/* Ignore unsupported features (userspace built against newer kernel) */
1954 	features = uffdio_api.features & UFFD_API_FEATURES;
1955 	ret = -EPERM;
1956 	if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
1957 		goto err_out;
1958 	/* report all available features and ioctls to userland */
1959 	uffdio_api.features = UFFD_API_FEATURES;
1960 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1961 	uffdio_api.features &=
1962 		~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
1963 #endif
1964 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1965 	uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
1966 #endif
1967 #ifndef CONFIG_PTE_MARKER_UFFD_WP
1968 	uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
1969 #endif
1970 	uffdio_api.ioctls = UFFD_API_IOCTLS;
1971 	ret = -EFAULT;
1972 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1973 		goto out;
1974 
1975 	/* only enable the requested features for this uffd context */
1976 	ctx_features = uffd_ctx_features(features);
1977 	ret = -EINVAL;
1978 	if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
1979 		goto err_out;
1980 
1981 	ret = 0;
1982 out:
1983 	return ret;
1984 err_out:
1985 	memset(&uffdio_api, 0, sizeof(uffdio_api));
1986 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1987 		ret = -EFAULT;
1988 	goto out;
1989 }
1990 
1991 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1992 			      unsigned long arg)
1993 {
1994 	int ret = -EINVAL;
1995 	struct userfaultfd_ctx *ctx = file->private_data;
1996 
1997 	if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
1998 		return -EINVAL;
1999 
2000 	switch(cmd) {
2001 	case UFFDIO_API:
2002 		ret = userfaultfd_api(ctx, arg);
2003 		break;
2004 	case UFFDIO_REGISTER:
2005 		ret = userfaultfd_register(ctx, arg);
2006 		break;
2007 	case UFFDIO_UNREGISTER:
2008 		ret = userfaultfd_unregister(ctx, arg);
2009 		break;
2010 	case UFFDIO_WAKE:
2011 		ret = userfaultfd_wake(ctx, arg);
2012 		break;
2013 	case UFFDIO_COPY:
2014 		ret = userfaultfd_copy(ctx, arg);
2015 		break;
2016 	case UFFDIO_ZEROPAGE:
2017 		ret = userfaultfd_zeropage(ctx, arg);
2018 		break;
2019 	case UFFDIO_WRITEPROTECT:
2020 		ret = userfaultfd_writeprotect(ctx, arg);
2021 		break;
2022 	case UFFDIO_CONTINUE:
2023 		ret = userfaultfd_continue(ctx, arg);
2024 		break;
2025 	}
2026 	return ret;
2027 }
2028 
2029 #ifdef CONFIG_PROC_FS
2030 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
2031 {
2032 	struct userfaultfd_ctx *ctx = f->private_data;
2033 	wait_queue_entry_t *wq;
2034 	unsigned long pending = 0, total = 0;
2035 
2036 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
2037 	list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
2038 		pending++;
2039 		total++;
2040 	}
2041 	list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
2042 		total++;
2043 	}
2044 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
2045 
2046 	/*
2047 	 * If more protocols will be added, there will be all shown
2048 	 * separated by a space. Like this:
2049 	 *	protocols: aa:... bb:...
2050 	 */
2051 	seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
2052 		   pending, total, UFFD_API, ctx->features,
2053 		   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
2054 }
2055 #endif
2056 
2057 static const struct file_operations userfaultfd_fops = {
2058 #ifdef CONFIG_PROC_FS
2059 	.show_fdinfo	= userfaultfd_show_fdinfo,
2060 #endif
2061 	.release	= userfaultfd_release,
2062 	.poll		= userfaultfd_poll,
2063 	.read		= userfaultfd_read,
2064 	.unlocked_ioctl = userfaultfd_ioctl,
2065 	.compat_ioctl	= compat_ptr_ioctl,
2066 	.llseek		= noop_llseek,
2067 };
2068 
2069 static void init_once_userfaultfd_ctx(void *mem)
2070 {
2071 	struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
2072 
2073 	init_waitqueue_head(&ctx->fault_pending_wqh);
2074 	init_waitqueue_head(&ctx->fault_wqh);
2075 	init_waitqueue_head(&ctx->event_wqh);
2076 	init_waitqueue_head(&ctx->fd_wqh);
2077 	seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
2078 }
2079 
2080 static int new_userfaultfd(int flags)
2081 {
2082 	struct userfaultfd_ctx *ctx;
2083 	int fd;
2084 
2085 	BUG_ON(!current->mm);
2086 
2087 	/* Check the UFFD_* constants for consistency.  */
2088 	BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
2089 	BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
2090 	BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
2091 
2092 	if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
2093 		return -EINVAL;
2094 
2095 	ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
2096 	if (!ctx)
2097 		return -ENOMEM;
2098 
2099 	refcount_set(&ctx->refcount, 1);
2100 	ctx->flags = flags;
2101 	ctx->features = 0;
2102 	ctx->released = false;
2103 	atomic_set(&ctx->mmap_changing, 0);
2104 	ctx->mm = current->mm;
2105 	/* prevent the mm struct to be freed */
2106 	mmgrab(ctx->mm);
2107 
2108 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
2109 			O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
2110 	if (fd < 0) {
2111 		mmdrop(ctx->mm);
2112 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
2113 	}
2114 	return fd;
2115 }
2116 
2117 static inline bool userfaultfd_syscall_allowed(int flags)
2118 {
2119 	/* Userspace-only page faults are always allowed */
2120 	if (flags & UFFD_USER_MODE_ONLY)
2121 		return true;
2122 
2123 	/*
2124 	 * The user is requesting a userfaultfd which can handle kernel faults.
2125 	 * Privileged users are always allowed to do this.
2126 	 */
2127 	if (capable(CAP_SYS_PTRACE))
2128 		return true;
2129 
2130 	/* Otherwise, access to kernel fault handling is sysctl controlled. */
2131 	return sysctl_unprivileged_userfaultfd;
2132 }
2133 
2134 SYSCALL_DEFINE1(userfaultfd, int, flags)
2135 {
2136 	if (!userfaultfd_syscall_allowed(flags))
2137 		return -EPERM;
2138 
2139 	return new_userfaultfd(flags);
2140 }
2141 
2142 static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
2143 {
2144 	if (cmd != USERFAULTFD_IOC_NEW)
2145 		return -EINVAL;
2146 
2147 	return new_userfaultfd(flags);
2148 }
2149 
2150 static const struct file_operations userfaultfd_dev_fops = {
2151 	.unlocked_ioctl = userfaultfd_dev_ioctl,
2152 	.compat_ioctl = userfaultfd_dev_ioctl,
2153 	.owner = THIS_MODULE,
2154 	.llseek = noop_llseek,
2155 };
2156 
2157 static struct miscdevice userfaultfd_misc = {
2158 	.minor = MISC_DYNAMIC_MINOR,
2159 	.name = "userfaultfd",
2160 	.fops = &userfaultfd_dev_fops
2161 };
2162 
2163 static int __init userfaultfd_init(void)
2164 {
2165 	int ret;
2166 
2167 	ret = misc_register(&userfaultfd_misc);
2168 	if (ret)
2169 		return ret;
2170 
2171 	userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
2172 						sizeof(struct userfaultfd_ctx),
2173 						0,
2174 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2175 						init_once_userfaultfd_ctx);
2176 	return 0;
2177 }
2178 __initcall(userfaultfd_init);
2179