xref: /openbmc/linux/fs/userfaultfd.c (revision 62e59c4e)
1 /*
2  *  fs/userfaultfd.c
3  *
4  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
5  *  Copyright (C) 2008-2009 Red Hat, Inc.
6  *  Copyright (C) 2015  Red Hat, Inc.
7  *
8  *  This work is licensed under the terms of the GNU GPL, version 2. See
9  *  the COPYING file in the top-level directory.
10  *
11  *  Some part derived from fs/eventfd.c (anon inode setup) and
12  *  mm/ksm.c (mm hashing).
13  */
14 
15 #include <linux/list.h>
16 #include <linux/hashtable.h>
17 #include <linux/sched/signal.h>
18 #include <linux/sched/mm.h>
19 #include <linux/mm.h>
20 #include <linux/poll.h>
21 #include <linux/slab.h>
22 #include <linux/seq_file.h>
23 #include <linux/file.h>
24 #include <linux/bug.h>
25 #include <linux/anon_inodes.h>
26 #include <linux/syscalls.h>
27 #include <linux/userfaultfd_k.h>
28 #include <linux/mempolicy.h>
29 #include <linux/ioctl.h>
30 #include <linux/security.h>
31 #include <linux/hugetlb.h>
32 
33 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
34 
35 enum userfaultfd_state {
36 	UFFD_STATE_WAIT_API,
37 	UFFD_STATE_RUNNING,
38 };
39 
40 /*
41  * Start with fault_pending_wqh and fault_wqh so they're more likely
42  * to be in the same cacheline.
43  */
44 struct userfaultfd_ctx {
45 	/* waitqueue head for the pending (i.e. not read) userfaults */
46 	wait_queue_head_t fault_pending_wqh;
47 	/* waitqueue head for the userfaults */
48 	wait_queue_head_t fault_wqh;
49 	/* waitqueue head for the pseudo fd to wakeup poll/read */
50 	wait_queue_head_t fd_wqh;
51 	/* waitqueue head for events */
52 	wait_queue_head_t event_wqh;
53 	/* a refile sequence protected by fault_pending_wqh lock */
54 	struct seqcount refile_seq;
55 	/* pseudo fd refcounting */
56 	refcount_t refcount;
57 	/* userfaultfd syscall flags */
58 	unsigned int flags;
59 	/* features requested from the userspace */
60 	unsigned int features;
61 	/* state machine */
62 	enum userfaultfd_state state;
63 	/* released */
64 	bool released;
65 	/* memory mappings are changing because of non-cooperative event */
66 	bool mmap_changing;
67 	/* mm with one ore more vmas attached to this userfaultfd_ctx */
68 	struct mm_struct *mm;
69 };
70 
71 struct userfaultfd_fork_ctx {
72 	struct userfaultfd_ctx *orig;
73 	struct userfaultfd_ctx *new;
74 	struct list_head list;
75 };
76 
77 struct userfaultfd_unmap_ctx {
78 	struct userfaultfd_ctx *ctx;
79 	unsigned long start;
80 	unsigned long end;
81 	struct list_head list;
82 };
83 
84 struct userfaultfd_wait_queue {
85 	struct uffd_msg msg;
86 	wait_queue_entry_t wq;
87 	struct userfaultfd_ctx *ctx;
88 	bool waken;
89 };
90 
91 struct userfaultfd_wake_range {
92 	unsigned long start;
93 	unsigned long len;
94 };
95 
96 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
97 				     int wake_flags, void *key)
98 {
99 	struct userfaultfd_wake_range *range = key;
100 	int ret;
101 	struct userfaultfd_wait_queue *uwq;
102 	unsigned long start, len;
103 
104 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
105 	ret = 0;
106 	/* len == 0 means wake all */
107 	start = range->start;
108 	len = range->len;
109 	if (len && (start > uwq->msg.arg.pagefault.address ||
110 		    start + len <= uwq->msg.arg.pagefault.address))
111 		goto out;
112 	WRITE_ONCE(uwq->waken, true);
113 	/*
114 	 * The Program-Order guarantees provided by the scheduler
115 	 * ensure uwq->waken is visible before the task is woken.
116 	 */
117 	ret = wake_up_state(wq->private, mode);
118 	if (ret) {
119 		/*
120 		 * Wake only once, autoremove behavior.
121 		 *
122 		 * After the effect of list_del_init is visible to the other
123 		 * CPUs, the waitqueue may disappear from under us, see the
124 		 * !list_empty_careful() in handle_userfault().
125 		 *
126 		 * try_to_wake_up() has an implicit smp_mb(), and the
127 		 * wq->private is read before calling the extern function
128 		 * "wake_up_state" (which in turns calls try_to_wake_up).
129 		 */
130 		list_del_init(&wq->entry);
131 	}
132 out:
133 	return ret;
134 }
135 
136 /**
137  * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
138  * context.
139  * @ctx: [in] Pointer to the userfaultfd context.
140  */
141 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
142 {
143 	refcount_inc(&ctx->refcount);
144 }
145 
146 /**
147  * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
148  * context.
149  * @ctx: [in] Pointer to userfaultfd context.
150  *
151  * The userfaultfd context reference must have been previously acquired either
152  * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
153  */
154 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
155 {
156 	if (refcount_dec_and_test(&ctx->refcount)) {
157 		VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
158 		VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
159 		VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
160 		VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
161 		VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
162 		VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
163 		VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
164 		VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
165 		mmdrop(ctx->mm);
166 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
167 	}
168 }
169 
170 static inline void msg_init(struct uffd_msg *msg)
171 {
172 	BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
173 	/*
174 	 * Must use memset to zero out the paddings or kernel data is
175 	 * leaked to userland.
176 	 */
177 	memset(msg, 0, sizeof(struct uffd_msg));
178 }
179 
180 static inline struct uffd_msg userfault_msg(unsigned long address,
181 					    unsigned int flags,
182 					    unsigned long reason,
183 					    unsigned int features)
184 {
185 	struct uffd_msg msg;
186 	msg_init(&msg);
187 	msg.event = UFFD_EVENT_PAGEFAULT;
188 	msg.arg.pagefault.address = address;
189 	if (flags & FAULT_FLAG_WRITE)
190 		/*
191 		 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
192 		 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WRITE
193 		 * was not set in a UFFD_EVENT_PAGEFAULT, it means it
194 		 * was a read fault, otherwise if set it means it's
195 		 * a write fault.
196 		 */
197 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
198 	if (reason & VM_UFFD_WP)
199 		/*
200 		 * If UFFD_FEATURE_PAGEFAULT_FLAG_WP was set in the
201 		 * uffdio_api.features and UFFD_PAGEFAULT_FLAG_WP was
202 		 * not set in a UFFD_EVENT_PAGEFAULT, it means it was
203 		 * a missing fault, otherwise if set it means it's a
204 		 * write protect fault.
205 		 */
206 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
207 	if (features & UFFD_FEATURE_THREAD_ID)
208 		msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
209 	return msg;
210 }
211 
212 #ifdef CONFIG_HUGETLB_PAGE
213 /*
214  * Same functionality as userfaultfd_must_wait below with modifications for
215  * hugepmd ranges.
216  */
217 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
218 					 struct vm_area_struct *vma,
219 					 unsigned long address,
220 					 unsigned long flags,
221 					 unsigned long reason)
222 {
223 	struct mm_struct *mm = ctx->mm;
224 	pte_t *ptep, pte;
225 	bool ret = true;
226 
227 	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
228 
229 	ptep = huge_pte_offset(mm, address, vma_mmu_pagesize(vma));
230 
231 	if (!ptep)
232 		goto out;
233 
234 	ret = false;
235 	pte = huge_ptep_get(ptep);
236 
237 	/*
238 	 * Lockless access: we're in a wait_event so it's ok if it
239 	 * changes under us.
240 	 */
241 	if (huge_pte_none(pte))
242 		ret = true;
243 	if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
244 		ret = true;
245 out:
246 	return ret;
247 }
248 #else
249 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
250 					 struct vm_area_struct *vma,
251 					 unsigned long address,
252 					 unsigned long flags,
253 					 unsigned long reason)
254 {
255 	return false;	/* should never get here */
256 }
257 #endif /* CONFIG_HUGETLB_PAGE */
258 
259 /*
260  * Verify the pagetables are still not ok after having reigstered into
261  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
262  * userfault that has already been resolved, if userfaultfd_read and
263  * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
264  * threads.
265  */
266 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
267 					 unsigned long address,
268 					 unsigned long flags,
269 					 unsigned long reason)
270 {
271 	struct mm_struct *mm = ctx->mm;
272 	pgd_t *pgd;
273 	p4d_t *p4d;
274 	pud_t *pud;
275 	pmd_t *pmd, _pmd;
276 	pte_t *pte;
277 	bool ret = true;
278 
279 	VM_BUG_ON(!rwsem_is_locked(&mm->mmap_sem));
280 
281 	pgd = pgd_offset(mm, address);
282 	if (!pgd_present(*pgd))
283 		goto out;
284 	p4d = p4d_offset(pgd, address);
285 	if (!p4d_present(*p4d))
286 		goto out;
287 	pud = pud_offset(p4d, address);
288 	if (!pud_present(*pud))
289 		goto out;
290 	pmd = pmd_offset(pud, address);
291 	/*
292 	 * READ_ONCE must function as a barrier with narrower scope
293 	 * and it must be equivalent to:
294 	 *	_pmd = *pmd; barrier();
295 	 *
296 	 * This is to deal with the instability (as in
297 	 * pmd_trans_unstable) of the pmd.
298 	 */
299 	_pmd = READ_ONCE(*pmd);
300 	if (pmd_none(_pmd))
301 		goto out;
302 
303 	ret = false;
304 	if (!pmd_present(_pmd))
305 		goto out;
306 
307 	if (pmd_trans_huge(_pmd))
308 		goto out;
309 
310 	/*
311 	 * the pmd is stable (as in !pmd_trans_unstable) so we can re-read it
312 	 * and use the standard pte_offset_map() instead of parsing _pmd.
313 	 */
314 	pte = pte_offset_map(pmd, address);
315 	/*
316 	 * Lockless access: we're in a wait_event so it's ok if it
317 	 * changes under us.
318 	 */
319 	if (pte_none(*pte))
320 		ret = true;
321 	pte_unmap(pte);
322 
323 out:
324 	return ret;
325 }
326 
327 /*
328  * The locking rules involved in returning VM_FAULT_RETRY depending on
329  * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
330  * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
331  * recommendation in __lock_page_or_retry is not an understatement.
332  *
333  * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_sem must be released
334  * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
335  * not set.
336  *
337  * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
338  * set, VM_FAULT_RETRY can still be returned if and only if there are
339  * fatal_signal_pending()s, and the mmap_sem must be released before
340  * returning it.
341  */
342 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
343 {
344 	struct mm_struct *mm = vmf->vma->vm_mm;
345 	struct userfaultfd_ctx *ctx;
346 	struct userfaultfd_wait_queue uwq;
347 	vm_fault_t ret = VM_FAULT_SIGBUS;
348 	bool must_wait, return_to_userland;
349 	long blocking_state;
350 
351 	/*
352 	 * We don't do userfault handling for the final child pid update.
353 	 *
354 	 * We also don't do userfault handling during
355 	 * coredumping. hugetlbfs has the special
356 	 * follow_hugetlb_page() to skip missing pages in the
357 	 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
358 	 * the no_page_table() helper in follow_page_mask(), but the
359 	 * shmem_vm_ops->fault method is invoked even during
360 	 * coredumping without mmap_sem and it ends up here.
361 	 */
362 	if (current->flags & (PF_EXITING|PF_DUMPCORE))
363 		goto out;
364 
365 	/*
366 	 * Coredumping runs without mmap_sem so we can only check that
367 	 * the mmap_sem is held, if PF_DUMPCORE was not set.
368 	 */
369 	WARN_ON_ONCE(!rwsem_is_locked(&mm->mmap_sem));
370 
371 	ctx = vmf->vma->vm_userfaultfd_ctx.ctx;
372 	if (!ctx)
373 		goto out;
374 
375 	BUG_ON(ctx->mm != mm);
376 
377 	VM_BUG_ON(reason & ~(VM_UFFD_MISSING|VM_UFFD_WP));
378 	VM_BUG_ON(!(reason & VM_UFFD_MISSING) ^ !!(reason & VM_UFFD_WP));
379 
380 	if (ctx->features & UFFD_FEATURE_SIGBUS)
381 		goto out;
382 
383 	/*
384 	 * If it's already released don't get it. This avoids to loop
385 	 * in __get_user_pages if userfaultfd_release waits on the
386 	 * caller of handle_userfault to release the mmap_sem.
387 	 */
388 	if (unlikely(READ_ONCE(ctx->released))) {
389 		/*
390 		 * Don't return VM_FAULT_SIGBUS in this case, so a non
391 		 * cooperative manager can close the uffd after the
392 		 * last UFFDIO_COPY, without risking to trigger an
393 		 * involuntary SIGBUS if the process was starting the
394 		 * userfaultfd while the userfaultfd was still armed
395 		 * (but after the last UFFDIO_COPY). If the uffd
396 		 * wasn't already closed when the userfault reached
397 		 * this point, that would normally be solved by
398 		 * userfaultfd_must_wait returning 'false'.
399 		 *
400 		 * If we were to return VM_FAULT_SIGBUS here, the non
401 		 * cooperative manager would be instead forced to
402 		 * always call UFFDIO_UNREGISTER before it can safely
403 		 * close the uffd.
404 		 */
405 		ret = VM_FAULT_NOPAGE;
406 		goto out;
407 	}
408 
409 	/*
410 	 * Check that we can return VM_FAULT_RETRY.
411 	 *
412 	 * NOTE: it should become possible to return VM_FAULT_RETRY
413 	 * even if FAULT_FLAG_TRIED is set without leading to gup()
414 	 * -EBUSY failures, if the userfaultfd is to be extended for
415 	 * VM_UFFD_WP tracking and we intend to arm the userfault
416 	 * without first stopping userland access to the memory. For
417 	 * VM_UFFD_MISSING userfaults this is enough for now.
418 	 */
419 	if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
420 		/*
421 		 * Validate the invariant that nowait must allow retry
422 		 * to be sure not to return SIGBUS erroneously on
423 		 * nowait invocations.
424 		 */
425 		BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
426 #ifdef CONFIG_DEBUG_VM
427 		if (printk_ratelimit()) {
428 			printk(KERN_WARNING
429 			       "FAULT_FLAG_ALLOW_RETRY missing %x\n",
430 			       vmf->flags);
431 			dump_stack();
432 		}
433 #endif
434 		goto out;
435 	}
436 
437 	/*
438 	 * Handle nowait, not much to do other than tell it to retry
439 	 * and wait.
440 	 */
441 	ret = VM_FAULT_RETRY;
442 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
443 		goto out;
444 
445 	/* take the reference before dropping the mmap_sem */
446 	userfaultfd_ctx_get(ctx);
447 
448 	init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
449 	uwq.wq.private = current;
450 	uwq.msg = userfault_msg(vmf->address, vmf->flags, reason,
451 			ctx->features);
452 	uwq.ctx = ctx;
453 	uwq.waken = false;
454 
455 	return_to_userland =
456 		(vmf->flags & (FAULT_FLAG_USER|FAULT_FLAG_KILLABLE)) ==
457 		(FAULT_FLAG_USER|FAULT_FLAG_KILLABLE);
458 	blocking_state = return_to_userland ? TASK_INTERRUPTIBLE :
459 			 TASK_KILLABLE;
460 
461 	spin_lock(&ctx->fault_pending_wqh.lock);
462 	/*
463 	 * After the __add_wait_queue the uwq is visible to userland
464 	 * through poll/read().
465 	 */
466 	__add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
467 	/*
468 	 * The smp_mb() after __set_current_state prevents the reads
469 	 * following the spin_unlock to happen before the list_add in
470 	 * __add_wait_queue.
471 	 */
472 	set_current_state(blocking_state);
473 	spin_unlock(&ctx->fault_pending_wqh.lock);
474 
475 	if (!is_vm_hugetlb_page(vmf->vma))
476 		must_wait = userfaultfd_must_wait(ctx, vmf->address, vmf->flags,
477 						  reason);
478 	else
479 		must_wait = userfaultfd_huge_must_wait(ctx, vmf->vma,
480 						       vmf->address,
481 						       vmf->flags, reason);
482 	up_read(&mm->mmap_sem);
483 
484 	if (likely(must_wait && !READ_ONCE(ctx->released) &&
485 		   (return_to_userland ? !signal_pending(current) :
486 		    !fatal_signal_pending(current)))) {
487 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
488 		schedule();
489 		ret |= VM_FAULT_MAJOR;
490 
491 		/*
492 		 * False wakeups can orginate even from rwsem before
493 		 * up_read() however userfaults will wait either for a
494 		 * targeted wakeup on the specific uwq waitqueue from
495 		 * wake_userfault() or for signals or for uffd
496 		 * release.
497 		 */
498 		while (!READ_ONCE(uwq.waken)) {
499 			/*
500 			 * This needs the full smp_store_mb()
501 			 * guarantee as the state write must be
502 			 * visible to other CPUs before reading
503 			 * uwq.waken from other CPUs.
504 			 */
505 			set_current_state(blocking_state);
506 			if (READ_ONCE(uwq.waken) ||
507 			    READ_ONCE(ctx->released) ||
508 			    (return_to_userland ? signal_pending(current) :
509 			     fatal_signal_pending(current)))
510 				break;
511 			schedule();
512 		}
513 	}
514 
515 	__set_current_state(TASK_RUNNING);
516 
517 	if (return_to_userland) {
518 		if (signal_pending(current) &&
519 		    !fatal_signal_pending(current)) {
520 			/*
521 			 * If we got a SIGSTOP or SIGCONT and this is
522 			 * a normal userland page fault, just let
523 			 * userland return so the signal will be
524 			 * handled and gdb debugging works.  The page
525 			 * fault code immediately after we return from
526 			 * this function is going to release the
527 			 * mmap_sem and it's not depending on it
528 			 * (unlike gup would if we were not to return
529 			 * VM_FAULT_RETRY).
530 			 *
531 			 * If a fatal signal is pending we still take
532 			 * the streamlined VM_FAULT_RETRY failure path
533 			 * and there's no need to retake the mmap_sem
534 			 * in such case.
535 			 */
536 			down_read(&mm->mmap_sem);
537 			ret = VM_FAULT_NOPAGE;
538 		}
539 	}
540 
541 	/*
542 	 * Here we race with the list_del; list_add in
543 	 * userfaultfd_ctx_read(), however because we don't ever run
544 	 * list_del_init() to refile across the two lists, the prev
545 	 * and next pointers will never point to self. list_add also
546 	 * would never let any of the two pointers to point to
547 	 * self. So list_empty_careful won't risk to see both pointers
548 	 * pointing to self at any time during the list refile. The
549 	 * only case where list_del_init() is called is the full
550 	 * removal in the wake function and there we don't re-list_add
551 	 * and it's fine not to block on the spinlock. The uwq on this
552 	 * kernel stack can be released after the list_del_init.
553 	 */
554 	if (!list_empty_careful(&uwq.wq.entry)) {
555 		spin_lock(&ctx->fault_pending_wqh.lock);
556 		/*
557 		 * No need of list_del_init(), the uwq on the stack
558 		 * will be freed shortly anyway.
559 		 */
560 		list_del(&uwq.wq.entry);
561 		spin_unlock(&ctx->fault_pending_wqh.lock);
562 	}
563 
564 	/*
565 	 * ctx may go away after this if the userfault pseudo fd is
566 	 * already released.
567 	 */
568 	userfaultfd_ctx_put(ctx);
569 
570 out:
571 	return ret;
572 }
573 
574 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
575 					      struct userfaultfd_wait_queue *ewq)
576 {
577 	struct userfaultfd_ctx *release_new_ctx;
578 
579 	if (WARN_ON_ONCE(current->flags & PF_EXITING))
580 		goto out;
581 
582 	ewq->ctx = ctx;
583 	init_waitqueue_entry(&ewq->wq, current);
584 	release_new_ctx = NULL;
585 
586 	spin_lock(&ctx->event_wqh.lock);
587 	/*
588 	 * After the __add_wait_queue the uwq is visible to userland
589 	 * through poll/read().
590 	 */
591 	__add_wait_queue(&ctx->event_wqh, &ewq->wq);
592 	for (;;) {
593 		set_current_state(TASK_KILLABLE);
594 		if (ewq->msg.event == 0)
595 			break;
596 		if (READ_ONCE(ctx->released) ||
597 		    fatal_signal_pending(current)) {
598 			/*
599 			 * &ewq->wq may be queued in fork_event, but
600 			 * __remove_wait_queue ignores the head
601 			 * parameter. It would be a problem if it
602 			 * didn't.
603 			 */
604 			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
605 			if (ewq->msg.event == UFFD_EVENT_FORK) {
606 				struct userfaultfd_ctx *new;
607 
608 				new = (struct userfaultfd_ctx *)
609 					(unsigned long)
610 					ewq->msg.arg.reserved.reserved1;
611 				release_new_ctx = new;
612 			}
613 			break;
614 		}
615 
616 		spin_unlock(&ctx->event_wqh.lock);
617 
618 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
619 		schedule();
620 
621 		spin_lock(&ctx->event_wqh.lock);
622 	}
623 	__set_current_state(TASK_RUNNING);
624 	spin_unlock(&ctx->event_wqh.lock);
625 
626 	if (release_new_ctx) {
627 		struct vm_area_struct *vma;
628 		struct mm_struct *mm = release_new_ctx->mm;
629 
630 		/* the various vma->vm_userfaultfd_ctx still points to it */
631 		down_write(&mm->mmap_sem);
632 		/* no task can run (and in turn coredump) yet */
633 		VM_WARN_ON(!mmget_still_valid(mm));
634 		for (vma = mm->mmap; vma; vma = vma->vm_next)
635 			if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
636 				vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
637 				vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
638 			}
639 		up_write(&mm->mmap_sem);
640 
641 		userfaultfd_ctx_put(release_new_ctx);
642 	}
643 
644 	/*
645 	 * ctx may go away after this if the userfault pseudo fd is
646 	 * already released.
647 	 */
648 out:
649 	WRITE_ONCE(ctx->mmap_changing, false);
650 	userfaultfd_ctx_put(ctx);
651 }
652 
653 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
654 				       struct userfaultfd_wait_queue *ewq)
655 {
656 	ewq->msg.event = 0;
657 	wake_up_locked(&ctx->event_wqh);
658 	__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
659 }
660 
661 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
662 {
663 	struct userfaultfd_ctx *ctx = NULL, *octx;
664 	struct userfaultfd_fork_ctx *fctx;
665 
666 	octx = vma->vm_userfaultfd_ctx.ctx;
667 	if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
668 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
669 		vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
670 		return 0;
671 	}
672 
673 	list_for_each_entry(fctx, fcs, list)
674 		if (fctx->orig == octx) {
675 			ctx = fctx->new;
676 			break;
677 		}
678 
679 	if (!ctx) {
680 		fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
681 		if (!fctx)
682 			return -ENOMEM;
683 
684 		ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
685 		if (!ctx) {
686 			kfree(fctx);
687 			return -ENOMEM;
688 		}
689 
690 		refcount_set(&ctx->refcount, 1);
691 		ctx->flags = octx->flags;
692 		ctx->state = UFFD_STATE_RUNNING;
693 		ctx->features = octx->features;
694 		ctx->released = false;
695 		ctx->mmap_changing = false;
696 		ctx->mm = vma->vm_mm;
697 		mmgrab(ctx->mm);
698 
699 		userfaultfd_ctx_get(octx);
700 		WRITE_ONCE(octx->mmap_changing, true);
701 		fctx->orig = octx;
702 		fctx->new = ctx;
703 		list_add_tail(&fctx->list, fcs);
704 	}
705 
706 	vma->vm_userfaultfd_ctx.ctx = ctx;
707 	return 0;
708 }
709 
710 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
711 {
712 	struct userfaultfd_ctx *ctx = fctx->orig;
713 	struct userfaultfd_wait_queue ewq;
714 
715 	msg_init(&ewq.msg);
716 
717 	ewq.msg.event = UFFD_EVENT_FORK;
718 	ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
719 
720 	userfaultfd_event_wait_completion(ctx, &ewq);
721 }
722 
723 void dup_userfaultfd_complete(struct list_head *fcs)
724 {
725 	struct userfaultfd_fork_ctx *fctx, *n;
726 
727 	list_for_each_entry_safe(fctx, n, fcs, list) {
728 		dup_fctx(fctx);
729 		list_del(&fctx->list);
730 		kfree(fctx);
731 	}
732 }
733 
734 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
735 			     struct vm_userfaultfd_ctx *vm_ctx)
736 {
737 	struct userfaultfd_ctx *ctx;
738 
739 	ctx = vma->vm_userfaultfd_ctx.ctx;
740 
741 	if (!ctx)
742 		return;
743 
744 	if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
745 		vm_ctx->ctx = ctx;
746 		userfaultfd_ctx_get(ctx);
747 		WRITE_ONCE(ctx->mmap_changing, true);
748 	} else {
749 		/* Drop uffd context if remap feature not enabled */
750 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
751 		vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
752 	}
753 }
754 
755 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
756 				 unsigned long from, unsigned long to,
757 				 unsigned long len)
758 {
759 	struct userfaultfd_ctx *ctx = vm_ctx->ctx;
760 	struct userfaultfd_wait_queue ewq;
761 
762 	if (!ctx)
763 		return;
764 
765 	if (to & ~PAGE_MASK) {
766 		userfaultfd_ctx_put(ctx);
767 		return;
768 	}
769 
770 	msg_init(&ewq.msg);
771 
772 	ewq.msg.event = UFFD_EVENT_REMAP;
773 	ewq.msg.arg.remap.from = from;
774 	ewq.msg.arg.remap.to = to;
775 	ewq.msg.arg.remap.len = len;
776 
777 	userfaultfd_event_wait_completion(ctx, &ewq);
778 }
779 
780 bool userfaultfd_remove(struct vm_area_struct *vma,
781 			unsigned long start, unsigned long end)
782 {
783 	struct mm_struct *mm = vma->vm_mm;
784 	struct userfaultfd_ctx *ctx;
785 	struct userfaultfd_wait_queue ewq;
786 
787 	ctx = vma->vm_userfaultfd_ctx.ctx;
788 	if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
789 		return true;
790 
791 	userfaultfd_ctx_get(ctx);
792 	WRITE_ONCE(ctx->mmap_changing, true);
793 	up_read(&mm->mmap_sem);
794 
795 	msg_init(&ewq.msg);
796 
797 	ewq.msg.event = UFFD_EVENT_REMOVE;
798 	ewq.msg.arg.remove.start = start;
799 	ewq.msg.arg.remove.end = end;
800 
801 	userfaultfd_event_wait_completion(ctx, &ewq);
802 
803 	return false;
804 }
805 
806 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
807 			  unsigned long start, unsigned long end)
808 {
809 	struct userfaultfd_unmap_ctx *unmap_ctx;
810 
811 	list_for_each_entry(unmap_ctx, unmaps, list)
812 		if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
813 		    unmap_ctx->end == end)
814 			return true;
815 
816 	return false;
817 }
818 
819 int userfaultfd_unmap_prep(struct vm_area_struct *vma,
820 			   unsigned long start, unsigned long end,
821 			   struct list_head *unmaps)
822 {
823 	for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
824 		struct userfaultfd_unmap_ctx *unmap_ctx;
825 		struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
826 
827 		if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
828 		    has_unmap_ctx(ctx, unmaps, start, end))
829 			continue;
830 
831 		unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
832 		if (!unmap_ctx)
833 			return -ENOMEM;
834 
835 		userfaultfd_ctx_get(ctx);
836 		WRITE_ONCE(ctx->mmap_changing, true);
837 		unmap_ctx->ctx = ctx;
838 		unmap_ctx->start = start;
839 		unmap_ctx->end = end;
840 		list_add_tail(&unmap_ctx->list, unmaps);
841 	}
842 
843 	return 0;
844 }
845 
846 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
847 {
848 	struct userfaultfd_unmap_ctx *ctx, *n;
849 	struct userfaultfd_wait_queue ewq;
850 
851 	list_for_each_entry_safe(ctx, n, uf, list) {
852 		msg_init(&ewq.msg);
853 
854 		ewq.msg.event = UFFD_EVENT_UNMAP;
855 		ewq.msg.arg.remove.start = ctx->start;
856 		ewq.msg.arg.remove.end = ctx->end;
857 
858 		userfaultfd_event_wait_completion(ctx->ctx, &ewq);
859 
860 		list_del(&ctx->list);
861 		kfree(ctx);
862 	}
863 }
864 
865 static int userfaultfd_release(struct inode *inode, struct file *file)
866 {
867 	struct userfaultfd_ctx *ctx = file->private_data;
868 	struct mm_struct *mm = ctx->mm;
869 	struct vm_area_struct *vma, *prev;
870 	/* len == 0 means wake all */
871 	struct userfaultfd_wake_range range = { .len = 0, };
872 	unsigned long new_flags;
873 
874 	WRITE_ONCE(ctx->released, true);
875 
876 	if (!mmget_not_zero(mm))
877 		goto wakeup;
878 
879 	/*
880 	 * Flush page faults out of all CPUs. NOTE: all page faults
881 	 * must be retried without returning VM_FAULT_SIGBUS if
882 	 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
883 	 * changes while handle_userfault released the mmap_sem. So
884 	 * it's critical that released is set to true (above), before
885 	 * taking the mmap_sem for writing.
886 	 */
887 	down_write(&mm->mmap_sem);
888 	if (!mmget_still_valid(mm))
889 		goto skip_mm;
890 	prev = NULL;
891 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
892 		cond_resched();
893 		BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
894 		       !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
895 		if (vma->vm_userfaultfd_ctx.ctx != ctx) {
896 			prev = vma;
897 			continue;
898 		}
899 		new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
900 		prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
901 				 new_flags, vma->anon_vma,
902 				 vma->vm_file, vma->vm_pgoff,
903 				 vma_policy(vma),
904 				 NULL_VM_UFFD_CTX);
905 		if (prev)
906 			vma = prev;
907 		else
908 			prev = vma;
909 		vma->vm_flags = new_flags;
910 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
911 	}
912 skip_mm:
913 	up_write(&mm->mmap_sem);
914 	mmput(mm);
915 wakeup:
916 	/*
917 	 * After no new page faults can wait on this fault_*wqh, flush
918 	 * the last page faults that may have been already waiting on
919 	 * the fault_*wqh.
920 	 */
921 	spin_lock(&ctx->fault_pending_wqh.lock);
922 	__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
923 	__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
924 	spin_unlock(&ctx->fault_pending_wqh.lock);
925 
926 	/* Flush pending events that may still wait on event_wqh */
927 	wake_up_all(&ctx->event_wqh);
928 
929 	wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
930 	userfaultfd_ctx_put(ctx);
931 	return 0;
932 }
933 
934 /* fault_pending_wqh.lock must be hold by the caller */
935 static inline struct userfaultfd_wait_queue *find_userfault_in(
936 		wait_queue_head_t *wqh)
937 {
938 	wait_queue_entry_t *wq;
939 	struct userfaultfd_wait_queue *uwq;
940 
941 	lockdep_assert_held(&wqh->lock);
942 
943 	uwq = NULL;
944 	if (!waitqueue_active(wqh))
945 		goto out;
946 	/* walk in reverse to provide FIFO behavior to read userfaults */
947 	wq = list_last_entry(&wqh->head, typeof(*wq), entry);
948 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
949 out:
950 	return uwq;
951 }
952 
953 static inline struct userfaultfd_wait_queue *find_userfault(
954 		struct userfaultfd_ctx *ctx)
955 {
956 	return find_userfault_in(&ctx->fault_pending_wqh);
957 }
958 
959 static inline struct userfaultfd_wait_queue *find_userfault_evt(
960 		struct userfaultfd_ctx *ctx)
961 {
962 	return find_userfault_in(&ctx->event_wqh);
963 }
964 
965 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
966 {
967 	struct userfaultfd_ctx *ctx = file->private_data;
968 	__poll_t ret;
969 
970 	poll_wait(file, &ctx->fd_wqh, wait);
971 
972 	switch (ctx->state) {
973 	case UFFD_STATE_WAIT_API:
974 		return EPOLLERR;
975 	case UFFD_STATE_RUNNING:
976 		/*
977 		 * poll() never guarantees that read won't block.
978 		 * userfaults can be waken before they're read().
979 		 */
980 		if (unlikely(!(file->f_flags & O_NONBLOCK)))
981 			return EPOLLERR;
982 		/*
983 		 * lockless access to see if there are pending faults
984 		 * __pollwait last action is the add_wait_queue but
985 		 * the spin_unlock would allow the waitqueue_active to
986 		 * pass above the actual list_add inside
987 		 * add_wait_queue critical section. So use a full
988 		 * memory barrier to serialize the list_add write of
989 		 * add_wait_queue() with the waitqueue_active read
990 		 * below.
991 		 */
992 		ret = 0;
993 		smp_mb();
994 		if (waitqueue_active(&ctx->fault_pending_wqh))
995 			ret = EPOLLIN;
996 		else if (waitqueue_active(&ctx->event_wqh))
997 			ret = EPOLLIN;
998 
999 		return ret;
1000 	default:
1001 		WARN_ON_ONCE(1);
1002 		return EPOLLERR;
1003 	}
1004 }
1005 
1006 static const struct file_operations userfaultfd_fops;
1007 
1008 static int resolve_userfault_fork(struct userfaultfd_ctx *ctx,
1009 				  struct userfaultfd_ctx *new,
1010 				  struct uffd_msg *msg)
1011 {
1012 	int fd;
1013 
1014 	fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, new,
1015 			      O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS));
1016 	if (fd < 0)
1017 		return fd;
1018 
1019 	msg->arg.reserved.reserved1 = 0;
1020 	msg->arg.fork.ufd = fd;
1021 	return 0;
1022 }
1023 
1024 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1025 				    struct uffd_msg *msg)
1026 {
1027 	ssize_t ret;
1028 	DECLARE_WAITQUEUE(wait, current);
1029 	struct userfaultfd_wait_queue *uwq;
1030 	/*
1031 	 * Handling fork event requires sleeping operations, so
1032 	 * we drop the event_wqh lock, then do these ops, then
1033 	 * lock it back and wake up the waiter. While the lock is
1034 	 * dropped the ewq may go away so we keep track of it
1035 	 * carefully.
1036 	 */
1037 	LIST_HEAD(fork_event);
1038 	struct userfaultfd_ctx *fork_nctx = NULL;
1039 
1040 	/* always take the fd_wqh lock before the fault_pending_wqh lock */
1041 	spin_lock_irq(&ctx->fd_wqh.lock);
1042 	__add_wait_queue(&ctx->fd_wqh, &wait);
1043 	for (;;) {
1044 		set_current_state(TASK_INTERRUPTIBLE);
1045 		spin_lock(&ctx->fault_pending_wqh.lock);
1046 		uwq = find_userfault(ctx);
1047 		if (uwq) {
1048 			/*
1049 			 * Use a seqcount to repeat the lockless check
1050 			 * in wake_userfault() to avoid missing
1051 			 * wakeups because during the refile both
1052 			 * waitqueue could become empty if this is the
1053 			 * only userfault.
1054 			 */
1055 			write_seqcount_begin(&ctx->refile_seq);
1056 
1057 			/*
1058 			 * The fault_pending_wqh.lock prevents the uwq
1059 			 * to disappear from under us.
1060 			 *
1061 			 * Refile this userfault from
1062 			 * fault_pending_wqh to fault_wqh, it's not
1063 			 * pending anymore after we read it.
1064 			 *
1065 			 * Use list_del() by hand (as
1066 			 * userfaultfd_wake_function also uses
1067 			 * list_del_init() by hand) to be sure nobody
1068 			 * changes __remove_wait_queue() to use
1069 			 * list_del_init() in turn breaking the
1070 			 * !list_empty_careful() check in
1071 			 * handle_userfault(). The uwq->wq.head list
1072 			 * must never be empty at any time during the
1073 			 * refile, or the waitqueue could disappear
1074 			 * from under us. The "wait_queue_head_t"
1075 			 * parameter of __remove_wait_queue() is unused
1076 			 * anyway.
1077 			 */
1078 			list_del(&uwq->wq.entry);
1079 			add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1080 
1081 			write_seqcount_end(&ctx->refile_seq);
1082 
1083 			/* careful to always initialize msg if ret == 0 */
1084 			*msg = uwq->msg;
1085 			spin_unlock(&ctx->fault_pending_wqh.lock);
1086 			ret = 0;
1087 			break;
1088 		}
1089 		spin_unlock(&ctx->fault_pending_wqh.lock);
1090 
1091 		spin_lock(&ctx->event_wqh.lock);
1092 		uwq = find_userfault_evt(ctx);
1093 		if (uwq) {
1094 			*msg = uwq->msg;
1095 
1096 			if (uwq->msg.event == UFFD_EVENT_FORK) {
1097 				fork_nctx = (struct userfaultfd_ctx *)
1098 					(unsigned long)
1099 					uwq->msg.arg.reserved.reserved1;
1100 				list_move(&uwq->wq.entry, &fork_event);
1101 				/*
1102 				 * fork_nctx can be freed as soon as
1103 				 * we drop the lock, unless we take a
1104 				 * reference on it.
1105 				 */
1106 				userfaultfd_ctx_get(fork_nctx);
1107 				spin_unlock(&ctx->event_wqh.lock);
1108 				ret = 0;
1109 				break;
1110 			}
1111 
1112 			userfaultfd_event_complete(ctx, uwq);
1113 			spin_unlock(&ctx->event_wqh.lock);
1114 			ret = 0;
1115 			break;
1116 		}
1117 		spin_unlock(&ctx->event_wqh.lock);
1118 
1119 		if (signal_pending(current)) {
1120 			ret = -ERESTARTSYS;
1121 			break;
1122 		}
1123 		if (no_wait) {
1124 			ret = -EAGAIN;
1125 			break;
1126 		}
1127 		spin_unlock_irq(&ctx->fd_wqh.lock);
1128 		schedule();
1129 		spin_lock_irq(&ctx->fd_wqh.lock);
1130 	}
1131 	__remove_wait_queue(&ctx->fd_wqh, &wait);
1132 	__set_current_state(TASK_RUNNING);
1133 	spin_unlock_irq(&ctx->fd_wqh.lock);
1134 
1135 	if (!ret && msg->event == UFFD_EVENT_FORK) {
1136 		ret = resolve_userfault_fork(ctx, fork_nctx, msg);
1137 		spin_lock(&ctx->event_wqh.lock);
1138 		if (!list_empty(&fork_event)) {
1139 			/*
1140 			 * The fork thread didn't abort, so we can
1141 			 * drop the temporary refcount.
1142 			 */
1143 			userfaultfd_ctx_put(fork_nctx);
1144 
1145 			uwq = list_first_entry(&fork_event,
1146 					       typeof(*uwq),
1147 					       wq.entry);
1148 			/*
1149 			 * If fork_event list wasn't empty and in turn
1150 			 * the event wasn't already released by fork
1151 			 * (the event is allocated on fork kernel
1152 			 * stack), put the event back to its place in
1153 			 * the event_wq. fork_event head will be freed
1154 			 * as soon as we return so the event cannot
1155 			 * stay queued there no matter the current
1156 			 * "ret" value.
1157 			 */
1158 			list_del(&uwq->wq.entry);
1159 			__add_wait_queue(&ctx->event_wqh, &uwq->wq);
1160 
1161 			/*
1162 			 * Leave the event in the waitqueue and report
1163 			 * error to userland if we failed to resolve
1164 			 * the userfault fork.
1165 			 */
1166 			if (likely(!ret))
1167 				userfaultfd_event_complete(ctx, uwq);
1168 		} else {
1169 			/*
1170 			 * Here the fork thread aborted and the
1171 			 * refcount from the fork thread on fork_nctx
1172 			 * has already been released. We still hold
1173 			 * the reference we took before releasing the
1174 			 * lock above. If resolve_userfault_fork
1175 			 * failed we've to drop it because the
1176 			 * fork_nctx has to be freed in such case. If
1177 			 * it succeeded we'll hold it because the new
1178 			 * uffd references it.
1179 			 */
1180 			if (ret)
1181 				userfaultfd_ctx_put(fork_nctx);
1182 		}
1183 		spin_unlock(&ctx->event_wqh.lock);
1184 	}
1185 
1186 	return ret;
1187 }
1188 
1189 static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1190 				size_t count, loff_t *ppos)
1191 {
1192 	struct userfaultfd_ctx *ctx = file->private_data;
1193 	ssize_t _ret, ret = 0;
1194 	struct uffd_msg msg;
1195 	int no_wait = file->f_flags & O_NONBLOCK;
1196 
1197 	if (ctx->state == UFFD_STATE_WAIT_API)
1198 		return -EINVAL;
1199 
1200 	for (;;) {
1201 		if (count < sizeof(msg))
1202 			return ret ? ret : -EINVAL;
1203 		_ret = userfaultfd_ctx_read(ctx, no_wait, &msg);
1204 		if (_ret < 0)
1205 			return ret ? ret : _ret;
1206 		if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1207 			return ret ? ret : -EFAULT;
1208 		ret += sizeof(msg);
1209 		buf += sizeof(msg);
1210 		count -= sizeof(msg);
1211 		/*
1212 		 * Allow to read more than one fault at time but only
1213 		 * block if waiting for the very first one.
1214 		 */
1215 		no_wait = O_NONBLOCK;
1216 	}
1217 }
1218 
1219 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1220 			     struct userfaultfd_wake_range *range)
1221 {
1222 	spin_lock(&ctx->fault_pending_wqh.lock);
1223 	/* wake all in the range and autoremove */
1224 	if (waitqueue_active(&ctx->fault_pending_wqh))
1225 		__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1226 				     range);
1227 	if (waitqueue_active(&ctx->fault_wqh))
1228 		__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1229 	spin_unlock(&ctx->fault_pending_wqh.lock);
1230 }
1231 
1232 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1233 					   struct userfaultfd_wake_range *range)
1234 {
1235 	unsigned seq;
1236 	bool need_wakeup;
1237 
1238 	/*
1239 	 * To be sure waitqueue_active() is not reordered by the CPU
1240 	 * before the pagetable update, use an explicit SMP memory
1241 	 * barrier here. PT lock release or up_read(mmap_sem) still
1242 	 * have release semantics that can allow the
1243 	 * waitqueue_active() to be reordered before the pte update.
1244 	 */
1245 	smp_mb();
1246 
1247 	/*
1248 	 * Use waitqueue_active because it's very frequent to
1249 	 * change the address space atomically even if there are no
1250 	 * userfaults yet. So we take the spinlock only when we're
1251 	 * sure we've userfaults to wake.
1252 	 */
1253 	do {
1254 		seq = read_seqcount_begin(&ctx->refile_seq);
1255 		need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1256 			waitqueue_active(&ctx->fault_wqh);
1257 		cond_resched();
1258 	} while (read_seqcount_retry(&ctx->refile_seq, seq));
1259 	if (need_wakeup)
1260 		__wake_userfault(ctx, range);
1261 }
1262 
1263 static __always_inline int validate_range(struct mm_struct *mm,
1264 					  __u64 start, __u64 len)
1265 {
1266 	__u64 task_size = mm->task_size;
1267 
1268 	if (start & ~PAGE_MASK)
1269 		return -EINVAL;
1270 	if (len & ~PAGE_MASK)
1271 		return -EINVAL;
1272 	if (!len)
1273 		return -EINVAL;
1274 	if (start < mmap_min_addr)
1275 		return -EINVAL;
1276 	if (start >= task_size)
1277 		return -EINVAL;
1278 	if (len > task_size - start)
1279 		return -EINVAL;
1280 	return 0;
1281 }
1282 
1283 static inline bool vma_can_userfault(struct vm_area_struct *vma)
1284 {
1285 	return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) ||
1286 		vma_is_shmem(vma);
1287 }
1288 
1289 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1290 				unsigned long arg)
1291 {
1292 	struct mm_struct *mm = ctx->mm;
1293 	struct vm_area_struct *vma, *prev, *cur;
1294 	int ret;
1295 	struct uffdio_register uffdio_register;
1296 	struct uffdio_register __user *user_uffdio_register;
1297 	unsigned long vm_flags, new_flags;
1298 	bool found;
1299 	bool basic_ioctls;
1300 	unsigned long start, end, vma_end;
1301 
1302 	user_uffdio_register = (struct uffdio_register __user *) arg;
1303 
1304 	ret = -EFAULT;
1305 	if (copy_from_user(&uffdio_register, user_uffdio_register,
1306 			   sizeof(uffdio_register)-sizeof(__u64)))
1307 		goto out;
1308 
1309 	ret = -EINVAL;
1310 	if (!uffdio_register.mode)
1311 		goto out;
1312 	if (uffdio_register.mode & ~(UFFDIO_REGISTER_MODE_MISSING|
1313 				     UFFDIO_REGISTER_MODE_WP))
1314 		goto out;
1315 	vm_flags = 0;
1316 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1317 		vm_flags |= VM_UFFD_MISSING;
1318 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1319 		vm_flags |= VM_UFFD_WP;
1320 		/*
1321 		 * FIXME: remove the below error constraint by
1322 		 * implementing the wprotect tracking mode.
1323 		 */
1324 		ret = -EINVAL;
1325 		goto out;
1326 	}
1327 
1328 	ret = validate_range(mm, uffdio_register.range.start,
1329 			     uffdio_register.range.len);
1330 	if (ret)
1331 		goto out;
1332 
1333 	start = uffdio_register.range.start;
1334 	end = start + uffdio_register.range.len;
1335 
1336 	ret = -ENOMEM;
1337 	if (!mmget_not_zero(mm))
1338 		goto out;
1339 
1340 	down_write(&mm->mmap_sem);
1341 	if (!mmget_still_valid(mm))
1342 		goto out_unlock;
1343 	vma = find_vma_prev(mm, start, &prev);
1344 	if (!vma)
1345 		goto out_unlock;
1346 
1347 	/* check that there's at least one vma in the range */
1348 	ret = -EINVAL;
1349 	if (vma->vm_start >= end)
1350 		goto out_unlock;
1351 
1352 	/*
1353 	 * If the first vma contains huge pages, make sure start address
1354 	 * is aligned to huge page size.
1355 	 */
1356 	if (is_vm_hugetlb_page(vma)) {
1357 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1358 
1359 		if (start & (vma_hpagesize - 1))
1360 			goto out_unlock;
1361 	}
1362 
1363 	/*
1364 	 * Search for not compatible vmas.
1365 	 */
1366 	found = false;
1367 	basic_ioctls = false;
1368 	for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1369 		cond_resched();
1370 
1371 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1372 		       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1373 
1374 		/* check not compatible vmas */
1375 		ret = -EINVAL;
1376 		if (!vma_can_userfault(cur))
1377 			goto out_unlock;
1378 
1379 		/*
1380 		 * UFFDIO_COPY will fill file holes even without
1381 		 * PROT_WRITE. This check enforces that if this is a
1382 		 * MAP_SHARED, the process has write permission to the backing
1383 		 * file. If VM_MAYWRITE is set it also enforces that on a
1384 		 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1385 		 * F_WRITE_SEAL can be taken until the vma is destroyed.
1386 		 */
1387 		ret = -EPERM;
1388 		if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1389 			goto out_unlock;
1390 
1391 		/*
1392 		 * If this vma contains ending address, and huge pages
1393 		 * check alignment.
1394 		 */
1395 		if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1396 		    end > cur->vm_start) {
1397 			unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1398 
1399 			ret = -EINVAL;
1400 
1401 			if (end & (vma_hpagesize - 1))
1402 				goto out_unlock;
1403 		}
1404 
1405 		/*
1406 		 * Check that this vma isn't already owned by a
1407 		 * different userfaultfd. We can't allow more than one
1408 		 * userfaultfd to own a single vma simultaneously or we
1409 		 * wouldn't know which one to deliver the userfaults to.
1410 		 */
1411 		ret = -EBUSY;
1412 		if (cur->vm_userfaultfd_ctx.ctx &&
1413 		    cur->vm_userfaultfd_ctx.ctx != ctx)
1414 			goto out_unlock;
1415 
1416 		/*
1417 		 * Note vmas containing huge pages
1418 		 */
1419 		if (is_vm_hugetlb_page(cur))
1420 			basic_ioctls = true;
1421 
1422 		found = true;
1423 	}
1424 	BUG_ON(!found);
1425 
1426 	if (vma->vm_start < start)
1427 		prev = vma;
1428 
1429 	ret = 0;
1430 	do {
1431 		cond_resched();
1432 
1433 		BUG_ON(!vma_can_userfault(vma));
1434 		BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1435 		       vma->vm_userfaultfd_ctx.ctx != ctx);
1436 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1437 
1438 		/*
1439 		 * Nothing to do: this vma is already registered into this
1440 		 * userfaultfd and with the right tracking mode too.
1441 		 */
1442 		if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1443 		    (vma->vm_flags & vm_flags) == vm_flags)
1444 			goto skip;
1445 
1446 		if (vma->vm_start > start)
1447 			start = vma->vm_start;
1448 		vma_end = min(end, vma->vm_end);
1449 
1450 		new_flags = (vma->vm_flags & ~vm_flags) | vm_flags;
1451 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1452 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1453 				 vma_policy(vma),
1454 				 ((struct vm_userfaultfd_ctx){ ctx }));
1455 		if (prev) {
1456 			vma = prev;
1457 			goto next;
1458 		}
1459 		if (vma->vm_start < start) {
1460 			ret = split_vma(mm, vma, start, 1);
1461 			if (ret)
1462 				break;
1463 		}
1464 		if (vma->vm_end > end) {
1465 			ret = split_vma(mm, vma, end, 0);
1466 			if (ret)
1467 				break;
1468 		}
1469 	next:
1470 		/*
1471 		 * In the vma_merge() successful mprotect-like case 8:
1472 		 * the next vma was merged into the current one and
1473 		 * the current one has not been updated yet.
1474 		 */
1475 		vma->vm_flags = new_flags;
1476 		vma->vm_userfaultfd_ctx.ctx = ctx;
1477 
1478 	skip:
1479 		prev = vma;
1480 		start = vma->vm_end;
1481 		vma = vma->vm_next;
1482 	} while (vma && vma->vm_start < end);
1483 out_unlock:
1484 	up_write(&mm->mmap_sem);
1485 	mmput(mm);
1486 	if (!ret) {
1487 		/*
1488 		 * Now that we scanned all vmas we can already tell
1489 		 * userland which ioctls methods are guaranteed to
1490 		 * succeed on this range.
1491 		 */
1492 		if (put_user(basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1493 			     UFFD_API_RANGE_IOCTLS,
1494 			     &user_uffdio_register->ioctls))
1495 			ret = -EFAULT;
1496 	}
1497 out:
1498 	return ret;
1499 }
1500 
1501 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1502 				  unsigned long arg)
1503 {
1504 	struct mm_struct *mm = ctx->mm;
1505 	struct vm_area_struct *vma, *prev, *cur;
1506 	int ret;
1507 	struct uffdio_range uffdio_unregister;
1508 	unsigned long new_flags;
1509 	bool found;
1510 	unsigned long start, end, vma_end;
1511 	const void __user *buf = (void __user *)arg;
1512 
1513 	ret = -EFAULT;
1514 	if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1515 		goto out;
1516 
1517 	ret = validate_range(mm, uffdio_unregister.start,
1518 			     uffdio_unregister.len);
1519 	if (ret)
1520 		goto out;
1521 
1522 	start = uffdio_unregister.start;
1523 	end = start + uffdio_unregister.len;
1524 
1525 	ret = -ENOMEM;
1526 	if (!mmget_not_zero(mm))
1527 		goto out;
1528 
1529 	down_write(&mm->mmap_sem);
1530 	if (!mmget_still_valid(mm))
1531 		goto out_unlock;
1532 	vma = find_vma_prev(mm, start, &prev);
1533 	if (!vma)
1534 		goto out_unlock;
1535 
1536 	/* check that there's at least one vma in the range */
1537 	ret = -EINVAL;
1538 	if (vma->vm_start >= end)
1539 		goto out_unlock;
1540 
1541 	/*
1542 	 * If the first vma contains huge pages, make sure start address
1543 	 * is aligned to huge page size.
1544 	 */
1545 	if (is_vm_hugetlb_page(vma)) {
1546 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1547 
1548 		if (start & (vma_hpagesize - 1))
1549 			goto out_unlock;
1550 	}
1551 
1552 	/*
1553 	 * Search for not compatible vmas.
1554 	 */
1555 	found = false;
1556 	ret = -EINVAL;
1557 	for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
1558 		cond_resched();
1559 
1560 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1561 		       !!(cur->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
1562 
1563 		/*
1564 		 * Check not compatible vmas, not strictly required
1565 		 * here as not compatible vmas cannot have an
1566 		 * userfaultfd_ctx registered on them, but this
1567 		 * provides for more strict behavior to notice
1568 		 * unregistration errors.
1569 		 */
1570 		if (!vma_can_userfault(cur))
1571 			goto out_unlock;
1572 
1573 		found = true;
1574 	}
1575 	BUG_ON(!found);
1576 
1577 	if (vma->vm_start < start)
1578 		prev = vma;
1579 
1580 	ret = 0;
1581 	do {
1582 		cond_resched();
1583 
1584 		BUG_ON(!vma_can_userfault(vma));
1585 
1586 		/*
1587 		 * Nothing to do: this vma is already registered into this
1588 		 * userfaultfd and with the right tracking mode too.
1589 		 */
1590 		if (!vma->vm_userfaultfd_ctx.ctx)
1591 			goto skip;
1592 
1593 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1594 
1595 		if (vma->vm_start > start)
1596 			start = vma->vm_start;
1597 		vma_end = min(end, vma->vm_end);
1598 
1599 		if (userfaultfd_missing(vma)) {
1600 			/*
1601 			 * Wake any concurrent pending userfault while
1602 			 * we unregister, so they will not hang
1603 			 * permanently and it avoids userland to call
1604 			 * UFFDIO_WAKE explicitly.
1605 			 */
1606 			struct userfaultfd_wake_range range;
1607 			range.start = start;
1608 			range.len = vma_end - start;
1609 			wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1610 		}
1611 
1612 		new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
1613 		prev = vma_merge(mm, prev, start, vma_end, new_flags,
1614 				 vma->anon_vma, vma->vm_file, vma->vm_pgoff,
1615 				 vma_policy(vma),
1616 				 NULL_VM_UFFD_CTX);
1617 		if (prev) {
1618 			vma = prev;
1619 			goto next;
1620 		}
1621 		if (vma->vm_start < start) {
1622 			ret = split_vma(mm, vma, start, 1);
1623 			if (ret)
1624 				break;
1625 		}
1626 		if (vma->vm_end > end) {
1627 			ret = split_vma(mm, vma, end, 0);
1628 			if (ret)
1629 				break;
1630 		}
1631 	next:
1632 		/*
1633 		 * In the vma_merge() successful mprotect-like case 8:
1634 		 * the next vma was merged into the current one and
1635 		 * the current one has not been updated yet.
1636 		 */
1637 		vma->vm_flags = new_flags;
1638 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1639 
1640 	skip:
1641 		prev = vma;
1642 		start = vma->vm_end;
1643 		vma = vma->vm_next;
1644 	} while (vma && vma->vm_start < end);
1645 out_unlock:
1646 	up_write(&mm->mmap_sem);
1647 	mmput(mm);
1648 out:
1649 	return ret;
1650 }
1651 
1652 /*
1653  * userfaultfd_wake may be used in combination with the
1654  * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1655  */
1656 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1657 			    unsigned long arg)
1658 {
1659 	int ret;
1660 	struct uffdio_range uffdio_wake;
1661 	struct userfaultfd_wake_range range;
1662 	const void __user *buf = (void __user *)arg;
1663 
1664 	ret = -EFAULT;
1665 	if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1666 		goto out;
1667 
1668 	ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1669 	if (ret)
1670 		goto out;
1671 
1672 	range.start = uffdio_wake.start;
1673 	range.len = uffdio_wake.len;
1674 
1675 	/*
1676 	 * len == 0 means wake all and we don't want to wake all here,
1677 	 * so check it again to be sure.
1678 	 */
1679 	VM_BUG_ON(!range.len);
1680 
1681 	wake_userfault(ctx, &range);
1682 	ret = 0;
1683 
1684 out:
1685 	return ret;
1686 }
1687 
1688 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1689 			    unsigned long arg)
1690 {
1691 	__s64 ret;
1692 	struct uffdio_copy uffdio_copy;
1693 	struct uffdio_copy __user *user_uffdio_copy;
1694 	struct userfaultfd_wake_range range;
1695 
1696 	user_uffdio_copy = (struct uffdio_copy __user *) arg;
1697 
1698 	ret = -EAGAIN;
1699 	if (READ_ONCE(ctx->mmap_changing))
1700 		goto out;
1701 
1702 	ret = -EFAULT;
1703 	if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1704 			   /* don't copy "copy" last field */
1705 			   sizeof(uffdio_copy)-sizeof(__s64)))
1706 		goto out;
1707 
1708 	ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1709 	if (ret)
1710 		goto out;
1711 	/*
1712 	 * double check for wraparound just in case. copy_from_user()
1713 	 * will later check uffdio_copy.src + uffdio_copy.len to fit
1714 	 * in the userland range.
1715 	 */
1716 	ret = -EINVAL;
1717 	if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
1718 		goto out;
1719 	if (uffdio_copy.mode & ~UFFDIO_COPY_MODE_DONTWAKE)
1720 		goto out;
1721 	if (mmget_not_zero(ctx->mm)) {
1722 		ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1723 				   uffdio_copy.len, &ctx->mmap_changing);
1724 		mmput(ctx->mm);
1725 	} else {
1726 		return -ESRCH;
1727 	}
1728 	if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1729 		return -EFAULT;
1730 	if (ret < 0)
1731 		goto out;
1732 	BUG_ON(!ret);
1733 	/* len == 0 would wake all */
1734 	range.len = ret;
1735 	if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1736 		range.start = uffdio_copy.dst;
1737 		wake_userfault(ctx, &range);
1738 	}
1739 	ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1740 out:
1741 	return ret;
1742 }
1743 
1744 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1745 				unsigned long arg)
1746 {
1747 	__s64 ret;
1748 	struct uffdio_zeropage uffdio_zeropage;
1749 	struct uffdio_zeropage __user *user_uffdio_zeropage;
1750 	struct userfaultfd_wake_range range;
1751 
1752 	user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1753 
1754 	ret = -EAGAIN;
1755 	if (READ_ONCE(ctx->mmap_changing))
1756 		goto out;
1757 
1758 	ret = -EFAULT;
1759 	if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1760 			   /* don't copy "zeropage" last field */
1761 			   sizeof(uffdio_zeropage)-sizeof(__s64)))
1762 		goto out;
1763 
1764 	ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1765 			     uffdio_zeropage.range.len);
1766 	if (ret)
1767 		goto out;
1768 	ret = -EINVAL;
1769 	if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1770 		goto out;
1771 
1772 	if (mmget_not_zero(ctx->mm)) {
1773 		ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
1774 				     uffdio_zeropage.range.len,
1775 				     &ctx->mmap_changing);
1776 		mmput(ctx->mm);
1777 	} else {
1778 		return -ESRCH;
1779 	}
1780 	if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1781 		return -EFAULT;
1782 	if (ret < 0)
1783 		goto out;
1784 	/* len == 0 would wake all */
1785 	BUG_ON(!ret);
1786 	range.len = ret;
1787 	if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1788 		range.start = uffdio_zeropage.range.start;
1789 		wake_userfault(ctx, &range);
1790 	}
1791 	ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1792 out:
1793 	return ret;
1794 }
1795 
1796 static inline unsigned int uffd_ctx_features(__u64 user_features)
1797 {
1798 	/*
1799 	 * For the current set of features the bits just coincide
1800 	 */
1801 	return (unsigned int)user_features;
1802 }
1803 
1804 /*
1805  * userland asks for a certain API version and we return which bits
1806  * and ioctl commands are implemented in this kernel for such API
1807  * version or -EINVAL if unknown.
1808  */
1809 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
1810 			   unsigned long arg)
1811 {
1812 	struct uffdio_api uffdio_api;
1813 	void __user *buf = (void __user *)arg;
1814 	int ret;
1815 	__u64 features;
1816 
1817 	ret = -EINVAL;
1818 	if (ctx->state != UFFD_STATE_WAIT_API)
1819 		goto out;
1820 	ret = -EFAULT;
1821 	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
1822 		goto out;
1823 	features = uffdio_api.features;
1824 	if (uffdio_api.api != UFFD_API || (features & ~UFFD_API_FEATURES)) {
1825 		memset(&uffdio_api, 0, sizeof(uffdio_api));
1826 		if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1827 			goto out;
1828 		ret = -EINVAL;
1829 		goto out;
1830 	}
1831 	/* report all available features and ioctls to userland */
1832 	uffdio_api.features = UFFD_API_FEATURES;
1833 	uffdio_api.ioctls = UFFD_API_IOCTLS;
1834 	ret = -EFAULT;
1835 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
1836 		goto out;
1837 	ctx->state = UFFD_STATE_RUNNING;
1838 	/* only enable the requested features for this uffd context */
1839 	ctx->features = uffd_ctx_features(features);
1840 	ret = 0;
1841 out:
1842 	return ret;
1843 }
1844 
1845 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
1846 			      unsigned long arg)
1847 {
1848 	int ret = -EINVAL;
1849 	struct userfaultfd_ctx *ctx = file->private_data;
1850 
1851 	if (cmd != UFFDIO_API && ctx->state == UFFD_STATE_WAIT_API)
1852 		return -EINVAL;
1853 
1854 	switch(cmd) {
1855 	case UFFDIO_API:
1856 		ret = userfaultfd_api(ctx, arg);
1857 		break;
1858 	case UFFDIO_REGISTER:
1859 		ret = userfaultfd_register(ctx, arg);
1860 		break;
1861 	case UFFDIO_UNREGISTER:
1862 		ret = userfaultfd_unregister(ctx, arg);
1863 		break;
1864 	case UFFDIO_WAKE:
1865 		ret = userfaultfd_wake(ctx, arg);
1866 		break;
1867 	case UFFDIO_COPY:
1868 		ret = userfaultfd_copy(ctx, arg);
1869 		break;
1870 	case UFFDIO_ZEROPAGE:
1871 		ret = userfaultfd_zeropage(ctx, arg);
1872 		break;
1873 	}
1874 	return ret;
1875 }
1876 
1877 #ifdef CONFIG_PROC_FS
1878 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
1879 {
1880 	struct userfaultfd_ctx *ctx = f->private_data;
1881 	wait_queue_entry_t *wq;
1882 	unsigned long pending = 0, total = 0;
1883 
1884 	spin_lock(&ctx->fault_pending_wqh.lock);
1885 	list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
1886 		pending++;
1887 		total++;
1888 	}
1889 	list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
1890 		total++;
1891 	}
1892 	spin_unlock(&ctx->fault_pending_wqh.lock);
1893 
1894 	/*
1895 	 * If more protocols will be added, there will be all shown
1896 	 * separated by a space. Like this:
1897 	 *	protocols: aa:... bb:...
1898 	 */
1899 	seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
1900 		   pending, total, UFFD_API, ctx->features,
1901 		   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
1902 }
1903 #endif
1904 
1905 static const struct file_operations userfaultfd_fops = {
1906 #ifdef CONFIG_PROC_FS
1907 	.show_fdinfo	= userfaultfd_show_fdinfo,
1908 #endif
1909 	.release	= userfaultfd_release,
1910 	.poll		= userfaultfd_poll,
1911 	.read		= userfaultfd_read,
1912 	.unlocked_ioctl = userfaultfd_ioctl,
1913 	.compat_ioctl	= userfaultfd_ioctl,
1914 	.llseek		= noop_llseek,
1915 };
1916 
1917 static void init_once_userfaultfd_ctx(void *mem)
1918 {
1919 	struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
1920 
1921 	init_waitqueue_head(&ctx->fault_pending_wqh);
1922 	init_waitqueue_head(&ctx->fault_wqh);
1923 	init_waitqueue_head(&ctx->event_wqh);
1924 	init_waitqueue_head(&ctx->fd_wqh);
1925 	seqcount_init(&ctx->refile_seq);
1926 }
1927 
1928 SYSCALL_DEFINE1(userfaultfd, int, flags)
1929 {
1930 	struct userfaultfd_ctx *ctx;
1931 	int fd;
1932 
1933 	BUG_ON(!current->mm);
1934 
1935 	/* Check the UFFD_* constants for consistency.  */
1936 	BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
1937 	BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
1938 
1939 	if (flags & ~UFFD_SHARED_FCNTL_FLAGS)
1940 		return -EINVAL;
1941 
1942 	ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
1943 	if (!ctx)
1944 		return -ENOMEM;
1945 
1946 	refcount_set(&ctx->refcount, 1);
1947 	ctx->flags = flags;
1948 	ctx->features = 0;
1949 	ctx->state = UFFD_STATE_WAIT_API;
1950 	ctx->released = false;
1951 	ctx->mmap_changing = false;
1952 	ctx->mm = current->mm;
1953 	/* prevent the mm struct to be freed */
1954 	mmgrab(ctx->mm);
1955 
1956 	fd = anon_inode_getfd("[userfaultfd]", &userfaultfd_fops, ctx,
1957 			      O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS));
1958 	if (fd < 0) {
1959 		mmdrop(ctx->mm);
1960 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
1961 	}
1962 	return fd;
1963 }
1964 
1965 static int __init userfaultfd_init(void)
1966 {
1967 	userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
1968 						sizeof(struct userfaultfd_ctx),
1969 						0,
1970 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
1971 						init_once_userfaultfd_ctx);
1972 	return 0;
1973 }
1974 __initcall(userfaultfd_init);
1975