xref: /openbmc/linux/fs/userfaultfd.c (revision cd94cac4)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  fs/userfaultfd.c
4  *
5  *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
6  *  Copyright (C) 2008-2009 Red Hat, Inc.
7  *  Copyright (C) 2015  Red Hat, Inc.
8  *
9  *  Some part derived from fs/eventfd.c (anon inode setup) and
10  *  mm/ksm.c (mm hashing).
11  */
12 
13 #include <linux/list.h>
14 #include <linux/hashtable.h>
15 #include <linux/sched/signal.h>
16 #include <linux/sched/mm.h>
17 #include <linux/mm.h>
18 #include <linux/mm_inline.h>
19 #include <linux/mmu_notifier.h>
20 #include <linux/poll.h>
21 #include <linux/slab.h>
22 #include <linux/seq_file.h>
23 #include <linux/file.h>
24 #include <linux/bug.h>
25 #include <linux/anon_inodes.h>
26 #include <linux/syscalls.h>
27 #include <linux/userfaultfd_k.h>
28 #include <linux/mempolicy.h>
29 #include <linux/ioctl.h>
30 #include <linux/security.h>
31 #include <linux/hugetlb.h>
32 #include <linux/swapops.h>
33 #include <linux/miscdevice.h>
34 
35 static int sysctl_unprivileged_userfaultfd __read_mostly;
36 
37 #ifdef CONFIG_SYSCTL
38 static struct ctl_table vm_userfaultfd_table[] = {
39 	{
40 		.procname	= "unprivileged_userfaultfd",
41 		.data		= &sysctl_unprivileged_userfaultfd,
42 		.maxlen		= sizeof(sysctl_unprivileged_userfaultfd),
43 		.mode		= 0644,
44 		.proc_handler	= proc_dointvec_minmax,
45 		.extra1		= SYSCTL_ZERO,
46 		.extra2		= SYSCTL_ONE,
47 	},
48 	{ }
49 };
50 #endif
51 
52 static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly;
53 
54 /*
55  * Start with fault_pending_wqh and fault_wqh so they're more likely
56  * to be in the same cacheline.
57  *
58  * Locking order:
59  *	fd_wqh.lock
60  *		fault_pending_wqh.lock
61  *			fault_wqh.lock
62  *		event_wqh.lock
63  *
64  * To avoid deadlocks, IRQs must be disabled when taking any of the above locks,
65  * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's
66  * also taken in IRQ context.
67  */
68 struct userfaultfd_ctx {
69 	/* waitqueue head for the pending (i.e. not read) userfaults */
70 	wait_queue_head_t fault_pending_wqh;
71 	/* waitqueue head for the userfaults */
72 	wait_queue_head_t fault_wqh;
73 	/* waitqueue head for the pseudo fd to wakeup poll/read */
74 	wait_queue_head_t fd_wqh;
75 	/* waitqueue head for events */
76 	wait_queue_head_t event_wqh;
77 	/* a refile sequence protected by fault_pending_wqh lock */
78 	seqcount_spinlock_t refile_seq;
79 	/* pseudo fd refcounting */
80 	refcount_t refcount;
81 	/* userfaultfd syscall flags */
82 	unsigned int flags;
83 	/* features requested from the userspace */
84 	unsigned int features;
85 	/* released */
86 	bool released;
87 	/* memory mappings are changing because of non-cooperative event */
88 	atomic_t mmap_changing;
89 	/* mm with one ore more vmas attached to this userfaultfd_ctx */
90 	struct mm_struct *mm;
91 };
92 
93 struct userfaultfd_fork_ctx {
94 	struct userfaultfd_ctx *orig;
95 	struct userfaultfd_ctx *new;
96 	struct list_head list;
97 };
98 
99 struct userfaultfd_unmap_ctx {
100 	struct userfaultfd_ctx *ctx;
101 	unsigned long start;
102 	unsigned long end;
103 	struct list_head list;
104 };
105 
106 struct userfaultfd_wait_queue {
107 	struct uffd_msg msg;
108 	wait_queue_entry_t wq;
109 	struct userfaultfd_ctx *ctx;
110 	bool waken;
111 };
112 
113 struct userfaultfd_wake_range {
114 	unsigned long start;
115 	unsigned long len;
116 };
117 
118 /* internal indication that UFFD_API ioctl was successfully executed */
119 #define UFFD_FEATURE_INITIALIZED		(1u << 31)
120 
userfaultfd_is_initialized(struct userfaultfd_ctx * ctx)121 static bool userfaultfd_is_initialized(struct userfaultfd_ctx *ctx)
122 {
123 	return ctx->features & UFFD_FEATURE_INITIALIZED;
124 }
125 
126 /*
127  * Whether WP_UNPOPULATED is enabled on the uffd context.  It is only
128  * meaningful when userfaultfd_wp()==true on the vma and when it's
129  * anonymous.
130  */
userfaultfd_wp_unpopulated(struct vm_area_struct * vma)131 bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma)
132 {
133 	struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
134 
135 	if (!ctx)
136 		return false;
137 
138 	return ctx->features & UFFD_FEATURE_WP_UNPOPULATED;
139 }
140 
userfaultfd_set_vm_flags(struct vm_area_struct * vma,vm_flags_t flags)141 static void userfaultfd_set_vm_flags(struct vm_area_struct *vma,
142 				     vm_flags_t flags)
143 {
144 	const bool uffd_wp_changed = (vma->vm_flags ^ flags) & VM_UFFD_WP;
145 
146 	vm_flags_reset(vma, flags);
147 	/*
148 	 * For shared mappings, we want to enable writenotify while
149 	 * userfaultfd-wp is enabled (see vma_wants_writenotify()). We'll simply
150 	 * recalculate vma->vm_page_prot whenever userfaultfd-wp changes.
151 	 */
152 	if ((vma->vm_flags & VM_SHARED) && uffd_wp_changed)
153 		vma_set_page_prot(vma);
154 }
155 
userfaultfd_wake_function(wait_queue_entry_t * wq,unsigned mode,int wake_flags,void * key)156 static int userfaultfd_wake_function(wait_queue_entry_t *wq, unsigned mode,
157 				     int wake_flags, void *key)
158 {
159 	struct userfaultfd_wake_range *range = key;
160 	int ret;
161 	struct userfaultfd_wait_queue *uwq;
162 	unsigned long start, len;
163 
164 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
165 	ret = 0;
166 	/* len == 0 means wake all */
167 	start = range->start;
168 	len = range->len;
169 	if (len && (start > uwq->msg.arg.pagefault.address ||
170 		    start + len <= uwq->msg.arg.pagefault.address))
171 		goto out;
172 	WRITE_ONCE(uwq->waken, true);
173 	/*
174 	 * The Program-Order guarantees provided by the scheduler
175 	 * ensure uwq->waken is visible before the task is woken.
176 	 */
177 	ret = wake_up_state(wq->private, mode);
178 	if (ret) {
179 		/*
180 		 * Wake only once, autoremove behavior.
181 		 *
182 		 * After the effect of list_del_init is visible to the other
183 		 * CPUs, the waitqueue may disappear from under us, see the
184 		 * !list_empty_careful() in handle_userfault().
185 		 *
186 		 * try_to_wake_up() has an implicit smp_mb(), and the
187 		 * wq->private is read before calling the extern function
188 		 * "wake_up_state" (which in turns calls try_to_wake_up).
189 		 */
190 		list_del_init(&wq->entry);
191 	}
192 out:
193 	return ret;
194 }
195 
196 /**
197  * userfaultfd_ctx_get - Acquires a reference to the internal userfaultfd
198  * context.
199  * @ctx: [in] Pointer to the userfaultfd context.
200  */
userfaultfd_ctx_get(struct userfaultfd_ctx * ctx)201 static void userfaultfd_ctx_get(struct userfaultfd_ctx *ctx)
202 {
203 	refcount_inc(&ctx->refcount);
204 }
205 
206 /**
207  * userfaultfd_ctx_put - Releases a reference to the internal userfaultfd
208  * context.
209  * @ctx: [in] Pointer to userfaultfd context.
210  *
211  * The userfaultfd context reference must have been previously acquired either
212  * with userfaultfd_ctx_get() or userfaultfd_ctx_fdget().
213  */
userfaultfd_ctx_put(struct userfaultfd_ctx * ctx)214 static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx)
215 {
216 	if (refcount_dec_and_test(&ctx->refcount)) {
217 		VM_BUG_ON(spin_is_locked(&ctx->fault_pending_wqh.lock));
218 		VM_BUG_ON(waitqueue_active(&ctx->fault_pending_wqh));
219 		VM_BUG_ON(spin_is_locked(&ctx->fault_wqh.lock));
220 		VM_BUG_ON(waitqueue_active(&ctx->fault_wqh));
221 		VM_BUG_ON(spin_is_locked(&ctx->event_wqh.lock));
222 		VM_BUG_ON(waitqueue_active(&ctx->event_wqh));
223 		VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock));
224 		VM_BUG_ON(waitqueue_active(&ctx->fd_wqh));
225 		mmdrop(ctx->mm);
226 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
227 	}
228 }
229 
msg_init(struct uffd_msg * msg)230 static inline void msg_init(struct uffd_msg *msg)
231 {
232 	BUILD_BUG_ON(sizeof(struct uffd_msg) != 32);
233 	/*
234 	 * Must use memset to zero out the paddings or kernel data is
235 	 * leaked to userland.
236 	 */
237 	memset(msg, 0, sizeof(struct uffd_msg));
238 }
239 
userfault_msg(unsigned long address,unsigned long real_address,unsigned int flags,unsigned long reason,unsigned int features)240 static inline struct uffd_msg userfault_msg(unsigned long address,
241 					    unsigned long real_address,
242 					    unsigned int flags,
243 					    unsigned long reason,
244 					    unsigned int features)
245 {
246 	struct uffd_msg msg;
247 
248 	msg_init(&msg);
249 	msg.event = UFFD_EVENT_PAGEFAULT;
250 
251 	msg.arg.pagefault.address = (features & UFFD_FEATURE_EXACT_ADDRESS) ?
252 				    real_address : address;
253 
254 	/*
255 	 * These flags indicate why the userfault occurred:
256 	 * - UFFD_PAGEFAULT_FLAG_WP indicates a write protect fault.
257 	 * - UFFD_PAGEFAULT_FLAG_MINOR indicates a minor fault.
258 	 * - Neither of these flags being set indicates a MISSING fault.
259 	 *
260 	 * Separately, UFFD_PAGEFAULT_FLAG_WRITE indicates it was a write
261 	 * fault. Otherwise, it was a read fault.
262 	 */
263 	if (flags & FAULT_FLAG_WRITE)
264 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WRITE;
265 	if (reason & VM_UFFD_WP)
266 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_WP;
267 	if (reason & VM_UFFD_MINOR)
268 		msg.arg.pagefault.flags |= UFFD_PAGEFAULT_FLAG_MINOR;
269 	if (features & UFFD_FEATURE_THREAD_ID)
270 		msg.arg.pagefault.feat.ptid = task_pid_vnr(current);
271 	return msg;
272 }
273 
274 #ifdef CONFIG_HUGETLB_PAGE
275 /*
276  * Same functionality as userfaultfd_must_wait below with modifications for
277  * hugepmd ranges.
278  */
userfaultfd_huge_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)279 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
280 					      struct vm_fault *vmf,
281 					      unsigned long reason)
282 {
283 	struct vm_area_struct *vma = vmf->vma;
284 	pte_t *ptep, pte;
285 	bool ret = true;
286 
287 	assert_fault_locked(vmf);
288 
289 	ptep = hugetlb_walk(vma, vmf->address, vma_mmu_pagesize(vma));
290 	if (!ptep)
291 		goto out;
292 
293 	ret = false;
294 	pte = huge_ptep_get(ptep);
295 
296 	/*
297 	 * Lockless access: we're in a wait_event so it's ok if it
298 	 * changes under us.  PTE markers should be handled the same as none
299 	 * ptes here.
300 	 */
301 	if (huge_pte_none_mostly(pte))
302 		ret = true;
303 	if (!huge_pte_write(pte) && (reason & VM_UFFD_WP))
304 		ret = true;
305 out:
306 	return ret;
307 }
308 #else
userfaultfd_huge_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)309 static inline bool userfaultfd_huge_must_wait(struct userfaultfd_ctx *ctx,
310 					      struct vm_fault *vmf,
311 					      unsigned long reason)
312 {
313 	return false;	/* should never get here */
314 }
315 #endif /* CONFIG_HUGETLB_PAGE */
316 
317 /*
318  * Verify the pagetables are still not ok after having reigstered into
319  * the fault_pending_wqh to avoid userland having to UFFDIO_WAKE any
320  * userfault that has already been resolved, if userfaultfd_read and
321  * UFFDIO_COPY|ZEROPAGE are being run simultaneously on two different
322  * threads.
323  */
userfaultfd_must_wait(struct userfaultfd_ctx * ctx,struct vm_fault * vmf,unsigned long reason)324 static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
325 					 struct vm_fault *vmf,
326 					 unsigned long reason)
327 {
328 	struct mm_struct *mm = ctx->mm;
329 	unsigned long address = vmf->address;
330 	pgd_t *pgd;
331 	p4d_t *p4d;
332 	pud_t *pud;
333 	pmd_t *pmd, _pmd;
334 	pte_t *pte;
335 	pte_t ptent;
336 	bool ret = true;
337 
338 	assert_fault_locked(vmf);
339 
340 	pgd = pgd_offset(mm, address);
341 	if (!pgd_present(*pgd))
342 		goto out;
343 	p4d = p4d_offset(pgd, address);
344 	if (!p4d_present(*p4d))
345 		goto out;
346 	pud = pud_offset(p4d, address);
347 	if (!pud_present(*pud))
348 		goto out;
349 	pmd = pmd_offset(pud, address);
350 again:
351 	_pmd = pmdp_get_lockless(pmd);
352 	if (pmd_none(_pmd))
353 		goto out;
354 
355 	ret = false;
356 	if (!pmd_present(_pmd) || pmd_devmap(_pmd))
357 		goto out;
358 
359 	if (pmd_trans_huge(_pmd)) {
360 		if (!pmd_write(_pmd) && (reason & VM_UFFD_WP))
361 			ret = true;
362 		goto out;
363 	}
364 
365 	pte = pte_offset_map(pmd, address);
366 	if (!pte) {
367 		ret = true;
368 		goto again;
369 	}
370 	/*
371 	 * Lockless access: we're in a wait_event so it's ok if it
372 	 * changes under us.  PTE markers should be handled the same as none
373 	 * ptes here.
374 	 */
375 	ptent = ptep_get(pte);
376 	if (pte_none_mostly(ptent))
377 		ret = true;
378 	if (!pte_write(ptent) && (reason & VM_UFFD_WP))
379 		ret = true;
380 	pte_unmap(pte);
381 
382 out:
383 	return ret;
384 }
385 
userfaultfd_get_blocking_state(unsigned int flags)386 static inline unsigned int userfaultfd_get_blocking_state(unsigned int flags)
387 {
388 	if (flags & FAULT_FLAG_INTERRUPTIBLE)
389 		return TASK_INTERRUPTIBLE;
390 
391 	if (flags & FAULT_FLAG_KILLABLE)
392 		return TASK_KILLABLE;
393 
394 	return TASK_UNINTERRUPTIBLE;
395 }
396 
397 /*
398  * The locking rules involved in returning VM_FAULT_RETRY depending on
399  * FAULT_FLAG_ALLOW_RETRY, FAULT_FLAG_RETRY_NOWAIT and
400  * FAULT_FLAG_KILLABLE are not straightforward. The "Caution"
401  * recommendation in __lock_page_or_retry is not an understatement.
402  *
403  * If FAULT_FLAG_ALLOW_RETRY is set, the mmap_lock must be released
404  * before returning VM_FAULT_RETRY only if FAULT_FLAG_RETRY_NOWAIT is
405  * not set.
406  *
407  * If FAULT_FLAG_ALLOW_RETRY is set but FAULT_FLAG_KILLABLE is not
408  * set, VM_FAULT_RETRY can still be returned if and only if there are
409  * fatal_signal_pending()s, and the mmap_lock must be released before
410  * returning it.
411  */
handle_userfault(struct vm_fault * vmf,unsigned long reason)412 vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)
413 {
414 	struct vm_area_struct *vma = vmf->vma;
415 	struct mm_struct *mm = vma->vm_mm;
416 	struct userfaultfd_ctx *ctx;
417 	struct userfaultfd_wait_queue uwq;
418 	vm_fault_t ret = VM_FAULT_SIGBUS;
419 	bool must_wait;
420 	unsigned int blocking_state;
421 
422 	/*
423 	 * We don't do userfault handling for the final child pid update.
424 	 *
425 	 * We also don't do userfault handling during
426 	 * coredumping. hugetlbfs has the special
427 	 * hugetlb_follow_page_mask() to skip missing pages in the
428 	 * FOLL_DUMP case, anon memory also checks for FOLL_DUMP with
429 	 * the no_page_table() helper in follow_page_mask(), but the
430 	 * shmem_vm_ops->fault method is invoked even during
431 	 * coredumping and it ends up here.
432 	 */
433 	if (current->flags & (PF_EXITING|PF_DUMPCORE))
434 		goto out;
435 
436 	assert_fault_locked(vmf);
437 
438 	ctx = vma->vm_userfaultfd_ctx.ctx;
439 	if (!ctx)
440 		goto out;
441 
442 	BUG_ON(ctx->mm != mm);
443 
444 	/* Any unrecognized flag is a bug. */
445 	VM_BUG_ON(reason & ~__VM_UFFD_FLAGS);
446 	/* 0 or > 1 flags set is a bug; we expect exactly 1. */
447 	VM_BUG_ON(!reason || (reason & (reason - 1)));
448 
449 	if (ctx->features & UFFD_FEATURE_SIGBUS)
450 		goto out;
451 	if (!(vmf->flags & FAULT_FLAG_USER) && (ctx->flags & UFFD_USER_MODE_ONLY))
452 		goto out;
453 
454 	/*
455 	 * If it's already released don't get it. This avoids to loop
456 	 * in __get_user_pages if userfaultfd_release waits on the
457 	 * caller of handle_userfault to release the mmap_lock.
458 	 */
459 	if (unlikely(READ_ONCE(ctx->released))) {
460 		/*
461 		 * Don't return VM_FAULT_SIGBUS in this case, so a non
462 		 * cooperative manager can close the uffd after the
463 		 * last UFFDIO_COPY, without risking to trigger an
464 		 * involuntary SIGBUS if the process was starting the
465 		 * userfaultfd while the userfaultfd was still armed
466 		 * (but after the last UFFDIO_COPY). If the uffd
467 		 * wasn't already closed when the userfault reached
468 		 * this point, that would normally be solved by
469 		 * userfaultfd_must_wait returning 'false'.
470 		 *
471 		 * If we were to return VM_FAULT_SIGBUS here, the non
472 		 * cooperative manager would be instead forced to
473 		 * always call UFFDIO_UNREGISTER before it can safely
474 		 * close the uffd.
475 		 */
476 		ret = VM_FAULT_NOPAGE;
477 		goto out;
478 	}
479 
480 	/*
481 	 * Check that we can return VM_FAULT_RETRY.
482 	 *
483 	 * NOTE: it should become possible to return VM_FAULT_RETRY
484 	 * even if FAULT_FLAG_TRIED is set without leading to gup()
485 	 * -EBUSY failures, if the userfaultfd is to be extended for
486 	 * VM_UFFD_WP tracking and we intend to arm the userfault
487 	 * without first stopping userland access to the memory. For
488 	 * VM_UFFD_MISSING userfaults this is enough for now.
489 	 */
490 	if (unlikely(!(vmf->flags & FAULT_FLAG_ALLOW_RETRY))) {
491 		/*
492 		 * Validate the invariant that nowait must allow retry
493 		 * to be sure not to return SIGBUS erroneously on
494 		 * nowait invocations.
495 		 */
496 		BUG_ON(vmf->flags & FAULT_FLAG_RETRY_NOWAIT);
497 #ifdef CONFIG_DEBUG_VM
498 		if (printk_ratelimit()) {
499 			printk(KERN_WARNING
500 			       "FAULT_FLAG_ALLOW_RETRY missing %x\n",
501 			       vmf->flags);
502 			dump_stack();
503 		}
504 #endif
505 		goto out;
506 	}
507 
508 	/*
509 	 * Handle nowait, not much to do other than tell it to retry
510 	 * and wait.
511 	 */
512 	ret = VM_FAULT_RETRY;
513 	if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
514 		goto out;
515 
516 	/* take the reference before dropping the mmap_lock */
517 	userfaultfd_ctx_get(ctx);
518 
519 	init_waitqueue_func_entry(&uwq.wq, userfaultfd_wake_function);
520 	uwq.wq.private = current;
521 	uwq.msg = userfault_msg(vmf->address, vmf->real_address, vmf->flags,
522 				reason, ctx->features);
523 	uwq.ctx = ctx;
524 	uwq.waken = false;
525 
526 	blocking_state = userfaultfd_get_blocking_state(vmf->flags);
527 
528         /*
529          * Take the vma lock now, in order to safely call
530          * userfaultfd_huge_must_wait() later. Since acquiring the
531          * (sleepable) vma lock can modify the current task state, that
532          * must be before explicitly calling set_current_state().
533          */
534 	if (is_vm_hugetlb_page(vma))
535 		hugetlb_vma_lock_read(vma);
536 
537 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
538 	/*
539 	 * After the __add_wait_queue the uwq is visible to userland
540 	 * through poll/read().
541 	 */
542 	__add_wait_queue(&ctx->fault_pending_wqh, &uwq.wq);
543 	/*
544 	 * The smp_mb() after __set_current_state prevents the reads
545 	 * following the spin_unlock to happen before the list_add in
546 	 * __add_wait_queue.
547 	 */
548 	set_current_state(blocking_state);
549 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
550 
551 	if (!is_vm_hugetlb_page(vma))
552 		must_wait = userfaultfd_must_wait(ctx, vmf, reason);
553 	else
554 		must_wait = userfaultfd_huge_must_wait(ctx, vmf, reason);
555 	if (is_vm_hugetlb_page(vma))
556 		hugetlb_vma_unlock_read(vma);
557 	release_fault_lock(vmf);
558 
559 	if (likely(must_wait && !READ_ONCE(ctx->released))) {
560 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
561 		schedule();
562 	}
563 
564 	__set_current_state(TASK_RUNNING);
565 
566 	/*
567 	 * Here we race with the list_del; list_add in
568 	 * userfaultfd_ctx_read(), however because we don't ever run
569 	 * list_del_init() to refile across the two lists, the prev
570 	 * and next pointers will never point to self. list_add also
571 	 * would never let any of the two pointers to point to
572 	 * self. So list_empty_careful won't risk to see both pointers
573 	 * pointing to self at any time during the list refile. The
574 	 * only case where list_del_init() is called is the full
575 	 * removal in the wake function and there we don't re-list_add
576 	 * and it's fine not to block on the spinlock. The uwq on this
577 	 * kernel stack can be released after the list_del_init.
578 	 */
579 	if (!list_empty_careful(&uwq.wq.entry)) {
580 		spin_lock_irq(&ctx->fault_pending_wqh.lock);
581 		/*
582 		 * No need of list_del_init(), the uwq on the stack
583 		 * will be freed shortly anyway.
584 		 */
585 		list_del(&uwq.wq.entry);
586 		spin_unlock_irq(&ctx->fault_pending_wqh.lock);
587 	}
588 
589 	/*
590 	 * ctx may go away after this if the userfault pseudo fd is
591 	 * already released.
592 	 */
593 	userfaultfd_ctx_put(ctx);
594 
595 out:
596 	return ret;
597 }
598 
userfaultfd_event_wait_completion(struct userfaultfd_ctx * ctx,struct userfaultfd_wait_queue * ewq)599 static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
600 					      struct userfaultfd_wait_queue *ewq)
601 {
602 	struct userfaultfd_ctx *release_new_ctx;
603 
604 	if (WARN_ON_ONCE(current->flags & PF_EXITING))
605 		goto out;
606 
607 	ewq->ctx = ctx;
608 	init_waitqueue_entry(&ewq->wq, current);
609 	release_new_ctx = NULL;
610 
611 	spin_lock_irq(&ctx->event_wqh.lock);
612 	/*
613 	 * After the __add_wait_queue the uwq is visible to userland
614 	 * through poll/read().
615 	 */
616 	__add_wait_queue(&ctx->event_wqh, &ewq->wq);
617 	for (;;) {
618 		set_current_state(TASK_KILLABLE);
619 		if (ewq->msg.event == 0)
620 			break;
621 		if (READ_ONCE(ctx->released) ||
622 		    fatal_signal_pending(current)) {
623 			/*
624 			 * &ewq->wq may be queued in fork_event, but
625 			 * __remove_wait_queue ignores the head
626 			 * parameter. It would be a problem if it
627 			 * didn't.
628 			 */
629 			__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
630 			if (ewq->msg.event == UFFD_EVENT_FORK) {
631 				struct userfaultfd_ctx *new;
632 
633 				new = (struct userfaultfd_ctx *)
634 					(unsigned long)
635 					ewq->msg.arg.reserved.reserved1;
636 				release_new_ctx = new;
637 			}
638 			break;
639 		}
640 
641 		spin_unlock_irq(&ctx->event_wqh.lock);
642 
643 		wake_up_poll(&ctx->fd_wqh, EPOLLIN);
644 		schedule();
645 
646 		spin_lock_irq(&ctx->event_wqh.lock);
647 	}
648 	__set_current_state(TASK_RUNNING);
649 	spin_unlock_irq(&ctx->event_wqh.lock);
650 
651 	if (release_new_ctx) {
652 		struct vm_area_struct *vma;
653 		struct mm_struct *mm = release_new_ctx->mm;
654 		VMA_ITERATOR(vmi, mm, 0);
655 
656 		/* the various vma->vm_userfaultfd_ctx still points to it */
657 		mmap_write_lock(mm);
658 		for_each_vma(vmi, vma) {
659 			if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
660 				vma_start_write(vma);
661 				vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
662 				userfaultfd_set_vm_flags(vma,
663 							 vma->vm_flags & ~__VM_UFFD_FLAGS);
664 			}
665 		}
666 		mmap_write_unlock(mm);
667 
668 		userfaultfd_ctx_put(release_new_ctx);
669 	}
670 
671 	/*
672 	 * ctx may go away after this if the userfault pseudo fd is
673 	 * already released.
674 	 */
675 out:
676 	atomic_dec(&ctx->mmap_changing);
677 	VM_BUG_ON(atomic_read(&ctx->mmap_changing) < 0);
678 	userfaultfd_ctx_put(ctx);
679 }
680 
userfaultfd_event_complete(struct userfaultfd_ctx * ctx,struct userfaultfd_wait_queue * ewq)681 static void userfaultfd_event_complete(struct userfaultfd_ctx *ctx,
682 				       struct userfaultfd_wait_queue *ewq)
683 {
684 	ewq->msg.event = 0;
685 	wake_up_locked(&ctx->event_wqh);
686 	__remove_wait_queue(&ctx->event_wqh, &ewq->wq);
687 }
688 
dup_userfaultfd(struct vm_area_struct * vma,struct list_head * fcs)689 int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
690 {
691 	struct userfaultfd_ctx *ctx = NULL, *octx;
692 	struct userfaultfd_fork_ctx *fctx;
693 
694 	octx = vma->vm_userfaultfd_ctx.ctx;
695 	if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
696 		vma_start_write(vma);
697 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
698 		userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS);
699 		return 0;
700 	}
701 
702 	list_for_each_entry(fctx, fcs, list)
703 		if (fctx->orig == octx) {
704 			ctx = fctx->new;
705 			break;
706 		}
707 
708 	if (!ctx) {
709 		fctx = kmalloc(sizeof(*fctx), GFP_KERNEL);
710 		if (!fctx)
711 			return -ENOMEM;
712 
713 		ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
714 		if (!ctx) {
715 			kfree(fctx);
716 			return -ENOMEM;
717 		}
718 
719 		refcount_set(&ctx->refcount, 1);
720 		ctx->flags = octx->flags;
721 		ctx->features = octx->features;
722 		ctx->released = false;
723 		atomic_set(&ctx->mmap_changing, 0);
724 		ctx->mm = vma->vm_mm;
725 		mmgrab(ctx->mm);
726 
727 		userfaultfd_ctx_get(octx);
728 		atomic_inc(&octx->mmap_changing);
729 		fctx->orig = octx;
730 		fctx->new = ctx;
731 		list_add_tail(&fctx->list, fcs);
732 	}
733 
734 	vma->vm_userfaultfd_ctx.ctx = ctx;
735 	return 0;
736 }
737 
dup_fctx(struct userfaultfd_fork_ctx * fctx)738 static void dup_fctx(struct userfaultfd_fork_ctx *fctx)
739 {
740 	struct userfaultfd_ctx *ctx = fctx->orig;
741 	struct userfaultfd_wait_queue ewq;
742 
743 	msg_init(&ewq.msg);
744 
745 	ewq.msg.event = UFFD_EVENT_FORK;
746 	ewq.msg.arg.reserved.reserved1 = (unsigned long)fctx->new;
747 
748 	userfaultfd_event_wait_completion(ctx, &ewq);
749 }
750 
dup_userfaultfd_complete(struct list_head * fcs)751 void dup_userfaultfd_complete(struct list_head *fcs)
752 {
753 	struct userfaultfd_fork_ctx *fctx, *n;
754 
755 	list_for_each_entry_safe(fctx, n, fcs, list) {
756 		dup_fctx(fctx);
757 		list_del(&fctx->list);
758 		kfree(fctx);
759 	}
760 }
761 
mremap_userfaultfd_prep(struct vm_area_struct * vma,struct vm_userfaultfd_ctx * vm_ctx)762 void mremap_userfaultfd_prep(struct vm_area_struct *vma,
763 			     struct vm_userfaultfd_ctx *vm_ctx)
764 {
765 	struct userfaultfd_ctx *ctx;
766 
767 	ctx = vma->vm_userfaultfd_ctx.ctx;
768 
769 	if (!ctx)
770 		return;
771 
772 	if (ctx->features & UFFD_FEATURE_EVENT_REMAP) {
773 		vm_ctx->ctx = ctx;
774 		userfaultfd_ctx_get(ctx);
775 		atomic_inc(&ctx->mmap_changing);
776 	} else {
777 		/* Drop uffd context if remap feature not enabled */
778 		vma_start_write(vma);
779 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
780 		userfaultfd_set_vm_flags(vma, vma->vm_flags & ~__VM_UFFD_FLAGS);
781 	}
782 }
783 
mremap_userfaultfd_complete(struct vm_userfaultfd_ctx * vm_ctx,unsigned long from,unsigned long to,unsigned long len)784 void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *vm_ctx,
785 				 unsigned long from, unsigned long to,
786 				 unsigned long len)
787 {
788 	struct userfaultfd_ctx *ctx = vm_ctx->ctx;
789 	struct userfaultfd_wait_queue ewq;
790 
791 	if (!ctx)
792 		return;
793 
794 	if (to & ~PAGE_MASK) {
795 		userfaultfd_ctx_put(ctx);
796 		return;
797 	}
798 
799 	msg_init(&ewq.msg);
800 
801 	ewq.msg.event = UFFD_EVENT_REMAP;
802 	ewq.msg.arg.remap.from = from;
803 	ewq.msg.arg.remap.to = to;
804 	ewq.msg.arg.remap.len = len;
805 
806 	userfaultfd_event_wait_completion(ctx, &ewq);
807 }
808 
userfaultfd_remove(struct vm_area_struct * vma,unsigned long start,unsigned long end)809 bool userfaultfd_remove(struct vm_area_struct *vma,
810 			unsigned long start, unsigned long end)
811 {
812 	struct mm_struct *mm = vma->vm_mm;
813 	struct userfaultfd_ctx *ctx;
814 	struct userfaultfd_wait_queue ewq;
815 
816 	ctx = vma->vm_userfaultfd_ctx.ctx;
817 	if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_REMOVE))
818 		return true;
819 
820 	userfaultfd_ctx_get(ctx);
821 	atomic_inc(&ctx->mmap_changing);
822 	mmap_read_unlock(mm);
823 
824 	msg_init(&ewq.msg);
825 
826 	ewq.msg.event = UFFD_EVENT_REMOVE;
827 	ewq.msg.arg.remove.start = start;
828 	ewq.msg.arg.remove.end = end;
829 
830 	userfaultfd_event_wait_completion(ctx, &ewq);
831 
832 	return false;
833 }
834 
has_unmap_ctx(struct userfaultfd_ctx * ctx,struct list_head * unmaps,unsigned long start,unsigned long end)835 static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
836 			  unsigned long start, unsigned long end)
837 {
838 	struct userfaultfd_unmap_ctx *unmap_ctx;
839 
840 	list_for_each_entry(unmap_ctx, unmaps, list)
841 		if (unmap_ctx->ctx == ctx && unmap_ctx->start == start &&
842 		    unmap_ctx->end == end)
843 			return true;
844 
845 	return false;
846 }
847 
userfaultfd_unmap_prep(struct vm_area_struct * vma,unsigned long start,unsigned long end,struct list_head * unmaps)848 int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start,
849 			   unsigned long end, struct list_head *unmaps)
850 {
851 	struct userfaultfd_unmap_ctx *unmap_ctx;
852 	struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
853 
854 	if (!ctx || !(ctx->features & UFFD_FEATURE_EVENT_UNMAP) ||
855 	    has_unmap_ctx(ctx, unmaps, start, end))
856 		return 0;
857 
858 	unmap_ctx = kzalloc(sizeof(*unmap_ctx), GFP_KERNEL);
859 	if (!unmap_ctx)
860 		return -ENOMEM;
861 
862 	userfaultfd_ctx_get(ctx);
863 	atomic_inc(&ctx->mmap_changing);
864 	unmap_ctx->ctx = ctx;
865 	unmap_ctx->start = start;
866 	unmap_ctx->end = end;
867 	list_add_tail(&unmap_ctx->list, unmaps);
868 
869 	return 0;
870 }
871 
userfaultfd_unmap_complete(struct mm_struct * mm,struct list_head * uf)872 void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf)
873 {
874 	struct userfaultfd_unmap_ctx *ctx, *n;
875 	struct userfaultfd_wait_queue ewq;
876 
877 	list_for_each_entry_safe(ctx, n, uf, list) {
878 		msg_init(&ewq.msg);
879 
880 		ewq.msg.event = UFFD_EVENT_UNMAP;
881 		ewq.msg.arg.remove.start = ctx->start;
882 		ewq.msg.arg.remove.end = ctx->end;
883 
884 		userfaultfd_event_wait_completion(ctx->ctx, &ewq);
885 
886 		list_del(&ctx->list);
887 		kfree(ctx);
888 	}
889 }
890 
userfaultfd_release(struct inode * inode,struct file * file)891 static int userfaultfd_release(struct inode *inode, struct file *file)
892 {
893 	struct userfaultfd_ctx *ctx = file->private_data;
894 	struct mm_struct *mm = ctx->mm;
895 	struct vm_area_struct *vma, *prev;
896 	/* len == 0 means wake all */
897 	struct userfaultfd_wake_range range = { .len = 0, };
898 	unsigned long new_flags;
899 	VMA_ITERATOR(vmi, mm, 0);
900 
901 	WRITE_ONCE(ctx->released, true);
902 
903 	if (!mmget_not_zero(mm))
904 		goto wakeup;
905 
906 	/*
907 	 * Flush page faults out of all CPUs. NOTE: all page faults
908 	 * must be retried without returning VM_FAULT_SIGBUS if
909 	 * userfaultfd_ctx_get() succeeds but vma->vma_userfault_ctx
910 	 * changes while handle_userfault released the mmap_lock. So
911 	 * it's critical that released is set to true (above), before
912 	 * taking the mmap_lock for writing.
913 	 */
914 	mmap_write_lock(mm);
915 	prev = NULL;
916 	for_each_vma(vmi, vma) {
917 		cond_resched();
918 		BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
919 		       !!(vma->vm_flags & __VM_UFFD_FLAGS));
920 		if (vma->vm_userfaultfd_ctx.ctx != ctx) {
921 			prev = vma;
922 			continue;
923 		}
924 		/* Reset ptes for the whole vma range if wr-protected */
925 		if (userfaultfd_wp(vma))
926 			uffd_wp_range(vma, vma->vm_start,
927 				      vma->vm_end - vma->vm_start, false);
928 		new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
929 		prev = vma_merge(&vmi, mm, prev, vma->vm_start, vma->vm_end,
930 				 new_flags, vma->anon_vma,
931 				 vma->vm_file, vma->vm_pgoff,
932 				 vma_policy(vma),
933 				 NULL_VM_UFFD_CTX, anon_vma_name(vma));
934 		if (prev) {
935 			vma = prev;
936 		} else {
937 			prev = vma;
938 		}
939 
940 		vma_start_write(vma);
941 		userfaultfd_set_vm_flags(vma, new_flags);
942 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
943 	}
944 	mmap_write_unlock(mm);
945 	mmput(mm);
946 wakeup:
947 	/*
948 	 * After no new page faults can wait on this fault_*wqh, flush
949 	 * the last page faults that may have been already waiting on
950 	 * the fault_*wqh.
951 	 */
952 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
953 	__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL, &range);
954 	__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, &range);
955 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
956 
957 	/* Flush pending events that may still wait on event_wqh */
958 	wake_up_all(&ctx->event_wqh);
959 
960 	wake_up_poll(&ctx->fd_wqh, EPOLLHUP);
961 	userfaultfd_ctx_put(ctx);
962 	return 0;
963 }
964 
965 /* fault_pending_wqh.lock must be hold by the caller */
find_userfault_in(wait_queue_head_t * wqh)966 static inline struct userfaultfd_wait_queue *find_userfault_in(
967 		wait_queue_head_t *wqh)
968 {
969 	wait_queue_entry_t *wq;
970 	struct userfaultfd_wait_queue *uwq;
971 
972 	lockdep_assert_held(&wqh->lock);
973 
974 	uwq = NULL;
975 	if (!waitqueue_active(wqh))
976 		goto out;
977 	/* walk in reverse to provide FIFO behavior to read userfaults */
978 	wq = list_last_entry(&wqh->head, typeof(*wq), entry);
979 	uwq = container_of(wq, struct userfaultfd_wait_queue, wq);
980 out:
981 	return uwq;
982 }
983 
find_userfault(struct userfaultfd_ctx * ctx)984 static inline struct userfaultfd_wait_queue *find_userfault(
985 		struct userfaultfd_ctx *ctx)
986 {
987 	return find_userfault_in(&ctx->fault_pending_wqh);
988 }
989 
find_userfault_evt(struct userfaultfd_ctx * ctx)990 static inline struct userfaultfd_wait_queue *find_userfault_evt(
991 		struct userfaultfd_ctx *ctx)
992 {
993 	return find_userfault_in(&ctx->event_wqh);
994 }
995 
userfaultfd_poll(struct file * file,poll_table * wait)996 static __poll_t userfaultfd_poll(struct file *file, poll_table *wait)
997 {
998 	struct userfaultfd_ctx *ctx = file->private_data;
999 	__poll_t ret;
1000 
1001 	poll_wait(file, &ctx->fd_wqh, wait);
1002 
1003 	if (!userfaultfd_is_initialized(ctx))
1004 		return EPOLLERR;
1005 
1006 	/*
1007 	 * poll() never guarantees that read won't block.
1008 	 * userfaults can be waken before they're read().
1009 	 */
1010 	if (unlikely(!(file->f_flags & O_NONBLOCK)))
1011 		return EPOLLERR;
1012 	/*
1013 	 * lockless access to see if there are pending faults
1014 	 * __pollwait last action is the add_wait_queue but
1015 	 * the spin_unlock would allow the waitqueue_active to
1016 	 * pass above the actual list_add inside
1017 	 * add_wait_queue critical section. So use a full
1018 	 * memory barrier to serialize the list_add write of
1019 	 * add_wait_queue() with the waitqueue_active read
1020 	 * below.
1021 	 */
1022 	ret = 0;
1023 	smp_mb();
1024 	if (waitqueue_active(&ctx->fault_pending_wqh))
1025 		ret = EPOLLIN;
1026 	else if (waitqueue_active(&ctx->event_wqh))
1027 		ret = EPOLLIN;
1028 
1029 	return ret;
1030 }
1031 
1032 static const struct file_operations userfaultfd_fops;
1033 
resolve_userfault_fork(struct userfaultfd_ctx * new,struct inode * inode,struct uffd_msg * msg)1034 static int resolve_userfault_fork(struct userfaultfd_ctx *new,
1035 				  struct inode *inode,
1036 				  struct uffd_msg *msg)
1037 {
1038 	int fd;
1039 
1040 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
1041 			O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
1042 	if (fd < 0)
1043 		return fd;
1044 
1045 	msg->arg.reserved.reserved1 = 0;
1046 	msg->arg.fork.ufd = fd;
1047 	return 0;
1048 }
1049 
userfaultfd_ctx_read(struct userfaultfd_ctx * ctx,int no_wait,struct uffd_msg * msg,struct inode * inode)1050 static ssize_t userfaultfd_ctx_read(struct userfaultfd_ctx *ctx, int no_wait,
1051 				    struct uffd_msg *msg, struct inode *inode)
1052 {
1053 	ssize_t ret;
1054 	DECLARE_WAITQUEUE(wait, current);
1055 	struct userfaultfd_wait_queue *uwq;
1056 	/*
1057 	 * Handling fork event requires sleeping operations, so
1058 	 * we drop the event_wqh lock, then do these ops, then
1059 	 * lock it back and wake up the waiter. While the lock is
1060 	 * dropped the ewq may go away so we keep track of it
1061 	 * carefully.
1062 	 */
1063 	LIST_HEAD(fork_event);
1064 	struct userfaultfd_ctx *fork_nctx = NULL;
1065 
1066 	/* always take the fd_wqh lock before the fault_pending_wqh lock */
1067 	spin_lock_irq(&ctx->fd_wqh.lock);
1068 	__add_wait_queue(&ctx->fd_wqh, &wait);
1069 	for (;;) {
1070 		set_current_state(TASK_INTERRUPTIBLE);
1071 		spin_lock(&ctx->fault_pending_wqh.lock);
1072 		uwq = find_userfault(ctx);
1073 		if (uwq) {
1074 			/*
1075 			 * Use a seqcount to repeat the lockless check
1076 			 * in wake_userfault() to avoid missing
1077 			 * wakeups because during the refile both
1078 			 * waitqueue could become empty if this is the
1079 			 * only userfault.
1080 			 */
1081 			write_seqcount_begin(&ctx->refile_seq);
1082 
1083 			/*
1084 			 * The fault_pending_wqh.lock prevents the uwq
1085 			 * to disappear from under us.
1086 			 *
1087 			 * Refile this userfault from
1088 			 * fault_pending_wqh to fault_wqh, it's not
1089 			 * pending anymore after we read it.
1090 			 *
1091 			 * Use list_del() by hand (as
1092 			 * userfaultfd_wake_function also uses
1093 			 * list_del_init() by hand) to be sure nobody
1094 			 * changes __remove_wait_queue() to use
1095 			 * list_del_init() in turn breaking the
1096 			 * !list_empty_careful() check in
1097 			 * handle_userfault(). The uwq->wq.head list
1098 			 * must never be empty at any time during the
1099 			 * refile, or the waitqueue could disappear
1100 			 * from under us. The "wait_queue_head_t"
1101 			 * parameter of __remove_wait_queue() is unused
1102 			 * anyway.
1103 			 */
1104 			list_del(&uwq->wq.entry);
1105 			add_wait_queue(&ctx->fault_wqh, &uwq->wq);
1106 
1107 			write_seqcount_end(&ctx->refile_seq);
1108 
1109 			/* careful to always initialize msg if ret == 0 */
1110 			*msg = uwq->msg;
1111 			spin_unlock(&ctx->fault_pending_wqh.lock);
1112 			ret = 0;
1113 			break;
1114 		}
1115 		spin_unlock(&ctx->fault_pending_wqh.lock);
1116 
1117 		spin_lock(&ctx->event_wqh.lock);
1118 		uwq = find_userfault_evt(ctx);
1119 		if (uwq) {
1120 			*msg = uwq->msg;
1121 
1122 			if (uwq->msg.event == UFFD_EVENT_FORK) {
1123 				fork_nctx = (struct userfaultfd_ctx *)
1124 					(unsigned long)
1125 					uwq->msg.arg.reserved.reserved1;
1126 				list_move(&uwq->wq.entry, &fork_event);
1127 				/*
1128 				 * fork_nctx can be freed as soon as
1129 				 * we drop the lock, unless we take a
1130 				 * reference on it.
1131 				 */
1132 				userfaultfd_ctx_get(fork_nctx);
1133 				spin_unlock(&ctx->event_wqh.lock);
1134 				ret = 0;
1135 				break;
1136 			}
1137 
1138 			userfaultfd_event_complete(ctx, uwq);
1139 			spin_unlock(&ctx->event_wqh.lock);
1140 			ret = 0;
1141 			break;
1142 		}
1143 		spin_unlock(&ctx->event_wqh.lock);
1144 
1145 		if (signal_pending(current)) {
1146 			ret = -ERESTARTSYS;
1147 			break;
1148 		}
1149 		if (no_wait) {
1150 			ret = -EAGAIN;
1151 			break;
1152 		}
1153 		spin_unlock_irq(&ctx->fd_wqh.lock);
1154 		schedule();
1155 		spin_lock_irq(&ctx->fd_wqh.lock);
1156 	}
1157 	__remove_wait_queue(&ctx->fd_wqh, &wait);
1158 	__set_current_state(TASK_RUNNING);
1159 	spin_unlock_irq(&ctx->fd_wqh.lock);
1160 
1161 	if (!ret && msg->event == UFFD_EVENT_FORK) {
1162 		ret = resolve_userfault_fork(fork_nctx, inode, msg);
1163 		spin_lock_irq(&ctx->event_wqh.lock);
1164 		if (!list_empty(&fork_event)) {
1165 			/*
1166 			 * The fork thread didn't abort, so we can
1167 			 * drop the temporary refcount.
1168 			 */
1169 			userfaultfd_ctx_put(fork_nctx);
1170 
1171 			uwq = list_first_entry(&fork_event,
1172 					       typeof(*uwq),
1173 					       wq.entry);
1174 			/*
1175 			 * If fork_event list wasn't empty and in turn
1176 			 * the event wasn't already released by fork
1177 			 * (the event is allocated on fork kernel
1178 			 * stack), put the event back to its place in
1179 			 * the event_wq. fork_event head will be freed
1180 			 * as soon as we return so the event cannot
1181 			 * stay queued there no matter the current
1182 			 * "ret" value.
1183 			 */
1184 			list_del(&uwq->wq.entry);
1185 			__add_wait_queue(&ctx->event_wqh, &uwq->wq);
1186 
1187 			/*
1188 			 * Leave the event in the waitqueue and report
1189 			 * error to userland if we failed to resolve
1190 			 * the userfault fork.
1191 			 */
1192 			if (likely(!ret))
1193 				userfaultfd_event_complete(ctx, uwq);
1194 		} else {
1195 			/*
1196 			 * Here the fork thread aborted and the
1197 			 * refcount from the fork thread on fork_nctx
1198 			 * has already been released. We still hold
1199 			 * the reference we took before releasing the
1200 			 * lock above. If resolve_userfault_fork
1201 			 * failed we've to drop it because the
1202 			 * fork_nctx has to be freed in such case. If
1203 			 * it succeeded we'll hold it because the new
1204 			 * uffd references it.
1205 			 */
1206 			if (ret)
1207 				userfaultfd_ctx_put(fork_nctx);
1208 		}
1209 		spin_unlock_irq(&ctx->event_wqh.lock);
1210 	}
1211 
1212 	return ret;
1213 }
1214 
userfaultfd_read(struct file * file,char __user * buf,size_t count,loff_t * ppos)1215 static ssize_t userfaultfd_read(struct file *file, char __user *buf,
1216 				size_t count, loff_t *ppos)
1217 {
1218 	struct userfaultfd_ctx *ctx = file->private_data;
1219 	ssize_t _ret, ret = 0;
1220 	struct uffd_msg msg;
1221 	int no_wait = file->f_flags & O_NONBLOCK;
1222 	struct inode *inode = file_inode(file);
1223 
1224 	if (!userfaultfd_is_initialized(ctx))
1225 		return -EINVAL;
1226 
1227 	for (;;) {
1228 		if (count < sizeof(msg))
1229 			return ret ? ret : -EINVAL;
1230 		_ret = userfaultfd_ctx_read(ctx, no_wait, &msg, inode);
1231 		if (_ret < 0)
1232 			return ret ? ret : _ret;
1233 		if (copy_to_user((__u64 __user *) buf, &msg, sizeof(msg)))
1234 			return ret ? ret : -EFAULT;
1235 		ret += sizeof(msg);
1236 		buf += sizeof(msg);
1237 		count -= sizeof(msg);
1238 		/*
1239 		 * Allow to read more than one fault at time but only
1240 		 * block if waiting for the very first one.
1241 		 */
1242 		no_wait = O_NONBLOCK;
1243 	}
1244 }
1245 
__wake_userfault(struct userfaultfd_ctx * ctx,struct userfaultfd_wake_range * range)1246 static void __wake_userfault(struct userfaultfd_ctx *ctx,
1247 			     struct userfaultfd_wake_range *range)
1248 {
1249 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
1250 	/* wake all in the range and autoremove */
1251 	if (waitqueue_active(&ctx->fault_pending_wqh))
1252 		__wake_up_locked_key(&ctx->fault_pending_wqh, TASK_NORMAL,
1253 				     range);
1254 	if (waitqueue_active(&ctx->fault_wqh))
1255 		__wake_up(&ctx->fault_wqh, TASK_NORMAL, 1, range);
1256 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
1257 }
1258 
wake_userfault(struct userfaultfd_ctx * ctx,struct userfaultfd_wake_range * range)1259 static __always_inline void wake_userfault(struct userfaultfd_ctx *ctx,
1260 					   struct userfaultfd_wake_range *range)
1261 {
1262 	unsigned seq;
1263 	bool need_wakeup;
1264 
1265 	/*
1266 	 * To be sure waitqueue_active() is not reordered by the CPU
1267 	 * before the pagetable update, use an explicit SMP memory
1268 	 * barrier here. PT lock release or mmap_read_unlock(mm) still
1269 	 * have release semantics that can allow the
1270 	 * waitqueue_active() to be reordered before the pte update.
1271 	 */
1272 	smp_mb();
1273 
1274 	/*
1275 	 * Use waitqueue_active because it's very frequent to
1276 	 * change the address space atomically even if there are no
1277 	 * userfaults yet. So we take the spinlock only when we're
1278 	 * sure we've userfaults to wake.
1279 	 */
1280 	do {
1281 		seq = read_seqcount_begin(&ctx->refile_seq);
1282 		need_wakeup = waitqueue_active(&ctx->fault_pending_wqh) ||
1283 			waitqueue_active(&ctx->fault_wqh);
1284 		cond_resched();
1285 	} while (read_seqcount_retry(&ctx->refile_seq, seq));
1286 	if (need_wakeup)
1287 		__wake_userfault(ctx, range);
1288 }
1289 
validate_unaligned_range(struct mm_struct * mm,__u64 start,__u64 len)1290 static __always_inline int validate_unaligned_range(
1291 	struct mm_struct *mm, __u64 start, __u64 len)
1292 {
1293 	__u64 task_size = mm->task_size;
1294 
1295 	if (len & ~PAGE_MASK)
1296 		return -EINVAL;
1297 	if (!len)
1298 		return -EINVAL;
1299 	if (start < mmap_min_addr)
1300 		return -EINVAL;
1301 	if (start >= task_size)
1302 		return -EINVAL;
1303 	if (len > task_size - start)
1304 		return -EINVAL;
1305 	if (start + len <= start)
1306 		return -EINVAL;
1307 	return 0;
1308 }
1309 
validate_range(struct mm_struct * mm,__u64 start,__u64 len)1310 static __always_inline int validate_range(struct mm_struct *mm,
1311 					  __u64 start, __u64 len)
1312 {
1313 	if (start & ~PAGE_MASK)
1314 		return -EINVAL;
1315 
1316 	return validate_unaligned_range(mm, start, len);
1317 }
1318 
userfaultfd_register(struct userfaultfd_ctx * ctx,unsigned long arg)1319 static int userfaultfd_register(struct userfaultfd_ctx *ctx,
1320 				unsigned long arg)
1321 {
1322 	struct mm_struct *mm = ctx->mm;
1323 	struct vm_area_struct *vma, *prev, *cur;
1324 	int ret;
1325 	struct uffdio_register uffdio_register;
1326 	struct uffdio_register __user *user_uffdio_register;
1327 	unsigned long vm_flags, new_flags;
1328 	bool found;
1329 	bool basic_ioctls;
1330 	unsigned long start, end, vma_end;
1331 	struct vma_iterator vmi;
1332 	pgoff_t pgoff;
1333 
1334 	user_uffdio_register = (struct uffdio_register __user *) arg;
1335 
1336 	ret = -EFAULT;
1337 	if (copy_from_user(&uffdio_register, user_uffdio_register,
1338 			   sizeof(uffdio_register)-sizeof(__u64)))
1339 		goto out;
1340 
1341 	ret = -EINVAL;
1342 	if (!uffdio_register.mode)
1343 		goto out;
1344 	if (uffdio_register.mode & ~UFFD_API_REGISTER_MODES)
1345 		goto out;
1346 	vm_flags = 0;
1347 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MISSING)
1348 		vm_flags |= VM_UFFD_MISSING;
1349 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_WP) {
1350 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
1351 		goto out;
1352 #endif
1353 		vm_flags |= VM_UFFD_WP;
1354 	}
1355 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR) {
1356 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
1357 		goto out;
1358 #endif
1359 		vm_flags |= VM_UFFD_MINOR;
1360 	}
1361 
1362 	ret = validate_range(mm, uffdio_register.range.start,
1363 			     uffdio_register.range.len);
1364 	if (ret)
1365 		goto out;
1366 
1367 	start = uffdio_register.range.start;
1368 	end = start + uffdio_register.range.len;
1369 
1370 	ret = -ENOMEM;
1371 	if (!mmget_not_zero(mm))
1372 		goto out;
1373 
1374 	ret = -EINVAL;
1375 	mmap_write_lock(mm);
1376 	vma_iter_init(&vmi, mm, start);
1377 	vma = vma_find(&vmi, end);
1378 	if (!vma)
1379 		goto out_unlock;
1380 
1381 	/*
1382 	 * If the first vma contains huge pages, make sure start address
1383 	 * is aligned to huge page size.
1384 	 */
1385 	if (is_vm_hugetlb_page(vma)) {
1386 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1387 
1388 		if (start & (vma_hpagesize - 1))
1389 			goto out_unlock;
1390 	}
1391 
1392 	/*
1393 	 * Search for not compatible vmas.
1394 	 */
1395 	found = false;
1396 	basic_ioctls = false;
1397 	cur = vma;
1398 	do {
1399 		cond_resched();
1400 
1401 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1402 		       !!(cur->vm_flags & __VM_UFFD_FLAGS));
1403 
1404 		/* check not compatible vmas */
1405 		ret = -EINVAL;
1406 		if (!vma_can_userfault(cur, vm_flags))
1407 			goto out_unlock;
1408 
1409 		/*
1410 		 * UFFDIO_COPY will fill file holes even without
1411 		 * PROT_WRITE. This check enforces that if this is a
1412 		 * MAP_SHARED, the process has write permission to the backing
1413 		 * file. If VM_MAYWRITE is set it also enforces that on a
1414 		 * MAP_SHARED vma: there is no F_WRITE_SEAL and no further
1415 		 * F_WRITE_SEAL can be taken until the vma is destroyed.
1416 		 */
1417 		ret = -EPERM;
1418 		if (unlikely(!(cur->vm_flags & VM_MAYWRITE)))
1419 			goto out_unlock;
1420 
1421 		/*
1422 		 * If this vma contains ending address, and huge pages
1423 		 * check alignment.
1424 		 */
1425 		if (is_vm_hugetlb_page(cur) && end <= cur->vm_end &&
1426 		    end > cur->vm_start) {
1427 			unsigned long vma_hpagesize = vma_kernel_pagesize(cur);
1428 
1429 			ret = -EINVAL;
1430 
1431 			if (end & (vma_hpagesize - 1))
1432 				goto out_unlock;
1433 		}
1434 		if ((vm_flags & VM_UFFD_WP) && !(cur->vm_flags & VM_MAYWRITE))
1435 			goto out_unlock;
1436 
1437 		/*
1438 		 * Check that this vma isn't already owned by a
1439 		 * different userfaultfd. We can't allow more than one
1440 		 * userfaultfd to own a single vma simultaneously or we
1441 		 * wouldn't know which one to deliver the userfaults to.
1442 		 */
1443 		ret = -EBUSY;
1444 		if (cur->vm_userfaultfd_ctx.ctx &&
1445 		    cur->vm_userfaultfd_ctx.ctx != ctx)
1446 			goto out_unlock;
1447 
1448 		/*
1449 		 * Note vmas containing huge pages
1450 		 */
1451 		if (is_vm_hugetlb_page(cur))
1452 			basic_ioctls = true;
1453 
1454 		found = true;
1455 	} for_each_vma_range(vmi, cur, end);
1456 	BUG_ON(!found);
1457 
1458 	vma_iter_set(&vmi, start);
1459 	prev = vma_prev(&vmi);
1460 	if (vma->vm_start < start)
1461 		prev = vma;
1462 
1463 	ret = 0;
1464 	for_each_vma_range(vmi, vma, end) {
1465 		cond_resched();
1466 
1467 		BUG_ON(!vma_can_userfault(vma, vm_flags));
1468 		BUG_ON(vma->vm_userfaultfd_ctx.ctx &&
1469 		       vma->vm_userfaultfd_ctx.ctx != ctx);
1470 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1471 
1472 		/*
1473 		 * Nothing to do: this vma is already registered into this
1474 		 * userfaultfd and with the right tracking mode too.
1475 		 */
1476 		if (vma->vm_userfaultfd_ctx.ctx == ctx &&
1477 		    (vma->vm_flags & vm_flags) == vm_flags)
1478 			goto skip;
1479 
1480 		if (vma->vm_start > start)
1481 			start = vma->vm_start;
1482 		vma_end = min(end, vma->vm_end);
1483 
1484 		new_flags = (vma->vm_flags & ~__VM_UFFD_FLAGS) | vm_flags;
1485 		pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
1486 		prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
1487 				 vma->anon_vma, vma->vm_file, pgoff,
1488 				 vma_policy(vma),
1489 				 ((struct vm_userfaultfd_ctx){ ctx }),
1490 				 anon_vma_name(vma));
1491 		if (prev) {
1492 			/* vma_merge() invalidated the mas */
1493 			vma = prev;
1494 			goto next;
1495 		}
1496 		if (vma->vm_start < start) {
1497 			ret = split_vma(&vmi, vma, start, 1);
1498 			if (ret)
1499 				break;
1500 		}
1501 		if (vma->vm_end > end) {
1502 			ret = split_vma(&vmi, vma, end, 0);
1503 			if (ret)
1504 				break;
1505 		}
1506 	next:
1507 		/*
1508 		 * In the vma_merge() successful mprotect-like case 8:
1509 		 * the next vma was merged into the current one and
1510 		 * the current one has not been updated yet.
1511 		 */
1512 		vma_start_write(vma);
1513 		userfaultfd_set_vm_flags(vma, new_flags);
1514 		vma->vm_userfaultfd_ctx.ctx = ctx;
1515 
1516 		if (is_vm_hugetlb_page(vma) && uffd_disable_huge_pmd_share(vma))
1517 			hugetlb_unshare_all_pmds(vma);
1518 
1519 	skip:
1520 		prev = vma;
1521 		start = vma->vm_end;
1522 	}
1523 
1524 out_unlock:
1525 	mmap_write_unlock(mm);
1526 	mmput(mm);
1527 	if (!ret) {
1528 		__u64 ioctls_out;
1529 
1530 		ioctls_out = basic_ioctls ? UFFD_API_RANGE_IOCTLS_BASIC :
1531 		    UFFD_API_RANGE_IOCTLS;
1532 
1533 		/*
1534 		 * Declare the WP ioctl only if the WP mode is
1535 		 * specified and all checks passed with the range
1536 		 */
1537 		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_WP))
1538 			ioctls_out &= ~((__u64)1 << _UFFDIO_WRITEPROTECT);
1539 
1540 		/* CONTINUE ioctl is only supported for MINOR ranges. */
1541 		if (!(uffdio_register.mode & UFFDIO_REGISTER_MODE_MINOR))
1542 			ioctls_out &= ~((__u64)1 << _UFFDIO_CONTINUE);
1543 
1544 		/*
1545 		 * Now that we scanned all vmas we can already tell
1546 		 * userland which ioctls methods are guaranteed to
1547 		 * succeed on this range.
1548 		 */
1549 		if (put_user(ioctls_out, &user_uffdio_register->ioctls))
1550 			ret = -EFAULT;
1551 	}
1552 out:
1553 	return ret;
1554 }
1555 
userfaultfd_unregister(struct userfaultfd_ctx * ctx,unsigned long arg)1556 static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
1557 				  unsigned long arg)
1558 {
1559 	struct mm_struct *mm = ctx->mm;
1560 	struct vm_area_struct *vma, *prev, *cur;
1561 	int ret;
1562 	struct uffdio_range uffdio_unregister;
1563 	unsigned long new_flags;
1564 	bool found;
1565 	unsigned long start, end, vma_end;
1566 	const void __user *buf = (void __user *)arg;
1567 	struct vma_iterator vmi;
1568 	pgoff_t pgoff;
1569 
1570 	ret = -EFAULT;
1571 	if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
1572 		goto out;
1573 
1574 	ret = validate_range(mm, uffdio_unregister.start,
1575 			     uffdio_unregister.len);
1576 	if (ret)
1577 		goto out;
1578 
1579 	start = uffdio_unregister.start;
1580 	end = start + uffdio_unregister.len;
1581 
1582 	ret = -ENOMEM;
1583 	if (!mmget_not_zero(mm))
1584 		goto out;
1585 
1586 	mmap_write_lock(mm);
1587 	ret = -EINVAL;
1588 	vma_iter_init(&vmi, mm, start);
1589 	vma = vma_find(&vmi, end);
1590 	if (!vma)
1591 		goto out_unlock;
1592 
1593 	/*
1594 	 * If the first vma contains huge pages, make sure start address
1595 	 * is aligned to huge page size.
1596 	 */
1597 	if (is_vm_hugetlb_page(vma)) {
1598 		unsigned long vma_hpagesize = vma_kernel_pagesize(vma);
1599 
1600 		if (start & (vma_hpagesize - 1))
1601 			goto out_unlock;
1602 	}
1603 
1604 	/*
1605 	 * Search for not compatible vmas.
1606 	 */
1607 	found = false;
1608 	cur = vma;
1609 	do {
1610 		cond_resched();
1611 
1612 		BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
1613 		       !!(cur->vm_flags & __VM_UFFD_FLAGS));
1614 
1615 		/*
1616 		 * Check not compatible vmas, not strictly required
1617 		 * here as not compatible vmas cannot have an
1618 		 * userfaultfd_ctx registered on them, but this
1619 		 * provides for more strict behavior to notice
1620 		 * unregistration errors.
1621 		 */
1622 		if (!vma_can_userfault(cur, cur->vm_flags))
1623 			goto out_unlock;
1624 
1625 		found = true;
1626 	} for_each_vma_range(vmi, cur, end);
1627 	BUG_ON(!found);
1628 
1629 	vma_iter_set(&vmi, start);
1630 	prev = vma_prev(&vmi);
1631 	if (vma->vm_start < start)
1632 		prev = vma;
1633 
1634 	ret = 0;
1635 	for_each_vma_range(vmi, vma, end) {
1636 		cond_resched();
1637 
1638 		BUG_ON(!vma_can_userfault(vma, vma->vm_flags));
1639 
1640 		/*
1641 		 * Nothing to do: this vma is already registered into this
1642 		 * userfaultfd and with the right tracking mode too.
1643 		 */
1644 		if (!vma->vm_userfaultfd_ctx.ctx)
1645 			goto skip;
1646 
1647 		WARN_ON(!(vma->vm_flags & VM_MAYWRITE));
1648 
1649 		if (vma->vm_start > start)
1650 			start = vma->vm_start;
1651 		vma_end = min(end, vma->vm_end);
1652 
1653 		if (userfaultfd_missing(vma)) {
1654 			/*
1655 			 * Wake any concurrent pending userfault while
1656 			 * we unregister, so they will not hang
1657 			 * permanently and it avoids userland to call
1658 			 * UFFDIO_WAKE explicitly.
1659 			 */
1660 			struct userfaultfd_wake_range range;
1661 			range.start = start;
1662 			range.len = vma_end - start;
1663 			wake_userfault(vma->vm_userfaultfd_ctx.ctx, &range);
1664 		}
1665 
1666 		/* Reset ptes for the whole vma range if wr-protected */
1667 		if (userfaultfd_wp(vma))
1668 			uffd_wp_range(vma, start, vma_end - start, false);
1669 
1670 		new_flags = vma->vm_flags & ~__VM_UFFD_FLAGS;
1671 		pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
1672 		prev = vma_merge(&vmi, mm, prev, start, vma_end, new_flags,
1673 				 vma->anon_vma, vma->vm_file, pgoff,
1674 				 vma_policy(vma),
1675 				 NULL_VM_UFFD_CTX, anon_vma_name(vma));
1676 		if (prev) {
1677 			vma = prev;
1678 			goto next;
1679 		}
1680 		if (vma->vm_start < start) {
1681 			ret = split_vma(&vmi, vma, start, 1);
1682 			if (ret)
1683 				break;
1684 		}
1685 		if (vma->vm_end > end) {
1686 			ret = split_vma(&vmi, vma, end, 0);
1687 			if (ret)
1688 				break;
1689 		}
1690 	next:
1691 		/*
1692 		 * In the vma_merge() successful mprotect-like case 8:
1693 		 * the next vma was merged into the current one and
1694 		 * the current one has not been updated yet.
1695 		 */
1696 		vma_start_write(vma);
1697 		userfaultfd_set_vm_flags(vma, new_flags);
1698 		vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
1699 
1700 	skip:
1701 		prev = vma;
1702 		start = vma->vm_end;
1703 	}
1704 
1705 out_unlock:
1706 	mmap_write_unlock(mm);
1707 	mmput(mm);
1708 out:
1709 	return ret;
1710 }
1711 
1712 /*
1713  * userfaultfd_wake may be used in combination with the
1714  * UFFDIO_*_MODE_DONTWAKE to wakeup userfaults in batches.
1715  */
userfaultfd_wake(struct userfaultfd_ctx * ctx,unsigned long arg)1716 static int userfaultfd_wake(struct userfaultfd_ctx *ctx,
1717 			    unsigned long arg)
1718 {
1719 	int ret;
1720 	struct uffdio_range uffdio_wake;
1721 	struct userfaultfd_wake_range range;
1722 	const void __user *buf = (void __user *)arg;
1723 
1724 	ret = -EFAULT;
1725 	if (copy_from_user(&uffdio_wake, buf, sizeof(uffdio_wake)))
1726 		goto out;
1727 
1728 	ret = validate_range(ctx->mm, uffdio_wake.start, uffdio_wake.len);
1729 	if (ret)
1730 		goto out;
1731 
1732 	range.start = uffdio_wake.start;
1733 	range.len = uffdio_wake.len;
1734 
1735 	/*
1736 	 * len == 0 means wake all and we don't want to wake all here,
1737 	 * so check it again to be sure.
1738 	 */
1739 	VM_BUG_ON(!range.len);
1740 
1741 	wake_userfault(ctx, &range);
1742 	ret = 0;
1743 
1744 out:
1745 	return ret;
1746 }
1747 
userfaultfd_copy(struct userfaultfd_ctx * ctx,unsigned long arg)1748 static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
1749 			    unsigned long arg)
1750 {
1751 	__s64 ret;
1752 	struct uffdio_copy uffdio_copy;
1753 	struct uffdio_copy __user *user_uffdio_copy;
1754 	struct userfaultfd_wake_range range;
1755 	uffd_flags_t flags = 0;
1756 
1757 	user_uffdio_copy = (struct uffdio_copy __user *) arg;
1758 
1759 	ret = -EAGAIN;
1760 	if (atomic_read(&ctx->mmap_changing))
1761 		goto out;
1762 
1763 	ret = -EFAULT;
1764 	if (copy_from_user(&uffdio_copy, user_uffdio_copy,
1765 			   /* don't copy "copy" last field */
1766 			   sizeof(uffdio_copy)-sizeof(__s64)))
1767 		goto out;
1768 
1769 	ret = validate_unaligned_range(ctx->mm, uffdio_copy.src,
1770 				       uffdio_copy.len);
1771 	if (ret)
1772 		goto out;
1773 	ret = validate_range(ctx->mm, uffdio_copy.dst, uffdio_copy.len);
1774 	if (ret)
1775 		goto out;
1776 
1777 	ret = -EINVAL;
1778 	if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP))
1779 		goto out;
1780 	if (uffdio_copy.mode & UFFDIO_COPY_MODE_WP)
1781 		flags |= MFILL_ATOMIC_WP;
1782 	if (mmget_not_zero(ctx->mm)) {
1783 		ret = mfill_atomic_copy(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
1784 					uffdio_copy.len, &ctx->mmap_changing,
1785 					flags);
1786 		mmput(ctx->mm);
1787 	} else {
1788 		return -ESRCH;
1789 	}
1790 	if (unlikely(put_user(ret, &user_uffdio_copy->copy)))
1791 		return -EFAULT;
1792 	if (ret < 0)
1793 		goto out;
1794 	BUG_ON(!ret);
1795 	/* len == 0 would wake all */
1796 	range.len = ret;
1797 	if (!(uffdio_copy.mode & UFFDIO_COPY_MODE_DONTWAKE)) {
1798 		range.start = uffdio_copy.dst;
1799 		wake_userfault(ctx, &range);
1800 	}
1801 	ret = range.len == uffdio_copy.len ? 0 : -EAGAIN;
1802 out:
1803 	return ret;
1804 }
1805 
userfaultfd_zeropage(struct userfaultfd_ctx * ctx,unsigned long arg)1806 static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
1807 				unsigned long arg)
1808 {
1809 	__s64 ret;
1810 	struct uffdio_zeropage uffdio_zeropage;
1811 	struct uffdio_zeropage __user *user_uffdio_zeropage;
1812 	struct userfaultfd_wake_range range;
1813 
1814 	user_uffdio_zeropage = (struct uffdio_zeropage __user *) arg;
1815 
1816 	ret = -EAGAIN;
1817 	if (atomic_read(&ctx->mmap_changing))
1818 		goto out;
1819 
1820 	ret = -EFAULT;
1821 	if (copy_from_user(&uffdio_zeropage, user_uffdio_zeropage,
1822 			   /* don't copy "zeropage" last field */
1823 			   sizeof(uffdio_zeropage)-sizeof(__s64)))
1824 		goto out;
1825 
1826 	ret = validate_range(ctx->mm, uffdio_zeropage.range.start,
1827 			     uffdio_zeropage.range.len);
1828 	if (ret)
1829 		goto out;
1830 	ret = -EINVAL;
1831 	if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE)
1832 		goto out;
1833 
1834 	if (mmget_not_zero(ctx->mm)) {
1835 		ret = mfill_atomic_zeropage(ctx->mm, uffdio_zeropage.range.start,
1836 					   uffdio_zeropage.range.len,
1837 					   &ctx->mmap_changing);
1838 		mmput(ctx->mm);
1839 	} else {
1840 		return -ESRCH;
1841 	}
1842 	if (unlikely(put_user(ret, &user_uffdio_zeropage->zeropage)))
1843 		return -EFAULT;
1844 	if (ret < 0)
1845 		goto out;
1846 	/* len == 0 would wake all */
1847 	BUG_ON(!ret);
1848 	range.len = ret;
1849 	if (!(uffdio_zeropage.mode & UFFDIO_ZEROPAGE_MODE_DONTWAKE)) {
1850 		range.start = uffdio_zeropage.range.start;
1851 		wake_userfault(ctx, &range);
1852 	}
1853 	ret = range.len == uffdio_zeropage.range.len ? 0 : -EAGAIN;
1854 out:
1855 	return ret;
1856 }
1857 
userfaultfd_writeprotect(struct userfaultfd_ctx * ctx,unsigned long arg)1858 static int userfaultfd_writeprotect(struct userfaultfd_ctx *ctx,
1859 				    unsigned long arg)
1860 {
1861 	int ret;
1862 	struct uffdio_writeprotect uffdio_wp;
1863 	struct uffdio_writeprotect __user *user_uffdio_wp;
1864 	struct userfaultfd_wake_range range;
1865 	bool mode_wp, mode_dontwake;
1866 
1867 	if (atomic_read(&ctx->mmap_changing))
1868 		return -EAGAIN;
1869 
1870 	user_uffdio_wp = (struct uffdio_writeprotect __user *) arg;
1871 
1872 	if (copy_from_user(&uffdio_wp, user_uffdio_wp,
1873 			   sizeof(struct uffdio_writeprotect)))
1874 		return -EFAULT;
1875 
1876 	ret = validate_range(ctx->mm, uffdio_wp.range.start,
1877 			     uffdio_wp.range.len);
1878 	if (ret)
1879 		return ret;
1880 
1881 	if (uffdio_wp.mode & ~(UFFDIO_WRITEPROTECT_MODE_DONTWAKE |
1882 			       UFFDIO_WRITEPROTECT_MODE_WP))
1883 		return -EINVAL;
1884 
1885 	mode_wp = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_WP;
1886 	mode_dontwake = uffdio_wp.mode & UFFDIO_WRITEPROTECT_MODE_DONTWAKE;
1887 
1888 	if (mode_wp && mode_dontwake)
1889 		return -EINVAL;
1890 
1891 	if (mmget_not_zero(ctx->mm)) {
1892 		ret = mwriteprotect_range(ctx->mm, uffdio_wp.range.start,
1893 					  uffdio_wp.range.len, mode_wp,
1894 					  &ctx->mmap_changing);
1895 		mmput(ctx->mm);
1896 	} else {
1897 		return -ESRCH;
1898 	}
1899 
1900 	if (ret)
1901 		return ret;
1902 
1903 	if (!mode_wp && !mode_dontwake) {
1904 		range.start = uffdio_wp.range.start;
1905 		range.len = uffdio_wp.range.len;
1906 		wake_userfault(ctx, &range);
1907 	}
1908 	return ret;
1909 }
1910 
userfaultfd_continue(struct userfaultfd_ctx * ctx,unsigned long arg)1911 static int userfaultfd_continue(struct userfaultfd_ctx *ctx, unsigned long arg)
1912 {
1913 	__s64 ret;
1914 	struct uffdio_continue uffdio_continue;
1915 	struct uffdio_continue __user *user_uffdio_continue;
1916 	struct userfaultfd_wake_range range;
1917 	uffd_flags_t flags = 0;
1918 
1919 	user_uffdio_continue = (struct uffdio_continue __user *)arg;
1920 
1921 	ret = -EAGAIN;
1922 	if (atomic_read(&ctx->mmap_changing))
1923 		goto out;
1924 
1925 	ret = -EFAULT;
1926 	if (copy_from_user(&uffdio_continue, user_uffdio_continue,
1927 			   /* don't copy the output fields */
1928 			   sizeof(uffdio_continue) - (sizeof(__s64))))
1929 		goto out;
1930 
1931 	ret = validate_range(ctx->mm, uffdio_continue.range.start,
1932 			     uffdio_continue.range.len);
1933 	if (ret)
1934 		goto out;
1935 
1936 	ret = -EINVAL;
1937 	if (uffdio_continue.mode & ~(UFFDIO_CONTINUE_MODE_DONTWAKE |
1938 				     UFFDIO_CONTINUE_MODE_WP))
1939 		goto out;
1940 	if (uffdio_continue.mode & UFFDIO_CONTINUE_MODE_WP)
1941 		flags |= MFILL_ATOMIC_WP;
1942 
1943 	if (mmget_not_zero(ctx->mm)) {
1944 		ret = mfill_atomic_continue(ctx->mm, uffdio_continue.range.start,
1945 					    uffdio_continue.range.len,
1946 					    &ctx->mmap_changing, flags);
1947 		mmput(ctx->mm);
1948 	} else {
1949 		return -ESRCH;
1950 	}
1951 
1952 	if (unlikely(put_user(ret, &user_uffdio_continue->mapped)))
1953 		return -EFAULT;
1954 	if (ret < 0)
1955 		goto out;
1956 
1957 	/* len == 0 would wake all */
1958 	BUG_ON(!ret);
1959 	range.len = ret;
1960 	if (!(uffdio_continue.mode & UFFDIO_CONTINUE_MODE_DONTWAKE)) {
1961 		range.start = uffdio_continue.range.start;
1962 		wake_userfault(ctx, &range);
1963 	}
1964 	ret = range.len == uffdio_continue.range.len ? 0 : -EAGAIN;
1965 
1966 out:
1967 	return ret;
1968 }
1969 
userfaultfd_poison(struct userfaultfd_ctx * ctx,unsigned long arg)1970 static inline int userfaultfd_poison(struct userfaultfd_ctx *ctx, unsigned long arg)
1971 {
1972 	__s64 ret;
1973 	struct uffdio_poison uffdio_poison;
1974 	struct uffdio_poison __user *user_uffdio_poison;
1975 	struct userfaultfd_wake_range range;
1976 
1977 	user_uffdio_poison = (struct uffdio_poison __user *)arg;
1978 
1979 	ret = -EAGAIN;
1980 	if (atomic_read(&ctx->mmap_changing))
1981 		goto out;
1982 
1983 	ret = -EFAULT;
1984 	if (copy_from_user(&uffdio_poison, user_uffdio_poison,
1985 			   /* don't copy the output fields */
1986 			   sizeof(uffdio_poison) - (sizeof(__s64))))
1987 		goto out;
1988 
1989 	ret = validate_range(ctx->mm, uffdio_poison.range.start,
1990 			     uffdio_poison.range.len);
1991 	if (ret)
1992 		goto out;
1993 
1994 	ret = -EINVAL;
1995 	if (uffdio_poison.mode & ~UFFDIO_POISON_MODE_DONTWAKE)
1996 		goto out;
1997 
1998 	if (mmget_not_zero(ctx->mm)) {
1999 		ret = mfill_atomic_poison(ctx->mm, uffdio_poison.range.start,
2000 					  uffdio_poison.range.len,
2001 					  &ctx->mmap_changing, 0);
2002 		mmput(ctx->mm);
2003 	} else {
2004 		return -ESRCH;
2005 	}
2006 
2007 	if (unlikely(put_user(ret, &user_uffdio_poison->updated)))
2008 		return -EFAULT;
2009 	if (ret < 0)
2010 		goto out;
2011 
2012 	/* len == 0 would wake all */
2013 	BUG_ON(!ret);
2014 	range.len = ret;
2015 	if (!(uffdio_poison.mode & UFFDIO_POISON_MODE_DONTWAKE)) {
2016 		range.start = uffdio_poison.range.start;
2017 		wake_userfault(ctx, &range);
2018 	}
2019 	ret = range.len == uffdio_poison.range.len ? 0 : -EAGAIN;
2020 
2021 out:
2022 	return ret;
2023 }
2024 
uffd_ctx_features(__u64 user_features)2025 static inline unsigned int uffd_ctx_features(__u64 user_features)
2026 {
2027 	/*
2028 	 * For the current set of features the bits just coincide. Set
2029 	 * UFFD_FEATURE_INITIALIZED to mark the features as enabled.
2030 	 */
2031 	return (unsigned int)user_features | UFFD_FEATURE_INITIALIZED;
2032 }
2033 
2034 /*
2035  * userland asks for a certain API version and we return which bits
2036  * and ioctl commands are implemented in this kernel for such API
2037  * version or -EINVAL if unknown.
2038  */
userfaultfd_api(struct userfaultfd_ctx * ctx,unsigned long arg)2039 static int userfaultfd_api(struct userfaultfd_ctx *ctx,
2040 			   unsigned long arg)
2041 {
2042 	struct uffdio_api uffdio_api;
2043 	void __user *buf = (void __user *)arg;
2044 	unsigned int ctx_features;
2045 	int ret;
2046 	__u64 features;
2047 
2048 	ret = -EFAULT;
2049 	if (copy_from_user(&uffdio_api, buf, sizeof(uffdio_api)))
2050 		goto out;
2051 	features = uffdio_api.features;
2052 	ret = -EINVAL;
2053 	if (uffdio_api.api != UFFD_API)
2054 		goto err_out;
2055 	ret = -EPERM;
2056 	if ((features & UFFD_FEATURE_EVENT_FORK) && !capable(CAP_SYS_PTRACE))
2057 		goto err_out;
2058 	/* report all available features and ioctls to userland */
2059 	uffdio_api.features = UFFD_API_FEATURES;
2060 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
2061 	uffdio_api.features &=
2062 		~(UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM);
2063 #endif
2064 #ifndef CONFIG_HAVE_ARCH_USERFAULTFD_WP
2065 	uffdio_api.features &= ~UFFD_FEATURE_PAGEFAULT_FLAG_WP;
2066 #endif
2067 #ifndef CONFIG_PTE_MARKER_UFFD_WP
2068 	uffdio_api.features &= ~UFFD_FEATURE_WP_HUGETLBFS_SHMEM;
2069 	uffdio_api.features &= ~UFFD_FEATURE_WP_UNPOPULATED;
2070 #endif
2071 
2072 	ret = -EINVAL;
2073 	if (features & ~uffdio_api.features)
2074 		goto err_out;
2075 
2076 	uffdio_api.ioctls = UFFD_API_IOCTLS;
2077 	ret = -EFAULT;
2078 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
2079 		goto out;
2080 
2081 	/* only enable the requested features for this uffd context */
2082 	ctx_features = uffd_ctx_features(features);
2083 	ret = -EINVAL;
2084 	if (cmpxchg(&ctx->features, 0, ctx_features) != 0)
2085 		goto err_out;
2086 
2087 	ret = 0;
2088 out:
2089 	return ret;
2090 err_out:
2091 	memset(&uffdio_api, 0, sizeof(uffdio_api));
2092 	if (copy_to_user(buf, &uffdio_api, sizeof(uffdio_api)))
2093 		ret = -EFAULT;
2094 	goto out;
2095 }
2096 
userfaultfd_ioctl(struct file * file,unsigned cmd,unsigned long arg)2097 static long userfaultfd_ioctl(struct file *file, unsigned cmd,
2098 			      unsigned long arg)
2099 {
2100 	int ret = -EINVAL;
2101 	struct userfaultfd_ctx *ctx = file->private_data;
2102 
2103 	if (cmd != UFFDIO_API && !userfaultfd_is_initialized(ctx))
2104 		return -EINVAL;
2105 
2106 	switch(cmd) {
2107 	case UFFDIO_API:
2108 		ret = userfaultfd_api(ctx, arg);
2109 		break;
2110 	case UFFDIO_REGISTER:
2111 		ret = userfaultfd_register(ctx, arg);
2112 		break;
2113 	case UFFDIO_UNREGISTER:
2114 		ret = userfaultfd_unregister(ctx, arg);
2115 		break;
2116 	case UFFDIO_WAKE:
2117 		ret = userfaultfd_wake(ctx, arg);
2118 		break;
2119 	case UFFDIO_COPY:
2120 		ret = userfaultfd_copy(ctx, arg);
2121 		break;
2122 	case UFFDIO_ZEROPAGE:
2123 		ret = userfaultfd_zeropage(ctx, arg);
2124 		break;
2125 	case UFFDIO_WRITEPROTECT:
2126 		ret = userfaultfd_writeprotect(ctx, arg);
2127 		break;
2128 	case UFFDIO_CONTINUE:
2129 		ret = userfaultfd_continue(ctx, arg);
2130 		break;
2131 	case UFFDIO_POISON:
2132 		ret = userfaultfd_poison(ctx, arg);
2133 		break;
2134 	}
2135 	return ret;
2136 }
2137 
2138 #ifdef CONFIG_PROC_FS
userfaultfd_show_fdinfo(struct seq_file * m,struct file * f)2139 static void userfaultfd_show_fdinfo(struct seq_file *m, struct file *f)
2140 {
2141 	struct userfaultfd_ctx *ctx = f->private_data;
2142 	wait_queue_entry_t *wq;
2143 	unsigned long pending = 0, total = 0;
2144 
2145 	spin_lock_irq(&ctx->fault_pending_wqh.lock);
2146 	list_for_each_entry(wq, &ctx->fault_pending_wqh.head, entry) {
2147 		pending++;
2148 		total++;
2149 	}
2150 	list_for_each_entry(wq, &ctx->fault_wqh.head, entry) {
2151 		total++;
2152 	}
2153 	spin_unlock_irq(&ctx->fault_pending_wqh.lock);
2154 
2155 	/*
2156 	 * If more protocols will be added, there will be all shown
2157 	 * separated by a space. Like this:
2158 	 *	protocols: aa:... bb:...
2159 	 */
2160 	seq_printf(m, "pending:\t%lu\ntotal:\t%lu\nAPI:\t%Lx:%x:%Lx\n",
2161 		   pending, total, UFFD_API, ctx->features,
2162 		   UFFD_API_IOCTLS|UFFD_API_RANGE_IOCTLS);
2163 }
2164 #endif
2165 
2166 static const struct file_operations userfaultfd_fops = {
2167 #ifdef CONFIG_PROC_FS
2168 	.show_fdinfo	= userfaultfd_show_fdinfo,
2169 #endif
2170 	.release	= userfaultfd_release,
2171 	.poll		= userfaultfd_poll,
2172 	.read		= userfaultfd_read,
2173 	.unlocked_ioctl = userfaultfd_ioctl,
2174 	.compat_ioctl	= compat_ptr_ioctl,
2175 	.llseek		= noop_llseek,
2176 };
2177 
init_once_userfaultfd_ctx(void * mem)2178 static void init_once_userfaultfd_ctx(void *mem)
2179 {
2180 	struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem;
2181 
2182 	init_waitqueue_head(&ctx->fault_pending_wqh);
2183 	init_waitqueue_head(&ctx->fault_wqh);
2184 	init_waitqueue_head(&ctx->event_wqh);
2185 	init_waitqueue_head(&ctx->fd_wqh);
2186 	seqcount_spinlock_init(&ctx->refile_seq, &ctx->fault_pending_wqh.lock);
2187 }
2188 
new_userfaultfd(int flags)2189 static int new_userfaultfd(int flags)
2190 {
2191 	struct userfaultfd_ctx *ctx;
2192 	int fd;
2193 
2194 	BUG_ON(!current->mm);
2195 
2196 	/* Check the UFFD_* constants for consistency.  */
2197 	BUILD_BUG_ON(UFFD_USER_MODE_ONLY & UFFD_SHARED_FCNTL_FLAGS);
2198 	BUILD_BUG_ON(UFFD_CLOEXEC != O_CLOEXEC);
2199 	BUILD_BUG_ON(UFFD_NONBLOCK != O_NONBLOCK);
2200 
2201 	if (flags & ~(UFFD_SHARED_FCNTL_FLAGS | UFFD_USER_MODE_ONLY))
2202 		return -EINVAL;
2203 
2204 	ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL);
2205 	if (!ctx)
2206 		return -ENOMEM;
2207 
2208 	refcount_set(&ctx->refcount, 1);
2209 	ctx->flags = flags;
2210 	ctx->features = 0;
2211 	ctx->released = false;
2212 	atomic_set(&ctx->mmap_changing, 0);
2213 	ctx->mm = current->mm;
2214 	/* prevent the mm struct to be freed */
2215 	mmgrab(ctx->mm);
2216 
2217 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
2218 			O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
2219 	if (fd < 0) {
2220 		mmdrop(ctx->mm);
2221 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
2222 	}
2223 	return fd;
2224 }
2225 
userfaultfd_syscall_allowed(int flags)2226 static inline bool userfaultfd_syscall_allowed(int flags)
2227 {
2228 	/* Userspace-only page faults are always allowed */
2229 	if (flags & UFFD_USER_MODE_ONLY)
2230 		return true;
2231 
2232 	/*
2233 	 * The user is requesting a userfaultfd which can handle kernel faults.
2234 	 * Privileged users are always allowed to do this.
2235 	 */
2236 	if (capable(CAP_SYS_PTRACE))
2237 		return true;
2238 
2239 	/* Otherwise, access to kernel fault handling is sysctl controlled. */
2240 	return sysctl_unprivileged_userfaultfd;
2241 }
2242 
SYSCALL_DEFINE1(userfaultfd,int,flags)2243 SYSCALL_DEFINE1(userfaultfd, int, flags)
2244 {
2245 	if (!userfaultfd_syscall_allowed(flags))
2246 		return -EPERM;
2247 
2248 	return new_userfaultfd(flags);
2249 }
2250 
userfaultfd_dev_ioctl(struct file * file,unsigned int cmd,unsigned long flags)2251 static long userfaultfd_dev_ioctl(struct file *file, unsigned int cmd, unsigned long flags)
2252 {
2253 	if (cmd != USERFAULTFD_IOC_NEW)
2254 		return -EINVAL;
2255 
2256 	return new_userfaultfd(flags);
2257 }
2258 
2259 static const struct file_operations userfaultfd_dev_fops = {
2260 	.unlocked_ioctl = userfaultfd_dev_ioctl,
2261 	.compat_ioctl = userfaultfd_dev_ioctl,
2262 	.owner = THIS_MODULE,
2263 	.llseek = noop_llseek,
2264 };
2265 
2266 static struct miscdevice userfaultfd_misc = {
2267 	.minor = MISC_DYNAMIC_MINOR,
2268 	.name = "userfaultfd",
2269 	.fops = &userfaultfd_dev_fops
2270 };
2271 
userfaultfd_init(void)2272 static int __init userfaultfd_init(void)
2273 {
2274 	int ret;
2275 
2276 	ret = misc_register(&userfaultfd_misc);
2277 	if (ret)
2278 		return ret;
2279 
2280 	userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache",
2281 						sizeof(struct userfaultfd_ctx),
2282 						0,
2283 						SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2284 						init_once_userfaultfd_ctx);
2285 #ifdef CONFIG_SYSCTL
2286 	register_sysctl_init("vm", vm_userfaultfd_table);
2287 #endif
2288 	return 0;
2289 }
2290 __initcall(userfaultfd_init);
2291