xref: /openbmc/linux/mm/mmap.c (revision abfbd895)
1 /*
2  * mm/mmap.c
3  *
4  * Written by obz.
5  *
6  * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
7  */
8 
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 
11 #include <linux/kernel.h>
12 #include <linux/slab.h>
13 #include <linux/backing-dev.h>
14 #include <linux/mm.h>
15 #include <linux/vmacache.h>
16 #include <linux/shm.h>
17 #include <linux/mman.h>
18 #include <linux/pagemap.h>
19 #include <linux/swap.h>
20 #include <linux/syscalls.h>
21 #include <linux/capability.h>
22 #include <linux/init.h>
23 #include <linux/file.h>
24 #include <linux/fs.h>
25 #include <linux/personality.h>
26 #include <linux/security.h>
27 #include <linux/hugetlb.h>
28 #include <linux/profile.h>
29 #include <linux/export.h>
30 #include <linux/mount.h>
31 #include <linux/mempolicy.h>
32 #include <linux/rmap.h>
33 #include <linux/mmu_notifier.h>
34 #include <linux/mmdebug.h>
35 #include <linux/perf_event.h>
36 #include <linux/audit.h>
37 #include <linux/khugepaged.h>
38 #include <linux/uprobes.h>
39 #include <linux/rbtree_augmented.h>
40 #include <linux/sched/sysctl.h>
41 #include <linux/notifier.h>
42 #include <linux/memory.h>
43 #include <linux/printk.h>
44 #include <linux/userfaultfd_k.h>
45 
46 #include <asm/uaccess.h>
47 #include <asm/cacheflush.h>
48 #include <asm/tlb.h>
49 #include <asm/mmu_context.h>
50 
51 #include "internal.h"
52 
53 #ifndef arch_mmap_check
54 #define arch_mmap_check(addr, len, flags)	(0)
55 #endif
56 
57 #ifndef arch_rebalance_pgtables
58 #define arch_rebalance_pgtables(addr, len)		(addr)
59 #endif
60 
61 static void unmap_region(struct mm_struct *mm,
62 		struct vm_area_struct *vma, struct vm_area_struct *prev,
63 		unsigned long start, unsigned long end);
64 
65 /* description of effects of mapping type and prot in current implementation.
66  * this is due to the limited x86 page protection hardware.  The expected
67  * behavior is in parens:
68  *
69  * map_type	prot
70  *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
71  * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
72  *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
73  *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
74  *
75  * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
76  *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
77  *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
78  *
79  */
80 pgprot_t protection_map[16] = {
81 	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
82 	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
83 };
84 
85 pgprot_t vm_get_page_prot(unsigned long vm_flags)
86 {
87 	return __pgprot(pgprot_val(protection_map[vm_flags &
88 				(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
89 			pgprot_val(arch_vm_get_page_prot(vm_flags)));
90 }
91 EXPORT_SYMBOL(vm_get_page_prot);
92 
93 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
94 {
95 	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
96 }
97 
98 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
99 void vma_set_page_prot(struct vm_area_struct *vma)
100 {
101 	unsigned long vm_flags = vma->vm_flags;
102 
103 	vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
104 	if (vma_wants_writenotify(vma)) {
105 		vm_flags &= ~VM_SHARED;
106 		vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot,
107 						     vm_flags);
108 	}
109 }
110 
111 
112 int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
113 int sysctl_overcommit_ratio __read_mostly = 50;	/* default is 50% */
114 unsigned long sysctl_overcommit_kbytes __read_mostly;
115 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
116 unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
117 unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
118 /*
119  * Make sure vm_committed_as in one cacheline and not cacheline shared with
120  * other variables. It can be updated by several CPUs frequently.
121  */
122 struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
123 
124 /*
125  * The global memory commitment made in the system can be a metric
126  * that can be used to drive ballooning decisions when Linux is hosted
127  * as a guest. On Hyper-V, the host implements a policy engine for dynamically
128  * balancing memory across competing virtual machines that are hosted.
129  * Several metrics drive this policy engine including the guest reported
130  * memory commitment.
131  */
132 unsigned long vm_memory_committed(void)
133 {
134 	return percpu_counter_read_positive(&vm_committed_as);
135 }
136 EXPORT_SYMBOL_GPL(vm_memory_committed);
137 
138 /*
139  * Check that a process has enough memory to allocate a new virtual
140  * mapping. 0 means there is enough memory for the allocation to
141  * succeed and -ENOMEM implies there is not.
142  *
143  * We currently support three overcommit policies, which are set via the
144  * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
145  *
146  * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
147  * Additional code 2002 Jul 20 by Robert Love.
148  *
149  * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
150  *
151  * Note this is a helper function intended to be used by LSMs which
152  * wish to use this logic.
153  */
154 int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
155 {
156 	long free, allowed, reserve;
157 
158 	VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
159 			-(s64)vm_committed_as_batch * num_online_cpus(),
160 			"memory commitment underflow");
161 
162 	vm_acct_memory(pages);
163 
164 	/*
165 	 * Sometimes we want to use more memory than we have
166 	 */
167 	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
168 		return 0;
169 
170 	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
171 		free = global_page_state(NR_FREE_PAGES);
172 		free += global_page_state(NR_FILE_PAGES);
173 
174 		/*
175 		 * shmem pages shouldn't be counted as free in this
176 		 * case, they can't be purged, only swapped out, and
177 		 * that won't affect the overall amount of available
178 		 * memory in the system.
179 		 */
180 		free -= global_page_state(NR_SHMEM);
181 
182 		free += get_nr_swap_pages();
183 
184 		/*
185 		 * Any slabs which are created with the
186 		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
187 		 * which are reclaimable, under pressure.  The dentry
188 		 * cache and most inode caches should fall into this
189 		 */
190 		free += global_page_state(NR_SLAB_RECLAIMABLE);
191 
192 		/*
193 		 * Leave reserved pages. The pages are not for anonymous pages.
194 		 */
195 		if (free <= totalreserve_pages)
196 			goto error;
197 		else
198 			free -= totalreserve_pages;
199 
200 		/*
201 		 * Reserve some for root
202 		 */
203 		if (!cap_sys_admin)
204 			free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
205 
206 		if (free > pages)
207 			return 0;
208 
209 		goto error;
210 	}
211 
212 	allowed = vm_commit_limit();
213 	/*
214 	 * Reserve some for root
215 	 */
216 	if (!cap_sys_admin)
217 		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
218 
219 	/*
220 	 * Don't let a single process grow so big a user can't recover
221 	 */
222 	if (mm) {
223 		reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
224 		allowed -= min_t(long, mm->total_vm / 32, reserve);
225 	}
226 
227 	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
228 		return 0;
229 error:
230 	vm_unacct_memory(pages);
231 
232 	return -ENOMEM;
233 }
234 
235 /*
236  * Requires inode->i_mapping->i_mmap_rwsem
237  */
238 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
239 		struct file *file, struct address_space *mapping)
240 {
241 	if (vma->vm_flags & VM_DENYWRITE)
242 		atomic_inc(&file_inode(file)->i_writecount);
243 	if (vma->vm_flags & VM_SHARED)
244 		mapping_unmap_writable(mapping);
245 
246 	flush_dcache_mmap_lock(mapping);
247 	vma_interval_tree_remove(vma, &mapping->i_mmap);
248 	flush_dcache_mmap_unlock(mapping);
249 }
250 
251 /*
252  * Unlink a file-based vm structure from its interval tree, to hide
253  * vma from rmap and vmtruncate before freeing its page tables.
254  */
255 void unlink_file_vma(struct vm_area_struct *vma)
256 {
257 	struct file *file = vma->vm_file;
258 
259 	if (file) {
260 		struct address_space *mapping = file->f_mapping;
261 		i_mmap_lock_write(mapping);
262 		__remove_shared_vm_struct(vma, file, mapping);
263 		i_mmap_unlock_write(mapping);
264 	}
265 }
266 
267 /*
268  * Close a vm structure and free it, returning the next.
269  */
270 static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
271 {
272 	struct vm_area_struct *next = vma->vm_next;
273 
274 	might_sleep();
275 	if (vma->vm_ops && vma->vm_ops->close)
276 		vma->vm_ops->close(vma);
277 	if (vma->vm_file)
278 		fput(vma->vm_file);
279 	mpol_put(vma_policy(vma));
280 	kmem_cache_free(vm_area_cachep, vma);
281 	return next;
282 }
283 
284 static unsigned long do_brk(unsigned long addr, unsigned long len);
285 
286 SYSCALL_DEFINE1(brk, unsigned long, brk)
287 {
288 	unsigned long retval;
289 	unsigned long newbrk, oldbrk;
290 	struct mm_struct *mm = current->mm;
291 	unsigned long min_brk;
292 	bool populate;
293 
294 	down_write(&mm->mmap_sem);
295 
296 #ifdef CONFIG_COMPAT_BRK
297 	/*
298 	 * CONFIG_COMPAT_BRK can still be overridden by setting
299 	 * randomize_va_space to 2, which will still cause mm->start_brk
300 	 * to be arbitrarily shifted
301 	 */
302 	if (current->brk_randomized)
303 		min_brk = mm->start_brk;
304 	else
305 		min_brk = mm->end_data;
306 #else
307 	min_brk = mm->start_brk;
308 #endif
309 	if (brk < min_brk)
310 		goto out;
311 
312 	/*
313 	 * Check against rlimit here. If this check is done later after the test
314 	 * of oldbrk with newbrk then it can escape the test and let the data
315 	 * segment grow beyond its set limit the in case where the limit is
316 	 * not page aligned -Ram Gupta
317 	 */
318 	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
319 			      mm->end_data, mm->start_data))
320 		goto out;
321 
322 	newbrk = PAGE_ALIGN(brk);
323 	oldbrk = PAGE_ALIGN(mm->brk);
324 	if (oldbrk == newbrk)
325 		goto set_brk;
326 
327 	/* Always allow shrinking brk. */
328 	if (brk <= mm->brk) {
329 		if (!do_munmap(mm, newbrk, oldbrk-newbrk))
330 			goto set_brk;
331 		goto out;
332 	}
333 
334 	/* Check against existing mmap mappings. */
335 	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
336 		goto out;
337 
338 	/* Ok, looks good - let it rip. */
339 	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
340 		goto out;
341 
342 set_brk:
343 	mm->brk = brk;
344 	populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
345 	up_write(&mm->mmap_sem);
346 	if (populate)
347 		mm_populate(oldbrk, newbrk - oldbrk);
348 	return brk;
349 
350 out:
351 	retval = mm->brk;
352 	up_write(&mm->mmap_sem);
353 	return retval;
354 }
355 
356 static long vma_compute_subtree_gap(struct vm_area_struct *vma)
357 {
358 	unsigned long max, subtree_gap;
359 	max = vma->vm_start;
360 	if (vma->vm_prev)
361 		max -= vma->vm_prev->vm_end;
362 	if (vma->vm_rb.rb_left) {
363 		subtree_gap = rb_entry(vma->vm_rb.rb_left,
364 				struct vm_area_struct, vm_rb)->rb_subtree_gap;
365 		if (subtree_gap > max)
366 			max = subtree_gap;
367 	}
368 	if (vma->vm_rb.rb_right) {
369 		subtree_gap = rb_entry(vma->vm_rb.rb_right,
370 				struct vm_area_struct, vm_rb)->rb_subtree_gap;
371 		if (subtree_gap > max)
372 			max = subtree_gap;
373 	}
374 	return max;
375 }
376 
377 #ifdef CONFIG_DEBUG_VM_RB
378 static int browse_rb(struct rb_root *root)
379 {
380 	int i = 0, j, bug = 0;
381 	struct rb_node *nd, *pn = NULL;
382 	unsigned long prev = 0, pend = 0;
383 
384 	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
385 		struct vm_area_struct *vma;
386 		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
387 		if (vma->vm_start < prev) {
388 			pr_emerg("vm_start %lx < prev %lx\n",
389 				  vma->vm_start, prev);
390 			bug = 1;
391 		}
392 		if (vma->vm_start < pend) {
393 			pr_emerg("vm_start %lx < pend %lx\n",
394 				  vma->vm_start, pend);
395 			bug = 1;
396 		}
397 		if (vma->vm_start > vma->vm_end) {
398 			pr_emerg("vm_start %lx > vm_end %lx\n",
399 				  vma->vm_start, vma->vm_end);
400 			bug = 1;
401 		}
402 		if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
403 			pr_emerg("free gap %lx, correct %lx\n",
404 			       vma->rb_subtree_gap,
405 			       vma_compute_subtree_gap(vma));
406 			bug = 1;
407 		}
408 		i++;
409 		pn = nd;
410 		prev = vma->vm_start;
411 		pend = vma->vm_end;
412 	}
413 	j = 0;
414 	for (nd = pn; nd; nd = rb_prev(nd))
415 		j++;
416 	if (i != j) {
417 		pr_emerg("backwards %d, forwards %d\n", j, i);
418 		bug = 1;
419 	}
420 	return bug ? -1 : i;
421 }
422 
423 static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
424 {
425 	struct rb_node *nd;
426 
427 	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
428 		struct vm_area_struct *vma;
429 		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
430 		VM_BUG_ON_VMA(vma != ignore &&
431 			vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
432 			vma);
433 	}
434 }
435 
436 static void validate_mm(struct mm_struct *mm)
437 {
438 	int bug = 0;
439 	int i = 0;
440 	unsigned long highest_address = 0;
441 	struct vm_area_struct *vma = mm->mmap;
442 
443 	while (vma) {
444 		struct anon_vma_chain *avc;
445 
446 		vma_lock_anon_vma(vma);
447 		list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
448 			anon_vma_interval_tree_verify(avc);
449 		vma_unlock_anon_vma(vma);
450 		highest_address = vma->vm_end;
451 		vma = vma->vm_next;
452 		i++;
453 	}
454 	if (i != mm->map_count) {
455 		pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
456 		bug = 1;
457 	}
458 	if (highest_address != mm->highest_vm_end) {
459 		pr_emerg("mm->highest_vm_end %lx, found %lx\n",
460 			  mm->highest_vm_end, highest_address);
461 		bug = 1;
462 	}
463 	i = browse_rb(&mm->mm_rb);
464 	if (i != mm->map_count) {
465 		if (i != -1)
466 			pr_emerg("map_count %d rb %d\n", mm->map_count, i);
467 		bug = 1;
468 	}
469 	VM_BUG_ON_MM(bug, mm);
470 }
471 #else
472 #define validate_mm_rb(root, ignore) do { } while (0)
473 #define validate_mm(mm) do { } while (0)
474 #endif
475 
476 RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
477 		     unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
478 
479 /*
480  * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
481  * vma->vm_prev->vm_end values changed, without modifying the vma's position
482  * in the rbtree.
483  */
484 static void vma_gap_update(struct vm_area_struct *vma)
485 {
486 	/*
487 	 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
488 	 * function that does exacltly what we want.
489 	 */
490 	vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
491 }
492 
493 static inline void vma_rb_insert(struct vm_area_struct *vma,
494 				 struct rb_root *root)
495 {
496 	/* All rb_subtree_gap values must be consistent prior to insertion */
497 	validate_mm_rb(root, NULL);
498 
499 	rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
500 }
501 
502 static void vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
503 {
504 	/*
505 	 * All rb_subtree_gap values must be consistent prior to erase,
506 	 * with the possible exception of the vma being erased.
507 	 */
508 	validate_mm_rb(root, vma);
509 
510 	/*
511 	 * Note rb_erase_augmented is a fairly large inline function,
512 	 * so make sure we instantiate it only once with our desired
513 	 * augmented rbtree callbacks.
514 	 */
515 	rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
516 }
517 
518 /*
519  * vma has some anon_vma assigned, and is already inserted on that
520  * anon_vma's interval trees.
521  *
522  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
523  * vma must be removed from the anon_vma's interval trees using
524  * anon_vma_interval_tree_pre_update_vma().
525  *
526  * After the update, the vma will be reinserted using
527  * anon_vma_interval_tree_post_update_vma().
528  *
529  * The entire update must be protected by exclusive mmap_sem and by
530  * the root anon_vma's mutex.
531  */
532 static inline void
533 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
534 {
535 	struct anon_vma_chain *avc;
536 
537 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
538 		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
539 }
540 
541 static inline void
542 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
543 {
544 	struct anon_vma_chain *avc;
545 
546 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
547 		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
548 }
549 
550 static int find_vma_links(struct mm_struct *mm, unsigned long addr,
551 		unsigned long end, struct vm_area_struct **pprev,
552 		struct rb_node ***rb_link, struct rb_node **rb_parent)
553 {
554 	struct rb_node **__rb_link, *__rb_parent, *rb_prev;
555 
556 	__rb_link = &mm->mm_rb.rb_node;
557 	rb_prev = __rb_parent = NULL;
558 
559 	while (*__rb_link) {
560 		struct vm_area_struct *vma_tmp;
561 
562 		__rb_parent = *__rb_link;
563 		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
564 
565 		if (vma_tmp->vm_end > addr) {
566 			/* Fail if an existing vma overlaps the area */
567 			if (vma_tmp->vm_start < end)
568 				return -ENOMEM;
569 			__rb_link = &__rb_parent->rb_left;
570 		} else {
571 			rb_prev = __rb_parent;
572 			__rb_link = &__rb_parent->rb_right;
573 		}
574 	}
575 
576 	*pprev = NULL;
577 	if (rb_prev)
578 		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
579 	*rb_link = __rb_link;
580 	*rb_parent = __rb_parent;
581 	return 0;
582 }
583 
584 static unsigned long count_vma_pages_range(struct mm_struct *mm,
585 		unsigned long addr, unsigned long end)
586 {
587 	unsigned long nr_pages = 0;
588 	struct vm_area_struct *vma;
589 
590 	/* Find first overlaping mapping */
591 	vma = find_vma_intersection(mm, addr, end);
592 	if (!vma)
593 		return 0;
594 
595 	nr_pages = (min(end, vma->vm_end) -
596 		max(addr, vma->vm_start)) >> PAGE_SHIFT;
597 
598 	/* Iterate over the rest of the overlaps */
599 	for (vma = vma->vm_next; vma; vma = vma->vm_next) {
600 		unsigned long overlap_len;
601 
602 		if (vma->vm_start > end)
603 			break;
604 
605 		overlap_len = min(end, vma->vm_end) - vma->vm_start;
606 		nr_pages += overlap_len >> PAGE_SHIFT;
607 	}
608 
609 	return nr_pages;
610 }
611 
612 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
613 		struct rb_node **rb_link, struct rb_node *rb_parent)
614 {
615 	/* Update tracking information for the gap following the new vma. */
616 	if (vma->vm_next)
617 		vma_gap_update(vma->vm_next);
618 	else
619 		mm->highest_vm_end = vma->vm_end;
620 
621 	/*
622 	 * vma->vm_prev wasn't known when we followed the rbtree to find the
623 	 * correct insertion point for that vma. As a result, we could not
624 	 * update the vma vm_rb parents rb_subtree_gap values on the way down.
625 	 * So, we first insert the vma with a zero rb_subtree_gap value
626 	 * (to be consistent with what we did on the way down), and then
627 	 * immediately update the gap to the correct value. Finally we
628 	 * rebalance the rbtree after all augmented values have been set.
629 	 */
630 	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
631 	vma->rb_subtree_gap = 0;
632 	vma_gap_update(vma);
633 	vma_rb_insert(vma, &mm->mm_rb);
634 }
635 
636 static void __vma_link_file(struct vm_area_struct *vma)
637 {
638 	struct file *file;
639 
640 	file = vma->vm_file;
641 	if (file) {
642 		struct address_space *mapping = file->f_mapping;
643 
644 		if (vma->vm_flags & VM_DENYWRITE)
645 			atomic_dec(&file_inode(file)->i_writecount);
646 		if (vma->vm_flags & VM_SHARED)
647 			atomic_inc(&mapping->i_mmap_writable);
648 
649 		flush_dcache_mmap_lock(mapping);
650 		vma_interval_tree_insert(vma, &mapping->i_mmap);
651 		flush_dcache_mmap_unlock(mapping);
652 	}
653 }
654 
655 static void
656 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
657 	struct vm_area_struct *prev, struct rb_node **rb_link,
658 	struct rb_node *rb_parent)
659 {
660 	__vma_link_list(mm, vma, prev, rb_parent);
661 	__vma_link_rb(mm, vma, rb_link, rb_parent);
662 }
663 
664 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
665 			struct vm_area_struct *prev, struct rb_node **rb_link,
666 			struct rb_node *rb_parent)
667 {
668 	struct address_space *mapping = NULL;
669 
670 	if (vma->vm_file) {
671 		mapping = vma->vm_file->f_mapping;
672 		i_mmap_lock_write(mapping);
673 	}
674 
675 	__vma_link(mm, vma, prev, rb_link, rb_parent);
676 	__vma_link_file(vma);
677 
678 	if (mapping)
679 		i_mmap_unlock_write(mapping);
680 
681 	mm->map_count++;
682 	validate_mm(mm);
683 }
684 
685 /*
686  * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
687  * mm's list and rbtree.  It has already been inserted into the interval tree.
688  */
689 static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
690 {
691 	struct vm_area_struct *prev;
692 	struct rb_node **rb_link, *rb_parent;
693 
694 	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
695 			   &prev, &rb_link, &rb_parent))
696 		BUG();
697 	__vma_link(mm, vma, prev, rb_link, rb_parent);
698 	mm->map_count++;
699 }
700 
701 static inline void
702 __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
703 		struct vm_area_struct *prev)
704 {
705 	struct vm_area_struct *next;
706 
707 	vma_rb_erase(vma, &mm->mm_rb);
708 	prev->vm_next = next = vma->vm_next;
709 	if (next)
710 		next->vm_prev = prev;
711 
712 	/* Kill the cache */
713 	vmacache_invalidate(mm);
714 }
715 
716 /*
717  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
718  * is already present in an i_mmap tree without adjusting the tree.
719  * The following helper function should be used when such adjustments
720  * are necessary.  The "insert" vma (if any) is to be inserted
721  * before we drop the necessary locks.
722  */
723 int vma_adjust(struct vm_area_struct *vma, unsigned long start,
724 	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
725 {
726 	struct mm_struct *mm = vma->vm_mm;
727 	struct vm_area_struct *next = vma->vm_next;
728 	struct vm_area_struct *importer = NULL;
729 	struct address_space *mapping = NULL;
730 	struct rb_root *root = NULL;
731 	struct anon_vma *anon_vma = NULL;
732 	struct file *file = vma->vm_file;
733 	bool start_changed = false, end_changed = false;
734 	long adjust_next = 0;
735 	int remove_next = 0;
736 
737 	if (next && !insert) {
738 		struct vm_area_struct *exporter = NULL;
739 
740 		if (end >= next->vm_end) {
741 			/*
742 			 * vma expands, overlapping all the next, and
743 			 * perhaps the one after too (mprotect case 6).
744 			 */
745 again:			remove_next = 1 + (end > next->vm_end);
746 			end = next->vm_end;
747 			exporter = next;
748 			importer = vma;
749 		} else if (end > next->vm_start) {
750 			/*
751 			 * vma expands, overlapping part of the next:
752 			 * mprotect case 5 shifting the boundary up.
753 			 */
754 			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
755 			exporter = next;
756 			importer = vma;
757 		} else if (end < vma->vm_end) {
758 			/*
759 			 * vma shrinks, and !insert tells it's not
760 			 * split_vma inserting another: so it must be
761 			 * mprotect case 4 shifting the boundary down.
762 			 */
763 			adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
764 			exporter = vma;
765 			importer = next;
766 		}
767 
768 		/*
769 		 * Easily overlooked: when mprotect shifts the boundary,
770 		 * make sure the expanding vma has anon_vma set if the
771 		 * shrinking vma had, to cover any anon pages imported.
772 		 */
773 		if (exporter && exporter->anon_vma && !importer->anon_vma) {
774 			int error;
775 
776 			importer->anon_vma = exporter->anon_vma;
777 			error = anon_vma_clone(importer, exporter);
778 			if (error)
779 				return error;
780 		}
781 	}
782 
783 	if (file) {
784 		mapping = file->f_mapping;
785 		root = &mapping->i_mmap;
786 		uprobe_munmap(vma, vma->vm_start, vma->vm_end);
787 
788 		if (adjust_next)
789 			uprobe_munmap(next, next->vm_start, next->vm_end);
790 
791 		i_mmap_lock_write(mapping);
792 		if (insert) {
793 			/*
794 			 * Put into interval tree now, so instantiated pages
795 			 * are visible to arm/parisc __flush_dcache_page
796 			 * throughout; but we cannot insert into address
797 			 * space until vma start or end is updated.
798 			 */
799 			__vma_link_file(insert);
800 		}
801 	}
802 
803 	vma_adjust_trans_huge(vma, start, end, adjust_next);
804 
805 	anon_vma = vma->anon_vma;
806 	if (!anon_vma && adjust_next)
807 		anon_vma = next->anon_vma;
808 	if (anon_vma) {
809 		VM_BUG_ON_VMA(adjust_next && next->anon_vma &&
810 			  anon_vma != next->anon_vma, next);
811 		anon_vma_lock_write(anon_vma);
812 		anon_vma_interval_tree_pre_update_vma(vma);
813 		if (adjust_next)
814 			anon_vma_interval_tree_pre_update_vma(next);
815 	}
816 
817 	if (root) {
818 		flush_dcache_mmap_lock(mapping);
819 		vma_interval_tree_remove(vma, root);
820 		if (adjust_next)
821 			vma_interval_tree_remove(next, root);
822 	}
823 
824 	if (start != vma->vm_start) {
825 		vma->vm_start = start;
826 		start_changed = true;
827 	}
828 	if (end != vma->vm_end) {
829 		vma->vm_end = end;
830 		end_changed = true;
831 	}
832 	vma->vm_pgoff = pgoff;
833 	if (adjust_next) {
834 		next->vm_start += adjust_next << PAGE_SHIFT;
835 		next->vm_pgoff += adjust_next;
836 	}
837 
838 	if (root) {
839 		if (adjust_next)
840 			vma_interval_tree_insert(next, root);
841 		vma_interval_tree_insert(vma, root);
842 		flush_dcache_mmap_unlock(mapping);
843 	}
844 
845 	if (remove_next) {
846 		/*
847 		 * vma_merge has merged next into vma, and needs
848 		 * us to remove next before dropping the locks.
849 		 */
850 		__vma_unlink(mm, next, vma);
851 		if (file)
852 			__remove_shared_vm_struct(next, file, mapping);
853 	} else if (insert) {
854 		/*
855 		 * split_vma has split insert from vma, and needs
856 		 * us to insert it before dropping the locks
857 		 * (it may either follow vma or precede it).
858 		 */
859 		__insert_vm_struct(mm, insert);
860 	} else {
861 		if (start_changed)
862 			vma_gap_update(vma);
863 		if (end_changed) {
864 			if (!next)
865 				mm->highest_vm_end = end;
866 			else if (!adjust_next)
867 				vma_gap_update(next);
868 		}
869 	}
870 
871 	if (anon_vma) {
872 		anon_vma_interval_tree_post_update_vma(vma);
873 		if (adjust_next)
874 			anon_vma_interval_tree_post_update_vma(next);
875 		anon_vma_unlock_write(anon_vma);
876 	}
877 	if (mapping)
878 		i_mmap_unlock_write(mapping);
879 
880 	if (root) {
881 		uprobe_mmap(vma);
882 
883 		if (adjust_next)
884 			uprobe_mmap(next);
885 	}
886 
887 	if (remove_next) {
888 		if (file) {
889 			uprobe_munmap(next, next->vm_start, next->vm_end);
890 			fput(file);
891 		}
892 		if (next->anon_vma)
893 			anon_vma_merge(vma, next);
894 		mm->map_count--;
895 		mpol_put(vma_policy(next));
896 		kmem_cache_free(vm_area_cachep, next);
897 		/*
898 		 * In mprotect's case 6 (see comments on vma_merge),
899 		 * we must remove another next too. It would clutter
900 		 * up the code too much to do both in one go.
901 		 */
902 		next = vma->vm_next;
903 		if (remove_next == 2)
904 			goto again;
905 		else if (next)
906 			vma_gap_update(next);
907 		else
908 			mm->highest_vm_end = end;
909 	}
910 	if (insert && file)
911 		uprobe_mmap(insert);
912 
913 	validate_mm(mm);
914 
915 	return 0;
916 }
917 
918 /*
919  * If the vma has a ->close operation then the driver probably needs to release
920  * per-vma resources, so we don't attempt to merge those.
921  */
922 static inline int is_mergeable_vma(struct vm_area_struct *vma,
923 				struct file *file, unsigned long vm_flags,
924 				struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
925 {
926 	/*
927 	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
928 	 * match the flags but dirty bit -- the caller should mark
929 	 * merged VMA as dirty. If dirty bit won't be excluded from
930 	 * comparison, we increase pressue on the memory system forcing
931 	 * the kernel to generate new VMAs when old one could be
932 	 * extended instead.
933 	 */
934 	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
935 		return 0;
936 	if (vma->vm_file != file)
937 		return 0;
938 	if (vma->vm_ops && vma->vm_ops->close)
939 		return 0;
940 	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
941 		return 0;
942 	return 1;
943 }
944 
945 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
946 					struct anon_vma *anon_vma2,
947 					struct vm_area_struct *vma)
948 {
949 	/*
950 	 * The list_is_singular() test is to avoid merging VMA cloned from
951 	 * parents. This can improve scalability caused by anon_vma lock.
952 	 */
953 	if ((!anon_vma1 || !anon_vma2) && (!vma ||
954 		list_is_singular(&vma->anon_vma_chain)))
955 		return 1;
956 	return anon_vma1 == anon_vma2;
957 }
958 
959 /*
960  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
961  * in front of (at a lower virtual address and file offset than) the vma.
962  *
963  * We cannot merge two vmas if they have differently assigned (non-NULL)
964  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
965  *
966  * We don't check here for the merged mmap wrapping around the end of pagecache
967  * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
968  * wrap, nor mmaps which cover the final page at index -1UL.
969  */
970 static int
971 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
972 		     struct anon_vma *anon_vma, struct file *file,
973 		     pgoff_t vm_pgoff,
974 		     struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
975 {
976 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
977 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
978 		if (vma->vm_pgoff == vm_pgoff)
979 			return 1;
980 	}
981 	return 0;
982 }
983 
984 /*
985  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
986  * beyond (at a higher virtual address and file offset than) the vma.
987  *
988  * We cannot merge two vmas if they have differently assigned (non-NULL)
989  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
990  */
991 static int
992 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
993 		    struct anon_vma *anon_vma, struct file *file,
994 		    pgoff_t vm_pgoff,
995 		    struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
996 {
997 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
998 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
999 		pgoff_t vm_pglen;
1000 		vm_pglen = vma_pages(vma);
1001 		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
1002 			return 1;
1003 	}
1004 	return 0;
1005 }
1006 
1007 /*
1008  * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
1009  * whether that can be merged with its predecessor or its successor.
1010  * Or both (it neatly fills a hole).
1011  *
1012  * In most cases - when called for mmap, brk or mremap - [addr,end) is
1013  * certain not to be mapped by the time vma_merge is called; but when
1014  * called for mprotect, it is certain to be already mapped (either at
1015  * an offset within prev, or at the start of next), and the flags of
1016  * this area are about to be changed to vm_flags - and the no-change
1017  * case has already been eliminated.
1018  *
1019  * The following mprotect cases have to be considered, where AAAA is
1020  * the area passed down from mprotect_fixup, never extending beyond one
1021  * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
1022  *
1023  *     AAAA             AAAA                AAAA          AAAA
1024  *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
1025  *    cannot merge    might become    might become    might become
1026  *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
1027  *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
1028  *    mremap move:                                    PPPPNNNNNNNN 8
1029  *        AAAA
1030  *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
1031  *    might become    case 1 below    case 2 below    case 3 below
1032  *
1033  * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
1034  * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
1035  */
1036 struct vm_area_struct *vma_merge(struct mm_struct *mm,
1037 			struct vm_area_struct *prev, unsigned long addr,
1038 			unsigned long end, unsigned long vm_flags,
1039 			struct anon_vma *anon_vma, struct file *file,
1040 			pgoff_t pgoff, struct mempolicy *policy,
1041 			struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
1042 {
1043 	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1044 	struct vm_area_struct *area, *next;
1045 	int err;
1046 
1047 	/*
1048 	 * We later require that vma->vm_flags == vm_flags,
1049 	 * so this tests vma->vm_flags & VM_SPECIAL, too.
1050 	 */
1051 	if (vm_flags & VM_SPECIAL)
1052 		return NULL;
1053 
1054 	if (prev)
1055 		next = prev->vm_next;
1056 	else
1057 		next = mm->mmap;
1058 	area = next;
1059 	if (next && next->vm_end == end)		/* cases 6, 7, 8 */
1060 		next = next->vm_next;
1061 
1062 	/*
1063 	 * Can it merge with the predecessor?
1064 	 */
1065 	if (prev && prev->vm_end == addr &&
1066 			mpol_equal(vma_policy(prev), policy) &&
1067 			can_vma_merge_after(prev, vm_flags,
1068 					    anon_vma, file, pgoff,
1069 					    vm_userfaultfd_ctx)) {
1070 		/*
1071 		 * OK, it can.  Can we now merge in the successor as well?
1072 		 */
1073 		if (next && end == next->vm_start &&
1074 				mpol_equal(policy, vma_policy(next)) &&
1075 				can_vma_merge_before(next, vm_flags,
1076 						     anon_vma, file,
1077 						     pgoff+pglen,
1078 						     vm_userfaultfd_ctx) &&
1079 				is_mergeable_anon_vma(prev->anon_vma,
1080 						      next->anon_vma, NULL)) {
1081 							/* cases 1, 6 */
1082 			err = vma_adjust(prev, prev->vm_start,
1083 				next->vm_end, prev->vm_pgoff, NULL);
1084 		} else					/* cases 2, 5, 7 */
1085 			err = vma_adjust(prev, prev->vm_start,
1086 				end, prev->vm_pgoff, NULL);
1087 		if (err)
1088 			return NULL;
1089 		khugepaged_enter_vma_merge(prev, vm_flags);
1090 		return prev;
1091 	}
1092 
1093 	/*
1094 	 * Can this new request be merged in front of next?
1095 	 */
1096 	if (next && end == next->vm_start &&
1097 			mpol_equal(policy, vma_policy(next)) &&
1098 			can_vma_merge_before(next, vm_flags,
1099 					     anon_vma, file, pgoff+pglen,
1100 					     vm_userfaultfd_ctx)) {
1101 		if (prev && addr < prev->vm_end)	/* case 4 */
1102 			err = vma_adjust(prev, prev->vm_start,
1103 				addr, prev->vm_pgoff, NULL);
1104 		else					/* cases 3, 8 */
1105 			err = vma_adjust(area, addr, next->vm_end,
1106 				next->vm_pgoff - pglen, NULL);
1107 		if (err)
1108 			return NULL;
1109 		khugepaged_enter_vma_merge(area, vm_flags);
1110 		return area;
1111 	}
1112 
1113 	return NULL;
1114 }
1115 
1116 /*
1117  * Rough compatbility check to quickly see if it's even worth looking
1118  * at sharing an anon_vma.
1119  *
1120  * They need to have the same vm_file, and the flags can only differ
1121  * in things that mprotect may change.
1122  *
1123  * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1124  * we can merge the two vma's. For example, we refuse to merge a vma if
1125  * there is a vm_ops->close() function, because that indicates that the
1126  * driver is doing some kind of reference counting. But that doesn't
1127  * really matter for the anon_vma sharing case.
1128  */
1129 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1130 {
1131 	return a->vm_end == b->vm_start &&
1132 		mpol_equal(vma_policy(a), vma_policy(b)) &&
1133 		a->vm_file == b->vm_file &&
1134 		!((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
1135 		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1136 }
1137 
1138 /*
1139  * Do some basic sanity checking to see if we can re-use the anon_vma
1140  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1141  * the same as 'old', the other will be the new one that is trying
1142  * to share the anon_vma.
1143  *
1144  * NOTE! This runs with mm_sem held for reading, so it is possible that
1145  * the anon_vma of 'old' is concurrently in the process of being set up
1146  * by another page fault trying to merge _that_. But that's ok: if it
1147  * is being set up, that automatically means that it will be a singleton
1148  * acceptable for merging, so we can do all of this optimistically. But
1149  * we do that READ_ONCE() to make sure that we never re-load the pointer.
1150  *
1151  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1152  * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1153  * is to return an anon_vma that is "complex" due to having gone through
1154  * a fork).
1155  *
1156  * We also make sure that the two vma's are compatible (adjacent,
1157  * and with the same memory policies). That's all stable, even with just
1158  * a read lock on the mm_sem.
1159  */
1160 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1161 {
1162 	if (anon_vma_compatible(a, b)) {
1163 		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1164 
1165 		if (anon_vma && list_is_singular(&old->anon_vma_chain))
1166 			return anon_vma;
1167 	}
1168 	return NULL;
1169 }
1170 
1171 /*
1172  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1173  * neighbouring vmas for a suitable anon_vma, before it goes off
1174  * to allocate a new anon_vma.  It checks because a repetitive
1175  * sequence of mprotects and faults may otherwise lead to distinct
1176  * anon_vmas being allocated, preventing vma merge in subsequent
1177  * mprotect.
1178  */
1179 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1180 {
1181 	struct anon_vma *anon_vma;
1182 	struct vm_area_struct *near;
1183 
1184 	near = vma->vm_next;
1185 	if (!near)
1186 		goto try_prev;
1187 
1188 	anon_vma = reusable_anon_vma(near, vma, near);
1189 	if (anon_vma)
1190 		return anon_vma;
1191 try_prev:
1192 	near = vma->vm_prev;
1193 	if (!near)
1194 		goto none;
1195 
1196 	anon_vma = reusable_anon_vma(near, near, vma);
1197 	if (anon_vma)
1198 		return anon_vma;
1199 none:
1200 	/*
1201 	 * There's no absolute need to look only at touching neighbours:
1202 	 * we could search further afield for "compatible" anon_vmas.
1203 	 * But it would probably just be a waste of time searching,
1204 	 * or lead to too many vmas hanging off the same anon_vma.
1205 	 * We're trying to allow mprotect remerging later on,
1206 	 * not trying to minimize memory used for anon_vmas.
1207 	 */
1208 	return NULL;
1209 }
1210 
1211 #ifdef CONFIG_PROC_FS
1212 void vm_stat_account(struct mm_struct *mm, unsigned long flags,
1213 						struct file *file, long pages)
1214 {
1215 	const unsigned long stack_flags
1216 		= VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
1217 
1218 	mm->total_vm += pages;
1219 
1220 	if (file) {
1221 		mm->shared_vm += pages;
1222 		if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
1223 			mm->exec_vm += pages;
1224 	} else if (flags & stack_flags)
1225 		mm->stack_vm += pages;
1226 }
1227 #endif /* CONFIG_PROC_FS */
1228 
1229 /*
1230  * If a hint addr is less than mmap_min_addr change hint to be as
1231  * low as possible but still greater than mmap_min_addr
1232  */
1233 static inline unsigned long round_hint_to_min(unsigned long hint)
1234 {
1235 	hint &= PAGE_MASK;
1236 	if (((void *)hint != NULL) &&
1237 	    (hint < mmap_min_addr))
1238 		return PAGE_ALIGN(mmap_min_addr);
1239 	return hint;
1240 }
1241 
1242 static inline int mlock_future_check(struct mm_struct *mm,
1243 				     unsigned long flags,
1244 				     unsigned long len)
1245 {
1246 	unsigned long locked, lock_limit;
1247 
1248 	/*  mlock MCL_FUTURE? */
1249 	if (flags & VM_LOCKED) {
1250 		locked = len >> PAGE_SHIFT;
1251 		locked += mm->locked_vm;
1252 		lock_limit = rlimit(RLIMIT_MEMLOCK);
1253 		lock_limit >>= PAGE_SHIFT;
1254 		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1255 			return -EAGAIN;
1256 	}
1257 	return 0;
1258 }
1259 
1260 /*
1261  * The caller must hold down_write(&current->mm->mmap_sem).
1262  */
1263 unsigned long do_mmap(struct file *file, unsigned long addr,
1264 			unsigned long len, unsigned long prot,
1265 			unsigned long flags, vm_flags_t vm_flags,
1266 			unsigned long pgoff, unsigned long *populate)
1267 {
1268 	struct mm_struct *mm = current->mm;
1269 
1270 	*populate = 0;
1271 
1272 	if (!len)
1273 		return -EINVAL;
1274 
1275 	/*
1276 	 * Does the application expect PROT_READ to imply PROT_EXEC?
1277 	 *
1278 	 * (the exception is when the underlying filesystem is noexec
1279 	 *  mounted, in which case we dont add PROT_EXEC.)
1280 	 */
1281 	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1282 		if (!(file && path_noexec(&file->f_path)))
1283 			prot |= PROT_EXEC;
1284 
1285 	if (!(flags & MAP_FIXED))
1286 		addr = round_hint_to_min(addr);
1287 
1288 	/* Careful about overflows.. */
1289 	len = PAGE_ALIGN(len);
1290 	if (!len)
1291 		return -ENOMEM;
1292 
1293 	/* offset overflow? */
1294 	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1295 		return -EOVERFLOW;
1296 
1297 	/* Too many mappings? */
1298 	if (mm->map_count > sysctl_max_map_count)
1299 		return -ENOMEM;
1300 
1301 	/* Obtain the address to map to. we verify (or select) it and ensure
1302 	 * that it represents a valid section of the address space.
1303 	 */
1304 	addr = get_unmapped_area(file, addr, len, pgoff, flags);
1305 	if (offset_in_page(addr))
1306 		return addr;
1307 
1308 	/* Do simple checking here so the lower-level routines won't have
1309 	 * to. we assume access permissions have been handled by the open
1310 	 * of the memory object, so we don't do any here.
1311 	 */
1312 	vm_flags |= calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
1313 			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1314 
1315 	if (flags & MAP_LOCKED)
1316 		if (!can_do_mlock())
1317 			return -EPERM;
1318 
1319 	if (mlock_future_check(mm, vm_flags, len))
1320 		return -EAGAIN;
1321 
1322 	if (file) {
1323 		struct inode *inode = file_inode(file);
1324 
1325 		switch (flags & MAP_TYPE) {
1326 		case MAP_SHARED:
1327 			if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1328 				return -EACCES;
1329 
1330 			/*
1331 			 * Make sure we don't allow writing to an append-only
1332 			 * file..
1333 			 */
1334 			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1335 				return -EACCES;
1336 
1337 			/*
1338 			 * Make sure there are no mandatory locks on the file.
1339 			 */
1340 			if (locks_verify_locked(file))
1341 				return -EAGAIN;
1342 
1343 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1344 			if (!(file->f_mode & FMODE_WRITE))
1345 				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1346 
1347 			/* fall through */
1348 		case MAP_PRIVATE:
1349 			if (!(file->f_mode & FMODE_READ))
1350 				return -EACCES;
1351 			if (path_noexec(&file->f_path)) {
1352 				if (vm_flags & VM_EXEC)
1353 					return -EPERM;
1354 				vm_flags &= ~VM_MAYEXEC;
1355 			}
1356 
1357 			if (!file->f_op->mmap)
1358 				return -ENODEV;
1359 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1360 				return -EINVAL;
1361 			break;
1362 
1363 		default:
1364 			return -EINVAL;
1365 		}
1366 	} else {
1367 		switch (flags & MAP_TYPE) {
1368 		case MAP_SHARED:
1369 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1370 				return -EINVAL;
1371 			/*
1372 			 * Ignore pgoff.
1373 			 */
1374 			pgoff = 0;
1375 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1376 			break;
1377 		case MAP_PRIVATE:
1378 			/*
1379 			 * Set pgoff according to addr for anon_vma.
1380 			 */
1381 			pgoff = addr >> PAGE_SHIFT;
1382 			break;
1383 		default:
1384 			return -EINVAL;
1385 		}
1386 	}
1387 
1388 	/*
1389 	 * Set 'VM_NORESERVE' if we should not account for the
1390 	 * memory use of this mapping.
1391 	 */
1392 	if (flags & MAP_NORESERVE) {
1393 		/* We honor MAP_NORESERVE if allowed to overcommit */
1394 		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1395 			vm_flags |= VM_NORESERVE;
1396 
1397 		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
1398 		if (file && is_file_hugepages(file))
1399 			vm_flags |= VM_NORESERVE;
1400 	}
1401 
1402 	addr = mmap_region(file, addr, len, vm_flags, pgoff);
1403 	if (!IS_ERR_VALUE(addr) &&
1404 	    ((vm_flags & VM_LOCKED) ||
1405 	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1406 		*populate = len;
1407 	return addr;
1408 }
1409 
1410 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1411 		unsigned long, prot, unsigned long, flags,
1412 		unsigned long, fd, unsigned long, pgoff)
1413 {
1414 	struct file *file = NULL;
1415 	unsigned long retval;
1416 
1417 	if (!(flags & MAP_ANONYMOUS)) {
1418 		audit_mmap_fd(fd, flags);
1419 		file = fget(fd);
1420 		if (!file)
1421 			return -EBADF;
1422 		if (is_file_hugepages(file))
1423 			len = ALIGN(len, huge_page_size(hstate_file(file)));
1424 		retval = -EINVAL;
1425 		if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
1426 			goto out_fput;
1427 	} else if (flags & MAP_HUGETLB) {
1428 		struct user_struct *user = NULL;
1429 		struct hstate *hs;
1430 
1431 		hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
1432 		if (!hs)
1433 			return -EINVAL;
1434 
1435 		len = ALIGN(len, huge_page_size(hs));
1436 		/*
1437 		 * VM_NORESERVE is used because the reservations will be
1438 		 * taken when vm_ops->mmap() is called
1439 		 * A dummy user value is used because we are not locking
1440 		 * memory so no accounting is necessary
1441 		 */
1442 		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1443 				VM_NORESERVE,
1444 				&user, HUGETLB_ANONHUGE_INODE,
1445 				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1446 		if (IS_ERR(file))
1447 			return PTR_ERR(file);
1448 	}
1449 
1450 	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1451 
1452 	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1453 out_fput:
1454 	if (file)
1455 		fput(file);
1456 	return retval;
1457 }
1458 
1459 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1460 struct mmap_arg_struct {
1461 	unsigned long addr;
1462 	unsigned long len;
1463 	unsigned long prot;
1464 	unsigned long flags;
1465 	unsigned long fd;
1466 	unsigned long offset;
1467 };
1468 
1469 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1470 {
1471 	struct mmap_arg_struct a;
1472 
1473 	if (copy_from_user(&a, arg, sizeof(a)))
1474 		return -EFAULT;
1475 	if (offset_in_page(a.offset))
1476 		return -EINVAL;
1477 
1478 	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1479 			      a.offset >> PAGE_SHIFT);
1480 }
1481 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1482 
1483 /*
1484  * Some shared mappigns will want the pages marked read-only
1485  * to track write events. If so, we'll downgrade vm_page_prot
1486  * to the private version (using protection_map[] without the
1487  * VM_SHARED bit).
1488  */
1489 int vma_wants_writenotify(struct vm_area_struct *vma)
1490 {
1491 	vm_flags_t vm_flags = vma->vm_flags;
1492 	const struct vm_operations_struct *vm_ops = vma->vm_ops;
1493 
1494 	/* If it was private or non-writable, the write bit is already clear */
1495 	if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1496 		return 0;
1497 
1498 	/* The backer wishes to know when pages are first written to? */
1499 	if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
1500 		return 1;
1501 
1502 	/* The open routine did something to the protections that pgprot_modify
1503 	 * won't preserve? */
1504 	if (pgprot_val(vma->vm_page_prot) !=
1505 	    pgprot_val(vm_pgprot_modify(vma->vm_page_prot, vm_flags)))
1506 		return 0;
1507 
1508 	/* Do we need to track softdirty? */
1509 	if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
1510 		return 1;
1511 
1512 	/* Specialty mapping? */
1513 	if (vm_flags & VM_PFNMAP)
1514 		return 0;
1515 
1516 	/* Can the mapping track the dirty pages? */
1517 	return vma->vm_file && vma->vm_file->f_mapping &&
1518 		mapping_cap_account_dirty(vma->vm_file->f_mapping);
1519 }
1520 
1521 /*
1522  * We account for memory if it's a private writeable mapping,
1523  * not hugepages and VM_NORESERVE wasn't set.
1524  */
1525 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1526 {
1527 	/*
1528 	 * hugetlb has its own accounting separate from the core VM
1529 	 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1530 	 */
1531 	if (file && is_file_hugepages(file))
1532 		return 0;
1533 
1534 	return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1535 }
1536 
1537 unsigned long mmap_region(struct file *file, unsigned long addr,
1538 		unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
1539 {
1540 	struct mm_struct *mm = current->mm;
1541 	struct vm_area_struct *vma, *prev;
1542 	int error;
1543 	struct rb_node **rb_link, *rb_parent;
1544 	unsigned long charged = 0;
1545 
1546 	/* Check against address space limit. */
1547 	if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
1548 		unsigned long nr_pages;
1549 
1550 		/*
1551 		 * MAP_FIXED may remove pages of mappings that intersects with
1552 		 * requested mapping. Account for the pages it would unmap.
1553 		 */
1554 		if (!(vm_flags & MAP_FIXED))
1555 			return -ENOMEM;
1556 
1557 		nr_pages = count_vma_pages_range(mm, addr, addr + len);
1558 
1559 		if (!may_expand_vm(mm, (len >> PAGE_SHIFT) - nr_pages))
1560 			return -ENOMEM;
1561 	}
1562 
1563 	/* Clear old maps */
1564 	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
1565 			      &rb_parent)) {
1566 		if (do_munmap(mm, addr, len))
1567 			return -ENOMEM;
1568 	}
1569 
1570 	/*
1571 	 * Private writable mapping: check memory availability
1572 	 */
1573 	if (accountable_mapping(file, vm_flags)) {
1574 		charged = len >> PAGE_SHIFT;
1575 		if (security_vm_enough_memory_mm(mm, charged))
1576 			return -ENOMEM;
1577 		vm_flags |= VM_ACCOUNT;
1578 	}
1579 
1580 	/*
1581 	 * Can we just expand an old mapping?
1582 	 */
1583 	vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
1584 			NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
1585 	if (vma)
1586 		goto out;
1587 
1588 	/*
1589 	 * Determine the object being mapped and call the appropriate
1590 	 * specific mapper. the address has already been validated, but
1591 	 * not unmapped, but the maps are removed from the list.
1592 	 */
1593 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1594 	if (!vma) {
1595 		error = -ENOMEM;
1596 		goto unacct_error;
1597 	}
1598 
1599 	vma->vm_mm = mm;
1600 	vma->vm_start = addr;
1601 	vma->vm_end = addr + len;
1602 	vma->vm_flags = vm_flags;
1603 	vma->vm_page_prot = vm_get_page_prot(vm_flags);
1604 	vma->vm_pgoff = pgoff;
1605 	INIT_LIST_HEAD(&vma->anon_vma_chain);
1606 
1607 	if (file) {
1608 		if (vm_flags & VM_DENYWRITE) {
1609 			error = deny_write_access(file);
1610 			if (error)
1611 				goto free_vma;
1612 		}
1613 		if (vm_flags & VM_SHARED) {
1614 			error = mapping_map_writable(file->f_mapping);
1615 			if (error)
1616 				goto allow_write_and_free_vma;
1617 		}
1618 
1619 		/* ->mmap() can change vma->vm_file, but must guarantee that
1620 		 * vma_link() below can deny write-access if VM_DENYWRITE is set
1621 		 * and map writably if VM_SHARED is set. This usually means the
1622 		 * new file must not have been exposed to user-space, yet.
1623 		 */
1624 		vma->vm_file = get_file(file);
1625 		error = file->f_op->mmap(file, vma);
1626 		if (error)
1627 			goto unmap_and_free_vma;
1628 
1629 		/* Can addr have changed??
1630 		 *
1631 		 * Answer: Yes, several device drivers can do it in their
1632 		 *         f_op->mmap method. -DaveM
1633 		 * Bug: If addr is changed, prev, rb_link, rb_parent should
1634 		 *      be updated for vma_link()
1635 		 */
1636 		WARN_ON_ONCE(addr != vma->vm_start);
1637 
1638 		addr = vma->vm_start;
1639 		vm_flags = vma->vm_flags;
1640 	} else if (vm_flags & VM_SHARED) {
1641 		error = shmem_zero_setup(vma);
1642 		if (error)
1643 			goto free_vma;
1644 	}
1645 
1646 	vma_link(mm, vma, prev, rb_link, rb_parent);
1647 	/* Once vma denies write, undo our temporary denial count */
1648 	if (file) {
1649 		if (vm_flags & VM_SHARED)
1650 			mapping_unmap_writable(file->f_mapping);
1651 		if (vm_flags & VM_DENYWRITE)
1652 			allow_write_access(file);
1653 	}
1654 	file = vma->vm_file;
1655 out:
1656 	perf_event_mmap(vma);
1657 
1658 	vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1659 	if (vm_flags & VM_LOCKED) {
1660 		if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
1661 					vma == get_gate_vma(current->mm)))
1662 			mm->locked_vm += (len >> PAGE_SHIFT);
1663 		else
1664 			vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
1665 	}
1666 
1667 	if (file)
1668 		uprobe_mmap(vma);
1669 
1670 	/*
1671 	 * New (or expanded) vma always get soft dirty status.
1672 	 * Otherwise user-space soft-dirty page tracker won't
1673 	 * be able to distinguish situation when vma area unmapped,
1674 	 * then new mapped in-place (which must be aimed as
1675 	 * a completely new data area).
1676 	 */
1677 	vma->vm_flags |= VM_SOFTDIRTY;
1678 
1679 	vma_set_page_prot(vma);
1680 
1681 	return addr;
1682 
1683 unmap_and_free_vma:
1684 	vma->vm_file = NULL;
1685 	fput(file);
1686 
1687 	/* Undo any partial mapping done by a device driver. */
1688 	unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1689 	charged = 0;
1690 	if (vm_flags & VM_SHARED)
1691 		mapping_unmap_writable(file->f_mapping);
1692 allow_write_and_free_vma:
1693 	if (vm_flags & VM_DENYWRITE)
1694 		allow_write_access(file);
1695 free_vma:
1696 	kmem_cache_free(vm_area_cachep, vma);
1697 unacct_error:
1698 	if (charged)
1699 		vm_unacct_memory(charged);
1700 	return error;
1701 }
1702 
1703 unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1704 {
1705 	/*
1706 	 * We implement the search by looking for an rbtree node that
1707 	 * immediately follows a suitable gap. That is,
1708 	 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1709 	 * - gap_end   = vma->vm_start        >= info->low_limit  + length;
1710 	 * - gap_end - gap_start >= length
1711 	 */
1712 
1713 	struct mm_struct *mm = current->mm;
1714 	struct vm_area_struct *vma;
1715 	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1716 
1717 	/* Adjust search length to account for worst case alignment overhead */
1718 	length = info->length + info->align_mask;
1719 	if (length < info->length)
1720 		return -ENOMEM;
1721 
1722 	/* Adjust search limits by the desired length */
1723 	if (info->high_limit < length)
1724 		return -ENOMEM;
1725 	high_limit = info->high_limit - length;
1726 
1727 	if (info->low_limit > high_limit)
1728 		return -ENOMEM;
1729 	low_limit = info->low_limit + length;
1730 
1731 	/* Check if rbtree root looks promising */
1732 	if (RB_EMPTY_ROOT(&mm->mm_rb))
1733 		goto check_highest;
1734 	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1735 	if (vma->rb_subtree_gap < length)
1736 		goto check_highest;
1737 
1738 	while (true) {
1739 		/* Visit left subtree if it looks promising */
1740 		gap_end = vma->vm_start;
1741 		if (gap_end >= low_limit && vma->vm_rb.rb_left) {
1742 			struct vm_area_struct *left =
1743 				rb_entry(vma->vm_rb.rb_left,
1744 					 struct vm_area_struct, vm_rb);
1745 			if (left->rb_subtree_gap >= length) {
1746 				vma = left;
1747 				continue;
1748 			}
1749 		}
1750 
1751 		gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1752 check_current:
1753 		/* Check if current node has a suitable gap */
1754 		if (gap_start > high_limit)
1755 			return -ENOMEM;
1756 		if (gap_end >= low_limit && gap_end - gap_start >= length)
1757 			goto found;
1758 
1759 		/* Visit right subtree if it looks promising */
1760 		if (vma->vm_rb.rb_right) {
1761 			struct vm_area_struct *right =
1762 				rb_entry(vma->vm_rb.rb_right,
1763 					 struct vm_area_struct, vm_rb);
1764 			if (right->rb_subtree_gap >= length) {
1765 				vma = right;
1766 				continue;
1767 			}
1768 		}
1769 
1770 		/* Go back up the rbtree to find next candidate node */
1771 		while (true) {
1772 			struct rb_node *prev = &vma->vm_rb;
1773 			if (!rb_parent(prev))
1774 				goto check_highest;
1775 			vma = rb_entry(rb_parent(prev),
1776 				       struct vm_area_struct, vm_rb);
1777 			if (prev == vma->vm_rb.rb_left) {
1778 				gap_start = vma->vm_prev->vm_end;
1779 				gap_end = vma->vm_start;
1780 				goto check_current;
1781 			}
1782 		}
1783 	}
1784 
1785 check_highest:
1786 	/* Check highest gap, which does not precede any rbtree node */
1787 	gap_start = mm->highest_vm_end;
1788 	gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
1789 	if (gap_start > high_limit)
1790 		return -ENOMEM;
1791 
1792 found:
1793 	/* We found a suitable gap. Clip it with the original low_limit. */
1794 	if (gap_start < info->low_limit)
1795 		gap_start = info->low_limit;
1796 
1797 	/* Adjust gap address to the desired alignment */
1798 	gap_start += (info->align_offset - gap_start) & info->align_mask;
1799 
1800 	VM_BUG_ON(gap_start + info->length > info->high_limit);
1801 	VM_BUG_ON(gap_start + info->length > gap_end);
1802 	return gap_start;
1803 }
1804 
1805 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1806 {
1807 	struct mm_struct *mm = current->mm;
1808 	struct vm_area_struct *vma;
1809 	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1810 
1811 	/* Adjust search length to account for worst case alignment overhead */
1812 	length = info->length + info->align_mask;
1813 	if (length < info->length)
1814 		return -ENOMEM;
1815 
1816 	/*
1817 	 * Adjust search limits by the desired length.
1818 	 * See implementation comment at top of unmapped_area().
1819 	 */
1820 	gap_end = info->high_limit;
1821 	if (gap_end < length)
1822 		return -ENOMEM;
1823 	high_limit = gap_end - length;
1824 
1825 	if (info->low_limit > high_limit)
1826 		return -ENOMEM;
1827 	low_limit = info->low_limit + length;
1828 
1829 	/* Check highest gap, which does not precede any rbtree node */
1830 	gap_start = mm->highest_vm_end;
1831 	if (gap_start <= high_limit)
1832 		goto found_highest;
1833 
1834 	/* Check if rbtree root looks promising */
1835 	if (RB_EMPTY_ROOT(&mm->mm_rb))
1836 		return -ENOMEM;
1837 	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1838 	if (vma->rb_subtree_gap < length)
1839 		return -ENOMEM;
1840 
1841 	while (true) {
1842 		/* Visit right subtree if it looks promising */
1843 		gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1844 		if (gap_start <= high_limit && vma->vm_rb.rb_right) {
1845 			struct vm_area_struct *right =
1846 				rb_entry(vma->vm_rb.rb_right,
1847 					 struct vm_area_struct, vm_rb);
1848 			if (right->rb_subtree_gap >= length) {
1849 				vma = right;
1850 				continue;
1851 			}
1852 		}
1853 
1854 check_current:
1855 		/* Check if current node has a suitable gap */
1856 		gap_end = vma->vm_start;
1857 		if (gap_end < low_limit)
1858 			return -ENOMEM;
1859 		if (gap_start <= high_limit && gap_end - gap_start >= length)
1860 			goto found;
1861 
1862 		/* Visit left subtree if it looks promising */
1863 		if (vma->vm_rb.rb_left) {
1864 			struct vm_area_struct *left =
1865 				rb_entry(vma->vm_rb.rb_left,
1866 					 struct vm_area_struct, vm_rb);
1867 			if (left->rb_subtree_gap >= length) {
1868 				vma = left;
1869 				continue;
1870 			}
1871 		}
1872 
1873 		/* Go back up the rbtree to find next candidate node */
1874 		while (true) {
1875 			struct rb_node *prev = &vma->vm_rb;
1876 			if (!rb_parent(prev))
1877 				return -ENOMEM;
1878 			vma = rb_entry(rb_parent(prev),
1879 				       struct vm_area_struct, vm_rb);
1880 			if (prev == vma->vm_rb.rb_right) {
1881 				gap_start = vma->vm_prev ?
1882 					vma->vm_prev->vm_end : 0;
1883 				goto check_current;
1884 			}
1885 		}
1886 	}
1887 
1888 found:
1889 	/* We found a suitable gap. Clip it with the original high_limit. */
1890 	if (gap_end > info->high_limit)
1891 		gap_end = info->high_limit;
1892 
1893 found_highest:
1894 	/* Compute highest gap address at the desired alignment */
1895 	gap_end -= info->length;
1896 	gap_end -= (gap_end - info->align_offset) & info->align_mask;
1897 
1898 	VM_BUG_ON(gap_end < info->low_limit);
1899 	VM_BUG_ON(gap_end < gap_start);
1900 	return gap_end;
1901 }
1902 
1903 /* Get an address range which is currently unmapped.
1904  * For shmat() with addr=0.
1905  *
1906  * Ugly calling convention alert:
1907  * Return value with the low bits set means error value,
1908  * ie
1909  *	if (ret & ~PAGE_MASK)
1910  *		error = ret;
1911  *
1912  * This function "knows" that -ENOMEM has the bits set.
1913  */
1914 #ifndef HAVE_ARCH_UNMAPPED_AREA
1915 unsigned long
1916 arch_get_unmapped_area(struct file *filp, unsigned long addr,
1917 		unsigned long len, unsigned long pgoff, unsigned long flags)
1918 {
1919 	struct mm_struct *mm = current->mm;
1920 	struct vm_area_struct *vma;
1921 	struct vm_unmapped_area_info info;
1922 
1923 	if (len > TASK_SIZE - mmap_min_addr)
1924 		return -ENOMEM;
1925 
1926 	if (flags & MAP_FIXED)
1927 		return addr;
1928 
1929 	if (addr) {
1930 		addr = PAGE_ALIGN(addr);
1931 		vma = find_vma(mm, addr);
1932 		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1933 		    (!vma || addr + len <= vma->vm_start))
1934 			return addr;
1935 	}
1936 
1937 	info.flags = 0;
1938 	info.length = len;
1939 	info.low_limit = mm->mmap_base;
1940 	info.high_limit = TASK_SIZE;
1941 	info.align_mask = 0;
1942 	return vm_unmapped_area(&info);
1943 }
1944 #endif
1945 
1946 /*
1947  * This mmap-allocator allocates new areas top-down from below the
1948  * stack's low limit (the base):
1949  */
1950 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1951 unsigned long
1952 arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1953 			  const unsigned long len, const unsigned long pgoff,
1954 			  const unsigned long flags)
1955 {
1956 	struct vm_area_struct *vma;
1957 	struct mm_struct *mm = current->mm;
1958 	unsigned long addr = addr0;
1959 	struct vm_unmapped_area_info info;
1960 
1961 	/* requested length too big for entire address space */
1962 	if (len > TASK_SIZE - mmap_min_addr)
1963 		return -ENOMEM;
1964 
1965 	if (flags & MAP_FIXED)
1966 		return addr;
1967 
1968 	/* requesting a specific address */
1969 	if (addr) {
1970 		addr = PAGE_ALIGN(addr);
1971 		vma = find_vma(mm, addr);
1972 		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1973 				(!vma || addr + len <= vma->vm_start))
1974 			return addr;
1975 	}
1976 
1977 	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
1978 	info.length = len;
1979 	info.low_limit = max(PAGE_SIZE, mmap_min_addr);
1980 	info.high_limit = mm->mmap_base;
1981 	info.align_mask = 0;
1982 	addr = vm_unmapped_area(&info);
1983 
1984 	/*
1985 	 * A failed mmap() very likely causes application failure,
1986 	 * so fall back to the bottom-up function here. This scenario
1987 	 * can happen with large stack limits and large mmap()
1988 	 * allocations.
1989 	 */
1990 	if (offset_in_page(addr)) {
1991 		VM_BUG_ON(addr != -ENOMEM);
1992 		info.flags = 0;
1993 		info.low_limit = TASK_UNMAPPED_BASE;
1994 		info.high_limit = TASK_SIZE;
1995 		addr = vm_unmapped_area(&info);
1996 	}
1997 
1998 	return addr;
1999 }
2000 #endif
2001 
2002 unsigned long
2003 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
2004 		unsigned long pgoff, unsigned long flags)
2005 {
2006 	unsigned long (*get_area)(struct file *, unsigned long,
2007 				  unsigned long, unsigned long, unsigned long);
2008 
2009 	unsigned long error = arch_mmap_check(addr, len, flags);
2010 	if (error)
2011 		return error;
2012 
2013 	/* Careful about overflows.. */
2014 	if (len > TASK_SIZE)
2015 		return -ENOMEM;
2016 
2017 	get_area = current->mm->get_unmapped_area;
2018 	if (file && file->f_op->get_unmapped_area)
2019 		get_area = file->f_op->get_unmapped_area;
2020 	addr = get_area(file, addr, len, pgoff, flags);
2021 	if (IS_ERR_VALUE(addr))
2022 		return addr;
2023 
2024 	if (addr > TASK_SIZE - len)
2025 		return -ENOMEM;
2026 	if (offset_in_page(addr))
2027 		return -EINVAL;
2028 
2029 	addr = arch_rebalance_pgtables(addr, len);
2030 	error = security_mmap_addr(addr);
2031 	return error ? error : addr;
2032 }
2033 
2034 EXPORT_SYMBOL(get_unmapped_area);
2035 
2036 /* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
2037 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
2038 {
2039 	struct rb_node *rb_node;
2040 	struct vm_area_struct *vma;
2041 
2042 	/* Check the cache first. */
2043 	vma = vmacache_find(mm, addr);
2044 	if (likely(vma))
2045 		return vma;
2046 
2047 	rb_node = mm->mm_rb.rb_node;
2048 
2049 	while (rb_node) {
2050 		struct vm_area_struct *tmp;
2051 
2052 		tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2053 
2054 		if (tmp->vm_end > addr) {
2055 			vma = tmp;
2056 			if (tmp->vm_start <= addr)
2057 				break;
2058 			rb_node = rb_node->rb_left;
2059 		} else
2060 			rb_node = rb_node->rb_right;
2061 	}
2062 
2063 	if (vma)
2064 		vmacache_update(addr, vma);
2065 	return vma;
2066 }
2067 
2068 EXPORT_SYMBOL(find_vma);
2069 
2070 /*
2071  * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2072  */
2073 struct vm_area_struct *
2074 find_vma_prev(struct mm_struct *mm, unsigned long addr,
2075 			struct vm_area_struct **pprev)
2076 {
2077 	struct vm_area_struct *vma;
2078 
2079 	vma = find_vma(mm, addr);
2080 	if (vma) {
2081 		*pprev = vma->vm_prev;
2082 	} else {
2083 		struct rb_node *rb_node = mm->mm_rb.rb_node;
2084 		*pprev = NULL;
2085 		while (rb_node) {
2086 			*pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2087 			rb_node = rb_node->rb_right;
2088 		}
2089 	}
2090 	return vma;
2091 }
2092 
2093 /*
2094  * Verify that the stack growth is acceptable and
2095  * update accounting. This is shared with both the
2096  * grow-up and grow-down cases.
2097  */
2098 static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2099 {
2100 	struct mm_struct *mm = vma->vm_mm;
2101 	struct rlimit *rlim = current->signal->rlim;
2102 	unsigned long new_start, actual_size;
2103 
2104 	/* address space limit tests */
2105 	if (!may_expand_vm(mm, grow))
2106 		return -ENOMEM;
2107 
2108 	/* Stack limit test */
2109 	actual_size = size;
2110 	if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN)))
2111 		actual_size -= PAGE_SIZE;
2112 	if (actual_size > READ_ONCE(rlim[RLIMIT_STACK].rlim_cur))
2113 		return -ENOMEM;
2114 
2115 	/* mlock limit tests */
2116 	if (vma->vm_flags & VM_LOCKED) {
2117 		unsigned long locked;
2118 		unsigned long limit;
2119 		locked = mm->locked_vm + grow;
2120 		limit = READ_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
2121 		limit >>= PAGE_SHIFT;
2122 		if (locked > limit && !capable(CAP_IPC_LOCK))
2123 			return -ENOMEM;
2124 	}
2125 
2126 	/* Check to ensure the stack will not grow into a hugetlb-only region */
2127 	new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2128 			vma->vm_end - size;
2129 	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2130 		return -EFAULT;
2131 
2132 	/*
2133 	 * Overcommit..  This must be the final test, as it will
2134 	 * update security statistics.
2135 	 */
2136 	if (security_vm_enough_memory_mm(mm, grow))
2137 		return -ENOMEM;
2138 
2139 	return 0;
2140 }
2141 
2142 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2143 /*
2144  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2145  * vma is the last one with address > vma->vm_end.  Have to extend vma.
2146  */
2147 int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2148 {
2149 	struct mm_struct *mm = vma->vm_mm;
2150 	int error;
2151 
2152 	if (!(vma->vm_flags & VM_GROWSUP))
2153 		return -EFAULT;
2154 
2155 	/*
2156 	 * We must make sure the anon_vma is allocated
2157 	 * so that the anon_vma locking is not a noop.
2158 	 */
2159 	if (unlikely(anon_vma_prepare(vma)))
2160 		return -ENOMEM;
2161 	vma_lock_anon_vma(vma);
2162 
2163 	/*
2164 	 * vma->vm_start/vm_end cannot change under us because the caller
2165 	 * is required to hold the mmap_sem in read mode.  We need the
2166 	 * anon_vma lock to serialize against concurrent expand_stacks.
2167 	 * Also guard against wrapping around to address 0.
2168 	 */
2169 	if (address < PAGE_ALIGN(address+4))
2170 		address = PAGE_ALIGN(address+4);
2171 	else {
2172 		vma_unlock_anon_vma(vma);
2173 		return -ENOMEM;
2174 	}
2175 	error = 0;
2176 
2177 	/* Somebody else might have raced and expanded it already */
2178 	if (address > vma->vm_end) {
2179 		unsigned long size, grow;
2180 
2181 		size = address - vma->vm_start;
2182 		grow = (address - vma->vm_end) >> PAGE_SHIFT;
2183 
2184 		error = -ENOMEM;
2185 		if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2186 			error = acct_stack_growth(vma, size, grow);
2187 			if (!error) {
2188 				/*
2189 				 * vma_gap_update() doesn't support concurrent
2190 				 * updates, but we only hold a shared mmap_sem
2191 				 * lock here, so we need to protect against
2192 				 * concurrent vma expansions.
2193 				 * vma_lock_anon_vma() doesn't help here, as
2194 				 * we don't guarantee that all growable vmas
2195 				 * in a mm share the same root anon vma.
2196 				 * So, we reuse mm->page_table_lock to guard
2197 				 * against concurrent vma expansions.
2198 				 */
2199 				spin_lock(&mm->page_table_lock);
2200 				if (vma->vm_flags & VM_LOCKED)
2201 					mm->locked_vm += grow;
2202 				vm_stat_account(mm, vma->vm_flags,
2203 						vma->vm_file, grow);
2204 				anon_vma_interval_tree_pre_update_vma(vma);
2205 				vma->vm_end = address;
2206 				anon_vma_interval_tree_post_update_vma(vma);
2207 				if (vma->vm_next)
2208 					vma_gap_update(vma->vm_next);
2209 				else
2210 					mm->highest_vm_end = address;
2211 				spin_unlock(&mm->page_table_lock);
2212 
2213 				perf_event_mmap(vma);
2214 			}
2215 		}
2216 	}
2217 	vma_unlock_anon_vma(vma);
2218 	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2219 	validate_mm(mm);
2220 	return error;
2221 }
2222 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
2223 
2224 /*
2225  * vma is the first one with address < vma->vm_start.  Have to extend vma.
2226  */
2227 int expand_downwards(struct vm_area_struct *vma,
2228 				   unsigned long address)
2229 {
2230 	struct mm_struct *mm = vma->vm_mm;
2231 	int error;
2232 
2233 	/*
2234 	 * We must make sure the anon_vma is allocated
2235 	 * so that the anon_vma locking is not a noop.
2236 	 */
2237 	if (unlikely(anon_vma_prepare(vma)))
2238 		return -ENOMEM;
2239 
2240 	address &= PAGE_MASK;
2241 	error = security_mmap_addr(address);
2242 	if (error)
2243 		return error;
2244 
2245 	vma_lock_anon_vma(vma);
2246 
2247 	/*
2248 	 * vma->vm_start/vm_end cannot change under us because the caller
2249 	 * is required to hold the mmap_sem in read mode.  We need the
2250 	 * anon_vma lock to serialize against concurrent expand_stacks.
2251 	 */
2252 
2253 	/* Somebody else might have raced and expanded it already */
2254 	if (address < vma->vm_start) {
2255 		unsigned long size, grow;
2256 
2257 		size = vma->vm_end - address;
2258 		grow = (vma->vm_start - address) >> PAGE_SHIFT;
2259 
2260 		error = -ENOMEM;
2261 		if (grow <= vma->vm_pgoff) {
2262 			error = acct_stack_growth(vma, size, grow);
2263 			if (!error) {
2264 				/*
2265 				 * vma_gap_update() doesn't support concurrent
2266 				 * updates, but we only hold a shared mmap_sem
2267 				 * lock here, so we need to protect against
2268 				 * concurrent vma expansions.
2269 				 * vma_lock_anon_vma() doesn't help here, as
2270 				 * we don't guarantee that all growable vmas
2271 				 * in a mm share the same root anon vma.
2272 				 * So, we reuse mm->page_table_lock to guard
2273 				 * against concurrent vma expansions.
2274 				 */
2275 				spin_lock(&mm->page_table_lock);
2276 				if (vma->vm_flags & VM_LOCKED)
2277 					mm->locked_vm += grow;
2278 				vm_stat_account(mm, vma->vm_flags,
2279 						vma->vm_file, grow);
2280 				anon_vma_interval_tree_pre_update_vma(vma);
2281 				vma->vm_start = address;
2282 				vma->vm_pgoff -= grow;
2283 				anon_vma_interval_tree_post_update_vma(vma);
2284 				vma_gap_update(vma);
2285 				spin_unlock(&mm->page_table_lock);
2286 
2287 				perf_event_mmap(vma);
2288 			}
2289 		}
2290 	}
2291 	vma_unlock_anon_vma(vma);
2292 	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2293 	validate_mm(mm);
2294 	return error;
2295 }
2296 
2297 /*
2298  * Note how expand_stack() refuses to expand the stack all the way to
2299  * abut the next virtual mapping, *unless* that mapping itself is also
2300  * a stack mapping. We want to leave room for a guard page, after all
2301  * (the guard page itself is not added here, that is done by the
2302  * actual page faulting logic)
2303  *
2304  * This matches the behavior of the guard page logic (see mm/memory.c:
2305  * check_stack_guard_page()), which only allows the guard page to be
2306  * removed under these circumstances.
2307  */
2308 #ifdef CONFIG_STACK_GROWSUP
2309 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2310 {
2311 	struct vm_area_struct *next;
2312 
2313 	address &= PAGE_MASK;
2314 	next = vma->vm_next;
2315 	if (next && next->vm_start == address + PAGE_SIZE) {
2316 		if (!(next->vm_flags & VM_GROWSUP))
2317 			return -ENOMEM;
2318 	}
2319 	return expand_upwards(vma, address);
2320 }
2321 
2322 struct vm_area_struct *
2323 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2324 {
2325 	struct vm_area_struct *vma, *prev;
2326 
2327 	addr &= PAGE_MASK;
2328 	vma = find_vma_prev(mm, addr, &prev);
2329 	if (vma && (vma->vm_start <= addr))
2330 		return vma;
2331 	if (!prev || expand_stack(prev, addr))
2332 		return NULL;
2333 	if (prev->vm_flags & VM_LOCKED)
2334 		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
2335 	return prev;
2336 }
2337 #else
2338 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2339 {
2340 	struct vm_area_struct *prev;
2341 
2342 	address &= PAGE_MASK;
2343 	prev = vma->vm_prev;
2344 	if (prev && prev->vm_end == address) {
2345 		if (!(prev->vm_flags & VM_GROWSDOWN))
2346 			return -ENOMEM;
2347 	}
2348 	return expand_downwards(vma, address);
2349 }
2350 
2351 struct vm_area_struct *
2352 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2353 {
2354 	struct vm_area_struct *vma;
2355 	unsigned long start;
2356 
2357 	addr &= PAGE_MASK;
2358 	vma = find_vma(mm, addr);
2359 	if (!vma)
2360 		return NULL;
2361 	if (vma->vm_start <= addr)
2362 		return vma;
2363 	if (!(vma->vm_flags & VM_GROWSDOWN))
2364 		return NULL;
2365 	start = vma->vm_start;
2366 	if (expand_stack(vma, addr))
2367 		return NULL;
2368 	if (vma->vm_flags & VM_LOCKED)
2369 		populate_vma_page_range(vma, addr, start, NULL);
2370 	return vma;
2371 }
2372 #endif
2373 
2374 EXPORT_SYMBOL_GPL(find_extend_vma);
2375 
2376 /*
2377  * Ok - we have the memory areas we should free on the vma list,
2378  * so release them, and do the vma updates.
2379  *
2380  * Called with the mm semaphore held.
2381  */
2382 static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
2383 {
2384 	unsigned long nr_accounted = 0;
2385 
2386 	/* Update high watermark before we lower total_vm */
2387 	update_hiwater_vm(mm);
2388 	do {
2389 		long nrpages = vma_pages(vma);
2390 
2391 		if (vma->vm_flags & VM_ACCOUNT)
2392 			nr_accounted += nrpages;
2393 		vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
2394 		vma = remove_vma(vma);
2395 	} while (vma);
2396 	vm_unacct_memory(nr_accounted);
2397 	validate_mm(mm);
2398 }
2399 
2400 /*
2401  * Get rid of page table information in the indicated region.
2402  *
2403  * Called with the mm semaphore held.
2404  */
2405 static void unmap_region(struct mm_struct *mm,
2406 		struct vm_area_struct *vma, struct vm_area_struct *prev,
2407 		unsigned long start, unsigned long end)
2408 {
2409 	struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
2410 	struct mmu_gather tlb;
2411 
2412 	lru_add_drain();
2413 	tlb_gather_mmu(&tlb, mm, start, end);
2414 	update_hiwater_rss(mm);
2415 	unmap_vmas(&tlb, vma, start, end);
2416 	free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2417 				 next ? next->vm_start : USER_PGTABLES_CEILING);
2418 	tlb_finish_mmu(&tlb, start, end);
2419 }
2420 
2421 /*
2422  * Create a list of vma's touched by the unmap, removing them from the mm's
2423  * vma list as we go..
2424  */
2425 static void
2426 detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
2427 	struct vm_area_struct *prev, unsigned long end)
2428 {
2429 	struct vm_area_struct **insertion_point;
2430 	struct vm_area_struct *tail_vma = NULL;
2431 
2432 	insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2433 	vma->vm_prev = NULL;
2434 	do {
2435 		vma_rb_erase(vma, &mm->mm_rb);
2436 		mm->map_count--;
2437 		tail_vma = vma;
2438 		vma = vma->vm_next;
2439 	} while (vma && vma->vm_start < end);
2440 	*insertion_point = vma;
2441 	if (vma) {
2442 		vma->vm_prev = prev;
2443 		vma_gap_update(vma);
2444 	} else
2445 		mm->highest_vm_end = prev ? prev->vm_end : 0;
2446 	tail_vma->vm_next = NULL;
2447 
2448 	/* Kill the cache */
2449 	vmacache_invalidate(mm);
2450 }
2451 
2452 /*
2453  * __split_vma() bypasses sysctl_max_map_count checking.  We use this on the
2454  * munmap path where it doesn't make sense to fail.
2455  */
2456 static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2457 	      unsigned long addr, int new_below)
2458 {
2459 	struct vm_area_struct *new;
2460 	int err;
2461 
2462 	if (is_vm_hugetlb_page(vma) && (addr &
2463 					~(huge_page_mask(hstate_vma(vma)))))
2464 		return -EINVAL;
2465 
2466 	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2467 	if (!new)
2468 		return -ENOMEM;
2469 
2470 	/* most fields are the same, copy all, and then fixup */
2471 	*new = *vma;
2472 
2473 	INIT_LIST_HEAD(&new->anon_vma_chain);
2474 
2475 	if (new_below)
2476 		new->vm_end = addr;
2477 	else {
2478 		new->vm_start = addr;
2479 		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2480 	}
2481 
2482 	err = vma_dup_policy(vma, new);
2483 	if (err)
2484 		goto out_free_vma;
2485 
2486 	err = anon_vma_clone(new, vma);
2487 	if (err)
2488 		goto out_free_mpol;
2489 
2490 	if (new->vm_file)
2491 		get_file(new->vm_file);
2492 
2493 	if (new->vm_ops && new->vm_ops->open)
2494 		new->vm_ops->open(new);
2495 
2496 	if (new_below)
2497 		err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2498 			((addr - new->vm_start) >> PAGE_SHIFT), new);
2499 	else
2500 		err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2501 
2502 	/* Success. */
2503 	if (!err)
2504 		return 0;
2505 
2506 	/* Clean everything up if vma_adjust failed. */
2507 	if (new->vm_ops && new->vm_ops->close)
2508 		new->vm_ops->close(new);
2509 	if (new->vm_file)
2510 		fput(new->vm_file);
2511 	unlink_anon_vmas(new);
2512  out_free_mpol:
2513 	mpol_put(vma_policy(new));
2514  out_free_vma:
2515 	kmem_cache_free(vm_area_cachep, new);
2516 	return err;
2517 }
2518 
2519 /*
2520  * Split a vma into two pieces at address 'addr', a new vma is allocated
2521  * either for the first part or the tail.
2522  */
2523 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2524 	      unsigned long addr, int new_below)
2525 {
2526 	if (mm->map_count >= sysctl_max_map_count)
2527 		return -ENOMEM;
2528 
2529 	return __split_vma(mm, vma, addr, new_below);
2530 }
2531 
2532 /* Munmap is split into 2 main parts -- this part which finds
2533  * what needs doing, and the areas themselves, which do the
2534  * work.  This now handles partial unmappings.
2535  * Jeremy Fitzhardinge <jeremy@goop.org>
2536  */
2537 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2538 {
2539 	unsigned long end;
2540 	struct vm_area_struct *vma, *prev, *last;
2541 
2542 	if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
2543 		return -EINVAL;
2544 
2545 	len = PAGE_ALIGN(len);
2546 	if (len == 0)
2547 		return -EINVAL;
2548 
2549 	/* Find the first overlapping VMA */
2550 	vma = find_vma(mm, start);
2551 	if (!vma)
2552 		return 0;
2553 	prev = vma->vm_prev;
2554 	/* we have  start < vma->vm_end  */
2555 
2556 	/* if it doesn't overlap, we have nothing.. */
2557 	end = start + len;
2558 	if (vma->vm_start >= end)
2559 		return 0;
2560 
2561 	/*
2562 	 * If we need to split any vma, do it now to save pain later.
2563 	 *
2564 	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2565 	 * unmapped vm_area_struct will remain in use: so lower split_vma
2566 	 * places tmp vma above, and higher split_vma places tmp vma below.
2567 	 */
2568 	if (start > vma->vm_start) {
2569 		int error;
2570 
2571 		/*
2572 		 * Make sure that map_count on return from munmap() will
2573 		 * not exceed its limit; but let map_count go just above
2574 		 * its limit temporarily, to help free resources as expected.
2575 		 */
2576 		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2577 			return -ENOMEM;
2578 
2579 		error = __split_vma(mm, vma, start, 0);
2580 		if (error)
2581 			return error;
2582 		prev = vma;
2583 	}
2584 
2585 	/* Does it split the last one? */
2586 	last = find_vma(mm, end);
2587 	if (last && end > last->vm_start) {
2588 		int error = __split_vma(mm, last, end, 1);
2589 		if (error)
2590 			return error;
2591 	}
2592 	vma = prev ? prev->vm_next : mm->mmap;
2593 
2594 	/*
2595 	 * unlock any mlock()ed ranges before detaching vmas
2596 	 */
2597 	if (mm->locked_vm) {
2598 		struct vm_area_struct *tmp = vma;
2599 		while (tmp && tmp->vm_start < end) {
2600 			if (tmp->vm_flags & VM_LOCKED) {
2601 				mm->locked_vm -= vma_pages(tmp);
2602 				munlock_vma_pages_all(tmp);
2603 			}
2604 			tmp = tmp->vm_next;
2605 		}
2606 	}
2607 
2608 	/*
2609 	 * Remove the vma's, and unmap the actual pages
2610 	 */
2611 	detach_vmas_to_be_unmapped(mm, vma, prev, end);
2612 	unmap_region(mm, vma, prev, start, end);
2613 
2614 	arch_unmap(mm, vma, start, end);
2615 
2616 	/* Fix up all other VM information */
2617 	remove_vma_list(mm, vma);
2618 
2619 	return 0;
2620 }
2621 
2622 int vm_munmap(unsigned long start, size_t len)
2623 {
2624 	int ret;
2625 	struct mm_struct *mm = current->mm;
2626 
2627 	down_write(&mm->mmap_sem);
2628 	ret = do_munmap(mm, start, len);
2629 	up_write(&mm->mmap_sem);
2630 	return ret;
2631 }
2632 EXPORT_SYMBOL(vm_munmap);
2633 
2634 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2635 {
2636 	profile_munmap(addr);
2637 	return vm_munmap(addr, len);
2638 }
2639 
2640 
2641 /*
2642  * Emulation of deprecated remap_file_pages() syscall.
2643  */
2644 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
2645 		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
2646 {
2647 
2648 	struct mm_struct *mm = current->mm;
2649 	struct vm_area_struct *vma;
2650 	unsigned long populate = 0;
2651 	unsigned long ret = -EINVAL;
2652 	struct file *file;
2653 
2654 	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. "
2655 			"See Documentation/vm/remap_file_pages.txt.\n",
2656 			current->comm, current->pid);
2657 
2658 	if (prot)
2659 		return ret;
2660 	start = start & PAGE_MASK;
2661 	size = size & PAGE_MASK;
2662 
2663 	if (start + size <= start)
2664 		return ret;
2665 
2666 	/* Does pgoff wrap? */
2667 	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
2668 		return ret;
2669 
2670 	down_write(&mm->mmap_sem);
2671 	vma = find_vma(mm, start);
2672 
2673 	if (!vma || !(vma->vm_flags & VM_SHARED))
2674 		goto out;
2675 
2676 	if (start < vma->vm_start || start + size > vma->vm_end)
2677 		goto out;
2678 
2679 	if (pgoff == linear_page_index(vma, start)) {
2680 		ret = 0;
2681 		goto out;
2682 	}
2683 
2684 	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
2685 	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
2686 	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
2687 
2688 	flags &= MAP_NONBLOCK;
2689 	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
2690 	if (vma->vm_flags & VM_LOCKED) {
2691 		flags |= MAP_LOCKED;
2692 		/* drop PG_Mlocked flag for over-mapped range */
2693 		munlock_vma_pages_range(vma, start, start + size);
2694 	}
2695 
2696 	file = get_file(vma->vm_file);
2697 	ret = do_mmap_pgoff(vma->vm_file, start, size,
2698 			prot, flags, pgoff, &populate);
2699 	fput(file);
2700 out:
2701 	up_write(&mm->mmap_sem);
2702 	if (populate)
2703 		mm_populate(ret, populate);
2704 	if (!IS_ERR_VALUE(ret))
2705 		ret = 0;
2706 	return ret;
2707 }
2708 
2709 static inline void verify_mm_writelocked(struct mm_struct *mm)
2710 {
2711 #ifdef CONFIG_DEBUG_VM
2712 	if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2713 		WARN_ON(1);
2714 		up_read(&mm->mmap_sem);
2715 	}
2716 #endif
2717 }
2718 
2719 /*
2720  *  this is really a simplified "do_mmap".  it only handles
2721  *  anonymous maps.  eventually we may be able to do some
2722  *  brk-specific accounting here.
2723  */
2724 static unsigned long do_brk(unsigned long addr, unsigned long len)
2725 {
2726 	struct mm_struct *mm = current->mm;
2727 	struct vm_area_struct *vma, *prev;
2728 	unsigned long flags;
2729 	struct rb_node **rb_link, *rb_parent;
2730 	pgoff_t pgoff = addr >> PAGE_SHIFT;
2731 	int error;
2732 
2733 	len = PAGE_ALIGN(len);
2734 	if (!len)
2735 		return addr;
2736 
2737 	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2738 
2739 	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2740 	if (offset_in_page(error))
2741 		return error;
2742 
2743 	error = mlock_future_check(mm, mm->def_flags, len);
2744 	if (error)
2745 		return error;
2746 
2747 	/*
2748 	 * mm->mmap_sem is required to protect against another thread
2749 	 * changing the mappings in case we sleep.
2750 	 */
2751 	verify_mm_writelocked(mm);
2752 
2753 	/*
2754 	 * Clear old maps.  this also does some error checking for us
2755 	 */
2756 	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
2757 			      &rb_parent)) {
2758 		if (do_munmap(mm, addr, len))
2759 			return -ENOMEM;
2760 	}
2761 
2762 	/* Check against address space limits *after* clearing old maps... */
2763 	if (!may_expand_vm(mm, len >> PAGE_SHIFT))
2764 		return -ENOMEM;
2765 
2766 	if (mm->map_count > sysctl_max_map_count)
2767 		return -ENOMEM;
2768 
2769 	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
2770 		return -ENOMEM;
2771 
2772 	/* Can we just expand an old private anonymous mapping? */
2773 	vma = vma_merge(mm, prev, addr, addr + len, flags,
2774 			NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
2775 	if (vma)
2776 		goto out;
2777 
2778 	/*
2779 	 * create a vma struct for an anonymous mapping
2780 	 */
2781 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2782 	if (!vma) {
2783 		vm_unacct_memory(len >> PAGE_SHIFT);
2784 		return -ENOMEM;
2785 	}
2786 
2787 	INIT_LIST_HEAD(&vma->anon_vma_chain);
2788 	vma->vm_mm = mm;
2789 	vma->vm_start = addr;
2790 	vma->vm_end = addr + len;
2791 	vma->vm_pgoff = pgoff;
2792 	vma->vm_flags = flags;
2793 	vma->vm_page_prot = vm_get_page_prot(flags);
2794 	vma_link(mm, vma, prev, rb_link, rb_parent);
2795 out:
2796 	perf_event_mmap(vma);
2797 	mm->total_vm += len >> PAGE_SHIFT;
2798 	if (flags & VM_LOCKED)
2799 		mm->locked_vm += (len >> PAGE_SHIFT);
2800 	vma->vm_flags |= VM_SOFTDIRTY;
2801 	return addr;
2802 }
2803 
2804 unsigned long vm_brk(unsigned long addr, unsigned long len)
2805 {
2806 	struct mm_struct *mm = current->mm;
2807 	unsigned long ret;
2808 	bool populate;
2809 
2810 	down_write(&mm->mmap_sem);
2811 	ret = do_brk(addr, len);
2812 	populate = ((mm->def_flags & VM_LOCKED) != 0);
2813 	up_write(&mm->mmap_sem);
2814 	if (populate)
2815 		mm_populate(addr, len);
2816 	return ret;
2817 }
2818 EXPORT_SYMBOL(vm_brk);
2819 
2820 /* Release all mmaps. */
2821 void exit_mmap(struct mm_struct *mm)
2822 {
2823 	struct mmu_gather tlb;
2824 	struct vm_area_struct *vma;
2825 	unsigned long nr_accounted = 0;
2826 
2827 	/* mm's last user has gone, and its about to be pulled down */
2828 	mmu_notifier_release(mm);
2829 
2830 	if (mm->locked_vm) {
2831 		vma = mm->mmap;
2832 		while (vma) {
2833 			if (vma->vm_flags & VM_LOCKED)
2834 				munlock_vma_pages_all(vma);
2835 			vma = vma->vm_next;
2836 		}
2837 	}
2838 
2839 	arch_exit_mmap(mm);
2840 
2841 	vma = mm->mmap;
2842 	if (!vma)	/* Can happen if dup_mmap() received an OOM */
2843 		return;
2844 
2845 	lru_add_drain();
2846 	flush_cache_mm(mm);
2847 	tlb_gather_mmu(&tlb, mm, 0, -1);
2848 	/* update_hiwater_rss(mm) here? but nobody should be looking */
2849 	/* Use -1 here to ensure all VMAs in the mm are unmapped */
2850 	unmap_vmas(&tlb, vma, 0, -1);
2851 
2852 	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
2853 	tlb_finish_mmu(&tlb, 0, -1);
2854 
2855 	/*
2856 	 * Walk the list again, actually closing and freeing it,
2857 	 * with preemption enabled, without holding any MM locks.
2858 	 */
2859 	while (vma) {
2860 		if (vma->vm_flags & VM_ACCOUNT)
2861 			nr_accounted += vma_pages(vma);
2862 		vma = remove_vma(vma);
2863 	}
2864 	vm_unacct_memory(nr_accounted);
2865 }
2866 
2867 /* Insert vm structure into process list sorted by address
2868  * and into the inode's i_mmap tree.  If vm_file is non-NULL
2869  * then i_mmap_rwsem is taken here.
2870  */
2871 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
2872 {
2873 	struct vm_area_struct *prev;
2874 	struct rb_node **rb_link, *rb_parent;
2875 
2876 	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
2877 			   &prev, &rb_link, &rb_parent))
2878 		return -ENOMEM;
2879 	if ((vma->vm_flags & VM_ACCOUNT) &&
2880 	     security_vm_enough_memory_mm(mm, vma_pages(vma)))
2881 		return -ENOMEM;
2882 
2883 	/*
2884 	 * The vm_pgoff of a purely anonymous vma should be irrelevant
2885 	 * until its first write fault, when page's anon_vma and index
2886 	 * are set.  But now set the vm_pgoff it will almost certainly
2887 	 * end up with (unless mremap moves it elsewhere before that
2888 	 * first wfault), so /proc/pid/maps tells a consistent story.
2889 	 *
2890 	 * By setting it to reflect the virtual start address of the
2891 	 * vma, merges and splits can happen in a seamless way, just
2892 	 * using the existing file pgoff checks and manipulations.
2893 	 * Similarly in do_mmap_pgoff and in do_brk.
2894 	 */
2895 	if (vma_is_anonymous(vma)) {
2896 		BUG_ON(vma->anon_vma);
2897 		vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2898 	}
2899 
2900 	vma_link(mm, vma, prev, rb_link, rb_parent);
2901 	return 0;
2902 }
2903 
2904 /*
2905  * Copy the vma structure to a new location in the same mm,
2906  * prior to moving page table entries, to effect an mremap move.
2907  */
2908 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2909 	unsigned long addr, unsigned long len, pgoff_t pgoff,
2910 	bool *need_rmap_locks)
2911 {
2912 	struct vm_area_struct *vma = *vmap;
2913 	unsigned long vma_start = vma->vm_start;
2914 	struct mm_struct *mm = vma->vm_mm;
2915 	struct vm_area_struct *new_vma, *prev;
2916 	struct rb_node **rb_link, *rb_parent;
2917 	bool faulted_in_anon_vma = true;
2918 
2919 	/*
2920 	 * If anonymous vma has not yet been faulted, update new pgoff
2921 	 * to match new location, to increase its chance of merging.
2922 	 */
2923 	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
2924 		pgoff = addr >> PAGE_SHIFT;
2925 		faulted_in_anon_vma = false;
2926 	}
2927 
2928 	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
2929 		return NULL;	/* should never get here */
2930 	new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2931 			    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
2932 			    vma->vm_userfaultfd_ctx);
2933 	if (new_vma) {
2934 		/*
2935 		 * Source vma may have been merged into new_vma
2936 		 */
2937 		if (unlikely(vma_start >= new_vma->vm_start &&
2938 			     vma_start < new_vma->vm_end)) {
2939 			/*
2940 			 * The only way we can get a vma_merge with
2941 			 * self during an mremap is if the vma hasn't
2942 			 * been faulted in yet and we were allowed to
2943 			 * reset the dst vma->vm_pgoff to the
2944 			 * destination address of the mremap to allow
2945 			 * the merge to happen. mremap must change the
2946 			 * vm_pgoff linearity between src and dst vmas
2947 			 * (in turn preventing a vma_merge) to be
2948 			 * safe. It is only safe to keep the vm_pgoff
2949 			 * linear if there are no pages mapped yet.
2950 			 */
2951 			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
2952 			*vmap = vma = new_vma;
2953 		}
2954 		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
2955 	} else {
2956 		new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2957 		if (!new_vma)
2958 			goto out;
2959 		*new_vma = *vma;
2960 		new_vma->vm_start = addr;
2961 		new_vma->vm_end = addr + len;
2962 		new_vma->vm_pgoff = pgoff;
2963 		if (vma_dup_policy(vma, new_vma))
2964 			goto out_free_vma;
2965 		INIT_LIST_HEAD(&new_vma->anon_vma_chain);
2966 		if (anon_vma_clone(new_vma, vma))
2967 			goto out_free_mempol;
2968 		if (new_vma->vm_file)
2969 			get_file(new_vma->vm_file);
2970 		if (new_vma->vm_ops && new_vma->vm_ops->open)
2971 			new_vma->vm_ops->open(new_vma);
2972 		vma_link(mm, new_vma, prev, rb_link, rb_parent);
2973 		*need_rmap_locks = false;
2974 	}
2975 	return new_vma;
2976 
2977 out_free_mempol:
2978 	mpol_put(vma_policy(new_vma));
2979 out_free_vma:
2980 	kmem_cache_free(vm_area_cachep, new_vma);
2981 out:
2982 	return NULL;
2983 }
2984 
2985 /*
2986  * Return true if the calling process may expand its vm space by the passed
2987  * number of pages
2988  */
2989 int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2990 {
2991 	unsigned long cur = mm->total_vm;	/* pages */
2992 	unsigned long lim;
2993 
2994 	lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
2995 
2996 	if (cur + npages > lim)
2997 		return 0;
2998 	return 1;
2999 }
3000 
3001 static int special_mapping_fault(struct vm_area_struct *vma,
3002 				 struct vm_fault *vmf);
3003 
3004 /*
3005  * Having a close hook prevents vma merging regardless of flags.
3006  */
3007 static void special_mapping_close(struct vm_area_struct *vma)
3008 {
3009 }
3010 
3011 static const char *special_mapping_name(struct vm_area_struct *vma)
3012 {
3013 	return ((struct vm_special_mapping *)vma->vm_private_data)->name;
3014 }
3015 
3016 static const struct vm_operations_struct special_mapping_vmops = {
3017 	.close = special_mapping_close,
3018 	.fault = special_mapping_fault,
3019 	.name = special_mapping_name,
3020 };
3021 
3022 static const struct vm_operations_struct legacy_special_mapping_vmops = {
3023 	.close = special_mapping_close,
3024 	.fault = special_mapping_fault,
3025 };
3026 
3027 static int special_mapping_fault(struct vm_area_struct *vma,
3028 				struct vm_fault *vmf)
3029 {
3030 	pgoff_t pgoff;
3031 	struct page **pages;
3032 
3033 	if (vma->vm_ops == &legacy_special_mapping_vmops)
3034 		pages = vma->vm_private_data;
3035 	else
3036 		pages = ((struct vm_special_mapping *)vma->vm_private_data)->
3037 			pages;
3038 
3039 	for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
3040 		pgoff--;
3041 
3042 	if (*pages) {
3043 		struct page *page = *pages;
3044 		get_page(page);
3045 		vmf->page = page;
3046 		return 0;
3047 	}
3048 
3049 	return VM_FAULT_SIGBUS;
3050 }
3051 
3052 static struct vm_area_struct *__install_special_mapping(
3053 	struct mm_struct *mm,
3054 	unsigned long addr, unsigned long len,
3055 	unsigned long vm_flags, void *priv,
3056 	const struct vm_operations_struct *ops)
3057 {
3058 	int ret;
3059 	struct vm_area_struct *vma;
3060 
3061 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
3062 	if (unlikely(vma == NULL))
3063 		return ERR_PTR(-ENOMEM);
3064 
3065 	INIT_LIST_HEAD(&vma->anon_vma_chain);
3066 	vma->vm_mm = mm;
3067 	vma->vm_start = addr;
3068 	vma->vm_end = addr + len;
3069 
3070 	vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
3071 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3072 
3073 	vma->vm_ops = ops;
3074 	vma->vm_private_data = priv;
3075 
3076 	ret = insert_vm_struct(mm, vma);
3077 	if (ret)
3078 		goto out;
3079 
3080 	mm->total_vm += len >> PAGE_SHIFT;
3081 
3082 	perf_event_mmap(vma);
3083 
3084 	return vma;
3085 
3086 out:
3087 	kmem_cache_free(vm_area_cachep, vma);
3088 	return ERR_PTR(ret);
3089 }
3090 
3091 /*
3092  * Called with mm->mmap_sem held for writing.
3093  * Insert a new vma covering the given region, with the given flags.
3094  * Its pages are supplied by the given array of struct page *.
3095  * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
3096  * The region past the last page supplied will always produce SIGBUS.
3097  * The array pointer and the pages it points to are assumed to stay alive
3098  * for as long as this mapping might exist.
3099  */
3100 struct vm_area_struct *_install_special_mapping(
3101 	struct mm_struct *mm,
3102 	unsigned long addr, unsigned long len,
3103 	unsigned long vm_flags, const struct vm_special_mapping *spec)
3104 {
3105 	return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
3106 					&special_mapping_vmops);
3107 }
3108 
3109 int install_special_mapping(struct mm_struct *mm,
3110 			    unsigned long addr, unsigned long len,
3111 			    unsigned long vm_flags, struct page **pages)
3112 {
3113 	struct vm_area_struct *vma = __install_special_mapping(
3114 		mm, addr, len, vm_flags, (void *)pages,
3115 		&legacy_special_mapping_vmops);
3116 
3117 	return PTR_ERR_OR_ZERO(vma);
3118 }
3119 
3120 static DEFINE_MUTEX(mm_all_locks_mutex);
3121 
3122 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
3123 {
3124 	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3125 		/*
3126 		 * The LSB of head.next can't change from under us
3127 		 * because we hold the mm_all_locks_mutex.
3128 		 */
3129 		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
3130 		/*
3131 		 * We can safely modify head.next after taking the
3132 		 * anon_vma->root->rwsem. If some other vma in this mm shares
3133 		 * the same anon_vma we won't take it again.
3134 		 *
3135 		 * No need of atomic instructions here, head.next
3136 		 * can't change from under us thanks to the
3137 		 * anon_vma->root->rwsem.
3138 		 */
3139 		if (__test_and_set_bit(0, (unsigned long *)
3140 				       &anon_vma->root->rb_root.rb_node))
3141 			BUG();
3142 	}
3143 }
3144 
3145 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
3146 {
3147 	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3148 		/*
3149 		 * AS_MM_ALL_LOCKS can't change from under us because
3150 		 * we hold the mm_all_locks_mutex.
3151 		 *
3152 		 * Operations on ->flags have to be atomic because
3153 		 * even if AS_MM_ALL_LOCKS is stable thanks to the
3154 		 * mm_all_locks_mutex, there may be other cpus
3155 		 * changing other bitflags in parallel to us.
3156 		 */
3157 		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
3158 			BUG();
3159 		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
3160 	}
3161 }
3162 
3163 /*
3164  * This operation locks against the VM for all pte/vma/mm related
3165  * operations that could ever happen on a certain mm. This includes
3166  * vmtruncate, try_to_unmap, and all page faults.
3167  *
3168  * The caller must take the mmap_sem in write mode before calling
3169  * mm_take_all_locks(). The caller isn't allowed to release the
3170  * mmap_sem until mm_drop_all_locks() returns.
3171  *
3172  * mmap_sem in write mode is required in order to block all operations
3173  * that could modify pagetables and free pages without need of
3174  * altering the vma layout. It's also needed in write mode to avoid new
3175  * anon_vmas to be associated with existing vmas.
3176  *
3177  * A single task can't take more than one mm_take_all_locks() in a row
3178  * or it would deadlock.
3179  *
3180  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3181  * mapping->flags avoid to take the same lock twice, if more than one
3182  * vma in this mm is backed by the same anon_vma or address_space.
3183  *
3184  * We can take all the locks in random order because the VM code
3185  * taking i_mmap_rwsem or anon_vma->rwsem outside the mmap_sem never
3186  * takes more than one of them in a row. Secondly we're protected
3187  * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3188  *
3189  * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3190  * that may have to take thousand of locks.
3191  *
3192  * mm_take_all_locks() can fail if it's interrupted by signals.
3193  */
3194 int mm_take_all_locks(struct mm_struct *mm)
3195 {
3196 	struct vm_area_struct *vma;
3197 	struct anon_vma_chain *avc;
3198 
3199 	BUG_ON(down_read_trylock(&mm->mmap_sem));
3200 
3201 	mutex_lock(&mm_all_locks_mutex);
3202 
3203 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3204 		if (signal_pending(current))
3205 			goto out_unlock;
3206 		if (vma->vm_file && vma->vm_file->f_mapping)
3207 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
3208 	}
3209 
3210 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3211 		if (signal_pending(current))
3212 			goto out_unlock;
3213 		if (vma->anon_vma)
3214 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3215 				vm_lock_anon_vma(mm, avc->anon_vma);
3216 	}
3217 
3218 	return 0;
3219 
3220 out_unlock:
3221 	mm_drop_all_locks(mm);
3222 	return -EINTR;
3223 }
3224 
3225 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3226 {
3227 	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3228 		/*
3229 		 * The LSB of head.next can't change to 0 from under
3230 		 * us because we hold the mm_all_locks_mutex.
3231 		 *
3232 		 * We must however clear the bitflag before unlocking
3233 		 * the vma so the users using the anon_vma->rb_root will
3234 		 * never see our bitflag.
3235 		 *
3236 		 * No need of atomic instructions here, head.next
3237 		 * can't change from under us until we release the
3238 		 * anon_vma->root->rwsem.
3239 		 */
3240 		if (!__test_and_clear_bit(0, (unsigned long *)
3241 					  &anon_vma->root->rb_root.rb_node))
3242 			BUG();
3243 		anon_vma_unlock_write(anon_vma);
3244 	}
3245 }
3246 
3247 static void vm_unlock_mapping(struct address_space *mapping)
3248 {
3249 	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3250 		/*
3251 		 * AS_MM_ALL_LOCKS can't change to 0 from under us
3252 		 * because we hold the mm_all_locks_mutex.
3253 		 */
3254 		i_mmap_unlock_write(mapping);
3255 		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3256 					&mapping->flags))
3257 			BUG();
3258 	}
3259 }
3260 
3261 /*
3262  * The mmap_sem cannot be released by the caller until
3263  * mm_drop_all_locks() returns.
3264  */
3265 void mm_drop_all_locks(struct mm_struct *mm)
3266 {
3267 	struct vm_area_struct *vma;
3268 	struct anon_vma_chain *avc;
3269 
3270 	BUG_ON(down_read_trylock(&mm->mmap_sem));
3271 	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3272 
3273 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3274 		if (vma->anon_vma)
3275 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3276 				vm_unlock_anon_vma(avc->anon_vma);
3277 		if (vma->vm_file && vma->vm_file->f_mapping)
3278 			vm_unlock_mapping(vma->vm_file->f_mapping);
3279 	}
3280 
3281 	mutex_unlock(&mm_all_locks_mutex);
3282 }
3283 
3284 /*
3285  * initialise the VMA slab
3286  */
3287 void __init mmap_init(void)
3288 {
3289 	int ret;
3290 
3291 	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
3292 	VM_BUG_ON(ret);
3293 }
3294 
3295 /*
3296  * Initialise sysctl_user_reserve_kbytes.
3297  *
3298  * This is intended to prevent a user from starting a single memory hogging
3299  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3300  * mode.
3301  *
3302  * The default value is min(3% of free memory, 128MB)
3303  * 128MB is enough to recover with sshd/login, bash, and top/kill.
3304  */
3305 static int init_user_reserve(void)
3306 {
3307 	unsigned long free_kbytes;
3308 
3309 	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3310 
3311 	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
3312 	return 0;
3313 }
3314 subsys_initcall(init_user_reserve);
3315 
3316 /*
3317  * Initialise sysctl_admin_reserve_kbytes.
3318  *
3319  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3320  * to log in and kill a memory hogging process.
3321  *
3322  * Systems with more than 256MB will reserve 8MB, enough to recover
3323  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3324  * only reserve 3% of free pages by default.
3325  */
3326 static int init_admin_reserve(void)
3327 {
3328 	unsigned long free_kbytes;
3329 
3330 	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3331 
3332 	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
3333 	return 0;
3334 }
3335 subsys_initcall(init_admin_reserve);
3336 
3337 /*
3338  * Reinititalise user and admin reserves if memory is added or removed.
3339  *
3340  * The default user reserve max is 128MB, and the default max for the
3341  * admin reserve is 8MB. These are usually, but not always, enough to
3342  * enable recovery from a memory hogging process using login/sshd, a shell,
3343  * and tools like top. It may make sense to increase or even disable the
3344  * reserve depending on the existence of swap or variations in the recovery
3345  * tools. So, the admin may have changed them.
3346  *
3347  * If memory is added and the reserves have been eliminated or increased above
3348  * the default max, then we'll trust the admin.
3349  *
3350  * If memory is removed and there isn't enough free memory, then we
3351  * need to reset the reserves.
3352  *
3353  * Otherwise keep the reserve set by the admin.
3354  */
3355 static int reserve_mem_notifier(struct notifier_block *nb,
3356 			     unsigned long action, void *data)
3357 {
3358 	unsigned long tmp, free_kbytes;
3359 
3360 	switch (action) {
3361 	case MEM_ONLINE:
3362 		/* Default max is 128MB. Leave alone if modified by operator. */
3363 		tmp = sysctl_user_reserve_kbytes;
3364 		if (0 < tmp && tmp < (1UL << 17))
3365 			init_user_reserve();
3366 
3367 		/* Default max is 8MB.  Leave alone if modified by operator. */
3368 		tmp = sysctl_admin_reserve_kbytes;
3369 		if (0 < tmp && tmp < (1UL << 13))
3370 			init_admin_reserve();
3371 
3372 		break;
3373 	case MEM_OFFLINE:
3374 		free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3375 
3376 		if (sysctl_user_reserve_kbytes > free_kbytes) {
3377 			init_user_reserve();
3378 			pr_info("vm.user_reserve_kbytes reset to %lu\n",
3379 				sysctl_user_reserve_kbytes);
3380 		}
3381 
3382 		if (sysctl_admin_reserve_kbytes > free_kbytes) {
3383 			init_admin_reserve();
3384 			pr_info("vm.admin_reserve_kbytes reset to %lu\n",
3385 				sysctl_admin_reserve_kbytes);
3386 		}
3387 		break;
3388 	default:
3389 		break;
3390 	}
3391 	return NOTIFY_OK;
3392 }
3393 
3394 static struct notifier_block reserve_mem_nb = {
3395 	.notifier_call = reserve_mem_notifier,
3396 };
3397 
3398 static int __meminit init_reserve_notifier(void)
3399 {
3400 	if (register_hotmemory_notifier(&reserve_mem_nb))
3401 		pr_err("Failed registering memory add/remove notifier for admin reserve\n");
3402 
3403 	return 0;
3404 }
3405 subsys_initcall(init_reserve_notifier);
3406