xref: /openbmc/linux/mm/mmap.c (revision b85d4594)
1 /*
2  * mm/mmap.c
3  *
4  * Written by obz.
5  *
6  * Address space accounting code	<alan@lxorguk.ukuu.org.uk>
7  */
8 
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 
11 #include <linux/kernel.h>
12 #include <linux/slab.h>
13 #include <linux/backing-dev.h>
14 #include <linux/mm.h>
15 #include <linux/vmacache.h>
16 #include <linux/shm.h>
17 #include <linux/mman.h>
18 #include <linux/pagemap.h>
19 #include <linux/swap.h>
20 #include <linux/syscalls.h>
21 #include <linux/capability.h>
22 #include <linux/init.h>
23 #include <linux/file.h>
24 #include <linux/fs.h>
25 #include <linux/personality.h>
26 #include <linux/security.h>
27 #include <linux/hugetlb.h>
28 #include <linux/profile.h>
29 #include <linux/export.h>
30 #include <linux/mount.h>
31 #include <linux/mempolicy.h>
32 #include <linux/rmap.h>
33 #include <linux/mmu_notifier.h>
34 #include <linux/mmdebug.h>
35 #include <linux/perf_event.h>
36 #include <linux/audit.h>
37 #include <linux/khugepaged.h>
38 #include <linux/uprobes.h>
39 #include <linux/rbtree_augmented.h>
40 #include <linux/sched/sysctl.h>
41 #include <linux/notifier.h>
42 #include <linux/memory.h>
43 #include <linux/printk.h>
44 #include <linux/userfaultfd_k.h>
45 
46 #include <asm/uaccess.h>
47 #include <asm/cacheflush.h>
48 #include <asm/tlb.h>
49 #include <asm/mmu_context.h>
50 
51 #include "internal.h"
52 
53 #ifndef arch_mmap_check
54 #define arch_mmap_check(addr, len, flags)	(0)
55 #endif
56 
57 #ifndef arch_rebalance_pgtables
58 #define arch_rebalance_pgtables(addr, len)		(addr)
59 #endif
60 
61 static void unmap_region(struct mm_struct *mm,
62 		struct vm_area_struct *vma, struct vm_area_struct *prev,
63 		unsigned long start, unsigned long end);
64 
65 /* description of effects of mapping type and prot in current implementation.
66  * this is due to the limited x86 page protection hardware.  The expected
67  * behavior is in parens:
68  *
69  * map_type	prot
70  *		PROT_NONE	PROT_READ	PROT_WRITE	PROT_EXEC
71  * MAP_SHARED	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
72  *		w: (no) no	w: (no) no	w: (yes) yes	w: (no) no
73  *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
74  *
75  * MAP_PRIVATE	r: (no) no	r: (yes) yes	r: (no) yes	r: (no) yes
76  *		w: (no) no	w: (no) no	w: (copy) copy	w: (no) no
77  *		x: (no) no	x: (no) yes	x: (no) yes	x: (yes) yes
78  *
79  */
80 pgprot_t protection_map[16] = {
81 	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
82 	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
83 };
84 
85 pgprot_t vm_get_page_prot(unsigned long vm_flags)
86 {
87 	return __pgprot(pgprot_val(protection_map[vm_flags &
88 				(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
89 			pgprot_val(arch_vm_get_page_prot(vm_flags)));
90 }
91 EXPORT_SYMBOL(vm_get_page_prot);
92 
93 static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
94 {
95 	return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
96 }
97 
98 /* Update vma->vm_page_prot to reflect vma->vm_flags. */
99 void vma_set_page_prot(struct vm_area_struct *vma)
100 {
101 	unsigned long vm_flags = vma->vm_flags;
102 
103 	vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
104 	if (vma_wants_writenotify(vma)) {
105 		vm_flags &= ~VM_SHARED;
106 		vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot,
107 						     vm_flags);
108 	}
109 }
110 
111 
112 int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;  /* heuristic overcommit */
113 int sysctl_overcommit_ratio __read_mostly = 50;	/* default is 50% */
114 unsigned long sysctl_overcommit_kbytes __read_mostly;
115 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
116 unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
117 unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
118 /*
119  * Make sure vm_committed_as in one cacheline and not cacheline shared with
120  * other variables. It can be updated by several CPUs frequently.
121  */
122 struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
123 
124 /*
125  * The global memory commitment made in the system can be a metric
126  * that can be used to drive ballooning decisions when Linux is hosted
127  * as a guest. On Hyper-V, the host implements a policy engine for dynamically
128  * balancing memory across competing virtual machines that are hosted.
129  * Several metrics drive this policy engine including the guest reported
130  * memory commitment.
131  */
132 unsigned long vm_memory_committed(void)
133 {
134 	return percpu_counter_read_positive(&vm_committed_as);
135 }
136 EXPORT_SYMBOL_GPL(vm_memory_committed);
137 
138 /*
139  * Check that a process has enough memory to allocate a new virtual
140  * mapping. 0 means there is enough memory for the allocation to
141  * succeed and -ENOMEM implies there is not.
142  *
143  * We currently support three overcommit policies, which are set via the
144  * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting
145  *
146  * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
147  * Additional code 2002 Jul 20 by Robert Love.
148  *
149  * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
150  *
151  * Note this is a helper function intended to be used by LSMs which
152  * wish to use this logic.
153  */
154 int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
155 {
156 	long free, allowed, reserve;
157 
158 	VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) <
159 			-(s64)vm_committed_as_batch * num_online_cpus(),
160 			"memory commitment underflow");
161 
162 	vm_acct_memory(pages);
163 
164 	/*
165 	 * Sometimes we want to use more memory than we have
166 	 */
167 	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
168 		return 0;
169 
170 	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
171 		free = global_page_state(NR_FREE_PAGES);
172 		free += global_page_state(NR_FILE_PAGES);
173 
174 		/*
175 		 * shmem pages shouldn't be counted as free in this
176 		 * case, they can't be purged, only swapped out, and
177 		 * that won't affect the overall amount of available
178 		 * memory in the system.
179 		 */
180 		free -= global_page_state(NR_SHMEM);
181 
182 		free += get_nr_swap_pages();
183 
184 		/*
185 		 * Any slabs which are created with the
186 		 * SLAB_RECLAIM_ACCOUNT flag claim to have contents
187 		 * which are reclaimable, under pressure.  The dentry
188 		 * cache and most inode caches should fall into this
189 		 */
190 		free += global_page_state(NR_SLAB_RECLAIMABLE);
191 
192 		/*
193 		 * Leave reserved pages. The pages are not for anonymous pages.
194 		 */
195 		if (free <= totalreserve_pages)
196 			goto error;
197 		else
198 			free -= totalreserve_pages;
199 
200 		/*
201 		 * Reserve some for root
202 		 */
203 		if (!cap_sys_admin)
204 			free -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
205 
206 		if (free > pages)
207 			return 0;
208 
209 		goto error;
210 	}
211 
212 	allowed = vm_commit_limit();
213 	/*
214 	 * Reserve some for root
215 	 */
216 	if (!cap_sys_admin)
217 		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
218 
219 	/*
220 	 * Don't let a single process grow so big a user can't recover
221 	 */
222 	if (mm) {
223 		reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
224 		allowed -= min_t(long, mm->total_vm / 32, reserve);
225 	}
226 
227 	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
228 		return 0;
229 error:
230 	vm_unacct_memory(pages);
231 
232 	return -ENOMEM;
233 }
234 
235 /*
236  * Requires inode->i_mapping->i_mmap_rwsem
237  */
238 static void __remove_shared_vm_struct(struct vm_area_struct *vma,
239 		struct file *file, struct address_space *mapping)
240 {
241 	if (vma->vm_flags & VM_DENYWRITE)
242 		atomic_inc(&file_inode(file)->i_writecount);
243 	if (vma->vm_flags & VM_SHARED)
244 		mapping_unmap_writable(mapping);
245 
246 	flush_dcache_mmap_lock(mapping);
247 	vma_interval_tree_remove(vma, &mapping->i_mmap);
248 	flush_dcache_mmap_unlock(mapping);
249 }
250 
251 /*
252  * Unlink a file-based vm structure from its interval tree, to hide
253  * vma from rmap and vmtruncate before freeing its page tables.
254  */
255 void unlink_file_vma(struct vm_area_struct *vma)
256 {
257 	struct file *file = vma->vm_file;
258 
259 	if (file) {
260 		struct address_space *mapping = file->f_mapping;
261 		i_mmap_lock_write(mapping);
262 		__remove_shared_vm_struct(vma, file, mapping);
263 		i_mmap_unlock_write(mapping);
264 	}
265 }
266 
267 /*
268  * Close a vm structure and free it, returning the next.
269  */
270 static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
271 {
272 	struct vm_area_struct *next = vma->vm_next;
273 
274 	might_sleep();
275 	if (vma->vm_ops && vma->vm_ops->close)
276 		vma->vm_ops->close(vma);
277 	if (vma->vm_file)
278 		fput(vma->vm_file);
279 	mpol_put(vma_policy(vma));
280 	kmem_cache_free(vm_area_cachep, vma);
281 	return next;
282 }
283 
284 static unsigned long do_brk(unsigned long addr, unsigned long len);
285 
286 SYSCALL_DEFINE1(brk, unsigned long, brk)
287 {
288 	unsigned long retval;
289 	unsigned long newbrk, oldbrk;
290 	struct mm_struct *mm = current->mm;
291 	unsigned long min_brk;
292 	bool populate;
293 
294 	down_write(&mm->mmap_sem);
295 
296 #ifdef CONFIG_COMPAT_BRK
297 	/*
298 	 * CONFIG_COMPAT_BRK can still be overridden by setting
299 	 * randomize_va_space to 2, which will still cause mm->start_brk
300 	 * to be arbitrarily shifted
301 	 */
302 	if (current->brk_randomized)
303 		min_brk = mm->start_brk;
304 	else
305 		min_brk = mm->end_data;
306 #else
307 	min_brk = mm->start_brk;
308 #endif
309 	if (brk < min_brk)
310 		goto out;
311 
312 	/*
313 	 * Check against rlimit here. If this check is done later after the test
314 	 * of oldbrk with newbrk then it can escape the test and let the data
315 	 * segment grow beyond its set limit the in case where the limit is
316 	 * not page aligned -Ram Gupta
317 	 */
318 	if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
319 			      mm->end_data, mm->start_data))
320 		goto out;
321 
322 	newbrk = PAGE_ALIGN(brk);
323 	oldbrk = PAGE_ALIGN(mm->brk);
324 	if (oldbrk == newbrk)
325 		goto set_brk;
326 
327 	/* Always allow shrinking brk. */
328 	if (brk <= mm->brk) {
329 		if (!do_munmap(mm, newbrk, oldbrk-newbrk))
330 			goto set_brk;
331 		goto out;
332 	}
333 
334 	/* Check against existing mmap mappings. */
335 	if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
336 		goto out;
337 
338 	/* Ok, looks good - let it rip. */
339 	if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
340 		goto out;
341 
342 set_brk:
343 	mm->brk = brk;
344 	populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
345 	up_write(&mm->mmap_sem);
346 	if (populate)
347 		mm_populate(oldbrk, newbrk - oldbrk);
348 	return brk;
349 
350 out:
351 	retval = mm->brk;
352 	up_write(&mm->mmap_sem);
353 	return retval;
354 }
355 
356 static long vma_compute_subtree_gap(struct vm_area_struct *vma)
357 {
358 	unsigned long max, subtree_gap;
359 	max = vma->vm_start;
360 	if (vma->vm_prev)
361 		max -= vma->vm_prev->vm_end;
362 	if (vma->vm_rb.rb_left) {
363 		subtree_gap = rb_entry(vma->vm_rb.rb_left,
364 				struct vm_area_struct, vm_rb)->rb_subtree_gap;
365 		if (subtree_gap > max)
366 			max = subtree_gap;
367 	}
368 	if (vma->vm_rb.rb_right) {
369 		subtree_gap = rb_entry(vma->vm_rb.rb_right,
370 				struct vm_area_struct, vm_rb)->rb_subtree_gap;
371 		if (subtree_gap > max)
372 			max = subtree_gap;
373 	}
374 	return max;
375 }
376 
377 #ifdef CONFIG_DEBUG_VM_RB
378 static int browse_rb(struct rb_root *root)
379 {
380 	int i = 0, j, bug = 0;
381 	struct rb_node *nd, *pn = NULL;
382 	unsigned long prev = 0, pend = 0;
383 
384 	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
385 		struct vm_area_struct *vma;
386 		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
387 		if (vma->vm_start < prev) {
388 			pr_emerg("vm_start %lx < prev %lx\n",
389 				  vma->vm_start, prev);
390 			bug = 1;
391 		}
392 		if (vma->vm_start < pend) {
393 			pr_emerg("vm_start %lx < pend %lx\n",
394 				  vma->vm_start, pend);
395 			bug = 1;
396 		}
397 		if (vma->vm_start > vma->vm_end) {
398 			pr_emerg("vm_start %lx > vm_end %lx\n",
399 				  vma->vm_start, vma->vm_end);
400 			bug = 1;
401 		}
402 		if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
403 			pr_emerg("free gap %lx, correct %lx\n",
404 			       vma->rb_subtree_gap,
405 			       vma_compute_subtree_gap(vma));
406 			bug = 1;
407 		}
408 		i++;
409 		pn = nd;
410 		prev = vma->vm_start;
411 		pend = vma->vm_end;
412 	}
413 	j = 0;
414 	for (nd = pn; nd; nd = rb_prev(nd))
415 		j++;
416 	if (i != j) {
417 		pr_emerg("backwards %d, forwards %d\n", j, i);
418 		bug = 1;
419 	}
420 	return bug ? -1 : i;
421 }
422 
423 static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
424 {
425 	struct rb_node *nd;
426 
427 	for (nd = rb_first(root); nd; nd = rb_next(nd)) {
428 		struct vm_area_struct *vma;
429 		vma = rb_entry(nd, struct vm_area_struct, vm_rb);
430 		VM_BUG_ON_VMA(vma != ignore &&
431 			vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
432 			vma);
433 	}
434 }
435 
436 static void validate_mm(struct mm_struct *mm)
437 {
438 	int bug = 0;
439 	int i = 0;
440 	unsigned long highest_address = 0;
441 	struct vm_area_struct *vma = mm->mmap;
442 
443 	while (vma) {
444 		struct anon_vma_chain *avc;
445 
446 		vma_lock_anon_vma(vma);
447 		list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
448 			anon_vma_interval_tree_verify(avc);
449 		vma_unlock_anon_vma(vma);
450 		highest_address = vma->vm_end;
451 		vma = vma->vm_next;
452 		i++;
453 	}
454 	if (i != mm->map_count) {
455 		pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
456 		bug = 1;
457 	}
458 	if (highest_address != mm->highest_vm_end) {
459 		pr_emerg("mm->highest_vm_end %lx, found %lx\n",
460 			  mm->highest_vm_end, highest_address);
461 		bug = 1;
462 	}
463 	i = browse_rb(&mm->mm_rb);
464 	if (i != mm->map_count) {
465 		if (i != -1)
466 			pr_emerg("map_count %d rb %d\n", mm->map_count, i);
467 		bug = 1;
468 	}
469 	VM_BUG_ON_MM(bug, mm);
470 }
471 #else
472 #define validate_mm_rb(root, ignore) do { } while (0)
473 #define validate_mm(mm) do { } while (0)
474 #endif
475 
476 RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
477 		     unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
478 
479 /*
480  * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
481  * vma->vm_prev->vm_end values changed, without modifying the vma's position
482  * in the rbtree.
483  */
484 static void vma_gap_update(struct vm_area_struct *vma)
485 {
486 	/*
487 	 * As it turns out, RB_DECLARE_CALLBACKS() already created a callback
488 	 * function that does exacltly what we want.
489 	 */
490 	vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
491 }
492 
493 static inline void vma_rb_insert(struct vm_area_struct *vma,
494 				 struct rb_root *root)
495 {
496 	/* All rb_subtree_gap values must be consistent prior to insertion */
497 	validate_mm_rb(root, NULL);
498 
499 	rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
500 }
501 
502 static void vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
503 {
504 	/*
505 	 * All rb_subtree_gap values must be consistent prior to erase,
506 	 * with the possible exception of the vma being erased.
507 	 */
508 	validate_mm_rb(root, vma);
509 
510 	/*
511 	 * Note rb_erase_augmented is a fairly large inline function,
512 	 * so make sure we instantiate it only once with our desired
513 	 * augmented rbtree callbacks.
514 	 */
515 	rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
516 }
517 
518 /*
519  * vma has some anon_vma assigned, and is already inserted on that
520  * anon_vma's interval trees.
521  *
522  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
523  * vma must be removed from the anon_vma's interval trees using
524  * anon_vma_interval_tree_pre_update_vma().
525  *
526  * After the update, the vma will be reinserted using
527  * anon_vma_interval_tree_post_update_vma().
528  *
529  * The entire update must be protected by exclusive mmap_sem and by
530  * the root anon_vma's mutex.
531  */
532 static inline void
533 anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
534 {
535 	struct anon_vma_chain *avc;
536 
537 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
538 		anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
539 }
540 
541 static inline void
542 anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
543 {
544 	struct anon_vma_chain *avc;
545 
546 	list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
547 		anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
548 }
549 
550 static int find_vma_links(struct mm_struct *mm, unsigned long addr,
551 		unsigned long end, struct vm_area_struct **pprev,
552 		struct rb_node ***rb_link, struct rb_node **rb_parent)
553 {
554 	struct rb_node **__rb_link, *__rb_parent, *rb_prev;
555 
556 	__rb_link = &mm->mm_rb.rb_node;
557 	rb_prev = __rb_parent = NULL;
558 
559 	while (*__rb_link) {
560 		struct vm_area_struct *vma_tmp;
561 
562 		__rb_parent = *__rb_link;
563 		vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
564 
565 		if (vma_tmp->vm_end > addr) {
566 			/* Fail if an existing vma overlaps the area */
567 			if (vma_tmp->vm_start < end)
568 				return -ENOMEM;
569 			__rb_link = &__rb_parent->rb_left;
570 		} else {
571 			rb_prev = __rb_parent;
572 			__rb_link = &__rb_parent->rb_right;
573 		}
574 	}
575 
576 	*pprev = NULL;
577 	if (rb_prev)
578 		*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
579 	*rb_link = __rb_link;
580 	*rb_parent = __rb_parent;
581 	return 0;
582 }
583 
584 static unsigned long count_vma_pages_range(struct mm_struct *mm,
585 		unsigned long addr, unsigned long end)
586 {
587 	unsigned long nr_pages = 0;
588 	struct vm_area_struct *vma;
589 
590 	/* Find first overlaping mapping */
591 	vma = find_vma_intersection(mm, addr, end);
592 	if (!vma)
593 		return 0;
594 
595 	nr_pages = (min(end, vma->vm_end) -
596 		max(addr, vma->vm_start)) >> PAGE_SHIFT;
597 
598 	/* Iterate over the rest of the overlaps */
599 	for (vma = vma->vm_next; vma; vma = vma->vm_next) {
600 		unsigned long overlap_len;
601 
602 		if (vma->vm_start > end)
603 			break;
604 
605 		overlap_len = min(end, vma->vm_end) - vma->vm_start;
606 		nr_pages += overlap_len >> PAGE_SHIFT;
607 	}
608 
609 	return nr_pages;
610 }
611 
612 void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
613 		struct rb_node **rb_link, struct rb_node *rb_parent)
614 {
615 	/* Update tracking information for the gap following the new vma. */
616 	if (vma->vm_next)
617 		vma_gap_update(vma->vm_next);
618 	else
619 		mm->highest_vm_end = vma->vm_end;
620 
621 	/*
622 	 * vma->vm_prev wasn't known when we followed the rbtree to find the
623 	 * correct insertion point for that vma. As a result, we could not
624 	 * update the vma vm_rb parents rb_subtree_gap values on the way down.
625 	 * So, we first insert the vma with a zero rb_subtree_gap value
626 	 * (to be consistent with what we did on the way down), and then
627 	 * immediately update the gap to the correct value. Finally we
628 	 * rebalance the rbtree after all augmented values have been set.
629 	 */
630 	rb_link_node(&vma->vm_rb, rb_parent, rb_link);
631 	vma->rb_subtree_gap = 0;
632 	vma_gap_update(vma);
633 	vma_rb_insert(vma, &mm->mm_rb);
634 }
635 
636 static void __vma_link_file(struct vm_area_struct *vma)
637 {
638 	struct file *file;
639 
640 	file = vma->vm_file;
641 	if (file) {
642 		struct address_space *mapping = file->f_mapping;
643 
644 		if (vma->vm_flags & VM_DENYWRITE)
645 			atomic_dec(&file_inode(file)->i_writecount);
646 		if (vma->vm_flags & VM_SHARED)
647 			atomic_inc(&mapping->i_mmap_writable);
648 
649 		flush_dcache_mmap_lock(mapping);
650 		vma_interval_tree_insert(vma, &mapping->i_mmap);
651 		flush_dcache_mmap_unlock(mapping);
652 	}
653 }
654 
655 static void
656 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
657 	struct vm_area_struct *prev, struct rb_node **rb_link,
658 	struct rb_node *rb_parent)
659 {
660 	__vma_link_list(mm, vma, prev, rb_parent);
661 	__vma_link_rb(mm, vma, rb_link, rb_parent);
662 }
663 
664 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
665 			struct vm_area_struct *prev, struct rb_node **rb_link,
666 			struct rb_node *rb_parent)
667 {
668 	struct address_space *mapping = NULL;
669 
670 	if (vma->vm_file) {
671 		mapping = vma->vm_file->f_mapping;
672 		i_mmap_lock_write(mapping);
673 	}
674 
675 	__vma_link(mm, vma, prev, rb_link, rb_parent);
676 	__vma_link_file(vma);
677 
678 	if (mapping)
679 		i_mmap_unlock_write(mapping);
680 
681 	mm->map_count++;
682 	validate_mm(mm);
683 }
684 
685 /*
686  * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
687  * mm's list and rbtree.  It has already been inserted into the interval tree.
688  */
689 static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
690 {
691 	struct vm_area_struct *prev;
692 	struct rb_node **rb_link, *rb_parent;
693 
694 	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
695 			   &prev, &rb_link, &rb_parent))
696 		BUG();
697 	__vma_link(mm, vma, prev, rb_link, rb_parent);
698 	mm->map_count++;
699 }
700 
701 static inline void
702 __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
703 		struct vm_area_struct *prev)
704 {
705 	struct vm_area_struct *next;
706 
707 	vma_rb_erase(vma, &mm->mm_rb);
708 	prev->vm_next = next = vma->vm_next;
709 	if (next)
710 		next->vm_prev = prev;
711 
712 	/* Kill the cache */
713 	vmacache_invalidate(mm);
714 }
715 
716 /*
717  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
718  * is already present in an i_mmap tree without adjusting the tree.
719  * The following helper function should be used when such adjustments
720  * are necessary.  The "insert" vma (if any) is to be inserted
721  * before we drop the necessary locks.
722  */
723 int vma_adjust(struct vm_area_struct *vma, unsigned long start,
724 	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
725 {
726 	struct mm_struct *mm = vma->vm_mm;
727 	struct vm_area_struct *next = vma->vm_next;
728 	struct vm_area_struct *importer = NULL;
729 	struct address_space *mapping = NULL;
730 	struct rb_root *root = NULL;
731 	struct anon_vma *anon_vma = NULL;
732 	struct file *file = vma->vm_file;
733 	bool start_changed = false, end_changed = false;
734 	long adjust_next = 0;
735 	int remove_next = 0;
736 
737 	if (next && !insert) {
738 		struct vm_area_struct *exporter = NULL;
739 
740 		if (end >= next->vm_end) {
741 			/*
742 			 * vma expands, overlapping all the next, and
743 			 * perhaps the one after too (mprotect case 6).
744 			 */
745 again:			remove_next = 1 + (end > next->vm_end);
746 			end = next->vm_end;
747 			exporter = next;
748 			importer = vma;
749 		} else if (end > next->vm_start) {
750 			/*
751 			 * vma expands, overlapping part of the next:
752 			 * mprotect case 5 shifting the boundary up.
753 			 */
754 			adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
755 			exporter = next;
756 			importer = vma;
757 		} else if (end < vma->vm_end) {
758 			/*
759 			 * vma shrinks, and !insert tells it's not
760 			 * split_vma inserting another: so it must be
761 			 * mprotect case 4 shifting the boundary down.
762 			 */
763 			adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
764 			exporter = vma;
765 			importer = next;
766 		}
767 
768 		/*
769 		 * Easily overlooked: when mprotect shifts the boundary,
770 		 * make sure the expanding vma has anon_vma set if the
771 		 * shrinking vma had, to cover any anon pages imported.
772 		 */
773 		if (exporter && exporter->anon_vma && !importer->anon_vma) {
774 			int error;
775 
776 			importer->anon_vma = exporter->anon_vma;
777 			error = anon_vma_clone(importer, exporter);
778 			if (error)
779 				return error;
780 		}
781 	}
782 
783 	if (file) {
784 		mapping = file->f_mapping;
785 		root = &mapping->i_mmap;
786 		uprobe_munmap(vma, vma->vm_start, vma->vm_end);
787 
788 		if (adjust_next)
789 			uprobe_munmap(next, next->vm_start, next->vm_end);
790 
791 		i_mmap_lock_write(mapping);
792 		if (insert) {
793 			/*
794 			 * Put into interval tree now, so instantiated pages
795 			 * are visible to arm/parisc __flush_dcache_page
796 			 * throughout; but we cannot insert into address
797 			 * space until vma start or end is updated.
798 			 */
799 			__vma_link_file(insert);
800 		}
801 	}
802 
803 	vma_adjust_trans_huge(vma, start, end, adjust_next);
804 
805 	anon_vma = vma->anon_vma;
806 	if (!anon_vma && adjust_next)
807 		anon_vma = next->anon_vma;
808 	if (anon_vma) {
809 		VM_BUG_ON_VMA(adjust_next && next->anon_vma &&
810 			  anon_vma != next->anon_vma, next);
811 		anon_vma_lock_write(anon_vma);
812 		anon_vma_interval_tree_pre_update_vma(vma);
813 		if (adjust_next)
814 			anon_vma_interval_tree_pre_update_vma(next);
815 	}
816 
817 	if (root) {
818 		flush_dcache_mmap_lock(mapping);
819 		vma_interval_tree_remove(vma, root);
820 		if (adjust_next)
821 			vma_interval_tree_remove(next, root);
822 	}
823 
824 	if (start != vma->vm_start) {
825 		vma->vm_start = start;
826 		start_changed = true;
827 	}
828 	if (end != vma->vm_end) {
829 		vma->vm_end = end;
830 		end_changed = true;
831 	}
832 	vma->vm_pgoff = pgoff;
833 	if (adjust_next) {
834 		next->vm_start += adjust_next << PAGE_SHIFT;
835 		next->vm_pgoff += adjust_next;
836 	}
837 
838 	if (root) {
839 		if (adjust_next)
840 			vma_interval_tree_insert(next, root);
841 		vma_interval_tree_insert(vma, root);
842 		flush_dcache_mmap_unlock(mapping);
843 	}
844 
845 	if (remove_next) {
846 		/*
847 		 * vma_merge has merged next into vma, and needs
848 		 * us to remove next before dropping the locks.
849 		 */
850 		__vma_unlink(mm, next, vma);
851 		if (file)
852 			__remove_shared_vm_struct(next, file, mapping);
853 	} else if (insert) {
854 		/*
855 		 * split_vma has split insert from vma, and needs
856 		 * us to insert it before dropping the locks
857 		 * (it may either follow vma or precede it).
858 		 */
859 		__insert_vm_struct(mm, insert);
860 	} else {
861 		if (start_changed)
862 			vma_gap_update(vma);
863 		if (end_changed) {
864 			if (!next)
865 				mm->highest_vm_end = end;
866 			else if (!adjust_next)
867 				vma_gap_update(next);
868 		}
869 	}
870 
871 	if (anon_vma) {
872 		anon_vma_interval_tree_post_update_vma(vma);
873 		if (adjust_next)
874 			anon_vma_interval_tree_post_update_vma(next);
875 		anon_vma_unlock_write(anon_vma);
876 	}
877 	if (mapping)
878 		i_mmap_unlock_write(mapping);
879 
880 	if (root) {
881 		uprobe_mmap(vma);
882 
883 		if (adjust_next)
884 			uprobe_mmap(next);
885 	}
886 
887 	if (remove_next) {
888 		if (file) {
889 			uprobe_munmap(next, next->vm_start, next->vm_end);
890 			fput(file);
891 		}
892 		if (next->anon_vma)
893 			anon_vma_merge(vma, next);
894 		mm->map_count--;
895 		mpol_put(vma_policy(next));
896 		kmem_cache_free(vm_area_cachep, next);
897 		/*
898 		 * In mprotect's case 6 (see comments on vma_merge),
899 		 * we must remove another next too. It would clutter
900 		 * up the code too much to do both in one go.
901 		 */
902 		next = vma->vm_next;
903 		if (remove_next == 2)
904 			goto again;
905 		else if (next)
906 			vma_gap_update(next);
907 		else
908 			mm->highest_vm_end = end;
909 	}
910 	if (insert && file)
911 		uprobe_mmap(insert);
912 
913 	validate_mm(mm);
914 
915 	return 0;
916 }
917 
918 /*
919  * If the vma has a ->close operation then the driver probably needs to release
920  * per-vma resources, so we don't attempt to merge those.
921  */
922 static inline int is_mergeable_vma(struct vm_area_struct *vma,
923 				struct file *file, unsigned long vm_flags,
924 				struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
925 {
926 	/*
927 	 * VM_SOFTDIRTY should not prevent from VMA merging, if we
928 	 * match the flags but dirty bit -- the caller should mark
929 	 * merged VMA as dirty. If dirty bit won't be excluded from
930 	 * comparison, we increase pressue on the memory system forcing
931 	 * the kernel to generate new VMAs when old one could be
932 	 * extended instead.
933 	 */
934 	if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
935 		return 0;
936 	if (vma->vm_file != file)
937 		return 0;
938 	if (vma->vm_ops && vma->vm_ops->close)
939 		return 0;
940 	if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
941 		return 0;
942 	return 1;
943 }
944 
945 static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
946 					struct anon_vma *anon_vma2,
947 					struct vm_area_struct *vma)
948 {
949 	/*
950 	 * The list_is_singular() test is to avoid merging VMA cloned from
951 	 * parents. This can improve scalability caused by anon_vma lock.
952 	 */
953 	if ((!anon_vma1 || !anon_vma2) && (!vma ||
954 		list_is_singular(&vma->anon_vma_chain)))
955 		return 1;
956 	return anon_vma1 == anon_vma2;
957 }
958 
959 /*
960  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
961  * in front of (at a lower virtual address and file offset than) the vma.
962  *
963  * We cannot merge two vmas if they have differently assigned (non-NULL)
964  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
965  *
966  * We don't check here for the merged mmap wrapping around the end of pagecache
967  * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
968  * wrap, nor mmaps which cover the final page at index -1UL.
969  */
970 static int
971 can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
972 		     struct anon_vma *anon_vma, struct file *file,
973 		     pgoff_t vm_pgoff,
974 		     struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
975 {
976 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
977 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
978 		if (vma->vm_pgoff == vm_pgoff)
979 			return 1;
980 	}
981 	return 0;
982 }
983 
984 /*
985  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
986  * beyond (at a higher virtual address and file offset than) the vma.
987  *
988  * We cannot merge two vmas if they have differently assigned (non-NULL)
989  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
990  */
991 static int
992 can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
993 		    struct anon_vma *anon_vma, struct file *file,
994 		    pgoff_t vm_pgoff,
995 		    struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
996 {
997 	if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
998 	    is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
999 		pgoff_t vm_pglen;
1000 		vm_pglen = vma_pages(vma);
1001 		if (vma->vm_pgoff + vm_pglen == vm_pgoff)
1002 			return 1;
1003 	}
1004 	return 0;
1005 }
1006 
1007 /*
1008  * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
1009  * whether that can be merged with its predecessor or its successor.
1010  * Or both (it neatly fills a hole).
1011  *
1012  * In most cases - when called for mmap, brk or mremap - [addr,end) is
1013  * certain not to be mapped by the time vma_merge is called; but when
1014  * called for mprotect, it is certain to be already mapped (either at
1015  * an offset within prev, or at the start of next), and the flags of
1016  * this area are about to be changed to vm_flags - and the no-change
1017  * case has already been eliminated.
1018  *
1019  * The following mprotect cases have to be considered, where AAAA is
1020  * the area passed down from mprotect_fixup, never extending beyond one
1021  * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
1022  *
1023  *     AAAA             AAAA                AAAA          AAAA
1024  *    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPPPNNNNNN    PPPPNNNNXXXX
1025  *    cannot merge    might become    might become    might become
1026  *                    PPNNNNNNNNNN    PPPPPPPPPPNN    PPPPPPPPPPPP 6 or
1027  *    mmap, brk or    case 4 below    case 5 below    PPPPPPPPXXXX 7 or
1028  *    mremap move:                                    PPPPNNNNNNNN 8
1029  *        AAAA
1030  *    PPPP    NNNN    PPPPPPPPPPPP    PPPPPPPPNNNN    PPPPNNNNNNNN
1031  *    might become    case 1 below    case 2 below    case 3 below
1032  *
1033  * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
1034  * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
1035  */
1036 struct vm_area_struct *vma_merge(struct mm_struct *mm,
1037 			struct vm_area_struct *prev, unsigned long addr,
1038 			unsigned long end, unsigned long vm_flags,
1039 			struct anon_vma *anon_vma, struct file *file,
1040 			pgoff_t pgoff, struct mempolicy *policy,
1041 			struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
1042 {
1043 	pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
1044 	struct vm_area_struct *area, *next;
1045 	int err;
1046 
1047 	/*
1048 	 * We later require that vma->vm_flags == vm_flags,
1049 	 * so this tests vma->vm_flags & VM_SPECIAL, too.
1050 	 */
1051 	if (vm_flags & VM_SPECIAL)
1052 		return NULL;
1053 
1054 	if (prev)
1055 		next = prev->vm_next;
1056 	else
1057 		next = mm->mmap;
1058 	area = next;
1059 	if (next && next->vm_end == end)		/* cases 6, 7, 8 */
1060 		next = next->vm_next;
1061 
1062 	/*
1063 	 * Can it merge with the predecessor?
1064 	 */
1065 	if (prev && prev->vm_end == addr &&
1066 			mpol_equal(vma_policy(prev), policy) &&
1067 			can_vma_merge_after(prev, vm_flags,
1068 					    anon_vma, file, pgoff,
1069 					    vm_userfaultfd_ctx)) {
1070 		/*
1071 		 * OK, it can.  Can we now merge in the successor as well?
1072 		 */
1073 		if (next && end == next->vm_start &&
1074 				mpol_equal(policy, vma_policy(next)) &&
1075 				can_vma_merge_before(next, vm_flags,
1076 						     anon_vma, file,
1077 						     pgoff+pglen,
1078 						     vm_userfaultfd_ctx) &&
1079 				is_mergeable_anon_vma(prev->anon_vma,
1080 						      next->anon_vma, NULL)) {
1081 							/* cases 1, 6 */
1082 			err = vma_adjust(prev, prev->vm_start,
1083 				next->vm_end, prev->vm_pgoff, NULL);
1084 		} else					/* cases 2, 5, 7 */
1085 			err = vma_adjust(prev, prev->vm_start,
1086 				end, prev->vm_pgoff, NULL);
1087 		if (err)
1088 			return NULL;
1089 		khugepaged_enter_vma_merge(prev, vm_flags);
1090 		return prev;
1091 	}
1092 
1093 	/*
1094 	 * Can this new request be merged in front of next?
1095 	 */
1096 	if (next && end == next->vm_start &&
1097 			mpol_equal(policy, vma_policy(next)) &&
1098 			can_vma_merge_before(next, vm_flags,
1099 					     anon_vma, file, pgoff+pglen,
1100 					     vm_userfaultfd_ctx)) {
1101 		if (prev && addr < prev->vm_end)	/* case 4 */
1102 			err = vma_adjust(prev, prev->vm_start,
1103 				addr, prev->vm_pgoff, NULL);
1104 		else					/* cases 3, 8 */
1105 			err = vma_adjust(area, addr, next->vm_end,
1106 				next->vm_pgoff - pglen, NULL);
1107 		if (err)
1108 			return NULL;
1109 		khugepaged_enter_vma_merge(area, vm_flags);
1110 		return area;
1111 	}
1112 
1113 	return NULL;
1114 }
1115 
1116 /*
1117  * Rough compatbility check to quickly see if it's even worth looking
1118  * at sharing an anon_vma.
1119  *
1120  * They need to have the same vm_file, and the flags can only differ
1121  * in things that mprotect may change.
1122  *
1123  * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
1124  * we can merge the two vma's. For example, we refuse to merge a vma if
1125  * there is a vm_ops->close() function, because that indicates that the
1126  * driver is doing some kind of reference counting. But that doesn't
1127  * really matter for the anon_vma sharing case.
1128  */
1129 static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
1130 {
1131 	return a->vm_end == b->vm_start &&
1132 		mpol_equal(vma_policy(a), vma_policy(b)) &&
1133 		a->vm_file == b->vm_file &&
1134 		!((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
1135 		b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
1136 }
1137 
1138 /*
1139  * Do some basic sanity checking to see if we can re-use the anon_vma
1140  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
1141  * the same as 'old', the other will be the new one that is trying
1142  * to share the anon_vma.
1143  *
1144  * NOTE! This runs with mm_sem held for reading, so it is possible that
1145  * the anon_vma of 'old' is concurrently in the process of being set up
1146  * by another page fault trying to merge _that_. But that's ok: if it
1147  * is being set up, that automatically means that it will be a singleton
1148  * acceptable for merging, so we can do all of this optimistically. But
1149  * we do that READ_ONCE() to make sure that we never re-load the pointer.
1150  *
1151  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
1152  * matters for the 'stable anon_vma' case (ie the thing we want to avoid
1153  * is to return an anon_vma that is "complex" due to having gone through
1154  * a fork).
1155  *
1156  * We also make sure that the two vma's are compatible (adjacent,
1157  * and with the same memory policies). That's all stable, even with just
1158  * a read lock on the mm_sem.
1159  */
1160 static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
1161 {
1162 	if (anon_vma_compatible(a, b)) {
1163 		struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
1164 
1165 		if (anon_vma && list_is_singular(&old->anon_vma_chain))
1166 			return anon_vma;
1167 	}
1168 	return NULL;
1169 }
1170 
1171 /*
1172  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
1173  * neighbouring vmas for a suitable anon_vma, before it goes off
1174  * to allocate a new anon_vma.  It checks because a repetitive
1175  * sequence of mprotects and faults may otherwise lead to distinct
1176  * anon_vmas being allocated, preventing vma merge in subsequent
1177  * mprotect.
1178  */
1179 struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
1180 {
1181 	struct anon_vma *anon_vma;
1182 	struct vm_area_struct *near;
1183 
1184 	near = vma->vm_next;
1185 	if (!near)
1186 		goto try_prev;
1187 
1188 	anon_vma = reusable_anon_vma(near, vma, near);
1189 	if (anon_vma)
1190 		return anon_vma;
1191 try_prev:
1192 	near = vma->vm_prev;
1193 	if (!near)
1194 		goto none;
1195 
1196 	anon_vma = reusable_anon_vma(near, near, vma);
1197 	if (anon_vma)
1198 		return anon_vma;
1199 none:
1200 	/*
1201 	 * There's no absolute need to look only at touching neighbours:
1202 	 * we could search further afield for "compatible" anon_vmas.
1203 	 * But it would probably just be a waste of time searching,
1204 	 * or lead to too many vmas hanging off the same anon_vma.
1205 	 * We're trying to allow mprotect remerging later on,
1206 	 * not trying to minimize memory used for anon_vmas.
1207 	 */
1208 	return NULL;
1209 }
1210 
1211 #ifdef CONFIG_PROC_FS
1212 void vm_stat_account(struct mm_struct *mm, unsigned long flags,
1213 						struct file *file, long pages)
1214 {
1215 	const unsigned long stack_flags
1216 		= VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
1217 
1218 	mm->total_vm += pages;
1219 
1220 	if (file) {
1221 		mm->shared_vm += pages;
1222 		if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
1223 			mm->exec_vm += pages;
1224 	} else if (flags & stack_flags)
1225 		mm->stack_vm += pages;
1226 }
1227 #endif /* CONFIG_PROC_FS */
1228 
1229 /*
1230  * If a hint addr is less than mmap_min_addr change hint to be as
1231  * low as possible but still greater than mmap_min_addr
1232  */
1233 static inline unsigned long round_hint_to_min(unsigned long hint)
1234 {
1235 	hint &= PAGE_MASK;
1236 	if (((void *)hint != NULL) &&
1237 	    (hint < mmap_min_addr))
1238 		return PAGE_ALIGN(mmap_min_addr);
1239 	return hint;
1240 }
1241 
1242 static inline int mlock_future_check(struct mm_struct *mm,
1243 				     unsigned long flags,
1244 				     unsigned long len)
1245 {
1246 	unsigned long locked, lock_limit;
1247 
1248 	/*  mlock MCL_FUTURE? */
1249 	if (flags & VM_LOCKED) {
1250 		locked = len >> PAGE_SHIFT;
1251 		locked += mm->locked_vm;
1252 		lock_limit = rlimit(RLIMIT_MEMLOCK);
1253 		lock_limit >>= PAGE_SHIFT;
1254 		if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1255 			return -EAGAIN;
1256 	}
1257 	return 0;
1258 }
1259 
1260 /*
1261  * The caller must hold down_write(&current->mm->mmap_sem).
1262  */
1263 unsigned long do_mmap(struct file *file, unsigned long addr,
1264 			unsigned long len, unsigned long prot,
1265 			unsigned long flags, vm_flags_t vm_flags,
1266 			unsigned long pgoff, unsigned long *populate)
1267 {
1268 	struct mm_struct *mm = current->mm;
1269 
1270 	*populate = 0;
1271 
1272 	if (!len)
1273 		return -EINVAL;
1274 
1275 	/*
1276 	 * Does the application expect PROT_READ to imply PROT_EXEC?
1277 	 *
1278 	 * (the exception is when the underlying filesystem is noexec
1279 	 *  mounted, in which case we dont add PROT_EXEC.)
1280 	 */
1281 	if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
1282 		if (!(file && path_noexec(&file->f_path)))
1283 			prot |= PROT_EXEC;
1284 
1285 	if (!(flags & MAP_FIXED))
1286 		addr = round_hint_to_min(addr);
1287 
1288 	/* Careful about overflows.. */
1289 	len = PAGE_ALIGN(len);
1290 	if (!len)
1291 		return -ENOMEM;
1292 
1293 	/* offset overflow? */
1294 	if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1295 		return -EOVERFLOW;
1296 
1297 	/* Too many mappings? */
1298 	if (mm->map_count > sysctl_max_map_count)
1299 		return -ENOMEM;
1300 
1301 	/* Obtain the address to map to. we verify (or select) it and ensure
1302 	 * that it represents a valid section of the address space.
1303 	 */
1304 	addr = get_unmapped_area(file, addr, len, pgoff, flags);
1305 	if (addr & ~PAGE_MASK)
1306 		return addr;
1307 
1308 	/* Do simple checking here so the lower-level routines won't have
1309 	 * to. we assume access permissions have been handled by the open
1310 	 * of the memory object, so we don't do any here.
1311 	 */
1312 	vm_flags |= calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
1313 			mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1314 
1315 	if (flags & MAP_LOCKED)
1316 		if (!can_do_mlock())
1317 			return -EPERM;
1318 
1319 	if (mlock_future_check(mm, vm_flags, len))
1320 		return -EAGAIN;
1321 
1322 	if (file) {
1323 		struct inode *inode = file_inode(file);
1324 
1325 		switch (flags & MAP_TYPE) {
1326 		case MAP_SHARED:
1327 			if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1328 				return -EACCES;
1329 
1330 			/*
1331 			 * Make sure we don't allow writing to an append-only
1332 			 * file..
1333 			 */
1334 			if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1335 				return -EACCES;
1336 
1337 			/*
1338 			 * Make sure there are no mandatory locks on the file.
1339 			 */
1340 			if (locks_verify_locked(file))
1341 				return -EAGAIN;
1342 
1343 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1344 			if (!(file->f_mode & FMODE_WRITE))
1345 				vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1346 
1347 			/* fall through */
1348 		case MAP_PRIVATE:
1349 			if (!(file->f_mode & FMODE_READ))
1350 				return -EACCES;
1351 			if (path_noexec(&file->f_path)) {
1352 				if (vm_flags & VM_EXEC)
1353 					return -EPERM;
1354 				vm_flags &= ~VM_MAYEXEC;
1355 			}
1356 
1357 			if (!file->f_op->mmap)
1358 				return -ENODEV;
1359 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1360 				return -EINVAL;
1361 			break;
1362 
1363 		default:
1364 			return -EINVAL;
1365 		}
1366 	} else {
1367 		switch (flags & MAP_TYPE) {
1368 		case MAP_SHARED:
1369 			if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1370 				return -EINVAL;
1371 			/*
1372 			 * Ignore pgoff.
1373 			 */
1374 			pgoff = 0;
1375 			vm_flags |= VM_SHARED | VM_MAYSHARE;
1376 			break;
1377 		case MAP_PRIVATE:
1378 			/*
1379 			 * Set pgoff according to addr for anon_vma.
1380 			 */
1381 			pgoff = addr >> PAGE_SHIFT;
1382 			break;
1383 		default:
1384 			return -EINVAL;
1385 		}
1386 	}
1387 
1388 	/*
1389 	 * Set 'VM_NORESERVE' if we should not account for the
1390 	 * memory use of this mapping.
1391 	 */
1392 	if (flags & MAP_NORESERVE) {
1393 		/* We honor MAP_NORESERVE if allowed to overcommit */
1394 		if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1395 			vm_flags |= VM_NORESERVE;
1396 
1397 		/* hugetlb applies strict overcommit unless MAP_NORESERVE */
1398 		if (file && is_file_hugepages(file))
1399 			vm_flags |= VM_NORESERVE;
1400 	}
1401 
1402 	addr = mmap_region(file, addr, len, vm_flags, pgoff);
1403 	if (!IS_ERR_VALUE(addr) &&
1404 	    ((vm_flags & VM_LOCKED) ||
1405 	     (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
1406 		*populate = len;
1407 	return addr;
1408 }
1409 
1410 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1411 		unsigned long, prot, unsigned long, flags,
1412 		unsigned long, fd, unsigned long, pgoff)
1413 {
1414 	struct file *file = NULL;
1415 	unsigned long retval = -EBADF;
1416 
1417 	if (!(flags & MAP_ANONYMOUS)) {
1418 		audit_mmap_fd(fd, flags);
1419 		file = fget(fd);
1420 		if (!file)
1421 			goto out;
1422 		if (is_file_hugepages(file))
1423 			len = ALIGN(len, huge_page_size(hstate_file(file)));
1424 		retval = -EINVAL;
1425 		if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
1426 			goto out_fput;
1427 	} else if (flags & MAP_HUGETLB) {
1428 		struct user_struct *user = NULL;
1429 		struct hstate *hs;
1430 
1431 		hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
1432 		if (!hs)
1433 			return -EINVAL;
1434 
1435 		len = ALIGN(len, huge_page_size(hs));
1436 		/*
1437 		 * VM_NORESERVE is used because the reservations will be
1438 		 * taken when vm_ops->mmap() is called
1439 		 * A dummy user value is used because we are not locking
1440 		 * memory so no accounting is necessary
1441 		 */
1442 		file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
1443 				VM_NORESERVE,
1444 				&user, HUGETLB_ANONHUGE_INODE,
1445 				(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
1446 		if (IS_ERR(file))
1447 			return PTR_ERR(file);
1448 	}
1449 
1450 	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1451 
1452 	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1453 out_fput:
1454 	if (file)
1455 		fput(file);
1456 out:
1457 	return retval;
1458 }
1459 
1460 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1461 struct mmap_arg_struct {
1462 	unsigned long addr;
1463 	unsigned long len;
1464 	unsigned long prot;
1465 	unsigned long flags;
1466 	unsigned long fd;
1467 	unsigned long offset;
1468 };
1469 
1470 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1471 {
1472 	struct mmap_arg_struct a;
1473 
1474 	if (copy_from_user(&a, arg, sizeof(a)))
1475 		return -EFAULT;
1476 	if (a.offset & ~PAGE_MASK)
1477 		return -EINVAL;
1478 
1479 	return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1480 			      a.offset >> PAGE_SHIFT);
1481 }
1482 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1483 
1484 /*
1485  * Some shared mappigns will want the pages marked read-only
1486  * to track write events. If so, we'll downgrade vm_page_prot
1487  * to the private version (using protection_map[] without the
1488  * VM_SHARED bit).
1489  */
1490 int vma_wants_writenotify(struct vm_area_struct *vma)
1491 {
1492 	vm_flags_t vm_flags = vma->vm_flags;
1493 	const struct vm_operations_struct *vm_ops = vma->vm_ops;
1494 
1495 	/* If it was private or non-writable, the write bit is already clear */
1496 	if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1497 		return 0;
1498 
1499 	/* The backer wishes to know when pages are first written to? */
1500 	if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
1501 		return 1;
1502 
1503 	/* The open routine did something to the protections that pgprot_modify
1504 	 * won't preserve? */
1505 	if (pgprot_val(vma->vm_page_prot) !=
1506 	    pgprot_val(vm_pgprot_modify(vma->vm_page_prot, vm_flags)))
1507 		return 0;
1508 
1509 	/* Do we need to track softdirty? */
1510 	if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
1511 		return 1;
1512 
1513 	/* Specialty mapping? */
1514 	if (vm_flags & VM_PFNMAP)
1515 		return 0;
1516 
1517 	/* Can the mapping track the dirty pages? */
1518 	return vma->vm_file && vma->vm_file->f_mapping &&
1519 		mapping_cap_account_dirty(vma->vm_file->f_mapping);
1520 }
1521 
1522 /*
1523  * We account for memory if it's a private writeable mapping,
1524  * not hugepages and VM_NORESERVE wasn't set.
1525  */
1526 static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
1527 {
1528 	/*
1529 	 * hugetlb has its own accounting separate from the core VM
1530 	 * VM_HUGETLB may not be set yet so we cannot check for that flag.
1531 	 */
1532 	if (file && is_file_hugepages(file))
1533 		return 0;
1534 
1535 	return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1536 }
1537 
1538 unsigned long mmap_region(struct file *file, unsigned long addr,
1539 		unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
1540 {
1541 	struct mm_struct *mm = current->mm;
1542 	struct vm_area_struct *vma, *prev;
1543 	int error;
1544 	struct rb_node **rb_link, *rb_parent;
1545 	unsigned long charged = 0;
1546 
1547 	/* Check against address space limit. */
1548 	if (!may_expand_vm(mm, len >> PAGE_SHIFT)) {
1549 		unsigned long nr_pages;
1550 
1551 		/*
1552 		 * MAP_FIXED may remove pages of mappings that intersects with
1553 		 * requested mapping. Account for the pages it would unmap.
1554 		 */
1555 		if (!(vm_flags & MAP_FIXED))
1556 			return -ENOMEM;
1557 
1558 		nr_pages = count_vma_pages_range(mm, addr, addr + len);
1559 
1560 		if (!may_expand_vm(mm, (len >> PAGE_SHIFT) - nr_pages))
1561 			return -ENOMEM;
1562 	}
1563 
1564 	/* Clear old maps */
1565 	error = -ENOMEM;
1566 	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
1567 			      &rb_parent)) {
1568 		if (do_munmap(mm, addr, len))
1569 			return -ENOMEM;
1570 	}
1571 
1572 	/*
1573 	 * Private writable mapping: check memory availability
1574 	 */
1575 	if (accountable_mapping(file, vm_flags)) {
1576 		charged = len >> PAGE_SHIFT;
1577 		if (security_vm_enough_memory_mm(mm, charged))
1578 			return -ENOMEM;
1579 		vm_flags |= VM_ACCOUNT;
1580 	}
1581 
1582 	/*
1583 	 * Can we just expand an old mapping?
1584 	 */
1585 	vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
1586 			NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
1587 	if (vma)
1588 		goto out;
1589 
1590 	/*
1591 	 * Determine the object being mapped and call the appropriate
1592 	 * specific mapper. the address has already been validated, but
1593 	 * not unmapped, but the maps are removed from the list.
1594 	 */
1595 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1596 	if (!vma) {
1597 		error = -ENOMEM;
1598 		goto unacct_error;
1599 	}
1600 
1601 	vma->vm_mm = mm;
1602 	vma->vm_start = addr;
1603 	vma->vm_end = addr + len;
1604 	vma->vm_flags = vm_flags;
1605 	vma->vm_page_prot = vm_get_page_prot(vm_flags);
1606 	vma->vm_pgoff = pgoff;
1607 	INIT_LIST_HEAD(&vma->anon_vma_chain);
1608 
1609 	if (file) {
1610 		if (vm_flags & VM_DENYWRITE) {
1611 			error = deny_write_access(file);
1612 			if (error)
1613 				goto free_vma;
1614 		}
1615 		if (vm_flags & VM_SHARED) {
1616 			error = mapping_map_writable(file->f_mapping);
1617 			if (error)
1618 				goto allow_write_and_free_vma;
1619 		}
1620 
1621 		/* ->mmap() can change vma->vm_file, but must guarantee that
1622 		 * vma_link() below can deny write-access if VM_DENYWRITE is set
1623 		 * and map writably if VM_SHARED is set. This usually means the
1624 		 * new file must not have been exposed to user-space, yet.
1625 		 */
1626 		vma->vm_file = get_file(file);
1627 		error = file->f_op->mmap(file, vma);
1628 		if (error)
1629 			goto unmap_and_free_vma;
1630 
1631 		/* Can addr have changed??
1632 		 *
1633 		 * Answer: Yes, several device drivers can do it in their
1634 		 *         f_op->mmap method. -DaveM
1635 		 * Bug: If addr is changed, prev, rb_link, rb_parent should
1636 		 *      be updated for vma_link()
1637 		 */
1638 		WARN_ON_ONCE(addr != vma->vm_start);
1639 
1640 		addr = vma->vm_start;
1641 		vm_flags = vma->vm_flags;
1642 	} else if (vm_flags & VM_SHARED) {
1643 		error = shmem_zero_setup(vma);
1644 		if (error)
1645 			goto free_vma;
1646 	}
1647 
1648 	vma_link(mm, vma, prev, rb_link, rb_parent);
1649 	/* Once vma denies write, undo our temporary denial count */
1650 	if (file) {
1651 		if (vm_flags & VM_SHARED)
1652 			mapping_unmap_writable(file->f_mapping);
1653 		if (vm_flags & VM_DENYWRITE)
1654 			allow_write_access(file);
1655 	}
1656 	file = vma->vm_file;
1657 out:
1658 	perf_event_mmap(vma);
1659 
1660 	vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1661 	if (vm_flags & VM_LOCKED) {
1662 		if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
1663 					vma == get_gate_vma(current->mm)))
1664 			mm->locked_vm += (len >> PAGE_SHIFT);
1665 		else
1666 			vma->vm_flags &= ~VM_LOCKED;
1667 	}
1668 
1669 	if (file)
1670 		uprobe_mmap(vma);
1671 
1672 	/*
1673 	 * New (or expanded) vma always get soft dirty status.
1674 	 * Otherwise user-space soft-dirty page tracker won't
1675 	 * be able to distinguish situation when vma area unmapped,
1676 	 * then new mapped in-place (which must be aimed as
1677 	 * a completely new data area).
1678 	 */
1679 	vma->vm_flags |= VM_SOFTDIRTY;
1680 
1681 	vma_set_page_prot(vma);
1682 
1683 	return addr;
1684 
1685 unmap_and_free_vma:
1686 	vma->vm_file = NULL;
1687 	fput(file);
1688 
1689 	/* Undo any partial mapping done by a device driver. */
1690 	unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1691 	charged = 0;
1692 	if (vm_flags & VM_SHARED)
1693 		mapping_unmap_writable(file->f_mapping);
1694 allow_write_and_free_vma:
1695 	if (vm_flags & VM_DENYWRITE)
1696 		allow_write_access(file);
1697 free_vma:
1698 	kmem_cache_free(vm_area_cachep, vma);
1699 unacct_error:
1700 	if (charged)
1701 		vm_unacct_memory(charged);
1702 	return error;
1703 }
1704 
1705 unsigned long unmapped_area(struct vm_unmapped_area_info *info)
1706 {
1707 	/*
1708 	 * We implement the search by looking for an rbtree node that
1709 	 * immediately follows a suitable gap. That is,
1710 	 * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
1711 	 * - gap_end   = vma->vm_start        >= info->low_limit  + length;
1712 	 * - gap_end - gap_start >= length
1713 	 */
1714 
1715 	struct mm_struct *mm = current->mm;
1716 	struct vm_area_struct *vma;
1717 	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1718 
1719 	/* Adjust search length to account for worst case alignment overhead */
1720 	length = info->length + info->align_mask;
1721 	if (length < info->length)
1722 		return -ENOMEM;
1723 
1724 	/* Adjust search limits by the desired length */
1725 	if (info->high_limit < length)
1726 		return -ENOMEM;
1727 	high_limit = info->high_limit - length;
1728 
1729 	if (info->low_limit > high_limit)
1730 		return -ENOMEM;
1731 	low_limit = info->low_limit + length;
1732 
1733 	/* Check if rbtree root looks promising */
1734 	if (RB_EMPTY_ROOT(&mm->mm_rb))
1735 		goto check_highest;
1736 	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1737 	if (vma->rb_subtree_gap < length)
1738 		goto check_highest;
1739 
1740 	while (true) {
1741 		/* Visit left subtree if it looks promising */
1742 		gap_end = vma->vm_start;
1743 		if (gap_end >= low_limit && vma->vm_rb.rb_left) {
1744 			struct vm_area_struct *left =
1745 				rb_entry(vma->vm_rb.rb_left,
1746 					 struct vm_area_struct, vm_rb);
1747 			if (left->rb_subtree_gap >= length) {
1748 				vma = left;
1749 				continue;
1750 			}
1751 		}
1752 
1753 		gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1754 check_current:
1755 		/* Check if current node has a suitable gap */
1756 		if (gap_start > high_limit)
1757 			return -ENOMEM;
1758 		if (gap_end >= low_limit && gap_end - gap_start >= length)
1759 			goto found;
1760 
1761 		/* Visit right subtree if it looks promising */
1762 		if (vma->vm_rb.rb_right) {
1763 			struct vm_area_struct *right =
1764 				rb_entry(vma->vm_rb.rb_right,
1765 					 struct vm_area_struct, vm_rb);
1766 			if (right->rb_subtree_gap >= length) {
1767 				vma = right;
1768 				continue;
1769 			}
1770 		}
1771 
1772 		/* Go back up the rbtree to find next candidate node */
1773 		while (true) {
1774 			struct rb_node *prev = &vma->vm_rb;
1775 			if (!rb_parent(prev))
1776 				goto check_highest;
1777 			vma = rb_entry(rb_parent(prev),
1778 				       struct vm_area_struct, vm_rb);
1779 			if (prev == vma->vm_rb.rb_left) {
1780 				gap_start = vma->vm_prev->vm_end;
1781 				gap_end = vma->vm_start;
1782 				goto check_current;
1783 			}
1784 		}
1785 	}
1786 
1787 check_highest:
1788 	/* Check highest gap, which does not precede any rbtree node */
1789 	gap_start = mm->highest_vm_end;
1790 	gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
1791 	if (gap_start > high_limit)
1792 		return -ENOMEM;
1793 
1794 found:
1795 	/* We found a suitable gap. Clip it with the original low_limit. */
1796 	if (gap_start < info->low_limit)
1797 		gap_start = info->low_limit;
1798 
1799 	/* Adjust gap address to the desired alignment */
1800 	gap_start += (info->align_offset - gap_start) & info->align_mask;
1801 
1802 	VM_BUG_ON(gap_start + info->length > info->high_limit);
1803 	VM_BUG_ON(gap_start + info->length > gap_end);
1804 	return gap_start;
1805 }
1806 
1807 unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
1808 {
1809 	struct mm_struct *mm = current->mm;
1810 	struct vm_area_struct *vma;
1811 	unsigned long length, low_limit, high_limit, gap_start, gap_end;
1812 
1813 	/* Adjust search length to account for worst case alignment overhead */
1814 	length = info->length + info->align_mask;
1815 	if (length < info->length)
1816 		return -ENOMEM;
1817 
1818 	/*
1819 	 * Adjust search limits by the desired length.
1820 	 * See implementation comment at top of unmapped_area().
1821 	 */
1822 	gap_end = info->high_limit;
1823 	if (gap_end < length)
1824 		return -ENOMEM;
1825 	high_limit = gap_end - length;
1826 
1827 	if (info->low_limit > high_limit)
1828 		return -ENOMEM;
1829 	low_limit = info->low_limit + length;
1830 
1831 	/* Check highest gap, which does not precede any rbtree node */
1832 	gap_start = mm->highest_vm_end;
1833 	if (gap_start <= high_limit)
1834 		goto found_highest;
1835 
1836 	/* Check if rbtree root looks promising */
1837 	if (RB_EMPTY_ROOT(&mm->mm_rb))
1838 		return -ENOMEM;
1839 	vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
1840 	if (vma->rb_subtree_gap < length)
1841 		return -ENOMEM;
1842 
1843 	while (true) {
1844 		/* Visit right subtree if it looks promising */
1845 		gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
1846 		if (gap_start <= high_limit && vma->vm_rb.rb_right) {
1847 			struct vm_area_struct *right =
1848 				rb_entry(vma->vm_rb.rb_right,
1849 					 struct vm_area_struct, vm_rb);
1850 			if (right->rb_subtree_gap >= length) {
1851 				vma = right;
1852 				continue;
1853 			}
1854 		}
1855 
1856 check_current:
1857 		/* Check if current node has a suitable gap */
1858 		gap_end = vma->vm_start;
1859 		if (gap_end < low_limit)
1860 			return -ENOMEM;
1861 		if (gap_start <= high_limit && gap_end - gap_start >= length)
1862 			goto found;
1863 
1864 		/* Visit left subtree if it looks promising */
1865 		if (vma->vm_rb.rb_left) {
1866 			struct vm_area_struct *left =
1867 				rb_entry(vma->vm_rb.rb_left,
1868 					 struct vm_area_struct, vm_rb);
1869 			if (left->rb_subtree_gap >= length) {
1870 				vma = left;
1871 				continue;
1872 			}
1873 		}
1874 
1875 		/* Go back up the rbtree to find next candidate node */
1876 		while (true) {
1877 			struct rb_node *prev = &vma->vm_rb;
1878 			if (!rb_parent(prev))
1879 				return -ENOMEM;
1880 			vma = rb_entry(rb_parent(prev),
1881 				       struct vm_area_struct, vm_rb);
1882 			if (prev == vma->vm_rb.rb_right) {
1883 				gap_start = vma->vm_prev ?
1884 					vma->vm_prev->vm_end : 0;
1885 				goto check_current;
1886 			}
1887 		}
1888 	}
1889 
1890 found:
1891 	/* We found a suitable gap. Clip it with the original high_limit. */
1892 	if (gap_end > info->high_limit)
1893 		gap_end = info->high_limit;
1894 
1895 found_highest:
1896 	/* Compute highest gap address at the desired alignment */
1897 	gap_end -= info->length;
1898 	gap_end -= (gap_end - info->align_offset) & info->align_mask;
1899 
1900 	VM_BUG_ON(gap_end < info->low_limit);
1901 	VM_BUG_ON(gap_end < gap_start);
1902 	return gap_end;
1903 }
1904 
1905 /* Get an address range which is currently unmapped.
1906  * For shmat() with addr=0.
1907  *
1908  * Ugly calling convention alert:
1909  * Return value with the low bits set means error value,
1910  * ie
1911  *	if (ret & ~PAGE_MASK)
1912  *		error = ret;
1913  *
1914  * This function "knows" that -ENOMEM has the bits set.
1915  */
1916 #ifndef HAVE_ARCH_UNMAPPED_AREA
1917 unsigned long
1918 arch_get_unmapped_area(struct file *filp, unsigned long addr,
1919 		unsigned long len, unsigned long pgoff, unsigned long flags)
1920 {
1921 	struct mm_struct *mm = current->mm;
1922 	struct vm_area_struct *vma;
1923 	struct vm_unmapped_area_info info;
1924 
1925 	if (len > TASK_SIZE - mmap_min_addr)
1926 		return -ENOMEM;
1927 
1928 	if (flags & MAP_FIXED)
1929 		return addr;
1930 
1931 	if (addr) {
1932 		addr = PAGE_ALIGN(addr);
1933 		vma = find_vma(mm, addr);
1934 		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1935 		    (!vma || addr + len <= vma->vm_start))
1936 			return addr;
1937 	}
1938 
1939 	info.flags = 0;
1940 	info.length = len;
1941 	info.low_limit = mm->mmap_base;
1942 	info.high_limit = TASK_SIZE;
1943 	info.align_mask = 0;
1944 	return vm_unmapped_area(&info);
1945 }
1946 #endif
1947 
1948 /*
1949  * This mmap-allocator allocates new areas top-down from below the
1950  * stack's low limit (the base):
1951  */
1952 #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1953 unsigned long
1954 arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1955 			  const unsigned long len, const unsigned long pgoff,
1956 			  const unsigned long flags)
1957 {
1958 	struct vm_area_struct *vma;
1959 	struct mm_struct *mm = current->mm;
1960 	unsigned long addr = addr0;
1961 	struct vm_unmapped_area_info info;
1962 
1963 	/* requested length too big for entire address space */
1964 	if (len > TASK_SIZE - mmap_min_addr)
1965 		return -ENOMEM;
1966 
1967 	if (flags & MAP_FIXED)
1968 		return addr;
1969 
1970 	/* requesting a specific address */
1971 	if (addr) {
1972 		addr = PAGE_ALIGN(addr);
1973 		vma = find_vma(mm, addr);
1974 		if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
1975 				(!vma || addr + len <= vma->vm_start))
1976 			return addr;
1977 	}
1978 
1979 	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
1980 	info.length = len;
1981 	info.low_limit = max(PAGE_SIZE, mmap_min_addr);
1982 	info.high_limit = mm->mmap_base;
1983 	info.align_mask = 0;
1984 	addr = vm_unmapped_area(&info);
1985 
1986 	/*
1987 	 * A failed mmap() very likely causes application failure,
1988 	 * so fall back to the bottom-up function here. This scenario
1989 	 * can happen with large stack limits and large mmap()
1990 	 * allocations.
1991 	 */
1992 	if (addr & ~PAGE_MASK) {
1993 		VM_BUG_ON(addr != -ENOMEM);
1994 		info.flags = 0;
1995 		info.low_limit = TASK_UNMAPPED_BASE;
1996 		info.high_limit = TASK_SIZE;
1997 		addr = vm_unmapped_area(&info);
1998 	}
1999 
2000 	return addr;
2001 }
2002 #endif
2003 
2004 unsigned long
2005 get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
2006 		unsigned long pgoff, unsigned long flags)
2007 {
2008 	unsigned long (*get_area)(struct file *, unsigned long,
2009 				  unsigned long, unsigned long, unsigned long);
2010 
2011 	unsigned long error = arch_mmap_check(addr, len, flags);
2012 	if (error)
2013 		return error;
2014 
2015 	/* Careful about overflows.. */
2016 	if (len > TASK_SIZE)
2017 		return -ENOMEM;
2018 
2019 	get_area = current->mm->get_unmapped_area;
2020 	if (file && file->f_op->get_unmapped_area)
2021 		get_area = file->f_op->get_unmapped_area;
2022 	addr = get_area(file, addr, len, pgoff, flags);
2023 	if (IS_ERR_VALUE(addr))
2024 		return addr;
2025 
2026 	if (addr > TASK_SIZE - len)
2027 		return -ENOMEM;
2028 	if (addr & ~PAGE_MASK)
2029 		return -EINVAL;
2030 
2031 	addr = arch_rebalance_pgtables(addr, len);
2032 	error = security_mmap_addr(addr);
2033 	return error ? error : addr;
2034 }
2035 
2036 EXPORT_SYMBOL(get_unmapped_area);
2037 
2038 /* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
2039 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
2040 {
2041 	struct rb_node *rb_node;
2042 	struct vm_area_struct *vma;
2043 
2044 	/* Check the cache first. */
2045 	vma = vmacache_find(mm, addr);
2046 	if (likely(vma))
2047 		return vma;
2048 
2049 	rb_node = mm->mm_rb.rb_node;
2050 	vma = NULL;
2051 
2052 	while (rb_node) {
2053 		struct vm_area_struct *tmp;
2054 
2055 		tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2056 
2057 		if (tmp->vm_end > addr) {
2058 			vma = tmp;
2059 			if (tmp->vm_start <= addr)
2060 				break;
2061 			rb_node = rb_node->rb_left;
2062 		} else
2063 			rb_node = rb_node->rb_right;
2064 	}
2065 
2066 	if (vma)
2067 		vmacache_update(addr, vma);
2068 	return vma;
2069 }
2070 
2071 EXPORT_SYMBOL(find_vma);
2072 
2073 /*
2074  * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
2075  */
2076 struct vm_area_struct *
2077 find_vma_prev(struct mm_struct *mm, unsigned long addr,
2078 			struct vm_area_struct **pprev)
2079 {
2080 	struct vm_area_struct *vma;
2081 
2082 	vma = find_vma(mm, addr);
2083 	if (vma) {
2084 		*pprev = vma->vm_prev;
2085 	} else {
2086 		struct rb_node *rb_node = mm->mm_rb.rb_node;
2087 		*pprev = NULL;
2088 		while (rb_node) {
2089 			*pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
2090 			rb_node = rb_node->rb_right;
2091 		}
2092 	}
2093 	return vma;
2094 }
2095 
2096 /*
2097  * Verify that the stack growth is acceptable and
2098  * update accounting. This is shared with both the
2099  * grow-up and grow-down cases.
2100  */
2101 static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
2102 {
2103 	struct mm_struct *mm = vma->vm_mm;
2104 	struct rlimit *rlim = current->signal->rlim;
2105 	unsigned long new_start, actual_size;
2106 
2107 	/* address space limit tests */
2108 	if (!may_expand_vm(mm, grow))
2109 		return -ENOMEM;
2110 
2111 	/* Stack limit test */
2112 	actual_size = size;
2113 	if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN)))
2114 		actual_size -= PAGE_SIZE;
2115 	if (actual_size > READ_ONCE(rlim[RLIMIT_STACK].rlim_cur))
2116 		return -ENOMEM;
2117 
2118 	/* mlock limit tests */
2119 	if (vma->vm_flags & VM_LOCKED) {
2120 		unsigned long locked;
2121 		unsigned long limit;
2122 		locked = mm->locked_vm + grow;
2123 		limit = READ_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
2124 		limit >>= PAGE_SHIFT;
2125 		if (locked > limit && !capable(CAP_IPC_LOCK))
2126 			return -ENOMEM;
2127 	}
2128 
2129 	/* Check to ensure the stack will not grow into a hugetlb-only region */
2130 	new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
2131 			vma->vm_end - size;
2132 	if (is_hugepage_only_range(vma->vm_mm, new_start, size))
2133 		return -EFAULT;
2134 
2135 	/*
2136 	 * Overcommit..  This must be the final test, as it will
2137 	 * update security statistics.
2138 	 */
2139 	if (security_vm_enough_memory_mm(mm, grow))
2140 		return -ENOMEM;
2141 
2142 	/* Ok, everything looks good - let it rip */
2143 	if (vma->vm_flags & VM_LOCKED)
2144 		mm->locked_vm += grow;
2145 	vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
2146 	return 0;
2147 }
2148 
2149 #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
2150 /*
2151  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
2152  * vma is the last one with address > vma->vm_end.  Have to extend vma.
2153  */
2154 int expand_upwards(struct vm_area_struct *vma, unsigned long address)
2155 {
2156 	int error;
2157 
2158 	if (!(vma->vm_flags & VM_GROWSUP))
2159 		return -EFAULT;
2160 
2161 	/*
2162 	 * We must make sure the anon_vma is allocated
2163 	 * so that the anon_vma locking is not a noop.
2164 	 */
2165 	if (unlikely(anon_vma_prepare(vma)))
2166 		return -ENOMEM;
2167 	vma_lock_anon_vma(vma);
2168 
2169 	/*
2170 	 * vma->vm_start/vm_end cannot change under us because the caller
2171 	 * is required to hold the mmap_sem in read mode.  We need the
2172 	 * anon_vma lock to serialize against concurrent expand_stacks.
2173 	 * Also guard against wrapping around to address 0.
2174 	 */
2175 	if (address < PAGE_ALIGN(address+4))
2176 		address = PAGE_ALIGN(address+4);
2177 	else {
2178 		vma_unlock_anon_vma(vma);
2179 		return -ENOMEM;
2180 	}
2181 	error = 0;
2182 
2183 	/* Somebody else might have raced and expanded it already */
2184 	if (address > vma->vm_end) {
2185 		unsigned long size, grow;
2186 
2187 		size = address - vma->vm_start;
2188 		grow = (address - vma->vm_end) >> PAGE_SHIFT;
2189 
2190 		error = -ENOMEM;
2191 		if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
2192 			error = acct_stack_growth(vma, size, grow);
2193 			if (!error) {
2194 				/*
2195 				 * vma_gap_update() doesn't support concurrent
2196 				 * updates, but we only hold a shared mmap_sem
2197 				 * lock here, so we need to protect against
2198 				 * concurrent vma expansions.
2199 				 * vma_lock_anon_vma() doesn't help here, as
2200 				 * we don't guarantee that all growable vmas
2201 				 * in a mm share the same root anon vma.
2202 				 * So, we reuse mm->page_table_lock to guard
2203 				 * against concurrent vma expansions.
2204 				 */
2205 				spin_lock(&vma->vm_mm->page_table_lock);
2206 				anon_vma_interval_tree_pre_update_vma(vma);
2207 				vma->vm_end = address;
2208 				anon_vma_interval_tree_post_update_vma(vma);
2209 				if (vma->vm_next)
2210 					vma_gap_update(vma->vm_next);
2211 				else
2212 					vma->vm_mm->highest_vm_end = address;
2213 				spin_unlock(&vma->vm_mm->page_table_lock);
2214 
2215 				perf_event_mmap(vma);
2216 			}
2217 		}
2218 	}
2219 	vma_unlock_anon_vma(vma);
2220 	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2221 	validate_mm(vma->vm_mm);
2222 	return error;
2223 }
2224 #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
2225 
2226 /*
2227  * vma is the first one with address < vma->vm_start.  Have to extend vma.
2228  */
2229 int expand_downwards(struct vm_area_struct *vma,
2230 				   unsigned long address)
2231 {
2232 	int error;
2233 
2234 	/*
2235 	 * We must make sure the anon_vma is allocated
2236 	 * so that the anon_vma locking is not a noop.
2237 	 */
2238 	if (unlikely(anon_vma_prepare(vma)))
2239 		return -ENOMEM;
2240 
2241 	address &= PAGE_MASK;
2242 	error = security_mmap_addr(address);
2243 	if (error)
2244 		return error;
2245 
2246 	vma_lock_anon_vma(vma);
2247 
2248 	/*
2249 	 * vma->vm_start/vm_end cannot change under us because the caller
2250 	 * is required to hold the mmap_sem in read mode.  We need the
2251 	 * anon_vma lock to serialize against concurrent expand_stacks.
2252 	 */
2253 
2254 	/* Somebody else might have raced and expanded it already */
2255 	if (address < vma->vm_start) {
2256 		unsigned long size, grow;
2257 
2258 		size = vma->vm_end - address;
2259 		grow = (vma->vm_start - address) >> PAGE_SHIFT;
2260 
2261 		error = -ENOMEM;
2262 		if (grow <= vma->vm_pgoff) {
2263 			error = acct_stack_growth(vma, size, grow);
2264 			if (!error) {
2265 				/*
2266 				 * vma_gap_update() doesn't support concurrent
2267 				 * updates, but we only hold a shared mmap_sem
2268 				 * lock here, so we need to protect against
2269 				 * concurrent vma expansions.
2270 				 * vma_lock_anon_vma() doesn't help here, as
2271 				 * we don't guarantee that all growable vmas
2272 				 * in a mm share the same root anon vma.
2273 				 * So, we reuse mm->page_table_lock to guard
2274 				 * against concurrent vma expansions.
2275 				 */
2276 				spin_lock(&vma->vm_mm->page_table_lock);
2277 				anon_vma_interval_tree_pre_update_vma(vma);
2278 				vma->vm_start = address;
2279 				vma->vm_pgoff -= grow;
2280 				anon_vma_interval_tree_post_update_vma(vma);
2281 				vma_gap_update(vma);
2282 				spin_unlock(&vma->vm_mm->page_table_lock);
2283 
2284 				perf_event_mmap(vma);
2285 			}
2286 		}
2287 	}
2288 	vma_unlock_anon_vma(vma);
2289 	khugepaged_enter_vma_merge(vma, vma->vm_flags);
2290 	validate_mm(vma->vm_mm);
2291 	return error;
2292 }
2293 
2294 /*
2295  * Note how expand_stack() refuses to expand the stack all the way to
2296  * abut the next virtual mapping, *unless* that mapping itself is also
2297  * a stack mapping. We want to leave room for a guard page, after all
2298  * (the guard page itself is not added here, that is done by the
2299  * actual page faulting logic)
2300  *
2301  * This matches the behavior of the guard page logic (see mm/memory.c:
2302  * check_stack_guard_page()), which only allows the guard page to be
2303  * removed under these circumstances.
2304  */
2305 #ifdef CONFIG_STACK_GROWSUP
2306 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2307 {
2308 	struct vm_area_struct *next;
2309 
2310 	address &= PAGE_MASK;
2311 	next = vma->vm_next;
2312 	if (next && next->vm_start == address + PAGE_SIZE) {
2313 		if (!(next->vm_flags & VM_GROWSUP))
2314 			return -ENOMEM;
2315 	}
2316 	return expand_upwards(vma, address);
2317 }
2318 
2319 struct vm_area_struct *
2320 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2321 {
2322 	struct vm_area_struct *vma, *prev;
2323 
2324 	addr &= PAGE_MASK;
2325 	vma = find_vma_prev(mm, addr, &prev);
2326 	if (vma && (vma->vm_start <= addr))
2327 		return vma;
2328 	if (!prev || expand_stack(prev, addr))
2329 		return NULL;
2330 	if (prev->vm_flags & VM_LOCKED)
2331 		populate_vma_page_range(prev, addr, prev->vm_end, NULL);
2332 	return prev;
2333 }
2334 #else
2335 int expand_stack(struct vm_area_struct *vma, unsigned long address)
2336 {
2337 	struct vm_area_struct *prev;
2338 
2339 	address &= PAGE_MASK;
2340 	prev = vma->vm_prev;
2341 	if (prev && prev->vm_end == address) {
2342 		if (!(prev->vm_flags & VM_GROWSDOWN))
2343 			return -ENOMEM;
2344 	}
2345 	return expand_downwards(vma, address);
2346 }
2347 
2348 struct vm_area_struct *
2349 find_extend_vma(struct mm_struct *mm, unsigned long addr)
2350 {
2351 	struct vm_area_struct *vma;
2352 	unsigned long start;
2353 
2354 	addr &= PAGE_MASK;
2355 	vma = find_vma(mm, addr);
2356 	if (!vma)
2357 		return NULL;
2358 	if (vma->vm_start <= addr)
2359 		return vma;
2360 	if (!(vma->vm_flags & VM_GROWSDOWN))
2361 		return NULL;
2362 	start = vma->vm_start;
2363 	if (expand_stack(vma, addr))
2364 		return NULL;
2365 	if (vma->vm_flags & VM_LOCKED)
2366 		populate_vma_page_range(vma, addr, start, NULL);
2367 	return vma;
2368 }
2369 #endif
2370 
2371 EXPORT_SYMBOL_GPL(find_extend_vma);
2372 
2373 /*
2374  * Ok - we have the memory areas we should free on the vma list,
2375  * so release them, and do the vma updates.
2376  *
2377  * Called with the mm semaphore held.
2378  */
2379 static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
2380 {
2381 	unsigned long nr_accounted = 0;
2382 
2383 	/* Update high watermark before we lower total_vm */
2384 	update_hiwater_vm(mm);
2385 	do {
2386 		long nrpages = vma_pages(vma);
2387 
2388 		if (vma->vm_flags & VM_ACCOUNT)
2389 			nr_accounted += nrpages;
2390 		vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
2391 		vma = remove_vma(vma);
2392 	} while (vma);
2393 	vm_unacct_memory(nr_accounted);
2394 	validate_mm(mm);
2395 }
2396 
2397 /*
2398  * Get rid of page table information in the indicated region.
2399  *
2400  * Called with the mm semaphore held.
2401  */
2402 static void unmap_region(struct mm_struct *mm,
2403 		struct vm_area_struct *vma, struct vm_area_struct *prev,
2404 		unsigned long start, unsigned long end)
2405 {
2406 	struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
2407 	struct mmu_gather tlb;
2408 
2409 	lru_add_drain();
2410 	tlb_gather_mmu(&tlb, mm, start, end);
2411 	update_hiwater_rss(mm);
2412 	unmap_vmas(&tlb, vma, start, end);
2413 	free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
2414 				 next ? next->vm_start : USER_PGTABLES_CEILING);
2415 	tlb_finish_mmu(&tlb, start, end);
2416 }
2417 
2418 /*
2419  * Create a list of vma's touched by the unmap, removing them from the mm's
2420  * vma list as we go..
2421  */
2422 static void
2423 detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
2424 	struct vm_area_struct *prev, unsigned long end)
2425 {
2426 	struct vm_area_struct **insertion_point;
2427 	struct vm_area_struct *tail_vma = NULL;
2428 
2429 	insertion_point = (prev ? &prev->vm_next : &mm->mmap);
2430 	vma->vm_prev = NULL;
2431 	do {
2432 		vma_rb_erase(vma, &mm->mm_rb);
2433 		mm->map_count--;
2434 		tail_vma = vma;
2435 		vma = vma->vm_next;
2436 	} while (vma && vma->vm_start < end);
2437 	*insertion_point = vma;
2438 	if (vma) {
2439 		vma->vm_prev = prev;
2440 		vma_gap_update(vma);
2441 	} else
2442 		mm->highest_vm_end = prev ? prev->vm_end : 0;
2443 	tail_vma->vm_next = NULL;
2444 
2445 	/* Kill the cache */
2446 	vmacache_invalidate(mm);
2447 }
2448 
2449 /*
2450  * __split_vma() bypasses sysctl_max_map_count checking.  We use this on the
2451  * munmap path where it doesn't make sense to fail.
2452  */
2453 static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2454 	      unsigned long addr, int new_below)
2455 {
2456 	struct vm_area_struct *new;
2457 	int err;
2458 
2459 	if (is_vm_hugetlb_page(vma) && (addr &
2460 					~(huge_page_mask(hstate_vma(vma)))))
2461 		return -EINVAL;
2462 
2463 	new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2464 	if (!new)
2465 		return -ENOMEM;
2466 
2467 	/* most fields are the same, copy all, and then fixup */
2468 	*new = *vma;
2469 
2470 	INIT_LIST_HEAD(&new->anon_vma_chain);
2471 
2472 	if (new_below)
2473 		new->vm_end = addr;
2474 	else {
2475 		new->vm_start = addr;
2476 		new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
2477 	}
2478 
2479 	err = vma_dup_policy(vma, new);
2480 	if (err)
2481 		goto out_free_vma;
2482 
2483 	err = anon_vma_clone(new, vma);
2484 	if (err)
2485 		goto out_free_mpol;
2486 
2487 	if (new->vm_file)
2488 		get_file(new->vm_file);
2489 
2490 	if (new->vm_ops && new->vm_ops->open)
2491 		new->vm_ops->open(new);
2492 
2493 	if (new_below)
2494 		err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2495 			((addr - new->vm_start) >> PAGE_SHIFT), new);
2496 	else
2497 		err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2498 
2499 	/* Success. */
2500 	if (!err)
2501 		return 0;
2502 
2503 	/* Clean everything up if vma_adjust failed. */
2504 	if (new->vm_ops && new->vm_ops->close)
2505 		new->vm_ops->close(new);
2506 	if (new->vm_file)
2507 		fput(new->vm_file);
2508 	unlink_anon_vmas(new);
2509  out_free_mpol:
2510 	mpol_put(vma_policy(new));
2511  out_free_vma:
2512 	kmem_cache_free(vm_area_cachep, new);
2513 	return err;
2514 }
2515 
2516 /*
2517  * Split a vma into two pieces at address 'addr', a new vma is allocated
2518  * either for the first part or the tail.
2519  */
2520 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2521 	      unsigned long addr, int new_below)
2522 {
2523 	if (mm->map_count >= sysctl_max_map_count)
2524 		return -ENOMEM;
2525 
2526 	return __split_vma(mm, vma, addr, new_below);
2527 }
2528 
2529 /* Munmap is split into 2 main parts -- this part which finds
2530  * what needs doing, and the areas themselves, which do the
2531  * work.  This now handles partial unmappings.
2532  * Jeremy Fitzhardinge <jeremy@goop.org>
2533  */
2534 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2535 {
2536 	unsigned long end;
2537 	struct vm_area_struct *vma, *prev, *last;
2538 
2539 	if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
2540 		return -EINVAL;
2541 
2542 	len = PAGE_ALIGN(len);
2543 	if (len == 0)
2544 		return -EINVAL;
2545 
2546 	/* Find the first overlapping VMA */
2547 	vma = find_vma(mm, start);
2548 	if (!vma)
2549 		return 0;
2550 	prev = vma->vm_prev;
2551 	/* we have  start < vma->vm_end  */
2552 
2553 	/* if it doesn't overlap, we have nothing.. */
2554 	end = start + len;
2555 	if (vma->vm_start >= end)
2556 		return 0;
2557 
2558 	/*
2559 	 * If we need to split any vma, do it now to save pain later.
2560 	 *
2561 	 * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
2562 	 * unmapped vm_area_struct will remain in use: so lower split_vma
2563 	 * places tmp vma above, and higher split_vma places tmp vma below.
2564 	 */
2565 	if (start > vma->vm_start) {
2566 		int error;
2567 
2568 		/*
2569 		 * Make sure that map_count on return from munmap() will
2570 		 * not exceed its limit; but let map_count go just above
2571 		 * its limit temporarily, to help free resources as expected.
2572 		 */
2573 		if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2574 			return -ENOMEM;
2575 
2576 		error = __split_vma(mm, vma, start, 0);
2577 		if (error)
2578 			return error;
2579 		prev = vma;
2580 	}
2581 
2582 	/* Does it split the last one? */
2583 	last = find_vma(mm, end);
2584 	if (last && end > last->vm_start) {
2585 		int error = __split_vma(mm, last, end, 1);
2586 		if (error)
2587 			return error;
2588 	}
2589 	vma = prev ? prev->vm_next : mm->mmap;
2590 
2591 	/*
2592 	 * unlock any mlock()ed ranges before detaching vmas
2593 	 */
2594 	if (mm->locked_vm) {
2595 		struct vm_area_struct *tmp = vma;
2596 		while (tmp && tmp->vm_start < end) {
2597 			if (tmp->vm_flags & VM_LOCKED) {
2598 				mm->locked_vm -= vma_pages(tmp);
2599 				munlock_vma_pages_all(tmp);
2600 			}
2601 			tmp = tmp->vm_next;
2602 		}
2603 	}
2604 
2605 	/*
2606 	 * Remove the vma's, and unmap the actual pages
2607 	 */
2608 	detach_vmas_to_be_unmapped(mm, vma, prev, end);
2609 	unmap_region(mm, vma, prev, start, end);
2610 
2611 	arch_unmap(mm, vma, start, end);
2612 
2613 	/* Fix up all other VM information */
2614 	remove_vma_list(mm, vma);
2615 
2616 	return 0;
2617 }
2618 
2619 int vm_munmap(unsigned long start, size_t len)
2620 {
2621 	int ret;
2622 	struct mm_struct *mm = current->mm;
2623 
2624 	down_write(&mm->mmap_sem);
2625 	ret = do_munmap(mm, start, len);
2626 	up_write(&mm->mmap_sem);
2627 	return ret;
2628 }
2629 EXPORT_SYMBOL(vm_munmap);
2630 
2631 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2632 {
2633 	profile_munmap(addr);
2634 	return vm_munmap(addr, len);
2635 }
2636 
2637 
2638 /*
2639  * Emulation of deprecated remap_file_pages() syscall.
2640  */
2641 SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
2642 		unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
2643 {
2644 
2645 	struct mm_struct *mm = current->mm;
2646 	struct vm_area_struct *vma;
2647 	unsigned long populate = 0;
2648 	unsigned long ret = -EINVAL;
2649 	struct file *file;
2650 
2651 	pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. "
2652 			"See Documentation/vm/remap_file_pages.txt.\n",
2653 			current->comm, current->pid);
2654 
2655 	if (prot)
2656 		return ret;
2657 	start = start & PAGE_MASK;
2658 	size = size & PAGE_MASK;
2659 
2660 	if (start + size <= start)
2661 		return ret;
2662 
2663 	/* Does pgoff wrap? */
2664 	if (pgoff + (size >> PAGE_SHIFT) < pgoff)
2665 		return ret;
2666 
2667 	down_write(&mm->mmap_sem);
2668 	vma = find_vma(mm, start);
2669 
2670 	if (!vma || !(vma->vm_flags & VM_SHARED))
2671 		goto out;
2672 
2673 	if (start < vma->vm_start || start + size > vma->vm_end)
2674 		goto out;
2675 
2676 	if (pgoff == linear_page_index(vma, start)) {
2677 		ret = 0;
2678 		goto out;
2679 	}
2680 
2681 	prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
2682 	prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
2683 	prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
2684 
2685 	flags &= MAP_NONBLOCK;
2686 	flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
2687 	if (vma->vm_flags & VM_LOCKED) {
2688 		flags |= MAP_LOCKED;
2689 		/* drop PG_Mlocked flag for over-mapped range */
2690 		munlock_vma_pages_range(vma, start, start + size);
2691 	}
2692 
2693 	file = get_file(vma->vm_file);
2694 	ret = do_mmap_pgoff(vma->vm_file, start, size,
2695 			prot, flags, pgoff, &populate);
2696 	fput(file);
2697 out:
2698 	up_write(&mm->mmap_sem);
2699 	if (populate)
2700 		mm_populate(ret, populate);
2701 	if (!IS_ERR_VALUE(ret))
2702 		ret = 0;
2703 	return ret;
2704 }
2705 
2706 static inline void verify_mm_writelocked(struct mm_struct *mm)
2707 {
2708 #ifdef CONFIG_DEBUG_VM
2709 	if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2710 		WARN_ON(1);
2711 		up_read(&mm->mmap_sem);
2712 	}
2713 #endif
2714 }
2715 
2716 /*
2717  *  this is really a simplified "do_mmap".  it only handles
2718  *  anonymous maps.  eventually we may be able to do some
2719  *  brk-specific accounting here.
2720  */
2721 static unsigned long do_brk(unsigned long addr, unsigned long len)
2722 {
2723 	struct mm_struct *mm = current->mm;
2724 	struct vm_area_struct *vma, *prev;
2725 	unsigned long flags;
2726 	struct rb_node **rb_link, *rb_parent;
2727 	pgoff_t pgoff = addr >> PAGE_SHIFT;
2728 	int error;
2729 
2730 	len = PAGE_ALIGN(len);
2731 	if (!len)
2732 		return addr;
2733 
2734 	flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2735 
2736 	error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2737 	if (error & ~PAGE_MASK)
2738 		return error;
2739 
2740 	error = mlock_future_check(mm, mm->def_flags, len);
2741 	if (error)
2742 		return error;
2743 
2744 	/*
2745 	 * mm->mmap_sem is required to protect against another thread
2746 	 * changing the mappings in case we sleep.
2747 	 */
2748 	verify_mm_writelocked(mm);
2749 
2750 	/*
2751 	 * Clear old maps.  this also does some error checking for us
2752 	 */
2753 	while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
2754 			      &rb_parent)) {
2755 		if (do_munmap(mm, addr, len))
2756 			return -ENOMEM;
2757 	}
2758 
2759 	/* Check against address space limits *after* clearing old maps... */
2760 	if (!may_expand_vm(mm, len >> PAGE_SHIFT))
2761 		return -ENOMEM;
2762 
2763 	if (mm->map_count > sysctl_max_map_count)
2764 		return -ENOMEM;
2765 
2766 	if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
2767 		return -ENOMEM;
2768 
2769 	/* Can we just expand an old private anonymous mapping? */
2770 	vma = vma_merge(mm, prev, addr, addr + len, flags,
2771 			NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
2772 	if (vma)
2773 		goto out;
2774 
2775 	/*
2776 	 * create a vma struct for an anonymous mapping
2777 	 */
2778 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2779 	if (!vma) {
2780 		vm_unacct_memory(len >> PAGE_SHIFT);
2781 		return -ENOMEM;
2782 	}
2783 
2784 	INIT_LIST_HEAD(&vma->anon_vma_chain);
2785 	vma->vm_mm = mm;
2786 	vma->vm_start = addr;
2787 	vma->vm_end = addr + len;
2788 	vma->vm_pgoff = pgoff;
2789 	vma->vm_flags = flags;
2790 	vma->vm_page_prot = vm_get_page_prot(flags);
2791 	vma_link(mm, vma, prev, rb_link, rb_parent);
2792 out:
2793 	perf_event_mmap(vma);
2794 	mm->total_vm += len >> PAGE_SHIFT;
2795 	if (flags & VM_LOCKED)
2796 		mm->locked_vm += (len >> PAGE_SHIFT);
2797 	vma->vm_flags |= VM_SOFTDIRTY;
2798 	return addr;
2799 }
2800 
2801 unsigned long vm_brk(unsigned long addr, unsigned long len)
2802 {
2803 	struct mm_struct *mm = current->mm;
2804 	unsigned long ret;
2805 	bool populate;
2806 
2807 	down_write(&mm->mmap_sem);
2808 	ret = do_brk(addr, len);
2809 	populate = ((mm->def_flags & VM_LOCKED) != 0);
2810 	up_write(&mm->mmap_sem);
2811 	if (populate)
2812 		mm_populate(addr, len);
2813 	return ret;
2814 }
2815 EXPORT_SYMBOL(vm_brk);
2816 
2817 /* Release all mmaps. */
2818 void exit_mmap(struct mm_struct *mm)
2819 {
2820 	struct mmu_gather tlb;
2821 	struct vm_area_struct *vma;
2822 	unsigned long nr_accounted = 0;
2823 
2824 	/* mm's last user has gone, and its about to be pulled down */
2825 	mmu_notifier_release(mm);
2826 
2827 	if (mm->locked_vm) {
2828 		vma = mm->mmap;
2829 		while (vma) {
2830 			if (vma->vm_flags & VM_LOCKED)
2831 				munlock_vma_pages_all(vma);
2832 			vma = vma->vm_next;
2833 		}
2834 	}
2835 
2836 	arch_exit_mmap(mm);
2837 
2838 	vma = mm->mmap;
2839 	if (!vma)	/* Can happen if dup_mmap() received an OOM */
2840 		return;
2841 
2842 	lru_add_drain();
2843 	flush_cache_mm(mm);
2844 	tlb_gather_mmu(&tlb, mm, 0, -1);
2845 	/* update_hiwater_rss(mm) here? but nobody should be looking */
2846 	/* Use -1 here to ensure all VMAs in the mm are unmapped */
2847 	unmap_vmas(&tlb, vma, 0, -1);
2848 
2849 	free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
2850 	tlb_finish_mmu(&tlb, 0, -1);
2851 
2852 	/*
2853 	 * Walk the list again, actually closing and freeing it,
2854 	 * with preemption enabled, without holding any MM locks.
2855 	 */
2856 	while (vma) {
2857 		if (vma->vm_flags & VM_ACCOUNT)
2858 			nr_accounted += vma_pages(vma);
2859 		vma = remove_vma(vma);
2860 	}
2861 	vm_unacct_memory(nr_accounted);
2862 }
2863 
2864 /* Insert vm structure into process list sorted by address
2865  * and into the inode's i_mmap tree.  If vm_file is non-NULL
2866  * then i_mmap_rwsem is taken here.
2867  */
2868 int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
2869 {
2870 	struct vm_area_struct *prev;
2871 	struct rb_node **rb_link, *rb_parent;
2872 
2873 	if (find_vma_links(mm, vma->vm_start, vma->vm_end,
2874 			   &prev, &rb_link, &rb_parent))
2875 		return -ENOMEM;
2876 	if ((vma->vm_flags & VM_ACCOUNT) &&
2877 	     security_vm_enough_memory_mm(mm, vma_pages(vma)))
2878 		return -ENOMEM;
2879 
2880 	/*
2881 	 * The vm_pgoff of a purely anonymous vma should be irrelevant
2882 	 * until its first write fault, when page's anon_vma and index
2883 	 * are set.  But now set the vm_pgoff it will almost certainly
2884 	 * end up with (unless mremap moves it elsewhere before that
2885 	 * first wfault), so /proc/pid/maps tells a consistent story.
2886 	 *
2887 	 * By setting it to reflect the virtual start address of the
2888 	 * vma, merges and splits can happen in a seamless way, just
2889 	 * using the existing file pgoff checks and manipulations.
2890 	 * Similarly in do_mmap_pgoff and in do_brk.
2891 	 */
2892 	if (vma_is_anonymous(vma)) {
2893 		BUG_ON(vma->anon_vma);
2894 		vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2895 	}
2896 
2897 	vma_link(mm, vma, prev, rb_link, rb_parent);
2898 	return 0;
2899 }
2900 
2901 /*
2902  * Copy the vma structure to a new location in the same mm,
2903  * prior to moving page table entries, to effect an mremap move.
2904  */
2905 struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2906 	unsigned long addr, unsigned long len, pgoff_t pgoff,
2907 	bool *need_rmap_locks)
2908 {
2909 	struct vm_area_struct *vma = *vmap;
2910 	unsigned long vma_start = vma->vm_start;
2911 	struct mm_struct *mm = vma->vm_mm;
2912 	struct vm_area_struct *new_vma, *prev;
2913 	struct rb_node **rb_link, *rb_parent;
2914 	bool faulted_in_anon_vma = true;
2915 
2916 	/*
2917 	 * If anonymous vma has not yet been faulted, update new pgoff
2918 	 * to match new location, to increase its chance of merging.
2919 	 */
2920 	if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
2921 		pgoff = addr >> PAGE_SHIFT;
2922 		faulted_in_anon_vma = false;
2923 	}
2924 
2925 	if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
2926 		return NULL;	/* should never get here */
2927 	new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2928 			    vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
2929 			    vma->vm_userfaultfd_ctx);
2930 	if (new_vma) {
2931 		/*
2932 		 * Source vma may have been merged into new_vma
2933 		 */
2934 		if (unlikely(vma_start >= new_vma->vm_start &&
2935 			     vma_start < new_vma->vm_end)) {
2936 			/*
2937 			 * The only way we can get a vma_merge with
2938 			 * self during an mremap is if the vma hasn't
2939 			 * been faulted in yet and we were allowed to
2940 			 * reset the dst vma->vm_pgoff to the
2941 			 * destination address of the mremap to allow
2942 			 * the merge to happen. mremap must change the
2943 			 * vm_pgoff linearity between src and dst vmas
2944 			 * (in turn preventing a vma_merge) to be
2945 			 * safe. It is only safe to keep the vm_pgoff
2946 			 * linear if there are no pages mapped yet.
2947 			 */
2948 			VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
2949 			*vmap = vma = new_vma;
2950 		}
2951 		*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
2952 	} else {
2953 		new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2954 		if (!new_vma)
2955 			goto out;
2956 		*new_vma = *vma;
2957 		new_vma->vm_start = addr;
2958 		new_vma->vm_end = addr + len;
2959 		new_vma->vm_pgoff = pgoff;
2960 		if (vma_dup_policy(vma, new_vma))
2961 			goto out_free_vma;
2962 		INIT_LIST_HEAD(&new_vma->anon_vma_chain);
2963 		if (anon_vma_clone(new_vma, vma))
2964 			goto out_free_mempol;
2965 		if (new_vma->vm_file)
2966 			get_file(new_vma->vm_file);
2967 		if (new_vma->vm_ops && new_vma->vm_ops->open)
2968 			new_vma->vm_ops->open(new_vma);
2969 		vma_link(mm, new_vma, prev, rb_link, rb_parent);
2970 		*need_rmap_locks = false;
2971 	}
2972 	return new_vma;
2973 
2974 out_free_mempol:
2975 	mpol_put(vma_policy(new_vma));
2976 out_free_vma:
2977 	kmem_cache_free(vm_area_cachep, new_vma);
2978 out:
2979 	return NULL;
2980 }
2981 
2982 /*
2983  * Return true if the calling process may expand its vm space by the passed
2984  * number of pages
2985  */
2986 int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2987 {
2988 	unsigned long cur = mm->total_vm;	/* pages */
2989 	unsigned long lim;
2990 
2991 	lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
2992 
2993 	if (cur + npages > lim)
2994 		return 0;
2995 	return 1;
2996 }
2997 
2998 static int special_mapping_fault(struct vm_area_struct *vma,
2999 				 struct vm_fault *vmf);
3000 
3001 /*
3002  * Having a close hook prevents vma merging regardless of flags.
3003  */
3004 static void special_mapping_close(struct vm_area_struct *vma)
3005 {
3006 }
3007 
3008 static const char *special_mapping_name(struct vm_area_struct *vma)
3009 {
3010 	return ((struct vm_special_mapping *)vma->vm_private_data)->name;
3011 }
3012 
3013 static const struct vm_operations_struct special_mapping_vmops = {
3014 	.close = special_mapping_close,
3015 	.fault = special_mapping_fault,
3016 	.name = special_mapping_name,
3017 };
3018 
3019 static const struct vm_operations_struct legacy_special_mapping_vmops = {
3020 	.close = special_mapping_close,
3021 	.fault = special_mapping_fault,
3022 };
3023 
3024 static int special_mapping_fault(struct vm_area_struct *vma,
3025 				struct vm_fault *vmf)
3026 {
3027 	pgoff_t pgoff;
3028 	struct page **pages;
3029 
3030 	if (vma->vm_ops == &legacy_special_mapping_vmops)
3031 		pages = vma->vm_private_data;
3032 	else
3033 		pages = ((struct vm_special_mapping *)vma->vm_private_data)->
3034 			pages;
3035 
3036 	for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
3037 		pgoff--;
3038 
3039 	if (*pages) {
3040 		struct page *page = *pages;
3041 		get_page(page);
3042 		vmf->page = page;
3043 		return 0;
3044 	}
3045 
3046 	return VM_FAULT_SIGBUS;
3047 }
3048 
3049 static struct vm_area_struct *__install_special_mapping(
3050 	struct mm_struct *mm,
3051 	unsigned long addr, unsigned long len,
3052 	unsigned long vm_flags, const struct vm_operations_struct *ops,
3053 	void *priv)
3054 {
3055 	int ret;
3056 	struct vm_area_struct *vma;
3057 
3058 	vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
3059 	if (unlikely(vma == NULL))
3060 		return ERR_PTR(-ENOMEM);
3061 
3062 	INIT_LIST_HEAD(&vma->anon_vma_chain);
3063 	vma->vm_mm = mm;
3064 	vma->vm_start = addr;
3065 	vma->vm_end = addr + len;
3066 
3067 	vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
3068 	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
3069 
3070 	vma->vm_ops = ops;
3071 	vma->vm_private_data = priv;
3072 
3073 	ret = insert_vm_struct(mm, vma);
3074 	if (ret)
3075 		goto out;
3076 
3077 	mm->total_vm += len >> PAGE_SHIFT;
3078 
3079 	perf_event_mmap(vma);
3080 
3081 	return vma;
3082 
3083 out:
3084 	kmem_cache_free(vm_area_cachep, vma);
3085 	return ERR_PTR(ret);
3086 }
3087 
3088 /*
3089  * Called with mm->mmap_sem held for writing.
3090  * Insert a new vma covering the given region, with the given flags.
3091  * Its pages are supplied by the given array of struct page *.
3092  * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
3093  * The region past the last page supplied will always produce SIGBUS.
3094  * The array pointer and the pages it points to are assumed to stay alive
3095  * for as long as this mapping might exist.
3096  */
3097 struct vm_area_struct *_install_special_mapping(
3098 	struct mm_struct *mm,
3099 	unsigned long addr, unsigned long len,
3100 	unsigned long vm_flags, const struct vm_special_mapping *spec)
3101 {
3102 	return __install_special_mapping(mm, addr, len, vm_flags,
3103 					 &special_mapping_vmops, (void *)spec);
3104 }
3105 
3106 int install_special_mapping(struct mm_struct *mm,
3107 			    unsigned long addr, unsigned long len,
3108 			    unsigned long vm_flags, struct page **pages)
3109 {
3110 	struct vm_area_struct *vma = __install_special_mapping(
3111 		mm, addr, len, vm_flags, &legacy_special_mapping_vmops,
3112 		(void *)pages);
3113 
3114 	return PTR_ERR_OR_ZERO(vma);
3115 }
3116 
3117 static DEFINE_MUTEX(mm_all_locks_mutex);
3118 
3119 static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
3120 {
3121 	if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3122 		/*
3123 		 * The LSB of head.next can't change from under us
3124 		 * because we hold the mm_all_locks_mutex.
3125 		 */
3126 		down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
3127 		/*
3128 		 * We can safely modify head.next after taking the
3129 		 * anon_vma->root->rwsem. If some other vma in this mm shares
3130 		 * the same anon_vma we won't take it again.
3131 		 *
3132 		 * No need of atomic instructions here, head.next
3133 		 * can't change from under us thanks to the
3134 		 * anon_vma->root->rwsem.
3135 		 */
3136 		if (__test_and_set_bit(0, (unsigned long *)
3137 				       &anon_vma->root->rb_root.rb_node))
3138 			BUG();
3139 	}
3140 }
3141 
3142 static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
3143 {
3144 	if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3145 		/*
3146 		 * AS_MM_ALL_LOCKS can't change from under us because
3147 		 * we hold the mm_all_locks_mutex.
3148 		 *
3149 		 * Operations on ->flags have to be atomic because
3150 		 * even if AS_MM_ALL_LOCKS is stable thanks to the
3151 		 * mm_all_locks_mutex, there may be other cpus
3152 		 * changing other bitflags in parallel to us.
3153 		 */
3154 		if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
3155 			BUG();
3156 		down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
3157 	}
3158 }
3159 
3160 /*
3161  * This operation locks against the VM for all pte/vma/mm related
3162  * operations that could ever happen on a certain mm. This includes
3163  * vmtruncate, try_to_unmap, and all page faults.
3164  *
3165  * The caller must take the mmap_sem in write mode before calling
3166  * mm_take_all_locks(). The caller isn't allowed to release the
3167  * mmap_sem until mm_drop_all_locks() returns.
3168  *
3169  * mmap_sem in write mode is required in order to block all operations
3170  * that could modify pagetables and free pages without need of
3171  * altering the vma layout. It's also needed in write mode to avoid new
3172  * anon_vmas to be associated with existing vmas.
3173  *
3174  * A single task can't take more than one mm_take_all_locks() in a row
3175  * or it would deadlock.
3176  *
3177  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
3178  * mapping->flags avoid to take the same lock twice, if more than one
3179  * vma in this mm is backed by the same anon_vma or address_space.
3180  *
3181  * We can take all the locks in random order because the VM code
3182  * taking i_mmap_rwsem or anon_vma->rwsem outside the mmap_sem never
3183  * takes more than one of them in a row. Secondly we're protected
3184  * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
3185  *
3186  * mm_take_all_locks() and mm_drop_all_locks are expensive operations
3187  * that may have to take thousand of locks.
3188  *
3189  * mm_take_all_locks() can fail if it's interrupted by signals.
3190  */
3191 int mm_take_all_locks(struct mm_struct *mm)
3192 {
3193 	struct vm_area_struct *vma;
3194 	struct anon_vma_chain *avc;
3195 
3196 	BUG_ON(down_read_trylock(&mm->mmap_sem));
3197 
3198 	mutex_lock(&mm_all_locks_mutex);
3199 
3200 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3201 		if (signal_pending(current))
3202 			goto out_unlock;
3203 		if (vma->vm_file && vma->vm_file->f_mapping)
3204 			vm_lock_mapping(mm, vma->vm_file->f_mapping);
3205 	}
3206 
3207 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3208 		if (signal_pending(current))
3209 			goto out_unlock;
3210 		if (vma->anon_vma)
3211 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3212 				vm_lock_anon_vma(mm, avc->anon_vma);
3213 	}
3214 
3215 	return 0;
3216 
3217 out_unlock:
3218 	mm_drop_all_locks(mm);
3219 	return -EINTR;
3220 }
3221 
3222 static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
3223 {
3224 	if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
3225 		/*
3226 		 * The LSB of head.next can't change to 0 from under
3227 		 * us because we hold the mm_all_locks_mutex.
3228 		 *
3229 		 * We must however clear the bitflag before unlocking
3230 		 * the vma so the users using the anon_vma->rb_root will
3231 		 * never see our bitflag.
3232 		 *
3233 		 * No need of atomic instructions here, head.next
3234 		 * can't change from under us until we release the
3235 		 * anon_vma->root->rwsem.
3236 		 */
3237 		if (!__test_and_clear_bit(0, (unsigned long *)
3238 					  &anon_vma->root->rb_root.rb_node))
3239 			BUG();
3240 		anon_vma_unlock_write(anon_vma);
3241 	}
3242 }
3243 
3244 static void vm_unlock_mapping(struct address_space *mapping)
3245 {
3246 	if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
3247 		/*
3248 		 * AS_MM_ALL_LOCKS can't change to 0 from under us
3249 		 * because we hold the mm_all_locks_mutex.
3250 		 */
3251 		i_mmap_unlock_write(mapping);
3252 		if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
3253 					&mapping->flags))
3254 			BUG();
3255 	}
3256 }
3257 
3258 /*
3259  * The mmap_sem cannot be released by the caller until
3260  * mm_drop_all_locks() returns.
3261  */
3262 void mm_drop_all_locks(struct mm_struct *mm)
3263 {
3264 	struct vm_area_struct *vma;
3265 	struct anon_vma_chain *avc;
3266 
3267 	BUG_ON(down_read_trylock(&mm->mmap_sem));
3268 	BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
3269 
3270 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
3271 		if (vma->anon_vma)
3272 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
3273 				vm_unlock_anon_vma(avc->anon_vma);
3274 		if (vma->vm_file && vma->vm_file->f_mapping)
3275 			vm_unlock_mapping(vma->vm_file->f_mapping);
3276 	}
3277 
3278 	mutex_unlock(&mm_all_locks_mutex);
3279 }
3280 
3281 /*
3282  * initialise the VMA slab
3283  */
3284 void __init mmap_init(void)
3285 {
3286 	int ret;
3287 
3288 	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
3289 	VM_BUG_ON(ret);
3290 }
3291 
3292 /*
3293  * Initialise sysctl_user_reserve_kbytes.
3294  *
3295  * This is intended to prevent a user from starting a single memory hogging
3296  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
3297  * mode.
3298  *
3299  * The default value is min(3% of free memory, 128MB)
3300  * 128MB is enough to recover with sshd/login, bash, and top/kill.
3301  */
3302 static int init_user_reserve(void)
3303 {
3304 	unsigned long free_kbytes;
3305 
3306 	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3307 
3308 	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
3309 	return 0;
3310 }
3311 subsys_initcall(init_user_reserve);
3312 
3313 /*
3314  * Initialise sysctl_admin_reserve_kbytes.
3315  *
3316  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
3317  * to log in and kill a memory hogging process.
3318  *
3319  * Systems with more than 256MB will reserve 8MB, enough to recover
3320  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
3321  * only reserve 3% of free pages by default.
3322  */
3323 static int init_admin_reserve(void)
3324 {
3325 	unsigned long free_kbytes;
3326 
3327 	free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3328 
3329 	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
3330 	return 0;
3331 }
3332 subsys_initcall(init_admin_reserve);
3333 
3334 /*
3335  * Reinititalise user and admin reserves if memory is added or removed.
3336  *
3337  * The default user reserve max is 128MB, and the default max for the
3338  * admin reserve is 8MB. These are usually, but not always, enough to
3339  * enable recovery from a memory hogging process using login/sshd, a shell,
3340  * and tools like top. It may make sense to increase or even disable the
3341  * reserve depending on the existence of swap or variations in the recovery
3342  * tools. So, the admin may have changed them.
3343  *
3344  * If memory is added and the reserves have been eliminated or increased above
3345  * the default max, then we'll trust the admin.
3346  *
3347  * If memory is removed and there isn't enough free memory, then we
3348  * need to reset the reserves.
3349  *
3350  * Otherwise keep the reserve set by the admin.
3351  */
3352 static int reserve_mem_notifier(struct notifier_block *nb,
3353 			     unsigned long action, void *data)
3354 {
3355 	unsigned long tmp, free_kbytes;
3356 
3357 	switch (action) {
3358 	case MEM_ONLINE:
3359 		/* Default max is 128MB. Leave alone if modified by operator. */
3360 		tmp = sysctl_user_reserve_kbytes;
3361 		if (0 < tmp && tmp < (1UL << 17))
3362 			init_user_reserve();
3363 
3364 		/* Default max is 8MB.  Leave alone if modified by operator. */
3365 		tmp = sysctl_admin_reserve_kbytes;
3366 		if (0 < tmp && tmp < (1UL << 13))
3367 			init_admin_reserve();
3368 
3369 		break;
3370 	case MEM_OFFLINE:
3371 		free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
3372 
3373 		if (sysctl_user_reserve_kbytes > free_kbytes) {
3374 			init_user_reserve();
3375 			pr_info("vm.user_reserve_kbytes reset to %lu\n",
3376 				sysctl_user_reserve_kbytes);
3377 		}
3378 
3379 		if (sysctl_admin_reserve_kbytes > free_kbytes) {
3380 			init_admin_reserve();
3381 			pr_info("vm.admin_reserve_kbytes reset to %lu\n",
3382 				sysctl_admin_reserve_kbytes);
3383 		}
3384 		break;
3385 	default:
3386 		break;
3387 	}
3388 	return NOTIFY_OK;
3389 }
3390 
3391 static struct notifier_block reserve_mem_nb = {
3392 	.notifier_call = reserve_mem_notifier,
3393 };
3394 
3395 static int __meminit init_reserve_notifier(void)
3396 {
3397 	if (register_hotmemory_notifier(&reserve_mem_nb))
3398 		pr_err("Failed registering memory add/remove notifier for admin reserve\n");
3399 
3400 	return 0;
3401 }
3402 subsys_initcall(init_reserve_notifier);
3403