xref: /openbmc/linux/mm/nommu.c (revision 1d202d14)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/nommu.c
4  *
5  *  Replacement code for mm functions to support CPU's that don't
6  *  have any form of memory management unit (thus no virtual memory).
7  *
8  *  See Documentation/admin-guide/mm/nommu-mmap.rst
9  *
10  *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
11  *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
12  *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
13  *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
14  *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
15  */
16 
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 
19 #include <linux/export.h>
20 #include <linux/mm.h>
21 #include <linux/sched/mm.h>
22 #include <linux/vmacache.h>
23 #include <linux/mman.h>
24 #include <linux/swap.h>
25 #include <linux/file.h>
26 #include <linux/highmem.h>
27 #include <linux/pagemap.h>
28 #include <linux/slab.h>
29 #include <linux/vmalloc.h>
30 #include <linux/backing-dev.h>
31 #include <linux/compiler.h>
32 #include <linux/mount.h>
33 #include <linux/personality.h>
34 #include <linux/security.h>
35 #include <linux/syscalls.h>
36 #include <linux/audit.h>
37 #include <linux/printk.h>
38 
39 #include <linux/uaccess.h>
40 #include <asm/tlb.h>
41 #include <asm/tlbflush.h>
42 #include <asm/mmu_context.h>
43 #include "internal.h"
44 
45 void *high_memory;
46 EXPORT_SYMBOL(high_memory);
47 struct page *mem_map;
48 unsigned long max_mapnr;
49 EXPORT_SYMBOL(max_mapnr);
50 unsigned long highest_memmap_pfn;
51 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
52 int heap_stack_gap = 0;
53 
54 atomic_long_t mmap_pages_allocated;
55 
56 EXPORT_SYMBOL(mem_map);
57 
58 /* list of mapped, potentially shareable regions */
59 static struct kmem_cache *vm_region_jar;
60 struct rb_root nommu_region_tree = RB_ROOT;
61 DECLARE_RWSEM(nommu_region_sem);
62 
63 const struct vm_operations_struct generic_file_vm_ops = {
64 };
65 
66 /*
67  * Return the total memory allocated for this pointer, not
68  * just what the caller asked for.
69  *
70  * Doesn't have to be accurate, i.e. may have races.
71  */
72 unsigned int kobjsize(const void *objp)
73 {
74 	struct page *page;
75 
76 	/*
77 	 * If the object we have should not have ksize performed on it,
78 	 * return size of 0
79 	 */
80 	if (!objp || !virt_addr_valid(objp))
81 		return 0;
82 
83 	page = virt_to_head_page(objp);
84 
85 	/*
86 	 * If the allocator sets PageSlab, we know the pointer came from
87 	 * kmalloc().
88 	 */
89 	if (PageSlab(page))
90 		return ksize(objp);
91 
92 	/*
93 	 * If it's not a compound page, see if we have a matching VMA
94 	 * region. This test is intentionally done in reverse order,
95 	 * so if there's no VMA, we still fall through and hand back
96 	 * PAGE_SIZE for 0-order pages.
97 	 */
98 	if (!PageCompound(page)) {
99 		struct vm_area_struct *vma;
100 
101 		vma = find_vma(current->mm, (unsigned long)objp);
102 		if (vma)
103 			return vma->vm_end - vma->vm_start;
104 	}
105 
106 	/*
107 	 * The ksize() function is only guaranteed to work for pointers
108 	 * returned by kmalloc(). So handle arbitrary pointers here.
109 	 */
110 	return page_size(page);
111 }
112 
113 /**
114  * follow_pfn - look up PFN at a user virtual address
115  * @vma: memory mapping
116  * @address: user virtual address
117  * @pfn: location to store found PFN
118  *
119  * Only IO mappings and raw PFN mappings are allowed.
120  *
121  * Returns zero and the pfn at @pfn on success, -ve otherwise.
122  */
123 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
124 	unsigned long *pfn)
125 {
126 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
127 		return -EINVAL;
128 
129 	*pfn = address >> PAGE_SHIFT;
130 	return 0;
131 }
132 EXPORT_SYMBOL(follow_pfn);
133 
134 LIST_HEAD(vmap_area_list);
135 
136 void vfree(const void *addr)
137 {
138 	kfree(addr);
139 }
140 EXPORT_SYMBOL(vfree);
141 
142 void *__vmalloc(unsigned long size, gfp_t gfp_mask)
143 {
144 	/*
145 	 *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
146 	 * returns only a logical address.
147 	 */
148 	return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
149 }
150 EXPORT_SYMBOL(__vmalloc);
151 
152 void *__vmalloc_node_range(unsigned long size, unsigned long align,
153 		unsigned long start, unsigned long end, gfp_t gfp_mask,
154 		pgprot_t prot, unsigned long vm_flags, int node,
155 		const void *caller)
156 {
157 	return __vmalloc(size, gfp_mask);
158 }
159 
160 void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask,
161 		int node, const void *caller)
162 {
163 	return __vmalloc(size, gfp_mask);
164 }
165 
166 static void *__vmalloc_user_flags(unsigned long size, gfp_t flags)
167 {
168 	void *ret;
169 
170 	ret = __vmalloc(size, flags);
171 	if (ret) {
172 		struct vm_area_struct *vma;
173 
174 		mmap_write_lock(current->mm);
175 		vma = find_vma(current->mm, (unsigned long)ret);
176 		if (vma)
177 			vma->vm_flags |= VM_USERMAP;
178 		mmap_write_unlock(current->mm);
179 	}
180 
181 	return ret;
182 }
183 
184 void *vmalloc_user(unsigned long size)
185 {
186 	return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO);
187 }
188 EXPORT_SYMBOL(vmalloc_user);
189 
190 struct page *vmalloc_to_page(const void *addr)
191 {
192 	return virt_to_page(addr);
193 }
194 EXPORT_SYMBOL(vmalloc_to_page);
195 
196 unsigned long vmalloc_to_pfn(const void *addr)
197 {
198 	return page_to_pfn(virt_to_page(addr));
199 }
200 EXPORT_SYMBOL(vmalloc_to_pfn);
201 
202 long vread(char *buf, char *addr, unsigned long count)
203 {
204 	/* Don't allow overflow */
205 	if ((unsigned long) buf + count < count)
206 		count = -(unsigned long) buf;
207 
208 	memcpy(buf, addr, count);
209 	return count;
210 }
211 
212 /*
213  *	vmalloc  -  allocate virtually contiguous memory
214  *
215  *	@size:		allocation size
216  *
217  *	Allocate enough pages to cover @size from the page level
218  *	allocator and map them into contiguous kernel virtual space.
219  *
220  *	For tight control over page level allocator and protection flags
221  *	use __vmalloc() instead.
222  */
223 void *vmalloc(unsigned long size)
224 {
225 	return __vmalloc(size, GFP_KERNEL);
226 }
227 EXPORT_SYMBOL(vmalloc);
228 
229 void *vmalloc_huge(unsigned long size, gfp_t gfp_mask) __weak __alias(__vmalloc);
230 
231 /*
232  *	vzalloc - allocate virtually contiguous memory with zero fill
233  *
234  *	@size:		allocation size
235  *
236  *	Allocate enough pages to cover @size from the page level
237  *	allocator and map them into contiguous kernel virtual space.
238  *	The memory allocated is set to zero.
239  *
240  *	For tight control over page level allocator and protection flags
241  *	use __vmalloc() instead.
242  */
243 void *vzalloc(unsigned long size)
244 {
245 	return __vmalloc(size, GFP_KERNEL | __GFP_ZERO);
246 }
247 EXPORT_SYMBOL(vzalloc);
248 
249 /**
250  * vmalloc_node - allocate memory on a specific node
251  * @size:	allocation size
252  * @node:	numa node
253  *
254  * Allocate enough pages to cover @size from the page level
255  * allocator and map them into contiguous kernel virtual space.
256  *
257  * For tight control over page level allocator and protection flags
258  * use __vmalloc() instead.
259  */
260 void *vmalloc_node(unsigned long size, int node)
261 {
262 	return vmalloc(size);
263 }
264 EXPORT_SYMBOL(vmalloc_node);
265 
266 /**
267  * vzalloc_node - allocate memory on a specific node with zero fill
268  * @size:	allocation size
269  * @node:	numa node
270  *
271  * Allocate enough pages to cover @size from the page level
272  * allocator and map them into contiguous kernel virtual space.
273  * The memory allocated is set to zero.
274  *
275  * For tight control over page level allocator and protection flags
276  * use __vmalloc() instead.
277  */
278 void *vzalloc_node(unsigned long size, int node)
279 {
280 	return vzalloc(size);
281 }
282 EXPORT_SYMBOL(vzalloc_node);
283 
284 /**
285  * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
286  *	@size:		allocation size
287  *
288  *	Allocate enough 32bit PA addressable pages to cover @size from the
289  *	page level allocator and map them into contiguous kernel virtual space.
290  */
291 void *vmalloc_32(unsigned long size)
292 {
293 	return __vmalloc(size, GFP_KERNEL);
294 }
295 EXPORT_SYMBOL(vmalloc_32);
296 
297 /**
298  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
299  *	@size:		allocation size
300  *
301  * The resulting memory area is 32bit addressable and zeroed so it can be
302  * mapped to userspace without leaking data.
303  *
304  * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
305  * remap_vmalloc_range() are permissible.
306  */
307 void *vmalloc_32_user(unsigned long size)
308 {
309 	/*
310 	 * We'll have to sort out the ZONE_DMA bits for 64-bit,
311 	 * but for now this can simply use vmalloc_user() directly.
312 	 */
313 	return vmalloc_user(size);
314 }
315 EXPORT_SYMBOL(vmalloc_32_user);
316 
317 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
318 {
319 	BUG();
320 	return NULL;
321 }
322 EXPORT_SYMBOL(vmap);
323 
324 void vunmap(const void *addr)
325 {
326 	BUG();
327 }
328 EXPORT_SYMBOL(vunmap);
329 
330 void *vm_map_ram(struct page **pages, unsigned int count, int node)
331 {
332 	BUG();
333 	return NULL;
334 }
335 EXPORT_SYMBOL(vm_map_ram);
336 
337 void vm_unmap_ram(const void *mem, unsigned int count)
338 {
339 	BUG();
340 }
341 EXPORT_SYMBOL(vm_unmap_ram);
342 
343 void vm_unmap_aliases(void)
344 {
345 }
346 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
347 
348 void free_vm_area(struct vm_struct *area)
349 {
350 	BUG();
351 }
352 EXPORT_SYMBOL_GPL(free_vm_area);
353 
354 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
355 		   struct page *page)
356 {
357 	return -EINVAL;
358 }
359 EXPORT_SYMBOL(vm_insert_page);
360 
361 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
362 			unsigned long num)
363 {
364 	return -EINVAL;
365 }
366 EXPORT_SYMBOL(vm_map_pages);
367 
368 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
369 				unsigned long num)
370 {
371 	return -EINVAL;
372 }
373 EXPORT_SYMBOL(vm_map_pages_zero);
374 
375 /*
376  *  sys_brk() for the most part doesn't need the global kernel
377  *  lock, except when an application is doing something nasty
378  *  like trying to un-brk an area that has already been mapped
379  *  to a regular file.  in this case, the unmapping will need
380  *  to invoke file system routines that need the global lock.
381  */
382 SYSCALL_DEFINE1(brk, unsigned long, brk)
383 {
384 	struct mm_struct *mm = current->mm;
385 
386 	if (brk < mm->start_brk || brk > mm->context.end_brk)
387 		return mm->brk;
388 
389 	if (mm->brk == brk)
390 		return mm->brk;
391 
392 	/*
393 	 * Always allow shrinking brk
394 	 */
395 	if (brk <= mm->brk) {
396 		mm->brk = brk;
397 		return brk;
398 	}
399 
400 	/*
401 	 * Ok, looks good - let it rip.
402 	 */
403 	flush_icache_user_range(mm->brk, brk);
404 	return mm->brk = brk;
405 }
406 
407 /*
408  * initialise the percpu counter for VM and region record slabs
409  */
410 void __init mmap_init(void)
411 {
412 	int ret;
413 
414 	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
415 	VM_BUG_ON(ret);
416 	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
417 }
418 
419 /*
420  * validate the region tree
421  * - the caller must hold the region lock
422  */
423 #ifdef CONFIG_DEBUG_NOMMU_REGIONS
424 static noinline void validate_nommu_regions(void)
425 {
426 	struct vm_region *region, *last;
427 	struct rb_node *p, *lastp;
428 
429 	lastp = rb_first(&nommu_region_tree);
430 	if (!lastp)
431 		return;
432 
433 	last = rb_entry(lastp, struct vm_region, vm_rb);
434 	BUG_ON(last->vm_end <= last->vm_start);
435 	BUG_ON(last->vm_top < last->vm_end);
436 
437 	while ((p = rb_next(lastp))) {
438 		region = rb_entry(p, struct vm_region, vm_rb);
439 		last = rb_entry(lastp, struct vm_region, vm_rb);
440 
441 		BUG_ON(region->vm_end <= region->vm_start);
442 		BUG_ON(region->vm_top < region->vm_end);
443 		BUG_ON(region->vm_start < last->vm_top);
444 
445 		lastp = p;
446 	}
447 }
448 #else
449 static void validate_nommu_regions(void)
450 {
451 }
452 #endif
453 
454 /*
455  * add a region into the global tree
456  */
457 static void add_nommu_region(struct vm_region *region)
458 {
459 	struct vm_region *pregion;
460 	struct rb_node **p, *parent;
461 
462 	validate_nommu_regions();
463 
464 	parent = NULL;
465 	p = &nommu_region_tree.rb_node;
466 	while (*p) {
467 		parent = *p;
468 		pregion = rb_entry(parent, struct vm_region, vm_rb);
469 		if (region->vm_start < pregion->vm_start)
470 			p = &(*p)->rb_left;
471 		else if (region->vm_start > pregion->vm_start)
472 			p = &(*p)->rb_right;
473 		else if (pregion == region)
474 			return;
475 		else
476 			BUG();
477 	}
478 
479 	rb_link_node(&region->vm_rb, parent, p);
480 	rb_insert_color(&region->vm_rb, &nommu_region_tree);
481 
482 	validate_nommu_regions();
483 }
484 
485 /*
486  * delete a region from the global tree
487  */
488 static void delete_nommu_region(struct vm_region *region)
489 {
490 	BUG_ON(!nommu_region_tree.rb_node);
491 
492 	validate_nommu_regions();
493 	rb_erase(&region->vm_rb, &nommu_region_tree);
494 	validate_nommu_regions();
495 }
496 
497 /*
498  * free a contiguous series of pages
499  */
500 static void free_page_series(unsigned long from, unsigned long to)
501 {
502 	for (; from < to; from += PAGE_SIZE) {
503 		struct page *page = virt_to_page(from);
504 
505 		atomic_long_dec(&mmap_pages_allocated);
506 		put_page(page);
507 	}
508 }
509 
510 /*
511  * release a reference to a region
512  * - the caller must hold the region semaphore for writing, which this releases
513  * - the region may not have been added to the tree yet, in which case vm_top
514  *   will equal vm_start
515  */
516 static void __put_nommu_region(struct vm_region *region)
517 	__releases(nommu_region_sem)
518 {
519 	BUG_ON(!nommu_region_tree.rb_node);
520 
521 	if (--region->vm_usage == 0) {
522 		if (region->vm_top > region->vm_start)
523 			delete_nommu_region(region);
524 		up_write(&nommu_region_sem);
525 
526 		if (region->vm_file)
527 			fput(region->vm_file);
528 
529 		/* IO memory and memory shared directly out of the pagecache
530 		 * from ramfs/tmpfs mustn't be released here */
531 		if (region->vm_flags & VM_MAPPED_COPY)
532 			free_page_series(region->vm_start, region->vm_top);
533 		kmem_cache_free(vm_region_jar, region);
534 	} else {
535 		up_write(&nommu_region_sem);
536 	}
537 }
538 
539 /*
540  * release a reference to a region
541  */
542 static void put_nommu_region(struct vm_region *region)
543 {
544 	down_write(&nommu_region_sem);
545 	__put_nommu_region(region);
546 }
547 
548 /*
549  * add a VMA into a process's mm_struct in the appropriate place in the list
550  * and tree and add to the address space's page tree also if not an anonymous
551  * page
552  * - should be called with mm->mmap_lock held writelocked
553  */
554 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
555 {
556 	struct vm_area_struct *pvma, *prev;
557 	struct address_space *mapping;
558 	struct rb_node **p, *parent, *rb_prev;
559 
560 	BUG_ON(!vma->vm_region);
561 
562 	mm->map_count++;
563 	vma->vm_mm = mm;
564 
565 	/* add the VMA to the mapping */
566 	if (vma->vm_file) {
567 		mapping = vma->vm_file->f_mapping;
568 
569 		i_mmap_lock_write(mapping);
570 		flush_dcache_mmap_lock(mapping);
571 		vma_interval_tree_insert(vma, &mapping->i_mmap);
572 		flush_dcache_mmap_unlock(mapping);
573 		i_mmap_unlock_write(mapping);
574 	}
575 
576 	/* add the VMA to the tree */
577 	parent = rb_prev = NULL;
578 	p = &mm->mm_rb.rb_node;
579 	while (*p) {
580 		parent = *p;
581 		pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
582 
583 		/* sort by: start addr, end addr, VMA struct addr in that order
584 		 * (the latter is necessary as we may get identical VMAs) */
585 		if (vma->vm_start < pvma->vm_start)
586 			p = &(*p)->rb_left;
587 		else if (vma->vm_start > pvma->vm_start) {
588 			rb_prev = parent;
589 			p = &(*p)->rb_right;
590 		} else if (vma->vm_end < pvma->vm_end)
591 			p = &(*p)->rb_left;
592 		else if (vma->vm_end > pvma->vm_end) {
593 			rb_prev = parent;
594 			p = &(*p)->rb_right;
595 		} else if (vma < pvma)
596 			p = &(*p)->rb_left;
597 		else if (vma > pvma) {
598 			rb_prev = parent;
599 			p = &(*p)->rb_right;
600 		} else
601 			BUG();
602 	}
603 
604 	rb_link_node(&vma->vm_rb, parent, p);
605 	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
606 
607 	/* add VMA to the VMA list also */
608 	prev = NULL;
609 	if (rb_prev)
610 		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
611 
612 	__vma_link_list(mm, vma, prev);
613 }
614 
615 /*
616  * delete a VMA from its owning mm_struct and address space
617  */
618 static void delete_vma_from_mm(struct vm_area_struct *vma)
619 {
620 	int i;
621 	struct address_space *mapping;
622 	struct mm_struct *mm = vma->vm_mm;
623 	struct task_struct *curr = current;
624 
625 	mm->map_count--;
626 	for (i = 0; i < VMACACHE_SIZE; i++) {
627 		/* if the vma is cached, invalidate the entire cache */
628 		if (curr->vmacache.vmas[i] == vma) {
629 			vmacache_invalidate(mm);
630 			break;
631 		}
632 	}
633 
634 	/* remove the VMA from the mapping */
635 	if (vma->vm_file) {
636 		mapping = vma->vm_file->f_mapping;
637 
638 		i_mmap_lock_write(mapping);
639 		flush_dcache_mmap_lock(mapping);
640 		vma_interval_tree_remove(vma, &mapping->i_mmap);
641 		flush_dcache_mmap_unlock(mapping);
642 		i_mmap_unlock_write(mapping);
643 	}
644 
645 	/* remove from the MM's tree and list */
646 	rb_erase(&vma->vm_rb, &mm->mm_rb);
647 
648 	__vma_unlink_list(mm, vma);
649 }
650 
651 /*
652  * destroy a VMA record
653  */
654 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
655 {
656 	if (vma->vm_ops && vma->vm_ops->close)
657 		vma->vm_ops->close(vma);
658 	if (vma->vm_file)
659 		fput(vma->vm_file);
660 	put_nommu_region(vma->vm_region);
661 	vm_area_free(vma);
662 }
663 
664 /*
665  * look up the first VMA in which addr resides, NULL if none
666  * - should be called with mm->mmap_lock at least held readlocked
667  */
668 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
669 {
670 	struct vm_area_struct *vma;
671 
672 	/* check the cache first */
673 	vma = vmacache_find(mm, addr);
674 	if (likely(vma))
675 		return vma;
676 
677 	/* trawl the list (there may be multiple mappings in which addr
678 	 * resides) */
679 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
680 		if (vma->vm_start > addr)
681 			return NULL;
682 		if (vma->vm_end > addr) {
683 			vmacache_update(addr, vma);
684 			return vma;
685 		}
686 	}
687 
688 	return NULL;
689 }
690 EXPORT_SYMBOL(find_vma);
691 
692 /*
693  * find a VMA
694  * - we don't extend stack VMAs under NOMMU conditions
695  */
696 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
697 {
698 	return find_vma(mm, addr);
699 }
700 
701 /*
702  * expand a stack to a given address
703  * - not supported under NOMMU conditions
704  */
705 int expand_stack(struct vm_area_struct *vma, unsigned long address)
706 {
707 	return -ENOMEM;
708 }
709 
710 /*
711  * look up the first VMA exactly that exactly matches addr
712  * - should be called with mm->mmap_lock at least held readlocked
713  */
714 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
715 					     unsigned long addr,
716 					     unsigned long len)
717 {
718 	struct vm_area_struct *vma;
719 	unsigned long end = addr + len;
720 
721 	/* check the cache first */
722 	vma = vmacache_find_exact(mm, addr, end);
723 	if (vma)
724 		return vma;
725 
726 	/* trawl the list (there may be multiple mappings in which addr
727 	 * resides) */
728 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
729 		if (vma->vm_start < addr)
730 			continue;
731 		if (vma->vm_start > addr)
732 			return NULL;
733 		if (vma->vm_end == end) {
734 			vmacache_update(addr, vma);
735 			return vma;
736 		}
737 	}
738 
739 	return NULL;
740 }
741 
742 /*
743  * determine whether a mapping should be permitted and, if so, what sort of
744  * mapping we're capable of supporting
745  */
746 static int validate_mmap_request(struct file *file,
747 				 unsigned long addr,
748 				 unsigned long len,
749 				 unsigned long prot,
750 				 unsigned long flags,
751 				 unsigned long pgoff,
752 				 unsigned long *_capabilities)
753 {
754 	unsigned long capabilities, rlen;
755 	int ret;
756 
757 	/* do the simple checks first */
758 	if (flags & MAP_FIXED)
759 		return -EINVAL;
760 
761 	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
762 	    (flags & MAP_TYPE) != MAP_SHARED)
763 		return -EINVAL;
764 
765 	if (!len)
766 		return -EINVAL;
767 
768 	/* Careful about overflows.. */
769 	rlen = PAGE_ALIGN(len);
770 	if (!rlen || rlen > TASK_SIZE)
771 		return -ENOMEM;
772 
773 	/* offset overflow? */
774 	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
775 		return -EOVERFLOW;
776 
777 	if (file) {
778 		/* files must support mmap */
779 		if (!file->f_op->mmap)
780 			return -ENODEV;
781 
782 		/* work out if what we've got could possibly be shared
783 		 * - we support chardevs that provide their own "memory"
784 		 * - we support files/blockdevs that are memory backed
785 		 */
786 		if (file->f_op->mmap_capabilities) {
787 			capabilities = file->f_op->mmap_capabilities(file);
788 		} else {
789 			/* no explicit capabilities set, so assume some
790 			 * defaults */
791 			switch (file_inode(file)->i_mode & S_IFMT) {
792 			case S_IFREG:
793 			case S_IFBLK:
794 				capabilities = NOMMU_MAP_COPY;
795 				break;
796 
797 			case S_IFCHR:
798 				capabilities =
799 					NOMMU_MAP_DIRECT |
800 					NOMMU_MAP_READ |
801 					NOMMU_MAP_WRITE;
802 				break;
803 
804 			default:
805 				return -EINVAL;
806 			}
807 		}
808 
809 		/* eliminate any capabilities that we can't support on this
810 		 * device */
811 		if (!file->f_op->get_unmapped_area)
812 			capabilities &= ~NOMMU_MAP_DIRECT;
813 		if (!(file->f_mode & FMODE_CAN_READ))
814 			capabilities &= ~NOMMU_MAP_COPY;
815 
816 		/* The file shall have been opened with read permission. */
817 		if (!(file->f_mode & FMODE_READ))
818 			return -EACCES;
819 
820 		if (flags & MAP_SHARED) {
821 			/* do checks for writing, appending and locking */
822 			if ((prot & PROT_WRITE) &&
823 			    !(file->f_mode & FMODE_WRITE))
824 				return -EACCES;
825 
826 			if (IS_APPEND(file_inode(file)) &&
827 			    (file->f_mode & FMODE_WRITE))
828 				return -EACCES;
829 
830 			if (!(capabilities & NOMMU_MAP_DIRECT))
831 				return -ENODEV;
832 
833 			/* we mustn't privatise shared mappings */
834 			capabilities &= ~NOMMU_MAP_COPY;
835 		} else {
836 			/* we're going to read the file into private memory we
837 			 * allocate */
838 			if (!(capabilities & NOMMU_MAP_COPY))
839 				return -ENODEV;
840 
841 			/* we don't permit a private writable mapping to be
842 			 * shared with the backing device */
843 			if (prot & PROT_WRITE)
844 				capabilities &= ~NOMMU_MAP_DIRECT;
845 		}
846 
847 		if (capabilities & NOMMU_MAP_DIRECT) {
848 			if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
849 			    ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
850 			    ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
851 			    ) {
852 				capabilities &= ~NOMMU_MAP_DIRECT;
853 				if (flags & MAP_SHARED) {
854 					pr_warn("MAP_SHARED not completely supported on !MMU\n");
855 					return -EINVAL;
856 				}
857 			}
858 		}
859 
860 		/* handle executable mappings and implied executable
861 		 * mappings */
862 		if (path_noexec(&file->f_path)) {
863 			if (prot & PROT_EXEC)
864 				return -EPERM;
865 		} else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
866 			/* handle implication of PROT_EXEC by PROT_READ */
867 			if (current->personality & READ_IMPLIES_EXEC) {
868 				if (capabilities & NOMMU_MAP_EXEC)
869 					prot |= PROT_EXEC;
870 			}
871 		} else if ((prot & PROT_READ) &&
872 			 (prot & PROT_EXEC) &&
873 			 !(capabilities & NOMMU_MAP_EXEC)
874 			 ) {
875 			/* backing file is not executable, try to copy */
876 			capabilities &= ~NOMMU_MAP_DIRECT;
877 		}
878 	} else {
879 		/* anonymous mappings are always memory backed and can be
880 		 * privately mapped
881 		 */
882 		capabilities = NOMMU_MAP_COPY;
883 
884 		/* handle PROT_EXEC implication by PROT_READ */
885 		if ((prot & PROT_READ) &&
886 		    (current->personality & READ_IMPLIES_EXEC))
887 			prot |= PROT_EXEC;
888 	}
889 
890 	/* allow the security API to have its say */
891 	ret = security_mmap_addr(addr);
892 	if (ret < 0)
893 		return ret;
894 
895 	/* looks okay */
896 	*_capabilities = capabilities;
897 	return 0;
898 }
899 
900 /*
901  * we've determined that we can make the mapping, now translate what we
902  * now know into VMA flags
903  */
904 static unsigned long determine_vm_flags(struct file *file,
905 					unsigned long prot,
906 					unsigned long flags,
907 					unsigned long capabilities)
908 {
909 	unsigned long vm_flags;
910 
911 	vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
912 	/* vm_flags |= mm->def_flags; */
913 
914 	if (!(capabilities & NOMMU_MAP_DIRECT)) {
915 		/* attempt to share read-only copies of mapped file chunks */
916 		vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
917 		if (file && !(prot & PROT_WRITE))
918 			vm_flags |= VM_MAYSHARE;
919 	} else {
920 		/* overlay a shareable mapping on the backing device or inode
921 		 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
922 		 * romfs/cramfs */
923 		vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
924 		if (flags & MAP_SHARED)
925 			vm_flags |= VM_SHARED;
926 	}
927 
928 	/* refuse to let anyone share private mappings with this process if
929 	 * it's being traced - otherwise breakpoints set in it may interfere
930 	 * with another untraced process
931 	 */
932 	if ((flags & MAP_PRIVATE) && current->ptrace)
933 		vm_flags &= ~VM_MAYSHARE;
934 
935 	return vm_flags;
936 }
937 
938 /*
939  * set up a shared mapping on a file (the driver or filesystem provides and
940  * pins the storage)
941  */
942 static int do_mmap_shared_file(struct vm_area_struct *vma)
943 {
944 	int ret;
945 
946 	ret = call_mmap(vma->vm_file, vma);
947 	if (ret == 0) {
948 		vma->vm_region->vm_top = vma->vm_region->vm_end;
949 		return 0;
950 	}
951 	if (ret != -ENOSYS)
952 		return ret;
953 
954 	/* getting -ENOSYS indicates that direct mmap isn't possible (as
955 	 * opposed to tried but failed) so we can only give a suitable error as
956 	 * it's not possible to make a private copy if MAP_SHARED was given */
957 	return -ENODEV;
958 }
959 
960 /*
961  * set up a private mapping or an anonymous shared mapping
962  */
963 static int do_mmap_private(struct vm_area_struct *vma,
964 			   struct vm_region *region,
965 			   unsigned long len,
966 			   unsigned long capabilities)
967 {
968 	unsigned long total, point;
969 	void *base;
970 	int ret, order;
971 
972 	/* invoke the file's mapping function so that it can keep track of
973 	 * shared mappings on devices or memory
974 	 * - VM_MAYSHARE will be set if it may attempt to share
975 	 */
976 	if (capabilities & NOMMU_MAP_DIRECT) {
977 		ret = call_mmap(vma->vm_file, vma);
978 		if (ret == 0) {
979 			/* shouldn't return success if we're not sharing */
980 			BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
981 			vma->vm_region->vm_top = vma->vm_region->vm_end;
982 			return 0;
983 		}
984 		if (ret != -ENOSYS)
985 			return ret;
986 
987 		/* getting an ENOSYS error indicates that direct mmap isn't
988 		 * possible (as opposed to tried but failed) so we'll try to
989 		 * make a private copy of the data and map that instead */
990 	}
991 
992 
993 	/* allocate some memory to hold the mapping
994 	 * - note that this may not return a page-aligned address if the object
995 	 *   we're allocating is smaller than a page
996 	 */
997 	order = get_order(len);
998 	total = 1 << order;
999 	point = len >> PAGE_SHIFT;
1000 
1001 	/* we don't want to allocate a power-of-2 sized page set */
1002 	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
1003 		total = point;
1004 
1005 	base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
1006 	if (!base)
1007 		goto enomem;
1008 
1009 	atomic_long_add(total, &mmap_pages_allocated);
1010 
1011 	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
1012 	region->vm_start = (unsigned long) base;
1013 	region->vm_end   = region->vm_start + len;
1014 	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1015 
1016 	vma->vm_start = region->vm_start;
1017 	vma->vm_end   = region->vm_start + len;
1018 
1019 	if (vma->vm_file) {
1020 		/* read the contents of a file into the copy */
1021 		loff_t fpos;
1022 
1023 		fpos = vma->vm_pgoff;
1024 		fpos <<= PAGE_SHIFT;
1025 
1026 		ret = kernel_read(vma->vm_file, base, len, &fpos);
1027 		if (ret < 0)
1028 			goto error_free;
1029 
1030 		/* clear the last little bit */
1031 		if (ret < len)
1032 			memset(base + ret, 0, len - ret);
1033 
1034 	} else {
1035 		vma_set_anonymous(vma);
1036 	}
1037 
1038 	return 0;
1039 
1040 error_free:
1041 	free_page_series(region->vm_start, region->vm_top);
1042 	region->vm_start = vma->vm_start = 0;
1043 	region->vm_end   = vma->vm_end = 0;
1044 	region->vm_top   = 0;
1045 	return ret;
1046 
1047 enomem:
1048 	pr_err("Allocation of length %lu from process %d (%s) failed\n",
1049 	       len, current->pid, current->comm);
1050 	show_free_areas(0, NULL);
1051 	return -ENOMEM;
1052 }
1053 
1054 /*
1055  * handle mapping creation for uClinux
1056  */
1057 unsigned long do_mmap(struct file *file,
1058 			unsigned long addr,
1059 			unsigned long len,
1060 			unsigned long prot,
1061 			unsigned long flags,
1062 			unsigned long pgoff,
1063 			unsigned long *populate,
1064 			struct list_head *uf)
1065 {
1066 	struct vm_area_struct *vma;
1067 	struct vm_region *region;
1068 	struct rb_node *rb;
1069 	vm_flags_t vm_flags;
1070 	unsigned long capabilities, result;
1071 	int ret;
1072 
1073 	*populate = 0;
1074 
1075 	/* decide whether we should attempt the mapping, and if so what sort of
1076 	 * mapping */
1077 	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1078 				    &capabilities);
1079 	if (ret < 0)
1080 		return ret;
1081 
1082 	/* we ignore the address hint */
1083 	addr = 0;
1084 	len = PAGE_ALIGN(len);
1085 
1086 	/* we've determined that we can make the mapping, now translate what we
1087 	 * now know into VMA flags */
1088 	vm_flags = determine_vm_flags(file, prot, flags, capabilities);
1089 
1090 	/* we're going to need to record the mapping */
1091 	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1092 	if (!region)
1093 		goto error_getting_region;
1094 
1095 	vma = vm_area_alloc(current->mm);
1096 	if (!vma)
1097 		goto error_getting_vma;
1098 
1099 	region->vm_usage = 1;
1100 	region->vm_flags = vm_flags;
1101 	region->vm_pgoff = pgoff;
1102 
1103 	vma->vm_flags = vm_flags;
1104 	vma->vm_pgoff = pgoff;
1105 
1106 	if (file) {
1107 		region->vm_file = get_file(file);
1108 		vma->vm_file = get_file(file);
1109 	}
1110 
1111 	down_write(&nommu_region_sem);
1112 
1113 	/* if we want to share, we need to check for regions created by other
1114 	 * mmap() calls that overlap with our proposed mapping
1115 	 * - we can only share with a superset match on most regular files
1116 	 * - shared mappings on character devices and memory backed files are
1117 	 *   permitted to overlap inexactly as far as we are concerned for in
1118 	 *   these cases, sharing is handled in the driver or filesystem rather
1119 	 *   than here
1120 	 */
1121 	if (vm_flags & VM_MAYSHARE) {
1122 		struct vm_region *pregion;
1123 		unsigned long pglen, rpglen, pgend, rpgend, start;
1124 
1125 		pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1126 		pgend = pgoff + pglen;
1127 
1128 		for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1129 			pregion = rb_entry(rb, struct vm_region, vm_rb);
1130 
1131 			if (!(pregion->vm_flags & VM_MAYSHARE))
1132 				continue;
1133 
1134 			/* search for overlapping mappings on the same file */
1135 			if (file_inode(pregion->vm_file) !=
1136 			    file_inode(file))
1137 				continue;
1138 
1139 			if (pregion->vm_pgoff >= pgend)
1140 				continue;
1141 
1142 			rpglen = pregion->vm_end - pregion->vm_start;
1143 			rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1144 			rpgend = pregion->vm_pgoff + rpglen;
1145 			if (pgoff >= rpgend)
1146 				continue;
1147 
1148 			/* handle inexactly overlapping matches between
1149 			 * mappings */
1150 			if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1151 			    !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1152 				/* new mapping is not a subset of the region */
1153 				if (!(capabilities & NOMMU_MAP_DIRECT))
1154 					goto sharing_violation;
1155 				continue;
1156 			}
1157 
1158 			/* we've found a region we can share */
1159 			pregion->vm_usage++;
1160 			vma->vm_region = pregion;
1161 			start = pregion->vm_start;
1162 			start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1163 			vma->vm_start = start;
1164 			vma->vm_end = start + len;
1165 
1166 			if (pregion->vm_flags & VM_MAPPED_COPY)
1167 				vma->vm_flags |= VM_MAPPED_COPY;
1168 			else {
1169 				ret = do_mmap_shared_file(vma);
1170 				if (ret < 0) {
1171 					vma->vm_region = NULL;
1172 					vma->vm_start = 0;
1173 					vma->vm_end = 0;
1174 					pregion->vm_usage--;
1175 					pregion = NULL;
1176 					goto error_just_free;
1177 				}
1178 			}
1179 			fput(region->vm_file);
1180 			kmem_cache_free(vm_region_jar, region);
1181 			region = pregion;
1182 			result = start;
1183 			goto share;
1184 		}
1185 
1186 		/* obtain the address at which to make a shared mapping
1187 		 * - this is the hook for quasi-memory character devices to
1188 		 *   tell us the location of a shared mapping
1189 		 */
1190 		if (capabilities & NOMMU_MAP_DIRECT) {
1191 			addr = file->f_op->get_unmapped_area(file, addr, len,
1192 							     pgoff, flags);
1193 			if (IS_ERR_VALUE(addr)) {
1194 				ret = addr;
1195 				if (ret != -ENOSYS)
1196 					goto error_just_free;
1197 
1198 				/* the driver refused to tell us where to site
1199 				 * the mapping so we'll have to attempt to copy
1200 				 * it */
1201 				ret = -ENODEV;
1202 				if (!(capabilities & NOMMU_MAP_COPY))
1203 					goto error_just_free;
1204 
1205 				capabilities &= ~NOMMU_MAP_DIRECT;
1206 			} else {
1207 				vma->vm_start = region->vm_start = addr;
1208 				vma->vm_end = region->vm_end = addr + len;
1209 			}
1210 		}
1211 	}
1212 
1213 	vma->vm_region = region;
1214 
1215 	/* set up the mapping
1216 	 * - the region is filled in if NOMMU_MAP_DIRECT is still set
1217 	 */
1218 	if (file && vma->vm_flags & VM_SHARED)
1219 		ret = do_mmap_shared_file(vma);
1220 	else
1221 		ret = do_mmap_private(vma, region, len, capabilities);
1222 	if (ret < 0)
1223 		goto error_just_free;
1224 	add_nommu_region(region);
1225 
1226 	/* clear anonymous mappings that don't ask for uninitialized data */
1227 	if (!vma->vm_file &&
1228 	    (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) ||
1229 	     !(flags & MAP_UNINITIALIZED)))
1230 		memset((void *)region->vm_start, 0,
1231 		       region->vm_end - region->vm_start);
1232 
1233 	/* okay... we have a mapping; now we have to register it */
1234 	result = vma->vm_start;
1235 
1236 	current->mm->total_vm += len >> PAGE_SHIFT;
1237 
1238 share:
1239 	add_vma_to_mm(current->mm, vma);
1240 
1241 	/* we flush the region from the icache only when the first executable
1242 	 * mapping of it is made  */
1243 	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1244 		flush_icache_user_range(region->vm_start, region->vm_end);
1245 		region->vm_icache_flushed = true;
1246 	}
1247 
1248 	up_write(&nommu_region_sem);
1249 
1250 	return result;
1251 
1252 error_just_free:
1253 	up_write(&nommu_region_sem);
1254 error:
1255 	if (region->vm_file)
1256 		fput(region->vm_file);
1257 	kmem_cache_free(vm_region_jar, region);
1258 	if (vma->vm_file)
1259 		fput(vma->vm_file);
1260 	vm_area_free(vma);
1261 	return ret;
1262 
1263 sharing_violation:
1264 	up_write(&nommu_region_sem);
1265 	pr_warn("Attempt to share mismatched mappings\n");
1266 	ret = -EINVAL;
1267 	goto error;
1268 
1269 error_getting_vma:
1270 	kmem_cache_free(vm_region_jar, region);
1271 	pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
1272 			len, current->pid);
1273 	show_free_areas(0, NULL);
1274 	return -ENOMEM;
1275 
1276 error_getting_region:
1277 	pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
1278 			len, current->pid);
1279 	show_free_areas(0, NULL);
1280 	return -ENOMEM;
1281 }
1282 
1283 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
1284 			      unsigned long prot, unsigned long flags,
1285 			      unsigned long fd, unsigned long pgoff)
1286 {
1287 	struct file *file = NULL;
1288 	unsigned long retval = -EBADF;
1289 
1290 	audit_mmap_fd(fd, flags);
1291 	if (!(flags & MAP_ANONYMOUS)) {
1292 		file = fget(fd);
1293 		if (!file)
1294 			goto out;
1295 	}
1296 
1297 	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1298 
1299 	if (file)
1300 		fput(file);
1301 out:
1302 	return retval;
1303 }
1304 
1305 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1306 		unsigned long, prot, unsigned long, flags,
1307 		unsigned long, fd, unsigned long, pgoff)
1308 {
1309 	return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
1310 }
1311 
1312 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1313 struct mmap_arg_struct {
1314 	unsigned long addr;
1315 	unsigned long len;
1316 	unsigned long prot;
1317 	unsigned long flags;
1318 	unsigned long fd;
1319 	unsigned long offset;
1320 };
1321 
1322 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1323 {
1324 	struct mmap_arg_struct a;
1325 
1326 	if (copy_from_user(&a, arg, sizeof(a)))
1327 		return -EFAULT;
1328 	if (offset_in_page(a.offset))
1329 		return -EINVAL;
1330 
1331 	return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1332 			       a.offset >> PAGE_SHIFT);
1333 }
1334 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1335 
1336 /*
1337  * split a vma into two pieces at address 'addr', a new vma is allocated either
1338  * for the first part or the tail.
1339  */
1340 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1341 	      unsigned long addr, int new_below)
1342 {
1343 	struct vm_area_struct *new;
1344 	struct vm_region *region;
1345 	unsigned long npages;
1346 
1347 	/* we're only permitted to split anonymous regions (these should have
1348 	 * only a single usage on the region) */
1349 	if (vma->vm_file)
1350 		return -ENOMEM;
1351 
1352 	if (mm->map_count >= sysctl_max_map_count)
1353 		return -ENOMEM;
1354 
1355 	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1356 	if (!region)
1357 		return -ENOMEM;
1358 
1359 	new = vm_area_dup(vma);
1360 	if (!new) {
1361 		kmem_cache_free(vm_region_jar, region);
1362 		return -ENOMEM;
1363 	}
1364 
1365 	/* most fields are the same, copy all, and then fixup */
1366 	*region = *vma->vm_region;
1367 	new->vm_region = region;
1368 
1369 	npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1370 
1371 	if (new_below) {
1372 		region->vm_top = region->vm_end = new->vm_end = addr;
1373 	} else {
1374 		region->vm_start = new->vm_start = addr;
1375 		region->vm_pgoff = new->vm_pgoff += npages;
1376 	}
1377 
1378 	if (new->vm_ops && new->vm_ops->open)
1379 		new->vm_ops->open(new);
1380 
1381 	delete_vma_from_mm(vma);
1382 	down_write(&nommu_region_sem);
1383 	delete_nommu_region(vma->vm_region);
1384 	if (new_below) {
1385 		vma->vm_region->vm_start = vma->vm_start = addr;
1386 		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1387 	} else {
1388 		vma->vm_region->vm_end = vma->vm_end = addr;
1389 		vma->vm_region->vm_top = addr;
1390 	}
1391 	add_nommu_region(vma->vm_region);
1392 	add_nommu_region(new->vm_region);
1393 	up_write(&nommu_region_sem);
1394 	add_vma_to_mm(mm, vma);
1395 	add_vma_to_mm(mm, new);
1396 	return 0;
1397 }
1398 
1399 /*
1400  * shrink a VMA by removing the specified chunk from either the beginning or
1401  * the end
1402  */
1403 static int shrink_vma(struct mm_struct *mm,
1404 		      struct vm_area_struct *vma,
1405 		      unsigned long from, unsigned long to)
1406 {
1407 	struct vm_region *region;
1408 
1409 	/* adjust the VMA's pointers, which may reposition it in the MM's tree
1410 	 * and list */
1411 	delete_vma_from_mm(vma);
1412 	if (from > vma->vm_start)
1413 		vma->vm_end = from;
1414 	else
1415 		vma->vm_start = to;
1416 	add_vma_to_mm(mm, vma);
1417 
1418 	/* cut the backing region down to size */
1419 	region = vma->vm_region;
1420 	BUG_ON(region->vm_usage != 1);
1421 
1422 	down_write(&nommu_region_sem);
1423 	delete_nommu_region(region);
1424 	if (from > region->vm_start) {
1425 		to = region->vm_top;
1426 		region->vm_top = region->vm_end = from;
1427 	} else {
1428 		region->vm_start = to;
1429 	}
1430 	add_nommu_region(region);
1431 	up_write(&nommu_region_sem);
1432 
1433 	free_page_series(from, to);
1434 	return 0;
1435 }
1436 
1437 /*
1438  * release a mapping
1439  * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1440  *   VMA, though it need not cover the whole VMA
1441  */
1442 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
1443 {
1444 	struct vm_area_struct *vma;
1445 	unsigned long end;
1446 	int ret;
1447 
1448 	len = PAGE_ALIGN(len);
1449 	if (len == 0)
1450 		return -EINVAL;
1451 
1452 	end = start + len;
1453 
1454 	/* find the first potentially overlapping VMA */
1455 	vma = find_vma(mm, start);
1456 	if (!vma) {
1457 		static int limit;
1458 		if (limit < 5) {
1459 			pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
1460 					current->pid, current->comm,
1461 					start, start + len - 1);
1462 			limit++;
1463 		}
1464 		return -EINVAL;
1465 	}
1466 
1467 	/* we're allowed to split an anonymous VMA but not a file-backed one */
1468 	if (vma->vm_file) {
1469 		do {
1470 			if (start > vma->vm_start)
1471 				return -EINVAL;
1472 			if (end == vma->vm_end)
1473 				goto erase_whole_vma;
1474 			vma = vma->vm_next;
1475 		} while (vma);
1476 		return -EINVAL;
1477 	} else {
1478 		/* the chunk must be a subset of the VMA found */
1479 		if (start == vma->vm_start && end == vma->vm_end)
1480 			goto erase_whole_vma;
1481 		if (start < vma->vm_start || end > vma->vm_end)
1482 			return -EINVAL;
1483 		if (offset_in_page(start))
1484 			return -EINVAL;
1485 		if (end != vma->vm_end && offset_in_page(end))
1486 			return -EINVAL;
1487 		if (start != vma->vm_start && end != vma->vm_end) {
1488 			ret = split_vma(mm, vma, start, 1);
1489 			if (ret < 0)
1490 				return ret;
1491 		}
1492 		return shrink_vma(mm, vma, start, end);
1493 	}
1494 
1495 erase_whole_vma:
1496 	delete_vma_from_mm(vma);
1497 	delete_vma(mm, vma);
1498 	return 0;
1499 }
1500 
1501 int vm_munmap(unsigned long addr, size_t len)
1502 {
1503 	struct mm_struct *mm = current->mm;
1504 	int ret;
1505 
1506 	mmap_write_lock(mm);
1507 	ret = do_munmap(mm, addr, len, NULL);
1508 	mmap_write_unlock(mm);
1509 	return ret;
1510 }
1511 EXPORT_SYMBOL(vm_munmap);
1512 
1513 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1514 {
1515 	return vm_munmap(addr, len);
1516 }
1517 
1518 /*
1519  * release all the mappings made in a process's VM space
1520  */
1521 void exit_mmap(struct mm_struct *mm)
1522 {
1523 	struct vm_area_struct *vma;
1524 
1525 	if (!mm)
1526 		return;
1527 
1528 	mm->total_vm = 0;
1529 
1530 	while ((vma = mm->mmap)) {
1531 		mm->mmap = vma->vm_next;
1532 		delete_vma_from_mm(vma);
1533 		delete_vma(mm, vma);
1534 		cond_resched();
1535 	}
1536 }
1537 
1538 int vm_brk(unsigned long addr, unsigned long len)
1539 {
1540 	return -ENOMEM;
1541 }
1542 
1543 /*
1544  * expand (or shrink) an existing mapping, potentially moving it at the same
1545  * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1546  *
1547  * under NOMMU conditions, we only permit changing a mapping's size, and only
1548  * as long as it stays within the region allocated by do_mmap_private() and the
1549  * block is not shareable
1550  *
1551  * MREMAP_FIXED is not supported under NOMMU conditions
1552  */
1553 static unsigned long do_mremap(unsigned long addr,
1554 			unsigned long old_len, unsigned long new_len,
1555 			unsigned long flags, unsigned long new_addr)
1556 {
1557 	struct vm_area_struct *vma;
1558 
1559 	/* insanity checks first */
1560 	old_len = PAGE_ALIGN(old_len);
1561 	new_len = PAGE_ALIGN(new_len);
1562 	if (old_len == 0 || new_len == 0)
1563 		return (unsigned long) -EINVAL;
1564 
1565 	if (offset_in_page(addr))
1566 		return -EINVAL;
1567 
1568 	if (flags & MREMAP_FIXED && new_addr != addr)
1569 		return (unsigned long) -EINVAL;
1570 
1571 	vma = find_vma_exact(current->mm, addr, old_len);
1572 	if (!vma)
1573 		return (unsigned long) -EINVAL;
1574 
1575 	if (vma->vm_end != vma->vm_start + old_len)
1576 		return (unsigned long) -EFAULT;
1577 
1578 	if (vma->vm_flags & VM_MAYSHARE)
1579 		return (unsigned long) -EPERM;
1580 
1581 	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1582 		return (unsigned long) -ENOMEM;
1583 
1584 	/* all checks complete - do it */
1585 	vma->vm_end = vma->vm_start + new_len;
1586 	return vma->vm_start;
1587 }
1588 
1589 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1590 		unsigned long, new_len, unsigned long, flags,
1591 		unsigned long, new_addr)
1592 {
1593 	unsigned long ret;
1594 
1595 	mmap_write_lock(current->mm);
1596 	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1597 	mmap_write_unlock(current->mm);
1598 	return ret;
1599 }
1600 
1601 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
1602 			 unsigned int foll_flags)
1603 {
1604 	return NULL;
1605 }
1606 
1607 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1608 		unsigned long pfn, unsigned long size, pgprot_t prot)
1609 {
1610 	if (addr != (pfn << PAGE_SHIFT))
1611 		return -EINVAL;
1612 
1613 	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1614 	return 0;
1615 }
1616 EXPORT_SYMBOL(remap_pfn_range);
1617 
1618 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1619 {
1620 	unsigned long pfn = start >> PAGE_SHIFT;
1621 	unsigned long vm_len = vma->vm_end - vma->vm_start;
1622 
1623 	pfn += vma->vm_pgoff;
1624 	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1625 }
1626 EXPORT_SYMBOL(vm_iomap_memory);
1627 
1628 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1629 			unsigned long pgoff)
1630 {
1631 	unsigned int size = vma->vm_end - vma->vm_start;
1632 
1633 	if (!(vma->vm_flags & VM_USERMAP))
1634 		return -EINVAL;
1635 
1636 	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1637 	vma->vm_end = vma->vm_start + size;
1638 
1639 	return 0;
1640 }
1641 EXPORT_SYMBOL(remap_vmalloc_range);
1642 
1643 vm_fault_t filemap_fault(struct vm_fault *vmf)
1644 {
1645 	BUG();
1646 	return 0;
1647 }
1648 EXPORT_SYMBOL(filemap_fault);
1649 
1650 vm_fault_t filemap_map_pages(struct vm_fault *vmf,
1651 		pgoff_t start_pgoff, pgoff_t end_pgoff)
1652 {
1653 	BUG();
1654 	return 0;
1655 }
1656 EXPORT_SYMBOL(filemap_map_pages);
1657 
1658 int __access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf,
1659 		       int len, unsigned int gup_flags)
1660 {
1661 	struct vm_area_struct *vma;
1662 	int write = gup_flags & FOLL_WRITE;
1663 
1664 	if (mmap_read_lock_killable(mm))
1665 		return 0;
1666 
1667 	/* the access must start within one of the target process's mappings */
1668 	vma = find_vma(mm, addr);
1669 	if (vma) {
1670 		/* don't overrun this mapping */
1671 		if (addr + len >= vma->vm_end)
1672 			len = vma->vm_end - addr;
1673 
1674 		/* only read or write mappings where it is permitted */
1675 		if (write && vma->vm_flags & VM_MAYWRITE)
1676 			copy_to_user_page(vma, NULL, addr,
1677 					 (void *) addr, buf, len);
1678 		else if (!write && vma->vm_flags & VM_MAYREAD)
1679 			copy_from_user_page(vma, NULL, addr,
1680 					    buf, (void *) addr, len);
1681 		else
1682 			len = 0;
1683 	} else {
1684 		len = 0;
1685 	}
1686 
1687 	mmap_read_unlock(mm);
1688 
1689 	return len;
1690 }
1691 
1692 /**
1693  * access_remote_vm - access another process' address space
1694  * @mm:		the mm_struct of the target address space
1695  * @addr:	start address to access
1696  * @buf:	source or destination buffer
1697  * @len:	number of bytes to transfer
1698  * @gup_flags:	flags modifying lookup behaviour
1699  *
1700  * The caller must hold a reference on @mm.
1701  */
1702 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1703 		void *buf, int len, unsigned int gup_flags)
1704 {
1705 	return __access_remote_vm(mm, addr, buf, len, gup_flags);
1706 }
1707 
1708 /*
1709  * Access another process' address space.
1710  * - source/target buffer must be kernel space
1711  */
1712 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1713 		unsigned int gup_flags)
1714 {
1715 	struct mm_struct *mm;
1716 
1717 	if (addr + len < addr)
1718 		return 0;
1719 
1720 	mm = get_task_mm(tsk);
1721 	if (!mm)
1722 		return 0;
1723 
1724 	len = __access_remote_vm(mm, addr, buf, len, gup_flags);
1725 
1726 	mmput(mm);
1727 	return len;
1728 }
1729 EXPORT_SYMBOL_GPL(access_process_vm);
1730 
1731 /**
1732  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1733  * @inode: The inode to check
1734  * @size: The current filesize of the inode
1735  * @newsize: The proposed filesize of the inode
1736  *
1737  * Check the shared mappings on an inode on behalf of a shrinking truncate to
1738  * make sure that any outstanding VMAs aren't broken and then shrink the
1739  * vm_regions that extend beyond so that do_mmap() doesn't
1740  * automatically grant mappings that are too large.
1741  */
1742 int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1743 				size_t newsize)
1744 {
1745 	struct vm_area_struct *vma;
1746 	struct vm_region *region;
1747 	pgoff_t low, high;
1748 	size_t r_size, r_top;
1749 
1750 	low = newsize >> PAGE_SHIFT;
1751 	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1752 
1753 	down_write(&nommu_region_sem);
1754 	i_mmap_lock_read(inode->i_mapping);
1755 
1756 	/* search for VMAs that fall within the dead zone */
1757 	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
1758 		/* found one - only interested if it's shared out of the page
1759 		 * cache */
1760 		if (vma->vm_flags & VM_SHARED) {
1761 			i_mmap_unlock_read(inode->i_mapping);
1762 			up_write(&nommu_region_sem);
1763 			return -ETXTBSY; /* not quite true, but near enough */
1764 		}
1765 	}
1766 
1767 	/* reduce any regions that overlap the dead zone - if in existence,
1768 	 * these will be pointed to by VMAs that don't overlap the dead zone
1769 	 *
1770 	 * we don't check for any regions that start beyond the EOF as there
1771 	 * shouldn't be any
1772 	 */
1773 	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
1774 		if (!(vma->vm_flags & VM_SHARED))
1775 			continue;
1776 
1777 		region = vma->vm_region;
1778 		r_size = region->vm_top - region->vm_start;
1779 		r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1780 
1781 		if (r_top > newsize) {
1782 			region->vm_top -= r_top - newsize;
1783 			if (region->vm_end > region->vm_top)
1784 				region->vm_end = region->vm_top;
1785 		}
1786 	}
1787 
1788 	i_mmap_unlock_read(inode->i_mapping);
1789 	up_write(&nommu_region_sem);
1790 	return 0;
1791 }
1792 
1793 /*
1794  * Initialise sysctl_user_reserve_kbytes.
1795  *
1796  * This is intended to prevent a user from starting a single memory hogging
1797  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1798  * mode.
1799  *
1800  * The default value is min(3% of free memory, 128MB)
1801  * 128MB is enough to recover with sshd/login, bash, and top/kill.
1802  */
1803 static int __meminit init_user_reserve(void)
1804 {
1805 	unsigned long free_kbytes;
1806 
1807 	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1808 
1809 	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
1810 	return 0;
1811 }
1812 subsys_initcall(init_user_reserve);
1813 
1814 /*
1815  * Initialise sysctl_admin_reserve_kbytes.
1816  *
1817  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1818  * to log in and kill a memory hogging process.
1819  *
1820  * Systems with more than 256MB will reserve 8MB, enough to recover
1821  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1822  * only reserve 3% of free pages by default.
1823  */
1824 static int __meminit init_admin_reserve(void)
1825 {
1826 	unsigned long free_kbytes;
1827 
1828 	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1829 
1830 	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
1831 	return 0;
1832 }
1833 subsys_initcall(init_admin_reserve);
1834