xref: /openbmc/linux/mm/nommu.c (revision 78bb17f7)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/nommu.c
4  *
5  *  Replacement code for mm functions to support CPU's that don't
6  *  have any form of memory management unit (thus no virtual memory).
7  *
8  *  See Documentation/nommu-mmap.txt
9  *
10  *  Copyright (c) 2004-2008 David Howells <dhowells@redhat.com>
11  *  Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com>
12  *  Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org>
13  *  Copyright (c) 2002      Greg Ungerer <gerg@snapgear.com>
14  *  Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org>
15  */
16 
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 
19 #include <linux/export.h>
20 #include <linux/mm.h>
21 #include <linux/sched/mm.h>
22 #include <linux/vmacache.h>
23 #include <linux/mman.h>
24 #include <linux/swap.h>
25 #include <linux/file.h>
26 #include <linux/highmem.h>
27 #include <linux/pagemap.h>
28 #include <linux/slab.h>
29 #include <linux/vmalloc.h>
30 #include <linux/blkdev.h>
31 #include <linux/backing-dev.h>
32 #include <linux/compiler.h>
33 #include <linux/mount.h>
34 #include <linux/personality.h>
35 #include <linux/security.h>
36 #include <linux/syscalls.h>
37 #include <linux/audit.h>
38 #include <linux/printk.h>
39 
40 #include <linux/uaccess.h>
41 #include <asm/tlb.h>
42 #include <asm/tlbflush.h>
43 #include <asm/mmu_context.h>
44 #include "internal.h"
45 
46 void *high_memory;
47 EXPORT_SYMBOL(high_memory);
48 struct page *mem_map;
49 unsigned long max_mapnr;
50 EXPORT_SYMBOL(max_mapnr);
51 unsigned long highest_memmap_pfn;
52 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS;
53 int heap_stack_gap = 0;
54 
55 atomic_long_t mmap_pages_allocated;
56 
57 EXPORT_SYMBOL(mem_map);
58 
59 /* list of mapped, potentially shareable regions */
60 static struct kmem_cache *vm_region_jar;
61 struct rb_root nommu_region_tree = RB_ROOT;
62 DECLARE_RWSEM(nommu_region_sem);
63 
64 const struct vm_operations_struct generic_file_vm_ops = {
65 };
66 
67 /*
68  * Return the total memory allocated for this pointer, not
69  * just what the caller asked for.
70  *
71  * Doesn't have to be accurate, i.e. may have races.
72  */
73 unsigned int kobjsize(const void *objp)
74 {
75 	struct page *page;
76 
77 	/*
78 	 * If the object we have should not have ksize performed on it,
79 	 * return size of 0
80 	 */
81 	if (!objp || !virt_addr_valid(objp))
82 		return 0;
83 
84 	page = virt_to_head_page(objp);
85 
86 	/*
87 	 * If the allocator sets PageSlab, we know the pointer came from
88 	 * kmalloc().
89 	 */
90 	if (PageSlab(page))
91 		return ksize(objp);
92 
93 	/*
94 	 * If it's not a compound page, see if we have a matching VMA
95 	 * region. This test is intentionally done in reverse order,
96 	 * so if there's no VMA, we still fall through and hand back
97 	 * PAGE_SIZE for 0-order pages.
98 	 */
99 	if (!PageCompound(page)) {
100 		struct vm_area_struct *vma;
101 
102 		vma = find_vma(current->mm, (unsigned long)objp);
103 		if (vma)
104 			return vma->vm_end - vma->vm_start;
105 	}
106 
107 	/*
108 	 * The ksize() function is only guaranteed to work for pointers
109 	 * returned by kmalloc(). So handle arbitrary pointers here.
110 	 */
111 	return page_size(page);
112 }
113 
114 /**
115  * follow_pfn - look up PFN at a user virtual address
116  * @vma: memory mapping
117  * @address: user virtual address
118  * @pfn: location to store found PFN
119  *
120  * Only IO mappings and raw PFN mappings are allowed.
121  *
122  * Returns zero and the pfn at @pfn on success, -ve otherwise.
123  */
124 int follow_pfn(struct vm_area_struct *vma, unsigned long address,
125 	unsigned long *pfn)
126 {
127 	if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
128 		return -EINVAL;
129 
130 	*pfn = address >> PAGE_SHIFT;
131 	return 0;
132 }
133 EXPORT_SYMBOL(follow_pfn);
134 
135 LIST_HEAD(vmap_area_list);
136 
137 void vfree(const void *addr)
138 {
139 	kfree(addr);
140 }
141 EXPORT_SYMBOL(vfree);
142 
143 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
144 {
145 	/*
146 	 *  You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc()
147 	 * returns only a logical address.
148 	 */
149 	return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM);
150 }
151 EXPORT_SYMBOL(__vmalloc);
152 
153 void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags)
154 {
155 	return __vmalloc(size, flags, PAGE_KERNEL);
156 }
157 
158 static void *__vmalloc_user_flags(unsigned long size, gfp_t flags)
159 {
160 	void *ret;
161 
162 	ret = __vmalloc(size, flags, PAGE_KERNEL);
163 	if (ret) {
164 		struct vm_area_struct *vma;
165 
166 		down_write(&current->mm->mmap_sem);
167 		vma = find_vma(current->mm, (unsigned long)ret);
168 		if (vma)
169 			vma->vm_flags |= VM_USERMAP;
170 		up_write(&current->mm->mmap_sem);
171 	}
172 
173 	return ret;
174 }
175 
176 void *vmalloc_user(unsigned long size)
177 {
178 	return __vmalloc_user_flags(size, GFP_KERNEL | __GFP_ZERO);
179 }
180 EXPORT_SYMBOL(vmalloc_user);
181 
182 void *vmalloc_user_node_flags(unsigned long size, int node, gfp_t flags)
183 {
184 	return __vmalloc_user_flags(size, flags | __GFP_ZERO);
185 }
186 EXPORT_SYMBOL(vmalloc_user_node_flags);
187 
188 struct page *vmalloc_to_page(const void *addr)
189 {
190 	return virt_to_page(addr);
191 }
192 EXPORT_SYMBOL(vmalloc_to_page);
193 
194 unsigned long vmalloc_to_pfn(const void *addr)
195 {
196 	return page_to_pfn(virt_to_page(addr));
197 }
198 EXPORT_SYMBOL(vmalloc_to_pfn);
199 
200 long vread(char *buf, char *addr, unsigned long count)
201 {
202 	/* Don't allow overflow */
203 	if ((unsigned long) buf + count < count)
204 		count = -(unsigned long) buf;
205 
206 	memcpy(buf, addr, count);
207 	return count;
208 }
209 
210 long vwrite(char *buf, char *addr, unsigned long count)
211 {
212 	/* Don't allow overflow */
213 	if ((unsigned long) addr + count < count)
214 		count = -(unsigned long) addr;
215 
216 	memcpy(addr, buf, count);
217 	return count;
218 }
219 
220 /*
221  *	vmalloc  -  allocate virtually contiguous memory
222  *
223  *	@size:		allocation size
224  *
225  *	Allocate enough pages to cover @size from the page level
226  *	allocator and map them into contiguous kernel virtual space.
227  *
228  *	For tight control over page level allocator and protection flags
229  *	use __vmalloc() instead.
230  */
231 void *vmalloc(unsigned long size)
232 {
233        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
234 }
235 EXPORT_SYMBOL(vmalloc);
236 
237 /*
238  *	vzalloc - allocate virtually contiguous memory with zero fill
239  *
240  *	@size:		allocation size
241  *
242  *	Allocate enough pages to cover @size from the page level
243  *	allocator and map them into contiguous kernel virtual space.
244  *	The memory allocated is set to zero.
245  *
246  *	For tight control over page level allocator and protection flags
247  *	use __vmalloc() instead.
248  */
249 void *vzalloc(unsigned long size)
250 {
251 	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
252 			PAGE_KERNEL);
253 }
254 EXPORT_SYMBOL(vzalloc);
255 
256 /**
257  * vmalloc_node - allocate memory on a specific node
258  * @size:	allocation size
259  * @node:	numa node
260  *
261  * Allocate enough pages to cover @size from the page level
262  * allocator and map them into contiguous kernel virtual space.
263  *
264  * For tight control over page level allocator and protection flags
265  * use __vmalloc() instead.
266  */
267 void *vmalloc_node(unsigned long size, int node)
268 {
269 	return vmalloc(size);
270 }
271 EXPORT_SYMBOL(vmalloc_node);
272 
273 /**
274  * vzalloc_node - allocate memory on a specific node with zero fill
275  * @size:	allocation size
276  * @node:	numa node
277  *
278  * Allocate enough pages to cover @size from the page level
279  * allocator and map them into contiguous kernel virtual space.
280  * The memory allocated is set to zero.
281  *
282  * For tight control over page level allocator and protection flags
283  * use __vmalloc() instead.
284  */
285 void *vzalloc_node(unsigned long size, int node)
286 {
287 	return vzalloc(size);
288 }
289 EXPORT_SYMBOL(vzalloc_node);
290 
291 /**
292  *	vmalloc_exec  -  allocate virtually contiguous, executable memory
293  *	@size:		allocation size
294  *
295  *	Kernel-internal function to allocate enough pages to cover @size
296  *	the page level allocator and map them into contiguous and
297  *	executable kernel virtual space.
298  *
299  *	For tight control over page level allocator and protection flags
300  *	use __vmalloc() instead.
301  */
302 
303 void *vmalloc_exec(unsigned long size)
304 {
305 	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
306 }
307 
308 /**
309  * vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
310  *	@size:		allocation size
311  *
312  *	Allocate enough 32bit PA addressable pages to cover @size from the
313  *	page level allocator and map them into contiguous kernel virtual space.
314  */
315 void *vmalloc_32(unsigned long size)
316 {
317 	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
318 }
319 EXPORT_SYMBOL(vmalloc_32);
320 
321 /**
322  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
323  *	@size:		allocation size
324  *
325  * The resulting memory area is 32bit addressable and zeroed so it can be
326  * mapped to userspace without leaking data.
327  *
328  * VM_USERMAP is set on the corresponding VMA so that subsequent calls to
329  * remap_vmalloc_range() are permissible.
330  */
331 void *vmalloc_32_user(unsigned long size)
332 {
333 	/*
334 	 * We'll have to sort out the ZONE_DMA bits for 64-bit,
335 	 * but for now this can simply use vmalloc_user() directly.
336 	 */
337 	return vmalloc_user(size);
338 }
339 EXPORT_SYMBOL(vmalloc_32_user);
340 
341 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot)
342 {
343 	BUG();
344 	return NULL;
345 }
346 EXPORT_SYMBOL(vmap);
347 
348 void vunmap(const void *addr)
349 {
350 	BUG();
351 }
352 EXPORT_SYMBOL(vunmap);
353 
354 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
355 {
356 	BUG();
357 	return NULL;
358 }
359 EXPORT_SYMBOL(vm_map_ram);
360 
361 void vm_unmap_ram(const void *mem, unsigned int count)
362 {
363 	BUG();
364 }
365 EXPORT_SYMBOL(vm_unmap_ram);
366 
367 void vm_unmap_aliases(void)
368 {
369 }
370 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
371 
372 /*
373  * Implement a stub for vmalloc_sync_[un]mapping() if the architecture
374  * chose not to have one.
375  */
376 void __weak vmalloc_sync_mappings(void)
377 {
378 }
379 
380 void __weak vmalloc_sync_unmappings(void)
381 {
382 }
383 
384 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
385 {
386 	BUG();
387 	return NULL;
388 }
389 EXPORT_SYMBOL_GPL(alloc_vm_area);
390 
391 void free_vm_area(struct vm_struct *area)
392 {
393 	BUG();
394 }
395 EXPORT_SYMBOL_GPL(free_vm_area);
396 
397 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
398 		   struct page *page)
399 {
400 	return -EINVAL;
401 }
402 EXPORT_SYMBOL(vm_insert_page);
403 
404 int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
405 			unsigned long num)
406 {
407 	return -EINVAL;
408 }
409 EXPORT_SYMBOL(vm_map_pages);
410 
411 int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
412 				unsigned long num)
413 {
414 	return -EINVAL;
415 }
416 EXPORT_SYMBOL(vm_map_pages_zero);
417 
418 /*
419  *  sys_brk() for the most part doesn't need the global kernel
420  *  lock, except when an application is doing something nasty
421  *  like trying to un-brk an area that has already been mapped
422  *  to a regular file.  in this case, the unmapping will need
423  *  to invoke file system routines that need the global lock.
424  */
425 SYSCALL_DEFINE1(brk, unsigned long, brk)
426 {
427 	struct mm_struct *mm = current->mm;
428 
429 	if (brk < mm->start_brk || brk > mm->context.end_brk)
430 		return mm->brk;
431 
432 	if (mm->brk == brk)
433 		return mm->brk;
434 
435 	/*
436 	 * Always allow shrinking brk
437 	 */
438 	if (brk <= mm->brk) {
439 		mm->brk = brk;
440 		return brk;
441 	}
442 
443 	/*
444 	 * Ok, looks good - let it rip.
445 	 */
446 	flush_icache_range(mm->brk, brk);
447 	return mm->brk = brk;
448 }
449 
450 /*
451  * initialise the percpu counter for VM and region record slabs
452  */
453 void __init mmap_init(void)
454 {
455 	int ret;
456 
457 	ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
458 	VM_BUG_ON(ret);
459 	vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT);
460 }
461 
462 /*
463  * validate the region tree
464  * - the caller must hold the region lock
465  */
466 #ifdef CONFIG_DEBUG_NOMMU_REGIONS
467 static noinline void validate_nommu_regions(void)
468 {
469 	struct vm_region *region, *last;
470 	struct rb_node *p, *lastp;
471 
472 	lastp = rb_first(&nommu_region_tree);
473 	if (!lastp)
474 		return;
475 
476 	last = rb_entry(lastp, struct vm_region, vm_rb);
477 	BUG_ON(last->vm_end <= last->vm_start);
478 	BUG_ON(last->vm_top < last->vm_end);
479 
480 	while ((p = rb_next(lastp))) {
481 		region = rb_entry(p, struct vm_region, vm_rb);
482 		last = rb_entry(lastp, struct vm_region, vm_rb);
483 
484 		BUG_ON(region->vm_end <= region->vm_start);
485 		BUG_ON(region->vm_top < region->vm_end);
486 		BUG_ON(region->vm_start < last->vm_top);
487 
488 		lastp = p;
489 	}
490 }
491 #else
492 static void validate_nommu_regions(void)
493 {
494 }
495 #endif
496 
497 /*
498  * add a region into the global tree
499  */
500 static void add_nommu_region(struct vm_region *region)
501 {
502 	struct vm_region *pregion;
503 	struct rb_node **p, *parent;
504 
505 	validate_nommu_regions();
506 
507 	parent = NULL;
508 	p = &nommu_region_tree.rb_node;
509 	while (*p) {
510 		parent = *p;
511 		pregion = rb_entry(parent, struct vm_region, vm_rb);
512 		if (region->vm_start < pregion->vm_start)
513 			p = &(*p)->rb_left;
514 		else if (region->vm_start > pregion->vm_start)
515 			p = &(*p)->rb_right;
516 		else if (pregion == region)
517 			return;
518 		else
519 			BUG();
520 	}
521 
522 	rb_link_node(&region->vm_rb, parent, p);
523 	rb_insert_color(&region->vm_rb, &nommu_region_tree);
524 
525 	validate_nommu_regions();
526 }
527 
528 /*
529  * delete a region from the global tree
530  */
531 static void delete_nommu_region(struct vm_region *region)
532 {
533 	BUG_ON(!nommu_region_tree.rb_node);
534 
535 	validate_nommu_regions();
536 	rb_erase(&region->vm_rb, &nommu_region_tree);
537 	validate_nommu_regions();
538 }
539 
540 /*
541  * free a contiguous series of pages
542  */
543 static void free_page_series(unsigned long from, unsigned long to)
544 {
545 	for (; from < to; from += PAGE_SIZE) {
546 		struct page *page = virt_to_page(from);
547 
548 		atomic_long_dec(&mmap_pages_allocated);
549 		put_page(page);
550 	}
551 }
552 
553 /*
554  * release a reference to a region
555  * - the caller must hold the region semaphore for writing, which this releases
556  * - the region may not have been added to the tree yet, in which case vm_top
557  *   will equal vm_start
558  */
559 static void __put_nommu_region(struct vm_region *region)
560 	__releases(nommu_region_sem)
561 {
562 	BUG_ON(!nommu_region_tree.rb_node);
563 
564 	if (--region->vm_usage == 0) {
565 		if (region->vm_top > region->vm_start)
566 			delete_nommu_region(region);
567 		up_write(&nommu_region_sem);
568 
569 		if (region->vm_file)
570 			fput(region->vm_file);
571 
572 		/* IO memory and memory shared directly out of the pagecache
573 		 * from ramfs/tmpfs mustn't be released here */
574 		if (region->vm_flags & VM_MAPPED_COPY)
575 			free_page_series(region->vm_start, region->vm_top);
576 		kmem_cache_free(vm_region_jar, region);
577 	} else {
578 		up_write(&nommu_region_sem);
579 	}
580 }
581 
582 /*
583  * release a reference to a region
584  */
585 static void put_nommu_region(struct vm_region *region)
586 {
587 	down_write(&nommu_region_sem);
588 	__put_nommu_region(region);
589 }
590 
591 /*
592  * add a VMA into a process's mm_struct in the appropriate place in the list
593  * and tree and add to the address space's page tree also if not an anonymous
594  * page
595  * - should be called with mm->mmap_sem held writelocked
596  */
597 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma)
598 {
599 	struct vm_area_struct *pvma, *prev;
600 	struct address_space *mapping;
601 	struct rb_node **p, *parent, *rb_prev;
602 
603 	BUG_ON(!vma->vm_region);
604 
605 	mm->map_count++;
606 	vma->vm_mm = mm;
607 
608 	/* add the VMA to the mapping */
609 	if (vma->vm_file) {
610 		mapping = vma->vm_file->f_mapping;
611 
612 		i_mmap_lock_write(mapping);
613 		flush_dcache_mmap_lock(mapping);
614 		vma_interval_tree_insert(vma, &mapping->i_mmap);
615 		flush_dcache_mmap_unlock(mapping);
616 		i_mmap_unlock_write(mapping);
617 	}
618 
619 	/* add the VMA to the tree */
620 	parent = rb_prev = NULL;
621 	p = &mm->mm_rb.rb_node;
622 	while (*p) {
623 		parent = *p;
624 		pvma = rb_entry(parent, struct vm_area_struct, vm_rb);
625 
626 		/* sort by: start addr, end addr, VMA struct addr in that order
627 		 * (the latter is necessary as we may get identical VMAs) */
628 		if (vma->vm_start < pvma->vm_start)
629 			p = &(*p)->rb_left;
630 		else if (vma->vm_start > pvma->vm_start) {
631 			rb_prev = parent;
632 			p = &(*p)->rb_right;
633 		} else if (vma->vm_end < pvma->vm_end)
634 			p = &(*p)->rb_left;
635 		else if (vma->vm_end > pvma->vm_end) {
636 			rb_prev = parent;
637 			p = &(*p)->rb_right;
638 		} else if (vma < pvma)
639 			p = &(*p)->rb_left;
640 		else if (vma > pvma) {
641 			rb_prev = parent;
642 			p = &(*p)->rb_right;
643 		} else
644 			BUG();
645 	}
646 
647 	rb_link_node(&vma->vm_rb, parent, p);
648 	rb_insert_color(&vma->vm_rb, &mm->mm_rb);
649 
650 	/* add VMA to the VMA list also */
651 	prev = NULL;
652 	if (rb_prev)
653 		prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
654 
655 	__vma_link_list(mm, vma, prev);
656 }
657 
658 /*
659  * delete a VMA from its owning mm_struct and address space
660  */
661 static void delete_vma_from_mm(struct vm_area_struct *vma)
662 {
663 	int i;
664 	struct address_space *mapping;
665 	struct mm_struct *mm = vma->vm_mm;
666 	struct task_struct *curr = current;
667 
668 	mm->map_count--;
669 	for (i = 0; i < VMACACHE_SIZE; i++) {
670 		/* if the vma is cached, invalidate the entire cache */
671 		if (curr->vmacache.vmas[i] == vma) {
672 			vmacache_invalidate(mm);
673 			break;
674 		}
675 	}
676 
677 	/* remove the VMA from the mapping */
678 	if (vma->vm_file) {
679 		mapping = vma->vm_file->f_mapping;
680 
681 		i_mmap_lock_write(mapping);
682 		flush_dcache_mmap_lock(mapping);
683 		vma_interval_tree_remove(vma, &mapping->i_mmap);
684 		flush_dcache_mmap_unlock(mapping);
685 		i_mmap_unlock_write(mapping);
686 	}
687 
688 	/* remove from the MM's tree and list */
689 	rb_erase(&vma->vm_rb, &mm->mm_rb);
690 
691 	__vma_unlink_list(mm, vma);
692 }
693 
694 /*
695  * destroy a VMA record
696  */
697 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma)
698 {
699 	if (vma->vm_ops && vma->vm_ops->close)
700 		vma->vm_ops->close(vma);
701 	if (vma->vm_file)
702 		fput(vma->vm_file);
703 	put_nommu_region(vma->vm_region);
704 	vm_area_free(vma);
705 }
706 
707 /*
708  * look up the first VMA in which addr resides, NULL if none
709  * - should be called with mm->mmap_sem at least held readlocked
710  */
711 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
712 {
713 	struct vm_area_struct *vma;
714 
715 	/* check the cache first */
716 	vma = vmacache_find(mm, addr);
717 	if (likely(vma))
718 		return vma;
719 
720 	/* trawl the list (there may be multiple mappings in which addr
721 	 * resides) */
722 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
723 		if (vma->vm_start > addr)
724 			return NULL;
725 		if (vma->vm_end > addr) {
726 			vmacache_update(addr, vma);
727 			return vma;
728 		}
729 	}
730 
731 	return NULL;
732 }
733 EXPORT_SYMBOL(find_vma);
734 
735 /*
736  * find a VMA
737  * - we don't extend stack VMAs under NOMMU conditions
738  */
739 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
740 {
741 	return find_vma(mm, addr);
742 }
743 
744 /*
745  * expand a stack to a given address
746  * - not supported under NOMMU conditions
747  */
748 int expand_stack(struct vm_area_struct *vma, unsigned long address)
749 {
750 	return -ENOMEM;
751 }
752 
753 /*
754  * look up the first VMA exactly that exactly matches addr
755  * - should be called with mm->mmap_sem at least held readlocked
756  */
757 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm,
758 					     unsigned long addr,
759 					     unsigned long len)
760 {
761 	struct vm_area_struct *vma;
762 	unsigned long end = addr + len;
763 
764 	/* check the cache first */
765 	vma = vmacache_find_exact(mm, addr, end);
766 	if (vma)
767 		return vma;
768 
769 	/* trawl the list (there may be multiple mappings in which addr
770 	 * resides) */
771 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
772 		if (vma->vm_start < addr)
773 			continue;
774 		if (vma->vm_start > addr)
775 			return NULL;
776 		if (vma->vm_end == end) {
777 			vmacache_update(addr, vma);
778 			return vma;
779 		}
780 	}
781 
782 	return NULL;
783 }
784 
785 /*
786  * determine whether a mapping should be permitted and, if so, what sort of
787  * mapping we're capable of supporting
788  */
789 static int validate_mmap_request(struct file *file,
790 				 unsigned long addr,
791 				 unsigned long len,
792 				 unsigned long prot,
793 				 unsigned long flags,
794 				 unsigned long pgoff,
795 				 unsigned long *_capabilities)
796 {
797 	unsigned long capabilities, rlen;
798 	int ret;
799 
800 	/* do the simple checks first */
801 	if (flags & MAP_FIXED)
802 		return -EINVAL;
803 
804 	if ((flags & MAP_TYPE) != MAP_PRIVATE &&
805 	    (flags & MAP_TYPE) != MAP_SHARED)
806 		return -EINVAL;
807 
808 	if (!len)
809 		return -EINVAL;
810 
811 	/* Careful about overflows.. */
812 	rlen = PAGE_ALIGN(len);
813 	if (!rlen || rlen > TASK_SIZE)
814 		return -ENOMEM;
815 
816 	/* offset overflow? */
817 	if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff)
818 		return -EOVERFLOW;
819 
820 	if (file) {
821 		/* files must support mmap */
822 		if (!file->f_op->mmap)
823 			return -ENODEV;
824 
825 		/* work out if what we've got could possibly be shared
826 		 * - we support chardevs that provide their own "memory"
827 		 * - we support files/blockdevs that are memory backed
828 		 */
829 		if (file->f_op->mmap_capabilities) {
830 			capabilities = file->f_op->mmap_capabilities(file);
831 		} else {
832 			/* no explicit capabilities set, so assume some
833 			 * defaults */
834 			switch (file_inode(file)->i_mode & S_IFMT) {
835 			case S_IFREG:
836 			case S_IFBLK:
837 				capabilities = NOMMU_MAP_COPY;
838 				break;
839 
840 			case S_IFCHR:
841 				capabilities =
842 					NOMMU_MAP_DIRECT |
843 					NOMMU_MAP_READ |
844 					NOMMU_MAP_WRITE;
845 				break;
846 
847 			default:
848 				return -EINVAL;
849 			}
850 		}
851 
852 		/* eliminate any capabilities that we can't support on this
853 		 * device */
854 		if (!file->f_op->get_unmapped_area)
855 			capabilities &= ~NOMMU_MAP_DIRECT;
856 		if (!(file->f_mode & FMODE_CAN_READ))
857 			capabilities &= ~NOMMU_MAP_COPY;
858 
859 		/* The file shall have been opened with read permission. */
860 		if (!(file->f_mode & FMODE_READ))
861 			return -EACCES;
862 
863 		if (flags & MAP_SHARED) {
864 			/* do checks for writing, appending and locking */
865 			if ((prot & PROT_WRITE) &&
866 			    !(file->f_mode & FMODE_WRITE))
867 				return -EACCES;
868 
869 			if (IS_APPEND(file_inode(file)) &&
870 			    (file->f_mode & FMODE_WRITE))
871 				return -EACCES;
872 
873 			if (locks_verify_locked(file))
874 				return -EAGAIN;
875 
876 			if (!(capabilities & NOMMU_MAP_DIRECT))
877 				return -ENODEV;
878 
879 			/* we mustn't privatise shared mappings */
880 			capabilities &= ~NOMMU_MAP_COPY;
881 		} else {
882 			/* we're going to read the file into private memory we
883 			 * allocate */
884 			if (!(capabilities & NOMMU_MAP_COPY))
885 				return -ENODEV;
886 
887 			/* we don't permit a private writable mapping to be
888 			 * shared with the backing device */
889 			if (prot & PROT_WRITE)
890 				capabilities &= ~NOMMU_MAP_DIRECT;
891 		}
892 
893 		if (capabilities & NOMMU_MAP_DIRECT) {
894 			if (((prot & PROT_READ)  && !(capabilities & NOMMU_MAP_READ))  ||
895 			    ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) ||
896 			    ((prot & PROT_EXEC)  && !(capabilities & NOMMU_MAP_EXEC))
897 			    ) {
898 				capabilities &= ~NOMMU_MAP_DIRECT;
899 				if (flags & MAP_SHARED) {
900 					pr_warn("MAP_SHARED not completely supported on !MMU\n");
901 					return -EINVAL;
902 				}
903 			}
904 		}
905 
906 		/* handle executable mappings and implied executable
907 		 * mappings */
908 		if (path_noexec(&file->f_path)) {
909 			if (prot & PROT_EXEC)
910 				return -EPERM;
911 		} else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) {
912 			/* handle implication of PROT_EXEC by PROT_READ */
913 			if (current->personality & READ_IMPLIES_EXEC) {
914 				if (capabilities & NOMMU_MAP_EXEC)
915 					prot |= PROT_EXEC;
916 			}
917 		} else if ((prot & PROT_READ) &&
918 			 (prot & PROT_EXEC) &&
919 			 !(capabilities & NOMMU_MAP_EXEC)
920 			 ) {
921 			/* backing file is not executable, try to copy */
922 			capabilities &= ~NOMMU_MAP_DIRECT;
923 		}
924 	} else {
925 		/* anonymous mappings are always memory backed and can be
926 		 * privately mapped
927 		 */
928 		capabilities = NOMMU_MAP_COPY;
929 
930 		/* handle PROT_EXEC implication by PROT_READ */
931 		if ((prot & PROT_READ) &&
932 		    (current->personality & READ_IMPLIES_EXEC))
933 			prot |= PROT_EXEC;
934 	}
935 
936 	/* allow the security API to have its say */
937 	ret = security_mmap_addr(addr);
938 	if (ret < 0)
939 		return ret;
940 
941 	/* looks okay */
942 	*_capabilities = capabilities;
943 	return 0;
944 }
945 
946 /*
947  * we've determined that we can make the mapping, now translate what we
948  * now know into VMA flags
949  */
950 static unsigned long determine_vm_flags(struct file *file,
951 					unsigned long prot,
952 					unsigned long flags,
953 					unsigned long capabilities)
954 {
955 	unsigned long vm_flags;
956 
957 	vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags);
958 	/* vm_flags |= mm->def_flags; */
959 
960 	if (!(capabilities & NOMMU_MAP_DIRECT)) {
961 		/* attempt to share read-only copies of mapped file chunks */
962 		vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
963 		if (file && !(prot & PROT_WRITE))
964 			vm_flags |= VM_MAYSHARE;
965 	} else {
966 		/* overlay a shareable mapping on the backing device or inode
967 		 * if possible - used for chardevs, ramfs/tmpfs/shmfs and
968 		 * romfs/cramfs */
969 		vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS);
970 		if (flags & MAP_SHARED)
971 			vm_flags |= VM_SHARED;
972 	}
973 
974 	/* refuse to let anyone share private mappings with this process if
975 	 * it's being traced - otherwise breakpoints set in it may interfere
976 	 * with another untraced process
977 	 */
978 	if ((flags & MAP_PRIVATE) && current->ptrace)
979 		vm_flags &= ~VM_MAYSHARE;
980 
981 	return vm_flags;
982 }
983 
984 /*
985  * set up a shared mapping on a file (the driver or filesystem provides and
986  * pins the storage)
987  */
988 static int do_mmap_shared_file(struct vm_area_struct *vma)
989 {
990 	int ret;
991 
992 	ret = call_mmap(vma->vm_file, vma);
993 	if (ret == 0) {
994 		vma->vm_region->vm_top = vma->vm_region->vm_end;
995 		return 0;
996 	}
997 	if (ret != -ENOSYS)
998 		return ret;
999 
1000 	/* getting -ENOSYS indicates that direct mmap isn't possible (as
1001 	 * opposed to tried but failed) so we can only give a suitable error as
1002 	 * it's not possible to make a private copy if MAP_SHARED was given */
1003 	return -ENODEV;
1004 }
1005 
1006 /*
1007  * set up a private mapping or an anonymous shared mapping
1008  */
1009 static int do_mmap_private(struct vm_area_struct *vma,
1010 			   struct vm_region *region,
1011 			   unsigned long len,
1012 			   unsigned long capabilities)
1013 {
1014 	unsigned long total, point;
1015 	void *base;
1016 	int ret, order;
1017 
1018 	/* invoke the file's mapping function so that it can keep track of
1019 	 * shared mappings on devices or memory
1020 	 * - VM_MAYSHARE will be set if it may attempt to share
1021 	 */
1022 	if (capabilities & NOMMU_MAP_DIRECT) {
1023 		ret = call_mmap(vma->vm_file, vma);
1024 		if (ret == 0) {
1025 			/* shouldn't return success if we're not sharing */
1026 			BUG_ON(!(vma->vm_flags & VM_MAYSHARE));
1027 			vma->vm_region->vm_top = vma->vm_region->vm_end;
1028 			return 0;
1029 		}
1030 		if (ret != -ENOSYS)
1031 			return ret;
1032 
1033 		/* getting an ENOSYS error indicates that direct mmap isn't
1034 		 * possible (as opposed to tried but failed) so we'll try to
1035 		 * make a private copy of the data and map that instead */
1036 	}
1037 
1038 
1039 	/* allocate some memory to hold the mapping
1040 	 * - note that this may not return a page-aligned address if the object
1041 	 *   we're allocating is smaller than a page
1042 	 */
1043 	order = get_order(len);
1044 	total = 1 << order;
1045 	point = len >> PAGE_SHIFT;
1046 
1047 	/* we don't want to allocate a power-of-2 sized page set */
1048 	if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages)
1049 		total = point;
1050 
1051 	base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL);
1052 	if (!base)
1053 		goto enomem;
1054 
1055 	atomic_long_add(total, &mmap_pages_allocated);
1056 
1057 	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
1058 	region->vm_start = (unsigned long) base;
1059 	region->vm_end   = region->vm_start + len;
1060 	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
1061 
1062 	vma->vm_start = region->vm_start;
1063 	vma->vm_end   = region->vm_start + len;
1064 
1065 	if (vma->vm_file) {
1066 		/* read the contents of a file into the copy */
1067 		loff_t fpos;
1068 
1069 		fpos = vma->vm_pgoff;
1070 		fpos <<= PAGE_SHIFT;
1071 
1072 		ret = kernel_read(vma->vm_file, base, len, &fpos);
1073 		if (ret < 0)
1074 			goto error_free;
1075 
1076 		/* clear the last little bit */
1077 		if (ret < len)
1078 			memset(base + ret, 0, len - ret);
1079 
1080 	} else {
1081 		vma_set_anonymous(vma);
1082 	}
1083 
1084 	return 0;
1085 
1086 error_free:
1087 	free_page_series(region->vm_start, region->vm_top);
1088 	region->vm_start = vma->vm_start = 0;
1089 	region->vm_end   = vma->vm_end = 0;
1090 	region->vm_top   = 0;
1091 	return ret;
1092 
1093 enomem:
1094 	pr_err("Allocation of length %lu from process %d (%s) failed\n",
1095 	       len, current->pid, current->comm);
1096 	show_free_areas(0, NULL);
1097 	return -ENOMEM;
1098 }
1099 
1100 /*
1101  * handle mapping creation for uClinux
1102  */
1103 unsigned long do_mmap(struct file *file,
1104 			unsigned long addr,
1105 			unsigned long len,
1106 			unsigned long prot,
1107 			unsigned long flags,
1108 			vm_flags_t vm_flags,
1109 			unsigned long pgoff,
1110 			unsigned long *populate,
1111 			struct list_head *uf)
1112 {
1113 	struct vm_area_struct *vma;
1114 	struct vm_region *region;
1115 	struct rb_node *rb;
1116 	unsigned long capabilities, result;
1117 	int ret;
1118 
1119 	*populate = 0;
1120 
1121 	/* decide whether we should attempt the mapping, and if so what sort of
1122 	 * mapping */
1123 	ret = validate_mmap_request(file, addr, len, prot, flags, pgoff,
1124 				    &capabilities);
1125 	if (ret < 0)
1126 		return ret;
1127 
1128 	/* we ignore the address hint */
1129 	addr = 0;
1130 	len = PAGE_ALIGN(len);
1131 
1132 	/* we've determined that we can make the mapping, now translate what we
1133 	 * now know into VMA flags */
1134 	vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
1135 
1136 	/* we're going to need to record the mapping */
1137 	region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL);
1138 	if (!region)
1139 		goto error_getting_region;
1140 
1141 	vma = vm_area_alloc(current->mm);
1142 	if (!vma)
1143 		goto error_getting_vma;
1144 
1145 	region->vm_usage = 1;
1146 	region->vm_flags = vm_flags;
1147 	region->vm_pgoff = pgoff;
1148 
1149 	vma->vm_flags = vm_flags;
1150 	vma->vm_pgoff = pgoff;
1151 
1152 	if (file) {
1153 		region->vm_file = get_file(file);
1154 		vma->vm_file = get_file(file);
1155 	}
1156 
1157 	down_write(&nommu_region_sem);
1158 
1159 	/* if we want to share, we need to check for regions created by other
1160 	 * mmap() calls that overlap with our proposed mapping
1161 	 * - we can only share with a superset match on most regular files
1162 	 * - shared mappings on character devices and memory backed files are
1163 	 *   permitted to overlap inexactly as far as we are concerned for in
1164 	 *   these cases, sharing is handled in the driver or filesystem rather
1165 	 *   than here
1166 	 */
1167 	if (vm_flags & VM_MAYSHARE) {
1168 		struct vm_region *pregion;
1169 		unsigned long pglen, rpglen, pgend, rpgend, start;
1170 
1171 		pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1172 		pgend = pgoff + pglen;
1173 
1174 		for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) {
1175 			pregion = rb_entry(rb, struct vm_region, vm_rb);
1176 
1177 			if (!(pregion->vm_flags & VM_MAYSHARE))
1178 				continue;
1179 
1180 			/* search for overlapping mappings on the same file */
1181 			if (file_inode(pregion->vm_file) !=
1182 			    file_inode(file))
1183 				continue;
1184 
1185 			if (pregion->vm_pgoff >= pgend)
1186 				continue;
1187 
1188 			rpglen = pregion->vm_end - pregion->vm_start;
1189 			rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT;
1190 			rpgend = pregion->vm_pgoff + rpglen;
1191 			if (pgoff >= rpgend)
1192 				continue;
1193 
1194 			/* handle inexactly overlapping matches between
1195 			 * mappings */
1196 			if ((pregion->vm_pgoff != pgoff || rpglen != pglen) &&
1197 			    !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) {
1198 				/* new mapping is not a subset of the region */
1199 				if (!(capabilities & NOMMU_MAP_DIRECT))
1200 					goto sharing_violation;
1201 				continue;
1202 			}
1203 
1204 			/* we've found a region we can share */
1205 			pregion->vm_usage++;
1206 			vma->vm_region = pregion;
1207 			start = pregion->vm_start;
1208 			start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT;
1209 			vma->vm_start = start;
1210 			vma->vm_end = start + len;
1211 
1212 			if (pregion->vm_flags & VM_MAPPED_COPY)
1213 				vma->vm_flags |= VM_MAPPED_COPY;
1214 			else {
1215 				ret = do_mmap_shared_file(vma);
1216 				if (ret < 0) {
1217 					vma->vm_region = NULL;
1218 					vma->vm_start = 0;
1219 					vma->vm_end = 0;
1220 					pregion->vm_usage--;
1221 					pregion = NULL;
1222 					goto error_just_free;
1223 				}
1224 			}
1225 			fput(region->vm_file);
1226 			kmem_cache_free(vm_region_jar, region);
1227 			region = pregion;
1228 			result = start;
1229 			goto share;
1230 		}
1231 
1232 		/* obtain the address at which to make a shared mapping
1233 		 * - this is the hook for quasi-memory character devices to
1234 		 *   tell us the location of a shared mapping
1235 		 */
1236 		if (capabilities & NOMMU_MAP_DIRECT) {
1237 			addr = file->f_op->get_unmapped_area(file, addr, len,
1238 							     pgoff, flags);
1239 			if (IS_ERR_VALUE(addr)) {
1240 				ret = addr;
1241 				if (ret != -ENOSYS)
1242 					goto error_just_free;
1243 
1244 				/* the driver refused to tell us where to site
1245 				 * the mapping so we'll have to attempt to copy
1246 				 * it */
1247 				ret = -ENODEV;
1248 				if (!(capabilities & NOMMU_MAP_COPY))
1249 					goto error_just_free;
1250 
1251 				capabilities &= ~NOMMU_MAP_DIRECT;
1252 			} else {
1253 				vma->vm_start = region->vm_start = addr;
1254 				vma->vm_end = region->vm_end = addr + len;
1255 			}
1256 		}
1257 	}
1258 
1259 	vma->vm_region = region;
1260 
1261 	/* set up the mapping
1262 	 * - the region is filled in if NOMMU_MAP_DIRECT is still set
1263 	 */
1264 	if (file && vma->vm_flags & VM_SHARED)
1265 		ret = do_mmap_shared_file(vma);
1266 	else
1267 		ret = do_mmap_private(vma, region, len, capabilities);
1268 	if (ret < 0)
1269 		goto error_just_free;
1270 	add_nommu_region(region);
1271 
1272 	/* clear anonymous mappings that don't ask for uninitialized data */
1273 	if (!vma->vm_file &&
1274 	    (!IS_ENABLED(CONFIG_MMAP_ALLOW_UNINITIALIZED) ||
1275 	     !(flags & MAP_UNINITIALIZED)))
1276 		memset((void *)region->vm_start, 0,
1277 		       region->vm_end - region->vm_start);
1278 
1279 	/* okay... we have a mapping; now we have to register it */
1280 	result = vma->vm_start;
1281 
1282 	current->mm->total_vm += len >> PAGE_SHIFT;
1283 
1284 share:
1285 	add_vma_to_mm(current->mm, vma);
1286 
1287 	/* we flush the region from the icache only when the first executable
1288 	 * mapping of it is made  */
1289 	if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) {
1290 		flush_icache_range(region->vm_start, region->vm_end);
1291 		region->vm_icache_flushed = true;
1292 	}
1293 
1294 	up_write(&nommu_region_sem);
1295 
1296 	return result;
1297 
1298 error_just_free:
1299 	up_write(&nommu_region_sem);
1300 error:
1301 	if (region->vm_file)
1302 		fput(region->vm_file);
1303 	kmem_cache_free(vm_region_jar, region);
1304 	if (vma->vm_file)
1305 		fput(vma->vm_file);
1306 	vm_area_free(vma);
1307 	return ret;
1308 
1309 sharing_violation:
1310 	up_write(&nommu_region_sem);
1311 	pr_warn("Attempt to share mismatched mappings\n");
1312 	ret = -EINVAL;
1313 	goto error;
1314 
1315 error_getting_vma:
1316 	kmem_cache_free(vm_region_jar, region);
1317 	pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n",
1318 			len, current->pid);
1319 	show_free_areas(0, NULL);
1320 	return -ENOMEM;
1321 
1322 error_getting_region:
1323 	pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n",
1324 			len, current->pid);
1325 	show_free_areas(0, NULL);
1326 	return -ENOMEM;
1327 }
1328 
1329 unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
1330 			      unsigned long prot, unsigned long flags,
1331 			      unsigned long fd, unsigned long pgoff)
1332 {
1333 	struct file *file = NULL;
1334 	unsigned long retval = -EBADF;
1335 
1336 	audit_mmap_fd(fd, flags);
1337 	if (!(flags & MAP_ANONYMOUS)) {
1338 		file = fget(fd);
1339 		if (!file)
1340 			goto out;
1341 	}
1342 
1343 	flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1344 
1345 	retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1346 
1347 	if (file)
1348 		fput(file);
1349 out:
1350 	return retval;
1351 }
1352 
1353 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1354 		unsigned long, prot, unsigned long, flags,
1355 		unsigned long, fd, unsigned long, pgoff)
1356 {
1357 	return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
1358 }
1359 
1360 #ifdef __ARCH_WANT_SYS_OLD_MMAP
1361 struct mmap_arg_struct {
1362 	unsigned long addr;
1363 	unsigned long len;
1364 	unsigned long prot;
1365 	unsigned long flags;
1366 	unsigned long fd;
1367 	unsigned long offset;
1368 };
1369 
1370 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1371 {
1372 	struct mmap_arg_struct a;
1373 
1374 	if (copy_from_user(&a, arg, sizeof(a)))
1375 		return -EFAULT;
1376 	if (offset_in_page(a.offset))
1377 		return -EINVAL;
1378 
1379 	return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1380 			       a.offset >> PAGE_SHIFT);
1381 }
1382 #endif /* __ARCH_WANT_SYS_OLD_MMAP */
1383 
1384 /*
1385  * split a vma into two pieces at address 'addr', a new vma is allocated either
1386  * for the first part or the tail.
1387  */
1388 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
1389 	      unsigned long addr, int new_below)
1390 {
1391 	struct vm_area_struct *new;
1392 	struct vm_region *region;
1393 	unsigned long npages;
1394 
1395 	/* we're only permitted to split anonymous regions (these should have
1396 	 * only a single usage on the region) */
1397 	if (vma->vm_file)
1398 		return -ENOMEM;
1399 
1400 	if (mm->map_count >= sysctl_max_map_count)
1401 		return -ENOMEM;
1402 
1403 	region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL);
1404 	if (!region)
1405 		return -ENOMEM;
1406 
1407 	new = vm_area_dup(vma);
1408 	if (!new) {
1409 		kmem_cache_free(vm_region_jar, region);
1410 		return -ENOMEM;
1411 	}
1412 
1413 	/* most fields are the same, copy all, and then fixup */
1414 	*region = *vma->vm_region;
1415 	new->vm_region = region;
1416 
1417 	npages = (addr - vma->vm_start) >> PAGE_SHIFT;
1418 
1419 	if (new_below) {
1420 		region->vm_top = region->vm_end = new->vm_end = addr;
1421 	} else {
1422 		region->vm_start = new->vm_start = addr;
1423 		region->vm_pgoff = new->vm_pgoff += npages;
1424 	}
1425 
1426 	if (new->vm_ops && new->vm_ops->open)
1427 		new->vm_ops->open(new);
1428 
1429 	delete_vma_from_mm(vma);
1430 	down_write(&nommu_region_sem);
1431 	delete_nommu_region(vma->vm_region);
1432 	if (new_below) {
1433 		vma->vm_region->vm_start = vma->vm_start = addr;
1434 		vma->vm_region->vm_pgoff = vma->vm_pgoff += npages;
1435 	} else {
1436 		vma->vm_region->vm_end = vma->vm_end = addr;
1437 		vma->vm_region->vm_top = addr;
1438 	}
1439 	add_nommu_region(vma->vm_region);
1440 	add_nommu_region(new->vm_region);
1441 	up_write(&nommu_region_sem);
1442 	add_vma_to_mm(mm, vma);
1443 	add_vma_to_mm(mm, new);
1444 	return 0;
1445 }
1446 
1447 /*
1448  * shrink a VMA by removing the specified chunk from either the beginning or
1449  * the end
1450  */
1451 static int shrink_vma(struct mm_struct *mm,
1452 		      struct vm_area_struct *vma,
1453 		      unsigned long from, unsigned long to)
1454 {
1455 	struct vm_region *region;
1456 
1457 	/* adjust the VMA's pointers, which may reposition it in the MM's tree
1458 	 * and list */
1459 	delete_vma_from_mm(vma);
1460 	if (from > vma->vm_start)
1461 		vma->vm_end = from;
1462 	else
1463 		vma->vm_start = to;
1464 	add_vma_to_mm(mm, vma);
1465 
1466 	/* cut the backing region down to size */
1467 	region = vma->vm_region;
1468 	BUG_ON(region->vm_usage != 1);
1469 
1470 	down_write(&nommu_region_sem);
1471 	delete_nommu_region(region);
1472 	if (from > region->vm_start) {
1473 		to = region->vm_top;
1474 		region->vm_top = region->vm_end = from;
1475 	} else {
1476 		region->vm_start = to;
1477 	}
1478 	add_nommu_region(region);
1479 	up_write(&nommu_region_sem);
1480 
1481 	free_page_series(from, to);
1482 	return 0;
1483 }
1484 
1485 /*
1486  * release a mapping
1487  * - under NOMMU conditions the chunk to be unmapped must be backed by a single
1488  *   VMA, though it need not cover the whole VMA
1489  */
1490 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
1491 {
1492 	struct vm_area_struct *vma;
1493 	unsigned long end;
1494 	int ret;
1495 
1496 	len = PAGE_ALIGN(len);
1497 	if (len == 0)
1498 		return -EINVAL;
1499 
1500 	end = start + len;
1501 
1502 	/* find the first potentially overlapping VMA */
1503 	vma = find_vma(mm, start);
1504 	if (!vma) {
1505 		static int limit;
1506 		if (limit < 5) {
1507 			pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n",
1508 					current->pid, current->comm,
1509 					start, start + len - 1);
1510 			limit++;
1511 		}
1512 		return -EINVAL;
1513 	}
1514 
1515 	/* we're allowed to split an anonymous VMA but not a file-backed one */
1516 	if (vma->vm_file) {
1517 		do {
1518 			if (start > vma->vm_start)
1519 				return -EINVAL;
1520 			if (end == vma->vm_end)
1521 				goto erase_whole_vma;
1522 			vma = vma->vm_next;
1523 		} while (vma);
1524 		return -EINVAL;
1525 	} else {
1526 		/* the chunk must be a subset of the VMA found */
1527 		if (start == vma->vm_start && end == vma->vm_end)
1528 			goto erase_whole_vma;
1529 		if (start < vma->vm_start || end > vma->vm_end)
1530 			return -EINVAL;
1531 		if (offset_in_page(start))
1532 			return -EINVAL;
1533 		if (end != vma->vm_end && offset_in_page(end))
1534 			return -EINVAL;
1535 		if (start != vma->vm_start && end != vma->vm_end) {
1536 			ret = split_vma(mm, vma, start, 1);
1537 			if (ret < 0)
1538 				return ret;
1539 		}
1540 		return shrink_vma(mm, vma, start, end);
1541 	}
1542 
1543 erase_whole_vma:
1544 	delete_vma_from_mm(vma);
1545 	delete_vma(mm, vma);
1546 	return 0;
1547 }
1548 EXPORT_SYMBOL(do_munmap);
1549 
1550 int vm_munmap(unsigned long addr, size_t len)
1551 {
1552 	struct mm_struct *mm = current->mm;
1553 	int ret;
1554 
1555 	down_write(&mm->mmap_sem);
1556 	ret = do_munmap(mm, addr, len, NULL);
1557 	up_write(&mm->mmap_sem);
1558 	return ret;
1559 }
1560 EXPORT_SYMBOL(vm_munmap);
1561 
1562 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
1563 {
1564 	return vm_munmap(addr, len);
1565 }
1566 
1567 /*
1568  * release all the mappings made in a process's VM space
1569  */
1570 void exit_mmap(struct mm_struct *mm)
1571 {
1572 	struct vm_area_struct *vma;
1573 
1574 	if (!mm)
1575 		return;
1576 
1577 	mm->total_vm = 0;
1578 
1579 	while ((vma = mm->mmap)) {
1580 		mm->mmap = vma->vm_next;
1581 		delete_vma_from_mm(vma);
1582 		delete_vma(mm, vma);
1583 		cond_resched();
1584 	}
1585 }
1586 
1587 int vm_brk(unsigned long addr, unsigned long len)
1588 {
1589 	return -ENOMEM;
1590 }
1591 
1592 /*
1593  * expand (or shrink) an existing mapping, potentially moving it at the same
1594  * time (controlled by the MREMAP_MAYMOVE flag and available VM space)
1595  *
1596  * under NOMMU conditions, we only permit changing a mapping's size, and only
1597  * as long as it stays within the region allocated by do_mmap_private() and the
1598  * block is not shareable
1599  *
1600  * MREMAP_FIXED is not supported under NOMMU conditions
1601  */
1602 static unsigned long do_mremap(unsigned long addr,
1603 			unsigned long old_len, unsigned long new_len,
1604 			unsigned long flags, unsigned long new_addr)
1605 {
1606 	struct vm_area_struct *vma;
1607 
1608 	/* insanity checks first */
1609 	old_len = PAGE_ALIGN(old_len);
1610 	new_len = PAGE_ALIGN(new_len);
1611 	if (old_len == 0 || new_len == 0)
1612 		return (unsigned long) -EINVAL;
1613 
1614 	if (offset_in_page(addr))
1615 		return -EINVAL;
1616 
1617 	if (flags & MREMAP_FIXED && new_addr != addr)
1618 		return (unsigned long) -EINVAL;
1619 
1620 	vma = find_vma_exact(current->mm, addr, old_len);
1621 	if (!vma)
1622 		return (unsigned long) -EINVAL;
1623 
1624 	if (vma->vm_end != vma->vm_start + old_len)
1625 		return (unsigned long) -EFAULT;
1626 
1627 	if (vma->vm_flags & VM_MAYSHARE)
1628 		return (unsigned long) -EPERM;
1629 
1630 	if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start)
1631 		return (unsigned long) -ENOMEM;
1632 
1633 	/* all checks complete - do it */
1634 	vma->vm_end = vma->vm_start + new_len;
1635 	return vma->vm_start;
1636 }
1637 
1638 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
1639 		unsigned long, new_len, unsigned long, flags,
1640 		unsigned long, new_addr)
1641 {
1642 	unsigned long ret;
1643 
1644 	down_write(&current->mm->mmap_sem);
1645 	ret = do_mremap(addr, old_len, new_len, flags, new_addr);
1646 	up_write(&current->mm->mmap_sem);
1647 	return ret;
1648 }
1649 
1650 struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
1651 			 unsigned int foll_flags)
1652 {
1653 	return NULL;
1654 }
1655 
1656 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
1657 		unsigned long pfn, unsigned long size, pgprot_t prot)
1658 {
1659 	if (addr != (pfn << PAGE_SHIFT))
1660 		return -EINVAL;
1661 
1662 	vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
1663 	return 0;
1664 }
1665 EXPORT_SYMBOL(remap_pfn_range);
1666 
1667 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len)
1668 {
1669 	unsigned long pfn = start >> PAGE_SHIFT;
1670 	unsigned long vm_len = vma->vm_end - vma->vm_start;
1671 
1672 	pfn += vma->vm_pgoff;
1673 	return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot);
1674 }
1675 EXPORT_SYMBOL(vm_iomap_memory);
1676 
1677 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
1678 			unsigned long pgoff)
1679 {
1680 	unsigned int size = vma->vm_end - vma->vm_start;
1681 
1682 	if (!(vma->vm_flags & VM_USERMAP))
1683 		return -EINVAL;
1684 
1685 	vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT));
1686 	vma->vm_end = vma->vm_start + size;
1687 
1688 	return 0;
1689 }
1690 EXPORT_SYMBOL(remap_vmalloc_range);
1691 
1692 unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr,
1693 	unsigned long len, unsigned long pgoff, unsigned long flags)
1694 {
1695 	return -ENOMEM;
1696 }
1697 
1698 vm_fault_t filemap_fault(struct vm_fault *vmf)
1699 {
1700 	BUG();
1701 	return 0;
1702 }
1703 EXPORT_SYMBOL(filemap_fault);
1704 
1705 void filemap_map_pages(struct vm_fault *vmf,
1706 		pgoff_t start_pgoff, pgoff_t end_pgoff)
1707 {
1708 	BUG();
1709 }
1710 EXPORT_SYMBOL(filemap_map_pages);
1711 
1712 int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
1713 		unsigned long addr, void *buf, int len, unsigned int gup_flags)
1714 {
1715 	struct vm_area_struct *vma;
1716 	int write = gup_flags & FOLL_WRITE;
1717 
1718 	if (down_read_killable(&mm->mmap_sem))
1719 		return 0;
1720 
1721 	/* the access must start within one of the target process's mappings */
1722 	vma = find_vma(mm, addr);
1723 	if (vma) {
1724 		/* don't overrun this mapping */
1725 		if (addr + len >= vma->vm_end)
1726 			len = vma->vm_end - addr;
1727 
1728 		/* only read or write mappings where it is permitted */
1729 		if (write && vma->vm_flags & VM_MAYWRITE)
1730 			copy_to_user_page(vma, NULL, addr,
1731 					 (void *) addr, buf, len);
1732 		else if (!write && vma->vm_flags & VM_MAYREAD)
1733 			copy_from_user_page(vma, NULL, addr,
1734 					    buf, (void *) addr, len);
1735 		else
1736 			len = 0;
1737 	} else {
1738 		len = 0;
1739 	}
1740 
1741 	up_read(&mm->mmap_sem);
1742 
1743 	return len;
1744 }
1745 
1746 /**
1747  * access_remote_vm - access another process' address space
1748  * @mm:		the mm_struct of the target address space
1749  * @addr:	start address to access
1750  * @buf:	source or destination buffer
1751  * @len:	number of bytes to transfer
1752  * @gup_flags:	flags modifying lookup behaviour
1753  *
1754  * The caller must hold a reference on @mm.
1755  */
1756 int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1757 		void *buf, int len, unsigned int gup_flags)
1758 {
1759 	return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags);
1760 }
1761 
1762 /*
1763  * Access another process' address space.
1764  * - source/target buffer must be kernel space
1765  */
1766 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len,
1767 		unsigned int gup_flags)
1768 {
1769 	struct mm_struct *mm;
1770 
1771 	if (addr + len < addr)
1772 		return 0;
1773 
1774 	mm = get_task_mm(tsk);
1775 	if (!mm)
1776 		return 0;
1777 
1778 	len = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags);
1779 
1780 	mmput(mm);
1781 	return len;
1782 }
1783 EXPORT_SYMBOL_GPL(access_process_vm);
1784 
1785 /**
1786  * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1787  * @inode: The inode to check
1788  * @size: The current filesize of the inode
1789  * @newsize: The proposed filesize of the inode
1790  *
1791  * Check the shared mappings on an inode on behalf of a shrinking truncate to
1792  * make sure that that any outstanding VMAs aren't broken and then shrink the
1793  * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't
1794  * automatically grant mappings that are too large.
1795  */
1796 int nommu_shrink_inode_mappings(struct inode *inode, size_t size,
1797 				size_t newsize)
1798 {
1799 	struct vm_area_struct *vma;
1800 	struct vm_region *region;
1801 	pgoff_t low, high;
1802 	size_t r_size, r_top;
1803 
1804 	low = newsize >> PAGE_SHIFT;
1805 	high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1806 
1807 	down_write(&nommu_region_sem);
1808 	i_mmap_lock_read(inode->i_mapping);
1809 
1810 	/* search for VMAs that fall within the dead zone */
1811 	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) {
1812 		/* found one - only interested if it's shared out of the page
1813 		 * cache */
1814 		if (vma->vm_flags & VM_SHARED) {
1815 			i_mmap_unlock_read(inode->i_mapping);
1816 			up_write(&nommu_region_sem);
1817 			return -ETXTBSY; /* not quite true, but near enough */
1818 		}
1819 	}
1820 
1821 	/* reduce any regions that overlap the dead zone - if in existence,
1822 	 * these will be pointed to by VMAs that don't overlap the dead zone
1823 	 *
1824 	 * we don't check for any regions that start beyond the EOF as there
1825 	 * shouldn't be any
1826 	 */
1827 	vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) {
1828 		if (!(vma->vm_flags & VM_SHARED))
1829 			continue;
1830 
1831 		region = vma->vm_region;
1832 		r_size = region->vm_top - region->vm_start;
1833 		r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size;
1834 
1835 		if (r_top > newsize) {
1836 			region->vm_top -= r_top - newsize;
1837 			if (region->vm_end > region->vm_top)
1838 				region->vm_end = region->vm_top;
1839 		}
1840 	}
1841 
1842 	i_mmap_unlock_read(inode->i_mapping);
1843 	up_write(&nommu_region_sem);
1844 	return 0;
1845 }
1846 
1847 /*
1848  * Initialise sysctl_user_reserve_kbytes.
1849  *
1850  * This is intended to prevent a user from starting a single memory hogging
1851  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
1852  * mode.
1853  *
1854  * The default value is min(3% of free memory, 128MB)
1855  * 128MB is enough to recover with sshd/login, bash, and top/kill.
1856  */
1857 static int __meminit init_user_reserve(void)
1858 {
1859 	unsigned long free_kbytes;
1860 
1861 	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1862 
1863 	sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
1864 	return 0;
1865 }
1866 subsys_initcall(init_user_reserve);
1867 
1868 /*
1869  * Initialise sysctl_admin_reserve_kbytes.
1870  *
1871  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
1872  * to log in and kill a memory hogging process.
1873  *
1874  * Systems with more than 256MB will reserve 8MB, enough to recover
1875  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
1876  * only reserve 3% of free pages by default.
1877  */
1878 static int __meminit init_admin_reserve(void)
1879 {
1880 	unsigned long free_kbytes;
1881 
1882 	free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
1883 
1884 	sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
1885 	return 0;
1886 }
1887 subsys_initcall(init_admin_reserve);
1888