1 // SPDX-License-Identifier: GPL-2.0-only
20 #include <linux/elf-randomize.h>
33  * kfree_const - conditionally free memory
46  * kstrdup - allocate space for and copy an existing string
70  * kstrdup_const - conditionally duplicate an existing const string
90  * kstrndup - allocate space for and copy an existing string
118  * kmemdup - duplicate region of memory
139  * kvmemdup - duplicate region of memory
160  * kmemdup_nul - Create a NUL-terminated string from unterminated data
165  * Return: newly allocated copy of @s with NUL-termination or %NULL in
185  * memdup_user - duplicate memory region from user space
199 		return ERR_PTR(-ENOMEM);  in memdup_user()
203 		return ERR_PTR(-EFAULT);  in memdup_user()
211  * vmemdup_user - duplicate memory region from user space
225 		return ERR_PTR(-ENOMEM);  in vmemdup_user()
229 		return ERR_PTR(-EFAULT);  in vmemdup_user()
237  * strndup_user - duplicate an existing string from user space
251 		return ERR_PTR(-EFAULT);  in strndup_user()
254 		return ERR_PTR(-EINVAL);  in strndup_user()
261 	p[length - 1] = '\0';  in strndup_user()
268  * memdup_user_nul - duplicate memory region from user space and NUL-terminate
286 		return ERR_PTR(-ENOMEM);  in memdup_user_nul()
290 		return ERR_PTR(-EFAULT);  in memdup_user_nul()
303 	return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));  in vma_is_stack_for_current()
313 	swap(vma->vm_file, file);  in vma_set_file()
319 #define STACK_RND_MASK (0x7ff >> (PAGE_SHIFT - 12))     /* 8MB of VA */
326 	if (current->flags & PF_RANDOMIZE) {  in randomize_stack_top()
334 	return PAGE_ALIGN(stack_top) - random_variable;  in randomize_stack_top()
339  * randomize_page - Generate a random, page aligned address
355 		range -= PAGE_ALIGN(start) - start;  in randomize_page()
359 	if (start > ULONG_MAX - range)  in randomize_page()
360 		range = ULONG_MAX - start;  in randomize_page()
375 		return randomize_page(mm->brk, SZ_32M);  in arch_randomize_brk()
377 	return randomize_page(mm->brk, SZ_1G);  in arch_randomize_brk()
386 		rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1);  in arch_mmap_rnd()
389 		rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1);  in arch_mmap_rnd()
396 	if (current->personality & ADDR_COMPAT_LAYOUT)  in mmap_is_legacy()
399 	/* On parisc the stack always grows up - so a unlimited stack should  in mmap_is_legacy()
401 	if (rlim_stack->rlim_cur == RLIM_INFINITY &&  in mmap_is_legacy()
424 	return PAGE_ALIGN_DOWN(mmap_upper_limit(rlim_stack) - rnd);  in mmap_base()
426 	unsigned long gap = rlim_stack->rlim_cur;  in mmap_base()
430 	if (current->flags & PF_RANDOMIZE)  in mmap_base()
442 	return PAGE_ALIGN(STACK_TOP - gap - rnd);  in mmap_base()
450 	if (current->flags & PF_RANDOMIZE)  in arch_pick_mmap_layout()
454 		mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;  in arch_pick_mmap_layout()
455 		mm->get_unmapped_area = arch_get_unmapped_area;  in arch_pick_mmap_layout()
457 		mm->mmap_base = mmap_base(random_factor, rlim_stack);  in arch_pick_mmap_layout()
458 		mm->get_unmapped_area = arch_get_unmapped_area_topdown;  in arch_pick_mmap_layout()
464 	mm->mmap_base = TASK_UNMAPPED_BASE;  in arch_pick_mmap_layout()
465 	mm->get_unmapped_area = arch_get_unmapped_area;  in arch_pick_mmap_layout()
470  * __account_locked_vm - account locked pages to an mm's locked_vm
482  * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
492 	locked_vm = mm->locked_vm;  in __account_locked_vm()
497 				ret = -ENOMEM;  in __account_locked_vm()
500 			mm->locked_vm = locked_vm + pages;  in __account_locked_vm()
503 		mm->locked_vm = locked_vm - pages;  in __account_locked_vm()
506 	pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,  in __account_locked_vm()
507 		 (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,  in __account_locked_vm()
509 		 ret ? " - exceeded" : "");  in __account_locked_vm()
516  * account_locked_vm - account locked pages to an mm's locked_vm
521  * Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
525  * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
548 	struct mm_struct *mm = current->mm;  in vm_mmap_pgoff()
555 			return -EINTR;  in vm_mmap_pgoff()
571 		return -EINVAL;  in vm_mmap()
573 		return -EINVAL;  in vm_mmap()
580  * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
581  * failure, fall back to non-contiguous (vmalloc) allocation.
583  * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
604 	 * However make sure that larger requests are not too disruptive - no  in kvmalloc_node()
626 	/* non-sleeping allocations are not supported by vmalloc */  in kvmalloc_node()
649  * kvfree() - Free memory.
656  * Context: Either preemptible task context or not-NMI interrupt.
668  * kvfree_sensitive - Free a data object containing sensitive information.
701  * __vmalloc_array - allocate memory for a virtually contiguous array.
717  * vmalloc_array - allocate memory for a virtually contiguous array.
728  * __vcalloc - allocate and zero memory for a virtually contiguous array.
740  * vcalloc - allocate and zero memory for a virtually contiguous array.
752 	unsigned long mapping = (unsigned long)folio->mapping;  in folio_anon_vma()
756 	return (void *)(mapping - PAGE_MAPPING_ANON);  in folio_anon_vma()
760  * folio_mapping - Find the mapping where this folio is stored.
780 		return swap_address_space(folio->swap);  in folio_mapping()
782 	mapping = folio->mapping;  in folio_mapping()
791  * folio_copy - Copy the contents of one folio to another.
797  * Can be called in atomic context for order-0 folios, but if the folio is
840 	int new_policy = -1;  in overcommit_policy_handler()
858 		if (ret || new_policy == -1)  in overcommit_policy_handler()
891 		allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);  in vm_commit_limit()
893 		allowed = ((totalram_pages() - hugetlb_total_pages())  in vm_commit_limit()
909  * as a guest. On Hyper-V, the host implements a policy engine for dynamically
928  * succeed and -ENOMEM implies there is not.
931  * vm.overcommit_memory sysctl.  See Documentation/mm/overcommit-accounting.rst
964 		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);  in __vm_enough_memory()
970 		long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);  in __vm_enough_memory()
972 		allowed -= min_t(long, mm->total_vm / 32, reserve);  in __vm_enough_memory()
979 			    __func__, current->pid, current->comm);  in __vm_enough_memory()
982 	return -ENOMEM;  in __vm_enough_memory()
986  * get_cmdline() - copy the cmdline value to a buffer.
1003 	if (!mm->arg_end)  in get_cmdline()
1006 	spin_lock(&mm->arg_lock);  in get_cmdline()
1007 	arg_start = mm->arg_start;  in get_cmdline()
1008 	arg_end = mm->arg_end;  in get_cmdline()
1009 	env_start = mm->env_start;  in get_cmdline()
1010 	env_end = mm->env_end;  in get_cmdline()
1011 	spin_unlock(&mm->arg_lock);  in get_cmdline()
1013 	len = arg_end - arg_start;  in get_cmdline()
1024 	if (res > 0 && buffer[res-1] != '\0' && len < buflen) {  in get_cmdline()
1029 			len = env_end - env_start;  in get_cmdline()
1030 			if (len > buflen - res)  in get_cmdline()
1031 				len = buflen - res;  in get_cmdline()
1059  * mem_dump_obj - Print available provenance information
1065  * For example, for a slab-cache object, the slab name is printed, and,
1082 		type = "non-slab/vmalloc memory";  in mem_dump_obj()
1086 		type = "zero-size pointer";  in mem_dump_obj()
1088 		type = "non-paged memory";  in mem_dump_obj()
1096  * A driver might set a page logically offline -- PageOffline() -- and
1100  * Some special PFN walkers -- i.e., /proc/kcore -- read content of random