444  * @addr: start of the VM area to unmap
445  * @end: end of the VM area to unmap (non-inclusive)
618  * @addr: start of the VM area to map
619  * @end: end of the VM area to map (non-inclusive)
764  * find a lowest match of free area.
1111  * free area is inserted. If VA has been merged, it is
1159 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1190 			/* Point to the new merged area. */  in __merge_or_add_vmap_area()
1485  * Returns a start address of the newly allocated area, if success.
1562 	 * when fit type of free area is NE_FIT_TYPE. It guarantees that  in preload_this_cpu_lock()
1759 		 * Finally insert or merge lazily-freed area. It is  in __purge_vmap_area_lazy()
1813  * Free a vmap area, caller ensuring that the area has been unmapped,
1845  * Free and unmap a vmap area
1917 #define VMAP_RAM		0x1 /* indicates vm_map_ram area*/
2460  * vm_area_add_early - add vmap area early during boot
2463  * This function is used to add fixed kernel vm area to vmlist before
2486  * vm_area_register_early - register vmap area early during boot
2490  * This function is used to register kernel vm area before
2593 	struct vm_struct *area;  in __get_vm_area_node()  local
2605 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);  in __get_vm_area_node()
2606 	if (unlikely(!area))  in __get_vm_area_node()
2614 		kfree(area);  in __get_vm_area_node()
2618 	setup_vmalloc_vm(area, va, flags, caller);  in __get_vm_area_node()
2629 		area->addr = kasan_unpoison_vmalloc(area->addr, requested_size,  in __get_vm_area_node()
2632 	return area;  in __get_vm_area_node()
2644  * get_vm_area - reserve a contiguous kernel virtual area
2645  * @size:	 size of the area
2648  * Search an area of @size in the kernel virtual mapping area,
2649  * and reserved it for out purposes.  Returns the area descriptor
2652  * Return: the area descriptor on success or %NULL on failure.
2671  * find_vm_area - find a continuous kernel virtual area
2674  * Search for the kernel VM area starting at @addr, and return it.
2678  * Return: the area descriptor on success or %NULL on failure.
2692  * remove_vm_area - find and remove a continuous kernel virtual area
2695  * Search for the kernel VM area starting at @addr, and remove it.
2696  * This function returns the found VM area, but using it is NOT safe
2699  * Return: the area descriptor on success or %NULL on failure.
2726 static inline void set_area_direct_map(const struct vm_struct *area,  in set_area_direct_map()  argument
2732 	for (i = 0; i < area->nr_pages; i++)  in set_area_direct_map()
2733 		if (page_address(area->pages[i]))  in set_area_direct_map()
2734 			set_direct_map(area->pages[i]);  in set_area_direct_map()
2740 static void vm_reset_perms(struct vm_struct *area)  in vm_reset_perms()  argument
2743 	unsigned int page_order = vm_area_page_order(area);  in vm_reset_perms()
2751 	for (i = 0; i < area->nr_pages; i += 1U << page_order) {  in vm_reset_perms()
2752 		unsigned long addr = (unsigned long)page_address(area->pages[i]);  in vm_reset_perms()
2769 	set_area_direct_map(area, set_direct_map_invalid_noflush);  in vm_reset_perms()
2771 	set_area_direct_map(area, set_direct_map_default_noflush);  in vm_reset_perms()
2811  * Free the virtually continuous memory area starting at @addr, as obtained
2843 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",  in vfree()
2874  * Free the virtually contiguous memory area starting at @addr,
2890 		WARN(1, KERN_ERR "Trying to vunmap() nonexistent vm area (%p)\n",  in vunmap()
2911  * Return: the address of the area or %NULL on failure
2916 	struct vm_struct *area;  in vmap()  local
2936 	area = get_vm_area_caller(size, flags, __builtin_return_address(0));  in vmap()
2937 	if (!area)  in vmap()
2940 	addr = (unsigned long)area->addr;  in vmap()
2943 		vunmap(area->addr);  in vmap()
2948 		area->pages = pages;  in vmap()
2949 		area->nr_pages = count;  in vmap()
2951 	return area->addr;  in vmap()
2990 	struct vm_struct *area;  in vmap_pfn()  local
2992 	area = get_vm_area_caller(count * PAGE_SIZE, VM_IOREMAP,  in vmap_pfn()
2994 	if (!area)  in vmap_pfn()
2996 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,  in vmap_pfn()
2998 		free_vm_area(area);  in vmap_pfn()
3002 	flush_cache_vmap((unsigned long)area->addr,  in vmap_pfn()
3003 			 (unsigned long)area->addr + count * PAGE_SIZE);  in vmap_pfn()
3005 	return area->addr;  in vmap_pfn()
3112 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,  in __vmalloc_area_node()  argument
3118 	unsigned long addr = (unsigned long)area->addr;  in __vmalloc_area_node()
3119 	unsigned long size = get_vm_area_size(area);  in __vmalloc_area_node()
3133 		area->pages = __vmalloc_node(array_size, 1, nested_gfp, node,  in __vmalloc_area_node()
3134 					area->caller);  in __vmalloc_area_node()
3136 		area->pages = kmalloc_node(array_size, nested_gfp, node);  in __vmalloc_area_node()
3139 	if (!area->pages) {  in __vmalloc_area_node()
3143 		free_vm_area(area);  in __vmalloc_area_node()
3147 	set_vm_area_page_order(area, page_shift - PAGE_SHIFT);  in __vmalloc_area_node()
3148 	page_order = vm_area_page_order(area);  in __vmalloc_area_node()
3150 	area->nr_pages = vm_area_alloc_pages(gfp_mask | __GFP_NOWARN,  in __vmalloc_area_node()
3151 		node, page_order, nr_small_pages, area->pages);  in __vmalloc_area_node()
3153 	atomic_long_add(area->nr_pages, &nr_vmalloc_pages);  in __vmalloc_area_node()
3157 		for (i = 0; i < area->nr_pages; i++)  in __vmalloc_area_node()
3158 			mod_memcg_page_state(area->pages[i], MEMCG_VMALLOC, 1);  in __vmalloc_area_node()
3165 	if (area->nr_pages != nr_small_pages) {  in __vmalloc_area_node()
3179 				area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3193 		ret = vmap_pages_range(addr, addr + size, prot, area->pages,  in __vmalloc_area_node()
3207 			area->nr_pages * PAGE_SIZE);  in __vmalloc_area_node()
3211 	return area->addr;  in __vmalloc_area_node()
3214 	vfree(area->addr);  in __vmalloc_area_node()
3222  * @start:		  vm area range start
3223  * @end:		  vm area range end
3226  * @vm_flags:		  additional vm area flags (e.g. %VM_NO_GUARD)
3244  * Return: the address of the area or %NULL on failure
3251 	struct vm_struct *area;  in __vmalloc_node_range()  local
3291 	area = __get_vm_area_node(real_size, align, shift, VM_ALLOC |  in __vmalloc_node_range()
3294 	if (!area) {  in __vmalloc_node_range()
3331 	ret = __vmalloc_area_node(area, gfp_mask, prot, shift, node);  in __vmalloc_node_range()
3348 	area->addr = kasan_unpoison_vmalloc(area->addr, real_size, kasan_flags);  in __vmalloc_node_range()
3355 	clear_vm_uninitialized_flag(area);  in __vmalloc_node_range()
3359 		kmemleak_vmalloc(area, size, gfp_mask);  in __vmalloc_node_range()
3361 	return area->addr;  in __vmalloc_node_range()
3478  * The resulting memory area is zeroed so it can be mapped to userspace
3562  * The resulting memory area is 32bit addressable and zeroed so it can be
3621 		 * To do safe access to this _mapped_ area, we need lock. But  in aligned_vread_iter()
3660 	 * If it's area created by vm_map_ram() interface directly, but  in vmap_ram_vread_iter()
3670 	 * Area is split into regions and tracked with vmap_block, read out  in vmap_ram_vread_iter()
3730  * vread_iter() - read vmalloc area in a safe way to an iterator.
3735  * This function checks that addr is a valid vmalloc'ed area, and
3736  * copy data from that area to a given buffer. If the given memory range
3738  * proper area of @buf. If there are memory holes, they'll be zero-filled.
3739  * IOREMAP area is treated as memory hole and no copy is done.
3742  * vm_struct area, returns 0. @buf should be kernel's buffer.
3745  * should know vmalloc() area is valid and can use memcpy().
3746  * This is for routines which have to access vmalloc area without
3751  * include any intersection with valid vmalloc area
3786 		 * VMAP_BLOCK indicates a sub-type of vm_map_ram area, need  in vread_iter()
3825 		else /* IOREMAP area is treated as memory hole */  in vread_iter()
3852  * @size:		size of map area
3856  * This function checks that @kaddr is a valid vmalloc'ed area,
3867 	struct vm_struct *area;  in remap_vmalloc_range_partial()  local
3879 	area = find_vm_area(kaddr);  in remap_vmalloc_range_partial()
3880 	if (!area)  in remap_vmalloc_range_partial()
3883 	if (!(area->flags & (VM_USERMAP | VM_DMA_COHERENT)))  in remap_vmalloc_range_partial()
3887 	    end_index > get_vm_area_size(area))  in remap_vmalloc_range_partial()
3917  * This function checks that addr is a valid vmalloc'ed area, and
3932 void free_vm_area(struct vm_struct *area)  in free_vm_area()  argument
3935 	ret = remove_vm_area(area->addr);  in free_vm_area()
3936 	BUG_ON(ret != area);  in free_vm_area()
3937 	kfree(area);  in free_vm_area()
3951  * Returns: vmap_area if it is found. If there is no such area
4011  * @offsets: array containing offset of each area
4012  * @sizes: array containing size of each area
4029  * base address is pulled down to fit the area. Scanning is repeated till
4041 	int area, area2, last_area, term_area;  in pcpu_get_vm_areas()  local
4047 	for (last_area = 0, area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4048 		start = offsets[area];  in pcpu_get_vm_areas()
4049 		end = start + sizes[area];  in pcpu_get_vm_areas()
4052 		BUG_ON(!IS_ALIGNED(offsets[area], align));  in pcpu_get_vm_areas()
4053 		BUG_ON(!IS_ALIGNED(sizes[area], align));  in pcpu_get_vm_areas()
4055 		/* detect the area with the highest address */  in pcpu_get_vm_areas()
4057 			last_area = area;  in pcpu_get_vm_areas()
4059 		for (area2 = area + 1; area2 < nr_vms; area2++) {  in pcpu_get_vm_areas()
4078 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4079 		vas[area] = kmem_cache_zalloc(vmap_area_cachep, GFP_KERNEL);  in pcpu_get_vm_areas()
4080 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);  in pcpu_get_vm_areas()
4081 		if (!vas[area] || !vms[area])  in pcpu_get_vm_areas()
4087 	/* start scanning - we scan from the top, begin with the last area */  in pcpu_get_vm_areas()
4088 	area = term_area = last_area;  in pcpu_get_vm_areas()
4089 	start = offsets[area];  in pcpu_get_vm_areas()
4090 	end = start + sizes[area];  in pcpu_get_vm_areas()
4115 			term_area = area;  in pcpu_get_vm_areas()
4125 			term_area = area;  in pcpu_get_vm_areas()
4130 		 * This area fits, move on to the previous one.  If  in pcpu_get_vm_areas()
4133 		area = (area + nr_vms - 1) % nr_vms;  in pcpu_get_vm_areas()
4134 		if (area == term_area)  in pcpu_get_vm_areas()
4137 		start = offsets[area];  in pcpu_get_vm_areas()
4138 		end = start + sizes[area];  in pcpu_get_vm_areas()
4143 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4146 		start = base + offsets[area];  in pcpu_get_vm_areas()
4147 		size = sizes[area];  in pcpu_get_vm_areas()
4161 		/* Allocated area. */  in pcpu_get_vm_areas()
4162 		va = vas[area];  in pcpu_get_vm_areas()
4170 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4171 		if (kasan_populate_vmalloc(vas[area]->va_start, sizes[area]))  in pcpu_get_vm_areas()
4177 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4178 		insert_vmap_area(vas[area], &vmap_area_root, &vmap_area_list);  in pcpu_get_vm_areas()
4180 		setup_vmalloc_vm_locked(vms[area], vas[area], VM_ALLOC,  in pcpu_get_vm_areas()
4191 	for (area = 0; area < nr_vms; area++)  in pcpu_get_vm_areas()
4192 		vms[area]->addr = kasan_unpoison_vmalloc(vms[area]->addr,  in pcpu_get_vm_areas()
4193 				vms[area]->size, KASAN_VMALLOC_PROT_NORMAL);  in pcpu_get_vm_areas()
4205 	while (area--) {  in pcpu_get_vm_areas()
4206 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4207 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4208 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4213 		vas[area] = NULL;  in pcpu_get_vm_areas()
4223 		for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4224 			if (vas[area])  in pcpu_get_vm_areas()
4227 			vas[area] = kmem_cache_zalloc(  in pcpu_get_vm_areas()
4229 			if (!vas[area])  in pcpu_get_vm_areas()
4237 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4238 		if (vas[area])  in pcpu_get_vm_areas()
4239 			kmem_cache_free(vmap_area_cachep, vas[area]);  in pcpu_get_vm_areas()
4241 		kfree(vms[area]);  in pcpu_get_vm_areas()
4255 	for (area = 0; area < nr_vms; area++) {  in pcpu_get_vm_areas()
4256 		orig_start = vas[area]->va_start;  in pcpu_get_vm_areas()
4257 		orig_end = vas[area]->va_end;  in pcpu_get_vm_areas()
4258 		va = merge_or_add_vmap_area_augment(vas[area], &free_vmap_area_root,  in pcpu_get_vm_areas()
4263 		vas[area] = NULL;  in pcpu_get_vm_areas()
4264 		kfree(vms[area]);  in pcpu_get_vm_areas()