xref: /openbmc/linux/mm/vmalloc.c (revision a06c488d)
1 /*
2  *  linux/mm/vmalloc.c
3  *
4  *  Copyright (C) 1993  Linus Torvalds
5  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
8  *  Numa awareness, Christoph Lameter, SGI, June 2005
9  */
10 
11 #include <linux/vmalloc.h>
12 #include <linux/mm.h>
13 #include <linux/module.h>
14 #include <linux/highmem.h>
15 #include <linux/sched.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/interrupt.h>
19 #include <linux/proc_fs.h>
20 #include <linux/seq_file.h>
21 #include <linux/debugobjects.h>
22 #include <linux/kallsyms.h>
23 #include <linux/list.h>
24 #include <linux/rbtree.h>
25 #include <linux/radix-tree.h>
26 #include <linux/rcupdate.h>
27 #include <linux/pfn.h>
28 #include <linux/kmemleak.h>
29 #include <linux/atomic.h>
30 #include <linux/compiler.h>
31 #include <linux/llist.h>
32 #include <linux/bitops.h>
33 
34 #include <asm/uaccess.h>
35 #include <asm/tlbflush.h>
36 #include <asm/shmparam.h>
37 
38 #include "internal.h"
39 
40 struct vfree_deferred {
41 	struct llist_head list;
42 	struct work_struct wq;
43 };
44 static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
45 
46 static void __vunmap(const void *, int);
47 
48 static void free_work(struct work_struct *w)
49 {
50 	struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
51 	struct llist_node *llnode = llist_del_all(&p->list);
52 	while (llnode) {
53 		void *p = llnode;
54 		llnode = llist_next(llnode);
55 		__vunmap(p, 1);
56 	}
57 }
58 
59 /*** Page table manipulation functions ***/
60 
61 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
62 {
63 	pte_t *pte;
64 
65 	pte = pte_offset_kernel(pmd, addr);
66 	do {
67 		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
68 		WARN_ON(!pte_none(ptent) && !pte_present(ptent));
69 	} while (pte++, addr += PAGE_SIZE, addr != end);
70 }
71 
72 static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
73 {
74 	pmd_t *pmd;
75 	unsigned long next;
76 
77 	pmd = pmd_offset(pud, addr);
78 	do {
79 		next = pmd_addr_end(addr, end);
80 		if (pmd_clear_huge(pmd))
81 			continue;
82 		if (pmd_none_or_clear_bad(pmd))
83 			continue;
84 		vunmap_pte_range(pmd, addr, next);
85 	} while (pmd++, addr = next, addr != end);
86 }
87 
88 static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
89 {
90 	pud_t *pud;
91 	unsigned long next;
92 
93 	pud = pud_offset(pgd, addr);
94 	do {
95 		next = pud_addr_end(addr, end);
96 		if (pud_clear_huge(pud))
97 			continue;
98 		if (pud_none_or_clear_bad(pud))
99 			continue;
100 		vunmap_pmd_range(pud, addr, next);
101 	} while (pud++, addr = next, addr != end);
102 }
103 
104 static void vunmap_page_range(unsigned long addr, unsigned long end)
105 {
106 	pgd_t *pgd;
107 	unsigned long next;
108 
109 	BUG_ON(addr >= end);
110 	pgd = pgd_offset_k(addr);
111 	do {
112 		next = pgd_addr_end(addr, end);
113 		if (pgd_none_or_clear_bad(pgd))
114 			continue;
115 		vunmap_pud_range(pgd, addr, next);
116 	} while (pgd++, addr = next, addr != end);
117 }
118 
119 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
120 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
121 {
122 	pte_t *pte;
123 
124 	/*
125 	 * nr is a running index into the array which helps higher level
126 	 * callers keep track of where we're up to.
127 	 */
128 
129 	pte = pte_alloc_kernel(pmd, addr);
130 	if (!pte)
131 		return -ENOMEM;
132 	do {
133 		struct page *page = pages[*nr];
134 
135 		if (WARN_ON(!pte_none(*pte)))
136 			return -EBUSY;
137 		if (WARN_ON(!page))
138 			return -ENOMEM;
139 		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
140 		(*nr)++;
141 	} while (pte++, addr += PAGE_SIZE, addr != end);
142 	return 0;
143 }
144 
145 static int vmap_pmd_range(pud_t *pud, unsigned long addr,
146 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
147 {
148 	pmd_t *pmd;
149 	unsigned long next;
150 
151 	pmd = pmd_alloc(&init_mm, pud, addr);
152 	if (!pmd)
153 		return -ENOMEM;
154 	do {
155 		next = pmd_addr_end(addr, end);
156 		if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
157 			return -ENOMEM;
158 	} while (pmd++, addr = next, addr != end);
159 	return 0;
160 }
161 
162 static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
163 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)
164 {
165 	pud_t *pud;
166 	unsigned long next;
167 
168 	pud = pud_alloc(&init_mm, pgd, addr);
169 	if (!pud)
170 		return -ENOMEM;
171 	do {
172 		next = pud_addr_end(addr, end);
173 		if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
174 			return -ENOMEM;
175 	} while (pud++, addr = next, addr != end);
176 	return 0;
177 }
178 
179 /*
180  * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
181  * will have pfns corresponding to the "pages" array.
182  *
183  * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
184  */
185 static int vmap_page_range_noflush(unsigned long start, unsigned long end,
186 				   pgprot_t prot, struct page **pages)
187 {
188 	pgd_t *pgd;
189 	unsigned long next;
190 	unsigned long addr = start;
191 	int err = 0;
192 	int nr = 0;
193 
194 	BUG_ON(addr >= end);
195 	pgd = pgd_offset_k(addr);
196 	do {
197 		next = pgd_addr_end(addr, end);
198 		err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
199 		if (err)
200 			return err;
201 	} while (pgd++, addr = next, addr != end);
202 
203 	return nr;
204 }
205 
206 static int vmap_page_range(unsigned long start, unsigned long end,
207 			   pgprot_t prot, struct page **pages)
208 {
209 	int ret;
210 
211 	ret = vmap_page_range_noflush(start, end, prot, pages);
212 	flush_cache_vmap(start, end);
213 	return ret;
214 }
215 
216 int is_vmalloc_or_module_addr(const void *x)
217 {
218 	/*
219 	 * ARM, x86-64 and sparc64 put modules in a special place,
220 	 * and fall back on vmalloc() if that fails. Others
221 	 * just put it in the vmalloc space.
222 	 */
223 #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
224 	unsigned long addr = (unsigned long)x;
225 	if (addr >= MODULES_VADDR && addr < MODULES_END)
226 		return 1;
227 #endif
228 	return is_vmalloc_addr(x);
229 }
230 
231 /*
232  * Walk a vmap address to the struct page it maps.
233  */
234 struct page *vmalloc_to_page(const void *vmalloc_addr)
235 {
236 	unsigned long addr = (unsigned long) vmalloc_addr;
237 	struct page *page = NULL;
238 	pgd_t *pgd = pgd_offset_k(addr);
239 
240 	/*
241 	 * XXX we might need to change this if we add VIRTUAL_BUG_ON for
242 	 * architectures that do not vmalloc module space
243 	 */
244 	VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
245 
246 	if (!pgd_none(*pgd)) {
247 		pud_t *pud = pud_offset(pgd, addr);
248 		if (!pud_none(*pud)) {
249 			pmd_t *pmd = pmd_offset(pud, addr);
250 			if (!pmd_none(*pmd)) {
251 				pte_t *ptep, pte;
252 
253 				ptep = pte_offset_map(pmd, addr);
254 				pte = *ptep;
255 				if (pte_present(pte))
256 					page = pte_page(pte);
257 				pte_unmap(ptep);
258 			}
259 		}
260 	}
261 	return page;
262 }
263 EXPORT_SYMBOL(vmalloc_to_page);
264 
265 /*
266  * Map a vmalloc()-space virtual address to the physical page frame number.
267  */
268 unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
269 {
270 	return page_to_pfn(vmalloc_to_page(vmalloc_addr));
271 }
272 EXPORT_SYMBOL(vmalloc_to_pfn);
273 
274 
275 /*** Global kva allocator ***/
276 
277 #define VM_LAZY_FREE	0x01
278 #define VM_LAZY_FREEING	0x02
279 #define VM_VM_AREA	0x04
280 
281 static DEFINE_SPINLOCK(vmap_area_lock);
282 /* Export for kexec only */
283 LIST_HEAD(vmap_area_list);
284 static struct rb_root vmap_area_root = RB_ROOT;
285 
286 /* The vmap cache globals are protected by vmap_area_lock */
287 static struct rb_node *free_vmap_cache;
288 static unsigned long cached_hole_size;
289 static unsigned long cached_vstart;
290 static unsigned long cached_align;
291 
292 static unsigned long vmap_area_pcpu_hole;
293 
294 static struct vmap_area *__find_vmap_area(unsigned long addr)
295 {
296 	struct rb_node *n = vmap_area_root.rb_node;
297 
298 	while (n) {
299 		struct vmap_area *va;
300 
301 		va = rb_entry(n, struct vmap_area, rb_node);
302 		if (addr < va->va_start)
303 			n = n->rb_left;
304 		else if (addr >= va->va_end)
305 			n = n->rb_right;
306 		else
307 			return va;
308 	}
309 
310 	return NULL;
311 }
312 
313 static void __insert_vmap_area(struct vmap_area *va)
314 {
315 	struct rb_node **p = &vmap_area_root.rb_node;
316 	struct rb_node *parent = NULL;
317 	struct rb_node *tmp;
318 
319 	while (*p) {
320 		struct vmap_area *tmp_va;
321 
322 		parent = *p;
323 		tmp_va = rb_entry(parent, struct vmap_area, rb_node);
324 		if (va->va_start < tmp_va->va_end)
325 			p = &(*p)->rb_left;
326 		else if (va->va_end > tmp_va->va_start)
327 			p = &(*p)->rb_right;
328 		else
329 			BUG();
330 	}
331 
332 	rb_link_node(&va->rb_node, parent, p);
333 	rb_insert_color(&va->rb_node, &vmap_area_root);
334 
335 	/* address-sort this list */
336 	tmp = rb_prev(&va->rb_node);
337 	if (tmp) {
338 		struct vmap_area *prev;
339 		prev = rb_entry(tmp, struct vmap_area, rb_node);
340 		list_add_rcu(&va->list, &prev->list);
341 	} else
342 		list_add_rcu(&va->list, &vmap_area_list);
343 }
344 
345 static void purge_vmap_area_lazy(void);
346 
347 /*
348  * Allocate a region of KVA of the specified size and alignment, within the
349  * vstart and vend.
350  */
351 static struct vmap_area *alloc_vmap_area(unsigned long size,
352 				unsigned long align,
353 				unsigned long vstart, unsigned long vend,
354 				int node, gfp_t gfp_mask)
355 {
356 	struct vmap_area *va;
357 	struct rb_node *n;
358 	unsigned long addr;
359 	int purged = 0;
360 	struct vmap_area *first;
361 
362 	BUG_ON(!size);
363 	BUG_ON(offset_in_page(size));
364 	BUG_ON(!is_power_of_2(align));
365 
366 	va = kmalloc_node(sizeof(struct vmap_area),
367 			gfp_mask & GFP_RECLAIM_MASK, node);
368 	if (unlikely(!va))
369 		return ERR_PTR(-ENOMEM);
370 
371 	/*
372 	 * Only scan the relevant parts containing pointers to other objects
373 	 * to avoid false negatives.
374 	 */
375 	kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
376 
377 retry:
378 	spin_lock(&vmap_area_lock);
379 	/*
380 	 * Invalidate cache if we have more permissive parameters.
381 	 * cached_hole_size notes the largest hole noticed _below_
382 	 * the vmap_area cached in free_vmap_cache: if size fits
383 	 * into that hole, we want to scan from vstart to reuse
384 	 * the hole instead of allocating above free_vmap_cache.
385 	 * Note that __free_vmap_area may update free_vmap_cache
386 	 * without updating cached_hole_size or cached_align.
387 	 */
388 	if (!free_vmap_cache ||
389 			size < cached_hole_size ||
390 			vstart < cached_vstart ||
391 			align < cached_align) {
392 nocache:
393 		cached_hole_size = 0;
394 		free_vmap_cache = NULL;
395 	}
396 	/* record if we encounter less permissive parameters */
397 	cached_vstart = vstart;
398 	cached_align = align;
399 
400 	/* find starting point for our search */
401 	if (free_vmap_cache) {
402 		first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
403 		addr = ALIGN(first->va_end, align);
404 		if (addr < vstart)
405 			goto nocache;
406 		if (addr + size < addr)
407 			goto overflow;
408 
409 	} else {
410 		addr = ALIGN(vstart, align);
411 		if (addr + size < addr)
412 			goto overflow;
413 
414 		n = vmap_area_root.rb_node;
415 		first = NULL;
416 
417 		while (n) {
418 			struct vmap_area *tmp;
419 			tmp = rb_entry(n, struct vmap_area, rb_node);
420 			if (tmp->va_end >= addr) {
421 				first = tmp;
422 				if (tmp->va_start <= addr)
423 					break;
424 				n = n->rb_left;
425 			} else
426 				n = n->rb_right;
427 		}
428 
429 		if (!first)
430 			goto found;
431 	}
432 
433 	/* from the starting point, walk areas until a suitable hole is found */
434 	while (addr + size > first->va_start && addr + size <= vend) {
435 		if (addr + cached_hole_size < first->va_start)
436 			cached_hole_size = first->va_start - addr;
437 		addr = ALIGN(first->va_end, align);
438 		if (addr + size < addr)
439 			goto overflow;
440 
441 		if (list_is_last(&first->list, &vmap_area_list))
442 			goto found;
443 
444 		first = list_next_entry(first, list);
445 	}
446 
447 found:
448 	if (addr + size > vend)
449 		goto overflow;
450 
451 	va->va_start = addr;
452 	va->va_end = addr + size;
453 	va->flags = 0;
454 	__insert_vmap_area(va);
455 	free_vmap_cache = &va->rb_node;
456 	spin_unlock(&vmap_area_lock);
457 
458 	BUG_ON(!IS_ALIGNED(va->va_start, align));
459 	BUG_ON(va->va_start < vstart);
460 	BUG_ON(va->va_end > vend);
461 
462 	return va;
463 
464 overflow:
465 	spin_unlock(&vmap_area_lock);
466 	if (!purged) {
467 		purge_vmap_area_lazy();
468 		purged = 1;
469 		goto retry;
470 	}
471 	if (printk_ratelimit())
472 		pr_warn("vmap allocation for size %lu failed: "
473 			"use vmalloc=<size> to increase size.\n", size);
474 	kfree(va);
475 	return ERR_PTR(-EBUSY);
476 }
477 
478 static void __free_vmap_area(struct vmap_area *va)
479 {
480 	BUG_ON(RB_EMPTY_NODE(&va->rb_node));
481 
482 	if (free_vmap_cache) {
483 		if (va->va_end < cached_vstart) {
484 			free_vmap_cache = NULL;
485 		} else {
486 			struct vmap_area *cache;
487 			cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
488 			if (va->va_start <= cache->va_start) {
489 				free_vmap_cache = rb_prev(&va->rb_node);
490 				/*
491 				 * We don't try to update cached_hole_size or
492 				 * cached_align, but it won't go very wrong.
493 				 */
494 			}
495 		}
496 	}
497 	rb_erase(&va->rb_node, &vmap_area_root);
498 	RB_CLEAR_NODE(&va->rb_node);
499 	list_del_rcu(&va->list);
500 
501 	/*
502 	 * Track the highest possible candidate for pcpu area
503 	 * allocation.  Areas outside of vmalloc area can be returned
504 	 * here too, consider only end addresses which fall inside
505 	 * vmalloc area proper.
506 	 */
507 	if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
508 		vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
509 
510 	kfree_rcu(va, rcu_head);
511 }
512 
513 /*
514  * Free a region of KVA allocated by alloc_vmap_area
515  */
516 static void free_vmap_area(struct vmap_area *va)
517 {
518 	spin_lock(&vmap_area_lock);
519 	__free_vmap_area(va);
520 	spin_unlock(&vmap_area_lock);
521 }
522 
523 /*
524  * Clear the pagetable entries of a given vmap_area
525  */
526 static void unmap_vmap_area(struct vmap_area *va)
527 {
528 	vunmap_page_range(va->va_start, va->va_end);
529 }
530 
531 static void vmap_debug_free_range(unsigned long start, unsigned long end)
532 {
533 	/*
534 	 * Unmap page tables and force a TLB flush immediately if
535 	 * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
536 	 * bugs similarly to those in linear kernel virtual address
537 	 * space after a page has been freed.
538 	 *
539 	 * All the lazy freeing logic is still retained, in order to
540 	 * minimise intrusiveness of this debugging feature.
541 	 *
542 	 * This is going to be *slow* (linear kernel virtual address
543 	 * debugging doesn't do a broadcast TLB flush so it is a lot
544 	 * faster).
545 	 */
546 #ifdef CONFIG_DEBUG_PAGEALLOC
547 	vunmap_page_range(start, end);
548 	flush_tlb_kernel_range(start, end);
549 #endif
550 }
551 
552 /*
553  * lazy_max_pages is the maximum amount of virtual address space we gather up
554  * before attempting to purge with a TLB flush.
555  *
556  * There is a tradeoff here: a larger number will cover more kernel page tables
557  * and take slightly longer to purge, but it will linearly reduce the number of
558  * global TLB flushes that must be performed. It would seem natural to scale
559  * this number up linearly with the number of CPUs (because vmapping activity
560  * could also scale linearly with the number of CPUs), however it is likely
561  * that in practice, workloads might be constrained in other ways that mean
562  * vmap activity will not scale linearly with CPUs. Also, I want to be
563  * conservative and not introduce a big latency on huge systems, so go with
564  * a less aggressive log scale. It will still be an improvement over the old
565  * code, and it will be simple to change the scale factor if we find that it
566  * becomes a problem on bigger systems.
567  */
568 static unsigned long lazy_max_pages(void)
569 {
570 	unsigned int log;
571 
572 	log = fls(num_online_cpus());
573 
574 	return log * (32UL * 1024 * 1024 / PAGE_SIZE);
575 }
576 
577 static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
578 
579 /* for per-CPU blocks */
580 static void purge_fragmented_blocks_allcpus(void);
581 
582 /*
583  * called before a call to iounmap() if the caller wants vm_area_struct's
584  * immediately freed.
585  */
586 void set_iounmap_nonlazy(void)
587 {
588 	atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
589 }
590 
591 /*
592  * Purges all lazily-freed vmap areas.
593  *
594  * If sync is 0 then don't purge if there is already a purge in progress.
595  * If force_flush is 1, then flush kernel TLBs between *start and *end even
596  * if we found no lazy vmap areas to unmap (callers can use this to optimise
597  * their own TLB flushing).
598  * Returns with *start = min(*start, lowest purged address)
599  *              *end = max(*end, highest purged address)
600  */
601 static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
602 					int sync, int force_flush)
603 {
604 	static DEFINE_SPINLOCK(purge_lock);
605 	LIST_HEAD(valist);
606 	struct vmap_area *va;
607 	struct vmap_area *n_va;
608 	int nr = 0;
609 
610 	/*
611 	 * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
612 	 * should not expect such behaviour. This just simplifies locking for
613 	 * the case that isn't actually used at the moment anyway.
614 	 */
615 	if (!sync && !force_flush) {
616 		if (!spin_trylock(&purge_lock))
617 			return;
618 	} else
619 		spin_lock(&purge_lock);
620 
621 	if (sync)
622 		purge_fragmented_blocks_allcpus();
623 
624 	rcu_read_lock();
625 	list_for_each_entry_rcu(va, &vmap_area_list, list) {
626 		if (va->flags & VM_LAZY_FREE) {
627 			if (va->va_start < *start)
628 				*start = va->va_start;
629 			if (va->va_end > *end)
630 				*end = va->va_end;
631 			nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
632 			list_add_tail(&va->purge_list, &valist);
633 			va->flags |= VM_LAZY_FREEING;
634 			va->flags &= ~VM_LAZY_FREE;
635 		}
636 	}
637 	rcu_read_unlock();
638 
639 	if (nr)
640 		atomic_sub(nr, &vmap_lazy_nr);
641 
642 	if (nr || force_flush)
643 		flush_tlb_kernel_range(*start, *end);
644 
645 	if (nr) {
646 		spin_lock(&vmap_area_lock);
647 		list_for_each_entry_safe(va, n_va, &valist, purge_list)
648 			__free_vmap_area(va);
649 		spin_unlock(&vmap_area_lock);
650 	}
651 	spin_unlock(&purge_lock);
652 }
653 
654 /*
655  * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
656  * is already purging.
657  */
658 static void try_purge_vmap_area_lazy(void)
659 {
660 	unsigned long start = ULONG_MAX, end = 0;
661 
662 	__purge_vmap_area_lazy(&start, &end, 0, 0);
663 }
664 
665 /*
666  * Kick off a purge of the outstanding lazy areas.
667  */
668 static void purge_vmap_area_lazy(void)
669 {
670 	unsigned long start = ULONG_MAX, end = 0;
671 
672 	__purge_vmap_area_lazy(&start, &end, 1, 0);
673 }
674 
675 /*
676  * Free a vmap area, caller ensuring that the area has been unmapped
677  * and flush_cache_vunmap had been called for the correct range
678  * previously.
679  */
680 static void free_vmap_area_noflush(struct vmap_area *va)
681 {
682 	va->flags |= VM_LAZY_FREE;
683 	atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
684 	if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
685 		try_purge_vmap_area_lazy();
686 }
687 
688 /*
689  * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
690  * called for the correct range previously.
691  */
692 static void free_unmap_vmap_area_noflush(struct vmap_area *va)
693 {
694 	unmap_vmap_area(va);
695 	free_vmap_area_noflush(va);
696 }
697 
698 /*
699  * Free and unmap a vmap area
700  */
701 static void free_unmap_vmap_area(struct vmap_area *va)
702 {
703 	flush_cache_vunmap(va->va_start, va->va_end);
704 	free_unmap_vmap_area_noflush(va);
705 }
706 
707 static struct vmap_area *find_vmap_area(unsigned long addr)
708 {
709 	struct vmap_area *va;
710 
711 	spin_lock(&vmap_area_lock);
712 	va = __find_vmap_area(addr);
713 	spin_unlock(&vmap_area_lock);
714 
715 	return va;
716 }
717 
718 static void free_unmap_vmap_area_addr(unsigned long addr)
719 {
720 	struct vmap_area *va;
721 
722 	va = find_vmap_area(addr);
723 	BUG_ON(!va);
724 	free_unmap_vmap_area(va);
725 }
726 
727 
728 /*** Per cpu kva allocator ***/
729 
730 /*
731  * vmap space is limited especially on 32 bit architectures. Ensure there is
732  * room for at least 16 percpu vmap blocks per CPU.
733  */
734 /*
735  * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
736  * to #define VMALLOC_SPACE		(VMALLOC_END-VMALLOC_START). Guess
737  * instead (we just need a rough idea)
738  */
739 #if BITS_PER_LONG == 32
740 #define VMALLOC_SPACE		(128UL*1024*1024)
741 #else
742 #define VMALLOC_SPACE		(128UL*1024*1024*1024)
743 #endif
744 
745 #define VMALLOC_PAGES		(VMALLOC_SPACE / PAGE_SIZE)
746 #define VMAP_MAX_ALLOC		BITS_PER_LONG	/* 256K with 4K pages */
747 #define VMAP_BBMAP_BITS_MAX	1024	/* 4MB with 4K pages */
748 #define VMAP_BBMAP_BITS_MIN	(VMAP_MAX_ALLOC*2)
749 #define VMAP_MIN(x, y)		((x) < (y) ? (x) : (y)) /* can't use min() */
750 #define VMAP_MAX(x, y)		((x) > (y) ? (x) : (y)) /* can't use max() */
751 #define VMAP_BBMAP_BITS		\
752 		VMAP_MIN(VMAP_BBMAP_BITS_MAX,	\
753 		VMAP_MAX(VMAP_BBMAP_BITS_MIN,	\
754 			VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
755 
756 #define VMAP_BLOCK_SIZE		(VMAP_BBMAP_BITS * PAGE_SIZE)
757 
758 static bool vmap_initialized __read_mostly = false;
759 
760 struct vmap_block_queue {
761 	spinlock_t lock;
762 	struct list_head free;
763 };
764 
765 struct vmap_block {
766 	spinlock_t lock;
767 	struct vmap_area *va;
768 	unsigned long free, dirty;
769 	unsigned long dirty_min, dirty_max; /*< dirty range */
770 	struct list_head free_list;
771 	struct rcu_head rcu_head;
772 	struct list_head purge;
773 };
774 
775 /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
776 static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
777 
778 /*
779  * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
780  * in the free path. Could get rid of this if we change the API to return a
781  * "cookie" from alloc, to be passed to free. But no big deal yet.
782  */
783 static DEFINE_SPINLOCK(vmap_block_tree_lock);
784 static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
785 
786 /*
787  * We should probably have a fallback mechanism to allocate virtual memory
788  * out of partially filled vmap blocks. However vmap block sizing should be
789  * fairly reasonable according to the vmalloc size, so it shouldn't be a
790  * big problem.
791  */
792 
793 static unsigned long addr_to_vb_idx(unsigned long addr)
794 {
795 	addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
796 	addr /= VMAP_BLOCK_SIZE;
797 	return addr;
798 }
799 
800 static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
801 {
802 	unsigned long addr;
803 
804 	addr = va_start + (pages_off << PAGE_SHIFT);
805 	BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
806 	return (void *)addr;
807 }
808 
809 /**
810  * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
811  *                  block. Of course pages number can't exceed VMAP_BBMAP_BITS
812  * @order:    how many 2^order pages should be occupied in newly allocated block
813  * @gfp_mask: flags for the page level allocator
814  *
815  * Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
816  */
817 static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
818 {
819 	struct vmap_block_queue *vbq;
820 	struct vmap_block *vb;
821 	struct vmap_area *va;
822 	unsigned long vb_idx;
823 	int node, err;
824 	void *vaddr;
825 
826 	node = numa_node_id();
827 
828 	vb = kmalloc_node(sizeof(struct vmap_block),
829 			gfp_mask & GFP_RECLAIM_MASK, node);
830 	if (unlikely(!vb))
831 		return ERR_PTR(-ENOMEM);
832 
833 	va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
834 					VMALLOC_START, VMALLOC_END,
835 					node, gfp_mask);
836 	if (IS_ERR(va)) {
837 		kfree(vb);
838 		return ERR_CAST(va);
839 	}
840 
841 	err = radix_tree_preload(gfp_mask);
842 	if (unlikely(err)) {
843 		kfree(vb);
844 		free_vmap_area(va);
845 		return ERR_PTR(err);
846 	}
847 
848 	vaddr = vmap_block_vaddr(va->va_start, 0);
849 	spin_lock_init(&vb->lock);
850 	vb->va = va;
851 	/* At least something should be left free */
852 	BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
853 	vb->free = VMAP_BBMAP_BITS - (1UL << order);
854 	vb->dirty = 0;
855 	vb->dirty_min = VMAP_BBMAP_BITS;
856 	vb->dirty_max = 0;
857 	INIT_LIST_HEAD(&vb->free_list);
858 
859 	vb_idx = addr_to_vb_idx(va->va_start);
860 	spin_lock(&vmap_block_tree_lock);
861 	err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
862 	spin_unlock(&vmap_block_tree_lock);
863 	BUG_ON(err);
864 	radix_tree_preload_end();
865 
866 	vbq = &get_cpu_var(vmap_block_queue);
867 	spin_lock(&vbq->lock);
868 	list_add_tail_rcu(&vb->free_list, &vbq->free);
869 	spin_unlock(&vbq->lock);
870 	put_cpu_var(vmap_block_queue);
871 
872 	return vaddr;
873 }
874 
875 static void free_vmap_block(struct vmap_block *vb)
876 {
877 	struct vmap_block *tmp;
878 	unsigned long vb_idx;
879 
880 	vb_idx = addr_to_vb_idx(vb->va->va_start);
881 	spin_lock(&vmap_block_tree_lock);
882 	tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
883 	spin_unlock(&vmap_block_tree_lock);
884 	BUG_ON(tmp != vb);
885 
886 	free_vmap_area_noflush(vb->va);
887 	kfree_rcu(vb, rcu_head);
888 }
889 
890 static void purge_fragmented_blocks(int cpu)
891 {
892 	LIST_HEAD(purge);
893 	struct vmap_block *vb;
894 	struct vmap_block *n_vb;
895 	struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
896 
897 	rcu_read_lock();
898 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
899 
900 		if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
901 			continue;
902 
903 		spin_lock(&vb->lock);
904 		if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
905 			vb->free = 0; /* prevent further allocs after releasing lock */
906 			vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
907 			vb->dirty_min = 0;
908 			vb->dirty_max = VMAP_BBMAP_BITS;
909 			spin_lock(&vbq->lock);
910 			list_del_rcu(&vb->free_list);
911 			spin_unlock(&vbq->lock);
912 			spin_unlock(&vb->lock);
913 			list_add_tail(&vb->purge, &purge);
914 		} else
915 			spin_unlock(&vb->lock);
916 	}
917 	rcu_read_unlock();
918 
919 	list_for_each_entry_safe(vb, n_vb, &purge, purge) {
920 		list_del(&vb->purge);
921 		free_vmap_block(vb);
922 	}
923 }
924 
925 static void purge_fragmented_blocks_allcpus(void)
926 {
927 	int cpu;
928 
929 	for_each_possible_cpu(cpu)
930 		purge_fragmented_blocks(cpu);
931 }
932 
933 static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
934 {
935 	struct vmap_block_queue *vbq;
936 	struct vmap_block *vb;
937 	void *vaddr = NULL;
938 	unsigned int order;
939 
940 	BUG_ON(offset_in_page(size));
941 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
942 	if (WARN_ON(size == 0)) {
943 		/*
944 		 * Allocating 0 bytes isn't what caller wants since
945 		 * get_order(0) returns funny result. Just warn and terminate
946 		 * early.
947 		 */
948 		return NULL;
949 	}
950 	order = get_order(size);
951 
952 	rcu_read_lock();
953 	vbq = &get_cpu_var(vmap_block_queue);
954 	list_for_each_entry_rcu(vb, &vbq->free, free_list) {
955 		unsigned long pages_off;
956 
957 		spin_lock(&vb->lock);
958 		if (vb->free < (1UL << order)) {
959 			spin_unlock(&vb->lock);
960 			continue;
961 		}
962 
963 		pages_off = VMAP_BBMAP_BITS - vb->free;
964 		vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
965 		vb->free -= 1UL << order;
966 		if (vb->free == 0) {
967 			spin_lock(&vbq->lock);
968 			list_del_rcu(&vb->free_list);
969 			spin_unlock(&vbq->lock);
970 		}
971 
972 		spin_unlock(&vb->lock);
973 		break;
974 	}
975 
976 	put_cpu_var(vmap_block_queue);
977 	rcu_read_unlock();
978 
979 	/* Allocate new block if nothing was found */
980 	if (!vaddr)
981 		vaddr = new_vmap_block(order, gfp_mask);
982 
983 	return vaddr;
984 }
985 
986 static void vb_free(const void *addr, unsigned long size)
987 {
988 	unsigned long offset;
989 	unsigned long vb_idx;
990 	unsigned int order;
991 	struct vmap_block *vb;
992 
993 	BUG_ON(offset_in_page(size));
994 	BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
995 
996 	flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
997 
998 	order = get_order(size);
999 
1000 	offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
1001 	offset >>= PAGE_SHIFT;
1002 
1003 	vb_idx = addr_to_vb_idx((unsigned long)addr);
1004 	rcu_read_lock();
1005 	vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
1006 	rcu_read_unlock();
1007 	BUG_ON(!vb);
1008 
1009 	vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
1010 
1011 	spin_lock(&vb->lock);
1012 
1013 	/* Expand dirty range */
1014 	vb->dirty_min = min(vb->dirty_min, offset);
1015 	vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
1016 
1017 	vb->dirty += 1UL << order;
1018 	if (vb->dirty == VMAP_BBMAP_BITS) {
1019 		BUG_ON(vb->free);
1020 		spin_unlock(&vb->lock);
1021 		free_vmap_block(vb);
1022 	} else
1023 		spin_unlock(&vb->lock);
1024 }
1025 
1026 /**
1027  * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
1028  *
1029  * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
1030  * to amortize TLB flushing overheads. What this means is that any page you
1031  * have now, may, in a former life, have been mapped into kernel virtual
1032  * address by the vmap layer and so there might be some CPUs with TLB entries
1033  * still referencing that page (additional to the regular 1:1 kernel mapping).
1034  *
1035  * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
1036  * be sure that none of the pages we have control over will have any aliases
1037  * from the vmap layer.
1038  */
1039 void vm_unmap_aliases(void)
1040 {
1041 	unsigned long start = ULONG_MAX, end = 0;
1042 	int cpu;
1043 	int flush = 0;
1044 
1045 	if (unlikely(!vmap_initialized))
1046 		return;
1047 
1048 	for_each_possible_cpu(cpu) {
1049 		struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
1050 		struct vmap_block *vb;
1051 
1052 		rcu_read_lock();
1053 		list_for_each_entry_rcu(vb, &vbq->free, free_list) {
1054 			spin_lock(&vb->lock);
1055 			if (vb->dirty) {
1056 				unsigned long va_start = vb->va->va_start;
1057 				unsigned long s, e;
1058 
1059 				s = va_start + (vb->dirty_min << PAGE_SHIFT);
1060 				e = va_start + (vb->dirty_max << PAGE_SHIFT);
1061 
1062 				start = min(s, start);
1063 				end   = max(e, end);
1064 
1065 				flush = 1;
1066 			}
1067 			spin_unlock(&vb->lock);
1068 		}
1069 		rcu_read_unlock();
1070 	}
1071 
1072 	__purge_vmap_area_lazy(&start, &end, 1, flush);
1073 }
1074 EXPORT_SYMBOL_GPL(vm_unmap_aliases);
1075 
1076 /**
1077  * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
1078  * @mem: the pointer returned by vm_map_ram
1079  * @count: the count passed to that vm_map_ram call (cannot unmap partial)
1080  */
1081 void vm_unmap_ram(const void *mem, unsigned int count)
1082 {
1083 	unsigned long size = count << PAGE_SHIFT;
1084 	unsigned long addr = (unsigned long)mem;
1085 
1086 	BUG_ON(!addr);
1087 	BUG_ON(addr < VMALLOC_START);
1088 	BUG_ON(addr > VMALLOC_END);
1089 	BUG_ON(!IS_ALIGNED(addr, PAGE_SIZE));
1090 
1091 	debug_check_no_locks_freed(mem, size);
1092 	vmap_debug_free_range(addr, addr+size);
1093 
1094 	if (likely(count <= VMAP_MAX_ALLOC))
1095 		vb_free(mem, size);
1096 	else
1097 		free_unmap_vmap_area_addr(addr);
1098 }
1099 EXPORT_SYMBOL(vm_unmap_ram);
1100 
1101 /**
1102  * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
1103  * @pages: an array of pointers to the pages to be mapped
1104  * @count: number of pages
1105  * @node: prefer to allocate data structures on this node
1106  * @prot: memory protection to use. PAGE_KERNEL for regular RAM
1107  *
1108  * If you use this function for less than VMAP_MAX_ALLOC pages, it could be
1109  * faster than vmap so it's good.  But if you mix long-life and short-life
1110  * objects with vm_map_ram(), it could consume lots of address space through
1111  * fragmentation (especially on a 32bit machine).  You could see failures in
1112  * the end.  Please use this function for short-lived objects.
1113  *
1114  * Returns: a pointer to the address that has been mapped, or %NULL on failure
1115  */
1116 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
1117 {
1118 	unsigned long size = count << PAGE_SHIFT;
1119 	unsigned long addr;
1120 	void *mem;
1121 
1122 	if (likely(count <= VMAP_MAX_ALLOC)) {
1123 		mem = vb_alloc(size, GFP_KERNEL);
1124 		if (IS_ERR(mem))
1125 			return NULL;
1126 		addr = (unsigned long)mem;
1127 	} else {
1128 		struct vmap_area *va;
1129 		va = alloc_vmap_area(size, PAGE_SIZE,
1130 				VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
1131 		if (IS_ERR(va))
1132 			return NULL;
1133 
1134 		addr = va->va_start;
1135 		mem = (void *)addr;
1136 	}
1137 	if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
1138 		vm_unmap_ram(mem, count);
1139 		return NULL;
1140 	}
1141 	return mem;
1142 }
1143 EXPORT_SYMBOL(vm_map_ram);
1144 
1145 static struct vm_struct *vmlist __initdata;
1146 /**
1147  * vm_area_add_early - add vmap area early during boot
1148  * @vm: vm_struct to add
1149  *
1150  * This function is used to add fixed kernel vm area to vmlist before
1151  * vmalloc_init() is called.  @vm->addr, @vm->size, and @vm->flags
1152  * should contain proper values and the other fields should be zero.
1153  *
1154  * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1155  */
1156 void __init vm_area_add_early(struct vm_struct *vm)
1157 {
1158 	struct vm_struct *tmp, **p;
1159 
1160 	BUG_ON(vmap_initialized);
1161 	for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
1162 		if (tmp->addr >= vm->addr) {
1163 			BUG_ON(tmp->addr < vm->addr + vm->size);
1164 			break;
1165 		} else
1166 			BUG_ON(tmp->addr + tmp->size > vm->addr);
1167 	}
1168 	vm->next = *p;
1169 	*p = vm;
1170 }
1171 
1172 /**
1173  * vm_area_register_early - register vmap area early during boot
1174  * @vm: vm_struct to register
1175  * @align: requested alignment
1176  *
1177  * This function is used to register kernel vm area before
1178  * vmalloc_init() is called.  @vm->size and @vm->flags should contain
1179  * proper values on entry and other fields should be zero.  On return,
1180  * vm->addr contains the allocated address.
1181  *
1182  * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
1183  */
1184 void __init vm_area_register_early(struct vm_struct *vm, size_t align)
1185 {
1186 	static size_t vm_init_off __initdata;
1187 	unsigned long addr;
1188 
1189 	addr = ALIGN(VMALLOC_START + vm_init_off, align);
1190 	vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
1191 
1192 	vm->addr = (void *)addr;
1193 
1194 	vm_area_add_early(vm);
1195 }
1196 
1197 void __init vmalloc_init(void)
1198 {
1199 	struct vmap_area *va;
1200 	struct vm_struct *tmp;
1201 	int i;
1202 
1203 	for_each_possible_cpu(i) {
1204 		struct vmap_block_queue *vbq;
1205 		struct vfree_deferred *p;
1206 
1207 		vbq = &per_cpu(vmap_block_queue, i);
1208 		spin_lock_init(&vbq->lock);
1209 		INIT_LIST_HEAD(&vbq->free);
1210 		p = &per_cpu(vfree_deferred, i);
1211 		init_llist_head(&p->list);
1212 		INIT_WORK(&p->wq, free_work);
1213 	}
1214 
1215 	/* Import existing vmlist entries. */
1216 	for (tmp = vmlist; tmp; tmp = tmp->next) {
1217 		va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
1218 		va->flags = VM_VM_AREA;
1219 		va->va_start = (unsigned long)tmp->addr;
1220 		va->va_end = va->va_start + tmp->size;
1221 		va->vm = tmp;
1222 		__insert_vmap_area(va);
1223 	}
1224 
1225 	vmap_area_pcpu_hole = VMALLOC_END;
1226 
1227 	vmap_initialized = true;
1228 }
1229 
1230 /**
1231  * map_kernel_range_noflush - map kernel VM area with the specified pages
1232  * @addr: start of the VM area to map
1233  * @size: size of the VM area to map
1234  * @prot: page protection flags to use
1235  * @pages: pages to map
1236  *
1237  * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size
1238  * specify should have been allocated using get_vm_area() and its
1239  * friends.
1240  *
1241  * NOTE:
1242  * This function does NOT do any cache flushing.  The caller is
1243  * responsible for calling flush_cache_vmap() on to-be-mapped areas
1244  * before calling this function.
1245  *
1246  * RETURNS:
1247  * The number of pages mapped on success, -errno on failure.
1248  */
1249 int map_kernel_range_noflush(unsigned long addr, unsigned long size,
1250 			     pgprot_t prot, struct page **pages)
1251 {
1252 	return vmap_page_range_noflush(addr, addr + size, prot, pages);
1253 }
1254 
1255 /**
1256  * unmap_kernel_range_noflush - unmap kernel VM area
1257  * @addr: start of the VM area to unmap
1258  * @size: size of the VM area to unmap
1259  *
1260  * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size
1261  * specify should have been allocated using get_vm_area() and its
1262  * friends.
1263  *
1264  * NOTE:
1265  * This function does NOT do any cache flushing.  The caller is
1266  * responsible for calling flush_cache_vunmap() on to-be-mapped areas
1267  * before calling this function and flush_tlb_kernel_range() after.
1268  */
1269 void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
1270 {
1271 	vunmap_page_range(addr, addr + size);
1272 }
1273 EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
1274 
1275 /**
1276  * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
1277  * @addr: start of the VM area to unmap
1278  * @size: size of the VM area to unmap
1279  *
1280  * Similar to unmap_kernel_range_noflush() but flushes vcache before
1281  * the unmapping and tlb after.
1282  */
1283 void unmap_kernel_range(unsigned long addr, unsigned long size)
1284 {
1285 	unsigned long end = addr + size;
1286 
1287 	flush_cache_vunmap(addr, end);
1288 	vunmap_page_range(addr, end);
1289 	flush_tlb_kernel_range(addr, end);
1290 }
1291 EXPORT_SYMBOL_GPL(unmap_kernel_range);
1292 
1293 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages)
1294 {
1295 	unsigned long addr = (unsigned long)area->addr;
1296 	unsigned long end = addr + get_vm_area_size(area);
1297 	int err;
1298 
1299 	err = vmap_page_range(addr, end, prot, pages);
1300 
1301 	return err > 0 ? 0 : err;
1302 }
1303 EXPORT_SYMBOL_GPL(map_vm_area);
1304 
1305 static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
1306 			      unsigned long flags, const void *caller)
1307 {
1308 	spin_lock(&vmap_area_lock);
1309 	vm->flags = flags;
1310 	vm->addr = (void *)va->va_start;
1311 	vm->size = va->va_end - va->va_start;
1312 	vm->caller = caller;
1313 	va->vm = vm;
1314 	va->flags |= VM_VM_AREA;
1315 	spin_unlock(&vmap_area_lock);
1316 }
1317 
1318 static void clear_vm_uninitialized_flag(struct vm_struct *vm)
1319 {
1320 	/*
1321 	 * Before removing VM_UNINITIALIZED,
1322 	 * we should make sure that vm has proper values.
1323 	 * Pair with smp_rmb() in show_numa_info().
1324 	 */
1325 	smp_wmb();
1326 	vm->flags &= ~VM_UNINITIALIZED;
1327 }
1328 
1329 static struct vm_struct *__get_vm_area_node(unsigned long size,
1330 		unsigned long align, unsigned long flags, unsigned long start,
1331 		unsigned long end, int node, gfp_t gfp_mask, const void *caller)
1332 {
1333 	struct vmap_area *va;
1334 	struct vm_struct *area;
1335 
1336 	BUG_ON(in_interrupt());
1337 	if (flags & VM_IOREMAP)
1338 		align = 1ul << clamp_t(int, fls_long(size),
1339 				       PAGE_SHIFT, IOREMAP_MAX_ORDER);
1340 
1341 	size = PAGE_ALIGN(size);
1342 	if (unlikely(!size))
1343 		return NULL;
1344 
1345 	area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
1346 	if (unlikely(!area))
1347 		return NULL;
1348 
1349 	if (!(flags & VM_NO_GUARD))
1350 		size += PAGE_SIZE;
1351 
1352 	va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
1353 	if (IS_ERR(va)) {
1354 		kfree(area);
1355 		return NULL;
1356 	}
1357 
1358 	setup_vmalloc_vm(area, va, flags, caller);
1359 
1360 	return area;
1361 }
1362 
1363 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
1364 				unsigned long start, unsigned long end)
1365 {
1366 	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1367 				  GFP_KERNEL, __builtin_return_address(0));
1368 }
1369 EXPORT_SYMBOL_GPL(__get_vm_area);
1370 
1371 struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
1372 				       unsigned long start, unsigned long end,
1373 				       const void *caller)
1374 {
1375 	return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
1376 				  GFP_KERNEL, caller);
1377 }
1378 
1379 /**
1380  *	get_vm_area  -  reserve a contiguous kernel virtual area
1381  *	@size:		size of the area
1382  *	@flags:		%VM_IOREMAP for I/O mappings or VM_ALLOC
1383  *
1384  *	Search an area of @size in the kernel virtual mapping area,
1385  *	and reserved it for out purposes.  Returns the area descriptor
1386  *	on success or %NULL on failure.
1387  */
1388 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
1389 {
1390 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
1391 				  NUMA_NO_NODE, GFP_KERNEL,
1392 				  __builtin_return_address(0));
1393 }
1394 
1395 struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
1396 				const void *caller)
1397 {
1398 	return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
1399 				  NUMA_NO_NODE, GFP_KERNEL, caller);
1400 }
1401 
1402 /**
1403  *	find_vm_area  -  find a continuous kernel virtual area
1404  *	@addr:		base address
1405  *
1406  *	Search for the kernel VM area starting at @addr, and return it.
1407  *	It is up to the caller to do all required locking to keep the returned
1408  *	pointer valid.
1409  */
1410 struct vm_struct *find_vm_area(const void *addr)
1411 {
1412 	struct vmap_area *va;
1413 
1414 	va = find_vmap_area((unsigned long)addr);
1415 	if (va && va->flags & VM_VM_AREA)
1416 		return va->vm;
1417 
1418 	return NULL;
1419 }
1420 
1421 /**
1422  *	remove_vm_area  -  find and remove a continuous kernel virtual area
1423  *	@addr:		base address
1424  *
1425  *	Search for the kernel VM area starting at @addr, and remove it.
1426  *	This function returns the found VM area, but using it is NOT safe
1427  *	on SMP machines, except for its size or flags.
1428  */
1429 struct vm_struct *remove_vm_area(const void *addr)
1430 {
1431 	struct vmap_area *va;
1432 
1433 	va = find_vmap_area((unsigned long)addr);
1434 	if (va && va->flags & VM_VM_AREA) {
1435 		struct vm_struct *vm = va->vm;
1436 
1437 		spin_lock(&vmap_area_lock);
1438 		va->vm = NULL;
1439 		va->flags &= ~VM_VM_AREA;
1440 		spin_unlock(&vmap_area_lock);
1441 
1442 		vmap_debug_free_range(va->va_start, va->va_end);
1443 		kasan_free_shadow(vm);
1444 		free_unmap_vmap_area(va);
1445 
1446 		return vm;
1447 	}
1448 	return NULL;
1449 }
1450 
1451 static void __vunmap(const void *addr, int deallocate_pages)
1452 {
1453 	struct vm_struct *area;
1454 
1455 	if (!addr)
1456 		return;
1457 
1458 	if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
1459 			addr))
1460 		return;
1461 
1462 	area = remove_vm_area(addr);
1463 	if (unlikely(!area)) {
1464 		WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
1465 				addr);
1466 		return;
1467 	}
1468 
1469 	debug_check_no_locks_freed(addr, get_vm_area_size(area));
1470 	debug_check_no_obj_freed(addr, get_vm_area_size(area));
1471 
1472 	if (deallocate_pages) {
1473 		int i;
1474 
1475 		for (i = 0; i < area->nr_pages; i++) {
1476 			struct page *page = area->pages[i];
1477 
1478 			BUG_ON(!page);
1479 			__free_kmem_pages(page, 0);
1480 		}
1481 
1482 		kvfree(area->pages);
1483 	}
1484 
1485 	kfree(area);
1486 	return;
1487 }
1488 
1489 /**
1490  *	vfree  -  release memory allocated by vmalloc()
1491  *	@addr:		memory base address
1492  *
1493  *	Free the virtually continuous memory area starting at @addr, as
1494  *	obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
1495  *	NULL, no operation is performed.
1496  *
1497  *	Must not be called in NMI context (strictly speaking, only if we don't
1498  *	have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
1499  *	conventions for vfree() arch-depenedent would be a really bad idea)
1500  *
1501  *	NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
1502  */
1503 void vfree(const void *addr)
1504 {
1505 	BUG_ON(in_nmi());
1506 
1507 	kmemleak_free(addr);
1508 
1509 	if (!addr)
1510 		return;
1511 	if (unlikely(in_interrupt())) {
1512 		struct vfree_deferred *p = this_cpu_ptr(&vfree_deferred);
1513 		if (llist_add((struct llist_node *)addr, &p->list))
1514 			schedule_work(&p->wq);
1515 	} else
1516 		__vunmap(addr, 1);
1517 }
1518 EXPORT_SYMBOL(vfree);
1519 
1520 /**
1521  *	vunmap  -  release virtual mapping obtained by vmap()
1522  *	@addr:		memory base address
1523  *
1524  *	Free the virtually contiguous memory area starting at @addr,
1525  *	which was created from the page array passed to vmap().
1526  *
1527  *	Must not be called in interrupt context.
1528  */
1529 void vunmap(const void *addr)
1530 {
1531 	BUG_ON(in_interrupt());
1532 	might_sleep();
1533 	if (addr)
1534 		__vunmap(addr, 0);
1535 }
1536 EXPORT_SYMBOL(vunmap);
1537 
1538 /**
1539  *	vmap  -  map an array of pages into virtually contiguous space
1540  *	@pages:		array of page pointers
1541  *	@count:		number of pages to map
1542  *	@flags:		vm_area->flags
1543  *	@prot:		page protection for the mapping
1544  *
1545  *	Maps @count pages from @pages into contiguous kernel virtual
1546  *	space.
1547  */
1548 void *vmap(struct page **pages, unsigned int count,
1549 		unsigned long flags, pgprot_t prot)
1550 {
1551 	struct vm_struct *area;
1552 
1553 	might_sleep();
1554 
1555 	if (count > totalram_pages)
1556 		return NULL;
1557 
1558 	area = get_vm_area_caller((count << PAGE_SHIFT), flags,
1559 					__builtin_return_address(0));
1560 	if (!area)
1561 		return NULL;
1562 
1563 	if (map_vm_area(area, prot, pages)) {
1564 		vunmap(area->addr);
1565 		return NULL;
1566 	}
1567 
1568 	return area->addr;
1569 }
1570 EXPORT_SYMBOL(vmap);
1571 
1572 static void *__vmalloc_node(unsigned long size, unsigned long align,
1573 			    gfp_t gfp_mask, pgprot_t prot,
1574 			    int node, const void *caller);
1575 static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
1576 				 pgprot_t prot, int node)
1577 {
1578 	const int order = 0;
1579 	struct page **pages;
1580 	unsigned int nr_pages, array_size, i;
1581 	const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
1582 	const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN;
1583 
1584 	nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
1585 	array_size = (nr_pages * sizeof(struct page *));
1586 
1587 	area->nr_pages = nr_pages;
1588 	/* Please note that the recursion is strictly bounded. */
1589 	if (array_size > PAGE_SIZE) {
1590 		pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
1591 				PAGE_KERNEL, node, area->caller);
1592 	} else {
1593 		pages = kmalloc_node(array_size, nested_gfp, node);
1594 	}
1595 	area->pages = pages;
1596 	if (!area->pages) {
1597 		remove_vm_area(area->addr);
1598 		kfree(area);
1599 		return NULL;
1600 	}
1601 
1602 	for (i = 0; i < area->nr_pages; i++) {
1603 		struct page *page;
1604 
1605 		if (node == NUMA_NO_NODE)
1606 			page = alloc_kmem_pages(alloc_mask, order);
1607 		else
1608 			page = alloc_kmem_pages_node(node, alloc_mask, order);
1609 
1610 		if (unlikely(!page)) {
1611 			/* Successfully allocated i pages, free them in __vunmap() */
1612 			area->nr_pages = i;
1613 			goto fail;
1614 		}
1615 		area->pages[i] = page;
1616 		if (gfpflags_allow_blocking(gfp_mask))
1617 			cond_resched();
1618 	}
1619 
1620 	if (map_vm_area(area, prot, pages))
1621 		goto fail;
1622 	return area->addr;
1623 
1624 fail:
1625 	warn_alloc_failed(gfp_mask, order,
1626 			  "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
1627 			  (area->nr_pages*PAGE_SIZE), area->size);
1628 	vfree(area->addr);
1629 	return NULL;
1630 }
1631 
1632 /**
1633  *	__vmalloc_node_range  -  allocate virtually contiguous memory
1634  *	@size:		allocation size
1635  *	@align:		desired alignment
1636  *	@start:		vm area range start
1637  *	@end:		vm area range end
1638  *	@gfp_mask:	flags for the page level allocator
1639  *	@prot:		protection mask for the allocated pages
1640  *	@vm_flags:	additional vm area flags (e.g. %VM_NO_GUARD)
1641  *	@node:		node to use for allocation or NUMA_NO_NODE
1642  *	@caller:	caller's return address
1643  *
1644  *	Allocate enough pages to cover @size from the page level
1645  *	allocator with @gfp_mask flags.  Map them into contiguous
1646  *	kernel virtual space, using a pagetable protection of @prot.
1647  */
1648 void *__vmalloc_node_range(unsigned long size, unsigned long align,
1649 			unsigned long start, unsigned long end, gfp_t gfp_mask,
1650 			pgprot_t prot, unsigned long vm_flags, int node,
1651 			const void *caller)
1652 {
1653 	struct vm_struct *area;
1654 	void *addr;
1655 	unsigned long real_size = size;
1656 
1657 	size = PAGE_ALIGN(size);
1658 	if (!size || (size >> PAGE_SHIFT) > totalram_pages)
1659 		goto fail;
1660 
1661 	area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |
1662 				vm_flags, start, end, node, gfp_mask, caller);
1663 	if (!area)
1664 		goto fail;
1665 
1666 	addr = __vmalloc_area_node(area, gfp_mask, prot, node);
1667 	if (!addr)
1668 		return NULL;
1669 
1670 	/*
1671 	 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
1672 	 * flag. It means that vm_struct is not fully initialized.
1673 	 * Now, it is fully initialized, so remove this flag here.
1674 	 */
1675 	clear_vm_uninitialized_flag(area);
1676 
1677 	/*
1678 	 * A ref_count = 2 is needed because vm_struct allocated in
1679 	 * __get_vm_area_node() contains a reference to the virtual address of
1680 	 * the vmalloc'ed block.
1681 	 */
1682 	kmemleak_alloc(addr, real_size, 2, gfp_mask);
1683 
1684 	return addr;
1685 
1686 fail:
1687 	warn_alloc_failed(gfp_mask, 0,
1688 			  "vmalloc: allocation failure: %lu bytes\n",
1689 			  real_size);
1690 	return NULL;
1691 }
1692 
1693 /**
1694  *	__vmalloc_node  -  allocate virtually contiguous memory
1695  *	@size:		allocation size
1696  *	@align:		desired alignment
1697  *	@gfp_mask:	flags for the page level allocator
1698  *	@prot:		protection mask for the allocated pages
1699  *	@node:		node to use for allocation or NUMA_NO_NODE
1700  *	@caller:	caller's return address
1701  *
1702  *	Allocate enough pages to cover @size from the page level
1703  *	allocator with @gfp_mask flags.  Map them into contiguous
1704  *	kernel virtual space, using a pagetable protection of @prot.
1705  */
1706 static void *__vmalloc_node(unsigned long size, unsigned long align,
1707 			    gfp_t gfp_mask, pgprot_t prot,
1708 			    int node, const void *caller)
1709 {
1710 	return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
1711 				gfp_mask, prot, 0, node, caller);
1712 }
1713 
1714 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
1715 {
1716 	return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
1717 				__builtin_return_address(0));
1718 }
1719 EXPORT_SYMBOL(__vmalloc);
1720 
1721 static inline void *__vmalloc_node_flags(unsigned long size,
1722 					int node, gfp_t flags)
1723 {
1724 	return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
1725 					node, __builtin_return_address(0));
1726 }
1727 
1728 /**
1729  *	vmalloc  -  allocate virtually contiguous memory
1730  *	@size:		allocation size
1731  *	Allocate enough pages to cover @size from the page level
1732  *	allocator and map them into contiguous kernel virtual space.
1733  *
1734  *	For tight control over page level allocator and protection flags
1735  *	use __vmalloc() instead.
1736  */
1737 void *vmalloc(unsigned long size)
1738 {
1739 	return __vmalloc_node_flags(size, NUMA_NO_NODE,
1740 				    GFP_KERNEL | __GFP_HIGHMEM);
1741 }
1742 EXPORT_SYMBOL(vmalloc);
1743 
1744 /**
1745  *	vzalloc - allocate virtually contiguous memory with zero fill
1746  *	@size:	allocation size
1747  *	Allocate enough pages to cover @size from the page level
1748  *	allocator and map them into contiguous kernel virtual space.
1749  *	The memory allocated is set to zero.
1750  *
1751  *	For tight control over page level allocator and protection flags
1752  *	use __vmalloc() instead.
1753  */
1754 void *vzalloc(unsigned long size)
1755 {
1756 	return __vmalloc_node_flags(size, NUMA_NO_NODE,
1757 				GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1758 }
1759 EXPORT_SYMBOL(vzalloc);
1760 
1761 /**
1762  * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
1763  * @size: allocation size
1764  *
1765  * The resulting memory area is zeroed so it can be mapped to userspace
1766  * without leaking data.
1767  */
1768 void *vmalloc_user(unsigned long size)
1769 {
1770 	struct vm_struct *area;
1771 	void *ret;
1772 
1773 	ret = __vmalloc_node(size, SHMLBA,
1774 			     GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
1775 			     PAGE_KERNEL, NUMA_NO_NODE,
1776 			     __builtin_return_address(0));
1777 	if (ret) {
1778 		area = find_vm_area(ret);
1779 		area->flags |= VM_USERMAP;
1780 	}
1781 	return ret;
1782 }
1783 EXPORT_SYMBOL(vmalloc_user);
1784 
1785 /**
1786  *	vmalloc_node  -  allocate memory on a specific node
1787  *	@size:		allocation size
1788  *	@node:		numa node
1789  *
1790  *	Allocate enough pages to cover @size from the page level
1791  *	allocator and map them into contiguous kernel virtual space.
1792  *
1793  *	For tight control over page level allocator and protection flags
1794  *	use __vmalloc() instead.
1795  */
1796 void *vmalloc_node(unsigned long size, int node)
1797 {
1798 	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
1799 					node, __builtin_return_address(0));
1800 }
1801 EXPORT_SYMBOL(vmalloc_node);
1802 
1803 /**
1804  * vzalloc_node - allocate memory on a specific node with zero fill
1805  * @size:	allocation size
1806  * @node:	numa node
1807  *
1808  * Allocate enough pages to cover @size from the page level
1809  * allocator and map them into contiguous kernel virtual space.
1810  * The memory allocated is set to zero.
1811  *
1812  * For tight control over page level allocator and protection flags
1813  * use __vmalloc_node() instead.
1814  */
1815 void *vzalloc_node(unsigned long size, int node)
1816 {
1817 	return __vmalloc_node_flags(size, node,
1818 			 GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
1819 }
1820 EXPORT_SYMBOL(vzalloc_node);
1821 
1822 #ifndef PAGE_KERNEL_EXEC
1823 # define PAGE_KERNEL_EXEC PAGE_KERNEL
1824 #endif
1825 
1826 /**
1827  *	vmalloc_exec  -  allocate virtually contiguous, executable memory
1828  *	@size:		allocation size
1829  *
1830  *	Kernel-internal function to allocate enough pages to cover @size
1831  *	the page level allocator and map them into contiguous and
1832  *	executable kernel virtual space.
1833  *
1834  *	For tight control over page level allocator and protection flags
1835  *	use __vmalloc() instead.
1836  */
1837 
1838 void *vmalloc_exec(unsigned long size)
1839 {
1840 	return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
1841 			      NUMA_NO_NODE, __builtin_return_address(0));
1842 }
1843 
1844 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
1845 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
1846 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
1847 #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
1848 #else
1849 #define GFP_VMALLOC32 GFP_KERNEL
1850 #endif
1851 
1852 /**
1853  *	vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
1854  *	@size:		allocation size
1855  *
1856  *	Allocate enough 32bit PA addressable pages to cover @size from the
1857  *	page level allocator and map them into contiguous kernel virtual space.
1858  */
1859 void *vmalloc_32(unsigned long size)
1860 {
1861 	return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
1862 			      NUMA_NO_NODE, __builtin_return_address(0));
1863 }
1864 EXPORT_SYMBOL(vmalloc_32);
1865 
1866 /**
1867  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
1868  *	@size:		allocation size
1869  *
1870  * The resulting memory area is 32bit addressable and zeroed so it can be
1871  * mapped to userspace without leaking data.
1872  */
1873 void *vmalloc_32_user(unsigned long size)
1874 {
1875 	struct vm_struct *area;
1876 	void *ret;
1877 
1878 	ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
1879 			     NUMA_NO_NODE, __builtin_return_address(0));
1880 	if (ret) {
1881 		area = find_vm_area(ret);
1882 		area->flags |= VM_USERMAP;
1883 	}
1884 	return ret;
1885 }
1886 EXPORT_SYMBOL(vmalloc_32_user);
1887 
1888 /*
1889  * small helper routine , copy contents to buf from addr.
1890  * If the page is not present, fill zero.
1891  */
1892 
1893 static int aligned_vread(char *buf, char *addr, unsigned long count)
1894 {
1895 	struct page *p;
1896 	int copied = 0;
1897 
1898 	while (count) {
1899 		unsigned long offset, length;
1900 
1901 		offset = offset_in_page(addr);
1902 		length = PAGE_SIZE - offset;
1903 		if (length > count)
1904 			length = count;
1905 		p = vmalloc_to_page(addr);
1906 		/*
1907 		 * To do safe access to this _mapped_ area, we need
1908 		 * lock. But adding lock here means that we need to add
1909 		 * overhead of vmalloc()/vfree() calles for this _debug_
1910 		 * interface, rarely used. Instead of that, we'll use
1911 		 * kmap() and get small overhead in this access function.
1912 		 */
1913 		if (p) {
1914 			/*
1915 			 * we can expect USER0 is not used (see vread/vwrite's
1916 			 * function description)
1917 			 */
1918 			void *map = kmap_atomic(p);
1919 			memcpy(buf, map + offset, length);
1920 			kunmap_atomic(map);
1921 		} else
1922 			memset(buf, 0, length);
1923 
1924 		addr += length;
1925 		buf += length;
1926 		copied += length;
1927 		count -= length;
1928 	}
1929 	return copied;
1930 }
1931 
1932 static int aligned_vwrite(char *buf, char *addr, unsigned long count)
1933 {
1934 	struct page *p;
1935 	int copied = 0;
1936 
1937 	while (count) {
1938 		unsigned long offset, length;
1939 
1940 		offset = offset_in_page(addr);
1941 		length = PAGE_SIZE - offset;
1942 		if (length > count)
1943 			length = count;
1944 		p = vmalloc_to_page(addr);
1945 		/*
1946 		 * To do safe access to this _mapped_ area, we need
1947 		 * lock. But adding lock here means that we need to add
1948 		 * overhead of vmalloc()/vfree() calles for this _debug_
1949 		 * interface, rarely used. Instead of that, we'll use
1950 		 * kmap() and get small overhead in this access function.
1951 		 */
1952 		if (p) {
1953 			/*
1954 			 * we can expect USER0 is not used (see vread/vwrite's
1955 			 * function description)
1956 			 */
1957 			void *map = kmap_atomic(p);
1958 			memcpy(map + offset, buf, length);
1959 			kunmap_atomic(map);
1960 		}
1961 		addr += length;
1962 		buf += length;
1963 		copied += length;
1964 		count -= length;
1965 	}
1966 	return copied;
1967 }
1968 
1969 /**
1970  *	vread() -  read vmalloc area in a safe way.
1971  *	@buf:		buffer for reading data
1972  *	@addr:		vm address.
1973  *	@count:		number of bytes to be read.
1974  *
1975  *	Returns # of bytes which addr and buf should be increased.
1976  *	(same number to @count). Returns 0 if [addr...addr+count) doesn't
1977  *	includes any intersect with alive vmalloc area.
1978  *
1979  *	This function checks that addr is a valid vmalloc'ed area, and
1980  *	copy data from that area to a given buffer. If the given memory range
1981  *	of [addr...addr+count) includes some valid address, data is copied to
1982  *	proper area of @buf. If there are memory holes, they'll be zero-filled.
1983  *	IOREMAP area is treated as memory hole and no copy is done.
1984  *
1985  *	If [addr...addr+count) doesn't includes any intersects with alive
1986  *	vm_struct area, returns 0. @buf should be kernel's buffer.
1987  *
1988  *	Note: In usual ops, vread() is never necessary because the caller
1989  *	should know vmalloc() area is valid and can use memcpy().
1990  *	This is for routines which have to access vmalloc area without
1991  *	any informaion, as /dev/kmem.
1992  *
1993  */
1994 
1995 long vread(char *buf, char *addr, unsigned long count)
1996 {
1997 	struct vmap_area *va;
1998 	struct vm_struct *vm;
1999 	char *vaddr, *buf_start = buf;
2000 	unsigned long buflen = count;
2001 	unsigned long n;
2002 
2003 	/* Don't allow overflow */
2004 	if ((unsigned long) addr + count < count)
2005 		count = -(unsigned long) addr;
2006 
2007 	spin_lock(&vmap_area_lock);
2008 	list_for_each_entry(va, &vmap_area_list, list) {
2009 		if (!count)
2010 			break;
2011 
2012 		if (!(va->flags & VM_VM_AREA))
2013 			continue;
2014 
2015 		vm = va->vm;
2016 		vaddr = (char *) vm->addr;
2017 		if (addr >= vaddr + get_vm_area_size(vm))
2018 			continue;
2019 		while (addr < vaddr) {
2020 			if (count == 0)
2021 				goto finished;
2022 			*buf = '\0';
2023 			buf++;
2024 			addr++;
2025 			count--;
2026 		}
2027 		n = vaddr + get_vm_area_size(vm) - addr;
2028 		if (n > count)
2029 			n = count;
2030 		if (!(vm->flags & VM_IOREMAP))
2031 			aligned_vread(buf, addr, n);
2032 		else /* IOREMAP area is treated as memory hole */
2033 			memset(buf, 0, n);
2034 		buf += n;
2035 		addr += n;
2036 		count -= n;
2037 	}
2038 finished:
2039 	spin_unlock(&vmap_area_lock);
2040 
2041 	if (buf == buf_start)
2042 		return 0;
2043 	/* zero-fill memory holes */
2044 	if (buf != buf_start + buflen)
2045 		memset(buf, 0, buflen - (buf - buf_start));
2046 
2047 	return buflen;
2048 }
2049 
2050 /**
2051  *	vwrite() -  write vmalloc area in a safe way.
2052  *	@buf:		buffer for source data
2053  *	@addr:		vm address.
2054  *	@count:		number of bytes to be read.
2055  *
2056  *	Returns # of bytes which addr and buf should be incresed.
2057  *	(same number to @count).
2058  *	If [addr...addr+count) doesn't includes any intersect with valid
2059  *	vmalloc area, returns 0.
2060  *
2061  *	This function checks that addr is a valid vmalloc'ed area, and
2062  *	copy data from a buffer to the given addr. If specified range of
2063  *	[addr...addr+count) includes some valid address, data is copied from
2064  *	proper area of @buf. If there are memory holes, no copy to hole.
2065  *	IOREMAP area is treated as memory hole and no copy is done.
2066  *
2067  *	If [addr...addr+count) doesn't includes any intersects with alive
2068  *	vm_struct area, returns 0. @buf should be kernel's buffer.
2069  *
2070  *	Note: In usual ops, vwrite() is never necessary because the caller
2071  *	should know vmalloc() area is valid and can use memcpy().
2072  *	This is for routines which have to access vmalloc area without
2073  *	any informaion, as /dev/kmem.
2074  */
2075 
2076 long vwrite(char *buf, char *addr, unsigned long count)
2077 {
2078 	struct vmap_area *va;
2079 	struct vm_struct *vm;
2080 	char *vaddr;
2081 	unsigned long n, buflen;
2082 	int copied = 0;
2083 
2084 	/* Don't allow overflow */
2085 	if ((unsigned long) addr + count < count)
2086 		count = -(unsigned long) addr;
2087 	buflen = count;
2088 
2089 	spin_lock(&vmap_area_lock);
2090 	list_for_each_entry(va, &vmap_area_list, list) {
2091 		if (!count)
2092 			break;
2093 
2094 		if (!(va->flags & VM_VM_AREA))
2095 			continue;
2096 
2097 		vm = va->vm;
2098 		vaddr = (char *) vm->addr;
2099 		if (addr >= vaddr + get_vm_area_size(vm))
2100 			continue;
2101 		while (addr < vaddr) {
2102 			if (count == 0)
2103 				goto finished;
2104 			buf++;
2105 			addr++;
2106 			count--;
2107 		}
2108 		n = vaddr + get_vm_area_size(vm) - addr;
2109 		if (n > count)
2110 			n = count;
2111 		if (!(vm->flags & VM_IOREMAP)) {
2112 			aligned_vwrite(buf, addr, n);
2113 			copied++;
2114 		}
2115 		buf += n;
2116 		addr += n;
2117 		count -= n;
2118 	}
2119 finished:
2120 	spin_unlock(&vmap_area_lock);
2121 	if (!copied)
2122 		return 0;
2123 	return buflen;
2124 }
2125 
2126 /**
2127  *	remap_vmalloc_range_partial  -  map vmalloc pages to userspace
2128  *	@vma:		vma to cover
2129  *	@uaddr:		target user address to start at
2130  *	@kaddr:		virtual address of vmalloc kernel memory
2131  *	@size:		size of map area
2132  *
2133  *	Returns:	0 for success, -Exxx on failure
2134  *
2135  *	This function checks that @kaddr is a valid vmalloc'ed area,
2136  *	and that it is big enough to cover the range starting at
2137  *	@uaddr in @vma. Will return failure if that criteria isn't
2138  *	met.
2139  *
2140  *	Similar to remap_pfn_range() (see mm/memory.c)
2141  */
2142 int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
2143 				void *kaddr, unsigned long size)
2144 {
2145 	struct vm_struct *area;
2146 
2147 	size = PAGE_ALIGN(size);
2148 
2149 	if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
2150 		return -EINVAL;
2151 
2152 	area = find_vm_area(kaddr);
2153 	if (!area)
2154 		return -EINVAL;
2155 
2156 	if (!(area->flags & VM_USERMAP))
2157 		return -EINVAL;
2158 
2159 	if (kaddr + size > area->addr + area->size)
2160 		return -EINVAL;
2161 
2162 	do {
2163 		struct page *page = vmalloc_to_page(kaddr);
2164 		int ret;
2165 
2166 		ret = vm_insert_page(vma, uaddr, page);
2167 		if (ret)
2168 			return ret;
2169 
2170 		uaddr += PAGE_SIZE;
2171 		kaddr += PAGE_SIZE;
2172 		size -= PAGE_SIZE;
2173 	} while (size > 0);
2174 
2175 	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
2176 
2177 	return 0;
2178 }
2179 EXPORT_SYMBOL(remap_vmalloc_range_partial);
2180 
2181 /**
2182  *	remap_vmalloc_range  -  map vmalloc pages to userspace
2183  *	@vma:		vma to cover (map full range of vma)
2184  *	@addr:		vmalloc memory
2185  *	@pgoff:		number of pages into addr before first page to map
2186  *
2187  *	Returns:	0 for success, -Exxx on failure
2188  *
2189  *	This function checks that addr is a valid vmalloc'ed area, and
2190  *	that it is big enough to cover the vma. Will return failure if
2191  *	that criteria isn't met.
2192  *
2193  *	Similar to remap_pfn_range() (see mm/memory.c)
2194  */
2195 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
2196 						unsigned long pgoff)
2197 {
2198 	return remap_vmalloc_range_partial(vma, vma->vm_start,
2199 					   addr + (pgoff << PAGE_SHIFT),
2200 					   vma->vm_end - vma->vm_start);
2201 }
2202 EXPORT_SYMBOL(remap_vmalloc_range);
2203 
2204 /*
2205  * Implement a stub for vmalloc_sync_all() if the architecture chose not to
2206  * have one.
2207  */
2208 void __weak vmalloc_sync_all(void)
2209 {
2210 }
2211 
2212 
2213 static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
2214 {
2215 	pte_t ***p = data;
2216 
2217 	if (p) {
2218 		*(*p) = pte;
2219 		(*p)++;
2220 	}
2221 	return 0;
2222 }
2223 
2224 /**
2225  *	alloc_vm_area - allocate a range of kernel address space
2226  *	@size:		size of the area
2227  *	@ptes:		returns the PTEs for the address space
2228  *
2229  *	Returns:	NULL on failure, vm_struct on success
2230  *
2231  *	This function reserves a range of kernel address space, and
2232  *	allocates pagetables to map that range.  No actual mappings
2233  *	are created.
2234  *
2235  *	If @ptes is non-NULL, pointers to the PTEs (in init_mm)
2236  *	allocated for the VM area are returned.
2237  */
2238 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
2239 {
2240 	struct vm_struct *area;
2241 
2242 	area = get_vm_area_caller(size, VM_IOREMAP,
2243 				__builtin_return_address(0));
2244 	if (area == NULL)
2245 		return NULL;
2246 
2247 	/*
2248 	 * This ensures that page tables are constructed for this region
2249 	 * of kernel virtual address space and mapped into init_mm.
2250 	 */
2251 	if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
2252 				size, f, ptes ? &ptes : NULL)) {
2253 		free_vm_area(area);
2254 		return NULL;
2255 	}
2256 
2257 	return area;
2258 }
2259 EXPORT_SYMBOL_GPL(alloc_vm_area);
2260 
2261 void free_vm_area(struct vm_struct *area)
2262 {
2263 	struct vm_struct *ret;
2264 	ret = remove_vm_area(area->addr);
2265 	BUG_ON(ret != area);
2266 	kfree(area);
2267 }
2268 EXPORT_SYMBOL_GPL(free_vm_area);
2269 
2270 #ifdef CONFIG_SMP
2271 static struct vmap_area *node_to_va(struct rb_node *n)
2272 {
2273 	return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
2274 }
2275 
2276 /**
2277  * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
2278  * @end: target address
2279  * @pnext: out arg for the next vmap_area
2280  * @pprev: out arg for the previous vmap_area
2281  *
2282  * Returns: %true if either or both of next and prev are found,
2283  *	    %false if no vmap_area exists
2284  *
2285  * Find vmap_areas end addresses of which enclose @end.  ie. if not
2286  * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
2287  */
2288 static bool pvm_find_next_prev(unsigned long end,
2289 			       struct vmap_area **pnext,
2290 			       struct vmap_area **pprev)
2291 {
2292 	struct rb_node *n = vmap_area_root.rb_node;
2293 	struct vmap_area *va = NULL;
2294 
2295 	while (n) {
2296 		va = rb_entry(n, struct vmap_area, rb_node);
2297 		if (end < va->va_end)
2298 			n = n->rb_left;
2299 		else if (end > va->va_end)
2300 			n = n->rb_right;
2301 		else
2302 			break;
2303 	}
2304 
2305 	if (!va)
2306 		return false;
2307 
2308 	if (va->va_end > end) {
2309 		*pnext = va;
2310 		*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2311 	} else {
2312 		*pprev = va;
2313 		*pnext = node_to_va(rb_next(&(*pprev)->rb_node));
2314 	}
2315 	return true;
2316 }
2317 
2318 /**
2319  * pvm_determine_end - find the highest aligned address between two vmap_areas
2320  * @pnext: in/out arg for the next vmap_area
2321  * @pprev: in/out arg for the previous vmap_area
2322  * @align: alignment
2323  *
2324  * Returns: determined end address
2325  *
2326  * Find the highest aligned address between *@pnext and *@pprev below
2327  * VMALLOC_END.  *@pnext and *@pprev are adjusted so that the aligned
2328  * down address is between the end addresses of the two vmap_areas.
2329  *
2330  * Please note that the address returned by this function may fall
2331  * inside *@pnext vmap_area.  The caller is responsible for checking
2332  * that.
2333  */
2334 static unsigned long pvm_determine_end(struct vmap_area **pnext,
2335 				       struct vmap_area **pprev,
2336 				       unsigned long align)
2337 {
2338 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2339 	unsigned long addr;
2340 
2341 	if (*pnext)
2342 		addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
2343 	else
2344 		addr = vmalloc_end;
2345 
2346 	while (*pprev && (*pprev)->va_end > addr) {
2347 		*pnext = *pprev;
2348 		*pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
2349 	}
2350 
2351 	return addr;
2352 }
2353 
2354 /**
2355  * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
2356  * @offsets: array containing offset of each area
2357  * @sizes: array containing size of each area
2358  * @nr_vms: the number of areas to allocate
2359  * @align: alignment, all entries in @offsets and @sizes must be aligned to this
2360  *
2361  * Returns: kmalloc'd vm_struct pointer array pointing to allocated
2362  *	    vm_structs on success, %NULL on failure
2363  *
2364  * Percpu allocator wants to use congruent vm areas so that it can
2365  * maintain the offsets among percpu areas.  This function allocates
2366  * congruent vmalloc areas for it with GFP_KERNEL.  These areas tend to
2367  * be scattered pretty far, distance between two areas easily going up
2368  * to gigabytes.  To avoid interacting with regular vmallocs, these
2369  * areas are allocated from top.
2370  *
2371  * Despite its complicated look, this allocator is rather simple.  It
2372  * does everything top-down and scans areas from the end looking for
2373  * matching slot.  While scanning, if any of the areas overlaps with
2374  * existing vmap_area, the base address is pulled down to fit the
2375  * area.  Scanning is repeated till all the areas fit and then all
2376  * necessary data structres are inserted and the result is returned.
2377  */
2378 struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
2379 				     const size_t *sizes, int nr_vms,
2380 				     size_t align)
2381 {
2382 	const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
2383 	const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
2384 	struct vmap_area **vas, *prev, *next;
2385 	struct vm_struct **vms;
2386 	int area, area2, last_area, term_area;
2387 	unsigned long base, start, end, last_end;
2388 	bool purged = false;
2389 
2390 	/* verify parameters and allocate data structures */
2391 	BUG_ON(offset_in_page(align) || !is_power_of_2(align));
2392 	for (last_area = 0, area = 0; area < nr_vms; area++) {
2393 		start = offsets[area];
2394 		end = start + sizes[area];
2395 
2396 		/* is everything aligned properly? */
2397 		BUG_ON(!IS_ALIGNED(offsets[area], align));
2398 		BUG_ON(!IS_ALIGNED(sizes[area], align));
2399 
2400 		/* detect the area with the highest address */
2401 		if (start > offsets[last_area])
2402 			last_area = area;
2403 
2404 		for (area2 = 0; area2 < nr_vms; area2++) {
2405 			unsigned long start2 = offsets[area2];
2406 			unsigned long end2 = start2 + sizes[area2];
2407 
2408 			if (area2 == area)
2409 				continue;
2410 
2411 			BUG_ON(start2 >= start && start2 < end);
2412 			BUG_ON(end2 <= end && end2 > start);
2413 		}
2414 	}
2415 	last_end = offsets[last_area] + sizes[last_area];
2416 
2417 	if (vmalloc_end - vmalloc_start < last_end) {
2418 		WARN_ON(true);
2419 		return NULL;
2420 	}
2421 
2422 	vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
2423 	vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
2424 	if (!vas || !vms)
2425 		goto err_free2;
2426 
2427 	for (area = 0; area < nr_vms; area++) {
2428 		vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
2429 		vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
2430 		if (!vas[area] || !vms[area])
2431 			goto err_free;
2432 	}
2433 retry:
2434 	spin_lock(&vmap_area_lock);
2435 
2436 	/* start scanning - we scan from the top, begin with the last area */
2437 	area = term_area = last_area;
2438 	start = offsets[area];
2439 	end = start + sizes[area];
2440 
2441 	if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
2442 		base = vmalloc_end - last_end;
2443 		goto found;
2444 	}
2445 	base = pvm_determine_end(&next, &prev, align) - end;
2446 
2447 	while (true) {
2448 		BUG_ON(next && next->va_end <= base + end);
2449 		BUG_ON(prev && prev->va_end > base + end);
2450 
2451 		/*
2452 		 * base might have underflowed, add last_end before
2453 		 * comparing.
2454 		 */
2455 		if (base + last_end < vmalloc_start + last_end) {
2456 			spin_unlock(&vmap_area_lock);
2457 			if (!purged) {
2458 				purge_vmap_area_lazy();
2459 				purged = true;
2460 				goto retry;
2461 			}
2462 			goto err_free;
2463 		}
2464 
2465 		/*
2466 		 * If next overlaps, move base downwards so that it's
2467 		 * right below next and then recheck.
2468 		 */
2469 		if (next && next->va_start < base + end) {
2470 			base = pvm_determine_end(&next, &prev, align) - end;
2471 			term_area = area;
2472 			continue;
2473 		}
2474 
2475 		/*
2476 		 * If prev overlaps, shift down next and prev and move
2477 		 * base so that it's right below new next and then
2478 		 * recheck.
2479 		 */
2480 		if (prev && prev->va_end > base + start)  {
2481 			next = prev;
2482 			prev = node_to_va(rb_prev(&next->rb_node));
2483 			base = pvm_determine_end(&next, &prev, align) - end;
2484 			term_area = area;
2485 			continue;
2486 		}
2487 
2488 		/*
2489 		 * This area fits, move on to the previous one.  If
2490 		 * the previous one is the terminal one, we're done.
2491 		 */
2492 		area = (area + nr_vms - 1) % nr_vms;
2493 		if (area == term_area)
2494 			break;
2495 		start = offsets[area];
2496 		end = start + sizes[area];
2497 		pvm_find_next_prev(base + end, &next, &prev);
2498 	}
2499 found:
2500 	/* we've found a fitting base, insert all va's */
2501 	for (area = 0; area < nr_vms; area++) {
2502 		struct vmap_area *va = vas[area];
2503 
2504 		va->va_start = base + offsets[area];
2505 		va->va_end = va->va_start + sizes[area];
2506 		__insert_vmap_area(va);
2507 	}
2508 
2509 	vmap_area_pcpu_hole = base + offsets[last_area];
2510 
2511 	spin_unlock(&vmap_area_lock);
2512 
2513 	/* insert all vm's */
2514 	for (area = 0; area < nr_vms; area++)
2515 		setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
2516 				 pcpu_get_vm_areas);
2517 
2518 	kfree(vas);
2519 	return vms;
2520 
2521 err_free:
2522 	for (area = 0; area < nr_vms; area++) {
2523 		kfree(vas[area]);
2524 		kfree(vms[area]);
2525 	}
2526 err_free2:
2527 	kfree(vas);
2528 	kfree(vms);
2529 	return NULL;
2530 }
2531 
2532 /**
2533  * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
2534  * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
2535  * @nr_vms: the number of allocated areas
2536  *
2537  * Free vm_structs and the array allocated by pcpu_get_vm_areas().
2538  */
2539 void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
2540 {
2541 	int i;
2542 
2543 	for (i = 0; i < nr_vms; i++)
2544 		free_vm_area(vms[i]);
2545 	kfree(vms);
2546 }
2547 #endif	/* CONFIG_SMP */
2548 
2549 #ifdef CONFIG_PROC_FS
2550 static void *s_start(struct seq_file *m, loff_t *pos)
2551 	__acquires(&vmap_area_lock)
2552 {
2553 	loff_t n = *pos;
2554 	struct vmap_area *va;
2555 
2556 	spin_lock(&vmap_area_lock);
2557 	va = list_first_entry(&vmap_area_list, typeof(*va), list);
2558 	while (n > 0 && &va->list != &vmap_area_list) {
2559 		n--;
2560 		va = list_next_entry(va, list);
2561 	}
2562 	if (!n && &va->list != &vmap_area_list)
2563 		return va;
2564 
2565 	return NULL;
2566 
2567 }
2568 
2569 static void *s_next(struct seq_file *m, void *p, loff_t *pos)
2570 {
2571 	struct vmap_area *va = p, *next;
2572 
2573 	++*pos;
2574 	next = list_next_entry(va, list);
2575 	if (&next->list != &vmap_area_list)
2576 		return next;
2577 
2578 	return NULL;
2579 }
2580 
2581 static void s_stop(struct seq_file *m, void *p)
2582 	__releases(&vmap_area_lock)
2583 {
2584 	spin_unlock(&vmap_area_lock);
2585 }
2586 
2587 static void show_numa_info(struct seq_file *m, struct vm_struct *v)
2588 {
2589 	if (IS_ENABLED(CONFIG_NUMA)) {
2590 		unsigned int nr, *counters = m->private;
2591 
2592 		if (!counters)
2593 			return;
2594 
2595 		if (v->flags & VM_UNINITIALIZED)
2596 			return;
2597 		/* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
2598 		smp_rmb();
2599 
2600 		memset(counters, 0, nr_node_ids * sizeof(unsigned int));
2601 
2602 		for (nr = 0; nr < v->nr_pages; nr++)
2603 			counters[page_to_nid(v->pages[nr])]++;
2604 
2605 		for_each_node_state(nr, N_HIGH_MEMORY)
2606 			if (counters[nr])
2607 				seq_printf(m, " N%u=%u", nr, counters[nr]);
2608 	}
2609 }
2610 
2611 static int s_show(struct seq_file *m, void *p)
2612 {
2613 	struct vmap_area *va = p;
2614 	struct vm_struct *v;
2615 
2616 	/*
2617 	 * s_show can encounter race with remove_vm_area, !VM_VM_AREA on
2618 	 * behalf of vmap area is being tear down or vm_map_ram allocation.
2619 	 */
2620 	if (!(va->flags & VM_VM_AREA))
2621 		return 0;
2622 
2623 	v = va->vm;
2624 
2625 	seq_printf(m, "0x%pK-0x%pK %7ld",
2626 		v->addr, v->addr + v->size, v->size);
2627 
2628 	if (v->caller)
2629 		seq_printf(m, " %pS", v->caller);
2630 
2631 	if (v->nr_pages)
2632 		seq_printf(m, " pages=%d", v->nr_pages);
2633 
2634 	if (v->phys_addr)
2635 		seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
2636 
2637 	if (v->flags & VM_IOREMAP)
2638 		seq_puts(m, " ioremap");
2639 
2640 	if (v->flags & VM_ALLOC)
2641 		seq_puts(m, " vmalloc");
2642 
2643 	if (v->flags & VM_MAP)
2644 		seq_puts(m, " vmap");
2645 
2646 	if (v->flags & VM_USERMAP)
2647 		seq_puts(m, " user");
2648 
2649 	if (is_vmalloc_addr(v->pages))
2650 		seq_puts(m, " vpages");
2651 
2652 	show_numa_info(m, v);
2653 	seq_putc(m, '\n');
2654 	return 0;
2655 }
2656 
2657 static const struct seq_operations vmalloc_op = {
2658 	.start = s_start,
2659 	.next = s_next,
2660 	.stop = s_stop,
2661 	.show = s_show,
2662 };
2663 
2664 static int vmalloc_open(struct inode *inode, struct file *file)
2665 {
2666 	if (IS_ENABLED(CONFIG_NUMA))
2667 		return seq_open_private(file, &vmalloc_op,
2668 					nr_node_ids * sizeof(unsigned int));
2669 	else
2670 		return seq_open(file, &vmalloc_op);
2671 }
2672 
2673 static const struct file_operations proc_vmalloc_operations = {
2674 	.open		= vmalloc_open,
2675 	.read		= seq_read,
2676 	.llseek		= seq_lseek,
2677 	.release	= seq_release_private,
2678 };
2679 
2680 static int __init proc_vmalloc_init(void)
2681 {
2682 	proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
2683 	return 0;
2684 }
2685 module_init(proc_vmalloc_init);
2686 
2687 #endif
2688 
2689