xref: /openbmc/linux/mm/vmalloc.c (revision d5cb9783536a41df9f9cba5b0a1d78047ed787f7)
1 /*
2  *  linux/mm/vmalloc.c
3  *
4  *  Copyright (C) 1993  Linus Torvalds
5  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
6  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
7  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
8  *  Numa awareness, Christoph Lameter, SGI, June 2005
9  */
10 
11 #include <linux/mm.h>
12 #include <linux/module.h>
13 #include <linux/highmem.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 #include <linux/interrupt.h>
17 
18 #include <linux/vmalloc.h>
19 
20 #include <asm/uaccess.h>
21 #include <asm/tlbflush.h>
22 
23 
24 DEFINE_RWLOCK(vmlist_lock);
25 struct vm_struct *vmlist;
26 
27 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
28 {
29 	pte_t *pte;
30 
31 	pte = pte_offset_kernel(pmd, addr);
32 	do {
33 		pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
34 		WARN_ON(!pte_none(ptent) && !pte_present(ptent));
35 	} while (pte++, addr += PAGE_SIZE, addr != end);
36 }
37 
38 static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
39 						unsigned long end)
40 {
41 	pmd_t *pmd;
42 	unsigned long next;
43 
44 	pmd = pmd_offset(pud, addr);
45 	do {
46 		next = pmd_addr_end(addr, end);
47 		if (pmd_none_or_clear_bad(pmd))
48 			continue;
49 		vunmap_pte_range(pmd, addr, next);
50 	} while (pmd++, addr = next, addr != end);
51 }
52 
53 static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
54 						unsigned long end)
55 {
56 	pud_t *pud;
57 	unsigned long next;
58 
59 	pud = pud_offset(pgd, addr);
60 	do {
61 		next = pud_addr_end(addr, end);
62 		if (pud_none_or_clear_bad(pud))
63 			continue;
64 		vunmap_pmd_range(pud, addr, next);
65 	} while (pud++, addr = next, addr != end);
66 }
67 
68 void unmap_vm_area(struct vm_struct *area)
69 {
70 	pgd_t *pgd;
71 	unsigned long next;
72 	unsigned long addr = (unsigned long) area->addr;
73 	unsigned long end = addr + area->size;
74 
75 	BUG_ON(addr >= end);
76 	pgd = pgd_offset_k(addr);
77 	flush_cache_vunmap(addr, end);
78 	do {
79 		next = pgd_addr_end(addr, end);
80 		if (pgd_none_or_clear_bad(pgd))
81 			continue;
82 		vunmap_pud_range(pgd, addr, next);
83 	} while (pgd++, addr = next, addr != end);
84 	flush_tlb_kernel_range((unsigned long) area->addr, end);
85 }
86 
87 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
88 			unsigned long end, pgprot_t prot, struct page ***pages)
89 {
90 	pte_t *pte;
91 
92 	pte = pte_alloc_kernel(pmd, addr);
93 	if (!pte)
94 		return -ENOMEM;
95 	do {
96 		struct page *page = **pages;
97 		WARN_ON(!pte_none(*pte));
98 		if (!page)
99 			return -ENOMEM;
100 		set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
101 		(*pages)++;
102 	} while (pte++, addr += PAGE_SIZE, addr != end);
103 	return 0;
104 }
105 
106 static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
107 			unsigned long end, pgprot_t prot, struct page ***pages)
108 {
109 	pmd_t *pmd;
110 	unsigned long next;
111 
112 	pmd = pmd_alloc(&init_mm, pud, addr);
113 	if (!pmd)
114 		return -ENOMEM;
115 	do {
116 		next = pmd_addr_end(addr, end);
117 		if (vmap_pte_range(pmd, addr, next, prot, pages))
118 			return -ENOMEM;
119 	} while (pmd++, addr = next, addr != end);
120 	return 0;
121 }
122 
123 static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
124 			unsigned long end, pgprot_t prot, struct page ***pages)
125 {
126 	pud_t *pud;
127 	unsigned long next;
128 
129 	pud = pud_alloc(&init_mm, pgd, addr);
130 	if (!pud)
131 		return -ENOMEM;
132 	do {
133 		next = pud_addr_end(addr, end);
134 		if (vmap_pmd_range(pud, addr, next, prot, pages))
135 			return -ENOMEM;
136 	} while (pud++, addr = next, addr != end);
137 	return 0;
138 }
139 
140 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
141 {
142 	pgd_t *pgd;
143 	unsigned long next;
144 	unsigned long addr = (unsigned long) area->addr;
145 	unsigned long end = addr + area->size - PAGE_SIZE;
146 	int err;
147 
148 	BUG_ON(addr >= end);
149 	pgd = pgd_offset_k(addr);
150 	do {
151 		next = pgd_addr_end(addr, end);
152 		err = vmap_pud_range(pgd, addr, next, prot, pages);
153 		if (err)
154 			break;
155 	} while (pgd++, addr = next, addr != end);
156 	flush_cache_vmap((unsigned long) area->addr, end);
157 	return err;
158 }
159 
160 struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
161 				unsigned long start, unsigned long end, int node)
162 {
163 	struct vm_struct **p, *tmp, *area;
164 	unsigned long align = 1;
165 	unsigned long addr;
166 
167 	if (flags & VM_IOREMAP) {
168 		int bit = fls(size);
169 
170 		if (bit > IOREMAP_MAX_ORDER)
171 			bit = IOREMAP_MAX_ORDER;
172 		else if (bit < PAGE_SHIFT)
173 			bit = PAGE_SHIFT;
174 
175 		align = 1ul << bit;
176 	}
177 	addr = ALIGN(start, align);
178 	size = PAGE_ALIGN(size);
179 
180 	area = kmalloc_node(sizeof(*area), GFP_KERNEL, node);
181 	if (unlikely(!area))
182 		return NULL;
183 
184 	if (unlikely(!size)) {
185 		kfree (area);
186 		return NULL;
187 	}
188 
189 	/*
190 	 * We always allocate a guard page.
191 	 */
192 	size += PAGE_SIZE;
193 
194 	write_lock(&vmlist_lock);
195 	for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
196 		if ((unsigned long)tmp->addr < addr) {
197 			if((unsigned long)tmp->addr + tmp->size >= addr)
198 				addr = ALIGN(tmp->size +
199 					     (unsigned long)tmp->addr, align);
200 			continue;
201 		}
202 		if ((size + addr) < addr)
203 			goto out;
204 		if (size + addr <= (unsigned long)tmp->addr)
205 			goto found;
206 		addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
207 		if (addr > end - size)
208 			goto out;
209 	}
210 
211 found:
212 	area->next = *p;
213 	*p = area;
214 
215 	area->flags = flags;
216 	area->addr = (void *)addr;
217 	area->size = size;
218 	area->pages = NULL;
219 	area->nr_pages = 0;
220 	area->phys_addr = 0;
221 	write_unlock(&vmlist_lock);
222 
223 	return area;
224 
225 out:
226 	write_unlock(&vmlist_lock);
227 	kfree(area);
228 	if (printk_ratelimit())
229 		printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
230 	return NULL;
231 }
232 
233 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
234 				unsigned long start, unsigned long end)
235 {
236 	return __get_vm_area_node(size, flags, start, end, -1);
237 }
238 
239 /**
240  *	get_vm_area  -  reserve a contingous kernel virtual area
241  *
242  *	@size:		size of the area
243  *	@flags:		%VM_IOREMAP for I/O mappings or VM_ALLOC
244  *
245  *	Search an area of @size in the kernel virtual mapping area,
246  *	and reserved it for out purposes.  Returns the area descriptor
247  *	on success or %NULL on failure.
248  */
249 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
250 {
251 	return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
252 }
253 
254 struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags, int node)
255 {
256 	return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node);
257 }
258 
259 /* Caller must hold vmlist_lock */
260 struct vm_struct *__remove_vm_area(void *addr)
261 {
262 	struct vm_struct **p, *tmp;
263 
264 	for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
265 		 if (tmp->addr == addr)
266 			 goto found;
267 	}
268 	return NULL;
269 
270 found:
271 	unmap_vm_area(tmp);
272 	*p = tmp->next;
273 
274 	/*
275 	 * Remove the guard page.
276 	 */
277 	tmp->size -= PAGE_SIZE;
278 	return tmp;
279 }
280 
281 /**
282  *	remove_vm_area  -  find and remove a contingous kernel virtual area
283  *
284  *	@addr:		base address
285  *
286  *	Search for the kernel VM area starting at @addr, and remove it.
287  *	This function returns the found VM area, but using it is NOT safe
288  *	on SMP machines, except for its size or flags.
289  */
290 struct vm_struct *remove_vm_area(void *addr)
291 {
292 	struct vm_struct *v;
293 	write_lock(&vmlist_lock);
294 	v = __remove_vm_area(addr);
295 	write_unlock(&vmlist_lock);
296 	return v;
297 }
298 
299 void __vunmap(void *addr, int deallocate_pages)
300 {
301 	struct vm_struct *area;
302 
303 	if (!addr)
304 		return;
305 
306 	if ((PAGE_SIZE-1) & (unsigned long)addr) {
307 		printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
308 		WARN_ON(1);
309 		return;
310 	}
311 
312 	area = remove_vm_area(addr);
313 	if (unlikely(!area)) {
314 		printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
315 				addr);
316 		WARN_ON(1);
317 		return;
318 	}
319 
320 	if (deallocate_pages) {
321 		int i;
322 
323 		for (i = 0; i < area->nr_pages; i++) {
324 			if (unlikely(!area->pages[i]))
325 				BUG();
326 			__free_page(area->pages[i]);
327 		}
328 
329 		if (area->nr_pages > PAGE_SIZE/sizeof(struct page *))
330 			vfree(area->pages);
331 		else
332 			kfree(area->pages);
333 	}
334 
335 	kfree(area);
336 	return;
337 }
338 
339 /**
340  *	vfree  -  release memory allocated by vmalloc()
341  *
342  *	@addr:		memory base address
343  *
344  *	Free the virtually contiguous memory area starting at @addr, as
345  *	obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
346  *	NULL, no operation is performed.
347  *
348  *	Must not be called in interrupt context.
349  */
350 void vfree(void *addr)
351 {
352 	BUG_ON(in_interrupt());
353 	__vunmap(addr, 1);
354 }
355 EXPORT_SYMBOL(vfree);
356 
357 /**
358  *	vunmap  -  release virtual mapping obtained by vmap()
359  *
360  *	@addr:		memory base address
361  *
362  *	Free the virtually contiguous memory area starting at @addr,
363  *	which was created from the page array passed to vmap().
364  *
365  *	Must not be called in interrupt context.
366  */
367 void vunmap(void *addr)
368 {
369 	BUG_ON(in_interrupt());
370 	__vunmap(addr, 0);
371 }
372 EXPORT_SYMBOL(vunmap);
373 
374 /**
375  *	vmap  -  map an array of pages into virtually contiguous space
376  *
377  *	@pages:		array of page pointers
378  *	@count:		number of pages to map
379  *	@flags:		vm_area->flags
380  *	@prot:		page protection for the mapping
381  *
382  *	Maps @count pages from @pages into contiguous kernel virtual
383  *	space.
384  */
385 void *vmap(struct page **pages, unsigned int count,
386 		unsigned long flags, pgprot_t prot)
387 {
388 	struct vm_struct *area;
389 
390 	if (count > num_physpages)
391 		return NULL;
392 
393 	area = get_vm_area((count << PAGE_SHIFT), flags);
394 	if (!area)
395 		return NULL;
396 	if (map_vm_area(area, prot, &pages)) {
397 		vunmap(area->addr);
398 		return NULL;
399 	}
400 
401 	return area->addr;
402 }
403 EXPORT_SYMBOL(vmap);
404 
405 void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
406 				pgprot_t prot, int node)
407 {
408 	struct page **pages;
409 	unsigned int nr_pages, array_size, i;
410 
411 	nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
412 	array_size = (nr_pages * sizeof(struct page *));
413 
414 	area->nr_pages = nr_pages;
415 	/* Please note that the recursion is strictly bounded. */
416 	if (array_size > PAGE_SIZE)
417 		pages = __vmalloc_node(array_size, gfp_mask, PAGE_KERNEL, node);
418 	else
419 		pages = kmalloc_node(array_size, (gfp_mask & ~__GFP_HIGHMEM), node);
420 	area->pages = pages;
421 	if (!area->pages) {
422 		remove_vm_area(area->addr);
423 		kfree(area);
424 		return NULL;
425 	}
426 	memset(area->pages, 0, array_size);
427 
428 	for (i = 0; i < area->nr_pages; i++) {
429 		if (node < 0)
430 			area->pages[i] = alloc_page(gfp_mask);
431 		else
432 			area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
433 		if (unlikely(!area->pages[i])) {
434 			/* Successfully allocated i pages, free them in __vunmap() */
435 			area->nr_pages = i;
436 			goto fail;
437 		}
438 	}
439 
440 	if (map_vm_area(area, prot, &pages))
441 		goto fail;
442 	return area->addr;
443 
444 fail:
445 	vfree(area->addr);
446 	return NULL;
447 }
448 
449 void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
450 {
451 	return __vmalloc_area_node(area, gfp_mask, prot, -1);
452 }
453 
454 /**
455  *	__vmalloc_node  -  allocate virtually contiguous memory
456  *
457  *	@size:		allocation size
458  *	@gfp_mask:	flags for the page level allocator
459  *	@prot:		protection mask for the allocated pages
460  *	@node		node to use for allocation or -1
461  *
462  *	Allocate enough pages to cover @size from the page level
463  *	allocator with @gfp_mask flags.  Map them into contiguous
464  *	kernel virtual space, using a pagetable protection of @prot.
465  */
466 void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
467 			int node)
468 {
469 	struct vm_struct *area;
470 
471 	size = PAGE_ALIGN(size);
472 	if (!size || (size >> PAGE_SHIFT) > num_physpages)
473 		return NULL;
474 
475 	area = get_vm_area_node(size, VM_ALLOC, node);
476 	if (!area)
477 		return NULL;
478 
479 	return __vmalloc_area_node(area, gfp_mask, prot, node);
480 }
481 EXPORT_SYMBOL(__vmalloc_node);
482 
483 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
484 {
485 	return __vmalloc_node(size, gfp_mask, prot, -1);
486 }
487 EXPORT_SYMBOL(__vmalloc);
488 
489 /**
490  *	vmalloc  -  allocate virtually contiguous memory
491  *
492  *	@size:		allocation size
493  *
494  *	Allocate enough pages to cover @size from the page level
495  *	allocator and map them into contiguous kernel virtual space.
496  *
497  *	For tight cotrol over page level allocator and protection flags
498  *	use __vmalloc() instead.
499  */
500 void *vmalloc(unsigned long size)
501 {
502        return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
503 }
504 EXPORT_SYMBOL(vmalloc);
505 
506 /**
507  *	vmalloc_node  -  allocate memory on a specific node
508  *
509  *	@size:		allocation size
510  *	@node;		numa node
511  *
512  *	Allocate enough pages to cover @size from the page level
513  *	allocator and map them into contiguous kernel virtual space.
514  *
515  *	For tight cotrol over page level allocator and protection flags
516  *	use __vmalloc() instead.
517  */
518 void *vmalloc_node(unsigned long size, int node)
519 {
520        return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
521 }
522 EXPORT_SYMBOL(vmalloc_node);
523 
524 #ifndef PAGE_KERNEL_EXEC
525 # define PAGE_KERNEL_EXEC PAGE_KERNEL
526 #endif
527 
528 /**
529  *	vmalloc_exec  -  allocate virtually contiguous, executable memory
530  *
531  *	@size:		allocation size
532  *
533  *	Kernel-internal function to allocate enough pages to cover @size
534  *	the page level allocator and map them into contiguous and
535  *	executable kernel virtual space.
536  *
537  *	For tight cotrol over page level allocator and protection flags
538  *	use __vmalloc() instead.
539  */
540 
541 void *vmalloc_exec(unsigned long size)
542 {
543 	return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
544 }
545 
546 /**
547  *	vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
548  *
549  *	@size:		allocation size
550  *
551  *	Allocate enough 32bit PA addressable pages to cover @size from the
552  *	page level allocator and map them into contiguous kernel virtual space.
553  */
554 void *vmalloc_32(unsigned long size)
555 {
556 	return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
557 }
558 EXPORT_SYMBOL(vmalloc_32);
559 
560 long vread(char *buf, char *addr, unsigned long count)
561 {
562 	struct vm_struct *tmp;
563 	char *vaddr, *buf_start = buf;
564 	unsigned long n;
565 
566 	/* Don't allow overflow */
567 	if ((unsigned long) addr + count < count)
568 		count = -(unsigned long) addr;
569 
570 	read_lock(&vmlist_lock);
571 	for (tmp = vmlist; tmp; tmp = tmp->next) {
572 		vaddr = (char *) tmp->addr;
573 		if (addr >= vaddr + tmp->size - PAGE_SIZE)
574 			continue;
575 		while (addr < vaddr) {
576 			if (count == 0)
577 				goto finished;
578 			*buf = '\0';
579 			buf++;
580 			addr++;
581 			count--;
582 		}
583 		n = vaddr + tmp->size - PAGE_SIZE - addr;
584 		do {
585 			if (count == 0)
586 				goto finished;
587 			*buf = *addr;
588 			buf++;
589 			addr++;
590 			count--;
591 		} while (--n > 0);
592 	}
593 finished:
594 	read_unlock(&vmlist_lock);
595 	return buf - buf_start;
596 }
597 
598 long vwrite(char *buf, char *addr, unsigned long count)
599 {
600 	struct vm_struct *tmp;
601 	char *vaddr, *buf_start = buf;
602 	unsigned long n;
603 
604 	/* Don't allow overflow */
605 	if ((unsigned long) addr + count < count)
606 		count = -(unsigned long) addr;
607 
608 	read_lock(&vmlist_lock);
609 	for (tmp = vmlist; tmp; tmp = tmp->next) {
610 		vaddr = (char *) tmp->addr;
611 		if (addr >= vaddr + tmp->size - PAGE_SIZE)
612 			continue;
613 		while (addr < vaddr) {
614 			if (count == 0)
615 				goto finished;
616 			buf++;
617 			addr++;
618 			count--;
619 		}
620 		n = vaddr + tmp->size - PAGE_SIZE - addr;
621 		do {
622 			if (count == 0)
623 				goto finished;
624 			*addr = *buf;
625 			buf++;
626 			addr++;
627 			count--;
628 		} while (--n > 0);
629 	}
630 finished:
631 	read_unlock(&vmlist_lock);
632 	return buf - buf_start;
633 }
634