xref: /openbmc/linux/mm/dmapool.c (revision 8440bb9b)
1 /*
2  * DMA Pool allocator
3  *
4  * Copyright 2001 David Brownell
5  * Copyright 2007 Intel Corporation
6  *   Author: Matthew Wilcox <willy@linux.intel.com>
7  *
8  * This software may be redistributed and/or modified under the terms of
9  * the GNU General Public License ("GPL") version 2 as published by the
10  * Free Software Foundation.
11  *
12  * This allocator returns small blocks of a given size which are DMA-able by
13  * the given device.  It uses the dma_alloc_coherent page allocator to get
14  * new pages, then splits them up into blocks of the required size.
15  * Many older drivers still have their own code to do this.
16  *
17  * The current design of this allocator is fairly simple.  The pool is
18  * represented by the 'struct dma_pool' which keeps a doubly-linked list of
19  * allocated pages.  Each page in the page_list is split into blocks of at
20  * least 'size' bytes.  Free blocks are tracked in an unsorted singly-linked
21  * list of free blocks within the page.  Used blocks aren't tracked, but we
22  * keep a count of how many are currently allocated from each page.
23  */
24 
25 #include <linux/device.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/dmapool.h>
28 #include <linux/kernel.h>
29 #include <linux/list.h>
30 #include <linux/export.h>
31 #include <linux/mutex.h>
32 #include <linux/poison.h>
33 #include <linux/sched.h>
34 #include <linux/slab.h>
35 #include <linux/stat.h>
36 #include <linux/spinlock.h>
37 #include <linux/string.h>
38 #include <linux/types.h>
39 #include <linux/wait.h>
40 
41 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
42 #define DMAPOOL_DEBUG 1
43 #endif
44 
45 struct dma_pool {		/* the pool */
46 	struct list_head page_list;
47 	spinlock_t lock;
48 	size_t size;
49 	struct device *dev;
50 	size_t allocation;
51 	size_t boundary;
52 	char name[32];
53 	struct list_head pools;
54 };
55 
56 struct dma_page {		/* cacheable header for 'allocation' bytes */
57 	struct list_head page_list;
58 	void *vaddr;
59 	dma_addr_t dma;
60 	unsigned int in_use;
61 	unsigned int offset;
62 };
63 
64 static DEFINE_MUTEX(pools_lock);
65 static DEFINE_MUTEX(pools_reg_lock);
66 
67 static ssize_t
68 show_pools(struct device *dev, struct device_attribute *attr, char *buf)
69 {
70 	unsigned temp;
71 	unsigned size;
72 	char *next;
73 	struct dma_page *page;
74 	struct dma_pool *pool;
75 
76 	next = buf;
77 	size = PAGE_SIZE;
78 
79 	temp = scnprintf(next, size, "poolinfo - 0.1\n");
80 	size -= temp;
81 	next += temp;
82 
83 	mutex_lock(&pools_lock);
84 	list_for_each_entry(pool, &dev->dma_pools, pools) {
85 		unsigned pages = 0;
86 		unsigned blocks = 0;
87 
88 		spin_lock_irq(&pool->lock);
89 		list_for_each_entry(page, &pool->page_list, page_list) {
90 			pages++;
91 			blocks += page->in_use;
92 		}
93 		spin_unlock_irq(&pool->lock);
94 
95 		/* per-pool info, no real statistics yet */
96 		temp = scnprintf(next, size, "%-16s %4u %4zu %4zu %2u\n",
97 				 pool->name, blocks,
98 				 pages * (pool->allocation / pool->size),
99 				 pool->size, pages);
100 		size -= temp;
101 		next += temp;
102 	}
103 	mutex_unlock(&pools_lock);
104 
105 	return PAGE_SIZE - size;
106 }
107 
108 static DEVICE_ATTR(pools, 0444, show_pools, NULL);
109 
110 /**
111  * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
112  * @name: name of pool, for diagnostics
113  * @dev: device that will be doing the DMA
114  * @size: size of the blocks in this pool.
115  * @align: alignment requirement for blocks; must be a power of two
116  * @boundary: returned blocks won't cross this power of two boundary
117  * Context: not in_interrupt()
118  *
119  * Given one of these pools, dma_pool_alloc()
120  * may be used to allocate memory.  Such memory will all have "consistent"
121  * DMA mappings, accessible by the device and its driver without using
122  * cache flushing primitives.  The actual size of blocks allocated may be
123  * larger than requested because of alignment.
124  *
125  * If @boundary is nonzero, objects returned from dma_pool_alloc() won't
126  * cross that size boundary.  This is useful for devices which have
127  * addressing restrictions on individual DMA transfers, such as not crossing
128  * boundaries of 4KBytes.
129  *
130  * Return: a dma allocation pool with the requested characteristics, or
131  * %NULL if one can't be created.
132  */
133 struct dma_pool *dma_pool_create(const char *name, struct device *dev,
134 				 size_t size, size_t align, size_t boundary)
135 {
136 	struct dma_pool *retval;
137 	size_t allocation;
138 	bool empty = false;
139 
140 	if (align == 0)
141 		align = 1;
142 	else if (align & (align - 1))
143 		return NULL;
144 
145 	if (size == 0)
146 		return NULL;
147 	else if (size < 4)
148 		size = 4;
149 
150 	if ((size % align) != 0)
151 		size = ALIGN(size, align);
152 
153 	allocation = max_t(size_t, size, PAGE_SIZE);
154 
155 	if (!boundary)
156 		boundary = allocation;
157 	else if ((boundary < size) || (boundary & (boundary - 1)))
158 		return NULL;
159 
160 	retval = kmalloc_node(sizeof(*retval), GFP_KERNEL, dev_to_node(dev));
161 	if (!retval)
162 		return retval;
163 
164 	strlcpy(retval->name, name, sizeof(retval->name));
165 
166 	retval->dev = dev;
167 
168 	INIT_LIST_HEAD(&retval->page_list);
169 	spin_lock_init(&retval->lock);
170 	retval->size = size;
171 	retval->boundary = boundary;
172 	retval->allocation = allocation;
173 
174 	INIT_LIST_HEAD(&retval->pools);
175 
176 	/*
177 	 * pools_lock ensures that the ->dma_pools list does not get corrupted.
178 	 * pools_reg_lock ensures that there is not a race between
179 	 * dma_pool_create() and dma_pool_destroy() or within dma_pool_create()
180 	 * when the first invocation of dma_pool_create() failed on
181 	 * device_create_file() and the second assumes that it has been done (I
182 	 * know it is a short window).
183 	 */
184 	mutex_lock(&pools_reg_lock);
185 	mutex_lock(&pools_lock);
186 	if (list_empty(&dev->dma_pools))
187 		empty = true;
188 	list_add(&retval->pools, &dev->dma_pools);
189 	mutex_unlock(&pools_lock);
190 	if (empty) {
191 		int err;
192 
193 		err = device_create_file(dev, &dev_attr_pools);
194 		if (err) {
195 			mutex_lock(&pools_lock);
196 			list_del(&retval->pools);
197 			mutex_unlock(&pools_lock);
198 			mutex_unlock(&pools_reg_lock);
199 			kfree(retval);
200 			return NULL;
201 		}
202 	}
203 	mutex_unlock(&pools_reg_lock);
204 	return retval;
205 }
206 EXPORT_SYMBOL(dma_pool_create);
207 
208 static void pool_initialise_page(struct dma_pool *pool, struct dma_page *page)
209 {
210 	unsigned int offset = 0;
211 	unsigned int next_boundary = pool->boundary;
212 
213 	do {
214 		unsigned int next = offset + pool->size;
215 		if (unlikely((next + pool->size) >= next_boundary)) {
216 			next = next_boundary;
217 			next_boundary += pool->boundary;
218 		}
219 		*(int *)(page->vaddr + offset) = next;
220 		offset = next;
221 	} while (offset < pool->allocation);
222 }
223 
224 static struct dma_page *pool_alloc_page(struct dma_pool *pool, gfp_t mem_flags)
225 {
226 	struct dma_page *page;
227 
228 	page = kmalloc(sizeof(*page), mem_flags);
229 	if (!page)
230 		return NULL;
231 	page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,
232 					 &page->dma, mem_flags);
233 	if (page->vaddr) {
234 #ifdef	DMAPOOL_DEBUG
235 		memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
236 #endif
237 		pool_initialise_page(pool, page);
238 		page->in_use = 0;
239 		page->offset = 0;
240 	} else {
241 		kfree(page);
242 		page = NULL;
243 	}
244 	return page;
245 }
246 
247 static inline bool is_page_busy(struct dma_page *page)
248 {
249 	return page->in_use != 0;
250 }
251 
252 static void pool_free_page(struct dma_pool *pool, struct dma_page *page)
253 {
254 	dma_addr_t dma = page->dma;
255 
256 #ifdef	DMAPOOL_DEBUG
257 	memset(page->vaddr, POOL_POISON_FREED, pool->allocation);
258 #endif
259 	dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma);
260 	list_del(&page->page_list);
261 	kfree(page);
262 }
263 
264 /**
265  * dma_pool_destroy - destroys a pool of dma memory blocks.
266  * @pool: dma pool that will be destroyed
267  * Context: !in_interrupt()
268  *
269  * Caller guarantees that no more memory from the pool is in use,
270  * and that nothing will try to use the pool after this call.
271  */
272 void dma_pool_destroy(struct dma_pool *pool)
273 {
274 	bool empty = false;
275 
276 	if (unlikely(!pool))
277 		return;
278 
279 	mutex_lock(&pools_reg_lock);
280 	mutex_lock(&pools_lock);
281 	list_del(&pool->pools);
282 	if (pool->dev && list_empty(&pool->dev->dma_pools))
283 		empty = true;
284 	mutex_unlock(&pools_lock);
285 	if (empty)
286 		device_remove_file(pool->dev, &dev_attr_pools);
287 	mutex_unlock(&pools_reg_lock);
288 
289 	while (!list_empty(&pool->page_list)) {
290 		struct dma_page *page;
291 		page = list_entry(pool->page_list.next,
292 				  struct dma_page, page_list);
293 		if (is_page_busy(page)) {
294 			if (pool->dev)
295 				dev_err(pool->dev,
296 					"dma_pool_destroy %s, %p busy\n",
297 					pool->name, page->vaddr);
298 			else
299 				pr_err("dma_pool_destroy %s, %p busy\n",
300 				       pool->name, page->vaddr);
301 			/* leak the still-in-use consistent memory */
302 			list_del(&page->page_list);
303 			kfree(page);
304 		} else
305 			pool_free_page(pool, page);
306 	}
307 
308 	kfree(pool);
309 }
310 EXPORT_SYMBOL(dma_pool_destroy);
311 
312 /**
313  * dma_pool_alloc - get a block of consistent memory
314  * @pool: dma pool that will produce the block
315  * @mem_flags: GFP_* bitmask
316  * @handle: pointer to dma address of block
317  *
318  * Return: the kernel virtual address of a currently unused block,
319  * and reports its dma address through the handle.
320  * If such a memory block can't be allocated, %NULL is returned.
321  */
322 void *dma_pool_alloc(struct dma_pool *pool, gfp_t mem_flags,
323 		     dma_addr_t *handle)
324 {
325 	unsigned long flags;
326 	struct dma_page *page;
327 	size_t offset;
328 	void *retval;
329 
330 	might_sleep_if(gfpflags_allow_blocking(mem_flags));
331 
332 	spin_lock_irqsave(&pool->lock, flags);
333 	list_for_each_entry(page, &pool->page_list, page_list) {
334 		if (page->offset < pool->allocation)
335 			goto ready;
336 	}
337 
338 	/* pool_alloc_page() might sleep, so temporarily drop &pool->lock */
339 	spin_unlock_irqrestore(&pool->lock, flags);
340 
341 	page = pool_alloc_page(pool, mem_flags & (~__GFP_ZERO));
342 	if (!page)
343 		return NULL;
344 
345 	spin_lock_irqsave(&pool->lock, flags);
346 
347 	list_add(&page->page_list, &pool->page_list);
348  ready:
349 	page->in_use++;
350 	offset = page->offset;
351 	page->offset = *(int *)(page->vaddr + offset);
352 	retval = offset + page->vaddr;
353 	*handle = offset + page->dma;
354 #ifdef	DMAPOOL_DEBUG
355 	{
356 		int i;
357 		u8 *data = retval;
358 		/* page->offset is stored in first 4 bytes */
359 		for (i = sizeof(page->offset); i < pool->size; i++) {
360 			if (data[i] == POOL_POISON_FREED)
361 				continue;
362 			if (pool->dev)
363 				dev_err(pool->dev,
364 					"dma_pool_alloc %s, %p (corrupted)\n",
365 					pool->name, retval);
366 			else
367 				pr_err("dma_pool_alloc %s, %p (corrupted)\n",
368 					pool->name, retval);
369 
370 			/*
371 			 * Dump the first 4 bytes even if they are not
372 			 * POOL_POISON_FREED
373 			 */
374 			print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1,
375 					data, pool->size, 1);
376 			break;
377 		}
378 	}
379 	if (!(mem_flags & __GFP_ZERO))
380 		memset(retval, POOL_POISON_ALLOCATED, pool->size);
381 #endif
382 	spin_unlock_irqrestore(&pool->lock, flags);
383 
384 	if (mem_flags & __GFP_ZERO)
385 		memset(retval, 0, pool->size);
386 
387 	return retval;
388 }
389 EXPORT_SYMBOL(dma_pool_alloc);
390 
391 static struct dma_page *pool_find_page(struct dma_pool *pool, dma_addr_t dma)
392 {
393 	struct dma_page *page;
394 
395 	list_for_each_entry(page, &pool->page_list, page_list) {
396 		if (dma < page->dma)
397 			continue;
398 		if ((dma - page->dma) < pool->allocation)
399 			return page;
400 	}
401 	return NULL;
402 }
403 
404 /**
405  * dma_pool_free - put block back into dma pool
406  * @pool: the dma pool holding the block
407  * @vaddr: virtual address of block
408  * @dma: dma address of block
409  *
410  * Caller promises neither device nor driver will again touch this block
411  * unless it is first re-allocated.
412  */
413 void dma_pool_free(struct dma_pool *pool, void *vaddr, dma_addr_t dma)
414 {
415 	struct dma_page *page;
416 	unsigned long flags;
417 	unsigned int offset;
418 
419 	spin_lock_irqsave(&pool->lock, flags);
420 	page = pool_find_page(pool, dma);
421 	if (!page) {
422 		spin_unlock_irqrestore(&pool->lock, flags);
423 		if (pool->dev)
424 			dev_err(pool->dev,
425 				"dma_pool_free %s, %p/%lx (bad dma)\n",
426 				pool->name, vaddr, (unsigned long)dma);
427 		else
428 			pr_err("dma_pool_free %s, %p/%lx (bad dma)\n",
429 			       pool->name, vaddr, (unsigned long)dma);
430 		return;
431 	}
432 
433 	offset = vaddr - page->vaddr;
434 #ifdef	DMAPOOL_DEBUG
435 	if ((dma - page->dma) != offset) {
436 		spin_unlock_irqrestore(&pool->lock, flags);
437 		if (pool->dev)
438 			dev_err(pool->dev,
439 				"dma_pool_free %s, %p (bad vaddr)/%pad\n",
440 				pool->name, vaddr, &dma);
441 		else
442 			pr_err("dma_pool_free %s, %p (bad vaddr)/%pad\n",
443 			       pool->name, vaddr, &dma);
444 		return;
445 	}
446 	{
447 		unsigned int chain = page->offset;
448 		while (chain < pool->allocation) {
449 			if (chain != offset) {
450 				chain = *(int *)(page->vaddr + chain);
451 				continue;
452 			}
453 			spin_unlock_irqrestore(&pool->lock, flags);
454 			if (pool->dev)
455 				dev_err(pool->dev, "dma_pool_free %s, dma %pad already free\n",
456 					pool->name, &dma);
457 			else
458 				pr_err("dma_pool_free %s, dma %pad already free\n",
459 				       pool->name, &dma);
460 			return;
461 		}
462 	}
463 	memset(vaddr, POOL_POISON_FREED, pool->size);
464 #endif
465 
466 	page->in_use--;
467 	*(int *)vaddr = page->offset;
468 	page->offset = offset;
469 	/*
470 	 * Resist a temptation to do
471 	 *    if (!is_page_busy(page)) pool_free_page(pool, page);
472 	 * Better have a few empty pages hang around.
473 	 */
474 	spin_unlock_irqrestore(&pool->lock, flags);
475 }
476 EXPORT_SYMBOL(dma_pool_free);
477 
478 /*
479  * Managed DMA pool
480  */
481 static void dmam_pool_release(struct device *dev, void *res)
482 {
483 	struct dma_pool *pool = *(struct dma_pool **)res;
484 
485 	dma_pool_destroy(pool);
486 }
487 
488 static int dmam_pool_match(struct device *dev, void *res, void *match_data)
489 {
490 	return *(struct dma_pool **)res == match_data;
491 }
492 
493 /**
494  * dmam_pool_create - Managed dma_pool_create()
495  * @name: name of pool, for diagnostics
496  * @dev: device that will be doing the DMA
497  * @size: size of the blocks in this pool.
498  * @align: alignment requirement for blocks; must be a power of two
499  * @allocation: returned blocks won't cross this boundary (or zero)
500  *
501  * Managed dma_pool_create().  DMA pool created with this function is
502  * automatically destroyed on driver detach.
503  *
504  * Return: a managed dma allocation pool with the requested
505  * characteristics, or %NULL if one can't be created.
506  */
507 struct dma_pool *dmam_pool_create(const char *name, struct device *dev,
508 				  size_t size, size_t align, size_t allocation)
509 {
510 	struct dma_pool **ptr, *pool;
511 
512 	ptr = devres_alloc(dmam_pool_release, sizeof(*ptr), GFP_KERNEL);
513 	if (!ptr)
514 		return NULL;
515 
516 	pool = *ptr = dma_pool_create(name, dev, size, align, allocation);
517 	if (pool)
518 		devres_add(dev, ptr);
519 	else
520 		devres_free(ptr);
521 
522 	return pool;
523 }
524 EXPORT_SYMBOL(dmam_pool_create);
525 
526 /**
527  * dmam_pool_destroy - Managed dma_pool_destroy()
528  * @pool: dma pool that will be destroyed
529  *
530  * Managed dma_pool_destroy().
531  */
532 void dmam_pool_destroy(struct dma_pool *pool)
533 {
534 	struct device *dev = pool->dev;
535 
536 	WARN_ON(devres_release(dev, dmam_pool_release, dmam_pool_match, pool));
537 }
538 EXPORT_SYMBOL(dmam_pool_destroy);
539