24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
32 	pr_err("BUG: mempool element poison mismatch\n");  in poison_error()
34 	pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");  in poison_error()
36 		pr_cont("%x ", *(u8 *)(element + i));  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
43 	u8 *obj = element;  in __check_element()
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
61 		__check_element(pool, element, (size_t)pool->pool_data);  in check_element()
63 		__check_element(pool, element, kmem_cache_size(pool->pool_data));  in check_element()
67 		void *addr = kmap_atomic((struct page *)element);  in check_element()
74 static void __poison_element(void *element, size_t size)  in __poison_element()  argument
76 	u8 *obj = element;  in __poison_element()
82 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
86 		__poison_element(element, (size_t)pool->pool_data);  in poison_element()
88 		__poison_element(element, kmem_cache_size(pool->pool_data));  in poison_element()
92 		void *addr = kmap_atomic((struct page *)element);  in poison_element()
99 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
102 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
107 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
110 		kasan_slab_free_mempool(element);  in kasan_poison_element()
112 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
116 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
119 		kasan_unpoison_range(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
121 		kasan_unpoison_range(element, kmem_cache_size(pool->pool_data));  in kasan_unpoison_element()
123 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
127 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
130 	poison_element(pool, element);  in add_element()
131 	kasan_poison_element(pool, element);  in add_element()
132 	pool->elements[pool->curr_nr++] = element;  in add_element()
137 	void *element = pool->elements[--pool->curr_nr];  in remove_element()  local
140 	kasan_unpoison_element(pool, element);  in remove_element()
141 	check_element(pool, element);  in remove_element()
142 	return element;  in remove_element()
159 		void *element = remove_element(pool);  in mempool_exit()  local
160 		pool->free(element, pool->pool_data);  in mempool_exit()
205 		void *element;  in mempool_init_node()  local
207 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
208 		if (unlikely(!element)) {  in mempool_init_node()
212 		add_element(pool, element);  in mempool_init_node()
224  * @alloc_fn:  user-defined element-allocation function.
225  * @free_fn:   user-defined element-freeing function.
246  * @alloc_fn:  user-defined element-allocation function.
247  * @free_fn:   user-defined element-freeing function.
306 	void *element;  in mempool_resize()  local
316 			element = remove_element(pool);  in mempool_resize()
318 			pool->free(element, pool->pool_data);  in mempool_resize()
347 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
348 		if (!element)  in mempool_resize()
352 			add_element(pool, element);  in mempool_resize()
355 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
367  * mempool_alloc - allocate an element from a specific memory pool
378  * Return: pointer to the allocated element or %NULL on error.
382 	void *element;  in mempool_alloc()  local
398 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
399 	if (likely(element != NULL))  in mempool_alloc()
400 		return element;  in mempool_alloc()
404 		element = remove_element(pool);  in mempool_alloc()
412 		kmemleak_update_trace(element);  in mempool_alloc()
413 		return element;  in mempool_alloc()
432 	/* Let's wait for someone else to return an element to @pool */  in mempool_alloc()
450  * mempool_free - return an element to the pool.
451  * @element:   pool element pointer.
457 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
461 	if (unlikely(element == NULL))  in mempool_free()
466 	 * for @element and the following @pool->curr_nr.  This ensures  in mempool_free()
468 	 * allocation of @element.  This is necessary for fringe cases  in mempool_free()
469 	 * where @element was passed to this task without going through  in mempool_free()
487 	 * allocation of @element, any task which decremented curr_nr below  in mempool_free()
490 	 * to min_nr after the allocation of @element, the elements  in mempool_free()
500 			add_element(pool, element);  in mempool_free()
507 	pool->free(element, pool->pool_data);  in mempool_free()
522 void mempool_free_slab(void *element, void *pool_data)  in mempool_free_slab()  argument
525 	kmem_cache_free(mem, element);  in mempool_free_slab()
540 void mempool_kfree(void *element, void *pool_data)  in mempool_kfree()  argument
542 	kfree(element);  in mempool_kfree()
557 void mempool_free_pages(void *element, void *pool_data)  in mempool_free_pages()  argument
560 	__free_pages(element, order);  in mempool_free_pages()