24 static void poison_error(mempool_t *pool, void *element, size_t size,  in poison_error()  argument
27 	const int nr = pool->curr_nr;  in poison_error()
33 	pr_err("Mempool %p size %zu\n", pool, size);  in poison_error()
41 static void __check_element(mempool_t *pool, void *element, size_t size)  in __check_element()  argument
50 			poison_error(pool, element, size, i);  in __check_element()
57 static void check_element(mempool_t *pool, void *element)  in check_element()  argument
60 	if (pool->free == mempool_kfree) {  in check_element()
61 		__check_element(pool, element, (size_t)pool->pool_data);  in check_element()
62 	} else if (pool->free == mempool_free_slab) {  in check_element()
63 		__check_element(pool, element, kmem_cache_size(pool->pool_data));  in check_element()
64 	} else if (pool->free == mempool_free_pages) {  in check_element()
66 		int order = (int)(long)pool->pool_data;  in check_element()
69 		__check_element(pool, addr, 1UL << (PAGE_SHIFT + order));  in check_element()
82 static void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
85 	if (pool->alloc == mempool_kmalloc) {  in poison_element()
86 		__poison_element(element, (size_t)pool->pool_data);  in poison_element()
87 	} else if (pool->alloc == mempool_alloc_slab) {  in poison_element()
88 		__poison_element(element, kmem_cache_size(pool->pool_data));  in poison_element()
89 	} else if (pool->alloc == mempool_alloc_pages) {  in poison_element()
91 		int order = (int)(long)pool->pool_data;  in poison_element()
99 static inline void check_element(mempool_t *pool, void *element)  in check_element()  argument
102 static inline void poison_element(mempool_t *pool, void *element)  in poison_element()  argument
107 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)  in kasan_poison_element()  argument
109 	if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)  in kasan_poison_element()
111 	else if (pool->alloc == mempool_alloc_pages)  in kasan_poison_element()
112 		kasan_poison_pages(element, (unsigned long)pool->pool_data,  in kasan_poison_element()
116 static void kasan_unpoison_element(mempool_t *pool, void *element)  in kasan_unpoison_element()  argument
118 	if (pool->alloc == mempool_kmalloc)  in kasan_unpoison_element()
119 		kasan_unpoison_range(element, (size_t)pool->pool_data);  in kasan_unpoison_element()
120 	else if (pool->alloc == mempool_alloc_slab)  in kasan_unpoison_element()
121 		kasan_unpoison_range(element, kmem_cache_size(pool->pool_data));  in kasan_unpoison_element()
122 	else if (pool->alloc == mempool_alloc_pages)  in kasan_unpoison_element()
123 		kasan_unpoison_pages(element, (unsigned long)pool->pool_data,  in kasan_unpoison_element()
127 static __always_inline void add_element(mempool_t *pool, void *element)  in add_element()  argument
129 	BUG_ON(pool->curr_nr >= pool->min_nr);  in add_element()
130 	poison_element(pool, element);  in add_element()
131 	kasan_poison_element(pool, element);  in add_element()
132 	pool->elements[pool->curr_nr++] = element;  in add_element()
135 static void *remove_element(mempool_t *pool)  in remove_element()  argument
137 	void *element = pool->elements[--pool->curr_nr];  in remove_element()
139 	BUG_ON(pool->curr_nr < 0);  in remove_element()
140 	kasan_unpoison_element(pool, element);  in remove_element()
141 	check_element(pool, element);  in remove_element()
156 void mempool_exit(mempool_t *pool)  in mempool_exit()  argument
158 	while (pool->curr_nr) {  in mempool_exit()
159 		void *element = remove_element(pool);  in mempool_exit()
160 		pool->free(element, pool->pool_data);  in mempool_exit()
162 	kfree(pool->elements);  in mempool_exit()
163 	pool->elements = NULL;  in mempool_exit()
175 void mempool_destroy(mempool_t *pool)  in mempool_destroy()  argument
177 	if (unlikely(!pool))  in mempool_destroy()
180 	mempool_exit(pool);  in mempool_destroy()
181 	kfree(pool);  in mempool_destroy()
185 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init_node()  argument
189 	spin_lock_init(&pool->lock);  in mempool_init_node()
190 	pool->min_nr	= min_nr;  in mempool_init_node()
191 	pool->pool_data = pool_data;  in mempool_init_node()
192 	pool->alloc	= alloc_fn;  in mempool_init_node()
193 	pool->free	= free_fn;  in mempool_init_node()
194 	init_waitqueue_head(&pool->wait);  in mempool_init_node()
196 	pool->elements = kmalloc_array_node(min_nr, sizeof(void *),  in mempool_init_node()
198 	if (!pool->elements)  in mempool_init_node()
204 	while (pool->curr_nr < pool->min_nr) {  in mempool_init_node()
207 		element = pool->alloc(gfp_mask, pool->pool_data);  in mempool_init_node()
209 			mempool_exit(pool);  in mempool_init_node()
212 		add_element(pool, element);  in mempool_init_node()
233 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,  in mempool_init()  argument
236 	return mempool_init_node(pool, min_nr, alloc_fn, free_fn,  in mempool_init()
270 	mempool_t *pool;  in mempool_create_node()  local
272 	pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);  in mempool_create_node()
273 	if (!pool)  in mempool_create_node()
276 	if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,  in mempool_create_node()
278 		kfree(pool);  in mempool_create_node()
282 	return pool;  in mempool_create_node()
304 int mempool_resize(mempool_t *pool, int new_min_nr)  in mempool_resize()  argument
313 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
314 	if (new_min_nr <= pool->min_nr) {  in mempool_resize()
315 		while (new_min_nr < pool->curr_nr) {  in mempool_resize()
316 			element = remove_element(pool);  in mempool_resize()
317 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
318 			pool->free(element, pool->pool_data);  in mempool_resize()
319 			spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
321 		pool->min_nr = new_min_nr;  in mempool_resize()
324 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
332 	spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
333 	if (unlikely(new_min_nr <= pool->min_nr)) {  in mempool_resize()
335 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
339 	memcpy(new_elements, pool->elements,  in mempool_resize()
340 			pool->curr_nr * sizeof(*new_elements));  in mempool_resize()
341 	kfree(pool->elements);  in mempool_resize()
342 	pool->elements = new_elements;  in mempool_resize()
343 	pool->min_nr = new_min_nr;  in mempool_resize()
345 	while (pool->curr_nr < pool->min_nr) {  in mempool_resize()
346 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
347 		element = pool->alloc(GFP_KERNEL, pool->pool_data);  in mempool_resize()
350 		spin_lock_irqsave(&pool->lock, flags);  in mempool_resize()
351 		if (pool->curr_nr < pool->min_nr) {  in mempool_resize()
352 			add_element(pool, element);  in mempool_resize()
354 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
355 			pool->free(element, pool->pool_data);	/* Raced */  in mempool_resize()
360 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_resize()
380 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)  in mempool_alloc()  argument
398 	element = pool->alloc(gfp_temp, pool->pool_data);  in mempool_alloc()
402 	spin_lock_irqsave(&pool->lock, flags);  in mempool_alloc()
403 	if (likely(pool->curr_nr)) {  in mempool_alloc()
404 		element = remove_element(pool);  in mempool_alloc()
405 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
421 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
428 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
434 	prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);  in mempool_alloc()
436 	spin_unlock_irqrestore(&pool->lock, flags);  in mempool_alloc()
444 	finish_wait(&pool->wait, &wait);  in mempool_alloc()
457 void mempool_free(void *element, mempool_t *pool)  in mempool_free()  argument
497 	if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {  in mempool_free()
498 		spin_lock_irqsave(&pool->lock, flags);  in mempool_free()
499 		if (likely(pool->curr_nr < pool->min_nr)) {  in mempool_free()
500 			add_element(pool, element);  in mempool_free()
501 			spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
502 			wake_up(&pool->wait);  in mempool_free()
505 		spin_unlock_irqrestore(&pool->lock, flags);  in mempool_free()
507 	pool->free(element, pool->pool_data);  in mempool_free()