1 /* 2 * linux/mm/mempool.c 3 * 4 * memory buffer pool support. Such pools are mostly used 5 * for guaranteed, deadlock-free memory allocations during 6 * extreme VM load. 7 * 8 * started by Ingo Molnar, Copyright (C) 2001 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/slab.h> 13 #include <linux/export.h> 14 #include <linux/mempool.h> 15 #include <linux/blkdev.h> 16 #include <linux/writeback.h> 17 18 static void add_element(mempool_t *pool, void *element) 19 { 20 BUG_ON(pool->curr_nr >= pool->min_nr); 21 pool->elements[pool->curr_nr++] = element; 22 } 23 24 static void *remove_element(mempool_t *pool) 25 { 26 BUG_ON(pool->curr_nr <= 0); 27 return pool->elements[--pool->curr_nr]; 28 } 29 30 /** 31 * mempool_destroy - deallocate a memory pool 32 * @pool: pointer to the memory pool which was allocated via 33 * mempool_create(). 34 * 35 * Free all reserved elements in @pool and @pool itself. This function 36 * only sleeps if the free_fn() function sleeps. 37 */ 38 void mempool_destroy(mempool_t *pool) 39 { 40 while (pool->curr_nr) { 41 void *element = remove_element(pool); 42 pool->free(element, pool->pool_data); 43 } 44 kfree(pool->elements); 45 kfree(pool); 46 } 47 EXPORT_SYMBOL(mempool_destroy); 48 49 /** 50 * mempool_create - create a memory pool 51 * @min_nr: the minimum number of elements guaranteed to be 52 * allocated for this pool. 53 * @alloc_fn: user-defined element-allocation function. 54 * @free_fn: user-defined element-freeing function. 55 * @pool_data: optional private data available to the user-defined functions. 56 * 57 * this function creates and allocates a guaranteed size, preallocated 58 * memory pool. The pool can be used from the mempool_alloc() and mempool_free() 59 * functions. This function might sleep. Both the alloc_fn() and the free_fn() 60 * functions might sleep - as long as the mempool_alloc() function is not called 61 * from IRQ contexts. 62 */ 63 mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn, 64 mempool_free_t *free_fn, void *pool_data) 65 { 66 return mempool_create_node(min_nr,alloc_fn,free_fn, pool_data,-1); 67 } 68 EXPORT_SYMBOL(mempool_create); 69 70 mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn, 71 mempool_free_t *free_fn, void *pool_data, int node_id) 72 { 73 mempool_t *pool; 74 pool = kmalloc_node(sizeof(*pool), GFP_KERNEL | __GFP_ZERO, node_id); 75 if (!pool) 76 return NULL; 77 pool->elements = kmalloc_node(min_nr * sizeof(void *), 78 GFP_KERNEL, node_id); 79 if (!pool->elements) { 80 kfree(pool); 81 return NULL; 82 } 83 spin_lock_init(&pool->lock); 84 pool->min_nr = min_nr; 85 pool->pool_data = pool_data; 86 init_waitqueue_head(&pool->wait); 87 pool->alloc = alloc_fn; 88 pool->free = free_fn; 89 90 /* 91 * First pre-allocate the guaranteed number of buffers. 92 */ 93 while (pool->curr_nr < pool->min_nr) { 94 void *element; 95 96 element = pool->alloc(GFP_KERNEL, pool->pool_data); 97 if (unlikely(!element)) { 98 mempool_destroy(pool); 99 return NULL; 100 } 101 add_element(pool, element); 102 } 103 return pool; 104 } 105 EXPORT_SYMBOL(mempool_create_node); 106 107 /** 108 * mempool_resize - resize an existing memory pool 109 * @pool: pointer to the memory pool which was allocated via 110 * mempool_create(). 111 * @new_min_nr: the new minimum number of elements guaranteed to be 112 * allocated for this pool. 113 * @gfp_mask: the usual allocation bitmask. 114 * 115 * This function shrinks/grows the pool. In the case of growing, 116 * it cannot be guaranteed that the pool will be grown to the new 117 * size immediately, but new mempool_free() calls will refill it. 118 * 119 * Note, the caller must guarantee that no mempool_destroy is called 120 * while this function is running. mempool_alloc() & mempool_free() 121 * might be called (eg. from IRQ contexts) while this function executes. 122 */ 123 int mempool_resize(mempool_t *pool, int new_min_nr, gfp_t gfp_mask) 124 { 125 void *element; 126 void **new_elements; 127 unsigned long flags; 128 129 BUG_ON(new_min_nr <= 0); 130 131 spin_lock_irqsave(&pool->lock, flags); 132 if (new_min_nr <= pool->min_nr) { 133 while (new_min_nr < pool->curr_nr) { 134 element = remove_element(pool); 135 spin_unlock_irqrestore(&pool->lock, flags); 136 pool->free(element, pool->pool_data); 137 spin_lock_irqsave(&pool->lock, flags); 138 } 139 pool->min_nr = new_min_nr; 140 goto out_unlock; 141 } 142 spin_unlock_irqrestore(&pool->lock, flags); 143 144 /* Grow the pool */ 145 new_elements = kmalloc(new_min_nr * sizeof(*new_elements), gfp_mask); 146 if (!new_elements) 147 return -ENOMEM; 148 149 spin_lock_irqsave(&pool->lock, flags); 150 if (unlikely(new_min_nr <= pool->min_nr)) { 151 /* Raced, other resize will do our work */ 152 spin_unlock_irqrestore(&pool->lock, flags); 153 kfree(new_elements); 154 goto out; 155 } 156 memcpy(new_elements, pool->elements, 157 pool->curr_nr * sizeof(*new_elements)); 158 kfree(pool->elements); 159 pool->elements = new_elements; 160 pool->min_nr = new_min_nr; 161 162 while (pool->curr_nr < pool->min_nr) { 163 spin_unlock_irqrestore(&pool->lock, flags); 164 element = pool->alloc(gfp_mask, pool->pool_data); 165 if (!element) 166 goto out; 167 spin_lock_irqsave(&pool->lock, flags); 168 if (pool->curr_nr < pool->min_nr) { 169 add_element(pool, element); 170 } else { 171 spin_unlock_irqrestore(&pool->lock, flags); 172 pool->free(element, pool->pool_data); /* Raced */ 173 goto out; 174 } 175 } 176 out_unlock: 177 spin_unlock_irqrestore(&pool->lock, flags); 178 out: 179 return 0; 180 } 181 EXPORT_SYMBOL(mempool_resize); 182 183 /** 184 * mempool_alloc - allocate an element from a specific memory pool 185 * @pool: pointer to the memory pool which was allocated via 186 * mempool_create(). 187 * @gfp_mask: the usual allocation bitmask. 188 * 189 * this function only sleeps if the alloc_fn() function sleeps or 190 * returns NULL. Note that due to preallocation, this function 191 * *never* fails when called from process contexts. (it might 192 * fail if called from an IRQ context.) 193 */ 194 void * mempool_alloc(mempool_t *pool, gfp_t gfp_mask) 195 { 196 void *element; 197 unsigned long flags; 198 wait_queue_t wait; 199 gfp_t gfp_temp; 200 201 might_sleep_if(gfp_mask & __GFP_WAIT); 202 203 gfp_mask |= __GFP_NOMEMALLOC; /* don't allocate emergency reserves */ 204 gfp_mask |= __GFP_NORETRY; /* don't loop in __alloc_pages */ 205 gfp_mask |= __GFP_NOWARN; /* failures are OK */ 206 207 gfp_temp = gfp_mask & ~(__GFP_WAIT|__GFP_IO); 208 209 repeat_alloc: 210 211 element = pool->alloc(gfp_temp, pool->pool_data); 212 if (likely(element != NULL)) 213 return element; 214 215 spin_lock_irqsave(&pool->lock, flags); 216 if (likely(pool->curr_nr)) { 217 element = remove_element(pool); 218 spin_unlock_irqrestore(&pool->lock, flags); 219 /* paired with rmb in mempool_free(), read comment there */ 220 smp_wmb(); 221 return element; 222 } 223 224 /* 225 * We use gfp mask w/o __GFP_WAIT or IO for the first round. If 226 * alloc failed with that and @pool was empty, retry immediately. 227 */ 228 if (gfp_temp != gfp_mask) { 229 spin_unlock_irqrestore(&pool->lock, flags); 230 gfp_temp = gfp_mask; 231 goto repeat_alloc; 232 } 233 234 /* We must not sleep if !__GFP_WAIT */ 235 if (!(gfp_mask & __GFP_WAIT)) { 236 spin_unlock_irqrestore(&pool->lock, flags); 237 return NULL; 238 } 239 240 /* Let's wait for someone else to return an element to @pool */ 241 init_wait(&wait); 242 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE); 243 244 spin_unlock_irqrestore(&pool->lock, flags); 245 246 /* 247 * FIXME: this should be io_schedule(). The timeout is there as a 248 * workaround for some DM problems in 2.6.18. 249 */ 250 io_schedule_timeout(5*HZ); 251 252 finish_wait(&pool->wait, &wait); 253 goto repeat_alloc; 254 } 255 EXPORT_SYMBOL(mempool_alloc); 256 257 /** 258 * mempool_free - return an element to the pool. 259 * @element: pool element pointer. 260 * @pool: pointer to the memory pool which was allocated via 261 * mempool_create(). 262 * 263 * this function only sleeps if the free_fn() function sleeps. 264 */ 265 void mempool_free(void *element, mempool_t *pool) 266 { 267 unsigned long flags; 268 269 if (unlikely(element == NULL)) 270 return; 271 272 /* 273 * Paired with the wmb in mempool_alloc(). The preceding read is 274 * for @element and the following @pool->curr_nr. This ensures 275 * that the visible value of @pool->curr_nr is from after the 276 * allocation of @element. This is necessary for fringe cases 277 * where @element was passed to this task without going through 278 * barriers. 279 * 280 * For example, assume @p is %NULL at the beginning and one task 281 * performs "p = mempool_alloc(...);" while another task is doing 282 * "while (!p) cpu_relax(); mempool_free(p, ...);". This function 283 * may end up using curr_nr value which is from before allocation 284 * of @p without the following rmb. 285 */ 286 smp_rmb(); 287 288 /* 289 * For correctness, we need a test which is guaranteed to trigger 290 * if curr_nr + #allocated == min_nr. Testing curr_nr < min_nr 291 * without locking achieves that and refilling as soon as possible 292 * is desirable. 293 * 294 * Because curr_nr visible here is always a value after the 295 * allocation of @element, any task which decremented curr_nr below 296 * min_nr is guaranteed to see curr_nr < min_nr unless curr_nr gets 297 * incremented to min_nr afterwards. If curr_nr gets incremented 298 * to min_nr after the allocation of @element, the elements 299 * allocated after that are subject to the same guarantee. 300 * 301 * Waiters happen iff curr_nr is 0 and the above guarantee also 302 * ensures that there will be frees which return elements to the 303 * pool waking up the waiters. 304 */ 305 if (pool->curr_nr < pool->min_nr) { 306 spin_lock_irqsave(&pool->lock, flags); 307 if (pool->curr_nr < pool->min_nr) { 308 add_element(pool, element); 309 spin_unlock_irqrestore(&pool->lock, flags); 310 wake_up(&pool->wait); 311 return; 312 } 313 spin_unlock_irqrestore(&pool->lock, flags); 314 } 315 pool->free(element, pool->pool_data); 316 } 317 EXPORT_SYMBOL(mempool_free); 318 319 /* 320 * A commonly used alloc and free fn. 321 */ 322 void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data) 323 { 324 struct kmem_cache *mem = pool_data; 325 return kmem_cache_alloc(mem, gfp_mask); 326 } 327 EXPORT_SYMBOL(mempool_alloc_slab); 328 329 void mempool_free_slab(void *element, void *pool_data) 330 { 331 struct kmem_cache *mem = pool_data; 332 kmem_cache_free(mem, element); 333 } 334 EXPORT_SYMBOL(mempool_free_slab); 335 336 /* 337 * A commonly used alloc and free fn that kmalloc/kfrees the amount of memory 338 * specified by pool_data 339 */ 340 void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data) 341 { 342 size_t size = (size_t)pool_data; 343 return kmalloc(size, gfp_mask); 344 } 345 EXPORT_SYMBOL(mempool_kmalloc); 346 347 void mempool_kfree(void *element, void *pool_data) 348 { 349 kfree(element); 350 } 351 EXPORT_SYMBOL(mempool_kfree); 352 353 /* 354 * A simple mempool-backed page allocator that allocates pages 355 * of the order specified by pool_data. 356 */ 357 void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data) 358 { 359 int order = (int)(long)pool_data; 360 return alloc_pages(gfp_mask, order); 361 } 362 EXPORT_SYMBOL(mempool_alloc_pages); 363 364 void mempool_free_pages(void *element, void *pool_data) 365 { 366 int order = (int)(long)pool_data; 367 __free_pages(element, order); 368 } 369 EXPORT_SYMBOL(mempool_free_pages); 370