13bd94003SHeinz Mauelshagen // SPDX-License-Identifier: GPL-2.0-only
2742c8fdcSJoe Thornber /*
3742c8fdcSJoe Thornber * Copyright (C) 2012 Red Hat, Inc.
4742c8fdcSJoe Thornber *
5742c8fdcSJoe Thornber * This file is released under the GPL.
6742c8fdcSJoe Thornber */
7742c8fdcSJoe Thornber
8742c8fdcSJoe Thornber #include "dm.h"
9742c8fdcSJoe Thornber #include "dm-bio-prison-v1.h"
10742c8fdcSJoe Thornber #include "dm-bio-prison-v2.h"
11742c8fdcSJoe Thornber
12742c8fdcSJoe Thornber #include <linux/spinlock.h>
13742c8fdcSJoe Thornber #include <linux/mempool.h>
14742c8fdcSJoe Thornber #include <linux/module.h>
15742c8fdcSJoe Thornber #include <linux/slab.h>
16742c8fdcSJoe Thornber
17742c8fdcSJoe Thornber /*----------------------------------------------------------------*/
18742c8fdcSJoe Thornber
19742c8fdcSJoe Thornber #define MIN_CELLS 1024
20742c8fdcSJoe Thornber
21e2dd8acaSJoe Thornber struct prison_region {
22742c8fdcSJoe Thornber spinlock_t lock;
23e2dd8acaSJoe Thornber struct rb_root cell;
24e2dd8acaSJoe Thornber } ____cacheline_aligned_in_smp;
25e2dd8acaSJoe Thornber
26e2dd8acaSJoe Thornber struct dm_bio_prison {
2772d711c8SMike Snitzer mempool_t cell_pool;
28c6273411SMike Snitzer unsigned int num_locks;
29b6279f82SMike Snitzer struct prison_region regions[];
30742c8fdcSJoe Thornber };
31742c8fdcSJoe Thornber
32742c8fdcSJoe Thornber static struct kmem_cache *_cell_cache;
33742c8fdcSJoe Thornber
34742c8fdcSJoe Thornber /*----------------------------------------------------------------*/
35742c8fdcSJoe Thornber
36742c8fdcSJoe Thornber /*
37742c8fdcSJoe Thornber * @nr_cells should be the number of cells you want in use _concurrently_.
38742c8fdcSJoe Thornber * Don't confuse it with the number of distinct keys.
39742c8fdcSJoe Thornber */
dm_bio_prison_create(void)40742c8fdcSJoe Thornber struct dm_bio_prison *dm_bio_prison_create(void)
41742c8fdcSJoe Thornber {
426f1c819cSKent Overstreet int ret;
43b6279f82SMike Snitzer unsigned int i, num_locks;
44b6279f82SMike Snitzer struct dm_bio_prison *prison;
45742c8fdcSJoe Thornber
46b6279f82SMike Snitzer num_locks = dm_num_hash_locks();
47b6279f82SMike Snitzer prison = kzalloc(struct_size(prison, regions, num_locks), GFP_KERNEL);
48742c8fdcSJoe Thornber if (!prison)
49742c8fdcSJoe Thornber return NULL;
50b6279f82SMike Snitzer prison->num_locks = num_locks;
51742c8fdcSJoe Thornber
52c6273411SMike Snitzer for (i = 0; i < prison->num_locks; i++) {
53e2dd8acaSJoe Thornber spin_lock_init(&prison->regions[i].lock);
54e2dd8acaSJoe Thornber prison->regions[i].cell = RB_ROOT;
55e2dd8acaSJoe Thornber }
56742c8fdcSJoe Thornber
576f1c819cSKent Overstreet ret = mempool_init_slab_pool(&prison->cell_pool, MIN_CELLS, _cell_cache);
586f1c819cSKent Overstreet if (ret) {
59742c8fdcSJoe Thornber kfree(prison);
60742c8fdcSJoe Thornber return NULL;
61742c8fdcSJoe Thornber }
62742c8fdcSJoe Thornber
63742c8fdcSJoe Thornber return prison;
64742c8fdcSJoe Thornber }
65742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_bio_prison_create);
66742c8fdcSJoe Thornber
dm_bio_prison_destroy(struct dm_bio_prison * prison)67742c8fdcSJoe Thornber void dm_bio_prison_destroy(struct dm_bio_prison *prison)
68742c8fdcSJoe Thornber {
696f1c819cSKent Overstreet mempool_exit(&prison->cell_pool);
70742c8fdcSJoe Thornber kfree(prison);
71742c8fdcSJoe Thornber }
72742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_bio_prison_destroy);
73742c8fdcSJoe Thornber
dm_bio_prison_alloc_cell(struct dm_bio_prison * prison,gfp_t gfp)74742c8fdcSJoe Thornber struct dm_bio_prison_cell *dm_bio_prison_alloc_cell(struct dm_bio_prison *prison, gfp_t gfp)
75742c8fdcSJoe Thornber {
766f1c819cSKent Overstreet return mempool_alloc(&prison->cell_pool, gfp);
77742c8fdcSJoe Thornber }
78742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_bio_prison_alloc_cell);
79742c8fdcSJoe Thornber
dm_bio_prison_free_cell(struct dm_bio_prison * prison,struct dm_bio_prison_cell * cell)80742c8fdcSJoe Thornber void dm_bio_prison_free_cell(struct dm_bio_prison *prison,
81742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell)
82742c8fdcSJoe Thornber {
836f1c819cSKent Overstreet mempool_free(cell, &prison->cell_pool);
84742c8fdcSJoe Thornber }
85742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_bio_prison_free_cell);
86742c8fdcSJoe Thornber
__setup_new_cell(struct dm_cell_key * key,struct bio * holder,struct dm_bio_prison_cell * cell)87742c8fdcSJoe Thornber static void __setup_new_cell(struct dm_cell_key *key,
88742c8fdcSJoe Thornber struct bio *holder,
89742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell)
90742c8fdcSJoe Thornber {
91742c8fdcSJoe Thornber memcpy(&cell->key, key, sizeof(cell->key));
92742c8fdcSJoe Thornber cell->holder = holder;
93742c8fdcSJoe Thornber bio_list_init(&cell->bios);
94742c8fdcSJoe Thornber }
95742c8fdcSJoe Thornber
cmp_keys(struct dm_cell_key * lhs,struct dm_cell_key * rhs)96742c8fdcSJoe Thornber static int cmp_keys(struct dm_cell_key *lhs,
97742c8fdcSJoe Thornber struct dm_cell_key *rhs)
98742c8fdcSJoe Thornber {
99742c8fdcSJoe Thornber if (lhs->virtual < rhs->virtual)
100742c8fdcSJoe Thornber return -1;
101742c8fdcSJoe Thornber
102742c8fdcSJoe Thornber if (lhs->virtual > rhs->virtual)
103742c8fdcSJoe Thornber return 1;
104742c8fdcSJoe Thornber
105742c8fdcSJoe Thornber if (lhs->dev < rhs->dev)
106742c8fdcSJoe Thornber return -1;
107742c8fdcSJoe Thornber
108742c8fdcSJoe Thornber if (lhs->dev > rhs->dev)
109742c8fdcSJoe Thornber return 1;
110742c8fdcSJoe Thornber
111742c8fdcSJoe Thornber if (lhs->block_end <= rhs->block_begin)
112742c8fdcSJoe Thornber return -1;
113742c8fdcSJoe Thornber
114742c8fdcSJoe Thornber if (lhs->block_begin >= rhs->block_end)
115742c8fdcSJoe Thornber return 1;
116742c8fdcSJoe Thornber
117742c8fdcSJoe Thornber return 0;
118742c8fdcSJoe Thornber }
119742c8fdcSJoe Thornber
lock_nr(struct dm_cell_key * key,unsigned int num_locks)120*363b7fd7SJoe Thornber static inline unsigned int lock_nr(struct dm_cell_key *key, unsigned int num_locks)
121e2dd8acaSJoe Thornber {
122*363b7fd7SJoe Thornber return dm_hash_locks_index((key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT),
123*363b7fd7SJoe Thornber num_locks);
124e2dd8acaSJoe Thornber }
125e2dd8acaSJoe Thornber
dm_cell_key_has_valid_range(struct dm_cell_key * key)1263f8d3f54SMike Snitzer bool dm_cell_key_has_valid_range(struct dm_cell_key *key)
127e2dd8acaSJoe Thornber {
1283f8d3f54SMike Snitzer if (WARN_ON_ONCE(key->block_end - key->block_begin > BIO_PRISON_MAX_RANGE))
1293f8d3f54SMike Snitzer return false;
1303f8d3f54SMike Snitzer if (WARN_ON_ONCE((key->block_begin >> BIO_PRISON_MAX_RANGE_SHIFT) !=
1313f8d3f54SMike Snitzer (key->block_end - 1) >> BIO_PRISON_MAX_RANGE_SHIFT))
1323f8d3f54SMike Snitzer return false;
1333f8d3f54SMike Snitzer
1343f8d3f54SMike Snitzer return true;
135e2dd8acaSJoe Thornber }
1363f8d3f54SMike Snitzer EXPORT_SYMBOL(dm_cell_key_has_valid_range);
137e2dd8acaSJoe Thornber
__bio_detain(struct rb_root * root,struct dm_cell_key * key,struct bio * inmate,struct dm_bio_prison_cell * cell_prealloc,struct dm_bio_prison_cell ** cell_result)138e2dd8acaSJoe Thornber static int __bio_detain(struct rb_root *root,
139742c8fdcSJoe Thornber struct dm_cell_key *key,
140742c8fdcSJoe Thornber struct bio *inmate,
141742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell_prealloc,
142742c8fdcSJoe Thornber struct dm_bio_prison_cell **cell_result)
143742c8fdcSJoe Thornber {
144742c8fdcSJoe Thornber int r;
145e2dd8acaSJoe Thornber struct rb_node **new = &root->rb_node, *parent = NULL;
146742c8fdcSJoe Thornber
147742c8fdcSJoe Thornber while (*new) {
148742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell =
1496e333d0bSGeliang Tang rb_entry(*new, struct dm_bio_prison_cell, node);
150742c8fdcSJoe Thornber
151742c8fdcSJoe Thornber r = cmp_keys(key, &cell->key);
152742c8fdcSJoe Thornber
153742c8fdcSJoe Thornber parent = *new;
154742c8fdcSJoe Thornber if (r < 0)
155742c8fdcSJoe Thornber new = &((*new)->rb_left);
156742c8fdcSJoe Thornber else if (r > 0)
157742c8fdcSJoe Thornber new = &((*new)->rb_right);
158742c8fdcSJoe Thornber else {
159742c8fdcSJoe Thornber if (inmate)
160742c8fdcSJoe Thornber bio_list_add(&cell->bios, inmate);
161742c8fdcSJoe Thornber *cell_result = cell;
162742c8fdcSJoe Thornber return 1;
163742c8fdcSJoe Thornber }
164742c8fdcSJoe Thornber }
165742c8fdcSJoe Thornber
166742c8fdcSJoe Thornber __setup_new_cell(key, inmate, cell_prealloc);
167742c8fdcSJoe Thornber *cell_result = cell_prealloc;
168742c8fdcSJoe Thornber
169742c8fdcSJoe Thornber rb_link_node(&cell_prealloc->node, parent, new);
170e2dd8acaSJoe Thornber rb_insert_color(&cell_prealloc->node, root);
171742c8fdcSJoe Thornber
172742c8fdcSJoe Thornber return 0;
173742c8fdcSJoe Thornber }
174742c8fdcSJoe Thornber
bio_detain(struct dm_bio_prison * prison,struct dm_cell_key * key,struct bio * inmate,struct dm_bio_prison_cell * cell_prealloc,struct dm_bio_prison_cell ** cell_result)175742c8fdcSJoe Thornber static int bio_detain(struct dm_bio_prison *prison,
176742c8fdcSJoe Thornber struct dm_cell_key *key,
177742c8fdcSJoe Thornber struct bio *inmate,
178742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell_prealloc,
179742c8fdcSJoe Thornber struct dm_bio_prison_cell **cell_result)
180742c8fdcSJoe Thornber {
181742c8fdcSJoe Thornber int r;
182c6273411SMike Snitzer unsigned l = lock_nr(key, prison->num_locks);
183742c8fdcSJoe Thornber
184e2dd8acaSJoe Thornber spin_lock_irq(&prison->regions[l].lock);
185e2dd8acaSJoe Thornber r = __bio_detain(&prison->regions[l].cell, key, inmate, cell_prealloc, cell_result);
186e2dd8acaSJoe Thornber spin_unlock_irq(&prison->regions[l].lock);
187742c8fdcSJoe Thornber
188742c8fdcSJoe Thornber return r;
189742c8fdcSJoe Thornber }
190742c8fdcSJoe Thornber
dm_bio_detain(struct dm_bio_prison * prison,struct dm_cell_key * key,struct bio * inmate,struct dm_bio_prison_cell * cell_prealloc,struct dm_bio_prison_cell ** cell_result)191742c8fdcSJoe Thornber int dm_bio_detain(struct dm_bio_prison *prison,
192742c8fdcSJoe Thornber struct dm_cell_key *key,
193742c8fdcSJoe Thornber struct bio *inmate,
194742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell_prealloc,
195742c8fdcSJoe Thornber struct dm_bio_prison_cell **cell_result)
196742c8fdcSJoe Thornber {
197742c8fdcSJoe Thornber return bio_detain(prison, key, inmate, cell_prealloc, cell_result);
198742c8fdcSJoe Thornber }
199742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_bio_detain);
200742c8fdcSJoe Thornber
dm_get_cell(struct dm_bio_prison * prison,struct dm_cell_key * key,struct dm_bio_prison_cell * cell_prealloc,struct dm_bio_prison_cell ** cell_result)201742c8fdcSJoe Thornber int dm_get_cell(struct dm_bio_prison *prison,
202742c8fdcSJoe Thornber struct dm_cell_key *key,
203742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell_prealloc,
204742c8fdcSJoe Thornber struct dm_bio_prison_cell **cell_result)
205742c8fdcSJoe Thornber {
206742c8fdcSJoe Thornber return bio_detain(prison, key, NULL, cell_prealloc, cell_result);
207742c8fdcSJoe Thornber }
208742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_get_cell);
209742c8fdcSJoe Thornber
210742c8fdcSJoe Thornber /*
211742c8fdcSJoe Thornber * @inmates must have been initialised prior to this call
212742c8fdcSJoe Thornber */
__cell_release(struct rb_root * root,struct dm_bio_prison_cell * cell,struct bio_list * inmates)213e2dd8acaSJoe Thornber static void __cell_release(struct rb_root *root,
214742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell,
215742c8fdcSJoe Thornber struct bio_list *inmates)
216742c8fdcSJoe Thornber {
217e2dd8acaSJoe Thornber rb_erase(&cell->node, root);
218742c8fdcSJoe Thornber
219742c8fdcSJoe Thornber if (inmates) {
220742c8fdcSJoe Thornber if (cell->holder)
221742c8fdcSJoe Thornber bio_list_add(inmates, cell->holder);
222742c8fdcSJoe Thornber bio_list_merge(inmates, &cell->bios);
223742c8fdcSJoe Thornber }
224742c8fdcSJoe Thornber }
225742c8fdcSJoe Thornber
dm_cell_release(struct dm_bio_prison * prison,struct dm_bio_prison_cell * cell,struct bio_list * bios)226742c8fdcSJoe Thornber void dm_cell_release(struct dm_bio_prison *prison,
227742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell,
228742c8fdcSJoe Thornber struct bio_list *bios)
229742c8fdcSJoe Thornber {
230c6273411SMike Snitzer unsigned l = lock_nr(&cell->key, prison->num_locks);
231e2dd8acaSJoe Thornber
232e2dd8acaSJoe Thornber spin_lock_irq(&prison->regions[l].lock);
233e2dd8acaSJoe Thornber __cell_release(&prison->regions[l].cell, cell, bios);
234e2dd8acaSJoe Thornber spin_unlock_irq(&prison->regions[l].lock);
235742c8fdcSJoe Thornber }
236742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_cell_release);
237742c8fdcSJoe Thornber
238742c8fdcSJoe Thornber /*
239742c8fdcSJoe Thornber * Sometimes we don't want the holder, just the additional bios.
240742c8fdcSJoe Thornber */
__cell_release_no_holder(struct rb_root * root,struct dm_bio_prison_cell * cell,struct bio_list * inmates)241e2dd8acaSJoe Thornber static void __cell_release_no_holder(struct rb_root *root,
242742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell,
243742c8fdcSJoe Thornber struct bio_list *inmates)
244742c8fdcSJoe Thornber {
245e2dd8acaSJoe Thornber rb_erase(&cell->node, root);
246742c8fdcSJoe Thornber bio_list_merge(inmates, &cell->bios);
247742c8fdcSJoe Thornber }
248742c8fdcSJoe Thornber
dm_cell_release_no_holder(struct dm_bio_prison * prison,struct dm_bio_prison_cell * cell,struct bio_list * inmates)249742c8fdcSJoe Thornber void dm_cell_release_no_holder(struct dm_bio_prison *prison,
250742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell,
251742c8fdcSJoe Thornber struct bio_list *inmates)
252742c8fdcSJoe Thornber {
253c6273411SMike Snitzer unsigned l = lock_nr(&cell->key, prison->num_locks);
254742c8fdcSJoe Thornber unsigned long flags;
255742c8fdcSJoe Thornber
256e2dd8acaSJoe Thornber spin_lock_irqsave(&prison->regions[l].lock, flags);
257e2dd8acaSJoe Thornber __cell_release_no_holder(&prison->regions[l].cell, cell, inmates);
258e2dd8acaSJoe Thornber spin_unlock_irqrestore(&prison->regions[l].lock, flags);
259742c8fdcSJoe Thornber }
260742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_cell_release_no_holder);
261742c8fdcSJoe Thornber
dm_cell_error(struct dm_bio_prison * prison,struct dm_bio_prison_cell * cell,blk_status_t error)262742c8fdcSJoe Thornber void dm_cell_error(struct dm_bio_prison *prison,
2634e4cbee9SChristoph Hellwig struct dm_bio_prison_cell *cell, blk_status_t error)
264742c8fdcSJoe Thornber {
265742c8fdcSJoe Thornber struct bio_list bios;
266742c8fdcSJoe Thornber struct bio *bio;
267742c8fdcSJoe Thornber
268742c8fdcSJoe Thornber bio_list_init(&bios);
269742c8fdcSJoe Thornber dm_cell_release(prison, cell, &bios);
270742c8fdcSJoe Thornber
271742c8fdcSJoe Thornber while ((bio = bio_list_pop(&bios))) {
2724e4cbee9SChristoph Hellwig bio->bi_status = error;
273742c8fdcSJoe Thornber bio_endio(bio);
274742c8fdcSJoe Thornber }
275742c8fdcSJoe Thornber }
276742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_cell_error);
277742c8fdcSJoe Thornber
dm_cell_visit_release(struct dm_bio_prison * prison,void (* visit_fn)(void *,struct dm_bio_prison_cell *),void * context,struct dm_bio_prison_cell * cell)278742c8fdcSJoe Thornber void dm_cell_visit_release(struct dm_bio_prison *prison,
279742c8fdcSJoe Thornber void (*visit_fn)(void *, struct dm_bio_prison_cell *),
280742c8fdcSJoe Thornber void *context,
281742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell)
282742c8fdcSJoe Thornber {
283c6273411SMike Snitzer unsigned l = lock_nr(&cell->key, prison->num_locks);
284e2dd8acaSJoe Thornber spin_lock_irq(&prison->regions[l].lock);
285742c8fdcSJoe Thornber visit_fn(context, cell);
286e2dd8acaSJoe Thornber rb_erase(&cell->node, &prison->regions[l].cell);
287e2dd8acaSJoe Thornber spin_unlock_irq(&prison->regions[l].lock);
288742c8fdcSJoe Thornber }
289742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_cell_visit_release);
290742c8fdcSJoe Thornber
__promote_or_release(struct rb_root * root,struct dm_bio_prison_cell * cell)291e2dd8acaSJoe Thornber static int __promote_or_release(struct rb_root *root,
292742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell)
293742c8fdcSJoe Thornber {
294742c8fdcSJoe Thornber if (bio_list_empty(&cell->bios)) {
295e2dd8acaSJoe Thornber rb_erase(&cell->node, root);
296742c8fdcSJoe Thornber return 1;
297742c8fdcSJoe Thornber }
298742c8fdcSJoe Thornber
299742c8fdcSJoe Thornber cell->holder = bio_list_pop(&cell->bios);
300742c8fdcSJoe Thornber return 0;
301742c8fdcSJoe Thornber }
302742c8fdcSJoe Thornber
dm_cell_promote_or_release(struct dm_bio_prison * prison,struct dm_bio_prison_cell * cell)303742c8fdcSJoe Thornber int dm_cell_promote_or_release(struct dm_bio_prison *prison,
304742c8fdcSJoe Thornber struct dm_bio_prison_cell *cell)
305742c8fdcSJoe Thornber {
306742c8fdcSJoe Thornber int r;
307c6273411SMike Snitzer unsigned l = lock_nr(&cell->key, prison->num_locks);
308742c8fdcSJoe Thornber
309e2dd8acaSJoe Thornber spin_lock_irq(&prison->regions[l].lock);
310e2dd8acaSJoe Thornber r = __promote_or_release(&prison->regions[l].cell, cell);
311e2dd8acaSJoe Thornber spin_unlock_irq(&prison->regions[l].lock);
312742c8fdcSJoe Thornber
313742c8fdcSJoe Thornber return r;
314742c8fdcSJoe Thornber }
315742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_cell_promote_or_release);
316742c8fdcSJoe Thornber
317742c8fdcSJoe Thornber /*----------------------------------------------------------------*/
318742c8fdcSJoe Thornber
319742c8fdcSJoe Thornber #define DEFERRED_SET_SIZE 64
320742c8fdcSJoe Thornber
321742c8fdcSJoe Thornber struct dm_deferred_entry {
322742c8fdcSJoe Thornber struct dm_deferred_set *ds;
32386a3238cSHeinz Mauelshagen unsigned int count;
324742c8fdcSJoe Thornber struct list_head work_items;
325742c8fdcSJoe Thornber };
326742c8fdcSJoe Thornber
327742c8fdcSJoe Thornber struct dm_deferred_set {
328742c8fdcSJoe Thornber spinlock_t lock;
32986a3238cSHeinz Mauelshagen unsigned int current_entry;
33086a3238cSHeinz Mauelshagen unsigned int sweeper;
331742c8fdcSJoe Thornber struct dm_deferred_entry entries[DEFERRED_SET_SIZE];
332742c8fdcSJoe Thornber };
333742c8fdcSJoe Thornber
dm_deferred_set_create(void)334742c8fdcSJoe Thornber struct dm_deferred_set *dm_deferred_set_create(void)
335742c8fdcSJoe Thornber {
336742c8fdcSJoe Thornber int i;
337742c8fdcSJoe Thornber struct dm_deferred_set *ds;
338742c8fdcSJoe Thornber
339742c8fdcSJoe Thornber ds = kmalloc(sizeof(*ds), GFP_KERNEL);
340742c8fdcSJoe Thornber if (!ds)
341742c8fdcSJoe Thornber return NULL;
342742c8fdcSJoe Thornber
343742c8fdcSJoe Thornber spin_lock_init(&ds->lock);
344742c8fdcSJoe Thornber ds->current_entry = 0;
345742c8fdcSJoe Thornber ds->sweeper = 0;
346742c8fdcSJoe Thornber for (i = 0; i < DEFERRED_SET_SIZE; i++) {
347742c8fdcSJoe Thornber ds->entries[i].ds = ds;
348742c8fdcSJoe Thornber ds->entries[i].count = 0;
349742c8fdcSJoe Thornber INIT_LIST_HEAD(&ds->entries[i].work_items);
350742c8fdcSJoe Thornber }
351742c8fdcSJoe Thornber
352742c8fdcSJoe Thornber return ds;
353742c8fdcSJoe Thornber }
354742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_deferred_set_create);
355742c8fdcSJoe Thornber
dm_deferred_set_destroy(struct dm_deferred_set * ds)356742c8fdcSJoe Thornber void dm_deferred_set_destroy(struct dm_deferred_set *ds)
357742c8fdcSJoe Thornber {
358742c8fdcSJoe Thornber kfree(ds);
359742c8fdcSJoe Thornber }
360742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_deferred_set_destroy);
361742c8fdcSJoe Thornber
dm_deferred_entry_inc(struct dm_deferred_set * ds)362742c8fdcSJoe Thornber struct dm_deferred_entry *dm_deferred_entry_inc(struct dm_deferred_set *ds)
363742c8fdcSJoe Thornber {
364742c8fdcSJoe Thornber unsigned long flags;
365742c8fdcSJoe Thornber struct dm_deferred_entry *entry;
366742c8fdcSJoe Thornber
367742c8fdcSJoe Thornber spin_lock_irqsave(&ds->lock, flags);
368742c8fdcSJoe Thornber entry = ds->entries + ds->current_entry;
369742c8fdcSJoe Thornber entry->count++;
370742c8fdcSJoe Thornber spin_unlock_irqrestore(&ds->lock, flags);
371742c8fdcSJoe Thornber
372742c8fdcSJoe Thornber return entry;
373742c8fdcSJoe Thornber }
374742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_deferred_entry_inc);
375742c8fdcSJoe Thornber
ds_next(unsigned int index)37686a3238cSHeinz Mauelshagen static unsigned int ds_next(unsigned int index)
377742c8fdcSJoe Thornber {
378742c8fdcSJoe Thornber return (index + 1) % DEFERRED_SET_SIZE;
379742c8fdcSJoe Thornber }
380742c8fdcSJoe Thornber
__sweep(struct dm_deferred_set * ds,struct list_head * head)381742c8fdcSJoe Thornber static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
382742c8fdcSJoe Thornber {
383742c8fdcSJoe Thornber while ((ds->sweeper != ds->current_entry) &&
384742c8fdcSJoe Thornber !ds->entries[ds->sweeper].count) {
385742c8fdcSJoe Thornber list_splice_init(&ds->entries[ds->sweeper].work_items, head);
386742c8fdcSJoe Thornber ds->sweeper = ds_next(ds->sweeper);
387742c8fdcSJoe Thornber }
388742c8fdcSJoe Thornber
389742c8fdcSJoe Thornber if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
390742c8fdcSJoe Thornber list_splice_init(&ds->entries[ds->sweeper].work_items, head);
391742c8fdcSJoe Thornber }
392742c8fdcSJoe Thornber
dm_deferred_entry_dec(struct dm_deferred_entry * entry,struct list_head * head)393742c8fdcSJoe Thornber void dm_deferred_entry_dec(struct dm_deferred_entry *entry, struct list_head *head)
394742c8fdcSJoe Thornber {
395742c8fdcSJoe Thornber unsigned long flags;
396742c8fdcSJoe Thornber
397742c8fdcSJoe Thornber spin_lock_irqsave(&entry->ds->lock, flags);
398742c8fdcSJoe Thornber BUG_ON(!entry->count);
399742c8fdcSJoe Thornber --entry->count;
400742c8fdcSJoe Thornber __sweep(entry->ds, head);
401742c8fdcSJoe Thornber spin_unlock_irqrestore(&entry->ds->lock, flags);
402742c8fdcSJoe Thornber }
403742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_deferred_entry_dec);
404742c8fdcSJoe Thornber
405742c8fdcSJoe Thornber /*
406742c8fdcSJoe Thornber * Returns 1 if deferred or 0 if no pending items to delay job.
407742c8fdcSJoe Thornber */
dm_deferred_set_add_work(struct dm_deferred_set * ds,struct list_head * work)408742c8fdcSJoe Thornber int dm_deferred_set_add_work(struct dm_deferred_set *ds, struct list_head *work)
409742c8fdcSJoe Thornber {
410742c8fdcSJoe Thornber int r = 1;
41186a3238cSHeinz Mauelshagen unsigned int next_entry;
412742c8fdcSJoe Thornber
413235bc861SMikulas Patocka spin_lock_irq(&ds->lock);
414742c8fdcSJoe Thornber if ((ds->sweeper == ds->current_entry) &&
415742c8fdcSJoe Thornber !ds->entries[ds->current_entry].count)
416742c8fdcSJoe Thornber r = 0;
417742c8fdcSJoe Thornber else {
418742c8fdcSJoe Thornber list_add(work, &ds->entries[ds->current_entry].work_items);
419742c8fdcSJoe Thornber next_entry = ds_next(ds->current_entry);
420742c8fdcSJoe Thornber if (!ds->entries[next_entry].count)
421742c8fdcSJoe Thornber ds->current_entry = next_entry;
422742c8fdcSJoe Thornber }
423235bc861SMikulas Patocka spin_unlock_irq(&ds->lock);
424742c8fdcSJoe Thornber
425742c8fdcSJoe Thornber return r;
426742c8fdcSJoe Thornber }
427742c8fdcSJoe Thornber EXPORT_SYMBOL_GPL(dm_deferred_set_add_work);
428742c8fdcSJoe Thornber
429742c8fdcSJoe Thornber /*----------------------------------------------------------------*/
430742c8fdcSJoe Thornber
dm_bio_prison_init_v1(void)431742c8fdcSJoe Thornber static int __init dm_bio_prison_init_v1(void)
432742c8fdcSJoe Thornber {
433742c8fdcSJoe Thornber _cell_cache = KMEM_CACHE(dm_bio_prison_cell, 0);
434742c8fdcSJoe Thornber if (!_cell_cache)
435742c8fdcSJoe Thornber return -ENOMEM;
436742c8fdcSJoe Thornber
437742c8fdcSJoe Thornber return 0;
438742c8fdcSJoe Thornber }
439742c8fdcSJoe Thornber
dm_bio_prison_exit_v1(void)440742c8fdcSJoe Thornber static void dm_bio_prison_exit_v1(void)
441742c8fdcSJoe Thornber {
442742c8fdcSJoe Thornber kmem_cache_destroy(_cell_cache);
443742c8fdcSJoe Thornber _cell_cache = NULL;
444742c8fdcSJoe Thornber }
445742c8fdcSJoe Thornber
446742c8fdcSJoe Thornber static int (*_inits[])(void) __initdata = {
447742c8fdcSJoe Thornber dm_bio_prison_init_v1,
448742c8fdcSJoe Thornber dm_bio_prison_init_v2,
449742c8fdcSJoe Thornber };
450742c8fdcSJoe Thornber
451742c8fdcSJoe Thornber static void (*_exits[])(void) = {
452742c8fdcSJoe Thornber dm_bio_prison_exit_v1,
453742c8fdcSJoe Thornber dm_bio_prison_exit_v2,
454742c8fdcSJoe Thornber };
455742c8fdcSJoe Thornber
dm_bio_prison_init(void)456742c8fdcSJoe Thornber static int __init dm_bio_prison_init(void)
457742c8fdcSJoe Thornber {
458742c8fdcSJoe Thornber const int count = ARRAY_SIZE(_inits);
459742c8fdcSJoe Thornber
460742c8fdcSJoe Thornber int r, i;
461742c8fdcSJoe Thornber
462742c8fdcSJoe Thornber for (i = 0; i < count; i++) {
463742c8fdcSJoe Thornber r = _inits[i]();
464742c8fdcSJoe Thornber if (r)
465742c8fdcSJoe Thornber goto bad;
466742c8fdcSJoe Thornber }
467742c8fdcSJoe Thornber
468742c8fdcSJoe Thornber return 0;
469742c8fdcSJoe Thornber
470742c8fdcSJoe Thornber bad:
471742c8fdcSJoe Thornber while (i--)
472742c8fdcSJoe Thornber _exits[i]();
473742c8fdcSJoe Thornber
474742c8fdcSJoe Thornber return r;
475742c8fdcSJoe Thornber }
476742c8fdcSJoe Thornber
dm_bio_prison_exit(void)477742c8fdcSJoe Thornber static void __exit dm_bio_prison_exit(void)
478742c8fdcSJoe Thornber {
479742c8fdcSJoe Thornber int i = ARRAY_SIZE(_exits);
480742c8fdcSJoe Thornber
481742c8fdcSJoe Thornber while (i--)
482742c8fdcSJoe Thornber _exits[i]();
483742c8fdcSJoe Thornber }
484742c8fdcSJoe Thornber
485742c8fdcSJoe Thornber /*
486742c8fdcSJoe Thornber * module hooks
487742c8fdcSJoe Thornber */
488742c8fdcSJoe Thornber module_init(dm_bio_prison_init);
489742c8fdcSJoe Thornber module_exit(dm_bio_prison_exit);
490742c8fdcSJoe Thornber
491742c8fdcSJoe Thornber MODULE_DESCRIPTION(DM_NAME " bio prison");
492742c8fdcSJoe Thornber MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
493742c8fdcSJoe Thornber MODULE_LICENSE("GPL");
494