dm.c (64f52b0e31489b46465cff2e61ab2e1f60a3b4eb) dm.c (dde1e1ec4c08c3b7a4cee6cdad53948680d9fe39)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 135 unchanged lines hidden (view full) ---

144
145#define DM_NUMA_NODE NUMA_NO_NODE
146static int dm_numa_node = DM_NUMA_NODE;
147
148/*
149 * For mempools pre-allocation at the table loading time.
150 */
151struct dm_md_mempools {
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 135 unchanged lines hidden (view full) ---

144
145#define DM_NUMA_NODE NUMA_NO_NODE
146static int dm_numa_node = DM_NUMA_NODE;
147
148/*
149 * For mempools pre-allocation at the table loading time.
150 */
151struct dm_md_mempools {
152 mempool_t *io_pool;
153 struct bio_set *bs;
154 struct bio_set *io_bs;
155};
156
157struct table_device {
158 struct list_head list;
159 refcount_t count;
160 struct dm_dev dm_dev;
161};
162
152 struct bio_set *bs;
153 struct bio_set *io_bs;
154};
155
156struct table_device {
157 struct list_head list;
158 refcount_t count;
159 struct dm_dev dm_dev;
160};
161
163static struct kmem_cache *_io_cache;
164static struct kmem_cache *_rq_tio_cache;
165static struct kmem_cache *_rq_cache;
166
167/*
168 * Bio-based DM's mempools' reserved IOs set by the user.
169 */
170#define RESERVED_BIO_BASED_IOS 16
171static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;

--- 50 unchanged lines hidden (view full) ---

222 return __dm_get_module_param_int(&dm_numa_node,
223 DM_NUMA_NODE, num_online_nodes() - 1);
224}
225
226static int __init local_init(void)
227{
228 int r = -ENOMEM;
229
162static struct kmem_cache *_rq_tio_cache;
163static struct kmem_cache *_rq_cache;
164
165/*
166 * Bio-based DM's mempools' reserved IOs set by the user.
167 */
168#define RESERVED_BIO_BASED_IOS 16
169static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;

--- 50 unchanged lines hidden (view full) ---

220 return __dm_get_module_param_int(&dm_numa_node,
221 DM_NUMA_NODE, num_online_nodes() - 1);
222}
223
224static int __init local_init(void)
225{
226 int r = -ENOMEM;
227
230 /* allocate a slab for the dm_ios */
231 _io_cache = KMEM_CACHE(dm_io, 0);
232 if (!_io_cache)
233 return r;
234
235 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
236 if (!_rq_tio_cache)
228 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
229 if (!_rq_tio_cache)
237 goto out_free_io_cache;
230 return r;
238
239 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
240 __alignof__(struct request), 0, NULL);
241 if (!_rq_cache)
242 goto out_free_rq_tio_cache;
243
244 r = dm_uevent_init();
245 if (r)

--- 18 unchanged lines hidden (view full) ---

264out_free_workqueue:
265 destroy_workqueue(deferred_remove_workqueue);
266out_uevent_exit:
267 dm_uevent_exit();
268out_free_rq_cache:
269 kmem_cache_destroy(_rq_cache);
270out_free_rq_tio_cache:
271 kmem_cache_destroy(_rq_tio_cache);
231
232 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
233 __alignof__(struct request), 0, NULL);
234 if (!_rq_cache)
235 goto out_free_rq_tio_cache;
236
237 r = dm_uevent_init();
238 if (r)

--- 18 unchanged lines hidden (view full) ---

257out_free_workqueue:
258 destroy_workqueue(deferred_remove_workqueue);
259out_uevent_exit:
260 dm_uevent_exit();
261out_free_rq_cache:
262 kmem_cache_destroy(_rq_cache);
263out_free_rq_tio_cache:
264 kmem_cache_destroy(_rq_tio_cache);
272out_free_io_cache:
273 kmem_cache_destroy(_io_cache);
274
275 return r;
276}
277
278static void local_exit(void)
279{
280 flush_scheduled_work();
281 destroy_workqueue(deferred_remove_workqueue);
282
283 kmem_cache_destroy(_rq_cache);
284 kmem_cache_destroy(_rq_tio_cache);
265
266 return r;
267}
268
269static void local_exit(void)
270{
271 flush_scheduled_work();
272 destroy_workqueue(deferred_remove_workqueue);
273
274 kmem_cache_destroy(_rq_cache);
275 kmem_cache_destroy(_rq_tio_cache);
285 kmem_cache_destroy(_io_cache);
286 unregister_blkdev(_major, _name);
287 dm_uevent_exit();
288
289 _major = 0;
290
291 DMINFO("cleaned up");
292}
293

--- 1399 unchanged lines hidden (view full) ---

1693}
1694
1695static void cleanup_mapped_device(struct mapped_device *md)
1696{
1697 if (md->wq)
1698 destroy_workqueue(md->wq);
1699 if (md->kworker_task)
1700 kthread_stop(md->kworker_task);
276 unregister_blkdev(_major, _name);
277 dm_uevent_exit();
278
279 _major = 0;
280
281 DMINFO("cleaned up");
282}
283

--- 1399 unchanged lines hidden (view full) ---

1683}
1684
1685static void cleanup_mapped_device(struct mapped_device *md)
1686{
1687 if (md->wq)
1688 destroy_workqueue(md->wq);
1689 if (md->kworker_task)
1690 kthread_stop(md->kworker_task);
1701 mempool_destroy(md->io_pool);
1702 if (md->bs)
1703 bioset_free(md->bs);
1704 if (md->io_bs)
1705 bioset_free(md->io_bs);
1706
1707 if (md->dax_dev) {
1708 kill_dax(md->dax_dev);
1709 put_dax(md->dax_dev);

--- 166 unchanged lines hidden (view full) ---

1876 if (md->bs) {
1877 bioset_free(md->bs);
1878 md->bs = NULL;
1879 }
1880 if (md->io_bs) {
1881 bioset_free(md->io_bs);
1882 md->io_bs = NULL;
1883 }
1691 if (md->bs)
1692 bioset_free(md->bs);
1693 if (md->io_bs)
1694 bioset_free(md->io_bs);
1695
1696 if (md->dax_dev) {
1697 kill_dax(md->dax_dev);
1698 put_dax(md->dax_dev);

--- 166 unchanged lines hidden (view full) ---

1865 if (md->bs) {
1866 bioset_free(md->bs);
1867 md->bs = NULL;
1868 }
1869 if (md->io_bs) {
1870 bioset_free(md->io_bs);
1871 md->io_bs = NULL;
1872 }
1884 if (md->io_pool) {
1885 /*
1886 * Reload io_pool because pool_size may have changed
1887 * because a different table was loaded.
1888 */
1889 mempool_destroy(md->io_pool);
1890 md->io_pool = NULL;
1891 }
1892
1893 } else if (md->bs) {
1894 /*
1895 * There's no need to reload with request-based dm
1896 * because the size of front_pad doesn't change.
1897 * Note for future: If you are to reload bioset,
1898 * prep-ed requests in the queue may refer
1899 * to bio from the old bioset, so you must walk
1900 * through the queue to unprep.
1901 */
1902 goto out;
1903 }
1904
1873
1874 } else if (md->bs) {
1875 /*
1876 * There's no need to reload with request-based dm
1877 * because the size of front_pad doesn't change.
1878 * Note for future: If you are to reload bioset,
1879 * prep-ed requests in the queue may refer
1880 * to bio from the old bioset, so you must walk
1881 * through the queue to unprep.
1882 */
1883 goto out;
1884 }
1885
1905 BUG_ON(!p || md->io_pool || md->bs || md->io_bs);
1886 BUG_ON(!p || md->bs || md->io_bs);
1906
1887
1907 md->io_pool = p->io_pool;
1908 p->io_pool = NULL;
1909 md->bs = p->bs;
1910 p->bs = NULL;
1911 md->io_bs = p->io_bs;
1912 p->io_bs = NULL;
1913out:
1914 /* mempool bind completed, no longer need any mempools in the table */
1915 dm_table_free_md_mempools(t);
1916}

--- 894 unchanged lines hidden (view full) ---

2811 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2812 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2813 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
2814 pools->io_bs = bioset_create(pool_size, io_front_pad, 0);
2815 if (!pools->io_bs)
2816 goto out;
2817 if (integrity && bioset_integrity_create(pools->io_bs, pool_size))
2818 goto out;
1888 md->bs = p->bs;
1889 p->bs = NULL;
1890 md->io_bs = p->io_bs;
1891 p->io_bs = NULL;
1892out:
1893 /* mempool bind completed, no longer need any mempools in the table */
1894 dm_table_free_md_mempools(t);
1895}

--- 894 unchanged lines hidden (view full) ---

2790 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2791 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2792 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio);
2793 pools->io_bs = bioset_create(pool_size, io_front_pad, 0);
2794 if (!pools->io_bs)
2795 goto out;
2796 if (integrity && bioset_integrity_create(pools->io_bs, pool_size))
2797 goto out;
2819 pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
2820 if (!pools->io_pool)
2821 goto out;
2822 break;
2823 case DM_TYPE_REQUEST_BASED:
2824 case DM_TYPE_MQ_REQUEST_BASED:
2825 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2826 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2827 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2828 break;
2829 default:

--- 15 unchanged lines hidden (view full) ---

2845 return NULL;
2846}
2847
2848void dm_free_md_mempools(struct dm_md_mempools *pools)
2849{
2850 if (!pools)
2851 return;
2852
2798 break;
2799 case DM_TYPE_REQUEST_BASED:
2800 case DM_TYPE_MQ_REQUEST_BASED:
2801 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2802 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2803 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2804 break;
2805 default:

--- 15 unchanged lines hidden (view full) ---

2821 return NULL;
2822}
2823
2824void dm_free_md_mempools(struct dm_md_mempools *pools)
2825{
2826 if (!pools)
2827 return;
2828
2853 mempool_destroy(pools->io_pool);
2854
2855 if (pools->bs)
2856 bioset_free(pools->bs);
2857 if (pools->io_bs)
2858 bioset_free(pools->io_bs);
2859
2860 kfree(pools);
2861}
2862

--- 198 unchanged lines hidden --- 		2829 if (pools->bs)
2830 bioset_free(pools->bs);
2831 if (pools->io_bs)
2832 bioset_free(pools->io_bs);
2833
2834 kfree(pools);
2835}
2836

--- 198 unchanged lines hidden ---