dm.c (e980f62353c697cbf0c4325e43df6e44399aeb64) dm.c (d67a5f4b5947aba4bfe9a80a2b86079c215ca755)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 77 unchanged lines hidden (view full) ---

86#define DM_NUMA_NODE NUMA_NO_NODE
87static int dm_numa_node = DM_NUMA_NODE;
88
89/*
90 * For mempools pre-allocation at the table loading time.
91 */
92struct dm_md_mempools {
93 mempool_t *io_pool;
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 77 unchanged lines hidden (view full) ---

86#define DM_NUMA_NODE NUMA_NO_NODE
87static int dm_numa_node = DM_NUMA_NODE;
88
89/*
90 * For mempools pre-allocation at the table loading time.
91 */
92struct dm_md_mempools {
93 mempool_t *io_pool;
94 mempool_t *rq_pool;
94 struct bio_set *bs;
95};
96
97struct table_device {
98 struct list_head list;
99 atomic_t count;
100 struct dm_dev dm_dev;
101};

--- 358 unchanged lines hidden (view full) ---

460 int r;
461
462 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
463 if (r < 0)
464 return r;
465
466 if (r > 0) {
467 /*
95 struct bio_set *bs;
96};
97
98struct table_device {
99 struct list_head list;
100 atomic_t count;
101 struct dm_dev dm_dev;
102};

--- 358 unchanged lines hidden (view full) ---

461 int r;
462
463 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
464 if (r < 0)
465 return r;
466
467 if (r > 0) {
468 /*
468 * Target determined this ioctl is being issued against a
469 * subset of the parent bdev; require extra privileges.
469 * Target determined this ioctl is being issued against
470 * a logical partition of the parent bdev; so extra
471 * validation is needed.
470 */
472 */
471 if (!capable(CAP_SYS_RAWIO)) {
472 DMWARN_LIMIT(
473 "%s: sending ioctl %x to DM device without required privilege.",
474 current->comm, cmd);
475 r = -ENOIOCTLCMD;
473 r = scsi_verify_blk_ioctl(NULL, cmd);
474 if (r)
476 goto out;
475 goto out;
477 }
478 }
479
480 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
481out:
482 bdput(bdev);
483 return r;
484}
485

--- 483 unchanged lines hidden (view full) ---

969 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
970 BUG_ON(bi_size > *tio->len_ptr);
971 BUG_ON(n_sectors > bi_size);
972 *tio->len_ptr -= bi_size - n_sectors;
973 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
974}
975EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
976
476 }
477
478 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
479out:
480 bdput(bdev);
481 return r;
482}
483

--- 483 unchanged lines hidden (view full) ---

967 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
968 BUG_ON(bi_size > *tio->len_ptr);
969 BUG_ON(n_sectors > bi_size);
970 *tio->len_ptr -= bi_size - n_sectors;
971 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
972}
973EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
974
975/*
976 * Flush current->bio_list when the target map method blocks.
977 * This fixes deadlocks in snapshot and possibly in other targets.
978 */
979struct dm_offload {
980 struct blk_plug plug;
981 struct blk_plug_cb cb;
982};
983
984static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
985{
986 struct dm_offload *o = container_of(cb, struct dm_offload, cb);
987 struct bio_list list;
988 struct bio *bio;
989
990 INIT_LIST_HEAD(&o->cb.list);
991
992 if (unlikely(!current->bio_list))
993 return;
994
995 list = *current->bio_list;
996 bio_list_init(current->bio_list);
997
998 while ((bio = bio_list_pop(&list))) {
999 struct bio_set *bs = bio->bi_pool;
1000 if (unlikely(!bs) || bs == fs_bio_set) {
1001 bio_list_add(current->bio_list, bio);
1002 continue;
1003 }
1004
1005 spin_lock(&bs->rescue_lock);
1006 bio_list_add(&bs->rescue_list, bio);
1007 queue_work(bs->rescue_workqueue, &bs->rescue_work);
1008 spin_unlock(&bs->rescue_lock);
1009 }
1010}
1011
1012static void dm_offload_start(struct dm_offload *o)
1013{
1014 blk_start_plug(&o->plug);
1015 o->cb.callback = flush_current_bio_list;
1016 list_add(&o->cb.list, &current->plug->cb_list);
1017}
1018
1019static void dm_offload_end(struct dm_offload *o)
1020{
1021 list_del(&o->cb.list);
1022 blk_finish_plug(&o->plug);
1023}
1024
977static void __map_bio(struct dm_target_io *tio)
978{
979 int r;
980 sector_t sector;
1025static void __map_bio(struct dm_target_io *tio)
1026{
1027 int r;
1028 sector_t sector;
1029 struct dm_offload o;
981 struct bio *clone = &tio->clone;
982 struct dm_target *ti = tio->ti;
983
984 clone->bi_end_io = clone_endio;
985
986 /*
987 * Map the clone. If r == 0 we don't need to do
988 * anything, the target has assumed ownership of
989 * this io.
990 */
991 atomic_inc(&tio->io->io_count);
992 sector = clone->bi_iter.bi_sector;
1030 struct bio *clone = &tio->clone;
1031 struct dm_target *ti = tio->ti;
1032
1033 clone->bi_end_io = clone_endio;
1034
1035 /*
1036 * Map the clone. If r == 0 we don't need to do
1037 * anything, the target has assumed ownership of
1038 * this io.
1039 */
1040 atomic_inc(&tio->io->io_count);
1041 sector = clone->bi_iter.bi_sector;
1042
1043 dm_offload_start(&o);
993 r = ti->type->map(ti, clone);
1044 r = ti->type->map(ti, clone);
1045 dm_offload_end(&o);
1046
994 if (r == DM_MAPIO_REMAPPED) {
995 /* the bio has been remapped so dispatch it */
996
997 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
998 tio->io->bio->bi_bdev->bd_dev, sector);
999
1000 generic_make_request(clone);
1001 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {

--- 309 unchanged lines hidden (view full) ---

1311 struct dm_table *map;
1312
1313 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1314 if (dm_request_based(md)) {
1315 /*
1316 * With request-based DM we only need to check the
1317 * top-level queue for congestion.
1318 */
1047 if (r == DM_MAPIO_REMAPPED) {
1048 /* the bio has been remapped so dispatch it */
1049
1050 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
1051 tio->io->bio->bi_bdev->bd_dev, sector);
1052
1053 generic_make_request(clone);
1054 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {

--- 309 unchanged lines hidden (view full) ---

1364 struct dm_table *map;
1365
1366 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1367 if (dm_request_based(md)) {
1368 /*
1369 * With request-based DM we only need to check the
1370 * top-level queue for congestion.
1371 */
1319 r = md->queue->backing_dev_info->wb.state & bdi_bits;
1372 r = md->queue->backing_dev_info.wb.state & bdi_bits;
1320 } else {
1321 map = dm_get_live_table_fast(md);
1322 if (map)
1323 r = dm_table_any_congested(map, bdi_bits);
1324 dm_put_live_table_fast(md);
1325 }
1326 }
1327

--- 66 unchanged lines hidden (view full) ---

1394 */
1395 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1396
1397 /*
1398 * Initialize data that will only be used by a non-blk-mq DM queue
1399 * - must do so here (in alloc_dev callchain) before queue is used
1400 */
1401 md->queue->queuedata = md;
1373 } else {
1374 map = dm_get_live_table_fast(md);
1375 if (map)
1376 r = dm_table_any_congested(map, bdi_bits);
1377 dm_put_live_table_fast(md);
1378 }
1379 }
1380

--- 66 unchanged lines hidden (view full) ---

1447 */
1448 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1449
1450 /*
1451 * Initialize data that will only be used by a non-blk-mq DM queue
1452 * - must do so here (in alloc_dev callchain) before queue is used
1453 */
1454 md->queue->queuedata = md;
1402 md->queue->backing_dev_info->congested_data = md;
1455 md->queue->backing_dev_info.congested_data = md;
1403}
1404
1405void dm_init_normal_md_queue(struct mapped_device *md)
1406{
1407 md->use_blk_mq = false;
1408 dm_init_md_queue(md);
1409
1410 /*
1411 * Initialize aspects of queue that aren't relevant for blk-mq
1412 */
1456}
1457
1458void dm_init_normal_md_queue(struct mapped_device *md)
1459{
1460 md->use_blk_mq = false;
1461 dm_init_md_queue(md);
1462
1463 /*
1464 * Initialize aspects of queue that aren't relevant for blk-mq
1465 */
1413 md->queue->backing_dev_info->congested_fn = dm_any_congested;
1466 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1414 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1415}
1416
1417static void cleanup_mapped_device(struct mapped_device *md)
1418{
1419 if (md->wq)
1420 destroy_workqueue(md->wq);
1421 if (md->kworker_task)
1422 kthread_stop(md->kworker_task);
1423 mempool_destroy(md->io_pool);
1467 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1468}
1469
1470static void cleanup_mapped_device(struct mapped_device *md)
1471{
1472 if (md->wq)
1473 destroy_workqueue(md->wq);
1474 if (md->kworker_task)
1475 kthread_stop(md->kworker_task);
1476 mempool_destroy(md->io_pool);
1477 mempool_destroy(md->rq_pool);
1424 if (md->bs)
1425 bioset_free(md->bs);
1426
1427 if (md->disk) {
1428 spin_lock(&_minor_lock);
1429 md->disk->private_data = NULL;
1430 spin_unlock(&_minor_lock);
1431 del_gendisk(md->disk);

--- 159 unchanged lines hidden (view full) ---

1591 * Note for future: If you are to reload bioset,
1592 * prep-ed requests in the queue may refer
1593 * to bio from the old bioset, so you must walk
1594 * through the queue to unprep.
1595 */
1596 goto out;
1597 }
1598
1478 if (md->bs)
1479 bioset_free(md->bs);
1480
1481 if (md->disk) {
1482 spin_lock(&_minor_lock);
1483 md->disk->private_data = NULL;
1484 spin_unlock(&_minor_lock);
1485 del_gendisk(md->disk);

--- 159 unchanged lines hidden (view full) ---

1645 * Note for future: If you are to reload bioset,
1646 * prep-ed requests in the queue may refer
1647 * to bio from the old bioset, so you must walk
1648 * through the queue to unprep.
1649 */
1650 goto out;
1651 }
1652
1599 BUG_ON(!p || md->io_pool || md->bs);
1653 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
1600
1601 md->io_pool = p->io_pool;
1602 p->io_pool = NULL;
1654
1655 md->io_pool = p->io_pool;
1656 p->io_pool = NULL;
1657 md->rq_pool = p->rq_pool;
1658 p->rq_pool = NULL;
1603 md->bs = p->bs;
1604 p->bs = NULL;
1605
1606out:
1607 /* mempool bind completed, no longer need any mempools in the table */
1608 dm_table_free_md_mempools(t);
1609}
1610

--- 160 unchanged lines hidden (view full) ---

1771 */
1772int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
1773{
1774 int r;
1775 unsigned type = dm_get_md_type(md);
1776
1777 switch (type) {
1778 case DM_TYPE_REQUEST_BASED:
1659 md->bs = p->bs;
1660 p->bs = NULL;
1661
1662out:
1663 /* mempool bind completed, no longer need any mempools in the table */
1664 dm_table_free_md_mempools(t);
1665}
1666

--- 160 unchanged lines hidden (view full) ---

1827 */
1828int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
1829{
1830 int r;
1831 unsigned type = dm_get_md_type(md);
1832
1833 switch (type) {
1834 case DM_TYPE_REQUEST_BASED:
1779 r = dm_old_init_request_queue(md, t);
1835 r = dm_old_init_request_queue(md);
1780 if (r) {
1781 DMERR("Cannot initialize queue for request-based mapped device");
1782 return r;
1783 }
1784 break;
1785 case DM_TYPE_MQ_REQUEST_BASED:
1786 r = dm_mq_init_request_queue(md, t);
1787 if (r) {

--- 699 unchanged lines hidden (view full) ---

2487 return __noflush_suspending(dm_table_get_md(ti->table));
2488}
2489EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2490
2491struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
2492 unsigned integrity, unsigned per_io_data_size)
2493{
2494 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
1836 if (r) {
1837 DMERR("Cannot initialize queue for request-based mapped device");
1838 return r;
1839 }
1840 break;
1841 case DM_TYPE_MQ_REQUEST_BASED:
1842 r = dm_mq_init_request_queue(md, t);
1843 if (r) {

--- 699 unchanged lines hidden (view full) ---

2543 return __noflush_suspending(dm_table_get_md(ti->table));
2544}
2545EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2546
2547struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
2548 unsigned integrity, unsigned per_io_data_size)
2549{
2550 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2551 struct kmem_cache *cachep = NULL;
2495 unsigned int pool_size = 0;
2496 unsigned int front_pad;
2497
2498 if (!pools)
2499 return NULL;
2500
2501 switch (type) {
2502 case DM_TYPE_BIO_BASED:
2503 case DM_TYPE_DAX_BIO_BASED:
2552 unsigned int pool_size = 0;
2553 unsigned int front_pad;
2554
2555 if (!pools)
2556 return NULL;
2557
2558 switch (type) {
2559 case DM_TYPE_BIO_BASED:
2560 case DM_TYPE_DAX_BIO_BASED:
2561 cachep = _io_cache;
2504 pool_size = dm_get_reserved_bio_based_ios();
2505 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2562 pool_size = dm_get_reserved_bio_based_ios();
2563 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2506
2507 pools->io_pool = mempool_create_slab_pool(pool_size, _io_cache);
2508 if (!pools->io_pool)
2509 goto out;
2510 break;
2511 case DM_TYPE_REQUEST_BASED:
2564 break;
2565 case DM_TYPE_REQUEST_BASED:
2512 case DM_TYPE_MQ_REQUEST_BASED:
2566 cachep = _rq_tio_cache;
2513 pool_size = dm_get_reserved_rq_based_ios();
2567 pool_size = dm_get_reserved_rq_based_ios();
2568 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
2569 if (!pools->rq_pool)
2570 goto out;
2571 /* fall through to setup remaining rq-based pools */
2572 case DM_TYPE_MQ_REQUEST_BASED:
2573 if (!pool_size)
2574 pool_size = dm_get_reserved_rq_based_ios();
2514 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2515 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2516 break;
2517 default:
2518 BUG();
2519 }
2520
2575 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2576 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2577 break;
2578 default:
2579 BUG();
2580 }
2581
2582 if (cachep) {
2583 pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
2584 if (!pools->io_pool)
2585 goto out;
2586 }
2587
2521 pools->bs = bioset_create_nobvec(pool_size, front_pad);
2522 if (!pools->bs)
2523 goto out;
2524
2525 if (integrity && bioset_integrity_create(pools->bs, pool_size))
2526 goto out;
2527
2528 return pools;

--- 5 unchanged lines hidden (view full) ---

2534}
2535
2536void dm_free_md_mempools(struct dm_md_mempools *pools)
2537{
2538 if (!pools)
2539 return;
2540
2541 mempool_destroy(pools->io_pool);
2588 pools->bs = bioset_create_nobvec(pool_size, front_pad);
2589 if (!pools->bs)
2590 goto out;
2591
2592 if (integrity && bioset_integrity_create(pools->bs, pool_size))
2593 goto out;
2594
2595 return pools;

--- 5 unchanged lines hidden (view full) ---

2601}
2602
2603void dm_free_md_mempools(struct dm_md_mempools *pools)
2604{
2605 if (!pools)
2606 return;
2607
2608 mempool_destroy(pools->io_pool);
2609 mempool_destroy(pools->rq_pool);
2542
2543 if (pools->bs)
2544 bioset_free(pools->bs);
2545
2546 kfree(pools);
2547}
2548
2549struct dm_pr {

--- 193 unchanged lines hidden --- 		2610
2611 if (pools->bs)
2612 bioset_free(pools->bs);
2613
2614 kfree(pools);
2615}
2616
2617struct dm_pr {

--- 193 unchanged lines hidden ---