dm.c (6c182cd88d179cbbd06f4f8a8a19b6977940753f) dm.c (83d5e5b0af907d46d241a86d9e44003b3f0accbd)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"

--- 103 unchanged lines hidden (view full) ---

112#define DMF_SUSPENDED 1
113#define DMF_FROZEN 2
114#define DMF_FREEING 3
115#define DMF_DELETING 4
116#define DMF_NOFLUSH_SUSPENDING 5
117#define DMF_MERGE_IS_OPTIONAL 6
118
119/*
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"

--- 103 unchanged lines hidden (view full) ---

112#define DMF_SUSPENDED 1
113#define DMF_FROZEN 2
114#define DMF_FREEING 3
115#define DMF_DELETING 4
116#define DMF_NOFLUSH_SUSPENDING 5
117#define DMF_MERGE_IS_OPTIONAL 6
118
119/*
120 * A dummy definition to make RCU happy.
121 * struct dm_table should never be dereferenced in this file.
122 */
123struct dm_table {
124 int undefined__;
125};
126
127/*
120 * Work processed by per-device workqueue.
121 */
122struct mapped_device {
128 * Work processed by per-device workqueue.
129 */
130struct mapped_device {
123 struct rw_semaphore io_lock;
131 struct srcu_struct io_barrier;
124 struct mutex suspend_lock;
132 struct mutex suspend_lock;
125 rwlock_t map_lock;
126 atomic_t holders;
127 atomic_t open_count;
128
129 unsigned long flags;
130
131 struct request_queue *queue;
132 unsigned type;
133 /* Protect queue and type against concurrent access. */

--- 17 unchanged lines hidden (view full) ---

151
152 /*
153 * Processing queue (flush)
154 */
155 struct workqueue_struct *wq;
156
157 /*
158 * The current mapping.
133 atomic_t holders;
134 atomic_t open_count;
135
136 unsigned long flags;
137
138 struct request_queue *queue;
139 unsigned type;
140 /* Protect queue and type against concurrent access. */

--- 17 unchanged lines hidden (view full) ---

158
159 /*
160 * Processing queue (flush)
161 */
162 struct workqueue_struct *wq;
163
164 /*
165 * The current mapping.
166 * Use dm_get_live_table{_fast} or take suspend_lock for
167 * dereference.
159 */
160 struct dm_table *map;
161
162 /*
163 * io objects are allocated from here.
164 */
165 mempool_t *io_pool;
166

--- 214 unchanged lines hidden (view full) ---

381
382 return dm_get_geometry(md, geo);
383}
384
385static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
386 unsigned int cmd, unsigned long arg)
387{
388 struct mapped_device *md = bdev->bd_disk->private_data;
168 */
169 struct dm_table *map;
170
171 /*
172 * io objects are allocated from here.
173 */
174 mempool_t *io_pool;
175

--- 214 unchanged lines hidden (view full) ---

390
391 return dm_get_geometry(md, geo);
392}
393
394static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
395 unsigned int cmd, unsigned long arg)
396{
397 struct mapped_device *md = bdev->bd_disk->private_data;
398 int srcu_idx;
389 struct dm_table *map;
390 struct dm_target *tgt;
391 int r = -ENOTTY;
392
393retry:
399 struct dm_table *map;
400 struct dm_target *tgt;
401 int r = -ENOTTY;
402
403retry:
394 map = dm_get_live_table(md);
404 map = dm_get_live_table(md, &srcu_idx);
405
395 if (!map || !dm_table_get_size(map))
396 goto out;
397
398 /* We only support devices that have a single target */
399 if (dm_table_get_num_targets(map) != 1)
400 goto out;
401
402 tgt = dm_table_get_target(map, 0);
403
404 if (dm_suspended_md(md)) {
405 r = -EAGAIN;
406 goto out;
407 }
408
409 if (tgt->type->ioctl)
410 r = tgt->type->ioctl(tgt, cmd, arg);
411
412out:
406 if (!map || !dm_table_get_size(map))
407 goto out;
408
409 /* We only support devices that have a single target */
410 if (dm_table_get_num_targets(map) != 1)
411 goto out;
412
413 tgt = dm_table_get_target(map, 0);
414
415 if (dm_suspended_md(md)) {
416 r = -EAGAIN;
417 goto out;
418 }
419
420 if (tgt->type->ioctl)
421 r = tgt->type->ioctl(tgt, cmd, arg);
422
423out:
413 dm_table_put(map);
424 dm_put_live_table(md, srcu_idx);
414
415 if (r == -ENOTCONN) {
416 msleep(10);
417 goto retry;
418 }
419
420 return r;
421}

--- 82 unchanged lines hidden (view full) ---

504 bio_list_add(&md->deferred, bio);
505 spin_unlock_irqrestore(&md->deferred_lock, flags);
506 queue_work(md->wq, &md->work);
507}
508
509/*
510 * Everyone (including functions in this file), should use this
511 * function to access the md->map field, and make sure they call
425
426 if (r == -ENOTCONN) {
427 msleep(10);
428 goto retry;
429 }
430
431 return r;
432}

--- 82 unchanged lines hidden (view full) ---

515 bio_list_add(&md->deferred, bio);
516 spin_unlock_irqrestore(&md->deferred_lock, flags);
517 queue_work(md->wq, &md->work);
518}
519
520/*
521 * Everyone (including functions in this file), should use this
522 * function to access the md->map field, and make sure they call
512 * dm_table_put() when finished.
523 * dm_put_live_table() when finished.
513 */
524 */
514struct dm_table *dm_get_live_table(struct mapped_device *md)
525struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
515{
526{
516 struct dm_table *t;
517 unsigned long flags;
527 *srcu_idx = srcu_read_lock(&md->io_barrier);
518
528
519 read_lock_irqsave(&md->map_lock, flags);
520 t = md->map;
521 if (t)
522 dm_table_get(t);
523 read_unlock_irqrestore(&md->map_lock, flags);
529 return srcu_dereference(md->map, &md->io_barrier);
530}
524
531
525 return t;
532void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
533{
534 srcu_read_unlock(&md->io_barrier, srcu_idx);
526}
527
535}
536
537void dm_sync_table(struct mapped_device *md)
538{
539 synchronize_srcu(&md->io_barrier);
540 synchronize_rcu_expedited();
541}
542
528/*
543/*
544 * A fast alternative to dm_get_live_table/dm_put_live_table.
545 * The caller must not block between these two functions.
546 */
547static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
548{
549 rcu_read_lock();
550 return rcu_dereference(md->map);
551}
552
553static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
554{
555 rcu_read_unlock();
556}
557
558/*
529 * Get the geometry associated with a dm device
530 */
531int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
532{
533 *geo = md->geometry;
534
535 return 0;
536}

--- 814 unchanged lines hidden (view full) ---

1351 * Handle a bvec that must be split between two or more targets.
1352 */
1353 return __split_bvec_across_targets(ci, ti, max);
1354}
1355
1356/*
1357 * Entry point to split a bio into clones and submit them to the targets.
1358 */
559 * Get the geometry associated with a dm device
560 */
561int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
562{
563 *geo = md->geometry;
564
565 return 0;
566}

--- 814 unchanged lines hidden (view full) ---

1381 * Handle a bvec that must be split between two or more targets.
1382 */
1383 return __split_bvec_across_targets(ci, ti, max);
1384}
1385
1386/*
1387 * Entry point to split a bio into clones and submit them to the targets.
1388 */
1359static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
1389static void __split_and_process_bio(struct mapped_device *md,
1390 struct dm_table *map, struct bio *bio)
1360{
1361 struct clone_info ci;
1362 int error = 0;
1363
1391{
1392 struct clone_info ci;
1393 int error = 0;
1394
1364 ci.map = dm_get_live_table(md);
1365 if (unlikely(!ci.map)) {
1395 if (unlikely(!map)) {
1366 bio_io_error(bio);
1367 return;
1368 }
1369
1396 bio_io_error(bio);
1397 return;
1398 }
1399
1400 ci.map = map;
1370 ci.md = md;
1371 ci.io = alloc_io(md);
1372 ci.io->error = 0;
1373 atomic_set(&ci.io->io_count, 1);
1374 ci.io->bio = bio;
1375 ci.io->md = md;
1376 spin_lock_init(&ci.io->endio_lock);
1377 ci.sector = bio->bi_sector;

--- 10 unchanged lines hidden (view full) ---

1388 ci.bio = bio;
1389 ci.sector_count = bio_sectors(bio);
1390 while (ci.sector_count && !error)
1391 error = __split_and_process_non_flush(&ci);
1392 }
1393
1394 /* drop the extra reference count */
1395 dec_pending(ci.io, error);
1401 ci.md = md;
1402 ci.io = alloc_io(md);
1403 ci.io->error = 0;
1404 atomic_set(&ci.io->io_count, 1);
1405 ci.io->bio = bio;
1406 ci.io->md = md;
1407 spin_lock_init(&ci.io->endio_lock);
1408 ci.sector = bio->bi_sector;

--- 10 unchanged lines hidden (view full) ---

1419 ci.bio = bio;
1420 ci.sector_count = bio_sectors(bio);
1421 while (ci.sector_count && !error)
1422 error = __split_and_process_non_flush(&ci);
1423 }
1424
1425 /* drop the extra reference count */
1426 dec_pending(ci.io, error);
1396 dm_table_put(ci.map);
1397}
1398/*-----------------------------------------------------------------
1399 * CRUD END
1400 *---------------------------------------------------------------*/
1401
1402static int dm_merge_bvec(struct request_queue *q,
1403 struct bvec_merge_data *bvm,
1404 struct bio_vec *biovec)
1405{
1406 struct mapped_device *md = q->queuedata;
1427}
1428/*-----------------------------------------------------------------
1429 * CRUD END
1430 *---------------------------------------------------------------*/
1431
1432static int dm_merge_bvec(struct request_queue *q,
1433 struct bvec_merge_data *bvm,
1434 struct bio_vec *biovec)
1435{
1436 struct mapped_device *md = q->queuedata;
1407 struct dm_table *map = dm_get_live_table(md);
1437 struct dm_table *map = dm_get_live_table_fast(md);
1408 struct dm_target *ti;
1409 sector_t max_sectors;
1410 int max_size = 0;
1411
1412 if (unlikely(!map))
1413 goto out;
1414
1415 ti = dm_table_find_target(map, bvm->bi_sector);
1416 if (!dm_target_is_valid(ti))
1438 struct dm_target *ti;
1439 sector_t max_sectors;
1440 int max_size = 0;
1441
1442 if (unlikely(!map))
1443 goto out;
1444
1445 ti = dm_table_find_target(map, bvm->bi_sector);
1446 if (!dm_target_is_valid(ti))
1417 goto out_table;
1447 goto out;
1418
1419 /*
1420 * Find maximum amount of I/O that won't need splitting
1421 */
1422 max_sectors = min(max_io_len(bvm->bi_sector, ti),
1423 (sector_t) BIO_MAX_SECTORS);
1424 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1425 if (max_size < 0)

--- 12 unchanged lines hidden (view full) ---

1438 * queue_max_hw_sectors), then we can't allow bios with multiple vector
1439 * entries. So always set max_size to 0, and the code below allows
1440 * just one page.
1441 */
1442 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1443
1444 max_size = 0;
1445
1448
1449 /*
1450 * Find maximum amount of I/O that won't need splitting
1451 */
1452 max_sectors = min(max_io_len(bvm->bi_sector, ti),
1453 (sector_t) BIO_MAX_SECTORS);
1454 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1455 if (max_size < 0)

--- 12 unchanged lines hidden (view full) ---

1468 * queue_max_hw_sectors), then we can't allow bios with multiple vector
1469 * entries. So always set max_size to 0, and the code below allows
1470 * just one page.
1471 */
1472 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
1473
1474 max_size = 0;
1475
1446out_table:
1447 dm_table_put(map);
1448
1449out:
1476out:
1477 dm_put_live_table_fast(md);
1450 /*
1451 * Always allow an entire first page
1452 */
1453 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1454 max_size = biovec->bv_len;
1455
1456 return max_size;
1457}
1458
1459/*
1460 * The request function that just remaps the bio built up by
1461 * dm_merge_bvec.
1462 */
1463static void _dm_request(struct request_queue *q, struct bio *bio)
1464{
1465 int rw = bio_data_dir(bio);
1466 struct mapped_device *md = q->queuedata;
1467 int cpu;
1478 /*
1479 * Always allow an entire first page
1480 */
1481 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1482 max_size = biovec->bv_len;
1483
1484 return max_size;
1485}
1486
1487/*
1488 * The request function that just remaps the bio built up by
1489 * dm_merge_bvec.
1490 */
1491static void _dm_request(struct request_queue *q, struct bio *bio)
1492{
1493 int rw = bio_data_dir(bio);
1494 struct mapped_device *md = q->queuedata;
1495 int cpu;
1496 int srcu_idx;
1497 struct dm_table *map;
1468
1498
1469 down_read(&md->io_lock);
1499 map = dm_get_live_table(md, &srcu_idx);
1470
1471 cpu = part_stat_lock();
1472 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1473 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1474 part_stat_unlock();
1475
1476 /* if we're suspended, we have to queue this io for later */
1477 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1500
1501 cpu = part_stat_lock();
1502 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1503 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1504 part_stat_unlock();
1505
1506 /* if we're suspended, we have to queue this io for later */
1507 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1478 up_read(&md->io_lock);
1508 dm_put_live_table(md, srcu_idx);
1479
1480 if (bio_rw(bio) != READA)
1481 queue_io(md, bio);
1482 else
1483 bio_io_error(bio);
1484 return;
1485 }
1486
1509
1510 if (bio_rw(bio) != READA)
1511 queue_io(md, bio);
1512 else
1513 bio_io_error(bio);
1514 return;
1515 }
1516
1487 __split_and_process_bio(md, bio);
1488 up_read(&md->io_lock);
1517 __split_and_process_bio(md, map, bio);
1518 dm_put_live_table(md, srcu_idx);
1489 return;
1490}
1491
1492static int dm_request_based(struct mapped_device *md)
1493{
1494 return blk_queue_stackable(md->queue);
1495}
1496

--- 169 unchanged lines hidden (view full) ---

1666
1667/*
1668 * q->request_fn for request-based dm.
1669 * Called with the queue lock held.
1670 */
1671static void dm_request_fn(struct request_queue *q)
1672{
1673 struct mapped_device *md = q->queuedata;
1519 return;
1520}
1521
1522static int dm_request_based(struct mapped_device *md)
1523{
1524 return blk_queue_stackable(md->queue);
1525}
1526

--- 169 unchanged lines hidden (view full) ---

1696
1697/*
1698 * q->request_fn for request-based dm.
1699 * Called with the queue lock held.
1700 */
1701static void dm_request_fn(struct request_queue *q)
1702{
1703 struct mapped_device *md = q->queuedata;
1674 struct dm_table *map = dm_get_live_table(md);
1704 int srcu_idx;
1705 struct dm_table *map = dm_get_live_table(md, &srcu_idx);
1675 struct dm_target *ti;
1676 struct request *rq, *clone;
1677 sector_t pos;
1678
1679 /*
1680 * For suspend, check blk_queue_stopped() and increment
1681 * ->pending within a single queue_lock not to increment the
1682 * number of in-flight I/Os after the queue is stopped in

--- 38 unchanged lines hidden (view full) ---

1721
1722requeued:
1723 BUG_ON(!irqs_disabled());
1724 spin_lock(q->queue_lock);
1725
1726delay_and_out:
1727 blk_delay_queue(q, HZ / 10);
1728out:
1706 struct dm_target *ti;
1707 struct request *rq, *clone;
1708 sector_t pos;
1709
1710 /*
1711 * For suspend, check blk_queue_stopped() and increment
1712 * ->pending within a single queue_lock not to increment the
1713 * number of in-flight I/Os after the queue is stopped in

--- 38 unchanged lines hidden (view full) ---

1752
1753requeued:
1754 BUG_ON(!irqs_disabled());
1755 spin_lock(q->queue_lock);
1756
1757delay_and_out:
1758 blk_delay_queue(q, HZ / 10);
1759out:
1729 dm_table_put(map);
1760 dm_put_live_table(md, srcu_idx);
1730}
1731
1732int dm_underlying_device_busy(struct request_queue *q)
1733{
1734 return blk_lld_busy(q);
1735}
1736EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1737
1738static int dm_lld_busy(struct request_queue *q)
1739{
1740 int r;
1741 struct mapped_device *md = q->queuedata;
1761}
1762
1763int dm_underlying_device_busy(struct request_queue *q)
1764{
1765 return blk_lld_busy(q);
1766}
1767EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1768
1769static int dm_lld_busy(struct request_queue *q)
1770{
1771 int r;
1772 struct mapped_device *md = q->queuedata;
1742 struct dm_table *map = dm_get_live_table(md);
1773 struct dm_table *map = dm_get_live_table_fast(md);
1743
1744 if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1745 r = 1;
1746 else
1747 r = dm_table_any_busy_target(map);
1748
1774
1775 if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1776 r = 1;
1777 else
1778 r = dm_table_any_busy_target(map);
1779
1749 dm_table_put(map);
1780 dm_put_live_table_fast(md);
1750
1751 return r;
1752}
1753
1754static int dm_any_congested(void *congested_data, int bdi_bits)
1755{
1756 int r = bdi_bits;
1757 struct mapped_device *md = congested_data;
1758 struct dm_table *map;
1759
1760 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1781
1782 return r;
1783}
1784
1785static int dm_any_congested(void *congested_data, int bdi_bits)
1786{
1787 int r = bdi_bits;
1788 struct mapped_device *md = congested_data;
1789 struct dm_table *map;
1790
1791 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1761 map = dm_get_live_table(md);
1792 map = dm_get_live_table_fast(md);
1762 if (map) {
1763 /*
1764 * Request-based dm cares about only own queue for
1765 * the query about congestion status of request_queue
1766 */
1767 if (dm_request_based(md))
1768 r = md->queue->backing_dev_info.state &
1769 bdi_bits;
1770 else
1771 r = dm_table_any_congested(map, bdi_bits);
1793 if (map) {
1794 /*
1795 * Request-based dm cares about only own queue for
1796 * the query about congestion status of request_queue
1797 */
1798 if (dm_request_based(md))
1799 r = md->queue->backing_dev_info.state &
1800 bdi_bits;
1801 else
1802 r = dm_table_any_congested(map, bdi_bits);
1772
1773 dm_table_put(map);
1774 }
1803 }
1804 dm_put_live_table_fast(md);
1775 }
1776
1777 return r;
1778}
1779
1780/*-----------------------------------------------------------------
1781 * An IDR is used to keep track of allocated minor numbers.
1782 *---------------------------------------------------------------*/

--- 88 unchanged lines hidden (view full) ---

1871 /* get a minor number for the dev */
1872 if (minor == DM_ANY_MINOR)
1873 r = next_free_minor(&minor);
1874 else
1875 r = specific_minor(minor);
1876 if (r < 0)
1877 goto bad_minor;
1878
1805 }
1806
1807 return r;
1808}
1809
1810/*-----------------------------------------------------------------
1811 * An IDR is used to keep track of allocated minor numbers.
1812 *---------------------------------------------------------------*/

--- 88 unchanged lines hidden (view full) ---

1901 /* get a minor number for the dev */
1902 if (minor == DM_ANY_MINOR)
1903 r = next_free_minor(&minor);
1904 else
1905 r = specific_minor(minor);
1906 if (r < 0)
1907 goto bad_minor;
1908
1909 r = init_srcu_struct(&md->io_barrier);
1910 if (r < 0)
1911 goto bad_io_barrier;
1912
1879 md->type = DM_TYPE_NONE;
1913 md->type = DM_TYPE_NONE;
1880 init_rwsem(&md->io_lock);
1881 mutex_init(&md->suspend_lock);
1882 mutex_init(&md->type_lock);
1883 spin_lock_init(&md->deferred_lock);
1914 mutex_init(&md->suspend_lock);
1915 mutex_init(&md->type_lock);
1916 spin_lock_init(&md->deferred_lock);
1884 rwlock_init(&md->map_lock);
1885 atomic_set(&md->holders, 1);
1886 atomic_set(&md->open_count, 0);
1887 atomic_set(&md->event_nr, 0);
1888 atomic_set(&md->uevent_seq, 0);
1889 INIT_LIST_HEAD(&md->uevent_list);
1890 spin_lock_init(&md->uevent_lock);
1891
1892 md->queue = blk_alloc_queue(GFP_KERNEL);

--- 46 unchanged lines hidden (view full) ---

1939bad_bdev:
1940 destroy_workqueue(md->wq);
1941bad_thread:
1942 del_gendisk(md->disk);
1943 put_disk(md->disk);
1944bad_disk:
1945 blk_cleanup_queue(md->queue);
1946bad_queue:
1917 atomic_set(&md->holders, 1);
1918 atomic_set(&md->open_count, 0);
1919 atomic_set(&md->event_nr, 0);
1920 atomic_set(&md->uevent_seq, 0);
1921 INIT_LIST_HEAD(&md->uevent_list);
1922 spin_lock_init(&md->uevent_lock);
1923
1924 md->queue = blk_alloc_queue(GFP_KERNEL);

--- 46 unchanged lines hidden (view full) ---

1971bad_bdev:
1972 destroy_workqueue(md->wq);
1973bad_thread:
1974 del_gendisk(md->disk);
1975 put_disk(md->disk);
1976bad_disk:
1977 blk_cleanup_queue(md->queue);
1978bad_queue:
1979 cleanup_srcu_struct(&md->io_barrier);
1980bad_io_barrier:
1947 free_minor(minor);
1948bad_minor:
1949 module_put(THIS_MODULE);
1950bad_module_get:
1951 kfree(md);
1952 return NULL;
1953}
1954

--- 7 unchanged lines hidden (view full) ---

1962 bdput(md->bdev);
1963 destroy_workqueue(md->wq);
1964 if (md->io_pool)
1965 mempool_destroy(md->io_pool);
1966 if (md->bs)
1967 bioset_free(md->bs);
1968 blk_integrity_unregister(md->disk);
1969 del_gendisk(md->disk);
1981 free_minor(minor);
1982bad_minor:
1983 module_put(THIS_MODULE);
1984bad_module_get:
1985 kfree(md);
1986 return NULL;
1987}
1988

--- 7 unchanged lines hidden (view full) ---

1996 bdput(md->bdev);
1997 destroy_workqueue(md->wq);
1998 if (md->io_pool)
1999 mempool_destroy(md->io_pool);
2000 if (md->bs)
2001 bioset_free(md->bs);
2002 blk_integrity_unregister(md->disk);
2003 del_gendisk(md->disk);
2004 cleanup_srcu_struct(&md->io_barrier);
1970 free_minor(minor);
1971
1972 spin_lock(&_minor_lock);
1973 md->disk->private_data = NULL;
1974 spin_unlock(&_minor_lock);
1975
1976 put_disk(md->disk);
1977 blk_cleanup_queue(md->queue);

--- 126 unchanged lines hidden (view full) ---

2104 * Returns old map, which caller must destroy.
2105 */
2106static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2107 struct queue_limits *limits)
2108{
2109 struct dm_table *old_map;
2110 struct request_queue *q = md->queue;
2111 sector_t size;
2005 free_minor(minor);
2006
2007 spin_lock(&_minor_lock);
2008 md->disk->private_data = NULL;
2009 spin_unlock(&_minor_lock);
2010
2011 put_disk(md->disk);
2012 blk_cleanup_queue(md->queue);

--- 126 unchanged lines hidden (view full) ---

2139 * Returns old map, which caller must destroy.
2140 */
2141static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2142 struct queue_limits *limits)
2143{
2144 struct dm_table *old_map;
2145 struct request_queue *q = md->queue;
2146 sector_t size;
2112 unsigned long flags;
2113 int merge_is_optional;
2114
2115 size = dm_table_get_size(t);
2116
2117 /*
2118 * Wipe any geometry if the size of the table changed.
2119 */
2120 if (size != get_capacity(md->disk))

--- 12 unchanged lines hidden (view full) ---

2133 */
2134 if (dm_table_request_based(t) && !blk_queue_stopped(q))
2135 stop_queue(q);
2136
2137 __bind_mempools(md, t);
2138
2139 merge_is_optional = dm_table_merge_is_optional(t);
2140
2147 int merge_is_optional;
2148
2149 size = dm_table_get_size(t);
2150
2151 /*
2152 * Wipe any geometry if the size of the table changed.
2153 */
2154 if (size != get_capacity(md->disk))

--- 12 unchanged lines hidden (view full) ---

2167 */
2168 if (dm_table_request_based(t) && !blk_queue_stopped(q))
2169 stop_queue(q);
2170
2171 __bind_mempools(md, t);
2172
2173 merge_is_optional = dm_table_merge_is_optional(t);
2174
2141 write_lock_irqsave(&md->map_lock, flags);
2142 old_map = md->map;
2175 old_map = md->map;
2143 md->map = t;
2176 rcu_assign_pointer(md->map, t);
2144 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2145
2146 dm_table_set_restrictions(t, q, limits);
2147 if (merge_is_optional)
2148 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2149 else
2150 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2177 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2178
2179 dm_table_set_restrictions(t, q, limits);
2180 if (merge_is_optional)
2181 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2182 else
2183 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2151 write_unlock_irqrestore(&md->map_lock, flags);
2184 dm_sync_table(md);
2152
2153 return old_map;
2154}
2155
2156/*
2157 * Returns unbound table for the caller to free.
2158 */
2159static struct dm_table *__unbind(struct mapped_device *md)
2160{
2161 struct dm_table *map = md->map;
2185
2186 return old_map;
2187}
2188
2189/*
2190 * Returns unbound table for the caller to free.
2191 */
2192static struct dm_table *__unbind(struct mapped_device *md)
2193{
2194 struct dm_table *map = md->map;
2162 unsigned long flags;
2163
2164 if (!map)
2165 return NULL;
2166
2167 dm_table_event_callback(map, NULL, NULL);
2195
2196 if (!map)
2197 return NULL;
2198
2199 dm_table_event_callback(map, NULL, NULL);
2168 write_lock_irqsave(&md->map_lock, flags);
2169 md->map = NULL;
2170 write_unlock_irqrestore(&md->map_lock, flags);
2200 rcu_assign_pointer(md->map, NULL);
2201 dm_sync_table(md);
2171
2172 return map;
2173}
2174
2175/*
2176 * Constructor for a new device.
2177 */
2178int dm_create(int minor, struct mapped_device **result)

--- 135 unchanged lines hidden (view full) ---

2314{
2315 return md->name;
2316}
2317EXPORT_SYMBOL_GPL(dm_device_name);
2318
2319static void __dm_destroy(struct mapped_device *md, bool wait)
2320{
2321 struct dm_table *map;
2202
2203 return map;
2204}
2205
2206/*
2207 * Constructor for a new device.
2208 */
2209int dm_create(int minor, struct mapped_device **result)

--- 135 unchanged lines hidden (view full) ---

2345{
2346 return md->name;
2347}
2348EXPORT_SYMBOL_GPL(dm_device_name);
2349
2350static void __dm_destroy(struct mapped_device *md, bool wait)
2351{
2352 struct dm_table *map;
2353 int srcu_idx;
2322
2323 might_sleep();
2324
2325 spin_lock(&_minor_lock);
2354
2355 might_sleep();
2356
2357 spin_lock(&_minor_lock);
2326 map = dm_get_live_table(md);
2358 map = dm_get_live_table(md, &srcu_idx);
2327 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2328 set_bit(DMF_FREEING, &md->flags);
2329 spin_unlock(&_minor_lock);
2330
2331 if (!dm_suspended_md(md)) {
2332 dm_table_presuspend_targets(map);
2333 dm_table_postsuspend_targets(map);
2334 }
2335
2359 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2360 set_bit(DMF_FREEING, &md->flags);
2361 spin_unlock(&_minor_lock);
2362
2363 if (!dm_suspended_md(md)) {
2364 dm_table_presuspend_targets(map);
2365 dm_table_postsuspend_targets(map);
2366 }
2367
2368 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2369 dm_put_live_table(md, srcu_idx);
2370
2336 /*
2337 * Rare, but there may be I/O requests still going to complete,
2338 * for example. Wait for all references to disappear.
2339 * No one should increment the reference count of the mapped_device,
2340 * after the mapped_device state becomes DMF_FREEING.
2341 */
2342 if (wait)
2343 while (atomic_read(&md->holders))
2344 msleep(1);
2345 else if (atomic_read(&md->holders))
2346 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2347 dm_device_name(md), atomic_read(&md->holders));
2348
2349 dm_sysfs_exit(md);
2371 /*
2372 * Rare, but there may be I/O requests still going to complete,
2373 * for example. Wait for all references to disappear.
2374 * No one should increment the reference count of the mapped_device,
2375 * after the mapped_device state becomes DMF_FREEING.
2376 */
2377 if (wait)
2378 while (atomic_read(&md->holders))
2379 msleep(1);
2380 else if (atomic_read(&md->holders))
2381 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2382 dm_device_name(md), atomic_read(&md->holders));
2383
2384 dm_sysfs_exit(md);
2350 dm_table_put(map);
2351 dm_table_destroy(__unbind(md));
2352 free_dev(md);
2353}
2354
2355void dm_destroy(struct mapped_device *md)
2356{
2357 __dm_destroy(md, true);
2358}

--- 40 unchanged lines hidden (view full) ---

2399/*
2400 * Process the deferred bios
2401 */
2402static void dm_wq_work(struct work_struct *work)
2403{
2404 struct mapped_device *md = container_of(work, struct mapped_device,
2405 work);
2406 struct bio *c;
2385 dm_table_destroy(__unbind(md));
2386 free_dev(md);
2387}
2388
2389void dm_destroy(struct mapped_device *md)
2390{
2391 __dm_destroy(md, true);
2392}

--- 40 unchanged lines hidden (view full) ---

2433/*
2434 * Process the deferred bios
2435 */
2436static void dm_wq_work(struct work_struct *work)
2437{
2438 struct mapped_device *md = container_of(work, struct mapped_device,
2439 work);
2440 struct bio *c;
2441 int srcu_idx;
2442 struct dm_table *map;
2407
2443
2408 down_read(&md->io_lock);
2444 map = dm_get_live_table(md, &srcu_idx);
2409
2410 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2411 spin_lock_irq(&md->deferred_lock);
2412 c = bio_list_pop(&md->deferred);
2413 spin_unlock_irq(&md->deferred_lock);
2414
2415 if (!c)
2416 break;
2417
2445
2446 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2447 spin_lock_irq(&md->deferred_lock);
2448 c = bio_list_pop(&md->deferred);
2449 spin_unlock_irq(&md->deferred_lock);
2450
2451 if (!c)
2452 break;
2453
2418 up_read(&md->io_lock);
2419
2420 if (dm_request_based(md))
2421 generic_make_request(c);
2422 else
2454 if (dm_request_based(md))
2455 generic_make_request(c);
2456 else
2423 __split_and_process_bio(md, c);
2424
2425 down_read(&md->io_lock);
2457 __split_and_process_bio(md, map, c);
2426 }
2427
2458 }
2459
2428 up_read(&md->io_lock);
2460 dm_put_live_table(md, srcu_idx);
2429}
2430
2431static void dm_queue_flush(struct mapped_device *md)
2432{
2433 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2434 smp_mb__after_clear_bit();
2435 queue_work(md->wq, &md->work);
2436}

--- 15 unchanged lines hidden (view full) ---

2452
2453 /*
2454 * If the new table has no data devices, retain the existing limits.
2455 * This helps multipath with queue_if_no_path if all paths disappear,
2456 * then new I/O is queued based on these limits, and then some paths
2457 * reappear.
2458 */
2459 if (dm_table_has_no_data_devices(table)) {
2461}
2462
2463static void dm_queue_flush(struct mapped_device *md)
2464{
2465 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2466 smp_mb__after_clear_bit();
2467 queue_work(md->wq, &md->work);
2468}

--- 15 unchanged lines hidden (view full) ---

2484
2485 /*
2486 * If the new table has no data devices, retain the existing limits.
2487 * This helps multipath with queue_if_no_path if all paths disappear,
2488 * then new I/O is queued based on these limits, and then some paths
2489 * reappear.
2490 */
2491 if (dm_table_has_no_data_devices(table)) {
2460 live_map = dm_get_live_table(md);
2492 live_map = dm_get_live_table_fast(md);
2461 if (live_map)
2462 limits = md->queue->limits;
2493 if (live_map)
2494 limits = md->queue->limits;
2463 dm_table_put(live_map);
2495 dm_put_live_table_fast(md);
2464 }
2465
2466 if (!live_map) {
2467 r = dm_calculate_queue_limits(table, &limits);
2468 if (r) {
2469 map = ERR_PTR(r);
2470 goto out;
2471 }

--- 63 unchanged lines hidden (view full) ---

2535
2536 mutex_lock(&md->suspend_lock);
2537
2538 if (dm_suspended_md(md)) {
2539 r = -EINVAL;
2540 goto out_unlock;
2541 }
2542
2496 }
2497
2498 if (!live_map) {
2499 r = dm_calculate_queue_limits(table, &limits);
2500 if (r) {
2501 map = ERR_PTR(r);
2502 goto out;
2503 }

--- 63 unchanged lines hidden (view full) ---

2567
2568 mutex_lock(&md->suspend_lock);
2569
2570 if (dm_suspended_md(md)) {
2571 r = -EINVAL;
2572 goto out_unlock;
2573 }
2574
2543 map = dm_get_live_table(md);
2575 map = md->map;
2544
2545 /*
2546 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2547 * This flag is cleared before dm_suspend returns.
2548 */
2549 if (noflush)
2550 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2551

--- 4 unchanged lines hidden (view full) ---

2556 * Flush I/O to the device.
2557 * Any I/O submitted after lock_fs() may not be flushed.
2558 * noflush takes precedence over do_lockfs.
2559 * (lock_fs() flushes I/Os and waits for them to complete.)
2560 */
2561 if (!noflush && do_lockfs) {
2562 r = lock_fs(md);
2563 if (r)
2576
2577 /*
2578 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2579 * This flag is cleared before dm_suspend returns.
2580 */
2581 if (noflush)
2582 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2583

--- 4 unchanged lines hidden (view full) ---

2588 * Flush I/O to the device.
2589 * Any I/O submitted after lock_fs() may not be flushed.
2590 * noflush takes precedence over do_lockfs.
2591 * (lock_fs() flushes I/Os and waits for them to complete.)
2592 */
2593 if (!noflush && do_lockfs) {
2594 r = lock_fs(md);
2595 if (r)
2564 goto out;
2596 goto out_unlock;
2565 }
2566
2567 /*
2568 * Here we must make sure that no processes are submitting requests
2569 * to target drivers i.e. no one may be executing
2570 * __split_and_process_bio. This is called from dm_request and
2571 * dm_wq_work.
2572 *
2573 * To get all processes out of __split_and_process_bio in dm_request,
2574 * we take the write lock. To prevent any process from reentering
2575 * __split_and_process_bio from dm_request and quiesce the thread
2576 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2577 * flush_workqueue(md->wq).
2578 */
2597 }
2598
2599 /*
2600 * Here we must make sure that no processes are submitting requests
2601 * to target drivers i.e. no one may be executing
2602 * __split_and_process_bio. This is called from dm_request and
2603 * dm_wq_work.
2604 *
2605 * To get all processes out of __split_and_process_bio in dm_request,
2606 * we take the write lock. To prevent any process from reentering
2607 * __split_and_process_bio from dm_request and quiesce the thread
2608 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2609 * flush_workqueue(md->wq).
2610 */
2579 down_write(&md->io_lock);
2580 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2611 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2581 up_write(&md->io_lock);
2612 synchronize_srcu(&md->io_barrier);
2582
2583 /*
2584 * Stop md->queue before flushing md->wq in case request-based
2585 * dm defers requests to md->wq from md->queue.
2586 */
2587 if (dm_request_based(md))
2588 stop_queue(md->queue);
2589
2590 flush_workqueue(md->wq);
2591
2592 /*
2593 * At this point no more requests are entering target request routines.
2594 * We call dm_wait_for_completion to wait for all existing requests
2595 * to finish.
2596 */
2597 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
2598
2613
2614 /*
2615 * Stop md->queue before flushing md->wq in case request-based
2616 * dm defers requests to md->wq from md->queue.
2617 */
2618 if (dm_request_based(md))
2619 stop_queue(md->queue);
2620
2621 flush_workqueue(md->wq);
2622
2623 /*
2624 * At this point no more requests are entering target request routines.
2625 * We call dm_wait_for_completion to wait for all existing requests
2626 * to finish.
2627 */
2628 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
2629
2599 down_write(&md->io_lock);
2600 if (noflush)
2601 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2630 if (noflush)
2631 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2602 up_write(&md->io_lock);
2632 synchronize_srcu(&md->io_barrier);
2603
2604 /* were we interrupted ? */
2605 if (r < 0) {
2606 dm_queue_flush(md);
2607
2608 if (dm_request_based(md))
2609 start_queue(md->queue);
2610
2611 unlock_fs(md);
2633
2634 /* were we interrupted ? */
2635 if (r < 0) {
2636 dm_queue_flush(md);
2637
2638 if (dm_request_based(md))
2639 start_queue(md->queue);
2640
2641 unlock_fs(md);
2612 goto out; /* pushback list is already flushed, so skip flush */
2642 goto out_unlock; /* pushback list is already flushed, so skip flush */
2613 }
2614
2615 /*
2616 * If dm_wait_for_completion returned 0, the device is completely
2617 * quiescent now. There is no request-processing activity. All new
2618 * requests are being added to md->deferred list.
2619 */
2620
2621 set_bit(DMF_SUSPENDED, &md->flags);
2622
2623 dm_table_postsuspend_targets(map);
2624
2643 }
2644
2645 /*
2646 * If dm_wait_for_completion returned 0, the device is completely
2647 * quiescent now. There is no request-processing activity. All new
2648 * requests are being added to md->deferred list.
2649 */
2650
2651 set_bit(DMF_SUSPENDED, &md->flags);
2652
2653 dm_table_postsuspend_targets(map);
2654
2625out:
2626 dm_table_put(map);
2627
2628out_unlock:
2629 mutex_unlock(&md->suspend_lock);
2630 return r;
2631}
2632
2633int dm_resume(struct mapped_device *md)
2634{
2635 int r = -EINVAL;
2636 struct dm_table *map = NULL;
2637
2638 mutex_lock(&md->suspend_lock);
2639 if (!dm_suspended_md(md))
2640 goto out;
2641
2655out_unlock:
2656 mutex_unlock(&md->suspend_lock);
2657 return r;
2658}
2659
2660int dm_resume(struct mapped_device *md)
2661{
2662 int r = -EINVAL;
2663 struct dm_table *map = NULL;
2664
2665 mutex_lock(&md->suspend_lock);
2666 if (!dm_suspended_md(md))
2667 goto out;
2668
2642 map = dm_get_live_table(md);
2669 map = md->map;
2643 if (!map || !dm_table_get_size(map))
2644 goto out;
2645
2646 r = dm_table_resume_targets(map);
2647 if (r)
2648 goto out;
2649
2650 dm_queue_flush(md);

--- 7 unchanged lines hidden (view full) ---

2658 start_queue(md->queue);
2659
2660 unlock_fs(md);
2661
2662 clear_bit(DMF_SUSPENDED, &md->flags);
2663
2664 r = 0;
2665out:
2670 if (!map || !dm_table_get_size(map))
2671 goto out;
2672
2673 r = dm_table_resume_targets(map);
2674 if (r)
2675 goto out;
2676
2677 dm_queue_flush(md);

--- 7 unchanged lines hidden (view full) ---

2685 start_queue(md->queue);
2686
2687 unlock_fs(md);
2688
2689 clear_bit(DMF_SUSPENDED, &md->flags);
2690
2691 r = 0;
2692out:
2666 dm_table_put(map);
2667 mutex_unlock(&md->suspend_lock);
2668
2669 return r;
2670}
2671
2672/*-----------------------------------------------------------------
2673 * Event notification.
2674 *---------------------------------------------------------------*/

--- 169 unchanged lines hidden --- 		2693 mutex_unlock(&md->suspend_lock);
2694
2695 return r;
2696}
2697
2698/*-----------------------------------------------------------------
2699 * Event notification.
2700 *---------------------------------------------------------------*/

--- 169 unchanged lines hidden ---