dm.c (82f6cdcc3676c68abaad2aac51a32f4e5d67d01e) dm.c (655f3aad7aa4858d06cdaca6c4b14635cc3c0eba)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 80 unchanged lines hidden (view full) ---

89
90static inline struct dm_target_io *clone_to_tio(struct bio *clone)
91{
92 return container_of(clone, struct dm_target_io, clone);
93}
94
95void *dm_per_bio_data(struct bio *bio, size_t data_size)
96{
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 80 unchanged lines hidden (view full) ---

89
90static inline struct dm_target_io *clone_to_tio(struct bio *clone)
91{
92 return container_of(clone, struct dm_target_io, clone);
93}
94
95void *dm_per_bio_data(struct bio *bio, size_t data_size)
96{
97 if (!clone_to_tio(bio)->inside_dm_io)
97 if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO))
98 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
99 return (char *)bio - DM_IO_BIO_OFFSET - data_size;
100}
101EXPORT_SYMBOL_GPL(dm_per_bio_data);
102
103struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
104{
105 struct dm_io *io = (struct dm_io *)((char *)data + data_size);

--- 427 unchanged lines hidden (view full) ---

533
534static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
535{
536 /* Must account IO to DM device in terms of orig_bio */
537 struct bio *bio = io->orig_bio;
538
539 /*
540 * Ensure IO accounting is only ever started once.
98 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
99 return (char *)bio - DM_IO_BIO_OFFSET - data_size;
100}
101EXPORT_SYMBOL_GPL(dm_per_bio_data);
102
103struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
104{
105 struct dm_io *io = (struct dm_io *)((char *)data + data_size);

--- 427 unchanged lines hidden (view full) ---

533
534static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
535{
536 /* Must account IO to DM device in terms of orig_bio */
537 struct bio *bio = io->orig_bio;
538
539 /*
540 * Ensure IO accounting is only ever started once.
541 * Expect no possibility for race unless is_duplicate_bio.
541 * Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO.
542 */
542 */
543 if (!clone || likely(!clone_to_tio(clone)->is_duplicate_bio)) {
543 if (!clone ||
544 likely(!dm_tio_flagged(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO))) {
544 if (WARN_ON_ONCE(dm_io_flagged(io, DM_IO_ACCOUNTED)))
545 return;
546 dm_io_set_flag(io, DM_IO_ACCOUNTED);
547 } else {
548 unsigned long flags;
549 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
550 return;
545 if (WARN_ON_ONCE(dm_io_flagged(io, DM_IO_ACCOUNTED)))
546 return;
547 dm_io_set_flag(io, DM_IO_ACCOUNTED);
548 } else {
549 unsigned long flags;
550 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
551 return;
551 /* Can afford locking given is_duplicate_bio */
552 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
552 spin_lock_irqsave(&io->startio_lock, flags);
553 dm_io_set_flag(io, DM_IO_ACCOUNTED);
554 spin_unlock_irqrestore(&io->startio_lock, flags);
555 }
556
557 __dm_start_io_acct(io, bio);
558}
559

--- 6 unchanged lines hidden (view full) ---

566{
567 struct dm_io *io;
568 struct dm_target_io *tio;
569 struct bio *clone;
570
571 clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs);
572
573 tio = clone_to_tio(clone);
553 spin_lock_irqsave(&io->startio_lock, flags);
554 dm_io_set_flag(io, DM_IO_ACCOUNTED);
555 spin_unlock_irqrestore(&io->startio_lock, flags);
556 }
557
558 __dm_start_io_acct(io, bio);
559}
560

--- 6 unchanged lines hidden (view full) ---

567{
568 struct dm_io *io;
569 struct dm_target_io *tio;
570 struct bio *clone;
571
572 clone = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, &md->io_bs);
573
574 tio = clone_to_tio(clone);
574 tio->inside_dm_io = true;
575 tio->flags = 0;
576 dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO);
575 tio->io = NULL;
576
577 io = container_of(tio, struct dm_io, tio);
578 io->magic = DM_IO_MAGIC;
579 io->status = 0;
580 atomic_set(&io->io_count, 1);
581 this_cpu_inc(*md->pending_io);
582 io->orig_bio = NULL;

--- 30 unchanged lines hidden (view full) ---

613 gfp_mask, &ci->io->md->bs);
614 if (!clone)
615 return NULL;
616
617 /* REQ_DM_POLL_LIST shouldn't be inherited */
618 clone->bi_opf &= ~REQ_DM_POLL_LIST;
619
620 tio = clone_to_tio(clone);
577 tio->io = NULL;
578
579 io = container_of(tio, struct dm_io, tio);
580 io->magic = DM_IO_MAGIC;
581 io->status = 0;
582 atomic_set(&io->io_count, 1);
583 this_cpu_inc(*md->pending_io);
584 io->orig_bio = NULL;

--- 30 unchanged lines hidden (view full) ---

615 gfp_mask, &ci->io->md->bs);
616 if (!clone)
617 return NULL;
618
619 /* REQ_DM_POLL_LIST shouldn't be inherited */
620 clone->bi_opf &= ~REQ_DM_POLL_LIST;
621
622 tio = clone_to_tio(clone);
621 tio->inside_dm_io = false;
623 tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */
622 }
623
624 tio->magic = DM_TIO_MAGIC;
625 tio->io = ci->io;
626 tio->ti = ti;
627 tio->target_bio_nr = target_bio_nr;
624 }
625
626 tio->magic = DM_TIO_MAGIC;
627 tio->io = ci->io;
628 tio->ti = ti;
629 tio->target_bio_nr = target_bio_nr;
628 tio->is_duplicate_bio = false;
629 tio->len_ptr = len;
630 tio->old_sector = 0;
631
632 if (len) {
633 clone->bi_iter.bi_size = to_bytes(*len);
634 if (bio_integrity(clone))
635 bio_integrity_trim(clone);
636 }
637
638 return clone;
639}
640
641static void free_tio(struct bio *clone)
642{
630 tio->len_ptr = len;
631 tio->old_sector = 0;
632
633 if (len) {
634 clone->bi_iter.bi_size = to_bytes(*len);
635 if (bio_integrity(clone))
636 bio_integrity_trim(clone);
637 }
638
639 return clone;
640}
641
642static void free_tio(struct bio *clone)
643{
643 if (clone_to_tio(clone)->inside_dm_io)
644 if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO))
644 return;
645 bio_put(clone);
646}
647
648/*
649 * Add the bio to the list of deferred io.
650 */
651static void queue_io(struct mapped_device *md, struct bio *bio)

--- 260 unchanged lines hidden (view full) ---

912 } else {
913 /* done with normal IO or empty flush */
914 if (io_error)
915 bio->bi_status = io_error;
916 bio_endio(bio);
917 }
918}
919
645 return;
646 bio_put(clone);
647}
648
649/*
650 * Add the bio to the list of deferred io.
651 */
652static void queue_io(struct mapped_device *md, struct bio *bio)

--- 260 unchanged lines hidden (view full) ---

913 } else {
914 /* done with normal IO or empty flush */
915 if (io_error)
916 bio->bi_status = io_error;
917 bio_endio(bio);
918 }
919}
920
921static inline bool dm_tio_is_normal(struct dm_target_io *tio)
922{
923 return (dm_tio_flagged(tio, DM_TIO_INSIDE_DM_IO) &&
924 !dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
925}
926
920/*
921 * Decrements the number of outstanding ios that a bio has been
922 * cloned into, completing the original io if necc.
923 */
924void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
925{
926 /* Push-back supersedes any I/O errors */
927 if (unlikely(error)) {

--- 247 unchanged lines hidden (view full) ---

1175 * the partially processed part (the sum of regions 1+2) must be the same for all
1176 * copies of the bio.
1177 */
1178void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1179{
1180 struct dm_target_io *tio = clone_to_tio(bio);
1181 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1182
927/*
928 * Decrements the number of outstanding ios that a bio has been
929 * cloned into, completing the original io if necc.
930 */
931void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
932{
933 /* Push-back supersedes any I/O errors */
934 if (unlikely(error)) {

--- 247 unchanged lines hidden (view full) ---

1182 * the partially processed part (the sum of regions 1+2) must be the same for all
1183 * copies of the bio.
1184 */
1185void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1186{
1187 struct dm_target_io *tio = clone_to_tio(bio);
1188 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1189
1183 BUG_ON(tio->is_duplicate_bio);
1190 BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
1184 BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1185 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1186 BUG_ON(bi_size > *tio->len_ptr);
1187 BUG_ON(n_sectors > bi_size);
1188
1189 *tio->len_ptr -= bi_size - n_sectors;
1190 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1191}

--- 165 unchanged lines hidden (view full) ---

1357 struct bio_list blist = BIO_EMPTY_LIST;
1358 struct bio *clone;
1359
1360 switch (num_bios) {
1361 case 0:
1362 break;
1363 case 1:
1364 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1191 BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1192 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1193 BUG_ON(bi_size > *tio->len_ptr);
1194 BUG_ON(n_sectors > bi_size);
1195
1196 *tio->len_ptr -= bi_size - n_sectors;
1197 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1198}

--- 165 unchanged lines hidden (view full) ---

1364 struct bio_list blist = BIO_EMPTY_LIST;
1365 struct bio *clone;
1366
1367 switch (num_bios) {
1368 case 0:
1369 break;
1370 case 1:
1371 clone = alloc_tio(ci, ti, 0, len, GFP_NOIO);
1365 clone_to_tio(clone)->is_duplicate_bio = true;
1372 dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
1366 __map_bio(clone);
1367 break;
1368 default:
1369 alloc_multiple_bios(&blist, ci, ti, num_bios, len);
1370 while ((clone = bio_list_pop(&blist))) {
1373 __map_bio(clone);
1374 break;
1375 default:
1376 alloc_multiple_bios(&blist, ci, ti, num_bios, len);
1377 while ((clone = bio_list_pop(&blist))) {
1371 clone_to_tio(clone)->is_duplicate_bio = true;
1378 dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO);
1372 __map_bio(clone);
1373 }
1374 break;
1375 }
1376}
1377
1378static void __send_empty_flush(struct clone_info *ci)
1379{

--- 263 unchanged lines hidden (view full) ---

1643 dm_split_and_process_bio(md, map, bio);
1644out:
1645 dm_put_live_table(md, srcu_idx);
1646}
1647
1648static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
1649 unsigned int flags)
1650{
1379 __map_bio(clone);
1380 }
1381 break;
1382 }
1383}
1384
1385static void __send_empty_flush(struct clone_info *ci)
1386{

--- 263 unchanged lines hidden (view full) ---

1650 dm_split_and_process_bio(md, map, bio);
1651out:
1652 dm_put_live_table(md, srcu_idx);
1653}
1654
1655static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
1656 unsigned int flags)
1657{
1651 WARN_ON_ONCE(!io->tio.inside_dm_io);
1658 WARN_ON_ONCE(!dm_tio_is_normal(&io->tio));
1652
1653 /* don't poll if the mapped io is done */
1654 if (atomic_read(&io->io_count) > 1)
1655 bio_poll(&io->tio.clone, iob, flags);
1656
1657 /* bio_poll holds the last reference */
1658 return atomic_read(&io->io_count) == 1;
1659}

--- 1519 unchanged lines hidden --- 		1659
1660 /* don't poll if the mapped io is done */
1661 if (atomic_read(&io->io_count) > 1)
1662 bio_poll(&io->tio.clone, iob, flags);
1663
1664 /* bio_poll holds the last reference */
1665 return atomic_read(&io->io_count) == 1;
1666}

--- 1519 unchanged lines hidden ---