dm.c (4857abf66434d96dec1a9209bbfbf55e2ee359fa) dm.c (3b03f7c1242c754f0c474b37eec7d79107b9f375)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 524 unchanged lines hidden (view full) ---

533
534static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
535{
536 /* Must account IO to DM device in terms of orig_bio */
537 struct bio *bio = io->orig_bio;
538
539 /*
540 * Ensure IO accounting is only ever started once.
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 524 unchanged lines hidden (view full) ---

533
534static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
535{
536 /* Must account IO to DM device in terms of orig_bio */
537 struct bio *bio = io->orig_bio;
538
539 /*
540 * Ensure IO accounting is only ever started once.
541 * Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO.
542 */
541 */
543 if (!clone ||
544 likely(!dm_tio_flagged(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO))) {
545 if (WARN_ON_ONCE(dm_io_flagged(io, DM_IO_ACCOUNTED)))
546 return;
542 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
543 return;
544
545 /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */
546 if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) {
547 dm_io_set_flag(io, DM_IO_ACCOUNTED);
548 } else {
549 unsigned long flags;
547 dm_io_set_flag(io, DM_IO_ACCOUNTED);
548 } else {
549 unsigned long flags;
550 if (dm_io_flagged(io, DM_IO_ACCOUNTED))
551 return;
552 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
553 spin_lock_irqsave(&io->lock, flags);
554 dm_io_set_flag(io, DM_IO_ACCOUNTED);
555 spin_unlock_irqrestore(&io->lock, flags);
556 }
557
558 __dm_start_io_acct(io, bio);
559}

--- 358 unchanged lines hidden (view full) ---

918 } else {
919 /* done with normal IO or empty flush */
920 if (io_error)
921 bio->bi_status = io_error;
922 bio_endio(bio);
923 }
924}
925
550 /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */
551 spin_lock_irqsave(&io->lock, flags);
552 dm_io_set_flag(io, DM_IO_ACCOUNTED);
553 spin_unlock_irqrestore(&io->lock, flags);
554 }
555
556 __dm_start_io_acct(io, bio);
557}

--- 358 unchanged lines hidden (view full) ---

916 } else {
917 /* done with normal IO or empty flush */
918 if (io_error)
919 bio->bi_status = io_error;
920 bio_endio(bio);
921 }
922}
923
926static inline bool dm_tio_is_normal(struct dm_target_io *tio)
927{
928 return (dm_tio_flagged(tio, DM_TIO_INSIDE_DM_IO) &&
929 !dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO));
930}
931
932/*
933 * Decrements the number of outstanding ios that a bio has been
934 * cloned into, completing the original io if necc.
935 */
936static inline void __dm_io_dec_pending(struct dm_io *io)
937{
938 if (atomic_dec_and_test(&io->io_count))
939 dm_io_complete(io);

--- 2241 unchanged lines hidden --- 		924/*
925 * Decrements the number of outstanding ios that a bio has been
926 * cloned into, completing the original io if necc.
927 */
928static inline void __dm_io_dec_pending(struct dm_io *io)
929{
930 if (atomic_dec_and_test(&io->io_count))
931 dm_io_complete(io);

--- 2241 unchanged lines hidden ---