dm.c (9f6dc633761006f974701d4c88da71ab68670749) dm.c (8d394bc4adf588ca4a0650745167cb83f86c18c9)
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 471 unchanged lines hidden (view full) ---

480}
481
482u64 dm_start_time_ns_from_clone(struct bio *bio)
483{
484 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
485}
486EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
487
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"

--- 471 unchanged lines hidden (view full) ---

480}
481
482u64 dm_start_time_ns_from_clone(struct bio *bio)
483{
484 return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time);
485}
486EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
487
488static void start_io_acct(struct dm_io *io)
488static bool bio_is_flush_with_data(struct bio *bio)
489{
489{
490 struct mapped_device *md = io->md;
491 struct bio *bio = io->orig_bio;
490 return ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size);
491}
492
492
493 bio_start_io_acct_time(bio, io->start_time);
493static void dm_io_acct(bool end, struct mapped_device *md, struct bio *bio,
494 unsigned long start_time, struct dm_stats_aux *stats_aux)
495{
496 bool is_flush_with_data;
497 unsigned int bi_size;
498
499 /* If REQ_PREFLUSH set save any payload but do not account it */
500 is_flush_with_data = bio_is_flush_with_data(bio);
501 if (is_flush_with_data) {
502 bi_size = bio->bi_iter.bi_size;
503 bio->bi_iter.bi_size = 0;
504 }
505
506 if (!end)
507 bio_start_io_acct_time(bio, start_time);
508 else
509 bio_end_io_acct(bio, start_time);
510
494 if (unlikely(dm_stats_used(&md->stats)))
495 dm_stats_account_io(&md->stats, bio_data_dir(bio),
496 bio->bi_iter.bi_sector, bio_sectors(bio),
511 if (unlikely(dm_stats_used(&md->stats)))
512 dm_stats_account_io(&md->stats, bio_data_dir(bio),
513 bio->bi_iter.bi_sector, bio_sectors(bio),
497 false, 0, &io->stats_aux);
514 end, start_time, stats_aux);
515
516 /* Restore bio's payload so it does get accounted upon requeue */
517 if (is_flush_with_data)
518 bio->bi_iter.bi_size = bi_size;
498}
499
519}
520
521static void start_io_acct(struct dm_io *io)
522{
523 dm_io_acct(false, io->md, io->orig_bio, io->start_time, &io->stats_aux);
524}
525
500static void end_io_acct(struct mapped_device *md, struct bio *bio,
501 unsigned long start_time, struct dm_stats_aux *stats_aux)
502{
526static void end_io_acct(struct mapped_device *md, struct bio *bio,
527 unsigned long start_time, struct dm_stats_aux *stats_aux)
528{
503 unsigned long duration = jiffies - start_time;
504
505 bio_end_io_acct(bio, start_time);
506
507 if (unlikely(dm_stats_used(&md->stats)))
508 dm_stats_account_io(&md->stats, bio_data_dir(bio),
509 bio->bi_iter.bi_sector, bio_sectors(bio),
510 true, duration, stats_aux);
529 dm_io_acct(true, md, bio, start_time, stats_aux);
511}
512
513static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
514{
515 struct dm_io *io;
516 struct dm_target_io *tio;
517 struct bio *clone;
518

--- 310 unchanged lines hidden (view full) ---

829
830 /* nudge anyone waiting on suspend queue */
831 if (unlikely(wq_has_sleeper(&md->wait)))
832 wake_up(&md->wait);
833
834 if (io_error == BLK_STS_DM_REQUEUE)
835 return;
836
530}
531
532static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
533{
534 struct dm_io *io;
535 struct dm_target_io *tio;
536 struct bio *clone;
537

--- 310 unchanged lines hidden (view full) ---

848
849 /* nudge anyone waiting on suspend queue */
850 if (unlikely(wq_has_sleeper(&md->wait)))
851 wake_up(&md->wait);
852
853 if (io_error == BLK_STS_DM_REQUEUE)
854 return;
855
837 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
856 if (bio_is_flush_with_data(bio)) {
838 /*
839 * Preflush done for flush with data, reissue
840 * without REQ_PREFLUSH.
841 */
842 bio->bi_opf &= ~REQ_PREFLUSH;
843 queue_io(md, bio);
844 } else {
845 /* done with normal IO or empty flush */

--- 2107 unchanged lines hidden --- 		857 /*
858 * Preflush done for flush with data, reissue
859 * without REQ_PREFLUSH.
860 */
861 bio->bi_opf &= ~REQ_PREFLUSH;
862 queue_io(md, bio);
863 } else {
864 /* done with normal IO or empty flush */

--- 2107 unchanged lines hidden ---