dm-mpath.c (ddb9e13af3bba3f8c36ccee0eb9563a82b425c12) dm-mpath.c (ac514ffc968bf14649dd0e048447dc966ee49555)
1/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include <linux/device-mapper.h>

--- 50 unchanged lines hidden (view full) ---

59 unsigned nr_pgpaths; /* Number of paths in PG */
60 struct list_head pgpaths;
61
62 bool bypassed:1; /* Temporarily bypass this PG? */
63};
64
65/* Multipath context */
66struct multipath {
1/*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include <linux/device-mapper.h>

--- 50 unchanged lines hidden (view full) ---

59 unsigned nr_pgpaths; /* Number of paths in PG */
60 struct list_head pgpaths;
61
62 bool bypassed:1; /* Temporarily bypass this PG? */
63};
64
65/* Multipath context */
66struct multipath {
67 struct list_head list;
68 struct dm_target *ti;
67 unsigned long flags; /* Multipath state flags */
69
68
70 const char *hw_handler_name;
71 char *hw_handler_params;
72
73 spinlock_t lock;
69 spinlock_t lock;
70 enum dm_queue_mode queue_mode;
74
71
75 unsigned nr_priority_groups;
76 struct list_head priority_groups;
77
78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
79
80 struct pgpath *current_pgpath;
81 struct priority_group *current_pg;
82 struct priority_group *next_pg; /* Switch to this PG if set */
83
72 struct pgpath *current_pgpath;
73 struct priority_group *current_pg;
74 struct priority_group *next_pg; /* Switch to this PG if set */
75
84 unsigned long flags; /* Multipath state flags */
76 atomic_t nr_valid_paths; /* Total number of usable paths */
77 unsigned nr_priority_groups;
78 struct list_head priority_groups;
85
79
80 const char *hw_handler_name;
81 char *hw_handler_params;
82 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
86 unsigned pg_init_retries; /* Number of times to retry pg_init */
87 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
83 unsigned pg_init_retries; /* Number of times to retry pg_init */
84 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
88
89 atomic_t nr_valid_paths; /* Total number of usable paths */
90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
91 atomic_t pg_init_count; /* Number of times pg_init called */
92
85 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
86 atomic_t pg_init_count; /* Number of times pg_init called */
87
93 enum dm_queue_mode queue_mode;
94
95 struct mutex work_mutex;
96 struct work_struct trigger_event;
88 struct mutex work_mutex;
89 struct work_struct trigger_event;
90 struct dm_target *ti;
97
98 struct work_struct process_queued_bios;
99 struct bio_list queued_bios;
100};
101
102/*
103 * Context information attached to each io we process.
104 */

--- 25 unchanged lines hidden (view full) ---

130/*-----------------------------------------------
131 * Allocation routines
132 *-----------------------------------------------*/
133
134static struct pgpath *alloc_pgpath(void)
135{
136 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
137
91
92 struct work_struct process_queued_bios;
93 struct bio_list queued_bios;
94};
95
96/*
97 * Context information attached to each io we process.
98 */

--- 25 unchanged lines hidden (view full) ---

124/*-----------------------------------------------
125 * Allocation routines
126 *-----------------------------------------------*/
127
128static struct pgpath *alloc_pgpath(void)
129{
130 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
131
138 if (pgpath) {
139 pgpath->is_active = true;
140 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path_work);
141 }
132 if (!pgpath)
133 return NULL;
142
134
135 pgpath->is_active = true;
136
143 return pgpath;
144}
145
146static void free_pgpath(struct pgpath *pgpath)
147{
148 kfree(pgpath);
149}
150

--- 37 unchanged lines hidden (view full) ---

188static struct multipath *alloc_multipath(struct dm_target *ti)
189{
190 struct multipath *m;
191
192 m = kzalloc(sizeof(*m), GFP_KERNEL);
193 if (m) {
194 INIT_LIST_HEAD(&m->priority_groups);
195 spin_lock_init(&m->lock);
137 return pgpath;
138}
139
140static void free_pgpath(struct pgpath *pgpath)
141{
142 kfree(pgpath);
143}
144

--- 37 unchanged lines hidden (view full) ---

182static struct multipath *alloc_multipath(struct dm_target *ti)
183{
184 struct multipath *m;
185
186 m = kzalloc(sizeof(*m), GFP_KERNEL);
187 if (m) {
188 INIT_LIST_HEAD(&m->priority_groups);
189 spin_lock_init(&m->lock);
196 set_bit(MPATHF_QUEUE_IO, &m->flags);
197 atomic_set(&m->nr_valid_paths, 0);
190 atomic_set(&m->nr_valid_paths, 0);
198 atomic_set(&m->pg_init_in_progress, 0);
199 atomic_set(&m->pg_init_count, 0);
200 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
201 INIT_WORK(&m->trigger_event, trigger_event);
191 INIT_WORK(&m->trigger_event, trigger_event);
202 init_waitqueue_head(&m->pg_init_wait);
203 mutex_init(&m->work_mutex);
204
205 m->queue_mode = DM_TYPE_NONE;
206
207 m->ti = ti;
208 ti->private = m;
209 }
210

--- 5 unchanged lines hidden (view full) ---

216 if (m->queue_mode == DM_TYPE_NONE) {
217 /*
218 * Default to request-based.
219 */
220 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
221 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
222 else
223 m->queue_mode = DM_TYPE_REQUEST_BASED;
192 mutex_init(&m->work_mutex);
193
194 m->queue_mode = DM_TYPE_NONE;
195
196 m->ti = ti;
197 ti->private = m;
198 }
199

--- 5 unchanged lines hidden (view full) ---

205 if (m->queue_mode == DM_TYPE_NONE) {
206 /*
207 * Default to request-based.
208 */
209 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
210 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
211 else
212 m->queue_mode = DM_TYPE_REQUEST_BASED;
224 } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
213
214 } else if (m->queue_mode == DM_TYPE_BIO_BASED ||
215 m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
225 INIT_WORK(&m->process_queued_bios, process_queued_bios);
216 INIT_WORK(&m->process_queued_bios, process_queued_bios);
226 /*
227 * bio-based doesn't support any direct scsi_dh management;
228 * it just discovers if a scsi_dh is attached.
229 */
230 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
217
218 if (m->queue_mode == DM_TYPE_BIO_BASED) {
219 /*
220 * bio-based doesn't support any direct scsi_dh management;
221 * it just discovers if a scsi_dh is attached.
222 */
223 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
224 }
231 }
232
225 }
226
227 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
228 set_bit(MPATHF_QUEUE_IO, &m->flags);
229 atomic_set(&m->pg_init_in_progress, 0);
230 atomic_set(&m->pg_init_count, 0);
231 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
232 init_waitqueue_head(&m->pg_init_wait);
233 }
234
233 dm_table_set_type(ti->table, m->queue_mode);
234
235 return 0;
236}
237
238static void free_multipath(struct multipath *m)
239{
240 struct priority_group *pg, *tmp;
241
242 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
243 list_del(&pg->list);
244 free_priority_group(pg, m->ti);
245 }
246
247 kfree(m->hw_handler_name);
248 kfree(m->hw_handler_params);
235 dm_table_set_type(ti->table, m->queue_mode);
236
237 return 0;
238}
239
240static void free_multipath(struct multipath *m)
241{
242 struct priority_group *pg, *tmp;
243
244 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
245 list_del(&pg->list);
246 free_priority_group(pg, m->ti);
247 }
248
249 kfree(m->hw_handler_name);
250 kfree(m->hw_handler_params);
251 mutex_destroy(&m->work_mutex);
249 kfree(m);
250}
251
252static struct dm_mpath_io *get_mpio(union map_info *info)
253{
254 return info->ptr;
255}
256
257static size_t multipath_per_bio_data_size(void)
258{
259 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
260}
261
262static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
263{
264 return dm_per_bio_data(bio, multipath_per_bio_data_size());
265}
266
252 kfree(m);
253}
254
255static struct dm_mpath_io *get_mpio(union map_info *info)
256{
257 return info->ptr;
258}
259
260static size_t multipath_per_bio_data_size(void)
261{
262 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
263}
264
265static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
266{
267 return dm_per_bio_data(bio, multipath_per_bio_data_size());
268}
269
267static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio)
270static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
268{
269 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
271{
272 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
270 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
271 void *bio_details = mpio + 1;
273 void *bio_details = mpio + 1;
272
273 return bio_details;
274}
275
274 return bio_details;
275}
276
276static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p,
277 struct dm_bio_details **bio_details_p)
277static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
278{
279 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
278{
279 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
280 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio);
280 struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
281
281
282 memset(mpio, 0, sizeof(*mpio));
283 memset(bio_details, 0, sizeof(*bio_details));
284 dm_bio_record(bio_details, bio);
282 mpio->nr_bytes = bio->bi_iter.bi_size;
283 mpio->pgpath = NULL;
284 *mpio_p = mpio;
285
285
286 if (mpio_p)
287 *mpio_p = mpio;
288 if (bio_details_p)
289 *bio_details_p = bio_details;
286 dm_bio_record(bio_details, bio);
290}
291
292/*-----------------------------------------------
293 * Path selection
294 *-----------------------------------------------*/
295
296static int __pg_init_all_paths(struct multipath *m)
297{

--- 37 unchanged lines hidden (view full) ---

335
336 return ret;
337}
338
339static void __switch_pg(struct multipath *m, struct priority_group *pg)
340{
341 m->current_pg = pg;
342
287}
288
289/*-----------------------------------------------
290 * Path selection
291 *-----------------------------------------------*/
292
293static int __pg_init_all_paths(struct multipath *m)
294{

--- 37 unchanged lines hidden (view full) ---

332
333 return ret;
334}
335
336static void __switch_pg(struct multipath *m, struct priority_group *pg)
337{
338 m->current_pg = pg;
339
340 if (m->queue_mode == DM_TYPE_NVME_BIO_BASED)
341 return;
342
343 /* Must we initialise the PG first, and queue I/O till it's ready? */
344 if (m->hw_handler_name) {
345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
346 set_bit(MPATHF_QUEUE_IO, &m->flags);
347 } else {
348 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
349 clear_bit(MPATHF_QUEUE_IO, &m->flags);
350 }

--- 29 unchanged lines hidden (view full) ---

380static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
381{
382 unsigned long flags;
383 struct priority_group *pg;
384 struct pgpath *pgpath;
385 unsigned bypassed = 1;
386
387 if (!atomic_read(&m->nr_valid_paths)) {
343 /* Must we initialise the PG first, and queue I/O till it's ready? */
344 if (m->hw_handler_name) {
345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
346 set_bit(MPATHF_QUEUE_IO, &m->flags);
347 } else {
348 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
349 clear_bit(MPATHF_QUEUE_IO, &m->flags);
350 }

--- 29 unchanged lines hidden (view full) ---

380static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
381{
382 unsigned long flags;
383 struct priority_group *pg;
384 struct pgpath *pgpath;
385 unsigned bypassed = 1;
386
387 if (!atomic_read(&m->nr_valid_paths)) {
388 clear_bit(MPATHF_QUEUE_IO, &m->flags);
388 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED)
389 clear_bit(MPATHF_QUEUE_IO, &m->flags);
389 goto failed;
390 }
391
392 /* Were we instructed to switch PG? */
393 if (READ_ONCE(m->next_pg)) {
394 spin_lock_irqsave(&m->lock, flags);
395 pg = m->next_pg;
396 if (!pg) {

--- 114 unchanged lines hidden (view full) ---

511
512 if (!pgpath) {
513 if (must_push_back_rq(m))
514 return DM_MAPIO_DELAY_REQUEUE;
515 dm_report_EIO(m); /* Failed */
516 return DM_MAPIO_KILL;
517 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
518 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
390 goto failed;
391 }
392
393 /* Were we instructed to switch PG? */
394 if (READ_ONCE(m->next_pg)) {
395 spin_lock_irqsave(&m->lock, flags);
396 pg = m->next_pg;
397 if (!pg) {

--- 114 unchanged lines hidden (view full) ---

512
513 if (!pgpath) {
514 if (must_push_back_rq(m))
515 return DM_MAPIO_DELAY_REQUEUE;
516 dm_report_EIO(m); /* Failed */
517 return DM_MAPIO_KILL;
518 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
519 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
519 if (pg_init_all_paths(m))
520 return DM_MAPIO_DELAY_REQUEUE;
521 return DM_MAPIO_REQUEUE;
520 pg_init_all_paths(m);
521 return DM_MAPIO_DELAY_REQUEUE;
522 }
523
522 }
523
524 memset(mpio, 0, sizeof(*mpio));
525 mpio->pgpath = pgpath;
526 mpio->nr_bytes = nr_bytes;
527
528 bdev = pgpath->path.dev->bdev;
529 q = bdev_get_queue(bdev);
530 clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
531 if (IS_ERR(clone)) {
532 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
524 mpio->pgpath = pgpath;
525 mpio->nr_bytes = nr_bytes;
526
527 bdev = pgpath->path.dev->bdev;
528 q = bdev_get_queue(bdev);
529 clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
530 if (IS_ERR(clone)) {
531 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
533 bool queue_dying = blk_queue_dying(q);
534 if (queue_dying) {
532 if (blk_queue_dying(q)) {
535 atomic_inc(&m->pg_init_in_progress);
536 activate_or_offline_path(pgpath);
533 atomic_inc(&m->pg_init_in_progress);
534 activate_or_offline_path(pgpath);
535 return DM_MAPIO_DELAY_REQUEUE;
537 }
536 }
538 return DM_MAPIO_DELAY_REQUEUE;
537
538 /*
539 * blk-mq's SCHED_RESTART can cover this requeue, so we
540 * needn't deal with it by DELAY_REQUEUE. More importantly,
541 * we have to return DM_MAPIO_REQUEUE so that blk-mq can
542 * get the queue busy feedback (via BLK_STS_RESOURCE),
543 * otherwise I/O merging can suffer.
544 */
545 if (q->mq_ops)
546 return DM_MAPIO_REQUEUE;
547 else
548 return DM_MAPIO_DELAY_REQUEUE;
539 }
540 clone->bio = clone->biotail = NULL;
541 clone->rq_disk = bdev->bd_disk;
542 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
543 *__clone = clone;
544
545 if (pgpath->pg->ps.type->start_io)
546 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,

--- 5 unchanged lines hidden (view full) ---

552static void multipath_release_clone(struct request *clone)
553{
554 blk_put_request(clone);
555}
556
557/*
558 * Map cloned bios (bio-based multipath)
559 */
549 }
550 clone->bio = clone->biotail = NULL;
551 clone->rq_disk = bdev->bd_disk;
552 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
553 *__clone = clone;
554
555 if (pgpath->pg->ps.type->start_io)
556 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,

--- 5 unchanged lines hidden (view full) ---

562static void multipath_release_clone(struct request *clone)
563{
564 blk_put_request(clone);
565}
566
567/*
568 * Map cloned bios (bio-based multipath)
569 */
560static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
570
571static struct pgpath *__map_bio(struct multipath *m, struct bio *bio)
561{
572{
562 size_t nr_bytes = bio->bi_iter.bi_size;
563 struct pgpath *pgpath;
564 unsigned long flags;
565 bool queue_io;
566
567 /* Do we need to select a new pgpath? */
568 pgpath = READ_ONCE(m->current_pgpath);
569 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
570 if (!pgpath || !queue_io)
573 struct pgpath *pgpath;
574 unsigned long flags;
575 bool queue_io;
576
577 /* Do we need to select a new pgpath? */
578 pgpath = READ_ONCE(m->current_pgpath);
579 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
580 if (!pgpath || !queue_io)
571 pgpath = choose_pgpath(m, nr_bytes);
581 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
572
573 if ((pgpath && queue_io) ||
574 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
575 /* Queue for the daemon to resubmit */
576 spin_lock_irqsave(&m->lock, flags);
577 bio_list_add(&m->queued_bios, bio);
578 spin_unlock_irqrestore(&m->lock, flags);
582
583 if ((pgpath && queue_io) ||
584 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
585 /* Queue for the daemon to resubmit */
586 spin_lock_irqsave(&m->lock, flags);
587 bio_list_add(&m->queued_bios, bio);
588 spin_unlock_irqrestore(&m->lock, flags);
589
579 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
580 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
581 pg_init_all_paths(m);
582 else if (!queue_io)
583 queue_work(kmultipathd, &m->process_queued_bios);
590 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
591 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
592 pg_init_all_paths(m);
593 else if (!queue_io)
594 queue_work(kmultipathd, &m->process_queued_bios);
584 return DM_MAPIO_SUBMITTED;
595
596 return ERR_PTR(-EAGAIN);
585 }
586
597 }
598
599 return pgpath;
600}
601
602static struct pgpath *__map_bio_nvme(struct multipath *m, struct bio *bio)
603{
604 struct pgpath *pgpath;
605 unsigned long flags;
606
607 /* Do we need to select a new pgpath? */
608 /*
609 * FIXME: currently only switching path if no path (due to failure, etc)
610 * - which negates the point of using a path selector
611 */
612 pgpath = READ_ONCE(m->current_pgpath);
613 if (!pgpath)
614 pgpath = choose_pgpath(m, bio->bi_iter.bi_size);
615
587 if (!pgpath) {
616 if (!pgpath) {
617 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
618 /* Queue for the daemon to resubmit */
619 spin_lock_irqsave(&m->lock, flags);
620 bio_list_add(&m->queued_bios, bio);
621 spin_unlock_irqrestore(&m->lock, flags);
622 queue_work(kmultipathd, &m->process_queued_bios);
623
624 return ERR_PTR(-EAGAIN);
625 }
626 return NULL;
627 }
628
629 return pgpath;
630}
631
632static int __multipath_map_bio(struct multipath *m, struct bio *bio,
633 struct dm_mpath_io *mpio)
634{
635 struct pgpath *pgpath;
636
637 if (m->queue_mode == DM_TYPE_NVME_BIO_BASED)
638 pgpath = __map_bio_nvme(m, bio);
639 else
640 pgpath = __map_bio(m, bio);
641
642 if (IS_ERR(pgpath))
643 return DM_MAPIO_SUBMITTED;
644
645 if (!pgpath) {
588 if (must_push_back_bio(m))
589 return DM_MAPIO_REQUEUE;
590 dm_report_EIO(m);
591 return DM_MAPIO_KILL;
592 }
593
594 mpio->pgpath = pgpath;
646 if (must_push_back_bio(m))
647 return DM_MAPIO_REQUEUE;
648 dm_report_EIO(m);
649 return DM_MAPIO_KILL;
650 }
651
652 mpio->pgpath = pgpath;
595 mpio->nr_bytes = nr_bytes;
596
597 bio->bi_status = 0;
598 bio_set_dev(bio, pgpath->path.dev->bdev);
599 bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
600
601 if (pgpath->pg->ps.type->start_io)
602 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
603 &pgpath->path,
653
654 bio->bi_status = 0;
655 bio_set_dev(bio, pgpath->path.dev->bdev);
656 bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
657
658 if (pgpath->pg->ps.type->start_io)
659 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
660 &pgpath->path,
604 nr_bytes);
661 mpio->nr_bytes);
605 return DM_MAPIO_REMAPPED;
606}
607
608static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
609{
610 struct multipath *m = ti->private;
611 struct dm_mpath_io *mpio = NULL;
612
662 return DM_MAPIO_REMAPPED;
663}
664
665static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
666{
667 struct multipath *m = ti->private;
668 struct dm_mpath_io *mpio = NULL;
669
613 multipath_init_per_bio_data(bio, &mpio, NULL);
614
670 multipath_init_per_bio_data(bio, &mpio);
615 return __multipath_map_bio(m, bio, mpio);
616}
617
618static void process_queued_io_list(struct multipath *m)
619{
620 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
621 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
671 return __multipath_map_bio(m, bio, mpio);
672}
673
674static void process_queued_io_list(struct multipath *m)
675{
676 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
677 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
622 else if (m->queue_mode == DM_TYPE_BIO_BASED)
678 else if (m->queue_mode == DM_TYPE_BIO_BASED ||
679 m->queue_mode == DM_TYPE_NVME_BIO_BASED)
623 queue_work(kmultipathd, &m->process_queued_bios);
624}
625
626static void process_queued_bios(struct work_struct *work)
627{
628 int r;
629 unsigned long flags;
630 struct bio *bio;

--- 13 unchanged lines hidden (view full) ---

644
645 bio_list_merge(&bios, &m->queued_bios);
646 bio_list_init(&m->queued_bios);
647
648 spin_unlock_irqrestore(&m->lock, flags);
649
650 blk_start_plug(&plug);
651 while ((bio = bio_list_pop(&bios))) {
680 queue_work(kmultipathd, &m->process_queued_bios);
681}
682
683static void process_queued_bios(struct work_struct *work)
684{
685 int r;
686 unsigned long flags;
687 struct bio *bio;

--- 13 unchanged lines hidden (view full) ---

701
702 bio_list_merge(&bios, &m->queued_bios);
703 bio_list_init(&m->queued_bios);
704
705 spin_unlock_irqrestore(&m->lock, flags);
706
707 blk_start_plug(&plug);
708 while ((bio = bio_list_pop(&bios))) {
652 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio));
709 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
710 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
711 r = __multipath_map_bio(m, bio, mpio);
653 switch (r) {
654 case DM_MAPIO_KILL:
655 bio->bi_status = BLK_STS_IOERR;
656 bio_endio(bio);
657 break;
658 case DM_MAPIO_REQUEUE:
659 bio->bi_status = BLK_STS_DM_REQUEUE;
660 bio_endio(bio);

--- 86 unchanged lines hidden (view full) ---

747 }
748
749 pg->ps.type = pst;
750 dm_consume_args(as, ps_argc);
751
752 return 0;
753}
754
712 switch (r) {
713 case DM_MAPIO_KILL:
714 bio->bi_status = BLK_STS_IOERR;
715 bio_endio(bio);
716 break;
717 case DM_MAPIO_REQUEUE:
718 bio->bi_status = BLK_STS_DM_REQUEUE;
719 bio_endio(bio);

--- 86 unchanged lines hidden (view full) ---

806 }
807
808 pg->ps.type = pst;
809 dm_consume_args(as, ps_argc);
810
811 return 0;
812}
813
755static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
756 struct dm_target *ti)
814static int setup_scsi_dh(struct block_device *bdev, struct multipath *m, char **error)
757{
815{
758 int r;
759 struct pgpath *p;
760 struct multipath *m = ti->private;
761 struct request_queue *q = NULL;
816 struct request_queue *q = bdev_get_queue(bdev);
762 const char *attached_handler_name;
817 const char *attached_handler_name;
818 int r;
763
819
764 /* we need at least a path arg */
765 if (as->argc < 1) {
766 ti->error = "no device given";
767 return ERR_PTR(-EINVAL);
768 }
769
770 p = alloc_pgpath();
771 if (!p)
772 return ERR_PTR(-ENOMEM);
773
774 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
775 &p->path.dev);
776 if (r) {
777 ti->error = "error getting device";
778 goto bad;
779 }
780
781 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name)
782 q = bdev_get_queue(p->path.dev->bdev);
783
784 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
785retain:
786 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
787 if (attached_handler_name) {
788 /*
789 * Clear any hw_handler_params associated with a
790 * handler that isn't already attached.
791 */

--- 14 unchanged lines hidden (view full) ---

806 }
807
808 if (m->hw_handler_name) {
809 r = scsi_dh_attach(q, m->hw_handler_name);
810 if (r == -EBUSY) {
811 char b[BDEVNAME_SIZE];
812
813 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
820 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
821retain:
822 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
823 if (attached_handler_name) {
824 /*
825 * Clear any hw_handler_params associated with a
826 * handler that isn't already attached.
827 */

--- 14 unchanged lines hidden (view full) ---

842 }
843
844 if (m->hw_handler_name) {
845 r = scsi_dh_attach(q, m->hw_handler_name);
846 if (r == -EBUSY) {
847 char b[BDEVNAME_SIZE];
848
849 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
814 bdevname(p->path.dev->bdev, b));
850 bdevname(bdev, b));
815 goto retain;
816 }
817 if (r < 0) {
851 goto retain;
852 }
853 if (r < 0) {
818 ti->error = "error attaching hardware handler";
819 dm_put_device(ti, p->path.dev);
820 goto bad;
854 *error = "error attaching hardware handler";
855 return r;
821 }
822
823 if (m->hw_handler_params) {
824 r = scsi_dh_set_params(q, m->hw_handler_params);
825 if (r < 0) {
856 }
857
858 if (m->hw_handler_params) {
859 r = scsi_dh_set_params(q, m->hw_handler_params);
860 if (r < 0) {
826 ti->error = "unable to set hardware "
827 "handler parameters";
828 dm_put_device(ti, p->path.dev);
829 goto bad;
861 *error = "unable to set hardware handler parameters";
862 return r;
830 }
831 }
832 }
833
863 }
864 }
865 }
866
867 return 0;
868}
869
870static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
871 struct dm_target *ti)
872{
873 int r;
874 struct pgpath *p;
875 struct multipath *m = ti->private;
876
877 /* we need at least a path arg */
878 if (as->argc < 1) {
879 ti->error = "no device given";
880 return ERR_PTR(-EINVAL);
881 }
882
883 p = alloc_pgpath();
884 if (!p)
885 return ERR_PTR(-ENOMEM);
886
887 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
888 &p->path.dev);
889 if (r) {
890 ti->error = "error getting device";
891 goto bad;
892 }
893
894 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
895 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
896 r = setup_scsi_dh(p->path.dev->bdev, m, &ti->error);
897 if (r) {
898 dm_put_device(ti, p->path.dev);
899 goto bad;
900 }
901 }
902
834 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
835 if (r) {
836 dm_put_device(ti, p->path.dev);
837 goto bad;
838 }
839
840 return p;
903 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
904 if (r) {
905 dm_put_device(ti, p->path.dev);
906 goto bad;
907 }
908
909 return p;
841
842 bad:
843 free_pgpath(p);
844 return ERR_PTR(r);
845}
846
847static struct priority_group *parse_priority_group(struct dm_arg_set *as,
848 struct multipath *m)
849{

--- 78 unchanged lines hidden (view full) ---

928 };
929
930 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
931 return -EINVAL;
932
933 if (!hw_argc)
934 return 0;
935
910 bad:
911 free_pgpath(p);
912 return ERR_PTR(r);
913}
914
915static struct priority_group *parse_priority_group(struct dm_arg_set *as,
916 struct multipath *m)
917{

--- 78 unchanged lines hidden (view full) ---

996 };
997
998 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
999 return -EINVAL;
1000
1001 if (!hw_argc)
1002 return 0;
1003
936 if (m->queue_mode == DM_TYPE_BIO_BASED) {
1004 if (m->queue_mode == DM_TYPE_BIO_BASED ||
1005 m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
937 dm_consume_args(as, hw_argc);
938 DMERR("bio-based multipath doesn't allow hardware handler args");
939 return 0;
940 }
941
942 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
943 if (!m->hw_handler_name)
944 return -EINVAL;

--- 72 unchanged lines hidden (view full) ---

1017 }
1018
1019 if (!strcasecmp(arg_name, "queue_mode") &&
1020 (argc >= 1)) {
1021 const char *queue_mode_name = dm_shift_arg(as);
1022
1023 if (!strcasecmp(queue_mode_name, "bio"))
1024 m->queue_mode = DM_TYPE_BIO_BASED;
1006 dm_consume_args(as, hw_argc);
1007 DMERR("bio-based multipath doesn't allow hardware handler args");
1008 return 0;
1009 }
1010
1011 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
1012 if (!m->hw_handler_name)
1013 return -EINVAL;

--- 72 unchanged lines hidden (view full) ---

1086 }
1087
1088 if (!strcasecmp(arg_name, "queue_mode") &&
1089 (argc >= 1)) {
1090 const char *queue_mode_name = dm_shift_arg(as);
1091
1092 if (!strcasecmp(queue_mode_name, "bio"))
1093 m->queue_mode = DM_TYPE_BIO_BASED;
1094 else if (!strcasecmp(queue_mode_name, "nvme"))
1095 m->queue_mode = DM_TYPE_NVME_BIO_BASED;
1025 else if (!strcasecmp(queue_mode_name, "rq"))
1026 m->queue_mode = DM_TYPE_REQUEST_BASED;
1027 else if (!strcasecmp(queue_mode_name, "mq"))
1028 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1029 else {
1030 ti->error = "Unknown 'queue_mode' requested";
1031 r = -EINVAL;
1032 }

--- 84 unchanged lines hidden (view full) ---

1117 r = -EINVAL;
1118 goto bad;
1119 }
1120
1121 ti->num_flush_bios = 1;
1122 ti->num_discard_bios = 1;
1123 ti->num_write_same_bios = 1;
1124 ti->num_write_zeroes_bios = 1;
1096 else if (!strcasecmp(queue_mode_name, "rq"))
1097 m->queue_mode = DM_TYPE_REQUEST_BASED;
1098 else if (!strcasecmp(queue_mode_name, "mq"))
1099 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1100 else {
1101 ti->error = "Unknown 'queue_mode' requested";
1102 r = -EINVAL;
1103 }

--- 84 unchanged lines hidden (view full) ---

1188 r = -EINVAL;
1189 goto bad;
1190 }
1191
1192 ti->num_flush_bios = 1;
1193 ti->num_discard_bios = 1;
1194 ti->num_write_same_bios = 1;
1195 ti->num_write_zeroes_bios = 1;
1125 if (m->queue_mode == DM_TYPE_BIO_BASED)
1196 if (m->queue_mode == DM_TYPE_BIO_BASED || m->queue_mode == DM_TYPE_NVME_BIO_BASED)
1126 ti->per_io_data_size = multipath_per_bio_data_size();
1127 else
1128 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1129
1130 return 0;
1131
1132 bad:
1133 free_multipath(m);

--- 12 unchanged lines hidden (view full) ---

1146
1147 io_schedule();
1148 }
1149 finish_wait(&m->pg_init_wait, &wait);
1150}
1151
1152static void flush_multipath_work(struct multipath *m)
1153{
1197 ti->per_io_data_size = multipath_per_bio_data_size();
1198 else
1199 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1200
1201 return 0;
1202
1203 bad:
1204 free_multipath(m);

--- 12 unchanged lines hidden (view full) ---

1217
1218 io_schedule();
1219 }
1220 finish_wait(&m->pg_init_wait, &wait);
1221}
1222
1223static void flush_multipath_work(struct multipath *m)
1224{
1154 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1155 smp_mb__after_atomic();
1225 if (m->hw_handler_name) {
1226 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1227 smp_mb__after_atomic();
1156
1228
1157 flush_workqueue(kmpath_handlerd);
1158 multipath_wait_for_pg_init_completion(m);
1229 flush_workqueue(kmpath_handlerd);
1230 multipath_wait_for_pg_init_completion(m);
1231
1232 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1233 smp_mb__after_atomic();
1234 }
1235
1159 flush_workqueue(kmultipathd);
1160 flush_work(&m->trigger_event);
1236 flush_workqueue(kmultipathd);
1237 flush_work(&m->trigger_event);
1161
1162 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1163 smp_mb__after_atomic();
1164}
1165
1166static void multipath_dtr(struct dm_target *ti)
1167{
1168 struct multipath *m = ti->private;
1169
1170 flush_multipath_work(m);
1171 free_multipath(m);

--- 334 unchanged lines hidden (view full) ---

1506 * don't have bio clones.)
1507 * Instead of queueing the clone request here, we queue the original
1508 * request into dm core, which will remake a clone request and
1509 * clone bios for it and resubmit it later.
1510 */
1511 if (error && !noretry_error(error)) {
1512 struct multipath *m = ti->private;
1513
1238}
1239
1240static void multipath_dtr(struct dm_target *ti)
1241{
1242 struct multipath *m = ti->private;
1243
1244 flush_multipath_work(m);
1245 free_multipath(m);

--- 334 unchanged lines hidden (view full) ---

1580 * don't have bio clones.)
1581 * Instead of queueing the clone request here, we queue the original
1582 * request into dm core, which will remake a clone request and
1583 * clone bios for it and resubmit it later.
1584 */
1585 if (error && !noretry_error(error)) {
1586 struct multipath *m = ti->private;
1587
1514 r = DM_ENDIO_REQUEUE;
1588 if (error == BLK_STS_RESOURCE)
1589 r = DM_ENDIO_DELAY_REQUEUE;
1590 else
1591 r = DM_ENDIO_REQUEUE;
1515
1516 if (pgpath)
1517 fail_path(pgpath);
1518
1519 if (atomic_read(&m->nr_valid_paths) == 0 &&
1520 !must_push_back_rq(m)) {
1521 if (error == BLK_STS_IOERR)
1522 dm_report_EIO(m);

--- 8 unchanged lines hidden (view full) ---

1531 if (ps->type->end_io)
1532 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1533 }
1534
1535 return r;
1536}
1537
1538static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1592
1593 if (pgpath)
1594 fail_path(pgpath);
1595
1596 if (atomic_read(&m->nr_valid_paths) == 0 &&
1597 !must_push_back_rq(m)) {
1598 if (error == BLK_STS_IOERR)
1599 dm_report_EIO(m);

--- 8 unchanged lines hidden (view full) ---

1608 if (ps->type->end_io)
1609 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1610 }
1611
1612 return r;
1613}
1614
1615static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1539 blk_status_t *error)
1616 blk_status_t *error)
1540{
1541 struct multipath *m = ti->private;
1542 struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1543 struct pgpath *pgpath = mpio->pgpath;
1544 unsigned long flags;
1545 int r = DM_ENDIO_DONE;
1546
1547 if (!*error || noretry_error(*error))

--- 8 unchanged lines hidden (view full) ---

1556 r = DM_ENDIO_REQUEUE;
1557 } else {
1558 dm_report_EIO(m);
1559 *error = BLK_STS_IOERR;
1560 }
1561 goto done;
1562 }
1563
1617{
1618 struct multipath *m = ti->private;
1619 struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1620 struct pgpath *pgpath = mpio->pgpath;
1621 unsigned long flags;
1622 int r = DM_ENDIO_DONE;
1623
1624 if (!*error || noretry_error(*error))

--- 8 unchanged lines hidden (view full) ---

1633 r = DM_ENDIO_REQUEUE;
1634 } else {
1635 dm_report_EIO(m);
1636 *error = BLK_STS_IOERR;
1637 }
1638 goto done;
1639 }
1640
1564 /* Queue for the daemon to resubmit */
1565 dm_bio_restore(get_bio_details_from_bio(clone), clone);
1566
1567 spin_lock_irqsave(&m->lock, flags);
1568 bio_list_add(&m->queued_bios, clone);
1569 spin_unlock_irqrestore(&m->lock, flags);
1570 if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1571 queue_work(kmultipathd, &m->process_queued_bios);
1572
1573 r = DM_ENDIO_INCOMPLETE;
1574done:

--- 91 unchanged lines hidden (view full) ---

1666 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1667 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1668 DMEMIT("retain_attached_hw_handler ");
1669 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1670 switch(m->queue_mode) {
1671 case DM_TYPE_BIO_BASED:
1672 DMEMIT("queue_mode bio ");
1673 break;
1641 spin_lock_irqsave(&m->lock, flags);
1642 bio_list_add(&m->queued_bios, clone);
1643 spin_unlock_irqrestore(&m->lock, flags);
1644 if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1645 queue_work(kmultipathd, &m->process_queued_bios);
1646
1647 r = DM_ENDIO_INCOMPLETE;
1648done:

--- 91 unchanged lines hidden (view full) ---

1740 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1741 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1742 DMEMIT("retain_attached_hw_handler ");
1743 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1744 switch(m->queue_mode) {
1745 case DM_TYPE_BIO_BASED:
1746 DMEMIT("queue_mode bio ");
1747 break;
1748 case DM_TYPE_NVME_BIO_BASED:
1749 DMEMIT("queue_mode nvme ");
1750 break;
1674 case DM_TYPE_MQ_REQUEST_BASED:
1675 DMEMIT("queue_mode mq ");
1676 break;
1677 default:
1678 WARN_ON_ONCE(true);
1679 break;
1680 }
1681 }

--- 365 unchanged lines hidden --- 		1751 case DM_TYPE_MQ_REQUEST_BASED:
1752 DMEMIT("queue_mode mq ");
1753 break;
1754 default:
1755 WARN_ON_ONCE(true);
1756 break;
1757 }
1758 }

--- 365 unchanged lines hidden ---