1 /* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 10 #include "dm-rq.h" 11 #include "dm-bio-record.h" 12 #include "dm-path-selector.h" 13 #include "dm-uevent.h" 14 15 #include <linux/blkdev.h> 16 #include <linux/ctype.h> 17 #include <linux/init.h> 18 #include <linux/mempool.h> 19 #include <linux/module.h> 20 #include <linux/pagemap.h> 21 #include <linux/slab.h> 22 #include <linux/time.h> 23 #include <linux/workqueue.h> 24 #include <linux/delay.h> 25 #include <scsi/scsi_dh.h> 26 #include <linux/atomic.h> 27 #include <linux/blk-mq.h> 28 29 #define DM_MSG_PREFIX "multipath" 30 #define DM_PG_INIT_DELAY_MSECS 2000 31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1) 32 33 /* Path properties */ 34 struct pgpath { 35 struct list_head list; 36 37 struct priority_group *pg; /* Owning PG */ 38 unsigned fail_count; /* Cumulative failure count */ 39 40 struct dm_path path; 41 struct delayed_work activate_path; 42 43 bool is_active:1; /* Path status */ 44 }; 45 46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 47 48 /* 49 * Paths are grouped into Priority Groups and numbered from 1 upwards. 50 * Each has a path selector which controls which path gets used. 51 */ 52 struct priority_group { 53 struct list_head list; 54 55 struct multipath *m; /* Owning multipath instance */ 56 struct path_selector ps; 57 58 unsigned pg_num; /* Reference number */ 59 unsigned nr_pgpaths; /* Number of paths in PG */ 60 struct list_head pgpaths; 61 62 bool bypassed:1; /* Temporarily bypass this PG? */ 63 }; 64 65 /* Multipath context */ 66 struct multipath { 67 struct list_head list; 68 struct dm_target *ti; 69 70 const char *hw_handler_name; 71 char *hw_handler_params; 72 73 spinlock_t lock; 74 75 unsigned nr_priority_groups; 76 struct list_head priority_groups; 77 78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */ 79 80 struct pgpath *current_pgpath; 81 struct priority_group *current_pg; 82 struct priority_group *next_pg; /* Switch to this PG if set */ 83 84 unsigned long flags; /* Multipath state flags */ 85 86 unsigned pg_init_retries; /* Number of times to retry pg_init */ 87 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */ 88 89 atomic_t nr_valid_paths; /* Total number of usable paths */ 90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */ 91 atomic_t pg_init_count; /* Number of times pg_init called */ 92 93 enum dm_queue_mode queue_mode; 94 95 struct mutex work_mutex; 96 struct work_struct trigger_event; 97 98 struct work_struct process_queued_bios; 99 struct bio_list queued_bios; 100 }; 101 102 /* 103 * Context information attached to each io we process. 104 */ 105 struct dm_mpath_io { 106 struct pgpath *pgpath; 107 size_t nr_bytes; 108 }; 109 110 typedef int (*action_fn) (struct pgpath *pgpath); 111 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 113 static void trigger_event(struct work_struct *work); 114 static void activate_or_offline_path(struct pgpath *pgpath); 115 static void activate_path_work(struct work_struct *work); 116 static void process_queued_bios(struct work_struct *work); 117 118 /*----------------------------------------------- 119 * Multipath state flags. 120 *-----------------------------------------------*/ 121 122 #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */ 123 #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */ 124 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */ 125 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */ 126 #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */ 127 #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */ 128 #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */ 129 130 /*----------------------------------------------- 131 * Allocation routines 132 *-----------------------------------------------*/ 133 134 static struct pgpath *alloc_pgpath(void) 135 { 136 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 137 138 if (pgpath) { 139 pgpath->is_active = true; 140 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path_work); 141 } 142 143 return pgpath; 144 } 145 146 static void free_pgpath(struct pgpath *pgpath) 147 { 148 kfree(pgpath); 149 } 150 151 static struct priority_group *alloc_priority_group(void) 152 { 153 struct priority_group *pg; 154 155 pg = kzalloc(sizeof(*pg), GFP_KERNEL); 156 157 if (pg) 158 INIT_LIST_HEAD(&pg->pgpaths); 159 160 return pg; 161 } 162 163 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 164 { 165 struct pgpath *pgpath, *tmp; 166 167 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 168 list_del(&pgpath->list); 169 dm_put_device(ti, pgpath->path.dev); 170 free_pgpath(pgpath); 171 } 172 } 173 174 static void free_priority_group(struct priority_group *pg, 175 struct dm_target *ti) 176 { 177 struct path_selector *ps = &pg->ps; 178 179 if (ps->type) { 180 ps->type->destroy(ps); 181 dm_put_path_selector(ps->type); 182 } 183 184 free_pgpaths(&pg->pgpaths, ti); 185 kfree(pg); 186 } 187 188 static struct multipath *alloc_multipath(struct dm_target *ti) 189 { 190 struct multipath *m; 191 192 m = kzalloc(sizeof(*m), GFP_KERNEL); 193 if (m) { 194 INIT_LIST_HEAD(&m->priority_groups); 195 spin_lock_init(&m->lock); 196 set_bit(MPATHF_QUEUE_IO, &m->flags); 197 atomic_set(&m->nr_valid_paths, 0); 198 atomic_set(&m->pg_init_in_progress, 0); 199 atomic_set(&m->pg_init_count, 0); 200 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 201 INIT_WORK(&m->trigger_event, trigger_event); 202 init_waitqueue_head(&m->pg_init_wait); 203 mutex_init(&m->work_mutex); 204 205 m->queue_mode = DM_TYPE_NONE; 206 207 m->ti = ti; 208 ti->private = m; 209 } 210 211 return m; 212 } 213 214 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) 215 { 216 if (m->queue_mode == DM_TYPE_NONE) { 217 /* 218 * Default to request-based. 219 */ 220 if (dm_use_blk_mq(dm_table_get_md(ti->table))) 221 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 222 else 223 m->queue_mode = DM_TYPE_REQUEST_BASED; 224 } else if (m->queue_mode == DM_TYPE_BIO_BASED) { 225 INIT_WORK(&m->process_queued_bios, process_queued_bios); 226 /* 227 * bio-based doesn't support any direct scsi_dh management; 228 * it just discovers if a scsi_dh is attached. 229 */ 230 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 231 } 232 233 dm_table_set_type(ti->table, m->queue_mode); 234 235 return 0; 236 } 237 238 static void free_multipath(struct multipath *m) 239 { 240 struct priority_group *pg, *tmp; 241 242 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 243 list_del(&pg->list); 244 free_priority_group(pg, m->ti); 245 } 246 247 kfree(m->hw_handler_name); 248 kfree(m->hw_handler_params); 249 kfree(m); 250 } 251 252 static struct dm_mpath_io *get_mpio(union map_info *info) 253 { 254 return info->ptr; 255 } 256 257 static size_t multipath_per_bio_data_size(void) 258 { 259 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); 260 } 261 262 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio) 263 { 264 return dm_per_bio_data(bio, multipath_per_bio_data_size()); 265 } 266 267 static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio) 268 { 269 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */ 270 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 271 void *bio_details = mpio + 1; 272 273 return bio_details; 274 } 275 276 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p) 277 { 278 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 279 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio); 280 281 mpio->nr_bytes = bio->bi_iter.bi_size; 282 mpio->pgpath = NULL; 283 *mpio_p = mpio; 284 285 dm_bio_record(bio_details, bio); 286 } 287 288 /*----------------------------------------------- 289 * Path selection 290 *-----------------------------------------------*/ 291 292 static int __pg_init_all_paths(struct multipath *m) 293 { 294 struct pgpath *pgpath; 295 unsigned long pg_init_delay = 0; 296 297 lockdep_assert_held(&m->lock); 298 299 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 300 return 0; 301 302 atomic_inc(&m->pg_init_count); 303 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 304 305 /* Check here to reset pg_init_required */ 306 if (!m->current_pg) 307 return 0; 308 309 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags)) 310 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ? 311 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS); 312 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) { 313 /* Skip failed paths */ 314 if (!pgpath->is_active) 315 continue; 316 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path, 317 pg_init_delay)) 318 atomic_inc(&m->pg_init_in_progress); 319 } 320 return atomic_read(&m->pg_init_in_progress); 321 } 322 323 static int pg_init_all_paths(struct multipath *m) 324 { 325 int ret; 326 unsigned long flags; 327 328 spin_lock_irqsave(&m->lock, flags); 329 ret = __pg_init_all_paths(m); 330 spin_unlock_irqrestore(&m->lock, flags); 331 332 return ret; 333 } 334 335 static void __switch_pg(struct multipath *m, struct priority_group *pg) 336 { 337 m->current_pg = pg; 338 339 /* Must we initialise the PG first, and queue I/O till it's ready? */ 340 if (m->hw_handler_name) { 341 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 342 set_bit(MPATHF_QUEUE_IO, &m->flags); 343 } else { 344 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 345 clear_bit(MPATHF_QUEUE_IO, &m->flags); 346 } 347 348 atomic_set(&m->pg_init_count, 0); 349 } 350 351 static struct pgpath *choose_path_in_pg(struct multipath *m, 352 struct priority_group *pg, 353 size_t nr_bytes) 354 { 355 unsigned long flags; 356 struct dm_path *path; 357 struct pgpath *pgpath; 358 359 path = pg->ps.type->select_path(&pg->ps, nr_bytes); 360 if (!path) 361 return ERR_PTR(-ENXIO); 362 363 pgpath = path_to_pgpath(path); 364 365 if (unlikely(READ_ONCE(m->current_pg) != pg)) { 366 /* Only update current_pgpath if pg changed */ 367 spin_lock_irqsave(&m->lock, flags); 368 m->current_pgpath = pgpath; 369 __switch_pg(m, pg); 370 spin_unlock_irqrestore(&m->lock, flags); 371 } 372 373 return pgpath; 374 } 375 376 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes) 377 { 378 unsigned long flags; 379 struct priority_group *pg; 380 struct pgpath *pgpath; 381 unsigned bypassed = 1; 382 383 if (!atomic_read(&m->nr_valid_paths)) { 384 clear_bit(MPATHF_QUEUE_IO, &m->flags); 385 goto failed; 386 } 387 388 /* Were we instructed to switch PG? */ 389 if (READ_ONCE(m->next_pg)) { 390 spin_lock_irqsave(&m->lock, flags); 391 pg = m->next_pg; 392 if (!pg) { 393 spin_unlock_irqrestore(&m->lock, flags); 394 goto check_current_pg; 395 } 396 m->next_pg = NULL; 397 spin_unlock_irqrestore(&m->lock, flags); 398 pgpath = choose_path_in_pg(m, pg, nr_bytes); 399 if (!IS_ERR_OR_NULL(pgpath)) 400 return pgpath; 401 } 402 403 /* Don't change PG until it has no remaining paths */ 404 check_current_pg: 405 pg = READ_ONCE(m->current_pg); 406 if (pg) { 407 pgpath = choose_path_in_pg(m, pg, nr_bytes); 408 if (!IS_ERR_OR_NULL(pgpath)) 409 return pgpath; 410 } 411 412 /* 413 * Loop through priority groups until we find a valid path. 414 * First time we skip PGs marked 'bypassed'. 415 * Second time we only try the ones we skipped, but set 416 * pg_init_delay_retry so we do not hammer controllers. 417 */ 418 do { 419 list_for_each_entry(pg, &m->priority_groups, list) { 420 if (pg->bypassed == !!bypassed) 421 continue; 422 pgpath = choose_path_in_pg(m, pg, nr_bytes); 423 if (!IS_ERR_OR_NULL(pgpath)) { 424 if (!bypassed) 425 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 426 return pgpath; 427 } 428 } 429 } while (bypassed--); 430 431 failed: 432 spin_lock_irqsave(&m->lock, flags); 433 m->current_pgpath = NULL; 434 m->current_pg = NULL; 435 spin_unlock_irqrestore(&m->lock, flags); 436 437 return NULL; 438 } 439 440 /* 441 * dm_report_EIO() is a macro instead of a function to make pr_debug() 442 * report the function name and line number of the function from which 443 * it has been invoked. 444 */ 445 #define dm_report_EIO(m) \ 446 do { \ 447 struct mapped_device *md = dm_table_get_md((m)->ti->table); \ 448 \ 449 pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \ 450 dm_device_name(md), \ 451 test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags), \ 452 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \ 453 dm_noflush_suspending((m)->ti)); \ 454 } while (0) 455 456 /* 457 * Check whether bios must be queued in the device-mapper core rather 458 * than here in the target. 459 * 460 * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold 461 * the same value then we are not between multipath_presuspend() 462 * and multipath_resume() calls and we have no need to check 463 * for the DMF_NOFLUSH_SUSPENDING flag. 464 */ 465 static bool __must_push_back(struct multipath *m, unsigned long flags) 466 { 467 return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) != 468 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) && 469 dm_noflush_suspending(m->ti)); 470 } 471 472 /* 473 * Following functions use READ_ONCE to get atomic access to 474 * all m->flags to avoid taking spinlock 475 */ 476 static bool must_push_back_rq(struct multipath *m) 477 { 478 unsigned long flags = READ_ONCE(m->flags); 479 return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags); 480 } 481 482 static bool must_push_back_bio(struct multipath *m) 483 { 484 unsigned long flags = READ_ONCE(m->flags); 485 return __must_push_back(m, flags); 486 } 487 488 /* 489 * Map cloned requests (request-based multipath) 490 */ 491 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq, 492 union map_info *map_context, 493 struct request **__clone) 494 { 495 struct multipath *m = ti->private; 496 size_t nr_bytes = blk_rq_bytes(rq); 497 struct pgpath *pgpath; 498 struct block_device *bdev; 499 struct dm_mpath_io *mpio = get_mpio(map_context); 500 struct request_queue *q; 501 struct request *clone; 502 503 /* Do we need to select a new pgpath? */ 504 pgpath = READ_ONCE(m->current_pgpath); 505 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags)) 506 pgpath = choose_pgpath(m, nr_bytes); 507 508 if (!pgpath) { 509 if (must_push_back_rq(m)) 510 return DM_MAPIO_DELAY_REQUEUE; 511 dm_report_EIO(m); /* Failed */ 512 return DM_MAPIO_KILL; 513 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) || 514 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 515 if (pg_init_all_paths(m)) 516 return DM_MAPIO_DELAY_REQUEUE; 517 return DM_MAPIO_REQUEUE; 518 } 519 520 mpio->pgpath = pgpath; 521 mpio->nr_bytes = nr_bytes; 522 523 bdev = pgpath->path.dev->bdev; 524 q = bdev_get_queue(bdev); 525 clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC); 526 if (IS_ERR(clone)) { 527 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */ 528 bool queue_dying = blk_queue_dying(q); 529 if (queue_dying) { 530 atomic_inc(&m->pg_init_in_progress); 531 activate_or_offline_path(pgpath); 532 } 533 return DM_MAPIO_DELAY_REQUEUE; 534 } 535 clone->bio = clone->biotail = NULL; 536 clone->rq_disk = bdev->bd_disk; 537 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; 538 *__clone = clone; 539 540 if (pgpath->pg->ps.type->start_io) 541 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 542 &pgpath->path, 543 nr_bytes); 544 return DM_MAPIO_REMAPPED; 545 } 546 547 static void multipath_release_clone(struct request *clone) 548 { 549 blk_put_request(clone); 550 } 551 552 /* 553 * Map cloned bios (bio-based multipath) 554 */ 555 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio) 556 { 557 struct pgpath *pgpath; 558 unsigned long flags; 559 bool queue_io; 560 561 /* Do we need to select a new pgpath? */ 562 pgpath = READ_ONCE(m->current_pgpath); 563 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags); 564 if (!pgpath || !queue_io) 565 pgpath = choose_pgpath(m, mpio->nr_bytes); 566 567 if ((pgpath && queue_io) || 568 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) { 569 /* Queue for the daemon to resubmit */ 570 spin_lock_irqsave(&m->lock, flags); 571 bio_list_add(&m->queued_bios, bio); 572 spin_unlock_irqrestore(&m->lock, flags); 573 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */ 574 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 575 pg_init_all_paths(m); 576 else if (!queue_io) 577 queue_work(kmultipathd, &m->process_queued_bios); 578 return DM_MAPIO_SUBMITTED; 579 } 580 581 if (!pgpath) { 582 if (must_push_back_bio(m)) 583 return DM_MAPIO_REQUEUE; 584 dm_report_EIO(m); 585 return DM_MAPIO_KILL; 586 } 587 588 mpio->pgpath = pgpath; 589 590 bio->bi_status = 0; 591 bio_set_dev(bio, pgpath->path.dev->bdev); 592 bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 593 594 if (pgpath->pg->ps.type->start_io) 595 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 596 &pgpath->path, 597 mpio->nr_bytes); 598 return DM_MAPIO_REMAPPED; 599 } 600 601 static int multipath_map_bio(struct dm_target *ti, struct bio *bio) 602 { 603 struct multipath *m = ti->private; 604 struct dm_mpath_io *mpio = NULL; 605 606 multipath_init_per_bio_data(bio, &mpio); 607 return __multipath_map_bio(m, bio, mpio); 608 } 609 610 static void process_queued_io_list(struct multipath *m) 611 { 612 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED) 613 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table)); 614 else if (m->queue_mode == DM_TYPE_BIO_BASED) 615 queue_work(kmultipathd, &m->process_queued_bios); 616 } 617 618 static void process_queued_bios(struct work_struct *work) 619 { 620 int r; 621 unsigned long flags; 622 struct bio *bio; 623 struct bio_list bios; 624 struct blk_plug plug; 625 struct multipath *m = 626 container_of(work, struct multipath, process_queued_bios); 627 628 bio_list_init(&bios); 629 630 spin_lock_irqsave(&m->lock, flags); 631 632 if (bio_list_empty(&m->queued_bios)) { 633 spin_unlock_irqrestore(&m->lock, flags); 634 return; 635 } 636 637 bio_list_merge(&bios, &m->queued_bios); 638 bio_list_init(&m->queued_bios); 639 640 spin_unlock_irqrestore(&m->lock, flags); 641 642 blk_start_plug(&plug); 643 while ((bio = bio_list_pop(&bios))) { 644 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio)); 645 switch (r) { 646 case DM_MAPIO_KILL: 647 bio->bi_status = BLK_STS_IOERR; 648 bio_endio(bio); 649 break; 650 case DM_MAPIO_REQUEUE: 651 bio->bi_status = BLK_STS_DM_REQUEUE; 652 bio_endio(bio); 653 break; 654 case DM_MAPIO_REMAPPED: 655 generic_make_request(bio); 656 break; 657 case 0: 658 break; 659 default: 660 WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r); 661 } 662 } 663 blk_finish_plug(&plug); 664 } 665 666 /* 667 * If we run out of usable paths, should we queue I/O or error it? 668 */ 669 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path, 670 bool save_old_value) 671 { 672 unsigned long flags; 673 674 spin_lock_irqsave(&m->lock, flags); 675 assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags, 676 (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) || 677 (!save_old_value && queue_if_no_path)); 678 assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path); 679 spin_unlock_irqrestore(&m->lock, flags); 680 681 if (!queue_if_no_path) { 682 dm_table_run_md_queue_async(m->ti->table); 683 process_queued_io_list(m); 684 } 685 686 return 0; 687 } 688 689 /* 690 * An event is triggered whenever a path is taken out of use. 691 * Includes path failure and PG bypass. 692 */ 693 static void trigger_event(struct work_struct *work) 694 { 695 struct multipath *m = 696 container_of(work, struct multipath, trigger_event); 697 698 dm_table_event(m->ti->table); 699 } 700 701 /*----------------------------------------------------------------- 702 * Constructor/argument parsing: 703 * <#multipath feature args> [<arg>]* 704 * <#hw_handler args> [hw_handler [<arg>]*] 705 * <#priority groups> 706 * <initial priority group> 707 * [<selector> <#selector args> [<arg>]* 708 * <#paths> <#per-path selector args> 709 * [<path> [<arg>]* ]+ ]+ 710 *---------------------------------------------------------------*/ 711 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, 712 struct dm_target *ti) 713 { 714 int r; 715 struct path_selector_type *pst; 716 unsigned ps_argc; 717 718 static const struct dm_arg _args[] = { 719 {0, 1024, "invalid number of path selector args"}, 720 }; 721 722 pst = dm_get_path_selector(dm_shift_arg(as)); 723 if (!pst) { 724 ti->error = "unknown path selector type"; 725 return -EINVAL; 726 } 727 728 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error); 729 if (r) { 730 dm_put_path_selector(pst); 731 return -EINVAL; 732 } 733 734 r = pst->create(&pg->ps, ps_argc, as->argv); 735 if (r) { 736 dm_put_path_selector(pst); 737 ti->error = "path selector constructor failed"; 738 return r; 739 } 740 741 pg->ps.type = pst; 742 dm_consume_args(as, ps_argc); 743 744 return 0; 745 } 746 747 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps, 748 struct dm_target *ti) 749 { 750 int r; 751 struct pgpath *p; 752 struct multipath *m = ti->private; 753 struct request_queue *q = NULL; 754 const char *attached_handler_name; 755 756 /* we need at least a path arg */ 757 if (as->argc < 1) { 758 ti->error = "no device given"; 759 return ERR_PTR(-EINVAL); 760 } 761 762 p = alloc_pgpath(); 763 if (!p) 764 return ERR_PTR(-ENOMEM); 765 766 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), 767 &p->path.dev); 768 if (r) { 769 ti->error = "error getting device"; 770 goto bad; 771 } 772 773 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name) 774 q = bdev_get_queue(p->path.dev->bdev); 775 776 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) { 777 retain: 778 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL); 779 if (attached_handler_name) { 780 /* 781 * Clear any hw_handler_params associated with a 782 * handler that isn't already attached. 783 */ 784 if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) { 785 kfree(m->hw_handler_params); 786 m->hw_handler_params = NULL; 787 } 788 789 /* 790 * Reset hw_handler_name to match the attached handler 791 * 792 * NB. This modifies the table line to show the actual 793 * handler instead of the original table passed in. 794 */ 795 kfree(m->hw_handler_name); 796 m->hw_handler_name = attached_handler_name; 797 } 798 } 799 800 if (m->hw_handler_name) { 801 r = scsi_dh_attach(q, m->hw_handler_name); 802 if (r == -EBUSY) { 803 char b[BDEVNAME_SIZE]; 804 805 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n", 806 bdevname(p->path.dev->bdev, b)); 807 goto retain; 808 } 809 if (r < 0) { 810 ti->error = "error attaching hardware handler"; 811 dm_put_device(ti, p->path.dev); 812 goto bad; 813 } 814 815 if (m->hw_handler_params) { 816 r = scsi_dh_set_params(q, m->hw_handler_params); 817 if (r < 0) { 818 ti->error = "unable to set hardware " 819 "handler parameters"; 820 dm_put_device(ti, p->path.dev); 821 goto bad; 822 } 823 } 824 } 825 826 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 827 if (r) { 828 dm_put_device(ti, p->path.dev); 829 goto bad; 830 } 831 832 return p; 833 834 bad: 835 free_pgpath(p); 836 return ERR_PTR(r); 837 } 838 839 static struct priority_group *parse_priority_group(struct dm_arg_set *as, 840 struct multipath *m) 841 { 842 static const struct dm_arg _args[] = { 843 {1, 1024, "invalid number of paths"}, 844 {0, 1024, "invalid number of selector args"} 845 }; 846 847 int r; 848 unsigned i, nr_selector_args, nr_args; 849 struct priority_group *pg; 850 struct dm_target *ti = m->ti; 851 852 if (as->argc < 2) { 853 as->argc = 0; 854 ti->error = "not enough priority group arguments"; 855 return ERR_PTR(-EINVAL); 856 } 857 858 pg = alloc_priority_group(); 859 if (!pg) { 860 ti->error = "couldn't allocate priority group"; 861 return ERR_PTR(-ENOMEM); 862 } 863 pg->m = m; 864 865 r = parse_path_selector(as, pg, ti); 866 if (r) 867 goto bad; 868 869 /* 870 * read the paths 871 */ 872 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error); 873 if (r) 874 goto bad; 875 876 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error); 877 if (r) 878 goto bad; 879 880 nr_args = 1 + nr_selector_args; 881 for (i = 0; i < pg->nr_pgpaths; i++) { 882 struct pgpath *pgpath; 883 struct dm_arg_set path_args; 884 885 if (as->argc < nr_args) { 886 ti->error = "not enough path parameters"; 887 r = -EINVAL; 888 goto bad; 889 } 890 891 path_args.argc = nr_args; 892 path_args.argv = as->argv; 893 894 pgpath = parse_path(&path_args, &pg->ps, ti); 895 if (IS_ERR(pgpath)) { 896 r = PTR_ERR(pgpath); 897 goto bad; 898 } 899 900 pgpath->pg = pg; 901 list_add_tail(&pgpath->list, &pg->pgpaths); 902 dm_consume_args(as, nr_args); 903 } 904 905 return pg; 906 907 bad: 908 free_priority_group(pg, ti); 909 return ERR_PTR(r); 910 } 911 912 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) 913 { 914 unsigned hw_argc; 915 int ret; 916 struct dm_target *ti = m->ti; 917 918 static const struct dm_arg _args[] = { 919 {0, 1024, "invalid number of hardware handler args"}, 920 }; 921 922 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error)) 923 return -EINVAL; 924 925 if (!hw_argc) 926 return 0; 927 928 if (m->queue_mode == DM_TYPE_BIO_BASED) { 929 dm_consume_args(as, hw_argc); 930 DMERR("bio-based multipath doesn't allow hardware handler args"); 931 return 0; 932 } 933 934 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); 935 if (!m->hw_handler_name) 936 return -EINVAL; 937 938 if (hw_argc > 1) { 939 char *p; 940 int i, j, len = 4; 941 942 for (i = 0; i <= hw_argc - 2; i++) 943 len += strlen(as->argv[i]) + 1; 944 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); 945 if (!p) { 946 ti->error = "memory allocation failed"; 947 ret = -ENOMEM; 948 goto fail; 949 } 950 j = sprintf(p, "%d", hw_argc - 1); 951 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1) 952 j = sprintf(p, "%s", as->argv[i]); 953 } 954 dm_consume_args(as, hw_argc - 1); 955 956 return 0; 957 fail: 958 kfree(m->hw_handler_name); 959 m->hw_handler_name = NULL; 960 return ret; 961 } 962 963 static int parse_features(struct dm_arg_set *as, struct multipath *m) 964 { 965 int r; 966 unsigned argc; 967 struct dm_target *ti = m->ti; 968 const char *arg_name; 969 970 static const struct dm_arg _args[] = { 971 {0, 8, "invalid number of feature args"}, 972 {1, 50, "pg_init_retries must be between 1 and 50"}, 973 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, 974 }; 975 976 r = dm_read_arg_group(_args, as, &argc, &ti->error); 977 if (r) 978 return -EINVAL; 979 980 if (!argc) 981 return 0; 982 983 do { 984 arg_name = dm_shift_arg(as); 985 argc--; 986 987 if (!strcasecmp(arg_name, "queue_if_no_path")) { 988 r = queue_if_no_path(m, true, false); 989 continue; 990 } 991 992 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) { 993 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 994 continue; 995 } 996 997 if (!strcasecmp(arg_name, "pg_init_retries") && 998 (argc >= 1)) { 999 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error); 1000 argc--; 1001 continue; 1002 } 1003 1004 if (!strcasecmp(arg_name, "pg_init_delay_msecs") && 1005 (argc >= 1)) { 1006 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error); 1007 argc--; 1008 continue; 1009 } 1010 1011 if (!strcasecmp(arg_name, "queue_mode") && 1012 (argc >= 1)) { 1013 const char *queue_mode_name = dm_shift_arg(as); 1014 1015 if (!strcasecmp(queue_mode_name, "bio")) 1016 m->queue_mode = DM_TYPE_BIO_BASED; 1017 else if (!strcasecmp(queue_mode_name, "rq")) 1018 m->queue_mode = DM_TYPE_REQUEST_BASED; 1019 else if (!strcasecmp(queue_mode_name, "mq")) 1020 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 1021 else { 1022 ti->error = "Unknown 'queue_mode' requested"; 1023 r = -EINVAL; 1024 } 1025 argc--; 1026 continue; 1027 } 1028 1029 ti->error = "Unrecognised multipath feature request"; 1030 r = -EINVAL; 1031 } while (argc && !r); 1032 1033 return r; 1034 } 1035 1036 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) 1037 { 1038 /* target arguments */ 1039 static const struct dm_arg _args[] = { 1040 {0, 1024, "invalid number of priority groups"}, 1041 {0, 1024, "invalid initial priority group number"}, 1042 }; 1043 1044 int r; 1045 struct multipath *m; 1046 struct dm_arg_set as; 1047 unsigned pg_count = 0; 1048 unsigned next_pg_num; 1049 1050 as.argc = argc; 1051 as.argv = argv; 1052 1053 m = alloc_multipath(ti); 1054 if (!m) { 1055 ti->error = "can't allocate multipath"; 1056 return -EINVAL; 1057 } 1058 1059 r = parse_features(&as, m); 1060 if (r) 1061 goto bad; 1062 1063 r = alloc_multipath_stage2(ti, m); 1064 if (r) 1065 goto bad; 1066 1067 r = parse_hw_handler(&as, m); 1068 if (r) 1069 goto bad; 1070 1071 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error); 1072 if (r) 1073 goto bad; 1074 1075 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error); 1076 if (r) 1077 goto bad; 1078 1079 if ((!m->nr_priority_groups && next_pg_num) || 1080 (m->nr_priority_groups && !next_pg_num)) { 1081 ti->error = "invalid initial priority group"; 1082 r = -EINVAL; 1083 goto bad; 1084 } 1085 1086 /* parse the priority groups */ 1087 while (as.argc) { 1088 struct priority_group *pg; 1089 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths); 1090 1091 pg = parse_priority_group(&as, m); 1092 if (IS_ERR(pg)) { 1093 r = PTR_ERR(pg); 1094 goto bad; 1095 } 1096 1097 nr_valid_paths += pg->nr_pgpaths; 1098 atomic_set(&m->nr_valid_paths, nr_valid_paths); 1099 1100 list_add_tail(&pg->list, &m->priority_groups); 1101 pg_count++; 1102 pg->pg_num = pg_count; 1103 if (!--next_pg_num) 1104 m->next_pg = pg; 1105 } 1106 1107 if (pg_count != m->nr_priority_groups) { 1108 ti->error = "priority group count mismatch"; 1109 r = -EINVAL; 1110 goto bad; 1111 } 1112 1113 ti->num_flush_bios = 1; 1114 ti->num_discard_bios = 1; 1115 ti->num_write_same_bios = 1; 1116 ti->num_write_zeroes_bios = 1; 1117 if (m->queue_mode == DM_TYPE_BIO_BASED) 1118 ti->per_io_data_size = multipath_per_bio_data_size(); 1119 else 1120 ti->per_io_data_size = sizeof(struct dm_mpath_io); 1121 1122 return 0; 1123 1124 bad: 1125 free_multipath(m); 1126 return r; 1127 } 1128 1129 static void multipath_wait_for_pg_init_completion(struct multipath *m) 1130 { 1131 DEFINE_WAIT(wait); 1132 1133 while (1) { 1134 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE); 1135 1136 if (!atomic_read(&m->pg_init_in_progress)) 1137 break; 1138 1139 io_schedule(); 1140 } 1141 finish_wait(&m->pg_init_wait, &wait); 1142 } 1143 1144 static void flush_multipath_work(struct multipath *m) 1145 { 1146 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1147 smp_mb__after_atomic(); 1148 1149 flush_workqueue(kmpath_handlerd); 1150 multipath_wait_for_pg_init_completion(m); 1151 flush_workqueue(kmultipathd); 1152 flush_work(&m->trigger_event); 1153 1154 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1155 smp_mb__after_atomic(); 1156 } 1157 1158 static void multipath_dtr(struct dm_target *ti) 1159 { 1160 struct multipath *m = ti->private; 1161 1162 flush_multipath_work(m); 1163 free_multipath(m); 1164 } 1165 1166 /* 1167 * Take a path out of use. 1168 */ 1169 static int fail_path(struct pgpath *pgpath) 1170 { 1171 unsigned long flags; 1172 struct multipath *m = pgpath->pg->m; 1173 1174 spin_lock_irqsave(&m->lock, flags); 1175 1176 if (!pgpath->is_active) 1177 goto out; 1178 1179 DMWARN("Failing path %s.", pgpath->path.dev->name); 1180 1181 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 1182 pgpath->is_active = false; 1183 pgpath->fail_count++; 1184 1185 atomic_dec(&m->nr_valid_paths); 1186 1187 if (pgpath == m->current_pgpath) 1188 m->current_pgpath = NULL; 1189 1190 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 1191 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths)); 1192 1193 schedule_work(&m->trigger_event); 1194 1195 out: 1196 spin_unlock_irqrestore(&m->lock, flags); 1197 1198 return 0; 1199 } 1200 1201 /* 1202 * Reinstate a previously-failed path 1203 */ 1204 static int reinstate_path(struct pgpath *pgpath) 1205 { 1206 int r = 0, run_queue = 0; 1207 unsigned long flags; 1208 struct multipath *m = pgpath->pg->m; 1209 unsigned nr_valid_paths; 1210 1211 spin_lock_irqsave(&m->lock, flags); 1212 1213 if (pgpath->is_active) 1214 goto out; 1215 1216 DMWARN("Reinstating path %s.", pgpath->path.dev->name); 1217 1218 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 1219 if (r) 1220 goto out; 1221 1222 pgpath->is_active = true; 1223 1224 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths); 1225 if (nr_valid_paths == 1) { 1226 m->current_pgpath = NULL; 1227 run_queue = 1; 1228 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) { 1229 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work)) 1230 atomic_inc(&m->pg_init_in_progress); 1231 } 1232 1233 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 1234 pgpath->path.dev->name, nr_valid_paths); 1235 1236 schedule_work(&m->trigger_event); 1237 1238 out: 1239 spin_unlock_irqrestore(&m->lock, flags); 1240 if (run_queue) { 1241 dm_table_run_md_queue_async(m->ti->table); 1242 process_queued_io_list(m); 1243 } 1244 1245 return r; 1246 } 1247 1248 /* 1249 * Fail or reinstate all paths that match the provided struct dm_dev. 1250 */ 1251 static int action_dev(struct multipath *m, struct dm_dev *dev, 1252 action_fn action) 1253 { 1254 int r = -EINVAL; 1255 struct pgpath *pgpath; 1256 struct priority_group *pg; 1257 1258 list_for_each_entry(pg, &m->priority_groups, list) { 1259 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1260 if (pgpath->path.dev == dev) 1261 r = action(pgpath); 1262 } 1263 } 1264 1265 return r; 1266 } 1267 1268 /* 1269 * Temporarily try to avoid having to use the specified PG 1270 */ 1271 static void bypass_pg(struct multipath *m, struct priority_group *pg, 1272 bool bypassed) 1273 { 1274 unsigned long flags; 1275 1276 spin_lock_irqsave(&m->lock, flags); 1277 1278 pg->bypassed = bypassed; 1279 m->current_pgpath = NULL; 1280 m->current_pg = NULL; 1281 1282 spin_unlock_irqrestore(&m->lock, flags); 1283 1284 schedule_work(&m->trigger_event); 1285 } 1286 1287 /* 1288 * Switch to using the specified PG from the next I/O that gets mapped 1289 */ 1290 static int switch_pg_num(struct multipath *m, const char *pgstr) 1291 { 1292 struct priority_group *pg; 1293 unsigned pgnum; 1294 unsigned long flags; 1295 char dummy; 1296 1297 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1298 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1299 DMWARN("invalid PG number supplied to switch_pg_num"); 1300 return -EINVAL; 1301 } 1302 1303 spin_lock_irqsave(&m->lock, flags); 1304 list_for_each_entry(pg, &m->priority_groups, list) { 1305 pg->bypassed = false; 1306 if (--pgnum) 1307 continue; 1308 1309 m->current_pgpath = NULL; 1310 m->current_pg = NULL; 1311 m->next_pg = pg; 1312 } 1313 spin_unlock_irqrestore(&m->lock, flags); 1314 1315 schedule_work(&m->trigger_event); 1316 return 0; 1317 } 1318 1319 /* 1320 * Set/clear bypassed status of a PG. 1321 * PGs are numbered upwards from 1 in the order they were declared. 1322 */ 1323 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed) 1324 { 1325 struct priority_group *pg; 1326 unsigned pgnum; 1327 char dummy; 1328 1329 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1330 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1331 DMWARN("invalid PG number supplied to bypass_pg"); 1332 return -EINVAL; 1333 } 1334 1335 list_for_each_entry(pg, &m->priority_groups, list) { 1336 if (!--pgnum) 1337 break; 1338 } 1339 1340 bypass_pg(m, pg, bypassed); 1341 return 0; 1342 } 1343 1344 /* 1345 * Should we retry pg_init immediately? 1346 */ 1347 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1348 { 1349 unsigned long flags; 1350 bool limit_reached = false; 1351 1352 spin_lock_irqsave(&m->lock, flags); 1353 1354 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries && 1355 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 1356 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 1357 else 1358 limit_reached = true; 1359 1360 spin_unlock_irqrestore(&m->lock, flags); 1361 1362 return limit_reached; 1363 } 1364 1365 static void pg_init_done(void *data, int errors) 1366 { 1367 struct pgpath *pgpath = data; 1368 struct priority_group *pg = pgpath->pg; 1369 struct multipath *m = pg->m; 1370 unsigned long flags; 1371 bool delay_retry = false; 1372 1373 /* device or driver problems */ 1374 switch (errors) { 1375 case SCSI_DH_OK: 1376 break; 1377 case SCSI_DH_NOSYS: 1378 if (!m->hw_handler_name) { 1379 errors = 0; 1380 break; 1381 } 1382 DMERR("Could not failover the device: Handler scsi_dh_%s " 1383 "Error %d.", m->hw_handler_name, errors); 1384 /* 1385 * Fail path for now, so we do not ping pong 1386 */ 1387 fail_path(pgpath); 1388 break; 1389 case SCSI_DH_DEV_TEMP_BUSY: 1390 /* 1391 * Probably doing something like FW upgrade on the 1392 * controller so try the other pg. 1393 */ 1394 bypass_pg(m, pg, true); 1395 break; 1396 case SCSI_DH_RETRY: 1397 /* Wait before retrying. */ 1398 delay_retry = 1; 1399 /* fall through */ 1400 case SCSI_DH_IMM_RETRY: 1401 case SCSI_DH_RES_TEMP_UNAVAIL: 1402 if (pg_init_limit_reached(m, pgpath)) 1403 fail_path(pgpath); 1404 errors = 0; 1405 break; 1406 case SCSI_DH_DEV_OFFLINED: 1407 default: 1408 /* 1409 * We probably do not want to fail the path for a device 1410 * error, but this is what the old dm did. In future 1411 * patches we can do more advanced handling. 1412 */ 1413 fail_path(pgpath); 1414 } 1415 1416 spin_lock_irqsave(&m->lock, flags); 1417 if (errors) { 1418 if (pgpath == m->current_pgpath) { 1419 DMERR("Could not failover device. Error %d.", errors); 1420 m->current_pgpath = NULL; 1421 m->current_pg = NULL; 1422 } 1423 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1424 pg->bypassed = false; 1425 1426 if (atomic_dec_return(&m->pg_init_in_progress) > 0) 1427 /* Activations of other paths are still on going */ 1428 goto out; 1429 1430 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 1431 if (delay_retry) 1432 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1433 else 1434 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1435 1436 if (__pg_init_all_paths(m)) 1437 goto out; 1438 } 1439 clear_bit(MPATHF_QUEUE_IO, &m->flags); 1440 1441 process_queued_io_list(m); 1442 1443 /* 1444 * Wake up any thread waiting to suspend. 1445 */ 1446 wake_up(&m->pg_init_wait); 1447 1448 out: 1449 spin_unlock_irqrestore(&m->lock, flags); 1450 } 1451 1452 static void activate_or_offline_path(struct pgpath *pgpath) 1453 { 1454 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1455 1456 if (pgpath->is_active && !blk_queue_dying(q)) 1457 scsi_dh_activate(q, pg_init_done, pgpath); 1458 else 1459 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED); 1460 } 1461 1462 static void activate_path_work(struct work_struct *work) 1463 { 1464 struct pgpath *pgpath = 1465 container_of(work, struct pgpath, activate_path.work); 1466 1467 activate_or_offline_path(pgpath); 1468 } 1469 1470 static int noretry_error(blk_status_t error) 1471 { 1472 switch (error) { 1473 case BLK_STS_NOTSUPP: 1474 case BLK_STS_NOSPC: 1475 case BLK_STS_TARGET: 1476 case BLK_STS_NEXUS: 1477 case BLK_STS_MEDIUM: 1478 return 1; 1479 } 1480 1481 /* Anything else could be a path failure, so should be retried */ 1482 return 0; 1483 } 1484 1485 static int multipath_end_io(struct dm_target *ti, struct request *clone, 1486 blk_status_t error, union map_info *map_context) 1487 { 1488 struct dm_mpath_io *mpio = get_mpio(map_context); 1489 struct pgpath *pgpath = mpio->pgpath; 1490 int r = DM_ENDIO_DONE; 1491 1492 /* 1493 * We don't queue any clone request inside the multipath target 1494 * during end I/O handling, since those clone requests don't have 1495 * bio clones. If we queue them inside the multipath target, 1496 * we need to make bio clones, that requires memory allocation. 1497 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests 1498 * don't have bio clones.) 1499 * Instead of queueing the clone request here, we queue the original 1500 * request into dm core, which will remake a clone request and 1501 * clone bios for it and resubmit it later. 1502 */ 1503 if (error && !noretry_error(error)) { 1504 struct multipath *m = ti->private; 1505 1506 r = DM_ENDIO_REQUEUE; 1507 1508 if (pgpath) 1509 fail_path(pgpath); 1510 1511 if (atomic_read(&m->nr_valid_paths) == 0 && 1512 !must_push_back_rq(m)) { 1513 if (error == BLK_STS_IOERR) 1514 dm_report_EIO(m); 1515 /* complete with the original error */ 1516 r = DM_ENDIO_DONE; 1517 } 1518 } 1519 1520 if (pgpath) { 1521 struct path_selector *ps = &pgpath->pg->ps; 1522 1523 if (ps->type->end_io) 1524 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1525 } 1526 1527 return r; 1528 } 1529 1530 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, 1531 blk_status_t *error) 1532 { 1533 struct multipath *m = ti->private; 1534 struct dm_mpath_io *mpio = get_mpio_from_bio(clone); 1535 struct pgpath *pgpath = mpio->pgpath; 1536 unsigned long flags; 1537 int r = DM_ENDIO_DONE; 1538 1539 if (!*error || noretry_error(*error)) 1540 goto done; 1541 1542 if (pgpath) 1543 fail_path(pgpath); 1544 1545 if (atomic_read(&m->nr_valid_paths) == 0 && 1546 !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1547 if (must_push_back_bio(m)) { 1548 r = DM_ENDIO_REQUEUE; 1549 } else { 1550 dm_report_EIO(m); 1551 *error = BLK_STS_IOERR; 1552 } 1553 goto done; 1554 } 1555 1556 /* Queue for the daemon to resubmit */ 1557 dm_bio_restore(get_bio_details_from_bio(clone), clone); 1558 1559 spin_lock_irqsave(&m->lock, flags); 1560 bio_list_add(&m->queued_bios, clone); 1561 spin_unlock_irqrestore(&m->lock, flags); 1562 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) 1563 queue_work(kmultipathd, &m->process_queued_bios); 1564 1565 r = DM_ENDIO_INCOMPLETE; 1566 done: 1567 if (pgpath) { 1568 struct path_selector *ps = &pgpath->pg->ps; 1569 1570 if (ps->type->end_io) 1571 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1572 } 1573 1574 return r; 1575 } 1576 1577 /* 1578 * Suspend can't complete until all the I/O is processed so if 1579 * the last path fails we must error any remaining I/O. 1580 * Note that if the freeze_bdev fails while suspending, the 1581 * queue_if_no_path state is lost - userspace should reset it. 1582 */ 1583 static void multipath_presuspend(struct dm_target *ti) 1584 { 1585 struct multipath *m = ti->private; 1586 1587 queue_if_no_path(m, false, true); 1588 } 1589 1590 static void multipath_postsuspend(struct dm_target *ti) 1591 { 1592 struct multipath *m = ti->private; 1593 1594 mutex_lock(&m->work_mutex); 1595 flush_multipath_work(m); 1596 mutex_unlock(&m->work_mutex); 1597 } 1598 1599 /* 1600 * Restore the queue_if_no_path setting. 1601 */ 1602 static void multipath_resume(struct dm_target *ti) 1603 { 1604 struct multipath *m = ti->private; 1605 unsigned long flags; 1606 1607 spin_lock_irqsave(&m->lock, flags); 1608 assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, 1609 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)); 1610 spin_unlock_irqrestore(&m->lock, flags); 1611 } 1612 1613 /* 1614 * Info output has the following format: 1615 * num_multipath_feature_args [multipath_feature_args]* 1616 * num_handler_status_args [handler_status_args]* 1617 * num_groups init_group_number 1618 * [A|D|E num_ps_status_args [ps_status_args]* 1619 * num_paths num_selector_args 1620 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1621 * 1622 * Table output has the following format (identical to the constructor string): 1623 * num_feature_args [features_args]* 1624 * num_handler_args hw_handler [hw_handler_args]* 1625 * num_groups init_group_number 1626 * [priority selector-name num_ps_args [ps_args]* 1627 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1628 */ 1629 static void multipath_status(struct dm_target *ti, status_type_t type, 1630 unsigned status_flags, char *result, unsigned maxlen) 1631 { 1632 int sz = 0; 1633 unsigned long flags; 1634 struct multipath *m = ti->private; 1635 struct priority_group *pg; 1636 struct pgpath *p; 1637 unsigned pg_num; 1638 char state; 1639 1640 spin_lock_irqsave(&m->lock, flags); 1641 1642 /* Features */ 1643 if (type == STATUSTYPE_INFO) 1644 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags), 1645 atomic_read(&m->pg_init_count)); 1646 else { 1647 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) + 1648 (m->pg_init_retries > 0) * 2 + 1649 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + 1650 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) + 1651 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2); 1652 1653 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1654 DMEMIT("queue_if_no_path "); 1655 if (m->pg_init_retries) 1656 DMEMIT("pg_init_retries %u ", m->pg_init_retries); 1657 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) 1658 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); 1659 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) 1660 DMEMIT("retain_attached_hw_handler "); 1661 if (m->queue_mode != DM_TYPE_REQUEST_BASED) { 1662 switch(m->queue_mode) { 1663 case DM_TYPE_BIO_BASED: 1664 DMEMIT("queue_mode bio "); 1665 break; 1666 case DM_TYPE_MQ_REQUEST_BASED: 1667 DMEMIT("queue_mode mq "); 1668 break; 1669 default: 1670 WARN_ON_ONCE(true); 1671 break; 1672 } 1673 } 1674 } 1675 1676 if (!m->hw_handler_name || type == STATUSTYPE_INFO) 1677 DMEMIT("0 "); 1678 else 1679 DMEMIT("1 %s ", m->hw_handler_name); 1680 1681 DMEMIT("%u ", m->nr_priority_groups); 1682 1683 if (m->next_pg) 1684 pg_num = m->next_pg->pg_num; 1685 else if (m->current_pg) 1686 pg_num = m->current_pg->pg_num; 1687 else 1688 pg_num = (m->nr_priority_groups ? 1 : 0); 1689 1690 DMEMIT("%u ", pg_num); 1691 1692 switch (type) { 1693 case STATUSTYPE_INFO: 1694 list_for_each_entry(pg, &m->priority_groups, list) { 1695 if (pg->bypassed) 1696 state = 'D'; /* Disabled */ 1697 else if (pg == m->current_pg) 1698 state = 'A'; /* Currently Active */ 1699 else 1700 state = 'E'; /* Enabled */ 1701 1702 DMEMIT("%c ", state); 1703 1704 if (pg->ps.type->status) 1705 sz += pg->ps.type->status(&pg->ps, NULL, type, 1706 result + sz, 1707 maxlen - sz); 1708 else 1709 DMEMIT("0 "); 1710 1711 DMEMIT("%u %u ", pg->nr_pgpaths, 1712 pg->ps.type->info_args); 1713 1714 list_for_each_entry(p, &pg->pgpaths, list) { 1715 DMEMIT("%s %s %u ", p->path.dev->name, 1716 p->is_active ? "A" : "F", 1717 p->fail_count); 1718 if (pg->ps.type->status) 1719 sz += pg->ps.type->status(&pg->ps, 1720 &p->path, type, result + sz, 1721 maxlen - sz); 1722 } 1723 } 1724 break; 1725 1726 case STATUSTYPE_TABLE: 1727 list_for_each_entry(pg, &m->priority_groups, list) { 1728 DMEMIT("%s ", pg->ps.type->name); 1729 1730 if (pg->ps.type->status) 1731 sz += pg->ps.type->status(&pg->ps, NULL, type, 1732 result + sz, 1733 maxlen - sz); 1734 else 1735 DMEMIT("0 "); 1736 1737 DMEMIT("%u %u ", pg->nr_pgpaths, 1738 pg->ps.type->table_args); 1739 1740 list_for_each_entry(p, &pg->pgpaths, list) { 1741 DMEMIT("%s ", p->path.dev->name); 1742 if (pg->ps.type->status) 1743 sz += pg->ps.type->status(&pg->ps, 1744 &p->path, type, result + sz, 1745 maxlen - sz); 1746 } 1747 } 1748 break; 1749 } 1750 1751 spin_unlock_irqrestore(&m->lock, flags); 1752 } 1753 1754 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 1755 { 1756 int r = -EINVAL; 1757 struct dm_dev *dev; 1758 struct multipath *m = ti->private; 1759 action_fn action; 1760 1761 mutex_lock(&m->work_mutex); 1762 1763 if (dm_suspended(ti)) { 1764 r = -EBUSY; 1765 goto out; 1766 } 1767 1768 if (argc == 1) { 1769 if (!strcasecmp(argv[0], "queue_if_no_path")) { 1770 r = queue_if_no_path(m, true, false); 1771 goto out; 1772 } else if (!strcasecmp(argv[0], "fail_if_no_path")) { 1773 r = queue_if_no_path(m, false, false); 1774 goto out; 1775 } 1776 } 1777 1778 if (argc != 2) { 1779 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc); 1780 goto out; 1781 } 1782 1783 if (!strcasecmp(argv[0], "disable_group")) { 1784 r = bypass_pg_num(m, argv[1], true); 1785 goto out; 1786 } else if (!strcasecmp(argv[0], "enable_group")) { 1787 r = bypass_pg_num(m, argv[1], false); 1788 goto out; 1789 } else if (!strcasecmp(argv[0], "switch_group")) { 1790 r = switch_pg_num(m, argv[1]); 1791 goto out; 1792 } else if (!strcasecmp(argv[0], "reinstate_path")) 1793 action = reinstate_path; 1794 else if (!strcasecmp(argv[0], "fail_path")) 1795 action = fail_path; 1796 else { 1797 DMWARN("Unrecognised multipath message received: %s", argv[0]); 1798 goto out; 1799 } 1800 1801 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev); 1802 if (r) { 1803 DMWARN("message: error getting device %s", 1804 argv[1]); 1805 goto out; 1806 } 1807 1808 r = action_dev(m, dev, action); 1809 1810 dm_put_device(ti, dev); 1811 1812 out: 1813 mutex_unlock(&m->work_mutex); 1814 return r; 1815 } 1816 1817 static int multipath_prepare_ioctl(struct dm_target *ti, 1818 struct block_device **bdev, fmode_t *mode) 1819 { 1820 struct multipath *m = ti->private; 1821 struct pgpath *current_pgpath; 1822 int r; 1823 1824 current_pgpath = READ_ONCE(m->current_pgpath); 1825 if (!current_pgpath) 1826 current_pgpath = choose_pgpath(m, 0); 1827 1828 if (current_pgpath) { 1829 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) { 1830 *bdev = current_pgpath->path.dev->bdev; 1831 *mode = current_pgpath->path.dev->mode; 1832 r = 0; 1833 } else { 1834 /* pg_init has not started or completed */ 1835 r = -ENOTCONN; 1836 } 1837 } else { 1838 /* No path is available */ 1839 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1840 r = -ENOTCONN; 1841 else 1842 r = -EIO; 1843 } 1844 1845 if (r == -ENOTCONN) { 1846 if (!READ_ONCE(m->current_pg)) { 1847 /* Path status changed, redo selection */ 1848 (void) choose_pgpath(m, 0); 1849 } 1850 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1851 pg_init_all_paths(m); 1852 dm_table_run_md_queue_async(m->ti->table); 1853 process_queued_io_list(m); 1854 } 1855 1856 /* 1857 * Only pass ioctls through if the device sizes match exactly. 1858 */ 1859 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) 1860 return 1; 1861 return r; 1862 } 1863 1864 static int multipath_iterate_devices(struct dm_target *ti, 1865 iterate_devices_callout_fn fn, void *data) 1866 { 1867 struct multipath *m = ti->private; 1868 struct priority_group *pg; 1869 struct pgpath *p; 1870 int ret = 0; 1871 1872 list_for_each_entry(pg, &m->priority_groups, list) { 1873 list_for_each_entry(p, &pg->pgpaths, list) { 1874 ret = fn(ti, p->path.dev, ti->begin, ti->len, data); 1875 if (ret) 1876 goto out; 1877 } 1878 } 1879 1880 out: 1881 return ret; 1882 } 1883 1884 static int pgpath_busy(struct pgpath *pgpath) 1885 { 1886 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1887 1888 return blk_lld_busy(q); 1889 } 1890 1891 /* 1892 * We return "busy", only when we can map I/Os but underlying devices 1893 * are busy (so even if we map I/Os now, the I/Os will wait on 1894 * the underlying queue). 1895 * In other words, if we want to kill I/Os or queue them inside us 1896 * due to map unavailability, we don't return "busy". Otherwise, 1897 * dm core won't give us the I/Os and we can't do what we want. 1898 */ 1899 static int multipath_busy(struct dm_target *ti) 1900 { 1901 bool busy = false, has_active = false; 1902 struct multipath *m = ti->private; 1903 struct priority_group *pg, *next_pg; 1904 struct pgpath *pgpath; 1905 1906 /* pg_init in progress */ 1907 if (atomic_read(&m->pg_init_in_progress)) 1908 return true; 1909 1910 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */ 1911 if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1912 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED); 1913 1914 /* Guess which priority_group will be used at next mapping time */ 1915 pg = READ_ONCE(m->current_pg); 1916 next_pg = READ_ONCE(m->next_pg); 1917 if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg)) 1918 pg = next_pg; 1919 1920 if (!pg) { 1921 /* 1922 * We don't know which pg will be used at next mapping time. 1923 * We don't call choose_pgpath() here to avoid to trigger 1924 * pg_init just by busy checking. 1925 * So we don't know whether underlying devices we will be using 1926 * at next mapping time are busy or not. Just try mapping. 1927 */ 1928 return busy; 1929 } 1930 1931 /* 1932 * If there is one non-busy active path at least, the path selector 1933 * will be able to select it. So we consider such a pg as not busy. 1934 */ 1935 busy = true; 1936 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1937 if (pgpath->is_active) { 1938 has_active = true; 1939 if (!pgpath_busy(pgpath)) { 1940 busy = false; 1941 break; 1942 } 1943 } 1944 } 1945 1946 if (!has_active) { 1947 /* 1948 * No active path in this pg, so this pg won't be used and 1949 * the current_pg will be changed at next mapping time. 1950 * We need to try mapping to determine it. 1951 */ 1952 busy = false; 1953 } 1954 1955 return busy; 1956 } 1957 1958 /*----------------------------------------------------------------- 1959 * Module setup 1960 *---------------------------------------------------------------*/ 1961 static struct target_type multipath_target = { 1962 .name = "multipath", 1963 .version = {1, 12, 0}, 1964 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, 1965 .module = THIS_MODULE, 1966 .ctr = multipath_ctr, 1967 .dtr = multipath_dtr, 1968 .clone_and_map_rq = multipath_clone_and_map, 1969 .release_clone_rq = multipath_release_clone, 1970 .rq_end_io = multipath_end_io, 1971 .map = multipath_map_bio, 1972 .end_io = multipath_end_io_bio, 1973 .presuspend = multipath_presuspend, 1974 .postsuspend = multipath_postsuspend, 1975 .resume = multipath_resume, 1976 .status = multipath_status, 1977 .message = multipath_message, 1978 .prepare_ioctl = multipath_prepare_ioctl, 1979 .iterate_devices = multipath_iterate_devices, 1980 .busy = multipath_busy, 1981 }; 1982 1983 static int __init dm_multipath_init(void) 1984 { 1985 int r; 1986 1987 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 1988 if (!kmultipathd) { 1989 DMERR("failed to create workqueue kmpathd"); 1990 r = -ENOMEM; 1991 goto bad_alloc_kmultipathd; 1992 } 1993 1994 /* 1995 * A separate workqueue is used to handle the device handlers 1996 * to avoid overloading existing workqueue. Overloading the 1997 * old workqueue would also create a bottleneck in the 1998 * path of the storage hardware device activation. 1999 */ 2000 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd", 2001 WQ_MEM_RECLAIM); 2002 if (!kmpath_handlerd) { 2003 DMERR("failed to create workqueue kmpath_handlerd"); 2004 r = -ENOMEM; 2005 goto bad_alloc_kmpath_handlerd; 2006 } 2007 2008 r = dm_register_target(&multipath_target); 2009 if (r < 0) { 2010 DMERR("request-based register failed %d", r); 2011 r = -EINVAL; 2012 goto bad_register_target; 2013 } 2014 2015 return 0; 2016 2017 bad_register_target: 2018 destroy_workqueue(kmpath_handlerd); 2019 bad_alloc_kmpath_handlerd: 2020 destroy_workqueue(kmultipathd); 2021 bad_alloc_kmultipathd: 2022 return r; 2023 } 2024 2025 static void __exit dm_multipath_exit(void) 2026 { 2027 destroy_workqueue(kmpath_handlerd); 2028 destroy_workqueue(kmultipathd); 2029 2030 dm_unregister_target(&multipath_target); 2031 } 2032 2033 module_init(dm_multipath_init); 2034 module_exit(dm_multipath_exit); 2035 2036 MODULE_DESCRIPTION(DM_NAME " multipath target"); 2037 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 2038 MODULE_LICENSE("GPL"); 2039