1 /* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 10 #include "dm-rq.h" 11 #include "dm-bio-record.h" 12 #include "dm-path-selector.h" 13 #include "dm-uevent.h" 14 15 #include <linux/blkdev.h> 16 #include <linux/ctype.h> 17 #include <linux/init.h> 18 #include <linux/mempool.h> 19 #include <linux/module.h> 20 #include <linux/pagemap.h> 21 #include <linux/slab.h> 22 #include <linux/time.h> 23 #include <linux/workqueue.h> 24 #include <linux/delay.h> 25 #include <scsi/scsi_dh.h> 26 #include <linux/atomic.h> 27 #include <linux/blk-mq.h> 28 29 #define DM_MSG_PREFIX "multipath" 30 #define DM_PG_INIT_DELAY_MSECS 2000 31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1) 32 33 /* Path properties */ 34 struct pgpath { 35 struct list_head list; 36 37 struct priority_group *pg; /* Owning PG */ 38 unsigned fail_count; /* Cumulative failure count */ 39 40 struct dm_path path; 41 struct delayed_work activate_path; 42 43 bool is_active:1; /* Path status */ 44 }; 45 46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 47 48 /* 49 * Paths are grouped into Priority Groups and numbered from 1 upwards. 50 * Each has a path selector which controls which path gets used. 51 */ 52 struct priority_group { 53 struct list_head list; 54 55 struct multipath *m; /* Owning multipath instance */ 56 struct path_selector ps; 57 58 unsigned pg_num; /* Reference number */ 59 unsigned nr_pgpaths; /* Number of paths in PG */ 60 struct list_head pgpaths; 61 62 bool bypassed:1; /* Temporarily bypass this PG? */ 63 }; 64 65 /* Multipath context */ 66 struct multipath { 67 struct list_head list; 68 struct dm_target *ti; 69 70 const char *hw_handler_name; 71 char *hw_handler_params; 72 73 spinlock_t lock; 74 75 unsigned nr_priority_groups; 76 struct list_head priority_groups; 77 78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */ 79 80 struct pgpath *current_pgpath; 81 struct priority_group *current_pg; 82 struct priority_group *next_pg; /* Switch to this PG if set */ 83 84 unsigned long flags; /* Multipath state flags */ 85 86 unsigned pg_init_retries; /* Number of times to retry pg_init */ 87 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */ 88 89 atomic_t nr_valid_paths; /* Total number of usable paths */ 90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */ 91 atomic_t pg_init_count; /* Number of times pg_init called */ 92 93 unsigned queue_mode; 94 95 struct mutex work_mutex; 96 struct work_struct trigger_event; 97 98 struct work_struct process_queued_bios; 99 struct bio_list queued_bios; 100 }; 101 102 /* 103 * Context information attached to each io we process. 104 */ 105 struct dm_mpath_io { 106 struct pgpath *pgpath; 107 size_t nr_bytes; 108 }; 109 110 typedef int (*action_fn) (struct pgpath *pgpath); 111 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 113 static void trigger_event(struct work_struct *work); 114 static void activate_path(struct work_struct *work); 115 static void process_queued_bios(struct work_struct *work); 116 117 /*----------------------------------------------- 118 * Multipath state flags. 119 *-----------------------------------------------*/ 120 121 #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */ 122 #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */ 123 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */ 124 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */ 125 #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */ 126 #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */ 127 #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */ 128 129 /*----------------------------------------------- 130 * Allocation routines 131 *-----------------------------------------------*/ 132 133 static struct pgpath *alloc_pgpath(void) 134 { 135 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 136 137 if (pgpath) { 138 pgpath->is_active = true; 139 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path); 140 } 141 142 return pgpath; 143 } 144 145 static void free_pgpath(struct pgpath *pgpath) 146 { 147 kfree(pgpath); 148 } 149 150 static struct priority_group *alloc_priority_group(void) 151 { 152 struct priority_group *pg; 153 154 pg = kzalloc(sizeof(*pg), GFP_KERNEL); 155 156 if (pg) 157 INIT_LIST_HEAD(&pg->pgpaths); 158 159 return pg; 160 } 161 162 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 163 { 164 struct pgpath *pgpath, *tmp; 165 166 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 167 list_del(&pgpath->list); 168 dm_put_device(ti, pgpath->path.dev); 169 free_pgpath(pgpath); 170 } 171 } 172 173 static void free_priority_group(struct priority_group *pg, 174 struct dm_target *ti) 175 { 176 struct path_selector *ps = &pg->ps; 177 178 if (ps->type) { 179 ps->type->destroy(ps); 180 dm_put_path_selector(ps->type); 181 } 182 183 free_pgpaths(&pg->pgpaths, ti); 184 kfree(pg); 185 } 186 187 static struct multipath *alloc_multipath(struct dm_target *ti) 188 { 189 struct multipath *m; 190 191 m = kzalloc(sizeof(*m), GFP_KERNEL); 192 if (m) { 193 INIT_LIST_HEAD(&m->priority_groups); 194 spin_lock_init(&m->lock); 195 set_bit(MPATHF_QUEUE_IO, &m->flags); 196 atomic_set(&m->nr_valid_paths, 0); 197 atomic_set(&m->pg_init_in_progress, 0); 198 atomic_set(&m->pg_init_count, 0); 199 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 200 INIT_WORK(&m->trigger_event, trigger_event); 201 init_waitqueue_head(&m->pg_init_wait); 202 mutex_init(&m->work_mutex); 203 204 m->queue_mode = DM_TYPE_NONE; 205 206 m->ti = ti; 207 ti->private = m; 208 } 209 210 return m; 211 } 212 213 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) 214 { 215 if (m->queue_mode == DM_TYPE_NONE) { 216 /* 217 * Default to request-based. 218 */ 219 if (dm_use_blk_mq(dm_table_get_md(ti->table))) 220 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 221 else 222 m->queue_mode = DM_TYPE_REQUEST_BASED; 223 } else if (m->queue_mode == DM_TYPE_BIO_BASED) { 224 INIT_WORK(&m->process_queued_bios, process_queued_bios); 225 /* 226 * bio-based doesn't support any direct scsi_dh management; 227 * it just discovers if a scsi_dh is attached. 228 */ 229 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 230 } 231 232 dm_table_set_type(ti->table, m->queue_mode); 233 234 return 0; 235 } 236 237 static void free_multipath(struct multipath *m) 238 { 239 struct priority_group *pg, *tmp; 240 241 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 242 list_del(&pg->list); 243 free_priority_group(pg, m->ti); 244 } 245 246 kfree(m->hw_handler_name); 247 kfree(m->hw_handler_params); 248 kfree(m); 249 } 250 251 static struct dm_mpath_io *get_mpio(union map_info *info) 252 { 253 return info->ptr; 254 } 255 256 static size_t multipath_per_bio_data_size(void) 257 { 258 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); 259 } 260 261 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio) 262 { 263 return dm_per_bio_data(bio, multipath_per_bio_data_size()); 264 } 265 266 static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio) 267 { 268 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */ 269 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 270 void *bio_details = mpio + 1; 271 272 return bio_details; 273 } 274 275 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p, 276 struct dm_bio_details **bio_details_p) 277 { 278 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 279 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio); 280 281 memset(mpio, 0, sizeof(*mpio)); 282 memset(bio_details, 0, sizeof(*bio_details)); 283 dm_bio_record(bio_details, bio); 284 285 if (mpio_p) 286 *mpio_p = mpio; 287 if (bio_details_p) 288 *bio_details_p = bio_details; 289 } 290 291 /*----------------------------------------------- 292 * Path selection 293 *-----------------------------------------------*/ 294 295 static int __pg_init_all_paths(struct multipath *m) 296 { 297 struct pgpath *pgpath; 298 unsigned long pg_init_delay = 0; 299 300 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 301 return 0; 302 303 atomic_inc(&m->pg_init_count); 304 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 305 306 /* Check here to reset pg_init_required */ 307 if (!m->current_pg) 308 return 0; 309 310 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags)) 311 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ? 312 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS); 313 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) { 314 /* Skip failed paths */ 315 if (!pgpath->is_active) 316 continue; 317 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path, 318 pg_init_delay)) 319 atomic_inc(&m->pg_init_in_progress); 320 } 321 return atomic_read(&m->pg_init_in_progress); 322 } 323 324 static void pg_init_all_paths(struct multipath *m) 325 { 326 unsigned long flags; 327 328 spin_lock_irqsave(&m->lock, flags); 329 __pg_init_all_paths(m); 330 spin_unlock_irqrestore(&m->lock, flags); 331 } 332 333 static void __switch_pg(struct multipath *m, struct priority_group *pg) 334 { 335 m->current_pg = pg; 336 337 /* Must we initialise the PG first, and queue I/O till it's ready? */ 338 if (m->hw_handler_name) { 339 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 340 set_bit(MPATHF_QUEUE_IO, &m->flags); 341 } else { 342 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 343 clear_bit(MPATHF_QUEUE_IO, &m->flags); 344 } 345 346 atomic_set(&m->pg_init_count, 0); 347 } 348 349 static struct pgpath *choose_path_in_pg(struct multipath *m, 350 struct priority_group *pg, 351 size_t nr_bytes) 352 { 353 unsigned long flags; 354 struct dm_path *path; 355 struct pgpath *pgpath; 356 357 path = pg->ps.type->select_path(&pg->ps, nr_bytes); 358 if (!path) 359 return ERR_PTR(-ENXIO); 360 361 pgpath = path_to_pgpath(path); 362 363 if (unlikely(lockless_dereference(m->current_pg) != pg)) { 364 /* Only update current_pgpath if pg changed */ 365 spin_lock_irqsave(&m->lock, flags); 366 m->current_pgpath = pgpath; 367 __switch_pg(m, pg); 368 spin_unlock_irqrestore(&m->lock, flags); 369 } 370 371 return pgpath; 372 } 373 374 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes) 375 { 376 unsigned long flags; 377 struct priority_group *pg; 378 struct pgpath *pgpath; 379 unsigned bypassed = 1; 380 381 if (!atomic_read(&m->nr_valid_paths)) { 382 clear_bit(MPATHF_QUEUE_IO, &m->flags); 383 goto failed; 384 } 385 386 /* Were we instructed to switch PG? */ 387 if (lockless_dereference(m->next_pg)) { 388 spin_lock_irqsave(&m->lock, flags); 389 pg = m->next_pg; 390 if (!pg) { 391 spin_unlock_irqrestore(&m->lock, flags); 392 goto check_current_pg; 393 } 394 m->next_pg = NULL; 395 spin_unlock_irqrestore(&m->lock, flags); 396 pgpath = choose_path_in_pg(m, pg, nr_bytes); 397 if (!IS_ERR_OR_NULL(pgpath)) 398 return pgpath; 399 } 400 401 /* Don't change PG until it has no remaining paths */ 402 check_current_pg: 403 pg = lockless_dereference(m->current_pg); 404 if (pg) { 405 pgpath = choose_path_in_pg(m, pg, nr_bytes); 406 if (!IS_ERR_OR_NULL(pgpath)) 407 return pgpath; 408 } 409 410 /* 411 * Loop through priority groups until we find a valid path. 412 * First time we skip PGs marked 'bypassed'. 413 * Second time we only try the ones we skipped, but set 414 * pg_init_delay_retry so we do not hammer controllers. 415 */ 416 do { 417 list_for_each_entry(pg, &m->priority_groups, list) { 418 if (pg->bypassed == !!bypassed) 419 continue; 420 pgpath = choose_path_in_pg(m, pg, nr_bytes); 421 if (!IS_ERR_OR_NULL(pgpath)) { 422 if (!bypassed) 423 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 424 return pgpath; 425 } 426 } 427 } while (bypassed--); 428 429 failed: 430 spin_lock_irqsave(&m->lock, flags); 431 m->current_pgpath = NULL; 432 m->current_pg = NULL; 433 spin_unlock_irqrestore(&m->lock, flags); 434 435 return NULL; 436 } 437 438 /* 439 * Check whether bios must be queued in the device-mapper core rather 440 * than here in the target. 441 * 442 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the 443 * same value then we are not between multipath_presuspend() 444 * and multipath_resume() calls and we have no need to check 445 * for the DMF_NOFLUSH_SUSPENDING flag. 446 */ 447 static bool __must_push_back(struct multipath *m) 448 { 449 return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) != 450 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) && 451 dm_noflush_suspending(m->ti)); 452 } 453 454 static bool must_push_back_rq(struct multipath *m) 455 { 456 bool r; 457 unsigned long flags; 458 459 spin_lock_irqsave(&m->lock, flags); 460 r = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) || 461 __must_push_back(m)); 462 spin_unlock_irqrestore(&m->lock, flags); 463 464 return r; 465 } 466 467 static bool must_push_back_bio(struct multipath *m) 468 { 469 bool r; 470 unsigned long flags; 471 472 spin_lock_irqsave(&m->lock, flags); 473 r = __must_push_back(m); 474 spin_unlock_irqrestore(&m->lock, flags); 475 476 return r; 477 } 478 479 /* 480 * Map cloned requests (request-based multipath) 481 */ 482 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq, 483 union map_info *map_context, 484 struct request **__clone) 485 { 486 struct multipath *m = ti->private; 487 int r = DM_MAPIO_REQUEUE; 488 size_t nr_bytes = blk_rq_bytes(rq); 489 struct pgpath *pgpath; 490 struct block_device *bdev; 491 struct dm_mpath_io *mpio = get_mpio(map_context); 492 struct request *clone; 493 494 /* Do we need to select a new pgpath? */ 495 pgpath = lockless_dereference(m->current_pgpath); 496 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags)) 497 pgpath = choose_pgpath(m, nr_bytes); 498 499 if (!pgpath) { 500 if (must_push_back_rq(m)) 501 return DM_MAPIO_DELAY_REQUEUE; 502 return -EIO; /* Failed */ 503 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) || 504 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 505 pg_init_all_paths(m); 506 return r; 507 } 508 509 memset(mpio, 0, sizeof(*mpio)); 510 mpio->pgpath = pgpath; 511 mpio->nr_bytes = nr_bytes; 512 513 bdev = pgpath->path.dev->bdev; 514 515 clone = blk_get_request(bdev_get_queue(bdev), 516 rq->cmd_flags | REQ_NOMERGE, 517 GFP_ATOMIC); 518 if (IS_ERR(clone)) { 519 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */ 520 return r; 521 } 522 clone->bio = clone->biotail = NULL; 523 clone->rq_disk = bdev->bd_disk; 524 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; 525 *__clone = clone; 526 527 if (pgpath->pg->ps.type->start_io) 528 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 529 &pgpath->path, 530 nr_bytes); 531 return DM_MAPIO_REMAPPED; 532 } 533 534 static void multipath_release_clone(struct request *clone) 535 { 536 blk_put_request(clone); 537 } 538 539 /* 540 * Map cloned bios (bio-based multipath) 541 */ 542 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio) 543 { 544 size_t nr_bytes = bio->bi_iter.bi_size; 545 struct pgpath *pgpath; 546 unsigned long flags; 547 bool queue_io; 548 549 /* Do we need to select a new pgpath? */ 550 pgpath = lockless_dereference(m->current_pgpath); 551 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags); 552 if (!pgpath || !queue_io) 553 pgpath = choose_pgpath(m, nr_bytes); 554 555 if ((pgpath && queue_io) || 556 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) { 557 /* Queue for the daemon to resubmit */ 558 spin_lock_irqsave(&m->lock, flags); 559 bio_list_add(&m->queued_bios, bio); 560 spin_unlock_irqrestore(&m->lock, flags); 561 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */ 562 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 563 pg_init_all_paths(m); 564 else if (!queue_io) 565 queue_work(kmultipathd, &m->process_queued_bios); 566 return DM_MAPIO_SUBMITTED; 567 } 568 569 if (!pgpath) { 570 if (!must_push_back_bio(m)) 571 return -EIO; 572 return DM_MAPIO_REQUEUE; 573 } 574 575 mpio->pgpath = pgpath; 576 mpio->nr_bytes = nr_bytes; 577 578 bio->bi_error = 0; 579 bio->bi_bdev = pgpath->path.dev->bdev; 580 bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 581 582 if (pgpath->pg->ps.type->start_io) 583 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 584 &pgpath->path, 585 nr_bytes); 586 return DM_MAPIO_REMAPPED; 587 } 588 589 static int multipath_map_bio(struct dm_target *ti, struct bio *bio) 590 { 591 struct multipath *m = ti->private; 592 struct dm_mpath_io *mpio = NULL; 593 594 multipath_init_per_bio_data(bio, &mpio, NULL); 595 596 return __multipath_map_bio(m, bio, mpio); 597 } 598 599 static void process_queued_io_list(struct multipath *m) 600 { 601 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED) 602 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table)); 603 else if (m->queue_mode == DM_TYPE_BIO_BASED) 604 queue_work(kmultipathd, &m->process_queued_bios); 605 } 606 607 static void process_queued_bios(struct work_struct *work) 608 { 609 int r; 610 unsigned long flags; 611 struct bio *bio; 612 struct bio_list bios; 613 struct blk_plug plug; 614 struct multipath *m = 615 container_of(work, struct multipath, process_queued_bios); 616 617 bio_list_init(&bios); 618 619 spin_lock_irqsave(&m->lock, flags); 620 621 if (bio_list_empty(&m->queued_bios)) { 622 spin_unlock_irqrestore(&m->lock, flags); 623 return; 624 } 625 626 bio_list_merge(&bios, &m->queued_bios); 627 bio_list_init(&m->queued_bios); 628 629 spin_unlock_irqrestore(&m->lock, flags); 630 631 blk_start_plug(&plug); 632 while ((bio = bio_list_pop(&bios))) { 633 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio)); 634 if (r < 0 || r == DM_MAPIO_REQUEUE) { 635 bio->bi_error = r; 636 bio_endio(bio); 637 } else if (r == DM_MAPIO_REMAPPED) 638 generic_make_request(bio); 639 } 640 blk_finish_plug(&plug); 641 } 642 643 /* 644 * If we run out of usable paths, should we queue I/O or error it? 645 */ 646 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path, 647 bool save_old_value) 648 { 649 unsigned long flags; 650 651 spin_lock_irqsave(&m->lock, flags); 652 653 if (save_old_value) { 654 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 655 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 656 else 657 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 658 } else { 659 if (queue_if_no_path) 660 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 661 else 662 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 663 } 664 if (queue_if_no_path) 665 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 666 else 667 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 668 669 spin_unlock_irqrestore(&m->lock, flags); 670 671 if (!queue_if_no_path) { 672 dm_table_run_md_queue_async(m->ti->table); 673 process_queued_io_list(m); 674 } 675 676 return 0; 677 } 678 679 /* 680 * An event is triggered whenever a path is taken out of use. 681 * Includes path failure and PG bypass. 682 */ 683 static void trigger_event(struct work_struct *work) 684 { 685 struct multipath *m = 686 container_of(work, struct multipath, trigger_event); 687 688 dm_table_event(m->ti->table); 689 } 690 691 /*----------------------------------------------------------------- 692 * Constructor/argument parsing: 693 * <#multipath feature args> [<arg>]* 694 * <#hw_handler args> [hw_handler [<arg>]*] 695 * <#priority groups> 696 * <initial priority group> 697 * [<selector> <#selector args> [<arg>]* 698 * <#paths> <#per-path selector args> 699 * [<path> [<arg>]* ]+ ]+ 700 *---------------------------------------------------------------*/ 701 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, 702 struct dm_target *ti) 703 { 704 int r; 705 struct path_selector_type *pst; 706 unsigned ps_argc; 707 708 static struct dm_arg _args[] = { 709 {0, 1024, "invalid number of path selector args"}, 710 }; 711 712 pst = dm_get_path_selector(dm_shift_arg(as)); 713 if (!pst) { 714 ti->error = "unknown path selector type"; 715 return -EINVAL; 716 } 717 718 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error); 719 if (r) { 720 dm_put_path_selector(pst); 721 return -EINVAL; 722 } 723 724 r = pst->create(&pg->ps, ps_argc, as->argv); 725 if (r) { 726 dm_put_path_selector(pst); 727 ti->error = "path selector constructor failed"; 728 return r; 729 } 730 731 pg->ps.type = pst; 732 dm_consume_args(as, ps_argc); 733 734 return 0; 735 } 736 737 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps, 738 struct dm_target *ti) 739 { 740 int r; 741 struct pgpath *p; 742 struct multipath *m = ti->private; 743 struct request_queue *q = NULL; 744 const char *attached_handler_name; 745 746 /* we need at least a path arg */ 747 if (as->argc < 1) { 748 ti->error = "no device given"; 749 return ERR_PTR(-EINVAL); 750 } 751 752 p = alloc_pgpath(); 753 if (!p) 754 return ERR_PTR(-ENOMEM); 755 756 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), 757 &p->path.dev); 758 if (r) { 759 ti->error = "error getting device"; 760 goto bad; 761 } 762 763 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name) 764 q = bdev_get_queue(p->path.dev->bdev); 765 766 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) { 767 retain: 768 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL); 769 if (attached_handler_name) { 770 /* 771 * Clear any hw_handler_params associated with a 772 * handler that isn't already attached. 773 */ 774 if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) { 775 kfree(m->hw_handler_params); 776 m->hw_handler_params = NULL; 777 } 778 779 /* 780 * Reset hw_handler_name to match the attached handler 781 * 782 * NB. This modifies the table line to show the actual 783 * handler instead of the original table passed in. 784 */ 785 kfree(m->hw_handler_name); 786 m->hw_handler_name = attached_handler_name; 787 } 788 } 789 790 if (m->hw_handler_name) { 791 r = scsi_dh_attach(q, m->hw_handler_name); 792 if (r == -EBUSY) { 793 char b[BDEVNAME_SIZE]; 794 795 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n", 796 bdevname(p->path.dev->bdev, b)); 797 goto retain; 798 } 799 if (r < 0) { 800 ti->error = "error attaching hardware handler"; 801 dm_put_device(ti, p->path.dev); 802 goto bad; 803 } 804 805 if (m->hw_handler_params) { 806 r = scsi_dh_set_params(q, m->hw_handler_params); 807 if (r < 0) { 808 ti->error = "unable to set hardware " 809 "handler parameters"; 810 dm_put_device(ti, p->path.dev); 811 goto bad; 812 } 813 } 814 } 815 816 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 817 if (r) { 818 dm_put_device(ti, p->path.dev); 819 goto bad; 820 } 821 822 return p; 823 824 bad: 825 free_pgpath(p); 826 return ERR_PTR(r); 827 } 828 829 static struct priority_group *parse_priority_group(struct dm_arg_set *as, 830 struct multipath *m) 831 { 832 static struct dm_arg _args[] = { 833 {1, 1024, "invalid number of paths"}, 834 {0, 1024, "invalid number of selector args"} 835 }; 836 837 int r; 838 unsigned i, nr_selector_args, nr_args; 839 struct priority_group *pg; 840 struct dm_target *ti = m->ti; 841 842 if (as->argc < 2) { 843 as->argc = 0; 844 ti->error = "not enough priority group arguments"; 845 return ERR_PTR(-EINVAL); 846 } 847 848 pg = alloc_priority_group(); 849 if (!pg) { 850 ti->error = "couldn't allocate priority group"; 851 return ERR_PTR(-ENOMEM); 852 } 853 pg->m = m; 854 855 r = parse_path_selector(as, pg, ti); 856 if (r) 857 goto bad; 858 859 /* 860 * read the paths 861 */ 862 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error); 863 if (r) 864 goto bad; 865 866 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error); 867 if (r) 868 goto bad; 869 870 nr_args = 1 + nr_selector_args; 871 for (i = 0; i < pg->nr_pgpaths; i++) { 872 struct pgpath *pgpath; 873 struct dm_arg_set path_args; 874 875 if (as->argc < nr_args) { 876 ti->error = "not enough path parameters"; 877 r = -EINVAL; 878 goto bad; 879 } 880 881 path_args.argc = nr_args; 882 path_args.argv = as->argv; 883 884 pgpath = parse_path(&path_args, &pg->ps, ti); 885 if (IS_ERR(pgpath)) { 886 r = PTR_ERR(pgpath); 887 goto bad; 888 } 889 890 pgpath->pg = pg; 891 list_add_tail(&pgpath->list, &pg->pgpaths); 892 dm_consume_args(as, nr_args); 893 } 894 895 return pg; 896 897 bad: 898 free_priority_group(pg, ti); 899 return ERR_PTR(r); 900 } 901 902 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) 903 { 904 unsigned hw_argc; 905 int ret; 906 struct dm_target *ti = m->ti; 907 908 static struct dm_arg _args[] = { 909 {0, 1024, "invalid number of hardware handler args"}, 910 }; 911 912 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error)) 913 return -EINVAL; 914 915 if (!hw_argc) 916 return 0; 917 918 if (m->queue_mode == DM_TYPE_BIO_BASED) { 919 dm_consume_args(as, hw_argc); 920 DMERR("bio-based multipath doesn't allow hardware handler args"); 921 return 0; 922 } 923 924 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); 925 if (!m->hw_handler_name) 926 return -EINVAL; 927 928 if (hw_argc > 1) { 929 char *p; 930 int i, j, len = 4; 931 932 for (i = 0; i <= hw_argc - 2; i++) 933 len += strlen(as->argv[i]) + 1; 934 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); 935 if (!p) { 936 ti->error = "memory allocation failed"; 937 ret = -ENOMEM; 938 goto fail; 939 } 940 j = sprintf(p, "%d", hw_argc - 1); 941 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1) 942 j = sprintf(p, "%s", as->argv[i]); 943 } 944 dm_consume_args(as, hw_argc - 1); 945 946 return 0; 947 fail: 948 kfree(m->hw_handler_name); 949 m->hw_handler_name = NULL; 950 return ret; 951 } 952 953 static int parse_features(struct dm_arg_set *as, struct multipath *m) 954 { 955 int r; 956 unsigned argc; 957 struct dm_target *ti = m->ti; 958 const char *arg_name; 959 960 static struct dm_arg _args[] = { 961 {0, 8, "invalid number of feature args"}, 962 {1, 50, "pg_init_retries must be between 1 and 50"}, 963 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, 964 }; 965 966 r = dm_read_arg_group(_args, as, &argc, &ti->error); 967 if (r) 968 return -EINVAL; 969 970 if (!argc) 971 return 0; 972 973 do { 974 arg_name = dm_shift_arg(as); 975 argc--; 976 977 if (!strcasecmp(arg_name, "queue_if_no_path")) { 978 r = queue_if_no_path(m, true, false); 979 continue; 980 } 981 982 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) { 983 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 984 continue; 985 } 986 987 if (!strcasecmp(arg_name, "pg_init_retries") && 988 (argc >= 1)) { 989 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error); 990 argc--; 991 continue; 992 } 993 994 if (!strcasecmp(arg_name, "pg_init_delay_msecs") && 995 (argc >= 1)) { 996 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error); 997 argc--; 998 continue; 999 } 1000 1001 if (!strcasecmp(arg_name, "queue_mode") && 1002 (argc >= 1)) { 1003 const char *queue_mode_name = dm_shift_arg(as); 1004 1005 if (!strcasecmp(queue_mode_name, "bio")) 1006 m->queue_mode = DM_TYPE_BIO_BASED; 1007 else if (!strcasecmp(queue_mode_name, "rq")) 1008 m->queue_mode = DM_TYPE_REQUEST_BASED; 1009 else if (!strcasecmp(queue_mode_name, "mq")) 1010 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 1011 else { 1012 ti->error = "Unknown 'queue_mode' requested"; 1013 r = -EINVAL; 1014 } 1015 argc--; 1016 continue; 1017 } 1018 1019 ti->error = "Unrecognised multipath feature request"; 1020 r = -EINVAL; 1021 } while (argc && !r); 1022 1023 return r; 1024 } 1025 1026 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) 1027 { 1028 /* target arguments */ 1029 static struct dm_arg _args[] = { 1030 {0, 1024, "invalid number of priority groups"}, 1031 {0, 1024, "invalid initial priority group number"}, 1032 }; 1033 1034 int r; 1035 struct multipath *m; 1036 struct dm_arg_set as; 1037 unsigned pg_count = 0; 1038 unsigned next_pg_num; 1039 1040 as.argc = argc; 1041 as.argv = argv; 1042 1043 m = alloc_multipath(ti); 1044 if (!m) { 1045 ti->error = "can't allocate multipath"; 1046 return -EINVAL; 1047 } 1048 1049 r = parse_features(&as, m); 1050 if (r) 1051 goto bad; 1052 1053 r = alloc_multipath_stage2(ti, m); 1054 if (r) 1055 goto bad; 1056 1057 r = parse_hw_handler(&as, m); 1058 if (r) 1059 goto bad; 1060 1061 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error); 1062 if (r) 1063 goto bad; 1064 1065 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error); 1066 if (r) 1067 goto bad; 1068 1069 if ((!m->nr_priority_groups && next_pg_num) || 1070 (m->nr_priority_groups && !next_pg_num)) { 1071 ti->error = "invalid initial priority group"; 1072 r = -EINVAL; 1073 goto bad; 1074 } 1075 1076 /* parse the priority groups */ 1077 while (as.argc) { 1078 struct priority_group *pg; 1079 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths); 1080 1081 pg = parse_priority_group(&as, m); 1082 if (IS_ERR(pg)) { 1083 r = PTR_ERR(pg); 1084 goto bad; 1085 } 1086 1087 nr_valid_paths += pg->nr_pgpaths; 1088 atomic_set(&m->nr_valid_paths, nr_valid_paths); 1089 1090 list_add_tail(&pg->list, &m->priority_groups); 1091 pg_count++; 1092 pg->pg_num = pg_count; 1093 if (!--next_pg_num) 1094 m->next_pg = pg; 1095 } 1096 1097 if (pg_count != m->nr_priority_groups) { 1098 ti->error = "priority group count mismatch"; 1099 r = -EINVAL; 1100 goto bad; 1101 } 1102 1103 ti->num_flush_bios = 1; 1104 ti->num_discard_bios = 1; 1105 ti->num_write_same_bios = 1; 1106 if (m->queue_mode == DM_TYPE_BIO_BASED) 1107 ti->per_io_data_size = multipath_per_bio_data_size(); 1108 else 1109 ti->per_io_data_size = sizeof(struct dm_mpath_io); 1110 1111 return 0; 1112 1113 bad: 1114 free_multipath(m); 1115 return r; 1116 } 1117 1118 static void multipath_wait_for_pg_init_completion(struct multipath *m) 1119 { 1120 DEFINE_WAIT(wait); 1121 1122 while (1) { 1123 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE); 1124 1125 if (!atomic_read(&m->pg_init_in_progress)) 1126 break; 1127 1128 io_schedule(); 1129 } 1130 finish_wait(&m->pg_init_wait, &wait); 1131 } 1132 1133 static void flush_multipath_work(struct multipath *m) 1134 { 1135 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1136 smp_mb__after_atomic(); 1137 1138 flush_workqueue(kmpath_handlerd); 1139 multipath_wait_for_pg_init_completion(m); 1140 flush_workqueue(kmultipathd); 1141 flush_work(&m->trigger_event); 1142 1143 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1144 smp_mb__after_atomic(); 1145 } 1146 1147 static void multipath_dtr(struct dm_target *ti) 1148 { 1149 struct multipath *m = ti->private; 1150 1151 flush_multipath_work(m); 1152 free_multipath(m); 1153 } 1154 1155 /* 1156 * Take a path out of use. 1157 */ 1158 static int fail_path(struct pgpath *pgpath) 1159 { 1160 unsigned long flags; 1161 struct multipath *m = pgpath->pg->m; 1162 1163 spin_lock_irqsave(&m->lock, flags); 1164 1165 if (!pgpath->is_active) 1166 goto out; 1167 1168 DMWARN("Failing path %s.", pgpath->path.dev->name); 1169 1170 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 1171 pgpath->is_active = false; 1172 pgpath->fail_count++; 1173 1174 atomic_dec(&m->nr_valid_paths); 1175 1176 if (pgpath == m->current_pgpath) 1177 m->current_pgpath = NULL; 1178 1179 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 1180 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths)); 1181 1182 schedule_work(&m->trigger_event); 1183 1184 out: 1185 spin_unlock_irqrestore(&m->lock, flags); 1186 1187 return 0; 1188 } 1189 1190 /* 1191 * Reinstate a previously-failed path 1192 */ 1193 static int reinstate_path(struct pgpath *pgpath) 1194 { 1195 int r = 0, run_queue = 0; 1196 unsigned long flags; 1197 struct multipath *m = pgpath->pg->m; 1198 unsigned nr_valid_paths; 1199 1200 spin_lock_irqsave(&m->lock, flags); 1201 1202 if (pgpath->is_active) 1203 goto out; 1204 1205 DMWARN("Reinstating path %s.", pgpath->path.dev->name); 1206 1207 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 1208 if (r) 1209 goto out; 1210 1211 pgpath->is_active = true; 1212 1213 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths); 1214 if (nr_valid_paths == 1) { 1215 m->current_pgpath = NULL; 1216 run_queue = 1; 1217 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) { 1218 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work)) 1219 atomic_inc(&m->pg_init_in_progress); 1220 } 1221 1222 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 1223 pgpath->path.dev->name, nr_valid_paths); 1224 1225 schedule_work(&m->trigger_event); 1226 1227 out: 1228 spin_unlock_irqrestore(&m->lock, flags); 1229 if (run_queue) { 1230 dm_table_run_md_queue_async(m->ti->table); 1231 process_queued_io_list(m); 1232 } 1233 1234 return r; 1235 } 1236 1237 /* 1238 * Fail or reinstate all paths that match the provided struct dm_dev. 1239 */ 1240 static int action_dev(struct multipath *m, struct dm_dev *dev, 1241 action_fn action) 1242 { 1243 int r = -EINVAL; 1244 struct pgpath *pgpath; 1245 struct priority_group *pg; 1246 1247 list_for_each_entry(pg, &m->priority_groups, list) { 1248 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1249 if (pgpath->path.dev == dev) 1250 r = action(pgpath); 1251 } 1252 } 1253 1254 return r; 1255 } 1256 1257 /* 1258 * Temporarily try to avoid having to use the specified PG 1259 */ 1260 static void bypass_pg(struct multipath *m, struct priority_group *pg, 1261 bool bypassed) 1262 { 1263 unsigned long flags; 1264 1265 spin_lock_irqsave(&m->lock, flags); 1266 1267 pg->bypassed = bypassed; 1268 m->current_pgpath = NULL; 1269 m->current_pg = NULL; 1270 1271 spin_unlock_irqrestore(&m->lock, flags); 1272 1273 schedule_work(&m->trigger_event); 1274 } 1275 1276 /* 1277 * Switch to using the specified PG from the next I/O that gets mapped 1278 */ 1279 static int switch_pg_num(struct multipath *m, const char *pgstr) 1280 { 1281 struct priority_group *pg; 1282 unsigned pgnum; 1283 unsigned long flags; 1284 char dummy; 1285 1286 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1287 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1288 DMWARN("invalid PG number supplied to switch_pg_num"); 1289 return -EINVAL; 1290 } 1291 1292 spin_lock_irqsave(&m->lock, flags); 1293 list_for_each_entry(pg, &m->priority_groups, list) { 1294 pg->bypassed = false; 1295 if (--pgnum) 1296 continue; 1297 1298 m->current_pgpath = NULL; 1299 m->current_pg = NULL; 1300 m->next_pg = pg; 1301 } 1302 spin_unlock_irqrestore(&m->lock, flags); 1303 1304 schedule_work(&m->trigger_event); 1305 return 0; 1306 } 1307 1308 /* 1309 * Set/clear bypassed status of a PG. 1310 * PGs are numbered upwards from 1 in the order they were declared. 1311 */ 1312 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed) 1313 { 1314 struct priority_group *pg; 1315 unsigned pgnum; 1316 char dummy; 1317 1318 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1319 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1320 DMWARN("invalid PG number supplied to bypass_pg"); 1321 return -EINVAL; 1322 } 1323 1324 list_for_each_entry(pg, &m->priority_groups, list) { 1325 if (!--pgnum) 1326 break; 1327 } 1328 1329 bypass_pg(m, pg, bypassed); 1330 return 0; 1331 } 1332 1333 /* 1334 * Should we retry pg_init immediately? 1335 */ 1336 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1337 { 1338 unsigned long flags; 1339 bool limit_reached = false; 1340 1341 spin_lock_irqsave(&m->lock, flags); 1342 1343 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries && 1344 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 1345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 1346 else 1347 limit_reached = true; 1348 1349 spin_unlock_irqrestore(&m->lock, flags); 1350 1351 return limit_reached; 1352 } 1353 1354 static void pg_init_done(void *data, int errors) 1355 { 1356 struct pgpath *pgpath = data; 1357 struct priority_group *pg = pgpath->pg; 1358 struct multipath *m = pg->m; 1359 unsigned long flags; 1360 bool delay_retry = false; 1361 1362 /* device or driver problems */ 1363 switch (errors) { 1364 case SCSI_DH_OK: 1365 break; 1366 case SCSI_DH_NOSYS: 1367 if (!m->hw_handler_name) { 1368 errors = 0; 1369 break; 1370 } 1371 DMERR("Could not failover the device: Handler scsi_dh_%s " 1372 "Error %d.", m->hw_handler_name, errors); 1373 /* 1374 * Fail path for now, so we do not ping pong 1375 */ 1376 fail_path(pgpath); 1377 break; 1378 case SCSI_DH_DEV_TEMP_BUSY: 1379 /* 1380 * Probably doing something like FW upgrade on the 1381 * controller so try the other pg. 1382 */ 1383 bypass_pg(m, pg, true); 1384 break; 1385 case SCSI_DH_RETRY: 1386 /* Wait before retrying. */ 1387 delay_retry = 1; 1388 case SCSI_DH_IMM_RETRY: 1389 case SCSI_DH_RES_TEMP_UNAVAIL: 1390 if (pg_init_limit_reached(m, pgpath)) 1391 fail_path(pgpath); 1392 errors = 0; 1393 break; 1394 case SCSI_DH_DEV_OFFLINED: 1395 default: 1396 /* 1397 * We probably do not want to fail the path for a device 1398 * error, but this is what the old dm did. In future 1399 * patches we can do more advanced handling. 1400 */ 1401 fail_path(pgpath); 1402 } 1403 1404 spin_lock_irqsave(&m->lock, flags); 1405 if (errors) { 1406 if (pgpath == m->current_pgpath) { 1407 DMERR("Could not failover device. Error %d.", errors); 1408 m->current_pgpath = NULL; 1409 m->current_pg = NULL; 1410 } 1411 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1412 pg->bypassed = false; 1413 1414 if (atomic_dec_return(&m->pg_init_in_progress) > 0) 1415 /* Activations of other paths are still on going */ 1416 goto out; 1417 1418 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 1419 if (delay_retry) 1420 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1421 else 1422 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1423 1424 if (__pg_init_all_paths(m)) 1425 goto out; 1426 } 1427 clear_bit(MPATHF_QUEUE_IO, &m->flags); 1428 1429 process_queued_io_list(m); 1430 1431 /* 1432 * Wake up any thread waiting to suspend. 1433 */ 1434 wake_up(&m->pg_init_wait); 1435 1436 out: 1437 spin_unlock_irqrestore(&m->lock, flags); 1438 } 1439 1440 static void activate_path(struct work_struct *work) 1441 { 1442 struct pgpath *pgpath = 1443 container_of(work, struct pgpath, activate_path.work); 1444 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1445 1446 if (pgpath->is_active && !blk_queue_dying(q)) 1447 scsi_dh_activate(q, pg_init_done, pgpath); 1448 else 1449 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED); 1450 } 1451 1452 static int noretry_error(int error) 1453 { 1454 switch (error) { 1455 case -EBADE: 1456 /* 1457 * EBADE signals an reservation conflict. 1458 * We shouldn't fail the path here as we can communicate with 1459 * the target. We should failover to the next path, but in 1460 * doing so we might be causing a ping-pong between paths. 1461 * So just return the reservation conflict error. 1462 */ 1463 case -EOPNOTSUPP: 1464 case -EREMOTEIO: 1465 case -EILSEQ: 1466 case -ENODATA: 1467 case -ENOSPC: 1468 return 1; 1469 } 1470 1471 /* Anything else could be a path failure, so should be retried */ 1472 return 0; 1473 } 1474 1475 /* 1476 * end_io handling 1477 */ 1478 static int do_end_io(struct multipath *m, struct request *clone, 1479 int error, struct dm_mpath_io *mpio) 1480 { 1481 /* 1482 * We don't queue any clone request inside the multipath target 1483 * during end I/O handling, since those clone requests don't have 1484 * bio clones. If we queue them inside the multipath target, 1485 * we need to make bio clones, that requires memory allocation. 1486 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests 1487 * don't have bio clones.) 1488 * Instead of queueing the clone request here, we queue the original 1489 * request into dm core, which will remake a clone request and 1490 * clone bios for it and resubmit it later. 1491 */ 1492 int r = DM_ENDIO_REQUEUE; 1493 1494 if (!error && !clone->errors) 1495 return 0; /* I/O complete */ 1496 1497 if (noretry_error(error)) 1498 return error; 1499 1500 if (mpio->pgpath) 1501 fail_path(mpio->pgpath); 1502 1503 if (!atomic_read(&m->nr_valid_paths)) { 1504 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1505 if (!must_push_back_rq(m)) 1506 r = -EIO; 1507 } 1508 } 1509 1510 return r; 1511 } 1512 1513 static int multipath_end_io(struct dm_target *ti, struct request *clone, 1514 int error, union map_info *map_context) 1515 { 1516 struct multipath *m = ti->private; 1517 struct dm_mpath_io *mpio = get_mpio(map_context); 1518 struct pgpath *pgpath; 1519 struct path_selector *ps; 1520 int r; 1521 1522 BUG_ON(!mpio); 1523 1524 r = do_end_io(m, clone, error, mpio); 1525 pgpath = mpio->pgpath; 1526 if (pgpath) { 1527 ps = &pgpath->pg->ps; 1528 if (ps->type->end_io) 1529 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1530 } 1531 1532 return r; 1533 } 1534 1535 static int do_end_io_bio(struct multipath *m, struct bio *clone, 1536 int error, struct dm_mpath_io *mpio) 1537 { 1538 unsigned long flags; 1539 1540 if (!error) 1541 return 0; /* I/O complete */ 1542 1543 if (noretry_error(error)) 1544 return error; 1545 1546 if (mpio->pgpath) 1547 fail_path(mpio->pgpath); 1548 1549 if (!atomic_read(&m->nr_valid_paths)) { 1550 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1551 if (!must_push_back_bio(m)) 1552 return -EIO; 1553 return DM_ENDIO_REQUEUE; 1554 } 1555 } 1556 1557 /* Queue for the daemon to resubmit */ 1558 dm_bio_restore(get_bio_details_from_bio(clone), clone); 1559 1560 spin_lock_irqsave(&m->lock, flags); 1561 bio_list_add(&m->queued_bios, clone); 1562 spin_unlock_irqrestore(&m->lock, flags); 1563 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) 1564 queue_work(kmultipathd, &m->process_queued_bios); 1565 1566 return DM_ENDIO_INCOMPLETE; 1567 } 1568 1569 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, int error) 1570 { 1571 struct multipath *m = ti->private; 1572 struct dm_mpath_io *mpio = get_mpio_from_bio(clone); 1573 struct pgpath *pgpath; 1574 struct path_selector *ps; 1575 int r; 1576 1577 BUG_ON(!mpio); 1578 1579 r = do_end_io_bio(m, clone, error, mpio); 1580 pgpath = mpio->pgpath; 1581 if (pgpath) { 1582 ps = &pgpath->pg->ps; 1583 if (ps->type->end_io) 1584 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1585 } 1586 1587 return r; 1588 } 1589 1590 /* 1591 * Suspend can't complete until all the I/O is processed so if 1592 * the last path fails we must error any remaining I/O. 1593 * Note that if the freeze_bdev fails while suspending, the 1594 * queue_if_no_path state is lost - userspace should reset it. 1595 */ 1596 static void multipath_presuspend(struct dm_target *ti) 1597 { 1598 struct multipath *m = ti->private; 1599 1600 queue_if_no_path(m, false, true); 1601 } 1602 1603 static void multipath_postsuspend(struct dm_target *ti) 1604 { 1605 struct multipath *m = ti->private; 1606 1607 mutex_lock(&m->work_mutex); 1608 flush_multipath_work(m); 1609 mutex_unlock(&m->work_mutex); 1610 } 1611 1612 /* 1613 * Restore the queue_if_no_path setting. 1614 */ 1615 static void multipath_resume(struct dm_target *ti) 1616 { 1617 struct multipath *m = ti->private; 1618 unsigned long flags; 1619 1620 spin_lock_irqsave(&m->lock, flags); 1621 if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) 1622 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 1623 else 1624 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags); 1625 spin_unlock_irqrestore(&m->lock, flags); 1626 } 1627 1628 /* 1629 * Info output has the following format: 1630 * num_multipath_feature_args [multipath_feature_args]* 1631 * num_handler_status_args [handler_status_args]* 1632 * num_groups init_group_number 1633 * [A|D|E num_ps_status_args [ps_status_args]* 1634 * num_paths num_selector_args 1635 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1636 * 1637 * Table output has the following format (identical to the constructor string): 1638 * num_feature_args [features_args]* 1639 * num_handler_args hw_handler [hw_handler_args]* 1640 * num_groups init_group_number 1641 * [priority selector-name num_ps_args [ps_args]* 1642 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1643 */ 1644 static void multipath_status(struct dm_target *ti, status_type_t type, 1645 unsigned status_flags, char *result, unsigned maxlen) 1646 { 1647 int sz = 0; 1648 unsigned long flags; 1649 struct multipath *m = ti->private; 1650 struct priority_group *pg; 1651 struct pgpath *p; 1652 unsigned pg_num; 1653 char state; 1654 1655 spin_lock_irqsave(&m->lock, flags); 1656 1657 /* Features */ 1658 if (type == STATUSTYPE_INFO) 1659 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags), 1660 atomic_read(&m->pg_init_count)); 1661 else { 1662 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) + 1663 (m->pg_init_retries > 0) * 2 + 1664 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + 1665 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) + 1666 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2); 1667 1668 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1669 DMEMIT("queue_if_no_path "); 1670 if (m->pg_init_retries) 1671 DMEMIT("pg_init_retries %u ", m->pg_init_retries); 1672 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) 1673 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); 1674 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) 1675 DMEMIT("retain_attached_hw_handler "); 1676 if (m->queue_mode != DM_TYPE_REQUEST_BASED) { 1677 switch(m->queue_mode) { 1678 case DM_TYPE_BIO_BASED: 1679 DMEMIT("queue_mode bio "); 1680 break; 1681 case DM_TYPE_MQ_REQUEST_BASED: 1682 DMEMIT("queue_mode mq "); 1683 break; 1684 } 1685 } 1686 } 1687 1688 if (!m->hw_handler_name || type == STATUSTYPE_INFO) 1689 DMEMIT("0 "); 1690 else 1691 DMEMIT("1 %s ", m->hw_handler_name); 1692 1693 DMEMIT("%u ", m->nr_priority_groups); 1694 1695 if (m->next_pg) 1696 pg_num = m->next_pg->pg_num; 1697 else if (m->current_pg) 1698 pg_num = m->current_pg->pg_num; 1699 else 1700 pg_num = (m->nr_priority_groups ? 1 : 0); 1701 1702 DMEMIT("%u ", pg_num); 1703 1704 switch (type) { 1705 case STATUSTYPE_INFO: 1706 list_for_each_entry(pg, &m->priority_groups, list) { 1707 if (pg->bypassed) 1708 state = 'D'; /* Disabled */ 1709 else if (pg == m->current_pg) 1710 state = 'A'; /* Currently Active */ 1711 else 1712 state = 'E'; /* Enabled */ 1713 1714 DMEMIT("%c ", state); 1715 1716 if (pg->ps.type->status) 1717 sz += pg->ps.type->status(&pg->ps, NULL, type, 1718 result + sz, 1719 maxlen - sz); 1720 else 1721 DMEMIT("0 "); 1722 1723 DMEMIT("%u %u ", pg->nr_pgpaths, 1724 pg->ps.type->info_args); 1725 1726 list_for_each_entry(p, &pg->pgpaths, list) { 1727 DMEMIT("%s %s %u ", p->path.dev->name, 1728 p->is_active ? "A" : "F", 1729 p->fail_count); 1730 if (pg->ps.type->status) 1731 sz += pg->ps.type->status(&pg->ps, 1732 &p->path, type, result + sz, 1733 maxlen - sz); 1734 } 1735 } 1736 break; 1737 1738 case STATUSTYPE_TABLE: 1739 list_for_each_entry(pg, &m->priority_groups, list) { 1740 DMEMIT("%s ", pg->ps.type->name); 1741 1742 if (pg->ps.type->status) 1743 sz += pg->ps.type->status(&pg->ps, NULL, type, 1744 result + sz, 1745 maxlen - sz); 1746 else 1747 DMEMIT("0 "); 1748 1749 DMEMIT("%u %u ", pg->nr_pgpaths, 1750 pg->ps.type->table_args); 1751 1752 list_for_each_entry(p, &pg->pgpaths, list) { 1753 DMEMIT("%s ", p->path.dev->name); 1754 if (pg->ps.type->status) 1755 sz += pg->ps.type->status(&pg->ps, 1756 &p->path, type, result + sz, 1757 maxlen - sz); 1758 } 1759 } 1760 break; 1761 } 1762 1763 spin_unlock_irqrestore(&m->lock, flags); 1764 } 1765 1766 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 1767 { 1768 int r = -EINVAL; 1769 struct dm_dev *dev; 1770 struct multipath *m = ti->private; 1771 action_fn action; 1772 1773 mutex_lock(&m->work_mutex); 1774 1775 if (dm_suspended(ti)) { 1776 r = -EBUSY; 1777 goto out; 1778 } 1779 1780 if (argc == 1) { 1781 if (!strcasecmp(argv[0], "queue_if_no_path")) { 1782 r = queue_if_no_path(m, true, false); 1783 goto out; 1784 } else if (!strcasecmp(argv[0], "fail_if_no_path")) { 1785 r = queue_if_no_path(m, false, false); 1786 goto out; 1787 } 1788 } 1789 1790 if (argc != 2) { 1791 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc); 1792 goto out; 1793 } 1794 1795 if (!strcasecmp(argv[0], "disable_group")) { 1796 r = bypass_pg_num(m, argv[1], true); 1797 goto out; 1798 } else if (!strcasecmp(argv[0], "enable_group")) { 1799 r = bypass_pg_num(m, argv[1], false); 1800 goto out; 1801 } else if (!strcasecmp(argv[0], "switch_group")) { 1802 r = switch_pg_num(m, argv[1]); 1803 goto out; 1804 } else if (!strcasecmp(argv[0], "reinstate_path")) 1805 action = reinstate_path; 1806 else if (!strcasecmp(argv[0], "fail_path")) 1807 action = fail_path; 1808 else { 1809 DMWARN("Unrecognised multipath message received: %s", argv[0]); 1810 goto out; 1811 } 1812 1813 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev); 1814 if (r) { 1815 DMWARN("message: error getting device %s", 1816 argv[1]); 1817 goto out; 1818 } 1819 1820 r = action_dev(m, dev, action); 1821 1822 dm_put_device(ti, dev); 1823 1824 out: 1825 mutex_unlock(&m->work_mutex); 1826 return r; 1827 } 1828 1829 static int multipath_prepare_ioctl(struct dm_target *ti, 1830 struct block_device **bdev, fmode_t *mode) 1831 { 1832 struct multipath *m = ti->private; 1833 struct pgpath *current_pgpath; 1834 int r; 1835 1836 current_pgpath = lockless_dereference(m->current_pgpath); 1837 if (!current_pgpath) 1838 current_pgpath = choose_pgpath(m, 0); 1839 1840 if (current_pgpath) { 1841 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) { 1842 *bdev = current_pgpath->path.dev->bdev; 1843 *mode = current_pgpath->path.dev->mode; 1844 r = 0; 1845 } else { 1846 /* pg_init has not started or completed */ 1847 r = -ENOTCONN; 1848 } 1849 } else { 1850 /* No path is available */ 1851 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1852 r = -ENOTCONN; 1853 else 1854 r = -EIO; 1855 } 1856 1857 if (r == -ENOTCONN) { 1858 if (!lockless_dereference(m->current_pg)) { 1859 /* Path status changed, redo selection */ 1860 (void) choose_pgpath(m, 0); 1861 } 1862 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1863 pg_init_all_paths(m); 1864 dm_table_run_md_queue_async(m->ti->table); 1865 process_queued_io_list(m); 1866 } 1867 1868 /* 1869 * Only pass ioctls through if the device sizes match exactly. 1870 */ 1871 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) 1872 return 1; 1873 return r; 1874 } 1875 1876 static int multipath_iterate_devices(struct dm_target *ti, 1877 iterate_devices_callout_fn fn, void *data) 1878 { 1879 struct multipath *m = ti->private; 1880 struct priority_group *pg; 1881 struct pgpath *p; 1882 int ret = 0; 1883 1884 list_for_each_entry(pg, &m->priority_groups, list) { 1885 list_for_each_entry(p, &pg->pgpaths, list) { 1886 ret = fn(ti, p->path.dev, ti->begin, ti->len, data); 1887 if (ret) 1888 goto out; 1889 } 1890 } 1891 1892 out: 1893 return ret; 1894 } 1895 1896 static int pgpath_busy(struct pgpath *pgpath) 1897 { 1898 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1899 1900 return blk_lld_busy(q); 1901 } 1902 1903 /* 1904 * We return "busy", only when we can map I/Os but underlying devices 1905 * are busy (so even if we map I/Os now, the I/Os will wait on 1906 * the underlying queue). 1907 * In other words, if we want to kill I/Os or queue them inside us 1908 * due to map unavailability, we don't return "busy". Otherwise, 1909 * dm core won't give us the I/Os and we can't do what we want. 1910 */ 1911 static int multipath_busy(struct dm_target *ti) 1912 { 1913 bool busy = false, has_active = false; 1914 struct multipath *m = ti->private; 1915 struct priority_group *pg, *next_pg; 1916 struct pgpath *pgpath; 1917 1918 /* pg_init in progress */ 1919 if (atomic_read(&m->pg_init_in_progress)) 1920 return true; 1921 1922 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */ 1923 if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1924 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED); 1925 1926 /* Guess which priority_group will be used at next mapping time */ 1927 pg = lockless_dereference(m->current_pg); 1928 next_pg = lockless_dereference(m->next_pg); 1929 if (unlikely(!lockless_dereference(m->current_pgpath) && next_pg)) 1930 pg = next_pg; 1931 1932 if (!pg) { 1933 /* 1934 * We don't know which pg will be used at next mapping time. 1935 * We don't call choose_pgpath() here to avoid to trigger 1936 * pg_init just by busy checking. 1937 * So we don't know whether underlying devices we will be using 1938 * at next mapping time are busy or not. Just try mapping. 1939 */ 1940 return busy; 1941 } 1942 1943 /* 1944 * If there is one non-busy active path at least, the path selector 1945 * will be able to select it. So we consider such a pg as not busy. 1946 */ 1947 busy = true; 1948 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1949 if (pgpath->is_active) { 1950 has_active = true; 1951 if (!pgpath_busy(pgpath)) { 1952 busy = false; 1953 break; 1954 } 1955 } 1956 } 1957 1958 if (!has_active) { 1959 /* 1960 * No active path in this pg, so this pg won't be used and 1961 * the current_pg will be changed at next mapping time. 1962 * We need to try mapping to determine it. 1963 */ 1964 busy = false; 1965 } 1966 1967 return busy; 1968 } 1969 1970 /*----------------------------------------------------------------- 1971 * Module setup 1972 *---------------------------------------------------------------*/ 1973 static struct target_type multipath_target = { 1974 .name = "multipath", 1975 .version = {1, 12, 0}, 1976 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, 1977 .module = THIS_MODULE, 1978 .ctr = multipath_ctr, 1979 .dtr = multipath_dtr, 1980 .clone_and_map_rq = multipath_clone_and_map, 1981 .release_clone_rq = multipath_release_clone, 1982 .rq_end_io = multipath_end_io, 1983 .map = multipath_map_bio, 1984 .end_io = multipath_end_io_bio, 1985 .presuspend = multipath_presuspend, 1986 .postsuspend = multipath_postsuspend, 1987 .resume = multipath_resume, 1988 .status = multipath_status, 1989 .message = multipath_message, 1990 .prepare_ioctl = multipath_prepare_ioctl, 1991 .iterate_devices = multipath_iterate_devices, 1992 .busy = multipath_busy, 1993 }; 1994 1995 static int __init dm_multipath_init(void) 1996 { 1997 int r; 1998 1999 r = dm_register_target(&multipath_target); 2000 if (r < 0) { 2001 DMERR("request-based register failed %d", r); 2002 r = -EINVAL; 2003 goto bad_register_target; 2004 } 2005 2006 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 2007 if (!kmultipathd) { 2008 DMERR("failed to create workqueue kmpathd"); 2009 r = -ENOMEM; 2010 goto bad_alloc_kmultipathd; 2011 } 2012 2013 /* 2014 * A separate workqueue is used to handle the device handlers 2015 * to avoid overloading existing workqueue. Overloading the 2016 * old workqueue would also create a bottleneck in the 2017 * path of the storage hardware device activation. 2018 */ 2019 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd", 2020 WQ_MEM_RECLAIM); 2021 if (!kmpath_handlerd) { 2022 DMERR("failed to create workqueue kmpath_handlerd"); 2023 r = -ENOMEM; 2024 goto bad_alloc_kmpath_handlerd; 2025 } 2026 2027 return 0; 2028 2029 bad_alloc_kmpath_handlerd: 2030 destroy_workqueue(kmultipathd); 2031 bad_alloc_kmultipathd: 2032 dm_unregister_target(&multipath_target); 2033 bad_register_target: 2034 return r; 2035 } 2036 2037 static void __exit dm_multipath_exit(void) 2038 { 2039 destroy_workqueue(kmpath_handlerd); 2040 destroy_workqueue(kmultipathd); 2041 2042 dm_unregister_target(&multipath_target); 2043 } 2044 2045 module_init(dm_multipath_init); 2046 module_exit(dm_multipath_exit); 2047 2048 MODULE_DESCRIPTION(DM_NAME " multipath target"); 2049 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 2050 MODULE_LICENSE("GPL"); 2051