1 /* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 10 #include "dm-rq.h" 11 #include "dm-bio-record.h" 12 #include "dm-path-selector.h" 13 #include "dm-uevent.h" 14 15 #include <linux/blkdev.h> 16 #include <linux/ctype.h> 17 #include <linux/init.h> 18 #include <linux/mempool.h> 19 #include <linux/module.h> 20 #include <linux/pagemap.h> 21 #include <linux/slab.h> 22 #include <linux/time.h> 23 #include <linux/workqueue.h> 24 #include <linux/delay.h> 25 #include <scsi/scsi_dh.h> 26 #include <linux/atomic.h> 27 #include <linux/blk-mq.h> 28 29 #define DM_MSG_PREFIX "multipath" 30 #define DM_PG_INIT_DELAY_MSECS 2000 31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1) 32 33 /* Path properties */ 34 struct pgpath { 35 struct list_head list; 36 37 struct priority_group *pg; /* Owning PG */ 38 unsigned fail_count; /* Cumulative failure count */ 39 40 struct dm_path path; 41 struct delayed_work activate_path; 42 43 bool is_active:1; /* Path status */ 44 }; 45 46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 47 48 /* 49 * Paths are grouped into Priority Groups and numbered from 1 upwards. 50 * Each has a path selector which controls which path gets used. 51 */ 52 struct priority_group { 53 struct list_head list; 54 55 struct multipath *m; /* Owning multipath instance */ 56 struct path_selector ps; 57 58 unsigned pg_num; /* Reference number */ 59 unsigned nr_pgpaths; /* Number of paths in PG */ 60 struct list_head pgpaths; 61 62 bool bypassed:1; /* Temporarily bypass this PG? */ 63 }; 64 65 /* Multipath context */ 66 struct multipath { 67 struct list_head list; 68 struct dm_target *ti; 69 70 const char *hw_handler_name; 71 char *hw_handler_params; 72 73 spinlock_t lock; 74 75 unsigned nr_priority_groups; 76 struct list_head priority_groups; 77 78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */ 79 80 struct pgpath *current_pgpath; 81 struct priority_group *current_pg; 82 struct priority_group *next_pg; /* Switch to this PG if set */ 83 84 unsigned long flags; /* Multipath state flags */ 85 86 unsigned pg_init_retries; /* Number of times to retry pg_init */ 87 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */ 88 89 atomic_t nr_valid_paths; /* Total number of usable paths */ 90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */ 91 atomic_t pg_init_count; /* Number of times pg_init called */ 92 93 enum dm_queue_mode queue_mode; 94 95 struct mutex work_mutex; 96 struct work_struct trigger_event; 97 98 struct work_struct process_queued_bios; 99 struct bio_list queued_bios; 100 }; 101 102 /* 103 * Context information attached to each io we process. 104 */ 105 struct dm_mpath_io { 106 struct pgpath *pgpath; 107 size_t nr_bytes; 108 }; 109 110 typedef int (*action_fn) (struct pgpath *pgpath); 111 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 113 static void trigger_event(struct work_struct *work); 114 static void activate_or_offline_path(struct pgpath *pgpath); 115 static void activate_path_work(struct work_struct *work); 116 static void process_queued_bios(struct work_struct *work); 117 118 /*----------------------------------------------- 119 * Multipath state flags. 120 *-----------------------------------------------*/ 121 122 #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */ 123 #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */ 124 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */ 125 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */ 126 #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */ 127 #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */ 128 #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */ 129 130 /*----------------------------------------------- 131 * Allocation routines 132 *-----------------------------------------------*/ 133 134 static struct pgpath *alloc_pgpath(void) 135 { 136 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 137 138 if (pgpath) { 139 pgpath->is_active = true; 140 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path_work); 141 } 142 143 return pgpath; 144 } 145 146 static void free_pgpath(struct pgpath *pgpath) 147 { 148 kfree(pgpath); 149 } 150 151 static struct priority_group *alloc_priority_group(void) 152 { 153 struct priority_group *pg; 154 155 pg = kzalloc(sizeof(*pg), GFP_KERNEL); 156 157 if (pg) 158 INIT_LIST_HEAD(&pg->pgpaths); 159 160 return pg; 161 } 162 163 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 164 { 165 struct pgpath *pgpath, *tmp; 166 167 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 168 list_del(&pgpath->list); 169 dm_put_device(ti, pgpath->path.dev); 170 free_pgpath(pgpath); 171 } 172 } 173 174 static void free_priority_group(struct priority_group *pg, 175 struct dm_target *ti) 176 { 177 struct path_selector *ps = &pg->ps; 178 179 if (ps->type) { 180 ps->type->destroy(ps); 181 dm_put_path_selector(ps->type); 182 } 183 184 free_pgpaths(&pg->pgpaths, ti); 185 kfree(pg); 186 } 187 188 static struct multipath *alloc_multipath(struct dm_target *ti) 189 { 190 struct multipath *m; 191 192 m = kzalloc(sizeof(*m), GFP_KERNEL); 193 if (m) { 194 INIT_LIST_HEAD(&m->priority_groups); 195 spin_lock_init(&m->lock); 196 set_bit(MPATHF_QUEUE_IO, &m->flags); 197 atomic_set(&m->nr_valid_paths, 0); 198 atomic_set(&m->pg_init_in_progress, 0); 199 atomic_set(&m->pg_init_count, 0); 200 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT; 201 INIT_WORK(&m->trigger_event, trigger_event); 202 init_waitqueue_head(&m->pg_init_wait); 203 mutex_init(&m->work_mutex); 204 205 m->queue_mode = DM_TYPE_NONE; 206 207 m->ti = ti; 208 ti->private = m; 209 } 210 211 return m; 212 } 213 214 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m) 215 { 216 if (m->queue_mode == DM_TYPE_NONE) { 217 /* 218 * Default to request-based. 219 */ 220 if (dm_use_blk_mq(dm_table_get_md(ti->table))) 221 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 222 else 223 m->queue_mode = DM_TYPE_REQUEST_BASED; 224 } else if (m->queue_mode == DM_TYPE_BIO_BASED) { 225 INIT_WORK(&m->process_queued_bios, process_queued_bios); 226 /* 227 * bio-based doesn't support any direct scsi_dh management; 228 * it just discovers if a scsi_dh is attached. 229 */ 230 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 231 } 232 233 dm_table_set_type(ti->table, m->queue_mode); 234 235 return 0; 236 } 237 238 static void free_multipath(struct multipath *m) 239 { 240 struct priority_group *pg, *tmp; 241 242 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 243 list_del(&pg->list); 244 free_priority_group(pg, m->ti); 245 } 246 247 kfree(m->hw_handler_name); 248 kfree(m->hw_handler_params); 249 kfree(m); 250 } 251 252 static struct dm_mpath_io *get_mpio(union map_info *info) 253 { 254 return info->ptr; 255 } 256 257 static size_t multipath_per_bio_data_size(void) 258 { 259 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details); 260 } 261 262 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio) 263 { 264 return dm_per_bio_data(bio, multipath_per_bio_data_size()); 265 } 266 267 static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio) 268 { 269 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */ 270 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 271 void *bio_details = mpio + 1; 272 273 return bio_details; 274 } 275 276 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p, 277 struct dm_bio_details **bio_details_p) 278 { 279 struct dm_mpath_io *mpio = get_mpio_from_bio(bio); 280 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio); 281 282 memset(mpio, 0, sizeof(*mpio)); 283 memset(bio_details, 0, sizeof(*bio_details)); 284 dm_bio_record(bio_details, bio); 285 286 if (mpio_p) 287 *mpio_p = mpio; 288 if (bio_details_p) 289 *bio_details_p = bio_details; 290 } 291 292 /*----------------------------------------------- 293 * Path selection 294 *-----------------------------------------------*/ 295 296 static int __pg_init_all_paths(struct multipath *m) 297 { 298 struct pgpath *pgpath; 299 unsigned long pg_init_delay = 0; 300 301 lockdep_assert_held(&m->lock); 302 303 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 304 return 0; 305 306 atomic_inc(&m->pg_init_count); 307 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 308 309 /* Check here to reset pg_init_required */ 310 if (!m->current_pg) 311 return 0; 312 313 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags)) 314 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ? 315 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS); 316 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) { 317 /* Skip failed paths */ 318 if (!pgpath->is_active) 319 continue; 320 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path, 321 pg_init_delay)) 322 atomic_inc(&m->pg_init_in_progress); 323 } 324 return atomic_read(&m->pg_init_in_progress); 325 } 326 327 static int pg_init_all_paths(struct multipath *m) 328 { 329 int ret; 330 unsigned long flags; 331 332 spin_lock_irqsave(&m->lock, flags); 333 ret = __pg_init_all_paths(m); 334 spin_unlock_irqrestore(&m->lock, flags); 335 336 return ret; 337 } 338 339 static void __switch_pg(struct multipath *m, struct priority_group *pg) 340 { 341 m->current_pg = pg; 342 343 /* Must we initialise the PG first, and queue I/O till it's ready? */ 344 if (m->hw_handler_name) { 345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 346 set_bit(MPATHF_QUEUE_IO, &m->flags); 347 } else { 348 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 349 clear_bit(MPATHF_QUEUE_IO, &m->flags); 350 } 351 352 atomic_set(&m->pg_init_count, 0); 353 } 354 355 static struct pgpath *choose_path_in_pg(struct multipath *m, 356 struct priority_group *pg, 357 size_t nr_bytes) 358 { 359 unsigned long flags; 360 struct dm_path *path; 361 struct pgpath *pgpath; 362 363 path = pg->ps.type->select_path(&pg->ps, nr_bytes); 364 if (!path) 365 return ERR_PTR(-ENXIO); 366 367 pgpath = path_to_pgpath(path); 368 369 if (unlikely(lockless_dereference(m->current_pg) != pg)) { 370 /* Only update current_pgpath if pg changed */ 371 spin_lock_irqsave(&m->lock, flags); 372 m->current_pgpath = pgpath; 373 __switch_pg(m, pg); 374 spin_unlock_irqrestore(&m->lock, flags); 375 } 376 377 return pgpath; 378 } 379 380 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes) 381 { 382 unsigned long flags; 383 struct priority_group *pg; 384 struct pgpath *pgpath; 385 unsigned bypassed = 1; 386 387 if (!atomic_read(&m->nr_valid_paths)) { 388 clear_bit(MPATHF_QUEUE_IO, &m->flags); 389 goto failed; 390 } 391 392 /* Were we instructed to switch PG? */ 393 if (lockless_dereference(m->next_pg)) { 394 spin_lock_irqsave(&m->lock, flags); 395 pg = m->next_pg; 396 if (!pg) { 397 spin_unlock_irqrestore(&m->lock, flags); 398 goto check_current_pg; 399 } 400 m->next_pg = NULL; 401 spin_unlock_irqrestore(&m->lock, flags); 402 pgpath = choose_path_in_pg(m, pg, nr_bytes); 403 if (!IS_ERR_OR_NULL(pgpath)) 404 return pgpath; 405 } 406 407 /* Don't change PG until it has no remaining paths */ 408 check_current_pg: 409 pg = lockless_dereference(m->current_pg); 410 if (pg) { 411 pgpath = choose_path_in_pg(m, pg, nr_bytes); 412 if (!IS_ERR_OR_NULL(pgpath)) 413 return pgpath; 414 } 415 416 /* 417 * Loop through priority groups until we find a valid path. 418 * First time we skip PGs marked 'bypassed'. 419 * Second time we only try the ones we skipped, but set 420 * pg_init_delay_retry so we do not hammer controllers. 421 */ 422 do { 423 list_for_each_entry(pg, &m->priority_groups, list) { 424 if (pg->bypassed == !!bypassed) 425 continue; 426 pgpath = choose_path_in_pg(m, pg, nr_bytes); 427 if (!IS_ERR_OR_NULL(pgpath)) { 428 if (!bypassed) 429 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 430 return pgpath; 431 } 432 } 433 } while (bypassed--); 434 435 failed: 436 spin_lock_irqsave(&m->lock, flags); 437 m->current_pgpath = NULL; 438 m->current_pg = NULL; 439 spin_unlock_irqrestore(&m->lock, flags); 440 441 return NULL; 442 } 443 444 /* 445 * dm_report_EIO() is a macro instead of a function to make pr_debug() 446 * report the function name and line number of the function from which 447 * it has been invoked. 448 */ 449 #define dm_report_EIO(m) \ 450 do { \ 451 struct mapped_device *md = dm_table_get_md((m)->ti->table); \ 452 \ 453 pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \ 454 dm_device_name(md), \ 455 test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags), \ 456 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags), \ 457 dm_noflush_suspending((m)->ti)); \ 458 } while (0) 459 460 /* 461 * Map cloned requests (request-based multipath) 462 */ 463 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq, 464 union map_info *map_context, 465 struct request **__clone) 466 { 467 struct multipath *m = ti->private; 468 size_t nr_bytes = blk_rq_bytes(rq); 469 struct pgpath *pgpath; 470 struct block_device *bdev; 471 struct dm_mpath_io *mpio = get_mpio(map_context); 472 struct request_queue *q; 473 struct request *clone; 474 475 /* Do we need to select a new pgpath? */ 476 pgpath = lockless_dereference(m->current_pgpath); 477 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags)) 478 pgpath = choose_pgpath(m, nr_bytes); 479 480 if (!pgpath) { 481 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 482 return DM_MAPIO_DELAY_REQUEUE; 483 dm_report_EIO(m); /* Failed */ 484 return DM_MAPIO_KILL; 485 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) || 486 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 487 if (pg_init_all_paths(m)) 488 return DM_MAPIO_DELAY_REQUEUE; 489 return DM_MAPIO_REQUEUE; 490 } 491 492 memset(mpio, 0, sizeof(*mpio)); 493 mpio->pgpath = pgpath; 494 mpio->nr_bytes = nr_bytes; 495 496 bdev = pgpath->path.dev->bdev; 497 q = bdev_get_queue(bdev); 498 clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC); 499 if (IS_ERR(clone)) { 500 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */ 501 bool queue_dying = blk_queue_dying(q); 502 DMERR_LIMIT("blk_get_request() returned %ld%s - requeuing", 503 PTR_ERR(clone), queue_dying ? " (path offline)" : ""); 504 if (queue_dying) { 505 atomic_inc(&m->pg_init_in_progress); 506 activate_or_offline_path(pgpath); 507 return DM_MAPIO_REQUEUE; 508 } 509 return DM_MAPIO_DELAY_REQUEUE; 510 } 511 clone->bio = clone->biotail = NULL; 512 clone->rq_disk = bdev->bd_disk; 513 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT; 514 *__clone = clone; 515 516 if (pgpath->pg->ps.type->start_io) 517 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 518 &pgpath->path, 519 nr_bytes); 520 return DM_MAPIO_REMAPPED; 521 } 522 523 static void multipath_release_clone(struct request *clone) 524 { 525 blk_put_request(clone); 526 } 527 528 /* 529 * Map cloned bios (bio-based multipath) 530 */ 531 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio) 532 { 533 size_t nr_bytes = bio->bi_iter.bi_size; 534 struct pgpath *pgpath; 535 unsigned long flags; 536 bool queue_io; 537 538 /* Do we need to select a new pgpath? */ 539 pgpath = lockless_dereference(m->current_pgpath); 540 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags); 541 if (!pgpath || !queue_io) 542 pgpath = choose_pgpath(m, nr_bytes); 543 544 if ((pgpath && queue_io) || 545 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) { 546 /* Queue for the daemon to resubmit */ 547 spin_lock_irqsave(&m->lock, flags); 548 bio_list_add(&m->queued_bios, bio); 549 spin_unlock_irqrestore(&m->lock, flags); 550 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */ 551 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 552 pg_init_all_paths(m); 553 else if (!queue_io) 554 queue_work(kmultipathd, &m->process_queued_bios); 555 return DM_MAPIO_SUBMITTED; 556 } 557 558 if (!pgpath) { 559 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 560 return DM_MAPIO_REQUEUE; 561 dm_report_EIO(m); 562 return DM_MAPIO_KILL; 563 } 564 565 mpio->pgpath = pgpath; 566 mpio->nr_bytes = nr_bytes; 567 568 bio->bi_status = 0; 569 bio->bi_bdev = pgpath->path.dev->bdev; 570 bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 571 572 if (pgpath->pg->ps.type->start_io) 573 pgpath->pg->ps.type->start_io(&pgpath->pg->ps, 574 &pgpath->path, 575 nr_bytes); 576 return DM_MAPIO_REMAPPED; 577 } 578 579 static int multipath_map_bio(struct dm_target *ti, struct bio *bio) 580 { 581 struct multipath *m = ti->private; 582 struct dm_mpath_io *mpio = NULL; 583 584 multipath_init_per_bio_data(bio, &mpio, NULL); 585 586 return __multipath_map_bio(m, bio, mpio); 587 } 588 589 static void process_queued_io_list(struct multipath *m) 590 { 591 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED) 592 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table)); 593 else if (m->queue_mode == DM_TYPE_BIO_BASED) 594 queue_work(kmultipathd, &m->process_queued_bios); 595 } 596 597 static void process_queued_bios(struct work_struct *work) 598 { 599 int r; 600 unsigned long flags; 601 struct bio *bio; 602 struct bio_list bios; 603 struct blk_plug plug; 604 struct multipath *m = 605 container_of(work, struct multipath, process_queued_bios); 606 607 bio_list_init(&bios); 608 609 spin_lock_irqsave(&m->lock, flags); 610 611 if (bio_list_empty(&m->queued_bios)) { 612 spin_unlock_irqrestore(&m->lock, flags); 613 return; 614 } 615 616 bio_list_merge(&bios, &m->queued_bios); 617 bio_list_init(&m->queued_bios); 618 619 spin_unlock_irqrestore(&m->lock, flags); 620 621 blk_start_plug(&plug); 622 while ((bio = bio_list_pop(&bios))) { 623 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio)); 624 switch (r) { 625 case DM_MAPIO_KILL: 626 bio->bi_status = BLK_STS_IOERR; 627 bio_endio(bio); 628 break; 629 case DM_MAPIO_REQUEUE: 630 bio->bi_status = BLK_STS_DM_REQUEUE; 631 bio_endio(bio); 632 break; 633 case DM_MAPIO_REMAPPED: 634 generic_make_request(bio); 635 break; 636 } 637 } 638 blk_finish_plug(&plug); 639 } 640 641 static void assign_bit(bool value, long nr, unsigned long *addr) 642 { 643 if (value) 644 set_bit(nr, addr); 645 else 646 clear_bit(nr, addr); 647 } 648 649 /* 650 * If we run out of usable paths, should we queue I/O or error it? 651 */ 652 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path, 653 bool save_old_value) 654 { 655 unsigned long flags; 656 657 spin_lock_irqsave(&m->lock, flags); 658 assign_bit((save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) || 659 (!save_old_value && queue_if_no_path), 660 MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags); 661 assign_bit(queue_if_no_path || dm_noflush_suspending(m->ti), 662 MPATHF_QUEUE_IF_NO_PATH, &m->flags); 663 spin_unlock_irqrestore(&m->lock, flags); 664 665 if (!queue_if_no_path) { 666 dm_table_run_md_queue_async(m->ti->table); 667 process_queued_io_list(m); 668 } 669 670 return 0; 671 } 672 673 /* 674 * An event is triggered whenever a path is taken out of use. 675 * Includes path failure and PG bypass. 676 */ 677 static void trigger_event(struct work_struct *work) 678 { 679 struct multipath *m = 680 container_of(work, struct multipath, trigger_event); 681 682 dm_table_event(m->ti->table); 683 } 684 685 /*----------------------------------------------------------------- 686 * Constructor/argument parsing: 687 * <#multipath feature args> [<arg>]* 688 * <#hw_handler args> [hw_handler [<arg>]*] 689 * <#priority groups> 690 * <initial priority group> 691 * [<selector> <#selector args> [<arg>]* 692 * <#paths> <#per-path selector args> 693 * [<path> [<arg>]* ]+ ]+ 694 *---------------------------------------------------------------*/ 695 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg, 696 struct dm_target *ti) 697 { 698 int r; 699 struct path_selector_type *pst; 700 unsigned ps_argc; 701 702 static struct dm_arg _args[] = { 703 {0, 1024, "invalid number of path selector args"}, 704 }; 705 706 pst = dm_get_path_selector(dm_shift_arg(as)); 707 if (!pst) { 708 ti->error = "unknown path selector type"; 709 return -EINVAL; 710 } 711 712 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error); 713 if (r) { 714 dm_put_path_selector(pst); 715 return -EINVAL; 716 } 717 718 r = pst->create(&pg->ps, ps_argc, as->argv); 719 if (r) { 720 dm_put_path_selector(pst); 721 ti->error = "path selector constructor failed"; 722 return r; 723 } 724 725 pg->ps.type = pst; 726 dm_consume_args(as, ps_argc); 727 728 return 0; 729 } 730 731 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps, 732 struct dm_target *ti) 733 { 734 int r; 735 struct pgpath *p; 736 struct multipath *m = ti->private; 737 struct request_queue *q = NULL; 738 const char *attached_handler_name; 739 740 /* we need at least a path arg */ 741 if (as->argc < 1) { 742 ti->error = "no device given"; 743 return ERR_PTR(-EINVAL); 744 } 745 746 p = alloc_pgpath(); 747 if (!p) 748 return ERR_PTR(-ENOMEM); 749 750 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table), 751 &p->path.dev); 752 if (r) { 753 ti->error = "error getting device"; 754 goto bad; 755 } 756 757 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name) 758 q = bdev_get_queue(p->path.dev->bdev); 759 760 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) { 761 retain: 762 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL); 763 if (attached_handler_name) { 764 /* 765 * Clear any hw_handler_params associated with a 766 * handler that isn't already attached. 767 */ 768 if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) { 769 kfree(m->hw_handler_params); 770 m->hw_handler_params = NULL; 771 } 772 773 /* 774 * Reset hw_handler_name to match the attached handler 775 * 776 * NB. This modifies the table line to show the actual 777 * handler instead of the original table passed in. 778 */ 779 kfree(m->hw_handler_name); 780 m->hw_handler_name = attached_handler_name; 781 } 782 } 783 784 if (m->hw_handler_name) { 785 r = scsi_dh_attach(q, m->hw_handler_name); 786 if (r == -EBUSY) { 787 char b[BDEVNAME_SIZE]; 788 789 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n", 790 bdevname(p->path.dev->bdev, b)); 791 goto retain; 792 } 793 if (r < 0) { 794 ti->error = "error attaching hardware handler"; 795 dm_put_device(ti, p->path.dev); 796 goto bad; 797 } 798 799 if (m->hw_handler_params) { 800 r = scsi_dh_set_params(q, m->hw_handler_params); 801 if (r < 0) { 802 ti->error = "unable to set hardware " 803 "handler parameters"; 804 dm_put_device(ti, p->path.dev); 805 goto bad; 806 } 807 } 808 } 809 810 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 811 if (r) { 812 dm_put_device(ti, p->path.dev); 813 goto bad; 814 } 815 816 return p; 817 818 bad: 819 free_pgpath(p); 820 return ERR_PTR(r); 821 } 822 823 static struct priority_group *parse_priority_group(struct dm_arg_set *as, 824 struct multipath *m) 825 { 826 static struct dm_arg _args[] = { 827 {1, 1024, "invalid number of paths"}, 828 {0, 1024, "invalid number of selector args"} 829 }; 830 831 int r; 832 unsigned i, nr_selector_args, nr_args; 833 struct priority_group *pg; 834 struct dm_target *ti = m->ti; 835 836 if (as->argc < 2) { 837 as->argc = 0; 838 ti->error = "not enough priority group arguments"; 839 return ERR_PTR(-EINVAL); 840 } 841 842 pg = alloc_priority_group(); 843 if (!pg) { 844 ti->error = "couldn't allocate priority group"; 845 return ERR_PTR(-ENOMEM); 846 } 847 pg->m = m; 848 849 r = parse_path_selector(as, pg, ti); 850 if (r) 851 goto bad; 852 853 /* 854 * read the paths 855 */ 856 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error); 857 if (r) 858 goto bad; 859 860 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error); 861 if (r) 862 goto bad; 863 864 nr_args = 1 + nr_selector_args; 865 for (i = 0; i < pg->nr_pgpaths; i++) { 866 struct pgpath *pgpath; 867 struct dm_arg_set path_args; 868 869 if (as->argc < nr_args) { 870 ti->error = "not enough path parameters"; 871 r = -EINVAL; 872 goto bad; 873 } 874 875 path_args.argc = nr_args; 876 path_args.argv = as->argv; 877 878 pgpath = parse_path(&path_args, &pg->ps, ti); 879 if (IS_ERR(pgpath)) { 880 r = PTR_ERR(pgpath); 881 goto bad; 882 } 883 884 pgpath->pg = pg; 885 list_add_tail(&pgpath->list, &pg->pgpaths); 886 dm_consume_args(as, nr_args); 887 } 888 889 return pg; 890 891 bad: 892 free_priority_group(pg, ti); 893 return ERR_PTR(r); 894 } 895 896 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m) 897 { 898 unsigned hw_argc; 899 int ret; 900 struct dm_target *ti = m->ti; 901 902 static struct dm_arg _args[] = { 903 {0, 1024, "invalid number of hardware handler args"}, 904 }; 905 906 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error)) 907 return -EINVAL; 908 909 if (!hw_argc) 910 return 0; 911 912 if (m->queue_mode == DM_TYPE_BIO_BASED) { 913 dm_consume_args(as, hw_argc); 914 DMERR("bio-based multipath doesn't allow hardware handler args"); 915 return 0; 916 } 917 918 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL); 919 if (!m->hw_handler_name) 920 return -EINVAL; 921 922 if (hw_argc > 1) { 923 char *p; 924 int i, j, len = 4; 925 926 for (i = 0; i <= hw_argc - 2; i++) 927 len += strlen(as->argv[i]) + 1; 928 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL); 929 if (!p) { 930 ti->error = "memory allocation failed"; 931 ret = -ENOMEM; 932 goto fail; 933 } 934 j = sprintf(p, "%d", hw_argc - 1); 935 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1) 936 j = sprintf(p, "%s", as->argv[i]); 937 } 938 dm_consume_args(as, hw_argc - 1); 939 940 return 0; 941 fail: 942 kfree(m->hw_handler_name); 943 m->hw_handler_name = NULL; 944 return ret; 945 } 946 947 static int parse_features(struct dm_arg_set *as, struct multipath *m) 948 { 949 int r; 950 unsigned argc; 951 struct dm_target *ti = m->ti; 952 const char *arg_name; 953 954 static struct dm_arg _args[] = { 955 {0, 8, "invalid number of feature args"}, 956 {1, 50, "pg_init_retries must be between 1 and 50"}, 957 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"}, 958 }; 959 960 r = dm_read_arg_group(_args, as, &argc, &ti->error); 961 if (r) 962 return -EINVAL; 963 964 if (!argc) 965 return 0; 966 967 do { 968 arg_name = dm_shift_arg(as); 969 argc--; 970 971 if (!strcasecmp(arg_name, "queue_if_no_path")) { 972 r = queue_if_no_path(m, true, false); 973 continue; 974 } 975 976 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) { 977 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags); 978 continue; 979 } 980 981 if (!strcasecmp(arg_name, "pg_init_retries") && 982 (argc >= 1)) { 983 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error); 984 argc--; 985 continue; 986 } 987 988 if (!strcasecmp(arg_name, "pg_init_delay_msecs") && 989 (argc >= 1)) { 990 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error); 991 argc--; 992 continue; 993 } 994 995 if (!strcasecmp(arg_name, "queue_mode") && 996 (argc >= 1)) { 997 const char *queue_mode_name = dm_shift_arg(as); 998 999 if (!strcasecmp(queue_mode_name, "bio")) 1000 m->queue_mode = DM_TYPE_BIO_BASED; 1001 else if (!strcasecmp(queue_mode_name, "rq")) 1002 m->queue_mode = DM_TYPE_REQUEST_BASED; 1003 else if (!strcasecmp(queue_mode_name, "mq")) 1004 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED; 1005 else { 1006 ti->error = "Unknown 'queue_mode' requested"; 1007 r = -EINVAL; 1008 } 1009 argc--; 1010 continue; 1011 } 1012 1013 ti->error = "Unrecognised multipath feature request"; 1014 r = -EINVAL; 1015 } while (argc && !r); 1016 1017 return r; 1018 } 1019 1020 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv) 1021 { 1022 /* target arguments */ 1023 static struct dm_arg _args[] = { 1024 {0, 1024, "invalid number of priority groups"}, 1025 {0, 1024, "invalid initial priority group number"}, 1026 }; 1027 1028 int r; 1029 struct multipath *m; 1030 struct dm_arg_set as; 1031 unsigned pg_count = 0; 1032 unsigned next_pg_num; 1033 1034 as.argc = argc; 1035 as.argv = argv; 1036 1037 m = alloc_multipath(ti); 1038 if (!m) { 1039 ti->error = "can't allocate multipath"; 1040 return -EINVAL; 1041 } 1042 1043 r = parse_features(&as, m); 1044 if (r) 1045 goto bad; 1046 1047 r = alloc_multipath_stage2(ti, m); 1048 if (r) 1049 goto bad; 1050 1051 r = parse_hw_handler(&as, m); 1052 if (r) 1053 goto bad; 1054 1055 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error); 1056 if (r) 1057 goto bad; 1058 1059 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error); 1060 if (r) 1061 goto bad; 1062 1063 if ((!m->nr_priority_groups && next_pg_num) || 1064 (m->nr_priority_groups && !next_pg_num)) { 1065 ti->error = "invalid initial priority group"; 1066 r = -EINVAL; 1067 goto bad; 1068 } 1069 1070 /* parse the priority groups */ 1071 while (as.argc) { 1072 struct priority_group *pg; 1073 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths); 1074 1075 pg = parse_priority_group(&as, m); 1076 if (IS_ERR(pg)) { 1077 r = PTR_ERR(pg); 1078 goto bad; 1079 } 1080 1081 nr_valid_paths += pg->nr_pgpaths; 1082 atomic_set(&m->nr_valid_paths, nr_valid_paths); 1083 1084 list_add_tail(&pg->list, &m->priority_groups); 1085 pg_count++; 1086 pg->pg_num = pg_count; 1087 if (!--next_pg_num) 1088 m->next_pg = pg; 1089 } 1090 1091 if (pg_count != m->nr_priority_groups) { 1092 ti->error = "priority group count mismatch"; 1093 r = -EINVAL; 1094 goto bad; 1095 } 1096 1097 ti->num_flush_bios = 1; 1098 ti->num_discard_bios = 1; 1099 ti->num_write_same_bios = 1; 1100 ti->num_write_zeroes_bios = 1; 1101 if (m->queue_mode == DM_TYPE_BIO_BASED) 1102 ti->per_io_data_size = multipath_per_bio_data_size(); 1103 else 1104 ti->per_io_data_size = sizeof(struct dm_mpath_io); 1105 1106 return 0; 1107 1108 bad: 1109 free_multipath(m); 1110 return r; 1111 } 1112 1113 static void multipath_wait_for_pg_init_completion(struct multipath *m) 1114 { 1115 DEFINE_WAIT(wait); 1116 1117 while (1) { 1118 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE); 1119 1120 if (!atomic_read(&m->pg_init_in_progress)) 1121 break; 1122 1123 io_schedule(); 1124 } 1125 finish_wait(&m->pg_init_wait, &wait); 1126 } 1127 1128 static void flush_multipath_work(struct multipath *m) 1129 { 1130 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1131 smp_mb__after_atomic(); 1132 1133 flush_workqueue(kmpath_handlerd); 1134 multipath_wait_for_pg_init_completion(m); 1135 flush_workqueue(kmultipathd); 1136 flush_work(&m->trigger_event); 1137 1138 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags); 1139 smp_mb__after_atomic(); 1140 } 1141 1142 static void multipath_dtr(struct dm_target *ti) 1143 { 1144 struct multipath *m = ti->private; 1145 1146 flush_multipath_work(m); 1147 free_multipath(m); 1148 } 1149 1150 /* 1151 * Take a path out of use. 1152 */ 1153 static int fail_path(struct pgpath *pgpath) 1154 { 1155 unsigned long flags; 1156 struct multipath *m = pgpath->pg->m; 1157 1158 spin_lock_irqsave(&m->lock, flags); 1159 1160 if (!pgpath->is_active) 1161 goto out; 1162 1163 DMWARN("Failing path %s.", pgpath->path.dev->name); 1164 1165 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 1166 pgpath->is_active = false; 1167 pgpath->fail_count++; 1168 1169 atomic_dec(&m->nr_valid_paths); 1170 1171 if (pgpath == m->current_pgpath) 1172 m->current_pgpath = NULL; 1173 1174 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 1175 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths)); 1176 1177 schedule_work(&m->trigger_event); 1178 1179 out: 1180 spin_unlock_irqrestore(&m->lock, flags); 1181 1182 return 0; 1183 } 1184 1185 /* 1186 * Reinstate a previously-failed path 1187 */ 1188 static int reinstate_path(struct pgpath *pgpath) 1189 { 1190 int r = 0, run_queue = 0; 1191 unsigned long flags; 1192 struct multipath *m = pgpath->pg->m; 1193 unsigned nr_valid_paths; 1194 1195 spin_lock_irqsave(&m->lock, flags); 1196 1197 if (pgpath->is_active) 1198 goto out; 1199 1200 DMWARN("Reinstating path %s.", pgpath->path.dev->name); 1201 1202 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 1203 if (r) 1204 goto out; 1205 1206 pgpath->is_active = true; 1207 1208 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths); 1209 if (nr_valid_paths == 1) { 1210 m->current_pgpath = NULL; 1211 run_queue = 1; 1212 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) { 1213 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work)) 1214 atomic_inc(&m->pg_init_in_progress); 1215 } 1216 1217 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 1218 pgpath->path.dev->name, nr_valid_paths); 1219 1220 schedule_work(&m->trigger_event); 1221 1222 out: 1223 spin_unlock_irqrestore(&m->lock, flags); 1224 if (run_queue) { 1225 dm_table_run_md_queue_async(m->ti->table); 1226 process_queued_io_list(m); 1227 } 1228 1229 return r; 1230 } 1231 1232 /* 1233 * Fail or reinstate all paths that match the provided struct dm_dev. 1234 */ 1235 static int action_dev(struct multipath *m, struct dm_dev *dev, 1236 action_fn action) 1237 { 1238 int r = -EINVAL; 1239 struct pgpath *pgpath; 1240 struct priority_group *pg; 1241 1242 list_for_each_entry(pg, &m->priority_groups, list) { 1243 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1244 if (pgpath->path.dev == dev) 1245 r = action(pgpath); 1246 } 1247 } 1248 1249 return r; 1250 } 1251 1252 /* 1253 * Temporarily try to avoid having to use the specified PG 1254 */ 1255 static void bypass_pg(struct multipath *m, struct priority_group *pg, 1256 bool bypassed) 1257 { 1258 unsigned long flags; 1259 1260 spin_lock_irqsave(&m->lock, flags); 1261 1262 pg->bypassed = bypassed; 1263 m->current_pgpath = NULL; 1264 m->current_pg = NULL; 1265 1266 spin_unlock_irqrestore(&m->lock, flags); 1267 1268 schedule_work(&m->trigger_event); 1269 } 1270 1271 /* 1272 * Switch to using the specified PG from the next I/O that gets mapped 1273 */ 1274 static int switch_pg_num(struct multipath *m, const char *pgstr) 1275 { 1276 struct priority_group *pg; 1277 unsigned pgnum; 1278 unsigned long flags; 1279 char dummy; 1280 1281 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1282 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1283 DMWARN("invalid PG number supplied to switch_pg_num"); 1284 return -EINVAL; 1285 } 1286 1287 spin_lock_irqsave(&m->lock, flags); 1288 list_for_each_entry(pg, &m->priority_groups, list) { 1289 pg->bypassed = false; 1290 if (--pgnum) 1291 continue; 1292 1293 m->current_pgpath = NULL; 1294 m->current_pg = NULL; 1295 m->next_pg = pg; 1296 } 1297 spin_unlock_irqrestore(&m->lock, flags); 1298 1299 schedule_work(&m->trigger_event); 1300 return 0; 1301 } 1302 1303 /* 1304 * Set/clear bypassed status of a PG. 1305 * PGs are numbered upwards from 1 in the order they were declared. 1306 */ 1307 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed) 1308 { 1309 struct priority_group *pg; 1310 unsigned pgnum; 1311 char dummy; 1312 1313 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum || 1314 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) { 1315 DMWARN("invalid PG number supplied to bypass_pg"); 1316 return -EINVAL; 1317 } 1318 1319 list_for_each_entry(pg, &m->priority_groups, list) { 1320 if (!--pgnum) 1321 break; 1322 } 1323 1324 bypass_pg(m, pg, bypassed); 1325 return 0; 1326 } 1327 1328 /* 1329 * Should we retry pg_init immediately? 1330 */ 1331 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1332 { 1333 unsigned long flags; 1334 bool limit_reached = false; 1335 1336 spin_lock_irqsave(&m->lock, flags); 1337 1338 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries && 1339 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags)) 1340 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags); 1341 else 1342 limit_reached = true; 1343 1344 spin_unlock_irqrestore(&m->lock, flags); 1345 1346 return limit_reached; 1347 } 1348 1349 static void pg_init_done(void *data, int errors) 1350 { 1351 struct pgpath *pgpath = data; 1352 struct priority_group *pg = pgpath->pg; 1353 struct multipath *m = pg->m; 1354 unsigned long flags; 1355 bool delay_retry = false; 1356 1357 /* device or driver problems */ 1358 switch (errors) { 1359 case SCSI_DH_OK: 1360 break; 1361 case SCSI_DH_NOSYS: 1362 if (!m->hw_handler_name) { 1363 errors = 0; 1364 break; 1365 } 1366 DMERR("Could not failover the device: Handler scsi_dh_%s " 1367 "Error %d.", m->hw_handler_name, errors); 1368 /* 1369 * Fail path for now, so we do not ping pong 1370 */ 1371 fail_path(pgpath); 1372 break; 1373 case SCSI_DH_DEV_TEMP_BUSY: 1374 /* 1375 * Probably doing something like FW upgrade on the 1376 * controller so try the other pg. 1377 */ 1378 bypass_pg(m, pg, true); 1379 break; 1380 case SCSI_DH_RETRY: 1381 /* Wait before retrying. */ 1382 delay_retry = 1; 1383 case SCSI_DH_IMM_RETRY: 1384 case SCSI_DH_RES_TEMP_UNAVAIL: 1385 if (pg_init_limit_reached(m, pgpath)) 1386 fail_path(pgpath); 1387 errors = 0; 1388 break; 1389 case SCSI_DH_DEV_OFFLINED: 1390 default: 1391 /* 1392 * We probably do not want to fail the path for a device 1393 * error, but this is what the old dm did. In future 1394 * patches we can do more advanced handling. 1395 */ 1396 fail_path(pgpath); 1397 } 1398 1399 spin_lock_irqsave(&m->lock, flags); 1400 if (errors) { 1401 if (pgpath == m->current_pgpath) { 1402 DMERR("Could not failover device. Error %d.", errors); 1403 m->current_pgpath = NULL; 1404 m->current_pg = NULL; 1405 } 1406 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1407 pg->bypassed = false; 1408 1409 if (atomic_dec_return(&m->pg_init_in_progress) > 0) 1410 /* Activations of other paths are still on going */ 1411 goto out; 1412 1413 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) { 1414 if (delay_retry) 1415 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1416 else 1417 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags); 1418 1419 if (__pg_init_all_paths(m)) 1420 goto out; 1421 } 1422 clear_bit(MPATHF_QUEUE_IO, &m->flags); 1423 1424 process_queued_io_list(m); 1425 1426 /* 1427 * Wake up any thread waiting to suspend. 1428 */ 1429 wake_up(&m->pg_init_wait); 1430 1431 out: 1432 spin_unlock_irqrestore(&m->lock, flags); 1433 } 1434 1435 static void activate_or_offline_path(struct pgpath *pgpath) 1436 { 1437 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1438 1439 if (pgpath->is_active && !blk_queue_dying(q)) 1440 scsi_dh_activate(q, pg_init_done, pgpath); 1441 else 1442 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED); 1443 } 1444 1445 static void activate_path_work(struct work_struct *work) 1446 { 1447 struct pgpath *pgpath = 1448 container_of(work, struct pgpath, activate_path.work); 1449 1450 activate_or_offline_path(pgpath); 1451 } 1452 1453 static int noretry_error(blk_status_t error) 1454 { 1455 switch (error) { 1456 case BLK_STS_NOTSUPP: 1457 case BLK_STS_NOSPC: 1458 case BLK_STS_TARGET: 1459 case BLK_STS_NEXUS: 1460 case BLK_STS_MEDIUM: 1461 case BLK_STS_RESOURCE: 1462 return 1; 1463 } 1464 1465 /* Anything else could be a path failure, so should be retried */ 1466 return 0; 1467 } 1468 1469 static int multipath_end_io(struct dm_target *ti, struct request *clone, 1470 blk_status_t error, union map_info *map_context) 1471 { 1472 struct dm_mpath_io *mpio = get_mpio(map_context); 1473 struct pgpath *pgpath = mpio->pgpath; 1474 int r = DM_ENDIO_DONE; 1475 1476 /* 1477 * We don't queue any clone request inside the multipath target 1478 * during end I/O handling, since those clone requests don't have 1479 * bio clones. If we queue them inside the multipath target, 1480 * we need to make bio clones, that requires memory allocation. 1481 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests 1482 * don't have bio clones.) 1483 * Instead of queueing the clone request here, we queue the original 1484 * request into dm core, which will remake a clone request and 1485 * clone bios for it and resubmit it later. 1486 */ 1487 if (error && !noretry_error(error)) { 1488 struct multipath *m = ti->private; 1489 1490 r = DM_ENDIO_REQUEUE; 1491 1492 if (pgpath) 1493 fail_path(pgpath); 1494 1495 if (atomic_read(&m->nr_valid_paths) == 0 && 1496 !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1497 if (error == BLK_STS_IOERR) 1498 dm_report_EIO(m); 1499 /* complete with the original error */ 1500 r = DM_ENDIO_DONE; 1501 } 1502 } 1503 1504 if (pgpath) { 1505 struct path_selector *ps = &pgpath->pg->ps; 1506 1507 if (ps->type->end_io) 1508 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1509 } 1510 1511 return r; 1512 } 1513 1514 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, 1515 blk_status_t *error) 1516 { 1517 struct multipath *m = ti->private; 1518 struct dm_mpath_io *mpio = get_mpio_from_bio(clone); 1519 struct pgpath *pgpath = mpio->pgpath; 1520 unsigned long flags; 1521 int r = DM_ENDIO_DONE; 1522 1523 if (!*error || noretry_error(*error)) 1524 goto done; 1525 1526 if (pgpath) 1527 fail_path(pgpath); 1528 1529 if (atomic_read(&m->nr_valid_paths) == 0 && 1530 !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) { 1531 dm_report_EIO(m); 1532 *error = BLK_STS_IOERR; 1533 goto done; 1534 } 1535 1536 /* Queue for the daemon to resubmit */ 1537 dm_bio_restore(get_bio_details_from_bio(clone), clone); 1538 1539 spin_lock_irqsave(&m->lock, flags); 1540 bio_list_add(&m->queued_bios, clone); 1541 spin_unlock_irqrestore(&m->lock, flags); 1542 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) 1543 queue_work(kmultipathd, &m->process_queued_bios); 1544 1545 r = DM_ENDIO_INCOMPLETE; 1546 done: 1547 if (pgpath) { 1548 struct path_selector *ps = &pgpath->pg->ps; 1549 1550 if (ps->type->end_io) 1551 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes); 1552 } 1553 1554 return r; 1555 } 1556 1557 /* 1558 * Suspend can't complete until all the I/O is processed so if 1559 * the last path fails we must error any remaining I/O. 1560 * Note that if the freeze_bdev fails while suspending, the 1561 * queue_if_no_path state is lost - userspace should reset it. 1562 */ 1563 static void multipath_presuspend(struct dm_target *ti) 1564 { 1565 struct multipath *m = ti->private; 1566 1567 queue_if_no_path(m, false, true); 1568 } 1569 1570 static void multipath_postsuspend(struct dm_target *ti) 1571 { 1572 struct multipath *m = ti->private; 1573 1574 mutex_lock(&m->work_mutex); 1575 flush_multipath_work(m); 1576 mutex_unlock(&m->work_mutex); 1577 } 1578 1579 /* 1580 * Restore the queue_if_no_path setting. 1581 */ 1582 static void multipath_resume(struct dm_target *ti) 1583 { 1584 struct multipath *m = ti->private; 1585 unsigned long flags; 1586 1587 spin_lock_irqsave(&m->lock, flags); 1588 assign_bit(test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags), 1589 MPATHF_QUEUE_IF_NO_PATH, &m->flags); 1590 spin_unlock_irqrestore(&m->lock, flags); 1591 } 1592 1593 /* 1594 * Info output has the following format: 1595 * num_multipath_feature_args [multipath_feature_args]* 1596 * num_handler_status_args [handler_status_args]* 1597 * num_groups init_group_number 1598 * [A|D|E num_ps_status_args [ps_status_args]* 1599 * num_paths num_selector_args 1600 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1601 * 1602 * Table output has the following format (identical to the constructor string): 1603 * num_feature_args [features_args]* 1604 * num_handler_args hw_handler [hw_handler_args]* 1605 * num_groups init_group_number 1606 * [priority selector-name num_ps_args [ps_args]* 1607 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1608 */ 1609 static void multipath_status(struct dm_target *ti, status_type_t type, 1610 unsigned status_flags, char *result, unsigned maxlen) 1611 { 1612 int sz = 0; 1613 unsigned long flags; 1614 struct multipath *m = ti->private; 1615 struct priority_group *pg; 1616 struct pgpath *p; 1617 unsigned pg_num; 1618 char state; 1619 1620 spin_lock_irqsave(&m->lock, flags); 1621 1622 /* Features */ 1623 if (type == STATUSTYPE_INFO) 1624 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags), 1625 atomic_read(&m->pg_init_count)); 1626 else { 1627 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) + 1628 (m->pg_init_retries > 0) * 2 + 1629 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 + 1630 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) + 1631 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2); 1632 1633 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1634 DMEMIT("queue_if_no_path "); 1635 if (m->pg_init_retries) 1636 DMEMIT("pg_init_retries %u ", m->pg_init_retries); 1637 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) 1638 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs); 1639 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) 1640 DMEMIT("retain_attached_hw_handler "); 1641 if (m->queue_mode != DM_TYPE_REQUEST_BASED) { 1642 switch(m->queue_mode) { 1643 case DM_TYPE_BIO_BASED: 1644 DMEMIT("queue_mode bio "); 1645 break; 1646 case DM_TYPE_MQ_REQUEST_BASED: 1647 DMEMIT("queue_mode mq "); 1648 break; 1649 default: 1650 WARN_ON_ONCE(true); 1651 break; 1652 } 1653 } 1654 } 1655 1656 if (!m->hw_handler_name || type == STATUSTYPE_INFO) 1657 DMEMIT("0 "); 1658 else 1659 DMEMIT("1 %s ", m->hw_handler_name); 1660 1661 DMEMIT("%u ", m->nr_priority_groups); 1662 1663 if (m->next_pg) 1664 pg_num = m->next_pg->pg_num; 1665 else if (m->current_pg) 1666 pg_num = m->current_pg->pg_num; 1667 else 1668 pg_num = (m->nr_priority_groups ? 1 : 0); 1669 1670 DMEMIT("%u ", pg_num); 1671 1672 switch (type) { 1673 case STATUSTYPE_INFO: 1674 list_for_each_entry(pg, &m->priority_groups, list) { 1675 if (pg->bypassed) 1676 state = 'D'; /* Disabled */ 1677 else if (pg == m->current_pg) 1678 state = 'A'; /* Currently Active */ 1679 else 1680 state = 'E'; /* Enabled */ 1681 1682 DMEMIT("%c ", state); 1683 1684 if (pg->ps.type->status) 1685 sz += pg->ps.type->status(&pg->ps, NULL, type, 1686 result + sz, 1687 maxlen - sz); 1688 else 1689 DMEMIT("0 "); 1690 1691 DMEMIT("%u %u ", pg->nr_pgpaths, 1692 pg->ps.type->info_args); 1693 1694 list_for_each_entry(p, &pg->pgpaths, list) { 1695 DMEMIT("%s %s %u ", p->path.dev->name, 1696 p->is_active ? "A" : "F", 1697 p->fail_count); 1698 if (pg->ps.type->status) 1699 sz += pg->ps.type->status(&pg->ps, 1700 &p->path, type, result + sz, 1701 maxlen - sz); 1702 } 1703 } 1704 break; 1705 1706 case STATUSTYPE_TABLE: 1707 list_for_each_entry(pg, &m->priority_groups, list) { 1708 DMEMIT("%s ", pg->ps.type->name); 1709 1710 if (pg->ps.type->status) 1711 sz += pg->ps.type->status(&pg->ps, NULL, type, 1712 result + sz, 1713 maxlen - sz); 1714 else 1715 DMEMIT("0 "); 1716 1717 DMEMIT("%u %u ", pg->nr_pgpaths, 1718 pg->ps.type->table_args); 1719 1720 list_for_each_entry(p, &pg->pgpaths, list) { 1721 DMEMIT("%s ", p->path.dev->name); 1722 if (pg->ps.type->status) 1723 sz += pg->ps.type->status(&pg->ps, 1724 &p->path, type, result + sz, 1725 maxlen - sz); 1726 } 1727 } 1728 break; 1729 } 1730 1731 spin_unlock_irqrestore(&m->lock, flags); 1732 } 1733 1734 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 1735 { 1736 int r = -EINVAL; 1737 struct dm_dev *dev; 1738 struct multipath *m = ti->private; 1739 action_fn action; 1740 1741 mutex_lock(&m->work_mutex); 1742 1743 if (dm_suspended(ti)) { 1744 r = -EBUSY; 1745 goto out; 1746 } 1747 1748 if (argc == 1) { 1749 if (!strcasecmp(argv[0], "queue_if_no_path")) { 1750 r = queue_if_no_path(m, true, false); 1751 goto out; 1752 } else if (!strcasecmp(argv[0], "fail_if_no_path")) { 1753 r = queue_if_no_path(m, false, false); 1754 goto out; 1755 } 1756 } 1757 1758 if (argc != 2) { 1759 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc); 1760 goto out; 1761 } 1762 1763 if (!strcasecmp(argv[0], "disable_group")) { 1764 r = bypass_pg_num(m, argv[1], true); 1765 goto out; 1766 } else if (!strcasecmp(argv[0], "enable_group")) { 1767 r = bypass_pg_num(m, argv[1], false); 1768 goto out; 1769 } else if (!strcasecmp(argv[0], "switch_group")) { 1770 r = switch_pg_num(m, argv[1]); 1771 goto out; 1772 } else if (!strcasecmp(argv[0], "reinstate_path")) 1773 action = reinstate_path; 1774 else if (!strcasecmp(argv[0], "fail_path")) 1775 action = fail_path; 1776 else { 1777 DMWARN("Unrecognised multipath message received: %s", argv[0]); 1778 goto out; 1779 } 1780 1781 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev); 1782 if (r) { 1783 DMWARN("message: error getting device %s", 1784 argv[1]); 1785 goto out; 1786 } 1787 1788 r = action_dev(m, dev, action); 1789 1790 dm_put_device(ti, dev); 1791 1792 out: 1793 mutex_unlock(&m->work_mutex); 1794 return r; 1795 } 1796 1797 static int multipath_prepare_ioctl(struct dm_target *ti, 1798 struct block_device **bdev, fmode_t *mode) 1799 { 1800 struct multipath *m = ti->private; 1801 struct pgpath *current_pgpath; 1802 int r; 1803 1804 current_pgpath = lockless_dereference(m->current_pgpath); 1805 if (!current_pgpath) 1806 current_pgpath = choose_pgpath(m, 0); 1807 1808 if (current_pgpath) { 1809 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) { 1810 *bdev = current_pgpath->path.dev->bdev; 1811 *mode = current_pgpath->path.dev->mode; 1812 r = 0; 1813 } else { 1814 /* pg_init has not started or completed */ 1815 r = -ENOTCONN; 1816 } 1817 } else { 1818 /* No path is available */ 1819 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1820 r = -ENOTCONN; 1821 else 1822 r = -EIO; 1823 } 1824 1825 if (r == -ENOTCONN) { 1826 if (!lockless_dereference(m->current_pg)) { 1827 /* Path status changed, redo selection */ 1828 (void) choose_pgpath(m, 0); 1829 } 1830 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) 1831 pg_init_all_paths(m); 1832 dm_table_run_md_queue_async(m->ti->table); 1833 process_queued_io_list(m); 1834 } 1835 1836 /* 1837 * Only pass ioctls through if the device sizes match exactly. 1838 */ 1839 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT) 1840 return 1; 1841 return r; 1842 } 1843 1844 static int multipath_iterate_devices(struct dm_target *ti, 1845 iterate_devices_callout_fn fn, void *data) 1846 { 1847 struct multipath *m = ti->private; 1848 struct priority_group *pg; 1849 struct pgpath *p; 1850 int ret = 0; 1851 1852 list_for_each_entry(pg, &m->priority_groups, list) { 1853 list_for_each_entry(p, &pg->pgpaths, list) { 1854 ret = fn(ti, p->path.dev, ti->begin, ti->len, data); 1855 if (ret) 1856 goto out; 1857 } 1858 } 1859 1860 out: 1861 return ret; 1862 } 1863 1864 static int pgpath_busy(struct pgpath *pgpath) 1865 { 1866 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev); 1867 1868 return blk_lld_busy(q); 1869 } 1870 1871 /* 1872 * We return "busy", only when we can map I/Os but underlying devices 1873 * are busy (so even if we map I/Os now, the I/Os will wait on 1874 * the underlying queue). 1875 * In other words, if we want to kill I/Os or queue them inside us 1876 * due to map unavailability, we don't return "busy". Otherwise, 1877 * dm core won't give us the I/Os and we can't do what we want. 1878 */ 1879 static int multipath_busy(struct dm_target *ti) 1880 { 1881 bool busy = false, has_active = false; 1882 struct multipath *m = ti->private; 1883 struct priority_group *pg, *next_pg; 1884 struct pgpath *pgpath; 1885 1886 /* pg_init in progress */ 1887 if (atomic_read(&m->pg_init_in_progress)) 1888 return true; 1889 1890 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */ 1891 if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) 1892 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED); 1893 1894 /* Guess which priority_group will be used at next mapping time */ 1895 pg = lockless_dereference(m->current_pg); 1896 next_pg = lockless_dereference(m->next_pg); 1897 if (unlikely(!lockless_dereference(m->current_pgpath) && next_pg)) 1898 pg = next_pg; 1899 1900 if (!pg) { 1901 /* 1902 * We don't know which pg will be used at next mapping time. 1903 * We don't call choose_pgpath() here to avoid to trigger 1904 * pg_init just by busy checking. 1905 * So we don't know whether underlying devices we will be using 1906 * at next mapping time are busy or not. Just try mapping. 1907 */ 1908 return busy; 1909 } 1910 1911 /* 1912 * If there is one non-busy active path at least, the path selector 1913 * will be able to select it. So we consider such a pg as not busy. 1914 */ 1915 busy = true; 1916 list_for_each_entry(pgpath, &pg->pgpaths, list) { 1917 if (pgpath->is_active) { 1918 has_active = true; 1919 if (!pgpath_busy(pgpath)) { 1920 busy = false; 1921 break; 1922 } 1923 } 1924 } 1925 1926 if (!has_active) { 1927 /* 1928 * No active path in this pg, so this pg won't be used and 1929 * the current_pg will be changed at next mapping time. 1930 * We need to try mapping to determine it. 1931 */ 1932 busy = false; 1933 } 1934 1935 return busy; 1936 } 1937 1938 /*----------------------------------------------------------------- 1939 * Module setup 1940 *---------------------------------------------------------------*/ 1941 static struct target_type multipath_target = { 1942 .name = "multipath", 1943 .version = {1, 12, 0}, 1944 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, 1945 .module = THIS_MODULE, 1946 .ctr = multipath_ctr, 1947 .dtr = multipath_dtr, 1948 .clone_and_map_rq = multipath_clone_and_map, 1949 .release_clone_rq = multipath_release_clone, 1950 .rq_end_io = multipath_end_io, 1951 .map = multipath_map_bio, 1952 .end_io = multipath_end_io_bio, 1953 .presuspend = multipath_presuspend, 1954 .postsuspend = multipath_postsuspend, 1955 .resume = multipath_resume, 1956 .status = multipath_status, 1957 .message = multipath_message, 1958 .prepare_ioctl = multipath_prepare_ioctl, 1959 .iterate_devices = multipath_iterate_devices, 1960 .busy = multipath_busy, 1961 }; 1962 1963 static int __init dm_multipath_init(void) 1964 { 1965 int r; 1966 1967 r = dm_register_target(&multipath_target); 1968 if (r < 0) { 1969 DMERR("request-based register failed %d", r); 1970 r = -EINVAL; 1971 goto bad_register_target; 1972 } 1973 1974 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0); 1975 if (!kmultipathd) { 1976 DMERR("failed to create workqueue kmpathd"); 1977 r = -ENOMEM; 1978 goto bad_alloc_kmultipathd; 1979 } 1980 1981 /* 1982 * A separate workqueue is used to handle the device handlers 1983 * to avoid overloading existing workqueue. Overloading the 1984 * old workqueue would also create a bottleneck in the 1985 * path of the storage hardware device activation. 1986 */ 1987 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd", 1988 WQ_MEM_RECLAIM); 1989 if (!kmpath_handlerd) { 1990 DMERR("failed to create workqueue kmpath_handlerd"); 1991 r = -ENOMEM; 1992 goto bad_alloc_kmpath_handlerd; 1993 } 1994 1995 return 0; 1996 1997 bad_alloc_kmpath_handlerd: 1998 destroy_workqueue(kmultipathd); 1999 bad_alloc_kmultipathd: 2000 dm_unregister_target(&multipath_target); 2001 bad_register_target: 2002 return r; 2003 } 2004 2005 static void __exit dm_multipath_exit(void) 2006 { 2007 destroy_workqueue(kmpath_handlerd); 2008 destroy_workqueue(kmultipathd); 2009 2010 dm_unregister_target(&multipath_target); 2011 } 2012 2013 module_init(dm_multipath_init); 2014 module_exit(dm_multipath_exit); 2015 2016 MODULE_DESCRIPTION(DM_NAME " multipath target"); 2017 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 2018 MODULE_LICENSE("GPL"); 2019