1 /* 2 * Copyright (C) 2003 Sistina Software Limited. 3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8 #include <linux/device-mapper.h> 9 10 #include "dm-path-selector.h" 11 #include "dm-bio-record.h" 12 #include "dm-uevent.h" 13 14 #include <linux/ctype.h> 15 #include <linux/init.h> 16 #include <linux/mempool.h> 17 #include <linux/module.h> 18 #include <linux/pagemap.h> 19 #include <linux/slab.h> 20 #include <linux/time.h> 21 #include <linux/workqueue.h> 22 #include <scsi/scsi_dh.h> 23 #include <asm/atomic.h> 24 25 #define DM_MSG_PREFIX "multipath" 26 #define MESG_STR(x) x, sizeof(x) 27 28 /* Path properties */ 29 struct pgpath { 30 struct list_head list; 31 32 struct priority_group *pg; /* Owning PG */ 33 unsigned is_active; /* Path status */ 34 unsigned fail_count; /* Cumulative failure count */ 35 36 struct dm_path path; 37 struct work_struct deactivate_path; 38 }; 39 40 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path) 41 42 /* 43 * Paths are grouped into Priority Groups and numbered from 1 upwards. 44 * Each has a path selector which controls which path gets used. 45 */ 46 struct priority_group { 47 struct list_head list; 48 49 struct multipath *m; /* Owning multipath instance */ 50 struct path_selector ps; 51 52 unsigned pg_num; /* Reference number */ 53 unsigned bypassed; /* Temporarily bypass this PG? */ 54 55 unsigned nr_pgpaths; /* Number of paths in PG */ 56 struct list_head pgpaths; 57 }; 58 59 /* Multipath context */ 60 struct multipath { 61 struct list_head list; 62 struct dm_target *ti; 63 64 spinlock_t lock; 65 66 const char *hw_handler_name; 67 struct work_struct activate_path; 68 struct pgpath *pgpath_to_activate; 69 unsigned nr_priority_groups; 70 struct list_head priority_groups; 71 unsigned pg_init_required; /* pg_init needs calling? */ 72 unsigned pg_init_in_progress; /* Only one pg_init allowed at once */ 73 74 unsigned nr_valid_paths; /* Total number of usable paths */ 75 struct pgpath *current_pgpath; 76 struct priority_group *current_pg; 77 struct priority_group *next_pg; /* Switch to this PG if set */ 78 unsigned repeat_count; /* I/Os left before calling PS again */ 79 80 unsigned queue_io; /* Must we queue all I/O? */ 81 unsigned queue_if_no_path; /* Queue I/O if last path fails? */ 82 unsigned saved_queue_if_no_path;/* Saved state during suspension */ 83 unsigned pg_init_retries; /* Number of times to retry pg_init */ 84 unsigned pg_init_count; /* Number of times pg_init called */ 85 86 struct work_struct process_queued_ios; 87 struct bio_list queued_ios; 88 unsigned queue_size; 89 90 struct work_struct trigger_event; 91 92 /* 93 * We must use a mempool of dm_mpath_io structs so that we 94 * can resubmit bios on error. 95 */ 96 mempool_t *mpio_pool; 97 }; 98 99 /* 100 * Context information attached to each bio we process. 101 */ 102 struct dm_mpath_io { 103 struct pgpath *pgpath; 104 struct dm_bio_details details; 105 }; 106 107 typedef int (*action_fn) (struct pgpath *pgpath); 108 109 #define MIN_IOS 256 /* Mempool size */ 110 111 static struct kmem_cache *_mpio_cache; 112 113 static struct workqueue_struct *kmultipathd, *kmpath_handlerd; 114 static void process_queued_ios(struct work_struct *work); 115 static void trigger_event(struct work_struct *work); 116 static void activate_path(struct work_struct *work); 117 static void deactivate_path(struct work_struct *work); 118 119 120 /*----------------------------------------------- 121 * Allocation routines 122 *-----------------------------------------------*/ 123 124 static struct pgpath *alloc_pgpath(void) 125 { 126 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL); 127 128 if (pgpath) { 129 pgpath->is_active = 1; 130 INIT_WORK(&pgpath->deactivate_path, deactivate_path); 131 } 132 133 return pgpath; 134 } 135 136 static void free_pgpath(struct pgpath *pgpath) 137 { 138 kfree(pgpath); 139 } 140 141 static void deactivate_path(struct work_struct *work) 142 { 143 struct pgpath *pgpath = 144 container_of(work, struct pgpath, deactivate_path); 145 146 blk_abort_queue(pgpath->path.dev->bdev->bd_disk->queue); 147 } 148 149 static struct priority_group *alloc_priority_group(void) 150 { 151 struct priority_group *pg; 152 153 pg = kzalloc(sizeof(*pg), GFP_KERNEL); 154 155 if (pg) 156 INIT_LIST_HEAD(&pg->pgpaths); 157 158 return pg; 159 } 160 161 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti) 162 { 163 unsigned long flags; 164 struct pgpath *pgpath, *tmp; 165 struct multipath *m = ti->private; 166 167 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) { 168 list_del(&pgpath->list); 169 if (m->hw_handler_name) 170 scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev)); 171 dm_put_device(ti, pgpath->path.dev); 172 spin_lock_irqsave(&m->lock, flags); 173 if (m->pgpath_to_activate == pgpath) 174 m->pgpath_to_activate = NULL; 175 spin_unlock_irqrestore(&m->lock, flags); 176 free_pgpath(pgpath); 177 } 178 } 179 180 static void free_priority_group(struct priority_group *pg, 181 struct dm_target *ti) 182 { 183 struct path_selector *ps = &pg->ps; 184 185 if (ps->type) { 186 ps->type->destroy(ps); 187 dm_put_path_selector(ps->type); 188 } 189 190 free_pgpaths(&pg->pgpaths, ti); 191 kfree(pg); 192 } 193 194 static struct multipath *alloc_multipath(struct dm_target *ti) 195 { 196 struct multipath *m; 197 198 m = kzalloc(sizeof(*m), GFP_KERNEL); 199 if (m) { 200 INIT_LIST_HEAD(&m->priority_groups); 201 spin_lock_init(&m->lock); 202 m->queue_io = 1; 203 INIT_WORK(&m->process_queued_ios, process_queued_ios); 204 INIT_WORK(&m->trigger_event, trigger_event); 205 INIT_WORK(&m->activate_path, activate_path); 206 m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache); 207 if (!m->mpio_pool) { 208 kfree(m); 209 return NULL; 210 } 211 m->ti = ti; 212 ti->private = m; 213 } 214 215 return m; 216 } 217 218 static void free_multipath(struct multipath *m) 219 { 220 struct priority_group *pg, *tmp; 221 222 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) { 223 list_del(&pg->list); 224 free_priority_group(pg, m->ti); 225 } 226 227 kfree(m->hw_handler_name); 228 mempool_destroy(m->mpio_pool); 229 kfree(m); 230 } 231 232 233 /*----------------------------------------------- 234 * Path selection 235 *-----------------------------------------------*/ 236 237 static void __switch_pg(struct multipath *m, struct pgpath *pgpath) 238 { 239 m->current_pg = pgpath->pg; 240 241 /* Must we initialise the PG first, and queue I/O till it's ready? */ 242 if (m->hw_handler_name) { 243 m->pg_init_required = 1; 244 m->queue_io = 1; 245 } else { 246 m->pg_init_required = 0; 247 m->queue_io = 0; 248 } 249 250 m->pg_init_count = 0; 251 } 252 253 static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg) 254 { 255 struct dm_path *path; 256 257 path = pg->ps.type->select_path(&pg->ps, &m->repeat_count); 258 if (!path) 259 return -ENXIO; 260 261 m->current_pgpath = path_to_pgpath(path); 262 263 if (m->current_pg != pg) 264 __switch_pg(m, m->current_pgpath); 265 266 return 0; 267 } 268 269 static void __choose_pgpath(struct multipath *m) 270 { 271 struct priority_group *pg; 272 unsigned bypassed = 1; 273 274 if (!m->nr_valid_paths) 275 goto failed; 276 277 /* Were we instructed to switch PG? */ 278 if (m->next_pg) { 279 pg = m->next_pg; 280 m->next_pg = NULL; 281 if (!__choose_path_in_pg(m, pg)) 282 return; 283 } 284 285 /* Don't change PG until it has no remaining paths */ 286 if (m->current_pg && !__choose_path_in_pg(m, m->current_pg)) 287 return; 288 289 /* 290 * Loop through priority groups until we find a valid path. 291 * First time we skip PGs marked 'bypassed'. 292 * Second time we only try the ones we skipped. 293 */ 294 do { 295 list_for_each_entry(pg, &m->priority_groups, list) { 296 if (pg->bypassed == bypassed) 297 continue; 298 if (!__choose_path_in_pg(m, pg)) 299 return; 300 } 301 } while (bypassed--); 302 303 failed: 304 m->current_pgpath = NULL; 305 m->current_pg = NULL; 306 } 307 308 /* 309 * Check whether bios must be queued in the device-mapper core rather 310 * than here in the target. 311 * 312 * m->lock must be held on entry. 313 * 314 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the 315 * same value then we are not between multipath_presuspend() 316 * and multipath_resume() calls and we have no need to check 317 * for the DMF_NOFLUSH_SUSPENDING flag. 318 */ 319 static int __must_push_back(struct multipath *m) 320 { 321 return (m->queue_if_no_path != m->saved_queue_if_no_path && 322 dm_noflush_suspending(m->ti)); 323 } 324 325 static int map_io(struct multipath *m, struct bio *bio, 326 struct dm_mpath_io *mpio, unsigned was_queued) 327 { 328 int r = DM_MAPIO_REMAPPED; 329 unsigned long flags; 330 struct pgpath *pgpath; 331 332 spin_lock_irqsave(&m->lock, flags); 333 334 /* Do we need to select a new pgpath? */ 335 if (!m->current_pgpath || 336 (!m->queue_io && (m->repeat_count && --m->repeat_count == 0))) 337 __choose_pgpath(m); 338 339 pgpath = m->current_pgpath; 340 341 if (was_queued) 342 m->queue_size--; 343 344 if ((pgpath && m->queue_io) || 345 (!pgpath && m->queue_if_no_path)) { 346 /* Queue for the daemon to resubmit */ 347 bio_list_add(&m->queued_ios, bio); 348 m->queue_size++; 349 if ((m->pg_init_required && !m->pg_init_in_progress) || 350 !m->queue_io) 351 queue_work(kmultipathd, &m->process_queued_ios); 352 pgpath = NULL; 353 r = DM_MAPIO_SUBMITTED; 354 } else if (pgpath) 355 bio->bi_bdev = pgpath->path.dev->bdev; 356 else if (__must_push_back(m)) 357 r = DM_MAPIO_REQUEUE; 358 else 359 r = -EIO; /* Failed */ 360 361 mpio->pgpath = pgpath; 362 363 spin_unlock_irqrestore(&m->lock, flags); 364 365 return r; 366 } 367 368 /* 369 * If we run out of usable paths, should we queue I/O or error it? 370 */ 371 static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path, 372 unsigned save_old_value) 373 { 374 unsigned long flags; 375 376 spin_lock_irqsave(&m->lock, flags); 377 378 if (save_old_value) 379 m->saved_queue_if_no_path = m->queue_if_no_path; 380 else 381 m->saved_queue_if_no_path = queue_if_no_path; 382 m->queue_if_no_path = queue_if_no_path; 383 if (!m->queue_if_no_path && m->queue_size) 384 queue_work(kmultipathd, &m->process_queued_ios); 385 386 spin_unlock_irqrestore(&m->lock, flags); 387 388 return 0; 389 } 390 391 /*----------------------------------------------------------------- 392 * The multipath daemon is responsible for resubmitting queued ios. 393 *---------------------------------------------------------------*/ 394 395 static void dispatch_queued_ios(struct multipath *m) 396 { 397 int r; 398 unsigned long flags; 399 struct bio *bio = NULL, *next; 400 struct dm_mpath_io *mpio; 401 union map_info *info; 402 403 spin_lock_irqsave(&m->lock, flags); 404 bio = bio_list_get(&m->queued_ios); 405 spin_unlock_irqrestore(&m->lock, flags); 406 407 while (bio) { 408 next = bio->bi_next; 409 bio->bi_next = NULL; 410 411 info = dm_get_mapinfo(bio); 412 mpio = info->ptr; 413 414 r = map_io(m, bio, mpio, 1); 415 if (r < 0) 416 bio_endio(bio, r); 417 else if (r == DM_MAPIO_REMAPPED) 418 generic_make_request(bio); 419 else if (r == DM_MAPIO_REQUEUE) 420 bio_endio(bio, -EIO); 421 422 bio = next; 423 } 424 } 425 426 static void process_queued_ios(struct work_struct *work) 427 { 428 struct multipath *m = 429 container_of(work, struct multipath, process_queued_ios); 430 struct pgpath *pgpath = NULL; 431 unsigned init_required = 0, must_queue = 1; 432 unsigned long flags; 433 434 spin_lock_irqsave(&m->lock, flags); 435 436 if (!m->queue_size) 437 goto out; 438 439 if (!m->current_pgpath) 440 __choose_pgpath(m); 441 442 pgpath = m->current_pgpath; 443 444 if ((pgpath && !m->queue_io) || 445 (!pgpath && !m->queue_if_no_path)) 446 must_queue = 0; 447 448 if (m->pg_init_required && !m->pg_init_in_progress && pgpath) { 449 m->pgpath_to_activate = pgpath; 450 m->pg_init_count++; 451 m->pg_init_required = 0; 452 m->pg_init_in_progress = 1; 453 init_required = 1; 454 } 455 456 out: 457 spin_unlock_irqrestore(&m->lock, flags); 458 459 if (init_required) 460 queue_work(kmpath_handlerd, &m->activate_path); 461 462 if (!must_queue) 463 dispatch_queued_ios(m); 464 } 465 466 /* 467 * An event is triggered whenever a path is taken out of use. 468 * Includes path failure and PG bypass. 469 */ 470 static void trigger_event(struct work_struct *work) 471 { 472 struct multipath *m = 473 container_of(work, struct multipath, trigger_event); 474 475 dm_table_event(m->ti->table); 476 } 477 478 /*----------------------------------------------------------------- 479 * Constructor/argument parsing: 480 * <#multipath feature args> [<arg>]* 481 * <#hw_handler args> [hw_handler [<arg>]*] 482 * <#priority groups> 483 * <initial priority group> 484 * [<selector> <#selector args> [<arg>]* 485 * <#paths> <#per-path selector args> 486 * [<path> [<arg>]* ]+ ]+ 487 *---------------------------------------------------------------*/ 488 struct param { 489 unsigned min; 490 unsigned max; 491 char *error; 492 }; 493 494 static int read_param(struct param *param, char *str, unsigned *v, char **error) 495 { 496 if (!str || 497 (sscanf(str, "%u", v) != 1) || 498 (*v < param->min) || 499 (*v > param->max)) { 500 *error = param->error; 501 return -EINVAL; 502 } 503 504 return 0; 505 } 506 507 struct arg_set { 508 unsigned argc; 509 char **argv; 510 }; 511 512 static char *shift(struct arg_set *as) 513 { 514 char *r; 515 516 if (as->argc) { 517 as->argc--; 518 r = *as->argv; 519 as->argv++; 520 return r; 521 } 522 523 return NULL; 524 } 525 526 static void consume(struct arg_set *as, unsigned n) 527 { 528 BUG_ON (as->argc < n); 529 as->argc -= n; 530 as->argv += n; 531 } 532 533 static int parse_path_selector(struct arg_set *as, struct priority_group *pg, 534 struct dm_target *ti) 535 { 536 int r; 537 struct path_selector_type *pst; 538 unsigned ps_argc; 539 540 static struct param _params[] = { 541 {0, 1024, "invalid number of path selector args"}, 542 }; 543 544 pst = dm_get_path_selector(shift(as)); 545 if (!pst) { 546 ti->error = "unknown path selector type"; 547 return -EINVAL; 548 } 549 550 r = read_param(_params, shift(as), &ps_argc, &ti->error); 551 if (r) { 552 dm_put_path_selector(pst); 553 return -EINVAL; 554 } 555 556 r = pst->create(&pg->ps, ps_argc, as->argv); 557 if (r) { 558 dm_put_path_selector(pst); 559 ti->error = "path selector constructor failed"; 560 return r; 561 } 562 563 pg->ps.type = pst; 564 consume(as, ps_argc); 565 566 return 0; 567 } 568 569 static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps, 570 struct dm_target *ti) 571 { 572 int r; 573 struct pgpath *p; 574 struct multipath *m = ti->private; 575 576 /* we need at least a path arg */ 577 if (as->argc < 1) { 578 ti->error = "no device given"; 579 return ERR_PTR(-EINVAL); 580 } 581 582 p = alloc_pgpath(); 583 if (!p) 584 return ERR_PTR(-ENOMEM); 585 586 r = dm_get_device(ti, shift(as), ti->begin, ti->len, 587 dm_table_get_mode(ti->table), &p->path.dev); 588 if (r) { 589 ti->error = "error getting device"; 590 goto bad; 591 } 592 593 if (m->hw_handler_name) { 594 r = scsi_dh_attach(bdev_get_queue(p->path.dev->bdev), 595 m->hw_handler_name); 596 if (r < 0) { 597 dm_put_device(ti, p->path.dev); 598 goto bad; 599 } 600 } 601 602 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error); 603 if (r) { 604 dm_put_device(ti, p->path.dev); 605 goto bad; 606 } 607 608 return p; 609 610 bad: 611 free_pgpath(p); 612 return ERR_PTR(r); 613 } 614 615 static struct priority_group *parse_priority_group(struct arg_set *as, 616 struct multipath *m) 617 { 618 static struct param _params[] = { 619 {1, 1024, "invalid number of paths"}, 620 {0, 1024, "invalid number of selector args"} 621 }; 622 623 int r; 624 unsigned i, nr_selector_args, nr_params; 625 struct priority_group *pg; 626 struct dm_target *ti = m->ti; 627 628 if (as->argc < 2) { 629 as->argc = 0; 630 ti->error = "not enough priority group arguments"; 631 return ERR_PTR(-EINVAL); 632 } 633 634 pg = alloc_priority_group(); 635 if (!pg) { 636 ti->error = "couldn't allocate priority group"; 637 return ERR_PTR(-ENOMEM); 638 } 639 pg->m = m; 640 641 r = parse_path_selector(as, pg, ti); 642 if (r) 643 goto bad; 644 645 /* 646 * read the paths 647 */ 648 r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error); 649 if (r) 650 goto bad; 651 652 r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error); 653 if (r) 654 goto bad; 655 656 nr_params = 1 + nr_selector_args; 657 for (i = 0; i < pg->nr_pgpaths; i++) { 658 struct pgpath *pgpath; 659 struct arg_set path_args; 660 661 if (as->argc < nr_params) { 662 ti->error = "not enough path parameters"; 663 goto bad; 664 } 665 666 path_args.argc = nr_params; 667 path_args.argv = as->argv; 668 669 pgpath = parse_path(&path_args, &pg->ps, ti); 670 if (IS_ERR(pgpath)) { 671 r = PTR_ERR(pgpath); 672 goto bad; 673 } 674 675 pgpath->pg = pg; 676 list_add_tail(&pgpath->list, &pg->pgpaths); 677 consume(as, nr_params); 678 } 679 680 return pg; 681 682 bad: 683 free_priority_group(pg, ti); 684 return ERR_PTR(r); 685 } 686 687 static int parse_hw_handler(struct arg_set *as, struct multipath *m) 688 { 689 unsigned hw_argc; 690 struct dm_target *ti = m->ti; 691 692 static struct param _params[] = { 693 {0, 1024, "invalid number of hardware handler args"}, 694 }; 695 696 if (read_param(_params, shift(as), &hw_argc, &ti->error)) 697 return -EINVAL; 698 699 if (!hw_argc) 700 return 0; 701 702 m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL); 703 request_module("scsi_dh_%s", m->hw_handler_name); 704 if (scsi_dh_handler_exist(m->hw_handler_name) == 0) { 705 ti->error = "unknown hardware handler type"; 706 kfree(m->hw_handler_name); 707 m->hw_handler_name = NULL; 708 return -EINVAL; 709 } 710 711 if (hw_argc > 1) 712 DMWARN("Ignoring user-specified arguments for " 713 "hardware handler \"%s\"", m->hw_handler_name); 714 consume(as, hw_argc - 1); 715 716 return 0; 717 } 718 719 static int parse_features(struct arg_set *as, struct multipath *m) 720 { 721 int r; 722 unsigned argc; 723 struct dm_target *ti = m->ti; 724 const char *param_name; 725 726 static struct param _params[] = { 727 {0, 3, "invalid number of feature args"}, 728 {1, 50, "pg_init_retries must be between 1 and 50"}, 729 }; 730 731 r = read_param(_params, shift(as), &argc, &ti->error); 732 if (r) 733 return -EINVAL; 734 735 if (!argc) 736 return 0; 737 738 do { 739 param_name = shift(as); 740 argc--; 741 742 if (!strnicmp(param_name, MESG_STR("queue_if_no_path"))) { 743 r = queue_if_no_path(m, 1, 0); 744 continue; 745 } 746 747 if (!strnicmp(param_name, MESG_STR("pg_init_retries")) && 748 (argc >= 1)) { 749 r = read_param(_params + 1, shift(as), 750 &m->pg_init_retries, &ti->error); 751 argc--; 752 continue; 753 } 754 755 ti->error = "Unrecognised multipath feature request"; 756 r = -EINVAL; 757 } while (argc && !r); 758 759 return r; 760 } 761 762 static int multipath_ctr(struct dm_target *ti, unsigned int argc, 763 char **argv) 764 { 765 /* target parameters */ 766 static struct param _params[] = { 767 {1, 1024, "invalid number of priority groups"}, 768 {1, 1024, "invalid initial priority group number"}, 769 }; 770 771 int r; 772 struct multipath *m; 773 struct arg_set as; 774 unsigned pg_count = 0; 775 unsigned next_pg_num; 776 777 as.argc = argc; 778 as.argv = argv; 779 780 m = alloc_multipath(ti); 781 if (!m) { 782 ti->error = "can't allocate multipath"; 783 return -EINVAL; 784 } 785 786 r = parse_features(&as, m); 787 if (r) 788 goto bad; 789 790 r = parse_hw_handler(&as, m); 791 if (r) 792 goto bad; 793 794 r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error); 795 if (r) 796 goto bad; 797 798 r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error); 799 if (r) 800 goto bad; 801 802 /* parse the priority groups */ 803 while (as.argc) { 804 struct priority_group *pg; 805 806 pg = parse_priority_group(&as, m); 807 if (IS_ERR(pg)) { 808 r = PTR_ERR(pg); 809 goto bad; 810 } 811 812 m->nr_valid_paths += pg->nr_pgpaths; 813 list_add_tail(&pg->list, &m->priority_groups); 814 pg_count++; 815 pg->pg_num = pg_count; 816 if (!--next_pg_num) 817 m->next_pg = pg; 818 } 819 820 if (pg_count != m->nr_priority_groups) { 821 ti->error = "priority group count mismatch"; 822 r = -EINVAL; 823 goto bad; 824 } 825 826 return 0; 827 828 bad: 829 free_multipath(m); 830 return r; 831 } 832 833 static void multipath_dtr(struct dm_target *ti) 834 { 835 struct multipath *m = (struct multipath *) ti->private; 836 837 flush_workqueue(kmpath_handlerd); 838 flush_workqueue(kmultipathd); 839 free_multipath(m); 840 } 841 842 /* 843 * Map bios, recording original fields for later in case we have to resubmit 844 */ 845 static int multipath_map(struct dm_target *ti, struct bio *bio, 846 union map_info *map_context) 847 { 848 int r; 849 struct dm_mpath_io *mpio; 850 struct multipath *m = (struct multipath *) ti->private; 851 852 mpio = mempool_alloc(m->mpio_pool, GFP_NOIO); 853 dm_bio_record(&mpio->details, bio); 854 855 map_context->ptr = mpio; 856 bio->bi_rw |= (1 << BIO_RW_FAILFAST_TRANSPORT); 857 r = map_io(m, bio, mpio, 0); 858 if (r < 0 || r == DM_MAPIO_REQUEUE) 859 mempool_free(mpio, m->mpio_pool); 860 861 return r; 862 } 863 864 /* 865 * Take a path out of use. 866 */ 867 static int fail_path(struct pgpath *pgpath) 868 { 869 unsigned long flags; 870 struct multipath *m = pgpath->pg->m; 871 872 spin_lock_irqsave(&m->lock, flags); 873 874 if (!pgpath->is_active) 875 goto out; 876 877 DMWARN("Failing path %s.", pgpath->path.dev->name); 878 879 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path); 880 pgpath->is_active = 0; 881 pgpath->fail_count++; 882 883 m->nr_valid_paths--; 884 885 if (pgpath == m->current_pgpath) 886 m->current_pgpath = NULL; 887 888 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti, 889 pgpath->path.dev->name, m->nr_valid_paths); 890 891 schedule_work(&m->trigger_event); 892 queue_work(kmultipathd, &pgpath->deactivate_path); 893 894 out: 895 spin_unlock_irqrestore(&m->lock, flags); 896 897 return 0; 898 } 899 900 /* 901 * Reinstate a previously-failed path 902 */ 903 static int reinstate_path(struct pgpath *pgpath) 904 { 905 int r = 0; 906 unsigned long flags; 907 struct multipath *m = pgpath->pg->m; 908 909 spin_lock_irqsave(&m->lock, flags); 910 911 if (pgpath->is_active) 912 goto out; 913 914 if (!pgpath->pg->ps.type->reinstate_path) { 915 DMWARN("Reinstate path not supported by path selector %s", 916 pgpath->pg->ps.type->name); 917 r = -EINVAL; 918 goto out; 919 } 920 921 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path); 922 if (r) 923 goto out; 924 925 pgpath->is_active = 1; 926 927 m->current_pgpath = NULL; 928 if (!m->nr_valid_paths++ && m->queue_size) 929 queue_work(kmultipathd, &m->process_queued_ios); 930 931 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti, 932 pgpath->path.dev->name, m->nr_valid_paths); 933 934 schedule_work(&m->trigger_event); 935 936 out: 937 spin_unlock_irqrestore(&m->lock, flags); 938 939 return r; 940 } 941 942 /* 943 * Fail or reinstate all paths that match the provided struct dm_dev. 944 */ 945 static int action_dev(struct multipath *m, struct dm_dev *dev, 946 action_fn action) 947 { 948 int r = 0; 949 struct pgpath *pgpath; 950 struct priority_group *pg; 951 952 list_for_each_entry(pg, &m->priority_groups, list) { 953 list_for_each_entry(pgpath, &pg->pgpaths, list) { 954 if (pgpath->path.dev == dev) 955 r = action(pgpath); 956 } 957 } 958 959 return r; 960 } 961 962 /* 963 * Temporarily try to avoid having to use the specified PG 964 */ 965 static void bypass_pg(struct multipath *m, struct priority_group *pg, 966 int bypassed) 967 { 968 unsigned long flags; 969 970 spin_lock_irqsave(&m->lock, flags); 971 972 pg->bypassed = bypassed; 973 m->current_pgpath = NULL; 974 m->current_pg = NULL; 975 976 spin_unlock_irqrestore(&m->lock, flags); 977 978 schedule_work(&m->trigger_event); 979 } 980 981 /* 982 * Switch to using the specified PG from the next I/O that gets mapped 983 */ 984 static int switch_pg_num(struct multipath *m, const char *pgstr) 985 { 986 struct priority_group *pg; 987 unsigned pgnum; 988 unsigned long flags; 989 990 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum || 991 (pgnum > m->nr_priority_groups)) { 992 DMWARN("invalid PG number supplied to switch_pg_num"); 993 return -EINVAL; 994 } 995 996 spin_lock_irqsave(&m->lock, flags); 997 list_for_each_entry(pg, &m->priority_groups, list) { 998 pg->bypassed = 0; 999 if (--pgnum) 1000 continue; 1001 1002 m->current_pgpath = NULL; 1003 m->current_pg = NULL; 1004 m->next_pg = pg; 1005 } 1006 spin_unlock_irqrestore(&m->lock, flags); 1007 1008 schedule_work(&m->trigger_event); 1009 return 0; 1010 } 1011 1012 /* 1013 * Set/clear bypassed status of a PG. 1014 * PGs are numbered upwards from 1 in the order they were declared. 1015 */ 1016 static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed) 1017 { 1018 struct priority_group *pg; 1019 unsigned pgnum; 1020 1021 if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum || 1022 (pgnum > m->nr_priority_groups)) { 1023 DMWARN("invalid PG number supplied to bypass_pg"); 1024 return -EINVAL; 1025 } 1026 1027 list_for_each_entry(pg, &m->priority_groups, list) { 1028 if (!--pgnum) 1029 break; 1030 } 1031 1032 bypass_pg(m, pg, bypassed); 1033 return 0; 1034 } 1035 1036 /* 1037 * Should we retry pg_init immediately? 1038 */ 1039 static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath) 1040 { 1041 unsigned long flags; 1042 int limit_reached = 0; 1043 1044 spin_lock_irqsave(&m->lock, flags); 1045 1046 if (m->pg_init_count <= m->pg_init_retries) 1047 m->pg_init_required = 1; 1048 else 1049 limit_reached = 1; 1050 1051 spin_unlock_irqrestore(&m->lock, flags); 1052 1053 return limit_reached; 1054 } 1055 1056 static void pg_init_done(struct dm_path *path, int errors) 1057 { 1058 struct pgpath *pgpath = path_to_pgpath(path); 1059 struct priority_group *pg = pgpath->pg; 1060 struct multipath *m = pg->m; 1061 unsigned long flags; 1062 1063 /* device or driver problems */ 1064 switch (errors) { 1065 case SCSI_DH_OK: 1066 break; 1067 case SCSI_DH_NOSYS: 1068 if (!m->hw_handler_name) { 1069 errors = 0; 1070 break; 1071 } 1072 DMERR("Cannot failover device because scsi_dh_%s was not " 1073 "loaded.", m->hw_handler_name); 1074 /* 1075 * Fail path for now, so we do not ping pong 1076 */ 1077 fail_path(pgpath); 1078 break; 1079 case SCSI_DH_DEV_TEMP_BUSY: 1080 /* 1081 * Probably doing something like FW upgrade on the 1082 * controller so try the other pg. 1083 */ 1084 bypass_pg(m, pg, 1); 1085 break; 1086 /* TODO: For SCSI_DH_RETRY we should wait a couple seconds */ 1087 case SCSI_DH_RETRY: 1088 case SCSI_DH_IMM_RETRY: 1089 case SCSI_DH_RES_TEMP_UNAVAIL: 1090 if (pg_init_limit_reached(m, pgpath)) 1091 fail_path(pgpath); 1092 errors = 0; 1093 break; 1094 default: 1095 /* 1096 * We probably do not want to fail the path for a device 1097 * error, but this is what the old dm did. In future 1098 * patches we can do more advanced handling. 1099 */ 1100 fail_path(pgpath); 1101 } 1102 1103 spin_lock_irqsave(&m->lock, flags); 1104 if (errors) { 1105 DMERR("Could not failover device. Error %d.", errors); 1106 m->current_pgpath = NULL; 1107 m->current_pg = NULL; 1108 } else if (!m->pg_init_required) { 1109 m->queue_io = 0; 1110 pg->bypassed = 0; 1111 } 1112 1113 m->pg_init_in_progress = 0; 1114 queue_work(kmultipathd, &m->process_queued_ios); 1115 spin_unlock_irqrestore(&m->lock, flags); 1116 } 1117 1118 static void activate_path(struct work_struct *work) 1119 { 1120 int ret; 1121 struct multipath *m = 1122 container_of(work, struct multipath, activate_path); 1123 struct dm_path *path; 1124 unsigned long flags; 1125 1126 spin_lock_irqsave(&m->lock, flags); 1127 path = &m->pgpath_to_activate->path; 1128 m->pgpath_to_activate = NULL; 1129 spin_unlock_irqrestore(&m->lock, flags); 1130 if (!path) 1131 return; 1132 ret = scsi_dh_activate(bdev_get_queue(path->dev->bdev)); 1133 pg_init_done(path, ret); 1134 } 1135 1136 /* 1137 * end_io handling 1138 */ 1139 static int do_end_io(struct multipath *m, struct bio *bio, 1140 int error, struct dm_mpath_io *mpio) 1141 { 1142 unsigned long flags; 1143 1144 if (!error) 1145 return 0; /* I/O complete */ 1146 1147 if ((error == -EWOULDBLOCK) && bio_rw_ahead(bio)) 1148 return error; 1149 1150 if (error == -EOPNOTSUPP) 1151 return error; 1152 1153 spin_lock_irqsave(&m->lock, flags); 1154 if (!m->nr_valid_paths) { 1155 if (__must_push_back(m)) { 1156 spin_unlock_irqrestore(&m->lock, flags); 1157 return DM_ENDIO_REQUEUE; 1158 } else if (!m->queue_if_no_path) { 1159 spin_unlock_irqrestore(&m->lock, flags); 1160 return -EIO; 1161 } else { 1162 spin_unlock_irqrestore(&m->lock, flags); 1163 goto requeue; 1164 } 1165 } 1166 spin_unlock_irqrestore(&m->lock, flags); 1167 1168 if (mpio->pgpath) 1169 fail_path(mpio->pgpath); 1170 1171 requeue: 1172 dm_bio_restore(&mpio->details, bio); 1173 1174 /* queue for the daemon to resubmit or fail */ 1175 spin_lock_irqsave(&m->lock, flags); 1176 bio_list_add(&m->queued_ios, bio); 1177 m->queue_size++; 1178 if (!m->queue_io) 1179 queue_work(kmultipathd, &m->process_queued_ios); 1180 spin_unlock_irqrestore(&m->lock, flags); 1181 1182 return DM_ENDIO_INCOMPLETE; /* io not complete */ 1183 } 1184 1185 static int multipath_end_io(struct dm_target *ti, struct bio *bio, 1186 int error, union map_info *map_context) 1187 { 1188 struct multipath *m = ti->private; 1189 struct dm_mpath_io *mpio = map_context->ptr; 1190 struct pgpath *pgpath = mpio->pgpath; 1191 struct path_selector *ps; 1192 int r; 1193 1194 r = do_end_io(m, bio, error, mpio); 1195 if (pgpath) { 1196 ps = &pgpath->pg->ps; 1197 if (ps->type->end_io) 1198 ps->type->end_io(ps, &pgpath->path); 1199 } 1200 if (r != DM_ENDIO_INCOMPLETE) 1201 mempool_free(mpio, m->mpio_pool); 1202 1203 return r; 1204 } 1205 1206 /* 1207 * Suspend can't complete until all the I/O is processed so if 1208 * the last path fails we must error any remaining I/O. 1209 * Note that if the freeze_bdev fails while suspending, the 1210 * queue_if_no_path state is lost - userspace should reset it. 1211 */ 1212 static void multipath_presuspend(struct dm_target *ti) 1213 { 1214 struct multipath *m = (struct multipath *) ti->private; 1215 1216 queue_if_no_path(m, 0, 1); 1217 } 1218 1219 /* 1220 * Restore the queue_if_no_path setting. 1221 */ 1222 static void multipath_resume(struct dm_target *ti) 1223 { 1224 struct multipath *m = (struct multipath *) ti->private; 1225 unsigned long flags; 1226 1227 spin_lock_irqsave(&m->lock, flags); 1228 m->queue_if_no_path = m->saved_queue_if_no_path; 1229 spin_unlock_irqrestore(&m->lock, flags); 1230 } 1231 1232 /* 1233 * Info output has the following format: 1234 * num_multipath_feature_args [multipath_feature_args]* 1235 * num_handler_status_args [handler_status_args]* 1236 * num_groups init_group_number 1237 * [A|D|E num_ps_status_args [ps_status_args]* 1238 * num_paths num_selector_args 1239 * [path_dev A|F fail_count [selector_args]* ]+ ]+ 1240 * 1241 * Table output has the following format (identical to the constructor string): 1242 * num_feature_args [features_args]* 1243 * num_handler_args hw_handler [hw_handler_args]* 1244 * num_groups init_group_number 1245 * [priority selector-name num_ps_args [ps_args]* 1246 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+ 1247 */ 1248 static int multipath_status(struct dm_target *ti, status_type_t type, 1249 char *result, unsigned int maxlen) 1250 { 1251 int sz = 0; 1252 unsigned long flags; 1253 struct multipath *m = (struct multipath *) ti->private; 1254 struct priority_group *pg; 1255 struct pgpath *p; 1256 unsigned pg_num; 1257 char state; 1258 1259 spin_lock_irqsave(&m->lock, flags); 1260 1261 /* Features */ 1262 if (type == STATUSTYPE_INFO) 1263 DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count); 1264 else { 1265 DMEMIT("%u ", m->queue_if_no_path + 1266 (m->pg_init_retries > 0) * 2); 1267 if (m->queue_if_no_path) 1268 DMEMIT("queue_if_no_path "); 1269 if (m->pg_init_retries) 1270 DMEMIT("pg_init_retries %u ", m->pg_init_retries); 1271 } 1272 1273 if (!m->hw_handler_name || type == STATUSTYPE_INFO) 1274 DMEMIT("0 "); 1275 else 1276 DMEMIT("1 %s ", m->hw_handler_name); 1277 1278 DMEMIT("%u ", m->nr_priority_groups); 1279 1280 if (m->next_pg) 1281 pg_num = m->next_pg->pg_num; 1282 else if (m->current_pg) 1283 pg_num = m->current_pg->pg_num; 1284 else 1285 pg_num = 1; 1286 1287 DMEMIT("%u ", pg_num); 1288 1289 switch (type) { 1290 case STATUSTYPE_INFO: 1291 list_for_each_entry(pg, &m->priority_groups, list) { 1292 if (pg->bypassed) 1293 state = 'D'; /* Disabled */ 1294 else if (pg == m->current_pg) 1295 state = 'A'; /* Currently Active */ 1296 else 1297 state = 'E'; /* Enabled */ 1298 1299 DMEMIT("%c ", state); 1300 1301 if (pg->ps.type->status) 1302 sz += pg->ps.type->status(&pg->ps, NULL, type, 1303 result + sz, 1304 maxlen - sz); 1305 else 1306 DMEMIT("0 "); 1307 1308 DMEMIT("%u %u ", pg->nr_pgpaths, 1309 pg->ps.type->info_args); 1310 1311 list_for_each_entry(p, &pg->pgpaths, list) { 1312 DMEMIT("%s %s %u ", p->path.dev->name, 1313 p->is_active ? "A" : "F", 1314 p->fail_count); 1315 if (pg->ps.type->status) 1316 sz += pg->ps.type->status(&pg->ps, 1317 &p->path, type, result + sz, 1318 maxlen - sz); 1319 } 1320 } 1321 break; 1322 1323 case STATUSTYPE_TABLE: 1324 list_for_each_entry(pg, &m->priority_groups, list) { 1325 DMEMIT("%s ", pg->ps.type->name); 1326 1327 if (pg->ps.type->status) 1328 sz += pg->ps.type->status(&pg->ps, NULL, type, 1329 result + sz, 1330 maxlen - sz); 1331 else 1332 DMEMIT("0 "); 1333 1334 DMEMIT("%u %u ", pg->nr_pgpaths, 1335 pg->ps.type->table_args); 1336 1337 list_for_each_entry(p, &pg->pgpaths, list) { 1338 DMEMIT("%s ", p->path.dev->name); 1339 if (pg->ps.type->status) 1340 sz += pg->ps.type->status(&pg->ps, 1341 &p->path, type, result + sz, 1342 maxlen - sz); 1343 } 1344 } 1345 break; 1346 } 1347 1348 spin_unlock_irqrestore(&m->lock, flags); 1349 1350 return 0; 1351 } 1352 1353 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv) 1354 { 1355 int r; 1356 struct dm_dev *dev; 1357 struct multipath *m = (struct multipath *) ti->private; 1358 action_fn action; 1359 1360 if (argc == 1) { 1361 if (!strnicmp(argv[0], MESG_STR("queue_if_no_path"))) 1362 return queue_if_no_path(m, 1, 0); 1363 else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path"))) 1364 return queue_if_no_path(m, 0, 0); 1365 } 1366 1367 if (argc != 2) 1368 goto error; 1369 1370 if (!strnicmp(argv[0], MESG_STR("disable_group"))) 1371 return bypass_pg_num(m, argv[1], 1); 1372 else if (!strnicmp(argv[0], MESG_STR("enable_group"))) 1373 return bypass_pg_num(m, argv[1], 0); 1374 else if (!strnicmp(argv[0], MESG_STR("switch_group"))) 1375 return switch_pg_num(m, argv[1]); 1376 else if (!strnicmp(argv[0], MESG_STR("reinstate_path"))) 1377 action = reinstate_path; 1378 else if (!strnicmp(argv[0], MESG_STR("fail_path"))) 1379 action = fail_path; 1380 else 1381 goto error; 1382 1383 r = dm_get_device(ti, argv[1], ti->begin, ti->len, 1384 dm_table_get_mode(ti->table), &dev); 1385 if (r) { 1386 DMWARN("message: error getting device %s", 1387 argv[1]); 1388 return -EINVAL; 1389 } 1390 1391 r = action_dev(m, dev, action); 1392 1393 dm_put_device(ti, dev); 1394 1395 return r; 1396 1397 error: 1398 DMWARN("Unrecognised multipath message received."); 1399 return -EINVAL; 1400 } 1401 1402 static int multipath_ioctl(struct dm_target *ti, unsigned int cmd, 1403 unsigned long arg) 1404 { 1405 struct multipath *m = (struct multipath *) ti->private; 1406 struct block_device *bdev = NULL; 1407 fmode_t mode = 0; 1408 unsigned long flags; 1409 int r = 0; 1410 1411 spin_lock_irqsave(&m->lock, flags); 1412 1413 if (!m->current_pgpath) 1414 __choose_pgpath(m); 1415 1416 if (m->current_pgpath) { 1417 bdev = m->current_pgpath->path.dev->bdev; 1418 mode = m->current_pgpath->path.dev->mode; 1419 } 1420 1421 if (m->queue_io) 1422 r = -EAGAIN; 1423 else if (!bdev) 1424 r = -EIO; 1425 1426 spin_unlock_irqrestore(&m->lock, flags); 1427 1428 return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg); 1429 } 1430 1431 /*----------------------------------------------------------------- 1432 * Module setup 1433 *---------------------------------------------------------------*/ 1434 static struct target_type multipath_target = { 1435 .name = "multipath", 1436 .version = {1, 0, 5}, 1437 .module = THIS_MODULE, 1438 .ctr = multipath_ctr, 1439 .dtr = multipath_dtr, 1440 .map = multipath_map, 1441 .end_io = multipath_end_io, 1442 .presuspend = multipath_presuspend, 1443 .resume = multipath_resume, 1444 .status = multipath_status, 1445 .message = multipath_message, 1446 .ioctl = multipath_ioctl, 1447 }; 1448 1449 static int __init dm_multipath_init(void) 1450 { 1451 int r; 1452 1453 /* allocate a slab for the dm_ios */ 1454 _mpio_cache = KMEM_CACHE(dm_mpath_io, 0); 1455 if (!_mpio_cache) 1456 return -ENOMEM; 1457 1458 r = dm_register_target(&multipath_target); 1459 if (r < 0) { 1460 DMERR("register failed %d", r); 1461 kmem_cache_destroy(_mpio_cache); 1462 return -EINVAL; 1463 } 1464 1465 kmultipathd = create_workqueue("kmpathd"); 1466 if (!kmultipathd) { 1467 DMERR("failed to create workqueue kmpathd"); 1468 dm_unregister_target(&multipath_target); 1469 kmem_cache_destroy(_mpio_cache); 1470 return -ENOMEM; 1471 } 1472 1473 /* 1474 * A separate workqueue is used to handle the device handlers 1475 * to avoid overloading existing workqueue. Overloading the 1476 * old workqueue would also create a bottleneck in the 1477 * path of the storage hardware device activation. 1478 */ 1479 kmpath_handlerd = create_singlethread_workqueue("kmpath_handlerd"); 1480 if (!kmpath_handlerd) { 1481 DMERR("failed to create workqueue kmpath_handlerd"); 1482 destroy_workqueue(kmultipathd); 1483 dm_unregister_target(&multipath_target); 1484 kmem_cache_destroy(_mpio_cache); 1485 return -ENOMEM; 1486 } 1487 1488 DMINFO("version %u.%u.%u loaded", 1489 multipath_target.version[0], multipath_target.version[1], 1490 multipath_target.version[2]); 1491 1492 return r; 1493 } 1494 1495 static void __exit dm_multipath_exit(void) 1496 { 1497 destroy_workqueue(kmpath_handlerd); 1498 destroy_workqueue(kmultipathd); 1499 1500 dm_unregister_target(&multipath_target); 1501 kmem_cache_destroy(_mpio_cache); 1502 } 1503 1504 module_init(dm_multipath_init); 1505 module_exit(dm_multipath_exit); 1506 1507 MODULE_DESCRIPTION(DM_NAME " multipath target"); 1508 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>"); 1509 MODULE_LICENSE("GPL"); 1510