1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * multipath.c : Multiple Devices driver for Linux 4 * 5 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat 6 * 7 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman 8 * 9 * MULTIPATH management functions. 10 * 11 * derived from raid1.c. 12 */ 13 14 #include <linux/blkdev.h> 15 #include <linux/module.h> 16 #include <linux/raid/md_u.h> 17 #include <linux/seq_file.h> 18 #include <linux/slab.h> 19 #include "md.h" 20 #include "md-multipath.h" 21 22 #define MAX_WORK_PER_DISK 128 23 24 #define NR_RESERVED_BUFS 32 25 26 static int multipath_map (struct mpconf *conf) 27 { 28 int i, disks = conf->raid_disks; 29 30 /* 31 * Later we do read balancing on the read side 32 * now we use the first available disk. 33 */ 34 35 rcu_read_lock(); 36 for (i = 0; i < disks; i++) { 37 struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); 38 if (rdev && test_bit(In_sync, &rdev->flags) && 39 !test_bit(Faulty, &rdev->flags)) { 40 atomic_inc(&rdev->nr_pending); 41 rcu_read_unlock(); 42 return i; 43 } 44 } 45 rcu_read_unlock(); 46 47 pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n"); 48 return (-1); 49 } 50 51 static void multipath_reschedule_retry (struct multipath_bh *mp_bh) 52 { 53 unsigned long flags; 54 struct mddev *mddev = mp_bh->mddev; 55 struct mpconf *conf = mddev->private; 56 57 spin_lock_irqsave(&conf->device_lock, flags); 58 list_add(&mp_bh->retry_list, &conf->retry_list); 59 spin_unlock_irqrestore(&conf->device_lock, flags); 60 md_wakeup_thread(mddev->thread); 61 } 62 63 /* 64 * multipath_end_bh_io() is called when we have finished servicing a multipathed 65 * operation and are ready to return a success/failure code to the buffer 66 * cache layer. 67 */ 68 static void multipath_end_bh_io(struct multipath_bh *mp_bh, blk_status_t status) 69 { 70 struct bio *bio = mp_bh->master_bio; 71 struct mpconf *conf = mp_bh->mddev->private; 72 73 bio->bi_status = status; 74 bio_endio(bio); 75 mempool_free(mp_bh, &conf->pool); 76 } 77 78 static void multipath_end_request(struct bio *bio) 79 { 80 struct multipath_bh *mp_bh = bio->bi_private; 81 struct mpconf *conf = mp_bh->mddev->private; 82 struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev; 83 84 if (!bio->bi_status) 85 multipath_end_bh_io(mp_bh, 0); 86 else if (!(bio->bi_opf & REQ_RAHEAD)) { 87 /* 88 * oops, IO error: 89 */ 90 char b[BDEVNAME_SIZE]; 91 md_error (mp_bh->mddev, rdev); 92 pr_info("multipath: %s: rescheduling sector %llu\n", 93 bdevname(rdev->bdev,b), 94 (unsigned long long)bio->bi_iter.bi_sector); 95 multipath_reschedule_retry(mp_bh); 96 } else 97 multipath_end_bh_io(mp_bh, bio->bi_status); 98 rdev_dec_pending(rdev, conf->mddev); 99 } 100 101 static bool multipath_make_request(struct mddev *mddev, struct bio * bio) 102 { 103 struct mpconf *conf = mddev->private; 104 struct multipath_bh * mp_bh; 105 struct multipath_info *multipath; 106 107 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 108 md_flush_request(mddev, bio); 109 return true; 110 } 111 112 mp_bh = mempool_alloc(&conf->pool, GFP_NOIO); 113 114 mp_bh->master_bio = bio; 115 mp_bh->mddev = mddev; 116 117 mp_bh->path = multipath_map(conf); 118 if (mp_bh->path < 0) { 119 bio_io_error(bio); 120 mempool_free(mp_bh, &conf->pool); 121 return true; 122 } 123 multipath = conf->multipaths + mp_bh->path; 124 125 bio_init(&mp_bh->bio, NULL, 0); 126 __bio_clone_fast(&mp_bh->bio, bio); 127 128 mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset; 129 bio_set_dev(&mp_bh->bio, multipath->rdev->bdev); 130 mp_bh->bio.bi_opf |= REQ_FAILFAST_TRANSPORT; 131 mp_bh->bio.bi_end_io = multipath_end_request; 132 mp_bh->bio.bi_private = mp_bh; 133 mddev_check_writesame(mddev, &mp_bh->bio); 134 mddev_check_write_zeroes(mddev, &mp_bh->bio); 135 generic_make_request(&mp_bh->bio); 136 return true; 137 } 138 139 static void multipath_status(struct seq_file *seq, struct mddev *mddev) 140 { 141 struct mpconf *conf = mddev->private; 142 int i; 143 144 seq_printf (seq, " [%d/%d] [", conf->raid_disks, 145 conf->raid_disks - mddev->degraded); 146 rcu_read_lock(); 147 for (i = 0; i < conf->raid_disks; i++) { 148 struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); 149 seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 150 } 151 rcu_read_unlock(); 152 seq_putc(seq, ']'); 153 } 154 155 static int multipath_congested(struct mddev *mddev, int bits) 156 { 157 struct mpconf *conf = mddev->private; 158 int i, ret = 0; 159 160 rcu_read_lock(); 161 for (i = 0; i < mddev->raid_disks ; i++) { 162 struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev); 163 if (rdev && !test_bit(Faulty, &rdev->flags)) { 164 struct request_queue *q = bdev_get_queue(rdev->bdev); 165 166 ret |= bdi_congested(q->backing_dev_info, bits); 167 /* Just like multipath_map, we just check the 168 * first available device 169 */ 170 break; 171 } 172 } 173 rcu_read_unlock(); 174 return ret; 175 } 176 177 /* 178 * Careful, this can execute in IRQ contexts as well! 179 */ 180 static void multipath_error (struct mddev *mddev, struct md_rdev *rdev) 181 { 182 struct mpconf *conf = mddev->private; 183 char b[BDEVNAME_SIZE]; 184 185 if (conf->raid_disks - mddev->degraded <= 1) { 186 /* 187 * Uh oh, we can do nothing if this is our last path, but 188 * first check if this is a queued request for a device 189 * which has just failed. 190 */ 191 pr_warn("multipath: only one IO path left and IO error.\n"); 192 /* leave it active... it's all we have */ 193 return; 194 } 195 /* 196 * Mark disk as unusable 197 */ 198 if (test_and_clear_bit(In_sync, &rdev->flags)) { 199 unsigned long flags; 200 spin_lock_irqsave(&conf->device_lock, flags); 201 mddev->degraded++; 202 spin_unlock_irqrestore(&conf->device_lock, flags); 203 } 204 set_bit(Faulty, &rdev->flags); 205 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 206 pr_err("multipath: IO failure on %s, disabling IO path.\n" 207 "multipath: Operation continuing on %d IO paths.\n", 208 bdevname(rdev->bdev, b), 209 conf->raid_disks - mddev->degraded); 210 } 211 212 static void print_multipath_conf (struct mpconf *conf) 213 { 214 int i; 215 struct multipath_info *tmp; 216 217 pr_debug("MULTIPATH conf printout:\n"); 218 if (!conf) { 219 pr_debug("(conf==NULL)\n"); 220 return; 221 } 222 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 223 conf->raid_disks); 224 225 for (i = 0; i < conf->raid_disks; i++) { 226 char b[BDEVNAME_SIZE]; 227 tmp = conf->multipaths + i; 228 if (tmp->rdev) 229 pr_debug(" disk%d, o:%d, dev:%s\n", 230 i,!test_bit(Faulty, &tmp->rdev->flags), 231 bdevname(tmp->rdev->bdev,b)); 232 } 233 } 234 235 static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev) 236 { 237 struct mpconf *conf = mddev->private; 238 int err = -EEXIST; 239 int path; 240 struct multipath_info *p; 241 int first = 0; 242 int last = mddev->raid_disks - 1; 243 244 if (rdev->raid_disk >= 0) 245 first = last = rdev->raid_disk; 246 247 print_multipath_conf(conf); 248 249 for (path = first; path <= last; path++) 250 if ((p=conf->multipaths+path)->rdev == NULL) { 251 disk_stack_limits(mddev->gendisk, rdev->bdev, 252 rdev->data_offset << 9); 253 254 err = md_integrity_add_rdev(rdev, mddev); 255 if (err) 256 break; 257 spin_lock_irq(&conf->device_lock); 258 mddev->degraded--; 259 rdev->raid_disk = path; 260 set_bit(In_sync, &rdev->flags); 261 spin_unlock_irq(&conf->device_lock); 262 rcu_assign_pointer(p->rdev, rdev); 263 err = 0; 264 break; 265 } 266 267 print_multipath_conf(conf); 268 269 return err; 270 } 271 272 static int multipath_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 273 { 274 struct mpconf *conf = mddev->private; 275 int err = 0; 276 int number = rdev->raid_disk; 277 struct multipath_info *p = conf->multipaths + number; 278 279 print_multipath_conf(conf); 280 281 if (rdev == p->rdev) { 282 if (test_bit(In_sync, &rdev->flags) || 283 atomic_read(&rdev->nr_pending)) { 284 pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n", number); 285 err = -EBUSY; 286 goto abort; 287 } 288 p->rdev = NULL; 289 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 290 synchronize_rcu(); 291 if (atomic_read(&rdev->nr_pending)) { 292 /* lost the race, try later */ 293 err = -EBUSY; 294 p->rdev = rdev; 295 goto abort; 296 } 297 } 298 err = md_integrity_register(mddev); 299 } 300 abort: 301 302 print_multipath_conf(conf); 303 return err; 304 } 305 306 /* 307 * This is a kernel thread which: 308 * 309 * 1. Retries failed read operations on working multipaths. 310 * 2. Updates the raid superblock when problems encounter. 311 * 3. Performs writes following reads for array syncronising. 312 */ 313 314 static void multipathd(struct md_thread *thread) 315 { 316 struct mddev *mddev = thread->mddev; 317 struct multipath_bh *mp_bh; 318 struct bio *bio; 319 unsigned long flags; 320 struct mpconf *conf = mddev->private; 321 struct list_head *head = &conf->retry_list; 322 323 md_check_recovery(mddev); 324 for (;;) { 325 char b[BDEVNAME_SIZE]; 326 spin_lock_irqsave(&conf->device_lock, flags); 327 if (list_empty(head)) 328 break; 329 mp_bh = list_entry(head->prev, struct multipath_bh, retry_list); 330 list_del(head->prev); 331 spin_unlock_irqrestore(&conf->device_lock, flags); 332 333 bio = &mp_bh->bio; 334 bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector; 335 336 if ((mp_bh->path = multipath_map (conf))<0) { 337 pr_err("multipath: %s: unrecoverable IO read error for block %llu\n", 338 bio_devname(bio, b), 339 (unsigned long long)bio->bi_iter.bi_sector); 340 multipath_end_bh_io(mp_bh, BLK_STS_IOERR); 341 } else { 342 pr_err("multipath: %s: redirecting sector %llu to another IO path\n", 343 bio_devname(bio, b), 344 (unsigned long long)bio->bi_iter.bi_sector); 345 *bio = *(mp_bh->master_bio); 346 bio->bi_iter.bi_sector += 347 conf->multipaths[mp_bh->path].rdev->data_offset; 348 bio_set_dev(bio, conf->multipaths[mp_bh->path].rdev->bdev); 349 bio->bi_opf |= REQ_FAILFAST_TRANSPORT; 350 bio->bi_end_io = multipath_end_request; 351 bio->bi_private = mp_bh; 352 generic_make_request(bio); 353 } 354 } 355 spin_unlock_irqrestore(&conf->device_lock, flags); 356 } 357 358 static sector_t multipath_size(struct mddev *mddev, sector_t sectors, int raid_disks) 359 { 360 WARN_ONCE(sectors || raid_disks, 361 "%s does not support generic reshape\n", __func__); 362 363 return mddev->dev_sectors; 364 } 365 366 static int multipath_run (struct mddev *mddev) 367 { 368 struct mpconf *conf; 369 int disk_idx; 370 struct multipath_info *disk; 371 struct md_rdev *rdev; 372 int working_disks; 373 int ret; 374 375 if (md_check_no_bitmap(mddev)) 376 return -EINVAL; 377 378 if (mddev->level != LEVEL_MULTIPATH) { 379 pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n", 380 mdname(mddev), mddev->level); 381 goto out; 382 } 383 /* 384 * copy the already verified devices into our private MULTIPATH 385 * bookkeeping area. [whatever we allocate in multipath_run(), 386 * should be freed in multipath_free()] 387 */ 388 389 conf = kzalloc(sizeof(struct mpconf), GFP_KERNEL); 390 mddev->private = conf; 391 if (!conf) 392 goto out; 393 394 conf->multipaths = kcalloc(mddev->raid_disks, 395 sizeof(struct multipath_info), 396 GFP_KERNEL); 397 if (!conf->multipaths) 398 goto out_free_conf; 399 400 working_disks = 0; 401 rdev_for_each(rdev, mddev) { 402 disk_idx = rdev->raid_disk; 403 if (disk_idx < 0 || 404 disk_idx >= mddev->raid_disks) 405 continue; 406 407 disk = conf->multipaths + disk_idx; 408 disk->rdev = rdev; 409 disk_stack_limits(mddev->gendisk, rdev->bdev, 410 rdev->data_offset << 9); 411 412 if (!test_bit(Faulty, &rdev->flags)) 413 working_disks++; 414 } 415 416 conf->raid_disks = mddev->raid_disks; 417 conf->mddev = mddev; 418 spin_lock_init(&conf->device_lock); 419 INIT_LIST_HEAD(&conf->retry_list); 420 421 if (!working_disks) { 422 pr_warn("multipath: no operational IO paths for %s\n", 423 mdname(mddev)); 424 goto out_free_conf; 425 } 426 mddev->degraded = conf->raid_disks - working_disks; 427 428 ret = mempool_init_kmalloc_pool(&conf->pool, NR_RESERVED_BUFS, 429 sizeof(struct multipath_bh)); 430 if (ret) 431 goto out_free_conf; 432 433 mddev->thread = md_register_thread(multipathd, mddev, 434 "multipath"); 435 if (!mddev->thread) 436 goto out_free_conf; 437 438 pr_info("multipath: array %s active with %d out of %d IO paths\n", 439 mdname(mddev), conf->raid_disks - mddev->degraded, 440 mddev->raid_disks); 441 /* 442 * Ok, everything is just fine now 443 */ 444 md_set_array_sectors(mddev, multipath_size(mddev, 0, 0)); 445 446 if (md_integrity_register(mddev)) 447 goto out_free_conf; 448 449 return 0; 450 451 out_free_conf: 452 mempool_exit(&conf->pool); 453 kfree(conf->multipaths); 454 kfree(conf); 455 mddev->private = NULL; 456 out: 457 return -EIO; 458 } 459 460 static void multipath_free(struct mddev *mddev, void *priv) 461 { 462 struct mpconf *conf = priv; 463 464 mempool_exit(&conf->pool); 465 kfree(conf->multipaths); 466 kfree(conf); 467 } 468 469 static struct md_personality multipath_personality = 470 { 471 .name = "multipath", 472 .level = LEVEL_MULTIPATH, 473 .owner = THIS_MODULE, 474 .make_request = multipath_make_request, 475 .run = multipath_run, 476 .free = multipath_free, 477 .status = multipath_status, 478 .error_handler = multipath_error, 479 .hot_add_disk = multipath_add_disk, 480 .hot_remove_disk= multipath_remove_disk, 481 .size = multipath_size, 482 .congested = multipath_congested, 483 }; 484 485 static int __init multipath_init (void) 486 { 487 return register_md_personality (&multipath_personality); 488 } 489 490 static void __exit multipath_exit (void) 491 { 492 unregister_md_personality (&multipath_personality); 493 } 494 495 module_init(multipath_init); 496 module_exit(multipath_exit); 497 MODULE_LICENSE("GPL"); 498 MODULE_DESCRIPTION("simple multi-path personality for MD"); 499 MODULE_ALIAS("md-personality-7"); /* MULTIPATH */ 500 MODULE_ALIAS("md-multipath"); 501 MODULE_ALIAS("md-level--4"); 502