1 /* 2 * raid1.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat 5 * 6 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman 7 * 8 * RAID-1 management functions. 9 * 10 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000 11 * 12 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk> 13 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au> 14 * 15 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support 16 * bitmapped intelligence in resync: 17 * 18 * - bitmap marked during normal i/o 19 * - bitmap used to skip nondirty blocks during sync 20 * 21 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology: 22 * - persistent bitmap code 23 * 24 * This program is free software; you can redistribute it and/or modify 25 * it under the terms of the GNU General Public License as published by 26 * the Free Software Foundation; either version 2, or (at your option) 27 * any later version. 28 * 29 * You should have received a copy of the GNU General Public License 30 * (for example /usr/src/linux/COPYING); if not, write to the Free 31 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 32 */ 33 34 #include <linux/slab.h> 35 #include <linux/delay.h> 36 #include <linux/blkdev.h> 37 #include <linux/module.h> 38 #include <linux/seq_file.h> 39 #include <linux/ratelimit.h> 40 #include <linux/sched/signal.h> 41 42 #include <trace/events/block.h> 43 44 #include "md.h" 45 #include "raid1.h" 46 #include "bitmap.h" 47 48 #define UNSUPPORTED_MDDEV_FLAGS \ 49 ((1L << MD_HAS_JOURNAL) | \ 50 (1L << MD_JOURNAL_CLEAN) | \ 51 (1L << MD_HAS_PPL)) 52 53 /* 54 * Number of guaranteed r1bios in case of extreme VM load: 55 */ 56 #define NR_RAID1_BIOS 256 57 58 /* when we get a read error on a read-only array, we redirect to another 59 * device without failing the first device, or trying to over-write to 60 * correct the read error. To keep track of bad blocks on a per-bio 61 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 62 */ 63 #define IO_BLOCKED ((struct bio *)1) 64 /* When we successfully write to a known bad-block, we need to remove the 65 * bad-block marking which must be done from process context. So we record 66 * the success by setting devs[n].bio to IO_MADE_GOOD 67 */ 68 #define IO_MADE_GOOD ((struct bio *)2) 69 70 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 71 72 /* When there are this many requests queue to be written by 73 * the raid1 thread, we become 'congested' to provide back-pressure 74 * for writeback. 75 */ 76 static int max_queued_requests = 1024; 77 78 static void allow_barrier(struct r1conf *conf, sector_t sector_nr); 79 static void lower_barrier(struct r1conf *conf, sector_t sector_nr); 80 81 #define raid1_log(md, fmt, args...) \ 82 do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid1 " fmt, ##args); } while (0) 83 84 /* 85 * 'strct resync_pages' stores actual pages used for doing the resync 86 * IO, and it is per-bio, so make .bi_private points to it. 87 */ 88 static inline struct resync_pages *get_resync_pages(struct bio *bio) 89 { 90 return bio->bi_private; 91 } 92 93 /* 94 * for resync bio, r1bio pointer can be retrieved from the per-bio 95 * 'struct resync_pages'. 96 */ 97 static inline struct r1bio *get_resync_r1bio(struct bio *bio) 98 { 99 return get_resync_pages(bio)->raid_bio; 100 } 101 102 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) 103 { 104 struct pool_info *pi = data; 105 int size = offsetof(struct r1bio, bios[pi->raid_disks]); 106 107 /* allocate a r1bio with room for raid_disks entries in the bios array */ 108 return kzalloc(size, gfp_flags); 109 } 110 111 static void r1bio_pool_free(void *r1_bio, void *data) 112 { 113 kfree(r1_bio); 114 } 115 116 #define RESYNC_DEPTH 32 117 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) 118 #define RESYNC_WINDOW (RESYNC_BLOCK_SIZE * RESYNC_DEPTH) 119 #define RESYNC_WINDOW_SECTORS (RESYNC_WINDOW >> 9) 120 #define CLUSTER_RESYNC_WINDOW (16 * RESYNC_WINDOW) 121 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) 122 123 static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) 124 { 125 struct pool_info *pi = data; 126 struct r1bio *r1_bio; 127 struct bio *bio; 128 int need_pages; 129 int j; 130 struct resync_pages *rps; 131 132 r1_bio = r1bio_pool_alloc(gfp_flags, pi); 133 if (!r1_bio) 134 return NULL; 135 136 rps = kmalloc(sizeof(struct resync_pages) * pi->raid_disks, 137 gfp_flags); 138 if (!rps) 139 goto out_free_r1bio; 140 141 /* 142 * Allocate bios : 1 for reading, n-1 for writing 143 */ 144 for (j = pi->raid_disks ; j-- ; ) { 145 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 146 if (!bio) 147 goto out_free_bio; 148 r1_bio->bios[j] = bio; 149 } 150 /* 151 * Allocate RESYNC_PAGES data pages and attach them to 152 * the first bio. 153 * If this is a user-requested check/repair, allocate 154 * RESYNC_PAGES for each bio. 155 */ 156 if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) 157 need_pages = pi->raid_disks; 158 else 159 need_pages = 1; 160 for (j = 0; j < pi->raid_disks; j++) { 161 struct resync_pages *rp = &rps[j]; 162 163 bio = r1_bio->bios[j]; 164 165 if (j < need_pages) { 166 if (resync_alloc_pages(rp, gfp_flags)) 167 goto out_free_pages; 168 } else { 169 memcpy(rp, &rps[0], sizeof(*rp)); 170 resync_get_all_pages(rp); 171 } 172 173 rp->idx = 0; 174 rp->raid_bio = r1_bio; 175 bio->bi_private = rp; 176 } 177 178 r1_bio->master_bio = NULL; 179 180 return r1_bio; 181 182 out_free_pages: 183 while (--j >= 0) 184 resync_free_pages(&rps[j]); 185 186 out_free_bio: 187 while (++j < pi->raid_disks) 188 bio_put(r1_bio->bios[j]); 189 kfree(rps); 190 191 out_free_r1bio: 192 r1bio_pool_free(r1_bio, data); 193 return NULL; 194 } 195 196 static void r1buf_pool_free(void *__r1_bio, void *data) 197 { 198 struct pool_info *pi = data; 199 int i; 200 struct r1bio *r1bio = __r1_bio; 201 struct resync_pages *rp = NULL; 202 203 for (i = pi->raid_disks; i--; ) { 204 rp = get_resync_pages(r1bio->bios[i]); 205 resync_free_pages(rp); 206 bio_put(r1bio->bios[i]); 207 } 208 209 /* resync pages array stored in the 1st bio's .bi_private */ 210 kfree(rp); 211 212 r1bio_pool_free(r1bio, data); 213 } 214 215 static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio) 216 { 217 int i; 218 219 for (i = 0; i < conf->raid_disks * 2; i++) { 220 struct bio **bio = r1_bio->bios + i; 221 if (!BIO_SPECIAL(*bio)) 222 bio_put(*bio); 223 *bio = NULL; 224 } 225 } 226 227 static void free_r1bio(struct r1bio *r1_bio) 228 { 229 struct r1conf *conf = r1_bio->mddev->private; 230 231 put_all_bios(conf, r1_bio); 232 mempool_free(r1_bio, conf->r1bio_pool); 233 } 234 235 static void put_buf(struct r1bio *r1_bio) 236 { 237 struct r1conf *conf = r1_bio->mddev->private; 238 sector_t sect = r1_bio->sector; 239 int i; 240 241 for (i = 0; i < conf->raid_disks * 2; i++) { 242 struct bio *bio = r1_bio->bios[i]; 243 if (bio->bi_end_io) 244 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); 245 } 246 247 mempool_free(r1_bio, conf->r1buf_pool); 248 249 lower_barrier(conf, sect); 250 } 251 252 static void reschedule_retry(struct r1bio *r1_bio) 253 { 254 unsigned long flags; 255 struct mddev *mddev = r1_bio->mddev; 256 struct r1conf *conf = mddev->private; 257 int idx; 258 259 idx = sector_to_idx(r1_bio->sector); 260 spin_lock_irqsave(&conf->device_lock, flags); 261 list_add(&r1_bio->retry_list, &conf->retry_list); 262 atomic_inc(&conf->nr_queued[idx]); 263 spin_unlock_irqrestore(&conf->device_lock, flags); 264 265 wake_up(&conf->wait_barrier); 266 md_wakeup_thread(mddev->thread); 267 } 268 269 /* 270 * raid_end_bio_io() is called when we have finished servicing a mirrored 271 * operation and are ready to return a success/failure code to the buffer 272 * cache layer. 273 */ 274 static void call_bio_endio(struct r1bio *r1_bio) 275 { 276 struct bio *bio = r1_bio->master_bio; 277 struct r1conf *conf = r1_bio->mddev->private; 278 279 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) 280 bio->bi_status = BLK_STS_IOERR; 281 282 bio_endio(bio); 283 /* 284 * Wake up any possible resync thread that waits for the device 285 * to go idle. 286 */ 287 allow_barrier(conf, r1_bio->sector); 288 } 289 290 static void raid_end_bio_io(struct r1bio *r1_bio) 291 { 292 struct bio *bio = r1_bio->master_bio; 293 294 /* if nobody has done the final endio yet, do it now */ 295 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 296 pr_debug("raid1: sync end %s on sectors %llu-%llu\n", 297 (bio_data_dir(bio) == WRITE) ? "write" : "read", 298 (unsigned long long) bio->bi_iter.bi_sector, 299 (unsigned long long) bio_end_sector(bio) - 1); 300 301 call_bio_endio(r1_bio); 302 } 303 free_r1bio(r1_bio); 304 } 305 306 /* 307 * Update disk head position estimator based on IRQ completion info. 308 */ 309 static inline void update_head_pos(int disk, struct r1bio *r1_bio) 310 { 311 struct r1conf *conf = r1_bio->mddev->private; 312 313 conf->mirrors[disk].head_position = 314 r1_bio->sector + (r1_bio->sectors); 315 } 316 317 /* 318 * Find the disk number which triggered given bio 319 */ 320 static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio) 321 { 322 int mirror; 323 struct r1conf *conf = r1_bio->mddev->private; 324 int raid_disks = conf->raid_disks; 325 326 for (mirror = 0; mirror < raid_disks * 2; mirror++) 327 if (r1_bio->bios[mirror] == bio) 328 break; 329 330 BUG_ON(mirror == raid_disks * 2); 331 update_head_pos(mirror, r1_bio); 332 333 return mirror; 334 } 335 336 static void raid1_end_read_request(struct bio *bio) 337 { 338 int uptodate = !bio->bi_status; 339 struct r1bio *r1_bio = bio->bi_private; 340 struct r1conf *conf = r1_bio->mddev->private; 341 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; 342 343 /* 344 * this branch is our 'one mirror IO has finished' event handler: 345 */ 346 update_head_pos(r1_bio->read_disk, r1_bio); 347 348 if (uptodate) 349 set_bit(R1BIO_Uptodate, &r1_bio->state); 350 else if (test_bit(FailFast, &rdev->flags) && 351 test_bit(R1BIO_FailFast, &r1_bio->state)) 352 /* This was a fail-fast read so we definitely 353 * want to retry */ 354 ; 355 else { 356 /* If all other devices have failed, we want to return 357 * the error upwards rather than fail the last device. 358 * Here we redefine "uptodate" to mean "Don't want to retry" 359 */ 360 unsigned long flags; 361 spin_lock_irqsave(&conf->device_lock, flags); 362 if (r1_bio->mddev->degraded == conf->raid_disks || 363 (r1_bio->mddev->degraded == conf->raid_disks-1 && 364 test_bit(In_sync, &rdev->flags))) 365 uptodate = 1; 366 spin_unlock_irqrestore(&conf->device_lock, flags); 367 } 368 369 if (uptodate) { 370 raid_end_bio_io(r1_bio); 371 rdev_dec_pending(rdev, conf->mddev); 372 } else { 373 /* 374 * oops, read error: 375 */ 376 char b[BDEVNAME_SIZE]; 377 pr_err_ratelimited("md/raid1:%s: %s: rescheduling sector %llu\n", 378 mdname(conf->mddev), 379 bdevname(rdev->bdev, b), 380 (unsigned long long)r1_bio->sector); 381 set_bit(R1BIO_ReadError, &r1_bio->state); 382 reschedule_retry(r1_bio); 383 /* don't drop the reference on read_disk yet */ 384 } 385 } 386 387 static void close_write(struct r1bio *r1_bio) 388 { 389 /* it really is the end of this request */ 390 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { 391 bio_free_pages(r1_bio->behind_master_bio); 392 bio_put(r1_bio->behind_master_bio); 393 r1_bio->behind_master_bio = NULL; 394 } 395 /* clear the bitmap if all writes complete successfully */ 396 bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector, 397 r1_bio->sectors, 398 !test_bit(R1BIO_Degraded, &r1_bio->state), 399 test_bit(R1BIO_BehindIO, &r1_bio->state)); 400 md_write_end(r1_bio->mddev); 401 } 402 403 static void r1_bio_write_done(struct r1bio *r1_bio) 404 { 405 if (!atomic_dec_and_test(&r1_bio->remaining)) 406 return; 407 408 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 409 reschedule_retry(r1_bio); 410 else { 411 close_write(r1_bio); 412 if (test_bit(R1BIO_MadeGood, &r1_bio->state)) 413 reschedule_retry(r1_bio); 414 else 415 raid_end_bio_io(r1_bio); 416 } 417 } 418 419 static void raid1_end_write_request(struct bio *bio) 420 { 421 struct r1bio *r1_bio = bio->bi_private; 422 int behind = test_bit(R1BIO_BehindIO, &r1_bio->state); 423 struct r1conf *conf = r1_bio->mddev->private; 424 struct bio *to_put = NULL; 425 int mirror = find_bio_disk(r1_bio, bio); 426 struct md_rdev *rdev = conf->mirrors[mirror].rdev; 427 bool discard_error; 428 429 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD; 430 431 /* 432 * 'one mirror IO has finished' event handler: 433 */ 434 if (bio->bi_status && !discard_error) { 435 set_bit(WriteErrorSeen, &rdev->flags); 436 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 437 set_bit(MD_RECOVERY_NEEDED, & 438 conf->mddev->recovery); 439 440 if (test_bit(FailFast, &rdev->flags) && 441 (bio->bi_opf & MD_FAILFAST) && 442 /* We never try FailFast to WriteMostly devices */ 443 !test_bit(WriteMostly, &rdev->flags)) { 444 md_error(r1_bio->mddev, rdev); 445 if (!test_bit(Faulty, &rdev->flags)) 446 /* This is the only remaining device, 447 * We need to retry the write without 448 * FailFast 449 */ 450 set_bit(R1BIO_WriteError, &r1_bio->state); 451 else { 452 /* Finished with this branch */ 453 r1_bio->bios[mirror] = NULL; 454 to_put = bio; 455 } 456 } else 457 set_bit(R1BIO_WriteError, &r1_bio->state); 458 } else { 459 /* 460 * Set R1BIO_Uptodate in our master bio, so that we 461 * will return a good error code for to the higher 462 * levels even if IO on some other mirrored buffer 463 * fails. 464 * 465 * The 'master' represents the composite IO operation 466 * to user-side. So if something waits for IO, then it 467 * will wait for the 'master' bio. 468 */ 469 sector_t first_bad; 470 int bad_sectors; 471 472 r1_bio->bios[mirror] = NULL; 473 to_put = bio; 474 /* 475 * Do not set R1BIO_Uptodate if the current device is 476 * rebuilding or Faulty. This is because we cannot use 477 * such device for properly reading the data back (we could 478 * potentially use it, if the current write would have felt 479 * before rdev->recovery_offset, but for simplicity we don't 480 * check this here. 481 */ 482 if (test_bit(In_sync, &rdev->flags) && 483 !test_bit(Faulty, &rdev->flags)) 484 set_bit(R1BIO_Uptodate, &r1_bio->state); 485 486 /* Maybe we can clear some bad blocks. */ 487 if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors, 488 &first_bad, &bad_sectors) && !discard_error) { 489 r1_bio->bios[mirror] = IO_MADE_GOOD; 490 set_bit(R1BIO_MadeGood, &r1_bio->state); 491 } 492 } 493 494 if (behind) { 495 /* we release behind master bio when all write are done */ 496 if (r1_bio->behind_master_bio == bio) 497 to_put = NULL; 498 499 if (test_bit(WriteMostly, &rdev->flags)) 500 atomic_dec(&r1_bio->behind_remaining); 501 502 /* 503 * In behind mode, we ACK the master bio once the I/O 504 * has safely reached all non-writemostly 505 * disks. Setting the Returned bit ensures that this 506 * gets done only once -- we don't ever want to return 507 * -EIO here, instead we'll wait 508 */ 509 if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) && 510 test_bit(R1BIO_Uptodate, &r1_bio->state)) { 511 /* Maybe we can return now */ 512 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 513 struct bio *mbio = r1_bio->master_bio; 514 pr_debug("raid1: behind end write sectors" 515 " %llu-%llu\n", 516 (unsigned long long) mbio->bi_iter.bi_sector, 517 (unsigned long long) bio_end_sector(mbio) - 1); 518 call_bio_endio(r1_bio); 519 } 520 } 521 } 522 if (r1_bio->bios[mirror] == NULL) 523 rdev_dec_pending(rdev, conf->mddev); 524 525 /* 526 * Let's see if all mirrored write operations have finished 527 * already. 528 */ 529 r1_bio_write_done(r1_bio); 530 531 if (to_put) 532 bio_put(to_put); 533 } 534 535 static sector_t align_to_barrier_unit_end(sector_t start_sector, 536 sector_t sectors) 537 { 538 sector_t len; 539 540 WARN_ON(sectors == 0); 541 /* 542 * len is the number of sectors from start_sector to end of the 543 * barrier unit which start_sector belongs to. 544 */ 545 len = round_up(start_sector + 1, BARRIER_UNIT_SECTOR_SIZE) - 546 start_sector; 547 548 if (len > sectors) 549 len = sectors; 550 551 return len; 552 } 553 554 /* 555 * This routine returns the disk from which the requested read should 556 * be done. There is a per-array 'next expected sequential IO' sector 557 * number - if this matches on the next IO then we use the last disk. 558 * There is also a per-disk 'last know head position' sector that is 559 * maintained from IRQ contexts, both the normal and the resync IO 560 * completion handlers update this position correctly. If there is no 561 * perfect sequential match then we pick the disk whose head is closest. 562 * 563 * If there are 2 mirrors in the same 2 devices, performance degrades 564 * because position is mirror, not device based. 565 * 566 * The rdev for the device selected will have nr_pending incremented. 567 */ 568 static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sectors) 569 { 570 const sector_t this_sector = r1_bio->sector; 571 int sectors; 572 int best_good_sectors; 573 int best_disk, best_dist_disk, best_pending_disk; 574 int has_nonrot_disk; 575 int disk; 576 sector_t best_dist; 577 unsigned int min_pending; 578 struct md_rdev *rdev; 579 int choose_first; 580 int choose_next_idle; 581 582 rcu_read_lock(); 583 /* 584 * Check if we can balance. We can balance on the whole 585 * device if no resync is going on, or below the resync window. 586 * We take the first readable disk when above the resync window. 587 */ 588 retry: 589 sectors = r1_bio->sectors; 590 best_disk = -1; 591 best_dist_disk = -1; 592 best_dist = MaxSector; 593 best_pending_disk = -1; 594 min_pending = UINT_MAX; 595 best_good_sectors = 0; 596 has_nonrot_disk = 0; 597 choose_next_idle = 0; 598 clear_bit(R1BIO_FailFast, &r1_bio->state); 599 600 if ((conf->mddev->recovery_cp < this_sector + sectors) || 601 (mddev_is_clustered(conf->mddev) && 602 md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector, 603 this_sector + sectors))) 604 choose_first = 1; 605 else 606 choose_first = 0; 607 608 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { 609 sector_t dist; 610 sector_t first_bad; 611 int bad_sectors; 612 unsigned int pending; 613 bool nonrot; 614 615 rdev = rcu_dereference(conf->mirrors[disk].rdev); 616 if (r1_bio->bios[disk] == IO_BLOCKED 617 || rdev == NULL 618 || test_bit(Faulty, &rdev->flags)) 619 continue; 620 if (!test_bit(In_sync, &rdev->flags) && 621 rdev->recovery_offset < this_sector + sectors) 622 continue; 623 if (test_bit(WriteMostly, &rdev->flags)) { 624 /* Don't balance among write-mostly, just 625 * use the first as a last resort */ 626 if (best_dist_disk < 0) { 627 if (is_badblock(rdev, this_sector, sectors, 628 &first_bad, &bad_sectors)) { 629 if (first_bad <= this_sector) 630 /* Cannot use this */ 631 continue; 632 best_good_sectors = first_bad - this_sector; 633 } else 634 best_good_sectors = sectors; 635 best_dist_disk = disk; 636 best_pending_disk = disk; 637 } 638 continue; 639 } 640 /* This is a reasonable device to use. It might 641 * even be best. 642 */ 643 if (is_badblock(rdev, this_sector, sectors, 644 &first_bad, &bad_sectors)) { 645 if (best_dist < MaxSector) 646 /* already have a better device */ 647 continue; 648 if (first_bad <= this_sector) { 649 /* cannot read here. If this is the 'primary' 650 * device, then we must not read beyond 651 * bad_sectors from another device.. 652 */ 653 bad_sectors -= (this_sector - first_bad); 654 if (choose_first && sectors > bad_sectors) 655 sectors = bad_sectors; 656 if (best_good_sectors > sectors) 657 best_good_sectors = sectors; 658 659 } else { 660 sector_t good_sectors = first_bad - this_sector; 661 if (good_sectors > best_good_sectors) { 662 best_good_sectors = good_sectors; 663 best_disk = disk; 664 } 665 if (choose_first) 666 break; 667 } 668 continue; 669 } else { 670 if ((sectors > best_good_sectors) && (best_disk >= 0)) 671 best_disk = -1; 672 best_good_sectors = sectors; 673 } 674 675 if (best_disk >= 0) 676 /* At least two disks to choose from so failfast is OK */ 677 set_bit(R1BIO_FailFast, &r1_bio->state); 678 679 nonrot = blk_queue_nonrot(bdev_get_queue(rdev->bdev)); 680 has_nonrot_disk |= nonrot; 681 pending = atomic_read(&rdev->nr_pending); 682 dist = abs(this_sector - conf->mirrors[disk].head_position); 683 if (choose_first) { 684 best_disk = disk; 685 break; 686 } 687 /* Don't change to another disk for sequential reads */ 688 if (conf->mirrors[disk].next_seq_sect == this_sector 689 || dist == 0) { 690 int opt_iosize = bdev_io_opt(rdev->bdev) >> 9; 691 struct raid1_info *mirror = &conf->mirrors[disk]; 692 693 best_disk = disk; 694 /* 695 * If buffered sequential IO size exceeds optimal 696 * iosize, check if there is idle disk. If yes, choose 697 * the idle disk. read_balance could already choose an 698 * idle disk before noticing it's a sequential IO in 699 * this disk. This doesn't matter because this disk 700 * will idle, next time it will be utilized after the 701 * first disk has IO size exceeds optimal iosize. In 702 * this way, iosize of the first disk will be optimal 703 * iosize at least. iosize of the second disk might be 704 * small, but not a big deal since when the second disk 705 * starts IO, the first disk is likely still busy. 706 */ 707 if (nonrot && opt_iosize > 0 && 708 mirror->seq_start != MaxSector && 709 mirror->next_seq_sect > opt_iosize && 710 mirror->next_seq_sect - opt_iosize >= 711 mirror->seq_start) { 712 choose_next_idle = 1; 713 continue; 714 } 715 break; 716 } 717 718 if (choose_next_idle) 719 continue; 720 721 if (min_pending > pending) { 722 min_pending = pending; 723 best_pending_disk = disk; 724 } 725 726 if (dist < best_dist) { 727 best_dist = dist; 728 best_dist_disk = disk; 729 } 730 } 731 732 /* 733 * If all disks are rotational, choose the closest disk. If any disk is 734 * non-rotational, choose the disk with less pending request even the 735 * disk is rotational, which might/might not be optimal for raids with 736 * mixed ratation/non-rotational disks depending on workload. 737 */ 738 if (best_disk == -1) { 739 if (has_nonrot_disk || min_pending == 0) 740 best_disk = best_pending_disk; 741 else 742 best_disk = best_dist_disk; 743 } 744 745 if (best_disk >= 0) { 746 rdev = rcu_dereference(conf->mirrors[best_disk].rdev); 747 if (!rdev) 748 goto retry; 749 atomic_inc(&rdev->nr_pending); 750 sectors = best_good_sectors; 751 752 if (conf->mirrors[best_disk].next_seq_sect != this_sector) 753 conf->mirrors[best_disk].seq_start = this_sector; 754 755 conf->mirrors[best_disk].next_seq_sect = this_sector + sectors; 756 } 757 rcu_read_unlock(); 758 *max_sectors = sectors; 759 760 return best_disk; 761 } 762 763 static int raid1_congested(struct mddev *mddev, int bits) 764 { 765 struct r1conf *conf = mddev->private; 766 int i, ret = 0; 767 768 if ((bits & (1 << WB_async_congested)) && 769 conf->pending_count >= max_queued_requests) 770 return 1; 771 772 rcu_read_lock(); 773 for (i = 0; i < conf->raid_disks * 2; i++) { 774 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 775 if (rdev && !test_bit(Faulty, &rdev->flags)) { 776 struct request_queue *q = bdev_get_queue(rdev->bdev); 777 778 BUG_ON(!q); 779 780 /* Note the '|| 1' - when read_balance prefers 781 * non-congested targets, it can be removed 782 */ 783 if ((bits & (1 << WB_async_congested)) || 1) 784 ret |= bdi_congested(q->backing_dev_info, bits); 785 else 786 ret &= bdi_congested(q->backing_dev_info, bits); 787 } 788 } 789 rcu_read_unlock(); 790 return ret; 791 } 792 793 static void flush_bio_list(struct r1conf *conf, struct bio *bio) 794 { 795 /* flush any pending bitmap writes to disk before proceeding w/ I/O */ 796 bitmap_unplug(conf->mddev->bitmap); 797 wake_up(&conf->wait_barrier); 798 799 while (bio) { /* submit pending writes */ 800 struct bio *next = bio->bi_next; 801 struct md_rdev *rdev = (void*)bio->bi_bdev; 802 bio->bi_next = NULL; 803 bio->bi_bdev = rdev->bdev; 804 if (test_bit(Faulty, &rdev->flags)) { 805 bio->bi_status = BLK_STS_IOERR; 806 bio_endio(bio); 807 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 808 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 809 /* Just ignore it */ 810 bio_endio(bio); 811 else 812 generic_make_request(bio); 813 bio = next; 814 } 815 } 816 817 static void flush_pending_writes(struct r1conf *conf) 818 { 819 /* Any writes that have been queued but are awaiting 820 * bitmap updates get flushed here. 821 */ 822 spin_lock_irq(&conf->device_lock); 823 824 if (conf->pending_bio_list.head) { 825 struct bio *bio; 826 bio = bio_list_get(&conf->pending_bio_list); 827 conf->pending_count = 0; 828 spin_unlock_irq(&conf->device_lock); 829 flush_bio_list(conf, bio); 830 } else 831 spin_unlock_irq(&conf->device_lock); 832 } 833 834 /* Barriers.... 835 * Sometimes we need to suspend IO while we do something else, 836 * either some resync/recovery, or reconfigure the array. 837 * To do this we raise a 'barrier'. 838 * The 'barrier' is a counter that can be raised multiple times 839 * to count how many activities are happening which preclude 840 * normal IO. 841 * We can only raise the barrier if there is no pending IO. 842 * i.e. if nr_pending == 0. 843 * We choose only to raise the barrier if no-one is waiting for the 844 * barrier to go down. This means that as soon as an IO request 845 * is ready, no other operations which require a barrier will start 846 * until the IO request has had a chance. 847 * 848 * So: regular IO calls 'wait_barrier'. When that returns there 849 * is no backgroup IO happening, It must arrange to call 850 * allow_barrier when it has finished its IO. 851 * backgroup IO calls must call raise_barrier. Once that returns 852 * there is no normal IO happeing. It must arrange to call 853 * lower_barrier when the particular background IO completes. 854 */ 855 static void raise_barrier(struct r1conf *conf, sector_t sector_nr) 856 { 857 int idx = sector_to_idx(sector_nr); 858 859 spin_lock_irq(&conf->resync_lock); 860 861 /* Wait until no block IO is waiting */ 862 wait_event_lock_irq(conf->wait_barrier, 863 !atomic_read(&conf->nr_waiting[idx]), 864 conf->resync_lock); 865 866 /* block any new IO from starting */ 867 atomic_inc(&conf->barrier[idx]); 868 /* 869 * In raise_barrier() we firstly increase conf->barrier[idx] then 870 * check conf->nr_pending[idx]. In _wait_barrier() we firstly 871 * increase conf->nr_pending[idx] then check conf->barrier[idx]. 872 * A memory barrier here to make sure conf->nr_pending[idx] won't 873 * be fetched before conf->barrier[idx] is increased. Otherwise 874 * there will be a race between raise_barrier() and _wait_barrier(). 875 */ 876 smp_mb__after_atomic(); 877 878 /* For these conditions we must wait: 879 * A: while the array is in frozen state 880 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O 881 * existing in corresponding I/O barrier bucket. 882 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches 883 * max resync count which allowed on current I/O barrier bucket. 884 */ 885 wait_event_lock_irq(conf->wait_barrier, 886 !conf->array_frozen && 887 !atomic_read(&conf->nr_pending[idx]) && 888 atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH, 889 conf->resync_lock); 890 891 atomic_inc(&conf->nr_sync_pending); 892 spin_unlock_irq(&conf->resync_lock); 893 } 894 895 static void lower_barrier(struct r1conf *conf, sector_t sector_nr) 896 { 897 int idx = sector_to_idx(sector_nr); 898 899 BUG_ON(atomic_read(&conf->barrier[idx]) <= 0); 900 901 atomic_dec(&conf->barrier[idx]); 902 atomic_dec(&conf->nr_sync_pending); 903 wake_up(&conf->wait_barrier); 904 } 905 906 static void _wait_barrier(struct r1conf *conf, int idx) 907 { 908 /* 909 * We need to increase conf->nr_pending[idx] very early here, 910 * then raise_barrier() can be blocked when it waits for 911 * conf->nr_pending[idx] to be 0. Then we can avoid holding 912 * conf->resync_lock when there is no barrier raised in same 913 * barrier unit bucket. Also if the array is frozen, I/O 914 * should be blocked until array is unfrozen. 915 */ 916 atomic_inc(&conf->nr_pending[idx]); 917 /* 918 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then 919 * check conf->barrier[idx]. In raise_barrier() we firstly increase 920 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory 921 * barrier is necessary here to make sure conf->barrier[idx] won't be 922 * fetched before conf->nr_pending[idx] is increased. Otherwise there 923 * will be a race between _wait_barrier() and raise_barrier(). 924 */ 925 smp_mb__after_atomic(); 926 927 /* 928 * Don't worry about checking two atomic_t variables at same time 929 * here. If during we check conf->barrier[idx], the array is 930 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is 931 * 0, it is safe to return and make the I/O continue. Because the 932 * array is frozen, all I/O returned here will eventually complete 933 * or be queued, no race will happen. See code comment in 934 * frozen_array(). 935 */ 936 if (!READ_ONCE(conf->array_frozen) && 937 !atomic_read(&conf->barrier[idx])) 938 return; 939 940 /* 941 * After holding conf->resync_lock, conf->nr_pending[idx] 942 * should be decreased before waiting for barrier to drop. 943 * Otherwise, we may encounter a race condition because 944 * raise_barrer() might be waiting for conf->nr_pending[idx] 945 * to be 0 at same time. 946 */ 947 spin_lock_irq(&conf->resync_lock); 948 atomic_inc(&conf->nr_waiting[idx]); 949 atomic_dec(&conf->nr_pending[idx]); 950 /* 951 * In case freeze_array() is waiting for 952 * get_unqueued_pending() == extra 953 */ 954 wake_up(&conf->wait_barrier); 955 /* Wait for the barrier in same barrier unit bucket to drop. */ 956 wait_event_lock_irq(conf->wait_barrier, 957 !conf->array_frozen && 958 !atomic_read(&conf->barrier[idx]), 959 conf->resync_lock); 960 atomic_inc(&conf->nr_pending[idx]); 961 atomic_dec(&conf->nr_waiting[idx]); 962 spin_unlock_irq(&conf->resync_lock); 963 } 964 965 static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) 966 { 967 int idx = sector_to_idx(sector_nr); 968 969 /* 970 * Very similar to _wait_barrier(). The difference is, for read 971 * I/O we don't need wait for sync I/O, but if the whole array 972 * is frozen, the read I/O still has to wait until the array is 973 * unfrozen. Since there is no ordering requirement with 974 * conf->barrier[idx] here, memory barrier is unnecessary as well. 975 */ 976 atomic_inc(&conf->nr_pending[idx]); 977 978 if (!READ_ONCE(conf->array_frozen)) 979 return; 980 981 spin_lock_irq(&conf->resync_lock); 982 atomic_inc(&conf->nr_waiting[idx]); 983 atomic_dec(&conf->nr_pending[idx]); 984 /* 985 * In case freeze_array() is waiting for 986 * get_unqueued_pending() == extra 987 */ 988 wake_up(&conf->wait_barrier); 989 /* Wait for array to be unfrozen */ 990 wait_event_lock_irq(conf->wait_barrier, 991 !conf->array_frozen, 992 conf->resync_lock); 993 atomic_inc(&conf->nr_pending[idx]); 994 atomic_dec(&conf->nr_waiting[idx]); 995 spin_unlock_irq(&conf->resync_lock); 996 } 997 998 static void wait_barrier(struct r1conf *conf, sector_t sector_nr) 999 { 1000 int idx = sector_to_idx(sector_nr); 1001 1002 _wait_barrier(conf, idx); 1003 } 1004 1005 static void wait_all_barriers(struct r1conf *conf) 1006 { 1007 int idx; 1008 1009 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) 1010 _wait_barrier(conf, idx); 1011 } 1012 1013 static void _allow_barrier(struct r1conf *conf, int idx) 1014 { 1015 atomic_dec(&conf->nr_pending[idx]); 1016 wake_up(&conf->wait_barrier); 1017 } 1018 1019 static void allow_barrier(struct r1conf *conf, sector_t sector_nr) 1020 { 1021 int idx = sector_to_idx(sector_nr); 1022 1023 _allow_barrier(conf, idx); 1024 } 1025 1026 static void allow_all_barriers(struct r1conf *conf) 1027 { 1028 int idx; 1029 1030 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) 1031 _allow_barrier(conf, idx); 1032 } 1033 1034 /* conf->resync_lock should be held */ 1035 static int get_unqueued_pending(struct r1conf *conf) 1036 { 1037 int idx, ret; 1038 1039 ret = atomic_read(&conf->nr_sync_pending); 1040 for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) 1041 ret += atomic_read(&conf->nr_pending[idx]) - 1042 atomic_read(&conf->nr_queued[idx]); 1043 1044 return ret; 1045 } 1046 1047 static void freeze_array(struct r1conf *conf, int extra) 1048 { 1049 /* Stop sync I/O and normal I/O and wait for everything to 1050 * go quiet. 1051 * This is called in two situations: 1052 * 1) management command handlers (reshape, remove disk, quiesce). 1053 * 2) one normal I/O request failed. 1054 1055 * After array_frozen is set to 1, new sync IO will be blocked at 1056 * raise_barrier(), and new normal I/O will blocked at _wait_barrier() 1057 * or wait_read_barrier(). The flying I/Os will either complete or be 1058 * queued. When everything goes quite, there are only queued I/Os left. 1059 1060 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the 1061 * barrier bucket index which this I/O request hits. When all sync and 1062 * normal I/O are queued, sum of all conf->nr_pending[] will match sum 1063 * of all conf->nr_queued[]. But normal I/O failure is an exception, 1064 * in handle_read_error(), we may call freeze_array() before trying to 1065 * fix the read error. In this case, the error read I/O is not queued, 1066 * so get_unqueued_pending() == 1. 1067 * 1068 * Therefore before this function returns, we need to wait until 1069 * get_unqueued_pendings(conf) gets equal to extra. For 1070 * normal I/O context, extra is 1, in rested situations extra is 0. 1071 */ 1072 spin_lock_irq(&conf->resync_lock); 1073 conf->array_frozen = 1; 1074 raid1_log(conf->mddev, "wait freeze"); 1075 wait_event_lock_irq_cmd( 1076 conf->wait_barrier, 1077 get_unqueued_pending(conf) == extra, 1078 conf->resync_lock, 1079 flush_pending_writes(conf)); 1080 spin_unlock_irq(&conf->resync_lock); 1081 } 1082 static void unfreeze_array(struct r1conf *conf) 1083 { 1084 /* reverse the effect of the freeze */ 1085 spin_lock_irq(&conf->resync_lock); 1086 conf->array_frozen = 0; 1087 spin_unlock_irq(&conf->resync_lock); 1088 wake_up(&conf->wait_barrier); 1089 } 1090 1091 static struct bio *alloc_behind_master_bio(struct r1bio *r1_bio, 1092 struct bio *bio) 1093 { 1094 int size = bio->bi_iter.bi_size; 1095 unsigned vcnt = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1096 int i = 0; 1097 struct bio *behind_bio = NULL; 1098 1099 behind_bio = bio_alloc_mddev(GFP_NOIO, vcnt, r1_bio->mddev); 1100 if (!behind_bio) 1101 goto fail; 1102 1103 /* discard op, we don't support writezero/writesame yet */ 1104 if (!bio_has_data(bio)) 1105 goto skip_copy; 1106 1107 while (i < vcnt && size) { 1108 struct page *page; 1109 int len = min_t(int, PAGE_SIZE, size); 1110 1111 page = alloc_page(GFP_NOIO); 1112 if (unlikely(!page)) 1113 goto free_pages; 1114 1115 bio_add_page(behind_bio, page, len, 0); 1116 1117 size -= len; 1118 i++; 1119 } 1120 1121 bio_copy_data(behind_bio, bio); 1122 skip_copy: 1123 r1_bio->behind_master_bio = behind_bio;; 1124 set_bit(R1BIO_BehindIO, &r1_bio->state); 1125 1126 return behind_bio; 1127 1128 free_pages: 1129 pr_debug("%dB behind alloc failed, doing sync I/O\n", 1130 bio->bi_iter.bi_size); 1131 bio_free_pages(behind_bio); 1132 fail: 1133 return behind_bio; 1134 } 1135 1136 struct raid1_plug_cb { 1137 struct blk_plug_cb cb; 1138 struct bio_list pending; 1139 int pending_cnt; 1140 }; 1141 1142 static void raid1_unplug(struct blk_plug_cb *cb, bool from_schedule) 1143 { 1144 struct raid1_plug_cb *plug = container_of(cb, struct raid1_plug_cb, 1145 cb); 1146 struct mddev *mddev = plug->cb.data; 1147 struct r1conf *conf = mddev->private; 1148 struct bio *bio; 1149 1150 if (from_schedule || current->bio_list) { 1151 spin_lock_irq(&conf->device_lock); 1152 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1153 conf->pending_count += plug->pending_cnt; 1154 spin_unlock_irq(&conf->device_lock); 1155 wake_up(&conf->wait_barrier); 1156 md_wakeup_thread(mddev->thread); 1157 kfree(plug); 1158 return; 1159 } 1160 1161 /* we aren't scheduling, so we can do the write-out directly. */ 1162 bio = bio_list_get(&plug->pending); 1163 flush_bio_list(conf, bio); 1164 kfree(plug); 1165 } 1166 1167 static void init_r1bio(struct r1bio *r1_bio, struct mddev *mddev, struct bio *bio) 1168 { 1169 r1_bio->master_bio = bio; 1170 r1_bio->sectors = bio_sectors(bio); 1171 r1_bio->state = 0; 1172 r1_bio->mddev = mddev; 1173 r1_bio->sector = bio->bi_iter.bi_sector; 1174 } 1175 1176 static inline struct r1bio * 1177 alloc_r1bio(struct mddev *mddev, struct bio *bio) 1178 { 1179 struct r1conf *conf = mddev->private; 1180 struct r1bio *r1_bio; 1181 1182 r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO); 1183 /* Ensure no bio records IO_BLOCKED */ 1184 memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0])); 1185 init_r1bio(r1_bio, mddev, bio); 1186 return r1_bio; 1187 } 1188 1189 static void raid1_read_request(struct mddev *mddev, struct bio *bio, 1190 int max_read_sectors, struct r1bio *r1_bio) 1191 { 1192 struct r1conf *conf = mddev->private; 1193 struct raid1_info *mirror; 1194 struct bio *read_bio; 1195 struct bitmap *bitmap = mddev->bitmap; 1196 const int op = bio_op(bio); 1197 const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); 1198 int max_sectors; 1199 int rdisk; 1200 bool print_msg = !!r1_bio; 1201 char b[BDEVNAME_SIZE]; 1202 1203 /* 1204 * If r1_bio is set, we are blocking the raid1d thread 1205 * so there is a tiny risk of deadlock. So ask for 1206 * emergency memory if needed. 1207 */ 1208 gfp_t gfp = r1_bio ? (GFP_NOIO | __GFP_HIGH) : GFP_NOIO; 1209 1210 if (print_msg) { 1211 /* Need to get the block device name carefully */ 1212 struct md_rdev *rdev; 1213 rcu_read_lock(); 1214 rdev = rcu_dereference(conf->mirrors[r1_bio->read_disk].rdev); 1215 if (rdev) 1216 bdevname(rdev->bdev, b); 1217 else 1218 strcpy(b, "???"); 1219 rcu_read_unlock(); 1220 } 1221 1222 /* 1223 * Still need barrier for READ in case that whole 1224 * array is frozen. 1225 */ 1226 wait_read_barrier(conf, bio->bi_iter.bi_sector); 1227 1228 if (!r1_bio) 1229 r1_bio = alloc_r1bio(mddev, bio); 1230 else 1231 init_r1bio(r1_bio, mddev, bio); 1232 r1_bio->sectors = max_read_sectors; 1233 1234 /* 1235 * make_request() can abort the operation when read-ahead is being 1236 * used and no empty request is available. 1237 */ 1238 rdisk = read_balance(conf, r1_bio, &max_sectors); 1239 1240 if (rdisk < 0) { 1241 /* couldn't find anywhere to read from */ 1242 if (print_msg) { 1243 pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n", 1244 mdname(mddev), 1245 b, 1246 (unsigned long long)r1_bio->sector); 1247 } 1248 raid_end_bio_io(r1_bio); 1249 return; 1250 } 1251 mirror = conf->mirrors + rdisk; 1252 1253 if (print_msg) 1254 pr_info_ratelimited("md/raid1:%s: redirecting sector %llu to other mirror: %s\n", 1255 mdname(mddev), 1256 (unsigned long long)r1_bio->sector, 1257 bdevname(mirror->rdev->bdev, b)); 1258 1259 if (test_bit(WriteMostly, &mirror->rdev->flags) && 1260 bitmap) { 1261 /* 1262 * Reading from a write-mostly device must take care not to 1263 * over-take any writes that are 'behind' 1264 */ 1265 raid1_log(mddev, "wait behind writes"); 1266 wait_event(bitmap->behind_wait, 1267 atomic_read(&bitmap->behind_writes) == 0); 1268 } 1269 1270 if (max_sectors < bio_sectors(bio)) { 1271 struct bio *split = bio_split(bio, max_sectors, 1272 gfp, conf->bio_split); 1273 bio_chain(split, bio); 1274 generic_make_request(bio); 1275 bio = split; 1276 r1_bio->master_bio = bio; 1277 r1_bio->sectors = max_sectors; 1278 } 1279 1280 r1_bio->read_disk = rdisk; 1281 1282 read_bio = bio_clone_fast(bio, gfp, mddev->bio_set); 1283 1284 r1_bio->bios[rdisk] = read_bio; 1285 1286 read_bio->bi_iter.bi_sector = r1_bio->sector + 1287 mirror->rdev->data_offset; 1288 read_bio->bi_bdev = mirror->rdev->bdev; 1289 read_bio->bi_end_io = raid1_end_read_request; 1290 bio_set_op_attrs(read_bio, op, do_sync); 1291 if (test_bit(FailFast, &mirror->rdev->flags) && 1292 test_bit(R1BIO_FailFast, &r1_bio->state)) 1293 read_bio->bi_opf |= MD_FAILFAST; 1294 read_bio->bi_private = r1_bio; 1295 1296 if (mddev->gendisk) 1297 trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev), 1298 read_bio, disk_devt(mddev->gendisk), 1299 r1_bio->sector); 1300 1301 generic_make_request(read_bio); 1302 } 1303 1304 static void raid1_write_request(struct mddev *mddev, struct bio *bio, 1305 int max_write_sectors) 1306 { 1307 struct r1conf *conf = mddev->private; 1308 struct r1bio *r1_bio; 1309 int i, disks; 1310 struct bitmap *bitmap = mddev->bitmap; 1311 unsigned long flags; 1312 struct md_rdev *blocked_rdev; 1313 struct blk_plug_cb *cb; 1314 struct raid1_plug_cb *plug = NULL; 1315 int first_clone; 1316 int max_sectors; 1317 1318 /* 1319 * Register the new request and wait if the reconstruction 1320 * thread has put up a bar for new requests. 1321 * Continue immediately if no resync is active currently. 1322 */ 1323 1324 1325 if ((bio_end_sector(bio) > mddev->suspend_lo && 1326 bio->bi_iter.bi_sector < mddev->suspend_hi) || 1327 (mddev_is_clustered(mddev) && 1328 md_cluster_ops->area_resyncing(mddev, WRITE, 1329 bio->bi_iter.bi_sector, bio_end_sector(bio)))) { 1330 1331 /* 1332 * As the suspend_* range is controlled by userspace, we want 1333 * an interruptible wait. 1334 */ 1335 DEFINE_WAIT(w); 1336 for (;;) { 1337 sigset_t full, old; 1338 prepare_to_wait(&conf->wait_barrier, 1339 &w, TASK_INTERRUPTIBLE); 1340 if (bio_end_sector(bio) <= mddev->suspend_lo || 1341 bio->bi_iter.bi_sector >= mddev->suspend_hi || 1342 (mddev_is_clustered(mddev) && 1343 !md_cluster_ops->area_resyncing(mddev, WRITE, 1344 bio->bi_iter.bi_sector, 1345 bio_end_sector(bio)))) 1346 break; 1347 sigfillset(&full); 1348 sigprocmask(SIG_BLOCK, &full, &old); 1349 schedule(); 1350 sigprocmask(SIG_SETMASK, &old, NULL); 1351 } 1352 finish_wait(&conf->wait_barrier, &w); 1353 } 1354 wait_barrier(conf, bio->bi_iter.bi_sector); 1355 1356 r1_bio = alloc_r1bio(mddev, bio); 1357 r1_bio->sectors = max_write_sectors; 1358 1359 if (conf->pending_count >= max_queued_requests) { 1360 md_wakeup_thread(mddev->thread); 1361 raid1_log(mddev, "wait queued"); 1362 wait_event(conf->wait_barrier, 1363 conf->pending_count < max_queued_requests); 1364 } 1365 /* first select target devices under rcu_lock and 1366 * inc refcount on their rdev. Record them by setting 1367 * bios[x] to bio 1368 * If there are known/acknowledged bad blocks on any device on 1369 * which we have seen a write error, we want to avoid writing those 1370 * blocks. 1371 * This potentially requires several writes to write around 1372 * the bad blocks. Each set of writes gets it's own r1bio 1373 * with a set of bios attached. 1374 */ 1375 1376 disks = conf->raid_disks * 2; 1377 retry_write: 1378 blocked_rdev = NULL; 1379 rcu_read_lock(); 1380 max_sectors = r1_bio->sectors; 1381 for (i = 0; i < disks; i++) { 1382 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1383 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1384 atomic_inc(&rdev->nr_pending); 1385 blocked_rdev = rdev; 1386 break; 1387 } 1388 r1_bio->bios[i] = NULL; 1389 if (!rdev || test_bit(Faulty, &rdev->flags)) { 1390 if (i < conf->raid_disks) 1391 set_bit(R1BIO_Degraded, &r1_bio->state); 1392 continue; 1393 } 1394 1395 atomic_inc(&rdev->nr_pending); 1396 if (test_bit(WriteErrorSeen, &rdev->flags)) { 1397 sector_t first_bad; 1398 int bad_sectors; 1399 int is_bad; 1400 1401 is_bad = is_badblock(rdev, r1_bio->sector, max_sectors, 1402 &first_bad, &bad_sectors); 1403 if (is_bad < 0) { 1404 /* mustn't write here until the bad block is 1405 * acknowledged*/ 1406 set_bit(BlockedBadBlocks, &rdev->flags); 1407 blocked_rdev = rdev; 1408 break; 1409 } 1410 if (is_bad && first_bad <= r1_bio->sector) { 1411 /* Cannot write here at all */ 1412 bad_sectors -= (r1_bio->sector - first_bad); 1413 if (bad_sectors < max_sectors) 1414 /* mustn't write more than bad_sectors 1415 * to other devices yet 1416 */ 1417 max_sectors = bad_sectors; 1418 rdev_dec_pending(rdev, mddev); 1419 /* We don't set R1BIO_Degraded as that 1420 * only applies if the disk is 1421 * missing, so it might be re-added, 1422 * and we want to know to recover this 1423 * chunk. 1424 * In this case the device is here, 1425 * and the fact that this chunk is not 1426 * in-sync is recorded in the bad 1427 * block log 1428 */ 1429 continue; 1430 } 1431 if (is_bad) { 1432 int good_sectors = first_bad - r1_bio->sector; 1433 if (good_sectors < max_sectors) 1434 max_sectors = good_sectors; 1435 } 1436 } 1437 r1_bio->bios[i] = bio; 1438 } 1439 rcu_read_unlock(); 1440 1441 if (unlikely(blocked_rdev)) { 1442 /* Wait for this device to become unblocked */ 1443 int j; 1444 1445 for (j = 0; j < i; j++) 1446 if (r1_bio->bios[j]) 1447 rdev_dec_pending(conf->mirrors[j].rdev, mddev); 1448 r1_bio->state = 0; 1449 allow_barrier(conf, bio->bi_iter.bi_sector); 1450 raid1_log(mddev, "wait rdev %d blocked", blocked_rdev->raid_disk); 1451 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1452 wait_barrier(conf, bio->bi_iter.bi_sector); 1453 goto retry_write; 1454 } 1455 1456 if (max_sectors < bio_sectors(bio)) { 1457 struct bio *split = bio_split(bio, max_sectors, 1458 GFP_NOIO, conf->bio_split); 1459 bio_chain(split, bio); 1460 generic_make_request(bio); 1461 bio = split; 1462 r1_bio->master_bio = bio; 1463 r1_bio->sectors = max_sectors; 1464 } 1465 1466 atomic_set(&r1_bio->remaining, 1); 1467 atomic_set(&r1_bio->behind_remaining, 0); 1468 1469 first_clone = 1; 1470 1471 for (i = 0; i < disks; i++) { 1472 struct bio *mbio = NULL; 1473 if (!r1_bio->bios[i]) 1474 continue; 1475 1476 1477 if (first_clone) { 1478 /* do behind I/O ? 1479 * Not if there are too many, or cannot 1480 * allocate memory, or a reader on WriteMostly 1481 * is waiting for behind writes to flush */ 1482 if (bitmap && 1483 (atomic_read(&bitmap->behind_writes) 1484 < mddev->bitmap_info.max_write_behind) && 1485 !waitqueue_active(&bitmap->behind_wait)) { 1486 mbio = alloc_behind_master_bio(r1_bio, bio); 1487 } 1488 1489 bitmap_startwrite(bitmap, r1_bio->sector, 1490 r1_bio->sectors, 1491 test_bit(R1BIO_BehindIO, 1492 &r1_bio->state)); 1493 first_clone = 0; 1494 } 1495 1496 if (!mbio) { 1497 if (r1_bio->behind_master_bio) 1498 mbio = bio_clone_fast(r1_bio->behind_master_bio, 1499 GFP_NOIO, 1500 mddev->bio_set); 1501 else 1502 mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set); 1503 } 1504 1505 if (r1_bio->behind_master_bio) { 1506 if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags)) 1507 atomic_inc(&r1_bio->behind_remaining); 1508 } 1509 1510 r1_bio->bios[i] = mbio; 1511 1512 mbio->bi_iter.bi_sector = (r1_bio->sector + 1513 conf->mirrors[i].rdev->data_offset); 1514 mbio->bi_bdev = conf->mirrors[i].rdev->bdev; 1515 mbio->bi_end_io = raid1_end_write_request; 1516 mbio->bi_opf = bio_op(bio) | (bio->bi_opf & (REQ_SYNC | REQ_FUA)); 1517 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags) && 1518 !test_bit(WriteMostly, &conf->mirrors[i].rdev->flags) && 1519 conf->raid_disks - mddev->degraded > 1) 1520 mbio->bi_opf |= MD_FAILFAST; 1521 mbio->bi_private = r1_bio; 1522 1523 atomic_inc(&r1_bio->remaining); 1524 1525 if (mddev->gendisk) 1526 trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev), 1527 mbio, disk_devt(mddev->gendisk), 1528 r1_bio->sector); 1529 /* flush_pending_writes() needs access to the rdev so...*/ 1530 mbio->bi_bdev = (void*)conf->mirrors[i].rdev; 1531 1532 cb = blk_check_plugged(raid1_unplug, mddev, sizeof(*plug)); 1533 if (cb) 1534 plug = container_of(cb, struct raid1_plug_cb, cb); 1535 else 1536 plug = NULL; 1537 if (plug) { 1538 bio_list_add(&plug->pending, mbio); 1539 plug->pending_cnt++; 1540 } else { 1541 spin_lock_irqsave(&conf->device_lock, flags); 1542 bio_list_add(&conf->pending_bio_list, mbio); 1543 conf->pending_count++; 1544 spin_unlock_irqrestore(&conf->device_lock, flags); 1545 md_wakeup_thread(mddev->thread); 1546 } 1547 } 1548 1549 r1_bio_write_done(r1_bio); 1550 1551 /* In case raid1d snuck in to freeze_array */ 1552 wake_up(&conf->wait_barrier); 1553 } 1554 1555 static bool raid1_make_request(struct mddev *mddev, struct bio *bio) 1556 { 1557 sector_t sectors; 1558 1559 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1560 md_flush_request(mddev, bio); 1561 return true; 1562 } 1563 1564 /* 1565 * There is a limit to the maximum size, but 1566 * the read/write handler might find a lower limit 1567 * due to bad blocks. To avoid multiple splits, 1568 * we pass the maximum number of sectors down 1569 * and let the lower level perform the split. 1570 */ 1571 sectors = align_to_barrier_unit_end( 1572 bio->bi_iter.bi_sector, bio_sectors(bio)); 1573 1574 if (bio_data_dir(bio) == READ) 1575 raid1_read_request(mddev, bio, sectors, NULL); 1576 else { 1577 if (!md_write_start(mddev,bio)) 1578 return false; 1579 raid1_write_request(mddev, bio, sectors); 1580 } 1581 return true; 1582 } 1583 1584 static void raid1_status(struct seq_file *seq, struct mddev *mddev) 1585 { 1586 struct r1conf *conf = mddev->private; 1587 int i; 1588 1589 seq_printf(seq, " [%d/%d] [", conf->raid_disks, 1590 conf->raid_disks - mddev->degraded); 1591 rcu_read_lock(); 1592 for (i = 0; i < conf->raid_disks; i++) { 1593 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1594 seq_printf(seq, "%s", 1595 rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1596 } 1597 rcu_read_unlock(); 1598 seq_printf(seq, "]"); 1599 } 1600 1601 static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) 1602 { 1603 char b[BDEVNAME_SIZE]; 1604 struct r1conf *conf = mddev->private; 1605 unsigned long flags; 1606 1607 /* 1608 * If it is not operational, then we have already marked it as dead 1609 * else if it is the last working disks, ignore the error, let the 1610 * next level up know. 1611 * else mark the drive as failed 1612 */ 1613 spin_lock_irqsave(&conf->device_lock, flags); 1614 if (test_bit(In_sync, &rdev->flags) 1615 && (conf->raid_disks - mddev->degraded) == 1) { 1616 /* 1617 * Don't fail the drive, act as though we were just a 1618 * normal single drive. 1619 * However don't try a recovery from this drive as 1620 * it is very likely to fail. 1621 */ 1622 conf->recovery_disabled = mddev->recovery_disabled; 1623 spin_unlock_irqrestore(&conf->device_lock, flags); 1624 return; 1625 } 1626 set_bit(Blocked, &rdev->flags); 1627 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1628 mddev->degraded++; 1629 set_bit(Faulty, &rdev->flags); 1630 } else 1631 set_bit(Faulty, &rdev->flags); 1632 spin_unlock_irqrestore(&conf->device_lock, flags); 1633 /* 1634 * if recovery is running, make sure it aborts. 1635 */ 1636 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1637 set_mask_bits(&mddev->sb_flags, 0, 1638 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1639 pr_crit("md/raid1:%s: Disk failure on %s, disabling device.\n" 1640 "md/raid1:%s: Operation continuing on %d devices.\n", 1641 mdname(mddev), bdevname(rdev->bdev, b), 1642 mdname(mddev), conf->raid_disks - mddev->degraded); 1643 } 1644 1645 static void print_conf(struct r1conf *conf) 1646 { 1647 int i; 1648 1649 pr_debug("RAID1 conf printout:\n"); 1650 if (!conf) { 1651 pr_debug("(!conf)\n"); 1652 return; 1653 } 1654 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 1655 conf->raid_disks); 1656 1657 rcu_read_lock(); 1658 for (i = 0; i < conf->raid_disks; i++) { 1659 char b[BDEVNAME_SIZE]; 1660 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1661 if (rdev) 1662 pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n", 1663 i, !test_bit(In_sync, &rdev->flags), 1664 !test_bit(Faulty, &rdev->flags), 1665 bdevname(rdev->bdev,b)); 1666 } 1667 rcu_read_unlock(); 1668 } 1669 1670 static void close_sync(struct r1conf *conf) 1671 { 1672 wait_all_barriers(conf); 1673 allow_all_barriers(conf); 1674 1675 mempool_destroy(conf->r1buf_pool); 1676 conf->r1buf_pool = NULL; 1677 } 1678 1679 static int raid1_spare_active(struct mddev *mddev) 1680 { 1681 int i; 1682 struct r1conf *conf = mddev->private; 1683 int count = 0; 1684 unsigned long flags; 1685 1686 /* 1687 * Find all failed disks within the RAID1 configuration 1688 * and mark them readable. 1689 * Called under mddev lock, so rcu protection not needed. 1690 * device_lock used to avoid races with raid1_end_read_request 1691 * which expects 'In_sync' flags and ->degraded to be consistent. 1692 */ 1693 spin_lock_irqsave(&conf->device_lock, flags); 1694 for (i = 0; i < conf->raid_disks; i++) { 1695 struct md_rdev *rdev = conf->mirrors[i].rdev; 1696 struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev; 1697 if (repl 1698 && !test_bit(Candidate, &repl->flags) 1699 && repl->recovery_offset == MaxSector 1700 && !test_bit(Faulty, &repl->flags) 1701 && !test_and_set_bit(In_sync, &repl->flags)) { 1702 /* replacement has just become active */ 1703 if (!rdev || 1704 !test_and_clear_bit(In_sync, &rdev->flags)) 1705 count++; 1706 if (rdev) { 1707 /* Replaced device not technically 1708 * faulty, but we need to be sure 1709 * it gets removed and never re-added 1710 */ 1711 set_bit(Faulty, &rdev->flags); 1712 sysfs_notify_dirent_safe( 1713 rdev->sysfs_state); 1714 } 1715 } 1716 if (rdev 1717 && rdev->recovery_offset == MaxSector 1718 && !test_bit(Faulty, &rdev->flags) 1719 && !test_and_set_bit(In_sync, &rdev->flags)) { 1720 count++; 1721 sysfs_notify_dirent_safe(rdev->sysfs_state); 1722 } 1723 } 1724 mddev->degraded -= count; 1725 spin_unlock_irqrestore(&conf->device_lock, flags); 1726 1727 print_conf(conf); 1728 return count; 1729 } 1730 1731 static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1732 { 1733 struct r1conf *conf = mddev->private; 1734 int err = -EEXIST; 1735 int mirror = 0; 1736 struct raid1_info *p; 1737 int first = 0; 1738 int last = conf->raid_disks - 1; 1739 1740 if (mddev->recovery_disabled == conf->recovery_disabled) 1741 return -EBUSY; 1742 1743 if (md_integrity_add_rdev(rdev, mddev)) 1744 return -ENXIO; 1745 1746 if (rdev->raid_disk >= 0) 1747 first = last = rdev->raid_disk; 1748 1749 /* 1750 * find the disk ... but prefer rdev->saved_raid_disk 1751 * if possible. 1752 */ 1753 if (rdev->saved_raid_disk >= 0 && 1754 rdev->saved_raid_disk >= first && 1755 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1756 first = last = rdev->saved_raid_disk; 1757 1758 for (mirror = first; mirror <= last; mirror++) { 1759 p = conf->mirrors+mirror; 1760 if (!p->rdev) { 1761 1762 if (mddev->gendisk) 1763 disk_stack_limits(mddev->gendisk, rdev->bdev, 1764 rdev->data_offset << 9); 1765 1766 p->head_position = 0; 1767 rdev->raid_disk = mirror; 1768 err = 0; 1769 /* As all devices are equivalent, we don't need a full recovery 1770 * if this was recently any drive of the array 1771 */ 1772 if (rdev->saved_raid_disk < 0) 1773 conf->fullsync = 1; 1774 rcu_assign_pointer(p->rdev, rdev); 1775 break; 1776 } 1777 if (test_bit(WantReplacement, &p->rdev->flags) && 1778 p[conf->raid_disks].rdev == NULL) { 1779 /* Add this device as a replacement */ 1780 clear_bit(In_sync, &rdev->flags); 1781 set_bit(Replacement, &rdev->flags); 1782 rdev->raid_disk = mirror; 1783 err = 0; 1784 conf->fullsync = 1; 1785 rcu_assign_pointer(p[conf->raid_disks].rdev, rdev); 1786 break; 1787 } 1788 } 1789 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1790 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1791 print_conf(conf); 1792 return err; 1793 } 1794 1795 static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1796 { 1797 struct r1conf *conf = mddev->private; 1798 int err = 0; 1799 int number = rdev->raid_disk; 1800 struct raid1_info *p = conf->mirrors + number; 1801 1802 if (rdev != p->rdev) 1803 p = conf->mirrors + conf->raid_disks + number; 1804 1805 print_conf(conf); 1806 if (rdev == p->rdev) { 1807 if (test_bit(In_sync, &rdev->flags) || 1808 atomic_read(&rdev->nr_pending)) { 1809 err = -EBUSY; 1810 goto abort; 1811 } 1812 /* Only remove non-faulty devices if recovery 1813 * is not possible. 1814 */ 1815 if (!test_bit(Faulty, &rdev->flags) && 1816 mddev->recovery_disabled != conf->recovery_disabled && 1817 mddev->degraded < conf->raid_disks) { 1818 err = -EBUSY; 1819 goto abort; 1820 } 1821 p->rdev = NULL; 1822 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 1823 synchronize_rcu(); 1824 if (atomic_read(&rdev->nr_pending)) { 1825 /* lost the race, try later */ 1826 err = -EBUSY; 1827 p->rdev = rdev; 1828 goto abort; 1829 } 1830 } 1831 if (conf->mirrors[conf->raid_disks + number].rdev) { 1832 /* We just removed a device that is being replaced. 1833 * Move down the replacement. We drain all IO before 1834 * doing this to avoid confusion. 1835 */ 1836 struct md_rdev *repl = 1837 conf->mirrors[conf->raid_disks + number].rdev; 1838 freeze_array(conf, 0); 1839 clear_bit(Replacement, &repl->flags); 1840 p->rdev = repl; 1841 conf->mirrors[conf->raid_disks + number].rdev = NULL; 1842 unfreeze_array(conf); 1843 } 1844 1845 clear_bit(WantReplacement, &rdev->flags); 1846 err = md_integrity_register(mddev); 1847 } 1848 abort: 1849 1850 print_conf(conf); 1851 return err; 1852 } 1853 1854 static void end_sync_read(struct bio *bio) 1855 { 1856 struct r1bio *r1_bio = get_resync_r1bio(bio); 1857 1858 update_head_pos(r1_bio->read_disk, r1_bio); 1859 1860 /* 1861 * we have read a block, now it needs to be re-written, 1862 * or re-read if the read failed. 1863 * We don't do much here, just schedule handling by raid1d 1864 */ 1865 if (!bio->bi_status) 1866 set_bit(R1BIO_Uptodate, &r1_bio->state); 1867 1868 if (atomic_dec_and_test(&r1_bio->remaining)) 1869 reschedule_retry(r1_bio); 1870 } 1871 1872 static void end_sync_write(struct bio *bio) 1873 { 1874 int uptodate = !bio->bi_status; 1875 struct r1bio *r1_bio = get_resync_r1bio(bio); 1876 struct mddev *mddev = r1_bio->mddev; 1877 struct r1conf *conf = mddev->private; 1878 sector_t first_bad; 1879 int bad_sectors; 1880 struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev; 1881 1882 if (!uptodate) { 1883 sector_t sync_blocks = 0; 1884 sector_t s = r1_bio->sector; 1885 long sectors_to_go = r1_bio->sectors; 1886 /* make sure these bits doesn't get cleared. */ 1887 do { 1888 bitmap_end_sync(mddev->bitmap, s, 1889 &sync_blocks, 1); 1890 s += sync_blocks; 1891 sectors_to_go -= sync_blocks; 1892 } while (sectors_to_go > 0); 1893 set_bit(WriteErrorSeen, &rdev->flags); 1894 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 1895 set_bit(MD_RECOVERY_NEEDED, & 1896 mddev->recovery); 1897 set_bit(R1BIO_WriteError, &r1_bio->state); 1898 } else if (is_badblock(rdev, r1_bio->sector, r1_bio->sectors, 1899 &first_bad, &bad_sectors) && 1900 !is_badblock(conf->mirrors[r1_bio->read_disk].rdev, 1901 r1_bio->sector, 1902 r1_bio->sectors, 1903 &first_bad, &bad_sectors) 1904 ) 1905 set_bit(R1BIO_MadeGood, &r1_bio->state); 1906 1907 if (atomic_dec_and_test(&r1_bio->remaining)) { 1908 int s = r1_bio->sectors; 1909 if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 1910 test_bit(R1BIO_WriteError, &r1_bio->state)) 1911 reschedule_retry(r1_bio); 1912 else { 1913 put_buf(r1_bio); 1914 md_done_sync(mddev, s, uptodate); 1915 } 1916 } 1917 } 1918 1919 static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector, 1920 int sectors, struct page *page, int rw) 1921 { 1922 if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) 1923 /* success */ 1924 return 1; 1925 if (rw == WRITE) { 1926 set_bit(WriteErrorSeen, &rdev->flags); 1927 if (!test_and_set_bit(WantReplacement, 1928 &rdev->flags)) 1929 set_bit(MD_RECOVERY_NEEDED, & 1930 rdev->mddev->recovery); 1931 } 1932 /* need to record an error - either for the block or the device */ 1933 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 1934 md_error(rdev->mddev, rdev); 1935 return 0; 1936 } 1937 1938 static int fix_sync_read_error(struct r1bio *r1_bio) 1939 { 1940 /* Try some synchronous reads of other devices to get 1941 * good data, much like with normal read errors. Only 1942 * read into the pages we already have so we don't 1943 * need to re-issue the read request. 1944 * We don't need to freeze the array, because being in an 1945 * active sync request, there is no normal IO, and 1946 * no overlapping syncs. 1947 * We don't need to check is_badblock() again as we 1948 * made sure that anything with a bad block in range 1949 * will have bi_end_io clear. 1950 */ 1951 struct mddev *mddev = r1_bio->mddev; 1952 struct r1conf *conf = mddev->private; 1953 struct bio *bio = r1_bio->bios[r1_bio->read_disk]; 1954 struct page **pages = get_resync_pages(bio)->pages; 1955 sector_t sect = r1_bio->sector; 1956 int sectors = r1_bio->sectors; 1957 int idx = 0; 1958 struct md_rdev *rdev; 1959 1960 rdev = conf->mirrors[r1_bio->read_disk].rdev; 1961 if (test_bit(FailFast, &rdev->flags)) { 1962 /* Don't try recovering from here - just fail it 1963 * ... unless it is the last working device of course */ 1964 md_error(mddev, rdev); 1965 if (test_bit(Faulty, &rdev->flags)) 1966 /* Don't try to read from here, but make sure 1967 * put_buf does it's thing 1968 */ 1969 bio->bi_end_io = end_sync_write; 1970 } 1971 1972 while(sectors) { 1973 int s = sectors; 1974 int d = r1_bio->read_disk; 1975 int success = 0; 1976 int start; 1977 1978 if (s > (PAGE_SIZE>>9)) 1979 s = PAGE_SIZE >> 9; 1980 do { 1981 if (r1_bio->bios[d]->bi_end_io == end_sync_read) { 1982 /* No rcu protection needed here devices 1983 * can only be removed when no resync is 1984 * active, and resync is currently active 1985 */ 1986 rdev = conf->mirrors[d].rdev; 1987 if (sync_page_io(rdev, sect, s<<9, 1988 pages[idx], 1989 REQ_OP_READ, 0, false)) { 1990 success = 1; 1991 break; 1992 } 1993 } 1994 d++; 1995 if (d == conf->raid_disks * 2) 1996 d = 0; 1997 } while (!success && d != r1_bio->read_disk); 1998 1999 if (!success) { 2000 char b[BDEVNAME_SIZE]; 2001 int abort = 0; 2002 /* Cannot read from anywhere, this block is lost. 2003 * Record a bad block on each device. If that doesn't 2004 * work just disable and interrupt the recovery. 2005 * Don't fail devices as that won't really help. 2006 */ 2007 pr_crit_ratelimited("md/raid1:%s: %s: unrecoverable I/O read error for block %llu\n", 2008 mdname(mddev), 2009 bdevname(bio->bi_bdev, b), 2010 (unsigned long long)r1_bio->sector); 2011 for (d = 0; d < conf->raid_disks * 2; d++) { 2012 rdev = conf->mirrors[d].rdev; 2013 if (!rdev || test_bit(Faulty, &rdev->flags)) 2014 continue; 2015 if (!rdev_set_badblocks(rdev, sect, s, 0)) 2016 abort = 1; 2017 } 2018 if (abort) { 2019 conf->recovery_disabled = 2020 mddev->recovery_disabled; 2021 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2022 md_done_sync(mddev, r1_bio->sectors, 0); 2023 put_buf(r1_bio); 2024 return 0; 2025 } 2026 /* Try next page */ 2027 sectors -= s; 2028 sect += s; 2029 idx++; 2030 continue; 2031 } 2032 2033 start = d; 2034 /* write it back and re-read */ 2035 while (d != r1_bio->read_disk) { 2036 if (d == 0) 2037 d = conf->raid_disks * 2; 2038 d--; 2039 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 2040 continue; 2041 rdev = conf->mirrors[d].rdev; 2042 if (r1_sync_page_io(rdev, sect, s, 2043 pages[idx], 2044 WRITE) == 0) { 2045 r1_bio->bios[d]->bi_end_io = NULL; 2046 rdev_dec_pending(rdev, mddev); 2047 } 2048 } 2049 d = start; 2050 while (d != r1_bio->read_disk) { 2051 if (d == 0) 2052 d = conf->raid_disks * 2; 2053 d--; 2054 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 2055 continue; 2056 rdev = conf->mirrors[d].rdev; 2057 if (r1_sync_page_io(rdev, sect, s, 2058 pages[idx], 2059 READ) != 0) 2060 atomic_add(s, &rdev->corrected_errors); 2061 } 2062 sectors -= s; 2063 sect += s; 2064 idx ++; 2065 } 2066 set_bit(R1BIO_Uptodate, &r1_bio->state); 2067 bio->bi_status = 0; 2068 return 1; 2069 } 2070 2071 static void process_checks(struct r1bio *r1_bio) 2072 { 2073 /* We have read all readable devices. If we haven't 2074 * got the block, then there is no hope left. 2075 * If we have, then we want to do a comparison 2076 * and skip the write if everything is the same. 2077 * If any blocks failed to read, then we need to 2078 * attempt an over-write 2079 */ 2080 struct mddev *mddev = r1_bio->mddev; 2081 struct r1conf *conf = mddev->private; 2082 int primary; 2083 int i; 2084 int vcnt; 2085 2086 /* Fix variable parts of all bios */ 2087 vcnt = (r1_bio->sectors + PAGE_SIZE / 512 - 1) >> (PAGE_SHIFT - 9); 2088 for (i = 0; i < conf->raid_disks * 2; i++) { 2089 int j; 2090 int size; 2091 blk_status_t status; 2092 struct bio_vec *bi; 2093 struct bio *b = r1_bio->bios[i]; 2094 struct resync_pages *rp = get_resync_pages(b); 2095 if (b->bi_end_io != end_sync_read) 2096 continue; 2097 /* fixup the bio for reuse, but preserve errno */ 2098 status = b->bi_status; 2099 bio_reset(b); 2100 b->bi_status = status; 2101 b->bi_vcnt = vcnt; 2102 b->bi_iter.bi_size = r1_bio->sectors << 9; 2103 b->bi_iter.bi_sector = r1_bio->sector + 2104 conf->mirrors[i].rdev->data_offset; 2105 b->bi_bdev = conf->mirrors[i].rdev->bdev; 2106 b->bi_end_io = end_sync_read; 2107 rp->raid_bio = r1_bio; 2108 b->bi_private = rp; 2109 2110 size = b->bi_iter.bi_size; 2111 bio_for_each_segment_all(bi, b, j) { 2112 bi->bv_offset = 0; 2113 if (size > PAGE_SIZE) 2114 bi->bv_len = PAGE_SIZE; 2115 else 2116 bi->bv_len = size; 2117 size -= PAGE_SIZE; 2118 } 2119 } 2120 for (primary = 0; primary < conf->raid_disks * 2; primary++) 2121 if (r1_bio->bios[primary]->bi_end_io == end_sync_read && 2122 !r1_bio->bios[primary]->bi_status) { 2123 r1_bio->bios[primary]->bi_end_io = NULL; 2124 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); 2125 break; 2126 } 2127 r1_bio->read_disk = primary; 2128 for (i = 0; i < conf->raid_disks * 2; i++) { 2129 int j; 2130 struct bio *pbio = r1_bio->bios[primary]; 2131 struct bio *sbio = r1_bio->bios[i]; 2132 blk_status_t status = sbio->bi_status; 2133 struct page **ppages = get_resync_pages(pbio)->pages; 2134 struct page **spages = get_resync_pages(sbio)->pages; 2135 struct bio_vec *bi; 2136 int page_len[RESYNC_PAGES] = { 0 }; 2137 2138 if (sbio->bi_end_io != end_sync_read) 2139 continue; 2140 /* Now we can 'fixup' the error value */ 2141 sbio->bi_status = 0; 2142 2143 bio_for_each_segment_all(bi, sbio, j) 2144 page_len[j] = bi->bv_len; 2145 2146 if (!status) { 2147 for (j = vcnt; j-- ; ) { 2148 if (memcmp(page_address(ppages[j]), 2149 page_address(spages[j]), 2150 page_len[j])) 2151 break; 2152 } 2153 } else 2154 j = 0; 2155 if (j >= 0) 2156 atomic64_add(r1_bio->sectors, &mddev->resync_mismatches); 2157 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) 2158 && !status)) { 2159 /* No need to write to this device. */ 2160 sbio->bi_end_io = NULL; 2161 rdev_dec_pending(conf->mirrors[i].rdev, mddev); 2162 continue; 2163 } 2164 2165 bio_copy_data(sbio, pbio); 2166 } 2167 } 2168 2169 static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio) 2170 { 2171 struct r1conf *conf = mddev->private; 2172 int i; 2173 int disks = conf->raid_disks * 2; 2174 struct bio *wbio; 2175 2176 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) 2177 /* ouch - failed to read all of that. */ 2178 if (!fix_sync_read_error(r1_bio)) 2179 return; 2180 2181 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 2182 process_checks(r1_bio); 2183 2184 /* 2185 * schedule writes 2186 */ 2187 atomic_set(&r1_bio->remaining, 1); 2188 for (i = 0; i < disks ; i++) { 2189 wbio = r1_bio->bios[i]; 2190 if (wbio->bi_end_io == NULL || 2191 (wbio->bi_end_io == end_sync_read && 2192 (i == r1_bio->read_disk || 2193 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)))) 2194 continue; 2195 if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) 2196 continue; 2197 2198 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); 2199 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags)) 2200 wbio->bi_opf |= MD_FAILFAST; 2201 2202 wbio->bi_end_io = end_sync_write; 2203 atomic_inc(&r1_bio->remaining); 2204 md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio)); 2205 2206 generic_make_request(wbio); 2207 } 2208 2209 if (atomic_dec_and_test(&r1_bio->remaining)) { 2210 /* if we're here, all write(s) have completed, so clean up */ 2211 int s = r1_bio->sectors; 2212 if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2213 test_bit(R1BIO_WriteError, &r1_bio->state)) 2214 reschedule_retry(r1_bio); 2215 else { 2216 put_buf(r1_bio); 2217 md_done_sync(mddev, s, 1); 2218 } 2219 } 2220 } 2221 2222 /* 2223 * This is a kernel thread which: 2224 * 2225 * 1. Retries failed read operations on working mirrors. 2226 * 2. Updates the raid superblock when problems encounter. 2227 * 3. Performs writes following reads for array synchronising. 2228 */ 2229 2230 static void fix_read_error(struct r1conf *conf, int read_disk, 2231 sector_t sect, int sectors) 2232 { 2233 struct mddev *mddev = conf->mddev; 2234 while(sectors) { 2235 int s = sectors; 2236 int d = read_disk; 2237 int success = 0; 2238 int start; 2239 struct md_rdev *rdev; 2240 2241 if (s > (PAGE_SIZE>>9)) 2242 s = PAGE_SIZE >> 9; 2243 2244 do { 2245 sector_t first_bad; 2246 int bad_sectors; 2247 2248 rcu_read_lock(); 2249 rdev = rcu_dereference(conf->mirrors[d].rdev); 2250 if (rdev && 2251 (test_bit(In_sync, &rdev->flags) || 2252 (!test_bit(Faulty, &rdev->flags) && 2253 rdev->recovery_offset >= sect + s)) && 2254 is_badblock(rdev, sect, s, 2255 &first_bad, &bad_sectors) == 0) { 2256 atomic_inc(&rdev->nr_pending); 2257 rcu_read_unlock(); 2258 if (sync_page_io(rdev, sect, s<<9, 2259 conf->tmppage, REQ_OP_READ, 0, false)) 2260 success = 1; 2261 rdev_dec_pending(rdev, mddev); 2262 if (success) 2263 break; 2264 } else 2265 rcu_read_unlock(); 2266 d++; 2267 if (d == conf->raid_disks * 2) 2268 d = 0; 2269 } while (!success && d != read_disk); 2270 2271 if (!success) { 2272 /* Cannot read from anywhere - mark it bad */ 2273 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; 2274 if (!rdev_set_badblocks(rdev, sect, s, 0)) 2275 md_error(mddev, rdev); 2276 break; 2277 } 2278 /* write it back and re-read */ 2279 start = d; 2280 while (d != read_disk) { 2281 if (d==0) 2282 d = conf->raid_disks * 2; 2283 d--; 2284 rcu_read_lock(); 2285 rdev = rcu_dereference(conf->mirrors[d].rdev); 2286 if (rdev && 2287 !test_bit(Faulty, &rdev->flags)) { 2288 atomic_inc(&rdev->nr_pending); 2289 rcu_read_unlock(); 2290 r1_sync_page_io(rdev, sect, s, 2291 conf->tmppage, WRITE); 2292 rdev_dec_pending(rdev, mddev); 2293 } else 2294 rcu_read_unlock(); 2295 } 2296 d = start; 2297 while (d != read_disk) { 2298 char b[BDEVNAME_SIZE]; 2299 if (d==0) 2300 d = conf->raid_disks * 2; 2301 d--; 2302 rcu_read_lock(); 2303 rdev = rcu_dereference(conf->mirrors[d].rdev); 2304 if (rdev && 2305 !test_bit(Faulty, &rdev->flags)) { 2306 atomic_inc(&rdev->nr_pending); 2307 rcu_read_unlock(); 2308 if (r1_sync_page_io(rdev, sect, s, 2309 conf->tmppage, READ)) { 2310 atomic_add(s, &rdev->corrected_errors); 2311 pr_info("md/raid1:%s: read error corrected (%d sectors at %llu on %s)\n", 2312 mdname(mddev), s, 2313 (unsigned long long)(sect + 2314 rdev->data_offset), 2315 bdevname(rdev->bdev, b)); 2316 } 2317 rdev_dec_pending(rdev, mddev); 2318 } else 2319 rcu_read_unlock(); 2320 } 2321 sectors -= s; 2322 sect += s; 2323 } 2324 } 2325 2326 static int narrow_write_error(struct r1bio *r1_bio, int i) 2327 { 2328 struct mddev *mddev = r1_bio->mddev; 2329 struct r1conf *conf = mddev->private; 2330 struct md_rdev *rdev = conf->mirrors[i].rdev; 2331 2332 /* bio has the data to be written to device 'i' where 2333 * we just recently had a write error. 2334 * We repeatedly clone the bio and trim down to one block, 2335 * then try the write. Where the write fails we record 2336 * a bad block. 2337 * It is conceivable that the bio doesn't exactly align with 2338 * blocks. We must handle this somehow. 2339 * 2340 * We currently own a reference on the rdev. 2341 */ 2342 2343 int block_sectors; 2344 sector_t sector; 2345 int sectors; 2346 int sect_to_write = r1_bio->sectors; 2347 int ok = 1; 2348 2349 if (rdev->badblocks.shift < 0) 2350 return 0; 2351 2352 block_sectors = roundup(1 << rdev->badblocks.shift, 2353 bdev_logical_block_size(rdev->bdev) >> 9); 2354 sector = r1_bio->sector; 2355 sectors = ((sector + block_sectors) 2356 & ~(sector_t)(block_sectors - 1)) 2357 - sector; 2358 2359 while (sect_to_write) { 2360 struct bio *wbio; 2361 if (sectors > sect_to_write) 2362 sectors = sect_to_write; 2363 /* Write at 'sector' for 'sectors'*/ 2364 2365 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { 2366 wbio = bio_clone_fast(r1_bio->behind_master_bio, 2367 GFP_NOIO, 2368 mddev->bio_set); 2369 /* We really need a _all clone */ 2370 wbio->bi_iter = (struct bvec_iter){ 0 }; 2371 } else { 2372 wbio = bio_clone_fast(r1_bio->master_bio, GFP_NOIO, 2373 mddev->bio_set); 2374 } 2375 2376 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); 2377 wbio->bi_iter.bi_sector = r1_bio->sector; 2378 wbio->bi_iter.bi_size = r1_bio->sectors << 9; 2379 2380 bio_trim(wbio, sector - r1_bio->sector, sectors); 2381 wbio->bi_iter.bi_sector += rdev->data_offset; 2382 wbio->bi_bdev = rdev->bdev; 2383 2384 if (submit_bio_wait(wbio) < 0) 2385 /* failure! */ 2386 ok = rdev_set_badblocks(rdev, sector, 2387 sectors, 0) 2388 && ok; 2389 2390 bio_put(wbio); 2391 sect_to_write -= sectors; 2392 sector += sectors; 2393 sectors = block_sectors; 2394 } 2395 return ok; 2396 } 2397 2398 static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio) 2399 { 2400 int m; 2401 int s = r1_bio->sectors; 2402 for (m = 0; m < conf->raid_disks * 2 ; m++) { 2403 struct md_rdev *rdev = conf->mirrors[m].rdev; 2404 struct bio *bio = r1_bio->bios[m]; 2405 if (bio->bi_end_io == NULL) 2406 continue; 2407 if (!bio->bi_status && 2408 test_bit(R1BIO_MadeGood, &r1_bio->state)) { 2409 rdev_clear_badblocks(rdev, r1_bio->sector, s, 0); 2410 } 2411 if (bio->bi_status && 2412 test_bit(R1BIO_WriteError, &r1_bio->state)) { 2413 if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0)) 2414 md_error(conf->mddev, rdev); 2415 } 2416 } 2417 put_buf(r1_bio); 2418 md_done_sync(conf->mddev, s, 1); 2419 } 2420 2421 static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio) 2422 { 2423 int m, idx; 2424 bool fail = false; 2425 2426 for (m = 0; m < conf->raid_disks * 2 ; m++) 2427 if (r1_bio->bios[m] == IO_MADE_GOOD) { 2428 struct md_rdev *rdev = conf->mirrors[m].rdev; 2429 rdev_clear_badblocks(rdev, 2430 r1_bio->sector, 2431 r1_bio->sectors, 0); 2432 rdev_dec_pending(rdev, conf->mddev); 2433 } else if (r1_bio->bios[m] != NULL) { 2434 /* This drive got a write error. We need to 2435 * narrow down and record precise write 2436 * errors. 2437 */ 2438 fail = true; 2439 if (!narrow_write_error(r1_bio, m)) { 2440 md_error(conf->mddev, 2441 conf->mirrors[m].rdev); 2442 /* an I/O failed, we can't clear the bitmap */ 2443 set_bit(R1BIO_Degraded, &r1_bio->state); 2444 } 2445 rdev_dec_pending(conf->mirrors[m].rdev, 2446 conf->mddev); 2447 } 2448 if (fail) { 2449 spin_lock_irq(&conf->device_lock); 2450 list_add(&r1_bio->retry_list, &conf->bio_end_io_list); 2451 idx = sector_to_idx(r1_bio->sector); 2452 atomic_inc(&conf->nr_queued[idx]); 2453 spin_unlock_irq(&conf->device_lock); 2454 /* 2455 * In case freeze_array() is waiting for condition 2456 * get_unqueued_pending() == extra to be true. 2457 */ 2458 wake_up(&conf->wait_barrier); 2459 md_wakeup_thread(conf->mddev->thread); 2460 } else { 2461 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 2462 close_write(r1_bio); 2463 raid_end_bio_io(r1_bio); 2464 } 2465 } 2466 2467 static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) 2468 { 2469 struct mddev *mddev = conf->mddev; 2470 struct bio *bio; 2471 struct md_rdev *rdev; 2472 dev_t bio_dev; 2473 sector_t bio_sector; 2474 2475 clear_bit(R1BIO_ReadError, &r1_bio->state); 2476 /* we got a read error. Maybe the drive is bad. Maybe just 2477 * the block and we can fix it. 2478 * We freeze all other IO, and try reading the block from 2479 * other devices. When we find one, we re-write 2480 * and check it that fixes the read error. 2481 * This is all done synchronously while the array is 2482 * frozen 2483 */ 2484 2485 bio = r1_bio->bios[r1_bio->read_disk]; 2486 bio_dev = bio->bi_bdev->bd_dev; 2487 bio_sector = conf->mirrors[r1_bio->read_disk].rdev->data_offset + r1_bio->sector; 2488 bio_put(bio); 2489 r1_bio->bios[r1_bio->read_disk] = NULL; 2490 2491 rdev = conf->mirrors[r1_bio->read_disk].rdev; 2492 if (mddev->ro == 0 2493 && !test_bit(FailFast, &rdev->flags)) { 2494 freeze_array(conf, 1); 2495 fix_read_error(conf, r1_bio->read_disk, 2496 r1_bio->sector, r1_bio->sectors); 2497 unfreeze_array(conf); 2498 } else { 2499 r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; 2500 } 2501 2502 rdev_dec_pending(rdev, conf->mddev); 2503 allow_barrier(conf, r1_bio->sector); 2504 bio = r1_bio->master_bio; 2505 2506 /* Reuse the old r1_bio so that the IO_BLOCKED settings are preserved */ 2507 r1_bio->state = 0; 2508 raid1_read_request(mddev, bio, r1_bio->sectors, r1_bio); 2509 } 2510 2511 static void raid1d(struct md_thread *thread) 2512 { 2513 struct mddev *mddev = thread->mddev; 2514 struct r1bio *r1_bio; 2515 unsigned long flags; 2516 struct r1conf *conf = mddev->private; 2517 struct list_head *head = &conf->retry_list; 2518 struct blk_plug plug; 2519 int idx; 2520 2521 md_check_recovery(mddev); 2522 2523 if (!list_empty_careful(&conf->bio_end_io_list) && 2524 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2525 LIST_HEAD(tmp); 2526 spin_lock_irqsave(&conf->device_lock, flags); 2527 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) 2528 list_splice_init(&conf->bio_end_io_list, &tmp); 2529 spin_unlock_irqrestore(&conf->device_lock, flags); 2530 while (!list_empty(&tmp)) { 2531 r1_bio = list_first_entry(&tmp, struct r1bio, 2532 retry_list); 2533 list_del(&r1_bio->retry_list); 2534 idx = sector_to_idx(r1_bio->sector); 2535 atomic_dec(&conf->nr_queued[idx]); 2536 if (mddev->degraded) 2537 set_bit(R1BIO_Degraded, &r1_bio->state); 2538 if (test_bit(R1BIO_WriteError, &r1_bio->state)) 2539 close_write(r1_bio); 2540 raid_end_bio_io(r1_bio); 2541 } 2542 } 2543 2544 blk_start_plug(&plug); 2545 for (;;) { 2546 2547 flush_pending_writes(conf); 2548 2549 spin_lock_irqsave(&conf->device_lock, flags); 2550 if (list_empty(head)) { 2551 spin_unlock_irqrestore(&conf->device_lock, flags); 2552 break; 2553 } 2554 r1_bio = list_entry(head->prev, struct r1bio, retry_list); 2555 list_del(head->prev); 2556 idx = sector_to_idx(r1_bio->sector); 2557 atomic_dec(&conf->nr_queued[idx]); 2558 spin_unlock_irqrestore(&conf->device_lock, flags); 2559 2560 mddev = r1_bio->mddev; 2561 conf = mddev->private; 2562 if (test_bit(R1BIO_IsSync, &r1_bio->state)) { 2563 if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2564 test_bit(R1BIO_WriteError, &r1_bio->state)) 2565 handle_sync_write_finished(conf, r1_bio); 2566 else 2567 sync_request_write(mddev, r1_bio); 2568 } else if (test_bit(R1BIO_MadeGood, &r1_bio->state) || 2569 test_bit(R1BIO_WriteError, &r1_bio->state)) 2570 handle_write_finished(conf, r1_bio); 2571 else if (test_bit(R1BIO_ReadError, &r1_bio->state)) 2572 handle_read_error(conf, r1_bio); 2573 else 2574 WARN_ON_ONCE(1); 2575 2576 cond_resched(); 2577 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING)) 2578 md_check_recovery(mddev); 2579 } 2580 blk_finish_plug(&plug); 2581 } 2582 2583 static int init_resync(struct r1conf *conf) 2584 { 2585 int buffs; 2586 2587 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2588 BUG_ON(conf->r1buf_pool); 2589 conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free, 2590 conf->poolinfo); 2591 if (!conf->r1buf_pool) 2592 return -ENOMEM; 2593 return 0; 2594 } 2595 2596 /* 2597 * perform a "sync" on one "block" 2598 * 2599 * We need to make sure that no normal I/O request - particularly write 2600 * requests - conflict with active sync requests. 2601 * 2602 * This is achieved by tracking pending requests and a 'barrier' concept 2603 * that can be installed to exclude normal IO requests. 2604 */ 2605 2606 static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, 2607 int *skipped) 2608 { 2609 struct r1conf *conf = mddev->private; 2610 struct r1bio *r1_bio; 2611 struct bio *bio; 2612 sector_t max_sector, nr_sectors; 2613 int disk = -1; 2614 int i; 2615 int wonly = -1; 2616 int write_targets = 0, read_targets = 0; 2617 sector_t sync_blocks; 2618 int still_degraded = 0; 2619 int good_sectors = RESYNC_SECTORS; 2620 int min_bad = 0; /* number of sectors that are bad in all devices */ 2621 int idx = sector_to_idx(sector_nr); 2622 2623 if (!conf->r1buf_pool) 2624 if (init_resync(conf)) 2625 return 0; 2626 2627 max_sector = mddev->dev_sectors; 2628 if (sector_nr >= max_sector) { 2629 /* If we aborted, we need to abort the 2630 * sync on the 'current' bitmap chunk (there will 2631 * only be one in raid1 resync. 2632 * We can find the current addess in mddev->curr_resync 2633 */ 2634 if (mddev->curr_resync < max_sector) /* aborted */ 2635 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2636 &sync_blocks, 1); 2637 else /* completed sync */ 2638 conf->fullsync = 0; 2639 2640 bitmap_close_sync(mddev->bitmap); 2641 close_sync(conf); 2642 2643 if (mddev_is_clustered(mddev)) { 2644 conf->cluster_sync_low = 0; 2645 conf->cluster_sync_high = 0; 2646 } 2647 return 0; 2648 } 2649 2650 if (mddev->bitmap == NULL && 2651 mddev->recovery_cp == MaxSector && 2652 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2653 conf->fullsync == 0) { 2654 *skipped = 1; 2655 return max_sector - sector_nr; 2656 } 2657 /* before building a request, check if we can skip these blocks.. 2658 * This call the bitmap_start_sync doesn't actually record anything 2659 */ 2660 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 2661 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 2662 /* We can skip this block, and probably several more */ 2663 *skipped = 1; 2664 return sync_blocks; 2665 } 2666 2667 /* 2668 * If there is non-resync activity waiting for a turn, then let it 2669 * though before starting on this new sync request. 2670 */ 2671 if (atomic_read(&conf->nr_waiting[idx])) 2672 schedule_timeout_uninterruptible(1); 2673 2674 /* we are incrementing sector_nr below. To be safe, we check against 2675 * sector_nr + two times RESYNC_SECTORS 2676 */ 2677 2678 bitmap_cond_end_sync(mddev->bitmap, sector_nr, 2679 mddev_is_clustered(mddev) && (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); 2680 r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO); 2681 2682 raise_barrier(conf, sector_nr); 2683 2684 rcu_read_lock(); 2685 /* 2686 * If we get a correctably read error during resync or recovery, 2687 * we might want to read from a different device. So we 2688 * flag all drives that could conceivably be read from for READ, 2689 * and any others (which will be non-In_sync devices) for WRITE. 2690 * If a read fails, we try reading from something else for which READ 2691 * is OK. 2692 */ 2693 2694 r1_bio->mddev = mddev; 2695 r1_bio->sector = sector_nr; 2696 r1_bio->state = 0; 2697 set_bit(R1BIO_IsSync, &r1_bio->state); 2698 /* make sure good_sectors won't go across barrier unit boundary */ 2699 good_sectors = align_to_barrier_unit_end(sector_nr, good_sectors); 2700 2701 for (i = 0; i < conf->raid_disks * 2; i++) { 2702 struct md_rdev *rdev; 2703 bio = r1_bio->bios[i]; 2704 2705 rdev = rcu_dereference(conf->mirrors[i].rdev); 2706 if (rdev == NULL || 2707 test_bit(Faulty, &rdev->flags)) { 2708 if (i < conf->raid_disks) 2709 still_degraded = 1; 2710 } else if (!test_bit(In_sync, &rdev->flags)) { 2711 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 2712 bio->bi_end_io = end_sync_write; 2713 write_targets ++; 2714 } else { 2715 /* may need to read from here */ 2716 sector_t first_bad = MaxSector; 2717 int bad_sectors; 2718 2719 if (is_badblock(rdev, sector_nr, good_sectors, 2720 &first_bad, &bad_sectors)) { 2721 if (first_bad > sector_nr) 2722 good_sectors = first_bad - sector_nr; 2723 else { 2724 bad_sectors -= (sector_nr - first_bad); 2725 if (min_bad == 0 || 2726 min_bad > bad_sectors) 2727 min_bad = bad_sectors; 2728 } 2729 } 2730 if (sector_nr < first_bad) { 2731 if (test_bit(WriteMostly, &rdev->flags)) { 2732 if (wonly < 0) 2733 wonly = i; 2734 } else { 2735 if (disk < 0) 2736 disk = i; 2737 } 2738 bio_set_op_attrs(bio, REQ_OP_READ, 0); 2739 bio->bi_end_io = end_sync_read; 2740 read_targets++; 2741 } else if (!test_bit(WriteErrorSeen, &rdev->flags) && 2742 test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2743 !test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) { 2744 /* 2745 * The device is suitable for reading (InSync), 2746 * but has bad block(s) here. Let's try to correct them, 2747 * if we are doing resync or repair. Otherwise, leave 2748 * this device alone for this sync request. 2749 */ 2750 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 2751 bio->bi_end_io = end_sync_write; 2752 write_targets++; 2753 } 2754 } 2755 if (bio->bi_end_io) { 2756 atomic_inc(&rdev->nr_pending); 2757 bio->bi_iter.bi_sector = sector_nr + rdev->data_offset; 2758 bio->bi_bdev = rdev->bdev; 2759 if (test_bit(FailFast, &rdev->flags)) 2760 bio->bi_opf |= MD_FAILFAST; 2761 } 2762 } 2763 rcu_read_unlock(); 2764 if (disk < 0) 2765 disk = wonly; 2766 r1_bio->read_disk = disk; 2767 2768 if (read_targets == 0 && min_bad > 0) { 2769 /* These sectors are bad on all InSync devices, so we 2770 * need to mark them bad on all write targets 2771 */ 2772 int ok = 1; 2773 for (i = 0 ; i < conf->raid_disks * 2 ; i++) 2774 if (r1_bio->bios[i]->bi_end_io == end_sync_write) { 2775 struct md_rdev *rdev = conf->mirrors[i].rdev; 2776 ok = rdev_set_badblocks(rdev, sector_nr, 2777 min_bad, 0 2778 ) && ok; 2779 } 2780 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 2781 *skipped = 1; 2782 put_buf(r1_bio); 2783 2784 if (!ok) { 2785 /* Cannot record the badblocks, so need to 2786 * abort the resync. 2787 * If there are multiple read targets, could just 2788 * fail the really bad ones ??? 2789 */ 2790 conf->recovery_disabled = mddev->recovery_disabled; 2791 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2792 return 0; 2793 } else 2794 return min_bad; 2795 2796 } 2797 if (min_bad > 0 && min_bad < good_sectors) { 2798 /* only resync enough to reach the next bad->good 2799 * transition */ 2800 good_sectors = min_bad; 2801 } 2802 2803 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0) 2804 /* extra read targets are also write targets */ 2805 write_targets += read_targets-1; 2806 2807 if (write_targets == 0 || read_targets == 0) { 2808 /* There is nowhere to write, so all non-sync 2809 * drives must be failed - so we are finished 2810 */ 2811 sector_t rv; 2812 if (min_bad > 0) 2813 max_sector = sector_nr + min_bad; 2814 rv = max_sector - sector_nr; 2815 *skipped = 1; 2816 put_buf(r1_bio); 2817 return rv; 2818 } 2819 2820 if (max_sector > mddev->resync_max) 2821 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2822 if (max_sector > sector_nr + good_sectors) 2823 max_sector = sector_nr + good_sectors; 2824 nr_sectors = 0; 2825 sync_blocks = 0; 2826 do { 2827 struct page *page; 2828 int len = PAGE_SIZE; 2829 if (sector_nr + (len>>9) > max_sector) 2830 len = (max_sector - sector_nr) << 9; 2831 if (len == 0) 2832 break; 2833 if (sync_blocks == 0) { 2834 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 2835 &sync_blocks, still_degraded) && 2836 !conf->fullsync && 2837 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 2838 break; 2839 if ((len >> 9) > sync_blocks) 2840 len = sync_blocks<<9; 2841 } 2842 2843 for (i = 0 ; i < conf->raid_disks * 2; i++) { 2844 struct resync_pages *rp; 2845 2846 bio = r1_bio->bios[i]; 2847 rp = get_resync_pages(bio); 2848 if (bio->bi_end_io) { 2849 page = resync_fetch_page(rp, rp->idx++); 2850 2851 /* 2852 * won't fail because the vec table is big 2853 * enough to hold all these pages 2854 */ 2855 bio_add_page(bio, page, len, 0); 2856 } 2857 } 2858 nr_sectors += len>>9; 2859 sector_nr += len>>9; 2860 sync_blocks -= (len>>9); 2861 } while (get_resync_pages(r1_bio->bios[disk]->bi_private)->idx < RESYNC_PAGES); 2862 2863 r1_bio->sectors = nr_sectors; 2864 2865 if (mddev_is_clustered(mddev) && 2866 conf->cluster_sync_high < sector_nr + nr_sectors) { 2867 conf->cluster_sync_low = mddev->curr_resync_completed; 2868 conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS; 2869 /* Send resync message */ 2870 md_cluster_ops->resync_info_update(mddev, 2871 conf->cluster_sync_low, 2872 conf->cluster_sync_high); 2873 } 2874 2875 /* For a user-requested sync, we read all readable devices and do a 2876 * compare 2877 */ 2878 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 2879 atomic_set(&r1_bio->remaining, read_targets); 2880 for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) { 2881 bio = r1_bio->bios[i]; 2882 if (bio->bi_end_io == end_sync_read) { 2883 read_targets--; 2884 md_sync_acct(bio->bi_bdev, nr_sectors); 2885 if (read_targets == 1) 2886 bio->bi_opf &= ~MD_FAILFAST; 2887 generic_make_request(bio); 2888 } 2889 } 2890 } else { 2891 atomic_set(&r1_bio->remaining, 1); 2892 bio = r1_bio->bios[r1_bio->read_disk]; 2893 md_sync_acct(bio->bi_bdev, nr_sectors); 2894 if (read_targets == 1) 2895 bio->bi_opf &= ~MD_FAILFAST; 2896 generic_make_request(bio); 2897 2898 } 2899 return nr_sectors; 2900 } 2901 2902 static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks) 2903 { 2904 if (sectors) 2905 return sectors; 2906 2907 return mddev->dev_sectors; 2908 } 2909 2910 static struct r1conf *setup_conf(struct mddev *mddev) 2911 { 2912 struct r1conf *conf; 2913 int i; 2914 struct raid1_info *disk; 2915 struct md_rdev *rdev; 2916 int err = -ENOMEM; 2917 2918 conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL); 2919 if (!conf) 2920 goto abort; 2921 2922 conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR, 2923 sizeof(atomic_t), GFP_KERNEL); 2924 if (!conf->nr_pending) 2925 goto abort; 2926 2927 conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR, 2928 sizeof(atomic_t), GFP_KERNEL); 2929 if (!conf->nr_waiting) 2930 goto abort; 2931 2932 conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR, 2933 sizeof(atomic_t), GFP_KERNEL); 2934 if (!conf->nr_queued) 2935 goto abort; 2936 2937 conf->barrier = kcalloc(BARRIER_BUCKETS_NR, 2938 sizeof(atomic_t), GFP_KERNEL); 2939 if (!conf->barrier) 2940 goto abort; 2941 2942 conf->mirrors = kzalloc(sizeof(struct raid1_info) 2943 * mddev->raid_disks * 2, 2944 GFP_KERNEL); 2945 if (!conf->mirrors) 2946 goto abort; 2947 2948 conf->tmppage = alloc_page(GFP_KERNEL); 2949 if (!conf->tmppage) 2950 goto abort; 2951 2952 conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL); 2953 if (!conf->poolinfo) 2954 goto abort; 2955 conf->poolinfo->raid_disks = mddev->raid_disks * 2; 2956 conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc, 2957 r1bio_pool_free, 2958 conf->poolinfo); 2959 if (!conf->r1bio_pool) 2960 goto abort; 2961 2962 conf->bio_split = bioset_create(BIO_POOL_SIZE, 0, 0); 2963 if (!conf->bio_split) 2964 goto abort; 2965 2966 conf->poolinfo->mddev = mddev; 2967 2968 err = -EINVAL; 2969 spin_lock_init(&conf->device_lock); 2970 rdev_for_each(rdev, mddev) { 2971 int disk_idx = rdev->raid_disk; 2972 if (disk_idx >= mddev->raid_disks 2973 || disk_idx < 0) 2974 continue; 2975 if (test_bit(Replacement, &rdev->flags)) 2976 disk = conf->mirrors + mddev->raid_disks + disk_idx; 2977 else 2978 disk = conf->mirrors + disk_idx; 2979 2980 if (disk->rdev) 2981 goto abort; 2982 disk->rdev = rdev; 2983 disk->head_position = 0; 2984 disk->seq_start = MaxSector; 2985 } 2986 conf->raid_disks = mddev->raid_disks; 2987 conf->mddev = mddev; 2988 INIT_LIST_HEAD(&conf->retry_list); 2989 INIT_LIST_HEAD(&conf->bio_end_io_list); 2990 2991 spin_lock_init(&conf->resync_lock); 2992 init_waitqueue_head(&conf->wait_barrier); 2993 2994 bio_list_init(&conf->pending_bio_list); 2995 conf->pending_count = 0; 2996 conf->recovery_disabled = mddev->recovery_disabled - 1; 2997 2998 err = -EIO; 2999 for (i = 0; i < conf->raid_disks * 2; i++) { 3000 3001 disk = conf->mirrors + i; 3002 3003 if (i < conf->raid_disks && 3004 disk[conf->raid_disks].rdev) { 3005 /* This slot has a replacement. */ 3006 if (!disk->rdev) { 3007 /* No original, just make the replacement 3008 * a recovering spare 3009 */ 3010 disk->rdev = 3011 disk[conf->raid_disks].rdev; 3012 disk[conf->raid_disks].rdev = NULL; 3013 } else if (!test_bit(In_sync, &disk->rdev->flags)) 3014 /* Original is not in_sync - bad */ 3015 goto abort; 3016 } 3017 3018 if (!disk->rdev || 3019 !test_bit(In_sync, &disk->rdev->flags)) { 3020 disk->head_position = 0; 3021 if (disk->rdev && 3022 (disk->rdev->saved_raid_disk < 0)) 3023 conf->fullsync = 1; 3024 } 3025 } 3026 3027 err = -ENOMEM; 3028 conf->thread = md_register_thread(raid1d, mddev, "raid1"); 3029 if (!conf->thread) 3030 goto abort; 3031 3032 return conf; 3033 3034 abort: 3035 if (conf) { 3036 mempool_destroy(conf->r1bio_pool); 3037 kfree(conf->mirrors); 3038 safe_put_page(conf->tmppage); 3039 kfree(conf->poolinfo); 3040 kfree(conf->nr_pending); 3041 kfree(conf->nr_waiting); 3042 kfree(conf->nr_queued); 3043 kfree(conf->barrier); 3044 if (conf->bio_split) 3045 bioset_free(conf->bio_split); 3046 kfree(conf); 3047 } 3048 return ERR_PTR(err); 3049 } 3050 3051 static void raid1_free(struct mddev *mddev, void *priv); 3052 static int raid1_run(struct mddev *mddev) 3053 { 3054 struct r1conf *conf; 3055 int i; 3056 struct md_rdev *rdev; 3057 int ret; 3058 bool discard_supported = false; 3059 3060 if (mddev->level != 1) { 3061 pr_warn("md/raid1:%s: raid level not set to mirroring (%d)\n", 3062 mdname(mddev), mddev->level); 3063 return -EIO; 3064 } 3065 if (mddev->reshape_position != MaxSector) { 3066 pr_warn("md/raid1:%s: reshape_position set but not supported\n", 3067 mdname(mddev)); 3068 return -EIO; 3069 } 3070 if (mddev_init_writes_pending(mddev) < 0) 3071 return -ENOMEM; 3072 /* 3073 * copy the already verified devices into our private RAID1 3074 * bookkeeping area. [whatever we allocate in run(), 3075 * should be freed in raid1_free()] 3076 */ 3077 if (mddev->private == NULL) 3078 conf = setup_conf(mddev); 3079 else 3080 conf = mddev->private; 3081 3082 if (IS_ERR(conf)) 3083 return PTR_ERR(conf); 3084 3085 if (mddev->queue) { 3086 blk_queue_max_write_same_sectors(mddev->queue, 0); 3087 blk_queue_max_write_zeroes_sectors(mddev->queue, 0); 3088 } 3089 3090 rdev_for_each(rdev, mddev) { 3091 if (!mddev->gendisk) 3092 continue; 3093 disk_stack_limits(mddev->gendisk, rdev->bdev, 3094 rdev->data_offset << 9); 3095 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3096 discard_supported = true; 3097 } 3098 3099 mddev->degraded = 0; 3100 for (i=0; i < conf->raid_disks; i++) 3101 if (conf->mirrors[i].rdev == NULL || 3102 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) || 3103 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) 3104 mddev->degraded++; 3105 3106 if (conf->raid_disks - mddev->degraded == 1) 3107 mddev->recovery_cp = MaxSector; 3108 3109 if (mddev->recovery_cp != MaxSector) 3110 pr_info("md/raid1:%s: not clean -- starting background reconstruction\n", 3111 mdname(mddev)); 3112 pr_info("md/raid1:%s: active with %d out of %d mirrors\n", 3113 mdname(mddev), mddev->raid_disks - mddev->degraded, 3114 mddev->raid_disks); 3115 3116 /* 3117 * Ok, everything is just fine now 3118 */ 3119 mddev->thread = conf->thread; 3120 conf->thread = NULL; 3121 mddev->private = conf; 3122 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 3123 3124 md_set_array_sectors(mddev, raid1_size(mddev, 0, 0)); 3125 3126 if (mddev->queue) { 3127 if (discard_supported) 3128 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 3129 mddev->queue); 3130 else 3131 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 3132 mddev->queue); 3133 } 3134 3135 ret = md_integrity_register(mddev); 3136 if (ret) { 3137 md_unregister_thread(&mddev->thread); 3138 raid1_free(mddev, conf); 3139 } 3140 return ret; 3141 } 3142 3143 static void raid1_free(struct mddev *mddev, void *priv) 3144 { 3145 struct r1conf *conf = priv; 3146 3147 mempool_destroy(conf->r1bio_pool); 3148 kfree(conf->mirrors); 3149 safe_put_page(conf->tmppage); 3150 kfree(conf->poolinfo); 3151 kfree(conf->nr_pending); 3152 kfree(conf->nr_waiting); 3153 kfree(conf->nr_queued); 3154 kfree(conf->barrier); 3155 if (conf->bio_split) 3156 bioset_free(conf->bio_split); 3157 kfree(conf); 3158 } 3159 3160 static int raid1_resize(struct mddev *mddev, sector_t sectors) 3161 { 3162 /* no resync is happening, and there is enough space 3163 * on all devices, so we can resize. 3164 * We need to make sure resync covers any new space. 3165 * If the array is shrinking we should possibly wait until 3166 * any io in the removed space completes, but it hardly seems 3167 * worth it. 3168 */ 3169 sector_t newsize = raid1_size(mddev, sectors, 0); 3170 if (mddev->external_size && 3171 mddev->array_sectors > newsize) 3172 return -EINVAL; 3173 if (mddev->bitmap) { 3174 int ret = bitmap_resize(mddev->bitmap, newsize, 0, 0); 3175 if (ret) 3176 return ret; 3177 } 3178 md_set_array_sectors(mddev, newsize); 3179 if (sectors > mddev->dev_sectors && 3180 mddev->recovery_cp > mddev->dev_sectors) { 3181 mddev->recovery_cp = mddev->dev_sectors; 3182 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3183 } 3184 mddev->dev_sectors = sectors; 3185 mddev->resync_max_sectors = sectors; 3186 return 0; 3187 } 3188 3189 static int raid1_reshape(struct mddev *mddev) 3190 { 3191 /* We need to: 3192 * 1/ resize the r1bio_pool 3193 * 2/ resize conf->mirrors 3194 * 3195 * We allocate a new r1bio_pool if we can. 3196 * Then raise a device barrier and wait until all IO stops. 3197 * Then resize conf->mirrors and swap in the new r1bio pool. 3198 * 3199 * At the same time, we "pack" the devices so that all the missing 3200 * devices have the higher raid_disk numbers. 3201 */ 3202 mempool_t *newpool, *oldpool; 3203 struct pool_info *newpoolinfo; 3204 struct raid1_info *newmirrors; 3205 struct r1conf *conf = mddev->private; 3206 int cnt, raid_disks; 3207 unsigned long flags; 3208 int d, d2; 3209 3210 /* Cannot change chunk_size, layout, or level */ 3211 if (mddev->chunk_sectors != mddev->new_chunk_sectors || 3212 mddev->layout != mddev->new_layout || 3213 mddev->level != mddev->new_level) { 3214 mddev->new_chunk_sectors = mddev->chunk_sectors; 3215 mddev->new_layout = mddev->layout; 3216 mddev->new_level = mddev->level; 3217 return -EINVAL; 3218 } 3219 3220 if (!mddev_is_clustered(mddev)) 3221 md_allow_write(mddev); 3222 3223 raid_disks = mddev->raid_disks + mddev->delta_disks; 3224 3225 if (raid_disks < conf->raid_disks) { 3226 cnt=0; 3227 for (d= 0; d < conf->raid_disks; d++) 3228 if (conf->mirrors[d].rdev) 3229 cnt++; 3230 if (cnt > raid_disks) 3231 return -EBUSY; 3232 } 3233 3234 newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL); 3235 if (!newpoolinfo) 3236 return -ENOMEM; 3237 newpoolinfo->mddev = mddev; 3238 newpoolinfo->raid_disks = raid_disks * 2; 3239 3240 newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc, 3241 r1bio_pool_free, newpoolinfo); 3242 if (!newpool) { 3243 kfree(newpoolinfo); 3244 return -ENOMEM; 3245 } 3246 newmirrors = kzalloc(sizeof(struct raid1_info) * raid_disks * 2, 3247 GFP_KERNEL); 3248 if (!newmirrors) { 3249 kfree(newpoolinfo); 3250 mempool_destroy(newpool); 3251 return -ENOMEM; 3252 } 3253 3254 freeze_array(conf, 0); 3255 3256 /* ok, everything is stopped */ 3257 oldpool = conf->r1bio_pool; 3258 conf->r1bio_pool = newpool; 3259 3260 for (d = d2 = 0; d < conf->raid_disks; d++) { 3261 struct md_rdev *rdev = conf->mirrors[d].rdev; 3262 if (rdev && rdev->raid_disk != d2) { 3263 sysfs_unlink_rdev(mddev, rdev); 3264 rdev->raid_disk = d2; 3265 sysfs_unlink_rdev(mddev, rdev); 3266 if (sysfs_link_rdev(mddev, rdev)) 3267 pr_warn("md/raid1:%s: cannot register rd%d\n", 3268 mdname(mddev), rdev->raid_disk); 3269 } 3270 if (rdev) 3271 newmirrors[d2++].rdev = rdev; 3272 } 3273 kfree(conf->mirrors); 3274 conf->mirrors = newmirrors; 3275 kfree(conf->poolinfo); 3276 conf->poolinfo = newpoolinfo; 3277 3278 spin_lock_irqsave(&conf->device_lock, flags); 3279 mddev->degraded += (raid_disks - conf->raid_disks); 3280 spin_unlock_irqrestore(&conf->device_lock, flags); 3281 conf->raid_disks = mddev->raid_disks = raid_disks; 3282 mddev->delta_disks = 0; 3283 3284 unfreeze_array(conf); 3285 3286 set_bit(MD_RECOVERY_RECOVER, &mddev->recovery); 3287 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3288 md_wakeup_thread(mddev->thread); 3289 3290 mempool_destroy(oldpool); 3291 return 0; 3292 } 3293 3294 static void raid1_quiesce(struct mddev *mddev, int state) 3295 { 3296 struct r1conf *conf = mddev->private; 3297 3298 switch(state) { 3299 case 2: /* wake for suspend */ 3300 wake_up(&conf->wait_barrier); 3301 break; 3302 case 1: 3303 freeze_array(conf, 0); 3304 break; 3305 case 0: 3306 unfreeze_array(conf); 3307 break; 3308 } 3309 } 3310 3311 static void *raid1_takeover(struct mddev *mddev) 3312 { 3313 /* raid1 can take over: 3314 * raid5 with 2 devices, any layout or chunk size 3315 */ 3316 if (mddev->level == 5 && mddev->raid_disks == 2) { 3317 struct r1conf *conf; 3318 mddev->new_level = 1; 3319 mddev->new_layout = 0; 3320 mddev->new_chunk_sectors = 0; 3321 conf = setup_conf(mddev); 3322 if (!IS_ERR(conf)) { 3323 /* Array must appear to be quiesced */ 3324 conf->array_frozen = 1; 3325 mddev_clear_unsupported_flags(mddev, 3326 UNSUPPORTED_MDDEV_FLAGS); 3327 } 3328 return conf; 3329 } 3330 return ERR_PTR(-EINVAL); 3331 } 3332 3333 static struct md_personality raid1_personality = 3334 { 3335 .name = "raid1", 3336 .level = 1, 3337 .owner = THIS_MODULE, 3338 .make_request = raid1_make_request, 3339 .run = raid1_run, 3340 .free = raid1_free, 3341 .status = raid1_status, 3342 .error_handler = raid1_error, 3343 .hot_add_disk = raid1_add_disk, 3344 .hot_remove_disk= raid1_remove_disk, 3345 .spare_active = raid1_spare_active, 3346 .sync_request = raid1_sync_request, 3347 .resize = raid1_resize, 3348 .size = raid1_size, 3349 .check_reshape = raid1_reshape, 3350 .quiesce = raid1_quiesce, 3351 .takeover = raid1_takeover, 3352 .congested = raid1_congested, 3353 }; 3354 3355 static int __init raid_init(void) 3356 { 3357 return register_md_personality(&raid1_personality); 3358 } 3359 3360 static void raid_exit(void) 3361 { 3362 unregister_md_personality(&raid1_personality); 3363 } 3364 3365 module_init(raid_init); 3366 module_exit(raid_exit); 3367 MODULE_LICENSE("GPL"); 3368 MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD"); 3369 MODULE_ALIAS("md-personality-3"); /* RAID1 */ 3370 MODULE_ALIAS("md-raid1"); 3371 MODULE_ALIAS("md-level-1"); 3372 3373 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 3374