1 /* 2 * raid10.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 2000-2004 Neil Brown 5 * 6 * RAID-10 support for md. 7 * 8 * Base on code in raid1.c. See raid1.c for further copyright information. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 #include <linux/slab.h> 22 #include <linux/delay.h> 23 #include <linux/blkdev.h> 24 #include <linux/module.h> 25 #include <linux/seq_file.h> 26 #include <linux/ratelimit.h> 27 #include <linux/kthread.h> 28 #include <trace/events/block.h> 29 #include "md.h" 30 #include "raid10.h" 31 #include "raid0.h" 32 #include "md-bitmap.h" 33 34 /* 35 * RAID10 provides a combination of RAID0 and RAID1 functionality. 36 * The layout of data is defined by 37 * chunk_size 38 * raid_disks 39 * near_copies (stored in low byte of layout) 40 * far_copies (stored in second byte of layout) 41 * far_offset (stored in bit 16 of layout ) 42 * use_far_sets (stored in bit 17 of layout ) 43 * use_far_sets_bugfixed (stored in bit 18 of layout ) 44 * 45 * The data to be stored is divided into chunks using chunksize. Each device 46 * is divided into far_copies sections. In each section, chunks are laid out 47 * in a style similar to raid0, but near_copies copies of each chunk is stored 48 * (each on a different drive). The starting device for each section is offset 49 * near_copies from the starting device of the previous section. Thus there 50 * are (near_copies * far_copies) of each chunk, and each is on a different 51 * drive. near_copies and far_copies must be at least one, and their product 52 * is at most raid_disks. 53 * 54 * If far_offset is true, then the far_copies are handled a bit differently. 55 * The copies are still in different stripes, but instead of being very far 56 * apart on disk, there are adjacent stripes. 57 * 58 * The far and offset algorithms are handled slightly differently if 59 * 'use_far_sets' is true. In this case, the array's devices are grouped into 60 * sets that are (near_copies * far_copies) in size. The far copied stripes 61 * are still shifted by 'near_copies' devices, but this shifting stays confined 62 * to the set rather than the entire array. This is done to improve the number 63 * of device combinations that can fail without causing the array to fail. 64 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 65 * on a device): 66 * A B C D A B C D E 67 * ... ... 68 * D A B C E A B C D 69 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 70 * [A B] [C D] [A B] [C D E] 71 * |...| |...| |...| | ... | 72 * [B A] [D C] [B A] [E C D] 73 */ 74 75 /* 76 * Number of guaranteed r10bios in case of extreme VM load: 77 */ 78 #define NR_RAID10_BIOS 256 79 80 /* when we get a read error on a read-only array, we redirect to another 81 * device without failing the first device, or trying to over-write to 82 * correct the read error. To keep track of bad blocks on a per-bio 83 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 84 */ 85 #define IO_BLOCKED ((struct bio *)1) 86 /* When we successfully write to a known bad-block, we need to remove the 87 * bad-block marking which must be done from process context. So we record 88 * the success by setting devs[n].bio to IO_MADE_GOOD 89 */ 90 #define IO_MADE_GOOD ((struct bio *)2) 91 92 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 93 94 /* When there are this many requests queued to be written by 95 * the raid10 thread, we become 'congested' to provide back-pressure 96 * for writeback. 97 */ 98 static int max_queued_requests = 1024; 99 100 static void allow_barrier(struct r10conf *conf); 101 static void lower_barrier(struct r10conf *conf); 102 static int _enough(struct r10conf *conf, int previous, int ignore); 103 static int enough(struct r10conf *conf, int ignore); 104 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 105 int *skipped); 106 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 107 static void end_reshape_write(struct bio *bio); 108 static void end_reshape(struct r10conf *conf); 109 110 #define raid10_log(md, fmt, args...) \ 111 do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0) 112 113 #include "raid1-10.c" 114 115 /* 116 * for resync bio, r10bio pointer can be retrieved from the per-bio 117 * 'struct resync_pages'. 118 */ 119 static inline struct r10bio *get_resync_r10bio(struct bio *bio) 120 { 121 return get_resync_pages(bio)->raid_bio; 122 } 123 124 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 125 { 126 struct r10conf *conf = data; 127 int size = offsetof(struct r10bio, devs[conf->copies]); 128 129 /* allocate a r10bio with room for raid_disks entries in the 130 * bios array */ 131 return kzalloc(size, gfp_flags); 132 } 133 134 static void r10bio_pool_free(void *r10_bio, void *data) 135 { 136 kfree(r10_bio); 137 } 138 139 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) 140 /* amount of memory to reserve for resync requests */ 141 #define RESYNC_WINDOW (1024*1024) 142 /* maximum number of concurrent requests, memory permitting */ 143 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 144 #define CLUSTER_RESYNC_WINDOW (32 * RESYNC_WINDOW) 145 #define CLUSTER_RESYNC_WINDOW_SECTORS (CLUSTER_RESYNC_WINDOW >> 9) 146 147 /* 148 * When performing a resync, we need to read and compare, so 149 * we need as many pages are there are copies. 150 * When performing a recovery, we need 2 bios, one for read, 151 * one for write (we recover only one drive per r10buf) 152 * 153 */ 154 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 155 { 156 struct r10conf *conf = data; 157 struct r10bio *r10_bio; 158 struct bio *bio; 159 int j; 160 int nalloc, nalloc_rp; 161 struct resync_pages *rps; 162 163 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 164 if (!r10_bio) 165 return NULL; 166 167 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 168 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 169 nalloc = conf->copies; /* resync */ 170 else 171 nalloc = 2; /* recovery */ 172 173 /* allocate once for all bios */ 174 if (!conf->have_replacement) 175 nalloc_rp = nalloc; 176 else 177 nalloc_rp = nalloc * 2; 178 rps = kmalloc_array(nalloc_rp, sizeof(struct resync_pages), gfp_flags); 179 if (!rps) 180 goto out_free_r10bio; 181 182 /* 183 * Allocate bios. 184 */ 185 for (j = nalloc ; j-- ; ) { 186 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 187 if (!bio) 188 goto out_free_bio; 189 r10_bio->devs[j].bio = bio; 190 if (!conf->have_replacement) 191 continue; 192 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 193 if (!bio) 194 goto out_free_bio; 195 r10_bio->devs[j].repl_bio = bio; 196 } 197 /* 198 * Allocate RESYNC_PAGES data pages and attach them 199 * where needed. 200 */ 201 for (j = 0; j < nalloc; j++) { 202 struct bio *rbio = r10_bio->devs[j].repl_bio; 203 struct resync_pages *rp, *rp_repl; 204 205 rp = &rps[j]; 206 if (rbio) 207 rp_repl = &rps[nalloc + j]; 208 209 bio = r10_bio->devs[j].bio; 210 211 if (!j || test_bit(MD_RECOVERY_SYNC, 212 &conf->mddev->recovery)) { 213 if (resync_alloc_pages(rp, gfp_flags)) 214 goto out_free_pages; 215 } else { 216 memcpy(rp, &rps[0], sizeof(*rp)); 217 resync_get_all_pages(rp); 218 } 219 220 rp->raid_bio = r10_bio; 221 bio->bi_private = rp; 222 if (rbio) { 223 memcpy(rp_repl, rp, sizeof(*rp)); 224 rbio->bi_private = rp_repl; 225 } 226 } 227 228 return r10_bio; 229 230 out_free_pages: 231 while (--j >= 0) 232 resync_free_pages(&rps[j * 2]); 233 234 j = 0; 235 out_free_bio: 236 for ( ; j < nalloc; j++) { 237 if (r10_bio->devs[j].bio) 238 bio_put(r10_bio->devs[j].bio); 239 if (r10_bio->devs[j].repl_bio) 240 bio_put(r10_bio->devs[j].repl_bio); 241 } 242 kfree(rps); 243 out_free_r10bio: 244 r10bio_pool_free(r10_bio, conf); 245 return NULL; 246 } 247 248 static void r10buf_pool_free(void *__r10_bio, void *data) 249 { 250 struct r10conf *conf = data; 251 struct r10bio *r10bio = __r10_bio; 252 int j; 253 struct resync_pages *rp = NULL; 254 255 for (j = conf->copies; j--; ) { 256 struct bio *bio = r10bio->devs[j].bio; 257 258 if (bio) { 259 rp = get_resync_pages(bio); 260 resync_free_pages(rp); 261 bio_put(bio); 262 } 263 264 bio = r10bio->devs[j].repl_bio; 265 if (bio) 266 bio_put(bio); 267 } 268 269 /* resync pages array stored in the 1st bio's .bi_private */ 270 kfree(rp); 271 272 r10bio_pool_free(r10bio, conf); 273 } 274 275 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 276 { 277 int i; 278 279 for (i = 0; i < conf->copies; i++) { 280 struct bio **bio = & r10_bio->devs[i].bio; 281 if (!BIO_SPECIAL(*bio)) 282 bio_put(*bio); 283 *bio = NULL; 284 bio = &r10_bio->devs[i].repl_bio; 285 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 286 bio_put(*bio); 287 *bio = NULL; 288 } 289 } 290 291 static void free_r10bio(struct r10bio *r10_bio) 292 { 293 struct r10conf *conf = r10_bio->mddev->private; 294 295 put_all_bios(conf, r10_bio); 296 mempool_free(r10_bio, &conf->r10bio_pool); 297 } 298 299 static void put_buf(struct r10bio *r10_bio) 300 { 301 struct r10conf *conf = r10_bio->mddev->private; 302 303 mempool_free(r10_bio, &conf->r10buf_pool); 304 305 lower_barrier(conf); 306 } 307 308 static void reschedule_retry(struct r10bio *r10_bio) 309 { 310 unsigned long flags; 311 struct mddev *mddev = r10_bio->mddev; 312 struct r10conf *conf = mddev->private; 313 314 spin_lock_irqsave(&conf->device_lock, flags); 315 list_add(&r10_bio->retry_list, &conf->retry_list); 316 conf->nr_queued ++; 317 spin_unlock_irqrestore(&conf->device_lock, flags); 318 319 /* wake up frozen array... */ 320 wake_up(&conf->wait_barrier); 321 322 md_wakeup_thread(mddev->thread); 323 } 324 325 /* 326 * raid_end_bio_io() is called when we have finished servicing a mirrored 327 * operation and are ready to return a success/failure code to the buffer 328 * cache layer. 329 */ 330 static void raid_end_bio_io(struct r10bio *r10_bio) 331 { 332 struct bio *bio = r10_bio->master_bio; 333 struct r10conf *conf = r10_bio->mddev->private; 334 335 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 336 bio->bi_status = BLK_STS_IOERR; 337 338 bio_endio(bio); 339 /* 340 * Wake up any possible resync thread that waits for the device 341 * to go idle. 342 */ 343 allow_barrier(conf); 344 345 free_r10bio(r10_bio); 346 } 347 348 /* 349 * Update disk head position estimator based on IRQ completion info. 350 */ 351 static inline void update_head_pos(int slot, struct r10bio *r10_bio) 352 { 353 struct r10conf *conf = r10_bio->mddev->private; 354 355 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 356 r10_bio->devs[slot].addr + (r10_bio->sectors); 357 } 358 359 /* 360 * Find the disk number which triggered given bio 361 */ 362 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 363 struct bio *bio, int *slotp, int *replp) 364 { 365 int slot; 366 int repl = 0; 367 368 for (slot = 0; slot < conf->copies; slot++) { 369 if (r10_bio->devs[slot].bio == bio) 370 break; 371 if (r10_bio->devs[slot].repl_bio == bio) { 372 repl = 1; 373 break; 374 } 375 } 376 377 BUG_ON(slot == conf->copies); 378 update_head_pos(slot, r10_bio); 379 380 if (slotp) 381 *slotp = slot; 382 if (replp) 383 *replp = repl; 384 return r10_bio->devs[slot].devnum; 385 } 386 387 static void raid10_end_read_request(struct bio *bio) 388 { 389 int uptodate = !bio->bi_status; 390 struct r10bio *r10_bio = bio->bi_private; 391 int slot; 392 struct md_rdev *rdev; 393 struct r10conf *conf = r10_bio->mddev->private; 394 395 slot = r10_bio->read_slot; 396 rdev = r10_bio->devs[slot].rdev; 397 /* 398 * this branch is our 'one mirror IO has finished' event handler: 399 */ 400 update_head_pos(slot, r10_bio); 401 402 if (uptodate) { 403 /* 404 * Set R10BIO_Uptodate in our master bio, so that 405 * we will return a good error code to the higher 406 * levels even if IO on some other mirrored buffer fails. 407 * 408 * The 'master' represents the composite IO operation to 409 * user-side. So if something waits for IO, then it will 410 * wait for the 'master' bio. 411 */ 412 set_bit(R10BIO_Uptodate, &r10_bio->state); 413 } else { 414 /* If all other devices that store this block have 415 * failed, we want to return the error upwards rather 416 * than fail the last device. Here we redefine 417 * "uptodate" to mean "Don't want to retry" 418 */ 419 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), 420 rdev->raid_disk)) 421 uptodate = 1; 422 } 423 if (uptodate) { 424 raid_end_bio_io(r10_bio); 425 rdev_dec_pending(rdev, conf->mddev); 426 } else { 427 /* 428 * oops, read error - keep the refcount on the rdev 429 */ 430 char b[BDEVNAME_SIZE]; 431 pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n", 432 mdname(conf->mddev), 433 bdevname(rdev->bdev, b), 434 (unsigned long long)r10_bio->sector); 435 set_bit(R10BIO_ReadError, &r10_bio->state); 436 reschedule_retry(r10_bio); 437 } 438 } 439 440 static void close_write(struct r10bio *r10_bio) 441 { 442 /* clear the bitmap if all writes complete successfully */ 443 md_bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector, 444 r10_bio->sectors, 445 !test_bit(R10BIO_Degraded, &r10_bio->state), 446 0); 447 md_write_end(r10_bio->mddev); 448 } 449 450 static void one_write_done(struct r10bio *r10_bio) 451 { 452 if (atomic_dec_and_test(&r10_bio->remaining)) { 453 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 454 reschedule_retry(r10_bio); 455 else { 456 close_write(r10_bio); 457 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 458 reschedule_retry(r10_bio); 459 else 460 raid_end_bio_io(r10_bio); 461 } 462 } 463 } 464 465 static void raid10_end_write_request(struct bio *bio) 466 { 467 struct r10bio *r10_bio = bio->bi_private; 468 int dev; 469 int dec_rdev = 1; 470 struct r10conf *conf = r10_bio->mddev->private; 471 int slot, repl; 472 struct md_rdev *rdev = NULL; 473 struct bio *to_put = NULL; 474 bool discard_error; 475 476 discard_error = bio->bi_status && bio_op(bio) == REQ_OP_DISCARD; 477 478 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 479 480 if (repl) 481 rdev = conf->mirrors[dev].replacement; 482 if (!rdev) { 483 smp_rmb(); 484 repl = 0; 485 rdev = conf->mirrors[dev].rdev; 486 } 487 /* 488 * this branch is our 'one mirror IO has finished' event handler: 489 */ 490 if (bio->bi_status && !discard_error) { 491 if (repl) 492 /* Never record new bad blocks to replacement, 493 * just fail it. 494 */ 495 md_error(rdev->mddev, rdev); 496 else { 497 set_bit(WriteErrorSeen, &rdev->flags); 498 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 499 set_bit(MD_RECOVERY_NEEDED, 500 &rdev->mddev->recovery); 501 502 dec_rdev = 0; 503 if (test_bit(FailFast, &rdev->flags) && 504 (bio->bi_opf & MD_FAILFAST)) { 505 md_error(rdev->mddev, rdev); 506 if (!test_bit(Faulty, &rdev->flags)) 507 /* This is the only remaining device, 508 * We need to retry the write without 509 * FailFast 510 */ 511 set_bit(R10BIO_WriteError, &r10_bio->state); 512 else { 513 r10_bio->devs[slot].bio = NULL; 514 to_put = bio; 515 dec_rdev = 1; 516 } 517 } else 518 set_bit(R10BIO_WriteError, &r10_bio->state); 519 } 520 } else { 521 /* 522 * Set R10BIO_Uptodate in our master bio, so that 523 * we will return a good error code for to the higher 524 * levels even if IO on some other mirrored buffer fails. 525 * 526 * The 'master' represents the composite IO operation to 527 * user-side. So if something waits for IO, then it will 528 * wait for the 'master' bio. 529 */ 530 sector_t first_bad; 531 int bad_sectors; 532 533 /* 534 * Do not set R10BIO_Uptodate if the current device is 535 * rebuilding or Faulty. This is because we cannot use 536 * such device for properly reading the data back (we could 537 * potentially use it, if the current write would have felt 538 * before rdev->recovery_offset, but for simplicity we don't 539 * check this here. 540 */ 541 if (test_bit(In_sync, &rdev->flags) && 542 !test_bit(Faulty, &rdev->flags)) 543 set_bit(R10BIO_Uptodate, &r10_bio->state); 544 545 /* Maybe we can clear some bad blocks. */ 546 if (is_badblock(rdev, 547 r10_bio->devs[slot].addr, 548 r10_bio->sectors, 549 &first_bad, &bad_sectors) && !discard_error) { 550 bio_put(bio); 551 if (repl) 552 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 553 else 554 r10_bio->devs[slot].bio = IO_MADE_GOOD; 555 dec_rdev = 0; 556 set_bit(R10BIO_MadeGood, &r10_bio->state); 557 } 558 } 559 560 /* 561 * 562 * Let's see if all mirrored write operations have finished 563 * already. 564 */ 565 one_write_done(r10_bio); 566 if (dec_rdev) 567 rdev_dec_pending(rdev, conf->mddev); 568 if (to_put) 569 bio_put(to_put); 570 } 571 572 /* 573 * RAID10 layout manager 574 * As well as the chunksize and raid_disks count, there are two 575 * parameters: near_copies and far_copies. 576 * near_copies * far_copies must be <= raid_disks. 577 * Normally one of these will be 1. 578 * If both are 1, we get raid0. 579 * If near_copies == raid_disks, we get raid1. 580 * 581 * Chunks are laid out in raid0 style with near_copies copies of the 582 * first chunk, followed by near_copies copies of the next chunk and 583 * so on. 584 * If far_copies > 1, then after 1/far_copies of the array has been assigned 585 * as described above, we start again with a device offset of near_copies. 586 * So we effectively have another copy of the whole array further down all 587 * the drives, but with blocks on different drives. 588 * With this layout, and block is never stored twice on the one device. 589 * 590 * raid10_find_phys finds the sector offset of a given virtual sector 591 * on each device that it is on. 592 * 593 * raid10_find_virt does the reverse mapping, from a device and a 594 * sector offset to a virtual address 595 */ 596 597 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 598 { 599 int n,f; 600 sector_t sector; 601 sector_t chunk; 602 sector_t stripe; 603 int dev; 604 int slot = 0; 605 int last_far_set_start, last_far_set_size; 606 607 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 608 last_far_set_start *= geo->far_set_size; 609 610 last_far_set_size = geo->far_set_size; 611 last_far_set_size += (geo->raid_disks % geo->far_set_size); 612 613 /* now calculate first sector/dev */ 614 chunk = r10bio->sector >> geo->chunk_shift; 615 sector = r10bio->sector & geo->chunk_mask; 616 617 chunk *= geo->near_copies; 618 stripe = chunk; 619 dev = sector_div(stripe, geo->raid_disks); 620 if (geo->far_offset) 621 stripe *= geo->far_copies; 622 623 sector += stripe << geo->chunk_shift; 624 625 /* and calculate all the others */ 626 for (n = 0; n < geo->near_copies; n++) { 627 int d = dev; 628 int set; 629 sector_t s = sector; 630 r10bio->devs[slot].devnum = d; 631 r10bio->devs[slot].addr = s; 632 slot++; 633 634 for (f = 1; f < geo->far_copies; f++) { 635 set = d / geo->far_set_size; 636 d += geo->near_copies; 637 638 if ((geo->raid_disks % geo->far_set_size) && 639 (d > last_far_set_start)) { 640 d -= last_far_set_start; 641 d %= last_far_set_size; 642 d += last_far_set_start; 643 } else { 644 d %= geo->far_set_size; 645 d += geo->far_set_size * set; 646 } 647 s += geo->stride; 648 r10bio->devs[slot].devnum = d; 649 r10bio->devs[slot].addr = s; 650 slot++; 651 } 652 dev++; 653 if (dev >= geo->raid_disks) { 654 dev = 0; 655 sector += (geo->chunk_mask + 1); 656 } 657 } 658 } 659 660 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 661 { 662 struct geom *geo = &conf->geo; 663 664 if (conf->reshape_progress != MaxSector && 665 ((r10bio->sector >= conf->reshape_progress) != 666 conf->mddev->reshape_backwards)) { 667 set_bit(R10BIO_Previous, &r10bio->state); 668 geo = &conf->prev; 669 } else 670 clear_bit(R10BIO_Previous, &r10bio->state); 671 672 __raid10_find_phys(geo, r10bio); 673 } 674 675 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 676 { 677 sector_t offset, chunk, vchunk; 678 /* Never use conf->prev as this is only called during resync 679 * or recovery, so reshape isn't happening 680 */ 681 struct geom *geo = &conf->geo; 682 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 683 int far_set_size = geo->far_set_size; 684 int last_far_set_start; 685 686 if (geo->raid_disks % geo->far_set_size) { 687 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 688 last_far_set_start *= geo->far_set_size; 689 690 if (dev >= last_far_set_start) { 691 far_set_size = geo->far_set_size; 692 far_set_size += (geo->raid_disks % geo->far_set_size); 693 far_set_start = last_far_set_start; 694 } 695 } 696 697 offset = sector & geo->chunk_mask; 698 if (geo->far_offset) { 699 int fc; 700 chunk = sector >> geo->chunk_shift; 701 fc = sector_div(chunk, geo->far_copies); 702 dev -= fc * geo->near_copies; 703 if (dev < far_set_start) 704 dev += far_set_size; 705 } else { 706 while (sector >= geo->stride) { 707 sector -= geo->stride; 708 if (dev < (geo->near_copies + far_set_start)) 709 dev += far_set_size - geo->near_copies; 710 else 711 dev -= geo->near_copies; 712 } 713 chunk = sector >> geo->chunk_shift; 714 } 715 vchunk = chunk * geo->raid_disks + dev; 716 sector_div(vchunk, geo->near_copies); 717 return (vchunk << geo->chunk_shift) + offset; 718 } 719 720 /* 721 * This routine returns the disk from which the requested read should 722 * be done. There is a per-array 'next expected sequential IO' sector 723 * number - if this matches on the next IO then we use the last disk. 724 * There is also a per-disk 'last know head position' sector that is 725 * maintained from IRQ contexts, both the normal and the resync IO 726 * completion handlers update this position correctly. If there is no 727 * perfect sequential match then we pick the disk whose head is closest. 728 * 729 * If there are 2 mirrors in the same 2 devices, performance degrades 730 * because position is mirror, not device based. 731 * 732 * The rdev for the device selected will have nr_pending incremented. 733 */ 734 735 /* 736 * FIXME: possibly should rethink readbalancing and do it differently 737 * depending on near_copies / far_copies geometry. 738 */ 739 static struct md_rdev *read_balance(struct r10conf *conf, 740 struct r10bio *r10_bio, 741 int *max_sectors) 742 { 743 const sector_t this_sector = r10_bio->sector; 744 int disk, slot; 745 int sectors = r10_bio->sectors; 746 int best_good_sectors; 747 sector_t new_distance, best_dist; 748 struct md_rdev *best_rdev, *rdev = NULL; 749 int do_balance; 750 int best_slot; 751 struct geom *geo = &conf->geo; 752 753 raid10_find_phys(conf, r10_bio); 754 rcu_read_lock(); 755 best_slot = -1; 756 best_rdev = NULL; 757 best_dist = MaxSector; 758 best_good_sectors = 0; 759 do_balance = 1; 760 clear_bit(R10BIO_FailFast, &r10_bio->state); 761 /* 762 * Check if we can balance. We can balance on the whole 763 * device if no resync is going on (recovery is ok), or below 764 * the resync window. We take the first readable disk when 765 * above the resync window. 766 */ 767 if ((conf->mddev->recovery_cp < MaxSector 768 && (this_sector + sectors >= conf->next_resync)) || 769 (mddev_is_clustered(conf->mddev) && 770 md_cluster_ops->area_resyncing(conf->mddev, READ, this_sector, 771 this_sector + sectors))) 772 do_balance = 0; 773 774 for (slot = 0; slot < conf->copies ; slot++) { 775 sector_t first_bad; 776 int bad_sectors; 777 sector_t dev_sector; 778 779 if (r10_bio->devs[slot].bio == IO_BLOCKED) 780 continue; 781 disk = r10_bio->devs[slot].devnum; 782 rdev = rcu_dereference(conf->mirrors[disk].replacement); 783 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 784 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 785 rdev = rcu_dereference(conf->mirrors[disk].rdev); 786 if (rdev == NULL || 787 test_bit(Faulty, &rdev->flags)) 788 continue; 789 if (!test_bit(In_sync, &rdev->flags) && 790 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 791 continue; 792 793 dev_sector = r10_bio->devs[slot].addr; 794 if (is_badblock(rdev, dev_sector, sectors, 795 &first_bad, &bad_sectors)) { 796 if (best_dist < MaxSector) 797 /* Already have a better slot */ 798 continue; 799 if (first_bad <= dev_sector) { 800 /* Cannot read here. If this is the 801 * 'primary' device, then we must not read 802 * beyond 'bad_sectors' from another device. 803 */ 804 bad_sectors -= (dev_sector - first_bad); 805 if (!do_balance && sectors > bad_sectors) 806 sectors = bad_sectors; 807 if (best_good_sectors > sectors) 808 best_good_sectors = sectors; 809 } else { 810 sector_t good_sectors = 811 first_bad - dev_sector; 812 if (good_sectors > best_good_sectors) { 813 best_good_sectors = good_sectors; 814 best_slot = slot; 815 best_rdev = rdev; 816 } 817 if (!do_balance) 818 /* Must read from here */ 819 break; 820 } 821 continue; 822 } else 823 best_good_sectors = sectors; 824 825 if (!do_balance) 826 break; 827 828 if (best_slot >= 0) 829 /* At least 2 disks to choose from so failfast is OK */ 830 set_bit(R10BIO_FailFast, &r10_bio->state); 831 /* This optimisation is debatable, and completely destroys 832 * sequential read speed for 'far copies' arrays. So only 833 * keep it for 'near' arrays, and review those later. 834 */ 835 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending)) 836 new_distance = 0; 837 838 /* for far > 1 always use the lowest address */ 839 else if (geo->far_copies > 1) 840 new_distance = r10_bio->devs[slot].addr; 841 else 842 new_distance = abs(r10_bio->devs[slot].addr - 843 conf->mirrors[disk].head_position); 844 if (new_distance < best_dist) { 845 best_dist = new_distance; 846 best_slot = slot; 847 best_rdev = rdev; 848 } 849 } 850 if (slot >= conf->copies) { 851 slot = best_slot; 852 rdev = best_rdev; 853 } 854 855 if (slot >= 0) { 856 atomic_inc(&rdev->nr_pending); 857 r10_bio->read_slot = slot; 858 } else 859 rdev = NULL; 860 rcu_read_unlock(); 861 *max_sectors = best_good_sectors; 862 863 return rdev; 864 } 865 866 static int raid10_congested(struct mddev *mddev, int bits) 867 { 868 struct r10conf *conf = mddev->private; 869 int i, ret = 0; 870 871 if ((bits & (1 << WB_async_congested)) && 872 conf->pending_count >= max_queued_requests) 873 return 1; 874 875 rcu_read_lock(); 876 for (i = 0; 877 (i < conf->geo.raid_disks || i < conf->prev.raid_disks) 878 && ret == 0; 879 i++) { 880 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 881 if (rdev && !test_bit(Faulty, &rdev->flags)) { 882 struct request_queue *q = bdev_get_queue(rdev->bdev); 883 884 ret |= bdi_congested(q->backing_dev_info, bits); 885 } 886 } 887 rcu_read_unlock(); 888 return ret; 889 } 890 891 static void flush_pending_writes(struct r10conf *conf) 892 { 893 /* Any writes that have been queued but are awaiting 894 * bitmap updates get flushed here. 895 */ 896 spin_lock_irq(&conf->device_lock); 897 898 if (conf->pending_bio_list.head) { 899 struct blk_plug plug; 900 struct bio *bio; 901 902 bio = bio_list_get(&conf->pending_bio_list); 903 conf->pending_count = 0; 904 spin_unlock_irq(&conf->device_lock); 905 906 /* 907 * As this is called in a wait_event() loop (see freeze_array), 908 * current->state might be TASK_UNINTERRUPTIBLE which will 909 * cause a warning when we prepare to wait again. As it is 910 * rare that this path is taken, it is perfectly safe to force 911 * us to go around the wait_event() loop again, so the warning 912 * is a false-positive. Silence the warning by resetting 913 * thread state 914 */ 915 __set_current_state(TASK_RUNNING); 916 917 blk_start_plug(&plug); 918 /* flush any pending bitmap writes to disk 919 * before proceeding w/ I/O */ 920 md_bitmap_unplug(conf->mddev->bitmap); 921 wake_up(&conf->wait_barrier); 922 923 while (bio) { /* submit pending writes */ 924 struct bio *next = bio->bi_next; 925 struct md_rdev *rdev = (void*)bio->bi_disk; 926 bio->bi_next = NULL; 927 bio_set_dev(bio, rdev->bdev); 928 if (test_bit(Faulty, &rdev->flags)) { 929 bio_io_error(bio); 930 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 931 !blk_queue_discard(bio->bi_disk->queue))) 932 /* Just ignore it */ 933 bio_endio(bio); 934 else 935 generic_make_request(bio); 936 bio = next; 937 } 938 blk_finish_plug(&plug); 939 } else 940 spin_unlock_irq(&conf->device_lock); 941 } 942 943 /* Barriers.... 944 * Sometimes we need to suspend IO while we do something else, 945 * either some resync/recovery, or reconfigure the array. 946 * To do this we raise a 'barrier'. 947 * The 'barrier' is a counter that can be raised multiple times 948 * to count how many activities are happening which preclude 949 * normal IO. 950 * We can only raise the barrier if there is no pending IO. 951 * i.e. if nr_pending == 0. 952 * We choose only to raise the barrier if no-one is waiting for the 953 * barrier to go down. This means that as soon as an IO request 954 * is ready, no other operations which require a barrier will start 955 * until the IO request has had a chance. 956 * 957 * So: regular IO calls 'wait_barrier'. When that returns there 958 * is no backgroup IO happening, It must arrange to call 959 * allow_barrier when it has finished its IO. 960 * backgroup IO calls must call raise_barrier. Once that returns 961 * there is no normal IO happeing. It must arrange to call 962 * lower_barrier when the particular background IO completes. 963 */ 964 965 static void raise_barrier(struct r10conf *conf, int force) 966 { 967 BUG_ON(force && !conf->barrier); 968 spin_lock_irq(&conf->resync_lock); 969 970 /* Wait until no block IO is waiting (unless 'force') */ 971 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, 972 conf->resync_lock); 973 974 /* block any new IO from starting */ 975 conf->barrier++; 976 977 /* Now wait for all pending IO to complete */ 978 wait_event_lock_irq(conf->wait_barrier, 979 !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH, 980 conf->resync_lock); 981 982 spin_unlock_irq(&conf->resync_lock); 983 } 984 985 static void lower_barrier(struct r10conf *conf) 986 { 987 unsigned long flags; 988 spin_lock_irqsave(&conf->resync_lock, flags); 989 conf->barrier--; 990 spin_unlock_irqrestore(&conf->resync_lock, flags); 991 wake_up(&conf->wait_barrier); 992 } 993 994 static void wait_barrier(struct r10conf *conf) 995 { 996 spin_lock_irq(&conf->resync_lock); 997 if (conf->barrier) { 998 conf->nr_waiting++; 999 /* Wait for the barrier to drop. 1000 * However if there are already pending 1001 * requests (preventing the barrier from 1002 * rising completely), and the 1003 * pre-process bio queue isn't empty, 1004 * then don't wait, as we need to empty 1005 * that queue to get the nr_pending 1006 * count down. 1007 */ 1008 raid10_log(conf->mddev, "wait barrier"); 1009 wait_event_lock_irq(conf->wait_barrier, 1010 !conf->barrier || 1011 (atomic_read(&conf->nr_pending) && 1012 current->bio_list && 1013 (!bio_list_empty(¤t->bio_list[0]) || 1014 !bio_list_empty(¤t->bio_list[1]))), 1015 conf->resync_lock); 1016 conf->nr_waiting--; 1017 if (!conf->nr_waiting) 1018 wake_up(&conf->wait_barrier); 1019 } 1020 atomic_inc(&conf->nr_pending); 1021 spin_unlock_irq(&conf->resync_lock); 1022 } 1023 1024 static void allow_barrier(struct r10conf *conf) 1025 { 1026 if ((atomic_dec_and_test(&conf->nr_pending)) || 1027 (conf->array_freeze_pending)) 1028 wake_up(&conf->wait_barrier); 1029 } 1030 1031 static void freeze_array(struct r10conf *conf, int extra) 1032 { 1033 /* stop syncio and normal IO and wait for everything to 1034 * go quiet. 1035 * We increment barrier and nr_waiting, and then 1036 * wait until nr_pending match nr_queued+extra 1037 * This is called in the context of one normal IO request 1038 * that has failed. Thus any sync request that might be pending 1039 * will be blocked by nr_pending, and we need to wait for 1040 * pending IO requests to complete or be queued for re-try. 1041 * Thus the number queued (nr_queued) plus this request (extra) 1042 * must match the number of pending IOs (nr_pending) before 1043 * we continue. 1044 */ 1045 spin_lock_irq(&conf->resync_lock); 1046 conf->array_freeze_pending++; 1047 conf->barrier++; 1048 conf->nr_waiting++; 1049 wait_event_lock_irq_cmd(conf->wait_barrier, 1050 atomic_read(&conf->nr_pending) == conf->nr_queued+extra, 1051 conf->resync_lock, 1052 flush_pending_writes(conf)); 1053 1054 conf->array_freeze_pending--; 1055 spin_unlock_irq(&conf->resync_lock); 1056 } 1057 1058 static void unfreeze_array(struct r10conf *conf) 1059 { 1060 /* reverse the effect of the freeze */ 1061 spin_lock_irq(&conf->resync_lock); 1062 conf->barrier--; 1063 conf->nr_waiting--; 1064 wake_up(&conf->wait_barrier); 1065 spin_unlock_irq(&conf->resync_lock); 1066 } 1067 1068 static sector_t choose_data_offset(struct r10bio *r10_bio, 1069 struct md_rdev *rdev) 1070 { 1071 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 1072 test_bit(R10BIO_Previous, &r10_bio->state)) 1073 return rdev->data_offset; 1074 else 1075 return rdev->new_data_offset; 1076 } 1077 1078 struct raid10_plug_cb { 1079 struct blk_plug_cb cb; 1080 struct bio_list pending; 1081 int pending_cnt; 1082 }; 1083 1084 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1085 { 1086 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb, 1087 cb); 1088 struct mddev *mddev = plug->cb.data; 1089 struct r10conf *conf = mddev->private; 1090 struct bio *bio; 1091 1092 if (from_schedule || current->bio_list) { 1093 spin_lock_irq(&conf->device_lock); 1094 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1095 conf->pending_count += plug->pending_cnt; 1096 spin_unlock_irq(&conf->device_lock); 1097 wake_up(&conf->wait_barrier); 1098 md_wakeup_thread(mddev->thread); 1099 kfree(plug); 1100 return; 1101 } 1102 1103 /* we aren't scheduling, so we can do the write-out directly. */ 1104 bio = bio_list_get(&plug->pending); 1105 md_bitmap_unplug(mddev->bitmap); 1106 wake_up(&conf->wait_barrier); 1107 1108 while (bio) { /* submit pending writes */ 1109 struct bio *next = bio->bi_next; 1110 struct md_rdev *rdev = (void*)bio->bi_disk; 1111 bio->bi_next = NULL; 1112 bio_set_dev(bio, rdev->bdev); 1113 if (test_bit(Faulty, &rdev->flags)) { 1114 bio_io_error(bio); 1115 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 1116 !blk_queue_discard(bio->bi_disk->queue))) 1117 /* Just ignore it */ 1118 bio_endio(bio); 1119 else 1120 generic_make_request(bio); 1121 bio = next; 1122 } 1123 kfree(plug); 1124 } 1125 1126 static void raid10_read_request(struct mddev *mddev, struct bio *bio, 1127 struct r10bio *r10_bio) 1128 { 1129 struct r10conf *conf = mddev->private; 1130 struct bio *read_bio; 1131 const int op = bio_op(bio); 1132 const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); 1133 int max_sectors; 1134 sector_t sectors; 1135 struct md_rdev *rdev; 1136 char b[BDEVNAME_SIZE]; 1137 int slot = r10_bio->read_slot; 1138 struct md_rdev *err_rdev = NULL; 1139 gfp_t gfp = GFP_NOIO; 1140 1141 if (r10_bio->devs[slot].rdev) { 1142 /* 1143 * This is an error retry, but we cannot 1144 * safely dereference the rdev in the r10_bio, 1145 * we must use the one in conf. 1146 * If it has already been disconnected (unlikely) 1147 * we lose the device name in error messages. 1148 */ 1149 int disk; 1150 /* 1151 * As we are blocking raid10, it is a little safer to 1152 * use __GFP_HIGH. 1153 */ 1154 gfp = GFP_NOIO | __GFP_HIGH; 1155 1156 rcu_read_lock(); 1157 disk = r10_bio->devs[slot].devnum; 1158 err_rdev = rcu_dereference(conf->mirrors[disk].rdev); 1159 if (err_rdev) 1160 bdevname(err_rdev->bdev, b); 1161 else { 1162 strcpy(b, "???"); 1163 /* This never gets dereferenced */ 1164 err_rdev = r10_bio->devs[slot].rdev; 1165 } 1166 rcu_read_unlock(); 1167 } 1168 /* 1169 * Register the new request and wait if the reconstruction 1170 * thread has put up a bar for new requests. 1171 * Continue immediately if no resync is active currently. 1172 */ 1173 wait_barrier(conf); 1174 1175 sectors = r10_bio->sectors; 1176 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1177 bio->bi_iter.bi_sector < conf->reshape_progress && 1178 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1179 /* 1180 * IO spans the reshape position. Need to wait for reshape to 1181 * pass 1182 */ 1183 raid10_log(conf->mddev, "wait reshape"); 1184 allow_barrier(conf); 1185 wait_event(conf->wait_barrier, 1186 conf->reshape_progress <= bio->bi_iter.bi_sector || 1187 conf->reshape_progress >= bio->bi_iter.bi_sector + 1188 sectors); 1189 wait_barrier(conf); 1190 } 1191 1192 rdev = read_balance(conf, r10_bio, &max_sectors); 1193 if (!rdev) { 1194 if (err_rdev) { 1195 pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n", 1196 mdname(mddev), b, 1197 (unsigned long long)r10_bio->sector); 1198 } 1199 raid_end_bio_io(r10_bio); 1200 return; 1201 } 1202 if (err_rdev) 1203 pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n", 1204 mdname(mddev), 1205 bdevname(rdev->bdev, b), 1206 (unsigned long long)r10_bio->sector); 1207 if (max_sectors < bio_sectors(bio)) { 1208 struct bio *split = bio_split(bio, max_sectors, 1209 gfp, &conf->bio_split); 1210 bio_chain(split, bio); 1211 generic_make_request(bio); 1212 bio = split; 1213 r10_bio->master_bio = bio; 1214 r10_bio->sectors = max_sectors; 1215 } 1216 slot = r10_bio->read_slot; 1217 1218 read_bio = bio_clone_fast(bio, gfp, &mddev->bio_set); 1219 1220 r10_bio->devs[slot].bio = read_bio; 1221 r10_bio->devs[slot].rdev = rdev; 1222 1223 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + 1224 choose_data_offset(r10_bio, rdev); 1225 bio_set_dev(read_bio, rdev->bdev); 1226 read_bio->bi_end_io = raid10_end_read_request; 1227 bio_set_op_attrs(read_bio, op, do_sync); 1228 if (test_bit(FailFast, &rdev->flags) && 1229 test_bit(R10BIO_FailFast, &r10_bio->state)) 1230 read_bio->bi_opf |= MD_FAILFAST; 1231 read_bio->bi_private = r10_bio; 1232 1233 if (mddev->gendisk) 1234 trace_block_bio_remap(read_bio->bi_disk->queue, 1235 read_bio, disk_devt(mddev->gendisk), 1236 r10_bio->sector); 1237 generic_make_request(read_bio); 1238 return; 1239 } 1240 1241 static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, 1242 struct bio *bio, bool replacement, 1243 int n_copy) 1244 { 1245 const int op = bio_op(bio); 1246 const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); 1247 const unsigned long do_fua = (bio->bi_opf & REQ_FUA); 1248 unsigned long flags; 1249 struct blk_plug_cb *cb; 1250 struct raid10_plug_cb *plug = NULL; 1251 struct r10conf *conf = mddev->private; 1252 struct md_rdev *rdev; 1253 int devnum = r10_bio->devs[n_copy].devnum; 1254 struct bio *mbio; 1255 1256 if (replacement) { 1257 rdev = conf->mirrors[devnum].replacement; 1258 if (rdev == NULL) { 1259 /* Replacement just got moved to main 'rdev' */ 1260 smp_mb(); 1261 rdev = conf->mirrors[devnum].rdev; 1262 } 1263 } else 1264 rdev = conf->mirrors[devnum].rdev; 1265 1266 mbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set); 1267 if (replacement) 1268 r10_bio->devs[n_copy].repl_bio = mbio; 1269 else 1270 r10_bio->devs[n_copy].bio = mbio; 1271 1272 mbio->bi_iter.bi_sector = (r10_bio->devs[n_copy].addr + 1273 choose_data_offset(r10_bio, rdev)); 1274 bio_set_dev(mbio, rdev->bdev); 1275 mbio->bi_end_io = raid10_end_write_request; 1276 bio_set_op_attrs(mbio, op, do_sync | do_fua); 1277 if (!replacement && test_bit(FailFast, 1278 &conf->mirrors[devnum].rdev->flags) 1279 && enough(conf, devnum)) 1280 mbio->bi_opf |= MD_FAILFAST; 1281 mbio->bi_private = r10_bio; 1282 1283 if (conf->mddev->gendisk) 1284 trace_block_bio_remap(mbio->bi_disk->queue, 1285 mbio, disk_devt(conf->mddev->gendisk), 1286 r10_bio->sector); 1287 /* flush_pending_writes() needs access to the rdev so...*/ 1288 mbio->bi_disk = (void *)rdev; 1289 1290 atomic_inc(&r10_bio->remaining); 1291 1292 cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug)); 1293 if (cb) 1294 plug = container_of(cb, struct raid10_plug_cb, cb); 1295 else 1296 plug = NULL; 1297 if (plug) { 1298 bio_list_add(&plug->pending, mbio); 1299 plug->pending_cnt++; 1300 } else { 1301 spin_lock_irqsave(&conf->device_lock, flags); 1302 bio_list_add(&conf->pending_bio_list, mbio); 1303 conf->pending_count++; 1304 spin_unlock_irqrestore(&conf->device_lock, flags); 1305 md_wakeup_thread(mddev->thread); 1306 } 1307 } 1308 1309 static void raid10_write_request(struct mddev *mddev, struct bio *bio, 1310 struct r10bio *r10_bio) 1311 { 1312 struct r10conf *conf = mddev->private; 1313 int i; 1314 struct md_rdev *blocked_rdev; 1315 sector_t sectors; 1316 int max_sectors; 1317 1318 if ((mddev_is_clustered(mddev) && 1319 md_cluster_ops->area_resyncing(mddev, WRITE, 1320 bio->bi_iter.bi_sector, 1321 bio_end_sector(bio)))) { 1322 DEFINE_WAIT(w); 1323 for (;;) { 1324 prepare_to_wait(&conf->wait_barrier, 1325 &w, TASK_IDLE); 1326 if (!md_cluster_ops->area_resyncing(mddev, WRITE, 1327 bio->bi_iter.bi_sector, bio_end_sector(bio))) 1328 break; 1329 schedule(); 1330 } 1331 finish_wait(&conf->wait_barrier, &w); 1332 } 1333 1334 /* 1335 * Register the new request and wait if the reconstruction 1336 * thread has put up a bar for new requests. 1337 * Continue immediately if no resync is active currently. 1338 */ 1339 wait_barrier(conf); 1340 1341 sectors = r10_bio->sectors; 1342 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1343 bio->bi_iter.bi_sector < conf->reshape_progress && 1344 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1345 /* 1346 * IO spans the reshape position. Need to wait for reshape to 1347 * pass 1348 */ 1349 raid10_log(conf->mddev, "wait reshape"); 1350 allow_barrier(conf); 1351 wait_event(conf->wait_barrier, 1352 conf->reshape_progress <= bio->bi_iter.bi_sector || 1353 conf->reshape_progress >= bio->bi_iter.bi_sector + 1354 sectors); 1355 wait_barrier(conf); 1356 } 1357 1358 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1359 (mddev->reshape_backwards 1360 ? (bio->bi_iter.bi_sector < conf->reshape_safe && 1361 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) 1362 : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe && 1363 bio->bi_iter.bi_sector < conf->reshape_progress))) { 1364 /* Need to update reshape_position in metadata */ 1365 mddev->reshape_position = conf->reshape_progress; 1366 set_mask_bits(&mddev->sb_flags, 0, 1367 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1368 md_wakeup_thread(mddev->thread); 1369 raid10_log(conf->mddev, "wait reshape metadata"); 1370 wait_event(mddev->sb_wait, 1371 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)); 1372 1373 conf->reshape_safe = mddev->reshape_position; 1374 } 1375 1376 if (conf->pending_count >= max_queued_requests) { 1377 md_wakeup_thread(mddev->thread); 1378 raid10_log(mddev, "wait queued"); 1379 wait_event(conf->wait_barrier, 1380 conf->pending_count < max_queued_requests); 1381 } 1382 /* first select target devices under rcu_lock and 1383 * inc refcount on their rdev. Record them by setting 1384 * bios[x] to bio 1385 * If there are known/acknowledged bad blocks on any device 1386 * on which we have seen a write error, we want to avoid 1387 * writing to those blocks. This potentially requires several 1388 * writes to write around the bad blocks. Each set of writes 1389 * gets its own r10_bio with a set of bios attached. 1390 */ 1391 1392 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1393 raid10_find_phys(conf, r10_bio); 1394 retry_write: 1395 blocked_rdev = NULL; 1396 rcu_read_lock(); 1397 max_sectors = r10_bio->sectors; 1398 1399 for (i = 0; i < conf->copies; i++) { 1400 int d = r10_bio->devs[i].devnum; 1401 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 1402 struct md_rdev *rrdev = rcu_dereference( 1403 conf->mirrors[d].replacement); 1404 if (rdev == rrdev) 1405 rrdev = NULL; 1406 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1407 atomic_inc(&rdev->nr_pending); 1408 blocked_rdev = rdev; 1409 break; 1410 } 1411 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) { 1412 atomic_inc(&rrdev->nr_pending); 1413 blocked_rdev = rrdev; 1414 break; 1415 } 1416 if (rdev && (test_bit(Faulty, &rdev->flags))) 1417 rdev = NULL; 1418 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1419 rrdev = NULL; 1420 1421 r10_bio->devs[i].bio = NULL; 1422 r10_bio->devs[i].repl_bio = NULL; 1423 1424 if (!rdev && !rrdev) { 1425 set_bit(R10BIO_Degraded, &r10_bio->state); 1426 continue; 1427 } 1428 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1429 sector_t first_bad; 1430 sector_t dev_sector = r10_bio->devs[i].addr; 1431 int bad_sectors; 1432 int is_bad; 1433 1434 is_bad = is_badblock(rdev, dev_sector, max_sectors, 1435 &first_bad, &bad_sectors); 1436 if (is_bad < 0) { 1437 /* Mustn't write here until the bad block 1438 * is acknowledged 1439 */ 1440 atomic_inc(&rdev->nr_pending); 1441 set_bit(BlockedBadBlocks, &rdev->flags); 1442 blocked_rdev = rdev; 1443 break; 1444 } 1445 if (is_bad && first_bad <= dev_sector) { 1446 /* Cannot write here at all */ 1447 bad_sectors -= (dev_sector - first_bad); 1448 if (bad_sectors < max_sectors) 1449 /* Mustn't write more than bad_sectors 1450 * to other devices yet 1451 */ 1452 max_sectors = bad_sectors; 1453 /* We don't set R10BIO_Degraded as that 1454 * only applies if the disk is missing, 1455 * so it might be re-added, and we want to 1456 * know to recover this chunk. 1457 * In this case the device is here, and the 1458 * fact that this chunk is not in-sync is 1459 * recorded in the bad block log. 1460 */ 1461 continue; 1462 } 1463 if (is_bad) { 1464 int good_sectors = first_bad - dev_sector; 1465 if (good_sectors < max_sectors) 1466 max_sectors = good_sectors; 1467 } 1468 } 1469 if (rdev) { 1470 r10_bio->devs[i].bio = bio; 1471 atomic_inc(&rdev->nr_pending); 1472 } 1473 if (rrdev) { 1474 r10_bio->devs[i].repl_bio = bio; 1475 atomic_inc(&rrdev->nr_pending); 1476 } 1477 } 1478 rcu_read_unlock(); 1479 1480 if (unlikely(blocked_rdev)) { 1481 /* Have to wait for this device to get unblocked, then retry */ 1482 int j; 1483 int d; 1484 1485 for (j = 0; j < i; j++) { 1486 if (r10_bio->devs[j].bio) { 1487 d = r10_bio->devs[j].devnum; 1488 rdev_dec_pending(conf->mirrors[d].rdev, mddev); 1489 } 1490 if (r10_bio->devs[j].repl_bio) { 1491 struct md_rdev *rdev; 1492 d = r10_bio->devs[j].devnum; 1493 rdev = conf->mirrors[d].replacement; 1494 if (!rdev) { 1495 /* Race with remove_disk */ 1496 smp_mb(); 1497 rdev = conf->mirrors[d].rdev; 1498 } 1499 rdev_dec_pending(rdev, mddev); 1500 } 1501 } 1502 allow_barrier(conf); 1503 raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk); 1504 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1505 wait_barrier(conf); 1506 goto retry_write; 1507 } 1508 1509 if (max_sectors < r10_bio->sectors) 1510 r10_bio->sectors = max_sectors; 1511 1512 if (r10_bio->sectors < bio_sectors(bio)) { 1513 struct bio *split = bio_split(bio, r10_bio->sectors, 1514 GFP_NOIO, &conf->bio_split); 1515 bio_chain(split, bio); 1516 generic_make_request(bio); 1517 bio = split; 1518 r10_bio->master_bio = bio; 1519 } 1520 1521 atomic_set(&r10_bio->remaining, 1); 1522 md_bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0); 1523 1524 for (i = 0; i < conf->copies; i++) { 1525 if (r10_bio->devs[i].bio) 1526 raid10_write_one_disk(mddev, r10_bio, bio, false, i); 1527 if (r10_bio->devs[i].repl_bio) 1528 raid10_write_one_disk(mddev, r10_bio, bio, true, i); 1529 } 1530 one_write_done(r10_bio); 1531 } 1532 1533 static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) 1534 { 1535 struct r10conf *conf = mddev->private; 1536 struct r10bio *r10_bio; 1537 1538 r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); 1539 1540 r10_bio->master_bio = bio; 1541 r10_bio->sectors = sectors; 1542 1543 r10_bio->mddev = mddev; 1544 r10_bio->sector = bio->bi_iter.bi_sector; 1545 r10_bio->state = 0; 1546 memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->copies); 1547 1548 if (bio_data_dir(bio) == READ) 1549 raid10_read_request(mddev, bio, r10_bio); 1550 else 1551 raid10_write_request(mddev, bio, r10_bio); 1552 } 1553 1554 static bool raid10_make_request(struct mddev *mddev, struct bio *bio) 1555 { 1556 struct r10conf *conf = mddev->private; 1557 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1558 int chunk_sects = chunk_mask + 1; 1559 int sectors = bio_sectors(bio); 1560 1561 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1562 md_flush_request(mddev, bio); 1563 return true; 1564 } 1565 1566 if (!md_write_start(mddev, bio)) 1567 return false; 1568 1569 /* 1570 * If this request crosses a chunk boundary, we need to split 1571 * it. 1572 */ 1573 if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + 1574 sectors > chunk_sects 1575 && (conf->geo.near_copies < conf->geo.raid_disks 1576 || conf->prev.near_copies < 1577 conf->prev.raid_disks))) 1578 sectors = chunk_sects - 1579 (bio->bi_iter.bi_sector & 1580 (chunk_sects - 1)); 1581 __make_request(mddev, bio, sectors); 1582 1583 /* In case raid10d snuck in to freeze_array */ 1584 wake_up(&conf->wait_barrier); 1585 return true; 1586 } 1587 1588 static void raid10_status(struct seq_file *seq, struct mddev *mddev) 1589 { 1590 struct r10conf *conf = mddev->private; 1591 int i; 1592 1593 if (conf->geo.near_copies < conf->geo.raid_disks) 1594 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1595 if (conf->geo.near_copies > 1) 1596 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1597 if (conf->geo.far_copies > 1) { 1598 if (conf->geo.far_offset) 1599 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1600 else 1601 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1602 if (conf->geo.far_set_size != conf->geo.raid_disks) 1603 seq_printf(seq, " %d devices per set", conf->geo.far_set_size); 1604 } 1605 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1606 conf->geo.raid_disks - mddev->degraded); 1607 rcu_read_lock(); 1608 for (i = 0; i < conf->geo.raid_disks; i++) { 1609 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1610 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1611 } 1612 rcu_read_unlock(); 1613 seq_printf(seq, "]"); 1614 } 1615 1616 /* check if there are enough drives for 1617 * every block to appear on atleast one. 1618 * Don't consider the device numbered 'ignore' 1619 * as we might be about to remove it. 1620 */ 1621 static int _enough(struct r10conf *conf, int previous, int ignore) 1622 { 1623 int first = 0; 1624 int has_enough = 0; 1625 int disks, ncopies; 1626 if (previous) { 1627 disks = conf->prev.raid_disks; 1628 ncopies = conf->prev.near_copies; 1629 } else { 1630 disks = conf->geo.raid_disks; 1631 ncopies = conf->geo.near_copies; 1632 } 1633 1634 rcu_read_lock(); 1635 do { 1636 int n = conf->copies; 1637 int cnt = 0; 1638 int this = first; 1639 while (n--) { 1640 struct md_rdev *rdev; 1641 if (this != ignore && 1642 (rdev = rcu_dereference(conf->mirrors[this].rdev)) && 1643 test_bit(In_sync, &rdev->flags)) 1644 cnt++; 1645 this = (this+1) % disks; 1646 } 1647 if (cnt == 0) 1648 goto out; 1649 first = (first + ncopies) % disks; 1650 } while (first != 0); 1651 has_enough = 1; 1652 out: 1653 rcu_read_unlock(); 1654 return has_enough; 1655 } 1656 1657 static int enough(struct r10conf *conf, int ignore) 1658 { 1659 /* when calling 'enough', both 'prev' and 'geo' must 1660 * be stable. 1661 * This is ensured if ->reconfig_mutex or ->device_lock 1662 * is held. 1663 */ 1664 return _enough(conf, 0, ignore) && 1665 _enough(conf, 1, ignore); 1666 } 1667 1668 static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) 1669 { 1670 char b[BDEVNAME_SIZE]; 1671 struct r10conf *conf = mddev->private; 1672 unsigned long flags; 1673 1674 /* 1675 * If it is not operational, then we have already marked it as dead 1676 * else if it is the last working disks, ignore the error, let the 1677 * next level up know. 1678 * else mark the drive as failed 1679 */ 1680 spin_lock_irqsave(&conf->device_lock, flags); 1681 if (test_bit(In_sync, &rdev->flags) 1682 && !enough(conf, rdev->raid_disk)) { 1683 /* 1684 * Don't fail the drive, just return an IO error. 1685 */ 1686 spin_unlock_irqrestore(&conf->device_lock, flags); 1687 return; 1688 } 1689 if (test_and_clear_bit(In_sync, &rdev->flags)) 1690 mddev->degraded++; 1691 /* 1692 * If recovery is running, make sure it aborts. 1693 */ 1694 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1695 set_bit(Blocked, &rdev->flags); 1696 set_bit(Faulty, &rdev->flags); 1697 set_mask_bits(&mddev->sb_flags, 0, 1698 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1699 spin_unlock_irqrestore(&conf->device_lock, flags); 1700 pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n" 1701 "md/raid10:%s: Operation continuing on %d devices.\n", 1702 mdname(mddev), bdevname(rdev->bdev, b), 1703 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 1704 } 1705 1706 static void print_conf(struct r10conf *conf) 1707 { 1708 int i; 1709 struct md_rdev *rdev; 1710 1711 pr_debug("RAID10 conf printout:\n"); 1712 if (!conf) { 1713 pr_debug("(!conf)\n"); 1714 return; 1715 } 1716 pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 1717 conf->geo.raid_disks); 1718 1719 /* This is only called with ->reconfix_mutex held, so 1720 * rcu protection of rdev is not needed */ 1721 for (i = 0; i < conf->geo.raid_disks; i++) { 1722 char b[BDEVNAME_SIZE]; 1723 rdev = conf->mirrors[i].rdev; 1724 if (rdev) 1725 pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n", 1726 i, !test_bit(In_sync, &rdev->flags), 1727 !test_bit(Faulty, &rdev->flags), 1728 bdevname(rdev->bdev,b)); 1729 } 1730 } 1731 1732 static void close_sync(struct r10conf *conf) 1733 { 1734 wait_barrier(conf); 1735 allow_barrier(conf); 1736 1737 mempool_exit(&conf->r10buf_pool); 1738 } 1739 1740 static int raid10_spare_active(struct mddev *mddev) 1741 { 1742 int i; 1743 struct r10conf *conf = mddev->private; 1744 struct raid10_info *tmp; 1745 int count = 0; 1746 unsigned long flags; 1747 1748 /* 1749 * Find all non-in_sync disks within the RAID10 configuration 1750 * and mark them in_sync 1751 */ 1752 for (i = 0; i < conf->geo.raid_disks; i++) { 1753 tmp = conf->mirrors + i; 1754 if (tmp->replacement 1755 && tmp->replacement->recovery_offset == MaxSector 1756 && !test_bit(Faulty, &tmp->replacement->flags) 1757 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 1758 /* Replacement has just become active */ 1759 if (!tmp->rdev 1760 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 1761 count++; 1762 if (tmp->rdev) { 1763 /* Replaced device not technically faulty, 1764 * but we need to be sure it gets removed 1765 * and never re-added. 1766 */ 1767 set_bit(Faulty, &tmp->rdev->flags); 1768 sysfs_notify_dirent_safe( 1769 tmp->rdev->sysfs_state); 1770 } 1771 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 1772 } else if (tmp->rdev 1773 && tmp->rdev->recovery_offset == MaxSector 1774 && !test_bit(Faulty, &tmp->rdev->flags) 1775 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 1776 count++; 1777 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 1778 } 1779 } 1780 spin_lock_irqsave(&conf->device_lock, flags); 1781 mddev->degraded -= count; 1782 spin_unlock_irqrestore(&conf->device_lock, flags); 1783 1784 print_conf(conf); 1785 return count; 1786 } 1787 1788 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1789 { 1790 struct r10conf *conf = mddev->private; 1791 int err = -EEXIST; 1792 int mirror; 1793 int first = 0; 1794 int last = conf->geo.raid_disks - 1; 1795 1796 if (mddev->recovery_cp < MaxSector) 1797 /* only hot-add to in-sync arrays, as recovery is 1798 * very different from resync 1799 */ 1800 return -EBUSY; 1801 if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1)) 1802 return -EINVAL; 1803 1804 if (md_integrity_add_rdev(rdev, mddev)) 1805 return -ENXIO; 1806 1807 if (rdev->raid_disk >= 0) 1808 first = last = rdev->raid_disk; 1809 1810 if (rdev->saved_raid_disk >= first && 1811 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1812 mirror = rdev->saved_raid_disk; 1813 else 1814 mirror = first; 1815 for ( ; mirror <= last ; mirror++) { 1816 struct raid10_info *p = &conf->mirrors[mirror]; 1817 if (p->recovery_disabled == mddev->recovery_disabled) 1818 continue; 1819 if (p->rdev) { 1820 if (!test_bit(WantReplacement, &p->rdev->flags) || 1821 p->replacement != NULL) 1822 continue; 1823 clear_bit(In_sync, &rdev->flags); 1824 set_bit(Replacement, &rdev->flags); 1825 rdev->raid_disk = mirror; 1826 err = 0; 1827 if (mddev->gendisk) 1828 disk_stack_limits(mddev->gendisk, rdev->bdev, 1829 rdev->data_offset << 9); 1830 conf->fullsync = 1; 1831 rcu_assign_pointer(p->replacement, rdev); 1832 break; 1833 } 1834 1835 if (mddev->gendisk) 1836 disk_stack_limits(mddev->gendisk, rdev->bdev, 1837 rdev->data_offset << 9); 1838 1839 p->head_position = 0; 1840 p->recovery_disabled = mddev->recovery_disabled - 1; 1841 rdev->raid_disk = mirror; 1842 err = 0; 1843 if (rdev->saved_raid_disk != mirror) 1844 conf->fullsync = 1; 1845 rcu_assign_pointer(p->rdev, rdev); 1846 break; 1847 } 1848 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1849 blk_queue_flag_set(QUEUE_FLAG_DISCARD, mddev->queue); 1850 1851 print_conf(conf); 1852 return err; 1853 } 1854 1855 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1856 { 1857 struct r10conf *conf = mddev->private; 1858 int err = 0; 1859 int number = rdev->raid_disk; 1860 struct md_rdev **rdevp; 1861 struct raid10_info *p = conf->mirrors + number; 1862 1863 print_conf(conf); 1864 if (rdev == p->rdev) 1865 rdevp = &p->rdev; 1866 else if (rdev == p->replacement) 1867 rdevp = &p->replacement; 1868 else 1869 return 0; 1870 1871 if (test_bit(In_sync, &rdev->flags) || 1872 atomic_read(&rdev->nr_pending)) { 1873 err = -EBUSY; 1874 goto abort; 1875 } 1876 /* Only remove non-faulty devices if recovery 1877 * is not possible. 1878 */ 1879 if (!test_bit(Faulty, &rdev->flags) && 1880 mddev->recovery_disabled != p->recovery_disabled && 1881 (!p->replacement || p->replacement == rdev) && 1882 number < conf->geo.raid_disks && 1883 enough(conf, -1)) { 1884 err = -EBUSY; 1885 goto abort; 1886 } 1887 *rdevp = NULL; 1888 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 1889 synchronize_rcu(); 1890 if (atomic_read(&rdev->nr_pending)) { 1891 /* lost the race, try later */ 1892 err = -EBUSY; 1893 *rdevp = rdev; 1894 goto abort; 1895 } 1896 } 1897 if (p->replacement) { 1898 /* We must have just cleared 'rdev' */ 1899 p->rdev = p->replacement; 1900 clear_bit(Replacement, &p->replacement->flags); 1901 smp_mb(); /* Make sure other CPUs may see both as identical 1902 * but will never see neither -- if they are careful. 1903 */ 1904 p->replacement = NULL; 1905 } 1906 1907 clear_bit(WantReplacement, &rdev->flags); 1908 err = md_integrity_register(mddev); 1909 1910 abort: 1911 1912 print_conf(conf); 1913 return err; 1914 } 1915 1916 static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d) 1917 { 1918 struct r10conf *conf = r10_bio->mddev->private; 1919 1920 if (!bio->bi_status) 1921 set_bit(R10BIO_Uptodate, &r10_bio->state); 1922 else 1923 /* The write handler will notice the lack of 1924 * R10BIO_Uptodate and record any errors etc 1925 */ 1926 atomic_add(r10_bio->sectors, 1927 &conf->mirrors[d].rdev->corrected_errors); 1928 1929 /* for reconstruct, we always reschedule after a read. 1930 * for resync, only after all reads 1931 */ 1932 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 1933 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 1934 atomic_dec_and_test(&r10_bio->remaining)) { 1935 /* we have read all the blocks, 1936 * do the comparison in process context in raid10d 1937 */ 1938 reschedule_retry(r10_bio); 1939 } 1940 } 1941 1942 static void end_sync_read(struct bio *bio) 1943 { 1944 struct r10bio *r10_bio = get_resync_r10bio(bio); 1945 struct r10conf *conf = r10_bio->mddev->private; 1946 int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1947 1948 __end_sync_read(r10_bio, bio, d); 1949 } 1950 1951 static void end_reshape_read(struct bio *bio) 1952 { 1953 /* reshape read bio isn't allocated from r10buf_pool */ 1954 struct r10bio *r10_bio = bio->bi_private; 1955 1956 __end_sync_read(r10_bio, bio, r10_bio->read_slot); 1957 } 1958 1959 static void end_sync_request(struct r10bio *r10_bio) 1960 { 1961 struct mddev *mddev = r10_bio->mddev; 1962 1963 while (atomic_dec_and_test(&r10_bio->remaining)) { 1964 if (r10_bio->master_bio == NULL) { 1965 /* the primary of several recovery bios */ 1966 sector_t s = r10_bio->sectors; 1967 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1968 test_bit(R10BIO_WriteError, &r10_bio->state)) 1969 reschedule_retry(r10_bio); 1970 else 1971 put_buf(r10_bio); 1972 md_done_sync(mddev, s, 1); 1973 break; 1974 } else { 1975 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 1976 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1977 test_bit(R10BIO_WriteError, &r10_bio->state)) 1978 reschedule_retry(r10_bio); 1979 else 1980 put_buf(r10_bio); 1981 r10_bio = r10_bio2; 1982 } 1983 } 1984 } 1985 1986 static void end_sync_write(struct bio *bio) 1987 { 1988 struct r10bio *r10_bio = get_resync_r10bio(bio); 1989 struct mddev *mddev = r10_bio->mddev; 1990 struct r10conf *conf = mddev->private; 1991 int d; 1992 sector_t first_bad; 1993 int bad_sectors; 1994 int slot; 1995 int repl; 1996 struct md_rdev *rdev = NULL; 1997 1998 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1999 if (repl) 2000 rdev = conf->mirrors[d].replacement; 2001 else 2002 rdev = conf->mirrors[d].rdev; 2003 2004 if (bio->bi_status) { 2005 if (repl) 2006 md_error(mddev, rdev); 2007 else { 2008 set_bit(WriteErrorSeen, &rdev->flags); 2009 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2010 set_bit(MD_RECOVERY_NEEDED, 2011 &rdev->mddev->recovery); 2012 set_bit(R10BIO_WriteError, &r10_bio->state); 2013 } 2014 } else if (is_badblock(rdev, 2015 r10_bio->devs[slot].addr, 2016 r10_bio->sectors, 2017 &first_bad, &bad_sectors)) 2018 set_bit(R10BIO_MadeGood, &r10_bio->state); 2019 2020 rdev_dec_pending(rdev, mddev); 2021 2022 end_sync_request(r10_bio); 2023 } 2024 2025 /* 2026 * Note: sync and recover and handled very differently for raid10 2027 * This code is for resync. 2028 * For resync, we read through virtual addresses and read all blocks. 2029 * If there is any error, we schedule a write. The lowest numbered 2030 * drive is authoritative. 2031 * However requests come for physical address, so we need to map. 2032 * For every physical address there are raid_disks/copies virtual addresses, 2033 * which is always are least one, but is not necessarly an integer. 2034 * This means that a physical address can span multiple chunks, so we may 2035 * have to submit multiple io requests for a single sync request. 2036 */ 2037 /* 2038 * We check if all blocks are in-sync and only write to blocks that 2039 * aren't in sync 2040 */ 2041 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2042 { 2043 struct r10conf *conf = mddev->private; 2044 int i, first; 2045 struct bio *tbio, *fbio; 2046 int vcnt; 2047 struct page **tpages, **fpages; 2048 2049 atomic_set(&r10_bio->remaining, 1); 2050 2051 /* find the first device with a block */ 2052 for (i=0; i<conf->copies; i++) 2053 if (!r10_bio->devs[i].bio->bi_status) 2054 break; 2055 2056 if (i == conf->copies) 2057 goto done; 2058 2059 first = i; 2060 fbio = r10_bio->devs[i].bio; 2061 fbio->bi_iter.bi_size = r10_bio->sectors << 9; 2062 fbio->bi_iter.bi_idx = 0; 2063 fpages = get_resync_pages(fbio)->pages; 2064 2065 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 2066 /* now find blocks with errors */ 2067 for (i=0 ; i < conf->copies ; i++) { 2068 int j, d; 2069 struct md_rdev *rdev; 2070 struct resync_pages *rp; 2071 2072 tbio = r10_bio->devs[i].bio; 2073 2074 if (tbio->bi_end_io != end_sync_read) 2075 continue; 2076 if (i == first) 2077 continue; 2078 2079 tpages = get_resync_pages(tbio)->pages; 2080 d = r10_bio->devs[i].devnum; 2081 rdev = conf->mirrors[d].rdev; 2082 if (!r10_bio->devs[i].bio->bi_status) { 2083 /* We know that the bi_io_vec layout is the same for 2084 * both 'first' and 'i', so we just compare them. 2085 * All vec entries are PAGE_SIZE; 2086 */ 2087 int sectors = r10_bio->sectors; 2088 for (j = 0; j < vcnt; j++) { 2089 int len = PAGE_SIZE; 2090 if (sectors < (len / 512)) 2091 len = sectors * 512; 2092 if (memcmp(page_address(fpages[j]), 2093 page_address(tpages[j]), 2094 len)) 2095 break; 2096 sectors -= len/512; 2097 } 2098 if (j == vcnt) 2099 continue; 2100 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 2101 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 2102 /* Don't fix anything. */ 2103 continue; 2104 } else if (test_bit(FailFast, &rdev->flags)) { 2105 /* Just give up on this device */ 2106 md_error(rdev->mddev, rdev); 2107 continue; 2108 } 2109 /* Ok, we need to write this bio, either to correct an 2110 * inconsistency or to correct an unreadable block. 2111 * First we need to fixup bv_offset, bv_len and 2112 * bi_vecs, as the read request might have corrupted these 2113 */ 2114 rp = get_resync_pages(tbio); 2115 bio_reset(tbio); 2116 2117 md_bio_reset_resync_pages(tbio, rp, fbio->bi_iter.bi_size); 2118 2119 rp->raid_bio = r10_bio; 2120 tbio->bi_private = rp; 2121 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; 2122 tbio->bi_end_io = end_sync_write; 2123 bio_set_op_attrs(tbio, REQ_OP_WRITE, 0); 2124 2125 bio_copy_data(tbio, fbio); 2126 2127 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2128 atomic_inc(&r10_bio->remaining); 2129 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio)); 2130 2131 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags)) 2132 tbio->bi_opf |= MD_FAILFAST; 2133 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset; 2134 bio_set_dev(tbio, conf->mirrors[d].rdev->bdev); 2135 generic_make_request(tbio); 2136 } 2137 2138 /* Now write out to any replacement devices 2139 * that are active 2140 */ 2141 for (i = 0; i < conf->copies; i++) { 2142 int d; 2143 2144 tbio = r10_bio->devs[i].repl_bio; 2145 if (!tbio || !tbio->bi_end_io) 2146 continue; 2147 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2148 && r10_bio->devs[i].bio != fbio) 2149 bio_copy_data(tbio, fbio); 2150 d = r10_bio->devs[i].devnum; 2151 atomic_inc(&r10_bio->remaining); 2152 md_sync_acct(conf->mirrors[d].replacement->bdev, 2153 bio_sectors(tbio)); 2154 generic_make_request(tbio); 2155 } 2156 2157 done: 2158 if (atomic_dec_and_test(&r10_bio->remaining)) { 2159 md_done_sync(mddev, r10_bio->sectors, 1); 2160 put_buf(r10_bio); 2161 } 2162 } 2163 2164 /* 2165 * Now for the recovery code. 2166 * Recovery happens across physical sectors. 2167 * We recover all non-is_sync drives by finding the virtual address of 2168 * each, and then choose a working drive that also has that virt address. 2169 * There is a separate r10_bio for each non-in_sync drive. 2170 * Only the first two slots are in use. The first for reading, 2171 * The second for writing. 2172 * 2173 */ 2174 static void fix_recovery_read_error(struct r10bio *r10_bio) 2175 { 2176 /* We got a read error during recovery. 2177 * We repeat the read in smaller page-sized sections. 2178 * If a read succeeds, write it to the new device or record 2179 * a bad block if we cannot. 2180 * If a read fails, record a bad block on both old and 2181 * new devices. 2182 */ 2183 struct mddev *mddev = r10_bio->mddev; 2184 struct r10conf *conf = mddev->private; 2185 struct bio *bio = r10_bio->devs[0].bio; 2186 sector_t sect = 0; 2187 int sectors = r10_bio->sectors; 2188 int idx = 0; 2189 int dr = r10_bio->devs[0].devnum; 2190 int dw = r10_bio->devs[1].devnum; 2191 struct page **pages = get_resync_pages(bio)->pages; 2192 2193 while (sectors) { 2194 int s = sectors; 2195 struct md_rdev *rdev; 2196 sector_t addr; 2197 int ok; 2198 2199 if (s > (PAGE_SIZE>>9)) 2200 s = PAGE_SIZE >> 9; 2201 2202 rdev = conf->mirrors[dr].rdev; 2203 addr = r10_bio->devs[0].addr + sect, 2204 ok = sync_page_io(rdev, 2205 addr, 2206 s << 9, 2207 pages[idx], 2208 REQ_OP_READ, 0, false); 2209 if (ok) { 2210 rdev = conf->mirrors[dw].rdev; 2211 addr = r10_bio->devs[1].addr + sect; 2212 ok = sync_page_io(rdev, 2213 addr, 2214 s << 9, 2215 pages[idx], 2216 REQ_OP_WRITE, 0, false); 2217 if (!ok) { 2218 set_bit(WriteErrorSeen, &rdev->flags); 2219 if (!test_and_set_bit(WantReplacement, 2220 &rdev->flags)) 2221 set_bit(MD_RECOVERY_NEEDED, 2222 &rdev->mddev->recovery); 2223 } 2224 } 2225 if (!ok) { 2226 /* We don't worry if we cannot set a bad block - 2227 * it really is bad so there is no loss in not 2228 * recording it yet 2229 */ 2230 rdev_set_badblocks(rdev, addr, s, 0); 2231 2232 if (rdev != conf->mirrors[dw].rdev) { 2233 /* need bad block on destination too */ 2234 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2235 addr = r10_bio->devs[1].addr + sect; 2236 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2237 if (!ok) { 2238 /* just abort the recovery */ 2239 pr_notice("md/raid10:%s: recovery aborted due to read error\n", 2240 mdname(mddev)); 2241 2242 conf->mirrors[dw].recovery_disabled 2243 = mddev->recovery_disabled; 2244 set_bit(MD_RECOVERY_INTR, 2245 &mddev->recovery); 2246 break; 2247 } 2248 } 2249 } 2250 2251 sectors -= s; 2252 sect += s; 2253 idx++; 2254 } 2255 } 2256 2257 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2258 { 2259 struct r10conf *conf = mddev->private; 2260 int d; 2261 struct bio *wbio, *wbio2; 2262 2263 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2264 fix_recovery_read_error(r10_bio); 2265 end_sync_request(r10_bio); 2266 return; 2267 } 2268 2269 /* 2270 * share the pages with the first bio 2271 * and submit the write request 2272 */ 2273 d = r10_bio->devs[1].devnum; 2274 wbio = r10_bio->devs[1].bio; 2275 wbio2 = r10_bio->devs[1].repl_bio; 2276 /* Need to test wbio2->bi_end_io before we call 2277 * generic_make_request as if the former is NULL, 2278 * the latter is free to free wbio2. 2279 */ 2280 if (wbio2 && !wbio2->bi_end_io) 2281 wbio2 = NULL; 2282 if (wbio->bi_end_io) { 2283 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2284 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio)); 2285 generic_make_request(wbio); 2286 } 2287 if (wbio2) { 2288 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2289 md_sync_acct(conf->mirrors[d].replacement->bdev, 2290 bio_sectors(wbio2)); 2291 generic_make_request(wbio2); 2292 } 2293 } 2294 2295 /* 2296 * Used by fix_read_error() to decay the per rdev read_errors. 2297 * We halve the read error count for every hour that has elapsed 2298 * since the last recorded read error. 2299 * 2300 */ 2301 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2302 { 2303 long cur_time_mon; 2304 unsigned long hours_since_last; 2305 unsigned int read_errors = atomic_read(&rdev->read_errors); 2306 2307 cur_time_mon = ktime_get_seconds(); 2308 2309 if (rdev->last_read_error == 0) { 2310 /* first time we've seen a read error */ 2311 rdev->last_read_error = cur_time_mon; 2312 return; 2313 } 2314 2315 hours_since_last = (long)(cur_time_mon - 2316 rdev->last_read_error) / 3600; 2317 2318 rdev->last_read_error = cur_time_mon; 2319 2320 /* 2321 * if hours_since_last is > the number of bits in read_errors 2322 * just set read errors to 0. We do this to avoid 2323 * overflowing the shift of read_errors by hours_since_last. 2324 */ 2325 if (hours_since_last >= 8 * sizeof(read_errors)) 2326 atomic_set(&rdev->read_errors, 0); 2327 else 2328 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2329 } 2330 2331 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2332 int sectors, struct page *page, int rw) 2333 { 2334 sector_t first_bad; 2335 int bad_sectors; 2336 2337 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) 2338 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) 2339 return -1; 2340 if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) 2341 /* success */ 2342 return 1; 2343 if (rw == WRITE) { 2344 set_bit(WriteErrorSeen, &rdev->flags); 2345 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2346 set_bit(MD_RECOVERY_NEEDED, 2347 &rdev->mddev->recovery); 2348 } 2349 /* need to record an error - either for the block or the device */ 2350 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2351 md_error(rdev->mddev, rdev); 2352 return 0; 2353 } 2354 2355 /* 2356 * This is a kernel thread which: 2357 * 2358 * 1. Retries failed read operations on working mirrors. 2359 * 2. Updates the raid superblock when problems encounter. 2360 * 3. Performs writes following reads for array synchronising. 2361 */ 2362 2363 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2364 { 2365 int sect = 0; /* Offset from r10_bio->sector */ 2366 int sectors = r10_bio->sectors; 2367 struct md_rdev *rdev; 2368 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2369 int d = r10_bio->devs[r10_bio->read_slot].devnum; 2370 2371 /* still own a reference to this rdev, so it cannot 2372 * have been cleared recently. 2373 */ 2374 rdev = conf->mirrors[d].rdev; 2375 2376 if (test_bit(Faulty, &rdev->flags)) 2377 /* drive has already been failed, just ignore any 2378 more fix_read_error() attempts */ 2379 return; 2380 2381 check_decay_read_errors(mddev, rdev); 2382 atomic_inc(&rdev->read_errors); 2383 if (atomic_read(&rdev->read_errors) > max_read_errors) { 2384 char b[BDEVNAME_SIZE]; 2385 bdevname(rdev->bdev, b); 2386 2387 pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n", 2388 mdname(mddev), b, 2389 atomic_read(&rdev->read_errors), max_read_errors); 2390 pr_notice("md/raid10:%s: %s: Failing raid device\n", 2391 mdname(mddev), b); 2392 md_error(mddev, rdev); 2393 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED; 2394 return; 2395 } 2396 2397 while(sectors) { 2398 int s = sectors; 2399 int sl = r10_bio->read_slot; 2400 int success = 0; 2401 int start; 2402 2403 if (s > (PAGE_SIZE>>9)) 2404 s = PAGE_SIZE >> 9; 2405 2406 rcu_read_lock(); 2407 do { 2408 sector_t first_bad; 2409 int bad_sectors; 2410 2411 d = r10_bio->devs[sl].devnum; 2412 rdev = rcu_dereference(conf->mirrors[d].rdev); 2413 if (rdev && 2414 test_bit(In_sync, &rdev->flags) && 2415 !test_bit(Faulty, &rdev->flags) && 2416 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2417 &first_bad, &bad_sectors) == 0) { 2418 atomic_inc(&rdev->nr_pending); 2419 rcu_read_unlock(); 2420 success = sync_page_io(rdev, 2421 r10_bio->devs[sl].addr + 2422 sect, 2423 s<<9, 2424 conf->tmppage, 2425 REQ_OP_READ, 0, false); 2426 rdev_dec_pending(rdev, mddev); 2427 rcu_read_lock(); 2428 if (success) 2429 break; 2430 } 2431 sl++; 2432 if (sl == conf->copies) 2433 sl = 0; 2434 } while (!success && sl != r10_bio->read_slot); 2435 rcu_read_unlock(); 2436 2437 if (!success) { 2438 /* Cannot read from anywhere, just mark the block 2439 * as bad on the first device to discourage future 2440 * reads. 2441 */ 2442 int dn = r10_bio->devs[r10_bio->read_slot].devnum; 2443 rdev = conf->mirrors[dn].rdev; 2444 2445 if (!rdev_set_badblocks( 2446 rdev, 2447 r10_bio->devs[r10_bio->read_slot].addr 2448 + sect, 2449 s, 0)) { 2450 md_error(mddev, rdev); 2451 r10_bio->devs[r10_bio->read_slot].bio 2452 = IO_BLOCKED; 2453 } 2454 break; 2455 } 2456 2457 start = sl; 2458 /* write it back and re-read */ 2459 rcu_read_lock(); 2460 while (sl != r10_bio->read_slot) { 2461 char b[BDEVNAME_SIZE]; 2462 2463 if (sl==0) 2464 sl = conf->copies; 2465 sl--; 2466 d = r10_bio->devs[sl].devnum; 2467 rdev = rcu_dereference(conf->mirrors[d].rdev); 2468 if (!rdev || 2469 test_bit(Faulty, &rdev->flags) || 2470 !test_bit(In_sync, &rdev->flags)) 2471 continue; 2472 2473 atomic_inc(&rdev->nr_pending); 2474 rcu_read_unlock(); 2475 if (r10_sync_page_io(rdev, 2476 r10_bio->devs[sl].addr + 2477 sect, 2478 s, conf->tmppage, WRITE) 2479 == 0) { 2480 /* Well, this device is dead */ 2481 pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n", 2482 mdname(mddev), s, 2483 (unsigned long long)( 2484 sect + 2485 choose_data_offset(r10_bio, 2486 rdev)), 2487 bdevname(rdev->bdev, b)); 2488 pr_notice("md/raid10:%s: %s: failing drive\n", 2489 mdname(mddev), 2490 bdevname(rdev->bdev, b)); 2491 } 2492 rdev_dec_pending(rdev, mddev); 2493 rcu_read_lock(); 2494 } 2495 sl = start; 2496 while (sl != r10_bio->read_slot) { 2497 char b[BDEVNAME_SIZE]; 2498 2499 if (sl==0) 2500 sl = conf->copies; 2501 sl--; 2502 d = r10_bio->devs[sl].devnum; 2503 rdev = rcu_dereference(conf->mirrors[d].rdev); 2504 if (!rdev || 2505 test_bit(Faulty, &rdev->flags) || 2506 !test_bit(In_sync, &rdev->flags)) 2507 continue; 2508 2509 atomic_inc(&rdev->nr_pending); 2510 rcu_read_unlock(); 2511 switch (r10_sync_page_io(rdev, 2512 r10_bio->devs[sl].addr + 2513 sect, 2514 s, conf->tmppage, 2515 READ)) { 2516 case 0: 2517 /* Well, this device is dead */ 2518 pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n", 2519 mdname(mddev), s, 2520 (unsigned long long)( 2521 sect + 2522 choose_data_offset(r10_bio, rdev)), 2523 bdevname(rdev->bdev, b)); 2524 pr_notice("md/raid10:%s: %s: failing drive\n", 2525 mdname(mddev), 2526 bdevname(rdev->bdev, b)); 2527 break; 2528 case 1: 2529 pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n", 2530 mdname(mddev), s, 2531 (unsigned long long)( 2532 sect + 2533 choose_data_offset(r10_bio, rdev)), 2534 bdevname(rdev->bdev, b)); 2535 atomic_add(s, &rdev->corrected_errors); 2536 } 2537 2538 rdev_dec_pending(rdev, mddev); 2539 rcu_read_lock(); 2540 } 2541 rcu_read_unlock(); 2542 2543 sectors -= s; 2544 sect += s; 2545 } 2546 } 2547 2548 static int narrow_write_error(struct r10bio *r10_bio, int i) 2549 { 2550 struct bio *bio = r10_bio->master_bio; 2551 struct mddev *mddev = r10_bio->mddev; 2552 struct r10conf *conf = mddev->private; 2553 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2554 /* bio has the data to be written to slot 'i' where 2555 * we just recently had a write error. 2556 * We repeatedly clone the bio and trim down to one block, 2557 * then try the write. Where the write fails we record 2558 * a bad block. 2559 * It is conceivable that the bio doesn't exactly align with 2560 * blocks. We must handle this. 2561 * 2562 * We currently own a reference to the rdev. 2563 */ 2564 2565 int block_sectors; 2566 sector_t sector; 2567 int sectors; 2568 int sect_to_write = r10_bio->sectors; 2569 int ok = 1; 2570 2571 if (rdev->badblocks.shift < 0) 2572 return 0; 2573 2574 block_sectors = roundup(1 << rdev->badblocks.shift, 2575 bdev_logical_block_size(rdev->bdev) >> 9); 2576 sector = r10_bio->sector; 2577 sectors = ((r10_bio->sector + block_sectors) 2578 & ~(sector_t)(block_sectors - 1)) 2579 - sector; 2580 2581 while (sect_to_write) { 2582 struct bio *wbio; 2583 sector_t wsector; 2584 if (sectors > sect_to_write) 2585 sectors = sect_to_write; 2586 /* Write at 'sector' for 'sectors' */ 2587 wbio = bio_clone_fast(bio, GFP_NOIO, &mddev->bio_set); 2588 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors); 2589 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector); 2590 wbio->bi_iter.bi_sector = wsector + 2591 choose_data_offset(r10_bio, rdev); 2592 bio_set_dev(wbio, rdev->bdev); 2593 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); 2594 2595 if (submit_bio_wait(wbio) < 0) 2596 /* Failure! */ 2597 ok = rdev_set_badblocks(rdev, wsector, 2598 sectors, 0) 2599 && ok; 2600 2601 bio_put(wbio); 2602 sect_to_write -= sectors; 2603 sector += sectors; 2604 sectors = block_sectors; 2605 } 2606 return ok; 2607 } 2608 2609 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2610 { 2611 int slot = r10_bio->read_slot; 2612 struct bio *bio; 2613 struct r10conf *conf = mddev->private; 2614 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2615 2616 /* we got a read error. Maybe the drive is bad. Maybe just 2617 * the block and we can fix it. 2618 * We freeze all other IO, and try reading the block from 2619 * other devices. When we find one, we re-write 2620 * and check it that fixes the read error. 2621 * This is all done synchronously while the array is 2622 * frozen. 2623 */ 2624 bio = r10_bio->devs[slot].bio; 2625 bio_put(bio); 2626 r10_bio->devs[slot].bio = NULL; 2627 2628 if (mddev->ro) 2629 r10_bio->devs[slot].bio = IO_BLOCKED; 2630 else if (!test_bit(FailFast, &rdev->flags)) { 2631 freeze_array(conf, 1); 2632 fix_read_error(conf, mddev, r10_bio); 2633 unfreeze_array(conf); 2634 } else 2635 md_error(mddev, rdev); 2636 2637 rdev_dec_pending(rdev, mddev); 2638 allow_barrier(conf); 2639 r10_bio->state = 0; 2640 raid10_read_request(mddev, r10_bio->master_bio, r10_bio); 2641 } 2642 2643 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2644 { 2645 /* Some sort of write request has finished and it 2646 * succeeded in writing where we thought there was a 2647 * bad block. So forget the bad block. 2648 * Or possibly if failed and we need to record 2649 * a bad block. 2650 */ 2651 int m; 2652 struct md_rdev *rdev; 2653 2654 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2655 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2656 for (m = 0; m < conf->copies; m++) { 2657 int dev = r10_bio->devs[m].devnum; 2658 rdev = conf->mirrors[dev].rdev; 2659 if (r10_bio->devs[m].bio == NULL || 2660 r10_bio->devs[m].bio->bi_end_io == NULL) 2661 continue; 2662 if (!r10_bio->devs[m].bio->bi_status) { 2663 rdev_clear_badblocks( 2664 rdev, 2665 r10_bio->devs[m].addr, 2666 r10_bio->sectors, 0); 2667 } else { 2668 if (!rdev_set_badblocks( 2669 rdev, 2670 r10_bio->devs[m].addr, 2671 r10_bio->sectors, 0)) 2672 md_error(conf->mddev, rdev); 2673 } 2674 rdev = conf->mirrors[dev].replacement; 2675 if (r10_bio->devs[m].repl_bio == NULL || 2676 r10_bio->devs[m].repl_bio->bi_end_io == NULL) 2677 continue; 2678 2679 if (!r10_bio->devs[m].repl_bio->bi_status) { 2680 rdev_clear_badblocks( 2681 rdev, 2682 r10_bio->devs[m].addr, 2683 r10_bio->sectors, 0); 2684 } else { 2685 if (!rdev_set_badblocks( 2686 rdev, 2687 r10_bio->devs[m].addr, 2688 r10_bio->sectors, 0)) 2689 md_error(conf->mddev, rdev); 2690 } 2691 } 2692 put_buf(r10_bio); 2693 } else { 2694 bool fail = false; 2695 for (m = 0; m < conf->copies; m++) { 2696 int dev = r10_bio->devs[m].devnum; 2697 struct bio *bio = r10_bio->devs[m].bio; 2698 rdev = conf->mirrors[dev].rdev; 2699 if (bio == IO_MADE_GOOD) { 2700 rdev_clear_badblocks( 2701 rdev, 2702 r10_bio->devs[m].addr, 2703 r10_bio->sectors, 0); 2704 rdev_dec_pending(rdev, conf->mddev); 2705 } else if (bio != NULL && bio->bi_status) { 2706 fail = true; 2707 if (!narrow_write_error(r10_bio, m)) { 2708 md_error(conf->mddev, rdev); 2709 set_bit(R10BIO_Degraded, 2710 &r10_bio->state); 2711 } 2712 rdev_dec_pending(rdev, conf->mddev); 2713 } 2714 bio = r10_bio->devs[m].repl_bio; 2715 rdev = conf->mirrors[dev].replacement; 2716 if (rdev && bio == IO_MADE_GOOD) { 2717 rdev_clear_badblocks( 2718 rdev, 2719 r10_bio->devs[m].addr, 2720 r10_bio->sectors, 0); 2721 rdev_dec_pending(rdev, conf->mddev); 2722 } 2723 } 2724 if (fail) { 2725 spin_lock_irq(&conf->device_lock); 2726 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); 2727 conf->nr_queued++; 2728 spin_unlock_irq(&conf->device_lock); 2729 /* 2730 * In case freeze_array() is waiting for condition 2731 * nr_pending == nr_queued + extra to be true. 2732 */ 2733 wake_up(&conf->wait_barrier); 2734 md_wakeup_thread(conf->mddev->thread); 2735 } else { 2736 if (test_bit(R10BIO_WriteError, 2737 &r10_bio->state)) 2738 close_write(r10_bio); 2739 raid_end_bio_io(r10_bio); 2740 } 2741 } 2742 } 2743 2744 static void raid10d(struct md_thread *thread) 2745 { 2746 struct mddev *mddev = thread->mddev; 2747 struct r10bio *r10_bio; 2748 unsigned long flags; 2749 struct r10conf *conf = mddev->private; 2750 struct list_head *head = &conf->retry_list; 2751 struct blk_plug plug; 2752 2753 md_check_recovery(mddev); 2754 2755 if (!list_empty_careful(&conf->bio_end_io_list) && 2756 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2757 LIST_HEAD(tmp); 2758 spin_lock_irqsave(&conf->device_lock, flags); 2759 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2760 while (!list_empty(&conf->bio_end_io_list)) { 2761 list_move(conf->bio_end_io_list.prev, &tmp); 2762 conf->nr_queued--; 2763 } 2764 } 2765 spin_unlock_irqrestore(&conf->device_lock, flags); 2766 while (!list_empty(&tmp)) { 2767 r10_bio = list_first_entry(&tmp, struct r10bio, 2768 retry_list); 2769 list_del(&r10_bio->retry_list); 2770 if (mddev->degraded) 2771 set_bit(R10BIO_Degraded, &r10_bio->state); 2772 2773 if (test_bit(R10BIO_WriteError, 2774 &r10_bio->state)) 2775 close_write(r10_bio); 2776 raid_end_bio_io(r10_bio); 2777 } 2778 } 2779 2780 blk_start_plug(&plug); 2781 for (;;) { 2782 2783 flush_pending_writes(conf); 2784 2785 spin_lock_irqsave(&conf->device_lock, flags); 2786 if (list_empty(head)) { 2787 spin_unlock_irqrestore(&conf->device_lock, flags); 2788 break; 2789 } 2790 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 2791 list_del(head->prev); 2792 conf->nr_queued--; 2793 spin_unlock_irqrestore(&conf->device_lock, flags); 2794 2795 mddev = r10_bio->mddev; 2796 conf = mddev->private; 2797 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2798 test_bit(R10BIO_WriteError, &r10_bio->state)) 2799 handle_write_completed(conf, r10_bio); 2800 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 2801 reshape_request_write(mddev, r10_bio); 2802 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 2803 sync_request_write(mddev, r10_bio); 2804 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 2805 recovery_request_write(mddev, r10_bio); 2806 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 2807 handle_read_error(mddev, r10_bio); 2808 else 2809 WARN_ON_ONCE(1); 2810 2811 cond_resched(); 2812 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING)) 2813 md_check_recovery(mddev); 2814 } 2815 blk_finish_plug(&plug); 2816 } 2817 2818 static int init_resync(struct r10conf *conf) 2819 { 2820 int ret, buffs, i; 2821 2822 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2823 BUG_ON(mempool_initialized(&conf->r10buf_pool)); 2824 conf->have_replacement = 0; 2825 for (i = 0; i < conf->geo.raid_disks; i++) 2826 if (conf->mirrors[i].replacement) 2827 conf->have_replacement = 1; 2828 ret = mempool_init(&conf->r10buf_pool, buffs, 2829 r10buf_pool_alloc, r10buf_pool_free, conf); 2830 if (ret) 2831 return ret; 2832 conf->next_resync = 0; 2833 return 0; 2834 } 2835 2836 static struct r10bio *raid10_alloc_init_r10buf(struct r10conf *conf) 2837 { 2838 struct r10bio *r10bio = mempool_alloc(&conf->r10buf_pool, GFP_NOIO); 2839 struct rsync_pages *rp; 2840 struct bio *bio; 2841 int nalloc; 2842 int i; 2843 2844 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 2845 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 2846 nalloc = conf->copies; /* resync */ 2847 else 2848 nalloc = 2; /* recovery */ 2849 2850 for (i = 0; i < nalloc; i++) { 2851 bio = r10bio->devs[i].bio; 2852 rp = bio->bi_private; 2853 bio_reset(bio); 2854 bio->bi_private = rp; 2855 bio = r10bio->devs[i].repl_bio; 2856 if (bio) { 2857 rp = bio->bi_private; 2858 bio_reset(bio); 2859 bio->bi_private = rp; 2860 } 2861 } 2862 return r10bio; 2863 } 2864 2865 /* 2866 * Set cluster_sync_high since we need other nodes to add the 2867 * range [cluster_sync_low, cluster_sync_high] to suspend list. 2868 */ 2869 static void raid10_set_cluster_sync_high(struct r10conf *conf) 2870 { 2871 sector_t window_size; 2872 int extra_chunk, chunks; 2873 2874 /* 2875 * First, here we define "stripe" as a unit which across 2876 * all member devices one time, so we get chunks by use 2877 * raid_disks / near_copies. Otherwise, if near_copies is 2878 * close to raid_disks, then resync window could increases 2879 * linearly with the increase of raid_disks, which means 2880 * we will suspend a really large IO window while it is not 2881 * necessary. If raid_disks is not divisible by near_copies, 2882 * an extra chunk is needed to ensure the whole "stripe" is 2883 * covered. 2884 */ 2885 2886 chunks = conf->geo.raid_disks / conf->geo.near_copies; 2887 if (conf->geo.raid_disks % conf->geo.near_copies == 0) 2888 extra_chunk = 0; 2889 else 2890 extra_chunk = 1; 2891 window_size = (chunks + extra_chunk) * conf->mddev->chunk_sectors; 2892 2893 /* 2894 * At least use a 32M window to align with raid1's resync window 2895 */ 2896 window_size = (CLUSTER_RESYNC_WINDOW_SECTORS > window_size) ? 2897 CLUSTER_RESYNC_WINDOW_SECTORS : window_size; 2898 2899 conf->cluster_sync_high = conf->cluster_sync_low + window_size; 2900 } 2901 2902 /* 2903 * perform a "sync" on one "block" 2904 * 2905 * We need to make sure that no normal I/O request - particularly write 2906 * requests - conflict with active sync requests. 2907 * 2908 * This is achieved by tracking pending requests and a 'barrier' concept 2909 * that can be installed to exclude normal IO requests. 2910 * 2911 * Resync and recovery are handled very differently. 2912 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 2913 * 2914 * For resync, we iterate over virtual addresses, read all copies, 2915 * and update if there are differences. If only one copy is live, 2916 * skip it. 2917 * For recovery, we iterate over physical addresses, read a good 2918 * value for each non-in_sync drive, and over-write. 2919 * 2920 * So, for recovery we may have several outstanding complex requests for a 2921 * given address, one for each out-of-sync device. We model this by allocating 2922 * a number of r10_bio structures, one for each out-of-sync device. 2923 * As we setup these structures, we collect all bio's together into a list 2924 * which we then process collectively to add pages, and then process again 2925 * to pass to generic_make_request. 2926 * 2927 * The r10_bio structures are linked using a borrowed master_bio pointer. 2928 * This link is counted in ->remaining. When the r10_bio that points to NULL 2929 * has its remaining count decremented to 0, the whole complex operation 2930 * is complete. 2931 * 2932 */ 2933 2934 static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, 2935 int *skipped) 2936 { 2937 struct r10conf *conf = mddev->private; 2938 struct r10bio *r10_bio; 2939 struct bio *biolist = NULL, *bio; 2940 sector_t max_sector, nr_sectors; 2941 int i; 2942 int max_sync; 2943 sector_t sync_blocks; 2944 sector_t sectors_skipped = 0; 2945 int chunks_skipped = 0; 2946 sector_t chunk_mask = conf->geo.chunk_mask; 2947 int page_idx = 0; 2948 2949 if (!mempool_initialized(&conf->r10buf_pool)) 2950 if (init_resync(conf)) 2951 return 0; 2952 2953 /* 2954 * Allow skipping a full rebuild for incremental assembly 2955 * of a clean array, like RAID1 does. 2956 */ 2957 if (mddev->bitmap == NULL && 2958 mddev->recovery_cp == MaxSector && 2959 mddev->reshape_position == MaxSector && 2960 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2961 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2962 !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 2963 conf->fullsync == 0) { 2964 *skipped = 1; 2965 return mddev->dev_sectors - sector_nr; 2966 } 2967 2968 skipped: 2969 max_sector = mddev->dev_sectors; 2970 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2971 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2972 max_sector = mddev->resync_max_sectors; 2973 if (sector_nr >= max_sector) { 2974 conf->cluster_sync_low = 0; 2975 conf->cluster_sync_high = 0; 2976 2977 /* If we aborted, we need to abort the 2978 * sync on the 'current' bitmap chucks (there can 2979 * be several when recovering multiple devices). 2980 * as we may have started syncing it but not finished. 2981 * We can find the current address in 2982 * mddev->curr_resync, but for recovery, 2983 * we need to convert that to several 2984 * virtual addresses. 2985 */ 2986 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 2987 end_reshape(conf); 2988 close_sync(conf); 2989 return 0; 2990 } 2991 2992 if (mddev->curr_resync < max_sector) { /* aborted */ 2993 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 2994 md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2995 &sync_blocks, 1); 2996 else for (i = 0; i < conf->geo.raid_disks; i++) { 2997 sector_t sect = 2998 raid10_find_virt(conf, mddev->curr_resync, i); 2999 md_bitmap_end_sync(mddev->bitmap, sect, 3000 &sync_blocks, 1); 3001 } 3002 } else { 3003 /* completed sync */ 3004 if ((!mddev->bitmap || conf->fullsync) 3005 && conf->have_replacement 3006 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3007 /* Completed a full sync so the replacements 3008 * are now fully recovered. 3009 */ 3010 rcu_read_lock(); 3011 for (i = 0; i < conf->geo.raid_disks; i++) { 3012 struct md_rdev *rdev = 3013 rcu_dereference(conf->mirrors[i].replacement); 3014 if (rdev) 3015 rdev->recovery_offset = MaxSector; 3016 } 3017 rcu_read_unlock(); 3018 } 3019 conf->fullsync = 0; 3020 } 3021 md_bitmap_close_sync(mddev->bitmap); 3022 close_sync(conf); 3023 *skipped = 1; 3024 return sectors_skipped; 3025 } 3026 3027 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 3028 return reshape_request(mddev, sector_nr, skipped); 3029 3030 if (chunks_skipped >= conf->geo.raid_disks) { 3031 /* if there has been nothing to do on any drive, 3032 * then there is nothing to do at all.. 3033 */ 3034 *skipped = 1; 3035 return (max_sector - sector_nr) + sectors_skipped; 3036 } 3037 3038 if (max_sector > mddev->resync_max) 3039 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 3040 3041 /* make sure whole request will fit in a chunk - if chunks 3042 * are meaningful 3043 */ 3044 if (conf->geo.near_copies < conf->geo.raid_disks && 3045 max_sector > (sector_nr | chunk_mask)) 3046 max_sector = (sector_nr | chunk_mask) + 1; 3047 3048 /* 3049 * If there is non-resync activity waiting for a turn, then let it 3050 * though before starting on this new sync request. 3051 */ 3052 if (conf->nr_waiting) 3053 schedule_timeout_uninterruptible(1); 3054 3055 /* Again, very different code for resync and recovery. 3056 * Both must result in an r10bio with a list of bios that 3057 * have bi_end_io, bi_sector, bi_disk set, 3058 * and bi_private set to the r10bio. 3059 * For recovery, we may actually create several r10bios 3060 * with 2 bios in each, that correspond to the bios in the main one. 3061 * In this case, the subordinate r10bios link back through a 3062 * borrowed master_bio pointer, and the counter in the master 3063 * includes a ref from each subordinate. 3064 */ 3065 /* First, we decide what to do and set ->bi_end_io 3066 * To end_sync_read if we want to read, and 3067 * end_sync_write if we will want to write. 3068 */ 3069 3070 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 3071 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3072 /* recovery... the complicated one */ 3073 int j; 3074 r10_bio = NULL; 3075 3076 for (i = 0 ; i < conf->geo.raid_disks; i++) { 3077 int still_degraded; 3078 struct r10bio *rb2; 3079 sector_t sect; 3080 int must_sync; 3081 int any_working; 3082 struct raid10_info *mirror = &conf->mirrors[i]; 3083 struct md_rdev *mrdev, *mreplace; 3084 3085 rcu_read_lock(); 3086 mrdev = rcu_dereference(mirror->rdev); 3087 mreplace = rcu_dereference(mirror->replacement); 3088 3089 if ((mrdev == NULL || 3090 test_bit(Faulty, &mrdev->flags) || 3091 test_bit(In_sync, &mrdev->flags)) && 3092 (mreplace == NULL || 3093 test_bit(Faulty, &mreplace->flags))) { 3094 rcu_read_unlock(); 3095 continue; 3096 } 3097 3098 still_degraded = 0; 3099 /* want to reconstruct this device */ 3100 rb2 = r10_bio; 3101 sect = raid10_find_virt(conf, sector_nr, i); 3102 if (sect >= mddev->resync_max_sectors) { 3103 /* last stripe is not complete - don't 3104 * try to recover this sector. 3105 */ 3106 rcu_read_unlock(); 3107 continue; 3108 } 3109 if (mreplace && test_bit(Faulty, &mreplace->flags)) 3110 mreplace = NULL; 3111 /* Unless we are doing a full sync, or a replacement 3112 * we only need to recover the block if it is set in 3113 * the bitmap 3114 */ 3115 must_sync = md_bitmap_start_sync(mddev->bitmap, sect, 3116 &sync_blocks, 1); 3117 if (sync_blocks < max_sync) 3118 max_sync = sync_blocks; 3119 if (!must_sync && 3120 mreplace == NULL && 3121 !conf->fullsync) { 3122 /* yep, skip the sync_blocks here, but don't assume 3123 * that there will never be anything to do here 3124 */ 3125 chunks_skipped = -1; 3126 rcu_read_unlock(); 3127 continue; 3128 } 3129 atomic_inc(&mrdev->nr_pending); 3130 if (mreplace) 3131 atomic_inc(&mreplace->nr_pending); 3132 rcu_read_unlock(); 3133 3134 r10_bio = raid10_alloc_init_r10buf(conf); 3135 r10_bio->state = 0; 3136 raise_barrier(conf, rb2 != NULL); 3137 atomic_set(&r10_bio->remaining, 0); 3138 3139 r10_bio->master_bio = (struct bio*)rb2; 3140 if (rb2) 3141 atomic_inc(&rb2->remaining); 3142 r10_bio->mddev = mddev; 3143 set_bit(R10BIO_IsRecover, &r10_bio->state); 3144 r10_bio->sector = sect; 3145 3146 raid10_find_phys(conf, r10_bio); 3147 3148 /* Need to check if the array will still be 3149 * degraded 3150 */ 3151 rcu_read_lock(); 3152 for (j = 0; j < conf->geo.raid_disks; j++) { 3153 struct md_rdev *rdev = rcu_dereference( 3154 conf->mirrors[j].rdev); 3155 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3156 still_degraded = 1; 3157 break; 3158 } 3159 } 3160 3161 must_sync = md_bitmap_start_sync(mddev->bitmap, sect, 3162 &sync_blocks, still_degraded); 3163 3164 any_working = 0; 3165 for (j=0; j<conf->copies;j++) { 3166 int k; 3167 int d = r10_bio->devs[j].devnum; 3168 sector_t from_addr, to_addr; 3169 struct md_rdev *rdev = 3170 rcu_dereference(conf->mirrors[d].rdev); 3171 sector_t sector, first_bad; 3172 int bad_sectors; 3173 if (!rdev || 3174 !test_bit(In_sync, &rdev->flags)) 3175 continue; 3176 /* This is where we read from */ 3177 any_working = 1; 3178 sector = r10_bio->devs[j].addr; 3179 3180 if (is_badblock(rdev, sector, max_sync, 3181 &first_bad, &bad_sectors)) { 3182 if (first_bad > sector) 3183 max_sync = first_bad - sector; 3184 else { 3185 bad_sectors -= (sector 3186 - first_bad); 3187 if (max_sync > bad_sectors) 3188 max_sync = bad_sectors; 3189 continue; 3190 } 3191 } 3192 bio = r10_bio->devs[0].bio; 3193 bio->bi_next = biolist; 3194 biolist = bio; 3195 bio->bi_end_io = end_sync_read; 3196 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3197 if (test_bit(FailFast, &rdev->flags)) 3198 bio->bi_opf |= MD_FAILFAST; 3199 from_addr = r10_bio->devs[j].addr; 3200 bio->bi_iter.bi_sector = from_addr + 3201 rdev->data_offset; 3202 bio_set_dev(bio, rdev->bdev); 3203 atomic_inc(&rdev->nr_pending); 3204 /* and we write to 'i' (if not in_sync) */ 3205 3206 for (k=0; k<conf->copies; k++) 3207 if (r10_bio->devs[k].devnum == i) 3208 break; 3209 BUG_ON(k == conf->copies); 3210 to_addr = r10_bio->devs[k].addr; 3211 r10_bio->devs[0].devnum = d; 3212 r10_bio->devs[0].addr = from_addr; 3213 r10_bio->devs[1].devnum = i; 3214 r10_bio->devs[1].addr = to_addr; 3215 3216 if (!test_bit(In_sync, &mrdev->flags)) { 3217 bio = r10_bio->devs[1].bio; 3218 bio->bi_next = biolist; 3219 biolist = bio; 3220 bio->bi_end_io = end_sync_write; 3221 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3222 bio->bi_iter.bi_sector = to_addr 3223 + mrdev->data_offset; 3224 bio_set_dev(bio, mrdev->bdev); 3225 atomic_inc(&r10_bio->remaining); 3226 } else 3227 r10_bio->devs[1].bio->bi_end_io = NULL; 3228 3229 /* and maybe write to replacement */ 3230 bio = r10_bio->devs[1].repl_bio; 3231 if (bio) 3232 bio->bi_end_io = NULL; 3233 /* Note: if mreplace != NULL, then bio 3234 * cannot be NULL as r10buf_pool_alloc will 3235 * have allocated it. 3236 * So the second test here is pointless. 3237 * But it keeps semantic-checkers happy, and 3238 * this comment keeps human reviewers 3239 * happy. 3240 */ 3241 if (mreplace == NULL || bio == NULL || 3242 test_bit(Faulty, &mreplace->flags)) 3243 break; 3244 bio->bi_next = biolist; 3245 biolist = bio; 3246 bio->bi_end_io = end_sync_write; 3247 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3248 bio->bi_iter.bi_sector = to_addr + 3249 mreplace->data_offset; 3250 bio_set_dev(bio, mreplace->bdev); 3251 atomic_inc(&r10_bio->remaining); 3252 break; 3253 } 3254 rcu_read_unlock(); 3255 if (j == conf->copies) { 3256 /* Cannot recover, so abort the recovery or 3257 * record a bad block */ 3258 if (any_working) { 3259 /* problem is that there are bad blocks 3260 * on other device(s) 3261 */ 3262 int k; 3263 for (k = 0; k < conf->copies; k++) 3264 if (r10_bio->devs[k].devnum == i) 3265 break; 3266 if (!test_bit(In_sync, 3267 &mrdev->flags) 3268 && !rdev_set_badblocks( 3269 mrdev, 3270 r10_bio->devs[k].addr, 3271 max_sync, 0)) 3272 any_working = 0; 3273 if (mreplace && 3274 !rdev_set_badblocks( 3275 mreplace, 3276 r10_bio->devs[k].addr, 3277 max_sync, 0)) 3278 any_working = 0; 3279 } 3280 if (!any_working) { 3281 if (!test_and_set_bit(MD_RECOVERY_INTR, 3282 &mddev->recovery)) 3283 pr_warn("md/raid10:%s: insufficient working devices for recovery.\n", 3284 mdname(mddev)); 3285 mirror->recovery_disabled 3286 = mddev->recovery_disabled; 3287 } 3288 put_buf(r10_bio); 3289 if (rb2) 3290 atomic_dec(&rb2->remaining); 3291 r10_bio = rb2; 3292 rdev_dec_pending(mrdev, mddev); 3293 if (mreplace) 3294 rdev_dec_pending(mreplace, mddev); 3295 break; 3296 } 3297 rdev_dec_pending(mrdev, mddev); 3298 if (mreplace) 3299 rdev_dec_pending(mreplace, mddev); 3300 if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) { 3301 /* Only want this if there is elsewhere to 3302 * read from. 'j' is currently the first 3303 * readable copy. 3304 */ 3305 int targets = 1; 3306 for (; j < conf->copies; j++) { 3307 int d = r10_bio->devs[j].devnum; 3308 if (conf->mirrors[d].rdev && 3309 test_bit(In_sync, 3310 &conf->mirrors[d].rdev->flags)) 3311 targets++; 3312 } 3313 if (targets == 1) 3314 r10_bio->devs[0].bio->bi_opf 3315 &= ~MD_FAILFAST; 3316 } 3317 } 3318 if (biolist == NULL) { 3319 while (r10_bio) { 3320 struct r10bio *rb2 = r10_bio; 3321 r10_bio = (struct r10bio*) rb2->master_bio; 3322 rb2->master_bio = NULL; 3323 put_buf(rb2); 3324 } 3325 goto giveup; 3326 } 3327 } else { 3328 /* resync. Schedule a read for every block at this virt offset */ 3329 int count = 0; 3330 3331 /* 3332 * Since curr_resync_completed could probably not update in 3333 * time, and we will set cluster_sync_low based on it. 3334 * Let's check against "sector_nr + 2 * RESYNC_SECTORS" for 3335 * safety reason, which ensures curr_resync_completed is 3336 * updated in bitmap_cond_end_sync. 3337 */ 3338 md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, 3339 mddev_is_clustered(mddev) && 3340 (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); 3341 3342 if (!md_bitmap_start_sync(mddev->bitmap, sector_nr, 3343 &sync_blocks, mddev->degraded) && 3344 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3345 &mddev->recovery)) { 3346 /* We can skip this block */ 3347 *skipped = 1; 3348 return sync_blocks + sectors_skipped; 3349 } 3350 if (sync_blocks < max_sync) 3351 max_sync = sync_blocks; 3352 r10_bio = raid10_alloc_init_r10buf(conf); 3353 r10_bio->state = 0; 3354 3355 r10_bio->mddev = mddev; 3356 atomic_set(&r10_bio->remaining, 0); 3357 raise_barrier(conf, 0); 3358 conf->next_resync = sector_nr; 3359 3360 r10_bio->master_bio = NULL; 3361 r10_bio->sector = sector_nr; 3362 set_bit(R10BIO_IsSync, &r10_bio->state); 3363 raid10_find_phys(conf, r10_bio); 3364 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3365 3366 for (i = 0; i < conf->copies; i++) { 3367 int d = r10_bio->devs[i].devnum; 3368 sector_t first_bad, sector; 3369 int bad_sectors; 3370 struct md_rdev *rdev; 3371 3372 if (r10_bio->devs[i].repl_bio) 3373 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3374 3375 bio = r10_bio->devs[i].bio; 3376 bio->bi_status = BLK_STS_IOERR; 3377 rcu_read_lock(); 3378 rdev = rcu_dereference(conf->mirrors[d].rdev); 3379 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3380 rcu_read_unlock(); 3381 continue; 3382 } 3383 sector = r10_bio->devs[i].addr; 3384 if (is_badblock(rdev, sector, max_sync, 3385 &first_bad, &bad_sectors)) { 3386 if (first_bad > sector) 3387 max_sync = first_bad - sector; 3388 else { 3389 bad_sectors -= (sector - first_bad); 3390 if (max_sync > bad_sectors) 3391 max_sync = bad_sectors; 3392 rcu_read_unlock(); 3393 continue; 3394 } 3395 } 3396 atomic_inc(&rdev->nr_pending); 3397 atomic_inc(&r10_bio->remaining); 3398 bio->bi_next = biolist; 3399 biolist = bio; 3400 bio->bi_end_io = end_sync_read; 3401 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3402 if (test_bit(FailFast, &rdev->flags)) 3403 bio->bi_opf |= MD_FAILFAST; 3404 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3405 bio_set_dev(bio, rdev->bdev); 3406 count++; 3407 3408 rdev = rcu_dereference(conf->mirrors[d].replacement); 3409 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3410 rcu_read_unlock(); 3411 continue; 3412 } 3413 atomic_inc(&rdev->nr_pending); 3414 3415 /* Need to set up for writing to the replacement */ 3416 bio = r10_bio->devs[i].repl_bio; 3417 bio->bi_status = BLK_STS_IOERR; 3418 3419 sector = r10_bio->devs[i].addr; 3420 bio->bi_next = biolist; 3421 biolist = bio; 3422 bio->bi_end_io = end_sync_write; 3423 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3424 if (test_bit(FailFast, &rdev->flags)) 3425 bio->bi_opf |= MD_FAILFAST; 3426 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3427 bio_set_dev(bio, rdev->bdev); 3428 count++; 3429 rcu_read_unlock(); 3430 } 3431 3432 if (count < 2) { 3433 for (i=0; i<conf->copies; i++) { 3434 int d = r10_bio->devs[i].devnum; 3435 if (r10_bio->devs[i].bio->bi_end_io) 3436 rdev_dec_pending(conf->mirrors[d].rdev, 3437 mddev); 3438 if (r10_bio->devs[i].repl_bio && 3439 r10_bio->devs[i].repl_bio->bi_end_io) 3440 rdev_dec_pending( 3441 conf->mirrors[d].replacement, 3442 mddev); 3443 } 3444 put_buf(r10_bio); 3445 biolist = NULL; 3446 goto giveup; 3447 } 3448 } 3449 3450 nr_sectors = 0; 3451 if (sector_nr + max_sync < max_sector) 3452 max_sector = sector_nr + max_sync; 3453 do { 3454 struct page *page; 3455 int len = PAGE_SIZE; 3456 if (sector_nr + (len>>9) > max_sector) 3457 len = (max_sector - sector_nr) << 9; 3458 if (len == 0) 3459 break; 3460 for (bio= biolist ; bio ; bio=bio->bi_next) { 3461 struct resync_pages *rp = get_resync_pages(bio); 3462 page = resync_fetch_page(rp, page_idx); 3463 /* 3464 * won't fail because the vec table is big enough 3465 * to hold all these pages 3466 */ 3467 bio_add_page(bio, page, len, 0); 3468 } 3469 nr_sectors += len>>9; 3470 sector_nr += len>>9; 3471 } while (++page_idx < RESYNC_PAGES); 3472 r10_bio->sectors = nr_sectors; 3473 3474 if (mddev_is_clustered(mddev) && 3475 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3476 /* It is resync not recovery */ 3477 if (conf->cluster_sync_high < sector_nr + nr_sectors) { 3478 conf->cluster_sync_low = mddev->curr_resync_completed; 3479 raid10_set_cluster_sync_high(conf); 3480 /* Send resync message */ 3481 md_cluster_ops->resync_info_update(mddev, 3482 conf->cluster_sync_low, 3483 conf->cluster_sync_high); 3484 } 3485 } else if (mddev_is_clustered(mddev)) { 3486 /* This is recovery not resync */ 3487 sector_t sect_va1, sect_va2; 3488 bool broadcast_msg = false; 3489 3490 for (i = 0; i < conf->geo.raid_disks; i++) { 3491 /* 3492 * sector_nr is a device address for recovery, so we 3493 * need translate it to array address before compare 3494 * with cluster_sync_high. 3495 */ 3496 sect_va1 = raid10_find_virt(conf, sector_nr, i); 3497 3498 if (conf->cluster_sync_high < sect_va1 + nr_sectors) { 3499 broadcast_msg = true; 3500 /* 3501 * curr_resync_completed is similar as 3502 * sector_nr, so make the translation too. 3503 */ 3504 sect_va2 = raid10_find_virt(conf, 3505 mddev->curr_resync_completed, i); 3506 3507 if (conf->cluster_sync_low == 0 || 3508 conf->cluster_sync_low > sect_va2) 3509 conf->cluster_sync_low = sect_va2; 3510 } 3511 } 3512 if (broadcast_msg) { 3513 raid10_set_cluster_sync_high(conf); 3514 md_cluster_ops->resync_info_update(mddev, 3515 conf->cluster_sync_low, 3516 conf->cluster_sync_high); 3517 } 3518 } 3519 3520 while (biolist) { 3521 bio = biolist; 3522 biolist = biolist->bi_next; 3523 3524 bio->bi_next = NULL; 3525 r10_bio = get_resync_r10bio(bio); 3526 r10_bio->sectors = nr_sectors; 3527 3528 if (bio->bi_end_io == end_sync_read) { 3529 md_sync_acct_bio(bio, nr_sectors); 3530 bio->bi_status = 0; 3531 generic_make_request(bio); 3532 } 3533 } 3534 3535 if (sectors_skipped) 3536 /* pretend they weren't skipped, it makes 3537 * no important difference in this case 3538 */ 3539 md_done_sync(mddev, sectors_skipped, 1); 3540 3541 return sectors_skipped + nr_sectors; 3542 giveup: 3543 /* There is nowhere to write, so all non-sync 3544 * drives must be failed or in resync, all drives 3545 * have a bad block, so try the next chunk... 3546 */ 3547 if (sector_nr + max_sync < max_sector) 3548 max_sector = sector_nr + max_sync; 3549 3550 sectors_skipped += (max_sector - sector_nr); 3551 chunks_skipped ++; 3552 sector_nr = max_sector; 3553 goto skipped; 3554 } 3555 3556 static sector_t 3557 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3558 { 3559 sector_t size; 3560 struct r10conf *conf = mddev->private; 3561 3562 if (!raid_disks) 3563 raid_disks = min(conf->geo.raid_disks, 3564 conf->prev.raid_disks); 3565 if (!sectors) 3566 sectors = conf->dev_sectors; 3567 3568 size = sectors >> conf->geo.chunk_shift; 3569 sector_div(size, conf->geo.far_copies); 3570 size = size * raid_disks; 3571 sector_div(size, conf->geo.near_copies); 3572 3573 return size << conf->geo.chunk_shift; 3574 } 3575 3576 static void calc_sectors(struct r10conf *conf, sector_t size) 3577 { 3578 /* Calculate the number of sectors-per-device that will 3579 * actually be used, and set conf->dev_sectors and 3580 * conf->stride 3581 */ 3582 3583 size = size >> conf->geo.chunk_shift; 3584 sector_div(size, conf->geo.far_copies); 3585 size = size * conf->geo.raid_disks; 3586 sector_div(size, conf->geo.near_copies); 3587 /* 'size' is now the number of chunks in the array */ 3588 /* calculate "used chunks per device" */ 3589 size = size * conf->copies; 3590 3591 /* We need to round up when dividing by raid_disks to 3592 * get the stride size. 3593 */ 3594 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3595 3596 conf->dev_sectors = size << conf->geo.chunk_shift; 3597 3598 if (conf->geo.far_offset) 3599 conf->geo.stride = 1 << conf->geo.chunk_shift; 3600 else { 3601 sector_div(size, conf->geo.far_copies); 3602 conf->geo.stride = size << conf->geo.chunk_shift; 3603 } 3604 } 3605 3606 enum geo_type {geo_new, geo_old, geo_start}; 3607 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3608 { 3609 int nc, fc, fo; 3610 int layout, chunk, disks; 3611 switch (new) { 3612 case geo_old: 3613 layout = mddev->layout; 3614 chunk = mddev->chunk_sectors; 3615 disks = mddev->raid_disks - mddev->delta_disks; 3616 break; 3617 case geo_new: 3618 layout = mddev->new_layout; 3619 chunk = mddev->new_chunk_sectors; 3620 disks = mddev->raid_disks; 3621 break; 3622 default: /* avoid 'may be unused' warnings */ 3623 case geo_start: /* new when starting reshape - raid_disks not 3624 * updated yet. */ 3625 layout = mddev->new_layout; 3626 chunk = mddev->new_chunk_sectors; 3627 disks = mddev->raid_disks + mddev->delta_disks; 3628 break; 3629 } 3630 if (layout >> 19) 3631 return -1; 3632 if (chunk < (PAGE_SIZE >> 9) || 3633 !is_power_of_2(chunk)) 3634 return -2; 3635 nc = layout & 255; 3636 fc = (layout >> 8) & 255; 3637 fo = layout & (1<<16); 3638 geo->raid_disks = disks; 3639 geo->near_copies = nc; 3640 geo->far_copies = fc; 3641 geo->far_offset = fo; 3642 switch (layout >> 17) { 3643 case 0: /* original layout. simple but not always optimal */ 3644 geo->far_set_size = disks; 3645 break; 3646 case 1: /* "improved" layout which was buggy. Hopefully no-one is 3647 * actually using this, but leave code here just in case.*/ 3648 geo->far_set_size = disks/fc; 3649 WARN(geo->far_set_size < fc, 3650 "This RAID10 layout does not provide data safety - please backup and create new array\n"); 3651 break; 3652 case 2: /* "improved" layout fixed to match documentation */ 3653 geo->far_set_size = fc * nc; 3654 break; 3655 default: /* Not a valid layout */ 3656 return -1; 3657 } 3658 geo->chunk_mask = chunk - 1; 3659 geo->chunk_shift = ffz(~chunk); 3660 return nc*fc; 3661 } 3662 3663 static struct r10conf *setup_conf(struct mddev *mddev) 3664 { 3665 struct r10conf *conf = NULL; 3666 int err = -EINVAL; 3667 struct geom geo; 3668 int copies; 3669 3670 copies = setup_geo(&geo, mddev, geo_new); 3671 3672 if (copies == -2) { 3673 pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n", 3674 mdname(mddev), PAGE_SIZE); 3675 goto out; 3676 } 3677 3678 if (copies < 2 || copies > mddev->raid_disks) { 3679 pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3680 mdname(mddev), mddev->new_layout); 3681 goto out; 3682 } 3683 3684 err = -ENOMEM; 3685 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3686 if (!conf) 3687 goto out; 3688 3689 /* FIXME calc properly */ 3690 conf->mirrors = kcalloc(mddev->raid_disks + max(0, -mddev->delta_disks), 3691 sizeof(struct raid10_info), 3692 GFP_KERNEL); 3693 if (!conf->mirrors) 3694 goto out; 3695 3696 conf->tmppage = alloc_page(GFP_KERNEL); 3697 if (!conf->tmppage) 3698 goto out; 3699 3700 conf->geo = geo; 3701 conf->copies = copies; 3702 err = mempool_init(&conf->r10bio_pool, NR_RAID10_BIOS, r10bio_pool_alloc, 3703 r10bio_pool_free, conf); 3704 if (err) 3705 goto out; 3706 3707 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); 3708 if (err) 3709 goto out; 3710 3711 calc_sectors(conf, mddev->dev_sectors); 3712 if (mddev->reshape_position == MaxSector) { 3713 conf->prev = conf->geo; 3714 conf->reshape_progress = MaxSector; 3715 } else { 3716 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3717 err = -EINVAL; 3718 goto out; 3719 } 3720 conf->reshape_progress = mddev->reshape_position; 3721 if (conf->prev.far_offset) 3722 conf->prev.stride = 1 << conf->prev.chunk_shift; 3723 else 3724 /* far_copies must be 1 */ 3725 conf->prev.stride = conf->dev_sectors; 3726 } 3727 conf->reshape_safe = conf->reshape_progress; 3728 spin_lock_init(&conf->device_lock); 3729 INIT_LIST_HEAD(&conf->retry_list); 3730 INIT_LIST_HEAD(&conf->bio_end_io_list); 3731 3732 spin_lock_init(&conf->resync_lock); 3733 init_waitqueue_head(&conf->wait_barrier); 3734 atomic_set(&conf->nr_pending, 0); 3735 3736 err = -ENOMEM; 3737 conf->thread = md_register_thread(raid10d, mddev, "raid10"); 3738 if (!conf->thread) 3739 goto out; 3740 3741 conf->mddev = mddev; 3742 return conf; 3743 3744 out: 3745 if (conf) { 3746 mempool_exit(&conf->r10bio_pool); 3747 kfree(conf->mirrors); 3748 safe_put_page(conf->tmppage); 3749 bioset_exit(&conf->bio_split); 3750 kfree(conf); 3751 } 3752 return ERR_PTR(err); 3753 } 3754 3755 static int raid10_run(struct mddev *mddev) 3756 { 3757 struct r10conf *conf; 3758 int i, disk_idx, chunk_size; 3759 struct raid10_info *disk; 3760 struct md_rdev *rdev; 3761 sector_t size; 3762 sector_t min_offset_diff = 0; 3763 int first = 1; 3764 bool discard_supported = false; 3765 3766 if (mddev_init_writes_pending(mddev) < 0) 3767 return -ENOMEM; 3768 3769 if (mddev->private == NULL) { 3770 conf = setup_conf(mddev); 3771 if (IS_ERR(conf)) 3772 return PTR_ERR(conf); 3773 mddev->private = conf; 3774 } 3775 conf = mddev->private; 3776 if (!conf) 3777 goto out; 3778 3779 if (mddev_is_clustered(conf->mddev)) { 3780 int fc, fo; 3781 3782 fc = (mddev->layout >> 8) & 255; 3783 fo = mddev->layout & (1<<16); 3784 if (fc > 1 || fo > 0) { 3785 pr_err("only near layout is supported by clustered" 3786 " raid10\n"); 3787 goto out_free_conf; 3788 } 3789 } 3790 3791 mddev->thread = conf->thread; 3792 conf->thread = NULL; 3793 3794 chunk_size = mddev->chunk_sectors << 9; 3795 if (mddev->queue) { 3796 blk_queue_max_discard_sectors(mddev->queue, 3797 mddev->chunk_sectors); 3798 blk_queue_max_write_same_sectors(mddev->queue, 0); 3799 blk_queue_max_write_zeroes_sectors(mddev->queue, 0); 3800 blk_queue_io_min(mddev->queue, chunk_size); 3801 if (conf->geo.raid_disks % conf->geo.near_copies) 3802 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks); 3803 else 3804 blk_queue_io_opt(mddev->queue, chunk_size * 3805 (conf->geo.raid_disks / conf->geo.near_copies)); 3806 } 3807 3808 rdev_for_each(rdev, mddev) { 3809 long long diff; 3810 3811 disk_idx = rdev->raid_disk; 3812 if (disk_idx < 0) 3813 continue; 3814 if (disk_idx >= conf->geo.raid_disks && 3815 disk_idx >= conf->prev.raid_disks) 3816 continue; 3817 disk = conf->mirrors + disk_idx; 3818 3819 if (test_bit(Replacement, &rdev->flags)) { 3820 if (disk->replacement) 3821 goto out_free_conf; 3822 disk->replacement = rdev; 3823 } else { 3824 if (disk->rdev) 3825 goto out_free_conf; 3826 disk->rdev = rdev; 3827 } 3828 diff = (rdev->new_data_offset - rdev->data_offset); 3829 if (!mddev->reshape_backwards) 3830 diff = -diff; 3831 if (diff < 0) 3832 diff = 0; 3833 if (first || diff < min_offset_diff) 3834 min_offset_diff = diff; 3835 3836 if (mddev->gendisk) 3837 disk_stack_limits(mddev->gendisk, rdev->bdev, 3838 rdev->data_offset << 9); 3839 3840 disk->head_position = 0; 3841 3842 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3843 discard_supported = true; 3844 first = 0; 3845 } 3846 3847 if (mddev->queue) { 3848 if (discard_supported) 3849 blk_queue_flag_set(QUEUE_FLAG_DISCARD, 3850 mddev->queue); 3851 else 3852 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, 3853 mddev->queue); 3854 } 3855 /* need to check that every block has at least one working mirror */ 3856 if (!enough(conf, -1)) { 3857 pr_err("md/raid10:%s: not enough operational mirrors.\n", 3858 mdname(mddev)); 3859 goto out_free_conf; 3860 } 3861 3862 if (conf->reshape_progress != MaxSector) { 3863 /* must ensure that shape change is supported */ 3864 if (conf->geo.far_copies != 1 && 3865 conf->geo.far_offset == 0) 3866 goto out_free_conf; 3867 if (conf->prev.far_copies != 1 && 3868 conf->prev.far_offset == 0) 3869 goto out_free_conf; 3870 } 3871 3872 mddev->degraded = 0; 3873 for (i = 0; 3874 i < conf->geo.raid_disks 3875 || i < conf->prev.raid_disks; 3876 i++) { 3877 3878 disk = conf->mirrors + i; 3879 3880 if (!disk->rdev && disk->replacement) { 3881 /* The replacement is all we have - use it */ 3882 disk->rdev = disk->replacement; 3883 disk->replacement = NULL; 3884 clear_bit(Replacement, &disk->rdev->flags); 3885 } 3886 3887 if (!disk->rdev || 3888 !test_bit(In_sync, &disk->rdev->flags)) { 3889 disk->head_position = 0; 3890 mddev->degraded++; 3891 if (disk->rdev && 3892 disk->rdev->saved_raid_disk < 0) 3893 conf->fullsync = 1; 3894 } 3895 3896 if (disk->replacement && 3897 !test_bit(In_sync, &disk->replacement->flags) && 3898 disk->replacement->saved_raid_disk < 0) { 3899 conf->fullsync = 1; 3900 } 3901 3902 disk->recovery_disabled = mddev->recovery_disabled - 1; 3903 } 3904 3905 if (mddev->recovery_cp != MaxSector) 3906 pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n", 3907 mdname(mddev)); 3908 pr_info("md/raid10:%s: active with %d out of %d devices\n", 3909 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 3910 conf->geo.raid_disks); 3911 /* 3912 * Ok, everything is just fine now 3913 */ 3914 mddev->dev_sectors = conf->dev_sectors; 3915 size = raid10_size(mddev, 0, 0); 3916 md_set_array_sectors(mddev, size); 3917 mddev->resync_max_sectors = size; 3918 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 3919 3920 if (mddev->queue) { 3921 int stripe = conf->geo.raid_disks * 3922 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 3923 3924 /* Calculate max read-ahead size. 3925 * We need to readahead at least twice a whole stripe.... 3926 * maybe... 3927 */ 3928 stripe /= conf->geo.near_copies; 3929 if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe) 3930 mddev->queue->backing_dev_info->ra_pages = 2 * stripe; 3931 } 3932 3933 if (md_integrity_register(mddev)) 3934 goto out_free_conf; 3935 3936 if (conf->reshape_progress != MaxSector) { 3937 unsigned long before_length, after_length; 3938 3939 before_length = ((1 << conf->prev.chunk_shift) * 3940 conf->prev.far_copies); 3941 after_length = ((1 << conf->geo.chunk_shift) * 3942 conf->geo.far_copies); 3943 3944 if (max(before_length, after_length) > min_offset_diff) { 3945 /* This cannot work */ 3946 pr_warn("md/raid10: offset difference not enough to continue reshape\n"); 3947 goto out_free_conf; 3948 } 3949 conf->offset_diff = min_offset_diff; 3950 3951 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 3952 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 3953 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 3954 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 3955 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 3956 "reshape"); 3957 } 3958 3959 return 0; 3960 3961 out_free_conf: 3962 md_unregister_thread(&mddev->thread); 3963 mempool_exit(&conf->r10bio_pool); 3964 safe_put_page(conf->tmppage); 3965 kfree(conf->mirrors); 3966 kfree(conf); 3967 mddev->private = NULL; 3968 out: 3969 return -EIO; 3970 } 3971 3972 static void raid10_free(struct mddev *mddev, void *priv) 3973 { 3974 struct r10conf *conf = priv; 3975 3976 mempool_exit(&conf->r10bio_pool); 3977 safe_put_page(conf->tmppage); 3978 kfree(conf->mirrors); 3979 kfree(conf->mirrors_old); 3980 kfree(conf->mirrors_new); 3981 bioset_exit(&conf->bio_split); 3982 kfree(conf); 3983 } 3984 3985 static void raid10_quiesce(struct mddev *mddev, int quiesce) 3986 { 3987 struct r10conf *conf = mddev->private; 3988 3989 if (quiesce) 3990 raise_barrier(conf, 0); 3991 else 3992 lower_barrier(conf); 3993 } 3994 3995 static int raid10_resize(struct mddev *mddev, sector_t sectors) 3996 { 3997 /* Resize of 'far' arrays is not supported. 3998 * For 'near' and 'offset' arrays we can set the 3999 * number of sectors used to be an appropriate multiple 4000 * of the chunk size. 4001 * For 'offset', this is far_copies*chunksize. 4002 * For 'near' the multiplier is the LCM of 4003 * near_copies and raid_disks. 4004 * So if far_copies > 1 && !far_offset, fail. 4005 * Else find LCM(raid_disks, near_copy)*far_copies and 4006 * multiply by chunk_size. Then round to this number. 4007 * This is mostly done by raid10_size() 4008 */ 4009 struct r10conf *conf = mddev->private; 4010 sector_t oldsize, size; 4011 4012 if (mddev->reshape_position != MaxSector) 4013 return -EBUSY; 4014 4015 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 4016 return -EINVAL; 4017 4018 oldsize = raid10_size(mddev, 0, 0); 4019 size = raid10_size(mddev, sectors, 0); 4020 if (mddev->external_size && 4021 mddev->array_sectors > size) 4022 return -EINVAL; 4023 if (mddev->bitmap) { 4024 int ret = md_bitmap_resize(mddev->bitmap, size, 0, 0); 4025 if (ret) 4026 return ret; 4027 } 4028 md_set_array_sectors(mddev, size); 4029 if (sectors > mddev->dev_sectors && 4030 mddev->recovery_cp > oldsize) { 4031 mddev->recovery_cp = oldsize; 4032 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4033 } 4034 calc_sectors(conf, sectors); 4035 mddev->dev_sectors = conf->dev_sectors; 4036 mddev->resync_max_sectors = size; 4037 return 0; 4038 } 4039 4040 static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) 4041 { 4042 struct md_rdev *rdev; 4043 struct r10conf *conf; 4044 4045 if (mddev->degraded > 0) { 4046 pr_warn("md/raid10:%s: Error: degraded raid0!\n", 4047 mdname(mddev)); 4048 return ERR_PTR(-EINVAL); 4049 } 4050 sector_div(size, devs); 4051 4052 /* Set new parameters */ 4053 mddev->new_level = 10; 4054 /* new layout: far_copies = 1, near_copies = 2 */ 4055 mddev->new_layout = (1<<8) + 2; 4056 mddev->new_chunk_sectors = mddev->chunk_sectors; 4057 mddev->delta_disks = mddev->raid_disks; 4058 mddev->raid_disks *= 2; 4059 /* make sure it will be not marked as dirty */ 4060 mddev->recovery_cp = MaxSector; 4061 mddev->dev_sectors = size; 4062 4063 conf = setup_conf(mddev); 4064 if (!IS_ERR(conf)) { 4065 rdev_for_each(rdev, mddev) 4066 if (rdev->raid_disk >= 0) { 4067 rdev->new_raid_disk = rdev->raid_disk * 2; 4068 rdev->sectors = size; 4069 } 4070 conf->barrier = 1; 4071 } 4072 4073 return conf; 4074 } 4075 4076 static void *raid10_takeover(struct mddev *mddev) 4077 { 4078 struct r0conf *raid0_conf; 4079 4080 /* raid10 can take over: 4081 * raid0 - providing it has only two drives 4082 */ 4083 if (mddev->level == 0) { 4084 /* for raid0 takeover only one zone is supported */ 4085 raid0_conf = mddev->private; 4086 if (raid0_conf->nr_strip_zones > 1) { 4087 pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n", 4088 mdname(mddev)); 4089 return ERR_PTR(-EINVAL); 4090 } 4091 return raid10_takeover_raid0(mddev, 4092 raid0_conf->strip_zone->zone_end, 4093 raid0_conf->strip_zone->nb_dev); 4094 } 4095 return ERR_PTR(-EINVAL); 4096 } 4097 4098 static int raid10_check_reshape(struct mddev *mddev) 4099 { 4100 /* Called when there is a request to change 4101 * - layout (to ->new_layout) 4102 * - chunk size (to ->new_chunk_sectors) 4103 * - raid_disks (by delta_disks) 4104 * or when trying to restart a reshape that was ongoing. 4105 * 4106 * We need to validate the request and possibly allocate 4107 * space if that might be an issue later. 4108 * 4109 * Currently we reject any reshape of a 'far' mode array, 4110 * allow chunk size to change if new is generally acceptable, 4111 * allow raid_disks to increase, and allow 4112 * a switch between 'near' mode and 'offset' mode. 4113 */ 4114 struct r10conf *conf = mddev->private; 4115 struct geom geo; 4116 4117 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 4118 return -EINVAL; 4119 4120 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 4121 /* mustn't change number of copies */ 4122 return -EINVAL; 4123 if (geo.far_copies > 1 && !geo.far_offset) 4124 /* Cannot switch to 'far' mode */ 4125 return -EINVAL; 4126 4127 if (mddev->array_sectors & geo.chunk_mask) 4128 /* not factor of array size */ 4129 return -EINVAL; 4130 4131 if (!enough(conf, -1)) 4132 return -EINVAL; 4133 4134 kfree(conf->mirrors_new); 4135 conf->mirrors_new = NULL; 4136 if (mddev->delta_disks > 0) { 4137 /* allocate new 'mirrors' list */ 4138 conf->mirrors_new = 4139 kcalloc(mddev->raid_disks + mddev->delta_disks, 4140 sizeof(struct raid10_info), 4141 GFP_KERNEL); 4142 if (!conf->mirrors_new) 4143 return -ENOMEM; 4144 } 4145 return 0; 4146 } 4147 4148 /* 4149 * Need to check if array has failed when deciding whether to: 4150 * - start an array 4151 * - remove non-faulty devices 4152 * - add a spare 4153 * - allow a reshape 4154 * This determination is simple when no reshape is happening. 4155 * However if there is a reshape, we need to carefully check 4156 * both the before and after sections. 4157 * This is because some failed devices may only affect one 4158 * of the two sections, and some non-in_sync devices may 4159 * be insync in the section most affected by failed devices. 4160 */ 4161 static int calc_degraded(struct r10conf *conf) 4162 { 4163 int degraded, degraded2; 4164 int i; 4165 4166 rcu_read_lock(); 4167 degraded = 0; 4168 /* 'prev' section first */ 4169 for (i = 0; i < conf->prev.raid_disks; i++) { 4170 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4171 if (!rdev || test_bit(Faulty, &rdev->flags)) 4172 degraded++; 4173 else if (!test_bit(In_sync, &rdev->flags)) 4174 /* When we can reduce the number of devices in 4175 * an array, this might not contribute to 4176 * 'degraded'. It does now. 4177 */ 4178 degraded++; 4179 } 4180 rcu_read_unlock(); 4181 if (conf->geo.raid_disks == conf->prev.raid_disks) 4182 return degraded; 4183 rcu_read_lock(); 4184 degraded2 = 0; 4185 for (i = 0; i < conf->geo.raid_disks; i++) { 4186 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4187 if (!rdev || test_bit(Faulty, &rdev->flags)) 4188 degraded2++; 4189 else if (!test_bit(In_sync, &rdev->flags)) { 4190 /* If reshape is increasing the number of devices, 4191 * this section has already been recovered, so 4192 * it doesn't contribute to degraded. 4193 * else it does. 4194 */ 4195 if (conf->geo.raid_disks <= conf->prev.raid_disks) 4196 degraded2++; 4197 } 4198 } 4199 rcu_read_unlock(); 4200 if (degraded2 > degraded) 4201 return degraded2; 4202 return degraded; 4203 } 4204 4205 static int raid10_start_reshape(struct mddev *mddev) 4206 { 4207 /* A 'reshape' has been requested. This commits 4208 * the various 'new' fields and sets MD_RECOVER_RESHAPE 4209 * This also checks if there are enough spares and adds them 4210 * to the array. 4211 * We currently require enough spares to make the final 4212 * array non-degraded. We also require that the difference 4213 * between old and new data_offset - on each device - is 4214 * enough that we never risk over-writing. 4215 */ 4216 4217 unsigned long before_length, after_length; 4218 sector_t min_offset_diff = 0; 4219 int first = 1; 4220 struct geom new; 4221 struct r10conf *conf = mddev->private; 4222 struct md_rdev *rdev; 4223 int spares = 0; 4224 int ret; 4225 4226 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4227 return -EBUSY; 4228 4229 if (setup_geo(&new, mddev, geo_start) != conf->copies) 4230 return -EINVAL; 4231 4232 before_length = ((1 << conf->prev.chunk_shift) * 4233 conf->prev.far_copies); 4234 after_length = ((1 << conf->geo.chunk_shift) * 4235 conf->geo.far_copies); 4236 4237 rdev_for_each(rdev, mddev) { 4238 if (!test_bit(In_sync, &rdev->flags) 4239 && !test_bit(Faulty, &rdev->flags)) 4240 spares++; 4241 if (rdev->raid_disk >= 0) { 4242 long long diff = (rdev->new_data_offset 4243 - rdev->data_offset); 4244 if (!mddev->reshape_backwards) 4245 diff = -diff; 4246 if (diff < 0) 4247 diff = 0; 4248 if (first || diff < min_offset_diff) 4249 min_offset_diff = diff; 4250 first = 0; 4251 } 4252 } 4253 4254 if (max(before_length, after_length) > min_offset_diff) 4255 return -EINVAL; 4256 4257 if (spares < mddev->delta_disks) 4258 return -EINVAL; 4259 4260 conf->offset_diff = min_offset_diff; 4261 spin_lock_irq(&conf->device_lock); 4262 if (conf->mirrors_new) { 4263 memcpy(conf->mirrors_new, conf->mirrors, 4264 sizeof(struct raid10_info)*conf->prev.raid_disks); 4265 smp_mb(); 4266 kfree(conf->mirrors_old); 4267 conf->mirrors_old = conf->mirrors; 4268 conf->mirrors = conf->mirrors_new; 4269 conf->mirrors_new = NULL; 4270 } 4271 setup_geo(&conf->geo, mddev, geo_start); 4272 smp_mb(); 4273 if (mddev->reshape_backwards) { 4274 sector_t size = raid10_size(mddev, 0, 0); 4275 if (size < mddev->array_sectors) { 4276 spin_unlock_irq(&conf->device_lock); 4277 pr_warn("md/raid10:%s: array size must be reduce before number of disks\n", 4278 mdname(mddev)); 4279 return -EINVAL; 4280 } 4281 mddev->resync_max_sectors = size; 4282 conf->reshape_progress = size; 4283 } else 4284 conf->reshape_progress = 0; 4285 conf->reshape_safe = conf->reshape_progress; 4286 spin_unlock_irq(&conf->device_lock); 4287 4288 if (mddev->delta_disks && mddev->bitmap) { 4289 ret = md_bitmap_resize(mddev->bitmap, 4290 raid10_size(mddev, 0, conf->geo.raid_disks), 4291 0, 0); 4292 if (ret) 4293 goto abort; 4294 } 4295 if (mddev->delta_disks > 0) { 4296 rdev_for_each(rdev, mddev) 4297 if (rdev->raid_disk < 0 && 4298 !test_bit(Faulty, &rdev->flags)) { 4299 if (raid10_add_disk(mddev, rdev) == 0) { 4300 if (rdev->raid_disk >= 4301 conf->prev.raid_disks) 4302 set_bit(In_sync, &rdev->flags); 4303 else 4304 rdev->recovery_offset = 0; 4305 4306 if (sysfs_link_rdev(mddev, rdev)) 4307 /* Failure here is OK */; 4308 } 4309 } else if (rdev->raid_disk >= conf->prev.raid_disks 4310 && !test_bit(Faulty, &rdev->flags)) { 4311 /* This is a spare that was manually added */ 4312 set_bit(In_sync, &rdev->flags); 4313 } 4314 } 4315 /* When a reshape changes the number of devices, 4316 * ->degraded is measured against the larger of the 4317 * pre and post numbers. 4318 */ 4319 spin_lock_irq(&conf->device_lock); 4320 mddev->degraded = calc_degraded(conf); 4321 spin_unlock_irq(&conf->device_lock); 4322 mddev->raid_disks = conf->geo.raid_disks; 4323 mddev->reshape_position = conf->reshape_progress; 4324 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4325 4326 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4327 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4328 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 4329 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4330 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4331 4332 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4333 "reshape"); 4334 if (!mddev->sync_thread) { 4335 ret = -EAGAIN; 4336 goto abort; 4337 } 4338 conf->reshape_checkpoint = jiffies; 4339 md_wakeup_thread(mddev->sync_thread); 4340 md_new_event(mddev); 4341 return 0; 4342 4343 abort: 4344 mddev->recovery = 0; 4345 spin_lock_irq(&conf->device_lock); 4346 conf->geo = conf->prev; 4347 mddev->raid_disks = conf->geo.raid_disks; 4348 rdev_for_each(rdev, mddev) 4349 rdev->new_data_offset = rdev->data_offset; 4350 smp_wmb(); 4351 conf->reshape_progress = MaxSector; 4352 conf->reshape_safe = MaxSector; 4353 mddev->reshape_position = MaxSector; 4354 spin_unlock_irq(&conf->device_lock); 4355 return ret; 4356 } 4357 4358 /* Calculate the last device-address that could contain 4359 * any block from the chunk that includes the array-address 's' 4360 * and report the next address. 4361 * i.e. the address returned will be chunk-aligned and after 4362 * any data that is in the chunk containing 's'. 4363 */ 4364 static sector_t last_dev_address(sector_t s, struct geom *geo) 4365 { 4366 s = (s | geo->chunk_mask) + 1; 4367 s >>= geo->chunk_shift; 4368 s *= geo->near_copies; 4369 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4370 s *= geo->far_copies; 4371 s <<= geo->chunk_shift; 4372 return s; 4373 } 4374 4375 /* Calculate the first device-address that could contain 4376 * any block from the chunk that includes the array-address 's'. 4377 * This too will be the start of a chunk 4378 */ 4379 static sector_t first_dev_address(sector_t s, struct geom *geo) 4380 { 4381 s >>= geo->chunk_shift; 4382 s *= geo->near_copies; 4383 sector_div(s, geo->raid_disks); 4384 s *= geo->far_copies; 4385 s <<= geo->chunk_shift; 4386 return s; 4387 } 4388 4389 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4390 int *skipped) 4391 { 4392 /* We simply copy at most one chunk (smallest of old and new) 4393 * at a time, possibly less if that exceeds RESYNC_PAGES, 4394 * or we hit a bad block or something. 4395 * This might mean we pause for normal IO in the middle of 4396 * a chunk, but that is not a problem as mddev->reshape_position 4397 * can record any location. 4398 * 4399 * If we will want to write to a location that isn't 4400 * yet recorded as 'safe' (i.e. in metadata on disk) then 4401 * we need to flush all reshape requests and update the metadata. 4402 * 4403 * When reshaping forwards (e.g. to more devices), we interpret 4404 * 'safe' as the earliest block which might not have been copied 4405 * down yet. We divide this by previous stripe size and multiply 4406 * by previous stripe length to get lowest device offset that we 4407 * cannot write to yet. 4408 * We interpret 'sector_nr' as an address that we want to write to. 4409 * From this we use last_device_address() to find where we might 4410 * write to, and first_device_address on the 'safe' position. 4411 * If this 'next' write position is after the 'safe' position, 4412 * we must update the metadata to increase the 'safe' position. 4413 * 4414 * When reshaping backwards, we round in the opposite direction 4415 * and perform the reverse test: next write position must not be 4416 * less than current safe position. 4417 * 4418 * In all this the minimum difference in data offsets 4419 * (conf->offset_diff - always positive) allows a bit of slack, 4420 * so next can be after 'safe', but not by more than offset_diff 4421 * 4422 * We need to prepare all the bios here before we start any IO 4423 * to ensure the size we choose is acceptable to all devices. 4424 * The means one for each copy for write-out and an extra one for 4425 * read-in. 4426 * We store the read-in bio in ->master_bio and the others in 4427 * ->devs[x].bio and ->devs[x].repl_bio. 4428 */ 4429 struct r10conf *conf = mddev->private; 4430 struct r10bio *r10_bio; 4431 sector_t next, safe, last; 4432 int max_sectors; 4433 int nr_sectors; 4434 int s; 4435 struct md_rdev *rdev; 4436 int need_flush = 0; 4437 struct bio *blist; 4438 struct bio *bio, *read_bio; 4439 int sectors_done = 0; 4440 struct page **pages; 4441 4442 if (sector_nr == 0) { 4443 /* If restarting in the middle, skip the initial sectors */ 4444 if (mddev->reshape_backwards && 4445 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4446 sector_nr = (raid10_size(mddev, 0, 0) 4447 - conf->reshape_progress); 4448 } else if (!mddev->reshape_backwards && 4449 conf->reshape_progress > 0) 4450 sector_nr = conf->reshape_progress; 4451 if (sector_nr) { 4452 mddev->curr_resync_completed = sector_nr; 4453 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4454 *skipped = 1; 4455 return sector_nr; 4456 } 4457 } 4458 4459 /* We don't use sector_nr to track where we are up to 4460 * as that doesn't work well for ->reshape_backwards. 4461 * So just use ->reshape_progress. 4462 */ 4463 if (mddev->reshape_backwards) { 4464 /* 'next' is the earliest device address that we might 4465 * write to for this chunk in the new layout 4466 */ 4467 next = first_dev_address(conf->reshape_progress - 1, 4468 &conf->geo); 4469 4470 /* 'safe' is the last device address that we might read from 4471 * in the old layout after a restart 4472 */ 4473 safe = last_dev_address(conf->reshape_safe - 1, 4474 &conf->prev); 4475 4476 if (next + conf->offset_diff < safe) 4477 need_flush = 1; 4478 4479 last = conf->reshape_progress - 1; 4480 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4481 & conf->prev.chunk_mask); 4482 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last) 4483 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512; 4484 } else { 4485 /* 'next' is after the last device address that we 4486 * might write to for this chunk in the new layout 4487 */ 4488 next = last_dev_address(conf->reshape_progress, &conf->geo); 4489 4490 /* 'safe' is the earliest device address that we might 4491 * read from in the old layout after a restart 4492 */ 4493 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4494 4495 /* Need to update metadata if 'next' might be beyond 'safe' 4496 * as that would possibly corrupt data 4497 */ 4498 if (next > safe + conf->offset_diff) 4499 need_flush = 1; 4500 4501 sector_nr = conf->reshape_progress; 4502 last = sector_nr | (conf->geo.chunk_mask 4503 & conf->prev.chunk_mask); 4504 4505 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last) 4506 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1; 4507 } 4508 4509 if (need_flush || 4510 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4511 /* Need to update reshape_position in metadata */ 4512 wait_barrier(conf); 4513 mddev->reshape_position = conf->reshape_progress; 4514 if (mddev->reshape_backwards) 4515 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4516 - conf->reshape_progress; 4517 else 4518 mddev->curr_resync_completed = conf->reshape_progress; 4519 conf->reshape_checkpoint = jiffies; 4520 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4521 md_wakeup_thread(mddev->thread); 4522 wait_event(mddev->sb_wait, mddev->sb_flags == 0 || 4523 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4524 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 4525 allow_barrier(conf); 4526 return sectors_done; 4527 } 4528 conf->reshape_safe = mddev->reshape_position; 4529 allow_barrier(conf); 4530 } 4531 4532 raise_barrier(conf, 0); 4533 read_more: 4534 /* Now schedule reads for blocks from sector_nr to last */ 4535 r10_bio = raid10_alloc_init_r10buf(conf); 4536 r10_bio->state = 0; 4537 raise_barrier(conf, 1); 4538 atomic_set(&r10_bio->remaining, 0); 4539 r10_bio->mddev = mddev; 4540 r10_bio->sector = sector_nr; 4541 set_bit(R10BIO_IsReshape, &r10_bio->state); 4542 r10_bio->sectors = last - sector_nr + 1; 4543 rdev = read_balance(conf, r10_bio, &max_sectors); 4544 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4545 4546 if (!rdev) { 4547 /* Cannot read from here, so need to record bad blocks 4548 * on all the target devices. 4549 */ 4550 // FIXME 4551 mempool_free(r10_bio, &conf->r10buf_pool); 4552 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4553 return sectors_done; 4554 } 4555 4556 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev); 4557 4558 bio_set_dev(read_bio, rdev->bdev); 4559 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4560 + rdev->data_offset); 4561 read_bio->bi_private = r10_bio; 4562 read_bio->bi_end_io = end_reshape_read; 4563 bio_set_op_attrs(read_bio, REQ_OP_READ, 0); 4564 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); 4565 read_bio->bi_status = 0; 4566 read_bio->bi_vcnt = 0; 4567 read_bio->bi_iter.bi_size = 0; 4568 r10_bio->master_bio = read_bio; 4569 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4570 4571 /* Now find the locations in the new layout */ 4572 __raid10_find_phys(&conf->geo, r10_bio); 4573 4574 blist = read_bio; 4575 read_bio->bi_next = NULL; 4576 4577 rcu_read_lock(); 4578 for (s = 0; s < conf->copies*2; s++) { 4579 struct bio *b; 4580 int d = r10_bio->devs[s/2].devnum; 4581 struct md_rdev *rdev2; 4582 if (s&1) { 4583 rdev2 = rcu_dereference(conf->mirrors[d].replacement); 4584 b = r10_bio->devs[s/2].repl_bio; 4585 } else { 4586 rdev2 = rcu_dereference(conf->mirrors[d].rdev); 4587 b = r10_bio->devs[s/2].bio; 4588 } 4589 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4590 continue; 4591 4592 bio_set_dev(b, rdev2->bdev); 4593 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + 4594 rdev2->new_data_offset; 4595 b->bi_end_io = end_reshape_write; 4596 bio_set_op_attrs(b, REQ_OP_WRITE, 0); 4597 b->bi_next = blist; 4598 blist = b; 4599 } 4600 4601 /* Now add as many pages as possible to all of these bios. */ 4602 4603 nr_sectors = 0; 4604 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; 4605 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4606 struct page *page = pages[s / (PAGE_SIZE >> 9)]; 4607 int len = (max_sectors - s) << 9; 4608 if (len > PAGE_SIZE) 4609 len = PAGE_SIZE; 4610 for (bio = blist; bio ; bio = bio->bi_next) { 4611 /* 4612 * won't fail because the vec table is big enough 4613 * to hold all these pages 4614 */ 4615 bio_add_page(bio, page, len, 0); 4616 } 4617 sector_nr += len >> 9; 4618 nr_sectors += len >> 9; 4619 } 4620 rcu_read_unlock(); 4621 r10_bio->sectors = nr_sectors; 4622 4623 /* Now submit the read */ 4624 md_sync_acct_bio(read_bio, r10_bio->sectors); 4625 atomic_inc(&r10_bio->remaining); 4626 read_bio->bi_next = NULL; 4627 generic_make_request(read_bio); 4628 sector_nr += nr_sectors; 4629 sectors_done += nr_sectors; 4630 if (sector_nr <= last) 4631 goto read_more; 4632 4633 lower_barrier(conf); 4634 4635 /* Now that we have done the whole section we can 4636 * update reshape_progress 4637 */ 4638 if (mddev->reshape_backwards) 4639 conf->reshape_progress -= sectors_done; 4640 else 4641 conf->reshape_progress += sectors_done; 4642 4643 return sectors_done; 4644 } 4645 4646 static void end_reshape_request(struct r10bio *r10_bio); 4647 static int handle_reshape_read_error(struct mddev *mddev, 4648 struct r10bio *r10_bio); 4649 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4650 { 4651 /* Reshape read completed. Hopefully we have a block 4652 * to write out. 4653 * If we got a read error then we do sync 1-page reads from 4654 * elsewhere until we find the data - or give up. 4655 */ 4656 struct r10conf *conf = mddev->private; 4657 int s; 4658 4659 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4660 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4661 /* Reshape has been aborted */ 4662 md_done_sync(mddev, r10_bio->sectors, 0); 4663 return; 4664 } 4665 4666 /* We definitely have the data in the pages, schedule the 4667 * writes. 4668 */ 4669 atomic_set(&r10_bio->remaining, 1); 4670 for (s = 0; s < conf->copies*2; s++) { 4671 struct bio *b; 4672 int d = r10_bio->devs[s/2].devnum; 4673 struct md_rdev *rdev; 4674 rcu_read_lock(); 4675 if (s&1) { 4676 rdev = rcu_dereference(conf->mirrors[d].replacement); 4677 b = r10_bio->devs[s/2].repl_bio; 4678 } else { 4679 rdev = rcu_dereference(conf->mirrors[d].rdev); 4680 b = r10_bio->devs[s/2].bio; 4681 } 4682 if (!rdev || test_bit(Faulty, &rdev->flags)) { 4683 rcu_read_unlock(); 4684 continue; 4685 } 4686 atomic_inc(&rdev->nr_pending); 4687 rcu_read_unlock(); 4688 md_sync_acct_bio(b, r10_bio->sectors); 4689 atomic_inc(&r10_bio->remaining); 4690 b->bi_next = NULL; 4691 generic_make_request(b); 4692 } 4693 end_reshape_request(r10_bio); 4694 } 4695 4696 static void end_reshape(struct r10conf *conf) 4697 { 4698 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4699 return; 4700 4701 spin_lock_irq(&conf->device_lock); 4702 conf->prev = conf->geo; 4703 md_finish_reshape(conf->mddev); 4704 smp_wmb(); 4705 conf->reshape_progress = MaxSector; 4706 conf->reshape_safe = MaxSector; 4707 spin_unlock_irq(&conf->device_lock); 4708 4709 /* read-ahead size must cover two whole stripes, which is 4710 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4711 */ 4712 if (conf->mddev->queue) { 4713 int stripe = conf->geo.raid_disks * 4714 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE); 4715 stripe /= conf->geo.near_copies; 4716 if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe) 4717 conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe; 4718 } 4719 conf->fullsync = 0; 4720 } 4721 4722 static int handle_reshape_read_error(struct mddev *mddev, 4723 struct r10bio *r10_bio) 4724 { 4725 /* Use sync reads to get the blocks from somewhere else */ 4726 int sectors = r10_bio->sectors; 4727 struct r10conf *conf = mddev->private; 4728 struct r10bio *r10b; 4729 int slot = 0; 4730 int idx = 0; 4731 struct page **pages; 4732 4733 r10b = kmalloc(sizeof(*r10b) + 4734 sizeof(struct r10dev) * conf->copies, GFP_NOIO); 4735 if (!r10b) { 4736 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4737 return -ENOMEM; 4738 } 4739 4740 /* reshape IOs share pages from .devs[0].bio */ 4741 pages = get_resync_pages(r10_bio->devs[0].bio)->pages; 4742 4743 r10b->sector = r10_bio->sector; 4744 __raid10_find_phys(&conf->prev, r10b); 4745 4746 while (sectors) { 4747 int s = sectors; 4748 int success = 0; 4749 int first_slot = slot; 4750 4751 if (s > (PAGE_SIZE >> 9)) 4752 s = PAGE_SIZE >> 9; 4753 4754 rcu_read_lock(); 4755 while (!success) { 4756 int d = r10b->devs[slot].devnum; 4757 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4758 sector_t addr; 4759 if (rdev == NULL || 4760 test_bit(Faulty, &rdev->flags) || 4761 !test_bit(In_sync, &rdev->flags)) 4762 goto failed; 4763 4764 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 4765 atomic_inc(&rdev->nr_pending); 4766 rcu_read_unlock(); 4767 success = sync_page_io(rdev, 4768 addr, 4769 s << 9, 4770 pages[idx], 4771 REQ_OP_READ, 0, false); 4772 rdev_dec_pending(rdev, mddev); 4773 rcu_read_lock(); 4774 if (success) 4775 break; 4776 failed: 4777 slot++; 4778 if (slot >= conf->copies) 4779 slot = 0; 4780 if (slot == first_slot) 4781 break; 4782 } 4783 rcu_read_unlock(); 4784 if (!success) { 4785 /* couldn't read this block, must give up */ 4786 set_bit(MD_RECOVERY_INTR, 4787 &mddev->recovery); 4788 kfree(r10b); 4789 return -EIO; 4790 } 4791 sectors -= s; 4792 idx++; 4793 } 4794 kfree(r10b); 4795 return 0; 4796 } 4797 4798 static void end_reshape_write(struct bio *bio) 4799 { 4800 struct r10bio *r10_bio = get_resync_r10bio(bio); 4801 struct mddev *mddev = r10_bio->mddev; 4802 struct r10conf *conf = mddev->private; 4803 int d; 4804 int slot; 4805 int repl; 4806 struct md_rdev *rdev = NULL; 4807 4808 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 4809 if (repl) 4810 rdev = conf->mirrors[d].replacement; 4811 if (!rdev) { 4812 smp_mb(); 4813 rdev = conf->mirrors[d].rdev; 4814 } 4815 4816 if (bio->bi_status) { 4817 /* FIXME should record badblock */ 4818 md_error(mddev, rdev); 4819 } 4820 4821 rdev_dec_pending(rdev, mddev); 4822 end_reshape_request(r10_bio); 4823 } 4824 4825 static void end_reshape_request(struct r10bio *r10_bio) 4826 { 4827 if (!atomic_dec_and_test(&r10_bio->remaining)) 4828 return; 4829 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 4830 bio_put(r10_bio->master_bio); 4831 put_buf(r10_bio); 4832 } 4833 4834 static void raid10_finish_reshape(struct mddev *mddev) 4835 { 4836 struct r10conf *conf = mddev->private; 4837 4838 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4839 return; 4840 4841 if (mddev->delta_disks > 0) { 4842 if (mddev->recovery_cp > mddev->resync_max_sectors) { 4843 mddev->recovery_cp = mddev->resync_max_sectors; 4844 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4845 } 4846 mddev->resync_max_sectors = mddev->array_sectors; 4847 } else { 4848 int d; 4849 rcu_read_lock(); 4850 for (d = conf->geo.raid_disks ; 4851 d < conf->geo.raid_disks - mddev->delta_disks; 4852 d++) { 4853 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4854 if (rdev) 4855 clear_bit(In_sync, &rdev->flags); 4856 rdev = rcu_dereference(conf->mirrors[d].replacement); 4857 if (rdev) 4858 clear_bit(In_sync, &rdev->flags); 4859 } 4860 rcu_read_unlock(); 4861 } 4862 mddev->layout = mddev->new_layout; 4863 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 4864 mddev->reshape_position = MaxSector; 4865 mddev->delta_disks = 0; 4866 mddev->reshape_backwards = 0; 4867 } 4868 4869 static struct md_personality raid10_personality = 4870 { 4871 .name = "raid10", 4872 .level = 10, 4873 .owner = THIS_MODULE, 4874 .make_request = raid10_make_request, 4875 .run = raid10_run, 4876 .free = raid10_free, 4877 .status = raid10_status, 4878 .error_handler = raid10_error, 4879 .hot_add_disk = raid10_add_disk, 4880 .hot_remove_disk= raid10_remove_disk, 4881 .spare_active = raid10_spare_active, 4882 .sync_request = raid10_sync_request, 4883 .quiesce = raid10_quiesce, 4884 .size = raid10_size, 4885 .resize = raid10_resize, 4886 .takeover = raid10_takeover, 4887 .check_reshape = raid10_check_reshape, 4888 .start_reshape = raid10_start_reshape, 4889 .finish_reshape = raid10_finish_reshape, 4890 .congested = raid10_congested, 4891 }; 4892 4893 static int __init raid_init(void) 4894 { 4895 return register_md_personality(&raid10_personality); 4896 } 4897 4898 static void raid_exit(void) 4899 { 4900 unregister_md_personality(&raid10_personality); 4901 } 4902 4903 module_init(raid_init); 4904 module_exit(raid_exit); 4905 MODULE_LICENSE("GPL"); 4906 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 4907 MODULE_ALIAS("md-personality-9"); /* RAID10 */ 4908 MODULE_ALIAS("md-raid10"); 4909 MODULE_ALIAS("md-level-10"); 4910 4911 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 4912