1 /* 2 * raid10.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 2000-2004 Neil Brown 5 * 6 * RAID-10 support for md. 7 * 8 * Base on code in raid1.c. See raid1.c for further copyright information. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 #include <linux/slab.h> 22 #include <linux/delay.h> 23 #include <linux/blkdev.h> 24 #include <linux/module.h> 25 #include <linux/seq_file.h> 26 #include <linux/ratelimit.h> 27 #include <linux/kthread.h> 28 #include "md.h" 29 #include "raid10.h" 30 #include "raid0.h" 31 #include "bitmap.h" 32 33 /* 34 * RAID10 provides a combination of RAID0 and RAID1 functionality. 35 * The layout of data is defined by 36 * chunk_size 37 * raid_disks 38 * near_copies (stored in low byte of layout) 39 * far_copies (stored in second byte of layout) 40 * far_offset (stored in bit 16 of layout ) 41 * use_far_sets (stored in bit 17 of layout ) 42 * 43 * The data to be stored is divided into chunks using chunksize. Each device 44 * is divided into far_copies sections. In each section, chunks are laid out 45 * in a style similar to raid0, but near_copies copies of each chunk is stored 46 * (each on a different drive). The starting device for each section is offset 47 * near_copies from the starting device of the previous section. Thus there 48 * are (near_copies * far_copies) of each chunk, and each is on a different 49 * drive. near_copies and far_copies must be at least one, and their product 50 * is at most raid_disks. 51 * 52 * If far_offset is true, then the far_copies are handled a bit differently. 53 * The copies are still in different stripes, but instead of being very far 54 * apart on disk, there are adjacent stripes. 55 * 56 * The far and offset algorithms are handled slightly differently if 57 * 'use_far_sets' is true. In this case, the array's devices are grouped into 58 * sets that are (near_copies * far_copies) in size. The far copied stripes 59 * are still shifted by 'near_copies' devices, but this shifting stays confined 60 * to the set rather than the entire array. This is done to improve the number 61 * of device combinations that can fail without causing the array to fail. 62 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 63 * on a device): 64 * A B C D A B C D E 65 * ... ... 66 * D A B C E A B C D 67 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 68 * [A B] [C D] [A B] [C D E] 69 * |...| |...| |...| | ... | 70 * [B A] [D C] [B A] [E C D] 71 */ 72 73 /* 74 * Number of guaranteed r10bios in case of extreme VM load: 75 */ 76 #define NR_RAID10_BIOS 256 77 78 /* when we get a read error on a read-only array, we redirect to another 79 * device without failing the first device, or trying to over-write to 80 * correct the read error. To keep track of bad blocks on a per-bio 81 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 82 */ 83 #define IO_BLOCKED ((struct bio *)1) 84 /* When we successfully write to a known bad-block, we need to remove the 85 * bad-block marking which must be done from process context. So we record 86 * the success by setting devs[n].bio to IO_MADE_GOOD 87 */ 88 #define IO_MADE_GOOD ((struct bio *)2) 89 90 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 91 92 /* When there are this many requests queued to be written by 93 * the raid10 thread, we become 'congested' to provide back-pressure 94 * for writeback. 95 */ 96 static int max_queued_requests = 1024; 97 98 static void allow_barrier(struct r10conf *conf); 99 static void lower_barrier(struct r10conf *conf); 100 static int _enough(struct r10conf *conf, int previous, int ignore); 101 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 102 int *skipped); 103 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 104 static void end_reshape_write(struct bio *bio); 105 static void end_reshape(struct r10conf *conf); 106 107 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 108 { 109 struct r10conf *conf = data; 110 int size = offsetof(struct r10bio, devs[conf->copies]); 111 112 /* allocate a r10bio with room for raid_disks entries in the 113 * bios array */ 114 return kzalloc(size, gfp_flags); 115 } 116 117 static void r10bio_pool_free(void *r10_bio, void *data) 118 { 119 kfree(r10_bio); 120 } 121 122 /* Maximum size of each resync request */ 123 #define RESYNC_BLOCK_SIZE (64*1024) 124 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 125 /* amount of memory to reserve for resync requests */ 126 #define RESYNC_WINDOW (1024*1024) 127 /* maximum number of concurrent requests, memory permitting */ 128 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 129 130 /* 131 * When performing a resync, we need to read and compare, so 132 * we need as many pages are there are copies. 133 * When performing a recovery, we need 2 bios, one for read, 134 * one for write (we recover only one drive per r10buf) 135 * 136 */ 137 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 138 { 139 struct r10conf *conf = data; 140 struct page *page; 141 struct r10bio *r10_bio; 142 struct bio *bio; 143 int i, j; 144 int nalloc; 145 146 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 147 if (!r10_bio) 148 return NULL; 149 150 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 151 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 152 nalloc = conf->copies; /* resync */ 153 else 154 nalloc = 2; /* recovery */ 155 156 /* 157 * Allocate bios. 158 */ 159 for (j = nalloc ; j-- ; ) { 160 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 161 if (!bio) 162 goto out_free_bio; 163 r10_bio->devs[j].bio = bio; 164 if (!conf->have_replacement) 165 continue; 166 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 167 if (!bio) 168 goto out_free_bio; 169 r10_bio->devs[j].repl_bio = bio; 170 } 171 /* 172 * Allocate RESYNC_PAGES data pages and attach them 173 * where needed. 174 */ 175 for (j = 0 ; j < nalloc; j++) { 176 struct bio *rbio = r10_bio->devs[j].repl_bio; 177 bio = r10_bio->devs[j].bio; 178 for (i = 0; i < RESYNC_PAGES; i++) { 179 if (j > 0 && !test_bit(MD_RECOVERY_SYNC, 180 &conf->mddev->recovery)) { 181 /* we can share bv_page's during recovery 182 * and reshape */ 183 struct bio *rbio = r10_bio->devs[0].bio; 184 page = rbio->bi_io_vec[i].bv_page; 185 get_page(page); 186 } else 187 page = alloc_page(gfp_flags); 188 if (unlikely(!page)) 189 goto out_free_pages; 190 191 bio->bi_io_vec[i].bv_page = page; 192 if (rbio) 193 rbio->bi_io_vec[i].bv_page = page; 194 } 195 } 196 197 return r10_bio; 198 199 out_free_pages: 200 for ( ; i > 0 ; i--) 201 safe_put_page(bio->bi_io_vec[i-1].bv_page); 202 while (j--) 203 for (i = 0; i < RESYNC_PAGES ; i++) 204 safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page); 205 j = 0; 206 out_free_bio: 207 for ( ; j < nalloc; j++) { 208 if (r10_bio->devs[j].bio) 209 bio_put(r10_bio->devs[j].bio); 210 if (r10_bio->devs[j].repl_bio) 211 bio_put(r10_bio->devs[j].repl_bio); 212 } 213 r10bio_pool_free(r10_bio, conf); 214 return NULL; 215 } 216 217 static void r10buf_pool_free(void *__r10_bio, void *data) 218 { 219 int i; 220 struct r10conf *conf = data; 221 struct r10bio *r10bio = __r10_bio; 222 int j; 223 224 for (j=0; j < conf->copies; j++) { 225 struct bio *bio = r10bio->devs[j].bio; 226 if (bio) { 227 for (i = 0; i < RESYNC_PAGES; i++) { 228 safe_put_page(bio->bi_io_vec[i].bv_page); 229 bio->bi_io_vec[i].bv_page = NULL; 230 } 231 bio_put(bio); 232 } 233 bio = r10bio->devs[j].repl_bio; 234 if (bio) 235 bio_put(bio); 236 } 237 r10bio_pool_free(r10bio, conf); 238 } 239 240 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 241 { 242 int i; 243 244 for (i = 0; i < conf->copies; i++) { 245 struct bio **bio = & r10_bio->devs[i].bio; 246 if (!BIO_SPECIAL(*bio)) 247 bio_put(*bio); 248 *bio = NULL; 249 bio = &r10_bio->devs[i].repl_bio; 250 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 251 bio_put(*bio); 252 *bio = NULL; 253 } 254 } 255 256 static void free_r10bio(struct r10bio *r10_bio) 257 { 258 struct r10conf *conf = r10_bio->mddev->private; 259 260 put_all_bios(conf, r10_bio); 261 mempool_free(r10_bio, conf->r10bio_pool); 262 } 263 264 static void put_buf(struct r10bio *r10_bio) 265 { 266 struct r10conf *conf = r10_bio->mddev->private; 267 268 mempool_free(r10_bio, conf->r10buf_pool); 269 270 lower_barrier(conf); 271 } 272 273 static void reschedule_retry(struct r10bio *r10_bio) 274 { 275 unsigned long flags; 276 struct mddev *mddev = r10_bio->mddev; 277 struct r10conf *conf = mddev->private; 278 279 spin_lock_irqsave(&conf->device_lock, flags); 280 list_add(&r10_bio->retry_list, &conf->retry_list); 281 conf->nr_queued ++; 282 spin_unlock_irqrestore(&conf->device_lock, flags); 283 284 /* wake up frozen array... */ 285 wake_up(&conf->wait_barrier); 286 287 md_wakeup_thread(mddev->thread); 288 } 289 290 /* 291 * raid_end_bio_io() is called when we have finished servicing a mirrored 292 * operation and are ready to return a success/failure code to the buffer 293 * cache layer. 294 */ 295 static void raid_end_bio_io(struct r10bio *r10_bio) 296 { 297 struct bio *bio = r10_bio->master_bio; 298 int done; 299 struct r10conf *conf = r10_bio->mddev->private; 300 301 if (bio->bi_phys_segments) { 302 unsigned long flags; 303 spin_lock_irqsave(&conf->device_lock, flags); 304 bio->bi_phys_segments--; 305 done = (bio->bi_phys_segments == 0); 306 spin_unlock_irqrestore(&conf->device_lock, flags); 307 } else 308 done = 1; 309 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 310 bio->bi_error = -EIO; 311 if (done) { 312 bio_endio(bio); 313 /* 314 * Wake up any possible resync thread that waits for the device 315 * to go idle. 316 */ 317 allow_barrier(conf); 318 } 319 free_r10bio(r10_bio); 320 } 321 322 /* 323 * Update disk head position estimator based on IRQ completion info. 324 */ 325 static inline void update_head_pos(int slot, struct r10bio *r10_bio) 326 { 327 struct r10conf *conf = r10_bio->mddev->private; 328 329 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 330 r10_bio->devs[slot].addr + (r10_bio->sectors); 331 } 332 333 /* 334 * Find the disk number which triggered given bio 335 */ 336 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 337 struct bio *bio, int *slotp, int *replp) 338 { 339 int slot; 340 int repl = 0; 341 342 for (slot = 0; slot < conf->copies; slot++) { 343 if (r10_bio->devs[slot].bio == bio) 344 break; 345 if (r10_bio->devs[slot].repl_bio == bio) { 346 repl = 1; 347 break; 348 } 349 } 350 351 BUG_ON(slot == conf->copies); 352 update_head_pos(slot, r10_bio); 353 354 if (slotp) 355 *slotp = slot; 356 if (replp) 357 *replp = repl; 358 return r10_bio->devs[slot].devnum; 359 } 360 361 static void raid10_end_read_request(struct bio *bio) 362 { 363 int uptodate = !bio->bi_error; 364 struct r10bio *r10_bio = bio->bi_private; 365 int slot, dev; 366 struct md_rdev *rdev; 367 struct r10conf *conf = r10_bio->mddev->private; 368 369 slot = r10_bio->read_slot; 370 dev = r10_bio->devs[slot].devnum; 371 rdev = r10_bio->devs[slot].rdev; 372 /* 373 * this branch is our 'one mirror IO has finished' event handler: 374 */ 375 update_head_pos(slot, r10_bio); 376 377 if (uptodate) { 378 /* 379 * Set R10BIO_Uptodate in our master bio, so that 380 * we will return a good error code to the higher 381 * levels even if IO on some other mirrored buffer fails. 382 * 383 * The 'master' represents the composite IO operation to 384 * user-side. So if something waits for IO, then it will 385 * wait for the 'master' bio. 386 */ 387 set_bit(R10BIO_Uptodate, &r10_bio->state); 388 } else { 389 /* If all other devices that store this block have 390 * failed, we want to return the error upwards rather 391 * than fail the last device. Here we redefine 392 * "uptodate" to mean "Don't want to retry" 393 */ 394 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), 395 rdev->raid_disk)) 396 uptodate = 1; 397 } 398 if (uptodate) { 399 raid_end_bio_io(r10_bio); 400 rdev_dec_pending(rdev, conf->mddev); 401 } else { 402 /* 403 * oops, read error - keep the refcount on the rdev 404 */ 405 char b[BDEVNAME_SIZE]; 406 printk_ratelimited(KERN_ERR 407 "md/raid10:%s: %s: rescheduling sector %llu\n", 408 mdname(conf->mddev), 409 bdevname(rdev->bdev, b), 410 (unsigned long long)r10_bio->sector); 411 set_bit(R10BIO_ReadError, &r10_bio->state); 412 reschedule_retry(r10_bio); 413 } 414 } 415 416 static void close_write(struct r10bio *r10_bio) 417 { 418 /* clear the bitmap if all writes complete successfully */ 419 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector, 420 r10_bio->sectors, 421 !test_bit(R10BIO_Degraded, &r10_bio->state), 422 0); 423 md_write_end(r10_bio->mddev); 424 } 425 426 static void one_write_done(struct r10bio *r10_bio) 427 { 428 if (atomic_dec_and_test(&r10_bio->remaining)) { 429 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 430 reschedule_retry(r10_bio); 431 else { 432 close_write(r10_bio); 433 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 434 reschedule_retry(r10_bio); 435 else 436 raid_end_bio_io(r10_bio); 437 } 438 } 439 } 440 441 static void raid10_end_write_request(struct bio *bio) 442 { 443 struct r10bio *r10_bio = bio->bi_private; 444 int dev; 445 int dec_rdev = 1; 446 struct r10conf *conf = r10_bio->mddev->private; 447 int slot, repl; 448 struct md_rdev *rdev = NULL; 449 450 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 451 452 if (repl) 453 rdev = conf->mirrors[dev].replacement; 454 if (!rdev) { 455 smp_rmb(); 456 repl = 0; 457 rdev = conf->mirrors[dev].rdev; 458 } 459 /* 460 * this branch is our 'one mirror IO has finished' event handler: 461 */ 462 if (bio->bi_error) { 463 if (repl) 464 /* Never record new bad blocks to replacement, 465 * just fail it. 466 */ 467 md_error(rdev->mddev, rdev); 468 else { 469 set_bit(WriteErrorSeen, &rdev->flags); 470 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 471 set_bit(MD_RECOVERY_NEEDED, 472 &rdev->mddev->recovery); 473 set_bit(R10BIO_WriteError, &r10_bio->state); 474 dec_rdev = 0; 475 } 476 } else { 477 /* 478 * Set R10BIO_Uptodate in our master bio, so that 479 * we will return a good error code for to the higher 480 * levels even if IO on some other mirrored buffer fails. 481 * 482 * The 'master' represents the composite IO operation to 483 * user-side. So if something waits for IO, then it will 484 * wait for the 'master' bio. 485 */ 486 sector_t first_bad; 487 int bad_sectors; 488 489 /* 490 * Do not set R10BIO_Uptodate if the current device is 491 * rebuilding or Faulty. This is because we cannot use 492 * such device for properly reading the data back (we could 493 * potentially use it, if the current write would have felt 494 * before rdev->recovery_offset, but for simplicity we don't 495 * check this here. 496 */ 497 if (test_bit(In_sync, &rdev->flags) && 498 !test_bit(Faulty, &rdev->flags)) 499 set_bit(R10BIO_Uptodate, &r10_bio->state); 500 501 /* Maybe we can clear some bad blocks. */ 502 if (is_badblock(rdev, 503 r10_bio->devs[slot].addr, 504 r10_bio->sectors, 505 &first_bad, &bad_sectors)) { 506 bio_put(bio); 507 if (repl) 508 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 509 else 510 r10_bio->devs[slot].bio = IO_MADE_GOOD; 511 dec_rdev = 0; 512 set_bit(R10BIO_MadeGood, &r10_bio->state); 513 } 514 } 515 516 /* 517 * 518 * Let's see if all mirrored write operations have finished 519 * already. 520 */ 521 one_write_done(r10_bio); 522 if (dec_rdev) 523 rdev_dec_pending(rdev, conf->mddev); 524 } 525 526 /* 527 * RAID10 layout manager 528 * As well as the chunksize and raid_disks count, there are two 529 * parameters: near_copies and far_copies. 530 * near_copies * far_copies must be <= raid_disks. 531 * Normally one of these will be 1. 532 * If both are 1, we get raid0. 533 * If near_copies == raid_disks, we get raid1. 534 * 535 * Chunks are laid out in raid0 style with near_copies copies of the 536 * first chunk, followed by near_copies copies of the next chunk and 537 * so on. 538 * If far_copies > 1, then after 1/far_copies of the array has been assigned 539 * as described above, we start again with a device offset of near_copies. 540 * So we effectively have another copy of the whole array further down all 541 * the drives, but with blocks on different drives. 542 * With this layout, and block is never stored twice on the one device. 543 * 544 * raid10_find_phys finds the sector offset of a given virtual sector 545 * on each device that it is on. 546 * 547 * raid10_find_virt does the reverse mapping, from a device and a 548 * sector offset to a virtual address 549 */ 550 551 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 552 { 553 int n,f; 554 sector_t sector; 555 sector_t chunk; 556 sector_t stripe; 557 int dev; 558 int slot = 0; 559 int last_far_set_start, last_far_set_size; 560 561 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 562 last_far_set_start *= geo->far_set_size; 563 564 last_far_set_size = geo->far_set_size; 565 last_far_set_size += (geo->raid_disks % geo->far_set_size); 566 567 /* now calculate first sector/dev */ 568 chunk = r10bio->sector >> geo->chunk_shift; 569 sector = r10bio->sector & geo->chunk_mask; 570 571 chunk *= geo->near_copies; 572 stripe = chunk; 573 dev = sector_div(stripe, geo->raid_disks); 574 if (geo->far_offset) 575 stripe *= geo->far_copies; 576 577 sector += stripe << geo->chunk_shift; 578 579 /* and calculate all the others */ 580 for (n = 0; n < geo->near_copies; n++) { 581 int d = dev; 582 int set; 583 sector_t s = sector; 584 r10bio->devs[slot].devnum = d; 585 r10bio->devs[slot].addr = s; 586 slot++; 587 588 for (f = 1; f < geo->far_copies; f++) { 589 set = d / geo->far_set_size; 590 d += geo->near_copies; 591 592 if ((geo->raid_disks % geo->far_set_size) && 593 (d > last_far_set_start)) { 594 d -= last_far_set_start; 595 d %= last_far_set_size; 596 d += last_far_set_start; 597 } else { 598 d %= geo->far_set_size; 599 d += geo->far_set_size * set; 600 } 601 s += geo->stride; 602 r10bio->devs[slot].devnum = d; 603 r10bio->devs[slot].addr = s; 604 slot++; 605 } 606 dev++; 607 if (dev >= geo->raid_disks) { 608 dev = 0; 609 sector += (geo->chunk_mask + 1); 610 } 611 } 612 } 613 614 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 615 { 616 struct geom *geo = &conf->geo; 617 618 if (conf->reshape_progress != MaxSector && 619 ((r10bio->sector >= conf->reshape_progress) != 620 conf->mddev->reshape_backwards)) { 621 set_bit(R10BIO_Previous, &r10bio->state); 622 geo = &conf->prev; 623 } else 624 clear_bit(R10BIO_Previous, &r10bio->state); 625 626 __raid10_find_phys(geo, r10bio); 627 } 628 629 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 630 { 631 sector_t offset, chunk, vchunk; 632 /* Never use conf->prev as this is only called during resync 633 * or recovery, so reshape isn't happening 634 */ 635 struct geom *geo = &conf->geo; 636 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 637 int far_set_size = geo->far_set_size; 638 int last_far_set_start; 639 640 if (geo->raid_disks % geo->far_set_size) { 641 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 642 last_far_set_start *= geo->far_set_size; 643 644 if (dev >= last_far_set_start) { 645 far_set_size = geo->far_set_size; 646 far_set_size += (geo->raid_disks % geo->far_set_size); 647 far_set_start = last_far_set_start; 648 } 649 } 650 651 offset = sector & geo->chunk_mask; 652 if (geo->far_offset) { 653 int fc; 654 chunk = sector >> geo->chunk_shift; 655 fc = sector_div(chunk, geo->far_copies); 656 dev -= fc * geo->near_copies; 657 if (dev < far_set_start) 658 dev += far_set_size; 659 } else { 660 while (sector >= geo->stride) { 661 sector -= geo->stride; 662 if (dev < (geo->near_copies + far_set_start)) 663 dev += far_set_size - geo->near_copies; 664 else 665 dev -= geo->near_copies; 666 } 667 chunk = sector >> geo->chunk_shift; 668 } 669 vchunk = chunk * geo->raid_disks + dev; 670 sector_div(vchunk, geo->near_copies); 671 return (vchunk << geo->chunk_shift) + offset; 672 } 673 674 /* 675 * This routine returns the disk from which the requested read should 676 * be done. There is a per-array 'next expected sequential IO' sector 677 * number - if this matches on the next IO then we use the last disk. 678 * There is also a per-disk 'last know head position' sector that is 679 * maintained from IRQ contexts, both the normal and the resync IO 680 * completion handlers update this position correctly. If there is no 681 * perfect sequential match then we pick the disk whose head is closest. 682 * 683 * If there are 2 mirrors in the same 2 devices, performance degrades 684 * because position is mirror, not device based. 685 * 686 * The rdev for the device selected will have nr_pending incremented. 687 */ 688 689 /* 690 * FIXME: possibly should rethink readbalancing and do it differently 691 * depending on near_copies / far_copies geometry. 692 */ 693 static struct md_rdev *read_balance(struct r10conf *conf, 694 struct r10bio *r10_bio, 695 int *max_sectors) 696 { 697 const sector_t this_sector = r10_bio->sector; 698 int disk, slot; 699 int sectors = r10_bio->sectors; 700 int best_good_sectors; 701 sector_t new_distance, best_dist; 702 struct md_rdev *best_rdev, *rdev = NULL; 703 int do_balance; 704 int best_slot; 705 struct geom *geo = &conf->geo; 706 707 raid10_find_phys(conf, r10_bio); 708 rcu_read_lock(); 709 retry: 710 sectors = r10_bio->sectors; 711 best_slot = -1; 712 best_rdev = NULL; 713 best_dist = MaxSector; 714 best_good_sectors = 0; 715 do_balance = 1; 716 /* 717 * Check if we can balance. We can balance on the whole 718 * device if no resync is going on (recovery is ok), or below 719 * the resync window. We take the first readable disk when 720 * above the resync window. 721 */ 722 if (conf->mddev->recovery_cp < MaxSector 723 && (this_sector + sectors >= conf->next_resync)) 724 do_balance = 0; 725 726 for (slot = 0; slot < conf->copies ; slot++) { 727 sector_t first_bad; 728 int bad_sectors; 729 sector_t dev_sector; 730 731 if (r10_bio->devs[slot].bio == IO_BLOCKED) 732 continue; 733 disk = r10_bio->devs[slot].devnum; 734 rdev = rcu_dereference(conf->mirrors[disk].replacement); 735 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 736 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 737 rdev = rcu_dereference(conf->mirrors[disk].rdev); 738 if (rdev == NULL || 739 test_bit(Faulty, &rdev->flags)) 740 continue; 741 if (!test_bit(In_sync, &rdev->flags) && 742 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 743 continue; 744 745 dev_sector = r10_bio->devs[slot].addr; 746 if (is_badblock(rdev, dev_sector, sectors, 747 &first_bad, &bad_sectors)) { 748 if (best_dist < MaxSector) 749 /* Already have a better slot */ 750 continue; 751 if (first_bad <= dev_sector) { 752 /* Cannot read here. If this is the 753 * 'primary' device, then we must not read 754 * beyond 'bad_sectors' from another device. 755 */ 756 bad_sectors -= (dev_sector - first_bad); 757 if (!do_balance && sectors > bad_sectors) 758 sectors = bad_sectors; 759 if (best_good_sectors > sectors) 760 best_good_sectors = sectors; 761 } else { 762 sector_t good_sectors = 763 first_bad - dev_sector; 764 if (good_sectors > best_good_sectors) { 765 best_good_sectors = good_sectors; 766 best_slot = slot; 767 best_rdev = rdev; 768 } 769 if (!do_balance) 770 /* Must read from here */ 771 break; 772 } 773 continue; 774 } else 775 best_good_sectors = sectors; 776 777 if (!do_balance) 778 break; 779 780 /* This optimisation is debatable, and completely destroys 781 * sequential read speed for 'far copies' arrays. So only 782 * keep it for 'near' arrays, and review those later. 783 */ 784 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending)) 785 break; 786 787 /* for far > 1 always use the lowest address */ 788 if (geo->far_copies > 1) 789 new_distance = r10_bio->devs[slot].addr; 790 else 791 new_distance = abs(r10_bio->devs[slot].addr - 792 conf->mirrors[disk].head_position); 793 if (new_distance < best_dist) { 794 best_dist = new_distance; 795 best_slot = slot; 796 best_rdev = rdev; 797 } 798 } 799 if (slot >= conf->copies) { 800 slot = best_slot; 801 rdev = best_rdev; 802 } 803 804 if (slot >= 0) { 805 atomic_inc(&rdev->nr_pending); 806 if (test_bit(Faulty, &rdev->flags)) { 807 /* Cannot risk returning a device that failed 808 * before we inc'ed nr_pending 809 */ 810 rdev_dec_pending(rdev, conf->mddev); 811 goto retry; 812 } 813 r10_bio->read_slot = slot; 814 } else 815 rdev = NULL; 816 rcu_read_unlock(); 817 *max_sectors = best_good_sectors; 818 819 return rdev; 820 } 821 822 static int raid10_congested(struct mddev *mddev, int bits) 823 { 824 struct r10conf *conf = mddev->private; 825 int i, ret = 0; 826 827 if ((bits & (1 << WB_async_congested)) && 828 conf->pending_count >= max_queued_requests) 829 return 1; 830 831 rcu_read_lock(); 832 for (i = 0; 833 (i < conf->geo.raid_disks || i < conf->prev.raid_disks) 834 && ret == 0; 835 i++) { 836 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 837 if (rdev && !test_bit(Faulty, &rdev->flags)) { 838 struct request_queue *q = bdev_get_queue(rdev->bdev); 839 840 ret |= bdi_congested(&q->backing_dev_info, bits); 841 } 842 } 843 rcu_read_unlock(); 844 return ret; 845 } 846 847 static void flush_pending_writes(struct r10conf *conf) 848 { 849 /* Any writes that have been queued but are awaiting 850 * bitmap updates get flushed here. 851 */ 852 spin_lock_irq(&conf->device_lock); 853 854 if (conf->pending_bio_list.head) { 855 struct bio *bio; 856 bio = bio_list_get(&conf->pending_bio_list); 857 conf->pending_count = 0; 858 spin_unlock_irq(&conf->device_lock); 859 /* flush any pending bitmap writes to disk 860 * before proceeding w/ I/O */ 861 bitmap_unplug(conf->mddev->bitmap); 862 wake_up(&conf->wait_barrier); 863 864 while (bio) { /* submit pending writes */ 865 struct bio *next = bio->bi_next; 866 bio->bi_next = NULL; 867 if (unlikely((bio->bi_rw & REQ_DISCARD) && 868 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 869 /* Just ignore it */ 870 bio_endio(bio); 871 else 872 generic_make_request(bio); 873 bio = next; 874 } 875 } else 876 spin_unlock_irq(&conf->device_lock); 877 } 878 879 /* Barriers.... 880 * Sometimes we need to suspend IO while we do something else, 881 * either some resync/recovery, or reconfigure the array. 882 * To do this we raise a 'barrier'. 883 * The 'barrier' is a counter that can be raised multiple times 884 * to count how many activities are happening which preclude 885 * normal IO. 886 * We can only raise the barrier if there is no pending IO. 887 * i.e. if nr_pending == 0. 888 * We choose only to raise the barrier if no-one is waiting for the 889 * barrier to go down. This means that as soon as an IO request 890 * is ready, no other operations which require a barrier will start 891 * until the IO request has had a chance. 892 * 893 * So: regular IO calls 'wait_barrier'. When that returns there 894 * is no backgroup IO happening, It must arrange to call 895 * allow_barrier when it has finished its IO. 896 * backgroup IO calls must call raise_barrier. Once that returns 897 * there is no normal IO happeing. It must arrange to call 898 * lower_barrier when the particular background IO completes. 899 */ 900 901 static void raise_barrier(struct r10conf *conf, int force) 902 { 903 BUG_ON(force && !conf->barrier); 904 spin_lock_irq(&conf->resync_lock); 905 906 /* Wait until no block IO is waiting (unless 'force') */ 907 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, 908 conf->resync_lock); 909 910 /* block any new IO from starting */ 911 conf->barrier++; 912 913 /* Now wait for all pending IO to complete */ 914 wait_event_lock_irq(conf->wait_barrier, 915 !conf->nr_pending && conf->barrier < RESYNC_DEPTH, 916 conf->resync_lock); 917 918 spin_unlock_irq(&conf->resync_lock); 919 } 920 921 static void lower_barrier(struct r10conf *conf) 922 { 923 unsigned long flags; 924 spin_lock_irqsave(&conf->resync_lock, flags); 925 conf->barrier--; 926 spin_unlock_irqrestore(&conf->resync_lock, flags); 927 wake_up(&conf->wait_barrier); 928 } 929 930 static void wait_barrier(struct r10conf *conf) 931 { 932 spin_lock_irq(&conf->resync_lock); 933 if (conf->barrier) { 934 conf->nr_waiting++; 935 /* Wait for the barrier to drop. 936 * However if there are already pending 937 * requests (preventing the barrier from 938 * rising completely), and the 939 * pre-process bio queue isn't empty, 940 * then don't wait, as we need to empty 941 * that queue to get the nr_pending 942 * count down. 943 */ 944 wait_event_lock_irq(conf->wait_barrier, 945 !conf->barrier || 946 (conf->nr_pending && 947 current->bio_list && 948 !bio_list_empty(current->bio_list)), 949 conf->resync_lock); 950 conf->nr_waiting--; 951 } 952 conf->nr_pending++; 953 spin_unlock_irq(&conf->resync_lock); 954 } 955 956 static void allow_barrier(struct r10conf *conf) 957 { 958 unsigned long flags; 959 spin_lock_irqsave(&conf->resync_lock, flags); 960 conf->nr_pending--; 961 spin_unlock_irqrestore(&conf->resync_lock, flags); 962 wake_up(&conf->wait_barrier); 963 } 964 965 static void freeze_array(struct r10conf *conf, int extra) 966 { 967 /* stop syncio and normal IO and wait for everything to 968 * go quiet. 969 * We increment barrier and nr_waiting, and then 970 * wait until nr_pending match nr_queued+extra 971 * This is called in the context of one normal IO request 972 * that has failed. Thus any sync request that might be pending 973 * will be blocked by nr_pending, and we need to wait for 974 * pending IO requests to complete or be queued for re-try. 975 * Thus the number queued (nr_queued) plus this request (extra) 976 * must match the number of pending IOs (nr_pending) before 977 * we continue. 978 */ 979 spin_lock_irq(&conf->resync_lock); 980 conf->barrier++; 981 conf->nr_waiting++; 982 wait_event_lock_irq_cmd(conf->wait_barrier, 983 conf->nr_pending == conf->nr_queued+extra, 984 conf->resync_lock, 985 flush_pending_writes(conf)); 986 987 spin_unlock_irq(&conf->resync_lock); 988 } 989 990 static void unfreeze_array(struct r10conf *conf) 991 { 992 /* reverse the effect of the freeze */ 993 spin_lock_irq(&conf->resync_lock); 994 conf->barrier--; 995 conf->nr_waiting--; 996 wake_up(&conf->wait_barrier); 997 spin_unlock_irq(&conf->resync_lock); 998 } 999 1000 static sector_t choose_data_offset(struct r10bio *r10_bio, 1001 struct md_rdev *rdev) 1002 { 1003 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 1004 test_bit(R10BIO_Previous, &r10_bio->state)) 1005 return rdev->data_offset; 1006 else 1007 return rdev->new_data_offset; 1008 } 1009 1010 struct raid10_plug_cb { 1011 struct blk_plug_cb cb; 1012 struct bio_list pending; 1013 int pending_cnt; 1014 }; 1015 1016 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1017 { 1018 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb, 1019 cb); 1020 struct mddev *mddev = plug->cb.data; 1021 struct r10conf *conf = mddev->private; 1022 struct bio *bio; 1023 1024 if (from_schedule || current->bio_list) { 1025 spin_lock_irq(&conf->device_lock); 1026 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1027 conf->pending_count += plug->pending_cnt; 1028 spin_unlock_irq(&conf->device_lock); 1029 wake_up(&conf->wait_barrier); 1030 md_wakeup_thread(mddev->thread); 1031 kfree(plug); 1032 return; 1033 } 1034 1035 /* we aren't scheduling, so we can do the write-out directly. */ 1036 bio = bio_list_get(&plug->pending); 1037 bitmap_unplug(mddev->bitmap); 1038 wake_up(&conf->wait_barrier); 1039 1040 while (bio) { /* submit pending writes */ 1041 struct bio *next = bio->bi_next; 1042 bio->bi_next = NULL; 1043 if (unlikely((bio->bi_rw & REQ_DISCARD) && 1044 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1045 /* Just ignore it */ 1046 bio_endio(bio); 1047 else 1048 generic_make_request(bio); 1049 bio = next; 1050 } 1051 kfree(plug); 1052 } 1053 1054 static void __make_request(struct mddev *mddev, struct bio *bio) 1055 { 1056 struct r10conf *conf = mddev->private; 1057 struct r10bio *r10_bio; 1058 struct bio *read_bio; 1059 int i; 1060 const int rw = bio_data_dir(bio); 1061 const unsigned long do_sync = (bio->bi_rw & REQ_SYNC); 1062 const unsigned long do_fua = (bio->bi_rw & REQ_FUA); 1063 const unsigned long do_discard = (bio->bi_rw 1064 & (REQ_DISCARD | REQ_SECURE)); 1065 const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME); 1066 unsigned long flags; 1067 struct md_rdev *blocked_rdev; 1068 struct blk_plug_cb *cb; 1069 struct raid10_plug_cb *plug = NULL; 1070 int sectors_handled; 1071 int max_sectors; 1072 int sectors; 1073 1074 /* 1075 * Register the new request and wait if the reconstruction 1076 * thread has put up a bar for new requests. 1077 * Continue immediately if no resync is active currently. 1078 */ 1079 wait_barrier(conf); 1080 1081 sectors = bio_sectors(bio); 1082 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1083 bio->bi_iter.bi_sector < conf->reshape_progress && 1084 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1085 /* IO spans the reshape position. Need to wait for 1086 * reshape to pass 1087 */ 1088 allow_barrier(conf); 1089 wait_event(conf->wait_barrier, 1090 conf->reshape_progress <= bio->bi_iter.bi_sector || 1091 conf->reshape_progress >= bio->bi_iter.bi_sector + 1092 sectors); 1093 wait_barrier(conf); 1094 } 1095 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1096 bio_data_dir(bio) == WRITE && 1097 (mddev->reshape_backwards 1098 ? (bio->bi_iter.bi_sector < conf->reshape_safe && 1099 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) 1100 : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe && 1101 bio->bi_iter.bi_sector < conf->reshape_progress))) { 1102 /* Need to update reshape_position in metadata */ 1103 mddev->reshape_position = conf->reshape_progress; 1104 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1105 set_bit(MD_CHANGE_PENDING, &mddev->flags); 1106 md_wakeup_thread(mddev->thread); 1107 wait_event(mddev->sb_wait, 1108 !test_bit(MD_CHANGE_PENDING, &mddev->flags)); 1109 1110 conf->reshape_safe = mddev->reshape_position; 1111 } 1112 1113 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1114 1115 r10_bio->master_bio = bio; 1116 r10_bio->sectors = sectors; 1117 1118 r10_bio->mddev = mddev; 1119 r10_bio->sector = bio->bi_iter.bi_sector; 1120 r10_bio->state = 0; 1121 1122 /* We might need to issue multiple reads to different 1123 * devices if there are bad blocks around, so we keep 1124 * track of the number of reads in bio->bi_phys_segments. 1125 * If this is 0, there is only one r10_bio and no locking 1126 * will be needed when the request completes. If it is 1127 * non-zero, then it is the number of not-completed requests. 1128 */ 1129 bio->bi_phys_segments = 0; 1130 bio_clear_flag(bio, BIO_SEG_VALID); 1131 1132 if (rw == READ) { 1133 /* 1134 * read balancing logic: 1135 */ 1136 struct md_rdev *rdev; 1137 int slot; 1138 1139 read_again: 1140 rdev = read_balance(conf, r10_bio, &max_sectors); 1141 if (!rdev) { 1142 raid_end_bio_io(r10_bio); 1143 return; 1144 } 1145 slot = r10_bio->read_slot; 1146 1147 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1148 bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector, 1149 max_sectors); 1150 1151 r10_bio->devs[slot].bio = read_bio; 1152 r10_bio->devs[slot].rdev = rdev; 1153 1154 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + 1155 choose_data_offset(r10_bio, rdev); 1156 read_bio->bi_bdev = rdev->bdev; 1157 read_bio->bi_end_io = raid10_end_read_request; 1158 read_bio->bi_rw = READ | do_sync; 1159 read_bio->bi_private = r10_bio; 1160 1161 if (max_sectors < r10_bio->sectors) { 1162 /* Could not read all from this device, so we will 1163 * need another r10_bio. 1164 */ 1165 sectors_handled = (r10_bio->sector + max_sectors 1166 - bio->bi_iter.bi_sector); 1167 r10_bio->sectors = max_sectors; 1168 spin_lock_irq(&conf->device_lock); 1169 if (bio->bi_phys_segments == 0) 1170 bio->bi_phys_segments = 2; 1171 else 1172 bio->bi_phys_segments++; 1173 spin_unlock_irq(&conf->device_lock); 1174 /* Cannot call generic_make_request directly 1175 * as that will be queued in __generic_make_request 1176 * and subsequent mempool_alloc might block 1177 * waiting for it. so hand bio over to raid10d. 1178 */ 1179 reschedule_retry(r10_bio); 1180 1181 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1182 1183 r10_bio->master_bio = bio; 1184 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1185 r10_bio->state = 0; 1186 r10_bio->mddev = mddev; 1187 r10_bio->sector = bio->bi_iter.bi_sector + 1188 sectors_handled; 1189 goto read_again; 1190 } else 1191 generic_make_request(read_bio); 1192 return; 1193 } 1194 1195 /* 1196 * WRITE: 1197 */ 1198 if (conf->pending_count >= max_queued_requests) { 1199 md_wakeup_thread(mddev->thread); 1200 wait_event(conf->wait_barrier, 1201 conf->pending_count < max_queued_requests); 1202 } 1203 /* first select target devices under rcu_lock and 1204 * inc refcount on their rdev. Record them by setting 1205 * bios[x] to bio 1206 * If there are known/acknowledged bad blocks on any device 1207 * on which we have seen a write error, we want to avoid 1208 * writing to those blocks. This potentially requires several 1209 * writes to write around the bad blocks. Each set of writes 1210 * gets its own r10_bio with a set of bios attached. The number 1211 * of r10_bios is recored in bio->bi_phys_segments just as with 1212 * the read case. 1213 */ 1214 1215 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1216 raid10_find_phys(conf, r10_bio); 1217 retry_write: 1218 blocked_rdev = NULL; 1219 rcu_read_lock(); 1220 max_sectors = r10_bio->sectors; 1221 1222 for (i = 0; i < conf->copies; i++) { 1223 int d = r10_bio->devs[i].devnum; 1224 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 1225 struct md_rdev *rrdev = rcu_dereference( 1226 conf->mirrors[d].replacement); 1227 if (rdev == rrdev) 1228 rrdev = NULL; 1229 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1230 atomic_inc(&rdev->nr_pending); 1231 blocked_rdev = rdev; 1232 break; 1233 } 1234 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) { 1235 atomic_inc(&rrdev->nr_pending); 1236 blocked_rdev = rrdev; 1237 break; 1238 } 1239 if (rdev && (test_bit(Faulty, &rdev->flags))) 1240 rdev = NULL; 1241 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1242 rrdev = NULL; 1243 1244 r10_bio->devs[i].bio = NULL; 1245 r10_bio->devs[i].repl_bio = NULL; 1246 1247 if (!rdev && !rrdev) { 1248 set_bit(R10BIO_Degraded, &r10_bio->state); 1249 continue; 1250 } 1251 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1252 sector_t first_bad; 1253 sector_t dev_sector = r10_bio->devs[i].addr; 1254 int bad_sectors; 1255 int is_bad; 1256 1257 is_bad = is_badblock(rdev, dev_sector, 1258 max_sectors, 1259 &first_bad, &bad_sectors); 1260 if (is_bad < 0) { 1261 /* Mustn't write here until the bad block 1262 * is acknowledged 1263 */ 1264 atomic_inc(&rdev->nr_pending); 1265 set_bit(BlockedBadBlocks, &rdev->flags); 1266 blocked_rdev = rdev; 1267 break; 1268 } 1269 if (is_bad && first_bad <= dev_sector) { 1270 /* Cannot write here at all */ 1271 bad_sectors -= (dev_sector - first_bad); 1272 if (bad_sectors < max_sectors) 1273 /* Mustn't write more than bad_sectors 1274 * to other devices yet 1275 */ 1276 max_sectors = bad_sectors; 1277 /* We don't set R10BIO_Degraded as that 1278 * only applies if the disk is missing, 1279 * so it might be re-added, and we want to 1280 * know to recover this chunk. 1281 * In this case the device is here, and the 1282 * fact that this chunk is not in-sync is 1283 * recorded in the bad block log. 1284 */ 1285 continue; 1286 } 1287 if (is_bad) { 1288 int good_sectors = first_bad - dev_sector; 1289 if (good_sectors < max_sectors) 1290 max_sectors = good_sectors; 1291 } 1292 } 1293 if (rdev) { 1294 r10_bio->devs[i].bio = bio; 1295 atomic_inc(&rdev->nr_pending); 1296 } 1297 if (rrdev) { 1298 r10_bio->devs[i].repl_bio = bio; 1299 atomic_inc(&rrdev->nr_pending); 1300 } 1301 } 1302 rcu_read_unlock(); 1303 1304 if (unlikely(blocked_rdev)) { 1305 /* Have to wait for this device to get unblocked, then retry */ 1306 int j; 1307 int d; 1308 1309 for (j = 0; j < i; j++) { 1310 if (r10_bio->devs[j].bio) { 1311 d = r10_bio->devs[j].devnum; 1312 rdev_dec_pending(conf->mirrors[d].rdev, mddev); 1313 } 1314 if (r10_bio->devs[j].repl_bio) { 1315 struct md_rdev *rdev; 1316 d = r10_bio->devs[j].devnum; 1317 rdev = conf->mirrors[d].replacement; 1318 if (!rdev) { 1319 /* Race with remove_disk */ 1320 smp_mb(); 1321 rdev = conf->mirrors[d].rdev; 1322 } 1323 rdev_dec_pending(rdev, mddev); 1324 } 1325 } 1326 allow_barrier(conf); 1327 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1328 wait_barrier(conf); 1329 goto retry_write; 1330 } 1331 1332 if (max_sectors < r10_bio->sectors) { 1333 /* We are splitting this into multiple parts, so 1334 * we need to prepare for allocating another r10_bio. 1335 */ 1336 r10_bio->sectors = max_sectors; 1337 spin_lock_irq(&conf->device_lock); 1338 if (bio->bi_phys_segments == 0) 1339 bio->bi_phys_segments = 2; 1340 else 1341 bio->bi_phys_segments++; 1342 spin_unlock_irq(&conf->device_lock); 1343 } 1344 sectors_handled = r10_bio->sector + max_sectors - 1345 bio->bi_iter.bi_sector; 1346 1347 atomic_set(&r10_bio->remaining, 1); 1348 bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0); 1349 1350 for (i = 0; i < conf->copies; i++) { 1351 struct bio *mbio; 1352 int d = r10_bio->devs[i].devnum; 1353 if (r10_bio->devs[i].bio) { 1354 struct md_rdev *rdev = conf->mirrors[d].rdev; 1355 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1356 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1357 max_sectors); 1358 r10_bio->devs[i].bio = mbio; 1359 1360 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+ 1361 choose_data_offset(r10_bio, 1362 rdev)); 1363 mbio->bi_bdev = rdev->bdev; 1364 mbio->bi_end_io = raid10_end_write_request; 1365 mbio->bi_rw = 1366 WRITE | do_sync | do_fua | do_discard | do_same; 1367 mbio->bi_private = r10_bio; 1368 1369 atomic_inc(&r10_bio->remaining); 1370 1371 cb = blk_check_plugged(raid10_unplug, mddev, 1372 sizeof(*plug)); 1373 if (cb) 1374 plug = container_of(cb, struct raid10_plug_cb, 1375 cb); 1376 else 1377 plug = NULL; 1378 spin_lock_irqsave(&conf->device_lock, flags); 1379 if (plug) { 1380 bio_list_add(&plug->pending, mbio); 1381 plug->pending_cnt++; 1382 } else { 1383 bio_list_add(&conf->pending_bio_list, mbio); 1384 conf->pending_count++; 1385 } 1386 spin_unlock_irqrestore(&conf->device_lock, flags); 1387 if (!plug) 1388 md_wakeup_thread(mddev->thread); 1389 } 1390 1391 if (r10_bio->devs[i].repl_bio) { 1392 struct md_rdev *rdev = conf->mirrors[d].replacement; 1393 if (rdev == NULL) { 1394 /* Replacement just got moved to main 'rdev' */ 1395 smp_mb(); 1396 rdev = conf->mirrors[d].rdev; 1397 } 1398 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1399 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1400 max_sectors); 1401 r10_bio->devs[i].repl_bio = mbio; 1402 1403 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr + 1404 choose_data_offset( 1405 r10_bio, rdev)); 1406 mbio->bi_bdev = rdev->bdev; 1407 mbio->bi_end_io = raid10_end_write_request; 1408 mbio->bi_rw = 1409 WRITE | do_sync | do_fua | do_discard | do_same; 1410 mbio->bi_private = r10_bio; 1411 1412 atomic_inc(&r10_bio->remaining); 1413 spin_lock_irqsave(&conf->device_lock, flags); 1414 bio_list_add(&conf->pending_bio_list, mbio); 1415 conf->pending_count++; 1416 spin_unlock_irqrestore(&conf->device_lock, flags); 1417 if (!mddev_check_plugged(mddev)) 1418 md_wakeup_thread(mddev->thread); 1419 } 1420 } 1421 1422 /* Don't remove the bias on 'remaining' (one_write_done) until 1423 * after checking if we need to go around again. 1424 */ 1425 1426 if (sectors_handled < bio_sectors(bio)) { 1427 one_write_done(r10_bio); 1428 /* We need another r10_bio. It has already been counted 1429 * in bio->bi_phys_segments. 1430 */ 1431 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1432 1433 r10_bio->master_bio = bio; 1434 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1435 1436 r10_bio->mddev = mddev; 1437 r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled; 1438 r10_bio->state = 0; 1439 goto retry_write; 1440 } 1441 one_write_done(r10_bio); 1442 } 1443 1444 static void make_request(struct mddev *mddev, struct bio *bio) 1445 { 1446 struct r10conf *conf = mddev->private; 1447 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1448 int chunk_sects = chunk_mask + 1; 1449 1450 struct bio *split; 1451 1452 if (unlikely(bio->bi_rw & REQ_FLUSH)) { 1453 md_flush_request(mddev, bio); 1454 return; 1455 } 1456 1457 md_write_start(mddev, bio); 1458 1459 do { 1460 1461 /* 1462 * If this request crosses a chunk boundary, we need to split 1463 * it. 1464 */ 1465 if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + 1466 bio_sectors(bio) > chunk_sects 1467 && (conf->geo.near_copies < conf->geo.raid_disks 1468 || conf->prev.near_copies < 1469 conf->prev.raid_disks))) { 1470 split = bio_split(bio, chunk_sects - 1471 (bio->bi_iter.bi_sector & 1472 (chunk_sects - 1)), 1473 GFP_NOIO, fs_bio_set); 1474 bio_chain(split, bio); 1475 } else { 1476 split = bio; 1477 } 1478 1479 __make_request(mddev, split); 1480 } while (split != bio); 1481 1482 /* In case raid10d snuck in to freeze_array */ 1483 wake_up(&conf->wait_barrier); 1484 } 1485 1486 static void status(struct seq_file *seq, struct mddev *mddev) 1487 { 1488 struct r10conf *conf = mddev->private; 1489 int i; 1490 1491 if (conf->geo.near_copies < conf->geo.raid_disks) 1492 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1493 if (conf->geo.near_copies > 1) 1494 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1495 if (conf->geo.far_copies > 1) { 1496 if (conf->geo.far_offset) 1497 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1498 else 1499 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1500 } 1501 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1502 conf->geo.raid_disks - mddev->degraded); 1503 for (i = 0; i < conf->geo.raid_disks; i++) 1504 seq_printf(seq, "%s", 1505 conf->mirrors[i].rdev && 1506 test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_"); 1507 seq_printf(seq, "]"); 1508 } 1509 1510 /* check if there are enough drives for 1511 * every block to appear on atleast one. 1512 * Don't consider the device numbered 'ignore' 1513 * as we might be about to remove it. 1514 */ 1515 static int _enough(struct r10conf *conf, int previous, int ignore) 1516 { 1517 int first = 0; 1518 int has_enough = 0; 1519 int disks, ncopies; 1520 if (previous) { 1521 disks = conf->prev.raid_disks; 1522 ncopies = conf->prev.near_copies; 1523 } else { 1524 disks = conf->geo.raid_disks; 1525 ncopies = conf->geo.near_copies; 1526 } 1527 1528 rcu_read_lock(); 1529 do { 1530 int n = conf->copies; 1531 int cnt = 0; 1532 int this = first; 1533 while (n--) { 1534 struct md_rdev *rdev; 1535 if (this != ignore && 1536 (rdev = rcu_dereference(conf->mirrors[this].rdev)) && 1537 test_bit(In_sync, &rdev->flags)) 1538 cnt++; 1539 this = (this+1) % disks; 1540 } 1541 if (cnt == 0) 1542 goto out; 1543 first = (first + ncopies) % disks; 1544 } while (first != 0); 1545 has_enough = 1; 1546 out: 1547 rcu_read_unlock(); 1548 return has_enough; 1549 } 1550 1551 static int enough(struct r10conf *conf, int ignore) 1552 { 1553 /* when calling 'enough', both 'prev' and 'geo' must 1554 * be stable. 1555 * This is ensured if ->reconfig_mutex or ->device_lock 1556 * is held. 1557 */ 1558 return _enough(conf, 0, ignore) && 1559 _enough(conf, 1, ignore); 1560 } 1561 1562 static void error(struct mddev *mddev, struct md_rdev *rdev) 1563 { 1564 char b[BDEVNAME_SIZE]; 1565 struct r10conf *conf = mddev->private; 1566 unsigned long flags; 1567 1568 /* 1569 * If it is not operational, then we have already marked it as dead 1570 * else if it is the last working disks, ignore the error, let the 1571 * next level up know. 1572 * else mark the drive as failed 1573 */ 1574 spin_lock_irqsave(&conf->device_lock, flags); 1575 if (test_bit(In_sync, &rdev->flags) 1576 && !enough(conf, rdev->raid_disk)) { 1577 /* 1578 * Don't fail the drive, just return an IO error. 1579 */ 1580 spin_unlock_irqrestore(&conf->device_lock, flags); 1581 return; 1582 } 1583 if (test_and_clear_bit(In_sync, &rdev->flags)) 1584 mddev->degraded++; 1585 /* 1586 * If recovery is running, make sure it aborts. 1587 */ 1588 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1589 set_bit(Blocked, &rdev->flags); 1590 set_bit(Faulty, &rdev->flags); 1591 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1592 set_bit(MD_CHANGE_PENDING, &mddev->flags); 1593 spin_unlock_irqrestore(&conf->device_lock, flags); 1594 printk(KERN_ALERT 1595 "md/raid10:%s: Disk failure on %s, disabling device.\n" 1596 "md/raid10:%s: Operation continuing on %d devices.\n", 1597 mdname(mddev), bdevname(rdev->bdev, b), 1598 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 1599 } 1600 1601 static void print_conf(struct r10conf *conf) 1602 { 1603 int i; 1604 struct raid10_info *tmp; 1605 1606 printk(KERN_DEBUG "RAID10 conf printout:\n"); 1607 if (!conf) { 1608 printk(KERN_DEBUG "(!conf)\n"); 1609 return; 1610 } 1611 printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 1612 conf->geo.raid_disks); 1613 1614 for (i = 0; i < conf->geo.raid_disks; i++) { 1615 char b[BDEVNAME_SIZE]; 1616 tmp = conf->mirrors + i; 1617 if (tmp->rdev) 1618 printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n", 1619 i, !test_bit(In_sync, &tmp->rdev->flags), 1620 !test_bit(Faulty, &tmp->rdev->flags), 1621 bdevname(tmp->rdev->bdev,b)); 1622 } 1623 } 1624 1625 static void close_sync(struct r10conf *conf) 1626 { 1627 wait_barrier(conf); 1628 allow_barrier(conf); 1629 1630 mempool_destroy(conf->r10buf_pool); 1631 conf->r10buf_pool = NULL; 1632 } 1633 1634 static int raid10_spare_active(struct mddev *mddev) 1635 { 1636 int i; 1637 struct r10conf *conf = mddev->private; 1638 struct raid10_info *tmp; 1639 int count = 0; 1640 unsigned long flags; 1641 1642 /* 1643 * Find all non-in_sync disks within the RAID10 configuration 1644 * and mark them in_sync 1645 */ 1646 for (i = 0; i < conf->geo.raid_disks; i++) { 1647 tmp = conf->mirrors + i; 1648 if (tmp->replacement 1649 && tmp->replacement->recovery_offset == MaxSector 1650 && !test_bit(Faulty, &tmp->replacement->flags) 1651 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 1652 /* Replacement has just become active */ 1653 if (!tmp->rdev 1654 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 1655 count++; 1656 if (tmp->rdev) { 1657 /* Replaced device not technically faulty, 1658 * but we need to be sure it gets removed 1659 * and never re-added. 1660 */ 1661 set_bit(Faulty, &tmp->rdev->flags); 1662 sysfs_notify_dirent_safe( 1663 tmp->rdev->sysfs_state); 1664 } 1665 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 1666 } else if (tmp->rdev 1667 && tmp->rdev->recovery_offset == MaxSector 1668 && !test_bit(Faulty, &tmp->rdev->flags) 1669 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 1670 count++; 1671 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 1672 } 1673 } 1674 spin_lock_irqsave(&conf->device_lock, flags); 1675 mddev->degraded -= count; 1676 spin_unlock_irqrestore(&conf->device_lock, flags); 1677 1678 print_conf(conf); 1679 return count; 1680 } 1681 1682 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1683 { 1684 struct r10conf *conf = mddev->private; 1685 int err = -EEXIST; 1686 int mirror; 1687 int first = 0; 1688 int last = conf->geo.raid_disks - 1; 1689 1690 if (mddev->recovery_cp < MaxSector) 1691 /* only hot-add to in-sync arrays, as recovery is 1692 * very different from resync 1693 */ 1694 return -EBUSY; 1695 if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1)) 1696 return -EINVAL; 1697 1698 if (rdev->raid_disk >= 0) 1699 first = last = rdev->raid_disk; 1700 1701 if (rdev->saved_raid_disk >= first && 1702 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1703 mirror = rdev->saved_raid_disk; 1704 else 1705 mirror = first; 1706 for ( ; mirror <= last ; mirror++) { 1707 struct raid10_info *p = &conf->mirrors[mirror]; 1708 if (p->recovery_disabled == mddev->recovery_disabled) 1709 continue; 1710 if (p->rdev) { 1711 if (!test_bit(WantReplacement, &p->rdev->flags) || 1712 p->replacement != NULL) 1713 continue; 1714 clear_bit(In_sync, &rdev->flags); 1715 set_bit(Replacement, &rdev->flags); 1716 rdev->raid_disk = mirror; 1717 err = 0; 1718 if (mddev->gendisk) 1719 disk_stack_limits(mddev->gendisk, rdev->bdev, 1720 rdev->data_offset << 9); 1721 conf->fullsync = 1; 1722 rcu_assign_pointer(p->replacement, rdev); 1723 break; 1724 } 1725 1726 if (mddev->gendisk) 1727 disk_stack_limits(mddev->gendisk, rdev->bdev, 1728 rdev->data_offset << 9); 1729 1730 p->head_position = 0; 1731 p->recovery_disabled = mddev->recovery_disabled - 1; 1732 rdev->raid_disk = mirror; 1733 err = 0; 1734 if (rdev->saved_raid_disk != mirror) 1735 conf->fullsync = 1; 1736 rcu_assign_pointer(p->rdev, rdev); 1737 break; 1738 } 1739 md_integrity_add_rdev(rdev, mddev); 1740 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1741 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1742 1743 print_conf(conf); 1744 return err; 1745 } 1746 1747 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1748 { 1749 struct r10conf *conf = mddev->private; 1750 int err = 0; 1751 int number = rdev->raid_disk; 1752 struct md_rdev **rdevp; 1753 struct raid10_info *p = conf->mirrors + number; 1754 1755 print_conf(conf); 1756 if (rdev == p->rdev) 1757 rdevp = &p->rdev; 1758 else if (rdev == p->replacement) 1759 rdevp = &p->replacement; 1760 else 1761 return 0; 1762 1763 if (test_bit(In_sync, &rdev->flags) || 1764 atomic_read(&rdev->nr_pending)) { 1765 err = -EBUSY; 1766 goto abort; 1767 } 1768 /* Only remove faulty devices if recovery 1769 * is not possible. 1770 */ 1771 if (!test_bit(Faulty, &rdev->flags) && 1772 mddev->recovery_disabled != p->recovery_disabled && 1773 (!p->replacement || p->replacement == rdev) && 1774 number < conf->geo.raid_disks && 1775 enough(conf, -1)) { 1776 err = -EBUSY; 1777 goto abort; 1778 } 1779 *rdevp = NULL; 1780 synchronize_rcu(); 1781 if (atomic_read(&rdev->nr_pending)) { 1782 /* lost the race, try later */ 1783 err = -EBUSY; 1784 *rdevp = rdev; 1785 goto abort; 1786 } else if (p->replacement) { 1787 /* We must have just cleared 'rdev' */ 1788 p->rdev = p->replacement; 1789 clear_bit(Replacement, &p->replacement->flags); 1790 smp_mb(); /* Make sure other CPUs may see both as identical 1791 * but will never see neither -- if they are careful. 1792 */ 1793 p->replacement = NULL; 1794 clear_bit(WantReplacement, &rdev->flags); 1795 } else 1796 /* We might have just remove the Replacement as faulty 1797 * Clear the flag just in case 1798 */ 1799 clear_bit(WantReplacement, &rdev->flags); 1800 1801 err = md_integrity_register(mddev); 1802 1803 abort: 1804 1805 print_conf(conf); 1806 return err; 1807 } 1808 1809 static void end_sync_read(struct bio *bio) 1810 { 1811 struct r10bio *r10_bio = bio->bi_private; 1812 struct r10conf *conf = r10_bio->mddev->private; 1813 int d; 1814 1815 if (bio == r10_bio->master_bio) { 1816 /* this is a reshape read */ 1817 d = r10_bio->read_slot; /* really the read dev */ 1818 } else 1819 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1820 1821 if (!bio->bi_error) 1822 set_bit(R10BIO_Uptodate, &r10_bio->state); 1823 else 1824 /* The write handler will notice the lack of 1825 * R10BIO_Uptodate and record any errors etc 1826 */ 1827 atomic_add(r10_bio->sectors, 1828 &conf->mirrors[d].rdev->corrected_errors); 1829 1830 /* for reconstruct, we always reschedule after a read. 1831 * for resync, only after all reads 1832 */ 1833 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 1834 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 1835 atomic_dec_and_test(&r10_bio->remaining)) { 1836 /* we have read all the blocks, 1837 * do the comparison in process context in raid10d 1838 */ 1839 reschedule_retry(r10_bio); 1840 } 1841 } 1842 1843 static void end_sync_request(struct r10bio *r10_bio) 1844 { 1845 struct mddev *mddev = r10_bio->mddev; 1846 1847 while (atomic_dec_and_test(&r10_bio->remaining)) { 1848 if (r10_bio->master_bio == NULL) { 1849 /* the primary of several recovery bios */ 1850 sector_t s = r10_bio->sectors; 1851 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1852 test_bit(R10BIO_WriteError, &r10_bio->state)) 1853 reschedule_retry(r10_bio); 1854 else 1855 put_buf(r10_bio); 1856 md_done_sync(mddev, s, 1); 1857 break; 1858 } else { 1859 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 1860 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1861 test_bit(R10BIO_WriteError, &r10_bio->state)) 1862 reschedule_retry(r10_bio); 1863 else 1864 put_buf(r10_bio); 1865 r10_bio = r10_bio2; 1866 } 1867 } 1868 } 1869 1870 static void end_sync_write(struct bio *bio) 1871 { 1872 struct r10bio *r10_bio = bio->bi_private; 1873 struct mddev *mddev = r10_bio->mddev; 1874 struct r10conf *conf = mddev->private; 1875 int d; 1876 sector_t first_bad; 1877 int bad_sectors; 1878 int slot; 1879 int repl; 1880 struct md_rdev *rdev = NULL; 1881 1882 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1883 if (repl) 1884 rdev = conf->mirrors[d].replacement; 1885 else 1886 rdev = conf->mirrors[d].rdev; 1887 1888 if (bio->bi_error) { 1889 if (repl) 1890 md_error(mddev, rdev); 1891 else { 1892 set_bit(WriteErrorSeen, &rdev->flags); 1893 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 1894 set_bit(MD_RECOVERY_NEEDED, 1895 &rdev->mddev->recovery); 1896 set_bit(R10BIO_WriteError, &r10_bio->state); 1897 } 1898 } else if (is_badblock(rdev, 1899 r10_bio->devs[slot].addr, 1900 r10_bio->sectors, 1901 &first_bad, &bad_sectors)) 1902 set_bit(R10BIO_MadeGood, &r10_bio->state); 1903 1904 rdev_dec_pending(rdev, mddev); 1905 1906 end_sync_request(r10_bio); 1907 } 1908 1909 /* 1910 * Note: sync and recover and handled very differently for raid10 1911 * This code is for resync. 1912 * For resync, we read through virtual addresses and read all blocks. 1913 * If there is any error, we schedule a write. The lowest numbered 1914 * drive is authoritative. 1915 * However requests come for physical address, so we need to map. 1916 * For every physical address there are raid_disks/copies virtual addresses, 1917 * which is always are least one, but is not necessarly an integer. 1918 * This means that a physical address can span multiple chunks, so we may 1919 * have to submit multiple io requests for a single sync request. 1920 */ 1921 /* 1922 * We check if all blocks are in-sync and only write to blocks that 1923 * aren't in sync 1924 */ 1925 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 1926 { 1927 struct r10conf *conf = mddev->private; 1928 int i, first; 1929 struct bio *tbio, *fbio; 1930 int vcnt; 1931 1932 atomic_set(&r10_bio->remaining, 1); 1933 1934 /* find the first device with a block */ 1935 for (i=0; i<conf->copies; i++) 1936 if (!r10_bio->devs[i].bio->bi_error) 1937 break; 1938 1939 if (i == conf->copies) 1940 goto done; 1941 1942 first = i; 1943 fbio = r10_bio->devs[i].bio; 1944 1945 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 1946 /* now find blocks with errors */ 1947 for (i=0 ; i < conf->copies ; i++) { 1948 int j, d; 1949 1950 tbio = r10_bio->devs[i].bio; 1951 1952 if (tbio->bi_end_io != end_sync_read) 1953 continue; 1954 if (i == first) 1955 continue; 1956 if (!r10_bio->devs[i].bio->bi_error) { 1957 /* We know that the bi_io_vec layout is the same for 1958 * both 'first' and 'i', so we just compare them. 1959 * All vec entries are PAGE_SIZE; 1960 */ 1961 int sectors = r10_bio->sectors; 1962 for (j = 0; j < vcnt; j++) { 1963 int len = PAGE_SIZE; 1964 if (sectors < (len / 512)) 1965 len = sectors * 512; 1966 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page), 1967 page_address(tbio->bi_io_vec[j].bv_page), 1968 len)) 1969 break; 1970 sectors -= len/512; 1971 } 1972 if (j == vcnt) 1973 continue; 1974 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 1975 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 1976 /* Don't fix anything. */ 1977 continue; 1978 } 1979 /* Ok, we need to write this bio, either to correct an 1980 * inconsistency or to correct an unreadable block. 1981 * First we need to fixup bv_offset, bv_len and 1982 * bi_vecs, as the read request might have corrupted these 1983 */ 1984 bio_reset(tbio); 1985 1986 tbio->bi_vcnt = vcnt; 1987 tbio->bi_iter.bi_size = r10_bio->sectors << 9; 1988 tbio->bi_rw = WRITE; 1989 tbio->bi_private = r10_bio; 1990 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; 1991 tbio->bi_end_io = end_sync_write; 1992 1993 bio_copy_data(tbio, fbio); 1994 1995 d = r10_bio->devs[i].devnum; 1996 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 1997 atomic_inc(&r10_bio->remaining); 1998 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio)); 1999 2000 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset; 2001 tbio->bi_bdev = conf->mirrors[d].rdev->bdev; 2002 generic_make_request(tbio); 2003 } 2004 2005 /* Now write out to any replacement devices 2006 * that are active 2007 */ 2008 for (i = 0; i < conf->copies; i++) { 2009 int d; 2010 2011 tbio = r10_bio->devs[i].repl_bio; 2012 if (!tbio || !tbio->bi_end_io) 2013 continue; 2014 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2015 && r10_bio->devs[i].bio != fbio) 2016 bio_copy_data(tbio, fbio); 2017 d = r10_bio->devs[i].devnum; 2018 atomic_inc(&r10_bio->remaining); 2019 md_sync_acct(conf->mirrors[d].replacement->bdev, 2020 bio_sectors(tbio)); 2021 generic_make_request(tbio); 2022 } 2023 2024 done: 2025 if (atomic_dec_and_test(&r10_bio->remaining)) { 2026 md_done_sync(mddev, r10_bio->sectors, 1); 2027 put_buf(r10_bio); 2028 } 2029 } 2030 2031 /* 2032 * Now for the recovery code. 2033 * Recovery happens across physical sectors. 2034 * We recover all non-is_sync drives by finding the virtual address of 2035 * each, and then choose a working drive that also has that virt address. 2036 * There is a separate r10_bio for each non-in_sync drive. 2037 * Only the first two slots are in use. The first for reading, 2038 * The second for writing. 2039 * 2040 */ 2041 static void fix_recovery_read_error(struct r10bio *r10_bio) 2042 { 2043 /* We got a read error during recovery. 2044 * We repeat the read in smaller page-sized sections. 2045 * If a read succeeds, write it to the new device or record 2046 * a bad block if we cannot. 2047 * If a read fails, record a bad block on both old and 2048 * new devices. 2049 */ 2050 struct mddev *mddev = r10_bio->mddev; 2051 struct r10conf *conf = mddev->private; 2052 struct bio *bio = r10_bio->devs[0].bio; 2053 sector_t sect = 0; 2054 int sectors = r10_bio->sectors; 2055 int idx = 0; 2056 int dr = r10_bio->devs[0].devnum; 2057 int dw = r10_bio->devs[1].devnum; 2058 2059 while (sectors) { 2060 int s = sectors; 2061 struct md_rdev *rdev; 2062 sector_t addr; 2063 int ok; 2064 2065 if (s > (PAGE_SIZE>>9)) 2066 s = PAGE_SIZE >> 9; 2067 2068 rdev = conf->mirrors[dr].rdev; 2069 addr = r10_bio->devs[0].addr + sect, 2070 ok = sync_page_io(rdev, 2071 addr, 2072 s << 9, 2073 bio->bi_io_vec[idx].bv_page, 2074 READ, false); 2075 if (ok) { 2076 rdev = conf->mirrors[dw].rdev; 2077 addr = r10_bio->devs[1].addr + sect; 2078 ok = sync_page_io(rdev, 2079 addr, 2080 s << 9, 2081 bio->bi_io_vec[idx].bv_page, 2082 WRITE, false); 2083 if (!ok) { 2084 set_bit(WriteErrorSeen, &rdev->flags); 2085 if (!test_and_set_bit(WantReplacement, 2086 &rdev->flags)) 2087 set_bit(MD_RECOVERY_NEEDED, 2088 &rdev->mddev->recovery); 2089 } 2090 } 2091 if (!ok) { 2092 /* We don't worry if we cannot set a bad block - 2093 * it really is bad so there is no loss in not 2094 * recording it yet 2095 */ 2096 rdev_set_badblocks(rdev, addr, s, 0); 2097 2098 if (rdev != conf->mirrors[dw].rdev) { 2099 /* need bad block on destination too */ 2100 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2101 addr = r10_bio->devs[1].addr + sect; 2102 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2103 if (!ok) { 2104 /* just abort the recovery */ 2105 printk(KERN_NOTICE 2106 "md/raid10:%s: recovery aborted" 2107 " due to read error\n", 2108 mdname(mddev)); 2109 2110 conf->mirrors[dw].recovery_disabled 2111 = mddev->recovery_disabled; 2112 set_bit(MD_RECOVERY_INTR, 2113 &mddev->recovery); 2114 break; 2115 } 2116 } 2117 } 2118 2119 sectors -= s; 2120 sect += s; 2121 idx++; 2122 } 2123 } 2124 2125 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2126 { 2127 struct r10conf *conf = mddev->private; 2128 int d; 2129 struct bio *wbio, *wbio2; 2130 2131 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2132 fix_recovery_read_error(r10_bio); 2133 end_sync_request(r10_bio); 2134 return; 2135 } 2136 2137 /* 2138 * share the pages with the first bio 2139 * and submit the write request 2140 */ 2141 d = r10_bio->devs[1].devnum; 2142 wbio = r10_bio->devs[1].bio; 2143 wbio2 = r10_bio->devs[1].repl_bio; 2144 /* Need to test wbio2->bi_end_io before we call 2145 * generic_make_request as if the former is NULL, 2146 * the latter is free to free wbio2. 2147 */ 2148 if (wbio2 && !wbio2->bi_end_io) 2149 wbio2 = NULL; 2150 if (wbio->bi_end_io) { 2151 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2152 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio)); 2153 generic_make_request(wbio); 2154 } 2155 if (wbio2) { 2156 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2157 md_sync_acct(conf->mirrors[d].replacement->bdev, 2158 bio_sectors(wbio2)); 2159 generic_make_request(wbio2); 2160 } 2161 } 2162 2163 /* 2164 * Used by fix_read_error() to decay the per rdev read_errors. 2165 * We halve the read error count for every hour that has elapsed 2166 * since the last recorded read error. 2167 * 2168 */ 2169 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2170 { 2171 struct timespec cur_time_mon; 2172 unsigned long hours_since_last; 2173 unsigned int read_errors = atomic_read(&rdev->read_errors); 2174 2175 ktime_get_ts(&cur_time_mon); 2176 2177 if (rdev->last_read_error.tv_sec == 0 && 2178 rdev->last_read_error.tv_nsec == 0) { 2179 /* first time we've seen a read error */ 2180 rdev->last_read_error = cur_time_mon; 2181 return; 2182 } 2183 2184 hours_since_last = (cur_time_mon.tv_sec - 2185 rdev->last_read_error.tv_sec) / 3600; 2186 2187 rdev->last_read_error = cur_time_mon; 2188 2189 /* 2190 * if hours_since_last is > the number of bits in read_errors 2191 * just set read errors to 0. We do this to avoid 2192 * overflowing the shift of read_errors by hours_since_last. 2193 */ 2194 if (hours_since_last >= 8 * sizeof(read_errors)) 2195 atomic_set(&rdev->read_errors, 0); 2196 else 2197 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2198 } 2199 2200 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2201 int sectors, struct page *page, int rw) 2202 { 2203 sector_t first_bad; 2204 int bad_sectors; 2205 2206 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) 2207 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) 2208 return -1; 2209 if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) 2210 /* success */ 2211 return 1; 2212 if (rw == WRITE) { 2213 set_bit(WriteErrorSeen, &rdev->flags); 2214 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2215 set_bit(MD_RECOVERY_NEEDED, 2216 &rdev->mddev->recovery); 2217 } 2218 /* need to record an error - either for the block or the device */ 2219 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2220 md_error(rdev->mddev, rdev); 2221 return 0; 2222 } 2223 2224 /* 2225 * This is a kernel thread which: 2226 * 2227 * 1. Retries failed read operations on working mirrors. 2228 * 2. Updates the raid superblock when problems encounter. 2229 * 3. Performs writes following reads for array synchronising. 2230 */ 2231 2232 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2233 { 2234 int sect = 0; /* Offset from r10_bio->sector */ 2235 int sectors = r10_bio->sectors; 2236 struct md_rdev*rdev; 2237 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2238 int d = r10_bio->devs[r10_bio->read_slot].devnum; 2239 2240 /* still own a reference to this rdev, so it cannot 2241 * have been cleared recently. 2242 */ 2243 rdev = conf->mirrors[d].rdev; 2244 2245 if (test_bit(Faulty, &rdev->flags)) 2246 /* drive has already been failed, just ignore any 2247 more fix_read_error() attempts */ 2248 return; 2249 2250 check_decay_read_errors(mddev, rdev); 2251 atomic_inc(&rdev->read_errors); 2252 if (atomic_read(&rdev->read_errors) > max_read_errors) { 2253 char b[BDEVNAME_SIZE]; 2254 bdevname(rdev->bdev, b); 2255 2256 printk(KERN_NOTICE 2257 "md/raid10:%s: %s: Raid device exceeded " 2258 "read_error threshold [cur %d:max %d]\n", 2259 mdname(mddev), b, 2260 atomic_read(&rdev->read_errors), max_read_errors); 2261 printk(KERN_NOTICE 2262 "md/raid10:%s: %s: Failing raid device\n", 2263 mdname(mddev), b); 2264 md_error(mddev, conf->mirrors[d].rdev); 2265 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED; 2266 return; 2267 } 2268 2269 while(sectors) { 2270 int s = sectors; 2271 int sl = r10_bio->read_slot; 2272 int success = 0; 2273 int start; 2274 2275 if (s > (PAGE_SIZE>>9)) 2276 s = PAGE_SIZE >> 9; 2277 2278 rcu_read_lock(); 2279 do { 2280 sector_t first_bad; 2281 int bad_sectors; 2282 2283 d = r10_bio->devs[sl].devnum; 2284 rdev = rcu_dereference(conf->mirrors[d].rdev); 2285 if (rdev && 2286 test_bit(In_sync, &rdev->flags) && 2287 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2288 &first_bad, &bad_sectors) == 0) { 2289 atomic_inc(&rdev->nr_pending); 2290 rcu_read_unlock(); 2291 success = sync_page_io(rdev, 2292 r10_bio->devs[sl].addr + 2293 sect, 2294 s<<9, 2295 conf->tmppage, READ, false); 2296 rdev_dec_pending(rdev, mddev); 2297 rcu_read_lock(); 2298 if (success) 2299 break; 2300 } 2301 sl++; 2302 if (sl == conf->copies) 2303 sl = 0; 2304 } while (!success && sl != r10_bio->read_slot); 2305 rcu_read_unlock(); 2306 2307 if (!success) { 2308 /* Cannot read from anywhere, just mark the block 2309 * as bad on the first device to discourage future 2310 * reads. 2311 */ 2312 int dn = r10_bio->devs[r10_bio->read_slot].devnum; 2313 rdev = conf->mirrors[dn].rdev; 2314 2315 if (!rdev_set_badblocks( 2316 rdev, 2317 r10_bio->devs[r10_bio->read_slot].addr 2318 + sect, 2319 s, 0)) { 2320 md_error(mddev, rdev); 2321 r10_bio->devs[r10_bio->read_slot].bio 2322 = IO_BLOCKED; 2323 } 2324 break; 2325 } 2326 2327 start = sl; 2328 /* write it back and re-read */ 2329 rcu_read_lock(); 2330 while (sl != r10_bio->read_slot) { 2331 char b[BDEVNAME_SIZE]; 2332 2333 if (sl==0) 2334 sl = conf->copies; 2335 sl--; 2336 d = r10_bio->devs[sl].devnum; 2337 rdev = rcu_dereference(conf->mirrors[d].rdev); 2338 if (!rdev || 2339 !test_bit(In_sync, &rdev->flags)) 2340 continue; 2341 2342 atomic_inc(&rdev->nr_pending); 2343 rcu_read_unlock(); 2344 if (r10_sync_page_io(rdev, 2345 r10_bio->devs[sl].addr + 2346 sect, 2347 s, conf->tmppage, WRITE) 2348 == 0) { 2349 /* Well, this device is dead */ 2350 printk(KERN_NOTICE 2351 "md/raid10:%s: read correction " 2352 "write failed" 2353 " (%d sectors at %llu on %s)\n", 2354 mdname(mddev), s, 2355 (unsigned long long)( 2356 sect + 2357 choose_data_offset(r10_bio, 2358 rdev)), 2359 bdevname(rdev->bdev, b)); 2360 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2361 "drive\n", 2362 mdname(mddev), 2363 bdevname(rdev->bdev, b)); 2364 } 2365 rdev_dec_pending(rdev, mddev); 2366 rcu_read_lock(); 2367 } 2368 sl = start; 2369 while (sl != r10_bio->read_slot) { 2370 char b[BDEVNAME_SIZE]; 2371 2372 if (sl==0) 2373 sl = conf->copies; 2374 sl--; 2375 d = r10_bio->devs[sl].devnum; 2376 rdev = rcu_dereference(conf->mirrors[d].rdev); 2377 if (!rdev || 2378 !test_bit(In_sync, &rdev->flags)) 2379 continue; 2380 2381 atomic_inc(&rdev->nr_pending); 2382 rcu_read_unlock(); 2383 switch (r10_sync_page_io(rdev, 2384 r10_bio->devs[sl].addr + 2385 sect, 2386 s, conf->tmppage, 2387 READ)) { 2388 case 0: 2389 /* Well, this device is dead */ 2390 printk(KERN_NOTICE 2391 "md/raid10:%s: unable to read back " 2392 "corrected sectors" 2393 " (%d sectors at %llu on %s)\n", 2394 mdname(mddev), s, 2395 (unsigned long long)( 2396 sect + 2397 choose_data_offset(r10_bio, rdev)), 2398 bdevname(rdev->bdev, b)); 2399 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2400 "drive\n", 2401 mdname(mddev), 2402 bdevname(rdev->bdev, b)); 2403 break; 2404 case 1: 2405 printk(KERN_INFO 2406 "md/raid10:%s: read error corrected" 2407 " (%d sectors at %llu on %s)\n", 2408 mdname(mddev), s, 2409 (unsigned long long)( 2410 sect + 2411 choose_data_offset(r10_bio, rdev)), 2412 bdevname(rdev->bdev, b)); 2413 atomic_add(s, &rdev->corrected_errors); 2414 } 2415 2416 rdev_dec_pending(rdev, mddev); 2417 rcu_read_lock(); 2418 } 2419 rcu_read_unlock(); 2420 2421 sectors -= s; 2422 sect += s; 2423 } 2424 } 2425 2426 static int narrow_write_error(struct r10bio *r10_bio, int i) 2427 { 2428 struct bio *bio = r10_bio->master_bio; 2429 struct mddev *mddev = r10_bio->mddev; 2430 struct r10conf *conf = mddev->private; 2431 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2432 /* bio has the data to be written to slot 'i' where 2433 * we just recently had a write error. 2434 * We repeatedly clone the bio and trim down to one block, 2435 * then try the write. Where the write fails we record 2436 * a bad block. 2437 * It is conceivable that the bio doesn't exactly align with 2438 * blocks. We must handle this. 2439 * 2440 * We currently own a reference to the rdev. 2441 */ 2442 2443 int block_sectors; 2444 sector_t sector; 2445 int sectors; 2446 int sect_to_write = r10_bio->sectors; 2447 int ok = 1; 2448 2449 if (rdev->badblocks.shift < 0) 2450 return 0; 2451 2452 block_sectors = roundup(1 << rdev->badblocks.shift, 2453 bdev_logical_block_size(rdev->bdev) >> 9); 2454 sector = r10_bio->sector; 2455 sectors = ((r10_bio->sector + block_sectors) 2456 & ~(sector_t)(block_sectors - 1)) 2457 - sector; 2458 2459 while (sect_to_write) { 2460 struct bio *wbio; 2461 if (sectors > sect_to_write) 2462 sectors = sect_to_write; 2463 /* Write at 'sector' for 'sectors' */ 2464 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 2465 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors); 2466 wbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+ 2467 choose_data_offset(r10_bio, rdev) + 2468 (sector - r10_bio->sector)); 2469 wbio->bi_bdev = rdev->bdev; 2470 if (submit_bio_wait(WRITE, wbio) == 0) 2471 /* Failure! */ 2472 ok = rdev_set_badblocks(rdev, sector, 2473 sectors, 0) 2474 && ok; 2475 2476 bio_put(wbio); 2477 sect_to_write -= sectors; 2478 sector += sectors; 2479 sectors = block_sectors; 2480 } 2481 return ok; 2482 } 2483 2484 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2485 { 2486 int slot = r10_bio->read_slot; 2487 struct bio *bio; 2488 struct r10conf *conf = mddev->private; 2489 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2490 char b[BDEVNAME_SIZE]; 2491 unsigned long do_sync; 2492 int max_sectors; 2493 2494 /* we got a read error. Maybe the drive is bad. Maybe just 2495 * the block and we can fix it. 2496 * We freeze all other IO, and try reading the block from 2497 * other devices. When we find one, we re-write 2498 * and check it that fixes the read error. 2499 * This is all done synchronously while the array is 2500 * frozen. 2501 */ 2502 bio = r10_bio->devs[slot].bio; 2503 bdevname(bio->bi_bdev, b); 2504 bio_put(bio); 2505 r10_bio->devs[slot].bio = NULL; 2506 2507 if (mddev->ro == 0) { 2508 freeze_array(conf, 1); 2509 fix_read_error(conf, mddev, r10_bio); 2510 unfreeze_array(conf); 2511 } else 2512 r10_bio->devs[slot].bio = IO_BLOCKED; 2513 2514 rdev_dec_pending(rdev, mddev); 2515 2516 read_more: 2517 rdev = read_balance(conf, r10_bio, &max_sectors); 2518 if (rdev == NULL) { 2519 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O" 2520 " read error for block %llu\n", 2521 mdname(mddev), b, 2522 (unsigned long long)r10_bio->sector); 2523 raid_end_bio_io(r10_bio); 2524 return; 2525 } 2526 2527 do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC); 2528 slot = r10_bio->read_slot; 2529 printk_ratelimited( 2530 KERN_ERR 2531 "md/raid10:%s: %s: redirecting " 2532 "sector %llu to another mirror\n", 2533 mdname(mddev), 2534 bdevname(rdev->bdev, b), 2535 (unsigned long long)r10_bio->sector); 2536 bio = bio_clone_mddev(r10_bio->master_bio, 2537 GFP_NOIO, mddev); 2538 bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors); 2539 r10_bio->devs[slot].bio = bio; 2540 r10_bio->devs[slot].rdev = rdev; 2541 bio->bi_iter.bi_sector = r10_bio->devs[slot].addr 2542 + choose_data_offset(r10_bio, rdev); 2543 bio->bi_bdev = rdev->bdev; 2544 bio->bi_rw = READ | do_sync; 2545 bio->bi_private = r10_bio; 2546 bio->bi_end_io = raid10_end_read_request; 2547 if (max_sectors < r10_bio->sectors) { 2548 /* Drat - have to split this up more */ 2549 struct bio *mbio = r10_bio->master_bio; 2550 int sectors_handled = 2551 r10_bio->sector + max_sectors 2552 - mbio->bi_iter.bi_sector; 2553 r10_bio->sectors = max_sectors; 2554 spin_lock_irq(&conf->device_lock); 2555 if (mbio->bi_phys_segments == 0) 2556 mbio->bi_phys_segments = 2; 2557 else 2558 mbio->bi_phys_segments++; 2559 spin_unlock_irq(&conf->device_lock); 2560 generic_make_request(bio); 2561 2562 r10_bio = mempool_alloc(conf->r10bio_pool, 2563 GFP_NOIO); 2564 r10_bio->master_bio = mbio; 2565 r10_bio->sectors = bio_sectors(mbio) - sectors_handled; 2566 r10_bio->state = 0; 2567 set_bit(R10BIO_ReadError, 2568 &r10_bio->state); 2569 r10_bio->mddev = mddev; 2570 r10_bio->sector = mbio->bi_iter.bi_sector 2571 + sectors_handled; 2572 2573 goto read_more; 2574 } else 2575 generic_make_request(bio); 2576 } 2577 2578 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2579 { 2580 /* Some sort of write request has finished and it 2581 * succeeded in writing where we thought there was a 2582 * bad block. So forget the bad block. 2583 * Or possibly if failed and we need to record 2584 * a bad block. 2585 */ 2586 int m; 2587 struct md_rdev *rdev; 2588 2589 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2590 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2591 for (m = 0; m < conf->copies; m++) { 2592 int dev = r10_bio->devs[m].devnum; 2593 rdev = conf->mirrors[dev].rdev; 2594 if (r10_bio->devs[m].bio == NULL) 2595 continue; 2596 if (!r10_bio->devs[m].bio->bi_error) { 2597 rdev_clear_badblocks( 2598 rdev, 2599 r10_bio->devs[m].addr, 2600 r10_bio->sectors, 0); 2601 } else { 2602 if (!rdev_set_badblocks( 2603 rdev, 2604 r10_bio->devs[m].addr, 2605 r10_bio->sectors, 0)) 2606 md_error(conf->mddev, rdev); 2607 } 2608 rdev = conf->mirrors[dev].replacement; 2609 if (r10_bio->devs[m].repl_bio == NULL) 2610 continue; 2611 2612 if (!r10_bio->devs[m].repl_bio->bi_error) { 2613 rdev_clear_badblocks( 2614 rdev, 2615 r10_bio->devs[m].addr, 2616 r10_bio->sectors, 0); 2617 } else { 2618 if (!rdev_set_badblocks( 2619 rdev, 2620 r10_bio->devs[m].addr, 2621 r10_bio->sectors, 0)) 2622 md_error(conf->mddev, rdev); 2623 } 2624 } 2625 put_buf(r10_bio); 2626 } else { 2627 bool fail = false; 2628 for (m = 0; m < conf->copies; m++) { 2629 int dev = r10_bio->devs[m].devnum; 2630 struct bio *bio = r10_bio->devs[m].bio; 2631 rdev = conf->mirrors[dev].rdev; 2632 if (bio == IO_MADE_GOOD) { 2633 rdev_clear_badblocks( 2634 rdev, 2635 r10_bio->devs[m].addr, 2636 r10_bio->sectors, 0); 2637 rdev_dec_pending(rdev, conf->mddev); 2638 } else if (bio != NULL && bio->bi_error) { 2639 fail = true; 2640 if (!narrow_write_error(r10_bio, m)) { 2641 md_error(conf->mddev, rdev); 2642 set_bit(R10BIO_Degraded, 2643 &r10_bio->state); 2644 } 2645 rdev_dec_pending(rdev, conf->mddev); 2646 } 2647 bio = r10_bio->devs[m].repl_bio; 2648 rdev = conf->mirrors[dev].replacement; 2649 if (rdev && bio == IO_MADE_GOOD) { 2650 rdev_clear_badblocks( 2651 rdev, 2652 r10_bio->devs[m].addr, 2653 r10_bio->sectors, 0); 2654 rdev_dec_pending(rdev, conf->mddev); 2655 } 2656 } 2657 if (test_bit(R10BIO_WriteError, 2658 &r10_bio->state)) 2659 close_write(r10_bio); 2660 if (fail) { 2661 spin_lock_irq(&conf->device_lock); 2662 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); 2663 spin_unlock_irq(&conf->device_lock); 2664 md_wakeup_thread(conf->mddev->thread); 2665 } else 2666 raid_end_bio_io(r10_bio); 2667 } 2668 } 2669 2670 static void raid10d(struct md_thread *thread) 2671 { 2672 struct mddev *mddev = thread->mddev; 2673 struct r10bio *r10_bio; 2674 unsigned long flags; 2675 struct r10conf *conf = mddev->private; 2676 struct list_head *head = &conf->retry_list; 2677 struct blk_plug plug; 2678 2679 md_check_recovery(mddev); 2680 2681 if (!list_empty_careful(&conf->bio_end_io_list) && 2682 !test_bit(MD_CHANGE_PENDING, &mddev->flags)) { 2683 LIST_HEAD(tmp); 2684 spin_lock_irqsave(&conf->device_lock, flags); 2685 if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) { 2686 list_add(&tmp, &conf->bio_end_io_list); 2687 list_del_init(&conf->bio_end_io_list); 2688 } 2689 spin_unlock_irqrestore(&conf->device_lock, flags); 2690 while (!list_empty(&tmp)) { 2691 r10_bio = list_first_entry(&conf->bio_end_io_list, 2692 struct r10bio, retry_list); 2693 list_del(&r10_bio->retry_list); 2694 raid_end_bio_io(r10_bio); 2695 } 2696 } 2697 2698 blk_start_plug(&plug); 2699 for (;;) { 2700 2701 flush_pending_writes(conf); 2702 2703 spin_lock_irqsave(&conf->device_lock, flags); 2704 if (list_empty(head)) { 2705 spin_unlock_irqrestore(&conf->device_lock, flags); 2706 break; 2707 } 2708 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 2709 list_del(head->prev); 2710 conf->nr_queued--; 2711 spin_unlock_irqrestore(&conf->device_lock, flags); 2712 2713 mddev = r10_bio->mddev; 2714 conf = mddev->private; 2715 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2716 test_bit(R10BIO_WriteError, &r10_bio->state)) 2717 handle_write_completed(conf, r10_bio); 2718 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 2719 reshape_request_write(mddev, r10_bio); 2720 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 2721 sync_request_write(mddev, r10_bio); 2722 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 2723 recovery_request_write(mddev, r10_bio); 2724 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 2725 handle_read_error(mddev, r10_bio); 2726 else { 2727 /* just a partial read to be scheduled from a 2728 * separate context 2729 */ 2730 int slot = r10_bio->read_slot; 2731 generic_make_request(r10_bio->devs[slot].bio); 2732 } 2733 2734 cond_resched(); 2735 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) 2736 md_check_recovery(mddev); 2737 } 2738 blk_finish_plug(&plug); 2739 } 2740 2741 static int init_resync(struct r10conf *conf) 2742 { 2743 int buffs; 2744 int i; 2745 2746 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2747 BUG_ON(conf->r10buf_pool); 2748 conf->have_replacement = 0; 2749 for (i = 0; i < conf->geo.raid_disks; i++) 2750 if (conf->mirrors[i].replacement) 2751 conf->have_replacement = 1; 2752 conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf); 2753 if (!conf->r10buf_pool) 2754 return -ENOMEM; 2755 conf->next_resync = 0; 2756 return 0; 2757 } 2758 2759 /* 2760 * perform a "sync" on one "block" 2761 * 2762 * We need to make sure that no normal I/O request - particularly write 2763 * requests - conflict with active sync requests. 2764 * 2765 * This is achieved by tracking pending requests and a 'barrier' concept 2766 * that can be installed to exclude normal IO requests. 2767 * 2768 * Resync and recovery are handled very differently. 2769 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 2770 * 2771 * For resync, we iterate over virtual addresses, read all copies, 2772 * and update if there are differences. If only one copy is live, 2773 * skip it. 2774 * For recovery, we iterate over physical addresses, read a good 2775 * value for each non-in_sync drive, and over-write. 2776 * 2777 * So, for recovery we may have several outstanding complex requests for a 2778 * given address, one for each out-of-sync device. We model this by allocating 2779 * a number of r10_bio structures, one for each out-of-sync device. 2780 * As we setup these structures, we collect all bio's together into a list 2781 * which we then process collectively to add pages, and then process again 2782 * to pass to generic_make_request. 2783 * 2784 * The r10_bio structures are linked using a borrowed master_bio pointer. 2785 * This link is counted in ->remaining. When the r10_bio that points to NULL 2786 * has its remaining count decremented to 0, the whole complex operation 2787 * is complete. 2788 * 2789 */ 2790 2791 static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, 2792 int *skipped) 2793 { 2794 struct r10conf *conf = mddev->private; 2795 struct r10bio *r10_bio; 2796 struct bio *biolist = NULL, *bio; 2797 sector_t max_sector, nr_sectors; 2798 int i; 2799 int max_sync; 2800 sector_t sync_blocks; 2801 sector_t sectors_skipped = 0; 2802 int chunks_skipped = 0; 2803 sector_t chunk_mask = conf->geo.chunk_mask; 2804 2805 if (!conf->r10buf_pool) 2806 if (init_resync(conf)) 2807 return 0; 2808 2809 /* 2810 * Allow skipping a full rebuild for incremental assembly 2811 * of a clean array, like RAID1 does. 2812 */ 2813 if (mddev->bitmap == NULL && 2814 mddev->recovery_cp == MaxSector && 2815 mddev->reshape_position == MaxSector && 2816 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2817 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2818 !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 2819 conf->fullsync == 0) { 2820 *skipped = 1; 2821 return mddev->dev_sectors - sector_nr; 2822 } 2823 2824 skipped: 2825 max_sector = mddev->dev_sectors; 2826 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2827 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2828 max_sector = mddev->resync_max_sectors; 2829 if (sector_nr >= max_sector) { 2830 /* If we aborted, we need to abort the 2831 * sync on the 'current' bitmap chucks (there can 2832 * be several when recovering multiple devices). 2833 * as we may have started syncing it but not finished. 2834 * We can find the current address in 2835 * mddev->curr_resync, but for recovery, 2836 * we need to convert that to several 2837 * virtual addresses. 2838 */ 2839 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 2840 end_reshape(conf); 2841 close_sync(conf); 2842 return 0; 2843 } 2844 2845 if (mddev->curr_resync < max_sector) { /* aborted */ 2846 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 2847 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2848 &sync_blocks, 1); 2849 else for (i = 0; i < conf->geo.raid_disks; i++) { 2850 sector_t sect = 2851 raid10_find_virt(conf, mddev->curr_resync, i); 2852 bitmap_end_sync(mddev->bitmap, sect, 2853 &sync_blocks, 1); 2854 } 2855 } else { 2856 /* completed sync */ 2857 if ((!mddev->bitmap || conf->fullsync) 2858 && conf->have_replacement 2859 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2860 /* Completed a full sync so the replacements 2861 * are now fully recovered. 2862 */ 2863 for (i = 0; i < conf->geo.raid_disks; i++) 2864 if (conf->mirrors[i].replacement) 2865 conf->mirrors[i].replacement 2866 ->recovery_offset 2867 = MaxSector; 2868 } 2869 conf->fullsync = 0; 2870 } 2871 bitmap_close_sync(mddev->bitmap); 2872 close_sync(conf); 2873 *skipped = 1; 2874 return sectors_skipped; 2875 } 2876 2877 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2878 return reshape_request(mddev, sector_nr, skipped); 2879 2880 if (chunks_skipped >= conf->geo.raid_disks) { 2881 /* if there has been nothing to do on any drive, 2882 * then there is nothing to do at all.. 2883 */ 2884 *skipped = 1; 2885 return (max_sector - sector_nr) + sectors_skipped; 2886 } 2887 2888 if (max_sector > mddev->resync_max) 2889 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2890 2891 /* make sure whole request will fit in a chunk - if chunks 2892 * are meaningful 2893 */ 2894 if (conf->geo.near_copies < conf->geo.raid_disks && 2895 max_sector > (sector_nr | chunk_mask)) 2896 max_sector = (sector_nr | chunk_mask) + 1; 2897 2898 /* Again, very different code for resync and recovery. 2899 * Both must result in an r10bio with a list of bios that 2900 * have bi_end_io, bi_sector, bi_bdev set, 2901 * and bi_private set to the r10bio. 2902 * For recovery, we may actually create several r10bios 2903 * with 2 bios in each, that correspond to the bios in the main one. 2904 * In this case, the subordinate r10bios link back through a 2905 * borrowed master_bio pointer, and the counter in the master 2906 * includes a ref from each subordinate. 2907 */ 2908 /* First, we decide what to do and set ->bi_end_io 2909 * To end_sync_read if we want to read, and 2910 * end_sync_write if we will want to write. 2911 */ 2912 2913 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 2914 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2915 /* recovery... the complicated one */ 2916 int j; 2917 r10_bio = NULL; 2918 2919 for (i = 0 ; i < conf->geo.raid_disks; i++) { 2920 int still_degraded; 2921 struct r10bio *rb2; 2922 sector_t sect; 2923 int must_sync; 2924 int any_working; 2925 struct raid10_info *mirror = &conf->mirrors[i]; 2926 2927 if ((mirror->rdev == NULL || 2928 test_bit(In_sync, &mirror->rdev->flags)) 2929 && 2930 (mirror->replacement == NULL || 2931 test_bit(Faulty, 2932 &mirror->replacement->flags))) 2933 continue; 2934 2935 still_degraded = 0; 2936 /* want to reconstruct this device */ 2937 rb2 = r10_bio; 2938 sect = raid10_find_virt(conf, sector_nr, i); 2939 if (sect >= mddev->resync_max_sectors) { 2940 /* last stripe is not complete - don't 2941 * try to recover this sector. 2942 */ 2943 continue; 2944 } 2945 /* Unless we are doing a full sync, or a replacement 2946 * we only need to recover the block if it is set in 2947 * the bitmap 2948 */ 2949 must_sync = bitmap_start_sync(mddev->bitmap, sect, 2950 &sync_blocks, 1); 2951 if (sync_blocks < max_sync) 2952 max_sync = sync_blocks; 2953 if (!must_sync && 2954 mirror->replacement == NULL && 2955 !conf->fullsync) { 2956 /* yep, skip the sync_blocks here, but don't assume 2957 * that there will never be anything to do here 2958 */ 2959 chunks_skipped = -1; 2960 continue; 2961 } 2962 2963 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 2964 r10_bio->state = 0; 2965 raise_barrier(conf, rb2 != NULL); 2966 atomic_set(&r10_bio->remaining, 0); 2967 2968 r10_bio->master_bio = (struct bio*)rb2; 2969 if (rb2) 2970 atomic_inc(&rb2->remaining); 2971 r10_bio->mddev = mddev; 2972 set_bit(R10BIO_IsRecover, &r10_bio->state); 2973 r10_bio->sector = sect; 2974 2975 raid10_find_phys(conf, r10_bio); 2976 2977 /* Need to check if the array will still be 2978 * degraded 2979 */ 2980 for (j = 0; j < conf->geo.raid_disks; j++) 2981 if (conf->mirrors[j].rdev == NULL || 2982 test_bit(Faulty, &conf->mirrors[j].rdev->flags)) { 2983 still_degraded = 1; 2984 break; 2985 } 2986 2987 must_sync = bitmap_start_sync(mddev->bitmap, sect, 2988 &sync_blocks, still_degraded); 2989 2990 any_working = 0; 2991 for (j=0; j<conf->copies;j++) { 2992 int k; 2993 int d = r10_bio->devs[j].devnum; 2994 sector_t from_addr, to_addr; 2995 struct md_rdev *rdev; 2996 sector_t sector, first_bad; 2997 int bad_sectors; 2998 if (!conf->mirrors[d].rdev || 2999 !test_bit(In_sync, &conf->mirrors[d].rdev->flags)) 3000 continue; 3001 /* This is where we read from */ 3002 any_working = 1; 3003 rdev = conf->mirrors[d].rdev; 3004 sector = r10_bio->devs[j].addr; 3005 3006 if (is_badblock(rdev, sector, max_sync, 3007 &first_bad, &bad_sectors)) { 3008 if (first_bad > sector) 3009 max_sync = first_bad - sector; 3010 else { 3011 bad_sectors -= (sector 3012 - first_bad); 3013 if (max_sync > bad_sectors) 3014 max_sync = bad_sectors; 3015 continue; 3016 } 3017 } 3018 bio = r10_bio->devs[0].bio; 3019 bio_reset(bio); 3020 bio->bi_next = biolist; 3021 biolist = bio; 3022 bio->bi_private = r10_bio; 3023 bio->bi_end_io = end_sync_read; 3024 bio->bi_rw = READ; 3025 from_addr = r10_bio->devs[j].addr; 3026 bio->bi_iter.bi_sector = from_addr + 3027 rdev->data_offset; 3028 bio->bi_bdev = rdev->bdev; 3029 atomic_inc(&rdev->nr_pending); 3030 /* and we write to 'i' (if not in_sync) */ 3031 3032 for (k=0; k<conf->copies; k++) 3033 if (r10_bio->devs[k].devnum == i) 3034 break; 3035 BUG_ON(k == conf->copies); 3036 to_addr = r10_bio->devs[k].addr; 3037 r10_bio->devs[0].devnum = d; 3038 r10_bio->devs[0].addr = from_addr; 3039 r10_bio->devs[1].devnum = i; 3040 r10_bio->devs[1].addr = to_addr; 3041 3042 rdev = mirror->rdev; 3043 if (!test_bit(In_sync, &rdev->flags)) { 3044 bio = r10_bio->devs[1].bio; 3045 bio_reset(bio); 3046 bio->bi_next = biolist; 3047 biolist = bio; 3048 bio->bi_private = r10_bio; 3049 bio->bi_end_io = end_sync_write; 3050 bio->bi_rw = WRITE; 3051 bio->bi_iter.bi_sector = to_addr 3052 + rdev->data_offset; 3053 bio->bi_bdev = rdev->bdev; 3054 atomic_inc(&r10_bio->remaining); 3055 } else 3056 r10_bio->devs[1].bio->bi_end_io = NULL; 3057 3058 /* and maybe write to replacement */ 3059 bio = r10_bio->devs[1].repl_bio; 3060 if (bio) 3061 bio->bi_end_io = NULL; 3062 rdev = mirror->replacement; 3063 /* Note: if rdev != NULL, then bio 3064 * cannot be NULL as r10buf_pool_alloc will 3065 * have allocated it. 3066 * So the second test here is pointless. 3067 * But it keeps semantic-checkers happy, and 3068 * this comment keeps human reviewers 3069 * happy. 3070 */ 3071 if (rdev == NULL || bio == NULL || 3072 test_bit(Faulty, &rdev->flags)) 3073 break; 3074 bio_reset(bio); 3075 bio->bi_next = biolist; 3076 biolist = bio; 3077 bio->bi_private = r10_bio; 3078 bio->bi_end_io = end_sync_write; 3079 bio->bi_rw = WRITE; 3080 bio->bi_iter.bi_sector = to_addr + 3081 rdev->data_offset; 3082 bio->bi_bdev = rdev->bdev; 3083 atomic_inc(&r10_bio->remaining); 3084 break; 3085 } 3086 if (j == conf->copies) { 3087 /* Cannot recover, so abort the recovery or 3088 * record a bad block */ 3089 if (any_working) { 3090 /* problem is that there are bad blocks 3091 * on other device(s) 3092 */ 3093 int k; 3094 for (k = 0; k < conf->copies; k++) 3095 if (r10_bio->devs[k].devnum == i) 3096 break; 3097 if (!test_bit(In_sync, 3098 &mirror->rdev->flags) 3099 && !rdev_set_badblocks( 3100 mirror->rdev, 3101 r10_bio->devs[k].addr, 3102 max_sync, 0)) 3103 any_working = 0; 3104 if (mirror->replacement && 3105 !rdev_set_badblocks( 3106 mirror->replacement, 3107 r10_bio->devs[k].addr, 3108 max_sync, 0)) 3109 any_working = 0; 3110 } 3111 if (!any_working) { 3112 if (!test_and_set_bit(MD_RECOVERY_INTR, 3113 &mddev->recovery)) 3114 printk(KERN_INFO "md/raid10:%s: insufficient " 3115 "working devices for recovery.\n", 3116 mdname(mddev)); 3117 mirror->recovery_disabled 3118 = mddev->recovery_disabled; 3119 } 3120 put_buf(r10_bio); 3121 if (rb2) 3122 atomic_dec(&rb2->remaining); 3123 r10_bio = rb2; 3124 break; 3125 } 3126 } 3127 if (biolist == NULL) { 3128 while (r10_bio) { 3129 struct r10bio *rb2 = r10_bio; 3130 r10_bio = (struct r10bio*) rb2->master_bio; 3131 rb2->master_bio = NULL; 3132 put_buf(rb2); 3133 } 3134 goto giveup; 3135 } 3136 } else { 3137 /* resync. Schedule a read for every block at this virt offset */ 3138 int count = 0; 3139 3140 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 3141 3142 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 3143 &sync_blocks, mddev->degraded) && 3144 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3145 &mddev->recovery)) { 3146 /* We can skip this block */ 3147 *skipped = 1; 3148 return sync_blocks + sectors_skipped; 3149 } 3150 if (sync_blocks < max_sync) 3151 max_sync = sync_blocks; 3152 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3153 r10_bio->state = 0; 3154 3155 r10_bio->mddev = mddev; 3156 atomic_set(&r10_bio->remaining, 0); 3157 raise_barrier(conf, 0); 3158 conf->next_resync = sector_nr; 3159 3160 r10_bio->master_bio = NULL; 3161 r10_bio->sector = sector_nr; 3162 set_bit(R10BIO_IsSync, &r10_bio->state); 3163 raid10_find_phys(conf, r10_bio); 3164 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3165 3166 for (i = 0; i < conf->copies; i++) { 3167 int d = r10_bio->devs[i].devnum; 3168 sector_t first_bad, sector; 3169 int bad_sectors; 3170 3171 if (r10_bio->devs[i].repl_bio) 3172 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3173 3174 bio = r10_bio->devs[i].bio; 3175 bio_reset(bio); 3176 bio->bi_error = -EIO; 3177 if (conf->mirrors[d].rdev == NULL || 3178 test_bit(Faulty, &conf->mirrors[d].rdev->flags)) 3179 continue; 3180 sector = r10_bio->devs[i].addr; 3181 if (is_badblock(conf->mirrors[d].rdev, 3182 sector, max_sync, 3183 &first_bad, &bad_sectors)) { 3184 if (first_bad > sector) 3185 max_sync = first_bad - sector; 3186 else { 3187 bad_sectors -= (sector - first_bad); 3188 if (max_sync > bad_sectors) 3189 max_sync = bad_sectors; 3190 continue; 3191 } 3192 } 3193 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 3194 atomic_inc(&r10_bio->remaining); 3195 bio->bi_next = biolist; 3196 biolist = bio; 3197 bio->bi_private = r10_bio; 3198 bio->bi_end_io = end_sync_read; 3199 bio->bi_rw = READ; 3200 bio->bi_iter.bi_sector = sector + 3201 conf->mirrors[d].rdev->data_offset; 3202 bio->bi_bdev = conf->mirrors[d].rdev->bdev; 3203 count++; 3204 3205 if (conf->mirrors[d].replacement == NULL || 3206 test_bit(Faulty, 3207 &conf->mirrors[d].replacement->flags)) 3208 continue; 3209 3210 /* Need to set up for writing to the replacement */ 3211 bio = r10_bio->devs[i].repl_bio; 3212 bio_reset(bio); 3213 bio->bi_error = -EIO; 3214 3215 sector = r10_bio->devs[i].addr; 3216 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 3217 bio->bi_next = biolist; 3218 biolist = bio; 3219 bio->bi_private = r10_bio; 3220 bio->bi_end_io = end_sync_write; 3221 bio->bi_rw = WRITE; 3222 bio->bi_iter.bi_sector = sector + 3223 conf->mirrors[d].replacement->data_offset; 3224 bio->bi_bdev = conf->mirrors[d].replacement->bdev; 3225 count++; 3226 } 3227 3228 if (count < 2) { 3229 for (i=0; i<conf->copies; i++) { 3230 int d = r10_bio->devs[i].devnum; 3231 if (r10_bio->devs[i].bio->bi_end_io) 3232 rdev_dec_pending(conf->mirrors[d].rdev, 3233 mddev); 3234 if (r10_bio->devs[i].repl_bio && 3235 r10_bio->devs[i].repl_bio->bi_end_io) 3236 rdev_dec_pending( 3237 conf->mirrors[d].replacement, 3238 mddev); 3239 } 3240 put_buf(r10_bio); 3241 biolist = NULL; 3242 goto giveup; 3243 } 3244 } 3245 3246 nr_sectors = 0; 3247 if (sector_nr + max_sync < max_sector) 3248 max_sector = sector_nr + max_sync; 3249 do { 3250 struct page *page; 3251 int len = PAGE_SIZE; 3252 if (sector_nr + (len>>9) > max_sector) 3253 len = (max_sector - sector_nr) << 9; 3254 if (len == 0) 3255 break; 3256 for (bio= biolist ; bio ; bio=bio->bi_next) { 3257 struct bio *bio2; 3258 page = bio->bi_io_vec[bio->bi_vcnt].bv_page; 3259 if (bio_add_page(bio, page, len, 0)) 3260 continue; 3261 3262 /* stop here */ 3263 bio->bi_io_vec[bio->bi_vcnt].bv_page = page; 3264 for (bio2 = biolist; 3265 bio2 && bio2 != bio; 3266 bio2 = bio2->bi_next) { 3267 /* remove last page from this bio */ 3268 bio2->bi_vcnt--; 3269 bio2->bi_iter.bi_size -= len; 3270 bio_clear_flag(bio2, BIO_SEG_VALID); 3271 } 3272 goto bio_full; 3273 } 3274 nr_sectors += len>>9; 3275 sector_nr += len>>9; 3276 } while (biolist->bi_vcnt < RESYNC_PAGES); 3277 bio_full: 3278 r10_bio->sectors = nr_sectors; 3279 3280 while (biolist) { 3281 bio = biolist; 3282 biolist = biolist->bi_next; 3283 3284 bio->bi_next = NULL; 3285 r10_bio = bio->bi_private; 3286 r10_bio->sectors = nr_sectors; 3287 3288 if (bio->bi_end_io == end_sync_read) { 3289 md_sync_acct(bio->bi_bdev, nr_sectors); 3290 bio->bi_error = 0; 3291 generic_make_request(bio); 3292 } 3293 } 3294 3295 if (sectors_skipped) 3296 /* pretend they weren't skipped, it makes 3297 * no important difference in this case 3298 */ 3299 md_done_sync(mddev, sectors_skipped, 1); 3300 3301 return sectors_skipped + nr_sectors; 3302 giveup: 3303 /* There is nowhere to write, so all non-sync 3304 * drives must be failed or in resync, all drives 3305 * have a bad block, so try the next chunk... 3306 */ 3307 if (sector_nr + max_sync < max_sector) 3308 max_sector = sector_nr + max_sync; 3309 3310 sectors_skipped += (max_sector - sector_nr); 3311 chunks_skipped ++; 3312 sector_nr = max_sector; 3313 goto skipped; 3314 } 3315 3316 static sector_t 3317 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3318 { 3319 sector_t size; 3320 struct r10conf *conf = mddev->private; 3321 3322 if (!raid_disks) 3323 raid_disks = min(conf->geo.raid_disks, 3324 conf->prev.raid_disks); 3325 if (!sectors) 3326 sectors = conf->dev_sectors; 3327 3328 size = sectors >> conf->geo.chunk_shift; 3329 sector_div(size, conf->geo.far_copies); 3330 size = size * raid_disks; 3331 sector_div(size, conf->geo.near_copies); 3332 3333 return size << conf->geo.chunk_shift; 3334 } 3335 3336 static void calc_sectors(struct r10conf *conf, sector_t size) 3337 { 3338 /* Calculate the number of sectors-per-device that will 3339 * actually be used, and set conf->dev_sectors and 3340 * conf->stride 3341 */ 3342 3343 size = size >> conf->geo.chunk_shift; 3344 sector_div(size, conf->geo.far_copies); 3345 size = size * conf->geo.raid_disks; 3346 sector_div(size, conf->geo.near_copies); 3347 /* 'size' is now the number of chunks in the array */ 3348 /* calculate "used chunks per device" */ 3349 size = size * conf->copies; 3350 3351 /* We need to round up when dividing by raid_disks to 3352 * get the stride size. 3353 */ 3354 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3355 3356 conf->dev_sectors = size << conf->geo.chunk_shift; 3357 3358 if (conf->geo.far_offset) 3359 conf->geo.stride = 1 << conf->geo.chunk_shift; 3360 else { 3361 sector_div(size, conf->geo.far_copies); 3362 conf->geo.stride = size << conf->geo.chunk_shift; 3363 } 3364 } 3365 3366 enum geo_type {geo_new, geo_old, geo_start}; 3367 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3368 { 3369 int nc, fc, fo; 3370 int layout, chunk, disks; 3371 switch (new) { 3372 case geo_old: 3373 layout = mddev->layout; 3374 chunk = mddev->chunk_sectors; 3375 disks = mddev->raid_disks - mddev->delta_disks; 3376 break; 3377 case geo_new: 3378 layout = mddev->new_layout; 3379 chunk = mddev->new_chunk_sectors; 3380 disks = mddev->raid_disks; 3381 break; 3382 default: /* avoid 'may be unused' warnings */ 3383 case geo_start: /* new when starting reshape - raid_disks not 3384 * updated yet. */ 3385 layout = mddev->new_layout; 3386 chunk = mddev->new_chunk_sectors; 3387 disks = mddev->raid_disks + mddev->delta_disks; 3388 break; 3389 } 3390 if (layout >> 18) 3391 return -1; 3392 if (chunk < (PAGE_SIZE >> 9) || 3393 !is_power_of_2(chunk)) 3394 return -2; 3395 nc = layout & 255; 3396 fc = (layout >> 8) & 255; 3397 fo = layout & (1<<16); 3398 geo->raid_disks = disks; 3399 geo->near_copies = nc; 3400 geo->far_copies = fc; 3401 geo->far_offset = fo; 3402 geo->far_set_size = (layout & (1<<17)) ? disks / fc : disks; 3403 geo->chunk_mask = chunk - 1; 3404 geo->chunk_shift = ffz(~chunk); 3405 return nc*fc; 3406 } 3407 3408 static struct r10conf *setup_conf(struct mddev *mddev) 3409 { 3410 struct r10conf *conf = NULL; 3411 int err = -EINVAL; 3412 struct geom geo; 3413 int copies; 3414 3415 copies = setup_geo(&geo, mddev, geo_new); 3416 3417 if (copies == -2) { 3418 printk(KERN_ERR "md/raid10:%s: chunk size must be " 3419 "at least PAGE_SIZE(%ld) and be a power of 2.\n", 3420 mdname(mddev), PAGE_SIZE); 3421 goto out; 3422 } 3423 3424 if (copies < 2 || copies > mddev->raid_disks) { 3425 printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3426 mdname(mddev), mddev->new_layout); 3427 goto out; 3428 } 3429 3430 err = -ENOMEM; 3431 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3432 if (!conf) 3433 goto out; 3434 3435 /* FIXME calc properly */ 3436 conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks + 3437 max(0,-mddev->delta_disks)), 3438 GFP_KERNEL); 3439 if (!conf->mirrors) 3440 goto out; 3441 3442 conf->tmppage = alloc_page(GFP_KERNEL); 3443 if (!conf->tmppage) 3444 goto out; 3445 3446 conf->geo = geo; 3447 conf->copies = copies; 3448 conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc, 3449 r10bio_pool_free, conf); 3450 if (!conf->r10bio_pool) 3451 goto out; 3452 3453 calc_sectors(conf, mddev->dev_sectors); 3454 if (mddev->reshape_position == MaxSector) { 3455 conf->prev = conf->geo; 3456 conf->reshape_progress = MaxSector; 3457 } else { 3458 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3459 err = -EINVAL; 3460 goto out; 3461 } 3462 conf->reshape_progress = mddev->reshape_position; 3463 if (conf->prev.far_offset) 3464 conf->prev.stride = 1 << conf->prev.chunk_shift; 3465 else 3466 /* far_copies must be 1 */ 3467 conf->prev.stride = conf->dev_sectors; 3468 } 3469 conf->reshape_safe = conf->reshape_progress; 3470 spin_lock_init(&conf->device_lock); 3471 INIT_LIST_HEAD(&conf->retry_list); 3472 INIT_LIST_HEAD(&conf->bio_end_io_list); 3473 3474 spin_lock_init(&conf->resync_lock); 3475 init_waitqueue_head(&conf->wait_barrier); 3476 3477 conf->thread = md_register_thread(raid10d, mddev, "raid10"); 3478 if (!conf->thread) 3479 goto out; 3480 3481 conf->mddev = mddev; 3482 return conf; 3483 3484 out: 3485 if (err == -ENOMEM) 3486 printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n", 3487 mdname(mddev)); 3488 if (conf) { 3489 if (conf->r10bio_pool) 3490 mempool_destroy(conf->r10bio_pool); 3491 kfree(conf->mirrors); 3492 safe_put_page(conf->tmppage); 3493 kfree(conf); 3494 } 3495 return ERR_PTR(err); 3496 } 3497 3498 static int run(struct mddev *mddev) 3499 { 3500 struct r10conf *conf; 3501 int i, disk_idx, chunk_size; 3502 struct raid10_info *disk; 3503 struct md_rdev *rdev; 3504 sector_t size; 3505 sector_t min_offset_diff = 0; 3506 int first = 1; 3507 bool discard_supported = false; 3508 3509 if (mddev->private == NULL) { 3510 conf = setup_conf(mddev); 3511 if (IS_ERR(conf)) 3512 return PTR_ERR(conf); 3513 mddev->private = conf; 3514 } 3515 conf = mddev->private; 3516 if (!conf) 3517 goto out; 3518 3519 mddev->thread = conf->thread; 3520 conf->thread = NULL; 3521 3522 chunk_size = mddev->chunk_sectors << 9; 3523 if (mddev->queue) { 3524 blk_queue_max_discard_sectors(mddev->queue, 3525 mddev->chunk_sectors); 3526 blk_queue_max_write_same_sectors(mddev->queue, 0); 3527 blk_queue_io_min(mddev->queue, chunk_size); 3528 if (conf->geo.raid_disks % conf->geo.near_copies) 3529 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks); 3530 else 3531 blk_queue_io_opt(mddev->queue, chunk_size * 3532 (conf->geo.raid_disks / conf->geo.near_copies)); 3533 } 3534 3535 rdev_for_each(rdev, mddev) { 3536 long long diff; 3537 struct request_queue *q; 3538 3539 disk_idx = rdev->raid_disk; 3540 if (disk_idx < 0) 3541 continue; 3542 if (disk_idx >= conf->geo.raid_disks && 3543 disk_idx >= conf->prev.raid_disks) 3544 continue; 3545 disk = conf->mirrors + disk_idx; 3546 3547 if (test_bit(Replacement, &rdev->flags)) { 3548 if (disk->replacement) 3549 goto out_free_conf; 3550 disk->replacement = rdev; 3551 } else { 3552 if (disk->rdev) 3553 goto out_free_conf; 3554 disk->rdev = rdev; 3555 } 3556 q = bdev_get_queue(rdev->bdev); 3557 diff = (rdev->new_data_offset - rdev->data_offset); 3558 if (!mddev->reshape_backwards) 3559 diff = -diff; 3560 if (diff < 0) 3561 diff = 0; 3562 if (first || diff < min_offset_diff) 3563 min_offset_diff = diff; 3564 3565 if (mddev->gendisk) 3566 disk_stack_limits(mddev->gendisk, rdev->bdev, 3567 rdev->data_offset << 9); 3568 3569 disk->head_position = 0; 3570 3571 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3572 discard_supported = true; 3573 } 3574 3575 if (mddev->queue) { 3576 if (discard_supported) 3577 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 3578 mddev->queue); 3579 else 3580 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 3581 mddev->queue); 3582 } 3583 /* need to check that every block has at least one working mirror */ 3584 if (!enough(conf, -1)) { 3585 printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n", 3586 mdname(mddev)); 3587 goto out_free_conf; 3588 } 3589 3590 if (conf->reshape_progress != MaxSector) { 3591 /* must ensure that shape change is supported */ 3592 if (conf->geo.far_copies != 1 && 3593 conf->geo.far_offset == 0) 3594 goto out_free_conf; 3595 if (conf->prev.far_copies != 1 && 3596 conf->prev.far_offset == 0) 3597 goto out_free_conf; 3598 } 3599 3600 mddev->degraded = 0; 3601 for (i = 0; 3602 i < conf->geo.raid_disks 3603 || i < conf->prev.raid_disks; 3604 i++) { 3605 3606 disk = conf->mirrors + i; 3607 3608 if (!disk->rdev && disk->replacement) { 3609 /* The replacement is all we have - use it */ 3610 disk->rdev = disk->replacement; 3611 disk->replacement = NULL; 3612 clear_bit(Replacement, &disk->rdev->flags); 3613 } 3614 3615 if (!disk->rdev || 3616 !test_bit(In_sync, &disk->rdev->flags)) { 3617 disk->head_position = 0; 3618 mddev->degraded++; 3619 if (disk->rdev && 3620 disk->rdev->saved_raid_disk < 0) 3621 conf->fullsync = 1; 3622 } 3623 disk->recovery_disabled = mddev->recovery_disabled - 1; 3624 } 3625 3626 if (mddev->recovery_cp != MaxSector) 3627 printk(KERN_NOTICE "md/raid10:%s: not clean" 3628 " -- starting background reconstruction\n", 3629 mdname(mddev)); 3630 printk(KERN_INFO 3631 "md/raid10:%s: active with %d out of %d devices\n", 3632 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 3633 conf->geo.raid_disks); 3634 /* 3635 * Ok, everything is just fine now 3636 */ 3637 mddev->dev_sectors = conf->dev_sectors; 3638 size = raid10_size(mddev, 0, 0); 3639 md_set_array_sectors(mddev, size); 3640 mddev->resync_max_sectors = size; 3641 3642 if (mddev->queue) { 3643 int stripe = conf->geo.raid_disks * 3644 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 3645 3646 /* Calculate max read-ahead size. 3647 * We need to readahead at least twice a whole stripe.... 3648 * maybe... 3649 */ 3650 stripe /= conf->geo.near_copies; 3651 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 3652 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 3653 } 3654 3655 if (md_integrity_register(mddev)) 3656 goto out_free_conf; 3657 3658 if (conf->reshape_progress != MaxSector) { 3659 unsigned long before_length, after_length; 3660 3661 before_length = ((1 << conf->prev.chunk_shift) * 3662 conf->prev.far_copies); 3663 after_length = ((1 << conf->geo.chunk_shift) * 3664 conf->geo.far_copies); 3665 3666 if (max(before_length, after_length) > min_offset_diff) { 3667 /* This cannot work */ 3668 printk("md/raid10: offset difference not enough to continue reshape\n"); 3669 goto out_free_conf; 3670 } 3671 conf->offset_diff = min_offset_diff; 3672 3673 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 3674 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 3675 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 3676 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 3677 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 3678 "reshape"); 3679 } 3680 3681 return 0; 3682 3683 out_free_conf: 3684 md_unregister_thread(&mddev->thread); 3685 if (conf->r10bio_pool) 3686 mempool_destroy(conf->r10bio_pool); 3687 safe_put_page(conf->tmppage); 3688 kfree(conf->mirrors); 3689 kfree(conf); 3690 mddev->private = NULL; 3691 out: 3692 return -EIO; 3693 } 3694 3695 static void raid10_free(struct mddev *mddev, void *priv) 3696 { 3697 struct r10conf *conf = priv; 3698 3699 if (conf->r10bio_pool) 3700 mempool_destroy(conf->r10bio_pool); 3701 safe_put_page(conf->tmppage); 3702 kfree(conf->mirrors); 3703 kfree(conf->mirrors_old); 3704 kfree(conf->mirrors_new); 3705 kfree(conf); 3706 } 3707 3708 static void raid10_quiesce(struct mddev *mddev, int state) 3709 { 3710 struct r10conf *conf = mddev->private; 3711 3712 switch(state) { 3713 case 1: 3714 raise_barrier(conf, 0); 3715 break; 3716 case 0: 3717 lower_barrier(conf); 3718 break; 3719 } 3720 } 3721 3722 static int raid10_resize(struct mddev *mddev, sector_t sectors) 3723 { 3724 /* Resize of 'far' arrays is not supported. 3725 * For 'near' and 'offset' arrays we can set the 3726 * number of sectors used to be an appropriate multiple 3727 * of the chunk size. 3728 * For 'offset', this is far_copies*chunksize. 3729 * For 'near' the multiplier is the LCM of 3730 * near_copies and raid_disks. 3731 * So if far_copies > 1 && !far_offset, fail. 3732 * Else find LCM(raid_disks, near_copy)*far_copies and 3733 * multiply by chunk_size. Then round to this number. 3734 * This is mostly done by raid10_size() 3735 */ 3736 struct r10conf *conf = mddev->private; 3737 sector_t oldsize, size; 3738 3739 if (mddev->reshape_position != MaxSector) 3740 return -EBUSY; 3741 3742 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 3743 return -EINVAL; 3744 3745 oldsize = raid10_size(mddev, 0, 0); 3746 size = raid10_size(mddev, sectors, 0); 3747 if (mddev->external_size && 3748 mddev->array_sectors > size) 3749 return -EINVAL; 3750 if (mddev->bitmap) { 3751 int ret = bitmap_resize(mddev->bitmap, size, 0, 0); 3752 if (ret) 3753 return ret; 3754 } 3755 md_set_array_sectors(mddev, size); 3756 set_capacity(mddev->gendisk, mddev->array_sectors); 3757 revalidate_disk(mddev->gendisk); 3758 if (sectors > mddev->dev_sectors && 3759 mddev->recovery_cp > oldsize) { 3760 mddev->recovery_cp = oldsize; 3761 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3762 } 3763 calc_sectors(conf, sectors); 3764 mddev->dev_sectors = conf->dev_sectors; 3765 mddev->resync_max_sectors = size; 3766 return 0; 3767 } 3768 3769 static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) 3770 { 3771 struct md_rdev *rdev; 3772 struct r10conf *conf; 3773 3774 if (mddev->degraded > 0) { 3775 printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n", 3776 mdname(mddev)); 3777 return ERR_PTR(-EINVAL); 3778 } 3779 sector_div(size, devs); 3780 3781 /* Set new parameters */ 3782 mddev->new_level = 10; 3783 /* new layout: far_copies = 1, near_copies = 2 */ 3784 mddev->new_layout = (1<<8) + 2; 3785 mddev->new_chunk_sectors = mddev->chunk_sectors; 3786 mddev->delta_disks = mddev->raid_disks; 3787 mddev->raid_disks *= 2; 3788 /* make sure it will be not marked as dirty */ 3789 mddev->recovery_cp = MaxSector; 3790 mddev->dev_sectors = size; 3791 3792 conf = setup_conf(mddev); 3793 if (!IS_ERR(conf)) { 3794 rdev_for_each(rdev, mddev) 3795 if (rdev->raid_disk >= 0) { 3796 rdev->new_raid_disk = rdev->raid_disk * 2; 3797 rdev->sectors = size; 3798 } 3799 conf->barrier = 1; 3800 } 3801 3802 return conf; 3803 } 3804 3805 static void *raid10_takeover(struct mddev *mddev) 3806 { 3807 struct r0conf *raid0_conf; 3808 3809 /* raid10 can take over: 3810 * raid0 - providing it has only two drives 3811 */ 3812 if (mddev->level == 0) { 3813 /* for raid0 takeover only one zone is supported */ 3814 raid0_conf = mddev->private; 3815 if (raid0_conf->nr_strip_zones > 1) { 3816 printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0" 3817 " with more than one zone.\n", 3818 mdname(mddev)); 3819 return ERR_PTR(-EINVAL); 3820 } 3821 return raid10_takeover_raid0(mddev, 3822 raid0_conf->strip_zone->zone_end, 3823 raid0_conf->strip_zone->nb_dev); 3824 } 3825 return ERR_PTR(-EINVAL); 3826 } 3827 3828 static int raid10_check_reshape(struct mddev *mddev) 3829 { 3830 /* Called when there is a request to change 3831 * - layout (to ->new_layout) 3832 * - chunk size (to ->new_chunk_sectors) 3833 * - raid_disks (by delta_disks) 3834 * or when trying to restart a reshape that was ongoing. 3835 * 3836 * We need to validate the request and possibly allocate 3837 * space if that might be an issue later. 3838 * 3839 * Currently we reject any reshape of a 'far' mode array, 3840 * allow chunk size to change if new is generally acceptable, 3841 * allow raid_disks to increase, and allow 3842 * a switch between 'near' mode and 'offset' mode. 3843 */ 3844 struct r10conf *conf = mddev->private; 3845 struct geom geo; 3846 3847 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 3848 return -EINVAL; 3849 3850 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 3851 /* mustn't change number of copies */ 3852 return -EINVAL; 3853 if (geo.far_copies > 1 && !geo.far_offset) 3854 /* Cannot switch to 'far' mode */ 3855 return -EINVAL; 3856 3857 if (mddev->array_sectors & geo.chunk_mask) 3858 /* not factor of array size */ 3859 return -EINVAL; 3860 3861 if (!enough(conf, -1)) 3862 return -EINVAL; 3863 3864 kfree(conf->mirrors_new); 3865 conf->mirrors_new = NULL; 3866 if (mddev->delta_disks > 0) { 3867 /* allocate new 'mirrors' list */ 3868 conf->mirrors_new = kzalloc( 3869 sizeof(struct raid10_info) 3870 *(mddev->raid_disks + 3871 mddev->delta_disks), 3872 GFP_KERNEL); 3873 if (!conf->mirrors_new) 3874 return -ENOMEM; 3875 } 3876 return 0; 3877 } 3878 3879 /* 3880 * Need to check if array has failed when deciding whether to: 3881 * - start an array 3882 * - remove non-faulty devices 3883 * - add a spare 3884 * - allow a reshape 3885 * This determination is simple when no reshape is happening. 3886 * However if there is a reshape, we need to carefully check 3887 * both the before and after sections. 3888 * This is because some failed devices may only affect one 3889 * of the two sections, and some non-in_sync devices may 3890 * be insync in the section most affected by failed devices. 3891 */ 3892 static int calc_degraded(struct r10conf *conf) 3893 { 3894 int degraded, degraded2; 3895 int i; 3896 3897 rcu_read_lock(); 3898 degraded = 0; 3899 /* 'prev' section first */ 3900 for (i = 0; i < conf->prev.raid_disks; i++) { 3901 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 3902 if (!rdev || test_bit(Faulty, &rdev->flags)) 3903 degraded++; 3904 else if (!test_bit(In_sync, &rdev->flags)) 3905 /* When we can reduce the number of devices in 3906 * an array, this might not contribute to 3907 * 'degraded'. It does now. 3908 */ 3909 degraded++; 3910 } 3911 rcu_read_unlock(); 3912 if (conf->geo.raid_disks == conf->prev.raid_disks) 3913 return degraded; 3914 rcu_read_lock(); 3915 degraded2 = 0; 3916 for (i = 0; i < conf->geo.raid_disks; i++) { 3917 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 3918 if (!rdev || test_bit(Faulty, &rdev->flags)) 3919 degraded2++; 3920 else if (!test_bit(In_sync, &rdev->flags)) { 3921 /* If reshape is increasing the number of devices, 3922 * this section has already been recovered, so 3923 * it doesn't contribute to degraded. 3924 * else it does. 3925 */ 3926 if (conf->geo.raid_disks <= conf->prev.raid_disks) 3927 degraded2++; 3928 } 3929 } 3930 rcu_read_unlock(); 3931 if (degraded2 > degraded) 3932 return degraded2; 3933 return degraded; 3934 } 3935 3936 static int raid10_start_reshape(struct mddev *mddev) 3937 { 3938 /* A 'reshape' has been requested. This commits 3939 * the various 'new' fields and sets MD_RECOVER_RESHAPE 3940 * This also checks if there are enough spares and adds them 3941 * to the array. 3942 * We currently require enough spares to make the final 3943 * array non-degraded. We also require that the difference 3944 * between old and new data_offset - on each device - is 3945 * enough that we never risk over-writing. 3946 */ 3947 3948 unsigned long before_length, after_length; 3949 sector_t min_offset_diff = 0; 3950 int first = 1; 3951 struct geom new; 3952 struct r10conf *conf = mddev->private; 3953 struct md_rdev *rdev; 3954 int spares = 0; 3955 int ret; 3956 3957 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 3958 return -EBUSY; 3959 3960 if (setup_geo(&new, mddev, geo_start) != conf->copies) 3961 return -EINVAL; 3962 3963 before_length = ((1 << conf->prev.chunk_shift) * 3964 conf->prev.far_copies); 3965 after_length = ((1 << conf->geo.chunk_shift) * 3966 conf->geo.far_copies); 3967 3968 rdev_for_each(rdev, mddev) { 3969 if (!test_bit(In_sync, &rdev->flags) 3970 && !test_bit(Faulty, &rdev->flags)) 3971 spares++; 3972 if (rdev->raid_disk >= 0) { 3973 long long diff = (rdev->new_data_offset 3974 - rdev->data_offset); 3975 if (!mddev->reshape_backwards) 3976 diff = -diff; 3977 if (diff < 0) 3978 diff = 0; 3979 if (first || diff < min_offset_diff) 3980 min_offset_diff = diff; 3981 } 3982 } 3983 3984 if (max(before_length, after_length) > min_offset_diff) 3985 return -EINVAL; 3986 3987 if (spares < mddev->delta_disks) 3988 return -EINVAL; 3989 3990 conf->offset_diff = min_offset_diff; 3991 spin_lock_irq(&conf->device_lock); 3992 if (conf->mirrors_new) { 3993 memcpy(conf->mirrors_new, conf->mirrors, 3994 sizeof(struct raid10_info)*conf->prev.raid_disks); 3995 smp_mb(); 3996 kfree(conf->mirrors_old); 3997 conf->mirrors_old = conf->mirrors; 3998 conf->mirrors = conf->mirrors_new; 3999 conf->mirrors_new = NULL; 4000 } 4001 setup_geo(&conf->geo, mddev, geo_start); 4002 smp_mb(); 4003 if (mddev->reshape_backwards) { 4004 sector_t size = raid10_size(mddev, 0, 0); 4005 if (size < mddev->array_sectors) { 4006 spin_unlock_irq(&conf->device_lock); 4007 printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n", 4008 mdname(mddev)); 4009 return -EINVAL; 4010 } 4011 mddev->resync_max_sectors = size; 4012 conf->reshape_progress = size; 4013 } else 4014 conf->reshape_progress = 0; 4015 conf->reshape_safe = conf->reshape_progress; 4016 spin_unlock_irq(&conf->device_lock); 4017 4018 if (mddev->delta_disks && mddev->bitmap) { 4019 ret = bitmap_resize(mddev->bitmap, 4020 raid10_size(mddev, 0, 4021 conf->geo.raid_disks), 4022 0, 0); 4023 if (ret) 4024 goto abort; 4025 } 4026 if (mddev->delta_disks > 0) { 4027 rdev_for_each(rdev, mddev) 4028 if (rdev->raid_disk < 0 && 4029 !test_bit(Faulty, &rdev->flags)) { 4030 if (raid10_add_disk(mddev, rdev) == 0) { 4031 if (rdev->raid_disk >= 4032 conf->prev.raid_disks) 4033 set_bit(In_sync, &rdev->flags); 4034 else 4035 rdev->recovery_offset = 0; 4036 4037 if (sysfs_link_rdev(mddev, rdev)) 4038 /* Failure here is OK */; 4039 } 4040 } else if (rdev->raid_disk >= conf->prev.raid_disks 4041 && !test_bit(Faulty, &rdev->flags)) { 4042 /* This is a spare that was manually added */ 4043 set_bit(In_sync, &rdev->flags); 4044 } 4045 } 4046 /* When a reshape changes the number of devices, 4047 * ->degraded is measured against the larger of the 4048 * pre and post numbers. 4049 */ 4050 spin_lock_irq(&conf->device_lock); 4051 mddev->degraded = calc_degraded(conf); 4052 spin_unlock_irq(&conf->device_lock); 4053 mddev->raid_disks = conf->geo.raid_disks; 4054 mddev->reshape_position = conf->reshape_progress; 4055 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4056 4057 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4058 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4059 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 4060 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4061 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4062 4063 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4064 "reshape"); 4065 if (!mddev->sync_thread) { 4066 ret = -EAGAIN; 4067 goto abort; 4068 } 4069 conf->reshape_checkpoint = jiffies; 4070 md_wakeup_thread(mddev->sync_thread); 4071 md_new_event(mddev); 4072 return 0; 4073 4074 abort: 4075 mddev->recovery = 0; 4076 spin_lock_irq(&conf->device_lock); 4077 conf->geo = conf->prev; 4078 mddev->raid_disks = conf->geo.raid_disks; 4079 rdev_for_each(rdev, mddev) 4080 rdev->new_data_offset = rdev->data_offset; 4081 smp_wmb(); 4082 conf->reshape_progress = MaxSector; 4083 conf->reshape_safe = MaxSector; 4084 mddev->reshape_position = MaxSector; 4085 spin_unlock_irq(&conf->device_lock); 4086 return ret; 4087 } 4088 4089 /* Calculate the last device-address that could contain 4090 * any block from the chunk that includes the array-address 's' 4091 * and report the next address. 4092 * i.e. the address returned will be chunk-aligned and after 4093 * any data that is in the chunk containing 's'. 4094 */ 4095 static sector_t last_dev_address(sector_t s, struct geom *geo) 4096 { 4097 s = (s | geo->chunk_mask) + 1; 4098 s >>= geo->chunk_shift; 4099 s *= geo->near_copies; 4100 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4101 s *= geo->far_copies; 4102 s <<= geo->chunk_shift; 4103 return s; 4104 } 4105 4106 /* Calculate the first device-address that could contain 4107 * any block from the chunk that includes the array-address 's'. 4108 * This too will be the start of a chunk 4109 */ 4110 static sector_t first_dev_address(sector_t s, struct geom *geo) 4111 { 4112 s >>= geo->chunk_shift; 4113 s *= geo->near_copies; 4114 sector_div(s, geo->raid_disks); 4115 s *= geo->far_copies; 4116 s <<= geo->chunk_shift; 4117 return s; 4118 } 4119 4120 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4121 int *skipped) 4122 { 4123 /* We simply copy at most one chunk (smallest of old and new) 4124 * at a time, possibly less if that exceeds RESYNC_PAGES, 4125 * or we hit a bad block or something. 4126 * This might mean we pause for normal IO in the middle of 4127 * a chunk, but that is not a problem as mddev->reshape_position 4128 * can record any location. 4129 * 4130 * If we will want to write to a location that isn't 4131 * yet recorded as 'safe' (i.e. in metadata on disk) then 4132 * we need to flush all reshape requests and update the metadata. 4133 * 4134 * When reshaping forwards (e.g. to more devices), we interpret 4135 * 'safe' as the earliest block which might not have been copied 4136 * down yet. We divide this by previous stripe size and multiply 4137 * by previous stripe length to get lowest device offset that we 4138 * cannot write to yet. 4139 * We interpret 'sector_nr' as an address that we want to write to. 4140 * From this we use last_device_address() to find where we might 4141 * write to, and first_device_address on the 'safe' position. 4142 * If this 'next' write position is after the 'safe' position, 4143 * we must update the metadata to increase the 'safe' position. 4144 * 4145 * When reshaping backwards, we round in the opposite direction 4146 * and perform the reverse test: next write position must not be 4147 * less than current safe position. 4148 * 4149 * In all this the minimum difference in data offsets 4150 * (conf->offset_diff - always positive) allows a bit of slack, 4151 * so next can be after 'safe', but not by more than offset_diff 4152 * 4153 * We need to prepare all the bios here before we start any IO 4154 * to ensure the size we choose is acceptable to all devices. 4155 * The means one for each copy for write-out and an extra one for 4156 * read-in. 4157 * We store the read-in bio in ->master_bio and the others in 4158 * ->devs[x].bio and ->devs[x].repl_bio. 4159 */ 4160 struct r10conf *conf = mddev->private; 4161 struct r10bio *r10_bio; 4162 sector_t next, safe, last; 4163 int max_sectors; 4164 int nr_sectors; 4165 int s; 4166 struct md_rdev *rdev; 4167 int need_flush = 0; 4168 struct bio *blist; 4169 struct bio *bio, *read_bio; 4170 int sectors_done = 0; 4171 4172 if (sector_nr == 0) { 4173 /* If restarting in the middle, skip the initial sectors */ 4174 if (mddev->reshape_backwards && 4175 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4176 sector_nr = (raid10_size(mddev, 0, 0) 4177 - conf->reshape_progress); 4178 } else if (!mddev->reshape_backwards && 4179 conf->reshape_progress > 0) 4180 sector_nr = conf->reshape_progress; 4181 if (sector_nr) { 4182 mddev->curr_resync_completed = sector_nr; 4183 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4184 *skipped = 1; 4185 return sector_nr; 4186 } 4187 } 4188 4189 /* We don't use sector_nr to track where we are up to 4190 * as that doesn't work well for ->reshape_backwards. 4191 * So just use ->reshape_progress. 4192 */ 4193 if (mddev->reshape_backwards) { 4194 /* 'next' is the earliest device address that we might 4195 * write to for this chunk in the new layout 4196 */ 4197 next = first_dev_address(conf->reshape_progress - 1, 4198 &conf->geo); 4199 4200 /* 'safe' is the last device address that we might read from 4201 * in the old layout after a restart 4202 */ 4203 safe = last_dev_address(conf->reshape_safe - 1, 4204 &conf->prev); 4205 4206 if (next + conf->offset_diff < safe) 4207 need_flush = 1; 4208 4209 last = conf->reshape_progress - 1; 4210 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4211 & conf->prev.chunk_mask); 4212 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last) 4213 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512; 4214 } else { 4215 /* 'next' is after the last device address that we 4216 * might write to for this chunk in the new layout 4217 */ 4218 next = last_dev_address(conf->reshape_progress, &conf->geo); 4219 4220 /* 'safe' is the earliest device address that we might 4221 * read from in the old layout after a restart 4222 */ 4223 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4224 4225 /* Need to update metadata if 'next' might be beyond 'safe' 4226 * as that would possibly corrupt data 4227 */ 4228 if (next > safe + conf->offset_diff) 4229 need_flush = 1; 4230 4231 sector_nr = conf->reshape_progress; 4232 last = sector_nr | (conf->geo.chunk_mask 4233 & conf->prev.chunk_mask); 4234 4235 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last) 4236 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1; 4237 } 4238 4239 if (need_flush || 4240 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4241 /* Need to update reshape_position in metadata */ 4242 wait_barrier(conf); 4243 mddev->reshape_position = conf->reshape_progress; 4244 if (mddev->reshape_backwards) 4245 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4246 - conf->reshape_progress; 4247 else 4248 mddev->curr_resync_completed = conf->reshape_progress; 4249 conf->reshape_checkpoint = jiffies; 4250 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4251 md_wakeup_thread(mddev->thread); 4252 wait_event(mddev->sb_wait, mddev->flags == 0 || 4253 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4254 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 4255 allow_barrier(conf); 4256 return sectors_done; 4257 } 4258 conf->reshape_safe = mddev->reshape_position; 4259 allow_barrier(conf); 4260 } 4261 4262 read_more: 4263 /* Now schedule reads for blocks from sector_nr to last */ 4264 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 4265 r10_bio->state = 0; 4266 raise_barrier(conf, sectors_done != 0); 4267 atomic_set(&r10_bio->remaining, 0); 4268 r10_bio->mddev = mddev; 4269 r10_bio->sector = sector_nr; 4270 set_bit(R10BIO_IsReshape, &r10_bio->state); 4271 r10_bio->sectors = last - sector_nr + 1; 4272 rdev = read_balance(conf, r10_bio, &max_sectors); 4273 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4274 4275 if (!rdev) { 4276 /* Cannot read from here, so need to record bad blocks 4277 * on all the target devices. 4278 */ 4279 // FIXME 4280 mempool_free(r10_bio, conf->r10buf_pool); 4281 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4282 return sectors_done; 4283 } 4284 4285 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev); 4286 4287 read_bio->bi_bdev = rdev->bdev; 4288 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4289 + rdev->data_offset); 4290 read_bio->bi_private = r10_bio; 4291 read_bio->bi_end_io = end_sync_read; 4292 read_bio->bi_rw = READ; 4293 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); 4294 read_bio->bi_error = 0; 4295 read_bio->bi_vcnt = 0; 4296 read_bio->bi_iter.bi_size = 0; 4297 r10_bio->master_bio = read_bio; 4298 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4299 4300 /* Now find the locations in the new layout */ 4301 __raid10_find_phys(&conf->geo, r10_bio); 4302 4303 blist = read_bio; 4304 read_bio->bi_next = NULL; 4305 4306 for (s = 0; s < conf->copies*2; s++) { 4307 struct bio *b; 4308 int d = r10_bio->devs[s/2].devnum; 4309 struct md_rdev *rdev2; 4310 if (s&1) { 4311 rdev2 = conf->mirrors[d].replacement; 4312 b = r10_bio->devs[s/2].repl_bio; 4313 } else { 4314 rdev2 = conf->mirrors[d].rdev; 4315 b = r10_bio->devs[s/2].bio; 4316 } 4317 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4318 continue; 4319 4320 bio_reset(b); 4321 b->bi_bdev = rdev2->bdev; 4322 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + 4323 rdev2->new_data_offset; 4324 b->bi_private = r10_bio; 4325 b->bi_end_io = end_reshape_write; 4326 b->bi_rw = WRITE; 4327 b->bi_next = blist; 4328 blist = b; 4329 } 4330 4331 /* Now add as many pages as possible to all of these bios. */ 4332 4333 nr_sectors = 0; 4334 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4335 struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page; 4336 int len = (max_sectors - s) << 9; 4337 if (len > PAGE_SIZE) 4338 len = PAGE_SIZE; 4339 for (bio = blist; bio ; bio = bio->bi_next) { 4340 struct bio *bio2; 4341 if (bio_add_page(bio, page, len, 0)) 4342 continue; 4343 4344 /* Didn't fit, must stop */ 4345 for (bio2 = blist; 4346 bio2 && bio2 != bio; 4347 bio2 = bio2->bi_next) { 4348 /* Remove last page from this bio */ 4349 bio2->bi_vcnt--; 4350 bio2->bi_iter.bi_size -= len; 4351 bio_clear_flag(bio2, BIO_SEG_VALID); 4352 } 4353 goto bio_full; 4354 } 4355 sector_nr += len >> 9; 4356 nr_sectors += len >> 9; 4357 } 4358 bio_full: 4359 r10_bio->sectors = nr_sectors; 4360 4361 /* Now submit the read */ 4362 md_sync_acct(read_bio->bi_bdev, r10_bio->sectors); 4363 atomic_inc(&r10_bio->remaining); 4364 read_bio->bi_next = NULL; 4365 generic_make_request(read_bio); 4366 sector_nr += nr_sectors; 4367 sectors_done += nr_sectors; 4368 if (sector_nr <= last) 4369 goto read_more; 4370 4371 /* Now that we have done the whole section we can 4372 * update reshape_progress 4373 */ 4374 if (mddev->reshape_backwards) 4375 conf->reshape_progress -= sectors_done; 4376 else 4377 conf->reshape_progress += sectors_done; 4378 4379 return sectors_done; 4380 } 4381 4382 static void end_reshape_request(struct r10bio *r10_bio); 4383 static int handle_reshape_read_error(struct mddev *mddev, 4384 struct r10bio *r10_bio); 4385 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4386 { 4387 /* Reshape read completed. Hopefully we have a block 4388 * to write out. 4389 * If we got a read error then we do sync 1-page reads from 4390 * elsewhere until we find the data - or give up. 4391 */ 4392 struct r10conf *conf = mddev->private; 4393 int s; 4394 4395 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4396 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4397 /* Reshape has been aborted */ 4398 md_done_sync(mddev, r10_bio->sectors, 0); 4399 return; 4400 } 4401 4402 /* We definitely have the data in the pages, schedule the 4403 * writes. 4404 */ 4405 atomic_set(&r10_bio->remaining, 1); 4406 for (s = 0; s < conf->copies*2; s++) { 4407 struct bio *b; 4408 int d = r10_bio->devs[s/2].devnum; 4409 struct md_rdev *rdev; 4410 if (s&1) { 4411 rdev = conf->mirrors[d].replacement; 4412 b = r10_bio->devs[s/2].repl_bio; 4413 } else { 4414 rdev = conf->mirrors[d].rdev; 4415 b = r10_bio->devs[s/2].bio; 4416 } 4417 if (!rdev || test_bit(Faulty, &rdev->flags)) 4418 continue; 4419 atomic_inc(&rdev->nr_pending); 4420 md_sync_acct(b->bi_bdev, r10_bio->sectors); 4421 atomic_inc(&r10_bio->remaining); 4422 b->bi_next = NULL; 4423 generic_make_request(b); 4424 } 4425 end_reshape_request(r10_bio); 4426 } 4427 4428 static void end_reshape(struct r10conf *conf) 4429 { 4430 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4431 return; 4432 4433 spin_lock_irq(&conf->device_lock); 4434 conf->prev = conf->geo; 4435 md_finish_reshape(conf->mddev); 4436 smp_wmb(); 4437 conf->reshape_progress = MaxSector; 4438 conf->reshape_safe = MaxSector; 4439 spin_unlock_irq(&conf->device_lock); 4440 4441 /* read-ahead size must cover two whole stripes, which is 4442 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4443 */ 4444 if (conf->mddev->queue) { 4445 int stripe = conf->geo.raid_disks * 4446 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE); 4447 stripe /= conf->geo.near_copies; 4448 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4449 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4450 } 4451 conf->fullsync = 0; 4452 } 4453 4454 static int handle_reshape_read_error(struct mddev *mddev, 4455 struct r10bio *r10_bio) 4456 { 4457 /* Use sync reads to get the blocks from somewhere else */ 4458 int sectors = r10_bio->sectors; 4459 struct r10conf *conf = mddev->private; 4460 struct { 4461 struct r10bio r10_bio; 4462 struct r10dev devs[conf->copies]; 4463 } on_stack; 4464 struct r10bio *r10b = &on_stack.r10_bio; 4465 int slot = 0; 4466 int idx = 0; 4467 struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec; 4468 4469 r10b->sector = r10_bio->sector; 4470 __raid10_find_phys(&conf->prev, r10b); 4471 4472 while (sectors) { 4473 int s = sectors; 4474 int success = 0; 4475 int first_slot = slot; 4476 4477 if (s > (PAGE_SIZE >> 9)) 4478 s = PAGE_SIZE >> 9; 4479 4480 while (!success) { 4481 int d = r10b->devs[slot].devnum; 4482 struct md_rdev *rdev = conf->mirrors[d].rdev; 4483 sector_t addr; 4484 if (rdev == NULL || 4485 test_bit(Faulty, &rdev->flags) || 4486 !test_bit(In_sync, &rdev->flags)) 4487 goto failed; 4488 4489 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 4490 success = sync_page_io(rdev, 4491 addr, 4492 s << 9, 4493 bvec[idx].bv_page, 4494 READ, false); 4495 if (success) 4496 break; 4497 failed: 4498 slot++; 4499 if (slot >= conf->copies) 4500 slot = 0; 4501 if (slot == first_slot) 4502 break; 4503 } 4504 if (!success) { 4505 /* couldn't read this block, must give up */ 4506 set_bit(MD_RECOVERY_INTR, 4507 &mddev->recovery); 4508 return -EIO; 4509 } 4510 sectors -= s; 4511 idx++; 4512 } 4513 return 0; 4514 } 4515 4516 static void end_reshape_write(struct bio *bio) 4517 { 4518 struct r10bio *r10_bio = bio->bi_private; 4519 struct mddev *mddev = r10_bio->mddev; 4520 struct r10conf *conf = mddev->private; 4521 int d; 4522 int slot; 4523 int repl; 4524 struct md_rdev *rdev = NULL; 4525 4526 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 4527 if (repl) 4528 rdev = conf->mirrors[d].replacement; 4529 if (!rdev) { 4530 smp_mb(); 4531 rdev = conf->mirrors[d].rdev; 4532 } 4533 4534 if (bio->bi_error) { 4535 /* FIXME should record badblock */ 4536 md_error(mddev, rdev); 4537 } 4538 4539 rdev_dec_pending(rdev, mddev); 4540 end_reshape_request(r10_bio); 4541 } 4542 4543 static void end_reshape_request(struct r10bio *r10_bio) 4544 { 4545 if (!atomic_dec_and_test(&r10_bio->remaining)) 4546 return; 4547 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 4548 bio_put(r10_bio->master_bio); 4549 put_buf(r10_bio); 4550 } 4551 4552 static void raid10_finish_reshape(struct mddev *mddev) 4553 { 4554 struct r10conf *conf = mddev->private; 4555 4556 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4557 return; 4558 4559 if (mddev->delta_disks > 0) { 4560 sector_t size = raid10_size(mddev, 0, 0); 4561 md_set_array_sectors(mddev, size); 4562 if (mddev->recovery_cp > mddev->resync_max_sectors) { 4563 mddev->recovery_cp = mddev->resync_max_sectors; 4564 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4565 } 4566 mddev->resync_max_sectors = size; 4567 set_capacity(mddev->gendisk, mddev->array_sectors); 4568 revalidate_disk(mddev->gendisk); 4569 } else { 4570 int d; 4571 for (d = conf->geo.raid_disks ; 4572 d < conf->geo.raid_disks - mddev->delta_disks; 4573 d++) { 4574 struct md_rdev *rdev = conf->mirrors[d].rdev; 4575 if (rdev) 4576 clear_bit(In_sync, &rdev->flags); 4577 rdev = conf->mirrors[d].replacement; 4578 if (rdev) 4579 clear_bit(In_sync, &rdev->flags); 4580 } 4581 } 4582 mddev->layout = mddev->new_layout; 4583 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 4584 mddev->reshape_position = MaxSector; 4585 mddev->delta_disks = 0; 4586 mddev->reshape_backwards = 0; 4587 } 4588 4589 static struct md_personality raid10_personality = 4590 { 4591 .name = "raid10", 4592 .level = 10, 4593 .owner = THIS_MODULE, 4594 .make_request = make_request, 4595 .run = run, 4596 .free = raid10_free, 4597 .status = status, 4598 .error_handler = error, 4599 .hot_add_disk = raid10_add_disk, 4600 .hot_remove_disk= raid10_remove_disk, 4601 .spare_active = raid10_spare_active, 4602 .sync_request = sync_request, 4603 .quiesce = raid10_quiesce, 4604 .size = raid10_size, 4605 .resize = raid10_resize, 4606 .takeover = raid10_takeover, 4607 .check_reshape = raid10_check_reshape, 4608 .start_reshape = raid10_start_reshape, 4609 .finish_reshape = raid10_finish_reshape, 4610 .congested = raid10_congested, 4611 }; 4612 4613 static int __init raid_init(void) 4614 { 4615 return register_md_personality(&raid10_personality); 4616 } 4617 4618 static void raid_exit(void) 4619 { 4620 unregister_md_personality(&raid10_personality); 4621 } 4622 4623 module_init(raid_init); 4624 module_exit(raid_exit); 4625 MODULE_LICENSE("GPL"); 4626 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 4627 MODULE_ALIAS("md-personality-9"); /* RAID10 */ 4628 MODULE_ALIAS("md-raid10"); 4629 MODULE_ALIAS("md-level-10"); 4630 4631 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 4632