1 /* 2 * raid10.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 2000-2004 Neil Brown 5 * 6 * RAID-10 support for md. 7 * 8 * Base on code in raid1.c. See raid1.c for further copyright information. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 #include <linux/slab.h> 22 #include <linux/delay.h> 23 #include <linux/blkdev.h> 24 #include <linux/module.h> 25 #include <linux/seq_file.h> 26 #include <linux/ratelimit.h> 27 #include <linux/kthread.h> 28 #include "md.h" 29 #include "raid10.h" 30 #include "raid0.h" 31 #include "bitmap.h" 32 33 /* 34 * RAID10 provides a combination of RAID0 and RAID1 functionality. 35 * The layout of data is defined by 36 * chunk_size 37 * raid_disks 38 * near_copies (stored in low byte of layout) 39 * far_copies (stored in second byte of layout) 40 * far_offset (stored in bit 16 of layout ) 41 * use_far_sets (stored in bit 17 of layout ) 42 * use_far_sets_bugfixed (stored in bit 18 of layout ) 43 * 44 * The data to be stored is divided into chunks using chunksize. Each device 45 * is divided into far_copies sections. In each section, chunks are laid out 46 * in a style similar to raid0, but near_copies copies of each chunk is stored 47 * (each on a different drive). The starting device for each section is offset 48 * near_copies from the starting device of the previous section. Thus there 49 * are (near_copies * far_copies) of each chunk, and each is on a different 50 * drive. near_copies and far_copies must be at least one, and their product 51 * is at most raid_disks. 52 * 53 * If far_offset is true, then the far_copies are handled a bit differently. 54 * The copies are still in different stripes, but instead of being very far 55 * apart on disk, there are adjacent stripes. 56 * 57 * The far and offset algorithms are handled slightly differently if 58 * 'use_far_sets' is true. In this case, the array's devices are grouped into 59 * sets that are (near_copies * far_copies) in size. The far copied stripes 60 * are still shifted by 'near_copies' devices, but this shifting stays confined 61 * to the set rather than the entire array. This is done to improve the number 62 * of device combinations that can fail without causing the array to fail. 63 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 64 * on a device): 65 * A B C D A B C D E 66 * ... ... 67 * D A B C E A B C D 68 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 69 * [A B] [C D] [A B] [C D E] 70 * |...| |...| |...| | ... | 71 * [B A] [D C] [B A] [E C D] 72 */ 73 74 /* 75 * Number of guaranteed r10bios in case of extreme VM load: 76 */ 77 #define NR_RAID10_BIOS 256 78 79 /* when we get a read error on a read-only array, we redirect to another 80 * device without failing the first device, or trying to over-write to 81 * correct the read error. To keep track of bad blocks on a per-bio 82 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 83 */ 84 #define IO_BLOCKED ((struct bio *)1) 85 /* When we successfully write to a known bad-block, we need to remove the 86 * bad-block marking which must be done from process context. So we record 87 * the success by setting devs[n].bio to IO_MADE_GOOD 88 */ 89 #define IO_MADE_GOOD ((struct bio *)2) 90 91 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 92 93 /* When there are this many requests queued to be written by 94 * the raid10 thread, we become 'congested' to provide back-pressure 95 * for writeback. 96 */ 97 static int max_queued_requests = 1024; 98 99 static void allow_barrier(struct r10conf *conf); 100 static void lower_barrier(struct r10conf *conf); 101 static int _enough(struct r10conf *conf, int previous, int ignore); 102 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 103 int *skipped); 104 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 105 static void end_reshape_write(struct bio *bio); 106 static void end_reshape(struct r10conf *conf); 107 108 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 109 { 110 struct r10conf *conf = data; 111 int size = offsetof(struct r10bio, devs[conf->copies]); 112 113 /* allocate a r10bio with room for raid_disks entries in the 114 * bios array */ 115 return kzalloc(size, gfp_flags); 116 } 117 118 static void r10bio_pool_free(void *r10_bio, void *data) 119 { 120 kfree(r10_bio); 121 } 122 123 /* Maximum size of each resync request */ 124 #define RESYNC_BLOCK_SIZE (64*1024) 125 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 126 /* amount of memory to reserve for resync requests */ 127 #define RESYNC_WINDOW (1024*1024) 128 /* maximum number of concurrent requests, memory permitting */ 129 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 130 131 /* 132 * When performing a resync, we need to read and compare, so 133 * we need as many pages are there are copies. 134 * When performing a recovery, we need 2 bios, one for read, 135 * one for write (we recover only one drive per r10buf) 136 * 137 */ 138 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 139 { 140 struct r10conf *conf = data; 141 struct page *page; 142 struct r10bio *r10_bio; 143 struct bio *bio; 144 int i, j; 145 int nalloc; 146 147 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 148 if (!r10_bio) 149 return NULL; 150 151 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 152 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 153 nalloc = conf->copies; /* resync */ 154 else 155 nalloc = 2; /* recovery */ 156 157 /* 158 * Allocate bios. 159 */ 160 for (j = nalloc ; j-- ; ) { 161 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 162 if (!bio) 163 goto out_free_bio; 164 r10_bio->devs[j].bio = bio; 165 if (!conf->have_replacement) 166 continue; 167 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 168 if (!bio) 169 goto out_free_bio; 170 r10_bio->devs[j].repl_bio = bio; 171 } 172 /* 173 * Allocate RESYNC_PAGES data pages and attach them 174 * where needed. 175 */ 176 for (j = 0 ; j < nalloc; j++) { 177 struct bio *rbio = r10_bio->devs[j].repl_bio; 178 bio = r10_bio->devs[j].bio; 179 for (i = 0; i < RESYNC_PAGES; i++) { 180 if (j > 0 && !test_bit(MD_RECOVERY_SYNC, 181 &conf->mddev->recovery)) { 182 /* we can share bv_page's during recovery 183 * and reshape */ 184 struct bio *rbio = r10_bio->devs[0].bio; 185 page = rbio->bi_io_vec[i].bv_page; 186 get_page(page); 187 } else 188 page = alloc_page(gfp_flags); 189 if (unlikely(!page)) 190 goto out_free_pages; 191 192 bio->bi_io_vec[i].bv_page = page; 193 if (rbio) 194 rbio->bi_io_vec[i].bv_page = page; 195 } 196 } 197 198 return r10_bio; 199 200 out_free_pages: 201 for ( ; i > 0 ; i--) 202 safe_put_page(bio->bi_io_vec[i-1].bv_page); 203 while (j--) 204 for (i = 0; i < RESYNC_PAGES ; i++) 205 safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page); 206 j = 0; 207 out_free_bio: 208 for ( ; j < nalloc; j++) { 209 if (r10_bio->devs[j].bio) 210 bio_put(r10_bio->devs[j].bio); 211 if (r10_bio->devs[j].repl_bio) 212 bio_put(r10_bio->devs[j].repl_bio); 213 } 214 r10bio_pool_free(r10_bio, conf); 215 return NULL; 216 } 217 218 static void r10buf_pool_free(void *__r10_bio, void *data) 219 { 220 int i; 221 struct r10conf *conf = data; 222 struct r10bio *r10bio = __r10_bio; 223 int j; 224 225 for (j=0; j < conf->copies; j++) { 226 struct bio *bio = r10bio->devs[j].bio; 227 if (bio) { 228 for (i = 0; i < RESYNC_PAGES; i++) { 229 safe_put_page(bio->bi_io_vec[i].bv_page); 230 bio->bi_io_vec[i].bv_page = NULL; 231 } 232 bio_put(bio); 233 } 234 bio = r10bio->devs[j].repl_bio; 235 if (bio) 236 bio_put(bio); 237 } 238 r10bio_pool_free(r10bio, conf); 239 } 240 241 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 242 { 243 int i; 244 245 for (i = 0; i < conf->copies; i++) { 246 struct bio **bio = & r10_bio->devs[i].bio; 247 if (!BIO_SPECIAL(*bio)) 248 bio_put(*bio); 249 *bio = NULL; 250 bio = &r10_bio->devs[i].repl_bio; 251 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 252 bio_put(*bio); 253 *bio = NULL; 254 } 255 } 256 257 static void free_r10bio(struct r10bio *r10_bio) 258 { 259 struct r10conf *conf = r10_bio->mddev->private; 260 261 put_all_bios(conf, r10_bio); 262 mempool_free(r10_bio, conf->r10bio_pool); 263 } 264 265 static void put_buf(struct r10bio *r10_bio) 266 { 267 struct r10conf *conf = r10_bio->mddev->private; 268 269 mempool_free(r10_bio, conf->r10buf_pool); 270 271 lower_barrier(conf); 272 } 273 274 static void reschedule_retry(struct r10bio *r10_bio) 275 { 276 unsigned long flags; 277 struct mddev *mddev = r10_bio->mddev; 278 struct r10conf *conf = mddev->private; 279 280 spin_lock_irqsave(&conf->device_lock, flags); 281 list_add(&r10_bio->retry_list, &conf->retry_list); 282 conf->nr_queued ++; 283 spin_unlock_irqrestore(&conf->device_lock, flags); 284 285 /* wake up frozen array... */ 286 wake_up(&conf->wait_barrier); 287 288 md_wakeup_thread(mddev->thread); 289 } 290 291 /* 292 * raid_end_bio_io() is called when we have finished servicing a mirrored 293 * operation and are ready to return a success/failure code to the buffer 294 * cache layer. 295 */ 296 static void raid_end_bio_io(struct r10bio *r10_bio) 297 { 298 struct bio *bio = r10_bio->master_bio; 299 int done; 300 struct r10conf *conf = r10_bio->mddev->private; 301 302 if (bio->bi_phys_segments) { 303 unsigned long flags; 304 spin_lock_irqsave(&conf->device_lock, flags); 305 bio->bi_phys_segments--; 306 done = (bio->bi_phys_segments == 0); 307 spin_unlock_irqrestore(&conf->device_lock, flags); 308 } else 309 done = 1; 310 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 311 bio->bi_error = -EIO; 312 if (done) { 313 bio_endio(bio); 314 /* 315 * Wake up any possible resync thread that waits for the device 316 * to go idle. 317 */ 318 allow_barrier(conf); 319 } 320 free_r10bio(r10_bio); 321 } 322 323 /* 324 * Update disk head position estimator based on IRQ completion info. 325 */ 326 static inline void update_head_pos(int slot, struct r10bio *r10_bio) 327 { 328 struct r10conf *conf = r10_bio->mddev->private; 329 330 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 331 r10_bio->devs[slot].addr + (r10_bio->sectors); 332 } 333 334 /* 335 * Find the disk number which triggered given bio 336 */ 337 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 338 struct bio *bio, int *slotp, int *replp) 339 { 340 int slot; 341 int repl = 0; 342 343 for (slot = 0; slot < conf->copies; slot++) { 344 if (r10_bio->devs[slot].bio == bio) 345 break; 346 if (r10_bio->devs[slot].repl_bio == bio) { 347 repl = 1; 348 break; 349 } 350 } 351 352 BUG_ON(slot == conf->copies); 353 update_head_pos(slot, r10_bio); 354 355 if (slotp) 356 *slotp = slot; 357 if (replp) 358 *replp = repl; 359 return r10_bio->devs[slot].devnum; 360 } 361 362 static void raid10_end_read_request(struct bio *bio) 363 { 364 int uptodate = !bio->bi_error; 365 struct r10bio *r10_bio = bio->bi_private; 366 int slot, dev; 367 struct md_rdev *rdev; 368 struct r10conf *conf = r10_bio->mddev->private; 369 370 slot = r10_bio->read_slot; 371 dev = r10_bio->devs[slot].devnum; 372 rdev = r10_bio->devs[slot].rdev; 373 /* 374 * this branch is our 'one mirror IO has finished' event handler: 375 */ 376 update_head_pos(slot, r10_bio); 377 378 if (uptodate) { 379 /* 380 * Set R10BIO_Uptodate in our master bio, so that 381 * we will return a good error code to the higher 382 * levels even if IO on some other mirrored buffer fails. 383 * 384 * The 'master' represents the composite IO operation to 385 * user-side. So if something waits for IO, then it will 386 * wait for the 'master' bio. 387 */ 388 set_bit(R10BIO_Uptodate, &r10_bio->state); 389 } else { 390 /* If all other devices that store this block have 391 * failed, we want to return the error upwards rather 392 * than fail the last device. Here we redefine 393 * "uptodate" to mean "Don't want to retry" 394 */ 395 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), 396 rdev->raid_disk)) 397 uptodate = 1; 398 } 399 if (uptodate) { 400 raid_end_bio_io(r10_bio); 401 rdev_dec_pending(rdev, conf->mddev); 402 } else { 403 /* 404 * oops, read error - keep the refcount on the rdev 405 */ 406 char b[BDEVNAME_SIZE]; 407 printk_ratelimited(KERN_ERR 408 "md/raid10:%s: %s: rescheduling sector %llu\n", 409 mdname(conf->mddev), 410 bdevname(rdev->bdev, b), 411 (unsigned long long)r10_bio->sector); 412 set_bit(R10BIO_ReadError, &r10_bio->state); 413 reschedule_retry(r10_bio); 414 } 415 } 416 417 static void close_write(struct r10bio *r10_bio) 418 { 419 /* clear the bitmap if all writes complete successfully */ 420 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector, 421 r10_bio->sectors, 422 !test_bit(R10BIO_Degraded, &r10_bio->state), 423 0); 424 md_write_end(r10_bio->mddev); 425 } 426 427 static void one_write_done(struct r10bio *r10_bio) 428 { 429 if (atomic_dec_and_test(&r10_bio->remaining)) { 430 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 431 reschedule_retry(r10_bio); 432 else { 433 close_write(r10_bio); 434 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 435 reschedule_retry(r10_bio); 436 else 437 raid_end_bio_io(r10_bio); 438 } 439 } 440 } 441 442 static void raid10_end_write_request(struct bio *bio) 443 { 444 struct r10bio *r10_bio = bio->bi_private; 445 int dev; 446 int dec_rdev = 1; 447 struct r10conf *conf = r10_bio->mddev->private; 448 int slot, repl; 449 struct md_rdev *rdev = NULL; 450 451 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 452 453 if (repl) 454 rdev = conf->mirrors[dev].replacement; 455 if (!rdev) { 456 smp_rmb(); 457 repl = 0; 458 rdev = conf->mirrors[dev].rdev; 459 } 460 /* 461 * this branch is our 'one mirror IO has finished' event handler: 462 */ 463 if (bio->bi_error) { 464 if (repl) 465 /* Never record new bad blocks to replacement, 466 * just fail it. 467 */ 468 md_error(rdev->mddev, rdev); 469 else { 470 set_bit(WriteErrorSeen, &rdev->flags); 471 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 472 set_bit(MD_RECOVERY_NEEDED, 473 &rdev->mddev->recovery); 474 set_bit(R10BIO_WriteError, &r10_bio->state); 475 dec_rdev = 0; 476 } 477 } else { 478 /* 479 * Set R10BIO_Uptodate in our master bio, so that 480 * we will return a good error code for to the higher 481 * levels even if IO on some other mirrored buffer fails. 482 * 483 * The 'master' represents the composite IO operation to 484 * user-side. So if something waits for IO, then it will 485 * wait for the 'master' bio. 486 */ 487 sector_t first_bad; 488 int bad_sectors; 489 490 /* 491 * Do not set R10BIO_Uptodate if the current device is 492 * rebuilding or Faulty. This is because we cannot use 493 * such device for properly reading the data back (we could 494 * potentially use it, if the current write would have felt 495 * before rdev->recovery_offset, but for simplicity we don't 496 * check this here. 497 */ 498 if (test_bit(In_sync, &rdev->flags) && 499 !test_bit(Faulty, &rdev->flags)) 500 set_bit(R10BIO_Uptodate, &r10_bio->state); 501 502 /* Maybe we can clear some bad blocks. */ 503 if (is_badblock(rdev, 504 r10_bio->devs[slot].addr, 505 r10_bio->sectors, 506 &first_bad, &bad_sectors)) { 507 bio_put(bio); 508 if (repl) 509 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 510 else 511 r10_bio->devs[slot].bio = IO_MADE_GOOD; 512 dec_rdev = 0; 513 set_bit(R10BIO_MadeGood, &r10_bio->state); 514 } 515 } 516 517 /* 518 * 519 * Let's see if all mirrored write operations have finished 520 * already. 521 */ 522 one_write_done(r10_bio); 523 if (dec_rdev) 524 rdev_dec_pending(rdev, conf->mddev); 525 } 526 527 /* 528 * RAID10 layout manager 529 * As well as the chunksize and raid_disks count, there are two 530 * parameters: near_copies and far_copies. 531 * near_copies * far_copies must be <= raid_disks. 532 * Normally one of these will be 1. 533 * If both are 1, we get raid0. 534 * If near_copies == raid_disks, we get raid1. 535 * 536 * Chunks are laid out in raid0 style with near_copies copies of the 537 * first chunk, followed by near_copies copies of the next chunk and 538 * so on. 539 * If far_copies > 1, then after 1/far_copies of the array has been assigned 540 * as described above, we start again with a device offset of near_copies. 541 * So we effectively have another copy of the whole array further down all 542 * the drives, but with blocks on different drives. 543 * With this layout, and block is never stored twice on the one device. 544 * 545 * raid10_find_phys finds the sector offset of a given virtual sector 546 * on each device that it is on. 547 * 548 * raid10_find_virt does the reverse mapping, from a device and a 549 * sector offset to a virtual address 550 */ 551 552 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 553 { 554 int n,f; 555 sector_t sector; 556 sector_t chunk; 557 sector_t stripe; 558 int dev; 559 int slot = 0; 560 int last_far_set_start, last_far_set_size; 561 562 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 563 last_far_set_start *= geo->far_set_size; 564 565 last_far_set_size = geo->far_set_size; 566 last_far_set_size += (geo->raid_disks % geo->far_set_size); 567 568 /* now calculate first sector/dev */ 569 chunk = r10bio->sector >> geo->chunk_shift; 570 sector = r10bio->sector & geo->chunk_mask; 571 572 chunk *= geo->near_copies; 573 stripe = chunk; 574 dev = sector_div(stripe, geo->raid_disks); 575 if (geo->far_offset) 576 stripe *= geo->far_copies; 577 578 sector += stripe << geo->chunk_shift; 579 580 /* and calculate all the others */ 581 for (n = 0; n < geo->near_copies; n++) { 582 int d = dev; 583 int set; 584 sector_t s = sector; 585 r10bio->devs[slot].devnum = d; 586 r10bio->devs[slot].addr = s; 587 slot++; 588 589 for (f = 1; f < geo->far_copies; f++) { 590 set = d / geo->far_set_size; 591 d += geo->near_copies; 592 593 if ((geo->raid_disks % geo->far_set_size) && 594 (d > last_far_set_start)) { 595 d -= last_far_set_start; 596 d %= last_far_set_size; 597 d += last_far_set_start; 598 } else { 599 d %= geo->far_set_size; 600 d += geo->far_set_size * set; 601 } 602 s += geo->stride; 603 r10bio->devs[slot].devnum = d; 604 r10bio->devs[slot].addr = s; 605 slot++; 606 } 607 dev++; 608 if (dev >= geo->raid_disks) { 609 dev = 0; 610 sector += (geo->chunk_mask + 1); 611 } 612 } 613 } 614 615 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 616 { 617 struct geom *geo = &conf->geo; 618 619 if (conf->reshape_progress != MaxSector && 620 ((r10bio->sector >= conf->reshape_progress) != 621 conf->mddev->reshape_backwards)) { 622 set_bit(R10BIO_Previous, &r10bio->state); 623 geo = &conf->prev; 624 } else 625 clear_bit(R10BIO_Previous, &r10bio->state); 626 627 __raid10_find_phys(geo, r10bio); 628 } 629 630 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 631 { 632 sector_t offset, chunk, vchunk; 633 /* Never use conf->prev as this is only called during resync 634 * or recovery, so reshape isn't happening 635 */ 636 struct geom *geo = &conf->geo; 637 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 638 int far_set_size = geo->far_set_size; 639 int last_far_set_start; 640 641 if (geo->raid_disks % geo->far_set_size) { 642 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 643 last_far_set_start *= geo->far_set_size; 644 645 if (dev >= last_far_set_start) { 646 far_set_size = geo->far_set_size; 647 far_set_size += (geo->raid_disks % geo->far_set_size); 648 far_set_start = last_far_set_start; 649 } 650 } 651 652 offset = sector & geo->chunk_mask; 653 if (geo->far_offset) { 654 int fc; 655 chunk = sector >> geo->chunk_shift; 656 fc = sector_div(chunk, geo->far_copies); 657 dev -= fc * geo->near_copies; 658 if (dev < far_set_start) 659 dev += far_set_size; 660 } else { 661 while (sector >= geo->stride) { 662 sector -= geo->stride; 663 if (dev < (geo->near_copies + far_set_start)) 664 dev += far_set_size - geo->near_copies; 665 else 666 dev -= geo->near_copies; 667 } 668 chunk = sector >> geo->chunk_shift; 669 } 670 vchunk = chunk * geo->raid_disks + dev; 671 sector_div(vchunk, geo->near_copies); 672 return (vchunk << geo->chunk_shift) + offset; 673 } 674 675 /* 676 * This routine returns the disk from which the requested read should 677 * be done. There is a per-array 'next expected sequential IO' sector 678 * number - if this matches on the next IO then we use the last disk. 679 * There is also a per-disk 'last know head position' sector that is 680 * maintained from IRQ contexts, both the normal and the resync IO 681 * completion handlers update this position correctly. If there is no 682 * perfect sequential match then we pick the disk whose head is closest. 683 * 684 * If there are 2 mirrors in the same 2 devices, performance degrades 685 * because position is mirror, not device based. 686 * 687 * The rdev for the device selected will have nr_pending incremented. 688 */ 689 690 /* 691 * FIXME: possibly should rethink readbalancing and do it differently 692 * depending on near_copies / far_copies geometry. 693 */ 694 static struct md_rdev *read_balance(struct r10conf *conf, 695 struct r10bio *r10_bio, 696 int *max_sectors) 697 { 698 const sector_t this_sector = r10_bio->sector; 699 int disk, slot; 700 int sectors = r10_bio->sectors; 701 int best_good_sectors; 702 sector_t new_distance, best_dist; 703 struct md_rdev *best_rdev, *rdev = NULL; 704 int do_balance; 705 int best_slot; 706 struct geom *geo = &conf->geo; 707 708 raid10_find_phys(conf, r10_bio); 709 rcu_read_lock(); 710 sectors = r10_bio->sectors; 711 best_slot = -1; 712 best_rdev = NULL; 713 best_dist = MaxSector; 714 best_good_sectors = 0; 715 do_balance = 1; 716 /* 717 * Check if we can balance. We can balance on the whole 718 * device if no resync is going on (recovery is ok), or below 719 * the resync window. We take the first readable disk when 720 * above the resync window. 721 */ 722 if (conf->mddev->recovery_cp < MaxSector 723 && (this_sector + sectors >= conf->next_resync)) 724 do_balance = 0; 725 726 for (slot = 0; slot < conf->copies ; slot++) { 727 sector_t first_bad; 728 int bad_sectors; 729 sector_t dev_sector; 730 731 if (r10_bio->devs[slot].bio == IO_BLOCKED) 732 continue; 733 disk = r10_bio->devs[slot].devnum; 734 rdev = rcu_dereference(conf->mirrors[disk].replacement); 735 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 736 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 737 rdev = rcu_dereference(conf->mirrors[disk].rdev); 738 if (rdev == NULL || 739 test_bit(Faulty, &rdev->flags)) 740 continue; 741 if (!test_bit(In_sync, &rdev->flags) && 742 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 743 continue; 744 745 dev_sector = r10_bio->devs[slot].addr; 746 if (is_badblock(rdev, dev_sector, sectors, 747 &first_bad, &bad_sectors)) { 748 if (best_dist < MaxSector) 749 /* Already have a better slot */ 750 continue; 751 if (first_bad <= dev_sector) { 752 /* Cannot read here. If this is the 753 * 'primary' device, then we must not read 754 * beyond 'bad_sectors' from another device. 755 */ 756 bad_sectors -= (dev_sector - first_bad); 757 if (!do_balance && sectors > bad_sectors) 758 sectors = bad_sectors; 759 if (best_good_sectors > sectors) 760 best_good_sectors = sectors; 761 } else { 762 sector_t good_sectors = 763 first_bad - dev_sector; 764 if (good_sectors > best_good_sectors) { 765 best_good_sectors = good_sectors; 766 best_slot = slot; 767 best_rdev = rdev; 768 } 769 if (!do_balance) 770 /* Must read from here */ 771 break; 772 } 773 continue; 774 } else 775 best_good_sectors = sectors; 776 777 if (!do_balance) 778 break; 779 780 /* This optimisation is debatable, and completely destroys 781 * sequential read speed for 'far copies' arrays. So only 782 * keep it for 'near' arrays, and review those later. 783 */ 784 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending)) 785 break; 786 787 /* for far > 1 always use the lowest address */ 788 if (geo->far_copies > 1) 789 new_distance = r10_bio->devs[slot].addr; 790 else 791 new_distance = abs(r10_bio->devs[slot].addr - 792 conf->mirrors[disk].head_position); 793 if (new_distance < best_dist) { 794 best_dist = new_distance; 795 best_slot = slot; 796 best_rdev = rdev; 797 } 798 } 799 if (slot >= conf->copies) { 800 slot = best_slot; 801 rdev = best_rdev; 802 } 803 804 if (slot >= 0) { 805 atomic_inc(&rdev->nr_pending); 806 r10_bio->read_slot = slot; 807 } else 808 rdev = NULL; 809 rcu_read_unlock(); 810 *max_sectors = best_good_sectors; 811 812 return rdev; 813 } 814 815 static int raid10_congested(struct mddev *mddev, int bits) 816 { 817 struct r10conf *conf = mddev->private; 818 int i, ret = 0; 819 820 if ((bits & (1 << WB_async_congested)) && 821 conf->pending_count >= max_queued_requests) 822 return 1; 823 824 rcu_read_lock(); 825 for (i = 0; 826 (i < conf->geo.raid_disks || i < conf->prev.raid_disks) 827 && ret == 0; 828 i++) { 829 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 830 if (rdev && !test_bit(Faulty, &rdev->flags)) { 831 struct request_queue *q = bdev_get_queue(rdev->bdev); 832 833 ret |= bdi_congested(&q->backing_dev_info, bits); 834 } 835 } 836 rcu_read_unlock(); 837 return ret; 838 } 839 840 static void flush_pending_writes(struct r10conf *conf) 841 { 842 /* Any writes that have been queued but are awaiting 843 * bitmap updates get flushed here. 844 */ 845 spin_lock_irq(&conf->device_lock); 846 847 if (conf->pending_bio_list.head) { 848 struct bio *bio; 849 bio = bio_list_get(&conf->pending_bio_list); 850 conf->pending_count = 0; 851 spin_unlock_irq(&conf->device_lock); 852 /* flush any pending bitmap writes to disk 853 * before proceeding w/ I/O */ 854 bitmap_unplug(conf->mddev->bitmap); 855 wake_up(&conf->wait_barrier); 856 857 while (bio) { /* submit pending writes */ 858 struct bio *next = bio->bi_next; 859 bio->bi_next = NULL; 860 if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 861 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 862 /* Just ignore it */ 863 bio_endio(bio); 864 else 865 generic_make_request(bio); 866 bio = next; 867 } 868 } else 869 spin_unlock_irq(&conf->device_lock); 870 } 871 872 /* Barriers.... 873 * Sometimes we need to suspend IO while we do something else, 874 * either some resync/recovery, or reconfigure the array. 875 * To do this we raise a 'barrier'. 876 * The 'barrier' is a counter that can be raised multiple times 877 * to count how many activities are happening which preclude 878 * normal IO. 879 * We can only raise the barrier if there is no pending IO. 880 * i.e. if nr_pending == 0. 881 * We choose only to raise the barrier if no-one is waiting for the 882 * barrier to go down. This means that as soon as an IO request 883 * is ready, no other operations which require a barrier will start 884 * until the IO request has had a chance. 885 * 886 * So: regular IO calls 'wait_barrier'. When that returns there 887 * is no backgroup IO happening, It must arrange to call 888 * allow_barrier when it has finished its IO. 889 * backgroup IO calls must call raise_barrier. Once that returns 890 * there is no normal IO happeing. It must arrange to call 891 * lower_barrier when the particular background IO completes. 892 */ 893 894 static void raise_barrier(struct r10conf *conf, int force) 895 { 896 BUG_ON(force && !conf->barrier); 897 spin_lock_irq(&conf->resync_lock); 898 899 /* Wait until no block IO is waiting (unless 'force') */ 900 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, 901 conf->resync_lock); 902 903 /* block any new IO from starting */ 904 conf->barrier++; 905 906 /* Now wait for all pending IO to complete */ 907 wait_event_lock_irq(conf->wait_barrier, 908 !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH, 909 conf->resync_lock); 910 911 spin_unlock_irq(&conf->resync_lock); 912 } 913 914 static void lower_barrier(struct r10conf *conf) 915 { 916 unsigned long flags; 917 spin_lock_irqsave(&conf->resync_lock, flags); 918 conf->barrier--; 919 spin_unlock_irqrestore(&conf->resync_lock, flags); 920 wake_up(&conf->wait_barrier); 921 } 922 923 static void wait_barrier(struct r10conf *conf) 924 { 925 spin_lock_irq(&conf->resync_lock); 926 if (conf->barrier) { 927 conf->nr_waiting++; 928 /* Wait for the barrier to drop. 929 * However if there are already pending 930 * requests (preventing the barrier from 931 * rising completely), and the 932 * pre-process bio queue isn't empty, 933 * then don't wait, as we need to empty 934 * that queue to get the nr_pending 935 * count down. 936 */ 937 wait_event_lock_irq(conf->wait_barrier, 938 !conf->barrier || 939 (atomic_read(&conf->nr_pending) && 940 current->bio_list && 941 !bio_list_empty(current->bio_list)), 942 conf->resync_lock); 943 conf->nr_waiting--; 944 if (!conf->nr_waiting) 945 wake_up(&conf->wait_barrier); 946 } 947 atomic_inc(&conf->nr_pending); 948 spin_unlock_irq(&conf->resync_lock); 949 } 950 951 static void allow_barrier(struct r10conf *conf) 952 { 953 if ((atomic_dec_and_test(&conf->nr_pending)) || 954 (conf->array_freeze_pending)) 955 wake_up(&conf->wait_barrier); 956 } 957 958 static void freeze_array(struct r10conf *conf, int extra) 959 { 960 /* stop syncio and normal IO and wait for everything to 961 * go quiet. 962 * We increment barrier and nr_waiting, and then 963 * wait until nr_pending match nr_queued+extra 964 * This is called in the context of one normal IO request 965 * that has failed. Thus any sync request that might be pending 966 * will be blocked by nr_pending, and we need to wait for 967 * pending IO requests to complete or be queued for re-try. 968 * Thus the number queued (nr_queued) plus this request (extra) 969 * must match the number of pending IOs (nr_pending) before 970 * we continue. 971 */ 972 spin_lock_irq(&conf->resync_lock); 973 conf->array_freeze_pending++; 974 conf->barrier++; 975 conf->nr_waiting++; 976 wait_event_lock_irq_cmd(conf->wait_barrier, 977 atomic_read(&conf->nr_pending) == conf->nr_queued+extra, 978 conf->resync_lock, 979 flush_pending_writes(conf)); 980 981 conf->array_freeze_pending--; 982 spin_unlock_irq(&conf->resync_lock); 983 } 984 985 static void unfreeze_array(struct r10conf *conf) 986 { 987 /* reverse the effect of the freeze */ 988 spin_lock_irq(&conf->resync_lock); 989 conf->barrier--; 990 conf->nr_waiting--; 991 wake_up(&conf->wait_barrier); 992 spin_unlock_irq(&conf->resync_lock); 993 } 994 995 static sector_t choose_data_offset(struct r10bio *r10_bio, 996 struct md_rdev *rdev) 997 { 998 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 999 test_bit(R10BIO_Previous, &r10_bio->state)) 1000 return rdev->data_offset; 1001 else 1002 return rdev->new_data_offset; 1003 } 1004 1005 struct raid10_plug_cb { 1006 struct blk_plug_cb cb; 1007 struct bio_list pending; 1008 int pending_cnt; 1009 }; 1010 1011 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1012 { 1013 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb, 1014 cb); 1015 struct mddev *mddev = plug->cb.data; 1016 struct r10conf *conf = mddev->private; 1017 struct bio *bio; 1018 1019 if (from_schedule || current->bio_list) { 1020 spin_lock_irq(&conf->device_lock); 1021 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1022 conf->pending_count += plug->pending_cnt; 1023 spin_unlock_irq(&conf->device_lock); 1024 wake_up(&conf->wait_barrier); 1025 md_wakeup_thread(mddev->thread); 1026 kfree(plug); 1027 return; 1028 } 1029 1030 /* we aren't scheduling, so we can do the write-out directly. */ 1031 bio = bio_list_get(&plug->pending); 1032 bitmap_unplug(mddev->bitmap); 1033 wake_up(&conf->wait_barrier); 1034 1035 while (bio) { /* submit pending writes */ 1036 struct bio *next = bio->bi_next; 1037 bio->bi_next = NULL; 1038 if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 1039 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1040 /* Just ignore it */ 1041 bio_endio(bio); 1042 else 1043 generic_make_request(bio); 1044 bio = next; 1045 } 1046 kfree(plug); 1047 } 1048 1049 static void __make_request(struct mddev *mddev, struct bio *bio) 1050 { 1051 struct r10conf *conf = mddev->private; 1052 struct r10bio *r10_bio; 1053 struct bio *read_bio; 1054 int i; 1055 const int op = bio_op(bio); 1056 const int rw = bio_data_dir(bio); 1057 const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); 1058 const unsigned long do_fua = (bio->bi_opf & REQ_FUA); 1059 unsigned long flags; 1060 struct md_rdev *blocked_rdev; 1061 struct blk_plug_cb *cb; 1062 struct raid10_plug_cb *plug = NULL; 1063 int sectors_handled; 1064 int max_sectors; 1065 int sectors; 1066 1067 md_write_start(mddev, bio); 1068 1069 /* 1070 * Register the new request and wait if the reconstruction 1071 * thread has put up a bar for new requests. 1072 * Continue immediately if no resync is active currently. 1073 */ 1074 wait_barrier(conf); 1075 1076 sectors = bio_sectors(bio); 1077 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1078 bio->bi_iter.bi_sector < conf->reshape_progress && 1079 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1080 /* IO spans the reshape position. Need to wait for 1081 * reshape to pass 1082 */ 1083 allow_barrier(conf); 1084 wait_event(conf->wait_barrier, 1085 conf->reshape_progress <= bio->bi_iter.bi_sector || 1086 conf->reshape_progress >= bio->bi_iter.bi_sector + 1087 sectors); 1088 wait_barrier(conf); 1089 } 1090 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1091 bio_data_dir(bio) == WRITE && 1092 (mddev->reshape_backwards 1093 ? (bio->bi_iter.bi_sector < conf->reshape_safe && 1094 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) 1095 : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe && 1096 bio->bi_iter.bi_sector < conf->reshape_progress))) { 1097 /* Need to update reshape_position in metadata */ 1098 mddev->reshape_position = conf->reshape_progress; 1099 set_mask_bits(&mddev->flags, 0, 1100 BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING)); 1101 md_wakeup_thread(mddev->thread); 1102 wait_event(mddev->sb_wait, 1103 !test_bit(MD_CHANGE_PENDING, &mddev->flags)); 1104 1105 conf->reshape_safe = mddev->reshape_position; 1106 } 1107 1108 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1109 1110 r10_bio->master_bio = bio; 1111 r10_bio->sectors = sectors; 1112 1113 r10_bio->mddev = mddev; 1114 r10_bio->sector = bio->bi_iter.bi_sector; 1115 r10_bio->state = 0; 1116 1117 /* We might need to issue multiple reads to different 1118 * devices if there are bad blocks around, so we keep 1119 * track of the number of reads in bio->bi_phys_segments. 1120 * If this is 0, there is only one r10_bio and no locking 1121 * will be needed when the request completes. If it is 1122 * non-zero, then it is the number of not-completed requests. 1123 */ 1124 bio->bi_phys_segments = 0; 1125 bio_clear_flag(bio, BIO_SEG_VALID); 1126 1127 if (rw == READ) { 1128 /* 1129 * read balancing logic: 1130 */ 1131 struct md_rdev *rdev; 1132 int slot; 1133 1134 read_again: 1135 rdev = read_balance(conf, r10_bio, &max_sectors); 1136 if (!rdev) { 1137 raid_end_bio_io(r10_bio); 1138 return; 1139 } 1140 slot = r10_bio->read_slot; 1141 1142 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1143 bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector, 1144 max_sectors); 1145 1146 r10_bio->devs[slot].bio = read_bio; 1147 r10_bio->devs[slot].rdev = rdev; 1148 1149 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + 1150 choose_data_offset(r10_bio, rdev); 1151 read_bio->bi_bdev = rdev->bdev; 1152 read_bio->bi_end_io = raid10_end_read_request; 1153 bio_set_op_attrs(read_bio, op, do_sync); 1154 read_bio->bi_private = r10_bio; 1155 1156 if (max_sectors < r10_bio->sectors) { 1157 /* Could not read all from this device, so we will 1158 * need another r10_bio. 1159 */ 1160 sectors_handled = (r10_bio->sector + max_sectors 1161 - bio->bi_iter.bi_sector); 1162 r10_bio->sectors = max_sectors; 1163 spin_lock_irq(&conf->device_lock); 1164 if (bio->bi_phys_segments == 0) 1165 bio->bi_phys_segments = 2; 1166 else 1167 bio->bi_phys_segments++; 1168 spin_unlock_irq(&conf->device_lock); 1169 /* Cannot call generic_make_request directly 1170 * as that will be queued in __generic_make_request 1171 * and subsequent mempool_alloc might block 1172 * waiting for it. so hand bio over to raid10d. 1173 */ 1174 reschedule_retry(r10_bio); 1175 1176 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1177 1178 r10_bio->master_bio = bio; 1179 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1180 r10_bio->state = 0; 1181 r10_bio->mddev = mddev; 1182 r10_bio->sector = bio->bi_iter.bi_sector + 1183 sectors_handled; 1184 goto read_again; 1185 } else 1186 generic_make_request(read_bio); 1187 return; 1188 } 1189 1190 /* 1191 * WRITE: 1192 */ 1193 if (conf->pending_count >= max_queued_requests) { 1194 md_wakeup_thread(mddev->thread); 1195 wait_event(conf->wait_barrier, 1196 conf->pending_count < max_queued_requests); 1197 } 1198 /* first select target devices under rcu_lock and 1199 * inc refcount on their rdev. Record them by setting 1200 * bios[x] to bio 1201 * If there are known/acknowledged bad blocks on any device 1202 * on which we have seen a write error, we want to avoid 1203 * writing to those blocks. This potentially requires several 1204 * writes to write around the bad blocks. Each set of writes 1205 * gets its own r10_bio with a set of bios attached. The number 1206 * of r10_bios is recored in bio->bi_phys_segments just as with 1207 * the read case. 1208 */ 1209 1210 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1211 raid10_find_phys(conf, r10_bio); 1212 retry_write: 1213 blocked_rdev = NULL; 1214 rcu_read_lock(); 1215 max_sectors = r10_bio->sectors; 1216 1217 for (i = 0; i < conf->copies; i++) { 1218 int d = r10_bio->devs[i].devnum; 1219 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 1220 struct md_rdev *rrdev = rcu_dereference( 1221 conf->mirrors[d].replacement); 1222 if (rdev == rrdev) 1223 rrdev = NULL; 1224 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1225 atomic_inc(&rdev->nr_pending); 1226 blocked_rdev = rdev; 1227 break; 1228 } 1229 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) { 1230 atomic_inc(&rrdev->nr_pending); 1231 blocked_rdev = rrdev; 1232 break; 1233 } 1234 if (rdev && (test_bit(Faulty, &rdev->flags))) 1235 rdev = NULL; 1236 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1237 rrdev = NULL; 1238 1239 r10_bio->devs[i].bio = NULL; 1240 r10_bio->devs[i].repl_bio = NULL; 1241 1242 if (!rdev && !rrdev) { 1243 set_bit(R10BIO_Degraded, &r10_bio->state); 1244 continue; 1245 } 1246 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1247 sector_t first_bad; 1248 sector_t dev_sector = r10_bio->devs[i].addr; 1249 int bad_sectors; 1250 int is_bad; 1251 1252 is_bad = is_badblock(rdev, dev_sector, 1253 max_sectors, 1254 &first_bad, &bad_sectors); 1255 if (is_bad < 0) { 1256 /* Mustn't write here until the bad block 1257 * is acknowledged 1258 */ 1259 atomic_inc(&rdev->nr_pending); 1260 set_bit(BlockedBadBlocks, &rdev->flags); 1261 blocked_rdev = rdev; 1262 break; 1263 } 1264 if (is_bad && first_bad <= dev_sector) { 1265 /* Cannot write here at all */ 1266 bad_sectors -= (dev_sector - first_bad); 1267 if (bad_sectors < max_sectors) 1268 /* Mustn't write more than bad_sectors 1269 * to other devices yet 1270 */ 1271 max_sectors = bad_sectors; 1272 /* We don't set R10BIO_Degraded as that 1273 * only applies if the disk is missing, 1274 * so it might be re-added, and we want to 1275 * know to recover this chunk. 1276 * In this case the device is here, and the 1277 * fact that this chunk is not in-sync is 1278 * recorded in the bad block log. 1279 */ 1280 continue; 1281 } 1282 if (is_bad) { 1283 int good_sectors = first_bad - dev_sector; 1284 if (good_sectors < max_sectors) 1285 max_sectors = good_sectors; 1286 } 1287 } 1288 if (rdev) { 1289 r10_bio->devs[i].bio = bio; 1290 atomic_inc(&rdev->nr_pending); 1291 } 1292 if (rrdev) { 1293 r10_bio->devs[i].repl_bio = bio; 1294 atomic_inc(&rrdev->nr_pending); 1295 } 1296 } 1297 rcu_read_unlock(); 1298 1299 if (unlikely(blocked_rdev)) { 1300 /* Have to wait for this device to get unblocked, then retry */ 1301 int j; 1302 int d; 1303 1304 for (j = 0; j < i; j++) { 1305 if (r10_bio->devs[j].bio) { 1306 d = r10_bio->devs[j].devnum; 1307 rdev_dec_pending(conf->mirrors[d].rdev, mddev); 1308 } 1309 if (r10_bio->devs[j].repl_bio) { 1310 struct md_rdev *rdev; 1311 d = r10_bio->devs[j].devnum; 1312 rdev = conf->mirrors[d].replacement; 1313 if (!rdev) { 1314 /* Race with remove_disk */ 1315 smp_mb(); 1316 rdev = conf->mirrors[d].rdev; 1317 } 1318 rdev_dec_pending(rdev, mddev); 1319 } 1320 } 1321 allow_barrier(conf); 1322 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1323 wait_barrier(conf); 1324 goto retry_write; 1325 } 1326 1327 if (max_sectors < r10_bio->sectors) { 1328 /* We are splitting this into multiple parts, so 1329 * we need to prepare for allocating another r10_bio. 1330 */ 1331 r10_bio->sectors = max_sectors; 1332 spin_lock_irq(&conf->device_lock); 1333 if (bio->bi_phys_segments == 0) 1334 bio->bi_phys_segments = 2; 1335 else 1336 bio->bi_phys_segments++; 1337 spin_unlock_irq(&conf->device_lock); 1338 } 1339 sectors_handled = r10_bio->sector + max_sectors - 1340 bio->bi_iter.bi_sector; 1341 1342 atomic_set(&r10_bio->remaining, 1); 1343 bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0); 1344 1345 for (i = 0; i < conf->copies; i++) { 1346 struct bio *mbio; 1347 int d = r10_bio->devs[i].devnum; 1348 if (r10_bio->devs[i].bio) { 1349 struct md_rdev *rdev = conf->mirrors[d].rdev; 1350 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1351 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1352 max_sectors); 1353 r10_bio->devs[i].bio = mbio; 1354 1355 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+ 1356 choose_data_offset(r10_bio, 1357 rdev)); 1358 mbio->bi_bdev = rdev->bdev; 1359 mbio->bi_end_io = raid10_end_write_request; 1360 bio_set_op_attrs(mbio, op, do_sync | do_fua); 1361 mbio->bi_private = r10_bio; 1362 1363 atomic_inc(&r10_bio->remaining); 1364 1365 cb = blk_check_plugged(raid10_unplug, mddev, 1366 sizeof(*plug)); 1367 if (cb) 1368 plug = container_of(cb, struct raid10_plug_cb, 1369 cb); 1370 else 1371 plug = NULL; 1372 spin_lock_irqsave(&conf->device_lock, flags); 1373 if (plug) { 1374 bio_list_add(&plug->pending, mbio); 1375 plug->pending_cnt++; 1376 } else { 1377 bio_list_add(&conf->pending_bio_list, mbio); 1378 conf->pending_count++; 1379 } 1380 spin_unlock_irqrestore(&conf->device_lock, flags); 1381 if (!plug) 1382 md_wakeup_thread(mddev->thread); 1383 } 1384 1385 if (r10_bio->devs[i].repl_bio) { 1386 struct md_rdev *rdev = conf->mirrors[d].replacement; 1387 if (rdev == NULL) { 1388 /* Replacement just got moved to main 'rdev' */ 1389 smp_mb(); 1390 rdev = conf->mirrors[d].rdev; 1391 } 1392 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1393 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1394 max_sectors); 1395 r10_bio->devs[i].repl_bio = mbio; 1396 1397 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr + 1398 choose_data_offset( 1399 r10_bio, rdev)); 1400 mbio->bi_bdev = rdev->bdev; 1401 mbio->bi_end_io = raid10_end_write_request; 1402 bio_set_op_attrs(mbio, op, do_sync | do_fua); 1403 mbio->bi_private = r10_bio; 1404 1405 atomic_inc(&r10_bio->remaining); 1406 spin_lock_irqsave(&conf->device_lock, flags); 1407 bio_list_add(&conf->pending_bio_list, mbio); 1408 conf->pending_count++; 1409 spin_unlock_irqrestore(&conf->device_lock, flags); 1410 if (!mddev_check_plugged(mddev)) 1411 md_wakeup_thread(mddev->thread); 1412 } 1413 } 1414 1415 /* Don't remove the bias on 'remaining' (one_write_done) until 1416 * after checking if we need to go around again. 1417 */ 1418 1419 if (sectors_handled < bio_sectors(bio)) { 1420 one_write_done(r10_bio); 1421 /* We need another r10_bio. It has already been counted 1422 * in bio->bi_phys_segments. 1423 */ 1424 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1425 1426 r10_bio->master_bio = bio; 1427 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1428 1429 r10_bio->mddev = mddev; 1430 r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled; 1431 r10_bio->state = 0; 1432 goto retry_write; 1433 } 1434 one_write_done(r10_bio); 1435 } 1436 1437 static void raid10_make_request(struct mddev *mddev, struct bio *bio) 1438 { 1439 struct r10conf *conf = mddev->private; 1440 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1441 int chunk_sects = chunk_mask + 1; 1442 1443 struct bio *split; 1444 1445 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1446 md_flush_request(mddev, bio); 1447 return; 1448 } 1449 1450 do { 1451 1452 /* 1453 * If this request crosses a chunk boundary, we need to split 1454 * it. 1455 */ 1456 if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + 1457 bio_sectors(bio) > chunk_sects 1458 && (conf->geo.near_copies < conf->geo.raid_disks 1459 || conf->prev.near_copies < 1460 conf->prev.raid_disks))) { 1461 split = bio_split(bio, chunk_sects - 1462 (bio->bi_iter.bi_sector & 1463 (chunk_sects - 1)), 1464 GFP_NOIO, fs_bio_set); 1465 bio_chain(split, bio); 1466 } else { 1467 split = bio; 1468 } 1469 1470 __make_request(mddev, split); 1471 } while (split != bio); 1472 1473 /* In case raid10d snuck in to freeze_array */ 1474 wake_up(&conf->wait_barrier); 1475 } 1476 1477 static void raid10_status(struct seq_file *seq, struct mddev *mddev) 1478 { 1479 struct r10conf *conf = mddev->private; 1480 int i; 1481 1482 if (conf->geo.near_copies < conf->geo.raid_disks) 1483 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1484 if (conf->geo.near_copies > 1) 1485 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1486 if (conf->geo.far_copies > 1) { 1487 if (conf->geo.far_offset) 1488 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1489 else 1490 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1491 if (conf->geo.far_set_size != conf->geo.raid_disks) 1492 seq_printf(seq, " %d devices per set", conf->geo.far_set_size); 1493 } 1494 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1495 conf->geo.raid_disks - mddev->degraded); 1496 rcu_read_lock(); 1497 for (i = 0; i < conf->geo.raid_disks; i++) { 1498 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1499 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1500 } 1501 rcu_read_unlock(); 1502 seq_printf(seq, "]"); 1503 } 1504 1505 /* check if there are enough drives for 1506 * every block to appear on atleast one. 1507 * Don't consider the device numbered 'ignore' 1508 * as we might be about to remove it. 1509 */ 1510 static int _enough(struct r10conf *conf, int previous, int ignore) 1511 { 1512 int first = 0; 1513 int has_enough = 0; 1514 int disks, ncopies; 1515 if (previous) { 1516 disks = conf->prev.raid_disks; 1517 ncopies = conf->prev.near_copies; 1518 } else { 1519 disks = conf->geo.raid_disks; 1520 ncopies = conf->geo.near_copies; 1521 } 1522 1523 rcu_read_lock(); 1524 do { 1525 int n = conf->copies; 1526 int cnt = 0; 1527 int this = first; 1528 while (n--) { 1529 struct md_rdev *rdev; 1530 if (this != ignore && 1531 (rdev = rcu_dereference(conf->mirrors[this].rdev)) && 1532 test_bit(In_sync, &rdev->flags)) 1533 cnt++; 1534 this = (this+1) % disks; 1535 } 1536 if (cnt == 0) 1537 goto out; 1538 first = (first + ncopies) % disks; 1539 } while (first != 0); 1540 has_enough = 1; 1541 out: 1542 rcu_read_unlock(); 1543 return has_enough; 1544 } 1545 1546 static int enough(struct r10conf *conf, int ignore) 1547 { 1548 /* when calling 'enough', both 'prev' and 'geo' must 1549 * be stable. 1550 * This is ensured if ->reconfig_mutex or ->device_lock 1551 * is held. 1552 */ 1553 return _enough(conf, 0, ignore) && 1554 _enough(conf, 1, ignore); 1555 } 1556 1557 static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) 1558 { 1559 char b[BDEVNAME_SIZE]; 1560 struct r10conf *conf = mddev->private; 1561 unsigned long flags; 1562 1563 /* 1564 * If it is not operational, then we have already marked it as dead 1565 * else if it is the last working disks, ignore the error, let the 1566 * next level up know. 1567 * else mark the drive as failed 1568 */ 1569 spin_lock_irqsave(&conf->device_lock, flags); 1570 if (test_bit(In_sync, &rdev->flags) 1571 && !enough(conf, rdev->raid_disk)) { 1572 /* 1573 * Don't fail the drive, just return an IO error. 1574 */ 1575 spin_unlock_irqrestore(&conf->device_lock, flags); 1576 return; 1577 } 1578 if (test_and_clear_bit(In_sync, &rdev->flags)) 1579 mddev->degraded++; 1580 /* 1581 * If recovery is running, make sure it aborts. 1582 */ 1583 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1584 set_bit(Blocked, &rdev->flags); 1585 set_bit(Faulty, &rdev->flags); 1586 set_mask_bits(&mddev->flags, 0, 1587 BIT(MD_CHANGE_DEVS) | BIT(MD_CHANGE_PENDING)); 1588 spin_unlock_irqrestore(&conf->device_lock, flags); 1589 printk(KERN_ALERT 1590 "md/raid10:%s: Disk failure on %s, disabling device.\n" 1591 "md/raid10:%s: Operation continuing on %d devices.\n", 1592 mdname(mddev), bdevname(rdev->bdev, b), 1593 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 1594 } 1595 1596 static void print_conf(struct r10conf *conf) 1597 { 1598 int i; 1599 struct md_rdev *rdev; 1600 1601 printk(KERN_DEBUG "RAID10 conf printout:\n"); 1602 if (!conf) { 1603 printk(KERN_DEBUG "(!conf)\n"); 1604 return; 1605 } 1606 printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 1607 conf->geo.raid_disks); 1608 1609 /* This is only called with ->reconfix_mutex held, so 1610 * rcu protection of rdev is not needed */ 1611 for (i = 0; i < conf->geo.raid_disks; i++) { 1612 char b[BDEVNAME_SIZE]; 1613 rdev = conf->mirrors[i].rdev; 1614 if (rdev) 1615 printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n", 1616 i, !test_bit(In_sync, &rdev->flags), 1617 !test_bit(Faulty, &rdev->flags), 1618 bdevname(rdev->bdev,b)); 1619 } 1620 } 1621 1622 static void close_sync(struct r10conf *conf) 1623 { 1624 wait_barrier(conf); 1625 allow_barrier(conf); 1626 1627 mempool_destroy(conf->r10buf_pool); 1628 conf->r10buf_pool = NULL; 1629 } 1630 1631 static int raid10_spare_active(struct mddev *mddev) 1632 { 1633 int i; 1634 struct r10conf *conf = mddev->private; 1635 struct raid10_info *tmp; 1636 int count = 0; 1637 unsigned long flags; 1638 1639 /* 1640 * Find all non-in_sync disks within the RAID10 configuration 1641 * and mark them in_sync 1642 */ 1643 for (i = 0; i < conf->geo.raid_disks; i++) { 1644 tmp = conf->mirrors + i; 1645 if (tmp->replacement 1646 && tmp->replacement->recovery_offset == MaxSector 1647 && !test_bit(Faulty, &tmp->replacement->flags) 1648 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 1649 /* Replacement has just become active */ 1650 if (!tmp->rdev 1651 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 1652 count++; 1653 if (tmp->rdev) { 1654 /* Replaced device not technically faulty, 1655 * but we need to be sure it gets removed 1656 * and never re-added. 1657 */ 1658 set_bit(Faulty, &tmp->rdev->flags); 1659 sysfs_notify_dirent_safe( 1660 tmp->rdev->sysfs_state); 1661 } 1662 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 1663 } else if (tmp->rdev 1664 && tmp->rdev->recovery_offset == MaxSector 1665 && !test_bit(Faulty, &tmp->rdev->flags) 1666 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 1667 count++; 1668 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 1669 } 1670 } 1671 spin_lock_irqsave(&conf->device_lock, flags); 1672 mddev->degraded -= count; 1673 spin_unlock_irqrestore(&conf->device_lock, flags); 1674 1675 print_conf(conf); 1676 return count; 1677 } 1678 1679 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1680 { 1681 struct r10conf *conf = mddev->private; 1682 int err = -EEXIST; 1683 int mirror; 1684 int first = 0; 1685 int last = conf->geo.raid_disks - 1; 1686 1687 if (mddev->recovery_cp < MaxSector) 1688 /* only hot-add to in-sync arrays, as recovery is 1689 * very different from resync 1690 */ 1691 return -EBUSY; 1692 if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1)) 1693 return -EINVAL; 1694 1695 if (md_integrity_add_rdev(rdev, mddev)) 1696 return -ENXIO; 1697 1698 if (rdev->raid_disk >= 0) 1699 first = last = rdev->raid_disk; 1700 1701 if (rdev->saved_raid_disk >= first && 1702 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1703 mirror = rdev->saved_raid_disk; 1704 else 1705 mirror = first; 1706 for ( ; mirror <= last ; mirror++) { 1707 struct raid10_info *p = &conf->mirrors[mirror]; 1708 if (p->recovery_disabled == mddev->recovery_disabled) 1709 continue; 1710 if (p->rdev) { 1711 if (!test_bit(WantReplacement, &p->rdev->flags) || 1712 p->replacement != NULL) 1713 continue; 1714 clear_bit(In_sync, &rdev->flags); 1715 set_bit(Replacement, &rdev->flags); 1716 rdev->raid_disk = mirror; 1717 err = 0; 1718 if (mddev->gendisk) 1719 disk_stack_limits(mddev->gendisk, rdev->bdev, 1720 rdev->data_offset << 9); 1721 conf->fullsync = 1; 1722 rcu_assign_pointer(p->replacement, rdev); 1723 break; 1724 } 1725 1726 if (mddev->gendisk) 1727 disk_stack_limits(mddev->gendisk, rdev->bdev, 1728 rdev->data_offset << 9); 1729 1730 p->head_position = 0; 1731 p->recovery_disabled = mddev->recovery_disabled - 1; 1732 rdev->raid_disk = mirror; 1733 err = 0; 1734 if (rdev->saved_raid_disk != mirror) 1735 conf->fullsync = 1; 1736 rcu_assign_pointer(p->rdev, rdev); 1737 break; 1738 } 1739 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1740 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1741 1742 print_conf(conf); 1743 return err; 1744 } 1745 1746 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1747 { 1748 struct r10conf *conf = mddev->private; 1749 int err = 0; 1750 int number = rdev->raid_disk; 1751 struct md_rdev **rdevp; 1752 struct raid10_info *p = conf->mirrors + number; 1753 1754 print_conf(conf); 1755 if (rdev == p->rdev) 1756 rdevp = &p->rdev; 1757 else if (rdev == p->replacement) 1758 rdevp = &p->replacement; 1759 else 1760 return 0; 1761 1762 if (test_bit(In_sync, &rdev->flags) || 1763 atomic_read(&rdev->nr_pending)) { 1764 err = -EBUSY; 1765 goto abort; 1766 } 1767 /* Only remove non-faulty devices if recovery 1768 * is not possible. 1769 */ 1770 if (!test_bit(Faulty, &rdev->flags) && 1771 mddev->recovery_disabled != p->recovery_disabled && 1772 (!p->replacement || p->replacement == rdev) && 1773 number < conf->geo.raid_disks && 1774 enough(conf, -1)) { 1775 err = -EBUSY; 1776 goto abort; 1777 } 1778 *rdevp = NULL; 1779 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 1780 synchronize_rcu(); 1781 if (atomic_read(&rdev->nr_pending)) { 1782 /* lost the race, try later */ 1783 err = -EBUSY; 1784 *rdevp = rdev; 1785 goto abort; 1786 } 1787 } 1788 if (p->replacement) { 1789 /* We must have just cleared 'rdev' */ 1790 p->rdev = p->replacement; 1791 clear_bit(Replacement, &p->replacement->flags); 1792 smp_mb(); /* Make sure other CPUs may see both as identical 1793 * but will never see neither -- if they are careful. 1794 */ 1795 p->replacement = NULL; 1796 clear_bit(WantReplacement, &rdev->flags); 1797 } else 1798 /* We might have just remove the Replacement as faulty 1799 * Clear the flag just in case 1800 */ 1801 clear_bit(WantReplacement, &rdev->flags); 1802 1803 err = md_integrity_register(mddev); 1804 1805 abort: 1806 1807 print_conf(conf); 1808 return err; 1809 } 1810 1811 static void end_sync_read(struct bio *bio) 1812 { 1813 struct r10bio *r10_bio = bio->bi_private; 1814 struct r10conf *conf = r10_bio->mddev->private; 1815 int d; 1816 1817 if (bio == r10_bio->master_bio) { 1818 /* this is a reshape read */ 1819 d = r10_bio->read_slot; /* really the read dev */ 1820 } else 1821 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1822 1823 if (!bio->bi_error) 1824 set_bit(R10BIO_Uptodate, &r10_bio->state); 1825 else 1826 /* The write handler will notice the lack of 1827 * R10BIO_Uptodate and record any errors etc 1828 */ 1829 atomic_add(r10_bio->sectors, 1830 &conf->mirrors[d].rdev->corrected_errors); 1831 1832 /* for reconstruct, we always reschedule after a read. 1833 * for resync, only after all reads 1834 */ 1835 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 1836 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 1837 atomic_dec_and_test(&r10_bio->remaining)) { 1838 /* we have read all the blocks, 1839 * do the comparison in process context in raid10d 1840 */ 1841 reschedule_retry(r10_bio); 1842 } 1843 } 1844 1845 static void end_sync_request(struct r10bio *r10_bio) 1846 { 1847 struct mddev *mddev = r10_bio->mddev; 1848 1849 while (atomic_dec_and_test(&r10_bio->remaining)) { 1850 if (r10_bio->master_bio == NULL) { 1851 /* the primary of several recovery bios */ 1852 sector_t s = r10_bio->sectors; 1853 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1854 test_bit(R10BIO_WriteError, &r10_bio->state)) 1855 reschedule_retry(r10_bio); 1856 else 1857 put_buf(r10_bio); 1858 md_done_sync(mddev, s, 1); 1859 break; 1860 } else { 1861 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 1862 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1863 test_bit(R10BIO_WriteError, &r10_bio->state)) 1864 reschedule_retry(r10_bio); 1865 else 1866 put_buf(r10_bio); 1867 r10_bio = r10_bio2; 1868 } 1869 } 1870 } 1871 1872 static void end_sync_write(struct bio *bio) 1873 { 1874 struct r10bio *r10_bio = bio->bi_private; 1875 struct mddev *mddev = r10_bio->mddev; 1876 struct r10conf *conf = mddev->private; 1877 int d; 1878 sector_t first_bad; 1879 int bad_sectors; 1880 int slot; 1881 int repl; 1882 struct md_rdev *rdev = NULL; 1883 1884 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1885 if (repl) 1886 rdev = conf->mirrors[d].replacement; 1887 else 1888 rdev = conf->mirrors[d].rdev; 1889 1890 if (bio->bi_error) { 1891 if (repl) 1892 md_error(mddev, rdev); 1893 else { 1894 set_bit(WriteErrorSeen, &rdev->flags); 1895 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 1896 set_bit(MD_RECOVERY_NEEDED, 1897 &rdev->mddev->recovery); 1898 set_bit(R10BIO_WriteError, &r10_bio->state); 1899 } 1900 } else if (is_badblock(rdev, 1901 r10_bio->devs[slot].addr, 1902 r10_bio->sectors, 1903 &first_bad, &bad_sectors)) 1904 set_bit(R10BIO_MadeGood, &r10_bio->state); 1905 1906 rdev_dec_pending(rdev, mddev); 1907 1908 end_sync_request(r10_bio); 1909 } 1910 1911 /* 1912 * Note: sync and recover and handled very differently for raid10 1913 * This code is for resync. 1914 * For resync, we read through virtual addresses and read all blocks. 1915 * If there is any error, we schedule a write. The lowest numbered 1916 * drive is authoritative. 1917 * However requests come for physical address, so we need to map. 1918 * For every physical address there are raid_disks/copies virtual addresses, 1919 * which is always are least one, but is not necessarly an integer. 1920 * This means that a physical address can span multiple chunks, so we may 1921 * have to submit multiple io requests for a single sync request. 1922 */ 1923 /* 1924 * We check if all blocks are in-sync and only write to blocks that 1925 * aren't in sync 1926 */ 1927 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 1928 { 1929 struct r10conf *conf = mddev->private; 1930 int i, first; 1931 struct bio *tbio, *fbio; 1932 int vcnt; 1933 1934 atomic_set(&r10_bio->remaining, 1); 1935 1936 /* find the first device with a block */ 1937 for (i=0; i<conf->copies; i++) 1938 if (!r10_bio->devs[i].bio->bi_error) 1939 break; 1940 1941 if (i == conf->copies) 1942 goto done; 1943 1944 first = i; 1945 fbio = r10_bio->devs[i].bio; 1946 fbio->bi_iter.bi_size = r10_bio->sectors << 9; 1947 fbio->bi_iter.bi_idx = 0; 1948 1949 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 1950 /* now find blocks with errors */ 1951 for (i=0 ; i < conf->copies ; i++) { 1952 int j, d; 1953 1954 tbio = r10_bio->devs[i].bio; 1955 1956 if (tbio->bi_end_io != end_sync_read) 1957 continue; 1958 if (i == first) 1959 continue; 1960 if (!r10_bio->devs[i].bio->bi_error) { 1961 /* We know that the bi_io_vec layout is the same for 1962 * both 'first' and 'i', so we just compare them. 1963 * All vec entries are PAGE_SIZE; 1964 */ 1965 int sectors = r10_bio->sectors; 1966 for (j = 0; j < vcnt; j++) { 1967 int len = PAGE_SIZE; 1968 if (sectors < (len / 512)) 1969 len = sectors * 512; 1970 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page), 1971 page_address(tbio->bi_io_vec[j].bv_page), 1972 len)) 1973 break; 1974 sectors -= len/512; 1975 } 1976 if (j == vcnt) 1977 continue; 1978 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 1979 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 1980 /* Don't fix anything. */ 1981 continue; 1982 } 1983 /* Ok, we need to write this bio, either to correct an 1984 * inconsistency or to correct an unreadable block. 1985 * First we need to fixup bv_offset, bv_len and 1986 * bi_vecs, as the read request might have corrupted these 1987 */ 1988 bio_reset(tbio); 1989 1990 tbio->bi_vcnt = vcnt; 1991 tbio->bi_iter.bi_size = fbio->bi_iter.bi_size; 1992 tbio->bi_private = r10_bio; 1993 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; 1994 tbio->bi_end_io = end_sync_write; 1995 bio_set_op_attrs(tbio, REQ_OP_WRITE, 0); 1996 1997 bio_copy_data(tbio, fbio); 1998 1999 d = r10_bio->devs[i].devnum; 2000 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2001 atomic_inc(&r10_bio->remaining); 2002 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio)); 2003 2004 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset; 2005 tbio->bi_bdev = conf->mirrors[d].rdev->bdev; 2006 generic_make_request(tbio); 2007 } 2008 2009 /* Now write out to any replacement devices 2010 * that are active 2011 */ 2012 for (i = 0; i < conf->copies; i++) { 2013 int d; 2014 2015 tbio = r10_bio->devs[i].repl_bio; 2016 if (!tbio || !tbio->bi_end_io) 2017 continue; 2018 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2019 && r10_bio->devs[i].bio != fbio) 2020 bio_copy_data(tbio, fbio); 2021 d = r10_bio->devs[i].devnum; 2022 atomic_inc(&r10_bio->remaining); 2023 md_sync_acct(conf->mirrors[d].replacement->bdev, 2024 bio_sectors(tbio)); 2025 generic_make_request(tbio); 2026 } 2027 2028 done: 2029 if (atomic_dec_and_test(&r10_bio->remaining)) { 2030 md_done_sync(mddev, r10_bio->sectors, 1); 2031 put_buf(r10_bio); 2032 } 2033 } 2034 2035 /* 2036 * Now for the recovery code. 2037 * Recovery happens across physical sectors. 2038 * We recover all non-is_sync drives by finding the virtual address of 2039 * each, and then choose a working drive that also has that virt address. 2040 * There is a separate r10_bio for each non-in_sync drive. 2041 * Only the first two slots are in use. The first for reading, 2042 * The second for writing. 2043 * 2044 */ 2045 static void fix_recovery_read_error(struct r10bio *r10_bio) 2046 { 2047 /* We got a read error during recovery. 2048 * We repeat the read in smaller page-sized sections. 2049 * If a read succeeds, write it to the new device or record 2050 * a bad block if we cannot. 2051 * If a read fails, record a bad block on both old and 2052 * new devices. 2053 */ 2054 struct mddev *mddev = r10_bio->mddev; 2055 struct r10conf *conf = mddev->private; 2056 struct bio *bio = r10_bio->devs[0].bio; 2057 sector_t sect = 0; 2058 int sectors = r10_bio->sectors; 2059 int idx = 0; 2060 int dr = r10_bio->devs[0].devnum; 2061 int dw = r10_bio->devs[1].devnum; 2062 2063 while (sectors) { 2064 int s = sectors; 2065 struct md_rdev *rdev; 2066 sector_t addr; 2067 int ok; 2068 2069 if (s > (PAGE_SIZE>>9)) 2070 s = PAGE_SIZE >> 9; 2071 2072 rdev = conf->mirrors[dr].rdev; 2073 addr = r10_bio->devs[0].addr + sect, 2074 ok = sync_page_io(rdev, 2075 addr, 2076 s << 9, 2077 bio->bi_io_vec[idx].bv_page, 2078 REQ_OP_READ, 0, false); 2079 if (ok) { 2080 rdev = conf->mirrors[dw].rdev; 2081 addr = r10_bio->devs[1].addr + sect; 2082 ok = sync_page_io(rdev, 2083 addr, 2084 s << 9, 2085 bio->bi_io_vec[idx].bv_page, 2086 REQ_OP_WRITE, 0, false); 2087 if (!ok) { 2088 set_bit(WriteErrorSeen, &rdev->flags); 2089 if (!test_and_set_bit(WantReplacement, 2090 &rdev->flags)) 2091 set_bit(MD_RECOVERY_NEEDED, 2092 &rdev->mddev->recovery); 2093 } 2094 } 2095 if (!ok) { 2096 /* We don't worry if we cannot set a bad block - 2097 * it really is bad so there is no loss in not 2098 * recording it yet 2099 */ 2100 rdev_set_badblocks(rdev, addr, s, 0); 2101 2102 if (rdev != conf->mirrors[dw].rdev) { 2103 /* need bad block on destination too */ 2104 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2105 addr = r10_bio->devs[1].addr + sect; 2106 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2107 if (!ok) { 2108 /* just abort the recovery */ 2109 printk(KERN_NOTICE 2110 "md/raid10:%s: recovery aborted" 2111 " due to read error\n", 2112 mdname(mddev)); 2113 2114 conf->mirrors[dw].recovery_disabled 2115 = mddev->recovery_disabled; 2116 set_bit(MD_RECOVERY_INTR, 2117 &mddev->recovery); 2118 break; 2119 } 2120 } 2121 } 2122 2123 sectors -= s; 2124 sect += s; 2125 idx++; 2126 } 2127 } 2128 2129 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2130 { 2131 struct r10conf *conf = mddev->private; 2132 int d; 2133 struct bio *wbio, *wbio2; 2134 2135 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2136 fix_recovery_read_error(r10_bio); 2137 end_sync_request(r10_bio); 2138 return; 2139 } 2140 2141 /* 2142 * share the pages with the first bio 2143 * and submit the write request 2144 */ 2145 d = r10_bio->devs[1].devnum; 2146 wbio = r10_bio->devs[1].bio; 2147 wbio2 = r10_bio->devs[1].repl_bio; 2148 /* Need to test wbio2->bi_end_io before we call 2149 * generic_make_request as if the former is NULL, 2150 * the latter is free to free wbio2. 2151 */ 2152 if (wbio2 && !wbio2->bi_end_io) 2153 wbio2 = NULL; 2154 if (wbio->bi_end_io) { 2155 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2156 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio)); 2157 generic_make_request(wbio); 2158 } 2159 if (wbio2) { 2160 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2161 md_sync_acct(conf->mirrors[d].replacement->bdev, 2162 bio_sectors(wbio2)); 2163 generic_make_request(wbio2); 2164 } 2165 } 2166 2167 /* 2168 * Used by fix_read_error() to decay the per rdev read_errors. 2169 * We halve the read error count for every hour that has elapsed 2170 * since the last recorded read error. 2171 * 2172 */ 2173 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2174 { 2175 long cur_time_mon; 2176 unsigned long hours_since_last; 2177 unsigned int read_errors = atomic_read(&rdev->read_errors); 2178 2179 cur_time_mon = ktime_get_seconds(); 2180 2181 if (rdev->last_read_error == 0) { 2182 /* first time we've seen a read error */ 2183 rdev->last_read_error = cur_time_mon; 2184 return; 2185 } 2186 2187 hours_since_last = (long)(cur_time_mon - 2188 rdev->last_read_error) / 3600; 2189 2190 rdev->last_read_error = cur_time_mon; 2191 2192 /* 2193 * if hours_since_last is > the number of bits in read_errors 2194 * just set read errors to 0. We do this to avoid 2195 * overflowing the shift of read_errors by hours_since_last. 2196 */ 2197 if (hours_since_last >= 8 * sizeof(read_errors)) 2198 atomic_set(&rdev->read_errors, 0); 2199 else 2200 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2201 } 2202 2203 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2204 int sectors, struct page *page, int rw) 2205 { 2206 sector_t first_bad; 2207 int bad_sectors; 2208 2209 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) 2210 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) 2211 return -1; 2212 if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) 2213 /* success */ 2214 return 1; 2215 if (rw == WRITE) { 2216 set_bit(WriteErrorSeen, &rdev->flags); 2217 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2218 set_bit(MD_RECOVERY_NEEDED, 2219 &rdev->mddev->recovery); 2220 } 2221 /* need to record an error - either for the block or the device */ 2222 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2223 md_error(rdev->mddev, rdev); 2224 return 0; 2225 } 2226 2227 /* 2228 * This is a kernel thread which: 2229 * 2230 * 1. Retries failed read operations on working mirrors. 2231 * 2. Updates the raid superblock when problems encounter. 2232 * 3. Performs writes following reads for array synchronising. 2233 */ 2234 2235 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2236 { 2237 int sect = 0; /* Offset from r10_bio->sector */ 2238 int sectors = r10_bio->sectors; 2239 struct md_rdev*rdev; 2240 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2241 int d = r10_bio->devs[r10_bio->read_slot].devnum; 2242 2243 /* still own a reference to this rdev, so it cannot 2244 * have been cleared recently. 2245 */ 2246 rdev = conf->mirrors[d].rdev; 2247 2248 if (test_bit(Faulty, &rdev->flags)) 2249 /* drive has already been failed, just ignore any 2250 more fix_read_error() attempts */ 2251 return; 2252 2253 check_decay_read_errors(mddev, rdev); 2254 atomic_inc(&rdev->read_errors); 2255 if (atomic_read(&rdev->read_errors) > max_read_errors) { 2256 char b[BDEVNAME_SIZE]; 2257 bdevname(rdev->bdev, b); 2258 2259 printk(KERN_NOTICE 2260 "md/raid10:%s: %s: Raid device exceeded " 2261 "read_error threshold [cur %d:max %d]\n", 2262 mdname(mddev), b, 2263 atomic_read(&rdev->read_errors), max_read_errors); 2264 printk(KERN_NOTICE 2265 "md/raid10:%s: %s: Failing raid device\n", 2266 mdname(mddev), b); 2267 md_error(mddev, rdev); 2268 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED; 2269 return; 2270 } 2271 2272 while(sectors) { 2273 int s = sectors; 2274 int sl = r10_bio->read_slot; 2275 int success = 0; 2276 int start; 2277 2278 if (s > (PAGE_SIZE>>9)) 2279 s = PAGE_SIZE >> 9; 2280 2281 rcu_read_lock(); 2282 do { 2283 sector_t first_bad; 2284 int bad_sectors; 2285 2286 d = r10_bio->devs[sl].devnum; 2287 rdev = rcu_dereference(conf->mirrors[d].rdev); 2288 if (rdev && 2289 test_bit(In_sync, &rdev->flags) && 2290 !test_bit(Faulty, &rdev->flags) && 2291 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2292 &first_bad, &bad_sectors) == 0) { 2293 atomic_inc(&rdev->nr_pending); 2294 rcu_read_unlock(); 2295 success = sync_page_io(rdev, 2296 r10_bio->devs[sl].addr + 2297 sect, 2298 s<<9, 2299 conf->tmppage, 2300 REQ_OP_READ, 0, false); 2301 rdev_dec_pending(rdev, mddev); 2302 rcu_read_lock(); 2303 if (success) 2304 break; 2305 } 2306 sl++; 2307 if (sl == conf->copies) 2308 sl = 0; 2309 } while (!success && sl != r10_bio->read_slot); 2310 rcu_read_unlock(); 2311 2312 if (!success) { 2313 /* Cannot read from anywhere, just mark the block 2314 * as bad on the first device to discourage future 2315 * reads. 2316 */ 2317 int dn = r10_bio->devs[r10_bio->read_slot].devnum; 2318 rdev = conf->mirrors[dn].rdev; 2319 2320 if (!rdev_set_badblocks( 2321 rdev, 2322 r10_bio->devs[r10_bio->read_slot].addr 2323 + sect, 2324 s, 0)) { 2325 md_error(mddev, rdev); 2326 r10_bio->devs[r10_bio->read_slot].bio 2327 = IO_BLOCKED; 2328 } 2329 break; 2330 } 2331 2332 start = sl; 2333 /* write it back and re-read */ 2334 rcu_read_lock(); 2335 while (sl != r10_bio->read_slot) { 2336 char b[BDEVNAME_SIZE]; 2337 2338 if (sl==0) 2339 sl = conf->copies; 2340 sl--; 2341 d = r10_bio->devs[sl].devnum; 2342 rdev = rcu_dereference(conf->mirrors[d].rdev); 2343 if (!rdev || 2344 test_bit(Faulty, &rdev->flags) || 2345 !test_bit(In_sync, &rdev->flags)) 2346 continue; 2347 2348 atomic_inc(&rdev->nr_pending); 2349 rcu_read_unlock(); 2350 if (r10_sync_page_io(rdev, 2351 r10_bio->devs[sl].addr + 2352 sect, 2353 s, conf->tmppage, WRITE) 2354 == 0) { 2355 /* Well, this device is dead */ 2356 printk(KERN_NOTICE 2357 "md/raid10:%s: read correction " 2358 "write failed" 2359 " (%d sectors at %llu on %s)\n", 2360 mdname(mddev), s, 2361 (unsigned long long)( 2362 sect + 2363 choose_data_offset(r10_bio, 2364 rdev)), 2365 bdevname(rdev->bdev, b)); 2366 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2367 "drive\n", 2368 mdname(mddev), 2369 bdevname(rdev->bdev, b)); 2370 } 2371 rdev_dec_pending(rdev, mddev); 2372 rcu_read_lock(); 2373 } 2374 sl = start; 2375 while (sl != r10_bio->read_slot) { 2376 char b[BDEVNAME_SIZE]; 2377 2378 if (sl==0) 2379 sl = conf->copies; 2380 sl--; 2381 d = r10_bio->devs[sl].devnum; 2382 rdev = rcu_dereference(conf->mirrors[d].rdev); 2383 if (!rdev || 2384 test_bit(Faulty, &rdev->flags) || 2385 !test_bit(In_sync, &rdev->flags)) 2386 continue; 2387 2388 atomic_inc(&rdev->nr_pending); 2389 rcu_read_unlock(); 2390 switch (r10_sync_page_io(rdev, 2391 r10_bio->devs[sl].addr + 2392 sect, 2393 s, conf->tmppage, 2394 READ)) { 2395 case 0: 2396 /* Well, this device is dead */ 2397 printk(KERN_NOTICE 2398 "md/raid10:%s: unable to read back " 2399 "corrected sectors" 2400 " (%d sectors at %llu on %s)\n", 2401 mdname(mddev), s, 2402 (unsigned long long)( 2403 sect + 2404 choose_data_offset(r10_bio, rdev)), 2405 bdevname(rdev->bdev, b)); 2406 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2407 "drive\n", 2408 mdname(mddev), 2409 bdevname(rdev->bdev, b)); 2410 break; 2411 case 1: 2412 printk(KERN_INFO 2413 "md/raid10:%s: read error corrected" 2414 " (%d sectors at %llu on %s)\n", 2415 mdname(mddev), s, 2416 (unsigned long long)( 2417 sect + 2418 choose_data_offset(r10_bio, rdev)), 2419 bdevname(rdev->bdev, b)); 2420 atomic_add(s, &rdev->corrected_errors); 2421 } 2422 2423 rdev_dec_pending(rdev, mddev); 2424 rcu_read_lock(); 2425 } 2426 rcu_read_unlock(); 2427 2428 sectors -= s; 2429 sect += s; 2430 } 2431 } 2432 2433 static int narrow_write_error(struct r10bio *r10_bio, int i) 2434 { 2435 struct bio *bio = r10_bio->master_bio; 2436 struct mddev *mddev = r10_bio->mddev; 2437 struct r10conf *conf = mddev->private; 2438 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2439 /* bio has the data to be written to slot 'i' where 2440 * we just recently had a write error. 2441 * We repeatedly clone the bio and trim down to one block, 2442 * then try the write. Where the write fails we record 2443 * a bad block. 2444 * It is conceivable that the bio doesn't exactly align with 2445 * blocks. We must handle this. 2446 * 2447 * We currently own a reference to the rdev. 2448 */ 2449 2450 int block_sectors; 2451 sector_t sector; 2452 int sectors; 2453 int sect_to_write = r10_bio->sectors; 2454 int ok = 1; 2455 2456 if (rdev->badblocks.shift < 0) 2457 return 0; 2458 2459 block_sectors = roundup(1 << rdev->badblocks.shift, 2460 bdev_logical_block_size(rdev->bdev) >> 9); 2461 sector = r10_bio->sector; 2462 sectors = ((r10_bio->sector + block_sectors) 2463 & ~(sector_t)(block_sectors - 1)) 2464 - sector; 2465 2466 while (sect_to_write) { 2467 struct bio *wbio; 2468 sector_t wsector; 2469 if (sectors > sect_to_write) 2470 sectors = sect_to_write; 2471 /* Write at 'sector' for 'sectors' */ 2472 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 2473 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors); 2474 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector); 2475 wbio->bi_iter.bi_sector = wsector + 2476 choose_data_offset(r10_bio, rdev); 2477 wbio->bi_bdev = rdev->bdev; 2478 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); 2479 2480 if (submit_bio_wait(wbio) < 0) 2481 /* Failure! */ 2482 ok = rdev_set_badblocks(rdev, wsector, 2483 sectors, 0) 2484 && ok; 2485 2486 bio_put(wbio); 2487 sect_to_write -= sectors; 2488 sector += sectors; 2489 sectors = block_sectors; 2490 } 2491 return ok; 2492 } 2493 2494 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2495 { 2496 int slot = r10_bio->read_slot; 2497 struct bio *bio; 2498 struct r10conf *conf = mddev->private; 2499 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2500 char b[BDEVNAME_SIZE]; 2501 unsigned long do_sync; 2502 int max_sectors; 2503 2504 /* we got a read error. Maybe the drive is bad. Maybe just 2505 * the block and we can fix it. 2506 * We freeze all other IO, and try reading the block from 2507 * other devices. When we find one, we re-write 2508 * and check it that fixes the read error. 2509 * This is all done synchronously while the array is 2510 * frozen. 2511 */ 2512 bio = r10_bio->devs[slot].bio; 2513 bdevname(bio->bi_bdev, b); 2514 bio_put(bio); 2515 r10_bio->devs[slot].bio = NULL; 2516 2517 if (mddev->ro == 0) { 2518 freeze_array(conf, 1); 2519 fix_read_error(conf, mddev, r10_bio); 2520 unfreeze_array(conf); 2521 } else 2522 r10_bio->devs[slot].bio = IO_BLOCKED; 2523 2524 rdev_dec_pending(rdev, mddev); 2525 2526 read_more: 2527 rdev = read_balance(conf, r10_bio, &max_sectors); 2528 if (rdev == NULL) { 2529 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O" 2530 " read error for block %llu\n", 2531 mdname(mddev), b, 2532 (unsigned long long)r10_bio->sector); 2533 raid_end_bio_io(r10_bio); 2534 return; 2535 } 2536 2537 do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC); 2538 slot = r10_bio->read_slot; 2539 printk_ratelimited( 2540 KERN_ERR 2541 "md/raid10:%s: %s: redirecting " 2542 "sector %llu to another mirror\n", 2543 mdname(mddev), 2544 bdevname(rdev->bdev, b), 2545 (unsigned long long)r10_bio->sector); 2546 bio = bio_clone_mddev(r10_bio->master_bio, 2547 GFP_NOIO, mddev); 2548 bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors); 2549 r10_bio->devs[slot].bio = bio; 2550 r10_bio->devs[slot].rdev = rdev; 2551 bio->bi_iter.bi_sector = r10_bio->devs[slot].addr 2552 + choose_data_offset(r10_bio, rdev); 2553 bio->bi_bdev = rdev->bdev; 2554 bio_set_op_attrs(bio, REQ_OP_READ, do_sync); 2555 bio->bi_private = r10_bio; 2556 bio->bi_end_io = raid10_end_read_request; 2557 if (max_sectors < r10_bio->sectors) { 2558 /* Drat - have to split this up more */ 2559 struct bio *mbio = r10_bio->master_bio; 2560 int sectors_handled = 2561 r10_bio->sector + max_sectors 2562 - mbio->bi_iter.bi_sector; 2563 r10_bio->sectors = max_sectors; 2564 spin_lock_irq(&conf->device_lock); 2565 if (mbio->bi_phys_segments == 0) 2566 mbio->bi_phys_segments = 2; 2567 else 2568 mbio->bi_phys_segments++; 2569 spin_unlock_irq(&conf->device_lock); 2570 generic_make_request(bio); 2571 2572 r10_bio = mempool_alloc(conf->r10bio_pool, 2573 GFP_NOIO); 2574 r10_bio->master_bio = mbio; 2575 r10_bio->sectors = bio_sectors(mbio) - sectors_handled; 2576 r10_bio->state = 0; 2577 set_bit(R10BIO_ReadError, 2578 &r10_bio->state); 2579 r10_bio->mddev = mddev; 2580 r10_bio->sector = mbio->bi_iter.bi_sector 2581 + sectors_handled; 2582 2583 goto read_more; 2584 } else 2585 generic_make_request(bio); 2586 } 2587 2588 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2589 { 2590 /* Some sort of write request has finished and it 2591 * succeeded in writing where we thought there was a 2592 * bad block. So forget the bad block. 2593 * Or possibly if failed and we need to record 2594 * a bad block. 2595 */ 2596 int m; 2597 struct md_rdev *rdev; 2598 2599 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2600 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2601 for (m = 0; m < conf->copies; m++) { 2602 int dev = r10_bio->devs[m].devnum; 2603 rdev = conf->mirrors[dev].rdev; 2604 if (r10_bio->devs[m].bio == NULL) 2605 continue; 2606 if (!r10_bio->devs[m].bio->bi_error) { 2607 rdev_clear_badblocks( 2608 rdev, 2609 r10_bio->devs[m].addr, 2610 r10_bio->sectors, 0); 2611 } else { 2612 if (!rdev_set_badblocks( 2613 rdev, 2614 r10_bio->devs[m].addr, 2615 r10_bio->sectors, 0)) 2616 md_error(conf->mddev, rdev); 2617 } 2618 rdev = conf->mirrors[dev].replacement; 2619 if (r10_bio->devs[m].repl_bio == NULL) 2620 continue; 2621 2622 if (!r10_bio->devs[m].repl_bio->bi_error) { 2623 rdev_clear_badblocks( 2624 rdev, 2625 r10_bio->devs[m].addr, 2626 r10_bio->sectors, 0); 2627 } else { 2628 if (!rdev_set_badblocks( 2629 rdev, 2630 r10_bio->devs[m].addr, 2631 r10_bio->sectors, 0)) 2632 md_error(conf->mddev, rdev); 2633 } 2634 } 2635 put_buf(r10_bio); 2636 } else { 2637 bool fail = false; 2638 for (m = 0; m < conf->copies; m++) { 2639 int dev = r10_bio->devs[m].devnum; 2640 struct bio *bio = r10_bio->devs[m].bio; 2641 rdev = conf->mirrors[dev].rdev; 2642 if (bio == IO_MADE_GOOD) { 2643 rdev_clear_badblocks( 2644 rdev, 2645 r10_bio->devs[m].addr, 2646 r10_bio->sectors, 0); 2647 rdev_dec_pending(rdev, conf->mddev); 2648 } else if (bio != NULL && bio->bi_error) { 2649 fail = true; 2650 if (!narrow_write_error(r10_bio, m)) { 2651 md_error(conf->mddev, rdev); 2652 set_bit(R10BIO_Degraded, 2653 &r10_bio->state); 2654 } 2655 rdev_dec_pending(rdev, conf->mddev); 2656 } 2657 bio = r10_bio->devs[m].repl_bio; 2658 rdev = conf->mirrors[dev].replacement; 2659 if (rdev && bio == IO_MADE_GOOD) { 2660 rdev_clear_badblocks( 2661 rdev, 2662 r10_bio->devs[m].addr, 2663 r10_bio->sectors, 0); 2664 rdev_dec_pending(rdev, conf->mddev); 2665 } 2666 } 2667 if (fail) { 2668 spin_lock_irq(&conf->device_lock); 2669 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); 2670 conf->nr_queued++; 2671 spin_unlock_irq(&conf->device_lock); 2672 md_wakeup_thread(conf->mddev->thread); 2673 } else { 2674 if (test_bit(R10BIO_WriteError, 2675 &r10_bio->state)) 2676 close_write(r10_bio); 2677 raid_end_bio_io(r10_bio); 2678 } 2679 } 2680 } 2681 2682 static void raid10d(struct md_thread *thread) 2683 { 2684 struct mddev *mddev = thread->mddev; 2685 struct r10bio *r10_bio; 2686 unsigned long flags; 2687 struct r10conf *conf = mddev->private; 2688 struct list_head *head = &conf->retry_list; 2689 struct blk_plug plug; 2690 2691 md_check_recovery(mddev); 2692 2693 if (!list_empty_careful(&conf->bio_end_io_list) && 2694 !test_bit(MD_CHANGE_PENDING, &mddev->flags)) { 2695 LIST_HEAD(tmp); 2696 spin_lock_irqsave(&conf->device_lock, flags); 2697 if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) { 2698 while (!list_empty(&conf->bio_end_io_list)) { 2699 list_move(conf->bio_end_io_list.prev, &tmp); 2700 conf->nr_queued--; 2701 } 2702 } 2703 spin_unlock_irqrestore(&conf->device_lock, flags); 2704 while (!list_empty(&tmp)) { 2705 r10_bio = list_first_entry(&tmp, struct r10bio, 2706 retry_list); 2707 list_del(&r10_bio->retry_list); 2708 if (mddev->degraded) 2709 set_bit(R10BIO_Degraded, &r10_bio->state); 2710 2711 if (test_bit(R10BIO_WriteError, 2712 &r10_bio->state)) 2713 close_write(r10_bio); 2714 raid_end_bio_io(r10_bio); 2715 } 2716 } 2717 2718 blk_start_plug(&plug); 2719 for (;;) { 2720 2721 flush_pending_writes(conf); 2722 2723 spin_lock_irqsave(&conf->device_lock, flags); 2724 if (list_empty(head)) { 2725 spin_unlock_irqrestore(&conf->device_lock, flags); 2726 break; 2727 } 2728 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 2729 list_del(head->prev); 2730 conf->nr_queued--; 2731 spin_unlock_irqrestore(&conf->device_lock, flags); 2732 2733 mddev = r10_bio->mddev; 2734 conf = mddev->private; 2735 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2736 test_bit(R10BIO_WriteError, &r10_bio->state)) 2737 handle_write_completed(conf, r10_bio); 2738 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 2739 reshape_request_write(mddev, r10_bio); 2740 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 2741 sync_request_write(mddev, r10_bio); 2742 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 2743 recovery_request_write(mddev, r10_bio); 2744 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 2745 handle_read_error(mddev, r10_bio); 2746 else { 2747 /* just a partial read to be scheduled from a 2748 * separate context 2749 */ 2750 int slot = r10_bio->read_slot; 2751 generic_make_request(r10_bio->devs[slot].bio); 2752 } 2753 2754 cond_resched(); 2755 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) 2756 md_check_recovery(mddev); 2757 } 2758 blk_finish_plug(&plug); 2759 } 2760 2761 static int init_resync(struct r10conf *conf) 2762 { 2763 int buffs; 2764 int i; 2765 2766 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2767 BUG_ON(conf->r10buf_pool); 2768 conf->have_replacement = 0; 2769 for (i = 0; i < conf->geo.raid_disks; i++) 2770 if (conf->mirrors[i].replacement) 2771 conf->have_replacement = 1; 2772 conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf); 2773 if (!conf->r10buf_pool) 2774 return -ENOMEM; 2775 conf->next_resync = 0; 2776 return 0; 2777 } 2778 2779 /* 2780 * perform a "sync" on one "block" 2781 * 2782 * We need to make sure that no normal I/O request - particularly write 2783 * requests - conflict with active sync requests. 2784 * 2785 * This is achieved by tracking pending requests and a 'barrier' concept 2786 * that can be installed to exclude normal IO requests. 2787 * 2788 * Resync and recovery are handled very differently. 2789 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 2790 * 2791 * For resync, we iterate over virtual addresses, read all copies, 2792 * and update if there are differences. If only one copy is live, 2793 * skip it. 2794 * For recovery, we iterate over physical addresses, read a good 2795 * value for each non-in_sync drive, and over-write. 2796 * 2797 * So, for recovery we may have several outstanding complex requests for a 2798 * given address, one for each out-of-sync device. We model this by allocating 2799 * a number of r10_bio structures, one for each out-of-sync device. 2800 * As we setup these structures, we collect all bio's together into a list 2801 * which we then process collectively to add pages, and then process again 2802 * to pass to generic_make_request. 2803 * 2804 * The r10_bio structures are linked using a borrowed master_bio pointer. 2805 * This link is counted in ->remaining. When the r10_bio that points to NULL 2806 * has its remaining count decremented to 0, the whole complex operation 2807 * is complete. 2808 * 2809 */ 2810 2811 static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, 2812 int *skipped) 2813 { 2814 struct r10conf *conf = mddev->private; 2815 struct r10bio *r10_bio; 2816 struct bio *biolist = NULL, *bio; 2817 sector_t max_sector, nr_sectors; 2818 int i; 2819 int max_sync; 2820 sector_t sync_blocks; 2821 sector_t sectors_skipped = 0; 2822 int chunks_skipped = 0; 2823 sector_t chunk_mask = conf->geo.chunk_mask; 2824 2825 if (!conf->r10buf_pool) 2826 if (init_resync(conf)) 2827 return 0; 2828 2829 /* 2830 * Allow skipping a full rebuild for incremental assembly 2831 * of a clean array, like RAID1 does. 2832 */ 2833 if (mddev->bitmap == NULL && 2834 mddev->recovery_cp == MaxSector && 2835 mddev->reshape_position == MaxSector && 2836 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2837 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2838 !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 2839 conf->fullsync == 0) { 2840 *skipped = 1; 2841 return mddev->dev_sectors - sector_nr; 2842 } 2843 2844 skipped: 2845 max_sector = mddev->dev_sectors; 2846 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2847 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2848 max_sector = mddev->resync_max_sectors; 2849 if (sector_nr >= max_sector) { 2850 /* If we aborted, we need to abort the 2851 * sync on the 'current' bitmap chucks (there can 2852 * be several when recovering multiple devices). 2853 * as we may have started syncing it but not finished. 2854 * We can find the current address in 2855 * mddev->curr_resync, but for recovery, 2856 * we need to convert that to several 2857 * virtual addresses. 2858 */ 2859 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 2860 end_reshape(conf); 2861 close_sync(conf); 2862 return 0; 2863 } 2864 2865 if (mddev->curr_resync < max_sector) { /* aborted */ 2866 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 2867 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2868 &sync_blocks, 1); 2869 else for (i = 0; i < conf->geo.raid_disks; i++) { 2870 sector_t sect = 2871 raid10_find_virt(conf, mddev->curr_resync, i); 2872 bitmap_end_sync(mddev->bitmap, sect, 2873 &sync_blocks, 1); 2874 } 2875 } else { 2876 /* completed sync */ 2877 if ((!mddev->bitmap || conf->fullsync) 2878 && conf->have_replacement 2879 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2880 /* Completed a full sync so the replacements 2881 * are now fully recovered. 2882 */ 2883 rcu_read_lock(); 2884 for (i = 0; i < conf->geo.raid_disks; i++) { 2885 struct md_rdev *rdev = 2886 rcu_dereference(conf->mirrors[i].replacement); 2887 if (rdev) 2888 rdev->recovery_offset = MaxSector; 2889 } 2890 rcu_read_unlock(); 2891 } 2892 conf->fullsync = 0; 2893 } 2894 bitmap_close_sync(mddev->bitmap); 2895 close_sync(conf); 2896 *skipped = 1; 2897 return sectors_skipped; 2898 } 2899 2900 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2901 return reshape_request(mddev, sector_nr, skipped); 2902 2903 if (chunks_skipped >= conf->geo.raid_disks) { 2904 /* if there has been nothing to do on any drive, 2905 * then there is nothing to do at all.. 2906 */ 2907 *skipped = 1; 2908 return (max_sector - sector_nr) + sectors_skipped; 2909 } 2910 2911 if (max_sector > mddev->resync_max) 2912 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2913 2914 /* make sure whole request will fit in a chunk - if chunks 2915 * are meaningful 2916 */ 2917 if (conf->geo.near_copies < conf->geo.raid_disks && 2918 max_sector > (sector_nr | chunk_mask)) 2919 max_sector = (sector_nr | chunk_mask) + 1; 2920 2921 /* 2922 * If there is non-resync activity waiting for a turn, then let it 2923 * though before starting on this new sync request. 2924 */ 2925 if (conf->nr_waiting) 2926 schedule_timeout_uninterruptible(1); 2927 2928 /* Again, very different code for resync and recovery. 2929 * Both must result in an r10bio with a list of bios that 2930 * have bi_end_io, bi_sector, bi_bdev set, 2931 * and bi_private set to the r10bio. 2932 * For recovery, we may actually create several r10bios 2933 * with 2 bios in each, that correspond to the bios in the main one. 2934 * In this case, the subordinate r10bios link back through a 2935 * borrowed master_bio pointer, and the counter in the master 2936 * includes a ref from each subordinate. 2937 */ 2938 /* First, we decide what to do and set ->bi_end_io 2939 * To end_sync_read if we want to read, and 2940 * end_sync_write if we will want to write. 2941 */ 2942 2943 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 2944 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2945 /* recovery... the complicated one */ 2946 int j; 2947 r10_bio = NULL; 2948 2949 for (i = 0 ; i < conf->geo.raid_disks; i++) { 2950 int still_degraded; 2951 struct r10bio *rb2; 2952 sector_t sect; 2953 int must_sync; 2954 int any_working; 2955 struct raid10_info *mirror = &conf->mirrors[i]; 2956 struct md_rdev *mrdev, *mreplace; 2957 2958 rcu_read_lock(); 2959 mrdev = rcu_dereference(mirror->rdev); 2960 mreplace = rcu_dereference(mirror->replacement); 2961 2962 if ((mrdev == NULL || 2963 test_bit(Faulty, &mrdev->flags) || 2964 test_bit(In_sync, &mrdev->flags)) && 2965 (mreplace == NULL || 2966 test_bit(Faulty, &mreplace->flags))) { 2967 rcu_read_unlock(); 2968 continue; 2969 } 2970 2971 still_degraded = 0; 2972 /* want to reconstruct this device */ 2973 rb2 = r10_bio; 2974 sect = raid10_find_virt(conf, sector_nr, i); 2975 if (sect >= mddev->resync_max_sectors) { 2976 /* last stripe is not complete - don't 2977 * try to recover this sector. 2978 */ 2979 rcu_read_unlock(); 2980 continue; 2981 } 2982 if (mreplace && test_bit(Faulty, &mreplace->flags)) 2983 mreplace = NULL; 2984 /* Unless we are doing a full sync, or a replacement 2985 * we only need to recover the block if it is set in 2986 * the bitmap 2987 */ 2988 must_sync = bitmap_start_sync(mddev->bitmap, sect, 2989 &sync_blocks, 1); 2990 if (sync_blocks < max_sync) 2991 max_sync = sync_blocks; 2992 if (!must_sync && 2993 mreplace == NULL && 2994 !conf->fullsync) { 2995 /* yep, skip the sync_blocks here, but don't assume 2996 * that there will never be anything to do here 2997 */ 2998 chunks_skipped = -1; 2999 rcu_read_unlock(); 3000 continue; 3001 } 3002 atomic_inc(&mrdev->nr_pending); 3003 if (mreplace) 3004 atomic_inc(&mreplace->nr_pending); 3005 rcu_read_unlock(); 3006 3007 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3008 r10_bio->state = 0; 3009 raise_barrier(conf, rb2 != NULL); 3010 atomic_set(&r10_bio->remaining, 0); 3011 3012 r10_bio->master_bio = (struct bio*)rb2; 3013 if (rb2) 3014 atomic_inc(&rb2->remaining); 3015 r10_bio->mddev = mddev; 3016 set_bit(R10BIO_IsRecover, &r10_bio->state); 3017 r10_bio->sector = sect; 3018 3019 raid10_find_phys(conf, r10_bio); 3020 3021 /* Need to check if the array will still be 3022 * degraded 3023 */ 3024 rcu_read_lock(); 3025 for (j = 0; j < conf->geo.raid_disks; j++) { 3026 struct md_rdev *rdev = rcu_dereference( 3027 conf->mirrors[j].rdev); 3028 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3029 still_degraded = 1; 3030 break; 3031 } 3032 } 3033 3034 must_sync = bitmap_start_sync(mddev->bitmap, sect, 3035 &sync_blocks, still_degraded); 3036 3037 any_working = 0; 3038 for (j=0; j<conf->copies;j++) { 3039 int k; 3040 int d = r10_bio->devs[j].devnum; 3041 sector_t from_addr, to_addr; 3042 struct md_rdev *rdev = 3043 rcu_dereference(conf->mirrors[d].rdev); 3044 sector_t sector, first_bad; 3045 int bad_sectors; 3046 if (!rdev || 3047 !test_bit(In_sync, &rdev->flags)) 3048 continue; 3049 /* This is where we read from */ 3050 any_working = 1; 3051 sector = r10_bio->devs[j].addr; 3052 3053 if (is_badblock(rdev, sector, max_sync, 3054 &first_bad, &bad_sectors)) { 3055 if (first_bad > sector) 3056 max_sync = first_bad - sector; 3057 else { 3058 bad_sectors -= (sector 3059 - first_bad); 3060 if (max_sync > bad_sectors) 3061 max_sync = bad_sectors; 3062 continue; 3063 } 3064 } 3065 bio = r10_bio->devs[0].bio; 3066 bio_reset(bio); 3067 bio->bi_next = biolist; 3068 biolist = bio; 3069 bio->bi_private = r10_bio; 3070 bio->bi_end_io = end_sync_read; 3071 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3072 from_addr = r10_bio->devs[j].addr; 3073 bio->bi_iter.bi_sector = from_addr + 3074 rdev->data_offset; 3075 bio->bi_bdev = rdev->bdev; 3076 atomic_inc(&rdev->nr_pending); 3077 /* and we write to 'i' (if not in_sync) */ 3078 3079 for (k=0; k<conf->copies; k++) 3080 if (r10_bio->devs[k].devnum == i) 3081 break; 3082 BUG_ON(k == conf->copies); 3083 to_addr = r10_bio->devs[k].addr; 3084 r10_bio->devs[0].devnum = d; 3085 r10_bio->devs[0].addr = from_addr; 3086 r10_bio->devs[1].devnum = i; 3087 r10_bio->devs[1].addr = to_addr; 3088 3089 if (!test_bit(In_sync, &mrdev->flags)) { 3090 bio = r10_bio->devs[1].bio; 3091 bio_reset(bio); 3092 bio->bi_next = biolist; 3093 biolist = bio; 3094 bio->bi_private = r10_bio; 3095 bio->bi_end_io = end_sync_write; 3096 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3097 bio->bi_iter.bi_sector = to_addr 3098 + mrdev->data_offset; 3099 bio->bi_bdev = mrdev->bdev; 3100 atomic_inc(&r10_bio->remaining); 3101 } else 3102 r10_bio->devs[1].bio->bi_end_io = NULL; 3103 3104 /* and maybe write to replacement */ 3105 bio = r10_bio->devs[1].repl_bio; 3106 if (bio) 3107 bio->bi_end_io = NULL; 3108 /* Note: if mreplace != NULL, then bio 3109 * cannot be NULL as r10buf_pool_alloc will 3110 * have allocated it. 3111 * So the second test here is pointless. 3112 * But it keeps semantic-checkers happy, and 3113 * this comment keeps human reviewers 3114 * happy. 3115 */ 3116 if (mreplace == NULL || bio == NULL || 3117 test_bit(Faulty, &mreplace->flags)) 3118 break; 3119 bio_reset(bio); 3120 bio->bi_next = biolist; 3121 biolist = bio; 3122 bio->bi_private = r10_bio; 3123 bio->bi_end_io = end_sync_write; 3124 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3125 bio->bi_iter.bi_sector = to_addr + 3126 mreplace->data_offset; 3127 bio->bi_bdev = mreplace->bdev; 3128 atomic_inc(&r10_bio->remaining); 3129 break; 3130 } 3131 rcu_read_unlock(); 3132 if (j == conf->copies) { 3133 /* Cannot recover, so abort the recovery or 3134 * record a bad block */ 3135 if (any_working) { 3136 /* problem is that there are bad blocks 3137 * on other device(s) 3138 */ 3139 int k; 3140 for (k = 0; k < conf->copies; k++) 3141 if (r10_bio->devs[k].devnum == i) 3142 break; 3143 if (!test_bit(In_sync, 3144 &mrdev->flags) 3145 && !rdev_set_badblocks( 3146 mrdev, 3147 r10_bio->devs[k].addr, 3148 max_sync, 0)) 3149 any_working = 0; 3150 if (mreplace && 3151 !rdev_set_badblocks( 3152 mreplace, 3153 r10_bio->devs[k].addr, 3154 max_sync, 0)) 3155 any_working = 0; 3156 } 3157 if (!any_working) { 3158 if (!test_and_set_bit(MD_RECOVERY_INTR, 3159 &mddev->recovery)) 3160 printk(KERN_INFO "md/raid10:%s: insufficient " 3161 "working devices for recovery.\n", 3162 mdname(mddev)); 3163 mirror->recovery_disabled 3164 = mddev->recovery_disabled; 3165 } 3166 put_buf(r10_bio); 3167 if (rb2) 3168 atomic_dec(&rb2->remaining); 3169 r10_bio = rb2; 3170 rdev_dec_pending(mrdev, mddev); 3171 if (mreplace) 3172 rdev_dec_pending(mreplace, mddev); 3173 break; 3174 } 3175 rdev_dec_pending(mrdev, mddev); 3176 if (mreplace) 3177 rdev_dec_pending(mreplace, mddev); 3178 } 3179 if (biolist == NULL) { 3180 while (r10_bio) { 3181 struct r10bio *rb2 = r10_bio; 3182 r10_bio = (struct r10bio*) rb2->master_bio; 3183 rb2->master_bio = NULL; 3184 put_buf(rb2); 3185 } 3186 goto giveup; 3187 } 3188 } else { 3189 /* resync. Schedule a read for every block at this virt offset */ 3190 int count = 0; 3191 3192 bitmap_cond_end_sync(mddev->bitmap, sector_nr, 0); 3193 3194 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 3195 &sync_blocks, mddev->degraded) && 3196 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3197 &mddev->recovery)) { 3198 /* We can skip this block */ 3199 *skipped = 1; 3200 return sync_blocks + sectors_skipped; 3201 } 3202 if (sync_blocks < max_sync) 3203 max_sync = sync_blocks; 3204 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3205 r10_bio->state = 0; 3206 3207 r10_bio->mddev = mddev; 3208 atomic_set(&r10_bio->remaining, 0); 3209 raise_barrier(conf, 0); 3210 conf->next_resync = sector_nr; 3211 3212 r10_bio->master_bio = NULL; 3213 r10_bio->sector = sector_nr; 3214 set_bit(R10BIO_IsSync, &r10_bio->state); 3215 raid10_find_phys(conf, r10_bio); 3216 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3217 3218 for (i = 0; i < conf->copies; i++) { 3219 int d = r10_bio->devs[i].devnum; 3220 sector_t first_bad, sector; 3221 int bad_sectors; 3222 struct md_rdev *rdev; 3223 3224 if (r10_bio->devs[i].repl_bio) 3225 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3226 3227 bio = r10_bio->devs[i].bio; 3228 bio_reset(bio); 3229 bio->bi_error = -EIO; 3230 rcu_read_lock(); 3231 rdev = rcu_dereference(conf->mirrors[d].rdev); 3232 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3233 rcu_read_unlock(); 3234 continue; 3235 } 3236 sector = r10_bio->devs[i].addr; 3237 if (is_badblock(rdev, sector, max_sync, 3238 &first_bad, &bad_sectors)) { 3239 if (first_bad > sector) 3240 max_sync = first_bad - sector; 3241 else { 3242 bad_sectors -= (sector - first_bad); 3243 if (max_sync > bad_sectors) 3244 max_sync = bad_sectors; 3245 rcu_read_unlock(); 3246 continue; 3247 } 3248 } 3249 atomic_inc(&rdev->nr_pending); 3250 atomic_inc(&r10_bio->remaining); 3251 bio->bi_next = biolist; 3252 biolist = bio; 3253 bio->bi_private = r10_bio; 3254 bio->bi_end_io = end_sync_read; 3255 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3256 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3257 bio->bi_bdev = rdev->bdev; 3258 count++; 3259 3260 rdev = rcu_dereference(conf->mirrors[d].replacement); 3261 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3262 rcu_read_unlock(); 3263 continue; 3264 } 3265 atomic_inc(&rdev->nr_pending); 3266 rcu_read_unlock(); 3267 3268 /* Need to set up for writing to the replacement */ 3269 bio = r10_bio->devs[i].repl_bio; 3270 bio_reset(bio); 3271 bio->bi_error = -EIO; 3272 3273 sector = r10_bio->devs[i].addr; 3274 bio->bi_next = biolist; 3275 biolist = bio; 3276 bio->bi_private = r10_bio; 3277 bio->bi_end_io = end_sync_write; 3278 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3279 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3280 bio->bi_bdev = rdev->bdev; 3281 count++; 3282 } 3283 3284 if (count < 2) { 3285 for (i=0; i<conf->copies; i++) { 3286 int d = r10_bio->devs[i].devnum; 3287 if (r10_bio->devs[i].bio->bi_end_io) 3288 rdev_dec_pending(conf->mirrors[d].rdev, 3289 mddev); 3290 if (r10_bio->devs[i].repl_bio && 3291 r10_bio->devs[i].repl_bio->bi_end_io) 3292 rdev_dec_pending( 3293 conf->mirrors[d].replacement, 3294 mddev); 3295 } 3296 put_buf(r10_bio); 3297 biolist = NULL; 3298 goto giveup; 3299 } 3300 } 3301 3302 nr_sectors = 0; 3303 if (sector_nr + max_sync < max_sector) 3304 max_sector = sector_nr + max_sync; 3305 do { 3306 struct page *page; 3307 int len = PAGE_SIZE; 3308 if (sector_nr + (len>>9) > max_sector) 3309 len = (max_sector - sector_nr) << 9; 3310 if (len == 0) 3311 break; 3312 for (bio= biolist ; bio ; bio=bio->bi_next) { 3313 struct bio *bio2; 3314 page = bio->bi_io_vec[bio->bi_vcnt].bv_page; 3315 if (bio_add_page(bio, page, len, 0)) 3316 continue; 3317 3318 /* stop here */ 3319 bio->bi_io_vec[bio->bi_vcnt].bv_page = page; 3320 for (bio2 = biolist; 3321 bio2 && bio2 != bio; 3322 bio2 = bio2->bi_next) { 3323 /* remove last page from this bio */ 3324 bio2->bi_vcnt--; 3325 bio2->bi_iter.bi_size -= len; 3326 bio_clear_flag(bio2, BIO_SEG_VALID); 3327 } 3328 goto bio_full; 3329 } 3330 nr_sectors += len>>9; 3331 sector_nr += len>>9; 3332 } while (biolist->bi_vcnt < RESYNC_PAGES); 3333 bio_full: 3334 r10_bio->sectors = nr_sectors; 3335 3336 while (biolist) { 3337 bio = biolist; 3338 biolist = biolist->bi_next; 3339 3340 bio->bi_next = NULL; 3341 r10_bio = bio->bi_private; 3342 r10_bio->sectors = nr_sectors; 3343 3344 if (bio->bi_end_io == end_sync_read) { 3345 md_sync_acct(bio->bi_bdev, nr_sectors); 3346 bio->bi_error = 0; 3347 generic_make_request(bio); 3348 } 3349 } 3350 3351 if (sectors_skipped) 3352 /* pretend they weren't skipped, it makes 3353 * no important difference in this case 3354 */ 3355 md_done_sync(mddev, sectors_skipped, 1); 3356 3357 return sectors_skipped + nr_sectors; 3358 giveup: 3359 /* There is nowhere to write, so all non-sync 3360 * drives must be failed or in resync, all drives 3361 * have a bad block, so try the next chunk... 3362 */ 3363 if (sector_nr + max_sync < max_sector) 3364 max_sector = sector_nr + max_sync; 3365 3366 sectors_skipped += (max_sector - sector_nr); 3367 chunks_skipped ++; 3368 sector_nr = max_sector; 3369 goto skipped; 3370 } 3371 3372 static sector_t 3373 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3374 { 3375 sector_t size; 3376 struct r10conf *conf = mddev->private; 3377 3378 if (!raid_disks) 3379 raid_disks = min(conf->geo.raid_disks, 3380 conf->prev.raid_disks); 3381 if (!sectors) 3382 sectors = conf->dev_sectors; 3383 3384 size = sectors >> conf->geo.chunk_shift; 3385 sector_div(size, conf->geo.far_copies); 3386 size = size * raid_disks; 3387 sector_div(size, conf->geo.near_copies); 3388 3389 return size << conf->geo.chunk_shift; 3390 } 3391 3392 static void calc_sectors(struct r10conf *conf, sector_t size) 3393 { 3394 /* Calculate the number of sectors-per-device that will 3395 * actually be used, and set conf->dev_sectors and 3396 * conf->stride 3397 */ 3398 3399 size = size >> conf->geo.chunk_shift; 3400 sector_div(size, conf->geo.far_copies); 3401 size = size * conf->geo.raid_disks; 3402 sector_div(size, conf->geo.near_copies); 3403 /* 'size' is now the number of chunks in the array */ 3404 /* calculate "used chunks per device" */ 3405 size = size * conf->copies; 3406 3407 /* We need to round up when dividing by raid_disks to 3408 * get the stride size. 3409 */ 3410 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3411 3412 conf->dev_sectors = size << conf->geo.chunk_shift; 3413 3414 if (conf->geo.far_offset) 3415 conf->geo.stride = 1 << conf->geo.chunk_shift; 3416 else { 3417 sector_div(size, conf->geo.far_copies); 3418 conf->geo.stride = size << conf->geo.chunk_shift; 3419 } 3420 } 3421 3422 enum geo_type {geo_new, geo_old, geo_start}; 3423 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3424 { 3425 int nc, fc, fo; 3426 int layout, chunk, disks; 3427 switch (new) { 3428 case geo_old: 3429 layout = mddev->layout; 3430 chunk = mddev->chunk_sectors; 3431 disks = mddev->raid_disks - mddev->delta_disks; 3432 break; 3433 case geo_new: 3434 layout = mddev->new_layout; 3435 chunk = mddev->new_chunk_sectors; 3436 disks = mddev->raid_disks; 3437 break; 3438 default: /* avoid 'may be unused' warnings */ 3439 case geo_start: /* new when starting reshape - raid_disks not 3440 * updated yet. */ 3441 layout = mddev->new_layout; 3442 chunk = mddev->new_chunk_sectors; 3443 disks = mddev->raid_disks + mddev->delta_disks; 3444 break; 3445 } 3446 if (layout >> 19) 3447 return -1; 3448 if (chunk < (PAGE_SIZE >> 9) || 3449 !is_power_of_2(chunk)) 3450 return -2; 3451 nc = layout & 255; 3452 fc = (layout >> 8) & 255; 3453 fo = layout & (1<<16); 3454 geo->raid_disks = disks; 3455 geo->near_copies = nc; 3456 geo->far_copies = fc; 3457 geo->far_offset = fo; 3458 switch (layout >> 17) { 3459 case 0: /* original layout. simple but not always optimal */ 3460 geo->far_set_size = disks; 3461 break; 3462 case 1: /* "improved" layout which was buggy. Hopefully no-one is 3463 * actually using this, but leave code here just in case.*/ 3464 geo->far_set_size = disks/fc; 3465 WARN(geo->far_set_size < fc, 3466 "This RAID10 layout does not provide data safety - please backup and create new array\n"); 3467 break; 3468 case 2: /* "improved" layout fixed to match documentation */ 3469 geo->far_set_size = fc * nc; 3470 break; 3471 default: /* Not a valid layout */ 3472 return -1; 3473 } 3474 geo->chunk_mask = chunk - 1; 3475 geo->chunk_shift = ffz(~chunk); 3476 return nc*fc; 3477 } 3478 3479 static struct r10conf *setup_conf(struct mddev *mddev) 3480 { 3481 struct r10conf *conf = NULL; 3482 int err = -EINVAL; 3483 struct geom geo; 3484 int copies; 3485 3486 copies = setup_geo(&geo, mddev, geo_new); 3487 3488 if (copies == -2) { 3489 printk(KERN_ERR "md/raid10:%s: chunk size must be " 3490 "at least PAGE_SIZE(%ld) and be a power of 2.\n", 3491 mdname(mddev), PAGE_SIZE); 3492 goto out; 3493 } 3494 3495 if (copies < 2 || copies > mddev->raid_disks) { 3496 printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3497 mdname(mddev), mddev->new_layout); 3498 goto out; 3499 } 3500 3501 err = -ENOMEM; 3502 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3503 if (!conf) 3504 goto out; 3505 3506 /* FIXME calc properly */ 3507 conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks + 3508 max(0,-mddev->delta_disks)), 3509 GFP_KERNEL); 3510 if (!conf->mirrors) 3511 goto out; 3512 3513 conf->tmppage = alloc_page(GFP_KERNEL); 3514 if (!conf->tmppage) 3515 goto out; 3516 3517 conf->geo = geo; 3518 conf->copies = copies; 3519 conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc, 3520 r10bio_pool_free, conf); 3521 if (!conf->r10bio_pool) 3522 goto out; 3523 3524 calc_sectors(conf, mddev->dev_sectors); 3525 if (mddev->reshape_position == MaxSector) { 3526 conf->prev = conf->geo; 3527 conf->reshape_progress = MaxSector; 3528 } else { 3529 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3530 err = -EINVAL; 3531 goto out; 3532 } 3533 conf->reshape_progress = mddev->reshape_position; 3534 if (conf->prev.far_offset) 3535 conf->prev.stride = 1 << conf->prev.chunk_shift; 3536 else 3537 /* far_copies must be 1 */ 3538 conf->prev.stride = conf->dev_sectors; 3539 } 3540 conf->reshape_safe = conf->reshape_progress; 3541 spin_lock_init(&conf->device_lock); 3542 INIT_LIST_HEAD(&conf->retry_list); 3543 INIT_LIST_HEAD(&conf->bio_end_io_list); 3544 3545 spin_lock_init(&conf->resync_lock); 3546 init_waitqueue_head(&conf->wait_barrier); 3547 atomic_set(&conf->nr_pending, 0); 3548 3549 conf->thread = md_register_thread(raid10d, mddev, "raid10"); 3550 if (!conf->thread) 3551 goto out; 3552 3553 conf->mddev = mddev; 3554 return conf; 3555 3556 out: 3557 if (err == -ENOMEM) 3558 printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n", 3559 mdname(mddev)); 3560 if (conf) { 3561 mempool_destroy(conf->r10bio_pool); 3562 kfree(conf->mirrors); 3563 safe_put_page(conf->tmppage); 3564 kfree(conf); 3565 } 3566 return ERR_PTR(err); 3567 } 3568 3569 static int raid10_run(struct mddev *mddev) 3570 { 3571 struct r10conf *conf; 3572 int i, disk_idx, chunk_size; 3573 struct raid10_info *disk; 3574 struct md_rdev *rdev; 3575 sector_t size; 3576 sector_t min_offset_diff = 0; 3577 int first = 1; 3578 bool discard_supported = false; 3579 3580 if (mddev->private == NULL) { 3581 conf = setup_conf(mddev); 3582 if (IS_ERR(conf)) 3583 return PTR_ERR(conf); 3584 mddev->private = conf; 3585 } 3586 conf = mddev->private; 3587 if (!conf) 3588 goto out; 3589 3590 mddev->thread = conf->thread; 3591 conf->thread = NULL; 3592 3593 chunk_size = mddev->chunk_sectors << 9; 3594 if (mddev->queue) { 3595 blk_queue_max_discard_sectors(mddev->queue, 3596 mddev->chunk_sectors); 3597 blk_queue_max_write_same_sectors(mddev->queue, 0); 3598 blk_queue_io_min(mddev->queue, chunk_size); 3599 if (conf->geo.raid_disks % conf->geo.near_copies) 3600 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks); 3601 else 3602 blk_queue_io_opt(mddev->queue, chunk_size * 3603 (conf->geo.raid_disks / conf->geo.near_copies)); 3604 } 3605 3606 rdev_for_each(rdev, mddev) { 3607 long long diff; 3608 struct request_queue *q; 3609 3610 disk_idx = rdev->raid_disk; 3611 if (disk_idx < 0) 3612 continue; 3613 if (disk_idx >= conf->geo.raid_disks && 3614 disk_idx >= conf->prev.raid_disks) 3615 continue; 3616 disk = conf->mirrors + disk_idx; 3617 3618 if (test_bit(Replacement, &rdev->flags)) { 3619 if (disk->replacement) 3620 goto out_free_conf; 3621 disk->replacement = rdev; 3622 } else { 3623 if (disk->rdev) 3624 goto out_free_conf; 3625 disk->rdev = rdev; 3626 } 3627 q = bdev_get_queue(rdev->bdev); 3628 diff = (rdev->new_data_offset - rdev->data_offset); 3629 if (!mddev->reshape_backwards) 3630 diff = -diff; 3631 if (diff < 0) 3632 diff = 0; 3633 if (first || diff < min_offset_diff) 3634 min_offset_diff = diff; 3635 3636 if (mddev->gendisk) 3637 disk_stack_limits(mddev->gendisk, rdev->bdev, 3638 rdev->data_offset << 9); 3639 3640 disk->head_position = 0; 3641 3642 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3643 discard_supported = true; 3644 } 3645 3646 if (mddev->queue) { 3647 if (discard_supported) 3648 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 3649 mddev->queue); 3650 else 3651 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 3652 mddev->queue); 3653 } 3654 /* need to check that every block has at least one working mirror */ 3655 if (!enough(conf, -1)) { 3656 printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n", 3657 mdname(mddev)); 3658 goto out_free_conf; 3659 } 3660 3661 if (conf->reshape_progress != MaxSector) { 3662 /* must ensure that shape change is supported */ 3663 if (conf->geo.far_copies != 1 && 3664 conf->geo.far_offset == 0) 3665 goto out_free_conf; 3666 if (conf->prev.far_copies != 1 && 3667 conf->prev.far_offset == 0) 3668 goto out_free_conf; 3669 } 3670 3671 mddev->degraded = 0; 3672 for (i = 0; 3673 i < conf->geo.raid_disks 3674 || i < conf->prev.raid_disks; 3675 i++) { 3676 3677 disk = conf->mirrors + i; 3678 3679 if (!disk->rdev && disk->replacement) { 3680 /* The replacement is all we have - use it */ 3681 disk->rdev = disk->replacement; 3682 disk->replacement = NULL; 3683 clear_bit(Replacement, &disk->rdev->flags); 3684 } 3685 3686 if (!disk->rdev || 3687 !test_bit(In_sync, &disk->rdev->flags)) { 3688 disk->head_position = 0; 3689 mddev->degraded++; 3690 if (disk->rdev && 3691 disk->rdev->saved_raid_disk < 0) 3692 conf->fullsync = 1; 3693 } 3694 disk->recovery_disabled = mddev->recovery_disabled - 1; 3695 } 3696 3697 if (mddev->recovery_cp != MaxSector) 3698 printk(KERN_NOTICE "md/raid10:%s: not clean" 3699 " -- starting background reconstruction\n", 3700 mdname(mddev)); 3701 printk(KERN_INFO 3702 "md/raid10:%s: active with %d out of %d devices\n", 3703 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 3704 conf->geo.raid_disks); 3705 /* 3706 * Ok, everything is just fine now 3707 */ 3708 mddev->dev_sectors = conf->dev_sectors; 3709 size = raid10_size(mddev, 0, 0); 3710 md_set_array_sectors(mddev, size); 3711 mddev->resync_max_sectors = size; 3712 3713 if (mddev->queue) { 3714 int stripe = conf->geo.raid_disks * 3715 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 3716 3717 /* Calculate max read-ahead size. 3718 * We need to readahead at least twice a whole stripe.... 3719 * maybe... 3720 */ 3721 stripe /= conf->geo.near_copies; 3722 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 3723 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 3724 } 3725 3726 if (md_integrity_register(mddev)) 3727 goto out_free_conf; 3728 3729 if (conf->reshape_progress != MaxSector) { 3730 unsigned long before_length, after_length; 3731 3732 before_length = ((1 << conf->prev.chunk_shift) * 3733 conf->prev.far_copies); 3734 after_length = ((1 << conf->geo.chunk_shift) * 3735 conf->geo.far_copies); 3736 3737 if (max(before_length, after_length) > min_offset_diff) { 3738 /* This cannot work */ 3739 printk("md/raid10: offset difference not enough to continue reshape\n"); 3740 goto out_free_conf; 3741 } 3742 conf->offset_diff = min_offset_diff; 3743 3744 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 3745 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 3746 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 3747 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 3748 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 3749 "reshape"); 3750 } 3751 3752 return 0; 3753 3754 out_free_conf: 3755 md_unregister_thread(&mddev->thread); 3756 mempool_destroy(conf->r10bio_pool); 3757 safe_put_page(conf->tmppage); 3758 kfree(conf->mirrors); 3759 kfree(conf); 3760 mddev->private = NULL; 3761 out: 3762 return -EIO; 3763 } 3764 3765 static void raid10_free(struct mddev *mddev, void *priv) 3766 { 3767 struct r10conf *conf = priv; 3768 3769 mempool_destroy(conf->r10bio_pool); 3770 safe_put_page(conf->tmppage); 3771 kfree(conf->mirrors); 3772 kfree(conf->mirrors_old); 3773 kfree(conf->mirrors_new); 3774 kfree(conf); 3775 } 3776 3777 static void raid10_quiesce(struct mddev *mddev, int state) 3778 { 3779 struct r10conf *conf = mddev->private; 3780 3781 switch(state) { 3782 case 1: 3783 raise_barrier(conf, 0); 3784 break; 3785 case 0: 3786 lower_barrier(conf); 3787 break; 3788 } 3789 } 3790 3791 static int raid10_resize(struct mddev *mddev, sector_t sectors) 3792 { 3793 /* Resize of 'far' arrays is not supported. 3794 * For 'near' and 'offset' arrays we can set the 3795 * number of sectors used to be an appropriate multiple 3796 * of the chunk size. 3797 * For 'offset', this is far_copies*chunksize. 3798 * For 'near' the multiplier is the LCM of 3799 * near_copies and raid_disks. 3800 * So if far_copies > 1 && !far_offset, fail. 3801 * Else find LCM(raid_disks, near_copy)*far_copies and 3802 * multiply by chunk_size. Then round to this number. 3803 * This is mostly done by raid10_size() 3804 */ 3805 struct r10conf *conf = mddev->private; 3806 sector_t oldsize, size; 3807 3808 if (mddev->reshape_position != MaxSector) 3809 return -EBUSY; 3810 3811 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 3812 return -EINVAL; 3813 3814 oldsize = raid10_size(mddev, 0, 0); 3815 size = raid10_size(mddev, sectors, 0); 3816 if (mddev->external_size && 3817 mddev->array_sectors > size) 3818 return -EINVAL; 3819 if (mddev->bitmap) { 3820 int ret = bitmap_resize(mddev->bitmap, size, 0, 0); 3821 if (ret) 3822 return ret; 3823 } 3824 md_set_array_sectors(mddev, size); 3825 if (mddev->queue) { 3826 set_capacity(mddev->gendisk, mddev->array_sectors); 3827 revalidate_disk(mddev->gendisk); 3828 } 3829 if (sectors > mddev->dev_sectors && 3830 mddev->recovery_cp > oldsize) { 3831 mddev->recovery_cp = oldsize; 3832 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3833 } 3834 calc_sectors(conf, sectors); 3835 mddev->dev_sectors = conf->dev_sectors; 3836 mddev->resync_max_sectors = size; 3837 return 0; 3838 } 3839 3840 static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) 3841 { 3842 struct md_rdev *rdev; 3843 struct r10conf *conf; 3844 3845 if (mddev->degraded > 0) { 3846 printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n", 3847 mdname(mddev)); 3848 return ERR_PTR(-EINVAL); 3849 } 3850 sector_div(size, devs); 3851 3852 /* Set new parameters */ 3853 mddev->new_level = 10; 3854 /* new layout: far_copies = 1, near_copies = 2 */ 3855 mddev->new_layout = (1<<8) + 2; 3856 mddev->new_chunk_sectors = mddev->chunk_sectors; 3857 mddev->delta_disks = mddev->raid_disks; 3858 mddev->raid_disks *= 2; 3859 /* make sure it will be not marked as dirty */ 3860 mddev->recovery_cp = MaxSector; 3861 mddev->dev_sectors = size; 3862 3863 conf = setup_conf(mddev); 3864 if (!IS_ERR(conf)) { 3865 rdev_for_each(rdev, mddev) 3866 if (rdev->raid_disk >= 0) { 3867 rdev->new_raid_disk = rdev->raid_disk * 2; 3868 rdev->sectors = size; 3869 } 3870 conf->barrier = 1; 3871 } 3872 3873 return conf; 3874 } 3875 3876 static void *raid10_takeover(struct mddev *mddev) 3877 { 3878 struct r0conf *raid0_conf; 3879 3880 /* raid10 can take over: 3881 * raid0 - providing it has only two drives 3882 */ 3883 if (mddev->level == 0) { 3884 /* for raid0 takeover only one zone is supported */ 3885 raid0_conf = mddev->private; 3886 if (raid0_conf->nr_strip_zones > 1) { 3887 printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0" 3888 " with more than one zone.\n", 3889 mdname(mddev)); 3890 return ERR_PTR(-EINVAL); 3891 } 3892 return raid10_takeover_raid0(mddev, 3893 raid0_conf->strip_zone->zone_end, 3894 raid0_conf->strip_zone->nb_dev); 3895 } 3896 return ERR_PTR(-EINVAL); 3897 } 3898 3899 static int raid10_check_reshape(struct mddev *mddev) 3900 { 3901 /* Called when there is a request to change 3902 * - layout (to ->new_layout) 3903 * - chunk size (to ->new_chunk_sectors) 3904 * - raid_disks (by delta_disks) 3905 * or when trying to restart a reshape that was ongoing. 3906 * 3907 * We need to validate the request and possibly allocate 3908 * space if that might be an issue later. 3909 * 3910 * Currently we reject any reshape of a 'far' mode array, 3911 * allow chunk size to change if new is generally acceptable, 3912 * allow raid_disks to increase, and allow 3913 * a switch between 'near' mode and 'offset' mode. 3914 */ 3915 struct r10conf *conf = mddev->private; 3916 struct geom geo; 3917 3918 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 3919 return -EINVAL; 3920 3921 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 3922 /* mustn't change number of copies */ 3923 return -EINVAL; 3924 if (geo.far_copies > 1 && !geo.far_offset) 3925 /* Cannot switch to 'far' mode */ 3926 return -EINVAL; 3927 3928 if (mddev->array_sectors & geo.chunk_mask) 3929 /* not factor of array size */ 3930 return -EINVAL; 3931 3932 if (!enough(conf, -1)) 3933 return -EINVAL; 3934 3935 kfree(conf->mirrors_new); 3936 conf->mirrors_new = NULL; 3937 if (mddev->delta_disks > 0) { 3938 /* allocate new 'mirrors' list */ 3939 conf->mirrors_new = kzalloc( 3940 sizeof(struct raid10_info) 3941 *(mddev->raid_disks + 3942 mddev->delta_disks), 3943 GFP_KERNEL); 3944 if (!conf->mirrors_new) 3945 return -ENOMEM; 3946 } 3947 return 0; 3948 } 3949 3950 /* 3951 * Need to check if array has failed when deciding whether to: 3952 * - start an array 3953 * - remove non-faulty devices 3954 * - add a spare 3955 * - allow a reshape 3956 * This determination is simple when no reshape is happening. 3957 * However if there is a reshape, we need to carefully check 3958 * both the before and after sections. 3959 * This is because some failed devices may only affect one 3960 * of the two sections, and some non-in_sync devices may 3961 * be insync in the section most affected by failed devices. 3962 */ 3963 static int calc_degraded(struct r10conf *conf) 3964 { 3965 int degraded, degraded2; 3966 int i; 3967 3968 rcu_read_lock(); 3969 degraded = 0; 3970 /* 'prev' section first */ 3971 for (i = 0; i < conf->prev.raid_disks; i++) { 3972 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 3973 if (!rdev || test_bit(Faulty, &rdev->flags)) 3974 degraded++; 3975 else if (!test_bit(In_sync, &rdev->flags)) 3976 /* When we can reduce the number of devices in 3977 * an array, this might not contribute to 3978 * 'degraded'. It does now. 3979 */ 3980 degraded++; 3981 } 3982 rcu_read_unlock(); 3983 if (conf->geo.raid_disks == conf->prev.raid_disks) 3984 return degraded; 3985 rcu_read_lock(); 3986 degraded2 = 0; 3987 for (i = 0; i < conf->geo.raid_disks; i++) { 3988 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 3989 if (!rdev || test_bit(Faulty, &rdev->flags)) 3990 degraded2++; 3991 else if (!test_bit(In_sync, &rdev->flags)) { 3992 /* If reshape is increasing the number of devices, 3993 * this section has already been recovered, so 3994 * it doesn't contribute to degraded. 3995 * else it does. 3996 */ 3997 if (conf->geo.raid_disks <= conf->prev.raid_disks) 3998 degraded2++; 3999 } 4000 } 4001 rcu_read_unlock(); 4002 if (degraded2 > degraded) 4003 return degraded2; 4004 return degraded; 4005 } 4006 4007 static int raid10_start_reshape(struct mddev *mddev) 4008 { 4009 /* A 'reshape' has been requested. This commits 4010 * the various 'new' fields and sets MD_RECOVER_RESHAPE 4011 * This also checks if there are enough spares and adds them 4012 * to the array. 4013 * We currently require enough spares to make the final 4014 * array non-degraded. We also require that the difference 4015 * between old and new data_offset - on each device - is 4016 * enough that we never risk over-writing. 4017 */ 4018 4019 unsigned long before_length, after_length; 4020 sector_t min_offset_diff = 0; 4021 int first = 1; 4022 struct geom new; 4023 struct r10conf *conf = mddev->private; 4024 struct md_rdev *rdev; 4025 int spares = 0; 4026 int ret; 4027 4028 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4029 return -EBUSY; 4030 4031 if (setup_geo(&new, mddev, geo_start) != conf->copies) 4032 return -EINVAL; 4033 4034 before_length = ((1 << conf->prev.chunk_shift) * 4035 conf->prev.far_copies); 4036 after_length = ((1 << conf->geo.chunk_shift) * 4037 conf->geo.far_copies); 4038 4039 rdev_for_each(rdev, mddev) { 4040 if (!test_bit(In_sync, &rdev->flags) 4041 && !test_bit(Faulty, &rdev->flags)) 4042 spares++; 4043 if (rdev->raid_disk >= 0) { 4044 long long diff = (rdev->new_data_offset 4045 - rdev->data_offset); 4046 if (!mddev->reshape_backwards) 4047 diff = -diff; 4048 if (diff < 0) 4049 diff = 0; 4050 if (first || diff < min_offset_diff) 4051 min_offset_diff = diff; 4052 } 4053 } 4054 4055 if (max(before_length, after_length) > min_offset_diff) 4056 return -EINVAL; 4057 4058 if (spares < mddev->delta_disks) 4059 return -EINVAL; 4060 4061 conf->offset_diff = min_offset_diff; 4062 spin_lock_irq(&conf->device_lock); 4063 if (conf->mirrors_new) { 4064 memcpy(conf->mirrors_new, conf->mirrors, 4065 sizeof(struct raid10_info)*conf->prev.raid_disks); 4066 smp_mb(); 4067 kfree(conf->mirrors_old); 4068 conf->mirrors_old = conf->mirrors; 4069 conf->mirrors = conf->mirrors_new; 4070 conf->mirrors_new = NULL; 4071 } 4072 setup_geo(&conf->geo, mddev, geo_start); 4073 smp_mb(); 4074 if (mddev->reshape_backwards) { 4075 sector_t size = raid10_size(mddev, 0, 0); 4076 if (size < mddev->array_sectors) { 4077 spin_unlock_irq(&conf->device_lock); 4078 printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n", 4079 mdname(mddev)); 4080 return -EINVAL; 4081 } 4082 mddev->resync_max_sectors = size; 4083 conf->reshape_progress = size; 4084 } else 4085 conf->reshape_progress = 0; 4086 conf->reshape_safe = conf->reshape_progress; 4087 spin_unlock_irq(&conf->device_lock); 4088 4089 if (mddev->delta_disks && mddev->bitmap) { 4090 ret = bitmap_resize(mddev->bitmap, 4091 raid10_size(mddev, 0, 4092 conf->geo.raid_disks), 4093 0, 0); 4094 if (ret) 4095 goto abort; 4096 } 4097 if (mddev->delta_disks > 0) { 4098 rdev_for_each(rdev, mddev) 4099 if (rdev->raid_disk < 0 && 4100 !test_bit(Faulty, &rdev->flags)) { 4101 if (raid10_add_disk(mddev, rdev) == 0) { 4102 if (rdev->raid_disk >= 4103 conf->prev.raid_disks) 4104 set_bit(In_sync, &rdev->flags); 4105 else 4106 rdev->recovery_offset = 0; 4107 4108 if (sysfs_link_rdev(mddev, rdev)) 4109 /* Failure here is OK */; 4110 } 4111 } else if (rdev->raid_disk >= conf->prev.raid_disks 4112 && !test_bit(Faulty, &rdev->flags)) { 4113 /* This is a spare that was manually added */ 4114 set_bit(In_sync, &rdev->flags); 4115 } 4116 } 4117 /* When a reshape changes the number of devices, 4118 * ->degraded is measured against the larger of the 4119 * pre and post numbers. 4120 */ 4121 spin_lock_irq(&conf->device_lock); 4122 mddev->degraded = calc_degraded(conf); 4123 spin_unlock_irq(&conf->device_lock); 4124 mddev->raid_disks = conf->geo.raid_disks; 4125 mddev->reshape_position = conf->reshape_progress; 4126 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4127 4128 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4129 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4130 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 4131 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4132 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4133 4134 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4135 "reshape"); 4136 if (!mddev->sync_thread) { 4137 ret = -EAGAIN; 4138 goto abort; 4139 } 4140 conf->reshape_checkpoint = jiffies; 4141 md_wakeup_thread(mddev->sync_thread); 4142 md_new_event(mddev); 4143 return 0; 4144 4145 abort: 4146 mddev->recovery = 0; 4147 spin_lock_irq(&conf->device_lock); 4148 conf->geo = conf->prev; 4149 mddev->raid_disks = conf->geo.raid_disks; 4150 rdev_for_each(rdev, mddev) 4151 rdev->new_data_offset = rdev->data_offset; 4152 smp_wmb(); 4153 conf->reshape_progress = MaxSector; 4154 conf->reshape_safe = MaxSector; 4155 mddev->reshape_position = MaxSector; 4156 spin_unlock_irq(&conf->device_lock); 4157 return ret; 4158 } 4159 4160 /* Calculate the last device-address that could contain 4161 * any block from the chunk that includes the array-address 's' 4162 * and report the next address. 4163 * i.e. the address returned will be chunk-aligned and after 4164 * any data that is in the chunk containing 's'. 4165 */ 4166 static sector_t last_dev_address(sector_t s, struct geom *geo) 4167 { 4168 s = (s | geo->chunk_mask) + 1; 4169 s >>= geo->chunk_shift; 4170 s *= geo->near_copies; 4171 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4172 s *= geo->far_copies; 4173 s <<= geo->chunk_shift; 4174 return s; 4175 } 4176 4177 /* Calculate the first device-address that could contain 4178 * any block from the chunk that includes the array-address 's'. 4179 * This too will be the start of a chunk 4180 */ 4181 static sector_t first_dev_address(sector_t s, struct geom *geo) 4182 { 4183 s >>= geo->chunk_shift; 4184 s *= geo->near_copies; 4185 sector_div(s, geo->raid_disks); 4186 s *= geo->far_copies; 4187 s <<= geo->chunk_shift; 4188 return s; 4189 } 4190 4191 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4192 int *skipped) 4193 { 4194 /* We simply copy at most one chunk (smallest of old and new) 4195 * at a time, possibly less if that exceeds RESYNC_PAGES, 4196 * or we hit a bad block or something. 4197 * This might mean we pause for normal IO in the middle of 4198 * a chunk, but that is not a problem as mddev->reshape_position 4199 * can record any location. 4200 * 4201 * If we will want to write to a location that isn't 4202 * yet recorded as 'safe' (i.e. in metadata on disk) then 4203 * we need to flush all reshape requests and update the metadata. 4204 * 4205 * When reshaping forwards (e.g. to more devices), we interpret 4206 * 'safe' as the earliest block which might not have been copied 4207 * down yet. We divide this by previous stripe size and multiply 4208 * by previous stripe length to get lowest device offset that we 4209 * cannot write to yet. 4210 * We interpret 'sector_nr' as an address that we want to write to. 4211 * From this we use last_device_address() to find where we might 4212 * write to, and first_device_address on the 'safe' position. 4213 * If this 'next' write position is after the 'safe' position, 4214 * we must update the metadata to increase the 'safe' position. 4215 * 4216 * When reshaping backwards, we round in the opposite direction 4217 * and perform the reverse test: next write position must not be 4218 * less than current safe position. 4219 * 4220 * In all this the minimum difference in data offsets 4221 * (conf->offset_diff - always positive) allows a bit of slack, 4222 * so next can be after 'safe', but not by more than offset_diff 4223 * 4224 * We need to prepare all the bios here before we start any IO 4225 * to ensure the size we choose is acceptable to all devices. 4226 * The means one for each copy for write-out and an extra one for 4227 * read-in. 4228 * We store the read-in bio in ->master_bio and the others in 4229 * ->devs[x].bio and ->devs[x].repl_bio. 4230 */ 4231 struct r10conf *conf = mddev->private; 4232 struct r10bio *r10_bio; 4233 sector_t next, safe, last; 4234 int max_sectors; 4235 int nr_sectors; 4236 int s; 4237 struct md_rdev *rdev; 4238 int need_flush = 0; 4239 struct bio *blist; 4240 struct bio *bio, *read_bio; 4241 int sectors_done = 0; 4242 4243 if (sector_nr == 0) { 4244 /* If restarting in the middle, skip the initial sectors */ 4245 if (mddev->reshape_backwards && 4246 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4247 sector_nr = (raid10_size(mddev, 0, 0) 4248 - conf->reshape_progress); 4249 } else if (!mddev->reshape_backwards && 4250 conf->reshape_progress > 0) 4251 sector_nr = conf->reshape_progress; 4252 if (sector_nr) { 4253 mddev->curr_resync_completed = sector_nr; 4254 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4255 *skipped = 1; 4256 return sector_nr; 4257 } 4258 } 4259 4260 /* We don't use sector_nr to track where we are up to 4261 * as that doesn't work well for ->reshape_backwards. 4262 * So just use ->reshape_progress. 4263 */ 4264 if (mddev->reshape_backwards) { 4265 /* 'next' is the earliest device address that we might 4266 * write to for this chunk in the new layout 4267 */ 4268 next = first_dev_address(conf->reshape_progress - 1, 4269 &conf->geo); 4270 4271 /* 'safe' is the last device address that we might read from 4272 * in the old layout after a restart 4273 */ 4274 safe = last_dev_address(conf->reshape_safe - 1, 4275 &conf->prev); 4276 4277 if (next + conf->offset_diff < safe) 4278 need_flush = 1; 4279 4280 last = conf->reshape_progress - 1; 4281 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4282 & conf->prev.chunk_mask); 4283 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last) 4284 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512; 4285 } else { 4286 /* 'next' is after the last device address that we 4287 * might write to for this chunk in the new layout 4288 */ 4289 next = last_dev_address(conf->reshape_progress, &conf->geo); 4290 4291 /* 'safe' is the earliest device address that we might 4292 * read from in the old layout after a restart 4293 */ 4294 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4295 4296 /* Need to update metadata if 'next' might be beyond 'safe' 4297 * as that would possibly corrupt data 4298 */ 4299 if (next > safe + conf->offset_diff) 4300 need_flush = 1; 4301 4302 sector_nr = conf->reshape_progress; 4303 last = sector_nr | (conf->geo.chunk_mask 4304 & conf->prev.chunk_mask); 4305 4306 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last) 4307 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1; 4308 } 4309 4310 if (need_flush || 4311 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4312 /* Need to update reshape_position in metadata */ 4313 wait_barrier(conf); 4314 mddev->reshape_position = conf->reshape_progress; 4315 if (mddev->reshape_backwards) 4316 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4317 - conf->reshape_progress; 4318 else 4319 mddev->curr_resync_completed = conf->reshape_progress; 4320 conf->reshape_checkpoint = jiffies; 4321 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4322 md_wakeup_thread(mddev->thread); 4323 wait_event(mddev->sb_wait, mddev->flags == 0 || 4324 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4325 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 4326 allow_barrier(conf); 4327 return sectors_done; 4328 } 4329 conf->reshape_safe = mddev->reshape_position; 4330 allow_barrier(conf); 4331 } 4332 4333 read_more: 4334 /* Now schedule reads for blocks from sector_nr to last */ 4335 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 4336 r10_bio->state = 0; 4337 raise_barrier(conf, sectors_done != 0); 4338 atomic_set(&r10_bio->remaining, 0); 4339 r10_bio->mddev = mddev; 4340 r10_bio->sector = sector_nr; 4341 set_bit(R10BIO_IsReshape, &r10_bio->state); 4342 r10_bio->sectors = last - sector_nr + 1; 4343 rdev = read_balance(conf, r10_bio, &max_sectors); 4344 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4345 4346 if (!rdev) { 4347 /* Cannot read from here, so need to record bad blocks 4348 * on all the target devices. 4349 */ 4350 // FIXME 4351 mempool_free(r10_bio, conf->r10buf_pool); 4352 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4353 return sectors_done; 4354 } 4355 4356 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev); 4357 4358 read_bio->bi_bdev = rdev->bdev; 4359 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4360 + rdev->data_offset); 4361 read_bio->bi_private = r10_bio; 4362 read_bio->bi_end_io = end_sync_read; 4363 bio_set_op_attrs(read_bio, REQ_OP_READ, 0); 4364 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); 4365 read_bio->bi_error = 0; 4366 read_bio->bi_vcnt = 0; 4367 read_bio->bi_iter.bi_size = 0; 4368 r10_bio->master_bio = read_bio; 4369 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4370 4371 /* Now find the locations in the new layout */ 4372 __raid10_find_phys(&conf->geo, r10_bio); 4373 4374 blist = read_bio; 4375 read_bio->bi_next = NULL; 4376 4377 rcu_read_lock(); 4378 for (s = 0; s < conf->copies*2; s++) { 4379 struct bio *b; 4380 int d = r10_bio->devs[s/2].devnum; 4381 struct md_rdev *rdev2; 4382 if (s&1) { 4383 rdev2 = rcu_dereference(conf->mirrors[d].replacement); 4384 b = r10_bio->devs[s/2].repl_bio; 4385 } else { 4386 rdev2 = rcu_dereference(conf->mirrors[d].rdev); 4387 b = r10_bio->devs[s/2].bio; 4388 } 4389 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4390 continue; 4391 4392 bio_reset(b); 4393 b->bi_bdev = rdev2->bdev; 4394 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + 4395 rdev2->new_data_offset; 4396 b->bi_private = r10_bio; 4397 b->bi_end_io = end_reshape_write; 4398 bio_set_op_attrs(b, REQ_OP_WRITE, 0); 4399 b->bi_next = blist; 4400 blist = b; 4401 } 4402 4403 /* Now add as many pages as possible to all of these bios. */ 4404 4405 nr_sectors = 0; 4406 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4407 struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page; 4408 int len = (max_sectors - s) << 9; 4409 if (len > PAGE_SIZE) 4410 len = PAGE_SIZE; 4411 for (bio = blist; bio ; bio = bio->bi_next) { 4412 struct bio *bio2; 4413 if (bio_add_page(bio, page, len, 0)) 4414 continue; 4415 4416 /* Didn't fit, must stop */ 4417 for (bio2 = blist; 4418 bio2 && bio2 != bio; 4419 bio2 = bio2->bi_next) { 4420 /* Remove last page from this bio */ 4421 bio2->bi_vcnt--; 4422 bio2->bi_iter.bi_size -= len; 4423 bio_clear_flag(bio2, BIO_SEG_VALID); 4424 } 4425 goto bio_full; 4426 } 4427 sector_nr += len >> 9; 4428 nr_sectors += len >> 9; 4429 } 4430 bio_full: 4431 rcu_read_unlock(); 4432 r10_bio->sectors = nr_sectors; 4433 4434 /* Now submit the read */ 4435 md_sync_acct(read_bio->bi_bdev, r10_bio->sectors); 4436 atomic_inc(&r10_bio->remaining); 4437 read_bio->bi_next = NULL; 4438 generic_make_request(read_bio); 4439 sector_nr += nr_sectors; 4440 sectors_done += nr_sectors; 4441 if (sector_nr <= last) 4442 goto read_more; 4443 4444 /* Now that we have done the whole section we can 4445 * update reshape_progress 4446 */ 4447 if (mddev->reshape_backwards) 4448 conf->reshape_progress -= sectors_done; 4449 else 4450 conf->reshape_progress += sectors_done; 4451 4452 return sectors_done; 4453 } 4454 4455 static void end_reshape_request(struct r10bio *r10_bio); 4456 static int handle_reshape_read_error(struct mddev *mddev, 4457 struct r10bio *r10_bio); 4458 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4459 { 4460 /* Reshape read completed. Hopefully we have a block 4461 * to write out. 4462 * If we got a read error then we do sync 1-page reads from 4463 * elsewhere until we find the data - or give up. 4464 */ 4465 struct r10conf *conf = mddev->private; 4466 int s; 4467 4468 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4469 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4470 /* Reshape has been aborted */ 4471 md_done_sync(mddev, r10_bio->sectors, 0); 4472 return; 4473 } 4474 4475 /* We definitely have the data in the pages, schedule the 4476 * writes. 4477 */ 4478 atomic_set(&r10_bio->remaining, 1); 4479 for (s = 0; s < conf->copies*2; s++) { 4480 struct bio *b; 4481 int d = r10_bio->devs[s/2].devnum; 4482 struct md_rdev *rdev; 4483 rcu_read_lock(); 4484 if (s&1) { 4485 rdev = rcu_dereference(conf->mirrors[d].replacement); 4486 b = r10_bio->devs[s/2].repl_bio; 4487 } else { 4488 rdev = rcu_dereference(conf->mirrors[d].rdev); 4489 b = r10_bio->devs[s/2].bio; 4490 } 4491 if (!rdev || test_bit(Faulty, &rdev->flags)) { 4492 rcu_read_unlock(); 4493 continue; 4494 } 4495 atomic_inc(&rdev->nr_pending); 4496 rcu_read_unlock(); 4497 md_sync_acct(b->bi_bdev, r10_bio->sectors); 4498 atomic_inc(&r10_bio->remaining); 4499 b->bi_next = NULL; 4500 generic_make_request(b); 4501 } 4502 end_reshape_request(r10_bio); 4503 } 4504 4505 static void end_reshape(struct r10conf *conf) 4506 { 4507 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4508 return; 4509 4510 spin_lock_irq(&conf->device_lock); 4511 conf->prev = conf->geo; 4512 md_finish_reshape(conf->mddev); 4513 smp_wmb(); 4514 conf->reshape_progress = MaxSector; 4515 conf->reshape_safe = MaxSector; 4516 spin_unlock_irq(&conf->device_lock); 4517 4518 /* read-ahead size must cover two whole stripes, which is 4519 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4520 */ 4521 if (conf->mddev->queue) { 4522 int stripe = conf->geo.raid_disks * 4523 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE); 4524 stripe /= conf->geo.near_copies; 4525 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4526 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4527 } 4528 conf->fullsync = 0; 4529 } 4530 4531 static int handle_reshape_read_error(struct mddev *mddev, 4532 struct r10bio *r10_bio) 4533 { 4534 /* Use sync reads to get the blocks from somewhere else */ 4535 int sectors = r10_bio->sectors; 4536 struct r10conf *conf = mddev->private; 4537 struct { 4538 struct r10bio r10_bio; 4539 struct r10dev devs[conf->copies]; 4540 } on_stack; 4541 struct r10bio *r10b = &on_stack.r10_bio; 4542 int slot = 0; 4543 int idx = 0; 4544 struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec; 4545 4546 r10b->sector = r10_bio->sector; 4547 __raid10_find_phys(&conf->prev, r10b); 4548 4549 while (sectors) { 4550 int s = sectors; 4551 int success = 0; 4552 int first_slot = slot; 4553 4554 if (s > (PAGE_SIZE >> 9)) 4555 s = PAGE_SIZE >> 9; 4556 4557 rcu_read_lock(); 4558 while (!success) { 4559 int d = r10b->devs[slot].devnum; 4560 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4561 sector_t addr; 4562 if (rdev == NULL || 4563 test_bit(Faulty, &rdev->flags) || 4564 !test_bit(In_sync, &rdev->flags)) 4565 goto failed; 4566 4567 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 4568 atomic_inc(&rdev->nr_pending); 4569 rcu_read_unlock(); 4570 success = sync_page_io(rdev, 4571 addr, 4572 s << 9, 4573 bvec[idx].bv_page, 4574 REQ_OP_READ, 0, false); 4575 rdev_dec_pending(rdev, mddev); 4576 rcu_read_lock(); 4577 if (success) 4578 break; 4579 failed: 4580 slot++; 4581 if (slot >= conf->copies) 4582 slot = 0; 4583 if (slot == first_slot) 4584 break; 4585 } 4586 rcu_read_unlock(); 4587 if (!success) { 4588 /* couldn't read this block, must give up */ 4589 set_bit(MD_RECOVERY_INTR, 4590 &mddev->recovery); 4591 return -EIO; 4592 } 4593 sectors -= s; 4594 idx++; 4595 } 4596 return 0; 4597 } 4598 4599 static void end_reshape_write(struct bio *bio) 4600 { 4601 struct r10bio *r10_bio = bio->bi_private; 4602 struct mddev *mddev = r10_bio->mddev; 4603 struct r10conf *conf = mddev->private; 4604 int d; 4605 int slot; 4606 int repl; 4607 struct md_rdev *rdev = NULL; 4608 4609 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 4610 if (repl) 4611 rdev = conf->mirrors[d].replacement; 4612 if (!rdev) { 4613 smp_mb(); 4614 rdev = conf->mirrors[d].rdev; 4615 } 4616 4617 if (bio->bi_error) { 4618 /* FIXME should record badblock */ 4619 md_error(mddev, rdev); 4620 } 4621 4622 rdev_dec_pending(rdev, mddev); 4623 end_reshape_request(r10_bio); 4624 } 4625 4626 static void end_reshape_request(struct r10bio *r10_bio) 4627 { 4628 if (!atomic_dec_and_test(&r10_bio->remaining)) 4629 return; 4630 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 4631 bio_put(r10_bio->master_bio); 4632 put_buf(r10_bio); 4633 } 4634 4635 static void raid10_finish_reshape(struct mddev *mddev) 4636 { 4637 struct r10conf *conf = mddev->private; 4638 4639 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4640 return; 4641 4642 if (mddev->delta_disks > 0) { 4643 sector_t size = raid10_size(mddev, 0, 0); 4644 md_set_array_sectors(mddev, size); 4645 if (mddev->recovery_cp > mddev->resync_max_sectors) { 4646 mddev->recovery_cp = mddev->resync_max_sectors; 4647 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4648 } 4649 mddev->resync_max_sectors = size; 4650 if (mddev->queue) { 4651 set_capacity(mddev->gendisk, mddev->array_sectors); 4652 revalidate_disk(mddev->gendisk); 4653 } 4654 } else { 4655 int d; 4656 rcu_read_lock(); 4657 for (d = conf->geo.raid_disks ; 4658 d < conf->geo.raid_disks - mddev->delta_disks; 4659 d++) { 4660 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4661 if (rdev) 4662 clear_bit(In_sync, &rdev->flags); 4663 rdev = rcu_dereference(conf->mirrors[d].replacement); 4664 if (rdev) 4665 clear_bit(In_sync, &rdev->flags); 4666 } 4667 rcu_read_unlock(); 4668 } 4669 mddev->layout = mddev->new_layout; 4670 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 4671 mddev->reshape_position = MaxSector; 4672 mddev->delta_disks = 0; 4673 mddev->reshape_backwards = 0; 4674 } 4675 4676 static struct md_personality raid10_personality = 4677 { 4678 .name = "raid10", 4679 .level = 10, 4680 .owner = THIS_MODULE, 4681 .make_request = raid10_make_request, 4682 .run = raid10_run, 4683 .free = raid10_free, 4684 .status = raid10_status, 4685 .error_handler = raid10_error, 4686 .hot_add_disk = raid10_add_disk, 4687 .hot_remove_disk= raid10_remove_disk, 4688 .spare_active = raid10_spare_active, 4689 .sync_request = raid10_sync_request, 4690 .quiesce = raid10_quiesce, 4691 .size = raid10_size, 4692 .resize = raid10_resize, 4693 .takeover = raid10_takeover, 4694 .check_reshape = raid10_check_reshape, 4695 .start_reshape = raid10_start_reshape, 4696 .finish_reshape = raid10_finish_reshape, 4697 .congested = raid10_congested, 4698 }; 4699 4700 static int __init raid_init(void) 4701 { 4702 return register_md_personality(&raid10_personality); 4703 } 4704 4705 static void raid_exit(void) 4706 { 4707 unregister_md_personality(&raid10_personality); 4708 } 4709 4710 module_init(raid_init); 4711 module_exit(raid_exit); 4712 MODULE_LICENSE("GPL"); 4713 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 4714 MODULE_ALIAS("md-personality-9"); /* RAID10 */ 4715 MODULE_ALIAS("md-raid10"); 4716 MODULE_ALIAS("md-level-10"); 4717 4718 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 4719