1 /* 2 * raid10.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 2000-2004 Neil Brown 5 * 6 * RAID-10 support for md. 7 * 8 * Base on code in raid1.c. See raid1.c for further copyright information. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 #include <linux/slab.h> 22 #include <linux/delay.h> 23 #include <linux/blkdev.h> 24 #include <linux/module.h> 25 #include <linux/seq_file.h> 26 #include <linux/ratelimit.h> 27 #include <linux/kthread.h> 28 #include "md.h" 29 #include "raid10.h" 30 #include "raid0.h" 31 #include "bitmap.h" 32 33 /* 34 * RAID10 provides a combination of RAID0 and RAID1 functionality. 35 * The layout of data is defined by 36 * chunk_size 37 * raid_disks 38 * near_copies (stored in low byte of layout) 39 * far_copies (stored in second byte of layout) 40 * far_offset (stored in bit 16 of layout ) 41 * use_far_sets (stored in bit 17 of layout ) 42 * 43 * The data to be stored is divided into chunks using chunksize. Each device 44 * is divided into far_copies sections. In each section, chunks are laid out 45 * in a style similar to raid0, but near_copies copies of each chunk is stored 46 * (each on a different drive). The starting device for each section is offset 47 * near_copies from the starting device of the previous section. Thus there 48 * are (near_copies * far_copies) of each chunk, and each is on a different 49 * drive. near_copies and far_copies must be at least one, and their product 50 * is at most raid_disks. 51 * 52 * If far_offset is true, then the far_copies are handled a bit differently. 53 * The copies are still in different stripes, but instead of being very far 54 * apart on disk, there are adjacent stripes. 55 * 56 * The far and offset algorithms are handled slightly differently if 57 * 'use_far_sets' is true. In this case, the array's devices are grouped into 58 * sets that are (near_copies * far_copies) in size. The far copied stripes 59 * are still shifted by 'near_copies' devices, but this shifting stays confined 60 * to the set rather than the entire array. This is done to improve the number 61 * of device combinations that can fail without causing the array to fail. 62 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 63 * on a device): 64 * A B C D A B C D E 65 * ... ... 66 * D A B C E A B C D 67 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 68 * [A B] [C D] [A B] [C D E] 69 * |...| |...| |...| | ... | 70 * [B A] [D C] [B A] [E C D] 71 */ 72 73 /* 74 * Number of guaranteed r10bios in case of extreme VM load: 75 */ 76 #define NR_RAID10_BIOS 256 77 78 /* when we get a read error on a read-only array, we redirect to another 79 * device without failing the first device, or trying to over-write to 80 * correct the read error. To keep track of bad blocks on a per-bio 81 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 82 */ 83 #define IO_BLOCKED ((struct bio *)1) 84 /* When we successfully write to a known bad-block, we need to remove the 85 * bad-block marking which must be done from process context. So we record 86 * the success by setting devs[n].bio to IO_MADE_GOOD 87 */ 88 #define IO_MADE_GOOD ((struct bio *)2) 89 90 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 91 92 /* When there are this many requests queued to be written by 93 * the raid10 thread, we become 'congested' to provide back-pressure 94 * for writeback. 95 */ 96 static int max_queued_requests = 1024; 97 98 static void allow_barrier(struct r10conf *conf); 99 static void lower_barrier(struct r10conf *conf); 100 static int enough(struct r10conf *conf, int ignore); 101 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 102 int *skipped); 103 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 104 static void end_reshape_write(struct bio *bio, int error); 105 static void end_reshape(struct r10conf *conf); 106 107 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 108 { 109 struct r10conf *conf = data; 110 int size = offsetof(struct r10bio, devs[conf->copies]); 111 112 /* allocate a r10bio with room for raid_disks entries in the 113 * bios array */ 114 return kzalloc(size, gfp_flags); 115 } 116 117 static void r10bio_pool_free(void *r10_bio, void *data) 118 { 119 kfree(r10_bio); 120 } 121 122 /* Maximum size of each resync request */ 123 #define RESYNC_BLOCK_SIZE (64*1024) 124 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 125 /* amount of memory to reserve for resync requests */ 126 #define RESYNC_WINDOW (1024*1024) 127 /* maximum number of concurrent requests, memory permitting */ 128 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 129 130 /* 131 * When performing a resync, we need to read and compare, so 132 * we need as many pages are there are copies. 133 * When performing a recovery, we need 2 bios, one for read, 134 * one for write (we recover only one drive per r10buf) 135 * 136 */ 137 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 138 { 139 struct r10conf *conf = data; 140 struct page *page; 141 struct r10bio *r10_bio; 142 struct bio *bio; 143 int i, j; 144 int nalloc; 145 146 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 147 if (!r10_bio) 148 return NULL; 149 150 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 151 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 152 nalloc = conf->copies; /* resync */ 153 else 154 nalloc = 2; /* recovery */ 155 156 /* 157 * Allocate bios. 158 */ 159 for (j = nalloc ; j-- ; ) { 160 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 161 if (!bio) 162 goto out_free_bio; 163 r10_bio->devs[j].bio = bio; 164 if (!conf->have_replacement) 165 continue; 166 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 167 if (!bio) 168 goto out_free_bio; 169 r10_bio->devs[j].repl_bio = bio; 170 } 171 /* 172 * Allocate RESYNC_PAGES data pages and attach them 173 * where needed. 174 */ 175 for (j = 0 ; j < nalloc; j++) { 176 struct bio *rbio = r10_bio->devs[j].repl_bio; 177 bio = r10_bio->devs[j].bio; 178 for (i = 0; i < RESYNC_PAGES; i++) { 179 if (j > 0 && !test_bit(MD_RECOVERY_SYNC, 180 &conf->mddev->recovery)) { 181 /* we can share bv_page's during recovery 182 * and reshape */ 183 struct bio *rbio = r10_bio->devs[0].bio; 184 page = rbio->bi_io_vec[i].bv_page; 185 get_page(page); 186 } else 187 page = alloc_page(gfp_flags); 188 if (unlikely(!page)) 189 goto out_free_pages; 190 191 bio->bi_io_vec[i].bv_page = page; 192 if (rbio) 193 rbio->bi_io_vec[i].bv_page = page; 194 } 195 } 196 197 return r10_bio; 198 199 out_free_pages: 200 for ( ; i > 0 ; i--) 201 safe_put_page(bio->bi_io_vec[i-1].bv_page); 202 while (j--) 203 for (i = 0; i < RESYNC_PAGES ; i++) 204 safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page); 205 j = 0; 206 out_free_bio: 207 for ( ; j < nalloc; j++) { 208 if (r10_bio->devs[j].bio) 209 bio_put(r10_bio->devs[j].bio); 210 if (r10_bio->devs[j].repl_bio) 211 bio_put(r10_bio->devs[j].repl_bio); 212 } 213 r10bio_pool_free(r10_bio, conf); 214 return NULL; 215 } 216 217 static void r10buf_pool_free(void *__r10_bio, void *data) 218 { 219 int i; 220 struct r10conf *conf = data; 221 struct r10bio *r10bio = __r10_bio; 222 int j; 223 224 for (j=0; j < conf->copies; j++) { 225 struct bio *bio = r10bio->devs[j].bio; 226 if (bio) { 227 for (i = 0; i < RESYNC_PAGES; i++) { 228 safe_put_page(bio->bi_io_vec[i].bv_page); 229 bio->bi_io_vec[i].bv_page = NULL; 230 } 231 bio_put(bio); 232 } 233 bio = r10bio->devs[j].repl_bio; 234 if (bio) 235 bio_put(bio); 236 } 237 r10bio_pool_free(r10bio, conf); 238 } 239 240 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 241 { 242 int i; 243 244 for (i = 0; i < conf->copies; i++) { 245 struct bio **bio = & r10_bio->devs[i].bio; 246 if (!BIO_SPECIAL(*bio)) 247 bio_put(*bio); 248 *bio = NULL; 249 bio = &r10_bio->devs[i].repl_bio; 250 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 251 bio_put(*bio); 252 *bio = NULL; 253 } 254 } 255 256 static void free_r10bio(struct r10bio *r10_bio) 257 { 258 struct r10conf *conf = r10_bio->mddev->private; 259 260 put_all_bios(conf, r10_bio); 261 mempool_free(r10_bio, conf->r10bio_pool); 262 } 263 264 static void put_buf(struct r10bio *r10_bio) 265 { 266 struct r10conf *conf = r10_bio->mddev->private; 267 268 mempool_free(r10_bio, conf->r10buf_pool); 269 270 lower_barrier(conf); 271 } 272 273 static void reschedule_retry(struct r10bio *r10_bio) 274 { 275 unsigned long flags; 276 struct mddev *mddev = r10_bio->mddev; 277 struct r10conf *conf = mddev->private; 278 279 spin_lock_irqsave(&conf->device_lock, flags); 280 list_add(&r10_bio->retry_list, &conf->retry_list); 281 conf->nr_queued ++; 282 spin_unlock_irqrestore(&conf->device_lock, flags); 283 284 /* wake up frozen array... */ 285 wake_up(&conf->wait_barrier); 286 287 md_wakeup_thread(mddev->thread); 288 } 289 290 /* 291 * raid_end_bio_io() is called when we have finished servicing a mirrored 292 * operation and are ready to return a success/failure code to the buffer 293 * cache layer. 294 */ 295 static void raid_end_bio_io(struct r10bio *r10_bio) 296 { 297 struct bio *bio = r10_bio->master_bio; 298 int done; 299 struct r10conf *conf = r10_bio->mddev->private; 300 301 if (bio->bi_phys_segments) { 302 unsigned long flags; 303 spin_lock_irqsave(&conf->device_lock, flags); 304 bio->bi_phys_segments--; 305 done = (bio->bi_phys_segments == 0); 306 spin_unlock_irqrestore(&conf->device_lock, flags); 307 } else 308 done = 1; 309 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 310 clear_bit(BIO_UPTODATE, &bio->bi_flags); 311 if (done) { 312 bio_endio(bio, 0); 313 /* 314 * Wake up any possible resync thread that waits for the device 315 * to go idle. 316 */ 317 allow_barrier(conf); 318 } 319 free_r10bio(r10_bio); 320 } 321 322 /* 323 * Update disk head position estimator based on IRQ completion info. 324 */ 325 static inline void update_head_pos(int slot, struct r10bio *r10_bio) 326 { 327 struct r10conf *conf = r10_bio->mddev->private; 328 329 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 330 r10_bio->devs[slot].addr + (r10_bio->sectors); 331 } 332 333 /* 334 * Find the disk number which triggered given bio 335 */ 336 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 337 struct bio *bio, int *slotp, int *replp) 338 { 339 int slot; 340 int repl = 0; 341 342 for (slot = 0; slot < conf->copies; slot++) { 343 if (r10_bio->devs[slot].bio == bio) 344 break; 345 if (r10_bio->devs[slot].repl_bio == bio) { 346 repl = 1; 347 break; 348 } 349 } 350 351 BUG_ON(slot == conf->copies); 352 update_head_pos(slot, r10_bio); 353 354 if (slotp) 355 *slotp = slot; 356 if (replp) 357 *replp = repl; 358 return r10_bio->devs[slot].devnum; 359 } 360 361 static void raid10_end_read_request(struct bio *bio, int error) 362 { 363 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 364 struct r10bio *r10_bio = bio->bi_private; 365 int slot, dev; 366 struct md_rdev *rdev; 367 struct r10conf *conf = r10_bio->mddev->private; 368 369 370 slot = r10_bio->read_slot; 371 dev = r10_bio->devs[slot].devnum; 372 rdev = r10_bio->devs[slot].rdev; 373 /* 374 * this branch is our 'one mirror IO has finished' event handler: 375 */ 376 update_head_pos(slot, r10_bio); 377 378 if (uptodate) { 379 /* 380 * Set R10BIO_Uptodate in our master bio, so that 381 * we will return a good error code to the higher 382 * levels even if IO on some other mirrored buffer fails. 383 * 384 * The 'master' represents the composite IO operation to 385 * user-side. So if something waits for IO, then it will 386 * wait for the 'master' bio. 387 */ 388 set_bit(R10BIO_Uptodate, &r10_bio->state); 389 } else { 390 /* If all other devices that store this block have 391 * failed, we want to return the error upwards rather 392 * than fail the last device. Here we redefine 393 * "uptodate" to mean "Don't want to retry" 394 */ 395 unsigned long flags; 396 spin_lock_irqsave(&conf->device_lock, flags); 397 if (!enough(conf, rdev->raid_disk)) 398 uptodate = 1; 399 spin_unlock_irqrestore(&conf->device_lock, flags); 400 } 401 if (uptodate) { 402 raid_end_bio_io(r10_bio); 403 rdev_dec_pending(rdev, conf->mddev); 404 } else { 405 /* 406 * oops, read error - keep the refcount on the rdev 407 */ 408 char b[BDEVNAME_SIZE]; 409 printk_ratelimited(KERN_ERR 410 "md/raid10:%s: %s: rescheduling sector %llu\n", 411 mdname(conf->mddev), 412 bdevname(rdev->bdev, b), 413 (unsigned long long)r10_bio->sector); 414 set_bit(R10BIO_ReadError, &r10_bio->state); 415 reschedule_retry(r10_bio); 416 } 417 } 418 419 static void close_write(struct r10bio *r10_bio) 420 { 421 /* clear the bitmap if all writes complete successfully */ 422 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector, 423 r10_bio->sectors, 424 !test_bit(R10BIO_Degraded, &r10_bio->state), 425 0); 426 md_write_end(r10_bio->mddev); 427 } 428 429 static void one_write_done(struct r10bio *r10_bio) 430 { 431 if (atomic_dec_and_test(&r10_bio->remaining)) { 432 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 433 reschedule_retry(r10_bio); 434 else { 435 close_write(r10_bio); 436 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 437 reschedule_retry(r10_bio); 438 else 439 raid_end_bio_io(r10_bio); 440 } 441 } 442 } 443 444 static void raid10_end_write_request(struct bio *bio, int error) 445 { 446 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 447 struct r10bio *r10_bio = bio->bi_private; 448 int dev; 449 int dec_rdev = 1; 450 struct r10conf *conf = r10_bio->mddev->private; 451 int slot, repl; 452 struct md_rdev *rdev = NULL; 453 454 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 455 456 if (repl) 457 rdev = conf->mirrors[dev].replacement; 458 if (!rdev) { 459 smp_rmb(); 460 repl = 0; 461 rdev = conf->mirrors[dev].rdev; 462 } 463 /* 464 * this branch is our 'one mirror IO has finished' event handler: 465 */ 466 if (!uptodate) { 467 if (repl) 468 /* Never record new bad blocks to replacement, 469 * just fail it. 470 */ 471 md_error(rdev->mddev, rdev); 472 else { 473 set_bit(WriteErrorSeen, &rdev->flags); 474 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 475 set_bit(MD_RECOVERY_NEEDED, 476 &rdev->mddev->recovery); 477 set_bit(R10BIO_WriteError, &r10_bio->state); 478 dec_rdev = 0; 479 } 480 } else { 481 /* 482 * Set R10BIO_Uptodate in our master bio, so that 483 * we will return a good error code for to the higher 484 * levels even if IO on some other mirrored buffer fails. 485 * 486 * The 'master' represents the composite IO operation to 487 * user-side. So if something waits for IO, then it will 488 * wait for the 'master' bio. 489 */ 490 sector_t first_bad; 491 int bad_sectors; 492 493 set_bit(R10BIO_Uptodate, &r10_bio->state); 494 495 /* Maybe we can clear some bad blocks. */ 496 if (is_badblock(rdev, 497 r10_bio->devs[slot].addr, 498 r10_bio->sectors, 499 &first_bad, &bad_sectors)) { 500 bio_put(bio); 501 if (repl) 502 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 503 else 504 r10_bio->devs[slot].bio = IO_MADE_GOOD; 505 dec_rdev = 0; 506 set_bit(R10BIO_MadeGood, &r10_bio->state); 507 } 508 } 509 510 /* 511 * 512 * Let's see if all mirrored write operations have finished 513 * already. 514 */ 515 one_write_done(r10_bio); 516 if (dec_rdev) 517 rdev_dec_pending(rdev, conf->mddev); 518 } 519 520 /* 521 * RAID10 layout manager 522 * As well as the chunksize and raid_disks count, there are two 523 * parameters: near_copies and far_copies. 524 * near_copies * far_copies must be <= raid_disks. 525 * Normally one of these will be 1. 526 * If both are 1, we get raid0. 527 * If near_copies == raid_disks, we get raid1. 528 * 529 * Chunks are laid out in raid0 style with near_copies copies of the 530 * first chunk, followed by near_copies copies of the next chunk and 531 * so on. 532 * If far_copies > 1, then after 1/far_copies of the array has been assigned 533 * as described above, we start again with a device offset of near_copies. 534 * So we effectively have another copy of the whole array further down all 535 * the drives, but with blocks on different drives. 536 * With this layout, and block is never stored twice on the one device. 537 * 538 * raid10_find_phys finds the sector offset of a given virtual sector 539 * on each device that it is on. 540 * 541 * raid10_find_virt does the reverse mapping, from a device and a 542 * sector offset to a virtual address 543 */ 544 545 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 546 { 547 int n,f; 548 sector_t sector; 549 sector_t chunk; 550 sector_t stripe; 551 int dev; 552 int slot = 0; 553 int last_far_set_start, last_far_set_size; 554 555 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 556 last_far_set_start *= geo->far_set_size; 557 558 last_far_set_size = geo->far_set_size; 559 last_far_set_size += (geo->raid_disks % geo->far_set_size); 560 561 /* now calculate first sector/dev */ 562 chunk = r10bio->sector >> geo->chunk_shift; 563 sector = r10bio->sector & geo->chunk_mask; 564 565 chunk *= geo->near_copies; 566 stripe = chunk; 567 dev = sector_div(stripe, geo->raid_disks); 568 if (geo->far_offset) 569 stripe *= geo->far_copies; 570 571 sector += stripe << geo->chunk_shift; 572 573 /* and calculate all the others */ 574 for (n = 0; n < geo->near_copies; n++) { 575 int d = dev; 576 int set; 577 sector_t s = sector; 578 r10bio->devs[slot].devnum = d; 579 r10bio->devs[slot].addr = s; 580 slot++; 581 582 for (f = 1; f < geo->far_copies; f++) { 583 set = d / geo->far_set_size; 584 d += geo->near_copies; 585 586 if ((geo->raid_disks % geo->far_set_size) && 587 (d > last_far_set_start)) { 588 d -= last_far_set_start; 589 d %= last_far_set_size; 590 d += last_far_set_start; 591 } else { 592 d %= geo->far_set_size; 593 d += geo->far_set_size * set; 594 } 595 s += geo->stride; 596 r10bio->devs[slot].devnum = d; 597 r10bio->devs[slot].addr = s; 598 slot++; 599 } 600 dev++; 601 if (dev >= geo->raid_disks) { 602 dev = 0; 603 sector += (geo->chunk_mask + 1); 604 } 605 } 606 } 607 608 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 609 { 610 struct geom *geo = &conf->geo; 611 612 if (conf->reshape_progress != MaxSector && 613 ((r10bio->sector >= conf->reshape_progress) != 614 conf->mddev->reshape_backwards)) { 615 set_bit(R10BIO_Previous, &r10bio->state); 616 geo = &conf->prev; 617 } else 618 clear_bit(R10BIO_Previous, &r10bio->state); 619 620 __raid10_find_phys(geo, r10bio); 621 } 622 623 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 624 { 625 sector_t offset, chunk, vchunk; 626 /* Never use conf->prev as this is only called during resync 627 * or recovery, so reshape isn't happening 628 */ 629 struct geom *geo = &conf->geo; 630 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 631 int far_set_size = geo->far_set_size; 632 int last_far_set_start; 633 634 if (geo->raid_disks % geo->far_set_size) { 635 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 636 last_far_set_start *= geo->far_set_size; 637 638 if (dev >= last_far_set_start) { 639 far_set_size = geo->far_set_size; 640 far_set_size += (geo->raid_disks % geo->far_set_size); 641 far_set_start = last_far_set_start; 642 } 643 } 644 645 offset = sector & geo->chunk_mask; 646 if (geo->far_offset) { 647 int fc; 648 chunk = sector >> geo->chunk_shift; 649 fc = sector_div(chunk, geo->far_copies); 650 dev -= fc * geo->near_copies; 651 if (dev < far_set_start) 652 dev += far_set_size; 653 } else { 654 while (sector >= geo->stride) { 655 sector -= geo->stride; 656 if (dev < (geo->near_copies + far_set_start)) 657 dev += far_set_size - geo->near_copies; 658 else 659 dev -= geo->near_copies; 660 } 661 chunk = sector >> geo->chunk_shift; 662 } 663 vchunk = chunk * geo->raid_disks + dev; 664 sector_div(vchunk, geo->near_copies); 665 return (vchunk << geo->chunk_shift) + offset; 666 } 667 668 /** 669 * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged 670 * @q: request queue 671 * @bvm: properties of new bio 672 * @biovec: the request that could be merged to it. 673 * 674 * Return amount of bytes we can accept at this offset 675 * This requires checking for end-of-chunk if near_copies != raid_disks, 676 * and for subordinate merge_bvec_fns if merge_check_needed. 677 */ 678 static int raid10_mergeable_bvec(struct request_queue *q, 679 struct bvec_merge_data *bvm, 680 struct bio_vec *biovec) 681 { 682 struct mddev *mddev = q->queuedata; 683 struct r10conf *conf = mddev->private; 684 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 685 int max; 686 unsigned int chunk_sectors; 687 unsigned int bio_sectors = bvm->bi_size >> 9; 688 struct geom *geo = &conf->geo; 689 690 chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1; 691 if (conf->reshape_progress != MaxSector && 692 ((sector >= conf->reshape_progress) != 693 conf->mddev->reshape_backwards)) 694 geo = &conf->prev; 695 696 if (geo->near_copies < geo->raid_disks) { 697 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) 698 + bio_sectors)) << 9; 699 if (max < 0) 700 /* bio_add cannot handle a negative return */ 701 max = 0; 702 if (max <= biovec->bv_len && bio_sectors == 0) 703 return biovec->bv_len; 704 } else 705 max = biovec->bv_len; 706 707 if (mddev->merge_check_needed) { 708 struct { 709 struct r10bio r10_bio; 710 struct r10dev devs[conf->copies]; 711 } on_stack; 712 struct r10bio *r10_bio = &on_stack.r10_bio; 713 int s; 714 if (conf->reshape_progress != MaxSector) { 715 /* Cannot give any guidance during reshape */ 716 if (max <= biovec->bv_len && bio_sectors == 0) 717 return biovec->bv_len; 718 return 0; 719 } 720 r10_bio->sector = sector; 721 raid10_find_phys(conf, r10_bio); 722 rcu_read_lock(); 723 for (s = 0; s < conf->copies; s++) { 724 int disk = r10_bio->devs[s].devnum; 725 struct md_rdev *rdev = rcu_dereference( 726 conf->mirrors[disk].rdev); 727 if (rdev && !test_bit(Faulty, &rdev->flags)) { 728 struct request_queue *q = 729 bdev_get_queue(rdev->bdev); 730 if (q->merge_bvec_fn) { 731 bvm->bi_sector = r10_bio->devs[s].addr 732 + rdev->data_offset; 733 bvm->bi_bdev = rdev->bdev; 734 max = min(max, q->merge_bvec_fn( 735 q, bvm, biovec)); 736 } 737 } 738 rdev = rcu_dereference(conf->mirrors[disk].replacement); 739 if (rdev && !test_bit(Faulty, &rdev->flags)) { 740 struct request_queue *q = 741 bdev_get_queue(rdev->bdev); 742 if (q->merge_bvec_fn) { 743 bvm->bi_sector = r10_bio->devs[s].addr 744 + rdev->data_offset; 745 bvm->bi_bdev = rdev->bdev; 746 max = min(max, q->merge_bvec_fn( 747 q, bvm, biovec)); 748 } 749 } 750 } 751 rcu_read_unlock(); 752 } 753 return max; 754 } 755 756 /* 757 * This routine returns the disk from which the requested read should 758 * be done. There is a per-array 'next expected sequential IO' sector 759 * number - if this matches on the next IO then we use the last disk. 760 * There is also a per-disk 'last know head position' sector that is 761 * maintained from IRQ contexts, both the normal and the resync IO 762 * completion handlers update this position correctly. If there is no 763 * perfect sequential match then we pick the disk whose head is closest. 764 * 765 * If there are 2 mirrors in the same 2 devices, performance degrades 766 * because position is mirror, not device based. 767 * 768 * The rdev for the device selected will have nr_pending incremented. 769 */ 770 771 /* 772 * FIXME: possibly should rethink readbalancing and do it differently 773 * depending on near_copies / far_copies geometry. 774 */ 775 static struct md_rdev *read_balance(struct r10conf *conf, 776 struct r10bio *r10_bio, 777 int *max_sectors) 778 { 779 const sector_t this_sector = r10_bio->sector; 780 int disk, slot; 781 int sectors = r10_bio->sectors; 782 int best_good_sectors; 783 sector_t new_distance, best_dist; 784 struct md_rdev *best_rdev, *rdev = NULL; 785 int do_balance; 786 int best_slot; 787 struct geom *geo = &conf->geo; 788 789 raid10_find_phys(conf, r10_bio); 790 rcu_read_lock(); 791 retry: 792 sectors = r10_bio->sectors; 793 best_slot = -1; 794 best_rdev = NULL; 795 best_dist = MaxSector; 796 best_good_sectors = 0; 797 do_balance = 1; 798 /* 799 * Check if we can balance. We can balance on the whole 800 * device if no resync is going on (recovery is ok), or below 801 * the resync window. We take the first readable disk when 802 * above the resync window. 803 */ 804 if (conf->mddev->recovery_cp < MaxSector 805 && (this_sector + sectors >= conf->next_resync)) 806 do_balance = 0; 807 808 for (slot = 0; slot < conf->copies ; slot++) { 809 sector_t first_bad; 810 int bad_sectors; 811 sector_t dev_sector; 812 813 if (r10_bio->devs[slot].bio == IO_BLOCKED) 814 continue; 815 disk = r10_bio->devs[slot].devnum; 816 rdev = rcu_dereference(conf->mirrors[disk].replacement); 817 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 818 test_bit(Unmerged, &rdev->flags) || 819 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 820 rdev = rcu_dereference(conf->mirrors[disk].rdev); 821 if (rdev == NULL || 822 test_bit(Faulty, &rdev->flags) || 823 test_bit(Unmerged, &rdev->flags)) 824 continue; 825 if (!test_bit(In_sync, &rdev->flags) && 826 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 827 continue; 828 829 dev_sector = r10_bio->devs[slot].addr; 830 if (is_badblock(rdev, dev_sector, sectors, 831 &first_bad, &bad_sectors)) { 832 if (best_dist < MaxSector) 833 /* Already have a better slot */ 834 continue; 835 if (first_bad <= dev_sector) { 836 /* Cannot read here. If this is the 837 * 'primary' device, then we must not read 838 * beyond 'bad_sectors' from another device. 839 */ 840 bad_sectors -= (dev_sector - first_bad); 841 if (!do_balance && sectors > bad_sectors) 842 sectors = bad_sectors; 843 if (best_good_sectors > sectors) 844 best_good_sectors = sectors; 845 } else { 846 sector_t good_sectors = 847 first_bad - dev_sector; 848 if (good_sectors > best_good_sectors) { 849 best_good_sectors = good_sectors; 850 best_slot = slot; 851 best_rdev = rdev; 852 } 853 if (!do_balance) 854 /* Must read from here */ 855 break; 856 } 857 continue; 858 } else 859 best_good_sectors = sectors; 860 861 if (!do_balance) 862 break; 863 864 /* This optimisation is debatable, and completely destroys 865 * sequential read speed for 'far copies' arrays. So only 866 * keep it for 'near' arrays, and review those later. 867 */ 868 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending)) 869 break; 870 871 /* for far > 1 always use the lowest address */ 872 if (geo->far_copies > 1) 873 new_distance = r10_bio->devs[slot].addr; 874 else 875 new_distance = abs(r10_bio->devs[slot].addr - 876 conf->mirrors[disk].head_position); 877 if (new_distance < best_dist) { 878 best_dist = new_distance; 879 best_slot = slot; 880 best_rdev = rdev; 881 } 882 } 883 if (slot >= conf->copies) { 884 slot = best_slot; 885 rdev = best_rdev; 886 } 887 888 if (slot >= 0) { 889 atomic_inc(&rdev->nr_pending); 890 if (test_bit(Faulty, &rdev->flags)) { 891 /* Cannot risk returning a device that failed 892 * before we inc'ed nr_pending 893 */ 894 rdev_dec_pending(rdev, conf->mddev); 895 goto retry; 896 } 897 r10_bio->read_slot = slot; 898 } else 899 rdev = NULL; 900 rcu_read_unlock(); 901 *max_sectors = best_good_sectors; 902 903 return rdev; 904 } 905 906 int md_raid10_congested(struct mddev *mddev, int bits) 907 { 908 struct r10conf *conf = mddev->private; 909 int i, ret = 0; 910 911 if ((bits & (1 << BDI_async_congested)) && 912 conf->pending_count >= max_queued_requests) 913 return 1; 914 915 rcu_read_lock(); 916 for (i = 0; 917 (i < conf->geo.raid_disks || i < conf->prev.raid_disks) 918 && ret == 0; 919 i++) { 920 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 921 if (rdev && !test_bit(Faulty, &rdev->flags)) { 922 struct request_queue *q = bdev_get_queue(rdev->bdev); 923 924 ret |= bdi_congested(&q->backing_dev_info, bits); 925 } 926 } 927 rcu_read_unlock(); 928 return ret; 929 } 930 EXPORT_SYMBOL_GPL(md_raid10_congested); 931 932 static int raid10_congested(void *data, int bits) 933 { 934 struct mddev *mddev = data; 935 936 return mddev_congested(mddev, bits) || 937 md_raid10_congested(mddev, bits); 938 } 939 940 static void flush_pending_writes(struct r10conf *conf) 941 { 942 /* Any writes that have been queued but are awaiting 943 * bitmap updates get flushed here. 944 */ 945 spin_lock_irq(&conf->device_lock); 946 947 if (conf->pending_bio_list.head) { 948 struct bio *bio; 949 bio = bio_list_get(&conf->pending_bio_list); 950 conf->pending_count = 0; 951 spin_unlock_irq(&conf->device_lock); 952 /* flush any pending bitmap writes to disk 953 * before proceeding w/ I/O */ 954 bitmap_unplug(conf->mddev->bitmap); 955 wake_up(&conf->wait_barrier); 956 957 while (bio) { /* submit pending writes */ 958 struct bio *next = bio->bi_next; 959 bio->bi_next = NULL; 960 if (unlikely((bio->bi_rw & REQ_DISCARD) && 961 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 962 /* Just ignore it */ 963 bio_endio(bio, 0); 964 else 965 generic_make_request(bio); 966 bio = next; 967 } 968 } else 969 spin_unlock_irq(&conf->device_lock); 970 } 971 972 /* Barriers.... 973 * Sometimes we need to suspend IO while we do something else, 974 * either some resync/recovery, or reconfigure the array. 975 * To do this we raise a 'barrier'. 976 * The 'barrier' is a counter that can be raised multiple times 977 * to count how many activities are happening which preclude 978 * normal IO. 979 * We can only raise the barrier if there is no pending IO. 980 * i.e. if nr_pending == 0. 981 * We choose only to raise the barrier if no-one is waiting for the 982 * barrier to go down. This means that as soon as an IO request 983 * is ready, no other operations which require a barrier will start 984 * until the IO request has had a chance. 985 * 986 * So: regular IO calls 'wait_barrier'. When that returns there 987 * is no backgroup IO happening, It must arrange to call 988 * allow_barrier when it has finished its IO. 989 * backgroup IO calls must call raise_barrier. Once that returns 990 * there is no normal IO happeing. It must arrange to call 991 * lower_barrier when the particular background IO completes. 992 */ 993 994 static void raise_barrier(struct r10conf *conf, int force) 995 { 996 BUG_ON(force && !conf->barrier); 997 spin_lock_irq(&conf->resync_lock); 998 999 /* Wait until no block IO is waiting (unless 'force') */ 1000 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, 1001 conf->resync_lock); 1002 1003 /* block any new IO from starting */ 1004 conf->barrier++; 1005 1006 /* Now wait for all pending IO to complete */ 1007 wait_event_lock_irq(conf->wait_barrier, 1008 !conf->nr_pending && conf->barrier < RESYNC_DEPTH, 1009 conf->resync_lock); 1010 1011 spin_unlock_irq(&conf->resync_lock); 1012 } 1013 1014 static void lower_barrier(struct r10conf *conf) 1015 { 1016 unsigned long flags; 1017 spin_lock_irqsave(&conf->resync_lock, flags); 1018 conf->barrier--; 1019 spin_unlock_irqrestore(&conf->resync_lock, flags); 1020 wake_up(&conf->wait_barrier); 1021 } 1022 1023 static void wait_barrier(struct r10conf *conf) 1024 { 1025 spin_lock_irq(&conf->resync_lock); 1026 if (conf->barrier) { 1027 conf->nr_waiting++; 1028 /* Wait for the barrier to drop. 1029 * However if there are already pending 1030 * requests (preventing the barrier from 1031 * rising completely), and the 1032 * pre-process bio queue isn't empty, 1033 * then don't wait, as we need to empty 1034 * that queue to get the nr_pending 1035 * count down. 1036 */ 1037 wait_event_lock_irq(conf->wait_barrier, 1038 !conf->barrier || 1039 (conf->nr_pending && 1040 current->bio_list && 1041 !bio_list_empty(current->bio_list)), 1042 conf->resync_lock); 1043 conf->nr_waiting--; 1044 } 1045 conf->nr_pending++; 1046 spin_unlock_irq(&conf->resync_lock); 1047 } 1048 1049 static void allow_barrier(struct r10conf *conf) 1050 { 1051 unsigned long flags; 1052 spin_lock_irqsave(&conf->resync_lock, flags); 1053 conf->nr_pending--; 1054 spin_unlock_irqrestore(&conf->resync_lock, flags); 1055 wake_up(&conf->wait_barrier); 1056 } 1057 1058 static void freeze_array(struct r10conf *conf) 1059 { 1060 /* stop syncio and normal IO and wait for everything to 1061 * go quiet. 1062 * We increment barrier and nr_waiting, and then 1063 * wait until nr_pending match nr_queued+1 1064 * This is called in the context of one normal IO request 1065 * that has failed. Thus any sync request that might be pending 1066 * will be blocked by nr_pending, and we need to wait for 1067 * pending IO requests to complete or be queued for re-try. 1068 * Thus the number queued (nr_queued) plus this request (1) 1069 * must match the number of pending IOs (nr_pending) before 1070 * we continue. 1071 */ 1072 spin_lock_irq(&conf->resync_lock); 1073 conf->barrier++; 1074 conf->nr_waiting++; 1075 wait_event_lock_irq_cmd(conf->wait_barrier, 1076 conf->nr_pending == conf->nr_queued+1, 1077 conf->resync_lock, 1078 flush_pending_writes(conf)); 1079 1080 spin_unlock_irq(&conf->resync_lock); 1081 } 1082 1083 static void unfreeze_array(struct r10conf *conf) 1084 { 1085 /* reverse the effect of the freeze */ 1086 spin_lock_irq(&conf->resync_lock); 1087 conf->barrier--; 1088 conf->nr_waiting--; 1089 wake_up(&conf->wait_barrier); 1090 spin_unlock_irq(&conf->resync_lock); 1091 } 1092 1093 static sector_t choose_data_offset(struct r10bio *r10_bio, 1094 struct md_rdev *rdev) 1095 { 1096 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 1097 test_bit(R10BIO_Previous, &r10_bio->state)) 1098 return rdev->data_offset; 1099 else 1100 return rdev->new_data_offset; 1101 } 1102 1103 struct raid10_plug_cb { 1104 struct blk_plug_cb cb; 1105 struct bio_list pending; 1106 int pending_cnt; 1107 }; 1108 1109 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1110 { 1111 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb, 1112 cb); 1113 struct mddev *mddev = plug->cb.data; 1114 struct r10conf *conf = mddev->private; 1115 struct bio *bio; 1116 1117 if (from_schedule || current->bio_list) { 1118 spin_lock_irq(&conf->device_lock); 1119 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1120 conf->pending_count += plug->pending_cnt; 1121 spin_unlock_irq(&conf->device_lock); 1122 wake_up(&conf->wait_barrier); 1123 md_wakeup_thread(mddev->thread); 1124 kfree(plug); 1125 return; 1126 } 1127 1128 /* we aren't scheduling, so we can do the write-out directly. */ 1129 bio = bio_list_get(&plug->pending); 1130 bitmap_unplug(mddev->bitmap); 1131 wake_up(&conf->wait_barrier); 1132 1133 while (bio) { /* submit pending writes */ 1134 struct bio *next = bio->bi_next; 1135 bio->bi_next = NULL; 1136 if (unlikely((bio->bi_rw & REQ_DISCARD) && 1137 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1138 /* Just ignore it */ 1139 bio_endio(bio, 0); 1140 else 1141 generic_make_request(bio); 1142 bio = next; 1143 } 1144 kfree(plug); 1145 } 1146 1147 static void make_request(struct mddev *mddev, struct bio * bio) 1148 { 1149 struct r10conf *conf = mddev->private; 1150 struct r10bio *r10_bio; 1151 struct bio *read_bio; 1152 int i; 1153 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1154 int chunk_sects = chunk_mask + 1; 1155 const int rw = bio_data_dir(bio); 1156 const unsigned long do_sync = (bio->bi_rw & REQ_SYNC); 1157 const unsigned long do_fua = (bio->bi_rw & REQ_FUA); 1158 const unsigned long do_discard = (bio->bi_rw 1159 & (REQ_DISCARD | REQ_SECURE)); 1160 const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME); 1161 unsigned long flags; 1162 struct md_rdev *blocked_rdev; 1163 struct blk_plug_cb *cb; 1164 struct raid10_plug_cb *plug = NULL; 1165 int sectors_handled; 1166 int max_sectors; 1167 int sectors; 1168 1169 if (unlikely(bio->bi_rw & REQ_FLUSH)) { 1170 md_flush_request(mddev, bio); 1171 return; 1172 } 1173 1174 /* If this request crosses a chunk boundary, we need to 1175 * split it. This will only happen for 1 PAGE (or less) requests. 1176 */ 1177 if (unlikely((bio->bi_sector & chunk_mask) + bio_sectors(bio) 1178 > chunk_sects 1179 && (conf->geo.near_copies < conf->geo.raid_disks 1180 || conf->prev.near_copies < conf->prev.raid_disks))) { 1181 struct bio_pair *bp; 1182 /* Sanity check -- queue functions should prevent this happening */ 1183 if (bio_segments(bio) > 1) 1184 goto bad_map; 1185 /* This is a one page bio that upper layers 1186 * refuse to split for us, so we need to split it. 1187 */ 1188 bp = bio_split(bio, 1189 chunk_sects - (bio->bi_sector & (chunk_sects - 1)) ); 1190 1191 /* Each of these 'make_request' calls will call 'wait_barrier'. 1192 * If the first succeeds but the second blocks due to the resync 1193 * thread raising the barrier, we will deadlock because the 1194 * IO to the underlying device will be queued in generic_make_request 1195 * and will never complete, so will never reduce nr_pending. 1196 * So increment nr_waiting here so no new raise_barriers will 1197 * succeed, and so the second wait_barrier cannot block. 1198 */ 1199 spin_lock_irq(&conf->resync_lock); 1200 conf->nr_waiting++; 1201 spin_unlock_irq(&conf->resync_lock); 1202 1203 make_request(mddev, &bp->bio1); 1204 make_request(mddev, &bp->bio2); 1205 1206 spin_lock_irq(&conf->resync_lock); 1207 conf->nr_waiting--; 1208 wake_up(&conf->wait_barrier); 1209 spin_unlock_irq(&conf->resync_lock); 1210 1211 bio_pair_release(bp); 1212 return; 1213 bad_map: 1214 printk("md/raid10:%s: make_request bug: can't convert block across chunks" 1215 " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2, 1216 (unsigned long long)bio->bi_sector, bio_sectors(bio) / 2); 1217 1218 bio_io_error(bio); 1219 return; 1220 } 1221 1222 md_write_start(mddev, bio); 1223 1224 /* 1225 * Register the new request and wait if the reconstruction 1226 * thread has put up a bar for new requests. 1227 * Continue immediately if no resync is active currently. 1228 */ 1229 wait_barrier(conf); 1230 1231 sectors = bio_sectors(bio); 1232 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1233 bio->bi_sector < conf->reshape_progress && 1234 bio->bi_sector + sectors > conf->reshape_progress) { 1235 /* IO spans the reshape position. Need to wait for 1236 * reshape to pass 1237 */ 1238 allow_barrier(conf); 1239 wait_event(conf->wait_barrier, 1240 conf->reshape_progress <= bio->bi_sector || 1241 conf->reshape_progress >= bio->bi_sector + sectors); 1242 wait_barrier(conf); 1243 } 1244 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1245 bio_data_dir(bio) == WRITE && 1246 (mddev->reshape_backwards 1247 ? (bio->bi_sector < conf->reshape_safe && 1248 bio->bi_sector + sectors > conf->reshape_progress) 1249 : (bio->bi_sector + sectors > conf->reshape_safe && 1250 bio->bi_sector < conf->reshape_progress))) { 1251 /* Need to update reshape_position in metadata */ 1252 mddev->reshape_position = conf->reshape_progress; 1253 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1254 set_bit(MD_CHANGE_PENDING, &mddev->flags); 1255 md_wakeup_thread(mddev->thread); 1256 wait_event(mddev->sb_wait, 1257 !test_bit(MD_CHANGE_PENDING, &mddev->flags)); 1258 1259 conf->reshape_safe = mddev->reshape_position; 1260 } 1261 1262 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1263 1264 r10_bio->master_bio = bio; 1265 r10_bio->sectors = sectors; 1266 1267 r10_bio->mddev = mddev; 1268 r10_bio->sector = bio->bi_sector; 1269 r10_bio->state = 0; 1270 1271 /* We might need to issue multiple reads to different 1272 * devices if there are bad blocks around, so we keep 1273 * track of the number of reads in bio->bi_phys_segments. 1274 * If this is 0, there is only one r10_bio and no locking 1275 * will be needed when the request completes. If it is 1276 * non-zero, then it is the number of not-completed requests. 1277 */ 1278 bio->bi_phys_segments = 0; 1279 clear_bit(BIO_SEG_VALID, &bio->bi_flags); 1280 1281 if (rw == READ) { 1282 /* 1283 * read balancing logic: 1284 */ 1285 struct md_rdev *rdev; 1286 int slot; 1287 1288 read_again: 1289 rdev = read_balance(conf, r10_bio, &max_sectors); 1290 if (!rdev) { 1291 raid_end_bio_io(r10_bio); 1292 return; 1293 } 1294 slot = r10_bio->read_slot; 1295 1296 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1297 md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector, 1298 max_sectors); 1299 1300 r10_bio->devs[slot].bio = read_bio; 1301 r10_bio->devs[slot].rdev = rdev; 1302 1303 read_bio->bi_sector = r10_bio->devs[slot].addr + 1304 choose_data_offset(r10_bio, rdev); 1305 read_bio->bi_bdev = rdev->bdev; 1306 read_bio->bi_end_io = raid10_end_read_request; 1307 read_bio->bi_rw = READ | do_sync; 1308 read_bio->bi_private = r10_bio; 1309 1310 if (max_sectors < r10_bio->sectors) { 1311 /* Could not read all from this device, so we will 1312 * need another r10_bio. 1313 */ 1314 sectors_handled = (r10_bio->sectors + max_sectors 1315 - bio->bi_sector); 1316 r10_bio->sectors = max_sectors; 1317 spin_lock_irq(&conf->device_lock); 1318 if (bio->bi_phys_segments == 0) 1319 bio->bi_phys_segments = 2; 1320 else 1321 bio->bi_phys_segments++; 1322 spin_unlock(&conf->device_lock); 1323 /* Cannot call generic_make_request directly 1324 * as that will be queued in __generic_make_request 1325 * and subsequent mempool_alloc might block 1326 * waiting for it. so hand bio over to raid10d. 1327 */ 1328 reschedule_retry(r10_bio); 1329 1330 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1331 1332 r10_bio->master_bio = bio; 1333 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1334 r10_bio->state = 0; 1335 r10_bio->mddev = mddev; 1336 r10_bio->sector = bio->bi_sector + sectors_handled; 1337 goto read_again; 1338 } else 1339 generic_make_request(read_bio); 1340 return; 1341 } 1342 1343 /* 1344 * WRITE: 1345 */ 1346 if (conf->pending_count >= max_queued_requests) { 1347 md_wakeup_thread(mddev->thread); 1348 wait_event(conf->wait_barrier, 1349 conf->pending_count < max_queued_requests); 1350 } 1351 /* first select target devices under rcu_lock and 1352 * inc refcount on their rdev. Record them by setting 1353 * bios[x] to bio 1354 * If there are known/acknowledged bad blocks on any device 1355 * on which we have seen a write error, we want to avoid 1356 * writing to those blocks. This potentially requires several 1357 * writes to write around the bad blocks. Each set of writes 1358 * gets its own r10_bio with a set of bios attached. The number 1359 * of r10_bios is recored in bio->bi_phys_segments just as with 1360 * the read case. 1361 */ 1362 1363 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1364 raid10_find_phys(conf, r10_bio); 1365 retry_write: 1366 blocked_rdev = NULL; 1367 rcu_read_lock(); 1368 max_sectors = r10_bio->sectors; 1369 1370 for (i = 0; i < conf->copies; i++) { 1371 int d = r10_bio->devs[i].devnum; 1372 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 1373 struct md_rdev *rrdev = rcu_dereference( 1374 conf->mirrors[d].replacement); 1375 if (rdev == rrdev) 1376 rrdev = NULL; 1377 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1378 atomic_inc(&rdev->nr_pending); 1379 blocked_rdev = rdev; 1380 break; 1381 } 1382 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) { 1383 atomic_inc(&rrdev->nr_pending); 1384 blocked_rdev = rrdev; 1385 break; 1386 } 1387 if (rdev && (test_bit(Faulty, &rdev->flags) 1388 || test_bit(Unmerged, &rdev->flags))) 1389 rdev = NULL; 1390 if (rrdev && (test_bit(Faulty, &rrdev->flags) 1391 || test_bit(Unmerged, &rrdev->flags))) 1392 rrdev = NULL; 1393 1394 r10_bio->devs[i].bio = NULL; 1395 r10_bio->devs[i].repl_bio = NULL; 1396 1397 if (!rdev && !rrdev) { 1398 set_bit(R10BIO_Degraded, &r10_bio->state); 1399 continue; 1400 } 1401 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1402 sector_t first_bad; 1403 sector_t dev_sector = r10_bio->devs[i].addr; 1404 int bad_sectors; 1405 int is_bad; 1406 1407 is_bad = is_badblock(rdev, dev_sector, 1408 max_sectors, 1409 &first_bad, &bad_sectors); 1410 if (is_bad < 0) { 1411 /* Mustn't write here until the bad block 1412 * is acknowledged 1413 */ 1414 atomic_inc(&rdev->nr_pending); 1415 set_bit(BlockedBadBlocks, &rdev->flags); 1416 blocked_rdev = rdev; 1417 break; 1418 } 1419 if (is_bad && first_bad <= dev_sector) { 1420 /* Cannot write here at all */ 1421 bad_sectors -= (dev_sector - first_bad); 1422 if (bad_sectors < max_sectors) 1423 /* Mustn't write more than bad_sectors 1424 * to other devices yet 1425 */ 1426 max_sectors = bad_sectors; 1427 /* We don't set R10BIO_Degraded as that 1428 * only applies if the disk is missing, 1429 * so it might be re-added, and we want to 1430 * know to recover this chunk. 1431 * In this case the device is here, and the 1432 * fact that this chunk is not in-sync is 1433 * recorded in the bad block log. 1434 */ 1435 continue; 1436 } 1437 if (is_bad) { 1438 int good_sectors = first_bad - dev_sector; 1439 if (good_sectors < max_sectors) 1440 max_sectors = good_sectors; 1441 } 1442 } 1443 if (rdev) { 1444 r10_bio->devs[i].bio = bio; 1445 atomic_inc(&rdev->nr_pending); 1446 } 1447 if (rrdev) { 1448 r10_bio->devs[i].repl_bio = bio; 1449 atomic_inc(&rrdev->nr_pending); 1450 } 1451 } 1452 rcu_read_unlock(); 1453 1454 if (unlikely(blocked_rdev)) { 1455 /* Have to wait for this device to get unblocked, then retry */ 1456 int j; 1457 int d; 1458 1459 for (j = 0; j < i; j++) { 1460 if (r10_bio->devs[j].bio) { 1461 d = r10_bio->devs[j].devnum; 1462 rdev_dec_pending(conf->mirrors[d].rdev, mddev); 1463 } 1464 if (r10_bio->devs[j].repl_bio) { 1465 struct md_rdev *rdev; 1466 d = r10_bio->devs[j].devnum; 1467 rdev = conf->mirrors[d].replacement; 1468 if (!rdev) { 1469 /* Race with remove_disk */ 1470 smp_mb(); 1471 rdev = conf->mirrors[d].rdev; 1472 } 1473 rdev_dec_pending(rdev, mddev); 1474 } 1475 } 1476 allow_barrier(conf); 1477 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1478 wait_barrier(conf); 1479 goto retry_write; 1480 } 1481 1482 if (max_sectors < r10_bio->sectors) { 1483 /* We are splitting this into multiple parts, so 1484 * we need to prepare for allocating another r10_bio. 1485 */ 1486 r10_bio->sectors = max_sectors; 1487 spin_lock_irq(&conf->device_lock); 1488 if (bio->bi_phys_segments == 0) 1489 bio->bi_phys_segments = 2; 1490 else 1491 bio->bi_phys_segments++; 1492 spin_unlock_irq(&conf->device_lock); 1493 } 1494 sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector; 1495 1496 atomic_set(&r10_bio->remaining, 1); 1497 bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0); 1498 1499 for (i = 0; i < conf->copies; i++) { 1500 struct bio *mbio; 1501 int d = r10_bio->devs[i].devnum; 1502 if (r10_bio->devs[i].bio) { 1503 struct md_rdev *rdev = conf->mirrors[d].rdev; 1504 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1505 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector, 1506 max_sectors); 1507 r10_bio->devs[i].bio = mbio; 1508 1509 mbio->bi_sector = (r10_bio->devs[i].addr+ 1510 choose_data_offset(r10_bio, 1511 rdev)); 1512 mbio->bi_bdev = rdev->bdev; 1513 mbio->bi_end_io = raid10_end_write_request; 1514 mbio->bi_rw = 1515 WRITE | do_sync | do_fua | do_discard | do_same; 1516 mbio->bi_private = r10_bio; 1517 1518 atomic_inc(&r10_bio->remaining); 1519 1520 cb = blk_check_plugged(raid10_unplug, mddev, 1521 sizeof(*plug)); 1522 if (cb) 1523 plug = container_of(cb, struct raid10_plug_cb, 1524 cb); 1525 else 1526 plug = NULL; 1527 spin_lock_irqsave(&conf->device_lock, flags); 1528 if (plug) { 1529 bio_list_add(&plug->pending, mbio); 1530 plug->pending_cnt++; 1531 } else { 1532 bio_list_add(&conf->pending_bio_list, mbio); 1533 conf->pending_count++; 1534 } 1535 spin_unlock_irqrestore(&conf->device_lock, flags); 1536 if (!plug) 1537 md_wakeup_thread(mddev->thread); 1538 } 1539 1540 if (r10_bio->devs[i].repl_bio) { 1541 struct md_rdev *rdev = conf->mirrors[d].replacement; 1542 if (rdev == NULL) { 1543 /* Replacement just got moved to main 'rdev' */ 1544 smp_mb(); 1545 rdev = conf->mirrors[d].rdev; 1546 } 1547 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1548 md_trim_bio(mbio, r10_bio->sector - bio->bi_sector, 1549 max_sectors); 1550 r10_bio->devs[i].repl_bio = mbio; 1551 1552 mbio->bi_sector = (r10_bio->devs[i].addr + 1553 choose_data_offset( 1554 r10_bio, rdev)); 1555 mbio->bi_bdev = rdev->bdev; 1556 mbio->bi_end_io = raid10_end_write_request; 1557 mbio->bi_rw = 1558 WRITE | do_sync | do_fua | do_discard | do_same; 1559 mbio->bi_private = r10_bio; 1560 1561 atomic_inc(&r10_bio->remaining); 1562 spin_lock_irqsave(&conf->device_lock, flags); 1563 bio_list_add(&conf->pending_bio_list, mbio); 1564 conf->pending_count++; 1565 spin_unlock_irqrestore(&conf->device_lock, flags); 1566 if (!mddev_check_plugged(mddev)) 1567 md_wakeup_thread(mddev->thread); 1568 } 1569 } 1570 1571 /* Don't remove the bias on 'remaining' (one_write_done) until 1572 * after checking if we need to go around again. 1573 */ 1574 1575 if (sectors_handled < bio_sectors(bio)) { 1576 one_write_done(r10_bio); 1577 /* We need another r10_bio. It has already been counted 1578 * in bio->bi_phys_segments. 1579 */ 1580 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1581 1582 r10_bio->master_bio = bio; 1583 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1584 1585 r10_bio->mddev = mddev; 1586 r10_bio->sector = bio->bi_sector + sectors_handled; 1587 r10_bio->state = 0; 1588 goto retry_write; 1589 } 1590 one_write_done(r10_bio); 1591 1592 /* In case raid10d snuck in to freeze_array */ 1593 wake_up(&conf->wait_barrier); 1594 } 1595 1596 static void status(struct seq_file *seq, struct mddev *mddev) 1597 { 1598 struct r10conf *conf = mddev->private; 1599 int i; 1600 1601 if (conf->geo.near_copies < conf->geo.raid_disks) 1602 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1603 if (conf->geo.near_copies > 1) 1604 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1605 if (conf->geo.far_copies > 1) { 1606 if (conf->geo.far_offset) 1607 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1608 else 1609 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1610 } 1611 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1612 conf->geo.raid_disks - mddev->degraded); 1613 for (i = 0; i < conf->geo.raid_disks; i++) 1614 seq_printf(seq, "%s", 1615 conf->mirrors[i].rdev && 1616 test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_"); 1617 seq_printf(seq, "]"); 1618 } 1619 1620 /* check if there are enough drives for 1621 * every block to appear on atleast one. 1622 * Don't consider the device numbered 'ignore' 1623 * as we might be about to remove it. 1624 */ 1625 static int _enough(struct r10conf *conf, struct geom *geo, int ignore) 1626 { 1627 int first = 0; 1628 1629 do { 1630 int n = conf->copies; 1631 int cnt = 0; 1632 int this = first; 1633 while (n--) { 1634 if (conf->mirrors[this].rdev && 1635 this != ignore) 1636 cnt++; 1637 this = (this+1) % geo->raid_disks; 1638 } 1639 if (cnt == 0) 1640 return 0; 1641 first = (first + geo->near_copies) % geo->raid_disks; 1642 } while (first != 0); 1643 return 1; 1644 } 1645 1646 static int enough(struct r10conf *conf, int ignore) 1647 { 1648 return _enough(conf, &conf->geo, ignore) && 1649 _enough(conf, &conf->prev, ignore); 1650 } 1651 1652 static void error(struct mddev *mddev, struct md_rdev *rdev) 1653 { 1654 char b[BDEVNAME_SIZE]; 1655 struct r10conf *conf = mddev->private; 1656 1657 /* 1658 * If it is not operational, then we have already marked it as dead 1659 * else if it is the last working disks, ignore the error, let the 1660 * next level up know. 1661 * else mark the drive as failed 1662 */ 1663 if (test_bit(In_sync, &rdev->flags) 1664 && !enough(conf, rdev->raid_disk)) 1665 /* 1666 * Don't fail the drive, just return an IO error. 1667 */ 1668 return; 1669 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1670 unsigned long flags; 1671 spin_lock_irqsave(&conf->device_lock, flags); 1672 mddev->degraded++; 1673 spin_unlock_irqrestore(&conf->device_lock, flags); 1674 /* 1675 * if recovery is running, make sure it aborts. 1676 */ 1677 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1678 } 1679 set_bit(Blocked, &rdev->flags); 1680 set_bit(Faulty, &rdev->flags); 1681 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1682 printk(KERN_ALERT 1683 "md/raid10:%s: Disk failure on %s, disabling device.\n" 1684 "md/raid10:%s: Operation continuing on %d devices.\n", 1685 mdname(mddev), bdevname(rdev->bdev, b), 1686 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 1687 } 1688 1689 static void print_conf(struct r10conf *conf) 1690 { 1691 int i; 1692 struct raid10_info *tmp; 1693 1694 printk(KERN_DEBUG "RAID10 conf printout:\n"); 1695 if (!conf) { 1696 printk(KERN_DEBUG "(!conf)\n"); 1697 return; 1698 } 1699 printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 1700 conf->geo.raid_disks); 1701 1702 for (i = 0; i < conf->geo.raid_disks; i++) { 1703 char b[BDEVNAME_SIZE]; 1704 tmp = conf->mirrors + i; 1705 if (tmp->rdev) 1706 printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n", 1707 i, !test_bit(In_sync, &tmp->rdev->flags), 1708 !test_bit(Faulty, &tmp->rdev->flags), 1709 bdevname(tmp->rdev->bdev,b)); 1710 } 1711 } 1712 1713 static void close_sync(struct r10conf *conf) 1714 { 1715 wait_barrier(conf); 1716 allow_barrier(conf); 1717 1718 mempool_destroy(conf->r10buf_pool); 1719 conf->r10buf_pool = NULL; 1720 } 1721 1722 static int raid10_spare_active(struct mddev *mddev) 1723 { 1724 int i; 1725 struct r10conf *conf = mddev->private; 1726 struct raid10_info *tmp; 1727 int count = 0; 1728 unsigned long flags; 1729 1730 /* 1731 * Find all non-in_sync disks within the RAID10 configuration 1732 * and mark them in_sync 1733 */ 1734 for (i = 0; i < conf->geo.raid_disks; i++) { 1735 tmp = conf->mirrors + i; 1736 if (tmp->replacement 1737 && tmp->replacement->recovery_offset == MaxSector 1738 && !test_bit(Faulty, &tmp->replacement->flags) 1739 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 1740 /* Replacement has just become active */ 1741 if (!tmp->rdev 1742 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 1743 count++; 1744 if (tmp->rdev) { 1745 /* Replaced device not technically faulty, 1746 * but we need to be sure it gets removed 1747 * and never re-added. 1748 */ 1749 set_bit(Faulty, &tmp->rdev->flags); 1750 sysfs_notify_dirent_safe( 1751 tmp->rdev->sysfs_state); 1752 } 1753 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 1754 } else if (tmp->rdev 1755 && !test_bit(Faulty, &tmp->rdev->flags) 1756 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 1757 count++; 1758 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 1759 } 1760 } 1761 spin_lock_irqsave(&conf->device_lock, flags); 1762 mddev->degraded -= count; 1763 spin_unlock_irqrestore(&conf->device_lock, flags); 1764 1765 print_conf(conf); 1766 return count; 1767 } 1768 1769 1770 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1771 { 1772 struct r10conf *conf = mddev->private; 1773 int err = -EEXIST; 1774 int mirror; 1775 int first = 0; 1776 int last = conf->geo.raid_disks - 1; 1777 struct request_queue *q = bdev_get_queue(rdev->bdev); 1778 1779 if (mddev->recovery_cp < MaxSector) 1780 /* only hot-add to in-sync arrays, as recovery is 1781 * very different from resync 1782 */ 1783 return -EBUSY; 1784 if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1)) 1785 return -EINVAL; 1786 1787 if (rdev->raid_disk >= 0) 1788 first = last = rdev->raid_disk; 1789 1790 if (q->merge_bvec_fn) { 1791 set_bit(Unmerged, &rdev->flags); 1792 mddev->merge_check_needed = 1; 1793 } 1794 1795 if (rdev->saved_raid_disk >= first && 1796 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1797 mirror = rdev->saved_raid_disk; 1798 else 1799 mirror = first; 1800 for ( ; mirror <= last ; mirror++) { 1801 struct raid10_info *p = &conf->mirrors[mirror]; 1802 if (p->recovery_disabled == mddev->recovery_disabled) 1803 continue; 1804 if (p->rdev) { 1805 if (!test_bit(WantReplacement, &p->rdev->flags) || 1806 p->replacement != NULL) 1807 continue; 1808 clear_bit(In_sync, &rdev->flags); 1809 set_bit(Replacement, &rdev->flags); 1810 rdev->raid_disk = mirror; 1811 err = 0; 1812 disk_stack_limits(mddev->gendisk, rdev->bdev, 1813 rdev->data_offset << 9); 1814 conf->fullsync = 1; 1815 rcu_assign_pointer(p->replacement, rdev); 1816 break; 1817 } 1818 1819 disk_stack_limits(mddev->gendisk, rdev->bdev, 1820 rdev->data_offset << 9); 1821 1822 p->head_position = 0; 1823 p->recovery_disabled = mddev->recovery_disabled - 1; 1824 rdev->raid_disk = mirror; 1825 err = 0; 1826 if (rdev->saved_raid_disk != mirror) 1827 conf->fullsync = 1; 1828 rcu_assign_pointer(p->rdev, rdev); 1829 break; 1830 } 1831 if (err == 0 && test_bit(Unmerged, &rdev->flags)) { 1832 /* Some requests might not have seen this new 1833 * merge_bvec_fn. We must wait for them to complete 1834 * before merging the device fully. 1835 * First we make sure any code which has tested 1836 * our function has submitted the request, then 1837 * we wait for all outstanding requests to complete. 1838 */ 1839 synchronize_sched(); 1840 raise_barrier(conf, 0); 1841 lower_barrier(conf); 1842 clear_bit(Unmerged, &rdev->flags); 1843 } 1844 md_integrity_add_rdev(rdev, mddev); 1845 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1846 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1847 1848 print_conf(conf); 1849 return err; 1850 } 1851 1852 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1853 { 1854 struct r10conf *conf = mddev->private; 1855 int err = 0; 1856 int number = rdev->raid_disk; 1857 struct md_rdev **rdevp; 1858 struct raid10_info *p = conf->mirrors + number; 1859 1860 print_conf(conf); 1861 if (rdev == p->rdev) 1862 rdevp = &p->rdev; 1863 else if (rdev == p->replacement) 1864 rdevp = &p->replacement; 1865 else 1866 return 0; 1867 1868 if (test_bit(In_sync, &rdev->flags) || 1869 atomic_read(&rdev->nr_pending)) { 1870 err = -EBUSY; 1871 goto abort; 1872 } 1873 /* Only remove faulty devices if recovery 1874 * is not possible. 1875 */ 1876 if (!test_bit(Faulty, &rdev->flags) && 1877 mddev->recovery_disabled != p->recovery_disabled && 1878 (!p->replacement || p->replacement == rdev) && 1879 number < conf->geo.raid_disks && 1880 enough(conf, -1)) { 1881 err = -EBUSY; 1882 goto abort; 1883 } 1884 *rdevp = NULL; 1885 synchronize_rcu(); 1886 if (atomic_read(&rdev->nr_pending)) { 1887 /* lost the race, try later */ 1888 err = -EBUSY; 1889 *rdevp = rdev; 1890 goto abort; 1891 } else if (p->replacement) { 1892 /* We must have just cleared 'rdev' */ 1893 p->rdev = p->replacement; 1894 clear_bit(Replacement, &p->replacement->flags); 1895 smp_mb(); /* Make sure other CPUs may see both as identical 1896 * but will never see neither -- if they are careful. 1897 */ 1898 p->replacement = NULL; 1899 clear_bit(WantReplacement, &rdev->flags); 1900 } else 1901 /* We might have just remove the Replacement as faulty 1902 * Clear the flag just in case 1903 */ 1904 clear_bit(WantReplacement, &rdev->flags); 1905 1906 err = md_integrity_register(mddev); 1907 1908 abort: 1909 1910 print_conf(conf); 1911 return err; 1912 } 1913 1914 1915 static void end_sync_read(struct bio *bio, int error) 1916 { 1917 struct r10bio *r10_bio = bio->bi_private; 1918 struct r10conf *conf = r10_bio->mddev->private; 1919 int d; 1920 1921 if (bio == r10_bio->master_bio) { 1922 /* this is a reshape read */ 1923 d = r10_bio->read_slot; /* really the read dev */ 1924 } else 1925 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1926 1927 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 1928 set_bit(R10BIO_Uptodate, &r10_bio->state); 1929 else 1930 /* The write handler will notice the lack of 1931 * R10BIO_Uptodate and record any errors etc 1932 */ 1933 atomic_add(r10_bio->sectors, 1934 &conf->mirrors[d].rdev->corrected_errors); 1935 1936 /* for reconstruct, we always reschedule after a read. 1937 * for resync, only after all reads 1938 */ 1939 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 1940 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 1941 atomic_dec_and_test(&r10_bio->remaining)) { 1942 /* we have read all the blocks, 1943 * do the comparison in process context in raid10d 1944 */ 1945 reschedule_retry(r10_bio); 1946 } 1947 } 1948 1949 static void end_sync_request(struct r10bio *r10_bio) 1950 { 1951 struct mddev *mddev = r10_bio->mddev; 1952 1953 while (atomic_dec_and_test(&r10_bio->remaining)) { 1954 if (r10_bio->master_bio == NULL) { 1955 /* the primary of several recovery bios */ 1956 sector_t s = r10_bio->sectors; 1957 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1958 test_bit(R10BIO_WriteError, &r10_bio->state)) 1959 reschedule_retry(r10_bio); 1960 else 1961 put_buf(r10_bio); 1962 md_done_sync(mddev, s, 1); 1963 break; 1964 } else { 1965 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 1966 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1967 test_bit(R10BIO_WriteError, &r10_bio->state)) 1968 reschedule_retry(r10_bio); 1969 else 1970 put_buf(r10_bio); 1971 r10_bio = r10_bio2; 1972 } 1973 } 1974 } 1975 1976 static void end_sync_write(struct bio *bio, int error) 1977 { 1978 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1979 struct r10bio *r10_bio = bio->bi_private; 1980 struct mddev *mddev = r10_bio->mddev; 1981 struct r10conf *conf = mddev->private; 1982 int d; 1983 sector_t first_bad; 1984 int bad_sectors; 1985 int slot; 1986 int repl; 1987 struct md_rdev *rdev = NULL; 1988 1989 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1990 if (repl) 1991 rdev = conf->mirrors[d].replacement; 1992 else 1993 rdev = conf->mirrors[d].rdev; 1994 1995 if (!uptodate) { 1996 if (repl) 1997 md_error(mddev, rdev); 1998 else { 1999 set_bit(WriteErrorSeen, &rdev->flags); 2000 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2001 set_bit(MD_RECOVERY_NEEDED, 2002 &rdev->mddev->recovery); 2003 set_bit(R10BIO_WriteError, &r10_bio->state); 2004 } 2005 } else if (is_badblock(rdev, 2006 r10_bio->devs[slot].addr, 2007 r10_bio->sectors, 2008 &first_bad, &bad_sectors)) 2009 set_bit(R10BIO_MadeGood, &r10_bio->state); 2010 2011 rdev_dec_pending(rdev, mddev); 2012 2013 end_sync_request(r10_bio); 2014 } 2015 2016 /* 2017 * Note: sync and recover and handled very differently for raid10 2018 * This code is for resync. 2019 * For resync, we read through virtual addresses and read all blocks. 2020 * If there is any error, we schedule a write. The lowest numbered 2021 * drive is authoritative. 2022 * However requests come for physical address, so we need to map. 2023 * For every physical address there are raid_disks/copies virtual addresses, 2024 * which is always are least one, but is not necessarly an integer. 2025 * This means that a physical address can span multiple chunks, so we may 2026 * have to submit multiple io requests for a single sync request. 2027 */ 2028 /* 2029 * We check if all blocks are in-sync and only write to blocks that 2030 * aren't in sync 2031 */ 2032 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2033 { 2034 struct r10conf *conf = mddev->private; 2035 int i, first; 2036 struct bio *tbio, *fbio; 2037 int vcnt; 2038 2039 atomic_set(&r10_bio->remaining, 1); 2040 2041 /* find the first device with a block */ 2042 for (i=0; i<conf->copies; i++) 2043 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) 2044 break; 2045 2046 if (i == conf->copies) 2047 goto done; 2048 2049 first = i; 2050 fbio = r10_bio->devs[i].bio; 2051 2052 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 2053 /* now find blocks with errors */ 2054 for (i=0 ; i < conf->copies ; i++) { 2055 int j, d; 2056 2057 tbio = r10_bio->devs[i].bio; 2058 2059 if (tbio->bi_end_io != end_sync_read) 2060 continue; 2061 if (i == first) 2062 continue; 2063 if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) { 2064 /* We know that the bi_io_vec layout is the same for 2065 * both 'first' and 'i', so we just compare them. 2066 * All vec entries are PAGE_SIZE; 2067 */ 2068 for (j = 0; j < vcnt; j++) 2069 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page), 2070 page_address(tbio->bi_io_vec[j].bv_page), 2071 fbio->bi_io_vec[j].bv_len)) 2072 break; 2073 if (j == vcnt) 2074 continue; 2075 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 2076 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 2077 /* Don't fix anything. */ 2078 continue; 2079 } 2080 /* Ok, we need to write this bio, either to correct an 2081 * inconsistency or to correct an unreadable block. 2082 * First we need to fixup bv_offset, bv_len and 2083 * bi_vecs, as the read request might have corrupted these 2084 */ 2085 bio_reset(tbio); 2086 2087 tbio->bi_vcnt = vcnt; 2088 tbio->bi_size = r10_bio->sectors << 9; 2089 tbio->bi_rw = WRITE; 2090 tbio->bi_private = r10_bio; 2091 tbio->bi_sector = r10_bio->devs[i].addr; 2092 2093 for (j=0; j < vcnt ; j++) { 2094 tbio->bi_io_vec[j].bv_offset = 0; 2095 tbio->bi_io_vec[j].bv_len = PAGE_SIZE; 2096 2097 memcpy(page_address(tbio->bi_io_vec[j].bv_page), 2098 page_address(fbio->bi_io_vec[j].bv_page), 2099 PAGE_SIZE); 2100 } 2101 tbio->bi_end_io = end_sync_write; 2102 2103 d = r10_bio->devs[i].devnum; 2104 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2105 atomic_inc(&r10_bio->remaining); 2106 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio)); 2107 2108 tbio->bi_sector += conf->mirrors[d].rdev->data_offset; 2109 tbio->bi_bdev = conf->mirrors[d].rdev->bdev; 2110 generic_make_request(tbio); 2111 } 2112 2113 /* Now write out to any replacement devices 2114 * that are active 2115 */ 2116 for (i = 0; i < conf->copies; i++) { 2117 int j, d; 2118 2119 tbio = r10_bio->devs[i].repl_bio; 2120 if (!tbio || !tbio->bi_end_io) 2121 continue; 2122 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2123 && r10_bio->devs[i].bio != fbio) 2124 for (j = 0; j < vcnt; j++) 2125 memcpy(page_address(tbio->bi_io_vec[j].bv_page), 2126 page_address(fbio->bi_io_vec[j].bv_page), 2127 PAGE_SIZE); 2128 d = r10_bio->devs[i].devnum; 2129 atomic_inc(&r10_bio->remaining); 2130 md_sync_acct(conf->mirrors[d].replacement->bdev, 2131 bio_sectors(tbio)); 2132 generic_make_request(tbio); 2133 } 2134 2135 done: 2136 if (atomic_dec_and_test(&r10_bio->remaining)) { 2137 md_done_sync(mddev, r10_bio->sectors, 1); 2138 put_buf(r10_bio); 2139 } 2140 } 2141 2142 /* 2143 * Now for the recovery code. 2144 * Recovery happens across physical sectors. 2145 * We recover all non-is_sync drives by finding the virtual address of 2146 * each, and then choose a working drive that also has that virt address. 2147 * There is a separate r10_bio for each non-in_sync drive. 2148 * Only the first two slots are in use. The first for reading, 2149 * The second for writing. 2150 * 2151 */ 2152 static void fix_recovery_read_error(struct r10bio *r10_bio) 2153 { 2154 /* We got a read error during recovery. 2155 * We repeat the read in smaller page-sized sections. 2156 * If a read succeeds, write it to the new device or record 2157 * a bad block if we cannot. 2158 * If a read fails, record a bad block on both old and 2159 * new devices. 2160 */ 2161 struct mddev *mddev = r10_bio->mddev; 2162 struct r10conf *conf = mddev->private; 2163 struct bio *bio = r10_bio->devs[0].bio; 2164 sector_t sect = 0; 2165 int sectors = r10_bio->sectors; 2166 int idx = 0; 2167 int dr = r10_bio->devs[0].devnum; 2168 int dw = r10_bio->devs[1].devnum; 2169 2170 while (sectors) { 2171 int s = sectors; 2172 struct md_rdev *rdev; 2173 sector_t addr; 2174 int ok; 2175 2176 if (s > (PAGE_SIZE>>9)) 2177 s = PAGE_SIZE >> 9; 2178 2179 rdev = conf->mirrors[dr].rdev; 2180 addr = r10_bio->devs[0].addr + sect, 2181 ok = sync_page_io(rdev, 2182 addr, 2183 s << 9, 2184 bio->bi_io_vec[idx].bv_page, 2185 READ, false); 2186 if (ok) { 2187 rdev = conf->mirrors[dw].rdev; 2188 addr = r10_bio->devs[1].addr + sect; 2189 ok = sync_page_io(rdev, 2190 addr, 2191 s << 9, 2192 bio->bi_io_vec[idx].bv_page, 2193 WRITE, false); 2194 if (!ok) { 2195 set_bit(WriteErrorSeen, &rdev->flags); 2196 if (!test_and_set_bit(WantReplacement, 2197 &rdev->flags)) 2198 set_bit(MD_RECOVERY_NEEDED, 2199 &rdev->mddev->recovery); 2200 } 2201 } 2202 if (!ok) { 2203 /* We don't worry if we cannot set a bad block - 2204 * it really is bad so there is no loss in not 2205 * recording it yet 2206 */ 2207 rdev_set_badblocks(rdev, addr, s, 0); 2208 2209 if (rdev != conf->mirrors[dw].rdev) { 2210 /* need bad block on destination too */ 2211 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2212 addr = r10_bio->devs[1].addr + sect; 2213 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2214 if (!ok) { 2215 /* just abort the recovery */ 2216 printk(KERN_NOTICE 2217 "md/raid10:%s: recovery aborted" 2218 " due to read error\n", 2219 mdname(mddev)); 2220 2221 conf->mirrors[dw].recovery_disabled 2222 = mddev->recovery_disabled; 2223 set_bit(MD_RECOVERY_INTR, 2224 &mddev->recovery); 2225 break; 2226 } 2227 } 2228 } 2229 2230 sectors -= s; 2231 sect += s; 2232 idx++; 2233 } 2234 } 2235 2236 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2237 { 2238 struct r10conf *conf = mddev->private; 2239 int d; 2240 struct bio *wbio, *wbio2; 2241 2242 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2243 fix_recovery_read_error(r10_bio); 2244 end_sync_request(r10_bio); 2245 return; 2246 } 2247 2248 /* 2249 * share the pages with the first bio 2250 * and submit the write request 2251 */ 2252 d = r10_bio->devs[1].devnum; 2253 wbio = r10_bio->devs[1].bio; 2254 wbio2 = r10_bio->devs[1].repl_bio; 2255 if (wbio->bi_end_io) { 2256 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2257 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio)); 2258 generic_make_request(wbio); 2259 } 2260 if (wbio2 && wbio2->bi_end_io) { 2261 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2262 md_sync_acct(conf->mirrors[d].replacement->bdev, 2263 bio_sectors(wbio2)); 2264 generic_make_request(wbio2); 2265 } 2266 } 2267 2268 2269 /* 2270 * Used by fix_read_error() to decay the per rdev read_errors. 2271 * We halve the read error count for every hour that has elapsed 2272 * since the last recorded read error. 2273 * 2274 */ 2275 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2276 { 2277 struct timespec cur_time_mon; 2278 unsigned long hours_since_last; 2279 unsigned int read_errors = atomic_read(&rdev->read_errors); 2280 2281 ktime_get_ts(&cur_time_mon); 2282 2283 if (rdev->last_read_error.tv_sec == 0 && 2284 rdev->last_read_error.tv_nsec == 0) { 2285 /* first time we've seen a read error */ 2286 rdev->last_read_error = cur_time_mon; 2287 return; 2288 } 2289 2290 hours_since_last = (cur_time_mon.tv_sec - 2291 rdev->last_read_error.tv_sec) / 3600; 2292 2293 rdev->last_read_error = cur_time_mon; 2294 2295 /* 2296 * if hours_since_last is > the number of bits in read_errors 2297 * just set read errors to 0. We do this to avoid 2298 * overflowing the shift of read_errors by hours_since_last. 2299 */ 2300 if (hours_since_last >= 8 * sizeof(read_errors)) 2301 atomic_set(&rdev->read_errors, 0); 2302 else 2303 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2304 } 2305 2306 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2307 int sectors, struct page *page, int rw) 2308 { 2309 sector_t first_bad; 2310 int bad_sectors; 2311 2312 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) 2313 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) 2314 return -1; 2315 if (sync_page_io(rdev, sector, sectors << 9, page, rw, false)) 2316 /* success */ 2317 return 1; 2318 if (rw == WRITE) { 2319 set_bit(WriteErrorSeen, &rdev->flags); 2320 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2321 set_bit(MD_RECOVERY_NEEDED, 2322 &rdev->mddev->recovery); 2323 } 2324 /* need to record an error - either for the block or the device */ 2325 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2326 md_error(rdev->mddev, rdev); 2327 return 0; 2328 } 2329 2330 /* 2331 * This is a kernel thread which: 2332 * 2333 * 1. Retries failed read operations on working mirrors. 2334 * 2. Updates the raid superblock when problems encounter. 2335 * 3. Performs writes following reads for array synchronising. 2336 */ 2337 2338 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2339 { 2340 int sect = 0; /* Offset from r10_bio->sector */ 2341 int sectors = r10_bio->sectors; 2342 struct md_rdev*rdev; 2343 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2344 int d = r10_bio->devs[r10_bio->read_slot].devnum; 2345 2346 /* still own a reference to this rdev, so it cannot 2347 * have been cleared recently. 2348 */ 2349 rdev = conf->mirrors[d].rdev; 2350 2351 if (test_bit(Faulty, &rdev->flags)) 2352 /* drive has already been failed, just ignore any 2353 more fix_read_error() attempts */ 2354 return; 2355 2356 check_decay_read_errors(mddev, rdev); 2357 atomic_inc(&rdev->read_errors); 2358 if (atomic_read(&rdev->read_errors) > max_read_errors) { 2359 char b[BDEVNAME_SIZE]; 2360 bdevname(rdev->bdev, b); 2361 2362 printk(KERN_NOTICE 2363 "md/raid10:%s: %s: Raid device exceeded " 2364 "read_error threshold [cur %d:max %d]\n", 2365 mdname(mddev), b, 2366 atomic_read(&rdev->read_errors), max_read_errors); 2367 printk(KERN_NOTICE 2368 "md/raid10:%s: %s: Failing raid device\n", 2369 mdname(mddev), b); 2370 md_error(mddev, conf->mirrors[d].rdev); 2371 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED; 2372 return; 2373 } 2374 2375 while(sectors) { 2376 int s = sectors; 2377 int sl = r10_bio->read_slot; 2378 int success = 0; 2379 int start; 2380 2381 if (s > (PAGE_SIZE>>9)) 2382 s = PAGE_SIZE >> 9; 2383 2384 rcu_read_lock(); 2385 do { 2386 sector_t first_bad; 2387 int bad_sectors; 2388 2389 d = r10_bio->devs[sl].devnum; 2390 rdev = rcu_dereference(conf->mirrors[d].rdev); 2391 if (rdev && 2392 !test_bit(Unmerged, &rdev->flags) && 2393 test_bit(In_sync, &rdev->flags) && 2394 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2395 &first_bad, &bad_sectors) == 0) { 2396 atomic_inc(&rdev->nr_pending); 2397 rcu_read_unlock(); 2398 success = sync_page_io(rdev, 2399 r10_bio->devs[sl].addr + 2400 sect, 2401 s<<9, 2402 conf->tmppage, READ, false); 2403 rdev_dec_pending(rdev, mddev); 2404 rcu_read_lock(); 2405 if (success) 2406 break; 2407 } 2408 sl++; 2409 if (sl == conf->copies) 2410 sl = 0; 2411 } while (!success && sl != r10_bio->read_slot); 2412 rcu_read_unlock(); 2413 2414 if (!success) { 2415 /* Cannot read from anywhere, just mark the block 2416 * as bad on the first device to discourage future 2417 * reads. 2418 */ 2419 int dn = r10_bio->devs[r10_bio->read_slot].devnum; 2420 rdev = conf->mirrors[dn].rdev; 2421 2422 if (!rdev_set_badblocks( 2423 rdev, 2424 r10_bio->devs[r10_bio->read_slot].addr 2425 + sect, 2426 s, 0)) { 2427 md_error(mddev, rdev); 2428 r10_bio->devs[r10_bio->read_slot].bio 2429 = IO_BLOCKED; 2430 } 2431 break; 2432 } 2433 2434 start = sl; 2435 /* write it back and re-read */ 2436 rcu_read_lock(); 2437 while (sl != r10_bio->read_slot) { 2438 char b[BDEVNAME_SIZE]; 2439 2440 if (sl==0) 2441 sl = conf->copies; 2442 sl--; 2443 d = r10_bio->devs[sl].devnum; 2444 rdev = rcu_dereference(conf->mirrors[d].rdev); 2445 if (!rdev || 2446 test_bit(Unmerged, &rdev->flags) || 2447 !test_bit(In_sync, &rdev->flags)) 2448 continue; 2449 2450 atomic_inc(&rdev->nr_pending); 2451 rcu_read_unlock(); 2452 if (r10_sync_page_io(rdev, 2453 r10_bio->devs[sl].addr + 2454 sect, 2455 s, conf->tmppage, WRITE) 2456 == 0) { 2457 /* Well, this device is dead */ 2458 printk(KERN_NOTICE 2459 "md/raid10:%s: read correction " 2460 "write failed" 2461 " (%d sectors at %llu on %s)\n", 2462 mdname(mddev), s, 2463 (unsigned long long)( 2464 sect + 2465 choose_data_offset(r10_bio, 2466 rdev)), 2467 bdevname(rdev->bdev, b)); 2468 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2469 "drive\n", 2470 mdname(mddev), 2471 bdevname(rdev->bdev, b)); 2472 } 2473 rdev_dec_pending(rdev, mddev); 2474 rcu_read_lock(); 2475 } 2476 sl = start; 2477 while (sl != r10_bio->read_slot) { 2478 char b[BDEVNAME_SIZE]; 2479 2480 if (sl==0) 2481 sl = conf->copies; 2482 sl--; 2483 d = r10_bio->devs[sl].devnum; 2484 rdev = rcu_dereference(conf->mirrors[d].rdev); 2485 if (!rdev || 2486 !test_bit(In_sync, &rdev->flags)) 2487 continue; 2488 2489 atomic_inc(&rdev->nr_pending); 2490 rcu_read_unlock(); 2491 switch (r10_sync_page_io(rdev, 2492 r10_bio->devs[sl].addr + 2493 sect, 2494 s, conf->tmppage, 2495 READ)) { 2496 case 0: 2497 /* Well, this device is dead */ 2498 printk(KERN_NOTICE 2499 "md/raid10:%s: unable to read back " 2500 "corrected sectors" 2501 " (%d sectors at %llu on %s)\n", 2502 mdname(mddev), s, 2503 (unsigned long long)( 2504 sect + 2505 choose_data_offset(r10_bio, rdev)), 2506 bdevname(rdev->bdev, b)); 2507 printk(KERN_NOTICE "md/raid10:%s: %s: failing " 2508 "drive\n", 2509 mdname(mddev), 2510 bdevname(rdev->bdev, b)); 2511 break; 2512 case 1: 2513 printk(KERN_INFO 2514 "md/raid10:%s: read error corrected" 2515 " (%d sectors at %llu on %s)\n", 2516 mdname(mddev), s, 2517 (unsigned long long)( 2518 sect + 2519 choose_data_offset(r10_bio, rdev)), 2520 bdevname(rdev->bdev, b)); 2521 atomic_add(s, &rdev->corrected_errors); 2522 } 2523 2524 rdev_dec_pending(rdev, mddev); 2525 rcu_read_lock(); 2526 } 2527 rcu_read_unlock(); 2528 2529 sectors -= s; 2530 sect += s; 2531 } 2532 } 2533 2534 static int narrow_write_error(struct r10bio *r10_bio, int i) 2535 { 2536 struct bio *bio = r10_bio->master_bio; 2537 struct mddev *mddev = r10_bio->mddev; 2538 struct r10conf *conf = mddev->private; 2539 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2540 /* bio has the data to be written to slot 'i' where 2541 * we just recently had a write error. 2542 * We repeatedly clone the bio and trim down to one block, 2543 * then try the write. Where the write fails we record 2544 * a bad block. 2545 * It is conceivable that the bio doesn't exactly align with 2546 * blocks. We must handle this. 2547 * 2548 * We currently own a reference to the rdev. 2549 */ 2550 2551 int block_sectors; 2552 sector_t sector; 2553 int sectors; 2554 int sect_to_write = r10_bio->sectors; 2555 int ok = 1; 2556 2557 if (rdev->badblocks.shift < 0) 2558 return 0; 2559 2560 block_sectors = 1 << rdev->badblocks.shift; 2561 sector = r10_bio->sector; 2562 sectors = ((r10_bio->sector + block_sectors) 2563 & ~(sector_t)(block_sectors - 1)) 2564 - sector; 2565 2566 while (sect_to_write) { 2567 struct bio *wbio; 2568 if (sectors > sect_to_write) 2569 sectors = sect_to_write; 2570 /* Write at 'sector' for 'sectors' */ 2571 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 2572 md_trim_bio(wbio, sector - bio->bi_sector, sectors); 2573 wbio->bi_sector = (r10_bio->devs[i].addr+ 2574 choose_data_offset(r10_bio, rdev) + 2575 (sector - r10_bio->sector)); 2576 wbio->bi_bdev = rdev->bdev; 2577 if (submit_bio_wait(WRITE, wbio) == 0) 2578 /* Failure! */ 2579 ok = rdev_set_badblocks(rdev, sector, 2580 sectors, 0) 2581 && ok; 2582 2583 bio_put(wbio); 2584 sect_to_write -= sectors; 2585 sector += sectors; 2586 sectors = block_sectors; 2587 } 2588 return ok; 2589 } 2590 2591 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2592 { 2593 int slot = r10_bio->read_slot; 2594 struct bio *bio; 2595 struct r10conf *conf = mddev->private; 2596 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2597 char b[BDEVNAME_SIZE]; 2598 unsigned long do_sync; 2599 int max_sectors; 2600 2601 /* we got a read error. Maybe the drive is bad. Maybe just 2602 * the block and we can fix it. 2603 * We freeze all other IO, and try reading the block from 2604 * other devices. When we find one, we re-write 2605 * and check it that fixes the read error. 2606 * This is all done synchronously while the array is 2607 * frozen. 2608 */ 2609 bio = r10_bio->devs[slot].bio; 2610 bdevname(bio->bi_bdev, b); 2611 bio_put(bio); 2612 r10_bio->devs[slot].bio = NULL; 2613 2614 if (mddev->ro == 0) { 2615 freeze_array(conf); 2616 fix_read_error(conf, mddev, r10_bio); 2617 unfreeze_array(conf); 2618 } else 2619 r10_bio->devs[slot].bio = IO_BLOCKED; 2620 2621 rdev_dec_pending(rdev, mddev); 2622 2623 read_more: 2624 rdev = read_balance(conf, r10_bio, &max_sectors); 2625 if (rdev == NULL) { 2626 printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O" 2627 " read error for block %llu\n", 2628 mdname(mddev), b, 2629 (unsigned long long)r10_bio->sector); 2630 raid_end_bio_io(r10_bio); 2631 return; 2632 } 2633 2634 do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC); 2635 slot = r10_bio->read_slot; 2636 printk_ratelimited( 2637 KERN_ERR 2638 "md/raid10:%s: %s: redirecting " 2639 "sector %llu to another mirror\n", 2640 mdname(mddev), 2641 bdevname(rdev->bdev, b), 2642 (unsigned long long)r10_bio->sector); 2643 bio = bio_clone_mddev(r10_bio->master_bio, 2644 GFP_NOIO, mddev); 2645 md_trim_bio(bio, 2646 r10_bio->sector - bio->bi_sector, 2647 max_sectors); 2648 r10_bio->devs[slot].bio = bio; 2649 r10_bio->devs[slot].rdev = rdev; 2650 bio->bi_sector = r10_bio->devs[slot].addr 2651 + choose_data_offset(r10_bio, rdev); 2652 bio->bi_bdev = rdev->bdev; 2653 bio->bi_rw = READ | do_sync; 2654 bio->bi_private = r10_bio; 2655 bio->bi_end_io = raid10_end_read_request; 2656 if (max_sectors < r10_bio->sectors) { 2657 /* Drat - have to split this up more */ 2658 struct bio *mbio = r10_bio->master_bio; 2659 int sectors_handled = 2660 r10_bio->sector + max_sectors 2661 - mbio->bi_sector; 2662 r10_bio->sectors = max_sectors; 2663 spin_lock_irq(&conf->device_lock); 2664 if (mbio->bi_phys_segments == 0) 2665 mbio->bi_phys_segments = 2; 2666 else 2667 mbio->bi_phys_segments++; 2668 spin_unlock_irq(&conf->device_lock); 2669 generic_make_request(bio); 2670 2671 r10_bio = mempool_alloc(conf->r10bio_pool, 2672 GFP_NOIO); 2673 r10_bio->master_bio = mbio; 2674 r10_bio->sectors = bio_sectors(mbio) - sectors_handled; 2675 r10_bio->state = 0; 2676 set_bit(R10BIO_ReadError, 2677 &r10_bio->state); 2678 r10_bio->mddev = mddev; 2679 r10_bio->sector = mbio->bi_sector 2680 + sectors_handled; 2681 2682 goto read_more; 2683 } else 2684 generic_make_request(bio); 2685 } 2686 2687 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2688 { 2689 /* Some sort of write request has finished and it 2690 * succeeded in writing where we thought there was a 2691 * bad block. So forget the bad block. 2692 * Or possibly if failed and we need to record 2693 * a bad block. 2694 */ 2695 int m; 2696 struct md_rdev *rdev; 2697 2698 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2699 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2700 for (m = 0; m < conf->copies; m++) { 2701 int dev = r10_bio->devs[m].devnum; 2702 rdev = conf->mirrors[dev].rdev; 2703 if (r10_bio->devs[m].bio == NULL) 2704 continue; 2705 if (test_bit(BIO_UPTODATE, 2706 &r10_bio->devs[m].bio->bi_flags)) { 2707 rdev_clear_badblocks( 2708 rdev, 2709 r10_bio->devs[m].addr, 2710 r10_bio->sectors, 0); 2711 } else { 2712 if (!rdev_set_badblocks( 2713 rdev, 2714 r10_bio->devs[m].addr, 2715 r10_bio->sectors, 0)) 2716 md_error(conf->mddev, rdev); 2717 } 2718 rdev = conf->mirrors[dev].replacement; 2719 if (r10_bio->devs[m].repl_bio == NULL) 2720 continue; 2721 if (test_bit(BIO_UPTODATE, 2722 &r10_bio->devs[m].repl_bio->bi_flags)) { 2723 rdev_clear_badblocks( 2724 rdev, 2725 r10_bio->devs[m].addr, 2726 r10_bio->sectors, 0); 2727 } else { 2728 if (!rdev_set_badblocks( 2729 rdev, 2730 r10_bio->devs[m].addr, 2731 r10_bio->sectors, 0)) 2732 md_error(conf->mddev, rdev); 2733 } 2734 } 2735 put_buf(r10_bio); 2736 } else { 2737 for (m = 0; m < conf->copies; m++) { 2738 int dev = r10_bio->devs[m].devnum; 2739 struct bio *bio = r10_bio->devs[m].bio; 2740 rdev = conf->mirrors[dev].rdev; 2741 if (bio == IO_MADE_GOOD) { 2742 rdev_clear_badblocks( 2743 rdev, 2744 r10_bio->devs[m].addr, 2745 r10_bio->sectors, 0); 2746 rdev_dec_pending(rdev, conf->mddev); 2747 } else if (bio != NULL && 2748 !test_bit(BIO_UPTODATE, &bio->bi_flags)) { 2749 if (!narrow_write_error(r10_bio, m)) { 2750 md_error(conf->mddev, rdev); 2751 set_bit(R10BIO_Degraded, 2752 &r10_bio->state); 2753 } 2754 rdev_dec_pending(rdev, conf->mddev); 2755 } 2756 bio = r10_bio->devs[m].repl_bio; 2757 rdev = conf->mirrors[dev].replacement; 2758 if (rdev && bio == IO_MADE_GOOD) { 2759 rdev_clear_badblocks( 2760 rdev, 2761 r10_bio->devs[m].addr, 2762 r10_bio->sectors, 0); 2763 rdev_dec_pending(rdev, conf->mddev); 2764 } 2765 } 2766 if (test_bit(R10BIO_WriteError, 2767 &r10_bio->state)) 2768 close_write(r10_bio); 2769 raid_end_bio_io(r10_bio); 2770 } 2771 } 2772 2773 static void raid10d(struct md_thread *thread) 2774 { 2775 struct mddev *mddev = thread->mddev; 2776 struct r10bio *r10_bio; 2777 unsigned long flags; 2778 struct r10conf *conf = mddev->private; 2779 struct list_head *head = &conf->retry_list; 2780 struct blk_plug plug; 2781 2782 md_check_recovery(mddev); 2783 2784 blk_start_plug(&plug); 2785 for (;;) { 2786 2787 flush_pending_writes(conf); 2788 2789 spin_lock_irqsave(&conf->device_lock, flags); 2790 if (list_empty(head)) { 2791 spin_unlock_irqrestore(&conf->device_lock, flags); 2792 break; 2793 } 2794 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 2795 list_del(head->prev); 2796 conf->nr_queued--; 2797 spin_unlock_irqrestore(&conf->device_lock, flags); 2798 2799 mddev = r10_bio->mddev; 2800 conf = mddev->private; 2801 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2802 test_bit(R10BIO_WriteError, &r10_bio->state)) 2803 handle_write_completed(conf, r10_bio); 2804 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 2805 reshape_request_write(mddev, r10_bio); 2806 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 2807 sync_request_write(mddev, r10_bio); 2808 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 2809 recovery_request_write(mddev, r10_bio); 2810 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 2811 handle_read_error(mddev, r10_bio); 2812 else { 2813 /* just a partial read to be scheduled from a 2814 * separate context 2815 */ 2816 int slot = r10_bio->read_slot; 2817 generic_make_request(r10_bio->devs[slot].bio); 2818 } 2819 2820 cond_resched(); 2821 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) 2822 md_check_recovery(mddev); 2823 } 2824 blk_finish_plug(&plug); 2825 } 2826 2827 2828 static int init_resync(struct r10conf *conf) 2829 { 2830 int buffs; 2831 int i; 2832 2833 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2834 BUG_ON(conf->r10buf_pool); 2835 conf->have_replacement = 0; 2836 for (i = 0; i < conf->geo.raid_disks; i++) 2837 if (conf->mirrors[i].replacement) 2838 conf->have_replacement = 1; 2839 conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf); 2840 if (!conf->r10buf_pool) 2841 return -ENOMEM; 2842 conf->next_resync = 0; 2843 return 0; 2844 } 2845 2846 /* 2847 * perform a "sync" on one "block" 2848 * 2849 * We need to make sure that no normal I/O request - particularly write 2850 * requests - conflict with active sync requests. 2851 * 2852 * This is achieved by tracking pending requests and a 'barrier' concept 2853 * that can be installed to exclude normal IO requests. 2854 * 2855 * Resync and recovery are handled very differently. 2856 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 2857 * 2858 * For resync, we iterate over virtual addresses, read all copies, 2859 * and update if there are differences. If only one copy is live, 2860 * skip it. 2861 * For recovery, we iterate over physical addresses, read a good 2862 * value for each non-in_sync drive, and over-write. 2863 * 2864 * So, for recovery we may have several outstanding complex requests for a 2865 * given address, one for each out-of-sync device. We model this by allocating 2866 * a number of r10_bio structures, one for each out-of-sync device. 2867 * As we setup these structures, we collect all bio's together into a list 2868 * which we then process collectively to add pages, and then process again 2869 * to pass to generic_make_request. 2870 * 2871 * The r10_bio structures are linked using a borrowed master_bio pointer. 2872 * This link is counted in ->remaining. When the r10_bio that points to NULL 2873 * has its remaining count decremented to 0, the whole complex operation 2874 * is complete. 2875 * 2876 */ 2877 2878 static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, 2879 int *skipped, int go_faster) 2880 { 2881 struct r10conf *conf = mddev->private; 2882 struct r10bio *r10_bio; 2883 struct bio *biolist = NULL, *bio; 2884 sector_t max_sector, nr_sectors; 2885 int i; 2886 int max_sync; 2887 sector_t sync_blocks; 2888 sector_t sectors_skipped = 0; 2889 int chunks_skipped = 0; 2890 sector_t chunk_mask = conf->geo.chunk_mask; 2891 2892 if (!conf->r10buf_pool) 2893 if (init_resync(conf)) 2894 return 0; 2895 2896 /* 2897 * Allow skipping a full rebuild for incremental assembly 2898 * of a clean array, like RAID1 does. 2899 */ 2900 if (mddev->bitmap == NULL && 2901 mddev->recovery_cp == MaxSector && 2902 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2903 conf->fullsync == 0) { 2904 *skipped = 1; 2905 max_sector = mddev->dev_sectors; 2906 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2907 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2908 max_sector = mddev->resync_max_sectors; 2909 return max_sector - sector_nr; 2910 } 2911 2912 skipped: 2913 max_sector = mddev->dev_sectors; 2914 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2915 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2916 max_sector = mddev->resync_max_sectors; 2917 if (sector_nr >= max_sector) { 2918 /* If we aborted, we need to abort the 2919 * sync on the 'current' bitmap chucks (there can 2920 * be several when recovering multiple devices). 2921 * as we may have started syncing it but not finished. 2922 * We can find the current address in 2923 * mddev->curr_resync, but for recovery, 2924 * we need to convert that to several 2925 * virtual addresses. 2926 */ 2927 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 2928 end_reshape(conf); 2929 return 0; 2930 } 2931 2932 if (mddev->curr_resync < max_sector) { /* aborted */ 2933 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 2934 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2935 &sync_blocks, 1); 2936 else for (i = 0; i < conf->geo.raid_disks; i++) { 2937 sector_t sect = 2938 raid10_find_virt(conf, mddev->curr_resync, i); 2939 bitmap_end_sync(mddev->bitmap, sect, 2940 &sync_blocks, 1); 2941 } 2942 } else { 2943 /* completed sync */ 2944 if ((!mddev->bitmap || conf->fullsync) 2945 && conf->have_replacement 2946 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2947 /* Completed a full sync so the replacements 2948 * are now fully recovered. 2949 */ 2950 for (i = 0; i < conf->geo.raid_disks; i++) 2951 if (conf->mirrors[i].replacement) 2952 conf->mirrors[i].replacement 2953 ->recovery_offset 2954 = MaxSector; 2955 } 2956 conf->fullsync = 0; 2957 } 2958 bitmap_close_sync(mddev->bitmap); 2959 close_sync(conf); 2960 *skipped = 1; 2961 return sectors_skipped; 2962 } 2963 2964 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2965 return reshape_request(mddev, sector_nr, skipped); 2966 2967 if (chunks_skipped >= conf->geo.raid_disks) { 2968 /* if there has been nothing to do on any drive, 2969 * then there is nothing to do at all.. 2970 */ 2971 *skipped = 1; 2972 return (max_sector - sector_nr) + sectors_skipped; 2973 } 2974 2975 if (max_sector > mddev->resync_max) 2976 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2977 2978 /* make sure whole request will fit in a chunk - if chunks 2979 * are meaningful 2980 */ 2981 if (conf->geo.near_copies < conf->geo.raid_disks && 2982 max_sector > (sector_nr | chunk_mask)) 2983 max_sector = (sector_nr | chunk_mask) + 1; 2984 /* 2985 * If there is non-resync activity waiting for us then 2986 * put in a delay to throttle resync. 2987 */ 2988 if (!go_faster && conf->nr_waiting) 2989 msleep_interruptible(1000); 2990 2991 /* Again, very different code for resync and recovery. 2992 * Both must result in an r10bio with a list of bios that 2993 * have bi_end_io, bi_sector, bi_bdev set, 2994 * and bi_private set to the r10bio. 2995 * For recovery, we may actually create several r10bios 2996 * with 2 bios in each, that correspond to the bios in the main one. 2997 * In this case, the subordinate r10bios link back through a 2998 * borrowed master_bio pointer, and the counter in the master 2999 * includes a ref from each subordinate. 3000 */ 3001 /* First, we decide what to do and set ->bi_end_io 3002 * To end_sync_read if we want to read, and 3003 * end_sync_write if we will want to write. 3004 */ 3005 3006 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 3007 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3008 /* recovery... the complicated one */ 3009 int j; 3010 r10_bio = NULL; 3011 3012 for (i = 0 ; i < conf->geo.raid_disks; i++) { 3013 int still_degraded; 3014 struct r10bio *rb2; 3015 sector_t sect; 3016 int must_sync; 3017 int any_working; 3018 struct raid10_info *mirror = &conf->mirrors[i]; 3019 3020 if ((mirror->rdev == NULL || 3021 test_bit(In_sync, &mirror->rdev->flags)) 3022 && 3023 (mirror->replacement == NULL || 3024 test_bit(Faulty, 3025 &mirror->replacement->flags))) 3026 continue; 3027 3028 still_degraded = 0; 3029 /* want to reconstruct this device */ 3030 rb2 = r10_bio; 3031 sect = raid10_find_virt(conf, sector_nr, i); 3032 if (sect >= mddev->resync_max_sectors) { 3033 /* last stripe is not complete - don't 3034 * try to recover this sector. 3035 */ 3036 continue; 3037 } 3038 /* Unless we are doing a full sync, or a replacement 3039 * we only need to recover the block if it is set in 3040 * the bitmap 3041 */ 3042 must_sync = bitmap_start_sync(mddev->bitmap, sect, 3043 &sync_blocks, 1); 3044 if (sync_blocks < max_sync) 3045 max_sync = sync_blocks; 3046 if (!must_sync && 3047 mirror->replacement == NULL && 3048 !conf->fullsync) { 3049 /* yep, skip the sync_blocks here, but don't assume 3050 * that there will never be anything to do here 3051 */ 3052 chunks_skipped = -1; 3053 continue; 3054 } 3055 3056 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3057 raise_barrier(conf, rb2 != NULL); 3058 atomic_set(&r10_bio->remaining, 0); 3059 3060 r10_bio->master_bio = (struct bio*)rb2; 3061 if (rb2) 3062 atomic_inc(&rb2->remaining); 3063 r10_bio->mddev = mddev; 3064 set_bit(R10BIO_IsRecover, &r10_bio->state); 3065 r10_bio->sector = sect; 3066 3067 raid10_find_phys(conf, r10_bio); 3068 3069 /* Need to check if the array will still be 3070 * degraded 3071 */ 3072 for (j = 0; j < conf->geo.raid_disks; j++) 3073 if (conf->mirrors[j].rdev == NULL || 3074 test_bit(Faulty, &conf->mirrors[j].rdev->flags)) { 3075 still_degraded = 1; 3076 break; 3077 } 3078 3079 must_sync = bitmap_start_sync(mddev->bitmap, sect, 3080 &sync_blocks, still_degraded); 3081 3082 any_working = 0; 3083 for (j=0; j<conf->copies;j++) { 3084 int k; 3085 int d = r10_bio->devs[j].devnum; 3086 sector_t from_addr, to_addr; 3087 struct md_rdev *rdev; 3088 sector_t sector, first_bad; 3089 int bad_sectors; 3090 if (!conf->mirrors[d].rdev || 3091 !test_bit(In_sync, &conf->mirrors[d].rdev->flags)) 3092 continue; 3093 /* This is where we read from */ 3094 any_working = 1; 3095 rdev = conf->mirrors[d].rdev; 3096 sector = r10_bio->devs[j].addr; 3097 3098 if (is_badblock(rdev, sector, max_sync, 3099 &first_bad, &bad_sectors)) { 3100 if (first_bad > sector) 3101 max_sync = first_bad - sector; 3102 else { 3103 bad_sectors -= (sector 3104 - first_bad); 3105 if (max_sync > bad_sectors) 3106 max_sync = bad_sectors; 3107 continue; 3108 } 3109 } 3110 bio = r10_bio->devs[0].bio; 3111 bio_reset(bio); 3112 bio->bi_next = biolist; 3113 biolist = bio; 3114 bio->bi_private = r10_bio; 3115 bio->bi_end_io = end_sync_read; 3116 bio->bi_rw = READ; 3117 from_addr = r10_bio->devs[j].addr; 3118 bio->bi_sector = from_addr + rdev->data_offset; 3119 bio->bi_bdev = rdev->bdev; 3120 atomic_inc(&rdev->nr_pending); 3121 /* and we write to 'i' (if not in_sync) */ 3122 3123 for (k=0; k<conf->copies; k++) 3124 if (r10_bio->devs[k].devnum == i) 3125 break; 3126 BUG_ON(k == conf->copies); 3127 to_addr = r10_bio->devs[k].addr; 3128 r10_bio->devs[0].devnum = d; 3129 r10_bio->devs[0].addr = from_addr; 3130 r10_bio->devs[1].devnum = i; 3131 r10_bio->devs[1].addr = to_addr; 3132 3133 rdev = mirror->rdev; 3134 if (!test_bit(In_sync, &rdev->flags)) { 3135 bio = r10_bio->devs[1].bio; 3136 bio_reset(bio); 3137 bio->bi_next = biolist; 3138 biolist = bio; 3139 bio->bi_private = r10_bio; 3140 bio->bi_end_io = end_sync_write; 3141 bio->bi_rw = WRITE; 3142 bio->bi_sector = to_addr 3143 + rdev->data_offset; 3144 bio->bi_bdev = rdev->bdev; 3145 atomic_inc(&r10_bio->remaining); 3146 } else 3147 r10_bio->devs[1].bio->bi_end_io = NULL; 3148 3149 /* and maybe write to replacement */ 3150 bio = r10_bio->devs[1].repl_bio; 3151 if (bio) 3152 bio->bi_end_io = NULL; 3153 rdev = mirror->replacement; 3154 /* Note: if rdev != NULL, then bio 3155 * cannot be NULL as r10buf_pool_alloc will 3156 * have allocated it. 3157 * So the second test here is pointless. 3158 * But it keeps semantic-checkers happy, and 3159 * this comment keeps human reviewers 3160 * happy. 3161 */ 3162 if (rdev == NULL || bio == NULL || 3163 test_bit(Faulty, &rdev->flags)) 3164 break; 3165 bio_reset(bio); 3166 bio->bi_next = biolist; 3167 biolist = bio; 3168 bio->bi_private = r10_bio; 3169 bio->bi_end_io = end_sync_write; 3170 bio->bi_rw = WRITE; 3171 bio->bi_sector = to_addr + rdev->data_offset; 3172 bio->bi_bdev = rdev->bdev; 3173 atomic_inc(&r10_bio->remaining); 3174 break; 3175 } 3176 if (j == conf->copies) { 3177 /* Cannot recover, so abort the recovery or 3178 * record a bad block */ 3179 put_buf(r10_bio); 3180 if (rb2) 3181 atomic_dec(&rb2->remaining); 3182 r10_bio = rb2; 3183 if (any_working) { 3184 /* problem is that there are bad blocks 3185 * on other device(s) 3186 */ 3187 int k; 3188 for (k = 0; k < conf->copies; k++) 3189 if (r10_bio->devs[k].devnum == i) 3190 break; 3191 if (!test_bit(In_sync, 3192 &mirror->rdev->flags) 3193 && !rdev_set_badblocks( 3194 mirror->rdev, 3195 r10_bio->devs[k].addr, 3196 max_sync, 0)) 3197 any_working = 0; 3198 if (mirror->replacement && 3199 !rdev_set_badblocks( 3200 mirror->replacement, 3201 r10_bio->devs[k].addr, 3202 max_sync, 0)) 3203 any_working = 0; 3204 } 3205 if (!any_working) { 3206 if (!test_and_set_bit(MD_RECOVERY_INTR, 3207 &mddev->recovery)) 3208 printk(KERN_INFO "md/raid10:%s: insufficient " 3209 "working devices for recovery.\n", 3210 mdname(mddev)); 3211 mirror->recovery_disabled 3212 = mddev->recovery_disabled; 3213 } 3214 break; 3215 } 3216 } 3217 if (biolist == NULL) { 3218 while (r10_bio) { 3219 struct r10bio *rb2 = r10_bio; 3220 r10_bio = (struct r10bio*) rb2->master_bio; 3221 rb2->master_bio = NULL; 3222 put_buf(rb2); 3223 } 3224 goto giveup; 3225 } 3226 } else { 3227 /* resync. Schedule a read for every block at this virt offset */ 3228 int count = 0; 3229 3230 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 3231 3232 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 3233 &sync_blocks, mddev->degraded) && 3234 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3235 &mddev->recovery)) { 3236 /* We can skip this block */ 3237 *skipped = 1; 3238 return sync_blocks + sectors_skipped; 3239 } 3240 if (sync_blocks < max_sync) 3241 max_sync = sync_blocks; 3242 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3243 3244 r10_bio->mddev = mddev; 3245 atomic_set(&r10_bio->remaining, 0); 3246 raise_barrier(conf, 0); 3247 conf->next_resync = sector_nr; 3248 3249 r10_bio->master_bio = NULL; 3250 r10_bio->sector = sector_nr; 3251 set_bit(R10BIO_IsSync, &r10_bio->state); 3252 raid10_find_phys(conf, r10_bio); 3253 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3254 3255 for (i = 0; i < conf->copies; i++) { 3256 int d = r10_bio->devs[i].devnum; 3257 sector_t first_bad, sector; 3258 int bad_sectors; 3259 3260 if (r10_bio->devs[i].repl_bio) 3261 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3262 3263 bio = r10_bio->devs[i].bio; 3264 bio_reset(bio); 3265 clear_bit(BIO_UPTODATE, &bio->bi_flags); 3266 if (conf->mirrors[d].rdev == NULL || 3267 test_bit(Faulty, &conf->mirrors[d].rdev->flags)) 3268 continue; 3269 sector = r10_bio->devs[i].addr; 3270 if (is_badblock(conf->mirrors[d].rdev, 3271 sector, max_sync, 3272 &first_bad, &bad_sectors)) { 3273 if (first_bad > sector) 3274 max_sync = first_bad - sector; 3275 else { 3276 bad_sectors -= (sector - first_bad); 3277 if (max_sync > bad_sectors) 3278 max_sync = bad_sectors; 3279 continue; 3280 } 3281 } 3282 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 3283 atomic_inc(&r10_bio->remaining); 3284 bio->bi_next = biolist; 3285 biolist = bio; 3286 bio->bi_private = r10_bio; 3287 bio->bi_end_io = end_sync_read; 3288 bio->bi_rw = READ; 3289 bio->bi_sector = sector + 3290 conf->mirrors[d].rdev->data_offset; 3291 bio->bi_bdev = conf->mirrors[d].rdev->bdev; 3292 count++; 3293 3294 if (conf->mirrors[d].replacement == NULL || 3295 test_bit(Faulty, 3296 &conf->mirrors[d].replacement->flags)) 3297 continue; 3298 3299 /* Need to set up for writing to the replacement */ 3300 bio = r10_bio->devs[i].repl_bio; 3301 bio_reset(bio); 3302 clear_bit(BIO_UPTODATE, &bio->bi_flags); 3303 3304 sector = r10_bio->devs[i].addr; 3305 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 3306 bio->bi_next = biolist; 3307 biolist = bio; 3308 bio->bi_private = r10_bio; 3309 bio->bi_end_io = end_sync_write; 3310 bio->bi_rw = WRITE; 3311 bio->bi_sector = sector + 3312 conf->mirrors[d].replacement->data_offset; 3313 bio->bi_bdev = conf->mirrors[d].replacement->bdev; 3314 count++; 3315 } 3316 3317 if (count < 2) { 3318 for (i=0; i<conf->copies; i++) { 3319 int d = r10_bio->devs[i].devnum; 3320 if (r10_bio->devs[i].bio->bi_end_io) 3321 rdev_dec_pending(conf->mirrors[d].rdev, 3322 mddev); 3323 if (r10_bio->devs[i].repl_bio && 3324 r10_bio->devs[i].repl_bio->bi_end_io) 3325 rdev_dec_pending( 3326 conf->mirrors[d].replacement, 3327 mddev); 3328 } 3329 put_buf(r10_bio); 3330 biolist = NULL; 3331 goto giveup; 3332 } 3333 } 3334 3335 nr_sectors = 0; 3336 if (sector_nr + max_sync < max_sector) 3337 max_sector = sector_nr + max_sync; 3338 do { 3339 struct page *page; 3340 int len = PAGE_SIZE; 3341 if (sector_nr + (len>>9) > max_sector) 3342 len = (max_sector - sector_nr) << 9; 3343 if (len == 0) 3344 break; 3345 for (bio= biolist ; bio ; bio=bio->bi_next) { 3346 struct bio *bio2; 3347 page = bio->bi_io_vec[bio->bi_vcnt].bv_page; 3348 if (bio_add_page(bio, page, len, 0)) 3349 continue; 3350 3351 /* stop here */ 3352 bio->bi_io_vec[bio->bi_vcnt].bv_page = page; 3353 for (bio2 = biolist; 3354 bio2 && bio2 != bio; 3355 bio2 = bio2->bi_next) { 3356 /* remove last page from this bio */ 3357 bio2->bi_vcnt--; 3358 bio2->bi_size -= len; 3359 bio2->bi_flags &= ~(1<< BIO_SEG_VALID); 3360 } 3361 goto bio_full; 3362 } 3363 nr_sectors += len>>9; 3364 sector_nr += len>>9; 3365 } while (biolist->bi_vcnt < RESYNC_PAGES); 3366 bio_full: 3367 r10_bio->sectors = nr_sectors; 3368 3369 while (biolist) { 3370 bio = biolist; 3371 biolist = biolist->bi_next; 3372 3373 bio->bi_next = NULL; 3374 r10_bio = bio->bi_private; 3375 r10_bio->sectors = nr_sectors; 3376 3377 if (bio->bi_end_io == end_sync_read) { 3378 md_sync_acct(bio->bi_bdev, nr_sectors); 3379 generic_make_request(bio); 3380 } 3381 } 3382 3383 if (sectors_skipped) 3384 /* pretend they weren't skipped, it makes 3385 * no important difference in this case 3386 */ 3387 md_done_sync(mddev, sectors_skipped, 1); 3388 3389 return sectors_skipped + nr_sectors; 3390 giveup: 3391 /* There is nowhere to write, so all non-sync 3392 * drives must be failed or in resync, all drives 3393 * have a bad block, so try the next chunk... 3394 */ 3395 if (sector_nr + max_sync < max_sector) 3396 max_sector = sector_nr + max_sync; 3397 3398 sectors_skipped += (max_sector - sector_nr); 3399 chunks_skipped ++; 3400 sector_nr = max_sector; 3401 goto skipped; 3402 } 3403 3404 static sector_t 3405 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3406 { 3407 sector_t size; 3408 struct r10conf *conf = mddev->private; 3409 3410 if (!raid_disks) 3411 raid_disks = min(conf->geo.raid_disks, 3412 conf->prev.raid_disks); 3413 if (!sectors) 3414 sectors = conf->dev_sectors; 3415 3416 size = sectors >> conf->geo.chunk_shift; 3417 sector_div(size, conf->geo.far_copies); 3418 size = size * raid_disks; 3419 sector_div(size, conf->geo.near_copies); 3420 3421 return size << conf->geo.chunk_shift; 3422 } 3423 3424 static void calc_sectors(struct r10conf *conf, sector_t size) 3425 { 3426 /* Calculate the number of sectors-per-device that will 3427 * actually be used, and set conf->dev_sectors and 3428 * conf->stride 3429 */ 3430 3431 size = size >> conf->geo.chunk_shift; 3432 sector_div(size, conf->geo.far_copies); 3433 size = size * conf->geo.raid_disks; 3434 sector_div(size, conf->geo.near_copies); 3435 /* 'size' is now the number of chunks in the array */ 3436 /* calculate "used chunks per device" */ 3437 size = size * conf->copies; 3438 3439 /* We need to round up when dividing by raid_disks to 3440 * get the stride size. 3441 */ 3442 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3443 3444 conf->dev_sectors = size << conf->geo.chunk_shift; 3445 3446 if (conf->geo.far_offset) 3447 conf->geo.stride = 1 << conf->geo.chunk_shift; 3448 else { 3449 sector_div(size, conf->geo.far_copies); 3450 conf->geo.stride = size << conf->geo.chunk_shift; 3451 } 3452 } 3453 3454 enum geo_type {geo_new, geo_old, geo_start}; 3455 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3456 { 3457 int nc, fc, fo; 3458 int layout, chunk, disks; 3459 switch (new) { 3460 case geo_old: 3461 layout = mddev->layout; 3462 chunk = mddev->chunk_sectors; 3463 disks = mddev->raid_disks - mddev->delta_disks; 3464 break; 3465 case geo_new: 3466 layout = mddev->new_layout; 3467 chunk = mddev->new_chunk_sectors; 3468 disks = mddev->raid_disks; 3469 break; 3470 default: /* avoid 'may be unused' warnings */ 3471 case geo_start: /* new when starting reshape - raid_disks not 3472 * updated yet. */ 3473 layout = mddev->new_layout; 3474 chunk = mddev->new_chunk_sectors; 3475 disks = mddev->raid_disks + mddev->delta_disks; 3476 break; 3477 } 3478 if (layout >> 18) 3479 return -1; 3480 if (chunk < (PAGE_SIZE >> 9) || 3481 !is_power_of_2(chunk)) 3482 return -2; 3483 nc = layout & 255; 3484 fc = (layout >> 8) & 255; 3485 fo = layout & (1<<16); 3486 geo->raid_disks = disks; 3487 geo->near_copies = nc; 3488 geo->far_copies = fc; 3489 geo->far_offset = fo; 3490 geo->far_set_size = (layout & (1<<17)) ? disks / fc : disks; 3491 geo->chunk_mask = chunk - 1; 3492 geo->chunk_shift = ffz(~chunk); 3493 return nc*fc; 3494 } 3495 3496 static struct r10conf *setup_conf(struct mddev *mddev) 3497 { 3498 struct r10conf *conf = NULL; 3499 int err = -EINVAL; 3500 struct geom geo; 3501 int copies; 3502 3503 copies = setup_geo(&geo, mddev, geo_new); 3504 3505 if (copies == -2) { 3506 printk(KERN_ERR "md/raid10:%s: chunk size must be " 3507 "at least PAGE_SIZE(%ld) and be a power of 2.\n", 3508 mdname(mddev), PAGE_SIZE); 3509 goto out; 3510 } 3511 3512 if (copies < 2 || copies > mddev->raid_disks) { 3513 printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3514 mdname(mddev), mddev->new_layout); 3515 goto out; 3516 } 3517 3518 err = -ENOMEM; 3519 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3520 if (!conf) 3521 goto out; 3522 3523 /* FIXME calc properly */ 3524 conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks + 3525 max(0,mddev->delta_disks)), 3526 GFP_KERNEL); 3527 if (!conf->mirrors) 3528 goto out; 3529 3530 conf->tmppage = alloc_page(GFP_KERNEL); 3531 if (!conf->tmppage) 3532 goto out; 3533 3534 conf->geo = geo; 3535 conf->copies = copies; 3536 conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc, 3537 r10bio_pool_free, conf); 3538 if (!conf->r10bio_pool) 3539 goto out; 3540 3541 calc_sectors(conf, mddev->dev_sectors); 3542 if (mddev->reshape_position == MaxSector) { 3543 conf->prev = conf->geo; 3544 conf->reshape_progress = MaxSector; 3545 } else { 3546 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3547 err = -EINVAL; 3548 goto out; 3549 } 3550 conf->reshape_progress = mddev->reshape_position; 3551 if (conf->prev.far_offset) 3552 conf->prev.stride = 1 << conf->prev.chunk_shift; 3553 else 3554 /* far_copies must be 1 */ 3555 conf->prev.stride = conf->dev_sectors; 3556 } 3557 spin_lock_init(&conf->device_lock); 3558 INIT_LIST_HEAD(&conf->retry_list); 3559 3560 spin_lock_init(&conf->resync_lock); 3561 init_waitqueue_head(&conf->wait_barrier); 3562 3563 conf->thread = md_register_thread(raid10d, mddev, "raid10"); 3564 if (!conf->thread) 3565 goto out; 3566 3567 conf->mddev = mddev; 3568 return conf; 3569 3570 out: 3571 if (err == -ENOMEM) 3572 printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n", 3573 mdname(mddev)); 3574 if (conf) { 3575 if (conf->r10bio_pool) 3576 mempool_destroy(conf->r10bio_pool); 3577 kfree(conf->mirrors); 3578 safe_put_page(conf->tmppage); 3579 kfree(conf); 3580 } 3581 return ERR_PTR(err); 3582 } 3583 3584 static int run(struct mddev *mddev) 3585 { 3586 struct r10conf *conf; 3587 int i, disk_idx, chunk_size; 3588 struct raid10_info *disk; 3589 struct md_rdev *rdev; 3590 sector_t size; 3591 sector_t min_offset_diff = 0; 3592 int first = 1; 3593 bool discard_supported = false; 3594 3595 if (mddev->private == NULL) { 3596 conf = setup_conf(mddev); 3597 if (IS_ERR(conf)) 3598 return PTR_ERR(conf); 3599 mddev->private = conf; 3600 } 3601 conf = mddev->private; 3602 if (!conf) 3603 goto out; 3604 3605 mddev->thread = conf->thread; 3606 conf->thread = NULL; 3607 3608 chunk_size = mddev->chunk_sectors << 9; 3609 if (mddev->queue) { 3610 blk_queue_max_discard_sectors(mddev->queue, 3611 mddev->chunk_sectors); 3612 blk_queue_max_write_same_sectors(mddev->queue, 3613 mddev->chunk_sectors); 3614 blk_queue_io_min(mddev->queue, chunk_size); 3615 if (conf->geo.raid_disks % conf->geo.near_copies) 3616 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks); 3617 else 3618 blk_queue_io_opt(mddev->queue, chunk_size * 3619 (conf->geo.raid_disks / conf->geo.near_copies)); 3620 } 3621 3622 rdev_for_each(rdev, mddev) { 3623 long long diff; 3624 struct request_queue *q; 3625 3626 disk_idx = rdev->raid_disk; 3627 if (disk_idx < 0) 3628 continue; 3629 if (disk_idx >= conf->geo.raid_disks && 3630 disk_idx >= conf->prev.raid_disks) 3631 continue; 3632 disk = conf->mirrors + disk_idx; 3633 3634 if (test_bit(Replacement, &rdev->flags)) { 3635 if (disk->replacement) 3636 goto out_free_conf; 3637 disk->replacement = rdev; 3638 } else { 3639 if (disk->rdev) 3640 goto out_free_conf; 3641 disk->rdev = rdev; 3642 } 3643 q = bdev_get_queue(rdev->bdev); 3644 if (q->merge_bvec_fn) 3645 mddev->merge_check_needed = 1; 3646 diff = (rdev->new_data_offset - rdev->data_offset); 3647 if (!mddev->reshape_backwards) 3648 diff = -diff; 3649 if (diff < 0) 3650 diff = 0; 3651 if (first || diff < min_offset_diff) 3652 min_offset_diff = diff; 3653 3654 if (mddev->gendisk) 3655 disk_stack_limits(mddev->gendisk, rdev->bdev, 3656 rdev->data_offset << 9); 3657 3658 disk->head_position = 0; 3659 3660 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3661 discard_supported = true; 3662 } 3663 3664 if (mddev->queue) { 3665 if (discard_supported) 3666 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 3667 mddev->queue); 3668 else 3669 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 3670 mddev->queue); 3671 } 3672 /* need to check that every block has at least one working mirror */ 3673 if (!enough(conf, -1)) { 3674 printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n", 3675 mdname(mddev)); 3676 goto out_free_conf; 3677 } 3678 3679 if (conf->reshape_progress != MaxSector) { 3680 /* must ensure that shape change is supported */ 3681 if (conf->geo.far_copies != 1 && 3682 conf->geo.far_offset == 0) 3683 goto out_free_conf; 3684 if (conf->prev.far_copies != 1 && 3685 conf->geo.far_offset == 0) 3686 goto out_free_conf; 3687 } 3688 3689 mddev->degraded = 0; 3690 for (i = 0; 3691 i < conf->geo.raid_disks 3692 || i < conf->prev.raid_disks; 3693 i++) { 3694 3695 disk = conf->mirrors + i; 3696 3697 if (!disk->rdev && disk->replacement) { 3698 /* The replacement is all we have - use it */ 3699 disk->rdev = disk->replacement; 3700 disk->replacement = NULL; 3701 clear_bit(Replacement, &disk->rdev->flags); 3702 } 3703 3704 if (!disk->rdev || 3705 !test_bit(In_sync, &disk->rdev->flags)) { 3706 disk->head_position = 0; 3707 mddev->degraded++; 3708 if (disk->rdev) 3709 conf->fullsync = 1; 3710 } 3711 disk->recovery_disabled = mddev->recovery_disabled - 1; 3712 } 3713 3714 if (mddev->recovery_cp != MaxSector) 3715 printk(KERN_NOTICE "md/raid10:%s: not clean" 3716 " -- starting background reconstruction\n", 3717 mdname(mddev)); 3718 printk(KERN_INFO 3719 "md/raid10:%s: active with %d out of %d devices\n", 3720 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 3721 conf->geo.raid_disks); 3722 /* 3723 * Ok, everything is just fine now 3724 */ 3725 mddev->dev_sectors = conf->dev_sectors; 3726 size = raid10_size(mddev, 0, 0); 3727 md_set_array_sectors(mddev, size); 3728 mddev->resync_max_sectors = size; 3729 3730 if (mddev->queue) { 3731 int stripe = conf->geo.raid_disks * 3732 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 3733 mddev->queue->backing_dev_info.congested_fn = raid10_congested; 3734 mddev->queue->backing_dev_info.congested_data = mddev; 3735 3736 /* Calculate max read-ahead size. 3737 * We need to readahead at least twice a whole stripe.... 3738 * maybe... 3739 */ 3740 stripe /= conf->geo.near_copies; 3741 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 3742 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 3743 blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec); 3744 } 3745 3746 3747 if (md_integrity_register(mddev)) 3748 goto out_free_conf; 3749 3750 if (conf->reshape_progress != MaxSector) { 3751 unsigned long before_length, after_length; 3752 3753 before_length = ((1 << conf->prev.chunk_shift) * 3754 conf->prev.far_copies); 3755 after_length = ((1 << conf->geo.chunk_shift) * 3756 conf->geo.far_copies); 3757 3758 if (max(before_length, after_length) > min_offset_diff) { 3759 /* This cannot work */ 3760 printk("md/raid10: offset difference not enough to continue reshape\n"); 3761 goto out_free_conf; 3762 } 3763 conf->offset_diff = min_offset_diff; 3764 3765 conf->reshape_safe = conf->reshape_progress; 3766 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 3767 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 3768 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 3769 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 3770 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 3771 "reshape"); 3772 } 3773 3774 return 0; 3775 3776 out_free_conf: 3777 md_unregister_thread(&mddev->thread); 3778 if (conf->r10bio_pool) 3779 mempool_destroy(conf->r10bio_pool); 3780 safe_put_page(conf->tmppage); 3781 kfree(conf->mirrors); 3782 kfree(conf); 3783 mddev->private = NULL; 3784 out: 3785 return -EIO; 3786 } 3787 3788 static int stop(struct mddev *mddev) 3789 { 3790 struct r10conf *conf = mddev->private; 3791 3792 raise_barrier(conf, 0); 3793 lower_barrier(conf); 3794 3795 md_unregister_thread(&mddev->thread); 3796 if (mddev->queue) 3797 /* the unplug fn references 'conf'*/ 3798 blk_sync_queue(mddev->queue); 3799 3800 if (conf->r10bio_pool) 3801 mempool_destroy(conf->r10bio_pool); 3802 safe_put_page(conf->tmppage); 3803 kfree(conf->mirrors); 3804 kfree(conf); 3805 mddev->private = NULL; 3806 return 0; 3807 } 3808 3809 static void raid10_quiesce(struct mddev *mddev, int state) 3810 { 3811 struct r10conf *conf = mddev->private; 3812 3813 switch(state) { 3814 case 1: 3815 raise_barrier(conf, 0); 3816 break; 3817 case 0: 3818 lower_barrier(conf); 3819 break; 3820 } 3821 } 3822 3823 static int raid10_resize(struct mddev *mddev, sector_t sectors) 3824 { 3825 /* Resize of 'far' arrays is not supported. 3826 * For 'near' and 'offset' arrays we can set the 3827 * number of sectors used to be an appropriate multiple 3828 * of the chunk size. 3829 * For 'offset', this is far_copies*chunksize. 3830 * For 'near' the multiplier is the LCM of 3831 * near_copies and raid_disks. 3832 * So if far_copies > 1 && !far_offset, fail. 3833 * Else find LCM(raid_disks, near_copy)*far_copies and 3834 * multiply by chunk_size. Then round to this number. 3835 * This is mostly done by raid10_size() 3836 */ 3837 struct r10conf *conf = mddev->private; 3838 sector_t oldsize, size; 3839 3840 if (mddev->reshape_position != MaxSector) 3841 return -EBUSY; 3842 3843 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 3844 return -EINVAL; 3845 3846 oldsize = raid10_size(mddev, 0, 0); 3847 size = raid10_size(mddev, sectors, 0); 3848 if (mddev->external_size && 3849 mddev->array_sectors > size) 3850 return -EINVAL; 3851 if (mddev->bitmap) { 3852 int ret = bitmap_resize(mddev->bitmap, size, 0, 0); 3853 if (ret) 3854 return ret; 3855 } 3856 md_set_array_sectors(mddev, size); 3857 set_capacity(mddev->gendisk, mddev->array_sectors); 3858 revalidate_disk(mddev->gendisk); 3859 if (sectors > mddev->dev_sectors && 3860 mddev->recovery_cp > oldsize) { 3861 mddev->recovery_cp = oldsize; 3862 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3863 } 3864 calc_sectors(conf, sectors); 3865 mddev->dev_sectors = conf->dev_sectors; 3866 mddev->resync_max_sectors = size; 3867 return 0; 3868 } 3869 3870 static void *raid10_takeover_raid0(struct mddev *mddev) 3871 { 3872 struct md_rdev *rdev; 3873 struct r10conf *conf; 3874 3875 if (mddev->degraded > 0) { 3876 printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n", 3877 mdname(mddev)); 3878 return ERR_PTR(-EINVAL); 3879 } 3880 3881 /* Set new parameters */ 3882 mddev->new_level = 10; 3883 /* new layout: far_copies = 1, near_copies = 2 */ 3884 mddev->new_layout = (1<<8) + 2; 3885 mddev->new_chunk_sectors = mddev->chunk_sectors; 3886 mddev->delta_disks = mddev->raid_disks; 3887 mddev->raid_disks *= 2; 3888 /* make sure it will be not marked as dirty */ 3889 mddev->recovery_cp = MaxSector; 3890 3891 conf = setup_conf(mddev); 3892 if (!IS_ERR(conf)) { 3893 rdev_for_each(rdev, mddev) 3894 if (rdev->raid_disk >= 0) 3895 rdev->new_raid_disk = rdev->raid_disk * 2; 3896 conf->barrier = 1; 3897 } 3898 3899 return conf; 3900 } 3901 3902 static void *raid10_takeover(struct mddev *mddev) 3903 { 3904 struct r0conf *raid0_conf; 3905 3906 /* raid10 can take over: 3907 * raid0 - providing it has only two drives 3908 */ 3909 if (mddev->level == 0) { 3910 /* for raid0 takeover only one zone is supported */ 3911 raid0_conf = mddev->private; 3912 if (raid0_conf->nr_strip_zones > 1) { 3913 printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0" 3914 " with more than one zone.\n", 3915 mdname(mddev)); 3916 return ERR_PTR(-EINVAL); 3917 } 3918 return raid10_takeover_raid0(mddev); 3919 } 3920 return ERR_PTR(-EINVAL); 3921 } 3922 3923 static int raid10_check_reshape(struct mddev *mddev) 3924 { 3925 /* Called when there is a request to change 3926 * - layout (to ->new_layout) 3927 * - chunk size (to ->new_chunk_sectors) 3928 * - raid_disks (by delta_disks) 3929 * or when trying to restart a reshape that was ongoing. 3930 * 3931 * We need to validate the request and possibly allocate 3932 * space if that might be an issue later. 3933 * 3934 * Currently we reject any reshape of a 'far' mode array, 3935 * allow chunk size to change if new is generally acceptable, 3936 * allow raid_disks to increase, and allow 3937 * a switch between 'near' mode and 'offset' mode. 3938 */ 3939 struct r10conf *conf = mddev->private; 3940 struct geom geo; 3941 3942 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 3943 return -EINVAL; 3944 3945 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 3946 /* mustn't change number of copies */ 3947 return -EINVAL; 3948 if (geo.far_copies > 1 && !geo.far_offset) 3949 /* Cannot switch to 'far' mode */ 3950 return -EINVAL; 3951 3952 if (mddev->array_sectors & geo.chunk_mask) 3953 /* not factor of array size */ 3954 return -EINVAL; 3955 3956 if (!enough(conf, -1)) 3957 return -EINVAL; 3958 3959 kfree(conf->mirrors_new); 3960 conf->mirrors_new = NULL; 3961 if (mddev->delta_disks > 0) { 3962 /* allocate new 'mirrors' list */ 3963 conf->mirrors_new = kzalloc( 3964 sizeof(struct raid10_info) 3965 *(mddev->raid_disks + 3966 mddev->delta_disks), 3967 GFP_KERNEL); 3968 if (!conf->mirrors_new) 3969 return -ENOMEM; 3970 } 3971 return 0; 3972 } 3973 3974 /* 3975 * Need to check if array has failed when deciding whether to: 3976 * - start an array 3977 * - remove non-faulty devices 3978 * - add a spare 3979 * - allow a reshape 3980 * This determination is simple when no reshape is happening. 3981 * However if there is a reshape, we need to carefully check 3982 * both the before and after sections. 3983 * This is because some failed devices may only affect one 3984 * of the two sections, and some non-in_sync devices may 3985 * be insync in the section most affected by failed devices. 3986 */ 3987 static int calc_degraded(struct r10conf *conf) 3988 { 3989 int degraded, degraded2; 3990 int i; 3991 3992 rcu_read_lock(); 3993 degraded = 0; 3994 /* 'prev' section first */ 3995 for (i = 0; i < conf->prev.raid_disks; i++) { 3996 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 3997 if (!rdev || test_bit(Faulty, &rdev->flags)) 3998 degraded++; 3999 else if (!test_bit(In_sync, &rdev->flags)) 4000 /* When we can reduce the number of devices in 4001 * an array, this might not contribute to 4002 * 'degraded'. It does now. 4003 */ 4004 degraded++; 4005 } 4006 rcu_read_unlock(); 4007 if (conf->geo.raid_disks == conf->prev.raid_disks) 4008 return degraded; 4009 rcu_read_lock(); 4010 degraded2 = 0; 4011 for (i = 0; i < conf->geo.raid_disks; i++) { 4012 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4013 if (!rdev || test_bit(Faulty, &rdev->flags)) 4014 degraded2++; 4015 else if (!test_bit(In_sync, &rdev->flags)) { 4016 /* If reshape is increasing the number of devices, 4017 * this section has already been recovered, so 4018 * it doesn't contribute to degraded. 4019 * else it does. 4020 */ 4021 if (conf->geo.raid_disks <= conf->prev.raid_disks) 4022 degraded2++; 4023 } 4024 } 4025 rcu_read_unlock(); 4026 if (degraded2 > degraded) 4027 return degraded2; 4028 return degraded; 4029 } 4030 4031 static int raid10_start_reshape(struct mddev *mddev) 4032 { 4033 /* A 'reshape' has been requested. This commits 4034 * the various 'new' fields and sets MD_RECOVER_RESHAPE 4035 * This also checks if there are enough spares and adds them 4036 * to the array. 4037 * We currently require enough spares to make the final 4038 * array non-degraded. We also require that the difference 4039 * between old and new data_offset - on each device - is 4040 * enough that we never risk over-writing. 4041 */ 4042 4043 unsigned long before_length, after_length; 4044 sector_t min_offset_diff = 0; 4045 int first = 1; 4046 struct geom new; 4047 struct r10conf *conf = mddev->private; 4048 struct md_rdev *rdev; 4049 int spares = 0; 4050 int ret; 4051 4052 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4053 return -EBUSY; 4054 4055 if (setup_geo(&new, mddev, geo_start) != conf->copies) 4056 return -EINVAL; 4057 4058 before_length = ((1 << conf->prev.chunk_shift) * 4059 conf->prev.far_copies); 4060 after_length = ((1 << conf->geo.chunk_shift) * 4061 conf->geo.far_copies); 4062 4063 rdev_for_each(rdev, mddev) { 4064 if (!test_bit(In_sync, &rdev->flags) 4065 && !test_bit(Faulty, &rdev->flags)) 4066 spares++; 4067 if (rdev->raid_disk >= 0) { 4068 long long diff = (rdev->new_data_offset 4069 - rdev->data_offset); 4070 if (!mddev->reshape_backwards) 4071 diff = -diff; 4072 if (diff < 0) 4073 diff = 0; 4074 if (first || diff < min_offset_diff) 4075 min_offset_diff = diff; 4076 } 4077 } 4078 4079 if (max(before_length, after_length) > min_offset_diff) 4080 return -EINVAL; 4081 4082 if (spares < mddev->delta_disks) 4083 return -EINVAL; 4084 4085 conf->offset_diff = min_offset_diff; 4086 spin_lock_irq(&conf->device_lock); 4087 if (conf->mirrors_new) { 4088 memcpy(conf->mirrors_new, conf->mirrors, 4089 sizeof(struct raid10_info)*conf->prev.raid_disks); 4090 smp_mb(); 4091 kfree(conf->mirrors_old); /* FIXME and elsewhere */ 4092 conf->mirrors_old = conf->mirrors; 4093 conf->mirrors = conf->mirrors_new; 4094 conf->mirrors_new = NULL; 4095 } 4096 setup_geo(&conf->geo, mddev, geo_start); 4097 smp_mb(); 4098 if (mddev->reshape_backwards) { 4099 sector_t size = raid10_size(mddev, 0, 0); 4100 if (size < mddev->array_sectors) { 4101 spin_unlock_irq(&conf->device_lock); 4102 printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n", 4103 mdname(mddev)); 4104 return -EINVAL; 4105 } 4106 mddev->resync_max_sectors = size; 4107 conf->reshape_progress = size; 4108 } else 4109 conf->reshape_progress = 0; 4110 spin_unlock_irq(&conf->device_lock); 4111 4112 if (mddev->delta_disks && mddev->bitmap) { 4113 ret = bitmap_resize(mddev->bitmap, 4114 raid10_size(mddev, 0, 4115 conf->geo.raid_disks), 4116 0, 0); 4117 if (ret) 4118 goto abort; 4119 } 4120 if (mddev->delta_disks > 0) { 4121 rdev_for_each(rdev, mddev) 4122 if (rdev->raid_disk < 0 && 4123 !test_bit(Faulty, &rdev->flags)) { 4124 if (raid10_add_disk(mddev, rdev) == 0) { 4125 if (rdev->raid_disk >= 4126 conf->prev.raid_disks) 4127 set_bit(In_sync, &rdev->flags); 4128 else 4129 rdev->recovery_offset = 0; 4130 4131 if (sysfs_link_rdev(mddev, rdev)) 4132 /* Failure here is OK */; 4133 } 4134 } else if (rdev->raid_disk >= conf->prev.raid_disks 4135 && !test_bit(Faulty, &rdev->flags)) { 4136 /* This is a spare that was manually added */ 4137 set_bit(In_sync, &rdev->flags); 4138 } 4139 } 4140 /* When a reshape changes the number of devices, 4141 * ->degraded is measured against the larger of the 4142 * pre and post numbers. 4143 */ 4144 spin_lock_irq(&conf->device_lock); 4145 mddev->degraded = calc_degraded(conf); 4146 spin_unlock_irq(&conf->device_lock); 4147 mddev->raid_disks = conf->geo.raid_disks; 4148 mddev->reshape_position = conf->reshape_progress; 4149 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4150 4151 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4152 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4153 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4154 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4155 4156 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4157 "reshape"); 4158 if (!mddev->sync_thread) { 4159 ret = -EAGAIN; 4160 goto abort; 4161 } 4162 conf->reshape_checkpoint = jiffies; 4163 md_wakeup_thread(mddev->sync_thread); 4164 md_new_event(mddev); 4165 return 0; 4166 4167 abort: 4168 mddev->recovery = 0; 4169 spin_lock_irq(&conf->device_lock); 4170 conf->geo = conf->prev; 4171 mddev->raid_disks = conf->geo.raid_disks; 4172 rdev_for_each(rdev, mddev) 4173 rdev->new_data_offset = rdev->data_offset; 4174 smp_wmb(); 4175 conf->reshape_progress = MaxSector; 4176 mddev->reshape_position = MaxSector; 4177 spin_unlock_irq(&conf->device_lock); 4178 return ret; 4179 } 4180 4181 /* Calculate the last device-address that could contain 4182 * any block from the chunk that includes the array-address 's' 4183 * and report the next address. 4184 * i.e. the address returned will be chunk-aligned and after 4185 * any data that is in the chunk containing 's'. 4186 */ 4187 static sector_t last_dev_address(sector_t s, struct geom *geo) 4188 { 4189 s = (s | geo->chunk_mask) + 1; 4190 s >>= geo->chunk_shift; 4191 s *= geo->near_copies; 4192 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4193 s *= geo->far_copies; 4194 s <<= geo->chunk_shift; 4195 return s; 4196 } 4197 4198 /* Calculate the first device-address that could contain 4199 * any block from the chunk that includes the array-address 's'. 4200 * This too will be the start of a chunk 4201 */ 4202 static sector_t first_dev_address(sector_t s, struct geom *geo) 4203 { 4204 s >>= geo->chunk_shift; 4205 s *= geo->near_copies; 4206 sector_div(s, geo->raid_disks); 4207 s *= geo->far_copies; 4208 s <<= geo->chunk_shift; 4209 return s; 4210 } 4211 4212 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4213 int *skipped) 4214 { 4215 /* We simply copy at most one chunk (smallest of old and new) 4216 * at a time, possibly less if that exceeds RESYNC_PAGES, 4217 * or we hit a bad block or something. 4218 * This might mean we pause for normal IO in the middle of 4219 * a chunk, but that is not a problem was mddev->reshape_position 4220 * can record any location. 4221 * 4222 * If we will want to write to a location that isn't 4223 * yet recorded as 'safe' (i.e. in metadata on disk) then 4224 * we need to flush all reshape requests and update the metadata. 4225 * 4226 * When reshaping forwards (e.g. to more devices), we interpret 4227 * 'safe' as the earliest block which might not have been copied 4228 * down yet. We divide this by previous stripe size and multiply 4229 * by previous stripe length to get lowest device offset that we 4230 * cannot write to yet. 4231 * We interpret 'sector_nr' as an address that we want to write to. 4232 * From this we use last_device_address() to find where we might 4233 * write to, and first_device_address on the 'safe' position. 4234 * If this 'next' write position is after the 'safe' position, 4235 * we must update the metadata to increase the 'safe' position. 4236 * 4237 * When reshaping backwards, we round in the opposite direction 4238 * and perform the reverse test: next write position must not be 4239 * less than current safe position. 4240 * 4241 * In all this the minimum difference in data offsets 4242 * (conf->offset_diff - always positive) allows a bit of slack, 4243 * so next can be after 'safe', but not by more than offset_disk 4244 * 4245 * We need to prepare all the bios here before we start any IO 4246 * to ensure the size we choose is acceptable to all devices. 4247 * The means one for each copy for write-out and an extra one for 4248 * read-in. 4249 * We store the read-in bio in ->master_bio and the others in 4250 * ->devs[x].bio and ->devs[x].repl_bio. 4251 */ 4252 struct r10conf *conf = mddev->private; 4253 struct r10bio *r10_bio; 4254 sector_t next, safe, last; 4255 int max_sectors; 4256 int nr_sectors; 4257 int s; 4258 struct md_rdev *rdev; 4259 int need_flush = 0; 4260 struct bio *blist; 4261 struct bio *bio, *read_bio; 4262 int sectors_done = 0; 4263 4264 if (sector_nr == 0) { 4265 /* If restarting in the middle, skip the initial sectors */ 4266 if (mddev->reshape_backwards && 4267 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4268 sector_nr = (raid10_size(mddev, 0, 0) 4269 - conf->reshape_progress); 4270 } else if (!mddev->reshape_backwards && 4271 conf->reshape_progress > 0) 4272 sector_nr = conf->reshape_progress; 4273 if (sector_nr) { 4274 mddev->curr_resync_completed = sector_nr; 4275 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4276 *skipped = 1; 4277 return sector_nr; 4278 } 4279 } 4280 4281 /* We don't use sector_nr to track where we are up to 4282 * as that doesn't work well for ->reshape_backwards. 4283 * So just use ->reshape_progress. 4284 */ 4285 if (mddev->reshape_backwards) { 4286 /* 'next' is the earliest device address that we might 4287 * write to for this chunk in the new layout 4288 */ 4289 next = first_dev_address(conf->reshape_progress - 1, 4290 &conf->geo); 4291 4292 /* 'safe' is the last device address that we might read from 4293 * in the old layout after a restart 4294 */ 4295 safe = last_dev_address(conf->reshape_safe - 1, 4296 &conf->prev); 4297 4298 if (next + conf->offset_diff < safe) 4299 need_flush = 1; 4300 4301 last = conf->reshape_progress - 1; 4302 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4303 & conf->prev.chunk_mask); 4304 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last) 4305 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512; 4306 } else { 4307 /* 'next' is after the last device address that we 4308 * might write to for this chunk in the new layout 4309 */ 4310 next = last_dev_address(conf->reshape_progress, &conf->geo); 4311 4312 /* 'safe' is the earliest device address that we might 4313 * read from in the old layout after a restart 4314 */ 4315 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4316 4317 /* Need to update metadata if 'next' might be beyond 'safe' 4318 * as that would possibly corrupt data 4319 */ 4320 if (next > safe + conf->offset_diff) 4321 need_flush = 1; 4322 4323 sector_nr = conf->reshape_progress; 4324 last = sector_nr | (conf->geo.chunk_mask 4325 & conf->prev.chunk_mask); 4326 4327 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last) 4328 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1; 4329 } 4330 4331 if (need_flush || 4332 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4333 /* Need to update reshape_position in metadata */ 4334 wait_barrier(conf); 4335 mddev->reshape_position = conf->reshape_progress; 4336 if (mddev->reshape_backwards) 4337 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4338 - conf->reshape_progress; 4339 else 4340 mddev->curr_resync_completed = conf->reshape_progress; 4341 conf->reshape_checkpoint = jiffies; 4342 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4343 md_wakeup_thread(mddev->thread); 4344 wait_event(mddev->sb_wait, mddev->flags == 0 || 4345 kthread_should_stop()); 4346 conf->reshape_safe = mddev->reshape_position; 4347 allow_barrier(conf); 4348 } 4349 4350 read_more: 4351 /* Now schedule reads for blocks from sector_nr to last */ 4352 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 4353 raise_barrier(conf, sectors_done != 0); 4354 atomic_set(&r10_bio->remaining, 0); 4355 r10_bio->mddev = mddev; 4356 r10_bio->sector = sector_nr; 4357 set_bit(R10BIO_IsReshape, &r10_bio->state); 4358 r10_bio->sectors = last - sector_nr + 1; 4359 rdev = read_balance(conf, r10_bio, &max_sectors); 4360 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4361 4362 if (!rdev) { 4363 /* Cannot read from here, so need to record bad blocks 4364 * on all the target devices. 4365 */ 4366 // FIXME 4367 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4368 return sectors_done; 4369 } 4370 4371 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev); 4372 4373 read_bio->bi_bdev = rdev->bdev; 4374 read_bio->bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4375 + rdev->data_offset); 4376 read_bio->bi_private = r10_bio; 4377 read_bio->bi_end_io = end_sync_read; 4378 read_bio->bi_rw = READ; 4379 read_bio->bi_flags &= ~(BIO_POOL_MASK - 1); 4380 read_bio->bi_flags |= 1 << BIO_UPTODATE; 4381 read_bio->bi_vcnt = 0; 4382 read_bio->bi_size = 0; 4383 r10_bio->master_bio = read_bio; 4384 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4385 4386 /* Now find the locations in the new layout */ 4387 __raid10_find_phys(&conf->geo, r10_bio); 4388 4389 blist = read_bio; 4390 read_bio->bi_next = NULL; 4391 4392 for (s = 0; s < conf->copies*2; s++) { 4393 struct bio *b; 4394 int d = r10_bio->devs[s/2].devnum; 4395 struct md_rdev *rdev2; 4396 if (s&1) { 4397 rdev2 = conf->mirrors[d].replacement; 4398 b = r10_bio->devs[s/2].repl_bio; 4399 } else { 4400 rdev2 = conf->mirrors[d].rdev; 4401 b = r10_bio->devs[s/2].bio; 4402 } 4403 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4404 continue; 4405 4406 bio_reset(b); 4407 b->bi_bdev = rdev2->bdev; 4408 b->bi_sector = r10_bio->devs[s/2].addr + rdev2->new_data_offset; 4409 b->bi_private = r10_bio; 4410 b->bi_end_io = end_reshape_write; 4411 b->bi_rw = WRITE; 4412 b->bi_next = blist; 4413 blist = b; 4414 } 4415 4416 /* Now add as many pages as possible to all of these bios. */ 4417 4418 nr_sectors = 0; 4419 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4420 struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page; 4421 int len = (max_sectors - s) << 9; 4422 if (len > PAGE_SIZE) 4423 len = PAGE_SIZE; 4424 for (bio = blist; bio ; bio = bio->bi_next) { 4425 struct bio *bio2; 4426 if (bio_add_page(bio, page, len, 0)) 4427 continue; 4428 4429 /* Didn't fit, must stop */ 4430 for (bio2 = blist; 4431 bio2 && bio2 != bio; 4432 bio2 = bio2->bi_next) { 4433 /* Remove last page from this bio */ 4434 bio2->bi_vcnt--; 4435 bio2->bi_size -= len; 4436 bio2->bi_flags &= ~(1<<BIO_SEG_VALID); 4437 } 4438 goto bio_full; 4439 } 4440 sector_nr += len >> 9; 4441 nr_sectors += len >> 9; 4442 } 4443 bio_full: 4444 r10_bio->sectors = nr_sectors; 4445 4446 /* Now submit the read */ 4447 md_sync_acct(read_bio->bi_bdev, r10_bio->sectors); 4448 atomic_inc(&r10_bio->remaining); 4449 read_bio->bi_next = NULL; 4450 generic_make_request(read_bio); 4451 sector_nr += nr_sectors; 4452 sectors_done += nr_sectors; 4453 if (sector_nr <= last) 4454 goto read_more; 4455 4456 /* Now that we have done the whole section we can 4457 * update reshape_progress 4458 */ 4459 if (mddev->reshape_backwards) 4460 conf->reshape_progress -= sectors_done; 4461 else 4462 conf->reshape_progress += sectors_done; 4463 4464 return sectors_done; 4465 } 4466 4467 static void end_reshape_request(struct r10bio *r10_bio); 4468 static int handle_reshape_read_error(struct mddev *mddev, 4469 struct r10bio *r10_bio); 4470 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4471 { 4472 /* Reshape read completed. Hopefully we have a block 4473 * to write out. 4474 * If we got a read error then we do sync 1-page reads from 4475 * elsewhere until we find the data - or give up. 4476 */ 4477 struct r10conf *conf = mddev->private; 4478 int s; 4479 4480 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4481 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4482 /* Reshape has been aborted */ 4483 md_done_sync(mddev, r10_bio->sectors, 0); 4484 return; 4485 } 4486 4487 /* We definitely have the data in the pages, schedule the 4488 * writes. 4489 */ 4490 atomic_set(&r10_bio->remaining, 1); 4491 for (s = 0; s < conf->copies*2; s++) { 4492 struct bio *b; 4493 int d = r10_bio->devs[s/2].devnum; 4494 struct md_rdev *rdev; 4495 if (s&1) { 4496 rdev = conf->mirrors[d].replacement; 4497 b = r10_bio->devs[s/2].repl_bio; 4498 } else { 4499 rdev = conf->mirrors[d].rdev; 4500 b = r10_bio->devs[s/2].bio; 4501 } 4502 if (!rdev || test_bit(Faulty, &rdev->flags)) 4503 continue; 4504 atomic_inc(&rdev->nr_pending); 4505 md_sync_acct(b->bi_bdev, r10_bio->sectors); 4506 atomic_inc(&r10_bio->remaining); 4507 b->bi_next = NULL; 4508 generic_make_request(b); 4509 } 4510 end_reshape_request(r10_bio); 4511 } 4512 4513 static void end_reshape(struct r10conf *conf) 4514 { 4515 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4516 return; 4517 4518 spin_lock_irq(&conf->device_lock); 4519 conf->prev = conf->geo; 4520 md_finish_reshape(conf->mddev); 4521 smp_wmb(); 4522 conf->reshape_progress = MaxSector; 4523 spin_unlock_irq(&conf->device_lock); 4524 4525 /* read-ahead size must cover two whole stripes, which is 4526 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4527 */ 4528 if (conf->mddev->queue) { 4529 int stripe = conf->geo.raid_disks * 4530 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE); 4531 stripe /= conf->geo.near_copies; 4532 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4533 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4534 } 4535 conf->fullsync = 0; 4536 } 4537 4538 4539 static int handle_reshape_read_error(struct mddev *mddev, 4540 struct r10bio *r10_bio) 4541 { 4542 /* Use sync reads to get the blocks from somewhere else */ 4543 int sectors = r10_bio->sectors; 4544 struct r10conf *conf = mddev->private; 4545 struct { 4546 struct r10bio r10_bio; 4547 struct r10dev devs[conf->copies]; 4548 } on_stack; 4549 struct r10bio *r10b = &on_stack.r10_bio; 4550 int slot = 0; 4551 int idx = 0; 4552 struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec; 4553 4554 r10b->sector = r10_bio->sector; 4555 __raid10_find_phys(&conf->prev, r10b); 4556 4557 while (sectors) { 4558 int s = sectors; 4559 int success = 0; 4560 int first_slot = slot; 4561 4562 if (s > (PAGE_SIZE >> 9)) 4563 s = PAGE_SIZE >> 9; 4564 4565 while (!success) { 4566 int d = r10b->devs[slot].devnum; 4567 struct md_rdev *rdev = conf->mirrors[d].rdev; 4568 sector_t addr; 4569 if (rdev == NULL || 4570 test_bit(Faulty, &rdev->flags) || 4571 !test_bit(In_sync, &rdev->flags)) 4572 goto failed; 4573 4574 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 4575 success = sync_page_io(rdev, 4576 addr, 4577 s << 9, 4578 bvec[idx].bv_page, 4579 READ, false); 4580 if (success) 4581 break; 4582 failed: 4583 slot++; 4584 if (slot >= conf->copies) 4585 slot = 0; 4586 if (slot == first_slot) 4587 break; 4588 } 4589 if (!success) { 4590 /* couldn't read this block, must give up */ 4591 set_bit(MD_RECOVERY_INTR, 4592 &mddev->recovery); 4593 return -EIO; 4594 } 4595 sectors -= s; 4596 idx++; 4597 } 4598 return 0; 4599 } 4600 4601 static void end_reshape_write(struct bio *bio, int error) 4602 { 4603 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 4604 struct r10bio *r10_bio = bio->bi_private; 4605 struct mddev *mddev = r10_bio->mddev; 4606 struct r10conf *conf = mddev->private; 4607 int d; 4608 int slot; 4609 int repl; 4610 struct md_rdev *rdev = NULL; 4611 4612 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 4613 if (repl) 4614 rdev = conf->mirrors[d].replacement; 4615 if (!rdev) { 4616 smp_mb(); 4617 rdev = conf->mirrors[d].rdev; 4618 } 4619 4620 if (!uptodate) { 4621 /* FIXME should record badblock */ 4622 md_error(mddev, rdev); 4623 } 4624 4625 rdev_dec_pending(rdev, mddev); 4626 end_reshape_request(r10_bio); 4627 } 4628 4629 static void end_reshape_request(struct r10bio *r10_bio) 4630 { 4631 if (!atomic_dec_and_test(&r10_bio->remaining)) 4632 return; 4633 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 4634 bio_put(r10_bio->master_bio); 4635 put_buf(r10_bio); 4636 } 4637 4638 static void raid10_finish_reshape(struct mddev *mddev) 4639 { 4640 struct r10conf *conf = mddev->private; 4641 4642 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4643 return; 4644 4645 if (mddev->delta_disks > 0) { 4646 sector_t size = raid10_size(mddev, 0, 0); 4647 md_set_array_sectors(mddev, size); 4648 if (mddev->recovery_cp > mddev->resync_max_sectors) { 4649 mddev->recovery_cp = mddev->resync_max_sectors; 4650 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4651 } 4652 mddev->resync_max_sectors = size; 4653 set_capacity(mddev->gendisk, mddev->array_sectors); 4654 revalidate_disk(mddev->gendisk); 4655 } else { 4656 int d; 4657 for (d = conf->geo.raid_disks ; 4658 d < conf->geo.raid_disks - mddev->delta_disks; 4659 d++) { 4660 struct md_rdev *rdev = conf->mirrors[d].rdev; 4661 if (rdev) 4662 clear_bit(In_sync, &rdev->flags); 4663 rdev = conf->mirrors[d].replacement; 4664 if (rdev) 4665 clear_bit(In_sync, &rdev->flags); 4666 } 4667 } 4668 mddev->layout = mddev->new_layout; 4669 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 4670 mddev->reshape_position = MaxSector; 4671 mddev->delta_disks = 0; 4672 mddev->reshape_backwards = 0; 4673 } 4674 4675 static struct md_personality raid10_personality = 4676 { 4677 .name = "raid10", 4678 .level = 10, 4679 .owner = THIS_MODULE, 4680 .make_request = make_request, 4681 .run = run, 4682 .stop = stop, 4683 .status = status, 4684 .error_handler = error, 4685 .hot_add_disk = raid10_add_disk, 4686 .hot_remove_disk= raid10_remove_disk, 4687 .spare_active = raid10_spare_active, 4688 .sync_request = sync_request, 4689 .quiesce = raid10_quiesce, 4690 .size = raid10_size, 4691 .resize = raid10_resize, 4692 .takeover = raid10_takeover, 4693 .check_reshape = raid10_check_reshape, 4694 .start_reshape = raid10_start_reshape, 4695 .finish_reshape = raid10_finish_reshape, 4696 }; 4697 4698 static int __init raid_init(void) 4699 { 4700 return register_md_personality(&raid10_personality); 4701 } 4702 4703 static void raid_exit(void) 4704 { 4705 unregister_md_personality(&raid10_personality); 4706 } 4707 4708 module_init(raid_init); 4709 module_exit(raid_exit); 4710 MODULE_LICENSE("GPL"); 4711 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 4712 MODULE_ALIAS("md-personality-9"); /* RAID10 */ 4713 MODULE_ALIAS("md-raid10"); 4714 MODULE_ALIAS("md-level-10"); 4715 4716 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 4717