1 /* 2 * raid10.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 2000-2004 Neil Brown 5 * 6 * RAID-10 support for md. 7 * 8 * Base on code in raid1.c. See raid1.c for further copyright information. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 #include <linux/slab.h> 22 #include <linux/delay.h> 23 #include <linux/blkdev.h> 24 #include <linux/module.h> 25 #include <linux/seq_file.h> 26 #include <linux/ratelimit.h> 27 #include <linux/kthread.h> 28 #include <trace/events/block.h> 29 #include "md.h" 30 #include "raid10.h" 31 #include "raid0.h" 32 #include "bitmap.h" 33 34 /* 35 * RAID10 provides a combination of RAID0 and RAID1 functionality. 36 * The layout of data is defined by 37 * chunk_size 38 * raid_disks 39 * near_copies (stored in low byte of layout) 40 * far_copies (stored in second byte of layout) 41 * far_offset (stored in bit 16 of layout ) 42 * use_far_sets (stored in bit 17 of layout ) 43 * use_far_sets_bugfixed (stored in bit 18 of layout ) 44 * 45 * The data to be stored is divided into chunks using chunksize. Each device 46 * is divided into far_copies sections. In each section, chunks are laid out 47 * in a style similar to raid0, but near_copies copies of each chunk is stored 48 * (each on a different drive). The starting device for each section is offset 49 * near_copies from the starting device of the previous section. Thus there 50 * are (near_copies * far_copies) of each chunk, and each is on a different 51 * drive. near_copies and far_copies must be at least one, and their product 52 * is at most raid_disks. 53 * 54 * If far_offset is true, then the far_copies are handled a bit differently. 55 * The copies are still in different stripes, but instead of being very far 56 * apart on disk, there are adjacent stripes. 57 * 58 * The far and offset algorithms are handled slightly differently if 59 * 'use_far_sets' is true. In this case, the array's devices are grouped into 60 * sets that are (near_copies * far_copies) in size. The far copied stripes 61 * are still shifted by 'near_copies' devices, but this shifting stays confined 62 * to the set rather than the entire array. This is done to improve the number 63 * of device combinations that can fail without causing the array to fail. 64 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk 65 * on a device): 66 * A B C D A B C D E 67 * ... ... 68 * D A B C E A B C D 69 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s): 70 * [A B] [C D] [A B] [C D E] 71 * |...| |...| |...| | ... | 72 * [B A] [D C] [B A] [E C D] 73 */ 74 75 /* 76 * Number of guaranteed r10bios in case of extreme VM load: 77 */ 78 #define NR_RAID10_BIOS 256 79 80 /* when we get a read error on a read-only array, we redirect to another 81 * device without failing the first device, or trying to over-write to 82 * correct the read error. To keep track of bad blocks on a per-bio 83 * level, we store IO_BLOCKED in the appropriate 'bios' pointer 84 */ 85 #define IO_BLOCKED ((struct bio *)1) 86 /* When we successfully write to a known bad-block, we need to remove the 87 * bad-block marking which must be done from process context. So we record 88 * the success by setting devs[n].bio to IO_MADE_GOOD 89 */ 90 #define IO_MADE_GOOD ((struct bio *)2) 91 92 #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2) 93 94 /* When there are this many requests queued to be written by 95 * the raid10 thread, we become 'congested' to provide back-pressure 96 * for writeback. 97 */ 98 static int max_queued_requests = 1024; 99 100 static void allow_barrier(struct r10conf *conf); 101 static void lower_barrier(struct r10conf *conf); 102 static int _enough(struct r10conf *conf, int previous, int ignore); 103 static int enough(struct r10conf *conf, int ignore); 104 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 105 int *skipped); 106 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio); 107 static void end_reshape_write(struct bio *bio); 108 static void end_reshape(struct r10conf *conf); 109 110 #define raid10_log(md, fmt, args...) \ 111 do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0) 112 113 static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) 114 { 115 struct r10conf *conf = data; 116 int size = offsetof(struct r10bio, devs[conf->copies]); 117 118 /* allocate a r10bio with room for raid_disks entries in the 119 * bios array */ 120 return kzalloc(size, gfp_flags); 121 } 122 123 static void r10bio_pool_free(void *r10_bio, void *data) 124 { 125 kfree(r10_bio); 126 } 127 128 /* Maximum size of each resync request */ 129 #define RESYNC_BLOCK_SIZE (64*1024) 130 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 131 /* amount of memory to reserve for resync requests */ 132 #define RESYNC_WINDOW (1024*1024) 133 /* maximum number of concurrent requests, memory permitting */ 134 #define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE) 135 136 /* 137 * When performing a resync, we need to read and compare, so 138 * we need as many pages are there are copies. 139 * When performing a recovery, we need 2 bios, one for read, 140 * one for write (we recover only one drive per r10buf) 141 * 142 */ 143 static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) 144 { 145 struct r10conf *conf = data; 146 struct page *page; 147 struct r10bio *r10_bio; 148 struct bio *bio; 149 int i, j; 150 int nalloc; 151 152 r10_bio = r10bio_pool_alloc(gfp_flags, conf); 153 if (!r10_bio) 154 return NULL; 155 156 if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) || 157 test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery)) 158 nalloc = conf->copies; /* resync */ 159 else 160 nalloc = 2; /* recovery */ 161 162 /* 163 * Allocate bios. 164 */ 165 for (j = nalloc ; j-- ; ) { 166 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 167 if (!bio) 168 goto out_free_bio; 169 r10_bio->devs[j].bio = bio; 170 if (!conf->have_replacement) 171 continue; 172 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 173 if (!bio) 174 goto out_free_bio; 175 r10_bio->devs[j].repl_bio = bio; 176 } 177 /* 178 * Allocate RESYNC_PAGES data pages and attach them 179 * where needed. 180 */ 181 for (j = 0 ; j < nalloc; j++) { 182 struct bio *rbio = r10_bio->devs[j].repl_bio; 183 bio = r10_bio->devs[j].bio; 184 for (i = 0; i < RESYNC_PAGES; i++) { 185 if (j > 0 && !test_bit(MD_RECOVERY_SYNC, 186 &conf->mddev->recovery)) { 187 /* we can share bv_page's during recovery 188 * and reshape */ 189 struct bio *rbio = r10_bio->devs[0].bio; 190 page = rbio->bi_io_vec[i].bv_page; 191 get_page(page); 192 } else 193 page = alloc_page(gfp_flags); 194 if (unlikely(!page)) 195 goto out_free_pages; 196 197 bio->bi_io_vec[i].bv_page = page; 198 if (rbio) 199 rbio->bi_io_vec[i].bv_page = page; 200 } 201 } 202 203 return r10_bio; 204 205 out_free_pages: 206 for ( ; i > 0 ; i--) 207 safe_put_page(bio->bi_io_vec[i-1].bv_page); 208 while (j--) 209 for (i = 0; i < RESYNC_PAGES ; i++) 210 safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page); 211 j = 0; 212 out_free_bio: 213 for ( ; j < nalloc; j++) { 214 if (r10_bio->devs[j].bio) 215 bio_put(r10_bio->devs[j].bio); 216 if (r10_bio->devs[j].repl_bio) 217 bio_put(r10_bio->devs[j].repl_bio); 218 } 219 r10bio_pool_free(r10_bio, conf); 220 return NULL; 221 } 222 223 static void r10buf_pool_free(void *__r10_bio, void *data) 224 { 225 int i; 226 struct r10conf *conf = data; 227 struct r10bio *r10bio = __r10_bio; 228 int j; 229 230 for (j=0; j < conf->copies; j++) { 231 struct bio *bio = r10bio->devs[j].bio; 232 if (bio) { 233 for (i = 0; i < RESYNC_PAGES; i++) { 234 safe_put_page(bio->bi_io_vec[i].bv_page); 235 bio->bi_io_vec[i].bv_page = NULL; 236 } 237 bio_put(bio); 238 } 239 bio = r10bio->devs[j].repl_bio; 240 if (bio) 241 bio_put(bio); 242 } 243 r10bio_pool_free(r10bio, conf); 244 } 245 246 static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio) 247 { 248 int i; 249 250 for (i = 0; i < conf->copies; i++) { 251 struct bio **bio = & r10_bio->devs[i].bio; 252 if (!BIO_SPECIAL(*bio)) 253 bio_put(*bio); 254 *bio = NULL; 255 bio = &r10_bio->devs[i].repl_bio; 256 if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio)) 257 bio_put(*bio); 258 *bio = NULL; 259 } 260 } 261 262 static void free_r10bio(struct r10bio *r10_bio) 263 { 264 struct r10conf *conf = r10_bio->mddev->private; 265 266 put_all_bios(conf, r10_bio); 267 mempool_free(r10_bio, conf->r10bio_pool); 268 } 269 270 static void put_buf(struct r10bio *r10_bio) 271 { 272 struct r10conf *conf = r10_bio->mddev->private; 273 274 mempool_free(r10_bio, conf->r10buf_pool); 275 276 lower_barrier(conf); 277 } 278 279 static void reschedule_retry(struct r10bio *r10_bio) 280 { 281 unsigned long flags; 282 struct mddev *mddev = r10_bio->mddev; 283 struct r10conf *conf = mddev->private; 284 285 spin_lock_irqsave(&conf->device_lock, flags); 286 list_add(&r10_bio->retry_list, &conf->retry_list); 287 conf->nr_queued ++; 288 spin_unlock_irqrestore(&conf->device_lock, flags); 289 290 /* wake up frozen array... */ 291 wake_up(&conf->wait_barrier); 292 293 md_wakeup_thread(mddev->thread); 294 } 295 296 /* 297 * raid_end_bio_io() is called when we have finished servicing a mirrored 298 * operation and are ready to return a success/failure code to the buffer 299 * cache layer. 300 */ 301 static void raid_end_bio_io(struct r10bio *r10_bio) 302 { 303 struct bio *bio = r10_bio->master_bio; 304 int done; 305 struct r10conf *conf = r10_bio->mddev->private; 306 307 if (bio->bi_phys_segments) { 308 unsigned long flags; 309 spin_lock_irqsave(&conf->device_lock, flags); 310 bio->bi_phys_segments--; 311 done = (bio->bi_phys_segments == 0); 312 spin_unlock_irqrestore(&conf->device_lock, flags); 313 } else 314 done = 1; 315 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 316 bio->bi_error = -EIO; 317 if (done) { 318 bio_endio(bio); 319 /* 320 * Wake up any possible resync thread that waits for the device 321 * to go idle. 322 */ 323 allow_barrier(conf); 324 } 325 free_r10bio(r10_bio); 326 } 327 328 /* 329 * Update disk head position estimator based on IRQ completion info. 330 */ 331 static inline void update_head_pos(int slot, struct r10bio *r10_bio) 332 { 333 struct r10conf *conf = r10_bio->mddev->private; 334 335 conf->mirrors[r10_bio->devs[slot].devnum].head_position = 336 r10_bio->devs[slot].addr + (r10_bio->sectors); 337 } 338 339 /* 340 * Find the disk number which triggered given bio 341 */ 342 static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio, 343 struct bio *bio, int *slotp, int *replp) 344 { 345 int slot; 346 int repl = 0; 347 348 for (slot = 0; slot < conf->copies; slot++) { 349 if (r10_bio->devs[slot].bio == bio) 350 break; 351 if (r10_bio->devs[slot].repl_bio == bio) { 352 repl = 1; 353 break; 354 } 355 } 356 357 BUG_ON(slot == conf->copies); 358 update_head_pos(slot, r10_bio); 359 360 if (slotp) 361 *slotp = slot; 362 if (replp) 363 *replp = repl; 364 return r10_bio->devs[slot].devnum; 365 } 366 367 static void raid10_end_read_request(struct bio *bio) 368 { 369 int uptodate = !bio->bi_error; 370 struct r10bio *r10_bio = bio->bi_private; 371 int slot, dev; 372 struct md_rdev *rdev; 373 struct r10conf *conf = r10_bio->mddev->private; 374 375 slot = r10_bio->read_slot; 376 dev = r10_bio->devs[slot].devnum; 377 rdev = r10_bio->devs[slot].rdev; 378 /* 379 * this branch is our 'one mirror IO has finished' event handler: 380 */ 381 update_head_pos(slot, r10_bio); 382 383 if (uptodate) { 384 /* 385 * Set R10BIO_Uptodate in our master bio, so that 386 * we will return a good error code to the higher 387 * levels even if IO on some other mirrored buffer fails. 388 * 389 * The 'master' represents the composite IO operation to 390 * user-side. So if something waits for IO, then it will 391 * wait for the 'master' bio. 392 */ 393 set_bit(R10BIO_Uptodate, &r10_bio->state); 394 } else { 395 /* If all other devices that store this block have 396 * failed, we want to return the error upwards rather 397 * than fail the last device. Here we redefine 398 * "uptodate" to mean "Don't want to retry" 399 */ 400 if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state), 401 rdev->raid_disk)) 402 uptodate = 1; 403 } 404 if (uptodate) { 405 raid_end_bio_io(r10_bio); 406 rdev_dec_pending(rdev, conf->mddev); 407 } else { 408 /* 409 * oops, read error - keep the refcount on the rdev 410 */ 411 char b[BDEVNAME_SIZE]; 412 pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n", 413 mdname(conf->mddev), 414 bdevname(rdev->bdev, b), 415 (unsigned long long)r10_bio->sector); 416 set_bit(R10BIO_ReadError, &r10_bio->state); 417 reschedule_retry(r10_bio); 418 } 419 } 420 421 static void close_write(struct r10bio *r10_bio) 422 { 423 /* clear the bitmap if all writes complete successfully */ 424 bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector, 425 r10_bio->sectors, 426 !test_bit(R10BIO_Degraded, &r10_bio->state), 427 0); 428 md_write_end(r10_bio->mddev); 429 } 430 431 static void one_write_done(struct r10bio *r10_bio) 432 { 433 if (atomic_dec_and_test(&r10_bio->remaining)) { 434 if (test_bit(R10BIO_WriteError, &r10_bio->state)) 435 reschedule_retry(r10_bio); 436 else { 437 close_write(r10_bio); 438 if (test_bit(R10BIO_MadeGood, &r10_bio->state)) 439 reschedule_retry(r10_bio); 440 else 441 raid_end_bio_io(r10_bio); 442 } 443 } 444 } 445 446 static void raid10_end_write_request(struct bio *bio) 447 { 448 struct r10bio *r10_bio = bio->bi_private; 449 int dev; 450 int dec_rdev = 1; 451 struct r10conf *conf = r10_bio->mddev->private; 452 int slot, repl; 453 struct md_rdev *rdev = NULL; 454 struct bio *to_put = NULL; 455 bool discard_error; 456 457 discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD; 458 459 dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 460 461 if (repl) 462 rdev = conf->mirrors[dev].replacement; 463 if (!rdev) { 464 smp_rmb(); 465 repl = 0; 466 rdev = conf->mirrors[dev].rdev; 467 } 468 /* 469 * this branch is our 'one mirror IO has finished' event handler: 470 */ 471 if (bio->bi_error && !discard_error) { 472 if (repl) 473 /* Never record new bad blocks to replacement, 474 * just fail it. 475 */ 476 md_error(rdev->mddev, rdev); 477 else { 478 set_bit(WriteErrorSeen, &rdev->flags); 479 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 480 set_bit(MD_RECOVERY_NEEDED, 481 &rdev->mddev->recovery); 482 483 dec_rdev = 0; 484 if (test_bit(FailFast, &rdev->flags) && 485 (bio->bi_opf & MD_FAILFAST)) { 486 md_error(rdev->mddev, rdev); 487 if (!test_bit(Faulty, &rdev->flags)) 488 /* This is the only remaining device, 489 * We need to retry the write without 490 * FailFast 491 */ 492 set_bit(R10BIO_WriteError, &r10_bio->state); 493 else { 494 r10_bio->devs[slot].bio = NULL; 495 to_put = bio; 496 dec_rdev = 1; 497 } 498 } else 499 set_bit(R10BIO_WriteError, &r10_bio->state); 500 } 501 } else { 502 /* 503 * Set R10BIO_Uptodate in our master bio, so that 504 * we will return a good error code for to the higher 505 * levels even if IO on some other mirrored buffer fails. 506 * 507 * The 'master' represents the composite IO operation to 508 * user-side. So if something waits for IO, then it will 509 * wait for the 'master' bio. 510 */ 511 sector_t first_bad; 512 int bad_sectors; 513 514 /* 515 * Do not set R10BIO_Uptodate if the current device is 516 * rebuilding or Faulty. This is because we cannot use 517 * such device for properly reading the data back (we could 518 * potentially use it, if the current write would have felt 519 * before rdev->recovery_offset, but for simplicity we don't 520 * check this here. 521 */ 522 if (test_bit(In_sync, &rdev->flags) && 523 !test_bit(Faulty, &rdev->flags)) 524 set_bit(R10BIO_Uptodate, &r10_bio->state); 525 526 /* Maybe we can clear some bad blocks. */ 527 if (is_badblock(rdev, 528 r10_bio->devs[slot].addr, 529 r10_bio->sectors, 530 &first_bad, &bad_sectors) && !discard_error) { 531 bio_put(bio); 532 if (repl) 533 r10_bio->devs[slot].repl_bio = IO_MADE_GOOD; 534 else 535 r10_bio->devs[slot].bio = IO_MADE_GOOD; 536 dec_rdev = 0; 537 set_bit(R10BIO_MadeGood, &r10_bio->state); 538 } 539 } 540 541 /* 542 * 543 * Let's see if all mirrored write operations have finished 544 * already. 545 */ 546 one_write_done(r10_bio); 547 if (dec_rdev) 548 rdev_dec_pending(rdev, conf->mddev); 549 if (to_put) 550 bio_put(to_put); 551 } 552 553 /* 554 * RAID10 layout manager 555 * As well as the chunksize and raid_disks count, there are two 556 * parameters: near_copies and far_copies. 557 * near_copies * far_copies must be <= raid_disks. 558 * Normally one of these will be 1. 559 * If both are 1, we get raid0. 560 * If near_copies == raid_disks, we get raid1. 561 * 562 * Chunks are laid out in raid0 style with near_copies copies of the 563 * first chunk, followed by near_copies copies of the next chunk and 564 * so on. 565 * If far_copies > 1, then after 1/far_copies of the array has been assigned 566 * as described above, we start again with a device offset of near_copies. 567 * So we effectively have another copy of the whole array further down all 568 * the drives, but with blocks on different drives. 569 * With this layout, and block is never stored twice on the one device. 570 * 571 * raid10_find_phys finds the sector offset of a given virtual sector 572 * on each device that it is on. 573 * 574 * raid10_find_virt does the reverse mapping, from a device and a 575 * sector offset to a virtual address 576 */ 577 578 static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio) 579 { 580 int n,f; 581 sector_t sector; 582 sector_t chunk; 583 sector_t stripe; 584 int dev; 585 int slot = 0; 586 int last_far_set_start, last_far_set_size; 587 588 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 589 last_far_set_start *= geo->far_set_size; 590 591 last_far_set_size = geo->far_set_size; 592 last_far_set_size += (geo->raid_disks % geo->far_set_size); 593 594 /* now calculate first sector/dev */ 595 chunk = r10bio->sector >> geo->chunk_shift; 596 sector = r10bio->sector & geo->chunk_mask; 597 598 chunk *= geo->near_copies; 599 stripe = chunk; 600 dev = sector_div(stripe, geo->raid_disks); 601 if (geo->far_offset) 602 stripe *= geo->far_copies; 603 604 sector += stripe << geo->chunk_shift; 605 606 /* and calculate all the others */ 607 for (n = 0; n < geo->near_copies; n++) { 608 int d = dev; 609 int set; 610 sector_t s = sector; 611 r10bio->devs[slot].devnum = d; 612 r10bio->devs[slot].addr = s; 613 slot++; 614 615 for (f = 1; f < geo->far_copies; f++) { 616 set = d / geo->far_set_size; 617 d += geo->near_copies; 618 619 if ((geo->raid_disks % geo->far_set_size) && 620 (d > last_far_set_start)) { 621 d -= last_far_set_start; 622 d %= last_far_set_size; 623 d += last_far_set_start; 624 } else { 625 d %= geo->far_set_size; 626 d += geo->far_set_size * set; 627 } 628 s += geo->stride; 629 r10bio->devs[slot].devnum = d; 630 r10bio->devs[slot].addr = s; 631 slot++; 632 } 633 dev++; 634 if (dev >= geo->raid_disks) { 635 dev = 0; 636 sector += (geo->chunk_mask + 1); 637 } 638 } 639 } 640 641 static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio) 642 { 643 struct geom *geo = &conf->geo; 644 645 if (conf->reshape_progress != MaxSector && 646 ((r10bio->sector >= conf->reshape_progress) != 647 conf->mddev->reshape_backwards)) { 648 set_bit(R10BIO_Previous, &r10bio->state); 649 geo = &conf->prev; 650 } else 651 clear_bit(R10BIO_Previous, &r10bio->state); 652 653 __raid10_find_phys(geo, r10bio); 654 } 655 656 static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev) 657 { 658 sector_t offset, chunk, vchunk; 659 /* Never use conf->prev as this is only called during resync 660 * or recovery, so reshape isn't happening 661 */ 662 struct geom *geo = &conf->geo; 663 int far_set_start = (dev / geo->far_set_size) * geo->far_set_size; 664 int far_set_size = geo->far_set_size; 665 int last_far_set_start; 666 667 if (geo->raid_disks % geo->far_set_size) { 668 last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1; 669 last_far_set_start *= geo->far_set_size; 670 671 if (dev >= last_far_set_start) { 672 far_set_size = geo->far_set_size; 673 far_set_size += (geo->raid_disks % geo->far_set_size); 674 far_set_start = last_far_set_start; 675 } 676 } 677 678 offset = sector & geo->chunk_mask; 679 if (geo->far_offset) { 680 int fc; 681 chunk = sector >> geo->chunk_shift; 682 fc = sector_div(chunk, geo->far_copies); 683 dev -= fc * geo->near_copies; 684 if (dev < far_set_start) 685 dev += far_set_size; 686 } else { 687 while (sector >= geo->stride) { 688 sector -= geo->stride; 689 if (dev < (geo->near_copies + far_set_start)) 690 dev += far_set_size - geo->near_copies; 691 else 692 dev -= geo->near_copies; 693 } 694 chunk = sector >> geo->chunk_shift; 695 } 696 vchunk = chunk * geo->raid_disks + dev; 697 sector_div(vchunk, geo->near_copies); 698 return (vchunk << geo->chunk_shift) + offset; 699 } 700 701 /* 702 * This routine returns the disk from which the requested read should 703 * be done. There is a per-array 'next expected sequential IO' sector 704 * number - if this matches on the next IO then we use the last disk. 705 * There is also a per-disk 'last know head position' sector that is 706 * maintained from IRQ contexts, both the normal and the resync IO 707 * completion handlers update this position correctly. If there is no 708 * perfect sequential match then we pick the disk whose head is closest. 709 * 710 * If there are 2 mirrors in the same 2 devices, performance degrades 711 * because position is mirror, not device based. 712 * 713 * The rdev for the device selected will have nr_pending incremented. 714 */ 715 716 /* 717 * FIXME: possibly should rethink readbalancing and do it differently 718 * depending on near_copies / far_copies geometry. 719 */ 720 static struct md_rdev *read_balance(struct r10conf *conf, 721 struct r10bio *r10_bio, 722 int *max_sectors) 723 { 724 const sector_t this_sector = r10_bio->sector; 725 int disk, slot; 726 int sectors = r10_bio->sectors; 727 int best_good_sectors; 728 sector_t new_distance, best_dist; 729 struct md_rdev *best_rdev, *rdev = NULL; 730 int do_balance; 731 int best_slot; 732 struct geom *geo = &conf->geo; 733 734 raid10_find_phys(conf, r10_bio); 735 rcu_read_lock(); 736 sectors = r10_bio->sectors; 737 best_slot = -1; 738 best_rdev = NULL; 739 best_dist = MaxSector; 740 best_good_sectors = 0; 741 do_balance = 1; 742 clear_bit(R10BIO_FailFast, &r10_bio->state); 743 /* 744 * Check if we can balance. We can balance on the whole 745 * device if no resync is going on (recovery is ok), or below 746 * the resync window. We take the first readable disk when 747 * above the resync window. 748 */ 749 if (conf->mddev->recovery_cp < MaxSector 750 && (this_sector + sectors >= conf->next_resync)) 751 do_balance = 0; 752 753 for (slot = 0; slot < conf->copies ; slot++) { 754 sector_t first_bad; 755 int bad_sectors; 756 sector_t dev_sector; 757 758 if (r10_bio->devs[slot].bio == IO_BLOCKED) 759 continue; 760 disk = r10_bio->devs[slot].devnum; 761 rdev = rcu_dereference(conf->mirrors[disk].replacement); 762 if (rdev == NULL || test_bit(Faulty, &rdev->flags) || 763 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 764 rdev = rcu_dereference(conf->mirrors[disk].rdev); 765 if (rdev == NULL || 766 test_bit(Faulty, &rdev->flags)) 767 continue; 768 if (!test_bit(In_sync, &rdev->flags) && 769 r10_bio->devs[slot].addr + sectors > rdev->recovery_offset) 770 continue; 771 772 dev_sector = r10_bio->devs[slot].addr; 773 if (is_badblock(rdev, dev_sector, sectors, 774 &first_bad, &bad_sectors)) { 775 if (best_dist < MaxSector) 776 /* Already have a better slot */ 777 continue; 778 if (first_bad <= dev_sector) { 779 /* Cannot read here. If this is the 780 * 'primary' device, then we must not read 781 * beyond 'bad_sectors' from another device. 782 */ 783 bad_sectors -= (dev_sector - first_bad); 784 if (!do_balance && sectors > bad_sectors) 785 sectors = bad_sectors; 786 if (best_good_sectors > sectors) 787 best_good_sectors = sectors; 788 } else { 789 sector_t good_sectors = 790 first_bad - dev_sector; 791 if (good_sectors > best_good_sectors) { 792 best_good_sectors = good_sectors; 793 best_slot = slot; 794 best_rdev = rdev; 795 } 796 if (!do_balance) 797 /* Must read from here */ 798 break; 799 } 800 continue; 801 } else 802 best_good_sectors = sectors; 803 804 if (!do_balance) 805 break; 806 807 if (best_slot >= 0) 808 /* At least 2 disks to choose from so failfast is OK */ 809 set_bit(R10BIO_FailFast, &r10_bio->state); 810 /* This optimisation is debatable, and completely destroys 811 * sequential read speed for 'far copies' arrays. So only 812 * keep it for 'near' arrays, and review those later. 813 */ 814 if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending)) 815 new_distance = 0; 816 817 /* for far > 1 always use the lowest address */ 818 else if (geo->far_copies > 1) 819 new_distance = r10_bio->devs[slot].addr; 820 else 821 new_distance = abs(r10_bio->devs[slot].addr - 822 conf->mirrors[disk].head_position); 823 if (new_distance < best_dist) { 824 best_dist = new_distance; 825 best_slot = slot; 826 best_rdev = rdev; 827 } 828 } 829 if (slot >= conf->copies) { 830 slot = best_slot; 831 rdev = best_rdev; 832 } 833 834 if (slot >= 0) { 835 atomic_inc(&rdev->nr_pending); 836 r10_bio->read_slot = slot; 837 } else 838 rdev = NULL; 839 rcu_read_unlock(); 840 *max_sectors = best_good_sectors; 841 842 return rdev; 843 } 844 845 static int raid10_congested(struct mddev *mddev, int bits) 846 { 847 struct r10conf *conf = mddev->private; 848 int i, ret = 0; 849 850 if ((bits & (1 << WB_async_congested)) && 851 conf->pending_count >= max_queued_requests) 852 return 1; 853 854 rcu_read_lock(); 855 for (i = 0; 856 (i < conf->geo.raid_disks || i < conf->prev.raid_disks) 857 && ret == 0; 858 i++) { 859 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 860 if (rdev && !test_bit(Faulty, &rdev->flags)) { 861 struct request_queue *q = bdev_get_queue(rdev->bdev); 862 863 ret |= bdi_congested(&q->backing_dev_info, bits); 864 } 865 } 866 rcu_read_unlock(); 867 return ret; 868 } 869 870 static void flush_pending_writes(struct r10conf *conf) 871 { 872 /* Any writes that have been queued but are awaiting 873 * bitmap updates get flushed here. 874 */ 875 spin_lock_irq(&conf->device_lock); 876 877 if (conf->pending_bio_list.head) { 878 struct bio *bio; 879 bio = bio_list_get(&conf->pending_bio_list); 880 conf->pending_count = 0; 881 spin_unlock_irq(&conf->device_lock); 882 /* flush any pending bitmap writes to disk 883 * before proceeding w/ I/O */ 884 bitmap_unplug(conf->mddev->bitmap); 885 wake_up(&conf->wait_barrier); 886 887 while (bio) { /* submit pending writes */ 888 struct bio *next = bio->bi_next; 889 struct md_rdev *rdev = (void*)bio->bi_bdev; 890 bio->bi_next = NULL; 891 bio->bi_bdev = rdev->bdev; 892 if (test_bit(Faulty, &rdev->flags)) { 893 bio->bi_error = -EIO; 894 bio_endio(bio); 895 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 896 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 897 /* Just ignore it */ 898 bio_endio(bio); 899 else 900 generic_make_request(bio); 901 bio = next; 902 } 903 } else 904 spin_unlock_irq(&conf->device_lock); 905 } 906 907 /* Barriers.... 908 * Sometimes we need to suspend IO while we do something else, 909 * either some resync/recovery, or reconfigure the array. 910 * To do this we raise a 'barrier'. 911 * The 'barrier' is a counter that can be raised multiple times 912 * to count how many activities are happening which preclude 913 * normal IO. 914 * We can only raise the barrier if there is no pending IO. 915 * i.e. if nr_pending == 0. 916 * We choose only to raise the barrier if no-one is waiting for the 917 * barrier to go down. This means that as soon as an IO request 918 * is ready, no other operations which require a barrier will start 919 * until the IO request has had a chance. 920 * 921 * So: regular IO calls 'wait_barrier'. When that returns there 922 * is no backgroup IO happening, It must arrange to call 923 * allow_barrier when it has finished its IO. 924 * backgroup IO calls must call raise_barrier. Once that returns 925 * there is no normal IO happeing. It must arrange to call 926 * lower_barrier when the particular background IO completes. 927 */ 928 929 static void raise_barrier(struct r10conf *conf, int force) 930 { 931 BUG_ON(force && !conf->barrier); 932 spin_lock_irq(&conf->resync_lock); 933 934 /* Wait until no block IO is waiting (unless 'force') */ 935 wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting, 936 conf->resync_lock); 937 938 /* block any new IO from starting */ 939 conf->barrier++; 940 941 /* Now wait for all pending IO to complete */ 942 wait_event_lock_irq(conf->wait_barrier, 943 !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH, 944 conf->resync_lock); 945 946 spin_unlock_irq(&conf->resync_lock); 947 } 948 949 static void lower_barrier(struct r10conf *conf) 950 { 951 unsigned long flags; 952 spin_lock_irqsave(&conf->resync_lock, flags); 953 conf->barrier--; 954 spin_unlock_irqrestore(&conf->resync_lock, flags); 955 wake_up(&conf->wait_barrier); 956 } 957 958 static void wait_barrier(struct r10conf *conf) 959 { 960 spin_lock_irq(&conf->resync_lock); 961 if (conf->barrier) { 962 conf->nr_waiting++; 963 /* Wait for the barrier to drop. 964 * However if there are already pending 965 * requests (preventing the barrier from 966 * rising completely), and the 967 * pre-process bio queue isn't empty, 968 * then don't wait, as we need to empty 969 * that queue to get the nr_pending 970 * count down. 971 */ 972 raid10_log(conf->mddev, "wait barrier"); 973 wait_event_lock_irq(conf->wait_barrier, 974 !conf->barrier || 975 (atomic_read(&conf->nr_pending) && 976 current->bio_list && 977 !bio_list_empty(current->bio_list)), 978 conf->resync_lock); 979 conf->nr_waiting--; 980 if (!conf->nr_waiting) 981 wake_up(&conf->wait_barrier); 982 } 983 atomic_inc(&conf->nr_pending); 984 spin_unlock_irq(&conf->resync_lock); 985 } 986 987 static void allow_barrier(struct r10conf *conf) 988 { 989 if ((atomic_dec_and_test(&conf->nr_pending)) || 990 (conf->array_freeze_pending)) 991 wake_up(&conf->wait_barrier); 992 } 993 994 static void freeze_array(struct r10conf *conf, int extra) 995 { 996 /* stop syncio and normal IO and wait for everything to 997 * go quiet. 998 * We increment barrier and nr_waiting, and then 999 * wait until nr_pending match nr_queued+extra 1000 * This is called in the context of one normal IO request 1001 * that has failed. Thus any sync request that might be pending 1002 * will be blocked by nr_pending, and we need to wait for 1003 * pending IO requests to complete or be queued for re-try. 1004 * Thus the number queued (nr_queued) plus this request (extra) 1005 * must match the number of pending IOs (nr_pending) before 1006 * we continue. 1007 */ 1008 spin_lock_irq(&conf->resync_lock); 1009 conf->array_freeze_pending++; 1010 conf->barrier++; 1011 conf->nr_waiting++; 1012 wait_event_lock_irq_cmd(conf->wait_barrier, 1013 atomic_read(&conf->nr_pending) == conf->nr_queued+extra, 1014 conf->resync_lock, 1015 flush_pending_writes(conf)); 1016 1017 conf->array_freeze_pending--; 1018 spin_unlock_irq(&conf->resync_lock); 1019 } 1020 1021 static void unfreeze_array(struct r10conf *conf) 1022 { 1023 /* reverse the effect of the freeze */ 1024 spin_lock_irq(&conf->resync_lock); 1025 conf->barrier--; 1026 conf->nr_waiting--; 1027 wake_up(&conf->wait_barrier); 1028 spin_unlock_irq(&conf->resync_lock); 1029 } 1030 1031 static sector_t choose_data_offset(struct r10bio *r10_bio, 1032 struct md_rdev *rdev) 1033 { 1034 if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) || 1035 test_bit(R10BIO_Previous, &r10_bio->state)) 1036 return rdev->data_offset; 1037 else 1038 return rdev->new_data_offset; 1039 } 1040 1041 struct raid10_plug_cb { 1042 struct blk_plug_cb cb; 1043 struct bio_list pending; 1044 int pending_cnt; 1045 }; 1046 1047 static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) 1048 { 1049 struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb, 1050 cb); 1051 struct mddev *mddev = plug->cb.data; 1052 struct r10conf *conf = mddev->private; 1053 struct bio *bio; 1054 1055 if (from_schedule || current->bio_list) { 1056 spin_lock_irq(&conf->device_lock); 1057 bio_list_merge(&conf->pending_bio_list, &plug->pending); 1058 conf->pending_count += plug->pending_cnt; 1059 spin_unlock_irq(&conf->device_lock); 1060 wake_up(&conf->wait_barrier); 1061 md_wakeup_thread(mddev->thread); 1062 kfree(plug); 1063 return; 1064 } 1065 1066 /* we aren't scheduling, so we can do the write-out directly. */ 1067 bio = bio_list_get(&plug->pending); 1068 bitmap_unplug(mddev->bitmap); 1069 wake_up(&conf->wait_barrier); 1070 1071 while (bio) { /* submit pending writes */ 1072 struct bio *next = bio->bi_next; 1073 struct md_rdev *rdev = (void*)bio->bi_bdev; 1074 bio->bi_next = NULL; 1075 bio->bi_bdev = rdev->bdev; 1076 if (test_bit(Faulty, &rdev->flags)) { 1077 bio->bi_error = -EIO; 1078 bio_endio(bio); 1079 } else if (unlikely((bio_op(bio) == REQ_OP_DISCARD) && 1080 !blk_queue_discard(bdev_get_queue(bio->bi_bdev)))) 1081 /* Just ignore it */ 1082 bio_endio(bio); 1083 else 1084 generic_make_request(bio); 1085 bio = next; 1086 } 1087 kfree(plug); 1088 } 1089 1090 static void __make_request(struct mddev *mddev, struct bio *bio) 1091 { 1092 struct r10conf *conf = mddev->private; 1093 struct r10bio *r10_bio; 1094 struct bio *read_bio; 1095 int i; 1096 const int op = bio_op(bio); 1097 const int rw = bio_data_dir(bio); 1098 const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); 1099 const unsigned long do_fua = (bio->bi_opf & REQ_FUA); 1100 unsigned long flags; 1101 struct md_rdev *blocked_rdev; 1102 struct blk_plug_cb *cb; 1103 struct raid10_plug_cb *plug = NULL; 1104 int sectors_handled; 1105 int max_sectors; 1106 int sectors; 1107 1108 md_write_start(mddev, bio); 1109 1110 /* 1111 * Register the new request and wait if the reconstruction 1112 * thread has put up a bar for new requests. 1113 * Continue immediately if no resync is active currently. 1114 */ 1115 wait_barrier(conf); 1116 1117 sectors = bio_sectors(bio); 1118 while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1119 bio->bi_iter.bi_sector < conf->reshape_progress && 1120 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { 1121 /* IO spans the reshape position. Need to wait for 1122 * reshape to pass 1123 */ 1124 raid10_log(conf->mddev, "wait reshape"); 1125 allow_barrier(conf); 1126 wait_event(conf->wait_barrier, 1127 conf->reshape_progress <= bio->bi_iter.bi_sector || 1128 conf->reshape_progress >= bio->bi_iter.bi_sector + 1129 sectors); 1130 wait_barrier(conf); 1131 } 1132 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 1133 bio_data_dir(bio) == WRITE && 1134 (mddev->reshape_backwards 1135 ? (bio->bi_iter.bi_sector < conf->reshape_safe && 1136 bio->bi_iter.bi_sector + sectors > conf->reshape_progress) 1137 : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe && 1138 bio->bi_iter.bi_sector < conf->reshape_progress))) { 1139 /* Need to update reshape_position in metadata */ 1140 mddev->reshape_position = conf->reshape_progress; 1141 set_mask_bits(&mddev->sb_flags, 0, 1142 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1143 md_wakeup_thread(mddev->thread); 1144 raid10_log(conf->mddev, "wait reshape metadata"); 1145 wait_event(mddev->sb_wait, 1146 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)); 1147 1148 conf->reshape_safe = mddev->reshape_position; 1149 } 1150 1151 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1152 1153 r10_bio->master_bio = bio; 1154 r10_bio->sectors = sectors; 1155 1156 r10_bio->mddev = mddev; 1157 r10_bio->sector = bio->bi_iter.bi_sector; 1158 r10_bio->state = 0; 1159 1160 /* We might need to issue multiple reads to different 1161 * devices if there are bad blocks around, so we keep 1162 * track of the number of reads in bio->bi_phys_segments. 1163 * If this is 0, there is only one r10_bio and no locking 1164 * will be needed when the request completes. If it is 1165 * non-zero, then it is the number of not-completed requests. 1166 */ 1167 bio->bi_phys_segments = 0; 1168 bio_clear_flag(bio, BIO_SEG_VALID); 1169 1170 if (rw == READ) { 1171 /* 1172 * read balancing logic: 1173 */ 1174 struct md_rdev *rdev; 1175 int slot; 1176 1177 read_again: 1178 rdev = read_balance(conf, r10_bio, &max_sectors); 1179 if (!rdev) { 1180 raid_end_bio_io(r10_bio); 1181 return; 1182 } 1183 slot = r10_bio->read_slot; 1184 1185 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1186 bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector, 1187 max_sectors); 1188 1189 r10_bio->devs[slot].bio = read_bio; 1190 r10_bio->devs[slot].rdev = rdev; 1191 1192 read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr + 1193 choose_data_offset(r10_bio, rdev); 1194 read_bio->bi_bdev = rdev->bdev; 1195 read_bio->bi_end_io = raid10_end_read_request; 1196 bio_set_op_attrs(read_bio, op, do_sync); 1197 if (test_bit(FailFast, &rdev->flags) && 1198 test_bit(R10BIO_FailFast, &r10_bio->state)) 1199 read_bio->bi_opf |= MD_FAILFAST; 1200 read_bio->bi_private = r10_bio; 1201 1202 if (mddev->gendisk) 1203 trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev), 1204 read_bio, disk_devt(mddev->gendisk), 1205 r10_bio->sector); 1206 if (max_sectors < r10_bio->sectors) { 1207 /* Could not read all from this device, so we will 1208 * need another r10_bio. 1209 */ 1210 sectors_handled = (r10_bio->sector + max_sectors 1211 - bio->bi_iter.bi_sector); 1212 r10_bio->sectors = max_sectors; 1213 spin_lock_irq(&conf->device_lock); 1214 if (bio->bi_phys_segments == 0) 1215 bio->bi_phys_segments = 2; 1216 else 1217 bio->bi_phys_segments++; 1218 spin_unlock_irq(&conf->device_lock); 1219 /* Cannot call generic_make_request directly 1220 * as that will be queued in __generic_make_request 1221 * and subsequent mempool_alloc might block 1222 * waiting for it. so hand bio over to raid10d. 1223 */ 1224 reschedule_retry(r10_bio); 1225 1226 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1227 1228 r10_bio->master_bio = bio; 1229 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1230 r10_bio->state = 0; 1231 r10_bio->mddev = mddev; 1232 r10_bio->sector = bio->bi_iter.bi_sector + 1233 sectors_handled; 1234 goto read_again; 1235 } else 1236 generic_make_request(read_bio); 1237 return; 1238 } 1239 1240 /* 1241 * WRITE: 1242 */ 1243 if (conf->pending_count >= max_queued_requests) { 1244 md_wakeup_thread(mddev->thread); 1245 raid10_log(mddev, "wait queued"); 1246 wait_event(conf->wait_barrier, 1247 conf->pending_count < max_queued_requests); 1248 } 1249 /* first select target devices under rcu_lock and 1250 * inc refcount on their rdev. Record them by setting 1251 * bios[x] to bio 1252 * If there are known/acknowledged bad blocks on any device 1253 * on which we have seen a write error, we want to avoid 1254 * writing to those blocks. This potentially requires several 1255 * writes to write around the bad blocks. Each set of writes 1256 * gets its own r10_bio with a set of bios attached. The number 1257 * of r10_bios is recored in bio->bi_phys_segments just as with 1258 * the read case. 1259 */ 1260 1261 r10_bio->read_slot = -1; /* make sure repl_bio gets freed */ 1262 raid10_find_phys(conf, r10_bio); 1263 retry_write: 1264 blocked_rdev = NULL; 1265 rcu_read_lock(); 1266 max_sectors = r10_bio->sectors; 1267 1268 for (i = 0; i < conf->copies; i++) { 1269 int d = r10_bio->devs[i].devnum; 1270 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 1271 struct md_rdev *rrdev = rcu_dereference( 1272 conf->mirrors[d].replacement); 1273 if (rdev == rrdev) 1274 rrdev = NULL; 1275 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 1276 atomic_inc(&rdev->nr_pending); 1277 blocked_rdev = rdev; 1278 break; 1279 } 1280 if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) { 1281 atomic_inc(&rrdev->nr_pending); 1282 blocked_rdev = rrdev; 1283 break; 1284 } 1285 if (rdev && (test_bit(Faulty, &rdev->flags))) 1286 rdev = NULL; 1287 if (rrdev && (test_bit(Faulty, &rrdev->flags))) 1288 rrdev = NULL; 1289 1290 r10_bio->devs[i].bio = NULL; 1291 r10_bio->devs[i].repl_bio = NULL; 1292 1293 if (!rdev && !rrdev) { 1294 set_bit(R10BIO_Degraded, &r10_bio->state); 1295 continue; 1296 } 1297 if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) { 1298 sector_t first_bad; 1299 sector_t dev_sector = r10_bio->devs[i].addr; 1300 int bad_sectors; 1301 int is_bad; 1302 1303 is_bad = is_badblock(rdev, dev_sector, 1304 max_sectors, 1305 &first_bad, &bad_sectors); 1306 if (is_bad < 0) { 1307 /* Mustn't write here until the bad block 1308 * is acknowledged 1309 */ 1310 atomic_inc(&rdev->nr_pending); 1311 set_bit(BlockedBadBlocks, &rdev->flags); 1312 blocked_rdev = rdev; 1313 break; 1314 } 1315 if (is_bad && first_bad <= dev_sector) { 1316 /* Cannot write here at all */ 1317 bad_sectors -= (dev_sector - first_bad); 1318 if (bad_sectors < max_sectors) 1319 /* Mustn't write more than bad_sectors 1320 * to other devices yet 1321 */ 1322 max_sectors = bad_sectors; 1323 /* We don't set R10BIO_Degraded as that 1324 * only applies if the disk is missing, 1325 * so it might be re-added, and we want to 1326 * know to recover this chunk. 1327 * In this case the device is here, and the 1328 * fact that this chunk is not in-sync is 1329 * recorded in the bad block log. 1330 */ 1331 continue; 1332 } 1333 if (is_bad) { 1334 int good_sectors = first_bad - dev_sector; 1335 if (good_sectors < max_sectors) 1336 max_sectors = good_sectors; 1337 } 1338 } 1339 if (rdev) { 1340 r10_bio->devs[i].bio = bio; 1341 atomic_inc(&rdev->nr_pending); 1342 } 1343 if (rrdev) { 1344 r10_bio->devs[i].repl_bio = bio; 1345 atomic_inc(&rrdev->nr_pending); 1346 } 1347 } 1348 rcu_read_unlock(); 1349 1350 if (unlikely(blocked_rdev)) { 1351 /* Have to wait for this device to get unblocked, then retry */ 1352 int j; 1353 int d; 1354 1355 for (j = 0; j < i; j++) { 1356 if (r10_bio->devs[j].bio) { 1357 d = r10_bio->devs[j].devnum; 1358 rdev_dec_pending(conf->mirrors[d].rdev, mddev); 1359 } 1360 if (r10_bio->devs[j].repl_bio) { 1361 struct md_rdev *rdev; 1362 d = r10_bio->devs[j].devnum; 1363 rdev = conf->mirrors[d].replacement; 1364 if (!rdev) { 1365 /* Race with remove_disk */ 1366 smp_mb(); 1367 rdev = conf->mirrors[d].rdev; 1368 } 1369 rdev_dec_pending(rdev, mddev); 1370 } 1371 } 1372 allow_barrier(conf); 1373 raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk); 1374 md_wait_for_blocked_rdev(blocked_rdev, mddev); 1375 wait_barrier(conf); 1376 goto retry_write; 1377 } 1378 1379 if (max_sectors < r10_bio->sectors) { 1380 /* We are splitting this into multiple parts, so 1381 * we need to prepare for allocating another r10_bio. 1382 */ 1383 r10_bio->sectors = max_sectors; 1384 spin_lock_irq(&conf->device_lock); 1385 if (bio->bi_phys_segments == 0) 1386 bio->bi_phys_segments = 2; 1387 else 1388 bio->bi_phys_segments++; 1389 spin_unlock_irq(&conf->device_lock); 1390 } 1391 sectors_handled = r10_bio->sector + max_sectors - 1392 bio->bi_iter.bi_sector; 1393 1394 atomic_set(&r10_bio->remaining, 1); 1395 bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0); 1396 1397 for (i = 0; i < conf->copies; i++) { 1398 struct bio *mbio; 1399 int d = r10_bio->devs[i].devnum; 1400 if (r10_bio->devs[i].bio) { 1401 struct md_rdev *rdev = conf->mirrors[d].rdev; 1402 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1403 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1404 max_sectors); 1405 r10_bio->devs[i].bio = mbio; 1406 1407 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+ 1408 choose_data_offset(r10_bio, 1409 rdev)); 1410 mbio->bi_bdev = rdev->bdev; 1411 mbio->bi_end_io = raid10_end_write_request; 1412 bio_set_op_attrs(mbio, op, do_sync | do_fua); 1413 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags) && 1414 enough(conf, d)) 1415 mbio->bi_opf |= MD_FAILFAST; 1416 mbio->bi_private = r10_bio; 1417 1418 if (conf->mddev->gendisk) 1419 trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev), 1420 mbio, disk_devt(conf->mddev->gendisk), 1421 r10_bio->sector); 1422 /* flush_pending_writes() needs access to the rdev so...*/ 1423 mbio->bi_bdev = (void*)rdev; 1424 1425 atomic_inc(&r10_bio->remaining); 1426 1427 cb = blk_check_plugged(raid10_unplug, mddev, 1428 sizeof(*plug)); 1429 if (cb) 1430 plug = container_of(cb, struct raid10_plug_cb, 1431 cb); 1432 else 1433 plug = NULL; 1434 spin_lock_irqsave(&conf->device_lock, flags); 1435 if (plug) { 1436 bio_list_add(&plug->pending, mbio); 1437 plug->pending_cnt++; 1438 } else { 1439 bio_list_add(&conf->pending_bio_list, mbio); 1440 conf->pending_count++; 1441 } 1442 spin_unlock_irqrestore(&conf->device_lock, flags); 1443 if (!plug) 1444 md_wakeup_thread(mddev->thread); 1445 } 1446 1447 if (r10_bio->devs[i].repl_bio) { 1448 struct md_rdev *rdev = conf->mirrors[d].replacement; 1449 if (rdev == NULL) { 1450 /* Replacement just got moved to main 'rdev' */ 1451 smp_mb(); 1452 rdev = conf->mirrors[d].rdev; 1453 } 1454 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 1455 bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, 1456 max_sectors); 1457 r10_bio->devs[i].repl_bio = mbio; 1458 1459 mbio->bi_iter.bi_sector = (r10_bio->devs[i].addr + 1460 choose_data_offset( 1461 r10_bio, rdev)); 1462 mbio->bi_bdev = rdev->bdev; 1463 mbio->bi_end_io = raid10_end_write_request; 1464 bio_set_op_attrs(mbio, op, do_sync | do_fua); 1465 mbio->bi_private = r10_bio; 1466 1467 if (conf->mddev->gendisk) 1468 trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev), 1469 mbio, disk_devt(conf->mddev->gendisk), 1470 r10_bio->sector); 1471 /* flush_pending_writes() needs access to the rdev so...*/ 1472 mbio->bi_bdev = (void*)rdev; 1473 1474 atomic_inc(&r10_bio->remaining); 1475 spin_lock_irqsave(&conf->device_lock, flags); 1476 bio_list_add(&conf->pending_bio_list, mbio); 1477 conf->pending_count++; 1478 spin_unlock_irqrestore(&conf->device_lock, flags); 1479 if (!mddev_check_plugged(mddev)) 1480 md_wakeup_thread(mddev->thread); 1481 } 1482 } 1483 1484 /* Don't remove the bias on 'remaining' (one_write_done) until 1485 * after checking if we need to go around again. 1486 */ 1487 1488 if (sectors_handled < bio_sectors(bio)) { 1489 one_write_done(r10_bio); 1490 /* We need another r10_bio. It has already been counted 1491 * in bio->bi_phys_segments. 1492 */ 1493 r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); 1494 1495 r10_bio->master_bio = bio; 1496 r10_bio->sectors = bio_sectors(bio) - sectors_handled; 1497 1498 r10_bio->mddev = mddev; 1499 r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled; 1500 r10_bio->state = 0; 1501 goto retry_write; 1502 } 1503 one_write_done(r10_bio); 1504 } 1505 1506 static void raid10_make_request(struct mddev *mddev, struct bio *bio) 1507 { 1508 struct r10conf *conf = mddev->private; 1509 sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); 1510 int chunk_sects = chunk_mask + 1; 1511 1512 struct bio *split; 1513 1514 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { 1515 md_flush_request(mddev, bio); 1516 return; 1517 } 1518 1519 do { 1520 1521 /* 1522 * If this request crosses a chunk boundary, we need to split 1523 * it. 1524 */ 1525 if (unlikely((bio->bi_iter.bi_sector & chunk_mask) + 1526 bio_sectors(bio) > chunk_sects 1527 && (conf->geo.near_copies < conf->geo.raid_disks 1528 || conf->prev.near_copies < 1529 conf->prev.raid_disks))) { 1530 split = bio_split(bio, chunk_sects - 1531 (bio->bi_iter.bi_sector & 1532 (chunk_sects - 1)), 1533 GFP_NOIO, fs_bio_set); 1534 bio_chain(split, bio); 1535 } else { 1536 split = bio; 1537 } 1538 1539 __make_request(mddev, split); 1540 } while (split != bio); 1541 1542 /* In case raid10d snuck in to freeze_array */ 1543 wake_up(&conf->wait_barrier); 1544 } 1545 1546 static void raid10_status(struct seq_file *seq, struct mddev *mddev) 1547 { 1548 struct r10conf *conf = mddev->private; 1549 int i; 1550 1551 if (conf->geo.near_copies < conf->geo.raid_disks) 1552 seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2); 1553 if (conf->geo.near_copies > 1) 1554 seq_printf(seq, " %d near-copies", conf->geo.near_copies); 1555 if (conf->geo.far_copies > 1) { 1556 if (conf->geo.far_offset) 1557 seq_printf(seq, " %d offset-copies", conf->geo.far_copies); 1558 else 1559 seq_printf(seq, " %d far-copies", conf->geo.far_copies); 1560 if (conf->geo.far_set_size != conf->geo.raid_disks) 1561 seq_printf(seq, " %d devices per set", conf->geo.far_set_size); 1562 } 1563 seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks, 1564 conf->geo.raid_disks - mddev->degraded); 1565 rcu_read_lock(); 1566 for (i = 0; i < conf->geo.raid_disks; i++) { 1567 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 1568 seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 1569 } 1570 rcu_read_unlock(); 1571 seq_printf(seq, "]"); 1572 } 1573 1574 /* check if there are enough drives for 1575 * every block to appear on atleast one. 1576 * Don't consider the device numbered 'ignore' 1577 * as we might be about to remove it. 1578 */ 1579 static int _enough(struct r10conf *conf, int previous, int ignore) 1580 { 1581 int first = 0; 1582 int has_enough = 0; 1583 int disks, ncopies; 1584 if (previous) { 1585 disks = conf->prev.raid_disks; 1586 ncopies = conf->prev.near_copies; 1587 } else { 1588 disks = conf->geo.raid_disks; 1589 ncopies = conf->geo.near_copies; 1590 } 1591 1592 rcu_read_lock(); 1593 do { 1594 int n = conf->copies; 1595 int cnt = 0; 1596 int this = first; 1597 while (n--) { 1598 struct md_rdev *rdev; 1599 if (this != ignore && 1600 (rdev = rcu_dereference(conf->mirrors[this].rdev)) && 1601 test_bit(In_sync, &rdev->flags)) 1602 cnt++; 1603 this = (this+1) % disks; 1604 } 1605 if (cnt == 0) 1606 goto out; 1607 first = (first + ncopies) % disks; 1608 } while (first != 0); 1609 has_enough = 1; 1610 out: 1611 rcu_read_unlock(); 1612 return has_enough; 1613 } 1614 1615 static int enough(struct r10conf *conf, int ignore) 1616 { 1617 /* when calling 'enough', both 'prev' and 'geo' must 1618 * be stable. 1619 * This is ensured if ->reconfig_mutex or ->device_lock 1620 * is held. 1621 */ 1622 return _enough(conf, 0, ignore) && 1623 _enough(conf, 1, ignore); 1624 } 1625 1626 static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) 1627 { 1628 char b[BDEVNAME_SIZE]; 1629 struct r10conf *conf = mddev->private; 1630 unsigned long flags; 1631 1632 /* 1633 * If it is not operational, then we have already marked it as dead 1634 * else if it is the last working disks, ignore the error, let the 1635 * next level up know. 1636 * else mark the drive as failed 1637 */ 1638 spin_lock_irqsave(&conf->device_lock, flags); 1639 if (test_bit(In_sync, &rdev->flags) 1640 && !enough(conf, rdev->raid_disk)) { 1641 /* 1642 * Don't fail the drive, just return an IO error. 1643 */ 1644 spin_unlock_irqrestore(&conf->device_lock, flags); 1645 return; 1646 } 1647 if (test_and_clear_bit(In_sync, &rdev->flags)) 1648 mddev->degraded++; 1649 /* 1650 * If recovery is running, make sure it aborts. 1651 */ 1652 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1653 set_bit(Blocked, &rdev->flags); 1654 set_bit(Faulty, &rdev->flags); 1655 set_mask_bits(&mddev->sb_flags, 0, 1656 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 1657 spin_unlock_irqrestore(&conf->device_lock, flags); 1658 pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n" 1659 "md/raid10:%s: Operation continuing on %d devices.\n", 1660 mdname(mddev), bdevname(rdev->bdev, b), 1661 mdname(mddev), conf->geo.raid_disks - mddev->degraded); 1662 } 1663 1664 static void print_conf(struct r10conf *conf) 1665 { 1666 int i; 1667 struct md_rdev *rdev; 1668 1669 pr_debug("RAID10 conf printout:\n"); 1670 if (!conf) { 1671 pr_debug("(!conf)\n"); 1672 return; 1673 } 1674 pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded, 1675 conf->geo.raid_disks); 1676 1677 /* This is only called with ->reconfix_mutex held, so 1678 * rcu protection of rdev is not needed */ 1679 for (i = 0; i < conf->geo.raid_disks; i++) { 1680 char b[BDEVNAME_SIZE]; 1681 rdev = conf->mirrors[i].rdev; 1682 if (rdev) 1683 pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n", 1684 i, !test_bit(In_sync, &rdev->flags), 1685 !test_bit(Faulty, &rdev->flags), 1686 bdevname(rdev->bdev,b)); 1687 } 1688 } 1689 1690 static void close_sync(struct r10conf *conf) 1691 { 1692 wait_barrier(conf); 1693 allow_barrier(conf); 1694 1695 mempool_destroy(conf->r10buf_pool); 1696 conf->r10buf_pool = NULL; 1697 } 1698 1699 static int raid10_spare_active(struct mddev *mddev) 1700 { 1701 int i; 1702 struct r10conf *conf = mddev->private; 1703 struct raid10_info *tmp; 1704 int count = 0; 1705 unsigned long flags; 1706 1707 /* 1708 * Find all non-in_sync disks within the RAID10 configuration 1709 * and mark them in_sync 1710 */ 1711 for (i = 0; i < conf->geo.raid_disks; i++) { 1712 tmp = conf->mirrors + i; 1713 if (tmp->replacement 1714 && tmp->replacement->recovery_offset == MaxSector 1715 && !test_bit(Faulty, &tmp->replacement->flags) 1716 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 1717 /* Replacement has just become active */ 1718 if (!tmp->rdev 1719 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 1720 count++; 1721 if (tmp->rdev) { 1722 /* Replaced device not technically faulty, 1723 * but we need to be sure it gets removed 1724 * and never re-added. 1725 */ 1726 set_bit(Faulty, &tmp->rdev->flags); 1727 sysfs_notify_dirent_safe( 1728 tmp->rdev->sysfs_state); 1729 } 1730 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 1731 } else if (tmp->rdev 1732 && tmp->rdev->recovery_offset == MaxSector 1733 && !test_bit(Faulty, &tmp->rdev->flags) 1734 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 1735 count++; 1736 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 1737 } 1738 } 1739 spin_lock_irqsave(&conf->device_lock, flags); 1740 mddev->degraded -= count; 1741 spin_unlock_irqrestore(&conf->device_lock, flags); 1742 1743 print_conf(conf); 1744 return count; 1745 } 1746 1747 static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) 1748 { 1749 struct r10conf *conf = mddev->private; 1750 int err = -EEXIST; 1751 int mirror; 1752 int first = 0; 1753 int last = conf->geo.raid_disks - 1; 1754 1755 if (mddev->recovery_cp < MaxSector) 1756 /* only hot-add to in-sync arrays, as recovery is 1757 * very different from resync 1758 */ 1759 return -EBUSY; 1760 if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1)) 1761 return -EINVAL; 1762 1763 if (md_integrity_add_rdev(rdev, mddev)) 1764 return -ENXIO; 1765 1766 if (rdev->raid_disk >= 0) 1767 first = last = rdev->raid_disk; 1768 1769 if (rdev->saved_raid_disk >= first && 1770 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) 1771 mirror = rdev->saved_raid_disk; 1772 else 1773 mirror = first; 1774 for ( ; mirror <= last ; mirror++) { 1775 struct raid10_info *p = &conf->mirrors[mirror]; 1776 if (p->recovery_disabled == mddev->recovery_disabled) 1777 continue; 1778 if (p->rdev) { 1779 if (!test_bit(WantReplacement, &p->rdev->flags) || 1780 p->replacement != NULL) 1781 continue; 1782 clear_bit(In_sync, &rdev->flags); 1783 set_bit(Replacement, &rdev->flags); 1784 rdev->raid_disk = mirror; 1785 err = 0; 1786 if (mddev->gendisk) 1787 disk_stack_limits(mddev->gendisk, rdev->bdev, 1788 rdev->data_offset << 9); 1789 conf->fullsync = 1; 1790 rcu_assign_pointer(p->replacement, rdev); 1791 break; 1792 } 1793 1794 if (mddev->gendisk) 1795 disk_stack_limits(mddev->gendisk, rdev->bdev, 1796 rdev->data_offset << 9); 1797 1798 p->head_position = 0; 1799 p->recovery_disabled = mddev->recovery_disabled - 1; 1800 rdev->raid_disk = mirror; 1801 err = 0; 1802 if (rdev->saved_raid_disk != mirror) 1803 conf->fullsync = 1; 1804 rcu_assign_pointer(p->rdev, rdev); 1805 break; 1806 } 1807 if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev))) 1808 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue); 1809 1810 print_conf(conf); 1811 return err; 1812 } 1813 1814 static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 1815 { 1816 struct r10conf *conf = mddev->private; 1817 int err = 0; 1818 int number = rdev->raid_disk; 1819 struct md_rdev **rdevp; 1820 struct raid10_info *p = conf->mirrors + number; 1821 1822 print_conf(conf); 1823 if (rdev == p->rdev) 1824 rdevp = &p->rdev; 1825 else if (rdev == p->replacement) 1826 rdevp = &p->replacement; 1827 else 1828 return 0; 1829 1830 if (test_bit(In_sync, &rdev->flags) || 1831 atomic_read(&rdev->nr_pending)) { 1832 err = -EBUSY; 1833 goto abort; 1834 } 1835 /* Only remove non-faulty devices if recovery 1836 * is not possible. 1837 */ 1838 if (!test_bit(Faulty, &rdev->flags) && 1839 mddev->recovery_disabled != p->recovery_disabled && 1840 (!p->replacement || p->replacement == rdev) && 1841 number < conf->geo.raid_disks && 1842 enough(conf, -1)) { 1843 err = -EBUSY; 1844 goto abort; 1845 } 1846 *rdevp = NULL; 1847 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 1848 synchronize_rcu(); 1849 if (atomic_read(&rdev->nr_pending)) { 1850 /* lost the race, try later */ 1851 err = -EBUSY; 1852 *rdevp = rdev; 1853 goto abort; 1854 } 1855 } 1856 if (p->replacement) { 1857 /* We must have just cleared 'rdev' */ 1858 p->rdev = p->replacement; 1859 clear_bit(Replacement, &p->replacement->flags); 1860 smp_mb(); /* Make sure other CPUs may see both as identical 1861 * but will never see neither -- if they are careful. 1862 */ 1863 p->replacement = NULL; 1864 clear_bit(WantReplacement, &rdev->flags); 1865 } else 1866 /* We might have just remove the Replacement as faulty 1867 * Clear the flag just in case 1868 */ 1869 clear_bit(WantReplacement, &rdev->flags); 1870 1871 err = md_integrity_register(mddev); 1872 1873 abort: 1874 1875 print_conf(conf); 1876 return err; 1877 } 1878 1879 static void end_sync_read(struct bio *bio) 1880 { 1881 struct r10bio *r10_bio = bio->bi_private; 1882 struct r10conf *conf = r10_bio->mddev->private; 1883 int d; 1884 1885 if (bio == r10_bio->master_bio) { 1886 /* this is a reshape read */ 1887 d = r10_bio->read_slot; /* really the read dev */ 1888 } else 1889 d = find_bio_disk(conf, r10_bio, bio, NULL, NULL); 1890 1891 if (!bio->bi_error) 1892 set_bit(R10BIO_Uptodate, &r10_bio->state); 1893 else 1894 /* The write handler will notice the lack of 1895 * R10BIO_Uptodate and record any errors etc 1896 */ 1897 atomic_add(r10_bio->sectors, 1898 &conf->mirrors[d].rdev->corrected_errors); 1899 1900 /* for reconstruct, we always reschedule after a read. 1901 * for resync, only after all reads 1902 */ 1903 rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev); 1904 if (test_bit(R10BIO_IsRecover, &r10_bio->state) || 1905 atomic_dec_and_test(&r10_bio->remaining)) { 1906 /* we have read all the blocks, 1907 * do the comparison in process context in raid10d 1908 */ 1909 reschedule_retry(r10_bio); 1910 } 1911 } 1912 1913 static void end_sync_request(struct r10bio *r10_bio) 1914 { 1915 struct mddev *mddev = r10_bio->mddev; 1916 1917 while (atomic_dec_and_test(&r10_bio->remaining)) { 1918 if (r10_bio->master_bio == NULL) { 1919 /* the primary of several recovery bios */ 1920 sector_t s = r10_bio->sectors; 1921 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1922 test_bit(R10BIO_WriteError, &r10_bio->state)) 1923 reschedule_retry(r10_bio); 1924 else 1925 put_buf(r10_bio); 1926 md_done_sync(mddev, s, 1); 1927 break; 1928 } else { 1929 struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio; 1930 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 1931 test_bit(R10BIO_WriteError, &r10_bio->state)) 1932 reschedule_retry(r10_bio); 1933 else 1934 put_buf(r10_bio); 1935 r10_bio = r10_bio2; 1936 } 1937 } 1938 } 1939 1940 static void end_sync_write(struct bio *bio) 1941 { 1942 struct r10bio *r10_bio = bio->bi_private; 1943 struct mddev *mddev = r10_bio->mddev; 1944 struct r10conf *conf = mddev->private; 1945 int d; 1946 sector_t first_bad; 1947 int bad_sectors; 1948 int slot; 1949 int repl; 1950 struct md_rdev *rdev = NULL; 1951 1952 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 1953 if (repl) 1954 rdev = conf->mirrors[d].replacement; 1955 else 1956 rdev = conf->mirrors[d].rdev; 1957 1958 if (bio->bi_error) { 1959 if (repl) 1960 md_error(mddev, rdev); 1961 else { 1962 set_bit(WriteErrorSeen, &rdev->flags); 1963 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 1964 set_bit(MD_RECOVERY_NEEDED, 1965 &rdev->mddev->recovery); 1966 set_bit(R10BIO_WriteError, &r10_bio->state); 1967 } 1968 } else if (is_badblock(rdev, 1969 r10_bio->devs[slot].addr, 1970 r10_bio->sectors, 1971 &first_bad, &bad_sectors)) 1972 set_bit(R10BIO_MadeGood, &r10_bio->state); 1973 1974 rdev_dec_pending(rdev, mddev); 1975 1976 end_sync_request(r10_bio); 1977 } 1978 1979 /* 1980 * Note: sync and recover and handled very differently for raid10 1981 * This code is for resync. 1982 * For resync, we read through virtual addresses and read all blocks. 1983 * If there is any error, we schedule a write. The lowest numbered 1984 * drive is authoritative. 1985 * However requests come for physical address, so we need to map. 1986 * For every physical address there are raid_disks/copies virtual addresses, 1987 * which is always are least one, but is not necessarly an integer. 1988 * This means that a physical address can span multiple chunks, so we may 1989 * have to submit multiple io requests for a single sync request. 1990 */ 1991 /* 1992 * We check if all blocks are in-sync and only write to blocks that 1993 * aren't in sync 1994 */ 1995 static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) 1996 { 1997 struct r10conf *conf = mddev->private; 1998 int i, first; 1999 struct bio *tbio, *fbio; 2000 int vcnt; 2001 2002 atomic_set(&r10_bio->remaining, 1); 2003 2004 /* find the first device with a block */ 2005 for (i=0; i<conf->copies; i++) 2006 if (!r10_bio->devs[i].bio->bi_error) 2007 break; 2008 2009 if (i == conf->copies) 2010 goto done; 2011 2012 first = i; 2013 fbio = r10_bio->devs[i].bio; 2014 fbio->bi_iter.bi_size = r10_bio->sectors << 9; 2015 fbio->bi_iter.bi_idx = 0; 2016 2017 vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9); 2018 /* now find blocks with errors */ 2019 for (i=0 ; i < conf->copies ; i++) { 2020 int j, d; 2021 struct md_rdev *rdev; 2022 2023 tbio = r10_bio->devs[i].bio; 2024 2025 if (tbio->bi_end_io != end_sync_read) 2026 continue; 2027 if (i == first) 2028 continue; 2029 d = r10_bio->devs[i].devnum; 2030 rdev = conf->mirrors[d].rdev; 2031 if (!r10_bio->devs[i].bio->bi_error) { 2032 /* We know that the bi_io_vec layout is the same for 2033 * both 'first' and 'i', so we just compare them. 2034 * All vec entries are PAGE_SIZE; 2035 */ 2036 int sectors = r10_bio->sectors; 2037 for (j = 0; j < vcnt; j++) { 2038 int len = PAGE_SIZE; 2039 if (sectors < (len / 512)) 2040 len = sectors * 512; 2041 if (memcmp(page_address(fbio->bi_io_vec[j].bv_page), 2042 page_address(tbio->bi_io_vec[j].bv_page), 2043 len)) 2044 break; 2045 sectors -= len/512; 2046 } 2047 if (j == vcnt) 2048 continue; 2049 atomic64_add(r10_bio->sectors, &mddev->resync_mismatches); 2050 if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)) 2051 /* Don't fix anything. */ 2052 continue; 2053 } else if (test_bit(FailFast, &rdev->flags)) { 2054 /* Just give up on this device */ 2055 md_error(rdev->mddev, rdev); 2056 continue; 2057 } 2058 /* Ok, we need to write this bio, either to correct an 2059 * inconsistency or to correct an unreadable block. 2060 * First we need to fixup bv_offset, bv_len and 2061 * bi_vecs, as the read request might have corrupted these 2062 */ 2063 bio_reset(tbio); 2064 2065 tbio->bi_vcnt = vcnt; 2066 tbio->bi_iter.bi_size = fbio->bi_iter.bi_size; 2067 tbio->bi_private = r10_bio; 2068 tbio->bi_iter.bi_sector = r10_bio->devs[i].addr; 2069 tbio->bi_end_io = end_sync_write; 2070 bio_set_op_attrs(tbio, REQ_OP_WRITE, 0); 2071 2072 bio_copy_data(tbio, fbio); 2073 2074 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2075 atomic_inc(&r10_bio->remaining); 2076 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio)); 2077 2078 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags)) 2079 tbio->bi_opf |= MD_FAILFAST; 2080 tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset; 2081 tbio->bi_bdev = conf->mirrors[d].rdev->bdev; 2082 generic_make_request(tbio); 2083 } 2084 2085 /* Now write out to any replacement devices 2086 * that are active 2087 */ 2088 for (i = 0; i < conf->copies; i++) { 2089 int d; 2090 2091 tbio = r10_bio->devs[i].repl_bio; 2092 if (!tbio || !tbio->bi_end_io) 2093 continue; 2094 if (r10_bio->devs[i].bio->bi_end_io != end_sync_write 2095 && r10_bio->devs[i].bio != fbio) 2096 bio_copy_data(tbio, fbio); 2097 d = r10_bio->devs[i].devnum; 2098 atomic_inc(&r10_bio->remaining); 2099 md_sync_acct(conf->mirrors[d].replacement->bdev, 2100 bio_sectors(tbio)); 2101 generic_make_request(tbio); 2102 } 2103 2104 done: 2105 if (atomic_dec_and_test(&r10_bio->remaining)) { 2106 md_done_sync(mddev, r10_bio->sectors, 1); 2107 put_buf(r10_bio); 2108 } 2109 } 2110 2111 /* 2112 * Now for the recovery code. 2113 * Recovery happens across physical sectors. 2114 * We recover all non-is_sync drives by finding the virtual address of 2115 * each, and then choose a working drive that also has that virt address. 2116 * There is a separate r10_bio for each non-in_sync drive. 2117 * Only the first two slots are in use. The first for reading, 2118 * The second for writing. 2119 * 2120 */ 2121 static void fix_recovery_read_error(struct r10bio *r10_bio) 2122 { 2123 /* We got a read error during recovery. 2124 * We repeat the read in smaller page-sized sections. 2125 * If a read succeeds, write it to the new device or record 2126 * a bad block if we cannot. 2127 * If a read fails, record a bad block on both old and 2128 * new devices. 2129 */ 2130 struct mddev *mddev = r10_bio->mddev; 2131 struct r10conf *conf = mddev->private; 2132 struct bio *bio = r10_bio->devs[0].bio; 2133 sector_t sect = 0; 2134 int sectors = r10_bio->sectors; 2135 int idx = 0; 2136 int dr = r10_bio->devs[0].devnum; 2137 int dw = r10_bio->devs[1].devnum; 2138 2139 while (sectors) { 2140 int s = sectors; 2141 struct md_rdev *rdev; 2142 sector_t addr; 2143 int ok; 2144 2145 if (s > (PAGE_SIZE>>9)) 2146 s = PAGE_SIZE >> 9; 2147 2148 rdev = conf->mirrors[dr].rdev; 2149 addr = r10_bio->devs[0].addr + sect, 2150 ok = sync_page_io(rdev, 2151 addr, 2152 s << 9, 2153 bio->bi_io_vec[idx].bv_page, 2154 REQ_OP_READ, 0, false); 2155 if (ok) { 2156 rdev = conf->mirrors[dw].rdev; 2157 addr = r10_bio->devs[1].addr + sect; 2158 ok = sync_page_io(rdev, 2159 addr, 2160 s << 9, 2161 bio->bi_io_vec[idx].bv_page, 2162 REQ_OP_WRITE, 0, false); 2163 if (!ok) { 2164 set_bit(WriteErrorSeen, &rdev->flags); 2165 if (!test_and_set_bit(WantReplacement, 2166 &rdev->flags)) 2167 set_bit(MD_RECOVERY_NEEDED, 2168 &rdev->mddev->recovery); 2169 } 2170 } 2171 if (!ok) { 2172 /* We don't worry if we cannot set a bad block - 2173 * it really is bad so there is no loss in not 2174 * recording it yet 2175 */ 2176 rdev_set_badblocks(rdev, addr, s, 0); 2177 2178 if (rdev != conf->mirrors[dw].rdev) { 2179 /* need bad block on destination too */ 2180 struct md_rdev *rdev2 = conf->mirrors[dw].rdev; 2181 addr = r10_bio->devs[1].addr + sect; 2182 ok = rdev_set_badblocks(rdev2, addr, s, 0); 2183 if (!ok) { 2184 /* just abort the recovery */ 2185 pr_notice("md/raid10:%s: recovery aborted due to read error\n", 2186 mdname(mddev)); 2187 2188 conf->mirrors[dw].recovery_disabled 2189 = mddev->recovery_disabled; 2190 set_bit(MD_RECOVERY_INTR, 2191 &mddev->recovery); 2192 break; 2193 } 2194 } 2195 } 2196 2197 sectors -= s; 2198 sect += s; 2199 idx++; 2200 } 2201 } 2202 2203 static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio) 2204 { 2205 struct r10conf *conf = mddev->private; 2206 int d; 2207 struct bio *wbio, *wbio2; 2208 2209 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) { 2210 fix_recovery_read_error(r10_bio); 2211 end_sync_request(r10_bio); 2212 return; 2213 } 2214 2215 /* 2216 * share the pages with the first bio 2217 * and submit the write request 2218 */ 2219 d = r10_bio->devs[1].devnum; 2220 wbio = r10_bio->devs[1].bio; 2221 wbio2 = r10_bio->devs[1].repl_bio; 2222 /* Need to test wbio2->bi_end_io before we call 2223 * generic_make_request as if the former is NULL, 2224 * the latter is free to free wbio2. 2225 */ 2226 if (wbio2 && !wbio2->bi_end_io) 2227 wbio2 = NULL; 2228 if (wbio->bi_end_io) { 2229 atomic_inc(&conf->mirrors[d].rdev->nr_pending); 2230 md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio)); 2231 generic_make_request(wbio); 2232 } 2233 if (wbio2) { 2234 atomic_inc(&conf->mirrors[d].replacement->nr_pending); 2235 md_sync_acct(conf->mirrors[d].replacement->bdev, 2236 bio_sectors(wbio2)); 2237 generic_make_request(wbio2); 2238 } 2239 } 2240 2241 /* 2242 * Used by fix_read_error() to decay the per rdev read_errors. 2243 * We halve the read error count for every hour that has elapsed 2244 * since the last recorded read error. 2245 * 2246 */ 2247 static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev) 2248 { 2249 long cur_time_mon; 2250 unsigned long hours_since_last; 2251 unsigned int read_errors = atomic_read(&rdev->read_errors); 2252 2253 cur_time_mon = ktime_get_seconds(); 2254 2255 if (rdev->last_read_error == 0) { 2256 /* first time we've seen a read error */ 2257 rdev->last_read_error = cur_time_mon; 2258 return; 2259 } 2260 2261 hours_since_last = (long)(cur_time_mon - 2262 rdev->last_read_error) / 3600; 2263 2264 rdev->last_read_error = cur_time_mon; 2265 2266 /* 2267 * if hours_since_last is > the number of bits in read_errors 2268 * just set read errors to 0. We do this to avoid 2269 * overflowing the shift of read_errors by hours_since_last. 2270 */ 2271 if (hours_since_last >= 8 * sizeof(read_errors)) 2272 atomic_set(&rdev->read_errors, 0); 2273 else 2274 atomic_set(&rdev->read_errors, read_errors >> hours_since_last); 2275 } 2276 2277 static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector, 2278 int sectors, struct page *page, int rw) 2279 { 2280 sector_t first_bad; 2281 int bad_sectors; 2282 2283 if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors) 2284 && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags))) 2285 return -1; 2286 if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false)) 2287 /* success */ 2288 return 1; 2289 if (rw == WRITE) { 2290 set_bit(WriteErrorSeen, &rdev->flags); 2291 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2292 set_bit(MD_RECOVERY_NEEDED, 2293 &rdev->mddev->recovery); 2294 } 2295 /* need to record an error - either for the block or the device */ 2296 if (!rdev_set_badblocks(rdev, sector, sectors, 0)) 2297 md_error(rdev->mddev, rdev); 2298 return 0; 2299 } 2300 2301 /* 2302 * This is a kernel thread which: 2303 * 2304 * 1. Retries failed read operations on working mirrors. 2305 * 2. Updates the raid superblock when problems encounter. 2306 * 3. Performs writes following reads for array synchronising. 2307 */ 2308 2309 static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio) 2310 { 2311 int sect = 0; /* Offset from r10_bio->sector */ 2312 int sectors = r10_bio->sectors; 2313 struct md_rdev*rdev; 2314 int max_read_errors = atomic_read(&mddev->max_corr_read_errors); 2315 int d = r10_bio->devs[r10_bio->read_slot].devnum; 2316 2317 /* still own a reference to this rdev, so it cannot 2318 * have been cleared recently. 2319 */ 2320 rdev = conf->mirrors[d].rdev; 2321 2322 if (test_bit(Faulty, &rdev->flags)) 2323 /* drive has already been failed, just ignore any 2324 more fix_read_error() attempts */ 2325 return; 2326 2327 check_decay_read_errors(mddev, rdev); 2328 atomic_inc(&rdev->read_errors); 2329 if (atomic_read(&rdev->read_errors) > max_read_errors) { 2330 char b[BDEVNAME_SIZE]; 2331 bdevname(rdev->bdev, b); 2332 2333 pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n", 2334 mdname(mddev), b, 2335 atomic_read(&rdev->read_errors), max_read_errors); 2336 pr_notice("md/raid10:%s: %s: Failing raid device\n", 2337 mdname(mddev), b); 2338 md_error(mddev, rdev); 2339 r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED; 2340 return; 2341 } 2342 2343 while(sectors) { 2344 int s = sectors; 2345 int sl = r10_bio->read_slot; 2346 int success = 0; 2347 int start; 2348 2349 if (s > (PAGE_SIZE>>9)) 2350 s = PAGE_SIZE >> 9; 2351 2352 rcu_read_lock(); 2353 do { 2354 sector_t first_bad; 2355 int bad_sectors; 2356 2357 d = r10_bio->devs[sl].devnum; 2358 rdev = rcu_dereference(conf->mirrors[d].rdev); 2359 if (rdev && 2360 test_bit(In_sync, &rdev->flags) && 2361 !test_bit(Faulty, &rdev->flags) && 2362 is_badblock(rdev, r10_bio->devs[sl].addr + sect, s, 2363 &first_bad, &bad_sectors) == 0) { 2364 atomic_inc(&rdev->nr_pending); 2365 rcu_read_unlock(); 2366 success = sync_page_io(rdev, 2367 r10_bio->devs[sl].addr + 2368 sect, 2369 s<<9, 2370 conf->tmppage, 2371 REQ_OP_READ, 0, false); 2372 rdev_dec_pending(rdev, mddev); 2373 rcu_read_lock(); 2374 if (success) 2375 break; 2376 } 2377 sl++; 2378 if (sl == conf->copies) 2379 sl = 0; 2380 } while (!success && sl != r10_bio->read_slot); 2381 rcu_read_unlock(); 2382 2383 if (!success) { 2384 /* Cannot read from anywhere, just mark the block 2385 * as bad on the first device to discourage future 2386 * reads. 2387 */ 2388 int dn = r10_bio->devs[r10_bio->read_slot].devnum; 2389 rdev = conf->mirrors[dn].rdev; 2390 2391 if (!rdev_set_badblocks( 2392 rdev, 2393 r10_bio->devs[r10_bio->read_slot].addr 2394 + sect, 2395 s, 0)) { 2396 md_error(mddev, rdev); 2397 r10_bio->devs[r10_bio->read_slot].bio 2398 = IO_BLOCKED; 2399 } 2400 break; 2401 } 2402 2403 start = sl; 2404 /* write it back and re-read */ 2405 rcu_read_lock(); 2406 while (sl != r10_bio->read_slot) { 2407 char b[BDEVNAME_SIZE]; 2408 2409 if (sl==0) 2410 sl = conf->copies; 2411 sl--; 2412 d = r10_bio->devs[sl].devnum; 2413 rdev = rcu_dereference(conf->mirrors[d].rdev); 2414 if (!rdev || 2415 test_bit(Faulty, &rdev->flags) || 2416 !test_bit(In_sync, &rdev->flags)) 2417 continue; 2418 2419 atomic_inc(&rdev->nr_pending); 2420 rcu_read_unlock(); 2421 if (r10_sync_page_io(rdev, 2422 r10_bio->devs[sl].addr + 2423 sect, 2424 s, conf->tmppage, WRITE) 2425 == 0) { 2426 /* Well, this device is dead */ 2427 pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n", 2428 mdname(mddev), s, 2429 (unsigned long long)( 2430 sect + 2431 choose_data_offset(r10_bio, 2432 rdev)), 2433 bdevname(rdev->bdev, b)); 2434 pr_notice("md/raid10:%s: %s: failing drive\n", 2435 mdname(mddev), 2436 bdevname(rdev->bdev, b)); 2437 } 2438 rdev_dec_pending(rdev, mddev); 2439 rcu_read_lock(); 2440 } 2441 sl = start; 2442 while (sl != r10_bio->read_slot) { 2443 char b[BDEVNAME_SIZE]; 2444 2445 if (sl==0) 2446 sl = conf->copies; 2447 sl--; 2448 d = r10_bio->devs[sl].devnum; 2449 rdev = rcu_dereference(conf->mirrors[d].rdev); 2450 if (!rdev || 2451 test_bit(Faulty, &rdev->flags) || 2452 !test_bit(In_sync, &rdev->flags)) 2453 continue; 2454 2455 atomic_inc(&rdev->nr_pending); 2456 rcu_read_unlock(); 2457 switch (r10_sync_page_io(rdev, 2458 r10_bio->devs[sl].addr + 2459 sect, 2460 s, conf->tmppage, 2461 READ)) { 2462 case 0: 2463 /* Well, this device is dead */ 2464 pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n", 2465 mdname(mddev), s, 2466 (unsigned long long)( 2467 sect + 2468 choose_data_offset(r10_bio, rdev)), 2469 bdevname(rdev->bdev, b)); 2470 pr_notice("md/raid10:%s: %s: failing drive\n", 2471 mdname(mddev), 2472 bdevname(rdev->bdev, b)); 2473 break; 2474 case 1: 2475 pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n", 2476 mdname(mddev), s, 2477 (unsigned long long)( 2478 sect + 2479 choose_data_offset(r10_bio, rdev)), 2480 bdevname(rdev->bdev, b)); 2481 atomic_add(s, &rdev->corrected_errors); 2482 } 2483 2484 rdev_dec_pending(rdev, mddev); 2485 rcu_read_lock(); 2486 } 2487 rcu_read_unlock(); 2488 2489 sectors -= s; 2490 sect += s; 2491 } 2492 } 2493 2494 static int narrow_write_error(struct r10bio *r10_bio, int i) 2495 { 2496 struct bio *bio = r10_bio->master_bio; 2497 struct mddev *mddev = r10_bio->mddev; 2498 struct r10conf *conf = mddev->private; 2499 struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev; 2500 /* bio has the data to be written to slot 'i' where 2501 * we just recently had a write error. 2502 * We repeatedly clone the bio and trim down to one block, 2503 * then try the write. Where the write fails we record 2504 * a bad block. 2505 * It is conceivable that the bio doesn't exactly align with 2506 * blocks. We must handle this. 2507 * 2508 * We currently own a reference to the rdev. 2509 */ 2510 2511 int block_sectors; 2512 sector_t sector; 2513 int sectors; 2514 int sect_to_write = r10_bio->sectors; 2515 int ok = 1; 2516 2517 if (rdev->badblocks.shift < 0) 2518 return 0; 2519 2520 block_sectors = roundup(1 << rdev->badblocks.shift, 2521 bdev_logical_block_size(rdev->bdev) >> 9); 2522 sector = r10_bio->sector; 2523 sectors = ((r10_bio->sector + block_sectors) 2524 & ~(sector_t)(block_sectors - 1)) 2525 - sector; 2526 2527 while (sect_to_write) { 2528 struct bio *wbio; 2529 sector_t wsector; 2530 if (sectors > sect_to_write) 2531 sectors = sect_to_write; 2532 /* Write at 'sector' for 'sectors' */ 2533 wbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 2534 bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors); 2535 wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector); 2536 wbio->bi_iter.bi_sector = wsector + 2537 choose_data_offset(r10_bio, rdev); 2538 wbio->bi_bdev = rdev->bdev; 2539 bio_set_op_attrs(wbio, REQ_OP_WRITE, 0); 2540 2541 if (submit_bio_wait(wbio) < 0) 2542 /* Failure! */ 2543 ok = rdev_set_badblocks(rdev, wsector, 2544 sectors, 0) 2545 && ok; 2546 2547 bio_put(wbio); 2548 sect_to_write -= sectors; 2549 sector += sectors; 2550 sectors = block_sectors; 2551 } 2552 return ok; 2553 } 2554 2555 static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) 2556 { 2557 int slot = r10_bio->read_slot; 2558 struct bio *bio; 2559 struct r10conf *conf = mddev->private; 2560 struct md_rdev *rdev = r10_bio->devs[slot].rdev; 2561 char b[BDEVNAME_SIZE]; 2562 unsigned long do_sync; 2563 int max_sectors; 2564 dev_t bio_dev; 2565 sector_t bio_last_sector; 2566 2567 /* we got a read error. Maybe the drive is bad. Maybe just 2568 * the block and we can fix it. 2569 * We freeze all other IO, and try reading the block from 2570 * other devices. When we find one, we re-write 2571 * and check it that fixes the read error. 2572 * This is all done synchronously while the array is 2573 * frozen. 2574 */ 2575 bio = r10_bio->devs[slot].bio; 2576 bdevname(bio->bi_bdev, b); 2577 bio_dev = bio->bi_bdev->bd_dev; 2578 bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors; 2579 bio_put(bio); 2580 r10_bio->devs[slot].bio = NULL; 2581 2582 if (mddev->ro) 2583 r10_bio->devs[slot].bio = IO_BLOCKED; 2584 else if (!test_bit(FailFast, &rdev->flags)) { 2585 freeze_array(conf, 1); 2586 fix_read_error(conf, mddev, r10_bio); 2587 unfreeze_array(conf); 2588 } else 2589 md_error(mddev, rdev); 2590 2591 rdev_dec_pending(rdev, mddev); 2592 2593 read_more: 2594 rdev = read_balance(conf, r10_bio, &max_sectors); 2595 if (rdev == NULL) { 2596 pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n", 2597 mdname(mddev), b, 2598 (unsigned long long)r10_bio->sector); 2599 raid_end_bio_io(r10_bio); 2600 return; 2601 } 2602 2603 do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC); 2604 slot = r10_bio->read_slot; 2605 pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n", 2606 mdname(mddev), 2607 bdevname(rdev->bdev, b), 2608 (unsigned long long)r10_bio->sector); 2609 bio = bio_clone_mddev(r10_bio->master_bio, 2610 GFP_NOIO, mddev); 2611 bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors); 2612 r10_bio->devs[slot].bio = bio; 2613 r10_bio->devs[slot].rdev = rdev; 2614 bio->bi_iter.bi_sector = r10_bio->devs[slot].addr 2615 + choose_data_offset(r10_bio, rdev); 2616 bio->bi_bdev = rdev->bdev; 2617 bio_set_op_attrs(bio, REQ_OP_READ, do_sync); 2618 if (test_bit(FailFast, &rdev->flags) && 2619 test_bit(R10BIO_FailFast, &r10_bio->state)) 2620 bio->bi_opf |= MD_FAILFAST; 2621 bio->bi_private = r10_bio; 2622 bio->bi_end_io = raid10_end_read_request; 2623 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), 2624 bio, bio_dev, 2625 bio_last_sector - r10_bio->sectors); 2626 2627 if (max_sectors < r10_bio->sectors) { 2628 /* Drat - have to split this up more */ 2629 struct bio *mbio = r10_bio->master_bio; 2630 int sectors_handled = 2631 r10_bio->sector + max_sectors 2632 - mbio->bi_iter.bi_sector; 2633 r10_bio->sectors = max_sectors; 2634 spin_lock_irq(&conf->device_lock); 2635 if (mbio->bi_phys_segments == 0) 2636 mbio->bi_phys_segments = 2; 2637 else 2638 mbio->bi_phys_segments++; 2639 spin_unlock_irq(&conf->device_lock); 2640 generic_make_request(bio); 2641 2642 r10_bio = mempool_alloc(conf->r10bio_pool, 2643 GFP_NOIO); 2644 r10_bio->master_bio = mbio; 2645 r10_bio->sectors = bio_sectors(mbio) - sectors_handled; 2646 r10_bio->state = 0; 2647 set_bit(R10BIO_ReadError, 2648 &r10_bio->state); 2649 r10_bio->mddev = mddev; 2650 r10_bio->sector = mbio->bi_iter.bi_sector 2651 + sectors_handled; 2652 2653 goto read_more; 2654 } else 2655 generic_make_request(bio); 2656 } 2657 2658 static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio) 2659 { 2660 /* Some sort of write request has finished and it 2661 * succeeded in writing where we thought there was a 2662 * bad block. So forget the bad block. 2663 * Or possibly if failed and we need to record 2664 * a bad block. 2665 */ 2666 int m; 2667 struct md_rdev *rdev; 2668 2669 if (test_bit(R10BIO_IsSync, &r10_bio->state) || 2670 test_bit(R10BIO_IsRecover, &r10_bio->state)) { 2671 for (m = 0; m < conf->copies; m++) { 2672 int dev = r10_bio->devs[m].devnum; 2673 rdev = conf->mirrors[dev].rdev; 2674 if (r10_bio->devs[m].bio == NULL) 2675 continue; 2676 if (!r10_bio->devs[m].bio->bi_error) { 2677 rdev_clear_badblocks( 2678 rdev, 2679 r10_bio->devs[m].addr, 2680 r10_bio->sectors, 0); 2681 } else { 2682 if (!rdev_set_badblocks( 2683 rdev, 2684 r10_bio->devs[m].addr, 2685 r10_bio->sectors, 0)) 2686 md_error(conf->mddev, rdev); 2687 } 2688 rdev = conf->mirrors[dev].replacement; 2689 if (r10_bio->devs[m].repl_bio == NULL) 2690 continue; 2691 2692 if (!r10_bio->devs[m].repl_bio->bi_error) { 2693 rdev_clear_badblocks( 2694 rdev, 2695 r10_bio->devs[m].addr, 2696 r10_bio->sectors, 0); 2697 } else { 2698 if (!rdev_set_badblocks( 2699 rdev, 2700 r10_bio->devs[m].addr, 2701 r10_bio->sectors, 0)) 2702 md_error(conf->mddev, rdev); 2703 } 2704 } 2705 put_buf(r10_bio); 2706 } else { 2707 bool fail = false; 2708 for (m = 0; m < conf->copies; m++) { 2709 int dev = r10_bio->devs[m].devnum; 2710 struct bio *bio = r10_bio->devs[m].bio; 2711 rdev = conf->mirrors[dev].rdev; 2712 if (bio == IO_MADE_GOOD) { 2713 rdev_clear_badblocks( 2714 rdev, 2715 r10_bio->devs[m].addr, 2716 r10_bio->sectors, 0); 2717 rdev_dec_pending(rdev, conf->mddev); 2718 } else if (bio != NULL && bio->bi_error) { 2719 fail = true; 2720 if (!narrow_write_error(r10_bio, m)) { 2721 md_error(conf->mddev, rdev); 2722 set_bit(R10BIO_Degraded, 2723 &r10_bio->state); 2724 } 2725 rdev_dec_pending(rdev, conf->mddev); 2726 } 2727 bio = r10_bio->devs[m].repl_bio; 2728 rdev = conf->mirrors[dev].replacement; 2729 if (rdev && bio == IO_MADE_GOOD) { 2730 rdev_clear_badblocks( 2731 rdev, 2732 r10_bio->devs[m].addr, 2733 r10_bio->sectors, 0); 2734 rdev_dec_pending(rdev, conf->mddev); 2735 } 2736 } 2737 if (fail) { 2738 spin_lock_irq(&conf->device_lock); 2739 list_add(&r10_bio->retry_list, &conf->bio_end_io_list); 2740 conf->nr_queued++; 2741 spin_unlock_irq(&conf->device_lock); 2742 md_wakeup_thread(conf->mddev->thread); 2743 } else { 2744 if (test_bit(R10BIO_WriteError, 2745 &r10_bio->state)) 2746 close_write(r10_bio); 2747 raid_end_bio_io(r10_bio); 2748 } 2749 } 2750 } 2751 2752 static void raid10d(struct md_thread *thread) 2753 { 2754 struct mddev *mddev = thread->mddev; 2755 struct r10bio *r10_bio; 2756 unsigned long flags; 2757 struct r10conf *conf = mddev->private; 2758 struct list_head *head = &conf->retry_list; 2759 struct blk_plug plug; 2760 2761 md_check_recovery(mddev); 2762 2763 if (!list_empty_careful(&conf->bio_end_io_list) && 2764 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2765 LIST_HEAD(tmp); 2766 spin_lock_irqsave(&conf->device_lock, flags); 2767 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 2768 while (!list_empty(&conf->bio_end_io_list)) { 2769 list_move(conf->bio_end_io_list.prev, &tmp); 2770 conf->nr_queued--; 2771 } 2772 } 2773 spin_unlock_irqrestore(&conf->device_lock, flags); 2774 while (!list_empty(&tmp)) { 2775 r10_bio = list_first_entry(&tmp, struct r10bio, 2776 retry_list); 2777 list_del(&r10_bio->retry_list); 2778 if (mddev->degraded) 2779 set_bit(R10BIO_Degraded, &r10_bio->state); 2780 2781 if (test_bit(R10BIO_WriteError, 2782 &r10_bio->state)) 2783 close_write(r10_bio); 2784 raid_end_bio_io(r10_bio); 2785 } 2786 } 2787 2788 blk_start_plug(&plug); 2789 for (;;) { 2790 2791 flush_pending_writes(conf); 2792 2793 spin_lock_irqsave(&conf->device_lock, flags); 2794 if (list_empty(head)) { 2795 spin_unlock_irqrestore(&conf->device_lock, flags); 2796 break; 2797 } 2798 r10_bio = list_entry(head->prev, struct r10bio, retry_list); 2799 list_del(head->prev); 2800 conf->nr_queued--; 2801 spin_unlock_irqrestore(&conf->device_lock, flags); 2802 2803 mddev = r10_bio->mddev; 2804 conf = mddev->private; 2805 if (test_bit(R10BIO_MadeGood, &r10_bio->state) || 2806 test_bit(R10BIO_WriteError, &r10_bio->state)) 2807 handle_write_completed(conf, r10_bio); 2808 else if (test_bit(R10BIO_IsReshape, &r10_bio->state)) 2809 reshape_request_write(mddev, r10_bio); 2810 else if (test_bit(R10BIO_IsSync, &r10_bio->state)) 2811 sync_request_write(mddev, r10_bio); 2812 else if (test_bit(R10BIO_IsRecover, &r10_bio->state)) 2813 recovery_request_write(mddev, r10_bio); 2814 else if (test_bit(R10BIO_ReadError, &r10_bio->state)) 2815 handle_read_error(mddev, r10_bio); 2816 else { 2817 /* just a partial read to be scheduled from a 2818 * separate context 2819 */ 2820 int slot = r10_bio->read_slot; 2821 generic_make_request(r10_bio->devs[slot].bio); 2822 } 2823 2824 cond_resched(); 2825 if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING)) 2826 md_check_recovery(mddev); 2827 } 2828 blk_finish_plug(&plug); 2829 } 2830 2831 static int init_resync(struct r10conf *conf) 2832 { 2833 int buffs; 2834 int i; 2835 2836 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 2837 BUG_ON(conf->r10buf_pool); 2838 conf->have_replacement = 0; 2839 for (i = 0; i < conf->geo.raid_disks; i++) 2840 if (conf->mirrors[i].replacement) 2841 conf->have_replacement = 1; 2842 conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf); 2843 if (!conf->r10buf_pool) 2844 return -ENOMEM; 2845 conf->next_resync = 0; 2846 return 0; 2847 } 2848 2849 /* 2850 * perform a "sync" on one "block" 2851 * 2852 * We need to make sure that no normal I/O request - particularly write 2853 * requests - conflict with active sync requests. 2854 * 2855 * This is achieved by tracking pending requests and a 'barrier' concept 2856 * that can be installed to exclude normal IO requests. 2857 * 2858 * Resync and recovery are handled very differently. 2859 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery. 2860 * 2861 * For resync, we iterate over virtual addresses, read all copies, 2862 * and update if there are differences. If only one copy is live, 2863 * skip it. 2864 * For recovery, we iterate over physical addresses, read a good 2865 * value for each non-in_sync drive, and over-write. 2866 * 2867 * So, for recovery we may have several outstanding complex requests for a 2868 * given address, one for each out-of-sync device. We model this by allocating 2869 * a number of r10_bio structures, one for each out-of-sync device. 2870 * As we setup these structures, we collect all bio's together into a list 2871 * which we then process collectively to add pages, and then process again 2872 * to pass to generic_make_request. 2873 * 2874 * The r10_bio structures are linked using a borrowed master_bio pointer. 2875 * This link is counted in ->remaining. When the r10_bio that points to NULL 2876 * has its remaining count decremented to 0, the whole complex operation 2877 * is complete. 2878 * 2879 */ 2880 2881 static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, 2882 int *skipped) 2883 { 2884 struct r10conf *conf = mddev->private; 2885 struct r10bio *r10_bio; 2886 struct bio *biolist = NULL, *bio; 2887 sector_t max_sector, nr_sectors; 2888 int i; 2889 int max_sync; 2890 sector_t sync_blocks; 2891 sector_t sectors_skipped = 0; 2892 int chunks_skipped = 0; 2893 sector_t chunk_mask = conf->geo.chunk_mask; 2894 2895 if (!conf->r10buf_pool) 2896 if (init_resync(conf)) 2897 return 0; 2898 2899 /* 2900 * Allow skipping a full rebuild for incremental assembly 2901 * of a clean array, like RAID1 does. 2902 */ 2903 if (mddev->bitmap == NULL && 2904 mddev->recovery_cp == MaxSector && 2905 mddev->reshape_position == MaxSector && 2906 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && 2907 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 2908 !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && 2909 conf->fullsync == 0) { 2910 *skipped = 1; 2911 return mddev->dev_sectors - sector_nr; 2912 } 2913 2914 skipped: 2915 max_sector = mddev->dev_sectors; 2916 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) || 2917 test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2918 max_sector = mddev->resync_max_sectors; 2919 if (sector_nr >= max_sector) { 2920 /* If we aborted, we need to abort the 2921 * sync on the 'current' bitmap chucks (there can 2922 * be several when recovering multiple devices). 2923 * as we may have started syncing it but not finished. 2924 * We can find the current address in 2925 * mddev->curr_resync, but for recovery, 2926 * we need to convert that to several 2927 * virtual addresses. 2928 */ 2929 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 2930 end_reshape(conf); 2931 close_sync(conf); 2932 return 0; 2933 } 2934 2935 if (mddev->curr_resync < max_sector) { /* aborted */ 2936 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) 2937 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 2938 &sync_blocks, 1); 2939 else for (i = 0; i < conf->geo.raid_disks; i++) { 2940 sector_t sect = 2941 raid10_find_virt(conf, mddev->curr_resync, i); 2942 bitmap_end_sync(mddev->bitmap, sect, 2943 &sync_blocks, 1); 2944 } 2945 } else { 2946 /* completed sync */ 2947 if ((!mddev->bitmap || conf->fullsync) 2948 && conf->have_replacement 2949 && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 2950 /* Completed a full sync so the replacements 2951 * are now fully recovered. 2952 */ 2953 rcu_read_lock(); 2954 for (i = 0; i < conf->geo.raid_disks; i++) { 2955 struct md_rdev *rdev = 2956 rcu_dereference(conf->mirrors[i].replacement); 2957 if (rdev) 2958 rdev->recovery_offset = MaxSector; 2959 } 2960 rcu_read_unlock(); 2961 } 2962 conf->fullsync = 0; 2963 } 2964 bitmap_close_sync(mddev->bitmap); 2965 close_sync(conf); 2966 *skipped = 1; 2967 return sectors_skipped; 2968 } 2969 2970 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 2971 return reshape_request(mddev, sector_nr, skipped); 2972 2973 if (chunks_skipped >= conf->geo.raid_disks) { 2974 /* if there has been nothing to do on any drive, 2975 * then there is nothing to do at all.. 2976 */ 2977 *skipped = 1; 2978 return (max_sector - sector_nr) + sectors_skipped; 2979 } 2980 2981 if (max_sector > mddev->resync_max) 2982 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 2983 2984 /* make sure whole request will fit in a chunk - if chunks 2985 * are meaningful 2986 */ 2987 if (conf->geo.near_copies < conf->geo.raid_disks && 2988 max_sector > (sector_nr | chunk_mask)) 2989 max_sector = (sector_nr | chunk_mask) + 1; 2990 2991 /* 2992 * If there is non-resync activity waiting for a turn, then let it 2993 * though before starting on this new sync request. 2994 */ 2995 if (conf->nr_waiting) 2996 schedule_timeout_uninterruptible(1); 2997 2998 /* Again, very different code for resync and recovery. 2999 * Both must result in an r10bio with a list of bios that 3000 * have bi_end_io, bi_sector, bi_bdev set, 3001 * and bi_private set to the r10bio. 3002 * For recovery, we may actually create several r10bios 3003 * with 2 bios in each, that correspond to the bios in the main one. 3004 * In this case, the subordinate r10bios link back through a 3005 * borrowed master_bio pointer, and the counter in the master 3006 * includes a ref from each subordinate. 3007 */ 3008 /* First, we decide what to do and set ->bi_end_io 3009 * To end_sync_read if we want to read, and 3010 * end_sync_write if we will want to write. 3011 */ 3012 3013 max_sync = RESYNC_PAGES << (PAGE_SHIFT-9); 3014 if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3015 /* recovery... the complicated one */ 3016 int j; 3017 r10_bio = NULL; 3018 3019 for (i = 0 ; i < conf->geo.raid_disks; i++) { 3020 int still_degraded; 3021 struct r10bio *rb2; 3022 sector_t sect; 3023 int must_sync; 3024 int any_working; 3025 struct raid10_info *mirror = &conf->mirrors[i]; 3026 struct md_rdev *mrdev, *mreplace; 3027 3028 rcu_read_lock(); 3029 mrdev = rcu_dereference(mirror->rdev); 3030 mreplace = rcu_dereference(mirror->replacement); 3031 3032 if ((mrdev == NULL || 3033 test_bit(Faulty, &mrdev->flags) || 3034 test_bit(In_sync, &mrdev->flags)) && 3035 (mreplace == NULL || 3036 test_bit(Faulty, &mreplace->flags))) { 3037 rcu_read_unlock(); 3038 continue; 3039 } 3040 3041 still_degraded = 0; 3042 /* want to reconstruct this device */ 3043 rb2 = r10_bio; 3044 sect = raid10_find_virt(conf, sector_nr, i); 3045 if (sect >= mddev->resync_max_sectors) { 3046 /* last stripe is not complete - don't 3047 * try to recover this sector. 3048 */ 3049 rcu_read_unlock(); 3050 continue; 3051 } 3052 if (mreplace && test_bit(Faulty, &mreplace->flags)) 3053 mreplace = NULL; 3054 /* Unless we are doing a full sync, or a replacement 3055 * we only need to recover the block if it is set in 3056 * the bitmap 3057 */ 3058 must_sync = bitmap_start_sync(mddev->bitmap, sect, 3059 &sync_blocks, 1); 3060 if (sync_blocks < max_sync) 3061 max_sync = sync_blocks; 3062 if (!must_sync && 3063 mreplace == NULL && 3064 !conf->fullsync) { 3065 /* yep, skip the sync_blocks here, but don't assume 3066 * that there will never be anything to do here 3067 */ 3068 chunks_skipped = -1; 3069 rcu_read_unlock(); 3070 continue; 3071 } 3072 atomic_inc(&mrdev->nr_pending); 3073 if (mreplace) 3074 atomic_inc(&mreplace->nr_pending); 3075 rcu_read_unlock(); 3076 3077 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3078 r10_bio->state = 0; 3079 raise_barrier(conf, rb2 != NULL); 3080 atomic_set(&r10_bio->remaining, 0); 3081 3082 r10_bio->master_bio = (struct bio*)rb2; 3083 if (rb2) 3084 atomic_inc(&rb2->remaining); 3085 r10_bio->mddev = mddev; 3086 set_bit(R10BIO_IsRecover, &r10_bio->state); 3087 r10_bio->sector = sect; 3088 3089 raid10_find_phys(conf, r10_bio); 3090 3091 /* Need to check if the array will still be 3092 * degraded 3093 */ 3094 rcu_read_lock(); 3095 for (j = 0; j < conf->geo.raid_disks; j++) { 3096 struct md_rdev *rdev = rcu_dereference( 3097 conf->mirrors[j].rdev); 3098 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3099 still_degraded = 1; 3100 break; 3101 } 3102 } 3103 3104 must_sync = bitmap_start_sync(mddev->bitmap, sect, 3105 &sync_blocks, still_degraded); 3106 3107 any_working = 0; 3108 for (j=0; j<conf->copies;j++) { 3109 int k; 3110 int d = r10_bio->devs[j].devnum; 3111 sector_t from_addr, to_addr; 3112 struct md_rdev *rdev = 3113 rcu_dereference(conf->mirrors[d].rdev); 3114 sector_t sector, first_bad; 3115 int bad_sectors; 3116 if (!rdev || 3117 !test_bit(In_sync, &rdev->flags)) 3118 continue; 3119 /* This is where we read from */ 3120 any_working = 1; 3121 sector = r10_bio->devs[j].addr; 3122 3123 if (is_badblock(rdev, sector, max_sync, 3124 &first_bad, &bad_sectors)) { 3125 if (first_bad > sector) 3126 max_sync = first_bad - sector; 3127 else { 3128 bad_sectors -= (sector 3129 - first_bad); 3130 if (max_sync > bad_sectors) 3131 max_sync = bad_sectors; 3132 continue; 3133 } 3134 } 3135 bio = r10_bio->devs[0].bio; 3136 bio_reset(bio); 3137 bio->bi_next = biolist; 3138 biolist = bio; 3139 bio->bi_private = r10_bio; 3140 bio->bi_end_io = end_sync_read; 3141 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3142 if (test_bit(FailFast, &rdev->flags)) 3143 bio->bi_opf |= MD_FAILFAST; 3144 from_addr = r10_bio->devs[j].addr; 3145 bio->bi_iter.bi_sector = from_addr + 3146 rdev->data_offset; 3147 bio->bi_bdev = rdev->bdev; 3148 atomic_inc(&rdev->nr_pending); 3149 /* and we write to 'i' (if not in_sync) */ 3150 3151 for (k=0; k<conf->copies; k++) 3152 if (r10_bio->devs[k].devnum == i) 3153 break; 3154 BUG_ON(k == conf->copies); 3155 to_addr = r10_bio->devs[k].addr; 3156 r10_bio->devs[0].devnum = d; 3157 r10_bio->devs[0].addr = from_addr; 3158 r10_bio->devs[1].devnum = i; 3159 r10_bio->devs[1].addr = to_addr; 3160 3161 if (!test_bit(In_sync, &mrdev->flags)) { 3162 bio = r10_bio->devs[1].bio; 3163 bio_reset(bio); 3164 bio->bi_next = biolist; 3165 biolist = bio; 3166 bio->bi_private = r10_bio; 3167 bio->bi_end_io = end_sync_write; 3168 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3169 bio->bi_iter.bi_sector = to_addr 3170 + mrdev->data_offset; 3171 bio->bi_bdev = mrdev->bdev; 3172 atomic_inc(&r10_bio->remaining); 3173 } else 3174 r10_bio->devs[1].bio->bi_end_io = NULL; 3175 3176 /* and maybe write to replacement */ 3177 bio = r10_bio->devs[1].repl_bio; 3178 if (bio) 3179 bio->bi_end_io = NULL; 3180 /* Note: if mreplace != NULL, then bio 3181 * cannot be NULL as r10buf_pool_alloc will 3182 * have allocated it. 3183 * So the second test here is pointless. 3184 * But it keeps semantic-checkers happy, and 3185 * this comment keeps human reviewers 3186 * happy. 3187 */ 3188 if (mreplace == NULL || bio == NULL || 3189 test_bit(Faulty, &mreplace->flags)) 3190 break; 3191 bio_reset(bio); 3192 bio->bi_next = biolist; 3193 biolist = bio; 3194 bio->bi_private = r10_bio; 3195 bio->bi_end_io = end_sync_write; 3196 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3197 bio->bi_iter.bi_sector = to_addr + 3198 mreplace->data_offset; 3199 bio->bi_bdev = mreplace->bdev; 3200 atomic_inc(&r10_bio->remaining); 3201 break; 3202 } 3203 rcu_read_unlock(); 3204 if (j == conf->copies) { 3205 /* Cannot recover, so abort the recovery or 3206 * record a bad block */ 3207 if (any_working) { 3208 /* problem is that there are bad blocks 3209 * on other device(s) 3210 */ 3211 int k; 3212 for (k = 0; k < conf->copies; k++) 3213 if (r10_bio->devs[k].devnum == i) 3214 break; 3215 if (!test_bit(In_sync, 3216 &mrdev->flags) 3217 && !rdev_set_badblocks( 3218 mrdev, 3219 r10_bio->devs[k].addr, 3220 max_sync, 0)) 3221 any_working = 0; 3222 if (mreplace && 3223 !rdev_set_badblocks( 3224 mreplace, 3225 r10_bio->devs[k].addr, 3226 max_sync, 0)) 3227 any_working = 0; 3228 } 3229 if (!any_working) { 3230 if (!test_and_set_bit(MD_RECOVERY_INTR, 3231 &mddev->recovery)) 3232 pr_warn("md/raid10:%s: insufficient working devices for recovery.\n", 3233 mdname(mddev)); 3234 mirror->recovery_disabled 3235 = mddev->recovery_disabled; 3236 } 3237 put_buf(r10_bio); 3238 if (rb2) 3239 atomic_dec(&rb2->remaining); 3240 r10_bio = rb2; 3241 rdev_dec_pending(mrdev, mddev); 3242 if (mreplace) 3243 rdev_dec_pending(mreplace, mddev); 3244 break; 3245 } 3246 rdev_dec_pending(mrdev, mddev); 3247 if (mreplace) 3248 rdev_dec_pending(mreplace, mddev); 3249 if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) { 3250 /* Only want this if there is elsewhere to 3251 * read from. 'j' is currently the first 3252 * readable copy. 3253 */ 3254 int targets = 1; 3255 for (; j < conf->copies; j++) { 3256 int d = r10_bio->devs[j].devnum; 3257 if (conf->mirrors[d].rdev && 3258 test_bit(In_sync, 3259 &conf->mirrors[d].rdev->flags)) 3260 targets++; 3261 } 3262 if (targets == 1) 3263 r10_bio->devs[0].bio->bi_opf 3264 &= ~MD_FAILFAST; 3265 } 3266 } 3267 if (biolist == NULL) { 3268 while (r10_bio) { 3269 struct r10bio *rb2 = r10_bio; 3270 r10_bio = (struct r10bio*) rb2->master_bio; 3271 rb2->master_bio = NULL; 3272 put_buf(rb2); 3273 } 3274 goto giveup; 3275 } 3276 } else { 3277 /* resync. Schedule a read for every block at this virt offset */ 3278 int count = 0; 3279 3280 bitmap_cond_end_sync(mddev->bitmap, sector_nr, 0); 3281 3282 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 3283 &sync_blocks, mddev->degraded) && 3284 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, 3285 &mddev->recovery)) { 3286 /* We can skip this block */ 3287 *skipped = 1; 3288 return sync_blocks + sectors_skipped; 3289 } 3290 if (sync_blocks < max_sync) 3291 max_sync = sync_blocks; 3292 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 3293 r10_bio->state = 0; 3294 3295 r10_bio->mddev = mddev; 3296 atomic_set(&r10_bio->remaining, 0); 3297 raise_barrier(conf, 0); 3298 conf->next_resync = sector_nr; 3299 3300 r10_bio->master_bio = NULL; 3301 r10_bio->sector = sector_nr; 3302 set_bit(R10BIO_IsSync, &r10_bio->state); 3303 raid10_find_phys(conf, r10_bio); 3304 r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1; 3305 3306 for (i = 0; i < conf->copies; i++) { 3307 int d = r10_bio->devs[i].devnum; 3308 sector_t first_bad, sector; 3309 int bad_sectors; 3310 struct md_rdev *rdev; 3311 3312 if (r10_bio->devs[i].repl_bio) 3313 r10_bio->devs[i].repl_bio->bi_end_io = NULL; 3314 3315 bio = r10_bio->devs[i].bio; 3316 bio_reset(bio); 3317 bio->bi_error = -EIO; 3318 rcu_read_lock(); 3319 rdev = rcu_dereference(conf->mirrors[d].rdev); 3320 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3321 rcu_read_unlock(); 3322 continue; 3323 } 3324 sector = r10_bio->devs[i].addr; 3325 if (is_badblock(rdev, sector, max_sync, 3326 &first_bad, &bad_sectors)) { 3327 if (first_bad > sector) 3328 max_sync = first_bad - sector; 3329 else { 3330 bad_sectors -= (sector - first_bad); 3331 if (max_sync > bad_sectors) 3332 max_sync = bad_sectors; 3333 rcu_read_unlock(); 3334 continue; 3335 } 3336 } 3337 atomic_inc(&rdev->nr_pending); 3338 atomic_inc(&r10_bio->remaining); 3339 bio->bi_next = biolist; 3340 biolist = bio; 3341 bio->bi_private = r10_bio; 3342 bio->bi_end_io = end_sync_read; 3343 bio_set_op_attrs(bio, REQ_OP_READ, 0); 3344 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags)) 3345 bio->bi_opf |= MD_FAILFAST; 3346 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3347 bio->bi_bdev = rdev->bdev; 3348 count++; 3349 3350 rdev = rcu_dereference(conf->mirrors[d].replacement); 3351 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { 3352 rcu_read_unlock(); 3353 continue; 3354 } 3355 atomic_inc(&rdev->nr_pending); 3356 rcu_read_unlock(); 3357 3358 /* Need to set up for writing to the replacement */ 3359 bio = r10_bio->devs[i].repl_bio; 3360 bio_reset(bio); 3361 bio->bi_error = -EIO; 3362 3363 sector = r10_bio->devs[i].addr; 3364 bio->bi_next = biolist; 3365 biolist = bio; 3366 bio->bi_private = r10_bio; 3367 bio->bi_end_io = end_sync_write; 3368 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 3369 if (test_bit(FailFast, &conf->mirrors[d].rdev->flags)) 3370 bio->bi_opf |= MD_FAILFAST; 3371 bio->bi_iter.bi_sector = sector + rdev->data_offset; 3372 bio->bi_bdev = rdev->bdev; 3373 count++; 3374 } 3375 3376 if (count < 2) { 3377 for (i=0; i<conf->copies; i++) { 3378 int d = r10_bio->devs[i].devnum; 3379 if (r10_bio->devs[i].bio->bi_end_io) 3380 rdev_dec_pending(conf->mirrors[d].rdev, 3381 mddev); 3382 if (r10_bio->devs[i].repl_bio && 3383 r10_bio->devs[i].repl_bio->bi_end_io) 3384 rdev_dec_pending( 3385 conf->mirrors[d].replacement, 3386 mddev); 3387 } 3388 put_buf(r10_bio); 3389 biolist = NULL; 3390 goto giveup; 3391 } 3392 } 3393 3394 nr_sectors = 0; 3395 if (sector_nr + max_sync < max_sector) 3396 max_sector = sector_nr + max_sync; 3397 do { 3398 struct page *page; 3399 int len = PAGE_SIZE; 3400 if (sector_nr + (len>>9) > max_sector) 3401 len = (max_sector - sector_nr) << 9; 3402 if (len == 0) 3403 break; 3404 for (bio= biolist ; bio ; bio=bio->bi_next) { 3405 struct bio *bio2; 3406 page = bio->bi_io_vec[bio->bi_vcnt].bv_page; 3407 if (bio_add_page(bio, page, len, 0)) 3408 continue; 3409 3410 /* stop here */ 3411 bio->bi_io_vec[bio->bi_vcnt].bv_page = page; 3412 for (bio2 = biolist; 3413 bio2 && bio2 != bio; 3414 bio2 = bio2->bi_next) { 3415 /* remove last page from this bio */ 3416 bio2->bi_vcnt--; 3417 bio2->bi_iter.bi_size -= len; 3418 bio_clear_flag(bio2, BIO_SEG_VALID); 3419 } 3420 goto bio_full; 3421 } 3422 nr_sectors += len>>9; 3423 sector_nr += len>>9; 3424 } while (biolist->bi_vcnt < RESYNC_PAGES); 3425 bio_full: 3426 r10_bio->sectors = nr_sectors; 3427 3428 while (biolist) { 3429 bio = biolist; 3430 biolist = biolist->bi_next; 3431 3432 bio->bi_next = NULL; 3433 r10_bio = bio->bi_private; 3434 r10_bio->sectors = nr_sectors; 3435 3436 if (bio->bi_end_io == end_sync_read) { 3437 md_sync_acct(bio->bi_bdev, nr_sectors); 3438 bio->bi_error = 0; 3439 generic_make_request(bio); 3440 } 3441 } 3442 3443 if (sectors_skipped) 3444 /* pretend they weren't skipped, it makes 3445 * no important difference in this case 3446 */ 3447 md_done_sync(mddev, sectors_skipped, 1); 3448 3449 return sectors_skipped + nr_sectors; 3450 giveup: 3451 /* There is nowhere to write, so all non-sync 3452 * drives must be failed or in resync, all drives 3453 * have a bad block, so try the next chunk... 3454 */ 3455 if (sector_nr + max_sync < max_sector) 3456 max_sector = sector_nr + max_sync; 3457 3458 sectors_skipped += (max_sector - sector_nr); 3459 chunks_skipped ++; 3460 sector_nr = max_sector; 3461 goto skipped; 3462 } 3463 3464 static sector_t 3465 raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks) 3466 { 3467 sector_t size; 3468 struct r10conf *conf = mddev->private; 3469 3470 if (!raid_disks) 3471 raid_disks = min(conf->geo.raid_disks, 3472 conf->prev.raid_disks); 3473 if (!sectors) 3474 sectors = conf->dev_sectors; 3475 3476 size = sectors >> conf->geo.chunk_shift; 3477 sector_div(size, conf->geo.far_copies); 3478 size = size * raid_disks; 3479 sector_div(size, conf->geo.near_copies); 3480 3481 return size << conf->geo.chunk_shift; 3482 } 3483 3484 static void calc_sectors(struct r10conf *conf, sector_t size) 3485 { 3486 /* Calculate the number of sectors-per-device that will 3487 * actually be used, and set conf->dev_sectors and 3488 * conf->stride 3489 */ 3490 3491 size = size >> conf->geo.chunk_shift; 3492 sector_div(size, conf->geo.far_copies); 3493 size = size * conf->geo.raid_disks; 3494 sector_div(size, conf->geo.near_copies); 3495 /* 'size' is now the number of chunks in the array */ 3496 /* calculate "used chunks per device" */ 3497 size = size * conf->copies; 3498 3499 /* We need to round up when dividing by raid_disks to 3500 * get the stride size. 3501 */ 3502 size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks); 3503 3504 conf->dev_sectors = size << conf->geo.chunk_shift; 3505 3506 if (conf->geo.far_offset) 3507 conf->geo.stride = 1 << conf->geo.chunk_shift; 3508 else { 3509 sector_div(size, conf->geo.far_copies); 3510 conf->geo.stride = size << conf->geo.chunk_shift; 3511 } 3512 } 3513 3514 enum geo_type {geo_new, geo_old, geo_start}; 3515 static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new) 3516 { 3517 int nc, fc, fo; 3518 int layout, chunk, disks; 3519 switch (new) { 3520 case geo_old: 3521 layout = mddev->layout; 3522 chunk = mddev->chunk_sectors; 3523 disks = mddev->raid_disks - mddev->delta_disks; 3524 break; 3525 case geo_new: 3526 layout = mddev->new_layout; 3527 chunk = mddev->new_chunk_sectors; 3528 disks = mddev->raid_disks; 3529 break; 3530 default: /* avoid 'may be unused' warnings */ 3531 case geo_start: /* new when starting reshape - raid_disks not 3532 * updated yet. */ 3533 layout = mddev->new_layout; 3534 chunk = mddev->new_chunk_sectors; 3535 disks = mddev->raid_disks + mddev->delta_disks; 3536 break; 3537 } 3538 if (layout >> 19) 3539 return -1; 3540 if (chunk < (PAGE_SIZE >> 9) || 3541 !is_power_of_2(chunk)) 3542 return -2; 3543 nc = layout & 255; 3544 fc = (layout >> 8) & 255; 3545 fo = layout & (1<<16); 3546 geo->raid_disks = disks; 3547 geo->near_copies = nc; 3548 geo->far_copies = fc; 3549 geo->far_offset = fo; 3550 switch (layout >> 17) { 3551 case 0: /* original layout. simple but not always optimal */ 3552 geo->far_set_size = disks; 3553 break; 3554 case 1: /* "improved" layout which was buggy. Hopefully no-one is 3555 * actually using this, but leave code here just in case.*/ 3556 geo->far_set_size = disks/fc; 3557 WARN(geo->far_set_size < fc, 3558 "This RAID10 layout does not provide data safety - please backup and create new array\n"); 3559 break; 3560 case 2: /* "improved" layout fixed to match documentation */ 3561 geo->far_set_size = fc * nc; 3562 break; 3563 default: /* Not a valid layout */ 3564 return -1; 3565 } 3566 geo->chunk_mask = chunk - 1; 3567 geo->chunk_shift = ffz(~chunk); 3568 return nc*fc; 3569 } 3570 3571 static struct r10conf *setup_conf(struct mddev *mddev) 3572 { 3573 struct r10conf *conf = NULL; 3574 int err = -EINVAL; 3575 struct geom geo; 3576 int copies; 3577 3578 copies = setup_geo(&geo, mddev, geo_new); 3579 3580 if (copies == -2) { 3581 pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n", 3582 mdname(mddev), PAGE_SIZE); 3583 goto out; 3584 } 3585 3586 if (copies < 2 || copies > mddev->raid_disks) { 3587 pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n", 3588 mdname(mddev), mddev->new_layout); 3589 goto out; 3590 } 3591 3592 err = -ENOMEM; 3593 conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL); 3594 if (!conf) 3595 goto out; 3596 3597 /* FIXME calc properly */ 3598 conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks + 3599 max(0,-mddev->delta_disks)), 3600 GFP_KERNEL); 3601 if (!conf->mirrors) 3602 goto out; 3603 3604 conf->tmppage = alloc_page(GFP_KERNEL); 3605 if (!conf->tmppage) 3606 goto out; 3607 3608 conf->geo = geo; 3609 conf->copies = copies; 3610 conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc, 3611 r10bio_pool_free, conf); 3612 if (!conf->r10bio_pool) 3613 goto out; 3614 3615 calc_sectors(conf, mddev->dev_sectors); 3616 if (mddev->reshape_position == MaxSector) { 3617 conf->prev = conf->geo; 3618 conf->reshape_progress = MaxSector; 3619 } else { 3620 if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) { 3621 err = -EINVAL; 3622 goto out; 3623 } 3624 conf->reshape_progress = mddev->reshape_position; 3625 if (conf->prev.far_offset) 3626 conf->prev.stride = 1 << conf->prev.chunk_shift; 3627 else 3628 /* far_copies must be 1 */ 3629 conf->prev.stride = conf->dev_sectors; 3630 } 3631 conf->reshape_safe = conf->reshape_progress; 3632 spin_lock_init(&conf->device_lock); 3633 INIT_LIST_HEAD(&conf->retry_list); 3634 INIT_LIST_HEAD(&conf->bio_end_io_list); 3635 3636 spin_lock_init(&conf->resync_lock); 3637 init_waitqueue_head(&conf->wait_barrier); 3638 atomic_set(&conf->nr_pending, 0); 3639 3640 conf->thread = md_register_thread(raid10d, mddev, "raid10"); 3641 if (!conf->thread) 3642 goto out; 3643 3644 conf->mddev = mddev; 3645 return conf; 3646 3647 out: 3648 if (conf) { 3649 mempool_destroy(conf->r10bio_pool); 3650 kfree(conf->mirrors); 3651 safe_put_page(conf->tmppage); 3652 kfree(conf); 3653 } 3654 return ERR_PTR(err); 3655 } 3656 3657 static int raid10_run(struct mddev *mddev) 3658 { 3659 struct r10conf *conf; 3660 int i, disk_idx, chunk_size; 3661 struct raid10_info *disk; 3662 struct md_rdev *rdev; 3663 sector_t size; 3664 sector_t min_offset_diff = 0; 3665 int first = 1; 3666 bool discard_supported = false; 3667 3668 if (mddev->private == NULL) { 3669 conf = setup_conf(mddev); 3670 if (IS_ERR(conf)) 3671 return PTR_ERR(conf); 3672 mddev->private = conf; 3673 } 3674 conf = mddev->private; 3675 if (!conf) 3676 goto out; 3677 3678 mddev->thread = conf->thread; 3679 conf->thread = NULL; 3680 3681 chunk_size = mddev->chunk_sectors << 9; 3682 if (mddev->queue) { 3683 blk_queue_max_discard_sectors(mddev->queue, 3684 mddev->chunk_sectors); 3685 blk_queue_max_write_same_sectors(mddev->queue, 0); 3686 blk_queue_io_min(mddev->queue, chunk_size); 3687 if (conf->geo.raid_disks % conf->geo.near_copies) 3688 blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks); 3689 else 3690 blk_queue_io_opt(mddev->queue, chunk_size * 3691 (conf->geo.raid_disks / conf->geo.near_copies)); 3692 } 3693 3694 rdev_for_each(rdev, mddev) { 3695 long long diff; 3696 struct request_queue *q; 3697 3698 disk_idx = rdev->raid_disk; 3699 if (disk_idx < 0) 3700 continue; 3701 if (disk_idx >= conf->geo.raid_disks && 3702 disk_idx >= conf->prev.raid_disks) 3703 continue; 3704 disk = conf->mirrors + disk_idx; 3705 3706 if (test_bit(Replacement, &rdev->flags)) { 3707 if (disk->replacement) 3708 goto out_free_conf; 3709 disk->replacement = rdev; 3710 } else { 3711 if (disk->rdev) 3712 goto out_free_conf; 3713 disk->rdev = rdev; 3714 } 3715 q = bdev_get_queue(rdev->bdev); 3716 diff = (rdev->new_data_offset - rdev->data_offset); 3717 if (!mddev->reshape_backwards) 3718 diff = -diff; 3719 if (diff < 0) 3720 diff = 0; 3721 if (first || diff < min_offset_diff) 3722 min_offset_diff = diff; 3723 3724 if (mddev->gendisk) 3725 disk_stack_limits(mddev->gendisk, rdev->bdev, 3726 rdev->data_offset << 9); 3727 3728 disk->head_position = 0; 3729 3730 if (blk_queue_discard(bdev_get_queue(rdev->bdev))) 3731 discard_supported = true; 3732 } 3733 3734 if (mddev->queue) { 3735 if (discard_supported) 3736 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 3737 mddev->queue); 3738 else 3739 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 3740 mddev->queue); 3741 } 3742 /* need to check that every block has at least one working mirror */ 3743 if (!enough(conf, -1)) { 3744 pr_err("md/raid10:%s: not enough operational mirrors.\n", 3745 mdname(mddev)); 3746 goto out_free_conf; 3747 } 3748 3749 if (conf->reshape_progress != MaxSector) { 3750 /* must ensure that shape change is supported */ 3751 if (conf->geo.far_copies != 1 && 3752 conf->geo.far_offset == 0) 3753 goto out_free_conf; 3754 if (conf->prev.far_copies != 1 && 3755 conf->prev.far_offset == 0) 3756 goto out_free_conf; 3757 } 3758 3759 mddev->degraded = 0; 3760 for (i = 0; 3761 i < conf->geo.raid_disks 3762 || i < conf->prev.raid_disks; 3763 i++) { 3764 3765 disk = conf->mirrors + i; 3766 3767 if (!disk->rdev && disk->replacement) { 3768 /* The replacement is all we have - use it */ 3769 disk->rdev = disk->replacement; 3770 disk->replacement = NULL; 3771 clear_bit(Replacement, &disk->rdev->flags); 3772 } 3773 3774 if (!disk->rdev || 3775 !test_bit(In_sync, &disk->rdev->flags)) { 3776 disk->head_position = 0; 3777 mddev->degraded++; 3778 if (disk->rdev && 3779 disk->rdev->saved_raid_disk < 0) 3780 conf->fullsync = 1; 3781 } 3782 disk->recovery_disabled = mddev->recovery_disabled - 1; 3783 } 3784 3785 if (mddev->recovery_cp != MaxSector) 3786 pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n", 3787 mdname(mddev)); 3788 pr_info("md/raid10:%s: active with %d out of %d devices\n", 3789 mdname(mddev), conf->geo.raid_disks - mddev->degraded, 3790 conf->geo.raid_disks); 3791 /* 3792 * Ok, everything is just fine now 3793 */ 3794 mddev->dev_sectors = conf->dev_sectors; 3795 size = raid10_size(mddev, 0, 0); 3796 md_set_array_sectors(mddev, size); 3797 mddev->resync_max_sectors = size; 3798 set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 3799 3800 if (mddev->queue) { 3801 int stripe = conf->geo.raid_disks * 3802 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 3803 3804 /* Calculate max read-ahead size. 3805 * We need to readahead at least twice a whole stripe.... 3806 * maybe... 3807 */ 3808 stripe /= conf->geo.near_copies; 3809 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 3810 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 3811 } 3812 3813 if (md_integrity_register(mddev)) 3814 goto out_free_conf; 3815 3816 if (conf->reshape_progress != MaxSector) { 3817 unsigned long before_length, after_length; 3818 3819 before_length = ((1 << conf->prev.chunk_shift) * 3820 conf->prev.far_copies); 3821 after_length = ((1 << conf->geo.chunk_shift) * 3822 conf->geo.far_copies); 3823 3824 if (max(before_length, after_length) > min_offset_diff) { 3825 /* This cannot work */ 3826 pr_warn("md/raid10: offset difference not enough to continue reshape\n"); 3827 goto out_free_conf; 3828 } 3829 conf->offset_diff = min_offset_diff; 3830 3831 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 3832 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 3833 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 3834 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 3835 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 3836 "reshape"); 3837 } 3838 3839 return 0; 3840 3841 out_free_conf: 3842 md_unregister_thread(&mddev->thread); 3843 mempool_destroy(conf->r10bio_pool); 3844 safe_put_page(conf->tmppage); 3845 kfree(conf->mirrors); 3846 kfree(conf); 3847 mddev->private = NULL; 3848 out: 3849 return -EIO; 3850 } 3851 3852 static void raid10_free(struct mddev *mddev, void *priv) 3853 { 3854 struct r10conf *conf = priv; 3855 3856 mempool_destroy(conf->r10bio_pool); 3857 safe_put_page(conf->tmppage); 3858 kfree(conf->mirrors); 3859 kfree(conf->mirrors_old); 3860 kfree(conf->mirrors_new); 3861 kfree(conf); 3862 } 3863 3864 static void raid10_quiesce(struct mddev *mddev, int state) 3865 { 3866 struct r10conf *conf = mddev->private; 3867 3868 switch(state) { 3869 case 1: 3870 raise_barrier(conf, 0); 3871 break; 3872 case 0: 3873 lower_barrier(conf); 3874 break; 3875 } 3876 } 3877 3878 static int raid10_resize(struct mddev *mddev, sector_t sectors) 3879 { 3880 /* Resize of 'far' arrays is not supported. 3881 * For 'near' and 'offset' arrays we can set the 3882 * number of sectors used to be an appropriate multiple 3883 * of the chunk size. 3884 * For 'offset', this is far_copies*chunksize. 3885 * For 'near' the multiplier is the LCM of 3886 * near_copies and raid_disks. 3887 * So if far_copies > 1 && !far_offset, fail. 3888 * Else find LCM(raid_disks, near_copy)*far_copies and 3889 * multiply by chunk_size. Then round to this number. 3890 * This is mostly done by raid10_size() 3891 */ 3892 struct r10conf *conf = mddev->private; 3893 sector_t oldsize, size; 3894 3895 if (mddev->reshape_position != MaxSector) 3896 return -EBUSY; 3897 3898 if (conf->geo.far_copies > 1 && !conf->geo.far_offset) 3899 return -EINVAL; 3900 3901 oldsize = raid10_size(mddev, 0, 0); 3902 size = raid10_size(mddev, sectors, 0); 3903 if (mddev->external_size && 3904 mddev->array_sectors > size) 3905 return -EINVAL; 3906 if (mddev->bitmap) { 3907 int ret = bitmap_resize(mddev->bitmap, size, 0, 0); 3908 if (ret) 3909 return ret; 3910 } 3911 md_set_array_sectors(mddev, size); 3912 if (mddev->queue) { 3913 set_capacity(mddev->gendisk, mddev->array_sectors); 3914 revalidate_disk(mddev->gendisk); 3915 } 3916 if (sectors > mddev->dev_sectors && 3917 mddev->recovery_cp > oldsize) { 3918 mddev->recovery_cp = oldsize; 3919 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 3920 } 3921 calc_sectors(conf, sectors); 3922 mddev->dev_sectors = conf->dev_sectors; 3923 mddev->resync_max_sectors = size; 3924 return 0; 3925 } 3926 3927 static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) 3928 { 3929 struct md_rdev *rdev; 3930 struct r10conf *conf; 3931 3932 if (mddev->degraded > 0) { 3933 pr_warn("md/raid10:%s: Error: degraded raid0!\n", 3934 mdname(mddev)); 3935 return ERR_PTR(-EINVAL); 3936 } 3937 sector_div(size, devs); 3938 3939 /* Set new parameters */ 3940 mddev->new_level = 10; 3941 /* new layout: far_copies = 1, near_copies = 2 */ 3942 mddev->new_layout = (1<<8) + 2; 3943 mddev->new_chunk_sectors = mddev->chunk_sectors; 3944 mddev->delta_disks = mddev->raid_disks; 3945 mddev->raid_disks *= 2; 3946 /* make sure it will be not marked as dirty */ 3947 mddev->recovery_cp = MaxSector; 3948 mddev->dev_sectors = size; 3949 3950 conf = setup_conf(mddev); 3951 if (!IS_ERR(conf)) { 3952 rdev_for_each(rdev, mddev) 3953 if (rdev->raid_disk >= 0) { 3954 rdev->new_raid_disk = rdev->raid_disk * 2; 3955 rdev->sectors = size; 3956 } 3957 conf->barrier = 1; 3958 } 3959 3960 return conf; 3961 } 3962 3963 static void *raid10_takeover(struct mddev *mddev) 3964 { 3965 struct r0conf *raid0_conf; 3966 3967 /* raid10 can take over: 3968 * raid0 - providing it has only two drives 3969 */ 3970 if (mddev->level == 0) { 3971 /* for raid0 takeover only one zone is supported */ 3972 raid0_conf = mddev->private; 3973 if (raid0_conf->nr_strip_zones > 1) { 3974 pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n", 3975 mdname(mddev)); 3976 return ERR_PTR(-EINVAL); 3977 } 3978 return raid10_takeover_raid0(mddev, 3979 raid0_conf->strip_zone->zone_end, 3980 raid0_conf->strip_zone->nb_dev); 3981 } 3982 return ERR_PTR(-EINVAL); 3983 } 3984 3985 static int raid10_check_reshape(struct mddev *mddev) 3986 { 3987 /* Called when there is a request to change 3988 * - layout (to ->new_layout) 3989 * - chunk size (to ->new_chunk_sectors) 3990 * - raid_disks (by delta_disks) 3991 * or when trying to restart a reshape that was ongoing. 3992 * 3993 * We need to validate the request and possibly allocate 3994 * space if that might be an issue later. 3995 * 3996 * Currently we reject any reshape of a 'far' mode array, 3997 * allow chunk size to change if new is generally acceptable, 3998 * allow raid_disks to increase, and allow 3999 * a switch between 'near' mode and 'offset' mode. 4000 */ 4001 struct r10conf *conf = mddev->private; 4002 struct geom geo; 4003 4004 if (conf->geo.far_copies != 1 && !conf->geo.far_offset) 4005 return -EINVAL; 4006 4007 if (setup_geo(&geo, mddev, geo_start) != conf->copies) 4008 /* mustn't change number of copies */ 4009 return -EINVAL; 4010 if (geo.far_copies > 1 && !geo.far_offset) 4011 /* Cannot switch to 'far' mode */ 4012 return -EINVAL; 4013 4014 if (mddev->array_sectors & geo.chunk_mask) 4015 /* not factor of array size */ 4016 return -EINVAL; 4017 4018 if (!enough(conf, -1)) 4019 return -EINVAL; 4020 4021 kfree(conf->mirrors_new); 4022 conf->mirrors_new = NULL; 4023 if (mddev->delta_disks > 0) { 4024 /* allocate new 'mirrors' list */ 4025 conf->mirrors_new = kzalloc( 4026 sizeof(struct raid10_info) 4027 *(mddev->raid_disks + 4028 mddev->delta_disks), 4029 GFP_KERNEL); 4030 if (!conf->mirrors_new) 4031 return -ENOMEM; 4032 } 4033 return 0; 4034 } 4035 4036 /* 4037 * Need to check if array has failed when deciding whether to: 4038 * - start an array 4039 * - remove non-faulty devices 4040 * - add a spare 4041 * - allow a reshape 4042 * This determination is simple when no reshape is happening. 4043 * However if there is a reshape, we need to carefully check 4044 * both the before and after sections. 4045 * This is because some failed devices may only affect one 4046 * of the two sections, and some non-in_sync devices may 4047 * be insync in the section most affected by failed devices. 4048 */ 4049 static int calc_degraded(struct r10conf *conf) 4050 { 4051 int degraded, degraded2; 4052 int i; 4053 4054 rcu_read_lock(); 4055 degraded = 0; 4056 /* 'prev' section first */ 4057 for (i = 0; i < conf->prev.raid_disks; i++) { 4058 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4059 if (!rdev || test_bit(Faulty, &rdev->flags)) 4060 degraded++; 4061 else if (!test_bit(In_sync, &rdev->flags)) 4062 /* When we can reduce the number of devices in 4063 * an array, this might not contribute to 4064 * 'degraded'. It does now. 4065 */ 4066 degraded++; 4067 } 4068 rcu_read_unlock(); 4069 if (conf->geo.raid_disks == conf->prev.raid_disks) 4070 return degraded; 4071 rcu_read_lock(); 4072 degraded2 = 0; 4073 for (i = 0; i < conf->geo.raid_disks; i++) { 4074 struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev); 4075 if (!rdev || test_bit(Faulty, &rdev->flags)) 4076 degraded2++; 4077 else if (!test_bit(In_sync, &rdev->flags)) { 4078 /* If reshape is increasing the number of devices, 4079 * this section has already been recovered, so 4080 * it doesn't contribute to degraded. 4081 * else it does. 4082 */ 4083 if (conf->geo.raid_disks <= conf->prev.raid_disks) 4084 degraded2++; 4085 } 4086 } 4087 rcu_read_unlock(); 4088 if (degraded2 > degraded) 4089 return degraded2; 4090 return degraded; 4091 } 4092 4093 static int raid10_start_reshape(struct mddev *mddev) 4094 { 4095 /* A 'reshape' has been requested. This commits 4096 * the various 'new' fields and sets MD_RECOVER_RESHAPE 4097 * This also checks if there are enough spares and adds them 4098 * to the array. 4099 * We currently require enough spares to make the final 4100 * array non-degraded. We also require that the difference 4101 * between old and new data_offset - on each device - is 4102 * enough that we never risk over-writing. 4103 */ 4104 4105 unsigned long before_length, after_length; 4106 sector_t min_offset_diff = 0; 4107 int first = 1; 4108 struct geom new; 4109 struct r10conf *conf = mddev->private; 4110 struct md_rdev *rdev; 4111 int spares = 0; 4112 int ret; 4113 4114 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4115 return -EBUSY; 4116 4117 if (setup_geo(&new, mddev, geo_start) != conf->copies) 4118 return -EINVAL; 4119 4120 before_length = ((1 << conf->prev.chunk_shift) * 4121 conf->prev.far_copies); 4122 after_length = ((1 << conf->geo.chunk_shift) * 4123 conf->geo.far_copies); 4124 4125 rdev_for_each(rdev, mddev) { 4126 if (!test_bit(In_sync, &rdev->flags) 4127 && !test_bit(Faulty, &rdev->flags)) 4128 spares++; 4129 if (rdev->raid_disk >= 0) { 4130 long long diff = (rdev->new_data_offset 4131 - rdev->data_offset); 4132 if (!mddev->reshape_backwards) 4133 diff = -diff; 4134 if (diff < 0) 4135 diff = 0; 4136 if (first || diff < min_offset_diff) 4137 min_offset_diff = diff; 4138 } 4139 } 4140 4141 if (max(before_length, after_length) > min_offset_diff) 4142 return -EINVAL; 4143 4144 if (spares < mddev->delta_disks) 4145 return -EINVAL; 4146 4147 conf->offset_diff = min_offset_diff; 4148 spin_lock_irq(&conf->device_lock); 4149 if (conf->mirrors_new) { 4150 memcpy(conf->mirrors_new, conf->mirrors, 4151 sizeof(struct raid10_info)*conf->prev.raid_disks); 4152 smp_mb(); 4153 kfree(conf->mirrors_old); 4154 conf->mirrors_old = conf->mirrors; 4155 conf->mirrors = conf->mirrors_new; 4156 conf->mirrors_new = NULL; 4157 } 4158 setup_geo(&conf->geo, mddev, geo_start); 4159 smp_mb(); 4160 if (mddev->reshape_backwards) { 4161 sector_t size = raid10_size(mddev, 0, 0); 4162 if (size < mddev->array_sectors) { 4163 spin_unlock_irq(&conf->device_lock); 4164 pr_warn("md/raid10:%s: array size must be reduce before number of disks\n", 4165 mdname(mddev)); 4166 return -EINVAL; 4167 } 4168 mddev->resync_max_sectors = size; 4169 conf->reshape_progress = size; 4170 } else 4171 conf->reshape_progress = 0; 4172 conf->reshape_safe = conf->reshape_progress; 4173 spin_unlock_irq(&conf->device_lock); 4174 4175 if (mddev->delta_disks && mddev->bitmap) { 4176 ret = bitmap_resize(mddev->bitmap, 4177 raid10_size(mddev, 0, 4178 conf->geo.raid_disks), 4179 0, 0); 4180 if (ret) 4181 goto abort; 4182 } 4183 if (mddev->delta_disks > 0) { 4184 rdev_for_each(rdev, mddev) 4185 if (rdev->raid_disk < 0 && 4186 !test_bit(Faulty, &rdev->flags)) { 4187 if (raid10_add_disk(mddev, rdev) == 0) { 4188 if (rdev->raid_disk >= 4189 conf->prev.raid_disks) 4190 set_bit(In_sync, &rdev->flags); 4191 else 4192 rdev->recovery_offset = 0; 4193 4194 if (sysfs_link_rdev(mddev, rdev)) 4195 /* Failure here is OK */; 4196 } 4197 } else if (rdev->raid_disk >= conf->prev.raid_disks 4198 && !test_bit(Faulty, &rdev->flags)) { 4199 /* This is a spare that was manually added */ 4200 set_bit(In_sync, &rdev->flags); 4201 } 4202 } 4203 /* When a reshape changes the number of devices, 4204 * ->degraded is measured against the larger of the 4205 * pre and post numbers. 4206 */ 4207 spin_lock_irq(&conf->device_lock); 4208 mddev->degraded = calc_degraded(conf); 4209 spin_unlock_irq(&conf->device_lock); 4210 mddev->raid_disks = conf->geo.raid_disks; 4211 mddev->reshape_position = conf->reshape_progress; 4212 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4213 4214 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4215 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4216 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 4217 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4218 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4219 4220 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4221 "reshape"); 4222 if (!mddev->sync_thread) { 4223 ret = -EAGAIN; 4224 goto abort; 4225 } 4226 conf->reshape_checkpoint = jiffies; 4227 md_wakeup_thread(mddev->sync_thread); 4228 md_new_event(mddev); 4229 return 0; 4230 4231 abort: 4232 mddev->recovery = 0; 4233 spin_lock_irq(&conf->device_lock); 4234 conf->geo = conf->prev; 4235 mddev->raid_disks = conf->geo.raid_disks; 4236 rdev_for_each(rdev, mddev) 4237 rdev->new_data_offset = rdev->data_offset; 4238 smp_wmb(); 4239 conf->reshape_progress = MaxSector; 4240 conf->reshape_safe = MaxSector; 4241 mddev->reshape_position = MaxSector; 4242 spin_unlock_irq(&conf->device_lock); 4243 return ret; 4244 } 4245 4246 /* Calculate the last device-address that could contain 4247 * any block from the chunk that includes the array-address 's' 4248 * and report the next address. 4249 * i.e. the address returned will be chunk-aligned and after 4250 * any data that is in the chunk containing 's'. 4251 */ 4252 static sector_t last_dev_address(sector_t s, struct geom *geo) 4253 { 4254 s = (s | geo->chunk_mask) + 1; 4255 s >>= geo->chunk_shift; 4256 s *= geo->near_copies; 4257 s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks); 4258 s *= geo->far_copies; 4259 s <<= geo->chunk_shift; 4260 return s; 4261 } 4262 4263 /* Calculate the first device-address that could contain 4264 * any block from the chunk that includes the array-address 's'. 4265 * This too will be the start of a chunk 4266 */ 4267 static sector_t first_dev_address(sector_t s, struct geom *geo) 4268 { 4269 s >>= geo->chunk_shift; 4270 s *= geo->near_copies; 4271 sector_div(s, geo->raid_disks); 4272 s *= geo->far_copies; 4273 s <<= geo->chunk_shift; 4274 return s; 4275 } 4276 4277 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, 4278 int *skipped) 4279 { 4280 /* We simply copy at most one chunk (smallest of old and new) 4281 * at a time, possibly less if that exceeds RESYNC_PAGES, 4282 * or we hit a bad block or something. 4283 * This might mean we pause for normal IO in the middle of 4284 * a chunk, but that is not a problem as mddev->reshape_position 4285 * can record any location. 4286 * 4287 * If we will want to write to a location that isn't 4288 * yet recorded as 'safe' (i.e. in metadata on disk) then 4289 * we need to flush all reshape requests and update the metadata. 4290 * 4291 * When reshaping forwards (e.g. to more devices), we interpret 4292 * 'safe' as the earliest block which might not have been copied 4293 * down yet. We divide this by previous stripe size and multiply 4294 * by previous stripe length to get lowest device offset that we 4295 * cannot write to yet. 4296 * We interpret 'sector_nr' as an address that we want to write to. 4297 * From this we use last_device_address() to find where we might 4298 * write to, and first_device_address on the 'safe' position. 4299 * If this 'next' write position is after the 'safe' position, 4300 * we must update the metadata to increase the 'safe' position. 4301 * 4302 * When reshaping backwards, we round in the opposite direction 4303 * and perform the reverse test: next write position must not be 4304 * less than current safe position. 4305 * 4306 * In all this the minimum difference in data offsets 4307 * (conf->offset_diff - always positive) allows a bit of slack, 4308 * so next can be after 'safe', but not by more than offset_diff 4309 * 4310 * We need to prepare all the bios here before we start any IO 4311 * to ensure the size we choose is acceptable to all devices. 4312 * The means one for each copy for write-out and an extra one for 4313 * read-in. 4314 * We store the read-in bio in ->master_bio and the others in 4315 * ->devs[x].bio and ->devs[x].repl_bio. 4316 */ 4317 struct r10conf *conf = mddev->private; 4318 struct r10bio *r10_bio; 4319 sector_t next, safe, last; 4320 int max_sectors; 4321 int nr_sectors; 4322 int s; 4323 struct md_rdev *rdev; 4324 int need_flush = 0; 4325 struct bio *blist; 4326 struct bio *bio, *read_bio; 4327 int sectors_done = 0; 4328 4329 if (sector_nr == 0) { 4330 /* If restarting in the middle, skip the initial sectors */ 4331 if (mddev->reshape_backwards && 4332 conf->reshape_progress < raid10_size(mddev, 0, 0)) { 4333 sector_nr = (raid10_size(mddev, 0, 0) 4334 - conf->reshape_progress); 4335 } else if (!mddev->reshape_backwards && 4336 conf->reshape_progress > 0) 4337 sector_nr = conf->reshape_progress; 4338 if (sector_nr) { 4339 mddev->curr_resync_completed = sector_nr; 4340 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4341 *skipped = 1; 4342 return sector_nr; 4343 } 4344 } 4345 4346 /* We don't use sector_nr to track where we are up to 4347 * as that doesn't work well for ->reshape_backwards. 4348 * So just use ->reshape_progress. 4349 */ 4350 if (mddev->reshape_backwards) { 4351 /* 'next' is the earliest device address that we might 4352 * write to for this chunk in the new layout 4353 */ 4354 next = first_dev_address(conf->reshape_progress - 1, 4355 &conf->geo); 4356 4357 /* 'safe' is the last device address that we might read from 4358 * in the old layout after a restart 4359 */ 4360 safe = last_dev_address(conf->reshape_safe - 1, 4361 &conf->prev); 4362 4363 if (next + conf->offset_diff < safe) 4364 need_flush = 1; 4365 4366 last = conf->reshape_progress - 1; 4367 sector_nr = last & ~(sector_t)(conf->geo.chunk_mask 4368 & conf->prev.chunk_mask); 4369 if (sector_nr + RESYNC_BLOCK_SIZE/512 < last) 4370 sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512; 4371 } else { 4372 /* 'next' is after the last device address that we 4373 * might write to for this chunk in the new layout 4374 */ 4375 next = last_dev_address(conf->reshape_progress, &conf->geo); 4376 4377 /* 'safe' is the earliest device address that we might 4378 * read from in the old layout after a restart 4379 */ 4380 safe = first_dev_address(conf->reshape_safe, &conf->prev); 4381 4382 /* Need to update metadata if 'next' might be beyond 'safe' 4383 * as that would possibly corrupt data 4384 */ 4385 if (next > safe + conf->offset_diff) 4386 need_flush = 1; 4387 4388 sector_nr = conf->reshape_progress; 4389 last = sector_nr | (conf->geo.chunk_mask 4390 & conf->prev.chunk_mask); 4391 4392 if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last) 4393 last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1; 4394 } 4395 4396 if (need_flush || 4397 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4398 /* Need to update reshape_position in metadata */ 4399 wait_barrier(conf); 4400 mddev->reshape_position = conf->reshape_progress; 4401 if (mddev->reshape_backwards) 4402 mddev->curr_resync_completed = raid10_size(mddev, 0, 0) 4403 - conf->reshape_progress; 4404 else 4405 mddev->curr_resync_completed = conf->reshape_progress; 4406 conf->reshape_checkpoint = jiffies; 4407 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 4408 md_wakeup_thread(mddev->thread); 4409 wait_event(mddev->sb_wait, mddev->sb_flags == 0 || 4410 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4411 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 4412 allow_barrier(conf); 4413 return sectors_done; 4414 } 4415 conf->reshape_safe = mddev->reshape_position; 4416 allow_barrier(conf); 4417 } 4418 4419 read_more: 4420 /* Now schedule reads for blocks from sector_nr to last */ 4421 r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO); 4422 r10_bio->state = 0; 4423 raise_barrier(conf, sectors_done != 0); 4424 atomic_set(&r10_bio->remaining, 0); 4425 r10_bio->mddev = mddev; 4426 r10_bio->sector = sector_nr; 4427 set_bit(R10BIO_IsReshape, &r10_bio->state); 4428 r10_bio->sectors = last - sector_nr + 1; 4429 rdev = read_balance(conf, r10_bio, &max_sectors); 4430 BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state)); 4431 4432 if (!rdev) { 4433 /* Cannot read from here, so need to record bad blocks 4434 * on all the target devices. 4435 */ 4436 // FIXME 4437 mempool_free(r10_bio, conf->r10buf_pool); 4438 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 4439 return sectors_done; 4440 } 4441 4442 read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev); 4443 4444 read_bio->bi_bdev = rdev->bdev; 4445 read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr 4446 + rdev->data_offset); 4447 read_bio->bi_private = r10_bio; 4448 read_bio->bi_end_io = end_sync_read; 4449 bio_set_op_attrs(read_bio, REQ_OP_READ, 0); 4450 read_bio->bi_flags &= (~0UL << BIO_RESET_BITS); 4451 read_bio->bi_error = 0; 4452 read_bio->bi_vcnt = 0; 4453 read_bio->bi_iter.bi_size = 0; 4454 r10_bio->master_bio = read_bio; 4455 r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum; 4456 4457 /* Now find the locations in the new layout */ 4458 __raid10_find_phys(&conf->geo, r10_bio); 4459 4460 blist = read_bio; 4461 read_bio->bi_next = NULL; 4462 4463 rcu_read_lock(); 4464 for (s = 0; s < conf->copies*2; s++) { 4465 struct bio *b; 4466 int d = r10_bio->devs[s/2].devnum; 4467 struct md_rdev *rdev2; 4468 if (s&1) { 4469 rdev2 = rcu_dereference(conf->mirrors[d].replacement); 4470 b = r10_bio->devs[s/2].repl_bio; 4471 } else { 4472 rdev2 = rcu_dereference(conf->mirrors[d].rdev); 4473 b = r10_bio->devs[s/2].bio; 4474 } 4475 if (!rdev2 || test_bit(Faulty, &rdev2->flags)) 4476 continue; 4477 4478 bio_reset(b); 4479 b->bi_bdev = rdev2->bdev; 4480 b->bi_iter.bi_sector = r10_bio->devs[s/2].addr + 4481 rdev2->new_data_offset; 4482 b->bi_private = r10_bio; 4483 b->bi_end_io = end_reshape_write; 4484 bio_set_op_attrs(b, REQ_OP_WRITE, 0); 4485 b->bi_next = blist; 4486 blist = b; 4487 } 4488 4489 /* Now add as many pages as possible to all of these bios. */ 4490 4491 nr_sectors = 0; 4492 for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) { 4493 struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page; 4494 int len = (max_sectors - s) << 9; 4495 if (len > PAGE_SIZE) 4496 len = PAGE_SIZE; 4497 for (bio = blist; bio ; bio = bio->bi_next) { 4498 struct bio *bio2; 4499 if (bio_add_page(bio, page, len, 0)) 4500 continue; 4501 4502 /* Didn't fit, must stop */ 4503 for (bio2 = blist; 4504 bio2 && bio2 != bio; 4505 bio2 = bio2->bi_next) { 4506 /* Remove last page from this bio */ 4507 bio2->bi_vcnt--; 4508 bio2->bi_iter.bi_size -= len; 4509 bio_clear_flag(bio2, BIO_SEG_VALID); 4510 } 4511 goto bio_full; 4512 } 4513 sector_nr += len >> 9; 4514 nr_sectors += len >> 9; 4515 } 4516 bio_full: 4517 rcu_read_unlock(); 4518 r10_bio->sectors = nr_sectors; 4519 4520 /* Now submit the read */ 4521 md_sync_acct(read_bio->bi_bdev, r10_bio->sectors); 4522 atomic_inc(&r10_bio->remaining); 4523 read_bio->bi_next = NULL; 4524 generic_make_request(read_bio); 4525 sector_nr += nr_sectors; 4526 sectors_done += nr_sectors; 4527 if (sector_nr <= last) 4528 goto read_more; 4529 4530 /* Now that we have done the whole section we can 4531 * update reshape_progress 4532 */ 4533 if (mddev->reshape_backwards) 4534 conf->reshape_progress -= sectors_done; 4535 else 4536 conf->reshape_progress += sectors_done; 4537 4538 return sectors_done; 4539 } 4540 4541 static void end_reshape_request(struct r10bio *r10_bio); 4542 static int handle_reshape_read_error(struct mddev *mddev, 4543 struct r10bio *r10_bio); 4544 static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) 4545 { 4546 /* Reshape read completed. Hopefully we have a block 4547 * to write out. 4548 * If we got a read error then we do sync 1-page reads from 4549 * elsewhere until we find the data - or give up. 4550 */ 4551 struct r10conf *conf = mddev->private; 4552 int s; 4553 4554 if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) 4555 if (handle_reshape_read_error(mddev, r10_bio) < 0) { 4556 /* Reshape has been aborted */ 4557 md_done_sync(mddev, r10_bio->sectors, 0); 4558 return; 4559 } 4560 4561 /* We definitely have the data in the pages, schedule the 4562 * writes. 4563 */ 4564 atomic_set(&r10_bio->remaining, 1); 4565 for (s = 0; s < conf->copies*2; s++) { 4566 struct bio *b; 4567 int d = r10_bio->devs[s/2].devnum; 4568 struct md_rdev *rdev; 4569 rcu_read_lock(); 4570 if (s&1) { 4571 rdev = rcu_dereference(conf->mirrors[d].replacement); 4572 b = r10_bio->devs[s/2].repl_bio; 4573 } else { 4574 rdev = rcu_dereference(conf->mirrors[d].rdev); 4575 b = r10_bio->devs[s/2].bio; 4576 } 4577 if (!rdev || test_bit(Faulty, &rdev->flags)) { 4578 rcu_read_unlock(); 4579 continue; 4580 } 4581 atomic_inc(&rdev->nr_pending); 4582 rcu_read_unlock(); 4583 md_sync_acct(b->bi_bdev, r10_bio->sectors); 4584 atomic_inc(&r10_bio->remaining); 4585 b->bi_next = NULL; 4586 generic_make_request(b); 4587 } 4588 end_reshape_request(r10_bio); 4589 } 4590 4591 static void end_reshape(struct r10conf *conf) 4592 { 4593 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) 4594 return; 4595 4596 spin_lock_irq(&conf->device_lock); 4597 conf->prev = conf->geo; 4598 md_finish_reshape(conf->mddev); 4599 smp_wmb(); 4600 conf->reshape_progress = MaxSector; 4601 conf->reshape_safe = MaxSector; 4602 spin_unlock_irq(&conf->device_lock); 4603 4604 /* read-ahead size must cover two whole stripes, which is 4605 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4606 */ 4607 if (conf->mddev->queue) { 4608 int stripe = conf->geo.raid_disks * 4609 ((conf->mddev->chunk_sectors << 9) / PAGE_SIZE); 4610 stripe /= conf->geo.near_copies; 4611 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4612 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4613 } 4614 conf->fullsync = 0; 4615 } 4616 4617 static int handle_reshape_read_error(struct mddev *mddev, 4618 struct r10bio *r10_bio) 4619 { 4620 /* Use sync reads to get the blocks from somewhere else */ 4621 int sectors = r10_bio->sectors; 4622 struct r10conf *conf = mddev->private; 4623 struct { 4624 struct r10bio r10_bio; 4625 struct r10dev devs[conf->copies]; 4626 } on_stack; 4627 struct r10bio *r10b = &on_stack.r10_bio; 4628 int slot = 0; 4629 int idx = 0; 4630 struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec; 4631 4632 r10b->sector = r10_bio->sector; 4633 __raid10_find_phys(&conf->prev, r10b); 4634 4635 while (sectors) { 4636 int s = sectors; 4637 int success = 0; 4638 int first_slot = slot; 4639 4640 if (s > (PAGE_SIZE >> 9)) 4641 s = PAGE_SIZE >> 9; 4642 4643 rcu_read_lock(); 4644 while (!success) { 4645 int d = r10b->devs[slot].devnum; 4646 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4647 sector_t addr; 4648 if (rdev == NULL || 4649 test_bit(Faulty, &rdev->flags) || 4650 !test_bit(In_sync, &rdev->flags)) 4651 goto failed; 4652 4653 addr = r10b->devs[slot].addr + idx * PAGE_SIZE; 4654 atomic_inc(&rdev->nr_pending); 4655 rcu_read_unlock(); 4656 success = sync_page_io(rdev, 4657 addr, 4658 s << 9, 4659 bvec[idx].bv_page, 4660 REQ_OP_READ, 0, false); 4661 rdev_dec_pending(rdev, mddev); 4662 rcu_read_lock(); 4663 if (success) 4664 break; 4665 failed: 4666 slot++; 4667 if (slot >= conf->copies) 4668 slot = 0; 4669 if (slot == first_slot) 4670 break; 4671 } 4672 rcu_read_unlock(); 4673 if (!success) { 4674 /* couldn't read this block, must give up */ 4675 set_bit(MD_RECOVERY_INTR, 4676 &mddev->recovery); 4677 return -EIO; 4678 } 4679 sectors -= s; 4680 idx++; 4681 } 4682 return 0; 4683 } 4684 4685 static void end_reshape_write(struct bio *bio) 4686 { 4687 struct r10bio *r10_bio = bio->bi_private; 4688 struct mddev *mddev = r10_bio->mddev; 4689 struct r10conf *conf = mddev->private; 4690 int d; 4691 int slot; 4692 int repl; 4693 struct md_rdev *rdev = NULL; 4694 4695 d = find_bio_disk(conf, r10_bio, bio, &slot, &repl); 4696 if (repl) 4697 rdev = conf->mirrors[d].replacement; 4698 if (!rdev) { 4699 smp_mb(); 4700 rdev = conf->mirrors[d].rdev; 4701 } 4702 4703 if (bio->bi_error) { 4704 /* FIXME should record badblock */ 4705 md_error(mddev, rdev); 4706 } 4707 4708 rdev_dec_pending(rdev, mddev); 4709 end_reshape_request(r10_bio); 4710 } 4711 4712 static void end_reshape_request(struct r10bio *r10_bio) 4713 { 4714 if (!atomic_dec_and_test(&r10_bio->remaining)) 4715 return; 4716 md_done_sync(r10_bio->mddev, r10_bio->sectors, 1); 4717 bio_put(r10_bio->master_bio); 4718 put_buf(r10_bio); 4719 } 4720 4721 static void raid10_finish_reshape(struct mddev *mddev) 4722 { 4723 struct r10conf *conf = mddev->private; 4724 4725 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4726 return; 4727 4728 if (mddev->delta_disks > 0) { 4729 sector_t size = raid10_size(mddev, 0, 0); 4730 md_set_array_sectors(mddev, size); 4731 if (mddev->recovery_cp > mddev->resync_max_sectors) { 4732 mddev->recovery_cp = mddev->resync_max_sectors; 4733 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4734 } 4735 mddev->resync_max_sectors = size; 4736 if (mddev->queue) { 4737 set_capacity(mddev->gendisk, mddev->array_sectors); 4738 revalidate_disk(mddev->gendisk); 4739 } 4740 } else { 4741 int d; 4742 rcu_read_lock(); 4743 for (d = conf->geo.raid_disks ; 4744 d < conf->geo.raid_disks - mddev->delta_disks; 4745 d++) { 4746 struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev); 4747 if (rdev) 4748 clear_bit(In_sync, &rdev->flags); 4749 rdev = rcu_dereference(conf->mirrors[d].replacement); 4750 if (rdev) 4751 clear_bit(In_sync, &rdev->flags); 4752 } 4753 rcu_read_unlock(); 4754 } 4755 mddev->layout = mddev->new_layout; 4756 mddev->chunk_sectors = 1 << conf->geo.chunk_shift; 4757 mddev->reshape_position = MaxSector; 4758 mddev->delta_disks = 0; 4759 mddev->reshape_backwards = 0; 4760 } 4761 4762 static struct md_personality raid10_personality = 4763 { 4764 .name = "raid10", 4765 .level = 10, 4766 .owner = THIS_MODULE, 4767 .make_request = raid10_make_request, 4768 .run = raid10_run, 4769 .free = raid10_free, 4770 .status = raid10_status, 4771 .error_handler = raid10_error, 4772 .hot_add_disk = raid10_add_disk, 4773 .hot_remove_disk= raid10_remove_disk, 4774 .spare_active = raid10_spare_active, 4775 .sync_request = raid10_sync_request, 4776 .quiesce = raid10_quiesce, 4777 .size = raid10_size, 4778 .resize = raid10_resize, 4779 .takeover = raid10_takeover, 4780 .check_reshape = raid10_check_reshape, 4781 .start_reshape = raid10_start_reshape, 4782 .finish_reshape = raid10_finish_reshape, 4783 .congested = raid10_congested, 4784 }; 4785 4786 static int __init raid_init(void) 4787 { 4788 return register_md_personality(&raid10_personality); 4789 } 4790 4791 static void raid_exit(void) 4792 { 4793 unregister_md_personality(&raid10_personality); 4794 } 4795 4796 module_init(raid_init); 4797 module_exit(raid_exit); 4798 MODULE_LICENSE("GPL"); 4799 MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD"); 4800 MODULE_ALIAS("md-personality-9"); /* RAID10 */ 4801 MODULE_ALIAS("md-raid10"); 4802 MODULE_ALIAS("md-level-10"); 4803 4804 module_param(max_queued_requests, int, S_IRUGO|S_IWUSR); 4805