1 /* 2 * raid1.c : Multiple Devices driver for Linux 3 * 4 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat 5 * 6 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman 7 * 8 * RAID-1 management functions. 9 * 10 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000 11 * 12 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk> 13 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au> 14 * 15 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support 16 * bitmapped intelligence in resync: 17 * 18 * - bitmap marked during normal i/o 19 * - bitmap used to skip nondirty blocks during sync 20 * 21 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology: 22 * - persistent bitmap code 23 * 24 * This program is free software; you can redistribute it and/or modify 25 * it under the terms of the GNU General Public License as published by 26 * the Free Software Foundation; either version 2, or (at your option) 27 * any later version. 28 * 29 * You should have received a copy of the GNU General Public License 30 * (for example /usr/src/linux/COPYING); if not, write to the Free 31 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 32 */ 33 34 #include <linux/slab.h> 35 #include <linux/delay.h> 36 #include <linux/blkdev.h> 37 #include <linux/seq_file.h> 38 #include "md.h" 39 #include "raid1.h" 40 #include "bitmap.h" 41 42 #define DEBUG 0 43 #if DEBUG 44 #define PRINTK(x...) printk(x) 45 #else 46 #define PRINTK(x...) 47 #endif 48 49 /* 50 * Number of guaranteed r1bios in case of extreme VM load: 51 */ 52 #define NR_RAID1_BIOS 256 53 54 55 static void unplug_slaves(mddev_t *mddev); 56 57 static void allow_barrier(conf_t *conf); 58 static void lower_barrier(conf_t *conf); 59 60 static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) 61 { 62 struct pool_info *pi = data; 63 r1bio_t *r1_bio; 64 int size = offsetof(r1bio_t, bios[pi->raid_disks]); 65 66 /* allocate a r1bio with room for raid_disks entries in the bios array */ 67 r1_bio = kzalloc(size, gfp_flags); 68 if (!r1_bio && pi->mddev) 69 unplug_slaves(pi->mddev); 70 71 return r1_bio; 72 } 73 74 static void r1bio_pool_free(void *r1_bio, void *data) 75 { 76 kfree(r1_bio); 77 } 78 79 #define RESYNC_BLOCK_SIZE (64*1024) 80 //#define RESYNC_BLOCK_SIZE PAGE_SIZE 81 #define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9) 82 #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE) 83 #define RESYNC_WINDOW (2048*1024) 84 85 static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) 86 { 87 struct pool_info *pi = data; 88 struct page *page; 89 r1bio_t *r1_bio; 90 struct bio *bio; 91 int i, j; 92 93 r1_bio = r1bio_pool_alloc(gfp_flags, pi); 94 if (!r1_bio) { 95 unplug_slaves(pi->mddev); 96 return NULL; 97 } 98 99 /* 100 * Allocate bios : 1 for reading, n-1 for writing 101 */ 102 for (j = pi->raid_disks ; j-- ; ) { 103 bio = bio_kmalloc(gfp_flags, RESYNC_PAGES); 104 if (!bio) 105 goto out_free_bio; 106 r1_bio->bios[j] = bio; 107 } 108 /* 109 * Allocate RESYNC_PAGES data pages and attach them to 110 * the first bio. 111 * If this is a user-requested check/repair, allocate 112 * RESYNC_PAGES for each bio. 113 */ 114 if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) 115 j = pi->raid_disks; 116 else 117 j = 1; 118 while(j--) { 119 bio = r1_bio->bios[j]; 120 for (i = 0; i < RESYNC_PAGES; i++) { 121 page = alloc_page(gfp_flags); 122 if (unlikely(!page)) 123 goto out_free_pages; 124 125 bio->bi_io_vec[i].bv_page = page; 126 bio->bi_vcnt = i+1; 127 } 128 } 129 /* If not user-requests, copy the page pointers to all bios */ 130 if (!test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) { 131 for (i=0; i<RESYNC_PAGES ; i++) 132 for (j=1; j<pi->raid_disks; j++) 133 r1_bio->bios[j]->bi_io_vec[i].bv_page = 134 r1_bio->bios[0]->bi_io_vec[i].bv_page; 135 } 136 137 r1_bio->master_bio = NULL; 138 139 return r1_bio; 140 141 out_free_pages: 142 for (j=0 ; j < pi->raid_disks; j++) 143 for (i=0; i < r1_bio->bios[j]->bi_vcnt ; i++) 144 put_page(r1_bio->bios[j]->bi_io_vec[i].bv_page); 145 j = -1; 146 out_free_bio: 147 while ( ++j < pi->raid_disks ) 148 bio_put(r1_bio->bios[j]); 149 r1bio_pool_free(r1_bio, data); 150 return NULL; 151 } 152 153 static void r1buf_pool_free(void *__r1_bio, void *data) 154 { 155 struct pool_info *pi = data; 156 int i,j; 157 r1bio_t *r1bio = __r1_bio; 158 159 for (i = 0; i < RESYNC_PAGES; i++) 160 for (j = pi->raid_disks; j-- ;) { 161 if (j == 0 || 162 r1bio->bios[j]->bi_io_vec[i].bv_page != 163 r1bio->bios[0]->bi_io_vec[i].bv_page) 164 safe_put_page(r1bio->bios[j]->bi_io_vec[i].bv_page); 165 } 166 for (i=0 ; i < pi->raid_disks; i++) 167 bio_put(r1bio->bios[i]); 168 169 r1bio_pool_free(r1bio, data); 170 } 171 172 static void put_all_bios(conf_t *conf, r1bio_t *r1_bio) 173 { 174 int i; 175 176 for (i = 0; i < conf->raid_disks; i++) { 177 struct bio **bio = r1_bio->bios + i; 178 if (*bio && *bio != IO_BLOCKED) 179 bio_put(*bio); 180 *bio = NULL; 181 } 182 } 183 184 static void free_r1bio(r1bio_t *r1_bio) 185 { 186 conf_t *conf = r1_bio->mddev->private; 187 188 /* 189 * Wake up any possible resync thread that waits for the device 190 * to go idle. 191 */ 192 allow_barrier(conf); 193 194 put_all_bios(conf, r1_bio); 195 mempool_free(r1_bio, conf->r1bio_pool); 196 } 197 198 static void put_buf(r1bio_t *r1_bio) 199 { 200 conf_t *conf = r1_bio->mddev->private; 201 int i; 202 203 for (i=0; i<conf->raid_disks; i++) { 204 struct bio *bio = r1_bio->bios[i]; 205 if (bio->bi_end_io) 206 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); 207 } 208 209 mempool_free(r1_bio, conf->r1buf_pool); 210 211 lower_barrier(conf); 212 } 213 214 static void reschedule_retry(r1bio_t *r1_bio) 215 { 216 unsigned long flags; 217 mddev_t *mddev = r1_bio->mddev; 218 conf_t *conf = mddev->private; 219 220 spin_lock_irqsave(&conf->device_lock, flags); 221 list_add(&r1_bio->retry_list, &conf->retry_list); 222 conf->nr_queued ++; 223 spin_unlock_irqrestore(&conf->device_lock, flags); 224 225 wake_up(&conf->wait_barrier); 226 md_wakeup_thread(mddev->thread); 227 } 228 229 /* 230 * raid_end_bio_io() is called when we have finished servicing a mirrored 231 * operation and are ready to return a success/failure code to the buffer 232 * cache layer. 233 */ 234 static void raid_end_bio_io(r1bio_t *r1_bio) 235 { 236 struct bio *bio = r1_bio->master_bio; 237 238 /* if nobody has done the final endio yet, do it now */ 239 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 240 PRINTK(KERN_DEBUG "raid1: sync end %s on sectors %llu-%llu\n", 241 (bio_data_dir(bio) == WRITE) ? "write" : "read", 242 (unsigned long long) bio->bi_sector, 243 (unsigned long long) bio->bi_sector + 244 (bio->bi_size >> 9) - 1); 245 246 bio_endio(bio, 247 test_bit(R1BIO_Uptodate, &r1_bio->state) ? 0 : -EIO); 248 } 249 free_r1bio(r1_bio); 250 } 251 252 /* 253 * Update disk head position estimator based on IRQ completion info. 254 */ 255 static inline void update_head_pos(int disk, r1bio_t *r1_bio) 256 { 257 conf_t *conf = r1_bio->mddev->private; 258 259 conf->mirrors[disk].head_position = 260 r1_bio->sector + (r1_bio->sectors); 261 } 262 263 static void raid1_end_read_request(struct bio *bio, int error) 264 { 265 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 266 r1bio_t *r1_bio = bio->bi_private; 267 int mirror; 268 conf_t *conf = r1_bio->mddev->private; 269 270 mirror = r1_bio->read_disk; 271 /* 272 * this branch is our 'one mirror IO has finished' event handler: 273 */ 274 update_head_pos(mirror, r1_bio); 275 276 if (uptodate) 277 set_bit(R1BIO_Uptodate, &r1_bio->state); 278 else { 279 /* If all other devices have failed, we want to return 280 * the error upwards rather than fail the last device. 281 * Here we redefine "uptodate" to mean "Don't want to retry" 282 */ 283 unsigned long flags; 284 spin_lock_irqsave(&conf->device_lock, flags); 285 if (r1_bio->mddev->degraded == conf->raid_disks || 286 (r1_bio->mddev->degraded == conf->raid_disks-1 && 287 !test_bit(Faulty, &conf->mirrors[mirror].rdev->flags))) 288 uptodate = 1; 289 spin_unlock_irqrestore(&conf->device_lock, flags); 290 } 291 292 if (uptodate) 293 raid_end_bio_io(r1_bio); 294 else { 295 /* 296 * oops, read error: 297 */ 298 char b[BDEVNAME_SIZE]; 299 if (printk_ratelimit()) 300 printk(KERN_ERR "md/raid1:%s: %s: rescheduling sector %llu\n", 301 mdname(conf->mddev), 302 bdevname(conf->mirrors[mirror].rdev->bdev,b), (unsigned long long)r1_bio->sector); 303 reschedule_retry(r1_bio); 304 } 305 306 rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev); 307 } 308 309 static void r1_bio_write_done(r1bio_t *r1_bio, int vcnt, struct bio_vec *bv, 310 int behind) 311 { 312 if (atomic_dec_and_test(&r1_bio->remaining)) 313 { 314 /* it really is the end of this request */ 315 if (test_bit(R1BIO_BehindIO, &r1_bio->state)) { 316 /* free extra copy of the data pages */ 317 int i = vcnt; 318 while (i--) 319 safe_put_page(bv[i].bv_page); 320 } 321 /* clear the bitmap if all writes complete successfully */ 322 bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector, 323 r1_bio->sectors, 324 !test_bit(R1BIO_Degraded, &r1_bio->state), 325 behind); 326 md_write_end(r1_bio->mddev); 327 raid_end_bio_io(r1_bio); 328 } 329 } 330 331 static void raid1_end_write_request(struct bio *bio, int error) 332 { 333 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 334 r1bio_t *r1_bio = bio->bi_private; 335 int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state); 336 conf_t *conf = r1_bio->mddev->private; 337 struct bio *to_put = NULL; 338 339 340 for (mirror = 0; mirror < conf->raid_disks; mirror++) 341 if (r1_bio->bios[mirror] == bio) 342 break; 343 344 /* 345 * 'one mirror IO has finished' event handler: 346 */ 347 r1_bio->bios[mirror] = NULL; 348 to_put = bio; 349 if (!uptodate) { 350 md_error(r1_bio->mddev, conf->mirrors[mirror].rdev); 351 /* an I/O failed, we can't clear the bitmap */ 352 set_bit(R1BIO_Degraded, &r1_bio->state); 353 } else 354 /* 355 * Set R1BIO_Uptodate in our master bio, so that we 356 * will return a good error code for to the higher 357 * levels even if IO on some other mirrored buffer 358 * fails. 359 * 360 * The 'master' represents the composite IO operation 361 * to user-side. So if something waits for IO, then it 362 * will wait for the 'master' bio. 363 */ 364 set_bit(R1BIO_Uptodate, &r1_bio->state); 365 366 update_head_pos(mirror, r1_bio); 367 368 if (behind) { 369 if (test_bit(WriteMostly, &conf->mirrors[mirror].rdev->flags)) 370 atomic_dec(&r1_bio->behind_remaining); 371 372 /* 373 * In behind mode, we ACK the master bio once the I/O 374 * has safely reached all non-writemostly 375 * disks. Setting the Returned bit ensures that this 376 * gets done only once -- we don't ever want to return 377 * -EIO here, instead we'll wait 378 */ 379 if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) && 380 test_bit(R1BIO_Uptodate, &r1_bio->state)) { 381 /* Maybe we can return now */ 382 if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { 383 struct bio *mbio = r1_bio->master_bio; 384 PRINTK(KERN_DEBUG "raid1: behind end write sectors %llu-%llu\n", 385 (unsigned long long) mbio->bi_sector, 386 (unsigned long long) mbio->bi_sector + 387 (mbio->bi_size >> 9) - 1); 388 bio_endio(mbio, 0); 389 } 390 } 391 } 392 rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev); 393 394 /* 395 * Let's see if all mirrored write operations have finished 396 * already. 397 */ 398 r1_bio_write_done(r1_bio, bio->bi_vcnt, bio->bi_io_vec, behind); 399 400 if (to_put) 401 bio_put(to_put); 402 } 403 404 405 /* 406 * This routine returns the disk from which the requested read should 407 * be done. There is a per-array 'next expected sequential IO' sector 408 * number - if this matches on the next IO then we use the last disk. 409 * There is also a per-disk 'last know head position' sector that is 410 * maintained from IRQ contexts, both the normal and the resync IO 411 * completion handlers update this position correctly. If there is no 412 * perfect sequential match then we pick the disk whose head is closest. 413 * 414 * If there are 2 mirrors in the same 2 devices, performance degrades 415 * because position is mirror, not device based. 416 * 417 * The rdev for the device selected will have nr_pending incremented. 418 */ 419 static int read_balance(conf_t *conf, r1bio_t *r1_bio) 420 { 421 const sector_t this_sector = r1_bio->sector; 422 const int sectors = r1_bio->sectors; 423 int new_disk = -1; 424 int start_disk; 425 int i; 426 sector_t new_distance, current_distance; 427 mdk_rdev_t *rdev; 428 int choose_first; 429 430 rcu_read_lock(); 431 /* 432 * Check if we can balance. We can balance on the whole 433 * device if no resync is going on, or below the resync window. 434 * We take the first readable disk when above the resync window. 435 */ 436 retry: 437 if (conf->mddev->recovery_cp < MaxSector && 438 (this_sector + sectors >= conf->next_resync)) { 439 choose_first = 1; 440 start_disk = 0; 441 } else { 442 choose_first = 0; 443 start_disk = conf->last_used; 444 } 445 446 /* make sure the disk is operational */ 447 for (i = 0 ; i < conf->raid_disks ; i++) { 448 int disk = start_disk + i; 449 if (disk >= conf->raid_disks) 450 disk -= conf->raid_disks; 451 452 rdev = rcu_dereference(conf->mirrors[disk].rdev); 453 if (r1_bio->bios[disk] == IO_BLOCKED 454 || rdev == NULL 455 || !test_bit(In_sync, &rdev->flags)) 456 continue; 457 458 new_disk = disk; 459 if (!test_bit(WriteMostly, &rdev->flags)) 460 break; 461 } 462 463 if (new_disk < 0 || choose_first) 464 goto rb_out; 465 466 /* 467 * Don't change to another disk for sequential reads: 468 */ 469 if (conf->next_seq_sect == this_sector) 470 goto rb_out; 471 if (this_sector == conf->mirrors[new_disk].head_position) 472 goto rb_out; 473 474 current_distance = abs(this_sector 475 - conf->mirrors[new_disk].head_position); 476 477 /* look for a better disk - i.e. head is closer */ 478 start_disk = new_disk; 479 for (i = 1; i < conf->raid_disks; i++) { 480 int disk = start_disk + 1; 481 if (disk >= conf->raid_disks) 482 disk -= conf->raid_disks; 483 484 rdev = rcu_dereference(conf->mirrors[disk].rdev); 485 if (r1_bio->bios[disk] == IO_BLOCKED 486 || rdev == NULL 487 || !test_bit(In_sync, &rdev->flags) 488 || test_bit(WriteMostly, &rdev->flags)) 489 continue; 490 491 if (!atomic_read(&rdev->nr_pending)) { 492 new_disk = disk; 493 break; 494 } 495 new_distance = abs(this_sector - conf->mirrors[disk].head_position); 496 if (new_distance < current_distance) { 497 current_distance = new_distance; 498 new_disk = disk; 499 } 500 } 501 502 rb_out: 503 if (new_disk >= 0) { 504 rdev = rcu_dereference(conf->mirrors[new_disk].rdev); 505 if (!rdev) 506 goto retry; 507 atomic_inc(&rdev->nr_pending); 508 if (!test_bit(In_sync, &rdev->flags)) { 509 /* cannot risk returning a device that failed 510 * before we inc'ed nr_pending 511 */ 512 rdev_dec_pending(rdev, conf->mddev); 513 goto retry; 514 } 515 conf->next_seq_sect = this_sector + sectors; 516 conf->last_used = new_disk; 517 } 518 rcu_read_unlock(); 519 520 return new_disk; 521 } 522 523 static void unplug_slaves(mddev_t *mddev) 524 { 525 conf_t *conf = mddev->private; 526 int i; 527 528 rcu_read_lock(); 529 for (i=0; i<mddev->raid_disks; i++) { 530 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); 531 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { 532 struct request_queue *r_queue = bdev_get_queue(rdev->bdev); 533 534 atomic_inc(&rdev->nr_pending); 535 rcu_read_unlock(); 536 537 blk_unplug(r_queue); 538 539 rdev_dec_pending(rdev, mddev); 540 rcu_read_lock(); 541 } 542 } 543 rcu_read_unlock(); 544 } 545 546 static void raid1_unplug(struct request_queue *q) 547 { 548 mddev_t *mddev = q->queuedata; 549 550 unplug_slaves(mddev); 551 md_wakeup_thread(mddev->thread); 552 } 553 554 static int raid1_congested(void *data, int bits) 555 { 556 mddev_t *mddev = data; 557 conf_t *conf = mddev->private; 558 int i, ret = 0; 559 560 if (mddev_congested(mddev, bits)) 561 return 1; 562 563 rcu_read_lock(); 564 for (i = 0; i < mddev->raid_disks; i++) { 565 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); 566 if (rdev && !test_bit(Faulty, &rdev->flags)) { 567 struct request_queue *q = bdev_get_queue(rdev->bdev); 568 569 /* Note the '|| 1' - when read_balance prefers 570 * non-congested targets, it can be removed 571 */ 572 if ((bits & (1<<BDI_async_congested)) || 1) 573 ret |= bdi_congested(&q->backing_dev_info, bits); 574 else 575 ret &= bdi_congested(&q->backing_dev_info, bits); 576 } 577 } 578 rcu_read_unlock(); 579 return ret; 580 } 581 582 583 static int flush_pending_writes(conf_t *conf) 584 { 585 /* Any writes that have been queued but are awaiting 586 * bitmap updates get flushed here. 587 * We return 1 if any requests were actually submitted. 588 */ 589 int rv = 0; 590 591 spin_lock_irq(&conf->device_lock); 592 593 if (conf->pending_bio_list.head) { 594 struct bio *bio; 595 bio = bio_list_get(&conf->pending_bio_list); 596 blk_remove_plug(conf->mddev->queue); 597 spin_unlock_irq(&conf->device_lock); 598 /* flush any pending bitmap writes to 599 * disk before proceeding w/ I/O */ 600 bitmap_unplug(conf->mddev->bitmap); 601 602 while (bio) { /* submit pending writes */ 603 struct bio *next = bio->bi_next; 604 bio->bi_next = NULL; 605 generic_make_request(bio); 606 bio = next; 607 } 608 rv = 1; 609 } else 610 spin_unlock_irq(&conf->device_lock); 611 return rv; 612 } 613 614 /* Barriers.... 615 * Sometimes we need to suspend IO while we do something else, 616 * either some resync/recovery, or reconfigure the array. 617 * To do this we raise a 'barrier'. 618 * The 'barrier' is a counter that can be raised multiple times 619 * to count how many activities are happening which preclude 620 * normal IO. 621 * We can only raise the barrier if there is no pending IO. 622 * i.e. if nr_pending == 0. 623 * We choose only to raise the barrier if no-one is waiting for the 624 * barrier to go down. This means that as soon as an IO request 625 * is ready, no other operations which require a barrier will start 626 * until the IO request has had a chance. 627 * 628 * So: regular IO calls 'wait_barrier'. When that returns there 629 * is no backgroup IO happening, It must arrange to call 630 * allow_barrier when it has finished its IO. 631 * backgroup IO calls must call raise_barrier. Once that returns 632 * there is no normal IO happeing. It must arrange to call 633 * lower_barrier when the particular background IO completes. 634 */ 635 #define RESYNC_DEPTH 32 636 637 static void raise_barrier(conf_t *conf) 638 { 639 spin_lock_irq(&conf->resync_lock); 640 641 /* Wait until no block IO is waiting */ 642 wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting, 643 conf->resync_lock, 644 raid1_unplug(conf->mddev->queue)); 645 646 /* block any new IO from starting */ 647 conf->barrier++; 648 649 /* Now wait for all pending IO to complete */ 650 wait_event_lock_irq(conf->wait_barrier, 651 !conf->nr_pending && conf->barrier < RESYNC_DEPTH, 652 conf->resync_lock, 653 raid1_unplug(conf->mddev->queue)); 654 655 spin_unlock_irq(&conf->resync_lock); 656 } 657 658 static void lower_barrier(conf_t *conf) 659 { 660 unsigned long flags; 661 BUG_ON(conf->barrier <= 0); 662 spin_lock_irqsave(&conf->resync_lock, flags); 663 conf->barrier--; 664 spin_unlock_irqrestore(&conf->resync_lock, flags); 665 wake_up(&conf->wait_barrier); 666 } 667 668 static void wait_barrier(conf_t *conf) 669 { 670 spin_lock_irq(&conf->resync_lock); 671 if (conf->barrier) { 672 conf->nr_waiting++; 673 wait_event_lock_irq(conf->wait_barrier, !conf->barrier, 674 conf->resync_lock, 675 raid1_unplug(conf->mddev->queue)); 676 conf->nr_waiting--; 677 } 678 conf->nr_pending++; 679 spin_unlock_irq(&conf->resync_lock); 680 } 681 682 static void allow_barrier(conf_t *conf) 683 { 684 unsigned long flags; 685 spin_lock_irqsave(&conf->resync_lock, flags); 686 conf->nr_pending--; 687 spin_unlock_irqrestore(&conf->resync_lock, flags); 688 wake_up(&conf->wait_barrier); 689 } 690 691 static void freeze_array(conf_t *conf) 692 { 693 /* stop syncio and normal IO and wait for everything to 694 * go quite. 695 * We increment barrier and nr_waiting, and then 696 * wait until nr_pending match nr_queued+1 697 * This is called in the context of one normal IO request 698 * that has failed. Thus any sync request that might be pending 699 * will be blocked by nr_pending, and we need to wait for 700 * pending IO requests to complete or be queued for re-try. 701 * Thus the number queued (nr_queued) plus this request (1) 702 * must match the number of pending IOs (nr_pending) before 703 * we continue. 704 */ 705 spin_lock_irq(&conf->resync_lock); 706 conf->barrier++; 707 conf->nr_waiting++; 708 wait_event_lock_irq(conf->wait_barrier, 709 conf->nr_pending == conf->nr_queued+1, 710 conf->resync_lock, 711 ({ flush_pending_writes(conf); 712 raid1_unplug(conf->mddev->queue); })); 713 spin_unlock_irq(&conf->resync_lock); 714 } 715 static void unfreeze_array(conf_t *conf) 716 { 717 /* reverse the effect of the freeze */ 718 spin_lock_irq(&conf->resync_lock); 719 conf->barrier--; 720 conf->nr_waiting--; 721 wake_up(&conf->wait_barrier); 722 spin_unlock_irq(&conf->resync_lock); 723 } 724 725 726 /* duplicate the data pages for behind I/O 727 * We return a list of bio_vec rather than just page pointers 728 * as it makes freeing easier 729 */ 730 static struct bio_vec *alloc_behind_pages(struct bio *bio) 731 { 732 int i; 733 struct bio_vec *bvec; 734 struct bio_vec *pages = kzalloc(bio->bi_vcnt * sizeof(struct bio_vec), 735 GFP_NOIO); 736 if (unlikely(!pages)) 737 goto do_sync_io; 738 739 bio_for_each_segment(bvec, bio, i) { 740 pages[i].bv_page = alloc_page(GFP_NOIO); 741 if (unlikely(!pages[i].bv_page)) 742 goto do_sync_io; 743 memcpy(kmap(pages[i].bv_page) + bvec->bv_offset, 744 kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len); 745 kunmap(pages[i].bv_page); 746 kunmap(bvec->bv_page); 747 } 748 749 return pages; 750 751 do_sync_io: 752 if (pages) 753 for (i = 0; i < bio->bi_vcnt && pages[i].bv_page; i++) 754 put_page(pages[i].bv_page); 755 kfree(pages); 756 PRINTK("%dB behind alloc failed, doing sync I/O\n", bio->bi_size); 757 return NULL; 758 } 759 760 static int make_request(mddev_t *mddev, struct bio * bio) 761 { 762 conf_t *conf = mddev->private; 763 mirror_info_t *mirror; 764 r1bio_t *r1_bio; 765 struct bio *read_bio; 766 int i, targets = 0, disks; 767 struct bitmap *bitmap; 768 unsigned long flags; 769 struct bio_vec *behind_pages = NULL; 770 const int rw = bio_data_dir(bio); 771 const unsigned long do_sync = (bio->bi_rw & REQ_SYNC); 772 const unsigned long do_flush_fua = (bio->bi_rw & (REQ_FLUSH | REQ_FUA)); 773 mdk_rdev_t *blocked_rdev; 774 775 /* 776 * Register the new request and wait if the reconstruction 777 * thread has put up a bar for new requests. 778 * Continue immediately if no resync is active currently. 779 */ 780 781 md_write_start(mddev, bio); /* wait on superblock update early */ 782 783 if (bio_data_dir(bio) == WRITE && 784 bio->bi_sector + bio->bi_size/512 > mddev->suspend_lo && 785 bio->bi_sector < mddev->suspend_hi) { 786 /* As the suspend_* range is controlled by 787 * userspace, we want an interruptible 788 * wait. 789 */ 790 DEFINE_WAIT(w); 791 for (;;) { 792 flush_signals(current); 793 prepare_to_wait(&conf->wait_barrier, 794 &w, TASK_INTERRUPTIBLE); 795 if (bio->bi_sector + bio->bi_size/512 <= mddev->suspend_lo || 796 bio->bi_sector >= mddev->suspend_hi) 797 break; 798 schedule(); 799 } 800 finish_wait(&conf->wait_barrier, &w); 801 } 802 803 wait_barrier(conf); 804 805 bitmap = mddev->bitmap; 806 807 /* 808 * make_request() can abort the operation when READA is being 809 * used and no empty request is available. 810 * 811 */ 812 r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO); 813 814 r1_bio->master_bio = bio; 815 r1_bio->sectors = bio->bi_size >> 9; 816 r1_bio->state = 0; 817 r1_bio->mddev = mddev; 818 r1_bio->sector = bio->bi_sector; 819 820 if (rw == READ) { 821 /* 822 * read balancing logic: 823 */ 824 int rdisk = read_balance(conf, r1_bio); 825 826 if (rdisk < 0) { 827 /* couldn't find anywhere to read from */ 828 raid_end_bio_io(r1_bio); 829 return 0; 830 } 831 mirror = conf->mirrors + rdisk; 832 833 if (test_bit(WriteMostly, &mirror->rdev->flags) && 834 bitmap) { 835 /* Reading from a write-mostly device must 836 * take care not to over-take any writes 837 * that are 'behind' 838 */ 839 wait_event(bitmap->behind_wait, 840 atomic_read(&bitmap->behind_writes) == 0); 841 } 842 r1_bio->read_disk = rdisk; 843 844 read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev); 845 846 r1_bio->bios[rdisk] = read_bio; 847 848 read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset; 849 read_bio->bi_bdev = mirror->rdev->bdev; 850 read_bio->bi_end_io = raid1_end_read_request; 851 read_bio->bi_rw = READ | do_sync; 852 read_bio->bi_private = r1_bio; 853 854 generic_make_request(read_bio); 855 return 0; 856 } 857 858 /* 859 * WRITE: 860 */ 861 /* first select target devices under spinlock and 862 * inc refcount on their rdev. Record them by setting 863 * bios[x] to bio 864 */ 865 disks = conf->raid_disks; 866 retry_write: 867 blocked_rdev = NULL; 868 rcu_read_lock(); 869 for (i = 0; i < disks; i++) { 870 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); 871 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 872 atomic_inc(&rdev->nr_pending); 873 blocked_rdev = rdev; 874 break; 875 } 876 if (rdev && !test_bit(Faulty, &rdev->flags)) { 877 atomic_inc(&rdev->nr_pending); 878 if (test_bit(Faulty, &rdev->flags)) { 879 rdev_dec_pending(rdev, mddev); 880 r1_bio->bios[i] = NULL; 881 } else { 882 r1_bio->bios[i] = bio; 883 targets++; 884 } 885 } else 886 r1_bio->bios[i] = NULL; 887 } 888 rcu_read_unlock(); 889 890 if (unlikely(blocked_rdev)) { 891 /* Wait for this device to become unblocked */ 892 int j; 893 894 for (j = 0; j < i; j++) 895 if (r1_bio->bios[j]) 896 rdev_dec_pending(conf->mirrors[j].rdev, mddev); 897 898 allow_barrier(conf); 899 md_wait_for_blocked_rdev(blocked_rdev, mddev); 900 wait_barrier(conf); 901 goto retry_write; 902 } 903 904 BUG_ON(targets == 0); /* we never fail the last device */ 905 906 if (targets < conf->raid_disks) { 907 /* array is degraded, we will not clear the bitmap 908 * on I/O completion (see raid1_end_write_request) */ 909 set_bit(R1BIO_Degraded, &r1_bio->state); 910 } 911 912 /* do behind I/O ? 913 * Not if there are too many, or cannot allocate memory, 914 * or a reader on WriteMostly is waiting for behind writes 915 * to flush */ 916 if (bitmap && 917 (atomic_read(&bitmap->behind_writes) 918 < mddev->bitmap_info.max_write_behind) && 919 !waitqueue_active(&bitmap->behind_wait) && 920 (behind_pages = alloc_behind_pages(bio)) != NULL) 921 set_bit(R1BIO_BehindIO, &r1_bio->state); 922 923 atomic_set(&r1_bio->remaining, 1); 924 atomic_set(&r1_bio->behind_remaining, 0); 925 926 bitmap_startwrite(bitmap, bio->bi_sector, r1_bio->sectors, 927 test_bit(R1BIO_BehindIO, &r1_bio->state)); 928 for (i = 0; i < disks; i++) { 929 struct bio *mbio; 930 if (!r1_bio->bios[i]) 931 continue; 932 933 mbio = bio_clone_mddev(bio, GFP_NOIO, mddev); 934 r1_bio->bios[i] = mbio; 935 936 mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset; 937 mbio->bi_bdev = conf->mirrors[i].rdev->bdev; 938 mbio->bi_end_io = raid1_end_write_request; 939 mbio->bi_rw = WRITE | do_flush_fua | do_sync; 940 mbio->bi_private = r1_bio; 941 942 if (behind_pages) { 943 struct bio_vec *bvec; 944 int j; 945 946 /* Yes, I really want the '__' version so that 947 * we clear any unused pointer in the io_vec, rather 948 * than leave them unchanged. This is important 949 * because when we come to free the pages, we won't 950 * know the original bi_idx, so we just free 951 * them all 952 */ 953 __bio_for_each_segment(bvec, mbio, j, 0) 954 bvec->bv_page = behind_pages[j].bv_page; 955 if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags)) 956 atomic_inc(&r1_bio->behind_remaining); 957 } 958 959 atomic_inc(&r1_bio->remaining); 960 spin_lock_irqsave(&conf->device_lock, flags); 961 bio_list_add(&conf->pending_bio_list, mbio); 962 blk_plug_device(mddev->queue); 963 spin_unlock_irqrestore(&conf->device_lock, flags); 964 } 965 r1_bio_write_done(r1_bio, bio->bi_vcnt, behind_pages, behind_pages != NULL); 966 kfree(behind_pages); /* the behind pages are attached to the bios now */ 967 968 /* In case raid1d snuck in to freeze_array */ 969 wake_up(&conf->wait_barrier); 970 971 if (do_sync) 972 md_wakeup_thread(mddev->thread); 973 974 return 0; 975 } 976 977 static void status(struct seq_file *seq, mddev_t *mddev) 978 { 979 conf_t *conf = mddev->private; 980 int i; 981 982 seq_printf(seq, " [%d/%d] [", conf->raid_disks, 983 conf->raid_disks - mddev->degraded); 984 rcu_read_lock(); 985 for (i = 0; i < conf->raid_disks; i++) { 986 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); 987 seq_printf(seq, "%s", 988 rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 989 } 990 rcu_read_unlock(); 991 seq_printf(seq, "]"); 992 } 993 994 995 static void error(mddev_t *mddev, mdk_rdev_t *rdev) 996 { 997 char b[BDEVNAME_SIZE]; 998 conf_t *conf = mddev->private; 999 1000 /* 1001 * If it is not operational, then we have already marked it as dead 1002 * else if it is the last working disks, ignore the error, let the 1003 * next level up know. 1004 * else mark the drive as failed 1005 */ 1006 if (test_bit(In_sync, &rdev->flags) 1007 && (conf->raid_disks - mddev->degraded) == 1) { 1008 /* 1009 * Don't fail the drive, act as though we were just a 1010 * normal single drive. 1011 * However don't try a recovery from this drive as 1012 * it is very likely to fail. 1013 */ 1014 mddev->recovery_disabled = 1; 1015 return; 1016 } 1017 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1018 unsigned long flags; 1019 spin_lock_irqsave(&conf->device_lock, flags); 1020 mddev->degraded++; 1021 set_bit(Faulty, &rdev->flags); 1022 spin_unlock_irqrestore(&conf->device_lock, flags); 1023 /* 1024 * if recovery is running, make sure it aborts. 1025 */ 1026 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1027 } else 1028 set_bit(Faulty, &rdev->flags); 1029 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1030 printk(KERN_ALERT "md/raid1:%s: Disk failure on %s, disabling device.\n" 1031 KERN_ALERT "md/raid1:%s: Operation continuing on %d devices.\n", 1032 mdname(mddev), bdevname(rdev->bdev, b), 1033 mdname(mddev), conf->raid_disks - mddev->degraded); 1034 } 1035 1036 static void print_conf(conf_t *conf) 1037 { 1038 int i; 1039 1040 printk(KERN_DEBUG "RAID1 conf printout:\n"); 1041 if (!conf) { 1042 printk(KERN_DEBUG "(!conf)\n"); 1043 return; 1044 } 1045 printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, 1046 conf->raid_disks); 1047 1048 rcu_read_lock(); 1049 for (i = 0; i < conf->raid_disks; i++) { 1050 char b[BDEVNAME_SIZE]; 1051 mdk_rdev_t *rdev = rcu_dereference(conf->mirrors[i].rdev); 1052 if (rdev) 1053 printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n", 1054 i, !test_bit(In_sync, &rdev->flags), 1055 !test_bit(Faulty, &rdev->flags), 1056 bdevname(rdev->bdev,b)); 1057 } 1058 rcu_read_unlock(); 1059 } 1060 1061 static void close_sync(conf_t *conf) 1062 { 1063 wait_barrier(conf); 1064 allow_barrier(conf); 1065 1066 mempool_destroy(conf->r1buf_pool); 1067 conf->r1buf_pool = NULL; 1068 } 1069 1070 static int raid1_spare_active(mddev_t *mddev) 1071 { 1072 int i; 1073 conf_t *conf = mddev->private; 1074 int count = 0; 1075 unsigned long flags; 1076 1077 /* 1078 * Find all failed disks within the RAID1 configuration 1079 * and mark them readable. 1080 * Called under mddev lock, so rcu protection not needed. 1081 */ 1082 for (i = 0; i < conf->raid_disks; i++) { 1083 mdk_rdev_t *rdev = conf->mirrors[i].rdev; 1084 if (rdev 1085 && !test_bit(Faulty, &rdev->flags) 1086 && !test_and_set_bit(In_sync, &rdev->flags)) { 1087 count++; 1088 sysfs_notify_dirent(rdev->sysfs_state); 1089 } 1090 } 1091 spin_lock_irqsave(&conf->device_lock, flags); 1092 mddev->degraded -= count; 1093 spin_unlock_irqrestore(&conf->device_lock, flags); 1094 1095 print_conf(conf); 1096 return count; 1097 } 1098 1099 1100 static int raid1_add_disk(mddev_t *mddev, mdk_rdev_t *rdev) 1101 { 1102 conf_t *conf = mddev->private; 1103 int err = -EEXIST; 1104 int mirror = 0; 1105 mirror_info_t *p; 1106 int first = 0; 1107 int last = mddev->raid_disks - 1; 1108 1109 if (rdev->raid_disk >= 0) 1110 first = last = rdev->raid_disk; 1111 1112 for (mirror = first; mirror <= last; mirror++) 1113 if ( !(p=conf->mirrors+mirror)->rdev) { 1114 1115 disk_stack_limits(mddev->gendisk, rdev->bdev, 1116 rdev->data_offset << 9); 1117 /* as we don't honour merge_bvec_fn, we must 1118 * never risk violating it, so limit 1119 * ->max_segments to one lying with a single 1120 * page, as a one page request is never in 1121 * violation. 1122 */ 1123 if (rdev->bdev->bd_disk->queue->merge_bvec_fn) { 1124 blk_queue_max_segments(mddev->queue, 1); 1125 blk_queue_segment_boundary(mddev->queue, 1126 PAGE_CACHE_SIZE - 1); 1127 } 1128 1129 p->head_position = 0; 1130 rdev->raid_disk = mirror; 1131 err = 0; 1132 /* As all devices are equivalent, we don't need a full recovery 1133 * if this was recently any drive of the array 1134 */ 1135 if (rdev->saved_raid_disk < 0) 1136 conf->fullsync = 1; 1137 rcu_assign_pointer(p->rdev, rdev); 1138 break; 1139 } 1140 md_integrity_add_rdev(rdev, mddev); 1141 print_conf(conf); 1142 return err; 1143 } 1144 1145 static int raid1_remove_disk(mddev_t *mddev, int number) 1146 { 1147 conf_t *conf = mddev->private; 1148 int err = 0; 1149 mdk_rdev_t *rdev; 1150 mirror_info_t *p = conf->mirrors+ number; 1151 1152 print_conf(conf); 1153 rdev = p->rdev; 1154 if (rdev) { 1155 if (test_bit(In_sync, &rdev->flags) || 1156 atomic_read(&rdev->nr_pending)) { 1157 err = -EBUSY; 1158 goto abort; 1159 } 1160 /* Only remove non-faulty devices if recovery 1161 * is not possible. 1162 */ 1163 if (!test_bit(Faulty, &rdev->flags) && 1164 !mddev->recovery_disabled && 1165 mddev->degraded < conf->raid_disks) { 1166 err = -EBUSY; 1167 goto abort; 1168 } 1169 p->rdev = NULL; 1170 synchronize_rcu(); 1171 if (atomic_read(&rdev->nr_pending)) { 1172 /* lost the race, try later */ 1173 err = -EBUSY; 1174 p->rdev = rdev; 1175 goto abort; 1176 } 1177 md_integrity_register(mddev); 1178 } 1179 abort: 1180 1181 print_conf(conf); 1182 return err; 1183 } 1184 1185 1186 static void end_sync_read(struct bio *bio, int error) 1187 { 1188 r1bio_t *r1_bio = bio->bi_private; 1189 int i; 1190 1191 for (i=r1_bio->mddev->raid_disks; i--; ) 1192 if (r1_bio->bios[i] == bio) 1193 break; 1194 BUG_ON(i < 0); 1195 update_head_pos(i, r1_bio); 1196 /* 1197 * we have read a block, now it needs to be re-written, 1198 * or re-read if the read failed. 1199 * We don't do much here, just schedule handling by raid1d 1200 */ 1201 if (test_bit(BIO_UPTODATE, &bio->bi_flags)) 1202 set_bit(R1BIO_Uptodate, &r1_bio->state); 1203 1204 if (atomic_dec_and_test(&r1_bio->remaining)) 1205 reschedule_retry(r1_bio); 1206 } 1207 1208 static void end_sync_write(struct bio *bio, int error) 1209 { 1210 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1211 r1bio_t *r1_bio = bio->bi_private; 1212 mddev_t *mddev = r1_bio->mddev; 1213 conf_t *conf = mddev->private; 1214 int i; 1215 int mirror=0; 1216 1217 for (i = 0; i < conf->raid_disks; i++) 1218 if (r1_bio->bios[i] == bio) { 1219 mirror = i; 1220 break; 1221 } 1222 if (!uptodate) { 1223 sector_t sync_blocks = 0; 1224 sector_t s = r1_bio->sector; 1225 long sectors_to_go = r1_bio->sectors; 1226 /* make sure these bits doesn't get cleared. */ 1227 do { 1228 bitmap_end_sync(mddev->bitmap, s, 1229 &sync_blocks, 1); 1230 s += sync_blocks; 1231 sectors_to_go -= sync_blocks; 1232 } while (sectors_to_go > 0); 1233 md_error(mddev, conf->mirrors[mirror].rdev); 1234 } 1235 1236 update_head_pos(mirror, r1_bio); 1237 1238 if (atomic_dec_and_test(&r1_bio->remaining)) { 1239 sector_t s = r1_bio->sectors; 1240 put_buf(r1_bio); 1241 md_done_sync(mddev, s, uptodate); 1242 } 1243 } 1244 1245 static void sync_request_write(mddev_t *mddev, r1bio_t *r1_bio) 1246 { 1247 conf_t *conf = mddev->private; 1248 int i; 1249 int disks = conf->raid_disks; 1250 struct bio *bio, *wbio; 1251 1252 bio = r1_bio->bios[r1_bio->read_disk]; 1253 1254 1255 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 1256 /* We have read all readable devices. If we haven't 1257 * got the block, then there is no hope left. 1258 * If we have, then we want to do a comparison 1259 * and skip the write if everything is the same. 1260 * If any blocks failed to read, then we need to 1261 * attempt an over-write 1262 */ 1263 int primary; 1264 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) { 1265 for (i=0; i<mddev->raid_disks; i++) 1266 if (r1_bio->bios[i]->bi_end_io == end_sync_read) 1267 md_error(mddev, conf->mirrors[i].rdev); 1268 1269 md_done_sync(mddev, r1_bio->sectors, 1); 1270 put_buf(r1_bio); 1271 return; 1272 } 1273 for (primary=0; primary<mddev->raid_disks; primary++) 1274 if (r1_bio->bios[primary]->bi_end_io == end_sync_read && 1275 test_bit(BIO_UPTODATE, &r1_bio->bios[primary]->bi_flags)) { 1276 r1_bio->bios[primary]->bi_end_io = NULL; 1277 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); 1278 break; 1279 } 1280 r1_bio->read_disk = primary; 1281 for (i=0; i<mddev->raid_disks; i++) 1282 if (r1_bio->bios[i]->bi_end_io == end_sync_read) { 1283 int j; 1284 int vcnt = r1_bio->sectors >> (PAGE_SHIFT- 9); 1285 struct bio *pbio = r1_bio->bios[primary]; 1286 struct bio *sbio = r1_bio->bios[i]; 1287 1288 if (test_bit(BIO_UPTODATE, &sbio->bi_flags)) { 1289 for (j = vcnt; j-- ; ) { 1290 struct page *p, *s; 1291 p = pbio->bi_io_vec[j].bv_page; 1292 s = sbio->bi_io_vec[j].bv_page; 1293 if (memcmp(page_address(p), 1294 page_address(s), 1295 PAGE_SIZE)) 1296 break; 1297 } 1298 } else 1299 j = 0; 1300 if (j >= 0) 1301 mddev->resync_mismatches += r1_bio->sectors; 1302 if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery) 1303 && test_bit(BIO_UPTODATE, &sbio->bi_flags))) { 1304 sbio->bi_end_io = NULL; 1305 rdev_dec_pending(conf->mirrors[i].rdev, mddev); 1306 } else { 1307 /* fixup the bio for reuse */ 1308 int size; 1309 sbio->bi_vcnt = vcnt; 1310 sbio->bi_size = r1_bio->sectors << 9; 1311 sbio->bi_idx = 0; 1312 sbio->bi_phys_segments = 0; 1313 sbio->bi_flags &= ~(BIO_POOL_MASK - 1); 1314 sbio->bi_flags |= 1 << BIO_UPTODATE; 1315 sbio->bi_next = NULL; 1316 sbio->bi_sector = r1_bio->sector + 1317 conf->mirrors[i].rdev->data_offset; 1318 sbio->bi_bdev = conf->mirrors[i].rdev->bdev; 1319 size = sbio->bi_size; 1320 for (j = 0; j < vcnt ; j++) { 1321 struct bio_vec *bi; 1322 bi = &sbio->bi_io_vec[j]; 1323 bi->bv_offset = 0; 1324 if (size > PAGE_SIZE) 1325 bi->bv_len = PAGE_SIZE; 1326 else 1327 bi->bv_len = size; 1328 size -= PAGE_SIZE; 1329 memcpy(page_address(bi->bv_page), 1330 page_address(pbio->bi_io_vec[j].bv_page), 1331 PAGE_SIZE); 1332 } 1333 1334 } 1335 } 1336 } 1337 if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) { 1338 /* ouch - failed to read all of that. 1339 * Try some synchronous reads of other devices to get 1340 * good data, much like with normal read errors. Only 1341 * read into the pages we already have so we don't 1342 * need to re-issue the read request. 1343 * We don't need to freeze the array, because being in an 1344 * active sync request, there is no normal IO, and 1345 * no overlapping syncs. 1346 */ 1347 sector_t sect = r1_bio->sector; 1348 int sectors = r1_bio->sectors; 1349 int idx = 0; 1350 1351 while(sectors) { 1352 int s = sectors; 1353 int d = r1_bio->read_disk; 1354 int success = 0; 1355 mdk_rdev_t *rdev; 1356 1357 if (s > (PAGE_SIZE>>9)) 1358 s = PAGE_SIZE >> 9; 1359 do { 1360 if (r1_bio->bios[d]->bi_end_io == end_sync_read) { 1361 /* No rcu protection needed here devices 1362 * can only be removed when no resync is 1363 * active, and resync is currently active 1364 */ 1365 rdev = conf->mirrors[d].rdev; 1366 if (sync_page_io(rdev, 1367 sect + rdev->data_offset, 1368 s<<9, 1369 bio->bi_io_vec[idx].bv_page, 1370 READ)) { 1371 success = 1; 1372 break; 1373 } 1374 } 1375 d++; 1376 if (d == conf->raid_disks) 1377 d = 0; 1378 } while (!success && d != r1_bio->read_disk); 1379 1380 if (success) { 1381 int start = d; 1382 /* write it back and re-read */ 1383 set_bit(R1BIO_Uptodate, &r1_bio->state); 1384 while (d != r1_bio->read_disk) { 1385 if (d == 0) 1386 d = conf->raid_disks; 1387 d--; 1388 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 1389 continue; 1390 rdev = conf->mirrors[d].rdev; 1391 atomic_add(s, &rdev->corrected_errors); 1392 if (sync_page_io(rdev, 1393 sect + rdev->data_offset, 1394 s<<9, 1395 bio->bi_io_vec[idx].bv_page, 1396 WRITE) == 0) 1397 md_error(mddev, rdev); 1398 } 1399 d = start; 1400 while (d != r1_bio->read_disk) { 1401 if (d == 0) 1402 d = conf->raid_disks; 1403 d--; 1404 if (r1_bio->bios[d]->bi_end_io != end_sync_read) 1405 continue; 1406 rdev = conf->mirrors[d].rdev; 1407 if (sync_page_io(rdev, 1408 sect + rdev->data_offset, 1409 s<<9, 1410 bio->bi_io_vec[idx].bv_page, 1411 READ) == 0) 1412 md_error(mddev, rdev); 1413 } 1414 } else { 1415 char b[BDEVNAME_SIZE]; 1416 /* Cannot read from anywhere, array is toast */ 1417 md_error(mddev, conf->mirrors[r1_bio->read_disk].rdev); 1418 printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O read error" 1419 " for block %llu\n", 1420 mdname(mddev), 1421 bdevname(bio->bi_bdev, b), 1422 (unsigned long long)r1_bio->sector); 1423 md_done_sync(mddev, r1_bio->sectors, 0); 1424 put_buf(r1_bio); 1425 return; 1426 } 1427 sectors -= s; 1428 sect += s; 1429 idx ++; 1430 } 1431 } 1432 1433 /* 1434 * schedule writes 1435 */ 1436 atomic_set(&r1_bio->remaining, 1); 1437 for (i = 0; i < disks ; i++) { 1438 wbio = r1_bio->bios[i]; 1439 if (wbio->bi_end_io == NULL || 1440 (wbio->bi_end_io == end_sync_read && 1441 (i == r1_bio->read_disk || 1442 !test_bit(MD_RECOVERY_SYNC, &mddev->recovery)))) 1443 continue; 1444 1445 wbio->bi_rw = WRITE; 1446 wbio->bi_end_io = end_sync_write; 1447 atomic_inc(&r1_bio->remaining); 1448 md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9); 1449 1450 generic_make_request(wbio); 1451 } 1452 1453 if (atomic_dec_and_test(&r1_bio->remaining)) { 1454 /* if we're here, all write(s) have completed, so clean up */ 1455 md_done_sync(mddev, r1_bio->sectors, 1); 1456 put_buf(r1_bio); 1457 } 1458 } 1459 1460 /* 1461 * This is a kernel thread which: 1462 * 1463 * 1. Retries failed read operations on working mirrors. 1464 * 2. Updates the raid superblock when problems encounter. 1465 * 3. Performs writes following reads for array syncronising. 1466 */ 1467 1468 static void fix_read_error(conf_t *conf, int read_disk, 1469 sector_t sect, int sectors) 1470 { 1471 mddev_t *mddev = conf->mddev; 1472 while(sectors) { 1473 int s = sectors; 1474 int d = read_disk; 1475 int success = 0; 1476 int start; 1477 mdk_rdev_t *rdev; 1478 1479 if (s > (PAGE_SIZE>>9)) 1480 s = PAGE_SIZE >> 9; 1481 1482 do { 1483 /* Note: no rcu protection needed here 1484 * as this is synchronous in the raid1d thread 1485 * which is the thread that might remove 1486 * a device. If raid1d ever becomes multi-threaded.... 1487 */ 1488 rdev = conf->mirrors[d].rdev; 1489 if (rdev && 1490 test_bit(In_sync, &rdev->flags) && 1491 sync_page_io(rdev, 1492 sect + rdev->data_offset, 1493 s<<9, 1494 conf->tmppage, READ)) 1495 success = 1; 1496 else { 1497 d++; 1498 if (d == conf->raid_disks) 1499 d = 0; 1500 } 1501 } while (!success && d != read_disk); 1502 1503 if (!success) { 1504 /* Cannot read from anywhere -- bye bye array */ 1505 md_error(mddev, conf->mirrors[read_disk].rdev); 1506 break; 1507 } 1508 /* write it back and re-read */ 1509 start = d; 1510 while (d != read_disk) { 1511 if (d==0) 1512 d = conf->raid_disks; 1513 d--; 1514 rdev = conf->mirrors[d].rdev; 1515 if (rdev && 1516 test_bit(In_sync, &rdev->flags)) { 1517 if (sync_page_io(rdev, 1518 sect + rdev->data_offset, 1519 s<<9, conf->tmppage, WRITE) 1520 == 0) 1521 /* Well, this device is dead */ 1522 md_error(mddev, rdev); 1523 } 1524 } 1525 d = start; 1526 while (d != read_disk) { 1527 char b[BDEVNAME_SIZE]; 1528 if (d==0) 1529 d = conf->raid_disks; 1530 d--; 1531 rdev = conf->mirrors[d].rdev; 1532 if (rdev && 1533 test_bit(In_sync, &rdev->flags)) { 1534 if (sync_page_io(rdev, 1535 sect + rdev->data_offset, 1536 s<<9, conf->tmppage, READ) 1537 == 0) 1538 /* Well, this device is dead */ 1539 md_error(mddev, rdev); 1540 else { 1541 atomic_add(s, &rdev->corrected_errors); 1542 printk(KERN_INFO 1543 "md/raid1:%s: read error corrected " 1544 "(%d sectors at %llu on %s)\n", 1545 mdname(mddev), s, 1546 (unsigned long long)(sect + 1547 rdev->data_offset), 1548 bdevname(rdev->bdev, b)); 1549 } 1550 } 1551 } 1552 sectors -= s; 1553 sect += s; 1554 } 1555 } 1556 1557 static void raid1d(mddev_t *mddev) 1558 { 1559 r1bio_t *r1_bio; 1560 struct bio *bio; 1561 unsigned long flags; 1562 conf_t *conf = mddev->private; 1563 struct list_head *head = &conf->retry_list; 1564 int unplug=0; 1565 mdk_rdev_t *rdev; 1566 1567 md_check_recovery(mddev); 1568 1569 for (;;) { 1570 char b[BDEVNAME_SIZE]; 1571 1572 unplug += flush_pending_writes(conf); 1573 1574 spin_lock_irqsave(&conf->device_lock, flags); 1575 if (list_empty(head)) { 1576 spin_unlock_irqrestore(&conf->device_lock, flags); 1577 break; 1578 } 1579 r1_bio = list_entry(head->prev, r1bio_t, retry_list); 1580 list_del(head->prev); 1581 conf->nr_queued--; 1582 spin_unlock_irqrestore(&conf->device_lock, flags); 1583 1584 mddev = r1_bio->mddev; 1585 conf = mddev->private; 1586 if (test_bit(R1BIO_IsSync, &r1_bio->state)) { 1587 sync_request_write(mddev, r1_bio); 1588 unplug = 1; 1589 } else { 1590 int disk; 1591 1592 /* we got a read error. Maybe the drive is bad. Maybe just 1593 * the block and we can fix it. 1594 * We freeze all other IO, and try reading the block from 1595 * other devices. When we find one, we re-write 1596 * and check it that fixes the read error. 1597 * This is all done synchronously while the array is 1598 * frozen 1599 */ 1600 if (mddev->ro == 0) { 1601 freeze_array(conf); 1602 fix_read_error(conf, r1_bio->read_disk, 1603 r1_bio->sector, 1604 r1_bio->sectors); 1605 unfreeze_array(conf); 1606 } else 1607 md_error(mddev, 1608 conf->mirrors[r1_bio->read_disk].rdev); 1609 1610 bio = r1_bio->bios[r1_bio->read_disk]; 1611 if ((disk=read_balance(conf, r1_bio)) == -1) { 1612 printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O" 1613 " read error for block %llu\n", 1614 mdname(mddev), 1615 bdevname(bio->bi_bdev,b), 1616 (unsigned long long)r1_bio->sector); 1617 raid_end_bio_io(r1_bio); 1618 } else { 1619 const unsigned long do_sync = r1_bio->master_bio->bi_rw & REQ_SYNC; 1620 r1_bio->bios[r1_bio->read_disk] = 1621 mddev->ro ? IO_BLOCKED : NULL; 1622 r1_bio->read_disk = disk; 1623 bio_put(bio); 1624 bio = bio_clone_mddev(r1_bio->master_bio, 1625 GFP_NOIO, mddev); 1626 r1_bio->bios[r1_bio->read_disk] = bio; 1627 rdev = conf->mirrors[disk].rdev; 1628 if (printk_ratelimit()) 1629 printk(KERN_ERR "md/raid1:%s: redirecting sector %llu to" 1630 " other mirror: %s\n", 1631 mdname(mddev), 1632 (unsigned long long)r1_bio->sector, 1633 bdevname(rdev->bdev,b)); 1634 bio->bi_sector = r1_bio->sector + rdev->data_offset; 1635 bio->bi_bdev = rdev->bdev; 1636 bio->bi_end_io = raid1_end_read_request; 1637 bio->bi_rw = READ | do_sync; 1638 bio->bi_private = r1_bio; 1639 unplug = 1; 1640 generic_make_request(bio); 1641 } 1642 } 1643 cond_resched(); 1644 } 1645 if (unplug) 1646 unplug_slaves(mddev); 1647 } 1648 1649 1650 static int init_resync(conf_t *conf) 1651 { 1652 int buffs; 1653 1654 buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE; 1655 BUG_ON(conf->r1buf_pool); 1656 conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free, 1657 conf->poolinfo); 1658 if (!conf->r1buf_pool) 1659 return -ENOMEM; 1660 conf->next_resync = 0; 1661 return 0; 1662 } 1663 1664 /* 1665 * perform a "sync" on one "block" 1666 * 1667 * We need to make sure that no normal I/O request - particularly write 1668 * requests - conflict with active sync requests. 1669 * 1670 * This is achieved by tracking pending requests and a 'barrier' concept 1671 * that can be installed to exclude normal IO requests. 1672 */ 1673 1674 static sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster) 1675 { 1676 conf_t *conf = mddev->private; 1677 r1bio_t *r1_bio; 1678 struct bio *bio; 1679 sector_t max_sector, nr_sectors; 1680 int disk = -1; 1681 int i; 1682 int wonly = -1; 1683 int write_targets = 0, read_targets = 0; 1684 sector_t sync_blocks; 1685 int still_degraded = 0; 1686 1687 if (!conf->r1buf_pool) 1688 if (init_resync(conf)) 1689 return 0; 1690 1691 max_sector = mddev->dev_sectors; 1692 if (sector_nr >= max_sector) { 1693 /* If we aborted, we need to abort the 1694 * sync on the 'current' bitmap chunk (there will 1695 * only be one in raid1 resync. 1696 * We can find the current addess in mddev->curr_resync 1697 */ 1698 if (mddev->curr_resync < max_sector) /* aborted */ 1699 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 1700 &sync_blocks, 1); 1701 else /* completed sync */ 1702 conf->fullsync = 0; 1703 1704 bitmap_close_sync(mddev->bitmap); 1705 close_sync(conf); 1706 return 0; 1707 } 1708 1709 if (mddev->bitmap == NULL && 1710 mddev->recovery_cp == MaxSector && 1711 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 1712 conf->fullsync == 0) { 1713 *skipped = 1; 1714 return max_sector - sector_nr; 1715 } 1716 /* before building a request, check if we can skip these blocks.. 1717 * This call the bitmap_start_sync doesn't actually record anything 1718 */ 1719 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 1720 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 1721 /* We can skip this block, and probably several more */ 1722 *skipped = 1; 1723 return sync_blocks; 1724 } 1725 /* 1726 * If there is non-resync activity waiting for a turn, 1727 * and resync is going fast enough, 1728 * then let it though before starting on this new sync request. 1729 */ 1730 if (!go_faster && conf->nr_waiting) 1731 msleep_interruptible(1000); 1732 1733 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 1734 r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO); 1735 raise_barrier(conf); 1736 1737 conf->next_resync = sector_nr; 1738 1739 rcu_read_lock(); 1740 /* 1741 * If we get a correctably read error during resync or recovery, 1742 * we might want to read from a different device. So we 1743 * flag all drives that could conceivably be read from for READ, 1744 * and any others (which will be non-In_sync devices) for WRITE. 1745 * If a read fails, we try reading from something else for which READ 1746 * is OK. 1747 */ 1748 1749 r1_bio->mddev = mddev; 1750 r1_bio->sector = sector_nr; 1751 r1_bio->state = 0; 1752 set_bit(R1BIO_IsSync, &r1_bio->state); 1753 1754 for (i=0; i < conf->raid_disks; i++) { 1755 mdk_rdev_t *rdev; 1756 bio = r1_bio->bios[i]; 1757 1758 /* take from bio_init */ 1759 bio->bi_next = NULL; 1760 bio->bi_flags &= ~(BIO_POOL_MASK-1); 1761 bio->bi_flags |= 1 << BIO_UPTODATE; 1762 bio->bi_comp_cpu = -1; 1763 bio->bi_rw = READ; 1764 bio->bi_vcnt = 0; 1765 bio->bi_idx = 0; 1766 bio->bi_phys_segments = 0; 1767 bio->bi_size = 0; 1768 bio->bi_end_io = NULL; 1769 bio->bi_private = NULL; 1770 1771 rdev = rcu_dereference(conf->mirrors[i].rdev); 1772 if (rdev == NULL || 1773 test_bit(Faulty, &rdev->flags)) { 1774 still_degraded = 1; 1775 continue; 1776 } else if (!test_bit(In_sync, &rdev->flags)) { 1777 bio->bi_rw = WRITE; 1778 bio->bi_end_io = end_sync_write; 1779 write_targets ++; 1780 } else { 1781 /* may need to read from here */ 1782 bio->bi_rw = READ; 1783 bio->bi_end_io = end_sync_read; 1784 if (test_bit(WriteMostly, &rdev->flags)) { 1785 if (wonly < 0) 1786 wonly = i; 1787 } else { 1788 if (disk < 0) 1789 disk = i; 1790 } 1791 read_targets++; 1792 } 1793 atomic_inc(&rdev->nr_pending); 1794 bio->bi_sector = sector_nr + rdev->data_offset; 1795 bio->bi_bdev = rdev->bdev; 1796 bio->bi_private = r1_bio; 1797 } 1798 rcu_read_unlock(); 1799 if (disk < 0) 1800 disk = wonly; 1801 r1_bio->read_disk = disk; 1802 1803 if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0) 1804 /* extra read targets are also write targets */ 1805 write_targets += read_targets-1; 1806 1807 if (write_targets == 0 || read_targets == 0) { 1808 /* There is nowhere to write, so all non-sync 1809 * drives must be failed - so we are finished 1810 */ 1811 sector_t rv = max_sector - sector_nr; 1812 *skipped = 1; 1813 put_buf(r1_bio); 1814 return rv; 1815 } 1816 1817 if (max_sector > mddev->resync_max) 1818 max_sector = mddev->resync_max; /* Don't do IO beyond here */ 1819 nr_sectors = 0; 1820 sync_blocks = 0; 1821 do { 1822 struct page *page; 1823 int len = PAGE_SIZE; 1824 if (sector_nr + (len>>9) > max_sector) 1825 len = (max_sector - sector_nr) << 9; 1826 if (len == 0) 1827 break; 1828 if (sync_blocks == 0) { 1829 if (!bitmap_start_sync(mddev->bitmap, sector_nr, 1830 &sync_blocks, still_degraded) && 1831 !conf->fullsync && 1832 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) 1833 break; 1834 BUG_ON(sync_blocks < (PAGE_SIZE>>9)); 1835 if ((len >> 9) > sync_blocks) 1836 len = sync_blocks<<9; 1837 } 1838 1839 for (i=0 ; i < conf->raid_disks; i++) { 1840 bio = r1_bio->bios[i]; 1841 if (bio->bi_end_io) { 1842 page = bio->bi_io_vec[bio->bi_vcnt].bv_page; 1843 if (bio_add_page(bio, page, len, 0) == 0) { 1844 /* stop here */ 1845 bio->bi_io_vec[bio->bi_vcnt].bv_page = page; 1846 while (i > 0) { 1847 i--; 1848 bio = r1_bio->bios[i]; 1849 if (bio->bi_end_io==NULL) 1850 continue; 1851 /* remove last page from this bio */ 1852 bio->bi_vcnt--; 1853 bio->bi_size -= len; 1854 bio->bi_flags &= ~(1<< BIO_SEG_VALID); 1855 } 1856 goto bio_full; 1857 } 1858 } 1859 } 1860 nr_sectors += len>>9; 1861 sector_nr += len>>9; 1862 sync_blocks -= (len>>9); 1863 } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES); 1864 bio_full: 1865 r1_bio->sectors = nr_sectors; 1866 1867 /* For a user-requested sync, we read all readable devices and do a 1868 * compare 1869 */ 1870 if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { 1871 atomic_set(&r1_bio->remaining, read_targets); 1872 for (i=0; i<conf->raid_disks; i++) { 1873 bio = r1_bio->bios[i]; 1874 if (bio->bi_end_io == end_sync_read) { 1875 md_sync_acct(bio->bi_bdev, nr_sectors); 1876 generic_make_request(bio); 1877 } 1878 } 1879 } else { 1880 atomic_set(&r1_bio->remaining, 1); 1881 bio = r1_bio->bios[r1_bio->read_disk]; 1882 md_sync_acct(bio->bi_bdev, nr_sectors); 1883 generic_make_request(bio); 1884 1885 } 1886 return nr_sectors; 1887 } 1888 1889 static sector_t raid1_size(mddev_t *mddev, sector_t sectors, int raid_disks) 1890 { 1891 if (sectors) 1892 return sectors; 1893 1894 return mddev->dev_sectors; 1895 } 1896 1897 static conf_t *setup_conf(mddev_t *mddev) 1898 { 1899 conf_t *conf; 1900 int i; 1901 mirror_info_t *disk; 1902 mdk_rdev_t *rdev; 1903 int err = -ENOMEM; 1904 1905 conf = kzalloc(sizeof(conf_t), GFP_KERNEL); 1906 if (!conf) 1907 goto abort; 1908 1909 conf->mirrors = kzalloc(sizeof(struct mirror_info)*mddev->raid_disks, 1910 GFP_KERNEL); 1911 if (!conf->mirrors) 1912 goto abort; 1913 1914 conf->tmppage = alloc_page(GFP_KERNEL); 1915 if (!conf->tmppage) 1916 goto abort; 1917 1918 conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL); 1919 if (!conf->poolinfo) 1920 goto abort; 1921 conf->poolinfo->raid_disks = mddev->raid_disks; 1922 conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc, 1923 r1bio_pool_free, 1924 conf->poolinfo); 1925 if (!conf->r1bio_pool) 1926 goto abort; 1927 1928 conf->poolinfo->mddev = mddev; 1929 1930 spin_lock_init(&conf->device_lock); 1931 list_for_each_entry(rdev, &mddev->disks, same_set) { 1932 int disk_idx = rdev->raid_disk; 1933 if (disk_idx >= mddev->raid_disks 1934 || disk_idx < 0) 1935 continue; 1936 disk = conf->mirrors + disk_idx; 1937 1938 disk->rdev = rdev; 1939 1940 disk->head_position = 0; 1941 } 1942 conf->raid_disks = mddev->raid_disks; 1943 conf->mddev = mddev; 1944 INIT_LIST_HEAD(&conf->retry_list); 1945 1946 spin_lock_init(&conf->resync_lock); 1947 init_waitqueue_head(&conf->wait_barrier); 1948 1949 bio_list_init(&conf->pending_bio_list); 1950 1951 conf->last_used = -1; 1952 for (i = 0; i < conf->raid_disks; i++) { 1953 1954 disk = conf->mirrors + i; 1955 1956 if (!disk->rdev || 1957 !test_bit(In_sync, &disk->rdev->flags)) { 1958 disk->head_position = 0; 1959 if (disk->rdev) 1960 conf->fullsync = 1; 1961 } else if (conf->last_used < 0) 1962 /* 1963 * The first working device is used as a 1964 * starting point to read balancing. 1965 */ 1966 conf->last_used = i; 1967 } 1968 1969 err = -EIO; 1970 if (conf->last_used < 0) { 1971 printk(KERN_ERR "md/raid1:%s: no operational mirrors\n", 1972 mdname(mddev)); 1973 goto abort; 1974 } 1975 err = -ENOMEM; 1976 conf->thread = md_register_thread(raid1d, mddev, NULL); 1977 if (!conf->thread) { 1978 printk(KERN_ERR 1979 "md/raid1:%s: couldn't allocate thread\n", 1980 mdname(mddev)); 1981 goto abort; 1982 } 1983 1984 return conf; 1985 1986 abort: 1987 if (conf) { 1988 if (conf->r1bio_pool) 1989 mempool_destroy(conf->r1bio_pool); 1990 kfree(conf->mirrors); 1991 safe_put_page(conf->tmppage); 1992 kfree(conf->poolinfo); 1993 kfree(conf); 1994 } 1995 return ERR_PTR(err); 1996 } 1997 1998 static int run(mddev_t *mddev) 1999 { 2000 conf_t *conf; 2001 int i; 2002 mdk_rdev_t *rdev; 2003 2004 if (mddev->level != 1) { 2005 printk(KERN_ERR "md/raid1:%s: raid level not set to mirroring (%d)\n", 2006 mdname(mddev), mddev->level); 2007 return -EIO; 2008 } 2009 if (mddev->reshape_position != MaxSector) { 2010 printk(KERN_ERR "md/raid1:%s: reshape_position set but not supported\n", 2011 mdname(mddev)); 2012 return -EIO; 2013 } 2014 /* 2015 * copy the already verified devices into our private RAID1 2016 * bookkeeping area. [whatever we allocate in run(), 2017 * should be freed in stop()] 2018 */ 2019 if (mddev->private == NULL) 2020 conf = setup_conf(mddev); 2021 else 2022 conf = mddev->private; 2023 2024 if (IS_ERR(conf)) 2025 return PTR_ERR(conf); 2026 2027 mddev->queue->queue_lock = &conf->device_lock; 2028 list_for_each_entry(rdev, &mddev->disks, same_set) { 2029 disk_stack_limits(mddev->gendisk, rdev->bdev, 2030 rdev->data_offset << 9); 2031 /* as we don't honour merge_bvec_fn, we must never risk 2032 * violating it, so limit ->max_segments to 1 lying within 2033 * a single page, as a one page request is never in violation. 2034 */ 2035 if (rdev->bdev->bd_disk->queue->merge_bvec_fn) { 2036 blk_queue_max_segments(mddev->queue, 1); 2037 blk_queue_segment_boundary(mddev->queue, 2038 PAGE_CACHE_SIZE - 1); 2039 } 2040 } 2041 2042 mddev->degraded = 0; 2043 for (i=0; i < conf->raid_disks; i++) 2044 if (conf->mirrors[i].rdev == NULL || 2045 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) || 2046 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) 2047 mddev->degraded++; 2048 2049 if (conf->raid_disks - mddev->degraded == 1) 2050 mddev->recovery_cp = MaxSector; 2051 2052 if (mddev->recovery_cp != MaxSector) 2053 printk(KERN_NOTICE "md/raid1:%s: not clean" 2054 " -- starting background reconstruction\n", 2055 mdname(mddev)); 2056 printk(KERN_INFO 2057 "md/raid1:%s: active with %d out of %d mirrors\n", 2058 mdname(mddev), mddev->raid_disks - mddev->degraded, 2059 mddev->raid_disks); 2060 2061 /* 2062 * Ok, everything is just fine now 2063 */ 2064 mddev->thread = conf->thread; 2065 conf->thread = NULL; 2066 mddev->private = conf; 2067 2068 md_set_array_sectors(mddev, raid1_size(mddev, 0, 0)); 2069 2070 mddev->queue->unplug_fn = raid1_unplug; 2071 mddev->queue->backing_dev_info.congested_fn = raid1_congested; 2072 mddev->queue->backing_dev_info.congested_data = mddev; 2073 md_integrity_register(mddev); 2074 return 0; 2075 } 2076 2077 static int stop(mddev_t *mddev) 2078 { 2079 conf_t *conf = mddev->private; 2080 struct bitmap *bitmap = mddev->bitmap; 2081 2082 /* wait for behind writes to complete */ 2083 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { 2084 printk(KERN_INFO "md/raid1:%s: behind writes in progress - waiting to stop.\n", 2085 mdname(mddev)); 2086 /* need to kick something here to make sure I/O goes? */ 2087 wait_event(bitmap->behind_wait, 2088 atomic_read(&bitmap->behind_writes) == 0); 2089 } 2090 2091 raise_barrier(conf); 2092 lower_barrier(conf); 2093 2094 md_unregister_thread(mddev->thread); 2095 mddev->thread = NULL; 2096 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/ 2097 if (conf->r1bio_pool) 2098 mempool_destroy(conf->r1bio_pool); 2099 kfree(conf->mirrors); 2100 kfree(conf->poolinfo); 2101 kfree(conf); 2102 mddev->private = NULL; 2103 return 0; 2104 } 2105 2106 static int raid1_resize(mddev_t *mddev, sector_t sectors) 2107 { 2108 /* no resync is happening, and there is enough space 2109 * on all devices, so we can resize. 2110 * We need to make sure resync covers any new space. 2111 * If the array is shrinking we should possibly wait until 2112 * any io in the removed space completes, but it hardly seems 2113 * worth it. 2114 */ 2115 md_set_array_sectors(mddev, raid1_size(mddev, sectors, 0)); 2116 if (mddev->array_sectors > raid1_size(mddev, sectors, 0)) 2117 return -EINVAL; 2118 set_capacity(mddev->gendisk, mddev->array_sectors); 2119 revalidate_disk(mddev->gendisk); 2120 if (sectors > mddev->dev_sectors && 2121 mddev->recovery_cp == MaxSector) { 2122 mddev->recovery_cp = mddev->dev_sectors; 2123 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 2124 } 2125 mddev->dev_sectors = sectors; 2126 mddev->resync_max_sectors = sectors; 2127 return 0; 2128 } 2129 2130 static int raid1_reshape(mddev_t *mddev) 2131 { 2132 /* We need to: 2133 * 1/ resize the r1bio_pool 2134 * 2/ resize conf->mirrors 2135 * 2136 * We allocate a new r1bio_pool if we can. 2137 * Then raise a device barrier and wait until all IO stops. 2138 * Then resize conf->mirrors and swap in the new r1bio pool. 2139 * 2140 * At the same time, we "pack" the devices so that all the missing 2141 * devices have the higher raid_disk numbers. 2142 */ 2143 mempool_t *newpool, *oldpool; 2144 struct pool_info *newpoolinfo; 2145 mirror_info_t *newmirrors; 2146 conf_t *conf = mddev->private; 2147 int cnt, raid_disks; 2148 unsigned long flags; 2149 int d, d2, err; 2150 2151 /* Cannot change chunk_size, layout, or level */ 2152 if (mddev->chunk_sectors != mddev->new_chunk_sectors || 2153 mddev->layout != mddev->new_layout || 2154 mddev->level != mddev->new_level) { 2155 mddev->new_chunk_sectors = mddev->chunk_sectors; 2156 mddev->new_layout = mddev->layout; 2157 mddev->new_level = mddev->level; 2158 return -EINVAL; 2159 } 2160 2161 err = md_allow_write(mddev); 2162 if (err) 2163 return err; 2164 2165 raid_disks = mddev->raid_disks + mddev->delta_disks; 2166 2167 if (raid_disks < conf->raid_disks) { 2168 cnt=0; 2169 for (d= 0; d < conf->raid_disks; d++) 2170 if (conf->mirrors[d].rdev) 2171 cnt++; 2172 if (cnt > raid_disks) 2173 return -EBUSY; 2174 } 2175 2176 newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL); 2177 if (!newpoolinfo) 2178 return -ENOMEM; 2179 newpoolinfo->mddev = mddev; 2180 newpoolinfo->raid_disks = raid_disks; 2181 2182 newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc, 2183 r1bio_pool_free, newpoolinfo); 2184 if (!newpool) { 2185 kfree(newpoolinfo); 2186 return -ENOMEM; 2187 } 2188 newmirrors = kzalloc(sizeof(struct mirror_info) * raid_disks, GFP_KERNEL); 2189 if (!newmirrors) { 2190 kfree(newpoolinfo); 2191 mempool_destroy(newpool); 2192 return -ENOMEM; 2193 } 2194 2195 raise_barrier(conf); 2196 2197 /* ok, everything is stopped */ 2198 oldpool = conf->r1bio_pool; 2199 conf->r1bio_pool = newpool; 2200 2201 for (d = d2 = 0; d < conf->raid_disks; d++) { 2202 mdk_rdev_t *rdev = conf->mirrors[d].rdev; 2203 if (rdev && rdev->raid_disk != d2) { 2204 char nm[20]; 2205 sprintf(nm, "rd%d", rdev->raid_disk); 2206 sysfs_remove_link(&mddev->kobj, nm); 2207 rdev->raid_disk = d2; 2208 sprintf(nm, "rd%d", rdev->raid_disk); 2209 sysfs_remove_link(&mddev->kobj, nm); 2210 if (sysfs_create_link(&mddev->kobj, 2211 &rdev->kobj, nm)) 2212 printk(KERN_WARNING 2213 "md/raid1:%s: cannot register " 2214 "%s\n", 2215 mdname(mddev), nm); 2216 } 2217 if (rdev) 2218 newmirrors[d2++].rdev = rdev; 2219 } 2220 kfree(conf->mirrors); 2221 conf->mirrors = newmirrors; 2222 kfree(conf->poolinfo); 2223 conf->poolinfo = newpoolinfo; 2224 2225 spin_lock_irqsave(&conf->device_lock, flags); 2226 mddev->degraded += (raid_disks - conf->raid_disks); 2227 spin_unlock_irqrestore(&conf->device_lock, flags); 2228 conf->raid_disks = mddev->raid_disks = raid_disks; 2229 mddev->delta_disks = 0; 2230 2231 conf->last_used = 0; /* just make sure it is in-range */ 2232 lower_barrier(conf); 2233 2234 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 2235 md_wakeup_thread(mddev->thread); 2236 2237 mempool_destroy(oldpool); 2238 return 0; 2239 } 2240 2241 static void raid1_quiesce(mddev_t *mddev, int state) 2242 { 2243 conf_t *conf = mddev->private; 2244 2245 switch(state) { 2246 case 2: /* wake for suspend */ 2247 wake_up(&conf->wait_barrier); 2248 break; 2249 case 1: 2250 raise_barrier(conf); 2251 break; 2252 case 0: 2253 lower_barrier(conf); 2254 break; 2255 } 2256 } 2257 2258 static void *raid1_takeover(mddev_t *mddev) 2259 { 2260 /* raid1 can take over: 2261 * raid5 with 2 devices, any layout or chunk size 2262 */ 2263 if (mddev->level == 5 && mddev->raid_disks == 2) { 2264 conf_t *conf; 2265 mddev->new_level = 1; 2266 mddev->new_layout = 0; 2267 mddev->new_chunk_sectors = 0; 2268 conf = setup_conf(mddev); 2269 if (!IS_ERR(conf)) 2270 conf->barrier = 1; 2271 return conf; 2272 } 2273 return ERR_PTR(-EINVAL); 2274 } 2275 2276 static struct mdk_personality raid1_personality = 2277 { 2278 .name = "raid1", 2279 .level = 1, 2280 .owner = THIS_MODULE, 2281 .make_request = make_request, 2282 .run = run, 2283 .stop = stop, 2284 .status = status, 2285 .error_handler = error, 2286 .hot_add_disk = raid1_add_disk, 2287 .hot_remove_disk= raid1_remove_disk, 2288 .spare_active = raid1_spare_active, 2289 .sync_request = sync_request, 2290 .resize = raid1_resize, 2291 .size = raid1_size, 2292 .check_reshape = raid1_reshape, 2293 .quiesce = raid1_quiesce, 2294 .takeover = raid1_takeover, 2295 }; 2296 2297 static int __init raid_init(void) 2298 { 2299 return register_md_personality(&raid1_personality); 2300 } 2301 2302 static void raid_exit(void) 2303 { 2304 unregister_md_personality(&raid1_personality); 2305 } 2306 2307 module_init(raid_init); 2308 module_exit(raid_exit); 2309 MODULE_LICENSE("GPL"); 2310 MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD"); 2311 MODULE_ALIAS("md-personality-3"); /* RAID1 */ 2312 MODULE_ALIAS("md-raid1"); 2313 MODULE_ALIAS("md-level-1"); 2314