1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->bm_write is the number of the last batch successfully written. 31 * conf->bm_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is bm_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/async.h> 51 #include <linux/seq_file.h> 52 #include <linux/cpu.h> 53 #include <linux/slab.h> 54 #include "md.h" 55 #include "raid5.h" 56 #include "raid0.h" 57 #include "bitmap.h" 58 59 /* 60 * Stripe cache 61 */ 62 63 #define NR_STRIPES 256 64 #define STRIPE_SIZE PAGE_SIZE 65 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 66 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 67 #define IO_THRESHOLD 1 68 #define BYPASS_THRESHOLD 1 69 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 70 #define HASH_MASK (NR_HASH - 1) 71 72 #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) 73 74 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 75 * order without overlap. There may be several bio's per stripe+device, and 76 * a bio could span several devices. 77 * When walking this list for a particular stripe+device, we must never proceed 78 * beyond a bio that extends past this device, as the next bio might no longer 79 * be valid. 80 * This macro is used to determine the 'next' bio in the list, given the sector 81 * of the current stripe+device 82 */ 83 #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL) 84 /* 85 * The following can be used to debug the driver 86 */ 87 #define RAID5_PARANOIA 1 88 #if RAID5_PARANOIA && defined(CONFIG_SMP) 89 # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock) 90 #else 91 # define CHECK_DEVLOCK() 92 #endif 93 94 #ifdef DEBUG 95 #define inline 96 #define __inline__ 97 #endif 98 99 #define printk_rl(args...) ((void) (printk_ratelimit() && printk(args))) 100 101 /* 102 * We maintain a biased count of active stripes in the bottom 16 bits of 103 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 104 */ 105 static inline int raid5_bi_phys_segments(struct bio *bio) 106 { 107 return bio->bi_phys_segments & 0xffff; 108 } 109 110 static inline int raid5_bi_hw_segments(struct bio *bio) 111 { 112 return (bio->bi_phys_segments >> 16) & 0xffff; 113 } 114 115 static inline int raid5_dec_bi_phys_segments(struct bio *bio) 116 { 117 --bio->bi_phys_segments; 118 return raid5_bi_phys_segments(bio); 119 } 120 121 static inline int raid5_dec_bi_hw_segments(struct bio *bio) 122 { 123 unsigned short val = raid5_bi_hw_segments(bio); 124 125 --val; 126 bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio); 127 return val; 128 } 129 130 static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt) 131 { 132 bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16); 133 } 134 135 /* Find first data disk in a raid6 stripe */ 136 static inline int raid6_d0(struct stripe_head *sh) 137 { 138 if (sh->ddf_layout) 139 /* ddf always start from first device */ 140 return 0; 141 /* md starts just after Q block */ 142 if (sh->qd_idx == sh->disks - 1) 143 return 0; 144 else 145 return sh->qd_idx + 1; 146 } 147 static inline int raid6_next_disk(int disk, int raid_disks) 148 { 149 disk++; 150 return (disk < raid_disks) ? disk : 0; 151 } 152 153 /* When walking through the disks in a raid5, starting at raid6_d0, 154 * We need to map each disk to a 'slot', where the data disks are slot 155 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 156 * is raid_disks-1. This help does that mapping. 157 */ 158 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 159 int *count, int syndrome_disks) 160 { 161 int slot = *count; 162 163 if (sh->ddf_layout) 164 (*count)++; 165 if (idx == sh->pd_idx) 166 return syndrome_disks; 167 if (idx == sh->qd_idx) 168 return syndrome_disks + 1; 169 if (!sh->ddf_layout) 170 (*count)++; 171 return slot; 172 } 173 174 static void return_io(struct bio *return_bi) 175 { 176 struct bio *bi = return_bi; 177 while (bi) { 178 179 return_bi = bi->bi_next; 180 bi->bi_next = NULL; 181 bi->bi_size = 0; 182 bio_endio(bi, 0); 183 bi = return_bi; 184 } 185 } 186 187 static void print_raid5_conf (raid5_conf_t *conf); 188 189 static int stripe_operations_active(struct stripe_head *sh) 190 { 191 return sh->check_state || sh->reconstruct_state || 192 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 193 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 194 } 195 196 static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh) 197 { 198 if (atomic_dec_and_test(&sh->count)) { 199 BUG_ON(!list_empty(&sh->lru)); 200 BUG_ON(atomic_read(&conf->active_stripes)==0); 201 if (test_bit(STRIPE_HANDLE, &sh->state)) { 202 if (test_bit(STRIPE_DELAYED, &sh->state)) { 203 list_add_tail(&sh->lru, &conf->delayed_list); 204 plugger_set_plug(&conf->plug); 205 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 206 sh->bm_seq - conf->seq_write > 0) { 207 list_add_tail(&sh->lru, &conf->bitmap_list); 208 plugger_set_plug(&conf->plug); 209 } else { 210 clear_bit(STRIPE_BIT_DELAY, &sh->state); 211 list_add_tail(&sh->lru, &conf->handle_list); 212 } 213 md_wakeup_thread(conf->mddev->thread); 214 } else { 215 BUG_ON(stripe_operations_active(sh)); 216 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 217 atomic_dec(&conf->preread_active_stripes); 218 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) 219 md_wakeup_thread(conf->mddev->thread); 220 } 221 atomic_dec(&conf->active_stripes); 222 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 223 list_add_tail(&sh->lru, &conf->inactive_list); 224 wake_up(&conf->wait_for_stripe); 225 if (conf->retry_read_aligned) 226 md_wakeup_thread(conf->mddev->thread); 227 } 228 } 229 } 230 } 231 232 static void release_stripe(struct stripe_head *sh) 233 { 234 raid5_conf_t *conf = sh->raid_conf; 235 unsigned long flags; 236 237 spin_lock_irqsave(&conf->device_lock, flags); 238 __release_stripe(conf, sh); 239 spin_unlock_irqrestore(&conf->device_lock, flags); 240 } 241 242 static inline void remove_hash(struct stripe_head *sh) 243 { 244 pr_debug("remove_hash(), stripe %llu\n", 245 (unsigned long long)sh->sector); 246 247 hlist_del_init(&sh->hash); 248 } 249 250 static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh) 251 { 252 struct hlist_head *hp = stripe_hash(conf, sh->sector); 253 254 pr_debug("insert_hash(), stripe %llu\n", 255 (unsigned long long)sh->sector); 256 257 CHECK_DEVLOCK(); 258 hlist_add_head(&sh->hash, hp); 259 } 260 261 262 /* find an idle stripe, make sure it is unhashed, and return it. */ 263 static struct stripe_head *get_free_stripe(raid5_conf_t *conf) 264 { 265 struct stripe_head *sh = NULL; 266 struct list_head *first; 267 268 CHECK_DEVLOCK(); 269 if (list_empty(&conf->inactive_list)) 270 goto out; 271 first = conf->inactive_list.next; 272 sh = list_entry(first, struct stripe_head, lru); 273 list_del_init(first); 274 remove_hash(sh); 275 atomic_inc(&conf->active_stripes); 276 out: 277 return sh; 278 } 279 280 static void shrink_buffers(struct stripe_head *sh) 281 { 282 struct page *p; 283 int i; 284 int num = sh->raid_conf->pool_size; 285 286 for (i = 0; i < num ; i++) { 287 p = sh->dev[i].page; 288 if (!p) 289 continue; 290 sh->dev[i].page = NULL; 291 put_page(p); 292 } 293 } 294 295 static int grow_buffers(struct stripe_head *sh) 296 { 297 int i; 298 int num = sh->raid_conf->pool_size; 299 300 for (i = 0; i < num; i++) { 301 struct page *page; 302 303 if (!(page = alloc_page(GFP_KERNEL))) { 304 return 1; 305 } 306 sh->dev[i].page = page; 307 } 308 return 0; 309 } 310 311 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 312 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 313 struct stripe_head *sh); 314 315 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 316 { 317 raid5_conf_t *conf = sh->raid_conf; 318 int i; 319 320 BUG_ON(atomic_read(&sh->count) != 0); 321 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 322 BUG_ON(stripe_operations_active(sh)); 323 324 CHECK_DEVLOCK(); 325 pr_debug("init_stripe called, stripe %llu\n", 326 (unsigned long long)sh->sector); 327 328 remove_hash(sh); 329 330 sh->generation = conf->generation - previous; 331 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 332 sh->sector = sector; 333 stripe_set_idx(sector, conf, previous, sh); 334 sh->state = 0; 335 336 337 for (i = sh->disks; i--; ) { 338 struct r5dev *dev = &sh->dev[i]; 339 340 if (dev->toread || dev->read || dev->towrite || dev->written || 341 test_bit(R5_LOCKED, &dev->flags)) { 342 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 343 (unsigned long long)sh->sector, i, dev->toread, 344 dev->read, dev->towrite, dev->written, 345 test_bit(R5_LOCKED, &dev->flags)); 346 BUG(); 347 } 348 dev->flags = 0; 349 raid5_build_block(sh, i, previous); 350 } 351 insert_hash(conf, sh); 352 } 353 354 static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, 355 short generation) 356 { 357 struct stripe_head *sh; 358 struct hlist_node *hn; 359 360 CHECK_DEVLOCK(); 361 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 362 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) 363 if (sh->sector == sector && sh->generation == generation) 364 return sh; 365 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 366 return NULL; 367 } 368 369 /* 370 * Need to check if array has failed when deciding whether to: 371 * - start an array 372 * - remove non-faulty devices 373 * - add a spare 374 * - allow a reshape 375 * This determination is simple when no reshape is happening. 376 * However if there is a reshape, we need to carefully check 377 * both the before and after sections. 378 * This is because some failed devices may only affect one 379 * of the two sections, and some non-in_sync devices may 380 * be insync in the section most affected by failed devices. 381 */ 382 static int has_failed(raid5_conf_t *conf) 383 { 384 int degraded; 385 int i; 386 if (conf->mddev->reshape_position == MaxSector) 387 return conf->mddev->degraded > conf->max_degraded; 388 389 rcu_read_lock(); 390 degraded = 0; 391 for (i = 0; i < conf->previous_raid_disks; i++) { 392 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 393 if (!rdev || test_bit(Faulty, &rdev->flags)) 394 degraded++; 395 else if (test_bit(In_sync, &rdev->flags)) 396 ; 397 else 398 /* not in-sync or faulty. 399 * If the reshape increases the number of devices, 400 * this is being recovered by the reshape, so 401 * this 'previous' section is not in_sync. 402 * If the number of devices is being reduced however, 403 * the device can only be part of the array if 404 * we are reverting a reshape, so this section will 405 * be in-sync. 406 */ 407 if (conf->raid_disks >= conf->previous_raid_disks) 408 degraded++; 409 } 410 rcu_read_unlock(); 411 if (degraded > conf->max_degraded) 412 return 1; 413 rcu_read_lock(); 414 degraded = 0; 415 for (i = 0; i < conf->raid_disks; i++) { 416 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 417 if (!rdev || test_bit(Faulty, &rdev->flags)) 418 degraded++; 419 else if (test_bit(In_sync, &rdev->flags)) 420 ; 421 else 422 /* not in-sync or faulty. 423 * If reshape increases the number of devices, this 424 * section has already been recovered, else it 425 * almost certainly hasn't. 426 */ 427 if (conf->raid_disks <= conf->previous_raid_disks) 428 degraded++; 429 } 430 rcu_read_unlock(); 431 if (degraded > conf->max_degraded) 432 return 1; 433 return 0; 434 } 435 436 static void unplug_slaves(mddev_t *mddev); 437 438 static struct stripe_head * 439 get_active_stripe(raid5_conf_t *conf, sector_t sector, 440 int previous, int noblock, int noquiesce) 441 { 442 struct stripe_head *sh; 443 444 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 445 446 spin_lock_irq(&conf->device_lock); 447 448 do { 449 wait_event_lock_irq(conf->wait_for_stripe, 450 conf->quiesce == 0 || noquiesce, 451 conf->device_lock, /* nothing */); 452 sh = __find_stripe(conf, sector, conf->generation - previous); 453 if (!sh) { 454 if (!conf->inactive_blocked) 455 sh = get_free_stripe(conf); 456 if (noblock && sh == NULL) 457 break; 458 if (!sh) { 459 conf->inactive_blocked = 1; 460 wait_event_lock_irq(conf->wait_for_stripe, 461 !list_empty(&conf->inactive_list) && 462 (atomic_read(&conf->active_stripes) 463 < (conf->max_nr_stripes *3/4) 464 || !conf->inactive_blocked), 465 conf->device_lock, 466 md_raid5_unplug_device(conf) 467 ); 468 conf->inactive_blocked = 0; 469 } else 470 init_stripe(sh, sector, previous); 471 } else { 472 if (atomic_read(&sh->count)) { 473 BUG_ON(!list_empty(&sh->lru) 474 && !test_bit(STRIPE_EXPANDING, &sh->state)); 475 } else { 476 if (!test_bit(STRIPE_HANDLE, &sh->state)) 477 atomic_inc(&conf->active_stripes); 478 if (list_empty(&sh->lru) && 479 !test_bit(STRIPE_EXPANDING, &sh->state)) 480 BUG(); 481 list_del_init(&sh->lru); 482 } 483 } 484 } while (sh == NULL); 485 486 if (sh) 487 atomic_inc(&sh->count); 488 489 spin_unlock_irq(&conf->device_lock); 490 return sh; 491 } 492 493 static void 494 raid5_end_read_request(struct bio *bi, int error); 495 static void 496 raid5_end_write_request(struct bio *bi, int error); 497 498 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 499 { 500 raid5_conf_t *conf = sh->raid_conf; 501 int i, disks = sh->disks; 502 503 might_sleep(); 504 505 for (i = disks; i--; ) { 506 int rw; 507 struct bio *bi; 508 mdk_rdev_t *rdev; 509 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 510 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 511 rw = WRITE_FUA; 512 else 513 rw = WRITE; 514 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 515 rw = READ; 516 else 517 continue; 518 519 bi = &sh->dev[i].req; 520 521 bi->bi_rw = rw; 522 if (rw == WRITE) 523 bi->bi_end_io = raid5_end_write_request; 524 else 525 bi->bi_end_io = raid5_end_read_request; 526 527 rcu_read_lock(); 528 rdev = rcu_dereference(conf->disks[i].rdev); 529 if (rdev && test_bit(Faulty, &rdev->flags)) 530 rdev = NULL; 531 if (rdev) 532 atomic_inc(&rdev->nr_pending); 533 rcu_read_unlock(); 534 535 if (rdev) { 536 if (s->syncing || s->expanding || s->expanded) 537 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 538 539 set_bit(STRIPE_IO_STARTED, &sh->state); 540 541 bi->bi_bdev = rdev->bdev; 542 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 543 __func__, (unsigned long long)sh->sector, 544 bi->bi_rw, i); 545 atomic_inc(&sh->count); 546 bi->bi_sector = sh->sector + rdev->data_offset; 547 bi->bi_flags = 1 << BIO_UPTODATE; 548 bi->bi_vcnt = 1; 549 bi->bi_max_vecs = 1; 550 bi->bi_idx = 0; 551 bi->bi_io_vec = &sh->dev[i].vec; 552 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 553 bi->bi_io_vec[0].bv_offset = 0; 554 bi->bi_size = STRIPE_SIZE; 555 bi->bi_next = NULL; 556 if (rw == WRITE && 557 test_bit(R5_ReWrite, &sh->dev[i].flags)) 558 atomic_add(STRIPE_SECTORS, 559 &rdev->corrected_errors); 560 generic_make_request(bi); 561 } else { 562 if (rw == WRITE) 563 set_bit(STRIPE_DEGRADED, &sh->state); 564 pr_debug("skip op %ld on disc %d for sector %llu\n", 565 bi->bi_rw, i, (unsigned long long)sh->sector); 566 clear_bit(R5_LOCKED, &sh->dev[i].flags); 567 set_bit(STRIPE_HANDLE, &sh->state); 568 } 569 } 570 } 571 572 static struct dma_async_tx_descriptor * 573 async_copy_data(int frombio, struct bio *bio, struct page *page, 574 sector_t sector, struct dma_async_tx_descriptor *tx) 575 { 576 struct bio_vec *bvl; 577 struct page *bio_page; 578 int i; 579 int page_offset; 580 struct async_submit_ctl submit; 581 enum async_tx_flags flags = 0; 582 583 if (bio->bi_sector >= sector) 584 page_offset = (signed)(bio->bi_sector - sector) * 512; 585 else 586 page_offset = (signed)(sector - bio->bi_sector) * -512; 587 588 if (frombio) 589 flags |= ASYNC_TX_FENCE; 590 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 591 592 bio_for_each_segment(bvl, bio, i) { 593 int len = bio_iovec_idx(bio, i)->bv_len; 594 int clen; 595 int b_offset = 0; 596 597 if (page_offset < 0) { 598 b_offset = -page_offset; 599 page_offset += b_offset; 600 len -= b_offset; 601 } 602 603 if (len > 0 && page_offset + len > STRIPE_SIZE) 604 clen = STRIPE_SIZE - page_offset; 605 else 606 clen = len; 607 608 if (clen > 0) { 609 b_offset += bio_iovec_idx(bio, i)->bv_offset; 610 bio_page = bio_iovec_idx(bio, i)->bv_page; 611 if (frombio) 612 tx = async_memcpy(page, bio_page, page_offset, 613 b_offset, clen, &submit); 614 else 615 tx = async_memcpy(bio_page, page, b_offset, 616 page_offset, clen, &submit); 617 } 618 /* chain the operations */ 619 submit.depend_tx = tx; 620 621 if (clen < len) /* hit end of page */ 622 break; 623 page_offset += len; 624 } 625 626 return tx; 627 } 628 629 static void ops_complete_biofill(void *stripe_head_ref) 630 { 631 struct stripe_head *sh = stripe_head_ref; 632 struct bio *return_bi = NULL; 633 raid5_conf_t *conf = sh->raid_conf; 634 int i; 635 636 pr_debug("%s: stripe %llu\n", __func__, 637 (unsigned long long)sh->sector); 638 639 /* clear completed biofills */ 640 spin_lock_irq(&conf->device_lock); 641 for (i = sh->disks; i--; ) { 642 struct r5dev *dev = &sh->dev[i]; 643 644 /* acknowledge completion of a biofill operation */ 645 /* and check if we need to reply to a read request, 646 * new R5_Wantfill requests are held off until 647 * !STRIPE_BIOFILL_RUN 648 */ 649 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 650 struct bio *rbi, *rbi2; 651 652 BUG_ON(!dev->read); 653 rbi = dev->read; 654 dev->read = NULL; 655 while (rbi && rbi->bi_sector < 656 dev->sector + STRIPE_SECTORS) { 657 rbi2 = r5_next_bio(rbi, dev->sector); 658 if (!raid5_dec_bi_phys_segments(rbi)) { 659 rbi->bi_next = return_bi; 660 return_bi = rbi; 661 } 662 rbi = rbi2; 663 } 664 } 665 } 666 spin_unlock_irq(&conf->device_lock); 667 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 668 669 return_io(return_bi); 670 671 set_bit(STRIPE_HANDLE, &sh->state); 672 release_stripe(sh); 673 } 674 675 static void ops_run_biofill(struct stripe_head *sh) 676 { 677 struct dma_async_tx_descriptor *tx = NULL; 678 raid5_conf_t *conf = sh->raid_conf; 679 struct async_submit_ctl submit; 680 int i; 681 682 pr_debug("%s: stripe %llu\n", __func__, 683 (unsigned long long)sh->sector); 684 685 for (i = sh->disks; i--; ) { 686 struct r5dev *dev = &sh->dev[i]; 687 if (test_bit(R5_Wantfill, &dev->flags)) { 688 struct bio *rbi; 689 spin_lock_irq(&conf->device_lock); 690 dev->read = rbi = dev->toread; 691 dev->toread = NULL; 692 spin_unlock_irq(&conf->device_lock); 693 while (rbi && rbi->bi_sector < 694 dev->sector + STRIPE_SECTORS) { 695 tx = async_copy_data(0, rbi, dev->page, 696 dev->sector, tx); 697 rbi = r5_next_bio(rbi, dev->sector); 698 } 699 } 700 } 701 702 atomic_inc(&sh->count); 703 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 704 async_trigger_callback(&submit); 705 } 706 707 static void mark_target_uptodate(struct stripe_head *sh, int target) 708 { 709 struct r5dev *tgt; 710 711 if (target < 0) 712 return; 713 714 tgt = &sh->dev[target]; 715 set_bit(R5_UPTODATE, &tgt->flags); 716 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 717 clear_bit(R5_Wantcompute, &tgt->flags); 718 } 719 720 static void ops_complete_compute(void *stripe_head_ref) 721 { 722 struct stripe_head *sh = stripe_head_ref; 723 724 pr_debug("%s: stripe %llu\n", __func__, 725 (unsigned long long)sh->sector); 726 727 /* mark the computed target(s) as uptodate */ 728 mark_target_uptodate(sh, sh->ops.target); 729 mark_target_uptodate(sh, sh->ops.target2); 730 731 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 732 if (sh->check_state == check_state_compute_run) 733 sh->check_state = check_state_compute_result; 734 set_bit(STRIPE_HANDLE, &sh->state); 735 release_stripe(sh); 736 } 737 738 /* return a pointer to the address conversion region of the scribble buffer */ 739 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 740 struct raid5_percpu *percpu) 741 { 742 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 743 } 744 745 static struct dma_async_tx_descriptor * 746 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 747 { 748 int disks = sh->disks; 749 struct page **xor_srcs = percpu->scribble; 750 int target = sh->ops.target; 751 struct r5dev *tgt = &sh->dev[target]; 752 struct page *xor_dest = tgt->page; 753 int count = 0; 754 struct dma_async_tx_descriptor *tx; 755 struct async_submit_ctl submit; 756 int i; 757 758 pr_debug("%s: stripe %llu block: %d\n", 759 __func__, (unsigned long long)sh->sector, target); 760 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 761 762 for (i = disks; i--; ) 763 if (i != target) 764 xor_srcs[count++] = sh->dev[i].page; 765 766 atomic_inc(&sh->count); 767 768 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 769 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 770 if (unlikely(count == 1)) 771 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 772 else 773 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 774 775 return tx; 776 } 777 778 /* set_syndrome_sources - populate source buffers for gen_syndrome 779 * @srcs - (struct page *) array of size sh->disks 780 * @sh - stripe_head to parse 781 * 782 * Populates srcs in proper layout order for the stripe and returns the 783 * 'count' of sources to be used in a call to async_gen_syndrome. The P 784 * destination buffer is recorded in srcs[count] and the Q destination 785 * is recorded in srcs[count+1]]. 786 */ 787 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 788 { 789 int disks = sh->disks; 790 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 791 int d0_idx = raid6_d0(sh); 792 int count; 793 int i; 794 795 for (i = 0; i < disks; i++) 796 srcs[i] = NULL; 797 798 count = 0; 799 i = d0_idx; 800 do { 801 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 802 803 srcs[slot] = sh->dev[i].page; 804 i = raid6_next_disk(i, disks); 805 } while (i != d0_idx); 806 807 return syndrome_disks; 808 } 809 810 static struct dma_async_tx_descriptor * 811 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 812 { 813 int disks = sh->disks; 814 struct page **blocks = percpu->scribble; 815 int target; 816 int qd_idx = sh->qd_idx; 817 struct dma_async_tx_descriptor *tx; 818 struct async_submit_ctl submit; 819 struct r5dev *tgt; 820 struct page *dest; 821 int i; 822 int count; 823 824 if (sh->ops.target < 0) 825 target = sh->ops.target2; 826 else if (sh->ops.target2 < 0) 827 target = sh->ops.target; 828 else 829 /* we should only have one valid target */ 830 BUG(); 831 BUG_ON(target < 0); 832 pr_debug("%s: stripe %llu block: %d\n", 833 __func__, (unsigned long long)sh->sector, target); 834 835 tgt = &sh->dev[target]; 836 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 837 dest = tgt->page; 838 839 atomic_inc(&sh->count); 840 841 if (target == qd_idx) { 842 count = set_syndrome_sources(blocks, sh); 843 blocks[count] = NULL; /* regenerating p is not necessary */ 844 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 845 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 846 ops_complete_compute, sh, 847 to_addr_conv(sh, percpu)); 848 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 849 } else { 850 /* Compute any data- or p-drive using XOR */ 851 count = 0; 852 for (i = disks; i-- ; ) { 853 if (i == target || i == qd_idx) 854 continue; 855 blocks[count++] = sh->dev[i].page; 856 } 857 858 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 859 NULL, ops_complete_compute, sh, 860 to_addr_conv(sh, percpu)); 861 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 862 } 863 864 return tx; 865 } 866 867 static struct dma_async_tx_descriptor * 868 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 869 { 870 int i, count, disks = sh->disks; 871 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 872 int d0_idx = raid6_d0(sh); 873 int faila = -1, failb = -1; 874 int target = sh->ops.target; 875 int target2 = sh->ops.target2; 876 struct r5dev *tgt = &sh->dev[target]; 877 struct r5dev *tgt2 = &sh->dev[target2]; 878 struct dma_async_tx_descriptor *tx; 879 struct page **blocks = percpu->scribble; 880 struct async_submit_ctl submit; 881 882 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 883 __func__, (unsigned long long)sh->sector, target, target2); 884 BUG_ON(target < 0 || target2 < 0); 885 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 886 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 887 888 /* we need to open-code set_syndrome_sources to handle the 889 * slot number conversion for 'faila' and 'failb' 890 */ 891 for (i = 0; i < disks ; i++) 892 blocks[i] = NULL; 893 count = 0; 894 i = d0_idx; 895 do { 896 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 897 898 blocks[slot] = sh->dev[i].page; 899 900 if (i == target) 901 faila = slot; 902 if (i == target2) 903 failb = slot; 904 i = raid6_next_disk(i, disks); 905 } while (i != d0_idx); 906 907 BUG_ON(faila == failb); 908 if (failb < faila) 909 swap(faila, failb); 910 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 911 __func__, (unsigned long long)sh->sector, faila, failb); 912 913 atomic_inc(&sh->count); 914 915 if (failb == syndrome_disks+1) { 916 /* Q disk is one of the missing disks */ 917 if (faila == syndrome_disks) { 918 /* Missing P+Q, just recompute */ 919 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 920 ops_complete_compute, sh, 921 to_addr_conv(sh, percpu)); 922 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 923 STRIPE_SIZE, &submit); 924 } else { 925 struct page *dest; 926 int data_target; 927 int qd_idx = sh->qd_idx; 928 929 /* Missing D+Q: recompute D from P, then recompute Q */ 930 if (target == qd_idx) 931 data_target = target2; 932 else 933 data_target = target; 934 935 count = 0; 936 for (i = disks; i-- ; ) { 937 if (i == data_target || i == qd_idx) 938 continue; 939 blocks[count++] = sh->dev[i].page; 940 } 941 dest = sh->dev[data_target].page; 942 init_async_submit(&submit, 943 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 944 NULL, NULL, NULL, 945 to_addr_conv(sh, percpu)); 946 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 947 &submit); 948 949 count = set_syndrome_sources(blocks, sh); 950 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 951 ops_complete_compute, sh, 952 to_addr_conv(sh, percpu)); 953 return async_gen_syndrome(blocks, 0, count+2, 954 STRIPE_SIZE, &submit); 955 } 956 } else { 957 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 958 ops_complete_compute, sh, 959 to_addr_conv(sh, percpu)); 960 if (failb == syndrome_disks) { 961 /* We're missing D+P. */ 962 return async_raid6_datap_recov(syndrome_disks+2, 963 STRIPE_SIZE, faila, 964 blocks, &submit); 965 } else { 966 /* We're missing D+D. */ 967 return async_raid6_2data_recov(syndrome_disks+2, 968 STRIPE_SIZE, faila, failb, 969 blocks, &submit); 970 } 971 } 972 } 973 974 975 static void ops_complete_prexor(void *stripe_head_ref) 976 { 977 struct stripe_head *sh = stripe_head_ref; 978 979 pr_debug("%s: stripe %llu\n", __func__, 980 (unsigned long long)sh->sector); 981 } 982 983 static struct dma_async_tx_descriptor * 984 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 985 struct dma_async_tx_descriptor *tx) 986 { 987 int disks = sh->disks; 988 struct page **xor_srcs = percpu->scribble; 989 int count = 0, pd_idx = sh->pd_idx, i; 990 struct async_submit_ctl submit; 991 992 /* existing parity data subtracted */ 993 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 994 995 pr_debug("%s: stripe %llu\n", __func__, 996 (unsigned long long)sh->sector); 997 998 for (i = disks; i--; ) { 999 struct r5dev *dev = &sh->dev[i]; 1000 /* Only process blocks that are known to be uptodate */ 1001 if (test_bit(R5_Wantdrain, &dev->flags)) 1002 xor_srcs[count++] = dev->page; 1003 } 1004 1005 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1006 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 1007 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1008 1009 return tx; 1010 } 1011 1012 static struct dma_async_tx_descriptor * 1013 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1014 { 1015 int disks = sh->disks; 1016 int i; 1017 1018 pr_debug("%s: stripe %llu\n", __func__, 1019 (unsigned long long)sh->sector); 1020 1021 for (i = disks; i--; ) { 1022 struct r5dev *dev = &sh->dev[i]; 1023 struct bio *chosen; 1024 1025 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 1026 struct bio *wbi; 1027 1028 spin_lock(&sh->lock); 1029 chosen = dev->towrite; 1030 dev->towrite = NULL; 1031 BUG_ON(dev->written); 1032 wbi = dev->written = chosen; 1033 spin_unlock(&sh->lock); 1034 1035 while (wbi && wbi->bi_sector < 1036 dev->sector + STRIPE_SECTORS) { 1037 if (wbi->bi_rw & REQ_FUA) 1038 set_bit(R5_WantFUA, &dev->flags); 1039 tx = async_copy_data(1, wbi, dev->page, 1040 dev->sector, tx); 1041 wbi = r5_next_bio(wbi, dev->sector); 1042 } 1043 } 1044 } 1045 1046 return tx; 1047 } 1048 1049 static void ops_complete_reconstruct(void *stripe_head_ref) 1050 { 1051 struct stripe_head *sh = stripe_head_ref; 1052 int disks = sh->disks; 1053 int pd_idx = sh->pd_idx; 1054 int qd_idx = sh->qd_idx; 1055 int i; 1056 bool fua = false; 1057 1058 pr_debug("%s: stripe %llu\n", __func__, 1059 (unsigned long long)sh->sector); 1060 1061 for (i = disks; i--; ) 1062 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1063 1064 for (i = disks; i--; ) { 1065 struct r5dev *dev = &sh->dev[i]; 1066 1067 if (dev->written || i == pd_idx || i == qd_idx) { 1068 set_bit(R5_UPTODATE, &dev->flags); 1069 if (fua) 1070 set_bit(R5_WantFUA, &dev->flags); 1071 } 1072 } 1073 1074 if (sh->reconstruct_state == reconstruct_state_drain_run) 1075 sh->reconstruct_state = reconstruct_state_drain_result; 1076 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1077 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1078 else { 1079 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1080 sh->reconstruct_state = reconstruct_state_result; 1081 } 1082 1083 set_bit(STRIPE_HANDLE, &sh->state); 1084 release_stripe(sh); 1085 } 1086 1087 static void 1088 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1089 struct dma_async_tx_descriptor *tx) 1090 { 1091 int disks = sh->disks; 1092 struct page **xor_srcs = percpu->scribble; 1093 struct async_submit_ctl submit; 1094 int count = 0, pd_idx = sh->pd_idx, i; 1095 struct page *xor_dest; 1096 int prexor = 0; 1097 unsigned long flags; 1098 1099 pr_debug("%s: stripe %llu\n", __func__, 1100 (unsigned long long)sh->sector); 1101 1102 /* check if prexor is active which means only process blocks 1103 * that are part of a read-modify-write (written) 1104 */ 1105 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1106 prexor = 1; 1107 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1108 for (i = disks; i--; ) { 1109 struct r5dev *dev = &sh->dev[i]; 1110 if (dev->written) 1111 xor_srcs[count++] = dev->page; 1112 } 1113 } else { 1114 xor_dest = sh->dev[pd_idx].page; 1115 for (i = disks; i--; ) { 1116 struct r5dev *dev = &sh->dev[i]; 1117 if (i != pd_idx) 1118 xor_srcs[count++] = dev->page; 1119 } 1120 } 1121 1122 /* 1/ if we prexor'd then the dest is reused as a source 1123 * 2/ if we did not prexor then we are redoing the parity 1124 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1125 * for the synchronous xor case 1126 */ 1127 flags = ASYNC_TX_ACK | 1128 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1129 1130 atomic_inc(&sh->count); 1131 1132 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1133 to_addr_conv(sh, percpu)); 1134 if (unlikely(count == 1)) 1135 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1136 else 1137 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1138 } 1139 1140 static void 1141 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1142 struct dma_async_tx_descriptor *tx) 1143 { 1144 struct async_submit_ctl submit; 1145 struct page **blocks = percpu->scribble; 1146 int count; 1147 1148 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1149 1150 count = set_syndrome_sources(blocks, sh); 1151 1152 atomic_inc(&sh->count); 1153 1154 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1155 sh, to_addr_conv(sh, percpu)); 1156 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1157 } 1158 1159 static void ops_complete_check(void *stripe_head_ref) 1160 { 1161 struct stripe_head *sh = stripe_head_ref; 1162 1163 pr_debug("%s: stripe %llu\n", __func__, 1164 (unsigned long long)sh->sector); 1165 1166 sh->check_state = check_state_check_result; 1167 set_bit(STRIPE_HANDLE, &sh->state); 1168 release_stripe(sh); 1169 } 1170 1171 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1172 { 1173 int disks = sh->disks; 1174 int pd_idx = sh->pd_idx; 1175 int qd_idx = sh->qd_idx; 1176 struct page *xor_dest; 1177 struct page **xor_srcs = percpu->scribble; 1178 struct dma_async_tx_descriptor *tx; 1179 struct async_submit_ctl submit; 1180 int count; 1181 int i; 1182 1183 pr_debug("%s: stripe %llu\n", __func__, 1184 (unsigned long long)sh->sector); 1185 1186 count = 0; 1187 xor_dest = sh->dev[pd_idx].page; 1188 xor_srcs[count++] = xor_dest; 1189 for (i = disks; i--; ) { 1190 if (i == pd_idx || i == qd_idx) 1191 continue; 1192 xor_srcs[count++] = sh->dev[i].page; 1193 } 1194 1195 init_async_submit(&submit, 0, NULL, NULL, NULL, 1196 to_addr_conv(sh, percpu)); 1197 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1198 &sh->ops.zero_sum_result, &submit); 1199 1200 atomic_inc(&sh->count); 1201 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1202 tx = async_trigger_callback(&submit); 1203 } 1204 1205 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1206 { 1207 struct page **srcs = percpu->scribble; 1208 struct async_submit_ctl submit; 1209 int count; 1210 1211 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1212 (unsigned long long)sh->sector, checkp); 1213 1214 count = set_syndrome_sources(srcs, sh); 1215 if (!checkp) 1216 srcs[count] = NULL; 1217 1218 atomic_inc(&sh->count); 1219 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1220 sh, to_addr_conv(sh, percpu)); 1221 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1222 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1223 } 1224 1225 static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1226 { 1227 int overlap_clear = 0, i, disks = sh->disks; 1228 struct dma_async_tx_descriptor *tx = NULL; 1229 raid5_conf_t *conf = sh->raid_conf; 1230 int level = conf->level; 1231 struct raid5_percpu *percpu; 1232 unsigned long cpu; 1233 1234 cpu = get_cpu(); 1235 percpu = per_cpu_ptr(conf->percpu, cpu); 1236 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1237 ops_run_biofill(sh); 1238 overlap_clear++; 1239 } 1240 1241 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1242 if (level < 6) 1243 tx = ops_run_compute5(sh, percpu); 1244 else { 1245 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1246 tx = ops_run_compute6_1(sh, percpu); 1247 else 1248 tx = ops_run_compute6_2(sh, percpu); 1249 } 1250 /* terminate the chain if reconstruct is not set to be run */ 1251 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1252 async_tx_ack(tx); 1253 } 1254 1255 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1256 tx = ops_run_prexor(sh, percpu, tx); 1257 1258 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1259 tx = ops_run_biodrain(sh, tx); 1260 overlap_clear++; 1261 } 1262 1263 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1264 if (level < 6) 1265 ops_run_reconstruct5(sh, percpu, tx); 1266 else 1267 ops_run_reconstruct6(sh, percpu, tx); 1268 } 1269 1270 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1271 if (sh->check_state == check_state_run) 1272 ops_run_check_p(sh, percpu); 1273 else if (sh->check_state == check_state_run_q) 1274 ops_run_check_pq(sh, percpu, 0); 1275 else if (sh->check_state == check_state_run_pq) 1276 ops_run_check_pq(sh, percpu, 1); 1277 else 1278 BUG(); 1279 } 1280 1281 if (overlap_clear) 1282 for (i = disks; i--; ) { 1283 struct r5dev *dev = &sh->dev[i]; 1284 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1285 wake_up(&sh->raid_conf->wait_for_overlap); 1286 } 1287 put_cpu(); 1288 } 1289 1290 #ifdef CONFIG_MULTICORE_RAID456 1291 static void async_run_ops(void *param, async_cookie_t cookie) 1292 { 1293 struct stripe_head *sh = param; 1294 unsigned long ops_request = sh->ops.request; 1295 1296 clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state); 1297 wake_up(&sh->ops.wait_for_ops); 1298 1299 __raid_run_ops(sh, ops_request); 1300 release_stripe(sh); 1301 } 1302 1303 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1304 { 1305 /* since handle_stripe can be called outside of raid5d context 1306 * we need to ensure sh->ops.request is de-staged before another 1307 * request arrives 1308 */ 1309 wait_event(sh->ops.wait_for_ops, 1310 !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state)); 1311 sh->ops.request = ops_request; 1312 1313 atomic_inc(&sh->count); 1314 async_schedule(async_run_ops, sh); 1315 } 1316 #else 1317 #define raid_run_ops __raid_run_ops 1318 #endif 1319 1320 static int grow_one_stripe(raid5_conf_t *conf) 1321 { 1322 struct stripe_head *sh; 1323 sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); 1324 if (!sh) 1325 return 0; 1326 memset(sh, 0, sizeof(*sh) + (conf->pool_size-1)*sizeof(struct r5dev)); 1327 sh->raid_conf = conf; 1328 spin_lock_init(&sh->lock); 1329 #ifdef CONFIG_MULTICORE_RAID456 1330 init_waitqueue_head(&sh->ops.wait_for_ops); 1331 #endif 1332 1333 if (grow_buffers(sh)) { 1334 shrink_buffers(sh); 1335 kmem_cache_free(conf->slab_cache, sh); 1336 return 0; 1337 } 1338 /* we just created an active stripe so... */ 1339 atomic_set(&sh->count, 1); 1340 atomic_inc(&conf->active_stripes); 1341 INIT_LIST_HEAD(&sh->lru); 1342 release_stripe(sh); 1343 return 1; 1344 } 1345 1346 static int grow_stripes(raid5_conf_t *conf, int num) 1347 { 1348 struct kmem_cache *sc; 1349 int devs = max(conf->raid_disks, conf->previous_raid_disks); 1350 1351 if (conf->mddev->gendisk) 1352 sprintf(conf->cache_name[0], 1353 "raid%d-%s", conf->level, mdname(conf->mddev)); 1354 else 1355 sprintf(conf->cache_name[0], 1356 "raid%d-%p", conf->level, conf->mddev); 1357 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 1358 1359 conf->active_name = 0; 1360 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1361 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1362 0, 0, NULL); 1363 if (!sc) 1364 return 1; 1365 conf->slab_cache = sc; 1366 conf->pool_size = devs; 1367 while (num--) 1368 if (!grow_one_stripe(conf)) 1369 return 1; 1370 return 0; 1371 } 1372 1373 /** 1374 * scribble_len - return the required size of the scribble region 1375 * @num - total number of disks in the array 1376 * 1377 * The size must be enough to contain: 1378 * 1/ a struct page pointer for each device in the array +2 1379 * 2/ room to convert each entry in (1) to its corresponding dma 1380 * (dma_map_page()) or page (page_address()) address. 1381 * 1382 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1383 * calculate over all devices (not just the data blocks), using zeros in place 1384 * of the P and Q blocks. 1385 */ 1386 static size_t scribble_len(int num) 1387 { 1388 size_t len; 1389 1390 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1391 1392 return len; 1393 } 1394 1395 static int resize_stripes(raid5_conf_t *conf, int newsize) 1396 { 1397 /* Make all the stripes able to hold 'newsize' devices. 1398 * New slots in each stripe get 'page' set to a new page. 1399 * 1400 * This happens in stages: 1401 * 1/ create a new kmem_cache and allocate the required number of 1402 * stripe_heads. 1403 * 2/ gather all the old stripe_heads and tranfer the pages across 1404 * to the new stripe_heads. This will have the side effect of 1405 * freezing the array as once all stripe_heads have been collected, 1406 * no IO will be possible. Old stripe heads are freed once their 1407 * pages have been transferred over, and the old kmem_cache is 1408 * freed when all stripes are done. 1409 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1410 * we simple return a failre status - no need to clean anything up. 1411 * 4/ allocate new pages for the new slots in the new stripe_heads. 1412 * If this fails, we don't bother trying the shrink the 1413 * stripe_heads down again, we just leave them as they are. 1414 * As each stripe_head is processed the new one is released into 1415 * active service. 1416 * 1417 * Once step2 is started, we cannot afford to wait for a write, 1418 * so we use GFP_NOIO allocations. 1419 */ 1420 struct stripe_head *osh, *nsh; 1421 LIST_HEAD(newstripes); 1422 struct disk_info *ndisks; 1423 unsigned long cpu; 1424 int err; 1425 struct kmem_cache *sc; 1426 int i; 1427 1428 if (newsize <= conf->pool_size) 1429 return 0; /* never bother to shrink */ 1430 1431 err = md_allow_write(conf->mddev); 1432 if (err) 1433 return err; 1434 1435 /* Step 1 */ 1436 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1437 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1438 0, 0, NULL); 1439 if (!sc) 1440 return -ENOMEM; 1441 1442 for (i = conf->max_nr_stripes; i; i--) { 1443 nsh = kmem_cache_alloc(sc, GFP_KERNEL); 1444 if (!nsh) 1445 break; 1446 1447 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); 1448 1449 nsh->raid_conf = conf; 1450 spin_lock_init(&nsh->lock); 1451 #ifdef CONFIG_MULTICORE_RAID456 1452 init_waitqueue_head(&nsh->ops.wait_for_ops); 1453 #endif 1454 1455 list_add(&nsh->lru, &newstripes); 1456 } 1457 if (i) { 1458 /* didn't get enough, give up */ 1459 while (!list_empty(&newstripes)) { 1460 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1461 list_del(&nsh->lru); 1462 kmem_cache_free(sc, nsh); 1463 } 1464 kmem_cache_destroy(sc); 1465 return -ENOMEM; 1466 } 1467 /* Step 2 - Must use GFP_NOIO now. 1468 * OK, we have enough stripes, start collecting inactive 1469 * stripes and copying them over 1470 */ 1471 list_for_each_entry(nsh, &newstripes, lru) { 1472 spin_lock_irq(&conf->device_lock); 1473 wait_event_lock_irq(conf->wait_for_stripe, 1474 !list_empty(&conf->inactive_list), 1475 conf->device_lock, 1476 unplug_slaves(conf->mddev) 1477 ); 1478 osh = get_free_stripe(conf); 1479 spin_unlock_irq(&conf->device_lock); 1480 atomic_set(&nsh->count, 1); 1481 for(i=0; i<conf->pool_size; i++) 1482 nsh->dev[i].page = osh->dev[i].page; 1483 for( ; i<newsize; i++) 1484 nsh->dev[i].page = NULL; 1485 kmem_cache_free(conf->slab_cache, osh); 1486 } 1487 kmem_cache_destroy(conf->slab_cache); 1488 1489 /* Step 3. 1490 * At this point, we are holding all the stripes so the array 1491 * is completely stalled, so now is a good time to resize 1492 * conf->disks and the scribble region 1493 */ 1494 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1495 if (ndisks) { 1496 for (i=0; i<conf->raid_disks; i++) 1497 ndisks[i] = conf->disks[i]; 1498 kfree(conf->disks); 1499 conf->disks = ndisks; 1500 } else 1501 err = -ENOMEM; 1502 1503 get_online_cpus(); 1504 conf->scribble_len = scribble_len(newsize); 1505 for_each_present_cpu(cpu) { 1506 struct raid5_percpu *percpu; 1507 void *scribble; 1508 1509 percpu = per_cpu_ptr(conf->percpu, cpu); 1510 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1511 1512 if (scribble) { 1513 kfree(percpu->scribble); 1514 percpu->scribble = scribble; 1515 } else { 1516 err = -ENOMEM; 1517 break; 1518 } 1519 } 1520 put_online_cpus(); 1521 1522 /* Step 4, return new stripes to service */ 1523 while(!list_empty(&newstripes)) { 1524 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1525 list_del_init(&nsh->lru); 1526 1527 for (i=conf->raid_disks; i < newsize; i++) 1528 if (nsh->dev[i].page == NULL) { 1529 struct page *p = alloc_page(GFP_NOIO); 1530 nsh->dev[i].page = p; 1531 if (!p) 1532 err = -ENOMEM; 1533 } 1534 release_stripe(nsh); 1535 } 1536 /* critical section pass, GFP_NOIO no longer needed */ 1537 1538 conf->slab_cache = sc; 1539 conf->active_name = 1-conf->active_name; 1540 conf->pool_size = newsize; 1541 return err; 1542 } 1543 1544 static int drop_one_stripe(raid5_conf_t *conf) 1545 { 1546 struct stripe_head *sh; 1547 1548 spin_lock_irq(&conf->device_lock); 1549 sh = get_free_stripe(conf); 1550 spin_unlock_irq(&conf->device_lock); 1551 if (!sh) 1552 return 0; 1553 BUG_ON(atomic_read(&sh->count)); 1554 shrink_buffers(sh); 1555 kmem_cache_free(conf->slab_cache, sh); 1556 atomic_dec(&conf->active_stripes); 1557 return 1; 1558 } 1559 1560 static void shrink_stripes(raid5_conf_t *conf) 1561 { 1562 while (drop_one_stripe(conf)) 1563 ; 1564 1565 if (conf->slab_cache) 1566 kmem_cache_destroy(conf->slab_cache); 1567 conf->slab_cache = NULL; 1568 } 1569 1570 static void raid5_end_read_request(struct bio * bi, int error) 1571 { 1572 struct stripe_head *sh = bi->bi_private; 1573 raid5_conf_t *conf = sh->raid_conf; 1574 int disks = sh->disks, i; 1575 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1576 char b[BDEVNAME_SIZE]; 1577 mdk_rdev_t *rdev; 1578 1579 1580 for (i=0 ; i<disks; i++) 1581 if (bi == &sh->dev[i].req) 1582 break; 1583 1584 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1585 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1586 uptodate); 1587 if (i == disks) { 1588 BUG(); 1589 return; 1590 } 1591 1592 if (uptodate) { 1593 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1594 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1595 rdev = conf->disks[i].rdev; 1596 printk_rl(KERN_INFO "md/raid:%s: read error corrected" 1597 " (%lu sectors at %llu on %s)\n", 1598 mdname(conf->mddev), STRIPE_SECTORS, 1599 (unsigned long long)(sh->sector 1600 + rdev->data_offset), 1601 bdevname(rdev->bdev, b)); 1602 clear_bit(R5_ReadError, &sh->dev[i].flags); 1603 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1604 } 1605 if (atomic_read(&conf->disks[i].rdev->read_errors)) 1606 atomic_set(&conf->disks[i].rdev->read_errors, 0); 1607 } else { 1608 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b); 1609 int retry = 0; 1610 rdev = conf->disks[i].rdev; 1611 1612 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1613 atomic_inc(&rdev->read_errors); 1614 if (conf->mddev->degraded >= conf->max_degraded) 1615 printk_rl(KERN_WARNING 1616 "md/raid:%s: read error not correctable " 1617 "(sector %llu on %s).\n", 1618 mdname(conf->mddev), 1619 (unsigned long long)(sh->sector 1620 + rdev->data_offset), 1621 bdn); 1622 else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) 1623 /* Oh, no!!! */ 1624 printk_rl(KERN_WARNING 1625 "md/raid:%s: read error NOT corrected!! " 1626 "(sector %llu on %s).\n", 1627 mdname(conf->mddev), 1628 (unsigned long long)(sh->sector 1629 + rdev->data_offset), 1630 bdn); 1631 else if (atomic_read(&rdev->read_errors) 1632 > conf->max_nr_stripes) 1633 printk(KERN_WARNING 1634 "md/raid:%s: Too many read errors, failing device %s.\n", 1635 mdname(conf->mddev), bdn); 1636 else 1637 retry = 1; 1638 if (retry) 1639 set_bit(R5_ReadError, &sh->dev[i].flags); 1640 else { 1641 clear_bit(R5_ReadError, &sh->dev[i].flags); 1642 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1643 md_error(conf->mddev, rdev); 1644 } 1645 } 1646 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1647 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1648 set_bit(STRIPE_HANDLE, &sh->state); 1649 release_stripe(sh); 1650 } 1651 1652 static void raid5_end_write_request(struct bio *bi, int error) 1653 { 1654 struct stripe_head *sh = bi->bi_private; 1655 raid5_conf_t *conf = sh->raid_conf; 1656 int disks = sh->disks, i; 1657 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1658 1659 for (i=0 ; i<disks; i++) 1660 if (bi == &sh->dev[i].req) 1661 break; 1662 1663 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1664 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1665 uptodate); 1666 if (i == disks) { 1667 BUG(); 1668 return; 1669 } 1670 1671 if (!uptodate) 1672 md_error(conf->mddev, conf->disks[i].rdev); 1673 1674 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1675 1676 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1677 set_bit(STRIPE_HANDLE, &sh->state); 1678 release_stripe(sh); 1679 } 1680 1681 1682 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 1683 1684 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 1685 { 1686 struct r5dev *dev = &sh->dev[i]; 1687 1688 bio_init(&dev->req); 1689 dev->req.bi_io_vec = &dev->vec; 1690 dev->req.bi_vcnt++; 1691 dev->req.bi_max_vecs++; 1692 dev->vec.bv_page = dev->page; 1693 dev->vec.bv_len = STRIPE_SIZE; 1694 dev->vec.bv_offset = 0; 1695 1696 dev->req.bi_sector = sh->sector; 1697 dev->req.bi_private = sh; 1698 1699 dev->flags = 0; 1700 dev->sector = compute_blocknr(sh, i, previous); 1701 } 1702 1703 static void error(mddev_t *mddev, mdk_rdev_t *rdev) 1704 { 1705 char b[BDEVNAME_SIZE]; 1706 raid5_conf_t *conf = mddev->private; 1707 pr_debug("raid456: error called\n"); 1708 1709 if (!test_bit(Faulty, &rdev->flags)) { 1710 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1711 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1712 unsigned long flags; 1713 spin_lock_irqsave(&conf->device_lock, flags); 1714 mddev->degraded++; 1715 spin_unlock_irqrestore(&conf->device_lock, flags); 1716 /* 1717 * if recovery was running, make sure it aborts. 1718 */ 1719 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1720 } 1721 set_bit(Faulty, &rdev->flags); 1722 printk(KERN_ALERT 1723 "md/raid:%s: Disk failure on %s, disabling device.\n" 1724 KERN_ALERT 1725 "md/raid:%s: Operation continuing on %d devices.\n", 1726 mdname(mddev), 1727 bdevname(rdev->bdev, b), 1728 mdname(mddev), 1729 conf->raid_disks - mddev->degraded); 1730 } 1731 } 1732 1733 /* 1734 * Input: a 'big' sector number, 1735 * Output: index of the data and parity disk, and the sector # in them. 1736 */ 1737 static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector, 1738 int previous, int *dd_idx, 1739 struct stripe_head *sh) 1740 { 1741 sector_t stripe, stripe2; 1742 sector_t chunk_number; 1743 unsigned int chunk_offset; 1744 int pd_idx, qd_idx; 1745 int ddf_layout = 0; 1746 sector_t new_sector; 1747 int algorithm = previous ? conf->prev_algo 1748 : conf->algorithm; 1749 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1750 : conf->chunk_sectors; 1751 int raid_disks = previous ? conf->previous_raid_disks 1752 : conf->raid_disks; 1753 int data_disks = raid_disks - conf->max_degraded; 1754 1755 /* First compute the information on this sector */ 1756 1757 /* 1758 * Compute the chunk number and the sector offset inside the chunk 1759 */ 1760 chunk_offset = sector_div(r_sector, sectors_per_chunk); 1761 chunk_number = r_sector; 1762 1763 /* 1764 * Compute the stripe number 1765 */ 1766 stripe = chunk_number; 1767 *dd_idx = sector_div(stripe, data_disks); 1768 stripe2 = stripe; 1769 /* 1770 * Select the parity disk based on the user selected algorithm. 1771 */ 1772 pd_idx = qd_idx = ~0; 1773 switch(conf->level) { 1774 case 4: 1775 pd_idx = data_disks; 1776 break; 1777 case 5: 1778 switch (algorithm) { 1779 case ALGORITHM_LEFT_ASYMMETRIC: 1780 pd_idx = data_disks - sector_div(stripe2, raid_disks); 1781 if (*dd_idx >= pd_idx) 1782 (*dd_idx)++; 1783 break; 1784 case ALGORITHM_RIGHT_ASYMMETRIC: 1785 pd_idx = sector_div(stripe2, raid_disks); 1786 if (*dd_idx >= pd_idx) 1787 (*dd_idx)++; 1788 break; 1789 case ALGORITHM_LEFT_SYMMETRIC: 1790 pd_idx = data_disks - sector_div(stripe2, raid_disks); 1791 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1792 break; 1793 case ALGORITHM_RIGHT_SYMMETRIC: 1794 pd_idx = sector_div(stripe2, raid_disks); 1795 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1796 break; 1797 case ALGORITHM_PARITY_0: 1798 pd_idx = 0; 1799 (*dd_idx)++; 1800 break; 1801 case ALGORITHM_PARITY_N: 1802 pd_idx = data_disks; 1803 break; 1804 default: 1805 BUG(); 1806 } 1807 break; 1808 case 6: 1809 1810 switch (algorithm) { 1811 case ALGORITHM_LEFT_ASYMMETRIC: 1812 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1813 qd_idx = pd_idx + 1; 1814 if (pd_idx == raid_disks-1) { 1815 (*dd_idx)++; /* Q D D D P */ 1816 qd_idx = 0; 1817 } else if (*dd_idx >= pd_idx) 1818 (*dd_idx) += 2; /* D D P Q D */ 1819 break; 1820 case ALGORITHM_RIGHT_ASYMMETRIC: 1821 pd_idx = sector_div(stripe2, raid_disks); 1822 qd_idx = pd_idx + 1; 1823 if (pd_idx == raid_disks-1) { 1824 (*dd_idx)++; /* Q D D D P */ 1825 qd_idx = 0; 1826 } else if (*dd_idx >= pd_idx) 1827 (*dd_idx) += 2; /* D D P Q D */ 1828 break; 1829 case ALGORITHM_LEFT_SYMMETRIC: 1830 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1831 qd_idx = (pd_idx + 1) % raid_disks; 1832 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1833 break; 1834 case ALGORITHM_RIGHT_SYMMETRIC: 1835 pd_idx = sector_div(stripe2, raid_disks); 1836 qd_idx = (pd_idx + 1) % raid_disks; 1837 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 1838 break; 1839 1840 case ALGORITHM_PARITY_0: 1841 pd_idx = 0; 1842 qd_idx = 1; 1843 (*dd_idx) += 2; 1844 break; 1845 case ALGORITHM_PARITY_N: 1846 pd_idx = data_disks; 1847 qd_idx = data_disks + 1; 1848 break; 1849 1850 case ALGORITHM_ROTATING_ZERO_RESTART: 1851 /* Exactly the same as RIGHT_ASYMMETRIC, but or 1852 * of blocks for computing Q is different. 1853 */ 1854 pd_idx = sector_div(stripe2, raid_disks); 1855 qd_idx = pd_idx + 1; 1856 if (pd_idx == raid_disks-1) { 1857 (*dd_idx)++; /* Q D D D P */ 1858 qd_idx = 0; 1859 } else if (*dd_idx >= pd_idx) 1860 (*dd_idx) += 2; /* D D P Q D */ 1861 ddf_layout = 1; 1862 break; 1863 1864 case ALGORITHM_ROTATING_N_RESTART: 1865 /* Same a left_asymmetric, by first stripe is 1866 * D D D P Q rather than 1867 * Q D D D P 1868 */ 1869 stripe2 += 1; 1870 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1871 qd_idx = pd_idx + 1; 1872 if (pd_idx == raid_disks-1) { 1873 (*dd_idx)++; /* Q D D D P */ 1874 qd_idx = 0; 1875 } else if (*dd_idx >= pd_idx) 1876 (*dd_idx) += 2; /* D D P Q D */ 1877 ddf_layout = 1; 1878 break; 1879 1880 case ALGORITHM_ROTATING_N_CONTINUE: 1881 /* Same as left_symmetric but Q is before P */ 1882 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 1883 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 1884 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 1885 ddf_layout = 1; 1886 break; 1887 1888 case ALGORITHM_LEFT_ASYMMETRIC_6: 1889 /* RAID5 left_asymmetric, with Q on last device */ 1890 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 1891 if (*dd_idx >= pd_idx) 1892 (*dd_idx)++; 1893 qd_idx = raid_disks - 1; 1894 break; 1895 1896 case ALGORITHM_RIGHT_ASYMMETRIC_6: 1897 pd_idx = sector_div(stripe2, raid_disks-1); 1898 if (*dd_idx >= pd_idx) 1899 (*dd_idx)++; 1900 qd_idx = raid_disks - 1; 1901 break; 1902 1903 case ALGORITHM_LEFT_SYMMETRIC_6: 1904 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 1905 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1906 qd_idx = raid_disks - 1; 1907 break; 1908 1909 case ALGORITHM_RIGHT_SYMMETRIC_6: 1910 pd_idx = sector_div(stripe2, raid_disks-1); 1911 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 1912 qd_idx = raid_disks - 1; 1913 break; 1914 1915 case ALGORITHM_PARITY_0_6: 1916 pd_idx = 0; 1917 (*dd_idx)++; 1918 qd_idx = raid_disks - 1; 1919 break; 1920 1921 default: 1922 BUG(); 1923 } 1924 break; 1925 } 1926 1927 if (sh) { 1928 sh->pd_idx = pd_idx; 1929 sh->qd_idx = qd_idx; 1930 sh->ddf_layout = ddf_layout; 1931 } 1932 /* 1933 * Finally, compute the new sector number 1934 */ 1935 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 1936 return new_sector; 1937 } 1938 1939 1940 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 1941 { 1942 raid5_conf_t *conf = sh->raid_conf; 1943 int raid_disks = sh->disks; 1944 int data_disks = raid_disks - conf->max_degraded; 1945 sector_t new_sector = sh->sector, check; 1946 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 1947 : conf->chunk_sectors; 1948 int algorithm = previous ? conf->prev_algo 1949 : conf->algorithm; 1950 sector_t stripe; 1951 int chunk_offset; 1952 sector_t chunk_number; 1953 int dummy1, dd_idx = i; 1954 sector_t r_sector; 1955 struct stripe_head sh2; 1956 1957 1958 chunk_offset = sector_div(new_sector, sectors_per_chunk); 1959 stripe = new_sector; 1960 1961 if (i == sh->pd_idx) 1962 return 0; 1963 switch(conf->level) { 1964 case 4: break; 1965 case 5: 1966 switch (algorithm) { 1967 case ALGORITHM_LEFT_ASYMMETRIC: 1968 case ALGORITHM_RIGHT_ASYMMETRIC: 1969 if (i > sh->pd_idx) 1970 i--; 1971 break; 1972 case ALGORITHM_LEFT_SYMMETRIC: 1973 case ALGORITHM_RIGHT_SYMMETRIC: 1974 if (i < sh->pd_idx) 1975 i += raid_disks; 1976 i -= (sh->pd_idx + 1); 1977 break; 1978 case ALGORITHM_PARITY_0: 1979 i -= 1; 1980 break; 1981 case ALGORITHM_PARITY_N: 1982 break; 1983 default: 1984 BUG(); 1985 } 1986 break; 1987 case 6: 1988 if (i == sh->qd_idx) 1989 return 0; /* It is the Q disk */ 1990 switch (algorithm) { 1991 case ALGORITHM_LEFT_ASYMMETRIC: 1992 case ALGORITHM_RIGHT_ASYMMETRIC: 1993 case ALGORITHM_ROTATING_ZERO_RESTART: 1994 case ALGORITHM_ROTATING_N_RESTART: 1995 if (sh->pd_idx == raid_disks-1) 1996 i--; /* Q D D D P */ 1997 else if (i > sh->pd_idx) 1998 i -= 2; /* D D P Q D */ 1999 break; 2000 case ALGORITHM_LEFT_SYMMETRIC: 2001 case ALGORITHM_RIGHT_SYMMETRIC: 2002 if (sh->pd_idx == raid_disks-1) 2003 i--; /* Q D D D P */ 2004 else { 2005 /* D D P Q D */ 2006 if (i < sh->pd_idx) 2007 i += raid_disks; 2008 i -= (sh->pd_idx + 2); 2009 } 2010 break; 2011 case ALGORITHM_PARITY_0: 2012 i -= 2; 2013 break; 2014 case ALGORITHM_PARITY_N: 2015 break; 2016 case ALGORITHM_ROTATING_N_CONTINUE: 2017 /* Like left_symmetric, but P is before Q */ 2018 if (sh->pd_idx == 0) 2019 i--; /* P D D D Q */ 2020 else { 2021 /* D D Q P D */ 2022 if (i < sh->pd_idx) 2023 i += raid_disks; 2024 i -= (sh->pd_idx + 1); 2025 } 2026 break; 2027 case ALGORITHM_LEFT_ASYMMETRIC_6: 2028 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2029 if (i > sh->pd_idx) 2030 i--; 2031 break; 2032 case ALGORITHM_LEFT_SYMMETRIC_6: 2033 case ALGORITHM_RIGHT_SYMMETRIC_6: 2034 if (i < sh->pd_idx) 2035 i += data_disks + 1; 2036 i -= (sh->pd_idx + 1); 2037 break; 2038 case ALGORITHM_PARITY_0_6: 2039 i -= 1; 2040 break; 2041 default: 2042 BUG(); 2043 } 2044 break; 2045 } 2046 2047 chunk_number = stripe * data_disks + i; 2048 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2049 2050 check = raid5_compute_sector(conf, r_sector, 2051 previous, &dummy1, &sh2); 2052 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2053 || sh2.qd_idx != sh->qd_idx) { 2054 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2055 mdname(conf->mddev)); 2056 return 0; 2057 } 2058 return r_sector; 2059 } 2060 2061 2062 static void 2063 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2064 int rcw, int expand) 2065 { 2066 int i, pd_idx = sh->pd_idx, disks = sh->disks; 2067 raid5_conf_t *conf = sh->raid_conf; 2068 int level = conf->level; 2069 2070 if (rcw) { 2071 /* if we are not expanding this is a proper write request, and 2072 * there will be bios with new data to be drained into the 2073 * stripe cache 2074 */ 2075 if (!expand) { 2076 sh->reconstruct_state = reconstruct_state_drain_run; 2077 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2078 } else 2079 sh->reconstruct_state = reconstruct_state_run; 2080 2081 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2082 2083 for (i = disks; i--; ) { 2084 struct r5dev *dev = &sh->dev[i]; 2085 2086 if (dev->towrite) { 2087 set_bit(R5_LOCKED, &dev->flags); 2088 set_bit(R5_Wantdrain, &dev->flags); 2089 if (!expand) 2090 clear_bit(R5_UPTODATE, &dev->flags); 2091 s->locked++; 2092 } 2093 } 2094 if (s->locked + conf->max_degraded == disks) 2095 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2096 atomic_inc(&conf->pending_full_writes); 2097 } else { 2098 BUG_ON(level == 6); 2099 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2100 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2101 2102 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2103 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2104 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2105 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2106 2107 for (i = disks; i--; ) { 2108 struct r5dev *dev = &sh->dev[i]; 2109 if (i == pd_idx) 2110 continue; 2111 2112 if (dev->towrite && 2113 (test_bit(R5_UPTODATE, &dev->flags) || 2114 test_bit(R5_Wantcompute, &dev->flags))) { 2115 set_bit(R5_Wantdrain, &dev->flags); 2116 set_bit(R5_LOCKED, &dev->flags); 2117 clear_bit(R5_UPTODATE, &dev->flags); 2118 s->locked++; 2119 } 2120 } 2121 } 2122 2123 /* keep the parity disk(s) locked while asynchronous operations 2124 * are in flight 2125 */ 2126 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2127 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2128 s->locked++; 2129 2130 if (level == 6) { 2131 int qd_idx = sh->qd_idx; 2132 struct r5dev *dev = &sh->dev[qd_idx]; 2133 2134 set_bit(R5_LOCKED, &dev->flags); 2135 clear_bit(R5_UPTODATE, &dev->flags); 2136 s->locked++; 2137 } 2138 2139 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2140 __func__, (unsigned long long)sh->sector, 2141 s->locked, s->ops_request); 2142 } 2143 2144 /* 2145 * Each stripe/dev can have one or more bion attached. 2146 * toread/towrite point to the first in a chain. 2147 * The bi_next chain must be in order. 2148 */ 2149 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2150 { 2151 struct bio **bip; 2152 raid5_conf_t *conf = sh->raid_conf; 2153 int firstwrite=0; 2154 2155 pr_debug("adding bh b#%llu to stripe s#%llu\n", 2156 (unsigned long long)bi->bi_sector, 2157 (unsigned long long)sh->sector); 2158 2159 2160 spin_lock(&sh->lock); 2161 spin_lock_irq(&conf->device_lock); 2162 if (forwrite) { 2163 bip = &sh->dev[dd_idx].towrite; 2164 if (*bip == NULL && sh->dev[dd_idx].written == NULL) 2165 firstwrite = 1; 2166 } else 2167 bip = &sh->dev[dd_idx].toread; 2168 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 2169 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector) 2170 goto overlap; 2171 bip = & (*bip)->bi_next; 2172 } 2173 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9)) 2174 goto overlap; 2175 2176 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2177 if (*bip) 2178 bi->bi_next = *bip; 2179 *bip = bi; 2180 bi->bi_phys_segments++; 2181 spin_unlock_irq(&conf->device_lock); 2182 spin_unlock(&sh->lock); 2183 2184 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2185 (unsigned long long)bi->bi_sector, 2186 (unsigned long long)sh->sector, dd_idx); 2187 2188 if (conf->mddev->bitmap && firstwrite) { 2189 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2190 STRIPE_SECTORS, 0); 2191 sh->bm_seq = conf->seq_flush+1; 2192 set_bit(STRIPE_BIT_DELAY, &sh->state); 2193 } 2194 2195 if (forwrite) { 2196 /* check if page is covered */ 2197 sector_t sector = sh->dev[dd_idx].sector; 2198 for (bi=sh->dev[dd_idx].towrite; 2199 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2200 bi && bi->bi_sector <= sector; 2201 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2202 if (bi->bi_sector + (bi->bi_size>>9) >= sector) 2203 sector = bi->bi_sector + (bi->bi_size>>9); 2204 } 2205 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2206 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2207 } 2208 return 1; 2209 2210 overlap: 2211 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2212 spin_unlock_irq(&conf->device_lock); 2213 spin_unlock(&sh->lock); 2214 return 0; 2215 } 2216 2217 static void end_reshape(raid5_conf_t *conf); 2218 2219 static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous, 2220 struct stripe_head *sh) 2221 { 2222 int sectors_per_chunk = 2223 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2224 int dd_idx; 2225 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2226 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2227 2228 raid5_compute_sector(conf, 2229 stripe * (disks - conf->max_degraded) 2230 *sectors_per_chunk + chunk_offset, 2231 previous, 2232 &dd_idx, sh); 2233 } 2234 2235 static void 2236 handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh, 2237 struct stripe_head_state *s, int disks, 2238 struct bio **return_bi) 2239 { 2240 int i; 2241 for (i = disks; i--; ) { 2242 struct bio *bi; 2243 int bitmap_end = 0; 2244 2245 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2246 mdk_rdev_t *rdev; 2247 rcu_read_lock(); 2248 rdev = rcu_dereference(conf->disks[i].rdev); 2249 if (rdev && test_bit(In_sync, &rdev->flags)) 2250 /* multiple read failures in one stripe */ 2251 md_error(conf->mddev, rdev); 2252 rcu_read_unlock(); 2253 } 2254 spin_lock_irq(&conf->device_lock); 2255 /* fail all writes first */ 2256 bi = sh->dev[i].towrite; 2257 sh->dev[i].towrite = NULL; 2258 if (bi) { 2259 s->to_write--; 2260 bitmap_end = 1; 2261 } 2262 2263 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2264 wake_up(&conf->wait_for_overlap); 2265 2266 while (bi && bi->bi_sector < 2267 sh->dev[i].sector + STRIPE_SECTORS) { 2268 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2269 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2270 if (!raid5_dec_bi_phys_segments(bi)) { 2271 md_write_end(conf->mddev); 2272 bi->bi_next = *return_bi; 2273 *return_bi = bi; 2274 } 2275 bi = nextbi; 2276 } 2277 /* and fail all 'written' */ 2278 bi = sh->dev[i].written; 2279 sh->dev[i].written = NULL; 2280 if (bi) bitmap_end = 1; 2281 while (bi && bi->bi_sector < 2282 sh->dev[i].sector + STRIPE_SECTORS) { 2283 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2284 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2285 if (!raid5_dec_bi_phys_segments(bi)) { 2286 md_write_end(conf->mddev); 2287 bi->bi_next = *return_bi; 2288 *return_bi = bi; 2289 } 2290 bi = bi2; 2291 } 2292 2293 /* fail any reads if this device is non-operational and 2294 * the data has not reached the cache yet. 2295 */ 2296 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2297 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2298 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2299 bi = sh->dev[i].toread; 2300 sh->dev[i].toread = NULL; 2301 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2302 wake_up(&conf->wait_for_overlap); 2303 if (bi) s->to_read--; 2304 while (bi && bi->bi_sector < 2305 sh->dev[i].sector + STRIPE_SECTORS) { 2306 struct bio *nextbi = 2307 r5_next_bio(bi, sh->dev[i].sector); 2308 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2309 if (!raid5_dec_bi_phys_segments(bi)) { 2310 bi->bi_next = *return_bi; 2311 *return_bi = bi; 2312 } 2313 bi = nextbi; 2314 } 2315 } 2316 spin_unlock_irq(&conf->device_lock); 2317 if (bitmap_end) 2318 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2319 STRIPE_SECTORS, 0, 0); 2320 } 2321 2322 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2323 if (atomic_dec_and_test(&conf->pending_full_writes)) 2324 md_wakeup_thread(conf->mddev->thread); 2325 } 2326 2327 /* fetch_block5 - checks the given member device to see if its data needs 2328 * to be read or computed to satisfy a request. 2329 * 2330 * Returns 1 when no more member devices need to be checked, otherwise returns 2331 * 0 to tell the loop in handle_stripe_fill5 to continue 2332 */ 2333 static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s, 2334 int disk_idx, int disks) 2335 { 2336 struct r5dev *dev = &sh->dev[disk_idx]; 2337 struct r5dev *failed_dev = &sh->dev[s->failed_num]; 2338 2339 /* is the data in this block needed, and can we get it? */ 2340 if (!test_bit(R5_LOCKED, &dev->flags) && 2341 !test_bit(R5_UPTODATE, &dev->flags) && 2342 (dev->toread || 2343 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2344 s->syncing || s->expanding || 2345 (s->failed && 2346 (failed_dev->toread || 2347 (failed_dev->towrite && 2348 !test_bit(R5_OVERWRITE, &failed_dev->flags)))))) { 2349 /* We would like to get this block, possibly by computing it, 2350 * otherwise read it if the backing disk is insync 2351 */ 2352 if ((s->uptodate == disks - 1) && 2353 (s->failed && disk_idx == s->failed_num)) { 2354 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2355 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2356 set_bit(R5_Wantcompute, &dev->flags); 2357 sh->ops.target = disk_idx; 2358 sh->ops.target2 = -1; 2359 s->req_compute = 1; 2360 /* Careful: from this point on 'uptodate' is in the eye 2361 * of raid_run_ops which services 'compute' operations 2362 * before writes. R5_Wantcompute flags a block that will 2363 * be R5_UPTODATE by the time it is needed for a 2364 * subsequent operation. 2365 */ 2366 s->uptodate++; 2367 return 1; /* uptodate + compute == disks */ 2368 } else if (test_bit(R5_Insync, &dev->flags)) { 2369 set_bit(R5_LOCKED, &dev->flags); 2370 set_bit(R5_Wantread, &dev->flags); 2371 s->locked++; 2372 pr_debug("Reading block %d (sync=%d)\n", disk_idx, 2373 s->syncing); 2374 } 2375 } 2376 2377 return 0; 2378 } 2379 2380 /** 2381 * handle_stripe_fill5 - read or compute data to satisfy pending requests. 2382 */ 2383 static void handle_stripe_fill5(struct stripe_head *sh, 2384 struct stripe_head_state *s, int disks) 2385 { 2386 int i; 2387 2388 /* look for blocks to read/compute, skip this if a compute 2389 * is already in flight, or if the stripe contents are in the 2390 * midst of changing due to a write 2391 */ 2392 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2393 !sh->reconstruct_state) 2394 for (i = disks; i--; ) 2395 if (fetch_block5(sh, s, i, disks)) 2396 break; 2397 set_bit(STRIPE_HANDLE, &sh->state); 2398 } 2399 2400 /* fetch_block6 - checks the given member device to see if its data needs 2401 * to be read or computed to satisfy a request. 2402 * 2403 * Returns 1 when no more member devices need to be checked, otherwise returns 2404 * 0 to tell the loop in handle_stripe_fill6 to continue 2405 */ 2406 static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s, 2407 struct r6_state *r6s, int disk_idx, int disks) 2408 { 2409 struct r5dev *dev = &sh->dev[disk_idx]; 2410 struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]], 2411 &sh->dev[r6s->failed_num[1]] }; 2412 2413 if (!test_bit(R5_LOCKED, &dev->flags) && 2414 !test_bit(R5_UPTODATE, &dev->flags) && 2415 (dev->toread || 2416 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2417 s->syncing || s->expanding || 2418 (s->failed >= 1 && 2419 (fdev[0]->toread || s->to_write)) || 2420 (s->failed >= 2 && 2421 (fdev[1]->toread || s->to_write)))) { 2422 /* we would like to get this block, possibly by computing it, 2423 * otherwise read it if the backing disk is insync 2424 */ 2425 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2426 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2427 if ((s->uptodate == disks - 1) && 2428 (s->failed && (disk_idx == r6s->failed_num[0] || 2429 disk_idx == r6s->failed_num[1]))) { 2430 /* have disk failed, and we're requested to fetch it; 2431 * do compute it 2432 */ 2433 pr_debug("Computing stripe %llu block %d\n", 2434 (unsigned long long)sh->sector, disk_idx); 2435 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2436 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2437 set_bit(R5_Wantcompute, &dev->flags); 2438 sh->ops.target = disk_idx; 2439 sh->ops.target2 = -1; /* no 2nd target */ 2440 s->req_compute = 1; 2441 s->uptodate++; 2442 return 1; 2443 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2444 /* Computing 2-failure is *very* expensive; only 2445 * do it if failed >= 2 2446 */ 2447 int other; 2448 for (other = disks; other--; ) { 2449 if (other == disk_idx) 2450 continue; 2451 if (!test_bit(R5_UPTODATE, 2452 &sh->dev[other].flags)) 2453 break; 2454 } 2455 BUG_ON(other < 0); 2456 pr_debug("Computing stripe %llu blocks %d,%d\n", 2457 (unsigned long long)sh->sector, 2458 disk_idx, other); 2459 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2460 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2461 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2462 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2463 sh->ops.target = disk_idx; 2464 sh->ops.target2 = other; 2465 s->uptodate += 2; 2466 s->req_compute = 1; 2467 return 1; 2468 } else if (test_bit(R5_Insync, &dev->flags)) { 2469 set_bit(R5_LOCKED, &dev->flags); 2470 set_bit(R5_Wantread, &dev->flags); 2471 s->locked++; 2472 pr_debug("Reading block %d (sync=%d)\n", 2473 disk_idx, s->syncing); 2474 } 2475 } 2476 2477 return 0; 2478 } 2479 2480 /** 2481 * handle_stripe_fill6 - read or compute data to satisfy pending requests. 2482 */ 2483 static void handle_stripe_fill6(struct stripe_head *sh, 2484 struct stripe_head_state *s, struct r6_state *r6s, 2485 int disks) 2486 { 2487 int i; 2488 2489 /* look for blocks to read/compute, skip this if a compute 2490 * is already in flight, or if the stripe contents are in the 2491 * midst of changing due to a write 2492 */ 2493 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2494 !sh->reconstruct_state) 2495 for (i = disks; i--; ) 2496 if (fetch_block6(sh, s, r6s, i, disks)) 2497 break; 2498 set_bit(STRIPE_HANDLE, &sh->state); 2499 } 2500 2501 2502 /* handle_stripe_clean_event 2503 * any written block on an uptodate or failed drive can be returned. 2504 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2505 * never LOCKED, so we don't need to test 'failed' directly. 2506 */ 2507 static void handle_stripe_clean_event(raid5_conf_t *conf, 2508 struct stripe_head *sh, int disks, struct bio **return_bi) 2509 { 2510 int i; 2511 struct r5dev *dev; 2512 2513 for (i = disks; i--; ) 2514 if (sh->dev[i].written) { 2515 dev = &sh->dev[i]; 2516 if (!test_bit(R5_LOCKED, &dev->flags) && 2517 test_bit(R5_UPTODATE, &dev->flags)) { 2518 /* We can return any write requests */ 2519 struct bio *wbi, *wbi2; 2520 int bitmap_end = 0; 2521 pr_debug("Return write for disc %d\n", i); 2522 spin_lock_irq(&conf->device_lock); 2523 wbi = dev->written; 2524 dev->written = NULL; 2525 while (wbi && wbi->bi_sector < 2526 dev->sector + STRIPE_SECTORS) { 2527 wbi2 = r5_next_bio(wbi, dev->sector); 2528 if (!raid5_dec_bi_phys_segments(wbi)) { 2529 md_write_end(conf->mddev); 2530 wbi->bi_next = *return_bi; 2531 *return_bi = wbi; 2532 } 2533 wbi = wbi2; 2534 } 2535 if (dev->towrite == NULL) 2536 bitmap_end = 1; 2537 spin_unlock_irq(&conf->device_lock); 2538 if (bitmap_end) 2539 bitmap_endwrite(conf->mddev->bitmap, 2540 sh->sector, 2541 STRIPE_SECTORS, 2542 !test_bit(STRIPE_DEGRADED, &sh->state), 2543 0); 2544 } 2545 } 2546 2547 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2548 if (atomic_dec_and_test(&conf->pending_full_writes)) 2549 md_wakeup_thread(conf->mddev->thread); 2550 } 2551 2552 static void handle_stripe_dirtying5(raid5_conf_t *conf, 2553 struct stripe_head *sh, struct stripe_head_state *s, int disks) 2554 { 2555 int rmw = 0, rcw = 0, i; 2556 for (i = disks; i--; ) { 2557 /* would I have to read this buffer for read_modify_write */ 2558 struct r5dev *dev = &sh->dev[i]; 2559 if ((dev->towrite || i == sh->pd_idx) && 2560 !test_bit(R5_LOCKED, &dev->flags) && 2561 !(test_bit(R5_UPTODATE, &dev->flags) || 2562 test_bit(R5_Wantcompute, &dev->flags))) { 2563 if (test_bit(R5_Insync, &dev->flags)) 2564 rmw++; 2565 else 2566 rmw += 2*disks; /* cannot read it */ 2567 } 2568 /* Would I have to read this buffer for reconstruct_write */ 2569 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2570 !test_bit(R5_LOCKED, &dev->flags) && 2571 !(test_bit(R5_UPTODATE, &dev->flags) || 2572 test_bit(R5_Wantcompute, &dev->flags))) { 2573 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2574 else 2575 rcw += 2*disks; 2576 } 2577 } 2578 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2579 (unsigned long long)sh->sector, rmw, rcw); 2580 set_bit(STRIPE_HANDLE, &sh->state); 2581 if (rmw < rcw && rmw > 0) 2582 /* prefer read-modify-write, but need to get some data */ 2583 for (i = disks; i--; ) { 2584 struct r5dev *dev = &sh->dev[i]; 2585 if ((dev->towrite || i == sh->pd_idx) && 2586 !test_bit(R5_LOCKED, &dev->flags) && 2587 !(test_bit(R5_UPTODATE, &dev->flags) || 2588 test_bit(R5_Wantcompute, &dev->flags)) && 2589 test_bit(R5_Insync, &dev->flags)) { 2590 if ( 2591 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2592 pr_debug("Read_old block " 2593 "%d for r-m-w\n", i); 2594 set_bit(R5_LOCKED, &dev->flags); 2595 set_bit(R5_Wantread, &dev->flags); 2596 s->locked++; 2597 } else { 2598 set_bit(STRIPE_DELAYED, &sh->state); 2599 set_bit(STRIPE_HANDLE, &sh->state); 2600 } 2601 } 2602 } 2603 if (rcw <= rmw && rcw > 0) 2604 /* want reconstruct write, but need to get some data */ 2605 for (i = disks; i--; ) { 2606 struct r5dev *dev = &sh->dev[i]; 2607 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2608 i != sh->pd_idx && 2609 !test_bit(R5_LOCKED, &dev->flags) && 2610 !(test_bit(R5_UPTODATE, &dev->flags) || 2611 test_bit(R5_Wantcompute, &dev->flags)) && 2612 test_bit(R5_Insync, &dev->flags)) { 2613 if ( 2614 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2615 pr_debug("Read_old block " 2616 "%d for Reconstruct\n", i); 2617 set_bit(R5_LOCKED, &dev->flags); 2618 set_bit(R5_Wantread, &dev->flags); 2619 s->locked++; 2620 } else { 2621 set_bit(STRIPE_DELAYED, &sh->state); 2622 set_bit(STRIPE_HANDLE, &sh->state); 2623 } 2624 } 2625 } 2626 /* now if nothing is locked, and if we have enough data, 2627 * we can start a write request 2628 */ 2629 /* since handle_stripe can be called at any time we need to handle the 2630 * case where a compute block operation has been submitted and then a 2631 * subsequent call wants to start a write request. raid_run_ops only 2632 * handles the case where compute block and reconstruct are requested 2633 * simultaneously. If this is not the case then new writes need to be 2634 * held off until the compute completes. 2635 */ 2636 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2637 (s->locked == 0 && (rcw == 0 || rmw == 0) && 2638 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 2639 schedule_reconstruction(sh, s, rcw == 0, 0); 2640 } 2641 2642 static void handle_stripe_dirtying6(raid5_conf_t *conf, 2643 struct stripe_head *sh, struct stripe_head_state *s, 2644 struct r6_state *r6s, int disks) 2645 { 2646 int rcw = 0, pd_idx = sh->pd_idx, i; 2647 int qd_idx = sh->qd_idx; 2648 2649 set_bit(STRIPE_HANDLE, &sh->state); 2650 for (i = disks; i--; ) { 2651 struct r5dev *dev = &sh->dev[i]; 2652 /* check if we haven't enough data */ 2653 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2654 i != pd_idx && i != qd_idx && 2655 !test_bit(R5_LOCKED, &dev->flags) && 2656 !(test_bit(R5_UPTODATE, &dev->flags) || 2657 test_bit(R5_Wantcompute, &dev->flags))) { 2658 rcw++; 2659 if (!test_bit(R5_Insync, &dev->flags)) 2660 continue; /* it's a failed drive */ 2661 2662 if ( 2663 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2664 pr_debug("Read_old stripe %llu " 2665 "block %d for Reconstruct\n", 2666 (unsigned long long)sh->sector, i); 2667 set_bit(R5_LOCKED, &dev->flags); 2668 set_bit(R5_Wantread, &dev->flags); 2669 s->locked++; 2670 } else { 2671 pr_debug("Request delayed stripe %llu " 2672 "block %d for Reconstruct\n", 2673 (unsigned long long)sh->sector, i); 2674 set_bit(STRIPE_DELAYED, &sh->state); 2675 set_bit(STRIPE_HANDLE, &sh->state); 2676 } 2677 } 2678 } 2679 /* now if nothing is locked, and if we have enough data, we can start a 2680 * write request 2681 */ 2682 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 2683 s->locked == 0 && rcw == 0 && 2684 !test_bit(STRIPE_BIT_DELAY, &sh->state)) { 2685 schedule_reconstruction(sh, s, 1, 0); 2686 } 2687 } 2688 2689 static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh, 2690 struct stripe_head_state *s, int disks) 2691 { 2692 struct r5dev *dev = NULL; 2693 2694 set_bit(STRIPE_HANDLE, &sh->state); 2695 2696 switch (sh->check_state) { 2697 case check_state_idle: 2698 /* start a new check operation if there are no failures */ 2699 if (s->failed == 0) { 2700 BUG_ON(s->uptodate != disks); 2701 sh->check_state = check_state_run; 2702 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2703 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2704 s->uptodate--; 2705 break; 2706 } 2707 dev = &sh->dev[s->failed_num]; 2708 /* fall through */ 2709 case check_state_compute_result: 2710 sh->check_state = check_state_idle; 2711 if (!dev) 2712 dev = &sh->dev[sh->pd_idx]; 2713 2714 /* check that a write has not made the stripe insync */ 2715 if (test_bit(STRIPE_INSYNC, &sh->state)) 2716 break; 2717 2718 /* either failed parity check, or recovery is happening */ 2719 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 2720 BUG_ON(s->uptodate != disks); 2721 2722 set_bit(R5_LOCKED, &dev->flags); 2723 s->locked++; 2724 set_bit(R5_Wantwrite, &dev->flags); 2725 2726 clear_bit(STRIPE_DEGRADED, &sh->state); 2727 set_bit(STRIPE_INSYNC, &sh->state); 2728 break; 2729 case check_state_run: 2730 break; /* we will be called again upon completion */ 2731 case check_state_check_result: 2732 sh->check_state = check_state_idle; 2733 2734 /* if a failure occurred during the check operation, leave 2735 * STRIPE_INSYNC not set and let the stripe be handled again 2736 */ 2737 if (s->failed) 2738 break; 2739 2740 /* handle a successful check operation, if parity is correct 2741 * we are done. Otherwise update the mismatch count and repair 2742 * parity if !MD_RECOVERY_CHECK 2743 */ 2744 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 2745 /* parity is correct (on disc, 2746 * not in buffer any more) 2747 */ 2748 set_bit(STRIPE_INSYNC, &sh->state); 2749 else { 2750 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2751 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2752 /* don't try to repair!! */ 2753 set_bit(STRIPE_INSYNC, &sh->state); 2754 else { 2755 sh->check_state = check_state_compute_run; 2756 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2757 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2758 set_bit(R5_Wantcompute, 2759 &sh->dev[sh->pd_idx].flags); 2760 sh->ops.target = sh->pd_idx; 2761 sh->ops.target2 = -1; 2762 s->uptodate++; 2763 } 2764 } 2765 break; 2766 case check_state_compute_run: 2767 break; 2768 default: 2769 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2770 __func__, sh->check_state, 2771 (unsigned long long) sh->sector); 2772 BUG(); 2773 } 2774 } 2775 2776 2777 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh, 2778 struct stripe_head_state *s, 2779 struct r6_state *r6s, int disks) 2780 { 2781 int pd_idx = sh->pd_idx; 2782 int qd_idx = sh->qd_idx; 2783 struct r5dev *dev; 2784 2785 set_bit(STRIPE_HANDLE, &sh->state); 2786 2787 BUG_ON(s->failed > 2); 2788 2789 /* Want to check and possibly repair P and Q. 2790 * However there could be one 'failed' device, in which 2791 * case we can only check one of them, possibly using the 2792 * other to generate missing data 2793 */ 2794 2795 switch (sh->check_state) { 2796 case check_state_idle: 2797 /* start a new check operation if there are < 2 failures */ 2798 if (s->failed == r6s->q_failed) { 2799 /* The only possible failed device holds Q, so it 2800 * makes sense to check P (If anything else were failed, 2801 * we would have used P to recreate it). 2802 */ 2803 sh->check_state = check_state_run; 2804 } 2805 if (!r6s->q_failed && s->failed < 2) { 2806 /* Q is not failed, and we didn't use it to generate 2807 * anything, so it makes sense to check it 2808 */ 2809 if (sh->check_state == check_state_run) 2810 sh->check_state = check_state_run_pq; 2811 else 2812 sh->check_state = check_state_run_q; 2813 } 2814 2815 /* discard potentially stale zero_sum_result */ 2816 sh->ops.zero_sum_result = 0; 2817 2818 if (sh->check_state == check_state_run) { 2819 /* async_xor_zero_sum destroys the contents of P */ 2820 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2821 s->uptodate--; 2822 } 2823 if (sh->check_state >= check_state_run && 2824 sh->check_state <= check_state_run_pq) { 2825 /* async_syndrome_zero_sum preserves P and Q, so 2826 * no need to mark them !uptodate here 2827 */ 2828 set_bit(STRIPE_OP_CHECK, &s->ops_request); 2829 break; 2830 } 2831 2832 /* we have 2-disk failure */ 2833 BUG_ON(s->failed != 2); 2834 /* fall through */ 2835 case check_state_compute_result: 2836 sh->check_state = check_state_idle; 2837 2838 /* check that a write has not made the stripe insync */ 2839 if (test_bit(STRIPE_INSYNC, &sh->state)) 2840 break; 2841 2842 /* now write out any block on a failed drive, 2843 * or P or Q if they were recomputed 2844 */ 2845 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 2846 if (s->failed == 2) { 2847 dev = &sh->dev[r6s->failed_num[1]]; 2848 s->locked++; 2849 set_bit(R5_LOCKED, &dev->flags); 2850 set_bit(R5_Wantwrite, &dev->flags); 2851 } 2852 if (s->failed >= 1) { 2853 dev = &sh->dev[r6s->failed_num[0]]; 2854 s->locked++; 2855 set_bit(R5_LOCKED, &dev->flags); 2856 set_bit(R5_Wantwrite, &dev->flags); 2857 } 2858 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2859 dev = &sh->dev[pd_idx]; 2860 s->locked++; 2861 set_bit(R5_LOCKED, &dev->flags); 2862 set_bit(R5_Wantwrite, &dev->flags); 2863 } 2864 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2865 dev = &sh->dev[qd_idx]; 2866 s->locked++; 2867 set_bit(R5_LOCKED, &dev->flags); 2868 set_bit(R5_Wantwrite, &dev->flags); 2869 } 2870 clear_bit(STRIPE_DEGRADED, &sh->state); 2871 2872 set_bit(STRIPE_INSYNC, &sh->state); 2873 break; 2874 case check_state_run: 2875 case check_state_run_q: 2876 case check_state_run_pq: 2877 break; /* we will be called again upon completion */ 2878 case check_state_check_result: 2879 sh->check_state = check_state_idle; 2880 2881 /* handle a successful check operation, if parity is correct 2882 * we are done. Otherwise update the mismatch count and repair 2883 * parity if !MD_RECOVERY_CHECK 2884 */ 2885 if (sh->ops.zero_sum_result == 0) { 2886 /* both parities are correct */ 2887 if (!s->failed) 2888 set_bit(STRIPE_INSYNC, &sh->state); 2889 else { 2890 /* in contrast to the raid5 case we can validate 2891 * parity, but still have a failure to write 2892 * back 2893 */ 2894 sh->check_state = check_state_compute_result; 2895 /* Returning at this point means that we may go 2896 * off and bring p and/or q uptodate again so 2897 * we make sure to check zero_sum_result again 2898 * to verify if p or q need writeback 2899 */ 2900 } 2901 } else { 2902 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2903 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2904 /* don't try to repair!! */ 2905 set_bit(STRIPE_INSYNC, &sh->state); 2906 else { 2907 int *target = &sh->ops.target; 2908 2909 sh->ops.target = -1; 2910 sh->ops.target2 = -1; 2911 sh->check_state = check_state_compute_run; 2912 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2913 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2914 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 2915 set_bit(R5_Wantcompute, 2916 &sh->dev[pd_idx].flags); 2917 *target = pd_idx; 2918 target = &sh->ops.target2; 2919 s->uptodate++; 2920 } 2921 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 2922 set_bit(R5_Wantcompute, 2923 &sh->dev[qd_idx].flags); 2924 *target = qd_idx; 2925 s->uptodate++; 2926 } 2927 } 2928 } 2929 break; 2930 case check_state_compute_run: 2931 break; 2932 default: 2933 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 2934 __func__, sh->check_state, 2935 (unsigned long long) sh->sector); 2936 BUG(); 2937 } 2938 } 2939 2940 static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh, 2941 struct r6_state *r6s) 2942 { 2943 int i; 2944 2945 /* We have read all the blocks in this stripe and now we need to 2946 * copy some of them into a target stripe for expand. 2947 */ 2948 struct dma_async_tx_descriptor *tx = NULL; 2949 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2950 for (i = 0; i < sh->disks; i++) 2951 if (i != sh->pd_idx && i != sh->qd_idx) { 2952 int dd_idx, j; 2953 struct stripe_head *sh2; 2954 struct async_submit_ctl submit; 2955 2956 sector_t bn = compute_blocknr(sh, i, 1); 2957 sector_t s = raid5_compute_sector(conf, bn, 0, 2958 &dd_idx, NULL); 2959 sh2 = get_active_stripe(conf, s, 0, 1, 1); 2960 if (sh2 == NULL) 2961 /* so far only the early blocks of this stripe 2962 * have been requested. When later blocks 2963 * get requested, we will try again 2964 */ 2965 continue; 2966 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 2967 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 2968 /* must have already done this block */ 2969 release_stripe(sh2); 2970 continue; 2971 } 2972 2973 /* place all the copies on one channel */ 2974 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 2975 tx = async_memcpy(sh2->dev[dd_idx].page, 2976 sh->dev[i].page, 0, 0, STRIPE_SIZE, 2977 &submit); 2978 2979 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 2980 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 2981 for (j = 0; j < conf->raid_disks; j++) 2982 if (j != sh2->pd_idx && 2983 (!r6s || j != sh2->qd_idx) && 2984 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 2985 break; 2986 if (j == conf->raid_disks) { 2987 set_bit(STRIPE_EXPAND_READY, &sh2->state); 2988 set_bit(STRIPE_HANDLE, &sh2->state); 2989 } 2990 release_stripe(sh2); 2991 2992 } 2993 /* done submitting copies, wait for them to complete */ 2994 if (tx) { 2995 async_tx_ack(tx); 2996 dma_wait_for_async_tx(tx); 2997 } 2998 } 2999 3000 3001 /* 3002 * handle_stripe - do things to a stripe. 3003 * 3004 * We lock the stripe and then examine the state of various bits 3005 * to see what needs to be done. 3006 * Possible results: 3007 * return some read request which now have data 3008 * return some write requests which are safely on disc 3009 * schedule a read on some buffers 3010 * schedule a write of some buffers 3011 * return confirmation of parity correctness 3012 * 3013 * buffers are taken off read_list or write_list, and bh_cache buffers 3014 * get BH_Lock set before the stripe lock is released. 3015 * 3016 */ 3017 3018 static void handle_stripe5(struct stripe_head *sh) 3019 { 3020 raid5_conf_t *conf = sh->raid_conf; 3021 int disks = sh->disks, i; 3022 struct bio *return_bi = NULL; 3023 struct stripe_head_state s; 3024 struct r5dev *dev; 3025 mdk_rdev_t *blocked_rdev = NULL; 3026 int prexor; 3027 int dec_preread_active = 0; 3028 3029 memset(&s, 0, sizeof(s)); 3030 pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d " 3031 "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state, 3032 atomic_read(&sh->count), sh->pd_idx, sh->check_state, 3033 sh->reconstruct_state); 3034 3035 spin_lock(&sh->lock); 3036 clear_bit(STRIPE_HANDLE, &sh->state); 3037 clear_bit(STRIPE_DELAYED, &sh->state); 3038 3039 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 3040 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3041 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3042 3043 /* Now to look around and see what can be done */ 3044 rcu_read_lock(); 3045 for (i=disks; i--; ) { 3046 mdk_rdev_t *rdev; 3047 3048 dev = &sh->dev[i]; 3049 3050 pr_debug("check %d: state 0x%lx toread %p read %p write %p " 3051 "written %p\n", i, dev->flags, dev->toread, dev->read, 3052 dev->towrite, dev->written); 3053 3054 /* maybe we can request a biofill operation 3055 * 3056 * new wantfill requests are only permitted while 3057 * ops_complete_biofill is guaranteed to be inactive 3058 */ 3059 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3060 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3061 set_bit(R5_Wantfill, &dev->flags); 3062 3063 /* now count some things */ 3064 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 3065 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 3066 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++; 3067 3068 if (test_bit(R5_Wantfill, &dev->flags)) 3069 s.to_fill++; 3070 else if (dev->toread) 3071 s.to_read++; 3072 if (dev->towrite) { 3073 s.to_write++; 3074 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3075 s.non_overwrite++; 3076 } 3077 if (dev->written) 3078 s.written++; 3079 rdev = rcu_dereference(conf->disks[i].rdev); 3080 if (blocked_rdev == NULL && 3081 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 3082 blocked_rdev = rdev; 3083 atomic_inc(&rdev->nr_pending); 3084 } 3085 clear_bit(R5_Insync, &dev->flags); 3086 if (!rdev) 3087 /* Not in-sync */; 3088 else if (test_bit(In_sync, &rdev->flags)) 3089 set_bit(R5_Insync, &dev->flags); 3090 else { 3091 /* could be in-sync depending on recovery/reshape status */ 3092 if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3093 set_bit(R5_Insync, &dev->flags); 3094 } 3095 if (!test_bit(R5_Insync, &dev->flags)) { 3096 /* The ReadError flag will just be confusing now */ 3097 clear_bit(R5_ReadError, &dev->flags); 3098 clear_bit(R5_ReWrite, &dev->flags); 3099 } 3100 if (test_bit(R5_ReadError, &dev->flags)) 3101 clear_bit(R5_Insync, &dev->flags); 3102 if (!test_bit(R5_Insync, &dev->flags)) { 3103 s.failed++; 3104 s.failed_num = i; 3105 } 3106 } 3107 rcu_read_unlock(); 3108 3109 if (unlikely(blocked_rdev)) { 3110 if (s.syncing || s.expanding || s.expanded || 3111 s.to_write || s.written) { 3112 set_bit(STRIPE_HANDLE, &sh->state); 3113 goto unlock; 3114 } 3115 /* There is nothing for the blocked_rdev to block */ 3116 rdev_dec_pending(blocked_rdev, conf->mddev); 3117 blocked_rdev = NULL; 3118 } 3119 3120 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3121 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3122 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3123 } 3124 3125 pr_debug("locked=%d uptodate=%d to_read=%d" 3126 " to_write=%d failed=%d failed_num=%d\n", 3127 s.locked, s.uptodate, s.to_read, s.to_write, 3128 s.failed, s.failed_num); 3129 /* check if the array has lost two devices and, if so, some requests might 3130 * need to be failed 3131 */ 3132 if (s.failed > 1 && s.to_read+s.to_write+s.written) 3133 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3134 if (s.failed > 1 && s.syncing) { 3135 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3136 clear_bit(STRIPE_SYNCING, &sh->state); 3137 s.syncing = 0; 3138 } 3139 3140 /* might be able to return some write requests if the parity block 3141 * is safe, or on a failed drive 3142 */ 3143 dev = &sh->dev[sh->pd_idx]; 3144 if ( s.written && 3145 ((test_bit(R5_Insync, &dev->flags) && 3146 !test_bit(R5_LOCKED, &dev->flags) && 3147 test_bit(R5_UPTODATE, &dev->flags)) || 3148 (s.failed == 1 && s.failed_num == sh->pd_idx))) 3149 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3150 3151 /* Now we might consider reading some blocks, either to check/generate 3152 * parity, or to satisfy requests 3153 * or to load a block that is being partially written. 3154 */ 3155 if (s.to_read || s.non_overwrite || 3156 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3157 handle_stripe_fill5(sh, &s, disks); 3158 3159 /* Now we check to see if any write operations have recently 3160 * completed 3161 */ 3162 prexor = 0; 3163 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3164 prexor = 1; 3165 if (sh->reconstruct_state == reconstruct_state_drain_result || 3166 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3167 sh->reconstruct_state = reconstruct_state_idle; 3168 3169 /* All the 'written' buffers and the parity block are ready to 3170 * be written back to disk 3171 */ 3172 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3173 for (i = disks; i--; ) { 3174 dev = &sh->dev[i]; 3175 if (test_bit(R5_LOCKED, &dev->flags) && 3176 (i == sh->pd_idx || dev->written)) { 3177 pr_debug("Writing block %d\n", i); 3178 set_bit(R5_Wantwrite, &dev->flags); 3179 if (prexor) 3180 continue; 3181 if (!test_bit(R5_Insync, &dev->flags) || 3182 (i == sh->pd_idx && s.failed == 0)) 3183 set_bit(STRIPE_INSYNC, &sh->state); 3184 } 3185 } 3186 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3187 dec_preread_active = 1; 3188 } 3189 3190 /* Now to consider new write requests and what else, if anything 3191 * should be read. We do not handle new writes when: 3192 * 1/ A 'write' operation (copy+xor) is already in flight. 3193 * 2/ A 'check' operation is in flight, as it may clobber the parity 3194 * block. 3195 */ 3196 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3197 handle_stripe_dirtying5(conf, sh, &s, disks); 3198 3199 /* maybe we need to check and possibly fix the parity for this stripe 3200 * Any reads will already have been scheduled, so we just see if enough 3201 * data is available. The parity check is held off while parity 3202 * dependent operations are in flight. 3203 */ 3204 if (sh->check_state || 3205 (s.syncing && s.locked == 0 && 3206 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3207 !test_bit(STRIPE_INSYNC, &sh->state))) 3208 handle_parity_checks5(conf, sh, &s, disks); 3209 3210 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3211 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3212 clear_bit(STRIPE_SYNCING, &sh->state); 3213 } 3214 3215 /* If the failed drive is just a ReadError, then we might need to progress 3216 * the repair/check process 3217 */ 3218 if (s.failed == 1 && !conf->mddev->ro && 3219 test_bit(R5_ReadError, &sh->dev[s.failed_num].flags) 3220 && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags) 3221 && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags) 3222 ) { 3223 dev = &sh->dev[s.failed_num]; 3224 if (!test_bit(R5_ReWrite, &dev->flags)) { 3225 set_bit(R5_Wantwrite, &dev->flags); 3226 set_bit(R5_ReWrite, &dev->flags); 3227 set_bit(R5_LOCKED, &dev->flags); 3228 s.locked++; 3229 } else { 3230 /* let's read it back */ 3231 set_bit(R5_Wantread, &dev->flags); 3232 set_bit(R5_LOCKED, &dev->flags); 3233 s.locked++; 3234 } 3235 } 3236 3237 /* Finish reconstruct operations initiated by the expansion process */ 3238 if (sh->reconstruct_state == reconstruct_state_result) { 3239 struct stripe_head *sh2 3240 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3241 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3242 /* sh cannot be written until sh2 has been read. 3243 * so arrange for sh to be delayed a little 3244 */ 3245 set_bit(STRIPE_DELAYED, &sh->state); 3246 set_bit(STRIPE_HANDLE, &sh->state); 3247 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3248 &sh2->state)) 3249 atomic_inc(&conf->preread_active_stripes); 3250 release_stripe(sh2); 3251 goto unlock; 3252 } 3253 if (sh2) 3254 release_stripe(sh2); 3255 3256 sh->reconstruct_state = reconstruct_state_idle; 3257 clear_bit(STRIPE_EXPANDING, &sh->state); 3258 for (i = conf->raid_disks; i--; ) { 3259 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3260 set_bit(R5_LOCKED, &sh->dev[i].flags); 3261 s.locked++; 3262 } 3263 } 3264 3265 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3266 !sh->reconstruct_state) { 3267 /* Need to write out all blocks after computing parity */ 3268 sh->disks = conf->raid_disks; 3269 stripe_set_idx(sh->sector, conf, 0, sh); 3270 schedule_reconstruction(sh, &s, 1, 1); 3271 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3272 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3273 atomic_dec(&conf->reshape_stripes); 3274 wake_up(&conf->wait_for_overlap); 3275 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3276 } 3277 3278 if (s.expanding && s.locked == 0 && 3279 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3280 handle_stripe_expansion(conf, sh, NULL); 3281 3282 unlock: 3283 spin_unlock(&sh->lock); 3284 3285 /* wait for this device to become unblocked */ 3286 if (unlikely(blocked_rdev)) 3287 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3288 3289 if (s.ops_request) 3290 raid_run_ops(sh, s.ops_request); 3291 3292 ops_run_io(sh, &s); 3293 3294 if (dec_preread_active) { 3295 /* We delay this until after ops_run_io so that if make_request 3296 * is waiting on a flush, it won't continue until the writes 3297 * have actually been submitted. 3298 */ 3299 atomic_dec(&conf->preread_active_stripes); 3300 if (atomic_read(&conf->preread_active_stripes) < 3301 IO_THRESHOLD) 3302 md_wakeup_thread(conf->mddev->thread); 3303 } 3304 return_io(return_bi); 3305 } 3306 3307 static void handle_stripe6(struct stripe_head *sh) 3308 { 3309 raid5_conf_t *conf = sh->raid_conf; 3310 int disks = sh->disks; 3311 struct bio *return_bi = NULL; 3312 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx; 3313 struct stripe_head_state s; 3314 struct r6_state r6s; 3315 struct r5dev *dev, *pdev, *qdev; 3316 mdk_rdev_t *blocked_rdev = NULL; 3317 int dec_preread_active = 0; 3318 3319 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3320 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3321 (unsigned long long)sh->sector, sh->state, 3322 atomic_read(&sh->count), pd_idx, qd_idx, 3323 sh->check_state, sh->reconstruct_state); 3324 memset(&s, 0, sizeof(s)); 3325 3326 spin_lock(&sh->lock); 3327 clear_bit(STRIPE_HANDLE, &sh->state); 3328 clear_bit(STRIPE_DELAYED, &sh->state); 3329 3330 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 3331 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3332 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3333 /* Now to look around and see what can be done */ 3334 3335 rcu_read_lock(); 3336 for (i=disks; i--; ) { 3337 mdk_rdev_t *rdev; 3338 dev = &sh->dev[i]; 3339 3340 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3341 i, dev->flags, dev->toread, dev->towrite, dev->written); 3342 /* maybe we can reply to a read 3343 * 3344 * new wantfill requests are only permitted while 3345 * ops_complete_biofill is guaranteed to be inactive 3346 */ 3347 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3348 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3349 set_bit(R5_Wantfill, &dev->flags); 3350 3351 /* now count some things */ 3352 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 3353 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 3354 if (test_bit(R5_Wantcompute, &dev->flags)) { 3355 s.compute++; 3356 BUG_ON(s.compute > 2); 3357 } 3358 3359 if (test_bit(R5_Wantfill, &dev->flags)) { 3360 s.to_fill++; 3361 } else if (dev->toread) 3362 s.to_read++; 3363 if (dev->towrite) { 3364 s.to_write++; 3365 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3366 s.non_overwrite++; 3367 } 3368 if (dev->written) 3369 s.written++; 3370 rdev = rcu_dereference(conf->disks[i].rdev); 3371 if (blocked_rdev == NULL && 3372 rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 3373 blocked_rdev = rdev; 3374 atomic_inc(&rdev->nr_pending); 3375 } 3376 clear_bit(R5_Insync, &dev->flags); 3377 if (!rdev) 3378 /* Not in-sync */; 3379 else if (test_bit(In_sync, &rdev->flags)) 3380 set_bit(R5_Insync, &dev->flags); 3381 else { 3382 /* in sync if before recovery_offset */ 3383 if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3384 set_bit(R5_Insync, &dev->flags); 3385 } 3386 if (!test_bit(R5_Insync, &dev->flags)) { 3387 /* The ReadError flag will just be confusing now */ 3388 clear_bit(R5_ReadError, &dev->flags); 3389 clear_bit(R5_ReWrite, &dev->flags); 3390 } 3391 if (test_bit(R5_ReadError, &dev->flags)) 3392 clear_bit(R5_Insync, &dev->flags); 3393 if (!test_bit(R5_Insync, &dev->flags)) { 3394 if (s.failed < 2) 3395 r6s.failed_num[s.failed] = i; 3396 s.failed++; 3397 } 3398 } 3399 rcu_read_unlock(); 3400 3401 if (unlikely(blocked_rdev)) { 3402 if (s.syncing || s.expanding || s.expanded || 3403 s.to_write || s.written) { 3404 set_bit(STRIPE_HANDLE, &sh->state); 3405 goto unlock; 3406 } 3407 /* There is nothing for the blocked_rdev to block */ 3408 rdev_dec_pending(blocked_rdev, conf->mddev); 3409 blocked_rdev = NULL; 3410 } 3411 3412 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3413 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3414 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3415 } 3416 3417 pr_debug("locked=%d uptodate=%d to_read=%d" 3418 " to_write=%d failed=%d failed_num=%d,%d\n", 3419 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3420 r6s.failed_num[0], r6s.failed_num[1]); 3421 /* check if the array has lost >2 devices and, if so, some requests 3422 * might need to be failed 3423 */ 3424 if (s.failed > 2 && s.to_read+s.to_write+s.written) 3425 handle_failed_stripe(conf, sh, &s, disks, &return_bi); 3426 if (s.failed > 2 && s.syncing) { 3427 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3428 clear_bit(STRIPE_SYNCING, &sh->state); 3429 s.syncing = 0; 3430 } 3431 3432 /* 3433 * might be able to return some write requests if the parity blocks 3434 * are safe, or on a failed drive 3435 */ 3436 pdev = &sh->dev[pd_idx]; 3437 r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx) 3438 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx); 3439 qdev = &sh->dev[qd_idx]; 3440 r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx) 3441 || (s.failed >= 2 && r6s.failed_num[1] == qd_idx); 3442 3443 if ( s.written && 3444 ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3445 && !test_bit(R5_LOCKED, &pdev->flags) 3446 && test_bit(R5_UPTODATE, &pdev->flags)))) && 3447 ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3448 && !test_bit(R5_LOCKED, &qdev->flags) 3449 && test_bit(R5_UPTODATE, &qdev->flags))))) 3450 handle_stripe_clean_event(conf, sh, disks, &return_bi); 3451 3452 /* Now we might consider reading some blocks, either to check/generate 3453 * parity, or to satisfy requests 3454 * or to load a block that is being partially written. 3455 */ 3456 if (s.to_read || s.non_overwrite || (s.to_write && s.failed) || 3457 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding) 3458 handle_stripe_fill6(sh, &s, &r6s, disks); 3459 3460 /* Now we check to see if any write operations have recently 3461 * completed 3462 */ 3463 if (sh->reconstruct_state == reconstruct_state_drain_result) { 3464 3465 sh->reconstruct_state = reconstruct_state_idle; 3466 /* All the 'written' buffers and the parity blocks are ready to 3467 * be written back to disk 3468 */ 3469 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 3470 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags)); 3471 for (i = disks; i--; ) { 3472 dev = &sh->dev[i]; 3473 if (test_bit(R5_LOCKED, &dev->flags) && 3474 (i == sh->pd_idx || i == qd_idx || 3475 dev->written)) { 3476 pr_debug("Writing block %d\n", i); 3477 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3478 set_bit(R5_Wantwrite, &dev->flags); 3479 if (!test_bit(R5_Insync, &dev->flags) || 3480 ((i == sh->pd_idx || i == qd_idx) && 3481 s.failed == 0)) 3482 set_bit(STRIPE_INSYNC, &sh->state); 3483 } 3484 } 3485 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3486 dec_preread_active = 1; 3487 } 3488 3489 /* Now to consider new write requests and what else, if anything 3490 * should be read. We do not handle new writes when: 3491 * 1/ A 'write' operation (copy+gen_syndrome) is already in flight. 3492 * 2/ A 'check' operation is in flight, as it may clobber the parity 3493 * block. 3494 */ 3495 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3496 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks); 3497 3498 /* maybe we need to check and possibly fix the parity for this stripe 3499 * Any reads will already have been scheduled, so we just see if enough 3500 * data is available. The parity check is held off while parity 3501 * dependent operations are in flight. 3502 */ 3503 if (sh->check_state || 3504 (s.syncing && s.locked == 0 && 3505 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3506 !test_bit(STRIPE_INSYNC, &sh->state))) 3507 handle_parity_checks6(conf, sh, &s, &r6s, disks); 3508 3509 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3510 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3511 clear_bit(STRIPE_SYNCING, &sh->state); 3512 } 3513 3514 /* If the failed drives are just a ReadError, then we might need 3515 * to progress the repair/check process 3516 */ 3517 if (s.failed <= 2 && !conf->mddev->ro) 3518 for (i = 0; i < s.failed; i++) { 3519 dev = &sh->dev[r6s.failed_num[i]]; 3520 if (test_bit(R5_ReadError, &dev->flags) 3521 && !test_bit(R5_LOCKED, &dev->flags) 3522 && test_bit(R5_UPTODATE, &dev->flags) 3523 ) { 3524 if (!test_bit(R5_ReWrite, &dev->flags)) { 3525 set_bit(R5_Wantwrite, &dev->flags); 3526 set_bit(R5_ReWrite, &dev->flags); 3527 set_bit(R5_LOCKED, &dev->flags); 3528 s.locked++; 3529 } else { 3530 /* let's read it back */ 3531 set_bit(R5_Wantread, &dev->flags); 3532 set_bit(R5_LOCKED, &dev->flags); 3533 s.locked++; 3534 } 3535 } 3536 } 3537 3538 /* Finish reconstruct operations initiated by the expansion process */ 3539 if (sh->reconstruct_state == reconstruct_state_result) { 3540 sh->reconstruct_state = reconstruct_state_idle; 3541 clear_bit(STRIPE_EXPANDING, &sh->state); 3542 for (i = conf->raid_disks; i--; ) { 3543 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3544 set_bit(R5_LOCKED, &sh->dev[i].flags); 3545 s.locked++; 3546 } 3547 } 3548 3549 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3550 !sh->reconstruct_state) { 3551 struct stripe_head *sh2 3552 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3553 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) { 3554 /* sh cannot be written until sh2 has been read. 3555 * so arrange for sh to be delayed a little 3556 */ 3557 set_bit(STRIPE_DELAYED, &sh->state); 3558 set_bit(STRIPE_HANDLE, &sh->state); 3559 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3560 &sh2->state)) 3561 atomic_inc(&conf->preread_active_stripes); 3562 release_stripe(sh2); 3563 goto unlock; 3564 } 3565 if (sh2) 3566 release_stripe(sh2); 3567 3568 /* Need to write out all blocks after computing P&Q */ 3569 sh->disks = conf->raid_disks; 3570 stripe_set_idx(sh->sector, conf, 0, sh); 3571 schedule_reconstruction(sh, &s, 1, 1); 3572 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3573 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3574 atomic_dec(&conf->reshape_stripes); 3575 wake_up(&conf->wait_for_overlap); 3576 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3577 } 3578 3579 if (s.expanding && s.locked == 0 && 3580 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3581 handle_stripe_expansion(conf, sh, &r6s); 3582 3583 unlock: 3584 spin_unlock(&sh->lock); 3585 3586 /* wait for this device to become unblocked */ 3587 if (unlikely(blocked_rdev)) 3588 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3589 3590 if (s.ops_request) 3591 raid_run_ops(sh, s.ops_request); 3592 3593 ops_run_io(sh, &s); 3594 3595 3596 if (dec_preread_active) { 3597 /* We delay this until after ops_run_io so that if make_request 3598 * is waiting on a flush, it won't continue until the writes 3599 * have actually been submitted. 3600 */ 3601 atomic_dec(&conf->preread_active_stripes); 3602 if (atomic_read(&conf->preread_active_stripes) < 3603 IO_THRESHOLD) 3604 md_wakeup_thread(conf->mddev->thread); 3605 } 3606 3607 return_io(return_bi); 3608 } 3609 3610 static void handle_stripe(struct stripe_head *sh) 3611 { 3612 if (sh->raid_conf->level == 6) 3613 handle_stripe6(sh); 3614 else 3615 handle_stripe5(sh); 3616 } 3617 3618 static void raid5_activate_delayed(raid5_conf_t *conf) 3619 { 3620 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3621 while (!list_empty(&conf->delayed_list)) { 3622 struct list_head *l = conf->delayed_list.next; 3623 struct stripe_head *sh; 3624 sh = list_entry(l, struct stripe_head, lru); 3625 list_del_init(l); 3626 clear_bit(STRIPE_DELAYED, &sh->state); 3627 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3628 atomic_inc(&conf->preread_active_stripes); 3629 list_add_tail(&sh->lru, &conf->hold_list); 3630 } 3631 } else 3632 plugger_set_plug(&conf->plug); 3633 } 3634 3635 static void activate_bit_delay(raid5_conf_t *conf) 3636 { 3637 /* device_lock is held */ 3638 struct list_head head; 3639 list_add(&head, &conf->bitmap_list); 3640 list_del_init(&conf->bitmap_list); 3641 while (!list_empty(&head)) { 3642 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3643 list_del_init(&sh->lru); 3644 atomic_inc(&sh->count); 3645 __release_stripe(conf, sh); 3646 } 3647 } 3648 3649 static void unplug_slaves(mddev_t *mddev) 3650 { 3651 raid5_conf_t *conf = mddev->private; 3652 int i; 3653 int devs = max(conf->raid_disks, conf->previous_raid_disks); 3654 3655 rcu_read_lock(); 3656 for (i = 0; i < devs; i++) { 3657 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 3658 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { 3659 struct request_queue *r_queue = bdev_get_queue(rdev->bdev); 3660 3661 atomic_inc(&rdev->nr_pending); 3662 rcu_read_unlock(); 3663 3664 blk_unplug(r_queue); 3665 3666 rdev_dec_pending(rdev, mddev); 3667 rcu_read_lock(); 3668 } 3669 } 3670 rcu_read_unlock(); 3671 } 3672 3673 void md_raid5_unplug_device(raid5_conf_t *conf) 3674 { 3675 unsigned long flags; 3676 3677 spin_lock_irqsave(&conf->device_lock, flags); 3678 3679 if (plugger_remove_plug(&conf->plug)) { 3680 conf->seq_flush++; 3681 raid5_activate_delayed(conf); 3682 } 3683 md_wakeup_thread(conf->mddev->thread); 3684 3685 spin_unlock_irqrestore(&conf->device_lock, flags); 3686 3687 unplug_slaves(conf->mddev); 3688 } 3689 EXPORT_SYMBOL_GPL(md_raid5_unplug_device); 3690 3691 static void raid5_unplug(struct plug_handle *plug) 3692 { 3693 raid5_conf_t *conf = container_of(plug, raid5_conf_t, plug); 3694 md_raid5_unplug_device(conf); 3695 } 3696 3697 static void raid5_unplug_queue(struct request_queue *q) 3698 { 3699 mddev_t *mddev = q->queuedata; 3700 md_raid5_unplug_device(mddev->private); 3701 } 3702 3703 int md_raid5_congested(mddev_t *mddev, int bits) 3704 { 3705 raid5_conf_t *conf = mddev->private; 3706 3707 /* No difference between reads and writes. Just check 3708 * how busy the stripe_cache is 3709 */ 3710 3711 if (conf->inactive_blocked) 3712 return 1; 3713 if (conf->quiesce) 3714 return 1; 3715 if (list_empty_careful(&conf->inactive_list)) 3716 return 1; 3717 3718 return 0; 3719 } 3720 EXPORT_SYMBOL_GPL(md_raid5_congested); 3721 3722 static int raid5_congested(void *data, int bits) 3723 { 3724 mddev_t *mddev = data; 3725 3726 return mddev_congested(mddev, bits) || 3727 md_raid5_congested(mddev, bits); 3728 } 3729 3730 /* We want read requests to align with chunks where possible, 3731 * but write requests don't need to. 3732 */ 3733 static int raid5_mergeable_bvec(struct request_queue *q, 3734 struct bvec_merge_data *bvm, 3735 struct bio_vec *biovec) 3736 { 3737 mddev_t *mddev = q->queuedata; 3738 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 3739 int max; 3740 unsigned int chunk_sectors = mddev->chunk_sectors; 3741 unsigned int bio_sectors = bvm->bi_size >> 9; 3742 3743 if ((bvm->bi_rw & 1) == WRITE) 3744 return biovec->bv_len; /* always allow writes to be mergeable */ 3745 3746 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3747 chunk_sectors = mddev->new_chunk_sectors; 3748 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3749 if (max < 0) max = 0; 3750 if (max <= biovec->bv_len && bio_sectors == 0) 3751 return biovec->bv_len; 3752 else 3753 return max; 3754 } 3755 3756 3757 static int in_chunk_boundary(mddev_t *mddev, struct bio *bio) 3758 { 3759 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3760 unsigned int chunk_sectors = mddev->chunk_sectors; 3761 unsigned int bio_sectors = bio->bi_size >> 9; 3762 3763 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3764 chunk_sectors = mddev->new_chunk_sectors; 3765 return chunk_sectors >= 3766 ((sector & (chunk_sectors - 1)) + bio_sectors); 3767 } 3768 3769 /* 3770 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3771 * later sampled by raid5d. 3772 */ 3773 static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf) 3774 { 3775 unsigned long flags; 3776 3777 spin_lock_irqsave(&conf->device_lock, flags); 3778 3779 bi->bi_next = conf->retry_read_aligned_list; 3780 conf->retry_read_aligned_list = bi; 3781 3782 spin_unlock_irqrestore(&conf->device_lock, flags); 3783 md_wakeup_thread(conf->mddev->thread); 3784 } 3785 3786 3787 static struct bio *remove_bio_from_retry(raid5_conf_t *conf) 3788 { 3789 struct bio *bi; 3790 3791 bi = conf->retry_read_aligned; 3792 if (bi) { 3793 conf->retry_read_aligned = NULL; 3794 return bi; 3795 } 3796 bi = conf->retry_read_aligned_list; 3797 if(bi) { 3798 conf->retry_read_aligned_list = bi->bi_next; 3799 bi->bi_next = NULL; 3800 /* 3801 * this sets the active strip count to 1 and the processed 3802 * strip count to zero (upper 8 bits) 3803 */ 3804 bi->bi_phys_segments = 1; /* biased count of active stripes */ 3805 } 3806 3807 return bi; 3808 } 3809 3810 3811 /* 3812 * The "raid5_align_endio" should check if the read succeeded and if it 3813 * did, call bio_endio on the original bio (having bio_put the new bio 3814 * first). 3815 * If the read failed.. 3816 */ 3817 static void raid5_align_endio(struct bio *bi, int error) 3818 { 3819 struct bio* raid_bi = bi->bi_private; 3820 mddev_t *mddev; 3821 raid5_conf_t *conf; 3822 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 3823 mdk_rdev_t *rdev; 3824 3825 bio_put(bi); 3826 3827 rdev = (void*)raid_bi->bi_next; 3828 raid_bi->bi_next = NULL; 3829 mddev = rdev->mddev; 3830 conf = mddev->private; 3831 3832 rdev_dec_pending(rdev, conf->mddev); 3833 3834 if (!error && uptodate) { 3835 bio_endio(raid_bi, 0); 3836 if (atomic_dec_and_test(&conf->active_aligned_reads)) 3837 wake_up(&conf->wait_for_stripe); 3838 return; 3839 } 3840 3841 3842 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 3843 3844 add_bio_to_retry(raid_bi, conf); 3845 } 3846 3847 static int bio_fits_rdev(struct bio *bi) 3848 { 3849 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 3850 3851 if ((bi->bi_size>>9) > queue_max_sectors(q)) 3852 return 0; 3853 blk_recount_segments(q, bi); 3854 if (bi->bi_phys_segments > queue_max_segments(q)) 3855 return 0; 3856 3857 if (q->merge_bvec_fn) 3858 /* it's too hard to apply the merge_bvec_fn at this stage, 3859 * just just give up 3860 */ 3861 return 0; 3862 3863 return 1; 3864 } 3865 3866 3867 static int chunk_aligned_read(mddev_t *mddev, struct bio * raid_bio) 3868 { 3869 raid5_conf_t *conf = mddev->private; 3870 int dd_idx; 3871 struct bio* align_bi; 3872 mdk_rdev_t *rdev; 3873 3874 if (!in_chunk_boundary(mddev, raid_bio)) { 3875 pr_debug("chunk_aligned_read : non aligned\n"); 3876 return 0; 3877 } 3878 /* 3879 * use bio_clone_mddev to make a copy of the bio 3880 */ 3881 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 3882 if (!align_bi) 3883 return 0; 3884 /* 3885 * set bi_end_io to a new function, and set bi_private to the 3886 * original bio. 3887 */ 3888 align_bi->bi_end_io = raid5_align_endio; 3889 align_bi->bi_private = raid_bio; 3890 /* 3891 * compute position 3892 */ 3893 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector, 3894 0, 3895 &dd_idx, NULL); 3896 3897 rcu_read_lock(); 3898 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 3899 if (rdev && test_bit(In_sync, &rdev->flags)) { 3900 atomic_inc(&rdev->nr_pending); 3901 rcu_read_unlock(); 3902 raid_bio->bi_next = (void*)rdev; 3903 align_bi->bi_bdev = rdev->bdev; 3904 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 3905 align_bi->bi_sector += rdev->data_offset; 3906 3907 if (!bio_fits_rdev(align_bi)) { 3908 /* too big in some way */ 3909 bio_put(align_bi); 3910 rdev_dec_pending(rdev, mddev); 3911 return 0; 3912 } 3913 3914 spin_lock_irq(&conf->device_lock); 3915 wait_event_lock_irq(conf->wait_for_stripe, 3916 conf->quiesce == 0, 3917 conf->device_lock, /* nothing */); 3918 atomic_inc(&conf->active_aligned_reads); 3919 spin_unlock_irq(&conf->device_lock); 3920 3921 generic_make_request(align_bi); 3922 return 1; 3923 } else { 3924 rcu_read_unlock(); 3925 bio_put(align_bi); 3926 return 0; 3927 } 3928 } 3929 3930 /* __get_priority_stripe - get the next stripe to process 3931 * 3932 * Full stripe writes are allowed to pass preread active stripes up until 3933 * the bypass_threshold is exceeded. In general the bypass_count 3934 * increments when the handle_list is handled before the hold_list; however, it 3935 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 3936 * stripe with in flight i/o. The bypass_count will be reset when the 3937 * head of the hold_list has changed, i.e. the head was promoted to the 3938 * handle_list. 3939 */ 3940 static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf) 3941 { 3942 struct stripe_head *sh; 3943 3944 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 3945 __func__, 3946 list_empty(&conf->handle_list) ? "empty" : "busy", 3947 list_empty(&conf->hold_list) ? "empty" : "busy", 3948 atomic_read(&conf->pending_full_writes), conf->bypass_count); 3949 3950 if (!list_empty(&conf->handle_list)) { 3951 sh = list_entry(conf->handle_list.next, typeof(*sh), lru); 3952 3953 if (list_empty(&conf->hold_list)) 3954 conf->bypass_count = 0; 3955 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 3956 if (conf->hold_list.next == conf->last_hold) 3957 conf->bypass_count++; 3958 else { 3959 conf->last_hold = conf->hold_list.next; 3960 conf->bypass_count -= conf->bypass_threshold; 3961 if (conf->bypass_count < 0) 3962 conf->bypass_count = 0; 3963 } 3964 } 3965 } else if (!list_empty(&conf->hold_list) && 3966 ((conf->bypass_threshold && 3967 conf->bypass_count > conf->bypass_threshold) || 3968 atomic_read(&conf->pending_full_writes) == 0)) { 3969 sh = list_entry(conf->hold_list.next, 3970 typeof(*sh), lru); 3971 conf->bypass_count -= conf->bypass_threshold; 3972 if (conf->bypass_count < 0) 3973 conf->bypass_count = 0; 3974 } else 3975 return NULL; 3976 3977 list_del_init(&sh->lru); 3978 atomic_inc(&sh->count); 3979 BUG_ON(atomic_read(&sh->count) != 1); 3980 return sh; 3981 } 3982 3983 static int make_request(mddev_t *mddev, struct bio * bi) 3984 { 3985 raid5_conf_t *conf = mddev->private; 3986 int dd_idx; 3987 sector_t new_sector; 3988 sector_t logical_sector, last_sector; 3989 struct stripe_head *sh; 3990 const int rw = bio_data_dir(bi); 3991 int remaining; 3992 3993 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 3994 md_flush_request(mddev, bi); 3995 return 0; 3996 } 3997 3998 md_write_start(mddev, bi); 3999 4000 if (rw == READ && 4001 mddev->reshape_position == MaxSector && 4002 chunk_aligned_read(mddev,bi)) 4003 return 0; 4004 4005 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4006 last_sector = bi->bi_sector + (bi->bi_size>>9); 4007 bi->bi_next = NULL; 4008 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4009 4010 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 4011 DEFINE_WAIT(w); 4012 int disks, data_disks; 4013 int previous; 4014 4015 retry: 4016 previous = 0; 4017 disks = conf->raid_disks; 4018 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 4019 if (unlikely(conf->reshape_progress != MaxSector)) { 4020 /* spinlock is needed as reshape_progress may be 4021 * 64bit on a 32bit platform, and so it might be 4022 * possible to see a half-updated value 4023 * Ofcourse reshape_progress could change after 4024 * the lock is dropped, so once we get a reference 4025 * to the stripe that we think it is, we will have 4026 * to check again. 4027 */ 4028 spin_lock_irq(&conf->device_lock); 4029 if (mddev->delta_disks < 0 4030 ? logical_sector < conf->reshape_progress 4031 : logical_sector >= conf->reshape_progress) { 4032 disks = conf->previous_raid_disks; 4033 previous = 1; 4034 } else { 4035 if (mddev->delta_disks < 0 4036 ? logical_sector < conf->reshape_safe 4037 : logical_sector >= conf->reshape_safe) { 4038 spin_unlock_irq(&conf->device_lock); 4039 schedule(); 4040 goto retry; 4041 } 4042 } 4043 spin_unlock_irq(&conf->device_lock); 4044 } 4045 data_disks = disks - conf->max_degraded; 4046 4047 new_sector = raid5_compute_sector(conf, logical_sector, 4048 previous, 4049 &dd_idx, NULL); 4050 pr_debug("raid456: make_request, sector %llu logical %llu\n", 4051 (unsigned long long)new_sector, 4052 (unsigned long long)logical_sector); 4053 4054 sh = get_active_stripe(conf, new_sector, previous, 4055 (bi->bi_rw&RWA_MASK), 0); 4056 if (sh) { 4057 if (unlikely(previous)) { 4058 /* expansion might have moved on while waiting for a 4059 * stripe, so we must do the range check again. 4060 * Expansion could still move past after this 4061 * test, but as we are holding a reference to 4062 * 'sh', we know that if that happens, 4063 * STRIPE_EXPANDING will get set and the expansion 4064 * won't proceed until we finish with the stripe. 4065 */ 4066 int must_retry = 0; 4067 spin_lock_irq(&conf->device_lock); 4068 if (mddev->delta_disks < 0 4069 ? logical_sector >= conf->reshape_progress 4070 : logical_sector < conf->reshape_progress) 4071 /* mismatch, need to try again */ 4072 must_retry = 1; 4073 spin_unlock_irq(&conf->device_lock); 4074 if (must_retry) { 4075 release_stripe(sh); 4076 schedule(); 4077 goto retry; 4078 } 4079 } 4080 4081 if (bio_data_dir(bi) == WRITE && 4082 logical_sector >= mddev->suspend_lo && 4083 logical_sector < mddev->suspend_hi) { 4084 release_stripe(sh); 4085 /* As the suspend_* range is controlled by 4086 * userspace, we want an interruptible 4087 * wait. 4088 */ 4089 flush_signals(current); 4090 prepare_to_wait(&conf->wait_for_overlap, 4091 &w, TASK_INTERRUPTIBLE); 4092 if (logical_sector >= mddev->suspend_lo && 4093 logical_sector < mddev->suspend_hi) 4094 schedule(); 4095 goto retry; 4096 } 4097 4098 if (test_bit(STRIPE_EXPANDING, &sh->state) || 4099 !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) { 4100 /* Stripe is busy expanding or 4101 * add failed due to overlap. Flush everything 4102 * and wait a while 4103 */ 4104 md_raid5_unplug_device(conf); 4105 release_stripe(sh); 4106 schedule(); 4107 goto retry; 4108 } 4109 finish_wait(&conf->wait_for_overlap, &w); 4110 set_bit(STRIPE_HANDLE, &sh->state); 4111 clear_bit(STRIPE_DELAYED, &sh->state); 4112 if ((bi->bi_rw & REQ_SYNC) && 4113 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4114 atomic_inc(&conf->preread_active_stripes); 4115 release_stripe(sh); 4116 } else { 4117 /* cannot get stripe for read-ahead, just give-up */ 4118 clear_bit(BIO_UPTODATE, &bi->bi_flags); 4119 finish_wait(&conf->wait_for_overlap, &w); 4120 break; 4121 } 4122 4123 } 4124 spin_lock_irq(&conf->device_lock); 4125 remaining = raid5_dec_bi_phys_segments(bi); 4126 spin_unlock_irq(&conf->device_lock); 4127 if (remaining == 0) { 4128 4129 if ( rw == WRITE ) 4130 md_write_end(mddev); 4131 4132 bio_endio(bi, 0); 4133 } 4134 4135 return 0; 4136 } 4137 4138 static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks); 4139 4140 static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped) 4141 { 4142 /* reshaping is quite different to recovery/resync so it is 4143 * handled quite separately ... here. 4144 * 4145 * On each call to sync_request, we gather one chunk worth of 4146 * destination stripes and flag them as expanding. 4147 * Then we find all the source stripes and request reads. 4148 * As the reads complete, handle_stripe will copy the data 4149 * into the destination stripe and release that stripe. 4150 */ 4151 raid5_conf_t *conf = mddev->private; 4152 struct stripe_head *sh; 4153 sector_t first_sector, last_sector; 4154 int raid_disks = conf->previous_raid_disks; 4155 int data_disks = raid_disks - conf->max_degraded; 4156 int new_data_disks = conf->raid_disks - conf->max_degraded; 4157 int i; 4158 int dd_idx; 4159 sector_t writepos, readpos, safepos; 4160 sector_t stripe_addr; 4161 int reshape_sectors; 4162 struct list_head stripes; 4163 4164 if (sector_nr == 0) { 4165 /* If restarting in the middle, skip the initial sectors */ 4166 if (mddev->delta_disks < 0 && 4167 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4168 sector_nr = raid5_size(mddev, 0, 0) 4169 - conf->reshape_progress; 4170 } else if (mddev->delta_disks >= 0 && 4171 conf->reshape_progress > 0) 4172 sector_nr = conf->reshape_progress; 4173 sector_div(sector_nr, new_data_disks); 4174 if (sector_nr) { 4175 mddev->curr_resync_completed = sector_nr; 4176 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4177 *skipped = 1; 4178 return sector_nr; 4179 } 4180 } 4181 4182 /* We need to process a full chunk at a time. 4183 * If old and new chunk sizes differ, we need to process the 4184 * largest of these 4185 */ 4186 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4187 reshape_sectors = mddev->new_chunk_sectors; 4188 else 4189 reshape_sectors = mddev->chunk_sectors; 4190 4191 /* we update the metadata when there is more than 3Meg 4192 * in the block range (that is rather arbitrary, should 4193 * probably be time based) or when the data about to be 4194 * copied would over-write the source of the data at 4195 * the front of the range. 4196 * i.e. one new_stripe along from reshape_progress new_maps 4197 * to after where reshape_safe old_maps to 4198 */ 4199 writepos = conf->reshape_progress; 4200 sector_div(writepos, new_data_disks); 4201 readpos = conf->reshape_progress; 4202 sector_div(readpos, data_disks); 4203 safepos = conf->reshape_safe; 4204 sector_div(safepos, data_disks); 4205 if (mddev->delta_disks < 0) { 4206 writepos -= min_t(sector_t, reshape_sectors, writepos); 4207 readpos += reshape_sectors; 4208 safepos += reshape_sectors; 4209 } else { 4210 writepos += reshape_sectors; 4211 readpos -= min_t(sector_t, reshape_sectors, readpos); 4212 safepos -= min_t(sector_t, reshape_sectors, safepos); 4213 } 4214 4215 /* 'writepos' is the most advanced device address we might write. 4216 * 'readpos' is the least advanced device address we might read. 4217 * 'safepos' is the least address recorded in the metadata as having 4218 * been reshaped. 4219 * If 'readpos' is behind 'writepos', then there is no way that we can 4220 * ensure safety in the face of a crash - that must be done by userspace 4221 * making a backup of the data. So in that case there is no particular 4222 * rush to update metadata. 4223 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4224 * update the metadata to advance 'safepos' to match 'readpos' so that 4225 * we can be safe in the event of a crash. 4226 * So we insist on updating metadata if safepos is behind writepos and 4227 * readpos is beyond writepos. 4228 * In any case, update the metadata every 10 seconds. 4229 * Maybe that number should be configurable, but I'm not sure it is 4230 * worth it.... maybe it could be a multiple of safemode_delay??? 4231 */ 4232 if ((mddev->delta_disks < 0 4233 ? (safepos > writepos && readpos < writepos) 4234 : (safepos < writepos && readpos > writepos)) || 4235 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4236 /* Cannot proceed until we've updated the superblock... */ 4237 wait_event(conf->wait_for_overlap, 4238 atomic_read(&conf->reshape_stripes)==0); 4239 mddev->reshape_position = conf->reshape_progress; 4240 mddev->curr_resync_completed = mddev->curr_resync; 4241 conf->reshape_checkpoint = jiffies; 4242 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4243 md_wakeup_thread(mddev->thread); 4244 wait_event(mddev->sb_wait, mddev->flags == 0 || 4245 kthread_should_stop()); 4246 spin_lock_irq(&conf->device_lock); 4247 conf->reshape_safe = mddev->reshape_position; 4248 spin_unlock_irq(&conf->device_lock); 4249 wake_up(&conf->wait_for_overlap); 4250 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4251 } 4252 4253 if (mddev->delta_disks < 0) { 4254 BUG_ON(conf->reshape_progress == 0); 4255 stripe_addr = writepos; 4256 BUG_ON((mddev->dev_sectors & 4257 ~((sector_t)reshape_sectors - 1)) 4258 - reshape_sectors - stripe_addr 4259 != sector_nr); 4260 } else { 4261 BUG_ON(writepos != sector_nr + reshape_sectors); 4262 stripe_addr = sector_nr; 4263 } 4264 INIT_LIST_HEAD(&stripes); 4265 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4266 int j; 4267 int skipped_disk = 0; 4268 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4269 set_bit(STRIPE_EXPANDING, &sh->state); 4270 atomic_inc(&conf->reshape_stripes); 4271 /* If any of this stripe is beyond the end of the old 4272 * array, then we need to zero those blocks 4273 */ 4274 for (j=sh->disks; j--;) { 4275 sector_t s; 4276 if (j == sh->pd_idx) 4277 continue; 4278 if (conf->level == 6 && 4279 j == sh->qd_idx) 4280 continue; 4281 s = compute_blocknr(sh, j, 0); 4282 if (s < raid5_size(mddev, 0, 0)) { 4283 skipped_disk = 1; 4284 continue; 4285 } 4286 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4287 set_bit(R5_Expanded, &sh->dev[j].flags); 4288 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4289 } 4290 if (!skipped_disk) { 4291 set_bit(STRIPE_EXPAND_READY, &sh->state); 4292 set_bit(STRIPE_HANDLE, &sh->state); 4293 } 4294 list_add(&sh->lru, &stripes); 4295 } 4296 spin_lock_irq(&conf->device_lock); 4297 if (mddev->delta_disks < 0) 4298 conf->reshape_progress -= reshape_sectors * new_data_disks; 4299 else 4300 conf->reshape_progress += reshape_sectors * new_data_disks; 4301 spin_unlock_irq(&conf->device_lock); 4302 /* Ok, those stripe are ready. We can start scheduling 4303 * reads on the source stripes. 4304 * The source stripes are determined by mapping the first and last 4305 * block on the destination stripes. 4306 */ 4307 first_sector = 4308 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4309 1, &dd_idx, NULL); 4310 last_sector = 4311 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4312 * new_data_disks - 1), 4313 1, &dd_idx, NULL); 4314 if (last_sector >= mddev->dev_sectors) 4315 last_sector = mddev->dev_sectors - 1; 4316 while (first_sector <= last_sector) { 4317 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4318 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4319 set_bit(STRIPE_HANDLE, &sh->state); 4320 release_stripe(sh); 4321 first_sector += STRIPE_SECTORS; 4322 } 4323 /* Now that the sources are clearly marked, we can release 4324 * the destination stripes 4325 */ 4326 while (!list_empty(&stripes)) { 4327 sh = list_entry(stripes.next, struct stripe_head, lru); 4328 list_del_init(&sh->lru); 4329 release_stripe(sh); 4330 } 4331 /* If this takes us to the resync_max point where we have to pause, 4332 * then we need to write out the superblock. 4333 */ 4334 sector_nr += reshape_sectors; 4335 if ((sector_nr - mddev->curr_resync_completed) * 2 4336 >= mddev->resync_max - mddev->curr_resync_completed) { 4337 /* Cannot proceed until we've updated the superblock... */ 4338 wait_event(conf->wait_for_overlap, 4339 atomic_read(&conf->reshape_stripes) == 0); 4340 mddev->reshape_position = conf->reshape_progress; 4341 mddev->curr_resync_completed = mddev->curr_resync + reshape_sectors; 4342 conf->reshape_checkpoint = jiffies; 4343 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4344 md_wakeup_thread(mddev->thread); 4345 wait_event(mddev->sb_wait, 4346 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4347 || kthread_should_stop()); 4348 spin_lock_irq(&conf->device_lock); 4349 conf->reshape_safe = mddev->reshape_position; 4350 spin_unlock_irq(&conf->device_lock); 4351 wake_up(&conf->wait_for_overlap); 4352 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4353 } 4354 return reshape_sectors; 4355 } 4356 4357 /* FIXME go_faster isn't used */ 4358 static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster) 4359 { 4360 raid5_conf_t *conf = mddev->private; 4361 struct stripe_head *sh; 4362 sector_t max_sector = mddev->dev_sectors; 4363 sector_t sync_blocks; 4364 int still_degraded = 0; 4365 int i; 4366 4367 if (sector_nr >= max_sector) { 4368 /* just being told to finish up .. nothing much to do */ 4369 unplug_slaves(mddev); 4370 4371 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 4372 end_reshape(conf); 4373 return 0; 4374 } 4375 4376 if (mddev->curr_resync < max_sector) /* aborted */ 4377 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 4378 &sync_blocks, 1); 4379 else /* completed sync */ 4380 conf->fullsync = 0; 4381 bitmap_close_sync(mddev->bitmap); 4382 4383 return 0; 4384 } 4385 4386 /* Allow raid5_quiesce to complete */ 4387 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 4388 4389 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 4390 return reshape_request(mddev, sector_nr, skipped); 4391 4392 /* No need to check resync_max as we never do more than one 4393 * stripe, and as resync_max will always be on a chunk boundary, 4394 * if the check in md_do_sync didn't fire, there is no chance 4395 * of overstepping resync_max here 4396 */ 4397 4398 /* if there is too many failed drives and we are trying 4399 * to resync, then assert that we are finished, because there is 4400 * nothing we can do. 4401 */ 4402 if (mddev->degraded >= conf->max_degraded && 4403 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 4404 sector_t rv = mddev->dev_sectors - sector_nr; 4405 *skipped = 1; 4406 return rv; 4407 } 4408 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 4409 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 4410 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) { 4411 /* we can skip this block, and probably more */ 4412 sync_blocks /= STRIPE_SECTORS; 4413 *skipped = 1; 4414 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 4415 } 4416 4417 4418 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 4419 4420 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 4421 if (sh == NULL) { 4422 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 4423 /* make sure we don't swamp the stripe cache if someone else 4424 * is trying to get access 4425 */ 4426 schedule_timeout_uninterruptible(1); 4427 } 4428 /* Need to check if array will still be degraded after recovery/resync 4429 * We don't need to check the 'failed' flag as when that gets set, 4430 * recovery aborts. 4431 */ 4432 for (i = 0; i < conf->raid_disks; i++) 4433 if (conf->disks[i].rdev == NULL) 4434 still_degraded = 1; 4435 4436 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 4437 4438 spin_lock(&sh->lock); 4439 set_bit(STRIPE_SYNCING, &sh->state); 4440 clear_bit(STRIPE_INSYNC, &sh->state); 4441 spin_unlock(&sh->lock); 4442 4443 handle_stripe(sh); 4444 release_stripe(sh); 4445 4446 return STRIPE_SECTORS; 4447 } 4448 4449 static int retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio) 4450 { 4451 /* We may not be able to submit a whole bio at once as there 4452 * may not be enough stripe_heads available. 4453 * We cannot pre-allocate enough stripe_heads as we may need 4454 * more than exist in the cache (if we allow ever large chunks). 4455 * So we do one stripe head at a time and record in 4456 * ->bi_hw_segments how many have been done. 4457 * 4458 * We *know* that this entire raid_bio is in one chunk, so 4459 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 4460 */ 4461 struct stripe_head *sh; 4462 int dd_idx; 4463 sector_t sector, logical_sector, last_sector; 4464 int scnt = 0; 4465 int remaining; 4466 int handled = 0; 4467 4468 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4469 sector = raid5_compute_sector(conf, logical_sector, 4470 0, &dd_idx, NULL); 4471 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9); 4472 4473 for (; logical_sector < last_sector; 4474 logical_sector += STRIPE_SECTORS, 4475 sector += STRIPE_SECTORS, 4476 scnt++) { 4477 4478 if (scnt < raid5_bi_hw_segments(raid_bio)) 4479 /* already done this stripe */ 4480 continue; 4481 4482 sh = get_active_stripe(conf, sector, 0, 1, 0); 4483 4484 if (!sh) { 4485 /* failed to get a stripe - must wait */ 4486 raid5_set_bi_hw_segments(raid_bio, scnt); 4487 conf->retry_read_aligned = raid_bio; 4488 return handled; 4489 } 4490 4491 set_bit(R5_ReadError, &sh->dev[dd_idx].flags); 4492 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 4493 release_stripe(sh); 4494 raid5_set_bi_hw_segments(raid_bio, scnt); 4495 conf->retry_read_aligned = raid_bio; 4496 return handled; 4497 } 4498 4499 handle_stripe(sh); 4500 release_stripe(sh); 4501 handled++; 4502 } 4503 spin_lock_irq(&conf->device_lock); 4504 remaining = raid5_dec_bi_phys_segments(raid_bio); 4505 spin_unlock_irq(&conf->device_lock); 4506 if (remaining == 0) 4507 bio_endio(raid_bio, 0); 4508 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4509 wake_up(&conf->wait_for_stripe); 4510 return handled; 4511 } 4512 4513 4514 /* 4515 * This is our raid5 kernel thread. 4516 * 4517 * We scan the hash table for stripes which can be handled now. 4518 * During the scan, completed stripes are saved for us by the interrupt 4519 * handler, so that they will not have to wait for our next wakeup. 4520 */ 4521 static void raid5d(mddev_t *mddev) 4522 { 4523 struct stripe_head *sh; 4524 raid5_conf_t *conf = mddev->private; 4525 int handled; 4526 4527 pr_debug("+++ raid5d active\n"); 4528 4529 md_check_recovery(mddev); 4530 4531 handled = 0; 4532 spin_lock_irq(&conf->device_lock); 4533 while (1) { 4534 struct bio *bio; 4535 4536 if (conf->seq_flush != conf->seq_write) { 4537 int seq = conf->seq_flush; 4538 spin_unlock_irq(&conf->device_lock); 4539 bitmap_unplug(mddev->bitmap); 4540 spin_lock_irq(&conf->device_lock); 4541 conf->seq_write = seq; 4542 activate_bit_delay(conf); 4543 } 4544 4545 while ((bio = remove_bio_from_retry(conf))) { 4546 int ok; 4547 spin_unlock_irq(&conf->device_lock); 4548 ok = retry_aligned_read(conf, bio); 4549 spin_lock_irq(&conf->device_lock); 4550 if (!ok) 4551 break; 4552 handled++; 4553 } 4554 4555 sh = __get_priority_stripe(conf); 4556 4557 if (!sh) 4558 break; 4559 spin_unlock_irq(&conf->device_lock); 4560 4561 handled++; 4562 handle_stripe(sh); 4563 release_stripe(sh); 4564 cond_resched(); 4565 4566 spin_lock_irq(&conf->device_lock); 4567 } 4568 pr_debug("%d stripes handled\n", handled); 4569 4570 spin_unlock_irq(&conf->device_lock); 4571 4572 async_tx_issue_pending_all(); 4573 unplug_slaves(mddev); 4574 4575 pr_debug("--- raid5d inactive\n"); 4576 } 4577 4578 static ssize_t 4579 raid5_show_stripe_cache_size(mddev_t *mddev, char *page) 4580 { 4581 raid5_conf_t *conf = mddev->private; 4582 if (conf) 4583 return sprintf(page, "%d\n", conf->max_nr_stripes); 4584 else 4585 return 0; 4586 } 4587 4588 int 4589 raid5_set_cache_size(mddev_t *mddev, int size) 4590 { 4591 raid5_conf_t *conf = mddev->private; 4592 int err; 4593 4594 if (size <= 16 || size > 32768) 4595 return -EINVAL; 4596 while (size < conf->max_nr_stripes) { 4597 if (drop_one_stripe(conf)) 4598 conf->max_nr_stripes--; 4599 else 4600 break; 4601 } 4602 err = md_allow_write(mddev); 4603 if (err) 4604 return err; 4605 while (size > conf->max_nr_stripes) { 4606 if (grow_one_stripe(conf)) 4607 conf->max_nr_stripes++; 4608 else break; 4609 } 4610 return 0; 4611 } 4612 EXPORT_SYMBOL(raid5_set_cache_size); 4613 4614 static ssize_t 4615 raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len) 4616 { 4617 raid5_conf_t *conf = mddev->private; 4618 unsigned long new; 4619 int err; 4620 4621 if (len >= PAGE_SIZE) 4622 return -EINVAL; 4623 if (!conf) 4624 return -ENODEV; 4625 4626 if (strict_strtoul(page, 10, &new)) 4627 return -EINVAL; 4628 err = raid5_set_cache_size(mddev, new); 4629 if (err) 4630 return err; 4631 return len; 4632 } 4633 4634 static struct md_sysfs_entry 4635 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 4636 raid5_show_stripe_cache_size, 4637 raid5_store_stripe_cache_size); 4638 4639 static ssize_t 4640 raid5_show_preread_threshold(mddev_t *mddev, char *page) 4641 { 4642 raid5_conf_t *conf = mddev->private; 4643 if (conf) 4644 return sprintf(page, "%d\n", conf->bypass_threshold); 4645 else 4646 return 0; 4647 } 4648 4649 static ssize_t 4650 raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len) 4651 { 4652 raid5_conf_t *conf = mddev->private; 4653 unsigned long new; 4654 if (len >= PAGE_SIZE) 4655 return -EINVAL; 4656 if (!conf) 4657 return -ENODEV; 4658 4659 if (strict_strtoul(page, 10, &new)) 4660 return -EINVAL; 4661 if (new > conf->max_nr_stripes) 4662 return -EINVAL; 4663 conf->bypass_threshold = new; 4664 return len; 4665 } 4666 4667 static struct md_sysfs_entry 4668 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 4669 S_IRUGO | S_IWUSR, 4670 raid5_show_preread_threshold, 4671 raid5_store_preread_threshold); 4672 4673 static ssize_t 4674 stripe_cache_active_show(mddev_t *mddev, char *page) 4675 { 4676 raid5_conf_t *conf = mddev->private; 4677 if (conf) 4678 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 4679 else 4680 return 0; 4681 } 4682 4683 static struct md_sysfs_entry 4684 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 4685 4686 static struct attribute *raid5_attrs[] = { 4687 &raid5_stripecache_size.attr, 4688 &raid5_stripecache_active.attr, 4689 &raid5_preread_bypass_threshold.attr, 4690 NULL, 4691 }; 4692 static struct attribute_group raid5_attrs_group = { 4693 .name = NULL, 4694 .attrs = raid5_attrs, 4695 }; 4696 4697 static sector_t 4698 raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks) 4699 { 4700 raid5_conf_t *conf = mddev->private; 4701 4702 if (!sectors) 4703 sectors = mddev->dev_sectors; 4704 if (!raid_disks) 4705 /* size is defined by the smallest of previous and new size */ 4706 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 4707 4708 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 4709 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 4710 return sectors * (raid_disks - conf->max_degraded); 4711 } 4712 4713 static void raid5_free_percpu(raid5_conf_t *conf) 4714 { 4715 struct raid5_percpu *percpu; 4716 unsigned long cpu; 4717 4718 if (!conf->percpu) 4719 return; 4720 4721 get_online_cpus(); 4722 for_each_possible_cpu(cpu) { 4723 percpu = per_cpu_ptr(conf->percpu, cpu); 4724 safe_put_page(percpu->spare_page); 4725 kfree(percpu->scribble); 4726 } 4727 #ifdef CONFIG_HOTPLUG_CPU 4728 unregister_cpu_notifier(&conf->cpu_notify); 4729 #endif 4730 put_online_cpus(); 4731 4732 free_percpu(conf->percpu); 4733 } 4734 4735 static void free_conf(raid5_conf_t *conf) 4736 { 4737 shrink_stripes(conf); 4738 raid5_free_percpu(conf); 4739 kfree(conf->disks); 4740 kfree(conf->stripe_hashtbl); 4741 kfree(conf); 4742 } 4743 4744 #ifdef CONFIG_HOTPLUG_CPU 4745 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 4746 void *hcpu) 4747 { 4748 raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify); 4749 long cpu = (long)hcpu; 4750 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 4751 4752 switch (action) { 4753 case CPU_UP_PREPARE: 4754 case CPU_UP_PREPARE_FROZEN: 4755 if (conf->level == 6 && !percpu->spare_page) 4756 percpu->spare_page = alloc_page(GFP_KERNEL); 4757 if (!percpu->scribble) 4758 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4759 4760 if (!percpu->scribble || 4761 (conf->level == 6 && !percpu->spare_page)) { 4762 safe_put_page(percpu->spare_page); 4763 kfree(percpu->scribble); 4764 pr_err("%s: failed memory allocation for cpu%ld\n", 4765 __func__, cpu); 4766 return notifier_from_errno(-ENOMEM); 4767 } 4768 break; 4769 case CPU_DEAD: 4770 case CPU_DEAD_FROZEN: 4771 safe_put_page(percpu->spare_page); 4772 kfree(percpu->scribble); 4773 percpu->spare_page = NULL; 4774 percpu->scribble = NULL; 4775 break; 4776 default: 4777 break; 4778 } 4779 return NOTIFY_OK; 4780 } 4781 #endif 4782 4783 static int raid5_alloc_percpu(raid5_conf_t *conf) 4784 { 4785 unsigned long cpu; 4786 struct page *spare_page; 4787 struct raid5_percpu __percpu *allcpus; 4788 void *scribble; 4789 int err; 4790 4791 allcpus = alloc_percpu(struct raid5_percpu); 4792 if (!allcpus) 4793 return -ENOMEM; 4794 conf->percpu = allcpus; 4795 4796 get_online_cpus(); 4797 err = 0; 4798 for_each_present_cpu(cpu) { 4799 if (conf->level == 6) { 4800 spare_page = alloc_page(GFP_KERNEL); 4801 if (!spare_page) { 4802 err = -ENOMEM; 4803 break; 4804 } 4805 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page; 4806 } 4807 scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 4808 if (!scribble) { 4809 err = -ENOMEM; 4810 break; 4811 } 4812 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble; 4813 } 4814 #ifdef CONFIG_HOTPLUG_CPU 4815 conf->cpu_notify.notifier_call = raid456_cpu_notify; 4816 conf->cpu_notify.priority = 0; 4817 if (err == 0) 4818 err = register_cpu_notifier(&conf->cpu_notify); 4819 #endif 4820 put_online_cpus(); 4821 4822 return err; 4823 } 4824 4825 static raid5_conf_t *setup_conf(mddev_t *mddev) 4826 { 4827 raid5_conf_t *conf; 4828 int raid_disk, memory, max_disks; 4829 mdk_rdev_t *rdev; 4830 struct disk_info *disk; 4831 4832 if (mddev->new_level != 5 4833 && mddev->new_level != 4 4834 && mddev->new_level != 6) { 4835 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 4836 mdname(mddev), mddev->new_level); 4837 return ERR_PTR(-EIO); 4838 } 4839 if ((mddev->new_level == 5 4840 && !algorithm_valid_raid5(mddev->new_layout)) || 4841 (mddev->new_level == 6 4842 && !algorithm_valid_raid6(mddev->new_layout))) { 4843 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 4844 mdname(mddev), mddev->new_layout); 4845 return ERR_PTR(-EIO); 4846 } 4847 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 4848 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 4849 mdname(mddev), mddev->raid_disks); 4850 return ERR_PTR(-EINVAL); 4851 } 4852 4853 if (!mddev->new_chunk_sectors || 4854 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 4855 !is_power_of_2(mddev->new_chunk_sectors)) { 4856 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 4857 mdname(mddev), mddev->new_chunk_sectors << 9); 4858 return ERR_PTR(-EINVAL); 4859 } 4860 4861 conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL); 4862 if (conf == NULL) 4863 goto abort; 4864 spin_lock_init(&conf->device_lock); 4865 init_waitqueue_head(&conf->wait_for_stripe); 4866 init_waitqueue_head(&conf->wait_for_overlap); 4867 INIT_LIST_HEAD(&conf->handle_list); 4868 INIT_LIST_HEAD(&conf->hold_list); 4869 INIT_LIST_HEAD(&conf->delayed_list); 4870 INIT_LIST_HEAD(&conf->bitmap_list); 4871 INIT_LIST_HEAD(&conf->inactive_list); 4872 atomic_set(&conf->active_stripes, 0); 4873 atomic_set(&conf->preread_active_stripes, 0); 4874 atomic_set(&conf->active_aligned_reads, 0); 4875 conf->bypass_threshold = BYPASS_THRESHOLD; 4876 4877 conf->raid_disks = mddev->raid_disks; 4878 if (mddev->reshape_position == MaxSector) 4879 conf->previous_raid_disks = mddev->raid_disks; 4880 else 4881 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 4882 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 4883 conf->scribble_len = scribble_len(max_disks); 4884 4885 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 4886 GFP_KERNEL); 4887 if (!conf->disks) 4888 goto abort; 4889 4890 conf->mddev = mddev; 4891 4892 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 4893 goto abort; 4894 4895 conf->level = mddev->new_level; 4896 if (raid5_alloc_percpu(conf) != 0) 4897 goto abort; 4898 4899 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 4900 4901 list_for_each_entry(rdev, &mddev->disks, same_set) { 4902 raid_disk = rdev->raid_disk; 4903 if (raid_disk >= max_disks 4904 || raid_disk < 0) 4905 continue; 4906 disk = conf->disks + raid_disk; 4907 4908 disk->rdev = rdev; 4909 4910 if (test_bit(In_sync, &rdev->flags)) { 4911 char b[BDEVNAME_SIZE]; 4912 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 4913 " disk %d\n", 4914 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 4915 } else 4916 /* Cannot rely on bitmap to complete recovery */ 4917 conf->fullsync = 1; 4918 } 4919 4920 conf->chunk_sectors = mddev->new_chunk_sectors; 4921 conf->level = mddev->new_level; 4922 if (conf->level == 6) 4923 conf->max_degraded = 2; 4924 else 4925 conf->max_degraded = 1; 4926 conf->algorithm = mddev->new_layout; 4927 conf->max_nr_stripes = NR_STRIPES; 4928 conf->reshape_progress = mddev->reshape_position; 4929 if (conf->reshape_progress != MaxSector) { 4930 conf->prev_chunk_sectors = mddev->chunk_sectors; 4931 conf->prev_algo = mddev->layout; 4932 } 4933 4934 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 4935 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 4936 if (grow_stripes(conf, conf->max_nr_stripes)) { 4937 printk(KERN_ERR 4938 "md/raid:%s: couldn't allocate %dkB for buffers\n", 4939 mdname(mddev), memory); 4940 goto abort; 4941 } else 4942 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 4943 mdname(mddev), memory); 4944 4945 conf->thread = md_register_thread(raid5d, mddev, NULL); 4946 if (!conf->thread) { 4947 printk(KERN_ERR 4948 "md/raid:%s: couldn't allocate thread.\n", 4949 mdname(mddev)); 4950 goto abort; 4951 } 4952 4953 return conf; 4954 4955 abort: 4956 if (conf) { 4957 free_conf(conf); 4958 return ERR_PTR(-EIO); 4959 } else 4960 return ERR_PTR(-ENOMEM); 4961 } 4962 4963 4964 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 4965 { 4966 switch (algo) { 4967 case ALGORITHM_PARITY_0: 4968 if (raid_disk < max_degraded) 4969 return 1; 4970 break; 4971 case ALGORITHM_PARITY_N: 4972 if (raid_disk >= raid_disks - max_degraded) 4973 return 1; 4974 break; 4975 case ALGORITHM_PARITY_0_6: 4976 if (raid_disk == 0 || 4977 raid_disk == raid_disks - 1) 4978 return 1; 4979 break; 4980 case ALGORITHM_LEFT_ASYMMETRIC_6: 4981 case ALGORITHM_RIGHT_ASYMMETRIC_6: 4982 case ALGORITHM_LEFT_SYMMETRIC_6: 4983 case ALGORITHM_RIGHT_SYMMETRIC_6: 4984 if (raid_disk == raid_disks - 1) 4985 return 1; 4986 } 4987 return 0; 4988 } 4989 4990 static int run(mddev_t *mddev) 4991 { 4992 raid5_conf_t *conf; 4993 int working_disks = 0; 4994 int dirty_parity_disks = 0; 4995 mdk_rdev_t *rdev; 4996 sector_t reshape_offset = 0; 4997 4998 if (mddev->recovery_cp != MaxSector) 4999 printk(KERN_NOTICE "md/raid:%s: not clean" 5000 " -- starting background reconstruction\n", 5001 mdname(mddev)); 5002 if (mddev->reshape_position != MaxSector) { 5003 /* Check that we can continue the reshape. 5004 * Currently only disks can change, it must 5005 * increase, and we must be past the point where 5006 * a stripe over-writes itself 5007 */ 5008 sector_t here_new, here_old; 5009 int old_disks; 5010 int max_degraded = (mddev->level == 6 ? 2 : 1); 5011 5012 if (mddev->new_level != mddev->level) { 5013 printk(KERN_ERR "md/raid:%s: unsupported reshape " 5014 "required - aborting.\n", 5015 mdname(mddev)); 5016 return -EINVAL; 5017 } 5018 old_disks = mddev->raid_disks - mddev->delta_disks; 5019 /* reshape_position must be on a new-stripe boundary, and one 5020 * further up in new geometry must map after here in old 5021 * geometry. 5022 */ 5023 here_new = mddev->reshape_position; 5024 if (sector_div(here_new, mddev->new_chunk_sectors * 5025 (mddev->raid_disks - max_degraded))) { 5026 printk(KERN_ERR "md/raid:%s: reshape_position not " 5027 "on a stripe boundary\n", mdname(mddev)); 5028 return -EINVAL; 5029 } 5030 reshape_offset = here_new * mddev->new_chunk_sectors; 5031 /* here_new is the stripe we will write to */ 5032 here_old = mddev->reshape_position; 5033 sector_div(here_old, mddev->chunk_sectors * 5034 (old_disks-max_degraded)); 5035 /* here_old is the first stripe that we might need to read 5036 * from */ 5037 if (mddev->delta_disks == 0) { 5038 /* We cannot be sure it is safe to start an in-place 5039 * reshape. It is only safe if user-space if monitoring 5040 * and taking constant backups. 5041 * mdadm always starts a situation like this in 5042 * readonly mode so it can take control before 5043 * allowing any writes. So just check for that. 5044 */ 5045 if ((here_new * mddev->new_chunk_sectors != 5046 here_old * mddev->chunk_sectors) || 5047 mddev->ro == 0) { 5048 printk(KERN_ERR "md/raid:%s: in-place reshape must be started" 5049 " in read-only mode - aborting\n", 5050 mdname(mddev)); 5051 return -EINVAL; 5052 } 5053 } else if (mddev->delta_disks < 0 5054 ? (here_new * mddev->new_chunk_sectors <= 5055 here_old * mddev->chunk_sectors) 5056 : (here_new * mddev->new_chunk_sectors >= 5057 here_old * mddev->chunk_sectors)) { 5058 /* Reading from the same stripe as writing to - bad */ 5059 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 5060 "auto-recovery - aborting.\n", 5061 mdname(mddev)); 5062 return -EINVAL; 5063 } 5064 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 5065 mdname(mddev)); 5066 /* OK, we should be able to continue; */ 5067 } else { 5068 BUG_ON(mddev->level != mddev->new_level); 5069 BUG_ON(mddev->layout != mddev->new_layout); 5070 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 5071 BUG_ON(mddev->delta_disks != 0); 5072 } 5073 5074 if (mddev->private == NULL) 5075 conf = setup_conf(mddev); 5076 else 5077 conf = mddev->private; 5078 5079 if (IS_ERR(conf)) 5080 return PTR_ERR(conf); 5081 5082 mddev->thread = conf->thread; 5083 conf->thread = NULL; 5084 mddev->private = conf; 5085 5086 /* 5087 * 0 for a fully functional array, 1 or 2 for a degraded array. 5088 */ 5089 list_for_each_entry(rdev, &mddev->disks, same_set) { 5090 if (rdev->raid_disk < 0) 5091 continue; 5092 if (test_bit(In_sync, &rdev->flags)) { 5093 working_disks++; 5094 continue; 5095 } 5096 /* This disc is not fully in-sync. However if it 5097 * just stored parity (beyond the recovery_offset), 5098 * when we don't need to be concerned about the 5099 * array being dirty. 5100 * When reshape goes 'backwards', we never have 5101 * partially completed devices, so we only need 5102 * to worry about reshape going forwards. 5103 */ 5104 /* Hack because v0.91 doesn't store recovery_offset properly. */ 5105 if (mddev->major_version == 0 && 5106 mddev->minor_version > 90) 5107 rdev->recovery_offset = reshape_offset; 5108 5109 if (rdev->recovery_offset < reshape_offset) { 5110 /* We need to check old and new layout */ 5111 if (!only_parity(rdev->raid_disk, 5112 conf->algorithm, 5113 conf->raid_disks, 5114 conf->max_degraded)) 5115 continue; 5116 } 5117 if (!only_parity(rdev->raid_disk, 5118 conf->prev_algo, 5119 conf->previous_raid_disks, 5120 conf->max_degraded)) 5121 continue; 5122 dirty_parity_disks++; 5123 } 5124 5125 mddev->degraded = (max(conf->raid_disks, conf->previous_raid_disks) 5126 - working_disks); 5127 5128 if (has_failed(conf)) { 5129 printk(KERN_ERR "md/raid:%s: not enough operational devices" 5130 " (%d/%d failed)\n", 5131 mdname(mddev), mddev->degraded, conf->raid_disks); 5132 goto abort; 5133 } 5134 5135 /* device size must be a multiple of chunk size */ 5136 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 5137 mddev->resync_max_sectors = mddev->dev_sectors; 5138 5139 if (mddev->degraded > dirty_parity_disks && 5140 mddev->recovery_cp != MaxSector) { 5141 if (mddev->ok_start_degraded) 5142 printk(KERN_WARNING 5143 "md/raid:%s: starting dirty degraded array" 5144 " - data corruption possible.\n", 5145 mdname(mddev)); 5146 else { 5147 printk(KERN_ERR 5148 "md/raid:%s: cannot start dirty degraded array.\n", 5149 mdname(mddev)); 5150 goto abort; 5151 } 5152 } 5153 5154 if (mddev->degraded == 0) 5155 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 5156 " devices, algorithm %d\n", mdname(mddev), conf->level, 5157 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 5158 mddev->new_layout); 5159 else 5160 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 5161 " out of %d devices, algorithm %d\n", 5162 mdname(mddev), conf->level, 5163 mddev->raid_disks - mddev->degraded, 5164 mddev->raid_disks, mddev->new_layout); 5165 5166 print_raid5_conf(conf); 5167 5168 if (conf->reshape_progress != MaxSector) { 5169 conf->reshape_safe = conf->reshape_progress; 5170 atomic_set(&conf->reshape_stripes, 0); 5171 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5172 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5173 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5174 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5175 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5176 "reshape"); 5177 } 5178 5179 5180 /* Ok, everything is just fine now */ 5181 if (mddev->to_remove == &raid5_attrs_group) 5182 mddev->to_remove = NULL; 5183 else if (mddev->kobj.sd && 5184 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 5185 printk(KERN_WARNING 5186 "raid5: failed to create sysfs attributes for %s\n", 5187 mdname(mddev)); 5188 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5189 5190 plugger_init(&conf->plug, raid5_unplug); 5191 mddev->plug = &conf->plug; 5192 if (mddev->queue) { 5193 int chunk_size; 5194 /* read-ahead size must cover two whole stripes, which 5195 * is 2 * (datadisks) * chunksize where 'n' is the 5196 * number of raid devices 5197 */ 5198 int data_disks = conf->previous_raid_disks - conf->max_degraded; 5199 int stripe = data_disks * 5200 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 5201 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5202 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5203 5204 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 5205 5206 mddev->queue->backing_dev_info.congested_data = mddev; 5207 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 5208 mddev->queue->queue_lock = &conf->device_lock; 5209 mddev->queue->unplug_fn = raid5_unplug_queue; 5210 5211 chunk_size = mddev->chunk_sectors << 9; 5212 blk_queue_io_min(mddev->queue, chunk_size); 5213 blk_queue_io_opt(mddev->queue, chunk_size * 5214 (conf->raid_disks - conf->max_degraded)); 5215 5216 list_for_each_entry(rdev, &mddev->disks, same_set) 5217 disk_stack_limits(mddev->gendisk, rdev->bdev, 5218 rdev->data_offset << 9); 5219 } 5220 5221 return 0; 5222 abort: 5223 md_unregister_thread(mddev->thread); 5224 mddev->thread = NULL; 5225 if (conf) { 5226 print_raid5_conf(conf); 5227 free_conf(conf); 5228 } 5229 mddev->private = NULL; 5230 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 5231 return -EIO; 5232 } 5233 5234 static int stop(mddev_t *mddev) 5235 { 5236 raid5_conf_t *conf = mddev->private; 5237 5238 md_unregister_thread(mddev->thread); 5239 mddev->thread = NULL; 5240 if (mddev->queue) 5241 mddev->queue->backing_dev_info.congested_fn = NULL; 5242 plugger_flush(&conf->plug); /* the unplug fn references 'conf'*/ 5243 free_conf(conf); 5244 mddev->private = NULL; 5245 mddev->to_remove = &raid5_attrs_group; 5246 return 0; 5247 } 5248 5249 #ifdef DEBUG 5250 static void print_sh(struct seq_file *seq, struct stripe_head *sh) 5251 { 5252 int i; 5253 5254 seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n", 5255 (unsigned long long)sh->sector, sh->pd_idx, sh->state); 5256 seq_printf(seq, "sh %llu, count %d.\n", 5257 (unsigned long long)sh->sector, atomic_read(&sh->count)); 5258 seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector); 5259 for (i = 0; i < sh->disks; i++) { 5260 seq_printf(seq, "(cache%d: %p %ld) ", 5261 i, sh->dev[i].page, sh->dev[i].flags); 5262 } 5263 seq_printf(seq, "\n"); 5264 } 5265 5266 static void printall(struct seq_file *seq, raid5_conf_t *conf) 5267 { 5268 struct stripe_head *sh; 5269 struct hlist_node *hn; 5270 int i; 5271 5272 spin_lock_irq(&conf->device_lock); 5273 for (i = 0; i < NR_HASH; i++) { 5274 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) { 5275 if (sh->raid_conf != conf) 5276 continue; 5277 print_sh(seq, sh); 5278 } 5279 } 5280 spin_unlock_irq(&conf->device_lock); 5281 } 5282 #endif 5283 5284 static void status(struct seq_file *seq, mddev_t *mddev) 5285 { 5286 raid5_conf_t *conf = mddev->private; 5287 int i; 5288 5289 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 5290 mddev->chunk_sectors / 2, mddev->layout); 5291 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 5292 for (i = 0; i < conf->raid_disks; i++) 5293 seq_printf (seq, "%s", 5294 conf->disks[i].rdev && 5295 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 5296 seq_printf (seq, "]"); 5297 #ifdef DEBUG 5298 seq_printf (seq, "\n"); 5299 printall(seq, conf); 5300 #endif 5301 } 5302 5303 static void print_raid5_conf (raid5_conf_t *conf) 5304 { 5305 int i; 5306 struct disk_info *tmp; 5307 5308 printk(KERN_DEBUG "RAID conf printout:\n"); 5309 if (!conf) { 5310 printk("(conf==NULL)\n"); 5311 return; 5312 } 5313 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 5314 conf->raid_disks, 5315 conf->raid_disks - conf->mddev->degraded); 5316 5317 for (i = 0; i < conf->raid_disks; i++) { 5318 char b[BDEVNAME_SIZE]; 5319 tmp = conf->disks + i; 5320 if (tmp->rdev) 5321 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 5322 i, !test_bit(Faulty, &tmp->rdev->flags), 5323 bdevname(tmp->rdev->bdev, b)); 5324 } 5325 } 5326 5327 static int raid5_spare_active(mddev_t *mddev) 5328 { 5329 int i; 5330 raid5_conf_t *conf = mddev->private; 5331 struct disk_info *tmp; 5332 int count = 0; 5333 unsigned long flags; 5334 5335 for (i = 0; i < conf->raid_disks; i++) { 5336 tmp = conf->disks + i; 5337 if (tmp->rdev 5338 && tmp->rdev->recovery_offset == MaxSector 5339 && !test_bit(Faulty, &tmp->rdev->flags) 5340 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 5341 count++; 5342 sysfs_notify_dirent(tmp->rdev->sysfs_state); 5343 } 5344 } 5345 spin_lock_irqsave(&conf->device_lock, flags); 5346 mddev->degraded -= count; 5347 spin_unlock_irqrestore(&conf->device_lock, flags); 5348 print_raid5_conf(conf); 5349 return count; 5350 } 5351 5352 static int raid5_remove_disk(mddev_t *mddev, int number) 5353 { 5354 raid5_conf_t *conf = mddev->private; 5355 int err = 0; 5356 mdk_rdev_t *rdev; 5357 struct disk_info *p = conf->disks + number; 5358 5359 print_raid5_conf(conf); 5360 rdev = p->rdev; 5361 if (rdev) { 5362 if (number >= conf->raid_disks && 5363 conf->reshape_progress == MaxSector) 5364 clear_bit(In_sync, &rdev->flags); 5365 5366 if (test_bit(In_sync, &rdev->flags) || 5367 atomic_read(&rdev->nr_pending)) { 5368 err = -EBUSY; 5369 goto abort; 5370 } 5371 /* Only remove non-faulty devices if recovery 5372 * isn't possible. 5373 */ 5374 if (!test_bit(Faulty, &rdev->flags) && 5375 !has_failed(conf) && 5376 number < conf->raid_disks) { 5377 err = -EBUSY; 5378 goto abort; 5379 } 5380 p->rdev = NULL; 5381 synchronize_rcu(); 5382 if (atomic_read(&rdev->nr_pending)) { 5383 /* lost the race, try later */ 5384 err = -EBUSY; 5385 p->rdev = rdev; 5386 } 5387 } 5388 abort: 5389 5390 print_raid5_conf(conf); 5391 return err; 5392 } 5393 5394 static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev) 5395 { 5396 raid5_conf_t *conf = mddev->private; 5397 int err = -EEXIST; 5398 int disk; 5399 struct disk_info *p; 5400 int first = 0; 5401 int last = conf->raid_disks - 1; 5402 5403 if (has_failed(conf)) 5404 /* no point adding a device */ 5405 return -EINVAL; 5406 5407 if (rdev->raid_disk >= 0) 5408 first = last = rdev->raid_disk; 5409 5410 /* 5411 * find the disk ... but prefer rdev->saved_raid_disk 5412 * if possible. 5413 */ 5414 if (rdev->saved_raid_disk >= 0 && 5415 rdev->saved_raid_disk >= first && 5416 conf->disks[rdev->saved_raid_disk].rdev == NULL) 5417 disk = rdev->saved_raid_disk; 5418 else 5419 disk = first; 5420 for ( ; disk <= last ; disk++) 5421 if ((p=conf->disks + disk)->rdev == NULL) { 5422 clear_bit(In_sync, &rdev->flags); 5423 rdev->raid_disk = disk; 5424 err = 0; 5425 if (rdev->saved_raid_disk != disk) 5426 conf->fullsync = 1; 5427 rcu_assign_pointer(p->rdev, rdev); 5428 break; 5429 } 5430 print_raid5_conf(conf); 5431 return err; 5432 } 5433 5434 static int raid5_resize(mddev_t *mddev, sector_t sectors) 5435 { 5436 /* no resync is happening, and there is enough space 5437 * on all devices, so we can resize. 5438 * We need to make sure resync covers any new space. 5439 * If the array is shrinking we should possibly wait until 5440 * any io in the removed space completes, but it hardly seems 5441 * worth it. 5442 */ 5443 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5444 md_set_array_sectors(mddev, raid5_size(mddev, sectors, 5445 mddev->raid_disks)); 5446 if (mddev->array_sectors > 5447 raid5_size(mddev, sectors, mddev->raid_disks)) 5448 return -EINVAL; 5449 set_capacity(mddev->gendisk, mddev->array_sectors); 5450 revalidate_disk(mddev->gendisk); 5451 if (sectors > mddev->dev_sectors && mddev->recovery_cp == MaxSector) { 5452 mddev->recovery_cp = mddev->dev_sectors; 5453 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 5454 } 5455 mddev->dev_sectors = sectors; 5456 mddev->resync_max_sectors = sectors; 5457 return 0; 5458 } 5459 5460 static int check_stripe_cache(mddev_t *mddev) 5461 { 5462 /* Can only proceed if there are plenty of stripe_heads. 5463 * We need a minimum of one full stripe,, and for sensible progress 5464 * it is best to have about 4 times that. 5465 * If we require 4 times, then the default 256 4K stripe_heads will 5466 * allow for chunk sizes up to 256K, which is probably OK. 5467 * If the chunk size is greater, user-space should request more 5468 * stripe_heads first. 5469 */ 5470 raid5_conf_t *conf = mddev->private; 5471 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 5472 > conf->max_nr_stripes || 5473 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 5474 > conf->max_nr_stripes) { 5475 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 5476 mdname(mddev), 5477 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 5478 / STRIPE_SIZE)*4); 5479 return 0; 5480 } 5481 return 1; 5482 } 5483 5484 static int check_reshape(mddev_t *mddev) 5485 { 5486 raid5_conf_t *conf = mddev->private; 5487 5488 if (mddev->delta_disks == 0 && 5489 mddev->new_layout == mddev->layout && 5490 mddev->new_chunk_sectors == mddev->chunk_sectors) 5491 return 0; /* nothing to do */ 5492 if (mddev->bitmap) 5493 /* Cannot grow a bitmap yet */ 5494 return -EBUSY; 5495 if (has_failed(conf)) 5496 return -EINVAL; 5497 if (mddev->delta_disks < 0) { 5498 /* We might be able to shrink, but the devices must 5499 * be made bigger first. 5500 * For raid6, 4 is the minimum size. 5501 * Otherwise 2 is the minimum 5502 */ 5503 int min = 2; 5504 if (mddev->level == 6) 5505 min = 4; 5506 if (mddev->raid_disks + mddev->delta_disks < min) 5507 return -EINVAL; 5508 } 5509 5510 if (!check_stripe_cache(mddev)) 5511 return -ENOSPC; 5512 5513 return resize_stripes(conf, conf->raid_disks + mddev->delta_disks); 5514 } 5515 5516 static int raid5_start_reshape(mddev_t *mddev) 5517 { 5518 raid5_conf_t *conf = mddev->private; 5519 mdk_rdev_t *rdev; 5520 int spares = 0; 5521 int added_devices = 0; 5522 unsigned long flags; 5523 5524 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 5525 return -EBUSY; 5526 5527 if (!check_stripe_cache(mddev)) 5528 return -ENOSPC; 5529 5530 list_for_each_entry(rdev, &mddev->disks, same_set) 5531 if (rdev->raid_disk < 0 && 5532 !test_bit(Faulty, &rdev->flags)) 5533 spares++; 5534 5535 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 5536 /* Not enough devices even to make a degraded array 5537 * of that size 5538 */ 5539 return -EINVAL; 5540 5541 /* Refuse to reduce size of the array. Any reductions in 5542 * array size must be through explicit setting of array_size 5543 * attribute. 5544 */ 5545 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 5546 < mddev->array_sectors) { 5547 printk(KERN_ERR "md/raid:%s: array size must be reduced " 5548 "before number of disks\n", mdname(mddev)); 5549 return -EINVAL; 5550 } 5551 5552 atomic_set(&conf->reshape_stripes, 0); 5553 spin_lock_irq(&conf->device_lock); 5554 conf->previous_raid_disks = conf->raid_disks; 5555 conf->raid_disks += mddev->delta_disks; 5556 conf->prev_chunk_sectors = conf->chunk_sectors; 5557 conf->chunk_sectors = mddev->new_chunk_sectors; 5558 conf->prev_algo = conf->algorithm; 5559 conf->algorithm = mddev->new_layout; 5560 if (mddev->delta_disks < 0) 5561 conf->reshape_progress = raid5_size(mddev, 0, 0); 5562 else 5563 conf->reshape_progress = 0; 5564 conf->reshape_safe = conf->reshape_progress; 5565 conf->generation++; 5566 spin_unlock_irq(&conf->device_lock); 5567 5568 /* Add some new drives, as many as will fit. 5569 * We know there are enough to make the newly sized array work. 5570 * Don't add devices if we are reducing the number of 5571 * devices in the array. This is because it is not possible 5572 * to correctly record the "partially reconstructed" state of 5573 * such devices during the reshape and confusion could result. 5574 */ 5575 if (mddev->delta_disks >= 0) 5576 list_for_each_entry(rdev, &mddev->disks, same_set) 5577 if (rdev->raid_disk < 0 && 5578 !test_bit(Faulty, &rdev->flags)) { 5579 if (raid5_add_disk(mddev, rdev) == 0) { 5580 char nm[20]; 5581 if (rdev->raid_disk >= conf->previous_raid_disks) { 5582 set_bit(In_sync, &rdev->flags); 5583 added_devices++; 5584 } else 5585 rdev->recovery_offset = 0; 5586 sprintf(nm, "rd%d", rdev->raid_disk); 5587 if (sysfs_create_link(&mddev->kobj, 5588 &rdev->kobj, nm)) 5589 /* Failure here is OK */; 5590 } else 5591 break; 5592 } 5593 5594 /* When a reshape changes the number of devices, ->degraded 5595 * is measured against the larger of the pre and post number of 5596 * devices.*/ 5597 if (mddev->delta_disks > 0) { 5598 spin_lock_irqsave(&conf->device_lock, flags); 5599 mddev->degraded += (conf->raid_disks - conf->previous_raid_disks) 5600 - added_devices; 5601 spin_unlock_irqrestore(&conf->device_lock, flags); 5602 } 5603 mddev->raid_disks = conf->raid_disks; 5604 mddev->reshape_position = conf->reshape_progress; 5605 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5606 5607 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5608 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5609 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5610 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5611 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5612 "reshape"); 5613 if (!mddev->sync_thread) { 5614 mddev->recovery = 0; 5615 spin_lock_irq(&conf->device_lock); 5616 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 5617 conf->reshape_progress = MaxSector; 5618 spin_unlock_irq(&conf->device_lock); 5619 return -EAGAIN; 5620 } 5621 conf->reshape_checkpoint = jiffies; 5622 md_wakeup_thread(mddev->sync_thread); 5623 md_new_event(mddev); 5624 return 0; 5625 } 5626 5627 /* This is called from the reshape thread and should make any 5628 * changes needed in 'conf' 5629 */ 5630 static void end_reshape(raid5_conf_t *conf) 5631 { 5632 5633 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 5634 5635 spin_lock_irq(&conf->device_lock); 5636 conf->previous_raid_disks = conf->raid_disks; 5637 conf->reshape_progress = MaxSector; 5638 spin_unlock_irq(&conf->device_lock); 5639 wake_up(&conf->wait_for_overlap); 5640 5641 /* read-ahead size must cover two whole stripes, which is 5642 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 5643 */ 5644 if (conf->mddev->queue) { 5645 int data_disks = conf->raid_disks - conf->max_degraded; 5646 int stripe = data_disks * ((conf->chunk_sectors << 9) 5647 / PAGE_SIZE); 5648 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5649 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5650 } 5651 } 5652 } 5653 5654 /* This is called from the raid5d thread with mddev_lock held. 5655 * It makes config changes to the device. 5656 */ 5657 static void raid5_finish_reshape(mddev_t *mddev) 5658 { 5659 raid5_conf_t *conf = mddev->private; 5660 5661 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 5662 5663 if (mddev->delta_disks > 0) { 5664 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5665 set_capacity(mddev->gendisk, mddev->array_sectors); 5666 revalidate_disk(mddev->gendisk); 5667 } else { 5668 int d; 5669 mddev->degraded = conf->raid_disks; 5670 for (d = 0; d < conf->raid_disks ; d++) 5671 if (conf->disks[d].rdev && 5672 test_bit(In_sync, 5673 &conf->disks[d].rdev->flags)) 5674 mddev->degraded--; 5675 for (d = conf->raid_disks ; 5676 d < conf->raid_disks - mddev->delta_disks; 5677 d++) { 5678 mdk_rdev_t *rdev = conf->disks[d].rdev; 5679 if (rdev && raid5_remove_disk(mddev, d) == 0) { 5680 char nm[20]; 5681 sprintf(nm, "rd%d", rdev->raid_disk); 5682 sysfs_remove_link(&mddev->kobj, nm); 5683 rdev->raid_disk = -1; 5684 } 5685 } 5686 } 5687 mddev->layout = conf->algorithm; 5688 mddev->chunk_sectors = conf->chunk_sectors; 5689 mddev->reshape_position = MaxSector; 5690 mddev->delta_disks = 0; 5691 } 5692 } 5693 5694 static void raid5_quiesce(mddev_t *mddev, int state) 5695 { 5696 raid5_conf_t *conf = mddev->private; 5697 5698 switch(state) { 5699 case 2: /* resume for a suspend */ 5700 wake_up(&conf->wait_for_overlap); 5701 break; 5702 5703 case 1: /* stop all writes */ 5704 spin_lock_irq(&conf->device_lock); 5705 /* '2' tells resync/reshape to pause so that all 5706 * active stripes can drain 5707 */ 5708 conf->quiesce = 2; 5709 wait_event_lock_irq(conf->wait_for_stripe, 5710 atomic_read(&conf->active_stripes) == 0 && 5711 atomic_read(&conf->active_aligned_reads) == 0, 5712 conf->device_lock, /* nothing */); 5713 conf->quiesce = 1; 5714 spin_unlock_irq(&conf->device_lock); 5715 /* allow reshape to continue */ 5716 wake_up(&conf->wait_for_overlap); 5717 break; 5718 5719 case 0: /* re-enable writes */ 5720 spin_lock_irq(&conf->device_lock); 5721 conf->quiesce = 0; 5722 wake_up(&conf->wait_for_stripe); 5723 wake_up(&conf->wait_for_overlap); 5724 spin_unlock_irq(&conf->device_lock); 5725 break; 5726 } 5727 } 5728 5729 5730 static void *raid45_takeover_raid0(mddev_t *mddev, int level) 5731 { 5732 struct raid0_private_data *raid0_priv = mddev->private; 5733 5734 /* for raid0 takeover only one zone is supported */ 5735 if (raid0_priv->nr_strip_zones > 1) { 5736 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 5737 mdname(mddev)); 5738 return ERR_PTR(-EINVAL); 5739 } 5740 5741 mddev->new_level = level; 5742 mddev->new_layout = ALGORITHM_PARITY_N; 5743 mddev->new_chunk_sectors = mddev->chunk_sectors; 5744 mddev->raid_disks += 1; 5745 mddev->delta_disks = 1; 5746 /* make sure it will be not marked as dirty */ 5747 mddev->recovery_cp = MaxSector; 5748 5749 return setup_conf(mddev); 5750 } 5751 5752 5753 static void *raid5_takeover_raid1(mddev_t *mddev) 5754 { 5755 int chunksect; 5756 5757 if (mddev->raid_disks != 2 || 5758 mddev->degraded > 1) 5759 return ERR_PTR(-EINVAL); 5760 5761 /* Should check if there are write-behind devices? */ 5762 5763 chunksect = 64*2; /* 64K by default */ 5764 5765 /* The array must be an exact multiple of chunksize */ 5766 while (chunksect && (mddev->array_sectors & (chunksect-1))) 5767 chunksect >>= 1; 5768 5769 if ((chunksect<<9) < STRIPE_SIZE) 5770 /* array size does not allow a suitable chunk size */ 5771 return ERR_PTR(-EINVAL); 5772 5773 mddev->new_level = 5; 5774 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 5775 mddev->new_chunk_sectors = chunksect; 5776 5777 return setup_conf(mddev); 5778 } 5779 5780 static void *raid5_takeover_raid6(mddev_t *mddev) 5781 { 5782 int new_layout; 5783 5784 switch (mddev->layout) { 5785 case ALGORITHM_LEFT_ASYMMETRIC_6: 5786 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 5787 break; 5788 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5789 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 5790 break; 5791 case ALGORITHM_LEFT_SYMMETRIC_6: 5792 new_layout = ALGORITHM_LEFT_SYMMETRIC; 5793 break; 5794 case ALGORITHM_RIGHT_SYMMETRIC_6: 5795 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 5796 break; 5797 case ALGORITHM_PARITY_0_6: 5798 new_layout = ALGORITHM_PARITY_0; 5799 break; 5800 case ALGORITHM_PARITY_N: 5801 new_layout = ALGORITHM_PARITY_N; 5802 break; 5803 default: 5804 return ERR_PTR(-EINVAL); 5805 } 5806 mddev->new_level = 5; 5807 mddev->new_layout = new_layout; 5808 mddev->delta_disks = -1; 5809 mddev->raid_disks -= 1; 5810 return setup_conf(mddev); 5811 } 5812 5813 5814 static int raid5_check_reshape(mddev_t *mddev) 5815 { 5816 /* For a 2-drive array, the layout and chunk size can be changed 5817 * immediately as not restriping is needed. 5818 * For larger arrays we record the new value - after validation 5819 * to be used by a reshape pass. 5820 */ 5821 raid5_conf_t *conf = mddev->private; 5822 int new_chunk = mddev->new_chunk_sectors; 5823 5824 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 5825 return -EINVAL; 5826 if (new_chunk > 0) { 5827 if (!is_power_of_2(new_chunk)) 5828 return -EINVAL; 5829 if (new_chunk < (PAGE_SIZE>>9)) 5830 return -EINVAL; 5831 if (mddev->array_sectors & (new_chunk-1)) 5832 /* not factor of array size */ 5833 return -EINVAL; 5834 } 5835 5836 /* They look valid */ 5837 5838 if (mddev->raid_disks == 2) { 5839 /* can make the change immediately */ 5840 if (mddev->new_layout >= 0) { 5841 conf->algorithm = mddev->new_layout; 5842 mddev->layout = mddev->new_layout; 5843 } 5844 if (new_chunk > 0) { 5845 conf->chunk_sectors = new_chunk ; 5846 mddev->chunk_sectors = new_chunk; 5847 } 5848 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5849 md_wakeup_thread(mddev->thread); 5850 } 5851 return check_reshape(mddev); 5852 } 5853 5854 static int raid6_check_reshape(mddev_t *mddev) 5855 { 5856 int new_chunk = mddev->new_chunk_sectors; 5857 5858 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 5859 return -EINVAL; 5860 if (new_chunk > 0) { 5861 if (!is_power_of_2(new_chunk)) 5862 return -EINVAL; 5863 if (new_chunk < (PAGE_SIZE >> 9)) 5864 return -EINVAL; 5865 if (mddev->array_sectors & (new_chunk-1)) 5866 /* not factor of array size */ 5867 return -EINVAL; 5868 } 5869 5870 /* They look valid */ 5871 return check_reshape(mddev); 5872 } 5873 5874 static void *raid5_takeover(mddev_t *mddev) 5875 { 5876 /* raid5 can take over: 5877 * raid0 - if there is only one strip zone - make it a raid4 layout 5878 * raid1 - if there are two drives. We need to know the chunk size 5879 * raid4 - trivial - just use a raid4 layout. 5880 * raid6 - Providing it is a *_6 layout 5881 */ 5882 if (mddev->level == 0) 5883 return raid45_takeover_raid0(mddev, 5); 5884 if (mddev->level == 1) 5885 return raid5_takeover_raid1(mddev); 5886 if (mddev->level == 4) { 5887 mddev->new_layout = ALGORITHM_PARITY_N; 5888 mddev->new_level = 5; 5889 return setup_conf(mddev); 5890 } 5891 if (mddev->level == 6) 5892 return raid5_takeover_raid6(mddev); 5893 5894 return ERR_PTR(-EINVAL); 5895 } 5896 5897 static void *raid4_takeover(mddev_t *mddev) 5898 { 5899 /* raid4 can take over: 5900 * raid0 - if there is only one strip zone 5901 * raid5 - if layout is right 5902 */ 5903 if (mddev->level == 0) 5904 return raid45_takeover_raid0(mddev, 4); 5905 if (mddev->level == 5 && 5906 mddev->layout == ALGORITHM_PARITY_N) { 5907 mddev->new_layout = 0; 5908 mddev->new_level = 4; 5909 return setup_conf(mddev); 5910 } 5911 return ERR_PTR(-EINVAL); 5912 } 5913 5914 static struct mdk_personality raid5_personality; 5915 5916 static void *raid6_takeover(mddev_t *mddev) 5917 { 5918 /* Currently can only take over a raid5. We map the 5919 * personality to an equivalent raid6 personality 5920 * with the Q block at the end. 5921 */ 5922 int new_layout; 5923 5924 if (mddev->pers != &raid5_personality) 5925 return ERR_PTR(-EINVAL); 5926 if (mddev->degraded > 1) 5927 return ERR_PTR(-EINVAL); 5928 if (mddev->raid_disks > 253) 5929 return ERR_PTR(-EINVAL); 5930 if (mddev->raid_disks < 3) 5931 return ERR_PTR(-EINVAL); 5932 5933 switch (mddev->layout) { 5934 case ALGORITHM_LEFT_ASYMMETRIC: 5935 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 5936 break; 5937 case ALGORITHM_RIGHT_ASYMMETRIC: 5938 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 5939 break; 5940 case ALGORITHM_LEFT_SYMMETRIC: 5941 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 5942 break; 5943 case ALGORITHM_RIGHT_SYMMETRIC: 5944 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 5945 break; 5946 case ALGORITHM_PARITY_0: 5947 new_layout = ALGORITHM_PARITY_0_6; 5948 break; 5949 case ALGORITHM_PARITY_N: 5950 new_layout = ALGORITHM_PARITY_N; 5951 break; 5952 default: 5953 return ERR_PTR(-EINVAL); 5954 } 5955 mddev->new_level = 6; 5956 mddev->new_layout = new_layout; 5957 mddev->delta_disks = 1; 5958 mddev->raid_disks += 1; 5959 return setup_conf(mddev); 5960 } 5961 5962 5963 static struct mdk_personality raid6_personality = 5964 { 5965 .name = "raid6", 5966 .level = 6, 5967 .owner = THIS_MODULE, 5968 .make_request = make_request, 5969 .run = run, 5970 .stop = stop, 5971 .status = status, 5972 .error_handler = error, 5973 .hot_add_disk = raid5_add_disk, 5974 .hot_remove_disk= raid5_remove_disk, 5975 .spare_active = raid5_spare_active, 5976 .sync_request = sync_request, 5977 .resize = raid5_resize, 5978 .size = raid5_size, 5979 .check_reshape = raid6_check_reshape, 5980 .start_reshape = raid5_start_reshape, 5981 .finish_reshape = raid5_finish_reshape, 5982 .quiesce = raid5_quiesce, 5983 .takeover = raid6_takeover, 5984 }; 5985 static struct mdk_personality raid5_personality = 5986 { 5987 .name = "raid5", 5988 .level = 5, 5989 .owner = THIS_MODULE, 5990 .make_request = make_request, 5991 .run = run, 5992 .stop = stop, 5993 .status = status, 5994 .error_handler = error, 5995 .hot_add_disk = raid5_add_disk, 5996 .hot_remove_disk= raid5_remove_disk, 5997 .spare_active = raid5_spare_active, 5998 .sync_request = sync_request, 5999 .resize = raid5_resize, 6000 .size = raid5_size, 6001 .check_reshape = raid5_check_reshape, 6002 .start_reshape = raid5_start_reshape, 6003 .finish_reshape = raid5_finish_reshape, 6004 .quiesce = raid5_quiesce, 6005 .takeover = raid5_takeover, 6006 }; 6007 6008 static struct mdk_personality raid4_personality = 6009 { 6010 .name = "raid4", 6011 .level = 4, 6012 .owner = THIS_MODULE, 6013 .make_request = make_request, 6014 .run = run, 6015 .stop = stop, 6016 .status = status, 6017 .error_handler = error, 6018 .hot_add_disk = raid5_add_disk, 6019 .hot_remove_disk= raid5_remove_disk, 6020 .spare_active = raid5_spare_active, 6021 .sync_request = sync_request, 6022 .resize = raid5_resize, 6023 .size = raid5_size, 6024 .check_reshape = raid5_check_reshape, 6025 .start_reshape = raid5_start_reshape, 6026 .finish_reshape = raid5_finish_reshape, 6027 .quiesce = raid5_quiesce, 6028 .takeover = raid4_takeover, 6029 }; 6030 6031 static int __init raid5_init(void) 6032 { 6033 register_md_personality(&raid6_personality); 6034 register_md_personality(&raid5_personality); 6035 register_md_personality(&raid4_personality); 6036 return 0; 6037 } 6038 6039 static void raid5_exit(void) 6040 { 6041 unregister_md_personality(&raid6_personality); 6042 unregister_md_personality(&raid5_personality); 6043 unregister_md_personality(&raid4_personality); 6044 } 6045 6046 module_init(raid5_init); 6047 module_exit(raid5_exit); 6048 MODULE_LICENSE("GPL"); 6049 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 6050 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 6051 MODULE_ALIAS("md-raid5"); 6052 MODULE_ALIAS("md-raid4"); 6053 MODULE_ALIAS("md-level-5"); 6054 MODULE_ALIAS("md-level-4"); 6055 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 6056 MODULE_ALIAS("md-raid6"); 6057 MODULE_ALIAS("md-level-6"); 6058 6059 /* This used to be two separate modules, they were: */ 6060 MODULE_ALIAS("raid5"); 6061 MODULE_ALIAS("raid6"); 6062