1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->bm_write is the number of the last batch successfully written. 31 * conf->bm_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is bm_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/module.h> 47 #include <linux/slab.h> 48 #include <linux/highmem.h> 49 #include <linux/bitops.h> 50 #include <linux/kthread.h> 51 #include <asm/atomic.h> 52 #include "raid6.h" 53 54 #include <linux/raid/bitmap.h> 55 #include <linux/async_tx.h> 56 57 /* 58 * Stripe cache 59 */ 60 61 #define NR_STRIPES 256 62 #define STRIPE_SIZE PAGE_SIZE 63 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 64 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 65 #define IO_THRESHOLD 1 66 #define BYPASS_THRESHOLD 1 67 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 68 #define HASH_MASK (NR_HASH - 1) 69 70 #define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK])) 71 72 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 73 * order without overlap. There may be several bio's per stripe+device, and 74 * a bio could span several devices. 75 * When walking this list for a particular stripe+device, we must never proceed 76 * beyond a bio that extends past this device, as the next bio might no longer 77 * be valid. 78 * This macro is used to determine the 'next' bio in the list, given the sector 79 * of the current stripe+device 80 */ 81 #define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL) 82 /* 83 * The following can be used to debug the driver 84 */ 85 #define RAID5_PARANOIA 1 86 #if RAID5_PARANOIA && defined(CONFIG_SMP) 87 # define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock) 88 #else 89 # define CHECK_DEVLOCK() 90 #endif 91 92 #ifdef DEBUG 93 #define inline 94 #define __inline__ 95 #endif 96 97 #define printk_rl(args...) ((void) (printk_ratelimit() && printk(args))) 98 99 #if !RAID6_USE_EMPTY_ZERO_PAGE 100 /* In .bss so it's zeroed */ 101 const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256))); 102 #endif 103 104 static inline int raid6_next_disk(int disk, int raid_disks) 105 { 106 disk++; 107 return (disk < raid_disks) ? disk : 0; 108 } 109 110 static void return_io(struct bio *return_bi) 111 { 112 struct bio *bi = return_bi; 113 while (bi) { 114 115 return_bi = bi->bi_next; 116 bi->bi_next = NULL; 117 bi->bi_size = 0; 118 bi->bi_end_io(bi, 119 test_bit(BIO_UPTODATE, &bi->bi_flags) 120 ? 0 : -EIO); 121 bi = return_bi; 122 } 123 } 124 125 static void print_raid5_conf (raid5_conf_t *conf); 126 127 static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh) 128 { 129 if (atomic_dec_and_test(&sh->count)) { 130 BUG_ON(!list_empty(&sh->lru)); 131 BUG_ON(atomic_read(&conf->active_stripes)==0); 132 if (test_bit(STRIPE_HANDLE, &sh->state)) { 133 if (test_bit(STRIPE_DELAYED, &sh->state)) { 134 list_add_tail(&sh->lru, &conf->delayed_list); 135 blk_plug_device(conf->mddev->queue); 136 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 137 sh->bm_seq - conf->seq_write > 0) { 138 list_add_tail(&sh->lru, &conf->bitmap_list); 139 blk_plug_device(conf->mddev->queue); 140 } else { 141 clear_bit(STRIPE_BIT_DELAY, &sh->state); 142 list_add_tail(&sh->lru, &conf->handle_list); 143 } 144 md_wakeup_thread(conf->mddev->thread); 145 } else { 146 BUG_ON(sh->ops.pending); 147 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 148 atomic_dec(&conf->preread_active_stripes); 149 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) 150 md_wakeup_thread(conf->mddev->thread); 151 } 152 atomic_dec(&conf->active_stripes); 153 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 154 list_add_tail(&sh->lru, &conf->inactive_list); 155 wake_up(&conf->wait_for_stripe); 156 if (conf->retry_read_aligned) 157 md_wakeup_thread(conf->mddev->thread); 158 } 159 } 160 } 161 } 162 static void release_stripe(struct stripe_head *sh) 163 { 164 raid5_conf_t *conf = sh->raid_conf; 165 unsigned long flags; 166 167 spin_lock_irqsave(&conf->device_lock, flags); 168 __release_stripe(conf, sh); 169 spin_unlock_irqrestore(&conf->device_lock, flags); 170 } 171 172 static inline void remove_hash(struct stripe_head *sh) 173 { 174 pr_debug("remove_hash(), stripe %llu\n", 175 (unsigned long long)sh->sector); 176 177 hlist_del_init(&sh->hash); 178 } 179 180 static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh) 181 { 182 struct hlist_head *hp = stripe_hash(conf, sh->sector); 183 184 pr_debug("insert_hash(), stripe %llu\n", 185 (unsigned long long)sh->sector); 186 187 CHECK_DEVLOCK(); 188 hlist_add_head(&sh->hash, hp); 189 } 190 191 192 /* find an idle stripe, make sure it is unhashed, and return it. */ 193 static struct stripe_head *get_free_stripe(raid5_conf_t *conf) 194 { 195 struct stripe_head *sh = NULL; 196 struct list_head *first; 197 198 CHECK_DEVLOCK(); 199 if (list_empty(&conf->inactive_list)) 200 goto out; 201 first = conf->inactive_list.next; 202 sh = list_entry(first, struct stripe_head, lru); 203 list_del_init(first); 204 remove_hash(sh); 205 atomic_inc(&conf->active_stripes); 206 out: 207 return sh; 208 } 209 210 static void shrink_buffers(struct stripe_head *sh, int num) 211 { 212 struct page *p; 213 int i; 214 215 for (i=0; i<num ; i++) { 216 p = sh->dev[i].page; 217 if (!p) 218 continue; 219 sh->dev[i].page = NULL; 220 put_page(p); 221 } 222 } 223 224 static int grow_buffers(struct stripe_head *sh, int num) 225 { 226 int i; 227 228 for (i=0; i<num; i++) { 229 struct page *page; 230 231 if (!(page = alloc_page(GFP_KERNEL))) { 232 return 1; 233 } 234 sh->dev[i].page = page; 235 } 236 return 0; 237 } 238 239 static void raid5_build_block (struct stripe_head *sh, int i); 240 241 static void init_stripe(struct stripe_head *sh, sector_t sector, int pd_idx, int disks) 242 { 243 raid5_conf_t *conf = sh->raid_conf; 244 int i; 245 246 BUG_ON(atomic_read(&sh->count) != 0); 247 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 248 BUG_ON(sh->ops.pending || sh->ops.ack || sh->ops.complete); 249 250 CHECK_DEVLOCK(); 251 pr_debug("init_stripe called, stripe %llu\n", 252 (unsigned long long)sh->sector); 253 254 remove_hash(sh); 255 256 sh->sector = sector; 257 sh->pd_idx = pd_idx; 258 sh->state = 0; 259 260 sh->disks = disks; 261 262 for (i = sh->disks; i--; ) { 263 struct r5dev *dev = &sh->dev[i]; 264 265 if (dev->toread || dev->read || dev->towrite || dev->written || 266 test_bit(R5_LOCKED, &dev->flags)) { 267 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 268 (unsigned long long)sh->sector, i, dev->toread, 269 dev->read, dev->towrite, dev->written, 270 test_bit(R5_LOCKED, &dev->flags)); 271 BUG(); 272 } 273 dev->flags = 0; 274 raid5_build_block(sh, i); 275 } 276 insert_hash(conf, sh); 277 } 278 279 static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector, int disks) 280 { 281 struct stripe_head *sh; 282 struct hlist_node *hn; 283 284 CHECK_DEVLOCK(); 285 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 286 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash) 287 if (sh->sector == sector && sh->disks == disks) 288 return sh; 289 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 290 return NULL; 291 } 292 293 static void unplug_slaves(mddev_t *mddev); 294 static void raid5_unplug_device(struct request_queue *q); 295 296 static struct stripe_head *get_active_stripe(raid5_conf_t *conf, sector_t sector, int disks, 297 int pd_idx, int noblock) 298 { 299 struct stripe_head *sh; 300 301 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 302 303 spin_lock_irq(&conf->device_lock); 304 305 do { 306 wait_event_lock_irq(conf->wait_for_stripe, 307 conf->quiesce == 0, 308 conf->device_lock, /* nothing */); 309 sh = __find_stripe(conf, sector, disks); 310 if (!sh) { 311 if (!conf->inactive_blocked) 312 sh = get_free_stripe(conf); 313 if (noblock && sh == NULL) 314 break; 315 if (!sh) { 316 conf->inactive_blocked = 1; 317 wait_event_lock_irq(conf->wait_for_stripe, 318 !list_empty(&conf->inactive_list) && 319 (atomic_read(&conf->active_stripes) 320 < (conf->max_nr_stripes *3/4) 321 || !conf->inactive_blocked), 322 conf->device_lock, 323 raid5_unplug_device(conf->mddev->queue) 324 ); 325 conf->inactive_blocked = 0; 326 } else 327 init_stripe(sh, sector, pd_idx, disks); 328 } else { 329 if (atomic_read(&sh->count)) { 330 BUG_ON(!list_empty(&sh->lru)); 331 } else { 332 if (!test_bit(STRIPE_HANDLE, &sh->state)) 333 atomic_inc(&conf->active_stripes); 334 if (list_empty(&sh->lru) && 335 !test_bit(STRIPE_EXPANDING, &sh->state)) 336 BUG(); 337 list_del_init(&sh->lru); 338 } 339 } 340 } while (sh == NULL); 341 342 if (sh) 343 atomic_inc(&sh->count); 344 345 spin_unlock_irq(&conf->device_lock); 346 return sh; 347 } 348 349 /* test_and_ack_op() ensures that we only dequeue an operation once */ 350 #define test_and_ack_op(op, pend) \ 351 do { \ 352 if (test_bit(op, &sh->ops.pending) && \ 353 !test_bit(op, &sh->ops.complete)) { \ 354 if (test_and_set_bit(op, &sh->ops.ack)) \ 355 clear_bit(op, &pend); \ 356 else \ 357 ack++; \ 358 } else \ 359 clear_bit(op, &pend); \ 360 } while (0) 361 362 /* find new work to run, do not resubmit work that is already 363 * in flight 364 */ 365 static unsigned long get_stripe_work(struct stripe_head *sh) 366 { 367 unsigned long pending; 368 int ack = 0; 369 370 pending = sh->ops.pending; 371 372 test_and_ack_op(STRIPE_OP_BIOFILL, pending); 373 test_and_ack_op(STRIPE_OP_COMPUTE_BLK, pending); 374 test_and_ack_op(STRIPE_OP_PREXOR, pending); 375 test_and_ack_op(STRIPE_OP_BIODRAIN, pending); 376 test_and_ack_op(STRIPE_OP_POSTXOR, pending); 377 test_and_ack_op(STRIPE_OP_CHECK, pending); 378 if (test_and_clear_bit(STRIPE_OP_IO, &sh->ops.pending)) 379 ack++; 380 381 sh->ops.count -= ack; 382 if (unlikely(sh->ops.count < 0)) { 383 printk(KERN_ERR "pending: %#lx ops.pending: %#lx ops.ack: %#lx " 384 "ops.complete: %#lx\n", pending, sh->ops.pending, 385 sh->ops.ack, sh->ops.complete); 386 BUG(); 387 } 388 389 return pending; 390 } 391 392 static void 393 raid5_end_read_request(struct bio *bi, int error); 394 static void 395 raid5_end_write_request(struct bio *bi, int error); 396 397 static void ops_run_io(struct stripe_head *sh) 398 { 399 raid5_conf_t *conf = sh->raid_conf; 400 int i, disks = sh->disks; 401 402 might_sleep(); 403 404 set_bit(STRIPE_IO_STARTED, &sh->state); 405 for (i = disks; i--; ) { 406 int rw; 407 struct bio *bi; 408 mdk_rdev_t *rdev; 409 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) 410 rw = WRITE; 411 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 412 rw = READ; 413 else 414 continue; 415 416 bi = &sh->dev[i].req; 417 418 bi->bi_rw = rw; 419 if (rw == WRITE) 420 bi->bi_end_io = raid5_end_write_request; 421 else 422 bi->bi_end_io = raid5_end_read_request; 423 424 rcu_read_lock(); 425 rdev = rcu_dereference(conf->disks[i].rdev); 426 if (rdev && test_bit(Faulty, &rdev->flags)) 427 rdev = NULL; 428 if (rdev) 429 atomic_inc(&rdev->nr_pending); 430 rcu_read_unlock(); 431 432 if (rdev) { 433 if (test_bit(STRIPE_SYNCING, &sh->state) || 434 test_bit(STRIPE_EXPAND_SOURCE, &sh->state) || 435 test_bit(STRIPE_EXPAND_READY, &sh->state)) 436 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 437 438 bi->bi_bdev = rdev->bdev; 439 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 440 __func__, (unsigned long long)sh->sector, 441 bi->bi_rw, i); 442 atomic_inc(&sh->count); 443 bi->bi_sector = sh->sector + rdev->data_offset; 444 bi->bi_flags = 1 << BIO_UPTODATE; 445 bi->bi_vcnt = 1; 446 bi->bi_max_vecs = 1; 447 bi->bi_idx = 0; 448 bi->bi_io_vec = &sh->dev[i].vec; 449 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 450 bi->bi_io_vec[0].bv_offset = 0; 451 bi->bi_size = STRIPE_SIZE; 452 bi->bi_next = NULL; 453 if (rw == WRITE && 454 test_bit(R5_ReWrite, &sh->dev[i].flags)) 455 atomic_add(STRIPE_SECTORS, 456 &rdev->corrected_errors); 457 generic_make_request(bi); 458 } else { 459 if (rw == WRITE) 460 set_bit(STRIPE_DEGRADED, &sh->state); 461 pr_debug("skip op %ld on disc %d for sector %llu\n", 462 bi->bi_rw, i, (unsigned long long)sh->sector); 463 clear_bit(R5_LOCKED, &sh->dev[i].flags); 464 set_bit(STRIPE_HANDLE, &sh->state); 465 } 466 } 467 } 468 469 static struct dma_async_tx_descriptor * 470 async_copy_data(int frombio, struct bio *bio, struct page *page, 471 sector_t sector, struct dma_async_tx_descriptor *tx) 472 { 473 struct bio_vec *bvl; 474 struct page *bio_page; 475 int i; 476 int page_offset; 477 478 if (bio->bi_sector >= sector) 479 page_offset = (signed)(bio->bi_sector - sector) * 512; 480 else 481 page_offset = (signed)(sector - bio->bi_sector) * -512; 482 bio_for_each_segment(bvl, bio, i) { 483 int len = bio_iovec_idx(bio, i)->bv_len; 484 int clen; 485 int b_offset = 0; 486 487 if (page_offset < 0) { 488 b_offset = -page_offset; 489 page_offset += b_offset; 490 len -= b_offset; 491 } 492 493 if (len > 0 && page_offset + len > STRIPE_SIZE) 494 clen = STRIPE_SIZE - page_offset; 495 else 496 clen = len; 497 498 if (clen > 0) { 499 b_offset += bio_iovec_idx(bio, i)->bv_offset; 500 bio_page = bio_iovec_idx(bio, i)->bv_page; 501 if (frombio) 502 tx = async_memcpy(page, bio_page, page_offset, 503 b_offset, clen, 504 ASYNC_TX_DEP_ACK, 505 tx, NULL, NULL); 506 else 507 tx = async_memcpy(bio_page, page, b_offset, 508 page_offset, clen, 509 ASYNC_TX_DEP_ACK, 510 tx, NULL, NULL); 511 } 512 if (clen < len) /* hit end of page */ 513 break; 514 page_offset += len; 515 } 516 517 return tx; 518 } 519 520 static void ops_complete_biofill(void *stripe_head_ref) 521 { 522 struct stripe_head *sh = stripe_head_ref; 523 struct bio *return_bi = NULL; 524 raid5_conf_t *conf = sh->raid_conf; 525 int i; 526 527 pr_debug("%s: stripe %llu\n", __func__, 528 (unsigned long long)sh->sector); 529 530 /* clear completed biofills */ 531 for (i = sh->disks; i--; ) { 532 struct r5dev *dev = &sh->dev[i]; 533 534 /* acknowledge completion of a biofill operation */ 535 /* and check if we need to reply to a read request, 536 * new R5_Wantfill requests are held off until 537 * !test_bit(STRIPE_OP_BIOFILL, &sh->ops.pending) 538 */ 539 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 540 struct bio *rbi, *rbi2; 541 542 /* The access to dev->read is outside of the 543 * spin_lock_irq(&conf->device_lock), but is protected 544 * by the STRIPE_OP_BIOFILL pending bit 545 */ 546 BUG_ON(!dev->read); 547 rbi = dev->read; 548 dev->read = NULL; 549 while (rbi && rbi->bi_sector < 550 dev->sector + STRIPE_SECTORS) { 551 rbi2 = r5_next_bio(rbi, dev->sector); 552 spin_lock_irq(&conf->device_lock); 553 if (--rbi->bi_phys_segments == 0) { 554 rbi->bi_next = return_bi; 555 return_bi = rbi; 556 } 557 spin_unlock_irq(&conf->device_lock); 558 rbi = rbi2; 559 } 560 } 561 } 562 set_bit(STRIPE_OP_BIOFILL, &sh->ops.complete); 563 564 return_io(return_bi); 565 566 set_bit(STRIPE_HANDLE, &sh->state); 567 release_stripe(sh); 568 } 569 570 static void ops_run_biofill(struct stripe_head *sh) 571 { 572 struct dma_async_tx_descriptor *tx = NULL; 573 raid5_conf_t *conf = sh->raid_conf; 574 int i; 575 576 pr_debug("%s: stripe %llu\n", __func__, 577 (unsigned long long)sh->sector); 578 579 for (i = sh->disks; i--; ) { 580 struct r5dev *dev = &sh->dev[i]; 581 if (test_bit(R5_Wantfill, &dev->flags)) { 582 struct bio *rbi; 583 spin_lock_irq(&conf->device_lock); 584 dev->read = rbi = dev->toread; 585 dev->toread = NULL; 586 spin_unlock_irq(&conf->device_lock); 587 while (rbi && rbi->bi_sector < 588 dev->sector + STRIPE_SECTORS) { 589 tx = async_copy_data(0, rbi, dev->page, 590 dev->sector, tx); 591 rbi = r5_next_bio(rbi, dev->sector); 592 } 593 } 594 } 595 596 atomic_inc(&sh->count); 597 async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx, 598 ops_complete_biofill, sh); 599 } 600 601 static void ops_complete_compute5(void *stripe_head_ref) 602 { 603 struct stripe_head *sh = stripe_head_ref; 604 int target = sh->ops.target; 605 struct r5dev *tgt = &sh->dev[target]; 606 607 pr_debug("%s: stripe %llu\n", __func__, 608 (unsigned long long)sh->sector); 609 610 set_bit(R5_UPTODATE, &tgt->flags); 611 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 612 clear_bit(R5_Wantcompute, &tgt->flags); 613 set_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.complete); 614 set_bit(STRIPE_HANDLE, &sh->state); 615 release_stripe(sh); 616 } 617 618 static struct dma_async_tx_descriptor * 619 ops_run_compute5(struct stripe_head *sh, unsigned long pending) 620 { 621 /* kernel stack size limits the total number of disks */ 622 int disks = sh->disks; 623 struct page *xor_srcs[disks]; 624 int target = sh->ops.target; 625 struct r5dev *tgt = &sh->dev[target]; 626 struct page *xor_dest = tgt->page; 627 int count = 0; 628 struct dma_async_tx_descriptor *tx; 629 int i; 630 631 pr_debug("%s: stripe %llu block: %d\n", 632 __func__, (unsigned long long)sh->sector, target); 633 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 634 635 for (i = disks; i--; ) 636 if (i != target) 637 xor_srcs[count++] = sh->dev[i].page; 638 639 atomic_inc(&sh->count); 640 641 if (unlikely(count == 1)) 642 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, 643 0, NULL, ops_complete_compute5, sh); 644 else 645 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 646 ASYNC_TX_XOR_ZERO_DST, NULL, 647 ops_complete_compute5, sh); 648 649 /* ack now if postxor is not set to be run */ 650 if (tx && !test_bit(STRIPE_OP_POSTXOR, &pending)) 651 async_tx_ack(tx); 652 653 return tx; 654 } 655 656 static void ops_complete_prexor(void *stripe_head_ref) 657 { 658 struct stripe_head *sh = stripe_head_ref; 659 660 pr_debug("%s: stripe %llu\n", __func__, 661 (unsigned long long)sh->sector); 662 663 set_bit(STRIPE_OP_PREXOR, &sh->ops.complete); 664 } 665 666 static struct dma_async_tx_descriptor * 667 ops_run_prexor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 668 { 669 /* kernel stack size limits the total number of disks */ 670 int disks = sh->disks; 671 struct page *xor_srcs[disks]; 672 int count = 0, pd_idx = sh->pd_idx, i; 673 674 /* existing parity data subtracted */ 675 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 676 677 pr_debug("%s: stripe %llu\n", __func__, 678 (unsigned long long)sh->sector); 679 680 for (i = disks; i--; ) { 681 struct r5dev *dev = &sh->dev[i]; 682 /* Only process blocks that are known to be uptodate */ 683 if (dev->towrite && test_bit(R5_Wantprexor, &dev->flags)) 684 xor_srcs[count++] = dev->page; 685 } 686 687 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 688 ASYNC_TX_DEP_ACK | ASYNC_TX_XOR_DROP_DST, tx, 689 ops_complete_prexor, sh); 690 691 return tx; 692 } 693 694 static struct dma_async_tx_descriptor * 695 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx, 696 unsigned long pending) 697 { 698 int disks = sh->disks; 699 int pd_idx = sh->pd_idx, i; 700 701 /* check if prexor is active which means only process blocks 702 * that are part of a read-modify-write (Wantprexor) 703 */ 704 int prexor = test_bit(STRIPE_OP_PREXOR, &pending); 705 706 pr_debug("%s: stripe %llu\n", __func__, 707 (unsigned long long)sh->sector); 708 709 for (i = disks; i--; ) { 710 struct r5dev *dev = &sh->dev[i]; 711 struct bio *chosen; 712 int towrite; 713 714 towrite = 0; 715 if (prexor) { /* rmw */ 716 if (dev->towrite && 717 test_bit(R5_Wantprexor, &dev->flags)) 718 towrite = 1; 719 } else { /* rcw */ 720 if (i != pd_idx && dev->towrite && 721 test_bit(R5_LOCKED, &dev->flags)) 722 towrite = 1; 723 } 724 725 if (towrite) { 726 struct bio *wbi; 727 728 spin_lock(&sh->lock); 729 chosen = dev->towrite; 730 dev->towrite = NULL; 731 BUG_ON(dev->written); 732 wbi = dev->written = chosen; 733 spin_unlock(&sh->lock); 734 735 while (wbi && wbi->bi_sector < 736 dev->sector + STRIPE_SECTORS) { 737 tx = async_copy_data(1, wbi, dev->page, 738 dev->sector, tx); 739 wbi = r5_next_bio(wbi, dev->sector); 740 } 741 } 742 } 743 744 return tx; 745 } 746 747 static void ops_complete_postxor(void *stripe_head_ref) 748 { 749 struct stripe_head *sh = stripe_head_ref; 750 751 pr_debug("%s: stripe %llu\n", __func__, 752 (unsigned long long)sh->sector); 753 754 set_bit(STRIPE_OP_POSTXOR, &sh->ops.complete); 755 set_bit(STRIPE_HANDLE, &sh->state); 756 release_stripe(sh); 757 } 758 759 static void ops_complete_write(void *stripe_head_ref) 760 { 761 struct stripe_head *sh = stripe_head_ref; 762 int disks = sh->disks, i, pd_idx = sh->pd_idx; 763 764 pr_debug("%s: stripe %llu\n", __func__, 765 (unsigned long long)sh->sector); 766 767 for (i = disks; i--; ) { 768 struct r5dev *dev = &sh->dev[i]; 769 if (dev->written || i == pd_idx) 770 set_bit(R5_UPTODATE, &dev->flags); 771 } 772 773 set_bit(STRIPE_OP_BIODRAIN, &sh->ops.complete); 774 set_bit(STRIPE_OP_POSTXOR, &sh->ops.complete); 775 776 set_bit(STRIPE_HANDLE, &sh->state); 777 release_stripe(sh); 778 } 779 780 static void 781 ops_run_postxor(struct stripe_head *sh, struct dma_async_tx_descriptor *tx, 782 unsigned long pending) 783 { 784 /* kernel stack size limits the total number of disks */ 785 int disks = sh->disks; 786 struct page *xor_srcs[disks]; 787 788 int count = 0, pd_idx = sh->pd_idx, i; 789 struct page *xor_dest; 790 int prexor = test_bit(STRIPE_OP_PREXOR, &pending); 791 unsigned long flags; 792 dma_async_tx_callback callback; 793 794 pr_debug("%s: stripe %llu\n", __func__, 795 (unsigned long long)sh->sector); 796 797 /* check if prexor is active which means only process blocks 798 * that are part of a read-modify-write (written) 799 */ 800 if (prexor) { 801 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 802 for (i = disks; i--; ) { 803 struct r5dev *dev = &sh->dev[i]; 804 if (dev->written) 805 xor_srcs[count++] = dev->page; 806 } 807 } else { 808 xor_dest = sh->dev[pd_idx].page; 809 for (i = disks; i--; ) { 810 struct r5dev *dev = &sh->dev[i]; 811 if (i != pd_idx) 812 xor_srcs[count++] = dev->page; 813 } 814 } 815 816 /* check whether this postxor is part of a write */ 817 callback = test_bit(STRIPE_OP_BIODRAIN, &pending) ? 818 ops_complete_write : ops_complete_postxor; 819 820 /* 1/ if we prexor'd then the dest is reused as a source 821 * 2/ if we did not prexor then we are redoing the parity 822 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 823 * for the synchronous xor case 824 */ 825 flags = ASYNC_TX_DEP_ACK | ASYNC_TX_ACK | 826 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 827 828 atomic_inc(&sh->count); 829 830 if (unlikely(count == 1)) { 831 flags &= ~(ASYNC_TX_XOR_DROP_DST | ASYNC_TX_XOR_ZERO_DST); 832 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, 833 flags, tx, callback, sh); 834 } else 835 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 836 flags, tx, callback, sh); 837 } 838 839 static void ops_complete_check(void *stripe_head_ref) 840 { 841 struct stripe_head *sh = stripe_head_ref; 842 int pd_idx = sh->pd_idx; 843 844 pr_debug("%s: stripe %llu\n", __func__, 845 (unsigned long long)sh->sector); 846 847 if (test_and_clear_bit(STRIPE_OP_MOD_DMA_CHECK, &sh->ops.pending) && 848 sh->ops.zero_sum_result == 0) 849 set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 850 851 set_bit(STRIPE_OP_CHECK, &sh->ops.complete); 852 set_bit(STRIPE_HANDLE, &sh->state); 853 release_stripe(sh); 854 } 855 856 static void ops_run_check(struct stripe_head *sh) 857 { 858 /* kernel stack size limits the total number of disks */ 859 int disks = sh->disks; 860 struct page *xor_srcs[disks]; 861 struct dma_async_tx_descriptor *tx; 862 863 int count = 0, pd_idx = sh->pd_idx, i; 864 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 865 866 pr_debug("%s: stripe %llu\n", __func__, 867 (unsigned long long)sh->sector); 868 869 for (i = disks; i--; ) { 870 struct r5dev *dev = &sh->dev[i]; 871 if (i != pd_idx) 872 xor_srcs[count++] = dev->page; 873 } 874 875 tx = async_xor_zero_sum(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 876 &sh->ops.zero_sum_result, 0, NULL, NULL, NULL); 877 878 if (tx) 879 set_bit(STRIPE_OP_MOD_DMA_CHECK, &sh->ops.pending); 880 else 881 clear_bit(STRIPE_OP_MOD_DMA_CHECK, &sh->ops.pending); 882 883 atomic_inc(&sh->count); 884 tx = async_trigger_callback(ASYNC_TX_DEP_ACK | ASYNC_TX_ACK, tx, 885 ops_complete_check, sh); 886 } 887 888 static void raid5_run_ops(struct stripe_head *sh, unsigned long pending) 889 { 890 int overlap_clear = 0, i, disks = sh->disks; 891 struct dma_async_tx_descriptor *tx = NULL; 892 893 if (test_bit(STRIPE_OP_BIOFILL, &pending)) { 894 ops_run_biofill(sh); 895 overlap_clear++; 896 } 897 898 if (test_bit(STRIPE_OP_COMPUTE_BLK, &pending)) 899 tx = ops_run_compute5(sh, pending); 900 901 if (test_bit(STRIPE_OP_PREXOR, &pending)) 902 tx = ops_run_prexor(sh, tx); 903 904 if (test_bit(STRIPE_OP_BIODRAIN, &pending)) { 905 tx = ops_run_biodrain(sh, tx, pending); 906 overlap_clear++; 907 } 908 909 if (test_bit(STRIPE_OP_POSTXOR, &pending)) 910 ops_run_postxor(sh, tx, pending); 911 912 if (test_bit(STRIPE_OP_CHECK, &pending)) 913 ops_run_check(sh); 914 915 if (test_bit(STRIPE_OP_IO, &pending)) 916 ops_run_io(sh); 917 918 if (overlap_clear) 919 for (i = disks; i--; ) { 920 struct r5dev *dev = &sh->dev[i]; 921 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 922 wake_up(&sh->raid_conf->wait_for_overlap); 923 } 924 } 925 926 static int grow_one_stripe(raid5_conf_t *conf) 927 { 928 struct stripe_head *sh; 929 sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL); 930 if (!sh) 931 return 0; 932 memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev)); 933 sh->raid_conf = conf; 934 spin_lock_init(&sh->lock); 935 936 if (grow_buffers(sh, conf->raid_disks)) { 937 shrink_buffers(sh, conf->raid_disks); 938 kmem_cache_free(conf->slab_cache, sh); 939 return 0; 940 } 941 sh->disks = conf->raid_disks; 942 /* we just created an active stripe so... */ 943 atomic_set(&sh->count, 1); 944 atomic_inc(&conf->active_stripes); 945 INIT_LIST_HEAD(&sh->lru); 946 release_stripe(sh); 947 return 1; 948 } 949 950 static int grow_stripes(raid5_conf_t *conf, int num) 951 { 952 struct kmem_cache *sc; 953 int devs = conf->raid_disks; 954 955 sprintf(conf->cache_name[0], "raid5-%s", mdname(conf->mddev)); 956 sprintf(conf->cache_name[1], "raid5-%s-alt", mdname(conf->mddev)); 957 conf->active_name = 0; 958 sc = kmem_cache_create(conf->cache_name[conf->active_name], 959 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 960 0, 0, NULL); 961 if (!sc) 962 return 1; 963 conf->slab_cache = sc; 964 conf->pool_size = devs; 965 while (num--) 966 if (!grow_one_stripe(conf)) 967 return 1; 968 return 0; 969 } 970 971 #ifdef CONFIG_MD_RAID5_RESHAPE 972 static int resize_stripes(raid5_conf_t *conf, int newsize) 973 { 974 /* Make all the stripes able to hold 'newsize' devices. 975 * New slots in each stripe get 'page' set to a new page. 976 * 977 * This happens in stages: 978 * 1/ create a new kmem_cache and allocate the required number of 979 * stripe_heads. 980 * 2/ gather all the old stripe_heads and tranfer the pages across 981 * to the new stripe_heads. This will have the side effect of 982 * freezing the array as once all stripe_heads have been collected, 983 * no IO will be possible. Old stripe heads are freed once their 984 * pages have been transferred over, and the old kmem_cache is 985 * freed when all stripes are done. 986 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 987 * we simple return a failre status - no need to clean anything up. 988 * 4/ allocate new pages for the new slots in the new stripe_heads. 989 * If this fails, we don't bother trying the shrink the 990 * stripe_heads down again, we just leave them as they are. 991 * As each stripe_head is processed the new one is released into 992 * active service. 993 * 994 * Once step2 is started, we cannot afford to wait for a write, 995 * so we use GFP_NOIO allocations. 996 */ 997 struct stripe_head *osh, *nsh; 998 LIST_HEAD(newstripes); 999 struct disk_info *ndisks; 1000 int err = 0; 1001 struct kmem_cache *sc; 1002 int i; 1003 1004 if (newsize <= conf->pool_size) 1005 return 0; /* never bother to shrink */ 1006 1007 md_allow_write(conf->mddev); 1008 1009 /* Step 1 */ 1010 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1011 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1012 0, 0, NULL); 1013 if (!sc) 1014 return -ENOMEM; 1015 1016 for (i = conf->max_nr_stripes; i; i--) { 1017 nsh = kmem_cache_alloc(sc, GFP_KERNEL); 1018 if (!nsh) 1019 break; 1020 1021 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev)); 1022 1023 nsh->raid_conf = conf; 1024 spin_lock_init(&nsh->lock); 1025 1026 list_add(&nsh->lru, &newstripes); 1027 } 1028 if (i) { 1029 /* didn't get enough, give up */ 1030 while (!list_empty(&newstripes)) { 1031 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1032 list_del(&nsh->lru); 1033 kmem_cache_free(sc, nsh); 1034 } 1035 kmem_cache_destroy(sc); 1036 return -ENOMEM; 1037 } 1038 /* Step 2 - Must use GFP_NOIO now. 1039 * OK, we have enough stripes, start collecting inactive 1040 * stripes and copying them over 1041 */ 1042 list_for_each_entry(nsh, &newstripes, lru) { 1043 spin_lock_irq(&conf->device_lock); 1044 wait_event_lock_irq(conf->wait_for_stripe, 1045 !list_empty(&conf->inactive_list), 1046 conf->device_lock, 1047 unplug_slaves(conf->mddev) 1048 ); 1049 osh = get_free_stripe(conf); 1050 spin_unlock_irq(&conf->device_lock); 1051 atomic_set(&nsh->count, 1); 1052 for(i=0; i<conf->pool_size; i++) 1053 nsh->dev[i].page = osh->dev[i].page; 1054 for( ; i<newsize; i++) 1055 nsh->dev[i].page = NULL; 1056 kmem_cache_free(conf->slab_cache, osh); 1057 } 1058 kmem_cache_destroy(conf->slab_cache); 1059 1060 /* Step 3. 1061 * At this point, we are holding all the stripes so the array 1062 * is completely stalled, so now is a good time to resize 1063 * conf->disks. 1064 */ 1065 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1066 if (ndisks) { 1067 for (i=0; i<conf->raid_disks; i++) 1068 ndisks[i] = conf->disks[i]; 1069 kfree(conf->disks); 1070 conf->disks = ndisks; 1071 } else 1072 err = -ENOMEM; 1073 1074 /* Step 4, return new stripes to service */ 1075 while(!list_empty(&newstripes)) { 1076 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1077 list_del_init(&nsh->lru); 1078 for (i=conf->raid_disks; i < newsize; i++) 1079 if (nsh->dev[i].page == NULL) { 1080 struct page *p = alloc_page(GFP_NOIO); 1081 nsh->dev[i].page = p; 1082 if (!p) 1083 err = -ENOMEM; 1084 } 1085 release_stripe(nsh); 1086 } 1087 /* critical section pass, GFP_NOIO no longer needed */ 1088 1089 conf->slab_cache = sc; 1090 conf->active_name = 1-conf->active_name; 1091 conf->pool_size = newsize; 1092 return err; 1093 } 1094 #endif 1095 1096 static int drop_one_stripe(raid5_conf_t *conf) 1097 { 1098 struct stripe_head *sh; 1099 1100 spin_lock_irq(&conf->device_lock); 1101 sh = get_free_stripe(conf); 1102 spin_unlock_irq(&conf->device_lock); 1103 if (!sh) 1104 return 0; 1105 BUG_ON(atomic_read(&sh->count)); 1106 shrink_buffers(sh, conf->pool_size); 1107 kmem_cache_free(conf->slab_cache, sh); 1108 atomic_dec(&conf->active_stripes); 1109 return 1; 1110 } 1111 1112 static void shrink_stripes(raid5_conf_t *conf) 1113 { 1114 while (drop_one_stripe(conf)) 1115 ; 1116 1117 if (conf->slab_cache) 1118 kmem_cache_destroy(conf->slab_cache); 1119 conf->slab_cache = NULL; 1120 } 1121 1122 static void raid5_end_read_request(struct bio * bi, int error) 1123 { 1124 struct stripe_head *sh = bi->bi_private; 1125 raid5_conf_t *conf = sh->raid_conf; 1126 int disks = sh->disks, i; 1127 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1128 char b[BDEVNAME_SIZE]; 1129 mdk_rdev_t *rdev; 1130 1131 1132 for (i=0 ; i<disks; i++) 1133 if (bi == &sh->dev[i].req) 1134 break; 1135 1136 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1137 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1138 uptodate); 1139 if (i == disks) { 1140 BUG(); 1141 return; 1142 } 1143 1144 if (uptodate) { 1145 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1146 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1147 rdev = conf->disks[i].rdev; 1148 printk_rl(KERN_INFO "raid5:%s: read error corrected" 1149 " (%lu sectors at %llu on %s)\n", 1150 mdname(conf->mddev), STRIPE_SECTORS, 1151 (unsigned long long)(sh->sector 1152 + rdev->data_offset), 1153 bdevname(rdev->bdev, b)); 1154 clear_bit(R5_ReadError, &sh->dev[i].flags); 1155 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1156 } 1157 if (atomic_read(&conf->disks[i].rdev->read_errors)) 1158 atomic_set(&conf->disks[i].rdev->read_errors, 0); 1159 } else { 1160 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b); 1161 int retry = 0; 1162 rdev = conf->disks[i].rdev; 1163 1164 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1165 atomic_inc(&rdev->read_errors); 1166 if (conf->mddev->degraded) 1167 printk_rl(KERN_WARNING 1168 "raid5:%s: read error not correctable " 1169 "(sector %llu on %s).\n", 1170 mdname(conf->mddev), 1171 (unsigned long long)(sh->sector 1172 + rdev->data_offset), 1173 bdn); 1174 else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) 1175 /* Oh, no!!! */ 1176 printk_rl(KERN_WARNING 1177 "raid5:%s: read error NOT corrected!! " 1178 "(sector %llu on %s).\n", 1179 mdname(conf->mddev), 1180 (unsigned long long)(sh->sector 1181 + rdev->data_offset), 1182 bdn); 1183 else if (atomic_read(&rdev->read_errors) 1184 > conf->max_nr_stripes) 1185 printk(KERN_WARNING 1186 "raid5:%s: Too many read errors, failing device %s.\n", 1187 mdname(conf->mddev), bdn); 1188 else 1189 retry = 1; 1190 if (retry) 1191 set_bit(R5_ReadError, &sh->dev[i].flags); 1192 else { 1193 clear_bit(R5_ReadError, &sh->dev[i].flags); 1194 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1195 md_error(conf->mddev, rdev); 1196 } 1197 } 1198 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1199 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1200 set_bit(STRIPE_HANDLE, &sh->state); 1201 release_stripe(sh); 1202 } 1203 1204 static void raid5_end_write_request (struct bio *bi, int error) 1205 { 1206 struct stripe_head *sh = bi->bi_private; 1207 raid5_conf_t *conf = sh->raid_conf; 1208 int disks = sh->disks, i; 1209 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1210 1211 for (i=0 ; i<disks; i++) 1212 if (bi == &sh->dev[i].req) 1213 break; 1214 1215 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1216 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1217 uptodate); 1218 if (i == disks) { 1219 BUG(); 1220 return; 1221 } 1222 1223 if (!uptodate) 1224 md_error(conf->mddev, conf->disks[i].rdev); 1225 1226 rdev_dec_pending(conf->disks[i].rdev, conf->mddev); 1227 1228 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1229 set_bit(STRIPE_HANDLE, &sh->state); 1230 release_stripe(sh); 1231 } 1232 1233 1234 static sector_t compute_blocknr(struct stripe_head *sh, int i); 1235 1236 static void raid5_build_block (struct stripe_head *sh, int i) 1237 { 1238 struct r5dev *dev = &sh->dev[i]; 1239 1240 bio_init(&dev->req); 1241 dev->req.bi_io_vec = &dev->vec; 1242 dev->req.bi_vcnt++; 1243 dev->req.bi_max_vecs++; 1244 dev->vec.bv_page = dev->page; 1245 dev->vec.bv_len = STRIPE_SIZE; 1246 dev->vec.bv_offset = 0; 1247 1248 dev->req.bi_sector = sh->sector; 1249 dev->req.bi_private = sh; 1250 1251 dev->flags = 0; 1252 dev->sector = compute_blocknr(sh, i); 1253 } 1254 1255 static void error(mddev_t *mddev, mdk_rdev_t *rdev) 1256 { 1257 char b[BDEVNAME_SIZE]; 1258 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 1259 pr_debug("raid5: error called\n"); 1260 1261 if (!test_bit(Faulty, &rdev->flags)) { 1262 set_bit(MD_CHANGE_DEVS, &mddev->flags); 1263 if (test_and_clear_bit(In_sync, &rdev->flags)) { 1264 unsigned long flags; 1265 spin_lock_irqsave(&conf->device_lock, flags); 1266 mddev->degraded++; 1267 spin_unlock_irqrestore(&conf->device_lock, flags); 1268 /* 1269 * if recovery was running, make sure it aborts. 1270 */ 1271 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 1272 } 1273 set_bit(Faulty, &rdev->flags); 1274 printk (KERN_ALERT 1275 "raid5: Disk failure on %s, disabling device.\n" 1276 "raid5: Operation continuing on %d devices.\n", 1277 bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded); 1278 } 1279 } 1280 1281 /* 1282 * Input: a 'big' sector number, 1283 * Output: index of the data and parity disk, and the sector # in them. 1284 */ 1285 static sector_t raid5_compute_sector(sector_t r_sector, unsigned int raid_disks, 1286 unsigned int data_disks, unsigned int * dd_idx, 1287 unsigned int * pd_idx, raid5_conf_t *conf) 1288 { 1289 long stripe; 1290 unsigned long chunk_number; 1291 unsigned int chunk_offset; 1292 sector_t new_sector; 1293 int sectors_per_chunk = conf->chunk_size >> 9; 1294 1295 /* First compute the information on this sector */ 1296 1297 /* 1298 * Compute the chunk number and the sector offset inside the chunk 1299 */ 1300 chunk_offset = sector_div(r_sector, sectors_per_chunk); 1301 chunk_number = r_sector; 1302 BUG_ON(r_sector != chunk_number); 1303 1304 /* 1305 * Compute the stripe number 1306 */ 1307 stripe = chunk_number / data_disks; 1308 1309 /* 1310 * Compute the data disk and parity disk indexes inside the stripe 1311 */ 1312 *dd_idx = chunk_number % data_disks; 1313 1314 /* 1315 * Select the parity disk based on the user selected algorithm. 1316 */ 1317 switch(conf->level) { 1318 case 4: 1319 *pd_idx = data_disks; 1320 break; 1321 case 5: 1322 switch (conf->algorithm) { 1323 case ALGORITHM_LEFT_ASYMMETRIC: 1324 *pd_idx = data_disks - stripe % raid_disks; 1325 if (*dd_idx >= *pd_idx) 1326 (*dd_idx)++; 1327 break; 1328 case ALGORITHM_RIGHT_ASYMMETRIC: 1329 *pd_idx = stripe % raid_disks; 1330 if (*dd_idx >= *pd_idx) 1331 (*dd_idx)++; 1332 break; 1333 case ALGORITHM_LEFT_SYMMETRIC: 1334 *pd_idx = data_disks - stripe % raid_disks; 1335 *dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks; 1336 break; 1337 case ALGORITHM_RIGHT_SYMMETRIC: 1338 *pd_idx = stripe % raid_disks; 1339 *dd_idx = (*pd_idx + 1 + *dd_idx) % raid_disks; 1340 break; 1341 default: 1342 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1343 conf->algorithm); 1344 } 1345 break; 1346 case 6: 1347 1348 /**** FIX THIS ****/ 1349 switch (conf->algorithm) { 1350 case ALGORITHM_LEFT_ASYMMETRIC: 1351 *pd_idx = raid_disks - 1 - (stripe % raid_disks); 1352 if (*pd_idx == raid_disks-1) 1353 (*dd_idx)++; /* Q D D D P */ 1354 else if (*dd_idx >= *pd_idx) 1355 (*dd_idx) += 2; /* D D P Q D */ 1356 break; 1357 case ALGORITHM_RIGHT_ASYMMETRIC: 1358 *pd_idx = stripe % raid_disks; 1359 if (*pd_idx == raid_disks-1) 1360 (*dd_idx)++; /* Q D D D P */ 1361 else if (*dd_idx >= *pd_idx) 1362 (*dd_idx) += 2; /* D D P Q D */ 1363 break; 1364 case ALGORITHM_LEFT_SYMMETRIC: 1365 *pd_idx = raid_disks - 1 - (stripe % raid_disks); 1366 *dd_idx = (*pd_idx + 2 + *dd_idx) % raid_disks; 1367 break; 1368 case ALGORITHM_RIGHT_SYMMETRIC: 1369 *pd_idx = stripe % raid_disks; 1370 *dd_idx = (*pd_idx + 2 + *dd_idx) % raid_disks; 1371 break; 1372 default: 1373 printk (KERN_CRIT "raid6: unsupported algorithm %d\n", 1374 conf->algorithm); 1375 } 1376 break; 1377 } 1378 1379 /* 1380 * Finally, compute the new sector number 1381 */ 1382 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 1383 return new_sector; 1384 } 1385 1386 1387 static sector_t compute_blocknr(struct stripe_head *sh, int i) 1388 { 1389 raid5_conf_t *conf = sh->raid_conf; 1390 int raid_disks = sh->disks; 1391 int data_disks = raid_disks - conf->max_degraded; 1392 sector_t new_sector = sh->sector, check; 1393 int sectors_per_chunk = conf->chunk_size >> 9; 1394 sector_t stripe; 1395 int chunk_offset; 1396 int chunk_number, dummy1, dummy2, dd_idx = i; 1397 sector_t r_sector; 1398 1399 1400 chunk_offset = sector_div(new_sector, sectors_per_chunk); 1401 stripe = new_sector; 1402 BUG_ON(new_sector != stripe); 1403 1404 if (i == sh->pd_idx) 1405 return 0; 1406 switch(conf->level) { 1407 case 4: break; 1408 case 5: 1409 switch (conf->algorithm) { 1410 case ALGORITHM_LEFT_ASYMMETRIC: 1411 case ALGORITHM_RIGHT_ASYMMETRIC: 1412 if (i > sh->pd_idx) 1413 i--; 1414 break; 1415 case ALGORITHM_LEFT_SYMMETRIC: 1416 case ALGORITHM_RIGHT_SYMMETRIC: 1417 if (i < sh->pd_idx) 1418 i += raid_disks; 1419 i -= (sh->pd_idx + 1); 1420 break; 1421 default: 1422 printk(KERN_ERR "raid5: unsupported algorithm %d\n", 1423 conf->algorithm); 1424 } 1425 break; 1426 case 6: 1427 if (i == raid6_next_disk(sh->pd_idx, raid_disks)) 1428 return 0; /* It is the Q disk */ 1429 switch (conf->algorithm) { 1430 case ALGORITHM_LEFT_ASYMMETRIC: 1431 case ALGORITHM_RIGHT_ASYMMETRIC: 1432 if (sh->pd_idx == raid_disks-1) 1433 i--; /* Q D D D P */ 1434 else if (i > sh->pd_idx) 1435 i -= 2; /* D D P Q D */ 1436 break; 1437 case ALGORITHM_LEFT_SYMMETRIC: 1438 case ALGORITHM_RIGHT_SYMMETRIC: 1439 if (sh->pd_idx == raid_disks-1) 1440 i--; /* Q D D D P */ 1441 else { 1442 /* D D P Q D */ 1443 if (i < sh->pd_idx) 1444 i += raid_disks; 1445 i -= (sh->pd_idx + 2); 1446 } 1447 break; 1448 default: 1449 printk (KERN_CRIT "raid6: unsupported algorithm %d\n", 1450 conf->algorithm); 1451 } 1452 break; 1453 } 1454 1455 chunk_number = stripe * data_disks + i; 1456 r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset; 1457 1458 check = raid5_compute_sector (r_sector, raid_disks, data_disks, &dummy1, &dummy2, conf); 1459 if (check != sh->sector || dummy1 != dd_idx || dummy2 != sh->pd_idx) { 1460 printk(KERN_ERR "compute_blocknr: map not correct\n"); 1461 return 0; 1462 } 1463 return r_sector; 1464 } 1465 1466 1467 1468 /* 1469 * Copy data between a page in the stripe cache, and one or more bion 1470 * The page could align with the middle of the bio, or there could be 1471 * several bion, each with several bio_vecs, which cover part of the page 1472 * Multiple bion are linked together on bi_next. There may be extras 1473 * at the end of this list. We ignore them. 1474 */ 1475 static void copy_data(int frombio, struct bio *bio, 1476 struct page *page, 1477 sector_t sector) 1478 { 1479 char *pa = page_address(page); 1480 struct bio_vec *bvl; 1481 int i; 1482 int page_offset; 1483 1484 if (bio->bi_sector >= sector) 1485 page_offset = (signed)(bio->bi_sector - sector) * 512; 1486 else 1487 page_offset = (signed)(sector - bio->bi_sector) * -512; 1488 bio_for_each_segment(bvl, bio, i) { 1489 int len = bio_iovec_idx(bio,i)->bv_len; 1490 int clen; 1491 int b_offset = 0; 1492 1493 if (page_offset < 0) { 1494 b_offset = -page_offset; 1495 page_offset += b_offset; 1496 len -= b_offset; 1497 } 1498 1499 if (len > 0 && page_offset + len > STRIPE_SIZE) 1500 clen = STRIPE_SIZE - page_offset; 1501 else clen = len; 1502 1503 if (clen > 0) { 1504 char *ba = __bio_kmap_atomic(bio, i, KM_USER0); 1505 if (frombio) 1506 memcpy(pa+page_offset, ba+b_offset, clen); 1507 else 1508 memcpy(ba+b_offset, pa+page_offset, clen); 1509 __bio_kunmap_atomic(ba, KM_USER0); 1510 } 1511 if (clen < len) /* hit end of page */ 1512 break; 1513 page_offset += len; 1514 } 1515 } 1516 1517 #define check_xor() do { \ 1518 if (count == MAX_XOR_BLOCKS) { \ 1519 xor_blocks(count, STRIPE_SIZE, dest, ptr);\ 1520 count = 0; \ 1521 } \ 1522 } while(0) 1523 1524 static void compute_parity6(struct stripe_head *sh, int method) 1525 { 1526 raid6_conf_t *conf = sh->raid_conf; 1527 int i, pd_idx = sh->pd_idx, qd_idx, d0_idx, disks = sh->disks, count; 1528 struct bio *chosen; 1529 /**** FIX THIS: This could be very bad if disks is close to 256 ****/ 1530 void *ptrs[disks]; 1531 1532 qd_idx = raid6_next_disk(pd_idx, disks); 1533 d0_idx = raid6_next_disk(qd_idx, disks); 1534 1535 pr_debug("compute_parity, stripe %llu, method %d\n", 1536 (unsigned long long)sh->sector, method); 1537 1538 switch(method) { 1539 case READ_MODIFY_WRITE: 1540 BUG(); /* READ_MODIFY_WRITE N/A for RAID-6 */ 1541 case RECONSTRUCT_WRITE: 1542 for (i= disks; i-- ;) 1543 if ( i != pd_idx && i != qd_idx && sh->dev[i].towrite ) { 1544 chosen = sh->dev[i].towrite; 1545 sh->dev[i].towrite = NULL; 1546 1547 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 1548 wake_up(&conf->wait_for_overlap); 1549 1550 BUG_ON(sh->dev[i].written); 1551 sh->dev[i].written = chosen; 1552 } 1553 break; 1554 case CHECK_PARITY: 1555 BUG(); /* Not implemented yet */ 1556 } 1557 1558 for (i = disks; i--;) 1559 if (sh->dev[i].written) { 1560 sector_t sector = sh->dev[i].sector; 1561 struct bio *wbi = sh->dev[i].written; 1562 while (wbi && wbi->bi_sector < sector + STRIPE_SECTORS) { 1563 copy_data(1, wbi, sh->dev[i].page, sector); 1564 wbi = r5_next_bio(wbi, sector); 1565 } 1566 1567 set_bit(R5_LOCKED, &sh->dev[i].flags); 1568 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1569 } 1570 1571 // switch(method) { 1572 // case RECONSTRUCT_WRITE: 1573 // case CHECK_PARITY: 1574 // case UPDATE_PARITY: 1575 /* Note that unlike RAID-5, the ordering of the disks matters greatly. */ 1576 /* FIX: Is this ordering of drives even remotely optimal? */ 1577 count = 0; 1578 i = d0_idx; 1579 do { 1580 ptrs[count++] = page_address(sh->dev[i].page); 1581 if (count <= disks-2 && !test_bit(R5_UPTODATE, &sh->dev[i].flags)) 1582 printk("block %d/%d not uptodate on parity calc\n", i,count); 1583 i = raid6_next_disk(i, disks); 1584 } while ( i != d0_idx ); 1585 // break; 1586 // } 1587 1588 raid6_call.gen_syndrome(disks, STRIPE_SIZE, ptrs); 1589 1590 switch(method) { 1591 case RECONSTRUCT_WRITE: 1592 set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 1593 set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags); 1594 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 1595 set_bit(R5_LOCKED, &sh->dev[qd_idx].flags); 1596 break; 1597 case UPDATE_PARITY: 1598 set_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 1599 set_bit(R5_UPTODATE, &sh->dev[qd_idx].flags); 1600 break; 1601 } 1602 } 1603 1604 1605 /* Compute one missing block */ 1606 static void compute_block_1(struct stripe_head *sh, int dd_idx, int nozero) 1607 { 1608 int i, count, disks = sh->disks; 1609 void *ptr[MAX_XOR_BLOCKS], *dest, *p; 1610 int pd_idx = sh->pd_idx; 1611 int qd_idx = raid6_next_disk(pd_idx, disks); 1612 1613 pr_debug("compute_block_1, stripe %llu, idx %d\n", 1614 (unsigned long long)sh->sector, dd_idx); 1615 1616 if ( dd_idx == qd_idx ) { 1617 /* We're actually computing the Q drive */ 1618 compute_parity6(sh, UPDATE_PARITY); 1619 } else { 1620 dest = page_address(sh->dev[dd_idx].page); 1621 if (!nozero) memset(dest, 0, STRIPE_SIZE); 1622 count = 0; 1623 for (i = disks ; i--; ) { 1624 if (i == dd_idx || i == qd_idx) 1625 continue; 1626 p = page_address(sh->dev[i].page); 1627 if (test_bit(R5_UPTODATE, &sh->dev[i].flags)) 1628 ptr[count++] = p; 1629 else 1630 printk("compute_block() %d, stripe %llu, %d" 1631 " not present\n", dd_idx, 1632 (unsigned long long)sh->sector, i); 1633 1634 check_xor(); 1635 } 1636 if (count) 1637 xor_blocks(count, STRIPE_SIZE, dest, ptr); 1638 if (!nozero) set_bit(R5_UPTODATE, &sh->dev[dd_idx].flags); 1639 else clear_bit(R5_UPTODATE, &sh->dev[dd_idx].flags); 1640 } 1641 } 1642 1643 /* Compute two missing blocks */ 1644 static void compute_block_2(struct stripe_head *sh, int dd_idx1, int dd_idx2) 1645 { 1646 int i, count, disks = sh->disks; 1647 int pd_idx = sh->pd_idx; 1648 int qd_idx = raid6_next_disk(pd_idx, disks); 1649 int d0_idx = raid6_next_disk(qd_idx, disks); 1650 int faila, failb; 1651 1652 /* faila and failb are disk numbers relative to d0_idx */ 1653 /* pd_idx become disks-2 and qd_idx become disks-1 */ 1654 faila = (dd_idx1 < d0_idx) ? dd_idx1+(disks-d0_idx) : dd_idx1-d0_idx; 1655 failb = (dd_idx2 < d0_idx) ? dd_idx2+(disks-d0_idx) : dd_idx2-d0_idx; 1656 1657 BUG_ON(faila == failb); 1658 if ( failb < faila ) { int tmp = faila; faila = failb; failb = tmp; } 1659 1660 pr_debug("compute_block_2, stripe %llu, idx %d,%d (%d,%d)\n", 1661 (unsigned long long)sh->sector, dd_idx1, dd_idx2, faila, failb); 1662 1663 if ( failb == disks-1 ) { 1664 /* Q disk is one of the missing disks */ 1665 if ( faila == disks-2 ) { 1666 /* Missing P+Q, just recompute */ 1667 compute_parity6(sh, UPDATE_PARITY); 1668 return; 1669 } else { 1670 /* We're missing D+Q; recompute D from P */ 1671 compute_block_1(sh, (dd_idx1 == qd_idx) ? dd_idx2 : dd_idx1, 0); 1672 compute_parity6(sh, UPDATE_PARITY); /* Is this necessary? */ 1673 return; 1674 } 1675 } 1676 1677 /* We're missing D+P or D+D; build pointer table */ 1678 { 1679 /**** FIX THIS: This could be very bad if disks is close to 256 ****/ 1680 void *ptrs[disks]; 1681 1682 count = 0; 1683 i = d0_idx; 1684 do { 1685 ptrs[count++] = page_address(sh->dev[i].page); 1686 i = raid6_next_disk(i, disks); 1687 if (i != dd_idx1 && i != dd_idx2 && 1688 !test_bit(R5_UPTODATE, &sh->dev[i].flags)) 1689 printk("compute_2 with missing block %d/%d\n", count, i); 1690 } while ( i != d0_idx ); 1691 1692 if ( failb == disks-2 ) { 1693 /* We're missing D+P. */ 1694 raid6_datap_recov(disks, STRIPE_SIZE, faila, ptrs); 1695 } else { 1696 /* We're missing D+D. */ 1697 raid6_2data_recov(disks, STRIPE_SIZE, faila, failb, ptrs); 1698 } 1699 1700 /* Both the above update both missing blocks */ 1701 set_bit(R5_UPTODATE, &sh->dev[dd_idx1].flags); 1702 set_bit(R5_UPTODATE, &sh->dev[dd_idx2].flags); 1703 } 1704 } 1705 1706 static int 1707 handle_write_operations5(struct stripe_head *sh, int rcw, int expand) 1708 { 1709 int i, pd_idx = sh->pd_idx, disks = sh->disks; 1710 int locked = 0; 1711 1712 if (rcw) { 1713 /* if we are not expanding this is a proper write request, and 1714 * there will be bios with new data to be drained into the 1715 * stripe cache 1716 */ 1717 if (!expand) { 1718 set_bit(STRIPE_OP_BIODRAIN, &sh->ops.pending); 1719 sh->ops.count++; 1720 } 1721 1722 set_bit(STRIPE_OP_POSTXOR, &sh->ops.pending); 1723 sh->ops.count++; 1724 1725 for (i = disks; i--; ) { 1726 struct r5dev *dev = &sh->dev[i]; 1727 1728 if (dev->towrite) { 1729 set_bit(R5_LOCKED, &dev->flags); 1730 if (!expand) 1731 clear_bit(R5_UPTODATE, &dev->flags); 1732 locked++; 1733 } 1734 } 1735 if (locked + 1 == disks) 1736 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 1737 atomic_inc(&sh->raid_conf->pending_full_writes); 1738 } else { 1739 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 1740 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 1741 1742 set_bit(STRIPE_OP_PREXOR, &sh->ops.pending); 1743 set_bit(STRIPE_OP_BIODRAIN, &sh->ops.pending); 1744 set_bit(STRIPE_OP_POSTXOR, &sh->ops.pending); 1745 1746 sh->ops.count += 3; 1747 1748 for (i = disks; i--; ) { 1749 struct r5dev *dev = &sh->dev[i]; 1750 if (i == pd_idx) 1751 continue; 1752 1753 /* For a read-modify write there may be blocks that are 1754 * locked for reading while others are ready to be 1755 * written so we distinguish these blocks by the 1756 * R5_Wantprexor bit 1757 */ 1758 if (dev->towrite && 1759 (test_bit(R5_UPTODATE, &dev->flags) || 1760 test_bit(R5_Wantcompute, &dev->flags))) { 1761 set_bit(R5_Wantprexor, &dev->flags); 1762 set_bit(R5_LOCKED, &dev->flags); 1763 clear_bit(R5_UPTODATE, &dev->flags); 1764 locked++; 1765 } 1766 } 1767 } 1768 1769 /* keep the parity disk locked while asynchronous operations 1770 * are in flight 1771 */ 1772 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 1773 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 1774 locked++; 1775 1776 pr_debug("%s: stripe %llu locked: %d pending: %lx\n", 1777 __func__, (unsigned long long)sh->sector, 1778 locked, sh->ops.pending); 1779 1780 return locked; 1781 } 1782 1783 /* 1784 * Each stripe/dev can have one or more bion attached. 1785 * toread/towrite point to the first in a chain. 1786 * The bi_next chain must be in order. 1787 */ 1788 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 1789 { 1790 struct bio **bip; 1791 raid5_conf_t *conf = sh->raid_conf; 1792 int firstwrite=0; 1793 1794 pr_debug("adding bh b#%llu to stripe s#%llu\n", 1795 (unsigned long long)bi->bi_sector, 1796 (unsigned long long)sh->sector); 1797 1798 1799 spin_lock(&sh->lock); 1800 spin_lock_irq(&conf->device_lock); 1801 if (forwrite) { 1802 bip = &sh->dev[dd_idx].towrite; 1803 if (*bip == NULL && sh->dev[dd_idx].written == NULL) 1804 firstwrite = 1; 1805 } else 1806 bip = &sh->dev[dd_idx].toread; 1807 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 1808 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector) 1809 goto overlap; 1810 bip = & (*bip)->bi_next; 1811 } 1812 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9)) 1813 goto overlap; 1814 1815 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 1816 if (*bip) 1817 bi->bi_next = *bip; 1818 *bip = bi; 1819 bi->bi_phys_segments ++; 1820 spin_unlock_irq(&conf->device_lock); 1821 spin_unlock(&sh->lock); 1822 1823 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 1824 (unsigned long long)bi->bi_sector, 1825 (unsigned long long)sh->sector, dd_idx); 1826 1827 if (conf->mddev->bitmap && firstwrite) { 1828 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 1829 STRIPE_SECTORS, 0); 1830 sh->bm_seq = conf->seq_flush+1; 1831 set_bit(STRIPE_BIT_DELAY, &sh->state); 1832 } 1833 1834 if (forwrite) { 1835 /* check if page is covered */ 1836 sector_t sector = sh->dev[dd_idx].sector; 1837 for (bi=sh->dev[dd_idx].towrite; 1838 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 1839 bi && bi->bi_sector <= sector; 1840 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 1841 if (bi->bi_sector + (bi->bi_size>>9) >= sector) 1842 sector = bi->bi_sector + (bi->bi_size>>9); 1843 } 1844 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 1845 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 1846 } 1847 return 1; 1848 1849 overlap: 1850 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 1851 spin_unlock_irq(&conf->device_lock); 1852 spin_unlock(&sh->lock); 1853 return 0; 1854 } 1855 1856 static void end_reshape(raid5_conf_t *conf); 1857 1858 static int page_is_zero(struct page *p) 1859 { 1860 char *a = page_address(p); 1861 return ((*(u32*)a) == 0 && 1862 memcmp(a, a+4, STRIPE_SIZE-4)==0); 1863 } 1864 1865 static int stripe_to_pdidx(sector_t stripe, raid5_conf_t *conf, int disks) 1866 { 1867 int sectors_per_chunk = conf->chunk_size >> 9; 1868 int pd_idx, dd_idx; 1869 int chunk_offset = sector_div(stripe, sectors_per_chunk); 1870 1871 raid5_compute_sector(stripe * (disks - conf->max_degraded) 1872 *sectors_per_chunk + chunk_offset, 1873 disks, disks - conf->max_degraded, 1874 &dd_idx, &pd_idx, conf); 1875 return pd_idx; 1876 } 1877 1878 static void 1879 handle_requests_to_failed_array(raid5_conf_t *conf, struct stripe_head *sh, 1880 struct stripe_head_state *s, int disks, 1881 struct bio **return_bi) 1882 { 1883 int i; 1884 for (i = disks; i--; ) { 1885 struct bio *bi; 1886 int bitmap_end = 0; 1887 1888 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1889 mdk_rdev_t *rdev; 1890 rcu_read_lock(); 1891 rdev = rcu_dereference(conf->disks[i].rdev); 1892 if (rdev && test_bit(In_sync, &rdev->flags)) 1893 /* multiple read failures in one stripe */ 1894 md_error(conf->mddev, rdev); 1895 rcu_read_unlock(); 1896 } 1897 spin_lock_irq(&conf->device_lock); 1898 /* fail all writes first */ 1899 bi = sh->dev[i].towrite; 1900 sh->dev[i].towrite = NULL; 1901 if (bi) { 1902 s->to_write--; 1903 bitmap_end = 1; 1904 } 1905 1906 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 1907 wake_up(&conf->wait_for_overlap); 1908 1909 while (bi && bi->bi_sector < 1910 sh->dev[i].sector + STRIPE_SECTORS) { 1911 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 1912 clear_bit(BIO_UPTODATE, &bi->bi_flags); 1913 if (--bi->bi_phys_segments == 0) { 1914 md_write_end(conf->mddev); 1915 bi->bi_next = *return_bi; 1916 *return_bi = bi; 1917 } 1918 bi = nextbi; 1919 } 1920 /* and fail all 'written' */ 1921 bi = sh->dev[i].written; 1922 sh->dev[i].written = NULL; 1923 if (bi) bitmap_end = 1; 1924 while (bi && bi->bi_sector < 1925 sh->dev[i].sector + STRIPE_SECTORS) { 1926 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 1927 clear_bit(BIO_UPTODATE, &bi->bi_flags); 1928 if (--bi->bi_phys_segments == 0) { 1929 md_write_end(conf->mddev); 1930 bi->bi_next = *return_bi; 1931 *return_bi = bi; 1932 } 1933 bi = bi2; 1934 } 1935 1936 /* fail any reads if this device is non-operational and 1937 * the data has not reached the cache yet. 1938 */ 1939 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 1940 (!test_bit(R5_Insync, &sh->dev[i].flags) || 1941 test_bit(R5_ReadError, &sh->dev[i].flags))) { 1942 bi = sh->dev[i].toread; 1943 sh->dev[i].toread = NULL; 1944 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 1945 wake_up(&conf->wait_for_overlap); 1946 if (bi) s->to_read--; 1947 while (bi && bi->bi_sector < 1948 sh->dev[i].sector + STRIPE_SECTORS) { 1949 struct bio *nextbi = 1950 r5_next_bio(bi, sh->dev[i].sector); 1951 clear_bit(BIO_UPTODATE, &bi->bi_flags); 1952 if (--bi->bi_phys_segments == 0) { 1953 bi->bi_next = *return_bi; 1954 *return_bi = bi; 1955 } 1956 bi = nextbi; 1957 } 1958 } 1959 spin_unlock_irq(&conf->device_lock); 1960 if (bitmap_end) 1961 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 1962 STRIPE_SECTORS, 0, 0); 1963 } 1964 1965 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 1966 if (atomic_dec_and_test(&conf->pending_full_writes)) 1967 md_wakeup_thread(conf->mddev->thread); 1968 } 1969 1970 /* __handle_issuing_new_read_requests5 - returns 0 if there are no more disks 1971 * to process 1972 */ 1973 static int __handle_issuing_new_read_requests5(struct stripe_head *sh, 1974 struct stripe_head_state *s, int disk_idx, int disks) 1975 { 1976 struct r5dev *dev = &sh->dev[disk_idx]; 1977 struct r5dev *failed_dev = &sh->dev[s->failed_num]; 1978 1979 /* don't schedule compute operations or reads on the parity block while 1980 * a check is in flight 1981 */ 1982 if ((disk_idx == sh->pd_idx) && 1983 test_bit(STRIPE_OP_CHECK, &sh->ops.pending)) 1984 return ~0; 1985 1986 /* is the data in this block needed, and can we get it? */ 1987 if (!test_bit(R5_LOCKED, &dev->flags) && 1988 !test_bit(R5_UPTODATE, &dev->flags) && (dev->toread || 1989 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 1990 s->syncing || s->expanding || (s->failed && 1991 (failed_dev->toread || (failed_dev->towrite && 1992 !test_bit(R5_OVERWRITE, &failed_dev->flags) 1993 ))))) { 1994 /* 1/ We would like to get this block, possibly by computing it, 1995 * but we might not be able to. 1996 * 1997 * 2/ Since parity check operations potentially make the parity 1998 * block !uptodate it will need to be refreshed before any 1999 * compute operations on data disks are scheduled. 2000 * 2001 * 3/ We hold off parity block re-reads until check operations 2002 * have quiesced. 2003 */ 2004 if ((s->uptodate == disks - 1) && 2005 (s->failed && disk_idx == s->failed_num) && 2006 !test_bit(STRIPE_OP_CHECK, &sh->ops.pending)) { 2007 set_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending); 2008 set_bit(R5_Wantcompute, &dev->flags); 2009 sh->ops.target = disk_idx; 2010 s->req_compute = 1; 2011 sh->ops.count++; 2012 /* Careful: from this point on 'uptodate' is in the eye 2013 * of raid5_run_ops which services 'compute' operations 2014 * before writes. R5_Wantcompute flags a block that will 2015 * be R5_UPTODATE by the time it is needed for a 2016 * subsequent operation. 2017 */ 2018 s->uptodate++; 2019 return 0; /* uptodate + compute == disks */ 2020 } else if ((s->uptodate < disks - 1) && 2021 test_bit(R5_Insync, &dev->flags)) { 2022 /* Note: we hold off compute operations while checks are 2023 * in flight, but we still prefer 'compute' over 'read' 2024 * hence we only read if (uptodate < * disks-1) 2025 */ 2026 set_bit(R5_LOCKED, &dev->flags); 2027 set_bit(R5_Wantread, &dev->flags); 2028 if (!test_and_set_bit(STRIPE_OP_IO, &sh->ops.pending)) 2029 sh->ops.count++; 2030 s->locked++; 2031 pr_debug("Reading block %d (sync=%d)\n", disk_idx, 2032 s->syncing); 2033 } 2034 } 2035 2036 return ~0; 2037 } 2038 2039 static void handle_issuing_new_read_requests5(struct stripe_head *sh, 2040 struct stripe_head_state *s, int disks) 2041 { 2042 int i; 2043 2044 /* Clear completed compute operations. Parity recovery 2045 * (STRIPE_OP_MOD_REPAIR_PD) implies a write-back which is handled 2046 * later on in this routine 2047 */ 2048 if (test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.complete) && 2049 !test_bit(STRIPE_OP_MOD_REPAIR_PD, &sh->ops.pending)) { 2050 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.complete); 2051 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.ack); 2052 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending); 2053 } 2054 2055 /* look for blocks to read/compute, skip this if a compute 2056 * is already in flight, or if the stripe contents are in the 2057 * midst of changing due to a write 2058 */ 2059 if (!test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending) && 2060 !test_bit(STRIPE_OP_PREXOR, &sh->ops.pending) && 2061 !test_bit(STRIPE_OP_POSTXOR, &sh->ops.pending)) { 2062 for (i = disks; i--; ) 2063 if (__handle_issuing_new_read_requests5( 2064 sh, s, i, disks) == 0) 2065 break; 2066 } 2067 set_bit(STRIPE_HANDLE, &sh->state); 2068 } 2069 2070 static void handle_issuing_new_read_requests6(struct stripe_head *sh, 2071 struct stripe_head_state *s, struct r6_state *r6s, 2072 int disks) 2073 { 2074 int i; 2075 for (i = disks; i--; ) { 2076 struct r5dev *dev = &sh->dev[i]; 2077 if (!test_bit(R5_LOCKED, &dev->flags) && 2078 !test_bit(R5_UPTODATE, &dev->flags) && 2079 (dev->toread || (dev->towrite && 2080 !test_bit(R5_OVERWRITE, &dev->flags)) || 2081 s->syncing || s->expanding || 2082 (s->failed >= 1 && 2083 (sh->dev[r6s->failed_num[0]].toread || 2084 s->to_write)) || 2085 (s->failed >= 2 && 2086 (sh->dev[r6s->failed_num[1]].toread || 2087 s->to_write)))) { 2088 /* we would like to get this block, possibly 2089 * by computing it, but we might not be able to 2090 */ 2091 if ((s->uptodate == disks - 1) && 2092 (s->failed && (i == r6s->failed_num[0] || 2093 i == r6s->failed_num[1]))) { 2094 pr_debug("Computing stripe %llu block %d\n", 2095 (unsigned long long)sh->sector, i); 2096 compute_block_1(sh, i, 0); 2097 s->uptodate++; 2098 } else if ( s->uptodate == disks-2 && s->failed >= 2 ) { 2099 /* Computing 2-failure is *very* expensive; only 2100 * do it if failed >= 2 2101 */ 2102 int other; 2103 for (other = disks; other--; ) { 2104 if (other == i) 2105 continue; 2106 if (!test_bit(R5_UPTODATE, 2107 &sh->dev[other].flags)) 2108 break; 2109 } 2110 BUG_ON(other < 0); 2111 pr_debug("Computing stripe %llu blocks %d,%d\n", 2112 (unsigned long long)sh->sector, 2113 i, other); 2114 compute_block_2(sh, i, other); 2115 s->uptodate += 2; 2116 } else if (test_bit(R5_Insync, &dev->flags)) { 2117 set_bit(R5_LOCKED, &dev->flags); 2118 set_bit(R5_Wantread, &dev->flags); 2119 s->locked++; 2120 pr_debug("Reading block %d (sync=%d)\n", 2121 i, s->syncing); 2122 } 2123 } 2124 } 2125 set_bit(STRIPE_HANDLE, &sh->state); 2126 } 2127 2128 2129 /* handle_completed_write_requests 2130 * any written block on an uptodate or failed drive can be returned. 2131 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2132 * never LOCKED, so we don't need to test 'failed' directly. 2133 */ 2134 static void handle_completed_write_requests(raid5_conf_t *conf, 2135 struct stripe_head *sh, int disks, struct bio **return_bi) 2136 { 2137 int i; 2138 struct r5dev *dev; 2139 2140 for (i = disks; i--; ) 2141 if (sh->dev[i].written) { 2142 dev = &sh->dev[i]; 2143 if (!test_bit(R5_LOCKED, &dev->flags) && 2144 test_bit(R5_UPTODATE, &dev->flags)) { 2145 /* We can return any write requests */ 2146 struct bio *wbi, *wbi2; 2147 int bitmap_end = 0; 2148 pr_debug("Return write for disc %d\n", i); 2149 spin_lock_irq(&conf->device_lock); 2150 wbi = dev->written; 2151 dev->written = NULL; 2152 while (wbi && wbi->bi_sector < 2153 dev->sector + STRIPE_SECTORS) { 2154 wbi2 = r5_next_bio(wbi, dev->sector); 2155 if (--wbi->bi_phys_segments == 0) { 2156 md_write_end(conf->mddev); 2157 wbi->bi_next = *return_bi; 2158 *return_bi = wbi; 2159 } 2160 wbi = wbi2; 2161 } 2162 if (dev->towrite == NULL) 2163 bitmap_end = 1; 2164 spin_unlock_irq(&conf->device_lock); 2165 if (bitmap_end) 2166 bitmap_endwrite(conf->mddev->bitmap, 2167 sh->sector, 2168 STRIPE_SECTORS, 2169 !test_bit(STRIPE_DEGRADED, &sh->state), 2170 0); 2171 } 2172 } 2173 2174 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2175 if (atomic_dec_and_test(&conf->pending_full_writes)) 2176 md_wakeup_thread(conf->mddev->thread); 2177 } 2178 2179 static void handle_issuing_new_write_requests5(raid5_conf_t *conf, 2180 struct stripe_head *sh, struct stripe_head_state *s, int disks) 2181 { 2182 int rmw = 0, rcw = 0, i; 2183 for (i = disks; i--; ) { 2184 /* would I have to read this buffer for read_modify_write */ 2185 struct r5dev *dev = &sh->dev[i]; 2186 if ((dev->towrite || i == sh->pd_idx) && 2187 !test_bit(R5_LOCKED, &dev->flags) && 2188 !(test_bit(R5_UPTODATE, &dev->flags) || 2189 test_bit(R5_Wantcompute, &dev->flags))) { 2190 if (test_bit(R5_Insync, &dev->flags)) 2191 rmw++; 2192 else 2193 rmw += 2*disks; /* cannot read it */ 2194 } 2195 /* Would I have to read this buffer for reconstruct_write */ 2196 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2197 !test_bit(R5_LOCKED, &dev->flags) && 2198 !(test_bit(R5_UPTODATE, &dev->flags) || 2199 test_bit(R5_Wantcompute, &dev->flags))) { 2200 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2201 else 2202 rcw += 2*disks; 2203 } 2204 } 2205 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2206 (unsigned long long)sh->sector, rmw, rcw); 2207 set_bit(STRIPE_HANDLE, &sh->state); 2208 if (rmw < rcw && rmw > 0) 2209 /* prefer read-modify-write, but need to get some data */ 2210 for (i = disks; i--; ) { 2211 struct r5dev *dev = &sh->dev[i]; 2212 if ((dev->towrite || i == sh->pd_idx) && 2213 !test_bit(R5_LOCKED, &dev->flags) && 2214 !(test_bit(R5_UPTODATE, &dev->flags) || 2215 test_bit(R5_Wantcompute, &dev->flags)) && 2216 test_bit(R5_Insync, &dev->flags)) { 2217 if ( 2218 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2219 pr_debug("Read_old block " 2220 "%d for r-m-w\n", i); 2221 set_bit(R5_LOCKED, &dev->flags); 2222 set_bit(R5_Wantread, &dev->flags); 2223 if (!test_and_set_bit( 2224 STRIPE_OP_IO, &sh->ops.pending)) 2225 sh->ops.count++; 2226 s->locked++; 2227 } else { 2228 set_bit(STRIPE_DELAYED, &sh->state); 2229 set_bit(STRIPE_HANDLE, &sh->state); 2230 } 2231 } 2232 } 2233 if (rcw <= rmw && rcw > 0) 2234 /* want reconstruct write, but need to get some data */ 2235 for (i = disks; i--; ) { 2236 struct r5dev *dev = &sh->dev[i]; 2237 if (!test_bit(R5_OVERWRITE, &dev->flags) && 2238 i != sh->pd_idx && 2239 !test_bit(R5_LOCKED, &dev->flags) && 2240 !(test_bit(R5_UPTODATE, &dev->flags) || 2241 test_bit(R5_Wantcompute, &dev->flags)) && 2242 test_bit(R5_Insync, &dev->flags)) { 2243 if ( 2244 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2245 pr_debug("Read_old block " 2246 "%d for Reconstruct\n", i); 2247 set_bit(R5_LOCKED, &dev->flags); 2248 set_bit(R5_Wantread, &dev->flags); 2249 if (!test_and_set_bit( 2250 STRIPE_OP_IO, &sh->ops.pending)) 2251 sh->ops.count++; 2252 s->locked++; 2253 } else { 2254 set_bit(STRIPE_DELAYED, &sh->state); 2255 set_bit(STRIPE_HANDLE, &sh->state); 2256 } 2257 } 2258 } 2259 /* now if nothing is locked, and if we have enough data, 2260 * we can start a write request 2261 */ 2262 /* since handle_stripe can be called at any time we need to handle the 2263 * case where a compute block operation has been submitted and then a 2264 * subsequent call wants to start a write request. raid5_run_ops only 2265 * handles the case where compute block and postxor are requested 2266 * simultaneously. If this is not the case then new writes need to be 2267 * held off until the compute completes. 2268 */ 2269 if ((s->req_compute || 2270 !test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending)) && 2271 (s->locked == 0 && (rcw == 0 || rmw == 0) && 2272 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 2273 s->locked += handle_write_operations5(sh, rcw == 0, 0); 2274 } 2275 2276 static void handle_issuing_new_write_requests6(raid5_conf_t *conf, 2277 struct stripe_head *sh, struct stripe_head_state *s, 2278 struct r6_state *r6s, int disks) 2279 { 2280 int rcw = 0, must_compute = 0, pd_idx = sh->pd_idx, i; 2281 int qd_idx = r6s->qd_idx; 2282 for (i = disks; i--; ) { 2283 struct r5dev *dev = &sh->dev[i]; 2284 /* Would I have to read this buffer for reconstruct_write */ 2285 if (!test_bit(R5_OVERWRITE, &dev->flags) 2286 && i != pd_idx && i != qd_idx 2287 && (!test_bit(R5_LOCKED, &dev->flags) 2288 ) && 2289 !test_bit(R5_UPTODATE, &dev->flags)) { 2290 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2291 else { 2292 pr_debug("raid6: must_compute: " 2293 "disk %d flags=%#lx\n", i, dev->flags); 2294 must_compute++; 2295 } 2296 } 2297 } 2298 pr_debug("for sector %llu, rcw=%d, must_compute=%d\n", 2299 (unsigned long long)sh->sector, rcw, must_compute); 2300 set_bit(STRIPE_HANDLE, &sh->state); 2301 2302 if (rcw > 0) 2303 /* want reconstruct write, but need to get some data */ 2304 for (i = disks; i--; ) { 2305 struct r5dev *dev = &sh->dev[i]; 2306 if (!test_bit(R5_OVERWRITE, &dev->flags) 2307 && !(s->failed == 0 && (i == pd_idx || i == qd_idx)) 2308 && !test_bit(R5_LOCKED, &dev->flags) && 2309 !test_bit(R5_UPTODATE, &dev->flags) && 2310 test_bit(R5_Insync, &dev->flags)) { 2311 if ( 2312 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2313 pr_debug("Read_old stripe %llu " 2314 "block %d for Reconstruct\n", 2315 (unsigned long long)sh->sector, i); 2316 set_bit(R5_LOCKED, &dev->flags); 2317 set_bit(R5_Wantread, &dev->flags); 2318 s->locked++; 2319 } else { 2320 pr_debug("Request delayed stripe %llu " 2321 "block %d for Reconstruct\n", 2322 (unsigned long long)sh->sector, i); 2323 set_bit(STRIPE_DELAYED, &sh->state); 2324 set_bit(STRIPE_HANDLE, &sh->state); 2325 } 2326 } 2327 } 2328 /* now if nothing is locked, and if we have enough data, we can start a 2329 * write request 2330 */ 2331 if (s->locked == 0 && rcw == 0 && 2332 !test_bit(STRIPE_BIT_DELAY, &sh->state)) { 2333 if (must_compute > 0) { 2334 /* We have failed blocks and need to compute them */ 2335 switch (s->failed) { 2336 case 0: 2337 BUG(); 2338 case 1: 2339 compute_block_1(sh, r6s->failed_num[0], 0); 2340 break; 2341 case 2: 2342 compute_block_2(sh, r6s->failed_num[0], 2343 r6s->failed_num[1]); 2344 break; 2345 default: /* This request should have been failed? */ 2346 BUG(); 2347 } 2348 } 2349 2350 pr_debug("Computing parity for stripe %llu\n", 2351 (unsigned long long)sh->sector); 2352 compute_parity6(sh, RECONSTRUCT_WRITE); 2353 /* now every locked buffer is ready to be written */ 2354 for (i = disks; i--; ) 2355 if (test_bit(R5_LOCKED, &sh->dev[i].flags)) { 2356 pr_debug("Writing stripe %llu block %d\n", 2357 (unsigned long long)sh->sector, i); 2358 s->locked++; 2359 set_bit(R5_Wantwrite, &sh->dev[i].flags); 2360 } 2361 if (s->locked == disks) 2362 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2363 atomic_inc(&conf->pending_full_writes); 2364 /* after a RECONSTRUCT_WRITE, the stripe MUST be in-sync */ 2365 set_bit(STRIPE_INSYNC, &sh->state); 2366 2367 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2368 atomic_dec(&conf->preread_active_stripes); 2369 if (atomic_read(&conf->preread_active_stripes) < 2370 IO_THRESHOLD) 2371 md_wakeup_thread(conf->mddev->thread); 2372 } 2373 } 2374 } 2375 2376 static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh, 2377 struct stripe_head_state *s, int disks) 2378 { 2379 int canceled_check = 0; 2380 2381 set_bit(STRIPE_HANDLE, &sh->state); 2382 2383 /* complete a check operation */ 2384 if (test_and_clear_bit(STRIPE_OP_CHECK, &sh->ops.complete)) { 2385 clear_bit(STRIPE_OP_CHECK, &sh->ops.ack); 2386 clear_bit(STRIPE_OP_CHECK, &sh->ops.pending); 2387 if (s->failed == 0) { 2388 if (sh->ops.zero_sum_result == 0) 2389 /* parity is correct (on disc, 2390 * not in buffer any more) 2391 */ 2392 set_bit(STRIPE_INSYNC, &sh->state); 2393 else { 2394 conf->mddev->resync_mismatches += 2395 STRIPE_SECTORS; 2396 if (test_bit( 2397 MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2398 /* don't try to repair!! */ 2399 set_bit(STRIPE_INSYNC, &sh->state); 2400 else { 2401 set_bit(STRIPE_OP_COMPUTE_BLK, 2402 &sh->ops.pending); 2403 set_bit(STRIPE_OP_MOD_REPAIR_PD, 2404 &sh->ops.pending); 2405 set_bit(R5_Wantcompute, 2406 &sh->dev[sh->pd_idx].flags); 2407 sh->ops.target = sh->pd_idx; 2408 sh->ops.count++; 2409 s->uptodate++; 2410 } 2411 } 2412 } else 2413 canceled_check = 1; /* STRIPE_INSYNC is not set */ 2414 } 2415 2416 /* start a new check operation if there are no failures, the stripe is 2417 * not insync, and a repair is not in flight 2418 */ 2419 if (s->failed == 0 && 2420 !test_bit(STRIPE_INSYNC, &sh->state) && 2421 !test_bit(STRIPE_OP_MOD_REPAIR_PD, &sh->ops.pending)) { 2422 if (!test_and_set_bit(STRIPE_OP_CHECK, &sh->ops.pending)) { 2423 BUG_ON(s->uptodate != disks); 2424 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2425 sh->ops.count++; 2426 s->uptodate--; 2427 } 2428 } 2429 2430 /* check if we can clear a parity disk reconstruct */ 2431 if (test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.complete) && 2432 test_bit(STRIPE_OP_MOD_REPAIR_PD, &sh->ops.pending)) { 2433 2434 clear_bit(STRIPE_OP_MOD_REPAIR_PD, &sh->ops.pending); 2435 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.complete); 2436 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.ack); 2437 clear_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending); 2438 } 2439 2440 2441 /* Wait for check parity and compute block operations to complete 2442 * before write-back. If a failure occurred while the check operation 2443 * was in flight we need to cycle this stripe through handle_stripe 2444 * since the parity block may not be uptodate 2445 */ 2446 if (!canceled_check && !test_bit(STRIPE_INSYNC, &sh->state) && 2447 !test_bit(STRIPE_OP_CHECK, &sh->ops.pending) && 2448 !test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending)) { 2449 struct r5dev *dev; 2450 /* either failed parity check, or recovery is happening */ 2451 if (s->failed == 0) 2452 s->failed_num = sh->pd_idx; 2453 dev = &sh->dev[s->failed_num]; 2454 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 2455 BUG_ON(s->uptodate != disks); 2456 2457 set_bit(R5_LOCKED, &dev->flags); 2458 set_bit(R5_Wantwrite, &dev->flags); 2459 if (!test_and_set_bit(STRIPE_OP_IO, &sh->ops.pending)) 2460 sh->ops.count++; 2461 2462 clear_bit(STRIPE_DEGRADED, &sh->state); 2463 s->locked++; 2464 set_bit(STRIPE_INSYNC, &sh->state); 2465 } 2466 } 2467 2468 2469 static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh, 2470 struct stripe_head_state *s, 2471 struct r6_state *r6s, struct page *tmp_page, 2472 int disks) 2473 { 2474 int update_p = 0, update_q = 0; 2475 struct r5dev *dev; 2476 int pd_idx = sh->pd_idx; 2477 int qd_idx = r6s->qd_idx; 2478 2479 set_bit(STRIPE_HANDLE, &sh->state); 2480 2481 BUG_ON(s->failed > 2); 2482 BUG_ON(s->uptodate < disks); 2483 /* Want to check and possibly repair P and Q. 2484 * However there could be one 'failed' device, in which 2485 * case we can only check one of them, possibly using the 2486 * other to generate missing data 2487 */ 2488 2489 /* If !tmp_page, we cannot do the calculations, 2490 * but as we have set STRIPE_HANDLE, we will soon be called 2491 * by stripe_handle with a tmp_page - just wait until then. 2492 */ 2493 if (tmp_page) { 2494 if (s->failed == r6s->q_failed) { 2495 /* The only possible failed device holds 'Q', so it 2496 * makes sense to check P (If anything else were failed, 2497 * we would have used P to recreate it). 2498 */ 2499 compute_block_1(sh, pd_idx, 1); 2500 if (!page_is_zero(sh->dev[pd_idx].page)) { 2501 compute_block_1(sh, pd_idx, 0); 2502 update_p = 1; 2503 } 2504 } 2505 if (!r6s->q_failed && s->failed < 2) { 2506 /* q is not failed, and we didn't use it to generate 2507 * anything, so it makes sense to check it 2508 */ 2509 memcpy(page_address(tmp_page), 2510 page_address(sh->dev[qd_idx].page), 2511 STRIPE_SIZE); 2512 compute_parity6(sh, UPDATE_PARITY); 2513 if (memcmp(page_address(tmp_page), 2514 page_address(sh->dev[qd_idx].page), 2515 STRIPE_SIZE) != 0) { 2516 clear_bit(STRIPE_INSYNC, &sh->state); 2517 update_q = 1; 2518 } 2519 } 2520 if (update_p || update_q) { 2521 conf->mddev->resync_mismatches += STRIPE_SECTORS; 2522 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 2523 /* don't try to repair!! */ 2524 update_p = update_q = 0; 2525 } 2526 2527 /* now write out any block on a failed drive, 2528 * or P or Q if they need it 2529 */ 2530 2531 if (s->failed == 2) { 2532 dev = &sh->dev[r6s->failed_num[1]]; 2533 s->locked++; 2534 set_bit(R5_LOCKED, &dev->flags); 2535 set_bit(R5_Wantwrite, &dev->flags); 2536 } 2537 if (s->failed >= 1) { 2538 dev = &sh->dev[r6s->failed_num[0]]; 2539 s->locked++; 2540 set_bit(R5_LOCKED, &dev->flags); 2541 set_bit(R5_Wantwrite, &dev->flags); 2542 } 2543 2544 if (update_p) { 2545 dev = &sh->dev[pd_idx]; 2546 s->locked++; 2547 set_bit(R5_LOCKED, &dev->flags); 2548 set_bit(R5_Wantwrite, &dev->flags); 2549 } 2550 if (update_q) { 2551 dev = &sh->dev[qd_idx]; 2552 s->locked++; 2553 set_bit(R5_LOCKED, &dev->flags); 2554 set_bit(R5_Wantwrite, &dev->flags); 2555 } 2556 clear_bit(STRIPE_DEGRADED, &sh->state); 2557 2558 set_bit(STRIPE_INSYNC, &sh->state); 2559 } 2560 } 2561 2562 static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh, 2563 struct r6_state *r6s) 2564 { 2565 int i; 2566 2567 /* We have read all the blocks in this stripe and now we need to 2568 * copy some of them into a target stripe for expand. 2569 */ 2570 struct dma_async_tx_descriptor *tx = NULL; 2571 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2572 for (i = 0; i < sh->disks; i++) 2573 if (i != sh->pd_idx && (!r6s || i != r6s->qd_idx)) { 2574 int dd_idx, pd_idx, j; 2575 struct stripe_head *sh2; 2576 2577 sector_t bn = compute_blocknr(sh, i); 2578 sector_t s = raid5_compute_sector(bn, conf->raid_disks, 2579 conf->raid_disks - 2580 conf->max_degraded, &dd_idx, 2581 &pd_idx, conf); 2582 sh2 = get_active_stripe(conf, s, conf->raid_disks, 2583 pd_idx, 1); 2584 if (sh2 == NULL) 2585 /* so far only the early blocks of this stripe 2586 * have been requested. When later blocks 2587 * get requested, we will try again 2588 */ 2589 continue; 2590 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 2591 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 2592 /* must have already done this block */ 2593 release_stripe(sh2); 2594 continue; 2595 } 2596 2597 /* place all the copies on one channel */ 2598 tx = async_memcpy(sh2->dev[dd_idx].page, 2599 sh->dev[i].page, 0, 0, STRIPE_SIZE, 2600 ASYNC_TX_DEP_ACK, tx, NULL, NULL); 2601 2602 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 2603 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 2604 for (j = 0; j < conf->raid_disks; j++) 2605 if (j != sh2->pd_idx && 2606 (!r6s || j != raid6_next_disk(sh2->pd_idx, 2607 sh2->disks)) && 2608 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 2609 break; 2610 if (j == conf->raid_disks) { 2611 set_bit(STRIPE_EXPAND_READY, &sh2->state); 2612 set_bit(STRIPE_HANDLE, &sh2->state); 2613 } 2614 release_stripe(sh2); 2615 2616 } 2617 /* done submitting copies, wait for them to complete */ 2618 if (tx) { 2619 async_tx_ack(tx); 2620 dma_wait_for_async_tx(tx); 2621 } 2622 } 2623 2624 2625 /* 2626 * handle_stripe - do things to a stripe. 2627 * 2628 * We lock the stripe and then examine the state of various bits 2629 * to see what needs to be done. 2630 * Possible results: 2631 * return some read request which now have data 2632 * return some write requests which are safely on disc 2633 * schedule a read on some buffers 2634 * schedule a write of some buffers 2635 * return confirmation of parity correctness 2636 * 2637 * buffers are taken off read_list or write_list, and bh_cache buffers 2638 * get BH_Lock set before the stripe lock is released. 2639 * 2640 */ 2641 2642 static void handle_stripe5(struct stripe_head *sh) 2643 { 2644 raid5_conf_t *conf = sh->raid_conf; 2645 int disks = sh->disks, i; 2646 struct bio *return_bi = NULL; 2647 struct stripe_head_state s; 2648 struct r5dev *dev; 2649 unsigned long pending = 0; 2650 mdk_rdev_t *blocked_rdev = NULL; 2651 int prexor; 2652 2653 memset(&s, 0, sizeof(s)); 2654 pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d " 2655 "ops=%lx:%lx:%lx\n", (unsigned long long)sh->sector, sh->state, 2656 atomic_read(&sh->count), sh->pd_idx, 2657 sh->ops.pending, sh->ops.ack, sh->ops.complete); 2658 2659 spin_lock(&sh->lock); 2660 clear_bit(STRIPE_HANDLE, &sh->state); 2661 clear_bit(STRIPE_DELAYED, &sh->state); 2662 2663 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 2664 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2665 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 2666 /* Now to look around and see what can be done */ 2667 2668 /* clean-up completed biofill operations */ 2669 if (test_bit(STRIPE_OP_BIOFILL, &sh->ops.complete)) { 2670 clear_bit(STRIPE_OP_BIOFILL, &sh->ops.pending); 2671 clear_bit(STRIPE_OP_BIOFILL, &sh->ops.ack); 2672 clear_bit(STRIPE_OP_BIOFILL, &sh->ops.complete); 2673 } 2674 2675 rcu_read_lock(); 2676 for (i=disks; i--; ) { 2677 mdk_rdev_t *rdev; 2678 struct r5dev *dev = &sh->dev[i]; 2679 clear_bit(R5_Insync, &dev->flags); 2680 2681 pr_debug("check %d: state 0x%lx toread %p read %p write %p " 2682 "written %p\n", i, dev->flags, dev->toread, dev->read, 2683 dev->towrite, dev->written); 2684 2685 /* maybe we can request a biofill operation 2686 * 2687 * new wantfill requests are only permitted while 2688 * STRIPE_OP_BIOFILL is clear 2689 */ 2690 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 2691 !test_bit(STRIPE_OP_BIOFILL, &sh->ops.pending)) 2692 set_bit(R5_Wantfill, &dev->flags); 2693 2694 /* now count some things */ 2695 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 2696 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 2697 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++; 2698 2699 if (test_bit(R5_Wantfill, &dev->flags)) 2700 s.to_fill++; 2701 else if (dev->toread) 2702 s.to_read++; 2703 if (dev->towrite) { 2704 s.to_write++; 2705 if (!test_bit(R5_OVERWRITE, &dev->flags)) 2706 s.non_overwrite++; 2707 } 2708 if (dev->written) 2709 s.written++; 2710 rdev = rcu_dereference(conf->disks[i].rdev); 2711 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 2712 blocked_rdev = rdev; 2713 atomic_inc(&rdev->nr_pending); 2714 break; 2715 } 2716 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 2717 /* The ReadError flag will just be confusing now */ 2718 clear_bit(R5_ReadError, &dev->flags); 2719 clear_bit(R5_ReWrite, &dev->flags); 2720 } 2721 if (!rdev || !test_bit(In_sync, &rdev->flags) 2722 || test_bit(R5_ReadError, &dev->flags)) { 2723 s.failed++; 2724 s.failed_num = i; 2725 } else 2726 set_bit(R5_Insync, &dev->flags); 2727 } 2728 rcu_read_unlock(); 2729 2730 if (unlikely(blocked_rdev)) { 2731 set_bit(STRIPE_HANDLE, &sh->state); 2732 goto unlock; 2733 } 2734 2735 if (s.to_fill && !test_and_set_bit(STRIPE_OP_BIOFILL, &sh->ops.pending)) 2736 sh->ops.count++; 2737 2738 pr_debug("locked=%d uptodate=%d to_read=%d" 2739 " to_write=%d failed=%d failed_num=%d\n", 2740 s.locked, s.uptodate, s.to_read, s.to_write, 2741 s.failed, s.failed_num); 2742 /* check if the array has lost two devices and, if so, some requests might 2743 * need to be failed 2744 */ 2745 if (s.failed > 1 && s.to_read+s.to_write+s.written) 2746 handle_requests_to_failed_array(conf, sh, &s, disks, 2747 &return_bi); 2748 if (s.failed > 1 && s.syncing) { 2749 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 2750 clear_bit(STRIPE_SYNCING, &sh->state); 2751 s.syncing = 0; 2752 } 2753 2754 /* might be able to return some write requests if the parity block 2755 * is safe, or on a failed drive 2756 */ 2757 dev = &sh->dev[sh->pd_idx]; 2758 if ( s.written && 2759 ((test_bit(R5_Insync, &dev->flags) && 2760 !test_bit(R5_LOCKED, &dev->flags) && 2761 test_bit(R5_UPTODATE, &dev->flags)) || 2762 (s.failed == 1 && s.failed_num == sh->pd_idx))) 2763 handle_completed_write_requests(conf, sh, disks, &return_bi); 2764 2765 /* Now we might consider reading some blocks, either to check/generate 2766 * parity, or to satisfy requests 2767 * or to load a block that is being partially written. 2768 */ 2769 if (s.to_read || s.non_overwrite || 2770 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding || 2771 test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending)) 2772 handle_issuing_new_read_requests5(sh, &s, disks); 2773 2774 /* Now we check to see if any write operations have recently 2775 * completed 2776 */ 2777 2778 /* leave prexor set until postxor is done, allows us to distinguish 2779 * a rmw from a rcw during biodrain 2780 */ 2781 prexor = 0; 2782 if (test_bit(STRIPE_OP_PREXOR, &sh->ops.complete) && 2783 test_bit(STRIPE_OP_POSTXOR, &sh->ops.complete)) { 2784 2785 prexor = 1; 2786 clear_bit(STRIPE_OP_PREXOR, &sh->ops.complete); 2787 clear_bit(STRIPE_OP_PREXOR, &sh->ops.ack); 2788 clear_bit(STRIPE_OP_PREXOR, &sh->ops.pending); 2789 2790 for (i = disks; i--; ) 2791 clear_bit(R5_Wantprexor, &sh->dev[i].flags); 2792 } 2793 2794 /* if only POSTXOR is set then this is an 'expand' postxor */ 2795 if (test_bit(STRIPE_OP_BIODRAIN, &sh->ops.complete) && 2796 test_bit(STRIPE_OP_POSTXOR, &sh->ops.complete)) { 2797 2798 clear_bit(STRIPE_OP_BIODRAIN, &sh->ops.complete); 2799 clear_bit(STRIPE_OP_BIODRAIN, &sh->ops.ack); 2800 clear_bit(STRIPE_OP_BIODRAIN, &sh->ops.pending); 2801 2802 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.complete); 2803 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.ack); 2804 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.pending); 2805 2806 /* All the 'written' buffers and the parity block are ready to 2807 * be written back to disk 2808 */ 2809 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags)); 2810 for (i = disks; i--; ) { 2811 dev = &sh->dev[i]; 2812 if (test_bit(R5_LOCKED, &dev->flags) && 2813 (i == sh->pd_idx || dev->written)) { 2814 pr_debug("Writing block %d\n", i); 2815 set_bit(R5_Wantwrite, &dev->flags); 2816 if (!test_and_set_bit( 2817 STRIPE_OP_IO, &sh->ops.pending)) 2818 sh->ops.count++; 2819 if (prexor) 2820 continue; 2821 if (!test_bit(R5_Insync, &dev->flags) || 2822 (i == sh->pd_idx && s.failed == 0)) 2823 set_bit(STRIPE_INSYNC, &sh->state); 2824 } 2825 } 2826 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 2827 atomic_dec(&conf->preread_active_stripes); 2828 if (atomic_read(&conf->preread_active_stripes) < 2829 IO_THRESHOLD) 2830 md_wakeup_thread(conf->mddev->thread); 2831 } 2832 } 2833 2834 /* Now to consider new write requests and what else, if anything 2835 * should be read. We do not handle new writes when: 2836 * 1/ A 'write' operation (copy+xor) is already in flight. 2837 * 2/ A 'check' operation is in flight, as it may clobber the parity 2838 * block. 2839 */ 2840 if (s.to_write && !test_bit(STRIPE_OP_POSTXOR, &sh->ops.pending) && 2841 !test_bit(STRIPE_OP_CHECK, &sh->ops.pending)) 2842 handle_issuing_new_write_requests5(conf, sh, &s, disks); 2843 2844 /* maybe we need to check and possibly fix the parity for this stripe 2845 * Any reads will already have been scheduled, so we just see if enough 2846 * data is available. The parity check is held off while parity 2847 * dependent operations are in flight. 2848 */ 2849 if ((s.syncing && s.locked == 0 && 2850 !test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending) && 2851 !test_bit(STRIPE_INSYNC, &sh->state)) || 2852 test_bit(STRIPE_OP_CHECK, &sh->ops.pending) || 2853 test_bit(STRIPE_OP_MOD_REPAIR_PD, &sh->ops.pending)) 2854 handle_parity_checks5(conf, sh, &s, disks); 2855 2856 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 2857 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 2858 clear_bit(STRIPE_SYNCING, &sh->state); 2859 } 2860 2861 /* If the failed drive is just a ReadError, then we might need to progress 2862 * the repair/check process 2863 */ 2864 if (s.failed == 1 && !conf->mddev->ro && 2865 test_bit(R5_ReadError, &sh->dev[s.failed_num].flags) 2866 && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags) 2867 && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags) 2868 ) { 2869 dev = &sh->dev[s.failed_num]; 2870 if (!test_bit(R5_ReWrite, &dev->flags)) { 2871 set_bit(R5_Wantwrite, &dev->flags); 2872 if (!test_and_set_bit(STRIPE_OP_IO, &sh->ops.pending)) 2873 sh->ops.count++; 2874 set_bit(R5_ReWrite, &dev->flags); 2875 set_bit(R5_LOCKED, &dev->flags); 2876 s.locked++; 2877 } else { 2878 /* let's read it back */ 2879 set_bit(R5_Wantread, &dev->flags); 2880 if (!test_and_set_bit(STRIPE_OP_IO, &sh->ops.pending)) 2881 sh->ops.count++; 2882 set_bit(R5_LOCKED, &dev->flags); 2883 s.locked++; 2884 } 2885 } 2886 2887 /* Finish postxor operations initiated by the expansion 2888 * process 2889 */ 2890 if (test_bit(STRIPE_OP_POSTXOR, &sh->ops.complete) && 2891 !test_bit(STRIPE_OP_BIODRAIN, &sh->ops.pending)) { 2892 2893 clear_bit(STRIPE_EXPANDING, &sh->state); 2894 2895 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.pending); 2896 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.ack); 2897 clear_bit(STRIPE_OP_POSTXOR, &sh->ops.complete); 2898 2899 for (i = conf->raid_disks; i--; ) { 2900 set_bit(R5_Wantwrite, &sh->dev[i].flags); 2901 if (!test_and_set_bit(STRIPE_OP_IO, &sh->ops.pending)) 2902 sh->ops.count++; 2903 } 2904 } 2905 2906 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 2907 !test_bit(STRIPE_OP_POSTXOR, &sh->ops.pending)) { 2908 /* Need to write out all blocks after computing parity */ 2909 sh->disks = conf->raid_disks; 2910 sh->pd_idx = stripe_to_pdidx(sh->sector, conf, 2911 conf->raid_disks); 2912 s.locked += handle_write_operations5(sh, 1, 1); 2913 } else if (s.expanded && 2914 !test_bit(STRIPE_OP_POSTXOR, &sh->ops.pending)) { 2915 clear_bit(STRIPE_EXPAND_READY, &sh->state); 2916 atomic_dec(&conf->reshape_stripes); 2917 wake_up(&conf->wait_for_overlap); 2918 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 2919 } 2920 2921 if (s.expanding && s.locked == 0 && 2922 !test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending)) 2923 handle_stripe_expansion(conf, sh, NULL); 2924 2925 if (sh->ops.count) 2926 pending = get_stripe_work(sh); 2927 2928 unlock: 2929 spin_unlock(&sh->lock); 2930 2931 /* wait for this device to become unblocked */ 2932 if (unlikely(blocked_rdev)) 2933 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 2934 2935 if (pending) 2936 raid5_run_ops(sh, pending); 2937 2938 return_io(return_bi); 2939 2940 } 2941 2942 static void handle_stripe6(struct stripe_head *sh, struct page *tmp_page) 2943 { 2944 raid6_conf_t *conf = sh->raid_conf; 2945 int disks = sh->disks; 2946 struct bio *return_bi = NULL; 2947 int i, pd_idx = sh->pd_idx; 2948 struct stripe_head_state s; 2949 struct r6_state r6s; 2950 struct r5dev *dev, *pdev, *qdev; 2951 mdk_rdev_t *blocked_rdev = NULL; 2952 2953 r6s.qd_idx = raid6_next_disk(pd_idx, disks); 2954 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 2955 "pd_idx=%d, qd_idx=%d\n", 2956 (unsigned long long)sh->sector, sh->state, 2957 atomic_read(&sh->count), pd_idx, r6s.qd_idx); 2958 memset(&s, 0, sizeof(s)); 2959 2960 spin_lock(&sh->lock); 2961 clear_bit(STRIPE_HANDLE, &sh->state); 2962 clear_bit(STRIPE_DELAYED, &sh->state); 2963 2964 s.syncing = test_bit(STRIPE_SYNCING, &sh->state); 2965 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 2966 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 2967 /* Now to look around and see what can be done */ 2968 2969 rcu_read_lock(); 2970 for (i=disks; i--; ) { 2971 mdk_rdev_t *rdev; 2972 dev = &sh->dev[i]; 2973 clear_bit(R5_Insync, &dev->flags); 2974 2975 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 2976 i, dev->flags, dev->toread, dev->towrite, dev->written); 2977 /* maybe we can reply to a read */ 2978 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread) { 2979 struct bio *rbi, *rbi2; 2980 pr_debug("Return read for disc %d\n", i); 2981 spin_lock_irq(&conf->device_lock); 2982 rbi = dev->toread; 2983 dev->toread = NULL; 2984 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 2985 wake_up(&conf->wait_for_overlap); 2986 spin_unlock_irq(&conf->device_lock); 2987 while (rbi && rbi->bi_sector < dev->sector + STRIPE_SECTORS) { 2988 copy_data(0, rbi, dev->page, dev->sector); 2989 rbi2 = r5_next_bio(rbi, dev->sector); 2990 spin_lock_irq(&conf->device_lock); 2991 if (--rbi->bi_phys_segments == 0) { 2992 rbi->bi_next = return_bi; 2993 return_bi = rbi; 2994 } 2995 spin_unlock_irq(&conf->device_lock); 2996 rbi = rbi2; 2997 } 2998 } 2999 3000 /* now count some things */ 3001 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++; 3002 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++; 3003 3004 3005 if (dev->toread) 3006 s.to_read++; 3007 if (dev->towrite) { 3008 s.to_write++; 3009 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3010 s.non_overwrite++; 3011 } 3012 if (dev->written) 3013 s.written++; 3014 rdev = rcu_dereference(conf->disks[i].rdev); 3015 if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) { 3016 blocked_rdev = rdev; 3017 atomic_inc(&rdev->nr_pending); 3018 break; 3019 } 3020 if (!rdev || !test_bit(In_sync, &rdev->flags)) { 3021 /* The ReadError flag will just be confusing now */ 3022 clear_bit(R5_ReadError, &dev->flags); 3023 clear_bit(R5_ReWrite, &dev->flags); 3024 } 3025 if (!rdev || !test_bit(In_sync, &rdev->flags) 3026 || test_bit(R5_ReadError, &dev->flags)) { 3027 if (s.failed < 2) 3028 r6s.failed_num[s.failed] = i; 3029 s.failed++; 3030 } else 3031 set_bit(R5_Insync, &dev->flags); 3032 } 3033 rcu_read_unlock(); 3034 3035 if (unlikely(blocked_rdev)) { 3036 set_bit(STRIPE_HANDLE, &sh->state); 3037 goto unlock; 3038 } 3039 pr_debug("locked=%d uptodate=%d to_read=%d" 3040 " to_write=%d failed=%d failed_num=%d,%d\n", 3041 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3042 r6s.failed_num[0], r6s.failed_num[1]); 3043 /* check if the array has lost >2 devices and, if so, some requests 3044 * might need to be failed 3045 */ 3046 if (s.failed > 2 && s.to_read+s.to_write+s.written) 3047 handle_requests_to_failed_array(conf, sh, &s, disks, 3048 &return_bi); 3049 if (s.failed > 2 && s.syncing) { 3050 md_done_sync(conf->mddev, STRIPE_SECTORS,0); 3051 clear_bit(STRIPE_SYNCING, &sh->state); 3052 s.syncing = 0; 3053 } 3054 3055 /* 3056 * might be able to return some write requests if the parity blocks 3057 * are safe, or on a failed drive 3058 */ 3059 pdev = &sh->dev[pd_idx]; 3060 r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx) 3061 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx); 3062 qdev = &sh->dev[r6s.qd_idx]; 3063 r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == r6s.qd_idx) 3064 || (s.failed >= 2 && r6s.failed_num[1] == r6s.qd_idx); 3065 3066 if ( s.written && 3067 ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3068 && !test_bit(R5_LOCKED, &pdev->flags) 3069 && test_bit(R5_UPTODATE, &pdev->flags)))) && 3070 ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3071 && !test_bit(R5_LOCKED, &qdev->flags) 3072 && test_bit(R5_UPTODATE, &qdev->flags))))) 3073 handle_completed_write_requests(conf, sh, disks, &return_bi); 3074 3075 /* Now we might consider reading some blocks, either to check/generate 3076 * parity, or to satisfy requests 3077 * or to load a block that is being partially written. 3078 */ 3079 if (s.to_read || s.non_overwrite || (s.to_write && s.failed) || 3080 (s.syncing && (s.uptodate < disks)) || s.expanding) 3081 handle_issuing_new_read_requests6(sh, &s, &r6s, disks); 3082 3083 /* now to consider writing and what else, if anything should be read */ 3084 if (s.to_write) 3085 handle_issuing_new_write_requests6(conf, sh, &s, &r6s, disks); 3086 3087 /* maybe we need to check and possibly fix the parity for this stripe 3088 * Any reads will already have been scheduled, so we just see if enough 3089 * data is available 3090 */ 3091 if (s.syncing && s.locked == 0 && !test_bit(STRIPE_INSYNC, &sh->state)) 3092 handle_parity_checks6(conf, sh, &s, &r6s, tmp_page, disks); 3093 3094 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) { 3095 md_done_sync(conf->mddev, STRIPE_SECTORS,1); 3096 clear_bit(STRIPE_SYNCING, &sh->state); 3097 } 3098 3099 /* If the failed drives are just a ReadError, then we might need 3100 * to progress the repair/check process 3101 */ 3102 if (s.failed <= 2 && !conf->mddev->ro) 3103 for (i = 0; i < s.failed; i++) { 3104 dev = &sh->dev[r6s.failed_num[i]]; 3105 if (test_bit(R5_ReadError, &dev->flags) 3106 && !test_bit(R5_LOCKED, &dev->flags) 3107 && test_bit(R5_UPTODATE, &dev->flags) 3108 ) { 3109 if (!test_bit(R5_ReWrite, &dev->flags)) { 3110 set_bit(R5_Wantwrite, &dev->flags); 3111 set_bit(R5_ReWrite, &dev->flags); 3112 set_bit(R5_LOCKED, &dev->flags); 3113 } else { 3114 /* let's read it back */ 3115 set_bit(R5_Wantread, &dev->flags); 3116 set_bit(R5_LOCKED, &dev->flags); 3117 } 3118 } 3119 } 3120 3121 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state)) { 3122 /* Need to write out all blocks after computing P&Q */ 3123 sh->disks = conf->raid_disks; 3124 sh->pd_idx = stripe_to_pdidx(sh->sector, conf, 3125 conf->raid_disks); 3126 compute_parity6(sh, RECONSTRUCT_WRITE); 3127 for (i = conf->raid_disks ; i-- ; ) { 3128 set_bit(R5_LOCKED, &sh->dev[i].flags); 3129 s.locked++; 3130 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3131 } 3132 clear_bit(STRIPE_EXPANDING, &sh->state); 3133 } else if (s.expanded) { 3134 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3135 atomic_dec(&conf->reshape_stripes); 3136 wake_up(&conf->wait_for_overlap); 3137 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3138 } 3139 3140 if (s.expanding && s.locked == 0 && 3141 !test_bit(STRIPE_OP_COMPUTE_BLK, &sh->ops.pending)) 3142 handle_stripe_expansion(conf, sh, &r6s); 3143 3144 unlock: 3145 spin_unlock(&sh->lock); 3146 3147 /* wait for this device to become unblocked */ 3148 if (unlikely(blocked_rdev)) 3149 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev); 3150 3151 return_io(return_bi); 3152 3153 for (i=disks; i-- ;) { 3154 int rw; 3155 struct bio *bi; 3156 mdk_rdev_t *rdev; 3157 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) 3158 rw = WRITE; 3159 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 3160 rw = READ; 3161 else 3162 continue; 3163 3164 set_bit(STRIPE_IO_STARTED, &sh->state); 3165 3166 bi = &sh->dev[i].req; 3167 3168 bi->bi_rw = rw; 3169 if (rw == WRITE) 3170 bi->bi_end_io = raid5_end_write_request; 3171 else 3172 bi->bi_end_io = raid5_end_read_request; 3173 3174 rcu_read_lock(); 3175 rdev = rcu_dereference(conf->disks[i].rdev); 3176 if (rdev && test_bit(Faulty, &rdev->flags)) 3177 rdev = NULL; 3178 if (rdev) 3179 atomic_inc(&rdev->nr_pending); 3180 rcu_read_unlock(); 3181 3182 if (rdev) { 3183 if (s.syncing || s.expanding || s.expanded) 3184 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 3185 3186 bi->bi_bdev = rdev->bdev; 3187 pr_debug("for %llu schedule op %ld on disc %d\n", 3188 (unsigned long long)sh->sector, bi->bi_rw, i); 3189 atomic_inc(&sh->count); 3190 bi->bi_sector = sh->sector + rdev->data_offset; 3191 bi->bi_flags = 1 << BIO_UPTODATE; 3192 bi->bi_vcnt = 1; 3193 bi->bi_max_vecs = 1; 3194 bi->bi_idx = 0; 3195 bi->bi_io_vec = &sh->dev[i].vec; 3196 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 3197 bi->bi_io_vec[0].bv_offset = 0; 3198 bi->bi_size = STRIPE_SIZE; 3199 bi->bi_next = NULL; 3200 if (rw == WRITE && 3201 test_bit(R5_ReWrite, &sh->dev[i].flags)) 3202 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 3203 generic_make_request(bi); 3204 } else { 3205 if (rw == WRITE) 3206 set_bit(STRIPE_DEGRADED, &sh->state); 3207 pr_debug("skip op %ld on disc %d for sector %llu\n", 3208 bi->bi_rw, i, (unsigned long long)sh->sector); 3209 clear_bit(R5_LOCKED, &sh->dev[i].flags); 3210 set_bit(STRIPE_HANDLE, &sh->state); 3211 } 3212 } 3213 } 3214 3215 static void handle_stripe(struct stripe_head *sh, struct page *tmp_page) 3216 { 3217 if (sh->raid_conf->level == 6) 3218 handle_stripe6(sh, tmp_page); 3219 else 3220 handle_stripe5(sh); 3221 } 3222 3223 3224 3225 static void raid5_activate_delayed(raid5_conf_t *conf) 3226 { 3227 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3228 while (!list_empty(&conf->delayed_list)) { 3229 struct list_head *l = conf->delayed_list.next; 3230 struct stripe_head *sh; 3231 sh = list_entry(l, struct stripe_head, lru); 3232 list_del_init(l); 3233 clear_bit(STRIPE_DELAYED, &sh->state); 3234 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3235 atomic_inc(&conf->preread_active_stripes); 3236 list_add_tail(&sh->lru, &conf->hold_list); 3237 } 3238 } else 3239 blk_plug_device(conf->mddev->queue); 3240 } 3241 3242 static void activate_bit_delay(raid5_conf_t *conf) 3243 { 3244 /* device_lock is held */ 3245 struct list_head head; 3246 list_add(&head, &conf->bitmap_list); 3247 list_del_init(&conf->bitmap_list); 3248 while (!list_empty(&head)) { 3249 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3250 list_del_init(&sh->lru); 3251 atomic_inc(&sh->count); 3252 __release_stripe(conf, sh); 3253 } 3254 } 3255 3256 static void unplug_slaves(mddev_t *mddev) 3257 { 3258 raid5_conf_t *conf = mddev_to_conf(mddev); 3259 int i; 3260 3261 rcu_read_lock(); 3262 for (i=0; i<mddev->raid_disks; i++) { 3263 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev); 3264 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) { 3265 struct request_queue *r_queue = bdev_get_queue(rdev->bdev); 3266 3267 atomic_inc(&rdev->nr_pending); 3268 rcu_read_unlock(); 3269 3270 blk_unplug(r_queue); 3271 3272 rdev_dec_pending(rdev, mddev); 3273 rcu_read_lock(); 3274 } 3275 } 3276 rcu_read_unlock(); 3277 } 3278 3279 static void raid5_unplug_device(struct request_queue *q) 3280 { 3281 mddev_t *mddev = q->queuedata; 3282 raid5_conf_t *conf = mddev_to_conf(mddev); 3283 unsigned long flags; 3284 3285 spin_lock_irqsave(&conf->device_lock, flags); 3286 3287 if (blk_remove_plug(q)) { 3288 conf->seq_flush++; 3289 raid5_activate_delayed(conf); 3290 } 3291 md_wakeup_thread(mddev->thread); 3292 3293 spin_unlock_irqrestore(&conf->device_lock, flags); 3294 3295 unplug_slaves(mddev); 3296 } 3297 3298 static int raid5_congested(void *data, int bits) 3299 { 3300 mddev_t *mddev = data; 3301 raid5_conf_t *conf = mddev_to_conf(mddev); 3302 3303 /* No difference between reads and writes. Just check 3304 * how busy the stripe_cache is 3305 */ 3306 if (conf->inactive_blocked) 3307 return 1; 3308 if (conf->quiesce) 3309 return 1; 3310 if (list_empty_careful(&conf->inactive_list)) 3311 return 1; 3312 3313 return 0; 3314 } 3315 3316 /* We want read requests to align with chunks where possible, 3317 * but write requests don't need to. 3318 */ 3319 static int raid5_mergeable_bvec(struct request_queue *q, struct bio *bio, struct bio_vec *biovec) 3320 { 3321 mddev_t *mddev = q->queuedata; 3322 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3323 int max; 3324 unsigned int chunk_sectors = mddev->chunk_size >> 9; 3325 unsigned int bio_sectors = bio->bi_size >> 9; 3326 3327 if (bio_data_dir(bio) == WRITE) 3328 return biovec->bv_len; /* always allow writes to be mergeable */ 3329 3330 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3331 if (max < 0) max = 0; 3332 if (max <= biovec->bv_len && bio_sectors == 0) 3333 return biovec->bv_len; 3334 else 3335 return max; 3336 } 3337 3338 3339 static int in_chunk_boundary(mddev_t *mddev, struct bio *bio) 3340 { 3341 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3342 unsigned int chunk_sectors = mddev->chunk_size >> 9; 3343 unsigned int bio_sectors = bio->bi_size >> 9; 3344 3345 return chunk_sectors >= 3346 ((sector & (chunk_sectors - 1)) + bio_sectors); 3347 } 3348 3349 /* 3350 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3351 * later sampled by raid5d. 3352 */ 3353 static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf) 3354 { 3355 unsigned long flags; 3356 3357 spin_lock_irqsave(&conf->device_lock, flags); 3358 3359 bi->bi_next = conf->retry_read_aligned_list; 3360 conf->retry_read_aligned_list = bi; 3361 3362 spin_unlock_irqrestore(&conf->device_lock, flags); 3363 md_wakeup_thread(conf->mddev->thread); 3364 } 3365 3366 3367 static struct bio *remove_bio_from_retry(raid5_conf_t *conf) 3368 { 3369 struct bio *bi; 3370 3371 bi = conf->retry_read_aligned; 3372 if (bi) { 3373 conf->retry_read_aligned = NULL; 3374 return bi; 3375 } 3376 bi = conf->retry_read_aligned_list; 3377 if(bi) { 3378 conf->retry_read_aligned_list = bi->bi_next; 3379 bi->bi_next = NULL; 3380 bi->bi_phys_segments = 1; /* biased count of active stripes */ 3381 bi->bi_hw_segments = 0; /* count of processed stripes */ 3382 } 3383 3384 return bi; 3385 } 3386 3387 3388 /* 3389 * The "raid5_align_endio" should check if the read succeeded and if it 3390 * did, call bio_endio on the original bio (having bio_put the new bio 3391 * first). 3392 * If the read failed.. 3393 */ 3394 static void raid5_align_endio(struct bio *bi, int error) 3395 { 3396 struct bio* raid_bi = bi->bi_private; 3397 mddev_t *mddev; 3398 raid5_conf_t *conf; 3399 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 3400 mdk_rdev_t *rdev; 3401 3402 bio_put(bi); 3403 3404 mddev = raid_bi->bi_bdev->bd_disk->queue->queuedata; 3405 conf = mddev_to_conf(mddev); 3406 rdev = (void*)raid_bi->bi_next; 3407 raid_bi->bi_next = NULL; 3408 3409 rdev_dec_pending(rdev, conf->mddev); 3410 3411 if (!error && uptodate) { 3412 bio_endio(raid_bi, 0); 3413 if (atomic_dec_and_test(&conf->active_aligned_reads)) 3414 wake_up(&conf->wait_for_stripe); 3415 return; 3416 } 3417 3418 3419 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 3420 3421 add_bio_to_retry(raid_bi, conf); 3422 } 3423 3424 static int bio_fits_rdev(struct bio *bi) 3425 { 3426 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 3427 3428 if ((bi->bi_size>>9) > q->max_sectors) 3429 return 0; 3430 blk_recount_segments(q, bi); 3431 if (bi->bi_phys_segments > q->max_phys_segments || 3432 bi->bi_hw_segments > q->max_hw_segments) 3433 return 0; 3434 3435 if (q->merge_bvec_fn) 3436 /* it's too hard to apply the merge_bvec_fn at this stage, 3437 * just just give up 3438 */ 3439 return 0; 3440 3441 return 1; 3442 } 3443 3444 3445 static int chunk_aligned_read(struct request_queue *q, struct bio * raid_bio) 3446 { 3447 mddev_t *mddev = q->queuedata; 3448 raid5_conf_t *conf = mddev_to_conf(mddev); 3449 const unsigned int raid_disks = conf->raid_disks; 3450 const unsigned int data_disks = raid_disks - conf->max_degraded; 3451 unsigned int dd_idx, pd_idx; 3452 struct bio* align_bi; 3453 mdk_rdev_t *rdev; 3454 3455 if (!in_chunk_boundary(mddev, raid_bio)) { 3456 pr_debug("chunk_aligned_read : non aligned\n"); 3457 return 0; 3458 } 3459 /* 3460 * use bio_clone to make a copy of the bio 3461 */ 3462 align_bi = bio_clone(raid_bio, GFP_NOIO); 3463 if (!align_bi) 3464 return 0; 3465 /* 3466 * set bi_end_io to a new function, and set bi_private to the 3467 * original bio. 3468 */ 3469 align_bi->bi_end_io = raid5_align_endio; 3470 align_bi->bi_private = raid_bio; 3471 /* 3472 * compute position 3473 */ 3474 align_bi->bi_sector = raid5_compute_sector(raid_bio->bi_sector, 3475 raid_disks, 3476 data_disks, 3477 &dd_idx, 3478 &pd_idx, 3479 conf); 3480 3481 rcu_read_lock(); 3482 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 3483 if (rdev && test_bit(In_sync, &rdev->flags)) { 3484 atomic_inc(&rdev->nr_pending); 3485 rcu_read_unlock(); 3486 raid_bio->bi_next = (void*)rdev; 3487 align_bi->bi_bdev = rdev->bdev; 3488 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 3489 align_bi->bi_sector += rdev->data_offset; 3490 3491 if (!bio_fits_rdev(align_bi)) { 3492 /* too big in some way */ 3493 bio_put(align_bi); 3494 rdev_dec_pending(rdev, mddev); 3495 return 0; 3496 } 3497 3498 spin_lock_irq(&conf->device_lock); 3499 wait_event_lock_irq(conf->wait_for_stripe, 3500 conf->quiesce == 0, 3501 conf->device_lock, /* nothing */); 3502 atomic_inc(&conf->active_aligned_reads); 3503 spin_unlock_irq(&conf->device_lock); 3504 3505 generic_make_request(align_bi); 3506 return 1; 3507 } else { 3508 rcu_read_unlock(); 3509 bio_put(align_bi); 3510 return 0; 3511 } 3512 } 3513 3514 /* __get_priority_stripe - get the next stripe to process 3515 * 3516 * Full stripe writes are allowed to pass preread active stripes up until 3517 * the bypass_threshold is exceeded. In general the bypass_count 3518 * increments when the handle_list is handled before the hold_list; however, it 3519 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 3520 * stripe with in flight i/o. The bypass_count will be reset when the 3521 * head of the hold_list has changed, i.e. the head was promoted to the 3522 * handle_list. 3523 */ 3524 static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf) 3525 { 3526 struct stripe_head *sh; 3527 3528 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 3529 __func__, 3530 list_empty(&conf->handle_list) ? "empty" : "busy", 3531 list_empty(&conf->hold_list) ? "empty" : "busy", 3532 atomic_read(&conf->pending_full_writes), conf->bypass_count); 3533 3534 if (!list_empty(&conf->handle_list)) { 3535 sh = list_entry(conf->handle_list.next, typeof(*sh), lru); 3536 3537 if (list_empty(&conf->hold_list)) 3538 conf->bypass_count = 0; 3539 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 3540 if (conf->hold_list.next == conf->last_hold) 3541 conf->bypass_count++; 3542 else { 3543 conf->last_hold = conf->hold_list.next; 3544 conf->bypass_count -= conf->bypass_threshold; 3545 if (conf->bypass_count < 0) 3546 conf->bypass_count = 0; 3547 } 3548 } 3549 } else if (!list_empty(&conf->hold_list) && 3550 ((conf->bypass_threshold && 3551 conf->bypass_count > conf->bypass_threshold) || 3552 atomic_read(&conf->pending_full_writes) == 0)) { 3553 sh = list_entry(conf->hold_list.next, 3554 typeof(*sh), lru); 3555 conf->bypass_count -= conf->bypass_threshold; 3556 if (conf->bypass_count < 0) 3557 conf->bypass_count = 0; 3558 } else 3559 return NULL; 3560 3561 list_del_init(&sh->lru); 3562 atomic_inc(&sh->count); 3563 BUG_ON(atomic_read(&sh->count) != 1); 3564 return sh; 3565 } 3566 3567 static int make_request(struct request_queue *q, struct bio * bi) 3568 { 3569 mddev_t *mddev = q->queuedata; 3570 raid5_conf_t *conf = mddev_to_conf(mddev); 3571 unsigned int dd_idx, pd_idx; 3572 sector_t new_sector; 3573 sector_t logical_sector, last_sector; 3574 struct stripe_head *sh; 3575 const int rw = bio_data_dir(bi); 3576 int remaining; 3577 3578 if (unlikely(bio_barrier(bi))) { 3579 bio_endio(bi, -EOPNOTSUPP); 3580 return 0; 3581 } 3582 3583 md_write_start(mddev, bi); 3584 3585 disk_stat_inc(mddev->gendisk, ios[rw]); 3586 disk_stat_add(mddev->gendisk, sectors[rw], bio_sectors(bi)); 3587 3588 if (rw == READ && 3589 mddev->reshape_position == MaxSector && 3590 chunk_aligned_read(q,bi)) 3591 return 0; 3592 3593 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 3594 last_sector = bi->bi_sector + (bi->bi_size>>9); 3595 bi->bi_next = NULL; 3596 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 3597 3598 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 3599 DEFINE_WAIT(w); 3600 int disks, data_disks; 3601 3602 retry: 3603 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 3604 if (likely(conf->expand_progress == MaxSector)) 3605 disks = conf->raid_disks; 3606 else { 3607 /* spinlock is needed as expand_progress may be 3608 * 64bit on a 32bit platform, and so it might be 3609 * possible to see a half-updated value 3610 * Ofcourse expand_progress could change after 3611 * the lock is dropped, so once we get a reference 3612 * to the stripe that we think it is, we will have 3613 * to check again. 3614 */ 3615 spin_lock_irq(&conf->device_lock); 3616 disks = conf->raid_disks; 3617 if (logical_sector >= conf->expand_progress) 3618 disks = conf->previous_raid_disks; 3619 else { 3620 if (logical_sector >= conf->expand_lo) { 3621 spin_unlock_irq(&conf->device_lock); 3622 schedule(); 3623 goto retry; 3624 } 3625 } 3626 spin_unlock_irq(&conf->device_lock); 3627 } 3628 data_disks = disks - conf->max_degraded; 3629 3630 new_sector = raid5_compute_sector(logical_sector, disks, data_disks, 3631 &dd_idx, &pd_idx, conf); 3632 pr_debug("raid5: make_request, sector %llu logical %llu\n", 3633 (unsigned long long)new_sector, 3634 (unsigned long long)logical_sector); 3635 3636 sh = get_active_stripe(conf, new_sector, disks, pd_idx, (bi->bi_rw&RWA_MASK)); 3637 if (sh) { 3638 if (unlikely(conf->expand_progress != MaxSector)) { 3639 /* expansion might have moved on while waiting for a 3640 * stripe, so we must do the range check again. 3641 * Expansion could still move past after this 3642 * test, but as we are holding a reference to 3643 * 'sh', we know that if that happens, 3644 * STRIPE_EXPANDING will get set and the expansion 3645 * won't proceed until we finish with the stripe. 3646 */ 3647 int must_retry = 0; 3648 spin_lock_irq(&conf->device_lock); 3649 if (logical_sector < conf->expand_progress && 3650 disks == conf->previous_raid_disks) 3651 /* mismatch, need to try again */ 3652 must_retry = 1; 3653 spin_unlock_irq(&conf->device_lock); 3654 if (must_retry) { 3655 release_stripe(sh); 3656 goto retry; 3657 } 3658 } 3659 /* FIXME what if we get a false positive because these 3660 * are being updated. 3661 */ 3662 if (logical_sector >= mddev->suspend_lo && 3663 logical_sector < mddev->suspend_hi) { 3664 release_stripe(sh); 3665 schedule(); 3666 goto retry; 3667 } 3668 3669 if (test_bit(STRIPE_EXPANDING, &sh->state) || 3670 !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) { 3671 /* Stripe is busy expanding or 3672 * add failed due to overlap. Flush everything 3673 * and wait a while 3674 */ 3675 raid5_unplug_device(mddev->queue); 3676 release_stripe(sh); 3677 schedule(); 3678 goto retry; 3679 } 3680 finish_wait(&conf->wait_for_overlap, &w); 3681 set_bit(STRIPE_HANDLE, &sh->state); 3682 clear_bit(STRIPE_DELAYED, &sh->state); 3683 release_stripe(sh); 3684 } else { 3685 /* cannot get stripe for read-ahead, just give-up */ 3686 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3687 finish_wait(&conf->wait_for_overlap, &w); 3688 break; 3689 } 3690 3691 } 3692 spin_lock_irq(&conf->device_lock); 3693 remaining = --bi->bi_phys_segments; 3694 spin_unlock_irq(&conf->device_lock); 3695 if (remaining == 0) { 3696 3697 if ( rw == WRITE ) 3698 md_write_end(mddev); 3699 3700 bi->bi_end_io(bi, 3701 test_bit(BIO_UPTODATE, &bi->bi_flags) 3702 ? 0 : -EIO); 3703 } 3704 return 0; 3705 } 3706 3707 static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped) 3708 { 3709 /* reshaping is quite different to recovery/resync so it is 3710 * handled quite separately ... here. 3711 * 3712 * On each call to sync_request, we gather one chunk worth of 3713 * destination stripes and flag them as expanding. 3714 * Then we find all the source stripes and request reads. 3715 * As the reads complete, handle_stripe will copy the data 3716 * into the destination stripe and release that stripe. 3717 */ 3718 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 3719 struct stripe_head *sh; 3720 int pd_idx; 3721 sector_t first_sector, last_sector; 3722 int raid_disks = conf->previous_raid_disks; 3723 int data_disks = raid_disks - conf->max_degraded; 3724 int new_data_disks = conf->raid_disks - conf->max_degraded; 3725 int i; 3726 int dd_idx; 3727 sector_t writepos, safepos, gap; 3728 3729 if (sector_nr == 0 && 3730 conf->expand_progress != 0) { 3731 /* restarting in the middle, skip the initial sectors */ 3732 sector_nr = conf->expand_progress; 3733 sector_div(sector_nr, new_data_disks); 3734 *skipped = 1; 3735 return sector_nr; 3736 } 3737 3738 /* we update the metadata when there is more than 3Meg 3739 * in the block range (that is rather arbitrary, should 3740 * probably be time based) or when the data about to be 3741 * copied would over-write the source of the data at 3742 * the front of the range. 3743 * i.e. one new_stripe forward from expand_progress new_maps 3744 * to after where expand_lo old_maps to 3745 */ 3746 writepos = conf->expand_progress + 3747 conf->chunk_size/512*(new_data_disks); 3748 sector_div(writepos, new_data_disks); 3749 safepos = conf->expand_lo; 3750 sector_div(safepos, data_disks); 3751 gap = conf->expand_progress - conf->expand_lo; 3752 3753 if (writepos >= safepos || 3754 gap > (new_data_disks)*3000*2 /*3Meg*/) { 3755 /* Cannot proceed until we've updated the superblock... */ 3756 wait_event(conf->wait_for_overlap, 3757 atomic_read(&conf->reshape_stripes)==0); 3758 mddev->reshape_position = conf->expand_progress; 3759 set_bit(MD_CHANGE_DEVS, &mddev->flags); 3760 md_wakeup_thread(mddev->thread); 3761 wait_event(mddev->sb_wait, mddev->flags == 0 || 3762 kthread_should_stop()); 3763 spin_lock_irq(&conf->device_lock); 3764 conf->expand_lo = mddev->reshape_position; 3765 spin_unlock_irq(&conf->device_lock); 3766 wake_up(&conf->wait_for_overlap); 3767 } 3768 3769 for (i=0; i < conf->chunk_size/512; i+= STRIPE_SECTORS) { 3770 int j; 3771 int skipped = 0; 3772 pd_idx = stripe_to_pdidx(sector_nr+i, conf, conf->raid_disks); 3773 sh = get_active_stripe(conf, sector_nr+i, 3774 conf->raid_disks, pd_idx, 0); 3775 set_bit(STRIPE_EXPANDING, &sh->state); 3776 atomic_inc(&conf->reshape_stripes); 3777 /* If any of this stripe is beyond the end of the old 3778 * array, then we need to zero those blocks 3779 */ 3780 for (j=sh->disks; j--;) { 3781 sector_t s; 3782 if (j == sh->pd_idx) 3783 continue; 3784 if (conf->level == 6 && 3785 j == raid6_next_disk(sh->pd_idx, sh->disks)) 3786 continue; 3787 s = compute_blocknr(sh, j); 3788 if (s < (mddev->array_size<<1)) { 3789 skipped = 1; 3790 continue; 3791 } 3792 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 3793 set_bit(R5_Expanded, &sh->dev[j].flags); 3794 set_bit(R5_UPTODATE, &sh->dev[j].flags); 3795 } 3796 if (!skipped) { 3797 set_bit(STRIPE_EXPAND_READY, &sh->state); 3798 set_bit(STRIPE_HANDLE, &sh->state); 3799 } 3800 release_stripe(sh); 3801 } 3802 spin_lock_irq(&conf->device_lock); 3803 conf->expand_progress = (sector_nr + i) * new_data_disks; 3804 spin_unlock_irq(&conf->device_lock); 3805 /* Ok, those stripe are ready. We can start scheduling 3806 * reads on the source stripes. 3807 * The source stripes are determined by mapping the first and last 3808 * block on the destination stripes. 3809 */ 3810 first_sector = 3811 raid5_compute_sector(sector_nr*(new_data_disks), 3812 raid_disks, data_disks, 3813 &dd_idx, &pd_idx, conf); 3814 last_sector = 3815 raid5_compute_sector((sector_nr+conf->chunk_size/512) 3816 *(new_data_disks) -1, 3817 raid_disks, data_disks, 3818 &dd_idx, &pd_idx, conf); 3819 if (last_sector >= (mddev->size<<1)) 3820 last_sector = (mddev->size<<1)-1; 3821 while (first_sector <= last_sector) { 3822 pd_idx = stripe_to_pdidx(first_sector, conf, 3823 conf->previous_raid_disks); 3824 sh = get_active_stripe(conf, first_sector, 3825 conf->previous_raid_disks, pd_idx, 0); 3826 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3827 set_bit(STRIPE_HANDLE, &sh->state); 3828 release_stripe(sh); 3829 first_sector += STRIPE_SECTORS; 3830 } 3831 /* If this takes us to the resync_max point where we have to pause, 3832 * then we need to write out the superblock. 3833 */ 3834 sector_nr += conf->chunk_size>>9; 3835 if (sector_nr >= mddev->resync_max) { 3836 /* Cannot proceed until we've updated the superblock... */ 3837 wait_event(conf->wait_for_overlap, 3838 atomic_read(&conf->reshape_stripes) == 0); 3839 mddev->reshape_position = conf->expand_progress; 3840 set_bit(MD_CHANGE_DEVS, &mddev->flags); 3841 md_wakeup_thread(mddev->thread); 3842 wait_event(mddev->sb_wait, 3843 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 3844 || kthread_should_stop()); 3845 spin_lock_irq(&conf->device_lock); 3846 conf->expand_lo = mddev->reshape_position; 3847 spin_unlock_irq(&conf->device_lock); 3848 wake_up(&conf->wait_for_overlap); 3849 } 3850 return conf->chunk_size>>9; 3851 } 3852 3853 /* FIXME go_faster isn't used */ 3854 static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster) 3855 { 3856 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 3857 struct stripe_head *sh; 3858 int pd_idx; 3859 int raid_disks = conf->raid_disks; 3860 sector_t max_sector = mddev->size << 1; 3861 int sync_blocks; 3862 int still_degraded = 0; 3863 int i; 3864 3865 if (sector_nr >= max_sector) { 3866 /* just being told to finish up .. nothing much to do */ 3867 unplug_slaves(mddev); 3868 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 3869 end_reshape(conf); 3870 return 0; 3871 } 3872 3873 if (mddev->curr_resync < max_sector) /* aborted */ 3874 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 3875 &sync_blocks, 1); 3876 else /* completed sync */ 3877 conf->fullsync = 0; 3878 bitmap_close_sync(mddev->bitmap); 3879 3880 return 0; 3881 } 3882 3883 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 3884 return reshape_request(mddev, sector_nr, skipped); 3885 3886 /* No need to check resync_max as we never do more than one 3887 * stripe, and as resync_max will always be on a chunk boundary, 3888 * if the check in md_do_sync didn't fire, there is no chance 3889 * of overstepping resync_max here 3890 */ 3891 3892 /* if there is too many failed drives and we are trying 3893 * to resync, then assert that we are finished, because there is 3894 * nothing we can do. 3895 */ 3896 if (mddev->degraded >= conf->max_degraded && 3897 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 3898 sector_t rv = (mddev->size << 1) - sector_nr; 3899 *skipped = 1; 3900 return rv; 3901 } 3902 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 3903 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 3904 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) { 3905 /* we can skip this block, and probably more */ 3906 sync_blocks /= STRIPE_SECTORS; 3907 *skipped = 1; 3908 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 3909 } 3910 3911 3912 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 3913 3914 pd_idx = stripe_to_pdidx(sector_nr, conf, raid_disks); 3915 sh = get_active_stripe(conf, sector_nr, raid_disks, pd_idx, 1); 3916 if (sh == NULL) { 3917 sh = get_active_stripe(conf, sector_nr, raid_disks, pd_idx, 0); 3918 /* make sure we don't swamp the stripe cache if someone else 3919 * is trying to get access 3920 */ 3921 schedule_timeout_uninterruptible(1); 3922 } 3923 /* Need to check if array will still be degraded after recovery/resync 3924 * We don't need to check the 'failed' flag as when that gets set, 3925 * recovery aborts. 3926 */ 3927 for (i=0; i<mddev->raid_disks; i++) 3928 if (conf->disks[i].rdev == NULL) 3929 still_degraded = 1; 3930 3931 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 3932 3933 spin_lock(&sh->lock); 3934 set_bit(STRIPE_SYNCING, &sh->state); 3935 clear_bit(STRIPE_INSYNC, &sh->state); 3936 spin_unlock(&sh->lock); 3937 3938 handle_stripe(sh, NULL); 3939 release_stripe(sh); 3940 3941 return STRIPE_SECTORS; 3942 } 3943 3944 static int retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio) 3945 { 3946 /* We may not be able to submit a whole bio at once as there 3947 * may not be enough stripe_heads available. 3948 * We cannot pre-allocate enough stripe_heads as we may need 3949 * more than exist in the cache (if we allow ever large chunks). 3950 * So we do one stripe head at a time and record in 3951 * ->bi_hw_segments how many have been done. 3952 * 3953 * We *know* that this entire raid_bio is in one chunk, so 3954 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 3955 */ 3956 struct stripe_head *sh; 3957 int dd_idx, pd_idx; 3958 sector_t sector, logical_sector, last_sector; 3959 int scnt = 0; 3960 int remaining; 3961 int handled = 0; 3962 3963 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 3964 sector = raid5_compute_sector( logical_sector, 3965 conf->raid_disks, 3966 conf->raid_disks - conf->max_degraded, 3967 &dd_idx, 3968 &pd_idx, 3969 conf); 3970 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9); 3971 3972 for (; logical_sector < last_sector; 3973 logical_sector += STRIPE_SECTORS, 3974 sector += STRIPE_SECTORS, 3975 scnt++) { 3976 3977 if (scnt < raid_bio->bi_hw_segments) 3978 /* already done this stripe */ 3979 continue; 3980 3981 sh = get_active_stripe(conf, sector, conf->raid_disks, pd_idx, 1); 3982 3983 if (!sh) { 3984 /* failed to get a stripe - must wait */ 3985 raid_bio->bi_hw_segments = scnt; 3986 conf->retry_read_aligned = raid_bio; 3987 return handled; 3988 } 3989 3990 set_bit(R5_ReadError, &sh->dev[dd_idx].flags); 3991 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 3992 release_stripe(sh); 3993 raid_bio->bi_hw_segments = scnt; 3994 conf->retry_read_aligned = raid_bio; 3995 return handled; 3996 } 3997 3998 handle_stripe(sh, NULL); 3999 release_stripe(sh); 4000 handled++; 4001 } 4002 spin_lock_irq(&conf->device_lock); 4003 remaining = --raid_bio->bi_phys_segments; 4004 spin_unlock_irq(&conf->device_lock); 4005 if (remaining == 0) { 4006 4007 raid_bio->bi_end_io(raid_bio, 4008 test_bit(BIO_UPTODATE, &raid_bio->bi_flags) 4009 ? 0 : -EIO); 4010 } 4011 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4012 wake_up(&conf->wait_for_stripe); 4013 return handled; 4014 } 4015 4016 4017 4018 /* 4019 * This is our raid5 kernel thread. 4020 * 4021 * We scan the hash table for stripes which can be handled now. 4022 * During the scan, completed stripes are saved for us by the interrupt 4023 * handler, so that they will not have to wait for our next wakeup. 4024 */ 4025 static void raid5d(mddev_t *mddev) 4026 { 4027 struct stripe_head *sh; 4028 raid5_conf_t *conf = mddev_to_conf(mddev); 4029 int handled; 4030 4031 pr_debug("+++ raid5d active\n"); 4032 4033 md_check_recovery(mddev); 4034 4035 handled = 0; 4036 spin_lock_irq(&conf->device_lock); 4037 while (1) { 4038 struct bio *bio; 4039 4040 if (conf->seq_flush != conf->seq_write) { 4041 int seq = conf->seq_flush; 4042 spin_unlock_irq(&conf->device_lock); 4043 bitmap_unplug(mddev->bitmap); 4044 spin_lock_irq(&conf->device_lock); 4045 conf->seq_write = seq; 4046 activate_bit_delay(conf); 4047 } 4048 4049 while ((bio = remove_bio_from_retry(conf))) { 4050 int ok; 4051 spin_unlock_irq(&conf->device_lock); 4052 ok = retry_aligned_read(conf, bio); 4053 spin_lock_irq(&conf->device_lock); 4054 if (!ok) 4055 break; 4056 handled++; 4057 } 4058 4059 sh = __get_priority_stripe(conf); 4060 4061 if (!sh) { 4062 async_tx_issue_pending_all(); 4063 break; 4064 } 4065 spin_unlock_irq(&conf->device_lock); 4066 4067 handled++; 4068 handle_stripe(sh, conf->spare_page); 4069 release_stripe(sh); 4070 4071 spin_lock_irq(&conf->device_lock); 4072 } 4073 pr_debug("%d stripes handled\n", handled); 4074 4075 spin_unlock_irq(&conf->device_lock); 4076 4077 unplug_slaves(mddev); 4078 4079 pr_debug("--- raid5d inactive\n"); 4080 } 4081 4082 static ssize_t 4083 raid5_show_stripe_cache_size(mddev_t *mddev, char *page) 4084 { 4085 raid5_conf_t *conf = mddev_to_conf(mddev); 4086 if (conf) 4087 return sprintf(page, "%d\n", conf->max_nr_stripes); 4088 else 4089 return 0; 4090 } 4091 4092 static ssize_t 4093 raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len) 4094 { 4095 raid5_conf_t *conf = mddev_to_conf(mddev); 4096 unsigned long new; 4097 if (len >= PAGE_SIZE) 4098 return -EINVAL; 4099 if (!conf) 4100 return -ENODEV; 4101 4102 if (strict_strtoul(page, 10, &new)) 4103 return -EINVAL; 4104 if (new <= 16 || new > 32768) 4105 return -EINVAL; 4106 while (new < conf->max_nr_stripes) { 4107 if (drop_one_stripe(conf)) 4108 conf->max_nr_stripes--; 4109 else 4110 break; 4111 } 4112 md_allow_write(mddev); 4113 while (new > conf->max_nr_stripes) { 4114 if (grow_one_stripe(conf)) 4115 conf->max_nr_stripes++; 4116 else break; 4117 } 4118 return len; 4119 } 4120 4121 static struct md_sysfs_entry 4122 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 4123 raid5_show_stripe_cache_size, 4124 raid5_store_stripe_cache_size); 4125 4126 static ssize_t 4127 raid5_show_preread_threshold(mddev_t *mddev, char *page) 4128 { 4129 raid5_conf_t *conf = mddev_to_conf(mddev); 4130 if (conf) 4131 return sprintf(page, "%d\n", conf->bypass_threshold); 4132 else 4133 return 0; 4134 } 4135 4136 static ssize_t 4137 raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len) 4138 { 4139 raid5_conf_t *conf = mddev_to_conf(mddev); 4140 unsigned long new; 4141 if (len >= PAGE_SIZE) 4142 return -EINVAL; 4143 if (!conf) 4144 return -ENODEV; 4145 4146 if (strict_strtoul(page, 10, &new)) 4147 return -EINVAL; 4148 if (new > conf->max_nr_stripes) 4149 return -EINVAL; 4150 conf->bypass_threshold = new; 4151 return len; 4152 } 4153 4154 static struct md_sysfs_entry 4155 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 4156 S_IRUGO | S_IWUSR, 4157 raid5_show_preread_threshold, 4158 raid5_store_preread_threshold); 4159 4160 static ssize_t 4161 stripe_cache_active_show(mddev_t *mddev, char *page) 4162 { 4163 raid5_conf_t *conf = mddev_to_conf(mddev); 4164 if (conf) 4165 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 4166 else 4167 return 0; 4168 } 4169 4170 static struct md_sysfs_entry 4171 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 4172 4173 static struct attribute *raid5_attrs[] = { 4174 &raid5_stripecache_size.attr, 4175 &raid5_stripecache_active.attr, 4176 &raid5_preread_bypass_threshold.attr, 4177 NULL, 4178 }; 4179 static struct attribute_group raid5_attrs_group = { 4180 .name = NULL, 4181 .attrs = raid5_attrs, 4182 }; 4183 4184 static int run(mddev_t *mddev) 4185 { 4186 raid5_conf_t *conf; 4187 int raid_disk, memory; 4188 mdk_rdev_t *rdev; 4189 struct disk_info *disk; 4190 struct list_head *tmp; 4191 int working_disks = 0; 4192 4193 if (mddev->level != 5 && mddev->level != 4 && mddev->level != 6) { 4194 printk(KERN_ERR "raid5: %s: raid level not set to 4/5/6 (%d)\n", 4195 mdname(mddev), mddev->level); 4196 return -EIO; 4197 } 4198 4199 if (mddev->reshape_position != MaxSector) { 4200 /* Check that we can continue the reshape. 4201 * Currently only disks can change, it must 4202 * increase, and we must be past the point where 4203 * a stripe over-writes itself 4204 */ 4205 sector_t here_new, here_old; 4206 int old_disks; 4207 int max_degraded = (mddev->level == 5 ? 1 : 2); 4208 4209 if (mddev->new_level != mddev->level || 4210 mddev->new_layout != mddev->layout || 4211 mddev->new_chunk != mddev->chunk_size) { 4212 printk(KERN_ERR "raid5: %s: unsupported reshape " 4213 "required - aborting.\n", 4214 mdname(mddev)); 4215 return -EINVAL; 4216 } 4217 if (mddev->delta_disks <= 0) { 4218 printk(KERN_ERR "raid5: %s: unsupported reshape " 4219 "(reduce disks) required - aborting.\n", 4220 mdname(mddev)); 4221 return -EINVAL; 4222 } 4223 old_disks = mddev->raid_disks - mddev->delta_disks; 4224 /* reshape_position must be on a new-stripe boundary, and one 4225 * further up in new geometry must map after here in old 4226 * geometry. 4227 */ 4228 here_new = mddev->reshape_position; 4229 if (sector_div(here_new, (mddev->chunk_size>>9)* 4230 (mddev->raid_disks - max_degraded))) { 4231 printk(KERN_ERR "raid5: reshape_position not " 4232 "on a stripe boundary\n"); 4233 return -EINVAL; 4234 } 4235 /* here_new is the stripe we will write to */ 4236 here_old = mddev->reshape_position; 4237 sector_div(here_old, (mddev->chunk_size>>9)* 4238 (old_disks-max_degraded)); 4239 /* here_old is the first stripe that we might need to read 4240 * from */ 4241 if (here_new >= here_old) { 4242 /* Reading from the same stripe as writing to - bad */ 4243 printk(KERN_ERR "raid5: reshape_position too early for " 4244 "auto-recovery - aborting.\n"); 4245 return -EINVAL; 4246 } 4247 printk(KERN_INFO "raid5: reshape will continue\n"); 4248 /* OK, we should be able to continue; */ 4249 } 4250 4251 4252 mddev->private = kzalloc(sizeof (raid5_conf_t), GFP_KERNEL); 4253 if ((conf = mddev->private) == NULL) 4254 goto abort; 4255 if (mddev->reshape_position == MaxSector) { 4256 conf->previous_raid_disks = conf->raid_disks = mddev->raid_disks; 4257 } else { 4258 conf->raid_disks = mddev->raid_disks; 4259 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 4260 } 4261 4262 conf->disks = kzalloc(conf->raid_disks * sizeof(struct disk_info), 4263 GFP_KERNEL); 4264 if (!conf->disks) 4265 goto abort; 4266 4267 conf->mddev = mddev; 4268 4269 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 4270 goto abort; 4271 4272 if (mddev->level == 6) { 4273 conf->spare_page = alloc_page(GFP_KERNEL); 4274 if (!conf->spare_page) 4275 goto abort; 4276 } 4277 spin_lock_init(&conf->device_lock); 4278 mddev->queue->queue_lock = &conf->device_lock; 4279 init_waitqueue_head(&conf->wait_for_stripe); 4280 init_waitqueue_head(&conf->wait_for_overlap); 4281 INIT_LIST_HEAD(&conf->handle_list); 4282 INIT_LIST_HEAD(&conf->hold_list); 4283 INIT_LIST_HEAD(&conf->delayed_list); 4284 INIT_LIST_HEAD(&conf->bitmap_list); 4285 INIT_LIST_HEAD(&conf->inactive_list); 4286 atomic_set(&conf->active_stripes, 0); 4287 atomic_set(&conf->preread_active_stripes, 0); 4288 atomic_set(&conf->active_aligned_reads, 0); 4289 conf->bypass_threshold = BYPASS_THRESHOLD; 4290 4291 pr_debug("raid5: run(%s) called.\n", mdname(mddev)); 4292 4293 rdev_for_each(rdev, tmp, mddev) { 4294 raid_disk = rdev->raid_disk; 4295 if (raid_disk >= conf->raid_disks 4296 || raid_disk < 0) 4297 continue; 4298 disk = conf->disks + raid_disk; 4299 4300 disk->rdev = rdev; 4301 4302 if (test_bit(In_sync, &rdev->flags)) { 4303 char b[BDEVNAME_SIZE]; 4304 printk(KERN_INFO "raid5: device %s operational as raid" 4305 " disk %d\n", bdevname(rdev->bdev,b), 4306 raid_disk); 4307 working_disks++; 4308 } 4309 } 4310 4311 /* 4312 * 0 for a fully functional array, 1 or 2 for a degraded array. 4313 */ 4314 mddev->degraded = conf->raid_disks - working_disks; 4315 conf->mddev = mddev; 4316 conf->chunk_size = mddev->chunk_size; 4317 conf->level = mddev->level; 4318 if (conf->level == 6) 4319 conf->max_degraded = 2; 4320 else 4321 conf->max_degraded = 1; 4322 conf->algorithm = mddev->layout; 4323 conf->max_nr_stripes = NR_STRIPES; 4324 conf->expand_progress = mddev->reshape_position; 4325 4326 /* device size must be a multiple of chunk size */ 4327 mddev->size &= ~(mddev->chunk_size/1024 -1); 4328 mddev->resync_max_sectors = mddev->size << 1; 4329 4330 if (conf->level == 6 && conf->raid_disks < 4) { 4331 printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n", 4332 mdname(mddev), conf->raid_disks); 4333 goto abort; 4334 } 4335 if (!conf->chunk_size || conf->chunk_size % 4) { 4336 printk(KERN_ERR "raid5: invalid chunk size %d for %s\n", 4337 conf->chunk_size, mdname(mddev)); 4338 goto abort; 4339 } 4340 if (conf->algorithm > ALGORITHM_RIGHT_SYMMETRIC) { 4341 printk(KERN_ERR 4342 "raid5: unsupported parity algorithm %d for %s\n", 4343 conf->algorithm, mdname(mddev)); 4344 goto abort; 4345 } 4346 if (mddev->degraded > conf->max_degraded) { 4347 printk(KERN_ERR "raid5: not enough operational devices for %s" 4348 " (%d/%d failed)\n", 4349 mdname(mddev), mddev->degraded, conf->raid_disks); 4350 goto abort; 4351 } 4352 4353 if (mddev->degraded > 0 && 4354 mddev->recovery_cp != MaxSector) { 4355 if (mddev->ok_start_degraded) 4356 printk(KERN_WARNING 4357 "raid5: starting dirty degraded array: %s" 4358 "- data corruption possible.\n", 4359 mdname(mddev)); 4360 else { 4361 printk(KERN_ERR 4362 "raid5: cannot start dirty degraded array for %s\n", 4363 mdname(mddev)); 4364 goto abort; 4365 } 4366 } 4367 4368 { 4369 mddev->thread = md_register_thread(raid5d, mddev, "%s_raid5"); 4370 if (!mddev->thread) { 4371 printk(KERN_ERR 4372 "raid5: couldn't allocate thread for %s\n", 4373 mdname(mddev)); 4374 goto abort; 4375 } 4376 } 4377 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 4378 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 4379 if (grow_stripes(conf, conf->max_nr_stripes)) { 4380 printk(KERN_ERR 4381 "raid5: couldn't allocate %dkB for buffers\n", memory); 4382 shrink_stripes(conf); 4383 md_unregister_thread(mddev->thread); 4384 goto abort; 4385 } else 4386 printk(KERN_INFO "raid5: allocated %dkB for %s\n", 4387 memory, mdname(mddev)); 4388 4389 if (mddev->degraded == 0) 4390 printk("raid5: raid level %d set %s active with %d out of %d" 4391 " devices, algorithm %d\n", conf->level, mdname(mddev), 4392 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 4393 conf->algorithm); 4394 else 4395 printk(KERN_ALERT "raid5: raid level %d set %s active with %d" 4396 " out of %d devices, algorithm %d\n", conf->level, 4397 mdname(mddev), mddev->raid_disks - mddev->degraded, 4398 mddev->raid_disks, conf->algorithm); 4399 4400 print_raid5_conf(conf); 4401 4402 if (conf->expand_progress != MaxSector) { 4403 printk("...ok start reshape thread\n"); 4404 conf->expand_lo = conf->expand_progress; 4405 atomic_set(&conf->reshape_stripes, 0); 4406 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4407 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4408 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4409 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4410 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4411 "%s_reshape"); 4412 } 4413 4414 /* read-ahead size must cover two whole stripes, which is 4415 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4416 */ 4417 { 4418 int data_disks = conf->previous_raid_disks - conf->max_degraded; 4419 int stripe = data_disks * 4420 (mddev->chunk_size / PAGE_SIZE); 4421 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4422 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4423 } 4424 4425 /* Ok, everything is just fine now */ 4426 if (sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 4427 printk(KERN_WARNING 4428 "raid5: failed to create sysfs attributes for %s\n", 4429 mdname(mddev)); 4430 4431 mddev->queue->unplug_fn = raid5_unplug_device; 4432 mddev->queue->backing_dev_info.congested_data = mddev; 4433 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 4434 4435 mddev->array_size = mddev->size * (conf->previous_raid_disks - 4436 conf->max_degraded); 4437 4438 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 4439 4440 return 0; 4441 abort: 4442 if (conf) { 4443 print_raid5_conf(conf); 4444 safe_put_page(conf->spare_page); 4445 kfree(conf->disks); 4446 kfree(conf->stripe_hashtbl); 4447 kfree(conf); 4448 } 4449 mddev->private = NULL; 4450 printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev)); 4451 return -EIO; 4452 } 4453 4454 4455 4456 static int stop(mddev_t *mddev) 4457 { 4458 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 4459 4460 md_unregister_thread(mddev->thread); 4461 mddev->thread = NULL; 4462 shrink_stripes(conf); 4463 kfree(conf->stripe_hashtbl); 4464 mddev->queue->backing_dev_info.congested_fn = NULL; 4465 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/ 4466 sysfs_remove_group(&mddev->kobj, &raid5_attrs_group); 4467 kfree(conf->disks); 4468 kfree(conf); 4469 mddev->private = NULL; 4470 return 0; 4471 } 4472 4473 #ifdef DEBUG 4474 static void print_sh (struct seq_file *seq, struct stripe_head *sh) 4475 { 4476 int i; 4477 4478 seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n", 4479 (unsigned long long)sh->sector, sh->pd_idx, sh->state); 4480 seq_printf(seq, "sh %llu, count %d.\n", 4481 (unsigned long long)sh->sector, atomic_read(&sh->count)); 4482 seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector); 4483 for (i = 0; i < sh->disks; i++) { 4484 seq_printf(seq, "(cache%d: %p %ld) ", 4485 i, sh->dev[i].page, sh->dev[i].flags); 4486 } 4487 seq_printf(seq, "\n"); 4488 } 4489 4490 static void printall (struct seq_file *seq, raid5_conf_t *conf) 4491 { 4492 struct stripe_head *sh; 4493 struct hlist_node *hn; 4494 int i; 4495 4496 spin_lock_irq(&conf->device_lock); 4497 for (i = 0; i < NR_HASH; i++) { 4498 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) { 4499 if (sh->raid_conf != conf) 4500 continue; 4501 print_sh(seq, sh); 4502 } 4503 } 4504 spin_unlock_irq(&conf->device_lock); 4505 } 4506 #endif 4507 4508 static void status (struct seq_file *seq, mddev_t *mddev) 4509 { 4510 raid5_conf_t *conf = (raid5_conf_t *) mddev->private; 4511 int i; 4512 4513 seq_printf (seq, " level %d, %dk chunk, algorithm %d", mddev->level, mddev->chunk_size >> 10, mddev->layout); 4514 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 4515 for (i = 0; i < conf->raid_disks; i++) 4516 seq_printf (seq, "%s", 4517 conf->disks[i].rdev && 4518 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 4519 seq_printf (seq, "]"); 4520 #ifdef DEBUG 4521 seq_printf (seq, "\n"); 4522 printall(seq, conf); 4523 #endif 4524 } 4525 4526 static void print_raid5_conf (raid5_conf_t *conf) 4527 { 4528 int i; 4529 struct disk_info *tmp; 4530 4531 printk("RAID5 conf printout:\n"); 4532 if (!conf) { 4533 printk("(conf==NULL)\n"); 4534 return; 4535 } 4536 printk(" --- rd:%d wd:%d\n", conf->raid_disks, 4537 conf->raid_disks - conf->mddev->degraded); 4538 4539 for (i = 0; i < conf->raid_disks; i++) { 4540 char b[BDEVNAME_SIZE]; 4541 tmp = conf->disks + i; 4542 if (tmp->rdev) 4543 printk(" disk %d, o:%d, dev:%s\n", 4544 i, !test_bit(Faulty, &tmp->rdev->flags), 4545 bdevname(tmp->rdev->bdev,b)); 4546 } 4547 } 4548 4549 static int raid5_spare_active(mddev_t *mddev) 4550 { 4551 int i; 4552 raid5_conf_t *conf = mddev->private; 4553 struct disk_info *tmp; 4554 4555 for (i = 0; i < conf->raid_disks; i++) { 4556 tmp = conf->disks + i; 4557 if (tmp->rdev 4558 && !test_bit(Faulty, &tmp->rdev->flags) 4559 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 4560 unsigned long flags; 4561 spin_lock_irqsave(&conf->device_lock, flags); 4562 mddev->degraded--; 4563 spin_unlock_irqrestore(&conf->device_lock, flags); 4564 } 4565 } 4566 print_raid5_conf(conf); 4567 return 0; 4568 } 4569 4570 static int raid5_remove_disk(mddev_t *mddev, int number) 4571 { 4572 raid5_conf_t *conf = mddev->private; 4573 int err = 0; 4574 mdk_rdev_t *rdev; 4575 struct disk_info *p = conf->disks + number; 4576 4577 print_raid5_conf(conf); 4578 rdev = p->rdev; 4579 if (rdev) { 4580 if (test_bit(In_sync, &rdev->flags) || 4581 atomic_read(&rdev->nr_pending)) { 4582 err = -EBUSY; 4583 goto abort; 4584 } 4585 /* Only remove non-faulty devices if recovery 4586 * isn't possible. 4587 */ 4588 if (!test_bit(Faulty, &rdev->flags) && 4589 mddev->degraded <= conf->max_degraded) { 4590 err = -EBUSY; 4591 goto abort; 4592 } 4593 p->rdev = NULL; 4594 synchronize_rcu(); 4595 if (atomic_read(&rdev->nr_pending)) { 4596 /* lost the race, try later */ 4597 err = -EBUSY; 4598 p->rdev = rdev; 4599 } 4600 } 4601 abort: 4602 4603 print_raid5_conf(conf); 4604 return err; 4605 } 4606 4607 static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev) 4608 { 4609 raid5_conf_t *conf = mddev->private; 4610 int found = 0; 4611 int disk; 4612 struct disk_info *p; 4613 4614 if (mddev->degraded > conf->max_degraded) 4615 /* no point adding a device */ 4616 return 0; 4617 4618 /* 4619 * find the disk ... but prefer rdev->saved_raid_disk 4620 * if possible. 4621 */ 4622 if (rdev->saved_raid_disk >= 0 && 4623 conf->disks[rdev->saved_raid_disk].rdev == NULL) 4624 disk = rdev->saved_raid_disk; 4625 else 4626 disk = 0; 4627 for ( ; disk < conf->raid_disks; disk++) 4628 if ((p=conf->disks + disk)->rdev == NULL) { 4629 clear_bit(In_sync, &rdev->flags); 4630 rdev->raid_disk = disk; 4631 found = 1; 4632 if (rdev->saved_raid_disk != disk) 4633 conf->fullsync = 1; 4634 rcu_assign_pointer(p->rdev, rdev); 4635 break; 4636 } 4637 print_raid5_conf(conf); 4638 return found; 4639 } 4640 4641 static int raid5_resize(mddev_t *mddev, sector_t sectors) 4642 { 4643 /* no resync is happening, and there is enough space 4644 * on all devices, so we can resize. 4645 * We need to make sure resync covers any new space. 4646 * If the array is shrinking we should possibly wait until 4647 * any io in the removed space completes, but it hardly seems 4648 * worth it. 4649 */ 4650 raid5_conf_t *conf = mddev_to_conf(mddev); 4651 4652 sectors &= ~((sector_t)mddev->chunk_size/512 - 1); 4653 mddev->array_size = (sectors * (mddev->raid_disks-conf->max_degraded))>>1; 4654 set_capacity(mddev->gendisk, mddev->array_size << 1); 4655 mddev->changed = 1; 4656 if (sectors/2 > mddev->size && mddev->recovery_cp == MaxSector) { 4657 mddev->recovery_cp = mddev->size << 1; 4658 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 4659 } 4660 mddev->size = sectors /2; 4661 mddev->resync_max_sectors = sectors; 4662 return 0; 4663 } 4664 4665 #ifdef CONFIG_MD_RAID5_RESHAPE 4666 static int raid5_check_reshape(mddev_t *mddev) 4667 { 4668 raid5_conf_t *conf = mddev_to_conf(mddev); 4669 int err; 4670 4671 if (mddev->delta_disks < 0 || 4672 mddev->new_level != mddev->level) 4673 return -EINVAL; /* Cannot shrink array or change level yet */ 4674 if (mddev->delta_disks == 0) 4675 return 0; /* nothing to do */ 4676 4677 /* Can only proceed if there are plenty of stripe_heads. 4678 * We need a minimum of one full stripe,, and for sensible progress 4679 * it is best to have about 4 times that. 4680 * If we require 4 times, then the default 256 4K stripe_heads will 4681 * allow for chunk sizes up to 256K, which is probably OK. 4682 * If the chunk size is greater, user-space should request more 4683 * stripe_heads first. 4684 */ 4685 if ((mddev->chunk_size / STRIPE_SIZE) * 4 > conf->max_nr_stripes || 4686 (mddev->new_chunk / STRIPE_SIZE) * 4 > conf->max_nr_stripes) { 4687 printk(KERN_WARNING "raid5: reshape: not enough stripes. Needed %lu\n", 4688 (mddev->chunk_size / STRIPE_SIZE)*4); 4689 return -ENOSPC; 4690 } 4691 4692 err = resize_stripes(conf, conf->raid_disks + mddev->delta_disks); 4693 if (err) 4694 return err; 4695 4696 if (mddev->degraded > conf->max_degraded) 4697 return -EINVAL; 4698 /* looks like we might be able to manage this */ 4699 return 0; 4700 } 4701 4702 static int raid5_start_reshape(mddev_t *mddev) 4703 { 4704 raid5_conf_t *conf = mddev_to_conf(mddev); 4705 mdk_rdev_t *rdev; 4706 struct list_head *rtmp; 4707 int spares = 0; 4708 int added_devices = 0; 4709 unsigned long flags; 4710 4711 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 4712 return -EBUSY; 4713 4714 rdev_for_each(rdev, rtmp, mddev) 4715 if (rdev->raid_disk < 0 && 4716 !test_bit(Faulty, &rdev->flags)) 4717 spares++; 4718 4719 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 4720 /* Not enough devices even to make a degraded array 4721 * of that size 4722 */ 4723 return -EINVAL; 4724 4725 atomic_set(&conf->reshape_stripes, 0); 4726 spin_lock_irq(&conf->device_lock); 4727 conf->previous_raid_disks = conf->raid_disks; 4728 conf->raid_disks += mddev->delta_disks; 4729 conf->expand_progress = 0; 4730 conf->expand_lo = 0; 4731 spin_unlock_irq(&conf->device_lock); 4732 4733 /* Add some new drives, as many as will fit. 4734 * We know there are enough to make the newly sized array work. 4735 */ 4736 rdev_for_each(rdev, rtmp, mddev) 4737 if (rdev->raid_disk < 0 && 4738 !test_bit(Faulty, &rdev->flags)) { 4739 if (raid5_add_disk(mddev, rdev)) { 4740 char nm[20]; 4741 set_bit(In_sync, &rdev->flags); 4742 added_devices++; 4743 rdev->recovery_offset = 0; 4744 sprintf(nm, "rd%d", rdev->raid_disk); 4745 if (sysfs_create_link(&mddev->kobj, 4746 &rdev->kobj, nm)) 4747 printk(KERN_WARNING 4748 "raid5: failed to create " 4749 " link %s for %s\n", 4750 nm, mdname(mddev)); 4751 } else 4752 break; 4753 } 4754 4755 spin_lock_irqsave(&conf->device_lock, flags); 4756 mddev->degraded = (conf->raid_disks - conf->previous_raid_disks) - added_devices; 4757 spin_unlock_irqrestore(&conf->device_lock, flags); 4758 mddev->raid_disks = conf->raid_disks; 4759 mddev->reshape_position = 0; 4760 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4761 4762 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 4763 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 4764 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 4765 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 4766 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 4767 "%s_reshape"); 4768 if (!mddev->sync_thread) { 4769 mddev->recovery = 0; 4770 spin_lock_irq(&conf->device_lock); 4771 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 4772 conf->expand_progress = MaxSector; 4773 spin_unlock_irq(&conf->device_lock); 4774 return -EAGAIN; 4775 } 4776 md_wakeup_thread(mddev->sync_thread); 4777 md_new_event(mddev); 4778 return 0; 4779 } 4780 #endif 4781 4782 static void end_reshape(raid5_conf_t *conf) 4783 { 4784 struct block_device *bdev; 4785 4786 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 4787 conf->mddev->array_size = conf->mddev->size * 4788 (conf->raid_disks - conf->max_degraded); 4789 set_capacity(conf->mddev->gendisk, conf->mddev->array_size << 1); 4790 conf->mddev->changed = 1; 4791 4792 bdev = bdget_disk(conf->mddev->gendisk, 0); 4793 if (bdev) { 4794 mutex_lock(&bdev->bd_inode->i_mutex); 4795 i_size_write(bdev->bd_inode, (loff_t)conf->mddev->array_size << 10); 4796 mutex_unlock(&bdev->bd_inode->i_mutex); 4797 bdput(bdev); 4798 } 4799 spin_lock_irq(&conf->device_lock); 4800 conf->expand_progress = MaxSector; 4801 spin_unlock_irq(&conf->device_lock); 4802 conf->mddev->reshape_position = MaxSector; 4803 4804 /* read-ahead size must cover two whole stripes, which is 4805 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 4806 */ 4807 { 4808 int data_disks = conf->previous_raid_disks - conf->max_degraded; 4809 int stripe = data_disks * 4810 (conf->mddev->chunk_size / PAGE_SIZE); 4811 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 4812 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 4813 } 4814 } 4815 } 4816 4817 static void raid5_quiesce(mddev_t *mddev, int state) 4818 { 4819 raid5_conf_t *conf = mddev_to_conf(mddev); 4820 4821 switch(state) { 4822 case 2: /* resume for a suspend */ 4823 wake_up(&conf->wait_for_overlap); 4824 break; 4825 4826 case 1: /* stop all writes */ 4827 spin_lock_irq(&conf->device_lock); 4828 conf->quiesce = 1; 4829 wait_event_lock_irq(conf->wait_for_stripe, 4830 atomic_read(&conf->active_stripes) == 0 && 4831 atomic_read(&conf->active_aligned_reads) == 0, 4832 conf->device_lock, /* nothing */); 4833 spin_unlock_irq(&conf->device_lock); 4834 break; 4835 4836 case 0: /* re-enable writes */ 4837 spin_lock_irq(&conf->device_lock); 4838 conf->quiesce = 0; 4839 wake_up(&conf->wait_for_stripe); 4840 wake_up(&conf->wait_for_overlap); 4841 spin_unlock_irq(&conf->device_lock); 4842 break; 4843 } 4844 } 4845 4846 static struct mdk_personality raid6_personality = 4847 { 4848 .name = "raid6", 4849 .level = 6, 4850 .owner = THIS_MODULE, 4851 .make_request = make_request, 4852 .run = run, 4853 .stop = stop, 4854 .status = status, 4855 .error_handler = error, 4856 .hot_add_disk = raid5_add_disk, 4857 .hot_remove_disk= raid5_remove_disk, 4858 .spare_active = raid5_spare_active, 4859 .sync_request = sync_request, 4860 .resize = raid5_resize, 4861 #ifdef CONFIG_MD_RAID5_RESHAPE 4862 .check_reshape = raid5_check_reshape, 4863 .start_reshape = raid5_start_reshape, 4864 #endif 4865 .quiesce = raid5_quiesce, 4866 }; 4867 static struct mdk_personality raid5_personality = 4868 { 4869 .name = "raid5", 4870 .level = 5, 4871 .owner = THIS_MODULE, 4872 .make_request = make_request, 4873 .run = run, 4874 .stop = stop, 4875 .status = status, 4876 .error_handler = error, 4877 .hot_add_disk = raid5_add_disk, 4878 .hot_remove_disk= raid5_remove_disk, 4879 .spare_active = raid5_spare_active, 4880 .sync_request = sync_request, 4881 .resize = raid5_resize, 4882 #ifdef CONFIG_MD_RAID5_RESHAPE 4883 .check_reshape = raid5_check_reshape, 4884 .start_reshape = raid5_start_reshape, 4885 #endif 4886 .quiesce = raid5_quiesce, 4887 }; 4888 4889 static struct mdk_personality raid4_personality = 4890 { 4891 .name = "raid4", 4892 .level = 4, 4893 .owner = THIS_MODULE, 4894 .make_request = make_request, 4895 .run = run, 4896 .stop = stop, 4897 .status = status, 4898 .error_handler = error, 4899 .hot_add_disk = raid5_add_disk, 4900 .hot_remove_disk= raid5_remove_disk, 4901 .spare_active = raid5_spare_active, 4902 .sync_request = sync_request, 4903 .resize = raid5_resize, 4904 #ifdef CONFIG_MD_RAID5_RESHAPE 4905 .check_reshape = raid5_check_reshape, 4906 .start_reshape = raid5_start_reshape, 4907 #endif 4908 .quiesce = raid5_quiesce, 4909 }; 4910 4911 static int __init raid5_init(void) 4912 { 4913 int e; 4914 4915 e = raid6_select_algo(); 4916 if ( e ) 4917 return e; 4918 register_md_personality(&raid6_personality); 4919 register_md_personality(&raid5_personality); 4920 register_md_personality(&raid4_personality); 4921 return 0; 4922 } 4923 4924 static void raid5_exit(void) 4925 { 4926 unregister_md_personality(&raid6_personality); 4927 unregister_md_personality(&raid5_personality); 4928 unregister_md_personality(&raid4_personality); 4929 } 4930 4931 module_init(raid5_init); 4932 module_exit(raid5_exit); 4933 MODULE_LICENSE("GPL"); 4934 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 4935 MODULE_ALIAS("md-raid5"); 4936 MODULE_ALIAS("md-raid4"); 4937 MODULE_ALIAS("md-level-5"); 4938 MODULE_ALIAS("md-level-4"); 4939 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 4940 MODULE_ALIAS("md-raid6"); 4941 MODULE_ALIAS("md-level-6"); 4942 4943 /* This used to be two separate modules, they were: */ 4944 MODULE_ALIAS("raid5"); 4945 MODULE_ALIAS("raid6"); 4946