1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include <linux/nodemask.h> 57 #include <trace/events/block.h> 58 59 #include "md.h" 60 #include "raid5.h" 61 #include "raid0.h" 62 #include "bitmap.h" 63 64 #define cpu_to_group(cpu) cpu_to_node(cpu) 65 #define ANY_GROUP NUMA_NO_NODE 66 67 static struct workqueue_struct *raid5_wq; 68 /* 69 * Stripe cache 70 */ 71 72 #define NR_STRIPES 256 73 #define STRIPE_SIZE PAGE_SIZE 74 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 75 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 76 #define IO_THRESHOLD 1 77 #define BYPASS_THRESHOLD 1 78 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 79 #define HASH_MASK (NR_HASH - 1) 80 #define MAX_STRIPE_BATCH 8 81 82 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 83 { 84 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 85 return &conf->stripe_hashtbl[hash]; 86 } 87 88 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 89 * order without overlap. There may be several bio's per stripe+device, and 90 * a bio could span several devices. 91 * When walking this list for a particular stripe+device, we must never proceed 92 * beyond a bio that extends past this device, as the next bio might no longer 93 * be valid. 94 * This function is used to determine the 'next' bio in the list, given the sector 95 * of the current stripe+device 96 */ 97 static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) 98 { 99 int sectors = bio_sectors(bio); 100 if (bio->bi_sector + sectors < sector + STRIPE_SECTORS) 101 return bio->bi_next; 102 else 103 return NULL; 104 } 105 106 /* 107 * We maintain a biased count of active stripes in the bottom 16 bits of 108 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 109 */ 110 static inline int raid5_bi_processed_stripes(struct bio *bio) 111 { 112 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 113 return (atomic_read(segments) >> 16) & 0xffff; 114 } 115 116 static inline int raid5_dec_bi_active_stripes(struct bio *bio) 117 { 118 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 119 return atomic_sub_return(1, segments) & 0xffff; 120 } 121 122 static inline void raid5_inc_bi_active_stripes(struct bio *bio) 123 { 124 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 125 atomic_inc(segments); 126 } 127 128 static inline void raid5_set_bi_processed_stripes(struct bio *bio, 129 unsigned int cnt) 130 { 131 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 132 int old, new; 133 134 do { 135 old = atomic_read(segments); 136 new = (old & 0xffff) | (cnt << 16); 137 } while (atomic_cmpxchg(segments, old, new) != old); 138 } 139 140 static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) 141 { 142 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 143 atomic_set(segments, cnt); 144 } 145 146 /* Find first data disk in a raid6 stripe */ 147 static inline int raid6_d0(struct stripe_head *sh) 148 { 149 if (sh->ddf_layout) 150 /* ddf always start from first device */ 151 return 0; 152 /* md starts just after Q block */ 153 if (sh->qd_idx == sh->disks - 1) 154 return 0; 155 else 156 return sh->qd_idx + 1; 157 } 158 static inline int raid6_next_disk(int disk, int raid_disks) 159 { 160 disk++; 161 return (disk < raid_disks) ? disk : 0; 162 } 163 164 /* When walking through the disks in a raid5, starting at raid6_d0, 165 * We need to map each disk to a 'slot', where the data disks are slot 166 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 167 * is raid_disks-1. This help does that mapping. 168 */ 169 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 170 int *count, int syndrome_disks) 171 { 172 int slot = *count; 173 174 if (sh->ddf_layout) 175 (*count)++; 176 if (idx == sh->pd_idx) 177 return syndrome_disks; 178 if (idx == sh->qd_idx) 179 return syndrome_disks + 1; 180 if (!sh->ddf_layout) 181 (*count)++; 182 return slot; 183 } 184 185 static void return_io(struct bio *return_bi) 186 { 187 struct bio *bi = return_bi; 188 while (bi) { 189 190 return_bi = bi->bi_next; 191 bi->bi_next = NULL; 192 bi->bi_size = 0; 193 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 194 bi, 0); 195 bio_endio(bi, 0); 196 bi = return_bi; 197 } 198 } 199 200 static void print_raid5_conf (struct r5conf *conf); 201 202 static int stripe_operations_active(struct stripe_head *sh) 203 { 204 return sh->check_state || sh->reconstruct_state || 205 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 206 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 207 } 208 209 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) 210 { 211 struct r5conf *conf = sh->raid_conf; 212 struct r5worker_group *group; 213 int thread_cnt; 214 int i, cpu = sh->cpu; 215 216 if (!cpu_online(cpu)) { 217 cpu = cpumask_any(cpu_online_mask); 218 sh->cpu = cpu; 219 } 220 221 if (list_empty(&sh->lru)) { 222 struct r5worker_group *group; 223 group = conf->worker_groups + cpu_to_group(cpu); 224 list_add_tail(&sh->lru, &group->handle_list); 225 group->stripes_cnt++; 226 sh->group = group; 227 } 228 229 if (conf->worker_cnt_per_group == 0) { 230 md_wakeup_thread(conf->mddev->thread); 231 return; 232 } 233 234 group = conf->worker_groups + cpu_to_group(sh->cpu); 235 236 group->workers[0].working = true; 237 /* at least one worker should run to avoid race */ 238 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); 239 240 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; 241 /* wakeup more workers */ 242 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { 243 if (group->workers[i].working == false) { 244 group->workers[i].working = true; 245 queue_work_on(sh->cpu, raid5_wq, 246 &group->workers[i].work); 247 thread_cnt--; 248 } 249 } 250 } 251 252 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh) 253 { 254 BUG_ON(!list_empty(&sh->lru)); 255 BUG_ON(atomic_read(&conf->active_stripes)==0); 256 if (test_bit(STRIPE_HANDLE, &sh->state)) { 257 if (test_bit(STRIPE_DELAYED, &sh->state) && 258 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 259 list_add_tail(&sh->lru, &conf->delayed_list); 260 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 261 sh->bm_seq - conf->seq_write > 0) 262 list_add_tail(&sh->lru, &conf->bitmap_list); 263 else { 264 clear_bit(STRIPE_DELAYED, &sh->state); 265 clear_bit(STRIPE_BIT_DELAY, &sh->state); 266 if (conf->worker_cnt_per_group == 0) { 267 list_add_tail(&sh->lru, &conf->handle_list); 268 } else { 269 raid5_wakeup_stripe_thread(sh); 270 return; 271 } 272 } 273 md_wakeup_thread(conf->mddev->thread); 274 } else { 275 BUG_ON(stripe_operations_active(sh)); 276 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 277 if (atomic_dec_return(&conf->preread_active_stripes) 278 < IO_THRESHOLD) 279 md_wakeup_thread(conf->mddev->thread); 280 atomic_dec(&conf->active_stripes); 281 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 282 list_add_tail(&sh->lru, &conf->inactive_list); 283 wake_up(&conf->wait_for_stripe); 284 if (conf->retry_read_aligned) 285 md_wakeup_thread(conf->mddev->thread); 286 } 287 } 288 } 289 290 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh) 291 { 292 if (atomic_dec_and_test(&sh->count)) 293 do_release_stripe(conf, sh); 294 } 295 296 static struct llist_node *llist_reverse_order(struct llist_node *head) 297 { 298 struct llist_node *new_head = NULL; 299 300 while (head) { 301 struct llist_node *tmp = head; 302 head = head->next; 303 tmp->next = new_head; 304 new_head = tmp; 305 } 306 307 return new_head; 308 } 309 310 /* should hold conf->device_lock already */ 311 static int release_stripe_list(struct r5conf *conf) 312 { 313 struct stripe_head *sh; 314 int count = 0; 315 struct llist_node *head; 316 317 head = llist_del_all(&conf->released_stripes); 318 head = llist_reverse_order(head); 319 while (head) { 320 sh = llist_entry(head, struct stripe_head, release_list); 321 head = llist_next(head); 322 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ 323 smp_mb(); 324 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); 325 /* 326 * Don't worry the bit is set here, because if the bit is set 327 * again, the count is always > 1. This is true for 328 * STRIPE_ON_UNPLUG_LIST bit too. 329 */ 330 __release_stripe(conf, sh); 331 count++; 332 } 333 334 return count; 335 } 336 337 static void release_stripe(struct stripe_head *sh) 338 { 339 struct r5conf *conf = sh->raid_conf; 340 unsigned long flags; 341 bool wakeup; 342 343 if (test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) 344 goto slow_path; 345 wakeup = llist_add(&sh->release_list, &conf->released_stripes); 346 if (wakeup) 347 md_wakeup_thread(conf->mddev->thread); 348 return; 349 slow_path: 350 local_irq_save(flags); 351 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ 352 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { 353 do_release_stripe(conf, sh); 354 spin_unlock(&conf->device_lock); 355 } 356 local_irq_restore(flags); 357 } 358 359 static inline void remove_hash(struct stripe_head *sh) 360 { 361 pr_debug("remove_hash(), stripe %llu\n", 362 (unsigned long long)sh->sector); 363 364 hlist_del_init(&sh->hash); 365 } 366 367 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 368 { 369 struct hlist_head *hp = stripe_hash(conf, sh->sector); 370 371 pr_debug("insert_hash(), stripe %llu\n", 372 (unsigned long long)sh->sector); 373 374 hlist_add_head(&sh->hash, hp); 375 } 376 377 378 /* find an idle stripe, make sure it is unhashed, and return it. */ 379 static struct stripe_head *get_free_stripe(struct r5conf *conf) 380 { 381 struct stripe_head *sh = NULL; 382 struct list_head *first; 383 384 if (list_empty(&conf->inactive_list)) 385 goto out; 386 first = conf->inactive_list.next; 387 sh = list_entry(first, struct stripe_head, lru); 388 list_del_init(first); 389 remove_hash(sh); 390 atomic_inc(&conf->active_stripes); 391 out: 392 return sh; 393 } 394 395 static void shrink_buffers(struct stripe_head *sh) 396 { 397 struct page *p; 398 int i; 399 int num = sh->raid_conf->pool_size; 400 401 for (i = 0; i < num ; i++) { 402 p = sh->dev[i].page; 403 if (!p) 404 continue; 405 sh->dev[i].page = NULL; 406 put_page(p); 407 } 408 } 409 410 static int grow_buffers(struct stripe_head *sh) 411 { 412 int i; 413 int num = sh->raid_conf->pool_size; 414 415 for (i = 0; i < num; i++) { 416 struct page *page; 417 418 if (!(page = alloc_page(GFP_KERNEL))) { 419 return 1; 420 } 421 sh->dev[i].page = page; 422 } 423 return 0; 424 } 425 426 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 427 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 428 struct stripe_head *sh); 429 430 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 431 { 432 struct r5conf *conf = sh->raid_conf; 433 int i; 434 435 BUG_ON(atomic_read(&sh->count) != 0); 436 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 437 BUG_ON(stripe_operations_active(sh)); 438 439 pr_debug("init_stripe called, stripe %llu\n", 440 (unsigned long long)sh->sector); 441 442 remove_hash(sh); 443 444 sh->generation = conf->generation - previous; 445 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 446 sh->sector = sector; 447 stripe_set_idx(sector, conf, previous, sh); 448 sh->state = 0; 449 450 451 for (i = sh->disks; i--; ) { 452 struct r5dev *dev = &sh->dev[i]; 453 454 if (dev->toread || dev->read || dev->towrite || dev->written || 455 test_bit(R5_LOCKED, &dev->flags)) { 456 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 457 (unsigned long long)sh->sector, i, dev->toread, 458 dev->read, dev->towrite, dev->written, 459 test_bit(R5_LOCKED, &dev->flags)); 460 WARN_ON(1); 461 } 462 dev->flags = 0; 463 raid5_build_block(sh, i, previous); 464 } 465 insert_hash(conf, sh); 466 sh->cpu = smp_processor_id(); 467 } 468 469 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 470 short generation) 471 { 472 struct stripe_head *sh; 473 474 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 475 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) 476 if (sh->sector == sector && sh->generation == generation) 477 return sh; 478 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 479 return NULL; 480 } 481 482 /* 483 * Need to check if array has failed when deciding whether to: 484 * - start an array 485 * - remove non-faulty devices 486 * - add a spare 487 * - allow a reshape 488 * This determination is simple when no reshape is happening. 489 * However if there is a reshape, we need to carefully check 490 * both the before and after sections. 491 * This is because some failed devices may only affect one 492 * of the two sections, and some non-in_sync devices may 493 * be insync in the section most affected by failed devices. 494 */ 495 static int calc_degraded(struct r5conf *conf) 496 { 497 int degraded, degraded2; 498 int i; 499 500 rcu_read_lock(); 501 degraded = 0; 502 for (i = 0; i < conf->previous_raid_disks; i++) { 503 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 504 if (rdev && test_bit(Faulty, &rdev->flags)) 505 rdev = rcu_dereference(conf->disks[i].replacement); 506 if (!rdev || test_bit(Faulty, &rdev->flags)) 507 degraded++; 508 else if (test_bit(In_sync, &rdev->flags)) 509 ; 510 else 511 /* not in-sync or faulty. 512 * If the reshape increases the number of devices, 513 * this is being recovered by the reshape, so 514 * this 'previous' section is not in_sync. 515 * If the number of devices is being reduced however, 516 * the device can only be part of the array if 517 * we are reverting a reshape, so this section will 518 * be in-sync. 519 */ 520 if (conf->raid_disks >= conf->previous_raid_disks) 521 degraded++; 522 } 523 rcu_read_unlock(); 524 if (conf->raid_disks == conf->previous_raid_disks) 525 return degraded; 526 rcu_read_lock(); 527 degraded2 = 0; 528 for (i = 0; i < conf->raid_disks; i++) { 529 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 530 if (rdev && test_bit(Faulty, &rdev->flags)) 531 rdev = rcu_dereference(conf->disks[i].replacement); 532 if (!rdev || test_bit(Faulty, &rdev->flags)) 533 degraded2++; 534 else if (test_bit(In_sync, &rdev->flags)) 535 ; 536 else 537 /* not in-sync or faulty. 538 * If reshape increases the number of devices, this 539 * section has already been recovered, else it 540 * almost certainly hasn't. 541 */ 542 if (conf->raid_disks <= conf->previous_raid_disks) 543 degraded2++; 544 } 545 rcu_read_unlock(); 546 if (degraded2 > degraded) 547 return degraded2; 548 return degraded; 549 } 550 551 static int has_failed(struct r5conf *conf) 552 { 553 int degraded; 554 555 if (conf->mddev->reshape_position == MaxSector) 556 return conf->mddev->degraded > conf->max_degraded; 557 558 degraded = calc_degraded(conf); 559 if (degraded > conf->max_degraded) 560 return 1; 561 return 0; 562 } 563 564 static struct stripe_head * 565 get_active_stripe(struct r5conf *conf, sector_t sector, 566 int previous, int noblock, int noquiesce) 567 { 568 struct stripe_head *sh; 569 570 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 571 572 spin_lock_irq(&conf->device_lock); 573 574 do { 575 wait_event_lock_irq(conf->wait_for_stripe, 576 conf->quiesce == 0 || noquiesce, 577 conf->device_lock); 578 sh = __find_stripe(conf, sector, conf->generation - previous); 579 if (!sh) { 580 if (!conf->inactive_blocked) 581 sh = get_free_stripe(conf); 582 if (noblock && sh == NULL) 583 break; 584 if (!sh) { 585 conf->inactive_blocked = 1; 586 wait_event_lock_irq(conf->wait_for_stripe, 587 !list_empty(&conf->inactive_list) && 588 (atomic_read(&conf->active_stripes) 589 < (conf->max_nr_stripes *3/4) 590 || !conf->inactive_blocked), 591 conf->device_lock); 592 conf->inactive_blocked = 0; 593 } else 594 init_stripe(sh, sector, previous); 595 } else { 596 if (atomic_read(&sh->count)) { 597 BUG_ON(!list_empty(&sh->lru) 598 && !test_bit(STRIPE_EXPANDING, &sh->state) 599 && !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state) 600 && !test_bit(STRIPE_ON_RELEASE_LIST, &sh->state)); 601 } else { 602 if (!test_bit(STRIPE_HANDLE, &sh->state)) 603 atomic_inc(&conf->active_stripes); 604 if (list_empty(&sh->lru) && 605 !test_bit(STRIPE_EXPANDING, &sh->state)) 606 BUG(); 607 list_del_init(&sh->lru); 608 if (sh->group) { 609 sh->group->stripes_cnt--; 610 sh->group = NULL; 611 } 612 } 613 } 614 } while (sh == NULL); 615 616 if (sh) 617 atomic_inc(&sh->count); 618 619 spin_unlock_irq(&conf->device_lock); 620 return sh; 621 } 622 623 /* Determine if 'data_offset' or 'new_data_offset' should be used 624 * in this stripe_head. 625 */ 626 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 627 { 628 sector_t progress = conf->reshape_progress; 629 /* Need a memory barrier to make sure we see the value 630 * of conf->generation, or ->data_offset that was set before 631 * reshape_progress was updated. 632 */ 633 smp_rmb(); 634 if (progress == MaxSector) 635 return 0; 636 if (sh->generation == conf->generation - 1) 637 return 0; 638 /* We are in a reshape, and this is a new-generation stripe, 639 * so use new_data_offset. 640 */ 641 return 1; 642 } 643 644 static void 645 raid5_end_read_request(struct bio *bi, int error); 646 static void 647 raid5_end_write_request(struct bio *bi, int error); 648 649 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 650 { 651 struct r5conf *conf = sh->raid_conf; 652 int i, disks = sh->disks; 653 654 might_sleep(); 655 656 for (i = disks; i--; ) { 657 int rw; 658 int replace_only = 0; 659 struct bio *bi, *rbi; 660 struct md_rdev *rdev, *rrdev = NULL; 661 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 662 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 663 rw = WRITE_FUA; 664 else 665 rw = WRITE; 666 if (test_bit(R5_Discard, &sh->dev[i].flags)) 667 rw |= REQ_DISCARD; 668 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 669 rw = READ; 670 else if (test_and_clear_bit(R5_WantReplace, 671 &sh->dev[i].flags)) { 672 rw = WRITE; 673 replace_only = 1; 674 } else 675 continue; 676 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) 677 rw |= REQ_SYNC; 678 679 bi = &sh->dev[i].req; 680 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 681 682 rcu_read_lock(); 683 rrdev = rcu_dereference(conf->disks[i].replacement); 684 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 685 rdev = rcu_dereference(conf->disks[i].rdev); 686 if (!rdev) { 687 rdev = rrdev; 688 rrdev = NULL; 689 } 690 if (rw & WRITE) { 691 if (replace_only) 692 rdev = NULL; 693 if (rdev == rrdev) 694 /* We raced and saw duplicates */ 695 rrdev = NULL; 696 } else { 697 if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev) 698 rdev = rrdev; 699 rrdev = NULL; 700 } 701 702 if (rdev && test_bit(Faulty, &rdev->flags)) 703 rdev = NULL; 704 if (rdev) 705 atomic_inc(&rdev->nr_pending); 706 if (rrdev && test_bit(Faulty, &rrdev->flags)) 707 rrdev = NULL; 708 if (rrdev) 709 atomic_inc(&rrdev->nr_pending); 710 rcu_read_unlock(); 711 712 /* We have already checked bad blocks for reads. Now 713 * need to check for writes. We never accept write errors 714 * on the replacement, so we don't to check rrdev. 715 */ 716 while ((rw & WRITE) && rdev && 717 test_bit(WriteErrorSeen, &rdev->flags)) { 718 sector_t first_bad; 719 int bad_sectors; 720 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 721 &first_bad, &bad_sectors); 722 if (!bad) 723 break; 724 725 if (bad < 0) { 726 set_bit(BlockedBadBlocks, &rdev->flags); 727 if (!conf->mddev->external && 728 conf->mddev->flags) { 729 /* It is very unlikely, but we might 730 * still need to write out the 731 * bad block log - better give it 732 * a chance*/ 733 md_check_recovery(conf->mddev); 734 } 735 /* 736 * Because md_wait_for_blocked_rdev 737 * will dec nr_pending, we must 738 * increment it first. 739 */ 740 atomic_inc(&rdev->nr_pending); 741 md_wait_for_blocked_rdev(rdev, conf->mddev); 742 } else { 743 /* Acknowledged bad block - skip the write */ 744 rdev_dec_pending(rdev, conf->mddev); 745 rdev = NULL; 746 } 747 } 748 749 if (rdev) { 750 if (s->syncing || s->expanding || s->expanded 751 || s->replacing) 752 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 753 754 set_bit(STRIPE_IO_STARTED, &sh->state); 755 756 bio_reset(bi); 757 bi->bi_bdev = rdev->bdev; 758 bi->bi_rw = rw; 759 bi->bi_end_io = (rw & WRITE) 760 ? raid5_end_write_request 761 : raid5_end_read_request; 762 bi->bi_private = sh; 763 764 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 765 __func__, (unsigned long long)sh->sector, 766 bi->bi_rw, i); 767 atomic_inc(&sh->count); 768 if (use_new_offset(conf, sh)) 769 bi->bi_sector = (sh->sector 770 + rdev->new_data_offset); 771 else 772 bi->bi_sector = (sh->sector 773 + rdev->data_offset); 774 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 775 bi->bi_rw |= REQ_FLUSH; 776 777 bi->bi_vcnt = 1; 778 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 779 bi->bi_io_vec[0].bv_offset = 0; 780 bi->bi_size = STRIPE_SIZE; 781 /* 782 * If this is discard request, set bi_vcnt 0. We don't 783 * want to confuse SCSI because SCSI will replace payload 784 */ 785 if (rw & REQ_DISCARD) 786 bi->bi_vcnt = 0; 787 if (rrdev) 788 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 789 790 if (conf->mddev->gendisk) 791 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), 792 bi, disk_devt(conf->mddev->gendisk), 793 sh->dev[i].sector); 794 generic_make_request(bi); 795 } 796 if (rrdev) { 797 if (s->syncing || s->expanding || s->expanded 798 || s->replacing) 799 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 800 801 set_bit(STRIPE_IO_STARTED, &sh->state); 802 803 bio_reset(rbi); 804 rbi->bi_bdev = rrdev->bdev; 805 rbi->bi_rw = rw; 806 BUG_ON(!(rw & WRITE)); 807 rbi->bi_end_io = raid5_end_write_request; 808 rbi->bi_private = sh; 809 810 pr_debug("%s: for %llu schedule op %ld on " 811 "replacement disc %d\n", 812 __func__, (unsigned long long)sh->sector, 813 rbi->bi_rw, i); 814 atomic_inc(&sh->count); 815 if (use_new_offset(conf, sh)) 816 rbi->bi_sector = (sh->sector 817 + rrdev->new_data_offset); 818 else 819 rbi->bi_sector = (sh->sector 820 + rrdev->data_offset); 821 rbi->bi_vcnt = 1; 822 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 823 rbi->bi_io_vec[0].bv_offset = 0; 824 rbi->bi_size = STRIPE_SIZE; 825 /* 826 * If this is discard request, set bi_vcnt 0. We don't 827 * want to confuse SCSI because SCSI will replace payload 828 */ 829 if (rw & REQ_DISCARD) 830 rbi->bi_vcnt = 0; 831 if (conf->mddev->gendisk) 832 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), 833 rbi, disk_devt(conf->mddev->gendisk), 834 sh->dev[i].sector); 835 generic_make_request(rbi); 836 } 837 if (!rdev && !rrdev) { 838 if (rw & WRITE) 839 set_bit(STRIPE_DEGRADED, &sh->state); 840 pr_debug("skip op %ld on disc %d for sector %llu\n", 841 bi->bi_rw, i, (unsigned long long)sh->sector); 842 clear_bit(R5_LOCKED, &sh->dev[i].flags); 843 set_bit(STRIPE_HANDLE, &sh->state); 844 } 845 } 846 } 847 848 static struct dma_async_tx_descriptor * 849 async_copy_data(int frombio, struct bio *bio, struct page *page, 850 sector_t sector, struct dma_async_tx_descriptor *tx) 851 { 852 struct bio_vec *bvl; 853 struct page *bio_page; 854 int i; 855 int page_offset; 856 struct async_submit_ctl submit; 857 enum async_tx_flags flags = 0; 858 859 if (bio->bi_sector >= sector) 860 page_offset = (signed)(bio->bi_sector - sector) * 512; 861 else 862 page_offset = (signed)(sector - bio->bi_sector) * -512; 863 864 if (frombio) 865 flags |= ASYNC_TX_FENCE; 866 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 867 868 bio_for_each_segment(bvl, bio, i) { 869 int len = bvl->bv_len; 870 int clen; 871 int b_offset = 0; 872 873 if (page_offset < 0) { 874 b_offset = -page_offset; 875 page_offset += b_offset; 876 len -= b_offset; 877 } 878 879 if (len > 0 && page_offset + len > STRIPE_SIZE) 880 clen = STRIPE_SIZE - page_offset; 881 else 882 clen = len; 883 884 if (clen > 0) { 885 b_offset += bvl->bv_offset; 886 bio_page = bvl->bv_page; 887 if (frombio) 888 tx = async_memcpy(page, bio_page, page_offset, 889 b_offset, clen, &submit); 890 else 891 tx = async_memcpy(bio_page, page, b_offset, 892 page_offset, clen, &submit); 893 } 894 /* chain the operations */ 895 submit.depend_tx = tx; 896 897 if (clen < len) /* hit end of page */ 898 break; 899 page_offset += len; 900 } 901 902 return tx; 903 } 904 905 static void ops_complete_biofill(void *stripe_head_ref) 906 { 907 struct stripe_head *sh = stripe_head_ref; 908 struct bio *return_bi = NULL; 909 int i; 910 911 pr_debug("%s: stripe %llu\n", __func__, 912 (unsigned long long)sh->sector); 913 914 /* clear completed biofills */ 915 for (i = sh->disks; i--; ) { 916 struct r5dev *dev = &sh->dev[i]; 917 918 /* acknowledge completion of a biofill operation */ 919 /* and check if we need to reply to a read request, 920 * new R5_Wantfill requests are held off until 921 * !STRIPE_BIOFILL_RUN 922 */ 923 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 924 struct bio *rbi, *rbi2; 925 926 BUG_ON(!dev->read); 927 rbi = dev->read; 928 dev->read = NULL; 929 while (rbi && rbi->bi_sector < 930 dev->sector + STRIPE_SECTORS) { 931 rbi2 = r5_next_bio(rbi, dev->sector); 932 if (!raid5_dec_bi_active_stripes(rbi)) { 933 rbi->bi_next = return_bi; 934 return_bi = rbi; 935 } 936 rbi = rbi2; 937 } 938 } 939 } 940 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 941 942 return_io(return_bi); 943 944 set_bit(STRIPE_HANDLE, &sh->state); 945 release_stripe(sh); 946 } 947 948 static void ops_run_biofill(struct stripe_head *sh) 949 { 950 struct dma_async_tx_descriptor *tx = NULL; 951 struct async_submit_ctl submit; 952 int i; 953 954 pr_debug("%s: stripe %llu\n", __func__, 955 (unsigned long long)sh->sector); 956 957 for (i = sh->disks; i--; ) { 958 struct r5dev *dev = &sh->dev[i]; 959 if (test_bit(R5_Wantfill, &dev->flags)) { 960 struct bio *rbi; 961 spin_lock_irq(&sh->stripe_lock); 962 dev->read = rbi = dev->toread; 963 dev->toread = NULL; 964 spin_unlock_irq(&sh->stripe_lock); 965 while (rbi && rbi->bi_sector < 966 dev->sector + STRIPE_SECTORS) { 967 tx = async_copy_data(0, rbi, dev->page, 968 dev->sector, tx); 969 rbi = r5_next_bio(rbi, dev->sector); 970 } 971 } 972 } 973 974 atomic_inc(&sh->count); 975 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 976 async_trigger_callback(&submit); 977 } 978 979 static void mark_target_uptodate(struct stripe_head *sh, int target) 980 { 981 struct r5dev *tgt; 982 983 if (target < 0) 984 return; 985 986 tgt = &sh->dev[target]; 987 set_bit(R5_UPTODATE, &tgt->flags); 988 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 989 clear_bit(R5_Wantcompute, &tgt->flags); 990 } 991 992 static void ops_complete_compute(void *stripe_head_ref) 993 { 994 struct stripe_head *sh = stripe_head_ref; 995 996 pr_debug("%s: stripe %llu\n", __func__, 997 (unsigned long long)sh->sector); 998 999 /* mark the computed target(s) as uptodate */ 1000 mark_target_uptodate(sh, sh->ops.target); 1001 mark_target_uptodate(sh, sh->ops.target2); 1002 1003 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 1004 if (sh->check_state == check_state_compute_run) 1005 sh->check_state = check_state_compute_result; 1006 set_bit(STRIPE_HANDLE, &sh->state); 1007 release_stripe(sh); 1008 } 1009 1010 /* return a pointer to the address conversion region of the scribble buffer */ 1011 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 1012 struct raid5_percpu *percpu) 1013 { 1014 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 1015 } 1016 1017 static struct dma_async_tx_descriptor * 1018 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 1019 { 1020 int disks = sh->disks; 1021 struct page **xor_srcs = percpu->scribble; 1022 int target = sh->ops.target; 1023 struct r5dev *tgt = &sh->dev[target]; 1024 struct page *xor_dest = tgt->page; 1025 int count = 0; 1026 struct dma_async_tx_descriptor *tx; 1027 struct async_submit_ctl submit; 1028 int i; 1029 1030 pr_debug("%s: stripe %llu block: %d\n", 1031 __func__, (unsigned long long)sh->sector, target); 1032 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1033 1034 for (i = disks; i--; ) 1035 if (i != target) 1036 xor_srcs[count++] = sh->dev[i].page; 1037 1038 atomic_inc(&sh->count); 1039 1040 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 1041 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 1042 if (unlikely(count == 1)) 1043 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1044 else 1045 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1046 1047 return tx; 1048 } 1049 1050 /* set_syndrome_sources - populate source buffers for gen_syndrome 1051 * @srcs - (struct page *) array of size sh->disks 1052 * @sh - stripe_head to parse 1053 * 1054 * Populates srcs in proper layout order for the stripe and returns the 1055 * 'count' of sources to be used in a call to async_gen_syndrome. The P 1056 * destination buffer is recorded in srcs[count] and the Q destination 1057 * is recorded in srcs[count+1]]. 1058 */ 1059 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 1060 { 1061 int disks = sh->disks; 1062 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 1063 int d0_idx = raid6_d0(sh); 1064 int count; 1065 int i; 1066 1067 for (i = 0; i < disks; i++) 1068 srcs[i] = NULL; 1069 1070 count = 0; 1071 i = d0_idx; 1072 do { 1073 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1074 1075 srcs[slot] = sh->dev[i].page; 1076 i = raid6_next_disk(i, disks); 1077 } while (i != d0_idx); 1078 1079 return syndrome_disks; 1080 } 1081 1082 static struct dma_async_tx_descriptor * 1083 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 1084 { 1085 int disks = sh->disks; 1086 struct page **blocks = percpu->scribble; 1087 int target; 1088 int qd_idx = sh->qd_idx; 1089 struct dma_async_tx_descriptor *tx; 1090 struct async_submit_ctl submit; 1091 struct r5dev *tgt; 1092 struct page *dest; 1093 int i; 1094 int count; 1095 1096 if (sh->ops.target < 0) 1097 target = sh->ops.target2; 1098 else if (sh->ops.target2 < 0) 1099 target = sh->ops.target; 1100 else 1101 /* we should only have one valid target */ 1102 BUG(); 1103 BUG_ON(target < 0); 1104 pr_debug("%s: stripe %llu block: %d\n", 1105 __func__, (unsigned long long)sh->sector, target); 1106 1107 tgt = &sh->dev[target]; 1108 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1109 dest = tgt->page; 1110 1111 atomic_inc(&sh->count); 1112 1113 if (target == qd_idx) { 1114 count = set_syndrome_sources(blocks, sh); 1115 blocks[count] = NULL; /* regenerating p is not necessary */ 1116 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 1117 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1118 ops_complete_compute, sh, 1119 to_addr_conv(sh, percpu)); 1120 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1121 } else { 1122 /* Compute any data- or p-drive using XOR */ 1123 count = 0; 1124 for (i = disks; i-- ; ) { 1125 if (i == target || i == qd_idx) 1126 continue; 1127 blocks[count++] = sh->dev[i].page; 1128 } 1129 1130 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1131 NULL, ops_complete_compute, sh, 1132 to_addr_conv(sh, percpu)); 1133 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 1134 } 1135 1136 return tx; 1137 } 1138 1139 static struct dma_async_tx_descriptor * 1140 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 1141 { 1142 int i, count, disks = sh->disks; 1143 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 1144 int d0_idx = raid6_d0(sh); 1145 int faila = -1, failb = -1; 1146 int target = sh->ops.target; 1147 int target2 = sh->ops.target2; 1148 struct r5dev *tgt = &sh->dev[target]; 1149 struct r5dev *tgt2 = &sh->dev[target2]; 1150 struct dma_async_tx_descriptor *tx; 1151 struct page **blocks = percpu->scribble; 1152 struct async_submit_ctl submit; 1153 1154 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 1155 __func__, (unsigned long long)sh->sector, target, target2); 1156 BUG_ON(target < 0 || target2 < 0); 1157 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1158 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 1159 1160 /* we need to open-code set_syndrome_sources to handle the 1161 * slot number conversion for 'faila' and 'failb' 1162 */ 1163 for (i = 0; i < disks ; i++) 1164 blocks[i] = NULL; 1165 count = 0; 1166 i = d0_idx; 1167 do { 1168 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1169 1170 blocks[slot] = sh->dev[i].page; 1171 1172 if (i == target) 1173 faila = slot; 1174 if (i == target2) 1175 failb = slot; 1176 i = raid6_next_disk(i, disks); 1177 } while (i != d0_idx); 1178 1179 BUG_ON(faila == failb); 1180 if (failb < faila) 1181 swap(faila, failb); 1182 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1183 __func__, (unsigned long long)sh->sector, faila, failb); 1184 1185 atomic_inc(&sh->count); 1186 1187 if (failb == syndrome_disks+1) { 1188 /* Q disk is one of the missing disks */ 1189 if (faila == syndrome_disks) { 1190 /* Missing P+Q, just recompute */ 1191 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1192 ops_complete_compute, sh, 1193 to_addr_conv(sh, percpu)); 1194 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1195 STRIPE_SIZE, &submit); 1196 } else { 1197 struct page *dest; 1198 int data_target; 1199 int qd_idx = sh->qd_idx; 1200 1201 /* Missing D+Q: recompute D from P, then recompute Q */ 1202 if (target == qd_idx) 1203 data_target = target2; 1204 else 1205 data_target = target; 1206 1207 count = 0; 1208 for (i = disks; i-- ; ) { 1209 if (i == data_target || i == qd_idx) 1210 continue; 1211 blocks[count++] = sh->dev[i].page; 1212 } 1213 dest = sh->dev[data_target].page; 1214 init_async_submit(&submit, 1215 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1216 NULL, NULL, NULL, 1217 to_addr_conv(sh, percpu)); 1218 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1219 &submit); 1220 1221 count = set_syndrome_sources(blocks, sh); 1222 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1223 ops_complete_compute, sh, 1224 to_addr_conv(sh, percpu)); 1225 return async_gen_syndrome(blocks, 0, count+2, 1226 STRIPE_SIZE, &submit); 1227 } 1228 } else { 1229 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1230 ops_complete_compute, sh, 1231 to_addr_conv(sh, percpu)); 1232 if (failb == syndrome_disks) { 1233 /* We're missing D+P. */ 1234 return async_raid6_datap_recov(syndrome_disks+2, 1235 STRIPE_SIZE, faila, 1236 blocks, &submit); 1237 } else { 1238 /* We're missing D+D. */ 1239 return async_raid6_2data_recov(syndrome_disks+2, 1240 STRIPE_SIZE, faila, failb, 1241 blocks, &submit); 1242 } 1243 } 1244 } 1245 1246 1247 static void ops_complete_prexor(void *stripe_head_ref) 1248 { 1249 struct stripe_head *sh = stripe_head_ref; 1250 1251 pr_debug("%s: stripe %llu\n", __func__, 1252 (unsigned long long)sh->sector); 1253 } 1254 1255 static struct dma_async_tx_descriptor * 1256 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 1257 struct dma_async_tx_descriptor *tx) 1258 { 1259 int disks = sh->disks; 1260 struct page **xor_srcs = percpu->scribble; 1261 int count = 0, pd_idx = sh->pd_idx, i; 1262 struct async_submit_ctl submit; 1263 1264 /* existing parity data subtracted */ 1265 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1266 1267 pr_debug("%s: stripe %llu\n", __func__, 1268 (unsigned long long)sh->sector); 1269 1270 for (i = disks; i--; ) { 1271 struct r5dev *dev = &sh->dev[i]; 1272 /* Only process blocks that are known to be uptodate */ 1273 if (test_bit(R5_Wantdrain, &dev->flags)) 1274 xor_srcs[count++] = dev->page; 1275 } 1276 1277 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1278 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 1279 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1280 1281 return tx; 1282 } 1283 1284 static struct dma_async_tx_descriptor * 1285 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1286 { 1287 int disks = sh->disks; 1288 int i; 1289 1290 pr_debug("%s: stripe %llu\n", __func__, 1291 (unsigned long long)sh->sector); 1292 1293 for (i = disks; i--; ) { 1294 struct r5dev *dev = &sh->dev[i]; 1295 struct bio *chosen; 1296 1297 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 1298 struct bio *wbi; 1299 1300 spin_lock_irq(&sh->stripe_lock); 1301 chosen = dev->towrite; 1302 dev->towrite = NULL; 1303 BUG_ON(dev->written); 1304 wbi = dev->written = chosen; 1305 spin_unlock_irq(&sh->stripe_lock); 1306 1307 while (wbi && wbi->bi_sector < 1308 dev->sector + STRIPE_SECTORS) { 1309 if (wbi->bi_rw & REQ_FUA) 1310 set_bit(R5_WantFUA, &dev->flags); 1311 if (wbi->bi_rw & REQ_SYNC) 1312 set_bit(R5_SyncIO, &dev->flags); 1313 if (wbi->bi_rw & REQ_DISCARD) 1314 set_bit(R5_Discard, &dev->flags); 1315 else 1316 tx = async_copy_data(1, wbi, dev->page, 1317 dev->sector, tx); 1318 wbi = r5_next_bio(wbi, dev->sector); 1319 } 1320 } 1321 } 1322 1323 return tx; 1324 } 1325 1326 static void ops_complete_reconstruct(void *stripe_head_ref) 1327 { 1328 struct stripe_head *sh = stripe_head_ref; 1329 int disks = sh->disks; 1330 int pd_idx = sh->pd_idx; 1331 int qd_idx = sh->qd_idx; 1332 int i; 1333 bool fua = false, sync = false, discard = false; 1334 1335 pr_debug("%s: stripe %llu\n", __func__, 1336 (unsigned long long)sh->sector); 1337 1338 for (i = disks; i--; ) { 1339 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1340 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); 1341 discard |= test_bit(R5_Discard, &sh->dev[i].flags); 1342 } 1343 1344 for (i = disks; i--; ) { 1345 struct r5dev *dev = &sh->dev[i]; 1346 1347 if (dev->written || i == pd_idx || i == qd_idx) { 1348 if (!discard) 1349 set_bit(R5_UPTODATE, &dev->flags); 1350 if (fua) 1351 set_bit(R5_WantFUA, &dev->flags); 1352 if (sync) 1353 set_bit(R5_SyncIO, &dev->flags); 1354 } 1355 } 1356 1357 if (sh->reconstruct_state == reconstruct_state_drain_run) 1358 sh->reconstruct_state = reconstruct_state_drain_result; 1359 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1360 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1361 else { 1362 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1363 sh->reconstruct_state = reconstruct_state_result; 1364 } 1365 1366 set_bit(STRIPE_HANDLE, &sh->state); 1367 release_stripe(sh); 1368 } 1369 1370 static void 1371 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1372 struct dma_async_tx_descriptor *tx) 1373 { 1374 int disks = sh->disks; 1375 struct page **xor_srcs = percpu->scribble; 1376 struct async_submit_ctl submit; 1377 int count = 0, pd_idx = sh->pd_idx, i; 1378 struct page *xor_dest; 1379 int prexor = 0; 1380 unsigned long flags; 1381 1382 pr_debug("%s: stripe %llu\n", __func__, 1383 (unsigned long long)sh->sector); 1384 1385 for (i = 0; i < sh->disks; i++) { 1386 if (pd_idx == i) 1387 continue; 1388 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1389 break; 1390 } 1391 if (i >= sh->disks) { 1392 atomic_inc(&sh->count); 1393 set_bit(R5_Discard, &sh->dev[pd_idx].flags); 1394 ops_complete_reconstruct(sh); 1395 return; 1396 } 1397 /* check if prexor is active which means only process blocks 1398 * that are part of a read-modify-write (written) 1399 */ 1400 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1401 prexor = 1; 1402 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1403 for (i = disks; i--; ) { 1404 struct r5dev *dev = &sh->dev[i]; 1405 if (dev->written) 1406 xor_srcs[count++] = dev->page; 1407 } 1408 } else { 1409 xor_dest = sh->dev[pd_idx].page; 1410 for (i = disks; i--; ) { 1411 struct r5dev *dev = &sh->dev[i]; 1412 if (i != pd_idx) 1413 xor_srcs[count++] = dev->page; 1414 } 1415 } 1416 1417 /* 1/ if we prexor'd then the dest is reused as a source 1418 * 2/ if we did not prexor then we are redoing the parity 1419 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1420 * for the synchronous xor case 1421 */ 1422 flags = ASYNC_TX_ACK | 1423 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1424 1425 atomic_inc(&sh->count); 1426 1427 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1428 to_addr_conv(sh, percpu)); 1429 if (unlikely(count == 1)) 1430 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1431 else 1432 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1433 } 1434 1435 static void 1436 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1437 struct dma_async_tx_descriptor *tx) 1438 { 1439 struct async_submit_ctl submit; 1440 struct page **blocks = percpu->scribble; 1441 int count, i; 1442 1443 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1444 1445 for (i = 0; i < sh->disks; i++) { 1446 if (sh->pd_idx == i || sh->qd_idx == i) 1447 continue; 1448 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1449 break; 1450 } 1451 if (i >= sh->disks) { 1452 atomic_inc(&sh->count); 1453 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 1454 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 1455 ops_complete_reconstruct(sh); 1456 return; 1457 } 1458 1459 count = set_syndrome_sources(blocks, sh); 1460 1461 atomic_inc(&sh->count); 1462 1463 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1464 sh, to_addr_conv(sh, percpu)); 1465 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1466 } 1467 1468 static void ops_complete_check(void *stripe_head_ref) 1469 { 1470 struct stripe_head *sh = stripe_head_ref; 1471 1472 pr_debug("%s: stripe %llu\n", __func__, 1473 (unsigned long long)sh->sector); 1474 1475 sh->check_state = check_state_check_result; 1476 set_bit(STRIPE_HANDLE, &sh->state); 1477 release_stripe(sh); 1478 } 1479 1480 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1481 { 1482 int disks = sh->disks; 1483 int pd_idx = sh->pd_idx; 1484 int qd_idx = sh->qd_idx; 1485 struct page *xor_dest; 1486 struct page **xor_srcs = percpu->scribble; 1487 struct dma_async_tx_descriptor *tx; 1488 struct async_submit_ctl submit; 1489 int count; 1490 int i; 1491 1492 pr_debug("%s: stripe %llu\n", __func__, 1493 (unsigned long long)sh->sector); 1494 1495 count = 0; 1496 xor_dest = sh->dev[pd_idx].page; 1497 xor_srcs[count++] = xor_dest; 1498 for (i = disks; i--; ) { 1499 if (i == pd_idx || i == qd_idx) 1500 continue; 1501 xor_srcs[count++] = sh->dev[i].page; 1502 } 1503 1504 init_async_submit(&submit, 0, NULL, NULL, NULL, 1505 to_addr_conv(sh, percpu)); 1506 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1507 &sh->ops.zero_sum_result, &submit); 1508 1509 atomic_inc(&sh->count); 1510 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1511 tx = async_trigger_callback(&submit); 1512 } 1513 1514 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1515 { 1516 struct page **srcs = percpu->scribble; 1517 struct async_submit_ctl submit; 1518 int count; 1519 1520 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1521 (unsigned long long)sh->sector, checkp); 1522 1523 count = set_syndrome_sources(srcs, sh); 1524 if (!checkp) 1525 srcs[count] = NULL; 1526 1527 atomic_inc(&sh->count); 1528 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1529 sh, to_addr_conv(sh, percpu)); 1530 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1531 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1532 } 1533 1534 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1535 { 1536 int overlap_clear = 0, i, disks = sh->disks; 1537 struct dma_async_tx_descriptor *tx = NULL; 1538 struct r5conf *conf = sh->raid_conf; 1539 int level = conf->level; 1540 struct raid5_percpu *percpu; 1541 unsigned long cpu; 1542 1543 cpu = get_cpu(); 1544 percpu = per_cpu_ptr(conf->percpu, cpu); 1545 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1546 ops_run_biofill(sh); 1547 overlap_clear++; 1548 } 1549 1550 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1551 if (level < 6) 1552 tx = ops_run_compute5(sh, percpu); 1553 else { 1554 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1555 tx = ops_run_compute6_1(sh, percpu); 1556 else 1557 tx = ops_run_compute6_2(sh, percpu); 1558 } 1559 /* terminate the chain if reconstruct is not set to be run */ 1560 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1561 async_tx_ack(tx); 1562 } 1563 1564 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1565 tx = ops_run_prexor(sh, percpu, tx); 1566 1567 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1568 tx = ops_run_biodrain(sh, tx); 1569 overlap_clear++; 1570 } 1571 1572 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1573 if (level < 6) 1574 ops_run_reconstruct5(sh, percpu, tx); 1575 else 1576 ops_run_reconstruct6(sh, percpu, tx); 1577 } 1578 1579 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1580 if (sh->check_state == check_state_run) 1581 ops_run_check_p(sh, percpu); 1582 else if (sh->check_state == check_state_run_q) 1583 ops_run_check_pq(sh, percpu, 0); 1584 else if (sh->check_state == check_state_run_pq) 1585 ops_run_check_pq(sh, percpu, 1); 1586 else 1587 BUG(); 1588 } 1589 1590 if (overlap_clear) 1591 for (i = disks; i--; ) { 1592 struct r5dev *dev = &sh->dev[i]; 1593 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1594 wake_up(&sh->raid_conf->wait_for_overlap); 1595 } 1596 put_cpu(); 1597 } 1598 1599 static int grow_one_stripe(struct r5conf *conf) 1600 { 1601 struct stripe_head *sh; 1602 sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL); 1603 if (!sh) 1604 return 0; 1605 1606 sh->raid_conf = conf; 1607 1608 spin_lock_init(&sh->stripe_lock); 1609 1610 if (grow_buffers(sh)) { 1611 shrink_buffers(sh); 1612 kmem_cache_free(conf->slab_cache, sh); 1613 return 0; 1614 } 1615 /* we just created an active stripe so... */ 1616 atomic_set(&sh->count, 1); 1617 atomic_inc(&conf->active_stripes); 1618 INIT_LIST_HEAD(&sh->lru); 1619 release_stripe(sh); 1620 return 1; 1621 } 1622 1623 static int grow_stripes(struct r5conf *conf, int num) 1624 { 1625 struct kmem_cache *sc; 1626 int devs = max(conf->raid_disks, conf->previous_raid_disks); 1627 1628 if (conf->mddev->gendisk) 1629 sprintf(conf->cache_name[0], 1630 "raid%d-%s", conf->level, mdname(conf->mddev)); 1631 else 1632 sprintf(conf->cache_name[0], 1633 "raid%d-%p", conf->level, conf->mddev); 1634 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 1635 1636 conf->active_name = 0; 1637 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1638 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1639 0, 0, NULL); 1640 if (!sc) 1641 return 1; 1642 conf->slab_cache = sc; 1643 conf->pool_size = devs; 1644 while (num--) 1645 if (!grow_one_stripe(conf)) 1646 return 1; 1647 return 0; 1648 } 1649 1650 /** 1651 * scribble_len - return the required size of the scribble region 1652 * @num - total number of disks in the array 1653 * 1654 * The size must be enough to contain: 1655 * 1/ a struct page pointer for each device in the array +2 1656 * 2/ room to convert each entry in (1) to its corresponding dma 1657 * (dma_map_page()) or page (page_address()) address. 1658 * 1659 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1660 * calculate over all devices (not just the data blocks), using zeros in place 1661 * of the P and Q blocks. 1662 */ 1663 static size_t scribble_len(int num) 1664 { 1665 size_t len; 1666 1667 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1668 1669 return len; 1670 } 1671 1672 static int resize_stripes(struct r5conf *conf, int newsize) 1673 { 1674 /* Make all the stripes able to hold 'newsize' devices. 1675 * New slots in each stripe get 'page' set to a new page. 1676 * 1677 * This happens in stages: 1678 * 1/ create a new kmem_cache and allocate the required number of 1679 * stripe_heads. 1680 * 2/ gather all the old stripe_heads and transfer the pages across 1681 * to the new stripe_heads. This will have the side effect of 1682 * freezing the array as once all stripe_heads have been collected, 1683 * no IO will be possible. Old stripe heads are freed once their 1684 * pages have been transferred over, and the old kmem_cache is 1685 * freed when all stripes are done. 1686 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1687 * we simple return a failre status - no need to clean anything up. 1688 * 4/ allocate new pages for the new slots in the new stripe_heads. 1689 * If this fails, we don't bother trying the shrink the 1690 * stripe_heads down again, we just leave them as they are. 1691 * As each stripe_head is processed the new one is released into 1692 * active service. 1693 * 1694 * Once step2 is started, we cannot afford to wait for a write, 1695 * so we use GFP_NOIO allocations. 1696 */ 1697 struct stripe_head *osh, *nsh; 1698 LIST_HEAD(newstripes); 1699 struct disk_info *ndisks; 1700 unsigned long cpu; 1701 int err; 1702 struct kmem_cache *sc; 1703 int i; 1704 1705 if (newsize <= conf->pool_size) 1706 return 0; /* never bother to shrink */ 1707 1708 err = md_allow_write(conf->mddev); 1709 if (err) 1710 return err; 1711 1712 /* Step 1 */ 1713 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1714 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1715 0, 0, NULL); 1716 if (!sc) 1717 return -ENOMEM; 1718 1719 for (i = conf->max_nr_stripes; i; i--) { 1720 nsh = kmem_cache_zalloc(sc, GFP_KERNEL); 1721 if (!nsh) 1722 break; 1723 1724 nsh->raid_conf = conf; 1725 spin_lock_init(&nsh->stripe_lock); 1726 1727 list_add(&nsh->lru, &newstripes); 1728 } 1729 if (i) { 1730 /* didn't get enough, give up */ 1731 while (!list_empty(&newstripes)) { 1732 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1733 list_del(&nsh->lru); 1734 kmem_cache_free(sc, nsh); 1735 } 1736 kmem_cache_destroy(sc); 1737 return -ENOMEM; 1738 } 1739 /* Step 2 - Must use GFP_NOIO now. 1740 * OK, we have enough stripes, start collecting inactive 1741 * stripes and copying them over 1742 */ 1743 list_for_each_entry(nsh, &newstripes, lru) { 1744 spin_lock_irq(&conf->device_lock); 1745 wait_event_lock_irq(conf->wait_for_stripe, 1746 !list_empty(&conf->inactive_list), 1747 conf->device_lock); 1748 osh = get_free_stripe(conf); 1749 spin_unlock_irq(&conf->device_lock); 1750 atomic_set(&nsh->count, 1); 1751 for(i=0; i<conf->pool_size; i++) 1752 nsh->dev[i].page = osh->dev[i].page; 1753 for( ; i<newsize; i++) 1754 nsh->dev[i].page = NULL; 1755 kmem_cache_free(conf->slab_cache, osh); 1756 } 1757 kmem_cache_destroy(conf->slab_cache); 1758 1759 /* Step 3. 1760 * At this point, we are holding all the stripes so the array 1761 * is completely stalled, so now is a good time to resize 1762 * conf->disks and the scribble region 1763 */ 1764 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1765 if (ndisks) { 1766 for (i=0; i<conf->raid_disks; i++) 1767 ndisks[i] = conf->disks[i]; 1768 kfree(conf->disks); 1769 conf->disks = ndisks; 1770 } else 1771 err = -ENOMEM; 1772 1773 get_online_cpus(); 1774 conf->scribble_len = scribble_len(newsize); 1775 for_each_present_cpu(cpu) { 1776 struct raid5_percpu *percpu; 1777 void *scribble; 1778 1779 percpu = per_cpu_ptr(conf->percpu, cpu); 1780 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1781 1782 if (scribble) { 1783 kfree(percpu->scribble); 1784 percpu->scribble = scribble; 1785 } else { 1786 err = -ENOMEM; 1787 break; 1788 } 1789 } 1790 put_online_cpus(); 1791 1792 /* Step 4, return new stripes to service */ 1793 while(!list_empty(&newstripes)) { 1794 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1795 list_del_init(&nsh->lru); 1796 1797 for (i=conf->raid_disks; i < newsize; i++) 1798 if (nsh->dev[i].page == NULL) { 1799 struct page *p = alloc_page(GFP_NOIO); 1800 nsh->dev[i].page = p; 1801 if (!p) 1802 err = -ENOMEM; 1803 } 1804 release_stripe(nsh); 1805 } 1806 /* critical section pass, GFP_NOIO no longer needed */ 1807 1808 conf->slab_cache = sc; 1809 conf->active_name = 1-conf->active_name; 1810 conf->pool_size = newsize; 1811 return err; 1812 } 1813 1814 static int drop_one_stripe(struct r5conf *conf) 1815 { 1816 struct stripe_head *sh; 1817 1818 spin_lock_irq(&conf->device_lock); 1819 sh = get_free_stripe(conf); 1820 spin_unlock_irq(&conf->device_lock); 1821 if (!sh) 1822 return 0; 1823 BUG_ON(atomic_read(&sh->count)); 1824 shrink_buffers(sh); 1825 kmem_cache_free(conf->slab_cache, sh); 1826 atomic_dec(&conf->active_stripes); 1827 return 1; 1828 } 1829 1830 static void shrink_stripes(struct r5conf *conf) 1831 { 1832 while (drop_one_stripe(conf)) 1833 ; 1834 1835 if (conf->slab_cache) 1836 kmem_cache_destroy(conf->slab_cache); 1837 conf->slab_cache = NULL; 1838 } 1839 1840 static void raid5_end_read_request(struct bio * bi, int error) 1841 { 1842 struct stripe_head *sh = bi->bi_private; 1843 struct r5conf *conf = sh->raid_conf; 1844 int disks = sh->disks, i; 1845 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1846 char b[BDEVNAME_SIZE]; 1847 struct md_rdev *rdev = NULL; 1848 sector_t s; 1849 1850 for (i=0 ; i<disks; i++) 1851 if (bi == &sh->dev[i].req) 1852 break; 1853 1854 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1855 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1856 uptodate); 1857 if (i == disks) { 1858 BUG(); 1859 return; 1860 } 1861 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1862 /* If replacement finished while this request was outstanding, 1863 * 'replacement' might be NULL already. 1864 * In that case it moved down to 'rdev'. 1865 * rdev is not removed until all requests are finished. 1866 */ 1867 rdev = conf->disks[i].replacement; 1868 if (!rdev) 1869 rdev = conf->disks[i].rdev; 1870 1871 if (use_new_offset(conf, sh)) 1872 s = sh->sector + rdev->new_data_offset; 1873 else 1874 s = sh->sector + rdev->data_offset; 1875 if (uptodate) { 1876 set_bit(R5_UPTODATE, &sh->dev[i].flags); 1877 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 1878 /* Note that this cannot happen on a 1879 * replacement device. We just fail those on 1880 * any error 1881 */ 1882 printk_ratelimited( 1883 KERN_INFO 1884 "md/raid:%s: read error corrected" 1885 " (%lu sectors at %llu on %s)\n", 1886 mdname(conf->mddev), STRIPE_SECTORS, 1887 (unsigned long long)s, 1888 bdevname(rdev->bdev, b)); 1889 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 1890 clear_bit(R5_ReadError, &sh->dev[i].flags); 1891 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1892 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 1893 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 1894 1895 if (atomic_read(&rdev->read_errors)) 1896 atomic_set(&rdev->read_errors, 0); 1897 } else { 1898 const char *bdn = bdevname(rdev->bdev, b); 1899 int retry = 0; 1900 int set_bad = 0; 1901 1902 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 1903 atomic_inc(&rdev->read_errors); 1904 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1905 printk_ratelimited( 1906 KERN_WARNING 1907 "md/raid:%s: read error on replacement device " 1908 "(sector %llu on %s).\n", 1909 mdname(conf->mddev), 1910 (unsigned long long)s, 1911 bdn); 1912 else if (conf->mddev->degraded >= conf->max_degraded) { 1913 set_bad = 1; 1914 printk_ratelimited( 1915 KERN_WARNING 1916 "md/raid:%s: read error not correctable " 1917 "(sector %llu on %s).\n", 1918 mdname(conf->mddev), 1919 (unsigned long long)s, 1920 bdn); 1921 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { 1922 /* Oh, no!!! */ 1923 set_bad = 1; 1924 printk_ratelimited( 1925 KERN_WARNING 1926 "md/raid:%s: read error NOT corrected!! " 1927 "(sector %llu on %s).\n", 1928 mdname(conf->mddev), 1929 (unsigned long long)s, 1930 bdn); 1931 } else if (atomic_read(&rdev->read_errors) 1932 > conf->max_nr_stripes) 1933 printk(KERN_WARNING 1934 "md/raid:%s: Too many read errors, failing device %s.\n", 1935 mdname(conf->mddev), bdn); 1936 else 1937 retry = 1; 1938 if (retry) 1939 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { 1940 set_bit(R5_ReadError, &sh->dev[i].flags); 1941 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 1942 } else 1943 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); 1944 else { 1945 clear_bit(R5_ReadError, &sh->dev[i].flags); 1946 clear_bit(R5_ReWrite, &sh->dev[i].flags); 1947 if (!(set_bad 1948 && test_bit(In_sync, &rdev->flags) 1949 && rdev_set_badblocks( 1950 rdev, sh->sector, STRIPE_SECTORS, 0))) 1951 md_error(conf->mddev, rdev); 1952 } 1953 } 1954 rdev_dec_pending(rdev, conf->mddev); 1955 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1956 set_bit(STRIPE_HANDLE, &sh->state); 1957 release_stripe(sh); 1958 } 1959 1960 static void raid5_end_write_request(struct bio *bi, int error) 1961 { 1962 struct stripe_head *sh = bi->bi_private; 1963 struct r5conf *conf = sh->raid_conf; 1964 int disks = sh->disks, i; 1965 struct md_rdev *uninitialized_var(rdev); 1966 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1967 sector_t first_bad; 1968 int bad_sectors; 1969 int replacement = 0; 1970 1971 for (i = 0 ; i < disks; i++) { 1972 if (bi == &sh->dev[i].req) { 1973 rdev = conf->disks[i].rdev; 1974 break; 1975 } 1976 if (bi == &sh->dev[i].rreq) { 1977 rdev = conf->disks[i].replacement; 1978 if (rdev) 1979 replacement = 1; 1980 else 1981 /* rdev was removed and 'replacement' 1982 * replaced it. rdev is not removed 1983 * until all requests are finished. 1984 */ 1985 rdev = conf->disks[i].rdev; 1986 break; 1987 } 1988 } 1989 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 1990 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1991 uptodate); 1992 if (i == disks) { 1993 BUG(); 1994 return; 1995 } 1996 1997 if (replacement) { 1998 if (!uptodate) 1999 md_error(conf->mddev, rdev); 2000 else if (is_badblock(rdev, sh->sector, 2001 STRIPE_SECTORS, 2002 &first_bad, &bad_sectors)) 2003 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2004 } else { 2005 if (!uptodate) { 2006 set_bit(WriteErrorSeen, &rdev->flags); 2007 set_bit(R5_WriteError, &sh->dev[i].flags); 2008 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2009 set_bit(MD_RECOVERY_NEEDED, 2010 &rdev->mddev->recovery); 2011 } else if (is_badblock(rdev, sh->sector, 2012 STRIPE_SECTORS, 2013 &first_bad, &bad_sectors)) { 2014 set_bit(R5_MadeGood, &sh->dev[i].flags); 2015 if (test_bit(R5_ReadError, &sh->dev[i].flags)) 2016 /* That was a successful write so make 2017 * sure it looks like we already did 2018 * a re-write. 2019 */ 2020 set_bit(R5_ReWrite, &sh->dev[i].flags); 2021 } 2022 } 2023 rdev_dec_pending(rdev, conf->mddev); 2024 2025 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2026 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2027 set_bit(STRIPE_HANDLE, &sh->state); 2028 release_stripe(sh); 2029 } 2030 2031 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 2032 2033 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 2034 { 2035 struct r5dev *dev = &sh->dev[i]; 2036 2037 bio_init(&dev->req); 2038 dev->req.bi_io_vec = &dev->vec; 2039 dev->req.bi_vcnt++; 2040 dev->req.bi_max_vecs++; 2041 dev->req.bi_private = sh; 2042 dev->vec.bv_page = dev->page; 2043 2044 bio_init(&dev->rreq); 2045 dev->rreq.bi_io_vec = &dev->rvec; 2046 dev->rreq.bi_vcnt++; 2047 dev->rreq.bi_max_vecs++; 2048 dev->rreq.bi_private = sh; 2049 dev->rvec.bv_page = dev->page; 2050 2051 dev->flags = 0; 2052 dev->sector = compute_blocknr(sh, i, previous); 2053 } 2054 2055 static void error(struct mddev *mddev, struct md_rdev *rdev) 2056 { 2057 char b[BDEVNAME_SIZE]; 2058 struct r5conf *conf = mddev->private; 2059 unsigned long flags; 2060 pr_debug("raid456: error called\n"); 2061 2062 spin_lock_irqsave(&conf->device_lock, flags); 2063 clear_bit(In_sync, &rdev->flags); 2064 mddev->degraded = calc_degraded(conf); 2065 spin_unlock_irqrestore(&conf->device_lock, flags); 2066 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2067 2068 set_bit(Blocked, &rdev->flags); 2069 set_bit(Faulty, &rdev->flags); 2070 set_bit(MD_CHANGE_DEVS, &mddev->flags); 2071 printk(KERN_ALERT 2072 "md/raid:%s: Disk failure on %s, disabling device.\n" 2073 "md/raid:%s: Operation continuing on %d devices.\n", 2074 mdname(mddev), 2075 bdevname(rdev->bdev, b), 2076 mdname(mddev), 2077 conf->raid_disks - mddev->degraded); 2078 } 2079 2080 /* 2081 * Input: a 'big' sector number, 2082 * Output: index of the data and parity disk, and the sector # in them. 2083 */ 2084 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 2085 int previous, int *dd_idx, 2086 struct stripe_head *sh) 2087 { 2088 sector_t stripe, stripe2; 2089 sector_t chunk_number; 2090 unsigned int chunk_offset; 2091 int pd_idx, qd_idx; 2092 int ddf_layout = 0; 2093 sector_t new_sector; 2094 int algorithm = previous ? conf->prev_algo 2095 : conf->algorithm; 2096 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2097 : conf->chunk_sectors; 2098 int raid_disks = previous ? conf->previous_raid_disks 2099 : conf->raid_disks; 2100 int data_disks = raid_disks - conf->max_degraded; 2101 2102 /* First compute the information on this sector */ 2103 2104 /* 2105 * Compute the chunk number and the sector offset inside the chunk 2106 */ 2107 chunk_offset = sector_div(r_sector, sectors_per_chunk); 2108 chunk_number = r_sector; 2109 2110 /* 2111 * Compute the stripe number 2112 */ 2113 stripe = chunk_number; 2114 *dd_idx = sector_div(stripe, data_disks); 2115 stripe2 = stripe; 2116 /* 2117 * Select the parity disk based on the user selected algorithm. 2118 */ 2119 pd_idx = qd_idx = -1; 2120 switch(conf->level) { 2121 case 4: 2122 pd_idx = data_disks; 2123 break; 2124 case 5: 2125 switch (algorithm) { 2126 case ALGORITHM_LEFT_ASYMMETRIC: 2127 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2128 if (*dd_idx >= pd_idx) 2129 (*dd_idx)++; 2130 break; 2131 case ALGORITHM_RIGHT_ASYMMETRIC: 2132 pd_idx = sector_div(stripe2, raid_disks); 2133 if (*dd_idx >= pd_idx) 2134 (*dd_idx)++; 2135 break; 2136 case ALGORITHM_LEFT_SYMMETRIC: 2137 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2138 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2139 break; 2140 case ALGORITHM_RIGHT_SYMMETRIC: 2141 pd_idx = sector_div(stripe2, raid_disks); 2142 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2143 break; 2144 case ALGORITHM_PARITY_0: 2145 pd_idx = 0; 2146 (*dd_idx)++; 2147 break; 2148 case ALGORITHM_PARITY_N: 2149 pd_idx = data_disks; 2150 break; 2151 default: 2152 BUG(); 2153 } 2154 break; 2155 case 6: 2156 2157 switch (algorithm) { 2158 case ALGORITHM_LEFT_ASYMMETRIC: 2159 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2160 qd_idx = pd_idx + 1; 2161 if (pd_idx == raid_disks-1) { 2162 (*dd_idx)++; /* Q D D D P */ 2163 qd_idx = 0; 2164 } else if (*dd_idx >= pd_idx) 2165 (*dd_idx) += 2; /* D D P Q D */ 2166 break; 2167 case ALGORITHM_RIGHT_ASYMMETRIC: 2168 pd_idx = sector_div(stripe2, raid_disks); 2169 qd_idx = pd_idx + 1; 2170 if (pd_idx == raid_disks-1) { 2171 (*dd_idx)++; /* Q D D D P */ 2172 qd_idx = 0; 2173 } else if (*dd_idx >= pd_idx) 2174 (*dd_idx) += 2; /* D D P Q D */ 2175 break; 2176 case ALGORITHM_LEFT_SYMMETRIC: 2177 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2178 qd_idx = (pd_idx + 1) % raid_disks; 2179 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2180 break; 2181 case ALGORITHM_RIGHT_SYMMETRIC: 2182 pd_idx = sector_div(stripe2, raid_disks); 2183 qd_idx = (pd_idx + 1) % raid_disks; 2184 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2185 break; 2186 2187 case ALGORITHM_PARITY_0: 2188 pd_idx = 0; 2189 qd_idx = 1; 2190 (*dd_idx) += 2; 2191 break; 2192 case ALGORITHM_PARITY_N: 2193 pd_idx = data_disks; 2194 qd_idx = data_disks + 1; 2195 break; 2196 2197 case ALGORITHM_ROTATING_ZERO_RESTART: 2198 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2199 * of blocks for computing Q is different. 2200 */ 2201 pd_idx = sector_div(stripe2, raid_disks); 2202 qd_idx = pd_idx + 1; 2203 if (pd_idx == raid_disks-1) { 2204 (*dd_idx)++; /* Q D D D P */ 2205 qd_idx = 0; 2206 } else if (*dd_idx >= pd_idx) 2207 (*dd_idx) += 2; /* D D P Q D */ 2208 ddf_layout = 1; 2209 break; 2210 2211 case ALGORITHM_ROTATING_N_RESTART: 2212 /* Same a left_asymmetric, by first stripe is 2213 * D D D P Q rather than 2214 * Q D D D P 2215 */ 2216 stripe2 += 1; 2217 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2218 qd_idx = pd_idx + 1; 2219 if (pd_idx == raid_disks-1) { 2220 (*dd_idx)++; /* Q D D D P */ 2221 qd_idx = 0; 2222 } else if (*dd_idx >= pd_idx) 2223 (*dd_idx) += 2; /* D D P Q D */ 2224 ddf_layout = 1; 2225 break; 2226 2227 case ALGORITHM_ROTATING_N_CONTINUE: 2228 /* Same as left_symmetric but Q is before P */ 2229 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2230 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2231 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2232 ddf_layout = 1; 2233 break; 2234 2235 case ALGORITHM_LEFT_ASYMMETRIC_6: 2236 /* RAID5 left_asymmetric, with Q on last device */ 2237 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2238 if (*dd_idx >= pd_idx) 2239 (*dd_idx)++; 2240 qd_idx = raid_disks - 1; 2241 break; 2242 2243 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2244 pd_idx = sector_div(stripe2, raid_disks-1); 2245 if (*dd_idx >= pd_idx) 2246 (*dd_idx)++; 2247 qd_idx = raid_disks - 1; 2248 break; 2249 2250 case ALGORITHM_LEFT_SYMMETRIC_6: 2251 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2252 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2253 qd_idx = raid_disks - 1; 2254 break; 2255 2256 case ALGORITHM_RIGHT_SYMMETRIC_6: 2257 pd_idx = sector_div(stripe2, raid_disks-1); 2258 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2259 qd_idx = raid_disks - 1; 2260 break; 2261 2262 case ALGORITHM_PARITY_0_6: 2263 pd_idx = 0; 2264 (*dd_idx)++; 2265 qd_idx = raid_disks - 1; 2266 break; 2267 2268 default: 2269 BUG(); 2270 } 2271 break; 2272 } 2273 2274 if (sh) { 2275 sh->pd_idx = pd_idx; 2276 sh->qd_idx = qd_idx; 2277 sh->ddf_layout = ddf_layout; 2278 } 2279 /* 2280 * Finally, compute the new sector number 2281 */ 2282 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2283 return new_sector; 2284 } 2285 2286 2287 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 2288 { 2289 struct r5conf *conf = sh->raid_conf; 2290 int raid_disks = sh->disks; 2291 int data_disks = raid_disks - conf->max_degraded; 2292 sector_t new_sector = sh->sector, check; 2293 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2294 : conf->chunk_sectors; 2295 int algorithm = previous ? conf->prev_algo 2296 : conf->algorithm; 2297 sector_t stripe; 2298 int chunk_offset; 2299 sector_t chunk_number; 2300 int dummy1, dd_idx = i; 2301 sector_t r_sector; 2302 struct stripe_head sh2; 2303 2304 2305 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2306 stripe = new_sector; 2307 2308 if (i == sh->pd_idx) 2309 return 0; 2310 switch(conf->level) { 2311 case 4: break; 2312 case 5: 2313 switch (algorithm) { 2314 case ALGORITHM_LEFT_ASYMMETRIC: 2315 case ALGORITHM_RIGHT_ASYMMETRIC: 2316 if (i > sh->pd_idx) 2317 i--; 2318 break; 2319 case ALGORITHM_LEFT_SYMMETRIC: 2320 case ALGORITHM_RIGHT_SYMMETRIC: 2321 if (i < sh->pd_idx) 2322 i += raid_disks; 2323 i -= (sh->pd_idx + 1); 2324 break; 2325 case ALGORITHM_PARITY_0: 2326 i -= 1; 2327 break; 2328 case ALGORITHM_PARITY_N: 2329 break; 2330 default: 2331 BUG(); 2332 } 2333 break; 2334 case 6: 2335 if (i == sh->qd_idx) 2336 return 0; /* It is the Q disk */ 2337 switch (algorithm) { 2338 case ALGORITHM_LEFT_ASYMMETRIC: 2339 case ALGORITHM_RIGHT_ASYMMETRIC: 2340 case ALGORITHM_ROTATING_ZERO_RESTART: 2341 case ALGORITHM_ROTATING_N_RESTART: 2342 if (sh->pd_idx == raid_disks-1) 2343 i--; /* Q D D D P */ 2344 else if (i > sh->pd_idx) 2345 i -= 2; /* D D P Q D */ 2346 break; 2347 case ALGORITHM_LEFT_SYMMETRIC: 2348 case ALGORITHM_RIGHT_SYMMETRIC: 2349 if (sh->pd_idx == raid_disks-1) 2350 i--; /* Q D D D P */ 2351 else { 2352 /* D D P Q D */ 2353 if (i < sh->pd_idx) 2354 i += raid_disks; 2355 i -= (sh->pd_idx + 2); 2356 } 2357 break; 2358 case ALGORITHM_PARITY_0: 2359 i -= 2; 2360 break; 2361 case ALGORITHM_PARITY_N: 2362 break; 2363 case ALGORITHM_ROTATING_N_CONTINUE: 2364 /* Like left_symmetric, but P is before Q */ 2365 if (sh->pd_idx == 0) 2366 i--; /* P D D D Q */ 2367 else { 2368 /* D D Q P D */ 2369 if (i < sh->pd_idx) 2370 i += raid_disks; 2371 i -= (sh->pd_idx + 1); 2372 } 2373 break; 2374 case ALGORITHM_LEFT_ASYMMETRIC_6: 2375 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2376 if (i > sh->pd_idx) 2377 i--; 2378 break; 2379 case ALGORITHM_LEFT_SYMMETRIC_6: 2380 case ALGORITHM_RIGHT_SYMMETRIC_6: 2381 if (i < sh->pd_idx) 2382 i += data_disks + 1; 2383 i -= (sh->pd_idx + 1); 2384 break; 2385 case ALGORITHM_PARITY_0_6: 2386 i -= 1; 2387 break; 2388 default: 2389 BUG(); 2390 } 2391 break; 2392 } 2393 2394 chunk_number = stripe * data_disks + i; 2395 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2396 2397 check = raid5_compute_sector(conf, r_sector, 2398 previous, &dummy1, &sh2); 2399 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2400 || sh2.qd_idx != sh->qd_idx) { 2401 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2402 mdname(conf->mddev)); 2403 return 0; 2404 } 2405 return r_sector; 2406 } 2407 2408 2409 static void 2410 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2411 int rcw, int expand) 2412 { 2413 int i, pd_idx = sh->pd_idx, disks = sh->disks; 2414 struct r5conf *conf = sh->raid_conf; 2415 int level = conf->level; 2416 2417 if (rcw) { 2418 2419 for (i = disks; i--; ) { 2420 struct r5dev *dev = &sh->dev[i]; 2421 2422 if (dev->towrite) { 2423 set_bit(R5_LOCKED, &dev->flags); 2424 set_bit(R5_Wantdrain, &dev->flags); 2425 if (!expand) 2426 clear_bit(R5_UPTODATE, &dev->flags); 2427 s->locked++; 2428 } 2429 } 2430 /* if we are not expanding this is a proper write request, and 2431 * there will be bios with new data to be drained into the 2432 * stripe cache 2433 */ 2434 if (!expand) { 2435 if (!s->locked) 2436 /* False alarm, nothing to do */ 2437 return; 2438 sh->reconstruct_state = reconstruct_state_drain_run; 2439 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2440 } else 2441 sh->reconstruct_state = reconstruct_state_run; 2442 2443 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2444 2445 if (s->locked + conf->max_degraded == disks) 2446 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2447 atomic_inc(&conf->pending_full_writes); 2448 } else { 2449 BUG_ON(level == 6); 2450 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2451 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2452 2453 for (i = disks; i--; ) { 2454 struct r5dev *dev = &sh->dev[i]; 2455 if (i == pd_idx) 2456 continue; 2457 2458 if (dev->towrite && 2459 (test_bit(R5_UPTODATE, &dev->flags) || 2460 test_bit(R5_Wantcompute, &dev->flags))) { 2461 set_bit(R5_Wantdrain, &dev->flags); 2462 set_bit(R5_LOCKED, &dev->flags); 2463 clear_bit(R5_UPTODATE, &dev->flags); 2464 s->locked++; 2465 } 2466 } 2467 if (!s->locked) 2468 /* False alarm - nothing to do */ 2469 return; 2470 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2471 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2472 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2473 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2474 } 2475 2476 /* keep the parity disk(s) locked while asynchronous operations 2477 * are in flight 2478 */ 2479 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2480 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2481 s->locked++; 2482 2483 if (level == 6) { 2484 int qd_idx = sh->qd_idx; 2485 struct r5dev *dev = &sh->dev[qd_idx]; 2486 2487 set_bit(R5_LOCKED, &dev->flags); 2488 clear_bit(R5_UPTODATE, &dev->flags); 2489 s->locked++; 2490 } 2491 2492 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2493 __func__, (unsigned long long)sh->sector, 2494 s->locked, s->ops_request); 2495 } 2496 2497 /* 2498 * Each stripe/dev can have one or more bion attached. 2499 * toread/towrite point to the first in a chain. 2500 * The bi_next chain must be in order. 2501 */ 2502 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2503 { 2504 struct bio **bip; 2505 struct r5conf *conf = sh->raid_conf; 2506 int firstwrite=0; 2507 2508 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2509 (unsigned long long)bi->bi_sector, 2510 (unsigned long long)sh->sector); 2511 2512 /* 2513 * If several bio share a stripe. The bio bi_phys_segments acts as a 2514 * reference count to avoid race. The reference count should already be 2515 * increased before this function is called (for example, in 2516 * make_request()), so other bio sharing this stripe will not free the 2517 * stripe. If a stripe is owned by one stripe, the stripe lock will 2518 * protect it. 2519 */ 2520 spin_lock_irq(&sh->stripe_lock); 2521 if (forwrite) { 2522 bip = &sh->dev[dd_idx].towrite; 2523 if (*bip == NULL) 2524 firstwrite = 1; 2525 } else 2526 bip = &sh->dev[dd_idx].toread; 2527 while (*bip && (*bip)->bi_sector < bi->bi_sector) { 2528 if (bio_end_sector(*bip) > bi->bi_sector) 2529 goto overlap; 2530 bip = & (*bip)->bi_next; 2531 } 2532 if (*bip && (*bip)->bi_sector < bio_end_sector(bi)) 2533 goto overlap; 2534 2535 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2536 if (*bip) 2537 bi->bi_next = *bip; 2538 *bip = bi; 2539 raid5_inc_bi_active_stripes(bi); 2540 2541 if (forwrite) { 2542 /* check if page is covered */ 2543 sector_t sector = sh->dev[dd_idx].sector; 2544 for (bi=sh->dev[dd_idx].towrite; 2545 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2546 bi && bi->bi_sector <= sector; 2547 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2548 if (bio_end_sector(bi) >= sector) 2549 sector = bio_end_sector(bi); 2550 } 2551 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2552 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2553 } 2554 2555 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2556 (unsigned long long)(*bip)->bi_sector, 2557 (unsigned long long)sh->sector, dd_idx); 2558 spin_unlock_irq(&sh->stripe_lock); 2559 2560 if (conf->mddev->bitmap && firstwrite) { 2561 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2562 STRIPE_SECTORS, 0); 2563 sh->bm_seq = conf->seq_flush+1; 2564 set_bit(STRIPE_BIT_DELAY, &sh->state); 2565 } 2566 return 1; 2567 2568 overlap: 2569 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2570 spin_unlock_irq(&sh->stripe_lock); 2571 return 0; 2572 } 2573 2574 static void end_reshape(struct r5conf *conf); 2575 2576 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 2577 struct stripe_head *sh) 2578 { 2579 int sectors_per_chunk = 2580 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2581 int dd_idx; 2582 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2583 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2584 2585 raid5_compute_sector(conf, 2586 stripe * (disks - conf->max_degraded) 2587 *sectors_per_chunk + chunk_offset, 2588 previous, 2589 &dd_idx, sh); 2590 } 2591 2592 static void 2593 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 2594 struct stripe_head_state *s, int disks, 2595 struct bio **return_bi) 2596 { 2597 int i; 2598 for (i = disks; i--; ) { 2599 struct bio *bi; 2600 int bitmap_end = 0; 2601 2602 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2603 struct md_rdev *rdev; 2604 rcu_read_lock(); 2605 rdev = rcu_dereference(conf->disks[i].rdev); 2606 if (rdev && test_bit(In_sync, &rdev->flags)) 2607 atomic_inc(&rdev->nr_pending); 2608 else 2609 rdev = NULL; 2610 rcu_read_unlock(); 2611 if (rdev) { 2612 if (!rdev_set_badblocks( 2613 rdev, 2614 sh->sector, 2615 STRIPE_SECTORS, 0)) 2616 md_error(conf->mddev, rdev); 2617 rdev_dec_pending(rdev, conf->mddev); 2618 } 2619 } 2620 spin_lock_irq(&sh->stripe_lock); 2621 /* fail all writes first */ 2622 bi = sh->dev[i].towrite; 2623 sh->dev[i].towrite = NULL; 2624 spin_unlock_irq(&sh->stripe_lock); 2625 if (bi) 2626 bitmap_end = 1; 2627 2628 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2629 wake_up(&conf->wait_for_overlap); 2630 2631 while (bi && bi->bi_sector < 2632 sh->dev[i].sector + STRIPE_SECTORS) { 2633 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2634 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2635 if (!raid5_dec_bi_active_stripes(bi)) { 2636 md_write_end(conf->mddev); 2637 bi->bi_next = *return_bi; 2638 *return_bi = bi; 2639 } 2640 bi = nextbi; 2641 } 2642 if (bitmap_end) 2643 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2644 STRIPE_SECTORS, 0, 0); 2645 bitmap_end = 0; 2646 /* and fail all 'written' */ 2647 bi = sh->dev[i].written; 2648 sh->dev[i].written = NULL; 2649 if (bi) bitmap_end = 1; 2650 while (bi && bi->bi_sector < 2651 sh->dev[i].sector + STRIPE_SECTORS) { 2652 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2653 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2654 if (!raid5_dec_bi_active_stripes(bi)) { 2655 md_write_end(conf->mddev); 2656 bi->bi_next = *return_bi; 2657 *return_bi = bi; 2658 } 2659 bi = bi2; 2660 } 2661 2662 /* fail any reads if this device is non-operational and 2663 * the data has not reached the cache yet. 2664 */ 2665 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2666 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2667 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2668 spin_lock_irq(&sh->stripe_lock); 2669 bi = sh->dev[i].toread; 2670 sh->dev[i].toread = NULL; 2671 spin_unlock_irq(&sh->stripe_lock); 2672 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2673 wake_up(&conf->wait_for_overlap); 2674 while (bi && bi->bi_sector < 2675 sh->dev[i].sector + STRIPE_SECTORS) { 2676 struct bio *nextbi = 2677 r5_next_bio(bi, sh->dev[i].sector); 2678 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2679 if (!raid5_dec_bi_active_stripes(bi)) { 2680 bi->bi_next = *return_bi; 2681 *return_bi = bi; 2682 } 2683 bi = nextbi; 2684 } 2685 } 2686 if (bitmap_end) 2687 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2688 STRIPE_SECTORS, 0, 0); 2689 /* If we were in the middle of a write the parity block might 2690 * still be locked - so just clear all R5_LOCKED flags 2691 */ 2692 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2693 } 2694 2695 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2696 if (atomic_dec_and_test(&conf->pending_full_writes)) 2697 md_wakeup_thread(conf->mddev->thread); 2698 } 2699 2700 static void 2701 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 2702 struct stripe_head_state *s) 2703 { 2704 int abort = 0; 2705 int i; 2706 2707 clear_bit(STRIPE_SYNCING, &sh->state); 2708 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 2709 wake_up(&conf->wait_for_overlap); 2710 s->syncing = 0; 2711 s->replacing = 0; 2712 /* There is nothing more to do for sync/check/repair. 2713 * Don't even need to abort as that is handled elsewhere 2714 * if needed, and not always wanted e.g. if there is a known 2715 * bad block here. 2716 * For recover/replace we need to record a bad block on all 2717 * non-sync devices, or abort the recovery 2718 */ 2719 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 2720 /* During recovery devices cannot be removed, so 2721 * locking and refcounting of rdevs is not needed 2722 */ 2723 for (i = 0; i < conf->raid_disks; i++) { 2724 struct md_rdev *rdev = conf->disks[i].rdev; 2725 if (rdev 2726 && !test_bit(Faulty, &rdev->flags) 2727 && !test_bit(In_sync, &rdev->flags) 2728 && !rdev_set_badblocks(rdev, sh->sector, 2729 STRIPE_SECTORS, 0)) 2730 abort = 1; 2731 rdev = conf->disks[i].replacement; 2732 if (rdev 2733 && !test_bit(Faulty, &rdev->flags) 2734 && !test_bit(In_sync, &rdev->flags) 2735 && !rdev_set_badblocks(rdev, sh->sector, 2736 STRIPE_SECTORS, 0)) 2737 abort = 1; 2738 } 2739 if (abort) 2740 conf->recovery_disabled = 2741 conf->mddev->recovery_disabled; 2742 } 2743 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 2744 } 2745 2746 static int want_replace(struct stripe_head *sh, int disk_idx) 2747 { 2748 struct md_rdev *rdev; 2749 int rv = 0; 2750 /* Doing recovery so rcu locking not required */ 2751 rdev = sh->raid_conf->disks[disk_idx].replacement; 2752 if (rdev 2753 && !test_bit(Faulty, &rdev->flags) 2754 && !test_bit(In_sync, &rdev->flags) 2755 && (rdev->recovery_offset <= sh->sector 2756 || rdev->mddev->recovery_cp <= sh->sector)) 2757 rv = 1; 2758 2759 return rv; 2760 } 2761 2762 /* fetch_block - checks the given member device to see if its data needs 2763 * to be read or computed to satisfy a request. 2764 * 2765 * Returns 1 when no more member devices need to be checked, otherwise returns 2766 * 0 to tell the loop in handle_stripe_fill to continue 2767 */ 2768 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 2769 int disk_idx, int disks) 2770 { 2771 struct r5dev *dev = &sh->dev[disk_idx]; 2772 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 2773 &sh->dev[s->failed_num[1]] }; 2774 2775 /* is the data in this block needed, and can we get it? */ 2776 if (!test_bit(R5_LOCKED, &dev->flags) && 2777 !test_bit(R5_UPTODATE, &dev->flags) && 2778 (dev->toread || 2779 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2780 s->syncing || s->expanding || 2781 (s->replacing && want_replace(sh, disk_idx)) || 2782 (s->failed >= 1 && fdev[0]->toread) || 2783 (s->failed >= 2 && fdev[1]->toread) || 2784 (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite && 2785 !test_bit(R5_OVERWRITE, &fdev[0]->flags)) || 2786 (sh->raid_conf->level == 6 && s->failed && s->to_write))) { 2787 /* we would like to get this block, possibly by computing it, 2788 * otherwise read it if the backing disk is insync 2789 */ 2790 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2791 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2792 if ((s->uptodate == disks - 1) && 2793 (s->failed && (disk_idx == s->failed_num[0] || 2794 disk_idx == s->failed_num[1]))) { 2795 /* have disk failed, and we're requested to fetch it; 2796 * do compute it 2797 */ 2798 pr_debug("Computing stripe %llu block %d\n", 2799 (unsigned long long)sh->sector, disk_idx); 2800 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2801 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2802 set_bit(R5_Wantcompute, &dev->flags); 2803 sh->ops.target = disk_idx; 2804 sh->ops.target2 = -1; /* no 2nd target */ 2805 s->req_compute = 1; 2806 /* Careful: from this point on 'uptodate' is in the eye 2807 * of raid_run_ops which services 'compute' operations 2808 * before writes. R5_Wantcompute flags a block that will 2809 * be R5_UPTODATE by the time it is needed for a 2810 * subsequent operation. 2811 */ 2812 s->uptodate++; 2813 return 1; 2814 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2815 /* Computing 2-failure is *very* expensive; only 2816 * do it if failed >= 2 2817 */ 2818 int other; 2819 for (other = disks; other--; ) { 2820 if (other == disk_idx) 2821 continue; 2822 if (!test_bit(R5_UPTODATE, 2823 &sh->dev[other].flags)) 2824 break; 2825 } 2826 BUG_ON(other < 0); 2827 pr_debug("Computing stripe %llu blocks %d,%d\n", 2828 (unsigned long long)sh->sector, 2829 disk_idx, other); 2830 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2831 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2832 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2833 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2834 sh->ops.target = disk_idx; 2835 sh->ops.target2 = other; 2836 s->uptodate += 2; 2837 s->req_compute = 1; 2838 return 1; 2839 } else if (test_bit(R5_Insync, &dev->flags)) { 2840 set_bit(R5_LOCKED, &dev->flags); 2841 set_bit(R5_Wantread, &dev->flags); 2842 s->locked++; 2843 pr_debug("Reading block %d (sync=%d)\n", 2844 disk_idx, s->syncing); 2845 } 2846 } 2847 2848 return 0; 2849 } 2850 2851 /** 2852 * handle_stripe_fill - read or compute data to satisfy pending requests. 2853 */ 2854 static void handle_stripe_fill(struct stripe_head *sh, 2855 struct stripe_head_state *s, 2856 int disks) 2857 { 2858 int i; 2859 2860 /* look for blocks to read/compute, skip this if a compute 2861 * is already in flight, or if the stripe contents are in the 2862 * midst of changing due to a write 2863 */ 2864 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 2865 !sh->reconstruct_state) 2866 for (i = disks; i--; ) 2867 if (fetch_block(sh, s, i, disks)) 2868 break; 2869 set_bit(STRIPE_HANDLE, &sh->state); 2870 } 2871 2872 2873 /* handle_stripe_clean_event 2874 * any written block on an uptodate or failed drive can be returned. 2875 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 2876 * never LOCKED, so we don't need to test 'failed' directly. 2877 */ 2878 static void handle_stripe_clean_event(struct r5conf *conf, 2879 struct stripe_head *sh, int disks, struct bio **return_bi) 2880 { 2881 int i; 2882 struct r5dev *dev; 2883 int discard_pending = 0; 2884 2885 for (i = disks; i--; ) 2886 if (sh->dev[i].written) { 2887 dev = &sh->dev[i]; 2888 if (!test_bit(R5_LOCKED, &dev->flags) && 2889 (test_bit(R5_UPTODATE, &dev->flags) || 2890 test_bit(R5_Discard, &dev->flags))) { 2891 /* We can return any write requests */ 2892 struct bio *wbi, *wbi2; 2893 pr_debug("Return write for disc %d\n", i); 2894 if (test_and_clear_bit(R5_Discard, &dev->flags)) 2895 clear_bit(R5_UPTODATE, &dev->flags); 2896 wbi = dev->written; 2897 dev->written = NULL; 2898 while (wbi && wbi->bi_sector < 2899 dev->sector + STRIPE_SECTORS) { 2900 wbi2 = r5_next_bio(wbi, dev->sector); 2901 if (!raid5_dec_bi_active_stripes(wbi)) { 2902 md_write_end(conf->mddev); 2903 wbi->bi_next = *return_bi; 2904 *return_bi = wbi; 2905 } 2906 wbi = wbi2; 2907 } 2908 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2909 STRIPE_SECTORS, 2910 !test_bit(STRIPE_DEGRADED, &sh->state), 2911 0); 2912 } else if (test_bit(R5_Discard, &dev->flags)) 2913 discard_pending = 1; 2914 } 2915 if (!discard_pending && 2916 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { 2917 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 2918 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 2919 if (sh->qd_idx >= 0) { 2920 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 2921 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); 2922 } 2923 /* now that discard is done we can proceed with any sync */ 2924 clear_bit(STRIPE_DISCARD, &sh->state); 2925 /* 2926 * SCSI discard will change some bio fields and the stripe has 2927 * no updated data, so remove it from hash list and the stripe 2928 * will be reinitialized 2929 */ 2930 spin_lock_irq(&conf->device_lock); 2931 remove_hash(sh); 2932 spin_unlock_irq(&conf->device_lock); 2933 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) 2934 set_bit(STRIPE_HANDLE, &sh->state); 2935 2936 } 2937 2938 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2939 if (atomic_dec_and_test(&conf->pending_full_writes)) 2940 md_wakeup_thread(conf->mddev->thread); 2941 } 2942 2943 static void handle_stripe_dirtying(struct r5conf *conf, 2944 struct stripe_head *sh, 2945 struct stripe_head_state *s, 2946 int disks) 2947 { 2948 int rmw = 0, rcw = 0, i; 2949 sector_t recovery_cp = conf->mddev->recovery_cp; 2950 2951 /* RAID6 requires 'rcw' in current implementation. 2952 * Otherwise, check whether resync is now happening or should start. 2953 * If yes, then the array is dirty (after unclean shutdown or 2954 * initial creation), so parity in some stripes might be inconsistent. 2955 * In this case, we need to always do reconstruct-write, to ensure 2956 * that in case of drive failure or read-error correction, we 2957 * generate correct data from the parity. 2958 */ 2959 if (conf->max_degraded == 2 || 2960 (recovery_cp < MaxSector && sh->sector >= recovery_cp)) { 2961 /* Calculate the real rcw later - for now make it 2962 * look like rcw is cheaper 2963 */ 2964 rcw = 1; rmw = 2; 2965 pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n", 2966 conf->max_degraded, (unsigned long long)recovery_cp, 2967 (unsigned long long)sh->sector); 2968 } else for (i = disks; i--; ) { 2969 /* would I have to read this buffer for read_modify_write */ 2970 struct r5dev *dev = &sh->dev[i]; 2971 if ((dev->towrite || i == sh->pd_idx) && 2972 !test_bit(R5_LOCKED, &dev->flags) && 2973 !(test_bit(R5_UPTODATE, &dev->flags) || 2974 test_bit(R5_Wantcompute, &dev->flags))) { 2975 if (test_bit(R5_Insync, &dev->flags)) 2976 rmw++; 2977 else 2978 rmw += 2*disks; /* cannot read it */ 2979 } 2980 /* Would I have to read this buffer for reconstruct_write */ 2981 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 2982 !test_bit(R5_LOCKED, &dev->flags) && 2983 !(test_bit(R5_UPTODATE, &dev->flags) || 2984 test_bit(R5_Wantcompute, &dev->flags))) { 2985 if (test_bit(R5_Insync, &dev->flags)) rcw++; 2986 else 2987 rcw += 2*disks; 2988 } 2989 } 2990 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 2991 (unsigned long long)sh->sector, rmw, rcw); 2992 set_bit(STRIPE_HANDLE, &sh->state); 2993 if (rmw < rcw && rmw > 0) { 2994 /* prefer read-modify-write, but need to get some data */ 2995 if (conf->mddev->queue) 2996 blk_add_trace_msg(conf->mddev->queue, 2997 "raid5 rmw %llu %d", 2998 (unsigned long long)sh->sector, rmw); 2999 for (i = disks; i--; ) { 3000 struct r5dev *dev = &sh->dev[i]; 3001 if ((dev->towrite || i == sh->pd_idx) && 3002 !test_bit(R5_LOCKED, &dev->flags) && 3003 !(test_bit(R5_UPTODATE, &dev->flags) || 3004 test_bit(R5_Wantcompute, &dev->flags)) && 3005 test_bit(R5_Insync, &dev->flags)) { 3006 if ( 3007 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3008 pr_debug("Read_old block " 3009 "%d for r-m-w\n", i); 3010 set_bit(R5_LOCKED, &dev->flags); 3011 set_bit(R5_Wantread, &dev->flags); 3012 s->locked++; 3013 } else { 3014 set_bit(STRIPE_DELAYED, &sh->state); 3015 set_bit(STRIPE_HANDLE, &sh->state); 3016 } 3017 } 3018 } 3019 } 3020 if (rcw <= rmw && rcw > 0) { 3021 /* want reconstruct write, but need to get some data */ 3022 int qread =0; 3023 rcw = 0; 3024 for (i = disks; i--; ) { 3025 struct r5dev *dev = &sh->dev[i]; 3026 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3027 i != sh->pd_idx && i != sh->qd_idx && 3028 !test_bit(R5_LOCKED, &dev->flags) && 3029 !(test_bit(R5_UPTODATE, &dev->flags) || 3030 test_bit(R5_Wantcompute, &dev->flags))) { 3031 rcw++; 3032 if (!test_bit(R5_Insync, &dev->flags)) 3033 continue; /* it's a failed drive */ 3034 if ( 3035 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 3036 pr_debug("Read_old block " 3037 "%d for Reconstruct\n", i); 3038 set_bit(R5_LOCKED, &dev->flags); 3039 set_bit(R5_Wantread, &dev->flags); 3040 s->locked++; 3041 qread++; 3042 } else { 3043 set_bit(STRIPE_DELAYED, &sh->state); 3044 set_bit(STRIPE_HANDLE, &sh->state); 3045 } 3046 } 3047 } 3048 if (rcw && conf->mddev->queue) 3049 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", 3050 (unsigned long long)sh->sector, 3051 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); 3052 } 3053 /* now if nothing is locked, and if we have enough data, 3054 * we can start a write request 3055 */ 3056 /* since handle_stripe can be called at any time we need to handle the 3057 * case where a compute block operation has been submitted and then a 3058 * subsequent call wants to start a write request. raid_run_ops only 3059 * handles the case where compute block and reconstruct are requested 3060 * simultaneously. If this is not the case then new writes need to be 3061 * held off until the compute completes. 3062 */ 3063 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 3064 (s->locked == 0 && (rcw == 0 || rmw == 0) && 3065 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 3066 schedule_reconstruction(sh, s, rcw == 0, 0); 3067 } 3068 3069 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 3070 struct stripe_head_state *s, int disks) 3071 { 3072 struct r5dev *dev = NULL; 3073 3074 set_bit(STRIPE_HANDLE, &sh->state); 3075 3076 switch (sh->check_state) { 3077 case check_state_idle: 3078 /* start a new check operation if there are no failures */ 3079 if (s->failed == 0) { 3080 BUG_ON(s->uptodate != disks); 3081 sh->check_state = check_state_run; 3082 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3083 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3084 s->uptodate--; 3085 break; 3086 } 3087 dev = &sh->dev[s->failed_num[0]]; 3088 /* fall through */ 3089 case check_state_compute_result: 3090 sh->check_state = check_state_idle; 3091 if (!dev) 3092 dev = &sh->dev[sh->pd_idx]; 3093 3094 /* check that a write has not made the stripe insync */ 3095 if (test_bit(STRIPE_INSYNC, &sh->state)) 3096 break; 3097 3098 /* either failed parity check, or recovery is happening */ 3099 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3100 BUG_ON(s->uptodate != disks); 3101 3102 set_bit(R5_LOCKED, &dev->flags); 3103 s->locked++; 3104 set_bit(R5_Wantwrite, &dev->flags); 3105 3106 clear_bit(STRIPE_DEGRADED, &sh->state); 3107 set_bit(STRIPE_INSYNC, &sh->state); 3108 break; 3109 case check_state_run: 3110 break; /* we will be called again upon completion */ 3111 case check_state_check_result: 3112 sh->check_state = check_state_idle; 3113 3114 /* if a failure occurred during the check operation, leave 3115 * STRIPE_INSYNC not set and let the stripe be handled again 3116 */ 3117 if (s->failed) 3118 break; 3119 3120 /* handle a successful check operation, if parity is correct 3121 * we are done. Otherwise update the mismatch count and repair 3122 * parity if !MD_RECOVERY_CHECK 3123 */ 3124 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 3125 /* parity is correct (on disc, 3126 * not in buffer any more) 3127 */ 3128 set_bit(STRIPE_INSYNC, &sh->state); 3129 else { 3130 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3131 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3132 /* don't try to repair!! */ 3133 set_bit(STRIPE_INSYNC, &sh->state); 3134 else { 3135 sh->check_state = check_state_compute_run; 3136 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3137 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3138 set_bit(R5_Wantcompute, 3139 &sh->dev[sh->pd_idx].flags); 3140 sh->ops.target = sh->pd_idx; 3141 sh->ops.target2 = -1; 3142 s->uptodate++; 3143 } 3144 } 3145 break; 3146 case check_state_compute_run: 3147 break; 3148 default: 3149 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3150 __func__, sh->check_state, 3151 (unsigned long long) sh->sector); 3152 BUG(); 3153 } 3154 } 3155 3156 3157 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 3158 struct stripe_head_state *s, 3159 int disks) 3160 { 3161 int pd_idx = sh->pd_idx; 3162 int qd_idx = sh->qd_idx; 3163 struct r5dev *dev; 3164 3165 set_bit(STRIPE_HANDLE, &sh->state); 3166 3167 BUG_ON(s->failed > 2); 3168 3169 /* Want to check and possibly repair P and Q. 3170 * However there could be one 'failed' device, in which 3171 * case we can only check one of them, possibly using the 3172 * other to generate missing data 3173 */ 3174 3175 switch (sh->check_state) { 3176 case check_state_idle: 3177 /* start a new check operation if there are < 2 failures */ 3178 if (s->failed == s->q_failed) { 3179 /* The only possible failed device holds Q, so it 3180 * makes sense to check P (If anything else were failed, 3181 * we would have used P to recreate it). 3182 */ 3183 sh->check_state = check_state_run; 3184 } 3185 if (!s->q_failed && s->failed < 2) { 3186 /* Q is not failed, and we didn't use it to generate 3187 * anything, so it makes sense to check it 3188 */ 3189 if (sh->check_state == check_state_run) 3190 sh->check_state = check_state_run_pq; 3191 else 3192 sh->check_state = check_state_run_q; 3193 } 3194 3195 /* discard potentially stale zero_sum_result */ 3196 sh->ops.zero_sum_result = 0; 3197 3198 if (sh->check_state == check_state_run) { 3199 /* async_xor_zero_sum destroys the contents of P */ 3200 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 3201 s->uptodate--; 3202 } 3203 if (sh->check_state >= check_state_run && 3204 sh->check_state <= check_state_run_pq) { 3205 /* async_syndrome_zero_sum preserves P and Q, so 3206 * no need to mark them !uptodate here 3207 */ 3208 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3209 break; 3210 } 3211 3212 /* we have 2-disk failure */ 3213 BUG_ON(s->failed != 2); 3214 /* fall through */ 3215 case check_state_compute_result: 3216 sh->check_state = check_state_idle; 3217 3218 /* check that a write has not made the stripe insync */ 3219 if (test_bit(STRIPE_INSYNC, &sh->state)) 3220 break; 3221 3222 /* now write out any block on a failed drive, 3223 * or P or Q if they were recomputed 3224 */ 3225 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 3226 if (s->failed == 2) { 3227 dev = &sh->dev[s->failed_num[1]]; 3228 s->locked++; 3229 set_bit(R5_LOCKED, &dev->flags); 3230 set_bit(R5_Wantwrite, &dev->flags); 3231 } 3232 if (s->failed >= 1) { 3233 dev = &sh->dev[s->failed_num[0]]; 3234 s->locked++; 3235 set_bit(R5_LOCKED, &dev->flags); 3236 set_bit(R5_Wantwrite, &dev->flags); 3237 } 3238 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3239 dev = &sh->dev[pd_idx]; 3240 s->locked++; 3241 set_bit(R5_LOCKED, &dev->flags); 3242 set_bit(R5_Wantwrite, &dev->flags); 3243 } 3244 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3245 dev = &sh->dev[qd_idx]; 3246 s->locked++; 3247 set_bit(R5_LOCKED, &dev->flags); 3248 set_bit(R5_Wantwrite, &dev->flags); 3249 } 3250 clear_bit(STRIPE_DEGRADED, &sh->state); 3251 3252 set_bit(STRIPE_INSYNC, &sh->state); 3253 break; 3254 case check_state_run: 3255 case check_state_run_q: 3256 case check_state_run_pq: 3257 break; /* we will be called again upon completion */ 3258 case check_state_check_result: 3259 sh->check_state = check_state_idle; 3260 3261 /* handle a successful check operation, if parity is correct 3262 * we are done. Otherwise update the mismatch count and repair 3263 * parity if !MD_RECOVERY_CHECK 3264 */ 3265 if (sh->ops.zero_sum_result == 0) { 3266 /* both parities are correct */ 3267 if (!s->failed) 3268 set_bit(STRIPE_INSYNC, &sh->state); 3269 else { 3270 /* in contrast to the raid5 case we can validate 3271 * parity, but still have a failure to write 3272 * back 3273 */ 3274 sh->check_state = check_state_compute_result; 3275 /* Returning at this point means that we may go 3276 * off and bring p and/or q uptodate again so 3277 * we make sure to check zero_sum_result again 3278 * to verify if p or q need writeback 3279 */ 3280 } 3281 } else { 3282 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3283 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3284 /* don't try to repair!! */ 3285 set_bit(STRIPE_INSYNC, &sh->state); 3286 else { 3287 int *target = &sh->ops.target; 3288 3289 sh->ops.target = -1; 3290 sh->ops.target2 = -1; 3291 sh->check_state = check_state_compute_run; 3292 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3293 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3294 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3295 set_bit(R5_Wantcompute, 3296 &sh->dev[pd_idx].flags); 3297 *target = pd_idx; 3298 target = &sh->ops.target2; 3299 s->uptodate++; 3300 } 3301 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3302 set_bit(R5_Wantcompute, 3303 &sh->dev[qd_idx].flags); 3304 *target = qd_idx; 3305 s->uptodate++; 3306 } 3307 } 3308 } 3309 break; 3310 case check_state_compute_run: 3311 break; 3312 default: 3313 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3314 __func__, sh->check_state, 3315 (unsigned long long) sh->sector); 3316 BUG(); 3317 } 3318 } 3319 3320 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 3321 { 3322 int i; 3323 3324 /* We have read all the blocks in this stripe and now we need to 3325 * copy some of them into a target stripe for expand. 3326 */ 3327 struct dma_async_tx_descriptor *tx = NULL; 3328 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3329 for (i = 0; i < sh->disks; i++) 3330 if (i != sh->pd_idx && i != sh->qd_idx) { 3331 int dd_idx, j; 3332 struct stripe_head *sh2; 3333 struct async_submit_ctl submit; 3334 3335 sector_t bn = compute_blocknr(sh, i, 1); 3336 sector_t s = raid5_compute_sector(conf, bn, 0, 3337 &dd_idx, NULL); 3338 sh2 = get_active_stripe(conf, s, 0, 1, 1); 3339 if (sh2 == NULL) 3340 /* so far only the early blocks of this stripe 3341 * have been requested. When later blocks 3342 * get requested, we will try again 3343 */ 3344 continue; 3345 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 3346 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 3347 /* must have already done this block */ 3348 release_stripe(sh2); 3349 continue; 3350 } 3351 3352 /* place all the copies on one channel */ 3353 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 3354 tx = async_memcpy(sh2->dev[dd_idx].page, 3355 sh->dev[i].page, 0, 0, STRIPE_SIZE, 3356 &submit); 3357 3358 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 3359 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 3360 for (j = 0; j < conf->raid_disks; j++) 3361 if (j != sh2->pd_idx && 3362 j != sh2->qd_idx && 3363 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 3364 break; 3365 if (j == conf->raid_disks) { 3366 set_bit(STRIPE_EXPAND_READY, &sh2->state); 3367 set_bit(STRIPE_HANDLE, &sh2->state); 3368 } 3369 release_stripe(sh2); 3370 3371 } 3372 /* done submitting copies, wait for them to complete */ 3373 async_tx_quiesce(&tx); 3374 } 3375 3376 /* 3377 * handle_stripe - do things to a stripe. 3378 * 3379 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 3380 * state of various bits to see what needs to be done. 3381 * Possible results: 3382 * return some read requests which now have data 3383 * return some write requests which are safely on storage 3384 * schedule a read on some buffers 3385 * schedule a write of some buffers 3386 * return confirmation of parity correctness 3387 * 3388 */ 3389 3390 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 3391 { 3392 struct r5conf *conf = sh->raid_conf; 3393 int disks = sh->disks; 3394 struct r5dev *dev; 3395 int i; 3396 int do_recovery = 0; 3397 3398 memset(s, 0, sizeof(*s)); 3399 3400 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3401 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3402 s->failed_num[0] = -1; 3403 s->failed_num[1] = -1; 3404 3405 /* Now to look around and see what can be done */ 3406 rcu_read_lock(); 3407 for (i=disks; i--; ) { 3408 struct md_rdev *rdev; 3409 sector_t first_bad; 3410 int bad_sectors; 3411 int is_bad = 0; 3412 3413 dev = &sh->dev[i]; 3414 3415 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3416 i, dev->flags, 3417 dev->toread, dev->towrite, dev->written); 3418 /* maybe we can reply to a read 3419 * 3420 * new wantfill requests are only permitted while 3421 * ops_complete_biofill is guaranteed to be inactive 3422 */ 3423 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3424 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3425 set_bit(R5_Wantfill, &dev->flags); 3426 3427 /* now count some things */ 3428 if (test_bit(R5_LOCKED, &dev->flags)) 3429 s->locked++; 3430 if (test_bit(R5_UPTODATE, &dev->flags)) 3431 s->uptodate++; 3432 if (test_bit(R5_Wantcompute, &dev->flags)) { 3433 s->compute++; 3434 BUG_ON(s->compute > 2); 3435 } 3436 3437 if (test_bit(R5_Wantfill, &dev->flags)) 3438 s->to_fill++; 3439 else if (dev->toread) 3440 s->to_read++; 3441 if (dev->towrite) { 3442 s->to_write++; 3443 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3444 s->non_overwrite++; 3445 } 3446 if (dev->written) 3447 s->written++; 3448 /* Prefer to use the replacement for reads, but only 3449 * if it is recovered enough and has no bad blocks. 3450 */ 3451 rdev = rcu_dereference(conf->disks[i].replacement); 3452 if (rdev && !test_bit(Faulty, &rdev->flags) && 3453 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 3454 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3455 &first_bad, &bad_sectors)) 3456 set_bit(R5_ReadRepl, &dev->flags); 3457 else { 3458 if (rdev) 3459 set_bit(R5_NeedReplace, &dev->flags); 3460 rdev = rcu_dereference(conf->disks[i].rdev); 3461 clear_bit(R5_ReadRepl, &dev->flags); 3462 } 3463 if (rdev && test_bit(Faulty, &rdev->flags)) 3464 rdev = NULL; 3465 if (rdev) { 3466 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3467 &first_bad, &bad_sectors); 3468 if (s->blocked_rdev == NULL 3469 && (test_bit(Blocked, &rdev->flags) 3470 || is_bad < 0)) { 3471 if (is_bad < 0) 3472 set_bit(BlockedBadBlocks, 3473 &rdev->flags); 3474 s->blocked_rdev = rdev; 3475 atomic_inc(&rdev->nr_pending); 3476 } 3477 } 3478 clear_bit(R5_Insync, &dev->flags); 3479 if (!rdev) 3480 /* Not in-sync */; 3481 else if (is_bad) { 3482 /* also not in-sync */ 3483 if (!test_bit(WriteErrorSeen, &rdev->flags) && 3484 test_bit(R5_UPTODATE, &dev->flags)) { 3485 /* treat as in-sync, but with a read error 3486 * which we can now try to correct 3487 */ 3488 set_bit(R5_Insync, &dev->flags); 3489 set_bit(R5_ReadError, &dev->flags); 3490 } 3491 } else if (test_bit(In_sync, &rdev->flags)) 3492 set_bit(R5_Insync, &dev->flags); 3493 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3494 /* in sync if before recovery_offset */ 3495 set_bit(R5_Insync, &dev->flags); 3496 else if (test_bit(R5_UPTODATE, &dev->flags) && 3497 test_bit(R5_Expanded, &dev->flags)) 3498 /* If we've reshaped into here, we assume it is Insync. 3499 * We will shortly update recovery_offset to make 3500 * it official. 3501 */ 3502 set_bit(R5_Insync, &dev->flags); 3503 3504 if (rdev && test_bit(R5_WriteError, &dev->flags)) { 3505 /* This flag does not apply to '.replacement' 3506 * only to .rdev, so make sure to check that*/ 3507 struct md_rdev *rdev2 = rcu_dereference( 3508 conf->disks[i].rdev); 3509 if (rdev2 == rdev) 3510 clear_bit(R5_Insync, &dev->flags); 3511 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3512 s->handle_bad_blocks = 1; 3513 atomic_inc(&rdev2->nr_pending); 3514 } else 3515 clear_bit(R5_WriteError, &dev->flags); 3516 } 3517 if (rdev && test_bit(R5_MadeGood, &dev->flags)) { 3518 /* This flag does not apply to '.replacement' 3519 * only to .rdev, so make sure to check that*/ 3520 struct md_rdev *rdev2 = rcu_dereference( 3521 conf->disks[i].rdev); 3522 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3523 s->handle_bad_blocks = 1; 3524 atomic_inc(&rdev2->nr_pending); 3525 } else 3526 clear_bit(R5_MadeGood, &dev->flags); 3527 } 3528 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 3529 struct md_rdev *rdev2 = rcu_dereference( 3530 conf->disks[i].replacement); 3531 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3532 s->handle_bad_blocks = 1; 3533 atomic_inc(&rdev2->nr_pending); 3534 } else 3535 clear_bit(R5_MadeGoodRepl, &dev->flags); 3536 } 3537 if (!test_bit(R5_Insync, &dev->flags)) { 3538 /* The ReadError flag will just be confusing now */ 3539 clear_bit(R5_ReadError, &dev->flags); 3540 clear_bit(R5_ReWrite, &dev->flags); 3541 } 3542 if (test_bit(R5_ReadError, &dev->flags)) 3543 clear_bit(R5_Insync, &dev->flags); 3544 if (!test_bit(R5_Insync, &dev->flags)) { 3545 if (s->failed < 2) 3546 s->failed_num[s->failed] = i; 3547 s->failed++; 3548 if (rdev && !test_bit(Faulty, &rdev->flags)) 3549 do_recovery = 1; 3550 } 3551 } 3552 if (test_bit(STRIPE_SYNCING, &sh->state)) { 3553 /* If there is a failed device being replaced, 3554 * we must be recovering. 3555 * else if we are after recovery_cp, we must be syncing 3556 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 3557 * else we can only be replacing 3558 * sync and recovery both need to read all devices, and so 3559 * use the same flag. 3560 */ 3561 if (do_recovery || 3562 sh->sector >= conf->mddev->recovery_cp || 3563 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 3564 s->syncing = 1; 3565 else 3566 s->replacing = 1; 3567 } 3568 rcu_read_unlock(); 3569 } 3570 3571 static void handle_stripe(struct stripe_head *sh) 3572 { 3573 struct stripe_head_state s; 3574 struct r5conf *conf = sh->raid_conf; 3575 int i; 3576 int prexor; 3577 int disks = sh->disks; 3578 struct r5dev *pdev, *qdev; 3579 3580 clear_bit(STRIPE_HANDLE, &sh->state); 3581 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 3582 /* already being handled, ensure it gets handled 3583 * again when current action finishes */ 3584 set_bit(STRIPE_HANDLE, &sh->state); 3585 return; 3586 } 3587 3588 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3589 spin_lock(&sh->stripe_lock); 3590 /* Cannot process 'sync' concurrently with 'discard' */ 3591 if (!test_bit(STRIPE_DISCARD, &sh->state) && 3592 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3593 set_bit(STRIPE_SYNCING, &sh->state); 3594 clear_bit(STRIPE_INSYNC, &sh->state); 3595 clear_bit(STRIPE_REPLACED, &sh->state); 3596 } 3597 spin_unlock(&sh->stripe_lock); 3598 } 3599 clear_bit(STRIPE_DELAYED, &sh->state); 3600 3601 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3602 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3603 (unsigned long long)sh->sector, sh->state, 3604 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 3605 sh->check_state, sh->reconstruct_state); 3606 3607 analyse_stripe(sh, &s); 3608 3609 if (s.handle_bad_blocks) { 3610 set_bit(STRIPE_HANDLE, &sh->state); 3611 goto finish; 3612 } 3613 3614 if (unlikely(s.blocked_rdev)) { 3615 if (s.syncing || s.expanding || s.expanded || 3616 s.replacing || s.to_write || s.written) { 3617 set_bit(STRIPE_HANDLE, &sh->state); 3618 goto finish; 3619 } 3620 /* There is nothing for the blocked_rdev to block */ 3621 rdev_dec_pending(s.blocked_rdev, conf->mddev); 3622 s.blocked_rdev = NULL; 3623 } 3624 3625 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3626 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3627 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3628 } 3629 3630 pr_debug("locked=%d uptodate=%d to_read=%d" 3631 " to_write=%d failed=%d failed_num=%d,%d\n", 3632 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3633 s.failed_num[0], s.failed_num[1]); 3634 /* check if the array has lost more than max_degraded devices and, 3635 * if so, some requests might need to be failed. 3636 */ 3637 if (s.failed > conf->max_degraded) { 3638 sh->check_state = 0; 3639 sh->reconstruct_state = 0; 3640 if (s.to_read+s.to_write+s.written) 3641 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 3642 if (s.syncing + s.replacing) 3643 handle_failed_sync(conf, sh, &s); 3644 } 3645 3646 /* Now we check to see if any write operations have recently 3647 * completed 3648 */ 3649 prexor = 0; 3650 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3651 prexor = 1; 3652 if (sh->reconstruct_state == reconstruct_state_drain_result || 3653 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3654 sh->reconstruct_state = reconstruct_state_idle; 3655 3656 /* All the 'written' buffers and the parity block are ready to 3657 * be written back to disk 3658 */ 3659 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && 3660 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); 3661 BUG_ON(sh->qd_idx >= 0 && 3662 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && 3663 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); 3664 for (i = disks; i--; ) { 3665 struct r5dev *dev = &sh->dev[i]; 3666 if (test_bit(R5_LOCKED, &dev->flags) && 3667 (i == sh->pd_idx || i == sh->qd_idx || 3668 dev->written)) { 3669 pr_debug("Writing block %d\n", i); 3670 set_bit(R5_Wantwrite, &dev->flags); 3671 if (prexor) 3672 continue; 3673 if (!test_bit(R5_Insync, &dev->flags) || 3674 ((i == sh->pd_idx || i == sh->qd_idx) && 3675 s.failed == 0)) 3676 set_bit(STRIPE_INSYNC, &sh->state); 3677 } 3678 } 3679 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3680 s.dec_preread_active = 1; 3681 } 3682 3683 /* 3684 * might be able to return some write requests if the parity blocks 3685 * are safe, or on a failed drive 3686 */ 3687 pdev = &sh->dev[sh->pd_idx]; 3688 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 3689 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 3690 qdev = &sh->dev[sh->qd_idx]; 3691 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 3692 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 3693 || conf->level < 6; 3694 3695 if (s.written && 3696 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3697 && !test_bit(R5_LOCKED, &pdev->flags) 3698 && (test_bit(R5_UPTODATE, &pdev->flags) || 3699 test_bit(R5_Discard, &pdev->flags))))) && 3700 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3701 && !test_bit(R5_LOCKED, &qdev->flags) 3702 && (test_bit(R5_UPTODATE, &qdev->flags) || 3703 test_bit(R5_Discard, &qdev->flags)))))) 3704 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 3705 3706 /* Now we might consider reading some blocks, either to check/generate 3707 * parity, or to satisfy requests 3708 * or to load a block that is being partially written. 3709 */ 3710 if (s.to_read || s.non_overwrite 3711 || (conf->level == 6 && s.to_write && s.failed) 3712 || (s.syncing && (s.uptodate + s.compute < disks)) 3713 || s.replacing 3714 || s.expanding) 3715 handle_stripe_fill(sh, &s, disks); 3716 3717 /* Now to consider new write requests and what else, if anything 3718 * should be read. We do not handle new writes when: 3719 * 1/ A 'write' operation (copy+xor) is already in flight. 3720 * 2/ A 'check' operation is in flight, as it may clobber the parity 3721 * block. 3722 */ 3723 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3724 handle_stripe_dirtying(conf, sh, &s, disks); 3725 3726 /* maybe we need to check and possibly fix the parity for this stripe 3727 * Any reads will already have been scheduled, so we just see if enough 3728 * data is available. The parity check is held off while parity 3729 * dependent operations are in flight. 3730 */ 3731 if (sh->check_state || 3732 (s.syncing && s.locked == 0 && 3733 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3734 !test_bit(STRIPE_INSYNC, &sh->state))) { 3735 if (conf->level == 6) 3736 handle_parity_checks6(conf, sh, &s, disks); 3737 else 3738 handle_parity_checks5(conf, sh, &s, disks); 3739 } 3740 3741 if ((s.replacing || s.syncing) && s.locked == 0 3742 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) 3743 && !test_bit(STRIPE_REPLACED, &sh->state)) { 3744 /* Write out to replacement devices where possible */ 3745 for (i = 0; i < conf->raid_disks; i++) 3746 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 3747 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); 3748 set_bit(R5_WantReplace, &sh->dev[i].flags); 3749 set_bit(R5_LOCKED, &sh->dev[i].flags); 3750 s.locked++; 3751 } 3752 if (s.replacing) 3753 set_bit(STRIPE_INSYNC, &sh->state); 3754 set_bit(STRIPE_REPLACED, &sh->state); 3755 } 3756 if ((s.syncing || s.replacing) && s.locked == 0 && 3757 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3758 test_bit(STRIPE_INSYNC, &sh->state)) { 3759 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3760 clear_bit(STRIPE_SYNCING, &sh->state); 3761 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 3762 wake_up(&conf->wait_for_overlap); 3763 } 3764 3765 /* If the failed drives are just a ReadError, then we might need 3766 * to progress the repair/check process 3767 */ 3768 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 3769 for (i = 0; i < s.failed; i++) { 3770 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 3771 if (test_bit(R5_ReadError, &dev->flags) 3772 && !test_bit(R5_LOCKED, &dev->flags) 3773 && test_bit(R5_UPTODATE, &dev->flags) 3774 ) { 3775 if (!test_bit(R5_ReWrite, &dev->flags)) { 3776 set_bit(R5_Wantwrite, &dev->flags); 3777 set_bit(R5_ReWrite, &dev->flags); 3778 set_bit(R5_LOCKED, &dev->flags); 3779 s.locked++; 3780 } else { 3781 /* let's read it back */ 3782 set_bit(R5_Wantread, &dev->flags); 3783 set_bit(R5_LOCKED, &dev->flags); 3784 s.locked++; 3785 } 3786 } 3787 } 3788 3789 3790 /* Finish reconstruct operations initiated by the expansion process */ 3791 if (sh->reconstruct_state == reconstruct_state_result) { 3792 struct stripe_head *sh_src 3793 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3794 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 3795 /* sh cannot be written until sh_src has been read. 3796 * so arrange for sh to be delayed a little 3797 */ 3798 set_bit(STRIPE_DELAYED, &sh->state); 3799 set_bit(STRIPE_HANDLE, &sh->state); 3800 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3801 &sh_src->state)) 3802 atomic_inc(&conf->preread_active_stripes); 3803 release_stripe(sh_src); 3804 goto finish; 3805 } 3806 if (sh_src) 3807 release_stripe(sh_src); 3808 3809 sh->reconstruct_state = reconstruct_state_idle; 3810 clear_bit(STRIPE_EXPANDING, &sh->state); 3811 for (i = conf->raid_disks; i--; ) { 3812 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3813 set_bit(R5_LOCKED, &sh->dev[i].flags); 3814 s.locked++; 3815 } 3816 } 3817 3818 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3819 !sh->reconstruct_state) { 3820 /* Need to write out all blocks after computing parity */ 3821 sh->disks = conf->raid_disks; 3822 stripe_set_idx(sh->sector, conf, 0, sh); 3823 schedule_reconstruction(sh, &s, 1, 1); 3824 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3825 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3826 atomic_dec(&conf->reshape_stripes); 3827 wake_up(&conf->wait_for_overlap); 3828 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3829 } 3830 3831 if (s.expanding && s.locked == 0 && 3832 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3833 handle_stripe_expansion(conf, sh); 3834 3835 finish: 3836 /* wait for this device to become unblocked */ 3837 if (unlikely(s.blocked_rdev)) { 3838 if (conf->mddev->external) 3839 md_wait_for_blocked_rdev(s.blocked_rdev, 3840 conf->mddev); 3841 else 3842 /* Internal metadata will immediately 3843 * be written by raid5d, so we don't 3844 * need to wait here. 3845 */ 3846 rdev_dec_pending(s.blocked_rdev, 3847 conf->mddev); 3848 } 3849 3850 if (s.handle_bad_blocks) 3851 for (i = disks; i--; ) { 3852 struct md_rdev *rdev; 3853 struct r5dev *dev = &sh->dev[i]; 3854 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 3855 /* We own a safe reference to the rdev */ 3856 rdev = conf->disks[i].rdev; 3857 if (!rdev_set_badblocks(rdev, sh->sector, 3858 STRIPE_SECTORS, 0)) 3859 md_error(conf->mddev, rdev); 3860 rdev_dec_pending(rdev, conf->mddev); 3861 } 3862 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 3863 rdev = conf->disks[i].rdev; 3864 rdev_clear_badblocks(rdev, sh->sector, 3865 STRIPE_SECTORS, 0); 3866 rdev_dec_pending(rdev, conf->mddev); 3867 } 3868 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 3869 rdev = conf->disks[i].replacement; 3870 if (!rdev) 3871 /* rdev have been moved down */ 3872 rdev = conf->disks[i].rdev; 3873 rdev_clear_badblocks(rdev, sh->sector, 3874 STRIPE_SECTORS, 0); 3875 rdev_dec_pending(rdev, conf->mddev); 3876 } 3877 } 3878 3879 if (s.ops_request) 3880 raid_run_ops(sh, s.ops_request); 3881 3882 ops_run_io(sh, &s); 3883 3884 if (s.dec_preread_active) { 3885 /* We delay this until after ops_run_io so that if make_request 3886 * is waiting on a flush, it won't continue until the writes 3887 * have actually been submitted. 3888 */ 3889 atomic_dec(&conf->preread_active_stripes); 3890 if (atomic_read(&conf->preread_active_stripes) < 3891 IO_THRESHOLD) 3892 md_wakeup_thread(conf->mddev->thread); 3893 } 3894 3895 return_io(s.return_bi); 3896 3897 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 3898 } 3899 3900 static void raid5_activate_delayed(struct r5conf *conf) 3901 { 3902 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 3903 while (!list_empty(&conf->delayed_list)) { 3904 struct list_head *l = conf->delayed_list.next; 3905 struct stripe_head *sh; 3906 sh = list_entry(l, struct stripe_head, lru); 3907 list_del_init(l); 3908 clear_bit(STRIPE_DELAYED, &sh->state); 3909 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3910 atomic_inc(&conf->preread_active_stripes); 3911 list_add_tail(&sh->lru, &conf->hold_list); 3912 raid5_wakeup_stripe_thread(sh); 3913 } 3914 } 3915 } 3916 3917 static void activate_bit_delay(struct r5conf *conf) 3918 { 3919 /* device_lock is held */ 3920 struct list_head head; 3921 list_add(&head, &conf->bitmap_list); 3922 list_del_init(&conf->bitmap_list); 3923 while (!list_empty(&head)) { 3924 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 3925 list_del_init(&sh->lru); 3926 atomic_inc(&sh->count); 3927 __release_stripe(conf, sh); 3928 } 3929 } 3930 3931 int md_raid5_congested(struct mddev *mddev, int bits) 3932 { 3933 struct r5conf *conf = mddev->private; 3934 3935 /* No difference between reads and writes. Just check 3936 * how busy the stripe_cache is 3937 */ 3938 3939 if (conf->inactive_blocked) 3940 return 1; 3941 if (conf->quiesce) 3942 return 1; 3943 if (list_empty_careful(&conf->inactive_list)) 3944 return 1; 3945 3946 return 0; 3947 } 3948 EXPORT_SYMBOL_GPL(md_raid5_congested); 3949 3950 static int raid5_congested(void *data, int bits) 3951 { 3952 struct mddev *mddev = data; 3953 3954 return mddev_congested(mddev, bits) || 3955 md_raid5_congested(mddev, bits); 3956 } 3957 3958 /* We want read requests to align with chunks where possible, 3959 * but write requests don't need to. 3960 */ 3961 static int raid5_mergeable_bvec(struct request_queue *q, 3962 struct bvec_merge_data *bvm, 3963 struct bio_vec *biovec) 3964 { 3965 struct mddev *mddev = q->queuedata; 3966 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 3967 int max; 3968 unsigned int chunk_sectors = mddev->chunk_sectors; 3969 unsigned int bio_sectors = bvm->bi_size >> 9; 3970 3971 if ((bvm->bi_rw & 1) == WRITE) 3972 return biovec->bv_len; /* always allow writes to be mergeable */ 3973 3974 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3975 chunk_sectors = mddev->new_chunk_sectors; 3976 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 3977 if (max < 0) max = 0; 3978 if (max <= biovec->bv_len && bio_sectors == 0) 3979 return biovec->bv_len; 3980 else 3981 return max; 3982 } 3983 3984 3985 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 3986 { 3987 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev); 3988 unsigned int chunk_sectors = mddev->chunk_sectors; 3989 unsigned int bio_sectors = bio_sectors(bio); 3990 3991 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 3992 chunk_sectors = mddev->new_chunk_sectors; 3993 return chunk_sectors >= 3994 ((sector & (chunk_sectors - 1)) + bio_sectors); 3995 } 3996 3997 /* 3998 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 3999 * later sampled by raid5d. 4000 */ 4001 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 4002 { 4003 unsigned long flags; 4004 4005 spin_lock_irqsave(&conf->device_lock, flags); 4006 4007 bi->bi_next = conf->retry_read_aligned_list; 4008 conf->retry_read_aligned_list = bi; 4009 4010 spin_unlock_irqrestore(&conf->device_lock, flags); 4011 md_wakeup_thread(conf->mddev->thread); 4012 } 4013 4014 4015 static struct bio *remove_bio_from_retry(struct r5conf *conf) 4016 { 4017 struct bio *bi; 4018 4019 bi = conf->retry_read_aligned; 4020 if (bi) { 4021 conf->retry_read_aligned = NULL; 4022 return bi; 4023 } 4024 bi = conf->retry_read_aligned_list; 4025 if(bi) { 4026 conf->retry_read_aligned_list = bi->bi_next; 4027 bi->bi_next = NULL; 4028 /* 4029 * this sets the active strip count to 1 and the processed 4030 * strip count to zero (upper 8 bits) 4031 */ 4032 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ 4033 } 4034 4035 return bi; 4036 } 4037 4038 4039 /* 4040 * The "raid5_align_endio" should check if the read succeeded and if it 4041 * did, call bio_endio on the original bio (having bio_put the new bio 4042 * first). 4043 * If the read failed.. 4044 */ 4045 static void raid5_align_endio(struct bio *bi, int error) 4046 { 4047 struct bio* raid_bi = bi->bi_private; 4048 struct mddev *mddev; 4049 struct r5conf *conf; 4050 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 4051 struct md_rdev *rdev; 4052 4053 bio_put(bi); 4054 4055 rdev = (void*)raid_bi->bi_next; 4056 raid_bi->bi_next = NULL; 4057 mddev = rdev->mddev; 4058 conf = mddev->private; 4059 4060 rdev_dec_pending(rdev, conf->mddev); 4061 4062 if (!error && uptodate) { 4063 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4064 raid_bi, 0); 4065 bio_endio(raid_bi, 0); 4066 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4067 wake_up(&conf->wait_for_stripe); 4068 return; 4069 } 4070 4071 4072 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 4073 4074 add_bio_to_retry(raid_bi, conf); 4075 } 4076 4077 static int bio_fits_rdev(struct bio *bi) 4078 { 4079 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 4080 4081 if (bio_sectors(bi) > queue_max_sectors(q)) 4082 return 0; 4083 blk_recount_segments(q, bi); 4084 if (bi->bi_phys_segments > queue_max_segments(q)) 4085 return 0; 4086 4087 if (q->merge_bvec_fn) 4088 /* it's too hard to apply the merge_bvec_fn at this stage, 4089 * just just give up 4090 */ 4091 return 0; 4092 4093 return 1; 4094 } 4095 4096 4097 static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) 4098 { 4099 struct r5conf *conf = mddev->private; 4100 int dd_idx; 4101 struct bio* align_bi; 4102 struct md_rdev *rdev; 4103 sector_t end_sector; 4104 4105 if (!in_chunk_boundary(mddev, raid_bio)) { 4106 pr_debug("chunk_aligned_read : non aligned\n"); 4107 return 0; 4108 } 4109 /* 4110 * use bio_clone_mddev to make a copy of the bio 4111 */ 4112 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 4113 if (!align_bi) 4114 return 0; 4115 /* 4116 * set bi_end_io to a new function, and set bi_private to the 4117 * original bio. 4118 */ 4119 align_bi->bi_end_io = raid5_align_endio; 4120 align_bi->bi_private = raid_bio; 4121 /* 4122 * compute position 4123 */ 4124 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector, 4125 0, 4126 &dd_idx, NULL); 4127 4128 end_sector = bio_end_sector(align_bi); 4129 rcu_read_lock(); 4130 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 4131 if (!rdev || test_bit(Faulty, &rdev->flags) || 4132 rdev->recovery_offset < end_sector) { 4133 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 4134 if (rdev && 4135 (test_bit(Faulty, &rdev->flags) || 4136 !(test_bit(In_sync, &rdev->flags) || 4137 rdev->recovery_offset >= end_sector))) 4138 rdev = NULL; 4139 } 4140 if (rdev) { 4141 sector_t first_bad; 4142 int bad_sectors; 4143 4144 atomic_inc(&rdev->nr_pending); 4145 rcu_read_unlock(); 4146 raid_bio->bi_next = (void*)rdev; 4147 align_bi->bi_bdev = rdev->bdev; 4148 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID); 4149 4150 if (!bio_fits_rdev(align_bi) || 4151 is_badblock(rdev, align_bi->bi_sector, bio_sectors(align_bi), 4152 &first_bad, &bad_sectors)) { 4153 /* too big in some way, or has a known bad block */ 4154 bio_put(align_bi); 4155 rdev_dec_pending(rdev, mddev); 4156 return 0; 4157 } 4158 4159 /* No reshape active, so we can trust rdev->data_offset */ 4160 align_bi->bi_sector += rdev->data_offset; 4161 4162 spin_lock_irq(&conf->device_lock); 4163 wait_event_lock_irq(conf->wait_for_stripe, 4164 conf->quiesce == 0, 4165 conf->device_lock); 4166 atomic_inc(&conf->active_aligned_reads); 4167 spin_unlock_irq(&conf->device_lock); 4168 4169 if (mddev->gendisk) 4170 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), 4171 align_bi, disk_devt(mddev->gendisk), 4172 raid_bio->bi_sector); 4173 generic_make_request(align_bi); 4174 return 1; 4175 } else { 4176 rcu_read_unlock(); 4177 bio_put(align_bi); 4178 return 0; 4179 } 4180 } 4181 4182 /* __get_priority_stripe - get the next stripe to process 4183 * 4184 * Full stripe writes are allowed to pass preread active stripes up until 4185 * the bypass_threshold is exceeded. In general the bypass_count 4186 * increments when the handle_list is handled before the hold_list; however, it 4187 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 4188 * stripe with in flight i/o. The bypass_count will be reset when the 4189 * head of the hold_list has changed, i.e. the head was promoted to the 4190 * handle_list. 4191 */ 4192 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) 4193 { 4194 struct stripe_head *sh = NULL, *tmp; 4195 struct list_head *handle_list = NULL; 4196 struct r5worker_group *wg = NULL; 4197 4198 if (conf->worker_cnt_per_group == 0) { 4199 handle_list = &conf->handle_list; 4200 } else if (group != ANY_GROUP) { 4201 handle_list = &conf->worker_groups[group].handle_list; 4202 wg = &conf->worker_groups[group]; 4203 } else { 4204 int i; 4205 for (i = 0; i < conf->group_cnt; i++) { 4206 handle_list = &conf->worker_groups[i].handle_list; 4207 wg = &conf->worker_groups[i]; 4208 if (!list_empty(handle_list)) 4209 break; 4210 } 4211 } 4212 4213 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 4214 __func__, 4215 list_empty(handle_list) ? "empty" : "busy", 4216 list_empty(&conf->hold_list) ? "empty" : "busy", 4217 atomic_read(&conf->pending_full_writes), conf->bypass_count); 4218 4219 if (!list_empty(handle_list)) { 4220 sh = list_entry(handle_list->next, typeof(*sh), lru); 4221 4222 if (list_empty(&conf->hold_list)) 4223 conf->bypass_count = 0; 4224 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 4225 if (conf->hold_list.next == conf->last_hold) 4226 conf->bypass_count++; 4227 else { 4228 conf->last_hold = conf->hold_list.next; 4229 conf->bypass_count -= conf->bypass_threshold; 4230 if (conf->bypass_count < 0) 4231 conf->bypass_count = 0; 4232 } 4233 } 4234 } else if (!list_empty(&conf->hold_list) && 4235 ((conf->bypass_threshold && 4236 conf->bypass_count > conf->bypass_threshold) || 4237 atomic_read(&conf->pending_full_writes) == 0)) { 4238 4239 list_for_each_entry(tmp, &conf->hold_list, lru) { 4240 if (conf->worker_cnt_per_group == 0 || 4241 group == ANY_GROUP || 4242 !cpu_online(tmp->cpu) || 4243 cpu_to_group(tmp->cpu) == group) { 4244 sh = tmp; 4245 break; 4246 } 4247 } 4248 4249 if (sh) { 4250 conf->bypass_count -= conf->bypass_threshold; 4251 if (conf->bypass_count < 0) 4252 conf->bypass_count = 0; 4253 } 4254 wg = NULL; 4255 } 4256 4257 if (!sh) 4258 return NULL; 4259 4260 if (wg) { 4261 wg->stripes_cnt--; 4262 sh->group = NULL; 4263 } 4264 list_del_init(&sh->lru); 4265 atomic_inc(&sh->count); 4266 BUG_ON(atomic_read(&sh->count) != 1); 4267 return sh; 4268 } 4269 4270 struct raid5_plug_cb { 4271 struct blk_plug_cb cb; 4272 struct list_head list; 4273 }; 4274 4275 static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) 4276 { 4277 struct raid5_plug_cb *cb = container_of( 4278 blk_cb, struct raid5_plug_cb, cb); 4279 struct stripe_head *sh; 4280 struct mddev *mddev = cb->cb.data; 4281 struct r5conf *conf = mddev->private; 4282 int cnt = 0; 4283 4284 if (cb->list.next && !list_empty(&cb->list)) { 4285 spin_lock_irq(&conf->device_lock); 4286 while (!list_empty(&cb->list)) { 4287 sh = list_first_entry(&cb->list, struct stripe_head, lru); 4288 list_del_init(&sh->lru); 4289 /* 4290 * avoid race release_stripe_plug() sees 4291 * STRIPE_ON_UNPLUG_LIST clear but the stripe 4292 * is still in our list 4293 */ 4294 smp_mb__before_clear_bit(); 4295 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); 4296 /* 4297 * STRIPE_ON_RELEASE_LIST could be set here. In that 4298 * case, the count is always > 1 here 4299 */ 4300 __release_stripe(conf, sh); 4301 cnt++; 4302 } 4303 spin_unlock_irq(&conf->device_lock); 4304 } 4305 if (mddev->queue) 4306 trace_block_unplug(mddev->queue, cnt, !from_schedule); 4307 kfree(cb); 4308 } 4309 4310 static void release_stripe_plug(struct mddev *mddev, 4311 struct stripe_head *sh) 4312 { 4313 struct blk_plug_cb *blk_cb = blk_check_plugged( 4314 raid5_unplug, mddev, 4315 sizeof(struct raid5_plug_cb)); 4316 struct raid5_plug_cb *cb; 4317 4318 if (!blk_cb) { 4319 release_stripe(sh); 4320 return; 4321 } 4322 4323 cb = container_of(blk_cb, struct raid5_plug_cb, cb); 4324 4325 if (cb->list.next == NULL) 4326 INIT_LIST_HEAD(&cb->list); 4327 4328 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) 4329 list_add_tail(&sh->lru, &cb->list); 4330 else 4331 release_stripe(sh); 4332 } 4333 4334 static void make_discard_request(struct mddev *mddev, struct bio *bi) 4335 { 4336 struct r5conf *conf = mddev->private; 4337 sector_t logical_sector, last_sector; 4338 struct stripe_head *sh; 4339 int remaining; 4340 int stripe_sectors; 4341 4342 if (mddev->reshape_position != MaxSector) 4343 /* Skip discard while reshape is happening */ 4344 return; 4345 4346 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4347 last_sector = bi->bi_sector + (bi->bi_size>>9); 4348 4349 bi->bi_next = NULL; 4350 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4351 4352 stripe_sectors = conf->chunk_sectors * 4353 (conf->raid_disks - conf->max_degraded); 4354 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, 4355 stripe_sectors); 4356 sector_div(last_sector, stripe_sectors); 4357 4358 logical_sector *= conf->chunk_sectors; 4359 last_sector *= conf->chunk_sectors; 4360 4361 for (; logical_sector < last_sector; 4362 logical_sector += STRIPE_SECTORS) { 4363 DEFINE_WAIT(w); 4364 int d; 4365 again: 4366 sh = get_active_stripe(conf, logical_sector, 0, 0, 0); 4367 prepare_to_wait(&conf->wait_for_overlap, &w, 4368 TASK_UNINTERRUPTIBLE); 4369 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4370 if (test_bit(STRIPE_SYNCING, &sh->state)) { 4371 release_stripe(sh); 4372 schedule(); 4373 goto again; 4374 } 4375 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4376 spin_lock_irq(&sh->stripe_lock); 4377 for (d = 0; d < conf->raid_disks; d++) { 4378 if (d == sh->pd_idx || d == sh->qd_idx) 4379 continue; 4380 if (sh->dev[d].towrite || sh->dev[d].toread) { 4381 set_bit(R5_Overlap, &sh->dev[d].flags); 4382 spin_unlock_irq(&sh->stripe_lock); 4383 release_stripe(sh); 4384 schedule(); 4385 goto again; 4386 } 4387 } 4388 set_bit(STRIPE_DISCARD, &sh->state); 4389 finish_wait(&conf->wait_for_overlap, &w); 4390 for (d = 0; d < conf->raid_disks; d++) { 4391 if (d == sh->pd_idx || d == sh->qd_idx) 4392 continue; 4393 sh->dev[d].towrite = bi; 4394 set_bit(R5_OVERWRITE, &sh->dev[d].flags); 4395 raid5_inc_bi_active_stripes(bi); 4396 } 4397 spin_unlock_irq(&sh->stripe_lock); 4398 if (conf->mddev->bitmap) { 4399 for (d = 0; 4400 d < conf->raid_disks - conf->max_degraded; 4401 d++) 4402 bitmap_startwrite(mddev->bitmap, 4403 sh->sector, 4404 STRIPE_SECTORS, 4405 0); 4406 sh->bm_seq = conf->seq_flush + 1; 4407 set_bit(STRIPE_BIT_DELAY, &sh->state); 4408 } 4409 4410 set_bit(STRIPE_HANDLE, &sh->state); 4411 clear_bit(STRIPE_DELAYED, &sh->state); 4412 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4413 atomic_inc(&conf->preread_active_stripes); 4414 release_stripe_plug(mddev, sh); 4415 } 4416 4417 remaining = raid5_dec_bi_active_stripes(bi); 4418 if (remaining == 0) { 4419 md_write_end(mddev); 4420 bio_endio(bi, 0); 4421 } 4422 } 4423 4424 static void make_request(struct mddev *mddev, struct bio * bi) 4425 { 4426 struct r5conf *conf = mddev->private; 4427 int dd_idx; 4428 sector_t new_sector; 4429 sector_t logical_sector, last_sector; 4430 struct stripe_head *sh; 4431 const int rw = bio_data_dir(bi); 4432 int remaining; 4433 4434 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 4435 md_flush_request(mddev, bi); 4436 return; 4437 } 4438 4439 md_write_start(mddev, bi); 4440 4441 if (rw == READ && 4442 mddev->reshape_position == MaxSector && 4443 chunk_aligned_read(mddev,bi)) 4444 return; 4445 4446 if (unlikely(bi->bi_rw & REQ_DISCARD)) { 4447 make_discard_request(mddev, bi); 4448 return; 4449 } 4450 4451 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4452 last_sector = bio_end_sector(bi); 4453 bi->bi_next = NULL; 4454 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4455 4456 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 4457 DEFINE_WAIT(w); 4458 int previous; 4459 int seq; 4460 4461 retry: 4462 seq = read_seqcount_begin(&conf->gen_lock); 4463 previous = 0; 4464 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 4465 if (unlikely(conf->reshape_progress != MaxSector)) { 4466 /* spinlock is needed as reshape_progress may be 4467 * 64bit on a 32bit platform, and so it might be 4468 * possible to see a half-updated value 4469 * Of course reshape_progress could change after 4470 * the lock is dropped, so once we get a reference 4471 * to the stripe that we think it is, we will have 4472 * to check again. 4473 */ 4474 spin_lock_irq(&conf->device_lock); 4475 if (mddev->reshape_backwards 4476 ? logical_sector < conf->reshape_progress 4477 : logical_sector >= conf->reshape_progress) { 4478 previous = 1; 4479 } else { 4480 if (mddev->reshape_backwards 4481 ? logical_sector < conf->reshape_safe 4482 : logical_sector >= conf->reshape_safe) { 4483 spin_unlock_irq(&conf->device_lock); 4484 schedule(); 4485 goto retry; 4486 } 4487 } 4488 spin_unlock_irq(&conf->device_lock); 4489 } 4490 4491 new_sector = raid5_compute_sector(conf, logical_sector, 4492 previous, 4493 &dd_idx, NULL); 4494 pr_debug("raid456: make_request, sector %llu logical %llu\n", 4495 (unsigned long long)new_sector, 4496 (unsigned long long)logical_sector); 4497 4498 sh = get_active_stripe(conf, new_sector, previous, 4499 (bi->bi_rw&RWA_MASK), 0); 4500 if (sh) { 4501 if (unlikely(previous)) { 4502 /* expansion might have moved on while waiting for a 4503 * stripe, so we must do the range check again. 4504 * Expansion could still move past after this 4505 * test, but as we are holding a reference to 4506 * 'sh', we know that if that happens, 4507 * STRIPE_EXPANDING will get set and the expansion 4508 * won't proceed until we finish with the stripe. 4509 */ 4510 int must_retry = 0; 4511 spin_lock_irq(&conf->device_lock); 4512 if (mddev->reshape_backwards 4513 ? logical_sector >= conf->reshape_progress 4514 : logical_sector < conf->reshape_progress) 4515 /* mismatch, need to try again */ 4516 must_retry = 1; 4517 spin_unlock_irq(&conf->device_lock); 4518 if (must_retry) { 4519 release_stripe(sh); 4520 schedule(); 4521 goto retry; 4522 } 4523 } 4524 if (read_seqcount_retry(&conf->gen_lock, seq)) { 4525 /* Might have got the wrong stripe_head 4526 * by accident 4527 */ 4528 release_stripe(sh); 4529 goto retry; 4530 } 4531 4532 if (rw == WRITE && 4533 logical_sector >= mddev->suspend_lo && 4534 logical_sector < mddev->suspend_hi) { 4535 release_stripe(sh); 4536 /* As the suspend_* range is controlled by 4537 * userspace, we want an interruptible 4538 * wait. 4539 */ 4540 flush_signals(current); 4541 prepare_to_wait(&conf->wait_for_overlap, 4542 &w, TASK_INTERRUPTIBLE); 4543 if (logical_sector >= mddev->suspend_lo && 4544 logical_sector < mddev->suspend_hi) 4545 schedule(); 4546 goto retry; 4547 } 4548 4549 if (test_bit(STRIPE_EXPANDING, &sh->state) || 4550 !add_stripe_bio(sh, bi, dd_idx, rw)) { 4551 /* Stripe is busy expanding or 4552 * add failed due to overlap. Flush everything 4553 * and wait a while 4554 */ 4555 md_wakeup_thread(mddev->thread); 4556 release_stripe(sh); 4557 schedule(); 4558 goto retry; 4559 } 4560 finish_wait(&conf->wait_for_overlap, &w); 4561 set_bit(STRIPE_HANDLE, &sh->state); 4562 clear_bit(STRIPE_DELAYED, &sh->state); 4563 if ((bi->bi_rw & REQ_SYNC) && 4564 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4565 atomic_inc(&conf->preread_active_stripes); 4566 release_stripe_plug(mddev, sh); 4567 } else { 4568 /* cannot get stripe for read-ahead, just give-up */ 4569 clear_bit(BIO_UPTODATE, &bi->bi_flags); 4570 finish_wait(&conf->wait_for_overlap, &w); 4571 break; 4572 } 4573 } 4574 4575 remaining = raid5_dec_bi_active_stripes(bi); 4576 if (remaining == 0) { 4577 4578 if ( rw == WRITE ) 4579 md_write_end(mddev); 4580 4581 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 4582 bi, 0); 4583 bio_endio(bi, 0); 4584 } 4585 } 4586 4587 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 4588 4589 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 4590 { 4591 /* reshaping is quite different to recovery/resync so it is 4592 * handled quite separately ... here. 4593 * 4594 * On each call to sync_request, we gather one chunk worth of 4595 * destination stripes and flag them as expanding. 4596 * Then we find all the source stripes and request reads. 4597 * As the reads complete, handle_stripe will copy the data 4598 * into the destination stripe and release that stripe. 4599 */ 4600 struct r5conf *conf = mddev->private; 4601 struct stripe_head *sh; 4602 sector_t first_sector, last_sector; 4603 int raid_disks = conf->previous_raid_disks; 4604 int data_disks = raid_disks - conf->max_degraded; 4605 int new_data_disks = conf->raid_disks - conf->max_degraded; 4606 int i; 4607 int dd_idx; 4608 sector_t writepos, readpos, safepos; 4609 sector_t stripe_addr; 4610 int reshape_sectors; 4611 struct list_head stripes; 4612 4613 if (sector_nr == 0) { 4614 /* If restarting in the middle, skip the initial sectors */ 4615 if (mddev->reshape_backwards && 4616 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4617 sector_nr = raid5_size(mddev, 0, 0) 4618 - conf->reshape_progress; 4619 } else if (!mddev->reshape_backwards && 4620 conf->reshape_progress > 0) 4621 sector_nr = conf->reshape_progress; 4622 sector_div(sector_nr, new_data_disks); 4623 if (sector_nr) { 4624 mddev->curr_resync_completed = sector_nr; 4625 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4626 *skipped = 1; 4627 return sector_nr; 4628 } 4629 } 4630 4631 /* We need to process a full chunk at a time. 4632 * If old and new chunk sizes differ, we need to process the 4633 * largest of these 4634 */ 4635 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4636 reshape_sectors = mddev->new_chunk_sectors; 4637 else 4638 reshape_sectors = mddev->chunk_sectors; 4639 4640 /* We update the metadata at least every 10 seconds, or when 4641 * the data about to be copied would over-write the source of 4642 * the data at the front of the range. i.e. one new_stripe 4643 * along from reshape_progress new_maps to after where 4644 * reshape_safe old_maps to 4645 */ 4646 writepos = conf->reshape_progress; 4647 sector_div(writepos, new_data_disks); 4648 readpos = conf->reshape_progress; 4649 sector_div(readpos, data_disks); 4650 safepos = conf->reshape_safe; 4651 sector_div(safepos, data_disks); 4652 if (mddev->reshape_backwards) { 4653 writepos -= min_t(sector_t, reshape_sectors, writepos); 4654 readpos += reshape_sectors; 4655 safepos += reshape_sectors; 4656 } else { 4657 writepos += reshape_sectors; 4658 readpos -= min_t(sector_t, reshape_sectors, readpos); 4659 safepos -= min_t(sector_t, reshape_sectors, safepos); 4660 } 4661 4662 /* Having calculated the 'writepos' possibly use it 4663 * to set 'stripe_addr' which is where we will write to. 4664 */ 4665 if (mddev->reshape_backwards) { 4666 BUG_ON(conf->reshape_progress == 0); 4667 stripe_addr = writepos; 4668 BUG_ON((mddev->dev_sectors & 4669 ~((sector_t)reshape_sectors - 1)) 4670 - reshape_sectors - stripe_addr 4671 != sector_nr); 4672 } else { 4673 BUG_ON(writepos != sector_nr + reshape_sectors); 4674 stripe_addr = sector_nr; 4675 } 4676 4677 /* 'writepos' is the most advanced device address we might write. 4678 * 'readpos' is the least advanced device address we might read. 4679 * 'safepos' is the least address recorded in the metadata as having 4680 * been reshaped. 4681 * If there is a min_offset_diff, these are adjusted either by 4682 * increasing the safepos/readpos if diff is negative, or 4683 * increasing writepos if diff is positive. 4684 * If 'readpos' is then behind 'writepos', there is no way that we can 4685 * ensure safety in the face of a crash - that must be done by userspace 4686 * making a backup of the data. So in that case there is no particular 4687 * rush to update metadata. 4688 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4689 * update the metadata to advance 'safepos' to match 'readpos' so that 4690 * we can be safe in the event of a crash. 4691 * So we insist on updating metadata if safepos is behind writepos and 4692 * readpos is beyond writepos. 4693 * In any case, update the metadata every 10 seconds. 4694 * Maybe that number should be configurable, but I'm not sure it is 4695 * worth it.... maybe it could be a multiple of safemode_delay??? 4696 */ 4697 if (conf->min_offset_diff < 0) { 4698 safepos += -conf->min_offset_diff; 4699 readpos += -conf->min_offset_diff; 4700 } else 4701 writepos += conf->min_offset_diff; 4702 4703 if ((mddev->reshape_backwards 4704 ? (safepos > writepos && readpos < writepos) 4705 : (safepos < writepos && readpos > writepos)) || 4706 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4707 /* Cannot proceed until we've updated the superblock... */ 4708 wait_event(conf->wait_for_overlap, 4709 atomic_read(&conf->reshape_stripes)==0); 4710 mddev->reshape_position = conf->reshape_progress; 4711 mddev->curr_resync_completed = sector_nr; 4712 conf->reshape_checkpoint = jiffies; 4713 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4714 md_wakeup_thread(mddev->thread); 4715 wait_event(mddev->sb_wait, mddev->flags == 0 || 4716 kthread_should_stop()); 4717 spin_lock_irq(&conf->device_lock); 4718 conf->reshape_safe = mddev->reshape_position; 4719 spin_unlock_irq(&conf->device_lock); 4720 wake_up(&conf->wait_for_overlap); 4721 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4722 } 4723 4724 INIT_LIST_HEAD(&stripes); 4725 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4726 int j; 4727 int skipped_disk = 0; 4728 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4729 set_bit(STRIPE_EXPANDING, &sh->state); 4730 atomic_inc(&conf->reshape_stripes); 4731 /* If any of this stripe is beyond the end of the old 4732 * array, then we need to zero those blocks 4733 */ 4734 for (j=sh->disks; j--;) { 4735 sector_t s; 4736 if (j == sh->pd_idx) 4737 continue; 4738 if (conf->level == 6 && 4739 j == sh->qd_idx) 4740 continue; 4741 s = compute_blocknr(sh, j, 0); 4742 if (s < raid5_size(mddev, 0, 0)) { 4743 skipped_disk = 1; 4744 continue; 4745 } 4746 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4747 set_bit(R5_Expanded, &sh->dev[j].flags); 4748 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4749 } 4750 if (!skipped_disk) { 4751 set_bit(STRIPE_EXPAND_READY, &sh->state); 4752 set_bit(STRIPE_HANDLE, &sh->state); 4753 } 4754 list_add(&sh->lru, &stripes); 4755 } 4756 spin_lock_irq(&conf->device_lock); 4757 if (mddev->reshape_backwards) 4758 conf->reshape_progress -= reshape_sectors * new_data_disks; 4759 else 4760 conf->reshape_progress += reshape_sectors * new_data_disks; 4761 spin_unlock_irq(&conf->device_lock); 4762 /* Ok, those stripe are ready. We can start scheduling 4763 * reads on the source stripes. 4764 * The source stripes are determined by mapping the first and last 4765 * block on the destination stripes. 4766 */ 4767 first_sector = 4768 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4769 1, &dd_idx, NULL); 4770 last_sector = 4771 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4772 * new_data_disks - 1), 4773 1, &dd_idx, NULL); 4774 if (last_sector >= mddev->dev_sectors) 4775 last_sector = mddev->dev_sectors - 1; 4776 while (first_sector <= last_sector) { 4777 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4778 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4779 set_bit(STRIPE_HANDLE, &sh->state); 4780 release_stripe(sh); 4781 first_sector += STRIPE_SECTORS; 4782 } 4783 /* Now that the sources are clearly marked, we can release 4784 * the destination stripes 4785 */ 4786 while (!list_empty(&stripes)) { 4787 sh = list_entry(stripes.next, struct stripe_head, lru); 4788 list_del_init(&sh->lru); 4789 release_stripe(sh); 4790 } 4791 /* If this takes us to the resync_max point where we have to pause, 4792 * then we need to write out the superblock. 4793 */ 4794 sector_nr += reshape_sectors; 4795 if ((sector_nr - mddev->curr_resync_completed) * 2 4796 >= mddev->resync_max - mddev->curr_resync_completed) { 4797 /* Cannot proceed until we've updated the superblock... */ 4798 wait_event(conf->wait_for_overlap, 4799 atomic_read(&conf->reshape_stripes) == 0); 4800 mddev->reshape_position = conf->reshape_progress; 4801 mddev->curr_resync_completed = sector_nr; 4802 conf->reshape_checkpoint = jiffies; 4803 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4804 md_wakeup_thread(mddev->thread); 4805 wait_event(mddev->sb_wait, 4806 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4807 || kthread_should_stop()); 4808 spin_lock_irq(&conf->device_lock); 4809 conf->reshape_safe = mddev->reshape_position; 4810 spin_unlock_irq(&conf->device_lock); 4811 wake_up(&conf->wait_for_overlap); 4812 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4813 } 4814 return reshape_sectors; 4815 } 4816 4817 /* FIXME go_faster isn't used */ 4818 static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster) 4819 { 4820 struct r5conf *conf = mddev->private; 4821 struct stripe_head *sh; 4822 sector_t max_sector = mddev->dev_sectors; 4823 sector_t sync_blocks; 4824 int still_degraded = 0; 4825 int i; 4826 4827 if (sector_nr >= max_sector) { 4828 /* just being told to finish up .. nothing much to do */ 4829 4830 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 4831 end_reshape(conf); 4832 return 0; 4833 } 4834 4835 if (mddev->curr_resync < max_sector) /* aborted */ 4836 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 4837 &sync_blocks, 1); 4838 else /* completed sync */ 4839 conf->fullsync = 0; 4840 bitmap_close_sync(mddev->bitmap); 4841 4842 return 0; 4843 } 4844 4845 /* Allow raid5_quiesce to complete */ 4846 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 4847 4848 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 4849 return reshape_request(mddev, sector_nr, skipped); 4850 4851 /* No need to check resync_max as we never do more than one 4852 * stripe, and as resync_max will always be on a chunk boundary, 4853 * if the check in md_do_sync didn't fire, there is no chance 4854 * of overstepping resync_max here 4855 */ 4856 4857 /* if there is too many failed drives and we are trying 4858 * to resync, then assert that we are finished, because there is 4859 * nothing we can do. 4860 */ 4861 if (mddev->degraded >= conf->max_degraded && 4862 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 4863 sector_t rv = mddev->dev_sectors - sector_nr; 4864 *skipped = 1; 4865 return rv; 4866 } 4867 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 4868 !conf->fullsync && 4869 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 4870 sync_blocks >= STRIPE_SECTORS) { 4871 /* we can skip this block, and probably more */ 4872 sync_blocks /= STRIPE_SECTORS; 4873 *skipped = 1; 4874 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 4875 } 4876 4877 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 4878 4879 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 4880 if (sh == NULL) { 4881 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 4882 /* make sure we don't swamp the stripe cache if someone else 4883 * is trying to get access 4884 */ 4885 schedule_timeout_uninterruptible(1); 4886 } 4887 /* Need to check if array will still be degraded after recovery/resync 4888 * We don't need to check the 'failed' flag as when that gets set, 4889 * recovery aborts. 4890 */ 4891 for (i = 0; i < conf->raid_disks; i++) 4892 if (conf->disks[i].rdev == NULL) 4893 still_degraded = 1; 4894 4895 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 4896 4897 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 4898 4899 handle_stripe(sh); 4900 release_stripe(sh); 4901 4902 return STRIPE_SECTORS; 4903 } 4904 4905 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 4906 { 4907 /* We may not be able to submit a whole bio at once as there 4908 * may not be enough stripe_heads available. 4909 * We cannot pre-allocate enough stripe_heads as we may need 4910 * more than exist in the cache (if we allow ever large chunks). 4911 * So we do one stripe head at a time and record in 4912 * ->bi_hw_segments how many have been done. 4913 * 4914 * We *know* that this entire raid_bio is in one chunk, so 4915 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 4916 */ 4917 struct stripe_head *sh; 4918 int dd_idx; 4919 sector_t sector, logical_sector, last_sector; 4920 int scnt = 0; 4921 int remaining; 4922 int handled = 0; 4923 4924 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4925 sector = raid5_compute_sector(conf, logical_sector, 4926 0, &dd_idx, NULL); 4927 last_sector = bio_end_sector(raid_bio); 4928 4929 for (; logical_sector < last_sector; 4930 logical_sector += STRIPE_SECTORS, 4931 sector += STRIPE_SECTORS, 4932 scnt++) { 4933 4934 if (scnt < raid5_bi_processed_stripes(raid_bio)) 4935 /* already done this stripe */ 4936 continue; 4937 4938 sh = get_active_stripe(conf, sector, 0, 1, 0); 4939 4940 if (!sh) { 4941 /* failed to get a stripe - must wait */ 4942 raid5_set_bi_processed_stripes(raid_bio, scnt); 4943 conf->retry_read_aligned = raid_bio; 4944 return handled; 4945 } 4946 4947 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 4948 release_stripe(sh); 4949 raid5_set_bi_processed_stripes(raid_bio, scnt); 4950 conf->retry_read_aligned = raid_bio; 4951 return handled; 4952 } 4953 4954 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); 4955 handle_stripe(sh); 4956 release_stripe(sh); 4957 handled++; 4958 } 4959 remaining = raid5_dec_bi_active_stripes(raid_bio); 4960 if (remaining == 0) { 4961 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 4962 raid_bio, 0); 4963 bio_endio(raid_bio, 0); 4964 } 4965 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4966 wake_up(&conf->wait_for_stripe); 4967 return handled; 4968 } 4969 4970 static int handle_active_stripes(struct r5conf *conf, int group, 4971 struct r5worker *worker) 4972 { 4973 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; 4974 int i, batch_size = 0; 4975 4976 while (batch_size < MAX_STRIPE_BATCH && 4977 (sh = __get_priority_stripe(conf, group)) != NULL) 4978 batch[batch_size++] = sh; 4979 4980 if (batch_size == 0) 4981 return batch_size; 4982 spin_unlock_irq(&conf->device_lock); 4983 4984 for (i = 0; i < batch_size; i++) 4985 handle_stripe(batch[i]); 4986 4987 cond_resched(); 4988 4989 spin_lock_irq(&conf->device_lock); 4990 for (i = 0; i < batch_size; i++) 4991 __release_stripe(conf, batch[i]); 4992 return batch_size; 4993 } 4994 4995 static void raid5_do_work(struct work_struct *work) 4996 { 4997 struct r5worker *worker = container_of(work, struct r5worker, work); 4998 struct r5worker_group *group = worker->group; 4999 struct r5conf *conf = group->conf; 5000 int group_id = group - conf->worker_groups; 5001 int handled; 5002 struct blk_plug plug; 5003 5004 pr_debug("+++ raid5worker active\n"); 5005 5006 blk_start_plug(&plug); 5007 handled = 0; 5008 spin_lock_irq(&conf->device_lock); 5009 while (1) { 5010 int batch_size, released; 5011 5012 released = release_stripe_list(conf); 5013 5014 batch_size = handle_active_stripes(conf, group_id, worker); 5015 worker->working = false; 5016 if (!batch_size && !released) 5017 break; 5018 handled += batch_size; 5019 } 5020 pr_debug("%d stripes handled\n", handled); 5021 5022 spin_unlock_irq(&conf->device_lock); 5023 blk_finish_plug(&plug); 5024 5025 pr_debug("--- raid5worker inactive\n"); 5026 } 5027 5028 /* 5029 * This is our raid5 kernel thread. 5030 * 5031 * We scan the hash table for stripes which can be handled now. 5032 * During the scan, completed stripes are saved for us by the interrupt 5033 * handler, so that they will not have to wait for our next wakeup. 5034 */ 5035 static void raid5d(struct md_thread *thread) 5036 { 5037 struct mddev *mddev = thread->mddev; 5038 struct r5conf *conf = mddev->private; 5039 int handled; 5040 struct blk_plug plug; 5041 5042 pr_debug("+++ raid5d active\n"); 5043 5044 md_check_recovery(mddev); 5045 5046 blk_start_plug(&plug); 5047 handled = 0; 5048 spin_lock_irq(&conf->device_lock); 5049 while (1) { 5050 struct bio *bio; 5051 int batch_size, released; 5052 5053 released = release_stripe_list(conf); 5054 5055 if ( 5056 !list_empty(&conf->bitmap_list)) { 5057 /* Now is a good time to flush some bitmap updates */ 5058 conf->seq_flush++; 5059 spin_unlock_irq(&conf->device_lock); 5060 bitmap_unplug(mddev->bitmap); 5061 spin_lock_irq(&conf->device_lock); 5062 conf->seq_write = conf->seq_flush; 5063 activate_bit_delay(conf); 5064 } 5065 raid5_activate_delayed(conf); 5066 5067 while ((bio = remove_bio_from_retry(conf))) { 5068 int ok; 5069 spin_unlock_irq(&conf->device_lock); 5070 ok = retry_aligned_read(conf, bio); 5071 spin_lock_irq(&conf->device_lock); 5072 if (!ok) 5073 break; 5074 handled++; 5075 } 5076 5077 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL); 5078 if (!batch_size && !released) 5079 break; 5080 handled += batch_size; 5081 5082 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) { 5083 spin_unlock_irq(&conf->device_lock); 5084 md_check_recovery(mddev); 5085 spin_lock_irq(&conf->device_lock); 5086 } 5087 } 5088 pr_debug("%d stripes handled\n", handled); 5089 5090 spin_unlock_irq(&conf->device_lock); 5091 5092 async_tx_issue_pending_all(); 5093 blk_finish_plug(&plug); 5094 5095 pr_debug("--- raid5d inactive\n"); 5096 } 5097 5098 static ssize_t 5099 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 5100 { 5101 struct r5conf *conf = mddev->private; 5102 if (conf) 5103 return sprintf(page, "%d\n", conf->max_nr_stripes); 5104 else 5105 return 0; 5106 } 5107 5108 int 5109 raid5_set_cache_size(struct mddev *mddev, int size) 5110 { 5111 struct r5conf *conf = mddev->private; 5112 int err; 5113 5114 if (size <= 16 || size > 32768) 5115 return -EINVAL; 5116 while (size < conf->max_nr_stripes) { 5117 if (drop_one_stripe(conf)) 5118 conf->max_nr_stripes--; 5119 else 5120 break; 5121 } 5122 err = md_allow_write(mddev); 5123 if (err) 5124 return err; 5125 while (size > conf->max_nr_stripes) { 5126 if (grow_one_stripe(conf)) 5127 conf->max_nr_stripes++; 5128 else break; 5129 } 5130 return 0; 5131 } 5132 EXPORT_SYMBOL(raid5_set_cache_size); 5133 5134 static ssize_t 5135 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 5136 { 5137 struct r5conf *conf = mddev->private; 5138 unsigned long new; 5139 int err; 5140 5141 if (len >= PAGE_SIZE) 5142 return -EINVAL; 5143 if (!conf) 5144 return -ENODEV; 5145 5146 if (kstrtoul(page, 10, &new)) 5147 return -EINVAL; 5148 err = raid5_set_cache_size(mddev, new); 5149 if (err) 5150 return err; 5151 return len; 5152 } 5153 5154 static struct md_sysfs_entry 5155 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 5156 raid5_show_stripe_cache_size, 5157 raid5_store_stripe_cache_size); 5158 5159 static ssize_t 5160 raid5_show_preread_threshold(struct mddev *mddev, char *page) 5161 { 5162 struct r5conf *conf = mddev->private; 5163 if (conf) 5164 return sprintf(page, "%d\n", conf->bypass_threshold); 5165 else 5166 return 0; 5167 } 5168 5169 static ssize_t 5170 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 5171 { 5172 struct r5conf *conf = mddev->private; 5173 unsigned long new; 5174 if (len >= PAGE_SIZE) 5175 return -EINVAL; 5176 if (!conf) 5177 return -ENODEV; 5178 5179 if (kstrtoul(page, 10, &new)) 5180 return -EINVAL; 5181 if (new > conf->max_nr_stripes) 5182 return -EINVAL; 5183 conf->bypass_threshold = new; 5184 return len; 5185 } 5186 5187 static struct md_sysfs_entry 5188 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 5189 S_IRUGO | S_IWUSR, 5190 raid5_show_preread_threshold, 5191 raid5_store_preread_threshold); 5192 5193 static ssize_t 5194 stripe_cache_active_show(struct mddev *mddev, char *page) 5195 { 5196 struct r5conf *conf = mddev->private; 5197 if (conf) 5198 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 5199 else 5200 return 0; 5201 } 5202 5203 static struct md_sysfs_entry 5204 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 5205 5206 static ssize_t 5207 raid5_show_group_thread_cnt(struct mddev *mddev, char *page) 5208 { 5209 struct r5conf *conf = mddev->private; 5210 if (conf) 5211 return sprintf(page, "%d\n", conf->worker_cnt_per_group); 5212 else 5213 return 0; 5214 } 5215 5216 static int alloc_thread_groups(struct r5conf *conf, int cnt); 5217 static ssize_t 5218 raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) 5219 { 5220 struct r5conf *conf = mddev->private; 5221 unsigned long new; 5222 int err; 5223 struct r5worker_group *old_groups; 5224 int old_group_cnt; 5225 5226 if (len >= PAGE_SIZE) 5227 return -EINVAL; 5228 if (!conf) 5229 return -ENODEV; 5230 5231 if (kstrtoul(page, 10, &new)) 5232 return -EINVAL; 5233 5234 if (new == conf->worker_cnt_per_group) 5235 return len; 5236 5237 mddev_suspend(mddev); 5238 5239 old_groups = conf->worker_groups; 5240 old_group_cnt = conf->worker_cnt_per_group; 5241 5242 conf->worker_groups = NULL; 5243 err = alloc_thread_groups(conf, new); 5244 if (err) { 5245 conf->worker_groups = old_groups; 5246 conf->worker_cnt_per_group = old_group_cnt; 5247 } else { 5248 if (old_groups) 5249 kfree(old_groups[0].workers); 5250 kfree(old_groups); 5251 } 5252 5253 mddev_resume(mddev); 5254 5255 if (err) 5256 return err; 5257 return len; 5258 } 5259 5260 static struct md_sysfs_entry 5261 raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, 5262 raid5_show_group_thread_cnt, 5263 raid5_store_group_thread_cnt); 5264 5265 static struct attribute *raid5_attrs[] = { 5266 &raid5_stripecache_size.attr, 5267 &raid5_stripecache_active.attr, 5268 &raid5_preread_bypass_threshold.attr, 5269 &raid5_group_thread_cnt.attr, 5270 NULL, 5271 }; 5272 static struct attribute_group raid5_attrs_group = { 5273 .name = NULL, 5274 .attrs = raid5_attrs, 5275 }; 5276 5277 static int alloc_thread_groups(struct r5conf *conf, int cnt) 5278 { 5279 int i, j; 5280 ssize_t size; 5281 struct r5worker *workers; 5282 5283 conf->worker_cnt_per_group = cnt; 5284 if (cnt == 0) { 5285 conf->worker_groups = NULL; 5286 return 0; 5287 } 5288 conf->group_cnt = num_possible_nodes(); 5289 size = sizeof(struct r5worker) * cnt; 5290 workers = kzalloc(size * conf->group_cnt, GFP_NOIO); 5291 conf->worker_groups = kzalloc(sizeof(struct r5worker_group) * 5292 conf->group_cnt, GFP_NOIO); 5293 if (!conf->worker_groups || !workers) { 5294 kfree(workers); 5295 kfree(conf->worker_groups); 5296 conf->worker_groups = NULL; 5297 return -ENOMEM; 5298 } 5299 5300 for (i = 0; i < conf->group_cnt; i++) { 5301 struct r5worker_group *group; 5302 5303 group = &conf->worker_groups[i]; 5304 INIT_LIST_HEAD(&group->handle_list); 5305 group->conf = conf; 5306 group->workers = workers + i * cnt; 5307 5308 for (j = 0; j < cnt; j++) { 5309 group->workers[j].group = group; 5310 INIT_WORK(&group->workers[j].work, raid5_do_work); 5311 } 5312 } 5313 5314 return 0; 5315 } 5316 5317 static void free_thread_groups(struct r5conf *conf) 5318 { 5319 if (conf->worker_groups) 5320 kfree(conf->worker_groups[0].workers); 5321 kfree(conf->worker_groups); 5322 conf->worker_groups = NULL; 5323 } 5324 5325 static sector_t 5326 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 5327 { 5328 struct r5conf *conf = mddev->private; 5329 5330 if (!sectors) 5331 sectors = mddev->dev_sectors; 5332 if (!raid_disks) 5333 /* size is defined by the smallest of previous and new size */ 5334 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 5335 5336 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5337 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 5338 return sectors * (raid_disks - conf->max_degraded); 5339 } 5340 5341 static void raid5_free_percpu(struct r5conf *conf) 5342 { 5343 struct raid5_percpu *percpu; 5344 unsigned long cpu; 5345 5346 if (!conf->percpu) 5347 return; 5348 5349 get_online_cpus(); 5350 for_each_possible_cpu(cpu) { 5351 percpu = per_cpu_ptr(conf->percpu, cpu); 5352 safe_put_page(percpu->spare_page); 5353 kfree(percpu->scribble); 5354 } 5355 #ifdef CONFIG_HOTPLUG_CPU 5356 unregister_cpu_notifier(&conf->cpu_notify); 5357 #endif 5358 put_online_cpus(); 5359 5360 free_percpu(conf->percpu); 5361 } 5362 5363 static void free_conf(struct r5conf *conf) 5364 { 5365 free_thread_groups(conf); 5366 shrink_stripes(conf); 5367 raid5_free_percpu(conf); 5368 kfree(conf->disks); 5369 kfree(conf->stripe_hashtbl); 5370 kfree(conf); 5371 } 5372 5373 #ifdef CONFIG_HOTPLUG_CPU 5374 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 5375 void *hcpu) 5376 { 5377 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); 5378 long cpu = (long)hcpu; 5379 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 5380 5381 switch (action) { 5382 case CPU_UP_PREPARE: 5383 case CPU_UP_PREPARE_FROZEN: 5384 if (conf->level == 6 && !percpu->spare_page) 5385 percpu->spare_page = alloc_page(GFP_KERNEL); 5386 if (!percpu->scribble) 5387 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 5388 5389 if (!percpu->scribble || 5390 (conf->level == 6 && !percpu->spare_page)) { 5391 safe_put_page(percpu->spare_page); 5392 kfree(percpu->scribble); 5393 pr_err("%s: failed memory allocation for cpu%ld\n", 5394 __func__, cpu); 5395 return notifier_from_errno(-ENOMEM); 5396 } 5397 break; 5398 case CPU_DEAD: 5399 case CPU_DEAD_FROZEN: 5400 safe_put_page(percpu->spare_page); 5401 kfree(percpu->scribble); 5402 percpu->spare_page = NULL; 5403 percpu->scribble = NULL; 5404 break; 5405 default: 5406 break; 5407 } 5408 return NOTIFY_OK; 5409 } 5410 #endif 5411 5412 static int raid5_alloc_percpu(struct r5conf *conf) 5413 { 5414 unsigned long cpu; 5415 struct page *spare_page; 5416 struct raid5_percpu __percpu *allcpus; 5417 void *scribble; 5418 int err; 5419 5420 allcpus = alloc_percpu(struct raid5_percpu); 5421 if (!allcpus) 5422 return -ENOMEM; 5423 conf->percpu = allcpus; 5424 5425 get_online_cpus(); 5426 err = 0; 5427 for_each_present_cpu(cpu) { 5428 if (conf->level == 6) { 5429 spare_page = alloc_page(GFP_KERNEL); 5430 if (!spare_page) { 5431 err = -ENOMEM; 5432 break; 5433 } 5434 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page; 5435 } 5436 scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 5437 if (!scribble) { 5438 err = -ENOMEM; 5439 break; 5440 } 5441 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble; 5442 } 5443 #ifdef CONFIG_HOTPLUG_CPU 5444 conf->cpu_notify.notifier_call = raid456_cpu_notify; 5445 conf->cpu_notify.priority = 0; 5446 if (err == 0) 5447 err = register_cpu_notifier(&conf->cpu_notify); 5448 #endif 5449 put_online_cpus(); 5450 5451 return err; 5452 } 5453 5454 static struct r5conf *setup_conf(struct mddev *mddev) 5455 { 5456 struct r5conf *conf; 5457 int raid_disk, memory, max_disks; 5458 struct md_rdev *rdev; 5459 struct disk_info *disk; 5460 char pers_name[6]; 5461 5462 if (mddev->new_level != 5 5463 && mddev->new_level != 4 5464 && mddev->new_level != 6) { 5465 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 5466 mdname(mddev), mddev->new_level); 5467 return ERR_PTR(-EIO); 5468 } 5469 if ((mddev->new_level == 5 5470 && !algorithm_valid_raid5(mddev->new_layout)) || 5471 (mddev->new_level == 6 5472 && !algorithm_valid_raid6(mddev->new_layout))) { 5473 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 5474 mdname(mddev), mddev->new_layout); 5475 return ERR_PTR(-EIO); 5476 } 5477 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 5478 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 5479 mdname(mddev), mddev->raid_disks); 5480 return ERR_PTR(-EINVAL); 5481 } 5482 5483 if (!mddev->new_chunk_sectors || 5484 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 5485 !is_power_of_2(mddev->new_chunk_sectors)) { 5486 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 5487 mdname(mddev), mddev->new_chunk_sectors << 9); 5488 return ERR_PTR(-EINVAL); 5489 } 5490 5491 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 5492 if (conf == NULL) 5493 goto abort; 5494 /* Don't enable multi-threading by default*/ 5495 if (alloc_thread_groups(conf, 0)) 5496 goto abort; 5497 spin_lock_init(&conf->device_lock); 5498 seqcount_init(&conf->gen_lock); 5499 init_waitqueue_head(&conf->wait_for_stripe); 5500 init_waitqueue_head(&conf->wait_for_overlap); 5501 INIT_LIST_HEAD(&conf->handle_list); 5502 INIT_LIST_HEAD(&conf->hold_list); 5503 INIT_LIST_HEAD(&conf->delayed_list); 5504 INIT_LIST_HEAD(&conf->bitmap_list); 5505 INIT_LIST_HEAD(&conf->inactive_list); 5506 init_llist_head(&conf->released_stripes); 5507 atomic_set(&conf->active_stripes, 0); 5508 atomic_set(&conf->preread_active_stripes, 0); 5509 atomic_set(&conf->active_aligned_reads, 0); 5510 conf->bypass_threshold = BYPASS_THRESHOLD; 5511 conf->recovery_disabled = mddev->recovery_disabled - 1; 5512 5513 conf->raid_disks = mddev->raid_disks; 5514 if (mddev->reshape_position == MaxSector) 5515 conf->previous_raid_disks = mddev->raid_disks; 5516 else 5517 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 5518 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 5519 conf->scribble_len = scribble_len(max_disks); 5520 5521 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 5522 GFP_KERNEL); 5523 if (!conf->disks) 5524 goto abort; 5525 5526 conf->mddev = mddev; 5527 5528 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 5529 goto abort; 5530 5531 conf->level = mddev->new_level; 5532 if (raid5_alloc_percpu(conf) != 0) 5533 goto abort; 5534 5535 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 5536 5537 rdev_for_each(rdev, mddev) { 5538 raid_disk = rdev->raid_disk; 5539 if (raid_disk >= max_disks 5540 || raid_disk < 0) 5541 continue; 5542 disk = conf->disks + raid_disk; 5543 5544 if (test_bit(Replacement, &rdev->flags)) { 5545 if (disk->replacement) 5546 goto abort; 5547 disk->replacement = rdev; 5548 } else { 5549 if (disk->rdev) 5550 goto abort; 5551 disk->rdev = rdev; 5552 } 5553 5554 if (test_bit(In_sync, &rdev->flags)) { 5555 char b[BDEVNAME_SIZE]; 5556 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 5557 " disk %d\n", 5558 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 5559 } else if (rdev->saved_raid_disk != raid_disk) 5560 /* Cannot rely on bitmap to complete recovery */ 5561 conf->fullsync = 1; 5562 } 5563 5564 conf->chunk_sectors = mddev->new_chunk_sectors; 5565 conf->level = mddev->new_level; 5566 if (conf->level == 6) 5567 conf->max_degraded = 2; 5568 else 5569 conf->max_degraded = 1; 5570 conf->algorithm = mddev->new_layout; 5571 conf->max_nr_stripes = NR_STRIPES; 5572 conf->reshape_progress = mddev->reshape_position; 5573 if (conf->reshape_progress != MaxSector) { 5574 conf->prev_chunk_sectors = mddev->chunk_sectors; 5575 conf->prev_algo = mddev->layout; 5576 } 5577 5578 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 5579 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 5580 if (grow_stripes(conf, conf->max_nr_stripes)) { 5581 printk(KERN_ERR 5582 "md/raid:%s: couldn't allocate %dkB for buffers\n", 5583 mdname(mddev), memory); 5584 goto abort; 5585 } else 5586 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 5587 mdname(mddev), memory); 5588 5589 sprintf(pers_name, "raid%d", mddev->new_level); 5590 conf->thread = md_register_thread(raid5d, mddev, pers_name); 5591 if (!conf->thread) { 5592 printk(KERN_ERR 5593 "md/raid:%s: couldn't allocate thread.\n", 5594 mdname(mddev)); 5595 goto abort; 5596 } 5597 5598 return conf; 5599 5600 abort: 5601 if (conf) { 5602 free_conf(conf); 5603 return ERR_PTR(-EIO); 5604 } else 5605 return ERR_PTR(-ENOMEM); 5606 } 5607 5608 5609 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 5610 { 5611 switch (algo) { 5612 case ALGORITHM_PARITY_0: 5613 if (raid_disk < max_degraded) 5614 return 1; 5615 break; 5616 case ALGORITHM_PARITY_N: 5617 if (raid_disk >= raid_disks - max_degraded) 5618 return 1; 5619 break; 5620 case ALGORITHM_PARITY_0_6: 5621 if (raid_disk == 0 || 5622 raid_disk == raid_disks - 1) 5623 return 1; 5624 break; 5625 case ALGORITHM_LEFT_ASYMMETRIC_6: 5626 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5627 case ALGORITHM_LEFT_SYMMETRIC_6: 5628 case ALGORITHM_RIGHT_SYMMETRIC_6: 5629 if (raid_disk == raid_disks - 1) 5630 return 1; 5631 } 5632 return 0; 5633 } 5634 5635 static int run(struct mddev *mddev) 5636 { 5637 struct r5conf *conf; 5638 int working_disks = 0; 5639 int dirty_parity_disks = 0; 5640 struct md_rdev *rdev; 5641 sector_t reshape_offset = 0; 5642 int i; 5643 long long min_offset_diff = 0; 5644 int first = 1; 5645 5646 if (mddev->recovery_cp != MaxSector) 5647 printk(KERN_NOTICE "md/raid:%s: not clean" 5648 " -- starting background reconstruction\n", 5649 mdname(mddev)); 5650 5651 rdev_for_each(rdev, mddev) { 5652 long long diff; 5653 if (rdev->raid_disk < 0) 5654 continue; 5655 diff = (rdev->new_data_offset - rdev->data_offset); 5656 if (first) { 5657 min_offset_diff = diff; 5658 first = 0; 5659 } else if (mddev->reshape_backwards && 5660 diff < min_offset_diff) 5661 min_offset_diff = diff; 5662 else if (!mddev->reshape_backwards && 5663 diff > min_offset_diff) 5664 min_offset_diff = diff; 5665 } 5666 5667 if (mddev->reshape_position != MaxSector) { 5668 /* Check that we can continue the reshape. 5669 * Difficulties arise if the stripe we would write to 5670 * next is at or after the stripe we would read from next. 5671 * For a reshape that changes the number of devices, this 5672 * is only possible for a very short time, and mdadm makes 5673 * sure that time appears to have past before assembling 5674 * the array. So we fail if that time hasn't passed. 5675 * For a reshape that keeps the number of devices the same 5676 * mdadm must be monitoring the reshape can keeping the 5677 * critical areas read-only and backed up. It will start 5678 * the array in read-only mode, so we check for that. 5679 */ 5680 sector_t here_new, here_old; 5681 int old_disks; 5682 int max_degraded = (mddev->level == 6 ? 2 : 1); 5683 5684 if (mddev->new_level != mddev->level) { 5685 printk(KERN_ERR "md/raid:%s: unsupported reshape " 5686 "required - aborting.\n", 5687 mdname(mddev)); 5688 return -EINVAL; 5689 } 5690 old_disks = mddev->raid_disks - mddev->delta_disks; 5691 /* reshape_position must be on a new-stripe boundary, and one 5692 * further up in new geometry must map after here in old 5693 * geometry. 5694 */ 5695 here_new = mddev->reshape_position; 5696 if (sector_div(here_new, mddev->new_chunk_sectors * 5697 (mddev->raid_disks - max_degraded))) { 5698 printk(KERN_ERR "md/raid:%s: reshape_position not " 5699 "on a stripe boundary\n", mdname(mddev)); 5700 return -EINVAL; 5701 } 5702 reshape_offset = here_new * mddev->new_chunk_sectors; 5703 /* here_new is the stripe we will write to */ 5704 here_old = mddev->reshape_position; 5705 sector_div(here_old, mddev->chunk_sectors * 5706 (old_disks-max_degraded)); 5707 /* here_old is the first stripe that we might need to read 5708 * from */ 5709 if (mddev->delta_disks == 0) { 5710 if ((here_new * mddev->new_chunk_sectors != 5711 here_old * mddev->chunk_sectors)) { 5712 printk(KERN_ERR "md/raid:%s: reshape position is" 5713 " confused - aborting\n", mdname(mddev)); 5714 return -EINVAL; 5715 } 5716 /* We cannot be sure it is safe to start an in-place 5717 * reshape. It is only safe if user-space is monitoring 5718 * and taking constant backups. 5719 * mdadm always starts a situation like this in 5720 * readonly mode so it can take control before 5721 * allowing any writes. So just check for that. 5722 */ 5723 if (abs(min_offset_diff) >= mddev->chunk_sectors && 5724 abs(min_offset_diff) >= mddev->new_chunk_sectors) 5725 /* not really in-place - so OK */; 5726 else if (mddev->ro == 0) { 5727 printk(KERN_ERR "md/raid:%s: in-place reshape " 5728 "must be started in read-only mode " 5729 "- aborting\n", 5730 mdname(mddev)); 5731 return -EINVAL; 5732 } 5733 } else if (mddev->reshape_backwards 5734 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= 5735 here_old * mddev->chunk_sectors) 5736 : (here_new * mddev->new_chunk_sectors >= 5737 here_old * mddev->chunk_sectors + (-min_offset_diff))) { 5738 /* Reading from the same stripe as writing to - bad */ 5739 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 5740 "auto-recovery - aborting.\n", 5741 mdname(mddev)); 5742 return -EINVAL; 5743 } 5744 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 5745 mdname(mddev)); 5746 /* OK, we should be able to continue; */ 5747 } else { 5748 BUG_ON(mddev->level != mddev->new_level); 5749 BUG_ON(mddev->layout != mddev->new_layout); 5750 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 5751 BUG_ON(mddev->delta_disks != 0); 5752 } 5753 5754 if (mddev->private == NULL) 5755 conf = setup_conf(mddev); 5756 else 5757 conf = mddev->private; 5758 5759 if (IS_ERR(conf)) 5760 return PTR_ERR(conf); 5761 5762 conf->min_offset_diff = min_offset_diff; 5763 mddev->thread = conf->thread; 5764 conf->thread = NULL; 5765 mddev->private = conf; 5766 5767 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 5768 i++) { 5769 rdev = conf->disks[i].rdev; 5770 if (!rdev && conf->disks[i].replacement) { 5771 /* The replacement is all we have yet */ 5772 rdev = conf->disks[i].replacement; 5773 conf->disks[i].replacement = NULL; 5774 clear_bit(Replacement, &rdev->flags); 5775 conf->disks[i].rdev = rdev; 5776 } 5777 if (!rdev) 5778 continue; 5779 if (conf->disks[i].replacement && 5780 conf->reshape_progress != MaxSector) { 5781 /* replacements and reshape simply do not mix. */ 5782 printk(KERN_ERR "md: cannot handle concurrent " 5783 "replacement and reshape.\n"); 5784 goto abort; 5785 } 5786 if (test_bit(In_sync, &rdev->flags)) { 5787 working_disks++; 5788 continue; 5789 } 5790 /* This disc is not fully in-sync. However if it 5791 * just stored parity (beyond the recovery_offset), 5792 * when we don't need to be concerned about the 5793 * array being dirty. 5794 * When reshape goes 'backwards', we never have 5795 * partially completed devices, so we only need 5796 * to worry about reshape going forwards. 5797 */ 5798 /* Hack because v0.91 doesn't store recovery_offset properly. */ 5799 if (mddev->major_version == 0 && 5800 mddev->minor_version > 90) 5801 rdev->recovery_offset = reshape_offset; 5802 5803 if (rdev->recovery_offset < reshape_offset) { 5804 /* We need to check old and new layout */ 5805 if (!only_parity(rdev->raid_disk, 5806 conf->algorithm, 5807 conf->raid_disks, 5808 conf->max_degraded)) 5809 continue; 5810 } 5811 if (!only_parity(rdev->raid_disk, 5812 conf->prev_algo, 5813 conf->previous_raid_disks, 5814 conf->max_degraded)) 5815 continue; 5816 dirty_parity_disks++; 5817 } 5818 5819 /* 5820 * 0 for a fully functional array, 1 or 2 for a degraded array. 5821 */ 5822 mddev->degraded = calc_degraded(conf); 5823 5824 if (has_failed(conf)) { 5825 printk(KERN_ERR "md/raid:%s: not enough operational devices" 5826 " (%d/%d failed)\n", 5827 mdname(mddev), mddev->degraded, conf->raid_disks); 5828 goto abort; 5829 } 5830 5831 /* device size must be a multiple of chunk size */ 5832 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 5833 mddev->resync_max_sectors = mddev->dev_sectors; 5834 5835 if (mddev->degraded > dirty_parity_disks && 5836 mddev->recovery_cp != MaxSector) { 5837 if (mddev->ok_start_degraded) 5838 printk(KERN_WARNING 5839 "md/raid:%s: starting dirty degraded array" 5840 " - data corruption possible.\n", 5841 mdname(mddev)); 5842 else { 5843 printk(KERN_ERR 5844 "md/raid:%s: cannot start dirty degraded array.\n", 5845 mdname(mddev)); 5846 goto abort; 5847 } 5848 } 5849 5850 if (mddev->degraded == 0) 5851 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 5852 " devices, algorithm %d\n", mdname(mddev), conf->level, 5853 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 5854 mddev->new_layout); 5855 else 5856 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 5857 " out of %d devices, algorithm %d\n", 5858 mdname(mddev), conf->level, 5859 mddev->raid_disks - mddev->degraded, 5860 mddev->raid_disks, mddev->new_layout); 5861 5862 print_raid5_conf(conf); 5863 5864 if (conf->reshape_progress != MaxSector) { 5865 conf->reshape_safe = conf->reshape_progress; 5866 atomic_set(&conf->reshape_stripes, 0); 5867 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 5868 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 5869 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 5870 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 5871 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 5872 "reshape"); 5873 } 5874 5875 5876 /* Ok, everything is just fine now */ 5877 if (mddev->to_remove == &raid5_attrs_group) 5878 mddev->to_remove = NULL; 5879 else if (mddev->kobj.sd && 5880 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 5881 printk(KERN_WARNING 5882 "raid5: failed to create sysfs attributes for %s\n", 5883 mdname(mddev)); 5884 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 5885 5886 if (mddev->queue) { 5887 int chunk_size; 5888 bool discard_supported = true; 5889 /* read-ahead size must cover two whole stripes, which 5890 * is 2 * (datadisks) * chunksize where 'n' is the 5891 * number of raid devices 5892 */ 5893 int data_disks = conf->previous_raid_disks - conf->max_degraded; 5894 int stripe = data_disks * 5895 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 5896 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 5897 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 5898 5899 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 5900 5901 mddev->queue->backing_dev_info.congested_data = mddev; 5902 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 5903 5904 chunk_size = mddev->chunk_sectors << 9; 5905 blk_queue_io_min(mddev->queue, chunk_size); 5906 blk_queue_io_opt(mddev->queue, chunk_size * 5907 (conf->raid_disks - conf->max_degraded)); 5908 /* 5909 * We can only discard a whole stripe. It doesn't make sense to 5910 * discard data disk but write parity disk 5911 */ 5912 stripe = stripe * PAGE_SIZE; 5913 /* Round up to power of 2, as discard handling 5914 * currently assumes that */ 5915 while ((stripe-1) & stripe) 5916 stripe = (stripe | (stripe-1)) + 1; 5917 mddev->queue->limits.discard_alignment = stripe; 5918 mddev->queue->limits.discard_granularity = stripe; 5919 /* 5920 * unaligned part of discard request will be ignored, so can't 5921 * guarantee discard_zerors_data 5922 */ 5923 mddev->queue->limits.discard_zeroes_data = 0; 5924 5925 blk_queue_max_write_same_sectors(mddev->queue, 0); 5926 5927 rdev_for_each(rdev, mddev) { 5928 disk_stack_limits(mddev->gendisk, rdev->bdev, 5929 rdev->data_offset << 9); 5930 disk_stack_limits(mddev->gendisk, rdev->bdev, 5931 rdev->new_data_offset << 9); 5932 /* 5933 * discard_zeroes_data is required, otherwise data 5934 * could be lost. Consider a scenario: discard a stripe 5935 * (the stripe could be inconsistent if 5936 * discard_zeroes_data is 0); write one disk of the 5937 * stripe (the stripe could be inconsistent again 5938 * depending on which disks are used to calculate 5939 * parity); the disk is broken; The stripe data of this 5940 * disk is lost. 5941 */ 5942 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || 5943 !bdev_get_queue(rdev->bdev)-> 5944 limits.discard_zeroes_data) 5945 discard_supported = false; 5946 } 5947 5948 if (discard_supported && 5949 mddev->queue->limits.max_discard_sectors >= stripe && 5950 mddev->queue->limits.discard_granularity >= stripe) 5951 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 5952 mddev->queue); 5953 else 5954 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 5955 mddev->queue); 5956 } 5957 5958 return 0; 5959 abort: 5960 md_unregister_thread(&mddev->thread); 5961 print_raid5_conf(conf); 5962 free_conf(conf); 5963 mddev->private = NULL; 5964 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 5965 return -EIO; 5966 } 5967 5968 static int stop(struct mddev *mddev) 5969 { 5970 struct r5conf *conf = mddev->private; 5971 5972 md_unregister_thread(&mddev->thread); 5973 if (mddev->queue) 5974 mddev->queue->backing_dev_info.congested_fn = NULL; 5975 free_conf(conf); 5976 mddev->private = NULL; 5977 mddev->to_remove = &raid5_attrs_group; 5978 return 0; 5979 } 5980 5981 static void status(struct seq_file *seq, struct mddev *mddev) 5982 { 5983 struct r5conf *conf = mddev->private; 5984 int i; 5985 5986 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 5987 mddev->chunk_sectors / 2, mddev->layout); 5988 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 5989 for (i = 0; i < conf->raid_disks; i++) 5990 seq_printf (seq, "%s", 5991 conf->disks[i].rdev && 5992 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 5993 seq_printf (seq, "]"); 5994 } 5995 5996 static void print_raid5_conf (struct r5conf *conf) 5997 { 5998 int i; 5999 struct disk_info *tmp; 6000 6001 printk(KERN_DEBUG "RAID conf printout:\n"); 6002 if (!conf) { 6003 printk("(conf==NULL)\n"); 6004 return; 6005 } 6006 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 6007 conf->raid_disks, 6008 conf->raid_disks - conf->mddev->degraded); 6009 6010 for (i = 0; i < conf->raid_disks; i++) { 6011 char b[BDEVNAME_SIZE]; 6012 tmp = conf->disks + i; 6013 if (tmp->rdev) 6014 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 6015 i, !test_bit(Faulty, &tmp->rdev->flags), 6016 bdevname(tmp->rdev->bdev, b)); 6017 } 6018 } 6019 6020 static int raid5_spare_active(struct mddev *mddev) 6021 { 6022 int i; 6023 struct r5conf *conf = mddev->private; 6024 struct disk_info *tmp; 6025 int count = 0; 6026 unsigned long flags; 6027 6028 for (i = 0; i < conf->raid_disks; i++) { 6029 tmp = conf->disks + i; 6030 if (tmp->replacement 6031 && tmp->replacement->recovery_offset == MaxSector 6032 && !test_bit(Faulty, &tmp->replacement->flags) 6033 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 6034 /* Replacement has just become active. */ 6035 if (!tmp->rdev 6036 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 6037 count++; 6038 if (tmp->rdev) { 6039 /* Replaced device not technically faulty, 6040 * but we need to be sure it gets removed 6041 * and never re-added. 6042 */ 6043 set_bit(Faulty, &tmp->rdev->flags); 6044 sysfs_notify_dirent_safe( 6045 tmp->rdev->sysfs_state); 6046 } 6047 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 6048 } else if (tmp->rdev 6049 && tmp->rdev->recovery_offset == MaxSector 6050 && !test_bit(Faulty, &tmp->rdev->flags) 6051 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 6052 count++; 6053 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 6054 } 6055 } 6056 spin_lock_irqsave(&conf->device_lock, flags); 6057 mddev->degraded = calc_degraded(conf); 6058 spin_unlock_irqrestore(&conf->device_lock, flags); 6059 print_raid5_conf(conf); 6060 return count; 6061 } 6062 6063 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 6064 { 6065 struct r5conf *conf = mddev->private; 6066 int err = 0; 6067 int number = rdev->raid_disk; 6068 struct md_rdev **rdevp; 6069 struct disk_info *p = conf->disks + number; 6070 6071 print_raid5_conf(conf); 6072 if (rdev == p->rdev) 6073 rdevp = &p->rdev; 6074 else if (rdev == p->replacement) 6075 rdevp = &p->replacement; 6076 else 6077 return 0; 6078 6079 if (number >= conf->raid_disks && 6080 conf->reshape_progress == MaxSector) 6081 clear_bit(In_sync, &rdev->flags); 6082 6083 if (test_bit(In_sync, &rdev->flags) || 6084 atomic_read(&rdev->nr_pending)) { 6085 err = -EBUSY; 6086 goto abort; 6087 } 6088 /* Only remove non-faulty devices if recovery 6089 * isn't possible. 6090 */ 6091 if (!test_bit(Faulty, &rdev->flags) && 6092 mddev->recovery_disabled != conf->recovery_disabled && 6093 !has_failed(conf) && 6094 (!p->replacement || p->replacement == rdev) && 6095 number < conf->raid_disks) { 6096 err = -EBUSY; 6097 goto abort; 6098 } 6099 *rdevp = NULL; 6100 synchronize_rcu(); 6101 if (atomic_read(&rdev->nr_pending)) { 6102 /* lost the race, try later */ 6103 err = -EBUSY; 6104 *rdevp = rdev; 6105 } else if (p->replacement) { 6106 /* We must have just cleared 'rdev' */ 6107 p->rdev = p->replacement; 6108 clear_bit(Replacement, &p->replacement->flags); 6109 smp_mb(); /* Make sure other CPUs may see both as identical 6110 * but will never see neither - if they are careful 6111 */ 6112 p->replacement = NULL; 6113 clear_bit(WantReplacement, &rdev->flags); 6114 } else 6115 /* We might have just removed the Replacement as faulty- 6116 * clear the bit just in case 6117 */ 6118 clear_bit(WantReplacement, &rdev->flags); 6119 abort: 6120 6121 print_raid5_conf(conf); 6122 return err; 6123 } 6124 6125 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 6126 { 6127 struct r5conf *conf = mddev->private; 6128 int err = -EEXIST; 6129 int disk; 6130 struct disk_info *p; 6131 int first = 0; 6132 int last = conf->raid_disks - 1; 6133 6134 if (mddev->recovery_disabled == conf->recovery_disabled) 6135 return -EBUSY; 6136 6137 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 6138 /* no point adding a device */ 6139 return -EINVAL; 6140 6141 if (rdev->raid_disk >= 0) 6142 first = last = rdev->raid_disk; 6143 6144 /* 6145 * find the disk ... but prefer rdev->saved_raid_disk 6146 * if possible. 6147 */ 6148 if (rdev->saved_raid_disk >= 0 && 6149 rdev->saved_raid_disk >= first && 6150 conf->disks[rdev->saved_raid_disk].rdev == NULL) 6151 first = rdev->saved_raid_disk; 6152 6153 for (disk = first; disk <= last; disk++) { 6154 p = conf->disks + disk; 6155 if (p->rdev == NULL) { 6156 clear_bit(In_sync, &rdev->flags); 6157 rdev->raid_disk = disk; 6158 err = 0; 6159 if (rdev->saved_raid_disk != disk) 6160 conf->fullsync = 1; 6161 rcu_assign_pointer(p->rdev, rdev); 6162 goto out; 6163 } 6164 } 6165 for (disk = first; disk <= last; disk++) { 6166 p = conf->disks + disk; 6167 if (test_bit(WantReplacement, &p->rdev->flags) && 6168 p->replacement == NULL) { 6169 clear_bit(In_sync, &rdev->flags); 6170 set_bit(Replacement, &rdev->flags); 6171 rdev->raid_disk = disk; 6172 err = 0; 6173 conf->fullsync = 1; 6174 rcu_assign_pointer(p->replacement, rdev); 6175 break; 6176 } 6177 } 6178 out: 6179 print_raid5_conf(conf); 6180 return err; 6181 } 6182 6183 static int raid5_resize(struct mddev *mddev, sector_t sectors) 6184 { 6185 /* no resync is happening, and there is enough space 6186 * on all devices, so we can resize. 6187 * We need to make sure resync covers any new space. 6188 * If the array is shrinking we should possibly wait until 6189 * any io in the removed space completes, but it hardly seems 6190 * worth it. 6191 */ 6192 sector_t newsize; 6193 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 6194 newsize = raid5_size(mddev, sectors, mddev->raid_disks); 6195 if (mddev->external_size && 6196 mddev->array_sectors > newsize) 6197 return -EINVAL; 6198 if (mddev->bitmap) { 6199 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); 6200 if (ret) 6201 return ret; 6202 } 6203 md_set_array_sectors(mddev, newsize); 6204 set_capacity(mddev->gendisk, mddev->array_sectors); 6205 revalidate_disk(mddev->gendisk); 6206 if (sectors > mddev->dev_sectors && 6207 mddev->recovery_cp > mddev->dev_sectors) { 6208 mddev->recovery_cp = mddev->dev_sectors; 6209 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 6210 } 6211 mddev->dev_sectors = sectors; 6212 mddev->resync_max_sectors = sectors; 6213 return 0; 6214 } 6215 6216 static int check_stripe_cache(struct mddev *mddev) 6217 { 6218 /* Can only proceed if there are plenty of stripe_heads. 6219 * We need a minimum of one full stripe,, and for sensible progress 6220 * it is best to have about 4 times that. 6221 * If we require 4 times, then the default 256 4K stripe_heads will 6222 * allow for chunk sizes up to 256K, which is probably OK. 6223 * If the chunk size is greater, user-space should request more 6224 * stripe_heads first. 6225 */ 6226 struct r5conf *conf = mddev->private; 6227 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 6228 > conf->max_nr_stripes || 6229 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 6230 > conf->max_nr_stripes) { 6231 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 6232 mdname(mddev), 6233 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 6234 / STRIPE_SIZE)*4); 6235 return 0; 6236 } 6237 return 1; 6238 } 6239 6240 static int check_reshape(struct mddev *mddev) 6241 { 6242 struct r5conf *conf = mddev->private; 6243 6244 if (mddev->delta_disks == 0 && 6245 mddev->new_layout == mddev->layout && 6246 mddev->new_chunk_sectors == mddev->chunk_sectors) 6247 return 0; /* nothing to do */ 6248 if (has_failed(conf)) 6249 return -EINVAL; 6250 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { 6251 /* We might be able to shrink, but the devices must 6252 * be made bigger first. 6253 * For raid6, 4 is the minimum size. 6254 * Otherwise 2 is the minimum 6255 */ 6256 int min = 2; 6257 if (mddev->level == 6) 6258 min = 4; 6259 if (mddev->raid_disks + mddev->delta_disks < min) 6260 return -EINVAL; 6261 } 6262 6263 if (!check_stripe_cache(mddev)) 6264 return -ENOSPC; 6265 6266 return resize_stripes(conf, (conf->previous_raid_disks 6267 + mddev->delta_disks)); 6268 } 6269 6270 static int raid5_start_reshape(struct mddev *mddev) 6271 { 6272 struct r5conf *conf = mddev->private; 6273 struct md_rdev *rdev; 6274 int spares = 0; 6275 unsigned long flags; 6276 6277 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 6278 return -EBUSY; 6279 6280 if (!check_stripe_cache(mddev)) 6281 return -ENOSPC; 6282 6283 if (has_failed(conf)) 6284 return -EINVAL; 6285 6286 rdev_for_each(rdev, mddev) { 6287 if (!test_bit(In_sync, &rdev->flags) 6288 && !test_bit(Faulty, &rdev->flags)) 6289 spares++; 6290 } 6291 6292 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 6293 /* Not enough devices even to make a degraded array 6294 * of that size 6295 */ 6296 return -EINVAL; 6297 6298 /* Refuse to reduce size of the array. Any reductions in 6299 * array size must be through explicit setting of array_size 6300 * attribute. 6301 */ 6302 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 6303 < mddev->array_sectors) { 6304 printk(KERN_ERR "md/raid:%s: array size must be reduced " 6305 "before number of disks\n", mdname(mddev)); 6306 return -EINVAL; 6307 } 6308 6309 atomic_set(&conf->reshape_stripes, 0); 6310 spin_lock_irq(&conf->device_lock); 6311 write_seqcount_begin(&conf->gen_lock); 6312 conf->previous_raid_disks = conf->raid_disks; 6313 conf->raid_disks += mddev->delta_disks; 6314 conf->prev_chunk_sectors = conf->chunk_sectors; 6315 conf->chunk_sectors = mddev->new_chunk_sectors; 6316 conf->prev_algo = conf->algorithm; 6317 conf->algorithm = mddev->new_layout; 6318 conf->generation++; 6319 /* Code that selects data_offset needs to see the generation update 6320 * if reshape_progress has been set - so a memory barrier needed. 6321 */ 6322 smp_mb(); 6323 if (mddev->reshape_backwards) 6324 conf->reshape_progress = raid5_size(mddev, 0, 0); 6325 else 6326 conf->reshape_progress = 0; 6327 conf->reshape_safe = conf->reshape_progress; 6328 write_seqcount_end(&conf->gen_lock); 6329 spin_unlock_irq(&conf->device_lock); 6330 6331 /* Now make sure any requests that proceeded on the assumption 6332 * the reshape wasn't running - like Discard or Read - have 6333 * completed. 6334 */ 6335 mddev_suspend(mddev); 6336 mddev_resume(mddev); 6337 6338 /* Add some new drives, as many as will fit. 6339 * We know there are enough to make the newly sized array work. 6340 * Don't add devices if we are reducing the number of 6341 * devices in the array. This is because it is not possible 6342 * to correctly record the "partially reconstructed" state of 6343 * such devices during the reshape and confusion could result. 6344 */ 6345 if (mddev->delta_disks >= 0) { 6346 rdev_for_each(rdev, mddev) 6347 if (rdev->raid_disk < 0 && 6348 !test_bit(Faulty, &rdev->flags)) { 6349 if (raid5_add_disk(mddev, rdev) == 0) { 6350 if (rdev->raid_disk 6351 >= conf->previous_raid_disks) 6352 set_bit(In_sync, &rdev->flags); 6353 else 6354 rdev->recovery_offset = 0; 6355 6356 if (sysfs_link_rdev(mddev, rdev)) 6357 /* Failure here is OK */; 6358 } 6359 } else if (rdev->raid_disk >= conf->previous_raid_disks 6360 && !test_bit(Faulty, &rdev->flags)) { 6361 /* This is a spare that was manually added */ 6362 set_bit(In_sync, &rdev->flags); 6363 } 6364 6365 /* When a reshape changes the number of devices, 6366 * ->degraded is measured against the larger of the 6367 * pre and post number of devices. 6368 */ 6369 spin_lock_irqsave(&conf->device_lock, flags); 6370 mddev->degraded = calc_degraded(conf); 6371 spin_unlock_irqrestore(&conf->device_lock, flags); 6372 } 6373 mddev->raid_disks = conf->raid_disks; 6374 mddev->reshape_position = conf->reshape_progress; 6375 set_bit(MD_CHANGE_DEVS, &mddev->flags); 6376 6377 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6378 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6379 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6380 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6381 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6382 "reshape"); 6383 if (!mddev->sync_thread) { 6384 mddev->recovery = 0; 6385 spin_lock_irq(&conf->device_lock); 6386 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 6387 rdev_for_each(rdev, mddev) 6388 rdev->new_data_offset = rdev->data_offset; 6389 smp_wmb(); 6390 conf->reshape_progress = MaxSector; 6391 mddev->reshape_position = MaxSector; 6392 spin_unlock_irq(&conf->device_lock); 6393 return -EAGAIN; 6394 } 6395 conf->reshape_checkpoint = jiffies; 6396 md_wakeup_thread(mddev->sync_thread); 6397 md_new_event(mddev); 6398 return 0; 6399 } 6400 6401 /* This is called from the reshape thread and should make any 6402 * changes needed in 'conf' 6403 */ 6404 static void end_reshape(struct r5conf *conf) 6405 { 6406 6407 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 6408 struct md_rdev *rdev; 6409 6410 spin_lock_irq(&conf->device_lock); 6411 conf->previous_raid_disks = conf->raid_disks; 6412 rdev_for_each(rdev, conf->mddev) 6413 rdev->data_offset = rdev->new_data_offset; 6414 smp_wmb(); 6415 conf->reshape_progress = MaxSector; 6416 spin_unlock_irq(&conf->device_lock); 6417 wake_up(&conf->wait_for_overlap); 6418 6419 /* read-ahead size must cover two whole stripes, which is 6420 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 6421 */ 6422 if (conf->mddev->queue) { 6423 int data_disks = conf->raid_disks - conf->max_degraded; 6424 int stripe = data_disks * ((conf->chunk_sectors << 9) 6425 / PAGE_SIZE); 6426 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6427 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6428 } 6429 } 6430 } 6431 6432 /* This is called from the raid5d thread with mddev_lock held. 6433 * It makes config changes to the device. 6434 */ 6435 static void raid5_finish_reshape(struct mddev *mddev) 6436 { 6437 struct r5conf *conf = mddev->private; 6438 6439 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 6440 6441 if (mddev->delta_disks > 0) { 6442 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6443 set_capacity(mddev->gendisk, mddev->array_sectors); 6444 revalidate_disk(mddev->gendisk); 6445 } else { 6446 int d; 6447 spin_lock_irq(&conf->device_lock); 6448 mddev->degraded = calc_degraded(conf); 6449 spin_unlock_irq(&conf->device_lock); 6450 for (d = conf->raid_disks ; 6451 d < conf->raid_disks - mddev->delta_disks; 6452 d++) { 6453 struct md_rdev *rdev = conf->disks[d].rdev; 6454 if (rdev) 6455 clear_bit(In_sync, &rdev->flags); 6456 rdev = conf->disks[d].replacement; 6457 if (rdev) 6458 clear_bit(In_sync, &rdev->flags); 6459 } 6460 } 6461 mddev->layout = conf->algorithm; 6462 mddev->chunk_sectors = conf->chunk_sectors; 6463 mddev->reshape_position = MaxSector; 6464 mddev->delta_disks = 0; 6465 mddev->reshape_backwards = 0; 6466 } 6467 } 6468 6469 static void raid5_quiesce(struct mddev *mddev, int state) 6470 { 6471 struct r5conf *conf = mddev->private; 6472 6473 switch(state) { 6474 case 2: /* resume for a suspend */ 6475 wake_up(&conf->wait_for_overlap); 6476 break; 6477 6478 case 1: /* stop all writes */ 6479 spin_lock_irq(&conf->device_lock); 6480 /* '2' tells resync/reshape to pause so that all 6481 * active stripes can drain 6482 */ 6483 conf->quiesce = 2; 6484 wait_event_lock_irq(conf->wait_for_stripe, 6485 atomic_read(&conf->active_stripes) == 0 && 6486 atomic_read(&conf->active_aligned_reads) == 0, 6487 conf->device_lock); 6488 conf->quiesce = 1; 6489 spin_unlock_irq(&conf->device_lock); 6490 /* allow reshape to continue */ 6491 wake_up(&conf->wait_for_overlap); 6492 break; 6493 6494 case 0: /* re-enable writes */ 6495 spin_lock_irq(&conf->device_lock); 6496 conf->quiesce = 0; 6497 wake_up(&conf->wait_for_stripe); 6498 wake_up(&conf->wait_for_overlap); 6499 spin_unlock_irq(&conf->device_lock); 6500 break; 6501 } 6502 } 6503 6504 6505 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 6506 { 6507 struct r0conf *raid0_conf = mddev->private; 6508 sector_t sectors; 6509 6510 /* for raid0 takeover only one zone is supported */ 6511 if (raid0_conf->nr_strip_zones > 1) { 6512 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 6513 mdname(mddev)); 6514 return ERR_PTR(-EINVAL); 6515 } 6516 6517 sectors = raid0_conf->strip_zone[0].zone_end; 6518 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 6519 mddev->dev_sectors = sectors; 6520 mddev->new_level = level; 6521 mddev->new_layout = ALGORITHM_PARITY_N; 6522 mddev->new_chunk_sectors = mddev->chunk_sectors; 6523 mddev->raid_disks += 1; 6524 mddev->delta_disks = 1; 6525 /* make sure it will be not marked as dirty */ 6526 mddev->recovery_cp = MaxSector; 6527 6528 return setup_conf(mddev); 6529 } 6530 6531 6532 static void *raid5_takeover_raid1(struct mddev *mddev) 6533 { 6534 int chunksect; 6535 6536 if (mddev->raid_disks != 2 || 6537 mddev->degraded > 1) 6538 return ERR_PTR(-EINVAL); 6539 6540 /* Should check if there are write-behind devices? */ 6541 6542 chunksect = 64*2; /* 64K by default */ 6543 6544 /* The array must be an exact multiple of chunksize */ 6545 while (chunksect && (mddev->array_sectors & (chunksect-1))) 6546 chunksect >>= 1; 6547 6548 if ((chunksect<<9) < STRIPE_SIZE) 6549 /* array size does not allow a suitable chunk size */ 6550 return ERR_PTR(-EINVAL); 6551 6552 mddev->new_level = 5; 6553 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 6554 mddev->new_chunk_sectors = chunksect; 6555 6556 return setup_conf(mddev); 6557 } 6558 6559 static void *raid5_takeover_raid6(struct mddev *mddev) 6560 { 6561 int new_layout; 6562 6563 switch (mddev->layout) { 6564 case ALGORITHM_LEFT_ASYMMETRIC_6: 6565 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 6566 break; 6567 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6568 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 6569 break; 6570 case ALGORITHM_LEFT_SYMMETRIC_6: 6571 new_layout = ALGORITHM_LEFT_SYMMETRIC; 6572 break; 6573 case ALGORITHM_RIGHT_SYMMETRIC_6: 6574 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 6575 break; 6576 case ALGORITHM_PARITY_0_6: 6577 new_layout = ALGORITHM_PARITY_0; 6578 break; 6579 case ALGORITHM_PARITY_N: 6580 new_layout = ALGORITHM_PARITY_N; 6581 break; 6582 default: 6583 return ERR_PTR(-EINVAL); 6584 } 6585 mddev->new_level = 5; 6586 mddev->new_layout = new_layout; 6587 mddev->delta_disks = -1; 6588 mddev->raid_disks -= 1; 6589 return setup_conf(mddev); 6590 } 6591 6592 6593 static int raid5_check_reshape(struct mddev *mddev) 6594 { 6595 /* For a 2-drive array, the layout and chunk size can be changed 6596 * immediately as not restriping is needed. 6597 * For larger arrays we record the new value - after validation 6598 * to be used by a reshape pass. 6599 */ 6600 struct r5conf *conf = mddev->private; 6601 int new_chunk = mddev->new_chunk_sectors; 6602 6603 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 6604 return -EINVAL; 6605 if (new_chunk > 0) { 6606 if (!is_power_of_2(new_chunk)) 6607 return -EINVAL; 6608 if (new_chunk < (PAGE_SIZE>>9)) 6609 return -EINVAL; 6610 if (mddev->array_sectors & (new_chunk-1)) 6611 /* not factor of array size */ 6612 return -EINVAL; 6613 } 6614 6615 /* They look valid */ 6616 6617 if (mddev->raid_disks == 2) { 6618 /* can make the change immediately */ 6619 if (mddev->new_layout >= 0) { 6620 conf->algorithm = mddev->new_layout; 6621 mddev->layout = mddev->new_layout; 6622 } 6623 if (new_chunk > 0) { 6624 conf->chunk_sectors = new_chunk ; 6625 mddev->chunk_sectors = new_chunk; 6626 } 6627 set_bit(MD_CHANGE_DEVS, &mddev->flags); 6628 md_wakeup_thread(mddev->thread); 6629 } 6630 return check_reshape(mddev); 6631 } 6632 6633 static int raid6_check_reshape(struct mddev *mddev) 6634 { 6635 int new_chunk = mddev->new_chunk_sectors; 6636 6637 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 6638 return -EINVAL; 6639 if (new_chunk > 0) { 6640 if (!is_power_of_2(new_chunk)) 6641 return -EINVAL; 6642 if (new_chunk < (PAGE_SIZE >> 9)) 6643 return -EINVAL; 6644 if (mddev->array_sectors & (new_chunk-1)) 6645 /* not factor of array size */ 6646 return -EINVAL; 6647 } 6648 6649 /* They look valid */ 6650 return check_reshape(mddev); 6651 } 6652 6653 static void *raid5_takeover(struct mddev *mddev) 6654 { 6655 /* raid5 can take over: 6656 * raid0 - if there is only one strip zone - make it a raid4 layout 6657 * raid1 - if there are two drives. We need to know the chunk size 6658 * raid4 - trivial - just use a raid4 layout. 6659 * raid6 - Providing it is a *_6 layout 6660 */ 6661 if (mddev->level == 0) 6662 return raid45_takeover_raid0(mddev, 5); 6663 if (mddev->level == 1) 6664 return raid5_takeover_raid1(mddev); 6665 if (mddev->level == 4) { 6666 mddev->new_layout = ALGORITHM_PARITY_N; 6667 mddev->new_level = 5; 6668 return setup_conf(mddev); 6669 } 6670 if (mddev->level == 6) 6671 return raid5_takeover_raid6(mddev); 6672 6673 return ERR_PTR(-EINVAL); 6674 } 6675 6676 static void *raid4_takeover(struct mddev *mddev) 6677 { 6678 /* raid4 can take over: 6679 * raid0 - if there is only one strip zone 6680 * raid5 - if layout is right 6681 */ 6682 if (mddev->level == 0) 6683 return raid45_takeover_raid0(mddev, 4); 6684 if (mddev->level == 5 && 6685 mddev->layout == ALGORITHM_PARITY_N) { 6686 mddev->new_layout = 0; 6687 mddev->new_level = 4; 6688 return setup_conf(mddev); 6689 } 6690 return ERR_PTR(-EINVAL); 6691 } 6692 6693 static struct md_personality raid5_personality; 6694 6695 static void *raid6_takeover(struct mddev *mddev) 6696 { 6697 /* Currently can only take over a raid5. We map the 6698 * personality to an equivalent raid6 personality 6699 * with the Q block at the end. 6700 */ 6701 int new_layout; 6702 6703 if (mddev->pers != &raid5_personality) 6704 return ERR_PTR(-EINVAL); 6705 if (mddev->degraded > 1) 6706 return ERR_PTR(-EINVAL); 6707 if (mddev->raid_disks > 253) 6708 return ERR_PTR(-EINVAL); 6709 if (mddev->raid_disks < 3) 6710 return ERR_PTR(-EINVAL); 6711 6712 switch (mddev->layout) { 6713 case ALGORITHM_LEFT_ASYMMETRIC: 6714 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 6715 break; 6716 case ALGORITHM_RIGHT_ASYMMETRIC: 6717 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 6718 break; 6719 case ALGORITHM_LEFT_SYMMETRIC: 6720 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 6721 break; 6722 case ALGORITHM_RIGHT_SYMMETRIC: 6723 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 6724 break; 6725 case ALGORITHM_PARITY_0: 6726 new_layout = ALGORITHM_PARITY_0_6; 6727 break; 6728 case ALGORITHM_PARITY_N: 6729 new_layout = ALGORITHM_PARITY_N; 6730 break; 6731 default: 6732 return ERR_PTR(-EINVAL); 6733 } 6734 mddev->new_level = 6; 6735 mddev->new_layout = new_layout; 6736 mddev->delta_disks = 1; 6737 mddev->raid_disks += 1; 6738 return setup_conf(mddev); 6739 } 6740 6741 6742 static struct md_personality raid6_personality = 6743 { 6744 .name = "raid6", 6745 .level = 6, 6746 .owner = THIS_MODULE, 6747 .make_request = make_request, 6748 .run = run, 6749 .stop = stop, 6750 .status = status, 6751 .error_handler = error, 6752 .hot_add_disk = raid5_add_disk, 6753 .hot_remove_disk= raid5_remove_disk, 6754 .spare_active = raid5_spare_active, 6755 .sync_request = sync_request, 6756 .resize = raid5_resize, 6757 .size = raid5_size, 6758 .check_reshape = raid6_check_reshape, 6759 .start_reshape = raid5_start_reshape, 6760 .finish_reshape = raid5_finish_reshape, 6761 .quiesce = raid5_quiesce, 6762 .takeover = raid6_takeover, 6763 }; 6764 static struct md_personality raid5_personality = 6765 { 6766 .name = "raid5", 6767 .level = 5, 6768 .owner = THIS_MODULE, 6769 .make_request = make_request, 6770 .run = run, 6771 .stop = stop, 6772 .status = status, 6773 .error_handler = error, 6774 .hot_add_disk = raid5_add_disk, 6775 .hot_remove_disk= raid5_remove_disk, 6776 .spare_active = raid5_spare_active, 6777 .sync_request = sync_request, 6778 .resize = raid5_resize, 6779 .size = raid5_size, 6780 .check_reshape = raid5_check_reshape, 6781 .start_reshape = raid5_start_reshape, 6782 .finish_reshape = raid5_finish_reshape, 6783 .quiesce = raid5_quiesce, 6784 .takeover = raid5_takeover, 6785 }; 6786 6787 static struct md_personality raid4_personality = 6788 { 6789 .name = "raid4", 6790 .level = 4, 6791 .owner = THIS_MODULE, 6792 .make_request = make_request, 6793 .run = run, 6794 .stop = stop, 6795 .status = status, 6796 .error_handler = error, 6797 .hot_add_disk = raid5_add_disk, 6798 .hot_remove_disk= raid5_remove_disk, 6799 .spare_active = raid5_spare_active, 6800 .sync_request = sync_request, 6801 .resize = raid5_resize, 6802 .size = raid5_size, 6803 .check_reshape = raid5_check_reshape, 6804 .start_reshape = raid5_start_reshape, 6805 .finish_reshape = raid5_finish_reshape, 6806 .quiesce = raid5_quiesce, 6807 .takeover = raid4_takeover, 6808 }; 6809 6810 static int __init raid5_init(void) 6811 { 6812 raid5_wq = alloc_workqueue("raid5wq", 6813 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 6814 if (!raid5_wq) 6815 return -ENOMEM; 6816 register_md_personality(&raid6_personality); 6817 register_md_personality(&raid5_personality); 6818 register_md_personality(&raid4_personality); 6819 return 0; 6820 } 6821 6822 static void raid5_exit(void) 6823 { 6824 unregister_md_personality(&raid6_personality); 6825 unregister_md_personality(&raid5_personality); 6826 unregister_md_personality(&raid4_personality); 6827 destroy_workqueue(raid5_wq); 6828 } 6829 6830 module_init(raid5_init); 6831 module_exit(raid5_exit); 6832 MODULE_LICENSE("GPL"); 6833 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 6834 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 6835 MODULE_ALIAS("md-raid5"); 6836 MODULE_ALIAS("md-raid4"); 6837 MODULE_ALIAS("md-level-5"); 6838 MODULE_ALIAS("md-level-4"); 6839 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 6840 MODULE_ALIAS("md-raid6"); 6841 MODULE_ALIAS("md-level-6"); 6842 6843 /* This used to be two separate modules, they were: */ 6844 MODULE_ALIAS("raid5"); 6845 MODULE_ALIAS("raid6"); 6846