1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include <linux/nodemask.h> 57 #include <trace/events/block.h> 58 59 #include "md.h" 60 #include "raid5.h" 61 #include "raid0.h" 62 #include "bitmap.h" 63 64 #define cpu_to_group(cpu) cpu_to_node(cpu) 65 #define ANY_GROUP NUMA_NO_NODE 66 67 static bool devices_handle_discard_safely = false; 68 module_param(devices_handle_discard_safely, bool, 0644); 69 MODULE_PARM_DESC(devices_handle_discard_safely, 70 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); 71 static struct workqueue_struct *raid5_wq; 72 /* 73 * Stripe cache 74 */ 75 76 #define NR_STRIPES 256 77 #define STRIPE_SIZE PAGE_SIZE 78 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 79 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 80 #define IO_THRESHOLD 1 81 #define BYPASS_THRESHOLD 1 82 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 83 #define HASH_MASK (NR_HASH - 1) 84 #define MAX_STRIPE_BATCH 8 85 86 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 87 { 88 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 89 return &conf->stripe_hashtbl[hash]; 90 } 91 92 static inline int stripe_hash_locks_hash(sector_t sect) 93 { 94 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; 95 } 96 97 static inline void lock_device_hash_lock(struct r5conf *conf, int hash) 98 { 99 spin_lock_irq(conf->hash_locks + hash); 100 spin_lock(&conf->device_lock); 101 } 102 103 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) 104 { 105 spin_unlock(&conf->device_lock); 106 spin_unlock_irq(conf->hash_locks + hash); 107 } 108 109 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) 110 { 111 int i; 112 local_irq_disable(); 113 spin_lock(conf->hash_locks); 114 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 115 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); 116 spin_lock(&conf->device_lock); 117 } 118 119 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) 120 { 121 int i; 122 spin_unlock(&conf->device_lock); 123 for (i = NR_STRIPE_HASH_LOCKS; i; i--) 124 spin_unlock(conf->hash_locks + i - 1); 125 local_irq_enable(); 126 } 127 128 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 129 * order without overlap. There may be several bio's per stripe+device, and 130 * a bio could span several devices. 131 * When walking this list for a particular stripe+device, we must never proceed 132 * beyond a bio that extends past this device, as the next bio might no longer 133 * be valid. 134 * This function is used to determine the 'next' bio in the list, given the sector 135 * of the current stripe+device 136 */ 137 static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) 138 { 139 int sectors = bio_sectors(bio); 140 if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) 141 return bio->bi_next; 142 else 143 return NULL; 144 } 145 146 /* 147 * We maintain a biased count of active stripes in the bottom 16 bits of 148 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 149 */ 150 static inline int raid5_bi_processed_stripes(struct bio *bio) 151 { 152 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 153 return (atomic_read(segments) >> 16) & 0xffff; 154 } 155 156 static inline int raid5_dec_bi_active_stripes(struct bio *bio) 157 { 158 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 159 return atomic_sub_return(1, segments) & 0xffff; 160 } 161 162 static inline void raid5_inc_bi_active_stripes(struct bio *bio) 163 { 164 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 165 atomic_inc(segments); 166 } 167 168 static inline void raid5_set_bi_processed_stripes(struct bio *bio, 169 unsigned int cnt) 170 { 171 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 172 int old, new; 173 174 do { 175 old = atomic_read(segments); 176 new = (old & 0xffff) | (cnt << 16); 177 } while (atomic_cmpxchg(segments, old, new) != old); 178 } 179 180 static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) 181 { 182 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 183 atomic_set(segments, cnt); 184 } 185 186 /* Find first data disk in a raid6 stripe */ 187 static inline int raid6_d0(struct stripe_head *sh) 188 { 189 if (sh->ddf_layout) 190 /* ddf always start from first device */ 191 return 0; 192 /* md starts just after Q block */ 193 if (sh->qd_idx == sh->disks - 1) 194 return 0; 195 else 196 return sh->qd_idx + 1; 197 } 198 static inline int raid6_next_disk(int disk, int raid_disks) 199 { 200 disk++; 201 return (disk < raid_disks) ? disk : 0; 202 } 203 204 /* When walking through the disks in a raid5, starting at raid6_d0, 205 * We need to map each disk to a 'slot', where the data disks are slot 206 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 207 * is raid_disks-1. This help does that mapping. 208 */ 209 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 210 int *count, int syndrome_disks) 211 { 212 int slot = *count; 213 214 if (sh->ddf_layout) 215 (*count)++; 216 if (idx == sh->pd_idx) 217 return syndrome_disks; 218 if (idx == sh->qd_idx) 219 return syndrome_disks + 1; 220 if (!sh->ddf_layout) 221 (*count)++; 222 return slot; 223 } 224 225 static void return_io(struct bio *return_bi) 226 { 227 struct bio *bi = return_bi; 228 while (bi) { 229 230 return_bi = bi->bi_next; 231 bi->bi_next = NULL; 232 bi->bi_iter.bi_size = 0; 233 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 234 bi, 0); 235 bio_endio(bi, 0); 236 bi = return_bi; 237 } 238 } 239 240 static void print_raid5_conf (struct r5conf *conf); 241 242 static int stripe_operations_active(struct stripe_head *sh) 243 { 244 return sh->check_state || sh->reconstruct_state || 245 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 246 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 247 } 248 249 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) 250 { 251 struct r5conf *conf = sh->raid_conf; 252 struct r5worker_group *group; 253 int thread_cnt; 254 int i, cpu = sh->cpu; 255 256 if (!cpu_online(cpu)) { 257 cpu = cpumask_any(cpu_online_mask); 258 sh->cpu = cpu; 259 } 260 261 if (list_empty(&sh->lru)) { 262 struct r5worker_group *group; 263 group = conf->worker_groups + cpu_to_group(cpu); 264 list_add_tail(&sh->lru, &group->handle_list); 265 group->stripes_cnt++; 266 sh->group = group; 267 } 268 269 if (conf->worker_cnt_per_group == 0) { 270 md_wakeup_thread(conf->mddev->thread); 271 return; 272 } 273 274 group = conf->worker_groups + cpu_to_group(sh->cpu); 275 276 group->workers[0].working = true; 277 /* at least one worker should run to avoid race */ 278 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); 279 280 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; 281 /* wakeup more workers */ 282 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { 283 if (group->workers[i].working == false) { 284 group->workers[i].working = true; 285 queue_work_on(sh->cpu, raid5_wq, 286 &group->workers[i].work); 287 thread_cnt--; 288 } 289 } 290 } 291 292 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, 293 struct list_head *temp_inactive_list) 294 { 295 BUG_ON(!list_empty(&sh->lru)); 296 BUG_ON(atomic_read(&conf->active_stripes)==0); 297 if (test_bit(STRIPE_HANDLE, &sh->state)) { 298 if (test_bit(STRIPE_DELAYED, &sh->state) && 299 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 300 list_add_tail(&sh->lru, &conf->delayed_list); 301 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 302 sh->bm_seq - conf->seq_write > 0) 303 list_add_tail(&sh->lru, &conf->bitmap_list); 304 else { 305 clear_bit(STRIPE_DELAYED, &sh->state); 306 clear_bit(STRIPE_BIT_DELAY, &sh->state); 307 if (conf->worker_cnt_per_group == 0) { 308 list_add_tail(&sh->lru, &conf->handle_list); 309 } else { 310 raid5_wakeup_stripe_thread(sh); 311 return; 312 } 313 } 314 md_wakeup_thread(conf->mddev->thread); 315 } else { 316 BUG_ON(stripe_operations_active(sh)); 317 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 318 if (atomic_dec_return(&conf->preread_active_stripes) 319 < IO_THRESHOLD) 320 md_wakeup_thread(conf->mddev->thread); 321 atomic_dec(&conf->active_stripes); 322 if (!test_bit(STRIPE_EXPANDING, &sh->state)) 323 list_add_tail(&sh->lru, temp_inactive_list); 324 } 325 } 326 327 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, 328 struct list_head *temp_inactive_list) 329 { 330 if (atomic_dec_and_test(&sh->count)) 331 do_release_stripe(conf, sh, temp_inactive_list); 332 } 333 334 /* 335 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list 336 * 337 * Be careful: Only one task can add/delete stripes from temp_inactive_list at 338 * given time. Adding stripes only takes device lock, while deleting stripes 339 * only takes hash lock. 340 */ 341 static void release_inactive_stripe_list(struct r5conf *conf, 342 struct list_head *temp_inactive_list, 343 int hash) 344 { 345 int size; 346 bool do_wakeup = false; 347 unsigned long flags; 348 349 if (hash == NR_STRIPE_HASH_LOCKS) { 350 size = NR_STRIPE_HASH_LOCKS; 351 hash = NR_STRIPE_HASH_LOCKS - 1; 352 } else 353 size = 1; 354 while (size) { 355 struct list_head *list = &temp_inactive_list[size - 1]; 356 357 /* 358 * We don't hold any lock here yet, get_active_stripe() might 359 * remove stripes from the list 360 */ 361 if (!list_empty_careful(list)) { 362 spin_lock_irqsave(conf->hash_locks + hash, flags); 363 if (list_empty(conf->inactive_list + hash) && 364 !list_empty(list)) 365 atomic_dec(&conf->empty_inactive_list_nr); 366 list_splice_tail_init(list, conf->inactive_list + hash); 367 do_wakeup = true; 368 spin_unlock_irqrestore(conf->hash_locks + hash, flags); 369 } 370 size--; 371 hash--; 372 } 373 374 if (do_wakeup) { 375 wake_up(&conf->wait_for_stripe); 376 if (conf->retry_read_aligned) 377 md_wakeup_thread(conf->mddev->thread); 378 } 379 } 380 381 /* should hold conf->device_lock already */ 382 static int release_stripe_list(struct r5conf *conf, 383 struct list_head *temp_inactive_list) 384 { 385 struct stripe_head *sh; 386 int count = 0; 387 struct llist_node *head; 388 389 head = llist_del_all(&conf->released_stripes); 390 head = llist_reverse_order(head); 391 while (head) { 392 int hash; 393 394 sh = llist_entry(head, struct stripe_head, release_list); 395 head = llist_next(head); 396 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ 397 smp_mb(); 398 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); 399 /* 400 * Don't worry the bit is set here, because if the bit is set 401 * again, the count is always > 1. This is true for 402 * STRIPE_ON_UNPLUG_LIST bit too. 403 */ 404 hash = sh->hash_lock_index; 405 __release_stripe(conf, sh, &temp_inactive_list[hash]); 406 count++; 407 } 408 409 return count; 410 } 411 412 static void release_stripe(struct stripe_head *sh) 413 { 414 struct r5conf *conf = sh->raid_conf; 415 unsigned long flags; 416 struct list_head list; 417 int hash; 418 bool wakeup; 419 420 /* Avoid release_list until the last reference. 421 */ 422 if (atomic_add_unless(&sh->count, -1, 1)) 423 return; 424 425 if (unlikely(!conf->mddev->thread) || 426 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) 427 goto slow_path; 428 wakeup = llist_add(&sh->release_list, &conf->released_stripes); 429 if (wakeup) 430 md_wakeup_thread(conf->mddev->thread); 431 return; 432 slow_path: 433 local_irq_save(flags); 434 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ 435 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { 436 INIT_LIST_HEAD(&list); 437 hash = sh->hash_lock_index; 438 do_release_stripe(conf, sh, &list); 439 spin_unlock(&conf->device_lock); 440 release_inactive_stripe_list(conf, &list, hash); 441 } 442 local_irq_restore(flags); 443 } 444 445 static inline void remove_hash(struct stripe_head *sh) 446 { 447 pr_debug("remove_hash(), stripe %llu\n", 448 (unsigned long long)sh->sector); 449 450 hlist_del_init(&sh->hash); 451 } 452 453 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 454 { 455 struct hlist_head *hp = stripe_hash(conf, sh->sector); 456 457 pr_debug("insert_hash(), stripe %llu\n", 458 (unsigned long long)sh->sector); 459 460 hlist_add_head(&sh->hash, hp); 461 } 462 463 /* find an idle stripe, make sure it is unhashed, and return it. */ 464 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) 465 { 466 struct stripe_head *sh = NULL; 467 struct list_head *first; 468 469 if (list_empty(conf->inactive_list + hash)) 470 goto out; 471 first = (conf->inactive_list + hash)->next; 472 sh = list_entry(first, struct stripe_head, lru); 473 list_del_init(first); 474 remove_hash(sh); 475 atomic_inc(&conf->active_stripes); 476 BUG_ON(hash != sh->hash_lock_index); 477 if (list_empty(conf->inactive_list + hash)) 478 atomic_inc(&conf->empty_inactive_list_nr); 479 out: 480 return sh; 481 } 482 483 static void shrink_buffers(struct stripe_head *sh) 484 { 485 struct page *p; 486 int i; 487 int num = sh->raid_conf->pool_size; 488 489 for (i = 0; i < num ; i++) { 490 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); 491 p = sh->dev[i].page; 492 if (!p) 493 continue; 494 sh->dev[i].page = NULL; 495 put_page(p); 496 } 497 } 498 499 static int grow_buffers(struct stripe_head *sh) 500 { 501 int i; 502 int num = sh->raid_conf->pool_size; 503 504 for (i = 0; i < num; i++) { 505 struct page *page; 506 507 if (!(page = alloc_page(GFP_KERNEL))) { 508 return 1; 509 } 510 sh->dev[i].page = page; 511 sh->dev[i].orig_page = page; 512 } 513 return 0; 514 } 515 516 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 517 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 518 struct stripe_head *sh); 519 520 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 521 { 522 struct r5conf *conf = sh->raid_conf; 523 int i, seq; 524 525 BUG_ON(atomic_read(&sh->count) != 0); 526 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 527 BUG_ON(stripe_operations_active(sh)); 528 529 pr_debug("init_stripe called, stripe %llu\n", 530 (unsigned long long)sector); 531 retry: 532 seq = read_seqcount_begin(&conf->gen_lock); 533 sh->generation = conf->generation - previous; 534 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 535 sh->sector = sector; 536 stripe_set_idx(sector, conf, previous, sh); 537 sh->state = 0; 538 539 for (i = sh->disks; i--; ) { 540 struct r5dev *dev = &sh->dev[i]; 541 542 if (dev->toread || dev->read || dev->towrite || dev->written || 543 test_bit(R5_LOCKED, &dev->flags)) { 544 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 545 (unsigned long long)sh->sector, i, dev->toread, 546 dev->read, dev->towrite, dev->written, 547 test_bit(R5_LOCKED, &dev->flags)); 548 WARN_ON(1); 549 } 550 dev->flags = 0; 551 raid5_build_block(sh, i, previous); 552 } 553 if (read_seqcount_retry(&conf->gen_lock, seq)) 554 goto retry; 555 insert_hash(conf, sh); 556 sh->cpu = smp_processor_id(); 557 } 558 559 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 560 short generation) 561 { 562 struct stripe_head *sh; 563 564 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 565 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) 566 if (sh->sector == sector && sh->generation == generation) 567 return sh; 568 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 569 return NULL; 570 } 571 572 /* 573 * Need to check if array has failed when deciding whether to: 574 * - start an array 575 * - remove non-faulty devices 576 * - add a spare 577 * - allow a reshape 578 * This determination is simple when no reshape is happening. 579 * However if there is a reshape, we need to carefully check 580 * both the before and after sections. 581 * This is because some failed devices may only affect one 582 * of the two sections, and some non-in_sync devices may 583 * be insync in the section most affected by failed devices. 584 */ 585 static int calc_degraded(struct r5conf *conf) 586 { 587 int degraded, degraded2; 588 int i; 589 590 rcu_read_lock(); 591 degraded = 0; 592 for (i = 0; i < conf->previous_raid_disks; i++) { 593 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 594 if (rdev && test_bit(Faulty, &rdev->flags)) 595 rdev = rcu_dereference(conf->disks[i].replacement); 596 if (!rdev || test_bit(Faulty, &rdev->flags)) 597 degraded++; 598 else if (test_bit(In_sync, &rdev->flags)) 599 ; 600 else 601 /* not in-sync or faulty. 602 * If the reshape increases the number of devices, 603 * this is being recovered by the reshape, so 604 * this 'previous' section is not in_sync. 605 * If the number of devices is being reduced however, 606 * the device can only be part of the array if 607 * we are reverting a reshape, so this section will 608 * be in-sync. 609 */ 610 if (conf->raid_disks >= conf->previous_raid_disks) 611 degraded++; 612 } 613 rcu_read_unlock(); 614 if (conf->raid_disks == conf->previous_raid_disks) 615 return degraded; 616 rcu_read_lock(); 617 degraded2 = 0; 618 for (i = 0; i < conf->raid_disks; i++) { 619 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 620 if (rdev && test_bit(Faulty, &rdev->flags)) 621 rdev = rcu_dereference(conf->disks[i].replacement); 622 if (!rdev || test_bit(Faulty, &rdev->flags)) 623 degraded2++; 624 else if (test_bit(In_sync, &rdev->flags)) 625 ; 626 else 627 /* not in-sync or faulty. 628 * If reshape increases the number of devices, this 629 * section has already been recovered, else it 630 * almost certainly hasn't. 631 */ 632 if (conf->raid_disks <= conf->previous_raid_disks) 633 degraded2++; 634 } 635 rcu_read_unlock(); 636 if (degraded2 > degraded) 637 return degraded2; 638 return degraded; 639 } 640 641 static int has_failed(struct r5conf *conf) 642 { 643 int degraded; 644 645 if (conf->mddev->reshape_position == MaxSector) 646 return conf->mddev->degraded > conf->max_degraded; 647 648 degraded = calc_degraded(conf); 649 if (degraded > conf->max_degraded) 650 return 1; 651 return 0; 652 } 653 654 static struct stripe_head * 655 get_active_stripe(struct r5conf *conf, sector_t sector, 656 int previous, int noblock, int noquiesce) 657 { 658 struct stripe_head *sh; 659 int hash = stripe_hash_locks_hash(sector); 660 661 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 662 663 spin_lock_irq(conf->hash_locks + hash); 664 665 do { 666 wait_event_lock_irq(conf->wait_for_stripe, 667 conf->quiesce == 0 || noquiesce, 668 *(conf->hash_locks + hash)); 669 sh = __find_stripe(conf, sector, conf->generation - previous); 670 if (!sh) { 671 if (!conf->inactive_blocked) 672 sh = get_free_stripe(conf, hash); 673 if (noblock && sh == NULL) 674 break; 675 if (!sh) { 676 conf->inactive_blocked = 1; 677 wait_event_lock_irq( 678 conf->wait_for_stripe, 679 !list_empty(conf->inactive_list + hash) && 680 (atomic_read(&conf->active_stripes) 681 < (conf->max_nr_stripes * 3 / 4) 682 || !conf->inactive_blocked), 683 *(conf->hash_locks + hash)); 684 conf->inactive_blocked = 0; 685 } else { 686 init_stripe(sh, sector, previous); 687 atomic_inc(&sh->count); 688 } 689 } else if (!atomic_inc_not_zero(&sh->count)) { 690 spin_lock(&conf->device_lock); 691 if (!atomic_read(&sh->count)) { 692 if (!test_bit(STRIPE_HANDLE, &sh->state)) 693 atomic_inc(&conf->active_stripes); 694 BUG_ON(list_empty(&sh->lru) && 695 !test_bit(STRIPE_EXPANDING, &sh->state)); 696 list_del_init(&sh->lru); 697 if (sh->group) { 698 sh->group->stripes_cnt--; 699 sh->group = NULL; 700 } 701 } 702 atomic_inc(&sh->count); 703 spin_unlock(&conf->device_lock); 704 } 705 } while (sh == NULL); 706 707 spin_unlock_irq(conf->hash_locks + hash); 708 return sh; 709 } 710 711 /* Determine if 'data_offset' or 'new_data_offset' should be used 712 * in this stripe_head. 713 */ 714 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 715 { 716 sector_t progress = conf->reshape_progress; 717 /* Need a memory barrier to make sure we see the value 718 * of conf->generation, or ->data_offset that was set before 719 * reshape_progress was updated. 720 */ 721 smp_rmb(); 722 if (progress == MaxSector) 723 return 0; 724 if (sh->generation == conf->generation - 1) 725 return 0; 726 /* We are in a reshape, and this is a new-generation stripe, 727 * so use new_data_offset. 728 */ 729 return 1; 730 } 731 732 static void 733 raid5_end_read_request(struct bio *bi, int error); 734 static void 735 raid5_end_write_request(struct bio *bi, int error); 736 737 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 738 { 739 struct r5conf *conf = sh->raid_conf; 740 int i, disks = sh->disks; 741 742 might_sleep(); 743 744 for (i = disks; i--; ) { 745 int rw; 746 int replace_only = 0; 747 struct bio *bi, *rbi; 748 struct md_rdev *rdev, *rrdev = NULL; 749 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 750 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 751 rw = WRITE_FUA; 752 else 753 rw = WRITE; 754 if (test_bit(R5_Discard, &sh->dev[i].flags)) 755 rw |= REQ_DISCARD; 756 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 757 rw = READ; 758 else if (test_and_clear_bit(R5_WantReplace, 759 &sh->dev[i].flags)) { 760 rw = WRITE; 761 replace_only = 1; 762 } else 763 continue; 764 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) 765 rw |= REQ_SYNC; 766 767 bi = &sh->dev[i].req; 768 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 769 770 rcu_read_lock(); 771 rrdev = rcu_dereference(conf->disks[i].replacement); 772 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 773 rdev = rcu_dereference(conf->disks[i].rdev); 774 if (!rdev) { 775 rdev = rrdev; 776 rrdev = NULL; 777 } 778 if (rw & WRITE) { 779 if (replace_only) 780 rdev = NULL; 781 if (rdev == rrdev) 782 /* We raced and saw duplicates */ 783 rrdev = NULL; 784 } else { 785 if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev) 786 rdev = rrdev; 787 rrdev = NULL; 788 } 789 790 if (rdev && test_bit(Faulty, &rdev->flags)) 791 rdev = NULL; 792 if (rdev) 793 atomic_inc(&rdev->nr_pending); 794 if (rrdev && test_bit(Faulty, &rrdev->flags)) 795 rrdev = NULL; 796 if (rrdev) 797 atomic_inc(&rrdev->nr_pending); 798 rcu_read_unlock(); 799 800 /* We have already checked bad blocks for reads. Now 801 * need to check for writes. We never accept write errors 802 * on the replacement, so we don't to check rrdev. 803 */ 804 while ((rw & WRITE) && rdev && 805 test_bit(WriteErrorSeen, &rdev->flags)) { 806 sector_t first_bad; 807 int bad_sectors; 808 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 809 &first_bad, &bad_sectors); 810 if (!bad) 811 break; 812 813 if (bad < 0) { 814 set_bit(BlockedBadBlocks, &rdev->flags); 815 if (!conf->mddev->external && 816 conf->mddev->flags) { 817 /* It is very unlikely, but we might 818 * still need to write out the 819 * bad block log - better give it 820 * a chance*/ 821 md_check_recovery(conf->mddev); 822 } 823 /* 824 * Because md_wait_for_blocked_rdev 825 * will dec nr_pending, we must 826 * increment it first. 827 */ 828 atomic_inc(&rdev->nr_pending); 829 md_wait_for_blocked_rdev(rdev, conf->mddev); 830 } else { 831 /* Acknowledged bad block - skip the write */ 832 rdev_dec_pending(rdev, conf->mddev); 833 rdev = NULL; 834 } 835 } 836 837 if (rdev) { 838 if (s->syncing || s->expanding || s->expanded 839 || s->replacing) 840 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 841 842 set_bit(STRIPE_IO_STARTED, &sh->state); 843 844 bio_reset(bi); 845 bi->bi_bdev = rdev->bdev; 846 bi->bi_rw = rw; 847 bi->bi_end_io = (rw & WRITE) 848 ? raid5_end_write_request 849 : raid5_end_read_request; 850 bi->bi_private = sh; 851 852 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 853 __func__, (unsigned long long)sh->sector, 854 bi->bi_rw, i); 855 atomic_inc(&sh->count); 856 if (use_new_offset(conf, sh)) 857 bi->bi_iter.bi_sector = (sh->sector 858 + rdev->new_data_offset); 859 else 860 bi->bi_iter.bi_sector = (sh->sector 861 + rdev->data_offset); 862 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 863 bi->bi_rw |= REQ_NOMERGE; 864 865 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 866 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 867 sh->dev[i].vec.bv_page = sh->dev[i].page; 868 bi->bi_vcnt = 1; 869 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 870 bi->bi_io_vec[0].bv_offset = 0; 871 bi->bi_iter.bi_size = STRIPE_SIZE; 872 /* 873 * If this is discard request, set bi_vcnt 0. We don't 874 * want to confuse SCSI because SCSI will replace payload 875 */ 876 if (rw & REQ_DISCARD) 877 bi->bi_vcnt = 0; 878 if (rrdev) 879 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 880 881 if (conf->mddev->gendisk) 882 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), 883 bi, disk_devt(conf->mddev->gendisk), 884 sh->dev[i].sector); 885 generic_make_request(bi); 886 } 887 if (rrdev) { 888 if (s->syncing || s->expanding || s->expanded 889 || s->replacing) 890 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 891 892 set_bit(STRIPE_IO_STARTED, &sh->state); 893 894 bio_reset(rbi); 895 rbi->bi_bdev = rrdev->bdev; 896 rbi->bi_rw = rw; 897 BUG_ON(!(rw & WRITE)); 898 rbi->bi_end_io = raid5_end_write_request; 899 rbi->bi_private = sh; 900 901 pr_debug("%s: for %llu schedule op %ld on " 902 "replacement disc %d\n", 903 __func__, (unsigned long long)sh->sector, 904 rbi->bi_rw, i); 905 atomic_inc(&sh->count); 906 if (use_new_offset(conf, sh)) 907 rbi->bi_iter.bi_sector = (sh->sector 908 + rrdev->new_data_offset); 909 else 910 rbi->bi_iter.bi_sector = (sh->sector 911 + rrdev->data_offset); 912 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 913 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 914 sh->dev[i].rvec.bv_page = sh->dev[i].page; 915 rbi->bi_vcnt = 1; 916 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 917 rbi->bi_io_vec[0].bv_offset = 0; 918 rbi->bi_iter.bi_size = STRIPE_SIZE; 919 /* 920 * If this is discard request, set bi_vcnt 0. We don't 921 * want to confuse SCSI because SCSI will replace payload 922 */ 923 if (rw & REQ_DISCARD) 924 rbi->bi_vcnt = 0; 925 if (conf->mddev->gendisk) 926 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), 927 rbi, disk_devt(conf->mddev->gendisk), 928 sh->dev[i].sector); 929 generic_make_request(rbi); 930 } 931 if (!rdev && !rrdev) { 932 if (rw & WRITE) 933 set_bit(STRIPE_DEGRADED, &sh->state); 934 pr_debug("skip op %ld on disc %d for sector %llu\n", 935 bi->bi_rw, i, (unsigned long long)sh->sector); 936 clear_bit(R5_LOCKED, &sh->dev[i].flags); 937 set_bit(STRIPE_HANDLE, &sh->state); 938 } 939 } 940 } 941 942 static struct dma_async_tx_descriptor * 943 async_copy_data(int frombio, struct bio *bio, struct page **page, 944 sector_t sector, struct dma_async_tx_descriptor *tx, 945 struct stripe_head *sh) 946 { 947 struct bio_vec bvl; 948 struct bvec_iter iter; 949 struct page *bio_page; 950 int page_offset; 951 struct async_submit_ctl submit; 952 enum async_tx_flags flags = 0; 953 954 if (bio->bi_iter.bi_sector >= sector) 955 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; 956 else 957 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; 958 959 if (frombio) 960 flags |= ASYNC_TX_FENCE; 961 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 962 963 bio_for_each_segment(bvl, bio, iter) { 964 int len = bvl.bv_len; 965 int clen; 966 int b_offset = 0; 967 968 if (page_offset < 0) { 969 b_offset = -page_offset; 970 page_offset += b_offset; 971 len -= b_offset; 972 } 973 974 if (len > 0 && page_offset + len > STRIPE_SIZE) 975 clen = STRIPE_SIZE - page_offset; 976 else 977 clen = len; 978 979 if (clen > 0) { 980 b_offset += bvl.bv_offset; 981 bio_page = bvl.bv_page; 982 if (frombio) { 983 if (sh->raid_conf->skip_copy && 984 b_offset == 0 && page_offset == 0 && 985 clen == STRIPE_SIZE) 986 *page = bio_page; 987 else 988 tx = async_memcpy(*page, bio_page, page_offset, 989 b_offset, clen, &submit); 990 } else 991 tx = async_memcpy(bio_page, *page, b_offset, 992 page_offset, clen, &submit); 993 } 994 /* chain the operations */ 995 submit.depend_tx = tx; 996 997 if (clen < len) /* hit end of page */ 998 break; 999 page_offset += len; 1000 } 1001 1002 return tx; 1003 } 1004 1005 static void ops_complete_biofill(void *stripe_head_ref) 1006 { 1007 struct stripe_head *sh = stripe_head_ref; 1008 struct bio *return_bi = NULL; 1009 int i; 1010 1011 pr_debug("%s: stripe %llu\n", __func__, 1012 (unsigned long long)sh->sector); 1013 1014 /* clear completed biofills */ 1015 for (i = sh->disks; i--; ) { 1016 struct r5dev *dev = &sh->dev[i]; 1017 1018 /* acknowledge completion of a biofill operation */ 1019 /* and check if we need to reply to a read request, 1020 * new R5_Wantfill requests are held off until 1021 * !STRIPE_BIOFILL_RUN 1022 */ 1023 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 1024 struct bio *rbi, *rbi2; 1025 1026 BUG_ON(!dev->read); 1027 rbi = dev->read; 1028 dev->read = NULL; 1029 while (rbi && rbi->bi_iter.bi_sector < 1030 dev->sector + STRIPE_SECTORS) { 1031 rbi2 = r5_next_bio(rbi, dev->sector); 1032 if (!raid5_dec_bi_active_stripes(rbi)) { 1033 rbi->bi_next = return_bi; 1034 return_bi = rbi; 1035 } 1036 rbi = rbi2; 1037 } 1038 } 1039 } 1040 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 1041 1042 return_io(return_bi); 1043 1044 set_bit(STRIPE_HANDLE, &sh->state); 1045 release_stripe(sh); 1046 } 1047 1048 static void ops_run_biofill(struct stripe_head *sh) 1049 { 1050 struct dma_async_tx_descriptor *tx = NULL; 1051 struct async_submit_ctl submit; 1052 int i; 1053 1054 pr_debug("%s: stripe %llu\n", __func__, 1055 (unsigned long long)sh->sector); 1056 1057 for (i = sh->disks; i--; ) { 1058 struct r5dev *dev = &sh->dev[i]; 1059 if (test_bit(R5_Wantfill, &dev->flags)) { 1060 struct bio *rbi; 1061 spin_lock_irq(&sh->stripe_lock); 1062 dev->read = rbi = dev->toread; 1063 dev->toread = NULL; 1064 spin_unlock_irq(&sh->stripe_lock); 1065 while (rbi && rbi->bi_iter.bi_sector < 1066 dev->sector + STRIPE_SECTORS) { 1067 tx = async_copy_data(0, rbi, &dev->page, 1068 dev->sector, tx, sh); 1069 rbi = r5_next_bio(rbi, dev->sector); 1070 } 1071 } 1072 } 1073 1074 atomic_inc(&sh->count); 1075 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 1076 async_trigger_callback(&submit); 1077 } 1078 1079 static void mark_target_uptodate(struct stripe_head *sh, int target) 1080 { 1081 struct r5dev *tgt; 1082 1083 if (target < 0) 1084 return; 1085 1086 tgt = &sh->dev[target]; 1087 set_bit(R5_UPTODATE, &tgt->flags); 1088 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1089 clear_bit(R5_Wantcompute, &tgt->flags); 1090 } 1091 1092 static void ops_complete_compute(void *stripe_head_ref) 1093 { 1094 struct stripe_head *sh = stripe_head_ref; 1095 1096 pr_debug("%s: stripe %llu\n", __func__, 1097 (unsigned long long)sh->sector); 1098 1099 /* mark the computed target(s) as uptodate */ 1100 mark_target_uptodate(sh, sh->ops.target); 1101 mark_target_uptodate(sh, sh->ops.target2); 1102 1103 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 1104 if (sh->check_state == check_state_compute_run) 1105 sh->check_state = check_state_compute_result; 1106 set_bit(STRIPE_HANDLE, &sh->state); 1107 release_stripe(sh); 1108 } 1109 1110 /* return a pointer to the address conversion region of the scribble buffer */ 1111 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 1112 struct raid5_percpu *percpu) 1113 { 1114 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 1115 } 1116 1117 static struct dma_async_tx_descriptor * 1118 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 1119 { 1120 int disks = sh->disks; 1121 struct page **xor_srcs = percpu->scribble; 1122 int target = sh->ops.target; 1123 struct r5dev *tgt = &sh->dev[target]; 1124 struct page *xor_dest = tgt->page; 1125 int count = 0; 1126 struct dma_async_tx_descriptor *tx; 1127 struct async_submit_ctl submit; 1128 int i; 1129 1130 pr_debug("%s: stripe %llu block: %d\n", 1131 __func__, (unsigned long long)sh->sector, target); 1132 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1133 1134 for (i = disks; i--; ) 1135 if (i != target) 1136 xor_srcs[count++] = sh->dev[i].page; 1137 1138 atomic_inc(&sh->count); 1139 1140 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 1141 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 1142 if (unlikely(count == 1)) 1143 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1144 else 1145 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1146 1147 return tx; 1148 } 1149 1150 /* set_syndrome_sources - populate source buffers for gen_syndrome 1151 * @srcs - (struct page *) array of size sh->disks 1152 * @sh - stripe_head to parse 1153 * 1154 * Populates srcs in proper layout order for the stripe and returns the 1155 * 'count' of sources to be used in a call to async_gen_syndrome. The P 1156 * destination buffer is recorded in srcs[count] and the Q destination 1157 * is recorded in srcs[count+1]]. 1158 */ 1159 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 1160 { 1161 int disks = sh->disks; 1162 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 1163 int d0_idx = raid6_d0(sh); 1164 int count; 1165 int i; 1166 1167 for (i = 0; i < disks; i++) 1168 srcs[i] = NULL; 1169 1170 count = 0; 1171 i = d0_idx; 1172 do { 1173 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1174 1175 srcs[slot] = sh->dev[i].page; 1176 i = raid6_next_disk(i, disks); 1177 } while (i != d0_idx); 1178 1179 return syndrome_disks; 1180 } 1181 1182 static struct dma_async_tx_descriptor * 1183 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 1184 { 1185 int disks = sh->disks; 1186 struct page **blocks = percpu->scribble; 1187 int target; 1188 int qd_idx = sh->qd_idx; 1189 struct dma_async_tx_descriptor *tx; 1190 struct async_submit_ctl submit; 1191 struct r5dev *tgt; 1192 struct page *dest; 1193 int i; 1194 int count; 1195 1196 if (sh->ops.target < 0) 1197 target = sh->ops.target2; 1198 else if (sh->ops.target2 < 0) 1199 target = sh->ops.target; 1200 else 1201 /* we should only have one valid target */ 1202 BUG(); 1203 BUG_ON(target < 0); 1204 pr_debug("%s: stripe %llu block: %d\n", 1205 __func__, (unsigned long long)sh->sector, target); 1206 1207 tgt = &sh->dev[target]; 1208 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1209 dest = tgt->page; 1210 1211 atomic_inc(&sh->count); 1212 1213 if (target == qd_idx) { 1214 count = set_syndrome_sources(blocks, sh); 1215 blocks[count] = NULL; /* regenerating p is not necessary */ 1216 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 1217 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1218 ops_complete_compute, sh, 1219 to_addr_conv(sh, percpu)); 1220 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1221 } else { 1222 /* Compute any data- or p-drive using XOR */ 1223 count = 0; 1224 for (i = disks; i-- ; ) { 1225 if (i == target || i == qd_idx) 1226 continue; 1227 blocks[count++] = sh->dev[i].page; 1228 } 1229 1230 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1231 NULL, ops_complete_compute, sh, 1232 to_addr_conv(sh, percpu)); 1233 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 1234 } 1235 1236 return tx; 1237 } 1238 1239 static struct dma_async_tx_descriptor * 1240 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 1241 { 1242 int i, count, disks = sh->disks; 1243 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 1244 int d0_idx = raid6_d0(sh); 1245 int faila = -1, failb = -1; 1246 int target = sh->ops.target; 1247 int target2 = sh->ops.target2; 1248 struct r5dev *tgt = &sh->dev[target]; 1249 struct r5dev *tgt2 = &sh->dev[target2]; 1250 struct dma_async_tx_descriptor *tx; 1251 struct page **blocks = percpu->scribble; 1252 struct async_submit_ctl submit; 1253 1254 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 1255 __func__, (unsigned long long)sh->sector, target, target2); 1256 BUG_ON(target < 0 || target2 < 0); 1257 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1258 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 1259 1260 /* we need to open-code set_syndrome_sources to handle the 1261 * slot number conversion for 'faila' and 'failb' 1262 */ 1263 for (i = 0; i < disks ; i++) 1264 blocks[i] = NULL; 1265 count = 0; 1266 i = d0_idx; 1267 do { 1268 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1269 1270 blocks[slot] = sh->dev[i].page; 1271 1272 if (i == target) 1273 faila = slot; 1274 if (i == target2) 1275 failb = slot; 1276 i = raid6_next_disk(i, disks); 1277 } while (i != d0_idx); 1278 1279 BUG_ON(faila == failb); 1280 if (failb < faila) 1281 swap(faila, failb); 1282 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1283 __func__, (unsigned long long)sh->sector, faila, failb); 1284 1285 atomic_inc(&sh->count); 1286 1287 if (failb == syndrome_disks+1) { 1288 /* Q disk is one of the missing disks */ 1289 if (faila == syndrome_disks) { 1290 /* Missing P+Q, just recompute */ 1291 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1292 ops_complete_compute, sh, 1293 to_addr_conv(sh, percpu)); 1294 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1295 STRIPE_SIZE, &submit); 1296 } else { 1297 struct page *dest; 1298 int data_target; 1299 int qd_idx = sh->qd_idx; 1300 1301 /* Missing D+Q: recompute D from P, then recompute Q */ 1302 if (target == qd_idx) 1303 data_target = target2; 1304 else 1305 data_target = target; 1306 1307 count = 0; 1308 for (i = disks; i-- ; ) { 1309 if (i == data_target || i == qd_idx) 1310 continue; 1311 blocks[count++] = sh->dev[i].page; 1312 } 1313 dest = sh->dev[data_target].page; 1314 init_async_submit(&submit, 1315 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1316 NULL, NULL, NULL, 1317 to_addr_conv(sh, percpu)); 1318 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1319 &submit); 1320 1321 count = set_syndrome_sources(blocks, sh); 1322 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1323 ops_complete_compute, sh, 1324 to_addr_conv(sh, percpu)); 1325 return async_gen_syndrome(blocks, 0, count+2, 1326 STRIPE_SIZE, &submit); 1327 } 1328 } else { 1329 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1330 ops_complete_compute, sh, 1331 to_addr_conv(sh, percpu)); 1332 if (failb == syndrome_disks) { 1333 /* We're missing D+P. */ 1334 return async_raid6_datap_recov(syndrome_disks+2, 1335 STRIPE_SIZE, faila, 1336 blocks, &submit); 1337 } else { 1338 /* We're missing D+D. */ 1339 return async_raid6_2data_recov(syndrome_disks+2, 1340 STRIPE_SIZE, faila, failb, 1341 blocks, &submit); 1342 } 1343 } 1344 } 1345 1346 static void ops_complete_prexor(void *stripe_head_ref) 1347 { 1348 struct stripe_head *sh = stripe_head_ref; 1349 1350 pr_debug("%s: stripe %llu\n", __func__, 1351 (unsigned long long)sh->sector); 1352 } 1353 1354 static struct dma_async_tx_descriptor * 1355 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 1356 struct dma_async_tx_descriptor *tx) 1357 { 1358 int disks = sh->disks; 1359 struct page **xor_srcs = percpu->scribble; 1360 int count = 0, pd_idx = sh->pd_idx, i; 1361 struct async_submit_ctl submit; 1362 1363 /* existing parity data subtracted */ 1364 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1365 1366 pr_debug("%s: stripe %llu\n", __func__, 1367 (unsigned long long)sh->sector); 1368 1369 for (i = disks; i--; ) { 1370 struct r5dev *dev = &sh->dev[i]; 1371 /* Only process blocks that are known to be uptodate */ 1372 if (test_bit(R5_Wantdrain, &dev->flags)) 1373 xor_srcs[count++] = dev->page; 1374 } 1375 1376 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1377 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 1378 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1379 1380 return tx; 1381 } 1382 1383 static struct dma_async_tx_descriptor * 1384 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1385 { 1386 int disks = sh->disks; 1387 int i; 1388 1389 pr_debug("%s: stripe %llu\n", __func__, 1390 (unsigned long long)sh->sector); 1391 1392 for (i = disks; i--; ) { 1393 struct r5dev *dev = &sh->dev[i]; 1394 struct bio *chosen; 1395 1396 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 1397 struct bio *wbi; 1398 1399 spin_lock_irq(&sh->stripe_lock); 1400 chosen = dev->towrite; 1401 dev->towrite = NULL; 1402 BUG_ON(dev->written); 1403 wbi = dev->written = chosen; 1404 spin_unlock_irq(&sh->stripe_lock); 1405 WARN_ON(dev->page != dev->orig_page); 1406 1407 while (wbi && wbi->bi_iter.bi_sector < 1408 dev->sector + STRIPE_SECTORS) { 1409 if (wbi->bi_rw & REQ_FUA) 1410 set_bit(R5_WantFUA, &dev->flags); 1411 if (wbi->bi_rw & REQ_SYNC) 1412 set_bit(R5_SyncIO, &dev->flags); 1413 if (wbi->bi_rw & REQ_DISCARD) 1414 set_bit(R5_Discard, &dev->flags); 1415 else { 1416 tx = async_copy_data(1, wbi, &dev->page, 1417 dev->sector, tx, sh); 1418 if (dev->page != dev->orig_page) { 1419 set_bit(R5_SkipCopy, &dev->flags); 1420 clear_bit(R5_UPTODATE, &dev->flags); 1421 clear_bit(R5_OVERWRITE, &dev->flags); 1422 } 1423 } 1424 wbi = r5_next_bio(wbi, dev->sector); 1425 } 1426 } 1427 } 1428 1429 return tx; 1430 } 1431 1432 static void ops_complete_reconstruct(void *stripe_head_ref) 1433 { 1434 struct stripe_head *sh = stripe_head_ref; 1435 int disks = sh->disks; 1436 int pd_idx = sh->pd_idx; 1437 int qd_idx = sh->qd_idx; 1438 int i; 1439 bool fua = false, sync = false, discard = false; 1440 1441 pr_debug("%s: stripe %llu\n", __func__, 1442 (unsigned long long)sh->sector); 1443 1444 for (i = disks; i--; ) { 1445 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1446 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); 1447 discard |= test_bit(R5_Discard, &sh->dev[i].flags); 1448 } 1449 1450 for (i = disks; i--; ) { 1451 struct r5dev *dev = &sh->dev[i]; 1452 1453 if (dev->written || i == pd_idx || i == qd_idx) { 1454 if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) 1455 set_bit(R5_UPTODATE, &dev->flags); 1456 if (fua) 1457 set_bit(R5_WantFUA, &dev->flags); 1458 if (sync) 1459 set_bit(R5_SyncIO, &dev->flags); 1460 } 1461 } 1462 1463 if (sh->reconstruct_state == reconstruct_state_drain_run) 1464 sh->reconstruct_state = reconstruct_state_drain_result; 1465 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1466 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1467 else { 1468 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1469 sh->reconstruct_state = reconstruct_state_result; 1470 } 1471 1472 set_bit(STRIPE_HANDLE, &sh->state); 1473 release_stripe(sh); 1474 } 1475 1476 static void 1477 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1478 struct dma_async_tx_descriptor *tx) 1479 { 1480 int disks = sh->disks; 1481 struct page **xor_srcs = percpu->scribble; 1482 struct async_submit_ctl submit; 1483 int count = 0, pd_idx = sh->pd_idx, i; 1484 struct page *xor_dest; 1485 int prexor = 0; 1486 unsigned long flags; 1487 1488 pr_debug("%s: stripe %llu\n", __func__, 1489 (unsigned long long)sh->sector); 1490 1491 for (i = 0; i < sh->disks; i++) { 1492 if (pd_idx == i) 1493 continue; 1494 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1495 break; 1496 } 1497 if (i >= sh->disks) { 1498 atomic_inc(&sh->count); 1499 set_bit(R5_Discard, &sh->dev[pd_idx].flags); 1500 ops_complete_reconstruct(sh); 1501 return; 1502 } 1503 /* check if prexor is active which means only process blocks 1504 * that are part of a read-modify-write (written) 1505 */ 1506 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1507 prexor = 1; 1508 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1509 for (i = disks; i--; ) { 1510 struct r5dev *dev = &sh->dev[i]; 1511 if (dev->written) 1512 xor_srcs[count++] = dev->page; 1513 } 1514 } else { 1515 xor_dest = sh->dev[pd_idx].page; 1516 for (i = disks; i--; ) { 1517 struct r5dev *dev = &sh->dev[i]; 1518 if (i != pd_idx) 1519 xor_srcs[count++] = dev->page; 1520 } 1521 } 1522 1523 /* 1/ if we prexor'd then the dest is reused as a source 1524 * 2/ if we did not prexor then we are redoing the parity 1525 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1526 * for the synchronous xor case 1527 */ 1528 flags = ASYNC_TX_ACK | 1529 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1530 1531 atomic_inc(&sh->count); 1532 1533 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1534 to_addr_conv(sh, percpu)); 1535 if (unlikely(count == 1)) 1536 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1537 else 1538 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1539 } 1540 1541 static void 1542 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1543 struct dma_async_tx_descriptor *tx) 1544 { 1545 struct async_submit_ctl submit; 1546 struct page **blocks = percpu->scribble; 1547 int count, i; 1548 1549 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1550 1551 for (i = 0; i < sh->disks; i++) { 1552 if (sh->pd_idx == i || sh->qd_idx == i) 1553 continue; 1554 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1555 break; 1556 } 1557 if (i >= sh->disks) { 1558 atomic_inc(&sh->count); 1559 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 1560 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 1561 ops_complete_reconstruct(sh); 1562 return; 1563 } 1564 1565 count = set_syndrome_sources(blocks, sh); 1566 1567 atomic_inc(&sh->count); 1568 1569 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1570 sh, to_addr_conv(sh, percpu)); 1571 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1572 } 1573 1574 static void ops_complete_check(void *stripe_head_ref) 1575 { 1576 struct stripe_head *sh = stripe_head_ref; 1577 1578 pr_debug("%s: stripe %llu\n", __func__, 1579 (unsigned long long)sh->sector); 1580 1581 sh->check_state = check_state_check_result; 1582 set_bit(STRIPE_HANDLE, &sh->state); 1583 release_stripe(sh); 1584 } 1585 1586 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1587 { 1588 int disks = sh->disks; 1589 int pd_idx = sh->pd_idx; 1590 int qd_idx = sh->qd_idx; 1591 struct page *xor_dest; 1592 struct page **xor_srcs = percpu->scribble; 1593 struct dma_async_tx_descriptor *tx; 1594 struct async_submit_ctl submit; 1595 int count; 1596 int i; 1597 1598 pr_debug("%s: stripe %llu\n", __func__, 1599 (unsigned long long)sh->sector); 1600 1601 count = 0; 1602 xor_dest = sh->dev[pd_idx].page; 1603 xor_srcs[count++] = xor_dest; 1604 for (i = disks; i--; ) { 1605 if (i == pd_idx || i == qd_idx) 1606 continue; 1607 xor_srcs[count++] = sh->dev[i].page; 1608 } 1609 1610 init_async_submit(&submit, 0, NULL, NULL, NULL, 1611 to_addr_conv(sh, percpu)); 1612 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1613 &sh->ops.zero_sum_result, &submit); 1614 1615 atomic_inc(&sh->count); 1616 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1617 tx = async_trigger_callback(&submit); 1618 } 1619 1620 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1621 { 1622 struct page **srcs = percpu->scribble; 1623 struct async_submit_ctl submit; 1624 int count; 1625 1626 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1627 (unsigned long long)sh->sector, checkp); 1628 1629 count = set_syndrome_sources(srcs, sh); 1630 if (!checkp) 1631 srcs[count] = NULL; 1632 1633 atomic_inc(&sh->count); 1634 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1635 sh, to_addr_conv(sh, percpu)); 1636 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1637 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1638 } 1639 1640 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1641 { 1642 int overlap_clear = 0, i, disks = sh->disks; 1643 struct dma_async_tx_descriptor *tx = NULL; 1644 struct r5conf *conf = sh->raid_conf; 1645 int level = conf->level; 1646 struct raid5_percpu *percpu; 1647 unsigned long cpu; 1648 1649 cpu = get_cpu(); 1650 percpu = per_cpu_ptr(conf->percpu, cpu); 1651 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1652 ops_run_biofill(sh); 1653 overlap_clear++; 1654 } 1655 1656 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1657 if (level < 6) 1658 tx = ops_run_compute5(sh, percpu); 1659 else { 1660 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1661 tx = ops_run_compute6_1(sh, percpu); 1662 else 1663 tx = ops_run_compute6_2(sh, percpu); 1664 } 1665 /* terminate the chain if reconstruct is not set to be run */ 1666 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1667 async_tx_ack(tx); 1668 } 1669 1670 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1671 tx = ops_run_prexor(sh, percpu, tx); 1672 1673 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1674 tx = ops_run_biodrain(sh, tx); 1675 overlap_clear++; 1676 } 1677 1678 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1679 if (level < 6) 1680 ops_run_reconstruct5(sh, percpu, tx); 1681 else 1682 ops_run_reconstruct6(sh, percpu, tx); 1683 } 1684 1685 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1686 if (sh->check_state == check_state_run) 1687 ops_run_check_p(sh, percpu); 1688 else if (sh->check_state == check_state_run_q) 1689 ops_run_check_pq(sh, percpu, 0); 1690 else if (sh->check_state == check_state_run_pq) 1691 ops_run_check_pq(sh, percpu, 1); 1692 else 1693 BUG(); 1694 } 1695 1696 if (overlap_clear) 1697 for (i = disks; i--; ) { 1698 struct r5dev *dev = &sh->dev[i]; 1699 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1700 wake_up(&sh->raid_conf->wait_for_overlap); 1701 } 1702 put_cpu(); 1703 } 1704 1705 static int grow_one_stripe(struct r5conf *conf, int hash) 1706 { 1707 struct stripe_head *sh; 1708 sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL); 1709 if (!sh) 1710 return 0; 1711 1712 sh->raid_conf = conf; 1713 1714 spin_lock_init(&sh->stripe_lock); 1715 1716 if (grow_buffers(sh)) { 1717 shrink_buffers(sh); 1718 kmem_cache_free(conf->slab_cache, sh); 1719 return 0; 1720 } 1721 sh->hash_lock_index = hash; 1722 /* we just created an active stripe so... */ 1723 atomic_set(&sh->count, 1); 1724 atomic_inc(&conf->active_stripes); 1725 INIT_LIST_HEAD(&sh->lru); 1726 release_stripe(sh); 1727 return 1; 1728 } 1729 1730 static int grow_stripes(struct r5conf *conf, int num) 1731 { 1732 struct kmem_cache *sc; 1733 int devs = max(conf->raid_disks, conf->previous_raid_disks); 1734 int hash; 1735 1736 if (conf->mddev->gendisk) 1737 sprintf(conf->cache_name[0], 1738 "raid%d-%s", conf->level, mdname(conf->mddev)); 1739 else 1740 sprintf(conf->cache_name[0], 1741 "raid%d-%p", conf->level, conf->mddev); 1742 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 1743 1744 conf->active_name = 0; 1745 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1746 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1747 0, 0, NULL); 1748 if (!sc) 1749 return 1; 1750 conf->slab_cache = sc; 1751 conf->pool_size = devs; 1752 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 1753 while (num--) { 1754 if (!grow_one_stripe(conf, hash)) 1755 return 1; 1756 conf->max_nr_stripes++; 1757 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS; 1758 } 1759 return 0; 1760 } 1761 1762 /** 1763 * scribble_len - return the required size of the scribble region 1764 * @num - total number of disks in the array 1765 * 1766 * The size must be enough to contain: 1767 * 1/ a struct page pointer for each device in the array +2 1768 * 2/ room to convert each entry in (1) to its corresponding dma 1769 * (dma_map_page()) or page (page_address()) address. 1770 * 1771 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1772 * calculate over all devices (not just the data blocks), using zeros in place 1773 * of the P and Q blocks. 1774 */ 1775 static size_t scribble_len(int num) 1776 { 1777 size_t len; 1778 1779 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1780 1781 return len; 1782 } 1783 1784 static int resize_stripes(struct r5conf *conf, int newsize) 1785 { 1786 /* Make all the stripes able to hold 'newsize' devices. 1787 * New slots in each stripe get 'page' set to a new page. 1788 * 1789 * This happens in stages: 1790 * 1/ create a new kmem_cache and allocate the required number of 1791 * stripe_heads. 1792 * 2/ gather all the old stripe_heads and transfer the pages across 1793 * to the new stripe_heads. This will have the side effect of 1794 * freezing the array as once all stripe_heads have been collected, 1795 * no IO will be possible. Old stripe heads are freed once their 1796 * pages have been transferred over, and the old kmem_cache is 1797 * freed when all stripes are done. 1798 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1799 * we simple return a failre status - no need to clean anything up. 1800 * 4/ allocate new pages for the new slots in the new stripe_heads. 1801 * If this fails, we don't bother trying the shrink the 1802 * stripe_heads down again, we just leave them as they are. 1803 * As each stripe_head is processed the new one is released into 1804 * active service. 1805 * 1806 * Once step2 is started, we cannot afford to wait for a write, 1807 * so we use GFP_NOIO allocations. 1808 */ 1809 struct stripe_head *osh, *nsh; 1810 LIST_HEAD(newstripes); 1811 struct disk_info *ndisks; 1812 unsigned long cpu; 1813 int err; 1814 struct kmem_cache *sc; 1815 int i; 1816 int hash, cnt; 1817 1818 if (newsize <= conf->pool_size) 1819 return 0; /* never bother to shrink */ 1820 1821 err = md_allow_write(conf->mddev); 1822 if (err) 1823 return err; 1824 1825 /* Step 1 */ 1826 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1827 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1828 0, 0, NULL); 1829 if (!sc) 1830 return -ENOMEM; 1831 1832 for (i = conf->max_nr_stripes; i; i--) { 1833 nsh = kmem_cache_zalloc(sc, GFP_KERNEL); 1834 if (!nsh) 1835 break; 1836 1837 nsh->raid_conf = conf; 1838 spin_lock_init(&nsh->stripe_lock); 1839 1840 list_add(&nsh->lru, &newstripes); 1841 } 1842 if (i) { 1843 /* didn't get enough, give up */ 1844 while (!list_empty(&newstripes)) { 1845 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1846 list_del(&nsh->lru); 1847 kmem_cache_free(sc, nsh); 1848 } 1849 kmem_cache_destroy(sc); 1850 return -ENOMEM; 1851 } 1852 /* Step 2 - Must use GFP_NOIO now. 1853 * OK, we have enough stripes, start collecting inactive 1854 * stripes and copying them over 1855 */ 1856 hash = 0; 1857 cnt = 0; 1858 list_for_each_entry(nsh, &newstripes, lru) { 1859 lock_device_hash_lock(conf, hash); 1860 wait_event_cmd(conf->wait_for_stripe, 1861 !list_empty(conf->inactive_list + hash), 1862 unlock_device_hash_lock(conf, hash), 1863 lock_device_hash_lock(conf, hash)); 1864 osh = get_free_stripe(conf, hash); 1865 unlock_device_hash_lock(conf, hash); 1866 atomic_set(&nsh->count, 1); 1867 for(i=0; i<conf->pool_size; i++) { 1868 nsh->dev[i].page = osh->dev[i].page; 1869 nsh->dev[i].orig_page = osh->dev[i].page; 1870 } 1871 for( ; i<newsize; i++) 1872 nsh->dev[i].page = NULL; 1873 nsh->hash_lock_index = hash; 1874 kmem_cache_free(conf->slab_cache, osh); 1875 cnt++; 1876 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + 1877 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { 1878 hash++; 1879 cnt = 0; 1880 } 1881 } 1882 kmem_cache_destroy(conf->slab_cache); 1883 1884 /* Step 3. 1885 * At this point, we are holding all the stripes so the array 1886 * is completely stalled, so now is a good time to resize 1887 * conf->disks and the scribble region 1888 */ 1889 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1890 if (ndisks) { 1891 for (i=0; i<conf->raid_disks; i++) 1892 ndisks[i] = conf->disks[i]; 1893 kfree(conf->disks); 1894 conf->disks = ndisks; 1895 } else 1896 err = -ENOMEM; 1897 1898 get_online_cpus(); 1899 conf->scribble_len = scribble_len(newsize); 1900 for_each_present_cpu(cpu) { 1901 struct raid5_percpu *percpu; 1902 void *scribble; 1903 1904 percpu = per_cpu_ptr(conf->percpu, cpu); 1905 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1906 1907 if (scribble) { 1908 kfree(percpu->scribble); 1909 percpu->scribble = scribble; 1910 } else { 1911 err = -ENOMEM; 1912 break; 1913 } 1914 } 1915 put_online_cpus(); 1916 1917 /* Step 4, return new stripes to service */ 1918 while(!list_empty(&newstripes)) { 1919 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1920 list_del_init(&nsh->lru); 1921 1922 for (i=conf->raid_disks; i < newsize; i++) 1923 if (nsh->dev[i].page == NULL) { 1924 struct page *p = alloc_page(GFP_NOIO); 1925 nsh->dev[i].page = p; 1926 nsh->dev[i].orig_page = p; 1927 if (!p) 1928 err = -ENOMEM; 1929 } 1930 release_stripe(nsh); 1931 } 1932 /* critical section pass, GFP_NOIO no longer needed */ 1933 1934 conf->slab_cache = sc; 1935 conf->active_name = 1-conf->active_name; 1936 conf->pool_size = newsize; 1937 return err; 1938 } 1939 1940 static int drop_one_stripe(struct r5conf *conf, int hash) 1941 { 1942 struct stripe_head *sh; 1943 1944 spin_lock_irq(conf->hash_locks + hash); 1945 sh = get_free_stripe(conf, hash); 1946 spin_unlock_irq(conf->hash_locks + hash); 1947 if (!sh) 1948 return 0; 1949 BUG_ON(atomic_read(&sh->count)); 1950 shrink_buffers(sh); 1951 kmem_cache_free(conf->slab_cache, sh); 1952 atomic_dec(&conf->active_stripes); 1953 return 1; 1954 } 1955 1956 static void shrink_stripes(struct r5conf *conf) 1957 { 1958 int hash; 1959 for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++) 1960 while (drop_one_stripe(conf, hash)) 1961 ; 1962 1963 if (conf->slab_cache) 1964 kmem_cache_destroy(conf->slab_cache); 1965 conf->slab_cache = NULL; 1966 } 1967 1968 static void raid5_end_read_request(struct bio * bi, int error) 1969 { 1970 struct stripe_head *sh = bi->bi_private; 1971 struct r5conf *conf = sh->raid_conf; 1972 int disks = sh->disks, i; 1973 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1974 char b[BDEVNAME_SIZE]; 1975 struct md_rdev *rdev = NULL; 1976 sector_t s; 1977 1978 for (i=0 ; i<disks; i++) 1979 if (bi == &sh->dev[i].req) 1980 break; 1981 1982 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1983 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1984 uptodate); 1985 if (i == disks) { 1986 BUG(); 1987 return; 1988 } 1989 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1990 /* If replacement finished while this request was outstanding, 1991 * 'replacement' might be NULL already. 1992 * In that case it moved down to 'rdev'. 1993 * rdev is not removed until all requests are finished. 1994 */ 1995 rdev = conf->disks[i].replacement; 1996 if (!rdev) 1997 rdev = conf->disks[i].rdev; 1998 1999 if (use_new_offset(conf, sh)) 2000 s = sh->sector + rdev->new_data_offset; 2001 else 2002 s = sh->sector + rdev->data_offset; 2003 if (uptodate) { 2004 set_bit(R5_UPTODATE, &sh->dev[i].flags); 2005 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2006 /* Note that this cannot happen on a 2007 * replacement device. We just fail those on 2008 * any error 2009 */ 2010 printk_ratelimited( 2011 KERN_INFO 2012 "md/raid:%s: read error corrected" 2013 " (%lu sectors at %llu on %s)\n", 2014 mdname(conf->mddev), STRIPE_SECTORS, 2015 (unsigned long long)s, 2016 bdevname(rdev->bdev, b)); 2017 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 2018 clear_bit(R5_ReadError, &sh->dev[i].flags); 2019 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2020 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2021 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2022 2023 if (atomic_read(&rdev->read_errors)) 2024 atomic_set(&rdev->read_errors, 0); 2025 } else { 2026 const char *bdn = bdevname(rdev->bdev, b); 2027 int retry = 0; 2028 int set_bad = 0; 2029 2030 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 2031 atomic_inc(&rdev->read_errors); 2032 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2033 printk_ratelimited( 2034 KERN_WARNING 2035 "md/raid:%s: read error on replacement device " 2036 "(sector %llu on %s).\n", 2037 mdname(conf->mddev), 2038 (unsigned long long)s, 2039 bdn); 2040 else if (conf->mddev->degraded >= conf->max_degraded) { 2041 set_bad = 1; 2042 printk_ratelimited( 2043 KERN_WARNING 2044 "md/raid:%s: read error not correctable " 2045 "(sector %llu on %s).\n", 2046 mdname(conf->mddev), 2047 (unsigned long long)s, 2048 bdn); 2049 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { 2050 /* Oh, no!!! */ 2051 set_bad = 1; 2052 printk_ratelimited( 2053 KERN_WARNING 2054 "md/raid:%s: read error NOT corrected!! " 2055 "(sector %llu on %s).\n", 2056 mdname(conf->mddev), 2057 (unsigned long long)s, 2058 bdn); 2059 } else if (atomic_read(&rdev->read_errors) 2060 > conf->max_nr_stripes) 2061 printk(KERN_WARNING 2062 "md/raid:%s: Too many read errors, failing device %s.\n", 2063 mdname(conf->mddev), bdn); 2064 else 2065 retry = 1; 2066 if (set_bad && test_bit(In_sync, &rdev->flags) 2067 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2068 retry = 1; 2069 if (retry) 2070 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { 2071 set_bit(R5_ReadError, &sh->dev[i].flags); 2072 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2073 } else 2074 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2075 else { 2076 clear_bit(R5_ReadError, &sh->dev[i].flags); 2077 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2078 if (!(set_bad 2079 && test_bit(In_sync, &rdev->flags) 2080 && rdev_set_badblocks( 2081 rdev, sh->sector, STRIPE_SECTORS, 0))) 2082 md_error(conf->mddev, rdev); 2083 } 2084 } 2085 rdev_dec_pending(rdev, conf->mddev); 2086 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2087 set_bit(STRIPE_HANDLE, &sh->state); 2088 release_stripe(sh); 2089 } 2090 2091 static void raid5_end_write_request(struct bio *bi, int error) 2092 { 2093 struct stripe_head *sh = bi->bi_private; 2094 struct r5conf *conf = sh->raid_conf; 2095 int disks = sh->disks, i; 2096 struct md_rdev *uninitialized_var(rdev); 2097 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 2098 sector_t first_bad; 2099 int bad_sectors; 2100 int replacement = 0; 2101 2102 for (i = 0 ; i < disks; i++) { 2103 if (bi == &sh->dev[i].req) { 2104 rdev = conf->disks[i].rdev; 2105 break; 2106 } 2107 if (bi == &sh->dev[i].rreq) { 2108 rdev = conf->disks[i].replacement; 2109 if (rdev) 2110 replacement = 1; 2111 else 2112 /* rdev was removed and 'replacement' 2113 * replaced it. rdev is not removed 2114 * until all requests are finished. 2115 */ 2116 rdev = conf->disks[i].rdev; 2117 break; 2118 } 2119 } 2120 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 2121 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2122 uptodate); 2123 if (i == disks) { 2124 BUG(); 2125 return; 2126 } 2127 2128 if (replacement) { 2129 if (!uptodate) 2130 md_error(conf->mddev, rdev); 2131 else if (is_badblock(rdev, sh->sector, 2132 STRIPE_SECTORS, 2133 &first_bad, &bad_sectors)) 2134 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2135 } else { 2136 if (!uptodate) { 2137 set_bit(STRIPE_DEGRADED, &sh->state); 2138 set_bit(WriteErrorSeen, &rdev->flags); 2139 set_bit(R5_WriteError, &sh->dev[i].flags); 2140 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2141 set_bit(MD_RECOVERY_NEEDED, 2142 &rdev->mddev->recovery); 2143 } else if (is_badblock(rdev, sh->sector, 2144 STRIPE_SECTORS, 2145 &first_bad, &bad_sectors)) { 2146 set_bit(R5_MadeGood, &sh->dev[i].flags); 2147 if (test_bit(R5_ReadError, &sh->dev[i].flags)) 2148 /* That was a successful write so make 2149 * sure it looks like we already did 2150 * a re-write. 2151 */ 2152 set_bit(R5_ReWrite, &sh->dev[i].flags); 2153 } 2154 } 2155 rdev_dec_pending(rdev, conf->mddev); 2156 2157 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2158 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2159 set_bit(STRIPE_HANDLE, &sh->state); 2160 release_stripe(sh); 2161 } 2162 2163 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 2164 2165 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 2166 { 2167 struct r5dev *dev = &sh->dev[i]; 2168 2169 bio_init(&dev->req); 2170 dev->req.bi_io_vec = &dev->vec; 2171 dev->req.bi_max_vecs = 1; 2172 dev->req.bi_private = sh; 2173 2174 bio_init(&dev->rreq); 2175 dev->rreq.bi_io_vec = &dev->rvec; 2176 dev->rreq.bi_max_vecs = 1; 2177 dev->rreq.bi_private = sh; 2178 2179 dev->flags = 0; 2180 dev->sector = compute_blocknr(sh, i, previous); 2181 } 2182 2183 static void error(struct mddev *mddev, struct md_rdev *rdev) 2184 { 2185 char b[BDEVNAME_SIZE]; 2186 struct r5conf *conf = mddev->private; 2187 unsigned long flags; 2188 pr_debug("raid456: error called\n"); 2189 2190 spin_lock_irqsave(&conf->device_lock, flags); 2191 clear_bit(In_sync, &rdev->flags); 2192 mddev->degraded = calc_degraded(conf); 2193 spin_unlock_irqrestore(&conf->device_lock, flags); 2194 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2195 2196 set_bit(Blocked, &rdev->flags); 2197 set_bit(Faulty, &rdev->flags); 2198 set_bit(MD_CHANGE_DEVS, &mddev->flags); 2199 printk(KERN_ALERT 2200 "md/raid:%s: Disk failure on %s, disabling device.\n" 2201 "md/raid:%s: Operation continuing on %d devices.\n", 2202 mdname(mddev), 2203 bdevname(rdev->bdev, b), 2204 mdname(mddev), 2205 conf->raid_disks - mddev->degraded); 2206 } 2207 2208 /* 2209 * Input: a 'big' sector number, 2210 * Output: index of the data and parity disk, and the sector # in them. 2211 */ 2212 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 2213 int previous, int *dd_idx, 2214 struct stripe_head *sh) 2215 { 2216 sector_t stripe, stripe2; 2217 sector_t chunk_number; 2218 unsigned int chunk_offset; 2219 int pd_idx, qd_idx; 2220 int ddf_layout = 0; 2221 sector_t new_sector; 2222 int algorithm = previous ? conf->prev_algo 2223 : conf->algorithm; 2224 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2225 : conf->chunk_sectors; 2226 int raid_disks = previous ? conf->previous_raid_disks 2227 : conf->raid_disks; 2228 int data_disks = raid_disks - conf->max_degraded; 2229 2230 /* First compute the information on this sector */ 2231 2232 /* 2233 * Compute the chunk number and the sector offset inside the chunk 2234 */ 2235 chunk_offset = sector_div(r_sector, sectors_per_chunk); 2236 chunk_number = r_sector; 2237 2238 /* 2239 * Compute the stripe number 2240 */ 2241 stripe = chunk_number; 2242 *dd_idx = sector_div(stripe, data_disks); 2243 stripe2 = stripe; 2244 /* 2245 * Select the parity disk based on the user selected algorithm. 2246 */ 2247 pd_idx = qd_idx = -1; 2248 switch(conf->level) { 2249 case 4: 2250 pd_idx = data_disks; 2251 break; 2252 case 5: 2253 switch (algorithm) { 2254 case ALGORITHM_LEFT_ASYMMETRIC: 2255 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2256 if (*dd_idx >= pd_idx) 2257 (*dd_idx)++; 2258 break; 2259 case ALGORITHM_RIGHT_ASYMMETRIC: 2260 pd_idx = sector_div(stripe2, raid_disks); 2261 if (*dd_idx >= pd_idx) 2262 (*dd_idx)++; 2263 break; 2264 case ALGORITHM_LEFT_SYMMETRIC: 2265 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2266 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2267 break; 2268 case ALGORITHM_RIGHT_SYMMETRIC: 2269 pd_idx = sector_div(stripe2, raid_disks); 2270 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2271 break; 2272 case ALGORITHM_PARITY_0: 2273 pd_idx = 0; 2274 (*dd_idx)++; 2275 break; 2276 case ALGORITHM_PARITY_N: 2277 pd_idx = data_disks; 2278 break; 2279 default: 2280 BUG(); 2281 } 2282 break; 2283 case 6: 2284 2285 switch (algorithm) { 2286 case ALGORITHM_LEFT_ASYMMETRIC: 2287 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2288 qd_idx = pd_idx + 1; 2289 if (pd_idx == raid_disks-1) { 2290 (*dd_idx)++; /* Q D D D P */ 2291 qd_idx = 0; 2292 } else if (*dd_idx >= pd_idx) 2293 (*dd_idx) += 2; /* D D P Q D */ 2294 break; 2295 case ALGORITHM_RIGHT_ASYMMETRIC: 2296 pd_idx = sector_div(stripe2, raid_disks); 2297 qd_idx = pd_idx + 1; 2298 if (pd_idx == raid_disks-1) { 2299 (*dd_idx)++; /* Q D D D P */ 2300 qd_idx = 0; 2301 } else if (*dd_idx >= pd_idx) 2302 (*dd_idx) += 2; /* D D P Q D */ 2303 break; 2304 case ALGORITHM_LEFT_SYMMETRIC: 2305 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2306 qd_idx = (pd_idx + 1) % raid_disks; 2307 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2308 break; 2309 case ALGORITHM_RIGHT_SYMMETRIC: 2310 pd_idx = sector_div(stripe2, raid_disks); 2311 qd_idx = (pd_idx + 1) % raid_disks; 2312 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2313 break; 2314 2315 case ALGORITHM_PARITY_0: 2316 pd_idx = 0; 2317 qd_idx = 1; 2318 (*dd_idx) += 2; 2319 break; 2320 case ALGORITHM_PARITY_N: 2321 pd_idx = data_disks; 2322 qd_idx = data_disks + 1; 2323 break; 2324 2325 case ALGORITHM_ROTATING_ZERO_RESTART: 2326 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2327 * of blocks for computing Q is different. 2328 */ 2329 pd_idx = sector_div(stripe2, raid_disks); 2330 qd_idx = pd_idx + 1; 2331 if (pd_idx == raid_disks-1) { 2332 (*dd_idx)++; /* Q D D D P */ 2333 qd_idx = 0; 2334 } else if (*dd_idx >= pd_idx) 2335 (*dd_idx) += 2; /* D D P Q D */ 2336 ddf_layout = 1; 2337 break; 2338 2339 case ALGORITHM_ROTATING_N_RESTART: 2340 /* Same a left_asymmetric, by first stripe is 2341 * D D D P Q rather than 2342 * Q D D D P 2343 */ 2344 stripe2 += 1; 2345 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2346 qd_idx = pd_idx + 1; 2347 if (pd_idx == raid_disks-1) { 2348 (*dd_idx)++; /* Q D D D P */ 2349 qd_idx = 0; 2350 } else if (*dd_idx >= pd_idx) 2351 (*dd_idx) += 2; /* D D P Q D */ 2352 ddf_layout = 1; 2353 break; 2354 2355 case ALGORITHM_ROTATING_N_CONTINUE: 2356 /* Same as left_symmetric but Q is before P */ 2357 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2358 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2359 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2360 ddf_layout = 1; 2361 break; 2362 2363 case ALGORITHM_LEFT_ASYMMETRIC_6: 2364 /* RAID5 left_asymmetric, with Q on last device */ 2365 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2366 if (*dd_idx >= pd_idx) 2367 (*dd_idx)++; 2368 qd_idx = raid_disks - 1; 2369 break; 2370 2371 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2372 pd_idx = sector_div(stripe2, raid_disks-1); 2373 if (*dd_idx >= pd_idx) 2374 (*dd_idx)++; 2375 qd_idx = raid_disks - 1; 2376 break; 2377 2378 case ALGORITHM_LEFT_SYMMETRIC_6: 2379 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2380 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2381 qd_idx = raid_disks - 1; 2382 break; 2383 2384 case ALGORITHM_RIGHT_SYMMETRIC_6: 2385 pd_idx = sector_div(stripe2, raid_disks-1); 2386 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2387 qd_idx = raid_disks - 1; 2388 break; 2389 2390 case ALGORITHM_PARITY_0_6: 2391 pd_idx = 0; 2392 (*dd_idx)++; 2393 qd_idx = raid_disks - 1; 2394 break; 2395 2396 default: 2397 BUG(); 2398 } 2399 break; 2400 } 2401 2402 if (sh) { 2403 sh->pd_idx = pd_idx; 2404 sh->qd_idx = qd_idx; 2405 sh->ddf_layout = ddf_layout; 2406 } 2407 /* 2408 * Finally, compute the new sector number 2409 */ 2410 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2411 return new_sector; 2412 } 2413 2414 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 2415 { 2416 struct r5conf *conf = sh->raid_conf; 2417 int raid_disks = sh->disks; 2418 int data_disks = raid_disks - conf->max_degraded; 2419 sector_t new_sector = sh->sector, check; 2420 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2421 : conf->chunk_sectors; 2422 int algorithm = previous ? conf->prev_algo 2423 : conf->algorithm; 2424 sector_t stripe; 2425 int chunk_offset; 2426 sector_t chunk_number; 2427 int dummy1, dd_idx = i; 2428 sector_t r_sector; 2429 struct stripe_head sh2; 2430 2431 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2432 stripe = new_sector; 2433 2434 if (i == sh->pd_idx) 2435 return 0; 2436 switch(conf->level) { 2437 case 4: break; 2438 case 5: 2439 switch (algorithm) { 2440 case ALGORITHM_LEFT_ASYMMETRIC: 2441 case ALGORITHM_RIGHT_ASYMMETRIC: 2442 if (i > sh->pd_idx) 2443 i--; 2444 break; 2445 case ALGORITHM_LEFT_SYMMETRIC: 2446 case ALGORITHM_RIGHT_SYMMETRIC: 2447 if (i < sh->pd_idx) 2448 i += raid_disks; 2449 i -= (sh->pd_idx + 1); 2450 break; 2451 case ALGORITHM_PARITY_0: 2452 i -= 1; 2453 break; 2454 case ALGORITHM_PARITY_N: 2455 break; 2456 default: 2457 BUG(); 2458 } 2459 break; 2460 case 6: 2461 if (i == sh->qd_idx) 2462 return 0; /* It is the Q disk */ 2463 switch (algorithm) { 2464 case ALGORITHM_LEFT_ASYMMETRIC: 2465 case ALGORITHM_RIGHT_ASYMMETRIC: 2466 case ALGORITHM_ROTATING_ZERO_RESTART: 2467 case ALGORITHM_ROTATING_N_RESTART: 2468 if (sh->pd_idx == raid_disks-1) 2469 i--; /* Q D D D P */ 2470 else if (i > sh->pd_idx) 2471 i -= 2; /* D D P Q D */ 2472 break; 2473 case ALGORITHM_LEFT_SYMMETRIC: 2474 case ALGORITHM_RIGHT_SYMMETRIC: 2475 if (sh->pd_idx == raid_disks-1) 2476 i--; /* Q D D D P */ 2477 else { 2478 /* D D P Q D */ 2479 if (i < sh->pd_idx) 2480 i += raid_disks; 2481 i -= (sh->pd_idx + 2); 2482 } 2483 break; 2484 case ALGORITHM_PARITY_0: 2485 i -= 2; 2486 break; 2487 case ALGORITHM_PARITY_N: 2488 break; 2489 case ALGORITHM_ROTATING_N_CONTINUE: 2490 /* Like left_symmetric, but P is before Q */ 2491 if (sh->pd_idx == 0) 2492 i--; /* P D D D Q */ 2493 else { 2494 /* D D Q P D */ 2495 if (i < sh->pd_idx) 2496 i += raid_disks; 2497 i -= (sh->pd_idx + 1); 2498 } 2499 break; 2500 case ALGORITHM_LEFT_ASYMMETRIC_6: 2501 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2502 if (i > sh->pd_idx) 2503 i--; 2504 break; 2505 case ALGORITHM_LEFT_SYMMETRIC_6: 2506 case ALGORITHM_RIGHT_SYMMETRIC_6: 2507 if (i < sh->pd_idx) 2508 i += data_disks + 1; 2509 i -= (sh->pd_idx + 1); 2510 break; 2511 case ALGORITHM_PARITY_0_6: 2512 i -= 1; 2513 break; 2514 default: 2515 BUG(); 2516 } 2517 break; 2518 } 2519 2520 chunk_number = stripe * data_disks + i; 2521 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2522 2523 check = raid5_compute_sector(conf, r_sector, 2524 previous, &dummy1, &sh2); 2525 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2526 || sh2.qd_idx != sh->qd_idx) { 2527 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2528 mdname(conf->mddev)); 2529 return 0; 2530 } 2531 return r_sector; 2532 } 2533 2534 static void 2535 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2536 int rcw, int expand) 2537 { 2538 int i, pd_idx = sh->pd_idx, disks = sh->disks; 2539 struct r5conf *conf = sh->raid_conf; 2540 int level = conf->level; 2541 2542 if (rcw) { 2543 2544 for (i = disks; i--; ) { 2545 struct r5dev *dev = &sh->dev[i]; 2546 2547 if (dev->towrite) { 2548 set_bit(R5_LOCKED, &dev->flags); 2549 set_bit(R5_Wantdrain, &dev->flags); 2550 if (!expand) 2551 clear_bit(R5_UPTODATE, &dev->flags); 2552 s->locked++; 2553 } 2554 } 2555 /* if we are not expanding this is a proper write request, and 2556 * there will be bios with new data to be drained into the 2557 * stripe cache 2558 */ 2559 if (!expand) { 2560 if (!s->locked) 2561 /* False alarm, nothing to do */ 2562 return; 2563 sh->reconstruct_state = reconstruct_state_drain_run; 2564 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2565 } else 2566 sh->reconstruct_state = reconstruct_state_run; 2567 2568 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2569 2570 if (s->locked + conf->max_degraded == disks) 2571 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2572 atomic_inc(&conf->pending_full_writes); 2573 } else { 2574 BUG_ON(level == 6); 2575 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2576 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2577 2578 for (i = disks; i--; ) { 2579 struct r5dev *dev = &sh->dev[i]; 2580 if (i == pd_idx) 2581 continue; 2582 2583 if (dev->towrite && 2584 (test_bit(R5_UPTODATE, &dev->flags) || 2585 test_bit(R5_Wantcompute, &dev->flags))) { 2586 set_bit(R5_Wantdrain, &dev->flags); 2587 set_bit(R5_LOCKED, &dev->flags); 2588 clear_bit(R5_UPTODATE, &dev->flags); 2589 s->locked++; 2590 } 2591 } 2592 if (!s->locked) 2593 /* False alarm - nothing to do */ 2594 return; 2595 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2596 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2597 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2598 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2599 } 2600 2601 /* keep the parity disk(s) locked while asynchronous operations 2602 * are in flight 2603 */ 2604 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2605 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2606 s->locked++; 2607 2608 if (level == 6) { 2609 int qd_idx = sh->qd_idx; 2610 struct r5dev *dev = &sh->dev[qd_idx]; 2611 2612 set_bit(R5_LOCKED, &dev->flags); 2613 clear_bit(R5_UPTODATE, &dev->flags); 2614 s->locked++; 2615 } 2616 2617 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2618 __func__, (unsigned long long)sh->sector, 2619 s->locked, s->ops_request); 2620 } 2621 2622 /* 2623 * Each stripe/dev can have one or more bion attached. 2624 * toread/towrite point to the first in a chain. 2625 * The bi_next chain must be in order. 2626 */ 2627 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2628 { 2629 struct bio **bip; 2630 struct r5conf *conf = sh->raid_conf; 2631 int firstwrite=0; 2632 2633 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2634 (unsigned long long)bi->bi_iter.bi_sector, 2635 (unsigned long long)sh->sector); 2636 2637 /* 2638 * If several bio share a stripe. The bio bi_phys_segments acts as a 2639 * reference count to avoid race. The reference count should already be 2640 * increased before this function is called (for example, in 2641 * make_request()), so other bio sharing this stripe will not free the 2642 * stripe. If a stripe is owned by one stripe, the stripe lock will 2643 * protect it. 2644 */ 2645 spin_lock_irq(&sh->stripe_lock); 2646 if (forwrite) { 2647 bip = &sh->dev[dd_idx].towrite; 2648 if (*bip == NULL) 2649 firstwrite = 1; 2650 } else 2651 bip = &sh->dev[dd_idx].toread; 2652 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { 2653 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) 2654 goto overlap; 2655 bip = & (*bip)->bi_next; 2656 } 2657 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) 2658 goto overlap; 2659 2660 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2661 if (*bip) 2662 bi->bi_next = *bip; 2663 *bip = bi; 2664 raid5_inc_bi_active_stripes(bi); 2665 2666 if (forwrite) { 2667 /* check if page is covered */ 2668 sector_t sector = sh->dev[dd_idx].sector; 2669 for (bi=sh->dev[dd_idx].towrite; 2670 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2671 bi && bi->bi_iter.bi_sector <= sector; 2672 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2673 if (bio_end_sector(bi) >= sector) 2674 sector = bio_end_sector(bi); 2675 } 2676 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2677 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2678 } 2679 2680 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2681 (unsigned long long)(*bip)->bi_iter.bi_sector, 2682 (unsigned long long)sh->sector, dd_idx); 2683 spin_unlock_irq(&sh->stripe_lock); 2684 2685 if (conf->mddev->bitmap && firstwrite) { 2686 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2687 STRIPE_SECTORS, 0); 2688 sh->bm_seq = conf->seq_flush+1; 2689 set_bit(STRIPE_BIT_DELAY, &sh->state); 2690 } 2691 return 1; 2692 2693 overlap: 2694 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2695 spin_unlock_irq(&sh->stripe_lock); 2696 return 0; 2697 } 2698 2699 static void end_reshape(struct r5conf *conf); 2700 2701 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 2702 struct stripe_head *sh) 2703 { 2704 int sectors_per_chunk = 2705 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2706 int dd_idx; 2707 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2708 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2709 2710 raid5_compute_sector(conf, 2711 stripe * (disks - conf->max_degraded) 2712 *sectors_per_chunk + chunk_offset, 2713 previous, 2714 &dd_idx, sh); 2715 } 2716 2717 static void 2718 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 2719 struct stripe_head_state *s, int disks, 2720 struct bio **return_bi) 2721 { 2722 int i; 2723 for (i = disks; i--; ) { 2724 struct bio *bi; 2725 int bitmap_end = 0; 2726 2727 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2728 struct md_rdev *rdev; 2729 rcu_read_lock(); 2730 rdev = rcu_dereference(conf->disks[i].rdev); 2731 if (rdev && test_bit(In_sync, &rdev->flags)) 2732 atomic_inc(&rdev->nr_pending); 2733 else 2734 rdev = NULL; 2735 rcu_read_unlock(); 2736 if (rdev) { 2737 if (!rdev_set_badblocks( 2738 rdev, 2739 sh->sector, 2740 STRIPE_SECTORS, 0)) 2741 md_error(conf->mddev, rdev); 2742 rdev_dec_pending(rdev, conf->mddev); 2743 } 2744 } 2745 spin_lock_irq(&sh->stripe_lock); 2746 /* fail all writes first */ 2747 bi = sh->dev[i].towrite; 2748 sh->dev[i].towrite = NULL; 2749 spin_unlock_irq(&sh->stripe_lock); 2750 if (bi) 2751 bitmap_end = 1; 2752 2753 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2754 wake_up(&conf->wait_for_overlap); 2755 2756 while (bi && bi->bi_iter.bi_sector < 2757 sh->dev[i].sector + STRIPE_SECTORS) { 2758 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2759 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2760 if (!raid5_dec_bi_active_stripes(bi)) { 2761 md_write_end(conf->mddev); 2762 bi->bi_next = *return_bi; 2763 *return_bi = bi; 2764 } 2765 bi = nextbi; 2766 } 2767 if (bitmap_end) 2768 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2769 STRIPE_SECTORS, 0, 0); 2770 bitmap_end = 0; 2771 /* and fail all 'written' */ 2772 bi = sh->dev[i].written; 2773 sh->dev[i].written = NULL; 2774 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { 2775 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 2776 sh->dev[i].page = sh->dev[i].orig_page; 2777 } 2778 2779 if (bi) bitmap_end = 1; 2780 while (bi && bi->bi_iter.bi_sector < 2781 sh->dev[i].sector + STRIPE_SECTORS) { 2782 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2783 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2784 if (!raid5_dec_bi_active_stripes(bi)) { 2785 md_write_end(conf->mddev); 2786 bi->bi_next = *return_bi; 2787 *return_bi = bi; 2788 } 2789 bi = bi2; 2790 } 2791 2792 /* fail any reads if this device is non-operational and 2793 * the data has not reached the cache yet. 2794 */ 2795 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2796 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2797 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2798 spin_lock_irq(&sh->stripe_lock); 2799 bi = sh->dev[i].toread; 2800 sh->dev[i].toread = NULL; 2801 spin_unlock_irq(&sh->stripe_lock); 2802 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2803 wake_up(&conf->wait_for_overlap); 2804 while (bi && bi->bi_iter.bi_sector < 2805 sh->dev[i].sector + STRIPE_SECTORS) { 2806 struct bio *nextbi = 2807 r5_next_bio(bi, sh->dev[i].sector); 2808 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2809 if (!raid5_dec_bi_active_stripes(bi)) { 2810 bi->bi_next = *return_bi; 2811 *return_bi = bi; 2812 } 2813 bi = nextbi; 2814 } 2815 } 2816 if (bitmap_end) 2817 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2818 STRIPE_SECTORS, 0, 0); 2819 /* If we were in the middle of a write the parity block might 2820 * still be locked - so just clear all R5_LOCKED flags 2821 */ 2822 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2823 } 2824 2825 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2826 if (atomic_dec_and_test(&conf->pending_full_writes)) 2827 md_wakeup_thread(conf->mddev->thread); 2828 } 2829 2830 static void 2831 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 2832 struct stripe_head_state *s) 2833 { 2834 int abort = 0; 2835 int i; 2836 2837 clear_bit(STRIPE_SYNCING, &sh->state); 2838 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 2839 wake_up(&conf->wait_for_overlap); 2840 s->syncing = 0; 2841 s->replacing = 0; 2842 /* There is nothing more to do for sync/check/repair. 2843 * Don't even need to abort as that is handled elsewhere 2844 * if needed, and not always wanted e.g. if there is a known 2845 * bad block here. 2846 * For recover/replace we need to record a bad block on all 2847 * non-sync devices, or abort the recovery 2848 */ 2849 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 2850 /* During recovery devices cannot be removed, so 2851 * locking and refcounting of rdevs is not needed 2852 */ 2853 for (i = 0; i < conf->raid_disks; i++) { 2854 struct md_rdev *rdev = conf->disks[i].rdev; 2855 if (rdev 2856 && !test_bit(Faulty, &rdev->flags) 2857 && !test_bit(In_sync, &rdev->flags) 2858 && !rdev_set_badblocks(rdev, sh->sector, 2859 STRIPE_SECTORS, 0)) 2860 abort = 1; 2861 rdev = conf->disks[i].replacement; 2862 if (rdev 2863 && !test_bit(Faulty, &rdev->flags) 2864 && !test_bit(In_sync, &rdev->flags) 2865 && !rdev_set_badblocks(rdev, sh->sector, 2866 STRIPE_SECTORS, 0)) 2867 abort = 1; 2868 } 2869 if (abort) 2870 conf->recovery_disabled = 2871 conf->mddev->recovery_disabled; 2872 } 2873 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 2874 } 2875 2876 static int want_replace(struct stripe_head *sh, int disk_idx) 2877 { 2878 struct md_rdev *rdev; 2879 int rv = 0; 2880 /* Doing recovery so rcu locking not required */ 2881 rdev = sh->raid_conf->disks[disk_idx].replacement; 2882 if (rdev 2883 && !test_bit(Faulty, &rdev->flags) 2884 && !test_bit(In_sync, &rdev->flags) 2885 && (rdev->recovery_offset <= sh->sector 2886 || rdev->mddev->recovery_cp <= sh->sector)) 2887 rv = 1; 2888 2889 return rv; 2890 } 2891 2892 /* fetch_block - checks the given member device to see if its data needs 2893 * to be read or computed to satisfy a request. 2894 * 2895 * Returns 1 when no more member devices need to be checked, otherwise returns 2896 * 0 to tell the loop in handle_stripe_fill to continue 2897 */ 2898 2899 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, 2900 int disk_idx, int disks) 2901 { 2902 struct r5dev *dev = &sh->dev[disk_idx]; 2903 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 2904 &sh->dev[s->failed_num[1]] }; 2905 int i; 2906 2907 2908 if (test_bit(R5_LOCKED, &dev->flags) || 2909 test_bit(R5_UPTODATE, &dev->flags)) 2910 /* No point reading this as we already have it or have 2911 * decided to get it. 2912 */ 2913 return 0; 2914 2915 if (dev->toread || 2916 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) 2917 /* We need this block to directly satisfy a request */ 2918 return 1; 2919 2920 if (s->syncing || s->expanding || 2921 (s->replacing && want_replace(sh, disk_idx))) 2922 /* When syncing, or expanding we read everything. 2923 * When replacing, we need the replaced block. 2924 */ 2925 return 1; 2926 2927 if ((s->failed >= 1 && fdev[0]->toread) || 2928 (s->failed >= 2 && fdev[1]->toread)) 2929 /* If we want to read from a failed device, then 2930 * we need to actually read every other device. 2931 */ 2932 return 1; 2933 2934 /* Sometimes neither read-modify-write nor reconstruct-write 2935 * cycles can work. In those cases we read every block we 2936 * can. Then the parity-update is certain to have enough to 2937 * work with. 2938 * This can only be a problem when we need to write something, 2939 * and some device has failed. If either of those tests 2940 * fail we need look no further. 2941 */ 2942 if (!s->failed || !s->to_write) 2943 return 0; 2944 2945 if (test_bit(R5_Insync, &dev->flags) && 2946 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 2947 /* Pre-reads at not permitted until after short delay 2948 * to gather multiple requests. However if this 2949 * device is no Insync, the block could only be be computed 2950 * and there is no need to delay that. 2951 */ 2952 return 0; 2953 2954 for (i = 0; i < s->failed; i++) { 2955 if (fdev[i]->towrite && 2956 !test_bit(R5_UPTODATE, &fdev[i]->flags) && 2957 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 2958 /* If we have a partial write to a failed 2959 * device, then we will need to reconstruct 2960 * the content of that device, so all other 2961 * devices must be read. 2962 */ 2963 return 1; 2964 } 2965 2966 /* If we are forced to do a reconstruct-write, either because 2967 * the current RAID6 implementation only supports that, or 2968 * or because parity cannot be trusted and we are currently 2969 * recovering it, there is extra need to be careful. 2970 * If one of the devices that we would need to read, because 2971 * it is not being overwritten (and maybe not written at all) 2972 * is missing/faulty, then we need to read everything we can. 2973 */ 2974 if (sh->raid_conf->level != 6 && 2975 sh->sector < sh->raid_conf->mddev->recovery_cp) 2976 /* reconstruct-write isn't being forced */ 2977 return 0; 2978 for (i = 0; i < s->failed; i++) { 2979 if (!test_bit(R5_UPTODATE, &fdev[i]->flags) && 2980 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 2981 return 1; 2982 } 2983 2984 return 0; 2985 } 2986 2987 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 2988 int disk_idx, int disks) 2989 { 2990 struct r5dev *dev = &sh->dev[disk_idx]; 2991 2992 /* is the data in this block needed, and can we get it? */ 2993 if (need_this_block(sh, s, disk_idx, disks)) { 2994 /* we would like to get this block, possibly by computing it, 2995 * otherwise read it if the backing disk is insync 2996 */ 2997 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2998 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2999 if ((s->uptodate == disks - 1) && 3000 (s->failed && (disk_idx == s->failed_num[0] || 3001 disk_idx == s->failed_num[1]))) { 3002 /* have disk failed, and we're requested to fetch it; 3003 * do compute it 3004 */ 3005 pr_debug("Computing stripe %llu block %d\n", 3006 (unsigned long long)sh->sector, disk_idx); 3007 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3008 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3009 set_bit(R5_Wantcompute, &dev->flags); 3010 sh->ops.target = disk_idx; 3011 sh->ops.target2 = -1; /* no 2nd target */ 3012 s->req_compute = 1; 3013 /* Careful: from this point on 'uptodate' is in the eye 3014 * of raid_run_ops which services 'compute' operations 3015 * before writes. R5_Wantcompute flags a block that will 3016 * be R5_UPTODATE by the time it is needed for a 3017 * subsequent operation. 3018 */ 3019 s->uptodate++; 3020 return 1; 3021 } else if (s->uptodate == disks-2 && s->failed >= 2) { 3022 /* Computing 2-failure is *very* expensive; only 3023 * do it if failed >= 2 3024 */ 3025 int other; 3026 for (other = disks; other--; ) { 3027 if (other == disk_idx) 3028 continue; 3029 if (!test_bit(R5_UPTODATE, 3030 &sh->dev[other].flags)) 3031 break; 3032 } 3033 BUG_ON(other < 0); 3034 pr_debug("Computing stripe %llu blocks %d,%d\n", 3035 (unsigned long long)sh->sector, 3036 disk_idx, other); 3037 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3038 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3039 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 3040 set_bit(R5_Wantcompute, &sh->dev[other].flags); 3041 sh->ops.target = disk_idx; 3042 sh->ops.target2 = other; 3043 s->uptodate += 2; 3044 s->req_compute = 1; 3045 return 1; 3046 } else if (test_bit(R5_Insync, &dev->flags)) { 3047 set_bit(R5_LOCKED, &dev->flags); 3048 set_bit(R5_Wantread, &dev->flags); 3049 s->locked++; 3050 pr_debug("Reading block %d (sync=%d)\n", 3051 disk_idx, s->syncing); 3052 } 3053 } 3054 3055 return 0; 3056 } 3057 3058 /** 3059 * handle_stripe_fill - read or compute data to satisfy pending requests. 3060 */ 3061 static void handle_stripe_fill(struct stripe_head *sh, 3062 struct stripe_head_state *s, 3063 int disks) 3064 { 3065 int i; 3066 3067 /* look for blocks to read/compute, skip this if a compute 3068 * is already in flight, or if the stripe contents are in the 3069 * midst of changing due to a write 3070 */ 3071 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 3072 !sh->reconstruct_state) 3073 for (i = disks; i--; ) 3074 if (fetch_block(sh, s, i, disks)) 3075 break; 3076 set_bit(STRIPE_HANDLE, &sh->state); 3077 } 3078 3079 /* handle_stripe_clean_event 3080 * any written block on an uptodate or failed drive can be returned. 3081 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 3082 * never LOCKED, so we don't need to test 'failed' directly. 3083 */ 3084 static void handle_stripe_clean_event(struct r5conf *conf, 3085 struct stripe_head *sh, int disks, struct bio **return_bi) 3086 { 3087 int i; 3088 struct r5dev *dev; 3089 int discard_pending = 0; 3090 3091 for (i = disks; i--; ) 3092 if (sh->dev[i].written) { 3093 dev = &sh->dev[i]; 3094 if (!test_bit(R5_LOCKED, &dev->flags) && 3095 (test_bit(R5_UPTODATE, &dev->flags) || 3096 test_bit(R5_Discard, &dev->flags) || 3097 test_bit(R5_SkipCopy, &dev->flags))) { 3098 /* We can return any write requests */ 3099 struct bio *wbi, *wbi2; 3100 pr_debug("Return write for disc %d\n", i); 3101 if (test_and_clear_bit(R5_Discard, &dev->flags)) 3102 clear_bit(R5_UPTODATE, &dev->flags); 3103 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { 3104 WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); 3105 dev->page = dev->orig_page; 3106 } 3107 wbi = dev->written; 3108 dev->written = NULL; 3109 while (wbi && wbi->bi_iter.bi_sector < 3110 dev->sector + STRIPE_SECTORS) { 3111 wbi2 = r5_next_bio(wbi, dev->sector); 3112 if (!raid5_dec_bi_active_stripes(wbi)) { 3113 md_write_end(conf->mddev); 3114 wbi->bi_next = *return_bi; 3115 *return_bi = wbi; 3116 } 3117 wbi = wbi2; 3118 } 3119 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3120 STRIPE_SECTORS, 3121 !test_bit(STRIPE_DEGRADED, &sh->state), 3122 0); 3123 } else if (test_bit(R5_Discard, &dev->flags)) 3124 discard_pending = 1; 3125 WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); 3126 WARN_ON(dev->page != dev->orig_page); 3127 } 3128 if (!discard_pending && 3129 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { 3130 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 3131 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3132 if (sh->qd_idx >= 0) { 3133 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 3134 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); 3135 } 3136 /* now that discard is done we can proceed with any sync */ 3137 clear_bit(STRIPE_DISCARD, &sh->state); 3138 /* 3139 * SCSI discard will change some bio fields and the stripe has 3140 * no updated data, so remove it from hash list and the stripe 3141 * will be reinitialized 3142 */ 3143 spin_lock_irq(&conf->device_lock); 3144 remove_hash(sh); 3145 spin_unlock_irq(&conf->device_lock); 3146 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) 3147 set_bit(STRIPE_HANDLE, &sh->state); 3148 3149 } 3150 3151 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3152 if (atomic_dec_and_test(&conf->pending_full_writes)) 3153 md_wakeup_thread(conf->mddev->thread); 3154 } 3155 3156 static void handle_stripe_dirtying(struct r5conf *conf, 3157 struct stripe_head *sh, 3158 struct stripe_head_state *s, 3159 int disks) 3160 { 3161 int rmw = 0, rcw = 0, i; 3162 sector_t recovery_cp = conf->mddev->recovery_cp; 3163 3164 /* RAID6 requires 'rcw' in current implementation. 3165 * Otherwise, check whether resync is now happening or should start. 3166 * If yes, then the array is dirty (after unclean shutdown or 3167 * initial creation), so parity in some stripes might be inconsistent. 3168 * In this case, we need to always do reconstruct-write, to ensure 3169 * that in case of drive failure or read-error correction, we 3170 * generate correct data from the parity. 3171 */ 3172 if (conf->max_degraded == 2 || 3173 (recovery_cp < MaxSector && sh->sector >= recovery_cp && 3174 s->failed == 0)) { 3175 /* Calculate the real rcw later - for now make it 3176 * look like rcw is cheaper 3177 */ 3178 rcw = 1; rmw = 2; 3179 pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n", 3180 conf->max_degraded, (unsigned long long)recovery_cp, 3181 (unsigned long long)sh->sector); 3182 } else for (i = disks; i--; ) { 3183 /* would I have to read this buffer for read_modify_write */ 3184 struct r5dev *dev = &sh->dev[i]; 3185 if ((dev->towrite || i == sh->pd_idx) && 3186 !test_bit(R5_LOCKED, &dev->flags) && 3187 !(test_bit(R5_UPTODATE, &dev->flags) || 3188 test_bit(R5_Wantcompute, &dev->flags))) { 3189 if (test_bit(R5_Insync, &dev->flags)) 3190 rmw++; 3191 else 3192 rmw += 2*disks; /* cannot read it */ 3193 } 3194 /* Would I have to read this buffer for reconstruct_write */ 3195 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 3196 !test_bit(R5_LOCKED, &dev->flags) && 3197 !(test_bit(R5_UPTODATE, &dev->flags) || 3198 test_bit(R5_Wantcompute, &dev->flags))) { 3199 if (test_bit(R5_Insync, &dev->flags)) 3200 rcw++; 3201 else 3202 rcw += 2*disks; 3203 } 3204 } 3205 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 3206 (unsigned long long)sh->sector, rmw, rcw); 3207 set_bit(STRIPE_HANDLE, &sh->state); 3208 if (rmw < rcw && rmw > 0) { 3209 /* prefer read-modify-write, but need to get some data */ 3210 if (conf->mddev->queue) 3211 blk_add_trace_msg(conf->mddev->queue, 3212 "raid5 rmw %llu %d", 3213 (unsigned long long)sh->sector, rmw); 3214 for (i = disks; i--; ) { 3215 struct r5dev *dev = &sh->dev[i]; 3216 if ((dev->towrite || i == sh->pd_idx) && 3217 !test_bit(R5_LOCKED, &dev->flags) && 3218 !(test_bit(R5_UPTODATE, &dev->flags) || 3219 test_bit(R5_Wantcompute, &dev->flags)) && 3220 test_bit(R5_Insync, &dev->flags)) { 3221 if (test_bit(STRIPE_PREREAD_ACTIVE, 3222 &sh->state)) { 3223 pr_debug("Read_old block %d for r-m-w\n", 3224 i); 3225 set_bit(R5_LOCKED, &dev->flags); 3226 set_bit(R5_Wantread, &dev->flags); 3227 s->locked++; 3228 } else { 3229 set_bit(STRIPE_DELAYED, &sh->state); 3230 set_bit(STRIPE_HANDLE, &sh->state); 3231 } 3232 } 3233 } 3234 } 3235 if (rcw <= rmw && rcw > 0) { 3236 /* want reconstruct write, but need to get some data */ 3237 int qread =0; 3238 rcw = 0; 3239 for (i = disks; i--; ) { 3240 struct r5dev *dev = &sh->dev[i]; 3241 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3242 i != sh->pd_idx && i != sh->qd_idx && 3243 !test_bit(R5_LOCKED, &dev->flags) && 3244 !(test_bit(R5_UPTODATE, &dev->flags) || 3245 test_bit(R5_Wantcompute, &dev->flags))) { 3246 rcw++; 3247 if (test_bit(R5_Insync, &dev->flags) && 3248 test_bit(STRIPE_PREREAD_ACTIVE, 3249 &sh->state)) { 3250 pr_debug("Read_old block " 3251 "%d for Reconstruct\n", i); 3252 set_bit(R5_LOCKED, &dev->flags); 3253 set_bit(R5_Wantread, &dev->flags); 3254 s->locked++; 3255 qread++; 3256 } else { 3257 set_bit(STRIPE_DELAYED, &sh->state); 3258 set_bit(STRIPE_HANDLE, &sh->state); 3259 } 3260 } 3261 } 3262 if (rcw && conf->mddev->queue) 3263 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", 3264 (unsigned long long)sh->sector, 3265 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); 3266 } 3267 3268 if (rcw > disks && rmw > disks && 3269 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3270 set_bit(STRIPE_DELAYED, &sh->state); 3271 3272 /* now if nothing is locked, and if we have enough data, 3273 * we can start a write request 3274 */ 3275 /* since handle_stripe can be called at any time we need to handle the 3276 * case where a compute block operation has been submitted and then a 3277 * subsequent call wants to start a write request. raid_run_ops only 3278 * handles the case where compute block and reconstruct are requested 3279 * simultaneously. If this is not the case then new writes need to be 3280 * held off until the compute completes. 3281 */ 3282 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 3283 (s->locked == 0 && (rcw == 0 || rmw == 0) && 3284 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 3285 schedule_reconstruction(sh, s, rcw == 0, 0); 3286 } 3287 3288 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 3289 struct stripe_head_state *s, int disks) 3290 { 3291 struct r5dev *dev = NULL; 3292 3293 set_bit(STRIPE_HANDLE, &sh->state); 3294 3295 switch (sh->check_state) { 3296 case check_state_idle: 3297 /* start a new check operation if there are no failures */ 3298 if (s->failed == 0) { 3299 BUG_ON(s->uptodate != disks); 3300 sh->check_state = check_state_run; 3301 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3302 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3303 s->uptodate--; 3304 break; 3305 } 3306 dev = &sh->dev[s->failed_num[0]]; 3307 /* fall through */ 3308 case check_state_compute_result: 3309 sh->check_state = check_state_idle; 3310 if (!dev) 3311 dev = &sh->dev[sh->pd_idx]; 3312 3313 /* check that a write has not made the stripe insync */ 3314 if (test_bit(STRIPE_INSYNC, &sh->state)) 3315 break; 3316 3317 /* either failed parity check, or recovery is happening */ 3318 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3319 BUG_ON(s->uptodate != disks); 3320 3321 set_bit(R5_LOCKED, &dev->flags); 3322 s->locked++; 3323 set_bit(R5_Wantwrite, &dev->flags); 3324 3325 clear_bit(STRIPE_DEGRADED, &sh->state); 3326 set_bit(STRIPE_INSYNC, &sh->state); 3327 break; 3328 case check_state_run: 3329 break; /* we will be called again upon completion */ 3330 case check_state_check_result: 3331 sh->check_state = check_state_idle; 3332 3333 /* if a failure occurred during the check operation, leave 3334 * STRIPE_INSYNC not set and let the stripe be handled again 3335 */ 3336 if (s->failed) 3337 break; 3338 3339 /* handle a successful check operation, if parity is correct 3340 * we are done. Otherwise update the mismatch count and repair 3341 * parity if !MD_RECOVERY_CHECK 3342 */ 3343 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 3344 /* parity is correct (on disc, 3345 * not in buffer any more) 3346 */ 3347 set_bit(STRIPE_INSYNC, &sh->state); 3348 else { 3349 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3350 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3351 /* don't try to repair!! */ 3352 set_bit(STRIPE_INSYNC, &sh->state); 3353 else { 3354 sh->check_state = check_state_compute_run; 3355 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3356 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3357 set_bit(R5_Wantcompute, 3358 &sh->dev[sh->pd_idx].flags); 3359 sh->ops.target = sh->pd_idx; 3360 sh->ops.target2 = -1; 3361 s->uptodate++; 3362 } 3363 } 3364 break; 3365 case check_state_compute_run: 3366 break; 3367 default: 3368 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3369 __func__, sh->check_state, 3370 (unsigned long long) sh->sector); 3371 BUG(); 3372 } 3373 } 3374 3375 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 3376 struct stripe_head_state *s, 3377 int disks) 3378 { 3379 int pd_idx = sh->pd_idx; 3380 int qd_idx = sh->qd_idx; 3381 struct r5dev *dev; 3382 3383 set_bit(STRIPE_HANDLE, &sh->state); 3384 3385 BUG_ON(s->failed > 2); 3386 3387 /* Want to check and possibly repair P and Q. 3388 * However there could be one 'failed' device, in which 3389 * case we can only check one of them, possibly using the 3390 * other to generate missing data 3391 */ 3392 3393 switch (sh->check_state) { 3394 case check_state_idle: 3395 /* start a new check operation if there are < 2 failures */ 3396 if (s->failed == s->q_failed) { 3397 /* The only possible failed device holds Q, so it 3398 * makes sense to check P (If anything else were failed, 3399 * we would have used P to recreate it). 3400 */ 3401 sh->check_state = check_state_run; 3402 } 3403 if (!s->q_failed && s->failed < 2) { 3404 /* Q is not failed, and we didn't use it to generate 3405 * anything, so it makes sense to check it 3406 */ 3407 if (sh->check_state == check_state_run) 3408 sh->check_state = check_state_run_pq; 3409 else 3410 sh->check_state = check_state_run_q; 3411 } 3412 3413 /* discard potentially stale zero_sum_result */ 3414 sh->ops.zero_sum_result = 0; 3415 3416 if (sh->check_state == check_state_run) { 3417 /* async_xor_zero_sum destroys the contents of P */ 3418 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 3419 s->uptodate--; 3420 } 3421 if (sh->check_state >= check_state_run && 3422 sh->check_state <= check_state_run_pq) { 3423 /* async_syndrome_zero_sum preserves P and Q, so 3424 * no need to mark them !uptodate here 3425 */ 3426 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3427 break; 3428 } 3429 3430 /* we have 2-disk failure */ 3431 BUG_ON(s->failed != 2); 3432 /* fall through */ 3433 case check_state_compute_result: 3434 sh->check_state = check_state_idle; 3435 3436 /* check that a write has not made the stripe insync */ 3437 if (test_bit(STRIPE_INSYNC, &sh->state)) 3438 break; 3439 3440 /* now write out any block on a failed drive, 3441 * or P or Q if they were recomputed 3442 */ 3443 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 3444 if (s->failed == 2) { 3445 dev = &sh->dev[s->failed_num[1]]; 3446 s->locked++; 3447 set_bit(R5_LOCKED, &dev->flags); 3448 set_bit(R5_Wantwrite, &dev->flags); 3449 } 3450 if (s->failed >= 1) { 3451 dev = &sh->dev[s->failed_num[0]]; 3452 s->locked++; 3453 set_bit(R5_LOCKED, &dev->flags); 3454 set_bit(R5_Wantwrite, &dev->flags); 3455 } 3456 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3457 dev = &sh->dev[pd_idx]; 3458 s->locked++; 3459 set_bit(R5_LOCKED, &dev->flags); 3460 set_bit(R5_Wantwrite, &dev->flags); 3461 } 3462 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3463 dev = &sh->dev[qd_idx]; 3464 s->locked++; 3465 set_bit(R5_LOCKED, &dev->flags); 3466 set_bit(R5_Wantwrite, &dev->flags); 3467 } 3468 clear_bit(STRIPE_DEGRADED, &sh->state); 3469 3470 set_bit(STRIPE_INSYNC, &sh->state); 3471 break; 3472 case check_state_run: 3473 case check_state_run_q: 3474 case check_state_run_pq: 3475 break; /* we will be called again upon completion */ 3476 case check_state_check_result: 3477 sh->check_state = check_state_idle; 3478 3479 /* handle a successful check operation, if parity is correct 3480 * we are done. Otherwise update the mismatch count and repair 3481 * parity if !MD_RECOVERY_CHECK 3482 */ 3483 if (sh->ops.zero_sum_result == 0) { 3484 /* both parities are correct */ 3485 if (!s->failed) 3486 set_bit(STRIPE_INSYNC, &sh->state); 3487 else { 3488 /* in contrast to the raid5 case we can validate 3489 * parity, but still have a failure to write 3490 * back 3491 */ 3492 sh->check_state = check_state_compute_result; 3493 /* Returning at this point means that we may go 3494 * off and bring p and/or q uptodate again so 3495 * we make sure to check zero_sum_result again 3496 * to verify if p or q need writeback 3497 */ 3498 } 3499 } else { 3500 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3501 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3502 /* don't try to repair!! */ 3503 set_bit(STRIPE_INSYNC, &sh->state); 3504 else { 3505 int *target = &sh->ops.target; 3506 3507 sh->ops.target = -1; 3508 sh->ops.target2 = -1; 3509 sh->check_state = check_state_compute_run; 3510 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3511 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3512 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3513 set_bit(R5_Wantcompute, 3514 &sh->dev[pd_idx].flags); 3515 *target = pd_idx; 3516 target = &sh->ops.target2; 3517 s->uptodate++; 3518 } 3519 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3520 set_bit(R5_Wantcompute, 3521 &sh->dev[qd_idx].flags); 3522 *target = qd_idx; 3523 s->uptodate++; 3524 } 3525 } 3526 } 3527 break; 3528 case check_state_compute_run: 3529 break; 3530 default: 3531 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3532 __func__, sh->check_state, 3533 (unsigned long long) sh->sector); 3534 BUG(); 3535 } 3536 } 3537 3538 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 3539 { 3540 int i; 3541 3542 /* We have read all the blocks in this stripe and now we need to 3543 * copy some of them into a target stripe for expand. 3544 */ 3545 struct dma_async_tx_descriptor *tx = NULL; 3546 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3547 for (i = 0; i < sh->disks; i++) 3548 if (i != sh->pd_idx && i != sh->qd_idx) { 3549 int dd_idx, j; 3550 struct stripe_head *sh2; 3551 struct async_submit_ctl submit; 3552 3553 sector_t bn = compute_blocknr(sh, i, 1); 3554 sector_t s = raid5_compute_sector(conf, bn, 0, 3555 &dd_idx, NULL); 3556 sh2 = get_active_stripe(conf, s, 0, 1, 1); 3557 if (sh2 == NULL) 3558 /* so far only the early blocks of this stripe 3559 * have been requested. When later blocks 3560 * get requested, we will try again 3561 */ 3562 continue; 3563 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 3564 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 3565 /* must have already done this block */ 3566 release_stripe(sh2); 3567 continue; 3568 } 3569 3570 /* place all the copies on one channel */ 3571 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 3572 tx = async_memcpy(sh2->dev[dd_idx].page, 3573 sh->dev[i].page, 0, 0, STRIPE_SIZE, 3574 &submit); 3575 3576 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 3577 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 3578 for (j = 0; j < conf->raid_disks; j++) 3579 if (j != sh2->pd_idx && 3580 j != sh2->qd_idx && 3581 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 3582 break; 3583 if (j == conf->raid_disks) { 3584 set_bit(STRIPE_EXPAND_READY, &sh2->state); 3585 set_bit(STRIPE_HANDLE, &sh2->state); 3586 } 3587 release_stripe(sh2); 3588 3589 } 3590 /* done submitting copies, wait for them to complete */ 3591 async_tx_quiesce(&tx); 3592 } 3593 3594 /* 3595 * handle_stripe - do things to a stripe. 3596 * 3597 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 3598 * state of various bits to see what needs to be done. 3599 * Possible results: 3600 * return some read requests which now have data 3601 * return some write requests which are safely on storage 3602 * schedule a read on some buffers 3603 * schedule a write of some buffers 3604 * return confirmation of parity correctness 3605 * 3606 */ 3607 3608 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 3609 { 3610 struct r5conf *conf = sh->raid_conf; 3611 int disks = sh->disks; 3612 struct r5dev *dev; 3613 int i; 3614 int do_recovery = 0; 3615 3616 memset(s, 0, sizeof(*s)); 3617 3618 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3619 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3620 s->failed_num[0] = -1; 3621 s->failed_num[1] = -1; 3622 3623 /* Now to look around and see what can be done */ 3624 rcu_read_lock(); 3625 for (i=disks; i--; ) { 3626 struct md_rdev *rdev; 3627 sector_t first_bad; 3628 int bad_sectors; 3629 int is_bad = 0; 3630 3631 dev = &sh->dev[i]; 3632 3633 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3634 i, dev->flags, 3635 dev->toread, dev->towrite, dev->written); 3636 /* maybe we can reply to a read 3637 * 3638 * new wantfill requests are only permitted while 3639 * ops_complete_biofill is guaranteed to be inactive 3640 */ 3641 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3642 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3643 set_bit(R5_Wantfill, &dev->flags); 3644 3645 /* now count some things */ 3646 if (test_bit(R5_LOCKED, &dev->flags)) 3647 s->locked++; 3648 if (test_bit(R5_UPTODATE, &dev->flags)) 3649 s->uptodate++; 3650 if (test_bit(R5_Wantcompute, &dev->flags)) { 3651 s->compute++; 3652 BUG_ON(s->compute > 2); 3653 } 3654 3655 if (test_bit(R5_Wantfill, &dev->flags)) 3656 s->to_fill++; 3657 else if (dev->toread) 3658 s->to_read++; 3659 if (dev->towrite) { 3660 s->to_write++; 3661 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3662 s->non_overwrite++; 3663 } 3664 if (dev->written) 3665 s->written++; 3666 /* Prefer to use the replacement for reads, but only 3667 * if it is recovered enough and has no bad blocks. 3668 */ 3669 rdev = rcu_dereference(conf->disks[i].replacement); 3670 if (rdev && !test_bit(Faulty, &rdev->flags) && 3671 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 3672 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3673 &first_bad, &bad_sectors)) 3674 set_bit(R5_ReadRepl, &dev->flags); 3675 else { 3676 if (rdev) 3677 set_bit(R5_NeedReplace, &dev->flags); 3678 rdev = rcu_dereference(conf->disks[i].rdev); 3679 clear_bit(R5_ReadRepl, &dev->flags); 3680 } 3681 if (rdev && test_bit(Faulty, &rdev->flags)) 3682 rdev = NULL; 3683 if (rdev) { 3684 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3685 &first_bad, &bad_sectors); 3686 if (s->blocked_rdev == NULL 3687 && (test_bit(Blocked, &rdev->flags) 3688 || is_bad < 0)) { 3689 if (is_bad < 0) 3690 set_bit(BlockedBadBlocks, 3691 &rdev->flags); 3692 s->blocked_rdev = rdev; 3693 atomic_inc(&rdev->nr_pending); 3694 } 3695 } 3696 clear_bit(R5_Insync, &dev->flags); 3697 if (!rdev) 3698 /* Not in-sync */; 3699 else if (is_bad) { 3700 /* also not in-sync */ 3701 if (!test_bit(WriteErrorSeen, &rdev->flags) && 3702 test_bit(R5_UPTODATE, &dev->flags)) { 3703 /* treat as in-sync, but with a read error 3704 * which we can now try to correct 3705 */ 3706 set_bit(R5_Insync, &dev->flags); 3707 set_bit(R5_ReadError, &dev->flags); 3708 } 3709 } else if (test_bit(In_sync, &rdev->flags)) 3710 set_bit(R5_Insync, &dev->flags); 3711 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3712 /* in sync if before recovery_offset */ 3713 set_bit(R5_Insync, &dev->flags); 3714 else if (test_bit(R5_UPTODATE, &dev->flags) && 3715 test_bit(R5_Expanded, &dev->flags)) 3716 /* If we've reshaped into here, we assume it is Insync. 3717 * We will shortly update recovery_offset to make 3718 * it official. 3719 */ 3720 set_bit(R5_Insync, &dev->flags); 3721 3722 if (test_bit(R5_WriteError, &dev->flags)) { 3723 /* This flag does not apply to '.replacement' 3724 * only to .rdev, so make sure to check that*/ 3725 struct md_rdev *rdev2 = rcu_dereference( 3726 conf->disks[i].rdev); 3727 if (rdev2 == rdev) 3728 clear_bit(R5_Insync, &dev->flags); 3729 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3730 s->handle_bad_blocks = 1; 3731 atomic_inc(&rdev2->nr_pending); 3732 } else 3733 clear_bit(R5_WriteError, &dev->flags); 3734 } 3735 if (test_bit(R5_MadeGood, &dev->flags)) { 3736 /* This flag does not apply to '.replacement' 3737 * only to .rdev, so make sure to check that*/ 3738 struct md_rdev *rdev2 = rcu_dereference( 3739 conf->disks[i].rdev); 3740 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3741 s->handle_bad_blocks = 1; 3742 atomic_inc(&rdev2->nr_pending); 3743 } else 3744 clear_bit(R5_MadeGood, &dev->flags); 3745 } 3746 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 3747 struct md_rdev *rdev2 = rcu_dereference( 3748 conf->disks[i].replacement); 3749 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3750 s->handle_bad_blocks = 1; 3751 atomic_inc(&rdev2->nr_pending); 3752 } else 3753 clear_bit(R5_MadeGoodRepl, &dev->flags); 3754 } 3755 if (!test_bit(R5_Insync, &dev->flags)) { 3756 /* The ReadError flag will just be confusing now */ 3757 clear_bit(R5_ReadError, &dev->flags); 3758 clear_bit(R5_ReWrite, &dev->flags); 3759 } 3760 if (test_bit(R5_ReadError, &dev->flags)) 3761 clear_bit(R5_Insync, &dev->flags); 3762 if (!test_bit(R5_Insync, &dev->flags)) { 3763 if (s->failed < 2) 3764 s->failed_num[s->failed] = i; 3765 s->failed++; 3766 if (rdev && !test_bit(Faulty, &rdev->flags)) 3767 do_recovery = 1; 3768 } 3769 } 3770 if (test_bit(STRIPE_SYNCING, &sh->state)) { 3771 /* If there is a failed device being replaced, 3772 * we must be recovering. 3773 * else if we are after recovery_cp, we must be syncing 3774 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 3775 * else we can only be replacing 3776 * sync and recovery both need to read all devices, and so 3777 * use the same flag. 3778 */ 3779 if (do_recovery || 3780 sh->sector >= conf->mddev->recovery_cp || 3781 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 3782 s->syncing = 1; 3783 else 3784 s->replacing = 1; 3785 } 3786 rcu_read_unlock(); 3787 } 3788 3789 static void handle_stripe(struct stripe_head *sh) 3790 { 3791 struct stripe_head_state s; 3792 struct r5conf *conf = sh->raid_conf; 3793 int i; 3794 int prexor; 3795 int disks = sh->disks; 3796 struct r5dev *pdev, *qdev; 3797 3798 clear_bit(STRIPE_HANDLE, &sh->state); 3799 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 3800 /* already being handled, ensure it gets handled 3801 * again when current action finishes */ 3802 set_bit(STRIPE_HANDLE, &sh->state); 3803 return; 3804 } 3805 3806 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3807 spin_lock(&sh->stripe_lock); 3808 /* Cannot process 'sync' concurrently with 'discard' */ 3809 if (!test_bit(STRIPE_DISCARD, &sh->state) && 3810 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3811 set_bit(STRIPE_SYNCING, &sh->state); 3812 clear_bit(STRIPE_INSYNC, &sh->state); 3813 clear_bit(STRIPE_REPLACED, &sh->state); 3814 } 3815 spin_unlock(&sh->stripe_lock); 3816 } 3817 clear_bit(STRIPE_DELAYED, &sh->state); 3818 3819 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3820 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3821 (unsigned long long)sh->sector, sh->state, 3822 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 3823 sh->check_state, sh->reconstruct_state); 3824 3825 analyse_stripe(sh, &s); 3826 3827 if (s.handle_bad_blocks) { 3828 set_bit(STRIPE_HANDLE, &sh->state); 3829 goto finish; 3830 } 3831 3832 if (unlikely(s.blocked_rdev)) { 3833 if (s.syncing || s.expanding || s.expanded || 3834 s.replacing || s.to_write || s.written) { 3835 set_bit(STRIPE_HANDLE, &sh->state); 3836 goto finish; 3837 } 3838 /* There is nothing for the blocked_rdev to block */ 3839 rdev_dec_pending(s.blocked_rdev, conf->mddev); 3840 s.blocked_rdev = NULL; 3841 } 3842 3843 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3844 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3845 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3846 } 3847 3848 pr_debug("locked=%d uptodate=%d to_read=%d" 3849 " to_write=%d failed=%d failed_num=%d,%d\n", 3850 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3851 s.failed_num[0], s.failed_num[1]); 3852 /* check if the array has lost more than max_degraded devices and, 3853 * if so, some requests might need to be failed. 3854 */ 3855 if (s.failed > conf->max_degraded) { 3856 sh->check_state = 0; 3857 sh->reconstruct_state = 0; 3858 if (s.to_read+s.to_write+s.written) 3859 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 3860 if (s.syncing + s.replacing) 3861 handle_failed_sync(conf, sh, &s); 3862 } 3863 3864 /* Now we check to see if any write operations have recently 3865 * completed 3866 */ 3867 prexor = 0; 3868 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3869 prexor = 1; 3870 if (sh->reconstruct_state == reconstruct_state_drain_result || 3871 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3872 sh->reconstruct_state = reconstruct_state_idle; 3873 3874 /* All the 'written' buffers and the parity block are ready to 3875 * be written back to disk 3876 */ 3877 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && 3878 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); 3879 BUG_ON(sh->qd_idx >= 0 && 3880 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && 3881 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); 3882 for (i = disks; i--; ) { 3883 struct r5dev *dev = &sh->dev[i]; 3884 if (test_bit(R5_LOCKED, &dev->flags) && 3885 (i == sh->pd_idx || i == sh->qd_idx || 3886 dev->written)) { 3887 pr_debug("Writing block %d\n", i); 3888 set_bit(R5_Wantwrite, &dev->flags); 3889 if (prexor) 3890 continue; 3891 if (s.failed > 1) 3892 continue; 3893 if (!test_bit(R5_Insync, &dev->flags) || 3894 ((i == sh->pd_idx || i == sh->qd_idx) && 3895 s.failed == 0)) 3896 set_bit(STRIPE_INSYNC, &sh->state); 3897 } 3898 } 3899 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3900 s.dec_preread_active = 1; 3901 } 3902 3903 /* 3904 * might be able to return some write requests if the parity blocks 3905 * are safe, or on a failed drive 3906 */ 3907 pdev = &sh->dev[sh->pd_idx]; 3908 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 3909 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 3910 qdev = &sh->dev[sh->qd_idx]; 3911 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 3912 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 3913 || conf->level < 6; 3914 3915 if (s.written && 3916 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3917 && !test_bit(R5_LOCKED, &pdev->flags) 3918 && (test_bit(R5_UPTODATE, &pdev->flags) || 3919 test_bit(R5_Discard, &pdev->flags))))) && 3920 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3921 && !test_bit(R5_LOCKED, &qdev->flags) 3922 && (test_bit(R5_UPTODATE, &qdev->flags) || 3923 test_bit(R5_Discard, &qdev->flags)))))) 3924 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 3925 3926 /* Now we might consider reading some blocks, either to check/generate 3927 * parity, or to satisfy requests 3928 * or to load a block that is being partially written. 3929 */ 3930 if (s.to_read || s.non_overwrite 3931 || (conf->level == 6 && s.to_write && s.failed) 3932 || (s.syncing && (s.uptodate + s.compute < disks)) 3933 || s.replacing 3934 || s.expanding) 3935 handle_stripe_fill(sh, &s, disks); 3936 3937 /* Now to consider new write requests and what else, if anything 3938 * should be read. We do not handle new writes when: 3939 * 1/ A 'write' operation (copy+xor) is already in flight. 3940 * 2/ A 'check' operation is in flight, as it may clobber the parity 3941 * block. 3942 */ 3943 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3944 handle_stripe_dirtying(conf, sh, &s, disks); 3945 3946 /* maybe we need to check and possibly fix the parity for this stripe 3947 * Any reads will already have been scheduled, so we just see if enough 3948 * data is available. The parity check is held off while parity 3949 * dependent operations are in flight. 3950 */ 3951 if (sh->check_state || 3952 (s.syncing && s.locked == 0 && 3953 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3954 !test_bit(STRIPE_INSYNC, &sh->state))) { 3955 if (conf->level == 6) 3956 handle_parity_checks6(conf, sh, &s, disks); 3957 else 3958 handle_parity_checks5(conf, sh, &s, disks); 3959 } 3960 3961 if ((s.replacing || s.syncing) && s.locked == 0 3962 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) 3963 && !test_bit(STRIPE_REPLACED, &sh->state)) { 3964 /* Write out to replacement devices where possible */ 3965 for (i = 0; i < conf->raid_disks; i++) 3966 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 3967 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); 3968 set_bit(R5_WantReplace, &sh->dev[i].flags); 3969 set_bit(R5_LOCKED, &sh->dev[i].flags); 3970 s.locked++; 3971 } 3972 if (s.replacing) 3973 set_bit(STRIPE_INSYNC, &sh->state); 3974 set_bit(STRIPE_REPLACED, &sh->state); 3975 } 3976 if ((s.syncing || s.replacing) && s.locked == 0 && 3977 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3978 test_bit(STRIPE_INSYNC, &sh->state)) { 3979 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3980 clear_bit(STRIPE_SYNCING, &sh->state); 3981 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 3982 wake_up(&conf->wait_for_overlap); 3983 } 3984 3985 /* If the failed drives are just a ReadError, then we might need 3986 * to progress the repair/check process 3987 */ 3988 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 3989 for (i = 0; i < s.failed; i++) { 3990 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 3991 if (test_bit(R5_ReadError, &dev->flags) 3992 && !test_bit(R5_LOCKED, &dev->flags) 3993 && test_bit(R5_UPTODATE, &dev->flags) 3994 ) { 3995 if (!test_bit(R5_ReWrite, &dev->flags)) { 3996 set_bit(R5_Wantwrite, &dev->flags); 3997 set_bit(R5_ReWrite, &dev->flags); 3998 set_bit(R5_LOCKED, &dev->flags); 3999 s.locked++; 4000 } else { 4001 /* let's read it back */ 4002 set_bit(R5_Wantread, &dev->flags); 4003 set_bit(R5_LOCKED, &dev->flags); 4004 s.locked++; 4005 } 4006 } 4007 } 4008 4009 /* Finish reconstruct operations initiated by the expansion process */ 4010 if (sh->reconstruct_state == reconstruct_state_result) { 4011 struct stripe_head *sh_src 4012 = get_active_stripe(conf, sh->sector, 1, 1, 1); 4013 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 4014 /* sh cannot be written until sh_src has been read. 4015 * so arrange for sh to be delayed a little 4016 */ 4017 set_bit(STRIPE_DELAYED, &sh->state); 4018 set_bit(STRIPE_HANDLE, &sh->state); 4019 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 4020 &sh_src->state)) 4021 atomic_inc(&conf->preread_active_stripes); 4022 release_stripe(sh_src); 4023 goto finish; 4024 } 4025 if (sh_src) 4026 release_stripe(sh_src); 4027 4028 sh->reconstruct_state = reconstruct_state_idle; 4029 clear_bit(STRIPE_EXPANDING, &sh->state); 4030 for (i = conf->raid_disks; i--; ) { 4031 set_bit(R5_Wantwrite, &sh->dev[i].flags); 4032 set_bit(R5_LOCKED, &sh->dev[i].flags); 4033 s.locked++; 4034 } 4035 } 4036 4037 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 4038 !sh->reconstruct_state) { 4039 /* Need to write out all blocks after computing parity */ 4040 sh->disks = conf->raid_disks; 4041 stripe_set_idx(sh->sector, conf, 0, sh); 4042 schedule_reconstruction(sh, &s, 1, 1); 4043 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 4044 clear_bit(STRIPE_EXPAND_READY, &sh->state); 4045 atomic_dec(&conf->reshape_stripes); 4046 wake_up(&conf->wait_for_overlap); 4047 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 4048 } 4049 4050 if (s.expanding && s.locked == 0 && 4051 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 4052 handle_stripe_expansion(conf, sh); 4053 4054 finish: 4055 /* wait for this device to become unblocked */ 4056 if (unlikely(s.blocked_rdev)) { 4057 if (conf->mddev->external) 4058 md_wait_for_blocked_rdev(s.blocked_rdev, 4059 conf->mddev); 4060 else 4061 /* Internal metadata will immediately 4062 * be written by raid5d, so we don't 4063 * need to wait here. 4064 */ 4065 rdev_dec_pending(s.blocked_rdev, 4066 conf->mddev); 4067 } 4068 4069 if (s.handle_bad_blocks) 4070 for (i = disks; i--; ) { 4071 struct md_rdev *rdev; 4072 struct r5dev *dev = &sh->dev[i]; 4073 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 4074 /* We own a safe reference to the rdev */ 4075 rdev = conf->disks[i].rdev; 4076 if (!rdev_set_badblocks(rdev, sh->sector, 4077 STRIPE_SECTORS, 0)) 4078 md_error(conf->mddev, rdev); 4079 rdev_dec_pending(rdev, conf->mddev); 4080 } 4081 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 4082 rdev = conf->disks[i].rdev; 4083 rdev_clear_badblocks(rdev, sh->sector, 4084 STRIPE_SECTORS, 0); 4085 rdev_dec_pending(rdev, conf->mddev); 4086 } 4087 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 4088 rdev = conf->disks[i].replacement; 4089 if (!rdev) 4090 /* rdev have been moved down */ 4091 rdev = conf->disks[i].rdev; 4092 rdev_clear_badblocks(rdev, sh->sector, 4093 STRIPE_SECTORS, 0); 4094 rdev_dec_pending(rdev, conf->mddev); 4095 } 4096 } 4097 4098 if (s.ops_request) 4099 raid_run_ops(sh, s.ops_request); 4100 4101 ops_run_io(sh, &s); 4102 4103 if (s.dec_preread_active) { 4104 /* We delay this until after ops_run_io so that if make_request 4105 * is waiting on a flush, it won't continue until the writes 4106 * have actually been submitted. 4107 */ 4108 atomic_dec(&conf->preread_active_stripes); 4109 if (atomic_read(&conf->preread_active_stripes) < 4110 IO_THRESHOLD) 4111 md_wakeup_thread(conf->mddev->thread); 4112 } 4113 4114 return_io(s.return_bi); 4115 4116 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4117 } 4118 4119 static void raid5_activate_delayed(struct r5conf *conf) 4120 { 4121 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 4122 while (!list_empty(&conf->delayed_list)) { 4123 struct list_head *l = conf->delayed_list.next; 4124 struct stripe_head *sh; 4125 sh = list_entry(l, struct stripe_head, lru); 4126 list_del_init(l); 4127 clear_bit(STRIPE_DELAYED, &sh->state); 4128 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4129 atomic_inc(&conf->preread_active_stripes); 4130 list_add_tail(&sh->lru, &conf->hold_list); 4131 raid5_wakeup_stripe_thread(sh); 4132 } 4133 } 4134 } 4135 4136 static void activate_bit_delay(struct r5conf *conf, 4137 struct list_head *temp_inactive_list) 4138 { 4139 /* device_lock is held */ 4140 struct list_head head; 4141 list_add(&head, &conf->bitmap_list); 4142 list_del_init(&conf->bitmap_list); 4143 while (!list_empty(&head)) { 4144 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 4145 int hash; 4146 list_del_init(&sh->lru); 4147 atomic_inc(&sh->count); 4148 hash = sh->hash_lock_index; 4149 __release_stripe(conf, sh, &temp_inactive_list[hash]); 4150 } 4151 } 4152 4153 static int raid5_congested(struct mddev *mddev, int bits) 4154 { 4155 struct r5conf *conf = mddev->private; 4156 4157 /* No difference between reads and writes. Just check 4158 * how busy the stripe_cache is 4159 */ 4160 4161 if (conf->inactive_blocked) 4162 return 1; 4163 if (conf->quiesce) 4164 return 1; 4165 if (atomic_read(&conf->empty_inactive_list_nr)) 4166 return 1; 4167 4168 return 0; 4169 } 4170 4171 /* We want read requests to align with chunks where possible, 4172 * but write requests don't need to. 4173 */ 4174 static int raid5_mergeable_bvec(struct mddev *mddev, 4175 struct bvec_merge_data *bvm, 4176 struct bio_vec *biovec) 4177 { 4178 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 4179 int max; 4180 unsigned int chunk_sectors = mddev->chunk_sectors; 4181 unsigned int bio_sectors = bvm->bi_size >> 9; 4182 4183 if ((bvm->bi_rw & 1) == WRITE) 4184 return biovec->bv_len; /* always allow writes to be mergeable */ 4185 4186 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4187 chunk_sectors = mddev->new_chunk_sectors; 4188 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 4189 if (max < 0) max = 0; 4190 if (max <= biovec->bv_len && bio_sectors == 0) 4191 return biovec->bv_len; 4192 else 4193 return max; 4194 } 4195 4196 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 4197 { 4198 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); 4199 unsigned int chunk_sectors = mddev->chunk_sectors; 4200 unsigned int bio_sectors = bio_sectors(bio); 4201 4202 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4203 chunk_sectors = mddev->new_chunk_sectors; 4204 return chunk_sectors >= 4205 ((sector & (chunk_sectors - 1)) + bio_sectors); 4206 } 4207 4208 /* 4209 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 4210 * later sampled by raid5d. 4211 */ 4212 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 4213 { 4214 unsigned long flags; 4215 4216 spin_lock_irqsave(&conf->device_lock, flags); 4217 4218 bi->bi_next = conf->retry_read_aligned_list; 4219 conf->retry_read_aligned_list = bi; 4220 4221 spin_unlock_irqrestore(&conf->device_lock, flags); 4222 md_wakeup_thread(conf->mddev->thread); 4223 } 4224 4225 static struct bio *remove_bio_from_retry(struct r5conf *conf) 4226 { 4227 struct bio *bi; 4228 4229 bi = conf->retry_read_aligned; 4230 if (bi) { 4231 conf->retry_read_aligned = NULL; 4232 return bi; 4233 } 4234 bi = conf->retry_read_aligned_list; 4235 if(bi) { 4236 conf->retry_read_aligned_list = bi->bi_next; 4237 bi->bi_next = NULL; 4238 /* 4239 * this sets the active strip count to 1 and the processed 4240 * strip count to zero (upper 8 bits) 4241 */ 4242 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ 4243 } 4244 4245 return bi; 4246 } 4247 4248 /* 4249 * The "raid5_align_endio" should check if the read succeeded and if it 4250 * did, call bio_endio on the original bio (having bio_put the new bio 4251 * first). 4252 * If the read failed.. 4253 */ 4254 static void raid5_align_endio(struct bio *bi, int error) 4255 { 4256 struct bio* raid_bi = bi->bi_private; 4257 struct mddev *mddev; 4258 struct r5conf *conf; 4259 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 4260 struct md_rdev *rdev; 4261 4262 bio_put(bi); 4263 4264 rdev = (void*)raid_bi->bi_next; 4265 raid_bi->bi_next = NULL; 4266 mddev = rdev->mddev; 4267 conf = mddev->private; 4268 4269 rdev_dec_pending(rdev, conf->mddev); 4270 4271 if (!error && uptodate) { 4272 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4273 raid_bi, 0); 4274 bio_endio(raid_bi, 0); 4275 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4276 wake_up(&conf->wait_for_stripe); 4277 return; 4278 } 4279 4280 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 4281 4282 add_bio_to_retry(raid_bi, conf); 4283 } 4284 4285 static int bio_fits_rdev(struct bio *bi) 4286 { 4287 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 4288 4289 if (bio_sectors(bi) > queue_max_sectors(q)) 4290 return 0; 4291 blk_recount_segments(q, bi); 4292 if (bi->bi_phys_segments > queue_max_segments(q)) 4293 return 0; 4294 4295 if (q->merge_bvec_fn) 4296 /* it's too hard to apply the merge_bvec_fn at this stage, 4297 * just just give up 4298 */ 4299 return 0; 4300 4301 return 1; 4302 } 4303 4304 static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) 4305 { 4306 struct r5conf *conf = mddev->private; 4307 int dd_idx; 4308 struct bio* align_bi; 4309 struct md_rdev *rdev; 4310 sector_t end_sector; 4311 4312 if (!in_chunk_boundary(mddev, raid_bio)) { 4313 pr_debug("chunk_aligned_read : non aligned\n"); 4314 return 0; 4315 } 4316 /* 4317 * use bio_clone_mddev to make a copy of the bio 4318 */ 4319 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 4320 if (!align_bi) 4321 return 0; 4322 /* 4323 * set bi_end_io to a new function, and set bi_private to the 4324 * original bio. 4325 */ 4326 align_bi->bi_end_io = raid5_align_endio; 4327 align_bi->bi_private = raid_bio; 4328 /* 4329 * compute position 4330 */ 4331 align_bi->bi_iter.bi_sector = 4332 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 4333 0, &dd_idx, NULL); 4334 4335 end_sector = bio_end_sector(align_bi); 4336 rcu_read_lock(); 4337 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 4338 if (!rdev || test_bit(Faulty, &rdev->flags) || 4339 rdev->recovery_offset < end_sector) { 4340 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 4341 if (rdev && 4342 (test_bit(Faulty, &rdev->flags) || 4343 !(test_bit(In_sync, &rdev->flags) || 4344 rdev->recovery_offset >= end_sector))) 4345 rdev = NULL; 4346 } 4347 if (rdev) { 4348 sector_t first_bad; 4349 int bad_sectors; 4350 4351 atomic_inc(&rdev->nr_pending); 4352 rcu_read_unlock(); 4353 raid_bio->bi_next = (void*)rdev; 4354 align_bi->bi_bdev = rdev->bdev; 4355 __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); 4356 4357 if (!bio_fits_rdev(align_bi) || 4358 is_badblock(rdev, align_bi->bi_iter.bi_sector, 4359 bio_sectors(align_bi), 4360 &first_bad, &bad_sectors)) { 4361 /* too big in some way, or has a known bad block */ 4362 bio_put(align_bi); 4363 rdev_dec_pending(rdev, mddev); 4364 return 0; 4365 } 4366 4367 /* No reshape active, so we can trust rdev->data_offset */ 4368 align_bi->bi_iter.bi_sector += rdev->data_offset; 4369 4370 spin_lock_irq(&conf->device_lock); 4371 wait_event_lock_irq(conf->wait_for_stripe, 4372 conf->quiesce == 0, 4373 conf->device_lock); 4374 atomic_inc(&conf->active_aligned_reads); 4375 spin_unlock_irq(&conf->device_lock); 4376 4377 if (mddev->gendisk) 4378 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), 4379 align_bi, disk_devt(mddev->gendisk), 4380 raid_bio->bi_iter.bi_sector); 4381 generic_make_request(align_bi); 4382 return 1; 4383 } else { 4384 rcu_read_unlock(); 4385 bio_put(align_bi); 4386 return 0; 4387 } 4388 } 4389 4390 /* __get_priority_stripe - get the next stripe to process 4391 * 4392 * Full stripe writes are allowed to pass preread active stripes up until 4393 * the bypass_threshold is exceeded. In general the bypass_count 4394 * increments when the handle_list is handled before the hold_list; however, it 4395 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 4396 * stripe with in flight i/o. The bypass_count will be reset when the 4397 * head of the hold_list has changed, i.e. the head was promoted to the 4398 * handle_list. 4399 */ 4400 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) 4401 { 4402 struct stripe_head *sh = NULL, *tmp; 4403 struct list_head *handle_list = NULL; 4404 struct r5worker_group *wg = NULL; 4405 4406 if (conf->worker_cnt_per_group == 0) { 4407 handle_list = &conf->handle_list; 4408 } else if (group != ANY_GROUP) { 4409 handle_list = &conf->worker_groups[group].handle_list; 4410 wg = &conf->worker_groups[group]; 4411 } else { 4412 int i; 4413 for (i = 0; i < conf->group_cnt; i++) { 4414 handle_list = &conf->worker_groups[i].handle_list; 4415 wg = &conf->worker_groups[i]; 4416 if (!list_empty(handle_list)) 4417 break; 4418 } 4419 } 4420 4421 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 4422 __func__, 4423 list_empty(handle_list) ? "empty" : "busy", 4424 list_empty(&conf->hold_list) ? "empty" : "busy", 4425 atomic_read(&conf->pending_full_writes), conf->bypass_count); 4426 4427 if (!list_empty(handle_list)) { 4428 sh = list_entry(handle_list->next, typeof(*sh), lru); 4429 4430 if (list_empty(&conf->hold_list)) 4431 conf->bypass_count = 0; 4432 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 4433 if (conf->hold_list.next == conf->last_hold) 4434 conf->bypass_count++; 4435 else { 4436 conf->last_hold = conf->hold_list.next; 4437 conf->bypass_count -= conf->bypass_threshold; 4438 if (conf->bypass_count < 0) 4439 conf->bypass_count = 0; 4440 } 4441 } 4442 } else if (!list_empty(&conf->hold_list) && 4443 ((conf->bypass_threshold && 4444 conf->bypass_count > conf->bypass_threshold) || 4445 atomic_read(&conf->pending_full_writes) == 0)) { 4446 4447 list_for_each_entry(tmp, &conf->hold_list, lru) { 4448 if (conf->worker_cnt_per_group == 0 || 4449 group == ANY_GROUP || 4450 !cpu_online(tmp->cpu) || 4451 cpu_to_group(tmp->cpu) == group) { 4452 sh = tmp; 4453 break; 4454 } 4455 } 4456 4457 if (sh) { 4458 conf->bypass_count -= conf->bypass_threshold; 4459 if (conf->bypass_count < 0) 4460 conf->bypass_count = 0; 4461 } 4462 wg = NULL; 4463 } 4464 4465 if (!sh) 4466 return NULL; 4467 4468 if (wg) { 4469 wg->stripes_cnt--; 4470 sh->group = NULL; 4471 } 4472 list_del_init(&sh->lru); 4473 BUG_ON(atomic_inc_return(&sh->count) != 1); 4474 return sh; 4475 } 4476 4477 struct raid5_plug_cb { 4478 struct blk_plug_cb cb; 4479 struct list_head list; 4480 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; 4481 }; 4482 4483 static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) 4484 { 4485 struct raid5_plug_cb *cb = container_of( 4486 blk_cb, struct raid5_plug_cb, cb); 4487 struct stripe_head *sh; 4488 struct mddev *mddev = cb->cb.data; 4489 struct r5conf *conf = mddev->private; 4490 int cnt = 0; 4491 int hash; 4492 4493 if (cb->list.next && !list_empty(&cb->list)) { 4494 spin_lock_irq(&conf->device_lock); 4495 while (!list_empty(&cb->list)) { 4496 sh = list_first_entry(&cb->list, struct stripe_head, lru); 4497 list_del_init(&sh->lru); 4498 /* 4499 * avoid race release_stripe_plug() sees 4500 * STRIPE_ON_UNPLUG_LIST clear but the stripe 4501 * is still in our list 4502 */ 4503 smp_mb__before_atomic(); 4504 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); 4505 /* 4506 * STRIPE_ON_RELEASE_LIST could be set here. In that 4507 * case, the count is always > 1 here 4508 */ 4509 hash = sh->hash_lock_index; 4510 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); 4511 cnt++; 4512 } 4513 spin_unlock_irq(&conf->device_lock); 4514 } 4515 release_inactive_stripe_list(conf, cb->temp_inactive_list, 4516 NR_STRIPE_HASH_LOCKS); 4517 if (mddev->queue) 4518 trace_block_unplug(mddev->queue, cnt, !from_schedule); 4519 kfree(cb); 4520 } 4521 4522 static void release_stripe_plug(struct mddev *mddev, 4523 struct stripe_head *sh) 4524 { 4525 struct blk_plug_cb *blk_cb = blk_check_plugged( 4526 raid5_unplug, mddev, 4527 sizeof(struct raid5_plug_cb)); 4528 struct raid5_plug_cb *cb; 4529 4530 if (!blk_cb) { 4531 release_stripe(sh); 4532 return; 4533 } 4534 4535 cb = container_of(blk_cb, struct raid5_plug_cb, cb); 4536 4537 if (cb->list.next == NULL) { 4538 int i; 4539 INIT_LIST_HEAD(&cb->list); 4540 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 4541 INIT_LIST_HEAD(cb->temp_inactive_list + i); 4542 } 4543 4544 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) 4545 list_add_tail(&sh->lru, &cb->list); 4546 else 4547 release_stripe(sh); 4548 } 4549 4550 static void make_discard_request(struct mddev *mddev, struct bio *bi) 4551 { 4552 struct r5conf *conf = mddev->private; 4553 sector_t logical_sector, last_sector; 4554 struct stripe_head *sh; 4555 int remaining; 4556 int stripe_sectors; 4557 4558 if (mddev->reshape_position != MaxSector) 4559 /* Skip discard while reshape is happening */ 4560 return; 4561 4562 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4563 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); 4564 4565 bi->bi_next = NULL; 4566 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4567 4568 stripe_sectors = conf->chunk_sectors * 4569 (conf->raid_disks - conf->max_degraded); 4570 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, 4571 stripe_sectors); 4572 sector_div(last_sector, stripe_sectors); 4573 4574 logical_sector *= conf->chunk_sectors; 4575 last_sector *= conf->chunk_sectors; 4576 4577 for (; logical_sector < last_sector; 4578 logical_sector += STRIPE_SECTORS) { 4579 DEFINE_WAIT(w); 4580 int d; 4581 again: 4582 sh = get_active_stripe(conf, logical_sector, 0, 0, 0); 4583 prepare_to_wait(&conf->wait_for_overlap, &w, 4584 TASK_UNINTERRUPTIBLE); 4585 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4586 if (test_bit(STRIPE_SYNCING, &sh->state)) { 4587 release_stripe(sh); 4588 schedule(); 4589 goto again; 4590 } 4591 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4592 spin_lock_irq(&sh->stripe_lock); 4593 for (d = 0; d < conf->raid_disks; d++) { 4594 if (d == sh->pd_idx || d == sh->qd_idx) 4595 continue; 4596 if (sh->dev[d].towrite || sh->dev[d].toread) { 4597 set_bit(R5_Overlap, &sh->dev[d].flags); 4598 spin_unlock_irq(&sh->stripe_lock); 4599 release_stripe(sh); 4600 schedule(); 4601 goto again; 4602 } 4603 } 4604 set_bit(STRIPE_DISCARD, &sh->state); 4605 finish_wait(&conf->wait_for_overlap, &w); 4606 for (d = 0; d < conf->raid_disks; d++) { 4607 if (d == sh->pd_idx || d == sh->qd_idx) 4608 continue; 4609 sh->dev[d].towrite = bi; 4610 set_bit(R5_OVERWRITE, &sh->dev[d].flags); 4611 raid5_inc_bi_active_stripes(bi); 4612 } 4613 spin_unlock_irq(&sh->stripe_lock); 4614 if (conf->mddev->bitmap) { 4615 for (d = 0; 4616 d < conf->raid_disks - conf->max_degraded; 4617 d++) 4618 bitmap_startwrite(mddev->bitmap, 4619 sh->sector, 4620 STRIPE_SECTORS, 4621 0); 4622 sh->bm_seq = conf->seq_flush + 1; 4623 set_bit(STRIPE_BIT_DELAY, &sh->state); 4624 } 4625 4626 set_bit(STRIPE_HANDLE, &sh->state); 4627 clear_bit(STRIPE_DELAYED, &sh->state); 4628 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4629 atomic_inc(&conf->preread_active_stripes); 4630 release_stripe_plug(mddev, sh); 4631 } 4632 4633 remaining = raid5_dec_bi_active_stripes(bi); 4634 if (remaining == 0) { 4635 md_write_end(mddev); 4636 bio_endio(bi, 0); 4637 } 4638 } 4639 4640 static void make_request(struct mddev *mddev, struct bio * bi) 4641 { 4642 struct r5conf *conf = mddev->private; 4643 int dd_idx; 4644 sector_t new_sector; 4645 sector_t logical_sector, last_sector; 4646 struct stripe_head *sh; 4647 const int rw = bio_data_dir(bi); 4648 int remaining; 4649 DEFINE_WAIT(w); 4650 bool do_prepare; 4651 4652 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 4653 md_flush_request(mddev, bi); 4654 return; 4655 } 4656 4657 md_write_start(mddev, bi); 4658 4659 if (rw == READ && 4660 mddev->reshape_position == MaxSector && 4661 chunk_aligned_read(mddev,bi)) 4662 return; 4663 4664 if (unlikely(bi->bi_rw & REQ_DISCARD)) { 4665 make_discard_request(mddev, bi); 4666 return; 4667 } 4668 4669 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4670 last_sector = bio_end_sector(bi); 4671 bi->bi_next = NULL; 4672 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4673 4674 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 4675 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 4676 int previous; 4677 int seq; 4678 4679 do_prepare = false; 4680 retry: 4681 seq = read_seqcount_begin(&conf->gen_lock); 4682 previous = 0; 4683 if (do_prepare) 4684 prepare_to_wait(&conf->wait_for_overlap, &w, 4685 TASK_UNINTERRUPTIBLE); 4686 if (unlikely(conf->reshape_progress != MaxSector)) { 4687 /* spinlock is needed as reshape_progress may be 4688 * 64bit on a 32bit platform, and so it might be 4689 * possible to see a half-updated value 4690 * Of course reshape_progress could change after 4691 * the lock is dropped, so once we get a reference 4692 * to the stripe that we think it is, we will have 4693 * to check again. 4694 */ 4695 spin_lock_irq(&conf->device_lock); 4696 if (mddev->reshape_backwards 4697 ? logical_sector < conf->reshape_progress 4698 : logical_sector >= conf->reshape_progress) { 4699 previous = 1; 4700 } else { 4701 if (mddev->reshape_backwards 4702 ? logical_sector < conf->reshape_safe 4703 : logical_sector >= conf->reshape_safe) { 4704 spin_unlock_irq(&conf->device_lock); 4705 schedule(); 4706 do_prepare = true; 4707 goto retry; 4708 } 4709 } 4710 spin_unlock_irq(&conf->device_lock); 4711 } 4712 4713 new_sector = raid5_compute_sector(conf, logical_sector, 4714 previous, 4715 &dd_idx, NULL); 4716 pr_debug("raid456: make_request, sector %llu logical %llu\n", 4717 (unsigned long long)new_sector, 4718 (unsigned long long)logical_sector); 4719 4720 sh = get_active_stripe(conf, new_sector, previous, 4721 (bi->bi_rw&RWA_MASK), 0); 4722 if (sh) { 4723 if (unlikely(previous)) { 4724 /* expansion might have moved on while waiting for a 4725 * stripe, so we must do the range check again. 4726 * Expansion could still move past after this 4727 * test, but as we are holding a reference to 4728 * 'sh', we know that if that happens, 4729 * STRIPE_EXPANDING will get set and the expansion 4730 * won't proceed until we finish with the stripe. 4731 */ 4732 int must_retry = 0; 4733 spin_lock_irq(&conf->device_lock); 4734 if (mddev->reshape_backwards 4735 ? logical_sector >= conf->reshape_progress 4736 : logical_sector < conf->reshape_progress) 4737 /* mismatch, need to try again */ 4738 must_retry = 1; 4739 spin_unlock_irq(&conf->device_lock); 4740 if (must_retry) { 4741 release_stripe(sh); 4742 schedule(); 4743 do_prepare = true; 4744 goto retry; 4745 } 4746 } 4747 if (read_seqcount_retry(&conf->gen_lock, seq)) { 4748 /* Might have got the wrong stripe_head 4749 * by accident 4750 */ 4751 release_stripe(sh); 4752 goto retry; 4753 } 4754 4755 if (rw == WRITE && 4756 logical_sector >= mddev->suspend_lo && 4757 logical_sector < mddev->suspend_hi) { 4758 release_stripe(sh); 4759 /* As the suspend_* range is controlled by 4760 * userspace, we want an interruptible 4761 * wait. 4762 */ 4763 flush_signals(current); 4764 prepare_to_wait(&conf->wait_for_overlap, 4765 &w, TASK_INTERRUPTIBLE); 4766 if (logical_sector >= mddev->suspend_lo && 4767 logical_sector < mddev->suspend_hi) { 4768 schedule(); 4769 do_prepare = true; 4770 } 4771 goto retry; 4772 } 4773 4774 if (test_bit(STRIPE_EXPANDING, &sh->state) || 4775 !add_stripe_bio(sh, bi, dd_idx, rw)) { 4776 /* Stripe is busy expanding or 4777 * add failed due to overlap. Flush everything 4778 * and wait a while 4779 */ 4780 md_wakeup_thread(mddev->thread); 4781 release_stripe(sh); 4782 schedule(); 4783 do_prepare = true; 4784 goto retry; 4785 } 4786 set_bit(STRIPE_HANDLE, &sh->state); 4787 clear_bit(STRIPE_DELAYED, &sh->state); 4788 if ((bi->bi_rw & REQ_SYNC) && 4789 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4790 atomic_inc(&conf->preread_active_stripes); 4791 release_stripe_plug(mddev, sh); 4792 } else { 4793 /* cannot get stripe for read-ahead, just give-up */ 4794 clear_bit(BIO_UPTODATE, &bi->bi_flags); 4795 break; 4796 } 4797 } 4798 finish_wait(&conf->wait_for_overlap, &w); 4799 4800 remaining = raid5_dec_bi_active_stripes(bi); 4801 if (remaining == 0) { 4802 4803 if ( rw == WRITE ) 4804 md_write_end(mddev); 4805 4806 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 4807 bi, 0); 4808 bio_endio(bi, 0); 4809 } 4810 } 4811 4812 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 4813 4814 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 4815 { 4816 /* reshaping is quite different to recovery/resync so it is 4817 * handled quite separately ... here. 4818 * 4819 * On each call to sync_request, we gather one chunk worth of 4820 * destination stripes and flag them as expanding. 4821 * Then we find all the source stripes and request reads. 4822 * As the reads complete, handle_stripe will copy the data 4823 * into the destination stripe and release that stripe. 4824 */ 4825 struct r5conf *conf = mddev->private; 4826 struct stripe_head *sh; 4827 sector_t first_sector, last_sector; 4828 int raid_disks = conf->previous_raid_disks; 4829 int data_disks = raid_disks - conf->max_degraded; 4830 int new_data_disks = conf->raid_disks - conf->max_degraded; 4831 int i; 4832 int dd_idx; 4833 sector_t writepos, readpos, safepos; 4834 sector_t stripe_addr; 4835 int reshape_sectors; 4836 struct list_head stripes; 4837 4838 if (sector_nr == 0) { 4839 /* If restarting in the middle, skip the initial sectors */ 4840 if (mddev->reshape_backwards && 4841 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4842 sector_nr = raid5_size(mddev, 0, 0) 4843 - conf->reshape_progress; 4844 } else if (!mddev->reshape_backwards && 4845 conf->reshape_progress > 0) 4846 sector_nr = conf->reshape_progress; 4847 sector_div(sector_nr, new_data_disks); 4848 if (sector_nr) { 4849 mddev->curr_resync_completed = sector_nr; 4850 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4851 *skipped = 1; 4852 return sector_nr; 4853 } 4854 } 4855 4856 /* We need to process a full chunk at a time. 4857 * If old and new chunk sizes differ, we need to process the 4858 * largest of these 4859 */ 4860 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4861 reshape_sectors = mddev->new_chunk_sectors; 4862 else 4863 reshape_sectors = mddev->chunk_sectors; 4864 4865 /* We update the metadata at least every 10 seconds, or when 4866 * the data about to be copied would over-write the source of 4867 * the data at the front of the range. i.e. one new_stripe 4868 * along from reshape_progress new_maps to after where 4869 * reshape_safe old_maps to 4870 */ 4871 writepos = conf->reshape_progress; 4872 sector_div(writepos, new_data_disks); 4873 readpos = conf->reshape_progress; 4874 sector_div(readpos, data_disks); 4875 safepos = conf->reshape_safe; 4876 sector_div(safepos, data_disks); 4877 if (mddev->reshape_backwards) { 4878 writepos -= min_t(sector_t, reshape_sectors, writepos); 4879 readpos += reshape_sectors; 4880 safepos += reshape_sectors; 4881 } else { 4882 writepos += reshape_sectors; 4883 readpos -= min_t(sector_t, reshape_sectors, readpos); 4884 safepos -= min_t(sector_t, reshape_sectors, safepos); 4885 } 4886 4887 /* Having calculated the 'writepos' possibly use it 4888 * to set 'stripe_addr' which is where we will write to. 4889 */ 4890 if (mddev->reshape_backwards) { 4891 BUG_ON(conf->reshape_progress == 0); 4892 stripe_addr = writepos; 4893 BUG_ON((mddev->dev_sectors & 4894 ~((sector_t)reshape_sectors - 1)) 4895 - reshape_sectors - stripe_addr 4896 != sector_nr); 4897 } else { 4898 BUG_ON(writepos != sector_nr + reshape_sectors); 4899 stripe_addr = sector_nr; 4900 } 4901 4902 /* 'writepos' is the most advanced device address we might write. 4903 * 'readpos' is the least advanced device address we might read. 4904 * 'safepos' is the least address recorded in the metadata as having 4905 * been reshaped. 4906 * If there is a min_offset_diff, these are adjusted either by 4907 * increasing the safepos/readpos if diff is negative, or 4908 * increasing writepos if diff is positive. 4909 * If 'readpos' is then behind 'writepos', there is no way that we can 4910 * ensure safety in the face of a crash - that must be done by userspace 4911 * making a backup of the data. So in that case there is no particular 4912 * rush to update metadata. 4913 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4914 * update the metadata to advance 'safepos' to match 'readpos' so that 4915 * we can be safe in the event of a crash. 4916 * So we insist on updating metadata if safepos is behind writepos and 4917 * readpos is beyond writepos. 4918 * In any case, update the metadata every 10 seconds. 4919 * Maybe that number should be configurable, but I'm not sure it is 4920 * worth it.... maybe it could be a multiple of safemode_delay??? 4921 */ 4922 if (conf->min_offset_diff < 0) { 4923 safepos += -conf->min_offset_diff; 4924 readpos += -conf->min_offset_diff; 4925 } else 4926 writepos += conf->min_offset_diff; 4927 4928 if ((mddev->reshape_backwards 4929 ? (safepos > writepos && readpos < writepos) 4930 : (safepos < writepos && readpos > writepos)) || 4931 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4932 /* Cannot proceed until we've updated the superblock... */ 4933 wait_event(conf->wait_for_overlap, 4934 atomic_read(&conf->reshape_stripes)==0 4935 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4936 if (atomic_read(&conf->reshape_stripes) != 0) 4937 return 0; 4938 mddev->reshape_position = conf->reshape_progress; 4939 mddev->curr_resync_completed = sector_nr; 4940 conf->reshape_checkpoint = jiffies; 4941 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4942 md_wakeup_thread(mddev->thread); 4943 wait_event(mddev->sb_wait, mddev->flags == 0 || 4944 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4945 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4946 return 0; 4947 spin_lock_irq(&conf->device_lock); 4948 conf->reshape_safe = mddev->reshape_position; 4949 spin_unlock_irq(&conf->device_lock); 4950 wake_up(&conf->wait_for_overlap); 4951 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4952 } 4953 4954 INIT_LIST_HEAD(&stripes); 4955 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4956 int j; 4957 int skipped_disk = 0; 4958 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4959 set_bit(STRIPE_EXPANDING, &sh->state); 4960 atomic_inc(&conf->reshape_stripes); 4961 /* If any of this stripe is beyond the end of the old 4962 * array, then we need to zero those blocks 4963 */ 4964 for (j=sh->disks; j--;) { 4965 sector_t s; 4966 if (j == sh->pd_idx) 4967 continue; 4968 if (conf->level == 6 && 4969 j == sh->qd_idx) 4970 continue; 4971 s = compute_blocknr(sh, j, 0); 4972 if (s < raid5_size(mddev, 0, 0)) { 4973 skipped_disk = 1; 4974 continue; 4975 } 4976 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4977 set_bit(R5_Expanded, &sh->dev[j].flags); 4978 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4979 } 4980 if (!skipped_disk) { 4981 set_bit(STRIPE_EXPAND_READY, &sh->state); 4982 set_bit(STRIPE_HANDLE, &sh->state); 4983 } 4984 list_add(&sh->lru, &stripes); 4985 } 4986 spin_lock_irq(&conf->device_lock); 4987 if (mddev->reshape_backwards) 4988 conf->reshape_progress -= reshape_sectors * new_data_disks; 4989 else 4990 conf->reshape_progress += reshape_sectors * new_data_disks; 4991 spin_unlock_irq(&conf->device_lock); 4992 /* Ok, those stripe are ready. We can start scheduling 4993 * reads on the source stripes. 4994 * The source stripes are determined by mapping the first and last 4995 * block on the destination stripes. 4996 */ 4997 first_sector = 4998 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4999 1, &dd_idx, NULL); 5000 last_sector = 5001 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 5002 * new_data_disks - 1), 5003 1, &dd_idx, NULL); 5004 if (last_sector >= mddev->dev_sectors) 5005 last_sector = mddev->dev_sectors - 1; 5006 while (first_sector <= last_sector) { 5007 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 5008 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 5009 set_bit(STRIPE_HANDLE, &sh->state); 5010 release_stripe(sh); 5011 first_sector += STRIPE_SECTORS; 5012 } 5013 /* Now that the sources are clearly marked, we can release 5014 * the destination stripes 5015 */ 5016 while (!list_empty(&stripes)) { 5017 sh = list_entry(stripes.next, struct stripe_head, lru); 5018 list_del_init(&sh->lru); 5019 release_stripe(sh); 5020 } 5021 /* If this takes us to the resync_max point where we have to pause, 5022 * then we need to write out the superblock. 5023 */ 5024 sector_nr += reshape_sectors; 5025 if ((sector_nr - mddev->curr_resync_completed) * 2 5026 >= mddev->resync_max - mddev->curr_resync_completed) { 5027 /* Cannot proceed until we've updated the superblock... */ 5028 wait_event(conf->wait_for_overlap, 5029 atomic_read(&conf->reshape_stripes) == 0 5030 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5031 if (atomic_read(&conf->reshape_stripes) != 0) 5032 goto ret; 5033 mddev->reshape_position = conf->reshape_progress; 5034 mddev->curr_resync_completed = sector_nr; 5035 conf->reshape_checkpoint = jiffies; 5036 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5037 md_wakeup_thread(mddev->thread); 5038 wait_event(mddev->sb_wait, 5039 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 5040 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5041 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5042 goto ret; 5043 spin_lock_irq(&conf->device_lock); 5044 conf->reshape_safe = mddev->reshape_position; 5045 spin_unlock_irq(&conf->device_lock); 5046 wake_up(&conf->wait_for_overlap); 5047 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5048 } 5049 ret: 5050 return reshape_sectors; 5051 } 5052 5053 /* FIXME go_faster isn't used */ 5054 static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster) 5055 { 5056 struct r5conf *conf = mddev->private; 5057 struct stripe_head *sh; 5058 sector_t max_sector = mddev->dev_sectors; 5059 sector_t sync_blocks; 5060 int still_degraded = 0; 5061 int i; 5062 5063 if (sector_nr >= max_sector) { 5064 /* just being told to finish up .. nothing much to do */ 5065 5066 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 5067 end_reshape(conf); 5068 return 0; 5069 } 5070 5071 if (mddev->curr_resync < max_sector) /* aborted */ 5072 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 5073 &sync_blocks, 1); 5074 else /* completed sync */ 5075 conf->fullsync = 0; 5076 bitmap_close_sync(mddev->bitmap); 5077 5078 return 0; 5079 } 5080 5081 /* Allow raid5_quiesce to complete */ 5082 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 5083 5084 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 5085 return reshape_request(mddev, sector_nr, skipped); 5086 5087 /* No need to check resync_max as we never do more than one 5088 * stripe, and as resync_max will always be on a chunk boundary, 5089 * if the check in md_do_sync didn't fire, there is no chance 5090 * of overstepping resync_max here 5091 */ 5092 5093 /* if there is too many failed drives and we are trying 5094 * to resync, then assert that we are finished, because there is 5095 * nothing we can do. 5096 */ 5097 if (mddev->degraded >= conf->max_degraded && 5098 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 5099 sector_t rv = mddev->dev_sectors - sector_nr; 5100 *skipped = 1; 5101 return rv; 5102 } 5103 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 5104 !conf->fullsync && 5105 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 5106 sync_blocks >= STRIPE_SECTORS) { 5107 /* we can skip this block, and probably more */ 5108 sync_blocks /= STRIPE_SECTORS; 5109 *skipped = 1; 5110 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 5111 } 5112 5113 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 5114 5115 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 5116 if (sh == NULL) { 5117 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 5118 /* make sure we don't swamp the stripe cache if someone else 5119 * is trying to get access 5120 */ 5121 schedule_timeout_uninterruptible(1); 5122 } 5123 /* Need to check if array will still be degraded after recovery/resync 5124 * Note in case of > 1 drive failures it's possible we're rebuilding 5125 * one drive while leaving another faulty drive in array. 5126 */ 5127 rcu_read_lock(); 5128 for (i = 0; i < conf->raid_disks; i++) { 5129 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); 5130 5131 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 5132 still_degraded = 1; 5133 } 5134 rcu_read_unlock(); 5135 5136 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 5137 5138 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 5139 set_bit(STRIPE_HANDLE, &sh->state); 5140 5141 release_stripe(sh); 5142 5143 return STRIPE_SECTORS; 5144 } 5145 5146 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 5147 { 5148 /* We may not be able to submit a whole bio at once as there 5149 * may not be enough stripe_heads available. 5150 * We cannot pre-allocate enough stripe_heads as we may need 5151 * more than exist in the cache (if we allow ever large chunks). 5152 * So we do one stripe head at a time and record in 5153 * ->bi_hw_segments how many have been done. 5154 * 5155 * We *know* that this entire raid_bio is in one chunk, so 5156 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 5157 */ 5158 struct stripe_head *sh; 5159 int dd_idx; 5160 sector_t sector, logical_sector, last_sector; 5161 int scnt = 0; 5162 int remaining; 5163 int handled = 0; 5164 5165 logical_sector = raid_bio->bi_iter.bi_sector & 5166 ~((sector_t)STRIPE_SECTORS-1); 5167 sector = raid5_compute_sector(conf, logical_sector, 5168 0, &dd_idx, NULL); 5169 last_sector = bio_end_sector(raid_bio); 5170 5171 for (; logical_sector < last_sector; 5172 logical_sector += STRIPE_SECTORS, 5173 sector += STRIPE_SECTORS, 5174 scnt++) { 5175 5176 if (scnt < raid5_bi_processed_stripes(raid_bio)) 5177 /* already done this stripe */ 5178 continue; 5179 5180 sh = get_active_stripe(conf, sector, 0, 1, 1); 5181 5182 if (!sh) { 5183 /* failed to get a stripe - must wait */ 5184 raid5_set_bi_processed_stripes(raid_bio, scnt); 5185 conf->retry_read_aligned = raid_bio; 5186 return handled; 5187 } 5188 5189 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 5190 release_stripe(sh); 5191 raid5_set_bi_processed_stripes(raid_bio, scnt); 5192 conf->retry_read_aligned = raid_bio; 5193 return handled; 5194 } 5195 5196 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); 5197 handle_stripe(sh); 5198 release_stripe(sh); 5199 handled++; 5200 } 5201 remaining = raid5_dec_bi_active_stripes(raid_bio); 5202 if (remaining == 0) { 5203 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 5204 raid_bio, 0); 5205 bio_endio(raid_bio, 0); 5206 } 5207 if (atomic_dec_and_test(&conf->active_aligned_reads)) 5208 wake_up(&conf->wait_for_stripe); 5209 return handled; 5210 } 5211 5212 static int handle_active_stripes(struct r5conf *conf, int group, 5213 struct r5worker *worker, 5214 struct list_head *temp_inactive_list) 5215 { 5216 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; 5217 int i, batch_size = 0, hash; 5218 bool release_inactive = false; 5219 5220 while (batch_size < MAX_STRIPE_BATCH && 5221 (sh = __get_priority_stripe(conf, group)) != NULL) 5222 batch[batch_size++] = sh; 5223 5224 if (batch_size == 0) { 5225 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5226 if (!list_empty(temp_inactive_list + i)) 5227 break; 5228 if (i == NR_STRIPE_HASH_LOCKS) 5229 return batch_size; 5230 release_inactive = true; 5231 } 5232 spin_unlock_irq(&conf->device_lock); 5233 5234 release_inactive_stripe_list(conf, temp_inactive_list, 5235 NR_STRIPE_HASH_LOCKS); 5236 5237 if (release_inactive) { 5238 spin_lock_irq(&conf->device_lock); 5239 return 0; 5240 } 5241 5242 for (i = 0; i < batch_size; i++) 5243 handle_stripe(batch[i]); 5244 5245 cond_resched(); 5246 5247 spin_lock_irq(&conf->device_lock); 5248 for (i = 0; i < batch_size; i++) { 5249 hash = batch[i]->hash_lock_index; 5250 __release_stripe(conf, batch[i], &temp_inactive_list[hash]); 5251 } 5252 return batch_size; 5253 } 5254 5255 static void raid5_do_work(struct work_struct *work) 5256 { 5257 struct r5worker *worker = container_of(work, struct r5worker, work); 5258 struct r5worker_group *group = worker->group; 5259 struct r5conf *conf = group->conf; 5260 int group_id = group - conf->worker_groups; 5261 int handled; 5262 struct blk_plug plug; 5263 5264 pr_debug("+++ raid5worker active\n"); 5265 5266 blk_start_plug(&plug); 5267 handled = 0; 5268 spin_lock_irq(&conf->device_lock); 5269 while (1) { 5270 int batch_size, released; 5271 5272 released = release_stripe_list(conf, worker->temp_inactive_list); 5273 5274 batch_size = handle_active_stripes(conf, group_id, worker, 5275 worker->temp_inactive_list); 5276 worker->working = false; 5277 if (!batch_size && !released) 5278 break; 5279 handled += batch_size; 5280 } 5281 pr_debug("%d stripes handled\n", handled); 5282 5283 spin_unlock_irq(&conf->device_lock); 5284 blk_finish_plug(&plug); 5285 5286 pr_debug("--- raid5worker inactive\n"); 5287 } 5288 5289 /* 5290 * This is our raid5 kernel thread. 5291 * 5292 * We scan the hash table for stripes which can be handled now. 5293 * During the scan, completed stripes are saved for us by the interrupt 5294 * handler, so that they will not have to wait for our next wakeup. 5295 */ 5296 static void raid5d(struct md_thread *thread) 5297 { 5298 struct mddev *mddev = thread->mddev; 5299 struct r5conf *conf = mddev->private; 5300 int handled; 5301 struct blk_plug plug; 5302 5303 pr_debug("+++ raid5d active\n"); 5304 5305 md_check_recovery(mddev); 5306 5307 blk_start_plug(&plug); 5308 handled = 0; 5309 spin_lock_irq(&conf->device_lock); 5310 while (1) { 5311 struct bio *bio; 5312 int batch_size, released; 5313 5314 released = release_stripe_list(conf, conf->temp_inactive_list); 5315 5316 if ( 5317 !list_empty(&conf->bitmap_list)) { 5318 /* Now is a good time to flush some bitmap updates */ 5319 conf->seq_flush++; 5320 spin_unlock_irq(&conf->device_lock); 5321 bitmap_unplug(mddev->bitmap); 5322 spin_lock_irq(&conf->device_lock); 5323 conf->seq_write = conf->seq_flush; 5324 activate_bit_delay(conf, conf->temp_inactive_list); 5325 } 5326 raid5_activate_delayed(conf); 5327 5328 while ((bio = remove_bio_from_retry(conf))) { 5329 int ok; 5330 spin_unlock_irq(&conf->device_lock); 5331 ok = retry_aligned_read(conf, bio); 5332 spin_lock_irq(&conf->device_lock); 5333 if (!ok) 5334 break; 5335 handled++; 5336 } 5337 5338 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, 5339 conf->temp_inactive_list); 5340 if (!batch_size && !released) 5341 break; 5342 handled += batch_size; 5343 5344 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) { 5345 spin_unlock_irq(&conf->device_lock); 5346 md_check_recovery(mddev); 5347 spin_lock_irq(&conf->device_lock); 5348 } 5349 } 5350 pr_debug("%d stripes handled\n", handled); 5351 5352 spin_unlock_irq(&conf->device_lock); 5353 5354 async_tx_issue_pending_all(); 5355 blk_finish_plug(&plug); 5356 5357 pr_debug("--- raid5d inactive\n"); 5358 } 5359 5360 static ssize_t 5361 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 5362 { 5363 struct r5conf *conf; 5364 int ret = 0; 5365 spin_lock(&mddev->lock); 5366 conf = mddev->private; 5367 if (conf) 5368 ret = sprintf(page, "%d\n", conf->max_nr_stripes); 5369 spin_unlock(&mddev->lock); 5370 return ret; 5371 } 5372 5373 int 5374 raid5_set_cache_size(struct mddev *mddev, int size) 5375 { 5376 struct r5conf *conf = mddev->private; 5377 int err; 5378 int hash; 5379 5380 if (size <= 16 || size > 32768) 5381 return -EINVAL; 5382 hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS; 5383 while (size < conf->max_nr_stripes) { 5384 if (drop_one_stripe(conf, hash)) 5385 conf->max_nr_stripes--; 5386 else 5387 break; 5388 hash--; 5389 if (hash < 0) 5390 hash = NR_STRIPE_HASH_LOCKS - 1; 5391 } 5392 err = md_allow_write(mddev); 5393 if (err) 5394 return err; 5395 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 5396 while (size > conf->max_nr_stripes) { 5397 if (grow_one_stripe(conf, hash)) 5398 conf->max_nr_stripes++; 5399 else break; 5400 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS; 5401 } 5402 return 0; 5403 } 5404 EXPORT_SYMBOL(raid5_set_cache_size); 5405 5406 static ssize_t 5407 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 5408 { 5409 struct r5conf *conf; 5410 unsigned long new; 5411 int err; 5412 5413 if (len >= PAGE_SIZE) 5414 return -EINVAL; 5415 if (kstrtoul(page, 10, &new)) 5416 return -EINVAL; 5417 err = mddev_lock(mddev); 5418 if (err) 5419 return err; 5420 conf = mddev->private; 5421 if (!conf) 5422 err = -ENODEV; 5423 else 5424 err = raid5_set_cache_size(mddev, new); 5425 mddev_unlock(mddev); 5426 5427 return err ?: len; 5428 } 5429 5430 static struct md_sysfs_entry 5431 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 5432 raid5_show_stripe_cache_size, 5433 raid5_store_stripe_cache_size); 5434 5435 static ssize_t 5436 raid5_show_preread_threshold(struct mddev *mddev, char *page) 5437 { 5438 struct r5conf *conf; 5439 int ret = 0; 5440 spin_lock(&mddev->lock); 5441 conf = mddev->private; 5442 if (conf) 5443 ret = sprintf(page, "%d\n", conf->bypass_threshold); 5444 spin_unlock(&mddev->lock); 5445 return ret; 5446 } 5447 5448 static ssize_t 5449 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 5450 { 5451 struct r5conf *conf; 5452 unsigned long new; 5453 int err; 5454 5455 if (len >= PAGE_SIZE) 5456 return -EINVAL; 5457 if (kstrtoul(page, 10, &new)) 5458 return -EINVAL; 5459 5460 err = mddev_lock(mddev); 5461 if (err) 5462 return err; 5463 conf = mddev->private; 5464 if (!conf) 5465 err = -ENODEV; 5466 else if (new > conf->max_nr_stripes) 5467 err = -EINVAL; 5468 else 5469 conf->bypass_threshold = new; 5470 mddev_unlock(mddev); 5471 return err ?: len; 5472 } 5473 5474 static struct md_sysfs_entry 5475 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 5476 S_IRUGO | S_IWUSR, 5477 raid5_show_preread_threshold, 5478 raid5_store_preread_threshold); 5479 5480 static ssize_t 5481 raid5_show_skip_copy(struct mddev *mddev, char *page) 5482 { 5483 struct r5conf *conf; 5484 int ret = 0; 5485 spin_lock(&mddev->lock); 5486 conf = mddev->private; 5487 if (conf) 5488 ret = sprintf(page, "%d\n", conf->skip_copy); 5489 spin_unlock(&mddev->lock); 5490 return ret; 5491 } 5492 5493 static ssize_t 5494 raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) 5495 { 5496 struct r5conf *conf; 5497 unsigned long new; 5498 int err; 5499 5500 if (len >= PAGE_SIZE) 5501 return -EINVAL; 5502 if (kstrtoul(page, 10, &new)) 5503 return -EINVAL; 5504 new = !!new; 5505 5506 err = mddev_lock(mddev); 5507 if (err) 5508 return err; 5509 conf = mddev->private; 5510 if (!conf) 5511 err = -ENODEV; 5512 else if (new != conf->skip_copy) { 5513 mddev_suspend(mddev); 5514 conf->skip_copy = new; 5515 if (new) 5516 mddev->queue->backing_dev_info.capabilities |= 5517 BDI_CAP_STABLE_WRITES; 5518 else 5519 mddev->queue->backing_dev_info.capabilities &= 5520 ~BDI_CAP_STABLE_WRITES; 5521 mddev_resume(mddev); 5522 } 5523 mddev_unlock(mddev); 5524 return err ?: len; 5525 } 5526 5527 static struct md_sysfs_entry 5528 raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, 5529 raid5_show_skip_copy, 5530 raid5_store_skip_copy); 5531 5532 static ssize_t 5533 stripe_cache_active_show(struct mddev *mddev, char *page) 5534 { 5535 struct r5conf *conf = mddev->private; 5536 if (conf) 5537 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 5538 else 5539 return 0; 5540 } 5541 5542 static struct md_sysfs_entry 5543 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 5544 5545 static ssize_t 5546 raid5_show_group_thread_cnt(struct mddev *mddev, char *page) 5547 { 5548 struct r5conf *conf; 5549 int ret = 0; 5550 spin_lock(&mddev->lock); 5551 conf = mddev->private; 5552 if (conf) 5553 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); 5554 spin_unlock(&mddev->lock); 5555 return ret; 5556 } 5557 5558 static int alloc_thread_groups(struct r5conf *conf, int cnt, 5559 int *group_cnt, 5560 int *worker_cnt_per_group, 5561 struct r5worker_group **worker_groups); 5562 static ssize_t 5563 raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) 5564 { 5565 struct r5conf *conf; 5566 unsigned long new; 5567 int err; 5568 struct r5worker_group *new_groups, *old_groups; 5569 int group_cnt, worker_cnt_per_group; 5570 5571 if (len >= PAGE_SIZE) 5572 return -EINVAL; 5573 if (kstrtoul(page, 10, &new)) 5574 return -EINVAL; 5575 5576 err = mddev_lock(mddev); 5577 if (err) 5578 return err; 5579 conf = mddev->private; 5580 if (!conf) 5581 err = -ENODEV; 5582 else if (new != conf->worker_cnt_per_group) { 5583 mddev_suspend(mddev); 5584 5585 old_groups = conf->worker_groups; 5586 if (old_groups) 5587 flush_workqueue(raid5_wq); 5588 5589 err = alloc_thread_groups(conf, new, 5590 &group_cnt, &worker_cnt_per_group, 5591 &new_groups); 5592 if (!err) { 5593 spin_lock_irq(&conf->device_lock); 5594 conf->group_cnt = group_cnt; 5595 conf->worker_cnt_per_group = worker_cnt_per_group; 5596 conf->worker_groups = new_groups; 5597 spin_unlock_irq(&conf->device_lock); 5598 5599 if (old_groups) 5600 kfree(old_groups[0].workers); 5601 kfree(old_groups); 5602 } 5603 mddev_resume(mddev); 5604 } 5605 mddev_unlock(mddev); 5606 5607 return err ?: len; 5608 } 5609 5610 static struct md_sysfs_entry 5611 raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, 5612 raid5_show_group_thread_cnt, 5613 raid5_store_group_thread_cnt); 5614 5615 static struct attribute *raid5_attrs[] = { 5616 &raid5_stripecache_size.attr, 5617 &raid5_stripecache_active.attr, 5618 &raid5_preread_bypass_threshold.attr, 5619 &raid5_group_thread_cnt.attr, 5620 &raid5_skip_copy.attr, 5621 NULL, 5622 }; 5623 static struct attribute_group raid5_attrs_group = { 5624 .name = NULL, 5625 .attrs = raid5_attrs, 5626 }; 5627 5628 static int alloc_thread_groups(struct r5conf *conf, int cnt, 5629 int *group_cnt, 5630 int *worker_cnt_per_group, 5631 struct r5worker_group **worker_groups) 5632 { 5633 int i, j, k; 5634 ssize_t size; 5635 struct r5worker *workers; 5636 5637 *worker_cnt_per_group = cnt; 5638 if (cnt == 0) { 5639 *group_cnt = 0; 5640 *worker_groups = NULL; 5641 return 0; 5642 } 5643 *group_cnt = num_possible_nodes(); 5644 size = sizeof(struct r5worker) * cnt; 5645 workers = kzalloc(size * *group_cnt, GFP_NOIO); 5646 *worker_groups = kzalloc(sizeof(struct r5worker_group) * 5647 *group_cnt, GFP_NOIO); 5648 if (!*worker_groups || !workers) { 5649 kfree(workers); 5650 kfree(*worker_groups); 5651 return -ENOMEM; 5652 } 5653 5654 for (i = 0; i < *group_cnt; i++) { 5655 struct r5worker_group *group; 5656 5657 group = &(*worker_groups)[i]; 5658 INIT_LIST_HEAD(&group->handle_list); 5659 group->conf = conf; 5660 group->workers = workers + i * cnt; 5661 5662 for (j = 0; j < cnt; j++) { 5663 struct r5worker *worker = group->workers + j; 5664 worker->group = group; 5665 INIT_WORK(&worker->work, raid5_do_work); 5666 5667 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) 5668 INIT_LIST_HEAD(worker->temp_inactive_list + k); 5669 } 5670 } 5671 5672 return 0; 5673 } 5674 5675 static void free_thread_groups(struct r5conf *conf) 5676 { 5677 if (conf->worker_groups) 5678 kfree(conf->worker_groups[0].workers); 5679 kfree(conf->worker_groups); 5680 conf->worker_groups = NULL; 5681 } 5682 5683 static sector_t 5684 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 5685 { 5686 struct r5conf *conf = mddev->private; 5687 5688 if (!sectors) 5689 sectors = mddev->dev_sectors; 5690 if (!raid_disks) 5691 /* size is defined by the smallest of previous and new size */ 5692 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 5693 5694 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5695 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 5696 return sectors * (raid_disks - conf->max_degraded); 5697 } 5698 5699 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 5700 { 5701 safe_put_page(percpu->spare_page); 5702 kfree(percpu->scribble); 5703 percpu->spare_page = NULL; 5704 percpu->scribble = NULL; 5705 } 5706 5707 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 5708 { 5709 if (conf->level == 6 && !percpu->spare_page) 5710 percpu->spare_page = alloc_page(GFP_KERNEL); 5711 if (!percpu->scribble) 5712 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 5713 5714 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { 5715 free_scratch_buffer(conf, percpu); 5716 return -ENOMEM; 5717 } 5718 5719 return 0; 5720 } 5721 5722 static void raid5_free_percpu(struct r5conf *conf) 5723 { 5724 unsigned long cpu; 5725 5726 if (!conf->percpu) 5727 return; 5728 5729 #ifdef CONFIG_HOTPLUG_CPU 5730 unregister_cpu_notifier(&conf->cpu_notify); 5731 #endif 5732 5733 get_online_cpus(); 5734 for_each_possible_cpu(cpu) 5735 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5736 put_online_cpus(); 5737 5738 free_percpu(conf->percpu); 5739 } 5740 5741 static void free_conf(struct r5conf *conf) 5742 { 5743 free_thread_groups(conf); 5744 shrink_stripes(conf); 5745 raid5_free_percpu(conf); 5746 kfree(conf->disks); 5747 kfree(conf->stripe_hashtbl); 5748 kfree(conf); 5749 } 5750 5751 #ifdef CONFIG_HOTPLUG_CPU 5752 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 5753 void *hcpu) 5754 { 5755 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); 5756 long cpu = (long)hcpu; 5757 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 5758 5759 switch (action) { 5760 case CPU_UP_PREPARE: 5761 case CPU_UP_PREPARE_FROZEN: 5762 if (alloc_scratch_buffer(conf, percpu)) { 5763 pr_err("%s: failed memory allocation for cpu%ld\n", 5764 __func__, cpu); 5765 return notifier_from_errno(-ENOMEM); 5766 } 5767 break; 5768 case CPU_DEAD: 5769 case CPU_DEAD_FROZEN: 5770 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5771 break; 5772 default: 5773 break; 5774 } 5775 return NOTIFY_OK; 5776 } 5777 #endif 5778 5779 static int raid5_alloc_percpu(struct r5conf *conf) 5780 { 5781 unsigned long cpu; 5782 int err = 0; 5783 5784 conf->percpu = alloc_percpu(struct raid5_percpu); 5785 if (!conf->percpu) 5786 return -ENOMEM; 5787 5788 #ifdef CONFIG_HOTPLUG_CPU 5789 conf->cpu_notify.notifier_call = raid456_cpu_notify; 5790 conf->cpu_notify.priority = 0; 5791 err = register_cpu_notifier(&conf->cpu_notify); 5792 if (err) 5793 return err; 5794 #endif 5795 5796 get_online_cpus(); 5797 for_each_present_cpu(cpu) { 5798 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5799 if (err) { 5800 pr_err("%s: failed memory allocation for cpu%ld\n", 5801 __func__, cpu); 5802 break; 5803 } 5804 } 5805 put_online_cpus(); 5806 5807 return err; 5808 } 5809 5810 static struct r5conf *setup_conf(struct mddev *mddev) 5811 { 5812 struct r5conf *conf; 5813 int raid_disk, memory, max_disks; 5814 struct md_rdev *rdev; 5815 struct disk_info *disk; 5816 char pers_name[6]; 5817 int i; 5818 int group_cnt, worker_cnt_per_group; 5819 struct r5worker_group *new_group; 5820 5821 if (mddev->new_level != 5 5822 && mddev->new_level != 4 5823 && mddev->new_level != 6) { 5824 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 5825 mdname(mddev), mddev->new_level); 5826 return ERR_PTR(-EIO); 5827 } 5828 if ((mddev->new_level == 5 5829 && !algorithm_valid_raid5(mddev->new_layout)) || 5830 (mddev->new_level == 6 5831 && !algorithm_valid_raid6(mddev->new_layout))) { 5832 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 5833 mdname(mddev), mddev->new_layout); 5834 return ERR_PTR(-EIO); 5835 } 5836 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 5837 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 5838 mdname(mddev), mddev->raid_disks); 5839 return ERR_PTR(-EINVAL); 5840 } 5841 5842 if (!mddev->new_chunk_sectors || 5843 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 5844 !is_power_of_2(mddev->new_chunk_sectors)) { 5845 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 5846 mdname(mddev), mddev->new_chunk_sectors << 9); 5847 return ERR_PTR(-EINVAL); 5848 } 5849 5850 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 5851 if (conf == NULL) 5852 goto abort; 5853 /* Don't enable multi-threading by default*/ 5854 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, 5855 &new_group)) { 5856 conf->group_cnt = group_cnt; 5857 conf->worker_cnt_per_group = worker_cnt_per_group; 5858 conf->worker_groups = new_group; 5859 } else 5860 goto abort; 5861 spin_lock_init(&conf->device_lock); 5862 seqcount_init(&conf->gen_lock); 5863 init_waitqueue_head(&conf->wait_for_stripe); 5864 init_waitqueue_head(&conf->wait_for_overlap); 5865 INIT_LIST_HEAD(&conf->handle_list); 5866 INIT_LIST_HEAD(&conf->hold_list); 5867 INIT_LIST_HEAD(&conf->delayed_list); 5868 INIT_LIST_HEAD(&conf->bitmap_list); 5869 init_llist_head(&conf->released_stripes); 5870 atomic_set(&conf->active_stripes, 0); 5871 atomic_set(&conf->preread_active_stripes, 0); 5872 atomic_set(&conf->active_aligned_reads, 0); 5873 conf->bypass_threshold = BYPASS_THRESHOLD; 5874 conf->recovery_disabled = mddev->recovery_disabled - 1; 5875 5876 conf->raid_disks = mddev->raid_disks; 5877 if (mddev->reshape_position == MaxSector) 5878 conf->previous_raid_disks = mddev->raid_disks; 5879 else 5880 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 5881 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 5882 conf->scribble_len = scribble_len(max_disks); 5883 5884 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 5885 GFP_KERNEL); 5886 if (!conf->disks) 5887 goto abort; 5888 5889 conf->mddev = mddev; 5890 5891 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 5892 goto abort; 5893 5894 /* We init hash_locks[0] separately to that it can be used 5895 * as the reference lock in the spin_lock_nest_lock() call 5896 * in lock_all_device_hash_locks_irq in order to convince 5897 * lockdep that we know what we are doing. 5898 */ 5899 spin_lock_init(conf->hash_locks); 5900 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 5901 spin_lock_init(conf->hash_locks + i); 5902 5903 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5904 INIT_LIST_HEAD(conf->inactive_list + i); 5905 5906 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5907 INIT_LIST_HEAD(conf->temp_inactive_list + i); 5908 5909 conf->level = mddev->new_level; 5910 if (raid5_alloc_percpu(conf) != 0) 5911 goto abort; 5912 5913 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 5914 5915 rdev_for_each(rdev, mddev) { 5916 raid_disk = rdev->raid_disk; 5917 if (raid_disk >= max_disks 5918 || raid_disk < 0) 5919 continue; 5920 disk = conf->disks + raid_disk; 5921 5922 if (test_bit(Replacement, &rdev->flags)) { 5923 if (disk->replacement) 5924 goto abort; 5925 disk->replacement = rdev; 5926 } else { 5927 if (disk->rdev) 5928 goto abort; 5929 disk->rdev = rdev; 5930 } 5931 5932 if (test_bit(In_sync, &rdev->flags)) { 5933 char b[BDEVNAME_SIZE]; 5934 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 5935 " disk %d\n", 5936 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 5937 } else if (rdev->saved_raid_disk != raid_disk) 5938 /* Cannot rely on bitmap to complete recovery */ 5939 conf->fullsync = 1; 5940 } 5941 5942 conf->chunk_sectors = mddev->new_chunk_sectors; 5943 conf->level = mddev->new_level; 5944 if (conf->level == 6) 5945 conf->max_degraded = 2; 5946 else 5947 conf->max_degraded = 1; 5948 conf->algorithm = mddev->new_layout; 5949 conf->reshape_progress = mddev->reshape_position; 5950 if (conf->reshape_progress != MaxSector) { 5951 conf->prev_chunk_sectors = mddev->chunk_sectors; 5952 conf->prev_algo = mddev->layout; 5953 } 5954 5955 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 5956 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 5957 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); 5958 if (grow_stripes(conf, NR_STRIPES)) { 5959 printk(KERN_ERR 5960 "md/raid:%s: couldn't allocate %dkB for buffers\n", 5961 mdname(mddev), memory); 5962 goto abort; 5963 } else 5964 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 5965 mdname(mddev), memory); 5966 5967 sprintf(pers_name, "raid%d", mddev->new_level); 5968 conf->thread = md_register_thread(raid5d, mddev, pers_name); 5969 if (!conf->thread) { 5970 printk(KERN_ERR 5971 "md/raid:%s: couldn't allocate thread.\n", 5972 mdname(mddev)); 5973 goto abort; 5974 } 5975 5976 return conf; 5977 5978 abort: 5979 if (conf) { 5980 free_conf(conf); 5981 return ERR_PTR(-EIO); 5982 } else 5983 return ERR_PTR(-ENOMEM); 5984 } 5985 5986 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 5987 { 5988 switch (algo) { 5989 case ALGORITHM_PARITY_0: 5990 if (raid_disk < max_degraded) 5991 return 1; 5992 break; 5993 case ALGORITHM_PARITY_N: 5994 if (raid_disk >= raid_disks - max_degraded) 5995 return 1; 5996 break; 5997 case ALGORITHM_PARITY_0_6: 5998 if (raid_disk == 0 || 5999 raid_disk == raid_disks - 1) 6000 return 1; 6001 break; 6002 case ALGORITHM_LEFT_ASYMMETRIC_6: 6003 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6004 case ALGORITHM_LEFT_SYMMETRIC_6: 6005 case ALGORITHM_RIGHT_SYMMETRIC_6: 6006 if (raid_disk == raid_disks - 1) 6007 return 1; 6008 } 6009 return 0; 6010 } 6011 6012 static int run(struct mddev *mddev) 6013 { 6014 struct r5conf *conf; 6015 int working_disks = 0; 6016 int dirty_parity_disks = 0; 6017 struct md_rdev *rdev; 6018 sector_t reshape_offset = 0; 6019 int i; 6020 long long min_offset_diff = 0; 6021 int first = 1; 6022 6023 if (mddev->recovery_cp != MaxSector) 6024 printk(KERN_NOTICE "md/raid:%s: not clean" 6025 " -- starting background reconstruction\n", 6026 mdname(mddev)); 6027 6028 rdev_for_each(rdev, mddev) { 6029 long long diff; 6030 if (rdev->raid_disk < 0) 6031 continue; 6032 diff = (rdev->new_data_offset - rdev->data_offset); 6033 if (first) { 6034 min_offset_diff = diff; 6035 first = 0; 6036 } else if (mddev->reshape_backwards && 6037 diff < min_offset_diff) 6038 min_offset_diff = diff; 6039 else if (!mddev->reshape_backwards && 6040 diff > min_offset_diff) 6041 min_offset_diff = diff; 6042 } 6043 6044 if (mddev->reshape_position != MaxSector) { 6045 /* Check that we can continue the reshape. 6046 * Difficulties arise if the stripe we would write to 6047 * next is at or after the stripe we would read from next. 6048 * For a reshape that changes the number of devices, this 6049 * is only possible for a very short time, and mdadm makes 6050 * sure that time appears to have past before assembling 6051 * the array. So we fail if that time hasn't passed. 6052 * For a reshape that keeps the number of devices the same 6053 * mdadm must be monitoring the reshape can keeping the 6054 * critical areas read-only and backed up. It will start 6055 * the array in read-only mode, so we check for that. 6056 */ 6057 sector_t here_new, here_old; 6058 int old_disks; 6059 int max_degraded = (mddev->level == 6 ? 2 : 1); 6060 6061 if (mddev->new_level != mddev->level) { 6062 printk(KERN_ERR "md/raid:%s: unsupported reshape " 6063 "required - aborting.\n", 6064 mdname(mddev)); 6065 return -EINVAL; 6066 } 6067 old_disks = mddev->raid_disks - mddev->delta_disks; 6068 /* reshape_position must be on a new-stripe boundary, and one 6069 * further up in new geometry must map after here in old 6070 * geometry. 6071 */ 6072 here_new = mddev->reshape_position; 6073 if (sector_div(here_new, mddev->new_chunk_sectors * 6074 (mddev->raid_disks - max_degraded))) { 6075 printk(KERN_ERR "md/raid:%s: reshape_position not " 6076 "on a stripe boundary\n", mdname(mddev)); 6077 return -EINVAL; 6078 } 6079 reshape_offset = here_new * mddev->new_chunk_sectors; 6080 /* here_new is the stripe we will write to */ 6081 here_old = mddev->reshape_position; 6082 sector_div(here_old, mddev->chunk_sectors * 6083 (old_disks-max_degraded)); 6084 /* here_old is the first stripe that we might need to read 6085 * from */ 6086 if (mddev->delta_disks == 0) { 6087 if ((here_new * mddev->new_chunk_sectors != 6088 here_old * mddev->chunk_sectors)) { 6089 printk(KERN_ERR "md/raid:%s: reshape position is" 6090 " confused - aborting\n", mdname(mddev)); 6091 return -EINVAL; 6092 } 6093 /* We cannot be sure it is safe to start an in-place 6094 * reshape. It is only safe if user-space is monitoring 6095 * and taking constant backups. 6096 * mdadm always starts a situation like this in 6097 * readonly mode so it can take control before 6098 * allowing any writes. So just check for that. 6099 */ 6100 if (abs(min_offset_diff) >= mddev->chunk_sectors && 6101 abs(min_offset_diff) >= mddev->new_chunk_sectors) 6102 /* not really in-place - so OK */; 6103 else if (mddev->ro == 0) { 6104 printk(KERN_ERR "md/raid:%s: in-place reshape " 6105 "must be started in read-only mode " 6106 "- aborting\n", 6107 mdname(mddev)); 6108 return -EINVAL; 6109 } 6110 } else if (mddev->reshape_backwards 6111 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= 6112 here_old * mddev->chunk_sectors) 6113 : (here_new * mddev->new_chunk_sectors >= 6114 here_old * mddev->chunk_sectors + (-min_offset_diff))) { 6115 /* Reading from the same stripe as writing to - bad */ 6116 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 6117 "auto-recovery - aborting.\n", 6118 mdname(mddev)); 6119 return -EINVAL; 6120 } 6121 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 6122 mdname(mddev)); 6123 /* OK, we should be able to continue; */ 6124 } else { 6125 BUG_ON(mddev->level != mddev->new_level); 6126 BUG_ON(mddev->layout != mddev->new_layout); 6127 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 6128 BUG_ON(mddev->delta_disks != 0); 6129 } 6130 6131 if (mddev->private == NULL) 6132 conf = setup_conf(mddev); 6133 else 6134 conf = mddev->private; 6135 6136 if (IS_ERR(conf)) 6137 return PTR_ERR(conf); 6138 6139 conf->min_offset_diff = min_offset_diff; 6140 mddev->thread = conf->thread; 6141 conf->thread = NULL; 6142 mddev->private = conf; 6143 6144 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 6145 i++) { 6146 rdev = conf->disks[i].rdev; 6147 if (!rdev && conf->disks[i].replacement) { 6148 /* The replacement is all we have yet */ 6149 rdev = conf->disks[i].replacement; 6150 conf->disks[i].replacement = NULL; 6151 clear_bit(Replacement, &rdev->flags); 6152 conf->disks[i].rdev = rdev; 6153 } 6154 if (!rdev) 6155 continue; 6156 if (conf->disks[i].replacement && 6157 conf->reshape_progress != MaxSector) { 6158 /* replacements and reshape simply do not mix. */ 6159 printk(KERN_ERR "md: cannot handle concurrent " 6160 "replacement and reshape.\n"); 6161 goto abort; 6162 } 6163 if (test_bit(In_sync, &rdev->flags)) { 6164 working_disks++; 6165 continue; 6166 } 6167 /* This disc is not fully in-sync. However if it 6168 * just stored parity (beyond the recovery_offset), 6169 * when we don't need to be concerned about the 6170 * array being dirty. 6171 * When reshape goes 'backwards', we never have 6172 * partially completed devices, so we only need 6173 * to worry about reshape going forwards. 6174 */ 6175 /* Hack because v0.91 doesn't store recovery_offset properly. */ 6176 if (mddev->major_version == 0 && 6177 mddev->minor_version > 90) 6178 rdev->recovery_offset = reshape_offset; 6179 6180 if (rdev->recovery_offset < reshape_offset) { 6181 /* We need to check old and new layout */ 6182 if (!only_parity(rdev->raid_disk, 6183 conf->algorithm, 6184 conf->raid_disks, 6185 conf->max_degraded)) 6186 continue; 6187 } 6188 if (!only_parity(rdev->raid_disk, 6189 conf->prev_algo, 6190 conf->previous_raid_disks, 6191 conf->max_degraded)) 6192 continue; 6193 dirty_parity_disks++; 6194 } 6195 6196 /* 6197 * 0 for a fully functional array, 1 or 2 for a degraded array. 6198 */ 6199 mddev->degraded = calc_degraded(conf); 6200 6201 if (has_failed(conf)) { 6202 printk(KERN_ERR "md/raid:%s: not enough operational devices" 6203 " (%d/%d failed)\n", 6204 mdname(mddev), mddev->degraded, conf->raid_disks); 6205 goto abort; 6206 } 6207 6208 /* device size must be a multiple of chunk size */ 6209 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 6210 mddev->resync_max_sectors = mddev->dev_sectors; 6211 6212 if (mddev->degraded > dirty_parity_disks && 6213 mddev->recovery_cp != MaxSector) { 6214 if (mddev->ok_start_degraded) 6215 printk(KERN_WARNING 6216 "md/raid:%s: starting dirty degraded array" 6217 " - data corruption possible.\n", 6218 mdname(mddev)); 6219 else { 6220 printk(KERN_ERR 6221 "md/raid:%s: cannot start dirty degraded array.\n", 6222 mdname(mddev)); 6223 goto abort; 6224 } 6225 } 6226 6227 if (mddev->degraded == 0) 6228 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 6229 " devices, algorithm %d\n", mdname(mddev), conf->level, 6230 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 6231 mddev->new_layout); 6232 else 6233 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 6234 " out of %d devices, algorithm %d\n", 6235 mdname(mddev), conf->level, 6236 mddev->raid_disks - mddev->degraded, 6237 mddev->raid_disks, mddev->new_layout); 6238 6239 print_raid5_conf(conf); 6240 6241 if (conf->reshape_progress != MaxSector) { 6242 conf->reshape_safe = conf->reshape_progress; 6243 atomic_set(&conf->reshape_stripes, 0); 6244 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6245 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6246 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6247 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6248 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6249 "reshape"); 6250 } 6251 6252 /* Ok, everything is just fine now */ 6253 if (mddev->to_remove == &raid5_attrs_group) 6254 mddev->to_remove = NULL; 6255 else if (mddev->kobj.sd && 6256 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 6257 printk(KERN_WARNING 6258 "raid5: failed to create sysfs attributes for %s\n", 6259 mdname(mddev)); 6260 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6261 6262 if (mddev->queue) { 6263 int chunk_size; 6264 bool discard_supported = true; 6265 /* read-ahead size must cover two whole stripes, which 6266 * is 2 * (datadisks) * chunksize where 'n' is the 6267 * number of raid devices 6268 */ 6269 int data_disks = conf->previous_raid_disks - conf->max_degraded; 6270 int stripe = data_disks * 6271 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 6272 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6273 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6274 6275 chunk_size = mddev->chunk_sectors << 9; 6276 blk_queue_io_min(mddev->queue, chunk_size); 6277 blk_queue_io_opt(mddev->queue, chunk_size * 6278 (conf->raid_disks - conf->max_degraded)); 6279 mddev->queue->limits.raid_partial_stripes_expensive = 1; 6280 /* 6281 * We can only discard a whole stripe. It doesn't make sense to 6282 * discard data disk but write parity disk 6283 */ 6284 stripe = stripe * PAGE_SIZE; 6285 /* Round up to power of 2, as discard handling 6286 * currently assumes that */ 6287 while ((stripe-1) & stripe) 6288 stripe = (stripe | (stripe-1)) + 1; 6289 mddev->queue->limits.discard_alignment = stripe; 6290 mddev->queue->limits.discard_granularity = stripe; 6291 /* 6292 * unaligned part of discard request will be ignored, so can't 6293 * guarantee discard_zeroes_data 6294 */ 6295 mddev->queue->limits.discard_zeroes_data = 0; 6296 6297 blk_queue_max_write_same_sectors(mddev->queue, 0); 6298 6299 rdev_for_each(rdev, mddev) { 6300 disk_stack_limits(mddev->gendisk, rdev->bdev, 6301 rdev->data_offset << 9); 6302 disk_stack_limits(mddev->gendisk, rdev->bdev, 6303 rdev->new_data_offset << 9); 6304 /* 6305 * discard_zeroes_data is required, otherwise data 6306 * could be lost. Consider a scenario: discard a stripe 6307 * (the stripe could be inconsistent if 6308 * discard_zeroes_data is 0); write one disk of the 6309 * stripe (the stripe could be inconsistent again 6310 * depending on which disks are used to calculate 6311 * parity); the disk is broken; The stripe data of this 6312 * disk is lost. 6313 */ 6314 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || 6315 !bdev_get_queue(rdev->bdev)-> 6316 limits.discard_zeroes_data) 6317 discard_supported = false; 6318 /* Unfortunately, discard_zeroes_data is not currently 6319 * a guarantee - just a hint. So we only allow DISCARD 6320 * if the sysadmin has confirmed that only safe devices 6321 * are in use by setting a module parameter. 6322 */ 6323 if (!devices_handle_discard_safely) { 6324 if (discard_supported) { 6325 pr_info("md/raid456: discard support disabled due to uncertainty.\n"); 6326 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); 6327 } 6328 discard_supported = false; 6329 } 6330 } 6331 6332 if (discard_supported && 6333 mddev->queue->limits.max_discard_sectors >= stripe && 6334 mddev->queue->limits.discard_granularity >= stripe) 6335 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 6336 mddev->queue); 6337 else 6338 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 6339 mddev->queue); 6340 } 6341 6342 return 0; 6343 abort: 6344 md_unregister_thread(&mddev->thread); 6345 print_raid5_conf(conf); 6346 free_conf(conf); 6347 mddev->private = NULL; 6348 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 6349 return -EIO; 6350 } 6351 6352 static void raid5_free(struct mddev *mddev, void *priv) 6353 { 6354 struct r5conf *conf = priv; 6355 6356 free_conf(conf); 6357 mddev->to_remove = &raid5_attrs_group; 6358 } 6359 6360 static void status(struct seq_file *seq, struct mddev *mddev) 6361 { 6362 struct r5conf *conf = mddev->private; 6363 int i; 6364 6365 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 6366 mddev->chunk_sectors / 2, mddev->layout); 6367 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 6368 for (i = 0; i < conf->raid_disks; i++) 6369 seq_printf (seq, "%s", 6370 conf->disks[i].rdev && 6371 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 6372 seq_printf (seq, "]"); 6373 } 6374 6375 static void print_raid5_conf (struct r5conf *conf) 6376 { 6377 int i; 6378 struct disk_info *tmp; 6379 6380 printk(KERN_DEBUG "RAID conf printout:\n"); 6381 if (!conf) { 6382 printk("(conf==NULL)\n"); 6383 return; 6384 } 6385 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 6386 conf->raid_disks, 6387 conf->raid_disks - conf->mddev->degraded); 6388 6389 for (i = 0; i < conf->raid_disks; i++) { 6390 char b[BDEVNAME_SIZE]; 6391 tmp = conf->disks + i; 6392 if (tmp->rdev) 6393 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 6394 i, !test_bit(Faulty, &tmp->rdev->flags), 6395 bdevname(tmp->rdev->bdev, b)); 6396 } 6397 } 6398 6399 static int raid5_spare_active(struct mddev *mddev) 6400 { 6401 int i; 6402 struct r5conf *conf = mddev->private; 6403 struct disk_info *tmp; 6404 int count = 0; 6405 unsigned long flags; 6406 6407 for (i = 0; i < conf->raid_disks; i++) { 6408 tmp = conf->disks + i; 6409 if (tmp->replacement 6410 && tmp->replacement->recovery_offset == MaxSector 6411 && !test_bit(Faulty, &tmp->replacement->flags) 6412 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 6413 /* Replacement has just become active. */ 6414 if (!tmp->rdev 6415 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 6416 count++; 6417 if (tmp->rdev) { 6418 /* Replaced device not technically faulty, 6419 * but we need to be sure it gets removed 6420 * and never re-added. 6421 */ 6422 set_bit(Faulty, &tmp->rdev->flags); 6423 sysfs_notify_dirent_safe( 6424 tmp->rdev->sysfs_state); 6425 } 6426 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 6427 } else if (tmp->rdev 6428 && tmp->rdev->recovery_offset == MaxSector 6429 && !test_bit(Faulty, &tmp->rdev->flags) 6430 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 6431 count++; 6432 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 6433 } 6434 } 6435 spin_lock_irqsave(&conf->device_lock, flags); 6436 mddev->degraded = calc_degraded(conf); 6437 spin_unlock_irqrestore(&conf->device_lock, flags); 6438 print_raid5_conf(conf); 6439 return count; 6440 } 6441 6442 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 6443 { 6444 struct r5conf *conf = mddev->private; 6445 int err = 0; 6446 int number = rdev->raid_disk; 6447 struct md_rdev **rdevp; 6448 struct disk_info *p = conf->disks + number; 6449 6450 print_raid5_conf(conf); 6451 if (rdev == p->rdev) 6452 rdevp = &p->rdev; 6453 else if (rdev == p->replacement) 6454 rdevp = &p->replacement; 6455 else 6456 return 0; 6457 6458 if (number >= conf->raid_disks && 6459 conf->reshape_progress == MaxSector) 6460 clear_bit(In_sync, &rdev->flags); 6461 6462 if (test_bit(In_sync, &rdev->flags) || 6463 atomic_read(&rdev->nr_pending)) { 6464 err = -EBUSY; 6465 goto abort; 6466 } 6467 /* Only remove non-faulty devices if recovery 6468 * isn't possible. 6469 */ 6470 if (!test_bit(Faulty, &rdev->flags) && 6471 mddev->recovery_disabled != conf->recovery_disabled && 6472 !has_failed(conf) && 6473 (!p->replacement || p->replacement == rdev) && 6474 number < conf->raid_disks) { 6475 err = -EBUSY; 6476 goto abort; 6477 } 6478 *rdevp = NULL; 6479 synchronize_rcu(); 6480 if (atomic_read(&rdev->nr_pending)) { 6481 /* lost the race, try later */ 6482 err = -EBUSY; 6483 *rdevp = rdev; 6484 } else if (p->replacement) { 6485 /* We must have just cleared 'rdev' */ 6486 p->rdev = p->replacement; 6487 clear_bit(Replacement, &p->replacement->flags); 6488 smp_mb(); /* Make sure other CPUs may see both as identical 6489 * but will never see neither - if they are careful 6490 */ 6491 p->replacement = NULL; 6492 clear_bit(WantReplacement, &rdev->flags); 6493 } else 6494 /* We might have just removed the Replacement as faulty- 6495 * clear the bit just in case 6496 */ 6497 clear_bit(WantReplacement, &rdev->flags); 6498 abort: 6499 6500 print_raid5_conf(conf); 6501 return err; 6502 } 6503 6504 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 6505 { 6506 struct r5conf *conf = mddev->private; 6507 int err = -EEXIST; 6508 int disk; 6509 struct disk_info *p; 6510 int first = 0; 6511 int last = conf->raid_disks - 1; 6512 6513 if (mddev->recovery_disabled == conf->recovery_disabled) 6514 return -EBUSY; 6515 6516 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 6517 /* no point adding a device */ 6518 return -EINVAL; 6519 6520 if (rdev->raid_disk >= 0) 6521 first = last = rdev->raid_disk; 6522 6523 /* 6524 * find the disk ... but prefer rdev->saved_raid_disk 6525 * if possible. 6526 */ 6527 if (rdev->saved_raid_disk >= 0 && 6528 rdev->saved_raid_disk >= first && 6529 conf->disks[rdev->saved_raid_disk].rdev == NULL) 6530 first = rdev->saved_raid_disk; 6531 6532 for (disk = first; disk <= last; disk++) { 6533 p = conf->disks + disk; 6534 if (p->rdev == NULL) { 6535 clear_bit(In_sync, &rdev->flags); 6536 rdev->raid_disk = disk; 6537 err = 0; 6538 if (rdev->saved_raid_disk != disk) 6539 conf->fullsync = 1; 6540 rcu_assign_pointer(p->rdev, rdev); 6541 goto out; 6542 } 6543 } 6544 for (disk = first; disk <= last; disk++) { 6545 p = conf->disks + disk; 6546 if (test_bit(WantReplacement, &p->rdev->flags) && 6547 p->replacement == NULL) { 6548 clear_bit(In_sync, &rdev->flags); 6549 set_bit(Replacement, &rdev->flags); 6550 rdev->raid_disk = disk; 6551 err = 0; 6552 conf->fullsync = 1; 6553 rcu_assign_pointer(p->replacement, rdev); 6554 break; 6555 } 6556 } 6557 out: 6558 print_raid5_conf(conf); 6559 return err; 6560 } 6561 6562 static int raid5_resize(struct mddev *mddev, sector_t sectors) 6563 { 6564 /* no resync is happening, and there is enough space 6565 * on all devices, so we can resize. 6566 * We need to make sure resync covers any new space. 6567 * If the array is shrinking we should possibly wait until 6568 * any io in the removed space completes, but it hardly seems 6569 * worth it. 6570 */ 6571 sector_t newsize; 6572 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 6573 newsize = raid5_size(mddev, sectors, mddev->raid_disks); 6574 if (mddev->external_size && 6575 mddev->array_sectors > newsize) 6576 return -EINVAL; 6577 if (mddev->bitmap) { 6578 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); 6579 if (ret) 6580 return ret; 6581 } 6582 md_set_array_sectors(mddev, newsize); 6583 set_capacity(mddev->gendisk, mddev->array_sectors); 6584 revalidate_disk(mddev->gendisk); 6585 if (sectors > mddev->dev_sectors && 6586 mddev->recovery_cp > mddev->dev_sectors) { 6587 mddev->recovery_cp = mddev->dev_sectors; 6588 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 6589 } 6590 mddev->dev_sectors = sectors; 6591 mddev->resync_max_sectors = sectors; 6592 return 0; 6593 } 6594 6595 static int check_stripe_cache(struct mddev *mddev) 6596 { 6597 /* Can only proceed if there are plenty of stripe_heads. 6598 * We need a minimum of one full stripe,, and for sensible progress 6599 * it is best to have about 4 times that. 6600 * If we require 4 times, then the default 256 4K stripe_heads will 6601 * allow for chunk sizes up to 256K, which is probably OK. 6602 * If the chunk size is greater, user-space should request more 6603 * stripe_heads first. 6604 */ 6605 struct r5conf *conf = mddev->private; 6606 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 6607 > conf->max_nr_stripes || 6608 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 6609 > conf->max_nr_stripes) { 6610 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 6611 mdname(mddev), 6612 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 6613 / STRIPE_SIZE)*4); 6614 return 0; 6615 } 6616 return 1; 6617 } 6618 6619 static int check_reshape(struct mddev *mddev) 6620 { 6621 struct r5conf *conf = mddev->private; 6622 6623 if (mddev->delta_disks == 0 && 6624 mddev->new_layout == mddev->layout && 6625 mddev->new_chunk_sectors == mddev->chunk_sectors) 6626 return 0; /* nothing to do */ 6627 if (has_failed(conf)) 6628 return -EINVAL; 6629 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { 6630 /* We might be able to shrink, but the devices must 6631 * be made bigger first. 6632 * For raid6, 4 is the minimum size. 6633 * Otherwise 2 is the minimum 6634 */ 6635 int min = 2; 6636 if (mddev->level == 6) 6637 min = 4; 6638 if (mddev->raid_disks + mddev->delta_disks < min) 6639 return -EINVAL; 6640 } 6641 6642 if (!check_stripe_cache(mddev)) 6643 return -ENOSPC; 6644 6645 return resize_stripes(conf, (conf->previous_raid_disks 6646 + mddev->delta_disks)); 6647 } 6648 6649 static int raid5_start_reshape(struct mddev *mddev) 6650 { 6651 struct r5conf *conf = mddev->private; 6652 struct md_rdev *rdev; 6653 int spares = 0; 6654 unsigned long flags; 6655 6656 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 6657 return -EBUSY; 6658 6659 if (!check_stripe_cache(mddev)) 6660 return -ENOSPC; 6661 6662 if (has_failed(conf)) 6663 return -EINVAL; 6664 6665 rdev_for_each(rdev, mddev) { 6666 if (!test_bit(In_sync, &rdev->flags) 6667 && !test_bit(Faulty, &rdev->flags)) 6668 spares++; 6669 } 6670 6671 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 6672 /* Not enough devices even to make a degraded array 6673 * of that size 6674 */ 6675 return -EINVAL; 6676 6677 /* Refuse to reduce size of the array. Any reductions in 6678 * array size must be through explicit setting of array_size 6679 * attribute. 6680 */ 6681 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 6682 < mddev->array_sectors) { 6683 printk(KERN_ERR "md/raid:%s: array size must be reduced " 6684 "before number of disks\n", mdname(mddev)); 6685 return -EINVAL; 6686 } 6687 6688 atomic_set(&conf->reshape_stripes, 0); 6689 spin_lock_irq(&conf->device_lock); 6690 write_seqcount_begin(&conf->gen_lock); 6691 conf->previous_raid_disks = conf->raid_disks; 6692 conf->raid_disks += mddev->delta_disks; 6693 conf->prev_chunk_sectors = conf->chunk_sectors; 6694 conf->chunk_sectors = mddev->new_chunk_sectors; 6695 conf->prev_algo = conf->algorithm; 6696 conf->algorithm = mddev->new_layout; 6697 conf->generation++; 6698 /* Code that selects data_offset needs to see the generation update 6699 * if reshape_progress has been set - so a memory barrier needed. 6700 */ 6701 smp_mb(); 6702 if (mddev->reshape_backwards) 6703 conf->reshape_progress = raid5_size(mddev, 0, 0); 6704 else 6705 conf->reshape_progress = 0; 6706 conf->reshape_safe = conf->reshape_progress; 6707 write_seqcount_end(&conf->gen_lock); 6708 spin_unlock_irq(&conf->device_lock); 6709 6710 /* Now make sure any requests that proceeded on the assumption 6711 * the reshape wasn't running - like Discard or Read - have 6712 * completed. 6713 */ 6714 mddev_suspend(mddev); 6715 mddev_resume(mddev); 6716 6717 /* Add some new drives, as many as will fit. 6718 * We know there are enough to make the newly sized array work. 6719 * Don't add devices if we are reducing the number of 6720 * devices in the array. This is because it is not possible 6721 * to correctly record the "partially reconstructed" state of 6722 * such devices during the reshape and confusion could result. 6723 */ 6724 if (mddev->delta_disks >= 0) { 6725 rdev_for_each(rdev, mddev) 6726 if (rdev->raid_disk < 0 && 6727 !test_bit(Faulty, &rdev->flags)) { 6728 if (raid5_add_disk(mddev, rdev) == 0) { 6729 if (rdev->raid_disk 6730 >= conf->previous_raid_disks) 6731 set_bit(In_sync, &rdev->flags); 6732 else 6733 rdev->recovery_offset = 0; 6734 6735 if (sysfs_link_rdev(mddev, rdev)) 6736 /* Failure here is OK */; 6737 } 6738 } else if (rdev->raid_disk >= conf->previous_raid_disks 6739 && !test_bit(Faulty, &rdev->flags)) { 6740 /* This is a spare that was manually added */ 6741 set_bit(In_sync, &rdev->flags); 6742 } 6743 6744 /* When a reshape changes the number of devices, 6745 * ->degraded is measured against the larger of the 6746 * pre and post number of devices. 6747 */ 6748 spin_lock_irqsave(&conf->device_lock, flags); 6749 mddev->degraded = calc_degraded(conf); 6750 spin_unlock_irqrestore(&conf->device_lock, flags); 6751 } 6752 mddev->raid_disks = conf->raid_disks; 6753 mddev->reshape_position = conf->reshape_progress; 6754 set_bit(MD_CHANGE_DEVS, &mddev->flags); 6755 6756 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6757 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6758 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6759 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6760 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6761 "reshape"); 6762 if (!mddev->sync_thread) { 6763 mddev->recovery = 0; 6764 spin_lock_irq(&conf->device_lock); 6765 write_seqcount_begin(&conf->gen_lock); 6766 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 6767 mddev->new_chunk_sectors = 6768 conf->chunk_sectors = conf->prev_chunk_sectors; 6769 mddev->new_layout = conf->algorithm = conf->prev_algo; 6770 rdev_for_each(rdev, mddev) 6771 rdev->new_data_offset = rdev->data_offset; 6772 smp_wmb(); 6773 conf->generation --; 6774 conf->reshape_progress = MaxSector; 6775 mddev->reshape_position = MaxSector; 6776 write_seqcount_end(&conf->gen_lock); 6777 spin_unlock_irq(&conf->device_lock); 6778 return -EAGAIN; 6779 } 6780 conf->reshape_checkpoint = jiffies; 6781 md_wakeup_thread(mddev->sync_thread); 6782 md_new_event(mddev); 6783 return 0; 6784 } 6785 6786 /* This is called from the reshape thread and should make any 6787 * changes needed in 'conf' 6788 */ 6789 static void end_reshape(struct r5conf *conf) 6790 { 6791 6792 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 6793 struct md_rdev *rdev; 6794 6795 spin_lock_irq(&conf->device_lock); 6796 conf->previous_raid_disks = conf->raid_disks; 6797 rdev_for_each(rdev, conf->mddev) 6798 rdev->data_offset = rdev->new_data_offset; 6799 smp_wmb(); 6800 conf->reshape_progress = MaxSector; 6801 spin_unlock_irq(&conf->device_lock); 6802 wake_up(&conf->wait_for_overlap); 6803 6804 /* read-ahead size must cover two whole stripes, which is 6805 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 6806 */ 6807 if (conf->mddev->queue) { 6808 int data_disks = conf->raid_disks - conf->max_degraded; 6809 int stripe = data_disks * ((conf->chunk_sectors << 9) 6810 / PAGE_SIZE); 6811 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6812 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6813 } 6814 } 6815 } 6816 6817 /* This is called from the raid5d thread with mddev_lock held. 6818 * It makes config changes to the device. 6819 */ 6820 static void raid5_finish_reshape(struct mddev *mddev) 6821 { 6822 struct r5conf *conf = mddev->private; 6823 6824 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 6825 6826 if (mddev->delta_disks > 0) { 6827 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6828 set_capacity(mddev->gendisk, mddev->array_sectors); 6829 revalidate_disk(mddev->gendisk); 6830 } else { 6831 int d; 6832 spin_lock_irq(&conf->device_lock); 6833 mddev->degraded = calc_degraded(conf); 6834 spin_unlock_irq(&conf->device_lock); 6835 for (d = conf->raid_disks ; 6836 d < conf->raid_disks - mddev->delta_disks; 6837 d++) { 6838 struct md_rdev *rdev = conf->disks[d].rdev; 6839 if (rdev) 6840 clear_bit(In_sync, &rdev->flags); 6841 rdev = conf->disks[d].replacement; 6842 if (rdev) 6843 clear_bit(In_sync, &rdev->flags); 6844 } 6845 } 6846 mddev->layout = conf->algorithm; 6847 mddev->chunk_sectors = conf->chunk_sectors; 6848 mddev->reshape_position = MaxSector; 6849 mddev->delta_disks = 0; 6850 mddev->reshape_backwards = 0; 6851 } 6852 } 6853 6854 static void raid5_quiesce(struct mddev *mddev, int state) 6855 { 6856 struct r5conf *conf = mddev->private; 6857 6858 switch(state) { 6859 case 2: /* resume for a suspend */ 6860 wake_up(&conf->wait_for_overlap); 6861 break; 6862 6863 case 1: /* stop all writes */ 6864 lock_all_device_hash_locks_irq(conf); 6865 /* '2' tells resync/reshape to pause so that all 6866 * active stripes can drain 6867 */ 6868 conf->quiesce = 2; 6869 wait_event_cmd(conf->wait_for_stripe, 6870 atomic_read(&conf->active_stripes) == 0 && 6871 atomic_read(&conf->active_aligned_reads) == 0, 6872 unlock_all_device_hash_locks_irq(conf), 6873 lock_all_device_hash_locks_irq(conf)); 6874 conf->quiesce = 1; 6875 unlock_all_device_hash_locks_irq(conf); 6876 /* allow reshape to continue */ 6877 wake_up(&conf->wait_for_overlap); 6878 break; 6879 6880 case 0: /* re-enable writes */ 6881 lock_all_device_hash_locks_irq(conf); 6882 conf->quiesce = 0; 6883 wake_up(&conf->wait_for_stripe); 6884 wake_up(&conf->wait_for_overlap); 6885 unlock_all_device_hash_locks_irq(conf); 6886 break; 6887 } 6888 } 6889 6890 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 6891 { 6892 struct r0conf *raid0_conf = mddev->private; 6893 sector_t sectors; 6894 6895 /* for raid0 takeover only one zone is supported */ 6896 if (raid0_conf->nr_strip_zones > 1) { 6897 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 6898 mdname(mddev)); 6899 return ERR_PTR(-EINVAL); 6900 } 6901 6902 sectors = raid0_conf->strip_zone[0].zone_end; 6903 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 6904 mddev->dev_sectors = sectors; 6905 mddev->new_level = level; 6906 mddev->new_layout = ALGORITHM_PARITY_N; 6907 mddev->new_chunk_sectors = mddev->chunk_sectors; 6908 mddev->raid_disks += 1; 6909 mddev->delta_disks = 1; 6910 /* make sure it will be not marked as dirty */ 6911 mddev->recovery_cp = MaxSector; 6912 6913 return setup_conf(mddev); 6914 } 6915 6916 static void *raid5_takeover_raid1(struct mddev *mddev) 6917 { 6918 int chunksect; 6919 6920 if (mddev->raid_disks != 2 || 6921 mddev->degraded > 1) 6922 return ERR_PTR(-EINVAL); 6923 6924 /* Should check if there are write-behind devices? */ 6925 6926 chunksect = 64*2; /* 64K by default */ 6927 6928 /* The array must be an exact multiple of chunksize */ 6929 while (chunksect && (mddev->array_sectors & (chunksect-1))) 6930 chunksect >>= 1; 6931 6932 if ((chunksect<<9) < STRIPE_SIZE) 6933 /* array size does not allow a suitable chunk size */ 6934 return ERR_PTR(-EINVAL); 6935 6936 mddev->new_level = 5; 6937 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 6938 mddev->new_chunk_sectors = chunksect; 6939 6940 return setup_conf(mddev); 6941 } 6942 6943 static void *raid5_takeover_raid6(struct mddev *mddev) 6944 { 6945 int new_layout; 6946 6947 switch (mddev->layout) { 6948 case ALGORITHM_LEFT_ASYMMETRIC_6: 6949 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 6950 break; 6951 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6952 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 6953 break; 6954 case ALGORITHM_LEFT_SYMMETRIC_6: 6955 new_layout = ALGORITHM_LEFT_SYMMETRIC; 6956 break; 6957 case ALGORITHM_RIGHT_SYMMETRIC_6: 6958 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 6959 break; 6960 case ALGORITHM_PARITY_0_6: 6961 new_layout = ALGORITHM_PARITY_0; 6962 break; 6963 case ALGORITHM_PARITY_N: 6964 new_layout = ALGORITHM_PARITY_N; 6965 break; 6966 default: 6967 return ERR_PTR(-EINVAL); 6968 } 6969 mddev->new_level = 5; 6970 mddev->new_layout = new_layout; 6971 mddev->delta_disks = -1; 6972 mddev->raid_disks -= 1; 6973 return setup_conf(mddev); 6974 } 6975 6976 static int raid5_check_reshape(struct mddev *mddev) 6977 { 6978 /* For a 2-drive array, the layout and chunk size can be changed 6979 * immediately as not restriping is needed. 6980 * For larger arrays we record the new value - after validation 6981 * to be used by a reshape pass. 6982 */ 6983 struct r5conf *conf = mddev->private; 6984 int new_chunk = mddev->new_chunk_sectors; 6985 6986 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 6987 return -EINVAL; 6988 if (new_chunk > 0) { 6989 if (!is_power_of_2(new_chunk)) 6990 return -EINVAL; 6991 if (new_chunk < (PAGE_SIZE>>9)) 6992 return -EINVAL; 6993 if (mddev->array_sectors & (new_chunk-1)) 6994 /* not factor of array size */ 6995 return -EINVAL; 6996 } 6997 6998 /* They look valid */ 6999 7000 if (mddev->raid_disks == 2) { 7001 /* can make the change immediately */ 7002 if (mddev->new_layout >= 0) { 7003 conf->algorithm = mddev->new_layout; 7004 mddev->layout = mddev->new_layout; 7005 } 7006 if (new_chunk > 0) { 7007 conf->chunk_sectors = new_chunk ; 7008 mddev->chunk_sectors = new_chunk; 7009 } 7010 set_bit(MD_CHANGE_DEVS, &mddev->flags); 7011 md_wakeup_thread(mddev->thread); 7012 } 7013 return check_reshape(mddev); 7014 } 7015 7016 static int raid6_check_reshape(struct mddev *mddev) 7017 { 7018 int new_chunk = mddev->new_chunk_sectors; 7019 7020 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 7021 return -EINVAL; 7022 if (new_chunk > 0) { 7023 if (!is_power_of_2(new_chunk)) 7024 return -EINVAL; 7025 if (new_chunk < (PAGE_SIZE >> 9)) 7026 return -EINVAL; 7027 if (mddev->array_sectors & (new_chunk-1)) 7028 /* not factor of array size */ 7029 return -EINVAL; 7030 } 7031 7032 /* They look valid */ 7033 return check_reshape(mddev); 7034 } 7035 7036 static void *raid5_takeover(struct mddev *mddev) 7037 { 7038 /* raid5 can take over: 7039 * raid0 - if there is only one strip zone - make it a raid4 layout 7040 * raid1 - if there are two drives. We need to know the chunk size 7041 * raid4 - trivial - just use a raid4 layout. 7042 * raid6 - Providing it is a *_6 layout 7043 */ 7044 if (mddev->level == 0) 7045 return raid45_takeover_raid0(mddev, 5); 7046 if (mddev->level == 1) 7047 return raid5_takeover_raid1(mddev); 7048 if (mddev->level == 4) { 7049 mddev->new_layout = ALGORITHM_PARITY_N; 7050 mddev->new_level = 5; 7051 return setup_conf(mddev); 7052 } 7053 if (mddev->level == 6) 7054 return raid5_takeover_raid6(mddev); 7055 7056 return ERR_PTR(-EINVAL); 7057 } 7058 7059 static void *raid4_takeover(struct mddev *mddev) 7060 { 7061 /* raid4 can take over: 7062 * raid0 - if there is only one strip zone 7063 * raid5 - if layout is right 7064 */ 7065 if (mddev->level == 0) 7066 return raid45_takeover_raid0(mddev, 4); 7067 if (mddev->level == 5 && 7068 mddev->layout == ALGORITHM_PARITY_N) { 7069 mddev->new_layout = 0; 7070 mddev->new_level = 4; 7071 return setup_conf(mddev); 7072 } 7073 return ERR_PTR(-EINVAL); 7074 } 7075 7076 static struct md_personality raid5_personality; 7077 7078 static void *raid6_takeover(struct mddev *mddev) 7079 { 7080 /* Currently can only take over a raid5. We map the 7081 * personality to an equivalent raid6 personality 7082 * with the Q block at the end. 7083 */ 7084 int new_layout; 7085 7086 if (mddev->pers != &raid5_personality) 7087 return ERR_PTR(-EINVAL); 7088 if (mddev->degraded > 1) 7089 return ERR_PTR(-EINVAL); 7090 if (mddev->raid_disks > 253) 7091 return ERR_PTR(-EINVAL); 7092 if (mddev->raid_disks < 3) 7093 return ERR_PTR(-EINVAL); 7094 7095 switch (mddev->layout) { 7096 case ALGORITHM_LEFT_ASYMMETRIC: 7097 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 7098 break; 7099 case ALGORITHM_RIGHT_ASYMMETRIC: 7100 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 7101 break; 7102 case ALGORITHM_LEFT_SYMMETRIC: 7103 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 7104 break; 7105 case ALGORITHM_RIGHT_SYMMETRIC: 7106 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 7107 break; 7108 case ALGORITHM_PARITY_0: 7109 new_layout = ALGORITHM_PARITY_0_6; 7110 break; 7111 case ALGORITHM_PARITY_N: 7112 new_layout = ALGORITHM_PARITY_N; 7113 break; 7114 default: 7115 return ERR_PTR(-EINVAL); 7116 } 7117 mddev->new_level = 6; 7118 mddev->new_layout = new_layout; 7119 mddev->delta_disks = 1; 7120 mddev->raid_disks += 1; 7121 return setup_conf(mddev); 7122 } 7123 7124 static struct md_personality raid6_personality = 7125 { 7126 .name = "raid6", 7127 .level = 6, 7128 .owner = THIS_MODULE, 7129 .make_request = make_request, 7130 .run = run, 7131 .free = raid5_free, 7132 .status = status, 7133 .error_handler = error, 7134 .hot_add_disk = raid5_add_disk, 7135 .hot_remove_disk= raid5_remove_disk, 7136 .spare_active = raid5_spare_active, 7137 .sync_request = sync_request, 7138 .resize = raid5_resize, 7139 .size = raid5_size, 7140 .check_reshape = raid6_check_reshape, 7141 .start_reshape = raid5_start_reshape, 7142 .finish_reshape = raid5_finish_reshape, 7143 .quiesce = raid5_quiesce, 7144 .takeover = raid6_takeover, 7145 .congested = raid5_congested, 7146 .mergeable_bvec = raid5_mergeable_bvec, 7147 }; 7148 static struct md_personality raid5_personality = 7149 { 7150 .name = "raid5", 7151 .level = 5, 7152 .owner = THIS_MODULE, 7153 .make_request = make_request, 7154 .run = run, 7155 .free = raid5_free, 7156 .status = status, 7157 .error_handler = error, 7158 .hot_add_disk = raid5_add_disk, 7159 .hot_remove_disk= raid5_remove_disk, 7160 .spare_active = raid5_spare_active, 7161 .sync_request = sync_request, 7162 .resize = raid5_resize, 7163 .size = raid5_size, 7164 .check_reshape = raid5_check_reshape, 7165 .start_reshape = raid5_start_reshape, 7166 .finish_reshape = raid5_finish_reshape, 7167 .quiesce = raid5_quiesce, 7168 .takeover = raid5_takeover, 7169 .congested = raid5_congested, 7170 .mergeable_bvec = raid5_mergeable_bvec, 7171 }; 7172 7173 static struct md_personality raid4_personality = 7174 { 7175 .name = "raid4", 7176 .level = 4, 7177 .owner = THIS_MODULE, 7178 .make_request = make_request, 7179 .run = run, 7180 .free = raid5_free, 7181 .status = status, 7182 .error_handler = error, 7183 .hot_add_disk = raid5_add_disk, 7184 .hot_remove_disk= raid5_remove_disk, 7185 .spare_active = raid5_spare_active, 7186 .sync_request = sync_request, 7187 .resize = raid5_resize, 7188 .size = raid5_size, 7189 .check_reshape = raid5_check_reshape, 7190 .start_reshape = raid5_start_reshape, 7191 .finish_reshape = raid5_finish_reshape, 7192 .quiesce = raid5_quiesce, 7193 .takeover = raid4_takeover, 7194 .congested = raid5_congested, 7195 .mergeable_bvec = raid5_mergeable_bvec, 7196 }; 7197 7198 static int __init raid5_init(void) 7199 { 7200 raid5_wq = alloc_workqueue("raid5wq", 7201 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 7202 if (!raid5_wq) 7203 return -ENOMEM; 7204 register_md_personality(&raid6_personality); 7205 register_md_personality(&raid5_personality); 7206 register_md_personality(&raid4_personality); 7207 return 0; 7208 } 7209 7210 static void raid5_exit(void) 7211 { 7212 unregister_md_personality(&raid6_personality); 7213 unregister_md_personality(&raid5_personality); 7214 unregister_md_personality(&raid4_personality); 7215 destroy_workqueue(raid5_wq); 7216 } 7217 7218 module_init(raid5_init); 7219 module_exit(raid5_exit); 7220 MODULE_LICENSE("GPL"); 7221 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 7222 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 7223 MODULE_ALIAS("md-raid5"); 7224 MODULE_ALIAS("md-raid4"); 7225 MODULE_ALIAS("md-level-5"); 7226 MODULE_ALIAS("md-level-4"); 7227 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 7228 MODULE_ALIAS("md-raid6"); 7229 MODULE_ALIAS("md-level-6"); 7230 7231 /* This used to be two separate modules, they were: */ 7232 MODULE_ALIAS("raid5"); 7233 MODULE_ALIAS("raid6"); 7234