1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include <linux/nodemask.h> 57 #include <linux/flex_array.h> 58 #include <trace/events/block.h> 59 60 #include "md.h" 61 #include "raid5.h" 62 #include "raid0.h" 63 #include "bitmap.h" 64 65 #define cpu_to_group(cpu) cpu_to_node(cpu) 66 #define ANY_GROUP NUMA_NO_NODE 67 68 static bool devices_handle_discard_safely = false; 69 module_param(devices_handle_discard_safely, bool, 0644); 70 MODULE_PARM_DESC(devices_handle_discard_safely, 71 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); 72 static struct workqueue_struct *raid5_wq; 73 /* 74 * Stripe cache 75 */ 76 77 #define NR_STRIPES 256 78 #define STRIPE_SIZE PAGE_SIZE 79 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 80 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 81 #define IO_THRESHOLD 1 82 #define BYPASS_THRESHOLD 1 83 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 84 #define HASH_MASK (NR_HASH - 1) 85 #define MAX_STRIPE_BATCH 8 86 87 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 88 { 89 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 90 return &conf->stripe_hashtbl[hash]; 91 } 92 93 static inline int stripe_hash_locks_hash(sector_t sect) 94 { 95 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; 96 } 97 98 static inline void lock_device_hash_lock(struct r5conf *conf, int hash) 99 { 100 spin_lock_irq(conf->hash_locks + hash); 101 spin_lock(&conf->device_lock); 102 } 103 104 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) 105 { 106 spin_unlock(&conf->device_lock); 107 spin_unlock_irq(conf->hash_locks + hash); 108 } 109 110 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) 111 { 112 int i; 113 local_irq_disable(); 114 spin_lock(conf->hash_locks); 115 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 116 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); 117 spin_lock(&conf->device_lock); 118 } 119 120 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) 121 { 122 int i; 123 spin_unlock(&conf->device_lock); 124 for (i = NR_STRIPE_HASH_LOCKS; i; i--) 125 spin_unlock(conf->hash_locks + i - 1); 126 local_irq_enable(); 127 } 128 129 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 130 * order without overlap. There may be several bio's per stripe+device, and 131 * a bio could span several devices. 132 * When walking this list for a particular stripe+device, we must never proceed 133 * beyond a bio that extends past this device, as the next bio might no longer 134 * be valid. 135 * This function is used to determine the 'next' bio in the list, given the sector 136 * of the current stripe+device 137 */ 138 static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) 139 { 140 int sectors = bio_sectors(bio); 141 if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) 142 return bio->bi_next; 143 else 144 return NULL; 145 } 146 147 /* 148 * We maintain a biased count of active stripes in the bottom 16 bits of 149 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 150 */ 151 static inline int raid5_bi_processed_stripes(struct bio *bio) 152 { 153 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 154 return (atomic_read(segments) >> 16) & 0xffff; 155 } 156 157 static inline int raid5_dec_bi_active_stripes(struct bio *bio) 158 { 159 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 160 return atomic_sub_return(1, segments) & 0xffff; 161 } 162 163 static inline void raid5_inc_bi_active_stripes(struct bio *bio) 164 { 165 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 166 atomic_inc(segments); 167 } 168 169 static inline void raid5_set_bi_processed_stripes(struct bio *bio, 170 unsigned int cnt) 171 { 172 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 173 int old, new; 174 175 do { 176 old = atomic_read(segments); 177 new = (old & 0xffff) | (cnt << 16); 178 } while (atomic_cmpxchg(segments, old, new) != old); 179 } 180 181 static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) 182 { 183 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 184 atomic_set(segments, cnt); 185 } 186 187 /* Find first data disk in a raid6 stripe */ 188 static inline int raid6_d0(struct stripe_head *sh) 189 { 190 if (sh->ddf_layout) 191 /* ddf always start from first device */ 192 return 0; 193 /* md starts just after Q block */ 194 if (sh->qd_idx == sh->disks - 1) 195 return 0; 196 else 197 return sh->qd_idx + 1; 198 } 199 static inline int raid6_next_disk(int disk, int raid_disks) 200 { 201 disk++; 202 return (disk < raid_disks) ? disk : 0; 203 } 204 205 /* When walking through the disks in a raid5, starting at raid6_d0, 206 * We need to map each disk to a 'slot', where the data disks are slot 207 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 208 * is raid_disks-1. This help does that mapping. 209 */ 210 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 211 int *count, int syndrome_disks) 212 { 213 int slot = *count; 214 215 if (sh->ddf_layout) 216 (*count)++; 217 if (idx == sh->pd_idx) 218 return syndrome_disks; 219 if (idx == sh->qd_idx) 220 return syndrome_disks + 1; 221 if (!sh->ddf_layout) 222 (*count)++; 223 return slot; 224 } 225 226 static void return_io(struct bio *return_bi) 227 { 228 struct bio *bi = return_bi; 229 while (bi) { 230 231 return_bi = bi->bi_next; 232 bi->bi_next = NULL; 233 bi->bi_iter.bi_size = 0; 234 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 235 bi, 0); 236 bio_endio(bi, 0); 237 bi = return_bi; 238 } 239 } 240 241 static void print_raid5_conf (struct r5conf *conf); 242 243 static int stripe_operations_active(struct stripe_head *sh) 244 { 245 return sh->check_state || sh->reconstruct_state || 246 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 247 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 248 } 249 250 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) 251 { 252 struct r5conf *conf = sh->raid_conf; 253 struct r5worker_group *group; 254 int thread_cnt; 255 int i, cpu = sh->cpu; 256 257 if (!cpu_online(cpu)) { 258 cpu = cpumask_any(cpu_online_mask); 259 sh->cpu = cpu; 260 } 261 262 if (list_empty(&sh->lru)) { 263 struct r5worker_group *group; 264 group = conf->worker_groups + cpu_to_group(cpu); 265 list_add_tail(&sh->lru, &group->handle_list); 266 group->stripes_cnt++; 267 sh->group = group; 268 } 269 270 if (conf->worker_cnt_per_group == 0) { 271 md_wakeup_thread(conf->mddev->thread); 272 return; 273 } 274 275 group = conf->worker_groups + cpu_to_group(sh->cpu); 276 277 group->workers[0].working = true; 278 /* at least one worker should run to avoid race */ 279 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); 280 281 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; 282 /* wakeup more workers */ 283 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { 284 if (group->workers[i].working == false) { 285 group->workers[i].working = true; 286 queue_work_on(sh->cpu, raid5_wq, 287 &group->workers[i].work); 288 thread_cnt--; 289 } 290 } 291 } 292 293 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, 294 struct list_head *temp_inactive_list) 295 { 296 BUG_ON(!list_empty(&sh->lru)); 297 BUG_ON(atomic_read(&conf->active_stripes)==0); 298 if (test_bit(STRIPE_HANDLE, &sh->state)) { 299 if (test_bit(STRIPE_DELAYED, &sh->state) && 300 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 301 list_add_tail(&sh->lru, &conf->delayed_list); 302 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 303 sh->bm_seq - conf->seq_write > 0) 304 list_add_tail(&sh->lru, &conf->bitmap_list); 305 else { 306 clear_bit(STRIPE_DELAYED, &sh->state); 307 clear_bit(STRIPE_BIT_DELAY, &sh->state); 308 if (conf->worker_cnt_per_group == 0) { 309 list_add_tail(&sh->lru, &conf->handle_list); 310 } else { 311 raid5_wakeup_stripe_thread(sh); 312 return; 313 } 314 } 315 md_wakeup_thread(conf->mddev->thread); 316 } else { 317 BUG_ON(stripe_operations_active(sh)); 318 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 319 if (atomic_dec_return(&conf->preread_active_stripes) 320 < IO_THRESHOLD) 321 md_wakeup_thread(conf->mddev->thread); 322 atomic_dec(&conf->active_stripes); 323 if (!test_bit(STRIPE_EXPANDING, &sh->state)) 324 list_add_tail(&sh->lru, temp_inactive_list); 325 } 326 } 327 328 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, 329 struct list_head *temp_inactive_list) 330 { 331 if (atomic_dec_and_test(&sh->count)) 332 do_release_stripe(conf, sh, temp_inactive_list); 333 } 334 335 /* 336 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list 337 * 338 * Be careful: Only one task can add/delete stripes from temp_inactive_list at 339 * given time. Adding stripes only takes device lock, while deleting stripes 340 * only takes hash lock. 341 */ 342 static void release_inactive_stripe_list(struct r5conf *conf, 343 struct list_head *temp_inactive_list, 344 int hash) 345 { 346 int size; 347 bool do_wakeup = false; 348 unsigned long flags; 349 350 if (hash == NR_STRIPE_HASH_LOCKS) { 351 size = NR_STRIPE_HASH_LOCKS; 352 hash = NR_STRIPE_HASH_LOCKS - 1; 353 } else 354 size = 1; 355 while (size) { 356 struct list_head *list = &temp_inactive_list[size - 1]; 357 358 /* 359 * We don't hold any lock here yet, get_active_stripe() might 360 * remove stripes from the list 361 */ 362 if (!list_empty_careful(list)) { 363 spin_lock_irqsave(conf->hash_locks + hash, flags); 364 if (list_empty(conf->inactive_list + hash) && 365 !list_empty(list)) 366 atomic_dec(&conf->empty_inactive_list_nr); 367 list_splice_tail_init(list, conf->inactive_list + hash); 368 do_wakeup = true; 369 spin_unlock_irqrestore(conf->hash_locks + hash, flags); 370 } 371 size--; 372 hash--; 373 } 374 375 if (do_wakeup) { 376 wake_up(&conf->wait_for_stripe); 377 if (conf->retry_read_aligned) 378 md_wakeup_thread(conf->mddev->thread); 379 } 380 } 381 382 /* should hold conf->device_lock already */ 383 static int release_stripe_list(struct r5conf *conf, 384 struct list_head *temp_inactive_list) 385 { 386 struct stripe_head *sh; 387 int count = 0; 388 struct llist_node *head; 389 390 head = llist_del_all(&conf->released_stripes); 391 head = llist_reverse_order(head); 392 while (head) { 393 int hash; 394 395 sh = llist_entry(head, struct stripe_head, release_list); 396 head = llist_next(head); 397 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ 398 smp_mb(); 399 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); 400 /* 401 * Don't worry the bit is set here, because if the bit is set 402 * again, the count is always > 1. This is true for 403 * STRIPE_ON_UNPLUG_LIST bit too. 404 */ 405 hash = sh->hash_lock_index; 406 __release_stripe(conf, sh, &temp_inactive_list[hash]); 407 count++; 408 } 409 410 return count; 411 } 412 413 static void release_stripe(struct stripe_head *sh) 414 { 415 struct r5conf *conf = sh->raid_conf; 416 unsigned long flags; 417 struct list_head list; 418 int hash; 419 bool wakeup; 420 421 /* Avoid release_list until the last reference. 422 */ 423 if (atomic_add_unless(&sh->count, -1, 1)) 424 return; 425 426 if (unlikely(!conf->mddev->thread) || 427 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) 428 goto slow_path; 429 wakeup = llist_add(&sh->release_list, &conf->released_stripes); 430 if (wakeup) 431 md_wakeup_thread(conf->mddev->thread); 432 return; 433 slow_path: 434 local_irq_save(flags); 435 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ 436 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { 437 INIT_LIST_HEAD(&list); 438 hash = sh->hash_lock_index; 439 do_release_stripe(conf, sh, &list); 440 spin_unlock(&conf->device_lock); 441 release_inactive_stripe_list(conf, &list, hash); 442 } 443 local_irq_restore(flags); 444 } 445 446 static inline void remove_hash(struct stripe_head *sh) 447 { 448 pr_debug("remove_hash(), stripe %llu\n", 449 (unsigned long long)sh->sector); 450 451 hlist_del_init(&sh->hash); 452 } 453 454 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 455 { 456 struct hlist_head *hp = stripe_hash(conf, sh->sector); 457 458 pr_debug("insert_hash(), stripe %llu\n", 459 (unsigned long long)sh->sector); 460 461 hlist_add_head(&sh->hash, hp); 462 } 463 464 /* find an idle stripe, make sure it is unhashed, and return it. */ 465 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) 466 { 467 struct stripe_head *sh = NULL; 468 struct list_head *first; 469 470 if (list_empty(conf->inactive_list + hash)) 471 goto out; 472 first = (conf->inactive_list + hash)->next; 473 sh = list_entry(first, struct stripe_head, lru); 474 list_del_init(first); 475 remove_hash(sh); 476 atomic_inc(&conf->active_stripes); 477 BUG_ON(hash != sh->hash_lock_index); 478 if (list_empty(conf->inactive_list + hash)) 479 atomic_inc(&conf->empty_inactive_list_nr); 480 out: 481 return sh; 482 } 483 484 static void shrink_buffers(struct stripe_head *sh) 485 { 486 struct page *p; 487 int i; 488 int num = sh->raid_conf->pool_size; 489 490 for (i = 0; i < num ; i++) { 491 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); 492 p = sh->dev[i].page; 493 if (!p) 494 continue; 495 sh->dev[i].page = NULL; 496 put_page(p); 497 } 498 } 499 500 static int grow_buffers(struct stripe_head *sh, gfp_t gfp) 501 { 502 int i; 503 int num = sh->raid_conf->pool_size; 504 505 for (i = 0; i < num; i++) { 506 struct page *page; 507 508 if (!(page = alloc_page(gfp))) { 509 return 1; 510 } 511 sh->dev[i].page = page; 512 sh->dev[i].orig_page = page; 513 } 514 return 0; 515 } 516 517 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 518 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 519 struct stripe_head *sh); 520 521 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 522 { 523 struct r5conf *conf = sh->raid_conf; 524 int i, seq; 525 526 BUG_ON(atomic_read(&sh->count) != 0); 527 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 528 BUG_ON(stripe_operations_active(sh)); 529 BUG_ON(sh->batch_head); 530 531 pr_debug("init_stripe called, stripe %llu\n", 532 (unsigned long long)sector); 533 retry: 534 seq = read_seqcount_begin(&conf->gen_lock); 535 sh->generation = conf->generation - previous; 536 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 537 sh->sector = sector; 538 stripe_set_idx(sector, conf, previous, sh); 539 sh->state = 0; 540 541 for (i = sh->disks; i--; ) { 542 struct r5dev *dev = &sh->dev[i]; 543 544 if (dev->toread || dev->read || dev->towrite || dev->written || 545 test_bit(R5_LOCKED, &dev->flags)) { 546 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 547 (unsigned long long)sh->sector, i, dev->toread, 548 dev->read, dev->towrite, dev->written, 549 test_bit(R5_LOCKED, &dev->flags)); 550 WARN_ON(1); 551 } 552 dev->flags = 0; 553 raid5_build_block(sh, i, previous); 554 } 555 if (read_seqcount_retry(&conf->gen_lock, seq)) 556 goto retry; 557 sh->overwrite_disks = 0; 558 insert_hash(conf, sh); 559 sh->cpu = smp_processor_id(); 560 set_bit(STRIPE_BATCH_READY, &sh->state); 561 } 562 563 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 564 short generation) 565 { 566 struct stripe_head *sh; 567 568 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 569 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) 570 if (sh->sector == sector && sh->generation == generation) 571 return sh; 572 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 573 return NULL; 574 } 575 576 /* 577 * Need to check if array has failed when deciding whether to: 578 * - start an array 579 * - remove non-faulty devices 580 * - add a spare 581 * - allow a reshape 582 * This determination is simple when no reshape is happening. 583 * However if there is a reshape, we need to carefully check 584 * both the before and after sections. 585 * This is because some failed devices may only affect one 586 * of the two sections, and some non-in_sync devices may 587 * be insync in the section most affected by failed devices. 588 */ 589 static int calc_degraded(struct r5conf *conf) 590 { 591 int degraded, degraded2; 592 int i; 593 594 rcu_read_lock(); 595 degraded = 0; 596 for (i = 0; i < conf->previous_raid_disks; i++) { 597 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 598 if (rdev && test_bit(Faulty, &rdev->flags)) 599 rdev = rcu_dereference(conf->disks[i].replacement); 600 if (!rdev || test_bit(Faulty, &rdev->flags)) 601 degraded++; 602 else if (test_bit(In_sync, &rdev->flags)) 603 ; 604 else 605 /* not in-sync or faulty. 606 * If the reshape increases the number of devices, 607 * this is being recovered by the reshape, so 608 * this 'previous' section is not in_sync. 609 * If the number of devices is being reduced however, 610 * the device can only be part of the array if 611 * we are reverting a reshape, so this section will 612 * be in-sync. 613 */ 614 if (conf->raid_disks >= conf->previous_raid_disks) 615 degraded++; 616 } 617 rcu_read_unlock(); 618 if (conf->raid_disks == conf->previous_raid_disks) 619 return degraded; 620 rcu_read_lock(); 621 degraded2 = 0; 622 for (i = 0; i < conf->raid_disks; i++) { 623 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 624 if (rdev && test_bit(Faulty, &rdev->flags)) 625 rdev = rcu_dereference(conf->disks[i].replacement); 626 if (!rdev || test_bit(Faulty, &rdev->flags)) 627 degraded2++; 628 else if (test_bit(In_sync, &rdev->flags)) 629 ; 630 else 631 /* not in-sync or faulty. 632 * If reshape increases the number of devices, this 633 * section has already been recovered, else it 634 * almost certainly hasn't. 635 */ 636 if (conf->raid_disks <= conf->previous_raid_disks) 637 degraded2++; 638 } 639 rcu_read_unlock(); 640 if (degraded2 > degraded) 641 return degraded2; 642 return degraded; 643 } 644 645 static int has_failed(struct r5conf *conf) 646 { 647 int degraded; 648 649 if (conf->mddev->reshape_position == MaxSector) 650 return conf->mddev->degraded > conf->max_degraded; 651 652 degraded = calc_degraded(conf); 653 if (degraded > conf->max_degraded) 654 return 1; 655 return 0; 656 } 657 658 static struct stripe_head * 659 get_active_stripe(struct r5conf *conf, sector_t sector, 660 int previous, int noblock, int noquiesce) 661 { 662 struct stripe_head *sh; 663 int hash = stripe_hash_locks_hash(sector); 664 665 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 666 667 spin_lock_irq(conf->hash_locks + hash); 668 669 do { 670 wait_event_lock_irq(conf->wait_for_stripe, 671 conf->quiesce == 0 || noquiesce, 672 *(conf->hash_locks + hash)); 673 sh = __find_stripe(conf, sector, conf->generation - previous); 674 if (!sh) { 675 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { 676 sh = get_free_stripe(conf, hash); 677 if (!sh && llist_empty(&conf->released_stripes) && 678 !test_bit(R5_DID_ALLOC, &conf->cache_state)) 679 set_bit(R5_ALLOC_MORE, 680 &conf->cache_state); 681 } 682 if (noblock && sh == NULL) 683 break; 684 if (!sh) { 685 set_bit(R5_INACTIVE_BLOCKED, 686 &conf->cache_state); 687 wait_event_lock_irq( 688 conf->wait_for_stripe, 689 !list_empty(conf->inactive_list + hash) && 690 (atomic_read(&conf->active_stripes) 691 < (conf->max_nr_stripes * 3 / 4) 692 || !test_bit(R5_INACTIVE_BLOCKED, 693 &conf->cache_state)), 694 *(conf->hash_locks + hash)); 695 clear_bit(R5_INACTIVE_BLOCKED, 696 &conf->cache_state); 697 } else { 698 init_stripe(sh, sector, previous); 699 atomic_inc(&sh->count); 700 } 701 } else if (!atomic_inc_not_zero(&sh->count)) { 702 spin_lock(&conf->device_lock); 703 if (!atomic_read(&sh->count)) { 704 if (!test_bit(STRIPE_HANDLE, &sh->state)) 705 atomic_inc(&conf->active_stripes); 706 BUG_ON(list_empty(&sh->lru) && 707 !test_bit(STRIPE_EXPANDING, &sh->state)); 708 list_del_init(&sh->lru); 709 if (sh->group) { 710 sh->group->stripes_cnt--; 711 sh->group = NULL; 712 } 713 } 714 atomic_inc(&sh->count); 715 spin_unlock(&conf->device_lock); 716 } 717 } while (sh == NULL); 718 719 spin_unlock_irq(conf->hash_locks + hash); 720 return sh; 721 } 722 723 static bool is_full_stripe_write(struct stripe_head *sh) 724 { 725 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); 726 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); 727 } 728 729 static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) 730 { 731 local_irq_disable(); 732 if (sh1 > sh2) { 733 spin_lock(&sh2->stripe_lock); 734 spin_lock_nested(&sh1->stripe_lock, 1); 735 } else { 736 spin_lock(&sh1->stripe_lock); 737 spin_lock_nested(&sh2->stripe_lock, 1); 738 } 739 } 740 741 static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) 742 { 743 spin_unlock(&sh1->stripe_lock); 744 spin_unlock(&sh2->stripe_lock); 745 local_irq_enable(); 746 } 747 748 /* Only freshly new full stripe normal write stripe can be added to a batch list */ 749 static bool stripe_can_batch(struct stripe_head *sh) 750 { 751 return test_bit(STRIPE_BATCH_READY, &sh->state) && 752 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && 753 is_full_stripe_write(sh); 754 } 755 756 /* we only do back search */ 757 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) 758 { 759 struct stripe_head *head; 760 sector_t head_sector, tmp_sec; 761 int hash; 762 int dd_idx; 763 764 if (!stripe_can_batch(sh)) 765 return; 766 /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */ 767 tmp_sec = sh->sector; 768 if (!sector_div(tmp_sec, conf->chunk_sectors)) 769 return; 770 head_sector = sh->sector - STRIPE_SECTORS; 771 772 hash = stripe_hash_locks_hash(head_sector); 773 spin_lock_irq(conf->hash_locks + hash); 774 head = __find_stripe(conf, head_sector, conf->generation); 775 if (head && !atomic_inc_not_zero(&head->count)) { 776 spin_lock(&conf->device_lock); 777 if (!atomic_read(&head->count)) { 778 if (!test_bit(STRIPE_HANDLE, &head->state)) 779 atomic_inc(&conf->active_stripes); 780 BUG_ON(list_empty(&head->lru) && 781 !test_bit(STRIPE_EXPANDING, &head->state)); 782 list_del_init(&head->lru); 783 if (head->group) { 784 head->group->stripes_cnt--; 785 head->group = NULL; 786 } 787 } 788 atomic_inc(&head->count); 789 spin_unlock(&conf->device_lock); 790 } 791 spin_unlock_irq(conf->hash_locks + hash); 792 793 if (!head) 794 return; 795 if (!stripe_can_batch(head)) 796 goto out; 797 798 lock_two_stripes(head, sh); 799 /* clear_batch_ready clear the flag */ 800 if (!stripe_can_batch(head) || !stripe_can_batch(sh)) 801 goto unlock_out; 802 803 if (sh->batch_head) 804 goto unlock_out; 805 806 dd_idx = 0; 807 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) 808 dd_idx++; 809 if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw) 810 goto unlock_out; 811 812 if (head->batch_head) { 813 spin_lock(&head->batch_head->batch_lock); 814 /* This batch list is already running */ 815 if (!stripe_can_batch(head)) { 816 spin_unlock(&head->batch_head->batch_lock); 817 goto unlock_out; 818 } 819 820 /* 821 * at this point, head's BATCH_READY could be cleared, but we 822 * can still add the stripe to batch list 823 */ 824 list_add(&sh->batch_list, &head->batch_list); 825 spin_unlock(&head->batch_head->batch_lock); 826 827 sh->batch_head = head->batch_head; 828 } else { 829 head->batch_head = head; 830 sh->batch_head = head->batch_head; 831 spin_lock(&head->batch_lock); 832 list_add_tail(&sh->batch_list, &head->batch_list); 833 spin_unlock(&head->batch_lock); 834 } 835 836 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 837 if (atomic_dec_return(&conf->preread_active_stripes) 838 < IO_THRESHOLD) 839 md_wakeup_thread(conf->mddev->thread); 840 841 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { 842 int seq = sh->bm_seq; 843 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && 844 sh->batch_head->bm_seq > seq) 845 seq = sh->batch_head->bm_seq; 846 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); 847 sh->batch_head->bm_seq = seq; 848 } 849 850 atomic_inc(&sh->count); 851 unlock_out: 852 unlock_two_stripes(head, sh); 853 out: 854 release_stripe(head); 855 } 856 857 /* Determine if 'data_offset' or 'new_data_offset' should be used 858 * in this stripe_head. 859 */ 860 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 861 { 862 sector_t progress = conf->reshape_progress; 863 /* Need a memory barrier to make sure we see the value 864 * of conf->generation, or ->data_offset that was set before 865 * reshape_progress was updated. 866 */ 867 smp_rmb(); 868 if (progress == MaxSector) 869 return 0; 870 if (sh->generation == conf->generation - 1) 871 return 0; 872 /* We are in a reshape, and this is a new-generation stripe, 873 * so use new_data_offset. 874 */ 875 return 1; 876 } 877 878 static void 879 raid5_end_read_request(struct bio *bi, int error); 880 static void 881 raid5_end_write_request(struct bio *bi, int error); 882 883 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 884 { 885 struct r5conf *conf = sh->raid_conf; 886 int i, disks = sh->disks; 887 struct stripe_head *head_sh = sh; 888 889 might_sleep(); 890 891 for (i = disks; i--; ) { 892 int rw; 893 int replace_only = 0; 894 struct bio *bi, *rbi; 895 struct md_rdev *rdev, *rrdev = NULL; 896 897 sh = head_sh; 898 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 899 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 900 rw = WRITE_FUA; 901 else 902 rw = WRITE; 903 if (test_bit(R5_Discard, &sh->dev[i].flags)) 904 rw |= REQ_DISCARD; 905 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 906 rw = READ; 907 else if (test_and_clear_bit(R5_WantReplace, 908 &sh->dev[i].flags)) { 909 rw = WRITE; 910 replace_only = 1; 911 } else 912 continue; 913 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) 914 rw |= REQ_SYNC; 915 916 again: 917 bi = &sh->dev[i].req; 918 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 919 920 rcu_read_lock(); 921 rrdev = rcu_dereference(conf->disks[i].replacement); 922 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 923 rdev = rcu_dereference(conf->disks[i].rdev); 924 if (!rdev) { 925 rdev = rrdev; 926 rrdev = NULL; 927 } 928 if (rw & WRITE) { 929 if (replace_only) 930 rdev = NULL; 931 if (rdev == rrdev) 932 /* We raced and saw duplicates */ 933 rrdev = NULL; 934 } else { 935 if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev) 936 rdev = rrdev; 937 rrdev = NULL; 938 } 939 940 if (rdev && test_bit(Faulty, &rdev->flags)) 941 rdev = NULL; 942 if (rdev) 943 atomic_inc(&rdev->nr_pending); 944 if (rrdev && test_bit(Faulty, &rrdev->flags)) 945 rrdev = NULL; 946 if (rrdev) 947 atomic_inc(&rrdev->nr_pending); 948 rcu_read_unlock(); 949 950 /* We have already checked bad blocks for reads. Now 951 * need to check for writes. We never accept write errors 952 * on the replacement, so we don't to check rrdev. 953 */ 954 while ((rw & WRITE) && rdev && 955 test_bit(WriteErrorSeen, &rdev->flags)) { 956 sector_t first_bad; 957 int bad_sectors; 958 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 959 &first_bad, &bad_sectors); 960 if (!bad) 961 break; 962 963 if (bad < 0) { 964 set_bit(BlockedBadBlocks, &rdev->flags); 965 if (!conf->mddev->external && 966 conf->mddev->flags) { 967 /* It is very unlikely, but we might 968 * still need to write out the 969 * bad block log - better give it 970 * a chance*/ 971 md_check_recovery(conf->mddev); 972 } 973 /* 974 * Because md_wait_for_blocked_rdev 975 * will dec nr_pending, we must 976 * increment it first. 977 */ 978 atomic_inc(&rdev->nr_pending); 979 md_wait_for_blocked_rdev(rdev, conf->mddev); 980 } else { 981 /* Acknowledged bad block - skip the write */ 982 rdev_dec_pending(rdev, conf->mddev); 983 rdev = NULL; 984 } 985 } 986 987 if (rdev) { 988 if (s->syncing || s->expanding || s->expanded 989 || s->replacing) 990 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 991 992 set_bit(STRIPE_IO_STARTED, &sh->state); 993 994 bio_reset(bi); 995 bi->bi_bdev = rdev->bdev; 996 bi->bi_rw = rw; 997 bi->bi_end_io = (rw & WRITE) 998 ? raid5_end_write_request 999 : raid5_end_read_request; 1000 bi->bi_private = sh; 1001 1002 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 1003 __func__, (unsigned long long)sh->sector, 1004 bi->bi_rw, i); 1005 atomic_inc(&sh->count); 1006 if (sh != head_sh) 1007 atomic_inc(&head_sh->count); 1008 if (use_new_offset(conf, sh)) 1009 bi->bi_iter.bi_sector = (sh->sector 1010 + rdev->new_data_offset); 1011 else 1012 bi->bi_iter.bi_sector = (sh->sector 1013 + rdev->data_offset); 1014 if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags)) 1015 bi->bi_rw |= REQ_NOMERGE; 1016 1017 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 1018 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 1019 sh->dev[i].vec.bv_page = sh->dev[i].page; 1020 bi->bi_vcnt = 1; 1021 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 1022 bi->bi_io_vec[0].bv_offset = 0; 1023 bi->bi_iter.bi_size = STRIPE_SIZE; 1024 /* 1025 * If this is discard request, set bi_vcnt 0. We don't 1026 * want to confuse SCSI because SCSI will replace payload 1027 */ 1028 if (rw & REQ_DISCARD) 1029 bi->bi_vcnt = 0; 1030 if (rrdev) 1031 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 1032 1033 if (conf->mddev->gendisk) 1034 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), 1035 bi, disk_devt(conf->mddev->gendisk), 1036 sh->dev[i].sector); 1037 generic_make_request(bi); 1038 } 1039 if (rrdev) { 1040 if (s->syncing || s->expanding || s->expanded 1041 || s->replacing) 1042 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 1043 1044 set_bit(STRIPE_IO_STARTED, &sh->state); 1045 1046 bio_reset(rbi); 1047 rbi->bi_bdev = rrdev->bdev; 1048 rbi->bi_rw = rw; 1049 BUG_ON(!(rw & WRITE)); 1050 rbi->bi_end_io = raid5_end_write_request; 1051 rbi->bi_private = sh; 1052 1053 pr_debug("%s: for %llu schedule op %ld on " 1054 "replacement disc %d\n", 1055 __func__, (unsigned long long)sh->sector, 1056 rbi->bi_rw, i); 1057 atomic_inc(&sh->count); 1058 if (sh != head_sh) 1059 atomic_inc(&head_sh->count); 1060 if (use_new_offset(conf, sh)) 1061 rbi->bi_iter.bi_sector = (sh->sector 1062 + rrdev->new_data_offset); 1063 else 1064 rbi->bi_iter.bi_sector = (sh->sector 1065 + rrdev->data_offset); 1066 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 1067 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 1068 sh->dev[i].rvec.bv_page = sh->dev[i].page; 1069 rbi->bi_vcnt = 1; 1070 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 1071 rbi->bi_io_vec[0].bv_offset = 0; 1072 rbi->bi_iter.bi_size = STRIPE_SIZE; 1073 /* 1074 * If this is discard request, set bi_vcnt 0. We don't 1075 * want to confuse SCSI because SCSI will replace payload 1076 */ 1077 if (rw & REQ_DISCARD) 1078 rbi->bi_vcnt = 0; 1079 if (conf->mddev->gendisk) 1080 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), 1081 rbi, disk_devt(conf->mddev->gendisk), 1082 sh->dev[i].sector); 1083 generic_make_request(rbi); 1084 } 1085 if (!rdev && !rrdev) { 1086 if (rw & WRITE) 1087 set_bit(STRIPE_DEGRADED, &sh->state); 1088 pr_debug("skip op %ld on disc %d for sector %llu\n", 1089 bi->bi_rw, i, (unsigned long long)sh->sector); 1090 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1091 set_bit(STRIPE_HANDLE, &sh->state); 1092 } 1093 1094 if (!head_sh->batch_head) 1095 continue; 1096 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1097 batch_list); 1098 if (sh != head_sh) 1099 goto again; 1100 } 1101 } 1102 1103 static struct dma_async_tx_descriptor * 1104 async_copy_data(int frombio, struct bio *bio, struct page **page, 1105 sector_t sector, struct dma_async_tx_descriptor *tx, 1106 struct stripe_head *sh) 1107 { 1108 struct bio_vec bvl; 1109 struct bvec_iter iter; 1110 struct page *bio_page; 1111 int page_offset; 1112 struct async_submit_ctl submit; 1113 enum async_tx_flags flags = 0; 1114 1115 if (bio->bi_iter.bi_sector >= sector) 1116 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; 1117 else 1118 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; 1119 1120 if (frombio) 1121 flags |= ASYNC_TX_FENCE; 1122 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 1123 1124 bio_for_each_segment(bvl, bio, iter) { 1125 int len = bvl.bv_len; 1126 int clen; 1127 int b_offset = 0; 1128 1129 if (page_offset < 0) { 1130 b_offset = -page_offset; 1131 page_offset += b_offset; 1132 len -= b_offset; 1133 } 1134 1135 if (len > 0 && page_offset + len > STRIPE_SIZE) 1136 clen = STRIPE_SIZE - page_offset; 1137 else 1138 clen = len; 1139 1140 if (clen > 0) { 1141 b_offset += bvl.bv_offset; 1142 bio_page = bvl.bv_page; 1143 if (frombio) { 1144 if (sh->raid_conf->skip_copy && 1145 b_offset == 0 && page_offset == 0 && 1146 clen == STRIPE_SIZE) 1147 *page = bio_page; 1148 else 1149 tx = async_memcpy(*page, bio_page, page_offset, 1150 b_offset, clen, &submit); 1151 } else 1152 tx = async_memcpy(bio_page, *page, b_offset, 1153 page_offset, clen, &submit); 1154 } 1155 /* chain the operations */ 1156 submit.depend_tx = tx; 1157 1158 if (clen < len) /* hit end of page */ 1159 break; 1160 page_offset += len; 1161 } 1162 1163 return tx; 1164 } 1165 1166 static void ops_complete_biofill(void *stripe_head_ref) 1167 { 1168 struct stripe_head *sh = stripe_head_ref; 1169 struct bio *return_bi = NULL; 1170 int i; 1171 1172 pr_debug("%s: stripe %llu\n", __func__, 1173 (unsigned long long)sh->sector); 1174 1175 /* clear completed biofills */ 1176 for (i = sh->disks; i--; ) { 1177 struct r5dev *dev = &sh->dev[i]; 1178 1179 /* acknowledge completion of a biofill operation */ 1180 /* and check if we need to reply to a read request, 1181 * new R5_Wantfill requests are held off until 1182 * !STRIPE_BIOFILL_RUN 1183 */ 1184 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 1185 struct bio *rbi, *rbi2; 1186 1187 BUG_ON(!dev->read); 1188 rbi = dev->read; 1189 dev->read = NULL; 1190 while (rbi && rbi->bi_iter.bi_sector < 1191 dev->sector + STRIPE_SECTORS) { 1192 rbi2 = r5_next_bio(rbi, dev->sector); 1193 if (!raid5_dec_bi_active_stripes(rbi)) { 1194 rbi->bi_next = return_bi; 1195 return_bi = rbi; 1196 } 1197 rbi = rbi2; 1198 } 1199 } 1200 } 1201 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 1202 1203 return_io(return_bi); 1204 1205 set_bit(STRIPE_HANDLE, &sh->state); 1206 release_stripe(sh); 1207 } 1208 1209 static void ops_run_biofill(struct stripe_head *sh) 1210 { 1211 struct dma_async_tx_descriptor *tx = NULL; 1212 struct async_submit_ctl submit; 1213 int i; 1214 1215 BUG_ON(sh->batch_head); 1216 pr_debug("%s: stripe %llu\n", __func__, 1217 (unsigned long long)sh->sector); 1218 1219 for (i = sh->disks; i--; ) { 1220 struct r5dev *dev = &sh->dev[i]; 1221 if (test_bit(R5_Wantfill, &dev->flags)) { 1222 struct bio *rbi; 1223 spin_lock_irq(&sh->stripe_lock); 1224 dev->read = rbi = dev->toread; 1225 dev->toread = NULL; 1226 spin_unlock_irq(&sh->stripe_lock); 1227 while (rbi && rbi->bi_iter.bi_sector < 1228 dev->sector + STRIPE_SECTORS) { 1229 tx = async_copy_data(0, rbi, &dev->page, 1230 dev->sector, tx, sh); 1231 rbi = r5_next_bio(rbi, dev->sector); 1232 } 1233 } 1234 } 1235 1236 atomic_inc(&sh->count); 1237 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 1238 async_trigger_callback(&submit); 1239 } 1240 1241 static void mark_target_uptodate(struct stripe_head *sh, int target) 1242 { 1243 struct r5dev *tgt; 1244 1245 if (target < 0) 1246 return; 1247 1248 tgt = &sh->dev[target]; 1249 set_bit(R5_UPTODATE, &tgt->flags); 1250 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1251 clear_bit(R5_Wantcompute, &tgt->flags); 1252 } 1253 1254 static void ops_complete_compute(void *stripe_head_ref) 1255 { 1256 struct stripe_head *sh = stripe_head_ref; 1257 1258 pr_debug("%s: stripe %llu\n", __func__, 1259 (unsigned long long)sh->sector); 1260 1261 /* mark the computed target(s) as uptodate */ 1262 mark_target_uptodate(sh, sh->ops.target); 1263 mark_target_uptodate(sh, sh->ops.target2); 1264 1265 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 1266 if (sh->check_state == check_state_compute_run) 1267 sh->check_state = check_state_compute_result; 1268 set_bit(STRIPE_HANDLE, &sh->state); 1269 release_stripe(sh); 1270 } 1271 1272 /* return a pointer to the address conversion region of the scribble buffer */ 1273 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 1274 struct raid5_percpu *percpu, int i) 1275 { 1276 void *addr; 1277 1278 addr = flex_array_get(percpu->scribble, i); 1279 return addr + sizeof(struct page *) * (sh->disks + 2); 1280 } 1281 1282 /* return a pointer to the address conversion region of the scribble buffer */ 1283 static struct page **to_addr_page(struct raid5_percpu *percpu, int i) 1284 { 1285 void *addr; 1286 1287 addr = flex_array_get(percpu->scribble, i); 1288 return addr; 1289 } 1290 1291 static struct dma_async_tx_descriptor * 1292 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 1293 { 1294 int disks = sh->disks; 1295 struct page **xor_srcs = to_addr_page(percpu, 0); 1296 int target = sh->ops.target; 1297 struct r5dev *tgt = &sh->dev[target]; 1298 struct page *xor_dest = tgt->page; 1299 int count = 0; 1300 struct dma_async_tx_descriptor *tx; 1301 struct async_submit_ctl submit; 1302 int i; 1303 1304 BUG_ON(sh->batch_head); 1305 1306 pr_debug("%s: stripe %llu block: %d\n", 1307 __func__, (unsigned long long)sh->sector, target); 1308 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1309 1310 for (i = disks; i--; ) 1311 if (i != target) 1312 xor_srcs[count++] = sh->dev[i].page; 1313 1314 atomic_inc(&sh->count); 1315 1316 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 1317 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); 1318 if (unlikely(count == 1)) 1319 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1320 else 1321 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1322 1323 return tx; 1324 } 1325 1326 /* set_syndrome_sources - populate source buffers for gen_syndrome 1327 * @srcs - (struct page *) array of size sh->disks 1328 * @sh - stripe_head to parse 1329 * 1330 * Populates srcs in proper layout order for the stripe and returns the 1331 * 'count' of sources to be used in a call to async_gen_syndrome. The P 1332 * destination buffer is recorded in srcs[count] and the Q destination 1333 * is recorded in srcs[count+1]]. 1334 */ 1335 static int set_syndrome_sources(struct page **srcs, 1336 struct stripe_head *sh, 1337 int srctype) 1338 { 1339 int disks = sh->disks; 1340 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 1341 int d0_idx = raid6_d0(sh); 1342 int count; 1343 int i; 1344 1345 for (i = 0; i < disks; i++) 1346 srcs[i] = NULL; 1347 1348 count = 0; 1349 i = d0_idx; 1350 do { 1351 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1352 struct r5dev *dev = &sh->dev[i]; 1353 1354 if (i == sh->qd_idx || i == sh->pd_idx || 1355 (srctype == SYNDROME_SRC_ALL) || 1356 (srctype == SYNDROME_SRC_WANT_DRAIN && 1357 test_bit(R5_Wantdrain, &dev->flags)) || 1358 (srctype == SYNDROME_SRC_WRITTEN && 1359 dev->written)) 1360 srcs[slot] = sh->dev[i].page; 1361 i = raid6_next_disk(i, disks); 1362 } while (i != d0_idx); 1363 1364 return syndrome_disks; 1365 } 1366 1367 static struct dma_async_tx_descriptor * 1368 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 1369 { 1370 int disks = sh->disks; 1371 struct page **blocks = to_addr_page(percpu, 0); 1372 int target; 1373 int qd_idx = sh->qd_idx; 1374 struct dma_async_tx_descriptor *tx; 1375 struct async_submit_ctl submit; 1376 struct r5dev *tgt; 1377 struct page *dest; 1378 int i; 1379 int count; 1380 1381 BUG_ON(sh->batch_head); 1382 if (sh->ops.target < 0) 1383 target = sh->ops.target2; 1384 else if (sh->ops.target2 < 0) 1385 target = sh->ops.target; 1386 else 1387 /* we should only have one valid target */ 1388 BUG(); 1389 BUG_ON(target < 0); 1390 pr_debug("%s: stripe %llu block: %d\n", 1391 __func__, (unsigned long long)sh->sector, target); 1392 1393 tgt = &sh->dev[target]; 1394 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1395 dest = tgt->page; 1396 1397 atomic_inc(&sh->count); 1398 1399 if (target == qd_idx) { 1400 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); 1401 blocks[count] = NULL; /* regenerating p is not necessary */ 1402 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 1403 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1404 ops_complete_compute, sh, 1405 to_addr_conv(sh, percpu, 0)); 1406 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1407 } else { 1408 /* Compute any data- or p-drive using XOR */ 1409 count = 0; 1410 for (i = disks; i-- ; ) { 1411 if (i == target || i == qd_idx) 1412 continue; 1413 blocks[count++] = sh->dev[i].page; 1414 } 1415 1416 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1417 NULL, ops_complete_compute, sh, 1418 to_addr_conv(sh, percpu, 0)); 1419 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 1420 } 1421 1422 return tx; 1423 } 1424 1425 static struct dma_async_tx_descriptor * 1426 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 1427 { 1428 int i, count, disks = sh->disks; 1429 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 1430 int d0_idx = raid6_d0(sh); 1431 int faila = -1, failb = -1; 1432 int target = sh->ops.target; 1433 int target2 = sh->ops.target2; 1434 struct r5dev *tgt = &sh->dev[target]; 1435 struct r5dev *tgt2 = &sh->dev[target2]; 1436 struct dma_async_tx_descriptor *tx; 1437 struct page **blocks = to_addr_page(percpu, 0); 1438 struct async_submit_ctl submit; 1439 1440 BUG_ON(sh->batch_head); 1441 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 1442 __func__, (unsigned long long)sh->sector, target, target2); 1443 BUG_ON(target < 0 || target2 < 0); 1444 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1445 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 1446 1447 /* we need to open-code set_syndrome_sources to handle the 1448 * slot number conversion for 'faila' and 'failb' 1449 */ 1450 for (i = 0; i < disks ; i++) 1451 blocks[i] = NULL; 1452 count = 0; 1453 i = d0_idx; 1454 do { 1455 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1456 1457 blocks[slot] = sh->dev[i].page; 1458 1459 if (i == target) 1460 faila = slot; 1461 if (i == target2) 1462 failb = slot; 1463 i = raid6_next_disk(i, disks); 1464 } while (i != d0_idx); 1465 1466 BUG_ON(faila == failb); 1467 if (failb < faila) 1468 swap(faila, failb); 1469 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1470 __func__, (unsigned long long)sh->sector, faila, failb); 1471 1472 atomic_inc(&sh->count); 1473 1474 if (failb == syndrome_disks+1) { 1475 /* Q disk is one of the missing disks */ 1476 if (faila == syndrome_disks) { 1477 /* Missing P+Q, just recompute */ 1478 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1479 ops_complete_compute, sh, 1480 to_addr_conv(sh, percpu, 0)); 1481 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1482 STRIPE_SIZE, &submit); 1483 } else { 1484 struct page *dest; 1485 int data_target; 1486 int qd_idx = sh->qd_idx; 1487 1488 /* Missing D+Q: recompute D from P, then recompute Q */ 1489 if (target == qd_idx) 1490 data_target = target2; 1491 else 1492 data_target = target; 1493 1494 count = 0; 1495 for (i = disks; i-- ; ) { 1496 if (i == data_target || i == qd_idx) 1497 continue; 1498 blocks[count++] = sh->dev[i].page; 1499 } 1500 dest = sh->dev[data_target].page; 1501 init_async_submit(&submit, 1502 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1503 NULL, NULL, NULL, 1504 to_addr_conv(sh, percpu, 0)); 1505 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1506 &submit); 1507 1508 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); 1509 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1510 ops_complete_compute, sh, 1511 to_addr_conv(sh, percpu, 0)); 1512 return async_gen_syndrome(blocks, 0, count+2, 1513 STRIPE_SIZE, &submit); 1514 } 1515 } else { 1516 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1517 ops_complete_compute, sh, 1518 to_addr_conv(sh, percpu, 0)); 1519 if (failb == syndrome_disks) { 1520 /* We're missing D+P. */ 1521 return async_raid6_datap_recov(syndrome_disks+2, 1522 STRIPE_SIZE, faila, 1523 blocks, &submit); 1524 } else { 1525 /* We're missing D+D. */ 1526 return async_raid6_2data_recov(syndrome_disks+2, 1527 STRIPE_SIZE, faila, failb, 1528 blocks, &submit); 1529 } 1530 } 1531 } 1532 1533 static void ops_complete_prexor(void *stripe_head_ref) 1534 { 1535 struct stripe_head *sh = stripe_head_ref; 1536 1537 pr_debug("%s: stripe %llu\n", __func__, 1538 (unsigned long long)sh->sector); 1539 } 1540 1541 static struct dma_async_tx_descriptor * 1542 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, 1543 struct dma_async_tx_descriptor *tx) 1544 { 1545 int disks = sh->disks; 1546 struct page **xor_srcs = to_addr_page(percpu, 0); 1547 int count = 0, pd_idx = sh->pd_idx, i; 1548 struct async_submit_ctl submit; 1549 1550 /* existing parity data subtracted */ 1551 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1552 1553 BUG_ON(sh->batch_head); 1554 pr_debug("%s: stripe %llu\n", __func__, 1555 (unsigned long long)sh->sector); 1556 1557 for (i = disks; i--; ) { 1558 struct r5dev *dev = &sh->dev[i]; 1559 /* Only process blocks that are known to be uptodate */ 1560 if (test_bit(R5_Wantdrain, &dev->flags)) 1561 xor_srcs[count++] = dev->page; 1562 } 1563 1564 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1565 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); 1566 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1567 1568 return tx; 1569 } 1570 1571 static struct dma_async_tx_descriptor * 1572 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, 1573 struct dma_async_tx_descriptor *tx) 1574 { 1575 struct page **blocks = to_addr_page(percpu, 0); 1576 int count; 1577 struct async_submit_ctl submit; 1578 1579 pr_debug("%s: stripe %llu\n", __func__, 1580 (unsigned long long)sh->sector); 1581 1582 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN); 1583 1584 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx, 1585 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); 1586 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1587 1588 return tx; 1589 } 1590 1591 static struct dma_async_tx_descriptor * 1592 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1593 { 1594 int disks = sh->disks; 1595 int i; 1596 struct stripe_head *head_sh = sh; 1597 1598 pr_debug("%s: stripe %llu\n", __func__, 1599 (unsigned long long)sh->sector); 1600 1601 for (i = disks; i--; ) { 1602 struct r5dev *dev; 1603 struct bio *chosen; 1604 1605 sh = head_sh; 1606 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) { 1607 struct bio *wbi; 1608 1609 again: 1610 dev = &sh->dev[i]; 1611 spin_lock_irq(&sh->stripe_lock); 1612 chosen = dev->towrite; 1613 dev->towrite = NULL; 1614 sh->overwrite_disks = 0; 1615 BUG_ON(dev->written); 1616 wbi = dev->written = chosen; 1617 spin_unlock_irq(&sh->stripe_lock); 1618 WARN_ON(dev->page != dev->orig_page); 1619 1620 while (wbi && wbi->bi_iter.bi_sector < 1621 dev->sector + STRIPE_SECTORS) { 1622 if (wbi->bi_rw & REQ_FUA) 1623 set_bit(R5_WantFUA, &dev->flags); 1624 if (wbi->bi_rw & REQ_SYNC) 1625 set_bit(R5_SyncIO, &dev->flags); 1626 if (wbi->bi_rw & REQ_DISCARD) 1627 set_bit(R5_Discard, &dev->flags); 1628 else { 1629 tx = async_copy_data(1, wbi, &dev->page, 1630 dev->sector, tx, sh); 1631 if (dev->page != dev->orig_page) { 1632 set_bit(R5_SkipCopy, &dev->flags); 1633 clear_bit(R5_UPTODATE, &dev->flags); 1634 clear_bit(R5_OVERWRITE, &dev->flags); 1635 } 1636 } 1637 wbi = r5_next_bio(wbi, dev->sector); 1638 } 1639 1640 if (head_sh->batch_head) { 1641 sh = list_first_entry(&sh->batch_list, 1642 struct stripe_head, 1643 batch_list); 1644 if (sh == head_sh) 1645 continue; 1646 goto again; 1647 } 1648 } 1649 } 1650 1651 return tx; 1652 } 1653 1654 static void ops_complete_reconstruct(void *stripe_head_ref) 1655 { 1656 struct stripe_head *sh = stripe_head_ref; 1657 int disks = sh->disks; 1658 int pd_idx = sh->pd_idx; 1659 int qd_idx = sh->qd_idx; 1660 int i; 1661 bool fua = false, sync = false, discard = false; 1662 1663 pr_debug("%s: stripe %llu\n", __func__, 1664 (unsigned long long)sh->sector); 1665 1666 for (i = disks; i--; ) { 1667 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1668 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); 1669 discard |= test_bit(R5_Discard, &sh->dev[i].flags); 1670 } 1671 1672 for (i = disks; i--; ) { 1673 struct r5dev *dev = &sh->dev[i]; 1674 1675 if (dev->written || i == pd_idx || i == qd_idx) { 1676 if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) 1677 set_bit(R5_UPTODATE, &dev->flags); 1678 if (fua) 1679 set_bit(R5_WantFUA, &dev->flags); 1680 if (sync) 1681 set_bit(R5_SyncIO, &dev->flags); 1682 } 1683 } 1684 1685 if (sh->reconstruct_state == reconstruct_state_drain_run) 1686 sh->reconstruct_state = reconstruct_state_drain_result; 1687 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1688 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1689 else { 1690 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1691 sh->reconstruct_state = reconstruct_state_result; 1692 } 1693 1694 set_bit(STRIPE_HANDLE, &sh->state); 1695 release_stripe(sh); 1696 } 1697 1698 static void 1699 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1700 struct dma_async_tx_descriptor *tx) 1701 { 1702 int disks = sh->disks; 1703 struct page **xor_srcs; 1704 struct async_submit_ctl submit; 1705 int count, pd_idx = sh->pd_idx, i; 1706 struct page *xor_dest; 1707 int prexor = 0; 1708 unsigned long flags; 1709 int j = 0; 1710 struct stripe_head *head_sh = sh; 1711 int last_stripe; 1712 1713 pr_debug("%s: stripe %llu\n", __func__, 1714 (unsigned long long)sh->sector); 1715 1716 for (i = 0; i < sh->disks; i++) { 1717 if (pd_idx == i) 1718 continue; 1719 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1720 break; 1721 } 1722 if (i >= sh->disks) { 1723 atomic_inc(&sh->count); 1724 set_bit(R5_Discard, &sh->dev[pd_idx].flags); 1725 ops_complete_reconstruct(sh); 1726 return; 1727 } 1728 again: 1729 count = 0; 1730 xor_srcs = to_addr_page(percpu, j); 1731 /* check if prexor is active which means only process blocks 1732 * that are part of a read-modify-write (written) 1733 */ 1734 if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1735 prexor = 1; 1736 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1737 for (i = disks; i--; ) { 1738 struct r5dev *dev = &sh->dev[i]; 1739 if (head_sh->dev[i].written) 1740 xor_srcs[count++] = dev->page; 1741 } 1742 } else { 1743 xor_dest = sh->dev[pd_idx].page; 1744 for (i = disks; i--; ) { 1745 struct r5dev *dev = &sh->dev[i]; 1746 if (i != pd_idx) 1747 xor_srcs[count++] = dev->page; 1748 } 1749 } 1750 1751 /* 1/ if we prexor'd then the dest is reused as a source 1752 * 2/ if we did not prexor then we are redoing the parity 1753 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1754 * for the synchronous xor case 1755 */ 1756 last_stripe = !head_sh->batch_head || 1757 list_first_entry(&sh->batch_list, 1758 struct stripe_head, batch_list) == head_sh; 1759 if (last_stripe) { 1760 flags = ASYNC_TX_ACK | 1761 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1762 1763 atomic_inc(&head_sh->count); 1764 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh, 1765 to_addr_conv(sh, percpu, j)); 1766 } else { 1767 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST; 1768 init_async_submit(&submit, flags, tx, NULL, NULL, 1769 to_addr_conv(sh, percpu, j)); 1770 } 1771 1772 if (unlikely(count == 1)) 1773 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1774 else 1775 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1776 if (!last_stripe) { 1777 j++; 1778 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1779 batch_list); 1780 goto again; 1781 } 1782 } 1783 1784 static void 1785 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1786 struct dma_async_tx_descriptor *tx) 1787 { 1788 struct async_submit_ctl submit; 1789 struct page **blocks; 1790 int count, i, j = 0; 1791 struct stripe_head *head_sh = sh; 1792 int last_stripe; 1793 int synflags; 1794 unsigned long txflags; 1795 1796 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1797 1798 for (i = 0; i < sh->disks; i++) { 1799 if (sh->pd_idx == i || sh->qd_idx == i) 1800 continue; 1801 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1802 break; 1803 } 1804 if (i >= sh->disks) { 1805 atomic_inc(&sh->count); 1806 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 1807 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 1808 ops_complete_reconstruct(sh); 1809 return; 1810 } 1811 1812 again: 1813 blocks = to_addr_page(percpu, j); 1814 1815 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1816 synflags = SYNDROME_SRC_WRITTEN; 1817 txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST; 1818 } else { 1819 synflags = SYNDROME_SRC_ALL; 1820 txflags = ASYNC_TX_ACK; 1821 } 1822 1823 count = set_syndrome_sources(blocks, sh, synflags); 1824 last_stripe = !head_sh->batch_head || 1825 list_first_entry(&sh->batch_list, 1826 struct stripe_head, batch_list) == head_sh; 1827 1828 if (last_stripe) { 1829 atomic_inc(&head_sh->count); 1830 init_async_submit(&submit, txflags, tx, ops_complete_reconstruct, 1831 head_sh, to_addr_conv(sh, percpu, j)); 1832 } else 1833 init_async_submit(&submit, 0, tx, NULL, NULL, 1834 to_addr_conv(sh, percpu, j)); 1835 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1836 if (!last_stripe) { 1837 j++; 1838 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1839 batch_list); 1840 goto again; 1841 } 1842 } 1843 1844 static void ops_complete_check(void *stripe_head_ref) 1845 { 1846 struct stripe_head *sh = stripe_head_ref; 1847 1848 pr_debug("%s: stripe %llu\n", __func__, 1849 (unsigned long long)sh->sector); 1850 1851 sh->check_state = check_state_check_result; 1852 set_bit(STRIPE_HANDLE, &sh->state); 1853 release_stripe(sh); 1854 } 1855 1856 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1857 { 1858 int disks = sh->disks; 1859 int pd_idx = sh->pd_idx; 1860 int qd_idx = sh->qd_idx; 1861 struct page *xor_dest; 1862 struct page **xor_srcs = to_addr_page(percpu, 0); 1863 struct dma_async_tx_descriptor *tx; 1864 struct async_submit_ctl submit; 1865 int count; 1866 int i; 1867 1868 pr_debug("%s: stripe %llu\n", __func__, 1869 (unsigned long long)sh->sector); 1870 1871 BUG_ON(sh->batch_head); 1872 count = 0; 1873 xor_dest = sh->dev[pd_idx].page; 1874 xor_srcs[count++] = xor_dest; 1875 for (i = disks; i--; ) { 1876 if (i == pd_idx || i == qd_idx) 1877 continue; 1878 xor_srcs[count++] = sh->dev[i].page; 1879 } 1880 1881 init_async_submit(&submit, 0, NULL, NULL, NULL, 1882 to_addr_conv(sh, percpu, 0)); 1883 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1884 &sh->ops.zero_sum_result, &submit); 1885 1886 atomic_inc(&sh->count); 1887 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1888 tx = async_trigger_callback(&submit); 1889 } 1890 1891 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1892 { 1893 struct page **srcs = to_addr_page(percpu, 0); 1894 struct async_submit_ctl submit; 1895 int count; 1896 1897 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1898 (unsigned long long)sh->sector, checkp); 1899 1900 BUG_ON(sh->batch_head); 1901 count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL); 1902 if (!checkp) 1903 srcs[count] = NULL; 1904 1905 atomic_inc(&sh->count); 1906 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1907 sh, to_addr_conv(sh, percpu, 0)); 1908 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1909 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1910 } 1911 1912 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1913 { 1914 int overlap_clear = 0, i, disks = sh->disks; 1915 struct dma_async_tx_descriptor *tx = NULL; 1916 struct r5conf *conf = sh->raid_conf; 1917 int level = conf->level; 1918 struct raid5_percpu *percpu; 1919 unsigned long cpu; 1920 1921 cpu = get_cpu(); 1922 percpu = per_cpu_ptr(conf->percpu, cpu); 1923 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1924 ops_run_biofill(sh); 1925 overlap_clear++; 1926 } 1927 1928 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1929 if (level < 6) 1930 tx = ops_run_compute5(sh, percpu); 1931 else { 1932 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1933 tx = ops_run_compute6_1(sh, percpu); 1934 else 1935 tx = ops_run_compute6_2(sh, percpu); 1936 } 1937 /* terminate the chain if reconstruct is not set to be run */ 1938 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1939 async_tx_ack(tx); 1940 } 1941 1942 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) { 1943 if (level < 6) 1944 tx = ops_run_prexor5(sh, percpu, tx); 1945 else 1946 tx = ops_run_prexor6(sh, percpu, tx); 1947 } 1948 1949 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1950 tx = ops_run_biodrain(sh, tx); 1951 overlap_clear++; 1952 } 1953 1954 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1955 if (level < 6) 1956 ops_run_reconstruct5(sh, percpu, tx); 1957 else 1958 ops_run_reconstruct6(sh, percpu, tx); 1959 } 1960 1961 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1962 if (sh->check_state == check_state_run) 1963 ops_run_check_p(sh, percpu); 1964 else if (sh->check_state == check_state_run_q) 1965 ops_run_check_pq(sh, percpu, 0); 1966 else if (sh->check_state == check_state_run_pq) 1967 ops_run_check_pq(sh, percpu, 1); 1968 else 1969 BUG(); 1970 } 1971 1972 if (overlap_clear && !sh->batch_head) 1973 for (i = disks; i--; ) { 1974 struct r5dev *dev = &sh->dev[i]; 1975 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1976 wake_up(&sh->raid_conf->wait_for_overlap); 1977 } 1978 put_cpu(); 1979 } 1980 1981 static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) 1982 { 1983 struct stripe_head *sh; 1984 1985 sh = kmem_cache_zalloc(sc, gfp); 1986 if (sh) { 1987 spin_lock_init(&sh->stripe_lock); 1988 spin_lock_init(&sh->batch_lock); 1989 INIT_LIST_HEAD(&sh->batch_list); 1990 INIT_LIST_HEAD(&sh->lru); 1991 atomic_set(&sh->count, 1); 1992 } 1993 return sh; 1994 } 1995 static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) 1996 { 1997 struct stripe_head *sh; 1998 1999 sh = alloc_stripe(conf->slab_cache, gfp); 2000 if (!sh) 2001 return 0; 2002 2003 sh->raid_conf = conf; 2004 2005 if (grow_buffers(sh, gfp)) { 2006 shrink_buffers(sh); 2007 kmem_cache_free(conf->slab_cache, sh); 2008 return 0; 2009 } 2010 sh->hash_lock_index = 2011 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 2012 /* we just created an active stripe so... */ 2013 atomic_inc(&conf->active_stripes); 2014 2015 release_stripe(sh); 2016 conf->max_nr_stripes++; 2017 return 1; 2018 } 2019 2020 static int grow_stripes(struct r5conf *conf, int num) 2021 { 2022 struct kmem_cache *sc; 2023 int devs = max(conf->raid_disks, conf->previous_raid_disks); 2024 2025 if (conf->mddev->gendisk) 2026 sprintf(conf->cache_name[0], 2027 "raid%d-%s", conf->level, mdname(conf->mddev)); 2028 else 2029 sprintf(conf->cache_name[0], 2030 "raid%d-%p", conf->level, conf->mddev); 2031 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 2032 2033 conf->active_name = 0; 2034 sc = kmem_cache_create(conf->cache_name[conf->active_name], 2035 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 2036 0, 0, NULL); 2037 if (!sc) 2038 return 1; 2039 conf->slab_cache = sc; 2040 conf->pool_size = devs; 2041 while (num--) 2042 if (!grow_one_stripe(conf, GFP_KERNEL)) 2043 return 1; 2044 2045 return 0; 2046 } 2047 2048 /** 2049 * scribble_len - return the required size of the scribble region 2050 * @num - total number of disks in the array 2051 * 2052 * The size must be enough to contain: 2053 * 1/ a struct page pointer for each device in the array +2 2054 * 2/ room to convert each entry in (1) to its corresponding dma 2055 * (dma_map_page()) or page (page_address()) address. 2056 * 2057 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 2058 * calculate over all devices (not just the data blocks), using zeros in place 2059 * of the P and Q blocks. 2060 */ 2061 static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) 2062 { 2063 struct flex_array *ret; 2064 size_t len; 2065 2066 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 2067 ret = flex_array_alloc(len, cnt, flags); 2068 if (!ret) 2069 return NULL; 2070 /* always prealloc all elements, so no locking is required */ 2071 if (flex_array_prealloc(ret, 0, cnt, flags)) { 2072 flex_array_free(ret); 2073 return NULL; 2074 } 2075 return ret; 2076 } 2077 2078 static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) 2079 { 2080 unsigned long cpu; 2081 int err = 0; 2082 2083 mddev_suspend(conf->mddev); 2084 get_online_cpus(); 2085 for_each_present_cpu(cpu) { 2086 struct raid5_percpu *percpu; 2087 struct flex_array *scribble; 2088 2089 percpu = per_cpu_ptr(conf->percpu, cpu); 2090 scribble = scribble_alloc(new_disks, 2091 new_sectors / STRIPE_SECTORS, 2092 GFP_NOIO); 2093 2094 if (scribble) { 2095 flex_array_free(percpu->scribble); 2096 percpu->scribble = scribble; 2097 } else { 2098 err = -ENOMEM; 2099 break; 2100 } 2101 } 2102 put_online_cpus(); 2103 mddev_resume(conf->mddev); 2104 return err; 2105 } 2106 2107 static int resize_stripes(struct r5conf *conf, int newsize) 2108 { 2109 /* Make all the stripes able to hold 'newsize' devices. 2110 * New slots in each stripe get 'page' set to a new page. 2111 * 2112 * This happens in stages: 2113 * 1/ create a new kmem_cache and allocate the required number of 2114 * stripe_heads. 2115 * 2/ gather all the old stripe_heads and transfer the pages across 2116 * to the new stripe_heads. This will have the side effect of 2117 * freezing the array as once all stripe_heads have been collected, 2118 * no IO will be possible. Old stripe heads are freed once their 2119 * pages have been transferred over, and the old kmem_cache is 2120 * freed when all stripes are done. 2121 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 2122 * we simple return a failre status - no need to clean anything up. 2123 * 4/ allocate new pages for the new slots in the new stripe_heads. 2124 * If this fails, we don't bother trying the shrink the 2125 * stripe_heads down again, we just leave them as they are. 2126 * As each stripe_head is processed the new one is released into 2127 * active service. 2128 * 2129 * Once step2 is started, we cannot afford to wait for a write, 2130 * so we use GFP_NOIO allocations. 2131 */ 2132 struct stripe_head *osh, *nsh; 2133 LIST_HEAD(newstripes); 2134 struct disk_info *ndisks; 2135 int err; 2136 struct kmem_cache *sc; 2137 int i; 2138 int hash, cnt; 2139 2140 if (newsize <= conf->pool_size) 2141 return 0; /* never bother to shrink */ 2142 2143 err = md_allow_write(conf->mddev); 2144 if (err) 2145 return err; 2146 2147 /* Step 1 */ 2148 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 2149 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 2150 0, 0, NULL); 2151 if (!sc) 2152 return -ENOMEM; 2153 2154 for (i = conf->max_nr_stripes; i; i--) { 2155 nsh = alloc_stripe(sc, GFP_KERNEL); 2156 if (!nsh) 2157 break; 2158 2159 nsh->raid_conf = conf; 2160 list_add(&nsh->lru, &newstripes); 2161 } 2162 if (i) { 2163 /* didn't get enough, give up */ 2164 while (!list_empty(&newstripes)) { 2165 nsh = list_entry(newstripes.next, struct stripe_head, lru); 2166 list_del(&nsh->lru); 2167 kmem_cache_free(sc, nsh); 2168 } 2169 kmem_cache_destroy(sc); 2170 return -ENOMEM; 2171 } 2172 /* Step 2 - Must use GFP_NOIO now. 2173 * OK, we have enough stripes, start collecting inactive 2174 * stripes and copying them over 2175 */ 2176 hash = 0; 2177 cnt = 0; 2178 list_for_each_entry(nsh, &newstripes, lru) { 2179 lock_device_hash_lock(conf, hash); 2180 wait_event_cmd(conf->wait_for_stripe, 2181 !list_empty(conf->inactive_list + hash), 2182 unlock_device_hash_lock(conf, hash), 2183 lock_device_hash_lock(conf, hash)); 2184 osh = get_free_stripe(conf, hash); 2185 unlock_device_hash_lock(conf, hash); 2186 2187 for(i=0; i<conf->pool_size; i++) { 2188 nsh->dev[i].page = osh->dev[i].page; 2189 nsh->dev[i].orig_page = osh->dev[i].page; 2190 } 2191 nsh->hash_lock_index = hash; 2192 kmem_cache_free(conf->slab_cache, osh); 2193 cnt++; 2194 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + 2195 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { 2196 hash++; 2197 cnt = 0; 2198 } 2199 } 2200 kmem_cache_destroy(conf->slab_cache); 2201 2202 /* Step 3. 2203 * At this point, we are holding all the stripes so the array 2204 * is completely stalled, so now is a good time to resize 2205 * conf->disks and the scribble region 2206 */ 2207 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 2208 if (ndisks) { 2209 for (i=0; i<conf->raid_disks; i++) 2210 ndisks[i] = conf->disks[i]; 2211 kfree(conf->disks); 2212 conf->disks = ndisks; 2213 } else 2214 err = -ENOMEM; 2215 2216 /* Step 4, return new stripes to service */ 2217 while(!list_empty(&newstripes)) { 2218 nsh = list_entry(newstripes.next, struct stripe_head, lru); 2219 list_del_init(&nsh->lru); 2220 2221 for (i=conf->raid_disks; i < newsize; i++) 2222 if (nsh->dev[i].page == NULL) { 2223 struct page *p = alloc_page(GFP_NOIO); 2224 nsh->dev[i].page = p; 2225 nsh->dev[i].orig_page = p; 2226 if (!p) 2227 err = -ENOMEM; 2228 } 2229 release_stripe(nsh); 2230 } 2231 /* critical section pass, GFP_NOIO no longer needed */ 2232 2233 conf->slab_cache = sc; 2234 conf->active_name = 1-conf->active_name; 2235 if (!err) 2236 conf->pool_size = newsize; 2237 return err; 2238 } 2239 2240 static int drop_one_stripe(struct r5conf *conf) 2241 { 2242 struct stripe_head *sh; 2243 int hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS; 2244 2245 spin_lock_irq(conf->hash_locks + hash); 2246 sh = get_free_stripe(conf, hash); 2247 spin_unlock_irq(conf->hash_locks + hash); 2248 if (!sh) 2249 return 0; 2250 BUG_ON(atomic_read(&sh->count)); 2251 shrink_buffers(sh); 2252 kmem_cache_free(conf->slab_cache, sh); 2253 atomic_dec(&conf->active_stripes); 2254 conf->max_nr_stripes--; 2255 return 1; 2256 } 2257 2258 static void shrink_stripes(struct r5conf *conf) 2259 { 2260 while (conf->max_nr_stripes && 2261 drop_one_stripe(conf)) 2262 ; 2263 2264 if (conf->slab_cache) 2265 kmem_cache_destroy(conf->slab_cache); 2266 conf->slab_cache = NULL; 2267 } 2268 2269 static void raid5_end_read_request(struct bio * bi, int error) 2270 { 2271 struct stripe_head *sh = bi->bi_private; 2272 struct r5conf *conf = sh->raid_conf; 2273 int disks = sh->disks, i; 2274 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 2275 char b[BDEVNAME_SIZE]; 2276 struct md_rdev *rdev = NULL; 2277 sector_t s; 2278 2279 for (i=0 ; i<disks; i++) 2280 if (bi == &sh->dev[i].req) 2281 break; 2282 2283 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 2284 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2285 uptodate); 2286 if (i == disks) { 2287 BUG(); 2288 return; 2289 } 2290 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2291 /* If replacement finished while this request was outstanding, 2292 * 'replacement' might be NULL already. 2293 * In that case it moved down to 'rdev'. 2294 * rdev is not removed until all requests are finished. 2295 */ 2296 rdev = conf->disks[i].replacement; 2297 if (!rdev) 2298 rdev = conf->disks[i].rdev; 2299 2300 if (use_new_offset(conf, sh)) 2301 s = sh->sector + rdev->new_data_offset; 2302 else 2303 s = sh->sector + rdev->data_offset; 2304 if (uptodate) { 2305 set_bit(R5_UPTODATE, &sh->dev[i].flags); 2306 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2307 /* Note that this cannot happen on a 2308 * replacement device. We just fail those on 2309 * any error 2310 */ 2311 printk_ratelimited( 2312 KERN_INFO 2313 "md/raid:%s: read error corrected" 2314 " (%lu sectors at %llu on %s)\n", 2315 mdname(conf->mddev), STRIPE_SECTORS, 2316 (unsigned long long)s, 2317 bdevname(rdev->bdev, b)); 2318 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 2319 clear_bit(R5_ReadError, &sh->dev[i].flags); 2320 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2321 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2322 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2323 2324 if (atomic_read(&rdev->read_errors)) 2325 atomic_set(&rdev->read_errors, 0); 2326 } else { 2327 const char *bdn = bdevname(rdev->bdev, b); 2328 int retry = 0; 2329 int set_bad = 0; 2330 2331 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 2332 atomic_inc(&rdev->read_errors); 2333 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2334 printk_ratelimited( 2335 KERN_WARNING 2336 "md/raid:%s: read error on replacement device " 2337 "(sector %llu on %s).\n", 2338 mdname(conf->mddev), 2339 (unsigned long long)s, 2340 bdn); 2341 else if (conf->mddev->degraded >= conf->max_degraded) { 2342 set_bad = 1; 2343 printk_ratelimited( 2344 KERN_WARNING 2345 "md/raid:%s: read error not correctable " 2346 "(sector %llu on %s).\n", 2347 mdname(conf->mddev), 2348 (unsigned long long)s, 2349 bdn); 2350 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { 2351 /* Oh, no!!! */ 2352 set_bad = 1; 2353 printk_ratelimited( 2354 KERN_WARNING 2355 "md/raid:%s: read error NOT corrected!! " 2356 "(sector %llu on %s).\n", 2357 mdname(conf->mddev), 2358 (unsigned long long)s, 2359 bdn); 2360 } else if (atomic_read(&rdev->read_errors) 2361 > conf->max_nr_stripes) 2362 printk(KERN_WARNING 2363 "md/raid:%s: Too many read errors, failing device %s.\n", 2364 mdname(conf->mddev), bdn); 2365 else 2366 retry = 1; 2367 if (set_bad && test_bit(In_sync, &rdev->flags) 2368 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2369 retry = 1; 2370 if (retry) 2371 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { 2372 set_bit(R5_ReadError, &sh->dev[i].flags); 2373 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2374 } else 2375 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2376 else { 2377 clear_bit(R5_ReadError, &sh->dev[i].flags); 2378 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2379 if (!(set_bad 2380 && test_bit(In_sync, &rdev->flags) 2381 && rdev_set_badblocks( 2382 rdev, sh->sector, STRIPE_SECTORS, 0))) 2383 md_error(conf->mddev, rdev); 2384 } 2385 } 2386 rdev_dec_pending(rdev, conf->mddev); 2387 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2388 set_bit(STRIPE_HANDLE, &sh->state); 2389 release_stripe(sh); 2390 } 2391 2392 static void raid5_end_write_request(struct bio *bi, int error) 2393 { 2394 struct stripe_head *sh = bi->bi_private; 2395 struct r5conf *conf = sh->raid_conf; 2396 int disks = sh->disks, i; 2397 struct md_rdev *uninitialized_var(rdev); 2398 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 2399 sector_t first_bad; 2400 int bad_sectors; 2401 int replacement = 0; 2402 2403 for (i = 0 ; i < disks; i++) { 2404 if (bi == &sh->dev[i].req) { 2405 rdev = conf->disks[i].rdev; 2406 break; 2407 } 2408 if (bi == &sh->dev[i].rreq) { 2409 rdev = conf->disks[i].replacement; 2410 if (rdev) 2411 replacement = 1; 2412 else 2413 /* rdev was removed and 'replacement' 2414 * replaced it. rdev is not removed 2415 * until all requests are finished. 2416 */ 2417 rdev = conf->disks[i].rdev; 2418 break; 2419 } 2420 } 2421 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 2422 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2423 uptodate); 2424 if (i == disks) { 2425 BUG(); 2426 return; 2427 } 2428 2429 if (replacement) { 2430 if (!uptodate) 2431 md_error(conf->mddev, rdev); 2432 else if (is_badblock(rdev, sh->sector, 2433 STRIPE_SECTORS, 2434 &first_bad, &bad_sectors)) 2435 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2436 } else { 2437 if (!uptodate) { 2438 set_bit(STRIPE_DEGRADED, &sh->state); 2439 set_bit(WriteErrorSeen, &rdev->flags); 2440 set_bit(R5_WriteError, &sh->dev[i].flags); 2441 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2442 set_bit(MD_RECOVERY_NEEDED, 2443 &rdev->mddev->recovery); 2444 } else if (is_badblock(rdev, sh->sector, 2445 STRIPE_SECTORS, 2446 &first_bad, &bad_sectors)) { 2447 set_bit(R5_MadeGood, &sh->dev[i].flags); 2448 if (test_bit(R5_ReadError, &sh->dev[i].flags)) 2449 /* That was a successful write so make 2450 * sure it looks like we already did 2451 * a re-write. 2452 */ 2453 set_bit(R5_ReWrite, &sh->dev[i].flags); 2454 } 2455 } 2456 rdev_dec_pending(rdev, conf->mddev); 2457 2458 if (sh->batch_head && !uptodate && !replacement) 2459 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); 2460 2461 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2462 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2463 set_bit(STRIPE_HANDLE, &sh->state); 2464 release_stripe(sh); 2465 2466 if (sh->batch_head && sh != sh->batch_head) 2467 release_stripe(sh->batch_head); 2468 } 2469 2470 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 2471 2472 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 2473 { 2474 struct r5dev *dev = &sh->dev[i]; 2475 2476 bio_init(&dev->req); 2477 dev->req.bi_io_vec = &dev->vec; 2478 dev->req.bi_max_vecs = 1; 2479 dev->req.bi_private = sh; 2480 2481 bio_init(&dev->rreq); 2482 dev->rreq.bi_io_vec = &dev->rvec; 2483 dev->rreq.bi_max_vecs = 1; 2484 dev->rreq.bi_private = sh; 2485 2486 dev->flags = 0; 2487 dev->sector = compute_blocknr(sh, i, previous); 2488 } 2489 2490 static void error(struct mddev *mddev, struct md_rdev *rdev) 2491 { 2492 char b[BDEVNAME_SIZE]; 2493 struct r5conf *conf = mddev->private; 2494 unsigned long flags; 2495 pr_debug("raid456: error called\n"); 2496 2497 spin_lock_irqsave(&conf->device_lock, flags); 2498 clear_bit(In_sync, &rdev->flags); 2499 mddev->degraded = calc_degraded(conf); 2500 spin_unlock_irqrestore(&conf->device_lock, flags); 2501 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2502 2503 set_bit(Blocked, &rdev->flags); 2504 set_bit(Faulty, &rdev->flags); 2505 set_bit(MD_CHANGE_DEVS, &mddev->flags); 2506 printk(KERN_ALERT 2507 "md/raid:%s: Disk failure on %s, disabling device.\n" 2508 "md/raid:%s: Operation continuing on %d devices.\n", 2509 mdname(mddev), 2510 bdevname(rdev->bdev, b), 2511 mdname(mddev), 2512 conf->raid_disks - mddev->degraded); 2513 } 2514 2515 /* 2516 * Input: a 'big' sector number, 2517 * Output: index of the data and parity disk, and the sector # in them. 2518 */ 2519 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 2520 int previous, int *dd_idx, 2521 struct stripe_head *sh) 2522 { 2523 sector_t stripe, stripe2; 2524 sector_t chunk_number; 2525 unsigned int chunk_offset; 2526 int pd_idx, qd_idx; 2527 int ddf_layout = 0; 2528 sector_t new_sector; 2529 int algorithm = previous ? conf->prev_algo 2530 : conf->algorithm; 2531 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2532 : conf->chunk_sectors; 2533 int raid_disks = previous ? conf->previous_raid_disks 2534 : conf->raid_disks; 2535 int data_disks = raid_disks - conf->max_degraded; 2536 2537 /* First compute the information on this sector */ 2538 2539 /* 2540 * Compute the chunk number and the sector offset inside the chunk 2541 */ 2542 chunk_offset = sector_div(r_sector, sectors_per_chunk); 2543 chunk_number = r_sector; 2544 2545 /* 2546 * Compute the stripe number 2547 */ 2548 stripe = chunk_number; 2549 *dd_idx = sector_div(stripe, data_disks); 2550 stripe2 = stripe; 2551 /* 2552 * Select the parity disk based on the user selected algorithm. 2553 */ 2554 pd_idx = qd_idx = -1; 2555 switch(conf->level) { 2556 case 4: 2557 pd_idx = data_disks; 2558 break; 2559 case 5: 2560 switch (algorithm) { 2561 case ALGORITHM_LEFT_ASYMMETRIC: 2562 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2563 if (*dd_idx >= pd_idx) 2564 (*dd_idx)++; 2565 break; 2566 case ALGORITHM_RIGHT_ASYMMETRIC: 2567 pd_idx = sector_div(stripe2, raid_disks); 2568 if (*dd_idx >= pd_idx) 2569 (*dd_idx)++; 2570 break; 2571 case ALGORITHM_LEFT_SYMMETRIC: 2572 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2573 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2574 break; 2575 case ALGORITHM_RIGHT_SYMMETRIC: 2576 pd_idx = sector_div(stripe2, raid_disks); 2577 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2578 break; 2579 case ALGORITHM_PARITY_0: 2580 pd_idx = 0; 2581 (*dd_idx)++; 2582 break; 2583 case ALGORITHM_PARITY_N: 2584 pd_idx = data_disks; 2585 break; 2586 default: 2587 BUG(); 2588 } 2589 break; 2590 case 6: 2591 2592 switch (algorithm) { 2593 case ALGORITHM_LEFT_ASYMMETRIC: 2594 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2595 qd_idx = pd_idx + 1; 2596 if (pd_idx == raid_disks-1) { 2597 (*dd_idx)++; /* Q D D D P */ 2598 qd_idx = 0; 2599 } else if (*dd_idx >= pd_idx) 2600 (*dd_idx) += 2; /* D D P Q D */ 2601 break; 2602 case ALGORITHM_RIGHT_ASYMMETRIC: 2603 pd_idx = sector_div(stripe2, raid_disks); 2604 qd_idx = pd_idx + 1; 2605 if (pd_idx == raid_disks-1) { 2606 (*dd_idx)++; /* Q D D D P */ 2607 qd_idx = 0; 2608 } else if (*dd_idx >= pd_idx) 2609 (*dd_idx) += 2; /* D D P Q D */ 2610 break; 2611 case ALGORITHM_LEFT_SYMMETRIC: 2612 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2613 qd_idx = (pd_idx + 1) % raid_disks; 2614 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2615 break; 2616 case ALGORITHM_RIGHT_SYMMETRIC: 2617 pd_idx = sector_div(stripe2, raid_disks); 2618 qd_idx = (pd_idx + 1) % raid_disks; 2619 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2620 break; 2621 2622 case ALGORITHM_PARITY_0: 2623 pd_idx = 0; 2624 qd_idx = 1; 2625 (*dd_idx) += 2; 2626 break; 2627 case ALGORITHM_PARITY_N: 2628 pd_idx = data_disks; 2629 qd_idx = data_disks + 1; 2630 break; 2631 2632 case ALGORITHM_ROTATING_ZERO_RESTART: 2633 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2634 * of blocks for computing Q is different. 2635 */ 2636 pd_idx = sector_div(stripe2, raid_disks); 2637 qd_idx = pd_idx + 1; 2638 if (pd_idx == raid_disks-1) { 2639 (*dd_idx)++; /* Q D D D P */ 2640 qd_idx = 0; 2641 } else if (*dd_idx >= pd_idx) 2642 (*dd_idx) += 2; /* D D P Q D */ 2643 ddf_layout = 1; 2644 break; 2645 2646 case ALGORITHM_ROTATING_N_RESTART: 2647 /* Same a left_asymmetric, by first stripe is 2648 * D D D P Q rather than 2649 * Q D D D P 2650 */ 2651 stripe2 += 1; 2652 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2653 qd_idx = pd_idx + 1; 2654 if (pd_idx == raid_disks-1) { 2655 (*dd_idx)++; /* Q D D D P */ 2656 qd_idx = 0; 2657 } else if (*dd_idx >= pd_idx) 2658 (*dd_idx) += 2; /* D D P Q D */ 2659 ddf_layout = 1; 2660 break; 2661 2662 case ALGORITHM_ROTATING_N_CONTINUE: 2663 /* Same as left_symmetric but Q is before P */ 2664 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2665 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2666 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2667 ddf_layout = 1; 2668 break; 2669 2670 case ALGORITHM_LEFT_ASYMMETRIC_6: 2671 /* RAID5 left_asymmetric, with Q on last device */ 2672 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2673 if (*dd_idx >= pd_idx) 2674 (*dd_idx)++; 2675 qd_idx = raid_disks - 1; 2676 break; 2677 2678 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2679 pd_idx = sector_div(stripe2, raid_disks-1); 2680 if (*dd_idx >= pd_idx) 2681 (*dd_idx)++; 2682 qd_idx = raid_disks - 1; 2683 break; 2684 2685 case ALGORITHM_LEFT_SYMMETRIC_6: 2686 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2687 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2688 qd_idx = raid_disks - 1; 2689 break; 2690 2691 case ALGORITHM_RIGHT_SYMMETRIC_6: 2692 pd_idx = sector_div(stripe2, raid_disks-1); 2693 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2694 qd_idx = raid_disks - 1; 2695 break; 2696 2697 case ALGORITHM_PARITY_0_6: 2698 pd_idx = 0; 2699 (*dd_idx)++; 2700 qd_idx = raid_disks - 1; 2701 break; 2702 2703 default: 2704 BUG(); 2705 } 2706 break; 2707 } 2708 2709 if (sh) { 2710 sh->pd_idx = pd_idx; 2711 sh->qd_idx = qd_idx; 2712 sh->ddf_layout = ddf_layout; 2713 } 2714 /* 2715 * Finally, compute the new sector number 2716 */ 2717 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2718 return new_sector; 2719 } 2720 2721 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 2722 { 2723 struct r5conf *conf = sh->raid_conf; 2724 int raid_disks = sh->disks; 2725 int data_disks = raid_disks - conf->max_degraded; 2726 sector_t new_sector = sh->sector, check; 2727 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2728 : conf->chunk_sectors; 2729 int algorithm = previous ? conf->prev_algo 2730 : conf->algorithm; 2731 sector_t stripe; 2732 int chunk_offset; 2733 sector_t chunk_number; 2734 int dummy1, dd_idx = i; 2735 sector_t r_sector; 2736 struct stripe_head sh2; 2737 2738 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2739 stripe = new_sector; 2740 2741 if (i == sh->pd_idx) 2742 return 0; 2743 switch(conf->level) { 2744 case 4: break; 2745 case 5: 2746 switch (algorithm) { 2747 case ALGORITHM_LEFT_ASYMMETRIC: 2748 case ALGORITHM_RIGHT_ASYMMETRIC: 2749 if (i > sh->pd_idx) 2750 i--; 2751 break; 2752 case ALGORITHM_LEFT_SYMMETRIC: 2753 case ALGORITHM_RIGHT_SYMMETRIC: 2754 if (i < sh->pd_idx) 2755 i += raid_disks; 2756 i -= (sh->pd_idx + 1); 2757 break; 2758 case ALGORITHM_PARITY_0: 2759 i -= 1; 2760 break; 2761 case ALGORITHM_PARITY_N: 2762 break; 2763 default: 2764 BUG(); 2765 } 2766 break; 2767 case 6: 2768 if (i == sh->qd_idx) 2769 return 0; /* It is the Q disk */ 2770 switch (algorithm) { 2771 case ALGORITHM_LEFT_ASYMMETRIC: 2772 case ALGORITHM_RIGHT_ASYMMETRIC: 2773 case ALGORITHM_ROTATING_ZERO_RESTART: 2774 case ALGORITHM_ROTATING_N_RESTART: 2775 if (sh->pd_idx == raid_disks-1) 2776 i--; /* Q D D D P */ 2777 else if (i > sh->pd_idx) 2778 i -= 2; /* D D P Q D */ 2779 break; 2780 case ALGORITHM_LEFT_SYMMETRIC: 2781 case ALGORITHM_RIGHT_SYMMETRIC: 2782 if (sh->pd_idx == raid_disks-1) 2783 i--; /* Q D D D P */ 2784 else { 2785 /* D D P Q D */ 2786 if (i < sh->pd_idx) 2787 i += raid_disks; 2788 i -= (sh->pd_idx + 2); 2789 } 2790 break; 2791 case ALGORITHM_PARITY_0: 2792 i -= 2; 2793 break; 2794 case ALGORITHM_PARITY_N: 2795 break; 2796 case ALGORITHM_ROTATING_N_CONTINUE: 2797 /* Like left_symmetric, but P is before Q */ 2798 if (sh->pd_idx == 0) 2799 i--; /* P D D D Q */ 2800 else { 2801 /* D D Q P D */ 2802 if (i < sh->pd_idx) 2803 i += raid_disks; 2804 i -= (sh->pd_idx + 1); 2805 } 2806 break; 2807 case ALGORITHM_LEFT_ASYMMETRIC_6: 2808 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2809 if (i > sh->pd_idx) 2810 i--; 2811 break; 2812 case ALGORITHM_LEFT_SYMMETRIC_6: 2813 case ALGORITHM_RIGHT_SYMMETRIC_6: 2814 if (i < sh->pd_idx) 2815 i += data_disks + 1; 2816 i -= (sh->pd_idx + 1); 2817 break; 2818 case ALGORITHM_PARITY_0_6: 2819 i -= 1; 2820 break; 2821 default: 2822 BUG(); 2823 } 2824 break; 2825 } 2826 2827 chunk_number = stripe * data_disks + i; 2828 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2829 2830 check = raid5_compute_sector(conf, r_sector, 2831 previous, &dummy1, &sh2); 2832 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2833 || sh2.qd_idx != sh->qd_idx) { 2834 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2835 mdname(conf->mddev)); 2836 return 0; 2837 } 2838 return r_sector; 2839 } 2840 2841 static void 2842 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2843 int rcw, int expand) 2844 { 2845 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; 2846 struct r5conf *conf = sh->raid_conf; 2847 int level = conf->level; 2848 2849 if (rcw) { 2850 2851 for (i = disks; i--; ) { 2852 struct r5dev *dev = &sh->dev[i]; 2853 2854 if (dev->towrite) { 2855 set_bit(R5_LOCKED, &dev->flags); 2856 set_bit(R5_Wantdrain, &dev->flags); 2857 if (!expand) 2858 clear_bit(R5_UPTODATE, &dev->flags); 2859 s->locked++; 2860 } 2861 } 2862 /* if we are not expanding this is a proper write request, and 2863 * there will be bios with new data to be drained into the 2864 * stripe cache 2865 */ 2866 if (!expand) { 2867 if (!s->locked) 2868 /* False alarm, nothing to do */ 2869 return; 2870 sh->reconstruct_state = reconstruct_state_drain_run; 2871 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2872 } else 2873 sh->reconstruct_state = reconstruct_state_run; 2874 2875 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2876 2877 if (s->locked + conf->max_degraded == disks) 2878 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2879 atomic_inc(&conf->pending_full_writes); 2880 } else { 2881 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2882 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2883 BUG_ON(level == 6 && 2884 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || 2885 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); 2886 2887 for (i = disks; i--; ) { 2888 struct r5dev *dev = &sh->dev[i]; 2889 if (i == pd_idx || i == qd_idx) 2890 continue; 2891 2892 if (dev->towrite && 2893 (test_bit(R5_UPTODATE, &dev->flags) || 2894 test_bit(R5_Wantcompute, &dev->flags))) { 2895 set_bit(R5_Wantdrain, &dev->flags); 2896 set_bit(R5_LOCKED, &dev->flags); 2897 clear_bit(R5_UPTODATE, &dev->flags); 2898 s->locked++; 2899 } 2900 } 2901 if (!s->locked) 2902 /* False alarm - nothing to do */ 2903 return; 2904 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2905 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2906 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2907 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2908 } 2909 2910 /* keep the parity disk(s) locked while asynchronous operations 2911 * are in flight 2912 */ 2913 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2914 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2915 s->locked++; 2916 2917 if (level == 6) { 2918 int qd_idx = sh->qd_idx; 2919 struct r5dev *dev = &sh->dev[qd_idx]; 2920 2921 set_bit(R5_LOCKED, &dev->flags); 2922 clear_bit(R5_UPTODATE, &dev->flags); 2923 s->locked++; 2924 } 2925 2926 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2927 __func__, (unsigned long long)sh->sector, 2928 s->locked, s->ops_request); 2929 } 2930 2931 /* 2932 * Each stripe/dev can have one or more bion attached. 2933 * toread/towrite point to the first in a chain. 2934 * The bi_next chain must be in order. 2935 */ 2936 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, 2937 int forwrite, int previous) 2938 { 2939 struct bio **bip; 2940 struct r5conf *conf = sh->raid_conf; 2941 int firstwrite=0; 2942 2943 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2944 (unsigned long long)bi->bi_iter.bi_sector, 2945 (unsigned long long)sh->sector); 2946 2947 /* 2948 * If several bio share a stripe. The bio bi_phys_segments acts as a 2949 * reference count to avoid race. The reference count should already be 2950 * increased before this function is called (for example, in 2951 * make_request()), so other bio sharing this stripe will not free the 2952 * stripe. If a stripe is owned by one stripe, the stripe lock will 2953 * protect it. 2954 */ 2955 spin_lock_irq(&sh->stripe_lock); 2956 /* Don't allow new IO added to stripes in batch list */ 2957 if (sh->batch_head) 2958 goto overlap; 2959 if (forwrite) { 2960 bip = &sh->dev[dd_idx].towrite; 2961 if (*bip == NULL) 2962 firstwrite = 1; 2963 } else 2964 bip = &sh->dev[dd_idx].toread; 2965 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { 2966 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) 2967 goto overlap; 2968 bip = & (*bip)->bi_next; 2969 } 2970 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) 2971 goto overlap; 2972 2973 if (!forwrite || previous) 2974 clear_bit(STRIPE_BATCH_READY, &sh->state); 2975 2976 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2977 if (*bip) 2978 bi->bi_next = *bip; 2979 *bip = bi; 2980 raid5_inc_bi_active_stripes(bi); 2981 2982 if (forwrite) { 2983 /* check if page is covered */ 2984 sector_t sector = sh->dev[dd_idx].sector; 2985 for (bi=sh->dev[dd_idx].towrite; 2986 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2987 bi && bi->bi_iter.bi_sector <= sector; 2988 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2989 if (bio_end_sector(bi) >= sector) 2990 sector = bio_end_sector(bi); 2991 } 2992 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2993 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) 2994 sh->overwrite_disks++; 2995 } 2996 2997 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2998 (unsigned long long)(*bip)->bi_iter.bi_sector, 2999 (unsigned long long)sh->sector, dd_idx); 3000 3001 if (conf->mddev->bitmap && firstwrite) { 3002 /* Cannot hold spinlock over bitmap_startwrite, 3003 * but must ensure this isn't added to a batch until 3004 * we have added to the bitmap and set bm_seq. 3005 * So set STRIPE_BITMAP_PENDING to prevent 3006 * batching. 3007 * If multiple add_stripe_bio() calls race here they 3008 * much all set STRIPE_BITMAP_PENDING. So only the first one 3009 * to complete "bitmap_startwrite" gets to set 3010 * STRIPE_BIT_DELAY. This is important as once a stripe 3011 * is added to a batch, STRIPE_BIT_DELAY cannot be changed 3012 * any more. 3013 */ 3014 set_bit(STRIPE_BITMAP_PENDING, &sh->state); 3015 spin_unlock_irq(&sh->stripe_lock); 3016 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 3017 STRIPE_SECTORS, 0); 3018 spin_lock_irq(&sh->stripe_lock); 3019 clear_bit(STRIPE_BITMAP_PENDING, &sh->state); 3020 if (!sh->batch_head) { 3021 sh->bm_seq = conf->seq_flush+1; 3022 set_bit(STRIPE_BIT_DELAY, &sh->state); 3023 } 3024 } 3025 spin_unlock_irq(&sh->stripe_lock); 3026 3027 if (stripe_can_batch(sh)) 3028 stripe_add_to_batch_list(conf, sh); 3029 return 1; 3030 3031 overlap: 3032 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 3033 spin_unlock_irq(&sh->stripe_lock); 3034 return 0; 3035 } 3036 3037 static void end_reshape(struct r5conf *conf); 3038 3039 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 3040 struct stripe_head *sh) 3041 { 3042 int sectors_per_chunk = 3043 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 3044 int dd_idx; 3045 int chunk_offset = sector_div(stripe, sectors_per_chunk); 3046 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 3047 3048 raid5_compute_sector(conf, 3049 stripe * (disks - conf->max_degraded) 3050 *sectors_per_chunk + chunk_offset, 3051 previous, 3052 &dd_idx, sh); 3053 } 3054 3055 static void 3056 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 3057 struct stripe_head_state *s, int disks, 3058 struct bio **return_bi) 3059 { 3060 int i; 3061 BUG_ON(sh->batch_head); 3062 for (i = disks; i--; ) { 3063 struct bio *bi; 3064 int bitmap_end = 0; 3065 3066 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 3067 struct md_rdev *rdev; 3068 rcu_read_lock(); 3069 rdev = rcu_dereference(conf->disks[i].rdev); 3070 if (rdev && test_bit(In_sync, &rdev->flags)) 3071 atomic_inc(&rdev->nr_pending); 3072 else 3073 rdev = NULL; 3074 rcu_read_unlock(); 3075 if (rdev) { 3076 if (!rdev_set_badblocks( 3077 rdev, 3078 sh->sector, 3079 STRIPE_SECTORS, 0)) 3080 md_error(conf->mddev, rdev); 3081 rdev_dec_pending(rdev, conf->mddev); 3082 } 3083 } 3084 spin_lock_irq(&sh->stripe_lock); 3085 /* fail all writes first */ 3086 bi = sh->dev[i].towrite; 3087 sh->dev[i].towrite = NULL; 3088 sh->overwrite_disks = 0; 3089 spin_unlock_irq(&sh->stripe_lock); 3090 if (bi) 3091 bitmap_end = 1; 3092 3093 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 3094 wake_up(&conf->wait_for_overlap); 3095 3096 while (bi && bi->bi_iter.bi_sector < 3097 sh->dev[i].sector + STRIPE_SECTORS) { 3098 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 3099 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3100 if (!raid5_dec_bi_active_stripes(bi)) { 3101 md_write_end(conf->mddev); 3102 bi->bi_next = *return_bi; 3103 *return_bi = bi; 3104 } 3105 bi = nextbi; 3106 } 3107 if (bitmap_end) 3108 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3109 STRIPE_SECTORS, 0, 0); 3110 bitmap_end = 0; 3111 /* and fail all 'written' */ 3112 bi = sh->dev[i].written; 3113 sh->dev[i].written = NULL; 3114 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { 3115 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 3116 sh->dev[i].page = sh->dev[i].orig_page; 3117 } 3118 3119 if (bi) bitmap_end = 1; 3120 while (bi && bi->bi_iter.bi_sector < 3121 sh->dev[i].sector + STRIPE_SECTORS) { 3122 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 3123 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3124 if (!raid5_dec_bi_active_stripes(bi)) { 3125 md_write_end(conf->mddev); 3126 bi->bi_next = *return_bi; 3127 *return_bi = bi; 3128 } 3129 bi = bi2; 3130 } 3131 3132 /* fail any reads if this device is non-operational and 3133 * the data has not reached the cache yet. 3134 */ 3135 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 3136 (!test_bit(R5_Insync, &sh->dev[i].flags) || 3137 test_bit(R5_ReadError, &sh->dev[i].flags))) { 3138 spin_lock_irq(&sh->stripe_lock); 3139 bi = sh->dev[i].toread; 3140 sh->dev[i].toread = NULL; 3141 spin_unlock_irq(&sh->stripe_lock); 3142 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 3143 wake_up(&conf->wait_for_overlap); 3144 while (bi && bi->bi_iter.bi_sector < 3145 sh->dev[i].sector + STRIPE_SECTORS) { 3146 struct bio *nextbi = 3147 r5_next_bio(bi, sh->dev[i].sector); 3148 clear_bit(BIO_UPTODATE, &bi->bi_flags); 3149 if (!raid5_dec_bi_active_stripes(bi)) { 3150 bi->bi_next = *return_bi; 3151 *return_bi = bi; 3152 } 3153 bi = nextbi; 3154 } 3155 } 3156 if (bitmap_end) 3157 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3158 STRIPE_SECTORS, 0, 0); 3159 /* If we were in the middle of a write the parity block might 3160 * still be locked - so just clear all R5_LOCKED flags 3161 */ 3162 clear_bit(R5_LOCKED, &sh->dev[i].flags); 3163 } 3164 3165 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3166 if (atomic_dec_and_test(&conf->pending_full_writes)) 3167 md_wakeup_thread(conf->mddev->thread); 3168 } 3169 3170 static void 3171 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 3172 struct stripe_head_state *s) 3173 { 3174 int abort = 0; 3175 int i; 3176 3177 BUG_ON(sh->batch_head); 3178 clear_bit(STRIPE_SYNCING, &sh->state); 3179 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 3180 wake_up(&conf->wait_for_overlap); 3181 s->syncing = 0; 3182 s->replacing = 0; 3183 /* There is nothing more to do for sync/check/repair. 3184 * Don't even need to abort as that is handled elsewhere 3185 * if needed, and not always wanted e.g. if there is a known 3186 * bad block here. 3187 * For recover/replace we need to record a bad block on all 3188 * non-sync devices, or abort the recovery 3189 */ 3190 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 3191 /* During recovery devices cannot be removed, so 3192 * locking and refcounting of rdevs is not needed 3193 */ 3194 for (i = 0; i < conf->raid_disks; i++) { 3195 struct md_rdev *rdev = conf->disks[i].rdev; 3196 if (rdev 3197 && !test_bit(Faulty, &rdev->flags) 3198 && !test_bit(In_sync, &rdev->flags) 3199 && !rdev_set_badblocks(rdev, sh->sector, 3200 STRIPE_SECTORS, 0)) 3201 abort = 1; 3202 rdev = conf->disks[i].replacement; 3203 if (rdev 3204 && !test_bit(Faulty, &rdev->flags) 3205 && !test_bit(In_sync, &rdev->flags) 3206 && !rdev_set_badblocks(rdev, sh->sector, 3207 STRIPE_SECTORS, 0)) 3208 abort = 1; 3209 } 3210 if (abort) 3211 conf->recovery_disabled = 3212 conf->mddev->recovery_disabled; 3213 } 3214 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 3215 } 3216 3217 static int want_replace(struct stripe_head *sh, int disk_idx) 3218 { 3219 struct md_rdev *rdev; 3220 int rv = 0; 3221 /* Doing recovery so rcu locking not required */ 3222 rdev = sh->raid_conf->disks[disk_idx].replacement; 3223 if (rdev 3224 && !test_bit(Faulty, &rdev->flags) 3225 && !test_bit(In_sync, &rdev->flags) 3226 && (rdev->recovery_offset <= sh->sector 3227 || rdev->mddev->recovery_cp <= sh->sector)) 3228 rv = 1; 3229 3230 return rv; 3231 } 3232 3233 /* fetch_block - checks the given member device to see if its data needs 3234 * to be read or computed to satisfy a request. 3235 * 3236 * Returns 1 when no more member devices need to be checked, otherwise returns 3237 * 0 to tell the loop in handle_stripe_fill to continue 3238 */ 3239 3240 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, 3241 int disk_idx, int disks) 3242 { 3243 struct r5dev *dev = &sh->dev[disk_idx]; 3244 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 3245 &sh->dev[s->failed_num[1]] }; 3246 int i; 3247 3248 3249 if (test_bit(R5_LOCKED, &dev->flags) || 3250 test_bit(R5_UPTODATE, &dev->flags)) 3251 /* No point reading this as we already have it or have 3252 * decided to get it. 3253 */ 3254 return 0; 3255 3256 if (dev->toread || 3257 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) 3258 /* We need this block to directly satisfy a request */ 3259 return 1; 3260 3261 if (s->syncing || s->expanding || 3262 (s->replacing && want_replace(sh, disk_idx))) 3263 /* When syncing, or expanding we read everything. 3264 * When replacing, we need the replaced block. 3265 */ 3266 return 1; 3267 3268 if ((s->failed >= 1 && fdev[0]->toread) || 3269 (s->failed >= 2 && fdev[1]->toread)) 3270 /* If we want to read from a failed device, then 3271 * we need to actually read every other device. 3272 */ 3273 return 1; 3274 3275 /* Sometimes neither read-modify-write nor reconstruct-write 3276 * cycles can work. In those cases we read every block we 3277 * can. Then the parity-update is certain to have enough to 3278 * work with. 3279 * This can only be a problem when we need to write something, 3280 * and some device has failed. If either of those tests 3281 * fail we need look no further. 3282 */ 3283 if (!s->failed || !s->to_write) 3284 return 0; 3285 3286 if (test_bit(R5_Insync, &dev->flags) && 3287 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3288 /* Pre-reads at not permitted until after short delay 3289 * to gather multiple requests. However if this 3290 * device is no Insync, the block could only be be computed 3291 * and there is no need to delay that. 3292 */ 3293 return 0; 3294 3295 for (i = 0; i < s->failed; i++) { 3296 if (fdev[i]->towrite && 3297 !test_bit(R5_UPTODATE, &fdev[i]->flags) && 3298 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 3299 /* If we have a partial write to a failed 3300 * device, then we will need to reconstruct 3301 * the content of that device, so all other 3302 * devices must be read. 3303 */ 3304 return 1; 3305 } 3306 3307 /* If we are forced to do a reconstruct-write, either because 3308 * the current RAID6 implementation only supports that, or 3309 * or because parity cannot be trusted and we are currently 3310 * recovering it, there is extra need to be careful. 3311 * If one of the devices that we would need to read, because 3312 * it is not being overwritten (and maybe not written at all) 3313 * is missing/faulty, then we need to read everything we can. 3314 */ 3315 if (sh->raid_conf->level != 6 && 3316 sh->sector < sh->raid_conf->mddev->recovery_cp) 3317 /* reconstruct-write isn't being forced */ 3318 return 0; 3319 for (i = 0; i < s->failed; i++) { 3320 if (s->failed_num[i] != sh->pd_idx && 3321 s->failed_num[i] != sh->qd_idx && 3322 !test_bit(R5_UPTODATE, &fdev[i]->flags) && 3323 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 3324 return 1; 3325 } 3326 3327 return 0; 3328 } 3329 3330 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 3331 int disk_idx, int disks) 3332 { 3333 struct r5dev *dev = &sh->dev[disk_idx]; 3334 3335 /* is the data in this block needed, and can we get it? */ 3336 if (need_this_block(sh, s, disk_idx, disks)) { 3337 /* we would like to get this block, possibly by computing it, 3338 * otherwise read it if the backing disk is insync 3339 */ 3340 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 3341 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 3342 BUG_ON(sh->batch_head); 3343 if ((s->uptodate == disks - 1) && 3344 (s->failed && (disk_idx == s->failed_num[0] || 3345 disk_idx == s->failed_num[1]))) { 3346 /* have disk failed, and we're requested to fetch it; 3347 * do compute it 3348 */ 3349 pr_debug("Computing stripe %llu block %d\n", 3350 (unsigned long long)sh->sector, disk_idx); 3351 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3352 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3353 set_bit(R5_Wantcompute, &dev->flags); 3354 sh->ops.target = disk_idx; 3355 sh->ops.target2 = -1; /* no 2nd target */ 3356 s->req_compute = 1; 3357 /* Careful: from this point on 'uptodate' is in the eye 3358 * of raid_run_ops which services 'compute' operations 3359 * before writes. R5_Wantcompute flags a block that will 3360 * be R5_UPTODATE by the time it is needed for a 3361 * subsequent operation. 3362 */ 3363 s->uptodate++; 3364 return 1; 3365 } else if (s->uptodate == disks-2 && s->failed >= 2) { 3366 /* Computing 2-failure is *very* expensive; only 3367 * do it if failed >= 2 3368 */ 3369 int other; 3370 for (other = disks; other--; ) { 3371 if (other == disk_idx) 3372 continue; 3373 if (!test_bit(R5_UPTODATE, 3374 &sh->dev[other].flags)) 3375 break; 3376 } 3377 BUG_ON(other < 0); 3378 pr_debug("Computing stripe %llu blocks %d,%d\n", 3379 (unsigned long long)sh->sector, 3380 disk_idx, other); 3381 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3382 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3383 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 3384 set_bit(R5_Wantcompute, &sh->dev[other].flags); 3385 sh->ops.target = disk_idx; 3386 sh->ops.target2 = other; 3387 s->uptodate += 2; 3388 s->req_compute = 1; 3389 return 1; 3390 } else if (test_bit(R5_Insync, &dev->flags)) { 3391 set_bit(R5_LOCKED, &dev->flags); 3392 set_bit(R5_Wantread, &dev->flags); 3393 s->locked++; 3394 pr_debug("Reading block %d (sync=%d)\n", 3395 disk_idx, s->syncing); 3396 } 3397 } 3398 3399 return 0; 3400 } 3401 3402 /** 3403 * handle_stripe_fill - read or compute data to satisfy pending requests. 3404 */ 3405 static void handle_stripe_fill(struct stripe_head *sh, 3406 struct stripe_head_state *s, 3407 int disks) 3408 { 3409 int i; 3410 3411 /* look for blocks to read/compute, skip this if a compute 3412 * is already in flight, or if the stripe contents are in the 3413 * midst of changing due to a write 3414 */ 3415 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 3416 !sh->reconstruct_state) 3417 for (i = disks; i--; ) 3418 if (fetch_block(sh, s, i, disks)) 3419 break; 3420 set_bit(STRIPE_HANDLE, &sh->state); 3421 } 3422 3423 static void break_stripe_batch_list(struct stripe_head *head_sh, 3424 unsigned long handle_flags); 3425 /* handle_stripe_clean_event 3426 * any written block on an uptodate or failed drive can be returned. 3427 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 3428 * never LOCKED, so we don't need to test 'failed' directly. 3429 */ 3430 static void handle_stripe_clean_event(struct r5conf *conf, 3431 struct stripe_head *sh, int disks, struct bio **return_bi) 3432 { 3433 int i; 3434 struct r5dev *dev; 3435 int discard_pending = 0; 3436 struct stripe_head *head_sh = sh; 3437 bool do_endio = false; 3438 3439 for (i = disks; i--; ) 3440 if (sh->dev[i].written) { 3441 dev = &sh->dev[i]; 3442 if (!test_bit(R5_LOCKED, &dev->flags) && 3443 (test_bit(R5_UPTODATE, &dev->flags) || 3444 test_bit(R5_Discard, &dev->flags) || 3445 test_bit(R5_SkipCopy, &dev->flags))) { 3446 /* We can return any write requests */ 3447 struct bio *wbi, *wbi2; 3448 pr_debug("Return write for disc %d\n", i); 3449 if (test_and_clear_bit(R5_Discard, &dev->flags)) 3450 clear_bit(R5_UPTODATE, &dev->flags); 3451 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { 3452 WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); 3453 } 3454 do_endio = true; 3455 3456 returnbi: 3457 dev->page = dev->orig_page; 3458 wbi = dev->written; 3459 dev->written = NULL; 3460 while (wbi && wbi->bi_iter.bi_sector < 3461 dev->sector + STRIPE_SECTORS) { 3462 wbi2 = r5_next_bio(wbi, dev->sector); 3463 if (!raid5_dec_bi_active_stripes(wbi)) { 3464 md_write_end(conf->mddev); 3465 wbi->bi_next = *return_bi; 3466 *return_bi = wbi; 3467 } 3468 wbi = wbi2; 3469 } 3470 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3471 STRIPE_SECTORS, 3472 !test_bit(STRIPE_DEGRADED, &sh->state), 3473 0); 3474 if (head_sh->batch_head) { 3475 sh = list_first_entry(&sh->batch_list, 3476 struct stripe_head, 3477 batch_list); 3478 if (sh != head_sh) { 3479 dev = &sh->dev[i]; 3480 goto returnbi; 3481 } 3482 } 3483 sh = head_sh; 3484 dev = &sh->dev[i]; 3485 } else if (test_bit(R5_Discard, &dev->flags)) 3486 discard_pending = 1; 3487 WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); 3488 WARN_ON(dev->page != dev->orig_page); 3489 } 3490 if (!discard_pending && 3491 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { 3492 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 3493 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3494 if (sh->qd_idx >= 0) { 3495 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 3496 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); 3497 } 3498 /* now that discard is done we can proceed with any sync */ 3499 clear_bit(STRIPE_DISCARD, &sh->state); 3500 /* 3501 * SCSI discard will change some bio fields and the stripe has 3502 * no updated data, so remove it from hash list and the stripe 3503 * will be reinitialized 3504 */ 3505 spin_lock_irq(&conf->device_lock); 3506 unhash: 3507 remove_hash(sh); 3508 if (head_sh->batch_head) { 3509 sh = list_first_entry(&sh->batch_list, 3510 struct stripe_head, batch_list); 3511 if (sh != head_sh) 3512 goto unhash; 3513 } 3514 spin_unlock_irq(&conf->device_lock); 3515 sh = head_sh; 3516 3517 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) 3518 set_bit(STRIPE_HANDLE, &sh->state); 3519 3520 } 3521 3522 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3523 if (atomic_dec_and_test(&conf->pending_full_writes)) 3524 md_wakeup_thread(conf->mddev->thread); 3525 3526 if (head_sh->batch_head && do_endio) 3527 break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS); 3528 } 3529 3530 static void handle_stripe_dirtying(struct r5conf *conf, 3531 struct stripe_head *sh, 3532 struct stripe_head_state *s, 3533 int disks) 3534 { 3535 int rmw = 0, rcw = 0, i; 3536 sector_t recovery_cp = conf->mddev->recovery_cp; 3537 3538 /* Check whether resync is now happening or should start. 3539 * If yes, then the array is dirty (after unclean shutdown or 3540 * initial creation), so parity in some stripes might be inconsistent. 3541 * In this case, we need to always do reconstruct-write, to ensure 3542 * that in case of drive failure or read-error correction, we 3543 * generate correct data from the parity. 3544 */ 3545 if (conf->rmw_level == PARITY_DISABLE_RMW || 3546 (recovery_cp < MaxSector && sh->sector >= recovery_cp && 3547 s->failed == 0)) { 3548 /* Calculate the real rcw later - for now make it 3549 * look like rcw is cheaper 3550 */ 3551 rcw = 1; rmw = 2; 3552 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", 3553 conf->rmw_level, (unsigned long long)recovery_cp, 3554 (unsigned long long)sh->sector); 3555 } else for (i = disks; i--; ) { 3556 /* would I have to read this buffer for read_modify_write */ 3557 struct r5dev *dev = &sh->dev[i]; 3558 if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && 3559 !test_bit(R5_LOCKED, &dev->flags) && 3560 !(test_bit(R5_UPTODATE, &dev->flags) || 3561 test_bit(R5_Wantcompute, &dev->flags))) { 3562 if (test_bit(R5_Insync, &dev->flags)) 3563 rmw++; 3564 else 3565 rmw += 2*disks; /* cannot read it */ 3566 } 3567 /* Would I have to read this buffer for reconstruct_write */ 3568 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3569 i != sh->pd_idx && i != sh->qd_idx && 3570 !test_bit(R5_LOCKED, &dev->flags) && 3571 !(test_bit(R5_UPTODATE, &dev->flags) || 3572 test_bit(R5_Wantcompute, &dev->flags))) { 3573 if (test_bit(R5_Insync, &dev->flags)) 3574 rcw++; 3575 else 3576 rcw += 2*disks; 3577 } 3578 } 3579 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 3580 (unsigned long long)sh->sector, rmw, rcw); 3581 set_bit(STRIPE_HANDLE, &sh->state); 3582 if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) { 3583 /* prefer read-modify-write, but need to get some data */ 3584 if (conf->mddev->queue) 3585 blk_add_trace_msg(conf->mddev->queue, 3586 "raid5 rmw %llu %d", 3587 (unsigned long long)sh->sector, rmw); 3588 for (i = disks; i--; ) { 3589 struct r5dev *dev = &sh->dev[i]; 3590 if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && 3591 !test_bit(R5_LOCKED, &dev->flags) && 3592 !(test_bit(R5_UPTODATE, &dev->flags) || 3593 test_bit(R5_Wantcompute, &dev->flags)) && 3594 test_bit(R5_Insync, &dev->flags)) { 3595 if (test_bit(STRIPE_PREREAD_ACTIVE, 3596 &sh->state)) { 3597 pr_debug("Read_old block %d for r-m-w\n", 3598 i); 3599 set_bit(R5_LOCKED, &dev->flags); 3600 set_bit(R5_Wantread, &dev->flags); 3601 s->locked++; 3602 } else { 3603 set_bit(STRIPE_DELAYED, &sh->state); 3604 set_bit(STRIPE_HANDLE, &sh->state); 3605 } 3606 } 3607 } 3608 } 3609 if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) { 3610 /* want reconstruct write, but need to get some data */ 3611 int qread =0; 3612 rcw = 0; 3613 for (i = disks; i--; ) { 3614 struct r5dev *dev = &sh->dev[i]; 3615 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3616 i != sh->pd_idx && i != sh->qd_idx && 3617 !test_bit(R5_LOCKED, &dev->flags) && 3618 !(test_bit(R5_UPTODATE, &dev->flags) || 3619 test_bit(R5_Wantcompute, &dev->flags))) { 3620 rcw++; 3621 if (test_bit(R5_Insync, &dev->flags) && 3622 test_bit(STRIPE_PREREAD_ACTIVE, 3623 &sh->state)) { 3624 pr_debug("Read_old block " 3625 "%d for Reconstruct\n", i); 3626 set_bit(R5_LOCKED, &dev->flags); 3627 set_bit(R5_Wantread, &dev->flags); 3628 s->locked++; 3629 qread++; 3630 } else { 3631 set_bit(STRIPE_DELAYED, &sh->state); 3632 set_bit(STRIPE_HANDLE, &sh->state); 3633 } 3634 } 3635 } 3636 if (rcw && conf->mddev->queue) 3637 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", 3638 (unsigned long long)sh->sector, 3639 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); 3640 } 3641 3642 if (rcw > disks && rmw > disks && 3643 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3644 set_bit(STRIPE_DELAYED, &sh->state); 3645 3646 /* now if nothing is locked, and if we have enough data, 3647 * we can start a write request 3648 */ 3649 /* since handle_stripe can be called at any time we need to handle the 3650 * case where a compute block operation has been submitted and then a 3651 * subsequent call wants to start a write request. raid_run_ops only 3652 * handles the case where compute block and reconstruct are requested 3653 * simultaneously. If this is not the case then new writes need to be 3654 * held off until the compute completes. 3655 */ 3656 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 3657 (s->locked == 0 && (rcw == 0 || rmw == 0) && 3658 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 3659 schedule_reconstruction(sh, s, rcw == 0, 0); 3660 } 3661 3662 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 3663 struct stripe_head_state *s, int disks) 3664 { 3665 struct r5dev *dev = NULL; 3666 3667 BUG_ON(sh->batch_head); 3668 set_bit(STRIPE_HANDLE, &sh->state); 3669 3670 switch (sh->check_state) { 3671 case check_state_idle: 3672 /* start a new check operation if there are no failures */ 3673 if (s->failed == 0) { 3674 BUG_ON(s->uptodate != disks); 3675 sh->check_state = check_state_run; 3676 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3677 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3678 s->uptodate--; 3679 break; 3680 } 3681 dev = &sh->dev[s->failed_num[0]]; 3682 /* fall through */ 3683 case check_state_compute_result: 3684 sh->check_state = check_state_idle; 3685 if (!dev) 3686 dev = &sh->dev[sh->pd_idx]; 3687 3688 /* check that a write has not made the stripe insync */ 3689 if (test_bit(STRIPE_INSYNC, &sh->state)) 3690 break; 3691 3692 /* either failed parity check, or recovery is happening */ 3693 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3694 BUG_ON(s->uptodate != disks); 3695 3696 set_bit(R5_LOCKED, &dev->flags); 3697 s->locked++; 3698 set_bit(R5_Wantwrite, &dev->flags); 3699 3700 clear_bit(STRIPE_DEGRADED, &sh->state); 3701 set_bit(STRIPE_INSYNC, &sh->state); 3702 break; 3703 case check_state_run: 3704 break; /* we will be called again upon completion */ 3705 case check_state_check_result: 3706 sh->check_state = check_state_idle; 3707 3708 /* if a failure occurred during the check operation, leave 3709 * STRIPE_INSYNC not set and let the stripe be handled again 3710 */ 3711 if (s->failed) 3712 break; 3713 3714 /* handle a successful check operation, if parity is correct 3715 * we are done. Otherwise update the mismatch count and repair 3716 * parity if !MD_RECOVERY_CHECK 3717 */ 3718 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 3719 /* parity is correct (on disc, 3720 * not in buffer any more) 3721 */ 3722 set_bit(STRIPE_INSYNC, &sh->state); 3723 else { 3724 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3725 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3726 /* don't try to repair!! */ 3727 set_bit(STRIPE_INSYNC, &sh->state); 3728 else { 3729 sh->check_state = check_state_compute_run; 3730 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3731 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3732 set_bit(R5_Wantcompute, 3733 &sh->dev[sh->pd_idx].flags); 3734 sh->ops.target = sh->pd_idx; 3735 sh->ops.target2 = -1; 3736 s->uptodate++; 3737 } 3738 } 3739 break; 3740 case check_state_compute_run: 3741 break; 3742 default: 3743 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3744 __func__, sh->check_state, 3745 (unsigned long long) sh->sector); 3746 BUG(); 3747 } 3748 } 3749 3750 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 3751 struct stripe_head_state *s, 3752 int disks) 3753 { 3754 int pd_idx = sh->pd_idx; 3755 int qd_idx = sh->qd_idx; 3756 struct r5dev *dev; 3757 3758 BUG_ON(sh->batch_head); 3759 set_bit(STRIPE_HANDLE, &sh->state); 3760 3761 BUG_ON(s->failed > 2); 3762 3763 /* Want to check and possibly repair P and Q. 3764 * However there could be one 'failed' device, in which 3765 * case we can only check one of them, possibly using the 3766 * other to generate missing data 3767 */ 3768 3769 switch (sh->check_state) { 3770 case check_state_idle: 3771 /* start a new check operation if there are < 2 failures */ 3772 if (s->failed == s->q_failed) { 3773 /* The only possible failed device holds Q, so it 3774 * makes sense to check P (If anything else were failed, 3775 * we would have used P to recreate it). 3776 */ 3777 sh->check_state = check_state_run; 3778 } 3779 if (!s->q_failed && s->failed < 2) { 3780 /* Q is not failed, and we didn't use it to generate 3781 * anything, so it makes sense to check it 3782 */ 3783 if (sh->check_state == check_state_run) 3784 sh->check_state = check_state_run_pq; 3785 else 3786 sh->check_state = check_state_run_q; 3787 } 3788 3789 /* discard potentially stale zero_sum_result */ 3790 sh->ops.zero_sum_result = 0; 3791 3792 if (sh->check_state == check_state_run) { 3793 /* async_xor_zero_sum destroys the contents of P */ 3794 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 3795 s->uptodate--; 3796 } 3797 if (sh->check_state >= check_state_run && 3798 sh->check_state <= check_state_run_pq) { 3799 /* async_syndrome_zero_sum preserves P and Q, so 3800 * no need to mark them !uptodate here 3801 */ 3802 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3803 break; 3804 } 3805 3806 /* we have 2-disk failure */ 3807 BUG_ON(s->failed != 2); 3808 /* fall through */ 3809 case check_state_compute_result: 3810 sh->check_state = check_state_idle; 3811 3812 /* check that a write has not made the stripe insync */ 3813 if (test_bit(STRIPE_INSYNC, &sh->state)) 3814 break; 3815 3816 /* now write out any block on a failed drive, 3817 * or P or Q if they were recomputed 3818 */ 3819 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 3820 if (s->failed == 2) { 3821 dev = &sh->dev[s->failed_num[1]]; 3822 s->locked++; 3823 set_bit(R5_LOCKED, &dev->flags); 3824 set_bit(R5_Wantwrite, &dev->flags); 3825 } 3826 if (s->failed >= 1) { 3827 dev = &sh->dev[s->failed_num[0]]; 3828 s->locked++; 3829 set_bit(R5_LOCKED, &dev->flags); 3830 set_bit(R5_Wantwrite, &dev->flags); 3831 } 3832 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3833 dev = &sh->dev[pd_idx]; 3834 s->locked++; 3835 set_bit(R5_LOCKED, &dev->flags); 3836 set_bit(R5_Wantwrite, &dev->flags); 3837 } 3838 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3839 dev = &sh->dev[qd_idx]; 3840 s->locked++; 3841 set_bit(R5_LOCKED, &dev->flags); 3842 set_bit(R5_Wantwrite, &dev->flags); 3843 } 3844 clear_bit(STRIPE_DEGRADED, &sh->state); 3845 3846 set_bit(STRIPE_INSYNC, &sh->state); 3847 break; 3848 case check_state_run: 3849 case check_state_run_q: 3850 case check_state_run_pq: 3851 break; /* we will be called again upon completion */ 3852 case check_state_check_result: 3853 sh->check_state = check_state_idle; 3854 3855 /* handle a successful check operation, if parity is correct 3856 * we are done. Otherwise update the mismatch count and repair 3857 * parity if !MD_RECOVERY_CHECK 3858 */ 3859 if (sh->ops.zero_sum_result == 0) { 3860 /* both parities are correct */ 3861 if (!s->failed) 3862 set_bit(STRIPE_INSYNC, &sh->state); 3863 else { 3864 /* in contrast to the raid5 case we can validate 3865 * parity, but still have a failure to write 3866 * back 3867 */ 3868 sh->check_state = check_state_compute_result; 3869 /* Returning at this point means that we may go 3870 * off and bring p and/or q uptodate again so 3871 * we make sure to check zero_sum_result again 3872 * to verify if p or q need writeback 3873 */ 3874 } 3875 } else { 3876 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3877 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3878 /* don't try to repair!! */ 3879 set_bit(STRIPE_INSYNC, &sh->state); 3880 else { 3881 int *target = &sh->ops.target; 3882 3883 sh->ops.target = -1; 3884 sh->ops.target2 = -1; 3885 sh->check_state = check_state_compute_run; 3886 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3887 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3888 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3889 set_bit(R5_Wantcompute, 3890 &sh->dev[pd_idx].flags); 3891 *target = pd_idx; 3892 target = &sh->ops.target2; 3893 s->uptodate++; 3894 } 3895 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3896 set_bit(R5_Wantcompute, 3897 &sh->dev[qd_idx].flags); 3898 *target = qd_idx; 3899 s->uptodate++; 3900 } 3901 } 3902 } 3903 break; 3904 case check_state_compute_run: 3905 break; 3906 default: 3907 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3908 __func__, sh->check_state, 3909 (unsigned long long) sh->sector); 3910 BUG(); 3911 } 3912 } 3913 3914 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 3915 { 3916 int i; 3917 3918 /* We have read all the blocks in this stripe and now we need to 3919 * copy some of them into a target stripe for expand. 3920 */ 3921 struct dma_async_tx_descriptor *tx = NULL; 3922 BUG_ON(sh->batch_head); 3923 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3924 for (i = 0; i < sh->disks; i++) 3925 if (i != sh->pd_idx && i != sh->qd_idx) { 3926 int dd_idx, j; 3927 struct stripe_head *sh2; 3928 struct async_submit_ctl submit; 3929 3930 sector_t bn = compute_blocknr(sh, i, 1); 3931 sector_t s = raid5_compute_sector(conf, bn, 0, 3932 &dd_idx, NULL); 3933 sh2 = get_active_stripe(conf, s, 0, 1, 1); 3934 if (sh2 == NULL) 3935 /* so far only the early blocks of this stripe 3936 * have been requested. When later blocks 3937 * get requested, we will try again 3938 */ 3939 continue; 3940 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 3941 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 3942 /* must have already done this block */ 3943 release_stripe(sh2); 3944 continue; 3945 } 3946 3947 /* place all the copies on one channel */ 3948 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 3949 tx = async_memcpy(sh2->dev[dd_idx].page, 3950 sh->dev[i].page, 0, 0, STRIPE_SIZE, 3951 &submit); 3952 3953 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 3954 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 3955 for (j = 0; j < conf->raid_disks; j++) 3956 if (j != sh2->pd_idx && 3957 j != sh2->qd_idx && 3958 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 3959 break; 3960 if (j == conf->raid_disks) { 3961 set_bit(STRIPE_EXPAND_READY, &sh2->state); 3962 set_bit(STRIPE_HANDLE, &sh2->state); 3963 } 3964 release_stripe(sh2); 3965 3966 } 3967 /* done submitting copies, wait for them to complete */ 3968 async_tx_quiesce(&tx); 3969 } 3970 3971 /* 3972 * handle_stripe - do things to a stripe. 3973 * 3974 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 3975 * state of various bits to see what needs to be done. 3976 * Possible results: 3977 * return some read requests which now have data 3978 * return some write requests which are safely on storage 3979 * schedule a read on some buffers 3980 * schedule a write of some buffers 3981 * return confirmation of parity correctness 3982 * 3983 */ 3984 3985 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 3986 { 3987 struct r5conf *conf = sh->raid_conf; 3988 int disks = sh->disks; 3989 struct r5dev *dev; 3990 int i; 3991 int do_recovery = 0; 3992 3993 memset(s, 0, sizeof(*s)); 3994 3995 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; 3996 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; 3997 s->failed_num[0] = -1; 3998 s->failed_num[1] = -1; 3999 4000 /* Now to look around and see what can be done */ 4001 rcu_read_lock(); 4002 for (i=disks; i--; ) { 4003 struct md_rdev *rdev; 4004 sector_t first_bad; 4005 int bad_sectors; 4006 int is_bad = 0; 4007 4008 dev = &sh->dev[i]; 4009 4010 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 4011 i, dev->flags, 4012 dev->toread, dev->towrite, dev->written); 4013 /* maybe we can reply to a read 4014 * 4015 * new wantfill requests are only permitted while 4016 * ops_complete_biofill is guaranteed to be inactive 4017 */ 4018 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 4019 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 4020 set_bit(R5_Wantfill, &dev->flags); 4021 4022 /* now count some things */ 4023 if (test_bit(R5_LOCKED, &dev->flags)) 4024 s->locked++; 4025 if (test_bit(R5_UPTODATE, &dev->flags)) 4026 s->uptodate++; 4027 if (test_bit(R5_Wantcompute, &dev->flags)) { 4028 s->compute++; 4029 BUG_ON(s->compute > 2); 4030 } 4031 4032 if (test_bit(R5_Wantfill, &dev->flags)) 4033 s->to_fill++; 4034 else if (dev->toread) 4035 s->to_read++; 4036 if (dev->towrite) { 4037 s->to_write++; 4038 if (!test_bit(R5_OVERWRITE, &dev->flags)) 4039 s->non_overwrite++; 4040 } 4041 if (dev->written) 4042 s->written++; 4043 /* Prefer to use the replacement for reads, but only 4044 * if it is recovered enough and has no bad blocks. 4045 */ 4046 rdev = rcu_dereference(conf->disks[i].replacement); 4047 if (rdev && !test_bit(Faulty, &rdev->flags) && 4048 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 4049 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 4050 &first_bad, &bad_sectors)) 4051 set_bit(R5_ReadRepl, &dev->flags); 4052 else { 4053 if (rdev) 4054 set_bit(R5_NeedReplace, &dev->flags); 4055 rdev = rcu_dereference(conf->disks[i].rdev); 4056 clear_bit(R5_ReadRepl, &dev->flags); 4057 } 4058 if (rdev && test_bit(Faulty, &rdev->flags)) 4059 rdev = NULL; 4060 if (rdev) { 4061 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 4062 &first_bad, &bad_sectors); 4063 if (s->blocked_rdev == NULL 4064 && (test_bit(Blocked, &rdev->flags) 4065 || is_bad < 0)) { 4066 if (is_bad < 0) 4067 set_bit(BlockedBadBlocks, 4068 &rdev->flags); 4069 s->blocked_rdev = rdev; 4070 atomic_inc(&rdev->nr_pending); 4071 } 4072 } 4073 clear_bit(R5_Insync, &dev->flags); 4074 if (!rdev) 4075 /* Not in-sync */; 4076 else if (is_bad) { 4077 /* also not in-sync */ 4078 if (!test_bit(WriteErrorSeen, &rdev->flags) && 4079 test_bit(R5_UPTODATE, &dev->flags)) { 4080 /* treat as in-sync, but with a read error 4081 * which we can now try to correct 4082 */ 4083 set_bit(R5_Insync, &dev->flags); 4084 set_bit(R5_ReadError, &dev->flags); 4085 } 4086 } else if (test_bit(In_sync, &rdev->flags)) 4087 set_bit(R5_Insync, &dev->flags); 4088 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 4089 /* in sync if before recovery_offset */ 4090 set_bit(R5_Insync, &dev->flags); 4091 else if (test_bit(R5_UPTODATE, &dev->flags) && 4092 test_bit(R5_Expanded, &dev->flags)) 4093 /* If we've reshaped into here, we assume it is Insync. 4094 * We will shortly update recovery_offset to make 4095 * it official. 4096 */ 4097 set_bit(R5_Insync, &dev->flags); 4098 4099 if (test_bit(R5_WriteError, &dev->flags)) { 4100 /* This flag does not apply to '.replacement' 4101 * only to .rdev, so make sure to check that*/ 4102 struct md_rdev *rdev2 = rcu_dereference( 4103 conf->disks[i].rdev); 4104 if (rdev2 == rdev) 4105 clear_bit(R5_Insync, &dev->flags); 4106 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4107 s->handle_bad_blocks = 1; 4108 atomic_inc(&rdev2->nr_pending); 4109 } else 4110 clear_bit(R5_WriteError, &dev->flags); 4111 } 4112 if (test_bit(R5_MadeGood, &dev->flags)) { 4113 /* This flag does not apply to '.replacement' 4114 * only to .rdev, so make sure to check that*/ 4115 struct md_rdev *rdev2 = rcu_dereference( 4116 conf->disks[i].rdev); 4117 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4118 s->handle_bad_blocks = 1; 4119 atomic_inc(&rdev2->nr_pending); 4120 } else 4121 clear_bit(R5_MadeGood, &dev->flags); 4122 } 4123 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 4124 struct md_rdev *rdev2 = rcu_dereference( 4125 conf->disks[i].replacement); 4126 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4127 s->handle_bad_blocks = 1; 4128 atomic_inc(&rdev2->nr_pending); 4129 } else 4130 clear_bit(R5_MadeGoodRepl, &dev->flags); 4131 } 4132 if (!test_bit(R5_Insync, &dev->flags)) { 4133 /* The ReadError flag will just be confusing now */ 4134 clear_bit(R5_ReadError, &dev->flags); 4135 clear_bit(R5_ReWrite, &dev->flags); 4136 } 4137 if (test_bit(R5_ReadError, &dev->flags)) 4138 clear_bit(R5_Insync, &dev->flags); 4139 if (!test_bit(R5_Insync, &dev->flags)) { 4140 if (s->failed < 2) 4141 s->failed_num[s->failed] = i; 4142 s->failed++; 4143 if (rdev && !test_bit(Faulty, &rdev->flags)) 4144 do_recovery = 1; 4145 } 4146 } 4147 if (test_bit(STRIPE_SYNCING, &sh->state)) { 4148 /* If there is a failed device being replaced, 4149 * we must be recovering. 4150 * else if we are after recovery_cp, we must be syncing 4151 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 4152 * else we can only be replacing 4153 * sync and recovery both need to read all devices, and so 4154 * use the same flag. 4155 */ 4156 if (do_recovery || 4157 sh->sector >= conf->mddev->recovery_cp || 4158 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 4159 s->syncing = 1; 4160 else 4161 s->replacing = 1; 4162 } 4163 rcu_read_unlock(); 4164 } 4165 4166 static int clear_batch_ready(struct stripe_head *sh) 4167 { 4168 /* Return '1' if this is a member of batch, or 4169 * '0' if it is a lone stripe or a head which can now be 4170 * handled. 4171 */ 4172 struct stripe_head *tmp; 4173 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) 4174 return (sh->batch_head && sh->batch_head != sh); 4175 spin_lock(&sh->stripe_lock); 4176 if (!sh->batch_head) { 4177 spin_unlock(&sh->stripe_lock); 4178 return 0; 4179 } 4180 4181 /* 4182 * this stripe could be added to a batch list before we check 4183 * BATCH_READY, skips it 4184 */ 4185 if (sh->batch_head != sh) { 4186 spin_unlock(&sh->stripe_lock); 4187 return 1; 4188 } 4189 spin_lock(&sh->batch_lock); 4190 list_for_each_entry(tmp, &sh->batch_list, batch_list) 4191 clear_bit(STRIPE_BATCH_READY, &tmp->state); 4192 spin_unlock(&sh->batch_lock); 4193 spin_unlock(&sh->stripe_lock); 4194 4195 /* 4196 * BATCH_READY is cleared, no new stripes can be added. 4197 * batch_list can be accessed without lock 4198 */ 4199 return 0; 4200 } 4201 4202 static void break_stripe_batch_list(struct stripe_head *head_sh, 4203 unsigned long handle_flags) 4204 { 4205 struct stripe_head *sh, *next; 4206 int i; 4207 int do_wakeup = 0; 4208 4209 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { 4210 4211 list_del_init(&sh->batch_list); 4212 4213 WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | 4214 (1 << STRIPE_SYNCING) | 4215 (1 << STRIPE_REPLACED) | 4216 (1 << STRIPE_PREREAD_ACTIVE) | 4217 (1 << STRIPE_DELAYED) | 4218 (1 << STRIPE_BIT_DELAY) | 4219 (1 << STRIPE_FULL_WRITE) | 4220 (1 << STRIPE_BIOFILL_RUN) | 4221 (1 << STRIPE_COMPUTE_RUN) | 4222 (1 << STRIPE_OPS_REQ_PENDING) | 4223 (1 << STRIPE_DISCARD) | 4224 (1 << STRIPE_BATCH_READY) | 4225 (1 << STRIPE_BATCH_ERR) | 4226 (1 << STRIPE_BITMAP_PENDING))); 4227 WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | 4228 (1 << STRIPE_REPLACED))); 4229 4230 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | 4231 (1 << STRIPE_DEGRADED)), 4232 head_sh->state & (1 << STRIPE_INSYNC)); 4233 4234 sh->check_state = head_sh->check_state; 4235 sh->reconstruct_state = head_sh->reconstruct_state; 4236 for (i = 0; i < sh->disks; i++) { 4237 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 4238 do_wakeup = 1; 4239 sh->dev[i].flags = head_sh->dev[i].flags & 4240 (~((1 << R5_WriteError) | (1 << R5_Overlap))); 4241 } 4242 spin_lock_irq(&sh->stripe_lock); 4243 sh->batch_head = NULL; 4244 spin_unlock_irq(&sh->stripe_lock); 4245 if (handle_flags == 0 || 4246 sh->state & handle_flags) 4247 set_bit(STRIPE_HANDLE, &sh->state); 4248 release_stripe(sh); 4249 } 4250 spin_lock_irq(&head_sh->stripe_lock); 4251 head_sh->batch_head = NULL; 4252 spin_unlock_irq(&head_sh->stripe_lock); 4253 for (i = 0; i < head_sh->disks; i++) 4254 if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) 4255 do_wakeup = 1; 4256 if (head_sh->state & handle_flags) 4257 set_bit(STRIPE_HANDLE, &head_sh->state); 4258 4259 if (do_wakeup) 4260 wake_up(&head_sh->raid_conf->wait_for_overlap); 4261 } 4262 4263 static void handle_stripe(struct stripe_head *sh) 4264 { 4265 struct stripe_head_state s; 4266 struct r5conf *conf = sh->raid_conf; 4267 int i; 4268 int prexor; 4269 int disks = sh->disks; 4270 struct r5dev *pdev, *qdev; 4271 4272 clear_bit(STRIPE_HANDLE, &sh->state); 4273 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 4274 /* already being handled, ensure it gets handled 4275 * again when current action finishes */ 4276 set_bit(STRIPE_HANDLE, &sh->state); 4277 return; 4278 } 4279 4280 if (clear_batch_ready(sh) ) { 4281 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4282 return; 4283 } 4284 4285 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) 4286 break_stripe_batch_list(sh, 0); 4287 4288 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { 4289 spin_lock(&sh->stripe_lock); 4290 /* Cannot process 'sync' concurrently with 'discard' */ 4291 if (!test_bit(STRIPE_DISCARD, &sh->state) && 4292 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 4293 set_bit(STRIPE_SYNCING, &sh->state); 4294 clear_bit(STRIPE_INSYNC, &sh->state); 4295 clear_bit(STRIPE_REPLACED, &sh->state); 4296 } 4297 spin_unlock(&sh->stripe_lock); 4298 } 4299 clear_bit(STRIPE_DELAYED, &sh->state); 4300 4301 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 4302 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 4303 (unsigned long long)sh->sector, sh->state, 4304 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 4305 sh->check_state, sh->reconstruct_state); 4306 4307 analyse_stripe(sh, &s); 4308 4309 if (s.handle_bad_blocks) { 4310 set_bit(STRIPE_HANDLE, &sh->state); 4311 goto finish; 4312 } 4313 4314 if (unlikely(s.blocked_rdev)) { 4315 if (s.syncing || s.expanding || s.expanded || 4316 s.replacing || s.to_write || s.written) { 4317 set_bit(STRIPE_HANDLE, &sh->state); 4318 goto finish; 4319 } 4320 /* There is nothing for the blocked_rdev to block */ 4321 rdev_dec_pending(s.blocked_rdev, conf->mddev); 4322 s.blocked_rdev = NULL; 4323 } 4324 4325 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 4326 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 4327 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 4328 } 4329 4330 pr_debug("locked=%d uptodate=%d to_read=%d" 4331 " to_write=%d failed=%d failed_num=%d,%d\n", 4332 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 4333 s.failed_num[0], s.failed_num[1]); 4334 /* check if the array has lost more than max_degraded devices and, 4335 * if so, some requests might need to be failed. 4336 */ 4337 if (s.failed > conf->max_degraded) { 4338 sh->check_state = 0; 4339 sh->reconstruct_state = 0; 4340 break_stripe_batch_list(sh, 0); 4341 if (s.to_read+s.to_write+s.written) 4342 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 4343 if (s.syncing + s.replacing) 4344 handle_failed_sync(conf, sh, &s); 4345 } 4346 4347 /* Now we check to see if any write operations have recently 4348 * completed 4349 */ 4350 prexor = 0; 4351 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 4352 prexor = 1; 4353 if (sh->reconstruct_state == reconstruct_state_drain_result || 4354 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 4355 sh->reconstruct_state = reconstruct_state_idle; 4356 4357 /* All the 'written' buffers and the parity block are ready to 4358 * be written back to disk 4359 */ 4360 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && 4361 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); 4362 BUG_ON(sh->qd_idx >= 0 && 4363 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && 4364 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); 4365 for (i = disks; i--; ) { 4366 struct r5dev *dev = &sh->dev[i]; 4367 if (test_bit(R5_LOCKED, &dev->flags) && 4368 (i == sh->pd_idx || i == sh->qd_idx || 4369 dev->written)) { 4370 pr_debug("Writing block %d\n", i); 4371 set_bit(R5_Wantwrite, &dev->flags); 4372 if (prexor) 4373 continue; 4374 if (s.failed > 1) 4375 continue; 4376 if (!test_bit(R5_Insync, &dev->flags) || 4377 ((i == sh->pd_idx || i == sh->qd_idx) && 4378 s.failed == 0)) 4379 set_bit(STRIPE_INSYNC, &sh->state); 4380 } 4381 } 4382 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4383 s.dec_preread_active = 1; 4384 } 4385 4386 /* 4387 * might be able to return some write requests if the parity blocks 4388 * are safe, or on a failed drive 4389 */ 4390 pdev = &sh->dev[sh->pd_idx]; 4391 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 4392 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 4393 qdev = &sh->dev[sh->qd_idx]; 4394 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 4395 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 4396 || conf->level < 6; 4397 4398 if (s.written && 4399 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 4400 && !test_bit(R5_LOCKED, &pdev->flags) 4401 && (test_bit(R5_UPTODATE, &pdev->flags) || 4402 test_bit(R5_Discard, &pdev->flags))))) && 4403 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 4404 && !test_bit(R5_LOCKED, &qdev->flags) 4405 && (test_bit(R5_UPTODATE, &qdev->flags) || 4406 test_bit(R5_Discard, &qdev->flags)))))) 4407 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 4408 4409 /* Now we might consider reading some blocks, either to check/generate 4410 * parity, or to satisfy requests 4411 * or to load a block that is being partially written. 4412 */ 4413 if (s.to_read || s.non_overwrite 4414 || (conf->level == 6 && s.to_write && s.failed) 4415 || (s.syncing && (s.uptodate + s.compute < disks)) 4416 || s.replacing 4417 || s.expanding) 4418 handle_stripe_fill(sh, &s, disks); 4419 4420 /* Now to consider new write requests and what else, if anything 4421 * should be read. We do not handle new writes when: 4422 * 1/ A 'write' operation (copy+xor) is already in flight. 4423 * 2/ A 'check' operation is in flight, as it may clobber the parity 4424 * block. 4425 */ 4426 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 4427 handle_stripe_dirtying(conf, sh, &s, disks); 4428 4429 /* maybe we need to check and possibly fix the parity for this stripe 4430 * Any reads will already have been scheduled, so we just see if enough 4431 * data is available. The parity check is held off while parity 4432 * dependent operations are in flight. 4433 */ 4434 if (sh->check_state || 4435 (s.syncing && s.locked == 0 && 4436 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 4437 !test_bit(STRIPE_INSYNC, &sh->state))) { 4438 if (conf->level == 6) 4439 handle_parity_checks6(conf, sh, &s, disks); 4440 else 4441 handle_parity_checks5(conf, sh, &s, disks); 4442 } 4443 4444 if ((s.replacing || s.syncing) && s.locked == 0 4445 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) 4446 && !test_bit(STRIPE_REPLACED, &sh->state)) { 4447 /* Write out to replacement devices where possible */ 4448 for (i = 0; i < conf->raid_disks; i++) 4449 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 4450 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); 4451 set_bit(R5_WantReplace, &sh->dev[i].flags); 4452 set_bit(R5_LOCKED, &sh->dev[i].flags); 4453 s.locked++; 4454 } 4455 if (s.replacing) 4456 set_bit(STRIPE_INSYNC, &sh->state); 4457 set_bit(STRIPE_REPLACED, &sh->state); 4458 } 4459 if ((s.syncing || s.replacing) && s.locked == 0 && 4460 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 4461 test_bit(STRIPE_INSYNC, &sh->state)) { 4462 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 4463 clear_bit(STRIPE_SYNCING, &sh->state); 4464 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 4465 wake_up(&conf->wait_for_overlap); 4466 } 4467 4468 /* If the failed drives are just a ReadError, then we might need 4469 * to progress the repair/check process 4470 */ 4471 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 4472 for (i = 0; i < s.failed; i++) { 4473 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 4474 if (test_bit(R5_ReadError, &dev->flags) 4475 && !test_bit(R5_LOCKED, &dev->flags) 4476 && test_bit(R5_UPTODATE, &dev->flags) 4477 ) { 4478 if (!test_bit(R5_ReWrite, &dev->flags)) { 4479 set_bit(R5_Wantwrite, &dev->flags); 4480 set_bit(R5_ReWrite, &dev->flags); 4481 set_bit(R5_LOCKED, &dev->flags); 4482 s.locked++; 4483 } else { 4484 /* let's read it back */ 4485 set_bit(R5_Wantread, &dev->flags); 4486 set_bit(R5_LOCKED, &dev->flags); 4487 s.locked++; 4488 } 4489 } 4490 } 4491 4492 /* Finish reconstruct operations initiated by the expansion process */ 4493 if (sh->reconstruct_state == reconstruct_state_result) { 4494 struct stripe_head *sh_src 4495 = get_active_stripe(conf, sh->sector, 1, 1, 1); 4496 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 4497 /* sh cannot be written until sh_src has been read. 4498 * so arrange for sh to be delayed a little 4499 */ 4500 set_bit(STRIPE_DELAYED, &sh->state); 4501 set_bit(STRIPE_HANDLE, &sh->state); 4502 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 4503 &sh_src->state)) 4504 atomic_inc(&conf->preread_active_stripes); 4505 release_stripe(sh_src); 4506 goto finish; 4507 } 4508 if (sh_src) 4509 release_stripe(sh_src); 4510 4511 sh->reconstruct_state = reconstruct_state_idle; 4512 clear_bit(STRIPE_EXPANDING, &sh->state); 4513 for (i = conf->raid_disks; i--; ) { 4514 set_bit(R5_Wantwrite, &sh->dev[i].flags); 4515 set_bit(R5_LOCKED, &sh->dev[i].flags); 4516 s.locked++; 4517 } 4518 } 4519 4520 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 4521 !sh->reconstruct_state) { 4522 /* Need to write out all blocks after computing parity */ 4523 sh->disks = conf->raid_disks; 4524 stripe_set_idx(sh->sector, conf, 0, sh); 4525 schedule_reconstruction(sh, &s, 1, 1); 4526 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 4527 clear_bit(STRIPE_EXPAND_READY, &sh->state); 4528 atomic_dec(&conf->reshape_stripes); 4529 wake_up(&conf->wait_for_overlap); 4530 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 4531 } 4532 4533 if (s.expanding && s.locked == 0 && 4534 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 4535 handle_stripe_expansion(conf, sh); 4536 4537 finish: 4538 /* wait for this device to become unblocked */ 4539 if (unlikely(s.blocked_rdev)) { 4540 if (conf->mddev->external) 4541 md_wait_for_blocked_rdev(s.blocked_rdev, 4542 conf->mddev); 4543 else 4544 /* Internal metadata will immediately 4545 * be written by raid5d, so we don't 4546 * need to wait here. 4547 */ 4548 rdev_dec_pending(s.blocked_rdev, 4549 conf->mddev); 4550 } 4551 4552 if (s.handle_bad_blocks) 4553 for (i = disks; i--; ) { 4554 struct md_rdev *rdev; 4555 struct r5dev *dev = &sh->dev[i]; 4556 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 4557 /* We own a safe reference to the rdev */ 4558 rdev = conf->disks[i].rdev; 4559 if (!rdev_set_badblocks(rdev, sh->sector, 4560 STRIPE_SECTORS, 0)) 4561 md_error(conf->mddev, rdev); 4562 rdev_dec_pending(rdev, conf->mddev); 4563 } 4564 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 4565 rdev = conf->disks[i].rdev; 4566 rdev_clear_badblocks(rdev, sh->sector, 4567 STRIPE_SECTORS, 0); 4568 rdev_dec_pending(rdev, conf->mddev); 4569 } 4570 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 4571 rdev = conf->disks[i].replacement; 4572 if (!rdev) 4573 /* rdev have been moved down */ 4574 rdev = conf->disks[i].rdev; 4575 rdev_clear_badblocks(rdev, sh->sector, 4576 STRIPE_SECTORS, 0); 4577 rdev_dec_pending(rdev, conf->mddev); 4578 } 4579 } 4580 4581 if (s.ops_request) 4582 raid_run_ops(sh, s.ops_request); 4583 4584 ops_run_io(sh, &s); 4585 4586 if (s.dec_preread_active) { 4587 /* We delay this until after ops_run_io so that if make_request 4588 * is waiting on a flush, it won't continue until the writes 4589 * have actually been submitted. 4590 */ 4591 atomic_dec(&conf->preread_active_stripes); 4592 if (atomic_read(&conf->preread_active_stripes) < 4593 IO_THRESHOLD) 4594 md_wakeup_thread(conf->mddev->thread); 4595 } 4596 4597 return_io(s.return_bi); 4598 4599 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4600 } 4601 4602 static void raid5_activate_delayed(struct r5conf *conf) 4603 { 4604 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 4605 while (!list_empty(&conf->delayed_list)) { 4606 struct list_head *l = conf->delayed_list.next; 4607 struct stripe_head *sh; 4608 sh = list_entry(l, struct stripe_head, lru); 4609 list_del_init(l); 4610 clear_bit(STRIPE_DELAYED, &sh->state); 4611 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4612 atomic_inc(&conf->preread_active_stripes); 4613 list_add_tail(&sh->lru, &conf->hold_list); 4614 raid5_wakeup_stripe_thread(sh); 4615 } 4616 } 4617 } 4618 4619 static void activate_bit_delay(struct r5conf *conf, 4620 struct list_head *temp_inactive_list) 4621 { 4622 /* device_lock is held */ 4623 struct list_head head; 4624 list_add(&head, &conf->bitmap_list); 4625 list_del_init(&conf->bitmap_list); 4626 while (!list_empty(&head)) { 4627 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 4628 int hash; 4629 list_del_init(&sh->lru); 4630 atomic_inc(&sh->count); 4631 hash = sh->hash_lock_index; 4632 __release_stripe(conf, sh, &temp_inactive_list[hash]); 4633 } 4634 } 4635 4636 static int raid5_congested(struct mddev *mddev, int bits) 4637 { 4638 struct r5conf *conf = mddev->private; 4639 4640 /* No difference between reads and writes. Just check 4641 * how busy the stripe_cache is 4642 */ 4643 4644 if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) 4645 return 1; 4646 if (conf->quiesce) 4647 return 1; 4648 if (atomic_read(&conf->empty_inactive_list_nr)) 4649 return 1; 4650 4651 return 0; 4652 } 4653 4654 /* We want read requests to align with chunks where possible, 4655 * but write requests don't need to. 4656 */ 4657 static int raid5_mergeable_bvec(struct mddev *mddev, 4658 struct bvec_merge_data *bvm, 4659 struct bio_vec *biovec) 4660 { 4661 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 4662 int max; 4663 unsigned int chunk_sectors = mddev->chunk_sectors; 4664 unsigned int bio_sectors = bvm->bi_size >> 9; 4665 4666 /* 4667 * always allow writes to be mergeable, read as well if array 4668 * is degraded as we'll go through stripe cache anyway. 4669 */ 4670 if ((bvm->bi_rw & 1) == WRITE || mddev->degraded) 4671 return biovec->bv_len; 4672 4673 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4674 chunk_sectors = mddev->new_chunk_sectors; 4675 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 4676 if (max < 0) max = 0; 4677 if (max <= biovec->bv_len && bio_sectors == 0) 4678 return biovec->bv_len; 4679 else 4680 return max; 4681 } 4682 4683 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 4684 { 4685 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); 4686 unsigned int chunk_sectors = mddev->chunk_sectors; 4687 unsigned int bio_sectors = bio_sectors(bio); 4688 4689 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4690 chunk_sectors = mddev->new_chunk_sectors; 4691 return chunk_sectors >= 4692 ((sector & (chunk_sectors - 1)) + bio_sectors); 4693 } 4694 4695 /* 4696 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 4697 * later sampled by raid5d. 4698 */ 4699 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 4700 { 4701 unsigned long flags; 4702 4703 spin_lock_irqsave(&conf->device_lock, flags); 4704 4705 bi->bi_next = conf->retry_read_aligned_list; 4706 conf->retry_read_aligned_list = bi; 4707 4708 spin_unlock_irqrestore(&conf->device_lock, flags); 4709 md_wakeup_thread(conf->mddev->thread); 4710 } 4711 4712 static struct bio *remove_bio_from_retry(struct r5conf *conf) 4713 { 4714 struct bio *bi; 4715 4716 bi = conf->retry_read_aligned; 4717 if (bi) { 4718 conf->retry_read_aligned = NULL; 4719 return bi; 4720 } 4721 bi = conf->retry_read_aligned_list; 4722 if(bi) { 4723 conf->retry_read_aligned_list = bi->bi_next; 4724 bi->bi_next = NULL; 4725 /* 4726 * this sets the active strip count to 1 and the processed 4727 * strip count to zero (upper 8 bits) 4728 */ 4729 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ 4730 } 4731 4732 return bi; 4733 } 4734 4735 /* 4736 * The "raid5_align_endio" should check if the read succeeded and if it 4737 * did, call bio_endio on the original bio (having bio_put the new bio 4738 * first). 4739 * If the read failed.. 4740 */ 4741 static void raid5_align_endio(struct bio *bi, int error) 4742 { 4743 struct bio* raid_bi = bi->bi_private; 4744 struct mddev *mddev; 4745 struct r5conf *conf; 4746 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 4747 struct md_rdev *rdev; 4748 4749 bio_put(bi); 4750 4751 rdev = (void*)raid_bi->bi_next; 4752 raid_bi->bi_next = NULL; 4753 mddev = rdev->mddev; 4754 conf = mddev->private; 4755 4756 rdev_dec_pending(rdev, conf->mddev); 4757 4758 if (!error && uptodate) { 4759 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4760 raid_bi, 0); 4761 bio_endio(raid_bi, 0); 4762 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4763 wake_up(&conf->wait_for_stripe); 4764 return; 4765 } 4766 4767 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 4768 4769 add_bio_to_retry(raid_bi, conf); 4770 } 4771 4772 static int bio_fits_rdev(struct bio *bi) 4773 { 4774 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 4775 4776 if (bio_sectors(bi) > queue_max_sectors(q)) 4777 return 0; 4778 blk_recount_segments(q, bi); 4779 if (bi->bi_phys_segments > queue_max_segments(q)) 4780 return 0; 4781 4782 if (q->merge_bvec_fn) 4783 /* it's too hard to apply the merge_bvec_fn at this stage, 4784 * just just give up 4785 */ 4786 return 0; 4787 4788 return 1; 4789 } 4790 4791 static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) 4792 { 4793 struct r5conf *conf = mddev->private; 4794 int dd_idx; 4795 struct bio* align_bi; 4796 struct md_rdev *rdev; 4797 sector_t end_sector; 4798 4799 if (!in_chunk_boundary(mddev, raid_bio)) { 4800 pr_debug("chunk_aligned_read : non aligned\n"); 4801 return 0; 4802 } 4803 /* 4804 * use bio_clone_mddev to make a copy of the bio 4805 */ 4806 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 4807 if (!align_bi) 4808 return 0; 4809 /* 4810 * set bi_end_io to a new function, and set bi_private to the 4811 * original bio. 4812 */ 4813 align_bi->bi_end_io = raid5_align_endio; 4814 align_bi->bi_private = raid_bio; 4815 /* 4816 * compute position 4817 */ 4818 align_bi->bi_iter.bi_sector = 4819 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 4820 0, &dd_idx, NULL); 4821 4822 end_sector = bio_end_sector(align_bi); 4823 rcu_read_lock(); 4824 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 4825 if (!rdev || test_bit(Faulty, &rdev->flags) || 4826 rdev->recovery_offset < end_sector) { 4827 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 4828 if (rdev && 4829 (test_bit(Faulty, &rdev->flags) || 4830 !(test_bit(In_sync, &rdev->flags) || 4831 rdev->recovery_offset >= end_sector))) 4832 rdev = NULL; 4833 } 4834 if (rdev) { 4835 sector_t first_bad; 4836 int bad_sectors; 4837 4838 atomic_inc(&rdev->nr_pending); 4839 rcu_read_unlock(); 4840 raid_bio->bi_next = (void*)rdev; 4841 align_bi->bi_bdev = rdev->bdev; 4842 __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); 4843 4844 if (!bio_fits_rdev(align_bi) || 4845 is_badblock(rdev, align_bi->bi_iter.bi_sector, 4846 bio_sectors(align_bi), 4847 &first_bad, &bad_sectors)) { 4848 /* too big in some way, or has a known bad block */ 4849 bio_put(align_bi); 4850 rdev_dec_pending(rdev, mddev); 4851 return 0; 4852 } 4853 4854 /* No reshape active, so we can trust rdev->data_offset */ 4855 align_bi->bi_iter.bi_sector += rdev->data_offset; 4856 4857 spin_lock_irq(&conf->device_lock); 4858 wait_event_lock_irq(conf->wait_for_stripe, 4859 conf->quiesce == 0, 4860 conf->device_lock); 4861 atomic_inc(&conf->active_aligned_reads); 4862 spin_unlock_irq(&conf->device_lock); 4863 4864 if (mddev->gendisk) 4865 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), 4866 align_bi, disk_devt(mddev->gendisk), 4867 raid_bio->bi_iter.bi_sector); 4868 generic_make_request(align_bi); 4869 return 1; 4870 } else { 4871 rcu_read_unlock(); 4872 bio_put(align_bi); 4873 return 0; 4874 } 4875 } 4876 4877 /* __get_priority_stripe - get the next stripe to process 4878 * 4879 * Full stripe writes are allowed to pass preread active stripes up until 4880 * the bypass_threshold is exceeded. In general the bypass_count 4881 * increments when the handle_list is handled before the hold_list; however, it 4882 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 4883 * stripe with in flight i/o. The bypass_count will be reset when the 4884 * head of the hold_list has changed, i.e. the head was promoted to the 4885 * handle_list. 4886 */ 4887 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) 4888 { 4889 struct stripe_head *sh = NULL, *tmp; 4890 struct list_head *handle_list = NULL; 4891 struct r5worker_group *wg = NULL; 4892 4893 if (conf->worker_cnt_per_group == 0) { 4894 handle_list = &conf->handle_list; 4895 } else if (group != ANY_GROUP) { 4896 handle_list = &conf->worker_groups[group].handle_list; 4897 wg = &conf->worker_groups[group]; 4898 } else { 4899 int i; 4900 for (i = 0; i < conf->group_cnt; i++) { 4901 handle_list = &conf->worker_groups[i].handle_list; 4902 wg = &conf->worker_groups[i]; 4903 if (!list_empty(handle_list)) 4904 break; 4905 } 4906 } 4907 4908 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 4909 __func__, 4910 list_empty(handle_list) ? "empty" : "busy", 4911 list_empty(&conf->hold_list) ? "empty" : "busy", 4912 atomic_read(&conf->pending_full_writes), conf->bypass_count); 4913 4914 if (!list_empty(handle_list)) { 4915 sh = list_entry(handle_list->next, typeof(*sh), lru); 4916 4917 if (list_empty(&conf->hold_list)) 4918 conf->bypass_count = 0; 4919 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 4920 if (conf->hold_list.next == conf->last_hold) 4921 conf->bypass_count++; 4922 else { 4923 conf->last_hold = conf->hold_list.next; 4924 conf->bypass_count -= conf->bypass_threshold; 4925 if (conf->bypass_count < 0) 4926 conf->bypass_count = 0; 4927 } 4928 } 4929 } else if (!list_empty(&conf->hold_list) && 4930 ((conf->bypass_threshold && 4931 conf->bypass_count > conf->bypass_threshold) || 4932 atomic_read(&conf->pending_full_writes) == 0)) { 4933 4934 list_for_each_entry(tmp, &conf->hold_list, lru) { 4935 if (conf->worker_cnt_per_group == 0 || 4936 group == ANY_GROUP || 4937 !cpu_online(tmp->cpu) || 4938 cpu_to_group(tmp->cpu) == group) { 4939 sh = tmp; 4940 break; 4941 } 4942 } 4943 4944 if (sh) { 4945 conf->bypass_count -= conf->bypass_threshold; 4946 if (conf->bypass_count < 0) 4947 conf->bypass_count = 0; 4948 } 4949 wg = NULL; 4950 } 4951 4952 if (!sh) 4953 return NULL; 4954 4955 if (wg) { 4956 wg->stripes_cnt--; 4957 sh->group = NULL; 4958 } 4959 list_del_init(&sh->lru); 4960 BUG_ON(atomic_inc_return(&sh->count) != 1); 4961 return sh; 4962 } 4963 4964 struct raid5_plug_cb { 4965 struct blk_plug_cb cb; 4966 struct list_head list; 4967 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; 4968 }; 4969 4970 static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) 4971 { 4972 struct raid5_plug_cb *cb = container_of( 4973 blk_cb, struct raid5_plug_cb, cb); 4974 struct stripe_head *sh; 4975 struct mddev *mddev = cb->cb.data; 4976 struct r5conf *conf = mddev->private; 4977 int cnt = 0; 4978 int hash; 4979 4980 if (cb->list.next && !list_empty(&cb->list)) { 4981 spin_lock_irq(&conf->device_lock); 4982 while (!list_empty(&cb->list)) { 4983 sh = list_first_entry(&cb->list, struct stripe_head, lru); 4984 list_del_init(&sh->lru); 4985 /* 4986 * avoid race release_stripe_plug() sees 4987 * STRIPE_ON_UNPLUG_LIST clear but the stripe 4988 * is still in our list 4989 */ 4990 smp_mb__before_atomic(); 4991 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); 4992 /* 4993 * STRIPE_ON_RELEASE_LIST could be set here. In that 4994 * case, the count is always > 1 here 4995 */ 4996 hash = sh->hash_lock_index; 4997 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); 4998 cnt++; 4999 } 5000 spin_unlock_irq(&conf->device_lock); 5001 } 5002 release_inactive_stripe_list(conf, cb->temp_inactive_list, 5003 NR_STRIPE_HASH_LOCKS); 5004 if (mddev->queue) 5005 trace_block_unplug(mddev->queue, cnt, !from_schedule); 5006 kfree(cb); 5007 } 5008 5009 static void release_stripe_plug(struct mddev *mddev, 5010 struct stripe_head *sh) 5011 { 5012 struct blk_plug_cb *blk_cb = blk_check_plugged( 5013 raid5_unplug, mddev, 5014 sizeof(struct raid5_plug_cb)); 5015 struct raid5_plug_cb *cb; 5016 5017 if (!blk_cb) { 5018 release_stripe(sh); 5019 return; 5020 } 5021 5022 cb = container_of(blk_cb, struct raid5_plug_cb, cb); 5023 5024 if (cb->list.next == NULL) { 5025 int i; 5026 INIT_LIST_HEAD(&cb->list); 5027 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5028 INIT_LIST_HEAD(cb->temp_inactive_list + i); 5029 } 5030 5031 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) 5032 list_add_tail(&sh->lru, &cb->list); 5033 else 5034 release_stripe(sh); 5035 } 5036 5037 static void make_discard_request(struct mddev *mddev, struct bio *bi) 5038 { 5039 struct r5conf *conf = mddev->private; 5040 sector_t logical_sector, last_sector; 5041 struct stripe_head *sh; 5042 int remaining; 5043 int stripe_sectors; 5044 5045 if (mddev->reshape_position != MaxSector) 5046 /* Skip discard while reshape is happening */ 5047 return; 5048 5049 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 5050 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); 5051 5052 bi->bi_next = NULL; 5053 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 5054 5055 stripe_sectors = conf->chunk_sectors * 5056 (conf->raid_disks - conf->max_degraded); 5057 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, 5058 stripe_sectors); 5059 sector_div(last_sector, stripe_sectors); 5060 5061 logical_sector *= conf->chunk_sectors; 5062 last_sector *= conf->chunk_sectors; 5063 5064 for (; logical_sector < last_sector; 5065 logical_sector += STRIPE_SECTORS) { 5066 DEFINE_WAIT(w); 5067 int d; 5068 again: 5069 sh = get_active_stripe(conf, logical_sector, 0, 0, 0); 5070 prepare_to_wait(&conf->wait_for_overlap, &w, 5071 TASK_UNINTERRUPTIBLE); 5072 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 5073 if (test_bit(STRIPE_SYNCING, &sh->state)) { 5074 release_stripe(sh); 5075 schedule(); 5076 goto again; 5077 } 5078 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 5079 spin_lock_irq(&sh->stripe_lock); 5080 for (d = 0; d < conf->raid_disks; d++) { 5081 if (d == sh->pd_idx || d == sh->qd_idx) 5082 continue; 5083 if (sh->dev[d].towrite || sh->dev[d].toread) { 5084 set_bit(R5_Overlap, &sh->dev[d].flags); 5085 spin_unlock_irq(&sh->stripe_lock); 5086 release_stripe(sh); 5087 schedule(); 5088 goto again; 5089 } 5090 } 5091 set_bit(STRIPE_DISCARD, &sh->state); 5092 finish_wait(&conf->wait_for_overlap, &w); 5093 sh->overwrite_disks = 0; 5094 for (d = 0; d < conf->raid_disks; d++) { 5095 if (d == sh->pd_idx || d == sh->qd_idx) 5096 continue; 5097 sh->dev[d].towrite = bi; 5098 set_bit(R5_OVERWRITE, &sh->dev[d].flags); 5099 raid5_inc_bi_active_stripes(bi); 5100 sh->overwrite_disks++; 5101 } 5102 spin_unlock_irq(&sh->stripe_lock); 5103 if (conf->mddev->bitmap) { 5104 for (d = 0; 5105 d < conf->raid_disks - conf->max_degraded; 5106 d++) 5107 bitmap_startwrite(mddev->bitmap, 5108 sh->sector, 5109 STRIPE_SECTORS, 5110 0); 5111 sh->bm_seq = conf->seq_flush + 1; 5112 set_bit(STRIPE_BIT_DELAY, &sh->state); 5113 } 5114 5115 set_bit(STRIPE_HANDLE, &sh->state); 5116 clear_bit(STRIPE_DELAYED, &sh->state); 5117 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 5118 atomic_inc(&conf->preread_active_stripes); 5119 release_stripe_plug(mddev, sh); 5120 } 5121 5122 remaining = raid5_dec_bi_active_stripes(bi); 5123 if (remaining == 0) { 5124 md_write_end(mddev); 5125 bio_endio(bi, 0); 5126 } 5127 } 5128 5129 static void make_request(struct mddev *mddev, struct bio * bi) 5130 { 5131 struct r5conf *conf = mddev->private; 5132 int dd_idx; 5133 sector_t new_sector; 5134 sector_t logical_sector, last_sector; 5135 struct stripe_head *sh; 5136 const int rw = bio_data_dir(bi); 5137 int remaining; 5138 DEFINE_WAIT(w); 5139 bool do_prepare; 5140 5141 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 5142 md_flush_request(mddev, bi); 5143 return; 5144 } 5145 5146 md_write_start(mddev, bi); 5147 5148 /* 5149 * If array is degraded, better not do chunk aligned read because 5150 * later we might have to read it again in order to reconstruct 5151 * data on failed drives. 5152 */ 5153 if (rw == READ && mddev->degraded == 0 && 5154 mddev->reshape_position == MaxSector && 5155 chunk_aligned_read(mddev,bi)) 5156 return; 5157 5158 if (unlikely(bi->bi_rw & REQ_DISCARD)) { 5159 make_discard_request(mddev, bi); 5160 return; 5161 } 5162 5163 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 5164 last_sector = bio_end_sector(bi); 5165 bi->bi_next = NULL; 5166 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 5167 5168 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 5169 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 5170 int previous; 5171 int seq; 5172 5173 do_prepare = false; 5174 retry: 5175 seq = read_seqcount_begin(&conf->gen_lock); 5176 previous = 0; 5177 if (do_prepare) 5178 prepare_to_wait(&conf->wait_for_overlap, &w, 5179 TASK_UNINTERRUPTIBLE); 5180 if (unlikely(conf->reshape_progress != MaxSector)) { 5181 /* spinlock is needed as reshape_progress may be 5182 * 64bit on a 32bit platform, and so it might be 5183 * possible to see a half-updated value 5184 * Of course reshape_progress could change after 5185 * the lock is dropped, so once we get a reference 5186 * to the stripe that we think it is, we will have 5187 * to check again. 5188 */ 5189 spin_lock_irq(&conf->device_lock); 5190 if (mddev->reshape_backwards 5191 ? logical_sector < conf->reshape_progress 5192 : logical_sector >= conf->reshape_progress) { 5193 previous = 1; 5194 } else { 5195 if (mddev->reshape_backwards 5196 ? logical_sector < conf->reshape_safe 5197 : logical_sector >= conf->reshape_safe) { 5198 spin_unlock_irq(&conf->device_lock); 5199 schedule(); 5200 do_prepare = true; 5201 goto retry; 5202 } 5203 } 5204 spin_unlock_irq(&conf->device_lock); 5205 } 5206 5207 new_sector = raid5_compute_sector(conf, logical_sector, 5208 previous, 5209 &dd_idx, NULL); 5210 pr_debug("raid456: make_request, sector %llu logical %llu\n", 5211 (unsigned long long)new_sector, 5212 (unsigned long long)logical_sector); 5213 5214 sh = get_active_stripe(conf, new_sector, previous, 5215 (bi->bi_rw&RWA_MASK), 0); 5216 if (sh) { 5217 if (unlikely(previous)) { 5218 /* expansion might have moved on while waiting for a 5219 * stripe, so we must do the range check again. 5220 * Expansion could still move past after this 5221 * test, but as we are holding a reference to 5222 * 'sh', we know that if that happens, 5223 * STRIPE_EXPANDING will get set and the expansion 5224 * won't proceed until we finish with the stripe. 5225 */ 5226 int must_retry = 0; 5227 spin_lock_irq(&conf->device_lock); 5228 if (mddev->reshape_backwards 5229 ? logical_sector >= conf->reshape_progress 5230 : logical_sector < conf->reshape_progress) 5231 /* mismatch, need to try again */ 5232 must_retry = 1; 5233 spin_unlock_irq(&conf->device_lock); 5234 if (must_retry) { 5235 release_stripe(sh); 5236 schedule(); 5237 do_prepare = true; 5238 goto retry; 5239 } 5240 } 5241 if (read_seqcount_retry(&conf->gen_lock, seq)) { 5242 /* Might have got the wrong stripe_head 5243 * by accident 5244 */ 5245 release_stripe(sh); 5246 goto retry; 5247 } 5248 5249 if (rw == WRITE && 5250 logical_sector >= mddev->suspend_lo && 5251 logical_sector < mddev->suspend_hi) { 5252 release_stripe(sh); 5253 /* As the suspend_* range is controlled by 5254 * userspace, we want an interruptible 5255 * wait. 5256 */ 5257 flush_signals(current); 5258 prepare_to_wait(&conf->wait_for_overlap, 5259 &w, TASK_INTERRUPTIBLE); 5260 if (logical_sector >= mddev->suspend_lo && 5261 logical_sector < mddev->suspend_hi) { 5262 schedule(); 5263 do_prepare = true; 5264 } 5265 goto retry; 5266 } 5267 5268 if (test_bit(STRIPE_EXPANDING, &sh->state) || 5269 !add_stripe_bio(sh, bi, dd_idx, rw, previous)) { 5270 /* Stripe is busy expanding or 5271 * add failed due to overlap. Flush everything 5272 * and wait a while 5273 */ 5274 md_wakeup_thread(mddev->thread); 5275 release_stripe(sh); 5276 schedule(); 5277 do_prepare = true; 5278 goto retry; 5279 } 5280 set_bit(STRIPE_HANDLE, &sh->state); 5281 clear_bit(STRIPE_DELAYED, &sh->state); 5282 if ((!sh->batch_head || sh == sh->batch_head) && 5283 (bi->bi_rw & REQ_SYNC) && 5284 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 5285 atomic_inc(&conf->preread_active_stripes); 5286 release_stripe_plug(mddev, sh); 5287 } else { 5288 /* cannot get stripe for read-ahead, just give-up */ 5289 clear_bit(BIO_UPTODATE, &bi->bi_flags); 5290 break; 5291 } 5292 } 5293 finish_wait(&conf->wait_for_overlap, &w); 5294 5295 remaining = raid5_dec_bi_active_stripes(bi); 5296 if (remaining == 0) { 5297 5298 if ( rw == WRITE ) 5299 md_write_end(mddev); 5300 5301 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 5302 bi, 0); 5303 bio_endio(bi, 0); 5304 } 5305 } 5306 5307 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 5308 5309 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 5310 { 5311 /* reshaping is quite different to recovery/resync so it is 5312 * handled quite separately ... here. 5313 * 5314 * On each call to sync_request, we gather one chunk worth of 5315 * destination stripes and flag them as expanding. 5316 * Then we find all the source stripes and request reads. 5317 * As the reads complete, handle_stripe will copy the data 5318 * into the destination stripe and release that stripe. 5319 */ 5320 struct r5conf *conf = mddev->private; 5321 struct stripe_head *sh; 5322 sector_t first_sector, last_sector; 5323 int raid_disks = conf->previous_raid_disks; 5324 int data_disks = raid_disks - conf->max_degraded; 5325 int new_data_disks = conf->raid_disks - conf->max_degraded; 5326 int i; 5327 int dd_idx; 5328 sector_t writepos, readpos, safepos; 5329 sector_t stripe_addr; 5330 int reshape_sectors; 5331 struct list_head stripes; 5332 5333 if (sector_nr == 0) { 5334 /* If restarting in the middle, skip the initial sectors */ 5335 if (mddev->reshape_backwards && 5336 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 5337 sector_nr = raid5_size(mddev, 0, 0) 5338 - conf->reshape_progress; 5339 } else if (!mddev->reshape_backwards && 5340 conf->reshape_progress > 0) 5341 sector_nr = conf->reshape_progress; 5342 sector_div(sector_nr, new_data_disks); 5343 if (sector_nr) { 5344 mddev->curr_resync_completed = sector_nr; 5345 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5346 *skipped = 1; 5347 return sector_nr; 5348 } 5349 } 5350 5351 /* We need to process a full chunk at a time. 5352 * If old and new chunk sizes differ, we need to process the 5353 * largest of these 5354 */ 5355 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 5356 reshape_sectors = mddev->new_chunk_sectors; 5357 else 5358 reshape_sectors = mddev->chunk_sectors; 5359 5360 /* We update the metadata at least every 10 seconds, or when 5361 * the data about to be copied would over-write the source of 5362 * the data at the front of the range. i.e. one new_stripe 5363 * along from reshape_progress new_maps to after where 5364 * reshape_safe old_maps to 5365 */ 5366 writepos = conf->reshape_progress; 5367 sector_div(writepos, new_data_disks); 5368 readpos = conf->reshape_progress; 5369 sector_div(readpos, data_disks); 5370 safepos = conf->reshape_safe; 5371 sector_div(safepos, data_disks); 5372 if (mddev->reshape_backwards) { 5373 writepos -= min_t(sector_t, reshape_sectors, writepos); 5374 readpos += reshape_sectors; 5375 safepos += reshape_sectors; 5376 } else { 5377 writepos += reshape_sectors; 5378 readpos -= min_t(sector_t, reshape_sectors, readpos); 5379 safepos -= min_t(sector_t, reshape_sectors, safepos); 5380 } 5381 5382 /* Having calculated the 'writepos' possibly use it 5383 * to set 'stripe_addr' which is where we will write to. 5384 */ 5385 if (mddev->reshape_backwards) { 5386 BUG_ON(conf->reshape_progress == 0); 5387 stripe_addr = writepos; 5388 BUG_ON((mddev->dev_sectors & 5389 ~((sector_t)reshape_sectors - 1)) 5390 - reshape_sectors - stripe_addr 5391 != sector_nr); 5392 } else { 5393 BUG_ON(writepos != sector_nr + reshape_sectors); 5394 stripe_addr = sector_nr; 5395 } 5396 5397 /* 'writepos' is the most advanced device address we might write. 5398 * 'readpos' is the least advanced device address we might read. 5399 * 'safepos' is the least address recorded in the metadata as having 5400 * been reshaped. 5401 * If there is a min_offset_diff, these are adjusted either by 5402 * increasing the safepos/readpos if diff is negative, or 5403 * increasing writepos if diff is positive. 5404 * If 'readpos' is then behind 'writepos', there is no way that we can 5405 * ensure safety in the face of a crash - that must be done by userspace 5406 * making a backup of the data. So in that case there is no particular 5407 * rush to update metadata. 5408 * Otherwise if 'safepos' is behind 'writepos', then we really need to 5409 * update the metadata to advance 'safepos' to match 'readpos' so that 5410 * we can be safe in the event of a crash. 5411 * So we insist on updating metadata if safepos is behind writepos and 5412 * readpos is beyond writepos. 5413 * In any case, update the metadata every 10 seconds. 5414 * Maybe that number should be configurable, but I'm not sure it is 5415 * worth it.... maybe it could be a multiple of safemode_delay??? 5416 */ 5417 if (conf->min_offset_diff < 0) { 5418 safepos += -conf->min_offset_diff; 5419 readpos += -conf->min_offset_diff; 5420 } else 5421 writepos += conf->min_offset_diff; 5422 5423 if ((mddev->reshape_backwards 5424 ? (safepos > writepos && readpos < writepos) 5425 : (safepos < writepos && readpos > writepos)) || 5426 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 5427 /* Cannot proceed until we've updated the superblock... */ 5428 wait_event(conf->wait_for_overlap, 5429 atomic_read(&conf->reshape_stripes)==0 5430 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5431 if (atomic_read(&conf->reshape_stripes) != 0) 5432 return 0; 5433 mddev->reshape_position = conf->reshape_progress; 5434 mddev->curr_resync_completed = sector_nr; 5435 conf->reshape_checkpoint = jiffies; 5436 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5437 md_wakeup_thread(mddev->thread); 5438 wait_event(mddev->sb_wait, mddev->flags == 0 || 5439 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5440 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5441 return 0; 5442 spin_lock_irq(&conf->device_lock); 5443 conf->reshape_safe = mddev->reshape_position; 5444 spin_unlock_irq(&conf->device_lock); 5445 wake_up(&conf->wait_for_overlap); 5446 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5447 } 5448 5449 INIT_LIST_HEAD(&stripes); 5450 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 5451 int j; 5452 int skipped_disk = 0; 5453 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 5454 set_bit(STRIPE_EXPANDING, &sh->state); 5455 atomic_inc(&conf->reshape_stripes); 5456 /* If any of this stripe is beyond the end of the old 5457 * array, then we need to zero those blocks 5458 */ 5459 for (j=sh->disks; j--;) { 5460 sector_t s; 5461 if (j == sh->pd_idx) 5462 continue; 5463 if (conf->level == 6 && 5464 j == sh->qd_idx) 5465 continue; 5466 s = compute_blocknr(sh, j, 0); 5467 if (s < raid5_size(mddev, 0, 0)) { 5468 skipped_disk = 1; 5469 continue; 5470 } 5471 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 5472 set_bit(R5_Expanded, &sh->dev[j].flags); 5473 set_bit(R5_UPTODATE, &sh->dev[j].flags); 5474 } 5475 if (!skipped_disk) { 5476 set_bit(STRIPE_EXPAND_READY, &sh->state); 5477 set_bit(STRIPE_HANDLE, &sh->state); 5478 } 5479 list_add(&sh->lru, &stripes); 5480 } 5481 spin_lock_irq(&conf->device_lock); 5482 if (mddev->reshape_backwards) 5483 conf->reshape_progress -= reshape_sectors * new_data_disks; 5484 else 5485 conf->reshape_progress += reshape_sectors * new_data_disks; 5486 spin_unlock_irq(&conf->device_lock); 5487 /* Ok, those stripe are ready. We can start scheduling 5488 * reads on the source stripes. 5489 * The source stripes are determined by mapping the first and last 5490 * block on the destination stripes. 5491 */ 5492 first_sector = 5493 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 5494 1, &dd_idx, NULL); 5495 last_sector = 5496 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 5497 * new_data_disks - 1), 5498 1, &dd_idx, NULL); 5499 if (last_sector >= mddev->dev_sectors) 5500 last_sector = mddev->dev_sectors - 1; 5501 while (first_sector <= last_sector) { 5502 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 5503 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 5504 set_bit(STRIPE_HANDLE, &sh->state); 5505 release_stripe(sh); 5506 first_sector += STRIPE_SECTORS; 5507 } 5508 /* Now that the sources are clearly marked, we can release 5509 * the destination stripes 5510 */ 5511 while (!list_empty(&stripes)) { 5512 sh = list_entry(stripes.next, struct stripe_head, lru); 5513 list_del_init(&sh->lru); 5514 release_stripe(sh); 5515 } 5516 /* If this takes us to the resync_max point where we have to pause, 5517 * then we need to write out the superblock. 5518 */ 5519 sector_nr += reshape_sectors; 5520 if ((sector_nr - mddev->curr_resync_completed) * 2 5521 >= mddev->resync_max - mddev->curr_resync_completed) { 5522 /* Cannot proceed until we've updated the superblock... */ 5523 wait_event(conf->wait_for_overlap, 5524 atomic_read(&conf->reshape_stripes) == 0 5525 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5526 if (atomic_read(&conf->reshape_stripes) != 0) 5527 goto ret; 5528 mddev->reshape_position = conf->reshape_progress; 5529 mddev->curr_resync_completed = sector_nr; 5530 conf->reshape_checkpoint = jiffies; 5531 set_bit(MD_CHANGE_DEVS, &mddev->flags); 5532 md_wakeup_thread(mddev->thread); 5533 wait_event(mddev->sb_wait, 5534 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 5535 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5536 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5537 goto ret; 5538 spin_lock_irq(&conf->device_lock); 5539 conf->reshape_safe = mddev->reshape_position; 5540 spin_unlock_irq(&conf->device_lock); 5541 wake_up(&conf->wait_for_overlap); 5542 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5543 } 5544 ret: 5545 return reshape_sectors; 5546 } 5547 5548 static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 5549 { 5550 struct r5conf *conf = mddev->private; 5551 struct stripe_head *sh; 5552 sector_t max_sector = mddev->dev_sectors; 5553 sector_t sync_blocks; 5554 int still_degraded = 0; 5555 int i; 5556 5557 if (sector_nr >= max_sector) { 5558 /* just being told to finish up .. nothing much to do */ 5559 5560 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 5561 end_reshape(conf); 5562 return 0; 5563 } 5564 5565 if (mddev->curr_resync < max_sector) /* aborted */ 5566 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 5567 &sync_blocks, 1); 5568 else /* completed sync */ 5569 conf->fullsync = 0; 5570 bitmap_close_sync(mddev->bitmap); 5571 5572 return 0; 5573 } 5574 5575 /* Allow raid5_quiesce to complete */ 5576 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 5577 5578 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 5579 return reshape_request(mddev, sector_nr, skipped); 5580 5581 /* No need to check resync_max as we never do more than one 5582 * stripe, and as resync_max will always be on a chunk boundary, 5583 * if the check in md_do_sync didn't fire, there is no chance 5584 * of overstepping resync_max here 5585 */ 5586 5587 /* if there is too many failed drives and we are trying 5588 * to resync, then assert that we are finished, because there is 5589 * nothing we can do. 5590 */ 5591 if (mddev->degraded >= conf->max_degraded && 5592 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 5593 sector_t rv = mddev->dev_sectors - sector_nr; 5594 *skipped = 1; 5595 return rv; 5596 } 5597 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 5598 !conf->fullsync && 5599 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 5600 sync_blocks >= STRIPE_SECTORS) { 5601 /* we can skip this block, and probably more */ 5602 sync_blocks /= STRIPE_SECTORS; 5603 *skipped = 1; 5604 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 5605 } 5606 5607 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 5608 5609 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 5610 if (sh == NULL) { 5611 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 5612 /* make sure we don't swamp the stripe cache if someone else 5613 * is trying to get access 5614 */ 5615 schedule_timeout_uninterruptible(1); 5616 } 5617 /* Need to check if array will still be degraded after recovery/resync 5618 * Note in case of > 1 drive failures it's possible we're rebuilding 5619 * one drive while leaving another faulty drive in array. 5620 */ 5621 rcu_read_lock(); 5622 for (i = 0; i < conf->raid_disks; i++) { 5623 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); 5624 5625 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 5626 still_degraded = 1; 5627 } 5628 rcu_read_unlock(); 5629 5630 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 5631 5632 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 5633 set_bit(STRIPE_HANDLE, &sh->state); 5634 5635 release_stripe(sh); 5636 5637 return STRIPE_SECTORS; 5638 } 5639 5640 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 5641 { 5642 /* We may not be able to submit a whole bio at once as there 5643 * may not be enough stripe_heads available. 5644 * We cannot pre-allocate enough stripe_heads as we may need 5645 * more than exist in the cache (if we allow ever large chunks). 5646 * So we do one stripe head at a time and record in 5647 * ->bi_hw_segments how many have been done. 5648 * 5649 * We *know* that this entire raid_bio is in one chunk, so 5650 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 5651 */ 5652 struct stripe_head *sh; 5653 int dd_idx; 5654 sector_t sector, logical_sector, last_sector; 5655 int scnt = 0; 5656 int remaining; 5657 int handled = 0; 5658 5659 logical_sector = raid_bio->bi_iter.bi_sector & 5660 ~((sector_t)STRIPE_SECTORS-1); 5661 sector = raid5_compute_sector(conf, logical_sector, 5662 0, &dd_idx, NULL); 5663 last_sector = bio_end_sector(raid_bio); 5664 5665 for (; logical_sector < last_sector; 5666 logical_sector += STRIPE_SECTORS, 5667 sector += STRIPE_SECTORS, 5668 scnt++) { 5669 5670 if (scnt < raid5_bi_processed_stripes(raid_bio)) 5671 /* already done this stripe */ 5672 continue; 5673 5674 sh = get_active_stripe(conf, sector, 0, 1, 1); 5675 5676 if (!sh) { 5677 /* failed to get a stripe - must wait */ 5678 raid5_set_bi_processed_stripes(raid_bio, scnt); 5679 conf->retry_read_aligned = raid_bio; 5680 return handled; 5681 } 5682 5683 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { 5684 release_stripe(sh); 5685 raid5_set_bi_processed_stripes(raid_bio, scnt); 5686 conf->retry_read_aligned = raid_bio; 5687 return handled; 5688 } 5689 5690 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); 5691 handle_stripe(sh); 5692 release_stripe(sh); 5693 handled++; 5694 } 5695 remaining = raid5_dec_bi_active_stripes(raid_bio); 5696 if (remaining == 0) { 5697 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 5698 raid_bio, 0); 5699 bio_endio(raid_bio, 0); 5700 } 5701 if (atomic_dec_and_test(&conf->active_aligned_reads)) 5702 wake_up(&conf->wait_for_stripe); 5703 return handled; 5704 } 5705 5706 static int handle_active_stripes(struct r5conf *conf, int group, 5707 struct r5worker *worker, 5708 struct list_head *temp_inactive_list) 5709 { 5710 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; 5711 int i, batch_size = 0, hash; 5712 bool release_inactive = false; 5713 5714 while (batch_size < MAX_STRIPE_BATCH && 5715 (sh = __get_priority_stripe(conf, group)) != NULL) 5716 batch[batch_size++] = sh; 5717 5718 if (batch_size == 0) { 5719 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5720 if (!list_empty(temp_inactive_list + i)) 5721 break; 5722 if (i == NR_STRIPE_HASH_LOCKS) 5723 return batch_size; 5724 release_inactive = true; 5725 } 5726 spin_unlock_irq(&conf->device_lock); 5727 5728 release_inactive_stripe_list(conf, temp_inactive_list, 5729 NR_STRIPE_HASH_LOCKS); 5730 5731 if (release_inactive) { 5732 spin_lock_irq(&conf->device_lock); 5733 return 0; 5734 } 5735 5736 for (i = 0; i < batch_size; i++) 5737 handle_stripe(batch[i]); 5738 5739 cond_resched(); 5740 5741 spin_lock_irq(&conf->device_lock); 5742 for (i = 0; i < batch_size; i++) { 5743 hash = batch[i]->hash_lock_index; 5744 __release_stripe(conf, batch[i], &temp_inactive_list[hash]); 5745 } 5746 return batch_size; 5747 } 5748 5749 static void raid5_do_work(struct work_struct *work) 5750 { 5751 struct r5worker *worker = container_of(work, struct r5worker, work); 5752 struct r5worker_group *group = worker->group; 5753 struct r5conf *conf = group->conf; 5754 int group_id = group - conf->worker_groups; 5755 int handled; 5756 struct blk_plug plug; 5757 5758 pr_debug("+++ raid5worker active\n"); 5759 5760 blk_start_plug(&plug); 5761 handled = 0; 5762 spin_lock_irq(&conf->device_lock); 5763 while (1) { 5764 int batch_size, released; 5765 5766 released = release_stripe_list(conf, worker->temp_inactive_list); 5767 5768 batch_size = handle_active_stripes(conf, group_id, worker, 5769 worker->temp_inactive_list); 5770 worker->working = false; 5771 if (!batch_size && !released) 5772 break; 5773 handled += batch_size; 5774 } 5775 pr_debug("%d stripes handled\n", handled); 5776 5777 spin_unlock_irq(&conf->device_lock); 5778 blk_finish_plug(&plug); 5779 5780 pr_debug("--- raid5worker inactive\n"); 5781 } 5782 5783 /* 5784 * This is our raid5 kernel thread. 5785 * 5786 * We scan the hash table for stripes which can be handled now. 5787 * During the scan, completed stripes are saved for us by the interrupt 5788 * handler, so that they will not have to wait for our next wakeup. 5789 */ 5790 static void raid5d(struct md_thread *thread) 5791 { 5792 struct mddev *mddev = thread->mddev; 5793 struct r5conf *conf = mddev->private; 5794 int handled; 5795 struct blk_plug plug; 5796 5797 pr_debug("+++ raid5d active\n"); 5798 5799 md_check_recovery(mddev); 5800 5801 blk_start_plug(&plug); 5802 handled = 0; 5803 spin_lock_irq(&conf->device_lock); 5804 while (1) { 5805 struct bio *bio; 5806 int batch_size, released; 5807 5808 released = release_stripe_list(conf, conf->temp_inactive_list); 5809 if (released) 5810 clear_bit(R5_DID_ALLOC, &conf->cache_state); 5811 5812 if ( 5813 !list_empty(&conf->bitmap_list)) { 5814 /* Now is a good time to flush some bitmap updates */ 5815 conf->seq_flush++; 5816 spin_unlock_irq(&conf->device_lock); 5817 bitmap_unplug(mddev->bitmap); 5818 spin_lock_irq(&conf->device_lock); 5819 conf->seq_write = conf->seq_flush; 5820 activate_bit_delay(conf, conf->temp_inactive_list); 5821 } 5822 raid5_activate_delayed(conf); 5823 5824 while ((bio = remove_bio_from_retry(conf))) { 5825 int ok; 5826 spin_unlock_irq(&conf->device_lock); 5827 ok = retry_aligned_read(conf, bio); 5828 spin_lock_irq(&conf->device_lock); 5829 if (!ok) 5830 break; 5831 handled++; 5832 } 5833 5834 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, 5835 conf->temp_inactive_list); 5836 if (!batch_size && !released) 5837 break; 5838 handled += batch_size; 5839 5840 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) { 5841 spin_unlock_irq(&conf->device_lock); 5842 md_check_recovery(mddev); 5843 spin_lock_irq(&conf->device_lock); 5844 } 5845 } 5846 pr_debug("%d stripes handled\n", handled); 5847 5848 spin_unlock_irq(&conf->device_lock); 5849 if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state)) { 5850 grow_one_stripe(conf, __GFP_NOWARN); 5851 /* Set flag even if allocation failed. This helps 5852 * slow down allocation requests when mem is short 5853 */ 5854 set_bit(R5_DID_ALLOC, &conf->cache_state); 5855 } 5856 5857 async_tx_issue_pending_all(); 5858 blk_finish_plug(&plug); 5859 5860 pr_debug("--- raid5d inactive\n"); 5861 } 5862 5863 static ssize_t 5864 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 5865 { 5866 struct r5conf *conf; 5867 int ret = 0; 5868 spin_lock(&mddev->lock); 5869 conf = mddev->private; 5870 if (conf) 5871 ret = sprintf(page, "%d\n", conf->min_nr_stripes); 5872 spin_unlock(&mddev->lock); 5873 return ret; 5874 } 5875 5876 int 5877 raid5_set_cache_size(struct mddev *mddev, int size) 5878 { 5879 struct r5conf *conf = mddev->private; 5880 int err; 5881 5882 if (size <= 16 || size > 32768) 5883 return -EINVAL; 5884 5885 conf->min_nr_stripes = size; 5886 while (size < conf->max_nr_stripes && 5887 drop_one_stripe(conf)) 5888 ; 5889 5890 5891 err = md_allow_write(mddev); 5892 if (err) 5893 return err; 5894 5895 while (size > conf->max_nr_stripes) 5896 if (!grow_one_stripe(conf, GFP_KERNEL)) 5897 break; 5898 5899 return 0; 5900 } 5901 EXPORT_SYMBOL(raid5_set_cache_size); 5902 5903 static ssize_t 5904 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 5905 { 5906 struct r5conf *conf; 5907 unsigned long new; 5908 int err; 5909 5910 if (len >= PAGE_SIZE) 5911 return -EINVAL; 5912 if (kstrtoul(page, 10, &new)) 5913 return -EINVAL; 5914 err = mddev_lock(mddev); 5915 if (err) 5916 return err; 5917 conf = mddev->private; 5918 if (!conf) 5919 err = -ENODEV; 5920 else 5921 err = raid5_set_cache_size(mddev, new); 5922 mddev_unlock(mddev); 5923 5924 return err ?: len; 5925 } 5926 5927 static struct md_sysfs_entry 5928 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 5929 raid5_show_stripe_cache_size, 5930 raid5_store_stripe_cache_size); 5931 5932 static ssize_t 5933 raid5_show_rmw_level(struct mddev *mddev, char *page) 5934 { 5935 struct r5conf *conf = mddev->private; 5936 if (conf) 5937 return sprintf(page, "%d\n", conf->rmw_level); 5938 else 5939 return 0; 5940 } 5941 5942 static ssize_t 5943 raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) 5944 { 5945 struct r5conf *conf = mddev->private; 5946 unsigned long new; 5947 5948 if (!conf) 5949 return -ENODEV; 5950 5951 if (len >= PAGE_SIZE) 5952 return -EINVAL; 5953 5954 if (kstrtoul(page, 10, &new)) 5955 return -EINVAL; 5956 5957 if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) 5958 return -EINVAL; 5959 5960 if (new != PARITY_DISABLE_RMW && 5961 new != PARITY_ENABLE_RMW && 5962 new != PARITY_PREFER_RMW) 5963 return -EINVAL; 5964 5965 conf->rmw_level = new; 5966 return len; 5967 } 5968 5969 static struct md_sysfs_entry 5970 raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR, 5971 raid5_show_rmw_level, 5972 raid5_store_rmw_level); 5973 5974 5975 static ssize_t 5976 raid5_show_preread_threshold(struct mddev *mddev, char *page) 5977 { 5978 struct r5conf *conf; 5979 int ret = 0; 5980 spin_lock(&mddev->lock); 5981 conf = mddev->private; 5982 if (conf) 5983 ret = sprintf(page, "%d\n", conf->bypass_threshold); 5984 spin_unlock(&mddev->lock); 5985 return ret; 5986 } 5987 5988 static ssize_t 5989 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 5990 { 5991 struct r5conf *conf; 5992 unsigned long new; 5993 int err; 5994 5995 if (len >= PAGE_SIZE) 5996 return -EINVAL; 5997 if (kstrtoul(page, 10, &new)) 5998 return -EINVAL; 5999 6000 err = mddev_lock(mddev); 6001 if (err) 6002 return err; 6003 conf = mddev->private; 6004 if (!conf) 6005 err = -ENODEV; 6006 else if (new > conf->min_nr_stripes) 6007 err = -EINVAL; 6008 else 6009 conf->bypass_threshold = new; 6010 mddev_unlock(mddev); 6011 return err ?: len; 6012 } 6013 6014 static struct md_sysfs_entry 6015 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 6016 S_IRUGO | S_IWUSR, 6017 raid5_show_preread_threshold, 6018 raid5_store_preread_threshold); 6019 6020 static ssize_t 6021 raid5_show_skip_copy(struct mddev *mddev, char *page) 6022 { 6023 struct r5conf *conf; 6024 int ret = 0; 6025 spin_lock(&mddev->lock); 6026 conf = mddev->private; 6027 if (conf) 6028 ret = sprintf(page, "%d\n", conf->skip_copy); 6029 spin_unlock(&mddev->lock); 6030 return ret; 6031 } 6032 6033 static ssize_t 6034 raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) 6035 { 6036 struct r5conf *conf; 6037 unsigned long new; 6038 int err; 6039 6040 if (len >= PAGE_SIZE) 6041 return -EINVAL; 6042 if (kstrtoul(page, 10, &new)) 6043 return -EINVAL; 6044 new = !!new; 6045 6046 err = mddev_lock(mddev); 6047 if (err) 6048 return err; 6049 conf = mddev->private; 6050 if (!conf) 6051 err = -ENODEV; 6052 else if (new != conf->skip_copy) { 6053 mddev_suspend(mddev); 6054 conf->skip_copy = new; 6055 if (new) 6056 mddev->queue->backing_dev_info.capabilities |= 6057 BDI_CAP_STABLE_WRITES; 6058 else 6059 mddev->queue->backing_dev_info.capabilities &= 6060 ~BDI_CAP_STABLE_WRITES; 6061 mddev_resume(mddev); 6062 } 6063 mddev_unlock(mddev); 6064 return err ?: len; 6065 } 6066 6067 static struct md_sysfs_entry 6068 raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, 6069 raid5_show_skip_copy, 6070 raid5_store_skip_copy); 6071 6072 static ssize_t 6073 stripe_cache_active_show(struct mddev *mddev, char *page) 6074 { 6075 struct r5conf *conf = mddev->private; 6076 if (conf) 6077 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 6078 else 6079 return 0; 6080 } 6081 6082 static struct md_sysfs_entry 6083 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 6084 6085 static ssize_t 6086 raid5_show_group_thread_cnt(struct mddev *mddev, char *page) 6087 { 6088 struct r5conf *conf; 6089 int ret = 0; 6090 spin_lock(&mddev->lock); 6091 conf = mddev->private; 6092 if (conf) 6093 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); 6094 spin_unlock(&mddev->lock); 6095 return ret; 6096 } 6097 6098 static int alloc_thread_groups(struct r5conf *conf, int cnt, 6099 int *group_cnt, 6100 int *worker_cnt_per_group, 6101 struct r5worker_group **worker_groups); 6102 static ssize_t 6103 raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) 6104 { 6105 struct r5conf *conf; 6106 unsigned long new; 6107 int err; 6108 struct r5worker_group *new_groups, *old_groups; 6109 int group_cnt, worker_cnt_per_group; 6110 6111 if (len >= PAGE_SIZE) 6112 return -EINVAL; 6113 if (kstrtoul(page, 10, &new)) 6114 return -EINVAL; 6115 6116 err = mddev_lock(mddev); 6117 if (err) 6118 return err; 6119 conf = mddev->private; 6120 if (!conf) 6121 err = -ENODEV; 6122 else if (new != conf->worker_cnt_per_group) { 6123 mddev_suspend(mddev); 6124 6125 old_groups = conf->worker_groups; 6126 if (old_groups) 6127 flush_workqueue(raid5_wq); 6128 6129 err = alloc_thread_groups(conf, new, 6130 &group_cnt, &worker_cnt_per_group, 6131 &new_groups); 6132 if (!err) { 6133 spin_lock_irq(&conf->device_lock); 6134 conf->group_cnt = group_cnt; 6135 conf->worker_cnt_per_group = worker_cnt_per_group; 6136 conf->worker_groups = new_groups; 6137 spin_unlock_irq(&conf->device_lock); 6138 6139 if (old_groups) 6140 kfree(old_groups[0].workers); 6141 kfree(old_groups); 6142 } 6143 mddev_resume(mddev); 6144 } 6145 mddev_unlock(mddev); 6146 6147 return err ?: len; 6148 } 6149 6150 static struct md_sysfs_entry 6151 raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, 6152 raid5_show_group_thread_cnt, 6153 raid5_store_group_thread_cnt); 6154 6155 static struct attribute *raid5_attrs[] = { 6156 &raid5_stripecache_size.attr, 6157 &raid5_stripecache_active.attr, 6158 &raid5_preread_bypass_threshold.attr, 6159 &raid5_group_thread_cnt.attr, 6160 &raid5_skip_copy.attr, 6161 &raid5_rmw_level.attr, 6162 NULL, 6163 }; 6164 static struct attribute_group raid5_attrs_group = { 6165 .name = NULL, 6166 .attrs = raid5_attrs, 6167 }; 6168 6169 static int alloc_thread_groups(struct r5conf *conf, int cnt, 6170 int *group_cnt, 6171 int *worker_cnt_per_group, 6172 struct r5worker_group **worker_groups) 6173 { 6174 int i, j, k; 6175 ssize_t size; 6176 struct r5worker *workers; 6177 6178 *worker_cnt_per_group = cnt; 6179 if (cnt == 0) { 6180 *group_cnt = 0; 6181 *worker_groups = NULL; 6182 return 0; 6183 } 6184 *group_cnt = num_possible_nodes(); 6185 size = sizeof(struct r5worker) * cnt; 6186 workers = kzalloc(size * *group_cnt, GFP_NOIO); 6187 *worker_groups = kzalloc(sizeof(struct r5worker_group) * 6188 *group_cnt, GFP_NOIO); 6189 if (!*worker_groups || !workers) { 6190 kfree(workers); 6191 kfree(*worker_groups); 6192 return -ENOMEM; 6193 } 6194 6195 for (i = 0; i < *group_cnt; i++) { 6196 struct r5worker_group *group; 6197 6198 group = &(*worker_groups)[i]; 6199 INIT_LIST_HEAD(&group->handle_list); 6200 group->conf = conf; 6201 group->workers = workers + i * cnt; 6202 6203 for (j = 0; j < cnt; j++) { 6204 struct r5worker *worker = group->workers + j; 6205 worker->group = group; 6206 INIT_WORK(&worker->work, raid5_do_work); 6207 6208 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) 6209 INIT_LIST_HEAD(worker->temp_inactive_list + k); 6210 } 6211 } 6212 6213 return 0; 6214 } 6215 6216 static void free_thread_groups(struct r5conf *conf) 6217 { 6218 if (conf->worker_groups) 6219 kfree(conf->worker_groups[0].workers); 6220 kfree(conf->worker_groups); 6221 conf->worker_groups = NULL; 6222 } 6223 6224 static sector_t 6225 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 6226 { 6227 struct r5conf *conf = mddev->private; 6228 6229 if (!sectors) 6230 sectors = mddev->dev_sectors; 6231 if (!raid_disks) 6232 /* size is defined by the smallest of previous and new size */ 6233 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 6234 6235 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 6236 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 6237 return sectors * (raid_disks - conf->max_degraded); 6238 } 6239 6240 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 6241 { 6242 safe_put_page(percpu->spare_page); 6243 if (percpu->scribble) 6244 flex_array_free(percpu->scribble); 6245 percpu->spare_page = NULL; 6246 percpu->scribble = NULL; 6247 } 6248 6249 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 6250 { 6251 if (conf->level == 6 && !percpu->spare_page) 6252 percpu->spare_page = alloc_page(GFP_KERNEL); 6253 if (!percpu->scribble) 6254 percpu->scribble = scribble_alloc(max(conf->raid_disks, 6255 conf->previous_raid_disks), 6256 max(conf->chunk_sectors, 6257 conf->prev_chunk_sectors) 6258 / STRIPE_SECTORS, 6259 GFP_KERNEL); 6260 6261 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { 6262 free_scratch_buffer(conf, percpu); 6263 return -ENOMEM; 6264 } 6265 6266 return 0; 6267 } 6268 6269 static void raid5_free_percpu(struct r5conf *conf) 6270 { 6271 unsigned long cpu; 6272 6273 if (!conf->percpu) 6274 return; 6275 6276 #ifdef CONFIG_HOTPLUG_CPU 6277 unregister_cpu_notifier(&conf->cpu_notify); 6278 #endif 6279 6280 get_online_cpus(); 6281 for_each_possible_cpu(cpu) 6282 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 6283 put_online_cpus(); 6284 6285 free_percpu(conf->percpu); 6286 } 6287 6288 static void free_conf(struct r5conf *conf) 6289 { 6290 if (conf->shrinker.seeks) 6291 unregister_shrinker(&conf->shrinker); 6292 free_thread_groups(conf); 6293 shrink_stripes(conf); 6294 raid5_free_percpu(conf); 6295 kfree(conf->disks); 6296 kfree(conf->stripe_hashtbl); 6297 kfree(conf); 6298 } 6299 6300 #ifdef CONFIG_HOTPLUG_CPU 6301 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 6302 void *hcpu) 6303 { 6304 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); 6305 long cpu = (long)hcpu; 6306 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 6307 6308 switch (action) { 6309 case CPU_UP_PREPARE: 6310 case CPU_UP_PREPARE_FROZEN: 6311 if (alloc_scratch_buffer(conf, percpu)) { 6312 pr_err("%s: failed memory allocation for cpu%ld\n", 6313 __func__, cpu); 6314 return notifier_from_errno(-ENOMEM); 6315 } 6316 break; 6317 case CPU_DEAD: 6318 case CPU_DEAD_FROZEN: 6319 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 6320 break; 6321 default: 6322 break; 6323 } 6324 return NOTIFY_OK; 6325 } 6326 #endif 6327 6328 static int raid5_alloc_percpu(struct r5conf *conf) 6329 { 6330 unsigned long cpu; 6331 int err = 0; 6332 6333 conf->percpu = alloc_percpu(struct raid5_percpu); 6334 if (!conf->percpu) 6335 return -ENOMEM; 6336 6337 #ifdef CONFIG_HOTPLUG_CPU 6338 conf->cpu_notify.notifier_call = raid456_cpu_notify; 6339 conf->cpu_notify.priority = 0; 6340 err = register_cpu_notifier(&conf->cpu_notify); 6341 if (err) 6342 return err; 6343 #endif 6344 6345 get_online_cpus(); 6346 for_each_present_cpu(cpu) { 6347 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 6348 if (err) { 6349 pr_err("%s: failed memory allocation for cpu%ld\n", 6350 __func__, cpu); 6351 break; 6352 } 6353 } 6354 put_online_cpus(); 6355 6356 return err; 6357 } 6358 6359 static unsigned long raid5_cache_scan(struct shrinker *shrink, 6360 struct shrink_control *sc) 6361 { 6362 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); 6363 int ret = 0; 6364 while (ret < sc->nr_to_scan) { 6365 if (drop_one_stripe(conf) == 0) 6366 return SHRINK_STOP; 6367 ret++; 6368 } 6369 return ret; 6370 } 6371 6372 static unsigned long raid5_cache_count(struct shrinker *shrink, 6373 struct shrink_control *sc) 6374 { 6375 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); 6376 6377 if (conf->max_nr_stripes < conf->min_nr_stripes) 6378 /* unlikely, but not impossible */ 6379 return 0; 6380 return conf->max_nr_stripes - conf->min_nr_stripes; 6381 } 6382 6383 static struct r5conf *setup_conf(struct mddev *mddev) 6384 { 6385 struct r5conf *conf; 6386 int raid_disk, memory, max_disks; 6387 struct md_rdev *rdev; 6388 struct disk_info *disk; 6389 char pers_name[6]; 6390 int i; 6391 int group_cnt, worker_cnt_per_group; 6392 struct r5worker_group *new_group; 6393 6394 if (mddev->new_level != 5 6395 && mddev->new_level != 4 6396 && mddev->new_level != 6) { 6397 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 6398 mdname(mddev), mddev->new_level); 6399 return ERR_PTR(-EIO); 6400 } 6401 if ((mddev->new_level == 5 6402 && !algorithm_valid_raid5(mddev->new_layout)) || 6403 (mddev->new_level == 6 6404 && !algorithm_valid_raid6(mddev->new_layout))) { 6405 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 6406 mdname(mddev), mddev->new_layout); 6407 return ERR_PTR(-EIO); 6408 } 6409 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 6410 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 6411 mdname(mddev), mddev->raid_disks); 6412 return ERR_PTR(-EINVAL); 6413 } 6414 6415 if (!mddev->new_chunk_sectors || 6416 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 6417 !is_power_of_2(mddev->new_chunk_sectors)) { 6418 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 6419 mdname(mddev), mddev->new_chunk_sectors << 9); 6420 return ERR_PTR(-EINVAL); 6421 } 6422 6423 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 6424 if (conf == NULL) 6425 goto abort; 6426 /* Don't enable multi-threading by default*/ 6427 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, 6428 &new_group)) { 6429 conf->group_cnt = group_cnt; 6430 conf->worker_cnt_per_group = worker_cnt_per_group; 6431 conf->worker_groups = new_group; 6432 } else 6433 goto abort; 6434 spin_lock_init(&conf->device_lock); 6435 seqcount_init(&conf->gen_lock); 6436 init_waitqueue_head(&conf->wait_for_stripe); 6437 init_waitqueue_head(&conf->wait_for_overlap); 6438 INIT_LIST_HEAD(&conf->handle_list); 6439 INIT_LIST_HEAD(&conf->hold_list); 6440 INIT_LIST_HEAD(&conf->delayed_list); 6441 INIT_LIST_HEAD(&conf->bitmap_list); 6442 init_llist_head(&conf->released_stripes); 6443 atomic_set(&conf->active_stripes, 0); 6444 atomic_set(&conf->preread_active_stripes, 0); 6445 atomic_set(&conf->active_aligned_reads, 0); 6446 conf->bypass_threshold = BYPASS_THRESHOLD; 6447 conf->recovery_disabled = mddev->recovery_disabled - 1; 6448 6449 conf->raid_disks = mddev->raid_disks; 6450 if (mddev->reshape_position == MaxSector) 6451 conf->previous_raid_disks = mddev->raid_disks; 6452 else 6453 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 6454 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 6455 6456 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 6457 GFP_KERNEL); 6458 if (!conf->disks) 6459 goto abort; 6460 6461 conf->mddev = mddev; 6462 6463 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 6464 goto abort; 6465 6466 /* We init hash_locks[0] separately to that it can be used 6467 * as the reference lock in the spin_lock_nest_lock() call 6468 * in lock_all_device_hash_locks_irq in order to convince 6469 * lockdep that we know what we are doing. 6470 */ 6471 spin_lock_init(conf->hash_locks); 6472 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 6473 spin_lock_init(conf->hash_locks + i); 6474 6475 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 6476 INIT_LIST_HEAD(conf->inactive_list + i); 6477 6478 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 6479 INIT_LIST_HEAD(conf->temp_inactive_list + i); 6480 6481 conf->level = mddev->new_level; 6482 conf->chunk_sectors = mddev->new_chunk_sectors; 6483 if (raid5_alloc_percpu(conf) != 0) 6484 goto abort; 6485 6486 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 6487 6488 rdev_for_each(rdev, mddev) { 6489 raid_disk = rdev->raid_disk; 6490 if (raid_disk >= max_disks 6491 || raid_disk < 0) 6492 continue; 6493 disk = conf->disks + raid_disk; 6494 6495 if (test_bit(Replacement, &rdev->flags)) { 6496 if (disk->replacement) 6497 goto abort; 6498 disk->replacement = rdev; 6499 } else { 6500 if (disk->rdev) 6501 goto abort; 6502 disk->rdev = rdev; 6503 } 6504 6505 if (test_bit(In_sync, &rdev->flags)) { 6506 char b[BDEVNAME_SIZE]; 6507 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 6508 " disk %d\n", 6509 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 6510 } else if (rdev->saved_raid_disk != raid_disk) 6511 /* Cannot rely on bitmap to complete recovery */ 6512 conf->fullsync = 1; 6513 } 6514 6515 conf->level = mddev->new_level; 6516 if (conf->level == 6) { 6517 conf->max_degraded = 2; 6518 if (raid6_call.xor_syndrome) 6519 conf->rmw_level = PARITY_ENABLE_RMW; 6520 else 6521 conf->rmw_level = PARITY_DISABLE_RMW; 6522 } else { 6523 conf->max_degraded = 1; 6524 conf->rmw_level = PARITY_ENABLE_RMW; 6525 } 6526 conf->algorithm = mddev->new_layout; 6527 conf->reshape_progress = mddev->reshape_position; 6528 if (conf->reshape_progress != MaxSector) { 6529 conf->prev_chunk_sectors = mddev->chunk_sectors; 6530 conf->prev_algo = mddev->layout; 6531 } 6532 6533 conf->min_nr_stripes = NR_STRIPES; 6534 memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + 6535 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 6536 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); 6537 if (grow_stripes(conf, conf->min_nr_stripes)) { 6538 printk(KERN_ERR 6539 "md/raid:%s: couldn't allocate %dkB for buffers\n", 6540 mdname(mddev), memory); 6541 goto abort; 6542 } else 6543 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 6544 mdname(mddev), memory); 6545 /* 6546 * Losing a stripe head costs more than the time to refill it, 6547 * it reduces the queue depth and so can hurt throughput. 6548 * So set it rather large, scaled by number of devices. 6549 */ 6550 conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; 6551 conf->shrinker.scan_objects = raid5_cache_scan; 6552 conf->shrinker.count_objects = raid5_cache_count; 6553 conf->shrinker.batch = 128; 6554 conf->shrinker.flags = 0; 6555 register_shrinker(&conf->shrinker); 6556 6557 sprintf(pers_name, "raid%d", mddev->new_level); 6558 conf->thread = md_register_thread(raid5d, mddev, pers_name); 6559 if (!conf->thread) { 6560 printk(KERN_ERR 6561 "md/raid:%s: couldn't allocate thread.\n", 6562 mdname(mddev)); 6563 goto abort; 6564 } 6565 6566 return conf; 6567 6568 abort: 6569 if (conf) { 6570 free_conf(conf); 6571 return ERR_PTR(-EIO); 6572 } else 6573 return ERR_PTR(-ENOMEM); 6574 } 6575 6576 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 6577 { 6578 switch (algo) { 6579 case ALGORITHM_PARITY_0: 6580 if (raid_disk < max_degraded) 6581 return 1; 6582 break; 6583 case ALGORITHM_PARITY_N: 6584 if (raid_disk >= raid_disks - max_degraded) 6585 return 1; 6586 break; 6587 case ALGORITHM_PARITY_0_6: 6588 if (raid_disk == 0 || 6589 raid_disk == raid_disks - 1) 6590 return 1; 6591 break; 6592 case ALGORITHM_LEFT_ASYMMETRIC_6: 6593 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6594 case ALGORITHM_LEFT_SYMMETRIC_6: 6595 case ALGORITHM_RIGHT_SYMMETRIC_6: 6596 if (raid_disk == raid_disks - 1) 6597 return 1; 6598 } 6599 return 0; 6600 } 6601 6602 static int run(struct mddev *mddev) 6603 { 6604 struct r5conf *conf; 6605 int working_disks = 0; 6606 int dirty_parity_disks = 0; 6607 struct md_rdev *rdev; 6608 sector_t reshape_offset = 0; 6609 int i; 6610 long long min_offset_diff = 0; 6611 int first = 1; 6612 6613 if (mddev->recovery_cp != MaxSector) 6614 printk(KERN_NOTICE "md/raid:%s: not clean" 6615 " -- starting background reconstruction\n", 6616 mdname(mddev)); 6617 6618 rdev_for_each(rdev, mddev) { 6619 long long diff; 6620 if (rdev->raid_disk < 0) 6621 continue; 6622 diff = (rdev->new_data_offset - rdev->data_offset); 6623 if (first) { 6624 min_offset_diff = diff; 6625 first = 0; 6626 } else if (mddev->reshape_backwards && 6627 diff < min_offset_diff) 6628 min_offset_diff = diff; 6629 else if (!mddev->reshape_backwards && 6630 diff > min_offset_diff) 6631 min_offset_diff = diff; 6632 } 6633 6634 if (mddev->reshape_position != MaxSector) { 6635 /* Check that we can continue the reshape. 6636 * Difficulties arise if the stripe we would write to 6637 * next is at or after the stripe we would read from next. 6638 * For a reshape that changes the number of devices, this 6639 * is only possible for a very short time, and mdadm makes 6640 * sure that time appears to have past before assembling 6641 * the array. So we fail if that time hasn't passed. 6642 * For a reshape that keeps the number of devices the same 6643 * mdadm must be monitoring the reshape can keeping the 6644 * critical areas read-only and backed up. It will start 6645 * the array in read-only mode, so we check for that. 6646 */ 6647 sector_t here_new, here_old; 6648 int old_disks; 6649 int max_degraded = (mddev->level == 6 ? 2 : 1); 6650 6651 if (mddev->new_level != mddev->level) { 6652 printk(KERN_ERR "md/raid:%s: unsupported reshape " 6653 "required - aborting.\n", 6654 mdname(mddev)); 6655 return -EINVAL; 6656 } 6657 old_disks = mddev->raid_disks - mddev->delta_disks; 6658 /* reshape_position must be on a new-stripe boundary, and one 6659 * further up in new geometry must map after here in old 6660 * geometry. 6661 */ 6662 here_new = mddev->reshape_position; 6663 if (sector_div(here_new, mddev->new_chunk_sectors * 6664 (mddev->raid_disks - max_degraded))) { 6665 printk(KERN_ERR "md/raid:%s: reshape_position not " 6666 "on a stripe boundary\n", mdname(mddev)); 6667 return -EINVAL; 6668 } 6669 reshape_offset = here_new * mddev->new_chunk_sectors; 6670 /* here_new is the stripe we will write to */ 6671 here_old = mddev->reshape_position; 6672 sector_div(here_old, mddev->chunk_sectors * 6673 (old_disks-max_degraded)); 6674 /* here_old is the first stripe that we might need to read 6675 * from */ 6676 if (mddev->delta_disks == 0) { 6677 if ((here_new * mddev->new_chunk_sectors != 6678 here_old * mddev->chunk_sectors)) { 6679 printk(KERN_ERR "md/raid:%s: reshape position is" 6680 " confused - aborting\n", mdname(mddev)); 6681 return -EINVAL; 6682 } 6683 /* We cannot be sure it is safe to start an in-place 6684 * reshape. It is only safe if user-space is monitoring 6685 * and taking constant backups. 6686 * mdadm always starts a situation like this in 6687 * readonly mode so it can take control before 6688 * allowing any writes. So just check for that. 6689 */ 6690 if (abs(min_offset_diff) >= mddev->chunk_sectors && 6691 abs(min_offset_diff) >= mddev->new_chunk_sectors) 6692 /* not really in-place - so OK */; 6693 else if (mddev->ro == 0) { 6694 printk(KERN_ERR "md/raid:%s: in-place reshape " 6695 "must be started in read-only mode " 6696 "- aborting\n", 6697 mdname(mddev)); 6698 return -EINVAL; 6699 } 6700 } else if (mddev->reshape_backwards 6701 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= 6702 here_old * mddev->chunk_sectors) 6703 : (here_new * mddev->new_chunk_sectors >= 6704 here_old * mddev->chunk_sectors + (-min_offset_diff))) { 6705 /* Reading from the same stripe as writing to - bad */ 6706 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 6707 "auto-recovery - aborting.\n", 6708 mdname(mddev)); 6709 return -EINVAL; 6710 } 6711 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 6712 mdname(mddev)); 6713 /* OK, we should be able to continue; */ 6714 } else { 6715 BUG_ON(mddev->level != mddev->new_level); 6716 BUG_ON(mddev->layout != mddev->new_layout); 6717 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 6718 BUG_ON(mddev->delta_disks != 0); 6719 } 6720 6721 if (mddev->private == NULL) 6722 conf = setup_conf(mddev); 6723 else 6724 conf = mddev->private; 6725 6726 if (IS_ERR(conf)) 6727 return PTR_ERR(conf); 6728 6729 conf->min_offset_diff = min_offset_diff; 6730 mddev->thread = conf->thread; 6731 conf->thread = NULL; 6732 mddev->private = conf; 6733 6734 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 6735 i++) { 6736 rdev = conf->disks[i].rdev; 6737 if (!rdev && conf->disks[i].replacement) { 6738 /* The replacement is all we have yet */ 6739 rdev = conf->disks[i].replacement; 6740 conf->disks[i].replacement = NULL; 6741 clear_bit(Replacement, &rdev->flags); 6742 conf->disks[i].rdev = rdev; 6743 } 6744 if (!rdev) 6745 continue; 6746 if (conf->disks[i].replacement && 6747 conf->reshape_progress != MaxSector) { 6748 /* replacements and reshape simply do not mix. */ 6749 printk(KERN_ERR "md: cannot handle concurrent " 6750 "replacement and reshape.\n"); 6751 goto abort; 6752 } 6753 if (test_bit(In_sync, &rdev->flags)) { 6754 working_disks++; 6755 continue; 6756 } 6757 /* This disc is not fully in-sync. However if it 6758 * just stored parity (beyond the recovery_offset), 6759 * when we don't need to be concerned about the 6760 * array being dirty. 6761 * When reshape goes 'backwards', we never have 6762 * partially completed devices, so we only need 6763 * to worry about reshape going forwards. 6764 */ 6765 /* Hack because v0.91 doesn't store recovery_offset properly. */ 6766 if (mddev->major_version == 0 && 6767 mddev->minor_version > 90) 6768 rdev->recovery_offset = reshape_offset; 6769 6770 if (rdev->recovery_offset < reshape_offset) { 6771 /* We need to check old and new layout */ 6772 if (!only_parity(rdev->raid_disk, 6773 conf->algorithm, 6774 conf->raid_disks, 6775 conf->max_degraded)) 6776 continue; 6777 } 6778 if (!only_parity(rdev->raid_disk, 6779 conf->prev_algo, 6780 conf->previous_raid_disks, 6781 conf->max_degraded)) 6782 continue; 6783 dirty_parity_disks++; 6784 } 6785 6786 /* 6787 * 0 for a fully functional array, 1 or 2 for a degraded array. 6788 */ 6789 mddev->degraded = calc_degraded(conf); 6790 6791 if (has_failed(conf)) { 6792 printk(KERN_ERR "md/raid:%s: not enough operational devices" 6793 " (%d/%d failed)\n", 6794 mdname(mddev), mddev->degraded, conf->raid_disks); 6795 goto abort; 6796 } 6797 6798 /* device size must be a multiple of chunk size */ 6799 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 6800 mddev->resync_max_sectors = mddev->dev_sectors; 6801 6802 if (mddev->degraded > dirty_parity_disks && 6803 mddev->recovery_cp != MaxSector) { 6804 if (mddev->ok_start_degraded) 6805 printk(KERN_WARNING 6806 "md/raid:%s: starting dirty degraded array" 6807 " - data corruption possible.\n", 6808 mdname(mddev)); 6809 else { 6810 printk(KERN_ERR 6811 "md/raid:%s: cannot start dirty degraded array.\n", 6812 mdname(mddev)); 6813 goto abort; 6814 } 6815 } 6816 6817 if (mddev->degraded == 0) 6818 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 6819 " devices, algorithm %d\n", mdname(mddev), conf->level, 6820 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 6821 mddev->new_layout); 6822 else 6823 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 6824 " out of %d devices, algorithm %d\n", 6825 mdname(mddev), conf->level, 6826 mddev->raid_disks - mddev->degraded, 6827 mddev->raid_disks, mddev->new_layout); 6828 6829 print_raid5_conf(conf); 6830 6831 if (conf->reshape_progress != MaxSector) { 6832 conf->reshape_safe = conf->reshape_progress; 6833 atomic_set(&conf->reshape_stripes, 0); 6834 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6835 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6836 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6837 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6838 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6839 "reshape"); 6840 } 6841 6842 /* Ok, everything is just fine now */ 6843 if (mddev->to_remove == &raid5_attrs_group) 6844 mddev->to_remove = NULL; 6845 else if (mddev->kobj.sd && 6846 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 6847 printk(KERN_WARNING 6848 "raid5: failed to create sysfs attributes for %s\n", 6849 mdname(mddev)); 6850 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6851 6852 if (mddev->queue) { 6853 int chunk_size; 6854 bool discard_supported = true; 6855 /* read-ahead size must cover two whole stripes, which 6856 * is 2 * (datadisks) * chunksize where 'n' is the 6857 * number of raid devices 6858 */ 6859 int data_disks = conf->previous_raid_disks - conf->max_degraded; 6860 int stripe = data_disks * 6861 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 6862 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6863 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6864 6865 chunk_size = mddev->chunk_sectors << 9; 6866 blk_queue_io_min(mddev->queue, chunk_size); 6867 blk_queue_io_opt(mddev->queue, chunk_size * 6868 (conf->raid_disks - conf->max_degraded)); 6869 mddev->queue->limits.raid_partial_stripes_expensive = 1; 6870 /* 6871 * We can only discard a whole stripe. It doesn't make sense to 6872 * discard data disk but write parity disk 6873 */ 6874 stripe = stripe * PAGE_SIZE; 6875 /* Round up to power of 2, as discard handling 6876 * currently assumes that */ 6877 while ((stripe-1) & stripe) 6878 stripe = (stripe | (stripe-1)) + 1; 6879 mddev->queue->limits.discard_alignment = stripe; 6880 mddev->queue->limits.discard_granularity = stripe; 6881 /* 6882 * unaligned part of discard request will be ignored, so can't 6883 * guarantee discard_zeroes_data 6884 */ 6885 mddev->queue->limits.discard_zeroes_data = 0; 6886 6887 blk_queue_max_write_same_sectors(mddev->queue, 0); 6888 6889 rdev_for_each(rdev, mddev) { 6890 disk_stack_limits(mddev->gendisk, rdev->bdev, 6891 rdev->data_offset << 9); 6892 disk_stack_limits(mddev->gendisk, rdev->bdev, 6893 rdev->new_data_offset << 9); 6894 /* 6895 * discard_zeroes_data is required, otherwise data 6896 * could be lost. Consider a scenario: discard a stripe 6897 * (the stripe could be inconsistent if 6898 * discard_zeroes_data is 0); write one disk of the 6899 * stripe (the stripe could be inconsistent again 6900 * depending on which disks are used to calculate 6901 * parity); the disk is broken; The stripe data of this 6902 * disk is lost. 6903 */ 6904 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || 6905 !bdev_get_queue(rdev->bdev)-> 6906 limits.discard_zeroes_data) 6907 discard_supported = false; 6908 /* Unfortunately, discard_zeroes_data is not currently 6909 * a guarantee - just a hint. So we only allow DISCARD 6910 * if the sysadmin has confirmed that only safe devices 6911 * are in use by setting a module parameter. 6912 */ 6913 if (!devices_handle_discard_safely) { 6914 if (discard_supported) { 6915 pr_info("md/raid456: discard support disabled due to uncertainty.\n"); 6916 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); 6917 } 6918 discard_supported = false; 6919 } 6920 } 6921 6922 if (discard_supported && 6923 mddev->queue->limits.max_discard_sectors >= stripe && 6924 mddev->queue->limits.discard_granularity >= stripe) 6925 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 6926 mddev->queue); 6927 else 6928 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 6929 mddev->queue); 6930 } 6931 6932 return 0; 6933 abort: 6934 md_unregister_thread(&mddev->thread); 6935 print_raid5_conf(conf); 6936 free_conf(conf); 6937 mddev->private = NULL; 6938 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 6939 return -EIO; 6940 } 6941 6942 static void raid5_free(struct mddev *mddev, void *priv) 6943 { 6944 struct r5conf *conf = priv; 6945 6946 free_conf(conf); 6947 mddev->to_remove = &raid5_attrs_group; 6948 } 6949 6950 static void status(struct seq_file *seq, struct mddev *mddev) 6951 { 6952 struct r5conf *conf = mddev->private; 6953 int i; 6954 6955 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 6956 mddev->chunk_sectors / 2, mddev->layout); 6957 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 6958 for (i = 0; i < conf->raid_disks; i++) 6959 seq_printf (seq, "%s", 6960 conf->disks[i].rdev && 6961 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 6962 seq_printf (seq, "]"); 6963 } 6964 6965 static void print_raid5_conf (struct r5conf *conf) 6966 { 6967 int i; 6968 struct disk_info *tmp; 6969 6970 printk(KERN_DEBUG "RAID conf printout:\n"); 6971 if (!conf) { 6972 printk("(conf==NULL)\n"); 6973 return; 6974 } 6975 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 6976 conf->raid_disks, 6977 conf->raid_disks - conf->mddev->degraded); 6978 6979 for (i = 0; i < conf->raid_disks; i++) { 6980 char b[BDEVNAME_SIZE]; 6981 tmp = conf->disks + i; 6982 if (tmp->rdev) 6983 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 6984 i, !test_bit(Faulty, &tmp->rdev->flags), 6985 bdevname(tmp->rdev->bdev, b)); 6986 } 6987 } 6988 6989 static int raid5_spare_active(struct mddev *mddev) 6990 { 6991 int i; 6992 struct r5conf *conf = mddev->private; 6993 struct disk_info *tmp; 6994 int count = 0; 6995 unsigned long flags; 6996 6997 for (i = 0; i < conf->raid_disks; i++) { 6998 tmp = conf->disks + i; 6999 if (tmp->replacement 7000 && tmp->replacement->recovery_offset == MaxSector 7001 && !test_bit(Faulty, &tmp->replacement->flags) 7002 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 7003 /* Replacement has just become active. */ 7004 if (!tmp->rdev 7005 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 7006 count++; 7007 if (tmp->rdev) { 7008 /* Replaced device not technically faulty, 7009 * but we need to be sure it gets removed 7010 * and never re-added. 7011 */ 7012 set_bit(Faulty, &tmp->rdev->flags); 7013 sysfs_notify_dirent_safe( 7014 tmp->rdev->sysfs_state); 7015 } 7016 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 7017 } else if (tmp->rdev 7018 && tmp->rdev->recovery_offset == MaxSector 7019 && !test_bit(Faulty, &tmp->rdev->flags) 7020 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 7021 count++; 7022 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 7023 } 7024 } 7025 spin_lock_irqsave(&conf->device_lock, flags); 7026 mddev->degraded = calc_degraded(conf); 7027 spin_unlock_irqrestore(&conf->device_lock, flags); 7028 print_raid5_conf(conf); 7029 return count; 7030 } 7031 7032 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 7033 { 7034 struct r5conf *conf = mddev->private; 7035 int err = 0; 7036 int number = rdev->raid_disk; 7037 struct md_rdev **rdevp; 7038 struct disk_info *p = conf->disks + number; 7039 7040 print_raid5_conf(conf); 7041 if (rdev == p->rdev) 7042 rdevp = &p->rdev; 7043 else if (rdev == p->replacement) 7044 rdevp = &p->replacement; 7045 else 7046 return 0; 7047 7048 if (number >= conf->raid_disks && 7049 conf->reshape_progress == MaxSector) 7050 clear_bit(In_sync, &rdev->flags); 7051 7052 if (test_bit(In_sync, &rdev->flags) || 7053 atomic_read(&rdev->nr_pending)) { 7054 err = -EBUSY; 7055 goto abort; 7056 } 7057 /* Only remove non-faulty devices if recovery 7058 * isn't possible. 7059 */ 7060 if (!test_bit(Faulty, &rdev->flags) && 7061 mddev->recovery_disabled != conf->recovery_disabled && 7062 !has_failed(conf) && 7063 (!p->replacement || p->replacement == rdev) && 7064 number < conf->raid_disks) { 7065 err = -EBUSY; 7066 goto abort; 7067 } 7068 *rdevp = NULL; 7069 synchronize_rcu(); 7070 if (atomic_read(&rdev->nr_pending)) { 7071 /* lost the race, try later */ 7072 err = -EBUSY; 7073 *rdevp = rdev; 7074 } else if (p->replacement) { 7075 /* We must have just cleared 'rdev' */ 7076 p->rdev = p->replacement; 7077 clear_bit(Replacement, &p->replacement->flags); 7078 smp_mb(); /* Make sure other CPUs may see both as identical 7079 * but will never see neither - if they are careful 7080 */ 7081 p->replacement = NULL; 7082 clear_bit(WantReplacement, &rdev->flags); 7083 } else 7084 /* We might have just removed the Replacement as faulty- 7085 * clear the bit just in case 7086 */ 7087 clear_bit(WantReplacement, &rdev->flags); 7088 abort: 7089 7090 print_raid5_conf(conf); 7091 return err; 7092 } 7093 7094 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 7095 { 7096 struct r5conf *conf = mddev->private; 7097 int err = -EEXIST; 7098 int disk; 7099 struct disk_info *p; 7100 int first = 0; 7101 int last = conf->raid_disks - 1; 7102 7103 if (mddev->recovery_disabled == conf->recovery_disabled) 7104 return -EBUSY; 7105 7106 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 7107 /* no point adding a device */ 7108 return -EINVAL; 7109 7110 if (rdev->raid_disk >= 0) 7111 first = last = rdev->raid_disk; 7112 7113 /* 7114 * find the disk ... but prefer rdev->saved_raid_disk 7115 * if possible. 7116 */ 7117 if (rdev->saved_raid_disk >= 0 && 7118 rdev->saved_raid_disk >= first && 7119 conf->disks[rdev->saved_raid_disk].rdev == NULL) 7120 first = rdev->saved_raid_disk; 7121 7122 for (disk = first; disk <= last; disk++) { 7123 p = conf->disks + disk; 7124 if (p->rdev == NULL) { 7125 clear_bit(In_sync, &rdev->flags); 7126 rdev->raid_disk = disk; 7127 err = 0; 7128 if (rdev->saved_raid_disk != disk) 7129 conf->fullsync = 1; 7130 rcu_assign_pointer(p->rdev, rdev); 7131 goto out; 7132 } 7133 } 7134 for (disk = first; disk <= last; disk++) { 7135 p = conf->disks + disk; 7136 if (test_bit(WantReplacement, &p->rdev->flags) && 7137 p->replacement == NULL) { 7138 clear_bit(In_sync, &rdev->flags); 7139 set_bit(Replacement, &rdev->flags); 7140 rdev->raid_disk = disk; 7141 err = 0; 7142 conf->fullsync = 1; 7143 rcu_assign_pointer(p->replacement, rdev); 7144 break; 7145 } 7146 } 7147 out: 7148 print_raid5_conf(conf); 7149 return err; 7150 } 7151 7152 static int raid5_resize(struct mddev *mddev, sector_t sectors) 7153 { 7154 /* no resync is happening, and there is enough space 7155 * on all devices, so we can resize. 7156 * We need to make sure resync covers any new space. 7157 * If the array is shrinking we should possibly wait until 7158 * any io in the removed space completes, but it hardly seems 7159 * worth it. 7160 */ 7161 sector_t newsize; 7162 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 7163 newsize = raid5_size(mddev, sectors, mddev->raid_disks); 7164 if (mddev->external_size && 7165 mddev->array_sectors > newsize) 7166 return -EINVAL; 7167 if (mddev->bitmap) { 7168 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); 7169 if (ret) 7170 return ret; 7171 } 7172 md_set_array_sectors(mddev, newsize); 7173 set_capacity(mddev->gendisk, mddev->array_sectors); 7174 revalidate_disk(mddev->gendisk); 7175 if (sectors > mddev->dev_sectors && 7176 mddev->recovery_cp > mddev->dev_sectors) { 7177 mddev->recovery_cp = mddev->dev_sectors; 7178 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 7179 } 7180 mddev->dev_sectors = sectors; 7181 mddev->resync_max_sectors = sectors; 7182 return 0; 7183 } 7184 7185 static int check_stripe_cache(struct mddev *mddev) 7186 { 7187 /* Can only proceed if there are plenty of stripe_heads. 7188 * We need a minimum of one full stripe,, and for sensible progress 7189 * it is best to have about 4 times that. 7190 * If we require 4 times, then the default 256 4K stripe_heads will 7191 * allow for chunk sizes up to 256K, which is probably OK. 7192 * If the chunk size is greater, user-space should request more 7193 * stripe_heads first. 7194 */ 7195 struct r5conf *conf = mddev->private; 7196 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 7197 > conf->min_nr_stripes || 7198 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 7199 > conf->min_nr_stripes) { 7200 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 7201 mdname(mddev), 7202 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 7203 / STRIPE_SIZE)*4); 7204 return 0; 7205 } 7206 return 1; 7207 } 7208 7209 static int check_reshape(struct mddev *mddev) 7210 { 7211 struct r5conf *conf = mddev->private; 7212 7213 if (mddev->delta_disks == 0 && 7214 mddev->new_layout == mddev->layout && 7215 mddev->new_chunk_sectors == mddev->chunk_sectors) 7216 return 0; /* nothing to do */ 7217 if (has_failed(conf)) 7218 return -EINVAL; 7219 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { 7220 /* We might be able to shrink, but the devices must 7221 * be made bigger first. 7222 * For raid6, 4 is the minimum size. 7223 * Otherwise 2 is the minimum 7224 */ 7225 int min = 2; 7226 if (mddev->level == 6) 7227 min = 4; 7228 if (mddev->raid_disks + mddev->delta_disks < min) 7229 return -EINVAL; 7230 } 7231 7232 if (!check_stripe_cache(mddev)) 7233 return -ENOSPC; 7234 7235 if (mddev->new_chunk_sectors > mddev->chunk_sectors || 7236 mddev->delta_disks > 0) 7237 if (resize_chunks(conf, 7238 conf->previous_raid_disks 7239 + max(0, mddev->delta_disks), 7240 max(mddev->new_chunk_sectors, 7241 mddev->chunk_sectors) 7242 ) < 0) 7243 return -ENOMEM; 7244 return resize_stripes(conf, (conf->previous_raid_disks 7245 + mddev->delta_disks)); 7246 } 7247 7248 static int raid5_start_reshape(struct mddev *mddev) 7249 { 7250 struct r5conf *conf = mddev->private; 7251 struct md_rdev *rdev; 7252 int spares = 0; 7253 unsigned long flags; 7254 7255 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 7256 return -EBUSY; 7257 7258 if (!check_stripe_cache(mddev)) 7259 return -ENOSPC; 7260 7261 if (has_failed(conf)) 7262 return -EINVAL; 7263 7264 rdev_for_each(rdev, mddev) { 7265 if (!test_bit(In_sync, &rdev->flags) 7266 && !test_bit(Faulty, &rdev->flags)) 7267 spares++; 7268 } 7269 7270 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 7271 /* Not enough devices even to make a degraded array 7272 * of that size 7273 */ 7274 return -EINVAL; 7275 7276 /* Refuse to reduce size of the array. Any reductions in 7277 * array size must be through explicit setting of array_size 7278 * attribute. 7279 */ 7280 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 7281 < mddev->array_sectors) { 7282 printk(KERN_ERR "md/raid:%s: array size must be reduced " 7283 "before number of disks\n", mdname(mddev)); 7284 return -EINVAL; 7285 } 7286 7287 atomic_set(&conf->reshape_stripes, 0); 7288 spin_lock_irq(&conf->device_lock); 7289 write_seqcount_begin(&conf->gen_lock); 7290 conf->previous_raid_disks = conf->raid_disks; 7291 conf->raid_disks += mddev->delta_disks; 7292 conf->prev_chunk_sectors = conf->chunk_sectors; 7293 conf->chunk_sectors = mddev->new_chunk_sectors; 7294 conf->prev_algo = conf->algorithm; 7295 conf->algorithm = mddev->new_layout; 7296 conf->generation++; 7297 /* Code that selects data_offset needs to see the generation update 7298 * if reshape_progress has been set - so a memory barrier needed. 7299 */ 7300 smp_mb(); 7301 if (mddev->reshape_backwards) 7302 conf->reshape_progress = raid5_size(mddev, 0, 0); 7303 else 7304 conf->reshape_progress = 0; 7305 conf->reshape_safe = conf->reshape_progress; 7306 write_seqcount_end(&conf->gen_lock); 7307 spin_unlock_irq(&conf->device_lock); 7308 7309 /* Now make sure any requests that proceeded on the assumption 7310 * the reshape wasn't running - like Discard or Read - have 7311 * completed. 7312 */ 7313 mddev_suspend(mddev); 7314 mddev_resume(mddev); 7315 7316 /* Add some new drives, as many as will fit. 7317 * We know there are enough to make the newly sized array work. 7318 * Don't add devices if we are reducing the number of 7319 * devices in the array. This is because it is not possible 7320 * to correctly record the "partially reconstructed" state of 7321 * such devices during the reshape and confusion could result. 7322 */ 7323 if (mddev->delta_disks >= 0) { 7324 rdev_for_each(rdev, mddev) 7325 if (rdev->raid_disk < 0 && 7326 !test_bit(Faulty, &rdev->flags)) { 7327 if (raid5_add_disk(mddev, rdev) == 0) { 7328 if (rdev->raid_disk 7329 >= conf->previous_raid_disks) 7330 set_bit(In_sync, &rdev->flags); 7331 else 7332 rdev->recovery_offset = 0; 7333 7334 if (sysfs_link_rdev(mddev, rdev)) 7335 /* Failure here is OK */; 7336 } 7337 } else if (rdev->raid_disk >= conf->previous_raid_disks 7338 && !test_bit(Faulty, &rdev->flags)) { 7339 /* This is a spare that was manually added */ 7340 set_bit(In_sync, &rdev->flags); 7341 } 7342 7343 /* When a reshape changes the number of devices, 7344 * ->degraded is measured against the larger of the 7345 * pre and post number of devices. 7346 */ 7347 spin_lock_irqsave(&conf->device_lock, flags); 7348 mddev->degraded = calc_degraded(conf); 7349 spin_unlock_irqrestore(&conf->device_lock, flags); 7350 } 7351 mddev->raid_disks = conf->raid_disks; 7352 mddev->reshape_position = conf->reshape_progress; 7353 set_bit(MD_CHANGE_DEVS, &mddev->flags); 7354 7355 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 7356 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 7357 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 7358 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 7359 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 7360 "reshape"); 7361 if (!mddev->sync_thread) { 7362 mddev->recovery = 0; 7363 spin_lock_irq(&conf->device_lock); 7364 write_seqcount_begin(&conf->gen_lock); 7365 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 7366 mddev->new_chunk_sectors = 7367 conf->chunk_sectors = conf->prev_chunk_sectors; 7368 mddev->new_layout = conf->algorithm = conf->prev_algo; 7369 rdev_for_each(rdev, mddev) 7370 rdev->new_data_offset = rdev->data_offset; 7371 smp_wmb(); 7372 conf->generation --; 7373 conf->reshape_progress = MaxSector; 7374 mddev->reshape_position = MaxSector; 7375 write_seqcount_end(&conf->gen_lock); 7376 spin_unlock_irq(&conf->device_lock); 7377 return -EAGAIN; 7378 } 7379 conf->reshape_checkpoint = jiffies; 7380 md_wakeup_thread(mddev->sync_thread); 7381 md_new_event(mddev); 7382 return 0; 7383 } 7384 7385 /* This is called from the reshape thread and should make any 7386 * changes needed in 'conf' 7387 */ 7388 static void end_reshape(struct r5conf *conf) 7389 { 7390 7391 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 7392 struct md_rdev *rdev; 7393 7394 spin_lock_irq(&conf->device_lock); 7395 conf->previous_raid_disks = conf->raid_disks; 7396 rdev_for_each(rdev, conf->mddev) 7397 rdev->data_offset = rdev->new_data_offset; 7398 smp_wmb(); 7399 conf->reshape_progress = MaxSector; 7400 spin_unlock_irq(&conf->device_lock); 7401 wake_up(&conf->wait_for_overlap); 7402 7403 /* read-ahead size must cover two whole stripes, which is 7404 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 7405 */ 7406 if (conf->mddev->queue) { 7407 int data_disks = conf->raid_disks - conf->max_degraded; 7408 int stripe = data_disks * ((conf->chunk_sectors << 9) 7409 / PAGE_SIZE); 7410 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 7411 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 7412 } 7413 } 7414 } 7415 7416 /* This is called from the raid5d thread with mddev_lock held. 7417 * It makes config changes to the device. 7418 */ 7419 static void raid5_finish_reshape(struct mddev *mddev) 7420 { 7421 struct r5conf *conf = mddev->private; 7422 7423 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 7424 7425 if (mddev->delta_disks > 0) { 7426 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 7427 set_capacity(mddev->gendisk, mddev->array_sectors); 7428 revalidate_disk(mddev->gendisk); 7429 } else { 7430 int d; 7431 spin_lock_irq(&conf->device_lock); 7432 mddev->degraded = calc_degraded(conf); 7433 spin_unlock_irq(&conf->device_lock); 7434 for (d = conf->raid_disks ; 7435 d < conf->raid_disks - mddev->delta_disks; 7436 d++) { 7437 struct md_rdev *rdev = conf->disks[d].rdev; 7438 if (rdev) 7439 clear_bit(In_sync, &rdev->flags); 7440 rdev = conf->disks[d].replacement; 7441 if (rdev) 7442 clear_bit(In_sync, &rdev->flags); 7443 } 7444 } 7445 mddev->layout = conf->algorithm; 7446 mddev->chunk_sectors = conf->chunk_sectors; 7447 mddev->reshape_position = MaxSector; 7448 mddev->delta_disks = 0; 7449 mddev->reshape_backwards = 0; 7450 } 7451 } 7452 7453 static void raid5_quiesce(struct mddev *mddev, int state) 7454 { 7455 struct r5conf *conf = mddev->private; 7456 7457 switch(state) { 7458 case 2: /* resume for a suspend */ 7459 wake_up(&conf->wait_for_overlap); 7460 break; 7461 7462 case 1: /* stop all writes */ 7463 lock_all_device_hash_locks_irq(conf); 7464 /* '2' tells resync/reshape to pause so that all 7465 * active stripes can drain 7466 */ 7467 conf->quiesce = 2; 7468 wait_event_cmd(conf->wait_for_stripe, 7469 atomic_read(&conf->active_stripes) == 0 && 7470 atomic_read(&conf->active_aligned_reads) == 0, 7471 unlock_all_device_hash_locks_irq(conf), 7472 lock_all_device_hash_locks_irq(conf)); 7473 conf->quiesce = 1; 7474 unlock_all_device_hash_locks_irq(conf); 7475 /* allow reshape to continue */ 7476 wake_up(&conf->wait_for_overlap); 7477 break; 7478 7479 case 0: /* re-enable writes */ 7480 lock_all_device_hash_locks_irq(conf); 7481 conf->quiesce = 0; 7482 wake_up(&conf->wait_for_stripe); 7483 wake_up(&conf->wait_for_overlap); 7484 unlock_all_device_hash_locks_irq(conf); 7485 break; 7486 } 7487 } 7488 7489 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 7490 { 7491 struct r0conf *raid0_conf = mddev->private; 7492 sector_t sectors; 7493 7494 /* for raid0 takeover only one zone is supported */ 7495 if (raid0_conf->nr_strip_zones > 1) { 7496 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 7497 mdname(mddev)); 7498 return ERR_PTR(-EINVAL); 7499 } 7500 7501 sectors = raid0_conf->strip_zone[0].zone_end; 7502 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 7503 mddev->dev_sectors = sectors; 7504 mddev->new_level = level; 7505 mddev->new_layout = ALGORITHM_PARITY_N; 7506 mddev->new_chunk_sectors = mddev->chunk_sectors; 7507 mddev->raid_disks += 1; 7508 mddev->delta_disks = 1; 7509 /* make sure it will be not marked as dirty */ 7510 mddev->recovery_cp = MaxSector; 7511 7512 return setup_conf(mddev); 7513 } 7514 7515 static void *raid5_takeover_raid1(struct mddev *mddev) 7516 { 7517 int chunksect; 7518 7519 if (mddev->raid_disks != 2 || 7520 mddev->degraded > 1) 7521 return ERR_PTR(-EINVAL); 7522 7523 /* Should check if there are write-behind devices? */ 7524 7525 chunksect = 64*2; /* 64K by default */ 7526 7527 /* The array must be an exact multiple of chunksize */ 7528 while (chunksect && (mddev->array_sectors & (chunksect-1))) 7529 chunksect >>= 1; 7530 7531 if ((chunksect<<9) < STRIPE_SIZE) 7532 /* array size does not allow a suitable chunk size */ 7533 return ERR_PTR(-EINVAL); 7534 7535 mddev->new_level = 5; 7536 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 7537 mddev->new_chunk_sectors = chunksect; 7538 7539 return setup_conf(mddev); 7540 } 7541 7542 static void *raid5_takeover_raid6(struct mddev *mddev) 7543 { 7544 int new_layout; 7545 7546 switch (mddev->layout) { 7547 case ALGORITHM_LEFT_ASYMMETRIC_6: 7548 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 7549 break; 7550 case ALGORITHM_RIGHT_ASYMMETRIC_6: 7551 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 7552 break; 7553 case ALGORITHM_LEFT_SYMMETRIC_6: 7554 new_layout = ALGORITHM_LEFT_SYMMETRIC; 7555 break; 7556 case ALGORITHM_RIGHT_SYMMETRIC_6: 7557 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 7558 break; 7559 case ALGORITHM_PARITY_0_6: 7560 new_layout = ALGORITHM_PARITY_0; 7561 break; 7562 case ALGORITHM_PARITY_N: 7563 new_layout = ALGORITHM_PARITY_N; 7564 break; 7565 default: 7566 return ERR_PTR(-EINVAL); 7567 } 7568 mddev->new_level = 5; 7569 mddev->new_layout = new_layout; 7570 mddev->delta_disks = -1; 7571 mddev->raid_disks -= 1; 7572 return setup_conf(mddev); 7573 } 7574 7575 static int raid5_check_reshape(struct mddev *mddev) 7576 { 7577 /* For a 2-drive array, the layout and chunk size can be changed 7578 * immediately as not restriping is needed. 7579 * For larger arrays we record the new value - after validation 7580 * to be used by a reshape pass. 7581 */ 7582 struct r5conf *conf = mddev->private; 7583 int new_chunk = mddev->new_chunk_sectors; 7584 7585 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 7586 return -EINVAL; 7587 if (new_chunk > 0) { 7588 if (!is_power_of_2(new_chunk)) 7589 return -EINVAL; 7590 if (new_chunk < (PAGE_SIZE>>9)) 7591 return -EINVAL; 7592 if (mddev->array_sectors & (new_chunk-1)) 7593 /* not factor of array size */ 7594 return -EINVAL; 7595 } 7596 7597 /* They look valid */ 7598 7599 if (mddev->raid_disks == 2) { 7600 /* can make the change immediately */ 7601 if (mddev->new_layout >= 0) { 7602 conf->algorithm = mddev->new_layout; 7603 mddev->layout = mddev->new_layout; 7604 } 7605 if (new_chunk > 0) { 7606 conf->chunk_sectors = new_chunk ; 7607 mddev->chunk_sectors = new_chunk; 7608 } 7609 set_bit(MD_CHANGE_DEVS, &mddev->flags); 7610 md_wakeup_thread(mddev->thread); 7611 } 7612 return check_reshape(mddev); 7613 } 7614 7615 static int raid6_check_reshape(struct mddev *mddev) 7616 { 7617 int new_chunk = mddev->new_chunk_sectors; 7618 7619 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 7620 return -EINVAL; 7621 if (new_chunk > 0) { 7622 if (!is_power_of_2(new_chunk)) 7623 return -EINVAL; 7624 if (new_chunk < (PAGE_SIZE >> 9)) 7625 return -EINVAL; 7626 if (mddev->array_sectors & (new_chunk-1)) 7627 /* not factor of array size */ 7628 return -EINVAL; 7629 } 7630 7631 /* They look valid */ 7632 return check_reshape(mddev); 7633 } 7634 7635 static void *raid5_takeover(struct mddev *mddev) 7636 { 7637 /* raid5 can take over: 7638 * raid0 - if there is only one strip zone - make it a raid4 layout 7639 * raid1 - if there are two drives. We need to know the chunk size 7640 * raid4 - trivial - just use a raid4 layout. 7641 * raid6 - Providing it is a *_6 layout 7642 */ 7643 if (mddev->level == 0) 7644 return raid45_takeover_raid0(mddev, 5); 7645 if (mddev->level == 1) 7646 return raid5_takeover_raid1(mddev); 7647 if (mddev->level == 4) { 7648 mddev->new_layout = ALGORITHM_PARITY_N; 7649 mddev->new_level = 5; 7650 return setup_conf(mddev); 7651 } 7652 if (mddev->level == 6) 7653 return raid5_takeover_raid6(mddev); 7654 7655 return ERR_PTR(-EINVAL); 7656 } 7657 7658 static void *raid4_takeover(struct mddev *mddev) 7659 { 7660 /* raid4 can take over: 7661 * raid0 - if there is only one strip zone 7662 * raid5 - if layout is right 7663 */ 7664 if (mddev->level == 0) 7665 return raid45_takeover_raid0(mddev, 4); 7666 if (mddev->level == 5 && 7667 mddev->layout == ALGORITHM_PARITY_N) { 7668 mddev->new_layout = 0; 7669 mddev->new_level = 4; 7670 return setup_conf(mddev); 7671 } 7672 return ERR_PTR(-EINVAL); 7673 } 7674 7675 static struct md_personality raid5_personality; 7676 7677 static void *raid6_takeover(struct mddev *mddev) 7678 { 7679 /* Currently can only take over a raid5. We map the 7680 * personality to an equivalent raid6 personality 7681 * with the Q block at the end. 7682 */ 7683 int new_layout; 7684 7685 if (mddev->pers != &raid5_personality) 7686 return ERR_PTR(-EINVAL); 7687 if (mddev->degraded > 1) 7688 return ERR_PTR(-EINVAL); 7689 if (mddev->raid_disks > 253) 7690 return ERR_PTR(-EINVAL); 7691 if (mddev->raid_disks < 3) 7692 return ERR_PTR(-EINVAL); 7693 7694 switch (mddev->layout) { 7695 case ALGORITHM_LEFT_ASYMMETRIC: 7696 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 7697 break; 7698 case ALGORITHM_RIGHT_ASYMMETRIC: 7699 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 7700 break; 7701 case ALGORITHM_LEFT_SYMMETRIC: 7702 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 7703 break; 7704 case ALGORITHM_RIGHT_SYMMETRIC: 7705 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 7706 break; 7707 case ALGORITHM_PARITY_0: 7708 new_layout = ALGORITHM_PARITY_0_6; 7709 break; 7710 case ALGORITHM_PARITY_N: 7711 new_layout = ALGORITHM_PARITY_N; 7712 break; 7713 default: 7714 return ERR_PTR(-EINVAL); 7715 } 7716 mddev->new_level = 6; 7717 mddev->new_layout = new_layout; 7718 mddev->delta_disks = 1; 7719 mddev->raid_disks += 1; 7720 return setup_conf(mddev); 7721 } 7722 7723 static struct md_personality raid6_personality = 7724 { 7725 .name = "raid6", 7726 .level = 6, 7727 .owner = THIS_MODULE, 7728 .make_request = make_request, 7729 .run = run, 7730 .free = raid5_free, 7731 .status = status, 7732 .error_handler = error, 7733 .hot_add_disk = raid5_add_disk, 7734 .hot_remove_disk= raid5_remove_disk, 7735 .spare_active = raid5_spare_active, 7736 .sync_request = sync_request, 7737 .resize = raid5_resize, 7738 .size = raid5_size, 7739 .check_reshape = raid6_check_reshape, 7740 .start_reshape = raid5_start_reshape, 7741 .finish_reshape = raid5_finish_reshape, 7742 .quiesce = raid5_quiesce, 7743 .takeover = raid6_takeover, 7744 .congested = raid5_congested, 7745 .mergeable_bvec = raid5_mergeable_bvec, 7746 }; 7747 static struct md_personality raid5_personality = 7748 { 7749 .name = "raid5", 7750 .level = 5, 7751 .owner = THIS_MODULE, 7752 .make_request = make_request, 7753 .run = run, 7754 .free = raid5_free, 7755 .status = status, 7756 .error_handler = error, 7757 .hot_add_disk = raid5_add_disk, 7758 .hot_remove_disk= raid5_remove_disk, 7759 .spare_active = raid5_spare_active, 7760 .sync_request = sync_request, 7761 .resize = raid5_resize, 7762 .size = raid5_size, 7763 .check_reshape = raid5_check_reshape, 7764 .start_reshape = raid5_start_reshape, 7765 .finish_reshape = raid5_finish_reshape, 7766 .quiesce = raid5_quiesce, 7767 .takeover = raid5_takeover, 7768 .congested = raid5_congested, 7769 .mergeable_bvec = raid5_mergeable_bvec, 7770 }; 7771 7772 static struct md_personality raid4_personality = 7773 { 7774 .name = "raid4", 7775 .level = 4, 7776 .owner = THIS_MODULE, 7777 .make_request = make_request, 7778 .run = run, 7779 .free = raid5_free, 7780 .status = status, 7781 .error_handler = error, 7782 .hot_add_disk = raid5_add_disk, 7783 .hot_remove_disk= raid5_remove_disk, 7784 .spare_active = raid5_spare_active, 7785 .sync_request = sync_request, 7786 .resize = raid5_resize, 7787 .size = raid5_size, 7788 .check_reshape = raid5_check_reshape, 7789 .start_reshape = raid5_start_reshape, 7790 .finish_reshape = raid5_finish_reshape, 7791 .quiesce = raid5_quiesce, 7792 .takeover = raid4_takeover, 7793 .congested = raid5_congested, 7794 .mergeable_bvec = raid5_mergeable_bvec, 7795 }; 7796 7797 static int __init raid5_init(void) 7798 { 7799 raid5_wq = alloc_workqueue("raid5wq", 7800 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 7801 if (!raid5_wq) 7802 return -ENOMEM; 7803 register_md_personality(&raid6_personality); 7804 register_md_personality(&raid5_personality); 7805 register_md_personality(&raid4_personality); 7806 return 0; 7807 } 7808 7809 static void raid5_exit(void) 7810 { 7811 unregister_md_personality(&raid6_personality); 7812 unregister_md_personality(&raid5_personality); 7813 unregister_md_personality(&raid4_personality); 7814 destroy_workqueue(raid5_wq); 7815 } 7816 7817 module_init(raid5_init); 7818 module_exit(raid5_exit); 7819 MODULE_LICENSE("GPL"); 7820 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 7821 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 7822 MODULE_ALIAS("md-raid5"); 7823 MODULE_ALIAS("md-raid4"); 7824 MODULE_ALIAS("md-level-5"); 7825 MODULE_ALIAS("md-level-4"); 7826 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 7827 MODULE_ALIAS("md-raid6"); 7828 MODULE_ALIAS("md-level-6"); 7829 7830 /* This used to be two separate modules, they were: */ 7831 MODULE_ALIAS("raid5"); 7832 MODULE_ALIAS("raid6"); 7833