1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include <linux/nodemask.h> 57 #include <trace/events/block.h> 58 59 #include "md.h" 60 #include "raid5.h" 61 #include "raid0.h" 62 #include "bitmap.h" 63 64 #define cpu_to_group(cpu) cpu_to_node(cpu) 65 #define ANY_GROUP NUMA_NO_NODE 66 67 static bool devices_handle_discard_safely = false; 68 module_param(devices_handle_discard_safely, bool, 0644); 69 MODULE_PARM_DESC(devices_handle_discard_safely, 70 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); 71 static struct workqueue_struct *raid5_wq; 72 /* 73 * Stripe cache 74 */ 75 76 #define NR_STRIPES 256 77 #define STRIPE_SIZE PAGE_SIZE 78 #define STRIPE_SHIFT (PAGE_SHIFT - 9) 79 #define STRIPE_SECTORS (STRIPE_SIZE>>9) 80 #define IO_THRESHOLD 1 81 #define BYPASS_THRESHOLD 1 82 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) 83 #define HASH_MASK (NR_HASH - 1) 84 #define MAX_STRIPE_BATCH 8 85 86 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 87 { 88 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 89 return &conf->stripe_hashtbl[hash]; 90 } 91 92 static inline int stripe_hash_locks_hash(sector_t sect) 93 { 94 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; 95 } 96 97 static inline void lock_device_hash_lock(struct r5conf *conf, int hash) 98 { 99 spin_lock_irq(conf->hash_locks + hash); 100 spin_lock(&conf->device_lock); 101 } 102 103 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) 104 { 105 spin_unlock(&conf->device_lock); 106 spin_unlock_irq(conf->hash_locks + hash); 107 } 108 109 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) 110 { 111 int i; 112 local_irq_disable(); 113 spin_lock(conf->hash_locks); 114 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 115 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); 116 spin_lock(&conf->device_lock); 117 } 118 119 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) 120 { 121 int i; 122 spin_unlock(&conf->device_lock); 123 for (i = NR_STRIPE_HASH_LOCKS; i; i--) 124 spin_unlock(conf->hash_locks + i - 1); 125 local_irq_enable(); 126 } 127 128 /* bio's attached to a stripe+device for I/O are linked together in bi_sector 129 * order without overlap. There may be several bio's per stripe+device, and 130 * a bio could span several devices. 131 * When walking this list for a particular stripe+device, we must never proceed 132 * beyond a bio that extends past this device, as the next bio might no longer 133 * be valid. 134 * This function is used to determine the 'next' bio in the list, given the sector 135 * of the current stripe+device 136 */ 137 static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) 138 { 139 int sectors = bio_sectors(bio); 140 if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) 141 return bio->bi_next; 142 else 143 return NULL; 144 } 145 146 /* 147 * We maintain a biased count of active stripes in the bottom 16 bits of 148 * bi_phys_segments, and a count of processed stripes in the upper 16 bits 149 */ 150 static inline int raid5_bi_processed_stripes(struct bio *bio) 151 { 152 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 153 return (atomic_read(segments) >> 16) & 0xffff; 154 } 155 156 static inline int raid5_dec_bi_active_stripes(struct bio *bio) 157 { 158 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 159 return atomic_sub_return(1, segments) & 0xffff; 160 } 161 162 static inline void raid5_inc_bi_active_stripes(struct bio *bio) 163 { 164 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 165 atomic_inc(segments); 166 } 167 168 static inline void raid5_set_bi_processed_stripes(struct bio *bio, 169 unsigned int cnt) 170 { 171 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 172 int old, new; 173 174 do { 175 old = atomic_read(segments); 176 new = (old & 0xffff) | (cnt << 16); 177 } while (atomic_cmpxchg(segments, old, new) != old); 178 } 179 180 static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) 181 { 182 atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; 183 atomic_set(segments, cnt); 184 } 185 186 /* Find first data disk in a raid6 stripe */ 187 static inline int raid6_d0(struct stripe_head *sh) 188 { 189 if (sh->ddf_layout) 190 /* ddf always start from first device */ 191 return 0; 192 /* md starts just after Q block */ 193 if (sh->qd_idx == sh->disks - 1) 194 return 0; 195 else 196 return sh->qd_idx + 1; 197 } 198 static inline int raid6_next_disk(int disk, int raid_disks) 199 { 200 disk++; 201 return (disk < raid_disks) ? disk : 0; 202 } 203 204 /* When walking through the disks in a raid5, starting at raid6_d0, 205 * We need to map each disk to a 'slot', where the data disks are slot 206 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 207 * is raid_disks-1. This help does that mapping. 208 */ 209 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 210 int *count, int syndrome_disks) 211 { 212 int slot = *count; 213 214 if (sh->ddf_layout) 215 (*count)++; 216 if (idx == sh->pd_idx) 217 return syndrome_disks; 218 if (idx == sh->qd_idx) 219 return syndrome_disks + 1; 220 if (!sh->ddf_layout) 221 (*count)++; 222 return slot; 223 } 224 225 static void return_io(struct bio *return_bi) 226 { 227 struct bio *bi = return_bi; 228 while (bi) { 229 230 return_bi = bi->bi_next; 231 bi->bi_next = NULL; 232 bi->bi_iter.bi_size = 0; 233 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 234 bi, 0); 235 bio_endio(bi, 0); 236 bi = return_bi; 237 } 238 } 239 240 static void print_raid5_conf (struct r5conf *conf); 241 242 static int stripe_operations_active(struct stripe_head *sh) 243 { 244 return sh->check_state || sh->reconstruct_state || 245 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 246 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 247 } 248 249 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) 250 { 251 struct r5conf *conf = sh->raid_conf; 252 struct r5worker_group *group; 253 int thread_cnt; 254 int i, cpu = sh->cpu; 255 256 if (!cpu_online(cpu)) { 257 cpu = cpumask_any(cpu_online_mask); 258 sh->cpu = cpu; 259 } 260 261 if (list_empty(&sh->lru)) { 262 struct r5worker_group *group; 263 group = conf->worker_groups + cpu_to_group(cpu); 264 list_add_tail(&sh->lru, &group->handle_list); 265 group->stripes_cnt++; 266 sh->group = group; 267 } 268 269 if (conf->worker_cnt_per_group == 0) { 270 md_wakeup_thread(conf->mddev->thread); 271 return; 272 } 273 274 group = conf->worker_groups + cpu_to_group(sh->cpu); 275 276 group->workers[0].working = true; 277 /* at least one worker should run to avoid race */ 278 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); 279 280 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; 281 /* wakeup more workers */ 282 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { 283 if (group->workers[i].working == false) { 284 group->workers[i].working = true; 285 queue_work_on(sh->cpu, raid5_wq, 286 &group->workers[i].work); 287 thread_cnt--; 288 } 289 } 290 } 291 292 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, 293 struct list_head *temp_inactive_list) 294 { 295 BUG_ON(!list_empty(&sh->lru)); 296 BUG_ON(atomic_read(&conf->active_stripes)==0); 297 if (test_bit(STRIPE_HANDLE, &sh->state)) { 298 if (test_bit(STRIPE_DELAYED, &sh->state) && 299 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) { 300 list_add_tail(&sh->lru, &conf->delayed_list); 301 if (atomic_read(&conf->preread_active_stripes) 302 < IO_THRESHOLD) 303 md_wakeup_thread(conf->mddev->thread); 304 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 305 sh->bm_seq - conf->seq_write > 0) 306 list_add_tail(&sh->lru, &conf->bitmap_list); 307 else { 308 clear_bit(STRIPE_DELAYED, &sh->state); 309 clear_bit(STRIPE_BIT_DELAY, &sh->state); 310 if (conf->worker_cnt_per_group == 0) { 311 list_add_tail(&sh->lru, &conf->handle_list); 312 } else { 313 raid5_wakeup_stripe_thread(sh); 314 return; 315 } 316 } 317 md_wakeup_thread(conf->mddev->thread); 318 } else { 319 BUG_ON(stripe_operations_active(sh)); 320 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 321 if (atomic_dec_return(&conf->preread_active_stripes) 322 < IO_THRESHOLD) 323 md_wakeup_thread(conf->mddev->thread); 324 atomic_dec(&conf->active_stripes); 325 if (!test_bit(STRIPE_EXPANDING, &sh->state)) 326 list_add_tail(&sh->lru, temp_inactive_list); 327 } 328 } 329 330 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, 331 struct list_head *temp_inactive_list) 332 { 333 if (atomic_dec_and_test(&sh->count)) 334 do_release_stripe(conf, sh, temp_inactive_list); 335 } 336 337 /* 338 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list 339 * 340 * Be careful: Only one task can add/delete stripes from temp_inactive_list at 341 * given time. Adding stripes only takes device lock, while deleting stripes 342 * only takes hash lock. 343 */ 344 static void release_inactive_stripe_list(struct r5conf *conf, 345 struct list_head *temp_inactive_list, 346 int hash) 347 { 348 int size; 349 bool do_wakeup = false; 350 unsigned long flags; 351 352 if (hash == NR_STRIPE_HASH_LOCKS) { 353 size = NR_STRIPE_HASH_LOCKS; 354 hash = NR_STRIPE_HASH_LOCKS - 1; 355 } else 356 size = 1; 357 while (size) { 358 struct list_head *list = &temp_inactive_list[size - 1]; 359 360 /* 361 * We don't hold any lock here yet, get_active_stripe() might 362 * remove stripes from the list 363 */ 364 if (!list_empty_careful(list)) { 365 spin_lock_irqsave(conf->hash_locks + hash, flags); 366 if (list_empty(conf->inactive_list + hash) && 367 !list_empty(list)) 368 atomic_dec(&conf->empty_inactive_list_nr); 369 list_splice_tail_init(list, conf->inactive_list + hash); 370 do_wakeup = true; 371 spin_unlock_irqrestore(conf->hash_locks + hash, flags); 372 } 373 size--; 374 hash--; 375 } 376 377 if (do_wakeup) { 378 wake_up(&conf->wait_for_stripe); 379 if (conf->retry_read_aligned) 380 md_wakeup_thread(conf->mddev->thread); 381 } 382 } 383 384 /* should hold conf->device_lock already */ 385 static int release_stripe_list(struct r5conf *conf, 386 struct list_head *temp_inactive_list) 387 { 388 struct stripe_head *sh; 389 int count = 0; 390 struct llist_node *head; 391 392 head = llist_del_all(&conf->released_stripes); 393 head = llist_reverse_order(head); 394 while (head) { 395 int hash; 396 397 sh = llist_entry(head, struct stripe_head, release_list); 398 head = llist_next(head); 399 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ 400 smp_mb(); 401 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); 402 /* 403 * Don't worry the bit is set here, because if the bit is set 404 * again, the count is always > 1. This is true for 405 * STRIPE_ON_UNPLUG_LIST bit too. 406 */ 407 hash = sh->hash_lock_index; 408 __release_stripe(conf, sh, &temp_inactive_list[hash]); 409 count++; 410 } 411 412 return count; 413 } 414 415 static void release_stripe(struct stripe_head *sh) 416 { 417 struct r5conf *conf = sh->raid_conf; 418 unsigned long flags; 419 struct list_head list; 420 int hash; 421 bool wakeup; 422 423 /* Avoid release_list until the last reference. 424 */ 425 if (atomic_add_unless(&sh->count, -1, 1)) 426 return; 427 428 if (unlikely(!conf->mddev->thread) || 429 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) 430 goto slow_path; 431 wakeup = llist_add(&sh->release_list, &conf->released_stripes); 432 if (wakeup) 433 md_wakeup_thread(conf->mddev->thread); 434 return; 435 slow_path: 436 local_irq_save(flags); 437 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ 438 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { 439 INIT_LIST_HEAD(&list); 440 hash = sh->hash_lock_index; 441 do_release_stripe(conf, sh, &list); 442 spin_unlock(&conf->device_lock); 443 release_inactive_stripe_list(conf, &list, hash); 444 } 445 local_irq_restore(flags); 446 } 447 448 static inline void remove_hash(struct stripe_head *sh) 449 { 450 pr_debug("remove_hash(), stripe %llu\n", 451 (unsigned long long)sh->sector); 452 453 hlist_del_init(&sh->hash); 454 } 455 456 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 457 { 458 struct hlist_head *hp = stripe_hash(conf, sh->sector); 459 460 pr_debug("insert_hash(), stripe %llu\n", 461 (unsigned long long)sh->sector); 462 463 hlist_add_head(&sh->hash, hp); 464 } 465 466 /* find an idle stripe, make sure it is unhashed, and return it. */ 467 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) 468 { 469 struct stripe_head *sh = NULL; 470 struct list_head *first; 471 472 if (list_empty(conf->inactive_list + hash)) 473 goto out; 474 first = (conf->inactive_list + hash)->next; 475 sh = list_entry(first, struct stripe_head, lru); 476 list_del_init(first); 477 remove_hash(sh); 478 atomic_inc(&conf->active_stripes); 479 BUG_ON(hash != sh->hash_lock_index); 480 if (list_empty(conf->inactive_list + hash)) 481 atomic_inc(&conf->empty_inactive_list_nr); 482 out: 483 return sh; 484 } 485 486 static void shrink_buffers(struct stripe_head *sh) 487 { 488 struct page *p; 489 int i; 490 int num = sh->raid_conf->pool_size; 491 492 for (i = 0; i < num ; i++) { 493 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); 494 p = sh->dev[i].page; 495 if (!p) 496 continue; 497 sh->dev[i].page = NULL; 498 put_page(p); 499 } 500 } 501 502 static int grow_buffers(struct stripe_head *sh) 503 { 504 int i; 505 int num = sh->raid_conf->pool_size; 506 507 for (i = 0; i < num; i++) { 508 struct page *page; 509 510 if (!(page = alloc_page(GFP_KERNEL))) { 511 return 1; 512 } 513 sh->dev[i].page = page; 514 sh->dev[i].orig_page = page; 515 } 516 return 0; 517 } 518 519 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 520 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 521 struct stripe_head *sh); 522 523 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 524 { 525 struct r5conf *conf = sh->raid_conf; 526 int i, seq; 527 528 BUG_ON(atomic_read(&sh->count) != 0); 529 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 530 BUG_ON(stripe_operations_active(sh)); 531 532 pr_debug("init_stripe called, stripe %llu\n", 533 (unsigned long long)sector); 534 retry: 535 seq = read_seqcount_begin(&conf->gen_lock); 536 sh->generation = conf->generation - previous; 537 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 538 sh->sector = sector; 539 stripe_set_idx(sector, conf, previous, sh); 540 sh->state = 0; 541 542 for (i = sh->disks; i--; ) { 543 struct r5dev *dev = &sh->dev[i]; 544 545 if (dev->toread || dev->read || dev->towrite || dev->written || 546 test_bit(R5_LOCKED, &dev->flags)) { 547 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", 548 (unsigned long long)sh->sector, i, dev->toread, 549 dev->read, dev->towrite, dev->written, 550 test_bit(R5_LOCKED, &dev->flags)); 551 WARN_ON(1); 552 } 553 dev->flags = 0; 554 raid5_build_block(sh, i, previous); 555 } 556 if (read_seqcount_retry(&conf->gen_lock, seq)) 557 goto retry; 558 insert_hash(conf, sh); 559 sh->cpu = smp_processor_id(); 560 } 561 562 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 563 short generation) 564 { 565 struct stripe_head *sh; 566 567 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 568 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) 569 if (sh->sector == sector && sh->generation == generation) 570 return sh; 571 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 572 return NULL; 573 } 574 575 /* 576 * Need to check if array has failed when deciding whether to: 577 * - start an array 578 * - remove non-faulty devices 579 * - add a spare 580 * - allow a reshape 581 * This determination is simple when no reshape is happening. 582 * However if there is a reshape, we need to carefully check 583 * both the before and after sections. 584 * This is because some failed devices may only affect one 585 * of the two sections, and some non-in_sync devices may 586 * be insync in the section most affected by failed devices. 587 */ 588 static int calc_degraded(struct r5conf *conf) 589 { 590 int degraded, degraded2; 591 int i; 592 593 rcu_read_lock(); 594 degraded = 0; 595 for (i = 0; i < conf->previous_raid_disks; i++) { 596 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 597 if (rdev && test_bit(Faulty, &rdev->flags)) 598 rdev = rcu_dereference(conf->disks[i].replacement); 599 if (!rdev || test_bit(Faulty, &rdev->flags)) 600 degraded++; 601 else if (test_bit(In_sync, &rdev->flags)) 602 ; 603 else 604 /* not in-sync or faulty. 605 * If the reshape increases the number of devices, 606 * this is being recovered by the reshape, so 607 * this 'previous' section is not in_sync. 608 * If the number of devices is being reduced however, 609 * the device can only be part of the array if 610 * we are reverting a reshape, so this section will 611 * be in-sync. 612 */ 613 if (conf->raid_disks >= conf->previous_raid_disks) 614 degraded++; 615 } 616 rcu_read_unlock(); 617 if (conf->raid_disks == conf->previous_raid_disks) 618 return degraded; 619 rcu_read_lock(); 620 degraded2 = 0; 621 for (i = 0; i < conf->raid_disks; i++) { 622 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 623 if (rdev && test_bit(Faulty, &rdev->flags)) 624 rdev = rcu_dereference(conf->disks[i].replacement); 625 if (!rdev || test_bit(Faulty, &rdev->flags)) 626 degraded2++; 627 else if (test_bit(In_sync, &rdev->flags)) 628 ; 629 else 630 /* not in-sync or faulty. 631 * If reshape increases the number of devices, this 632 * section has already been recovered, else it 633 * almost certainly hasn't. 634 */ 635 if (conf->raid_disks <= conf->previous_raid_disks) 636 degraded2++; 637 } 638 rcu_read_unlock(); 639 if (degraded2 > degraded) 640 return degraded2; 641 return degraded; 642 } 643 644 static int has_failed(struct r5conf *conf) 645 { 646 int degraded; 647 648 if (conf->mddev->reshape_position == MaxSector) 649 return conf->mddev->degraded > conf->max_degraded; 650 651 degraded = calc_degraded(conf); 652 if (degraded > conf->max_degraded) 653 return 1; 654 return 0; 655 } 656 657 static struct stripe_head * 658 get_active_stripe(struct r5conf *conf, sector_t sector, 659 int previous, int noblock, int noquiesce) 660 { 661 struct stripe_head *sh; 662 int hash = stripe_hash_locks_hash(sector); 663 664 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 665 666 spin_lock_irq(conf->hash_locks + hash); 667 668 do { 669 wait_event_lock_irq(conf->wait_for_stripe, 670 conf->quiesce == 0 || noquiesce, 671 *(conf->hash_locks + hash)); 672 sh = __find_stripe(conf, sector, conf->generation - previous); 673 if (!sh) { 674 if (!conf->inactive_blocked) 675 sh = get_free_stripe(conf, hash); 676 if (noblock && sh == NULL) 677 break; 678 if (!sh) { 679 conf->inactive_blocked = 1; 680 wait_event_lock_irq( 681 conf->wait_for_stripe, 682 !list_empty(conf->inactive_list + hash) && 683 (atomic_read(&conf->active_stripes) 684 < (conf->max_nr_stripes * 3 / 4) 685 || !conf->inactive_blocked), 686 *(conf->hash_locks + hash)); 687 conf->inactive_blocked = 0; 688 } else { 689 init_stripe(sh, sector, previous); 690 atomic_inc(&sh->count); 691 } 692 } else if (!atomic_inc_not_zero(&sh->count)) { 693 spin_lock(&conf->device_lock); 694 if (!atomic_read(&sh->count)) { 695 if (!test_bit(STRIPE_HANDLE, &sh->state)) 696 atomic_inc(&conf->active_stripes); 697 BUG_ON(list_empty(&sh->lru) && 698 !test_bit(STRIPE_EXPANDING, &sh->state)); 699 list_del_init(&sh->lru); 700 if (sh->group) { 701 sh->group->stripes_cnt--; 702 sh->group = NULL; 703 } 704 } 705 atomic_inc(&sh->count); 706 spin_unlock(&conf->device_lock); 707 } 708 } while (sh == NULL); 709 710 spin_unlock_irq(conf->hash_locks + hash); 711 return sh; 712 } 713 714 /* Determine if 'data_offset' or 'new_data_offset' should be used 715 * in this stripe_head. 716 */ 717 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 718 { 719 sector_t progress = conf->reshape_progress; 720 /* Need a memory barrier to make sure we see the value 721 * of conf->generation, or ->data_offset that was set before 722 * reshape_progress was updated. 723 */ 724 smp_rmb(); 725 if (progress == MaxSector) 726 return 0; 727 if (sh->generation == conf->generation - 1) 728 return 0; 729 /* We are in a reshape, and this is a new-generation stripe, 730 * so use new_data_offset. 731 */ 732 return 1; 733 } 734 735 static void 736 raid5_end_read_request(struct bio *bi, int error); 737 static void 738 raid5_end_write_request(struct bio *bi, int error); 739 740 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 741 { 742 struct r5conf *conf = sh->raid_conf; 743 int i, disks = sh->disks; 744 745 might_sleep(); 746 747 for (i = disks; i--; ) { 748 int rw; 749 int replace_only = 0; 750 struct bio *bi, *rbi; 751 struct md_rdev *rdev, *rrdev = NULL; 752 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 753 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 754 rw = WRITE_FUA; 755 else 756 rw = WRITE; 757 if (test_bit(R5_Discard, &sh->dev[i].flags)) 758 rw |= REQ_DISCARD; 759 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 760 rw = READ; 761 else if (test_and_clear_bit(R5_WantReplace, 762 &sh->dev[i].flags)) { 763 rw = WRITE; 764 replace_only = 1; 765 } else 766 continue; 767 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) 768 rw |= REQ_SYNC; 769 770 bi = &sh->dev[i].req; 771 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 772 773 rcu_read_lock(); 774 rrdev = rcu_dereference(conf->disks[i].replacement); 775 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 776 rdev = rcu_dereference(conf->disks[i].rdev); 777 if (!rdev) { 778 rdev = rrdev; 779 rrdev = NULL; 780 } 781 if (rw & WRITE) { 782 if (replace_only) 783 rdev = NULL; 784 if (rdev == rrdev) 785 /* We raced and saw duplicates */ 786 rrdev = NULL; 787 } else { 788 if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev) 789 rdev = rrdev; 790 rrdev = NULL; 791 } 792 793 if (rdev && test_bit(Faulty, &rdev->flags)) 794 rdev = NULL; 795 if (rdev) 796 atomic_inc(&rdev->nr_pending); 797 if (rrdev && test_bit(Faulty, &rrdev->flags)) 798 rrdev = NULL; 799 if (rrdev) 800 atomic_inc(&rrdev->nr_pending); 801 rcu_read_unlock(); 802 803 /* We have already checked bad blocks for reads. Now 804 * need to check for writes. We never accept write errors 805 * on the replacement, so we don't to check rrdev. 806 */ 807 while ((rw & WRITE) && rdev && 808 test_bit(WriteErrorSeen, &rdev->flags)) { 809 sector_t first_bad; 810 int bad_sectors; 811 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 812 &first_bad, &bad_sectors); 813 if (!bad) 814 break; 815 816 if (bad < 0) { 817 set_bit(BlockedBadBlocks, &rdev->flags); 818 if (!conf->mddev->external && 819 conf->mddev->flags) { 820 /* It is very unlikely, but we might 821 * still need to write out the 822 * bad block log - better give it 823 * a chance*/ 824 md_check_recovery(conf->mddev); 825 } 826 /* 827 * Because md_wait_for_blocked_rdev 828 * will dec nr_pending, we must 829 * increment it first. 830 */ 831 atomic_inc(&rdev->nr_pending); 832 md_wait_for_blocked_rdev(rdev, conf->mddev); 833 } else { 834 /* Acknowledged bad block - skip the write */ 835 rdev_dec_pending(rdev, conf->mddev); 836 rdev = NULL; 837 } 838 } 839 840 if (rdev) { 841 if (s->syncing || s->expanding || s->expanded 842 || s->replacing) 843 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 844 845 set_bit(STRIPE_IO_STARTED, &sh->state); 846 847 bio_reset(bi); 848 bi->bi_bdev = rdev->bdev; 849 bi->bi_rw = rw; 850 bi->bi_end_io = (rw & WRITE) 851 ? raid5_end_write_request 852 : raid5_end_read_request; 853 bi->bi_private = sh; 854 855 pr_debug("%s: for %llu schedule op %ld on disc %d\n", 856 __func__, (unsigned long long)sh->sector, 857 bi->bi_rw, i); 858 atomic_inc(&sh->count); 859 if (use_new_offset(conf, sh)) 860 bi->bi_iter.bi_sector = (sh->sector 861 + rdev->new_data_offset); 862 else 863 bi->bi_iter.bi_sector = (sh->sector 864 + rdev->data_offset); 865 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 866 bi->bi_rw |= REQ_NOMERGE; 867 868 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 869 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 870 sh->dev[i].vec.bv_page = sh->dev[i].page; 871 bi->bi_vcnt = 1; 872 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 873 bi->bi_io_vec[0].bv_offset = 0; 874 bi->bi_iter.bi_size = STRIPE_SIZE; 875 /* 876 * If this is discard request, set bi_vcnt 0. We don't 877 * want to confuse SCSI because SCSI will replace payload 878 */ 879 if (rw & REQ_DISCARD) 880 bi->bi_vcnt = 0; 881 if (rrdev) 882 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 883 884 if (conf->mddev->gendisk) 885 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), 886 bi, disk_devt(conf->mddev->gendisk), 887 sh->dev[i].sector); 888 generic_make_request(bi); 889 } 890 if (rrdev) { 891 if (s->syncing || s->expanding || s->expanded 892 || s->replacing) 893 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 894 895 set_bit(STRIPE_IO_STARTED, &sh->state); 896 897 bio_reset(rbi); 898 rbi->bi_bdev = rrdev->bdev; 899 rbi->bi_rw = rw; 900 BUG_ON(!(rw & WRITE)); 901 rbi->bi_end_io = raid5_end_write_request; 902 rbi->bi_private = sh; 903 904 pr_debug("%s: for %llu schedule op %ld on " 905 "replacement disc %d\n", 906 __func__, (unsigned long long)sh->sector, 907 rbi->bi_rw, i); 908 atomic_inc(&sh->count); 909 if (use_new_offset(conf, sh)) 910 rbi->bi_iter.bi_sector = (sh->sector 911 + rrdev->new_data_offset); 912 else 913 rbi->bi_iter.bi_sector = (sh->sector 914 + rrdev->data_offset); 915 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 916 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 917 sh->dev[i].rvec.bv_page = sh->dev[i].page; 918 rbi->bi_vcnt = 1; 919 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 920 rbi->bi_io_vec[0].bv_offset = 0; 921 rbi->bi_iter.bi_size = STRIPE_SIZE; 922 /* 923 * If this is discard request, set bi_vcnt 0. We don't 924 * want to confuse SCSI because SCSI will replace payload 925 */ 926 if (rw & REQ_DISCARD) 927 rbi->bi_vcnt = 0; 928 if (conf->mddev->gendisk) 929 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), 930 rbi, disk_devt(conf->mddev->gendisk), 931 sh->dev[i].sector); 932 generic_make_request(rbi); 933 } 934 if (!rdev && !rrdev) { 935 if (rw & WRITE) 936 set_bit(STRIPE_DEGRADED, &sh->state); 937 pr_debug("skip op %ld on disc %d for sector %llu\n", 938 bi->bi_rw, i, (unsigned long long)sh->sector); 939 clear_bit(R5_LOCKED, &sh->dev[i].flags); 940 set_bit(STRIPE_HANDLE, &sh->state); 941 } 942 } 943 } 944 945 static struct dma_async_tx_descriptor * 946 async_copy_data(int frombio, struct bio *bio, struct page **page, 947 sector_t sector, struct dma_async_tx_descriptor *tx, 948 struct stripe_head *sh) 949 { 950 struct bio_vec bvl; 951 struct bvec_iter iter; 952 struct page *bio_page; 953 int page_offset; 954 struct async_submit_ctl submit; 955 enum async_tx_flags flags = 0; 956 957 if (bio->bi_iter.bi_sector >= sector) 958 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; 959 else 960 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; 961 962 if (frombio) 963 flags |= ASYNC_TX_FENCE; 964 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 965 966 bio_for_each_segment(bvl, bio, iter) { 967 int len = bvl.bv_len; 968 int clen; 969 int b_offset = 0; 970 971 if (page_offset < 0) { 972 b_offset = -page_offset; 973 page_offset += b_offset; 974 len -= b_offset; 975 } 976 977 if (len > 0 && page_offset + len > STRIPE_SIZE) 978 clen = STRIPE_SIZE - page_offset; 979 else 980 clen = len; 981 982 if (clen > 0) { 983 b_offset += bvl.bv_offset; 984 bio_page = bvl.bv_page; 985 if (frombio) { 986 if (sh->raid_conf->skip_copy && 987 b_offset == 0 && page_offset == 0 && 988 clen == STRIPE_SIZE) 989 *page = bio_page; 990 else 991 tx = async_memcpy(*page, bio_page, page_offset, 992 b_offset, clen, &submit); 993 } else 994 tx = async_memcpy(bio_page, *page, b_offset, 995 page_offset, clen, &submit); 996 } 997 /* chain the operations */ 998 submit.depend_tx = tx; 999 1000 if (clen < len) /* hit end of page */ 1001 break; 1002 page_offset += len; 1003 } 1004 1005 return tx; 1006 } 1007 1008 static void ops_complete_biofill(void *stripe_head_ref) 1009 { 1010 struct stripe_head *sh = stripe_head_ref; 1011 struct bio *return_bi = NULL; 1012 int i; 1013 1014 pr_debug("%s: stripe %llu\n", __func__, 1015 (unsigned long long)sh->sector); 1016 1017 /* clear completed biofills */ 1018 for (i = sh->disks; i--; ) { 1019 struct r5dev *dev = &sh->dev[i]; 1020 1021 /* acknowledge completion of a biofill operation */ 1022 /* and check if we need to reply to a read request, 1023 * new R5_Wantfill requests are held off until 1024 * !STRIPE_BIOFILL_RUN 1025 */ 1026 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 1027 struct bio *rbi, *rbi2; 1028 1029 BUG_ON(!dev->read); 1030 rbi = dev->read; 1031 dev->read = NULL; 1032 while (rbi && rbi->bi_iter.bi_sector < 1033 dev->sector + STRIPE_SECTORS) { 1034 rbi2 = r5_next_bio(rbi, dev->sector); 1035 if (!raid5_dec_bi_active_stripes(rbi)) { 1036 rbi->bi_next = return_bi; 1037 return_bi = rbi; 1038 } 1039 rbi = rbi2; 1040 } 1041 } 1042 } 1043 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 1044 1045 return_io(return_bi); 1046 1047 set_bit(STRIPE_HANDLE, &sh->state); 1048 release_stripe(sh); 1049 } 1050 1051 static void ops_run_biofill(struct stripe_head *sh) 1052 { 1053 struct dma_async_tx_descriptor *tx = NULL; 1054 struct async_submit_ctl submit; 1055 int i; 1056 1057 pr_debug("%s: stripe %llu\n", __func__, 1058 (unsigned long long)sh->sector); 1059 1060 for (i = sh->disks; i--; ) { 1061 struct r5dev *dev = &sh->dev[i]; 1062 if (test_bit(R5_Wantfill, &dev->flags)) { 1063 struct bio *rbi; 1064 spin_lock_irq(&sh->stripe_lock); 1065 dev->read = rbi = dev->toread; 1066 dev->toread = NULL; 1067 spin_unlock_irq(&sh->stripe_lock); 1068 while (rbi && rbi->bi_iter.bi_sector < 1069 dev->sector + STRIPE_SECTORS) { 1070 tx = async_copy_data(0, rbi, &dev->page, 1071 dev->sector, tx, sh); 1072 rbi = r5_next_bio(rbi, dev->sector); 1073 } 1074 } 1075 } 1076 1077 atomic_inc(&sh->count); 1078 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 1079 async_trigger_callback(&submit); 1080 } 1081 1082 static void mark_target_uptodate(struct stripe_head *sh, int target) 1083 { 1084 struct r5dev *tgt; 1085 1086 if (target < 0) 1087 return; 1088 1089 tgt = &sh->dev[target]; 1090 set_bit(R5_UPTODATE, &tgt->flags); 1091 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1092 clear_bit(R5_Wantcompute, &tgt->flags); 1093 } 1094 1095 static void ops_complete_compute(void *stripe_head_ref) 1096 { 1097 struct stripe_head *sh = stripe_head_ref; 1098 1099 pr_debug("%s: stripe %llu\n", __func__, 1100 (unsigned long long)sh->sector); 1101 1102 /* mark the computed target(s) as uptodate */ 1103 mark_target_uptodate(sh, sh->ops.target); 1104 mark_target_uptodate(sh, sh->ops.target2); 1105 1106 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 1107 if (sh->check_state == check_state_compute_run) 1108 sh->check_state = check_state_compute_result; 1109 set_bit(STRIPE_HANDLE, &sh->state); 1110 release_stripe(sh); 1111 } 1112 1113 /* return a pointer to the address conversion region of the scribble buffer */ 1114 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 1115 struct raid5_percpu *percpu) 1116 { 1117 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); 1118 } 1119 1120 static struct dma_async_tx_descriptor * 1121 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 1122 { 1123 int disks = sh->disks; 1124 struct page **xor_srcs = percpu->scribble; 1125 int target = sh->ops.target; 1126 struct r5dev *tgt = &sh->dev[target]; 1127 struct page *xor_dest = tgt->page; 1128 int count = 0; 1129 struct dma_async_tx_descriptor *tx; 1130 struct async_submit_ctl submit; 1131 int i; 1132 1133 pr_debug("%s: stripe %llu block: %d\n", 1134 __func__, (unsigned long long)sh->sector, target); 1135 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1136 1137 for (i = disks; i--; ) 1138 if (i != target) 1139 xor_srcs[count++] = sh->dev[i].page; 1140 1141 atomic_inc(&sh->count); 1142 1143 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 1144 ops_complete_compute, sh, to_addr_conv(sh, percpu)); 1145 if (unlikely(count == 1)) 1146 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1147 else 1148 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1149 1150 return tx; 1151 } 1152 1153 /* set_syndrome_sources - populate source buffers for gen_syndrome 1154 * @srcs - (struct page *) array of size sh->disks 1155 * @sh - stripe_head to parse 1156 * 1157 * Populates srcs in proper layout order for the stripe and returns the 1158 * 'count' of sources to be used in a call to async_gen_syndrome. The P 1159 * destination buffer is recorded in srcs[count] and the Q destination 1160 * is recorded in srcs[count+1]]. 1161 */ 1162 static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) 1163 { 1164 int disks = sh->disks; 1165 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 1166 int d0_idx = raid6_d0(sh); 1167 int count; 1168 int i; 1169 1170 for (i = 0; i < disks; i++) 1171 srcs[i] = NULL; 1172 1173 count = 0; 1174 i = d0_idx; 1175 do { 1176 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1177 1178 srcs[slot] = sh->dev[i].page; 1179 i = raid6_next_disk(i, disks); 1180 } while (i != d0_idx); 1181 1182 return syndrome_disks; 1183 } 1184 1185 static struct dma_async_tx_descriptor * 1186 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 1187 { 1188 int disks = sh->disks; 1189 struct page **blocks = percpu->scribble; 1190 int target; 1191 int qd_idx = sh->qd_idx; 1192 struct dma_async_tx_descriptor *tx; 1193 struct async_submit_ctl submit; 1194 struct r5dev *tgt; 1195 struct page *dest; 1196 int i; 1197 int count; 1198 1199 if (sh->ops.target < 0) 1200 target = sh->ops.target2; 1201 else if (sh->ops.target2 < 0) 1202 target = sh->ops.target; 1203 else 1204 /* we should only have one valid target */ 1205 BUG(); 1206 BUG_ON(target < 0); 1207 pr_debug("%s: stripe %llu block: %d\n", 1208 __func__, (unsigned long long)sh->sector, target); 1209 1210 tgt = &sh->dev[target]; 1211 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1212 dest = tgt->page; 1213 1214 atomic_inc(&sh->count); 1215 1216 if (target == qd_idx) { 1217 count = set_syndrome_sources(blocks, sh); 1218 blocks[count] = NULL; /* regenerating p is not necessary */ 1219 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 1220 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1221 ops_complete_compute, sh, 1222 to_addr_conv(sh, percpu)); 1223 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1224 } else { 1225 /* Compute any data- or p-drive using XOR */ 1226 count = 0; 1227 for (i = disks; i-- ; ) { 1228 if (i == target || i == qd_idx) 1229 continue; 1230 blocks[count++] = sh->dev[i].page; 1231 } 1232 1233 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1234 NULL, ops_complete_compute, sh, 1235 to_addr_conv(sh, percpu)); 1236 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 1237 } 1238 1239 return tx; 1240 } 1241 1242 static struct dma_async_tx_descriptor * 1243 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 1244 { 1245 int i, count, disks = sh->disks; 1246 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 1247 int d0_idx = raid6_d0(sh); 1248 int faila = -1, failb = -1; 1249 int target = sh->ops.target; 1250 int target2 = sh->ops.target2; 1251 struct r5dev *tgt = &sh->dev[target]; 1252 struct r5dev *tgt2 = &sh->dev[target2]; 1253 struct dma_async_tx_descriptor *tx; 1254 struct page **blocks = percpu->scribble; 1255 struct async_submit_ctl submit; 1256 1257 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 1258 __func__, (unsigned long long)sh->sector, target, target2); 1259 BUG_ON(target < 0 || target2 < 0); 1260 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1261 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 1262 1263 /* we need to open-code set_syndrome_sources to handle the 1264 * slot number conversion for 'faila' and 'failb' 1265 */ 1266 for (i = 0; i < disks ; i++) 1267 blocks[i] = NULL; 1268 count = 0; 1269 i = d0_idx; 1270 do { 1271 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1272 1273 blocks[slot] = sh->dev[i].page; 1274 1275 if (i == target) 1276 faila = slot; 1277 if (i == target2) 1278 failb = slot; 1279 i = raid6_next_disk(i, disks); 1280 } while (i != d0_idx); 1281 1282 BUG_ON(faila == failb); 1283 if (failb < faila) 1284 swap(faila, failb); 1285 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1286 __func__, (unsigned long long)sh->sector, faila, failb); 1287 1288 atomic_inc(&sh->count); 1289 1290 if (failb == syndrome_disks+1) { 1291 /* Q disk is one of the missing disks */ 1292 if (faila == syndrome_disks) { 1293 /* Missing P+Q, just recompute */ 1294 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1295 ops_complete_compute, sh, 1296 to_addr_conv(sh, percpu)); 1297 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1298 STRIPE_SIZE, &submit); 1299 } else { 1300 struct page *dest; 1301 int data_target; 1302 int qd_idx = sh->qd_idx; 1303 1304 /* Missing D+Q: recompute D from P, then recompute Q */ 1305 if (target == qd_idx) 1306 data_target = target2; 1307 else 1308 data_target = target; 1309 1310 count = 0; 1311 for (i = disks; i-- ; ) { 1312 if (i == data_target || i == qd_idx) 1313 continue; 1314 blocks[count++] = sh->dev[i].page; 1315 } 1316 dest = sh->dev[data_target].page; 1317 init_async_submit(&submit, 1318 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1319 NULL, NULL, NULL, 1320 to_addr_conv(sh, percpu)); 1321 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1322 &submit); 1323 1324 count = set_syndrome_sources(blocks, sh); 1325 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1326 ops_complete_compute, sh, 1327 to_addr_conv(sh, percpu)); 1328 return async_gen_syndrome(blocks, 0, count+2, 1329 STRIPE_SIZE, &submit); 1330 } 1331 } else { 1332 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1333 ops_complete_compute, sh, 1334 to_addr_conv(sh, percpu)); 1335 if (failb == syndrome_disks) { 1336 /* We're missing D+P. */ 1337 return async_raid6_datap_recov(syndrome_disks+2, 1338 STRIPE_SIZE, faila, 1339 blocks, &submit); 1340 } else { 1341 /* We're missing D+D. */ 1342 return async_raid6_2data_recov(syndrome_disks+2, 1343 STRIPE_SIZE, faila, failb, 1344 blocks, &submit); 1345 } 1346 } 1347 } 1348 1349 static void ops_complete_prexor(void *stripe_head_ref) 1350 { 1351 struct stripe_head *sh = stripe_head_ref; 1352 1353 pr_debug("%s: stripe %llu\n", __func__, 1354 (unsigned long long)sh->sector); 1355 } 1356 1357 static struct dma_async_tx_descriptor * 1358 ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, 1359 struct dma_async_tx_descriptor *tx) 1360 { 1361 int disks = sh->disks; 1362 struct page **xor_srcs = percpu->scribble; 1363 int count = 0, pd_idx = sh->pd_idx, i; 1364 struct async_submit_ctl submit; 1365 1366 /* existing parity data subtracted */ 1367 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1368 1369 pr_debug("%s: stripe %llu\n", __func__, 1370 (unsigned long long)sh->sector); 1371 1372 for (i = disks; i--; ) { 1373 struct r5dev *dev = &sh->dev[i]; 1374 /* Only process blocks that are known to be uptodate */ 1375 if (test_bit(R5_Wantdrain, &dev->flags)) 1376 xor_srcs[count++] = dev->page; 1377 } 1378 1379 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1380 ops_complete_prexor, sh, to_addr_conv(sh, percpu)); 1381 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1382 1383 return tx; 1384 } 1385 1386 static struct dma_async_tx_descriptor * 1387 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1388 { 1389 int disks = sh->disks; 1390 int i; 1391 1392 pr_debug("%s: stripe %llu\n", __func__, 1393 (unsigned long long)sh->sector); 1394 1395 for (i = disks; i--; ) { 1396 struct r5dev *dev = &sh->dev[i]; 1397 struct bio *chosen; 1398 1399 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { 1400 struct bio *wbi; 1401 1402 spin_lock_irq(&sh->stripe_lock); 1403 chosen = dev->towrite; 1404 dev->towrite = NULL; 1405 BUG_ON(dev->written); 1406 wbi = dev->written = chosen; 1407 spin_unlock_irq(&sh->stripe_lock); 1408 WARN_ON(dev->page != dev->orig_page); 1409 1410 while (wbi && wbi->bi_iter.bi_sector < 1411 dev->sector + STRIPE_SECTORS) { 1412 if (wbi->bi_rw & REQ_FUA) 1413 set_bit(R5_WantFUA, &dev->flags); 1414 if (wbi->bi_rw & REQ_SYNC) 1415 set_bit(R5_SyncIO, &dev->flags); 1416 if (wbi->bi_rw & REQ_DISCARD) 1417 set_bit(R5_Discard, &dev->flags); 1418 else { 1419 tx = async_copy_data(1, wbi, &dev->page, 1420 dev->sector, tx, sh); 1421 if (dev->page != dev->orig_page) { 1422 set_bit(R5_SkipCopy, &dev->flags); 1423 clear_bit(R5_UPTODATE, &dev->flags); 1424 clear_bit(R5_OVERWRITE, &dev->flags); 1425 } 1426 } 1427 wbi = r5_next_bio(wbi, dev->sector); 1428 } 1429 } 1430 } 1431 1432 return tx; 1433 } 1434 1435 static void ops_complete_reconstruct(void *stripe_head_ref) 1436 { 1437 struct stripe_head *sh = stripe_head_ref; 1438 int disks = sh->disks; 1439 int pd_idx = sh->pd_idx; 1440 int qd_idx = sh->qd_idx; 1441 int i; 1442 bool fua = false, sync = false, discard = false; 1443 1444 pr_debug("%s: stripe %llu\n", __func__, 1445 (unsigned long long)sh->sector); 1446 1447 for (i = disks; i--; ) { 1448 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1449 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); 1450 discard |= test_bit(R5_Discard, &sh->dev[i].flags); 1451 } 1452 1453 for (i = disks; i--; ) { 1454 struct r5dev *dev = &sh->dev[i]; 1455 1456 if (dev->written || i == pd_idx || i == qd_idx) { 1457 if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) 1458 set_bit(R5_UPTODATE, &dev->flags); 1459 if (fua) 1460 set_bit(R5_WantFUA, &dev->flags); 1461 if (sync) 1462 set_bit(R5_SyncIO, &dev->flags); 1463 } 1464 } 1465 1466 if (sh->reconstruct_state == reconstruct_state_drain_run) 1467 sh->reconstruct_state = reconstruct_state_drain_result; 1468 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1469 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1470 else { 1471 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1472 sh->reconstruct_state = reconstruct_state_result; 1473 } 1474 1475 set_bit(STRIPE_HANDLE, &sh->state); 1476 release_stripe(sh); 1477 } 1478 1479 static void 1480 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1481 struct dma_async_tx_descriptor *tx) 1482 { 1483 int disks = sh->disks; 1484 struct page **xor_srcs = percpu->scribble; 1485 struct async_submit_ctl submit; 1486 int count = 0, pd_idx = sh->pd_idx, i; 1487 struct page *xor_dest; 1488 int prexor = 0; 1489 unsigned long flags; 1490 1491 pr_debug("%s: stripe %llu\n", __func__, 1492 (unsigned long long)sh->sector); 1493 1494 for (i = 0; i < sh->disks; i++) { 1495 if (pd_idx == i) 1496 continue; 1497 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1498 break; 1499 } 1500 if (i >= sh->disks) { 1501 atomic_inc(&sh->count); 1502 set_bit(R5_Discard, &sh->dev[pd_idx].flags); 1503 ops_complete_reconstruct(sh); 1504 return; 1505 } 1506 /* check if prexor is active which means only process blocks 1507 * that are part of a read-modify-write (written) 1508 */ 1509 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1510 prexor = 1; 1511 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1512 for (i = disks; i--; ) { 1513 struct r5dev *dev = &sh->dev[i]; 1514 if (dev->written) 1515 xor_srcs[count++] = dev->page; 1516 } 1517 } else { 1518 xor_dest = sh->dev[pd_idx].page; 1519 for (i = disks; i--; ) { 1520 struct r5dev *dev = &sh->dev[i]; 1521 if (i != pd_idx) 1522 xor_srcs[count++] = dev->page; 1523 } 1524 } 1525 1526 /* 1/ if we prexor'd then the dest is reused as a source 1527 * 2/ if we did not prexor then we are redoing the parity 1528 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1529 * for the synchronous xor case 1530 */ 1531 flags = ASYNC_TX_ACK | 1532 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1533 1534 atomic_inc(&sh->count); 1535 1536 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, 1537 to_addr_conv(sh, percpu)); 1538 if (unlikely(count == 1)) 1539 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1540 else 1541 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1542 } 1543 1544 static void 1545 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1546 struct dma_async_tx_descriptor *tx) 1547 { 1548 struct async_submit_ctl submit; 1549 struct page **blocks = percpu->scribble; 1550 int count, i; 1551 1552 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1553 1554 for (i = 0; i < sh->disks; i++) { 1555 if (sh->pd_idx == i || sh->qd_idx == i) 1556 continue; 1557 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1558 break; 1559 } 1560 if (i >= sh->disks) { 1561 atomic_inc(&sh->count); 1562 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 1563 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 1564 ops_complete_reconstruct(sh); 1565 return; 1566 } 1567 1568 count = set_syndrome_sources(blocks, sh); 1569 1570 atomic_inc(&sh->count); 1571 1572 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, 1573 sh, to_addr_conv(sh, percpu)); 1574 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1575 } 1576 1577 static void ops_complete_check(void *stripe_head_ref) 1578 { 1579 struct stripe_head *sh = stripe_head_ref; 1580 1581 pr_debug("%s: stripe %llu\n", __func__, 1582 (unsigned long long)sh->sector); 1583 1584 sh->check_state = check_state_check_result; 1585 set_bit(STRIPE_HANDLE, &sh->state); 1586 release_stripe(sh); 1587 } 1588 1589 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1590 { 1591 int disks = sh->disks; 1592 int pd_idx = sh->pd_idx; 1593 int qd_idx = sh->qd_idx; 1594 struct page *xor_dest; 1595 struct page **xor_srcs = percpu->scribble; 1596 struct dma_async_tx_descriptor *tx; 1597 struct async_submit_ctl submit; 1598 int count; 1599 int i; 1600 1601 pr_debug("%s: stripe %llu\n", __func__, 1602 (unsigned long long)sh->sector); 1603 1604 count = 0; 1605 xor_dest = sh->dev[pd_idx].page; 1606 xor_srcs[count++] = xor_dest; 1607 for (i = disks; i--; ) { 1608 if (i == pd_idx || i == qd_idx) 1609 continue; 1610 xor_srcs[count++] = sh->dev[i].page; 1611 } 1612 1613 init_async_submit(&submit, 0, NULL, NULL, NULL, 1614 to_addr_conv(sh, percpu)); 1615 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1616 &sh->ops.zero_sum_result, &submit); 1617 1618 atomic_inc(&sh->count); 1619 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1620 tx = async_trigger_callback(&submit); 1621 } 1622 1623 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1624 { 1625 struct page **srcs = percpu->scribble; 1626 struct async_submit_ctl submit; 1627 int count; 1628 1629 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1630 (unsigned long long)sh->sector, checkp); 1631 1632 count = set_syndrome_sources(srcs, sh); 1633 if (!checkp) 1634 srcs[count] = NULL; 1635 1636 atomic_inc(&sh->count); 1637 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1638 sh, to_addr_conv(sh, percpu)); 1639 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1640 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1641 } 1642 1643 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1644 { 1645 int overlap_clear = 0, i, disks = sh->disks; 1646 struct dma_async_tx_descriptor *tx = NULL; 1647 struct r5conf *conf = sh->raid_conf; 1648 int level = conf->level; 1649 struct raid5_percpu *percpu; 1650 unsigned long cpu; 1651 1652 cpu = get_cpu(); 1653 percpu = per_cpu_ptr(conf->percpu, cpu); 1654 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1655 ops_run_biofill(sh); 1656 overlap_clear++; 1657 } 1658 1659 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1660 if (level < 6) 1661 tx = ops_run_compute5(sh, percpu); 1662 else { 1663 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1664 tx = ops_run_compute6_1(sh, percpu); 1665 else 1666 tx = ops_run_compute6_2(sh, percpu); 1667 } 1668 /* terminate the chain if reconstruct is not set to be run */ 1669 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1670 async_tx_ack(tx); 1671 } 1672 1673 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) 1674 tx = ops_run_prexor(sh, percpu, tx); 1675 1676 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1677 tx = ops_run_biodrain(sh, tx); 1678 overlap_clear++; 1679 } 1680 1681 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1682 if (level < 6) 1683 ops_run_reconstruct5(sh, percpu, tx); 1684 else 1685 ops_run_reconstruct6(sh, percpu, tx); 1686 } 1687 1688 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1689 if (sh->check_state == check_state_run) 1690 ops_run_check_p(sh, percpu); 1691 else if (sh->check_state == check_state_run_q) 1692 ops_run_check_pq(sh, percpu, 0); 1693 else if (sh->check_state == check_state_run_pq) 1694 ops_run_check_pq(sh, percpu, 1); 1695 else 1696 BUG(); 1697 } 1698 1699 if (overlap_clear) 1700 for (i = disks; i--; ) { 1701 struct r5dev *dev = &sh->dev[i]; 1702 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1703 wake_up(&sh->raid_conf->wait_for_overlap); 1704 } 1705 put_cpu(); 1706 } 1707 1708 static int grow_one_stripe(struct r5conf *conf, int hash) 1709 { 1710 struct stripe_head *sh; 1711 sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL); 1712 if (!sh) 1713 return 0; 1714 1715 sh->raid_conf = conf; 1716 1717 spin_lock_init(&sh->stripe_lock); 1718 1719 if (grow_buffers(sh)) { 1720 shrink_buffers(sh); 1721 kmem_cache_free(conf->slab_cache, sh); 1722 return 0; 1723 } 1724 sh->hash_lock_index = hash; 1725 /* we just created an active stripe so... */ 1726 atomic_set(&sh->count, 1); 1727 atomic_inc(&conf->active_stripes); 1728 INIT_LIST_HEAD(&sh->lru); 1729 release_stripe(sh); 1730 return 1; 1731 } 1732 1733 static int grow_stripes(struct r5conf *conf, int num) 1734 { 1735 struct kmem_cache *sc; 1736 int devs = max(conf->raid_disks, conf->previous_raid_disks); 1737 int hash; 1738 1739 if (conf->mddev->gendisk) 1740 sprintf(conf->cache_name[0], 1741 "raid%d-%s", conf->level, mdname(conf->mddev)); 1742 else 1743 sprintf(conf->cache_name[0], 1744 "raid%d-%p", conf->level, conf->mddev); 1745 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 1746 1747 conf->active_name = 0; 1748 sc = kmem_cache_create(conf->cache_name[conf->active_name], 1749 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 1750 0, 0, NULL); 1751 if (!sc) 1752 return 1; 1753 conf->slab_cache = sc; 1754 conf->pool_size = devs; 1755 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 1756 while (num--) { 1757 if (!grow_one_stripe(conf, hash)) 1758 return 1; 1759 conf->max_nr_stripes++; 1760 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS; 1761 } 1762 return 0; 1763 } 1764 1765 /** 1766 * scribble_len - return the required size of the scribble region 1767 * @num - total number of disks in the array 1768 * 1769 * The size must be enough to contain: 1770 * 1/ a struct page pointer for each device in the array +2 1771 * 2/ room to convert each entry in (1) to its corresponding dma 1772 * (dma_map_page()) or page (page_address()) address. 1773 * 1774 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 1775 * calculate over all devices (not just the data blocks), using zeros in place 1776 * of the P and Q blocks. 1777 */ 1778 static size_t scribble_len(int num) 1779 { 1780 size_t len; 1781 1782 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 1783 1784 return len; 1785 } 1786 1787 static int resize_stripes(struct r5conf *conf, int newsize) 1788 { 1789 /* Make all the stripes able to hold 'newsize' devices. 1790 * New slots in each stripe get 'page' set to a new page. 1791 * 1792 * This happens in stages: 1793 * 1/ create a new kmem_cache and allocate the required number of 1794 * stripe_heads. 1795 * 2/ gather all the old stripe_heads and transfer the pages across 1796 * to the new stripe_heads. This will have the side effect of 1797 * freezing the array as once all stripe_heads have been collected, 1798 * no IO will be possible. Old stripe heads are freed once their 1799 * pages have been transferred over, and the old kmem_cache is 1800 * freed when all stripes are done. 1801 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 1802 * we simple return a failre status - no need to clean anything up. 1803 * 4/ allocate new pages for the new slots in the new stripe_heads. 1804 * If this fails, we don't bother trying the shrink the 1805 * stripe_heads down again, we just leave them as they are. 1806 * As each stripe_head is processed the new one is released into 1807 * active service. 1808 * 1809 * Once step2 is started, we cannot afford to wait for a write, 1810 * so we use GFP_NOIO allocations. 1811 */ 1812 struct stripe_head *osh, *nsh; 1813 LIST_HEAD(newstripes); 1814 struct disk_info *ndisks; 1815 unsigned long cpu; 1816 int err; 1817 struct kmem_cache *sc; 1818 int i; 1819 int hash, cnt; 1820 1821 if (newsize <= conf->pool_size) 1822 return 0; /* never bother to shrink */ 1823 1824 err = md_allow_write(conf->mddev); 1825 if (err) 1826 return err; 1827 1828 /* Step 1 */ 1829 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 1830 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 1831 0, 0, NULL); 1832 if (!sc) 1833 return -ENOMEM; 1834 1835 for (i = conf->max_nr_stripes; i; i--) { 1836 nsh = kmem_cache_zalloc(sc, GFP_KERNEL); 1837 if (!nsh) 1838 break; 1839 1840 nsh->raid_conf = conf; 1841 spin_lock_init(&nsh->stripe_lock); 1842 1843 list_add(&nsh->lru, &newstripes); 1844 } 1845 if (i) { 1846 /* didn't get enough, give up */ 1847 while (!list_empty(&newstripes)) { 1848 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1849 list_del(&nsh->lru); 1850 kmem_cache_free(sc, nsh); 1851 } 1852 kmem_cache_destroy(sc); 1853 return -ENOMEM; 1854 } 1855 /* Step 2 - Must use GFP_NOIO now. 1856 * OK, we have enough stripes, start collecting inactive 1857 * stripes and copying them over 1858 */ 1859 hash = 0; 1860 cnt = 0; 1861 list_for_each_entry(nsh, &newstripes, lru) { 1862 lock_device_hash_lock(conf, hash); 1863 wait_event_cmd(conf->wait_for_stripe, 1864 !list_empty(conf->inactive_list + hash), 1865 unlock_device_hash_lock(conf, hash), 1866 lock_device_hash_lock(conf, hash)); 1867 osh = get_free_stripe(conf, hash); 1868 unlock_device_hash_lock(conf, hash); 1869 atomic_set(&nsh->count, 1); 1870 for(i=0; i<conf->pool_size; i++) { 1871 nsh->dev[i].page = osh->dev[i].page; 1872 nsh->dev[i].orig_page = osh->dev[i].page; 1873 } 1874 for( ; i<newsize; i++) 1875 nsh->dev[i].page = NULL; 1876 nsh->hash_lock_index = hash; 1877 kmem_cache_free(conf->slab_cache, osh); 1878 cnt++; 1879 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + 1880 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { 1881 hash++; 1882 cnt = 0; 1883 } 1884 } 1885 kmem_cache_destroy(conf->slab_cache); 1886 1887 /* Step 3. 1888 * At this point, we are holding all the stripes so the array 1889 * is completely stalled, so now is a good time to resize 1890 * conf->disks and the scribble region 1891 */ 1892 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 1893 if (ndisks) { 1894 for (i=0; i<conf->raid_disks; i++) 1895 ndisks[i] = conf->disks[i]; 1896 kfree(conf->disks); 1897 conf->disks = ndisks; 1898 } else 1899 err = -ENOMEM; 1900 1901 get_online_cpus(); 1902 conf->scribble_len = scribble_len(newsize); 1903 for_each_present_cpu(cpu) { 1904 struct raid5_percpu *percpu; 1905 void *scribble; 1906 1907 percpu = per_cpu_ptr(conf->percpu, cpu); 1908 scribble = kmalloc(conf->scribble_len, GFP_NOIO); 1909 1910 if (scribble) { 1911 kfree(percpu->scribble); 1912 percpu->scribble = scribble; 1913 } else { 1914 err = -ENOMEM; 1915 break; 1916 } 1917 } 1918 put_online_cpus(); 1919 1920 /* Step 4, return new stripes to service */ 1921 while(!list_empty(&newstripes)) { 1922 nsh = list_entry(newstripes.next, struct stripe_head, lru); 1923 list_del_init(&nsh->lru); 1924 1925 for (i=conf->raid_disks; i < newsize; i++) 1926 if (nsh->dev[i].page == NULL) { 1927 struct page *p = alloc_page(GFP_NOIO); 1928 nsh->dev[i].page = p; 1929 nsh->dev[i].orig_page = p; 1930 if (!p) 1931 err = -ENOMEM; 1932 } 1933 release_stripe(nsh); 1934 } 1935 /* critical section pass, GFP_NOIO no longer needed */ 1936 1937 conf->slab_cache = sc; 1938 conf->active_name = 1-conf->active_name; 1939 conf->pool_size = newsize; 1940 return err; 1941 } 1942 1943 static int drop_one_stripe(struct r5conf *conf, int hash) 1944 { 1945 struct stripe_head *sh; 1946 1947 spin_lock_irq(conf->hash_locks + hash); 1948 sh = get_free_stripe(conf, hash); 1949 spin_unlock_irq(conf->hash_locks + hash); 1950 if (!sh) 1951 return 0; 1952 BUG_ON(atomic_read(&sh->count)); 1953 shrink_buffers(sh); 1954 kmem_cache_free(conf->slab_cache, sh); 1955 atomic_dec(&conf->active_stripes); 1956 return 1; 1957 } 1958 1959 static void shrink_stripes(struct r5conf *conf) 1960 { 1961 int hash; 1962 for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++) 1963 while (drop_one_stripe(conf, hash)) 1964 ; 1965 1966 if (conf->slab_cache) 1967 kmem_cache_destroy(conf->slab_cache); 1968 conf->slab_cache = NULL; 1969 } 1970 1971 static void raid5_end_read_request(struct bio * bi, int error) 1972 { 1973 struct stripe_head *sh = bi->bi_private; 1974 struct r5conf *conf = sh->raid_conf; 1975 int disks = sh->disks, i; 1976 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 1977 char b[BDEVNAME_SIZE]; 1978 struct md_rdev *rdev = NULL; 1979 sector_t s; 1980 1981 for (i=0 ; i<disks; i++) 1982 if (bi == &sh->dev[i].req) 1983 break; 1984 1985 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", 1986 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 1987 uptodate); 1988 if (i == disks) { 1989 BUG(); 1990 return; 1991 } 1992 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 1993 /* If replacement finished while this request was outstanding, 1994 * 'replacement' might be NULL already. 1995 * In that case it moved down to 'rdev'. 1996 * rdev is not removed until all requests are finished. 1997 */ 1998 rdev = conf->disks[i].replacement; 1999 if (!rdev) 2000 rdev = conf->disks[i].rdev; 2001 2002 if (use_new_offset(conf, sh)) 2003 s = sh->sector + rdev->new_data_offset; 2004 else 2005 s = sh->sector + rdev->data_offset; 2006 if (uptodate) { 2007 set_bit(R5_UPTODATE, &sh->dev[i].flags); 2008 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2009 /* Note that this cannot happen on a 2010 * replacement device. We just fail those on 2011 * any error 2012 */ 2013 printk_ratelimited( 2014 KERN_INFO 2015 "md/raid:%s: read error corrected" 2016 " (%lu sectors at %llu on %s)\n", 2017 mdname(conf->mddev), STRIPE_SECTORS, 2018 (unsigned long long)s, 2019 bdevname(rdev->bdev, b)); 2020 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 2021 clear_bit(R5_ReadError, &sh->dev[i].flags); 2022 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2023 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2024 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2025 2026 if (atomic_read(&rdev->read_errors)) 2027 atomic_set(&rdev->read_errors, 0); 2028 } else { 2029 const char *bdn = bdevname(rdev->bdev, b); 2030 int retry = 0; 2031 int set_bad = 0; 2032 2033 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 2034 atomic_inc(&rdev->read_errors); 2035 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2036 printk_ratelimited( 2037 KERN_WARNING 2038 "md/raid:%s: read error on replacement device " 2039 "(sector %llu on %s).\n", 2040 mdname(conf->mddev), 2041 (unsigned long long)s, 2042 bdn); 2043 else if (conf->mddev->degraded >= conf->max_degraded) { 2044 set_bad = 1; 2045 printk_ratelimited( 2046 KERN_WARNING 2047 "md/raid:%s: read error not correctable " 2048 "(sector %llu on %s).\n", 2049 mdname(conf->mddev), 2050 (unsigned long long)s, 2051 bdn); 2052 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { 2053 /* Oh, no!!! */ 2054 set_bad = 1; 2055 printk_ratelimited( 2056 KERN_WARNING 2057 "md/raid:%s: read error NOT corrected!! " 2058 "(sector %llu on %s).\n", 2059 mdname(conf->mddev), 2060 (unsigned long long)s, 2061 bdn); 2062 } else if (atomic_read(&rdev->read_errors) 2063 > conf->max_nr_stripes) 2064 printk(KERN_WARNING 2065 "md/raid:%s: Too many read errors, failing device %s.\n", 2066 mdname(conf->mddev), bdn); 2067 else 2068 retry = 1; 2069 if (set_bad && test_bit(In_sync, &rdev->flags) 2070 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2071 retry = 1; 2072 if (retry) 2073 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { 2074 set_bit(R5_ReadError, &sh->dev[i].flags); 2075 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2076 } else 2077 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2078 else { 2079 clear_bit(R5_ReadError, &sh->dev[i].flags); 2080 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2081 if (!(set_bad 2082 && test_bit(In_sync, &rdev->flags) 2083 && rdev_set_badblocks( 2084 rdev, sh->sector, STRIPE_SECTORS, 0))) 2085 md_error(conf->mddev, rdev); 2086 } 2087 } 2088 rdev_dec_pending(rdev, conf->mddev); 2089 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2090 set_bit(STRIPE_HANDLE, &sh->state); 2091 release_stripe(sh); 2092 } 2093 2094 static void raid5_end_write_request(struct bio *bi, int error) 2095 { 2096 struct stripe_head *sh = bi->bi_private; 2097 struct r5conf *conf = sh->raid_conf; 2098 int disks = sh->disks, i; 2099 struct md_rdev *uninitialized_var(rdev); 2100 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 2101 sector_t first_bad; 2102 int bad_sectors; 2103 int replacement = 0; 2104 2105 for (i = 0 ; i < disks; i++) { 2106 if (bi == &sh->dev[i].req) { 2107 rdev = conf->disks[i].rdev; 2108 break; 2109 } 2110 if (bi == &sh->dev[i].rreq) { 2111 rdev = conf->disks[i].replacement; 2112 if (rdev) 2113 replacement = 1; 2114 else 2115 /* rdev was removed and 'replacement' 2116 * replaced it. rdev is not removed 2117 * until all requests are finished. 2118 */ 2119 rdev = conf->disks[i].rdev; 2120 break; 2121 } 2122 } 2123 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", 2124 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2125 uptodate); 2126 if (i == disks) { 2127 BUG(); 2128 return; 2129 } 2130 2131 if (replacement) { 2132 if (!uptodate) 2133 md_error(conf->mddev, rdev); 2134 else if (is_badblock(rdev, sh->sector, 2135 STRIPE_SECTORS, 2136 &first_bad, &bad_sectors)) 2137 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2138 } else { 2139 if (!uptodate) { 2140 set_bit(STRIPE_DEGRADED, &sh->state); 2141 set_bit(WriteErrorSeen, &rdev->flags); 2142 set_bit(R5_WriteError, &sh->dev[i].flags); 2143 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2144 set_bit(MD_RECOVERY_NEEDED, 2145 &rdev->mddev->recovery); 2146 } else if (is_badblock(rdev, sh->sector, 2147 STRIPE_SECTORS, 2148 &first_bad, &bad_sectors)) { 2149 set_bit(R5_MadeGood, &sh->dev[i].flags); 2150 if (test_bit(R5_ReadError, &sh->dev[i].flags)) 2151 /* That was a successful write so make 2152 * sure it looks like we already did 2153 * a re-write. 2154 */ 2155 set_bit(R5_ReWrite, &sh->dev[i].flags); 2156 } 2157 } 2158 rdev_dec_pending(rdev, conf->mddev); 2159 2160 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2161 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2162 set_bit(STRIPE_HANDLE, &sh->state); 2163 release_stripe(sh); 2164 } 2165 2166 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); 2167 2168 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 2169 { 2170 struct r5dev *dev = &sh->dev[i]; 2171 2172 bio_init(&dev->req); 2173 dev->req.bi_io_vec = &dev->vec; 2174 dev->req.bi_max_vecs = 1; 2175 dev->req.bi_private = sh; 2176 2177 bio_init(&dev->rreq); 2178 dev->rreq.bi_io_vec = &dev->rvec; 2179 dev->rreq.bi_max_vecs = 1; 2180 dev->rreq.bi_private = sh; 2181 2182 dev->flags = 0; 2183 dev->sector = compute_blocknr(sh, i, previous); 2184 } 2185 2186 static void error(struct mddev *mddev, struct md_rdev *rdev) 2187 { 2188 char b[BDEVNAME_SIZE]; 2189 struct r5conf *conf = mddev->private; 2190 unsigned long flags; 2191 pr_debug("raid456: error called\n"); 2192 2193 spin_lock_irqsave(&conf->device_lock, flags); 2194 clear_bit(In_sync, &rdev->flags); 2195 mddev->degraded = calc_degraded(conf); 2196 spin_unlock_irqrestore(&conf->device_lock, flags); 2197 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2198 2199 set_bit(Blocked, &rdev->flags); 2200 set_bit(Faulty, &rdev->flags); 2201 set_bit(MD_CHANGE_DEVS, &mddev->flags); 2202 printk(KERN_ALERT 2203 "md/raid:%s: Disk failure on %s, disabling device.\n" 2204 "md/raid:%s: Operation continuing on %d devices.\n", 2205 mdname(mddev), 2206 bdevname(rdev->bdev, b), 2207 mdname(mddev), 2208 conf->raid_disks - mddev->degraded); 2209 } 2210 2211 /* 2212 * Input: a 'big' sector number, 2213 * Output: index of the data and parity disk, and the sector # in them. 2214 */ 2215 static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 2216 int previous, int *dd_idx, 2217 struct stripe_head *sh) 2218 { 2219 sector_t stripe, stripe2; 2220 sector_t chunk_number; 2221 unsigned int chunk_offset; 2222 int pd_idx, qd_idx; 2223 int ddf_layout = 0; 2224 sector_t new_sector; 2225 int algorithm = previous ? conf->prev_algo 2226 : conf->algorithm; 2227 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2228 : conf->chunk_sectors; 2229 int raid_disks = previous ? conf->previous_raid_disks 2230 : conf->raid_disks; 2231 int data_disks = raid_disks - conf->max_degraded; 2232 2233 /* First compute the information on this sector */ 2234 2235 /* 2236 * Compute the chunk number and the sector offset inside the chunk 2237 */ 2238 chunk_offset = sector_div(r_sector, sectors_per_chunk); 2239 chunk_number = r_sector; 2240 2241 /* 2242 * Compute the stripe number 2243 */ 2244 stripe = chunk_number; 2245 *dd_idx = sector_div(stripe, data_disks); 2246 stripe2 = stripe; 2247 /* 2248 * Select the parity disk based on the user selected algorithm. 2249 */ 2250 pd_idx = qd_idx = -1; 2251 switch(conf->level) { 2252 case 4: 2253 pd_idx = data_disks; 2254 break; 2255 case 5: 2256 switch (algorithm) { 2257 case ALGORITHM_LEFT_ASYMMETRIC: 2258 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2259 if (*dd_idx >= pd_idx) 2260 (*dd_idx)++; 2261 break; 2262 case ALGORITHM_RIGHT_ASYMMETRIC: 2263 pd_idx = sector_div(stripe2, raid_disks); 2264 if (*dd_idx >= pd_idx) 2265 (*dd_idx)++; 2266 break; 2267 case ALGORITHM_LEFT_SYMMETRIC: 2268 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2269 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2270 break; 2271 case ALGORITHM_RIGHT_SYMMETRIC: 2272 pd_idx = sector_div(stripe2, raid_disks); 2273 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2274 break; 2275 case ALGORITHM_PARITY_0: 2276 pd_idx = 0; 2277 (*dd_idx)++; 2278 break; 2279 case ALGORITHM_PARITY_N: 2280 pd_idx = data_disks; 2281 break; 2282 default: 2283 BUG(); 2284 } 2285 break; 2286 case 6: 2287 2288 switch (algorithm) { 2289 case ALGORITHM_LEFT_ASYMMETRIC: 2290 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2291 qd_idx = pd_idx + 1; 2292 if (pd_idx == raid_disks-1) { 2293 (*dd_idx)++; /* Q D D D P */ 2294 qd_idx = 0; 2295 } else if (*dd_idx >= pd_idx) 2296 (*dd_idx) += 2; /* D D P Q D */ 2297 break; 2298 case ALGORITHM_RIGHT_ASYMMETRIC: 2299 pd_idx = sector_div(stripe2, raid_disks); 2300 qd_idx = pd_idx + 1; 2301 if (pd_idx == raid_disks-1) { 2302 (*dd_idx)++; /* Q D D D P */ 2303 qd_idx = 0; 2304 } else if (*dd_idx >= pd_idx) 2305 (*dd_idx) += 2; /* D D P Q D */ 2306 break; 2307 case ALGORITHM_LEFT_SYMMETRIC: 2308 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2309 qd_idx = (pd_idx + 1) % raid_disks; 2310 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2311 break; 2312 case ALGORITHM_RIGHT_SYMMETRIC: 2313 pd_idx = sector_div(stripe2, raid_disks); 2314 qd_idx = (pd_idx + 1) % raid_disks; 2315 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2316 break; 2317 2318 case ALGORITHM_PARITY_0: 2319 pd_idx = 0; 2320 qd_idx = 1; 2321 (*dd_idx) += 2; 2322 break; 2323 case ALGORITHM_PARITY_N: 2324 pd_idx = data_disks; 2325 qd_idx = data_disks + 1; 2326 break; 2327 2328 case ALGORITHM_ROTATING_ZERO_RESTART: 2329 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2330 * of blocks for computing Q is different. 2331 */ 2332 pd_idx = sector_div(stripe2, raid_disks); 2333 qd_idx = pd_idx + 1; 2334 if (pd_idx == raid_disks-1) { 2335 (*dd_idx)++; /* Q D D D P */ 2336 qd_idx = 0; 2337 } else if (*dd_idx >= pd_idx) 2338 (*dd_idx) += 2; /* D D P Q D */ 2339 ddf_layout = 1; 2340 break; 2341 2342 case ALGORITHM_ROTATING_N_RESTART: 2343 /* Same a left_asymmetric, by first stripe is 2344 * D D D P Q rather than 2345 * Q D D D P 2346 */ 2347 stripe2 += 1; 2348 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2349 qd_idx = pd_idx + 1; 2350 if (pd_idx == raid_disks-1) { 2351 (*dd_idx)++; /* Q D D D P */ 2352 qd_idx = 0; 2353 } else if (*dd_idx >= pd_idx) 2354 (*dd_idx) += 2; /* D D P Q D */ 2355 ddf_layout = 1; 2356 break; 2357 2358 case ALGORITHM_ROTATING_N_CONTINUE: 2359 /* Same as left_symmetric but Q is before P */ 2360 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2361 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2362 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2363 ddf_layout = 1; 2364 break; 2365 2366 case ALGORITHM_LEFT_ASYMMETRIC_6: 2367 /* RAID5 left_asymmetric, with Q on last device */ 2368 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2369 if (*dd_idx >= pd_idx) 2370 (*dd_idx)++; 2371 qd_idx = raid_disks - 1; 2372 break; 2373 2374 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2375 pd_idx = sector_div(stripe2, raid_disks-1); 2376 if (*dd_idx >= pd_idx) 2377 (*dd_idx)++; 2378 qd_idx = raid_disks - 1; 2379 break; 2380 2381 case ALGORITHM_LEFT_SYMMETRIC_6: 2382 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2383 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2384 qd_idx = raid_disks - 1; 2385 break; 2386 2387 case ALGORITHM_RIGHT_SYMMETRIC_6: 2388 pd_idx = sector_div(stripe2, raid_disks-1); 2389 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2390 qd_idx = raid_disks - 1; 2391 break; 2392 2393 case ALGORITHM_PARITY_0_6: 2394 pd_idx = 0; 2395 (*dd_idx)++; 2396 qd_idx = raid_disks - 1; 2397 break; 2398 2399 default: 2400 BUG(); 2401 } 2402 break; 2403 } 2404 2405 if (sh) { 2406 sh->pd_idx = pd_idx; 2407 sh->qd_idx = qd_idx; 2408 sh->ddf_layout = ddf_layout; 2409 } 2410 /* 2411 * Finally, compute the new sector number 2412 */ 2413 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2414 return new_sector; 2415 } 2416 2417 static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) 2418 { 2419 struct r5conf *conf = sh->raid_conf; 2420 int raid_disks = sh->disks; 2421 int data_disks = raid_disks - conf->max_degraded; 2422 sector_t new_sector = sh->sector, check; 2423 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2424 : conf->chunk_sectors; 2425 int algorithm = previous ? conf->prev_algo 2426 : conf->algorithm; 2427 sector_t stripe; 2428 int chunk_offset; 2429 sector_t chunk_number; 2430 int dummy1, dd_idx = i; 2431 sector_t r_sector; 2432 struct stripe_head sh2; 2433 2434 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2435 stripe = new_sector; 2436 2437 if (i == sh->pd_idx) 2438 return 0; 2439 switch(conf->level) { 2440 case 4: break; 2441 case 5: 2442 switch (algorithm) { 2443 case ALGORITHM_LEFT_ASYMMETRIC: 2444 case ALGORITHM_RIGHT_ASYMMETRIC: 2445 if (i > sh->pd_idx) 2446 i--; 2447 break; 2448 case ALGORITHM_LEFT_SYMMETRIC: 2449 case ALGORITHM_RIGHT_SYMMETRIC: 2450 if (i < sh->pd_idx) 2451 i += raid_disks; 2452 i -= (sh->pd_idx + 1); 2453 break; 2454 case ALGORITHM_PARITY_0: 2455 i -= 1; 2456 break; 2457 case ALGORITHM_PARITY_N: 2458 break; 2459 default: 2460 BUG(); 2461 } 2462 break; 2463 case 6: 2464 if (i == sh->qd_idx) 2465 return 0; /* It is the Q disk */ 2466 switch (algorithm) { 2467 case ALGORITHM_LEFT_ASYMMETRIC: 2468 case ALGORITHM_RIGHT_ASYMMETRIC: 2469 case ALGORITHM_ROTATING_ZERO_RESTART: 2470 case ALGORITHM_ROTATING_N_RESTART: 2471 if (sh->pd_idx == raid_disks-1) 2472 i--; /* Q D D D P */ 2473 else if (i > sh->pd_idx) 2474 i -= 2; /* D D P Q D */ 2475 break; 2476 case ALGORITHM_LEFT_SYMMETRIC: 2477 case ALGORITHM_RIGHT_SYMMETRIC: 2478 if (sh->pd_idx == raid_disks-1) 2479 i--; /* Q D D D P */ 2480 else { 2481 /* D D P Q D */ 2482 if (i < sh->pd_idx) 2483 i += raid_disks; 2484 i -= (sh->pd_idx + 2); 2485 } 2486 break; 2487 case ALGORITHM_PARITY_0: 2488 i -= 2; 2489 break; 2490 case ALGORITHM_PARITY_N: 2491 break; 2492 case ALGORITHM_ROTATING_N_CONTINUE: 2493 /* Like left_symmetric, but P is before Q */ 2494 if (sh->pd_idx == 0) 2495 i--; /* P D D D Q */ 2496 else { 2497 /* D D Q P D */ 2498 if (i < sh->pd_idx) 2499 i += raid_disks; 2500 i -= (sh->pd_idx + 1); 2501 } 2502 break; 2503 case ALGORITHM_LEFT_ASYMMETRIC_6: 2504 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2505 if (i > sh->pd_idx) 2506 i--; 2507 break; 2508 case ALGORITHM_LEFT_SYMMETRIC_6: 2509 case ALGORITHM_RIGHT_SYMMETRIC_6: 2510 if (i < sh->pd_idx) 2511 i += data_disks + 1; 2512 i -= (sh->pd_idx + 1); 2513 break; 2514 case ALGORITHM_PARITY_0_6: 2515 i -= 1; 2516 break; 2517 default: 2518 BUG(); 2519 } 2520 break; 2521 } 2522 2523 chunk_number = stripe * data_disks + i; 2524 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2525 2526 check = raid5_compute_sector(conf, r_sector, 2527 previous, &dummy1, &sh2); 2528 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2529 || sh2.qd_idx != sh->qd_idx) { 2530 printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", 2531 mdname(conf->mddev)); 2532 return 0; 2533 } 2534 return r_sector; 2535 } 2536 2537 static void 2538 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2539 int rcw, int expand) 2540 { 2541 int i, pd_idx = sh->pd_idx, disks = sh->disks; 2542 struct r5conf *conf = sh->raid_conf; 2543 int level = conf->level; 2544 2545 if (rcw) { 2546 2547 for (i = disks; i--; ) { 2548 struct r5dev *dev = &sh->dev[i]; 2549 2550 if (dev->towrite) { 2551 set_bit(R5_LOCKED, &dev->flags); 2552 set_bit(R5_Wantdrain, &dev->flags); 2553 if (!expand) 2554 clear_bit(R5_UPTODATE, &dev->flags); 2555 s->locked++; 2556 } 2557 } 2558 /* if we are not expanding this is a proper write request, and 2559 * there will be bios with new data to be drained into the 2560 * stripe cache 2561 */ 2562 if (!expand) { 2563 if (!s->locked) 2564 /* False alarm, nothing to do */ 2565 return; 2566 sh->reconstruct_state = reconstruct_state_drain_run; 2567 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2568 } else 2569 sh->reconstruct_state = reconstruct_state_run; 2570 2571 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2572 2573 if (s->locked + conf->max_degraded == disks) 2574 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2575 atomic_inc(&conf->pending_full_writes); 2576 } else { 2577 BUG_ON(level == 6); 2578 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2579 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2580 2581 for (i = disks; i--; ) { 2582 struct r5dev *dev = &sh->dev[i]; 2583 if (i == pd_idx) 2584 continue; 2585 2586 if (dev->towrite && 2587 (test_bit(R5_UPTODATE, &dev->flags) || 2588 test_bit(R5_Wantcompute, &dev->flags))) { 2589 set_bit(R5_Wantdrain, &dev->flags); 2590 set_bit(R5_LOCKED, &dev->flags); 2591 clear_bit(R5_UPTODATE, &dev->flags); 2592 s->locked++; 2593 } 2594 } 2595 if (!s->locked) 2596 /* False alarm - nothing to do */ 2597 return; 2598 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2599 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2600 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2601 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2602 } 2603 2604 /* keep the parity disk(s) locked while asynchronous operations 2605 * are in flight 2606 */ 2607 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2608 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2609 s->locked++; 2610 2611 if (level == 6) { 2612 int qd_idx = sh->qd_idx; 2613 struct r5dev *dev = &sh->dev[qd_idx]; 2614 2615 set_bit(R5_LOCKED, &dev->flags); 2616 clear_bit(R5_UPTODATE, &dev->flags); 2617 s->locked++; 2618 } 2619 2620 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2621 __func__, (unsigned long long)sh->sector, 2622 s->locked, s->ops_request); 2623 } 2624 2625 /* 2626 * Each stripe/dev can have one or more bion attached. 2627 * toread/towrite point to the first in a chain. 2628 * The bi_next chain must be in order. 2629 */ 2630 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) 2631 { 2632 struct bio **bip; 2633 struct r5conf *conf = sh->raid_conf; 2634 int firstwrite=0; 2635 2636 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2637 (unsigned long long)bi->bi_iter.bi_sector, 2638 (unsigned long long)sh->sector); 2639 2640 /* 2641 * If several bio share a stripe. The bio bi_phys_segments acts as a 2642 * reference count to avoid race. The reference count should already be 2643 * increased before this function is called (for example, in 2644 * make_request()), so other bio sharing this stripe will not free the 2645 * stripe. If a stripe is owned by one stripe, the stripe lock will 2646 * protect it. 2647 */ 2648 spin_lock_irq(&sh->stripe_lock); 2649 if (forwrite) { 2650 bip = &sh->dev[dd_idx].towrite; 2651 if (*bip == NULL) 2652 firstwrite = 1; 2653 } else 2654 bip = &sh->dev[dd_idx].toread; 2655 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { 2656 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) 2657 goto overlap; 2658 bip = & (*bip)->bi_next; 2659 } 2660 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) 2661 goto overlap; 2662 2663 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 2664 if (*bip) 2665 bi->bi_next = *bip; 2666 *bip = bi; 2667 raid5_inc_bi_active_stripes(bi); 2668 2669 if (forwrite) { 2670 /* check if page is covered */ 2671 sector_t sector = sh->dev[dd_idx].sector; 2672 for (bi=sh->dev[dd_idx].towrite; 2673 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 2674 bi && bi->bi_iter.bi_sector <= sector; 2675 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 2676 if (bio_end_sector(bi) >= sector) 2677 sector = bio_end_sector(bi); 2678 } 2679 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 2680 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags); 2681 } 2682 2683 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 2684 (unsigned long long)(*bip)->bi_iter.bi_sector, 2685 (unsigned long long)sh->sector, dd_idx); 2686 spin_unlock_irq(&sh->stripe_lock); 2687 2688 if (conf->mddev->bitmap && firstwrite) { 2689 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 2690 STRIPE_SECTORS, 0); 2691 sh->bm_seq = conf->seq_flush+1; 2692 set_bit(STRIPE_BIT_DELAY, &sh->state); 2693 } 2694 return 1; 2695 2696 overlap: 2697 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 2698 spin_unlock_irq(&sh->stripe_lock); 2699 return 0; 2700 } 2701 2702 static void end_reshape(struct r5conf *conf); 2703 2704 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 2705 struct stripe_head *sh) 2706 { 2707 int sectors_per_chunk = 2708 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 2709 int dd_idx; 2710 int chunk_offset = sector_div(stripe, sectors_per_chunk); 2711 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 2712 2713 raid5_compute_sector(conf, 2714 stripe * (disks - conf->max_degraded) 2715 *sectors_per_chunk + chunk_offset, 2716 previous, 2717 &dd_idx, sh); 2718 } 2719 2720 static void 2721 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 2722 struct stripe_head_state *s, int disks, 2723 struct bio **return_bi) 2724 { 2725 int i; 2726 for (i = disks; i--; ) { 2727 struct bio *bi; 2728 int bitmap_end = 0; 2729 2730 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2731 struct md_rdev *rdev; 2732 rcu_read_lock(); 2733 rdev = rcu_dereference(conf->disks[i].rdev); 2734 if (rdev && test_bit(In_sync, &rdev->flags)) 2735 atomic_inc(&rdev->nr_pending); 2736 else 2737 rdev = NULL; 2738 rcu_read_unlock(); 2739 if (rdev) { 2740 if (!rdev_set_badblocks( 2741 rdev, 2742 sh->sector, 2743 STRIPE_SECTORS, 0)) 2744 md_error(conf->mddev, rdev); 2745 rdev_dec_pending(rdev, conf->mddev); 2746 } 2747 } 2748 spin_lock_irq(&sh->stripe_lock); 2749 /* fail all writes first */ 2750 bi = sh->dev[i].towrite; 2751 sh->dev[i].towrite = NULL; 2752 spin_unlock_irq(&sh->stripe_lock); 2753 if (bi) 2754 bitmap_end = 1; 2755 2756 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2757 wake_up(&conf->wait_for_overlap); 2758 2759 while (bi && bi->bi_iter.bi_sector < 2760 sh->dev[i].sector + STRIPE_SECTORS) { 2761 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 2762 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2763 if (!raid5_dec_bi_active_stripes(bi)) { 2764 md_write_end(conf->mddev); 2765 bi->bi_next = *return_bi; 2766 *return_bi = bi; 2767 } 2768 bi = nextbi; 2769 } 2770 if (bitmap_end) 2771 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2772 STRIPE_SECTORS, 0, 0); 2773 bitmap_end = 0; 2774 /* and fail all 'written' */ 2775 bi = sh->dev[i].written; 2776 sh->dev[i].written = NULL; 2777 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { 2778 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 2779 sh->dev[i].page = sh->dev[i].orig_page; 2780 } 2781 2782 if (bi) bitmap_end = 1; 2783 while (bi && bi->bi_iter.bi_sector < 2784 sh->dev[i].sector + STRIPE_SECTORS) { 2785 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 2786 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2787 if (!raid5_dec_bi_active_stripes(bi)) { 2788 md_write_end(conf->mddev); 2789 bi->bi_next = *return_bi; 2790 *return_bi = bi; 2791 } 2792 bi = bi2; 2793 } 2794 2795 /* fail any reads if this device is non-operational and 2796 * the data has not reached the cache yet. 2797 */ 2798 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 2799 (!test_bit(R5_Insync, &sh->dev[i].flags) || 2800 test_bit(R5_ReadError, &sh->dev[i].flags))) { 2801 spin_lock_irq(&sh->stripe_lock); 2802 bi = sh->dev[i].toread; 2803 sh->dev[i].toread = NULL; 2804 spin_unlock_irq(&sh->stripe_lock); 2805 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 2806 wake_up(&conf->wait_for_overlap); 2807 while (bi && bi->bi_iter.bi_sector < 2808 sh->dev[i].sector + STRIPE_SECTORS) { 2809 struct bio *nextbi = 2810 r5_next_bio(bi, sh->dev[i].sector); 2811 clear_bit(BIO_UPTODATE, &bi->bi_flags); 2812 if (!raid5_dec_bi_active_stripes(bi)) { 2813 bi->bi_next = *return_bi; 2814 *return_bi = bi; 2815 } 2816 bi = nextbi; 2817 } 2818 } 2819 if (bitmap_end) 2820 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 2821 STRIPE_SECTORS, 0, 0); 2822 /* If we were in the middle of a write the parity block might 2823 * still be locked - so just clear all R5_LOCKED flags 2824 */ 2825 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2826 } 2827 2828 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 2829 if (atomic_dec_and_test(&conf->pending_full_writes)) 2830 md_wakeup_thread(conf->mddev->thread); 2831 } 2832 2833 static void 2834 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 2835 struct stripe_head_state *s) 2836 { 2837 int abort = 0; 2838 int i; 2839 2840 clear_bit(STRIPE_SYNCING, &sh->state); 2841 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 2842 wake_up(&conf->wait_for_overlap); 2843 s->syncing = 0; 2844 s->replacing = 0; 2845 /* There is nothing more to do for sync/check/repair. 2846 * Don't even need to abort as that is handled elsewhere 2847 * if needed, and not always wanted e.g. if there is a known 2848 * bad block here. 2849 * For recover/replace we need to record a bad block on all 2850 * non-sync devices, or abort the recovery 2851 */ 2852 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 2853 /* During recovery devices cannot be removed, so 2854 * locking and refcounting of rdevs is not needed 2855 */ 2856 for (i = 0; i < conf->raid_disks; i++) { 2857 struct md_rdev *rdev = conf->disks[i].rdev; 2858 if (rdev 2859 && !test_bit(Faulty, &rdev->flags) 2860 && !test_bit(In_sync, &rdev->flags) 2861 && !rdev_set_badblocks(rdev, sh->sector, 2862 STRIPE_SECTORS, 0)) 2863 abort = 1; 2864 rdev = conf->disks[i].replacement; 2865 if (rdev 2866 && !test_bit(Faulty, &rdev->flags) 2867 && !test_bit(In_sync, &rdev->flags) 2868 && !rdev_set_badblocks(rdev, sh->sector, 2869 STRIPE_SECTORS, 0)) 2870 abort = 1; 2871 } 2872 if (abort) 2873 conf->recovery_disabled = 2874 conf->mddev->recovery_disabled; 2875 } 2876 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 2877 } 2878 2879 static int want_replace(struct stripe_head *sh, int disk_idx) 2880 { 2881 struct md_rdev *rdev; 2882 int rv = 0; 2883 /* Doing recovery so rcu locking not required */ 2884 rdev = sh->raid_conf->disks[disk_idx].replacement; 2885 if (rdev 2886 && !test_bit(Faulty, &rdev->flags) 2887 && !test_bit(In_sync, &rdev->flags) 2888 && (rdev->recovery_offset <= sh->sector 2889 || rdev->mddev->recovery_cp <= sh->sector)) 2890 rv = 1; 2891 2892 return rv; 2893 } 2894 2895 /* fetch_block - checks the given member device to see if its data needs 2896 * to be read or computed to satisfy a request. 2897 * 2898 * Returns 1 when no more member devices need to be checked, otherwise returns 2899 * 0 to tell the loop in handle_stripe_fill to continue 2900 */ 2901 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 2902 int disk_idx, int disks) 2903 { 2904 struct r5dev *dev = &sh->dev[disk_idx]; 2905 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 2906 &sh->dev[s->failed_num[1]] }; 2907 2908 /* is the data in this block needed, and can we get it? */ 2909 if (!test_bit(R5_LOCKED, &dev->flags) && 2910 !test_bit(R5_UPTODATE, &dev->flags) && 2911 (dev->toread || 2912 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) || 2913 s->syncing || s->expanding || 2914 (s->replacing && want_replace(sh, disk_idx)) || 2915 (s->failed >= 1 && fdev[0]->toread) || 2916 (s->failed >= 2 && fdev[1]->toread) || 2917 (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite && 2918 (!test_bit(R5_Insync, &dev->flags) || test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) && 2919 !test_bit(R5_OVERWRITE, &fdev[0]->flags)) || 2920 (sh->raid_conf->level == 6 && s->failed && s->to_write && 2921 s->to_write - s->non_overwrite < sh->raid_conf->raid_disks - 2 && 2922 (!test_bit(R5_Insync, &dev->flags) || test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))))) { 2923 /* we would like to get this block, possibly by computing it, 2924 * otherwise read it if the backing disk is insync 2925 */ 2926 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 2927 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 2928 if ((s->uptodate == disks - 1) && 2929 (s->failed && (disk_idx == s->failed_num[0] || 2930 disk_idx == s->failed_num[1]))) { 2931 /* have disk failed, and we're requested to fetch it; 2932 * do compute it 2933 */ 2934 pr_debug("Computing stripe %llu block %d\n", 2935 (unsigned long long)sh->sector, disk_idx); 2936 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2937 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2938 set_bit(R5_Wantcompute, &dev->flags); 2939 sh->ops.target = disk_idx; 2940 sh->ops.target2 = -1; /* no 2nd target */ 2941 s->req_compute = 1; 2942 /* Careful: from this point on 'uptodate' is in the eye 2943 * of raid_run_ops which services 'compute' operations 2944 * before writes. R5_Wantcompute flags a block that will 2945 * be R5_UPTODATE by the time it is needed for a 2946 * subsequent operation. 2947 */ 2948 s->uptodate++; 2949 return 1; 2950 } else if (s->uptodate == disks-2 && s->failed >= 2) { 2951 /* Computing 2-failure is *very* expensive; only 2952 * do it if failed >= 2 2953 */ 2954 int other; 2955 for (other = disks; other--; ) { 2956 if (other == disk_idx) 2957 continue; 2958 if (!test_bit(R5_UPTODATE, 2959 &sh->dev[other].flags)) 2960 break; 2961 } 2962 BUG_ON(other < 0); 2963 pr_debug("Computing stripe %llu blocks %d,%d\n", 2964 (unsigned long long)sh->sector, 2965 disk_idx, other); 2966 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 2967 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 2968 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 2969 set_bit(R5_Wantcompute, &sh->dev[other].flags); 2970 sh->ops.target = disk_idx; 2971 sh->ops.target2 = other; 2972 s->uptodate += 2; 2973 s->req_compute = 1; 2974 return 1; 2975 } else if (test_bit(R5_Insync, &dev->flags)) { 2976 set_bit(R5_LOCKED, &dev->flags); 2977 set_bit(R5_Wantread, &dev->flags); 2978 s->locked++; 2979 pr_debug("Reading block %d (sync=%d)\n", 2980 disk_idx, s->syncing); 2981 } 2982 } 2983 2984 return 0; 2985 } 2986 2987 /** 2988 * handle_stripe_fill - read or compute data to satisfy pending requests. 2989 */ 2990 static void handle_stripe_fill(struct stripe_head *sh, 2991 struct stripe_head_state *s, 2992 int disks) 2993 { 2994 int i; 2995 2996 /* look for blocks to read/compute, skip this if a compute 2997 * is already in flight, or if the stripe contents are in the 2998 * midst of changing due to a write 2999 */ 3000 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 3001 !sh->reconstruct_state) 3002 for (i = disks; i--; ) 3003 if (fetch_block(sh, s, i, disks)) 3004 break; 3005 set_bit(STRIPE_HANDLE, &sh->state); 3006 } 3007 3008 /* handle_stripe_clean_event 3009 * any written block on an uptodate or failed drive can be returned. 3010 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 3011 * never LOCKED, so we don't need to test 'failed' directly. 3012 */ 3013 static void handle_stripe_clean_event(struct r5conf *conf, 3014 struct stripe_head *sh, int disks, struct bio **return_bi) 3015 { 3016 int i; 3017 struct r5dev *dev; 3018 int discard_pending = 0; 3019 3020 for (i = disks; i--; ) 3021 if (sh->dev[i].written) { 3022 dev = &sh->dev[i]; 3023 if (!test_bit(R5_LOCKED, &dev->flags) && 3024 (test_bit(R5_UPTODATE, &dev->flags) || 3025 test_bit(R5_Discard, &dev->flags) || 3026 test_bit(R5_SkipCopy, &dev->flags))) { 3027 /* We can return any write requests */ 3028 struct bio *wbi, *wbi2; 3029 pr_debug("Return write for disc %d\n", i); 3030 if (test_and_clear_bit(R5_Discard, &dev->flags)) 3031 clear_bit(R5_UPTODATE, &dev->flags); 3032 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { 3033 WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); 3034 dev->page = dev->orig_page; 3035 } 3036 wbi = dev->written; 3037 dev->written = NULL; 3038 while (wbi && wbi->bi_iter.bi_sector < 3039 dev->sector + STRIPE_SECTORS) { 3040 wbi2 = r5_next_bio(wbi, dev->sector); 3041 if (!raid5_dec_bi_active_stripes(wbi)) { 3042 md_write_end(conf->mddev); 3043 wbi->bi_next = *return_bi; 3044 *return_bi = wbi; 3045 } 3046 wbi = wbi2; 3047 } 3048 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3049 STRIPE_SECTORS, 3050 !test_bit(STRIPE_DEGRADED, &sh->state), 3051 0); 3052 } else if (test_bit(R5_Discard, &dev->flags)) 3053 discard_pending = 1; 3054 WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); 3055 WARN_ON(dev->page != dev->orig_page); 3056 } 3057 if (!discard_pending && 3058 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { 3059 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 3060 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3061 if (sh->qd_idx >= 0) { 3062 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 3063 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); 3064 } 3065 /* now that discard is done we can proceed with any sync */ 3066 clear_bit(STRIPE_DISCARD, &sh->state); 3067 /* 3068 * SCSI discard will change some bio fields and the stripe has 3069 * no updated data, so remove it from hash list and the stripe 3070 * will be reinitialized 3071 */ 3072 spin_lock_irq(&conf->device_lock); 3073 remove_hash(sh); 3074 spin_unlock_irq(&conf->device_lock); 3075 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) 3076 set_bit(STRIPE_HANDLE, &sh->state); 3077 3078 } 3079 3080 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3081 if (atomic_dec_and_test(&conf->pending_full_writes)) 3082 md_wakeup_thread(conf->mddev->thread); 3083 } 3084 3085 static void handle_stripe_dirtying(struct r5conf *conf, 3086 struct stripe_head *sh, 3087 struct stripe_head_state *s, 3088 int disks) 3089 { 3090 int rmw = 0, rcw = 0, i; 3091 sector_t recovery_cp = conf->mddev->recovery_cp; 3092 3093 /* RAID6 requires 'rcw' in current implementation. 3094 * Otherwise, check whether resync is now happening or should start. 3095 * If yes, then the array is dirty (after unclean shutdown or 3096 * initial creation), so parity in some stripes might be inconsistent. 3097 * In this case, we need to always do reconstruct-write, to ensure 3098 * that in case of drive failure or read-error correction, we 3099 * generate correct data from the parity. 3100 */ 3101 if (conf->max_degraded == 2 || 3102 (recovery_cp < MaxSector && sh->sector >= recovery_cp)) { 3103 /* Calculate the real rcw later - for now make it 3104 * look like rcw is cheaper 3105 */ 3106 rcw = 1; rmw = 2; 3107 pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n", 3108 conf->max_degraded, (unsigned long long)recovery_cp, 3109 (unsigned long long)sh->sector); 3110 } else for (i = disks; i--; ) { 3111 /* would I have to read this buffer for read_modify_write */ 3112 struct r5dev *dev = &sh->dev[i]; 3113 if ((dev->towrite || i == sh->pd_idx) && 3114 !test_bit(R5_LOCKED, &dev->flags) && 3115 !(test_bit(R5_UPTODATE, &dev->flags) || 3116 test_bit(R5_Wantcompute, &dev->flags))) { 3117 if (test_bit(R5_Insync, &dev->flags)) 3118 rmw++; 3119 else 3120 rmw += 2*disks; /* cannot read it */ 3121 } 3122 /* Would I have to read this buffer for reconstruct_write */ 3123 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx && 3124 !test_bit(R5_LOCKED, &dev->flags) && 3125 !(test_bit(R5_UPTODATE, &dev->flags) || 3126 test_bit(R5_Wantcompute, &dev->flags))) { 3127 if (test_bit(R5_Insync, &dev->flags)) 3128 rcw++; 3129 else 3130 rcw += 2*disks; 3131 } 3132 } 3133 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 3134 (unsigned long long)sh->sector, rmw, rcw); 3135 set_bit(STRIPE_HANDLE, &sh->state); 3136 if (rmw < rcw && rmw > 0) { 3137 /* prefer read-modify-write, but need to get some data */ 3138 if (conf->mddev->queue) 3139 blk_add_trace_msg(conf->mddev->queue, 3140 "raid5 rmw %llu %d", 3141 (unsigned long long)sh->sector, rmw); 3142 for (i = disks; i--; ) { 3143 struct r5dev *dev = &sh->dev[i]; 3144 if ((dev->towrite || i == sh->pd_idx) && 3145 !test_bit(R5_LOCKED, &dev->flags) && 3146 !(test_bit(R5_UPTODATE, &dev->flags) || 3147 test_bit(R5_Wantcompute, &dev->flags)) && 3148 test_bit(R5_Insync, &dev->flags)) { 3149 if (test_bit(STRIPE_PREREAD_ACTIVE, 3150 &sh->state)) { 3151 pr_debug("Read_old block %d for r-m-w\n", 3152 i); 3153 set_bit(R5_LOCKED, &dev->flags); 3154 set_bit(R5_Wantread, &dev->flags); 3155 s->locked++; 3156 } else { 3157 set_bit(STRIPE_DELAYED, &sh->state); 3158 set_bit(STRIPE_HANDLE, &sh->state); 3159 } 3160 } 3161 } 3162 } 3163 if (rcw <= rmw && rcw > 0) { 3164 /* want reconstruct write, but need to get some data */ 3165 int qread =0; 3166 rcw = 0; 3167 for (i = disks; i--; ) { 3168 struct r5dev *dev = &sh->dev[i]; 3169 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3170 i != sh->pd_idx && i != sh->qd_idx && 3171 !test_bit(R5_LOCKED, &dev->flags) && 3172 !(test_bit(R5_UPTODATE, &dev->flags) || 3173 test_bit(R5_Wantcompute, &dev->flags))) { 3174 rcw++; 3175 if (test_bit(R5_Insync, &dev->flags) && 3176 test_bit(STRIPE_PREREAD_ACTIVE, 3177 &sh->state)) { 3178 pr_debug("Read_old block " 3179 "%d for Reconstruct\n", i); 3180 set_bit(R5_LOCKED, &dev->flags); 3181 set_bit(R5_Wantread, &dev->flags); 3182 s->locked++; 3183 qread++; 3184 } else { 3185 set_bit(STRIPE_DELAYED, &sh->state); 3186 set_bit(STRIPE_HANDLE, &sh->state); 3187 } 3188 } 3189 } 3190 if (rcw && conf->mddev->queue) 3191 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", 3192 (unsigned long long)sh->sector, 3193 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); 3194 } 3195 /* now if nothing is locked, and if we have enough data, 3196 * we can start a write request 3197 */ 3198 /* since handle_stripe can be called at any time we need to handle the 3199 * case where a compute block operation has been submitted and then a 3200 * subsequent call wants to start a write request. raid_run_ops only 3201 * handles the case where compute block and reconstruct are requested 3202 * simultaneously. If this is not the case then new writes need to be 3203 * held off until the compute completes. 3204 */ 3205 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 3206 (s->locked == 0 && (rcw == 0 || rmw == 0) && 3207 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 3208 schedule_reconstruction(sh, s, rcw == 0, 0); 3209 } 3210 3211 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 3212 struct stripe_head_state *s, int disks) 3213 { 3214 struct r5dev *dev = NULL; 3215 3216 set_bit(STRIPE_HANDLE, &sh->state); 3217 3218 switch (sh->check_state) { 3219 case check_state_idle: 3220 /* start a new check operation if there are no failures */ 3221 if (s->failed == 0) { 3222 BUG_ON(s->uptodate != disks); 3223 sh->check_state = check_state_run; 3224 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3225 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3226 s->uptodate--; 3227 break; 3228 } 3229 dev = &sh->dev[s->failed_num[0]]; 3230 /* fall through */ 3231 case check_state_compute_result: 3232 sh->check_state = check_state_idle; 3233 if (!dev) 3234 dev = &sh->dev[sh->pd_idx]; 3235 3236 /* check that a write has not made the stripe insync */ 3237 if (test_bit(STRIPE_INSYNC, &sh->state)) 3238 break; 3239 3240 /* either failed parity check, or recovery is happening */ 3241 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3242 BUG_ON(s->uptodate != disks); 3243 3244 set_bit(R5_LOCKED, &dev->flags); 3245 s->locked++; 3246 set_bit(R5_Wantwrite, &dev->flags); 3247 3248 clear_bit(STRIPE_DEGRADED, &sh->state); 3249 set_bit(STRIPE_INSYNC, &sh->state); 3250 break; 3251 case check_state_run: 3252 break; /* we will be called again upon completion */ 3253 case check_state_check_result: 3254 sh->check_state = check_state_idle; 3255 3256 /* if a failure occurred during the check operation, leave 3257 * STRIPE_INSYNC not set and let the stripe be handled again 3258 */ 3259 if (s->failed) 3260 break; 3261 3262 /* handle a successful check operation, if parity is correct 3263 * we are done. Otherwise update the mismatch count and repair 3264 * parity if !MD_RECOVERY_CHECK 3265 */ 3266 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 3267 /* parity is correct (on disc, 3268 * not in buffer any more) 3269 */ 3270 set_bit(STRIPE_INSYNC, &sh->state); 3271 else { 3272 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3273 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3274 /* don't try to repair!! */ 3275 set_bit(STRIPE_INSYNC, &sh->state); 3276 else { 3277 sh->check_state = check_state_compute_run; 3278 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3279 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3280 set_bit(R5_Wantcompute, 3281 &sh->dev[sh->pd_idx].flags); 3282 sh->ops.target = sh->pd_idx; 3283 sh->ops.target2 = -1; 3284 s->uptodate++; 3285 } 3286 } 3287 break; 3288 case check_state_compute_run: 3289 break; 3290 default: 3291 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3292 __func__, sh->check_state, 3293 (unsigned long long) sh->sector); 3294 BUG(); 3295 } 3296 } 3297 3298 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 3299 struct stripe_head_state *s, 3300 int disks) 3301 { 3302 int pd_idx = sh->pd_idx; 3303 int qd_idx = sh->qd_idx; 3304 struct r5dev *dev; 3305 3306 set_bit(STRIPE_HANDLE, &sh->state); 3307 3308 BUG_ON(s->failed > 2); 3309 3310 /* Want to check and possibly repair P and Q. 3311 * However there could be one 'failed' device, in which 3312 * case we can only check one of them, possibly using the 3313 * other to generate missing data 3314 */ 3315 3316 switch (sh->check_state) { 3317 case check_state_idle: 3318 /* start a new check operation if there are < 2 failures */ 3319 if (s->failed == s->q_failed) { 3320 /* The only possible failed device holds Q, so it 3321 * makes sense to check P (If anything else were failed, 3322 * we would have used P to recreate it). 3323 */ 3324 sh->check_state = check_state_run; 3325 } 3326 if (!s->q_failed && s->failed < 2) { 3327 /* Q is not failed, and we didn't use it to generate 3328 * anything, so it makes sense to check it 3329 */ 3330 if (sh->check_state == check_state_run) 3331 sh->check_state = check_state_run_pq; 3332 else 3333 sh->check_state = check_state_run_q; 3334 } 3335 3336 /* discard potentially stale zero_sum_result */ 3337 sh->ops.zero_sum_result = 0; 3338 3339 if (sh->check_state == check_state_run) { 3340 /* async_xor_zero_sum destroys the contents of P */ 3341 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 3342 s->uptodate--; 3343 } 3344 if (sh->check_state >= check_state_run && 3345 sh->check_state <= check_state_run_pq) { 3346 /* async_syndrome_zero_sum preserves P and Q, so 3347 * no need to mark them !uptodate here 3348 */ 3349 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3350 break; 3351 } 3352 3353 /* we have 2-disk failure */ 3354 BUG_ON(s->failed != 2); 3355 /* fall through */ 3356 case check_state_compute_result: 3357 sh->check_state = check_state_idle; 3358 3359 /* check that a write has not made the stripe insync */ 3360 if (test_bit(STRIPE_INSYNC, &sh->state)) 3361 break; 3362 3363 /* now write out any block on a failed drive, 3364 * or P or Q if they were recomputed 3365 */ 3366 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 3367 if (s->failed == 2) { 3368 dev = &sh->dev[s->failed_num[1]]; 3369 s->locked++; 3370 set_bit(R5_LOCKED, &dev->flags); 3371 set_bit(R5_Wantwrite, &dev->flags); 3372 } 3373 if (s->failed >= 1) { 3374 dev = &sh->dev[s->failed_num[0]]; 3375 s->locked++; 3376 set_bit(R5_LOCKED, &dev->flags); 3377 set_bit(R5_Wantwrite, &dev->flags); 3378 } 3379 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3380 dev = &sh->dev[pd_idx]; 3381 s->locked++; 3382 set_bit(R5_LOCKED, &dev->flags); 3383 set_bit(R5_Wantwrite, &dev->flags); 3384 } 3385 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3386 dev = &sh->dev[qd_idx]; 3387 s->locked++; 3388 set_bit(R5_LOCKED, &dev->flags); 3389 set_bit(R5_Wantwrite, &dev->flags); 3390 } 3391 clear_bit(STRIPE_DEGRADED, &sh->state); 3392 3393 set_bit(STRIPE_INSYNC, &sh->state); 3394 break; 3395 case check_state_run: 3396 case check_state_run_q: 3397 case check_state_run_pq: 3398 break; /* we will be called again upon completion */ 3399 case check_state_check_result: 3400 sh->check_state = check_state_idle; 3401 3402 /* handle a successful check operation, if parity is correct 3403 * we are done. Otherwise update the mismatch count and repair 3404 * parity if !MD_RECOVERY_CHECK 3405 */ 3406 if (sh->ops.zero_sum_result == 0) { 3407 /* both parities are correct */ 3408 if (!s->failed) 3409 set_bit(STRIPE_INSYNC, &sh->state); 3410 else { 3411 /* in contrast to the raid5 case we can validate 3412 * parity, but still have a failure to write 3413 * back 3414 */ 3415 sh->check_state = check_state_compute_result; 3416 /* Returning at this point means that we may go 3417 * off and bring p and/or q uptodate again so 3418 * we make sure to check zero_sum_result again 3419 * to verify if p or q need writeback 3420 */ 3421 } 3422 } else { 3423 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3424 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3425 /* don't try to repair!! */ 3426 set_bit(STRIPE_INSYNC, &sh->state); 3427 else { 3428 int *target = &sh->ops.target; 3429 3430 sh->ops.target = -1; 3431 sh->ops.target2 = -1; 3432 sh->check_state = check_state_compute_run; 3433 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3434 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3435 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3436 set_bit(R5_Wantcompute, 3437 &sh->dev[pd_idx].flags); 3438 *target = pd_idx; 3439 target = &sh->ops.target2; 3440 s->uptodate++; 3441 } 3442 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3443 set_bit(R5_Wantcompute, 3444 &sh->dev[qd_idx].flags); 3445 *target = qd_idx; 3446 s->uptodate++; 3447 } 3448 } 3449 } 3450 break; 3451 case check_state_compute_run: 3452 break; 3453 default: 3454 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", 3455 __func__, sh->check_state, 3456 (unsigned long long) sh->sector); 3457 BUG(); 3458 } 3459 } 3460 3461 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 3462 { 3463 int i; 3464 3465 /* We have read all the blocks in this stripe and now we need to 3466 * copy some of them into a target stripe for expand. 3467 */ 3468 struct dma_async_tx_descriptor *tx = NULL; 3469 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3470 for (i = 0; i < sh->disks; i++) 3471 if (i != sh->pd_idx && i != sh->qd_idx) { 3472 int dd_idx, j; 3473 struct stripe_head *sh2; 3474 struct async_submit_ctl submit; 3475 3476 sector_t bn = compute_blocknr(sh, i, 1); 3477 sector_t s = raid5_compute_sector(conf, bn, 0, 3478 &dd_idx, NULL); 3479 sh2 = get_active_stripe(conf, s, 0, 1, 1); 3480 if (sh2 == NULL) 3481 /* so far only the early blocks of this stripe 3482 * have been requested. When later blocks 3483 * get requested, we will try again 3484 */ 3485 continue; 3486 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 3487 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 3488 /* must have already done this block */ 3489 release_stripe(sh2); 3490 continue; 3491 } 3492 3493 /* place all the copies on one channel */ 3494 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 3495 tx = async_memcpy(sh2->dev[dd_idx].page, 3496 sh->dev[i].page, 0, 0, STRIPE_SIZE, 3497 &submit); 3498 3499 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 3500 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 3501 for (j = 0; j < conf->raid_disks; j++) 3502 if (j != sh2->pd_idx && 3503 j != sh2->qd_idx && 3504 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 3505 break; 3506 if (j == conf->raid_disks) { 3507 set_bit(STRIPE_EXPAND_READY, &sh2->state); 3508 set_bit(STRIPE_HANDLE, &sh2->state); 3509 } 3510 release_stripe(sh2); 3511 3512 } 3513 /* done submitting copies, wait for them to complete */ 3514 async_tx_quiesce(&tx); 3515 } 3516 3517 /* 3518 * handle_stripe - do things to a stripe. 3519 * 3520 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 3521 * state of various bits to see what needs to be done. 3522 * Possible results: 3523 * return some read requests which now have data 3524 * return some write requests which are safely on storage 3525 * schedule a read on some buffers 3526 * schedule a write of some buffers 3527 * return confirmation of parity correctness 3528 * 3529 */ 3530 3531 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 3532 { 3533 struct r5conf *conf = sh->raid_conf; 3534 int disks = sh->disks; 3535 struct r5dev *dev; 3536 int i; 3537 int do_recovery = 0; 3538 3539 memset(s, 0, sizeof(*s)); 3540 3541 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state); 3542 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state); 3543 s->failed_num[0] = -1; 3544 s->failed_num[1] = -1; 3545 3546 /* Now to look around and see what can be done */ 3547 rcu_read_lock(); 3548 for (i=disks; i--; ) { 3549 struct md_rdev *rdev; 3550 sector_t first_bad; 3551 int bad_sectors; 3552 int is_bad = 0; 3553 3554 dev = &sh->dev[i]; 3555 3556 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 3557 i, dev->flags, 3558 dev->toread, dev->towrite, dev->written); 3559 /* maybe we can reply to a read 3560 * 3561 * new wantfill requests are only permitted while 3562 * ops_complete_biofill is guaranteed to be inactive 3563 */ 3564 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 3565 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 3566 set_bit(R5_Wantfill, &dev->flags); 3567 3568 /* now count some things */ 3569 if (test_bit(R5_LOCKED, &dev->flags)) 3570 s->locked++; 3571 if (test_bit(R5_UPTODATE, &dev->flags)) 3572 s->uptodate++; 3573 if (test_bit(R5_Wantcompute, &dev->flags)) { 3574 s->compute++; 3575 BUG_ON(s->compute > 2); 3576 } 3577 3578 if (test_bit(R5_Wantfill, &dev->flags)) 3579 s->to_fill++; 3580 else if (dev->toread) 3581 s->to_read++; 3582 if (dev->towrite) { 3583 s->to_write++; 3584 if (!test_bit(R5_OVERWRITE, &dev->flags)) 3585 s->non_overwrite++; 3586 } 3587 if (dev->written) 3588 s->written++; 3589 /* Prefer to use the replacement for reads, but only 3590 * if it is recovered enough and has no bad blocks. 3591 */ 3592 rdev = rcu_dereference(conf->disks[i].replacement); 3593 if (rdev && !test_bit(Faulty, &rdev->flags) && 3594 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 3595 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3596 &first_bad, &bad_sectors)) 3597 set_bit(R5_ReadRepl, &dev->flags); 3598 else { 3599 if (rdev) 3600 set_bit(R5_NeedReplace, &dev->flags); 3601 rdev = rcu_dereference(conf->disks[i].rdev); 3602 clear_bit(R5_ReadRepl, &dev->flags); 3603 } 3604 if (rdev && test_bit(Faulty, &rdev->flags)) 3605 rdev = NULL; 3606 if (rdev) { 3607 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 3608 &first_bad, &bad_sectors); 3609 if (s->blocked_rdev == NULL 3610 && (test_bit(Blocked, &rdev->flags) 3611 || is_bad < 0)) { 3612 if (is_bad < 0) 3613 set_bit(BlockedBadBlocks, 3614 &rdev->flags); 3615 s->blocked_rdev = rdev; 3616 atomic_inc(&rdev->nr_pending); 3617 } 3618 } 3619 clear_bit(R5_Insync, &dev->flags); 3620 if (!rdev) 3621 /* Not in-sync */; 3622 else if (is_bad) { 3623 /* also not in-sync */ 3624 if (!test_bit(WriteErrorSeen, &rdev->flags) && 3625 test_bit(R5_UPTODATE, &dev->flags)) { 3626 /* treat as in-sync, but with a read error 3627 * which we can now try to correct 3628 */ 3629 set_bit(R5_Insync, &dev->flags); 3630 set_bit(R5_ReadError, &dev->flags); 3631 } 3632 } else if (test_bit(In_sync, &rdev->flags)) 3633 set_bit(R5_Insync, &dev->flags); 3634 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 3635 /* in sync if before recovery_offset */ 3636 set_bit(R5_Insync, &dev->flags); 3637 else if (test_bit(R5_UPTODATE, &dev->flags) && 3638 test_bit(R5_Expanded, &dev->flags)) 3639 /* If we've reshaped into here, we assume it is Insync. 3640 * We will shortly update recovery_offset to make 3641 * it official. 3642 */ 3643 set_bit(R5_Insync, &dev->flags); 3644 3645 if (test_bit(R5_WriteError, &dev->flags)) { 3646 /* This flag does not apply to '.replacement' 3647 * only to .rdev, so make sure to check that*/ 3648 struct md_rdev *rdev2 = rcu_dereference( 3649 conf->disks[i].rdev); 3650 if (rdev2 == rdev) 3651 clear_bit(R5_Insync, &dev->flags); 3652 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3653 s->handle_bad_blocks = 1; 3654 atomic_inc(&rdev2->nr_pending); 3655 } else 3656 clear_bit(R5_WriteError, &dev->flags); 3657 } 3658 if (test_bit(R5_MadeGood, &dev->flags)) { 3659 /* This flag does not apply to '.replacement' 3660 * only to .rdev, so make sure to check that*/ 3661 struct md_rdev *rdev2 = rcu_dereference( 3662 conf->disks[i].rdev); 3663 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3664 s->handle_bad_blocks = 1; 3665 atomic_inc(&rdev2->nr_pending); 3666 } else 3667 clear_bit(R5_MadeGood, &dev->flags); 3668 } 3669 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 3670 struct md_rdev *rdev2 = rcu_dereference( 3671 conf->disks[i].replacement); 3672 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 3673 s->handle_bad_blocks = 1; 3674 atomic_inc(&rdev2->nr_pending); 3675 } else 3676 clear_bit(R5_MadeGoodRepl, &dev->flags); 3677 } 3678 if (!test_bit(R5_Insync, &dev->flags)) { 3679 /* The ReadError flag will just be confusing now */ 3680 clear_bit(R5_ReadError, &dev->flags); 3681 clear_bit(R5_ReWrite, &dev->flags); 3682 } 3683 if (test_bit(R5_ReadError, &dev->flags)) 3684 clear_bit(R5_Insync, &dev->flags); 3685 if (!test_bit(R5_Insync, &dev->flags)) { 3686 if (s->failed < 2) 3687 s->failed_num[s->failed] = i; 3688 s->failed++; 3689 if (rdev && !test_bit(Faulty, &rdev->flags)) 3690 do_recovery = 1; 3691 } 3692 } 3693 if (test_bit(STRIPE_SYNCING, &sh->state)) { 3694 /* If there is a failed device being replaced, 3695 * we must be recovering. 3696 * else if we are after recovery_cp, we must be syncing 3697 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 3698 * else we can only be replacing 3699 * sync and recovery both need to read all devices, and so 3700 * use the same flag. 3701 */ 3702 if (do_recovery || 3703 sh->sector >= conf->mddev->recovery_cp || 3704 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 3705 s->syncing = 1; 3706 else 3707 s->replacing = 1; 3708 } 3709 rcu_read_unlock(); 3710 } 3711 3712 static void handle_stripe(struct stripe_head *sh) 3713 { 3714 struct stripe_head_state s; 3715 struct r5conf *conf = sh->raid_conf; 3716 int i; 3717 int prexor; 3718 int disks = sh->disks; 3719 struct r5dev *pdev, *qdev; 3720 3721 clear_bit(STRIPE_HANDLE, &sh->state); 3722 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 3723 /* already being handled, ensure it gets handled 3724 * again when current action finishes */ 3725 set_bit(STRIPE_HANDLE, &sh->state); 3726 return; 3727 } 3728 3729 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3730 spin_lock(&sh->stripe_lock); 3731 /* Cannot process 'sync' concurrently with 'discard' */ 3732 if (!test_bit(STRIPE_DISCARD, &sh->state) && 3733 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 3734 set_bit(STRIPE_SYNCING, &sh->state); 3735 clear_bit(STRIPE_INSYNC, &sh->state); 3736 clear_bit(STRIPE_REPLACED, &sh->state); 3737 } 3738 spin_unlock(&sh->stripe_lock); 3739 } 3740 clear_bit(STRIPE_DELAYED, &sh->state); 3741 3742 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 3743 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 3744 (unsigned long long)sh->sector, sh->state, 3745 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 3746 sh->check_state, sh->reconstruct_state); 3747 3748 analyse_stripe(sh, &s); 3749 3750 if (s.handle_bad_blocks) { 3751 set_bit(STRIPE_HANDLE, &sh->state); 3752 goto finish; 3753 } 3754 3755 if (unlikely(s.blocked_rdev)) { 3756 if (s.syncing || s.expanding || s.expanded || 3757 s.replacing || s.to_write || s.written) { 3758 set_bit(STRIPE_HANDLE, &sh->state); 3759 goto finish; 3760 } 3761 /* There is nothing for the blocked_rdev to block */ 3762 rdev_dec_pending(s.blocked_rdev, conf->mddev); 3763 s.blocked_rdev = NULL; 3764 } 3765 3766 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 3767 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 3768 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 3769 } 3770 3771 pr_debug("locked=%d uptodate=%d to_read=%d" 3772 " to_write=%d failed=%d failed_num=%d,%d\n", 3773 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 3774 s.failed_num[0], s.failed_num[1]); 3775 /* check if the array has lost more than max_degraded devices and, 3776 * if so, some requests might need to be failed. 3777 */ 3778 if (s.failed > conf->max_degraded) { 3779 sh->check_state = 0; 3780 sh->reconstruct_state = 0; 3781 if (s.to_read+s.to_write+s.written) 3782 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 3783 if (s.syncing + s.replacing) 3784 handle_failed_sync(conf, sh, &s); 3785 } 3786 3787 /* Now we check to see if any write operations have recently 3788 * completed 3789 */ 3790 prexor = 0; 3791 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 3792 prexor = 1; 3793 if (sh->reconstruct_state == reconstruct_state_drain_result || 3794 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 3795 sh->reconstruct_state = reconstruct_state_idle; 3796 3797 /* All the 'written' buffers and the parity block are ready to 3798 * be written back to disk 3799 */ 3800 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && 3801 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); 3802 BUG_ON(sh->qd_idx >= 0 && 3803 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && 3804 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); 3805 for (i = disks; i--; ) { 3806 struct r5dev *dev = &sh->dev[i]; 3807 if (test_bit(R5_LOCKED, &dev->flags) && 3808 (i == sh->pd_idx || i == sh->qd_idx || 3809 dev->written)) { 3810 pr_debug("Writing block %d\n", i); 3811 set_bit(R5_Wantwrite, &dev->flags); 3812 if (prexor) 3813 continue; 3814 if (s.failed > 1) 3815 continue; 3816 if (!test_bit(R5_Insync, &dev->flags) || 3817 ((i == sh->pd_idx || i == sh->qd_idx) && 3818 s.failed == 0)) 3819 set_bit(STRIPE_INSYNC, &sh->state); 3820 } 3821 } 3822 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3823 s.dec_preread_active = 1; 3824 } 3825 3826 /* 3827 * might be able to return some write requests if the parity blocks 3828 * are safe, or on a failed drive 3829 */ 3830 pdev = &sh->dev[sh->pd_idx]; 3831 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 3832 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 3833 qdev = &sh->dev[sh->qd_idx]; 3834 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 3835 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 3836 || conf->level < 6; 3837 3838 if (s.written && 3839 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 3840 && !test_bit(R5_LOCKED, &pdev->flags) 3841 && (test_bit(R5_UPTODATE, &pdev->flags) || 3842 test_bit(R5_Discard, &pdev->flags))))) && 3843 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 3844 && !test_bit(R5_LOCKED, &qdev->flags) 3845 && (test_bit(R5_UPTODATE, &qdev->flags) || 3846 test_bit(R5_Discard, &qdev->flags)))))) 3847 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 3848 3849 /* Now we might consider reading some blocks, either to check/generate 3850 * parity, or to satisfy requests 3851 * or to load a block that is being partially written. 3852 */ 3853 if (s.to_read || s.non_overwrite 3854 || (conf->level == 6 && s.to_write && s.failed) 3855 || (s.syncing && (s.uptodate + s.compute < disks)) 3856 || s.replacing 3857 || s.expanding) 3858 handle_stripe_fill(sh, &s, disks); 3859 3860 /* Now to consider new write requests and what else, if anything 3861 * should be read. We do not handle new writes when: 3862 * 1/ A 'write' operation (copy+xor) is already in flight. 3863 * 2/ A 'check' operation is in flight, as it may clobber the parity 3864 * block. 3865 */ 3866 if (s.to_write && !sh->reconstruct_state && !sh->check_state) 3867 handle_stripe_dirtying(conf, sh, &s, disks); 3868 3869 /* maybe we need to check and possibly fix the parity for this stripe 3870 * Any reads will already have been scheduled, so we just see if enough 3871 * data is available. The parity check is held off while parity 3872 * dependent operations are in flight. 3873 */ 3874 if (sh->check_state || 3875 (s.syncing && s.locked == 0 && 3876 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3877 !test_bit(STRIPE_INSYNC, &sh->state))) { 3878 if (conf->level == 6) 3879 handle_parity_checks6(conf, sh, &s, disks); 3880 else 3881 handle_parity_checks5(conf, sh, &s, disks); 3882 } 3883 3884 if ((s.replacing || s.syncing) && s.locked == 0 3885 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) 3886 && !test_bit(STRIPE_REPLACED, &sh->state)) { 3887 /* Write out to replacement devices where possible */ 3888 for (i = 0; i < conf->raid_disks; i++) 3889 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 3890 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); 3891 set_bit(R5_WantReplace, &sh->dev[i].flags); 3892 set_bit(R5_LOCKED, &sh->dev[i].flags); 3893 s.locked++; 3894 } 3895 if (s.replacing) 3896 set_bit(STRIPE_INSYNC, &sh->state); 3897 set_bit(STRIPE_REPLACED, &sh->state); 3898 } 3899 if ((s.syncing || s.replacing) && s.locked == 0 && 3900 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 3901 test_bit(STRIPE_INSYNC, &sh->state)) { 3902 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3903 clear_bit(STRIPE_SYNCING, &sh->state); 3904 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 3905 wake_up(&conf->wait_for_overlap); 3906 } 3907 3908 /* If the failed drives are just a ReadError, then we might need 3909 * to progress the repair/check process 3910 */ 3911 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 3912 for (i = 0; i < s.failed; i++) { 3913 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 3914 if (test_bit(R5_ReadError, &dev->flags) 3915 && !test_bit(R5_LOCKED, &dev->flags) 3916 && test_bit(R5_UPTODATE, &dev->flags) 3917 ) { 3918 if (!test_bit(R5_ReWrite, &dev->flags)) { 3919 set_bit(R5_Wantwrite, &dev->flags); 3920 set_bit(R5_ReWrite, &dev->flags); 3921 set_bit(R5_LOCKED, &dev->flags); 3922 s.locked++; 3923 } else { 3924 /* let's read it back */ 3925 set_bit(R5_Wantread, &dev->flags); 3926 set_bit(R5_LOCKED, &dev->flags); 3927 s.locked++; 3928 } 3929 } 3930 } 3931 3932 /* Finish reconstruct operations initiated by the expansion process */ 3933 if (sh->reconstruct_state == reconstruct_state_result) { 3934 struct stripe_head *sh_src 3935 = get_active_stripe(conf, sh->sector, 1, 1, 1); 3936 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 3937 /* sh cannot be written until sh_src has been read. 3938 * so arrange for sh to be delayed a little 3939 */ 3940 set_bit(STRIPE_DELAYED, &sh->state); 3941 set_bit(STRIPE_HANDLE, &sh->state); 3942 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 3943 &sh_src->state)) 3944 atomic_inc(&conf->preread_active_stripes); 3945 release_stripe(sh_src); 3946 goto finish; 3947 } 3948 if (sh_src) 3949 release_stripe(sh_src); 3950 3951 sh->reconstruct_state = reconstruct_state_idle; 3952 clear_bit(STRIPE_EXPANDING, &sh->state); 3953 for (i = conf->raid_disks; i--; ) { 3954 set_bit(R5_Wantwrite, &sh->dev[i].flags); 3955 set_bit(R5_LOCKED, &sh->dev[i].flags); 3956 s.locked++; 3957 } 3958 } 3959 3960 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 3961 !sh->reconstruct_state) { 3962 /* Need to write out all blocks after computing parity */ 3963 sh->disks = conf->raid_disks; 3964 stripe_set_idx(sh->sector, conf, 0, sh); 3965 schedule_reconstruction(sh, &s, 1, 1); 3966 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 3967 clear_bit(STRIPE_EXPAND_READY, &sh->state); 3968 atomic_dec(&conf->reshape_stripes); 3969 wake_up(&conf->wait_for_overlap); 3970 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 3971 } 3972 3973 if (s.expanding && s.locked == 0 && 3974 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 3975 handle_stripe_expansion(conf, sh); 3976 3977 finish: 3978 /* wait for this device to become unblocked */ 3979 if (unlikely(s.blocked_rdev)) { 3980 if (conf->mddev->external) 3981 md_wait_for_blocked_rdev(s.blocked_rdev, 3982 conf->mddev); 3983 else 3984 /* Internal metadata will immediately 3985 * be written by raid5d, so we don't 3986 * need to wait here. 3987 */ 3988 rdev_dec_pending(s.blocked_rdev, 3989 conf->mddev); 3990 } 3991 3992 if (s.handle_bad_blocks) 3993 for (i = disks; i--; ) { 3994 struct md_rdev *rdev; 3995 struct r5dev *dev = &sh->dev[i]; 3996 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 3997 /* We own a safe reference to the rdev */ 3998 rdev = conf->disks[i].rdev; 3999 if (!rdev_set_badblocks(rdev, sh->sector, 4000 STRIPE_SECTORS, 0)) 4001 md_error(conf->mddev, rdev); 4002 rdev_dec_pending(rdev, conf->mddev); 4003 } 4004 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 4005 rdev = conf->disks[i].rdev; 4006 rdev_clear_badblocks(rdev, sh->sector, 4007 STRIPE_SECTORS, 0); 4008 rdev_dec_pending(rdev, conf->mddev); 4009 } 4010 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 4011 rdev = conf->disks[i].replacement; 4012 if (!rdev) 4013 /* rdev have been moved down */ 4014 rdev = conf->disks[i].rdev; 4015 rdev_clear_badblocks(rdev, sh->sector, 4016 STRIPE_SECTORS, 0); 4017 rdev_dec_pending(rdev, conf->mddev); 4018 } 4019 } 4020 4021 if (s.ops_request) 4022 raid_run_ops(sh, s.ops_request); 4023 4024 ops_run_io(sh, &s); 4025 4026 if (s.dec_preread_active) { 4027 /* We delay this until after ops_run_io so that if make_request 4028 * is waiting on a flush, it won't continue until the writes 4029 * have actually been submitted. 4030 */ 4031 atomic_dec(&conf->preread_active_stripes); 4032 if (atomic_read(&conf->preread_active_stripes) < 4033 IO_THRESHOLD) 4034 md_wakeup_thread(conf->mddev->thread); 4035 } 4036 4037 return_io(s.return_bi); 4038 4039 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4040 } 4041 4042 static void raid5_activate_delayed(struct r5conf *conf) 4043 { 4044 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 4045 while (!list_empty(&conf->delayed_list)) { 4046 struct list_head *l = conf->delayed_list.next; 4047 struct stripe_head *sh; 4048 sh = list_entry(l, struct stripe_head, lru); 4049 list_del_init(l); 4050 clear_bit(STRIPE_DELAYED, &sh->state); 4051 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4052 atomic_inc(&conf->preread_active_stripes); 4053 list_add_tail(&sh->lru, &conf->hold_list); 4054 raid5_wakeup_stripe_thread(sh); 4055 } 4056 } 4057 } 4058 4059 static void activate_bit_delay(struct r5conf *conf, 4060 struct list_head *temp_inactive_list) 4061 { 4062 /* device_lock is held */ 4063 struct list_head head; 4064 list_add(&head, &conf->bitmap_list); 4065 list_del_init(&conf->bitmap_list); 4066 while (!list_empty(&head)) { 4067 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 4068 int hash; 4069 list_del_init(&sh->lru); 4070 atomic_inc(&sh->count); 4071 hash = sh->hash_lock_index; 4072 __release_stripe(conf, sh, &temp_inactive_list[hash]); 4073 } 4074 } 4075 4076 int md_raid5_congested(struct mddev *mddev, int bits) 4077 { 4078 struct r5conf *conf = mddev->private; 4079 4080 /* No difference between reads and writes. Just check 4081 * how busy the stripe_cache is 4082 */ 4083 4084 if (conf->inactive_blocked) 4085 return 1; 4086 if (conf->quiesce) 4087 return 1; 4088 if (atomic_read(&conf->empty_inactive_list_nr)) 4089 return 1; 4090 4091 return 0; 4092 } 4093 EXPORT_SYMBOL_GPL(md_raid5_congested); 4094 4095 static int raid5_congested(void *data, int bits) 4096 { 4097 struct mddev *mddev = data; 4098 4099 return mddev_congested(mddev, bits) || 4100 md_raid5_congested(mddev, bits); 4101 } 4102 4103 /* We want read requests to align with chunks where possible, 4104 * but write requests don't need to. 4105 */ 4106 static int raid5_mergeable_bvec(struct request_queue *q, 4107 struct bvec_merge_data *bvm, 4108 struct bio_vec *biovec) 4109 { 4110 struct mddev *mddev = q->queuedata; 4111 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); 4112 int max; 4113 unsigned int chunk_sectors = mddev->chunk_sectors; 4114 unsigned int bio_sectors = bvm->bi_size >> 9; 4115 4116 if ((bvm->bi_rw & 1) == WRITE) 4117 return biovec->bv_len; /* always allow writes to be mergeable */ 4118 4119 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4120 chunk_sectors = mddev->new_chunk_sectors; 4121 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; 4122 if (max < 0) max = 0; 4123 if (max <= biovec->bv_len && bio_sectors == 0) 4124 return biovec->bv_len; 4125 else 4126 return max; 4127 } 4128 4129 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 4130 { 4131 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); 4132 unsigned int chunk_sectors = mddev->chunk_sectors; 4133 unsigned int bio_sectors = bio_sectors(bio); 4134 4135 if (mddev->new_chunk_sectors < mddev->chunk_sectors) 4136 chunk_sectors = mddev->new_chunk_sectors; 4137 return chunk_sectors >= 4138 ((sector & (chunk_sectors - 1)) + bio_sectors); 4139 } 4140 4141 /* 4142 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 4143 * later sampled by raid5d. 4144 */ 4145 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 4146 { 4147 unsigned long flags; 4148 4149 spin_lock_irqsave(&conf->device_lock, flags); 4150 4151 bi->bi_next = conf->retry_read_aligned_list; 4152 conf->retry_read_aligned_list = bi; 4153 4154 spin_unlock_irqrestore(&conf->device_lock, flags); 4155 md_wakeup_thread(conf->mddev->thread); 4156 } 4157 4158 static struct bio *remove_bio_from_retry(struct r5conf *conf) 4159 { 4160 struct bio *bi; 4161 4162 bi = conf->retry_read_aligned; 4163 if (bi) { 4164 conf->retry_read_aligned = NULL; 4165 return bi; 4166 } 4167 bi = conf->retry_read_aligned_list; 4168 if(bi) { 4169 conf->retry_read_aligned_list = bi->bi_next; 4170 bi->bi_next = NULL; 4171 /* 4172 * this sets the active strip count to 1 and the processed 4173 * strip count to zero (upper 8 bits) 4174 */ 4175 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ 4176 } 4177 4178 return bi; 4179 } 4180 4181 /* 4182 * The "raid5_align_endio" should check if the read succeeded and if it 4183 * did, call bio_endio on the original bio (having bio_put the new bio 4184 * first). 4185 * If the read failed.. 4186 */ 4187 static void raid5_align_endio(struct bio *bi, int error) 4188 { 4189 struct bio* raid_bi = bi->bi_private; 4190 struct mddev *mddev; 4191 struct r5conf *conf; 4192 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); 4193 struct md_rdev *rdev; 4194 4195 bio_put(bi); 4196 4197 rdev = (void*)raid_bi->bi_next; 4198 raid_bi->bi_next = NULL; 4199 mddev = rdev->mddev; 4200 conf = mddev->private; 4201 4202 rdev_dec_pending(rdev, conf->mddev); 4203 4204 if (!error && uptodate) { 4205 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4206 raid_bi, 0); 4207 bio_endio(raid_bi, 0); 4208 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4209 wake_up(&conf->wait_for_stripe); 4210 return; 4211 } 4212 4213 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 4214 4215 add_bio_to_retry(raid_bi, conf); 4216 } 4217 4218 static int bio_fits_rdev(struct bio *bi) 4219 { 4220 struct request_queue *q = bdev_get_queue(bi->bi_bdev); 4221 4222 if (bio_sectors(bi) > queue_max_sectors(q)) 4223 return 0; 4224 blk_recount_segments(q, bi); 4225 if (bi->bi_phys_segments > queue_max_segments(q)) 4226 return 0; 4227 4228 if (q->merge_bvec_fn) 4229 /* it's too hard to apply the merge_bvec_fn at this stage, 4230 * just just give up 4231 */ 4232 return 0; 4233 4234 return 1; 4235 } 4236 4237 static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) 4238 { 4239 struct r5conf *conf = mddev->private; 4240 int dd_idx; 4241 struct bio* align_bi; 4242 struct md_rdev *rdev; 4243 sector_t end_sector; 4244 4245 if (!in_chunk_boundary(mddev, raid_bio)) { 4246 pr_debug("chunk_aligned_read : non aligned\n"); 4247 return 0; 4248 } 4249 /* 4250 * use bio_clone_mddev to make a copy of the bio 4251 */ 4252 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 4253 if (!align_bi) 4254 return 0; 4255 /* 4256 * set bi_end_io to a new function, and set bi_private to the 4257 * original bio. 4258 */ 4259 align_bi->bi_end_io = raid5_align_endio; 4260 align_bi->bi_private = raid_bio; 4261 /* 4262 * compute position 4263 */ 4264 align_bi->bi_iter.bi_sector = 4265 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 4266 0, &dd_idx, NULL); 4267 4268 end_sector = bio_end_sector(align_bi); 4269 rcu_read_lock(); 4270 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 4271 if (!rdev || test_bit(Faulty, &rdev->flags) || 4272 rdev->recovery_offset < end_sector) { 4273 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 4274 if (rdev && 4275 (test_bit(Faulty, &rdev->flags) || 4276 !(test_bit(In_sync, &rdev->flags) || 4277 rdev->recovery_offset >= end_sector))) 4278 rdev = NULL; 4279 } 4280 if (rdev) { 4281 sector_t first_bad; 4282 int bad_sectors; 4283 4284 atomic_inc(&rdev->nr_pending); 4285 rcu_read_unlock(); 4286 raid_bio->bi_next = (void*)rdev; 4287 align_bi->bi_bdev = rdev->bdev; 4288 __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); 4289 4290 if (!bio_fits_rdev(align_bi) || 4291 is_badblock(rdev, align_bi->bi_iter.bi_sector, 4292 bio_sectors(align_bi), 4293 &first_bad, &bad_sectors)) { 4294 /* too big in some way, or has a known bad block */ 4295 bio_put(align_bi); 4296 rdev_dec_pending(rdev, mddev); 4297 return 0; 4298 } 4299 4300 /* No reshape active, so we can trust rdev->data_offset */ 4301 align_bi->bi_iter.bi_sector += rdev->data_offset; 4302 4303 spin_lock_irq(&conf->device_lock); 4304 wait_event_lock_irq(conf->wait_for_stripe, 4305 conf->quiesce == 0, 4306 conf->device_lock); 4307 atomic_inc(&conf->active_aligned_reads); 4308 spin_unlock_irq(&conf->device_lock); 4309 4310 if (mddev->gendisk) 4311 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), 4312 align_bi, disk_devt(mddev->gendisk), 4313 raid_bio->bi_iter.bi_sector); 4314 generic_make_request(align_bi); 4315 return 1; 4316 } else { 4317 rcu_read_unlock(); 4318 bio_put(align_bi); 4319 return 0; 4320 } 4321 } 4322 4323 /* __get_priority_stripe - get the next stripe to process 4324 * 4325 * Full stripe writes are allowed to pass preread active stripes up until 4326 * the bypass_threshold is exceeded. In general the bypass_count 4327 * increments when the handle_list is handled before the hold_list; however, it 4328 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 4329 * stripe with in flight i/o. The bypass_count will be reset when the 4330 * head of the hold_list has changed, i.e. the head was promoted to the 4331 * handle_list. 4332 */ 4333 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) 4334 { 4335 struct stripe_head *sh = NULL, *tmp; 4336 struct list_head *handle_list = NULL; 4337 struct r5worker_group *wg = NULL; 4338 4339 if (conf->worker_cnt_per_group == 0) { 4340 handle_list = &conf->handle_list; 4341 } else if (group != ANY_GROUP) { 4342 handle_list = &conf->worker_groups[group].handle_list; 4343 wg = &conf->worker_groups[group]; 4344 } else { 4345 int i; 4346 for (i = 0; i < conf->group_cnt; i++) { 4347 handle_list = &conf->worker_groups[i].handle_list; 4348 wg = &conf->worker_groups[i]; 4349 if (!list_empty(handle_list)) 4350 break; 4351 } 4352 } 4353 4354 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 4355 __func__, 4356 list_empty(handle_list) ? "empty" : "busy", 4357 list_empty(&conf->hold_list) ? "empty" : "busy", 4358 atomic_read(&conf->pending_full_writes), conf->bypass_count); 4359 4360 if (!list_empty(handle_list)) { 4361 sh = list_entry(handle_list->next, typeof(*sh), lru); 4362 4363 if (list_empty(&conf->hold_list)) 4364 conf->bypass_count = 0; 4365 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 4366 if (conf->hold_list.next == conf->last_hold) 4367 conf->bypass_count++; 4368 else { 4369 conf->last_hold = conf->hold_list.next; 4370 conf->bypass_count -= conf->bypass_threshold; 4371 if (conf->bypass_count < 0) 4372 conf->bypass_count = 0; 4373 } 4374 } 4375 } else if (!list_empty(&conf->hold_list) && 4376 ((conf->bypass_threshold && 4377 conf->bypass_count > conf->bypass_threshold) || 4378 atomic_read(&conf->pending_full_writes) == 0)) { 4379 4380 list_for_each_entry(tmp, &conf->hold_list, lru) { 4381 if (conf->worker_cnt_per_group == 0 || 4382 group == ANY_GROUP || 4383 !cpu_online(tmp->cpu) || 4384 cpu_to_group(tmp->cpu) == group) { 4385 sh = tmp; 4386 break; 4387 } 4388 } 4389 4390 if (sh) { 4391 conf->bypass_count -= conf->bypass_threshold; 4392 if (conf->bypass_count < 0) 4393 conf->bypass_count = 0; 4394 } 4395 wg = NULL; 4396 } 4397 4398 if (!sh) 4399 return NULL; 4400 4401 if (wg) { 4402 wg->stripes_cnt--; 4403 sh->group = NULL; 4404 } 4405 list_del_init(&sh->lru); 4406 BUG_ON(atomic_inc_return(&sh->count) != 1); 4407 return sh; 4408 } 4409 4410 struct raid5_plug_cb { 4411 struct blk_plug_cb cb; 4412 struct list_head list; 4413 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; 4414 }; 4415 4416 static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) 4417 { 4418 struct raid5_plug_cb *cb = container_of( 4419 blk_cb, struct raid5_plug_cb, cb); 4420 struct stripe_head *sh; 4421 struct mddev *mddev = cb->cb.data; 4422 struct r5conf *conf = mddev->private; 4423 int cnt = 0; 4424 int hash; 4425 4426 if (cb->list.next && !list_empty(&cb->list)) { 4427 spin_lock_irq(&conf->device_lock); 4428 while (!list_empty(&cb->list)) { 4429 sh = list_first_entry(&cb->list, struct stripe_head, lru); 4430 list_del_init(&sh->lru); 4431 /* 4432 * avoid race release_stripe_plug() sees 4433 * STRIPE_ON_UNPLUG_LIST clear but the stripe 4434 * is still in our list 4435 */ 4436 smp_mb__before_atomic(); 4437 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); 4438 /* 4439 * STRIPE_ON_RELEASE_LIST could be set here. In that 4440 * case, the count is always > 1 here 4441 */ 4442 hash = sh->hash_lock_index; 4443 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); 4444 cnt++; 4445 } 4446 spin_unlock_irq(&conf->device_lock); 4447 } 4448 release_inactive_stripe_list(conf, cb->temp_inactive_list, 4449 NR_STRIPE_HASH_LOCKS); 4450 if (mddev->queue) 4451 trace_block_unplug(mddev->queue, cnt, !from_schedule); 4452 kfree(cb); 4453 } 4454 4455 static void release_stripe_plug(struct mddev *mddev, 4456 struct stripe_head *sh) 4457 { 4458 struct blk_plug_cb *blk_cb = blk_check_plugged( 4459 raid5_unplug, mddev, 4460 sizeof(struct raid5_plug_cb)); 4461 struct raid5_plug_cb *cb; 4462 4463 if (!blk_cb) { 4464 release_stripe(sh); 4465 return; 4466 } 4467 4468 cb = container_of(blk_cb, struct raid5_plug_cb, cb); 4469 4470 if (cb->list.next == NULL) { 4471 int i; 4472 INIT_LIST_HEAD(&cb->list); 4473 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 4474 INIT_LIST_HEAD(cb->temp_inactive_list + i); 4475 } 4476 4477 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) 4478 list_add_tail(&sh->lru, &cb->list); 4479 else 4480 release_stripe(sh); 4481 } 4482 4483 static void make_discard_request(struct mddev *mddev, struct bio *bi) 4484 { 4485 struct r5conf *conf = mddev->private; 4486 sector_t logical_sector, last_sector; 4487 struct stripe_head *sh; 4488 int remaining; 4489 int stripe_sectors; 4490 4491 if (mddev->reshape_position != MaxSector) 4492 /* Skip discard while reshape is happening */ 4493 return; 4494 4495 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4496 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); 4497 4498 bi->bi_next = NULL; 4499 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4500 4501 stripe_sectors = conf->chunk_sectors * 4502 (conf->raid_disks - conf->max_degraded); 4503 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, 4504 stripe_sectors); 4505 sector_div(last_sector, stripe_sectors); 4506 4507 logical_sector *= conf->chunk_sectors; 4508 last_sector *= conf->chunk_sectors; 4509 4510 for (; logical_sector < last_sector; 4511 logical_sector += STRIPE_SECTORS) { 4512 DEFINE_WAIT(w); 4513 int d; 4514 again: 4515 sh = get_active_stripe(conf, logical_sector, 0, 0, 0); 4516 prepare_to_wait(&conf->wait_for_overlap, &w, 4517 TASK_UNINTERRUPTIBLE); 4518 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4519 if (test_bit(STRIPE_SYNCING, &sh->state)) { 4520 release_stripe(sh); 4521 schedule(); 4522 goto again; 4523 } 4524 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 4525 spin_lock_irq(&sh->stripe_lock); 4526 for (d = 0; d < conf->raid_disks; d++) { 4527 if (d == sh->pd_idx || d == sh->qd_idx) 4528 continue; 4529 if (sh->dev[d].towrite || sh->dev[d].toread) { 4530 set_bit(R5_Overlap, &sh->dev[d].flags); 4531 spin_unlock_irq(&sh->stripe_lock); 4532 release_stripe(sh); 4533 schedule(); 4534 goto again; 4535 } 4536 } 4537 set_bit(STRIPE_DISCARD, &sh->state); 4538 finish_wait(&conf->wait_for_overlap, &w); 4539 for (d = 0; d < conf->raid_disks; d++) { 4540 if (d == sh->pd_idx || d == sh->qd_idx) 4541 continue; 4542 sh->dev[d].towrite = bi; 4543 set_bit(R5_OVERWRITE, &sh->dev[d].flags); 4544 raid5_inc_bi_active_stripes(bi); 4545 } 4546 spin_unlock_irq(&sh->stripe_lock); 4547 if (conf->mddev->bitmap) { 4548 for (d = 0; 4549 d < conf->raid_disks - conf->max_degraded; 4550 d++) 4551 bitmap_startwrite(mddev->bitmap, 4552 sh->sector, 4553 STRIPE_SECTORS, 4554 0); 4555 sh->bm_seq = conf->seq_flush + 1; 4556 set_bit(STRIPE_BIT_DELAY, &sh->state); 4557 } 4558 4559 set_bit(STRIPE_HANDLE, &sh->state); 4560 clear_bit(STRIPE_DELAYED, &sh->state); 4561 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4562 atomic_inc(&conf->preread_active_stripes); 4563 release_stripe_plug(mddev, sh); 4564 } 4565 4566 remaining = raid5_dec_bi_active_stripes(bi); 4567 if (remaining == 0) { 4568 md_write_end(mddev); 4569 bio_endio(bi, 0); 4570 } 4571 } 4572 4573 static void make_request(struct mddev *mddev, struct bio * bi) 4574 { 4575 struct r5conf *conf = mddev->private; 4576 int dd_idx; 4577 sector_t new_sector; 4578 sector_t logical_sector, last_sector; 4579 struct stripe_head *sh; 4580 const int rw = bio_data_dir(bi); 4581 int remaining; 4582 DEFINE_WAIT(w); 4583 bool do_prepare; 4584 4585 if (unlikely(bi->bi_rw & REQ_FLUSH)) { 4586 md_flush_request(mddev, bi); 4587 return; 4588 } 4589 4590 md_write_start(mddev, bi); 4591 4592 if (rw == READ && 4593 mddev->reshape_position == MaxSector && 4594 chunk_aligned_read(mddev,bi)) 4595 return; 4596 4597 if (unlikely(bi->bi_rw & REQ_DISCARD)) { 4598 make_discard_request(mddev, bi); 4599 return; 4600 } 4601 4602 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 4603 last_sector = bio_end_sector(bi); 4604 bi->bi_next = NULL; 4605 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 4606 4607 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 4608 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 4609 int previous; 4610 int seq; 4611 4612 do_prepare = false; 4613 retry: 4614 seq = read_seqcount_begin(&conf->gen_lock); 4615 previous = 0; 4616 if (do_prepare) 4617 prepare_to_wait(&conf->wait_for_overlap, &w, 4618 TASK_UNINTERRUPTIBLE); 4619 if (unlikely(conf->reshape_progress != MaxSector)) { 4620 /* spinlock is needed as reshape_progress may be 4621 * 64bit on a 32bit platform, and so it might be 4622 * possible to see a half-updated value 4623 * Of course reshape_progress could change after 4624 * the lock is dropped, so once we get a reference 4625 * to the stripe that we think it is, we will have 4626 * to check again. 4627 */ 4628 spin_lock_irq(&conf->device_lock); 4629 if (mddev->reshape_backwards 4630 ? logical_sector < conf->reshape_progress 4631 : logical_sector >= conf->reshape_progress) { 4632 previous = 1; 4633 } else { 4634 if (mddev->reshape_backwards 4635 ? logical_sector < conf->reshape_safe 4636 : logical_sector >= conf->reshape_safe) { 4637 spin_unlock_irq(&conf->device_lock); 4638 schedule(); 4639 do_prepare = true; 4640 goto retry; 4641 } 4642 } 4643 spin_unlock_irq(&conf->device_lock); 4644 } 4645 4646 new_sector = raid5_compute_sector(conf, logical_sector, 4647 previous, 4648 &dd_idx, NULL); 4649 pr_debug("raid456: make_request, sector %llu logical %llu\n", 4650 (unsigned long long)new_sector, 4651 (unsigned long long)logical_sector); 4652 4653 sh = get_active_stripe(conf, new_sector, previous, 4654 (bi->bi_rw&RWA_MASK), 0); 4655 if (sh) { 4656 if (unlikely(previous)) { 4657 /* expansion might have moved on while waiting for a 4658 * stripe, so we must do the range check again. 4659 * Expansion could still move past after this 4660 * test, but as we are holding a reference to 4661 * 'sh', we know that if that happens, 4662 * STRIPE_EXPANDING will get set and the expansion 4663 * won't proceed until we finish with the stripe. 4664 */ 4665 int must_retry = 0; 4666 spin_lock_irq(&conf->device_lock); 4667 if (mddev->reshape_backwards 4668 ? logical_sector >= conf->reshape_progress 4669 : logical_sector < conf->reshape_progress) 4670 /* mismatch, need to try again */ 4671 must_retry = 1; 4672 spin_unlock_irq(&conf->device_lock); 4673 if (must_retry) { 4674 release_stripe(sh); 4675 schedule(); 4676 do_prepare = true; 4677 goto retry; 4678 } 4679 } 4680 if (read_seqcount_retry(&conf->gen_lock, seq)) { 4681 /* Might have got the wrong stripe_head 4682 * by accident 4683 */ 4684 release_stripe(sh); 4685 goto retry; 4686 } 4687 4688 if (rw == WRITE && 4689 logical_sector >= mddev->suspend_lo && 4690 logical_sector < mddev->suspend_hi) { 4691 release_stripe(sh); 4692 /* As the suspend_* range is controlled by 4693 * userspace, we want an interruptible 4694 * wait. 4695 */ 4696 flush_signals(current); 4697 prepare_to_wait(&conf->wait_for_overlap, 4698 &w, TASK_INTERRUPTIBLE); 4699 if (logical_sector >= mddev->suspend_lo && 4700 logical_sector < mddev->suspend_hi) { 4701 schedule(); 4702 do_prepare = true; 4703 } 4704 goto retry; 4705 } 4706 4707 if (test_bit(STRIPE_EXPANDING, &sh->state) || 4708 !add_stripe_bio(sh, bi, dd_idx, rw)) { 4709 /* Stripe is busy expanding or 4710 * add failed due to overlap. Flush everything 4711 * and wait a while 4712 */ 4713 md_wakeup_thread(mddev->thread); 4714 release_stripe(sh); 4715 schedule(); 4716 do_prepare = true; 4717 goto retry; 4718 } 4719 set_bit(STRIPE_HANDLE, &sh->state); 4720 clear_bit(STRIPE_DELAYED, &sh->state); 4721 if ((bi->bi_rw & REQ_SYNC) && 4722 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4723 atomic_inc(&conf->preread_active_stripes); 4724 release_stripe_plug(mddev, sh); 4725 } else { 4726 /* cannot get stripe for read-ahead, just give-up */ 4727 clear_bit(BIO_UPTODATE, &bi->bi_flags); 4728 break; 4729 } 4730 } 4731 finish_wait(&conf->wait_for_overlap, &w); 4732 4733 remaining = raid5_dec_bi_active_stripes(bi); 4734 if (remaining == 0) { 4735 4736 if ( rw == WRITE ) 4737 md_write_end(mddev); 4738 4739 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 4740 bi, 0); 4741 bio_endio(bi, 0); 4742 } 4743 } 4744 4745 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 4746 4747 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 4748 { 4749 /* reshaping is quite different to recovery/resync so it is 4750 * handled quite separately ... here. 4751 * 4752 * On each call to sync_request, we gather one chunk worth of 4753 * destination stripes and flag them as expanding. 4754 * Then we find all the source stripes and request reads. 4755 * As the reads complete, handle_stripe will copy the data 4756 * into the destination stripe and release that stripe. 4757 */ 4758 struct r5conf *conf = mddev->private; 4759 struct stripe_head *sh; 4760 sector_t first_sector, last_sector; 4761 int raid_disks = conf->previous_raid_disks; 4762 int data_disks = raid_disks - conf->max_degraded; 4763 int new_data_disks = conf->raid_disks - conf->max_degraded; 4764 int i; 4765 int dd_idx; 4766 sector_t writepos, readpos, safepos; 4767 sector_t stripe_addr; 4768 int reshape_sectors; 4769 struct list_head stripes; 4770 4771 if (sector_nr == 0) { 4772 /* If restarting in the middle, skip the initial sectors */ 4773 if (mddev->reshape_backwards && 4774 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 4775 sector_nr = raid5_size(mddev, 0, 0) 4776 - conf->reshape_progress; 4777 } else if (!mddev->reshape_backwards && 4778 conf->reshape_progress > 0) 4779 sector_nr = conf->reshape_progress; 4780 sector_div(sector_nr, new_data_disks); 4781 if (sector_nr) { 4782 mddev->curr_resync_completed = sector_nr; 4783 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4784 *skipped = 1; 4785 return sector_nr; 4786 } 4787 } 4788 4789 /* We need to process a full chunk at a time. 4790 * If old and new chunk sizes differ, we need to process the 4791 * largest of these 4792 */ 4793 if (mddev->new_chunk_sectors > mddev->chunk_sectors) 4794 reshape_sectors = mddev->new_chunk_sectors; 4795 else 4796 reshape_sectors = mddev->chunk_sectors; 4797 4798 /* We update the metadata at least every 10 seconds, or when 4799 * the data about to be copied would over-write the source of 4800 * the data at the front of the range. i.e. one new_stripe 4801 * along from reshape_progress new_maps to after where 4802 * reshape_safe old_maps to 4803 */ 4804 writepos = conf->reshape_progress; 4805 sector_div(writepos, new_data_disks); 4806 readpos = conf->reshape_progress; 4807 sector_div(readpos, data_disks); 4808 safepos = conf->reshape_safe; 4809 sector_div(safepos, data_disks); 4810 if (mddev->reshape_backwards) { 4811 writepos -= min_t(sector_t, reshape_sectors, writepos); 4812 readpos += reshape_sectors; 4813 safepos += reshape_sectors; 4814 } else { 4815 writepos += reshape_sectors; 4816 readpos -= min_t(sector_t, reshape_sectors, readpos); 4817 safepos -= min_t(sector_t, reshape_sectors, safepos); 4818 } 4819 4820 /* Having calculated the 'writepos' possibly use it 4821 * to set 'stripe_addr' which is where we will write to. 4822 */ 4823 if (mddev->reshape_backwards) { 4824 BUG_ON(conf->reshape_progress == 0); 4825 stripe_addr = writepos; 4826 BUG_ON((mddev->dev_sectors & 4827 ~((sector_t)reshape_sectors - 1)) 4828 - reshape_sectors - stripe_addr 4829 != sector_nr); 4830 } else { 4831 BUG_ON(writepos != sector_nr + reshape_sectors); 4832 stripe_addr = sector_nr; 4833 } 4834 4835 /* 'writepos' is the most advanced device address we might write. 4836 * 'readpos' is the least advanced device address we might read. 4837 * 'safepos' is the least address recorded in the metadata as having 4838 * been reshaped. 4839 * If there is a min_offset_diff, these are adjusted either by 4840 * increasing the safepos/readpos if diff is negative, or 4841 * increasing writepos if diff is positive. 4842 * If 'readpos' is then behind 'writepos', there is no way that we can 4843 * ensure safety in the face of a crash - that must be done by userspace 4844 * making a backup of the data. So in that case there is no particular 4845 * rush to update metadata. 4846 * Otherwise if 'safepos' is behind 'writepos', then we really need to 4847 * update the metadata to advance 'safepos' to match 'readpos' so that 4848 * we can be safe in the event of a crash. 4849 * So we insist on updating metadata if safepos is behind writepos and 4850 * readpos is beyond writepos. 4851 * In any case, update the metadata every 10 seconds. 4852 * Maybe that number should be configurable, but I'm not sure it is 4853 * worth it.... maybe it could be a multiple of safemode_delay??? 4854 */ 4855 if (conf->min_offset_diff < 0) { 4856 safepos += -conf->min_offset_diff; 4857 readpos += -conf->min_offset_diff; 4858 } else 4859 writepos += conf->min_offset_diff; 4860 4861 if ((mddev->reshape_backwards 4862 ? (safepos > writepos && readpos < writepos) 4863 : (safepos < writepos && readpos > writepos)) || 4864 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 4865 /* Cannot proceed until we've updated the superblock... */ 4866 wait_event(conf->wait_for_overlap, 4867 atomic_read(&conf->reshape_stripes)==0 4868 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4869 if (atomic_read(&conf->reshape_stripes) != 0) 4870 return 0; 4871 mddev->reshape_position = conf->reshape_progress; 4872 mddev->curr_resync_completed = sector_nr; 4873 conf->reshape_checkpoint = jiffies; 4874 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4875 md_wakeup_thread(mddev->thread); 4876 wait_event(mddev->sb_wait, mddev->flags == 0 || 4877 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4878 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4879 return 0; 4880 spin_lock_irq(&conf->device_lock); 4881 conf->reshape_safe = mddev->reshape_position; 4882 spin_unlock_irq(&conf->device_lock); 4883 wake_up(&conf->wait_for_overlap); 4884 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4885 } 4886 4887 INIT_LIST_HEAD(&stripes); 4888 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 4889 int j; 4890 int skipped_disk = 0; 4891 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 4892 set_bit(STRIPE_EXPANDING, &sh->state); 4893 atomic_inc(&conf->reshape_stripes); 4894 /* If any of this stripe is beyond the end of the old 4895 * array, then we need to zero those blocks 4896 */ 4897 for (j=sh->disks; j--;) { 4898 sector_t s; 4899 if (j == sh->pd_idx) 4900 continue; 4901 if (conf->level == 6 && 4902 j == sh->qd_idx) 4903 continue; 4904 s = compute_blocknr(sh, j, 0); 4905 if (s < raid5_size(mddev, 0, 0)) { 4906 skipped_disk = 1; 4907 continue; 4908 } 4909 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 4910 set_bit(R5_Expanded, &sh->dev[j].flags); 4911 set_bit(R5_UPTODATE, &sh->dev[j].flags); 4912 } 4913 if (!skipped_disk) { 4914 set_bit(STRIPE_EXPAND_READY, &sh->state); 4915 set_bit(STRIPE_HANDLE, &sh->state); 4916 } 4917 list_add(&sh->lru, &stripes); 4918 } 4919 spin_lock_irq(&conf->device_lock); 4920 if (mddev->reshape_backwards) 4921 conf->reshape_progress -= reshape_sectors * new_data_disks; 4922 else 4923 conf->reshape_progress += reshape_sectors * new_data_disks; 4924 spin_unlock_irq(&conf->device_lock); 4925 /* Ok, those stripe are ready. We can start scheduling 4926 * reads on the source stripes. 4927 * The source stripes are determined by mapping the first and last 4928 * block on the destination stripes. 4929 */ 4930 first_sector = 4931 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 4932 1, &dd_idx, NULL); 4933 last_sector = 4934 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 4935 * new_data_disks - 1), 4936 1, &dd_idx, NULL); 4937 if (last_sector >= mddev->dev_sectors) 4938 last_sector = mddev->dev_sectors - 1; 4939 while (first_sector <= last_sector) { 4940 sh = get_active_stripe(conf, first_sector, 1, 0, 1); 4941 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4942 set_bit(STRIPE_HANDLE, &sh->state); 4943 release_stripe(sh); 4944 first_sector += STRIPE_SECTORS; 4945 } 4946 /* Now that the sources are clearly marked, we can release 4947 * the destination stripes 4948 */ 4949 while (!list_empty(&stripes)) { 4950 sh = list_entry(stripes.next, struct stripe_head, lru); 4951 list_del_init(&sh->lru); 4952 release_stripe(sh); 4953 } 4954 /* If this takes us to the resync_max point where we have to pause, 4955 * then we need to write out the superblock. 4956 */ 4957 sector_nr += reshape_sectors; 4958 if ((sector_nr - mddev->curr_resync_completed) * 2 4959 >= mddev->resync_max - mddev->curr_resync_completed) { 4960 /* Cannot proceed until we've updated the superblock... */ 4961 wait_event(conf->wait_for_overlap, 4962 atomic_read(&conf->reshape_stripes) == 0 4963 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4964 if (atomic_read(&conf->reshape_stripes) != 0) 4965 goto ret; 4966 mddev->reshape_position = conf->reshape_progress; 4967 mddev->curr_resync_completed = sector_nr; 4968 conf->reshape_checkpoint = jiffies; 4969 set_bit(MD_CHANGE_DEVS, &mddev->flags); 4970 md_wakeup_thread(mddev->thread); 4971 wait_event(mddev->sb_wait, 4972 !test_bit(MD_CHANGE_DEVS, &mddev->flags) 4973 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 4974 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 4975 goto ret; 4976 spin_lock_irq(&conf->device_lock); 4977 conf->reshape_safe = mddev->reshape_position; 4978 spin_unlock_irq(&conf->device_lock); 4979 wake_up(&conf->wait_for_overlap); 4980 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 4981 } 4982 ret: 4983 return reshape_sectors; 4984 } 4985 4986 /* FIXME go_faster isn't used */ 4987 static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster) 4988 { 4989 struct r5conf *conf = mddev->private; 4990 struct stripe_head *sh; 4991 sector_t max_sector = mddev->dev_sectors; 4992 sector_t sync_blocks; 4993 int still_degraded = 0; 4994 int i; 4995 4996 if (sector_nr >= max_sector) { 4997 /* just being told to finish up .. nothing much to do */ 4998 4999 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 5000 end_reshape(conf); 5001 return 0; 5002 } 5003 5004 if (mddev->curr_resync < max_sector) /* aborted */ 5005 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 5006 &sync_blocks, 1); 5007 else /* completed sync */ 5008 conf->fullsync = 0; 5009 bitmap_close_sync(mddev->bitmap); 5010 5011 return 0; 5012 } 5013 5014 /* Allow raid5_quiesce to complete */ 5015 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 5016 5017 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 5018 return reshape_request(mddev, sector_nr, skipped); 5019 5020 /* No need to check resync_max as we never do more than one 5021 * stripe, and as resync_max will always be on a chunk boundary, 5022 * if the check in md_do_sync didn't fire, there is no chance 5023 * of overstepping resync_max here 5024 */ 5025 5026 /* if there is too many failed drives and we are trying 5027 * to resync, then assert that we are finished, because there is 5028 * nothing we can do. 5029 */ 5030 if (mddev->degraded >= conf->max_degraded && 5031 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 5032 sector_t rv = mddev->dev_sectors - sector_nr; 5033 *skipped = 1; 5034 return rv; 5035 } 5036 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 5037 !conf->fullsync && 5038 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 5039 sync_blocks >= STRIPE_SECTORS) { 5040 /* we can skip this block, and probably more */ 5041 sync_blocks /= STRIPE_SECTORS; 5042 *skipped = 1; 5043 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 5044 } 5045 5046 bitmap_cond_end_sync(mddev->bitmap, sector_nr); 5047 5048 sh = get_active_stripe(conf, sector_nr, 0, 1, 0); 5049 if (sh == NULL) { 5050 sh = get_active_stripe(conf, sector_nr, 0, 0, 0); 5051 /* make sure we don't swamp the stripe cache if someone else 5052 * is trying to get access 5053 */ 5054 schedule_timeout_uninterruptible(1); 5055 } 5056 /* Need to check if array will still be degraded after recovery/resync 5057 * We don't need to check the 'failed' flag as when that gets set, 5058 * recovery aborts. 5059 */ 5060 for (i = 0; i < conf->raid_disks; i++) 5061 if (conf->disks[i].rdev == NULL) 5062 still_degraded = 1; 5063 5064 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 5065 5066 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 5067 set_bit(STRIPE_HANDLE, &sh->state); 5068 5069 release_stripe(sh); 5070 5071 return STRIPE_SECTORS; 5072 } 5073 5074 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 5075 { 5076 /* We may not be able to submit a whole bio at once as there 5077 * may not be enough stripe_heads available. 5078 * We cannot pre-allocate enough stripe_heads as we may need 5079 * more than exist in the cache (if we allow ever large chunks). 5080 * So we do one stripe head at a time and record in 5081 * ->bi_hw_segments how many have been done. 5082 * 5083 * We *know* that this entire raid_bio is in one chunk, so 5084 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 5085 */ 5086 struct stripe_head *sh; 5087 int dd_idx; 5088 sector_t sector, logical_sector, last_sector; 5089 int scnt = 0; 5090 int remaining; 5091 int handled = 0; 5092 5093 logical_sector = raid_bio->bi_iter.bi_sector & 5094 ~((sector_t)STRIPE_SECTORS-1); 5095 sector = raid5_compute_sector(conf, logical_sector, 5096 0, &dd_idx, NULL); 5097 last_sector = bio_end_sector(raid_bio); 5098 5099 for (; logical_sector < last_sector; 5100 logical_sector += STRIPE_SECTORS, 5101 sector += STRIPE_SECTORS, 5102 scnt++) { 5103 5104 if (scnt < raid5_bi_processed_stripes(raid_bio)) 5105 /* already done this stripe */ 5106 continue; 5107 5108 sh = get_active_stripe(conf, sector, 0, 1, 1); 5109 5110 if (!sh) { 5111 /* failed to get a stripe - must wait */ 5112 raid5_set_bi_processed_stripes(raid_bio, scnt); 5113 conf->retry_read_aligned = raid_bio; 5114 return handled; 5115 } 5116 5117 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) { 5118 release_stripe(sh); 5119 raid5_set_bi_processed_stripes(raid_bio, scnt); 5120 conf->retry_read_aligned = raid_bio; 5121 return handled; 5122 } 5123 5124 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); 5125 handle_stripe(sh); 5126 release_stripe(sh); 5127 handled++; 5128 } 5129 remaining = raid5_dec_bi_active_stripes(raid_bio); 5130 if (remaining == 0) { 5131 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 5132 raid_bio, 0); 5133 bio_endio(raid_bio, 0); 5134 } 5135 if (atomic_dec_and_test(&conf->active_aligned_reads)) 5136 wake_up(&conf->wait_for_stripe); 5137 return handled; 5138 } 5139 5140 static int handle_active_stripes(struct r5conf *conf, int group, 5141 struct r5worker *worker, 5142 struct list_head *temp_inactive_list) 5143 { 5144 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; 5145 int i, batch_size = 0, hash; 5146 bool release_inactive = false; 5147 5148 while (batch_size < MAX_STRIPE_BATCH && 5149 (sh = __get_priority_stripe(conf, group)) != NULL) 5150 batch[batch_size++] = sh; 5151 5152 if (batch_size == 0) { 5153 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5154 if (!list_empty(temp_inactive_list + i)) 5155 break; 5156 if (i == NR_STRIPE_HASH_LOCKS) 5157 return batch_size; 5158 release_inactive = true; 5159 } 5160 spin_unlock_irq(&conf->device_lock); 5161 5162 release_inactive_stripe_list(conf, temp_inactive_list, 5163 NR_STRIPE_HASH_LOCKS); 5164 5165 if (release_inactive) { 5166 spin_lock_irq(&conf->device_lock); 5167 return 0; 5168 } 5169 5170 for (i = 0; i < batch_size; i++) 5171 handle_stripe(batch[i]); 5172 5173 cond_resched(); 5174 5175 spin_lock_irq(&conf->device_lock); 5176 for (i = 0; i < batch_size; i++) { 5177 hash = batch[i]->hash_lock_index; 5178 __release_stripe(conf, batch[i], &temp_inactive_list[hash]); 5179 } 5180 return batch_size; 5181 } 5182 5183 static void raid5_do_work(struct work_struct *work) 5184 { 5185 struct r5worker *worker = container_of(work, struct r5worker, work); 5186 struct r5worker_group *group = worker->group; 5187 struct r5conf *conf = group->conf; 5188 int group_id = group - conf->worker_groups; 5189 int handled; 5190 struct blk_plug plug; 5191 5192 pr_debug("+++ raid5worker active\n"); 5193 5194 blk_start_plug(&plug); 5195 handled = 0; 5196 spin_lock_irq(&conf->device_lock); 5197 while (1) { 5198 int batch_size, released; 5199 5200 released = release_stripe_list(conf, worker->temp_inactive_list); 5201 5202 batch_size = handle_active_stripes(conf, group_id, worker, 5203 worker->temp_inactive_list); 5204 worker->working = false; 5205 if (!batch_size && !released) 5206 break; 5207 handled += batch_size; 5208 } 5209 pr_debug("%d stripes handled\n", handled); 5210 5211 spin_unlock_irq(&conf->device_lock); 5212 blk_finish_plug(&plug); 5213 5214 pr_debug("--- raid5worker inactive\n"); 5215 } 5216 5217 /* 5218 * This is our raid5 kernel thread. 5219 * 5220 * We scan the hash table for stripes which can be handled now. 5221 * During the scan, completed stripes are saved for us by the interrupt 5222 * handler, so that they will not have to wait for our next wakeup. 5223 */ 5224 static void raid5d(struct md_thread *thread) 5225 { 5226 struct mddev *mddev = thread->mddev; 5227 struct r5conf *conf = mddev->private; 5228 int handled; 5229 struct blk_plug plug; 5230 5231 pr_debug("+++ raid5d active\n"); 5232 5233 md_check_recovery(mddev); 5234 5235 blk_start_plug(&plug); 5236 handled = 0; 5237 spin_lock_irq(&conf->device_lock); 5238 while (1) { 5239 struct bio *bio; 5240 int batch_size, released; 5241 5242 released = release_stripe_list(conf, conf->temp_inactive_list); 5243 5244 if ( 5245 !list_empty(&conf->bitmap_list)) { 5246 /* Now is a good time to flush some bitmap updates */ 5247 conf->seq_flush++; 5248 spin_unlock_irq(&conf->device_lock); 5249 bitmap_unplug(mddev->bitmap); 5250 spin_lock_irq(&conf->device_lock); 5251 conf->seq_write = conf->seq_flush; 5252 activate_bit_delay(conf, conf->temp_inactive_list); 5253 } 5254 raid5_activate_delayed(conf); 5255 5256 while ((bio = remove_bio_from_retry(conf))) { 5257 int ok; 5258 spin_unlock_irq(&conf->device_lock); 5259 ok = retry_aligned_read(conf, bio); 5260 spin_lock_irq(&conf->device_lock); 5261 if (!ok) 5262 break; 5263 handled++; 5264 } 5265 5266 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, 5267 conf->temp_inactive_list); 5268 if (!batch_size && !released) 5269 break; 5270 handled += batch_size; 5271 5272 if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) { 5273 spin_unlock_irq(&conf->device_lock); 5274 md_check_recovery(mddev); 5275 spin_lock_irq(&conf->device_lock); 5276 } 5277 } 5278 pr_debug("%d stripes handled\n", handled); 5279 5280 spin_unlock_irq(&conf->device_lock); 5281 5282 async_tx_issue_pending_all(); 5283 blk_finish_plug(&plug); 5284 5285 pr_debug("--- raid5d inactive\n"); 5286 } 5287 5288 static ssize_t 5289 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 5290 { 5291 struct r5conf *conf = mddev->private; 5292 if (conf) 5293 return sprintf(page, "%d\n", conf->max_nr_stripes); 5294 else 5295 return 0; 5296 } 5297 5298 int 5299 raid5_set_cache_size(struct mddev *mddev, int size) 5300 { 5301 struct r5conf *conf = mddev->private; 5302 int err; 5303 int hash; 5304 5305 if (size <= 16 || size > 32768) 5306 return -EINVAL; 5307 hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS; 5308 while (size < conf->max_nr_stripes) { 5309 if (drop_one_stripe(conf, hash)) 5310 conf->max_nr_stripes--; 5311 else 5312 break; 5313 hash--; 5314 if (hash < 0) 5315 hash = NR_STRIPE_HASH_LOCKS - 1; 5316 } 5317 err = md_allow_write(mddev); 5318 if (err) 5319 return err; 5320 hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 5321 while (size > conf->max_nr_stripes) { 5322 if (grow_one_stripe(conf, hash)) 5323 conf->max_nr_stripes++; 5324 else break; 5325 hash = (hash + 1) % NR_STRIPE_HASH_LOCKS; 5326 } 5327 return 0; 5328 } 5329 EXPORT_SYMBOL(raid5_set_cache_size); 5330 5331 static ssize_t 5332 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 5333 { 5334 struct r5conf *conf = mddev->private; 5335 unsigned long new; 5336 int err; 5337 5338 if (len >= PAGE_SIZE) 5339 return -EINVAL; 5340 if (!conf) 5341 return -ENODEV; 5342 5343 if (kstrtoul(page, 10, &new)) 5344 return -EINVAL; 5345 err = raid5_set_cache_size(mddev, new); 5346 if (err) 5347 return err; 5348 return len; 5349 } 5350 5351 static struct md_sysfs_entry 5352 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 5353 raid5_show_stripe_cache_size, 5354 raid5_store_stripe_cache_size); 5355 5356 static ssize_t 5357 raid5_show_preread_threshold(struct mddev *mddev, char *page) 5358 { 5359 struct r5conf *conf = mddev->private; 5360 if (conf) 5361 return sprintf(page, "%d\n", conf->bypass_threshold); 5362 else 5363 return 0; 5364 } 5365 5366 static ssize_t 5367 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 5368 { 5369 struct r5conf *conf = mddev->private; 5370 unsigned long new; 5371 if (len >= PAGE_SIZE) 5372 return -EINVAL; 5373 if (!conf) 5374 return -ENODEV; 5375 5376 if (kstrtoul(page, 10, &new)) 5377 return -EINVAL; 5378 if (new > conf->max_nr_stripes) 5379 return -EINVAL; 5380 conf->bypass_threshold = new; 5381 return len; 5382 } 5383 5384 static struct md_sysfs_entry 5385 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 5386 S_IRUGO | S_IWUSR, 5387 raid5_show_preread_threshold, 5388 raid5_store_preread_threshold); 5389 5390 static ssize_t 5391 raid5_show_skip_copy(struct mddev *mddev, char *page) 5392 { 5393 struct r5conf *conf = mddev->private; 5394 if (conf) 5395 return sprintf(page, "%d\n", conf->skip_copy); 5396 else 5397 return 0; 5398 } 5399 5400 static ssize_t 5401 raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) 5402 { 5403 struct r5conf *conf = mddev->private; 5404 unsigned long new; 5405 if (len >= PAGE_SIZE) 5406 return -EINVAL; 5407 if (!conf) 5408 return -ENODEV; 5409 5410 if (kstrtoul(page, 10, &new)) 5411 return -EINVAL; 5412 new = !!new; 5413 if (new == conf->skip_copy) 5414 return len; 5415 5416 mddev_suspend(mddev); 5417 conf->skip_copy = new; 5418 if (new) 5419 mddev->queue->backing_dev_info.capabilities |= 5420 BDI_CAP_STABLE_WRITES; 5421 else 5422 mddev->queue->backing_dev_info.capabilities &= 5423 ~BDI_CAP_STABLE_WRITES; 5424 mddev_resume(mddev); 5425 return len; 5426 } 5427 5428 static struct md_sysfs_entry 5429 raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, 5430 raid5_show_skip_copy, 5431 raid5_store_skip_copy); 5432 5433 static ssize_t 5434 stripe_cache_active_show(struct mddev *mddev, char *page) 5435 { 5436 struct r5conf *conf = mddev->private; 5437 if (conf) 5438 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 5439 else 5440 return 0; 5441 } 5442 5443 static struct md_sysfs_entry 5444 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 5445 5446 static ssize_t 5447 raid5_show_group_thread_cnt(struct mddev *mddev, char *page) 5448 { 5449 struct r5conf *conf = mddev->private; 5450 if (conf) 5451 return sprintf(page, "%d\n", conf->worker_cnt_per_group); 5452 else 5453 return 0; 5454 } 5455 5456 static int alloc_thread_groups(struct r5conf *conf, int cnt, 5457 int *group_cnt, 5458 int *worker_cnt_per_group, 5459 struct r5worker_group **worker_groups); 5460 static ssize_t 5461 raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) 5462 { 5463 struct r5conf *conf = mddev->private; 5464 unsigned long new; 5465 int err; 5466 struct r5worker_group *new_groups, *old_groups; 5467 int group_cnt, worker_cnt_per_group; 5468 5469 if (len >= PAGE_SIZE) 5470 return -EINVAL; 5471 if (!conf) 5472 return -ENODEV; 5473 5474 if (kstrtoul(page, 10, &new)) 5475 return -EINVAL; 5476 5477 if (new == conf->worker_cnt_per_group) 5478 return len; 5479 5480 mddev_suspend(mddev); 5481 5482 old_groups = conf->worker_groups; 5483 if (old_groups) 5484 flush_workqueue(raid5_wq); 5485 5486 err = alloc_thread_groups(conf, new, 5487 &group_cnt, &worker_cnt_per_group, 5488 &new_groups); 5489 if (!err) { 5490 spin_lock_irq(&conf->device_lock); 5491 conf->group_cnt = group_cnt; 5492 conf->worker_cnt_per_group = worker_cnt_per_group; 5493 conf->worker_groups = new_groups; 5494 spin_unlock_irq(&conf->device_lock); 5495 5496 if (old_groups) 5497 kfree(old_groups[0].workers); 5498 kfree(old_groups); 5499 } 5500 5501 mddev_resume(mddev); 5502 5503 if (err) 5504 return err; 5505 return len; 5506 } 5507 5508 static struct md_sysfs_entry 5509 raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, 5510 raid5_show_group_thread_cnt, 5511 raid5_store_group_thread_cnt); 5512 5513 static struct attribute *raid5_attrs[] = { 5514 &raid5_stripecache_size.attr, 5515 &raid5_stripecache_active.attr, 5516 &raid5_preread_bypass_threshold.attr, 5517 &raid5_group_thread_cnt.attr, 5518 &raid5_skip_copy.attr, 5519 NULL, 5520 }; 5521 static struct attribute_group raid5_attrs_group = { 5522 .name = NULL, 5523 .attrs = raid5_attrs, 5524 }; 5525 5526 static int alloc_thread_groups(struct r5conf *conf, int cnt, 5527 int *group_cnt, 5528 int *worker_cnt_per_group, 5529 struct r5worker_group **worker_groups) 5530 { 5531 int i, j, k; 5532 ssize_t size; 5533 struct r5worker *workers; 5534 5535 *worker_cnt_per_group = cnt; 5536 if (cnt == 0) { 5537 *group_cnt = 0; 5538 *worker_groups = NULL; 5539 return 0; 5540 } 5541 *group_cnt = num_possible_nodes(); 5542 size = sizeof(struct r5worker) * cnt; 5543 workers = kzalloc(size * *group_cnt, GFP_NOIO); 5544 *worker_groups = kzalloc(sizeof(struct r5worker_group) * 5545 *group_cnt, GFP_NOIO); 5546 if (!*worker_groups || !workers) { 5547 kfree(workers); 5548 kfree(*worker_groups); 5549 return -ENOMEM; 5550 } 5551 5552 for (i = 0; i < *group_cnt; i++) { 5553 struct r5worker_group *group; 5554 5555 group = &(*worker_groups)[i]; 5556 INIT_LIST_HEAD(&group->handle_list); 5557 group->conf = conf; 5558 group->workers = workers + i * cnt; 5559 5560 for (j = 0; j < cnt; j++) { 5561 struct r5worker *worker = group->workers + j; 5562 worker->group = group; 5563 INIT_WORK(&worker->work, raid5_do_work); 5564 5565 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) 5566 INIT_LIST_HEAD(worker->temp_inactive_list + k); 5567 } 5568 } 5569 5570 return 0; 5571 } 5572 5573 static void free_thread_groups(struct r5conf *conf) 5574 { 5575 if (conf->worker_groups) 5576 kfree(conf->worker_groups[0].workers); 5577 kfree(conf->worker_groups); 5578 conf->worker_groups = NULL; 5579 } 5580 5581 static sector_t 5582 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 5583 { 5584 struct r5conf *conf = mddev->private; 5585 5586 if (!sectors) 5587 sectors = mddev->dev_sectors; 5588 if (!raid_disks) 5589 /* size is defined by the smallest of previous and new size */ 5590 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 5591 5592 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 5593 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); 5594 return sectors * (raid_disks - conf->max_degraded); 5595 } 5596 5597 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 5598 { 5599 safe_put_page(percpu->spare_page); 5600 kfree(percpu->scribble); 5601 percpu->spare_page = NULL; 5602 percpu->scribble = NULL; 5603 } 5604 5605 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 5606 { 5607 if (conf->level == 6 && !percpu->spare_page) 5608 percpu->spare_page = alloc_page(GFP_KERNEL); 5609 if (!percpu->scribble) 5610 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL); 5611 5612 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { 5613 free_scratch_buffer(conf, percpu); 5614 return -ENOMEM; 5615 } 5616 5617 return 0; 5618 } 5619 5620 static void raid5_free_percpu(struct r5conf *conf) 5621 { 5622 unsigned long cpu; 5623 5624 if (!conf->percpu) 5625 return; 5626 5627 #ifdef CONFIG_HOTPLUG_CPU 5628 unregister_cpu_notifier(&conf->cpu_notify); 5629 #endif 5630 5631 get_online_cpus(); 5632 for_each_possible_cpu(cpu) 5633 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5634 put_online_cpus(); 5635 5636 free_percpu(conf->percpu); 5637 } 5638 5639 static void free_conf(struct r5conf *conf) 5640 { 5641 free_thread_groups(conf); 5642 shrink_stripes(conf); 5643 raid5_free_percpu(conf); 5644 kfree(conf->disks); 5645 kfree(conf->stripe_hashtbl); 5646 kfree(conf); 5647 } 5648 5649 #ifdef CONFIG_HOTPLUG_CPU 5650 static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, 5651 void *hcpu) 5652 { 5653 struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); 5654 long cpu = (long)hcpu; 5655 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 5656 5657 switch (action) { 5658 case CPU_UP_PREPARE: 5659 case CPU_UP_PREPARE_FROZEN: 5660 if (alloc_scratch_buffer(conf, percpu)) { 5661 pr_err("%s: failed memory allocation for cpu%ld\n", 5662 __func__, cpu); 5663 return notifier_from_errno(-ENOMEM); 5664 } 5665 break; 5666 case CPU_DEAD: 5667 case CPU_DEAD_FROZEN: 5668 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5669 break; 5670 default: 5671 break; 5672 } 5673 return NOTIFY_OK; 5674 } 5675 #endif 5676 5677 static int raid5_alloc_percpu(struct r5conf *conf) 5678 { 5679 unsigned long cpu; 5680 int err = 0; 5681 5682 conf->percpu = alloc_percpu(struct raid5_percpu); 5683 if (!conf->percpu) 5684 return -ENOMEM; 5685 5686 #ifdef CONFIG_HOTPLUG_CPU 5687 conf->cpu_notify.notifier_call = raid456_cpu_notify; 5688 conf->cpu_notify.priority = 0; 5689 err = register_cpu_notifier(&conf->cpu_notify); 5690 if (err) 5691 return err; 5692 #endif 5693 5694 get_online_cpus(); 5695 for_each_present_cpu(cpu) { 5696 err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 5697 if (err) { 5698 pr_err("%s: failed memory allocation for cpu%ld\n", 5699 __func__, cpu); 5700 break; 5701 } 5702 } 5703 put_online_cpus(); 5704 5705 return err; 5706 } 5707 5708 static struct r5conf *setup_conf(struct mddev *mddev) 5709 { 5710 struct r5conf *conf; 5711 int raid_disk, memory, max_disks; 5712 struct md_rdev *rdev; 5713 struct disk_info *disk; 5714 char pers_name[6]; 5715 int i; 5716 int group_cnt, worker_cnt_per_group; 5717 struct r5worker_group *new_group; 5718 5719 if (mddev->new_level != 5 5720 && mddev->new_level != 4 5721 && mddev->new_level != 6) { 5722 printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", 5723 mdname(mddev), mddev->new_level); 5724 return ERR_PTR(-EIO); 5725 } 5726 if ((mddev->new_level == 5 5727 && !algorithm_valid_raid5(mddev->new_layout)) || 5728 (mddev->new_level == 6 5729 && !algorithm_valid_raid6(mddev->new_layout))) { 5730 printk(KERN_ERR "md/raid:%s: layout %d not supported\n", 5731 mdname(mddev), mddev->new_layout); 5732 return ERR_PTR(-EIO); 5733 } 5734 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 5735 printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", 5736 mdname(mddev), mddev->raid_disks); 5737 return ERR_PTR(-EINVAL); 5738 } 5739 5740 if (!mddev->new_chunk_sectors || 5741 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 5742 !is_power_of_2(mddev->new_chunk_sectors)) { 5743 printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", 5744 mdname(mddev), mddev->new_chunk_sectors << 9); 5745 return ERR_PTR(-EINVAL); 5746 } 5747 5748 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 5749 if (conf == NULL) 5750 goto abort; 5751 /* Don't enable multi-threading by default*/ 5752 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, 5753 &new_group)) { 5754 conf->group_cnt = group_cnt; 5755 conf->worker_cnt_per_group = worker_cnt_per_group; 5756 conf->worker_groups = new_group; 5757 } else 5758 goto abort; 5759 spin_lock_init(&conf->device_lock); 5760 seqcount_init(&conf->gen_lock); 5761 init_waitqueue_head(&conf->wait_for_stripe); 5762 init_waitqueue_head(&conf->wait_for_overlap); 5763 INIT_LIST_HEAD(&conf->handle_list); 5764 INIT_LIST_HEAD(&conf->hold_list); 5765 INIT_LIST_HEAD(&conf->delayed_list); 5766 INIT_LIST_HEAD(&conf->bitmap_list); 5767 init_llist_head(&conf->released_stripes); 5768 atomic_set(&conf->active_stripes, 0); 5769 atomic_set(&conf->preread_active_stripes, 0); 5770 atomic_set(&conf->active_aligned_reads, 0); 5771 conf->bypass_threshold = BYPASS_THRESHOLD; 5772 conf->recovery_disabled = mddev->recovery_disabled - 1; 5773 5774 conf->raid_disks = mddev->raid_disks; 5775 if (mddev->reshape_position == MaxSector) 5776 conf->previous_raid_disks = mddev->raid_disks; 5777 else 5778 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 5779 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 5780 conf->scribble_len = scribble_len(max_disks); 5781 5782 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 5783 GFP_KERNEL); 5784 if (!conf->disks) 5785 goto abort; 5786 5787 conf->mddev = mddev; 5788 5789 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 5790 goto abort; 5791 5792 /* We init hash_locks[0] separately to that it can be used 5793 * as the reference lock in the spin_lock_nest_lock() call 5794 * in lock_all_device_hash_locks_irq in order to convince 5795 * lockdep that we know what we are doing. 5796 */ 5797 spin_lock_init(conf->hash_locks); 5798 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 5799 spin_lock_init(conf->hash_locks + i); 5800 5801 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5802 INIT_LIST_HEAD(conf->inactive_list + i); 5803 5804 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5805 INIT_LIST_HEAD(conf->temp_inactive_list + i); 5806 5807 conf->level = mddev->new_level; 5808 if (raid5_alloc_percpu(conf) != 0) 5809 goto abort; 5810 5811 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 5812 5813 rdev_for_each(rdev, mddev) { 5814 raid_disk = rdev->raid_disk; 5815 if (raid_disk >= max_disks 5816 || raid_disk < 0) 5817 continue; 5818 disk = conf->disks + raid_disk; 5819 5820 if (test_bit(Replacement, &rdev->flags)) { 5821 if (disk->replacement) 5822 goto abort; 5823 disk->replacement = rdev; 5824 } else { 5825 if (disk->rdev) 5826 goto abort; 5827 disk->rdev = rdev; 5828 } 5829 5830 if (test_bit(In_sync, &rdev->flags)) { 5831 char b[BDEVNAME_SIZE]; 5832 printk(KERN_INFO "md/raid:%s: device %s operational as raid" 5833 " disk %d\n", 5834 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 5835 } else if (rdev->saved_raid_disk != raid_disk) 5836 /* Cannot rely on bitmap to complete recovery */ 5837 conf->fullsync = 1; 5838 } 5839 5840 conf->chunk_sectors = mddev->new_chunk_sectors; 5841 conf->level = mddev->new_level; 5842 if (conf->level == 6) 5843 conf->max_degraded = 2; 5844 else 5845 conf->max_degraded = 1; 5846 conf->algorithm = mddev->new_layout; 5847 conf->reshape_progress = mddev->reshape_position; 5848 if (conf->reshape_progress != MaxSector) { 5849 conf->prev_chunk_sectors = mddev->chunk_sectors; 5850 conf->prev_algo = mddev->layout; 5851 } 5852 5853 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) + 5854 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 5855 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); 5856 if (grow_stripes(conf, NR_STRIPES)) { 5857 printk(KERN_ERR 5858 "md/raid:%s: couldn't allocate %dkB for buffers\n", 5859 mdname(mddev), memory); 5860 goto abort; 5861 } else 5862 printk(KERN_INFO "md/raid:%s: allocated %dkB\n", 5863 mdname(mddev), memory); 5864 5865 sprintf(pers_name, "raid%d", mddev->new_level); 5866 conf->thread = md_register_thread(raid5d, mddev, pers_name); 5867 if (!conf->thread) { 5868 printk(KERN_ERR 5869 "md/raid:%s: couldn't allocate thread.\n", 5870 mdname(mddev)); 5871 goto abort; 5872 } 5873 5874 return conf; 5875 5876 abort: 5877 if (conf) { 5878 free_conf(conf); 5879 return ERR_PTR(-EIO); 5880 } else 5881 return ERR_PTR(-ENOMEM); 5882 } 5883 5884 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 5885 { 5886 switch (algo) { 5887 case ALGORITHM_PARITY_0: 5888 if (raid_disk < max_degraded) 5889 return 1; 5890 break; 5891 case ALGORITHM_PARITY_N: 5892 if (raid_disk >= raid_disks - max_degraded) 5893 return 1; 5894 break; 5895 case ALGORITHM_PARITY_0_6: 5896 if (raid_disk == 0 || 5897 raid_disk == raid_disks - 1) 5898 return 1; 5899 break; 5900 case ALGORITHM_LEFT_ASYMMETRIC_6: 5901 case ALGORITHM_RIGHT_ASYMMETRIC_6: 5902 case ALGORITHM_LEFT_SYMMETRIC_6: 5903 case ALGORITHM_RIGHT_SYMMETRIC_6: 5904 if (raid_disk == raid_disks - 1) 5905 return 1; 5906 } 5907 return 0; 5908 } 5909 5910 static int run(struct mddev *mddev) 5911 { 5912 struct r5conf *conf; 5913 int working_disks = 0; 5914 int dirty_parity_disks = 0; 5915 struct md_rdev *rdev; 5916 sector_t reshape_offset = 0; 5917 int i; 5918 long long min_offset_diff = 0; 5919 int first = 1; 5920 5921 if (mddev->recovery_cp != MaxSector) 5922 printk(KERN_NOTICE "md/raid:%s: not clean" 5923 " -- starting background reconstruction\n", 5924 mdname(mddev)); 5925 5926 rdev_for_each(rdev, mddev) { 5927 long long diff; 5928 if (rdev->raid_disk < 0) 5929 continue; 5930 diff = (rdev->new_data_offset - rdev->data_offset); 5931 if (first) { 5932 min_offset_diff = diff; 5933 first = 0; 5934 } else if (mddev->reshape_backwards && 5935 diff < min_offset_diff) 5936 min_offset_diff = diff; 5937 else if (!mddev->reshape_backwards && 5938 diff > min_offset_diff) 5939 min_offset_diff = diff; 5940 } 5941 5942 if (mddev->reshape_position != MaxSector) { 5943 /* Check that we can continue the reshape. 5944 * Difficulties arise if the stripe we would write to 5945 * next is at or after the stripe we would read from next. 5946 * For a reshape that changes the number of devices, this 5947 * is only possible for a very short time, and mdadm makes 5948 * sure that time appears to have past before assembling 5949 * the array. So we fail if that time hasn't passed. 5950 * For a reshape that keeps the number of devices the same 5951 * mdadm must be monitoring the reshape can keeping the 5952 * critical areas read-only and backed up. It will start 5953 * the array in read-only mode, so we check for that. 5954 */ 5955 sector_t here_new, here_old; 5956 int old_disks; 5957 int max_degraded = (mddev->level == 6 ? 2 : 1); 5958 5959 if (mddev->new_level != mddev->level) { 5960 printk(KERN_ERR "md/raid:%s: unsupported reshape " 5961 "required - aborting.\n", 5962 mdname(mddev)); 5963 return -EINVAL; 5964 } 5965 old_disks = mddev->raid_disks - mddev->delta_disks; 5966 /* reshape_position must be on a new-stripe boundary, and one 5967 * further up in new geometry must map after here in old 5968 * geometry. 5969 */ 5970 here_new = mddev->reshape_position; 5971 if (sector_div(here_new, mddev->new_chunk_sectors * 5972 (mddev->raid_disks - max_degraded))) { 5973 printk(KERN_ERR "md/raid:%s: reshape_position not " 5974 "on a stripe boundary\n", mdname(mddev)); 5975 return -EINVAL; 5976 } 5977 reshape_offset = here_new * mddev->new_chunk_sectors; 5978 /* here_new is the stripe we will write to */ 5979 here_old = mddev->reshape_position; 5980 sector_div(here_old, mddev->chunk_sectors * 5981 (old_disks-max_degraded)); 5982 /* here_old is the first stripe that we might need to read 5983 * from */ 5984 if (mddev->delta_disks == 0) { 5985 if ((here_new * mddev->new_chunk_sectors != 5986 here_old * mddev->chunk_sectors)) { 5987 printk(KERN_ERR "md/raid:%s: reshape position is" 5988 " confused - aborting\n", mdname(mddev)); 5989 return -EINVAL; 5990 } 5991 /* We cannot be sure it is safe to start an in-place 5992 * reshape. It is only safe if user-space is monitoring 5993 * and taking constant backups. 5994 * mdadm always starts a situation like this in 5995 * readonly mode so it can take control before 5996 * allowing any writes. So just check for that. 5997 */ 5998 if (abs(min_offset_diff) >= mddev->chunk_sectors && 5999 abs(min_offset_diff) >= mddev->new_chunk_sectors) 6000 /* not really in-place - so OK */; 6001 else if (mddev->ro == 0) { 6002 printk(KERN_ERR "md/raid:%s: in-place reshape " 6003 "must be started in read-only mode " 6004 "- aborting\n", 6005 mdname(mddev)); 6006 return -EINVAL; 6007 } 6008 } else if (mddev->reshape_backwards 6009 ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= 6010 here_old * mddev->chunk_sectors) 6011 : (here_new * mddev->new_chunk_sectors >= 6012 here_old * mddev->chunk_sectors + (-min_offset_diff))) { 6013 /* Reading from the same stripe as writing to - bad */ 6014 printk(KERN_ERR "md/raid:%s: reshape_position too early for " 6015 "auto-recovery - aborting.\n", 6016 mdname(mddev)); 6017 return -EINVAL; 6018 } 6019 printk(KERN_INFO "md/raid:%s: reshape will continue\n", 6020 mdname(mddev)); 6021 /* OK, we should be able to continue; */ 6022 } else { 6023 BUG_ON(mddev->level != mddev->new_level); 6024 BUG_ON(mddev->layout != mddev->new_layout); 6025 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 6026 BUG_ON(mddev->delta_disks != 0); 6027 } 6028 6029 if (mddev->private == NULL) 6030 conf = setup_conf(mddev); 6031 else 6032 conf = mddev->private; 6033 6034 if (IS_ERR(conf)) 6035 return PTR_ERR(conf); 6036 6037 conf->min_offset_diff = min_offset_diff; 6038 mddev->thread = conf->thread; 6039 conf->thread = NULL; 6040 mddev->private = conf; 6041 6042 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 6043 i++) { 6044 rdev = conf->disks[i].rdev; 6045 if (!rdev && conf->disks[i].replacement) { 6046 /* The replacement is all we have yet */ 6047 rdev = conf->disks[i].replacement; 6048 conf->disks[i].replacement = NULL; 6049 clear_bit(Replacement, &rdev->flags); 6050 conf->disks[i].rdev = rdev; 6051 } 6052 if (!rdev) 6053 continue; 6054 if (conf->disks[i].replacement && 6055 conf->reshape_progress != MaxSector) { 6056 /* replacements and reshape simply do not mix. */ 6057 printk(KERN_ERR "md: cannot handle concurrent " 6058 "replacement and reshape.\n"); 6059 goto abort; 6060 } 6061 if (test_bit(In_sync, &rdev->flags)) { 6062 working_disks++; 6063 continue; 6064 } 6065 /* This disc is not fully in-sync. However if it 6066 * just stored parity (beyond the recovery_offset), 6067 * when we don't need to be concerned about the 6068 * array being dirty. 6069 * When reshape goes 'backwards', we never have 6070 * partially completed devices, so we only need 6071 * to worry about reshape going forwards. 6072 */ 6073 /* Hack because v0.91 doesn't store recovery_offset properly. */ 6074 if (mddev->major_version == 0 && 6075 mddev->minor_version > 90) 6076 rdev->recovery_offset = reshape_offset; 6077 6078 if (rdev->recovery_offset < reshape_offset) { 6079 /* We need to check old and new layout */ 6080 if (!only_parity(rdev->raid_disk, 6081 conf->algorithm, 6082 conf->raid_disks, 6083 conf->max_degraded)) 6084 continue; 6085 } 6086 if (!only_parity(rdev->raid_disk, 6087 conf->prev_algo, 6088 conf->previous_raid_disks, 6089 conf->max_degraded)) 6090 continue; 6091 dirty_parity_disks++; 6092 } 6093 6094 /* 6095 * 0 for a fully functional array, 1 or 2 for a degraded array. 6096 */ 6097 mddev->degraded = calc_degraded(conf); 6098 6099 if (has_failed(conf)) { 6100 printk(KERN_ERR "md/raid:%s: not enough operational devices" 6101 " (%d/%d failed)\n", 6102 mdname(mddev), mddev->degraded, conf->raid_disks); 6103 goto abort; 6104 } 6105 6106 /* device size must be a multiple of chunk size */ 6107 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 6108 mddev->resync_max_sectors = mddev->dev_sectors; 6109 6110 if (mddev->degraded > dirty_parity_disks && 6111 mddev->recovery_cp != MaxSector) { 6112 if (mddev->ok_start_degraded) 6113 printk(KERN_WARNING 6114 "md/raid:%s: starting dirty degraded array" 6115 " - data corruption possible.\n", 6116 mdname(mddev)); 6117 else { 6118 printk(KERN_ERR 6119 "md/raid:%s: cannot start dirty degraded array.\n", 6120 mdname(mddev)); 6121 goto abort; 6122 } 6123 } 6124 6125 if (mddev->degraded == 0) 6126 printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" 6127 " devices, algorithm %d\n", mdname(mddev), conf->level, 6128 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 6129 mddev->new_layout); 6130 else 6131 printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" 6132 " out of %d devices, algorithm %d\n", 6133 mdname(mddev), conf->level, 6134 mddev->raid_disks - mddev->degraded, 6135 mddev->raid_disks, mddev->new_layout); 6136 6137 print_raid5_conf(conf); 6138 6139 if (conf->reshape_progress != MaxSector) { 6140 conf->reshape_safe = conf->reshape_progress; 6141 atomic_set(&conf->reshape_stripes, 0); 6142 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6143 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6144 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6145 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6146 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6147 "reshape"); 6148 } 6149 6150 /* Ok, everything is just fine now */ 6151 if (mddev->to_remove == &raid5_attrs_group) 6152 mddev->to_remove = NULL; 6153 else if (mddev->kobj.sd && 6154 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 6155 printk(KERN_WARNING 6156 "raid5: failed to create sysfs attributes for %s\n", 6157 mdname(mddev)); 6158 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6159 6160 if (mddev->queue) { 6161 int chunk_size; 6162 bool discard_supported = true; 6163 /* read-ahead size must cover two whole stripes, which 6164 * is 2 * (datadisks) * chunksize where 'n' is the 6165 * number of raid devices 6166 */ 6167 int data_disks = conf->previous_raid_disks - conf->max_degraded; 6168 int stripe = data_disks * 6169 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 6170 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6171 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6172 6173 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec); 6174 6175 mddev->queue->backing_dev_info.congested_data = mddev; 6176 mddev->queue->backing_dev_info.congested_fn = raid5_congested; 6177 6178 chunk_size = mddev->chunk_sectors << 9; 6179 blk_queue_io_min(mddev->queue, chunk_size); 6180 blk_queue_io_opt(mddev->queue, chunk_size * 6181 (conf->raid_disks - conf->max_degraded)); 6182 mddev->queue->limits.raid_partial_stripes_expensive = 1; 6183 /* 6184 * We can only discard a whole stripe. It doesn't make sense to 6185 * discard data disk but write parity disk 6186 */ 6187 stripe = stripe * PAGE_SIZE; 6188 /* Round up to power of 2, as discard handling 6189 * currently assumes that */ 6190 while ((stripe-1) & stripe) 6191 stripe = (stripe | (stripe-1)) + 1; 6192 mddev->queue->limits.discard_alignment = stripe; 6193 mddev->queue->limits.discard_granularity = stripe; 6194 /* 6195 * unaligned part of discard request will be ignored, so can't 6196 * guarantee discard_zeroes_data 6197 */ 6198 mddev->queue->limits.discard_zeroes_data = 0; 6199 6200 blk_queue_max_write_same_sectors(mddev->queue, 0); 6201 6202 rdev_for_each(rdev, mddev) { 6203 disk_stack_limits(mddev->gendisk, rdev->bdev, 6204 rdev->data_offset << 9); 6205 disk_stack_limits(mddev->gendisk, rdev->bdev, 6206 rdev->new_data_offset << 9); 6207 /* 6208 * discard_zeroes_data is required, otherwise data 6209 * could be lost. Consider a scenario: discard a stripe 6210 * (the stripe could be inconsistent if 6211 * discard_zeroes_data is 0); write one disk of the 6212 * stripe (the stripe could be inconsistent again 6213 * depending on which disks are used to calculate 6214 * parity); the disk is broken; The stripe data of this 6215 * disk is lost. 6216 */ 6217 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || 6218 !bdev_get_queue(rdev->bdev)-> 6219 limits.discard_zeroes_data) 6220 discard_supported = false; 6221 /* Unfortunately, discard_zeroes_data is not currently 6222 * a guarantee - just a hint. So we only allow DISCARD 6223 * if the sysadmin has confirmed that only safe devices 6224 * are in use by setting a module parameter. 6225 */ 6226 if (!devices_handle_discard_safely) { 6227 if (discard_supported) { 6228 pr_info("md/raid456: discard support disabled due to uncertainty.\n"); 6229 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); 6230 } 6231 discard_supported = false; 6232 } 6233 } 6234 6235 if (discard_supported && 6236 mddev->queue->limits.max_discard_sectors >= stripe && 6237 mddev->queue->limits.discard_granularity >= stripe) 6238 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 6239 mddev->queue); 6240 else 6241 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 6242 mddev->queue); 6243 } 6244 6245 return 0; 6246 abort: 6247 md_unregister_thread(&mddev->thread); 6248 print_raid5_conf(conf); 6249 free_conf(conf); 6250 mddev->private = NULL; 6251 printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); 6252 return -EIO; 6253 } 6254 6255 static int stop(struct mddev *mddev) 6256 { 6257 struct r5conf *conf = mddev->private; 6258 6259 md_unregister_thread(&mddev->thread); 6260 if (mddev->queue) 6261 mddev->queue->backing_dev_info.congested_fn = NULL; 6262 free_conf(conf); 6263 mddev->private = NULL; 6264 mddev->to_remove = &raid5_attrs_group; 6265 return 0; 6266 } 6267 6268 static void status(struct seq_file *seq, struct mddev *mddev) 6269 { 6270 struct r5conf *conf = mddev->private; 6271 int i; 6272 6273 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 6274 mddev->chunk_sectors / 2, mddev->layout); 6275 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 6276 for (i = 0; i < conf->raid_disks; i++) 6277 seq_printf (seq, "%s", 6278 conf->disks[i].rdev && 6279 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); 6280 seq_printf (seq, "]"); 6281 } 6282 6283 static void print_raid5_conf (struct r5conf *conf) 6284 { 6285 int i; 6286 struct disk_info *tmp; 6287 6288 printk(KERN_DEBUG "RAID conf printout:\n"); 6289 if (!conf) { 6290 printk("(conf==NULL)\n"); 6291 return; 6292 } 6293 printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, 6294 conf->raid_disks, 6295 conf->raid_disks - conf->mddev->degraded); 6296 6297 for (i = 0; i < conf->raid_disks; i++) { 6298 char b[BDEVNAME_SIZE]; 6299 tmp = conf->disks + i; 6300 if (tmp->rdev) 6301 printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", 6302 i, !test_bit(Faulty, &tmp->rdev->flags), 6303 bdevname(tmp->rdev->bdev, b)); 6304 } 6305 } 6306 6307 static int raid5_spare_active(struct mddev *mddev) 6308 { 6309 int i; 6310 struct r5conf *conf = mddev->private; 6311 struct disk_info *tmp; 6312 int count = 0; 6313 unsigned long flags; 6314 6315 for (i = 0; i < conf->raid_disks; i++) { 6316 tmp = conf->disks + i; 6317 if (tmp->replacement 6318 && tmp->replacement->recovery_offset == MaxSector 6319 && !test_bit(Faulty, &tmp->replacement->flags) 6320 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 6321 /* Replacement has just become active. */ 6322 if (!tmp->rdev 6323 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 6324 count++; 6325 if (tmp->rdev) { 6326 /* Replaced device not technically faulty, 6327 * but we need to be sure it gets removed 6328 * and never re-added. 6329 */ 6330 set_bit(Faulty, &tmp->rdev->flags); 6331 sysfs_notify_dirent_safe( 6332 tmp->rdev->sysfs_state); 6333 } 6334 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 6335 } else if (tmp->rdev 6336 && tmp->rdev->recovery_offset == MaxSector 6337 && !test_bit(Faulty, &tmp->rdev->flags) 6338 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 6339 count++; 6340 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 6341 } 6342 } 6343 spin_lock_irqsave(&conf->device_lock, flags); 6344 mddev->degraded = calc_degraded(conf); 6345 spin_unlock_irqrestore(&conf->device_lock, flags); 6346 print_raid5_conf(conf); 6347 return count; 6348 } 6349 6350 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 6351 { 6352 struct r5conf *conf = mddev->private; 6353 int err = 0; 6354 int number = rdev->raid_disk; 6355 struct md_rdev **rdevp; 6356 struct disk_info *p = conf->disks + number; 6357 6358 print_raid5_conf(conf); 6359 if (rdev == p->rdev) 6360 rdevp = &p->rdev; 6361 else if (rdev == p->replacement) 6362 rdevp = &p->replacement; 6363 else 6364 return 0; 6365 6366 if (number >= conf->raid_disks && 6367 conf->reshape_progress == MaxSector) 6368 clear_bit(In_sync, &rdev->flags); 6369 6370 if (test_bit(In_sync, &rdev->flags) || 6371 atomic_read(&rdev->nr_pending)) { 6372 err = -EBUSY; 6373 goto abort; 6374 } 6375 /* Only remove non-faulty devices if recovery 6376 * isn't possible. 6377 */ 6378 if (!test_bit(Faulty, &rdev->flags) && 6379 mddev->recovery_disabled != conf->recovery_disabled && 6380 !has_failed(conf) && 6381 (!p->replacement || p->replacement == rdev) && 6382 number < conf->raid_disks) { 6383 err = -EBUSY; 6384 goto abort; 6385 } 6386 *rdevp = NULL; 6387 synchronize_rcu(); 6388 if (atomic_read(&rdev->nr_pending)) { 6389 /* lost the race, try later */ 6390 err = -EBUSY; 6391 *rdevp = rdev; 6392 } else if (p->replacement) { 6393 /* We must have just cleared 'rdev' */ 6394 p->rdev = p->replacement; 6395 clear_bit(Replacement, &p->replacement->flags); 6396 smp_mb(); /* Make sure other CPUs may see both as identical 6397 * but will never see neither - if they are careful 6398 */ 6399 p->replacement = NULL; 6400 clear_bit(WantReplacement, &rdev->flags); 6401 } else 6402 /* We might have just removed the Replacement as faulty- 6403 * clear the bit just in case 6404 */ 6405 clear_bit(WantReplacement, &rdev->flags); 6406 abort: 6407 6408 print_raid5_conf(conf); 6409 return err; 6410 } 6411 6412 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 6413 { 6414 struct r5conf *conf = mddev->private; 6415 int err = -EEXIST; 6416 int disk; 6417 struct disk_info *p; 6418 int first = 0; 6419 int last = conf->raid_disks - 1; 6420 6421 if (mddev->recovery_disabled == conf->recovery_disabled) 6422 return -EBUSY; 6423 6424 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 6425 /* no point adding a device */ 6426 return -EINVAL; 6427 6428 if (rdev->raid_disk >= 0) 6429 first = last = rdev->raid_disk; 6430 6431 /* 6432 * find the disk ... but prefer rdev->saved_raid_disk 6433 * if possible. 6434 */ 6435 if (rdev->saved_raid_disk >= 0 && 6436 rdev->saved_raid_disk >= first && 6437 conf->disks[rdev->saved_raid_disk].rdev == NULL) 6438 first = rdev->saved_raid_disk; 6439 6440 for (disk = first; disk <= last; disk++) { 6441 p = conf->disks + disk; 6442 if (p->rdev == NULL) { 6443 clear_bit(In_sync, &rdev->flags); 6444 rdev->raid_disk = disk; 6445 err = 0; 6446 if (rdev->saved_raid_disk != disk) 6447 conf->fullsync = 1; 6448 rcu_assign_pointer(p->rdev, rdev); 6449 goto out; 6450 } 6451 } 6452 for (disk = first; disk <= last; disk++) { 6453 p = conf->disks + disk; 6454 if (test_bit(WantReplacement, &p->rdev->flags) && 6455 p->replacement == NULL) { 6456 clear_bit(In_sync, &rdev->flags); 6457 set_bit(Replacement, &rdev->flags); 6458 rdev->raid_disk = disk; 6459 err = 0; 6460 conf->fullsync = 1; 6461 rcu_assign_pointer(p->replacement, rdev); 6462 break; 6463 } 6464 } 6465 out: 6466 print_raid5_conf(conf); 6467 return err; 6468 } 6469 6470 static int raid5_resize(struct mddev *mddev, sector_t sectors) 6471 { 6472 /* no resync is happening, and there is enough space 6473 * on all devices, so we can resize. 6474 * We need to make sure resync covers any new space. 6475 * If the array is shrinking we should possibly wait until 6476 * any io in the removed space completes, but it hardly seems 6477 * worth it. 6478 */ 6479 sector_t newsize; 6480 sectors &= ~((sector_t)mddev->chunk_sectors - 1); 6481 newsize = raid5_size(mddev, sectors, mddev->raid_disks); 6482 if (mddev->external_size && 6483 mddev->array_sectors > newsize) 6484 return -EINVAL; 6485 if (mddev->bitmap) { 6486 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); 6487 if (ret) 6488 return ret; 6489 } 6490 md_set_array_sectors(mddev, newsize); 6491 set_capacity(mddev->gendisk, mddev->array_sectors); 6492 revalidate_disk(mddev->gendisk); 6493 if (sectors > mddev->dev_sectors && 6494 mddev->recovery_cp > mddev->dev_sectors) { 6495 mddev->recovery_cp = mddev->dev_sectors; 6496 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 6497 } 6498 mddev->dev_sectors = sectors; 6499 mddev->resync_max_sectors = sectors; 6500 return 0; 6501 } 6502 6503 static int check_stripe_cache(struct mddev *mddev) 6504 { 6505 /* Can only proceed if there are plenty of stripe_heads. 6506 * We need a minimum of one full stripe,, and for sensible progress 6507 * it is best to have about 4 times that. 6508 * If we require 4 times, then the default 256 4K stripe_heads will 6509 * allow for chunk sizes up to 256K, which is probably OK. 6510 * If the chunk size is greater, user-space should request more 6511 * stripe_heads first. 6512 */ 6513 struct r5conf *conf = mddev->private; 6514 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 6515 > conf->max_nr_stripes || 6516 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 6517 > conf->max_nr_stripes) { 6518 printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", 6519 mdname(mddev), 6520 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 6521 / STRIPE_SIZE)*4); 6522 return 0; 6523 } 6524 return 1; 6525 } 6526 6527 static int check_reshape(struct mddev *mddev) 6528 { 6529 struct r5conf *conf = mddev->private; 6530 6531 if (mddev->delta_disks == 0 && 6532 mddev->new_layout == mddev->layout && 6533 mddev->new_chunk_sectors == mddev->chunk_sectors) 6534 return 0; /* nothing to do */ 6535 if (has_failed(conf)) 6536 return -EINVAL; 6537 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { 6538 /* We might be able to shrink, but the devices must 6539 * be made bigger first. 6540 * For raid6, 4 is the minimum size. 6541 * Otherwise 2 is the minimum 6542 */ 6543 int min = 2; 6544 if (mddev->level == 6) 6545 min = 4; 6546 if (mddev->raid_disks + mddev->delta_disks < min) 6547 return -EINVAL; 6548 } 6549 6550 if (!check_stripe_cache(mddev)) 6551 return -ENOSPC; 6552 6553 return resize_stripes(conf, (conf->previous_raid_disks 6554 + mddev->delta_disks)); 6555 } 6556 6557 static int raid5_start_reshape(struct mddev *mddev) 6558 { 6559 struct r5conf *conf = mddev->private; 6560 struct md_rdev *rdev; 6561 int spares = 0; 6562 unsigned long flags; 6563 6564 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 6565 return -EBUSY; 6566 6567 if (!check_stripe_cache(mddev)) 6568 return -ENOSPC; 6569 6570 if (has_failed(conf)) 6571 return -EINVAL; 6572 6573 rdev_for_each(rdev, mddev) { 6574 if (!test_bit(In_sync, &rdev->flags) 6575 && !test_bit(Faulty, &rdev->flags)) 6576 spares++; 6577 } 6578 6579 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 6580 /* Not enough devices even to make a degraded array 6581 * of that size 6582 */ 6583 return -EINVAL; 6584 6585 /* Refuse to reduce size of the array. Any reductions in 6586 * array size must be through explicit setting of array_size 6587 * attribute. 6588 */ 6589 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 6590 < mddev->array_sectors) { 6591 printk(KERN_ERR "md/raid:%s: array size must be reduced " 6592 "before number of disks\n", mdname(mddev)); 6593 return -EINVAL; 6594 } 6595 6596 atomic_set(&conf->reshape_stripes, 0); 6597 spin_lock_irq(&conf->device_lock); 6598 write_seqcount_begin(&conf->gen_lock); 6599 conf->previous_raid_disks = conf->raid_disks; 6600 conf->raid_disks += mddev->delta_disks; 6601 conf->prev_chunk_sectors = conf->chunk_sectors; 6602 conf->chunk_sectors = mddev->new_chunk_sectors; 6603 conf->prev_algo = conf->algorithm; 6604 conf->algorithm = mddev->new_layout; 6605 conf->generation++; 6606 /* Code that selects data_offset needs to see the generation update 6607 * if reshape_progress has been set - so a memory barrier needed. 6608 */ 6609 smp_mb(); 6610 if (mddev->reshape_backwards) 6611 conf->reshape_progress = raid5_size(mddev, 0, 0); 6612 else 6613 conf->reshape_progress = 0; 6614 conf->reshape_safe = conf->reshape_progress; 6615 write_seqcount_end(&conf->gen_lock); 6616 spin_unlock_irq(&conf->device_lock); 6617 6618 /* Now make sure any requests that proceeded on the assumption 6619 * the reshape wasn't running - like Discard or Read - have 6620 * completed. 6621 */ 6622 mddev_suspend(mddev); 6623 mddev_resume(mddev); 6624 6625 /* Add some new drives, as many as will fit. 6626 * We know there are enough to make the newly sized array work. 6627 * Don't add devices if we are reducing the number of 6628 * devices in the array. This is because it is not possible 6629 * to correctly record the "partially reconstructed" state of 6630 * such devices during the reshape and confusion could result. 6631 */ 6632 if (mddev->delta_disks >= 0) { 6633 rdev_for_each(rdev, mddev) 6634 if (rdev->raid_disk < 0 && 6635 !test_bit(Faulty, &rdev->flags)) { 6636 if (raid5_add_disk(mddev, rdev) == 0) { 6637 if (rdev->raid_disk 6638 >= conf->previous_raid_disks) 6639 set_bit(In_sync, &rdev->flags); 6640 else 6641 rdev->recovery_offset = 0; 6642 6643 if (sysfs_link_rdev(mddev, rdev)) 6644 /* Failure here is OK */; 6645 } 6646 } else if (rdev->raid_disk >= conf->previous_raid_disks 6647 && !test_bit(Faulty, &rdev->flags)) { 6648 /* This is a spare that was manually added */ 6649 set_bit(In_sync, &rdev->flags); 6650 } 6651 6652 /* When a reshape changes the number of devices, 6653 * ->degraded is measured against the larger of the 6654 * pre and post number of devices. 6655 */ 6656 spin_lock_irqsave(&conf->device_lock, flags); 6657 mddev->degraded = calc_degraded(conf); 6658 spin_unlock_irqrestore(&conf->device_lock, flags); 6659 } 6660 mddev->raid_disks = conf->raid_disks; 6661 mddev->reshape_position = conf->reshape_progress; 6662 set_bit(MD_CHANGE_DEVS, &mddev->flags); 6663 6664 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 6665 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 6666 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 6667 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 6668 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 6669 "reshape"); 6670 if (!mddev->sync_thread) { 6671 mddev->recovery = 0; 6672 spin_lock_irq(&conf->device_lock); 6673 write_seqcount_begin(&conf->gen_lock); 6674 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 6675 mddev->new_chunk_sectors = 6676 conf->chunk_sectors = conf->prev_chunk_sectors; 6677 mddev->new_layout = conf->algorithm = conf->prev_algo; 6678 rdev_for_each(rdev, mddev) 6679 rdev->new_data_offset = rdev->data_offset; 6680 smp_wmb(); 6681 conf->generation --; 6682 conf->reshape_progress = MaxSector; 6683 mddev->reshape_position = MaxSector; 6684 write_seqcount_end(&conf->gen_lock); 6685 spin_unlock_irq(&conf->device_lock); 6686 return -EAGAIN; 6687 } 6688 conf->reshape_checkpoint = jiffies; 6689 md_wakeup_thread(mddev->sync_thread); 6690 md_new_event(mddev); 6691 return 0; 6692 } 6693 6694 /* This is called from the reshape thread and should make any 6695 * changes needed in 'conf' 6696 */ 6697 static void end_reshape(struct r5conf *conf) 6698 { 6699 6700 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 6701 struct md_rdev *rdev; 6702 6703 spin_lock_irq(&conf->device_lock); 6704 conf->previous_raid_disks = conf->raid_disks; 6705 rdev_for_each(rdev, conf->mddev) 6706 rdev->data_offset = rdev->new_data_offset; 6707 smp_wmb(); 6708 conf->reshape_progress = MaxSector; 6709 spin_unlock_irq(&conf->device_lock); 6710 wake_up(&conf->wait_for_overlap); 6711 6712 /* read-ahead size must cover two whole stripes, which is 6713 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 6714 */ 6715 if (conf->mddev->queue) { 6716 int data_disks = conf->raid_disks - conf->max_degraded; 6717 int stripe = data_disks * ((conf->chunk_sectors << 9) 6718 / PAGE_SIZE); 6719 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 6720 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 6721 } 6722 } 6723 } 6724 6725 /* This is called from the raid5d thread with mddev_lock held. 6726 * It makes config changes to the device. 6727 */ 6728 static void raid5_finish_reshape(struct mddev *mddev) 6729 { 6730 struct r5conf *conf = mddev->private; 6731 6732 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 6733 6734 if (mddev->delta_disks > 0) { 6735 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 6736 set_capacity(mddev->gendisk, mddev->array_sectors); 6737 revalidate_disk(mddev->gendisk); 6738 } else { 6739 int d; 6740 spin_lock_irq(&conf->device_lock); 6741 mddev->degraded = calc_degraded(conf); 6742 spin_unlock_irq(&conf->device_lock); 6743 for (d = conf->raid_disks ; 6744 d < conf->raid_disks - mddev->delta_disks; 6745 d++) { 6746 struct md_rdev *rdev = conf->disks[d].rdev; 6747 if (rdev) 6748 clear_bit(In_sync, &rdev->flags); 6749 rdev = conf->disks[d].replacement; 6750 if (rdev) 6751 clear_bit(In_sync, &rdev->flags); 6752 } 6753 } 6754 mddev->layout = conf->algorithm; 6755 mddev->chunk_sectors = conf->chunk_sectors; 6756 mddev->reshape_position = MaxSector; 6757 mddev->delta_disks = 0; 6758 mddev->reshape_backwards = 0; 6759 } 6760 } 6761 6762 static void raid5_quiesce(struct mddev *mddev, int state) 6763 { 6764 struct r5conf *conf = mddev->private; 6765 6766 switch(state) { 6767 case 2: /* resume for a suspend */ 6768 wake_up(&conf->wait_for_overlap); 6769 break; 6770 6771 case 1: /* stop all writes */ 6772 lock_all_device_hash_locks_irq(conf); 6773 /* '2' tells resync/reshape to pause so that all 6774 * active stripes can drain 6775 */ 6776 conf->quiesce = 2; 6777 wait_event_cmd(conf->wait_for_stripe, 6778 atomic_read(&conf->active_stripes) == 0 && 6779 atomic_read(&conf->active_aligned_reads) == 0, 6780 unlock_all_device_hash_locks_irq(conf), 6781 lock_all_device_hash_locks_irq(conf)); 6782 conf->quiesce = 1; 6783 unlock_all_device_hash_locks_irq(conf); 6784 /* allow reshape to continue */ 6785 wake_up(&conf->wait_for_overlap); 6786 break; 6787 6788 case 0: /* re-enable writes */ 6789 lock_all_device_hash_locks_irq(conf); 6790 conf->quiesce = 0; 6791 wake_up(&conf->wait_for_stripe); 6792 wake_up(&conf->wait_for_overlap); 6793 unlock_all_device_hash_locks_irq(conf); 6794 break; 6795 } 6796 } 6797 6798 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 6799 { 6800 struct r0conf *raid0_conf = mddev->private; 6801 sector_t sectors; 6802 6803 /* for raid0 takeover only one zone is supported */ 6804 if (raid0_conf->nr_strip_zones > 1) { 6805 printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", 6806 mdname(mddev)); 6807 return ERR_PTR(-EINVAL); 6808 } 6809 6810 sectors = raid0_conf->strip_zone[0].zone_end; 6811 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 6812 mddev->dev_sectors = sectors; 6813 mddev->new_level = level; 6814 mddev->new_layout = ALGORITHM_PARITY_N; 6815 mddev->new_chunk_sectors = mddev->chunk_sectors; 6816 mddev->raid_disks += 1; 6817 mddev->delta_disks = 1; 6818 /* make sure it will be not marked as dirty */ 6819 mddev->recovery_cp = MaxSector; 6820 6821 return setup_conf(mddev); 6822 } 6823 6824 static void *raid5_takeover_raid1(struct mddev *mddev) 6825 { 6826 int chunksect; 6827 6828 if (mddev->raid_disks != 2 || 6829 mddev->degraded > 1) 6830 return ERR_PTR(-EINVAL); 6831 6832 /* Should check if there are write-behind devices? */ 6833 6834 chunksect = 64*2; /* 64K by default */ 6835 6836 /* The array must be an exact multiple of chunksize */ 6837 while (chunksect && (mddev->array_sectors & (chunksect-1))) 6838 chunksect >>= 1; 6839 6840 if ((chunksect<<9) < STRIPE_SIZE) 6841 /* array size does not allow a suitable chunk size */ 6842 return ERR_PTR(-EINVAL); 6843 6844 mddev->new_level = 5; 6845 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 6846 mddev->new_chunk_sectors = chunksect; 6847 6848 return setup_conf(mddev); 6849 } 6850 6851 static void *raid5_takeover_raid6(struct mddev *mddev) 6852 { 6853 int new_layout; 6854 6855 switch (mddev->layout) { 6856 case ALGORITHM_LEFT_ASYMMETRIC_6: 6857 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 6858 break; 6859 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6860 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 6861 break; 6862 case ALGORITHM_LEFT_SYMMETRIC_6: 6863 new_layout = ALGORITHM_LEFT_SYMMETRIC; 6864 break; 6865 case ALGORITHM_RIGHT_SYMMETRIC_6: 6866 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 6867 break; 6868 case ALGORITHM_PARITY_0_6: 6869 new_layout = ALGORITHM_PARITY_0; 6870 break; 6871 case ALGORITHM_PARITY_N: 6872 new_layout = ALGORITHM_PARITY_N; 6873 break; 6874 default: 6875 return ERR_PTR(-EINVAL); 6876 } 6877 mddev->new_level = 5; 6878 mddev->new_layout = new_layout; 6879 mddev->delta_disks = -1; 6880 mddev->raid_disks -= 1; 6881 return setup_conf(mddev); 6882 } 6883 6884 static int raid5_check_reshape(struct mddev *mddev) 6885 { 6886 /* For a 2-drive array, the layout and chunk size can be changed 6887 * immediately as not restriping is needed. 6888 * For larger arrays we record the new value - after validation 6889 * to be used by a reshape pass. 6890 */ 6891 struct r5conf *conf = mddev->private; 6892 int new_chunk = mddev->new_chunk_sectors; 6893 6894 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 6895 return -EINVAL; 6896 if (new_chunk > 0) { 6897 if (!is_power_of_2(new_chunk)) 6898 return -EINVAL; 6899 if (new_chunk < (PAGE_SIZE>>9)) 6900 return -EINVAL; 6901 if (mddev->array_sectors & (new_chunk-1)) 6902 /* not factor of array size */ 6903 return -EINVAL; 6904 } 6905 6906 /* They look valid */ 6907 6908 if (mddev->raid_disks == 2) { 6909 /* can make the change immediately */ 6910 if (mddev->new_layout >= 0) { 6911 conf->algorithm = mddev->new_layout; 6912 mddev->layout = mddev->new_layout; 6913 } 6914 if (new_chunk > 0) { 6915 conf->chunk_sectors = new_chunk ; 6916 mddev->chunk_sectors = new_chunk; 6917 } 6918 set_bit(MD_CHANGE_DEVS, &mddev->flags); 6919 md_wakeup_thread(mddev->thread); 6920 } 6921 return check_reshape(mddev); 6922 } 6923 6924 static int raid6_check_reshape(struct mddev *mddev) 6925 { 6926 int new_chunk = mddev->new_chunk_sectors; 6927 6928 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 6929 return -EINVAL; 6930 if (new_chunk > 0) { 6931 if (!is_power_of_2(new_chunk)) 6932 return -EINVAL; 6933 if (new_chunk < (PAGE_SIZE >> 9)) 6934 return -EINVAL; 6935 if (mddev->array_sectors & (new_chunk-1)) 6936 /* not factor of array size */ 6937 return -EINVAL; 6938 } 6939 6940 /* They look valid */ 6941 return check_reshape(mddev); 6942 } 6943 6944 static void *raid5_takeover(struct mddev *mddev) 6945 { 6946 /* raid5 can take over: 6947 * raid0 - if there is only one strip zone - make it a raid4 layout 6948 * raid1 - if there are two drives. We need to know the chunk size 6949 * raid4 - trivial - just use a raid4 layout. 6950 * raid6 - Providing it is a *_6 layout 6951 */ 6952 if (mddev->level == 0) 6953 return raid45_takeover_raid0(mddev, 5); 6954 if (mddev->level == 1) 6955 return raid5_takeover_raid1(mddev); 6956 if (mddev->level == 4) { 6957 mddev->new_layout = ALGORITHM_PARITY_N; 6958 mddev->new_level = 5; 6959 return setup_conf(mddev); 6960 } 6961 if (mddev->level == 6) 6962 return raid5_takeover_raid6(mddev); 6963 6964 return ERR_PTR(-EINVAL); 6965 } 6966 6967 static void *raid4_takeover(struct mddev *mddev) 6968 { 6969 /* raid4 can take over: 6970 * raid0 - if there is only one strip zone 6971 * raid5 - if layout is right 6972 */ 6973 if (mddev->level == 0) 6974 return raid45_takeover_raid0(mddev, 4); 6975 if (mddev->level == 5 && 6976 mddev->layout == ALGORITHM_PARITY_N) { 6977 mddev->new_layout = 0; 6978 mddev->new_level = 4; 6979 return setup_conf(mddev); 6980 } 6981 return ERR_PTR(-EINVAL); 6982 } 6983 6984 static struct md_personality raid5_personality; 6985 6986 static void *raid6_takeover(struct mddev *mddev) 6987 { 6988 /* Currently can only take over a raid5. We map the 6989 * personality to an equivalent raid6 personality 6990 * with the Q block at the end. 6991 */ 6992 int new_layout; 6993 6994 if (mddev->pers != &raid5_personality) 6995 return ERR_PTR(-EINVAL); 6996 if (mddev->degraded > 1) 6997 return ERR_PTR(-EINVAL); 6998 if (mddev->raid_disks > 253) 6999 return ERR_PTR(-EINVAL); 7000 if (mddev->raid_disks < 3) 7001 return ERR_PTR(-EINVAL); 7002 7003 switch (mddev->layout) { 7004 case ALGORITHM_LEFT_ASYMMETRIC: 7005 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 7006 break; 7007 case ALGORITHM_RIGHT_ASYMMETRIC: 7008 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 7009 break; 7010 case ALGORITHM_LEFT_SYMMETRIC: 7011 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 7012 break; 7013 case ALGORITHM_RIGHT_SYMMETRIC: 7014 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 7015 break; 7016 case ALGORITHM_PARITY_0: 7017 new_layout = ALGORITHM_PARITY_0_6; 7018 break; 7019 case ALGORITHM_PARITY_N: 7020 new_layout = ALGORITHM_PARITY_N; 7021 break; 7022 default: 7023 return ERR_PTR(-EINVAL); 7024 } 7025 mddev->new_level = 6; 7026 mddev->new_layout = new_layout; 7027 mddev->delta_disks = 1; 7028 mddev->raid_disks += 1; 7029 return setup_conf(mddev); 7030 } 7031 7032 static struct md_personality raid6_personality = 7033 { 7034 .name = "raid6", 7035 .level = 6, 7036 .owner = THIS_MODULE, 7037 .make_request = make_request, 7038 .run = run, 7039 .stop = stop, 7040 .status = status, 7041 .error_handler = error, 7042 .hot_add_disk = raid5_add_disk, 7043 .hot_remove_disk= raid5_remove_disk, 7044 .spare_active = raid5_spare_active, 7045 .sync_request = sync_request, 7046 .resize = raid5_resize, 7047 .size = raid5_size, 7048 .check_reshape = raid6_check_reshape, 7049 .start_reshape = raid5_start_reshape, 7050 .finish_reshape = raid5_finish_reshape, 7051 .quiesce = raid5_quiesce, 7052 .takeover = raid6_takeover, 7053 }; 7054 static struct md_personality raid5_personality = 7055 { 7056 .name = "raid5", 7057 .level = 5, 7058 .owner = THIS_MODULE, 7059 .make_request = make_request, 7060 .run = run, 7061 .stop = stop, 7062 .status = status, 7063 .error_handler = error, 7064 .hot_add_disk = raid5_add_disk, 7065 .hot_remove_disk= raid5_remove_disk, 7066 .spare_active = raid5_spare_active, 7067 .sync_request = sync_request, 7068 .resize = raid5_resize, 7069 .size = raid5_size, 7070 .check_reshape = raid5_check_reshape, 7071 .start_reshape = raid5_start_reshape, 7072 .finish_reshape = raid5_finish_reshape, 7073 .quiesce = raid5_quiesce, 7074 .takeover = raid5_takeover, 7075 }; 7076 7077 static struct md_personality raid4_personality = 7078 { 7079 .name = "raid4", 7080 .level = 4, 7081 .owner = THIS_MODULE, 7082 .make_request = make_request, 7083 .run = run, 7084 .stop = stop, 7085 .status = status, 7086 .error_handler = error, 7087 .hot_add_disk = raid5_add_disk, 7088 .hot_remove_disk= raid5_remove_disk, 7089 .spare_active = raid5_spare_active, 7090 .sync_request = sync_request, 7091 .resize = raid5_resize, 7092 .size = raid5_size, 7093 .check_reshape = raid5_check_reshape, 7094 .start_reshape = raid5_start_reshape, 7095 .finish_reshape = raid5_finish_reshape, 7096 .quiesce = raid5_quiesce, 7097 .takeover = raid4_takeover, 7098 }; 7099 7100 static int __init raid5_init(void) 7101 { 7102 raid5_wq = alloc_workqueue("raid5wq", 7103 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 7104 if (!raid5_wq) 7105 return -ENOMEM; 7106 register_md_personality(&raid6_personality); 7107 register_md_personality(&raid5_personality); 7108 register_md_personality(&raid4_personality); 7109 return 0; 7110 } 7111 7112 static void raid5_exit(void) 7113 { 7114 unregister_md_personality(&raid6_personality); 7115 unregister_md_personality(&raid5_personality); 7116 unregister_md_personality(&raid4_personality); 7117 destroy_workqueue(raid5_wq); 7118 } 7119 7120 module_init(raid5_init); 7121 module_exit(raid5_exit); 7122 MODULE_LICENSE("GPL"); 7123 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 7124 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 7125 MODULE_ALIAS("md-raid5"); 7126 MODULE_ALIAS("md-raid4"); 7127 MODULE_ALIAS("md-level-5"); 7128 MODULE_ALIAS("md-level-4"); 7129 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 7130 MODULE_ALIAS("md-raid6"); 7131 MODULE_ALIAS("md-level-6"); 7132 7133 /* This used to be two separate modules, they were: */ 7134 MODULE_ALIAS("raid5"); 7135 MODULE_ALIAS("raid6"); 7136