1 /* 2 * raid5.c : Multiple Devices driver for Linux 3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman 4 * Copyright (C) 1999, 2000 Ingo Molnar 5 * Copyright (C) 2002, 2003 H. Peter Anvin 6 * 7 * RAID-4/5/6 management functions. 8 * Thanks to Penguin Computing for making the RAID-6 development possible 9 * by donating a test server! 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * You should have received a copy of the GNU General Public License 17 * (for example /usr/src/linux/COPYING); if not, write to the Free 18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 19 */ 20 21 /* 22 * BITMAP UNPLUGGING: 23 * 24 * The sequencing for updating the bitmap reliably is a little 25 * subtle (and I got it wrong the first time) so it deserves some 26 * explanation. 27 * 28 * We group bitmap updates into batches. Each batch has a number. 29 * We may write out several batches at once, but that isn't very important. 30 * conf->seq_write is the number of the last batch successfully written. 31 * conf->seq_flush is the number of the last batch that was closed to 32 * new additions. 33 * When we discover that we will need to write to any block in a stripe 34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq 35 * the number of the batch it will be in. This is seq_flush+1. 36 * When we are ready to do a write, if that batch hasn't been written yet, 37 * we plug the array and queue the stripe for later. 38 * When an unplug happens, we increment bm_flush, thus closing the current 39 * batch. 40 * When we notice that bm_flush > bm_write, we write out all pending updates 41 * to the bitmap, and advance bm_write to where bm_flush was. 42 * This may occasionally write a bit out twice, but is sure never to 43 * miss any bits. 44 */ 45 46 #include <linux/blkdev.h> 47 #include <linux/kthread.h> 48 #include <linux/raid/pq.h> 49 #include <linux/async_tx.h> 50 #include <linux/module.h> 51 #include <linux/async.h> 52 #include <linux/seq_file.h> 53 #include <linux/cpu.h> 54 #include <linux/slab.h> 55 #include <linux/ratelimit.h> 56 #include <linux/nodemask.h> 57 #include <linux/flex_array.h> 58 #include <trace/events/block.h> 59 60 #include "md.h" 61 #include "raid5.h" 62 #include "raid0.h" 63 #include "bitmap.h" 64 65 #define cpu_to_group(cpu) cpu_to_node(cpu) 66 #define ANY_GROUP NUMA_NO_NODE 67 68 static bool devices_handle_discard_safely = false; 69 module_param(devices_handle_discard_safely, bool, 0644); 70 MODULE_PARM_DESC(devices_handle_discard_safely, 71 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); 72 static struct workqueue_struct *raid5_wq; 73 74 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) 75 { 76 int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; 77 return &conf->stripe_hashtbl[hash]; 78 } 79 80 static inline int stripe_hash_locks_hash(sector_t sect) 81 { 82 return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; 83 } 84 85 static inline void lock_device_hash_lock(struct r5conf *conf, int hash) 86 { 87 spin_lock_irq(conf->hash_locks + hash); 88 spin_lock(&conf->device_lock); 89 } 90 91 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) 92 { 93 spin_unlock(&conf->device_lock); 94 spin_unlock_irq(conf->hash_locks + hash); 95 } 96 97 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) 98 { 99 int i; 100 local_irq_disable(); 101 spin_lock(conf->hash_locks); 102 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 103 spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); 104 spin_lock(&conf->device_lock); 105 } 106 107 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) 108 { 109 int i; 110 spin_unlock(&conf->device_lock); 111 for (i = NR_STRIPE_HASH_LOCKS; i; i--) 112 spin_unlock(conf->hash_locks + i - 1); 113 local_irq_enable(); 114 } 115 116 /* Find first data disk in a raid6 stripe */ 117 static inline int raid6_d0(struct stripe_head *sh) 118 { 119 if (sh->ddf_layout) 120 /* ddf always start from first device */ 121 return 0; 122 /* md starts just after Q block */ 123 if (sh->qd_idx == sh->disks - 1) 124 return 0; 125 else 126 return sh->qd_idx + 1; 127 } 128 static inline int raid6_next_disk(int disk, int raid_disks) 129 { 130 disk++; 131 return (disk < raid_disks) ? disk : 0; 132 } 133 134 /* When walking through the disks in a raid5, starting at raid6_d0, 135 * We need to map each disk to a 'slot', where the data disks are slot 136 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk 137 * is raid_disks-1. This help does that mapping. 138 */ 139 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, 140 int *count, int syndrome_disks) 141 { 142 int slot = *count; 143 144 if (sh->ddf_layout) 145 (*count)++; 146 if (idx == sh->pd_idx) 147 return syndrome_disks; 148 if (idx == sh->qd_idx) 149 return syndrome_disks + 1; 150 if (!sh->ddf_layout) 151 (*count)++; 152 return slot; 153 } 154 155 static void return_io(struct bio_list *return_bi) 156 { 157 struct bio *bi; 158 while ((bi = bio_list_pop(return_bi)) != NULL) { 159 bi->bi_iter.bi_size = 0; 160 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 161 bi, 0); 162 bio_endio(bi); 163 } 164 } 165 166 static void print_raid5_conf (struct r5conf *conf); 167 168 static int stripe_operations_active(struct stripe_head *sh) 169 { 170 return sh->check_state || sh->reconstruct_state || 171 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || 172 test_bit(STRIPE_COMPUTE_RUN, &sh->state); 173 } 174 175 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) 176 { 177 struct r5conf *conf = sh->raid_conf; 178 struct r5worker_group *group; 179 int thread_cnt; 180 int i, cpu = sh->cpu; 181 182 if (!cpu_online(cpu)) { 183 cpu = cpumask_any(cpu_online_mask); 184 sh->cpu = cpu; 185 } 186 187 if (list_empty(&sh->lru)) { 188 struct r5worker_group *group; 189 group = conf->worker_groups + cpu_to_group(cpu); 190 list_add_tail(&sh->lru, &group->handle_list); 191 group->stripes_cnt++; 192 sh->group = group; 193 } 194 195 if (conf->worker_cnt_per_group == 0) { 196 md_wakeup_thread(conf->mddev->thread); 197 return; 198 } 199 200 group = conf->worker_groups + cpu_to_group(sh->cpu); 201 202 group->workers[0].working = true; 203 /* at least one worker should run to avoid race */ 204 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); 205 206 thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; 207 /* wakeup more workers */ 208 for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { 209 if (group->workers[i].working == false) { 210 group->workers[i].working = true; 211 queue_work_on(sh->cpu, raid5_wq, 212 &group->workers[i].work); 213 thread_cnt--; 214 } 215 } 216 } 217 218 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, 219 struct list_head *temp_inactive_list) 220 { 221 int i; 222 int injournal = 0; /* number of date pages with R5_InJournal */ 223 224 BUG_ON(!list_empty(&sh->lru)); 225 BUG_ON(atomic_read(&conf->active_stripes)==0); 226 227 if (r5c_is_writeback(conf->log)) 228 for (i = sh->disks; i--; ) 229 if (test_bit(R5_InJournal, &sh->dev[i].flags)) 230 injournal++; 231 /* 232 * When quiesce in r5c write back, set STRIPE_HANDLE for stripes with 233 * data in journal, so they are not released to cached lists 234 */ 235 if (conf->quiesce && r5c_is_writeback(conf->log) && 236 !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0) { 237 if (test_bit(STRIPE_R5C_CACHING, &sh->state)) 238 r5c_make_stripe_write_out(sh); 239 set_bit(STRIPE_HANDLE, &sh->state); 240 } 241 242 if (test_bit(STRIPE_HANDLE, &sh->state)) { 243 if (test_bit(STRIPE_DELAYED, &sh->state) && 244 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 245 list_add_tail(&sh->lru, &conf->delayed_list); 246 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && 247 sh->bm_seq - conf->seq_write > 0) 248 list_add_tail(&sh->lru, &conf->bitmap_list); 249 else { 250 clear_bit(STRIPE_DELAYED, &sh->state); 251 clear_bit(STRIPE_BIT_DELAY, &sh->state); 252 if (conf->worker_cnt_per_group == 0) { 253 list_add_tail(&sh->lru, &conf->handle_list); 254 } else { 255 raid5_wakeup_stripe_thread(sh); 256 return; 257 } 258 } 259 md_wakeup_thread(conf->mddev->thread); 260 } else { 261 BUG_ON(stripe_operations_active(sh)); 262 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 263 if (atomic_dec_return(&conf->preread_active_stripes) 264 < IO_THRESHOLD) 265 md_wakeup_thread(conf->mddev->thread); 266 atomic_dec(&conf->active_stripes); 267 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { 268 if (!r5c_is_writeback(conf->log)) 269 list_add_tail(&sh->lru, temp_inactive_list); 270 else { 271 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags)); 272 if (injournal == 0) 273 list_add_tail(&sh->lru, temp_inactive_list); 274 else if (injournal == conf->raid_disks - conf->max_degraded) { 275 /* full stripe */ 276 if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) 277 atomic_inc(&conf->r5c_cached_full_stripes); 278 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) 279 atomic_dec(&conf->r5c_cached_partial_stripes); 280 list_add_tail(&sh->lru, &conf->r5c_full_stripe_list); 281 r5c_check_cached_full_stripe(conf); 282 } else { 283 /* partial stripe */ 284 if (!test_and_set_bit(STRIPE_R5C_PARTIAL_STRIPE, 285 &sh->state)) 286 atomic_inc(&conf->r5c_cached_partial_stripes); 287 list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list); 288 } 289 } 290 } 291 } 292 } 293 294 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, 295 struct list_head *temp_inactive_list) 296 { 297 if (atomic_dec_and_test(&sh->count)) 298 do_release_stripe(conf, sh, temp_inactive_list); 299 } 300 301 /* 302 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list 303 * 304 * Be careful: Only one task can add/delete stripes from temp_inactive_list at 305 * given time. Adding stripes only takes device lock, while deleting stripes 306 * only takes hash lock. 307 */ 308 static void release_inactive_stripe_list(struct r5conf *conf, 309 struct list_head *temp_inactive_list, 310 int hash) 311 { 312 int size; 313 bool do_wakeup = false; 314 unsigned long flags; 315 316 if (hash == NR_STRIPE_HASH_LOCKS) { 317 size = NR_STRIPE_HASH_LOCKS; 318 hash = NR_STRIPE_HASH_LOCKS - 1; 319 } else 320 size = 1; 321 while (size) { 322 struct list_head *list = &temp_inactive_list[size - 1]; 323 324 /* 325 * We don't hold any lock here yet, raid5_get_active_stripe() might 326 * remove stripes from the list 327 */ 328 if (!list_empty_careful(list)) { 329 spin_lock_irqsave(conf->hash_locks + hash, flags); 330 if (list_empty(conf->inactive_list + hash) && 331 !list_empty(list)) 332 atomic_dec(&conf->empty_inactive_list_nr); 333 list_splice_tail_init(list, conf->inactive_list + hash); 334 do_wakeup = true; 335 spin_unlock_irqrestore(conf->hash_locks + hash, flags); 336 } 337 size--; 338 hash--; 339 } 340 341 if (do_wakeup) { 342 wake_up(&conf->wait_for_stripe); 343 if (atomic_read(&conf->active_stripes) == 0) 344 wake_up(&conf->wait_for_quiescent); 345 if (conf->retry_read_aligned) 346 md_wakeup_thread(conf->mddev->thread); 347 } 348 } 349 350 /* should hold conf->device_lock already */ 351 static int release_stripe_list(struct r5conf *conf, 352 struct list_head *temp_inactive_list) 353 { 354 struct stripe_head *sh; 355 int count = 0; 356 struct llist_node *head; 357 358 head = llist_del_all(&conf->released_stripes); 359 head = llist_reverse_order(head); 360 while (head) { 361 int hash; 362 363 sh = llist_entry(head, struct stripe_head, release_list); 364 head = llist_next(head); 365 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ 366 smp_mb(); 367 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); 368 /* 369 * Don't worry the bit is set here, because if the bit is set 370 * again, the count is always > 1. This is true for 371 * STRIPE_ON_UNPLUG_LIST bit too. 372 */ 373 hash = sh->hash_lock_index; 374 __release_stripe(conf, sh, &temp_inactive_list[hash]); 375 count++; 376 } 377 378 return count; 379 } 380 381 void raid5_release_stripe(struct stripe_head *sh) 382 { 383 struct r5conf *conf = sh->raid_conf; 384 unsigned long flags; 385 struct list_head list; 386 int hash; 387 bool wakeup; 388 389 /* Avoid release_list until the last reference. 390 */ 391 if (atomic_add_unless(&sh->count, -1, 1)) 392 return; 393 394 if (unlikely(!conf->mddev->thread) || 395 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) 396 goto slow_path; 397 wakeup = llist_add(&sh->release_list, &conf->released_stripes); 398 if (wakeup) 399 md_wakeup_thread(conf->mddev->thread); 400 return; 401 slow_path: 402 local_irq_save(flags); 403 /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ 404 if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { 405 INIT_LIST_HEAD(&list); 406 hash = sh->hash_lock_index; 407 do_release_stripe(conf, sh, &list); 408 spin_unlock(&conf->device_lock); 409 release_inactive_stripe_list(conf, &list, hash); 410 } 411 local_irq_restore(flags); 412 } 413 414 static inline void remove_hash(struct stripe_head *sh) 415 { 416 pr_debug("remove_hash(), stripe %llu\n", 417 (unsigned long long)sh->sector); 418 419 hlist_del_init(&sh->hash); 420 } 421 422 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) 423 { 424 struct hlist_head *hp = stripe_hash(conf, sh->sector); 425 426 pr_debug("insert_hash(), stripe %llu\n", 427 (unsigned long long)sh->sector); 428 429 hlist_add_head(&sh->hash, hp); 430 } 431 432 /* find an idle stripe, make sure it is unhashed, and return it. */ 433 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) 434 { 435 struct stripe_head *sh = NULL; 436 struct list_head *first; 437 438 if (list_empty(conf->inactive_list + hash)) 439 goto out; 440 first = (conf->inactive_list + hash)->next; 441 sh = list_entry(first, struct stripe_head, lru); 442 list_del_init(first); 443 remove_hash(sh); 444 atomic_inc(&conf->active_stripes); 445 BUG_ON(hash != sh->hash_lock_index); 446 if (list_empty(conf->inactive_list + hash)) 447 atomic_inc(&conf->empty_inactive_list_nr); 448 out: 449 return sh; 450 } 451 452 static void shrink_buffers(struct stripe_head *sh) 453 { 454 struct page *p; 455 int i; 456 int num = sh->raid_conf->pool_size; 457 458 for (i = 0; i < num ; i++) { 459 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); 460 p = sh->dev[i].page; 461 if (!p) 462 continue; 463 sh->dev[i].page = NULL; 464 put_page(p); 465 } 466 } 467 468 static int grow_buffers(struct stripe_head *sh, gfp_t gfp) 469 { 470 int i; 471 int num = sh->raid_conf->pool_size; 472 473 for (i = 0; i < num; i++) { 474 struct page *page; 475 476 if (!(page = alloc_page(gfp))) { 477 return 1; 478 } 479 sh->dev[i].page = page; 480 sh->dev[i].orig_page = page; 481 } 482 return 0; 483 } 484 485 static void raid5_build_block(struct stripe_head *sh, int i, int previous); 486 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 487 struct stripe_head *sh); 488 489 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) 490 { 491 struct r5conf *conf = sh->raid_conf; 492 int i, seq; 493 494 BUG_ON(atomic_read(&sh->count) != 0); 495 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); 496 BUG_ON(stripe_operations_active(sh)); 497 BUG_ON(sh->batch_head); 498 499 pr_debug("init_stripe called, stripe %llu\n", 500 (unsigned long long)sector); 501 retry: 502 seq = read_seqcount_begin(&conf->gen_lock); 503 sh->generation = conf->generation - previous; 504 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; 505 sh->sector = sector; 506 stripe_set_idx(sector, conf, previous, sh); 507 sh->state = 0; 508 509 for (i = sh->disks; i--; ) { 510 struct r5dev *dev = &sh->dev[i]; 511 512 if (dev->toread || dev->read || dev->towrite || dev->written || 513 test_bit(R5_LOCKED, &dev->flags)) { 514 pr_err("sector=%llx i=%d %p %p %p %p %d\n", 515 (unsigned long long)sh->sector, i, dev->toread, 516 dev->read, dev->towrite, dev->written, 517 test_bit(R5_LOCKED, &dev->flags)); 518 WARN_ON(1); 519 } 520 dev->flags = 0; 521 raid5_build_block(sh, i, previous); 522 } 523 if (read_seqcount_retry(&conf->gen_lock, seq)) 524 goto retry; 525 sh->overwrite_disks = 0; 526 insert_hash(conf, sh); 527 sh->cpu = smp_processor_id(); 528 set_bit(STRIPE_BATCH_READY, &sh->state); 529 } 530 531 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, 532 short generation) 533 { 534 struct stripe_head *sh; 535 536 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); 537 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) 538 if (sh->sector == sector && sh->generation == generation) 539 return sh; 540 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); 541 return NULL; 542 } 543 544 /* 545 * Need to check if array has failed when deciding whether to: 546 * - start an array 547 * - remove non-faulty devices 548 * - add a spare 549 * - allow a reshape 550 * This determination is simple when no reshape is happening. 551 * However if there is a reshape, we need to carefully check 552 * both the before and after sections. 553 * This is because some failed devices may only affect one 554 * of the two sections, and some non-in_sync devices may 555 * be insync in the section most affected by failed devices. 556 */ 557 static int calc_degraded(struct r5conf *conf) 558 { 559 int degraded, degraded2; 560 int i; 561 562 rcu_read_lock(); 563 degraded = 0; 564 for (i = 0; i < conf->previous_raid_disks; i++) { 565 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 566 if (rdev && test_bit(Faulty, &rdev->flags)) 567 rdev = rcu_dereference(conf->disks[i].replacement); 568 if (!rdev || test_bit(Faulty, &rdev->flags)) 569 degraded++; 570 else if (test_bit(In_sync, &rdev->flags)) 571 ; 572 else 573 /* not in-sync or faulty. 574 * If the reshape increases the number of devices, 575 * this is being recovered by the reshape, so 576 * this 'previous' section is not in_sync. 577 * If the number of devices is being reduced however, 578 * the device can only be part of the array if 579 * we are reverting a reshape, so this section will 580 * be in-sync. 581 */ 582 if (conf->raid_disks >= conf->previous_raid_disks) 583 degraded++; 584 } 585 rcu_read_unlock(); 586 if (conf->raid_disks == conf->previous_raid_disks) 587 return degraded; 588 rcu_read_lock(); 589 degraded2 = 0; 590 for (i = 0; i < conf->raid_disks; i++) { 591 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 592 if (rdev && test_bit(Faulty, &rdev->flags)) 593 rdev = rcu_dereference(conf->disks[i].replacement); 594 if (!rdev || test_bit(Faulty, &rdev->flags)) 595 degraded2++; 596 else if (test_bit(In_sync, &rdev->flags)) 597 ; 598 else 599 /* not in-sync or faulty. 600 * If reshape increases the number of devices, this 601 * section has already been recovered, else it 602 * almost certainly hasn't. 603 */ 604 if (conf->raid_disks <= conf->previous_raid_disks) 605 degraded2++; 606 } 607 rcu_read_unlock(); 608 if (degraded2 > degraded) 609 return degraded2; 610 return degraded; 611 } 612 613 static int has_failed(struct r5conf *conf) 614 { 615 int degraded; 616 617 if (conf->mddev->reshape_position == MaxSector) 618 return conf->mddev->degraded > conf->max_degraded; 619 620 degraded = calc_degraded(conf); 621 if (degraded > conf->max_degraded) 622 return 1; 623 return 0; 624 } 625 626 struct stripe_head * 627 raid5_get_active_stripe(struct r5conf *conf, sector_t sector, 628 int previous, int noblock, int noquiesce) 629 { 630 struct stripe_head *sh; 631 int hash = stripe_hash_locks_hash(sector); 632 int inc_empty_inactive_list_flag; 633 634 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); 635 636 spin_lock_irq(conf->hash_locks + hash); 637 638 do { 639 wait_event_lock_irq(conf->wait_for_quiescent, 640 conf->quiesce == 0 || noquiesce, 641 *(conf->hash_locks + hash)); 642 sh = __find_stripe(conf, sector, conf->generation - previous); 643 if (!sh) { 644 if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { 645 sh = get_free_stripe(conf, hash); 646 if (!sh && !test_bit(R5_DID_ALLOC, 647 &conf->cache_state)) 648 set_bit(R5_ALLOC_MORE, 649 &conf->cache_state); 650 } 651 if (noblock && sh == NULL) 652 break; 653 654 r5c_check_stripe_cache_usage(conf); 655 if (!sh) { 656 set_bit(R5_INACTIVE_BLOCKED, 657 &conf->cache_state); 658 r5l_wake_reclaim(conf->log, 0); 659 wait_event_lock_irq( 660 conf->wait_for_stripe, 661 !list_empty(conf->inactive_list + hash) && 662 (atomic_read(&conf->active_stripes) 663 < (conf->max_nr_stripes * 3 / 4) 664 || !test_bit(R5_INACTIVE_BLOCKED, 665 &conf->cache_state)), 666 *(conf->hash_locks + hash)); 667 clear_bit(R5_INACTIVE_BLOCKED, 668 &conf->cache_state); 669 } else { 670 init_stripe(sh, sector, previous); 671 atomic_inc(&sh->count); 672 } 673 } else if (!atomic_inc_not_zero(&sh->count)) { 674 spin_lock(&conf->device_lock); 675 if (!atomic_read(&sh->count)) { 676 if (!test_bit(STRIPE_HANDLE, &sh->state)) 677 atomic_inc(&conf->active_stripes); 678 BUG_ON(list_empty(&sh->lru) && 679 !test_bit(STRIPE_EXPANDING, &sh->state)); 680 inc_empty_inactive_list_flag = 0; 681 if (!list_empty(conf->inactive_list + hash)) 682 inc_empty_inactive_list_flag = 1; 683 list_del_init(&sh->lru); 684 if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag) 685 atomic_inc(&conf->empty_inactive_list_nr); 686 if (sh->group) { 687 sh->group->stripes_cnt--; 688 sh->group = NULL; 689 } 690 } 691 atomic_inc(&sh->count); 692 spin_unlock(&conf->device_lock); 693 } 694 } while (sh == NULL); 695 696 spin_unlock_irq(conf->hash_locks + hash); 697 return sh; 698 } 699 700 static bool is_full_stripe_write(struct stripe_head *sh) 701 { 702 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); 703 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); 704 } 705 706 static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) 707 { 708 local_irq_disable(); 709 if (sh1 > sh2) { 710 spin_lock(&sh2->stripe_lock); 711 spin_lock_nested(&sh1->stripe_lock, 1); 712 } else { 713 spin_lock(&sh1->stripe_lock); 714 spin_lock_nested(&sh2->stripe_lock, 1); 715 } 716 } 717 718 static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) 719 { 720 spin_unlock(&sh1->stripe_lock); 721 spin_unlock(&sh2->stripe_lock); 722 local_irq_enable(); 723 } 724 725 /* Only freshly new full stripe normal write stripe can be added to a batch list */ 726 static bool stripe_can_batch(struct stripe_head *sh) 727 { 728 struct r5conf *conf = sh->raid_conf; 729 730 if (conf->log) 731 return false; 732 return test_bit(STRIPE_BATCH_READY, &sh->state) && 733 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && 734 is_full_stripe_write(sh); 735 } 736 737 /* we only do back search */ 738 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) 739 { 740 struct stripe_head *head; 741 sector_t head_sector, tmp_sec; 742 int hash; 743 int dd_idx; 744 int inc_empty_inactive_list_flag; 745 746 /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */ 747 tmp_sec = sh->sector; 748 if (!sector_div(tmp_sec, conf->chunk_sectors)) 749 return; 750 head_sector = sh->sector - STRIPE_SECTORS; 751 752 hash = stripe_hash_locks_hash(head_sector); 753 spin_lock_irq(conf->hash_locks + hash); 754 head = __find_stripe(conf, head_sector, conf->generation); 755 if (head && !atomic_inc_not_zero(&head->count)) { 756 spin_lock(&conf->device_lock); 757 if (!atomic_read(&head->count)) { 758 if (!test_bit(STRIPE_HANDLE, &head->state)) 759 atomic_inc(&conf->active_stripes); 760 BUG_ON(list_empty(&head->lru) && 761 !test_bit(STRIPE_EXPANDING, &head->state)); 762 inc_empty_inactive_list_flag = 0; 763 if (!list_empty(conf->inactive_list + hash)) 764 inc_empty_inactive_list_flag = 1; 765 list_del_init(&head->lru); 766 if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag) 767 atomic_inc(&conf->empty_inactive_list_nr); 768 if (head->group) { 769 head->group->stripes_cnt--; 770 head->group = NULL; 771 } 772 } 773 atomic_inc(&head->count); 774 spin_unlock(&conf->device_lock); 775 } 776 spin_unlock_irq(conf->hash_locks + hash); 777 778 if (!head) 779 return; 780 if (!stripe_can_batch(head)) 781 goto out; 782 783 lock_two_stripes(head, sh); 784 /* clear_batch_ready clear the flag */ 785 if (!stripe_can_batch(head) || !stripe_can_batch(sh)) 786 goto unlock_out; 787 788 if (sh->batch_head) 789 goto unlock_out; 790 791 dd_idx = 0; 792 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) 793 dd_idx++; 794 if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf || 795 bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite)) 796 goto unlock_out; 797 798 if (head->batch_head) { 799 spin_lock(&head->batch_head->batch_lock); 800 /* This batch list is already running */ 801 if (!stripe_can_batch(head)) { 802 spin_unlock(&head->batch_head->batch_lock); 803 goto unlock_out; 804 } 805 806 /* 807 * at this point, head's BATCH_READY could be cleared, but we 808 * can still add the stripe to batch list 809 */ 810 list_add(&sh->batch_list, &head->batch_list); 811 spin_unlock(&head->batch_head->batch_lock); 812 813 sh->batch_head = head->batch_head; 814 } else { 815 head->batch_head = head; 816 sh->batch_head = head->batch_head; 817 spin_lock(&head->batch_lock); 818 list_add_tail(&sh->batch_list, &head->batch_list); 819 spin_unlock(&head->batch_lock); 820 } 821 822 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 823 if (atomic_dec_return(&conf->preread_active_stripes) 824 < IO_THRESHOLD) 825 md_wakeup_thread(conf->mddev->thread); 826 827 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { 828 int seq = sh->bm_seq; 829 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && 830 sh->batch_head->bm_seq > seq) 831 seq = sh->batch_head->bm_seq; 832 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); 833 sh->batch_head->bm_seq = seq; 834 } 835 836 atomic_inc(&sh->count); 837 unlock_out: 838 unlock_two_stripes(head, sh); 839 out: 840 raid5_release_stripe(head); 841 } 842 843 /* Determine if 'data_offset' or 'new_data_offset' should be used 844 * in this stripe_head. 845 */ 846 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) 847 { 848 sector_t progress = conf->reshape_progress; 849 /* Need a memory barrier to make sure we see the value 850 * of conf->generation, or ->data_offset that was set before 851 * reshape_progress was updated. 852 */ 853 smp_rmb(); 854 if (progress == MaxSector) 855 return 0; 856 if (sh->generation == conf->generation - 1) 857 return 0; 858 /* We are in a reshape, and this is a new-generation stripe, 859 * so use new_data_offset. 860 */ 861 return 1; 862 } 863 864 static void 865 raid5_end_read_request(struct bio *bi); 866 static void 867 raid5_end_write_request(struct bio *bi); 868 869 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) 870 { 871 struct r5conf *conf = sh->raid_conf; 872 int i, disks = sh->disks; 873 struct stripe_head *head_sh = sh; 874 875 might_sleep(); 876 877 if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) { 878 /* writing out phase */ 879 if (s->waiting_extra_page) 880 return; 881 if (r5l_write_stripe(conf->log, sh) == 0) 882 return; 883 } else { /* caching phase */ 884 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) { 885 r5c_cache_data(conf->log, sh, s); 886 return; 887 } 888 } 889 890 for (i = disks; i--; ) { 891 int op, op_flags = 0; 892 int replace_only = 0; 893 struct bio *bi, *rbi; 894 struct md_rdev *rdev, *rrdev = NULL; 895 896 sh = head_sh; 897 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { 898 op = REQ_OP_WRITE; 899 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) 900 op_flags = REQ_FUA; 901 if (test_bit(R5_Discard, &sh->dev[i].flags)) 902 op = REQ_OP_DISCARD; 903 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) 904 op = REQ_OP_READ; 905 else if (test_and_clear_bit(R5_WantReplace, 906 &sh->dev[i].flags)) { 907 op = REQ_OP_WRITE; 908 replace_only = 1; 909 } else 910 continue; 911 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) 912 op_flags |= REQ_SYNC; 913 914 again: 915 bi = &sh->dev[i].req; 916 rbi = &sh->dev[i].rreq; /* For writing to replacement */ 917 918 rcu_read_lock(); 919 rrdev = rcu_dereference(conf->disks[i].replacement); 920 smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ 921 rdev = rcu_dereference(conf->disks[i].rdev); 922 if (!rdev) { 923 rdev = rrdev; 924 rrdev = NULL; 925 } 926 if (op_is_write(op)) { 927 if (replace_only) 928 rdev = NULL; 929 if (rdev == rrdev) 930 /* We raced and saw duplicates */ 931 rrdev = NULL; 932 } else { 933 if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev) 934 rdev = rrdev; 935 rrdev = NULL; 936 } 937 938 if (rdev && test_bit(Faulty, &rdev->flags)) 939 rdev = NULL; 940 if (rdev) 941 atomic_inc(&rdev->nr_pending); 942 if (rrdev && test_bit(Faulty, &rrdev->flags)) 943 rrdev = NULL; 944 if (rrdev) 945 atomic_inc(&rrdev->nr_pending); 946 rcu_read_unlock(); 947 948 /* We have already checked bad blocks for reads. Now 949 * need to check for writes. We never accept write errors 950 * on the replacement, so we don't to check rrdev. 951 */ 952 while (op_is_write(op) && rdev && 953 test_bit(WriteErrorSeen, &rdev->flags)) { 954 sector_t first_bad; 955 int bad_sectors; 956 int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 957 &first_bad, &bad_sectors); 958 if (!bad) 959 break; 960 961 if (bad < 0) { 962 set_bit(BlockedBadBlocks, &rdev->flags); 963 if (!conf->mddev->external && 964 conf->mddev->sb_flags) { 965 /* It is very unlikely, but we might 966 * still need to write out the 967 * bad block log - better give it 968 * a chance*/ 969 md_check_recovery(conf->mddev); 970 } 971 /* 972 * Because md_wait_for_blocked_rdev 973 * will dec nr_pending, we must 974 * increment it first. 975 */ 976 atomic_inc(&rdev->nr_pending); 977 md_wait_for_blocked_rdev(rdev, conf->mddev); 978 } else { 979 /* Acknowledged bad block - skip the write */ 980 rdev_dec_pending(rdev, conf->mddev); 981 rdev = NULL; 982 } 983 } 984 985 if (rdev) { 986 if (s->syncing || s->expanding || s->expanded 987 || s->replacing) 988 md_sync_acct(rdev->bdev, STRIPE_SECTORS); 989 990 set_bit(STRIPE_IO_STARTED, &sh->state); 991 992 bi->bi_bdev = rdev->bdev; 993 bio_set_op_attrs(bi, op, op_flags); 994 bi->bi_end_io = op_is_write(op) 995 ? raid5_end_write_request 996 : raid5_end_read_request; 997 bi->bi_private = sh; 998 999 pr_debug("%s: for %llu schedule op %d on disc %d\n", 1000 __func__, (unsigned long long)sh->sector, 1001 bi->bi_opf, i); 1002 atomic_inc(&sh->count); 1003 if (sh != head_sh) 1004 atomic_inc(&head_sh->count); 1005 if (use_new_offset(conf, sh)) 1006 bi->bi_iter.bi_sector = (sh->sector 1007 + rdev->new_data_offset); 1008 else 1009 bi->bi_iter.bi_sector = (sh->sector 1010 + rdev->data_offset); 1011 if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags)) 1012 bi->bi_opf |= REQ_NOMERGE; 1013 1014 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 1015 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 1016 sh->dev[i].vec.bv_page = sh->dev[i].page; 1017 bi->bi_vcnt = 1; 1018 bi->bi_io_vec[0].bv_len = STRIPE_SIZE; 1019 bi->bi_io_vec[0].bv_offset = 0; 1020 bi->bi_iter.bi_size = STRIPE_SIZE; 1021 /* 1022 * If this is discard request, set bi_vcnt 0. We don't 1023 * want to confuse SCSI because SCSI will replace payload 1024 */ 1025 if (op == REQ_OP_DISCARD) 1026 bi->bi_vcnt = 0; 1027 if (rrdev) 1028 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); 1029 1030 if (conf->mddev->gendisk) 1031 trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), 1032 bi, disk_devt(conf->mddev->gendisk), 1033 sh->dev[i].sector); 1034 generic_make_request(bi); 1035 } 1036 if (rrdev) { 1037 if (s->syncing || s->expanding || s->expanded 1038 || s->replacing) 1039 md_sync_acct(rrdev->bdev, STRIPE_SECTORS); 1040 1041 set_bit(STRIPE_IO_STARTED, &sh->state); 1042 1043 rbi->bi_bdev = rrdev->bdev; 1044 bio_set_op_attrs(rbi, op, op_flags); 1045 BUG_ON(!op_is_write(op)); 1046 rbi->bi_end_io = raid5_end_write_request; 1047 rbi->bi_private = sh; 1048 1049 pr_debug("%s: for %llu schedule op %d on " 1050 "replacement disc %d\n", 1051 __func__, (unsigned long long)sh->sector, 1052 rbi->bi_opf, i); 1053 atomic_inc(&sh->count); 1054 if (sh != head_sh) 1055 atomic_inc(&head_sh->count); 1056 if (use_new_offset(conf, sh)) 1057 rbi->bi_iter.bi_sector = (sh->sector 1058 + rrdev->new_data_offset); 1059 else 1060 rbi->bi_iter.bi_sector = (sh->sector 1061 + rrdev->data_offset); 1062 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) 1063 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 1064 sh->dev[i].rvec.bv_page = sh->dev[i].page; 1065 rbi->bi_vcnt = 1; 1066 rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; 1067 rbi->bi_io_vec[0].bv_offset = 0; 1068 rbi->bi_iter.bi_size = STRIPE_SIZE; 1069 /* 1070 * If this is discard request, set bi_vcnt 0. We don't 1071 * want to confuse SCSI because SCSI will replace payload 1072 */ 1073 if (op == REQ_OP_DISCARD) 1074 rbi->bi_vcnt = 0; 1075 if (conf->mddev->gendisk) 1076 trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), 1077 rbi, disk_devt(conf->mddev->gendisk), 1078 sh->dev[i].sector); 1079 generic_make_request(rbi); 1080 } 1081 if (!rdev && !rrdev) { 1082 if (op_is_write(op)) 1083 set_bit(STRIPE_DEGRADED, &sh->state); 1084 pr_debug("skip op %d on disc %d for sector %llu\n", 1085 bi->bi_opf, i, (unsigned long long)sh->sector); 1086 clear_bit(R5_LOCKED, &sh->dev[i].flags); 1087 set_bit(STRIPE_HANDLE, &sh->state); 1088 } 1089 1090 if (!head_sh->batch_head) 1091 continue; 1092 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1093 batch_list); 1094 if (sh != head_sh) 1095 goto again; 1096 } 1097 } 1098 1099 static struct dma_async_tx_descriptor * 1100 async_copy_data(int frombio, struct bio *bio, struct page **page, 1101 sector_t sector, struct dma_async_tx_descriptor *tx, 1102 struct stripe_head *sh, int no_skipcopy) 1103 { 1104 struct bio_vec bvl; 1105 struct bvec_iter iter; 1106 struct page *bio_page; 1107 int page_offset; 1108 struct async_submit_ctl submit; 1109 enum async_tx_flags flags = 0; 1110 1111 if (bio->bi_iter.bi_sector >= sector) 1112 page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; 1113 else 1114 page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; 1115 1116 if (frombio) 1117 flags |= ASYNC_TX_FENCE; 1118 init_async_submit(&submit, flags, tx, NULL, NULL, NULL); 1119 1120 bio_for_each_segment(bvl, bio, iter) { 1121 int len = bvl.bv_len; 1122 int clen; 1123 int b_offset = 0; 1124 1125 if (page_offset < 0) { 1126 b_offset = -page_offset; 1127 page_offset += b_offset; 1128 len -= b_offset; 1129 } 1130 1131 if (len > 0 && page_offset + len > STRIPE_SIZE) 1132 clen = STRIPE_SIZE - page_offset; 1133 else 1134 clen = len; 1135 1136 if (clen > 0) { 1137 b_offset += bvl.bv_offset; 1138 bio_page = bvl.bv_page; 1139 if (frombio) { 1140 if (sh->raid_conf->skip_copy && 1141 b_offset == 0 && page_offset == 0 && 1142 clen == STRIPE_SIZE && 1143 !no_skipcopy) 1144 *page = bio_page; 1145 else 1146 tx = async_memcpy(*page, bio_page, page_offset, 1147 b_offset, clen, &submit); 1148 } else 1149 tx = async_memcpy(bio_page, *page, b_offset, 1150 page_offset, clen, &submit); 1151 } 1152 /* chain the operations */ 1153 submit.depend_tx = tx; 1154 1155 if (clen < len) /* hit end of page */ 1156 break; 1157 page_offset += len; 1158 } 1159 1160 return tx; 1161 } 1162 1163 static void ops_complete_biofill(void *stripe_head_ref) 1164 { 1165 struct stripe_head *sh = stripe_head_ref; 1166 struct bio_list return_bi = BIO_EMPTY_LIST; 1167 int i; 1168 1169 pr_debug("%s: stripe %llu\n", __func__, 1170 (unsigned long long)sh->sector); 1171 1172 /* clear completed biofills */ 1173 for (i = sh->disks; i--; ) { 1174 struct r5dev *dev = &sh->dev[i]; 1175 1176 /* acknowledge completion of a biofill operation */ 1177 /* and check if we need to reply to a read request, 1178 * new R5_Wantfill requests are held off until 1179 * !STRIPE_BIOFILL_RUN 1180 */ 1181 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { 1182 struct bio *rbi, *rbi2; 1183 1184 BUG_ON(!dev->read); 1185 rbi = dev->read; 1186 dev->read = NULL; 1187 while (rbi && rbi->bi_iter.bi_sector < 1188 dev->sector + STRIPE_SECTORS) { 1189 rbi2 = r5_next_bio(rbi, dev->sector); 1190 if (!raid5_dec_bi_active_stripes(rbi)) 1191 bio_list_add(&return_bi, rbi); 1192 rbi = rbi2; 1193 } 1194 } 1195 } 1196 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); 1197 1198 return_io(&return_bi); 1199 1200 set_bit(STRIPE_HANDLE, &sh->state); 1201 raid5_release_stripe(sh); 1202 } 1203 1204 static void ops_run_biofill(struct stripe_head *sh) 1205 { 1206 struct dma_async_tx_descriptor *tx = NULL; 1207 struct async_submit_ctl submit; 1208 int i; 1209 1210 BUG_ON(sh->batch_head); 1211 pr_debug("%s: stripe %llu\n", __func__, 1212 (unsigned long long)sh->sector); 1213 1214 for (i = sh->disks; i--; ) { 1215 struct r5dev *dev = &sh->dev[i]; 1216 if (test_bit(R5_Wantfill, &dev->flags)) { 1217 struct bio *rbi; 1218 spin_lock_irq(&sh->stripe_lock); 1219 dev->read = rbi = dev->toread; 1220 dev->toread = NULL; 1221 spin_unlock_irq(&sh->stripe_lock); 1222 while (rbi && rbi->bi_iter.bi_sector < 1223 dev->sector + STRIPE_SECTORS) { 1224 tx = async_copy_data(0, rbi, &dev->page, 1225 dev->sector, tx, sh, 0); 1226 rbi = r5_next_bio(rbi, dev->sector); 1227 } 1228 } 1229 } 1230 1231 atomic_inc(&sh->count); 1232 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); 1233 async_trigger_callback(&submit); 1234 } 1235 1236 static void mark_target_uptodate(struct stripe_head *sh, int target) 1237 { 1238 struct r5dev *tgt; 1239 1240 if (target < 0) 1241 return; 1242 1243 tgt = &sh->dev[target]; 1244 set_bit(R5_UPTODATE, &tgt->flags); 1245 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1246 clear_bit(R5_Wantcompute, &tgt->flags); 1247 } 1248 1249 static void ops_complete_compute(void *stripe_head_ref) 1250 { 1251 struct stripe_head *sh = stripe_head_ref; 1252 1253 pr_debug("%s: stripe %llu\n", __func__, 1254 (unsigned long long)sh->sector); 1255 1256 /* mark the computed target(s) as uptodate */ 1257 mark_target_uptodate(sh, sh->ops.target); 1258 mark_target_uptodate(sh, sh->ops.target2); 1259 1260 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); 1261 if (sh->check_state == check_state_compute_run) 1262 sh->check_state = check_state_compute_result; 1263 set_bit(STRIPE_HANDLE, &sh->state); 1264 raid5_release_stripe(sh); 1265 } 1266 1267 /* return a pointer to the address conversion region of the scribble buffer */ 1268 static addr_conv_t *to_addr_conv(struct stripe_head *sh, 1269 struct raid5_percpu *percpu, int i) 1270 { 1271 void *addr; 1272 1273 addr = flex_array_get(percpu->scribble, i); 1274 return addr + sizeof(struct page *) * (sh->disks + 2); 1275 } 1276 1277 /* return a pointer to the address conversion region of the scribble buffer */ 1278 static struct page **to_addr_page(struct raid5_percpu *percpu, int i) 1279 { 1280 void *addr; 1281 1282 addr = flex_array_get(percpu->scribble, i); 1283 return addr; 1284 } 1285 1286 static struct dma_async_tx_descriptor * 1287 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) 1288 { 1289 int disks = sh->disks; 1290 struct page **xor_srcs = to_addr_page(percpu, 0); 1291 int target = sh->ops.target; 1292 struct r5dev *tgt = &sh->dev[target]; 1293 struct page *xor_dest = tgt->page; 1294 int count = 0; 1295 struct dma_async_tx_descriptor *tx; 1296 struct async_submit_ctl submit; 1297 int i; 1298 1299 BUG_ON(sh->batch_head); 1300 1301 pr_debug("%s: stripe %llu block: %d\n", 1302 __func__, (unsigned long long)sh->sector, target); 1303 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1304 1305 for (i = disks; i--; ) 1306 if (i != target) 1307 xor_srcs[count++] = sh->dev[i].page; 1308 1309 atomic_inc(&sh->count); 1310 1311 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, 1312 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); 1313 if (unlikely(count == 1)) 1314 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1315 else 1316 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1317 1318 return tx; 1319 } 1320 1321 /* set_syndrome_sources - populate source buffers for gen_syndrome 1322 * @srcs - (struct page *) array of size sh->disks 1323 * @sh - stripe_head to parse 1324 * 1325 * Populates srcs in proper layout order for the stripe and returns the 1326 * 'count' of sources to be used in a call to async_gen_syndrome. The P 1327 * destination buffer is recorded in srcs[count] and the Q destination 1328 * is recorded in srcs[count+1]]. 1329 */ 1330 static int set_syndrome_sources(struct page **srcs, 1331 struct stripe_head *sh, 1332 int srctype) 1333 { 1334 int disks = sh->disks; 1335 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); 1336 int d0_idx = raid6_d0(sh); 1337 int count; 1338 int i; 1339 1340 for (i = 0; i < disks; i++) 1341 srcs[i] = NULL; 1342 1343 count = 0; 1344 i = d0_idx; 1345 do { 1346 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1347 struct r5dev *dev = &sh->dev[i]; 1348 1349 if (i == sh->qd_idx || i == sh->pd_idx || 1350 (srctype == SYNDROME_SRC_ALL) || 1351 (srctype == SYNDROME_SRC_WANT_DRAIN && 1352 (test_bit(R5_Wantdrain, &dev->flags) || 1353 test_bit(R5_InJournal, &dev->flags))) || 1354 (srctype == SYNDROME_SRC_WRITTEN && 1355 dev->written)) { 1356 if (test_bit(R5_InJournal, &dev->flags)) 1357 srcs[slot] = sh->dev[i].orig_page; 1358 else 1359 srcs[slot] = sh->dev[i].page; 1360 } 1361 i = raid6_next_disk(i, disks); 1362 } while (i != d0_idx); 1363 1364 return syndrome_disks; 1365 } 1366 1367 static struct dma_async_tx_descriptor * 1368 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) 1369 { 1370 int disks = sh->disks; 1371 struct page **blocks = to_addr_page(percpu, 0); 1372 int target; 1373 int qd_idx = sh->qd_idx; 1374 struct dma_async_tx_descriptor *tx; 1375 struct async_submit_ctl submit; 1376 struct r5dev *tgt; 1377 struct page *dest; 1378 int i; 1379 int count; 1380 1381 BUG_ON(sh->batch_head); 1382 if (sh->ops.target < 0) 1383 target = sh->ops.target2; 1384 else if (sh->ops.target2 < 0) 1385 target = sh->ops.target; 1386 else 1387 /* we should only have one valid target */ 1388 BUG(); 1389 BUG_ON(target < 0); 1390 pr_debug("%s: stripe %llu block: %d\n", 1391 __func__, (unsigned long long)sh->sector, target); 1392 1393 tgt = &sh->dev[target]; 1394 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1395 dest = tgt->page; 1396 1397 atomic_inc(&sh->count); 1398 1399 if (target == qd_idx) { 1400 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); 1401 blocks[count] = NULL; /* regenerating p is not necessary */ 1402 BUG_ON(blocks[count+1] != dest); /* q should already be set */ 1403 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1404 ops_complete_compute, sh, 1405 to_addr_conv(sh, percpu, 0)); 1406 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1407 } else { 1408 /* Compute any data- or p-drive using XOR */ 1409 count = 0; 1410 for (i = disks; i-- ; ) { 1411 if (i == target || i == qd_idx) 1412 continue; 1413 blocks[count++] = sh->dev[i].page; 1414 } 1415 1416 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1417 NULL, ops_complete_compute, sh, 1418 to_addr_conv(sh, percpu, 0)); 1419 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); 1420 } 1421 1422 return tx; 1423 } 1424 1425 static struct dma_async_tx_descriptor * 1426 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) 1427 { 1428 int i, count, disks = sh->disks; 1429 int syndrome_disks = sh->ddf_layout ? disks : disks-2; 1430 int d0_idx = raid6_d0(sh); 1431 int faila = -1, failb = -1; 1432 int target = sh->ops.target; 1433 int target2 = sh->ops.target2; 1434 struct r5dev *tgt = &sh->dev[target]; 1435 struct r5dev *tgt2 = &sh->dev[target2]; 1436 struct dma_async_tx_descriptor *tx; 1437 struct page **blocks = to_addr_page(percpu, 0); 1438 struct async_submit_ctl submit; 1439 1440 BUG_ON(sh->batch_head); 1441 pr_debug("%s: stripe %llu block1: %d block2: %d\n", 1442 __func__, (unsigned long long)sh->sector, target, target2); 1443 BUG_ON(target < 0 || target2 < 0); 1444 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); 1445 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); 1446 1447 /* we need to open-code set_syndrome_sources to handle the 1448 * slot number conversion for 'faila' and 'failb' 1449 */ 1450 for (i = 0; i < disks ; i++) 1451 blocks[i] = NULL; 1452 count = 0; 1453 i = d0_idx; 1454 do { 1455 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); 1456 1457 blocks[slot] = sh->dev[i].page; 1458 1459 if (i == target) 1460 faila = slot; 1461 if (i == target2) 1462 failb = slot; 1463 i = raid6_next_disk(i, disks); 1464 } while (i != d0_idx); 1465 1466 BUG_ON(faila == failb); 1467 if (failb < faila) 1468 swap(faila, failb); 1469 pr_debug("%s: stripe: %llu faila: %d failb: %d\n", 1470 __func__, (unsigned long long)sh->sector, faila, failb); 1471 1472 atomic_inc(&sh->count); 1473 1474 if (failb == syndrome_disks+1) { 1475 /* Q disk is one of the missing disks */ 1476 if (faila == syndrome_disks) { 1477 /* Missing P+Q, just recompute */ 1478 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1479 ops_complete_compute, sh, 1480 to_addr_conv(sh, percpu, 0)); 1481 return async_gen_syndrome(blocks, 0, syndrome_disks+2, 1482 STRIPE_SIZE, &submit); 1483 } else { 1484 struct page *dest; 1485 int data_target; 1486 int qd_idx = sh->qd_idx; 1487 1488 /* Missing D+Q: recompute D from P, then recompute Q */ 1489 if (target == qd_idx) 1490 data_target = target2; 1491 else 1492 data_target = target; 1493 1494 count = 0; 1495 for (i = disks; i-- ; ) { 1496 if (i == data_target || i == qd_idx) 1497 continue; 1498 blocks[count++] = sh->dev[i].page; 1499 } 1500 dest = sh->dev[data_target].page; 1501 init_async_submit(&submit, 1502 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, 1503 NULL, NULL, NULL, 1504 to_addr_conv(sh, percpu, 0)); 1505 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, 1506 &submit); 1507 1508 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); 1509 init_async_submit(&submit, ASYNC_TX_FENCE, tx, 1510 ops_complete_compute, sh, 1511 to_addr_conv(sh, percpu, 0)); 1512 return async_gen_syndrome(blocks, 0, count+2, 1513 STRIPE_SIZE, &submit); 1514 } 1515 } else { 1516 init_async_submit(&submit, ASYNC_TX_FENCE, NULL, 1517 ops_complete_compute, sh, 1518 to_addr_conv(sh, percpu, 0)); 1519 if (failb == syndrome_disks) { 1520 /* We're missing D+P. */ 1521 return async_raid6_datap_recov(syndrome_disks+2, 1522 STRIPE_SIZE, faila, 1523 blocks, &submit); 1524 } else { 1525 /* We're missing D+D. */ 1526 return async_raid6_2data_recov(syndrome_disks+2, 1527 STRIPE_SIZE, faila, failb, 1528 blocks, &submit); 1529 } 1530 } 1531 } 1532 1533 static void ops_complete_prexor(void *stripe_head_ref) 1534 { 1535 struct stripe_head *sh = stripe_head_ref; 1536 1537 pr_debug("%s: stripe %llu\n", __func__, 1538 (unsigned long long)sh->sector); 1539 1540 if (r5c_is_writeback(sh->raid_conf->log)) 1541 /* 1542 * raid5-cache write back uses orig_page during prexor. 1543 * After prexor, it is time to free orig_page 1544 */ 1545 r5c_release_extra_page(sh); 1546 } 1547 1548 static struct dma_async_tx_descriptor * 1549 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, 1550 struct dma_async_tx_descriptor *tx) 1551 { 1552 int disks = sh->disks; 1553 struct page **xor_srcs = to_addr_page(percpu, 0); 1554 int count = 0, pd_idx = sh->pd_idx, i; 1555 struct async_submit_ctl submit; 1556 1557 /* existing parity data subtracted */ 1558 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1559 1560 BUG_ON(sh->batch_head); 1561 pr_debug("%s: stripe %llu\n", __func__, 1562 (unsigned long long)sh->sector); 1563 1564 for (i = disks; i--; ) { 1565 struct r5dev *dev = &sh->dev[i]; 1566 /* Only process blocks that are known to be uptodate */ 1567 if (test_bit(R5_InJournal, &dev->flags)) 1568 xor_srcs[count++] = dev->orig_page; 1569 else if (test_bit(R5_Wantdrain, &dev->flags)) 1570 xor_srcs[count++] = dev->page; 1571 } 1572 1573 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, 1574 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); 1575 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1576 1577 return tx; 1578 } 1579 1580 static struct dma_async_tx_descriptor * 1581 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, 1582 struct dma_async_tx_descriptor *tx) 1583 { 1584 struct page **blocks = to_addr_page(percpu, 0); 1585 int count; 1586 struct async_submit_ctl submit; 1587 1588 pr_debug("%s: stripe %llu\n", __func__, 1589 (unsigned long long)sh->sector); 1590 1591 count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN); 1592 1593 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx, 1594 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); 1595 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1596 1597 return tx; 1598 } 1599 1600 static struct dma_async_tx_descriptor * 1601 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) 1602 { 1603 struct r5conf *conf = sh->raid_conf; 1604 int disks = sh->disks; 1605 int i; 1606 struct stripe_head *head_sh = sh; 1607 1608 pr_debug("%s: stripe %llu\n", __func__, 1609 (unsigned long long)sh->sector); 1610 1611 for (i = disks; i--; ) { 1612 struct r5dev *dev; 1613 struct bio *chosen; 1614 1615 sh = head_sh; 1616 if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) { 1617 struct bio *wbi; 1618 1619 again: 1620 dev = &sh->dev[i]; 1621 /* 1622 * clear R5_InJournal, so when rewriting a page in 1623 * journal, it is not skipped by r5l_log_stripe() 1624 */ 1625 clear_bit(R5_InJournal, &dev->flags); 1626 spin_lock_irq(&sh->stripe_lock); 1627 chosen = dev->towrite; 1628 dev->towrite = NULL; 1629 sh->overwrite_disks = 0; 1630 BUG_ON(dev->written); 1631 wbi = dev->written = chosen; 1632 spin_unlock_irq(&sh->stripe_lock); 1633 WARN_ON(dev->page != dev->orig_page); 1634 1635 while (wbi && wbi->bi_iter.bi_sector < 1636 dev->sector + STRIPE_SECTORS) { 1637 if (wbi->bi_opf & REQ_FUA) 1638 set_bit(R5_WantFUA, &dev->flags); 1639 if (wbi->bi_opf & REQ_SYNC) 1640 set_bit(R5_SyncIO, &dev->flags); 1641 if (bio_op(wbi) == REQ_OP_DISCARD) 1642 set_bit(R5_Discard, &dev->flags); 1643 else { 1644 tx = async_copy_data(1, wbi, &dev->page, 1645 dev->sector, tx, sh, 1646 r5c_is_writeback(conf->log)); 1647 if (dev->page != dev->orig_page && 1648 !r5c_is_writeback(conf->log)) { 1649 set_bit(R5_SkipCopy, &dev->flags); 1650 clear_bit(R5_UPTODATE, &dev->flags); 1651 clear_bit(R5_OVERWRITE, &dev->flags); 1652 } 1653 } 1654 wbi = r5_next_bio(wbi, dev->sector); 1655 } 1656 1657 if (head_sh->batch_head) { 1658 sh = list_first_entry(&sh->batch_list, 1659 struct stripe_head, 1660 batch_list); 1661 if (sh == head_sh) 1662 continue; 1663 goto again; 1664 } 1665 } 1666 } 1667 1668 return tx; 1669 } 1670 1671 static void ops_complete_reconstruct(void *stripe_head_ref) 1672 { 1673 struct stripe_head *sh = stripe_head_ref; 1674 int disks = sh->disks; 1675 int pd_idx = sh->pd_idx; 1676 int qd_idx = sh->qd_idx; 1677 int i; 1678 bool fua = false, sync = false, discard = false; 1679 1680 pr_debug("%s: stripe %llu\n", __func__, 1681 (unsigned long long)sh->sector); 1682 1683 for (i = disks; i--; ) { 1684 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); 1685 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); 1686 discard |= test_bit(R5_Discard, &sh->dev[i].flags); 1687 } 1688 1689 for (i = disks; i--; ) { 1690 struct r5dev *dev = &sh->dev[i]; 1691 1692 if (dev->written || i == pd_idx || i == qd_idx) { 1693 if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) 1694 set_bit(R5_UPTODATE, &dev->flags); 1695 if (fua) 1696 set_bit(R5_WantFUA, &dev->flags); 1697 if (sync) 1698 set_bit(R5_SyncIO, &dev->flags); 1699 } 1700 } 1701 1702 if (sh->reconstruct_state == reconstruct_state_drain_run) 1703 sh->reconstruct_state = reconstruct_state_drain_result; 1704 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) 1705 sh->reconstruct_state = reconstruct_state_prexor_drain_result; 1706 else { 1707 BUG_ON(sh->reconstruct_state != reconstruct_state_run); 1708 sh->reconstruct_state = reconstruct_state_result; 1709 } 1710 1711 set_bit(STRIPE_HANDLE, &sh->state); 1712 raid5_release_stripe(sh); 1713 } 1714 1715 static void 1716 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, 1717 struct dma_async_tx_descriptor *tx) 1718 { 1719 int disks = sh->disks; 1720 struct page **xor_srcs; 1721 struct async_submit_ctl submit; 1722 int count, pd_idx = sh->pd_idx, i; 1723 struct page *xor_dest; 1724 int prexor = 0; 1725 unsigned long flags; 1726 int j = 0; 1727 struct stripe_head *head_sh = sh; 1728 int last_stripe; 1729 1730 pr_debug("%s: stripe %llu\n", __func__, 1731 (unsigned long long)sh->sector); 1732 1733 for (i = 0; i < sh->disks; i++) { 1734 if (pd_idx == i) 1735 continue; 1736 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1737 break; 1738 } 1739 if (i >= sh->disks) { 1740 atomic_inc(&sh->count); 1741 set_bit(R5_Discard, &sh->dev[pd_idx].flags); 1742 ops_complete_reconstruct(sh); 1743 return; 1744 } 1745 again: 1746 count = 0; 1747 xor_srcs = to_addr_page(percpu, j); 1748 /* check if prexor is active which means only process blocks 1749 * that are part of a read-modify-write (written) 1750 */ 1751 if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1752 prexor = 1; 1753 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; 1754 for (i = disks; i--; ) { 1755 struct r5dev *dev = &sh->dev[i]; 1756 if (head_sh->dev[i].written || 1757 test_bit(R5_InJournal, &head_sh->dev[i].flags)) 1758 xor_srcs[count++] = dev->page; 1759 } 1760 } else { 1761 xor_dest = sh->dev[pd_idx].page; 1762 for (i = disks; i--; ) { 1763 struct r5dev *dev = &sh->dev[i]; 1764 if (i != pd_idx) 1765 xor_srcs[count++] = dev->page; 1766 } 1767 } 1768 1769 /* 1/ if we prexor'd then the dest is reused as a source 1770 * 2/ if we did not prexor then we are redoing the parity 1771 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST 1772 * for the synchronous xor case 1773 */ 1774 last_stripe = !head_sh->batch_head || 1775 list_first_entry(&sh->batch_list, 1776 struct stripe_head, batch_list) == head_sh; 1777 if (last_stripe) { 1778 flags = ASYNC_TX_ACK | 1779 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); 1780 1781 atomic_inc(&head_sh->count); 1782 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh, 1783 to_addr_conv(sh, percpu, j)); 1784 } else { 1785 flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST; 1786 init_async_submit(&submit, flags, tx, NULL, NULL, 1787 to_addr_conv(sh, percpu, j)); 1788 } 1789 1790 if (unlikely(count == 1)) 1791 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); 1792 else 1793 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); 1794 if (!last_stripe) { 1795 j++; 1796 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1797 batch_list); 1798 goto again; 1799 } 1800 } 1801 1802 static void 1803 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, 1804 struct dma_async_tx_descriptor *tx) 1805 { 1806 struct async_submit_ctl submit; 1807 struct page **blocks; 1808 int count, i, j = 0; 1809 struct stripe_head *head_sh = sh; 1810 int last_stripe; 1811 int synflags; 1812 unsigned long txflags; 1813 1814 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); 1815 1816 for (i = 0; i < sh->disks; i++) { 1817 if (sh->pd_idx == i || sh->qd_idx == i) 1818 continue; 1819 if (!test_bit(R5_Discard, &sh->dev[i].flags)) 1820 break; 1821 } 1822 if (i >= sh->disks) { 1823 atomic_inc(&sh->count); 1824 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 1825 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 1826 ops_complete_reconstruct(sh); 1827 return; 1828 } 1829 1830 again: 1831 blocks = to_addr_page(percpu, j); 1832 1833 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { 1834 synflags = SYNDROME_SRC_WRITTEN; 1835 txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST; 1836 } else { 1837 synflags = SYNDROME_SRC_ALL; 1838 txflags = ASYNC_TX_ACK; 1839 } 1840 1841 count = set_syndrome_sources(blocks, sh, synflags); 1842 last_stripe = !head_sh->batch_head || 1843 list_first_entry(&sh->batch_list, 1844 struct stripe_head, batch_list) == head_sh; 1845 1846 if (last_stripe) { 1847 atomic_inc(&head_sh->count); 1848 init_async_submit(&submit, txflags, tx, ops_complete_reconstruct, 1849 head_sh, to_addr_conv(sh, percpu, j)); 1850 } else 1851 init_async_submit(&submit, 0, tx, NULL, NULL, 1852 to_addr_conv(sh, percpu, j)); 1853 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); 1854 if (!last_stripe) { 1855 j++; 1856 sh = list_first_entry(&sh->batch_list, struct stripe_head, 1857 batch_list); 1858 goto again; 1859 } 1860 } 1861 1862 static void ops_complete_check(void *stripe_head_ref) 1863 { 1864 struct stripe_head *sh = stripe_head_ref; 1865 1866 pr_debug("%s: stripe %llu\n", __func__, 1867 (unsigned long long)sh->sector); 1868 1869 sh->check_state = check_state_check_result; 1870 set_bit(STRIPE_HANDLE, &sh->state); 1871 raid5_release_stripe(sh); 1872 } 1873 1874 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) 1875 { 1876 int disks = sh->disks; 1877 int pd_idx = sh->pd_idx; 1878 int qd_idx = sh->qd_idx; 1879 struct page *xor_dest; 1880 struct page **xor_srcs = to_addr_page(percpu, 0); 1881 struct dma_async_tx_descriptor *tx; 1882 struct async_submit_ctl submit; 1883 int count; 1884 int i; 1885 1886 pr_debug("%s: stripe %llu\n", __func__, 1887 (unsigned long long)sh->sector); 1888 1889 BUG_ON(sh->batch_head); 1890 count = 0; 1891 xor_dest = sh->dev[pd_idx].page; 1892 xor_srcs[count++] = xor_dest; 1893 for (i = disks; i--; ) { 1894 if (i == pd_idx || i == qd_idx) 1895 continue; 1896 xor_srcs[count++] = sh->dev[i].page; 1897 } 1898 1899 init_async_submit(&submit, 0, NULL, NULL, NULL, 1900 to_addr_conv(sh, percpu, 0)); 1901 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, 1902 &sh->ops.zero_sum_result, &submit); 1903 1904 atomic_inc(&sh->count); 1905 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); 1906 tx = async_trigger_callback(&submit); 1907 } 1908 1909 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) 1910 { 1911 struct page **srcs = to_addr_page(percpu, 0); 1912 struct async_submit_ctl submit; 1913 int count; 1914 1915 pr_debug("%s: stripe %llu checkp: %d\n", __func__, 1916 (unsigned long long)sh->sector, checkp); 1917 1918 BUG_ON(sh->batch_head); 1919 count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL); 1920 if (!checkp) 1921 srcs[count] = NULL; 1922 1923 atomic_inc(&sh->count); 1924 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, 1925 sh, to_addr_conv(sh, percpu, 0)); 1926 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, 1927 &sh->ops.zero_sum_result, percpu->spare_page, &submit); 1928 } 1929 1930 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) 1931 { 1932 int overlap_clear = 0, i, disks = sh->disks; 1933 struct dma_async_tx_descriptor *tx = NULL; 1934 struct r5conf *conf = sh->raid_conf; 1935 int level = conf->level; 1936 struct raid5_percpu *percpu; 1937 unsigned long cpu; 1938 1939 cpu = get_cpu(); 1940 percpu = per_cpu_ptr(conf->percpu, cpu); 1941 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { 1942 ops_run_biofill(sh); 1943 overlap_clear++; 1944 } 1945 1946 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { 1947 if (level < 6) 1948 tx = ops_run_compute5(sh, percpu); 1949 else { 1950 if (sh->ops.target2 < 0 || sh->ops.target < 0) 1951 tx = ops_run_compute6_1(sh, percpu); 1952 else 1953 tx = ops_run_compute6_2(sh, percpu); 1954 } 1955 /* terminate the chain if reconstruct is not set to be run */ 1956 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) 1957 async_tx_ack(tx); 1958 } 1959 1960 if (test_bit(STRIPE_OP_PREXOR, &ops_request)) { 1961 if (level < 6) 1962 tx = ops_run_prexor5(sh, percpu, tx); 1963 else 1964 tx = ops_run_prexor6(sh, percpu, tx); 1965 } 1966 1967 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { 1968 tx = ops_run_biodrain(sh, tx); 1969 overlap_clear++; 1970 } 1971 1972 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { 1973 if (level < 6) 1974 ops_run_reconstruct5(sh, percpu, tx); 1975 else 1976 ops_run_reconstruct6(sh, percpu, tx); 1977 } 1978 1979 if (test_bit(STRIPE_OP_CHECK, &ops_request)) { 1980 if (sh->check_state == check_state_run) 1981 ops_run_check_p(sh, percpu); 1982 else if (sh->check_state == check_state_run_q) 1983 ops_run_check_pq(sh, percpu, 0); 1984 else if (sh->check_state == check_state_run_pq) 1985 ops_run_check_pq(sh, percpu, 1); 1986 else 1987 BUG(); 1988 } 1989 1990 if (overlap_clear && !sh->batch_head) 1991 for (i = disks; i--; ) { 1992 struct r5dev *dev = &sh->dev[i]; 1993 if (test_and_clear_bit(R5_Overlap, &dev->flags)) 1994 wake_up(&sh->raid_conf->wait_for_overlap); 1995 } 1996 put_cpu(); 1997 } 1998 1999 static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp, 2000 int disks) 2001 { 2002 struct stripe_head *sh; 2003 int i; 2004 2005 sh = kmem_cache_zalloc(sc, gfp); 2006 if (sh) { 2007 spin_lock_init(&sh->stripe_lock); 2008 spin_lock_init(&sh->batch_lock); 2009 INIT_LIST_HEAD(&sh->batch_list); 2010 INIT_LIST_HEAD(&sh->lru); 2011 INIT_LIST_HEAD(&sh->r5c); 2012 INIT_LIST_HEAD(&sh->log_list); 2013 atomic_set(&sh->count, 1); 2014 sh->log_start = MaxSector; 2015 for (i = 0; i < disks; i++) { 2016 struct r5dev *dev = &sh->dev[i]; 2017 2018 bio_init(&dev->req, &dev->vec, 1); 2019 bio_init(&dev->rreq, &dev->rvec, 1); 2020 } 2021 } 2022 return sh; 2023 } 2024 static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) 2025 { 2026 struct stripe_head *sh; 2027 2028 sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size); 2029 if (!sh) 2030 return 0; 2031 2032 sh->raid_conf = conf; 2033 2034 if (grow_buffers(sh, gfp)) { 2035 shrink_buffers(sh); 2036 kmem_cache_free(conf->slab_cache, sh); 2037 return 0; 2038 } 2039 sh->hash_lock_index = 2040 conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; 2041 /* we just created an active stripe so... */ 2042 atomic_inc(&conf->active_stripes); 2043 2044 raid5_release_stripe(sh); 2045 conf->max_nr_stripes++; 2046 return 1; 2047 } 2048 2049 static int grow_stripes(struct r5conf *conf, int num) 2050 { 2051 struct kmem_cache *sc; 2052 int devs = max(conf->raid_disks, conf->previous_raid_disks); 2053 2054 if (conf->mddev->gendisk) 2055 sprintf(conf->cache_name[0], 2056 "raid%d-%s", conf->level, mdname(conf->mddev)); 2057 else 2058 sprintf(conf->cache_name[0], 2059 "raid%d-%p", conf->level, conf->mddev); 2060 sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); 2061 2062 conf->active_name = 0; 2063 sc = kmem_cache_create(conf->cache_name[conf->active_name], 2064 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 2065 0, 0, NULL); 2066 if (!sc) 2067 return 1; 2068 conf->slab_cache = sc; 2069 conf->pool_size = devs; 2070 while (num--) 2071 if (!grow_one_stripe(conf, GFP_KERNEL)) 2072 return 1; 2073 2074 return 0; 2075 } 2076 2077 /** 2078 * scribble_len - return the required size of the scribble region 2079 * @num - total number of disks in the array 2080 * 2081 * The size must be enough to contain: 2082 * 1/ a struct page pointer for each device in the array +2 2083 * 2/ room to convert each entry in (1) to its corresponding dma 2084 * (dma_map_page()) or page (page_address()) address. 2085 * 2086 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we 2087 * calculate over all devices (not just the data blocks), using zeros in place 2088 * of the P and Q blocks. 2089 */ 2090 static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) 2091 { 2092 struct flex_array *ret; 2093 size_t len; 2094 2095 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); 2096 ret = flex_array_alloc(len, cnt, flags); 2097 if (!ret) 2098 return NULL; 2099 /* always prealloc all elements, so no locking is required */ 2100 if (flex_array_prealloc(ret, 0, cnt, flags)) { 2101 flex_array_free(ret); 2102 return NULL; 2103 } 2104 return ret; 2105 } 2106 2107 static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) 2108 { 2109 unsigned long cpu; 2110 int err = 0; 2111 2112 /* 2113 * Never shrink. And mddev_suspend() could deadlock if this is called 2114 * from raid5d. In that case, scribble_disks and scribble_sectors 2115 * should equal to new_disks and new_sectors 2116 */ 2117 if (conf->scribble_disks >= new_disks && 2118 conf->scribble_sectors >= new_sectors) 2119 return 0; 2120 mddev_suspend(conf->mddev); 2121 get_online_cpus(); 2122 for_each_present_cpu(cpu) { 2123 struct raid5_percpu *percpu; 2124 struct flex_array *scribble; 2125 2126 percpu = per_cpu_ptr(conf->percpu, cpu); 2127 scribble = scribble_alloc(new_disks, 2128 new_sectors / STRIPE_SECTORS, 2129 GFP_NOIO); 2130 2131 if (scribble) { 2132 flex_array_free(percpu->scribble); 2133 percpu->scribble = scribble; 2134 } else { 2135 err = -ENOMEM; 2136 break; 2137 } 2138 } 2139 put_online_cpus(); 2140 mddev_resume(conf->mddev); 2141 if (!err) { 2142 conf->scribble_disks = new_disks; 2143 conf->scribble_sectors = new_sectors; 2144 } 2145 return err; 2146 } 2147 2148 static int resize_stripes(struct r5conf *conf, int newsize) 2149 { 2150 /* Make all the stripes able to hold 'newsize' devices. 2151 * New slots in each stripe get 'page' set to a new page. 2152 * 2153 * This happens in stages: 2154 * 1/ create a new kmem_cache and allocate the required number of 2155 * stripe_heads. 2156 * 2/ gather all the old stripe_heads and transfer the pages across 2157 * to the new stripe_heads. This will have the side effect of 2158 * freezing the array as once all stripe_heads have been collected, 2159 * no IO will be possible. Old stripe heads are freed once their 2160 * pages have been transferred over, and the old kmem_cache is 2161 * freed when all stripes are done. 2162 * 3/ reallocate conf->disks to be suitable bigger. If this fails, 2163 * we simple return a failre status - no need to clean anything up. 2164 * 4/ allocate new pages for the new slots in the new stripe_heads. 2165 * If this fails, we don't bother trying the shrink the 2166 * stripe_heads down again, we just leave them as they are. 2167 * As each stripe_head is processed the new one is released into 2168 * active service. 2169 * 2170 * Once step2 is started, we cannot afford to wait for a write, 2171 * so we use GFP_NOIO allocations. 2172 */ 2173 struct stripe_head *osh, *nsh; 2174 LIST_HEAD(newstripes); 2175 struct disk_info *ndisks; 2176 int err; 2177 struct kmem_cache *sc; 2178 int i; 2179 int hash, cnt; 2180 2181 if (newsize <= conf->pool_size) 2182 return 0; /* never bother to shrink */ 2183 2184 err = md_allow_write(conf->mddev); 2185 if (err) 2186 return err; 2187 2188 /* Step 1 */ 2189 sc = kmem_cache_create(conf->cache_name[1-conf->active_name], 2190 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 2191 0, 0, NULL); 2192 if (!sc) 2193 return -ENOMEM; 2194 2195 /* Need to ensure auto-resizing doesn't interfere */ 2196 mutex_lock(&conf->cache_size_mutex); 2197 2198 for (i = conf->max_nr_stripes; i; i--) { 2199 nsh = alloc_stripe(sc, GFP_KERNEL, newsize); 2200 if (!nsh) 2201 break; 2202 2203 nsh->raid_conf = conf; 2204 list_add(&nsh->lru, &newstripes); 2205 } 2206 if (i) { 2207 /* didn't get enough, give up */ 2208 while (!list_empty(&newstripes)) { 2209 nsh = list_entry(newstripes.next, struct stripe_head, lru); 2210 list_del(&nsh->lru); 2211 kmem_cache_free(sc, nsh); 2212 } 2213 kmem_cache_destroy(sc); 2214 mutex_unlock(&conf->cache_size_mutex); 2215 return -ENOMEM; 2216 } 2217 /* Step 2 - Must use GFP_NOIO now. 2218 * OK, we have enough stripes, start collecting inactive 2219 * stripes and copying them over 2220 */ 2221 hash = 0; 2222 cnt = 0; 2223 list_for_each_entry(nsh, &newstripes, lru) { 2224 lock_device_hash_lock(conf, hash); 2225 wait_event_cmd(conf->wait_for_stripe, 2226 !list_empty(conf->inactive_list + hash), 2227 unlock_device_hash_lock(conf, hash), 2228 lock_device_hash_lock(conf, hash)); 2229 osh = get_free_stripe(conf, hash); 2230 unlock_device_hash_lock(conf, hash); 2231 2232 for(i=0; i<conf->pool_size; i++) { 2233 nsh->dev[i].page = osh->dev[i].page; 2234 nsh->dev[i].orig_page = osh->dev[i].page; 2235 } 2236 nsh->hash_lock_index = hash; 2237 kmem_cache_free(conf->slab_cache, osh); 2238 cnt++; 2239 if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + 2240 !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { 2241 hash++; 2242 cnt = 0; 2243 } 2244 } 2245 kmem_cache_destroy(conf->slab_cache); 2246 2247 /* Step 3. 2248 * At this point, we are holding all the stripes so the array 2249 * is completely stalled, so now is a good time to resize 2250 * conf->disks and the scribble region 2251 */ 2252 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); 2253 if (ndisks) { 2254 for (i = 0; i < conf->pool_size; i++) 2255 ndisks[i] = conf->disks[i]; 2256 2257 for (i = conf->pool_size; i < newsize; i++) { 2258 ndisks[i].extra_page = alloc_page(GFP_NOIO); 2259 if (!ndisks[i].extra_page) 2260 err = -ENOMEM; 2261 } 2262 2263 if (err) { 2264 for (i = conf->pool_size; i < newsize; i++) 2265 if (ndisks[i].extra_page) 2266 put_page(ndisks[i].extra_page); 2267 kfree(ndisks); 2268 } else { 2269 kfree(conf->disks); 2270 conf->disks = ndisks; 2271 } 2272 } else 2273 err = -ENOMEM; 2274 2275 mutex_unlock(&conf->cache_size_mutex); 2276 /* Step 4, return new stripes to service */ 2277 while(!list_empty(&newstripes)) { 2278 nsh = list_entry(newstripes.next, struct stripe_head, lru); 2279 list_del_init(&nsh->lru); 2280 2281 for (i=conf->raid_disks; i < newsize; i++) 2282 if (nsh->dev[i].page == NULL) { 2283 struct page *p = alloc_page(GFP_NOIO); 2284 nsh->dev[i].page = p; 2285 nsh->dev[i].orig_page = p; 2286 if (!p) 2287 err = -ENOMEM; 2288 } 2289 raid5_release_stripe(nsh); 2290 } 2291 /* critical section pass, GFP_NOIO no longer needed */ 2292 2293 conf->slab_cache = sc; 2294 conf->active_name = 1-conf->active_name; 2295 if (!err) 2296 conf->pool_size = newsize; 2297 return err; 2298 } 2299 2300 static int drop_one_stripe(struct r5conf *conf) 2301 { 2302 struct stripe_head *sh; 2303 int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK; 2304 2305 spin_lock_irq(conf->hash_locks + hash); 2306 sh = get_free_stripe(conf, hash); 2307 spin_unlock_irq(conf->hash_locks + hash); 2308 if (!sh) 2309 return 0; 2310 BUG_ON(atomic_read(&sh->count)); 2311 shrink_buffers(sh); 2312 kmem_cache_free(conf->slab_cache, sh); 2313 atomic_dec(&conf->active_stripes); 2314 conf->max_nr_stripes--; 2315 return 1; 2316 } 2317 2318 static void shrink_stripes(struct r5conf *conf) 2319 { 2320 while (conf->max_nr_stripes && 2321 drop_one_stripe(conf)) 2322 ; 2323 2324 kmem_cache_destroy(conf->slab_cache); 2325 conf->slab_cache = NULL; 2326 } 2327 2328 static void raid5_end_read_request(struct bio * bi) 2329 { 2330 struct stripe_head *sh = bi->bi_private; 2331 struct r5conf *conf = sh->raid_conf; 2332 int disks = sh->disks, i; 2333 char b[BDEVNAME_SIZE]; 2334 struct md_rdev *rdev = NULL; 2335 sector_t s; 2336 2337 for (i=0 ; i<disks; i++) 2338 if (bi == &sh->dev[i].req) 2339 break; 2340 2341 pr_debug("end_read_request %llu/%d, count: %d, error %d.\n", 2342 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2343 bi->bi_error); 2344 if (i == disks) { 2345 bio_reset(bi); 2346 BUG(); 2347 return; 2348 } 2349 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2350 /* If replacement finished while this request was outstanding, 2351 * 'replacement' might be NULL already. 2352 * In that case it moved down to 'rdev'. 2353 * rdev is not removed until all requests are finished. 2354 */ 2355 rdev = conf->disks[i].replacement; 2356 if (!rdev) 2357 rdev = conf->disks[i].rdev; 2358 2359 if (use_new_offset(conf, sh)) 2360 s = sh->sector + rdev->new_data_offset; 2361 else 2362 s = sh->sector + rdev->data_offset; 2363 if (!bi->bi_error) { 2364 set_bit(R5_UPTODATE, &sh->dev[i].flags); 2365 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 2366 /* Note that this cannot happen on a 2367 * replacement device. We just fail those on 2368 * any error 2369 */ 2370 pr_info_ratelimited( 2371 "md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n", 2372 mdname(conf->mddev), STRIPE_SECTORS, 2373 (unsigned long long)s, 2374 bdevname(rdev->bdev, b)); 2375 atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); 2376 clear_bit(R5_ReadError, &sh->dev[i].flags); 2377 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2378 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2379 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2380 2381 if (atomic_read(&rdev->read_errors)) 2382 atomic_set(&rdev->read_errors, 0); 2383 } else { 2384 const char *bdn = bdevname(rdev->bdev, b); 2385 int retry = 0; 2386 int set_bad = 0; 2387 2388 clear_bit(R5_UPTODATE, &sh->dev[i].flags); 2389 atomic_inc(&rdev->read_errors); 2390 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) 2391 pr_warn_ratelimited( 2392 "md/raid:%s: read error on replacement device (sector %llu on %s).\n", 2393 mdname(conf->mddev), 2394 (unsigned long long)s, 2395 bdn); 2396 else if (conf->mddev->degraded >= conf->max_degraded) { 2397 set_bad = 1; 2398 pr_warn_ratelimited( 2399 "md/raid:%s: read error not correctable (sector %llu on %s).\n", 2400 mdname(conf->mddev), 2401 (unsigned long long)s, 2402 bdn); 2403 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { 2404 /* Oh, no!!! */ 2405 set_bad = 1; 2406 pr_warn_ratelimited( 2407 "md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n", 2408 mdname(conf->mddev), 2409 (unsigned long long)s, 2410 bdn); 2411 } else if (atomic_read(&rdev->read_errors) 2412 > conf->max_nr_stripes) 2413 pr_warn("md/raid:%s: Too many read errors, failing device %s.\n", 2414 mdname(conf->mddev), bdn); 2415 else 2416 retry = 1; 2417 if (set_bad && test_bit(In_sync, &rdev->flags) 2418 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) 2419 retry = 1; 2420 if (retry) 2421 if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { 2422 set_bit(R5_ReadError, &sh->dev[i].flags); 2423 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2424 } else 2425 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); 2426 else { 2427 clear_bit(R5_ReadError, &sh->dev[i].flags); 2428 clear_bit(R5_ReWrite, &sh->dev[i].flags); 2429 if (!(set_bad 2430 && test_bit(In_sync, &rdev->flags) 2431 && rdev_set_badblocks( 2432 rdev, sh->sector, STRIPE_SECTORS, 0))) 2433 md_error(conf->mddev, rdev); 2434 } 2435 } 2436 rdev_dec_pending(rdev, conf->mddev); 2437 bio_reset(bi); 2438 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2439 set_bit(STRIPE_HANDLE, &sh->state); 2440 raid5_release_stripe(sh); 2441 } 2442 2443 static void raid5_end_write_request(struct bio *bi) 2444 { 2445 struct stripe_head *sh = bi->bi_private; 2446 struct r5conf *conf = sh->raid_conf; 2447 int disks = sh->disks, i; 2448 struct md_rdev *uninitialized_var(rdev); 2449 sector_t first_bad; 2450 int bad_sectors; 2451 int replacement = 0; 2452 2453 for (i = 0 ; i < disks; i++) { 2454 if (bi == &sh->dev[i].req) { 2455 rdev = conf->disks[i].rdev; 2456 break; 2457 } 2458 if (bi == &sh->dev[i].rreq) { 2459 rdev = conf->disks[i].replacement; 2460 if (rdev) 2461 replacement = 1; 2462 else 2463 /* rdev was removed and 'replacement' 2464 * replaced it. rdev is not removed 2465 * until all requests are finished. 2466 */ 2467 rdev = conf->disks[i].rdev; 2468 break; 2469 } 2470 } 2471 pr_debug("end_write_request %llu/%d, count %d, error: %d.\n", 2472 (unsigned long long)sh->sector, i, atomic_read(&sh->count), 2473 bi->bi_error); 2474 if (i == disks) { 2475 bio_reset(bi); 2476 BUG(); 2477 return; 2478 } 2479 2480 if (replacement) { 2481 if (bi->bi_error) 2482 md_error(conf->mddev, rdev); 2483 else if (is_badblock(rdev, sh->sector, 2484 STRIPE_SECTORS, 2485 &first_bad, &bad_sectors)) 2486 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); 2487 } else { 2488 if (bi->bi_error) { 2489 set_bit(STRIPE_DEGRADED, &sh->state); 2490 set_bit(WriteErrorSeen, &rdev->flags); 2491 set_bit(R5_WriteError, &sh->dev[i].flags); 2492 if (!test_and_set_bit(WantReplacement, &rdev->flags)) 2493 set_bit(MD_RECOVERY_NEEDED, 2494 &rdev->mddev->recovery); 2495 } else if (is_badblock(rdev, sh->sector, 2496 STRIPE_SECTORS, 2497 &first_bad, &bad_sectors)) { 2498 set_bit(R5_MadeGood, &sh->dev[i].flags); 2499 if (test_bit(R5_ReadError, &sh->dev[i].flags)) 2500 /* That was a successful write so make 2501 * sure it looks like we already did 2502 * a re-write. 2503 */ 2504 set_bit(R5_ReWrite, &sh->dev[i].flags); 2505 } 2506 } 2507 rdev_dec_pending(rdev, conf->mddev); 2508 2509 if (sh->batch_head && bi->bi_error && !replacement) 2510 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); 2511 2512 bio_reset(bi); 2513 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) 2514 clear_bit(R5_LOCKED, &sh->dev[i].flags); 2515 set_bit(STRIPE_HANDLE, &sh->state); 2516 raid5_release_stripe(sh); 2517 2518 if (sh->batch_head && sh != sh->batch_head) 2519 raid5_release_stripe(sh->batch_head); 2520 } 2521 2522 static void raid5_build_block(struct stripe_head *sh, int i, int previous) 2523 { 2524 struct r5dev *dev = &sh->dev[i]; 2525 2526 dev->flags = 0; 2527 dev->sector = raid5_compute_blocknr(sh, i, previous); 2528 } 2529 2530 static void raid5_error(struct mddev *mddev, struct md_rdev *rdev) 2531 { 2532 char b[BDEVNAME_SIZE]; 2533 struct r5conf *conf = mddev->private; 2534 unsigned long flags; 2535 pr_debug("raid456: error called\n"); 2536 2537 spin_lock_irqsave(&conf->device_lock, flags); 2538 clear_bit(In_sync, &rdev->flags); 2539 mddev->degraded = calc_degraded(conf); 2540 spin_unlock_irqrestore(&conf->device_lock, flags); 2541 set_bit(MD_RECOVERY_INTR, &mddev->recovery); 2542 2543 set_bit(Blocked, &rdev->flags); 2544 set_bit(Faulty, &rdev->flags); 2545 set_mask_bits(&mddev->sb_flags, 0, 2546 BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); 2547 pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n" 2548 "md/raid:%s: Operation continuing on %d devices.\n", 2549 mdname(mddev), 2550 bdevname(rdev->bdev, b), 2551 mdname(mddev), 2552 conf->raid_disks - mddev->degraded); 2553 } 2554 2555 /* 2556 * Input: a 'big' sector number, 2557 * Output: index of the data and parity disk, and the sector # in them. 2558 */ 2559 sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, 2560 int previous, int *dd_idx, 2561 struct stripe_head *sh) 2562 { 2563 sector_t stripe, stripe2; 2564 sector_t chunk_number; 2565 unsigned int chunk_offset; 2566 int pd_idx, qd_idx; 2567 int ddf_layout = 0; 2568 sector_t new_sector; 2569 int algorithm = previous ? conf->prev_algo 2570 : conf->algorithm; 2571 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2572 : conf->chunk_sectors; 2573 int raid_disks = previous ? conf->previous_raid_disks 2574 : conf->raid_disks; 2575 int data_disks = raid_disks - conf->max_degraded; 2576 2577 /* First compute the information on this sector */ 2578 2579 /* 2580 * Compute the chunk number and the sector offset inside the chunk 2581 */ 2582 chunk_offset = sector_div(r_sector, sectors_per_chunk); 2583 chunk_number = r_sector; 2584 2585 /* 2586 * Compute the stripe number 2587 */ 2588 stripe = chunk_number; 2589 *dd_idx = sector_div(stripe, data_disks); 2590 stripe2 = stripe; 2591 /* 2592 * Select the parity disk based on the user selected algorithm. 2593 */ 2594 pd_idx = qd_idx = -1; 2595 switch(conf->level) { 2596 case 4: 2597 pd_idx = data_disks; 2598 break; 2599 case 5: 2600 switch (algorithm) { 2601 case ALGORITHM_LEFT_ASYMMETRIC: 2602 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2603 if (*dd_idx >= pd_idx) 2604 (*dd_idx)++; 2605 break; 2606 case ALGORITHM_RIGHT_ASYMMETRIC: 2607 pd_idx = sector_div(stripe2, raid_disks); 2608 if (*dd_idx >= pd_idx) 2609 (*dd_idx)++; 2610 break; 2611 case ALGORITHM_LEFT_SYMMETRIC: 2612 pd_idx = data_disks - sector_div(stripe2, raid_disks); 2613 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2614 break; 2615 case ALGORITHM_RIGHT_SYMMETRIC: 2616 pd_idx = sector_div(stripe2, raid_disks); 2617 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2618 break; 2619 case ALGORITHM_PARITY_0: 2620 pd_idx = 0; 2621 (*dd_idx)++; 2622 break; 2623 case ALGORITHM_PARITY_N: 2624 pd_idx = data_disks; 2625 break; 2626 default: 2627 BUG(); 2628 } 2629 break; 2630 case 6: 2631 2632 switch (algorithm) { 2633 case ALGORITHM_LEFT_ASYMMETRIC: 2634 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2635 qd_idx = pd_idx + 1; 2636 if (pd_idx == raid_disks-1) { 2637 (*dd_idx)++; /* Q D D D P */ 2638 qd_idx = 0; 2639 } else if (*dd_idx >= pd_idx) 2640 (*dd_idx) += 2; /* D D P Q D */ 2641 break; 2642 case ALGORITHM_RIGHT_ASYMMETRIC: 2643 pd_idx = sector_div(stripe2, raid_disks); 2644 qd_idx = pd_idx + 1; 2645 if (pd_idx == raid_disks-1) { 2646 (*dd_idx)++; /* Q D D D P */ 2647 qd_idx = 0; 2648 } else if (*dd_idx >= pd_idx) 2649 (*dd_idx) += 2; /* D D P Q D */ 2650 break; 2651 case ALGORITHM_LEFT_SYMMETRIC: 2652 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2653 qd_idx = (pd_idx + 1) % raid_disks; 2654 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2655 break; 2656 case ALGORITHM_RIGHT_SYMMETRIC: 2657 pd_idx = sector_div(stripe2, raid_disks); 2658 qd_idx = (pd_idx + 1) % raid_disks; 2659 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; 2660 break; 2661 2662 case ALGORITHM_PARITY_0: 2663 pd_idx = 0; 2664 qd_idx = 1; 2665 (*dd_idx) += 2; 2666 break; 2667 case ALGORITHM_PARITY_N: 2668 pd_idx = data_disks; 2669 qd_idx = data_disks + 1; 2670 break; 2671 2672 case ALGORITHM_ROTATING_ZERO_RESTART: 2673 /* Exactly the same as RIGHT_ASYMMETRIC, but or 2674 * of blocks for computing Q is different. 2675 */ 2676 pd_idx = sector_div(stripe2, raid_disks); 2677 qd_idx = pd_idx + 1; 2678 if (pd_idx == raid_disks-1) { 2679 (*dd_idx)++; /* Q D D D P */ 2680 qd_idx = 0; 2681 } else if (*dd_idx >= pd_idx) 2682 (*dd_idx) += 2; /* D D P Q D */ 2683 ddf_layout = 1; 2684 break; 2685 2686 case ALGORITHM_ROTATING_N_RESTART: 2687 /* Same a left_asymmetric, by first stripe is 2688 * D D D P Q rather than 2689 * Q D D D P 2690 */ 2691 stripe2 += 1; 2692 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2693 qd_idx = pd_idx + 1; 2694 if (pd_idx == raid_disks-1) { 2695 (*dd_idx)++; /* Q D D D P */ 2696 qd_idx = 0; 2697 } else if (*dd_idx >= pd_idx) 2698 (*dd_idx) += 2; /* D D P Q D */ 2699 ddf_layout = 1; 2700 break; 2701 2702 case ALGORITHM_ROTATING_N_CONTINUE: 2703 /* Same as left_symmetric but Q is before P */ 2704 pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); 2705 qd_idx = (pd_idx + raid_disks - 1) % raid_disks; 2706 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; 2707 ddf_layout = 1; 2708 break; 2709 2710 case ALGORITHM_LEFT_ASYMMETRIC_6: 2711 /* RAID5 left_asymmetric, with Q on last device */ 2712 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2713 if (*dd_idx >= pd_idx) 2714 (*dd_idx)++; 2715 qd_idx = raid_disks - 1; 2716 break; 2717 2718 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2719 pd_idx = sector_div(stripe2, raid_disks-1); 2720 if (*dd_idx >= pd_idx) 2721 (*dd_idx)++; 2722 qd_idx = raid_disks - 1; 2723 break; 2724 2725 case ALGORITHM_LEFT_SYMMETRIC_6: 2726 pd_idx = data_disks - sector_div(stripe2, raid_disks-1); 2727 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2728 qd_idx = raid_disks - 1; 2729 break; 2730 2731 case ALGORITHM_RIGHT_SYMMETRIC_6: 2732 pd_idx = sector_div(stripe2, raid_disks-1); 2733 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); 2734 qd_idx = raid_disks - 1; 2735 break; 2736 2737 case ALGORITHM_PARITY_0_6: 2738 pd_idx = 0; 2739 (*dd_idx)++; 2740 qd_idx = raid_disks - 1; 2741 break; 2742 2743 default: 2744 BUG(); 2745 } 2746 break; 2747 } 2748 2749 if (sh) { 2750 sh->pd_idx = pd_idx; 2751 sh->qd_idx = qd_idx; 2752 sh->ddf_layout = ddf_layout; 2753 } 2754 /* 2755 * Finally, compute the new sector number 2756 */ 2757 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; 2758 return new_sector; 2759 } 2760 2761 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous) 2762 { 2763 struct r5conf *conf = sh->raid_conf; 2764 int raid_disks = sh->disks; 2765 int data_disks = raid_disks - conf->max_degraded; 2766 sector_t new_sector = sh->sector, check; 2767 int sectors_per_chunk = previous ? conf->prev_chunk_sectors 2768 : conf->chunk_sectors; 2769 int algorithm = previous ? conf->prev_algo 2770 : conf->algorithm; 2771 sector_t stripe; 2772 int chunk_offset; 2773 sector_t chunk_number; 2774 int dummy1, dd_idx = i; 2775 sector_t r_sector; 2776 struct stripe_head sh2; 2777 2778 chunk_offset = sector_div(new_sector, sectors_per_chunk); 2779 stripe = new_sector; 2780 2781 if (i == sh->pd_idx) 2782 return 0; 2783 switch(conf->level) { 2784 case 4: break; 2785 case 5: 2786 switch (algorithm) { 2787 case ALGORITHM_LEFT_ASYMMETRIC: 2788 case ALGORITHM_RIGHT_ASYMMETRIC: 2789 if (i > sh->pd_idx) 2790 i--; 2791 break; 2792 case ALGORITHM_LEFT_SYMMETRIC: 2793 case ALGORITHM_RIGHT_SYMMETRIC: 2794 if (i < sh->pd_idx) 2795 i += raid_disks; 2796 i -= (sh->pd_idx + 1); 2797 break; 2798 case ALGORITHM_PARITY_0: 2799 i -= 1; 2800 break; 2801 case ALGORITHM_PARITY_N: 2802 break; 2803 default: 2804 BUG(); 2805 } 2806 break; 2807 case 6: 2808 if (i == sh->qd_idx) 2809 return 0; /* It is the Q disk */ 2810 switch (algorithm) { 2811 case ALGORITHM_LEFT_ASYMMETRIC: 2812 case ALGORITHM_RIGHT_ASYMMETRIC: 2813 case ALGORITHM_ROTATING_ZERO_RESTART: 2814 case ALGORITHM_ROTATING_N_RESTART: 2815 if (sh->pd_idx == raid_disks-1) 2816 i--; /* Q D D D P */ 2817 else if (i > sh->pd_idx) 2818 i -= 2; /* D D P Q D */ 2819 break; 2820 case ALGORITHM_LEFT_SYMMETRIC: 2821 case ALGORITHM_RIGHT_SYMMETRIC: 2822 if (sh->pd_idx == raid_disks-1) 2823 i--; /* Q D D D P */ 2824 else { 2825 /* D D P Q D */ 2826 if (i < sh->pd_idx) 2827 i += raid_disks; 2828 i -= (sh->pd_idx + 2); 2829 } 2830 break; 2831 case ALGORITHM_PARITY_0: 2832 i -= 2; 2833 break; 2834 case ALGORITHM_PARITY_N: 2835 break; 2836 case ALGORITHM_ROTATING_N_CONTINUE: 2837 /* Like left_symmetric, but P is before Q */ 2838 if (sh->pd_idx == 0) 2839 i--; /* P D D D Q */ 2840 else { 2841 /* D D Q P D */ 2842 if (i < sh->pd_idx) 2843 i += raid_disks; 2844 i -= (sh->pd_idx + 1); 2845 } 2846 break; 2847 case ALGORITHM_LEFT_ASYMMETRIC_6: 2848 case ALGORITHM_RIGHT_ASYMMETRIC_6: 2849 if (i > sh->pd_idx) 2850 i--; 2851 break; 2852 case ALGORITHM_LEFT_SYMMETRIC_6: 2853 case ALGORITHM_RIGHT_SYMMETRIC_6: 2854 if (i < sh->pd_idx) 2855 i += data_disks + 1; 2856 i -= (sh->pd_idx + 1); 2857 break; 2858 case ALGORITHM_PARITY_0_6: 2859 i -= 1; 2860 break; 2861 default: 2862 BUG(); 2863 } 2864 break; 2865 } 2866 2867 chunk_number = stripe * data_disks + i; 2868 r_sector = chunk_number * sectors_per_chunk + chunk_offset; 2869 2870 check = raid5_compute_sector(conf, r_sector, 2871 previous, &dummy1, &sh2); 2872 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx 2873 || sh2.qd_idx != sh->qd_idx) { 2874 pr_warn("md/raid:%s: compute_blocknr: map not correct\n", 2875 mdname(conf->mddev)); 2876 return 0; 2877 } 2878 return r_sector; 2879 } 2880 2881 static void 2882 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, 2883 int rcw, int expand) 2884 { 2885 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; 2886 struct r5conf *conf = sh->raid_conf; 2887 int level = conf->level; 2888 2889 if (rcw) { 2890 /* 2891 * In some cases, handle_stripe_dirtying initially decided to 2892 * run rmw and allocates extra page for prexor. However, rcw is 2893 * cheaper later on. We need to free the extra page now, 2894 * because we won't be able to do that in ops_complete_prexor(). 2895 */ 2896 r5c_release_extra_page(sh); 2897 2898 for (i = disks; i--; ) { 2899 struct r5dev *dev = &sh->dev[i]; 2900 2901 if (dev->towrite) { 2902 set_bit(R5_LOCKED, &dev->flags); 2903 set_bit(R5_Wantdrain, &dev->flags); 2904 if (!expand) 2905 clear_bit(R5_UPTODATE, &dev->flags); 2906 s->locked++; 2907 } else if (test_bit(R5_InJournal, &dev->flags)) { 2908 set_bit(R5_LOCKED, &dev->flags); 2909 s->locked++; 2910 } 2911 } 2912 /* if we are not expanding this is a proper write request, and 2913 * there will be bios with new data to be drained into the 2914 * stripe cache 2915 */ 2916 if (!expand) { 2917 if (!s->locked) 2918 /* False alarm, nothing to do */ 2919 return; 2920 sh->reconstruct_state = reconstruct_state_drain_run; 2921 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2922 } else 2923 sh->reconstruct_state = reconstruct_state_run; 2924 2925 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2926 2927 if (s->locked + conf->max_degraded == disks) 2928 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) 2929 atomic_inc(&conf->pending_full_writes); 2930 } else { 2931 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || 2932 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); 2933 BUG_ON(level == 6 && 2934 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || 2935 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); 2936 2937 for (i = disks; i--; ) { 2938 struct r5dev *dev = &sh->dev[i]; 2939 if (i == pd_idx || i == qd_idx) 2940 continue; 2941 2942 if (dev->towrite && 2943 (test_bit(R5_UPTODATE, &dev->flags) || 2944 test_bit(R5_Wantcompute, &dev->flags))) { 2945 set_bit(R5_Wantdrain, &dev->flags); 2946 set_bit(R5_LOCKED, &dev->flags); 2947 clear_bit(R5_UPTODATE, &dev->flags); 2948 s->locked++; 2949 } else if (test_bit(R5_InJournal, &dev->flags)) { 2950 set_bit(R5_LOCKED, &dev->flags); 2951 s->locked++; 2952 } 2953 } 2954 if (!s->locked) 2955 /* False alarm - nothing to do */ 2956 return; 2957 sh->reconstruct_state = reconstruct_state_prexor_drain_run; 2958 set_bit(STRIPE_OP_PREXOR, &s->ops_request); 2959 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); 2960 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); 2961 } 2962 2963 /* keep the parity disk(s) locked while asynchronous operations 2964 * are in flight 2965 */ 2966 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); 2967 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 2968 s->locked++; 2969 2970 if (level == 6) { 2971 int qd_idx = sh->qd_idx; 2972 struct r5dev *dev = &sh->dev[qd_idx]; 2973 2974 set_bit(R5_LOCKED, &dev->flags); 2975 clear_bit(R5_UPTODATE, &dev->flags); 2976 s->locked++; 2977 } 2978 2979 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", 2980 __func__, (unsigned long long)sh->sector, 2981 s->locked, s->ops_request); 2982 } 2983 2984 /* 2985 * Each stripe/dev can have one or more bion attached. 2986 * toread/towrite point to the first in a chain. 2987 * The bi_next chain must be in order. 2988 */ 2989 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, 2990 int forwrite, int previous) 2991 { 2992 struct bio **bip; 2993 struct r5conf *conf = sh->raid_conf; 2994 int firstwrite=0; 2995 2996 pr_debug("adding bi b#%llu to stripe s#%llu\n", 2997 (unsigned long long)bi->bi_iter.bi_sector, 2998 (unsigned long long)sh->sector); 2999 3000 /* 3001 * If several bio share a stripe. The bio bi_phys_segments acts as a 3002 * reference count to avoid race. The reference count should already be 3003 * increased before this function is called (for example, in 3004 * raid5_make_request()), so other bio sharing this stripe will not free the 3005 * stripe. If a stripe is owned by one stripe, the stripe lock will 3006 * protect it. 3007 */ 3008 spin_lock_irq(&sh->stripe_lock); 3009 /* Don't allow new IO added to stripes in batch list */ 3010 if (sh->batch_head) 3011 goto overlap; 3012 if (forwrite) { 3013 bip = &sh->dev[dd_idx].towrite; 3014 if (*bip == NULL) 3015 firstwrite = 1; 3016 } else 3017 bip = &sh->dev[dd_idx].toread; 3018 while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { 3019 if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) 3020 goto overlap; 3021 bip = & (*bip)->bi_next; 3022 } 3023 if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) 3024 goto overlap; 3025 3026 if (!forwrite || previous) 3027 clear_bit(STRIPE_BATCH_READY, &sh->state); 3028 3029 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); 3030 if (*bip) 3031 bi->bi_next = *bip; 3032 *bip = bi; 3033 raid5_inc_bi_active_stripes(bi); 3034 3035 if (forwrite) { 3036 /* check if page is covered */ 3037 sector_t sector = sh->dev[dd_idx].sector; 3038 for (bi=sh->dev[dd_idx].towrite; 3039 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && 3040 bi && bi->bi_iter.bi_sector <= sector; 3041 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { 3042 if (bio_end_sector(bi) >= sector) 3043 sector = bio_end_sector(bi); 3044 } 3045 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) 3046 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) 3047 sh->overwrite_disks++; 3048 } 3049 3050 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", 3051 (unsigned long long)(*bip)->bi_iter.bi_sector, 3052 (unsigned long long)sh->sector, dd_idx); 3053 3054 if (conf->mddev->bitmap && firstwrite) { 3055 /* Cannot hold spinlock over bitmap_startwrite, 3056 * but must ensure this isn't added to a batch until 3057 * we have added to the bitmap and set bm_seq. 3058 * So set STRIPE_BITMAP_PENDING to prevent 3059 * batching. 3060 * If multiple add_stripe_bio() calls race here they 3061 * much all set STRIPE_BITMAP_PENDING. So only the first one 3062 * to complete "bitmap_startwrite" gets to set 3063 * STRIPE_BIT_DELAY. This is important as once a stripe 3064 * is added to a batch, STRIPE_BIT_DELAY cannot be changed 3065 * any more. 3066 */ 3067 set_bit(STRIPE_BITMAP_PENDING, &sh->state); 3068 spin_unlock_irq(&sh->stripe_lock); 3069 bitmap_startwrite(conf->mddev->bitmap, sh->sector, 3070 STRIPE_SECTORS, 0); 3071 spin_lock_irq(&sh->stripe_lock); 3072 clear_bit(STRIPE_BITMAP_PENDING, &sh->state); 3073 if (!sh->batch_head) { 3074 sh->bm_seq = conf->seq_flush+1; 3075 set_bit(STRIPE_BIT_DELAY, &sh->state); 3076 } 3077 } 3078 spin_unlock_irq(&sh->stripe_lock); 3079 3080 if (stripe_can_batch(sh)) 3081 stripe_add_to_batch_list(conf, sh); 3082 return 1; 3083 3084 overlap: 3085 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); 3086 spin_unlock_irq(&sh->stripe_lock); 3087 return 0; 3088 } 3089 3090 static void end_reshape(struct r5conf *conf); 3091 3092 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, 3093 struct stripe_head *sh) 3094 { 3095 int sectors_per_chunk = 3096 previous ? conf->prev_chunk_sectors : conf->chunk_sectors; 3097 int dd_idx; 3098 int chunk_offset = sector_div(stripe, sectors_per_chunk); 3099 int disks = previous ? conf->previous_raid_disks : conf->raid_disks; 3100 3101 raid5_compute_sector(conf, 3102 stripe * (disks - conf->max_degraded) 3103 *sectors_per_chunk + chunk_offset, 3104 previous, 3105 &dd_idx, sh); 3106 } 3107 3108 static void 3109 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, 3110 struct stripe_head_state *s, int disks, 3111 struct bio_list *return_bi) 3112 { 3113 int i; 3114 BUG_ON(sh->batch_head); 3115 for (i = disks; i--; ) { 3116 struct bio *bi; 3117 int bitmap_end = 0; 3118 3119 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { 3120 struct md_rdev *rdev; 3121 rcu_read_lock(); 3122 rdev = rcu_dereference(conf->disks[i].rdev); 3123 if (rdev && test_bit(In_sync, &rdev->flags) && 3124 !test_bit(Faulty, &rdev->flags)) 3125 atomic_inc(&rdev->nr_pending); 3126 else 3127 rdev = NULL; 3128 rcu_read_unlock(); 3129 if (rdev) { 3130 if (!rdev_set_badblocks( 3131 rdev, 3132 sh->sector, 3133 STRIPE_SECTORS, 0)) 3134 md_error(conf->mddev, rdev); 3135 rdev_dec_pending(rdev, conf->mddev); 3136 } 3137 } 3138 spin_lock_irq(&sh->stripe_lock); 3139 /* fail all writes first */ 3140 bi = sh->dev[i].towrite; 3141 sh->dev[i].towrite = NULL; 3142 sh->overwrite_disks = 0; 3143 spin_unlock_irq(&sh->stripe_lock); 3144 if (bi) 3145 bitmap_end = 1; 3146 3147 r5l_stripe_write_finished(sh); 3148 3149 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 3150 wake_up(&conf->wait_for_overlap); 3151 3152 while (bi && bi->bi_iter.bi_sector < 3153 sh->dev[i].sector + STRIPE_SECTORS) { 3154 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); 3155 3156 bi->bi_error = -EIO; 3157 if (!raid5_dec_bi_active_stripes(bi)) { 3158 md_write_end(conf->mddev); 3159 bio_list_add(return_bi, bi); 3160 } 3161 bi = nextbi; 3162 } 3163 if (bitmap_end) 3164 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3165 STRIPE_SECTORS, 0, 0); 3166 bitmap_end = 0; 3167 /* and fail all 'written' */ 3168 bi = sh->dev[i].written; 3169 sh->dev[i].written = NULL; 3170 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { 3171 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); 3172 sh->dev[i].page = sh->dev[i].orig_page; 3173 } 3174 3175 if (bi) bitmap_end = 1; 3176 while (bi && bi->bi_iter.bi_sector < 3177 sh->dev[i].sector + STRIPE_SECTORS) { 3178 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); 3179 3180 bi->bi_error = -EIO; 3181 if (!raid5_dec_bi_active_stripes(bi)) { 3182 md_write_end(conf->mddev); 3183 bio_list_add(return_bi, bi); 3184 } 3185 bi = bi2; 3186 } 3187 3188 /* fail any reads if this device is non-operational and 3189 * the data has not reached the cache yet. 3190 */ 3191 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && 3192 s->failed > conf->max_degraded && 3193 (!test_bit(R5_Insync, &sh->dev[i].flags) || 3194 test_bit(R5_ReadError, &sh->dev[i].flags))) { 3195 spin_lock_irq(&sh->stripe_lock); 3196 bi = sh->dev[i].toread; 3197 sh->dev[i].toread = NULL; 3198 spin_unlock_irq(&sh->stripe_lock); 3199 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 3200 wake_up(&conf->wait_for_overlap); 3201 if (bi) 3202 s->to_read--; 3203 while (bi && bi->bi_iter.bi_sector < 3204 sh->dev[i].sector + STRIPE_SECTORS) { 3205 struct bio *nextbi = 3206 r5_next_bio(bi, sh->dev[i].sector); 3207 3208 bi->bi_error = -EIO; 3209 if (!raid5_dec_bi_active_stripes(bi)) 3210 bio_list_add(return_bi, bi); 3211 bi = nextbi; 3212 } 3213 } 3214 if (bitmap_end) 3215 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3216 STRIPE_SECTORS, 0, 0); 3217 /* If we were in the middle of a write the parity block might 3218 * still be locked - so just clear all R5_LOCKED flags 3219 */ 3220 clear_bit(R5_LOCKED, &sh->dev[i].flags); 3221 } 3222 s->to_write = 0; 3223 s->written = 0; 3224 3225 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3226 if (atomic_dec_and_test(&conf->pending_full_writes)) 3227 md_wakeup_thread(conf->mddev->thread); 3228 } 3229 3230 static void 3231 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, 3232 struct stripe_head_state *s) 3233 { 3234 int abort = 0; 3235 int i; 3236 3237 BUG_ON(sh->batch_head); 3238 clear_bit(STRIPE_SYNCING, &sh->state); 3239 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 3240 wake_up(&conf->wait_for_overlap); 3241 s->syncing = 0; 3242 s->replacing = 0; 3243 /* There is nothing more to do for sync/check/repair. 3244 * Don't even need to abort as that is handled elsewhere 3245 * if needed, and not always wanted e.g. if there is a known 3246 * bad block here. 3247 * For recover/replace we need to record a bad block on all 3248 * non-sync devices, or abort the recovery 3249 */ 3250 if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { 3251 /* During recovery devices cannot be removed, so 3252 * locking and refcounting of rdevs is not needed 3253 */ 3254 rcu_read_lock(); 3255 for (i = 0; i < conf->raid_disks; i++) { 3256 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 3257 if (rdev 3258 && !test_bit(Faulty, &rdev->flags) 3259 && !test_bit(In_sync, &rdev->flags) 3260 && !rdev_set_badblocks(rdev, sh->sector, 3261 STRIPE_SECTORS, 0)) 3262 abort = 1; 3263 rdev = rcu_dereference(conf->disks[i].replacement); 3264 if (rdev 3265 && !test_bit(Faulty, &rdev->flags) 3266 && !test_bit(In_sync, &rdev->flags) 3267 && !rdev_set_badblocks(rdev, sh->sector, 3268 STRIPE_SECTORS, 0)) 3269 abort = 1; 3270 } 3271 rcu_read_unlock(); 3272 if (abort) 3273 conf->recovery_disabled = 3274 conf->mddev->recovery_disabled; 3275 } 3276 md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); 3277 } 3278 3279 static int want_replace(struct stripe_head *sh, int disk_idx) 3280 { 3281 struct md_rdev *rdev; 3282 int rv = 0; 3283 3284 rcu_read_lock(); 3285 rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement); 3286 if (rdev 3287 && !test_bit(Faulty, &rdev->flags) 3288 && !test_bit(In_sync, &rdev->flags) 3289 && (rdev->recovery_offset <= sh->sector 3290 || rdev->mddev->recovery_cp <= sh->sector)) 3291 rv = 1; 3292 rcu_read_unlock(); 3293 return rv; 3294 } 3295 3296 /* fetch_block - checks the given member device to see if its data needs 3297 * to be read or computed to satisfy a request. 3298 * 3299 * Returns 1 when no more member devices need to be checked, otherwise returns 3300 * 0 to tell the loop in handle_stripe_fill to continue 3301 */ 3302 3303 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, 3304 int disk_idx, int disks) 3305 { 3306 struct r5dev *dev = &sh->dev[disk_idx]; 3307 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], 3308 &sh->dev[s->failed_num[1]] }; 3309 int i; 3310 3311 3312 if (test_bit(R5_LOCKED, &dev->flags) || 3313 test_bit(R5_UPTODATE, &dev->flags)) 3314 /* No point reading this as we already have it or have 3315 * decided to get it. 3316 */ 3317 return 0; 3318 3319 if (dev->toread || 3320 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) 3321 /* We need this block to directly satisfy a request */ 3322 return 1; 3323 3324 if (s->syncing || s->expanding || 3325 (s->replacing && want_replace(sh, disk_idx))) 3326 /* When syncing, or expanding we read everything. 3327 * When replacing, we need the replaced block. 3328 */ 3329 return 1; 3330 3331 if ((s->failed >= 1 && fdev[0]->toread) || 3332 (s->failed >= 2 && fdev[1]->toread)) 3333 /* If we want to read from a failed device, then 3334 * we need to actually read every other device. 3335 */ 3336 return 1; 3337 3338 /* Sometimes neither read-modify-write nor reconstruct-write 3339 * cycles can work. In those cases we read every block we 3340 * can. Then the parity-update is certain to have enough to 3341 * work with. 3342 * This can only be a problem when we need to write something, 3343 * and some device has failed. If either of those tests 3344 * fail we need look no further. 3345 */ 3346 if (!s->failed || !s->to_write) 3347 return 0; 3348 3349 if (test_bit(R5_Insync, &dev->flags) && 3350 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3351 /* Pre-reads at not permitted until after short delay 3352 * to gather multiple requests. However if this 3353 * device is no Insync, the block could only be be computed 3354 * and there is no need to delay that. 3355 */ 3356 return 0; 3357 3358 for (i = 0; i < s->failed && i < 2; i++) { 3359 if (fdev[i]->towrite && 3360 !test_bit(R5_UPTODATE, &fdev[i]->flags) && 3361 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 3362 /* If we have a partial write to a failed 3363 * device, then we will need to reconstruct 3364 * the content of that device, so all other 3365 * devices must be read. 3366 */ 3367 return 1; 3368 } 3369 3370 /* If we are forced to do a reconstruct-write, either because 3371 * the current RAID6 implementation only supports that, or 3372 * or because parity cannot be trusted and we are currently 3373 * recovering it, there is extra need to be careful. 3374 * If one of the devices that we would need to read, because 3375 * it is not being overwritten (and maybe not written at all) 3376 * is missing/faulty, then we need to read everything we can. 3377 */ 3378 if (sh->raid_conf->level != 6 && 3379 sh->sector < sh->raid_conf->mddev->recovery_cp) 3380 /* reconstruct-write isn't being forced */ 3381 return 0; 3382 for (i = 0; i < s->failed && i < 2; i++) { 3383 if (s->failed_num[i] != sh->pd_idx && 3384 s->failed_num[i] != sh->qd_idx && 3385 !test_bit(R5_UPTODATE, &fdev[i]->flags) && 3386 !test_bit(R5_OVERWRITE, &fdev[i]->flags)) 3387 return 1; 3388 } 3389 3390 return 0; 3391 } 3392 3393 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, 3394 int disk_idx, int disks) 3395 { 3396 struct r5dev *dev = &sh->dev[disk_idx]; 3397 3398 /* is the data in this block needed, and can we get it? */ 3399 if (need_this_block(sh, s, disk_idx, disks)) { 3400 /* we would like to get this block, possibly by computing it, 3401 * otherwise read it if the backing disk is insync 3402 */ 3403 BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); 3404 BUG_ON(test_bit(R5_Wantread, &dev->flags)); 3405 BUG_ON(sh->batch_head); 3406 if ((s->uptodate == disks - 1) && 3407 (s->failed && (disk_idx == s->failed_num[0] || 3408 disk_idx == s->failed_num[1]))) { 3409 /* have disk failed, and we're requested to fetch it; 3410 * do compute it 3411 */ 3412 pr_debug("Computing stripe %llu block %d\n", 3413 (unsigned long long)sh->sector, disk_idx); 3414 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3415 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3416 set_bit(R5_Wantcompute, &dev->flags); 3417 sh->ops.target = disk_idx; 3418 sh->ops.target2 = -1; /* no 2nd target */ 3419 s->req_compute = 1; 3420 /* Careful: from this point on 'uptodate' is in the eye 3421 * of raid_run_ops which services 'compute' operations 3422 * before writes. R5_Wantcompute flags a block that will 3423 * be R5_UPTODATE by the time it is needed for a 3424 * subsequent operation. 3425 */ 3426 s->uptodate++; 3427 return 1; 3428 } else if (s->uptodate == disks-2 && s->failed >= 2) { 3429 /* Computing 2-failure is *very* expensive; only 3430 * do it if failed >= 2 3431 */ 3432 int other; 3433 for (other = disks; other--; ) { 3434 if (other == disk_idx) 3435 continue; 3436 if (!test_bit(R5_UPTODATE, 3437 &sh->dev[other].flags)) 3438 break; 3439 } 3440 BUG_ON(other < 0); 3441 pr_debug("Computing stripe %llu blocks %d,%d\n", 3442 (unsigned long long)sh->sector, 3443 disk_idx, other); 3444 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3445 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3446 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); 3447 set_bit(R5_Wantcompute, &sh->dev[other].flags); 3448 sh->ops.target = disk_idx; 3449 sh->ops.target2 = other; 3450 s->uptodate += 2; 3451 s->req_compute = 1; 3452 return 1; 3453 } else if (test_bit(R5_Insync, &dev->flags)) { 3454 set_bit(R5_LOCKED, &dev->flags); 3455 set_bit(R5_Wantread, &dev->flags); 3456 s->locked++; 3457 pr_debug("Reading block %d (sync=%d)\n", 3458 disk_idx, s->syncing); 3459 } 3460 } 3461 3462 return 0; 3463 } 3464 3465 /** 3466 * handle_stripe_fill - read or compute data to satisfy pending requests. 3467 */ 3468 static void handle_stripe_fill(struct stripe_head *sh, 3469 struct stripe_head_state *s, 3470 int disks) 3471 { 3472 int i; 3473 3474 /* look for blocks to read/compute, skip this if a compute 3475 * is already in flight, or if the stripe contents are in the 3476 * midst of changing due to a write 3477 */ 3478 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && 3479 !sh->reconstruct_state) 3480 for (i = disks; i--; ) 3481 if (fetch_block(sh, s, i, disks)) 3482 break; 3483 set_bit(STRIPE_HANDLE, &sh->state); 3484 } 3485 3486 static void break_stripe_batch_list(struct stripe_head *head_sh, 3487 unsigned long handle_flags); 3488 /* handle_stripe_clean_event 3489 * any written block on an uptodate or failed drive can be returned. 3490 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but 3491 * never LOCKED, so we don't need to test 'failed' directly. 3492 */ 3493 static void handle_stripe_clean_event(struct r5conf *conf, 3494 struct stripe_head *sh, int disks, struct bio_list *return_bi) 3495 { 3496 int i; 3497 struct r5dev *dev; 3498 int discard_pending = 0; 3499 struct stripe_head *head_sh = sh; 3500 bool do_endio = false; 3501 3502 for (i = disks; i--; ) 3503 if (sh->dev[i].written) { 3504 dev = &sh->dev[i]; 3505 if (!test_bit(R5_LOCKED, &dev->flags) && 3506 (test_bit(R5_UPTODATE, &dev->flags) || 3507 test_bit(R5_Discard, &dev->flags) || 3508 test_bit(R5_SkipCopy, &dev->flags))) { 3509 /* We can return any write requests */ 3510 struct bio *wbi, *wbi2; 3511 pr_debug("Return write for disc %d\n", i); 3512 if (test_and_clear_bit(R5_Discard, &dev->flags)) 3513 clear_bit(R5_UPTODATE, &dev->flags); 3514 if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { 3515 WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); 3516 } 3517 do_endio = true; 3518 3519 returnbi: 3520 dev->page = dev->orig_page; 3521 wbi = dev->written; 3522 dev->written = NULL; 3523 while (wbi && wbi->bi_iter.bi_sector < 3524 dev->sector + STRIPE_SECTORS) { 3525 wbi2 = r5_next_bio(wbi, dev->sector); 3526 if (!raid5_dec_bi_active_stripes(wbi)) { 3527 md_write_end(conf->mddev); 3528 bio_list_add(return_bi, wbi); 3529 } 3530 wbi = wbi2; 3531 } 3532 bitmap_endwrite(conf->mddev->bitmap, sh->sector, 3533 STRIPE_SECTORS, 3534 !test_bit(STRIPE_DEGRADED, &sh->state), 3535 0); 3536 if (head_sh->batch_head) { 3537 sh = list_first_entry(&sh->batch_list, 3538 struct stripe_head, 3539 batch_list); 3540 if (sh != head_sh) { 3541 dev = &sh->dev[i]; 3542 goto returnbi; 3543 } 3544 } 3545 sh = head_sh; 3546 dev = &sh->dev[i]; 3547 } else if (test_bit(R5_Discard, &dev->flags)) 3548 discard_pending = 1; 3549 } 3550 3551 r5l_stripe_write_finished(sh); 3552 3553 if (!discard_pending && 3554 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { 3555 int hash; 3556 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); 3557 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3558 if (sh->qd_idx >= 0) { 3559 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); 3560 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); 3561 } 3562 /* now that discard is done we can proceed with any sync */ 3563 clear_bit(STRIPE_DISCARD, &sh->state); 3564 /* 3565 * SCSI discard will change some bio fields and the stripe has 3566 * no updated data, so remove it from hash list and the stripe 3567 * will be reinitialized 3568 */ 3569 unhash: 3570 hash = sh->hash_lock_index; 3571 spin_lock_irq(conf->hash_locks + hash); 3572 remove_hash(sh); 3573 spin_unlock_irq(conf->hash_locks + hash); 3574 if (head_sh->batch_head) { 3575 sh = list_first_entry(&sh->batch_list, 3576 struct stripe_head, batch_list); 3577 if (sh != head_sh) 3578 goto unhash; 3579 } 3580 sh = head_sh; 3581 3582 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) 3583 set_bit(STRIPE_HANDLE, &sh->state); 3584 3585 } 3586 3587 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) 3588 if (atomic_dec_and_test(&conf->pending_full_writes)) 3589 md_wakeup_thread(conf->mddev->thread); 3590 3591 if (head_sh->batch_head && do_endio) 3592 break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS); 3593 } 3594 3595 static int handle_stripe_dirtying(struct r5conf *conf, 3596 struct stripe_head *sh, 3597 struct stripe_head_state *s, 3598 int disks) 3599 { 3600 int rmw = 0, rcw = 0, i; 3601 sector_t recovery_cp = conf->mddev->recovery_cp; 3602 3603 /* Check whether resync is now happening or should start. 3604 * If yes, then the array is dirty (after unclean shutdown or 3605 * initial creation), so parity in some stripes might be inconsistent. 3606 * In this case, we need to always do reconstruct-write, to ensure 3607 * that in case of drive failure or read-error correction, we 3608 * generate correct data from the parity. 3609 */ 3610 if (conf->rmw_level == PARITY_DISABLE_RMW || 3611 (recovery_cp < MaxSector && sh->sector >= recovery_cp && 3612 s->failed == 0)) { 3613 /* Calculate the real rcw later - for now make it 3614 * look like rcw is cheaper 3615 */ 3616 rcw = 1; rmw = 2; 3617 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", 3618 conf->rmw_level, (unsigned long long)recovery_cp, 3619 (unsigned long long)sh->sector); 3620 } else for (i = disks; i--; ) { 3621 /* would I have to read this buffer for read_modify_write */ 3622 struct r5dev *dev = &sh->dev[i]; 3623 if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx || 3624 test_bit(R5_InJournal, &dev->flags)) && 3625 !test_bit(R5_LOCKED, &dev->flags) && 3626 !((test_bit(R5_UPTODATE, &dev->flags) && 3627 (!test_bit(R5_InJournal, &dev->flags) || 3628 dev->page != dev->orig_page)) || 3629 test_bit(R5_Wantcompute, &dev->flags))) { 3630 if (test_bit(R5_Insync, &dev->flags)) 3631 rmw++; 3632 else 3633 rmw += 2*disks; /* cannot read it */ 3634 } 3635 /* Would I have to read this buffer for reconstruct_write */ 3636 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3637 i != sh->pd_idx && i != sh->qd_idx && 3638 !test_bit(R5_LOCKED, &dev->flags) && 3639 !(test_bit(R5_UPTODATE, &dev->flags) || 3640 test_bit(R5_InJournal, &dev->flags) || 3641 test_bit(R5_Wantcompute, &dev->flags))) { 3642 if (test_bit(R5_Insync, &dev->flags)) 3643 rcw++; 3644 else 3645 rcw += 2*disks; 3646 } 3647 } 3648 3649 pr_debug("for sector %llu, rmw=%d rcw=%d\n", 3650 (unsigned long long)sh->sector, rmw, rcw); 3651 set_bit(STRIPE_HANDLE, &sh->state); 3652 if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) { 3653 /* prefer read-modify-write, but need to get some data */ 3654 if (conf->mddev->queue) 3655 blk_add_trace_msg(conf->mddev->queue, 3656 "raid5 rmw %llu %d", 3657 (unsigned long long)sh->sector, rmw); 3658 for (i = disks; i--; ) { 3659 struct r5dev *dev = &sh->dev[i]; 3660 if (test_bit(R5_InJournal, &dev->flags) && 3661 dev->page == dev->orig_page && 3662 !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) { 3663 /* alloc page for prexor */ 3664 struct page *p = alloc_page(GFP_NOIO); 3665 3666 if (p) { 3667 dev->orig_page = p; 3668 continue; 3669 } 3670 3671 /* 3672 * alloc_page() failed, try use 3673 * disk_info->extra_page 3674 */ 3675 if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE, 3676 &conf->cache_state)) { 3677 r5c_use_extra_page(sh); 3678 break; 3679 } 3680 3681 /* extra_page in use, add to delayed_list */ 3682 set_bit(STRIPE_DELAYED, &sh->state); 3683 s->waiting_extra_page = 1; 3684 return -EAGAIN; 3685 } 3686 } 3687 3688 for (i = disks; i--; ) { 3689 struct r5dev *dev = &sh->dev[i]; 3690 if ((dev->towrite || 3691 i == sh->pd_idx || i == sh->qd_idx || 3692 test_bit(R5_InJournal, &dev->flags)) && 3693 !test_bit(R5_LOCKED, &dev->flags) && 3694 !((test_bit(R5_UPTODATE, &dev->flags) && 3695 (!test_bit(R5_InJournal, &dev->flags) || 3696 dev->page != dev->orig_page)) || 3697 test_bit(R5_Wantcompute, &dev->flags)) && 3698 test_bit(R5_Insync, &dev->flags)) { 3699 if (test_bit(STRIPE_PREREAD_ACTIVE, 3700 &sh->state)) { 3701 pr_debug("Read_old block %d for r-m-w\n", 3702 i); 3703 set_bit(R5_LOCKED, &dev->flags); 3704 set_bit(R5_Wantread, &dev->flags); 3705 s->locked++; 3706 } else { 3707 set_bit(STRIPE_DELAYED, &sh->state); 3708 set_bit(STRIPE_HANDLE, &sh->state); 3709 } 3710 } 3711 } 3712 } 3713 if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) { 3714 /* want reconstruct write, but need to get some data */ 3715 int qread =0; 3716 rcw = 0; 3717 for (i = disks; i--; ) { 3718 struct r5dev *dev = &sh->dev[i]; 3719 if (!test_bit(R5_OVERWRITE, &dev->flags) && 3720 i != sh->pd_idx && i != sh->qd_idx && 3721 !test_bit(R5_LOCKED, &dev->flags) && 3722 !(test_bit(R5_UPTODATE, &dev->flags) || 3723 test_bit(R5_InJournal, &dev->flags) || 3724 test_bit(R5_Wantcompute, &dev->flags))) { 3725 rcw++; 3726 if (test_bit(R5_Insync, &dev->flags) && 3727 test_bit(STRIPE_PREREAD_ACTIVE, 3728 &sh->state)) { 3729 pr_debug("Read_old block " 3730 "%d for Reconstruct\n", i); 3731 set_bit(R5_LOCKED, &dev->flags); 3732 set_bit(R5_Wantread, &dev->flags); 3733 s->locked++; 3734 qread++; 3735 } else { 3736 set_bit(STRIPE_DELAYED, &sh->state); 3737 set_bit(STRIPE_HANDLE, &sh->state); 3738 } 3739 } 3740 } 3741 if (rcw && conf->mddev->queue) 3742 blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", 3743 (unsigned long long)sh->sector, 3744 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); 3745 } 3746 3747 if (rcw > disks && rmw > disks && 3748 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 3749 set_bit(STRIPE_DELAYED, &sh->state); 3750 3751 /* now if nothing is locked, and if we have enough data, 3752 * we can start a write request 3753 */ 3754 /* since handle_stripe can be called at any time we need to handle the 3755 * case where a compute block operation has been submitted and then a 3756 * subsequent call wants to start a write request. raid_run_ops only 3757 * handles the case where compute block and reconstruct are requested 3758 * simultaneously. If this is not the case then new writes need to be 3759 * held off until the compute completes. 3760 */ 3761 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && 3762 (s->locked == 0 && (rcw == 0 || rmw == 0) && 3763 !test_bit(STRIPE_BIT_DELAY, &sh->state))) 3764 schedule_reconstruction(sh, s, rcw == 0, 0); 3765 return 0; 3766 } 3767 3768 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, 3769 struct stripe_head_state *s, int disks) 3770 { 3771 struct r5dev *dev = NULL; 3772 3773 BUG_ON(sh->batch_head); 3774 set_bit(STRIPE_HANDLE, &sh->state); 3775 3776 switch (sh->check_state) { 3777 case check_state_idle: 3778 /* start a new check operation if there are no failures */ 3779 if (s->failed == 0) { 3780 BUG_ON(s->uptodate != disks); 3781 sh->check_state = check_state_run; 3782 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3783 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); 3784 s->uptodate--; 3785 break; 3786 } 3787 dev = &sh->dev[s->failed_num[0]]; 3788 /* fall through */ 3789 case check_state_compute_result: 3790 sh->check_state = check_state_idle; 3791 if (!dev) 3792 dev = &sh->dev[sh->pd_idx]; 3793 3794 /* check that a write has not made the stripe insync */ 3795 if (test_bit(STRIPE_INSYNC, &sh->state)) 3796 break; 3797 3798 /* either failed parity check, or recovery is happening */ 3799 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); 3800 BUG_ON(s->uptodate != disks); 3801 3802 set_bit(R5_LOCKED, &dev->flags); 3803 s->locked++; 3804 set_bit(R5_Wantwrite, &dev->flags); 3805 3806 clear_bit(STRIPE_DEGRADED, &sh->state); 3807 set_bit(STRIPE_INSYNC, &sh->state); 3808 break; 3809 case check_state_run: 3810 break; /* we will be called again upon completion */ 3811 case check_state_check_result: 3812 sh->check_state = check_state_idle; 3813 3814 /* if a failure occurred during the check operation, leave 3815 * STRIPE_INSYNC not set and let the stripe be handled again 3816 */ 3817 if (s->failed) 3818 break; 3819 3820 /* handle a successful check operation, if parity is correct 3821 * we are done. Otherwise update the mismatch count and repair 3822 * parity if !MD_RECOVERY_CHECK 3823 */ 3824 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) 3825 /* parity is correct (on disc, 3826 * not in buffer any more) 3827 */ 3828 set_bit(STRIPE_INSYNC, &sh->state); 3829 else { 3830 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3831 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3832 /* don't try to repair!! */ 3833 set_bit(STRIPE_INSYNC, &sh->state); 3834 else { 3835 sh->check_state = check_state_compute_run; 3836 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3837 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3838 set_bit(R5_Wantcompute, 3839 &sh->dev[sh->pd_idx].flags); 3840 sh->ops.target = sh->pd_idx; 3841 sh->ops.target2 = -1; 3842 s->uptodate++; 3843 } 3844 } 3845 break; 3846 case check_state_compute_run: 3847 break; 3848 default: 3849 pr_err("%s: unknown check_state: %d sector: %llu\n", 3850 __func__, sh->check_state, 3851 (unsigned long long) sh->sector); 3852 BUG(); 3853 } 3854 } 3855 3856 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, 3857 struct stripe_head_state *s, 3858 int disks) 3859 { 3860 int pd_idx = sh->pd_idx; 3861 int qd_idx = sh->qd_idx; 3862 struct r5dev *dev; 3863 3864 BUG_ON(sh->batch_head); 3865 set_bit(STRIPE_HANDLE, &sh->state); 3866 3867 BUG_ON(s->failed > 2); 3868 3869 /* Want to check and possibly repair P and Q. 3870 * However there could be one 'failed' device, in which 3871 * case we can only check one of them, possibly using the 3872 * other to generate missing data 3873 */ 3874 3875 switch (sh->check_state) { 3876 case check_state_idle: 3877 /* start a new check operation if there are < 2 failures */ 3878 if (s->failed == s->q_failed) { 3879 /* The only possible failed device holds Q, so it 3880 * makes sense to check P (If anything else were failed, 3881 * we would have used P to recreate it). 3882 */ 3883 sh->check_state = check_state_run; 3884 } 3885 if (!s->q_failed && s->failed < 2) { 3886 /* Q is not failed, and we didn't use it to generate 3887 * anything, so it makes sense to check it 3888 */ 3889 if (sh->check_state == check_state_run) 3890 sh->check_state = check_state_run_pq; 3891 else 3892 sh->check_state = check_state_run_q; 3893 } 3894 3895 /* discard potentially stale zero_sum_result */ 3896 sh->ops.zero_sum_result = 0; 3897 3898 if (sh->check_state == check_state_run) { 3899 /* async_xor_zero_sum destroys the contents of P */ 3900 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); 3901 s->uptodate--; 3902 } 3903 if (sh->check_state >= check_state_run && 3904 sh->check_state <= check_state_run_pq) { 3905 /* async_syndrome_zero_sum preserves P and Q, so 3906 * no need to mark them !uptodate here 3907 */ 3908 set_bit(STRIPE_OP_CHECK, &s->ops_request); 3909 break; 3910 } 3911 3912 /* we have 2-disk failure */ 3913 BUG_ON(s->failed != 2); 3914 /* fall through */ 3915 case check_state_compute_result: 3916 sh->check_state = check_state_idle; 3917 3918 /* check that a write has not made the stripe insync */ 3919 if (test_bit(STRIPE_INSYNC, &sh->state)) 3920 break; 3921 3922 /* now write out any block on a failed drive, 3923 * or P or Q if they were recomputed 3924 */ 3925 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ 3926 if (s->failed == 2) { 3927 dev = &sh->dev[s->failed_num[1]]; 3928 s->locked++; 3929 set_bit(R5_LOCKED, &dev->flags); 3930 set_bit(R5_Wantwrite, &dev->flags); 3931 } 3932 if (s->failed >= 1) { 3933 dev = &sh->dev[s->failed_num[0]]; 3934 s->locked++; 3935 set_bit(R5_LOCKED, &dev->flags); 3936 set_bit(R5_Wantwrite, &dev->flags); 3937 } 3938 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3939 dev = &sh->dev[pd_idx]; 3940 s->locked++; 3941 set_bit(R5_LOCKED, &dev->flags); 3942 set_bit(R5_Wantwrite, &dev->flags); 3943 } 3944 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 3945 dev = &sh->dev[qd_idx]; 3946 s->locked++; 3947 set_bit(R5_LOCKED, &dev->flags); 3948 set_bit(R5_Wantwrite, &dev->flags); 3949 } 3950 clear_bit(STRIPE_DEGRADED, &sh->state); 3951 3952 set_bit(STRIPE_INSYNC, &sh->state); 3953 break; 3954 case check_state_run: 3955 case check_state_run_q: 3956 case check_state_run_pq: 3957 break; /* we will be called again upon completion */ 3958 case check_state_check_result: 3959 sh->check_state = check_state_idle; 3960 3961 /* handle a successful check operation, if parity is correct 3962 * we are done. Otherwise update the mismatch count and repair 3963 * parity if !MD_RECOVERY_CHECK 3964 */ 3965 if (sh->ops.zero_sum_result == 0) { 3966 /* both parities are correct */ 3967 if (!s->failed) 3968 set_bit(STRIPE_INSYNC, &sh->state); 3969 else { 3970 /* in contrast to the raid5 case we can validate 3971 * parity, but still have a failure to write 3972 * back 3973 */ 3974 sh->check_state = check_state_compute_result; 3975 /* Returning at this point means that we may go 3976 * off and bring p and/or q uptodate again so 3977 * we make sure to check zero_sum_result again 3978 * to verify if p or q need writeback 3979 */ 3980 } 3981 } else { 3982 atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); 3983 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) 3984 /* don't try to repair!! */ 3985 set_bit(STRIPE_INSYNC, &sh->state); 3986 else { 3987 int *target = &sh->ops.target; 3988 3989 sh->ops.target = -1; 3990 sh->ops.target2 = -1; 3991 sh->check_state = check_state_compute_run; 3992 set_bit(STRIPE_COMPUTE_RUN, &sh->state); 3993 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); 3994 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { 3995 set_bit(R5_Wantcompute, 3996 &sh->dev[pd_idx].flags); 3997 *target = pd_idx; 3998 target = &sh->ops.target2; 3999 s->uptodate++; 4000 } 4001 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { 4002 set_bit(R5_Wantcompute, 4003 &sh->dev[qd_idx].flags); 4004 *target = qd_idx; 4005 s->uptodate++; 4006 } 4007 } 4008 } 4009 break; 4010 case check_state_compute_run: 4011 break; 4012 default: 4013 pr_warn("%s: unknown check_state: %d sector: %llu\n", 4014 __func__, sh->check_state, 4015 (unsigned long long) sh->sector); 4016 BUG(); 4017 } 4018 } 4019 4020 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) 4021 { 4022 int i; 4023 4024 /* We have read all the blocks in this stripe and now we need to 4025 * copy some of them into a target stripe for expand. 4026 */ 4027 struct dma_async_tx_descriptor *tx = NULL; 4028 BUG_ON(sh->batch_head); 4029 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); 4030 for (i = 0; i < sh->disks; i++) 4031 if (i != sh->pd_idx && i != sh->qd_idx) { 4032 int dd_idx, j; 4033 struct stripe_head *sh2; 4034 struct async_submit_ctl submit; 4035 4036 sector_t bn = raid5_compute_blocknr(sh, i, 1); 4037 sector_t s = raid5_compute_sector(conf, bn, 0, 4038 &dd_idx, NULL); 4039 sh2 = raid5_get_active_stripe(conf, s, 0, 1, 1); 4040 if (sh2 == NULL) 4041 /* so far only the early blocks of this stripe 4042 * have been requested. When later blocks 4043 * get requested, we will try again 4044 */ 4045 continue; 4046 if (!test_bit(STRIPE_EXPANDING, &sh2->state) || 4047 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { 4048 /* must have already done this block */ 4049 raid5_release_stripe(sh2); 4050 continue; 4051 } 4052 4053 /* place all the copies on one channel */ 4054 init_async_submit(&submit, 0, tx, NULL, NULL, NULL); 4055 tx = async_memcpy(sh2->dev[dd_idx].page, 4056 sh->dev[i].page, 0, 0, STRIPE_SIZE, 4057 &submit); 4058 4059 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); 4060 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); 4061 for (j = 0; j < conf->raid_disks; j++) 4062 if (j != sh2->pd_idx && 4063 j != sh2->qd_idx && 4064 !test_bit(R5_Expanded, &sh2->dev[j].flags)) 4065 break; 4066 if (j == conf->raid_disks) { 4067 set_bit(STRIPE_EXPAND_READY, &sh2->state); 4068 set_bit(STRIPE_HANDLE, &sh2->state); 4069 } 4070 raid5_release_stripe(sh2); 4071 4072 } 4073 /* done submitting copies, wait for them to complete */ 4074 async_tx_quiesce(&tx); 4075 } 4076 4077 /* 4078 * handle_stripe - do things to a stripe. 4079 * 4080 * We lock the stripe by setting STRIPE_ACTIVE and then examine the 4081 * state of various bits to see what needs to be done. 4082 * Possible results: 4083 * return some read requests which now have data 4084 * return some write requests which are safely on storage 4085 * schedule a read on some buffers 4086 * schedule a write of some buffers 4087 * return confirmation of parity correctness 4088 * 4089 */ 4090 4091 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) 4092 { 4093 struct r5conf *conf = sh->raid_conf; 4094 int disks = sh->disks; 4095 struct r5dev *dev; 4096 int i; 4097 int do_recovery = 0; 4098 4099 memset(s, 0, sizeof(*s)); 4100 4101 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; 4102 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; 4103 s->failed_num[0] = -1; 4104 s->failed_num[1] = -1; 4105 s->log_failed = r5l_log_disk_error(conf); 4106 4107 /* Now to look around and see what can be done */ 4108 rcu_read_lock(); 4109 for (i=disks; i--; ) { 4110 struct md_rdev *rdev; 4111 sector_t first_bad; 4112 int bad_sectors; 4113 int is_bad = 0; 4114 4115 dev = &sh->dev[i]; 4116 4117 pr_debug("check %d: state 0x%lx read %p write %p written %p\n", 4118 i, dev->flags, 4119 dev->toread, dev->towrite, dev->written); 4120 /* maybe we can reply to a read 4121 * 4122 * new wantfill requests are only permitted while 4123 * ops_complete_biofill is guaranteed to be inactive 4124 */ 4125 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && 4126 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) 4127 set_bit(R5_Wantfill, &dev->flags); 4128 4129 /* now count some things */ 4130 if (test_bit(R5_LOCKED, &dev->flags)) 4131 s->locked++; 4132 if (test_bit(R5_UPTODATE, &dev->flags)) 4133 s->uptodate++; 4134 if (test_bit(R5_Wantcompute, &dev->flags)) { 4135 s->compute++; 4136 BUG_ON(s->compute > 2); 4137 } 4138 4139 if (test_bit(R5_Wantfill, &dev->flags)) 4140 s->to_fill++; 4141 else if (dev->toread) 4142 s->to_read++; 4143 if (dev->towrite) { 4144 s->to_write++; 4145 if (!test_bit(R5_OVERWRITE, &dev->flags)) 4146 s->non_overwrite++; 4147 } 4148 if (dev->written) 4149 s->written++; 4150 /* Prefer to use the replacement for reads, but only 4151 * if it is recovered enough and has no bad blocks. 4152 */ 4153 rdev = rcu_dereference(conf->disks[i].replacement); 4154 if (rdev && !test_bit(Faulty, &rdev->flags) && 4155 rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && 4156 !is_badblock(rdev, sh->sector, STRIPE_SECTORS, 4157 &first_bad, &bad_sectors)) 4158 set_bit(R5_ReadRepl, &dev->flags); 4159 else { 4160 if (rdev && !test_bit(Faulty, &rdev->flags)) 4161 set_bit(R5_NeedReplace, &dev->flags); 4162 else 4163 clear_bit(R5_NeedReplace, &dev->flags); 4164 rdev = rcu_dereference(conf->disks[i].rdev); 4165 clear_bit(R5_ReadRepl, &dev->flags); 4166 } 4167 if (rdev && test_bit(Faulty, &rdev->flags)) 4168 rdev = NULL; 4169 if (rdev) { 4170 is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, 4171 &first_bad, &bad_sectors); 4172 if (s->blocked_rdev == NULL 4173 && (test_bit(Blocked, &rdev->flags) 4174 || is_bad < 0)) { 4175 if (is_bad < 0) 4176 set_bit(BlockedBadBlocks, 4177 &rdev->flags); 4178 s->blocked_rdev = rdev; 4179 atomic_inc(&rdev->nr_pending); 4180 } 4181 } 4182 clear_bit(R5_Insync, &dev->flags); 4183 if (!rdev) 4184 /* Not in-sync */; 4185 else if (is_bad) { 4186 /* also not in-sync */ 4187 if (!test_bit(WriteErrorSeen, &rdev->flags) && 4188 test_bit(R5_UPTODATE, &dev->flags)) { 4189 /* treat as in-sync, but with a read error 4190 * which we can now try to correct 4191 */ 4192 set_bit(R5_Insync, &dev->flags); 4193 set_bit(R5_ReadError, &dev->flags); 4194 } 4195 } else if (test_bit(In_sync, &rdev->flags)) 4196 set_bit(R5_Insync, &dev->flags); 4197 else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) 4198 /* in sync if before recovery_offset */ 4199 set_bit(R5_Insync, &dev->flags); 4200 else if (test_bit(R5_UPTODATE, &dev->flags) && 4201 test_bit(R5_Expanded, &dev->flags)) 4202 /* If we've reshaped into here, we assume it is Insync. 4203 * We will shortly update recovery_offset to make 4204 * it official. 4205 */ 4206 set_bit(R5_Insync, &dev->flags); 4207 4208 if (test_bit(R5_WriteError, &dev->flags)) { 4209 /* This flag does not apply to '.replacement' 4210 * only to .rdev, so make sure to check that*/ 4211 struct md_rdev *rdev2 = rcu_dereference( 4212 conf->disks[i].rdev); 4213 if (rdev2 == rdev) 4214 clear_bit(R5_Insync, &dev->flags); 4215 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4216 s->handle_bad_blocks = 1; 4217 atomic_inc(&rdev2->nr_pending); 4218 } else 4219 clear_bit(R5_WriteError, &dev->flags); 4220 } 4221 if (test_bit(R5_MadeGood, &dev->flags)) { 4222 /* This flag does not apply to '.replacement' 4223 * only to .rdev, so make sure to check that*/ 4224 struct md_rdev *rdev2 = rcu_dereference( 4225 conf->disks[i].rdev); 4226 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4227 s->handle_bad_blocks = 1; 4228 atomic_inc(&rdev2->nr_pending); 4229 } else 4230 clear_bit(R5_MadeGood, &dev->flags); 4231 } 4232 if (test_bit(R5_MadeGoodRepl, &dev->flags)) { 4233 struct md_rdev *rdev2 = rcu_dereference( 4234 conf->disks[i].replacement); 4235 if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { 4236 s->handle_bad_blocks = 1; 4237 atomic_inc(&rdev2->nr_pending); 4238 } else 4239 clear_bit(R5_MadeGoodRepl, &dev->flags); 4240 } 4241 if (!test_bit(R5_Insync, &dev->flags)) { 4242 /* The ReadError flag will just be confusing now */ 4243 clear_bit(R5_ReadError, &dev->flags); 4244 clear_bit(R5_ReWrite, &dev->flags); 4245 } 4246 if (test_bit(R5_ReadError, &dev->flags)) 4247 clear_bit(R5_Insync, &dev->flags); 4248 if (!test_bit(R5_Insync, &dev->flags)) { 4249 if (s->failed < 2) 4250 s->failed_num[s->failed] = i; 4251 s->failed++; 4252 if (rdev && !test_bit(Faulty, &rdev->flags)) 4253 do_recovery = 1; 4254 } 4255 4256 if (test_bit(R5_InJournal, &dev->flags)) 4257 s->injournal++; 4258 if (test_bit(R5_InJournal, &dev->flags) && dev->written) 4259 s->just_cached++; 4260 } 4261 if (test_bit(STRIPE_SYNCING, &sh->state)) { 4262 /* If there is a failed device being replaced, 4263 * we must be recovering. 4264 * else if we are after recovery_cp, we must be syncing 4265 * else if MD_RECOVERY_REQUESTED is set, we also are syncing. 4266 * else we can only be replacing 4267 * sync and recovery both need to read all devices, and so 4268 * use the same flag. 4269 */ 4270 if (do_recovery || 4271 sh->sector >= conf->mddev->recovery_cp || 4272 test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) 4273 s->syncing = 1; 4274 else 4275 s->replacing = 1; 4276 } 4277 rcu_read_unlock(); 4278 } 4279 4280 static int clear_batch_ready(struct stripe_head *sh) 4281 { 4282 /* Return '1' if this is a member of batch, or 4283 * '0' if it is a lone stripe or a head which can now be 4284 * handled. 4285 */ 4286 struct stripe_head *tmp; 4287 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) 4288 return (sh->batch_head && sh->batch_head != sh); 4289 spin_lock(&sh->stripe_lock); 4290 if (!sh->batch_head) { 4291 spin_unlock(&sh->stripe_lock); 4292 return 0; 4293 } 4294 4295 /* 4296 * this stripe could be added to a batch list before we check 4297 * BATCH_READY, skips it 4298 */ 4299 if (sh->batch_head != sh) { 4300 spin_unlock(&sh->stripe_lock); 4301 return 1; 4302 } 4303 spin_lock(&sh->batch_lock); 4304 list_for_each_entry(tmp, &sh->batch_list, batch_list) 4305 clear_bit(STRIPE_BATCH_READY, &tmp->state); 4306 spin_unlock(&sh->batch_lock); 4307 spin_unlock(&sh->stripe_lock); 4308 4309 /* 4310 * BATCH_READY is cleared, no new stripes can be added. 4311 * batch_list can be accessed without lock 4312 */ 4313 return 0; 4314 } 4315 4316 static void break_stripe_batch_list(struct stripe_head *head_sh, 4317 unsigned long handle_flags) 4318 { 4319 struct stripe_head *sh, *next; 4320 int i; 4321 int do_wakeup = 0; 4322 4323 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { 4324 4325 list_del_init(&sh->batch_list); 4326 4327 WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | 4328 (1 << STRIPE_SYNCING) | 4329 (1 << STRIPE_REPLACED) | 4330 (1 << STRIPE_DELAYED) | 4331 (1 << STRIPE_BIT_DELAY) | 4332 (1 << STRIPE_FULL_WRITE) | 4333 (1 << STRIPE_BIOFILL_RUN) | 4334 (1 << STRIPE_COMPUTE_RUN) | 4335 (1 << STRIPE_OPS_REQ_PENDING) | 4336 (1 << STRIPE_DISCARD) | 4337 (1 << STRIPE_BATCH_READY) | 4338 (1 << STRIPE_BATCH_ERR) | 4339 (1 << STRIPE_BITMAP_PENDING)), 4340 "stripe state: %lx\n", sh->state); 4341 WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | 4342 (1 << STRIPE_REPLACED)), 4343 "head stripe state: %lx\n", head_sh->state); 4344 4345 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | 4346 (1 << STRIPE_PREREAD_ACTIVE) | 4347 (1 << STRIPE_DEGRADED)), 4348 head_sh->state & (1 << STRIPE_INSYNC)); 4349 4350 sh->check_state = head_sh->check_state; 4351 sh->reconstruct_state = head_sh->reconstruct_state; 4352 for (i = 0; i < sh->disks; i++) { 4353 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) 4354 do_wakeup = 1; 4355 sh->dev[i].flags = head_sh->dev[i].flags & 4356 (~((1 << R5_WriteError) | (1 << R5_Overlap))); 4357 } 4358 spin_lock_irq(&sh->stripe_lock); 4359 sh->batch_head = NULL; 4360 spin_unlock_irq(&sh->stripe_lock); 4361 if (handle_flags == 0 || 4362 sh->state & handle_flags) 4363 set_bit(STRIPE_HANDLE, &sh->state); 4364 raid5_release_stripe(sh); 4365 } 4366 spin_lock_irq(&head_sh->stripe_lock); 4367 head_sh->batch_head = NULL; 4368 spin_unlock_irq(&head_sh->stripe_lock); 4369 for (i = 0; i < head_sh->disks; i++) 4370 if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) 4371 do_wakeup = 1; 4372 if (head_sh->state & handle_flags) 4373 set_bit(STRIPE_HANDLE, &head_sh->state); 4374 4375 if (do_wakeup) 4376 wake_up(&head_sh->raid_conf->wait_for_overlap); 4377 } 4378 4379 static void handle_stripe(struct stripe_head *sh) 4380 { 4381 struct stripe_head_state s; 4382 struct r5conf *conf = sh->raid_conf; 4383 int i; 4384 int prexor; 4385 int disks = sh->disks; 4386 struct r5dev *pdev, *qdev; 4387 4388 clear_bit(STRIPE_HANDLE, &sh->state); 4389 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { 4390 /* already being handled, ensure it gets handled 4391 * again when current action finishes */ 4392 set_bit(STRIPE_HANDLE, &sh->state); 4393 return; 4394 } 4395 4396 if (clear_batch_ready(sh) ) { 4397 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4398 return; 4399 } 4400 4401 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) 4402 break_stripe_batch_list(sh, 0); 4403 4404 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { 4405 spin_lock(&sh->stripe_lock); 4406 /* Cannot process 'sync' concurrently with 'discard' */ 4407 if (!test_bit(STRIPE_DISCARD, &sh->state) && 4408 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { 4409 set_bit(STRIPE_SYNCING, &sh->state); 4410 clear_bit(STRIPE_INSYNC, &sh->state); 4411 clear_bit(STRIPE_REPLACED, &sh->state); 4412 } 4413 spin_unlock(&sh->stripe_lock); 4414 } 4415 clear_bit(STRIPE_DELAYED, &sh->state); 4416 4417 pr_debug("handling stripe %llu, state=%#lx cnt=%d, " 4418 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", 4419 (unsigned long long)sh->sector, sh->state, 4420 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, 4421 sh->check_state, sh->reconstruct_state); 4422 4423 analyse_stripe(sh, &s); 4424 4425 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) 4426 goto finish; 4427 4428 if (s.handle_bad_blocks) { 4429 set_bit(STRIPE_HANDLE, &sh->state); 4430 goto finish; 4431 } 4432 4433 if (unlikely(s.blocked_rdev)) { 4434 if (s.syncing || s.expanding || s.expanded || 4435 s.replacing || s.to_write || s.written) { 4436 set_bit(STRIPE_HANDLE, &sh->state); 4437 goto finish; 4438 } 4439 /* There is nothing for the blocked_rdev to block */ 4440 rdev_dec_pending(s.blocked_rdev, conf->mddev); 4441 s.blocked_rdev = NULL; 4442 } 4443 4444 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { 4445 set_bit(STRIPE_OP_BIOFILL, &s.ops_request); 4446 set_bit(STRIPE_BIOFILL_RUN, &sh->state); 4447 } 4448 4449 pr_debug("locked=%d uptodate=%d to_read=%d" 4450 " to_write=%d failed=%d failed_num=%d,%d\n", 4451 s.locked, s.uptodate, s.to_read, s.to_write, s.failed, 4452 s.failed_num[0], s.failed_num[1]); 4453 /* check if the array has lost more than max_degraded devices and, 4454 * if so, some requests might need to be failed. 4455 */ 4456 if (s.failed > conf->max_degraded || s.log_failed) { 4457 sh->check_state = 0; 4458 sh->reconstruct_state = 0; 4459 break_stripe_batch_list(sh, 0); 4460 if (s.to_read+s.to_write+s.written) 4461 handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); 4462 if (s.syncing + s.replacing) 4463 handle_failed_sync(conf, sh, &s); 4464 } 4465 4466 /* Now we check to see if any write operations have recently 4467 * completed 4468 */ 4469 prexor = 0; 4470 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) 4471 prexor = 1; 4472 if (sh->reconstruct_state == reconstruct_state_drain_result || 4473 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { 4474 sh->reconstruct_state = reconstruct_state_idle; 4475 4476 /* All the 'written' buffers and the parity block are ready to 4477 * be written back to disk 4478 */ 4479 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && 4480 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); 4481 BUG_ON(sh->qd_idx >= 0 && 4482 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && 4483 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); 4484 for (i = disks; i--; ) { 4485 struct r5dev *dev = &sh->dev[i]; 4486 if (test_bit(R5_LOCKED, &dev->flags) && 4487 (i == sh->pd_idx || i == sh->qd_idx || 4488 dev->written || test_bit(R5_InJournal, 4489 &dev->flags))) { 4490 pr_debug("Writing block %d\n", i); 4491 set_bit(R5_Wantwrite, &dev->flags); 4492 if (prexor) 4493 continue; 4494 if (s.failed > 1) 4495 continue; 4496 if (!test_bit(R5_Insync, &dev->flags) || 4497 ((i == sh->pd_idx || i == sh->qd_idx) && 4498 s.failed == 0)) 4499 set_bit(STRIPE_INSYNC, &sh->state); 4500 } 4501 } 4502 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4503 s.dec_preread_active = 1; 4504 } 4505 4506 /* 4507 * might be able to return some write requests if the parity blocks 4508 * are safe, or on a failed drive 4509 */ 4510 pdev = &sh->dev[sh->pd_idx]; 4511 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) 4512 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); 4513 qdev = &sh->dev[sh->qd_idx]; 4514 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) 4515 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) 4516 || conf->level < 6; 4517 4518 if (s.written && 4519 (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) 4520 && !test_bit(R5_LOCKED, &pdev->flags) 4521 && (test_bit(R5_UPTODATE, &pdev->flags) || 4522 test_bit(R5_Discard, &pdev->flags))))) && 4523 (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) 4524 && !test_bit(R5_LOCKED, &qdev->flags) 4525 && (test_bit(R5_UPTODATE, &qdev->flags) || 4526 test_bit(R5_Discard, &qdev->flags)))))) 4527 handle_stripe_clean_event(conf, sh, disks, &s.return_bi); 4528 4529 if (s.just_cached) 4530 r5c_handle_cached_data_endio(conf, sh, disks, &s.return_bi); 4531 r5l_stripe_write_finished(sh); 4532 4533 /* Now we might consider reading some blocks, either to check/generate 4534 * parity, or to satisfy requests 4535 * or to load a block that is being partially written. 4536 */ 4537 if (s.to_read || s.non_overwrite 4538 || (conf->level == 6 && s.to_write && s.failed) 4539 || (s.syncing && (s.uptodate + s.compute < disks)) 4540 || s.replacing 4541 || s.expanding) 4542 handle_stripe_fill(sh, &s, disks); 4543 4544 /* 4545 * When the stripe finishes full journal write cycle (write to journal 4546 * and raid disk), this is the clean up procedure so it is ready for 4547 * next operation. 4548 */ 4549 r5c_finish_stripe_write_out(conf, sh, &s); 4550 4551 /* 4552 * Now to consider new write requests, cache write back and what else, 4553 * if anything should be read. We do not handle new writes when: 4554 * 1/ A 'write' operation (copy+xor) is already in flight. 4555 * 2/ A 'check' operation is in flight, as it may clobber the parity 4556 * block. 4557 * 3/ A r5c cache log write is in flight. 4558 */ 4559 4560 if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) { 4561 if (!r5c_is_writeback(conf->log)) { 4562 if (s.to_write) 4563 handle_stripe_dirtying(conf, sh, &s, disks); 4564 } else { /* write back cache */ 4565 int ret = 0; 4566 4567 /* First, try handle writes in caching phase */ 4568 if (s.to_write) 4569 ret = r5c_try_caching_write(conf, sh, &s, 4570 disks); 4571 /* 4572 * If caching phase failed: ret == -EAGAIN 4573 * OR 4574 * stripe under reclaim: !caching && injournal 4575 * 4576 * fall back to handle_stripe_dirtying() 4577 */ 4578 if (ret == -EAGAIN || 4579 /* stripe under reclaim: !caching && injournal */ 4580 (!test_bit(STRIPE_R5C_CACHING, &sh->state) && 4581 s.injournal > 0)) { 4582 ret = handle_stripe_dirtying(conf, sh, &s, 4583 disks); 4584 if (ret == -EAGAIN) 4585 goto finish; 4586 } 4587 } 4588 } 4589 4590 /* maybe we need to check and possibly fix the parity for this stripe 4591 * Any reads will already have been scheduled, so we just see if enough 4592 * data is available. The parity check is held off while parity 4593 * dependent operations are in flight. 4594 */ 4595 if (sh->check_state || 4596 (s.syncing && s.locked == 0 && 4597 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 4598 !test_bit(STRIPE_INSYNC, &sh->state))) { 4599 if (conf->level == 6) 4600 handle_parity_checks6(conf, sh, &s, disks); 4601 else 4602 handle_parity_checks5(conf, sh, &s, disks); 4603 } 4604 4605 if ((s.replacing || s.syncing) && s.locked == 0 4606 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) 4607 && !test_bit(STRIPE_REPLACED, &sh->state)) { 4608 /* Write out to replacement devices where possible */ 4609 for (i = 0; i < conf->raid_disks; i++) 4610 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { 4611 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); 4612 set_bit(R5_WantReplace, &sh->dev[i].flags); 4613 set_bit(R5_LOCKED, &sh->dev[i].flags); 4614 s.locked++; 4615 } 4616 if (s.replacing) 4617 set_bit(STRIPE_INSYNC, &sh->state); 4618 set_bit(STRIPE_REPLACED, &sh->state); 4619 } 4620 if ((s.syncing || s.replacing) && s.locked == 0 && 4621 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && 4622 test_bit(STRIPE_INSYNC, &sh->state)) { 4623 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 4624 clear_bit(STRIPE_SYNCING, &sh->state); 4625 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) 4626 wake_up(&conf->wait_for_overlap); 4627 } 4628 4629 /* If the failed drives are just a ReadError, then we might need 4630 * to progress the repair/check process 4631 */ 4632 if (s.failed <= conf->max_degraded && !conf->mddev->ro) 4633 for (i = 0; i < s.failed; i++) { 4634 struct r5dev *dev = &sh->dev[s.failed_num[i]]; 4635 if (test_bit(R5_ReadError, &dev->flags) 4636 && !test_bit(R5_LOCKED, &dev->flags) 4637 && test_bit(R5_UPTODATE, &dev->flags) 4638 ) { 4639 if (!test_bit(R5_ReWrite, &dev->flags)) { 4640 set_bit(R5_Wantwrite, &dev->flags); 4641 set_bit(R5_ReWrite, &dev->flags); 4642 set_bit(R5_LOCKED, &dev->flags); 4643 s.locked++; 4644 } else { 4645 /* let's read it back */ 4646 set_bit(R5_Wantread, &dev->flags); 4647 set_bit(R5_LOCKED, &dev->flags); 4648 s.locked++; 4649 } 4650 } 4651 } 4652 4653 /* Finish reconstruct operations initiated by the expansion process */ 4654 if (sh->reconstruct_state == reconstruct_state_result) { 4655 struct stripe_head *sh_src 4656 = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1); 4657 if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { 4658 /* sh cannot be written until sh_src has been read. 4659 * so arrange for sh to be delayed a little 4660 */ 4661 set_bit(STRIPE_DELAYED, &sh->state); 4662 set_bit(STRIPE_HANDLE, &sh->state); 4663 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, 4664 &sh_src->state)) 4665 atomic_inc(&conf->preread_active_stripes); 4666 raid5_release_stripe(sh_src); 4667 goto finish; 4668 } 4669 if (sh_src) 4670 raid5_release_stripe(sh_src); 4671 4672 sh->reconstruct_state = reconstruct_state_idle; 4673 clear_bit(STRIPE_EXPANDING, &sh->state); 4674 for (i = conf->raid_disks; i--; ) { 4675 set_bit(R5_Wantwrite, &sh->dev[i].flags); 4676 set_bit(R5_LOCKED, &sh->dev[i].flags); 4677 s.locked++; 4678 } 4679 } 4680 4681 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && 4682 !sh->reconstruct_state) { 4683 /* Need to write out all blocks after computing parity */ 4684 sh->disks = conf->raid_disks; 4685 stripe_set_idx(sh->sector, conf, 0, sh); 4686 schedule_reconstruction(sh, &s, 1, 1); 4687 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { 4688 clear_bit(STRIPE_EXPAND_READY, &sh->state); 4689 atomic_dec(&conf->reshape_stripes); 4690 wake_up(&conf->wait_for_overlap); 4691 md_done_sync(conf->mddev, STRIPE_SECTORS, 1); 4692 } 4693 4694 if (s.expanding && s.locked == 0 && 4695 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) 4696 handle_stripe_expansion(conf, sh); 4697 4698 finish: 4699 /* wait for this device to become unblocked */ 4700 if (unlikely(s.blocked_rdev)) { 4701 if (conf->mddev->external) 4702 md_wait_for_blocked_rdev(s.blocked_rdev, 4703 conf->mddev); 4704 else 4705 /* Internal metadata will immediately 4706 * be written by raid5d, so we don't 4707 * need to wait here. 4708 */ 4709 rdev_dec_pending(s.blocked_rdev, 4710 conf->mddev); 4711 } 4712 4713 if (s.handle_bad_blocks) 4714 for (i = disks; i--; ) { 4715 struct md_rdev *rdev; 4716 struct r5dev *dev = &sh->dev[i]; 4717 if (test_and_clear_bit(R5_WriteError, &dev->flags)) { 4718 /* We own a safe reference to the rdev */ 4719 rdev = conf->disks[i].rdev; 4720 if (!rdev_set_badblocks(rdev, sh->sector, 4721 STRIPE_SECTORS, 0)) 4722 md_error(conf->mddev, rdev); 4723 rdev_dec_pending(rdev, conf->mddev); 4724 } 4725 if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { 4726 rdev = conf->disks[i].rdev; 4727 rdev_clear_badblocks(rdev, sh->sector, 4728 STRIPE_SECTORS, 0); 4729 rdev_dec_pending(rdev, conf->mddev); 4730 } 4731 if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { 4732 rdev = conf->disks[i].replacement; 4733 if (!rdev) 4734 /* rdev have been moved down */ 4735 rdev = conf->disks[i].rdev; 4736 rdev_clear_badblocks(rdev, sh->sector, 4737 STRIPE_SECTORS, 0); 4738 rdev_dec_pending(rdev, conf->mddev); 4739 } 4740 } 4741 4742 if (s.ops_request) 4743 raid_run_ops(sh, s.ops_request); 4744 4745 ops_run_io(sh, &s); 4746 4747 if (s.dec_preread_active) { 4748 /* We delay this until after ops_run_io so that if make_request 4749 * is waiting on a flush, it won't continue until the writes 4750 * have actually been submitted. 4751 */ 4752 atomic_dec(&conf->preread_active_stripes); 4753 if (atomic_read(&conf->preread_active_stripes) < 4754 IO_THRESHOLD) 4755 md_wakeup_thread(conf->mddev->thread); 4756 } 4757 4758 if (!bio_list_empty(&s.return_bi)) { 4759 if (test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) { 4760 spin_lock_irq(&conf->device_lock); 4761 bio_list_merge(&conf->return_bi, &s.return_bi); 4762 spin_unlock_irq(&conf->device_lock); 4763 md_wakeup_thread(conf->mddev->thread); 4764 } else 4765 return_io(&s.return_bi); 4766 } 4767 4768 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); 4769 } 4770 4771 static void raid5_activate_delayed(struct r5conf *conf) 4772 { 4773 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { 4774 while (!list_empty(&conf->delayed_list)) { 4775 struct list_head *l = conf->delayed_list.next; 4776 struct stripe_head *sh; 4777 sh = list_entry(l, struct stripe_head, lru); 4778 list_del_init(l); 4779 clear_bit(STRIPE_DELAYED, &sh->state); 4780 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 4781 atomic_inc(&conf->preread_active_stripes); 4782 list_add_tail(&sh->lru, &conf->hold_list); 4783 raid5_wakeup_stripe_thread(sh); 4784 } 4785 } 4786 } 4787 4788 static void activate_bit_delay(struct r5conf *conf, 4789 struct list_head *temp_inactive_list) 4790 { 4791 /* device_lock is held */ 4792 struct list_head head; 4793 list_add(&head, &conf->bitmap_list); 4794 list_del_init(&conf->bitmap_list); 4795 while (!list_empty(&head)) { 4796 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); 4797 int hash; 4798 list_del_init(&sh->lru); 4799 atomic_inc(&sh->count); 4800 hash = sh->hash_lock_index; 4801 __release_stripe(conf, sh, &temp_inactive_list[hash]); 4802 } 4803 } 4804 4805 static int raid5_congested(struct mddev *mddev, int bits) 4806 { 4807 struct r5conf *conf = mddev->private; 4808 4809 /* No difference between reads and writes. Just check 4810 * how busy the stripe_cache is 4811 */ 4812 4813 if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) 4814 return 1; 4815 4816 /* Also checks whether there is pressure on r5cache log space */ 4817 if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) 4818 return 1; 4819 if (conf->quiesce) 4820 return 1; 4821 if (atomic_read(&conf->empty_inactive_list_nr)) 4822 return 1; 4823 4824 return 0; 4825 } 4826 4827 static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) 4828 { 4829 struct r5conf *conf = mddev->private; 4830 sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); 4831 unsigned int chunk_sectors; 4832 unsigned int bio_sectors = bio_sectors(bio); 4833 4834 chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors); 4835 return chunk_sectors >= 4836 ((sector & (chunk_sectors - 1)) + bio_sectors); 4837 } 4838 4839 /* 4840 * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) 4841 * later sampled by raid5d. 4842 */ 4843 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) 4844 { 4845 unsigned long flags; 4846 4847 spin_lock_irqsave(&conf->device_lock, flags); 4848 4849 bi->bi_next = conf->retry_read_aligned_list; 4850 conf->retry_read_aligned_list = bi; 4851 4852 spin_unlock_irqrestore(&conf->device_lock, flags); 4853 md_wakeup_thread(conf->mddev->thread); 4854 } 4855 4856 static struct bio *remove_bio_from_retry(struct r5conf *conf) 4857 { 4858 struct bio *bi; 4859 4860 bi = conf->retry_read_aligned; 4861 if (bi) { 4862 conf->retry_read_aligned = NULL; 4863 return bi; 4864 } 4865 bi = conf->retry_read_aligned_list; 4866 if(bi) { 4867 conf->retry_read_aligned_list = bi->bi_next; 4868 bi->bi_next = NULL; 4869 /* 4870 * this sets the active strip count to 1 and the processed 4871 * strip count to zero (upper 8 bits) 4872 */ 4873 raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ 4874 } 4875 4876 return bi; 4877 } 4878 4879 /* 4880 * The "raid5_align_endio" should check if the read succeeded and if it 4881 * did, call bio_endio on the original bio (having bio_put the new bio 4882 * first). 4883 * If the read failed.. 4884 */ 4885 static void raid5_align_endio(struct bio *bi) 4886 { 4887 struct bio* raid_bi = bi->bi_private; 4888 struct mddev *mddev; 4889 struct r5conf *conf; 4890 struct md_rdev *rdev; 4891 int error = bi->bi_error; 4892 4893 bio_put(bi); 4894 4895 rdev = (void*)raid_bi->bi_next; 4896 raid_bi->bi_next = NULL; 4897 mddev = rdev->mddev; 4898 conf = mddev->private; 4899 4900 rdev_dec_pending(rdev, conf->mddev); 4901 4902 if (!error) { 4903 trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), 4904 raid_bi, 0); 4905 bio_endio(raid_bi); 4906 if (atomic_dec_and_test(&conf->active_aligned_reads)) 4907 wake_up(&conf->wait_for_quiescent); 4908 return; 4909 } 4910 4911 pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); 4912 4913 add_bio_to_retry(raid_bi, conf); 4914 } 4915 4916 static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) 4917 { 4918 struct r5conf *conf = mddev->private; 4919 int dd_idx; 4920 struct bio* align_bi; 4921 struct md_rdev *rdev; 4922 sector_t end_sector; 4923 4924 if (!in_chunk_boundary(mddev, raid_bio)) { 4925 pr_debug("%s: non aligned\n", __func__); 4926 return 0; 4927 } 4928 /* 4929 * use bio_clone_mddev to make a copy of the bio 4930 */ 4931 align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); 4932 if (!align_bi) 4933 return 0; 4934 /* 4935 * set bi_end_io to a new function, and set bi_private to the 4936 * original bio. 4937 */ 4938 align_bi->bi_end_io = raid5_align_endio; 4939 align_bi->bi_private = raid_bio; 4940 /* 4941 * compute position 4942 */ 4943 align_bi->bi_iter.bi_sector = 4944 raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 4945 0, &dd_idx, NULL); 4946 4947 end_sector = bio_end_sector(align_bi); 4948 rcu_read_lock(); 4949 rdev = rcu_dereference(conf->disks[dd_idx].replacement); 4950 if (!rdev || test_bit(Faulty, &rdev->flags) || 4951 rdev->recovery_offset < end_sector) { 4952 rdev = rcu_dereference(conf->disks[dd_idx].rdev); 4953 if (rdev && 4954 (test_bit(Faulty, &rdev->flags) || 4955 !(test_bit(In_sync, &rdev->flags) || 4956 rdev->recovery_offset >= end_sector))) 4957 rdev = NULL; 4958 } 4959 if (rdev) { 4960 sector_t first_bad; 4961 int bad_sectors; 4962 4963 atomic_inc(&rdev->nr_pending); 4964 rcu_read_unlock(); 4965 raid_bio->bi_next = (void*)rdev; 4966 align_bi->bi_bdev = rdev->bdev; 4967 bio_clear_flag(align_bi, BIO_SEG_VALID); 4968 4969 if (is_badblock(rdev, align_bi->bi_iter.bi_sector, 4970 bio_sectors(align_bi), 4971 &first_bad, &bad_sectors)) { 4972 bio_put(align_bi); 4973 rdev_dec_pending(rdev, mddev); 4974 return 0; 4975 } 4976 4977 /* No reshape active, so we can trust rdev->data_offset */ 4978 align_bi->bi_iter.bi_sector += rdev->data_offset; 4979 4980 spin_lock_irq(&conf->device_lock); 4981 wait_event_lock_irq(conf->wait_for_quiescent, 4982 conf->quiesce == 0, 4983 conf->device_lock); 4984 atomic_inc(&conf->active_aligned_reads); 4985 spin_unlock_irq(&conf->device_lock); 4986 4987 if (mddev->gendisk) 4988 trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), 4989 align_bi, disk_devt(mddev->gendisk), 4990 raid_bio->bi_iter.bi_sector); 4991 generic_make_request(align_bi); 4992 return 1; 4993 } else { 4994 rcu_read_unlock(); 4995 bio_put(align_bi); 4996 return 0; 4997 } 4998 } 4999 5000 static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio) 5001 { 5002 struct bio *split; 5003 5004 do { 5005 sector_t sector = raid_bio->bi_iter.bi_sector; 5006 unsigned chunk_sects = mddev->chunk_sectors; 5007 unsigned sectors = chunk_sects - (sector & (chunk_sects-1)); 5008 5009 if (sectors < bio_sectors(raid_bio)) { 5010 split = bio_split(raid_bio, sectors, GFP_NOIO, fs_bio_set); 5011 bio_chain(split, raid_bio); 5012 } else 5013 split = raid_bio; 5014 5015 if (!raid5_read_one_chunk(mddev, split)) { 5016 if (split != raid_bio) 5017 generic_make_request(raid_bio); 5018 return split; 5019 } 5020 } while (split != raid_bio); 5021 5022 return NULL; 5023 } 5024 5025 /* __get_priority_stripe - get the next stripe to process 5026 * 5027 * Full stripe writes are allowed to pass preread active stripes up until 5028 * the bypass_threshold is exceeded. In general the bypass_count 5029 * increments when the handle_list is handled before the hold_list; however, it 5030 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a 5031 * stripe with in flight i/o. The bypass_count will be reset when the 5032 * head of the hold_list has changed, i.e. the head was promoted to the 5033 * handle_list. 5034 */ 5035 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) 5036 { 5037 struct stripe_head *sh = NULL, *tmp; 5038 struct list_head *handle_list = NULL; 5039 struct r5worker_group *wg = NULL; 5040 5041 if (conf->worker_cnt_per_group == 0) { 5042 handle_list = &conf->handle_list; 5043 } else if (group != ANY_GROUP) { 5044 handle_list = &conf->worker_groups[group].handle_list; 5045 wg = &conf->worker_groups[group]; 5046 } else { 5047 int i; 5048 for (i = 0; i < conf->group_cnt; i++) { 5049 handle_list = &conf->worker_groups[i].handle_list; 5050 wg = &conf->worker_groups[i]; 5051 if (!list_empty(handle_list)) 5052 break; 5053 } 5054 } 5055 5056 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", 5057 __func__, 5058 list_empty(handle_list) ? "empty" : "busy", 5059 list_empty(&conf->hold_list) ? "empty" : "busy", 5060 atomic_read(&conf->pending_full_writes), conf->bypass_count); 5061 5062 if (!list_empty(handle_list)) { 5063 sh = list_entry(handle_list->next, typeof(*sh), lru); 5064 5065 if (list_empty(&conf->hold_list)) 5066 conf->bypass_count = 0; 5067 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { 5068 if (conf->hold_list.next == conf->last_hold) 5069 conf->bypass_count++; 5070 else { 5071 conf->last_hold = conf->hold_list.next; 5072 conf->bypass_count -= conf->bypass_threshold; 5073 if (conf->bypass_count < 0) 5074 conf->bypass_count = 0; 5075 } 5076 } 5077 } else if (!list_empty(&conf->hold_list) && 5078 ((conf->bypass_threshold && 5079 conf->bypass_count > conf->bypass_threshold) || 5080 atomic_read(&conf->pending_full_writes) == 0)) { 5081 5082 list_for_each_entry(tmp, &conf->hold_list, lru) { 5083 if (conf->worker_cnt_per_group == 0 || 5084 group == ANY_GROUP || 5085 !cpu_online(tmp->cpu) || 5086 cpu_to_group(tmp->cpu) == group) { 5087 sh = tmp; 5088 break; 5089 } 5090 } 5091 5092 if (sh) { 5093 conf->bypass_count -= conf->bypass_threshold; 5094 if (conf->bypass_count < 0) 5095 conf->bypass_count = 0; 5096 } 5097 wg = NULL; 5098 } 5099 5100 if (!sh) 5101 return NULL; 5102 5103 if (wg) { 5104 wg->stripes_cnt--; 5105 sh->group = NULL; 5106 } 5107 list_del_init(&sh->lru); 5108 BUG_ON(atomic_inc_return(&sh->count) != 1); 5109 return sh; 5110 } 5111 5112 struct raid5_plug_cb { 5113 struct blk_plug_cb cb; 5114 struct list_head list; 5115 struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; 5116 }; 5117 5118 static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) 5119 { 5120 struct raid5_plug_cb *cb = container_of( 5121 blk_cb, struct raid5_plug_cb, cb); 5122 struct stripe_head *sh; 5123 struct mddev *mddev = cb->cb.data; 5124 struct r5conf *conf = mddev->private; 5125 int cnt = 0; 5126 int hash; 5127 5128 if (cb->list.next && !list_empty(&cb->list)) { 5129 spin_lock_irq(&conf->device_lock); 5130 while (!list_empty(&cb->list)) { 5131 sh = list_first_entry(&cb->list, struct stripe_head, lru); 5132 list_del_init(&sh->lru); 5133 /* 5134 * avoid race release_stripe_plug() sees 5135 * STRIPE_ON_UNPLUG_LIST clear but the stripe 5136 * is still in our list 5137 */ 5138 smp_mb__before_atomic(); 5139 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); 5140 /* 5141 * STRIPE_ON_RELEASE_LIST could be set here. In that 5142 * case, the count is always > 1 here 5143 */ 5144 hash = sh->hash_lock_index; 5145 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); 5146 cnt++; 5147 } 5148 spin_unlock_irq(&conf->device_lock); 5149 } 5150 release_inactive_stripe_list(conf, cb->temp_inactive_list, 5151 NR_STRIPE_HASH_LOCKS); 5152 if (mddev->queue) 5153 trace_block_unplug(mddev->queue, cnt, !from_schedule); 5154 kfree(cb); 5155 } 5156 5157 static void release_stripe_plug(struct mddev *mddev, 5158 struct stripe_head *sh) 5159 { 5160 struct blk_plug_cb *blk_cb = blk_check_plugged( 5161 raid5_unplug, mddev, 5162 sizeof(struct raid5_plug_cb)); 5163 struct raid5_plug_cb *cb; 5164 5165 if (!blk_cb) { 5166 raid5_release_stripe(sh); 5167 return; 5168 } 5169 5170 cb = container_of(blk_cb, struct raid5_plug_cb, cb); 5171 5172 if (cb->list.next == NULL) { 5173 int i; 5174 INIT_LIST_HEAD(&cb->list); 5175 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5176 INIT_LIST_HEAD(cb->temp_inactive_list + i); 5177 } 5178 5179 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) 5180 list_add_tail(&sh->lru, &cb->list); 5181 else 5182 raid5_release_stripe(sh); 5183 } 5184 5185 static void make_discard_request(struct mddev *mddev, struct bio *bi) 5186 { 5187 struct r5conf *conf = mddev->private; 5188 sector_t logical_sector, last_sector; 5189 struct stripe_head *sh; 5190 int remaining; 5191 int stripe_sectors; 5192 5193 if (mddev->reshape_position != MaxSector) 5194 /* Skip discard while reshape is happening */ 5195 return; 5196 5197 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 5198 last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); 5199 5200 bi->bi_next = NULL; 5201 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 5202 5203 stripe_sectors = conf->chunk_sectors * 5204 (conf->raid_disks - conf->max_degraded); 5205 logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, 5206 stripe_sectors); 5207 sector_div(last_sector, stripe_sectors); 5208 5209 logical_sector *= conf->chunk_sectors; 5210 last_sector *= conf->chunk_sectors; 5211 5212 for (; logical_sector < last_sector; 5213 logical_sector += STRIPE_SECTORS) { 5214 DEFINE_WAIT(w); 5215 int d; 5216 again: 5217 sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0); 5218 prepare_to_wait(&conf->wait_for_overlap, &w, 5219 TASK_UNINTERRUPTIBLE); 5220 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 5221 if (test_bit(STRIPE_SYNCING, &sh->state)) { 5222 raid5_release_stripe(sh); 5223 schedule(); 5224 goto again; 5225 } 5226 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); 5227 spin_lock_irq(&sh->stripe_lock); 5228 for (d = 0; d < conf->raid_disks; d++) { 5229 if (d == sh->pd_idx || d == sh->qd_idx) 5230 continue; 5231 if (sh->dev[d].towrite || sh->dev[d].toread) { 5232 set_bit(R5_Overlap, &sh->dev[d].flags); 5233 spin_unlock_irq(&sh->stripe_lock); 5234 raid5_release_stripe(sh); 5235 schedule(); 5236 goto again; 5237 } 5238 } 5239 set_bit(STRIPE_DISCARD, &sh->state); 5240 finish_wait(&conf->wait_for_overlap, &w); 5241 sh->overwrite_disks = 0; 5242 for (d = 0; d < conf->raid_disks; d++) { 5243 if (d == sh->pd_idx || d == sh->qd_idx) 5244 continue; 5245 sh->dev[d].towrite = bi; 5246 set_bit(R5_OVERWRITE, &sh->dev[d].flags); 5247 raid5_inc_bi_active_stripes(bi); 5248 sh->overwrite_disks++; 5249 } 5250 spin_unlock_irq(&sh->stripe_lock); 5251 if (conf->mddev->bitmap) { 5252 for (d = 0; 5253 d < conf->raid_disks - conf->max_degraded; 5254 d++) 5255 bitmap_startwrite(mddev->bitmap, 5256 sh->sector, 5257 STRIPE_SECTORS, 5258 0); 5259 sh->bm_seq = conf->seq_flush + 1; 5260 set_bit(STRIPE_BIT_DELAY, &sh->state); 5261 } 5262 5263 set_bit(STRIPE_HANDLE, &sh->state); 5264 clear_bit(STRIPE_DELAYED, &sh->state); 5265 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 5266 atomic_inc(&conf->preread_active_stripes); 5267 release_stripe_plug(mddev, sh); 5268 } 5269 5270 remaining = raid5_dec_bi_active_stripes(bi); 5271 if (remaining == 0) { 5272 md_write_end(mddev); 5273 bio_endio(bi); 5274 } 5275 } 5276 5277 static void raid5_make_request(struct mddev *mddev, struct bio * bi) 5278 { 5279 struct r5conf *conf = mddev->private; 5280 int dd_idx; 5281 sector_t new_sector; 5282 sector_t logical_sector, last_sector; 5283 struct stripe_head *sh; 5284 const int rw = bio_data_dir(bi); 5285 int remaining; 5286 DEFINE_WAIT(w); 5287 bool do_prepare; 5288 bool do_flush = false; 5289 5290 if (unlikely(bi->bi_opf & REQ_PREFLUSH)) { 5291 int ret = r5l_handle_flush_request(conf->log, bi); 5292 5293 if (ret == 0) 5294 return; 5295 if (ret == -ENODEV) { 5296 md_flush_request(mddev, bi); 5297 return; 5298 } 5299 /* ret == -EAGAIN, fallback */ 5300 /* 5301 * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH, 5302 * we need to flush journal device 5303 */ 5304 do_flush = bi->bi_opf & REQ_PREFLUSH; 5305 } 5306 5307 md_write_start(mddev, bi); 5308 5309 /* 5310 * If array is degraded, better not do chunk aligned read because 5311 * later we might have to read it again in order to reconstruct 5312 * data on failed drives. 5313 */ 5314 if (rw == READ && mddev->degraded == 0 && 5315 !r5c_is_writeback(conf->log) && 5316 mddev->reshape_position == MaxSector) { 5317 bi = chunk_aligned_read(mddev, bi); 5318 if (!bi) 5319 return; 5320 } 5321 5322 if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) { 5323 make_discard_request(mddev, bi); 5324 return; 5325 } 5326 5327 logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); 5328 last_sector = bio_end_sector(bi); 5329 bi->bi_next = NULL; 5330 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ 5331 5332 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); 5333 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { 5334 int previous; 5335 int seq; 5336 5337 do_prepare = false; 5338 retry: 5339 seq = read_seqcount_begin(&conf->gen_lock); 5340 previous = 0; 5341 if (do_prepare) 5342 prepare_to_wait(&conf->wait_for_overlap, &w, 5343 TASK_UNINTERRUPTIBLE); 5344 if (unlikely(conf->reshape_progress != MaxSector)) { 5345 /* spinlock is needed as reshape_progress may be 5346 * 64bit on a 32bit platform, and so it might be 5347 * possible to see a half-updated value 5348 * Of course reshape_progress could change after 5349 * the lock is dropped, so once we get a reference 5350 * to the stripe that we think it is, we will have 5351 * to check again. 5352 */ 5353 spin_lock_irq(&conf->device_lock); 5354 if (mddev->reshape_backwards 5355 ? logical_sector < conf->reshape_progress 5356 : logical_sector >= conf->reshape_progress) { 5357 previous = 1; 5358 } else { 5359 if (mddev->reshape_backwards 5360 ? logical_sector < conf->reshape_safe 5361 : logical_sector >= conf->reshape_safe) { 5362 spin_unlock_irq(&conf->device_lock); 5363 schedule(); 5364 do_prepare = true; 5365 goto retry; 5366 } 5367 } 5368 spin_unlock_irq(&conf->device_lock); 5369 } 5370 5371 new_sector = raid5_compute_sector(conf, logical_sector, 5372 previous, 5373 &dd_idx, NULL); 5374 pr_debug("raid456: raid5_make_request, sector %llu logical %llu\n", 5375 (unsigned long long)new_sector, 5376 (unsigned long long)logical_sector); 5377 5378 sh = raid5_get_active_stripe(conf, new_sector, previous, 5379 (bi->bi_opf & REQ_RAHEAD), 0); 5380 if (sh) { 5381 if (unlikely(previous)) { 5382 /* expansion might have moved on while waiting for a 5383 * stripe, so we must do the range check again. 5384 * Expansion could still move past after this 5385 * test, but as we are holding a reference to 5386 * 'sh', we know that if that happens, 5387 * STRIPE_EXPANDING will get set and the expansion 5388 * won't proceed until we finish with the stripe. 5389 */ 5390 int must_retry = 0; 5391 spin_lock_irq(&conf->device_lock); 5392 if (mddev->reshape_backwards 5393 ? logical_sector >= conf->reshape_progress 5394 : logical_sector < conf->reshape_progress) 5395 /* mismatch, need to try again */ 5396 must_retry = 1; 5397 spin_unlock_irq(&conf->device_lock); 5398 if (must_retry) { 5399 raid5_release_stripe(sh); 5400 schedule(); 5401 do_prepare = true; 5402 goto retry; 5403 } 5404 } 5405 if (read_seqcount_retry(&conf->gen_lock, seq)) { 5406 /* Might have got the wrong stripe_head 5407 * by accident 5408 */ 5409 raid5_release_stripe(sh); 5410 goto retry; 5411 } 5412 5413 if (rw == WRITE && 5414 logical_sector >= mddev->suspend_lo && 5415 logical_sector < mddev->suspend_hi) { 5416 raid5_release_stripe(sh); 5417 /* As the suspend_* range is controlled by 5418 * userspace, we want an interruptible 5419 * wait. 5420 */ 5421 flush_signals(current); 5422 prepare_to_wait(&conf->wait_for_overlap, 5423 &w, TASK_INTERRUPTIBLE); 5424 if (logical_sector >= mddev->suspend_lo && 5425 logical_sector < mddev->suspend_hi) { 5426 schedule(); 5427 do_prepare = true; 5428 } 5429 goto retry; 5430 } 5431 5432 if (test_bit(STRIPE_EXPANDING, &sh->state) || 5433 !add_stripe_bio(sh, bi, dd_idx, rw, previous)) { 5434 /* Stripe is busy expanding or 5435 * add failed due to overlap. Flush everything 5436 * and wait a while 5437 */ 5438 md_wakeup_thread(mddev->thread); 5439 raid5_release_stripe(sh); 5440 schedule(); 5441 do_prepare = true; 5442 goto retry; 5443 } 5444 if (do_flush) { 5445 set_bit(STRIPE_R5C_PREFLUSH, &sh->state); 5446 /* we only need flush for one stripe */ 5447 do_flush = false; 5448 } 5449 5450 set_bit(STRIPE_HANDLE, &sh->state); 5451 clear_bit(STRIPE_DELAYED, &sh->state); 5452 if ((!sh->batch_head || sh == sh->batch_head) && 5453 (bi->bi_opf & REQ_SYNC) && 5454 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) 5455 atomic_inc(&conf->preread_active_stripes); 5456 release_stripe_plug(mddev, sh); 5457 } else { 5458 /* cannot get stripe for read-ahead, just give-up */ 5459 bi->bi_error = -EIO; 5460 break; 5461 } 5462 } 5463 finish_wait(&conf->wait_for_overlap, &w); 5464 5465 remaining = raid5_dec_bi_active_stripes(bi); 5466 if (remaining == 0) { 5467 5468 if ( rw == WRITE ) 5469 md_write_end(mddev); 5470 5471 trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), 5472 bi, 0); 5473 bio_endio(bi); 5474 } 5475 } 5476 5477 static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); 5478 5479 static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) 5480 { 5481 /* reshaping is quite different to recovery/resync so it is 5482 * handled quite separately ... here. 5483 * 5484 * On each call to sync_request, we gather one chunk worth of 5485 * destination stripes and flag them as expanding. 5486 * Then we find all the source stripes and request reads. 5487 * As the reads complete, handle_stripe will copy the data 5488 * into the destination stripe and release that stripe. 5489 */ 5490 struct r5conf *conf = mddev->private; 5491 struct stripe_head *sh; 5492 sector_t first_sector, last_sector; 5493 int raid_disks = conf->previous_raid_disks; 5494 int data_disks = raid_disks - conf->max_degraded; 5495 int new_data_disks = conf->raid_disks - conf->max_degraded; 5496 int i; 5497 int dd_idx; 5498 sector_t writepos, readpos, safepos; 5499 sector_t stripe_addr; 5500 int reshape_sectors; 5501 struct list_head stripes; 5502 sector_t retn; 5503 5504 if (sector_nr == 0) { 5505 /* If restarting in the middle, skip the initial sectors */ 5506 if (mddev->reshape_backwards && 5507 conf->reshape_progress < raid5_size(mddev, 0, 0)) { 5508 sector_nr = raid5_size(mddev, 0, 0) 5509 - conf->reshape_progress; 5510 } else if (mddev->reshape_backwards && 5511 conf->reshape_progress == MaxSector) { 5512 /* shouldn't happen, but just in case, finish up.*/ 5513 sector_nr = MaxSector; 5514 } else if (!mddev->reshape_backwards && 5515 conf->reshape_progress > 0) 5516 sector_nr = conf->reshape_progress; 5517 sector_div(sector_nr, new_data_disks); 5518 if (sector_nr) { 5519 mddev->curr_resync_completed = sector_nr; 5520 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5521 *skipped = 1; 5522 retn = sector_nr; 5523 goto finish; 5524 } 5525 } 5526 5527 /* We need to process a full chunk at a time. 5528 * If old and new chunk sizes differ, we need to process the 5529 * largest of these 5530 */ 5531 5532 reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors); 5533 5534 /* We update the metadata at least every 10 seconds, or when 5535 * the data about to be copied would over-write the source of 5536 * the data at the front of the range. i.e. one new_stripe 5537 * along from reshape_progress new_maps to after where 5538 * reshape_safe old_maps to 5539 */ 5540 writepos = conf->reshape_progress; 5541 sector_div(writepos, new_data_disks); 5542 readpos = conf->reshape_progress; 5543 sector_div(readpos, data_disks); 5544 safepos = conf->reshape_safe; 5545 sector_div(safepos, data_disks); 5546 if (mddev->reshape_backwards) { 5547 BUG_ON(writepos < reshape_sectors); 5548 writepos -= reshape_sectors; 5549 readpos += reshape_sectors; 5550 safepos += reshape_sectors; 5551 } else { 5552 writepos += reshape_sectors; 5553 /* readpos and safepos are worst-case calculations. 5554 * A negative number is overly pessimistic, and causes 5555 * obvious problems for unsigned storage. So clip to 0. 5556 */ 5557 readpos -= min_t(sector_t, reshape_sectors, readpos); 5558 safepos -= min_t(sector_t, reshape_sectors, safepos); 5559 } 5560 5561 /* Having calculated the 'writepos' possibly use it 5562 * to set 'stripe_addr' which is where we will write to. 5563 */ 5564 if (mddev->reshape_backwards) { 5565 BUG_ON(conf->reshape_progress == 0); 5566 stripe_addr = writepos; 5567 BUG_ON((mddev->dev_sectors & 5568 ~((sector_t)reshape_sectors - 1)) 5569 - reshape_sectors - stripe_addr 5570 != sector_nr); 5571 } else { 5572 BUG_ON(writepos != sector_nr + reshape_sectors); 5573 stripe_addr = sector_nr; 5574 } 5575 5576 /* 'writepos' is the most advanced device address we might write. 5577 * 'readpos' is the least advanced device address we might read. 5578 * 'safepos' is the least address recorded in the metadata as having 5579 * been reshaped. 5580 * If there is a min_offset_diff, these are adjusted either by 5581 * increasing the safepos/readpos if diff is negative, or 5582 * increasing writepos if diff is positive. 5583 * If 'readpos' is then behind 'writepos', there is no way that we can 5584 * ensure safety in the face of a crash - that must be done by userspace 5585 * making a backup of the data. So in that case there is no particular 5586 * rush to update metadata. 5587 * Otherwise if 'safepos' is behind 'writepos', then we really need to 5588 * update the metadata to advance 'safepos' to match 'readpos' so that 5589 * we can be safe in the event of a crash. 5590 * So we insist on updating metadata if safepos is behind writepos and 5591 * readpos is beyond writepos. 5592 * In any case, update the metadata every 10 seconds. 5593 * Maybe that number should be configurable, but I'm not sure it is 5594 * worth it.... maybe it could be a multiple of safemode_delay??? 5595 */ 5596 if (conf->min_offset_diff < 0) { 5597 safepos += -conf->min_offset_diff; 5598 readpos += -conf->min_offset_diff; 5599 } else 5600 writepos += conf->min_offset_diff; 5601 5602 if ((mddev->reshape_backwards 5603 ? (safepos > writepos && readpos < writepos) 5604 : (safepos < writepos && readpos > writepos)) || 5605 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { 5606 /* Cannot proceed until we've updated the superblock... */ 5607 wait_event(conf->wait_for_overlap, 5608 atomic_read(&conf->reshape_stripes)==0 5609 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5610 if (atomic_read(&conf->reshape_stripes) != 0) 5611 return 0; 5612 mddev->reshape_position = conf->reshape_progress; 5613 mddev->curr_resync_completed = sector_nr; 5614 conf->reshape_checkpoint = jiffies; 5615 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 5616 md_wakeup_thread(mddev->thread); 5617 wait_event(mddev->sb_wait, mddev->sb_flags == 0 || 5618 test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5619 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5620 return 0; 5621 spin_lock_irq(&conf->device_lock); 5622 conf->reshape_safe = mddev->reshape_position; 5623 spin_unlock_irq(&conf->device_lock); 5624 wake_up(&conf->wait_for_overlap); 5625 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5626 } 5627 5628 INIT_LIST_HEAD(&stripes); 5629 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { 5630 int j; 5631 int skipped_disk = 0; 5632 sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1); 5633 set_bit(STRIPE_EXPANDING, &sh->state); 5634 atomic_inc(&conf->reshape_stripes); 5635 /* If any of this stripe is beyond the end of the old 5636 * array, then we need to zero those blocks 5637 */ 5638 for (j=sh->disks; j--;) { 5639 sector_t s; 5640 if (j == sh->pd_idx) 5641 continue; 5642 if (conf->level == 6 && 5643 j == sh->qd_idx) 5644 continue; 5645 s = raid5_compute_blocknr(sh, j, 0); 5646 if (s < raid5_size(mddev, 0, 0)) { 5647 skipped_disk = 1; 5648 continue; 5649 } 5650 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); 5651 set_bit(R5_Expanded, &sh->dev[j].flags); 5652 set_bit(R5_UPTODATE, &sh->dev[j].flags); 5653 } 5654 if (!skipped_disk) { 5655 set_bit(STRIPE_EXPAND_READY, &sh->state); 5656 set_bit(STRIPE_HANDLE, &sh->state); 5657 } 5658 list_add(&sh->lru, &stripes); 5659 } 5660 spin_lock_irq(&conf->device_lock); 5661 if (mddev->reshape_backwards) 5662 conf->reshape_progress -= reshape_sectors * new_data_disks; 5663 else 5664 conf->reshape_progress += reshape_sectors * new_data_disks; 5665 spin_unlock_irq(&conf->device_lock); 5666 /* Ok, those stripe are ready. We can start scheduling 5667 * reads on the source stripes. 5668 * The source stripes are determined by mapping the first and last 5669 * block on the destination stripes. 5670 */ 5671 first_sector = 5672 raid5_compute_sector(conf, stripe_addr*(new_data_disks), 5673 1, &dd_idx, NULL); 5674 last_sector = 5675 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) 5676 * new_data_disks - 1), 5677 1, &dd_idx, NULL); 5678 if (last_sector >= mddev->dev_sectors) 5679 last_sector = mddev->dev_sectors - 1; 5680 while (first_sector <= last_sector) { 5681 sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1); 5682 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); 5683 set_bit(STRIPE_HANDLE, &sh->state); 5684 raid5_release_stripe(sh); 5685 first_sector += STRIPE_SECTORS; 5686 } 5687 /* Now that the sources are clearly marked, we can release 5688 * the destination stripes 5689 */ 5690 while (!list_empty(&stripes)) { 5691 sh = list_entry(stripes.next, struct stripe_head, lru); 5692 list_del_init(&sh->lru); 5693 raid5_release_stripe(sh); 5694 } 5695 /* If this takes us to the resync_max point where we have to pause, 5696 * then we need to write out the superblock. 5697 */ 5698 sector_nr += reshape_sectors; 5699 retn = reshape_sectors; 5700 finish: 5701 if (mddev->curr_resync_completed > mddev->resync_max || 5702 (sector_nr - mddev->curr_resync_completed) * 2 5703 >= mddev->resync_max - mddev->curr_resync_completed) { 5704 /* Cannot proceed until we've updated the superblock... */ 5705 wait_event(conf->wait_for_overlap, 5706 atomic_read(&conf->reshape_stripes) == 0 5707 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5708 if (atomic_read(&conf->reshape_stripes) != 0) 5709 goto ret; 5710 mddev->reshape_position = conf->reshape_progress; 5711 mddev->curr_resync_completed = sector_nr; 5712 conf->reshape_checkpoint = jiffies; 5713 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 5714 md_wakeup_thread(mddev->thread); 5715 wait_event(mddev->sb_wait, 5716 !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags) 5717 || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); 5718 if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) 5719 goto ret; 5720 spin_lock_irq(&conf->device_lock); 5721 conf->reshape_safe = mddev->reshape_position; 5722 spin_unlock_irq(&conf->device_lock); 5723 wake_up(&conf->wait_for_overlap); 5724 sysfs_notify(&mddev->kobj, NULL, "sync_completed"); 5725 } 5726 ret: 5727 return retn; 5728 } 5729 5730 static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_nr, 5731 int *skipped) 5732 { 5733 struct r5conf *conf = mddev->private; 5734 struct stripe_head *sh; 5735 sector_t max_sector = mddev->dev_sectors; 5736 sector_t sync_blocks; 5737 int still_degraded = 0; 5738 int i; 5739 5740 if (sector_nr >= max_sector) { 5741 /* just being told to finish up .. nothing much to do */ 5742 5743 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { 5744 end_reshape(conf); 5745 return 0; 5746 } 5747 5748 if (mddev->curr_resync < max_sector) /* aborted */ 5749 bitmap_end_sync(mddev->bitmap, mddev->curr_resync, 5750 &sync_blocks, 1); 5751 else /* completed sync */ 5752 conf->fullsync = 0; 5753 bitmap_close_sync(mddev->bitmap); 5754 5755 return 0; 5756 } 5757 5758 /* Allow raid5_quiesce to complete */ 5759 wait_event(conf->wait_for_overlap, conf->quiesce != 2); 5760 5761 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) 5762 return reshape_request(mddev, sector_nr, skipped); 5763 5764 /* No need to check resync_max as we never do more than one 5765 * stripe, and as resync_max will always be on a chunk boundary, 5766 * if the check in md_do_sync didn't fire, there is no chance 5767 * of overstepping resync_max here 5768 */ 5769 5770 /* if there is too many failed drives and we are trying 5771 * to resync, then assert that we are finished, because there is 5772 * nothing we can do. 5773 */ 5774 if (mddev->degraded >= conf->max_degraded && 5775 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { 5776 sector_t rv = mddev->dev_sectors - sector_nr; 5777 *skipped = 1; 5778 return rv; 5779 } 5780 if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && 5781 !conf->fullsync && 5782 !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && 5783 sync_blocks >= STRIPE_SECTORS) { 5784 /* we can skip this block, and probably more */ 5785 sync_blocks /= STRIPE_SECTORS; 5786 *skipped = 1; 5787 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ 5788 } 5789 5790 bitmap_cond_end_sync(mddev->bitmap, sector_nr, false); 5791 5792 sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0); 5793 if (sh == NULL) { 5794 sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0); 5795 /* make sure we don't swamp the stripe cache if someone else 5796 * is trying to get access 5797 */ 5798 schedule_timeout_uninterruptible(1); 5799 } 5800 /* Need to check if array will still be degraded after recovery/resync 5801 * Note in case of > 1 drive failures it's possible we're rebuilding 5802 * one drive while leaving another faulty drive in array. 5803 */ 5804 rcu_read_lock(); 5805 for (i = 0; i < conf->raid_disks; i++) { 5806 struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); 5807 5808 if (rdev == NULL || test_bit(Faulty, &rdev->flags)) 5809 still_degraded = 1; 5810 } 5811 rcu_read_unlock(); 5812 5813 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); 5814 5815 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); 5816 set_bit(STRIPE_HANDLE, &sh->state); 5817 5818 raid5_release_stripe(sh); 5819 5820 return STRIPE_SECTORS; 5821 } 5822 5823 static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) 5824 { 5825 /* We may not be able to submit a whole bio at once as there 5826 * may not be enough stripe_heads available. 5827 * We cannot pre-allocate enough stripe_heads as we may need 5828 * more than exist in the cache (if we allow ever large chunks). 5829 * So we do one stripe head at a time and record in 5830 * ->bi_hw_segments how many have been done. 5831 * 5832 * We *know* that this entire raid_bio is in one chunk, so 5833 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. 5834 */ 5835 struct stripe_head *sh; 5836 int dd_idx; 5837 sector_t sector, logical_sector, last_sector; 5838 int scnt = 0; 5839 int remaining; 5840 int handled = 0; 5841 5842 logical_sector = raid_bio->bi_iter.bi_sector & 5843 ~((sector_t)STRIPE_SECTORS-1); 5844 sector = raid5_compute_sector(conf, logical_sector, 5845 0, &dd_idx, NULL); 5846 last_sector = bio_end_sector(raid_bio); 5847 5848 for (; logical_sector < last_sector; 5849 logical_sector += STRIPE_SECTORS, 5850 sector += STRIPE_SECTORS, 5851 scnt++) { 5852 5853 if (scnt < raid5_bi_processed_stripes(raid_bio)) 5854 /* already done this stripe */ 5855 continue; 5856 5857 sh = raid5_get_active_stripe(conf, sector, 0, 1, 1); 5858 5859 if (!sh) { 5860 /* failed to get a stripe - must wait */ 5861 raid5_set_bi_processed_stripes(raid_bio, scnt); 5862 conf->retry_read_aligned = raid_bio; 5863 return handled; 5864 } 5865 5866 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { 5867 raid5_release_stripe(sh); 5868 raid5_set_bi_processed_stripes(raid_bio, scnt); 5869 conf->retry_read_aligned = raid_bio; 5870 return handled; 5871 } 5872 5873 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); 5874 handle_stripe(sh); 5875 raid5_release_stripe(sh); 5876 handled++; 5877 } 5878 remaining = raid5_dec_bi_active_stripes(raid_bio); 5879 if (remaining == 0) { 5880 trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), 5881 raid_bio, 0); 5882 bio_endio(raid_bio); 5883 } 5884 if (atomic_dec_and_test(&conf->active_aligned_reads)) 5885 wake_up(&conf->wait_for_quiescent); 5886 return handled; 5887 } 5888 5889 static int handle_active_stripes(struct r5conf *conf, int group, 5890 struct r5worker *worker, 5891 struct list_head *temp_inactive_list) 5892 { 5893 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; 5894 int i, batch_size = 0, hash; 5895 bool release_inactive = false; 5896 5897 while (batch_size < MAX_STRIPE_BATCH && 5898 (sh = __get_priority_stripe(conf, group)) != NULL) 5899 batch[batch_size++] = sh; 5900 5901 if (batch_size == 0) { 5902 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 5903 if (!list_empty(temp_inactive_list + i)) 5904 break; 5905 if (i == NR_STRIPE_HASH_LOCKS) { 5906 spin_unlock_irq(&conf->device_lock); 5907 r5l_flush_stripe_to_raid(conf->log); 5908 spin_lock_irq(&conf->device_lock); 5909 return batch_size; 5910 } 5911 release_inactive = true; 5912 } 5913 spin_unlock_irq(&conf->device_lock); 5914 5915 release_inactive_stripe_list(conf, temp_inactive_list, 5916 NR_STRIPE_HASH_LOCKS); 5917 5918 r5l_flush_stripe_to_raid(conf->log); 5919 if (release_inactive) { 5920 spin_lock_irq(&conf->device_lock); 5921 return 0; 5922 } 5923 5924 for (i = 0; i < batch_size; i++) 5925 handle_stripe(batch[i]); 5926 r5l_write_stripe_run(conf->log); 5927 5928 cond_resched(); 5929 5930 spin_lock_irq(&conf->device_lock); 5931 for (i = 0; i < batch_size; i++) { 5932 hash = batch[i]->hash_lock_index; 5933 __release_stripe(conf, batch[i], &temp_inactive_list[hash]); 5934 } 5935 return batch_size; 5936 } 5937 5938 static void raid5_do_work(struct work_struct *work) 5939 { 5940 struct r5worker *worker = container_of(work, struct r5worker, work); 5941 struct r5worker_group *group = worker->group; 5942 struct r5conf *conf = group->conf; 5943 int group_id = group - conf->worker_groups; 5944 int handled; 5945 struct blk_plug plug; 5946 5947 pr_debug("+++ raid5worker active\n"); 5948 5949 blk_start_plug(&plug); 5950 handled = 0; 5951 spin_lock_irq(&conf->device_lock); 5952 while (1) { 5953 int batch_size, released; 5954 5955 released = release_stripe_list(conf, worker->temp_inactive_list); 5956 5957 batch_size = handle_active_stripes(conf, group_id, worker, 5958 worker->temp_inactive_list); 5959 worker->working = false; 5960 if (!batch_size && !released) 5961 break; 5962 handled += batch_size; 5963 } 5964 pr_debug("%d stripes handled\n", handled); 5965 5966 spin_unlock_irq(&conf->device_lock); 5967 blk_finish_plug(&plug); 5968 5969 pr_debug("--- raid5worker inactive\n"); 5970 } 5971 5972 /* 5973 * This is our raid5 kernel thread. 5974 * 5975 * We scan the hash table for stripes which can be handled now. 5976 * During the scan, completed stripes are saved for us by the interrupt 5977 * handler, so that they will not have to wait for our next wakeup. 5978 */ 5979 static void raid5d(struct md_thread *thread) 5980 { 5981 struct mddev *mddev = thread->mddev; 5982 struct r5conf *conf = mddev->private; 5983 int handled; 5984 struct blk_plug plug; 5985 5986 pr_debug("+++ raid5d active\n"); 5987 5988 md_check_recovery(mddev); 5989 5990 if (!bio_list_empty(&conf->return_bi) && 5991 !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 5992 struct bio_list tmp = BIO_EMPTY_LIST; 5993 spin_lock_irq(&conf->device_lock); 5994 if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) { 5995 bio_list_merge(&tmp, &conf->return_bi); 5996 bio_list_init(&conf->return_bi); 5997 } 5998 spin_unlock_irq(&conf->device_lock); 5999 return_io(&tmp); 6000 } 6001 6002 blk_start_plug(&plug); 6003 handled = 0; 6004 spin_lock_irq(&conf->device_lock); 6005 while (1) { 6006 struct bio *bio; 6007 int batch_size, released; 6008 6009 released = release_stripe_list(conf, conf->temp_inactive_list); 6010 if (released) 6011 clear_bit(R5_DID_ALLOC, &conf->cache_state); 6012 6013 if ( 6014 !list_empty(&conf->bitmap_list)) { 6015 /* Now is a good time to flush some bitmap updates */ 6016 conf->seq_flush++; 6017 spin_unlock_irq(&conf->device_lock); 6018 bitmap_unplug(mddev->bitmap); 6019 spin_lock_irq(&conf->device_lock); 6020 conf->seq_write = conf->seq_flush; 6021 activate_bit_delay(conf, conf->temp_inactive_list); 6022 } 6023 raid5_activate_delayed(conf); 6024 6025 while ((bio = remove_bio_from_retry(conf))) { 6026 int ok; 6027 spin_unlock_irq(&conf->device_lock); 6028 ok = retry_aligned_read(conf, bio); 6029 spin_lock_irq(&conf->device_lock); 6030 if (!ok) 6031 break; 6032 handled++; 6033 } 6034 6035 batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, 6036 conf->temp_inactive_list); 6037 if (!batch_size && !released) 6038 break; 6039 handled += batch_size; 6040 6041 if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) { 6042 spin_unlock_irq(&conf->device_lock); 6043 md_check_recovery(mddev); 6044 spin_lock_irq(&conf->device_lock); 6045 } 6046 } 6047 pr_debug("%d stripes handled\n", handled); 6048 6049 spin_unlock_irq(&conf->device_lock); 6050 if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) && 6051 mutex_trylock(&conf->cache_size_mutex)) { 6052 grow_one_stripe(conf, __GFP_NOWARN); 6053 /* Set flag even if allocation failed. This helps 6054 * slow down allocation requests when mem is short 6055 */ 6056 set_bit(R5_DID_ALLOC, &conf->cache_state); 6057 mutex_unlock(&conf->cache_size_mutex); 6058 } 6059 6060 r5l_flush_stripe_to_raid(conf->log); 6061 6062 async_tx_issue_pending_all(); 6063 blk_finish_plug(&plug); 6064 6065 pr_debug("--- raid5d inactive\n"); 6066 } 6067 6068 static ssize_t 6069 raid5_show_stripe_cache_size(struct mddev *mddev, char *page) 6070 { 6071 struct r5conf *conf; 6072 int ret = 0; 6073 spin_lock(&mddev->lock); 6074 conf = mddev->private; 6075 if (conf) 6076 ret = sprintf(page, "%d\n", conf->min_nr_stripes); 6077 spin_unlock(&mddev->lock); 6078 return ret; 6079 } 6080 6081 int 6082 raid5_set_cache_size(struct mddev *mddev, int size) 6083 { 6084 struct r5conf *conf = mddev->private; 6085 int err; 6086 6087 if (size <= 16 || size > 32768) 6088 return -EINVAL; 6089 6090 conf->min_nr_stripes = size; 6091 mutex_lock(&conf->cache_size_mutex); 6092 while (size < conf->max_nr_stripes && 6093 drop_one_stripe(conf)) 6094 ; 6095 mutex_unlock(&conf->cache_size_mutex); 6096 6097 6098 err = md_allow_write(mddev); 6099 if (err) 6100 return err; 6101 6102 mutex_lock(&conf->cache_size_mutex); 6103 while (size > conf->max_nr_stripes) 6104 if (!grow_one_stripe(conf, GFP_KERNEL)) 6105 break; 6106 mutex_unlock(&conf->cache_size_mutex); 6107 6108 return 0; 6109 } 6110 EXPORT_SYMBOL(raid5_set_cache_size); 6111 6112 static ssize_t 6113 raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) 6114 { 6115 struct r5conf *conf; 6116 unsigned long new; 6117 int err; 6118 6119 if (len >= PAGE_SIZE) 6120 return -EINVAL; 6121 if (kstrtoul(page, 10, &new)) 6122 return -EINVAL; 6123 err = mddev_lock(mddev); 6124 if (err) 6125 return err; 6126 conf = mddev->private; 6127 if (!conf) 6128 err = -ENODEV; 6129 else 6130 err = raid5_set_cache_size(mddev, new); 6131 mddev_unlock(mddev); 6132 6133 return err ?: len; 6134 } 6135 6136 static struct md_sysfs_entry 6137 raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, 6138 raid5_show_stripe_cache_size, 6139 raid5_store_stripe_cache_size); 6140 6141 static ssize_t 6142 raid5_show_rmw_level(struct mddev *mddev, char *page) 6143 { 6144 struct r5conf *conf = mddev->private; 6145 if (conf) 6146 return sprintf(page, "%d\n", conf->rmw_level); 6147 else 6148 return 0; 6149 } 6150 6151 static ssize_t 6152 raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) 6153 { 6154 struct r5conf *conf = mddev->private; 6155 unsigned long new; 6156 6157 if (!conf) 6158 return -ENODEV; 6159 6160 if (len >= PAGE_SIZE) 6161 return -EINVAL; 6162 6163 if (kstrtoul(page, 10, &new)) 6164 return -EINVAL; 6165 6166 if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) 6167 return -EINVAL; 6168 6169 if (new != PARITY_DISABLE_RMW && 6170 new != PARITY_ENABLE_RMW && 6171 new != PARITY_PREFER_RMW) 6172 return -EINVAL; 6173 6174 conf->rmw_level = new; 6175 return len; 6176 } 6177 6178 static struct md_sysfs_entry 6179 raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR, 6180 raid5_show_rmw_level, 6181 raid5_store_rmw_level); 6182 6183 6184 static ssize_t 6185 raid5_show_preread_threshold(struct mddev *mddev, char *page) 6186 { 6187 struct r5conf *conf; 6188 int ret = 0; 6189 spin_lock(&mddev->lock); 6190 conf = mddev->private; 6191 if (conf) 6192 ret = sprintf(page, "%d\n", conf->bypass_threshold); 6193 spin_unlock(&mddev->lock); 6194 return ret; 6195 } 6196 6197 static ssize_t 6198 raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) 6199 { 6200 struct r5conf *conf; 6201 unsigned long new; 6202 int err; 6203 6204 if (len >= PAGE_SIZE) 6205 return -EINVAL; 6206 if (kstrtoul(page, 10, &new)) 6207 return -EINVAL; 6208 6209 err = mddev_lock(mddev); 6210 if (err) 6211 return err; 6212 conf = mddev->private; 6213 if (!conf) 6214 err = -ENODEV; 6215 else if (new > conf->min_nr_stripes) 6216 err = -EINVAL; 6217 else 6218 conf->bypass_threshold = new; 6219 mddev_unlock(mddev); 6220 return err ?: len; 6221 } 6222 6223 static struct md_sysfs_entry 6224 raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, 6225 S_IRUGO | S_IWUSR, 6226 raid5_show_preread_threshold, 6227 raid5_store_preread_threshold); 6228 6229 static ssize_t 6230 raid5_show_skip_copy(struct mddev *mddev, char *page) 6231 { 6232 struct r5conf *conf; 6233 int ret = 0; 6234 spin_lock(&mddev->lock); 6235 conf = mddev->private; 6236 if (conf) 6237 ret = sprintf(page, "%d\n", conf->skip_copy); 6238 spin_unlock(&mddev->lock); 6239 return ret; 6240 } 6241 6242 static ssize_t 6243 raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) 6244 { 6245 struct r5conf *conf; 6246 unsigned long new; 6247 int err; 6248 6249 if (len >= PAGE_SIZE) 6250 return -EINVAL; 6251 if (kstrtoul(page, 10, &new)) 6252 return -EINVAL; 6253 new = !!new; 6254 6255 err = mddev_lock(mddev); 6256 if (err) 6257 return err; 6258 conf = mddev->private; 6259 if (!conf) 6260 err = -ENODEV; 6261 else if (new != conf->skip_copy) { 6262 mddev_suspend(mddev); 6263 conf->skip_copy = new; 6264 if (new) 6265 mddev->queue->backing_dev_info.capabilities |= 6266 BDI_CAP_STABLE_WRITES; 6267 else 6268 mddev->queue->backing_dev_info.capabilities &= 6269 ~BDI_CAP_STABLE_WRITES; 6270 mddev_resume(mddev); 6271 } 6272 mddev_unlock(mddev); 6273 return err ?: len; 6274 } 6275 6276 static struct md_sysfs_entry 6277 raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, 6278 raid5_show_skip_copy, 6279 raid5_store_skip_copy); 6280 6281 static ssize_t 6282 stripe_cache_active_show(struct mddev *mddev, char *page) 6283 { 6284 struct r5conf *conf = mddev->private; 6285 if (conf) 6286 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); 6287 else 6288 return 0; 6289 } 6290 6291 static struct md_sysfs_entry 6292 raid5_stripecache_active = __ATTR_RO(stripe_cache_active); 6293 6294 static ssize_t 6295 raid5_show_group_thread_cnt(struct mddev *mddev, char *page) 6296 { 6297 struct r5conf *conf; 6298 int ret = 0; 6299 spin_lock(&mddev->lock); 6300 conf = mddev->private; 6301 if (conf) 6302 ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); 6303 spin_unlock(&mddev->lock); 6304 return ret; 6305 } 6306 6307 static int alloc_thread_groups(struct r5conf *conf, int cnt, 6308 int *group_cnt, 6309 int *worker_cnt_per_group, 6310 struct r5worker_group **worker_groups); 6311 static ssize_t 6312 raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) 6313 { 6314 struct r5conf *conf; 6315 unsigned long new; 6316 int err; 6317 struct r5worker_group *new_groups, *old_groups; 6318 int group_cnt, worker_cnt_per_group; 6319 6320 if (len >= PAGE_SIZE) 6321 return -EINVAL; 6322 if (kstrtoul(page, 10, &new)) 6323 return -EINVAL; 6324 6325 err = mddev_lock(mddev); 6326 if (err) 6327 return err; 6328 conf = mddev->private; 6329 if (!conf) 6330 err = -ENODEV; 6331 else if (new != conf->worker_cnt_per_group) { 6332 mddev_suspend(mddev); 6333 6334 old_groups = conf->worker_groups; 6335 if (old_groups) 6336 flush_workqueue(raid5_wq); 6337 6338 err = alloc_thread_groups(conf, new, 6339 &group_cnt, &worker_cnt_per_group, 6340 &new_groups); 6341 if (!err) { 6342 spin_lock_irq(&conf->device_lock); 6343 conf->group_cnt = group_cnt; 6344 conf->worker_cnt_per_group = worker_cnt_per_group; 6345 conf->worker_groups = new_groups; 6346 spin_unlock_irq(&conf->device_lock); 6347 6348 if (old_groups) 6349 kfree(old_groups[0].workers); 6350 kfree(old_groups); 6351 } 6352 mddev_resume(mddev); 6353 } 6354 mddev_unlock(mddev); 6355 6356 return err ?: len; 6357 } 6358 6359 static struct md_sysfs_entry 6360 raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, 6361 raid5_show_group_thread_cnt, 6362 raid5_store_group_thread_cnt); 6363 6364 static struct attribute *raid5_attrs[] = { 6365 &raid5_stripecache_size.attr, 6366 &raid5_stripecache_active.attr, 6367 &raid5_preread_bypass_threshold.attr, 6368 &raid5_group_thread_cnt.attr, 6369 &raid5_skip_copy.attr, 6370 &raid5_rmw_level.attr, 6371 &r5c_journal_mode.attr, 6372 NULL, 6373 }; 6374 static struct attribute_group raid5_attrs_group = { 6375 .name = NULL, 6376 .attrs = raid5_attrs, 6377 }; 6378 6379 static int alloc_thread_groups(struct r5conf *conf, int cnt, 6380 int *group_cnt, 6381 int *worker_cnt_per_group, 6382 struct r5worker_group **worker_groups) 6383 { 6384 int i, j, k; 6385 ssize_t size; 6386 struct r5worker *workers; 6387 6388 *worker_cnt_per_group = cnt; 6389 if (cnt == 0) { 6390 *group_cnt = 0; 6391 *worker_groups = NULL; 6392 return 0; 6393 } 6394 *group_cnt = num_possible_nodes(); 6395 size = sizeof(struct r5worker) * cnt; 6396 workers = kzalloc(size * *group_cnt, GFP_NOIO); 6397 *worker_groups = kzalloc(sizeof(struct r5worker_group) * 6398 *group_cnt, GFP_NOIO); 6399 if (!*worker_groups || !workers) { 6400 kfree(workers); 6401 kfree(*worker_groups); 6402 return -ENOMEM; 6403 } 6404 6405 for (i = 0; i < *group_cnt; i++) { 6406 struct r5worker_group *group; 6407 6408 group = &(*worker_groups)[i]; 6409 INIT_LIST_HEAD(&group->handle_list); 6410 group->conf = conf; 6411 group->workers = workers + i * cnt; 6412 6413 for (j = 0; j < cnt; j++) { 6414 struct r5worker *worker = group->workers + j; 6415 worker->group = group; 6416 INIT_WORK(&worker->work, raid5_do_work); 6417 6418 for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) 6419 INIT_LIST_HEAD(worker->temp_inactive_list + k); 6420 } 6421 } 6422 6423 return 0; 6424 } 6425 6426 static void free_thread_groups(struct r5conf *conf) 6427 { 6428 if (conf->worker_groups) 6429 kfree(conf->worker_groups[0].workers); 6430 kfree(conf->worker_groups); 6431 conf->worker_groups = NULL; 6432 } 6433 6434 static sector_t 6435 raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) 6436 { 6437 struct r5conf *conf = mddev->private; 6438 6439 if (!sectors) 6440 sectors = mddev->dev_sectors; 6441 if (!raid_disks) 6442 /* size is defined by the smallest of previous and new size */ 6443 raid_disks = min(conf->raid_disks, conf->previous_raid_disks); 6444 6445 sectors &= ~((sector_t)conf->chunk_sectors - 1); 6446 sectors &= ~((sector_t)conf->prev_chunk_sectors - 1); 6447 return sectors * (raid_disks - conf->max_degraded); 6448 } 6449 6450 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 6451 { 6452 safe_put_page(percpu->spare_page); 6453 if (percpu->scribble) 6454 flex_array_free(percpu->scribble); 6455 percpu->spare_page = NULL; 6456 percpu->scribble = NULL; 6457 } 6458 6459 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) 6460 { 6461 if (conf->level == 6 && !percpu->spare_page) 6462 percpu->spare_page = alloc_page(GFP_KERNEL); 6463 if (!percpu->scribble) 6464 percpu->scribble = scribble_alloc(max(conf->raid_disks, 6465 conf->previous_raid_disks), 6466 max(conf->chunk_sectors, 6467 conf->prev_chunk_sectors) 6468 / STRIPE_SECTORS, 6469 GFP_KERNEL); 6470 6471 if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { 6472 free_scratch_buffer(conf, percpu); 6473 return -ENOMEM; 6474 } 6475 6476 return 0; 6477 } 6478 6479 static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node) 6480 { 6481 struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node); 6482 6483 free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); 6484 return 0; 6485 } 6486 6487 static void raid5_free_percpu(struct r5conf *conf) 6488 { 6489 if (!conf->percpu) 6490 return; 6491 6492 cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node); 6493 free_percpu(conf->percpu); 6494 } 6495 6496 static void free_conf(struct r5conf *conf) 6497 { 6498 int i; 6499 6500 if (conf->log) 6501 r5l_exit_log(conf->log); 6502 if (conf->shrinker.nr_deferred) 6503 unregister_shrinker(&conf->shrinker); 6504 6505 free_thread_groups(conf); 6506 shrink_stripes(conf); 6507 raid5_free_percpu(conf); 6508 for (i = 0; i < conf->pool_size; i++) 6509 if (conf->disks[i].extra_page) 6510 put_page(conf->disks[i].extra_page); 6511 kfree(conf->disks); 6512 kfree(conf->stripe_hashtbl); 6513 kfree(conf); 6514 } 6515 6516 static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) 6517 { 6518 struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node); 6519 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); 6520 6521 if (alloc_scratch_buffer(conf, percpu)) { 6522 pr_warn("%s: failed memory allocation for cpu%u\n", 6523 __func__, cpu); 6524 return -ENOMEM; 6525 } 6526 return 0; 6527 } 6528 6529 static int raid5_alloc_percpu(struct r5conf *conf) 6530 { 6531 int err = 0; 6532 6533 conf->percpu = alloc_percpu(struct raid5_percpu); 6534 if (!conf->percpu) 6535 return -ENOMEM; 6536 6537 err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node); 6538 if (!err) { 6539 conf->scribble_disks = max(conf->raid_disks, 6540 conf->previous_raid_disks); 6541 conf->scribble_sectors = max(conf->chunk_sectors, 6542 conf->prev_chunk_sectors); 6543 } 6544 return err; 6545 } 6546 6547 static unsigned long raid5_cache_scan(struct shrinker *shrink, 6548 struct shrink_control *sc) 6549 { 6550 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); 6551 unsigned long ret = SHRINK_STOP; 6552 6553 if (mutex_trylock(&conf->cache_size_mutex)) { 6554 ret= 0; 6555 while (ret < sc->nr_to_scan && 6556 conf->max_nr_stripes > conf->min_nr_stripes) { 6557 if (drop_one_stripe(conf) == 0) { 6558 ret = SHRINK_STOP; 6559 break; 6560 } 6561 ret++; 6562 } 6563 mutex_unlock(&conf->cache_size_mutex); 6564 } 6565 return ret; 6566 } 6567 6568 static unsigned long raid5_cache_count(struct shrinker *shrink, 6569 struct shrink_control *sc) 6570 { 6571 struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); 6572 6573 if (conf->max_nr_stripes < conf->min_nr_stripes) 6574 /* unlikely, but not impossible */ 6575 return 0; 6576 return conf->max_nr_stripes - conf->min_nr_stripes; 6577 } 6578 6579 static struct r5conf *setup_conf(struct mddev *mddev) 6580 { 6581 struct r5conf *conf; 6582 int raid_disk, memory, max_disks; 6583 struct md_rdev *rdev; 6584 struct disk_info *disk; 6585 char pers_name[6]; 6586 int i; 6587 int group_cnt, worker_cnt_per_group; 6588 struct r5worker_group *new_group; 6589 6590 if (mddev->new_level != 5 6591 && mddev->new_level != 4 6592 && mddev->new_level != 6) { 6593 pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n", 6594 mdname(mddev), mddev->new_level); 6595 return ERR_PTR(-EIO); 6596 } 6597 if ((mddev->new_level == 5 6598 && !algorithm_valid_raid5(mddev->new_layout)) || 6599 (mddev->new_level == 6 6600 && !algorithm_valid_raid6(mddev->new_layout))) { 6601 pr_warn("md/raid:%s: layout %d not supported\n", 6602 mdname(mddev), mddev->new_layout); 6603 return ERR_PTR(-EIO); 6604 } 6605 if (mddev->new_level == 6 && mddev->raid_disks < 4) { 6606 pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n", 6607 mdname(mddev), mddev->raid_disks); 6608 return ERR_PTR(-EINVAL); 6609 } 6610 6611 if (!mddev->new_chunk_sectors || 6612 (mddev->new_chunk_sectors << 9) % PAGE_SIZE || 6613 !is_power_of_2(mddev->new_chunk_sectors)) { 6614 pr_warn("md/raid:%s: invalid chunk size %d\n", 6615 mdname(mddev), mddev->new_chunk_sectors << 9); 6616 return ERR_PTR(-EINVAL); 6617 } 6618 6619 conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); 6620 if (conf == NULL) 6621 goto abort; 6622 /* Don't enable multi-threading by default*/ 6623 if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, 6624 &new_group)) { 6625 conf->group_cnt = group_cnt; 6626 conf->worker_cnt_per_group = worker_cnt_per_group; 6627 conf->worker_groups = new_group; 6628 } else 6629 goto abort; 6630 spin_lock_init(&conf->device_lock); 6631 seqcount_init(&conf->gen_lock); 6632 mutex_init(&conf->cache_size_mutex); 6633 init_waitqueue_head(&conf->wait_for_quiescent); 6634 init_waitqueue_head(&conf->wait_for_stripe); 6635 init_waitqueue_head(&conf->wait_for_overlap); 6636 INIT_LIST_HEAD(&conf->handle_list); 6637 INIT_LIST_HEAD(&conf->hold_list); 6638 INIT_LIST_HEAD(&conf->delayed_list); 6639 INIT_LIST_HEAD(&conf->bitmap_list); 6640 bio_list_init(&conf->return_bi); 6641 init_llist_head(&conf->released_stripes); 6642 atomic_set(&conf->active_stripes, 0); 6643 atomic_set(&conf->preread_active_stripes, 0); 6644 atomic_set(&conf->active_aligned_reads, 0); 6645 conf->bypass_threshold = BYPASS_THRESHOLD; 6646 conf->recovery_disabled = mddev->recovery_disabled - 1; 6647 6648 conf->raid_disks = mddev->raid_disks; 6649 if (mddev->reshape_position == MaxSector) 6650 conf->previous_raid_disks = mddev->raid_disks; 6651 else 6652 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; 6653 max_disks = max(conf->raid_disks, conf->previous_raid_disks); 6654 6655 conf->disks = kzalloc(max_disks * sizeof(struct disk_info), 6656 GFP_KERNEL); 6657 6658 if (!conf->disks) 6659 goto abort; 6660 6661 for (i = 0; i < max_disks; i++) { 6662 conf->disks[i].extra_page = alloc_page(GFP_KERNEL); 6663 if (!conf->disks[i].extra_page) 6664 goto abort; 6665 } 6666 6667 conf->mddev = mddev; 6668 6669 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) 6670 goto abort; 6671 6672 /* We init hash_locks[0] separately to that it can be used 6673 * as the reference lock in the spin_lock_nest_lock() call 6674 * in lock_all_device_hash_locks_irq in order to convince 6675 * lockdep that we know what we are doing. 6676 */ 6677 spin_lock_init(conf->hash_locks); 6678 for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) 6679 spin_lock_init(conf->hash_locks + i); 6680 6681 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 6682 INIT_LIST_HEAD(conf->inactive_list + i); 6683 6684 for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) 6685 INIT_LIST_HEAD(conf->temp_inactive_list + i); 6686 6687 atomic_set(&conf->r5c_cached_full_stripes, 0); 6688 INIT_LIST_HEAD(&conf->r5c_full_stripe_list); 6689 atomic_set(&conf->r5c_cached_partial_stripes, 0); 6690 INIT_LIST_HEAD(&conf->r5c_partial_stripe_list); 6691 6692 conf->level = mddev->new_level; 6693 conf->chunk_sectors = mddev->new_chunk_sectors; 6694 if (raid5_alloc_percpu(conf) != 0) 6695 goto abort; 6696 6697 pr_debug("raid456: run(%s) called.\n", mdname(mddev)); 6698 6699 rdev_for_each(rdev, mddev) { 6700 raid_disk = rdev->raid_disk; 6701 if (raid_disk >= max_disks 6702 || raid_disk < 0 || test_bit(Journal, &rdev->flags)) 6703 continue; 6704 disk = conf->disks + raid_disk; 6705 6706 if (test_bit(Replacement, &rdev->flags)) { 6707 if (disk->replacement) 6708 goto abort; 6709 disk->replacement = rdev; 6710 } else { 6711 if (disk->rdev) 6712 goto abort; 6713 disk->rdev = rdev; 6714 } 6715 6716 if (test_bit(In_sync, &rdev->flags)) { 6717 char b[BDEVNAME_SIZE]; 6718 pr_info("md/raid:%s: device %s operational as raid disk %d\n", 6719 mdname(mddev), bdevname(rdev->bdev, b), raid_disk); 6720 } else if (rdev->saved_raid_disk != raid_disk) 6721 /* Cannot rely on bitmap to complete recovery */ 6722 conf->fullsync = 1; 6723 } 6724 6725 conf->level = mddev->new_level; 6726 if (conf->level == 6) { 6727 conf->max_degraded = 2; 6728 if (raid6_call.xor_syndrome) 6729 conf->rmw_level = PARITY_ENABLE_RMW; 6730 else 6731 conf->rmw_level = PARITY_DISABLE_RMW; 6732 } else { 6733 conf->max_degraded = 1; 6734 conf->rmw_level = PARITY_ENABLE_RMW; 6735 } 6736 conf->algorithm = mddev->new_layout; 6737 conf->reshape_progress = mddev->reshape_position; 6738 if (conf->reshape_progress != MaxSector) { 6739 conf->prev_chunk_sectors = mddev->chunk_sectors; 6740 conf->prev_algo = mddev->layout; 6741 } else { 6742 conf->prev_chunk_sectors = conf->chunk_sectors; 6743 conf->prev_algo = conf->algorithm; 6744 } 6745 6746 conf->min_nr_stripes = NR_STRIPES; 6747 if (mddev->reshape_position != MaxSector) { 6748 int stripes = max_t(int, 6749 ((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4, 6750 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4); 6751 conf->min_nr_stripes = max(NR_STRIPES, stripes); 6752 if (conf->min_nr_stripes != NR_STRIPES) 6753 pr_info("md/raid:%s: force stripe size %d for reshape\n", 6754 mdname(mddev), conf->min_nr_stripes); 6755 } 6756 memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + 6757 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; 6758 atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); 6759 if (grow_stripes(conf, conf->min_nr_stripes)) { 6760 pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n", 6761 mdname(mddev), memory); 6762 goto abort; 6763 } else 6764 pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory); 6765 /* 6766 * Losing a stripe head costs more than the time to refill it, 6767 * it reduces the queue depth and so can hurt throughput. 6768 * So set it rather large, scaled by number of devices. 6769 */ 6770 conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; 6771 conf->shrinker.scan_objects = raid5_cache_scan; 6772 conf->shrinker.count_objects = raid5_cache_count; 6773 conf->shrinker.batch = 128; 6774 conf->shrinker.flags = 0; 6775 if (register_shrinker(&conf->shrinker)) { 6776 pr_warn("md/raid:%s: couldn't register shrinker.\n", 6777 mdname(mddev)); 6778 goto abort; 6779 } 6780 6781 sprintf(pers_name, "raid%d", mddev->new_level); 6782 conf->thread = md_register_thread(raid5d, mddev, pers_name); 6783 if (!conf->thread) { 6784 pr_warn("md/raid:%s: couldn't allocate thread.\n", 6785 mdname(mddev)); 6786 goto abort; 6787 } 6788 6789 return conf; 6790 6791 abort: 6792 if (conf) { 6793 free_conf(conf); 6794 return ERR_PTR(-EIO); 6795 } else 6796 return ERR_PTR(-ENOMEM); 6797 } 6798 6799 static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) 6800 { 6801 switch (algo) { 6802 case ALGORITHM_PARITY_0: 6803 if (raid_disk < max_degraded) 6804 return 1; 6805 break; 6806 case ALGORITHM_PARITY_N: 6807 if (raid_disk >= raid_disks - max_degraded) 6808 return 1; 6809 break; 6810 case ALGORITHM_PARITY_0_6: 6811 if (raid_disk == 0 || 6812 raid_disk == raid_disks - 1) 6813 return 1; 6814 break; 6815 case ALGORITHM_LEFT_ASYMMETRIC_6: 6816 case ALGORITHM_RIGHT_ASYMMETRIC_6: 6817 case ALGORITHM_LEFT_SYMMETRIC_6: 6818 case ALGORITHM_RIGHT_SYMMETRIC_6: 6819 if (raid_disk == raid_disks - 1) 6820 return 1; 6821 } 6822 return 0; 6823 } 6824 6825 static int raid5_run(struct mddev *mddev) 6826 { 6827 struct r5conf *conf; 6828 int working_disks = 0; 6829 int dirty_parity_disks = 0; 6830 struct md_rdev *rdev; 6831 struct md_rdev *journal_dev = NULL; 6832 sector_t reshape_offset = 0; 6833 int i; 6834 long long min_offset_diff = 0; 6835 int first = 1; 6836 6837 if (mddev->recovery_cp != MaxSector) 6838 pr_notice("md/raid:%s: not clean -- starting background reconstruction\n", 6839 mdname(mddev)); 6840 6841 rdev_for_each(rdev, mddev) { 6842 long long diff; 6843 6844 if (test_bit(Journal, &rdev->flags)) { 6845 journal_dev = rdev; 6846 continue; 6847 } 6848 if (rdev->raid_disk < 0) 6849 continue; 6850 diff = (rdev->new_data_offset - rdev->data_offset); 6851 if (first) { 6852 min_offset_diff = diff; 6853 first = 0; 6854 } else if (mddev->reshape_backwards && 6855 diff < min_offset_diff) 6856 min_offset_diff = diff; 6857 else if (!mddev->reshape_backwards && 6858 diff > min_offset_diff) 6859 min_offset_diff = diff; 6860 } 6861 6862 if (mddev->reshape_position != MaxSector) { 6863 /* Check that we can continue the reshape. 6864 * Difficulties arise if the stripe we would write to 6865 * next is at or after the stripe we would read from next. 6866 * For a reshape that changes the number of devices, this 6867 * is only possible for a very short time, and mdadm makes 6868 * sure that time appears to have past before assembling 6869 * the array. So we fail if that time hasn't passed. 6870 * For a reshape that keeps the number of devices the same 6871 * mdadm must be monitoring the reshape can keeping the 6872 * critical areas read-only and backed up. It will start 6873 * the array in read-only mode, so we check for that. 6874 */ 6875 sector_t here_new, here_old; 6876 int old_disks; 6877 int max_degraded = (mddev->level == 6 ? 2 : 1); 6878 int chunk_sectors; 6879 int new_data_disks; 6880 6881 if (journal_dev) { 6882 pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n", 6883 mdname(mddev)); 6884 return -EINVAL; 6885 } 6886 6887 if (mddev->new_level != mddev->level) { 6888 pr_warn("md/raid:%s: unsupported reshape required - aborting.\n", 6889 mdname(mddev)); 6890 return -EINVAL; 6891 } 6892 old_disks = mddev->raid_disks - mddev->delta_disks; 6893 /* reshape_position must be on a new-stripe boundary, and one 6894 * further up in new geometry must map after here in old 6895 * geometry. 6896 * If the chunk sizes are different, then as we perform reshape 6897 * in units of the largest of the two, reshape_position needs 6898 * be a multiple of the largest chunk size times new data disks. 6899 */ 6900 here_new = mddev->reshape_position; 6901 chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors); 6902 new_data_disks = mddev->raid_disks - max_degraded; 6903 if (sector_div(here_new, chunk_sectors * new_data_disks)) { 6904 pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n", 6905 mdname(mddev)); 6906 return -EINVAL; 6907 } 6908 reshape_offset = here_new * chunk_sectors; 6909 /* here_new is the stripe we will write to */ 6910 here_old = mddev->reshape_position; 6911 sector_div(here_old, chunk_sectors * (old_disks-max_degraded)); 6912 /* here_old is the first stripe that we might need to read 6913 * from */ 6914 if (mddev->delta_disks == 0) { 6915 /* We cannot be sure it is safe to start an in-place 6916 * reshape. It is only safe if user-space is monitoring 6917 * and taking constant backups. 6918 * mdadm always starts a situation like this in 6919 * readonly mode so it can take control before 6920 * allowing any writes. So just check for that. 6921 */ 6922 if (abs(min_offset_diff) >= mddev->chunk_sectors && 6923 abs(min_offset_diff) >= mddev->new_chunk_sectors) 6924 /* not really in-place - so OK */; 6925 else if (mddev->ro == 0) { 6926 pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n", 6927 mdname(mddev)); 6928 return -EINVAL; 6929 } 6930 } else if (mddev->reshape_backwards 6931 ? (here_new * chunk_sectors + min_offset_diff <= 6932 here_old * chunk_sectors) 6933 : (here_new * chunk_sectors >= 6934 here_old * chunk_sectors + (-min_offset_diff))) { 6935 /* Reading from the same stripe as writing to - bad */ 6936 pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n", 6937 mdname(mddev)); 6938 return -EINVAL; 6939 } 6940 pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev)); 6941 /* OK, we should be able to continue; */ 6942 } else { 6943 BUG_ON(mddev->level != mddev->new_level); 6944 BUG_ON(mddev->layout != mddev->new_layout); 6945 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); 6946 BUG_ON(mddev->delta_disks != 0); 6947 } 6948 6949 if (mddev->private == NULL) 6950 conf = setup_conf(mddev); 6951 else 6952 conf = mddev->private; 6953 6954 if (IS_ERR(conf)) 6955 return PTR_ERR(conf); 6956 6957 if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) { 6958 if (!journal_dev) { 6959 pr_warn("md/raid:%s: journal disk is missing, force array readonly\n", 6960 mdname(mddev)); 6961 mddev->ro = 1; 6962 set_disk_ro(mddev->gendisk, 1); 6963 } else if (mddev->recovery_cp == MaxSector) 6964 set_bit(MD_JOURNAL_CLEAN, &mddev->flags); 6965 } 6966 6967 conf->min_offset_diff = min_offset_diff; 6968 mddev->thread = conf->thread; 6969 conf->thread = NULL; 6970 mddev->private = conf; 6971 6972 for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; 6973 i++) { 6974 rdev = conf->disks[i].rdev; 6975 if (!rdev && conf->disks[i].replacement) { 6976 /* The replacement is all we have yet */ 6977 rdev = conf->disks[i].replacement; 6978 conf->disks[i].replacement = NULL; 6979 clear_bit(Replacement, &rdev->flags); 6980 conf->disks[i].rdev = rdev; 6981 } 6982 if (!rdev) 6983 continue; 6984 if (conf->disks[i].replacement && 6985 conf->reshape_progress != MaxSector) { 6986 /* replacements and reshape simply do not mix. */ 6987 pr_warn("md: cannot handle concurrent replacement and reshape.\n"); 6988 goto abort; 6989 } 6990 if (test_bit(In_sync, &rdev->flags)) { 6991 working_disks++; 6992 continue; 6993 } 6994 /* This disc is not fully in-sync. However if it 6995 * just stored parity (beyond the recovery_offset), 6996 * when we don't need to be concerned about the 6997 * array being dirty. 6998 * When reshape goes 'backwards', we never have 6999 * partially completed devices, so we only need 7000 * to worry about reshape going forwards. 7001 */ 7002 /* Hack because v0.91 doesn't store recovery_offset properly. */ 7003 if (mddev->major_version == 0 && 7004 mddev->minor_version > 90) 7005 rdev->recovery_offset = reshape_offset; 7006 7007 if (rdev->recovery_offset < reshape_offset) { 7008 /* We need to check old and new layout */ 7009 if (!only_parity(rdev->raid_disk, 7010 conf->algorithm, 7011 conf->raid_disks, 7012 conf->max_degraded)) 7013 continue; 7014 } 7015 if (!only_parity(rdev->raid_disk, 7016 conf->prev_algo, 7017 conf->previous_raid_disks, 7018 conf->max_degraded)) 7019 continue; 7020 dirty_parity_disks++; 7021 } 7022 7023 /* 7024 * 0 for a fully functional array, 1 or 2 for a degraded array. 7025 */ 7026 mddev->degraded = calc_degraded(conf); 7027 7028 if (has_failed(conf)) { 7029 pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n", 7030 mdname(mddev), mddev->degraded, conf->raid_disks); 7031 goto abort; 7032 } 7033 7034 /* device size must be a multiple of chunk size */ 7035 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); 7036 mddev->resync_max_sectors = mddev->dev_sectors; 7037 7038 if (mddev->degraded > dirty_parity_disks && 7039 mddev->recovery_cp != MaxSector) { 7040 if (mddev->ok_start_degraded) 7041 pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n", 7042 mdname(mddev)); 7043 else { 7044 pr_crit("md/raid:%s: cannot start dirty degraded array.\n", 7045 mdname(mddev)); 7046 goto abort; 7047 } 7048 } 7049 7050 pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n", 7051 mdname(mddev), conf->level, 7052 mddev->raid_disks-mddev->degraded, mddev->raid_disks, 7053 mddev->new_layout); 7054 7055 print_raid5_conf(conf); 7056 7057 if (conf->reshape_progress != MaxSector) { 7058 conf->reshape_safe = conf->reshape_progress; 7059 atomic_set(&conf->reshape_stripes, 0); 7060 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 7061 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 7062 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 7063 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 7064 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 7065 "reshape"); 7066 } 7067 7068 /* Ok, everything is just fine now */ 7069 if (mddev->to_remove == &raid5_attrs_group) 7070 mddev->to_remove = NULL; 7071 else if (mddev->kobj.sd && 7072 sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) 7073 pr_warn("raid5: failed to create sysfs attributes for %s\n", 7074 mdname(mddev)); 7075 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 7076 7077 if (mddev->queue) { 7078 int chunk_size; 7079 bool discard_supported = true; 7080 /* read-ahead size must cover two whole stripes, which 7081 * is 2 * (datadisks) * chunksize where 'n' is the 7082 * number of raid devices 7083 */ 7084 int data_disks = conf->previous_raid_disks - conf->max_degraded; 7085 int stripe = data_disks * 7086 ((mddev->chunk_sectors << 9) / PAGE_SIZE); 7087 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 7088 mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 7089 7090 chunk_size = mddev->chunk_sectors << 9; 7091 blk_queue_io_min(mddev->queue, chunk_size); 7092 blk_queue_io_opt(mddev->queue, chunk_size * 7093 (conf->raid_disks - conf->max_degraded)); 7094 mddev->queue->limits.raid_partial_stripes_expensive = 1; 7095 /* 7096 * We can only discard a whole stripe. It doesn't make sense to 7097 * discard data disk but write parity disk 7098 */ 7099 stripe = stripe * PAGE_SIZE; 7100 /* Round up to power of 2, as discard handling 7101 * currently assumes that */ 7102 while ((stripe-1) & stripe) 7103 stripe = (stripe | (stripe-1)) + 1; 7104 mddev->queue->limits.discard_alignment = stripe; 7105 mddev->queue->limits.discard_granularity = stripe; 7106 7107 /* 7108 * We use 16-bit counter of active stripes in bi_phys_segments 7109 * (minus one for over-loaded initialization) 7110 */ 7111 blk_queue_max_hw_sectors(mddev->queue, 0xfffe * STRIPE_SECTORS); 7112 blk_queue_max_discard_sectors(mddev->queue, 7113 0xfffe * STRIPE_SECTORS); 7114 7115 /* 7116 * unaligned part of discard request will be ignored, so can't 7117 * guarantee discard_zeroes_data 7118 */ 7119 mddev->queue->limits.discard_zeroes_data = 0; 7120 7121 blk_queue_max_write_same_sectors(mddev->queue, 0); 7122 7123 rdev_for_each(rdev, mddev) { 7124 disk_stack_limits(mddev->gendisk, rdev->bdev, 7125 rdev->data_offset << 9); 7126 disk_stack_limits(mddev->gendisk, rdev->bdev, 7127 rdev->new_data_offset << 9); 7128 /* 7129 * discard_zeroes_data is required, otherwise data 7130 * could be lost. Consider a scenario: discard a stripe 7131 * (the stripe could be inconsistent if 7132 * discard_zeroes_data is 0); write one disk of the 7133 * stripe (the stripe could be inconsistent again 7134 * depending on which disks are used to calculate 7135 * parity); the disk is broken; The stripe data of this 7136 * disk is lost. 7137 */ 7138 if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || 7139 !bdev_get_queue(rdev->bdev)-> 7140 limits.discard_zeroes_data) 7141 discard_supported = false; 7142 /* Unfortunately, discard_zeroes_data is not currently 7143 * a guarantee - just a hint. So we only allow DISCARD 7144 * if the sysadmin has confirmed that only safe devices 7145 * are in use by setting a module parameter. 7146 */ 7147 if (!devices_handle_discard_safely) { 7148 if (discard_supported) { 7149 pr_info("md/raid456: discard support disabled due to uncertainty.\n"); 7150 pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); 7151 } 7152 discard_supported = false; 7153 } 7154 } 7155 7156 if (discard_supported && 7157 mddev->queue->limits.max_discard_sectors >= (stripe >> 9) && 7158 mddev->queue->limits.discard_granularity >= stripe) 7159 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, 7160 mddev->queue); 7161 else 7162 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, 7163 mddev->queue); 7164 7165 blk_queue_max_hw_sectors(mddev->queue, UINT_MAX); 7166 } 7167 7168 if (journal_dev) { 7169 char b[BDEVNAME_SIZE]; 7170 7171 pr_debug("md/raid:%s: using device %s as journal\n", 7172 mdname(mddev), bdevname(journal_dev->bdev, b)); 7173 if (r5l_init_log(conf, journal_dev)) 7174 goto abort; 7175 } 7176 7177 return 0; 7178 abort: 7179 md_unregister_thread(&mddev->thread); 7180 print_raid5_conf(conf); 7181 free_conf(conf); 7182 mddev->private = NULL; 7183 pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev)); 7184 return -EIO; 7185 } 7186 7187 static void raid5_free(struct mddev *mddev, void *priv) 7188 { 7189 struct r5conf *conf = priv; 7190 7191 free_conf(conf); 7192 mddev->to_remove = &raid5_attrs_group; 7193 } 7194 7195 static void raid5_status(struct seq_file *seq, struct mddev *mddev) 7196 { 7197 struct r5conf *conf = mddev->private; 7198 int i; 7199 7200 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, 7201 conf->chunk_sectors / 2, mddev->layout); 7202 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); 7203 rcu_read_lock(); 7204 for (i = 0; i < conf->raid_disks; i++) { 7205 struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); 7206 seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_"); 7207 } 7208 rcu_read_unlock(); 7209 seq_printf (seq, "]"); 7210 } 7211 7212 static void print_raid5_conf (struct r5conf *conf) 7213 { 7214 int i; 7215 struct disk_info *tmp; 7216 7217 pr_debug("RAID conf printout:\n"); 7218 if (!conf) { 7219 pr_debug("(conf==NULL)\n"); 7220 return; 7221 } 7222 pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level, 7223 conf->raid_disks, 7224 conf->raid_disks - conf->mddev->degraded); 7225 7226 for (i = 0; i < conf->raid_disks; i++) { 7227 char b[BDEVNAME_SIZE]; 7228 tmp = conf->disks + i; 7229 if (tmp->rdev) 7230 pr_debug(" disk %d, o:%d, dev:%s\n", 7231 i, !test_bit(Faulty, &tmp->rdev->flags), 7232 bdevname(tmp->rdev->bdev, b)); 7233 } 7234 } 7235 7236 static int raid5_spare_active(struct mddev *mddev) 7237 { 7238 int i; 7239 struct r5conf *conf = mddev->private; 7240 struct disk_info *tmp; 7241 int count = 0; 7242 unsigned long flags; 7243 7244 for (i = 0; i < conf->raid_disks; i++) { 7245 tmp = conf->disks + i; 7246 if (tmp->replacement 7247 && tmp->replacement->recovery_offset == MaxSector 7248 && !test_bit(Faulty, &tmp->replacement->flags) 7249 && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { 7250 /* Replacement has just become active. */ 7251 if (!tmp->rdev 7252 || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) 7253 count++; 7254 if (tmp->rdev) { 7255 /* Replaced device not technically faulty, 7256 * but we need to be sure it gets removed 7257 * and never re-added. 7258 */ 7259 set_bit(Faulty, &tmp->rdev->flags); 7260 sysfs_notify_dirent_safe( 7261 tmp->rdev->sysfs_state); 7262 } 7263 sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); 7264 } else if (tmp->rdev 7265 && tmp->rdev->recovery_offset == MaxSector 7266 && !test_bit(Faulty, &tmp->rdev->flags) 7267 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { 7268 count++; 7269 sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); 7270 } 7271 } 7272 spin_lock_irqsave(&conf->device_lock, flags); 7273 mddev->degraded = calc_degraded(conf); 7274 spin_unlock_irqrestore(&conf->device_lock, flags); 7275 print_raid5_conf(conf); 7276 return count; 7277 } 7278 7279 static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) 7280 { 7281 struct r5conf *conf = mddev->private; 7282 int err = 0; 7283 int number = rdev->raid_disk; 7284 struct md_rdev **rdevp; 7285 struct disk_info *p = conf->disks + number; 7286 7287 print_raid5_conf(conf); 7288 if (test_bit(Journal, &rdev->flags) && conf->log) { 7289 struct r5l_log *log; 7290 /* 7291 * we can't wait pending write here, as this is called in 7292 * raid5d, wait will deadlock. 7293 */ 7294 if (atomic_read(&mddev->writes_pending)) 7295 return -EBUSY; 7296 log = conf->log; 7297 conf->log = NULL; 7298 synchronize_rcu(); 7299 r5l_exit_log(log); 7300 return 0; 7301 } 7302 if (rdev == p->rdev) 7303 rdevp = &p->rdev; 7304 else if (rdev == p->replacement) 7305 rdevp = &p->replacement; 7306 else 7307 return 0; 7308 7309 if (number >= conf->raid_disks && 7310 conf->reshape_progress == MaxSector) 7311 clear_bit(In_sync, &rdev->flags); 7312 7313 if (test_bit(In_sync, &rdev->flags) || 7314 atomic_read(&rdev->nr_pending)) { 7315 err = -EBUSY; 7316 goto abort; 7317 } 7318 /* Only remove non-faulty devices if recovery 7319 * isn't possible. 7320 */ 7321 if (!test_bit(Faulty, &rdev->flags) && 7322 mddev->recovery_disabled != conf->recovery_disabled && 7323 !has_failed(conf) && 7324 (!p->replacement || p->replacement == rdev) && 7325 number < conf->raid_disks) { 7326 err = -EBUSY; 7327 goto abort; 7328 } 7329 *rdevp = NULL; 7330 if (!test_bit(RemoveSynchronized, &rdev->flags)) { 7331 synchronize_rcu(); 7332 if (atomic_read(&rdev->nr_pending)) { 7333 /* lost the race, try later */ 7334 err = -EBUSY; 7335 *rdevp = rdev; 7336 } 7337 } 7338 if (p->replacement) { 7339 /* We must have just cleared 'rdev' */ 7340 p->rdev = p->replacement; 7341 clear_bit(Replacement, &p->replacement->flags); 7342 smp_mb(); /* Make sure other CPUs may see both as identical 7343 * but will never see neither - if they are careful 7344 */ 7345 p->replacement = NULL; 7346 clear_bit(WantReplacement, &rdev->flags); 7347 } else 7348 /* We might have just removed the Replacement as faulty- 7349 * clear the bit just in case 7350 */ 7351 clear_bit(WantReplacement, &rdev->flags); 7352 abort: 7353 7354 print_raid5_conf(conf); 7355 return err; 7356 } 7357 7358 static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) 7359 { 7360 struct r5conf *conf = mddev->private; 7361 int err = -EEXIST; 7362 int disk; 7363 struct disk_info *p; 7364 int first = 0; 7365 int last = conf->raid_disks - 1; 7366 7367 if (test_bit(Journal, &rdev->flags)) { 7368 char b[BDEVNAME_SIZE]; 7369 if (conf->log) 7370 return -EBUSY; 7371 7372 rdev->raid_disk = 0; 7373 /* 7374 * The array is in readonly mode if journal is missing, so no 7375 * write requests running. We should be safe 7376 */ 7377 r5l_init_log(conf, rdev); 7378 pr_debug("md/raid:%s: using device %s as journal\n", 7379 mdname(mddev), bdevname(rdev->bdev, b)); 7380 return 0; 7381 } 7382 if (mddev->recovery_disabled == conf->recovery_disabled) 7383 return -EBUSY; 7384 7385 if (rdev->saved_raid_disk < 0 && has_failed(conf)) 7386 /* no point adding a device */ 7387 return -EINVAL; 7388 7389 if (rdev->raid_disk >= 0) 7390 first = last = rdev->raid_disk; 7391 7392 /* 7393 * find the disk ... but prefer rdev->saved_raid_disk 7394 * if possible. 7395 */ 7396 if (rdev->saved_raid_disk >= 0 && 7397 rdev->saved_raid_disk >= first && 7398 conf->disks[rdev->saved_raid_disk].rdev == NULL) 7399 first = rdev->saved_raid_disk; 7400 7401 for (disk = first; disk <= last; disk++) { 7402 p = conf->disks + disk; 7403 if (p->rdev == NULL) { 7404 clear_bit(In_sync, &rdev->flags); 7405 rdev->raid_disk = disk; 7406 err = 0; 7407 if (rdev->saved_raid_disk != disk) 7408 conf->fullsync = 1; 7409 rcu_assign_pointer(p->rdev, rdev); 7410 goto out; 7411 } 7412 } 7413 for (disk = first; disk <= last; disk++) { 7414 p = conf->disks + disk; 7415 if (test_bit(WantReplacement, &p->rdev->flags) && 7416 p->replacement == NULL) { 7417 clear_bit(In_sync, &rdev->flags); 7418 set_bit(Replacement, &rdev->flags); 7419 rdev->raid_disk = disk; 7420 err = 0; 7421 conf->fullsync = 1; 7422 rcu_assign_pointer(p->replacement, rdev); 7423 break; 7424 } 7425 } 7426 out: 7427 print_raid5_conf(conf); 7428 return err; 7429 } 7430 7431 static int raid5_resize(struct mddev *mddev, sector_t sectors) 7432 { 7433 /* no resync is happening, and there is enough space 7434 * on all devices, so we can resize. 7435 * We need to make sure resync covers any new space. 7436 * If the array is shrinking we should possibly wait until 7437 * any io in the removed space completes, but it hardly seems 7438 * worth it. 7439 */ 7440 sector_t newsize; 7441 struct r5conf *conf = mddev->private; 7442 7443 if (conf->log) 7444 return -EINVAL; 7445 sectors &= ~((sector_t)conf->chunk_sectors - 1); 7446 newsize = raid5_size(mddev, sectors, mddev->raid_disks); 7447 if (mddev->external_size && 7448 mddev->array_sectors > newsize) 7449 return -EINVAL; 7450 if (mddev->bitmap) { 7451 int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); 7452 if (ret) 7453 return ret; 7454 } 7455 md_set_array_sectors(mddev, newsize); 7456 set_capacity(mddev->gendisk, mddev->array_sectors); 7457 revalidate_disk(mddev->gendisk); 7458 if (sectors > mddev->dev_sectors && 7459 mddev->recovery_cp > mddev->dev_sectors) { 7460 mddev->recovery_cp = mddev->dev_sectors; 7461 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); 7462 } 7463 mddev->dev_sectors = sectors; 7464 mddev->resync_max_sectors = sectors; 7465 return 0; 7466 } 7467 7468 static int check_stripe_cache(struct mddev *mddev) 7469 { 7470 /* Can only proceed if there are plenty of stripe_heads. 7471 * We need a minimum of one full stripe,, and for sensible progress 7472 * it is best to have about 4 times that. 7473 * If we require 4 times, then the default 256 4K stripe_heads will 7474 * allow for chunk sizes up to 256K, which is probably OK. 7475 * If the chunk size is greater, user-space should request more 7476 * stripe_heads first. 7477 */ 7478 struct r5conf *conf = mddev->private; 7479 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 7480 > conf->min_nr_stripes || 7481 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 7482 > conf->min_nr_stripes) { 7483 pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n", 7484 mdname(mddev), 7485 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) 7486 / STRIPE_SIZE)*4); 7487 return 0; 7488 } 7489 return 1; 7490 } 7491 7492 static int check_reshape(struct mddev *mddev) 7493 { 7494 struct r5conf *conf = mddev->private; 7495 7496 if (conf->log) 7497 return -EINVAL; 7498 if (mddev->delta_disks == 0 && 7499 mddev->new_layout == mddev->layout && 7500 mddev->new_chunk_sectors == mddev->chunk_sectors) 7501 return 0; /* nothing to do */ 7502 if (has_failed(conf)) 7503 return -EINVAL; 7504 if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { 7505 /* We might be able to shrink, but the devices must 7506 * be made bigger first. 7507 * For raid6, 4 is the minimum size. 7508 * Otherwise 2 is the minimum 7509 */ 7510 int min = 2; 7511 if (mddev->level == 6) 7512 min = 4; 7513 if (mddev->raid_disks + mddev->delta_disks < min) 7514 return -EINVAL; 7515 } 7516 7517 if (!check_stripe_cache(mddev)) 7518 return -ENOSPC; 7519 7520 if (mddev->new_chunk_sectors > mddev->chunk_sectors || 7521 mddev->delta_disks > 0) 7522 if (resize_chunks(conf, 7523 conf->previous_raid_disks 7524 + max(0, mddev->delta_disks), 7525 max(mddev->new_chunk_sectors, 7526 mddev->chunk_sectors) 7527 ) < 0) 7528 return -ENOMEM; 7529 return resize_stripes(conf, (conf->previous_raid_disks 7530 + mddev->delta_disks)); 7531 } 7532 7533 static int raid5_start_reshape(struct mddev *mddev) 7534 { 7535 struct r5conf *conf = mddev->private; 7536 struct md_rdev *rdev; 7537 int spares = 0; 7538 unsigned long flags; 7539 7540 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) 7541 return -EBUSY; 7542 7543 if (!check_stripe_cache(mddev)) 7544 return -ENOSPC; 7545 7546 if (has_failed(conf)) 7547 return -EINVAL; 7548 7549 rdev_for_each(rdev, mddev) { 7550 if (!test_bit(In_sync, &rdev->flags) 7551 && !test_bit(Faulty, &rdev->flags)) 7552 spares++; 7553 } 7554 7555 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) 7556 /* Not enough devices even to make a degraded array 7557 * of that size 7558 */ 7559 return -EINVAL; 7560 7561 /* Refuse to reduce size of the array. Any reductions in 7562 * array size must be through explicit setting of array_size 7563 * attribute. 7564 */ 7565 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) 7566 < mddev->array_sectors) { 7567 pr_warn("md/raid:%s: array size must be reduced before number of disks\n", 7568 mdname(mddev)); 7569 return -EINVAL; 7570 } 7571 7572 atomic_set(&conf->reshape_stripes, 0); 7573 spin_lock_irq(&conf->device_lock); 7574 write_seqcount_begin(&conf->gen_lock); 7575 conf->previous_raid_disks = conf->raid_disks; 7576 conf->raid_disks += mddev->delta_disks; 7577 conf->prev_chunk_sectors = conf->chunk_sectors; 7578 conf->chunk_sectors = mddev->new_chunk_sectors; 7579 conf->prev_algo = conf->algorithm; 7580 conf->algorithm = mddev->new_layout; 7581 conf->generation++; 7582 /* Code that selects data_offset needs to see the generation update 7583 * if reshape_progress has been set - so a memory barrier needed. 7584 */ 7585 smp_mb(); 7586 if (mddev->reshape_backwards) 7587 conf->reshape_progress = raid5_size(mddev, 0, 0); 7588 else 7589 conf->reshape_progress = 0; 7590 conf->reshape_safe = conf->reshape_progress; 7591 write_seqcount_end(&conf->gen_lock); 7592 spin_unlock_irq(&conf->device_lock); 7593 7594 /* Now make sure any requests that proceeded on the assumption 7595 * the reshape wasn't running - like Discard or Read - have 7596 * completed. 7597 */ 7598 mddev_suspend(mddev); 7599 mddev_resume(mddev); 7600 7601 /* Add some new drives, as many as will fit. 7602 * We know there are enough to make the newly sized array work. 7603 * Don't add devices if we are reducing the number of 7604 * devices in the array. This is because it is not possible 7605 * to correctly record the "partially reconstructed" state of 7606 * such devices during the reshape and confusion could result. 7607 */ 7608 if (mddev->delta_disks >= 0) { 7609 rdev_for_each(rdev, mddev) 7610 if (rdev->raid_disk < 0 && 7611 !test_bit(Faulty, &rdev->flags)) { 7612 if (raid5_add_disk(mddev, rdev) == 0) { 7613 if (rdev->raid_disk 7614 >= conf->previous_raid_disks) 7615 set_bit(In_sync, &rdev->flags); 7616 else 7617 rdev->recovery_offset = 0; 7618 7619 if (sysfs_link_rdev(mddev, rdev)) 7620 /* Failure here is OK */; 7621 } 7622 } else if (rdev->raid_disk >= conf->previous_raid_disks 7623 && !test_bit(Faulty, &rdev->flags)) { 7624 /* This is a spare that was manually added */ 7625 set_bit(In_sync, &rdev->flags); 7626 } 7627 7628 /* When a reshape changes the number of devices, 7629 * ->degraded is measured against the larger of the 7630 * pre and post number of devices. 7631 */ 7632 spin_lock_irqsave(&conf->device_lock, flags); 7633 mddev->degraded = calc_degraded(conf); 7634 spin_unlock_irqrestore(&conf->device_lock, flags); 7635 } 7636 mddev->raid_disks = conf->raid_disks; 7637 mddev->reshape_position = conf->reshape_progress; 7638 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 7639 7640 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); 7641 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); 7642 clear_bit(MD_RECOVERY_DONE, &mddev->recovery); 7643 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); 7644 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); 7645 mddev->sync_thread = md_register_thread(md_do_sync, mddev, 7646 "reshape"); 7647 if (!mddev->sync_thread) { 7648 mddev->recovery = 0; 7649 spin_lock_irq(&conf->device_lock); 7650 write_seqcount_begin(&conf->gen_lock); 7651 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; 7652 mddev->new_chunk_sectors = 7653 conf->chunk_sectors = conf->prev_chunk_sectors; 7654 mddev->new_layout = conf->algorithm = conf->prev_algo; 7655 rdev_for_each(rdev, mddev) 7656 rdev->new_data_offset = rdev->data_offset; 7657 smp_wmb(); 7658 conf->generation --; 7659 conf->reshape_progress = MaxSector; 7660 mddev->reshape_position = MaxSector; 7661 write_seqcount_end(&conf->gen_lock); 7662 spin_unlock_irq(&conf->device_lock); 7663 return -EAGAIN; 7664 } 7665 conf->reshape_checkpoint = jiffies; 7666 md_wakeup_thread(mddev->sync_thread); 7667 md_new_event(mddev); 7668 return 0; 7669 } 7670 7671 /* This is called from the reshape thread and should make any 7672 * changes needed in 'conf' 7673 */ 7674 static void end_reshape(struct r5conf *conf) 7675 { 7676 7677 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { 7678 struct md_rdev *rdev; 7679 7680 spin_lock_irq(&conf->device_lock); 7681 conf->previous_raid_disks = conf->raid_disks; 7682 rdev_for_each(rdev, conf->mddev) 7683 rdev->data_offset = rdev->new_data_offset; 7684 smp_wmb(); 7685 conf->reshape_progress = MaxSector; 7686 conf->mddev->reshape_position = MaxSector; 7687 spin_unlock_irq(&conf->device_lock); 7688 wake_up(&conf->wait_for_overlap); 7689 7690 /* read-ahead size must cover two whole stripes, which is 7691 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices 7692 */ 7693 if (conf->mddev->queue) { 7694 int data_disks = conf->raid_disks - conf->max_degraded; 7695 int stripe = data_disks * ((conf->chunk_sectors << 9) 7696 / PAGE_SIZE); 7697 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) 7698 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; 7699 } 7700 } 7701 } 7702 7703 /* This is called from the raid5d thread with mddev_lock held. 7704 * It makes config changes to the device. 7705 */ 7706 static void raid5_finish_reshape(struct mddev *mddev) 7707 { 7708 struct r5conf *conf = mddev->private; 7709 7710 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { 7711 7712 if (mddev->delta_disks > 0) { 7713 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); 7714 if (mddev->queue) { 7715 set_capacity(mddev->gendisk, mddev->array_sectors); 7716 revalidate_disk(mddev->gendisk); 7717 } 7718 } else { 7719 int d; 7720 spin_lock_irq(&conf->device_lock); 7721 mddev->degraded = calc_degraded(conf); 7722 spin_unlock_irq(&conf->device_lock); 7723 for (d = conf->raid_disks ; 7724 d < conf->raid_disks - mddev->delta_disks; 7725 d++) { 7726 struct md_rdev *rdev = conf->disks[d].rdev; 7727 if (rdev) 7728 clear_bit(In_sync, &rdev->flags); 7729 rdev = conf->disks[d].replacement; 7730 if (rdev) 7731 clear_bit(In_sync, &rdev->flags); 7732 } 7733 } 7734 mddev->layout = conf->algorithm; 7735 mddev->chunk_sectors = conf->chunk_sectors; 7736 mddev->reshape_position = MaxSector; 7737 mddev->delta_disks = 0; 7738 mddev->reshape_backwards = 0; 7739 } 7740 } 7741 7742 static void raid5_quiesce(struct mddev *mddev, int state) 7743 { 7744 struct r5conf *conf = mddev->private; 7745 7746 switch(state) { 7747 case 2: /* resume for a suspend */ 7748 wake_up(&conf->wait_for_overlap); 7749 break; 7750 7751 case 1: /* stop all writes */ 7752 lock_all_device_hash_locks_irq(conf); 7753 /* '2' tells resync/reshape to pause so that all 7754 * active stripes can drain 7755 */ 7756 r5c_flush_cache(conf, INT_MAX); 7757 conf->quiesce = 2; 7758 wait_event_cmd(conf->wait_for_quiescent, 7759 atomic_read(&conf->active_stripes) == 0 && 7760 atomic_read(&conf->active_aligned_reads) == 0, 7761 unlock_all_device_hash_locks_irq(conf), 7762 lock_all_device_hash_locks_irq(conf)); 7763 conf->quiesce = 1; 7764 unlock_all_device_hash_locks_irq(conf); 7765 /* allow reshape to continue */ 7766 wake_up(&conf->wait_for_overlap); 7767 break; 7768 7769 case 0: /* re-enable writes */ 7770 lock_all_device_hash_locks_irq(conf); 7771 conf->quiesce = 0; 7772 wake_up(&conf->wait_for_quiescent); 7773 wake_up(&conf->wait_for_overlap); 7774 unlock_all_device_hash_locks_irq(conf); 7775 break; 7776 } 7777 r5l_quiesce(conf->log, state); 7778 } 7779 7780 static void *raid45_takeover_raid0(struct mddev *mddev, int level) 7781 { 7782 struct r0conf *raid0_conf = mddev->private; 7783 sector_t sectors; 7784 7785 /* for raid0 takeover only one zone is supported */ 7786 if (raid0_conf->nr_strip_zones > 1) { 7787 pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n", 7788 mdname(mddev)); 7789 return ERR_PTR(-EINVAL); 7790 } 7791 7792 sectors = raid0_conf->strip_zone[0].zone_end; 7793 sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); 7794 mddev->dev_sectors = sectors; 7795 mddev->new_level = level; 7796 mddev->new_layout = ALGORITHM_PARITY_N; 7797 mddev->new_chunk_sectors = mddev->chunk_sectors; 7798 mddev->raid_disks += 1; 7799 mddev->delta_disks = 1; 7800 /* make sure it will be not marked as dirty */ 7801 mddev->recovery_cp = MaxSector; 7802 7803 return setup_conf(mddev); 7804 } 7805 7806 static void *raid5_takeover_raid1(struct mddev *mddev) 7807 { 7808 int chunksect; 7809 void *ret; 7810 7811 if (mddev->raid_disks != 2 || 7812 mddev->degraded > 1) 7813 return ERR_PTR(-EINVAL); 7814 7815 /* Should check if there are write-behind devices? */ 7816 7817 chunksect = 64*2; /* 64K by default */ 7818 7819 /* The array must be an exact multiple of chunksize */ 7820 while (chunksect && (mddev->array_sectors & (chunksect-1))) 7821 chunksect >>= 1; 7822 7823 if ((chunksect<<9) < STRIPE_SIZE) 7824 /* array size does not allow a suitable chunk size */ 7825 return ERR_PTR(-EINVAL); 7826 7827 mddev->new_level = 5; 7828 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; 7829 mddev->new_chunk_sectors = chunksect; 7830 7831 ret = setup_conf(mddev); 7832 if (!IS_ERR_VALUE(ret)) 7833 clear_bit(MD_FAILFAST_SUPPORTED, &mddev->flags); 7834 return ret; 7835 } 7836 7837 static void *raid5_takeover_raid6(struct mddev *mddev) 7838 { 7839 int new_layout; 7840 7841 switch (mddev->layout) { 7842 case ALGORITHM_LEFT_ASYMMETRIC_6: 7843 new_layout = ALGORITHM_LEFT_ASYMMETRIC; 7844 break; 7845 case ALGORITHM_RIGHT_ASYMMETRIC_6: 7846 new_layout = ALGORITHM_RIGHT_ASYMMETRIC; 7847 break; 7848 case ALGORITHM_LEFT_SYMMETRIC_6: 7849 new_layout = ALGORITHM_LEFT_SYMMETRIC; 7850 break; 7851 case ALGORITHM_RIGHT_SYMMETRIC_6: 7852 new_layout = ALGORITHM_RIGHT_SYMMETRIC; 7853 break; 7854 case ALGORITHM_PARITY_0_6: 7855 new_layout = ALGORITHM_PARITY_0; 7856 break; 7857 case ALGORITHM_PARITY_N: 7858 new_layout = ALGORITHM_PARITY_N; 7859 break; 7860 default: 7861 return ERR_PTR(-EINVAL); 7862 } 7863 mddev->new_level = 5; 7864 mddev->new_layout = new_layout; 7865 mddev->delta_disks = -1; 7866 mddev->raid_disks -= 1; 7867 return setup_conf(mddev); 7868 } 7869 7870 static int raid5_check_reshape(struct mddev *mddev) 7871 { 7872 /* For a 2-drive array, the layout and chunk size can be changed 7873 * immediately as not restriping is needed. 7874 * For larger arrays we record the new value - after validation 7875 * to be used by a reshape pass. 7876 */ 7877 struct r5conf *conf = mddev->private; 7878 int new_chunk = mddev->new_chunk_sectors; 7879 7880 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) 7881 return -EINVAL; 7882 if (new_chunk > 0) { 7883 if (!is_power_of_2(new_chunk)) 7884 return -EINVAL; 7885 if (new_chunk < (PAGE_SIZE>>9)) 7886 return -EINVAL; 7887 if (mddev->array_sectors & (new_chunk-1)) 7888 /* not factor of array size */ 7889 return -EINVAL; 7890 } 7891 7892 /* They look valid */ 7893 7894 if (mddev->raid_disks == 2) { 7895 /* can make the change immediately */ 7896 if (mddev->new_layout >= 0) { 7897 conf->algorithm = mddev->new_layout; 7898 mddev->layout = mddev->new_layout; 7899 } 7900 if (new_chunk > 0) { 7901 conf->chunk_sectors = new_chunk ; 7902 mddev->chunk_sectors = new_chunk; 7903 } 7904 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); 7905 md_wakeup_thread(mddev->thread); 7906 } 7907 return check_reshape(mddev); 7908 } 7909 7910 static int raid6_check_reshape(struct mddev *mddev) 7911 { 7912 int new_chunk = mddev->new_chunk_sectors; 7913 7914 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) 7915 return -EINVAL; 7916 if (new_chunk > 0) { 7917 if (!is_power_of_2(new_chunk)) 7918 return -EINVAL; 7919 if (new_chunk < (PAGE_SIZE >> 9)) 7920 return -EINVAL; 7921 if (mddev->array_sectors & (new_chunk-1)) 7922 /* not factor of array size */ 7923 return -EINVAL; 7924 } 7925 7926 /* They look valid */ 7927 return check_reshape(mddev); 7928 } 7929 7930 static void *raid5_takeover(struct mddev *mddev) 7931 { 7932 /* raid5 can take over: 7933 * raid0 - if there is only one strip zone - make it a raid4 layout 7934 * raid1 - if there are two drives. We need to know the chunk size 7935 * raid4 - trivial - just use a raid4 layout. 7936 * raid6 - Providing it is a *_6 layout 7937 */ 7938 if (mddev->level == 0) 7939 return raid45_takeover_raid0(mddev, 5); 7940 if (mddev->level == 1) 7941 return raid5_takeover_raid1(mddev); 7942 if (mddev->level == 4) { 7943 mddev->new_layout = ALGORITHM_PARITY_N; 7944 mddev->new_level = 5; 7945 return setup_conf(mddev); 7946 } 7947 if (mddev->level == 6) 7948 return raid5_takeover_raid6(mddev); 7949 7950 return ERR_PTR(-EINVAL); 7951 } 7952 7953 static void *raid4_takeover(struct mddev *mddev) 7954 { 7955 /* raid4 can take over: 7956 * raid0 - if there is only one strip zone 7957 * raid5 - if layout is right 7958 */ 7959 if (mddev->level == 0) 7960 return raid45_takeover_raid0(mddev, 4); 7961 if (mddev->level == 5 && 7962 mddev->layout == ALGORITHM_PARITY_N) { 7963 mddev->new_layout = 0; 7964 mddev->new_level = 4; 7965 return setup_conf(mddev); 7966 } 7967 return ERR_PTR(-EINVAL); 7968 } 7969 7970 static struct md_personality raid5_personality; 7971 7972 static void *raid6_takeover(struct mddev *mddev) 7973 { 7974 /* Currently can only take over a raid5. We map the 7975 * personality to an equivalent raid6 personality 7976 * with the Q block at the end. 7977 */ 7978 int new_layout; 7979 7980 if (mddev->pers != &raid5_personality) 7981 return ERR_PTR(-EINVAL); 7982 if (mddev->degraded > 1) 7983 return ERR_PTR(-EINVAL); 7984 if (mddev->raid_disks > 253) 7985 return ERR_PTR(-EINVAL); 7986 if (mddev->raid_disks < 3) 7987 return ERR_PTR(-EINVAL); 7988 7989 switch (mddev->layout) { 7990 case ALGORITHM_LEFT_ASYMMETRIC: 7991 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; 7992 break; 7993 case ALGORITHM_RIGHT_ASYMMETRIC: 7994 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; 7995 break; 7996 case ALGORITHM_LEFT_SYMMETRIC: 7997 new_layout = ALGORITHM_LEFT_SYMMETRIC_6; 7998 break; 7999 case ALGORITHM_RIGHT_SYMMETRIC: 8000 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; 8001 break; 8002 case ALGORITHM_PARITY_0: 8003 new_layout = ALGORITHM_PARITY_0_6; 8004 break; 8005 case ALGORITHM_PARITY_N: 8006 new_layout = ALGORITHM_PARITY_N; 8007 break; 8008 default: 8009 return ERR_PTR(-EINVAL); 8010 } 8011 mddev->new_level = 6; 8012 mddev->new_layout = new_layout; 8013 mddev->delta_disks = 1; 8014 mddev->raid_disks += 1; 8015 return setup_conf(mddev); 8016 } 8017 8018 static struct md_personality raid6_personality = 8019 { 8020 .name = "raid6", 8021 .level = 6, 8022 .owner = THIS_MODULE, 8023 .make_request = raid5_make_request, 8024 .run = raid5_run, 8025 .free = raid5_free, 8026 .status = raid5_status, 8027 .error_handler = raid5_error, 8028 .hot_add_disk = raid5_add_disk, 8029 .hot_remove_disk= raid5_remove_disk, 8030 .spare_active = raid5_spare_active, 8031 .sync_request = raid5_sync_request, 8032 .resize = raid5_resize, 8033 .size = raid5_size, 8034 .check_reshape = raid6_check_reshape, 8035 .start_reshape = raid5_start_reshape, 8036 .finish_reshape = raid5_finish_reshape, 8037 .quiesce = raid5_quiesce, 8038 .takeover = raid6_takeover, 8039 .congested = raid5_congested, 8040 }; 8041 static struct md_personality raid5_personality = 8042 { 8043 .name = "raid5", 8044 .level = 5, 8045 .owner = THIS_MODULE, 8046 .make_request = raid5_make_request, 8047 .run = raid5_run, 8048 .free = raid5_free, 8049 .status = raid5_status, 8050 .error_handler = raid5_error, 8051 .hot_add_disk = raid5_add_disk, 8052 .hot_remove_disk= raid5_remove_disk, 8053 .spare_active = raid5_spare_active, 8054 .sync_request = raid5_sync_request, 8055 .resize = raid5_resize, 8056 .size = raid5_size, 8057 .check_reshape = raid5_check_reshape, 8058 .start_reshape = raid5_start_reshape, 8059 .finish_reshape = raid5_finish_reshape, 8060 .quiesce = raid5_quiesce, 8061 .takeover = raid5_takeover, 8062 .congested = raid5_congested, 8063 }; 8064 8065 static struct md_personality raid4_personality = 8066 { 8067 .name = "raid4", 8068 .level = 4, 8069 .owner = THIS_MODULE, 8070 .make_request = raid5_make_request, 8071 .run = raid5_run, 8072 .free = raid5_free, 8073 .status = raid5_status, 8074 .error_handler = raid5_error, 8075 .hot_add_disk = raid5_add_disk, 8076 .hot_remove_disk= raid5_remove_disk, 8077 .spare_active = raid5_spare_active, 8078 .sync_request = raid5_sync_request, 8079 .resize = raid5_resize, 8080 .size = raid5_size, 8081 .check_reshape = raid5_check_reshape, 8082 .start_reshape = raid5_start_reshape, 8083 .finish_reshape = raid5_finish_reshape, 8084 .quiesce = raid5_quiesce, 8085 .takeover = raid4_takeover, 8086 .congested = raid5_congested, 8087 }; 8088 8089 static int __init raid5_init(void) 8090 { 8091 int ret; 8092 8093 raid5_wq = alloc_workqueue("raid5wq", 8094 WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); 8095 if (!raid5_wq) 8096 return -ENOMEM; 8097 8098 ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE, 8099 "md/raid5:prepare", 8100 raid456_cpu_up_prepare, 8101 raid456_cpu_dead); 8102 if (ret) { 8103 destroy_workqueue(raid5_wq); 8104 return ret; 8105 } 8106 register_md_personality(&raid6_personality); 8107 register_md_personality(&raid5_personality); 8108 register_md_personality(&raid4_personality); 8109 return 0; 8110 } 8111 8112 static void raid5_exit(void) 8113 { 8114 unregister_md_personality(&raid6_personality); 8115 unregister_md_personality(&raid5_personality); 8116 unregister_md_personality(&raid4_personality); 8117 cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE); 8118 destroy_workqueue(raid5_wq); 8119 } 8120 8121 module_init(raid5_init); 8122 module_exit(raid5_exit); 8123 MODULE_LICENSE("GPL"); 8124 MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); 8125 MODULE_ALIAS("md-personality-4"); /* RAID5 */ 8126 MODULE_ALIAS("md-raid5"); 8127 MODULE_ALIAS("md-raid4"); 8128 MODULE_ALIAS("md-level-5"); 8129 MODULE_ALIAS("md-level-4"); 8130 MODULE_ALIAS("md-personality-8"); /* RAID6 */ 8131 MODULE_ALIAS("md-raid6"); 8132 MODULE_ALIAS("md-level-6"); 8133 8134 /* This used to be two separate modules, they were: */ 8135 MODULE_ALIAS("raid5"); 8136 MODULE_ALIAS("raid6"); 8137