Lines Matching full:sh
26 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
118 static inline int raid6_d0(struct stripe_head *sh) in raid6_d0() argument
120 if (sh->ddf_layout) in raid6_d0()
124 if (sh->qd_idx == sh->disks - 1) in raid6_d0()
127 return sh->qd_idx + 1; in raid6_d0()
140 static int raid6_idx_to_slot(int idx, struct stripe_head *sh, in raid6_idx_to_slot() argument
145 if (sh->ddf_layout) in raid6_idx_to_slot()
147 if (idx == sh->pd_idx) in raid6_idx_to_slot()
149 if (idx == sh->qd_idx) in raid6_idx_to_slot()
151 if (!sh->ddf_layout) in raid6_idx_to_slot()
158 static int stripe_operations_active(struct stripe_head *sh) in stripe_operations_active() argument
160 return sh->check_state || sh->reconstruct_state || in stripe_operations_active()
161 test_bit(STRIPE_BIOFILL_RUN, &sh->state) || in stripe_operations_active()
162 test_bit(STRIPE_COMPUTE_RUN, &sh->state); in stripe_operations_active()
165 static bool stripe_is_lowprio(struct stripe_head *sh) in stripe_is_lowprio() argument
167 return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) || in stripe_is_lowprio()
168 test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) && in stripe_is_lowprio()
169 !test_bit(STRIPE_R5C_CACHING, &sh->state); in stripe_is_lowprio()
172 static void raid5_wakeup_stripe_thread(struct stripe_head *sh) in raid5_wakeup_stripe_thread() argument
173 __must_hold(&sh->raid_conf->device_lock) in raid5_wakeup_stripe_thread()
175 struct r5conf *conf = sh->raid_conf; in raid5_wakeup_stripe_thread()
178 int i, cpu = sh->cpu; in raid5_wakeup_stripe_thread()
182 sh->cpu = cpu; in raid5_wakeup_stripe_thread()
185 if (list_empty(&sh->lru)) { in raid5_wakeup_stripe_thread()
188 if (stripe_is_lowprio(sh)) in raid5_wakeup_stripe_thread()
189 list_add_tail(&sh->lru, &group->loprio_list); in raid5_wakeup_stripe_thread()
191 list_add_tail(&sh->lru, &group->handle_list); in raid5_wakeup_stripe_thread()
193 sh->group = group; in raid5_wakeup_stripe_thread()
201 group = conf->worker_groups + cpu_to_group(sh->cpu); in raid5_wakeup_stripe_thread()
205 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); in raid5_wakeup_stripe_thread()
212 queue_work_on(sh->cpu, raid5_wq, in raid5_wakeup_stripe_thread()
219 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, in do_release_stripe() argument
226 BUG_ON(!list_empty(&sh->lru)); in do_release_stripe()
230 for (i = sh->disks; i--; ) in do_release_stripe()
231 if (test_bit(R5_InJournal, &sh->dev[i].flags)) in do_release_stripe()
240 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) || in do_release_stripe()
242 !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) { in do_release_stripe()
243 if (test_bit(STRIPE_R5C_CACHING, &sh->state)) in do_release_stripe()
244 r5c_make_stripe_write_out(sh); in do_release_stripe()
245 set_bit(STRIPE_HANDLE, &sh->state); in do_release_stripe()
248 if (test_bit(STRIPE_HANDLE, &sh->state)) { in do_release_stripe()
249 if (test_bit(STRIPE_DELAYED, &sh->state) && in do_release_stripe()
250 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in do_release_stripe()
251 list_add_tail(&sh->lru, &conf->delayed_list); in do_release_stripe()
252 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && in do_release_stripe()
253 sh->bm_seq - conf->seq_write > 0) in do_release_stripe()
254 list_add_tail(&sh->lru, &conf->bitmap_list); in do_release_stripe()
256 clear_bit(STRIPE_DELAYED, &sh->state); in do_release_stripe()
257 clear_bit(STRIPE_BIT_DELAY, &sh->state); in do_release_stripe()
259 if (stripe_is_lowprio(sh)) in do_release_stripe()
260 list_add_tail(&sh->lru, in do_release_stripe()
263 list_add_tail(&sh->lru, in do_release_stripe()
266 raid5_wakeup_stripe_thread(sh); in do_release_stripe()
272 BUG_ON(stripe_operations_active(sh)); in do_release_stripe()
273 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in do_release_stripe()
278 if (!test_bit(STRIPE_EXPANDING, &sh->state)) { in do_release_stripe()
280 list_add_tail(&sh->lru, temp_inactive_list); in do_release_stripe()
282 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags)); in do_release_stripe()
284 list_add_tail(&sh->lru, temp_inactive_list); in do_release_stripe()
287 if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) in do_release_stripe()
289 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) in do_release_stripe()
291 list_add_tail(&sh->lru, &conf->r5c_full_stripe_list); in do_release_stripe()
299 list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list); in do_release_stripe()
305 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, in __release_stripe() argument
309 if (atomic_dec_and_test(&sh->count)) in __release_stripe()
310 do_release_stripe(conf, sh, temp_inactive_list); in __release_stripe()
366 struct stripe_head *sh, *t; in release_stripe_list() local
372 llist_for_each_entry_safe(sh, t, head, release_list) { in release_stripe_list()
375 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ in release_stripe_list()
377 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); in release_stripe_list()
383 hash = sh->hash_lock_index; in release_stripe_list()
384 __release_stripe(conf, sh, &temp_inactive_list[hash]); in release_stripe_list()
391 void raid5_release_stripe(struct stripe_head *sh) in raid5_release_stripe() argument
393 struct r5conf *conf = sh->raid_conf; in raid5_release_stripe()
401 if (atomic_add_unless(&sh->count, -1, 1)) in raid5_release_stripe()
405 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) in raid5_release_stripe()
407 wakeup = llist_add(&sh->release_list, &conf->released_stripes); in raid5_release_stripe()
413 if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) { in raid5_release_stripe()
415 hash = sh->hash_lock_index; in raid5_release_stripe()
416 do_release_stripe(conf, sh, &list); in raid5_release_stripe()
422 static inline void remove_hash(struct stripe_head *sh) in remove_hash() argument
425 (unsigned long long)sh->sector); in remove_hash()
427 hlist_del_init(&sh->hash); in remove_hash()
430 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) in insert_hash() argument
432 struct hlist_head *hp = stripe_hash(conf, sh->sector); in insert_hash()
435 (unsigned long long)sh->sector); in insert_hash()
437 hlist_add_head(&sh->hash, hp); in insert_hash()
443 struct stripe_head *sh = NULL; in get_free_stripe() local
449 sh = list_entry(first, struct stripe_head, lru); in get_free_stripe()
451 remove_hash(sh); in get_free_stripe()
453 BUG_ON(hash != sh->hash_lock_index); in get_free_stripe()
457 return sh; in get_free_stripe()
461 static void free_stripe_pages(struct stripe_head *sh) in free_stripe_pages() argument
467 if (!sh->pages) in free_stripe_pages()
470 for (i = 0; i < sh->nr_pages; i++) { in free_stripe_pages()
471 p = sh->pages[i]; in free_stripe_pages()
474 sh->pages[i] = NULL; in free_stripe_pages()
478 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp) in alloc_stripe_pages() argument
483 for (i = 0; i < sh->nr_pages; i++) { in alloc_stripe_pages()
485 if (sh->pages[i]) in alloc_stripe_pages()
490 free_stripe_pages(sh); in alloc_stripe_pages()
493 sh->pages[i] = p; in alloc_stripe_pages()
499 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks) in init_stripe_shared_pages() argument
503 if (sh->pages) in init_stripe_shared_pages()
506 /* Each of the sh->dev[i] need one conf->stripe_size */ in init_stripe_shared_pages()
510 sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); in init_stripe_shared_pages()
511 if (!sh->pages) in init_stripe_shared_pages()
513 sh->nr_pages = nr_pages; in init_stripe_shared_pages()
514 sh->stripes_per_page = cnt; in init_stripe_shared_pages()
519 static void shrink_buffers(struct stripe_head *sh) in shrink_buffers() argument
522 int num = sh->raid_conf->pool_size; in shrink_buffers()
528 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); in shrink_buffers()
529 p = sh->dev[i].page; in shrink_buffers()
532 sh->dev[i].page = NULL; in shrink_buffers()
537 sh->dev[i].page = NULL; in shrink_buffers()
538 free_stripe_pages(sh); /* Free pages */ in shrink_buffers()
542 static int grow_buffers(struct stripe_head *sh, gfp_t gfp) in grow_buffers() argument
545 int num = sh->raid_conf->pool_size; in grow_buffers()
554 sh->dev[i].page = page; in grow_buffers()
555 sh->dev[i].orig_page = page; in grow_buffers()
556 sh->dev[i].offset = 0; in grow_buffers()
559 if (alloc_stripe_pages(sh, gfp)) in grow_buffers()
563 sh->dev[i].page = raid5_get_dev_page(sh, i); in grow_buffers()
564 sh->dev[i].orig_page = sh->dev[i].page; in grow_buffers()
565 sh->dev[i].offset = raid5_get_page_offset(sh, i); in grow_buffers()
572 struct stripe_head *sh);
574 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) in init_stripe() argument
576 struct r5conf *conf = sh->raid_conf; in init_stripe()
579 BUG_ON(atomic_read(&sh->count) != 0); in init_stripe()
580 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); in init_stripe()
581 BUG_ON(stripe_operations_active(sh)); in init_stripe()
582 BUG_ON(sh->batch_head); in init_stripe()
588 sh->generation = conf->generation - previous; in init_stripe()
589 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; in init_stripe()
590 sh->sector = sector; in init_stripe()
591 stripe_set_idx(sector, conf, previous, sh); in init_stripe()
592 sh->state = 0; in init_stripe()
594 for (i = sh->disks; i--; ) { in init_stripe()
595 struct r5dev *dev = &sh->dev[i]; in init_stripe()
600 (unsigned long long)sh->sector, i, dev->toread, in init_stripe()
606 dev->sector = raid5_compute_blocknr(sh, i, previous); in init_stripe()
610 sh->overwrite_disks = 0; in init_stripe()
611 insert_hash(conf, sh); in init_stripe()
612 sh->cpu = smp_processor_id(); in init_stripe()
613 set_bit(STRIPE_BATCH_READY, &sh->state); in init_stripe()
619 struct stripe_head *sh; in __find_stripe() local
622 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) in __find_stripe()
623 if (sh->sector == sector && sh->generation == generation) in __find_stripe()
624 return sh; in __find_stripe()
633 struct stripe_head *sh; in find_get_stripe() local
635 sh = __find_stripe(conf, sector, generation); in find_get_stripe()
636 if (!sh) in find_get_stripe()
639 if (atomic_inc_not_zero(&sh->count)) in find_get_stripe()
640 return sh; in find_get_stripe()
644 * be on a list (sh->lru). Must remove the stripe from the list that in find_get_stripe()
649 if (!atomic_read(&sh->count)) { in find_get_stripe()
650 if (!test_bit(STRIPE_HANDLE, &sh->state)) in find_get_stripe()
652 BUG_ON(list_empty(&sh->lru) && in find_get_stripe()
653 !test_bit(STRIPE_EXPANDING, &sh->state)); in find_get_stripe()
657 list_del_init(&sh->lru); in find_get_stripe()
661 if (sh->group) { in find_get_stripe()
662 sh->group->stripes_cnt--; in find_get_stripe()
663 sh->group = NULL; in find_get_stripe()
666 atomic_inc(&sh->count); in find_get_stripe()
669 return sh; in find_get_stripe()
806 struct stripe_head *sh; in raid5_get_active_stripe() local
833 sh = find_get_stripe(conf, sector, conf->generation - previous, in raid5_get_active_stripe()
835 if (sh) in raid5_get_active_stripe()
839 sh = get_free_stripe(conf, hash); in raid5_get_active_stripe()
840 if (sh) { in raid5_get_active_stripe()
842 init_stripe(sh, sector, previous); in raid5_get_active_stripe()
843 atomic_inc(&sh->count); in raid5_get_active_stripe()
870 return sh; in raid5_get_active_stripe()
873 static bool is_full_stripe_write(struct stripe_head *sh) in is_full_stripe_write() argument
875 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); in is_full_stripe_write()
876 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); in is_full_stripe_write()
901 static bool stripe_can_batch(struct stripe_head *sh) in stripe_can_batch() argument
903 struct r5conf *conf = sh->raid_conf; in stripe_can_batch()
907 return test_bit(STRIPE_BATCH_READY, &sh->state) && in stripe_can_batch()
908 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && in stripe_can_batch()
909 is_full_stripe_write(sh); in stripe_can_batch()
914 struct stripe_head *sh, struct stripe_head *last_sh) in stripe_add_to_batch_list() argument
922 tmp_sec = sh->sector; in stripe_add_to_batch_list()
925 head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf); in stripe_add_to_batch_list()
942 lock_two_stripes(head, sh); in stripe_add_to_batch_list()
944 if (!stripe_can_batch(head) || !stripe_can_batch(sh)) in stripe_add_to_batch_list()
947 if (sh->batch_head) in stripe_add_to_batch_list()
951 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) in stripe_add_to_batch_list()
953 if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf || in stripe_add_to_batch_list()
954 bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite)) in stripe_add_to_batch_list()
971 sh->batch_head = head->batch_head; in stripe_add_to_batch_list()
977 list_add(&sh->batch_list, &head->batch_list); in stripe_add_to_batch_list()
981 sh->batch_head = head->batch_head; in stripe_add_to_batch_list()
983 list_add_tail(&sh->batch_list, &head->batch_list); in stripe_add_to_batch_list()
987 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in stripe_add_to_batch_list()
992 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { in stripe_add_to_batch_list()
993 int seq = sh->bm_seq; in stripe_add_to_batch_list()
994 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && in stripe_add_to_batch_list()
995 sh->batch_head->bm_seq > seq) in stripe_add_to_batch_list()
996 seq = sh->batch_head->bm_seq; in stripe_add_to_batch_list()
997 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); in stripe_add_to_batch_list()
998 sh->batch_head->bm_seq = seq; in stripe_add_to_batch_list()
1001 atomic_inc(&sh->count); in stripe_add_to_batch_list()
1003 unlock_two_stripes(head, sh); in stripe_add_to_batch_list()
1011 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) in use_new_offset() argument
1021 if (sh->generation == conf->generation - 1) in use_new_offset()
1138 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) in ops_run_io() argument
1140 struct r5conf *conf = sh->raid_conf; in ops_run_io()
1141 int i, disks = sh->disks; in ops_run_io()
1142 struct stripe_head *head_sh = sh; in ops_run_io()
1149 if (log_stripe(sh, s) == 0) in ops_run_io()
1161 sh = head_sh; in ops_run_io()
1162 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { in ops_run_io()
1164 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) in ops_run_io()
1166 if (test_bit(R5_Discard, &sh->dev[i].flags)) in ops_run_io()
1168 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) in ops_run_io()
1171 &sh->dev[i].flags)) { in ops_run_io()
1176 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) in ops_run_io()
1180 dev = &sh->dev[i]; in ops_run_io()
1222 int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), in ops_run_io()
1256 set_bit(STRIPE_IO_STARTED, &sh->state); in ops_run_io()
1262 bi->bi_private = sh; in ops_run_io()
1265 __func__, (unsigned long long)sh->sector, in ops_run_io()
1267 atomic_inc(&sh->count); in ops_run_io()
1268 if (sh != head_sh) in ops_run_io()
1270 if (use_new_offset(conf, sh)) in ops_run_io()
1271 bi->bi_iter.bi_sector = (sh->sector in ops_run_io()
1274 bi->bi_iter.bi_sector = (sh->sector in ops_run_io()
1279 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) in ops_run_io()
1280 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); in ops_run_io()
1283 test_bit(R5_InJournal, &sh->dev[i].flags)) in ops_run_io()
1289 sh->dev[i].vec.bv_page = sh->dev[i].orig_page; in ops_run_io()
1291 sh->dev[i].vec.bv_page = sh->dev[i].page; in ops_run_io()
1294 bi->bi_io_vec[0].bv_offset = sh->dev[i].offset; in ops_run_io()
1303 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); in ops_run_io()
1308 sh->dev[i].sector); in ops_run_io()
1319 set_bit(STRIPE_IO_STARTED, &sh->state); in ops_run_io()
1324 rbi->bi_private = sh; in ops_run_io()
1328 __func__, (unsigned long long)sh->sector, in ops_run_io()
1330 atomic_inc(&sh->count); in ops_run_io()
1331 if (sh != head_sh) in ops_run_io()
1333 if (use_new_offset(conf, sh)) in ops_run_io()
1334 rbi->bi_iter.bi_sector = (sh->sector in ops_run_io()
1337 rbi->bi_iter.bi_sector = (sh->sector in ops_run_io()
1339 if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) in ops_run_io()
1340 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); in ops_run_io()
1341 sh->dev[i].rvec.bv_page = sh->dev[i].page; in ops_run_io()
1344 rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset; in ops_run_io()
1355 sh->dev[i].sector); in ops_run_io()
1363 set_bit(STRIPE_DEGRADED, &sh->state); in ops_run_io()
1365 bi->bi_opf, i, (unsigned long long)sh->sector); in ops_run_io()
1366 clear_bit(R5_LOCKED, &sh->dev[i].flags); in ops_run_io()
1367 set_bit(STRIPE_HANDLE, &sh->state); in ops_run_io()
1372 sh = list_first_entry(&sh->batch_list, struct stripe_head, in ops_run_io()
1374 if (sh != head_sh) in ops_run_io()
1385 struct stripe_head *sh, int no_skipcopy) in async_copy_data() argument
1393 struct r5conf *conf = sh->raid_conf; in async_copy_data()
1449 struct stripe_head *sh = stripe_head_ref; in ops_complete_biofill() local
1451 struct r5conf *conf = sh->raid_conf; in ops_complete_biofill()
1454 (unsigned long long)sh->sector); in ops_complete_biofill()
1457 for (i = sh->disks; i--; ) { in ops_complete_biofill()
1458 struct r5dev *dev = &sh->dev[i]; in ops_complete_biofill()
1479 clear_bit(STRIPE_BIOFILL_RUN, &sh->state); in ops_complete_biofill()
1481 set_bit(STRIPE_HANDLE, &sh->state); in ops_complete_biofill()
1482 raid5_release_stripe(sh); in ops_complete_biofill()
1485 static void ops_run_biofill(struct stripe_head *sh) in ops_run_biofill() argument
1490 struct r5conf *conf = sh->raid_conf; in ops_run_biofill()
1492 BUG_ON(sh->batch_head); in ops_run_biofill()
1494 (unsigned long long)sh->sector); in ops_run_biofill()
1496 for (i = sh->disks; i--; ) { in ops_run_biofill()
1497 struct r5dev *dev = &sh->dev[i]; in ops_run_biofill()
1500 spin_lock_irq(&sh->stripe_lock); in ops_run_biofill()
1503 spin_unlock_irq(&sh->stripe_lock); in ops_run_biofill()
1508 dev->sector, tx, sh, 0); in ops_run_biofill()
1514 atomic_inc(&sh->count); in ops_run_biofill()
1515 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); in ops_run_biofill()
1519 static void mark_target_uptodate(struct stripe_head *sh, int target) in mark_target_uptodate() argument
1526 tgt = &sh->dev[target]; in mark_target_uptodate()
1534 struct stripe_head *sh = stripe_head_ref; in ops_complete_compute() local
1537 (unsigned long long)sh->sector); in ops_complete_compute()
1540 mark_target_uptodate(sh, sh->ops.target); in ops_complete_compute()
1541 mark_target_uptodate(sh, sh->ops.target2); in ops_complete_compute()
1543 clear_bit(STRIPE_COMPUTE_RUN, &sh->state); in ops_complete_compute()
1544 if (sh->check_state == check_state_compute_run) in ops_complete_compute()
1545 sh->check_state = check_state_compute_result; in ops_complete_compute()
1546 set_bit(STRIPE_HANDLE, &sh->state); in ops_complete_compute()
1547 raid5_release_stripe(sh); in ops_complete_compute()
1557 static addr_conv_t *to_addr_conv(struct stripe_head *sh, in to_addr_conv() argument
1560 return (void *) (to_addr_page(percpu, i) + sh->disks + 2); in to_addr_conv()
1567 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu) in to_addr_offs() argument
1569 return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2); in to_addr_offs()
1573 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) in ops_run_compute5() argument
1575 int disks = sh->disks; in ops_run_compute5()
1577 unsigned int *off_srcs = to_addr_offs(sh, percpu); in ops_run_compute5()
1578 int target = sh->ops.target; in ops_run_compute5()
1579 struct r5dev *tgt = &sh->dev[target]; in ops_run_compute5()
1587 BUG_ON(sh->batch_head); in ops_run_compute5()
1590 __func__, (unsigned long long)sh->sector, target); in ops_run_compute5()
1595 off_srcs[count] = sh->dev[i].offset; in ops_run_compute5()
1596 xor_srcs[count++] = sh->dev[i].page; in ops_run_compute5()
1600 atomic_inc(&sh->count); in ops_run_compute5()
1603 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); in ops_run_compute5()
1606 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_compute5()
1609 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_compute5()
1615 * @srcs - (struct page *) array of size sh->disks
1617 * @sh - stripe_head to parse
1626 struct stripe_head *sh, in set_syndrome_sources() argument
1629 int disks = sh->disks; in set_syndrome_sources()
1630 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); in set_syndrome_sources()
1631 int d0_idx = raid6_d0(sh); in set_syndrome_sources()
1641 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); in set_syndrome_sources()
1642 struct r5dev *dev = &sh->dev[i]; in set_syndrome_sources()
1644 if (i == sh->qd_idx || i == sh->pd_idx || in set_syndrome_sources()
1653 srcs[slot] = sh->dev[i].orig_page; in set_syndrome_sources()
1655 srcs[slot] = sh->dev[i].page; in set_syndrome_sources()
1661 offs[slot] = sh->dev[i].offset; in set_syndrome_sources()
1670 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) in ops_run_compute6_1() argument
1672 int disks = sh->disks; in ops_run_compute6_1()
1674 unsigned int *offs = to_addr_offs(sh, percpu); in ops_run_compute6_1()
1676 int qd_idx = sh->qd_idx; in ops_run_compute6_1()
1685 BUG_ON(sh->batch_head); in ops_run_compute6_1()
1686 if (sh->ops.target < 0) in ops_run_compute6_1()
1687 target = sh->ops.target2; in ops_run_compute6_1()
1688 else if (sh->ops.target2 < 0) in ops_run_compute6_1()
1689 target = sh->ops.target; in ops_run_compute6_1()
1695 __func__, (unsigned long long)sh->sector, target); in ops_run_compute6_1()
1697 tgt = &sh->dev[target]; in ops_run_compute6_1()
1702 atomic_inc(&sh->count); in ops_run_compute6_1()
1705 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL); in ops_run_compute6_1()
1709 ops_complete_compute, sh, in ops_run_compute6_1()
1710 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_1()
1712 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_compute6_1()
1719 offs[count] = sh->dev[i].offset; in ops_run_compute6_1()
1720 blocks[count++] = sh->dev[i].page; in ops_run_compute6_1()
1724 NULL, ops_complete_compute, sh, in ops_run_compute6_1()
1725 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_1()
1727 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_compute6_1()
1734 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) in ops_run_compute6_2() argument
1736 int i, count, disks = sh->disks; in ops_run_compute6_2()
1737 int syndrome_disks = sh->ddf_layout ? disks : disks-2; in ops_run_compute6_2()
1738 int d0_idx = raid6_d0(sh); in ops_run_compute6_2()
1740 int target = sh->ops.target; in ops_run_compute6_2()
1741 int target2 = sh->ops.target2; in ops_run_compute6_2()
1742 struct r5dev *tgt = &sh->dev[target]; in ops_run_compute6_2()
1743 struct r5dev *tgt2 = &sh->dev[target2]; in ops_run_compute6_2()
1746 unsigned int *offs = to_addr_offs(sh, percpu); in ops_run_compute6_2()
1749 BUG_ON(sh->batch_head); in ops_run_compute6_2()
1751 __func__, (unsigned long long)sh->sector, target, target2); in ops_run_compute6_2()
1766 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); in ops_run_compute6_2()
1768 offs[slot] = sh->dev[i].offset; in ops_run_compute6_2()
1769 blocks[slot] = sh->dev[i].page; in ops_run_compute6_2()
1782 __func__, (unsigned long long)sh->sector, faila, failb); in ops_run_compute6_2()
1784 atomic_inc(&sh->count); in ops_run_compute6_2()
1791 ops_complete_compute, sh, in ops_run_compute6_2()
1792 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_2()
1794 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_compute6_2()
1800 int qd_idx = sh->qd_idx; in ops_run_compute6_2()
1812 offs[count] = sh->dev[i].offset; in ops_run_compute6_2()
1813 blocks[count++] = sh->dev[i].page; in ops_run_compute6_2()
1815 dest = sh->dev[data_target].page; in ops_run_compute6_2()
1816 dest_off = sh->dev[data_target].offset; in ops_run_compute6_2()
1820 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_2()
1822 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_compute6_2()
1825 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL); in ops_run_compute6_2()
1827 ops_complete_compute, sh, in ops_run_compute6_2()
1828 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_2()
1830 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_compute6_2()
1835 ops_complete_compute, sh, in ops_run_compute6_2()
1836 to_addr_conv(sh, percpu, 0)); in ops_run_compute6_2()
1840 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_compute6_2()
1846 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_compute6_2()
1855 struct stripe_head *sh = stripe_head_ref; in ops_complete_prexor() local
1858 (unsigned long long)sh->sector); in ops_complete_prexor()
1860 if (r5c_is_writeback(sh->raid_conf->log)) in ops_complete_prexor()
1865 r5c_release_extra_page(sh); in ops_complete_prexor()
1869 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, in ops_run_prexor5() argument
1872 int disks = sh->disks; in ops_run_prexor5()
1874 unsigned int *off_srcs = to_addr_offs(sh, percpu); in ops_run_prexor5()
1875 int count = 0, pd_idx = sh->pd_idx, i; in ops_run_prexor5()
1879 unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset; in ops_run_prexor5()
1880 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; in ops_run_prexor5()
1882 BUG_ON(sh->batch_head); in ops_run_prexor5()
1884 (unsigned long long)sh->sector); in ops_run_prexor5()
1887 struct r5dev *dev = &sh->dev[i]; in ops_run_prexor5()
1903 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); in ops_run_prexor5()
1905 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_prexor5()
1911 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, in ops_run_prexor6() argument
1915 unsigned int *offs = to_addr_offs(sh, percpu); in ops_run_prexor6()
1920 (unsigned long long)sh->sector); in ops_run_prexor6()
1922 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN); in ops_run_prexor6()
1925 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); in ops_run_prexor6()
1927 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_prexor6()
1933 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) in ops_run_biodrain() argument
1935 struct r5conf *conf = sh->raid_conf; in ops_run_biodrain()
1936 int disks = sh->disks; in ops_run_biodrain()
1938 struct stripe_head *head_sh = sh; in ops_run_biodrain()
1941 (unsigned long long)sh->sector); in ops_run_biodrain()
1947 sh = head_sh; in ops_run_biodrain()
1952 dev = &sh->dev[i]; in ops_run_biodrain()
1958 spin_lock_irq(&sh->stripe_lock); in ops_run_biodrain()
1961 sh->overwrite_disks = 0; in ops_run_biodrain()
1964 spin_unlock_irq(&sh->stripe_lock); in ops_run_biodrain()
1978 dev->sector, tx, sh, in ops_run_biodrain()
1991 sh = list_first_entry(&sh->batch_list, in ops_run_biodrain()
1994 if (sh == head_sh) in ops_run_biodrain()
2006 struct stripe_head *sh = stripe_head_ref; in ops_complete_reconstruct() local
2007 int disks = sh->disks; in ops_complete_reconstruct()
2008 int pd_idx = sh->pd_idx; in ops_complete_reconstruct()
2009 int qd_idx = sh->qd_idx; in ops_complete_reconstruct()
2014 (unsigned long long)sh->sector); in ops_complete_reconstruct()
2017 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); in ops_complete_reconstruct()
2018 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); in ops_complete_reconstruct()
2019 discard |= test_bit(R5_Discard, &sh->dev[i].flags); in ops_complete_reconstruct()
2023 struct r5dev *dev = &sh->dev[i]; in ops_complete_reconstruct()
2028 if (test_bit(STRIPE_EXPAND_READY, &sh->state)) in ops_complete_reconstruct()
2038 if (sh->reconstruct_state == reconstruct_state_drain_run) in ops_complete_reconstruct()
2039 sh->reconstruct_state = reconstruct_state_drain_result; in ops_complete_reconstruct()
2040 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) in ops_complete_reconstruct()
2041 sh->reconstruct_state = reconstruct_state_prexor_drain_result; in ops_complete_reconstruct()
2043 BUG_ON(sh->reconstruct_state != reconstruct_state_run); in ops_complete_reconstruct()
2044 sh->reconstruct_state = reconstruct_state_result; in ops_complete_reconstruct()
2047 set_bit(STRIPE_HANDLE, &sh->state); in ops_complete_reconstruct()
2048 raid5_release_stripe(sh); in ops_complete_reconstruct()
2052 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, in ops_run_reconstruct5() argument
2055 int disks = sh->disks; in ops_run_reconstruct5()
2059 int count, pd_idx = sh->pd_idx, i; in ops_run_reconstruct5()
2065 struct stripe_head *head_sh = sh; in ops_run_reconstruct5()
2069 (unsigned long long)sh->sector); in ops_run_reconstruct5()
2071 for (i = 0; i < sh->disks; i++) { in ops_run_reconstruct5()
2074 if (!test_bit(R5_Discard, &sh->dev[i].flags)) in ops_run_reconstruct5()
2077 if (i >= sh->disks) { in ops_run_reconstruct5()
2078 atomic_inc(&sh->count); in ops_run_reconstruct5()
2079 set_bit(R5_Discard, &sh->dev[pd_idx].flags); in ops_run_reconstruct5()
2080 ops_complete_reconstruct(sh); in ops_run_reconstruct5()
2086 off_srcs = to_addr_offs(sh, percpu); in ops_run_reconstruct5()
2092 off_dest = off_srcs[count] = sh->dev[pd_idx].offset; in ops_run_reconstruct5()
2093 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; in ops_run_reconstruct5()
2095 struct r5dev *dev = &sh->dev[i]; in ops_run_reconstruct5()
2103 xor_dest = sh->dev[pd_idx].page; in ops_run_reconstruct5()
2104 off_dest = sh->dev[pd_idx].offset; in ops_run_reconstruct5()
2106 struct r5dev *dev = &sh->dev[i]; in ops_run_reconstruct5()
2120 list_first_entry(&sh->batch_list, in ops_run_reconstruct5()
2128 to_addr_conv(sh, percpu, j)); in ops_run_reconstruct5()
2132 to_addr_conv(sh, percpu, j)); in ops_run_reconstruct5()
2137 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_reconstruct5()
2140 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_reconstruct5()
2143 sh = list_first_entry(&sh->batch_list, struct stripe_head, in ops_run_reconstruct5()
2150 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, in ops_run_reconstruct6() argument
2157 struct stripe_head *head_sh = sh; in ops_run_reconstruct6()
2162 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); in ops_run_reconstruct6()
2164 for (i = 0; i < sh->disks; i++) { in ops_run_reconstruct6()
2165 if (sh->pd_idx == i || sh->qd_idx == i) in ops_run_reconstruct6()
2167 if (!test_bit(R5_Discard, &sh->dev[i].flags)) in ops_run_reconstruct6()
2170 if (i >= sh->disks) { in ops_run_reconstruct6()
2171 atomic_inc(&sh->count); in ops_run_reconstruct6()
2172 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); in ops_run_reconstruct6()
2173 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); in ops_run_reconstruct6()
2174 ops_complete_reconstruct(sh); in ops_run_reconstruct6()
2180 offs = to_addr_offs(sh, percpu); in ops_run_reconstruct6()
2182 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { in ops_run_reconstruct6()
2190 count = set_syndrome_sources(blocks, offs, sh, synflags); in ops_run_reconstruct6()
2192 list_first_entry(&sh->batch_list, in ops_run_reconstruct6()
2198 head_sh, to_addr_conv(sh, percpu, j)); in ops_run_reconstruct6()
2201 to_addr_conv(sh, percpu, j)); in ops_run_reconstruct6()
2203 RAID5_STRIPE_SIZE(sh->raid_conf), &submit); in ops_run_reconstruct6()
2206 sh = list_first_entry(&sh->batch_list, struct stripe_head, in ops_run_reconstruct6()
2214 struct stripe_head *sh = stripe_head_ref; in ops_complete_check() local
2217 (unsigned long long)sh->sector); in ops_complete_check()
2219 sh->check_state = check_state_check_result; in ops_complete_check()
2220 set_bit(STRIPE_HANDLE, &sh->state); in ops_complete_check()
2221 raid5_release_stripe(sh); in ops_complete_check()
2224 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) in ops_run_check_p() argument
2226 int disks = sh->disks; in ops_run_check_p()
2227 int pd_idx = sh->pd_idx; in ops_run_check_p()
2228 int qd_idx = sh->qd_idx; in ops_run_check_p()
2232 unsigned int *off_srcs = to_addr_offs(sh, percpu); in ops_run_check_p()
2239 (unsigned long long)sh->sector); in ops_run_check_p()
2241 BUG_ON(sh->batch_head); in ops_run_check_p()
2243 xor_dest = sh->dev[pd_idx].page; in ops_run_check_p()
2244 off_dest = sh->dev[pd_idx].offset; in ops_run_check_p()
2250 off_srcs[count] = sh->dev[i].offset; in ops_run_check_p()
2251 xor_srcs[count++] = sh->dev[i].page; in ops_run_check_p()
2255 to_addr_conv(sh, percpu, 0)); in ops_run_check_p()
2257 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_check_p()
2258 &sh->ops.zero_sum_result, &submit); in ops_run_check_p()
2260 atomic_inc(&sh->count); in ops_run_check_p()
2261 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); in ops_run_check_p()
2265 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) in ops_run_check_pq() argument
2268 unsigned int *offs = to_addr_offs(sh, percpu); in ops_run_check_pq()
2273 (unsigned long long)sh->sector, checkp); in ops_run_check_pq()
2275 BUG_ON(sh->batch_head); in ops_run_check_pq()
2276 count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL); in ops_run_check_pq()
2280 atomic_inc(&sh->count); in ops_run_check_pq()
2282 sh, to_addr_conv(sh, percpu, 0)); in ops_run_check_pq()
2284 RAID5_STRIPE_SIZE(sh->raid_conf), in ops_run_check_pq()
2285 &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit); in ops_run_check_pq()
2288 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) in raid_run_ops() argument
2290 int overlap_clear = 0, i, disks = sh->disks; in raid_run_ops()
2292 struct r5conf *conf = sh->raid_conf; in raid_run_ops()
2299 ops_run_biofill(sh); in raid_run_ops()
2305 tx = ops_run_compute5(sh, percpu); in raid_run_ops()
2307 if (sh->ops.target2 < 0 || sh->ops.target < 0) in raid_run_ops()
2308 tx = ops_run_compute6_1(sh, percpu); in raid_run_ops()
2310 tx = ops_run_compute6_2(sh, percpu); in raid_run_ops()
2319 tx = ops_run_prexor5(sh, percpu, tx); in raid_run_ops()
2321 tx = ops_run_prexor6(sh, percpu, tx); in raid_run_ops()
2325 tx = ops_run_partial_parity(sh, percpu, tx); in raid_run_ops()
2328 tx = ops_run_biodrain(sh, tx); in raid_run_ops()
2334 ops_run_reconstruct5(sh, percpu, tx); in raid_run_ops()
2336 ops_run_reconstruct6(sh, percpu, tx); in raid_run_ops()
2340 if (sh->check_state == check_state_run) in raid_run_ops()
2341 ops_run_check_p(sh, percpu); in raid_run_ops()
2342 else if (sh->check_state == check_state_run_q) in raid_run_ops()
2343 ops_run_check_pq(sh, percpu, 0); in raid_run_ops()
2344 else if (sh->check_state == check_state_run_pq) in raid_run_ops()
2345 ops_run_check_pq(sh, percpu, 1); in raid_run_ops()
2350 if (overlap_clear && !sh->batch_head) { in raid_run_ops()
2352 struct r5dev *dev = &sh->dev[i]; in raid_run_ops()
2354 wake_up(&sh->raid_conf->wait_for_overlap); in raid_run_ops()
2360 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh) in free_stripe() argument
2363 kfree(sh->pages); in free_stripe()
2365 if (sh->ppl_page) in free_stripe()
2366 __free_page(sh->ppl_page); in free_stripe()
2367 kmem_cache_free(sc, sh); in free_stripe()
2373 struct stripe_head *sh; in alloc_stripe() local
2375 sh = kmem_cache_zalloc(sc, gfp); in alloc_stripe()
2376 if (sh) { in alloc_stripe()
2377 spin_lock_init(&sh->stripe_lock); in alloc_stripe()
2378 spin_lock_init(&sh->batch_lock); in alloc_stripe()
2379 INIT_LIST_HEAD(&sh->batch_list); in alloc_stripe()
2380 INIT_LIST_HEAD(&sh->lru); in alloc_stripe()
2381 INIT_LIST_HEAD(&sh->r5c); in alloc_stripe()
2382 INIT_LIST_HEAD(&sh->log_list); in alloc_stripe()
2383 atomic_set(&sh->count, 1); in alloc_stripe()
2384 sh->raid_conf = conf; in alloc_stripe()
2385 sh->log_start = MaxSector; in alloc_stripe()
2388 sh->ppl_page = alloc_page(gfp); in alloc_stripe()
2389 if (!sh->ppl_page) { in alloc_stripe()
2390 free_stripe(sc, sh); in alloc_stripe()
2395 if (init_stripe_shared_pages(sh, conf, disks)) { in alloc_stripe()
2396 free_stripe(sc, sh); in alloc_stripe()
2401 return sh; in alloc_stripe()
2405 struct stripe_head *sh; in grow_one_stripe() local
2407 sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf); in grow_one_stripe()
2408 if (!sh) in grow_one_stripe()
2411 if (grow_buffers(sh, gfp)) { in grow_one_stripe()
2412 shrink_buffers(sh); in grow_one_stripe()
2413 free_stripe(conf->slab_cache, sh); in grow_one_stripe()
2416 sh->hash_lock_index = in grow_one_stripe()
2421 raid5_release_stripe(sh); in grow_one_stripe()
2707 struct stripe_head *sh; in drop_one_stripe() local
2711 sh = get_free_stripe(conf, hash); in drop_one_stripe()
2713 if (!sh) in drop_one_stripe()
2715 BUG_ON(atomic_read(&sh->count)); in drop_one_stripe()
2716 shrink_buffers(sh); in drop_one_stripe()
2717 free_stripe(conf->slab_cache, sh); in drop_one_stripe()
2757 struct stripe_head *sh = bi->bi_private; in raid5_end_read_request() local
2758 struct r5conf *conf = sh->raid_conf; in raid5_end_read_request()
2759 int disks = sh->disks, i; in raid5_end_read_request()
2764 if (bi == &sh->dev[i].req) in raid5_end_read_request()
2768 (unsigned long long)sh->sector, i, atomic_read(&sh->count), in raid5_end_read_request()
2774 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) in raid5_end_read_request()
2784 if (use_new_offset(conf, sh)) in raid5_end_read_request()
2785 s = sh->sector + rdev->new_data_offset; in raid5_end_read_request()
2787 s = sh->sector + rdev->data_offset; in raid5_end_read_request()
2789 set_bit(R5_UPTODATE, &sh->dev[i].flags); in raid5_end_read_request()
2790 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { in raid5_end_read_request()
2801 clear_bit(R5_ReadError, &sh->dev[i].flags); in raid5_end_read_request()
2802 clear_bit(R5_ReWrite, &sh->dev[i].flags); in raid5_end_read_request()
2803 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) in raid5_end_read_request()
2804 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); in raid5_end_read_request()
2806 if (test_bit(R5_InJournal, &sh->dev[i].flags)) in raid5_end_read_request()
2811 set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags); in raid5_end_read_request()
2819 clear_bit(R5_UPTODATE, &sh->dev[i].flags); in raid5_end_read_request()
2822 if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) in raid5_end_read_request()
2835 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { in raid5_end_read_request()
2856 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) in raid5_end_read_request()
2859 if (sh->qd_idx >= 0 && sh->pd_idx == i) in raid5_end_read_request()
2860 set_bit(R5_ReadError, &sh->dev[i].flags); in raid5_end_read_request()
2861 else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { in raid5_end_read_request()
2862 set_bit(R5_ReadError, &sh->dev[i].flags); in raid5_end_read_request()
2863 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); in raid5_end_read_request()
2865 set_bit(R5_ReadNoMerge, &sh->dev[i].flags); in raid5_end_read_request()
2867 clear_bit(R5_ReadError, &sh->dev[i].flags); in raid5_end_read_request()
2868 clear_bit(R5_ReWrite, &sh->dev[i].flags); in raid5_end_read_request()
2872 rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0))) in raid5_end_read_request()
2878 clear_bit(R5_LOCKED, &sh->dev[i].flags); in raid5_end_read_request()
2879 set_bit(STRIPE_HANDLE, &sh->state); in raid5_end_read_request()
2880 raid5_release_stripe(sh); in raid5_end_read_request()
2885 struct stripe_head *sh = bi->bi_private; in raid5_end_write_request() local
2886 struct r5conf *conf = sh->raid_conf; in raid5_end_write_request()
2887 int disks = sh->disks, i; in raid5_end_write_request()
2894 if (bi == &sh->dev[i].req) { in raid5_end_write_request()
2898 if (bi == &sh->dev[i].rreq) { in raid5_end_write_request()
2912 (unsigned long long)sh->sector, i, atomic_read(&sh->count), in raid5_end_write_request()
2922 else if (is_badblock(rdev, sh->sector, in raid5_end_write_request()
2925 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); in raid5_end_write_request()
2928 set_bit(STRIPE_DEGRADED, &sh->state); in raid5_end_write_request()
2930 set_bit(R5_WriteError, &sh->dev[i].flags); in raid5_end_write_request()
2934 } else if (is_badblock(rdev, sh->sector, in raid5_end_write_request()
2937 set_bit(R5_MadeGood, &sh->dev[i].flags); in raid5_end_write_request()
2938 if (test_bit(R5_ReadError, &sh->dev[i].flags)) in raid5_end_write_request()
2943 set_bit(R5_ReWrite, &sh->dev[i].flags); in raid5_end_write_request()
2948 if (sh->batch_head && bi->bi_status && !replacement) in raid5_end_write_request()
2949 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); in raid5_end_write_request()
2952 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) in raid5_end_write_request()
2953 clear_bit(R5_LOCKED, &sh->dev[i].flags); in raid5_end_write_request()
2954 set_bit(STRIPE_HANDLE, &sh->state); in raid5_end_write_request()
2956 if (sh->batch_head && sh != sh->batch_head) in raid5_end_write_request()
2957 raid5_release_stripe(sh->batch_head); in raid5_end_write_request()
2958 raid5_release_stripe(sh); in raid5_end_write_request()
3001 struct stripe_head *sh) in raid5_compute_sector() argument
3189 if (sh) { in raid5_compute_sector()
3190 sh->pd_idx = pd_idx; in raid5_compute_sector()
3191 sh->qd_idx = qd_idx; in raid5_compute_sector()
3192 sh->ddf_layout = ddf_layout; in raid5_compute_sector()
3201 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous) in raid5_compute_blocknr() argument
3203 struct r5conf *conf = sh->raid_conf; in raid5_compute_blocknr()
3204 int raid_disks = sh->disks; in raid5_compute_blocknr()
3206 sector_t new_sector = sh->sector, check; in raid5_compute_blocknr()
3221 if (i == sh->pd_idx) in raid5_compute_blocknr()
3229 if (i > sh->pd_idx) in raid5_compute_blocknr()
3234 if (i < sh->pd_idx) in raid5_compute_blocknr()
3236 i -= (sh->pd_idx + 1); in raid5_compute_blocknr()
3248 if (i == sh->qd_idx) in raid5_compute_blocknr()
3255 if (sh->pd_idx == raid_disks-1) in raid5_compute_blocknr()
3257 else if (i > sh->pd_idx) in raid5_compute_blocknr()
3262 if (sh->pd_idx == raid_disks-1) in raid5_compute_blocknr()
3266 if (i < sh->pd_idx) in raid5_compute_blocknr()
3268 i -= (sh->pd_idx + 2); in raid5_compute_blocknr()
3278 if (sh->pd_idx == 0) in raid5_compute_blocknr()
3282 if (i < sh->pd_idx) in raid5_compute_blocknr()
3284 i -= (sh->pd_idx + 1); in raid5_compute_blocknr()
3289 if (i > sh->pd_idx) in raid5_compute_blocknr()
3294 if (i < sh->pd_idx) in raid5_compute_blocknr()
3296 i -= (sh->pd_idx + 1); in raid5_compute_blocknr()
3312 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx in raid5_compute_blocknr()
3313 || sh2.qd_idx != sh->qd_idx) { in raid5_compute_blocknr()
3378 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, in schedule_reconstruction() argument
3381 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; in schedule_reconstruction()
3382 struct r5conf *conf = sh->raid_conf; in schedule_reconstruction()
3392 r5c_release_extra_page(sh); in schedule_reconstruction()
3395 struct r5dev *dev = &sh->dev[i]; in schedule_reconstruction()
3416 sh->reconstruct_state = reconstruct_state_drain_run; in schedule_reconstruction()
3419 sh->reconstruct_state = reconstruct_state_run; in schedule_reconstruction()
3424 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) in schedule_reconstruction()
3427 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || in schedule_reconstruction()
3428 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); in schedule_reconstruction()
3430 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || in schedule_reconstruction()
3431 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); in schedule_reconstruction()
3434 struct r5dev *dev = &sh->dev[i]; in schedule_reconstruction()
3453 sh->reconstruct_state = reconstruct_state_prexor_drain_run; in schedule_reconstruction()
3462 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); in schedule_reconstruction()
3463 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); in schedule_reconstruction()
3467 int qd_idx = sh->qd_idx; in schedule_reconstruction()
3468 struct r5dev *dev = &sh->dev[qd_idx]; in schedule_reconstruction()
3475 if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page && in schedule_reconstruction()
3477 !test_bit(STRIPE_FULL_WRITE, &sh->state) && in schedule_reconstruction()
3478 test_bit(R5_Insync, &sh->dev[pd_idx].flags)) in schedule_reconstruction()
3482 __func__, (unsigned long long)sh->sector, in schedule_reconstruction()
3486 static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi, in stripe_bio_overlaps() argument
3489 struct r5conf *conf = sh->raid_conf; in stripe_bio_overlaps()
3493 bi->bi_iter.bi_sector, sh->sector); in stripe_bio_overlaps()
3496 if (sh->batch_head) in stripe_bio_overlaps()
3500 bip = &sh->dev[dd_idx].towrite; in stripe_bio_overlaps()
3502 bip = &sh->dev[dd_idx].toread; in stripe_bio_overlaps()
3527 for (i = 0; i < sh->disks; i++) { in stripe_bio_overlaps()
3528 if (i != sh->pd_idx && in stripe_bio_overlaps()
3529 (i == dd_idx || sh->dev[i].towrite)) { in stripe_bio_overlaps()
3530 sector = sh->dev[i].sector; in stripe_bio_overlaps()
3546 static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, in __add_stripe_bio() argument
3549 struct r5conf *conf = sh->raid_conf; in __add_stripe_bio()
3554 bip = &sh->dev[dd_idx].towrite; in __add_stripe_bio()
3558 bip = &sh->dev[dd_idx].toread; in __add_stripe_bio()
3565 clear_bit(STRIPE_BATCH_READY, &sh->state); in __add_stripe_bio()
3576 sector_t sector = sh->dev[dd_idx].sector; in __add_stripe_bio()
3577 for (bi=sh->dev[dd_idx].towrite; in __add_stripe_bio()
3578 sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) && in __add_stripe_bio()
3580 bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) { in __add_stripe_bio()
3584 if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf)) in __add_stripe_bio()
3585 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) in __add_stripe_bio()
3586 sh->overwrite_disks++; in __add_stripe_bio()
3590 (*bip)->bi_iter.bi_sector, sh->sector, dd_idx, in __add_stripe_bio()
3591 sh->dev[dd_idx].sector); in __add_stripe_bio()
3606 set_bit(STRIPE_BITMAP_PENDING, &sh->state); in __add_stripe_bio()
3607 spin_unlock_irq(&sh->stripe_lock); in __add_stripe_bio()
3608 md_bitmap_startwrite(conf->mddev->bitmap, sh->sector, in __add_stripe_bio()
3610 spin_lock_irq(&sh->stripe_lock); in __add_stripe_bio()
3611 clear_bit(STRIPE_BITMAP_PENDING, &sh->state); in __add_stripe_bio()
3612 if (!sh->batch_head) { in __add_stripe_bio()
3613 sh->bm_seq = conf->seq_flush+1; in __add_stripe_bio()
3614 set_bit(STRIPE_BIT_DELAY, &sh->state); in __add_stripe_bio()
3624 static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi, in add_stripe_bio() argument
3627 spin_lock_irq(&sh->stripe_lock); in add_stripe_bio()
3629 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) { in add_stripe_bio()
3630 set_bit(R5_Overlap, &sh->dev[dd_idx].flags); in add_stripe_bio()
3631 spin_unlock_irq(&sh->stripe_lock); in add_stripe_bio()
3635 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous); in add_stripe_bio()
3636 spin_unlock_irq(&sh->stripe_lock); in add_stripe_bio()
3643 struct stripe_head *sh) in stripe_set_idx() argument
3655 &dd_idx, sh); in stripe_set_idx()
3659 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, in handle_failed_stripe() argument
3663 BUG_ON(sh->batch_head); in handle_failed_stripe()
3668 if (test_bit(R5_ReadError, &sh->dev[i].flags)) { in handle_failed_stripe()
3681 sh->sector, in handle_failed_stripe()
3687 spin_lock_irq(&sh->stripe_lock); in handle_failed_stripe()
3689 bi = sh->dev[i].towrite; in handle_failed_stripe()
3690 sh->dev[i].towrite = NULL; in handle_failed_stripe()
3691 sh->overwrite_disks = 0; in handle_failed_stripe()
3692 spin_unlock_irq(&sh->stripe_lock); in handle_failed_stripe()
3696 log_stripe_write_finished(sh); in handle_failed_stripe()
3698 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) in handle_failed_stripe()
3702 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) { in handle_failed_stripe()
3703 struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector); in handle_failed_stripe()
3710 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_failed_stripe()
3714 bi = sh->dev[i].written; in handle_failed_stripe()
3715 sh->dev[i].written = NULL; in handle_failed_stripe()
3716 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { in handle_failed_stripe()
3717 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); in handle_failed_stripe()
3718 sh->dev[i].page = sh->dev[i].orig_page; in handle_failed_stripe()
3723 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) { in handle_failed_stripe()
3724 struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector); in handle_failed_stripe()
3734 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && in handle_failed_stripe()
3736 (!test_bit(R5_Insync, &sh->dev[i].flags) || in handle_failed_stripe()
3737 test_bit(R5_ReadError, &sh->dev[i].flags))) { in handle_failed_stripe()
3738 spin_lock_irq(&sh->stripe_lock); in handle_failed_stripe()
3739 bi = sh->dev[i].toread; in handle_failed_stripe()
3740 sh->dev[i].toread = NULL; in handle_failed_stripe()
3741 spin_unlock_irq(&sh->stripe_lock); in handle_failed_stripe()
3742 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) in handle_failed_stripe()
3747 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) { in handle_failed_stripe()
3749 r5_next_bio(conf, bi, sh->dev[i].sector); in handle_failed_stripe()
3756 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_failed_stripe()
3761 clear_bit(R5_LOCKED, &sh->dev[i].flags); in handle_failed_stripe()
3766 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) in handle_failed_stripe()
3772 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, in handle_failed_sync() argument
3778 BUG_ON(sh->batch_head); in handle_failed_sync()
3779 clear_bit(STRIPE_SYNCING, &sh->state); in handle_failed_sync()
3780 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) in handle_failed_sync()
3801 && !rdev_set_badblocks(rdev, sh->sector, in handle_failed_sync()
3808 && !rdev_set_badblocks(rdev, sh->sector, in handle_failed_sync()
3820 static int want_replace(struct stripe_head *sh, int disk_idx) in want_replace() argument
3826 rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement); in want_replace()
3830 && (rdev->recovery_offset <= sh->sector in want_replace()
3831 || rdev->mddev->recovery_cp <= sh->sector)) in want_replace()
3837 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, in need_this_block() argument
3840 struct r5dev *dev = &sh->dev[disk_idx]; in need_this_block()
3841 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], in need_this_block()
3842 &sh->dev[s->failed_num[1]] }; in need_this_block()
3844 bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW); in need_this_block()
3860 (s->replacing && want_replace(sh, disk_idx))) in need_this_block()
3885 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in need_this_block()
3906 s->failed_num[i] == sh->pd_idx || in need_this_block()
3907 s->failed_num[i] == sh->qd_idx) && in need_this_block()
3924 sh->sector < sh->raid_conf->mddev->recovery_cp) in need_this_block()
3928 if (s->failed_num[i] != sh->pd_idx && in need_this_block()
3929 s->failed_num[i] != sh->qd_idx && in need_this_block()
3944 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, in fetch_block() argument
3947 struct r5dev *dev = &sh->dev[disk_idx]; in fetch_block()
3950 if (need_this_block(sh, s, disk_idx, disks)) { in fetch_block()
3956 BUG_ON(sh->batch_head); in fetch_block()
3968 ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) || in fetch_block()
3975 (unsigned long long)sh->sector, disk_idx); in fetch_block()
3976 set_bit(STRIPE_COMPUTE_RUN, &sh->state); in fetch_block()
3979 sh->ops.target = disk_idx; in fetch_block()
3980 sh->ops.target2 = -1; /* no 2nd target */ in fetch_block()
3999 &sh->dev[other].flags)) in fetch_block()
4004 (unsigned long long)sh->sector, in fetch_block()
4006 set_bit(STRIPE_COMPUTE_RUN, &sh->state); in fetch_block()
4008 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); in fetch_block()
4009 set_bit(R5_Wantcompute, &sh->dev[other].flags); in fetch_block()
4010 sh->ops.target = disk_idx; in fetch_block()
4011 sh->ops.target2 = other; in fetch_block()
4030 static void handle_stripe_fill(struct stripe_head *sh, in handle_stripe_fill() argument
4040 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && in handle_stripe_fill()
4041 !sh->reconstruct_state) { in handle_stripe_fill()
4051 if (test_bit(STRIPE_R5C_CACHING, &sh->state)) in handle_stripe_fill()
4052 r5c_make_stripe_write_out(sh); in handle_stripe_fill()
4057 if (fetch_block(sh, s, i, disks)) in handle_stripe_fill()
4061 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe_fill()
4072 struct stripe_head *sh, int disks) in handle_stripe_clean_event() argument
4077 struct stripe_head *head_sh = sh; in handle_stripe_clean_event()
4081 if (sh->dev[i].written) { in handle_stripe_clean_event()
4082 dev = &sh->dev[i]; in handle_stripe_clean_event()
4108 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector, in handle_stripe_clean_event()
4110 !test_bit(STRIPE_DEGRADED, &sh->state), in handle_stripe_clean_event()
4113 sh = list_first_entry(&sh->batch_list, in handle_stripe_clean_event()
4116 if (sh != head_sh) { in handle_stripe_clean_event()
4117 dev = &sh->dev[i]; in handle_stripe_clean_event()
4121 sh = head_sh; in handle_stripe_clean_event()
4122 dev = &sh->dev[i]; in handle_stripe_clean_event()
4127 log_stripe_write_finished(sh); in handle_stripe_clean_event()
4130 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { in handle_stripe_clean_event()
4132 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); in handle_stripe_clean_event()
4133 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); in handle_stripe_clean_event()
4134 if (sh->qd_idx >= 0) { in handle_stripe_clean_event()
4135 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); in handle_stripe_clean_event()
4136 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); in handle_stripe_clean_event()
4139 clear_bit(STRIPE_DISCARD, &sh->state); in handle_stripe_clean_event()
4146 hash = sh->hash_lock_index; in handle_stripe_clean_event()
4148 remove_hash(sh); in handle_stripe_clean_event()
4151 sh = list_first_entry(&sh->batch_list, in handle_stripe_clean_event()
4153 if (sh != head_sh) in handle_stripe_clean_event()
4156 sh = head_sh; in handle_stripe_clean_event()
4158 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) in handle_stripe_clean_event()
4159 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe_clean_event()
4163 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) in handle_stripe_clean_event()
4187 struct stripe_head *sh, in handle_stripe_dirtying() argument
4202 (recovery_cp < MaxSector && sh->sector >= recovery_cp && in handle_stripe_dirtying()
4208 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", in handle_stripe_dirtying()
4210 (unsigned long long)sh->sector); in handle_stripe_dirtying()
4213 struct r5dev *dev = &sh->dev[i]; in handle_stripe_dirtying()
4215 i == sh->pd_idx || i == sh->qd_idx || in handle_stripe_dirtying()
4227 i != sh->pd_idx && i != sh->qd_idx && in handle_stripe_dirtying()
4239 (unsigned long long)sh->sector, sh->state, rmw, rcw); in handle_stripe_dirtying()
4240 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe_dirtying()
4246 (unsigned long long)sh->sector, rmw); in handle_stripe_dirtying()
4248 struct r5dev *dev = &sh->dev[i]; in handle_stripe_dirtying()
4251 !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) { in handle_stripe_dirtying()
4266 r5c_use_extra_page(sh); in handle_stripe_dirtying()
4271 set_bit(STRIPE_DELAYED, &sh->state); in handle_stripe_dirtying()
4278 struct r5dev *dev = &sh->dev[i]; in handle_stripe_dirtying()
4280 i == sh->pd_idx || i == sh->qd_idx || in handle_stripe_dirtying()
4287 &sh->state)) { in handle_stripe_dirtying()
4294 set_bit(STRIPE_DELAYED, &sh->state); in handle_stripe_dirtying()
4303 struct r5dev *dev = &sh->dev[i]; in handle_stripe_dirtying()
4305 i != sh->pd_idx && i != sh->qd_idx && in handle_stripe_dirtying()
4312 &sh->state)) { in handle_stripe_dirtying()
4320 set_bit(STRIPE_DELAYED, &sh->state); in handle_stripe_dirtying()
4325 (unsigned long long)sh->sector, in handle_stripe_dirtying()
4326 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); in handle_stripe_dirtying()
4330 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in handle_stripe_dirtying()
4331 set_bit(STRIPE_DELAYED, &sh->state); in handle_stripe_dirtying()
4343 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && in handle_stripe_dirtying()
4345 !test_bit(STRIPE_BIT_DELAY, &sh->state))) in handle_stripe_dirtying()
4346 schedule_reconstruction(sh, s, rcw == 0, 0); in handle_stripe_dirtying()
4350 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, in handle_parity_checks5() argument
4355 BUG_ON(sh->batch_head); in handle_parity_checks5()
4356 set_bit(STRIPE_HANDLE, &sh->state); in handle_parity_checks5()
4358 switch (sh->check_state) { in handle_parity_checks5()
4363 sh->check_state = check_state_run; in handle_parity_checks5()
4365 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); in handle_parity_checks5()
4369 dev = &sh->dev[s->failed_num[0]]; in handle_parity_checks5()
4372 sh->check_state = check_state_idle; in handle_parity_checks5()
4374 dev = &sh->dev[sh->pd_idx]; in handle_parity_checks5()
4377 if (test_bit(STRIPE_INSYNC, &sh->state)) in handle_parity_checks5()
4388 clear_bit(STRIPE_DEGRADED, &sh->state); in handle_parity_checks5()
4389 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks5()
4394 sh->check_state = check_state_idle; in handle_parity_checks5()
4406 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) in handle_parity_checks5()
4410 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks5()
4415 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks5()
4418 (unsigned long long) sh->sector, in handle_parity_checks5()
4419 (unsigned long long) sh->sector + in handle_parity_checks5()
4422 sh->check_state = check_state_compute_run; in handle_parity_checks5()
4423 set_bit(STRIPE_COMPUTE_RUN, &sh->state); in handle_parity_checks5()
4426 &sh->dev[sh->pd_idx].flags); in handle_parity_checks5()
4427 sh->ops.target = sh->pd_idx; in handle_parity_checks5()
4428 sh->ops.target2 = -1; in handle_parity_checks5()
4437 __func__, sh->check_state, in handle_parity_checks5()
4438 (unsigned long long) sh->sector); in handle_parity_checks5()
4443 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, in handle_parity_checks6() argument
4447 int pd_idx = sh->pd_idx; in handle_parity_checks6()
4448 int qd_idx = sh->qd_idx; in handle_parity_checks6()
4451 BUG_ON(sh->batch_head); in handle_parity_checks6()
4452 set_bit(STRIPE_HANDLE, &sh->state); in handle_parity_checks6()
4462 switch (sh->check_state) { in handle_parity_checks6()
4470 sh->check_state = check_state_run; in handle_parity_checks6()
4476 if (sh->check_state == check_state_run) in handle_parity_checks6()
4477 sh->check_state = check_state_run_pq; in handle_parity_checks6()
4479 sh->check_state = check_state_run_q; in handle_parity_checks6()
4483 sh->ops.zero_sum_result = 0; in handle_parity_checks6()
4485 if (sh->check_state == check_state_run) { in handle_parity_checks6()
4487 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); in handle_parity_checks6()
4490 if (sh->check_state >= check_state_run && in handle_parity_checks6()
4491 sh->check_state <= check_state_run_pq) { in handle_parity_checks6()
4503 sh->check_state = check_state_idle; in handle_parity_checks6()
4506 if (test_bit(STRIPE_INSYNC, &sh->state)) in handle_parity_checks6()
4514 dev = &sh->dev[s->failed_num[1]]; in handle_parity_checks6()
4520 dev = &sh->dev[s->failed_num[0]]; in handle_parity_checks6()
4525 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { in handle_parity_checks6()
4526 dev = &sh->dev[pd_idx]; in handle_parity_checks6()
4531 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { in handle_parity_checks6()
4532 dev = &sh->dev[qd_idx]; in handle_parity_checks6()
4540 dev - (struct r5dev *) &sh->dev)) { in handle_parity_checks6()
4545 clear_bit(STRIPE_DEGRADED, &sh->state); in handle_parity_checks6()
4547 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks6()
4554 sh->check_state = check_state_idle; in handle_parity_checks6()
4560 if (sh->ops.zero_sum_result == 0) { in handle_parity_checks6()
4563 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks6()
4569 sh->check_state = check_state_compute_result; in handle_parity_checks6()
4580 set_bit(STRIPE_INSYNC, &sh->state); in handle_parity_checks6()
4583 (unsigned long long) sh->sector, in handle_parity_checks6()
4584 (unsigned long long) sh->sector + in handle_parity_checks6()
4587 int *target = &sh->ops.target; in handle_parity_checks6()
4589 sh->ops.target = -1; in handle_parity_checks6()
4590 sh->ops.target2 = -1; in handle_parity_checks6()
4591 sh->check_state = check_state_compute_run; in handle_parity_checks6()
4592 set_bit(STRIPE_COMPUTE_RUN, &sh->state); in handle_parity_checks6()
4594 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { in handle_parity_checks6()
4596 &sh->dev[pd_idx].flags); in handle_parity_checks6()
4598 target = &sh->ops.target2; in handle_parity_checks6()
4601 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { in handle_parity_checks6()
4603 &sh->dev[qd_idx].flags); in handle_parity_checks6()
4614 __func__, sh->check_state, in handle_parity_checks6()
4615 (unsigned long long) sh->sector); in handle_parity_checks6()
4620 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) in handle_stripe_expansion() argument
4628 BUG_ON(sh->batch_head); in handle_stripe_expansion()
4629 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); in handle_stripe_expansion()
4630 for (i = 0; i < sh->disks; i++) in handle_stripe_expansion()
4631 if (i != sh->pd_idx && i != sh->qd_idx) { in handle_stripe_expansion()
4636 sector_t bn = raid5_compute_blocknr(sh, i, 1); in handle_stripe_expansion()
4657 sh->dev[i].page, sh2->dev[dd_idx].offset, in handle_stripe_expansion()
4658 sh->dev[i].offset, RAID5_STRIPE_SIZE(conf), in handle_stripe_expansion()
4693 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) in analyse_stripe() argument
4695 struct r5conf *conf = sh->raid_conf; in analyse_stripe()
4696 int disks = sh->disks; in analyse_stripe()
4703 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; in analyse_stripe()
4704 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; in analyse_stripe()
4717 dev = &sh->dev[i]; in analyse_stripe()
4728 !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) in analyse_stripe()
4757 rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && in analyse_stripe()
4758 !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), in analyse_stripe()
4772 is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), in analyse_stripe()
4799 else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset) in analyse_stripe()
4869 if (test_bit(STRIPE_SYNCING, &sh->state)) { in analyse_stripe()
4879 sh->sector >= conf->mddev->recovery_cp || in analyse_stripe()
4892 static int clear_batch_ready(struct stripe_head *sh) in clear_batch_ready() argument
4895 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) in clear_batch_ready()
4896 return (sh->batch_head && sh->batch_head != sh); in clear_batch_ready()
4897 spin_lock(&sh->stripe_lock); in clear_batch_ready()
4898 if (!sh->batch_head) { in clear_batch_ready()
4899 spin_unlock(&sh->stripe_lock); in clear_batch_ready()
4907 if (sh->batch_head != sh) { in clear_batch_ready()
4908 spin_unlock(&sh->stripe_lock); in clear_batch_ready()
4911 spin_lock(&sh->batch_lock); in clear_batch_ready()
4912 list_for_each_entry(tmp, &sh->batch_list, batch_list) in clear_batch_ready()
4914 spin_unlock(&sh->batch_lock); in clear_batch_ready()
4915 spin_unlock(&sh->stripe_lock); in clear_batch_ready()
4927 struct stripe_head *sh, *next; in break_stripe_batch_list() local
4931 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { in break_stripe_batch_list()
4933 list_del_init(&sh->batch_list); in break_stripe_batch_list()
4935 WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | in break_stripe_batch_list()
4947 "stripe state: %lx\n", sh->state); in break_stripe_batch_list()
4952 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | in break_stripe_batch_list()
4958 sh->check_state = head_sh->check_state; in break_stripe_batch_list()
4959 sh->reconstruct_state = head_sh->reconstruct_state; in break_stripe_batch_list()
4960 spin_lock_irq(&sh->stripe_lock); in break_stripe_batch_list()
4961 sh->batch_head = NULL; in break_stripe_batch_list()
4962 spin_unlock_irq(&sh->stripe_lock); in break_stripe_batch_list()
4963 for (i = 0; i < sh->disks; i++) { in break_stripe_batch_list()
4964 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) in break_stripe_batch_list()
4966 sh->dev[i].flags = head_sh->dev[i].flags & in break_stripe_batch_list()
4970 sh->state & handle_flags) in break_stripe_batch_list()
4971 set_bit(STRIPE_HANDLE, &sh->state); in break_stripe_batch_list()
4972 raid5_release_stripe(sh); in break_stripe_batch_list()
4987 static void handle_stripe(struct stripe_head *sh) in handle_stripe() argument
4990 struct r5conf *conf = sh->raid_conf; in handle_stripe()
4993 int disks = sh->disks; in handle_stripe()
4996 clear_bit(STRIPE_HANDLE, &sh->state); in handle_stripe()
5004 if (clear_batch_ready(sh)) in handle_stripe()
5007 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { in handle_stripe()
5010 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe()
5014 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) in handle_stripe()
5015 break_stripe_batch_list(sh, 0); in handle_stripe()
5017 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { in handle_stripe()
5018 spin_lock(&sh->stripe_lock); in handle_stripe()
5023 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) && in handle_stripe()
5024 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) && in handle_stripe()
5025 !test_bit(STRIPE_DISCARD, &sh->state) && in handle_stripe()
5026 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { in handle_stripe()
5027 set_bit(STRIPE_SYNCING, &sh->state); in handle_stripe()
5028 clear_bit(STRIPE_INSYNC, &sh->state); in handle_stripe()
5029 clear_bit(STRIPE_REPLACED, &sh->state); in handle_stripe()
5031 spin_unlock(&sh->stripe_lock); in handle_stripe()
5033 clear_bit(STRIPE_DELAYED, &sh->state); in handle_stripe()
5037 (unsigned long long)sh->sector, sh->state, in handle_stripe()
5038 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, in handle_stripe()
5039 sh->check_state, sh->reconstruct_state); in handle_stripe()
5041 analyse_stripe(sh, &s); in handle_stripe()
5043 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state)) in handle_stripe()
5048 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe()
5055 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe()
5063 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { in handle_stripe()
5065 set_bit(STRIPE_BIOFILL_RUN, &sh->state); in handle_stripe()
5081 sh->check_state = 0; in handle_stripe()
5082 sh->reconstruct_state = 0; in handle_stripe()
5083 break_stripe_batch_list(sh, 0); in handle_stripe()
5085 handle_failed_stripe(conf, sh, &s, disks); in handle_stripe()
5087 handle_failed_sync(conf, sh, &s); in handle_stripe()
5094 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) in handle_stripe()
5096 if (sh->reconstruct_state == reconstruct_state_drain_result || in handle_stripe()
5097 sh->reconstruct_state == reconstruct_state_prexor_drain_result) { in handle_stripe()
5098 sh->reconstruct_state = reconstruct_state_idle; in handle_stripe()
5103 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && in handle_stripe()
5104 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); in handle_stripe()
5105 BUG_ON(sh->qd_idx >= 0 && in handle_stripe()
5106 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && in handle_stripe()
5107 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); in handle_stripe()
5109 struct r5dev *dev = &sh->dev[i]; in handle_stripe()
5111 (i == sh->pd_idx || i == sh->qd_idx || in handle_stripe()
5121 ((i == sh->pd_idx || i == sh->qd_idx) && in handle_stripe()
5123 set_bit(STRIPE_INSYNC, &sh->state); in handle_stripe()
5126 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in handle_stripe()
5134 pdev = &sh->dev[sh->pd_idx]; in handle_stripe()
5135 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) in handle_stripe()
5136 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); in handle_stripe()
5137 qdev = &sh->dev[sh->qd_idx]; in handle_stripe()
5138 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) in handle_stripe()
5139 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) in handle_stripe()
5151 handle_stripe_clean_event(conf, sh, disks); in handle_stripe()
5154 r5c_handle_cached_data_endio(conf, sh, disks); in handle_stripe()
5155 log_stripe_write_finished(sh); in handle_stripe()
5166 handle_stripe_fill(sh, &s, disks); in handle_stripe()
5173 r5c_finish_stripe_write_out(conf, sh, &s); in handle_stripe()
5184 if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) { in handle_stripe()
5187 handle_stripe_dirtying(conf, sh, &s, disks); in handle_stripe()
5193 ret = r5c_try_caching_write(conf, sh, &s, in handle_stripe()
5204 (!test_bit(STRIPE_R5C_CACHING, &sh->state) && in handle_stripe()
5206 ret = handle_stripe_dirtying(conf, sh, &s, in handle_stripe()
5219 if (sh->check_state || in handle_stripe()
5221 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && in handle_stripe()
5222 !test_bit(STRIPE_INSYNC, &sh->state))) { in handle_stripe()
5224 handle_parity_checks6(conf, sh, &s, disks); in handle_stripe()
5226 handle_parity_checks5(conf, sh, &s, disks); in handle_stripe()
5230 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) in handle_stripe()
5231 && !test_bit(STRIPE_REPLACED, &sh->state)) { in handle_stripe()
5234 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { in handle_stripe()
5235 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); in handle_stripe()
5236 set_bit(R5_WantReplace, &sh->dev[i].flags); in handle_stripe()
5237 set_bit(R5_LOCKED, &sh->dev[i].flags); in handle_stripe()
5241 set_bit(STRIPE_INSYNC, &sh->state); in handle_stripe()
5242 set_bit(STRIPE_REPLACED, &sh->state); in handle_stripe()
5245 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && in handle_stripe()
5246 test_bit(STRIPE_INSYNC, &sh->state)) { in handle_stripe()
5248 clear_bit(STRIPE_SYNCING, &sh->state); in handle_stripe()
5249 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) in handle_stripe()
5258 struct r5dev *dev = &sh->dev[s.failed_num[i]]; in handle_stripe()
5275 if (sh->reconstruct_state == reconstruct_state_result) { in handle_stripe()
5277 = raid5_get_active_stripe(conf, NULL, sh->sector, in handle_stripe()
5281 /* sh cannot be written until sh_src has been read. in handle_stripe()
5282 * so arrange for sh to be delayed a little in handle_stripe()
5284 set_bit(STRIPE_DELAYED, &sh->state); in handle_stripe()
5285 set_bit(STRIPE_HANDLE, &sh->state); in handle_stripe()
5295 sh->reconstruct_state = reconstruct_state_idle; in handle_stripe()
5296 clear_bit(STRIPE_EXPANDING, &sh->state); in handle_stripe()
5298 set_bit(R5_Wantwrite, &sh->dev[i].flags); in handle_stripe()
5299 set_bit(R5_LOCKED, &sh->dev[i].flags); in handle_stripe()
5304 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && in handle_stripe()
5305 !sh->reconstruct_state) { in handle_stripe()
5307 sh->disks = conf->raid_disks; in handle_stripe()
5308 stripe_set_idx(sh->sector, conf, 0, sh); in handle_stripe()
5309 schedule_reconstruction(sh, &s, 1, 1); in handle_stripe()
5310 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { in handle_stripe()
5311 clear_bit(STRIPE_EXPAND_READY, &sh->state); in handle_stripe()
5318 !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) in handle_stripe()
5319 handle_stripe_expansion(conf, sh); in handle_stripe()
5339 struct r5dev *dev = &sh->dev[i]; in handle_stripe()
5343 if (!rdev_set_badblocks(rdev, sh->sector, in handle_stripe()
5350 rdev_clear_badblocks(rdev, sh->sector, in handle_stripe()
5359 rdev_clear_badblocks(rdev, sh->sector, in handle_stripe()
5366 raid_run_ops(sh, s.ops_request); in handle_stripe()
5368 ops_run_io(sh, &s); in handle_stripe()
5381 clear_bit_unlock(STRIPE_ACTIVE, &sh->state); in handle_stripe()
5390 struct stripe_head *sh; in raid5_activate_delayed() local
5391 sh = list_entry(l, struct stripe_head, lru); in raid5_activate_delayed()
5393 clear_bit(STRIPE_DELAYED, &sh->state); in raid5_activate_delayed()
5394 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in raid5_activate_delayed()
5396 list_add_tail(&sh->lru, &conf->hold_list); in raid5_activate_delayed()
5397 raid5_wakeup_stripe_thread(sh); in raid5_activate_delayed()
5410 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); in activate_bit_delay() local
5412 list_del_init(&sh->lru); in activate_bit_delay()
5413 atomic_inc(&sh->count); in activate_bit_delay()
5414 hash = sh->hash_lock_index; in activate_bit_delay()
5415 __release_stripe(conf, sh, &temp_inactive_list[hash]); in activate_bit_delay()
5618 struct stripe_head *sh, *tmp; in __get_priority_stripe() local
5628 sh = NULL; in __get_priority_stripe()
5654 sh = list_entry(handle_list->next, typeof(*sh), lru); in __get_priority_stripe()
5658 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { in __get_priority_stripe()
5678 sh = tmp; in __get_priority_stripe()
5683 if (sh) { in __get_priority_stripe()
5691 if (!sh) { in __get_priority_stripe()
5701 sh->group = NULL; in __get_priority_stripe()
5703 list_del_init(&sh->lru); in __get_priority_stripe()
5704 BUG_ON(atomic_inc_return(&sh->count) != 1); in __get_priority_stripe()
5705 return sh; in __get_priority_stripe()
5718 struct stripe_head *sh; in raid5_unplug() local
5727 sh = list_first_entry(&cb->list, struct stripe_head, lru); in raid5_unplug()
5728 list_del_init(&sh->lru); in raid5_unplug()
5735 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); in raid5_unplug()
5740 hash = sh->hash_lock_index; in raid5_unplug()
5741 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); in raid5_unplug()
5754 struct stripe_head *sh) in release_stripe_plug() argument
5762 raid5_release_stripe(sh); in release_stripe_plug()
5775 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) in release_stripe_plug()
5776 list_add_tail(&sh->lru, &cb->list); in release_stripe_plug()
5778 raid5_release_stripe(sh); in release_stripe_plug()
5785 struct stripe_head *sh; in make_discard_request() local
5815 sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0); in make_discard_request()
5818 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); in make_discard_request()
5819 if (test_bit(STRIPE_SYNCING, &sh->state)) { in make_discard_request()
5820 raid5_release_stripe(sh); in make_discard_request()
5824 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); in make_discard_request()
5825 spin_lock_irq(&sh->stripe_lock); in make_discard_request()
5827 if (d == sh->pd_idx || d == sh->qd_idx) in make_discard_request()
5829 if (sh->dev[d].towrite || sh->dev[d].toread) { in make_discard_request()
5830 set_bit(R5_Overlap, &sh->dev[d].flags); in make_discard_request()
5831 spin_unlock_irq(&sh->stripe_lock); in make_discard_request()
5832 raid5_release_stripe(sh); in make_discard_request()
5837 set_bit(STRIPE_DISCARD, &sh->state); in make_discard_request()
5839 sh->overwrite_disks = 0; in make_discard_request()
5841 if (d == sh->pd_idx || d == sh->qd_idx) in make_discard_request()
5843 sh->dev[d].towrite = bi; in make_discard_request()
5844 set_bit(R5_OVERWRITE, &sh->dev[d].flags); in make_discard_request()
5847 sh->overwrite_disks++; in make_discard_request()
5849 spin_unlock_irq(&sh->stripe_lock); in make_discard_request()
5855 sh->sector, in make_discard_request()
5858 sh->bm_seq = conf->seq_flush + 1; in make_discard_request()
5859 set_bit(STRIPE_BIT_DELAY, &sh->state); in make_discard_request()
5862 set_bit(STRIPE_HANDLE, &sh->state); in make_discard_request()
5863 clear_bit(STRIPE_DELAYED, &sh->state); in make_discard_request()
5864 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in make_discard_request()
5866 release_stripe_plug(mddev, sh); in make_discard_request()
5887 struct stripe_head *sh) in stripe_ahead_of_reshape() argument
5893 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) { in stripe_ahead_of_reshape()
5894 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) in stripe_ahead_of_reshape()
5897 min_sector = min(min_sector, sh->dev[dd_idx].sector); in stripe_ahead_of_reshape()
5898 max_sector = max(max_sector, sh->dev[dd_idx].sector); in stripe_ahead_of_reshape()
5914 struct stripe_request_ctx *ctx, struct stripe_head *sh, in add_all_stripe_bios() argument
5920 spin_lock_irq(&sh->stripe_lock); in add_all_stripe_bios()
5922 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) { in add_all_stripe_bios()
5923 struct r5dev *dev = &sh->dev[dd_idx]; in add_all_stripe_bios()
5925 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) in add_all_stripe_bios()
5932 if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) { in add_all_stripe_bios()
5942 for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) { in add_all_stripe_bios()
5943 struct r5dev *dev = &sh->dev[dd_idx]; in add_all_stripe_bios()
5945 if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) in add_all_stripe_bios()
5952 __add_stripe_bio(sh, bi, dd_idx, forwrite, previous); in add_all_stripe_bios()
5958 spin_unlock_irq(&sh->stripe_lock); in add_all_stripe_bios()
5981 struct stripe_head *sh; in make_stripe_request() local
6022 sh = raid5_get_active_stripe(conf, ctx, new_sector, flags); in make_stripe_request()
6023 if (unlikely(!sh)) { in make_stripe_request()
6030 stripe_ahead_of_reshape(mddev, conf, sh)) { in make_stripe_request()
6035 * 'sh', we know that if that happens, in make_stripe_request()
6049 if (test_bit(STRIPE_EXPANDING, &sh->state) || in make_stripe_request()
6050 !add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) { in make_stripe_request()
6060 if (stripe_can_batch(sh)) { in make_stripe_request()
6061 stripe_add_to_batch_list(conf, sh, ctx->batch_last); in make_stripe_request()
6064 atomic_inc(&sh->count); in make_stripe_request()
6065 ctx->batch_last = sh; in make_stripe_request()
6069 set_bit(STRIPE_R5C_PREFLUSH, &sh->state); in make_stripe_request()
6074 set_bit(STRIPE_HANDLE, &sh->state); in make_stripe_request()
6075 clear_bit(STRIPE_DELAYED, &sh->state); in make_stripe_request()
6076 if ((!sh->batch_head || sh == sh->batch_head) && in make_stripe_request()
6078 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) in make_stripe_request()
6081 release_stripe_plug(mddev, sh); in make_stripe_request()
6085 raid5_release_stripe(sh); in make_stripe_request()
6108 struct stripe_head sh; in raid5_bio_lowest_chunk_sector() local
6114 sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh); in raid5_bio_lowest_chunk_sector()
6123 while (dd_idx == sh.pd_idx || dd_idx == sh.qd_idx) in raid5_bio_lowest_chunk_sector()
6272 struct stripe_head *sh; in reshape_request() local
6429 sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i, in reshape_request()
6431 set_bit(STRIPE_EXPANDING, &sh->state); in reshape_request()
6436 for (j=sh->disks; j--;) { in reshape_request()
6438 if (j == sh->pd_idx) in reshape_request()
6441 j == sh->qd_idx) in reshape_request()
6443 s = raid5_compute_blocknr(sh, j, 0); in reshape_request()
6448 memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf)); in reshape_request()
6449 set_bit(R5_Expanded, &sh->dev[j].flags); in reshape_request()
6450 set_bit(R5_UPTODATE, &sh->dev[j].flags); in reshape_request()
6453 set_bit(STRIPE_EXPAND_READY, &sh->state); in reshape_request()
6454 set_bit(STRIPE_HANDLE, &sh->state); in reshape_request()
6456 list_add(&sh->lru, &stripes); in reshape_request()
6479 sh = raid5_get_active_stripe(conf, NULL, first_sector, in reshape_request()
6481 set_bit(STRIPE_EXPAND_SOURCE, &sh->state); in reshape_request()
6482 set_bit(STRIPE_HANDLE, &sh->state); in reshape_request()
6483 raid5_release_stripe(sh); in reshape_request()
6490 sh = list_entry(stripes.next, struct stripe_head, lru); in reshape_request()
6491 list_del_init(&sh->lru); in reshape_request()
6492 raid5_release_stripe(sh); in reshape_request()
6541 struct stripe_head *sh; in raid5_sync_request() local
6600 sh = raid5_get_active_stripe(conf, NULL, sector_nr, in raid5_sync_request()
6602 if (sh == NULL) { in raid5_sync_request()
6603 sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0); in raid5_sync_request()
6624 set_bit(STRIPE_SYNC_REQUESTED, &sh->state); in raid5_sync_request()
6625 set_bit(STRIPE_HANDLE, &sh->state); in raid5_sync_request()
6627 raid5_release_stripe(sh); in raid5_sync_request()
6645 struct stripe_head *sh; in retry_aligned_read() local
6666 sh = raid5_get_active_stripe(conf, NULL, sector, in retry_aligned_read()
6668 if (!sh) { in retry_aligned_read()
6675 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { in retry_aligned_read()
6676 raid5_release_stripe(sh); in retry_aligned_read()
6682 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); in retry_aligned_read()
6683 handle_stripe(sh); in retry_aligned_read()
6684 raid5_release_stripe(sh); in retry_aligned_read()
6700 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; in handle_active_stripes() local
6705 (sh = __get_priority_stripe(conf, group)) != NULL) in handle_active_stripes()
6706 batch[batch_size++] = sh; in handle_active_stripes()