1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2be453e77SMing Lei /* Maximum size of each resync request */
3be453e77SMing Lei #define RESYNC_BLOCK_SIZE (64*1024)
4be453e77SMing Lei #define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
5be453e77SMing Lei
63f677f9cSMarcos Paulo de Souza /*
73f677f9cSMarcos Paulo de Souza * Number of guaranteed raid bios in case of extreme VM load:
83f677f9cSMarcos Paulo de Souza */
93f677f9cSMarcos Paulo de Souza #define NR_RAID_BIOS 256
103f677f9cSMarcos Paulo de Souza
113f677f9cSMarcos Paulo de Souza /* when we get a read error on a read-only array, we redirect to another
123f677f9cSMarcos Paulo de Souza * device without failing the first device, or trying to over-write to
133f677f9cSMarcos Paulo de Souza * correct the read error. To keep track of bad blocks on a per-bio
143f677f9cSMarcos Paulo de Souza * level, we store IO_BLOCKED in the appropriate 'bios' pointer
153f677f9cSMarcos Paulo de Souza */
163f677f9cSMarcos Paulo de Souza #define IO_BLOCKED ((struct bio *)1)
173f677f9cSMarcos Paulo de Souza /* When we successfully write to a known bad-block, we need to remove the
183f677f9cSMarcos Paulo de Souza * bad-block marking which must be done from process context. So we record
193f677f9cSMarcos Paulo de Souza * the success by setting devs[n].bio to IO_MADE_GOOD
203f677f9cSMarcos Paulo de Souza */
213f677f9cSMarcos Paulo de Souza #define IO_MADE_GOOD ((struct bio *)2)
223f677f9cSMarcos Paulo de Souza
233f677f9cSMarcos Paulo de Souza #define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)
24460af1f9SYu Kuai #define MAX_PLUG_BIO 32
253f677f9cSMarcos Paulo de Souza
26be453e77SMing Lei /* for managing resync I/O pages */
27be453e77SMing Lei struct resync_pages {
28be453e77SMing Lei void *raid_bio;
29be453e77SMing Lei struct page *pages[RESYNC_PAGES];
30be453e77SMing Lei };
31be453e77SMing Lei
32daae161fSMariusz Tkaczyk struct raid1_plug_cb {
33daae161fSMariusz Tkaczyk struct blk_plug_cb cb;
34daae161fSMariusz Tkaczyk struct bio_list pending;
35460af1f9SYu Kuai unsigned int count;
36daae161fSMariusz Tkaczyk };
37daae161fSMariusz Tkaczyk
rbio_pool_free(void * rbio,void * data)38c7afa803SMarcos Paulo de Souza static void rbio_pool_free(void *rbio, void *data)
39c7afa803SMarcos Paulo de Souza {
40c7afa803SMarcos Paulo de Souza kfree(rbio);
41c7afa803SMarcos Paulo de Souza }
42c7afa803SMarcos Paulo de Souza
resync_alloc_pages(struct resync_pages * rp,gfp_t gfp_flags)43be453e77SMing Lei static inline int resync_alloc_pages(struct resync_pages *rp,
44be453e77SMing Lei gfp_t gfp_flags)
45be453e77SMing Lei {
46be453e77SMing Lei int i;
47be453e77SMing Lei
48be453e77SMing Lei for (i = 0; i < RESYNC_PAGES; i++) {
49be453e77SMing Lei rp->pages[i] = alloc_page(gfp_flags);
50be453e77SMing Lei if (!rp->pages[i])
51be453e77SMing Lei goto out_free;
52be453e77SMing Lei }
53be453e77SMing Lei
54be453e77SMing Lei return 0;
55be453e77SMing Lei
56be453e77SMing Lei out_free:
57be453e77SMing Lei while (--i >= 0)
58be453e77SMing Lei put_page(rp->pages[i]);
59be453e77SMing Lei return -ENOMEM;
60be453e77SMing Lei }
61be453e77SMing Lei
resync_free_pages(struct resync_pages * rp)62be453e77SMing Lei static inline void resync_free_pages(struct resync_pages *rp)
63be453e77SMing Lei {
64be453e77SMing Lei int i;
65be453e77SMing Lei
66be453e77SMing Lei for (i = 0; i < RESYNC_PAGES; i++)
67be453e77SMing Lei put_page(rp->pages[i]);
68be453e77SMing Lei }
69be453e77SMing Lei
resync_get_all_pages(struct resync_pages * rp)70be453e77SMing Lei static inline void resync_get_all_pages(struct resync_pages *rp)
71be453e77SMing Lei {
72be453e77SMing Lei int i;
73be453e77SMing Lei
74be453e77SMing Lei for (i = 0; i < RESYNC_PAGES; i++)
75be453e77SMing Lei get_page(rp->pages[i]);
76be453e77SMing Lei }
77be453e77SMing Lei
resync_fetch_page(struct resync_pages * rp,unsigned idx)78be453e77SMing Lei static inline struct page *resync_fetch_page(struct resync_pages *rp,
79be453e77SMing Lei unsigned idx)
80be453e77SMing Lei {
81be453e77SMing Lei if (WARN_ON_ONCE(idx >= RESYNC_PAGES))
82be453e77SMing Lei return NULL;
83be453e77SMing Lei return rp->pages[idx];
84be453e77SMing Lei }
85be453e77SMing Lei
86be453e77SMing Lei /*
87be453e77SMing Lei * 'strct resync_pages' stores actual pages used for doing the resync
88be453e77SMing Lei * IO, and it is per-bio, so make .bi_private points to it.
89be453e77SMing Lei */
get_resync_pages(struct bio * bio)90be453e77SMing Lei static inline struct resync_pages *get_resync_pages(struct bio *bio)
91be453e77SMing Lei {
92be453e77SMing Lei return bio->bi_private;
93be453e77SMing Lei }
94be453e77SMing Lei
95fb0eb5dfSMing Lei /* generally called after bio_reset() for reseting bvec */
md_bio_reset_resync_pages(struct bio * bio,struct resync_pages * rp,int size)96fb0eb5dfSMing Lei static void md_bio_reset_resync_pages(struct bio *bio, struct resync_pages *rp,
97fb0eb5dfSMing Lei int size)
98fb0eb5dfSMing Lei {
99fb0eb5dfSMing Lei int idx = 0;
100fb0eb5dfSMing Lei
101fb0eb5dfSMing Lei /* initialize bvec table again */
102fb0eb5dfSMing Lei do {
103fb0eb5dfSMing Lei struct page *page = resync_fetch_page(rp, idx);
104fb0eb5dfSMing Lei int len = min_t(int, size, PAGE_SIZE);
105fb0eb5dfSMing Lei
1060c67dd64SJohannes Thumshirn if (WARN_ON(!bio_add_page(bio, page, len, 0))) {
1070c67dd64SJohannes Thumshirn bio->bi_status = BLK_STS_RESOURCE;
1080c67dd64SJohannes Thumshirn bio_endio(bio);
1090c67dd64SJohannes Thumshirn return;
1100c67dd64SJohannes Thumshirn }
1110c67dd64SJohannes Thumshirn
112fb0eb5dfSMing Lei size -= len;
113fb0eb5dfSMing Lei } while (idx++ < RESYNC_PAGES && size > 0);
114fb0eb5dfSMing Lei }
1155ec6ca14SYu Kuai
1168295efbeSYu Kuai
raid1_submit_write(struct bio * bio)1178295efbeSYu Kuai static inline void raid1_submit_write(struct bio *bio)
1188295efbeSYu Kuai {
119*b5a99602SYu Kuai struct md_rdev *rdev = (void *)bio->bi_bdev;
1208295efbeSYu Kuai
1218295efbeSYu Kuai bio->bi_next = NULL;
1228295efbeSYu Kuai bio_set_dev(bio, rdev->bdev);
1238295efbeSYu Kuai if (test_bit(Faulty, &rdev->flags))
1248295efbeSYu Kuai bio_io_error(bio);
1258295efbeSYu Kuai else if (unlikely(bio_op(bio) == REQ_OP_DISCARD &&
1268295efbeSYu Kuai !bdev_max_discard_sectors(bio->bi_bdev)))
1278295efbeSYu Kuai /* Just ignore it */
1288295efbeSYu Kuai bio_endio(bio);
1298295efbeSYu Kuai else
1308295efbeSYu Kuai submit_bio_noacct(bio);
1318295efbeSYu Kuai }
1328295efbeSYu Kuai
raid1_add_bio_to_plug(struct mddev * mddev,struct bio * bio,blk_plug_cb_fn unplug,int copies)1335ec6ca14SYu Kuai static inline bool raid1_add_bio_to_plug(struct mddev *mddev, struct bio *bio,
134460af1f9SYu Kuai blk_plug_cb_fn unplug, int copies)
1355ec6ca14SYu Kuai {
1365ec6ca14SYu Kuai struct raid1_plug_cb *plug = NULL;
1377db922baSYu Kuai struct blk_plug_cb *cb;
1385ec6ca14SYu Kuai
1397db922baSYu Kuai /*
1407db922baSYu Kuai * If bitmap is not enabled, it's safe to submit the io directly, and
1417db922baSYu Kuai * this can get optimal performance.
1427db922baSYu Kuai */
1437db922baSYu Kuai if (!md_bitmap_enabled(mddev->bitmap)) {
1447db922baSYu Kuai raid1_submit_write(bio);
1457db922baSYu Kuai return true;
1467db922baSYu Kuai }
1477db922baSYu Kuai
1487db922baSYu Kuai cb = blk_check_plugged(unplug, mddev, sizeof(*plug));
1495ec6ca14SYu Kuai if (!cb)
1505ec6ca14SYu Kuai return false;
1515ec6ca14SYu Kuai
1525ec6ca14SYu Kuai plug = container_of(cb, struct raid1_plug_cb, cb);
1535ec6ca14SYu Kuai bio_list_add(&plug->pending, bio);
154460af1f9SYu Kuai if (++plug->count / MAX_PLUG_BIO >= copies) {
155460af1f9SYu Kuai list_del(&cb->list);
156460af1f9SYu Kuai cb->callback(cb, false);
157460af1f9SYu Kuai }
158460af1f9SYu Kuai
1595ec6ca14SYu Kuai
1605ec6ca14SYu Kuai return true;
1615ec6ca14SYu Kuai }
1629efcc2c3SYu Kuai
1639efcc2c3SYu Kuai /*
1649efcc2c3SYu Kuai * current->bio_list will be set under submit_bio() context, in this case bitmap
1659efcc2c3SYu Kuai * io will be added to the list and wait for current io submission to finish,
1669efcc2c3SYu Kuai * while current io submission must wait for bitmap io to be done. In order to
1679efcc2c3SYu Kuai * avoid such deadlock, submit bitmap io asynchronously.
1689efcc2c3SYu Kuai */
raid1_prepare_flush_writes(struct bitmap * bitmap)1699efcc2c3SYu Kuai static inline void raid1_prepare_flush_writes(struct bitmap *bitmap)
1709efcc2c3SYu Kuai {
1719efcc2c3SYu Kuai if (current->bio_list)
1729efcc2c3SYu Kuai md_bitmap_unplug_async(bitmap);
1739efcc2c3SYu Kuai else
1749efcc2c3SYu Kuai md_bitmap_unplug(bitmap);
1759efcc2c3SYu Kuai }
176