1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2719c555fSJens Axboe /* bounce buffer handling for block devices
3719c555fSJens Axboe *
4719c555fSJens Axboe * - Split from highmem.c
5719c555fSJens Axboe */
6719c555fSJens Axboe
7b1de0d13SMitchel Humpherys #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8b1de0d13SMitchel Humpherys
9719c555fSJens Axboe #include <linux/mm.h>
10719c555fSJens Axboe #include <linux/export.h>
11719c555fSJens Axboe #include <linux/swap.h>
12719c555fSJens Axboe #include <linux/gfp.h>
13719c555fSJens Axboe #include <linux/bio.h>
14719c555fSJens Axboe #include <linux/pagemap.h>
15719c555fSJens Axboe #include <linux/mempool.h>
16719c555fSJens Axboe #include <linux/blkdev.h>
1766114cadSTejun Heo #include <linux/backing-dev.h>
18719c555fSJens Axboe #include <linux/init.h>
19719c555fSJens Axboe #include <linux/hash.h>
20719c555fSJens Axboe #include <linux/highmem.h>
21b1de0d13SMitchel Humpherys #include <linux/printk.h>
22719c555fSJens Axboe #include <asm/tlbflush.h>
23719c555fSJens Axboe
24719c555fSJens Axboe #include <trace/events/block.h>
253bce016aSChristoph Hellwig #include "blk.h"
26672fdcf0SMing Lei #include "blk-cgroup.h"
27719c555fSJens Axboe
28719c555fSJens Axboe #define POOL_SIZE 64
29719c555fSJens Axboe #define ISA_POOL_SIZE 16
30719c555fSJens Axboe
31338aa96dSKent Overstreet static struct bio_set bounce_bio_set, bounce_bio_split;
32ce288e05SChristoph Hellwig static mempool_t page_pool;
33719c555fSJens Axboe
init_bounce_bioset(void)3452990a5fSJens Axboe static void init_bounce_bioset(void)
3552990a5fSJens Axboe {
3652990a5fSJens Axboe static bool bounce_bs_setup;
3752990a5fSJens Axboe int ret;
3852990a5fSJens Axboe
3952990a5fSJens Axboe if (bounce_bs_setup)
4052990a5fSJens Axboe return;
4152990a5fSJens Axboe
4252990a5fSJens Axboe ret = bioset_init(&bounce_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
4352990a5fSJens Axboe BUG_ON(ret);
4452990a5fSJens Axboe if (bioset_integrity_create(&bounce_bio_set, BIO_POOL_SIZE))
4552990a5fSJens Axboe BUG_ON(1);
4652990a5fSJens Axboe
4752990a5fSJens Axboe ret = bioset_init(&bounce_bio_split, BIO_POOL_SIZE, 0, 0);
4852990a5fSJens Axboe BUG_ON(ret);
4952990a5fSJens Axboe bounce_bs_setup = true;
5052990a5fSJens Axboe }
5152990a5fSJens Axboe
init_emergency_pool(void)52719c555fSJens Axboe static __init int init_emergency_pool(void)
53719c555fSJens Axboe {
54338aa96dSKent Overstreet int ret;
559bb33f24SChristoph Hellwig
569bb33f24SChristoph Hellwig #ifndef CONFIG_MEMORY_HOTPLUG
57719c555fSJens Axboe if (max_pfn <= max_low_pfn)
58719c555fSJens Axboe return 0;
59719c555fSJens Axboe #endif
60719c555fSJens Axboe
61338aa96dSKent Overstreet ret = mempool_init_page_pool(&page_pool, POOL_SIZE, 0);
62338aa96dSKent Overstreet BUG_ON(ret);
63b1de0d13SMitchel Humpherys pr_info("pool size: %d pages\n", POOL_SIZE);
64719c555fSJens Axboe
6552990a5fSJens Axboe init_bounce_bioset();
66719c555fSJens Axboe return 0;
67719c555fSJens Axboe }
68719c555fSJens Axboe
69719c555fSJens Axboe __initcall(init_emergency_pool);
70719c555fSJens Axboe
71719c555fSJens Axboe /*
72719c555fSJens Axboe * Simple bounce buffer support for highmem pages. Depending on the
73719c555fSJens Axboe * queue gfp mask set, *to may or may not be a highmem page. kmap it
74719c555fSJens Axboe * always, it will do the Right Thing
75719c555fSJens Axboe */
copy_to_high_bio_irq(struct bio * to,struct bio * from)76719c555fSJens Axboe static void copy_to_high_bio_irq(struct bio *to, struct bio *from)
77719c555fSJens Axboe {
783c892a09SMing Lei struct bio_vec tovec, fromvec;
79719c555fSJens Axboe struct bvec_iter iter;
803c892a09SMing Lei /*
813c892a09SMing Lei * The bio of @from is created by bounce, so we can iterate
823c892a09SMing Lei * its bvec from start to end, but the @from->bi_iter can't be
833c892a09SMing Lei * trusted because it might be changed by splitting.
843c892a09SMing Lei */
853c892a09SMing Lei struct bvec_iter from_iter = BVEC_ITER_ALL_INIT;
86719c555fSJens Axboe
87719c555fSJens Axboe bio_for_each_segment(tovec, to, iter) {
883c892a09SMing Lei fromvec = bio_iter_iovec(from, from_iter);
893c892a09SMing Lei if (tovec.bv_page != fromvec.bv_page) {
90719c555fSJens Axboe /*
91719c555fSJens Axboe * fromvec->bv_offset and fromvec->bv_len might have
92719c555fSJens Axboe * been modified by the block layer, so use the original
93719c555fSJens Axboe * copy, bounce_copy_vec already uses tovec->bv_len
94719c555fSJens Axboe */
95f434cdc7SChristoph Hellwig memcpy_to_bvec(&tovec, page_address(fromvec.bv_page) +
96f434cdc7SChristoph Hellwig tovec.bv_offset);
97719c555fSJens Axboe }
983c892a09SMing Lei bio_advance_iter(from, &from_iter, tovec.bv_len);
99719c555fSJens Axboe }
100719c555fSJens Axboe }
101719c555fSJens Axboe
bounce_end_io(struct bio * bio)102ce288e05SChristoph Hellwig static void bounce_end_io(struct bio *bio)
103719c555fSJens Axboe {
104719c555fSJens Axboe struct bio *bio_orig = bio->bi_private;
1057891f05cSMing Lei struct bio_vec *bvec, orig_vec;
1067891f05cSMing Lei struct bvec_iter orig_iter = bio_orig->bi_iter;
1076dc4f100SMing Lei struct bvec_iter_all iter_all;
108719c555fSJens Axboe
109719c555fSJens Axboe /*
110719c555fSJens Axboe * free up bounce indirect pages used
111719c555fSJens Axboe */
1122b070cfeSChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) {
1137891f05cSMing Lei orig_vec = bio_iter_iovec(bio_orig, orig_iter);
1147891f05cSMing Lei if (bvec->bv_page != orig_vec.bv_page) {
115719c555fSJens Axboe dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
116ce288e05SChristoph Hellwig mempool_free(bvec->bv_page, &page_pool);
117719c555fSJens Axboe }
1187891f05cSMing Lei bio_advance_iter(bio_orig, &orig_iter, orig_vec.bv_len);
1197891f05cSMing Lei }
120719c555fSJens Axboe
1214e4cbee9SChristoph Hellwig bio_orig->bi_status = bio->bi_status;
1224246a0b6SChristoph Hellwig bio_endio(bio_orig);
123719c555fSJens Axboe bio_put(bio);
124719c555fSJens Axboe }
125719c555fSJens Axboe
bounce_end_io_write(struct bio * bio)1264246a0b6SChristoph Hellwig static void bounce_end_io_write(struct bio *bio)
127719c555fSJens Axboe {
128ce288e05SChristoph Hellwig bounce_end_io(bio);
129719c555fSJens Axboe }
130719c555fSJens Axboe
bounce_end_io_read(struct bio * bio)131ce288e05SChristoph Hellwig static void bounce_end_io_read(struct bio *bio)
132719c555fSJens Axboe {
133719c555fSJens Axboe struct bio *bio_orig = bio->bi_private;
134719c555fSJens Axboe
1354e4cbee9SChristoph Hellwig if (!bio->bi_status)
136719c555fSJens Axboe copy_to_high_bio_irq(bio_orig, bio);
137719c555fSJens Axboe
138ce288e05SChristoph Hellwig bounce_end_io(bio);
139719c555fSJens Axboe }
140719c555fSJens Axboe
bounce_clone_bio(struct bio * bio_src)141ebfe4183SChristoph Hellwig static struct bio *bounce_clone_bio(struct bio *bio_src)
142c55183c9SChristoph Hellwig {
143c55183c9SChristoph Hellwig struct bvec_iter iter;
144c55183c9SChristoph Hellwig struct bio_vec bv;
145c55183c9SChristoph Hellwig struct bio *bio;
146c55183c9SChristoph Hellwig
147c55183c9SChristoph Hellwig /*
148c55183c9SChristoph Hellwig * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
149c55183c9SChristoph Hellwig * bio_src->bi_io_vec to bio->bi_io_vec.
150c55183c9SChristoph Hellwig *
151c55183c9SChristoph Hellwig * We can't do that anymore, because:
152c55183c9SChristoph Hellwig *
153c55183c9SChristoph Hellwig * - The point of cloning the biovec is to produce a bio with a biovec
154c55183c9SChristoph Hellwig * the caller can modify: bi_idx and bi_bvec_done should be 0.
155c55183c9SChristoph Hellwig *
156a8affc03SChristoph Hellwig * - The original bio could've had more than BIO_MAX_VECS biovecs; if
157c55183c9SChristoph Hellwig * we tried to clone the whole thing bio_alloc_bioset() would fail.
158c55183c9SChristoph Hellwig * But the clone should succeed as long as the number of biovecs we
159a8affc03SChristoph Hellwig * actually need to allocate is fewer than BIO_MAX_VECS.
160c55183c9SChristoph Hellwig *
161c55183c9SChristoph Hellwig * - Lastly, bi_vcnt should not be looked at or relied upon by code
162c55183c9SChristoph Hellwig * that does not own the bio - reason being drivers don't use it for
163c55183c9SChristoph Hellwig * iterating over the biovec anymore, so expecting it to be kept up
164c55183c9SChristoph Hellwig * to date (i.e. for clones that share the parent biovec) is just
165abfc426dSChristoph Hellwig * asking for trouble and would force extra work.
166c55183c9SChristoph Hellwig */
167609be106SChristoph Hellwig bio = bio_alloc_bioset(bio_src->bi_bdev, bio_segments(bio_src),
168609be106SChristoph Hellwig bio_src->bi_opf, GFP_NOIO, &bounce_bio_set);
16946bbf653SChristoph Hellwig if (bio_flagged(bio_src, BIO_REMAPPED))
17046bbf653SChristoph Hellwig bio_set_flag(bio, BIO_REMAPPED);
171ca474b73SHannes Reinecke bio->bi_ioprio = bio_src->bi_ioprio;
172c55183c9SChristoph Hellwig bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
173c55183c9SChristoph Hellwig bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
174c55183c9SChristoph Hellwig
175c55183c9SChristoph Hellwig switch (bio_op(bio)) {
176c55183c9SChristoph Hellwig case REQ_OP_DISCARD:
177c55183c9SChristoph Hellwig case REQ_OP_SECURE_ERASE:
178c55183c9SChristoph Hellwig case REQ_OP_WRITE_ZEROES:
179c55183c9SChristoph Hellwig break;
180c55183c9SChristoph Hellwig default:
181c55183c9SChristoph Hellwig bio_for_each_segment(bv, bio_src, iter)
182c55183c9SChristoph Hellwig bio->bi_io_vec[bio->bi_vcnt++] = bv;
183c55183c9SChristoph Hellwig break;
184c55183c9SChristoph Hellwig }
185c55183c9SChristoph Hellwig
186ebfe4183SChristoph Hellwig if (bio_crypt_clone(bio, bio_src, GFP_NOIO) < 0)
18707560151SEric Biggers goto err_put;
188a892c8d5SSatya Tangirala
18907560151SEric Biggers if (bio_integrity(bio_src) &&
190ebfe4183SChristoph Hellwig bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0)
19107560151SEric Biggers goto err_put;
192c55183c9SChristoph Hellwig
193db6638d7SDennis Zhou bio_clone_blkg_association(bio, bio_src);
1945bf9a1f3SDennis Zhou (Facebook)
195c55183c9SChristoph Hellwig return bio;
19607560151SEric Biggers
19707560151SEric Biggers err_put:
19807560151SEric Biggers bio_put(bio);
19907560151SEric Biggers return NULL;
200c55183c9SChristoph Hellwig }
201c55183c9SChristoph Hellwig
__blk_queue_bounce(struct bio * bio_orig,struct request_queue * q)202*51d798cdSChristoph Hellwig struct bio *__blk_queue_bounce(struct bio *bio_orig, struct request_queue *q)
203719c555fSJens Axboe {
204719c555fSJens Axboe struct bio *bio;
205*51d798cdSChristoph Hellwig int rw = bio_data_dir(bio_orig);
206719c555fSJens Axboe struct bio_vec *to, from;
207719c555fSJens Axboe struct bvec_iter iter;
2089cfe3ddeSKeith Busch unsigned i = 0, bytes = 0;
209a8821f3fSNeilBrown bool bounce = false;
2109cfe3ddeSKeith Busch int sectors;
211719c555fSJens Axboe
212*51d798cdSChristoph Hellwig bio_for_each_segment(from, bio_orig, iter) {
213a8affc03SChristoph Hellwig if (i++ < BIO_MAX_VECS)
2149cfe3ddeSKeith Busch bytes += from.bv_len;
2159bb33f24SChristoph Hellwig if (PageHighMem(from.bv_page))
216a8821f3fSNeilBrown bounce = true;
217a8821f3fSNeilBrown }
218a8821f3fSNeilBrown if (!bounce)
219*51d798cdSChristoph Hellwig return bio_orig;
220a8821f3fSNeilBrown
2219cfe3ddeSKeith Busch /*
2229cfe3ddeSKeith Busch * Individual bvecs might not be logical block aligned. Round down
2239cfe3ddeSKeith Busch * the split size so that each bio is properly block size aligned,
2249cfe3ddeSKeith Busch * even if we do not use the full hardware limits.
2259cfe3ddeSKeith Busch */
2269cfe3ddeSKeith Busch sectors = ALIGN_DOWN(bytes, queue_logical_block_size(q)) >>
2279cfe3ddeSKeith Busch SECTOR_SHIFT;
228*51d798cdSChristoph Hellwig if (sectors < bio_sectors(bio_orig)) {
229*51d798cdSChristoph Hellwig bio = bio_split(bio_orig, sectors, GFP_NOIO, &bounce_bio_split);
230*51d798cdSChristoph Hellwig bio_chain(bio, bio_orig);
231*51d798cdSChristoph Hellwig submit_bio_noacct(bio_orig);
232*51d798cdSChristoph Hellwig bio_orig = bio;
233a8821f3fSNeilBrown }
234*51d798cdSChristoph Hellwig bio = bounce_clone_bio(bio_orig);
235719c555fSJens Axboe
2368f4e80daSMing Lei /*
2378f4e80daSMing Lei * Bvec table can't be updated by bio_for_each_segment_all(),
2388f4e80daSMing Lei * so retrieve bvec from the table directly. This way is safe
2398f4e80daSMing Lei * because the 'bio' is single-page bvec.
2408f4e80daSMing Lei */
2418f4e80daSMing Lei for (i = 0, to = bio->bi_io_vec; i < bio->bi_vcnt; to++, i++) {
2424aebe859SChristoph Hellwig struct page *bounce_page;
243719c555fSJens Axboe
2444aebe859SChristoph Hellwig if (!PageHighMem(to->bv_page))
245719c555fSJens Axboe continue;
246719c555fSJens Axboe
2474aebe859SChristoph Hellwig bounce_page = mempool_alloc(&page_pool, GFP_NOIO);
2484aebe859SChristoph Hellwig inc_zone_page_state(bounce_page, NR_BOUNCE);
249719c555fSJens Axboe
250719c555fSJens Axboe if (rw == WRITE) {
2514aebe859SChristoph Hellwig flush_dcache_page(to->bv_page);
2524aebe859SChristoph Hellwig memcpy_from_bvec(page_address(bounce_page), to);
253719c555fSJens Axboe }
2544aebe859SChristoph Hellwig to->bv_page = bounce_page;
255719c555fSJens Axboe }
256719c555fSJens Axboe
257*51d798cdSChristoph Hellwig trace_block_bio_bounce(bio_orig);
258719c555fSJens Axboe
259719c555fSJens Axboe bio->bi_flags |= (1 << BIO_BOUNCED);
260719c555fSJens Axboe
261719c555fSJens Axboe if (rw == READ)
262719c555fSJens Axboe bio->bi_end_io = bounce_end_io_read;
263ce288e05SChristoph Hellwig else
264ce288e05SChristoph Hellwig bio->bi_end_io = bounce_end_io_write;
265719c555fSJens Axboe
266*51d798cdSChristoph Hellwig bio->bi_private = bio_orig;
267*51d798cdSChristoph Hellwig return bio;
268719c555fSJens Axboe }
269