xref: /openbmc/linux/block/blk-merge.c (revision f6970f83)
1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
2d6d48196SJens Axboe /*
3d6d48196SJens Axboe  * Functions related to segment and merge handling
4d6d48196SJens Axboe  */
5d6d48196SJens Axboe #include <linux/kernel.h>
6d6d48196SJens Axboe #include <linux/module.h>
7d6d48196SJens Axboe #include <linux/bio.h>
8d6d48196SJens Axboe #include <linux/blkdev.h>
9d6d48196SJens Axboe #include <linux/scatterlist.h>
10d6d48196SJens Axboe 
11cda22646SMike Krinkin #include <trace/events/block.h>
12cda22646SMike Krinkin 
13d6d48196SJens Axboe #include "blk.h"
14d6d48196SJens Axboe 
15e9907009SChristoph Hellwig /*
16e9907009SChristoph Hellwig  * Check if the two bvecs from two bios can be merged to one segment.  If yes,
17e9907009SChristoph Hellwig  * no need to check gap between the two bios since the 1st bio and the 1st bvec
18e9907009SChristoph Hellwig  * in the 2nd bio can be handled in one segment.
19e9907009SChristoph Hellwig  */
20e9907009SChristoph Hellwig static inline bool bios_segs_mergeable(struct request_queue *q,
21e9907009SChristoph Hellwig 		struct bio *prev, struct bio_vec *prev_last_bv,
22e9907009SChristoph Hellwig 		struct bio_vec *next_first_bv)
23e9907009SChristoph Hellwig {
243dccdae5SChristoph Hellwig 	if (!biovec_phys_mergeable(q, prev_last_bv, next_first_bv))
25e9907009SChristoph Hellwig 		return false;
26e9907009SChristoph Hellwig 	if (prev->bi_seg_back_size + next_first_bv->bv_len >
27e9907009SChristoph Hellwig 			queue_max_segment_size(q))
28e9907009SChristoph Hellwig 		return false;
29e9907009SChristoph Hellwig 	return true;
30e9907009SChristoph Hellwig }
31e9907009SChristoph Hellwig 
32e9907009SChristoph Hellwig static inline bool bio_will_gap(struct request_queue *q,
33e9907009SChristoph Hellwig 		struct request *prev_rq, struct bio *prev, struct bio *next)
34e9907009SChristoph Hellwig {
35e9907009SChristoph Hellwig 	struct bio_vec pb, nb;
36e9907009SChristoph Hellwig 
37e9907009SChristoph Hellwig 	if (!bio_has_data(prev) || !queue_virt_boundary(q))
38e9907009SChristoph Hellwig 		return false;
39e9907009SChristoph Hellwig 
40e9907009SChristoph Hellwig 	/*
41e9907009SChristoph Hellwig 	 * Don't merge if the 1st bio starts with non-zero offset, otherwise it
42e9907009SChristoph Hellwig 	 * is quite difficult to respect the sg gap limit.  We work hard to
43e9907009SChristoph Hellwig 	 * merge a huge number of small single bios in case of mkfs.
44e9907009SChristoph Hellwig 	 */
45e9907009SChristoph Hellwig 	if (prev_rq)
46e9907009SChristoph Hellwig 		bio_get_first_bvec(prev_rq->bio, &pb);
47e9907009SChristoph Hellwig 	else
48e9907009SChristoph Hellwig 		bio_get_first_bvec(prev, &pb);
49df376b2eSJohannes Thumshirn 	if (pb.bv_offset & queue_virt_boundary(q))
50e9907009SChristoph Hellwig 		return true;
51e9907009SChristoph Hellwig 
52e9907009SChristoph Hellwig 	/*
53e9907009SChristoph Hellwig 	 * We don't need to worry about the situation that the merged segment
54e9907009SChristoph Hellwig 	 * ends in unaligned virt boundary:
55e9907009SChristoph Hellwig 	 *
56e9907009SChristoph Hellwig 	 * - if 'pb' ends aligned, the merged segment ends aligned
57e9907009SChristoph Hellwig 	 * - if 'pb' ends unaligned, the next bio must include
58e9907009SChristoph Hellwig 	 *   one single bvec of 'nb', otherwise the 'nb' can't
59e9907009SChristoph Hellwig 	 *   merge with 'pb'
60e9907009SChristoph Hellwig 	 */
61e9907009SChristoph Hellwig 	bio_get_last_bvec(prev, &pb);
62e9907009SChristoph Hellwig 	bio_get_first_bvec(next, &nb);
63e9907009SChristoph Hellwig 	if (bios_segs_mergeable(q, prev, &pb, &nb))
64e9907009SChristoph Hellwig 		return false;
65e9907009SChristoph Hellwig 	return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
66e9907009SChristoph Hellwig }
67e9907009SChristoph Hellwig 
68e9907009SChristoph Hellwig static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
69e9907009SChristoph Hellwig {
70e9907009SChristoph Hellwig 	return bio_will_gap(req->q, req, req->biotail, bio);
71e9907009SChristoph Hellwig }
72e9907009SChristoph Hellwig 
73e9907009SChristoph Hellwig static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
74e9907009SChristoph Hellwig {
75e9907009SChristoph Hellwig 	return bio_will_gap(req->q, NULL, bio, req->bio);
76e9907009SChristoph Hellwig }
77e9907009SChristoph Hellwig 
7854efd50bSKent Overstreet static struct bio *blk_bio_discard_split(struct request_queue *q,
7954efd50bSKent Overstreet 					 struct bio *bio,
80bdced438SMing Lei 					 struct bio_set *bs,
81bdced438SMing Lei 					 unsigned *nsegs)
8254efd50bSKent Overstreet {
8354efd50bSKent Overstreet 	unsigned int max_discard_sectors, granularity;
8454efd50bSKent Overstreet 	int alignment;
8554efd50bSKent Overstreet 	sector_t tmp;
8654efd50bSKent Overstreet 	unsigned split_sectors;
8754efd50bSKent Overstreet 
88bdced438SMing Lei 	*nsegs = 1;
89bdced438SMing Lei 
9054efd50bSKent Overstreet 	/* Zero-sector (unknown) and one-sector granularities are the same.  */
9154efd50bSKent Overstreet 	granularity = max(q->limits.discard_granularity >> 9, 1U);
9254efd50bSKent Overstreet 
931adfc5e4SMing Lei 	max_discard_sectors = min(q->limits.max_discard_sectors,
941adfc5e4SMing Lei 			bio_allowed_max_sectors(q));
9554efd50bSKent Overstreet 	max_discard_sectors -= max_discard_sectors % granularity;
9654efd50bSKent Overstreet 
9754efd50bSKent Overstreet 	if (unlikely(!max_discard_sectors)) {
9854efd50bSKent Overstreet 		/* XXX: warn */
9954efd50bSKent Overstreet 		return NULL;
10054efd50bSKent Overstreet 	}
10154efd50bSKent Overstreet 
10254efd50bSKent Overstreet 	if (bio_sectors(bio) <= max_discard_sectors)
10354efd50bSKent Overstreet 		return NULL;
10454efd50bSKent Overstreet 
10554efd50bSKent Overstreet 	split_sectors = max_discard_sectors;
10654efd50bSKent Overstreet 
10754efd50bSKent Overstreet 	/*
10854efd50bSKent Overstreet 	 * If the next starting sector would be misaligned, stop the discard at
10954efd50bSKent Overstreet 	 * the previous aligned sector.
11054efd50bSKent Overstreet 	 */
11154efd50bSKent Overstreet 	alignment = (q->limits.discard_alignment >> 9) % granularity;
11254efd50bSKent Overstreet 
11354efd50bSKent Overstreet 	tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
11454efd50bSKent Overstreet 	tmp = sector_div(tmp, granularity);
11554efd50bSKent Overstreet 
11654efd50bSKent Overstreet 	if (split_sectors > tmp)
11754efd50bSKent Overstreet 		split_sectors -= tmp;
11854efd50bSKent Overstreet 
11954efd50bSKent Overstreet 	return bio_split(bio, split_sectors, GFP_NOIO, bs);
12054efd50bSKent Overstreet }
12154efd50bSKent Overstreet 
122885fa13fSChristoph Hellwig static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
123885fa13fSChristoph Hellwig 		struct bio *bio, struct bio_set *bs, unsigned *nsegs)
124885fa13fSChristoph Hellwig {
125885fa13fSChristoph Hellwig 	*nsegs = 1;
126885fa13fSChristoph Hellwig 
127885fa13fSChristoph Hellwig 	if (!q->limits.max_write_zeroes_sectors)
128885fa13fSChristoph Hellwig 		return NULL;
129885fa13fSChristoph Hellwig 
130885fa13fSChristoph Hellwig 	if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)
131885fa13fSChristoph Hellwig 		return NULL;
132885fa13fSChristoph Hellwig 
133885fa13fSChristoph Hellwig 	return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);
134885fa13fSChristoph Hellwig }
135885fa13fSChristoph Hellwig 
13654efd50bSKent Overstreet static struct bio *blk_bio_write_same_split(struct request_queue *q,
13754efd50bSKent Overstreet 					    struct bio *bio,
138bdced438SMing Lei 					    struct bio_set *bs,
139bdced438SMing Lei 					    unsigned *nsegs)
14054efd50bSKent Overstreet {
141bdced438SMing Lei 	*nsegs = 1;
142bdced438SMing Lei 
14354efd50bSKent Overstreet 	if (!q->limits.max_write_same_sectors)
14454efd50bSKent Overstreet 		return NULL;
14554efd50bSKent Overstreet 
14654efd50bSKent Overstreet 	if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
14754efd50bSKent Overstreet 		return NULL;
14854efd50bSKent Overstreet 
14954efd50bSKent Overstreet 	return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
15054efd50bSKent Overstreet }
15154efd50bSKent Overstreet 
152d0e5fbb0SMing Lei static inline unsigned get_max_io_size(struct request_queue *q,
153d0e5fbb0SMing Lei 				       struct bio *bio)
154d0e5fbb0SMing Lei {
155d0e5fbb0SMing Lei 	unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector);
156d0e5fbb0SMing Lei 	unsigned mask = queue_logical_block_size(q) - 1;
157d0e5fbb0SMing Lei 
158d0e5fbb0SMing Lei 	/* aligned to logical block size */
159d0e5fbb0SMing Lei 	sectors &= ~(mask >> 9);
160d0e5fbb0SMing Lei 
161d0e5fbb0SMing Lei 	return sectors;
162d0e5fbb0SMing Lei }
163d0e5fbb0SMing Lei 
164dcebd755SMing Lei static unsigned get_max_segment_size(struct request_queue *q,
165dcebd755SMing Lei 				     unsigned offset)
166dcebd755SMing Lei {
167dcebd755SMing Lei 	unsigned long mask = queue_segment_boundary(q);
168dcebd755SMing Lei 
169dcebd755SMing Lei 	/* default segment boundary mask means no boundary limit */
170dcebd755SMing Lei 	if (mask == BLK_SEG_BOUNDARY_MASK)
171dcebd755SMing Lei 		return queue_max_segment_size(q);
172dcebd755SMing Lei 
173dcebd755SMing Lei 	return min_t(unsigned long, mask - (mask & offset) + 1,
174dcebd755SMing Lei 		     queue_max_segment_size(q));
175dcebd755SMing Lei }
176dcebd755SMing Lei 
177dcebd755SMing Lei /*
178dcebd755SMing Lei  * Split the bvec @bv into segments, and update all kinds of
179dcebd755SMing Lei  * variables.
180dcebd755SMing Lei  */
181dcebd755SMing Lei static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
182dcebd755SMing Lei 		unsigned *nsegs, unsigned *last_seg_size,
18305b700baSMing Lei 		unsigned *front_seg_size, unsigned *sectors, unsigned max_segs)
184dcebd755SMing Lei {
185dcebd755SMing Lei 	unsigned len = bv->bv_len;
186dcebd755SMing Lei 	unsigned total_len = 0;
187dcebd755SMing Lei 	unsigned new_nsegs = 0, seg_size = 0;
188dcebd755SMing Lei 
189dcebd755SMing Lei 	/*
190dcebd755SMing Lei 	 * Multi-page bvec may be too big to hold in one segment, so the
191dcebd755SMing Lei 	 * current bvec has to be splitted as multiple segments.
192dcebd755SMing Lei 	 */
19305b700baSMing Lei 	while (len && new_nsegs + *nsegs < max_segs) {
194dcebd755SMing Lei 		seg_size = get_max_segment_size(q, bv->bv_offset + total_len);
195dcebd755SMing Lei 		seg_size = min(seg_size, len);
196dcebd755SMing Lei 
197dcebd755SMing Lei 		new_nsegs++;
198dcebd755SMing Lei 		total_len += seg_size;
199dcebd755SMing Lei 		len -= seg_size;
200dcebd755SMing Lei 
201dcebd755SMing Lei 		if ((bv->bv_offset + total_len) & queue_virt_boundary(q))
202dcebd755SMing Lei 			break;
203dcebd755SMing Lei 	}
204dcebd755SMing Lei 
205dcebd755SMing Lei 	if (!new_nsegs)
206dcebd755SMing Lei 		return !!len;
207dcebd755SMing Lei 
208dcebd755SMing Lei 	/* update front segment size */
209dcebd755SMing Lei 	if (!*nsegs) {
210dcebd755SMing Lei 		unsigned first_seg_size;
211dcebd755SMing Lei 
212dcebd755SMing Lei 		if (new_nsegs == 1)
213dcebd755SMing Lei 			first_seg_size = get_max_segment_size(q, bv->bv_offset);
214dcebd755SMing Lei 		else
215dcebd755SMing Lei 			first_seg_size = queue_max_segment_size(q);
216dcebd755SMing Lei 
217dcebd755SMing Lei 		if (*front_seg_size < first_seg_size)
218dcebd755SMing Lei 			*front_seg_size = first_seg_size;
219dcebd755SMing Lei 	}
220dcebd755SMing Lei 
221dcebd755SMing Lei 	/* update other varibles */
222dcebd755SMing Lei 	*last_seg_size = seg_size;
223dcebd755SMing Lei 	*nsegs += new_nsegs;
224dcebd755SMing Lei 	if (sectors)
225dcebd755SMing Lei 		*sectors += total_len >> 9;
226dcebd755SMing Lei 
227dcebd755SMing Lei 	/* split in the middle of the bvec if len != 0 */
228dcebd755SMing Lei 	return !!len;
229dcebd755SMing Lei }
230dcebd755SMing Lei 
23154efd50bSKent Overstreet static struct bio *blk_bio_segment_split(struct request_queue *q,
23254efd50bSKent Overstreet 					 struct bio *bio,
233bdced438SMing Lei 					 struct bio_set *bs,
234bdced438SMing Lei 					 unsigned *segs)
23554efd50bSKent Overstreet {
2365014c311SJens Axboe 	struct bio_vec bv, bvprv, *bvprvp = NULL;
23754efd50bSKent Overstreet 	struct bvec_iter iter;
2388ae12666SKent Overstreet 	unsigned seg_size = 0, nsegs = 0, sectors = 0;
23902e70742SMing Lei 	unsigned front_seg_size = bio->bi_seg_front_size;
24002e70742SMing Lei 	bool do_split = true;
24102e70742SMing Lei 	struct bio *new = NULL;
242d0e5fbb0SMing Lei 	const unsigned max_sectors = get_max_io_size(q, bio);
24305b700baSMing Lei 	const unsigned max_segs = queue_max_segments(q);
24454efd50bSKent Overstreet 
245dcebd755SMing Lei 	bio_for_each_bvec(bv, bio, iter) {
24654efd50bSKent Overstreet 		/*
24754efd50bSKent Overstreet 		 * If the queue doesn't support SG gaps and adding this
24854efd50bSKent Overstreet 		 * offset would create a gap, disallow it.
24954efd50bSKent Overstreet 		 */
2505014c311SJens Axboe 		if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
25154efd50bSKent Overstreet 			goto split;
25254efd50bSKent Overstreet 
253d0e5fbb0SMing Lei 		if (sectors + (bv.bv_len >> 9) > max_sectors) {
254e36f6204SKeith Busch 			/*
255e36f6204SKeith Busch 			 * Consider this a new segment if we're splitting in
256e36f6204SKeith Busch 			 * the middle of this vector.
257e36f6204SKeith Busch 			 */
25805b700baSMing Lei 			if (nsegs < max_segs &&
259d0e5fbb0SMing Lei 			    sectors < max_sectors) {
260dcebd755SMing Lei 				/* split in the middle of bvec */
261dcebd755SMing Lei 				bv.bv_len = (max_sectors - sectors) << 9;
262dcebd755SMing Lei 				bvec_split_segs(q, &bv, &nsegs,
263dcebd755SMing Lei 						&seg_size,
264dcebd755SMing Lei 						&front_seg_size,
26505b700baSMing Lei 						&sectors, max_segs);
266e36f6204SKeith Busch 			}
267e36f6204SKeith Busch 			goto split;
268e36f6204SKeith Busch 		}
269e36f6204SKeith Busch 
27005b700baSMing Lei 		if (nsegs == max_segs)
27154efd50bSKent Overstreet 			goto split;
27254efd50bSKent Overstreet 
27354efd50bSKent Overstreet 		bvprv = bv;
274578270bfSMing Lei 		bvprvp = &bvprv;
275dcebd755SMing Lei 
276bbcbbd56SMing Lei 		if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
277bbcbbd56SMing Lei 			nsegs++;
278bbcbbd56SMing Lei 			seg_size = bv.bv_len;
279bbcbbd56SMing Lei 			sectors += bv.bv_len >> 9;
280bbcbbd56SMing Lei 			if (nsegs == 1 && seg_size > front_seg_size)
281bbcbbd56SMing Lei 				front_seg_size = seg_size;
282bbcbbd56SMing Lei 		} else if (bvec_split_segs(q, &bv, &nsegs, &seg_size,
28305b700baSMing Lei 				    &front_seg_size, &sectors, max_segs)) {
284dcebd755SMing Lei 			goto split;
285bbcbbd56SMing Lei 		}
28654efd50bSKent Overstreet 	}
28754efd50bSKent Overstreet 
28802e70742SMing Lei 	do_split = false;
28954efd50bSKent Overstreet split:
290bdced438SMing Lei 	*segs = nsegs;
29102e70742SMing Lei 
29202e70742SMing Lei 	if (do_split) {
29302e70742SMing Lei 		new = bio_split(bio, sectors, GFP_NOIO, bs);
29402e70742SMing Lei 		if (new)
29502e70742SMing Lei 			bio = new;
29602e70742SMing Lei 	}
29702e70742SMing Lei 
29802e70742SMing Lei 	bio->bi_seg_front_size = front_seg_size;
29902e70742SMing Lei 	if (seg_size > bio->bi_seg_back_size)
30002e70742SMing Lei 		bio->bi_seg_back_size = seg_size;
30102e70742SMing Lei 
30202e70742SMing Lei 	return do_split ? new : NULL;
30354efd50bSKent Overstreet }
30454efd50bSKent Overstreet 
305af67c31fSNeilBrown void blk_queue_split(struct request_queue *q, struct bio **bio)
30654efd50bSKent Overstreet {
307bdced438SMing Lei 	struct bio *split, *res;
308bdced438SMing Lei 	unsigned nsegs;
30954efd50bSKent Overstreet 
3107afafc8aSAdrian Hunter 	switch (bio_op(*bio)) {
3117afafc8aSAdrian Hunter 	case REQ_OP_DISCARD:
3127afafc8aSAdrian Hunter 	case REQ_OP_SECURE_ERASE:
313338aa96dSKent Overstreet 		split = blk_bio_discard_split(q, *bio, &q->bio_split, &nsegs);
3147afafc8aSAdrian Hunter 		break;
315a6f0788eSChaitanya Kulkarni 	case REQ_OP_WRITE_ZEROES:
316338aa96dSKent Overstreet 		split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, &nsegs);
317a6f0788eSChaitanya Kulkarni 		break;
3187afafc8aSAdrian Hunter 	case REQ_OP_WRITE_SAME:
319338aa96dSKent Overstreet 		split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs);
3207afafc8aSAdrian Hunter 		break;
3217afafc8aSAdrian Hunter 	default:
322338aa96dSKent Overstreet 		split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs);
3237afafc8aSAdrian Hunter 		break;
3247afafc8aSAdrian Hunter 	}
325bdced438SMing Lei 
326bdced438SMing Lei 	/* physical segments can be figured out during splitting */
327bdced438SMing Lei 	res = split ? split : *bio;
328bdced438SMing Lei 	res->bi_phys_segments = nsegs;
329bdced438SMing Lei 	bio_set_flag(res, BIO_SEG_VALID);
33054efd50bSKent Overstreet 
33154efd50bSKent Overstreet 	if (split) {
3326ac45aebSMing Lei 		/* there isn't chance to merge the splitted bio */
3331eff9d32SJens Axboe 		split->bi_opf |= REQ_NOMERGE;
3346ac45aebSMing Lei 
335947b7ac1SJens Axboe 		/*
336947b7ac1SJens Axboe 		 * Since we're recursing into make_request here, ensure
337947b7ac1SJens Axboe 		 * that we mark this bio as already having entered the queue.
338947b7ac1SJens Axboe 		 * If not, and the queue is going away, we can get stuck
339947b7ac1SJens Axboe 		 * forever on waiting for the queue reference to drop. But
340947b7ac1SJens Axboe 		 * that will never happen, as we're already holding a
341947b7ac1SJens Axboe 		 * reference to it.
342947b7ac1SJens Axboe 		 */
343947b7ac1SJens Axboe 		bio_set_flag(*bio, BIO_QUEUE_ENTERED);
344947b7ac1SJens Axboe 
34554efd50bSKent Overstreet 		bio_chain(split, *bio);
346cda22646SMike Krinkin 		trace_block_split(q, split, (*bio)->bi_iter.bi_sector);
34754efd50bSKent Overstreet 		generic_make_request(*bio);
34854efd50bSKent Overstreet 		*bio = split;
34954efd50bSKent Overstreet 	}
35054efd50bSKent Overstreet }
35154efd50bSKent Overstreet EXPORT_SYMBOL(blk_queue_split);
35254efd50bSKent Overstreet 
3531e428079SJens Axboe static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
3542705c937SMing Lei 					     struct bio *bio)
355d6d48196SJens Axboe {
3567988613bSKent Overstreet 	struct bio_vec bv, bvprv = { NULL };
3571e428079SJens Axboe 	unsigned int seg_size, nr_phys_segs;
35849b1f22bSMing Lei 	unsigned front_seg_size;
35959247eaeSJens Axboe 	struct bio *fbio, *bbio;
3607988613bSKent Overstreet 	struct bvec_iter iter;
361f6970f83SMing Lei 	bool new_bio = false;
362d6d48196SJens Axboe 
3631e428079SJens Axboe 	if (!bio)
3641e428079SJens Axboe 		return 0;
365d6d48196SJens Axboe 
36649b1f22bSMing Lei 	front_seg_size = bio->bi_seg_front_size;
36749b1f22bSMing Lei 
368a6f0788eSChaitanya Kulkarni 	switch (bio_op(bio)) {
369a6f0788eSChaitanya Kulkarni 	case REQ_OP_DISCARD:
370a6f0788eSChaitanya Kulkarni 	case REQ_OP_SECURE_ERASE:
371a6f0788eSChaitanya Kulkarni 	case REQ_OP_WRITE_ZEROES:
372f9d03f96SChristoph Hellwig 		return 0;
373f9d03f96SChristoph Hellwig 	case REQ_OP_WRITE_SAME:
3745cb8850cSKent Overstreet 		return 1;
375a6f0788eSChaitanya Kulkarni 	}
3765cb8850cSKent Overstreet 
3771e428079SJens Axboe 	fbio = bio;
3785df97b91SMikulas Patocka 	seg_size = 0;
3792c8919deSAndi Kleen 	nr_phys_segs = 0;
3801e428079SJens Axboe 	for_each_bio(bio) {
381dcebd755SMing Lei 		bio_for_each_bvec(bv, bio, iter) {
382f6970f83SMing Lei 			if (new_bio) {
3837988613bSKent Overstreet 				if (seg_size + bv.bv_len
384ae03bf63SMartin K. Petersen 				    > queue_max_segment_size(q))
385d6d48196SJens Axboe 					goto new_segment;
3863dccdae5SChristoph Hellwig 				if (!biovec_phys_mergeable(q, &bvprv, &bv))
387d6d48196SJens Axboe 					goto new_segment;
388d6d48196SJens Axboe 
3897988613bSKent Overstreet 				seg_size += bv.bv_len;
390aaeee62cSMing Lei 
391aaeee62cSMing Lei 				if (nr_phys_segs == 1 && seg_size >
392aaeee62cSMing Lei 						front_seg_size)
393aaeee62cSMing Lei 					front_seg_size = seg_size;
394aaeee62cSMing Lei 
395d6d48196SJens Axboe 				continue;
396d6d48196SJens Axboe 			}
397d6d48196SJens Axboe new_segment:
398dcebd755SMing Lei 			bvec_split_segs(q, &bv, &nr_phys_segs, &seg_size,
39905b700baSMing Lei 					&front_seg_size, NULL, UINT_MAX);
400f6970f83SMing Lei 			new_bio = false;
401d6d48196SJens Axboe 		}
40259247eaeSJens Axboe 		bbio = bio;
403f6970f83SMing Lei 		bvprv = bv;
404f6970f83SMing Lei 		new_bio = true;
4051e428079SJens Axboe 	}
406d6d48196SJens Axboe 
407dcebd755SMing Lei 	fbio->bi_seg_front_size = front_seg_size;
40859247eaeSJens Axboe 	if (seg_size > bbio->bi_seg_back_size)
40959247eaeSJens Axboe 		bbio->bi_seg_back_size = seg_size;
4101e428079SJens Axboe 
4111e428079SJens Axboe 	return nr_phys_segs;
4121e428079SJens Axboe }
4131e428079SJens Axboe 
4141e428079SJens Axboe void blk_recalc_rq_segments(struct request *rq)
4151e428079SJens Axboe {
4162705c937SMing Lei 	rq->nr_phys_segments = __blk_recalc_rq_segments(rq->q, rq->bio);
417d6d48196SJens Axboe }
418d6d48196SJens Axboe 
419d6d48196SJens Axboe void blk_recount_segments(struct request_queue *q, struct bio *bio)
420d6d48196SJens Axboe {
421d6d48196SJens Axboe 	struct bio *nxt = bio->bi_next;
4221e428079SJens Axboe 
423d6d48196SJens Axboe 	bio->bi_next = NULL;
4242705c937SMing Lei 	bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
425d6d48196SJens Axboe 	bio->bi_next = nxt;
42605f1dd53SJens Axboe 
427b7c44ed9SJens Axboe 	bio_set_flag(bio, BIO_SEG_VALID);
428d6d48196SJens Axboe }
429d6d48196SJens Axboe 
430d6d48196SJens Axboe static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio,
431d6d48196SJens Axboe 				   struct bio *nxt)
432d6d48196SJens Axboe {
4332b8221e1SKent Overstreet 	struct bio_vec end_bv = { NULL }, nxt_bv;
434f619d254SKent Overstreet 
43586771427SFUJITA Tomonori 	if (bio->bi_seg_back_size + nxt->bi_seg_front_size >
436ae03bf63SMartin K. Petersen 	    queue_max_segment_size(q))
437d6d48196SJens Axboe 		return 0;
438d6d48196SJens Axboe 
439e17fc0a1SDavid Woodhouse 	if (!bio_has_data(bio))
440e17fc0a1SDavid Woodhouse 		return 1;
441e17fc0a1SDavid Woodhouse 
442e827091cSMing Lei 	bio_get_last_bvec(bio, &end_bv);
443e827091cSMing Lei 	bio_get_first_bvec(nxt, &nxt_bv);
444f619d254SKent Overstreet 
4453dccdae5SChristoph Hellwig 	return biovec_phys_mergeable(q, &end_bv, &nxt_bv);
446d6d48196SJens Axboe }
447d6d48196SJens Axboe 
44848d7727cSMing Lei static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
449862e5a5eSMing Lei 		struct scatterlist *sglist)
450862e5a5eSMing Lei {
451862e5a5eSMing Lei 	if (!*sg)
452862e5a5eSMing Lei 		return sglist;
453862e5a5eSMing Lei 
454862e5a5eSMing Lei 	/*
455862e5a5eSMing Lei 	 * If the driver previously mapped a shorter list, we could see a
456862e5a5eSMing Lei 	 * termination bit prematurely unless it fully inits the sg table
457862e5a5eSMing Lei 	 * on each mapping. We KNOW that there must be more entries here
458862e5a5eSMing Lei 	 * or the driver would be buggy, so force clear the termination bit
459862e5a5eSMing Lei 	 * to avoid doing a full sg_init_table() in drivers for each command.
460862e5a5eSMing Lei 	 */
461862e5a5eSMing Lei 	sg_unmark_end(*sg);
462862e5a5eSMing Lei 	return sg_next(*sg);
463862e5a5eSMing Lei }
464862e5a5eSMing Lei 
465862e5a5eSMing Lei static unsigned blk_bvec_map_sg(struct request_queue *q,
466862e5a5eSMing Lei 		struct bio_vec *bvec, struct scatterlist *sglist,
467862e5a5eSMing Lei 		struct scatterlist **sg)
468862e5a5eSMing Lei {
469862e5a5eSMing Lei 	unsigned nbytes = bvec->bv_len;
470862e5a5eSMing Lei 	unsigned nsegs = 0, total = 0, offset = 0;
471862e5a5eSMing Lei 
472862e5a5eSMing Lei 	while (nbytes > 0) {
473862e5a5eSMing Lei 		unsigned seg_size;
474862e5a5eSMing Lei 		struct page *pg;
475862e5a5eSMing Lei 		unsigned idx;
476862e5a5eSMing Lei 
477862e5a5eSMing Lei 		*sg = blk_next_sg(sg, sglist);
478862e5a5eSMing Lei 
479862e5a5eSMing Lei 		seg_size = get_max_segment_size(q, bvec->bv_offset + total);
480862e5a5eSMing Lei 		seg_size = min(nbytes, seg_size);
481862e5a5eSMing Lei 
482862e5a5eSMing Lei 		offset = (total + bvec->bv_offset) % PAGE_SIZE;
483862e5a5eSMing Lei 		idx = (total + bvec->bv_offset) / PAGE_SIZE;
4844d633062SMing Lei 		pg = bvec_nth_page(bvec->bv_page, idx);
485862e5a5eSMing Lei 
486862e5a5eSMing Lei 		sg_set_page(*sg, pg, seg_size, offset);
487862e5a5eSMing Lei 
488862e5a5eSMing Lei 		total += seg_size;
489862e5a5eSMing Lei 		nbytes -= seg_size;
490862e5a5eSMing Lei 		nsegs++;
491862e5a5eSMing Lei 	}
492862e5a5eSMing Lei 
493862e5a5eSMing Lei 	return nsegs;
494862e5a5eSMing Lei }
495862e5a5eSMing Lei 
49616e3e418SMing Lei static inline int __blk_bvec_map_sg(struct bio_vec bv,
49716e3e418SMing Lei 		struct scatterlist *sglist, struct scatterlist **sg)
49816e3e418SMing Lei {
49916e3e418SMing Lei 	*sg = blk_next_sg(sg, sglist);
50016e3e418SMing Lei 	sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
50116e3e418SMing Lei 	return 1;
50216e3e418SMing Lei }
50316e3e418SMing Lei 
504f6970f83SMing Lei /* only try to merge bvecs into one sg if they are from two bios */
505f6970f83SMing Lei static inline bool
506f6970f83SMing Lei __blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec,
507f6970f83SMing Lei 			   struct bio_vec *bvprv, struct scatterlist **sg)
508963ab9e5SAsias He {
509963ab9e5SAsias He 
510963ab9e5SAsias He 	int nbytes = bvec->bv_len;
511963ab9e5SAsias He 
512f6970f83SMing Lei 	if (!*sg)
513f6970f83SMing Lei 		return false;
514f6970f83SMing Lei 
515b4b6cb61SMing Lei 	if ((*sg)->length + nbytes > queue_max_segment_size(q))
516f6970f83SMing Lei 		return false;
517f6970f83SMing Lei 
5183dccdae5SChristoph Hellwig 	if (!biovec_phys_mergeable(q, bvprv, bvec))
519f6970f83SMing Lei 		return false;
520963ab9e5SAsias He 
521963ab9e5SAsias He 	(*sg)->length += nbytes;
522f6970f83SMing Lei 
523f6970f83SMing Lei 	return true;
524963ab9e5SAsias He }
525963ab9e5SAsias He 
5265cb8850cSKent Overstreet static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
5275cb8850cSKent Overstreet 			     struct scatterlist *sglist,
5285cb8850cSKent Overstreet 			     struct scatterlist **sg)
5295cb8850cSKent Overstreet {
5305cb8850cSKent Overstreet 	struct bio_vec bvec, bvprv = { NULL };
5315cb8850cSKent Overstreet 	struct bvec_iter iter;
53238417468SChristoph Hellwig 	int nsegs = 0;
533f6970f83SMing Lei 	bool new_bio = false;
5345cb8850cSKent Overstreet 
535f6970f83SMing Lei 	for_each_bio(bio) {
536f6970f83SMing Lei 		bio_for_each_bvec(bvec, bio, iter) {
537f6970f83SMing Lei 			/*
538f6970f83SMing Lei 			 * Only try to merge bvecs from two bios given we
539f6970f83SMing Lei 			 * have done bio internal merge when adding pages
540f6970f83SMing Lei 			 * to bio
541f6970f83SMing Lei 			 */
542f6970f83SMing Lei 			if (new_bio &&
543f6970f83SMing Lei 			    __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg))
544f6970f83SMing Lei 				goto next_bvec;
545f6970f83SMing Lei 
546f6970f83SMing Lei 			if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE)
547f6970f83SMing Lei 				nsegs += __blk_bvec_map_sg(bvec, sglist, sg);
548f6970f83SMing Lei 			else
549f6970f83SMing Lei 				nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg);
550f6970f83SMing Lei  next_bvec:
551f6970f83SMing Lei 			new_bio = false;
552f6970f83SMing Lei 		}
553f6970f83SMing Lei 		bvprv = bvec;
554f6970f83SMing Lei 		new_bio = true;
555f6970f83SMing Lei 	}
5565cb8850cSKent Overstreet 
5575cb8850cSKent Overstreet 	return nsegs;
5585cb8850cSKent Overstreet }
5595cb8850cSKent Overstreet 
560d6d48196SJens Axboe /*
561d6d48196SJens Axboe  * map a request to scatterlist, return number of sg entries setup. Caller
562d6d48196SJens Axboe  * must make sure sg can hold rq->nr_phys_segments entries
563d6d48196SJens Axboe  */
564d6d48196SJens Axboe int blk_rq_map_sg(struct request_queue *q, struct request *rq,
565d6d48196SJens Axboe 		  struct scatterlist *sglist)
566d6d48196SJens Axboe {
5675cb8850cSKent Overstreet 	struct scatterlist *sg = NULL;
5685cb8850cSKent Overstreet 	int nsegs = 0;
569d6d48196SJens Axboe 
570f9d03f96SChristoph Hellwig 	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
571cae6c2e5SMing Lei 		nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, &sg);
572f9d03f96SChristoph Hellwig 	else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME)
573cae6c2e5SMing Lei 		nsegs = __blk_bvec_map_sg(bio_iovec(rq->bio), sglist, &sg);
574f9d03f96SChristoph Hellwig 	else if (rq->bio)
5755cb8850cSKent Overstreet 		nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg);
576f18573abSFUJITA Tomonori 
577e8064021SChristoph Hellwig 	if (unlikely(rq->rq_flags & RQF_COPY_USER) &&
5782e46e8b2STejun Heo 	    (blk_rq_bytes(rq) & q->dma_pad_mask)) {
5792e46e8b2STejun Heo 		unsigned int pad_len =
5802e46e8b2STejun Heo 			(q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
581f18573abSFUJITA Tomonori 
582f18573abSFUJITA Tomonori 		sg->length += pad_len;
583f18573abSFUJITA Tomonori 		rq->extra_len += pad_len;
584f18573abSFUJITA Tomonori 	}
585f18573abSFUJITA Tomonori 
5862fb98e84STejun Heo 	if (q->dma_drain_size && q->dma_drain_needed(rq)) {
587a8ebb056SMike Christie 		if (op_is_write(req_op(rq)))
588db0a2e00STejun Heo 			memset(q->dma_drain_buffer, 0, q->dma_drain_size);
589db0a2e00STejun Heo 
590da81ed16SDan Williams 		sg_unmark_end(sg);
591d6d48196SJens Axboe 		sg = sg_next(sg);
592d6d48196SJens Axboe 		sg_set_page(sg, virt_to_page(q->dma_drain_buffer),
593d6d48196SJens Axboe 			    q->dma_drain_size,
594d6d48196SJens Axboe 			    ((unsigned long)q->dma_drain_buffer) &
595d6d48196SJens Axboe 			    (PAGE_SIZE - 1));
596d6d48196SJens Axboe 		nsegs++;
5977a85f889SFUJITA Tomonori 		rq->extra_len += q->dma_drain_size;
598d6d48196SJens Axboe 	}
599d6d48196SJens Axboe 
600d6d48196SJens Axboe 	if (sg)
601d6d48196SJens Axboe 		sg_mark_end(sg);
602d6d48196SJens Axboe 
60312e57f59SMing Lei 	/*
60412e57f59SMing Lei 	 * Something must have been wrong if the figured number of
60512e57f59SMing Lei 	 * segment is bigger than number of req's physical segments
60612e57f59SMing Lei 	 */
607f9d03f96SChristoph Hellwig 	WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
60812e57f59SMing Lei 
609d6d48196SJens Axboe 	return nsegs;
610d6d48196SJens Axboe }
611d6d48196SJens Axboe EXPORT_SYMBOL(blk_rq_map_sg);
612d6d48196SJens Axboe 
613d6d48196SJens Axboe static inline int ll_new_hw_segment(struct request_queue *q,
614d6d48196SJens Axboe 				    struct request *req,
615d6d48196SJens Axboe 				    struct bio *bio)
616d6d48196SJens Axboe {
617d6d48196SJens Axboe 	int nr_phys_segs = bio_phys_segments(q, bio);
618d6d48196SJens Axboe 
61913f05c8dSMartin K. Petersen 	if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q))
62013f05c8dSMartin K. Petersen 		goto no_merge;
62113f05c8dSMartin K. Petersen 
6224eaf99beSMartin K. Petersen 	if (blk_integrity_merge_bio(q, req, bio) == false)
62313f05c8dSMartin K. Petersen 		goto no_merge;
624d6d48196SJens Axboe 
625d6d48196SJens Axboe 	/*
626d6d48196SJens Axboe 	 * This will form the start of a new hw segment.  Bump both
627d6d48196SJens Axboe 	 * counters.
628d6d48196SJens Axboe 	 */
629d6d48196SJens Axboe 	req->nr_phys_segments += nr_phys_segs;
630d6d48196SJens Axboe 	return 1;
63113f05c8dSMartin K. Petersen 
63213f05c8dSMartin K. Petersen no_merge:
633e0c72300SRitesh Harjani 	req_set_nomerge(q, req);
63413f05c8dSMartin K. Petersen 	return 0;
635d6d48196SJens Axboe }
636d6d48196SJens Axboe 
637d6d48196SJens Axboe int ll_back_merge_fn(struct request_queue *q, struct request *req,
638d6d48196SJens Axboe 		     struct bio *bio)
639d6d48196SJens Axboe {
6405e7c4274SJens Axboe 	if (req_gap_back_merge(req, bio))
6415e7c4274SJens Axboe 		return 0;
6427f39add3SSagi Grimberg 	if (blk_integrity_rq(req) &&
6437f39add3SSagi Grimberg 	    integrity_req_gap_back_merge(req, bio))
6447f39add3SSagi Grimberg 		return 0;
645f31dc1cdSMartin K. Petersen 	if (blk_rq_sectors(req) + bio_sectors(bio) >
64617007f39SDamien Le Moal 	    blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
647e0c72300SRitesh Harjani 		req_set_nomerge(q, req);
648d6d48196SJens Axboe 		return 0;
649d6d48196SJens Axboe 	}
6502cdf79caSJens Axboe 	if (!bio_flagged(req->biotail, BIO_SEG_VALID))
651d6d48196SJens Axboe 		blk_recount_segments(q, req->biotail);
6522cdf79caSJens Axboe 	if (!bio_flagged(bio, BIO_SEG_VALID))
653d6d48196SJens Axboe 		blk_recount_segments(q, bio);
654d6d48196SJens Axboe 
655d6d48196SJens Axboe 	return ll_new_hw_segment(q, req, bio);
656d6d48196SJens Axboe }
657d6d48196SJens Axboe 
658d6d48196SJens Axboe int ll_front_merge_fn(struct request_queue *q, struct request *req,
659d6d48196SJens Axboe 		      struct bio *bio)
660d6d48196SJens Axboe {
6615e7c4274SJens Axboe 
6625e7c4274SJens Axboe 	if (req_gap_front_merge(req, bio))
6635e7c4274SJens Axboe 		return 0;
6647f39add3SSagi Grimberg 	if (blk_integrity_rq(req) &&
6657f39add3SSagi Grimberg 	    integrity_req_gap_front_merge(req, bio))
6667f39add3SSagi Grimberg 		return 0;
667f31dc1cdSMartin K. Petersen 	if (blk_rq_sectors(req) + bio_sectors(bio) >
66817007f39SDamien Le Moal 	    blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
669e0c72300SRitesh Harjani 		req_set_nomerge(q, req);
670d6d48196SJens Axboe 		return 0;
671d6d48196SJens Axboe 	}
6722cdf79caSJens Axboe 	if (!bio_flagged(bio, BIO_SEG_VALID))
673d6d48196SJens Axboe 		blk_recount_segments(q, bio);
6742cdf79caSJens Axboe 	if (!bio_flagged(req->bio, BIO_SEG_VALID))
675d6d48196SJens Axboe 		blk_recount_segments(q, req->bio);
676d6d48196SJens Axboe 
677d6d48196SJens Axboe 	return ll_new_hw_segment(q, req, bio);
678d6d48196SJens Axboe }
679d6d48196SJens Axboe 
680445251d0SJens Axboe static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
681445251d0SJens Axboe 		struct request *next)
682445251d0SJens Axboe {
683445251d0SJens Axboe 	unsigned short segments = blk_rq_nr_discard_segments(req);
684445251d0SJens Axboe 
685445251d0SJens Axboe 	if (segments >= queue_max_discard_segments(q))
686445251d0SJens Axboe 		goto no_merge;
687445251d0SJens Axboe 	if (blk_rq_sectors(req) + bio_sectors(next->bio) >
688445251d0SJens Axboe 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
689445251d0SJens Axboe 		goto no_merge;
690445251d0SJens Axboe 
691445251d0SJens Axboe 	req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
692445251d0SJens Axboe 	return true;
693445251d0SJens Axboe no_merge:
694445251d0SJens Axboe 	req_set_nomerge(q, req);
695445251d0SJens Axboe 	return false;
696445251d0SJens Axboe }
697445251d0SJens Axboe 
698d6d48196SJens Axboe static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
699d6d48196SJens Axboe 				struct request *next)
700d6d48196SJens Axboe {
701d6d48196SJens Axboe 	int total_phys_segments;
70286771427SFUJITA Tomonori 	unsigned int seg_size =
70386771427SFUJITA Tomonori 		req->biotail->bi_seg_back_size + next->bio->bi_seg_front_size;
704d6d48196SJens Axboe 
7055e7c4274SJens Axboe 	if (req_gap_back_merge(req, next->bio))
706854fbb9cSKeith Busch 		return 0;
707854fbb9cSKeith Busch 
708d6d48196SJens Axboe 	/*
709d6d48196SJens Axboe 	 * Will it become too large?
710d6d48196SJens Axboe 	 */
711f31dc1cdSMartin K. Petersen 	if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
71217007f39SDamien Le Moal 	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
713d6d48196SJens Axboe 		return 0;
714d6d48196SJens Axboe 
715d6d48196SJens Axboe 	total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
71686771427SFUJITA Tomonori 	if (blk_phys_contig_segment(q, req->biotail, next->bio)) {
71786771427SFUJITA Tomonori 		if (req->nr_phys_segments == 1)
71886771427SFUJITA Tomonori 			req->bio->bi_seg_front_size = seg_size;
71986771427SFUJITA Tomonori 		if (next->nr_phys_segments == 1)
72086771427SFUJITA Tomonori 			next->biotail->bi_seg_back_size = seg_size;
721d6d48196SJens Axboe 		total_phys_segments--;
72286771427SFUJITA Tomonori 	}
723d6d48196SJens Axboe 
7248a78362cSMartin K. Petersen 	if (total_phys_segments > queue_max_segments(q))
725d6d48196SJens Axboe 		return 0;
726d6d48196SJens Axboe 
7274eaf99beSMartin K. Petersen 	if (blk_integrity_merge_rq(q, req, next) == false)
72813f05c8dSMartin K. Petersen 		return 0;
72913f05c8dSMartin K. Petersen 
730d6d48196SJens Axboe 	/* Merge is OK... */
731d6d48196SJens Axboe 	req->nr_phys_segments = total_phys_segments;
732d6d48196SJens Axboe 	return 1;
733d6d48196SJens Axboe }
734d6d48196SJens Axboe 
73580a761fdSTejun Heo /**
73680a761fdSTejun Heo  * blk_rq_set_mixed_merge - mark a request as mixed merge
73780a761fdSTejun Heo  * @rq: request to mark as mixed merge
73880a761fdSTejun Heo  *
73980a761fdSTejun Heo  * Description:
74080a761fdSTejun Heo  *     @rq is about to be mixed merged.  Make sure the attributes
74180a761fdSTejun Heo  *     which can be mixed are set in each bio and mark @rq as mixed
74280a761fdSTejun Heo  *     merged.
74380a761fdSTejun Heo  */
74480a761fdSTejun Heo void blk_rq_set_mixed_merge(struct request *rq)
74580a761fdSTejun Heo {
74680a761fdSTejun Heo 	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
74780a761fdSTejun Heo 	struct bio *bio;
74880a761fdSTejun Heo 
749e8064021SChristoph Hellwig 	if (rq->rq_flags & RQF_MIXED_MERGE)
75080a761fdSTejun Heo 		return;
75180a761fdSTejun Heo 
75280a761fdSTejun Heo 	/*
75380a761fdSTejun Heo 	 * @rq will no longer represent mixable attributes for all the
75480a761fdSTejun Heo 	 * contained bios.  It will just track those of the first one.
75580a761fdSTejun Heo 	 * Distributes the attributs to each bio.
75680a761fdSTejun Heo 	 */
75780a761fdSTejun Heo 	for (bio = rq->bio; bio; bio = bio->bi_next) {
7581eff9d32SJens Axboe 		WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
7591eff9d32SJens Axboe 			     (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
7601eff9d32SJens Axboe 		bio->bi_opf |= ff;
76180a761fdSTejun Heo 	}
762e8064021SChristoph Hellwig 	rq->rq_flags |= RQF_MIXED_MERGE;
76380a761fdSTejun Heo }
76480a761fdSTejun Heo 
76526308eabSJerome Marchand static void blk_account_io_merge(struct request *req)
76626308eabSJerome Marchand {
76726308eabSJerome Marchand 	if (blk_do_io_stat(req)) {
76826308eabSJerome Marchand 		struct hd_struct *part;
76926308eabSJerome Marchand 
770112f158fSMike Snitzer 		part_stat_lock();
77109e099d4SJerome Marchand 		part = req->part;
77226308eabSJerome Marchand 
773d62e26b3SJens Axboe 		part_dec_in_flight(req->q, part, rq_data_dir(req));
77426308eabSJerome Marchand 
7756c23a968SJens Axboe 		hd_struct_put(part);
77626308eabSJerome Marchand 		part_stat_unlock();
77726308eabSJerome Marchand 	}
77826308eabSJerome Marchand }
77969840466SJianchao Wang /*
78069840466SJianchao Wang  * Two cases of handling DISCARD merge:
78169840466SJianchao Wang  * If max_discard_segments > 1, the driver takes every bio
78269840466SJianchao Wang  * as a range and send them to controller together. The ranges
78369840466SJianchao Wang  * needn't to be contiguous.
78469840466SJianchao Wang  * Otherwise, the bios/requests will be handled as same as
78569840466SJianchao Wang  * others which should be contiguous.
78669840466SJianchao Wang  */
78769840466SJianchao Wang static inline bool blk_discard_mergable(struct request *req)
78869840466SJianchao Wang {
78969840466SJianchao Wang 	if (req_op(req) == REQ_OP_DISCARD &&
79069840466SJianchao Wang 	    queue_max_discard_segments(req->q) > 1)
79169840466SJianchao Wang 		return true;
79269840466SJianchao Wang 	return false;
79369840466SJianchao Wang }
79469840466SJianchao Wang 
795e96c0d83SEric Biggers static enum elv_merge blk_try_req_merge(struct request *req,
796e96c0d83SEric Biggers 					struct request *next)
79769840466SJianchao Wang {
79869840466SJianchao Wang 	if (blk_discard_mergable(req))
79969840466SJianchao Wang 		return ELEVATOR_DISCARD_MERGE;
80069840466SJianchao Wang 	else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
80169840466SJianchao Wang 		return ELEVATOR_BACK_MERGE;
80269840466SJianchao Wang 
80369840466SJianchao Wang 	return ELEVATOR_NO_MERGE;
80469840466SJianchao Wang }
80526308eabSJerome Marchand 
806d6d48196SJens Axboe /*
807b973cb7eSJens Axboe  * For non-mq, this has to be called with the request spinlock acquired.
808b973cb7eSJens Axboe  * For mq with scheduling, the appropriate queue wide lock should be held.
809d6d48196SJens Axboe  */
810b973cb7eSJens Axboe static struct request *attempt_merge(struct request_queue *q,
811b973cb7eSJens Axboe 				     struct request *req, struct request *next)
812d6d48196SJens Axboe {
813d6d48196SJens Axboe 	if (!rq_mergeable(req) || !rq_mergeable(next))
814b973cb7eSJens Axboe 		return NULL;
815d6d48196SJens Axboe 
816288dab8aSChristoph Hellwig 	if (req_op(req) != req_op(next))
817b973cb7eSJens Axboe 		return NULL;
818f31dc1cdSMartin K. Petersen 
819d6d48196SJens Axboe 	if (rq_data_dir(req) != rq_data_dir(next)
8202081a56bSJens Axboe 	    || req->rq_disk != next->rq_disk)
821b973cb7eSJens Axboe 		return NULL;
822d6d48196SJens Axboe 
8238fe0d473SMike Christie 	if (req_op(req) == REQ_OP_WRITE_SAME &&
8244363ac7cSMartin K. Petersen 	    !blk_write_same_mergeable(req->bio, next->bio))
825b973cb7eSJens Axboe 		return NULL;
8264363ac7cSMartin K. Petersen 
827d6d48196SJens Axboe 	/*
828cb6934f8SJens Axboe 	 * Don't allow merge of different write hints, or for a hint with
829cb6934f8SJens Axboe 	 * non-hint IO.
830cb6934f8SJens Axboe 	 */
831cb6934f8SJens Axboe 	if (req->write_hint != next->write_hint)
832cb6934f8SJens Axboe 		return NULL;
833cb6934f8SJens Axboe 
834668ffc03SDamien Le Moal 	if (req->ioprio != next->ioprio)
835668ffc03SDamien Le Moal 		return NULL;
836668ffc03SDamien Le Moal 
837cb6934f8SJens Axboe 	/*
838d6d48196SJens Axboe 	 * If we are allowed to merge, then append bio list
839d6d48196SJens Axboe 	 * from next to rq and release next. merge_requests_fn
840d6d48196SJens Axboe 	 * will have updated segment counts, update sector
841445251d0SJens Axboe 	 * counts here. Handle DISCARDs separately, as they
842445251d0SJens Axboe 	 * have separate settings.
843d6d48196SJens Axboe 	 */
84469840466SJianchao Wang 
84569840466SJianchao Wang 	switch (blk_try_req_merge(req, next)) {
84669840466SJianchao Wang 	case ELEVATOR_DISCARD_MERGE:
847445251d0SJens Axboe 		if (!req_attempt_discard_merge(q, req, next))
848445251d0SJens Axboe 			return NULL;
84969840466SJianchao Wang 		break;
85069840466SJianchao Wang 	case ELEVATOR_BACK_MERGE:
85169840466SJianchao Wang 		if (!ll_merge_requests_fn(q, req, next))
852b973cb7eSJens Axboe 			return NULL;
85369840466SJianchao Wang 		break;
85469840466SJianchao Wang 	default:
85569840466SJianchao Wang 		return NULL;
85669840466SJianchao Wang 	}
857d6d48196SJens Axboe 
858d6d48196SJens Axboe 	/*
85980a761fdSTejun Heo 	 * If failfast settings disagree or any of the two is already
86080a761fdSTejun Heo 	 * a mixed merge, mark both as mixed before proceeding.  This
86180a761fdSTejun Heo 	 * makes sure that all involved bios have mixable attributes
86280a761fdSTejun Heo 	 * set properly.
86380a761fdSTejun Heo 	 */
864e8064021SChristoph Hellwig 	if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
86580a761fdSTejun Heo 	    (req->cmd_flags & REQ_FAILFAST_MASK) !=
86680a761fdSTejun Heo 	    (next->cmd_flags & REQ_FAILFAST_MASK)) {
86780a761fdSTejun Heo 		blk_rq_set_mixed_merge(req);
86880a761fdSTejun Heo 		blk_rq_set_mixed_merge(next);
86980a761fdSTejun Heo 	}
87080a761fdSTejun Heo 
87180a761fdSTejun Heo 	/*
872522a7775SOmar Sandoval 	 * At this point we have either done a back merge or front merge. We
873522a7775SOmar Sandoval 	 * need the smaller start_time_ns of the merged requests to be the
874522a7775SOmar Sandoval 	 * current request for accounting purposes.
875d6d48196SJens Axboe 	 */
876522a7775SOmar Sandoval 	if (next->start_time_ns < req->start_time_ns)
877522a7775SOmar Sandoval 		req->start_time_ns = next->start_time_ns;
878d6d48196SJens Axboe 
879d6d48196SJens Axboe 	req->biotail->bi_next = next->bio;
880d6d48196SJens Axboe 	req->biotail = next->biotail;
881d6d48196SJens Axboe 
882a2dec7b3STejun Heo 	req->__data_len += blk_rq_bytes(next);
883d6d48196SJens Axboe 
8842a5cf35cSMing Lei 	if (!blk_discard_mergable(req))
885d6d48196SJens Axboe 		elv_merge_requests(q, req, next);
886d6d48196SJens Axboe 
88742dad764SJerome Marchand 	/*
88842dad764SJerome Marchand 	 * 'next' is going away, so update stats accordingly
88942dad764SJerome Marchand 	 */
89042dad764SJerome Marchand 	blk_account_io_merge(next);
891d6d48196SJens Axboe 
892e4d750c9SJens Axboe 	/*
893e4d750c9SJens Axboe 	 * ownership of bio passed from next to req, return 'next' for
894e4d750c9SJens Axboe 	 * the caller to free
895e4d750c9SJens Axboe 	 */
8961cd96c24SBoaz Harrosh 	next->bio = NULL;
897b973cb7eSJens Axboe 	return next;
898d6d48196SJens Axboe }
899d6d48196SJens Axboe 
900b973cb7eSJens Axboe struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
901d6d48196SJens Axboe {
902d6d48196SJens Axboe 	struct request *next = elv_latter_request(q, rq);
903d6d48196SJens Axboe 
904d6d48196SJens Axboe 	if (next)
905d6d48196SJens Axboe 		return attempt_merge(q, rq, next);
906d6d48196SJens Axboe 
907b973cb7eSJens Axboe 	return NULL;
908d6d48196SJens Axboe }
909d6d48196SJens Axboe 
910b973cb7eSJens Axboe struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
911d6d48196SJens Axboe {
912d6d48196SJens Axboe 	struct request *prev = elv_former_request(q, rq);
913d6d48196SJens Axboe 
914d6d48196SJens Axboe 	if (prev)
915d6d48196SJens Axboe 		return attempt_merge(q, prev, rq);
916d6d48196SJens Axboe 
917b973cb7eSJens Axboe 	return NULL;
918d6d48196SJens Axboe }
9195e84ea3aSJens Axboe 
9205e84ea3aSJens Axboe int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
9215e84ea3aSJens Axboe 			  struct request *next)
9225e84ea3aSJens Axboe {
923e4d750c9SJens Axboe 	struct request *free;
92472ef799bSTahsin Erdogan 
925e4d750c9SJens Axboe 	free = attempt_merge(q, rq, next);
926e4d750c9SJens Axboe 	if (free) {
92792bc5a24SJens Axboe 		blk_put_request(free);
928e4d750c9SJens Axboe 		return 1;
929e4d750c9SJens Axboe 	}
930e4d750c9SJens Axboe 
931e4d750c9SJens Axboe 	return 0;
9325e84ea3aSJens Axboe }
933050c8ea8STejun Heo 
934050c8ea8STejun Heo bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
935050c8ea8STejun Heo {
936e2a60da7SMartin K. Petersen 	if (!rq_mergeable(rq) || !bio_mergeable(bio))
937050c8ea8STejun Heo 		return false;
938050c8ea8STejun Heo 
939288dab8aSChristoph Hellwig 	if (req_op(rq) != bio_op(bio))
940f31dc1cdSMartin K. Petersen 		return false;
941f31dc1cdSMartin K. Petersen 
942050c8ea8STejun Heo 	/* different data direction or already started, don't merge */
943050c8ea8STejun Heo 	if (bio_data_dir(bio) != rq_data_dir(rq))
944050c8ea8STejun Heo 		return false;
945050c8ea8STejun Heo 
9462081a56bSJens Axboe 	/* must be same device */
9472081a56bSJens Axboe 	if (rq->rq_disk != bio->bi_disk)
948050c8ea8STejun Heo 		return false;
949050c8ea8STejun Heo 
950050c8ea8STejun Heo 	/* only merge integrity protected bio into ditto rq */
9514eaf99beSMartin K. Petersen 	if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
952050c8ea8STejun Heo 		return false;
953050c8ea8STejun Heo 
9544363ac7cSMartin K. Petersen 	/* must be using the same buffer */
9558fe0d473SMike Christie 	if (req_op(rq) == REQ_OP_WRITE_SAME &&
9564363ac7cSMartin K. Petersen 	    !blk_write_same_mergeable(rq->bio, bio))
9574363ac7cSMartin K. Petersen 		return false;
9584363ac7cSMartin K. Petersen 
959cb6934f8SJens Axboe 	/*
960cb6934f8SJens Axboe 	 * Don't allow merge of different write hints, or for a hint with
961cb6934f8SJens Axboe 	 * non-hint IO.
962cb6934f8SJens Axboe 	 */
963cb6934f8SJens Axboe 	if (rq->write_hint != bio->bi_write_hint)
964cb6934f8SJens Axboe 		return false;
965cb6934f8SJens Axboe 
966668ffc03SDamien Le Moal 	if (rq->ioprio != bio_prio(bio))
967668ffc03SDamien Le Moal 		return false;
968668ffc03SDamien Le Moal 
969050c8ea8STejun Heo 	return true;
970050c8ea8STejun Heo }
971050c8ea8STejun Heo 
97234fe7c05SChristoph Hellwig enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
973050c8ea8STejun Heo {
97469840466SJianchao Wang 	if (blk_discard_mergable(rq))
9751e739730SChristoph Hellwig 		return ELEVATOR_DISCARD_MERGE;
9761e739730SChristoph Hellwig 	else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
977050c8ea8STejun Heo 		return ELEVATOR_BACK_MERGE;
9784f024f37SKent Overstreet 	else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
979050c8ea8STejun Heo 		return ELEVATOR_FRONT_MERGE;
980050c8ea8STejun Heo 	return ELEVATOR_NO_MERGE;
981050c8ea8STejun Heo }
982