1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 2d6d48196SJens Axboe /* 3d6d48196SJens Axboe * Functions related to segment and merge handling 4d6d48196SJens Axboe */ 5d6d48196SJens Axboe #include <linux/kernel.h> 6d6d48196SJens Axboe #include <linux/module.h> 7d6d48196SJens Axboe #include <linux/bio.h> 8d6d48196SJens Axboe #include <linux/blkdev.h> 9d6d48196SJens Axboe #include <linux/scatterlist.h> 10d6d48196SJens Axboe 11cda22646SMike Krinkin #include <trace/events/block.h> 12cda22646SMike Krinkin 13d6d48196SJens Axboe #include "blk.h" 14d6d48196SJens Axboe 15e9907009SChristoph Hellwig /* 16e9907009SChristoph Hellwig * Check if the two bvecs from two bios can be merged to one segment. If yes, 17e9907009SChristoph Hellwig * no need to check gap between the two bios since the 1st bio and the 1st bvec 18e9907009SChristoph Hellwig * in the 2nd bio can be handled in one segment. 19e9907009SChristoph Hellwig */ 20e9907009SChristoph Hellwig static inline bool bios_segs_mergeable(struct request_queue *q, 21e9907009SChristoph Hellwig struct bio *prev, struct bio_vec *prev_last_bv, 22e9907009SChristoph Hellwig struct bio_vec *next_first_bv) 23e9907009SChristoph Hellwig { 243dccdae5SChristoph Hellwig if (!biovec_phys_mergeable(q, prev_last_bv, next_first_bv)) 25e9907009SChristoph Hellwig return false; 26e9907009SChristoph Hellwig if (prev->bi_seg_back_size + next_first_bv->bv_len > 27e9907009SChristoph Hellwig queue_max_segment_size(q)) 28e9907009SChristoph Hellwig return false; 29e9907009SChristoph Hellwig return true; 30e9907009SChristoph Hellwig } 31e9907009SChristoph Hellwig 32e9907009SChristoph Hellwig static inline bool bio_will_gap(struct request_queue *q, 33e9907009SChristoph Hellwig struct request *prev_rq, struct bio *prev, struct bio *next) 34e9907009SChristoph Hellwig { 35e9907009SChristoph Hellwig struct bio_vec pb, nb; 36e9907009SChristoph Hellwig 37e9907009SChristoph Hellwig if (!bio_has_data(prev) || !queue_virt_boundary(q)) 38e9907009SChristoph Hellwig return false; 39e9907009SChristoph Hellwig 40e9907009SChristoph Hellwig /* 41e9907009SChristoph Hellwig * Don't merge if the 1st bio starts with non-zero offset, otherwise it 42e9907009SChristoph Hellwig * is quite difficult to respect the sg gap limit. We work hard to 43e9907009SChristoph Hellwig * merge a huge number of small single bios in case of mkfs. 44e9907009SChristoph Hellwig */ 45e9907009SChristoph Hellwig if (prev_rq) 46e9907009SChristoph Hellwig bio_get_first_bvec(prev_rq->bio, &pb); 47e9907009SChristoph Hellwig else 48e9907009SChristoph Hellwig bio_get_first_bvec(prev, &pb); 49df376b2eSJohannes Thumshirn if (pb.bv_offset & queue_virt_boundary(q)) 50e9907009SChristoph Hellwig return true; 51e9907009SChristoph Hellwig 52e9907009SChristoph Hellwig /* 53e9907009SChristoph Hellwig * We don't need to worry about the situation that the merged segment 54e9907009SChristoph Hellwig * ends in unaligned virt boundary: 55e9907009SChristoph Hellwig * 56e9907009SChristoph Hellwig * - if 'pb' ends aligned, the merged segment ends aligned 57e9907009SChristoph Hellwig * - if 'pb' ends unaligned, the next bio must include 58e9907009SChristoph Hellwig * one single bvec of 'nb', otherwise the 'nb' can't 59e9907009SChristoph Hellwig * merge with 'pb' 60e9907009SChristoph Hellwig */ 61e9907009SChristoph Hellwig bio_get_last_bvec(prev, &pb); 62e9907009SChristoph Hellwig bio_get_first_bvec(next, &nb); 63e9907009SChristoph Hellwig if (bios_segs_mergeable(q, prev, &pb, &nb)) 64e9907009SChristoph Hellwig return false; 65e9907009SChristoph Hellwig return __bvec_gap_to_prev(q, &pb, nb.bv_offset); 66e9907009SChristoph Hellwig } 67e9907009SChristoph Hellwig 68e9907009SChristoph Hellwig static inline bool req_gap_back_merge(struct request *req, struct bio *bio) 69e9907009SChristoph Hellwig { 70e9907009SChristoph Hellwig return bio_will_gap(req->q, req, req->biotail, bio); 71e9907009SChristoph Hellwig } 72e9907009SChristoph Hellwig 73e9907009SChristoph Hellwig static inline bool req_gap_front_merge(struct request *req, struct bio *bio) 74e9907009SChristoph Hellwig { 75e9907009SChristoph Hellwig return bio_will_gap(req->q, NULL, bio, req->bio); 76e9907009SChristoph Hellwig } 77e9907009SChristoph Hellwig 7854efd50bSKent Overstreet static struct bio *blk_bio_discard_split(struct request_queue *q, 7954efd50bSKent Overstreet struct bio *bio, 80bdced438SMing Lei struct bio_set *bs, 81bdced438SMing Lei unsigned *nsegs) 8254efd50bSKent Overstreet { 8354efd50bSKent Overstreet unsigned int max_discard_sectors, granularity; 8454efd50bSKent Overstreet int alignment; 8554efd50bSKent Overstreet sector_t tmp; 8654efd50bSKent Overstreet unsigned split_sectors; 8754efd50bSKent Overstreet 88bdced438SMing Lei *nsegs = 1; 89bdced438SMing Lei 9054efd50bSKent Overstreet /* Zero-sector (unknown) and one-sector granularities are the same. */ 9154efd50bSKent Overstreet granularity = max(q->limits.discard_granularity >> 9, 1U); 9254efd50bSKent Overstreet 931adfc5e4SMing Lei max_discard_sectors = min(q->limits.max_discard_sectors, 941adfc5e4SMing Lei bio_allowed_max_sectors(q)); 9554efd50bSKent Overstreet max_discard_sectors -= max_discard_sectors % granularity; 9654efd50bSKent Overstreet 9754efd50bSKent Overstreet if (unlikely(!max_discard_sectors)) { 9854efd50bSKent Overstreet /* XXX: warn */ 9954efd50bSKent Overstreet return NULL; 10054efd50bSKent Overstreet } 10154efd50bSKent Overstreet 10254efd50bSKent Overstreet if (bio_sectors(bio) <= max_discard_sectors) 10354efd50bSKent Overstreet return NULL; 10454efd50bSKent Overstreet 10554efd50bSKent Overstreet split_sectors = max_discard_sectors; 10654efd50bSKent Overstreet 10754efd50bSKent Overstreet /* 10854efd50bSKent Overstreet * If the next starting sector would be misaligned, stop the discard at 10954efd50bSKent Overstreet * the previous aligned sector. 11054efd50bSKent Overstreet */ 11154efd50bSKent Overstreet alignment = (q->limits.discard_alignment >> 9) % granularity; 11254efd50bSKent Overstreet 11354efd50bSKent Overstreet tmp = bio->bi_iter.bi_sector + split_sectors - alignment; 11454efd50bSKent Overstreet tmp = sector_div(tmp, granularity); 11554efd50bSKent Overstreet 11654efd50bSKent Overstreet if (split_sectors > tmp) 11754efd50bSKent Overstreet split_sectors -= tmp; 11854efd50bSKent Overstreet 11954efd50bSKent Overstreet return bio_split(bio, split_sectors, GFP_NOIO, bs); 12054efd50bSKent Overstreet } 12154efd50bSKent Overstreet 122885fa13fSChristoph Hellwig static struct bio *blk_bio_write_zeroes_split(struct request_queue *q, 123885fa13fSChristoph Hellwig struct bio *bio, struct bio_set *bs, unsigned *nsegs) 124885fa13fSChristoph Hellwig { 125885fa13fSChristoph Hellwig *nsegs = 1; 126885fa13fSChristoph Hellwig 127885fa13fSChristoph Hellwig if (!q->limits.max_write_zeroes_sectors) 128885fa13fSChristoph Hellwig return NULL; 129885fa13fSChristoph Hellwig 130885fa13fSChristoph Hellwig if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors) 131885fa13fSChristoph Hellwig return NULL; 132885fa13fSChristoph Hellwig 133885fa13fSChristoph Hellwig return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs); 134885fa13fSChristoph Hellwig } 135885fa13fSChristoph Hellwig 13654efd50bSKent Overstreet static struct bio *blk_bio_write_same_split(struct request_queue *q, 13754efd50bSKent Overstreet struct bio *bio, 138bdced438SMing Lei struct bio_set *bs, 139bdced438SMing Lei unsigned *nsegs) 14054efd50bSKent Overstreet { 141bdced438SMing Lei *nsegs = 1; 142bdced438SMing Lei 14354efd50bSKent Overstreet if (!q->limits.max_write_same_sectors) 14454efd50bSKent Overstreet return NULL; 14554efd50bSKent Overstreet 14654efd50bSKent Overstreet if (bio_sectors(bio) <= q->limits.max_write_same_sectors) 14754efd50bSKent Overstreet return NULL; 14854efd50bSKent Overstreet 14954efd50bSKent Overstreet return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs); 15054efd50bSKent Overstreet } 15154efd50bSKent Overstreet 152d0e5fbb0SMing Lei static inline unsigned get_max_io_size(struct request_queue *q, 153d0e5fbb0SMing Lei struct bio *bio) 154d0e5fbb0SMing Lei { 155d0e5fbb0SMing Lei unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); 156d0e5fbb0SMing Lei unsigned mask = queue_logical_block_size(q) - 1; 157d0e5fbb0SMing Lei 158d0e5fbb0SMing Lei /* aligned to logical block size */ 159d0e5fbb0SMing Lei sectors &= ~(mask >> 9); 160d0e5fbb0SMing Lei 161d0e5fbb0SMing Lei return sectors; 162d0e5fbb0SMing Lei } 163d0e5fbb0SMing Lei 164dcebd755SMing Lei static unsigned get_max_segment_size(struct request_queue *q, 165dcebd755SMing Lei unsigned offset) 166dcebd755SMing Lei { 167dcebd755SMing Lei unsigned long mask = queue_segment_boundary(q); 168dcebd755SMing Lei 169dcebd755SMing Lei /* default segment boundary mask means no boundary limit */ 170dcebd755SMing Lei if (mask == BLK_SEG_BOUNDARY_MASK) 171dcebd755SMing Lei return queue_max_segment_size(q); 172dcebd755SMing Lei 173dcebd755SMing Lei return min_t(unsigned long, mask - (mask & offset) + 1, 174dcebd755SMing Lei queue_max_segment_size(q)); 175dcebd755SMing Lei } 176dcebd755SMing Lei 177dcebd755SMing Lei /* 178dcebd755SMing Lei * Split the bvec @bv into segments, and update all kinds of 179dcebd755SMing Lei * variables. 180dcebd755SMing Lei */ 181dcebd755SMing Lei static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv, 182dcebd755SMing Lei unsigned *nsegs, unsigned *last_seg_size, 183dcebd755SMing Lei unsigned *front_seg_size, unsigned *sectors) 184dcebd755SMing Lei { 185dcebd755SMing Lei unsigned len = bv->bv_len; 186dcebd755SMing Lei unsigned total_len = 0; 187dcebd755SMing Lei unsigned new_nsegs = 0, seg_size = 0; 188dcebd755SMing Lei 189dcebd755SMing Lei /* 190dcebd755SMing Lei * Multi-page bvec may be too big to hold in one segment, so the 191dcebd755SMing Lei * current bvec has to be splitted as multiple segments. 192dcebd755SMing Lei */ 193dcebd755SMing Lei while (len && new_nsegs + *nsegs < queue_max_segments(q)) { 194dcebd755SMing Lei seg_size = get_max_segment_size(q, bv->bv_offset + total_len); 195dcebd755SMing Lei seg_size = min(seg_size, len); 196dcebd755SMing Lei 197dcebd755SMing Lei new_nsegs++; 198dcebd755SMing Lei total_len += seg_size; 199dcebd755SMing Lei len -= seg_size; 200dcebd755SMing Lei 201dcebd755SMing Lei if ((bv->bv_offset + total_len) & queue_virt_boundary(q)) 202dcebd755SMing Lei break; 203dcebd755SMing Lei } 204dcebd755SMing Lei 205dcebd755SMing Lei if (!new_nsegs) 206dcebd755SMing Lei return !!len; 207dcebd755SMing Lei 208dcebd755SMing Lei /* update front segment size */ 209dcebd755SMing Lei if (!*nsegs) { 210dcebd755SMing Lei unsigned first_seg_size; 211dcebd755SMing Lei 212dcebd755SMing Lei if (new_nsegs == 1) 213dcebd755SMing Lei first_seg_size = get_max_segment_size(q, bv->bv_offset); 214dcebd755SMing Lei else 215dcebd755SMing Lei first_seg_size = queue_max_segment_size(q); 216dcebd755SMing Lei 217dcebd755SMing Lei if (*front_seg_size < first_seg_size) 218dcebd755SMing Lei *front_seg_size = first_seg_size; 219dcebd755SMing Lei } 220dcebd755SMing Lei 221dcebd755SMing Lei /* update other varibles */ 222dcebd755SMing Lei *last_seg_size = seg_size; 223dcebd755SMing Lei *nsegs += new_nsegs; 224dcebd755SMing Lei if (sectors) 225dcebd755SMing Lei *sectors += total_len >> 9; 226dcebd755SMing Lei 227dcebd755SMing Lei /* split in the middle of the bvec if len != 0 */ 228dcebd755SMing Lei return !!len; 229dcebd755SMing Lei } 230dcebd755SMing Lei 23154efd50bSKent Overstreet static struct bio *blk_bio_segment_split(struct request_queue *q, 23254efd50bSKent Overstreet struct bio *bio, 233bdced438SMing Lei struct bio_set *bs, 234bdced438SMing Lei unsigned *segs) 23554efd50bSKent Overstreet { 2365014c311SJens Axboe struct bio_vec bv, bvprv, *bvprvp = NULL; 23754efd50bSKent Overstreet struct bvec_iter iter; 2388ae12666SKent Overstreet unsigned seg_size = 0, nsegs = 0, sectors = 0; 23902e70742SMing Lei unsigned front_seg_size = bio->bi_seg_front_size; 24002e70742SMing Lei bool do_split = true; 24102e70742SMing Lei struct bio *new = NULL; 242d0e5fbb0SMing Lei const unsigned max_sectors = get_max_io_size(q, bio); 24354efd50bSKent Overstreet 244dcebd755SMing Lei bio_for_each_bvec(bv, bio, iter) { 24554efd50bSKent Overstreet /* 24654efd50bSKent Overstreet * If the queue doesn't support SG gaps and adding this 24754efd50bSKent Overstreet * offset would create a gap, disallow it. 24854efd50bSKent Overstreet */ 2495014c311SJens Axboe if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) 25054efd50bSKent Overstreet goto split; 25154efd50bSKent Overstreet 252d0e5fbb0SMing Lei if (sectors + (bv.bv_len >> 9) > max_sectors) { 253e36f6204SKeith Busch /* 254e36f6204SKeith Busch * Consider this a new segment if we're splitting in 255e36f6204SKeith Busch * the middle of this vector. 256e36f6204SKeith Busch */ 257e36f6204SKeith Busch if (nsegs < queue_max_segments(q) && 258d0e5fbb0SMing Lei sectors < max_sectors) { 259dcebd755SMing Lei /* split in the middle of bvec */ 260dcebd755SMing Lei bv.bv_len = (max_sectors - sectors) << 9; 261dcebd755SMing Lei bvec_split_segs(q, &bv, &nsegs, 262dcebd755SMing Lei &seg_size, 263dcebd755SMing Lei &front_seg_size, 264dcebd755SMing Lei §ors); 265e36f6204SKeith Busch } 266e36f6204SKeith Busch goto split; 267e36f6204SKeith Busch } 268e36f6204SKeith Busch 26938417468SChristoph Hellwig if (bvprvp) { 270b4b6cb61SMing Lei if (seg_size + bv.bv_len > queue_max_segment_size(q)) 271b4b6cb61SMing Lei goto new_segment; 2723dccdae5SChristoph Hellwig if (!biovec_phys_mergeable(q, bvprvp, &bv)) 27354efd50bSKent Overstreet goto new_segment; 27454efd50bSKent Overstreet 27554efd50bSKent Overstreet seg_size += bv.bv_len; 27654efd50bSKent Overstreet bvprv = bv; 277578270bfSMing Lei bvprvp = &bvprv; 27852cc6eeaSMing Lei sectors += bv.bv_len >> 9; 279a88d32afSMing Lei 28054efd50bSKent Overstreet continue; 28154efd50bSKent Overstreet } 28254efd50bSKent Overstreet new_segment: 28354efd50bSKent Overstreet if (nsegs == queue_max_segments(q)) 28454efd50bSKent Overstreet goto split; 28554efd50bSKent Overstreet 28654efd50bSKent Overstreet bvprv = bv; 287578270bfSMing Lei bvprvp = &bvprv; 288dcebd755SMing Lei 289dcebd755SMing Lei if (bvec_split_segs(q, &bv, &nsegs, &seg_size, 290dcebd755SMing Lei &front_seg_size, §ors)) 291dcebd755SMing Lei goto split; 29202e70742SMing Lei 29354efd50bSKent Overstreet } 29454efd50bSKent Overstreet 29502e70742SMing Lei do_split = false; 29654efd50bSKent Overstreet split: 297bdced438SMing Lei *segs = nsegs; 29802e70742SMing Lei 29902e70742SMing Lei if (do_split) { 30002e70742SMing Lei new = bio_split(bio, sectors, GFP_NOIO, bs); 30102e70742SMing Lei if (new) 30202e70742SMing Lei bio = new; 30302e70742SMing Lei } 30402e70742SMing Lei 30502e70742SMing Lei bio->bi_seg_front_size = front_seg_size; 30602e70742SMing Lei if (seg_size > bio->bi_seg_back_size) 30702e70742SMing Lei bio->bi_seg_back_size = seg_size; 30802e70742SMing Lei 30902e70742SMing Lei return do_split ? new : NULL; 31054efd50bSKent Overstreet } 31154efd50bSKent Overstreet 312af67c31fSNeilBrown void blk_queue_split(struct request_queue *q, struct bio **bio) 31354efd50bSKent Overstreet { 314bdced438SMing Lei struct bio *split, *res; 315bdced438SMing Lei unsigned nsegs; 31654efd50bSKent Overstreet 3177afafc8aSAdrian Hunter switch (bio_op(*bio)) { 3187afafc8aSAdrian Hunter case REQ_OP_DISCARD: 3197afafc8aSAdrian Hunter case REQ_OP_SECURE_ERASE: 320338aa96dSKent Overstreet split = blk_bio_discard_split(q, *bio, &q->bio_split, &nsegs); 3217afafc8aSAdrian Hunter break; 322a6f0788eSChaitanya Kulkarni case REQ_OP_WRITE_ZEROES: 323338aa96dSKent Overstreet split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, &nsegs); 324a6f0788eSChaitanya Kulkarni break; 3257afafc8aSAdrian Hunter case REQ_OP_WRITE_SAME: 326338aa96dSKent Overstreet split = blk_bio_write_same_split(q, *bio, &q->bio_split, &nsegs); 3277afafc8aSAdrian Hunter break; 3287afafc8aSAdrian Hunter default: 329338aa96dSKent Overstreet split = blk_bio_segment_split(q, *bio, &q->bio_split, &nsegs); 3307afafc8aSAdrian Hunter break; 3317afafc8aSAdrian Hunter } 332bdced438SMing Lei 333bdced438SMing Lei /* physical segments can be figured out during splitting */ 334bdced438SMing Lei res = split ? split : *bio; 335bdced438SMing Lei res->bi_phys_segments = nsegs; 336bdced438SMing Lei bio_set_flag(res, BIO_SEG_VALID); 33754efd50bSKent Overstreet 33854efd50bSKent Overstreet if (split) { 3396ac45aebSMing Lei /* there isn't chance to merge the splitted bio */ 3401eff9d32SJens Axboe split->bi_opf |= REQ_NOMERGE; 3416ac45aebSMing Lei 342947b7ac1SJens Axboe /* 343947b7ac1SJens Axboe * Since we're recursing into make_request here, ensure 344947b7ac1SJens Axboe * that we mark this bio as already having entered the queue. 345947b7ac1SJens Axboe * If not, and the queue is going away, we can get stuck 346947b7ac1SJens Axboe * forever on waiting for the queue reference to drop. But 347947b7ac1SJens Axboe * that will never happen, as we're already holding a 348947b7ac1SJens Axboe * reference to it. 349947b7ac1SJens Axboe */ 350947b7ac1SJens Axboe bio_set_flag(*bio, BIO_QUEUE_ENTERED); 351947b7ac1SJens Axboe 35254efd50bSKent Overstreet bio_chain(split, *bio); 353cda22646SMike Krinkin trace_block_split(q, split, (*bio)->bi_iter.bi_sector); 35454efd50bSKent Overstreet generic_make_request(*bio); 35554efd50bSKent Overstreet *bio = split; 35654efd50bSKent Overstreet } 35754efd50bSKent Overstreet } 35854efd50bSKent Overstreet EXPORT_SYMBOL(blk_queue_split); 35954efd50bSKent Overstreet 3601e428079SJens Axboe static unsigned int __blk_recalc_rq_segments(struct request_queue *q, 36107388549SMing Lei struct bio *bio, 36207388549SMing Lei bool no_sg_merge) 363d6d48196SJens Axboe { 3647988613bSKent Overstreet struct bio_vec bv, bvprv = { NULL }; 36538417468SChristoph Hellwig int prev = 0; 3661e428079SJens Axboe unsigned int seg_size, nr_phys_segs; 367dcebd755SMing Lei unsigned front_seg_size = bio->bi_seg_front_size; 36859247eaeSJens Axboe struct bio *fbio, *bbio; 3697988613bSKent Overstreet struct bvec_iter iter; 370d6d48196SJens Axboe 3711e428079SJens Axboe if (!bio) 3721e428079SJens Axboe return 0; 373d6d48196SJens Axboe 374a6f0788eSChaitanya Kulkarni switch (bio_op(bio)) { 375a6f0788eSChaitanya Kulkarni case REQ_OP_DISCARD: 376a6f0788eSChaitanya Kulkarni case REQ_OP_SECURE_ERASE: 377a6f0788eSChaitanya Kulkarni case REQ_OP_WRITE_ZEROES: 378f9d03f96SChristoph Hellwig return 0; 379f9d03f96SChristoph Hellwig case REQ_OP_WRITE_SAME: 3805cb8850cSKent Overstreet return 1; 381a6f0788eSChaitanya Kulkarni } 3825cb8850cSKent Overstreet 3831e428079SJens Axboe fbio = bio; 3845df97b91SMikulas Patocka seg_size = 0; 3852c8919deSAndi Kleen nr_phys_segs = 0; 3861e428079SJens Axboe for_each_bio(bio) { 387dcebd755SMing Lei bio_for_each_bvec(bv, bio, iter) { 388d6d48196SJens Axboe /* 38905f1dd53SJens Axboe * If SG merging is disabled, each bio vector is 39005f1dd53SJens Axboe * a segment 39105f1dd53SJens Axboe */ 39205f1dd53SJens Axboe if (no_sg_merge) 39305f1dd53SJens Axboe goto new_segment; 39405f1dd53SJens Axboe 39538417468SChristoph Hellwig if (prev) { 3967988613bSKent Overstreet if (seg_size + bv.bv_len 397ae03bf63SMartin K. Petersen > queue_max_segment_size(q)) 398d6d48196SJens Axboe goto new_segment; 3993dccdae5SChristoph Hellwig if (!biovec_phys_mergeable(q, &bvprv, &bv)) 400d6d48196SJens Axboe goto new_segment; 401d6d48196SJens Axboe 4027988613bSKent Overstreet seg_size += bv.bv_len; 403d6d48196SJens Axboe bvprv = bv; 404d6d48196SJens Axboe continue; 405d6d48196SJens Axboe } 406d6d48196SJens Axboe new_segment: 407d6d48196SJens Axboe bvprv = bv; 40854efd50bSKent Overstreet prev = 1; 409dcebd755SMing Lei bvec_split_segs(q, &bv, &nr_phys_segs, &seg_size, 410dcebd755SMing Lei &front_seg_size, NULL); 411d6d48196SJens Axboe } 41259247eaeSJens Axboe bbio = bio; 4131e428079SJens Axboe } 414d6d48196SJens Axboe 415dcebd755SMing Lei fbio->bi_seg_front_size = front_seg_size; 41659247eaeSJens Axboe if (seg_size > bbio->bi_seg_back_size) 41759247eaeSJens Axboe bbio->bi_seg_back_size = seg_size; 4181e428079SJens Axboe 4191e428079SJens Axboe return nr_phys_segs; 4201e428079SJens Axboe } 4211e428079SJens Axboe 4221e428079SJens Axboe void blk_recalc_rq_segments(struct request *rq) 4231e428079SJens Axboe { 42407388549SMing Lei bool no_sg_merge = !!test_bit(QUEUE_FLAG_NO_SG_MERGE, 42507388549SMing Lei &rq->q->queue_flags); 42607388549SMing Lei 42707388549SMing Lei rq->nr_phys_segments = __blk_recalc_rq_segments(rq->q, rq->bio, 42807388549SMing Lei no_sg_merge); 429d6d48196SJens Axboe } 430d6d48196SJens Axboe 431d6d48196SJens Axboe void blk_recount_segments(struct request_queue *q, struct bio *bio) 432d6d48196SJens Axboe { 4331a67356eSMing Lei unsigned short seg_cnt = bio_segments(bio); 4347f60dcaaSMing Lei 4357f60dcaaSMing Lei if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags) && 4367f60dcaaSMing Lei (seg_cnt < queue_max_segments(q))) 4377f60dcaaSMing Lei bio->bi_phys_segments = seg_cnt; 43805f1dd53SJens Axboe else { 439d6d48196SJens Axboe struct bio *nxt = bio->bi_next; 4401e428079SJens Axboe 441d6d48196SJens Axboe bio->bi_next = NULL; 4427f60dcaaSMing Lei bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio, false); 443d6d48196SJens Axboe bio->bi_next = nxt; 44405f1dd53SJens Axboe } 44505f1dd53SJens Axboe 446b7c44ed9SJens Axboe bio_set_flag(bio, BIO_SEG_VALID); 447d6d48196SJens Axboe } 448d6d48196SJens Axboe 449d6d48196SJens Axboe static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio, 450d6d48196SJens Axboe struct bio *nxt) 451d6d48196SJens Axboe { 4522b8221e1SKent Overstreet struct bio_vec end_bv = { NULL }, nxt_bv; 453f619d254SKent Overstreet 45486771427SFUJITA Tomonori if (bio->bi_seg_back_size + nxt->bi_seg_front_size > 455ae03bf63SMartin K. Petersen queue_max_segment_size(q)) 456d6d48196SJens Axboe return 0; 457d6d48196SJens Axboe 458e17fc0a1SDavid Woodhouse if (!bio_has_data(bio)) 459e17fc0a1SDavid Woodhouse return 1; 460e17fc0a1SDavid Woodhouse 461e827091cSMing Lei bio_get_last_bvec(bio, &end_bv); 462e827091cSMing Lei bio_get_first_bvec(nxt, &nxt_bv); 463f619d254SKent Overstreet 4643dccdae5SChristoph Hellwig return biovec_phys_mergeable(q, &end_bv, &nxt_bv); 465d6d48196SJens Axboe } 466d6d48196SJens Axboe 467862e5a5eSMing Lei static struct scatterlist *blk_next_sg(struct scatterlist **sg, 468862e5a5eSMing Lei struct scatterlist *sglist) 469862e5a5eSMing Lei { 470862e5a5eSMing Lei if (!*sg) 471862e5a5eSMing Lei return sglist; 472862e5a5eSMing Lei 473862e5a5eSMing Lei /* 474862e5a5eSMing Lei * If the driver previously mapped a shorter list, we could see a 475862e5a5eSMing Lei * termination bit prematurely unless it fully inits the sg table 476862e5a5eSMing Lei * on each mapping. We KNOW that there must be more entries here 477862e5a5eSMing Lei * or the driver would be buggy, so force clear the termination bit 478862e5a5eSMing Lei * to avoid doing a full sg_init_table() in drivers for each command. 479862e5a5eSMing Lei */ 480862e5a5eSMing Lei sg_unmark_end(*sg); 481862e5a5eSMing Lei return sg_next(*sg); 482862e5a5eSMing Lei } 483862e5a5eSMing Lei 484862e5a5eSMing Lei static unsigned blk_bvec_map_sg(struct request_queue *q, 485862e5a5eSMing Lei struct bio_vec *bvec, struct scatterlist *sglist, 486862e5a5eSMing Lei struct scatterlist **sg) 487862e5a5eSMing Lei { 488862e5a5eSMing Lei unsigned nbytes = bvec->bv_len; 489862e5a5eSMing Lei unsigned nsegs = 0, total = 0, offset = 0; 490862e5a5eSMing Lei 491862e5a5eSMing Lei while (nbytes > 0) { 492862e5a5eSMing Lei unsigned seg_size; 493862e5a5eSMing Lei struct page *pg; 494862e5a5eSMing Lei unsigned idx; 495862e5a5eSMing Lei 496862e5a5eSMing Lei *sg = blk_next_sg(sg, sglist); 497862e5a5eSMing Lei 498862e5a5eSMing Lei seg_size = get_max_segment_size(q, bvec->bv_offset + total); 499862e5a5eSMing Lei seg_size = min(nbytes, seg_size); 500862e5a5eSMing Lei 501862e5a5eSMing Lei offset = (total + bvec->bv_offset) % PAGE_SIZE; 502862e5a5eSMing Lei idx = (total + bvec->bv_offset) / PAGE_SIZE; 503862e5a5eSMing Lei pg = nth_page(bvec->bv_page, idx); 504862e5a5eSMing Lei 505862e5a5eSMing Lei sg_set_page(*sg, pg, seg_size, offset); 506862e5a5eSMing Lei 507862e5a5eSMing Lei total += seg_size; 508862e5a5eSMing Lei nbytes -= seg_size; 509862e5a5eSMing Lei nsegs++; 510862e5a5eSMing Lei } 511862e5a5eSMing Lei 512862e5a5eSMing Lei return nsegs; 513862e5a5eSMing Lei } 514862e5a5eSMing Lei 5157988613bSKent Overstreet static inline void 516963ab9e5SAsias He __blk_segment_map_sg(struct request_queue *q, struct bio_vec *bvec, 5177988613bSKent Overstreet struct scatterlist *sglist, struct bio_vec *bvprv, 51838417468SChristoph Hellwig struct scatterlist **sg, int *nsegs) 519963ab9e5SAsias He { 520963ab9e5SAsias He 521963ab9e5SAsias He int nbytes = bvec->bv_len; 522963ab9e5SAsias He 52338417468SChristoph Hellwig if (*sg) { 524b4b6cb61SMing Lei if ((*sg)->length + nbytes > queue_max_segment_size(q)) 525b4b6cb61SMing Lei goto new_segment; 5263dccdae5SChristoph Hellwig if (!biovec_phys_mergeable(q, bvprv, bvec)) 527963ab9e5SAsias He goto new_segment; 528963ab9e5SAsias He 529963ab9e5SAsias He (*sg)->length += nbytes; 530963ab9e5SAsias He } else { 531963ab9e5SAsias He new_segment: 532862e5a5eSMing Lei (*nsegs) += blk_bvec_map_sg(q, bvec, sglist, sg); 533963ab9e5SAsias He } 5347988613bSKent Overstreet *bvprv = *bvec; 535963ab9e5SAsias He } 536963ab9e5SAsias He 537f9d03f96SChristoph Hellwig static inline int __blk_bvec_map_sg(struct request_queue *q, struct bio_vec bv, 538f9d03f96SChristoph Hellwig struct scatterlist *sglist, struct scatterlist **sg) 539f9d03f96SChristoph Hellwig { 540f9d03f96SChristoph Hellwig *sg = sglist; 541f9d03f96SChristoph Hellwig sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); 542f9d03f96SChristoph Hellwig return 1; 543f9d03f96SChristoph Hellwig } 544f9d03f96SChristoph Hellwig 5455cb8850cSKent Overstreet static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, 5465cb8850cSKent Overstreet struct scatterlist *sglist, 5475cb8850cSKent Overstreet struct scatterlist **sg) 5485cb8850cSKent Overstreet { 5495cb8850cSKent Overstreet struct bio_vec bvec, bvprv = { NULL }; 5505cb8850cSKent Overstreet struct bvec_iter iter; 55138417468SChristoph Hellwig int nsegs = 0; 5525cb8850cSKent Overstreet 5535cb8850cSKent Overstreet for_each_bio(bio) 554862e5a5eSMing Lei bio_for_each_bvec(bvec, bio, iter) 5555cb8850cSKent Overstreet __blk_segment_map_sg(q, &bvec, sglist, &bvprv, sg, 55638417468SChristoph Hellwig &nsegs); 5575cb8850cSKent Overstreet 5585cb8850cSKent Overstreet return nsegs; 5595cb8850cSKent Overstreet } 5605cb8850cSKent Overstreet 561d6d48196SJens Axboe /* 562d6d48196SJens Axboe * map a request to scatterlist, return number of sg entries setup. Caller 563d6d48196SJens Axboe * must make sure sg can hold rq->nr_phys_segments entries 564d6d48196SJens Axboe */ 565d6d48196SJens Axboe int blk_rq_map_sg(struct request_queue *q, struct request *rq, 566d6d48196SJens Axboe struct scatterlist *sglist) 567d6d48196SJens Axboe { 5685cb8850cSKent Overstreet struct scatterlist *sg = NULL; 5695cb8850cSKent Overstreet int nsegs = 0; 570d6d48196SJens Axboe 571f9d03f96SChristoph Hellwig if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 572f9d03f96SChristoph Hellwig nsegs = __blk_bvec_map_sg(q, rq->special_vec, sglist, &sg); 573f9d03f96SChristoph Hellwig else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME) 574f9d03f96SChristoph Hellwig nsegs = __blk_bvec_map_sg(q, bio_iovec(rq->bio), sglist, &sg); 575f9d03f96SChristoph Hellwig else if (rq->bio) 5765cb8850cSKent Overstreet nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg); 577f18573abSFUJITA Tomonori 578e8064021SChristoph Hellwig if (unlikely(rq->rq_flags & RQF_COPY_USER) && 5792e46e8b2STejun Heo (blk_rq_bytes(rq) & q->dma_pad_mask)) { 5802e46e8b2STejun Heo unsigned int pad_len = 5812e46e8b2STejun Heo (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1; 582f18573abSFUJITA Tomonori 583f18573abSFUJITA Tomonori sg->length += pad_len; 584f18573abSFUJITA Tomonori rq->extra_len += pad_len; 585f18573abSFUJITA Tomonori } 586f18573abSFUJITA Tomonori 5872fb98e84STejun Heo if (q->dma_drain_size && q->dma_drain_needed(rq)) { 588a8ebb056SMike Christie if (op_is_write(req_op(rq))) 589db0a2e00STejun Heo memset(q->dma_drain_buffer, 0, q->dma_drain_size); 590db0a2e00STejun Heo 591da81ed16SDan Williams sg_unmark_end(sg); 592d6d48196SJens Axboe sg = sg_next(sg); 593d6d48196SJens Axboe sg_set_page(sg, virt_to_page(q->dma_drain_buffer), 594d6d48196SJens Axboe q->dma_drain_size, 595d6d48196SJens Axboe ((unsigned long)q->dma_drain_buffer) & 596d6d48196SJens Axboe (PAGE_SIZE - 1)); 597d6d48196SJens Axboe nsegs++; 5987a85f889SFUJITA Tomonori rq->extra_len += q->dma_drain_size; 599d6d48196SJens Axboe } 600d6d48196SJens Axboe 601d6d48196SJens Axboe if (sg) 602d6d48196SJens Axboe sg_mark_end(sg); 603d6d48196SJens Axboe 60412e57f59SMing Lei /* 60512e57f59SMing Lei * Something must have been wrong if the figured number of 60612e57f59SMing Lei * segment is bigger than number of req's physical segments 60712e57f59SMing Lei */ 608f9d03f96SChristoph Hellwig WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); 60912e57f59SMing Lei 610d6d48196SJens Axboe return nsegs; 611d6d48196SJens Axboe } 612d6d48196SJens Axboe EXPORT_SYMBOL(blk_rq_map_sg); 613d6d48196SJens Axboe 614d6d48196SJens Axboe static inline int ll_new_hw_segment(struct request_queue *q, 615d6d48196SJens Axboe struct request *req, 616d6d48196SJens Axboe struct bio *bio) 617d6d48196SJens Axboe { 618d6d48196SJens Axboe int nr_phys_segs = bio_phys_segments(q, bio); 619d6d48196SJens Axboe 62013f05c8dSMartin K. Petersen if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(q)) 62113f05c8dSMartin K. Petersen goto no_merge; 62213f05c8dSMartin K. Petersen 6234eaf99beSMartin K. Petersen if (blk_integrity_merge_bio(q, req, bio) == false) 62413f05c8dSMartin K. Petersen goto no_merge; 625d6d48196SJens Axboe 626d6d48196SJens Axboe /* 627d6d48196SJens Axboe * This will form the start of a new hw segment. Bump both 628d6d48196SJens Axboe * counters. 629d6d48196SJens Axboe */ 630d6d48196SJens Axboe req->nr_phys_segments += nr_phys_segs; 631d6d48196SJens Axboe return 1; 63213f05c8dSMartin K. Petersen 63313f05c8dSMartin K. Petersen no_merge: 634e0c72300SRitesh Harjani req_set_nomerge(q, req); 63513f05c8dSMartin K. Petersen return 0; 636d6d48196SJens Axboe } 637d6d48196SJens Axboe 638d6d48196SJens Axboe int ll_back_merge_fn(struct request_queue *q, struct request *req, 639d6d48196SJens Axboe struct bio *bio) 640d6d48196SJens Axboe { 6415e7c4274SJens Axboe if (req_gap_back_merge(req, bio)) 6425e7c4274SJens Axboe return 0; 6437f39add3SSagi Grimberg if (blk_integrity_rq(req) && 6447f39add3SSagi Grimberg integrity_req_gap_back_merge(req, bio)) 6457f39add3SSagi Grimberg return 0; 646f31dc1cdSMartin K. Petersen if (blk_rq_sectors(req) + bio_sectors(bio) > 64717007f39SDamien Le Moal blk_rq_get_max_sectors(req, blk_rq_pos(req))) { 648e0c72300SRitesh Harjani req_set_nomerge(q, req); 649d6d48196SJens Axboe return 0; 650d6d48196SJens Axboe } 6512cdf79caSJens Axboe if (!bio_flagged(req->biotail, BIO_SEG_VALID)) 652d6d48196SJens Axboe blk_recount_segments(q, req->biotail); 6532cdf79caSJens Axboe if (!bio_flagged(bio, BIO_SEG_VALID)) 654d6d48196SJens Axboe blk_recount_segments(q, bio); 655d6d48196SJens Axboe 656d6d48196SJens Axboe return ll_new_hw_segment(q, req, bio); 657d6d48196SJens Axboe } 658d6d48196SJens Axboe 659d6d48196SJens Axboe int ll_front_merge_fn(struct request_queue *q, struct request *req, 660d6d48196SJens Axboe struct bio *bio) 661d6d48196SJens Axboe { 6625e7c4274SJens Axboe 6635e7c4274SJens Axboe if (req_gap_front_merge(req, bio)) 6645e7c4274SJens Axboe return 0; 6657f39add3SSagi Grimberg if (blk_integrity_rq(req) && 6667f39add3SSagi Grimberg integrity_req_gap_front_merge(req, bio)) 6677f39add3SSagi Grimberg return 0; 668f31dc1cdSMartin K. Petersen if (blk_rq_sectors(req) + bio_sectors(bio) > 66917007f39SDamien Le Moal blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { 670e0c72300SRitesh Harjani req_set_nomerge(q, req); 671d6d48196SJens Axboe return 0; 672d6d48196SJens Axboe } 6732cdf79caSJens Axboe if (!bio_flagged(bio, BIO_SEG_VALID)) 674d6d48196SJens Axboe blk_recount_segments(q, bio); 6752cdf79caSJens Axboe if (!bio_flagged(req->bio, BIO_SEG_VALID)) 676d6d48196SJens Axboe blk_recount_segments(q, req->bio); 677d6d48196SJens Axboe 678d6d48196SJens Axboe return ll_new_hw_segment(q, req, bio); 679d6d48196SJens Axboe } 680d6d48196SJens Axboe 681445251d0SJens Axboe static bool req_attempt_discard_merge(struct request_queue *q, struct request *req, 682445251d0SJens Axboe struct request *next) 683445251d0SJens Axboe { 684445251d0SJens Axboe unsigned short segments = blk_rq_nr_discard_segments(req); 685445251d0SJens Axboe 686445251d0SJens Axboe if (segments >= queue_max_discard_segments(q)) 687445251d0SJens Axboe goto no_merge; 688445251d0SJens Axboe if (blk_rq_sectors(req) + bio_sectors(next->bio) > 689445251d0SJens Axboe blk_rq_get_max_sectors(req, blk_rq_pos(req))) 690445251d0SJens Axboe goto no_merge; 691445251d0SJens Axboe 692445251d0SJens Axboe req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next); 693445251d0SJens Axboe return true; 694445251d0SJens Axboe no_merge: 695445251d0SJens Axboe req_set_nomerge(q, req); 696445251d0SJens Axboe return false; 697445251d0SJens Axboe } 698445251d0SJens Axboe 699d6d48196SJens Axboe static int ll_merge_requests_fn(struct request_queue *q, struct request *req, 700d6d48196SJens Axboe struct request *next) 701d6d48196SJens Axboe { 702d6d48196SJens Axboe int total_phys_segments; 70386771427SFUJITA Tomonori unsigned int seg_size = 70486771427SFUJITA Tomonori req->biotail->bi_seg_back_size + next->bio->bi_seg_front_size; 705d6d48196SJens Axboe 7065e7c4274SJens Axboe if (req_gap_back_merge(req, next->bio)) 707854fbb9cSKeith Busch return 0; 708854fbb9cSKeith Busch 709d6d48196SJens Axboe /* 710d6d48196SJens Axboe * Will it become too large? 711d6d48196SJens Axboe */ 712f31dc1cdSMartin K. Petersen if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > 71317007f39SDamien Le Moal blk_rq_get_max_sectors(req, blk_rq_pos(req))) 714d6d48196SJens Axboe return 0; 715d6d48196SJens Axboe 716d6d48196SJens Axboe total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; 71786771427SFUJITA Tomonori if (blk_phys_contig_segment(q, req->biotail, next->bio)) { 71886771427SFUJITA Tomonori if (req->nr_phys_segments == 1) 71986771427SFUJITA Tomonori req->bio->bi_seg_front_size = seg_size; 72086771427SFUJITA Tomonori if (next->nr_phys_segments == 1) 72186771427SFUJITA Tomonori next->biotail->bi_seg_back_size = seg_size; 722d6d48196SJens Axboe total_phys_segments--; 72386771427SFUJITA Tomonori } 724d6d48196SJens Axboe 7258a78362cSMartin K. Petersen if (total_phys_segments > queue_max_segments(q)) 726d6d48196SJens Axboe return 0; 727d6d48196SJens Axboe 7284eaf99beSMartin K. Petersen if (blk_integrity_merge_rq(q, req, next) == false) 72913f05c8dSMartin K. Petersen return 0; 73013f05c8dSMartin K. Petersen 731d6d48196SJens Axboe /* Merge is OK... */ 732d6d48196SJens Axboe req->nr_phys_segments = total_phys_segments; 733d6d48196SJens Axboe return 1; 734d6d48196SJens Axboe } 735d6d48196SJens Axboe 73680a761fdSTejun Heo /** 73780a761fdSTejun Heo * blk_rq_set_mixed_merge - mark a request as mixed merge 73880a761fdSTejun Heo * @rq: request to mark as mixed merge 73980a761fdSTejun Heo * 74080a761fdSTejun Heo * Description: 74180a761fdSTejun Heo * @rq is about to be mixed merged. Make sure the attributes 74280a761fdSTejun Heo * which can be mixed are set in each bio and mark @rq as mixed 74380a761fdSTejun Heo * merged. 74480a761fdSTejun Heo */ 74580a761fdSTejun Heo void blk_rq_set_mixed_merge(struct request *rq) 74680a761fdSTejun Heo { 74780a761fdSTejun Heo unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 74880a761fdSTejun Heo struct bio *bio; 74980a761fdSTejun Heo 750e8064021SChristoph Hellwig if (rq->rq_flags & RQF_MIXED_MERGE) 75180a761fdSTejun Heo return; 75280a761fdSTejun Heo 75380a761fdSTejun Heo /* 75480a761fdSTejun Heo * @rq will no longer represent mixable attributes for all the 75580a761fdSTejun Heo * contained bios. It will just track those of the first one. 75680a761fdSTejun Heo * Distributes the attributs to each bio. 75780a761fdSTejun Heo */ 75880a761fdSTejun Heo for (bio = rq->bio; bio; bio = bio->bi_next) { 7591eff9d32SJens Axboe WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) && 7601eff9d32SJens Axboe (bio->bi_opf & REQ_FAILFAST_MASK) != ff); 7611eff9d32SJens Axboe bio->bi_opf |= ff; 76280a761fdSTejun Heo } 763e8064021SChristoph Hellwig rq->rq_flags |= RQF_MIXED_MERGE; 76480a761fdSTejun Heo } 76580a761fdSTejun Heo 76626308eabSJerome Marchand static void blk_account_io_merge(struct request *req) 76726308eabSJerome Marchand { 76826308eabSJerome Marchand if (blk_do_io_stat(req)) { 76926308eabSJerome Marchand struct hd_struct *part; 77026308eabSJerome Marchand 771112f158fSMike Snitzer part_stat_lock(); 77209e099d4SJerome Marchand part = req->part; 77326308eabSJerome Marchand 774d62e26b3SJens Axboe part_dec_in_flight(req->q, part, rq_data_dir(req)); 77526308eabSJerome Marchand 7766c23a968SJens Axboe hd_struct_put(part); 77726308eabSJerome Marchand part_stat_unlock(); 77826308eabSJerome Marchand } 77926308eabSJerome Marchand } 78069840466SJianchao Wang /* 78169840466SJianchao Wang * Two cases of handling DISCARD merge: 78269840466SJianchao Wang * If max_discard_segments > 1, the driver takes every bio 78369840466SJianchao Wang * as a range and send them to controller together. The ranges 78469840466SJianchao Wang * needn't to be contiguous. 78569840466SJianchao Wang * Otherwise, the bios/requests will be handled as same as 78669840466SJianchao Wang * others which should be contiguous. 78769840466SJianchao Wang */ 78869840466SJianchao Wang static inline bool blk_discard_mergable(struct request *req) 78969840466SJianchao Wang { 79069840466SJianchao Wang if (req_op(req) == REQ_OP_DISCARD && 79169840466SJianchao Wang queue_max_discard_segments(req->q) > 1) 79269840466SJianchao Wang return true; 79369840466SJianchao Wang return false; 79469840466SJianchao Wang } 79569840466SJianchao Wang 796e96c0d83SEric Biggers static enum elv_merge blk_try_req_merge(struct request *req, 797e96c0d83SEric Biggers struct request *next) 79869840466SJianchao Wang { 79969840466SJianchao Wang if (blk_discard_mergable(req)) 80069840466SJianchao Wang return ELEVATOR_DISCARD_MERGE; 80169840466SJianchao Wang else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) 80269840466SJianchao Wang return ELEVATOR_BACK_MERGE; 80369840466SJianchao Wang 80469840466SJianchao Wang return ELEVATOR_NO_MERGE; 80569840466SJianchao Wang } 80626308eabSJerome Marchand 807d6d48196SJens Axboe /* 808b973cb7eSJens Axboe * For non-mq, this has to be called with the request spinlock acquired. 809b973cb7eSJens Axboe * For mq with scheduling, the appropriate queue wide lock should be held. 810d6d48196SJens Axboe */ 811b973cb7eSJens Axboe static struct request *attempt_merge(struct request_queue *q, 812b973cb7eSJens Axboe struct request *req, struct request *next) 813d6d48196SJens Axboe { 814d6d48196SJens Axboe if (!rq_mergeable(req) || !rq_mergeable(next)) 815b973cb7eSJens Axboe return NULL; 816d6d48196SJens Axboe 817288dab8aSChristoph Hellwig if (req_op(req) != req_op(next)) 818b973cb7eSJens Axboe return NULL; 819f31dc1cdSMartin K. Petersen 820d6d48196SJens Axboe if (rq_data_dir(req) != rq_data_dir(next) 8212081a56bSJens Axboe || req->rq_disk != next->rq_disk) 822b973cb7eSJens Axboe return NULL; 823d6d48196SJens Axboe 8248fe0d473SMike Christie if (req_op(req) == REQ_OP_WRITE_SAME && 8254363ac7cSMartin K. Petersen !blk_write_same_mergeable(req->bio, next->bio)) 826b973cb7eSJens Axboe return NULL; 8274363ac7cSMartin K. Petersen 828d6d48196SJens Axboe /* 829cb6934f8SJens Axboe * Don't allow merge of different write hints, or for a hint with 830cb6934f8SJens Axboe * non-hint IO. 831cb6934f8SJens Axboe */ 832cb6934f8SJens Axboe if (req->write_hint != next->write_hint) 833cb6934f8SJens Axboe return NULL; 834cb6934f8SJens Axboe 835668ffc03SDamien Le Moal if (req->ioprio != next->ioprio) 836668ffc03SDamien Le Moal return NULL; 837668ffc03SDamien Le Moal 838cb6934f8SJens Axboe /* 839d6d48196SJens Axboe * If we are allowed to merge, then append bio list 840d6d48196SJens Axboe * from next to rq and release next. merge_requests_fn 841d6d48196SJens Axboe * will have updated segment counts, update sector 842445251d0SJens Axboe * counts here. Handle DISCARDs separately, as they 843445251d0SJens Axboe * have separate settings. 844d6d48196SJens Axboe */ 84569840466SJianchao Wang 84669840466SJianchao Wang switch (blk_try_req_merge(req, next)) { 84769840466SJianchao Wang case ELEVATOR_DISCARD_MERGE: 848445251d0SJens Axboe if (!req_attempt_discard_merge(q, req, next)) 849445251d0SJens Axboe return NULL; 85069840466SJianchao Wang break; 85169840466SJianchao Wang case ELEVATOR_BACK_MERGE: 85269840466SJianchao Wang if (!ll_merge_requests_fn(q, req, next)) 853b973cb7eSJens Axboe return NULL; 85469840466SJianchao Wang break; 85569840466SJianchao Wang default: 85669840466SJianchao Wang return NULL; 85769840466SJianchao Wang } 858d6d48196SJens Axboe 859d6d48196SJens Axboe /* 86080a761fdSTejun Heo * If failfast settings disagree or any of the two is already 86180a761fdSTejun Heo * a mixed merge, mark both as mixed before proceeding. This 86280a761fdSTejun Heo * makes sure that all involved bios have mixable attributes 86380a761fdSTejun Heo * set properly. 86480a761fdSTejun Heo */ 865e8064021SChristoph Hellwig if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) || 86680a761fdSTejun Heo (req->cmd_flags & REQ_FAILFAST_MASK) != 86780a761fdSTejun Heo (next->cmd_flags & REQ_FAILFAST_MASK)) { 86880a761fdSTejun Heo blk_rq_set_mixed_merge(req); 86980a761fdSTejun Heo blk_rq_set_mixed_merge(next); 87080a761fdSTejun Heo } 87180a761fdSTejun Heo 87280a761fdSTejun Heo /* 873522a7775SOmar Sandoval * At this point we have either done a back merge or front merge. We 874522a7775SOmar Sandoval * need the smaller start_time_ns of the merged requests to be the 875522a7775SOmar Sandoval * current request for accounting purposes. 876d6d48196SJens Axboe */ 877522a7775SOmar Sandoval if (next->start_time_ns < req->start_time_ns) 878522a7775SOmar Sandoval req->start_time_ns = next->start_time_ns; 879d6d48196SJens Axboe 880d6d48196SJens Axboe req->biotail->bi_next = next->bio; 881d6d48196SJens Axboe req->biotail = next->biotail; 882d6d48196SJens Axboe 883a2dec7b3STejun Heo req->__data_len += blk_rq_bytes(next); 884d6d48196SJens Axboe 8852a5cf35cSMing Lei if (!blk_discard_mergable(req)) 886d6d48196SJens Axboe elv_merge_requests(q, req, next); 887d6d48196SJens Axboe 88842dad764SJerome Marchand /* 88942dad764SJerome Marchand * 'next' is going away, so update stats accordingly 89042dad764SJerome Marchand */ 89142dad764SJerome Marchand blk_account_io_merge(next); 892d6d48196SJens Axboe 893e4d750c9SJens Axboe /* 894e4d750c9SJens Axboe * ownership of bio passed from next to req, return 'next' for 895e4d750c9SJens Axboe * the caller to free 896e4d750c9SJens Axboe */ 8971cd96c24SBoaz Harrosh next->bio = NULL; 898b973cb7eSJens Axboe return next; 899d6d48196SJens Axboe } 900d6d48196SJens Axboe 901b973cb7eSJens Axboe struct request *attempt_back_merge(struct request_queue *q, struct request *rq) 902d6d48196SJens Axboe { 903d6d48196SJens Axboe struct request *next = elv_latter_request(q, rq); 904d6d48196SJens Axboe 905d6d48196SJens Axboe if (next) 906d6d48196SJens Axboe return attempt_merge(q, rq, next); 907d6d48196SJens Axboe 908b973cb7eSJens Axboe return NULL; 909d6d48196SJens Axboe } 910d6d48196SJens Axboe 911b973cb7eSJens Axboe struct request *attempt_front_merge(struct request_queue *q, struct request *rq) 912d6d48196SJens Axboe { 913d6d48196SJens Axboe struct request *prev = elv_former_request(q, rq); 914d6d48196SJens Axboe 915d6d48196SJens Axboe if (prev) 916d6d48196SJens Axboe return attempt_merge(q, prev, rq); 917d6d48196SJens Axboe 918b973cb7eSJens Axboe return NULL; 919d6d48196SJens Axboe } 9205e84ea3aSJens Axboe 9215e84ea3aSJens Axboe int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 9225e84ea3aSJens Axboe struct request *next) 9235e84ea3aSJens Axboe { 924e4d750c9SJens Axboe struct request *free; 92572ef799bSTahsin Erdogan 926e4d750c9SJens Axboe free = attempt_merge(q, rq, next); 927e4d750c9SJens Axboe if (free) { 92892bc5a24SJens Axboe blk_put_request(free); 929e4d750c9SJens Axboe return 1; 930e4d750c9SJens Axboe } 931e4d750c9SJens Axboe 932e4d750c9SJens Axboe return 0; 9335e84ea3aSJens Axboe } 934050c8ea8STejun Heo 935050c8ea8STejun Heo bool blk_rq_merge_ok(struct request *rq, struct bio *bio) 936050c8ea8STejun Heo { 937e2a60da7SMartin K. Petersen if (!rq_mergeable(rq) || !bio_mergeable(bio)) 938050c8ea8STejun Heo return false; 939050c8ea8STejun Heo 940288dab8aSChristoph Hellwig if (req_op(rq) != bio_op(bio)) 941f31dc1cdSMartin K. Petersen return false; 942f31dc1cdSMartin K. Petersen 943050c8ea8STejun Heo /* different data direction or already started, don't merge */ 944050c8ea8STejun Heo if (bio_data_dir(bio) != rq_data_dir(rq)) 945050c8ea8STejun Heo return false; 946050c8ea8STejun Heo 9472081a56bSJens Axboe /* must be same device */ 9482081a56bSJens Axboe if (rq->rq_disk != bio->bi_disk) 949050c8ea8STejun Heo return false; 950050c8ea8STejun Heo 951050c8ea8STejun Heo /* only merge integrity protected bio into ditto rq */ 9524eaf99beSMartin K. Petersen if (blk_integrity_merge_bio(rq->q, rq, bio) == false) 953050c8ea8STejun Heo return false; 954050c8ea8STejun Heo 9554363ac7cSMartin K. Petersen /* must be using the same buffer */ 9568fe0d473SMike Christie if (req_op(rq) == REQ_OP_WRITE_SAME && 9574363ac7cSMartin K. Petersen !blk_write_same_mergeable(rq->bio, bio)) 9584363ac7cSMartin K. Petersen return false; 9594363ac7cSMartin K. Petersen 960cb6934f8SJens Axboe /* 961cb6934f8SJens Axboe * Don't allow merge of different write hints, or for a hint with 962cb6934f8SJens Axboe * non-hint IO. 963cb6934f8SJens Axboe */ 964cb6934f8SJens Axboe if (rq->write_hint != bio->bi_write_hint) 965cb6934f8SJens Axboe return false; 966cb6934f8SJens Axboe 967668ffc03SDamien Le Moal if (rq->ioprio != bio_prio(bio)) 968668ffc03SDamien Le Moal return false; 969668ffc03SDamien Le Moal 970050c8ea8STejun Heo return true; 971050c8ea8STejun Heo } 972050c8ea8STejun Heo 97334fe7c05SChristoph Hellwig enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) 974050c8ea8STejun Heo { 97569840466SJianchao Wang if (blk_discard_mergable(rq)) 9761e739730SChristoph Hellwig return ELEVATOR_DISCARD_MERGE; 9771e739730SChristoph Hellwig else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) 978050c8ea8STejun Heo return ELEVATOR_BACK_MERGE; 9794f024f37SKent Overstreet else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) 980050c8ea8STejun Heo return ELEVATOR_FRONT_MERGE; 981050c8ea8STejun Heo return ELEVATOR_NO_MERGE; 982050c8ea8STejun Heo } 983