1d6d48196SJens Axboe /* 2d6d48196SJens Axboe * Functions related to segment and merge handling 3d6d48196SJens Axboe */ 4d6d48196SJens Axboe #include <linux/kernel.h> 5d6d48196SJens Axboe #include <linux/module.h> 6d6d48196SJens Axboe #include <linux/bio.h> 7d6d48196SJens Axboe #include <linux/blkdev.h> 8d6d48196SJens Axboe #include <linux/scatterlist.h> 9d6d48196SJens Axboe 10d6d48196SJens Axboe #include "blk.h" 11d6d48196SJens Axboe 12d6d48196SJens Axboe void blk_recalc_rq_sectors(struct request *rq, int nsect) 13d6d48196SJens Axboe { 14e17fc0a1SDavid Woodhouse if (blk_fs_request(rq) || blk_discard_rq(rq)) { 15d6d48196SJens Axboe rq->hard_sector += nsect; 16d6d48196SJens Axboe rq->hard_nr_sectors -= nsect; 17d6d48196SJens Axboe 18d6d48196SJens Axboe /* 19d6d48196SJens Axboe * Move the I/O submission pointers ahead if required. 20d6d48196SJens Axboe */ 21d6d48196SJens Axboe if ((rq->nr_sectors >= rq->hard_nr_sectors) && 22d6d48196SJens Axboe (rq->sector <= rq->hard_sector)) { 23d6d48196SJens Axboe rq->sector = rq->hard_sector; 24d6d48196SJens Axboe rq->nr_sectors = rq->hard_nr_sectors; 25d6d48196SJens Axboe rq->hard_cur_sectors = bio_cur_sectors(rq->bio); 26d6d48196SJens Axboe rq->current_nr_sectors = rq->hard_cur_sectors; 27d6d48196SJens Axboe rq->buffer = bio_data(rq->bio); 28d6d48196SJens Axboe } 29d6d48196SJens Axboe 30d6d48196SJens Axboe /* 31d6d48196SJens Axboe * if total number of sectors is less than the first segment 32d6d48196SJens Axboe * size, something has gone terribly wrong 33d6d48196SJens Axboe */ 34d6d48196SJens Axboe if (rq->nr_sectors < rq->current_nr_sectors) { 356728cb0eSJens Axboe printk(KERN_ERR "blk: request botched\n"); 36d6d48196SJens Axboe rq->nr_sectors = rq->current_nr_sectors; 37d6d48196SJens Axboe } 38d6d48196SJens Axboe } 39d6d48196SJens Axboe } 40d6d48196SJens Axboe 41d6d48196SJens Axboe void blk_recalc_rq_segments(struct request *rq) 42d6d48196SJens Axboe { 43d6d48196SJens Axboe int nr_phys_segs; 44d6d48196SJens Axboe int nr_hw_segs; 45d6d48196SJens Axboe unsigned int phys_size; 46d6d48196SJens Axboe unsigned int hw_size; 47d6d48196SJens Axboe struct bio_vec *bv, *bvprv = NULL; 48d6d48196SJens Axboe int seg_size; 49d6d48196SJens Axboe int hw_seg_size; 50d6d48196SJens Axboe int cluster; 51d6d48196SJens Axboe struct req_iterator iter; 52d6d48196SJens Axboe int high, highprv = 1; 53d6d48196SJens Axboe struct request_queue *q = rq->q; 54d6d48196SJens Axboe 55d6d48196SJens Axboe if (!rq->bio) 56d6d48196SJens Axboe return; 57d6d48196SJens Axboe 5875ad23bcSNick Piggin cluster = test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags); 59d6d48196SJens Axboe hw_seg_size = seg_size = 0; 60d6d48196SJens Axboe phys_size = hw_size = nr_phys_segs = nr_hw_segs = 0; 61d6d48196SJens Axboe rq_for_each_segment(bv, rq, iter) { 62d6d48196SJens Axboe /* 63d6d48196SJens Axboe * the trick here is making sure that a high page is never 64d6d48196SJens Axboe * considered part of another segment, since that might 65d6d48196SJens Axboe * change with the bounce page. 66d6d48196SJens Axboe */ 67d6d48196SJens Axboe high = page_to_pfn(bv->bv_page) > q->bounce_pfn; 68d6d48196SJens Axboe if (high || highprv) 69b8b3e16cSMikulas Patocka goto new_segment; 70d6d48196SJens Axboe if (cluster) { 71d6d48196SJens Axboe if (seg_size + bv->bv_len > q->max_segment_size) 72d6d48196SJens Axboe goto new_segment; 73d6d48196SJens Axboe if (!BIOVEC_PHYS_MERGEABLE(bvprv, bv)) 74d6d48196SJens Axboe goto new_segment; 75d6d48196SJens Axboe if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bv)) 76d6d48196SJens Axboe goto new_segment; 77d6d48196SJens Axboe 78d6d48196SJens Axboe seg_size += bv->bv_len; 79d6d48196SJens Axboe hw_seg_size += bv->bv_len; 80d6d48196SJens Axboe bvprv = bv; 81d6d48196SJens Axboe continue; 82d6d48196SJens Axboe } 83d6d48196SJens Axboe new_segment: 84d6d48196SJens Axboe if (nr_hw_segs == 1 && 85d6d48196SJens Axboe hw_seg_size > rq->bio->bi_hw_front_size) 86d6d48196SJens Axboe rq->bio->bi_hw_front_size = hw_seg_size; 87b8b3e16cSMikulas Patocka hw_seg_size = bv->bv_len; 88d6d48196SJens Axboe nr_hw_segs++; 89d6d48196SJens Axboe 90d6d48196SJens Axboe nr_phys_segs++; 91d6d48196SJens Axboe bvprv = bv; 92d6d48196SJens Axboe seg_size = bv->bv_len; 93d6d48196SJens Axboe highprv = high; 94d6d48196SJens Axboe } 95d6d48196SJens Axboe 96d6d48196SJens Axboe if (nr_hw_segs == 1 && 97d6d48196SJens Axboe hw_seg_size > rq->bio->bi_hw_front_size) 98d6d48196SJens Axboe rq->bio->bi_hw_front_size = hw_seg_size; 99d6d48196SJens Axboe if (hw_seg_size > rq->biotail->bi_hw_back_size) 100d6d48196SJens Axboe rq->biotail->bi_hw_back_size = hw_seg_size; 101d6d48196SJens Axboe rq->nr_phys_segments = nr_phys_segs; 102d6d48196SJens Axboe rq->nr_hw_segments = nr_hw_segs; 103d6d48196SJens Axboe } 104d6d48196SJens Axboe 105d6d48196SJens Axboe void blk_recount_segments(struct request_queue *q, struct bio *bio) 106d6d48196SJens Axboe { 107d6d48196SJens Axboe struct request rq; 108d6d48196SJens Axboe struct bio *nxt = bio->bi_next; 109d6d48196SJens Axboe rq.q = q; 110d6d48196SJens Axboe rq.bio = rq.biotail = bio; 111d6d48196SJens Axboe bio->bi_next = NULL; 112d6d48196SJens Axboe blk_recalc_rq_segments(&rq); 113d6d48196SJens Axboe bio->bi_next = nxt; 114d6d48196SJens Axboe bio->bi_phys_segments = rq.nr_phys_segments; 115d6d48196SJens Axboe bio->bi_hw_segments = rq.nr_hw_segments; 116d6d48196SJens Axboe bio->bi_flags |= (1 << BIO_SEG_VALID); 117d6d48196SJens Axboe } 118d6d48196SJens Axboe EXPORT_SYMBOL(blk_recount_segments); 119d6d48196SJens Axboe 120d6d48196SJens Axboe static int blk_phys_contig_segment(struct request_queue *q, struct bio *bio, 121d6d48196SJens Axboe struct bio *nxt) 122d6d48196SJens Axboe { 12375ad23bcSNick Piggin if (!test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags)) 124d6d48196SJens Axboe return 0; 125d6d48196SJens Axboe 126d6d48196SJens Axboe if (bio->bi_size + nxt->bi_size > q->max_segment_size) 127d6d48196SJens Axboe return 0; 128d6d48196SJens Axboe 129e17fc0a1SDavid Woodhouse if (!bio_has_data(bio)) 130e17fc0a1SDavid Woodhouse return 1; 131e17fc0a1SDavid Woodhouse 132e17fc0a1SDavid Woodhouse if (!BIOVEC_PHYS_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt))) 133e17fc0a1SDavid Woodhouse return 0; 134e17fc0a1SDavid Woodhouse 135d6d48196SJens Axboe /* 136e17fc0a1SDavid Woodhouse * bio and nxt are contiguous in memory; check if the queue allows 137d6d48196SJens Axboe * these two to be merged into one 138d6d48196SJens Axboe */ 139d6d48196SJens Axboe if (BIO_SEG_BOUNDARY(q, bio, nxt)) 140d6d48196SJens Axboe return 1; 141d6d48196SJens Axboe 142d6d48196SJens Axboe return 0; 143d6d48196SJens Axboe } 144d6d48196SJens Axboe 145d6d48196SJens Axboe /* 146d6d48196SJens Axboe * map a request to scatterlist, return number of sg entries setup. Caller 147d6d48196SJens Axboe * must make sure sg can hold rq->nr_phys_segments entries 148d6d48196SJens Axboe */ 149d6d48196SJens Axboe int blk_rq_map_sg(struct request_queue *q, struct request *rq, 150d6d48196SJens Axboe struct scatterlist *sglist) 151d6d48196SJens Axboe { 152d6d48196SJens Axboe struct bio_vec *bvec, *bvprv; 153d6d48196SJens Axboe struct req_iterator iter; 154d6d48196SJens Axboe struct scatterlist *sg; 155d6d48196SJens Axboe int nsegs, cluster; 156d6d48196SJens Axboe 157d6d48196SJens Axboe nsegs = 0; 15875ad23bcSNick Piggin cluster = test_bit(QUEUE_FLAG_CLUSTER, &q->queue_flags); 159d6d48196SJens Axboe 160d6d48196SJens Axboe /* 161d6d48196SJens Axboe * for each bio in rq 162d6d48196SJens Axboe */ 163d6d48196SJens Axboe bvprv = NULL; 164d6d48196SJens Axboe sg = NULL; 165d6d48196SJens Axboe rq_for_each_segment(bvec, rq, iter) { 166d6d48196SJens Axboe int nbytes = bvec->bv_len; 167d6d48196SJens Axboe 168d6d48196SJens Axboe if (bvprv && cluster) { 169d6d48196SJens Axboe if (sg->length + nbytes > q->max_segment_size) 170d6d48196SJens Axboe goto new_segment; 171d6d48196SJens Axboe 172d6d48196SJens Axboe if (!BIOVEC_PHYS_MERGEABLE(bvprv, bvec)) 173d6d48196SJens Axboe goto new_segment; 174d6d48196SJens Axboe if (!BIOVEC_SEG_BOUNDARY(q, bvprv, bvec)) 175d6d48196SJens Axboe goto new_segment; 176d6d48196SJens Axboe 177d6d48196SJens Axboe sg->length += nbytes; 178d6d48196SJens Axboe } else { 179d6d48196SJens Axboe new_segment: 180d6d48196SJens Axboe if (!sg) 181d6d48196SJens Axboe sg = sglist; 182d6d48196SJens Axboe else { 183d6d48196SJens Axboe /* 184d6d48196SJens Axboe * If the driver previously mapped a shorter 185d6d48196SJens Axboe * list, we could see a termination bit 186d6d48196SJens Axboe * prematurely unless it fully inits the sg 187d6d48196SJens Axboe * table on each mapping. We KNOW that there 188d6d48196SJens Axboe * must be more entries here or the driver 189d6d48196SJens Axboe * would be buggy, so force clear the 190d6d48196SJens Axboe * termination bit to avoid doing a full 191d6d48196SJens Axboe * sg_init_table() in drivers for each command. 192d6d48196SJens Axboe */ 193d6d48196SJens Axboe sg->page_link &= ~0x02; 194d6d48196SJens Axboe sg = sg_next(sg); 195d6d48196SJens Axboe } 196d6d48196SJens Axboe 197d6d48196SJens Axboe sg_set_page(sg, bvec->bv_page, nbytes, bvec->bv_offset); 198d6d48196SJens Axboe nsegs++; 199d6d48196SJens Axboe } 200d6d48196SJens Axboe bvprv = bvec; 201d6d48196SJens Axboe } /* segments in rq */ 202d6d48196SJens Axboe 203f18573abSFUJITA Tomonori 204f18573abSFUJITA Tomonori if (unlikely(rq->cmd_flags & REQ_COPY_USER) && 205f18573abSFUJITA Tomonori (rq->data_len & q->dma_pad_mask)) { 206f18573abSFUJITA Tomonori unsigned int pad_len = (q->dma_pad_mask & ~rq->data_len) + 1; 207f18573abSFUJITA Tomonori 208f18573abSFUJITA Tomonori sg->length += pad_len; 209f18573abSFUJITA Tomonori rq->extra_len += pad_len; 210f18573abSFUJITA Tomonori } 211f18573abSFUJITA Tomonori 2122fb98e84STejun Heo if (q->dma_drain_size && q->dma_drain_needed(rq)) { 213db0a2e00STejun Heo if (rq->cmd_flags & REQ_RW) 214db0a2e00STejun Heo memset(q->dma_drain_buffer, 0, q->dma_drain_size); 215db0a2e00STejun Heo 216d6d48196SJens Axboe sg->page_link &= ~0x02; 217d6d48196SJens Axboe sg = sg_next(sg); 218d6d48196SJens Axboe sg_set_page(sg, virt_to_page(q->dma_drain_buffer), 219d6d48196SJens Axboe q->dma_drain_size, 220d6d48196SJens Axboe ((unsigned long)q->dma_drain_buffer) & 221d6d48196SJens Axboe (PAGE_SIZE - 1)); 222d6d48196SJens Axboe nsegs++; 2237a85f889SFUJITA Tomonori rq->extra_len += q->dma_drain_size; 224d6d48196SJens Axboe } 225d6d48196SJens Axboe 226d6d48196SJens Axboe if (sg) 227d6d48196SJens Axboe sg_mark_end(sg); 228d6d48196SJens Axboe 229d6d48196SJens Axboe return nsegs; 230d6d48196SJens Axboe } 231d6d48196SJens Axboe EXPORT_SYMBOL(blk_rq_map_sg); 232d6d48196SJens Axboe 233d6d48196SJens Axboe static inline int ll_new_mergeable(struct request_queue *q, 234d6d48196SJens Axboe struct request *req, 235d6d48196SJens Axboe struct bio *bio) 236d6d48196SJens Axboe { 237d6d48196SJens Axboe int nr_phys_segs = bio_phys_segments(q, bio); 238d6d48196SJens Axboe 239d6d48196SJens Axboe if (req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) { 240d6d48196SJens Axboe req->cmd_flags |= REQ_NOMERGE; 241d6d48196SJens Axboe if (req == q->last_merge) 242d6d48196SJens Axboe q->last_merge = NULL; 243d6d48196SJens Axboe return 0; 244d6d48196SJens Axboe } 245d6d48196SJens Axboe 246d6d48196SJens Axboe /* 247d6d48196SJens Axboe * A hw segment is just getting larger, bump just the phys 248d6d48196SJens Axboe * counter. 249d6d48196SJens Axboe */ 250d6d48196SJens Axboe req->nr_phys_segments += nr_phys_segs; 251d6d48196SJens Axboe return 1; 252d6d48196SJens Axboe } 253d6d48196SJens Axboe 254d6d48196SJens Axboe static inline int ll_new_hw_segment(struct request_queue *q, 255d6d48196SJens Axboe struct request *req, 256d6d48196SJens Axboe struct bio *bio) 257d6d48196SJens Axboe { 258d6d48196SJens Axboe int nr_hw_segs = bio_hw_segments(q, bio); 259d6d48196SJens Axboe int nr_phys_segs = bio_phys_segments(q, bio); 260d6d48196SJens Axboe 261d6d48196SJens Axboe if (req->nr_hw_segments + nr_hw_segs > q->max_hw_segments 262d6d48196SJens Axboe || req->nr_phys_segments + nr_phys_segs > q->max_phys_segments) { 263d6d48196SJens Axboe req->cmd_flags |= REQ_NOMERGE; 264d6d48196SJens Axboe if (req == q->last_merge) 265d6d48196SJens Axboe q->last_merge = NULL; 266d6d48196SJens Axboe return 0; 267d6d48196SJens Axboe } 268d6d48196SJens Axboe 269d6d48196SJens Axboe /* 270d6d48196SJens Axboe * This will form the start of a new hw segment. Bump both 271d6d48196SJens Axboe * counters. 272d6d48196SJens Axboe */ 273d6d48196SJens Axboe req->nr_hw_segments += nr_hw_segs; 274d6d48196SJens Axboe req->nr_phys_segments += nr_phys_segs; 275d6d48196SJens Axboe return 1; 276d6d48196SJens Axboe } 277d6d48196SJens Axboe 278d6d48196SJens Axboe int ll_back_merge_fn(struct request_queue *q, struct request *req, 279d6d48196SJens Axboe struct bio *bio) 280d6d48196SJens Axboe { 281d6d48196SJens Axboe unsigned short max_sectors; 282d6d48196SJens Axboe 283d6d48196SJens Axboe if (unlikely(blk_pc_request(req))) 284d6d48196SJens Axboe max_sectors = q->max_hw_sectors; 285d6d48196SJens Axboe else 286d6d48196SJens Axboe max_sectors = q->max_sectors; 287d6d48196SJens Axboe 288d6d48196SJens Axboe if (req->nr_sectors + bio_sectors(bio) > max_sectors) { 289d6d48196SJens Axboe req->cmd_flags |= REQ_NOMERGE; 290d6d48196SJens Axboe if (req == q->last_merge) 291d6d48196SJens Axboe q->last_merge = NULL; 292d6d48196SJens Axboe return 0; 293d6d48196SJens Axboe } 2942cdf79caSJens Axboe if (!bio_flagged(req->biotail, BIO_SEG_VALID)) 295d6d48196SJens Axboe blk_recount_segments(q, req->biotail); 2962cdf79caSJens Axboe if (!bio_flagged(bio, BIO_SEG_VALID)) 297d6d48196SJens Axboe blk_recount_segments(q, bio); 298d6d48196SJens Axboe 299d6d48196SJens Axboe return ll_new_hw_segment(q, req, bio); 300d6d48196SJens Axboe } 301d6d48196SJens Axboe 302d6d48196SJens Axboe int ll_front_merge_fn(struct request_queue *q, struct request *req, 303d6d48196SJens Axboe struct bio *bio) 304d6d48196SJens Axboe { 305d6d48196SJens Axboe unsigned short max_sectors; 306d6d48196SJens Axboe 307d6d48196SJens Axboe if (unlikely(blk_pc_request(req))) 308d6d48196SJens Axboe max_sectors = q->max_hw_sectors; 309d6d48196SJens Axboe else 310d6d48196SJens Axboe max_sectors = q->max_sectors; 311d6d48196SJens Axboe 312d6d48196SJens Axboe 313d6d48196SJens Axboe if (req->nr_sectors + bio_sectors(bio) > max_sectors) { 314d6d48196SJens Axboe req->cmd_flags |= REQ_NOMERGE; 315d6d48196SJens Axboe if (req == q->last_merge) 316d6d48196SJens Axboe q->last_merge = NULL; 317d6d48196SJens Axboe return 0; 318d6d48196SJens Axboe } 3192cdf79caSJens Axboe if (!bio_flagged(bio, BIO_SEG_VALID)) 320d6d48196SJens Axboe blk_recount_segments(q, bio); 3212cdf79caSJens Axboe if (!bio_flagged(req->bio, BIO_SEG_VALID)) 322d6d48196SJens Axboe blk_recount_segments(q, req->bio); 323d6d48196SJens Axboe 324d6d48196SJens Axboe return ll_new_hw_segment(q, req, bio); 325d6d48196SJens Axboe } 326d6d48196SJens Axboe 327d6d48196SJens Axboe static int ll_merge_requests_fn(struct request_queue *q, struct request *req, 328d6d48196SJens Axboe struct request *next) 329d6d48196SJens Axboe { 330d6d48196SJens Axboe int total_phys_segments; 331d6d48196SJens Axboe int total_hw_segments; 332d6d48196SJens Axboe 333d6d48196SJens Axboe /* 334d6d48196SJens Axboe * First check if the either of the requests are re-queued 335d6d48196SJens Axboe * requests. Can't merge them if they are. 336d6d48196SJens Axboe */ 337d6d48196SJens Axboe if (req->special || next->special) 338d6d48196SJens Axboe return 0; 339d6d48196SJens Axboe 340d6d48196SJens Axboe /* 341d6d48196SJens Axboe * Will it become too large? 342d6d48196SJens Axboe */ 343d6d48196SJens Axboe if ((req->nr_sectors + next->nr_sectors) > q->max_sectors) 344d6d48196SJens Axboe return 0; 345d6d48196SJens Axboe 346d6d48196SJens Axboe total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; 347d6d48196SJens Axboe if (blk_phys_contig_segment(q, req->biotail, next->bio)) 348d6d48196SJens Axboe total_phys_segments--; 349d6d48196SJens Axboe 350d6d48196SJens Axboe if (total_phys_segments > q->max_phys_segments) 351d6d48196SJens Axboe return 0; 352d6d48196SJens Axboe 353d6d48196SJens Axboe total_hw_segments = req->nr_hw_segments + next->nr_hw_segments; 354d6d48196SJens Axboe 355d6d48196SJens Axboe if (total_hw_segments > q->max_hw_segments) 356d6d48196SJens Axboe return 0; 357d6d48196SJens Axboe 358d6d48196SJens Axboe /* Merge is OK... */ 359d6d48196SJens Axboe req->nr_phys_segments = total_phys_segments; 360d6d48196SJens Axboe req->nr_hw_segments = total_hw_segments; 361d6d48196SJens Axboe return 1; 362d6d48196SJens Axboe } 363d6d48196SJens Axboe 364d6d48196SJens Axboe /* 365d6d48196SJens Axboe * Has to be called with the request spinlock acquired 366d6d48196SJens Axboe */ 367d6d48196SJens Axboe static int attempt_merge(struct request_queue *q, struct request *req, 368d6d48196SJens Axboe struct request *next) 369d6d48196SJens Axboe { 370d6d48196SJens Axboe if (!rq_mergeable(req) || !rq_mergeable(next)) 371d6d48196SJens Axboe return 0; 372d6d48196SJens Axboe 373d6d48196SJens Axboe /* 374d6d48196SJens Axboe * not contiguous 375d6d48196SJens Axboe */ 376d6d48196SJens Axboe if (req->sector + req->nr_sectors != next->sector) 377d6d48196SJens Axboe return 0; 378d6d48196SJens Axboe 379d6d48196SJens Axboe if (rq_data_dir(req) != rq_data_dir(next) 380d6d48196SJens Axboe || req->rq_disk != next->rq_disk 381d6d48196SJens Axboe || next->special) 382d6d48196SJens Axboe return 0; 383d6d48196SJens Axboe 3847ba1ba12SMartin K. Petersen if (blk_integrity_rq(req) != blk_integrity_rq(next)) 3857ba1ba12SMartin K. Petersen return 0; 3867ba1ba12SMartin K. Petersen 387d6d48196SJens Axboe /* 388d6d48196SJens Axboe * If we are allowed to merge, then append bio list 389d6d48196SJens Axboe * from next to rq and release next. merge_requests_fn 390d6d48196SJens Axboe * will have updated segment counts, update sector 391d6d48196SJens Axboe * counts here. 392d6d48196SJens Axboe */ 393d6d48196SJens Axboe if (!ll_merge_requests_fn(q, req, next)) 394d6d48196SJens Axboe return 0; 395d6d48196SJens Axboe 396d6d48196SJens Axboe /* 397d6d48196SJens Axboe * At this point we have either done a back merge 398d6d48196SJens Axboe * or front merge. We need the smaller start_time of 399d6d48196SJens Axboe * the merged requests to be the current request 400d6d48196SJens Axboe * for accounting purposes. 401d6d48196SJens Axboe */ 402d6d48196SJens Axboe if (time_after(req->start_time, next->start_time)) 403d6d48196SJens Axboe req->start_time = next->start_time; 404d6d48196SJens Axboe 405d6d48196SJens Axboe req->biotail->bi_next = next->bio; 406d6d48196SJens Axboe req->biotail = next->biotail; 407d6d48196SJens Axboe 408d6d48196SJens Axboe req->nr_sectors = req->hard_nr_sectors += next->hard_nr_sectors; 409d6d48196SJens Axboe 410d6d48196SJens Axboe elv_merge_requests(q, req, next); 411d6d48196SJens Axboe 412d6d48196SJens Axboe if (req->rq_disk) { 4136f2576afSJerome Marchand struct hd_struct *part 4146f2576afSJerome Marchand = get_part(req->rq_disk, req->sector); 415d6d48196SJens Axboe disk_round_stats(req->rq_disk); 416d6d48196SJens Axboe req->rq_disk->in_flight--; 4176f2576afSJerome Marchand if (part) { 4186f2576afSJerome Marchand part_round_stats(part); 4196f2576afSJerome Marchand part->in_flight--; 4206f2576afSJerome Marchand } 421d6d48196SJens Axboe } 422d6d48196SJens Axboe 423d6d48196SJens Axboe req->ioprio = ioprio_best(req->ioprio, next->ioprio); 424d6d48196SJens Axboe 425d6d48196SJens Axboe __blk_put_request(q, next); 426d6d48196SJens Axboe return 1; 427d6d48196SJens Axboe } 428d6d48196SJens Axboe 429d6d48196SJens Axboe int attempt_back_merge(struct request_queue *q, struct request *rq) 430d6d48196SJens Axboe { 431d6d48196SJens Axboe struct request *next = elv_latter_request(q, rq); 432d6d48196SJens Axboe 433d6d48196SJens Axboe if (next) 434d6d48196SJens Axboe return attempt_merge(q, rq, next); 435d6d48196SJens Axboe 436d6d48196SJens Axboe return 0; 437d6d48196SJens Axboe } 438d6d48196SJens Axboe 439d6d48196SJens Axboe int attempt_front_merge(struct request_queue *q, struct request *rq) 440d6d48196SJens Axboe { 441d6d48196SJens Axboe struct request *prev = elv_former_request(q, rq); 442d6d48196SJens Axboe 443d6d48196SJens Axboe if (prev) 444d6d48196SJens Axboe return attempt_merge(q, prev, rq); 445d6d48196SJens Axboe 446d6d48196SJens Axboe return 0; 447d6d48196SJens Axboe } 448