1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to segment and merge handling 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include <trace/events/block.h> 12 13 #include "blk.h" 14 15 static inline bool bio_will_gap(struct request_queue *q, 16 struct request *prev_rq, struct bio *prev, struct bio *next) 17 { 18 struct bio_vec pb, nb; 19 20 if (!bio_has_data(prev) || !queue_virt_boundary(q)) 21 return false; 22 23 /* 24 * Don't merge if the 1st bio starts with non-zero offset, otherwise it 25 * is quite difficult to respect the sg gap limit. We work hard to 26 * merge a huge number of small single bios in case of mkfs. 27 */ 28 if (prev_rq) 29 bio_get_first_bvec(prev_rq->bio, &pb); 30 else 31 bio_get_first_bvec(prev, &pb); 32 if (pb.bv_offset & queue_virt_boundary(q)) 33 return true; 34 35 /* 36 * We don't need to worry about the situation that the merged segment 37 * ends in unaligned virt boundary: 38 * 39 * - if 'pb' ends aligned, the merged segment ends aligned 40 * - if 'pb' ends unaligned, the next bio must include 41 * one single bvec of 'nb', otherwise the 'nb' can't 42 * merge with 'pb' 43 */ 44 bio_get_last_bvec(prev, &pb); 45 bio_get_first_bvec(next, &nb); 46 if (biovec_phys_mergeable(q, &pb, &nb)) 47 return false; 48 return __bvec_gap_to_prev(q, &pb, nb.bv_offset); 49 } 50 51 static inline bool req_gap_back_merge(struct request *req, struct bio *bio) 52 { 53 return bio_will_gap(req->q, req, req->biotail, bio); 54 } 55 56 static inline bool req_gap_front_merge(struct request *req, struct bio *bio) 57 { 58 return bio_will_gap(req->q, NULL, bio, req->bio); 59 } 60 61 static struct bio *blk_bio_discard_split(struct request_queue *q, 62 struct bio *bio, 63 struct bio_set *bs, 64 unsigned *nsegs) 65 { 66 unsigned int max_discard_sectors, granularity; 67 int alignment; 68 sector_t tmp; 69 unsigned split_sectors; 70 71 *nsegs = 1; 72 73 /* Zero-sector (unknown) and one-sector granularities are the same. */ 74 granularity = max(q->limits.discard_granularity >> 9, 1U); 75 76 max_discard_sectors = min(q->limits.max_discard_sectors, 77 bio_allowed_max_sectors(q)); 78 max_discard_sectors -= max_discard_sectors % granularity; 79 80 if (unlikely(!max_discard_sectors)) { 81 /* XXX: warn */ 82 return NULL; 83 } 84 85 if (bio_sectors(bio) <= max_discard_sectors) 86 return NULL; 87 88 split_sectors = max_discard_sectors; 89 90 /* 91 * If the next starting sector would be misaligned, stop the discard at 92 * the previous aligned sector. 93 */ 94 alignment = (q->limits.discard_alignment >> 9) % granularity; 95 96 tmp = bio->bi_iter.bi_sector + split_sectors - alignment; 97 tmp = sector_div(tmp, granularity); 98 99 if (split_sectors > tmp) 100 split_sectors -= tmp; 101 102 return bio_split(bio, split_sectors, GFP_NOIO, bs); 103 } 104 105 static struct bio *blk_bio_write_zeroes_split(struct request_queue *q, 106 struct bio *bio, struct bio_set *bs, unsigned *nsegs) 107 { 108 *nsegs = 0; 109 110 if (!q->limits.max_write_zeroes_sectors) 111 return NULL; 112 113 if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors) 114 return NULL; 115 116 return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs); 117 } 118 119 static struct bio *blk_bio_write_same_split(struct request_queue *q, 120 struct bio *bio, 121 struct bio_set *bs, 122 unsigned *nsegs) 123 { 124 *nsegs = 1; 125 126 if (!q->limits.max_write_same_sectors) 127 return NULL; 128 129 if (bio_sectors(bio) <= q->limits.max_write_same_sectors) 130 return NULL; 131 132 return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs); 133 } 134 135 static inline unsigned get_max_io_size(struct request_queue *q, 136 struct bio *bio) 137 { 138 unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector); 139 unsigned mask = queue_logical_block_size(q) - 1; 140 141 /* aligned to logical block size */ 142 sectors &= ~(mask >> 9); 143 144 return sectors; 145 } 146 147 static unsigned get_max_segment_size(const struct request_queue *q, 148 unsigned offset) 149 { 150 unsigned long mask = queue_segment_boundary(q); 151 152 /* default segment boundary mask means no boundary limit */ 153 if (mask == BLK_SEG_BOUNDARY_MASK) 154 return queue_max_segment_size(q); 155 156 return min_t(unsigned long, mask - (mask & offset) + 1, 157 queue_max_segment_size(q)); 158 } 159 160 /* 161 * Split the bvec @bv into segments, and update all kinds of 162 * variables. 163 */ 164 static bool bvec_split_segs(const struct request_queue *q, 165 const struct bio_vec *bv, unsigned *nsegs, 166 unsigned *sectors, unsigned max_segs) 167 { 168 unsigned len = bv->bv_len; 169 unsigned total_len = 0; 170 unsigned new_nsegs = 0, seg_size = 0; 171 172 /* 173 * Multi-page bvec may be too big to hold in one segment, so the 174 * current bvec has to be splitted as multiple segments. 175 */ 176 while (len && new_nsegs + *nsegs < max_segs) { 177 seg_size = get_max_segment_size(q, bv->bv_offset + total_len); 178 seg_size = min(seg_size, len); 179 180 new_nsegs++; 181 total_len += seg_size; 182 len -= seg_size; 183 184 if ((bv->bv_offset + total_len) & queue_virt_boundary(q)) 185 break; 186 } 187 188 if (new_nsegs) { 189 *nsegs += new_nsegs; 190 if (sectors) 191 *sectors += total_len >> 9; 192 } 193 194 /* split in the middle of the bvec if len != 0 */ 195 return !!len; 196 } 197 198 static struct bio *blk_bio_segment_split(struct request_queue *q, 199 struct bio *bio, 200 struct bio_set *bs, 201 unsigned *segs) 202 { 203 struct bio_vec bv, bvprv, *bvprvp = NULL; 204 struct bvec_iter iter; 205 unsigned nsegs = 0, sectors = 0; 206 const unsigned max_sectors = get_max_io_size(q, bio); 207 const unsigned max_segs = queue_max_segments(q); 208 209 bio_for_each_bvec(bv, bio, iter) { 210 /* 211 * If the queue doesn't support SG gaps and adding this 212 * offset would create a gap, disallow it. 213 */ 214 if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) 215 goto split; 216 217 if (sectors + (bv.bv_len >> 9) > max_sectors) { 218 /* 219 * Consider this a new segment if we're splitting in 220 * the middle of this vector. 221 */ 222 if (nsegs < max_segs && 223 sectors < max_sectors) { 224 /* split in the middle of bvec */ 225 bv.bv_len = (max_sectors - sectors) << 9; 226 bvec_split_segs(q, &bv, &nsegs, 227 §ors, max_segs); 228 } 229 goto split; 230 } 231 232 if (nsegs == max_segs) 233 goto split; 234 235 bvprv = bv; 236 bvprvp = &bvprv; 237 238 if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) { 239 nsegs++; 240 sectors += bv.bv_len >> 9; 241 } else if (bvec_split_segs(q, &bv, &nsegs, §ors, 242 max_segs)) { 243 goto split; 244 } 245 } 246 247 *segs = nsegs; 248 return NULL; 249 split: 250 *segs = nsegs; 251 return bio_split(bio, sectors, GFP_NOIO, bs); 252 } 253 254 void __blk_queue_split(struct request_queue *q, struct bio **bio, 255 unsigned int *nr_segs) 256 { 257 struct bio *split; 258 259 switch (bio_op(*bio)) { 260 case REQ_OP_DISCARD: 261 case REQ_OP_SECURE_ERASE: 262 split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs); 263 break; 264 case REQ_OP_WRITE_ZEROES: 265 split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split, 266 nr_segs); 267 break; 268 case REQ_OP_WRITE_SAME: 269 split = blk_bio_write_same_split(q, *bio, &q->bio_split, 270 nr_segs); 271 break; 272 default: 273 split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs); 274 break; 275 } 276 277 if (split) { 278 /* there isn't chance to merge the splitted bio */ 279 split->bi_opf |= REQ_NOMERGE; 280 281 /* 282 * Since we're recursing into make_request here, ensure 283 * that we mark this bio as already having entered the queue. 284 * If not, and the queue is going away, we can get stuck 285 * forever on waiting for the queue reference to drop. But 286 * that will never happen, as we're already holding a 287 * reference to it. 288 */ 289 bio_set_flag(*bio, BIO_QUEUE_ENTERED); 290 291 bio_chain(split, *bio); 292 trace_block_split(q, split, (*bio)->bi_iter.bi_sector); 293 generic_make_request(*bio); 294 *bio = split; 295 } 296 } 297 298 void blk_queue_split(struct request_queue *q, struct bio **bio) 299 { 300 unsigned int nr_segs; 301 302 __blk_queue_split(q, bio, &nr_segs); 303 } 304 EXPORT_SYMBOL(blk_queue_split); 305 306 unsigned int blk_recalc_rq_segments(struct request *rq) 307 { 308 unsigned int nr_phys_segs = 0; 309 struct req_iterator iter; 310 struct bio_vec bv; 311 312 if (!rq->bio) 313 return 0; 314 315 switch (bio_op(rq->bio)) { 316 case REQ_OP_DISCARD: 317 case REQ_OP_SECURE_ERASE: 318 case REQ_OP_WRITE_ZEROES: 319 return 0; 320 case REQ_OP_WRITE_SAME: 321 return 1; 322 } 323 324 rq_for_each_bvec(bv, rq, iter) 325 bvec_split_segs(rq->q, &bv, &nr_phys_segs, NULL, UINT_MAX); 326 return nr_phys_segs; 327 } 328 329 static inline struct scatterlist *blk_next_sg(struct scatterlist **sg, 330 struct scatterlist *sglist) 331 { 332 if (!*sg) 333 return sglist; 334 335 /* 336 * If the driver previously mapped a shorter list, we could see a 337 * termination bit prematurely unless it fully inits the sg table 338 * on each mapping. We KNOW that there must be more entries here 339 * or the driver would be buggy, so force clear the termination bit 340 * to avoid doing a full sg_init_table() in drivers for each command. 341 */ 342 sg_unmark_end(*sg); 343 return sg_next(*sg); 344 } 345 346 static unsigned blk_bvec_map_sg(struct request_queue *q, 347 struct bio_vec *bvec, struct scatterlist *sglist, 348 struct scatterlist **sg) 349 { 350 unsigned nbytes = bvec->bv_len; 351 unsigned nsegs = 0, total = 0; 352 353 while (nbytes > 0) { 354 unsigned offset = bvec->bv_offset + total; 355 unsigned len = min(get_max_segment_size(q, offset), nbytes); 356 struct page *page = bvec->bv_page; 357 358 /* 359 * Unfortunately a fair number of drivers barf on scatterlists 360 * that have an offset larger than PAGE_SIZE, despite other 361 * subsystems dealing with that invariant just fine. For now 362 * stick to the legacy format where we never present those from 363 * the block layer, but the code below should be removed once 364 * these offenders (mostly MMC/SD drivers) are fixed. 365 */ 366 page += (offset >> PAGE_SHIFT); 367 offset &= ~PAGE_MASK; 368 369 *sg = blk_next_sg(sg, sglist); 370 sg_set_page(*sg, page, len, offset); 371 372 total += len; 373 nbytes -= len; 374 nsegs++; 375 } 376 377 return nsegs; 378 } 379 380 static inline int __blk_bvec_map_sg(struct bio_vec bv, 381 struct scatterlist *sglist, struct scatterlist **sg) 382 { 383 *sg = blk_next_sg(sg, sglist); 384 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); 385 return 1; 386 } 387 388 /* only try to merge bvecs into one sg if they are from two bios */ 389 static inline bool 390 __blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec, 391 struct bio_vec *bvprv, struct scatterlist **sg) 392 { 393 394 int nbytes = bvec->bv_len; 395 396 if (!*sg) 397 return false; 398 399 if ((*sg)->length + nbytes > queue_max_segment_size(q)) 400 return false; 401 402 if (!biovec_phys_mergeable(q, bvprv, bvec)) 403 return false; 404 405 (*sg)->length += nbytes; 406 407 return true; 408 } 409 410 static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio, 411 struct scatterlist *sglist, 412 struct scatterlist **sg) 413 { 414 struct bio_vec uninitialized_var(bvec), bvprv = { NULL }; 415 struct bvec_iter iter; 416 int nsegs = 0; 417 bool new_bio = false; 418 419 for_each_bio(bio) { 420 bio_for_each_bvec(bvec, bio, iter) { 421 /* 422 * Only try to merge bvecs from two bios given we 423 * have done bio internal merge when adding pages 424 * to bio 425 */ 426 if (new_bio && 427 __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg)) 428 goto next_bvec; 429 430 if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE) 431 nsegs += __blk_bvec_map_sg(bvec, sglist, sg); 432 else 433 nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg); 434 next_bvec: 435 new_bio = false; 436 } 437 if (likely(bio->bi_iter.bi_size)) { 438 bvprv = bvec; 439 new_bio = true; 440 } 441 } 442 443 return nsegs; 444 } 445 446 /* 447 * map a request to scatterlist, return number of sg entries setup. Caller 448 * must make sure sg can hold rq->nr_phys_segments entries 449 */ 450 int blk_rq_map_sg(struct request_queue *q, struct request *rq, 451 struct scatterlist *sglist) 452 { 453 struct scatterlist *sg = NULL; 454 int nsegs = 0; 455 456 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) 457 nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, &sg); 458 else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME) 459 nsegs = __blk_bvec_map_sg(bio_iovec(rq->bio), sglist, &sg); 460 else if (rq->bio) 461 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg); 462 463 if (unlikely(rq->rq_flags & RQF_COPY_USER) && 464 (blk_rq_bytes(rq) & q->dma_pad_mask)) { 465 unsigned int pad_len = 466 (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1; 467 468 sg->length += pad_len; 469 rq->extra_len += pad_len; 470 } 471 472 if (q->dma_drain_size && q->dma_drain_needed(rq)) { 473 if (op_is_write(req_op(rq))) 474 memset(q->dma_drain_buffer, 0, q->dma_drain_size); 475 476 sg_unmark_end(sg); 477 sg = sg_next(sg); 478 sg_set_page(sg, virt_to_page(q->dma_drain_buffer), 479 q->dma_drain_size, 480 ((unsigned long)q->dma_drain_buffer) & 481 (PAGE_SIZE - 1)); 482 nsegs++; 483 rq->extra_len += q->dma_drain_size; 484 } 485 486 if (sg) 487 sg_mark_end(sg); 488 489 /* 490 * Something must have been wrong if the figured number of 491 * segment is bigger than number of req's physical segments 492 */ 493 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq)); 494 495 return nsegs; 496 } 497 EXPORT_SYMBOL(blk_rq_map_sg); 498 499 static inline int ll_new_hw_segment(struct request *req, struct bio *bio, 500 unsigned int nr_phys_segs) 501 { 502 if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(req->q)) 503 goto no_merge; 504 505 if (blk_integrity_merge_bio(req->q, req, bio) == false) 506 goto no_merge; 507 508 /* 509 * This will form the start of a new hw segment. Bump both 510 * counters. 511 */ 512 req->nr_phys_segments += nr_phys_segs; 513 return 1; 514 515 no_merge: 516 req_set_nomerge(req->q, req); 517 return 0; 518 } 519 520 int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) 521 { 522 if (req_gap_back_merge(req, bio)) 523 return 0; 524 if (blk_integrity_rq(req) && 525 integrity_req_gap_back_merge(req, bio)) 526 return 0; 527 if (blk_rq_sectors(req) + bio_sectors(bio) > 528 blk_rq_get_max_sectors(req, blk_rq_pos(req))) { 529 req_set_nomerge(req->q, req); 530 return 0; 531 } 532 533 return ll_new_hw_segment(req, bio, nr_segs); 534 } 535 536 int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs) 537 { 538 if (req_gap_front_merge(req, bio)) 539 return 0; 540 if (blk_integrity_rq(req) && 541 integrity_req_gap_front_merge(req, bio)) 542 return 0; 543 if (blk_rq_sectors(req) + bio_sectors(bio) > 544 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) { 545 req_set_nomerge(req->q, req); 546 return 0; 547 } 548 549 return ll_new_hw_segment(req, bio, nr_segs); 550 } 551 552 static bool req_attempt_discard_merge(struct request_queue *q, struct request *req, 553 struct request *next) 554 { 555 unsigned short segments = blk_rq_nr_discard_segments(req); 556 557 if (segments >= queue_max_discard_segments(q)) 558 goto no_merge; 559 if (blk_rq_sectors(req) + bio_sectors(next->bio) > 560 blk_rq_get_max_sectors(req, blk_rq_pos(req))) 561 goto no_merge; 562 563 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next); 564 return true; 565 no_merge: 566 req_set_nomerge(q, req); 567 return false; 568 } 569 570 static int ll_merge_requests_fn(struct request_queue *q, struct request *req, 571 struct request *next) 572 { 573 int total_phys_segments; 574 575 if (req_gap_back_merge(req, next->bio)) 576 return 0; 577 578 /* 579 * Will it become too large? 580 */ 581 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) > 582 blk_rq_get_max_sectors(req, blk_rq_pos(req))) 583 return 0; 584 585 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments; 586 if (total_phys_segments > queue_max_segments(q)) 587 return 0; 588 589 if (blk_integrity_merge_rq(q, req, next) == false) 590 return 0; 591 592 /* Merge is OK... */ 593 req->nr_phys_segments = total_phys_segments; 594 return 1; 595 } 596 597 /** 598 * blk_rq_set_mixed_merge - mark a request as mixed merge 599 * @rq: request to mark as mixed merge 600 * 601 * Description: 602 * @rq is about to be mixed merged. Make sure the attributes 603 * which can be mixed are set in each bio and mark @rq as mixed 604 * merged. 605 */ 606 void blk_rq_set_mixed_merge(struct request *rq) 607 { 608 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; 609 struct bio *bio; 610 611 if (rq->rq_flags & RQF_MIXED_MERGE) 612 return; 613 614 /* 615 * @rq will no longer represent mixable attributes for all the 616 * contained bios. It will just track those of the first one. 617 * Distributes the attributs to each bio. 618 */ 619 for (bio = rq->bio; bio; bio = bio->bi_next) { 620 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) && 621 (bio->bi_opf & REQ_FAILFAST_MASK) != ff); 622 bio->bi_opf |= ff; 623 } 624 rq->rq_flags |= RQF_MIXED_MERGE; 625 } 626 627 static void blk_account_io_merge(struct request *req) 628 { 629 if (blk_do_io_stat(req)) { 630 struct hd_struct *part; 631 632 part_stat_lock(); 633 part = req->part; 634 635 part_dec_in_flight(req->q, part, rq_data_dir(req)); 636 637 hd_struct_put(part); 638 part_stat_unlock(); 639 } 640 } 641 /* 642 * Two cases of handling DISCARD merge: 643 * If max_discard_segments > 1, the driver takes every bio 644 * as a range and send them to controller together. The ranges 645 * needn't to be contiguous. 646 * Otherwise, the bios/requests will be handled as same as 647 * others which should be contiguous. 648 */ 649 static inline bool blk_discard_mergable(struct request *req) 650 { 651 if (req_op(req) == REQ_OP_DISCARD && 652 queue_max_discard_segments(req->q) > 1) 653 return true; 654 return false; 655 } 656 657 static enum elv_merge blk_try_req_merge(struct request *req, 658 struct request *next) 659 { 660 if (blk_discard_mergable(req)) 661 return ELEVATOR_DISCARD_MERGE; 662 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next)) 663 return ELEVATOR_BACK_MERGE; 664 665 return ELEVATOR_NO_MERGE; 666 } 667 668 /* 669 * For non-mq, this has to be called with the request spinlock acquired. 670 * For mq with scheduling, the appropriate queue wide lock should be held. 671 */ 672 static struct request *attempt_merge(struct request_queue *q, 673 struct request *req, struct request *next) 674 { 675 if (!rq_mergeable(req) || !rq_mergeable(next)) 676 return NULL; 677 678 if (req_op(req) != req_op(next)) 679 return NULL; 680 681 if (rq_data_dir(req) != rq_data_dir(next) 682 || req->rq_disk != next->rq_disk) 683 return NULL; 684 685 if (req_op(req) == REQ_OP_WRITE_SAME && 686 !blk_write_same_mergeable(req->bio, next->bio)) 687 return NULL; 688 689 /* 690 * Don't allow merge of different write hints, or for a hint with 691 * non-hint IO. 692 */ 693 if (req->write_hint != next->write_hint) 694 return NULL; 695 696 if (req->ioprio != next->ioprio) 697 return NULL; 698 699 /* 700 * If we are allowed to merge, then append bio list 701 * from next to rq and release next. merge_requests_fn 702 * will have updated segment counts, update sector 703 * counts here. Handle DISCARDs separately, as they 704 * have separate settings. 705 */ 706 707 switch (blk_try_req_merge(req, next)) { 708 case ELEVATOR_DISCARD_MERGE: 709 if (!req_attempt_discard_merge(q, req, next)) 710 return NULL; 711 break; 712 case ELEVATOR_BACK_MERGE: 713 if (!ll_merge_requests_fn(q, req, next)) 714 return NULL; 715 break; 716 default: 717 return NULL; 718 } 719 720 /* 721 * If failfast settings disagree or any of the two is already 722 * a mixed merge, mark both as mixed before proceeding. This 723 * makes sure that all involved bios have mixable attributes 724 * set properly. 725 */ 726 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) || 727 (req->cmd_flags & REQ_FAILFAST_MASK) != 728 (next->cmd_flags & REQ_FAILFAST_MASK)) { 729 blk_rq_set_mixed_merge(req); 730 blk_rq_set_mixed_merge(next); 731 } 732 733 /* 734 * At this point we have either done a back merge or front merge. We 735 * need the smaller start_time_ns of the merged requests to be the 736 * current request for accounting purposes. 737 */ 738 if (next->start_time_ns < req->start_time_ns) 739 req->start_time_ns = next->start_time_ns; 740 741 req->biotail->bi_next = next->bio; 742 req->biotail = next->biotail; 743 744 req->__data_len += blk_rq_bytes(next); 745 746 if (!blk_discard_mergable(req)) 747 elv_merge_requests(q, req, next); 748 749 /* 750 * 'next' is going away, so update stats accordingly 751 */ 752 blk_account_io_merge(next); 753 754 /* 755 * ownership of bio passed from next to req, return 'next' for 756 * the caller to free 757 */ 758 next->bio = NULL; 759 return next; 760 } 761 762 struct request *attempt_back_merge(struct request_queue *q, struct request *rq) 763 { 764 struct request *next = elv_latter_request(q, rq); 765 766 if (next) 767 return attempt_merge(q, rq, next); 768 769 return NULL; 770 } 771 772 struct request *attempt_front_merge(struct request_queue *q, struct request *rq) 773 { 774 struct request *prev = elv_former_request(q, rq); 775 776 if (prev) 777 return attempt_merge(q, prev, rq); 778 779 return NULL; 780 } 781 782 int blk_attempt_req_merge(struct request_queue *q, struct request *rq, 783 struct request *next) 784 { 785 struct request *free; 786 787 free = attempt_merge(q, rq, next); 788 if (free) { 789 blk_put_request(free); 790 return 1; 791 } 792 793 return 0; 794 } 795 796 bool blk_rq_merge_ok(struct request *rq, struct bio *bio) 797 { 798 if (!rq_mergeable(rq) || !bio_mergeable(bio)) 799 return false; 800 801 if (req_op(rq) != bio_op(bio)) 802 return false; 803 804 /* different data direction or already started, don't merge */ 805 if (bio_data_dir(bio) != rq_data_dir(rq)) 806 return false; 807 808 /* must be same device */ 809 if (rq->rq_disk != bio->bi_disk) 810 return false; 811 812 /* only merge integrity protected bio into ditto rq */ 813 if (blk_integrity_merge_bio(rq->q, rq, bio) == false) 814 return false; 815 816 /* must be using the same buffer */ 817 if (req_op(rq) == REQ_OP_WRITE_SAME && 818 !blk_write_same_mergeable(rq->bio, bio)) 819 return false; 820 821 /* 822 * Don't allow merge of different write hints, or for a hint with 823 * non-hint IO. 824 */ 825 if (rq->write_hint != bio->bi_write_hint) 826 return false; 827 828 if (rq->ioprio != bio_prio(bio)) 829 return false; 830 831 return true; 832 } 833 834 enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) 835 { 836 if (blk_discard_mergable(rq)) 837 return ELEVATOR_DISCARD_MERGE; 838 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) 839 return ELEVATOR_BACK_MERGE; 840 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) 841 return ELEVATOR_FRONT_MERGE; 842 return ELEVATOR_NO_MERGE; 843 } 844