1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> 4 */ 5 #include <linux/mm.h> 6 #include <linux/swap.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/uio.h> 10 #include <linux/iocontext.h> 11 #include <linux/slab.h> 12 #include <linux/init.h> 13 #include <linux/kernel.h> 14 #include <linux/export.h> 15 #include <linux/mempool.h> 16 #include <linux/workqueue.h> 17 #include <linux/cgroup.h> 18 #include <linux/blk-cgroup.h> 19 #include <linux/highmem.h> 20 #include <linux/sched/sysctl.h> 21 #include <linux/blk-crypto.h> 22 #include <linux/xarray.h> 23 24 #include <trace/events/block.h> 25 #include "blk.h" 26 #include "blk-rq-qos.h" 27 28 static struct biovec_slab { 29 int nr_vecs; 30 char *name; 31 struct kmem_cache *slab; 32 } bvec_slabs[] __read_mostly = { 33 { .nr_vecs = 16, .name = "biovec-16" }, 34 { .nr_vecs = 64, .name = "biovec-64" }, 35 { .nr_vecs = 128, .name = "biovec-128" }, 36 { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" }, 37 }; 38 39 static struct biovec_slab *biovec_slab(unsigned short nr_vecs) 40 { 41 switch (nr_vecs) { 42 /* smaller bios use inline vecs */ 43 case 5 ... 16: 44 return &bvec_slabs[0]; 45 case 17 ... 64: 46 return &bvec_slabs[1]; 47 case 65 ... 128: 48 return &bvec_slabs[2]; 49 case 129 ... BIO_MAX_VECS: 50 return &bvec_slabs[3]; 51 default: 52 BUG(); 53 return NULL; 54 } 55 } 56 57 /* 58 * fs_bio_set is the bio_set containing bio and iovec memory pools used by 59 * IO code that does not need private memory pools. 60 */ 61 struct bio_set fs_bio_set; 62 EXPORT_SYMBOL(fs_bio_set); 63 64 /* 65 * Our slab pool management 66 */ 67 struct bio_slab { 68 struct kmem_cache *slab; 69 unsigned int slab_ref; 70 unsigned int slab_size; 71 char name[8]; 72 }; 73 static DEFINE_MUTEX(bio_slab_lock); 74 static DEFINE_XARRAY(bio_slabs); 75 76 static struct bio_slab *create_bio_slab(unsigned int size) 77 { 78 struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL); 79 80 if (!bslab) 81 return NULL; 82 83 snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size); 84 bslab->slab = kmem_cache_create(bslab->name, size, 85 ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN, NULL); 86 if (!bslab->slab) 87 goto fail_alloc_slab; 88 89 bslab->slab_ref = 1; 90 bslab->slab_size = size; 91 92 if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL))) 93 return bslab; 94 95 kmem_cache_destroy(bslab->slab); 96 97 fail_alloc_slab: 98 kfree(bslab); 99 return NULL; 100 } 101 102 static inline unsigned int bs_bio_slab_size(struct bio_set *bs) 103 { 104 return bs->front_pad + sizeof(struct bio) + bs->back_pad; 105 } 106 107 static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) 108 { 109 unsigned int size = bs_bio_slab_size(bs); 110 struct bio_slab *bslab; 111 112 mutex_lock(&bio_slab_lock); 113 bslab = xa_load(&bio_slabs, size); 114 if (bslab) 115 bslab->slab_ref++; 116 else 117 bslab = create_bio_slab(size); 118 mutex_unlock(&bio_slab_lock); 119 120 if (bslab) 121 return bslab->slab; 122 return NULL; 123 } 124 125 static void bio_put_slab(struct bio_set *bs) 126 { 127 struct bio_slab *bslab = NULL; 128 unsigned int slab_size = bs_bio_slab_size(bs); 129 130 mutex_lock(&bio_slab_lock); 131 132 bslab = xa_load(&bio_slabs, slab_size); 133 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) 134 goto out; 135 136 WARN_ON_ONCE(bslab->slab != bs->bio_slab); 137 138 WARN_ON(!bslab->slab_ref); 139 140 if (--bslab->slab_ref) 141 goto out; 142 143 xa_erase(&bio_slabs, slab_size); 144 145 kmem_cache_destroy(bslab->slab); 146 kfree(bslab); 147 148 out: 149 mutex_unlock(&bio_slab_lock); 150 } 151 152 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) 153 { 154 BIO_BUG_ON(nr_vecs > BIO_MAX_VECS); 155 156 if (nr_vecs == BIO_MAX_VECS) 157 mempool_free(bv, pool); 158 else if (nr_vecs > BIO_INLINE_VECS) 159 kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); 160 } 161 162 /* 163 * Make the first allocation restricted and don't dump info on allocation 164 * failures, since we'll fall back to the mempool in case of failure. 165 */ 166 static inline gfp_t bvec_alloc_gfp(gfp_t gfp) 167 { 168 return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | 169 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 170 } 171 172 struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, 173 gfp_t gfp_mask) 174 { 175 struct biovec_slab *bvs = biovec_slab(*nr_vecs); 176 177 if (WARN_ON_ONCE(!bvs)) 178 return NULL; 179 180 /* 181 * Upgrade the nr_vecs request to take full advantage of the allocation. 182 * We also rely on this in the bvec_free path. 183 */ 184 *nr_vecs = bvs->nr_vecs; 185 186 /* 187 * Try a slab allocation first for all smaller allocations. If that 188 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool. 189 * The mempool is sized to handle up to BIO_MAX_VECS entries. 190 */ 191 if (*nr_vecs < BIO_MAX_VECS) { 192 struct bio_vec *bvl; 193 194 bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); 195 if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 196 return bvl; 197 *nr_vecs = BIO_MAX_VECS; 198 } 199 200 return mempool_alloc(pool, gfp_mask); 201 } 202 203 void bio_uninit(struct bio *bio) 204 { 205 #ifdef CONFIG_BLK_CGROUP 206 if (bio->bi_blkg) { 207 blkg_put(bio->bi_blkg); 208 bio->bi_blkg = NULL; 209 } 210 #endif 211 if (bio_integrity(bio)) 212 bio_integrity_free(bio); 213 214 bio_crypt_free_ctx(bio); 215 } 216 EXPORT_SYMBOL(bio_uninit); 217 218 static void bio_free(struct bio *bio) 219 { 220 struct bio_set *bs = bio->bi_pool; 221 void *p; 222 223 bio_uninit(bio); 224 225 if (bs) { 226 bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs); 227 228 /* 229 * If we have front padding, adjust the bio pointer before freeing 230 */ 231 p = bio; 232 p -= bs->front_pad; 233 234 mempool_free(p, &bs->bio_pool); 235 } else { 236 /* Bio was allocated by bio_kmalloc() */ 237 kfree(bio); 238 } 239 } 240 241 /* 242 * Users of this function have their own bio allocation. Subsequently, 243 * they must remember to pair any call to bio_init() with bio_uninit() 244 * when IO has completed, or when the bio is released. 245 */ 246 void bio_init(struct bio *bio, struct bio_vec *table, 247 unsigned short max_vecs) 248 { 249 memset(bio, 0, sizeof(*bio)); 250 atomic_set(&bio->__bi_remaining, 1); 251 atomic_set(&bio->__bi_cnt, 1); 252 253 bio->bi_io_vec = table; 254 bio->bi_max_vecs = max_vecs; 255 } 256 EXPORT_SYMBOL(bio_init); 257 258 /** 259 * bio_reset - reinitialize a bio 260 * @bio: bio to reset 261 * 262 * Description: 263 * After calling bio_reset(), @bio will be in the same state as a freshly 264 * allocated bio returned bio bio_alloc_bioset() - the only fields that are 265 * preserved are the ones that are initialized by bio_alloc_bioset(). See 266 * comment in struct bio. 267 */ 268 void bio_reset(struct bio *bio) 269 { 270 bio_uninit(bio); 271 memset(bio, 0, BIO_RESET_BYTES); 272 atomic_set(&bio->__bi_remaining, 1); 273 } 274 EXPORT_SYMBOL(bio_reset); 275 276 static struct bio *__bio_chain_endio(struct bio *bio) 277 { 278 struct bio *parent = bio->bi_private; 279 280 if (bio->bi_status && !parent->bi_status) 281 parent->bi_status = bio->bi_status; 282 bio_put(bio); 283 return parent; 284 } 285 286 static void bio_chain_endio(struct bio *bio) 287 { 288 bio_endio(__bio_chain_endio(bio)); 289 } 290 291 /** 292 * bio_chain - chain bio completions 293 * @bio: the target bio 294 * @parent: the parent bio of @bio 295 * 296 * The caller won't have a bi_end_io called when @bio completes - instead, 297 * @parent's bi_end_io won't be called until both @parent and @bio have 298 * completed; the chained bio will also be freed when it completes. 299 * 300 * The caller must not set bi_private or bi_end_io in @bio. 301 */ 302 void bio_chain(struct bio *bio, struct bio *parent) 303 { 304 BUG_ON(bio->bi_private || bio->bi_end_io); 305 306 bio->bi_private = parent; 307 bio->bi_end_io = bio_chain_endio; 308 bio_inc_remaining(parent); 309 } 310 EXPORT_SYMBOL(bio_chain); 311 312 static void bio_alloc_rescue(struct work_struct *work) 313 { 314 struct bio_set *bs = container_of(work, struct bio_set, rescue_work); 315 struct bio *bio; 316 317 while (1) { 318 spin_lock(&bs->rescue_lock); 319 bio = bio_list_pop(&bs->rescue_list); 320 spin_unlock(&bs->rescue_lock); 321 322 if (!bio) 323 break; 324 325 submit_bio_noacct(bio); 326 } 327 } 328 329 static void punt_bios_to_rescuer(struct bio_set *bs) 330 { 331 struct bio_list punt, nopunt; 332 struct bio *bio; 333 334 if (WARN_ON_ONCE(!bs->rescue_workqueue)) 335 return; 336 /* 337 * In order to guarantee forward progress we must punt only bios that 338 * were allocated from this bio_set; otherwise, if there was a bio on 339 * there for a stacking driver higher up in the stack, processing it 340 * could require allocating bios from this bio_set, and doing that from 341 * our own rescuer would be bad. 342 * 343 * Since bio lists are singly linked, pop them all instead of trying to 344 * remove from the middle of the list: 345 */ 346 347 bio_list_init(&punt); 348 bio_list_init(&nopunt); 349 350 while ((bio = bio_list_pop(¤t->bio_list[0]))) 351 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); 352 current->bio_list[0] = nopunt; 353 354 bio_list_init(&nopunt); 355 while ((bio = bio_list_pop(¤t->bio_list[1]))) 356 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); 357 current->bio_list[1] = nopunt; 358 359 spin_lock(&bs->rescue_lock); 360 bio_list_merge(&bs->rescue_list, &punt); 361 spin_unlock(&bs->rescue_lock); 362 363 queue_work(bs->rescue_workqueue, &bs->rescue_work); 364 } 365 366 /** 367 * bio_alloc_bioset - allocate a bio for I/O 368 * @gfp_mask: the GFP_* mask given to the slab allocator 369 * @nr_iovecs: number of iovecs to pre-allocate 370 * @bs: the bio_set to allocate from. 371 * 372 * Allocate a bio from the mempools in @bs. 373 * 374 * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to 375 * allocate a bio. This is due to the mempool guarantees. To make this work, 376 * callers must never allocate more than 1 bio at a time from the general pool. 377 * Callers that need to allocate more than 1 bio must always submit the 378 * previously allocated bio for IO before attempting to allocate a new one. 379 * Failure to do so can cause deadlocks under memory pressure. 380 * 381 * Note that when running under submit_bio_noacct() (i.e. any block driver), 382 * bios are not submitted until after you return - see the code in 383 * submit_bio_noacct() that converts recursion into iteration, to prevent 384 * stack overflows. 385 * 386 * This would normally mean allocating multiple bios under submit_bio_noacct() 387 * would be susceptible to deadlocks, but we have 388 * deadlock avoidance code that resubmits any blocked bios from a rescuer 389 * thread. 390 * 391 * However, we do not guarantee forward progress for allocations from other 392 * mempools. Doing multiple allocations from the same mempool under 393 * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad 394 * for per bio allocations. 395 * 396 * Returns: Pointer to new bio on success, NULL on failure. 397 */ 398 struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs, 399 struct bio_set *bs) 400 { 401 gfp_t saved_gfp = gfp_mask; 402 struct bio *bio; 403 void *p; 404 405 /* should not use nobvec bioset for nr_iovecs > 0 */ 406 if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0)) 407 return NULL; 408 409 /* 410 * submit_bio_noacct() converts recursion to iteration; this means if 411 * we're running beneath it, any bios we allocate and submit will not be 412 * submitted (and thus freed) until after we return. 413 * 414 * This exposes us to a potential deadlock if we allocate multiple bios 415 * from the same bio_set() while running underneath submit_bio_noacct(). 416 * If we were to allocate multiple bios (say a stacking block driver 417 * that was splitting bios), we would deadlock if we exhausted the 418 * mempool's reserve. 419 * 420 * We solve this, and guarantee forward progress, with a rescuer 421 * workqueue per bio_set. If we go to allocate and there are bios on 422 * current->bio_list, we first try the allocation without 423 * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be 424 * blocking to the rescuer workqueue before we retry with the original 425 * gfp_flags. 426 */ 427 if (current->bio_list && 428 (!bio_list_empty(¤t->bio_list[0]) || 429 !bio_list_empty(¤t->bio_list[1])) && 430 bs->rescue_workqueue) 431 gfp_mask &= ~__GFP_DIRECT_RECLAIM; 432 433 p = mempool_alloc(&bs->bio_pool, gfp_mask); 434 if (!p && gfp_mask != saved_gfp) { 435 punt_bios_to_rescuer(bs); 436 gfp_mask = saved_gfp; 437 p = mempool_alloc(&bs->bio_pool, gfp_mask); 438 } 439 if (unlikely(!p)) 440 return NULL; 441 442 bio = p + bs->front_pad; 443 if (nr_iovecs > BIO_INLINE_VECS) { 444 struct bio_vec *bvl = NULL; 445 446 bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); 447 if (!bvl && gfp_mask != saved_gfp) { 448 punt_bios_to_rescuer(bs); 449 gfp_mask = saved_gfp; 450 bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); 451 } 452 if (unlikely(!bvl)) 453 goto err_free; 454 455 bio_init(bio, bvl, nr_iovecs); 456 } else if (nr_iovecs) { 457 bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS); 458 } else { 459 bio_init(bio, NULL, 0); 460 } 461 462 bio->bi_pool = bs; 463 return bio; 464 465 err_free: 466 mempool_free(p, &bs->bio_pool); 467 return NULL; 468 } 469 EXPORT_SYMBOL(bio_alloc_bioset); 470 471 /** 472 * bio_kmalloc - kmalloc a bio for I/O 473 * @gfp_mask: the GFP_* mask given to the slab allocator 474 * @nr_iovecs: number of iovecs to pre-allocate 475 * 476 * Use kmalloc to allocate and initialize a bio. 477 * 478 * Returns: Pointer to new bio on success, NULL on failure. 479 */ 480 struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs) 481 { 482 struct bio *bio; 483 484 if (nr_iovecs > UIO_MAXIOV) 485 return NULL; 486 487 bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask); 488 if (unlikely(!bio)) 489 return NULL; 490 bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs); 491 bio->bi_pool = NULL; 492 return bio; 493 } 494 EXPORT_SYMBOL(bio_kmalloc); 495 496 void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start) 497 { 498 unsigned long flags; 499 struct bio_vec bv; 500 struct bvec_iter iter; 501 502 __bio_for_each_segment(bv, bio, iter, start) { 503 char *data = bvec_kmap_irq(&bv, &flags); 504 memset(data, 0, bv.bv_len); 505 flush_dcache_page(bv.bv_page); 506 bvec_kunmap_irq(data, &flags); 507 } 508 } 509 EXPORT_SYMBOL(zero_fill_bio_iter); 510 511 /** 512 * bio_truncate - truncate the bio to small size of @new_size 513 * @bio: the bio to be truncated 514 * @new_size: new size for truncating the bio 515 * 516 * Description: 517 * Truncate the bio to new size of @new_size. If bio_op(bio) is 518 * REQ_OP_READ, zero the truncated part. This function should only 519 * be used for handling corner cases, such as bio eod. 520 */ 521 void bio_truncate(struct bio *bio, unsigned new_size) 522 { 523 struct bio_vec bv; 524 struct bvec_iter iter; 525 unsigned int done = 0; 526 bool truncated = false; 527 528 if (new_size >= bio->bi_iter.bi_size) 529 return; 530 531 if (bio_op(bio) != REQ_OP_READ) 532 goto exit; 533 534 bio_for_each_segment(bv, bio, iter) { 535 if (done + bv.bv_len > new_size) { 536 unsigned offset; 537 538 if (!truncated) 539 offset = new_size - done; 540 else 541 offset = 0; 542 zero_user(bv.bv_page, offset, bv.bv_len - offset); 543 truncated = true; 544 } 545 done += bv.bv_len; 546 } 547 548 exit: 549 /* 550 * Don't touch bvec table here and make it really immutable, since 551 * fs bio user has to retrieve all pages via bio_for_each_segment_all 552 * in its .end_bio() callback. 553 * 554 * It is enough to truncate bio by updating .bi_size since we can make 555 * correct bvec with the updated .bi_size for drivers. 556 */ 557 bio->bi_iter.bi_size = new_size; 558 } 559 560 /** 561 * guard_bio_eod - truncate a BIO to fit the block device 562 * @bio: bio to truncate 563 * 564 * This allows us to do IO even on the odd last sectors of a device, even if the 565 * block size is some multiple of the physical sector size. 566 * 567 * We'll just truncate the bio to the size of the device, and clear the end of 568 * the buffer head manually. Truly out-of-range accesses will turn into actual 569 * I/O errors, this only handles the "we need to be able to do I/O at the final 570 * sector" case. 571 */ 572 void guard_bio_eod(struct bio *bio) 573 { 574 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); 575 576 if (!maxsector) 577 return; 578 579 /* 580 * If the *whole* IO is past the end of the device, 581 * let it through, and the IO layer will turn it into 582 * an EIO. 583 */ 584 if (unlikely(bio->bi_iter.bi_sector >= maxsector)) 585 return; 586 587 maxsector -= bio->bi_iter.bi_sector; 588 if (likely((bio->bi_iter.bi_size >> 9) <= maxsector)) 589 return; 590 591 bio_truncate(bio, maxsector << 9); 592 } 593 594 /** 595 * bio_put - release a reference to a bio 596 * @bio: bio to release reference to 597 * 598 * Description: 599 * Put a reference to a &struct bio, either one you have gotten with 600 * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it. 601 **/ 602 void bio_put(struct bio *bio) 603 { 604 if (!bio_flagged(bio, BIO_REFFED)) 605 bio_free(bio); 606 else { 607 BIO_BUG_ON(!atomic_read(&bio->__bi_cnt)); 608 609 /* 610 * last put frees it 611 */ 612 if (atomic_dec_and_test(&bio->__bi_cnt)) 613 bio_free(bio); 614 } 615 } 616 EXPORT_SYMBOL(bio_put); 617 618 /** 619 * __bio_clone_fast - clone a bio that shares the original bio's biovec 620 * @bio: destination bio 621 * @bio_src: bio to clone 622 * 623 * Clone a &bio. Caller will own the returned bio, but not 624 * the actual data it points to. Reference count of returned 625 * bio will be one. 626 * 627 * Caller must ensure that @bio_src is not freed before @bio. 628 */ 629 void __bio_clone_fast(struct bio *bio, struct bio *bio_src) 630 { 631 WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs); 632 633 /* 634 * most users will be overriding ->bi_bdev with a new target, 635 * so we don't set nor calculate new physical/hw segment counts here 636 */ 637 bio->bi_bdev = bio_src->bi_bdev; 638 bio_set_flag(bio, BIO_CLONED); 639 if (bio_flagged(bio_src, BIO_THROTTLED)) 640 bio_set_flag(bio, BIO_THROTTLED); 641 if (bio_flagged(bio_src, BIO_REMAPPED)) 642 bio_set_flag(bio, BIO_REMAPPED); 643 bio->bi_opf = bio_src->bi_opf; 644 bio->bi_ioprio = bio_src->bi_ioprio; 645 bio->bi_write_hint = bio_src->bi_write_hint; 646 bio->bi_iter = bio_src->bi_iter; 647 bio->bi_io_vec = bio_src->bi_io_vec; 648 649 bio_clone_blkg_association(bio, bio_src); 650 blkcg_bio_issue_init(bio); 651 } 652 EXPORT_SYMBOL(__bio_clone_fast); 653 654 /** 655 * bio_clone_fast - clone a bio that shares the original bio's biovec 656 * @bio: bio to clone 657 * @gfp_mask: allocation priority 658 * @bs: bio_set to allocate from 659 * 660 * Like __bio_clone_fast, only also allocates the returned bio 661 */ 662 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) 663 { 664 struct bio *b; 665 666 b = bio_alloc_bioset(gfp_mask, 0, bs); 667 if (!b) 668 return NULL; 669 670 __bio_clone_fast(b, bio); 671 672 if (bio_crypt_clone(b, bio, gfp_mask) < 0) 673 goto err_put; 674 675 if (bio_integrity(bio) && 676 bio_integrity_clone(b, bio, gfp_mask) < 0) 677 goto err_put; 678 679 return b; 680 681 err_put: 682 bio_put(b); 683 return NULL; 684 } 685 EXPORT_SYMBOL(bio_clone_fast); 686 687 const char *bio_devname(struct bio *bio, char *buf) 688 { 689 return bdevname(bio->bi_bdev, buf); 690 } 691 EXPORT_SYMBOL(bio_devname); 692 693 static inline bool page_is_mergeable(const struct bio_vec *bv, 694 struct page *page, unsigned int len, unsigned int off, 695 bool *same_page) 696 { 697 size_t bv_end = bv->bv_offset + bv->bv_len; 698 phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1; 699 phys_addr_t page_addr = page_to_phys(page); 700 701 if (vec_end_addr + 1 != page_addr + off) 702 return false; 703 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 704 return false; 705 706 *same_page = ((vec_end_addr & PAGE_MASK) == page_addr); 707 if (*same_page) 708 return true; 709 return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE); 710 } 711 712 /* 713 * Try to merge a page into a segment, while obeying the hardware segment 714 * size limit. This is not for normal read/write bios, but for passthrough 715 * or Zone Append operations that we can't split. 716 */ 717 static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio, 718 struct page *page, unsigned len, 719 unsigned offset, bool *same_page) 720 { 721 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 722 unsigned long mask = queue_segment_boundary(q); 723 phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset; 724 phys_addr_t addr2 = page_to_phys(page) + offset + len - 1; 725 726 if ((addr1 | mask) != (addr2 | mask)) 727 return false; 728 if (bv->bv_len + len > queue_max_segment_size(q)) 729 return false; 730 return __bio_try_merge_page(bio, page, len, offset, same_page); 731 } 732 733 /** 734 * bio_add_hw_page - attempt to add a page to a bio with hw constraints 735 * @q: the target queue 736 * @bio: destination bio 737 * @page: page to add 738 * @len: vec entry length 739 * @offset: vec entry offset 740 * @max_sectors: maximum number of sectors that can be added 741 * @same_page: return if the segment has been merged inside the same page 742 * 743 * Add a page to a bio while respecting the hardware max_sectors, max_segment 744 * and gap limitations. 745 */ 746 int bio_add_hw_page(struct request_queue *q, struct bio *bio, 747 struct page *page, unsigned int len, unsigned int offset, 748 unsigned int max_sectors, bool *same_page) 749 { 750 struct bio_vec *bvec; 751 752 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 753 return 0; 754 755 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) 756 return 0; 757 758 if (bio->bi_vcnt > 0) { 759 if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page)) 760 return len; 761 762 /* 763 * If the queue doesn't support SG gaps and adding this segment 764 * would create a gap, disallow it. 765 */ 766 bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; 767 if (bvec_gap_to_prev(q, bvec, offset)) 768 return 0; 769 } 770 771 if (bio_full(bio, len)) 772 return 0; 773 774 if (bio->bi_vcnt >= queue_max_segments(q)) 775 return 0; 776 777 bvec = &bio->bi_io_vec[bio->bi_vcnt]; 778 bvec->bv_page = page; 779 bvec->bv_len = len; 780 bvec->bv_offset = offset; 781 bio->bi_vcnt++; 782 bio->bi_iter.bi_size += len; 783 return len; 784 } 785 786 /** 787 * bio_add_pc_page - attempt to add page to passthrough bio 788 * @q: the target queue 789 * @bio: destination bio 790 * @page: page to add 791 * @len: vec entry length 792 * @offset: vec entry offset 793 * 794 * Attempt to add a page to the bio_vec maplist. This can fail for a 795 * number of reasons, such as the bio being full or target block device 796 * limitations. The target block device must allow bio's up to PAGE_SIZE, 797 * so it is always possible to add a single page to an empty bio. 798 * 799 * This should only be used by passthrough bios. 800 */ 801 int bio_add_pc_page(struct request_queue *q, struct bio *bio, 802 struct page *page, unsigned int len, unsigned int offset) 803 { 804 bool same_page = false; 805 return bio_add_hw_page(q, bio, page, len, offset, 806 queue_max_hw_sectors(q), &same_page); 807 } 808 EXPORT_SYMBOL(bio_add_pc_page); 809 810 /** 811 * bio_add_zone_append_page - attempt to add page to zone-append bio 812 * @bio: destination bio 813 * @page: page to add 814 * @len: vec entry length 815 * @offset: vec entry offset 816 * 817 * Attempt to add a page to the bio_vec maplist of a bio that will be submitted 818 * for a zone-append request. This can fail for a number of reasons, such as the 819 * bio being full or the target block device is not a zoned block device or 820 * other limitations of the target block device. The target block device must 821 * allow bio's up to PAGE_SIZE, so it is always possible to add a single page 822 * to an empty bio. 823 * 824 * Returns: number of bytes added to the bio, or 0 in case of a failure. 825 */ 826 int bio_add_zone_append_page(struct bio *bio, struct page *page, 827 unsigned int len, unsigned int offset) 828 { 829 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 830 bool same_page = false; 831 832 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND)) 833 return 0; 834 835 if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) 836 return 0; 837 838 return bio_add_hw_page(q, bio, page, len, offset, 839 queue_max_zone_append_sectors(q), &same_page); 840 } 841 EXPORT_SYMBOL_GPL(bio_add_zone_append_page); 842 843 /** 844 * __bio_try_merge_page - try appending data to an existing bvec. 845 * @bio: destination bio 846 * @page: start page to add 847 * @len: length of the data to add 848 * @off: offset of the data relative to @page 849 * @same_page: return if the segment has been merged inside the same page 850 * 851 * Try to add the data at @page + @off to the last bvec of @bio. This is a 852 * useful optimisation for file systems with a block size smaller than the 853 * page size. 854 * 855 * Warn if (@len, @off) crosses pages in case that @same_page is true. 856 * 857 * Return %true on success or %false on failure. 858 */ 859 bool __bio_try_merge_page(struct bio *bio, struct page *page, 860 unsigned int len, unsigned int off, bool *same_page) 861 { 862 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 863 return false; 864 865 if (bio->bi_vcnt > 0) { 866 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 867 868 if (page_is_mergeable(bv, page, len, off, same_page)) { 869 if (bio->bi_iter.bi_size > UINT_MAX - len) { 870 *same_page = false; 871 return false; 872 } 873 bv->bv_len += len; 874 bio->bi_iter.bi_size += len; 875 return true; 876 } 877 } 878 return false; 879 } 880 EXPORT_SYMBOL_GPL(__bio_try_merge_page); 881 882 /** 883 * __bio_add_page - add page(s) to a bio in a new segment 884 * @bio: destination bio 885 * @page: start page to add 886 * @len: length of the data to add, may cross pages 887 * @off: offset of the data relative to @page, may cross pages 888 * 889 * Add the data at @page + @off to @bio as a new bvec. The caller must ensure 890 * that @bio has space for another bvec. 891 */ 892 void __bio_add_page(struct bio *bio, struct page *page, 893 unsigned int len, unsigned int off) 894 { 895 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt]; 896 897 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 898 WARN_ON_ONCE(bio_full(bio, len)); 899 900 bv->bv_page = page; 901 bv->bv_offset = off; 902 bv->bv_len = len; 903 904 bio->bi_iter.bi_size += len; 905 bio->bi_vcnt++; 906 907 if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) 908 bio_set_flag(bio, BIO_WORKINGSET); 909 } 910 EXPORT_SYMBOL_GPL(__bio_add_page); 911 912 /** 913 * bio_add_page - attempt to add page(s) to bio 914 * @bio: destination bio 915 * @page: start page to add 916 * @len: vec entry length, may cross pages 917 * @offset: vec entry offset relative to @page, may cross pages 918 * 919 * Attempt to add page(s) to the bio_vec maplist. This will only fail 920 * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. 921 */ 922 int bio_add_page(struct bio *bio, struct page *page, 923 unsigned int len, unsigned int offset) 924 { 925 bool same_page = false; 926 927 if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) { 928 if (bio_full(bio, len)) 929 return 0; 930 __bio_add_page(bio, page, len, offset); 931 } 932 return len; 933 } 934 EXPORT_SYMBOL(bio_add_page); 935 936 void bio_release_pages(struct bio *bio, bool mark_dirty) 937 { 938 struct bvec_iter_all iter_all; 939 struct bio_vec *bvec; 940 941 if (bio_flagged(bio, BIO_NO_PAGE_REF)) 942 return; 943 944 bio_for_each_segment_all(bvec, bio, iter_all) { 945 if (mark_dirty && !PageCompound(bvec->bv_page)) 946 set_page_dirty_lock(bvec->bv_page); 947 put_page(bvec->bv_page); 948 } 949 } 950 EXPORT_SYMBOL_GPL(bio_release_pages); 951 952 static void __bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) 953 { 954 WARN_ON_ONCE(bio->bi_max_vecs); 955 956 bio->bi_vcnt = iter->nr_segs; 957 bio->bi_io_vec = (struct bio_vec *)iter->bvec; 958 bio->bi_iter.bi_bvec_done = iter->iov_offset; 959 bio->bi_iter.bi_size = iter->count; 960 bio_set_flag(bio, BIO_NO_PAGE_REF); 961 bio_set_flag(bio, BIO_CLONED); 962 } 963 964 static int bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) 965 { 966 __bio_iov_bvec_set(bio, iter); 967 iov_iter_advance(iter, iter->count); 968 return 0; 969 } 970 971 static int bio_iov_bvec_set_append(struct bio *bio, struct iov_iter *iter) 972 { 973 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 974 struct iov_iter i = *iter; 975 976 iov_iter_truncate(&i, queue_max_zone_append_sectors(q) << 9); 977 __bio_iov_bvec_set(bio, &i); 978 iov_iter_advance(iter, i.count); 979 return 0; 980 } 981 982 #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) 983 984 /** 985 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio 986 * @bio: bio to add pages to 987 * @iter: iov iterator describing the region to be mapped 988 * 989 * Pins pages from *iter and appends them to @bio's bvec array. The 990 * pages will have to be released using put_page() when done. 991 * For multi-segment *iter, this function only adds pages from the 992 * next non-empty segment of the iov iterator. 993 */ 994 static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 995 { 996 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 997 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; 998 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 999 struct page **pages = (struct page **)bv; 1000 bool same_page = false; 1001 ssize_t size, left; 1002 unsigned len, i; 1003 size_t offset; 1004 1005 /* 1006 * Move page array up in the allocated memory for the bio vecs as far as 1007 * possible so that we can start filling biovecs from the beginning 1008 * without overwriting the temporary page array. 1009 */ 1010 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1011 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 1012 1013 size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 1014 if (unlikely(size <= 0)) 1015 return size ? size : -EFAULT; 1016 1017 for (left = size, i = 0; left > 0; left -= len, i++) { 1018 struct page *page = pages[i]; 1019 1020 len = min_t(size_t, PAGE_SIZE - offset, left); 1021 1022 if (__bio_try_merge_page(bio, page, len, offset, &same_page)) { 1023 if (same_page) 1024 put_page(page); 1025 } else { 1026 if (WARN_ON_ONCE(bio_full(bio, len))) 1027 return -EINVAL; 1028 __bio_add_page(bio, page, len, offset); 1029 } 1030 offset = 0; 1031 } 1032 1033 iov_iter_advance(iter, size); 1034 return 0; 1035 } 1036 1037 static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) 1038 { 1039 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 1040 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; 1041 struct request_queue *q = bio->bi_bdev->bd_disk->queue; 1042 unsigned int max_append_sectors = queue_max_zone_append_sectors(q); 1043 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 1044 struct page **pages = (struct page **)bv; 1045 ssize_t size, left; 1046 unsigned len, i; 1047 size_t offset; 1048 int ret = 0; 1049 1050 if (WARN_ON_ONCE(!max_append_sectors)) 1051 return 0; 1052 1053 /* 1054 * Move page array up in the allocated memory for the bio vecs as far as 1055 * possible so that we can start filling biovecs from the beginning 1056 * without overwriting the temporary page array. 1057 */ 1058 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1059 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 1060 1061 size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 1062 if (unlikely(size <= 0)) 1063 return size ? size : -EFAULT; 1064 1065 for (left = size, i = 0; left > 0; left -= len, i++) { 1066 struct page *page = pages[i]; 1067 bool same_page = false; 1068 1069 len = min_t(size_t, PAGE_SIZE - offset, left); 1070 if (bio_add_hw_page(q, bio, page, len, offset, 1071 max_append_sectors, &same_page) != len) { 1072 ret = -EINVAL; 1073 break; 1074 } 1075 if (same_page) 1076 put_page(page); 1077 offset = 0; 1078 } 1079 1080 iov_iter_advance(iter, size - left); 1081 return ret; 1082 } 1083 1084 /** 1085 * bio_iov_iter_get_pages - add user or kernel pages to a bio 1086 * @bio: bio to add pages to 1087 * @iter: iov iterator describing the region to be added 1088 * 1089 * This takes either an iterator pointing to user memory, or one pointing to 1090 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and 1091 * map them into the kernel. On IO completion, the caller should put those 1092 * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided 1093 * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs 1094 * to ensure the bvecs and pages stay referenced until the submitted I/O is 1095 * completed by a call to ->ki_complete() or returns with an error other than 1096 * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF 1097 * on IO completion. If it isn't, then pages should be released. 1098 * 1099 * The function tries, but does not guarantee, to pin as many pages as 1100 * fit into the bio, or are requested in @iter, whatever is smaller. If 1101 * MM encounters an error pinning the requested pages, it stops. Error 1102 * is returned only if 0 pages could be pinned. 1103 * 1104 * It's intended for direct IO, so doesn't do PSI tracking, the caller is 1105 * responsible for setting BIO_WORKINGSET if necessary. 1106 */ 1107 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 1108 { 1109 int ret = 0; 1110 1111 if (iov_iter_is_bvec(iter)) { 1112 if (bio_op(bio) == REQ_OP_ZONE_APPEND) 1113 return bio_iov_bvec_set_append(bio, iter); 1114 return bio_iov_bvec_set(bio, iter); 1115 } 1116 1117 do { 1118 if (bio_op(bio) == REQ_OP_ZONE_APPEND) 1119 ret = __bio_iov_append_get_pages(bio, iter); 1120 else 1121 ret = __bio_iov_iter_get_pages(bio, iter); 1122 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); 1123 1124 /* don't account direct I/O as memory stall */ 1125 bio_clear_flag(bio, BIO_WORKINGSET); 1126 return bio->bi_vcnt ? 0 : ret; 1127 } 1128 EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); 1129 1130 static void submit_bio_wait_endio(struct bio *bio) 1131 { 1132 complete(bio->bi_private); 1133 } 1134 1135 /** 1136 * submit_bio_wait - submit a bio, and wait until it completes 1137 * @bio: The &struct bio which describes the I/O 1138 * 1139 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from 1140 * bio_endio() on failure. 1141 * 1142 * WARNING: Unlike to how submit_bio() is usually used, this function does not 1143 * result in bio reference to be consumed. The caller must drop the reference 1144 * on his own. 1145 */ 1146 int submit_bio_wait(struct bio *bio) 1147 { 1148 DECLARE_COMPLETION_ONSTACK_MAP(done, 1149 bio->bi_bdev->bd_disk->lockdep_map); 1150 unsigned long hang_check; 1151 1152 bio->bi_private = &done; 1153 bio->bi_end_io = submit_bio_wait_endio; 1154 bio->bi_opf |= REQ_SYNC; 1155 submit_bio(bio); 1156 1157 /* Prevent hang_check timer from firing at us during very long I/O */ 1158 hang_check = sysctl_hung_task_timeout_secs; 1159 if (hang_check) 1160 while (!wait_for_completion_io_timeout(&done, 1161 hang_check * (HZ/2))) 1162 ; 1163 else 1164 wait_for_completion_io(&done); 1165 1166 return blk_status_to_errno(bio->bi_status); 1167 } 1168 EXPORT_SYMBOL(submit_bio_wait); 1169 1170 /** 1171 * bio_advance - increment/complete a bio by some number of bytes 1172 * @bio: bio to advance 1173 * @bytes: number of bytes to complete 1174 * 1175 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to 1176 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will 1177 * be updated on the last bvec as well. 1178 * 1179 * @bio will then represent the remaining, uncompleted portion of the io. 1180 */ 1181 void bio_advance(struct bio *bio, unsigned bytes) 1182 { 1183 if (bio_integrity(bio)) 1184 bio_integrity_advance(bio, bytes); 1185 1186 bio_crypt_advance(bio, bytes); 1187 bio_advance_iter(bio, &bio->bi_iter, bytes); 1188 } 1189 EXPORT_SYMBOL(bio_advance); 1190 1191 void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, 1192 struct bio *src, struct bvec_iter *src_iter) 1193 { 1194 struct bio_vec src_bv, dst_bv; 1195 void *src_p, *dst_p; 1196 unsigned bytes; 1197 1198 while (src_iter->bi_size && dst_iter->bi_size) { 1199 src_bv = bio_iter_iovec(src, *src_iter); 1200 dst_bv = bio_iter_iovec(dst, *dst_iter); 1201 1202 bytes = min(src_bv.bv_len, dst_bv.bv_len); 1203 1204 src_p = kmap_atomic(src_bv.bv_page); 1205 dst_p = kmap_atomic(dst_bv.bv_page); 1206 1207 memcpy(dst_p + dst_bv.bv_offset, 1208 src_p + src_bv.bv_offset, 1209 bytes); 1210 1211 kunmap_atomic(dst_p); 1212 kunmap_atomic(src_p); 1213 1214 flush_dcache_page(dst_bv.bv_page); 1215 1216 bio_advance_iter_single(src, src_iter, bytes); 1217 bio_advance_iter_single(dst, dst_iter, bytes); 1218 } 1219 } 1220 EXPORT_SYMBOL(bio_copy_data_iter); 1221 1222 /** 1223 * bio_copy_data - copy contents of data buffers from one bio to another 1224 * @src: source bio 1225 * @dst: destination bio 1226 * 1227 * Stops when it reaches the end of either @src or @dst - that is, copies 1228 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios). 1229 */ 1230 void bio_copy_data(struct bio *dst, struct bio *src) 1231 { 1232 struct bvec_iter src_iter = src->bi_iter; 1233 struct bvec_iter dst_iter = dst->bi_iter; 1234 1235 bio_copy_data_iter(dst, &dst_iter, src, &src_iter); 1236 } 1237 EXPORT_SYMBOL(bio_copy_data); 1238 1239 /** 1240 * bio_list_copy_data - copy contents of data buffers from one chain of bios to 1241 * another 1242 * @src: source bio list 1243 * @dst: destination bio list 1244 * 1245 * Stops when it reaches the end of either the @src list or @dst list - that is, 1246 * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of 1247 * bios). 1248 */ 1249 void bio_list_copy_data(struct bio *dst, struct bio *src) 1250 { 1251 struct bvec_iter src_iter = src->bi_iter; 1252 struct bvec_iter dst_iter = dst->bi_iter; 1253 1254 while (1) { 1255 if (!src_iter.bi_size) { 1256 src = src->bi_next; 1257 if (!src) 1258 break; 1259 1260 src_iter = src->bi_iter; 1261 } 1262 1263 if (!dst_iter.bi_size) { 1264 dst = dst->bi_next; 1265 if (!dst) 1266 break; 1267 1268 dst_iter = dst->bi_iter; 1269 } 1270 1271 bio_copy_data_iter(dst, &dst_iter, src, &src_iter); 1272 } 1273 } 1274 EXPORT_SYMBOL(bio_list_copy_data); 1275 1276 void bio_free_pages(struct bio *bio) 1277 { 1278 struct bio_vec *bvec; 1279 struct bvec_iter_all iter_all; 1280 1281 bio_for_each_segment_all(bvec, bio, iter_all) 1282 __free_page(bvec->bv_page); 1283 } 1284 EXPORT_SYMBOL(bio_free_pages); 1285 1286 /* 1287 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions 1288 * for performing direct-IO in BIOs. 1289 * 1290 * The problem is that we cannot run set_page_dirty() from interrupt context 1291 * because the required locks are not interrupt-safe. So what we can do is to 1292 * mark the pages dirty _before_ performing IO. And in interrupt context, 1293 * check that the pages are still dirty. If so, fine. If not, redirty them 1294 * in process context. 1295 * 1296 * We special-case compound pages here: normally this means reads into hugetlb 1297 * pages. The logic in here doesn't really work right for compound pages 1298 * because the VM does not uniformly chase down the head page in all cases. 1299 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't 1300 * handle them at all. So we skip compound pages here at an early stage. 1301 * 1302 * Note that this code is very hard to test under normal circumstances because 1303 * direct-io pins the pages with get_user_pages(). This makes 1304 * is_page_cache_freeable return false, and the VM will not clean the pages. 1305 * But other code (eg, flusher threads) could clean the pages if they are mapped 1306 * pagecache. 1307 * 1308 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the 1309 * deferred bio dirtying paths. 1310 */ 1311 1312 /* 1313 * bio_set_pages_dirty() will mark all the bio's pages as dirty. 1314 */ 1315 void bio_set_pages_dirty(struct bio *bio) 1316 { 1317 struct bio_vec *bvec; 1318 struct bvec_iter_all iter_all; 1319 1320 bio_for_each_segment_all(bvec, bio, iter_all) { 1321 if (!PageCompound(bvec->bv_page)) 1322 set_page_dirty_lock(bvec->bv_page); 1323 } 1324 } 1325 1326 /* 1327 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. 1328 * If they are, then fine. If, however, some pages are clean then they must 1329 * have been written out during the direct-IO read. So we take another ref on 1330 * the BIO and re-dirty the pages in process context. 1331 * 1332 * It is expected that bio_check_pages_dirty() will wholly own the BIO from 1333 * here on. It will run one put_page() against each page and will run one 1334 * bio_put() against the BIO. 1335 */ 1336 1337 static void bio_dirty_fn(struct work_struct *work); 1338 1339 static DECLARE_WORK(bio_dirty_work, bio_dirty_fn); 1340 static DEFINE_SPINLOCK(bio_dirty_lock); 1341 static struct bio *bio_dirty_list; 1342 1343 /* 1344 * This runs in process context 1345 */ 1346 static void bio_dirty_fn(struct work_struct *work) 1347 { 1348 struct bio *bio, *next; 1349 1350 spin_lock_irq(&bio_dirty_lock); 1351 next = bio_dirty_list; 1352 bio_dirty_list = NULL; 1353 spin_unlock_irq(&bio_dirty_lock); 1354 1355 while ((bio = next) != NULL) { 1356 next = bio->bi_private; 1357 1358 bio_release_pages(bio, true); 1359 bio_put(bio); 1360 } 1361 } 1362 1363 void bio_check_pages_dirty(struct bio *bio) 1364 { 1365 struct bio_vec *bvec; 1366 unsigned long flags; 1367 struct bvec_iter_all iter_all; 1368 1369 bio_for_each_segment_all(bvec, bio, iter_all) { 1370 if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) 1371 goto defer; 1372 } 1373 1374 bio_release_pages(bio, false); 1375 bio_put(bio); 1376 return; 1377 defer: 1378 spin_lock_irqsave(&bio_dirty_lock, flags); 1379 bio->bi_private = bio_dirty_list; 1380 bio_dirty_list = bio; 1381 spin_unlock_irqrestore(&bio_dirty_lock, flags); 1382 schedule_work(&bio_dirty_work); 1383 } 1384 1385 static inline bool bio_remaining_done(struct bio *bio) 1386 { 1387 /* 1388 * If we're not chaining, then ->__bi_remaining is always 1 and 1389 * we always end io on the first invocation. 1390 */ 1391 if (!bio_flagged(bio, BIO_CHAIN)) 1392 return true; 1393 1394 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); 1395 1396 if (atomic_dec_and_test(&bio->__bi_remaining)) { 1397 bio_clear_flag(bio, BIO_CHAIN); 1398 return true; 1399 } 1400 1401 return false; 1402 } 1403 1404 /** 1405 * bio_endio - end I/O on a bio 1406 * @bio: bio 1407 * 1408 * Description: 1409 * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred 1410 * way to end I/O on a bio. No one should call bi_end_io() directly on a 1411 * bio unless they own it and thus know that it has an end_io function. 1412 * 1413 * bio_endio() can be called several times on a bio that has been chained 1414 * using bio_chain(). The ->bi_end_io() function will only be called the 1415 * last time. At this point the BLK_TA_COMPLETE tracing event will be 1416 * generated if BIO_TRACE_COMPLETION is set. 1417 **/ 1418 void bio_endio(struct bio *bio) 1419 { 1420 again: 1421 if (!bio_remaining_done(bio)) 1422 return; 1423 if (!bio_integrity_endio(bio)) 1424 return; 1425 1426 if (bio->bi_bdev) 1427 rq_qos_done_bio(bio->bi_bdev->bd_disk->queue, bio); 1428 1429 /* 1430 * Need to have a real endio function for chained bios, otherwise 1431 * various corner cases will break (like stacking block devices that 1432 * save/restore bi_end_io) - however, we want to avoid unbounded 1433 * recursion and blowing the stack. Tail call optimization would 1434 * handle this, but compiling with frame pointers also disables 1435 * gcc's sibling call optimization. 1436 */ 1437 if (bio->bi_end_io == bio_chain_endio) { 1438 bio = __bio_chain_endio(bio); 1439 goto again; 1440 } 1441 1442 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { 1443 trace_block_bio_complete(bio->bi_bdev->bd_disk->queue, bio); 1444 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 1445 } 1446 1447 blk_throtl_bio_endio(bio); 1448 /* release cgroup info */ 1449 bio_uninit(bio); 1450 if (bio->bi_end_io) 1451 bio->bi_end_io(bio); 1452 } 1453 EXPORT_SYMBOL(bio_endio); 1454 1455 /** 1456 * bio_split - split a bio 1457 * @bio: bio to split 1458 * @sectors: number of sectors to split from the front of @bio 1459 * @gfp: gfp mask 1460 * @bs: bio set to allocate from 1461 * 1462 * Allocates and returns a new bio which represents @sectors from the start of 1463 * @bio, and updates @bio to represent the remaining sectors. 1464 * 1465 * Unless this is a discard request the newly allocated bio will point 1466 * to @bio's bi_io_vec. It is the caller's responsibility to ensure that 1467 * neither @bio nor @bs are freed before the split bio. 1468 */ 1469 struct bio *bio_split(struct bio *bio, int sectors, 1470 gfp_t gfp, struct bio_set *bs) 1471 { 1472 struct bio *split; 1473 1474 BUG_ON(sectors <= 0); 1475 BUG_ON(sectors >= bio_sectors(bio)); 1476 1477 /* Zone append commands cannot be split */ 1478 if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) 1479 return NULL; 1480 1481 split = bio_clone_fast(bio, gfp, bs); 1482 if (!split) 1483 return NULL; 1484 1485 split->bi_iter.bi_size = sectors << 9; 1486 1487 if (bio_integrity(split)) 1488 bio_integrity_trim(split); 1489 1490 bio_advance(bio, split->bi_iter.bi_size); 1491 1492 if (bio_flagged(bio, BIO_TRACE_COMPLETION)) 1493 bio_set_flag(split, BIO_TRACE_COMPLETION); 1494 1495 return split; 1496 } 1497 EXPORT_SYMBOL(bio_split); 1498 1499 /** 1500 * bio_trim - trim a bio 1501 * @bio: bio to trim 1502 * @offset: number of sectors to trim from the front of @bio 1503 * @size: size we want to trim @bio to, in sectors 1504 */ 1505 void bio_trim(struct bio *bio, int offset, int size) 1506 { 1507 /* 'bio' is a cloned bio which we need to trim to match 1508 * the given offset and size. 1509 */ 1510 1511 size <<= 9; 1512 if (offset == 0 && size == bio->bi_iter.bi_size) 1513 return; 1514 1515 bio_advance(bio, offset << 9); 1516 bio->bi_iter.bi_size = size; 1517 1518 if (bio_integrity(bio)) 1519 bio_integrity_trim(bio); 1520 1521 } 1522 EXPORT_SYMBOL_GPL(bio_trim); 1523 1524 /* 1525 * create memory pools for biovec's in a bio_set. 1526 * use the global biovec slabs created for general use. 1527 */ 1528 int biovec_init_pool(mempool_t *pool, int pool_entries) 1529 { 1530 struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1; 1531 1532 return mempool_init_slab_pool(pool, pool_entries, bp->slab); 1533 } 1534 1535 /* 1536 * bioset_exit - exit a bioset initialized with bioset_init() 1537 * 1538 * May be called on a zeroed but uninitialized bioset (i.e. allocated with 1539 * kzalloc()). 1540 */ 1541 void bioset_exit(struct bio_set *bs) 1542 { 1543 if (bs->rescue_workqueue) 1544 destroy_workqueue(bs->rescue_workqueue); 1545 bs->rescue_workqueue = NULL; 1546 1547 mempool_exit(&bs->bio_pool); 1548 mempool_exit(&bs->bvec_pool); 1549 1550 bioset_integrity_free(bs); 1551 if (bs->bio_slab) 1552 bio_put_slab(bs); 1553 bs->bio_slab = NULL; 1554 } 1555 EXPORT_SYMBOL(bioset_exit); 1556 1557 /** 1558 * bioset_init - Initialize a bio_set 1559 * @bs: pool to initialize 1560 * @pool_size: Number of bio and bio_vecs to cache in the mempool 1561 * @front_pad: Number of bytes to allocate in front of the returned bio 1562 * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS 1563 * and %BIOSET_NEED_RESCUER 1564 * 1565 * Description: 1566 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller 1567 * to ask for a number of bytes to be allocated in front of the bio. 1568 * Front pad allocation is useful for embedding the bio inside 1569 * another structure, to avoid allocating extra data to go with the bio. 1570 * Note that the bio must be embedded at the END of that structure always, 1571 * or things will break badly. 1572 * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated 1573 * for allocating iovecs. This pool is not needed e.g. for bio_clone_fast(). 1574 * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to 1575 * dispatch queued requests when the mempool runs out of space. 1576 * 1577 */ 1578 int bioset_init(struct bio_set *bs, 1579 unsigned int pool_size, 1580 unsigned int front_pad, 1581 int flags) 1582 { 1583 bs->front_pad = front_pad; 1584 if (flags & BIOSET_NEED_BVECS) 1585 bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); 1586 else 1587 bs->back_pad = 0; 1588 1589 spin_lock_init(&bs->rescue_lock); 1590 bio_list_init(&bs->rescue_list); 1591 INIT_WORK(&bs->rescue_work, bio_alloc_rescue); 1592 1593 bs->bio_slab = bio_find_or_create_slab(bs); 1594 if (!bs->bio_slab) 1595 return -ENOMEM; 1596 1597 if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab)) 1598 goto bad; 1599 1600 if ((flags & BIOSET_NEED_BVECS) && 1601 biovec_init_pool(&bs->bvec_pool, pool_size)) 1602 goto bad; 1603 1604 if (!(flags & BIOSET_NEED_RESCUER)) 1605 return 0; 1606 1607 bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0); 1608 if (!bs->rescue_workqueue) 1609 goto bad; 1610 1611 return 0; 1612 bad: 1613 bioset_exit(bs); 1614 return -ENOMEM; 1615 } 1616 EXPORT_SYMBOL(bioset_init); 1617 1618 /* 1619 * Initialize and setup a new bio_set, based on the settings from 1620 * another bio_set. 1621 */ 1622 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src) 1623 { 1624 int flags; 1625 1626 flags = 0; 1627 if (src->bvec_pool.min_nr) 1628 flags |= BIOSET_NEED_BVECS; 1629 if (src->rescue_workqueue) 1630 flags |= BIOSET_NEED_RESCUER; 1631 1632 return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags); 1633 } 1634 EXPORT_SYMBOL(bioset_init_from_src); 1635 1636 static int __init init_bio(void) 1637 { 1638 int i; 1639 1640 bio_integrity_init(); 1641 1642 for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { 1643 struct biovec_slab *bvs = bvec_slabs + i; 1644 1645 bvs->slab = kmem_cache_create(bvs->name, 1646 bvs->nr_vecs * sizeof(struct bio_vec), 0, 1647 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); 1648 } 1649 1650 if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS)) 1651 panic("bio: can't allocate bios\n"); 1652 1653 if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE)) 1654 panic("bio: can't create integrity pool\n"); 1655 1656 return 0; 1657 } 1658 subsys_initcall(init_bio); 1659