1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk> 4 */ 5 #include <linux/mm.h> 6 #include <linux/swap.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/uio.h> 10 #include <linux/iocontext.h> 11 #include <linux/slab.h> 12 #include <linux/init.h> 13 #include <linux/kernel.h> 14 #include <linux/export.h> 15 #include <linux/mempool.h> 16 #include <linux/workqueue.h> 17 #include <linux/cgroup.h> 18 #include <linux/blk-cgroup.h> 19 #include <linux/highmem.h> 20 #include <linux/sched/sysctl.h> 21 #include <linux/blk-crypto.h> 22 #include <linux/xarray.h> 23 24 #include <trace/events/block.h> 25 #include "blk.h" 26 #include "blk-rq-qos.h" 27 28 struct bio_alloc_cache { 29 struct bio *free_list; 30 unsigned int nr; 31 }; 32 33 static struct biovec_slab { 34 int nr_vecs; 35 char *name; 36 struct kmem_cache *slab; 37 } bvec_slabs[] __read_mostly = { 38 { .nr_vecs = 16, .name = "biovec-16" }, 39 { .nr_vecs = 64, .name = "biovec-64" }, 40 { .nr_vecs = 128, .name = "biovec-128" }, 41 { .nr_vecs = BIO_MAX_VECS, .name = "biovec-max" }, 42 }; 43 44 static struct biovec_slab *biovec_slab(unsigned short nr_vecs) 45 { 46 switch (nr_vecs) { 47 /* smaller bios use inline vecs */ 48 case 5 ... 16: 49 return &bvec_slabs[0]; 50 case 17 ... 64: 51 return &bvec_slabs[1]; 52 case 65 ... 128: 53 return &bvec_slabs[2]; 54 case 129 ... BIO_MAX_VECS: 55 return &bvec_slabs[3]; 56 default: 57 BUG(); 58 return NULL; 59 } 60 } 61 62 /* 63 * fs_bio_set is the bio_set containing bio and iovec memory pools used by 64 * IO code that does not need private memory pools. 65 */ 66 struct bio_set fs_bio_set; 67 EXPORT_SYMBOL(fs_bio_set); 68 69 /* 70 * Our slab pool management 71 */ 72 struct bio_slab { 73 struct kmem_cache *slab; 74 unsigned int slab_ref; 75 unsigned int slab_size; 76 char name[8]; 77 }; 78 static DEFINE_MUTEX(bio_slab_lock); 79 static DEFINE_XARRAY(bio_slabs); 80 81 static struct bio_slab *create_bio_slab(unsigned int size) 82 { 83 struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL); 84 85 if (!bslab) 86 return NULL; 87 88 snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size); 89 bslab->slab = kmem_cache_create(bslab->name, size, 90 ARCH_KMALLOC_MINALIGN, 91 SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL); 92 if (!bslab->slab) 93 goto fail_alloc_slab; 94 95 bslab->slab_ref = 1; 96 bslab->slab_size = size; 97 98 if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL))) 99 return bslab; 100 101 kmem_cache_destroy(bslab->slab); 102 103 fail_alloc_slab: 104 kfree(bslab); 105 return NULL; 106 } 107 108 static inline unsigned int bs_bio_slab_size(struct bio_set *bs) 109 { 110 return bs->front_pad + sizeof(struct bio) + bs->back_pad; 111 } 112 113 static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) 114 { 115 unsigned int size = bs_bio_slab_size(bs); 116 struct bio_slab *bslab; 117 118 mutex_lock(&bio_slab_lock); 119 bslab = xa_load(&bio_slabs, size); 120 if (bslab) 121 bslab->slab_ref++; 122 else 123 bslab = create_bio_slab(size); 124 mutex_unlock(&bio_slab_lock); 125 126 if (bslab) 127 return bslab->slab; 128 return NULL; 129 } 130 131 static void bio_put_slab(struct bio_set *bs) 132 { 133 struct bio_slab *bslab = NULL; 134 unsigned int slab_size = bs_bio_slab_size(bs); 135 136 mutex_lock(&bio_slab_lock); 137 138 bslab = xa_load(&bio_slabs, slab_size); 139 if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) 140 goto out; 141 142 WARN_ON_ONCE(bslab->slab != bs->bio_slab); 143 144 WARN_ON(!bslab->slab_ref); 145 146 if (--bslab->slab_ref) 147 goto out; 148 149 xa_erase(&bio_slabs, slab_size); 150 151 kmem_cache_destroy(bslab->slab); 152 kfree(bslab); 153 154 out: 155 mutex_unlock(&bio_slab_lock); 156 } 157 158 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) 159 { 160 BUG_ON(nr_vecs > BIO_MAX_VECS); 161 162 if (nr_vecs == BIO_MAX_VECS) 163 mempool_free(bv, pool); 164 else if (nr_vecs > BIO_INLINE_VECS) 165 kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); 166 } 167 168 /* 169 * Make the first allocation restricted and don't dump info on allocation 170 * failures, since we'll fall back to the mempool in case of failure. 171 */ 172 static inline gfp_t bvec_alloc_gfp(gfp_t gfp) 173 { 174 return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | 175 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 176 } 177 178 struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, 179 gfp_t gfp_mask) 180 { 181 struct biovec_slab *bvs = biovec_slab(*nr_vecs); 182 183 if (WARN_ON_ONCE(!bvs)) 184 return NULL; 185 186 /* 187 * Upgrade the nr_vecs request to take full advantage of the allocation. 188 * We also rely on this in the bvec_free path. 189 */ 190 *nr_vecs = bvs->nr_vecs; 191 192 /* 193 * Try a slab allocation first for all smaller allocations. If that 194 * fails and __GFP_DIRECT_RECLAIM is set retry with the mempool. 195 * The mempool is sized to handle up to BIO_MAX_VECS entries. 196 */ 197 if (*nr_vecs < BIO_MAX_VECS) { 198 struct bio_vec *bvl; 199 200 bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); 201 if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) 202 return bvl; 203 *nr_vecs = BIO_MAX_VECS; 204 } 205 206 return mempool_alloc(pool, gfp_mask); 207 } 208 209 void bio_uninit(struct bio *bio) 210 { 211 #ifdef CONFIG_BLK_CGROUP 212 if (bio->bi_blkg) { 213 blkg_put(bio->bi_blkg); 214 bio->bi_blkg = NULL; 215 } 216 #endif 217 if (bio_integrity(bio)) 218 bio_integrity_free(bio); 219 220 bio_crypt_free_ctx(bio); 221 } 222 EXPORT_SYMBOL(bio_uninit); 223 224 static void bio_free(struct bio *bio) 225 { 226 struct bio_set *bs = bio->bi_pool; 227 void *p; 228 229 bio_uninit(bio); 230 231 if (bs) { 232 bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs); 233 234 /* 235 * If we have front padding, adjust the bio pointer before freeing 236 */ 237 p = bio; 238 p -= bs->front_pad; 239 240 mempool_free(p, &bs->bio_pool); 241 } else { 242 /* Bio was allocated by bio_kmalloc() */ 243 kfree(bio); 244 } 245 } 246 247 /* 248 * Users of this function have their own bio allocation. Subsequently, 249 * they must remember to pair any call to bio_init() with bio_uninit() 250 * when IO has completed, or when the bio is released. 251 */ 252 void bio_init(struct bio *bio, struct bio_vec *table, 253 unsigned short max_vecs) 254 { 255 bio->bi_next = NULL; 256 bio->bi_bdev = NULL; 257 bio->bi_opf = 0; 258 bio->bi_flags = 0; 259 bio->bi_ioprio = 0; 260 bio->bi_write_hint = 0; 261 bio->bi_status = 0; 262 bio->bi_iter.bi_sector = 0; 263 bio->bi_iter.bi_size = 0; 264 bio->bi_iter.bi_idx = 0; 265 bio->bi_iter.bi_bvec_done = 0; 266 bio->bi_end_io = NULL; 267 bio->bi_private = NULL; 268 #ifdef CONFIG_BLK_CGROUP 269 bio->bi_blkg = NULL; 270 bio->bi_issue.value = 0; 271 #ifdef CONFIG_BLK_CGROUP_IOCOST 272 bio->bi_iocost_cost = 0; 273 #endif 274 #endif 275 #ifdef CONFIG_BLK_INLINE_ENCRYPTION 276 bio->bi_crypt_context = NULL; 277 #endif 278 #ifdef CONFIG_BLK_DEV_INTEGRITY 279 bio->bi_integrity = NULL; 280 #endif 281 bio->bi_vcnt = 0; 282 283 atomic_set(&bio->__bi_remaining, 1); 284 atomic_set(&bio->__bi_cnt, 1); 285 bio->bi_cookie = BLK_QC_T_NONE; 286 287 bio->bi_max_vecs = max_vecs; 288 bio->bi_io_vec = table; 289 bio->bi_pool = NULL; 290 } 291 EXPORT_SYMBOL(bio_init); 292 293 /** 294 * bio_reset - reinitialize a bio 295 * @bio: bio to reset 296 * 297 * Description: 298 * After calling bio_reset(), @bio will be in the same state as a freshly 299 * allocated bio returned bio bio_alloc_bioset() - the only fields that are 300 * preserved are the ones that are initialized by bio_alloc_bioset(). See 301 * comment in struct bio. 302 */ 303 void bio_reset(struct bio *bio) 304 { 305 bio_uninit(bio); 306 memset(bio, 0, BIO_RESET_BYTES); 307 atomic_set(&bio->__bi_remaining, 1); 308 } 309 EXPORT_SYMBOL(bio_reset); 310 311 static struct bio *__bio_chain_endio(struct bio *bio) 312 { 313 struct bio *parent = bio->bi_private; 314 315 if (bio->bi_status && !parent->bi_status) 316 parent->bi_status = bio->bi_status; 317 bio_put(bio); 318 return parent; 319 } 320 321 static void bio_chain_endio(struct bio *bio) 322 { 323 bio_endio(__bio_chain_endio(bio)); 324 } 325 326 /** 327 * bio_chain - chain bio completions 328 * @bio: the target bio 329 * @parent: the parent bio of @bio 330 * 331 * The caller won't have a bi_end_io called when @bio completes - instead, 332 * @parent's bi_end_io won't be called until both @parent and @bio have 333 * completed; the chained bio will also be freed when it completes. 334 * 335 * The caller must not set bi_private or bi_end_io in @bio. 336 */ 337 void bio_chain(struct bio *bio, struct bio *parent) 338 { 339 BUG_ON(bio->bi_private || bio->bi_end_io); 340 341 bio->bi_private = parent; 342 bio->bi_end_io = bio_chain_endio; 343 bio_inc_remaining(parent); 344 } 345 EXPORT_SYMBOL(bio_chain); 346 347 struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp) 348 { 349 struct bio *new = bio_alloc(gfp, nr_pages); 350 351 if (bio) { 352 bio_chain(bio, new); 353 submit_bio(bio); 354 } 355 356 return new; 357 } 358 EXPORT_SYMBOL_GPL(blk_next_bio); 359 360 static void bio_alloc_rescue(struct work_struct *work) 361 { 362 struct bio_set *bs = container_of(work, struct bio_set, rescue_work); 363 struct bio *bio; 364 365 while (1) { 366 spin_lock(&bs->rescue_lock); 367 bio = bio_list_pop(&bs->rescue_list); 368 spin_unlock(&bs->rescue_lock); 369 370 if (!bio) 371 break; 372 373 submit_bio_noacct(bio); 374 } 375 } 376 377 static void punt_bios_to_rescuer(struct bio_set *bs) 378 { 379 struct bio_list punt, nopunt; 380 struct bio *bio; 381 382 if (WARN_ON_ONCE(!bs->rescue_workqueue)) 383 return; 384 /* 385 * In order to guarantee forward progress we must punt only bios that 386 * were allocated from this bio_set; otherwise, if there was a bio on 387 * there for a stacking driver higher up in the stack, processing it 388 * could require allocating bios from this bio_set, and doing that from 389 * our own rescuer would be bad. 390 * 391 * Since bio lists are singly linked, pop them all instead of trying to 392 * remove from the middle of the list: 393 */ 394 395 bio_list_init(&punt); 396 bio_list_init(&nopunt); 397 398 while ((bio = bio_list_pop(¤t->bio_list[0]))) 399 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); 400 current->bio_list[0] = nopunt; 401 402 bio_list_init(&nopunt); 403 while ((bio = bio_list_pop(¤t->bio_list[1]))) 404 bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); 405 current->bio_list[1] = nopunt; 406 407 spin_lock(&bs->rescue_lock); 408 bio_list_merge(&bs->rescue_list, &punt); 409 spin_unlock(&bs->rescue_lock); 410 411 queue_work(bs->rescue_workqueue, &bs->rescue_work); 412 } 413 414 /** 415 * bio_alloc_bioset - allocate a bio for I/O 416 * @gfp_mask: the GFP_* mask given to the slab allocator 417 * @nr_iovecs: number of iovecs to pre-allocate 418 * @bs: the bio_set to allocate from. 419 * 420 * Allocate a bio from the mempools in @bs. 421 * 422 * If %__GFP_DIRECT_RECLAIM is set then bio_alloc will always be able to 423 * allocate a bio. This is due to the mempool guarantees. To make this work, 424 * callers must never allocate more than 1 bio at a time from the general pool. 425 * Callers that need to allocate more than 1 bio must always submit the 426 * previously allocated bio for IO before attempting to allocate a new one. 427 * Failure to do so can cause deadlocks under memory pressure. 428 * 429 * Note that when running under submit_bio_noacct() (i.e. any block driver), 430 * bios are not submitted until after you return - see the code in 431 * submit_bio_noacct() that converts recursion into iteration, to prevent 432 * stack overflows. 433 * 434 * This would normally mean allocating multiple bios under submit_bio_noacct() 435 * would be susceptible to deadlocks, but we have 436 * deadlock avoidance code that resubmits any blocked bios from a rescuer 437 * thread. 438 * 439 * However, we do not guarantee forward progress for allocations from other 440 * mempools. Doing multiple allocations from the same mempool under 441 * submit_bio_noacct() should be avoided - instead, use bio_set's front_pad 442 * for per bio allocations. 443 * 444 * Returns: Pointer to new bio on success, NULL on failure. 445 */ 446 struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned short nr_iovecs, 447 struct bio_set *bs) 448 { 449 gfp_t saved_gfp = gfp_mask; 450 struct bio *bio; 451 void *p; 452 453 /* should not use nobvec bioset for nr_iovecs > 0 */ 454 if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_iovecs > 0)) 455 return NULL; 456 457 /* 458 * submit_bio_noacct() converts recursion to iteration; this means if 459 * we're running beneath it, any bios we allocate and submit will not be 460 * submitted (and thus freed) until after we return. 461 * 462 * This exposes us to a potential deadlock if we allocate multiple bios 463 * from the same bio_set() while running underneath submit_bio_noacct(). 464 * If we were to allocate multiple bios (say a stacking block driver 465 * that was splitting bios), we would deadlock if we exhausted the 466 * mempool's reserve. 467 * 468 * We solve this, and guarantee forward progress, with a rescuer 469 * workqueue per bio_set. If we go to allocate and there are bios on 470 * current->bio_list, we first try the allocation without 471 * __GFP_DIRECT_RECLAIM; if that fails, we punt those bios we would be 472 * blocking to the rescuer workqueue before we retry with the original 473 * gfp_flags. 474 */ 475 if (current->bio_list && 476 (!bio_list_empty(¤t->bio_list[0]) || 477 !bio_list_empty(¤t->bio_list[1])) && 478 bs->rescue_workqueue) 479 gfp_mask &= ~__GFP_DIRECT_RECLAIM; 480 481 p = mempool_alloc(&bs->bio_pool, gfp_mask); 482 if (!p && gfp_mask != saved_gfp) { 483 punt_bios_to_rescuer(bs); 484 gfp_mask = saved_gfp; 485 p = mempool_alloc(&bs->bio_pool, gfp_mask); 486 } 487 if (unlikely(!p)) 488 return NULL; 489 490 bio = p + bs->front_pad; 491 if (nr_iovecs > BIO_INLINE_VECS) { 492 struct bio_vec *bvl = NULL; 493 494 bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); 495 if (!bvl && gfp_mask != saved_gfp) { 496 punt_bios_to_rescuer(bs); 497 gfp_mask = saved_gfp; 498 bvl = bvec_alloc(&bs->bvec_pool, &nr_iovecs, gfp_mask); 499 } 500 if (unlikely(!bvl)) 501 goto err_free; 502 503 bio_init(bio, bvl, nr_iovecs); 504 } else if (nr_iovecs) { 505 bio_init(bio, bio->bi_inline_vecs, BIO_INLINE_VECS); 506 } else { 507 bio_init(bio, NULL, 0); 508 } 509 510 bio->bi_pool = bs; 511 return bio; 512 513 err_free: 514 mempool_free(p, &bs->bio_pool); 515 return NULL; 516 } 517 EXPORT_SYMBOL(bio_alloc_bioset); 518 519 /** 520 * bio_kmalloc - kmalloc a bio for I/O 521 * @gfp_mask: the GFP_* mask given to the slab allocator 522 * @nr_iovecs: number of iovecs to pre-allocate 523 * 524 * Use kmalloc to allocate and initialize a bio. 525 * 526 * Returns: Pointer to new bio on success, NULL on failure. 527 */ 528 struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned short nr_iovecs) 529 { 530 struct bio *bio; 531 532 if (nr_iovecs > UIO_MAXIOV) 533 return NULL; 534 535 bio = kmalloc(struct_size(bio, bi_inline_vecs, nr_iovecs), gfp_mask); 536 if (unlikely(!bio)) 537 return NULL; 538 bio_init(bio, nr_iovecs ? bio->bi_inline_vecs : NULL, nr_iovecs); 539 bio->bi_pool = NULL; 540 return bio; 541 } 542 EXPORT_SYMBOL(bio_kmalloc); 543 544 void zero_fill_bio(struct bio *bio) 545 { 546 struct bio_vec bv; 547 struct bvec_iter iter; 548 549 bio_for_each_segment(bv, bio, iter) 550 memzero_bvec(&bv); 551 } 552 EXPORT_SYMBOL(zero_fill_bio); 553 554 /** 555 * bio_truncate - truncate the bio to small size of @new_size 556 * @bio: the bio to be truncated 557 * @new_size: new size for truncating the bio 558 * 559 * Description: 560 * Truncate the bio to new size of @new_size. If bio_op(bio) is 561 * REQ_OP_READ, zero the truncated part. This function should only 562 * be used for handling corner cases, such as bio eod. 563 */ 564 static void bio_truncate(struct bio *bio, unsigned new_size) 565 { 566 struct bio_vec bv; 567 struct bvec_iter iter; 568 unsigned int done = 0; 569 bool truncated = false; 570 571 if (new_size >= bio->bi_iter.bi_size) 572 return; 573 574 if (bio_op(bio) != REQ_OP_READ) 575 goto exit; 576 577 bio_for_each_segment(bv, bio, iter) { 578 if (done + bv.bv_len > new_size) { 579 unsigned offset; 580 581 if (!truncated) 582 offset = new_size - done; 583 else 584 offset = 0; 585 zero_user(bv.bv_page, bv.bv_offset + offset, 586 bv.bv_len - offset); 587 truncated = true; 588 } 589 done += bv.bv_len; 590 } 591 592 exit: 593 /* 594 * Don't touch bvec table here and make it really immutable, since 595 * fs bio user has to retrieve all pages via bio_for_each_segment_all 596 * in its .end_bio() callback. 597 * 598 * It is enough to truncate bio by updating .bi_size since we can make 599 * correct bvec with the updated .bi_size for drivers. 600 */ 601 bio->bi_iter.bi_size = new_size; 602 } 603 604 /** 605 * guard_bio_eod - truncate a BIO to fit the block device 606 * @bio: bio to truncate 607 * 608 * This allows us to do IO even on the odd last sectors of a device, even if the 609 * block size is some multiple of the physical sector size. 610 * 611 * We'll just truncate the bio to the size of the device, and clear the end of 612 * the buffer head manually. Truly out-of-range accesses will turn into actual 613 * I/O errors, this only handles the "we need to be able to do I/O at the final 614 * sector" case. 615 */ 616 void guard_bio_eod(struct bio *bio) 617 { 618 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev); 619 620 if (!maxsector) 621 return; 622 623 /* 624 * If the *whole* IO is past the end of the device, 625 * let it through, and the IO layer will turn it into 626 * an EIO. 627 */ 628 if (unlikely(bio->bi_iter.bi_sector >= maxsector)) 629 return; 630 631 maxsector -= bio->bi_iter.bi_sector; 632 if (likely((bio->bi_iter.bi_size >> 9) <= maxsector)) 633 return; 634 635 bio_truncate(bio, maxsector << 9); 636 } 637 638 #define ALLOC_CACHE_MAX 512 639 #define ALLOC_CACHE_SLACK 64 640 641 static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, 642 unsigned int nr) 643 { 644 unsigned int i = 0; 645 struct bio *bio; 646 647 while ((bio = cache->free_list) != NULL) { 648 cache->free_list = bio->bi_next; 649 cache->nr--; 650 bio_free(bio); 651 if (++i == nr) 652 break; 653 } 654 } 655 656 static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node) 657 { 658 struct bio_set *bs; 659 660 bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead); 661 if (bs->cache) { 662 struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu); 663 664 bio_alloc_cache_prune(cache, -1U); 665 } 666 return 0; 667 } 668 669 static void bio_alloc_cache_destroy(struct bio_set *bs) 670 { 671 int cpu; 672 673 if (!bs->cache) 674 return; 675 676 cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); 677 for_each_possible_cpu(cpu) { 678 struct bio_alloc_cache *cache; 679 680 cache = per_cpu_ptr(bs->cache, cpu); 681 bio_alloc_cache_prune(cache, -1U); 682 } 683 free_percpu(bs->cache); 684 } 685 686 /** 687 * bio_put - release a reference to a bio 688 * @bio: bio to release reference to 689 * 690 * Description: 691 * Put a reference to a &struct bio, either one you have gotten with 692 * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it. 693 **/ 694 void bio_put(struct bio *bio) 695 { 696 if (unlikely(bio_flagged(bio, BIO_REFFED))) { 697 BUG_ON(!atomic_read(&bio->__bi_cnt)); 698 if (!atomic_dec_and_test(&bio->__bi_cnt)) 699 return; 700 } 701 702 if (bio_flagged(bio, BIO_PERCPU_CACHE)) { 703 struct bio_alloc_cache *cache; 704 705 bio_uninit(bio); 706 cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu()); 707 bio->bi_next = cache->free_list; 708 cache->free_list = bio; 709 if (++cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK) 710 bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK); 711 put_cpu(); 712 } else { 713 bio_free(bio); 714 } 715 } 716 EXPORT_SYMBOL(bio_put); 717 718 /** 719 * __bio_clone_fast - clone a bio that shares the original bio's biovec 720 * @bio: destination bio 721 * @bio_src: bio to clone 722 * 723 * Clone a &bio. Caller will own the returned bio, but not 724 * the actual data it points to. Reference count of returned 725 * bio will be one. 726 * 727 * Caller must ensure that @bio_src is not freed before @bio. 728 */ 729 void __bio_clone_fast(struct bio *bio, struct bio *bio_src) 730 { 731 WARN_ON_ONCE(bio->bi_pool && bio->bi_max_vecs); 732 733 /* 734 * most users will be overriding ->bi_bdev with a new target, 735 * so we don't set nor calculate new physical/hw segment counts here 736 */ 737 bio->bi_bdev = bio_src->bi_bdev; 738 bio_set_flag(bio, BIO_CLONED); 739 if (bio_flagged(bio_src, BIO_THROTTLED)) 740 bio_set_flag(bio, BIO_THROTTLED); 741 if (bio_flagged(bio_src, BIO_REMAPPED)) 742 bio_set_flag(bio, BIO_REMAPPED); 743 bio->bi_opf = bio_src->bi_opf; 744 bio->bi_ioprio = bio_src->bi_ioprio; 745 bio->bi_write_hint = bio_src->bi_write_hint; 746 bio->bi_iter = bio_src->bi_iter; 747 bio->bi_io_vec = bio_src->bi_io_vec; 748 749 bio_clone_blkg_association(bio, bio_src); 750 blkcg_bio_issue_init(bio); 751 } 752 EXPORT_SYMBOL(__bio_clone_fast); 753 754 /** 755 * bio_clone_fast - clone a bio that shares the original bio's biovec 756 * @bio: bio to clone 757 * @gfp_mask: allocation priority 758 * @bs: bio_set to allocate from 759 * 760 * Like __bio_clone_fast, only also allocates the returned bio 761 */ 762 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) 763 { 764 struct bio *b; 765 766 b = bio_alloc_bioset(gfp_mask, 0, bs); 767 if (!b) 768 return NULL; 769 770 __bio_clone_fast(b, bio); 771 772 if (bio_crypt_clone(b, bio, gfp_mask) < 0) 773 goto err_put; 774 775 if (bio_integrity(bio) && 776 bio_integrity_clone(b, bio, gfp_mask) < 0) 777 goto err_put; 778 779 return b; 780 781 err_put: 782 bio_put(b); 783 return NULL; 784 } 785 EXPORT_SYMBOL(bio_clone_fast); 786 787 const char *bio_devname(struct bio *bio, char *buf) 788 { 789 return bdevname(bio->bi_bdev, buf); 790 } 791 EXPORT_SYMBOL(bio_devname); 792 793 /** 794 * bio_full - check if the bio is full 795 * @bio: bio to check 796 * @len: length of one segment to be added 797 * 798 * Return true if @bio is full and one segment with @len bytes can't be 799 * added to the bio, otherwise return false 800 */ 801 static inline bool bio_full(struct bio *bio, unsigned len) 802 { 803 if (bio->bi_vcnt >= bio->bi_max_vecs) 804 return true; 805 if (bio->bi_iter.bi_size > UINT_MAX - len) 806 return true; 807 return false; 808 } 809 810 static inline bool page_is_mergeable(const struct bio_vec *bv, 811 struct page *page, unsigned int len, unsigned int off, 812 bool *same_page) 813 { 814 size_t bv_end = bv->bv_offset + bv->bv_len; 815 phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) + bv_end - 1; 816 phys_addr_t page_addr = page_to_phys(page); 817 818 if (vec_end_addr + 1 != page_addr + off) 819 return false; 820 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 821 return false; 822 823 *same_page = ((vec_end_addr & PAGE_MASK) == page_addr); 824 if (*same_page) 825 return true; 826 return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE); 827 } 828 829 /** 830 * __bio_try_merge_page - try appending data to an existing bvec. 831 * @bio: destination bio 832 * @page: start page to add 833 * @len: length of the data to add 834 * @off: offset of the data relative to @page 835 * @same_page: return if the segment has been merged inside the same page 836 * 837 * Try to add the data at @page + @off to the last bvec of @bio. This is a 838 * useful optimisation for file systems with a block size smaller than the 839 * page size. 840 * 841 * Warn if (@len, @off) crosses pages in case that @same_page is true. 842 * 843 * Return %true on success or %false on failure. 844 */ 845 static bool __bio_try_merge_page(struct bio *bio, struct page *page, 846 unsigned int len, unsigned int off, bool *same_page) 847 { 848 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 849 return false; 850 851 if (bio->bi_vcnt > 0) { 852 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 853 854 if (page_is_mergeable(bv, page, len, off, same_page)) { 855 if (bio->bi_iter.bi_size > UINT_MAX - len) { 856 *same_page = false; 857 return false; 858 } 859 bv->bv_len += len; 860 bio->bi_iter.bi_size += len; 861 return true; 862 } 863 } 864 return false; 865 } 866 867 /* 868 * Try to merge a page into a segment, while obeying the hardware segment 869 * size limit. This is not for normal read/write bios, but for passthrough 870 * or Zone Append operations that we can't split. 871 */ 872 static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio, 873 struct page *page, unsigned len, 874 unsigned offset, bool *same_page) 875 { 876 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 877 unsigned long mask = queue_segment_boundary(q); 878 phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset; 879 phys_addr_t addr2 = page_to_phys(page) + offset + len - 1; 880 881 if ((addr1 | mask) != (addr2 | mask)) 882 return false; 883 if (bv->bv_len + len > queue_max_segment_size(q)) 884 return false; 885 return __bio_try_merge_page(bio, page, len, offset, same_page); 886 } 887 888 /** 889 * bio_add_hw_page - attempt to add a page to a bio with hw constraints 890 * @q: the target queue 891 * @bio: destination bio 892 * @page: page to add 893 * @len: vec entry length 894 * @offset: vec entry offset 895 * @max_sectors: maximum number of sectors that can be added 896 * @same_page: return if the segment has been merged inside the same page 897 * 898 * Add a page to a bio while respecting the hardware max_sectors, max_segment 899 * and gap limitations. 900 */ 901 int bio_add_hw_page(struct request_queue *q, struct bio *bio, 902 struct page *page, unsigned int len, unsigned int offset, 903 unsigned int max_sectors, bool *same_page) 904 { 905 struct bio_vec *bvec; 906 907 if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) 908 return 0; 909 910 if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) 911 return 0; 912 913 if (bio->bi_vcnt > 0) { 914 if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page)) 915 return len; 916 917 /* 918 * If the queue doesn't support SG gaps and adding this segment 919 * would create a gap, disallow it. 920 */ 921 bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; 922 if (bvec_gap_to_prev(q, bvec, offset)) 923 return 0; 924 } 925 926 if (bio_full(bio, len)) 927 return 0; 928 929 if (bio->bi_vcnt >= queue_max_segments(q)) 930 return 0; 931 932 bvec = &bio->bi_io_vec[bio->bi_vcnt]; 933 bvec->bv_page = page; 934 bvec->bv_len = len; 935 bvec->bv_offset = offset; 936 bio->bi_vcnt++; 937 bio->bi_iter.bi_size += len; 938 return len; 939 } 940 941 /** 942 * bio_add_pc_page - attempt to add page to passthrough bio 943 * @q: the target queue 944 * @bio: destination bio 945 * @page: page to add 946 * @len: vec entry length 947 * @offset: vec entry offset 948 * 949 * Attempt to add a page to the bio_vec maplist. This can fail for a 950 * number of reasons, such as the bio being full or target block device 951 * limitations. The target block device must allow bio's up to PAGE_SIZE, 952 * so it is always possible to add a single page to an empty bio. 953 * 954 * This should only be used by passthrough bios. 955 */ 956 int bio_add_pc_page(struct request_queue *q, struct bio *bio, 957 struct page *page, unsigned int len, unsigned int offset) 958 { 959 bool same_page = false; 960 return bio_add_hw_page(q, bio, page, len, offset, 961 queue_max_hw_sectors(q), &same_page); 962 } 963 EXPORT_SYMBOL(bio_add_pc_page); 964 965 /** 966 * bio_add_zone_append_page - attempt to add page to zone-append bio 967 * @bio: destination bio 968 * @page: page to add 969 * @len: vec entry length 970 * @offset: vec entry offset 971 * 972 * Attempt to add a page to the bio_vec maplist of a bio that will be submitted 973 * for a zone-append request. This can fail for a number of reasons, such as the 974 * bio being full or the target block device is not a zoned block device or 975 * other limitations of the target block device. The target block device must 976 * allow bio's up to PAGE_SIZE, so it is always possible to add a single page 977 * to an empty bio. 978 * 979 * Returns: number of bytes added to the bio, or 0 in case of a failure. 980 */ 981 int bio_add_zone_append_page(struct bio *bio, struct page *page, 982 unsigned int len, unsigned int offset) 983 { 984 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 985 bool same_page = false; 986 987 if (WARN_ON_ONCE(bio_op(bio) != REQ_OP_ZONE_APPEND)) 988 return 0; 989 990 if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) 991 return 0; 992 993 return bio_add_hw_page(q, bio, page, len, offset, 994 queue_max_zone_append_sectors(q), &same_page); 995 } 996 EXPORT_SYMBOL_GPL(bio_add_zone_append_page); 997 998 /** 999 * __bio_add_page - add page(s) to a bio in a new segment 1000 * @bio: destination bio 1001 * @page: start page to add 1002 * @len: length of the data to add, may cross pages 1003 * @off: offset of the data relative to @page, may cross pages 1004 * 1005 * Add the data at @page + @off to @bio as a new bvec. The caller must ensure 1006 * that @bio has space for another bvec. 1007 */ 1008 void __bio_add_page(struct bio *bio, struct page *page, 1009 unsigned int len, unsigned int off) 1010 { 1011 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt]; 1012 1013 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 1014 WARN_ON_ONCE(bio_full(bio, len)); 1015 1016 bv->bv_page = page; 1017 bv->bv_offset = off; 1018 bv->bv_len = len; 1019 1020 bio->bi_iter.bi_size += len; 1021 bio->bi_vcnt++; 1022 1023 if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) 1024 bio_set_flag(bio, BIO_WORKINGSET); 1025 } 1026 EXPORT_SYMBOL_GPL(__bio_add_page); 1027 1028 /** 1029 * bio_add_page - attempt to add page(s) to bio 1030 * @bio: destination bio 1031 * @page: start page to add 1032 * @len: vec entry length, may cross pages 1033 * @offset: vec entry offset relative to @page, may cross pages 1034 * 1035 * Attempt to add page(s) to the bio_vec maplist. This will only fail 1036 * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. 1037 */ 1038 int bio_add_page(struct bio *bio, struct page *page, 1039 unsigned int len, unsigned int offset) 1040 { 1041 bool same_page = false; 1042 1043 if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) { 1044 if (bio_full(bio, len)) 1045 return 0; 1046 __bio_add_page(bio, page, len, offset); 1047 } 1048 return len; 1049 } 1050 EXPORT_SYMBOL(bio_add_page); 1051 1052 /** 1053 * bio_add_folio - Attempt to add part of a folio to a bio. 1054 * @bio: BIO to add to. 1055 * @folio: Folio to add. 1056 * @len: How many bytes from the folio to add. 1057 * @off: First byte in this folio to add. 1058 * 1059 * Filesystems that use folios can call this function instead of calling 1060 * bio_add_page() for each page in the folio. If @off is bigger than 1061 * PAGE_SIZE, this function can create a bio_vec that starts in a page 1062 * after the bv_page. BIOs do not support folios that are 4GiB or larger. 1063 * 1064 * Return: Whether the addition was successful. 1065 */ 1066 bool bio_add_folio(struct bio *bio, struct folio *folio, size_t len, 1067 size_t off) 1068 { 1069 if (len > UINT_MAX || off > UINT_MAX) 1070 return 0; 1071 return bio_add_page(bio, &folio->page, len, off) > 0; 1072 } 1073 1074 void __bio_release_pages(struct bio *bio, bool mark_dirty) 1075 { 1076 struct bvec_iter_all iter_all; 1077 struct bio_vec *bvec; 1078 1079 bio_for_each_segment_all(bvec, bio, iter_all) { 1080 if (mark_dirty && !PageCompound(bvec->bv_page)) 1081 set_page_dirty_lock(bvec->bv_page); 1082 put_page(bvec->bv_page); 1083 } 1084 } 1085 EXPORT_SYMBOL_GPL(__bio_release_pages); 1086 1087 void bio_iov_bvec_set(struct bio *bio, struct iov_iter *iter) 1088 { 1089 size_t size = iov_iter_count(iter); 1090 1091 WARN_ON_ONCE(bio->bi_max_vecs); 1092 1093 if (bio_op(bio) == REQ_OP_ZONE_APPEND) { 1094 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1095 size_t max_sectors = queue_max_zone_append_sectors(q); 1096 1097 size = min(size, max_sectors << SECTOR_SHIFT); 1098 } 1099 1100 bio->bi_vcnt = iter->nr_segs; 1101 bio->bi_io_vec = (struct bio_vec *)iter->bvec; 1102 bio->bi_iter.bi_bvec_done = iter->iov_offset; 1103 bio->bi_iter.bi_size = size; 1104 bio_set_flag(bio, BIO_NO_PAGE_REF); 1105 bio_set_flag(bio, BIO_CLONED); 1106 } 1107 1108 static void bio_put_pages(struct page **pages, size_t size, size_t off) 1109 { 1110 size_t i, nr = DIV_ROUND_UP(size + (off & ~PAGE_MASK), PAGE_SIZE); 1111 1112 for (i = 0; i < nr; i++) 1113 put_page(pages[i]); 1114 } 1115 1116 #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) 1117 1118 /** 1119 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio 1120 * @bio: bio to add pages to 1121 * @iter: iov iterator describing the region to be mapped 1122 * 1123 * Pins pages from *iter and appends them to @bio's bvec array. The 1124 * pages will have to be released using put_page() when done. 1125 * For multi-segment *iter, this function only adds pages from the 1126 * next non-empty segment of the iov iterator. 1127 */ 1128 static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 1129 { 1130 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 1131 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; 1132 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 1133 struct page **pages = (struct page **)bv; 1134 bool same_page = false; 1135 ssize_t size, left; 1136 unsigned len, i; 1137 size_t offset; 1138 1139 /* 1140 * Move page array up in the allocated memory for the bio vecs as far as 1141 * possible so that we can start filling biovecs from the beginning 1142 * without overwriting the temporary page array. 1143 */ 1144 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1145 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 1146 1147 size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 1148 if (unlikely(size <= 0)) 1149 return size ? size : -EFAULT; 1150 1151 for (left = size, i = 0; left > 0; left -= len, i++) { 1152 struct page *page = pages[i]; 1153 1154 len = min_t(size_t, PAGE_SIZE - offset, left); 1155 1156 if (__bio_try_merge_page(bio, page, len, offset, &same_page)) { 1157 if (same_page) 1158 put_page(page); 1159 } else { 1160 if (WARN_ON_ONCE(bio_full(bio, len))) { 1161 bio_put_pages(pages + i, left, offset); 1162 return -EINVAL; 1163 } 1164 __bio_add_page(bio, page, len, offset); 1165 } 1166 offset = 0; 1167 } 1168 1169 iov_iter_advance(iter, size); 1170 return 0; 1171 } 1172 1173 static int __bio_iov_append_get_pages(struct bio *bio, struct iov_iter *iter) 1174 { 1175 unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; 1176 unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; 1177 struct request_queue *q = bdev_get_queue(bio->bi_bdev); 1178 unsigned int max_append_sectors = queue_max_zone_append_sectors(q); 1179 struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; 1180 struct page **pages = (struct page **)bv; 1181 ssize_t size, left; 1182 unsigned len, i; 1183 size_t offset; 1184 int ret = 0; 1185 1186 if (WARN_ON_ONCE(!max_append_sectors)) 1187 return 0; 1188 1189 /* 1190 * Move page array up in the allocated memory for the bio vecs as far as 1191 * possible so that we can start filling biovecs from the beginning 1192 * without overwriting the temporary page array. 1193 */ 1194 BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2); 1195 pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); 1196 1197 size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset); 1198 if (unlikely(size <= 0)) 1199 return size ? size : -EFAULT; 1200 1201 for (left = size, i = 0; left > 0; left -= len, i++) { 1202 struct page *page = pages[i]; 1203 bool same_page = false; 1204 1205 len = min_t(size_t, PAGE_SIZE - offset, left); 1206 if (bio_add_hw_page(q, bio, page, len, offset, 1207 max_append_sectors, &same_page) != len) { 1208 bio_put_pages(pages + i, left, offset); 1209 ret = -EINVAL; 1210 break; 1211 } 1212 if (same_page) 1213 put_page(page); 1214 offset = 0; 1215 } 1216 1217 iov_iter_advance(iter, size - left); 1218 return ret; 1219 } 1220 1221 /** 1222 * bio_iov_iter_get_pages - add user or kernel pages to a bio 1223 * @bio: bio to add pages to 1224 * @iter: iov iterator describing the region to be added 1225 * 1226 * This takes either an iterator pointing to user memory, or one pointing to 1227 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and 1228 * map them into the kernel. On IO completion, the caller should put those 1229 * pages. For bvec based iterators bio_iov_iter_get_pages() uses the provided 1230 * bvecs rather than copying them. Hence anyone issuing kiocb based IO needs 1231 * to ensure the bvecs and pages stay referenced until the submitted I/O is 1232 * completed by a call to ->ki_complete() or returns with an error other than 1233 * -EIOCBQUEUED. The caller needs to check if the bio is flagged BIO_NO_PAGE_REF 1234 * on IO completion. If it isn't, then pages should be released. 1235 * 1236 * The function tries, but does not guarantee, to pin as many pages as 1237 * fit into the bio, or are requested in @iter, whatever is smaller. If 1238 * MM encounters an error pinning the requested pages, it stops. Error 1239 * is returned only if 0 pages could be pinned. 1240 * 1241 * It's intended for direct IO, so doesn't do PSI tracking, the caller is 1242 * responsible for setting BIO_WORKINGSET if necessary. 1243 */ 1244 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) 1245 { 1246 int ret = 0; 1247 1248 if (iov_iter_is_bvec(iter)) { 1249 bio_iov_bvec_set(bio, iter); 1250 iov_iter_advance(iter, bio->bi_iter.bi_size); 1251 return 0; 1252 } 1253 1254 do { 1255 if (bio_op(bio) == REQ_OP_ZONE_APPEND) 1256 ret = __bio_iov_append_get_pages(bio, iter); 1257 else 1258 ret = __bio_iov_iter_get_pages(bio, iter); 1259 } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); 1260 1261 /* don't account direct I/O as memory stall */ 1262 bio_clear_flag(bio, BIO_WORKINGSET); 1263 return bio->bi_vcnt ? 0 : ret; 1264 } 1265 EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); 1266 1267 static void submit_bio_wait_endio(struct bio *bio) 1268 { 1269 complete(bio->bi_private); 1270 } 1271 1272 /** 1273 * submit_bio_wait - submit a bio, and wait until it completes 1274 * @bio: The &struct bio which describes the I/O 1275 * 1276 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from 1277 * bio_endio() on failure. 1278 * 1279 * WARNING: Unlike to how submit_bio() is usually used, this function does not 1280 * result in bio reference to be consumed. The caller must drop the reference 1281 * on his own. 1282 */ 1283 int submit_bio_wait(struct bio *bio) 1284 { 1285 DECLARE_COMPLETION_ONSTACK_MAP(done, 1286 bio->bi_bdev->bd_disk->lockdep_map); 1287 unsigned long hang_check; 1288 1289 bio->bi_private = &done; 1290 bio->bi_end_io = submit_bio_wait_endio; 1291 bio->bi_opf |= REQ_SYNC; 1292 submit_bio(bio); 1293 1294 /* Prevent hang_check timer from firing at us during very long I/O */ 1295 hang_check = sysctl_hung_task_timeout_secs; 1296 if (hang_check) 1297 while (!wait_for_completion_io_timeout(&done, 1298 hang_check * (HZ/2))) 1299 ; 1300 else 1301 wait_for_completion_io(&done); 1302 1303 return blk_status_to_errno(bio->bi_status); 1304 } 1305 EXPORT_SYMBOL(submit_bio_wait); 1306 1307 void __bio_advance(struct bio *bio, unsigned bytes) 1308 { 1309 if (bio_integrity(bio)) 1310 bio_integrity_advance(bio, bytes); 1311 1312 bio_crypt_advance(bio, bytes); 1313 bio_advance_iter(bio, &bio->bi_iter, bytes); 1314 } 1315 EXPORT_SYMBOL(__bio_advance); 1316 1317 void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, 1318 struct bio *src, struct bvec_iter *src_iter) 1319 { 1320 while (src_iter->bi_size && dst_iter->bi_size) { 1321 struct bio_vec src_bv = bio_iter_iovec(src, *src_iter); 1322 struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter); 1323 unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); 1324 void *src_buf; 1325 1326 src_buf = bvec_kmap_local(&src_bv); 1327 memcpy_to_bvec(&dst_bv, src_buf); 1328 kunmap_local(src_buf); 1329 1330 bio_advance_iter_single(src, src_iter, bytes); 1331 bio_advance_iter_single(dst, dst_iter, bytes); 1332 } 1333 } 1334 EXPORT_SYMBOL(bio_copy_data_iter); 1335 1336 /** 1337 * bio_copy_data - copy contents of data buffers from one bio to another 1338 * @src: source bio 1339 * @dst: destination bio 1340 * 1341 * Stops when it reaches the end of either @src or @dst - that is, copies 1342 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios). 1343 */ 1344 void bio_copy_data(struct bio *dst, struct bio *src) 1345 { 1346 struct bvec_iter src_iter = src->bi_iter; 1347 struct bvec_iter dst_iter = dst->bi_iter; 1348 1349 bio_copy_data_iter(dst, &dst_iter, src, &src_iter); 1350 } 1351 EXPORT_SYMBOL(bio_copy_data); 1352 1353 void bio_free_pages(struct bio *bio) 1354 { 1355 struct bio_vec *bvec; 1356 struct bvec_iter_all iter_all; 1357 1358 bio_for_each_segment_all(bvec, bio, iter_all) 1359 __free_page(bvec->bv_page); 1360 } 1361 EXPORT_SYMBOL(bio_free_pages); 1362 1363 /* 1364 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions 1365 * for performing direct-IO in BIOs. 1366 * 1367 * The problem is that we cannot run set_page_dirty() from interrupt context 1368 * because the required locks are not interrupt-safe. So what we can do is to 1369 * mark the pages dirty _before_ performing IO. And in interrupt context, 1370 * check that the pages are still dirty. If so, fine. If not, redirty them 1371 * in process context. 1372 * 1373 * We special-case compound pages here: normally this means reads into hugetlb 1374 * pages. The logic in here doesn't really work right for compound pages 1375 * because the VM does not uniformly chase down the head page in all cases. 1376 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't 1377 * handle them at all. So we skip compound pages here at an early stage. 1378 * 1379 * Note that this code is very hard to test under normal circumstances because 1380 * direct-io pins the pages with get_user_pages(). This makes 1381 * is_page_cache_freeable return false, and the VM will not clean the pages. 1382 * But other code (eg, flusher threads) could clean the pages if they are mapped 1383 * pagecache. 1384 * 1385 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the 1386 * deferred bio dirtying paths. 1387 */ 1388 1389 /* 1390 * bio_set_pages_dirty() will mark all the bio's pages as dirty. 1391 */ 1392 void bio_set_pages_dirty(struct bio *bio) 1393 { 1394 struct bio_vec *bvec; 1395 struct bvec_iter_all iter_all; 1396 1397 bio_for_each_segment_all(bvec, bio, iter_all) { 1398 if (!PageCompound(bvec->bv_page)) 1399 set_page_dirty_lock(bvec->bv_page); 1400 } 1401 } 1402 1403 /* 1404 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty. 1405 * If they are, then fine. If, however, some pages are clean then they must 1406 * have been written out during the direct-IO read. So we take another ref on 1407 * the BIO and re-dirty the pages in process context. 1408 * 1409 * It is expected that bio_check_pages_dirty() will wholly own the BIO from 1410 * here on. It will run one put_page() against each page and will run one 1411 * bio_put() against the BIO. 1412 */ 1413 1414 static void bio_dirty_fn(struct work_struct *work); 1415 1416 static DECLARE_WORK(bio_dirty_work, bio_dirty_fn); 1417 static DEFINE_SPINLOCK(bio_dirty_lock); 1418 static struct bio *bio_dirty_list; 1419 1420 /* 1421 * This runs in process context 1422 */ 1423 static void bio_dirty_fn(struct work_struct *work) 1424 { 1425 struct bio *bio, *next; 1426 1427 spin_lock_irq(&bio_dirty_lock); 1428 next = bio_dirty_list; 1429 bio_dirty_list = NULL; 1430 spin_unlock_irq(&bio_dirty_lock); 1431 1432 while ((bio = next) != NULL) { 1433 next = bio->bi_private; 1434 1435 bio_release_pages(bio, true); 1436 bio_put(bio); 1437 } 1438 } 1439 1440 void bio_check_pages_dirty(struct bio *bio) 1441 { 1442 struct bio_vec *bvec; 1443 unsigned long flags; 1444 struct bvec_iter_all iter_all; 1445 1446 bio_for_each_segment_all(bvec, bio, iter_all) { 1447 if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) 1448 goto defer; 1449 } 1450 1451 bio_release_pages(bio, false); 1452 bio_put(bio); 1453 return; 1454 defer: 1455 spin_lock_irqsave(&bio_dirty_lock, flags); 1456 bio->bi_private = bio_dirty_list; 1457 bio_dirty_list = bio; 1458 spin_unlock_irqrestore(&bio_dirty_lock, flags); 1459 schedule_work(&bio_dirty_work); 1460 } 1461 1462 static inline bool bio_remaining_done(struct bio *bio) 1463 { 1464 /* 1465 * If we're not chaining, then ->__bi_remaining is always 1 and 1466 * we always end io on the first invocation. 1467 */ 1468 if (!bio_flagged(bio, BIO_CHAIN)) 1469 return true; 1470 1471 BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); 1472 1473 if (atomic_dec_and_test(&bio->__bi_remaining)) { 1474 bio_clear_flag(bio, BIO_CHAIN); 1475 return true; 1476 } 1477 1478 return false; 1479 } 1480 1481 /** 1482 * bio_endio - end I/O on a bio 1483 * @bio: bio 1484 * 1485 * Description: 1486 * bio_endio() will end I/O on the whole bio. bio_endio() is the preferred 1487 * way to end I/O on a bio. No one should call bi_end_io() directly on a 1488 * bio unless they own it and thus know that it has an end_io function. 1489 * 1490 * bio_endio() can be called several times on a bio that has been chained 1491 * using bio_chain(). The ->bi_end_io() function will only be called the 1492 * last time. 1493 **/ 1494 void bio_endio(struct bio *bio) 1495 { 1496 again: 1497 if (!bio_remaining_done(bio)) 1498 return; 1499 if (!bio_integrity_endio(bio)) 1500 return; 1501 1502 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACKED)) 1503 rq_qos_done_bio(bdev_get_queue(bio->bi_bdev), bio); 1504 1505 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { 1506 trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio); 1507 bio_clear_flag(bio, BIO_TRACE_COMPLETION); 1508 } 1509 1510 /* 1511 * Need to have a real endio function for chained bios, otherwise 1512 * various corner cases will break (like stacking block devices that 1513 * save/restore bi_end_io) - however, we want to avoid unbounded 1514 * recursion and blowing the stack. Tail call optimization would 1515 * handle this, but compiling with frame pointers also disables 1516 * gcc's sibling call optimization. 1517 */ 1518 if (bio->bi_end_io == bio_chain_endio) { 1519 bio = __bio_chain_endio(bio); 1520 goto again; 1521 } 1522 1523 blk_throtl_bio_endio(bio); 1524 /* release cgroup info */ 1525 bio_uninit(bio); 1526 if (bio->bi_end_io) 1527 bio->bi_end_io(bio); 1528 } 1529 EXPORT_SYMBOL(bio_endio); 1530 1531 /** 1532 * bio_split - split a bio 1533 * @bio: bio to split 1534 * @sectors: number of sectors to split from the front of @bio 1535 * @gfp: gfp mask 1536 * @bs: bio set to allocate from 1537 * 1538 * Allocates and returns a new bio which represents @sectors from the start of 1539 * @bio, and updates @bio to represent the remaining sectors. 1540 * 1541 * Unless this is a discard request the newly allocated bio will point 1542 * to @bio's bi_io_vec. It is the caller's responsibility to ensure that 1543 * neither @bio nor @bs are freed before the split bio. 1544 */ 1545 struct bio *bio_split(struct bio *bio, int sectors, 1546 gfp_t gfp, struct bio_set *bs) 1547 { 1548 struct bio *split; 1549 1550 BUG_ON(sectors <= 0); 1551 BUG_ON(sectors >= bio_sectors(bio)); 1552 1553 /* Zone append commands cannot be split */ 1554 if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) 1555 return NULL; 1556 1557 split = bio_clone_fast(bio, gfp, bs); 1558 if (!split) 1559 return NULL; 1560 1561 split->bi_iter.bi_size = sectors << 9; 1562 1563 if (bio_integrity(split)) 1564 bio_integrity_trim(split); 1565 1566 bio_advance(bio, split->bi_iter.bi_size); 1567 1568 if (bio_flagged(bio, BIO_TRACE_COMPLETION)) 1569 bio_set_flag(split, BIO_TRACE_COMPLETION); 1570 1571 return split; 1572 } 1573 EXPORT_SYMBOL(bio_split); 1574 1575 /** 1576 * bio_trim - trim a bio 1577 * @bio: bio to trim 1578 * @offset: number of sectors to trim from the front of @bio 1579 * @size: size we want to trim @bio to, in sectors 1580 * 1581 * This function is typically used for bios that are cloned and submitted 1582 * to the underlying device in parts. 1583 */ 1584 void bio_trim(struct bio *bio, sector_t offset, sector_t size) 1585 { 1586 if (WARN_ON_ONCE(offset > BIO_MAX_SECTORS || size > BIO_MAX_SECTORS || 1587 offset + size > bio->bi_iter.bi_size)) 1588 return; 1589 1590 size <<= 9; 1591 if (offset == 0 && size == bio->bi_iter.bi_size) 1592 return; 1593 1594 bio_advance(bio, offset << 9); 1595 bio->bi_iter.bi_size = size; 1596 1597 if (bio_integrity(bio)) 1598 bio_integrity_trim(bio); 1599 } 1600 EXPORT_SYMBOL_GPL(bio_trim); 1601 1602 /* 1603 * create memory pools for biovec's in a bio_set. 1604 * use the global biovec slabs created for general use. 1605 */ 1606 int biovec_init_pool(mempool_t *pool, int pool_entries) 1607 { 1608 struct biovec_slab *bp = bvec_slabs + ARRAY_SIZE(bvec_slabs) - 1; 1609 1610 return mempool_init_slab_pool(pool, pool_entries, bp->slab); 1611 } 1612 1613 /* 1614 * bioset_exit - exit a bioset initialized with bioset_init() 1615 * 1616 * May be called on a zeroed but uninitialized bioset (i.e. allocated with 1617 * kzalloc()). 1618 */ 1619 void bioset_exit(struct bio_set *bs) 1620 { 1621 bio_alloc_cache_destroy(bs); 1622 if (bs->rescue_workqueue) 1623 destroy_workqueue(bs->rescue_workqueue); 1624 bs->rescue_workqueue = NULL; 1625 1626 mempool_exit(&bs->bio_pool); 1627 mempool_exit(&bs->bvec_pool); 1628 1629 bioset_integrity_free(bs); 1630 if (bs->bio_slab) 1631 bio_put_slab(bs); 1632 bs->bio_slab = NULL; 1633 } 1634 EXPORT_SYMBOL(bioset_exit); 1635 1636 /** 1637 * bioset_init - Initialize a bio_set 1638 * @bs: pool to initialize 1639 * @pool_size: Number of bio and bio_vecs to cache in the mempool 1640 * @front_pad: Number of bytes to allocate in front of the returned bio 1641 * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS 1642 * and %BIOSET_NEED_RESCUER 1643 * 1644 * Description: 1645 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller 1646 * to ask for a number of bytes to be allocated in front of the bio. 1647 * Front pad allocation is useful for embedding the bio inside 1648 * another structure, to avoid allocating extra data to go with the bio. 1649 * Note that the bio must be embedded at the END of that structure always, 1650 * or things will break badly. 1651 * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated 1652 * for allocating iovecs. This pool is not needed e.g. for bio_clone_fast(). 1653 * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to 1654 * dispatch queued requests when the mempool runs out of space. 1655 * 1656 */ 1657 int bioset_init(struct bio_set *bs, 1658 unsigned int pool_size, 1659 unsigned int front_pad, 1660 int flags) 1661 { 1662 bs->front_pad = front_pad; 1663 if (flags & BIOSET_NEED_BVECS) 1664 bs->back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); 1665 else 1666 bs->back_pad = 0; 1667 1668 spin_lock_init(&bs->rescue_lock); 1669 bio_list_init(&bs->rescue_list); 1670 INIT_WORK(&bs->rescue_work, bio_alloc_rescue); 1671 1672 bs->bio_slab = bio_find_or_create_slab(bs); 1673 if (!bs->bio_slab) 1674 return -ENOMEM; 1675 1676 if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab)) 1677 goto bad; 1678 1679 if ((flags & BIOSET_NEED_BVECS) && 1680 biovec_init_pool(&bs->bvec_pool, pool_size)) 1681 goto bad; 1682 1683 if (flags & BIOSET_NEED_RESCUER) { 1684 bs->rescue_workqueue = alloc_workqueue("bioset", 1685 WQ_MEM_RECLAIM, 0); 1686 if (!bs->rescue_workqueue) 1687 goto bad; 1688 } 1689 if (flags & BIOSET_PERCPU_CACHE) { 1690 bs->cache = alloc_percpu(struct bio_alloc_cache); 1691 if (!bs->cache) 1692 goto bad; 1693 cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); 1694 } 1695 1696 return 0; 1697 bad: 1698 bioset_exit(bs); 1699 return -ENOMEM; 1700 } 1701 EXPORT_SYMBOL(bioset_init); 1702 1703 /* 1704 * Initialize and setup a new bio_set, based on the settings from 1705 * another bio_set. 1706 */ 1707 int bioset_init_from_src(struct bio_set *bs, struct bio_set *src) 1708 { 1709 int flags; 1710 1711 flags = 0; 1712 if (src->bvec_pool.min_nr) 1713 flags |= BIOSET_NEED_BVECS; 1714 if (src->rescue_workqueue) 1715 flags |= BIOSET_NEED_RESCUER; 1716 1717 return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags); 1718 } 1719 EXPORT_SYMBOL(bioset_init_from_src); 1720 1721 /** 1722 * bio_alloc_kiocb - Allocate a bio from bio_set based on kiocb 1723 * @kiocb: kiocb describing the IO 1724 * @nr_vecs: number of iovecs to pre-allocate 1725 * @bs: bio_set to allocate from 1726 * 1727 * Description: 1728 * Like @bio_alloc_bioset, but pass in the kiocb. The kiocb is only 1729 * used to check if we should dip into the per-cpu bio_set allocation 1730 * cache. The allocation uses GFP_KERNEL internally. On return, the 1731 * bio is marked BIO_PERCPU_CACHEABLE, and the final put of the bio 1732 * MUST be done from process context, not hard/soft IRQ. 1733 * 1734 */ 1735 struct bio *bio_alloc_kiocb(struct kiocb *kiocb, unsigned short nr_vecs, 1736 struct bio_set *bs) 1737 { 1738 struct bio_alloc_cache *cache; 1739 struct bio *bio; 1740 1741 if (!(kiocb->ki_flags & IOCB_ALLOC_CACHE) || nr_vecs > BIO_INLINE_VECS) 1742 return bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs); 1743 1744 cache = per_cpu_ptr(bs->cache, get_cpu()); 1745 if (cache->free_list) { 1746 bio = cache->free_list; 1747 cache->free_list = bio->bi_next; 1748 cache->nr--; 1749 put_cpu(); 1750 bio_init(bio, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs); 1751 bio->bi_pool = bs; 1752 bio_set_flag(bio, BIO_PERCPU_CACHE); 1753 return bio; 1754 } 1755 put_cpu(); 1756 bio = bio_alloc_bioset(GFP_KERNEL, nr_vecs, bs); 1757 bio_set_flag(bio, BIO_PERCPU_CACHE); 1758 return bio; 1759 } 1760 EXPORT_SYMBOL_GPL(bio_alloc_kiocb); 1761 1762 static int __init init_bio(void) 1763 { 1764 int i; 1765 1766 bio_integrity_init(); 1767 1768 for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { 1769 struct biovec_slab *bvs = bvec_slabs + i; 1770 1771 bvs->slab = kmem_cache_create(bvs->name, 1772 bvs->nr_vecs * sizeof(struct bio_vec), 0, 1773 SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); 1774 } 1775 1776 cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, 1777 bio_cpu_dead); 1778 1779 if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS)) 1780 panic("bio: can't allocate bios\n"); 1781 1782 if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE)) 1783 panic("bio: can't create integrity pool\n"); 1784 1785 return 0; 1786 } 1787 subsys_initcall(init_bio); 1788