bio.c (d4e0045c4ed300781d2d4cbab57d05ed5e665a37) bio.c (cb6934f8ea1a595902ca37e250e0917d4dd7b2a7)
1/*
2 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,

--- 295 unchanged lines hidden (view full) ---

304 atomic_set(&bio->__bi_remaining, 1);
305}
306EXPORT_SYMBOL(bio_reset);
307
308static struct bio *__bio_chain_endio(struct bio *bio)
309{
310 struct bio *parent = bio->bi_private;
311
1/*
2 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,

--- 295 unchanged lines hidden (view full) ---

304 atomic_set(&bio->__bi_remaining, 1);
305}
306EXPORT_SYMBOL(bio_reset);
307
308static struct bio *__bio_chain_endio(struct bio *bio)
309{
310 struct bio *parent = bio->bi_private;
311
312 if (!parent->bi_error)
313 parent->bi_error = bio->bi_error;
312 if (!parent->bi_status)
313 parent->bi_status = bio->bi_status;
314 bio_put(bio);
315 return parent;
316}
317
318static void bio_chain_endio(struct bio *bio)
319{
320 bio_endio(__bio_chain_endio(bio));
321}

--- 36 unchanged lines hidden (view full) ---

358 }
359}
360
361static void punt_bios_to_rescuer(struct bio_set *bs)
362{
363 struct bio_list punt, nopunt;
364 struct bio *bio;
365
314 bio_put(bio);
315 return parent;
316}
317
318static void bio_chain_endio(struct bio *bio)
319{
320 bio_endio(__bio_chain_endio(bio));
321}

--- 36 unchanged lines hidden (view full) ---

358 }
359}
360
361static void punt_bios_to_rescuer(struct bio_set *bs)
362{
363 struct bio_list punt, nopunt;
364 struct bio *bio;
365
366 if (WARN_ON_ONCE(!bs->rescue_workqueue))
367 return;
366 /*
367 * In order to guarantee forward progress we must punt only bios that
368 * were allocated from this bio_set; otherwise, if there was a bio on
369 * there for a stacking driver higher up in the stack, processing it
370 * could require allocating bios from this bio_set, and doing that from
371 * our own rescuer would be bad.
372 *
373 * Since bio lists are singly linked, pop them all instead of trying to

--- 95 unchanged lines hidden (view full) ---

469 * bios on current->bio_list, we first try the allocation
470 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
471 * bios we would be blocking to the rescuer workqueue before
472 * we retry with the original gfp_flags.
473 */
474
475 if (current->bio_list &&
476 (!bio_list_empty(&current->bio_list[0]) ||
368 /*
369 * In order to guarantee forward progress we must punt only bios that
370 * were allocated from this bio_set; otherwise, if there was a bio on
371 * there for a stacking driver higher up in the stack, processing it
372 * could require allocating bios from this bio_set, and doing that from
373 * our own rescuer would be bad.
374 *
375 * Since bio lists are singly linked, pop them all instead of trying to

--- 95 unchanged lines hidden (view full) ---

471 * bios on current->bio_list, we first try the allocation
472 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
473 * bios we would be blocking to the rescuer workqueue before
474 * we retry with the original gfp_flags.
475 */
476
477 if (current->bio_list &&
478 (!bio_list_empty(&current->bio_list[0]) ||
477 !bio_list_empty(&current->bio_list[1])))
479 !bio_list_empty(&current->bio_list[1])) &&
480 bs->rescue_workqueue)
478 gfp_mask &= ~__GFP_DIRECT_RECLAIM;
479
480 p = mempool_alloc(bs->bio_pool, gfp_mask);
481 if (!p && gfp_mask != saved_gfp) {
482 punt_bios_to_rescuer(bs);
483 gfp_mask = saved_gfp;
484 p = mempool_alloc(bs->bio_pool, gfp_mask);
485 }

--- 53 unchanged lines hidden (view full) ---

539EXPORT_SYMBOL(zero_fill_bio);
540
541/**
542 * bio_put - release a reference to a bio
543 * @bio: bio to release reference to
544 *
545 * Description:
546 * Put a reference to a &struct bio, either one you have gotten with
481 gfp_mask &= ~__GFP_DIRECT_RECLAIM;
482
483 p = mempool_alloc(bs->bio_pool, gfp_mask);
484 if (!p && gfp_mask != saved_gfp) {
485 punt_bios_to_rescuer(bs);
486 gfp_mask = saved_gfp;
487 p = mempool_alloc(bs->bio_pool, gfp_mask);
488 }

--- 53 unchanged lines hidden (view full) ---

542EXPORT_SYMBOL(zero_fill_bio);
543
544/**
545 * bio_put - release a reference to a bio
546 * @bio: bio to release reference to
547 *
548 * Description:
549 * Put a reference to a &struct bio, either one you have gotten with
547 * bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
550 * bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
548 **/
549void bio_put(struct bio *bio)
550{
551 if (!bio_flagged(bio, BIO_REFFED))
552 bio_free(bio);
553 else {
554 BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
555

--- 32 unchanged lines hidden (view full) ---

588
589 /*
590 * most users will be overriding ->bi_bdev with a new target,
591 * so we don't set nor calculate new physical/hw segment counts here
592 */
593 bio->bi_bdev = bio_src->bi_bdev;
594 bio_set_flag(bio, BIO_CLONED);
595 bio->bi_opf = bio_src->bi_opf;
551 **/
552void bio_put(struct bio *bio)
553{
554 if (!bio_flagged(bio, BIO_REFFED))
555 bio_free(bio);
556 else {
557 BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
558

--- 32 unchanged lines hidden (view full) ---

591
592 /*
593 * most users will be overriding ->bi_bdev with a new target,
594 * so we don't set nor calculate new physical/hw segment counts here
595 */
596 bio->bi_bdev = bio_src->bi_bdev;
597 bio_set_flag(bio, BIO_CLONED);
598 bio->bi_opf = bio_src->bi_opf;
599 bio->bi_write_hint = bio_src->bi_write_hint;
596 bio->bi_iter = bio_src->bi_iter;
597 bio->bi_io_vec = bio_src->bi_io_vec;
598
599 bio_clone_blkcg_association(bio, bio_src);
600}
601EXPORT_SYMBOL(__bio_clone_fast);
602
603/**

--- 67 unchanged lines hidden (view full) ---

671 * __bio_clone_fast() anyways.
672 */
673
674 bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
675 if (!bio)
676 return NULL;
677 bio->bi_bdev = bio_src->bi_bdev;
678 bio->bi_opf = bio_src->bi_opf;
600 bio->bi_iter = bio_src->bi_iter;
601 bio->bi_io_vec = bio_src->bi_io_vec;
602
603 bio_clone_blkcg_association(bio, bio_src);
604}
605EXPORT_SYMBOL(__bio_clone_fast);
606
607/**

--- 67 unchanged lines hidden (view full) ---

675 * __bio_clone_fast() anyways.
676 */
677
678 bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
679 if (!bio)
680 return NULL;
681 bio->bi_bdev = bio_src->bi_bdev;
682 bio->bi_opf = bio_src->bi_opf;
683 bio->bi_write_hint = bio_src->bi_write_hint;
679 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
680 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
681
682 switch (bio_op(bio)) {
683 case REQ_OP_DISCARD:
684 case REQ_OP_SECURE_ERASE:
685 case REQ_OP_WRITE_ZEROES:
686 break;

--- 226 unchanged lines hidden (view full) ---

913 struct completion event;
914 int error;
915};
916
917static void submit_bio_wait_endio(struct bio *bio)
918{
919 struct submit_bio_ret *ret = bio->bi_private;
920
684 bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector;
685 bio->bi_iter.bi_size = bio_src->bi_iter.bi_size;
686
687 switch (bio_op(bio)) {
688 case REQ_OP_DISCARD:
689 case REQ_OP_SECURE_ERASE:
690 case REQ_OP_WRITE_ZEROES:
691 break;

--- 226 unchanged lines hidden (view full) ---

918 struct completion event;
919 int error;
920};
921
922static void submit_bio_wait_endio(struct bio *bio)
923{
924 struct submit_bio_ret *ret = bio->bi_private;
925
921 ret->error = bio->bi_error;
926 ret->error = blk_status_to_errno(bio->bi_status);
922 complete(&ret->event);
923}
924
925/**
926 * submit_bio_wait - submit a bio, and wait until it completes
927 * @bio: The &struct bio which describes the I/O
928 *
929 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from

--- 882 unchanged lines hidden (view full) ---

1812 * gcc's sibling call optimization.
1813 */
1814 if (bio->bi_end_io == bio_chain_endio) {
1815 bio = __bio_chain_endio(bio);
1816 goto again;
1817 }
1818
1819 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
927 complete(&ret->event);
928}
929
930/**
931 * submit_bio_wait - submit a bio, and wait until it completes
932 * @bio: The &struct bio which describes the I/O
933 *
934 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from

--- 882 unchanged lines hidden (view full) ---

1817 * gcc's sibling call optimization.
1818 */
1819 if (bio->bi_end_io == bio_chain_endio) {
1820 bio = __bio_chain_endio(bio);
1821 goto again;
1822 }
1823
1824 if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1820 trace_block_bio_complete(bdev_get_queue(bio->bi_bdev),
1821 bio, bio->bi_error);
1825 trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio,
1826 blk_status_to_errno(bio->bi_status));
1822 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1823 }
1824
1825 blk_throtl_bio_endio(bio);
1826 if (bio->bi_end_io)
1827 bio->bi_end_io(bio);
1828}
1829EXPORT_SYMBOL(bio_endio);

--- 86 unchanged lines hidden (view full) ---

1916
1917 bioset_integrity_free(bs);
1918 bio_put_slab(bs);
1919
1920 kfree(bs);
1921}
1922EXPORT_SYMBOL(bioset_free);
1923
1827 bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1828 }
1829
1830 blk_throtl_bio_endio(bio);
1831 if (bio->bi_end_io)
1832 bio->bi_end_io(bio);
1833}
1834EXPORT_SYMBOL(bio_endio);

--- 86 unchanged lines hidden (view full) ---

1921
1922 bioset_integrity_free(bs);
1923 bio_put_slab(bs);
1924
1925 kfree(bs);
1926}
1927EXPORT_SYMBOL(bioset_free);
1928
1924static struct bio_set *__bioset_create(unsigned int pool_size,
1925 unsigned int front_pad,
1926 bool create_bvec_pool)
1929/**
1930 * bioset_create - Create a bio_set
1931 * @pool_size: Number of bio and bio_vecs to cache in the mempool
1932 * @front_pad: Number of bytes to allocate in front of the returned bio
1933 * @flags: Flags to modify behavior, currently %BIOSET_NEED_BVECS
1934 * and %BIOSET_NEED_RESCUER
1935 *
1936 * Description:
1937 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1938 * to ask for a number of bytes to be allocated in front of the bio.
1939 * Front pad allocation is useful for embedding the bio inside
1940 * another structure, to avoid allocating extra data to go with the bio.
1941 * Note that the bio must be embedded at the END of that structure always,
1942 * or things will break badly.
1943 * If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
1944 * for allocating iovecs. This pool is not needed e.g. for bio_clone_fast().
1945 * If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to
1946 * dispatch queued requests when the mempool runs out of space.
1947 *
1948 */
1949struct bio_set *bioset_create(unsigned int pool_size,
1950 unsigned int front_pad,
1951 int flags)
1927{
1928 unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1929 struct bio_set *bs;
1930
1931 bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1932 if (!bs)
1933 return NULL;
1934

--- 8 unchanged lines hidden (view full) ---

1943 kfree(bs);
1944 return NULL;
1945 }
1946
1947 bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1948 if (!bs->bio_pool)
1949 goto bad;
1950
1952{
1953 unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1954 struct bio_set *bs;
1955
1956 bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1957 if (!bs)
1958 return NULL;
1959

--- 8 unchanged lines hidden (view full) ---

1968 kfree(bs);
1969 return NULL;
1970 }
1971
1972 bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1973 if (!bs->bio_pool)
1974 goto bad;
1975
1951 if (create_bvec_pool) {
1976 if (flags & BIOSET_NEED_BVECS) {
1952 bs->bvec_pool = biovec_create_pool(pool_size);
1953 if (!bs->bvec_pool)
1954 goto bad;
1955 }
1956
1977 bs->bvec_pool = biovec_create_pool(pool_size);
1978 if (!bs->bvec_pool)
1979 goto bad;
1980 }
1981
1982 if (!(flags & BIOSET_NEED_RESCUER))
1983 return bs;
1984
1957 bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1958 if (!bs->rescue_workqueue)
1959 goto bad;
1960
1961 return bs;
1962bad:
1963 bioset_free(bs);
1964 return NULL;
1965}
1985 bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1986 if (!bs->rescue_workqueue)
1987 goto bad;
1988
1989 return bs;
1990bad:
1991 bioset_free(bs);
1992 return NULL;
1993}
1966
1967/**
1968 * bioset_create - Create a bio_set
1969 * @pool_size: Number of bio and bio_vecs to cache in the mempool
1970 * @front_pad: Number of bytes to allocate in front of the returned bio
1971 *
1972 * Description:
1973 * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1974 * to ask for a number of bytes to be allocated in front of the bio.
1975 * Front pad allocation is useful for embedding the bio inside
1976 * another structure, to avoid allocating extra data to go with the bio.
1977 * Note that the bio must be embedded at the END of that structure always,
1978 * or things will break badly.
1979 */
1980struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1981{
1982 return __bioset_create(pool_size, front_pad, true);
1983}
1984EXPORT_SYMBOL(bioset_create);
1985
1994EXPORT_SYMBOL(bioset_create);
1995
1986/**
1987 * bioset_create_nobvec - Create a bio_set without bio_vec mempool
1988 * @pool_size: Number of bio to cache in the mempool
1989 * @front_pad: Number of bytes to allocate in front of the returned bio
1990 *
1991 * Description:
1992 * Same functionality as bioset_create() except that mempool is not
1993 * created for bio_vecs. Saving some memory for bio_clone_fast() users.
1994 */
1995struct bio_set *bioset_create_nobvec(unsigned int pool_size, unsigned int front_pad)
1996{
1997 return __bioset_create(pool_size, front_pad, false);
1998}
1999EXPORT_SYMBOL(bioset_create_nobvec);
2000
2001#ifdef CONFIG_BLK_CGROUP
2002
2003/**
2004 * bio_associate_blkcg - associate a bio with the specified blkcg
2005 * @bio: target bio
2006 * @blkcg_css: css of the blkcg to associate
2007 *
2008 * Associate @bio with the blkcg specified by @blkcg_css. Block layer will

--- 98 unchanged lines hidden (view full) ---

2107 bio_slab_nr = 0;
2108 bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
2109 if (!bio_slabs)
2110 panic("bio: can't allocate bios\n");
2111
2112 bio_integrity_init();
2113 biovec_init_slabs();
2114
1996#ifdef CONFIG_BLK_CGROUP
1997
1998/**
1999 * bio_associate_blkcg - associate a bio with the specified blkcg
2000 * @bio: target bio
2001 * @blkcg_css: css of the blkcg to associate
2002 *
2003 * Associate @bio with the blkcg specified by @blkcg_css. Block layer will

--- 98 unchanged lines hidden (view full) ---

2102 bio_slab_nr = 0;
2103 bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
2104 if (!bio_slabs)
2105 panic("bio: can't allocate bios\n");
2106
2107 bio_integrity_init();
2108 biovec_init_slabs();
2109
2115 fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
2110 fs_bio_set = bioset_create(BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
2116 if (!fs_bio_set)
2117 panic("bio: can't allocate bios\n");
2118
2119 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2120 panic("bio: can't create integrity pool\n");
2121
2122 return 0;
2123}
2124subsys_initcall(init_bio);
2111 if (!fs_bio_set)
2112 panic("bio: can't allocate bios\n");
2113
2114 if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2115 panic("bio: can't create integrity pool\n");
2116
2117 return 0;
2118}
2119subsys_initcall(init_bio);