xref: /openbmc/linux/block/bio.c (revision 3a83f4677539bce8eaa2bca9ee9c20e172d7ab04)
1 /*
2  * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
3  *
4  * This program is free software; you can redistribute it and/or modify
5  * it under the terms of the GNU General Public License version 2 as
6  * published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public Licens
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
16  *
17  */
18 #include <linux/mm.h>
19 #include <linux/swap.h>
20 #include <linux/bio.h>
21 #include <linux/blkdev.h>
22 #include <linux/uio.h>
23 #include <linux/iocontext.h>
24 #include <linux/slab.h>
25 #include <linux/init.h>
26 #include <linux/kernel.h>
27 #include <linux/export.h>
28 #include <linux/mempool.h>
29 #include <linux/workqueue.h>
30 #include <linux/cgroup.h>
31 
32 #include <trace/events/block.h>
33 
34 /*
35  * Test patch to inline a certain number of bi_io_vec's inside the bio
36  * itself, to shrink a bio data allocation from two mempool calls to one
37  */
38 #define BIO_INLINE_VECS		4
39 
40 /*
41  * if you change this list, also change bvec_alloc or things will
42  * break badly! cannot be bigger than what you can fit into an
43  * unsigned short
44  */
45 #define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
46 static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
47 	BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
48 };
49 #undef BV
50 
51 /*
52  * fs_bio_set is the bio_set containing bio and iovec memory pools used by
53  * IO code that does not need private memory pools.
54  */
55 struct bio_set *fs_bio_set;
56 EXPORT_SYMBOL(fs_bio_set);
57 
58 /*
59  * Our slab pool management
60  */
61 struct bio_slab {
62 	struct kmem_cache *slab;
63 	unsigned int slab_ref;
64 	unsigned int slab_size;
65 	char name[8];
66 };
67 static DEFINE_MUTEX(bio_slab_lock);
68 static struct bio_slab *bio_slabs;
69 static unsigned int bio_slab_nr, bio_slab_max;
70 
71 static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
72 {
73 	unsigned int sz = sizeof(struct bio) + extra_size;
74 	struct kmem_cache *slab = NULL;
75 	struct bio_slab *bslab, *new_bio_slabs;
76 	unsigned int new_bio_slab_max;
77 	unsigned int i, entry = -1;
78 
79 	mutex_lock(&bio_slab_lock);
80 
81 	i = 0;
82 	while (i < bio_slab_nr) {
83 		bslab = &bio_slabs[i];
84 
85 		if (!bslab->slab && entry == -1)
86 			entry = i;
87 		else if (bslab->slab_size == sz) {
88 			slab = bslab->slab;
89 			bslab->slab_ref++;
90 			break;
91 		}
92 		i++;
93 	}
94 
95 	if (slab)
96 		goto out_unlock;
97 
98 	if (bio_slab_nr == bio_slab_max && entry == -1) {
99 		new_bio_slab_max = bio_slab_max << 1;
100 		new_bio_slabs = krealloc(bio_slabs,
101 					 new_bio_slab_max * sizeof(struct bio_slab),
102 					 GFP_KERNEL);
103 		if (!new_bio_slabs)
104 			goto out_unlock;
105 		bio_slab_max = new_bio_slab_max;
106 		bio_slabs = new_bio_slabs;
107 	}
108 	if (entry == -1)
109 		entry = bio_slab_nr++;
110 
111 	bslab = &bio_slabs[entry];
112 
113 	snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
114 	slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
115 				 SLAB_HWCACHE_ALIGN, NULL);
116 	if (!slab)
117 		goto out_unlock;
118 
119 	bslab->slab = slab;
120 	bslab->slab_ref = 1;
121 	bslab->slab_size = sz;
122 out_unlock:
123 	mutex_unlock(&bio_slab_lock);
124 	return slab;
125 }
126 
127 static void bio_put_slab(struct bio_set *bs)
128 {
129 	struct bio_slab *bslab = NULL;
130 	unsigned int i;
131 
132 	mutex_lock(&bio_slab_lock);
133 
134 	for (i = 0; i < bio_slab_nr; i++) {
135 		if (bs->bio_slab == bio_slabs[i].slab) {
136 			bslab = &bio_slabs[i];
137 			break;
138 		}
139 	}
140 
141 	if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
142 		goto out;
143 
144 	WARN_ON(!bslab->slab_ref);
145 
146 	if (--bslab->slab_ref)
147 		goto out;
148 
149 	kmem_cache_destroy(bslab->slab);
150 	bslab->slab = NULL;
151 
152 out:
153 	mutex_unlock(&bio_slab_lock);
154 }
155 
156 unsigned int bvec_nr_vecs(unsigned short idx)
157 {
158 	return bvec_slabs[idx].nr_vecs;
159 }
160 
161 void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
162 {
163 	if (!idx)
164 		return;
165 	idx--;
166 
167 	BIO_BUG_ON(idx >= BVEC_POOL_NR);
168 
169 	if (idx == BVEC_POOL_MAX) {
170 		mempool_free(bv, pool);
171 	} else {
172 		struct biovec_slab *bvs = bvec_slabs + idx;
173 
174 		kmem_cache_free(bvs->slab, bv);
175 	}
176 }
177 
178 struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
179 			   mempool_t *pool)
180 {
181 	struct bio_vec *bvl;
182 
183 	/*
184 	 * see comment near bvec_array define!
185 	 */
186 	switch (nr) {
187 	case 1:
188 		*idx = 0;
189 		break;
190 	case 2 ... 4:
191 		*idx = 1;
192 		break;
193 	case 5 ... 16:
194 		*idx = 2;
195 		break;
196 	case 17 ... 64:
197 		*idx = 3;
198 		break;
199 	case 65 ... 128:
200 		*idx = 4;
201 		break;
202 	case 129 ... BIO_MAX_PAGES:
203 		*idx = 5;
204 		break;
205 	default:
206 		return NULL;
207 	}
208 
209 	/*
210 	 * idx now points to the pool we want to allocate from. only the
211 	 * 1-vec entry pool is mempool backed.
212 	 */
213 	if (*idx == BVEC_POOL_MAX) {
214 fallback:
215 		bvl = mempool_alloc(pool, gfp_mask);
216 	} else {
217 		struct biovec_slab *bvs = bvec_slabs + *idx;
218 		gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
219 
220 		/*
221 		 * Make this allocation restricted and don't dump info on
222 		 * allocation failures, since we'll fallback to the mempool
223 		 * in case of failure.
224 		 */
225 		__gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
226 
227 		/*
228 		 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
229 		 * is set, retry with the 1-entry mempool
230 		 */
231 		bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
232 		if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
233 			*idx = BVEC_POOL_MAX;
234 			goto fallback;
235 		}
236 	}
237 
238 	(*idx)++;
239 	return bvl;
240 }
241 
242 static void __bio_free(struct bio *bio)
243 {
244 	bio_disassociate_task(bio);
245 
246 	if (bio_integrity(bio))
247 		bio_integrity_free(bio);
248 }
249 
250 static void bio_free(struct bio *bio)
251 {
252 	struct bio_set *bs = bio->bi_pool;
253 	void *p;
254 
255 	__bio_free(bio);
256 
257 	if (bs) {
258 		bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
259 
260 		/*
261 		 * If we have front padding, adjust the bio pointer before freeing
262 		 */
263 		p = bio;
264 		p -= bs->front_pad;
265 
266 		mempool_free(p, bs->bio_pool);
267 	} else {
268 		/* Bio was allocated by bio_kmalloc() */
269 		kfree(bio);
270 	}
271 }
272 
273 void bio_init(struct bio *bio, struct bio_vec *table,
274 	      unsigned short max_vecs)
275 {
276 	memset(bio, 0, sizeof(*bio));
277 	atomic_set(&bio->__bi_remaining, 1);
278 	atomic_set(&bio->__bi_cnt, 1);
279 
280 	bio->bi_io_vec = table;
281 	bio->bi_max_vecs = max_vecs;
282 }
283 EXPORT_SYMBOL(bio_init);
284 
285 /**
286  * bio_reset - reinitialize a bio
287  * @bio:	bio to reset
288  *
289  * Description:
290  *   After calling bio_reset(), @bio will be in the same state as a freshly
291  *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
292  *   preserved are the ones that are initialized by bio_alloc_bioset(). See
293  *   comment in struct bio.
294  */
295 void bio_reset(struct bio *bio)
296 {
297 	unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
298 
299 	__bio_free(bio);
300 
301 	memset(bio, 0, BIO_RESET_BYTES);
302 	bio->bi_flags = flags;
303 	atomic_set(&bio->__bi_remaining, 1);
304 }
305 EXPORT_SYMBOL(bio_reset);
306 
307 static struct bio *__bio_chain_endio(struct bio *bio)
308 {
309 	struct bio *parent = bio->bi_private;
310 
311 	if (!parent->bi_error)
312 		parent->bi_error = bio->bi_error;
313 	bio_put(bio);
314 	return parent;
315 }
316 
317 static void bio_chain_endio(struct bio *bio)
318 {
319 	bio_endio(__bio_chain_endio(bio));
320 }
321 
322 /**
323  * bio_chain - chain bio completions
324  * @bio: the target bio
325  * @parent: the @bio's parent bio
326  *
327  * The caller won't have a bi_end_io called when @bio completes - instead,
328  * @parent's bi_end_io won't be called until both @parent and @bio have
329  * completed; the chained bio will also be freed when it completes.
330  *
331  * The caller must not set bi_private or bi_end_io in @bio.
332  */
333 void bio_chain(struct bio *bio, struct bio *parent)
334 {
335 	BUG_ON(bio->bi_private || bio->bi_end_io);
336 
337 	bio->bi_private = parent;
338 	bio->bi_end_io	= bio_chain_endio;
339 	bio_inc_remaining(parent);
340 }
341 EXPORT_SYMBOL(bio_chain);
342 
343 static void bio_alloc_rescue(struct work_struct *work)
344 {
345 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
346 	struct bio *bio;
347 
348 	while (1) {
349 		spin_lock(&bs->rescue_lock);
350 		bio = bio_list_pop(&bs->rescue_list);
351 		spin_unlock(&bs->rescue_lock);
352 
353 		if (!bio)
354 			break;
355 
356 		generic_make_request(bio);
357 	}
358 }
359 
360 static void punt_bios_to_rescuer(struct bio_set *bs)
361 {
362 	struct bio_list punt, nopunt;
363 	struct bio *bio;
364 
365 	/*
366 	 * In order to guarantee forward progress we must punt only bios that
367 	 * were allocated from this bio_set; otherwise, if there was a bio on
368 	 * there for a stacking driver higher up in the stack, processing it
369 	 * could require allocating bios from this bio_set, and doing that from
370 	 * our own rescuer would be bad.
371 	 *
372 	 * Since bio lists are singly linked, pop them all instead of trying to
373 	 * remove from the middle of the list:
374 	 */
375 
376 	bio_list_init(&punt);
377 	bio_list_init(&nopunt);
378 
379 	while ((bio = bio_list_pop(current->bio_list)))
380 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
381 
382 	*current->bio_list = nopunt;
383 
384 	spin_lock(&bs->rescue_lock);
385 	bio_list_merge(&bs->rescue_list, &punt);
386 	spin_unlock(&bs->rescue_lock);
387 
388 	queue_work(bs->rescue_workqueue, &bs->rescue_work);
389 }
390 
391 /**
392  * bio_alloc_bioset - allocate a bio for I/O
393  * @gfp_mask:   the GFP_ mask given to the slab allocator
394  * @nr_iovecs:	number of iovecs to pre-allocate
395  * @bs:		the bio_set to allocate from.
396  *
397  * Description:
398  *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
399  *   backed by the @bs's mempool.
400  *
401  *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
402  *   always be able to allocate a bio. This is due to the mempool guarantees.
403  *   To make this work, callers must never allocate more than 1 bio at a time
404  *   from this pool. Callers that need to allocate more than 1 bio must always
405  *   submit the previously allocated bio for IO before attempting to allocate
406  *   a new one. Failure to do so can cause deadlocks under memory pressure.
407  *
408  *   Note that when running under generic_make_request() (i.e. any block
409  *   driver), bios are not submitted until after you return - see the code in
410  *   generic_make_request() that converts recursion into iteration, to prevent
411  *   stack overflows.
412  *
413  *   This would normally mean allocating multiple bios under
414  *   generic_make_request() would be susceptible to deadlocks, but we have
415  *   deadlock avoidance code that resubmits any blocked bios from a rescuer
416  *   thread.
417  *
418  *   However, we do not guarantee forward progress for allocations from other
419  *   mempools. Doing multiple allocations from the same mempool under
420  *   generic_make_request() should be avoided - instead, use bio_set's front_pad
421  *   for per bio allocations.
422  *
423  *   RETURNS:
424  *   Pointer to new bio on success, NULL on failure.
425  */
426 struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)
427 {
428 	gfp_t saved_gfp = gfp_mask;
429 	unsigned front_pad;
430 	unsigned inline_vecs;
431 	struct bio_vec *bvl = NULL;
432 	struct bio *bio;
433 	void *p;
434 
435 	if (!bs) {
436 		if (nr_iovecs > UIO_MAXIOV)
437 			return NULL;
438 
439 		p = kmalloc(sizeof(struct bio) +
440 			    nr_iovecs * sizeof(struct bio_vec),
441 			    gfp_mask);
442 		front_pad = 0;
443 		inline_vecs = nr_iovecs;
444 	} else {
445 		/* should not use nobvec bioset for nr_iovecs > 0 */
446 		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))
447 			return NULL;
448 		/*
449 		 * generic_make_request() converts recursion to iteration; this
450 		 * means if we're running beneath it, any bios we allocate and
451 		 * submit will not be submitted (and thus freed) until after we
452 		 * return.
453 		 *
454 		 * This exposes us to a potential deadlock if we allocate
455 		 * multiple bios from the same bio_set() while running
456 		 * underneath generic_make_request(). If we were to allocate
457 		 * multiple bios (say a stacking block driver that was splitting
458 		 * bios), we would deadlock if we exhausted the mempool's
459 		 * reserve.
460 		 *
461 		 * We solve this, and guarantee forward progress, with a rescuer
462 		 * workqueue per bio_set. If we go to allocate and there are
463 		 * bios on current->bio_list, we first try the allocation
464 		 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
465 		 * bios we would be blocking to the rescuer workqueue before
466 		 * we retry with the original gfp_flags.
467 		 */
468 
469 		if (current->bio_list && !bio_list_empty(current->bio_list))
470 			gfp_mask &= ~__GFP_DIRECT_RECLAIM;
471 
472 		p = mempool_alloc(bs->bio_pool, gfp_mask);
473 		if (!p && gfp_mask != saved_gfp) {
474 			punt_bios_to_rescuer(bs);
475 			gfp_mask = saved_gfp;
476 			p = mempool_alloc(bs->bio_pool, gfp_mask);
477 		}
478 
479 		front_pad = bs->front_pad;
480 		inline_vecs = BIO_INLINE_VECS;
481 	}
482 
483 	if (unlikely(!p))
484 		return NULL;
485 
486 	bio = p + front_pad;
487 	bio_init(bio, NULL, 0);
488 
489 	if (nr_iovecs > inline_vecs) {
490 		unsigned long idx = 0;
491 
492 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
493 		if (!bvl && gfp_mask != saved_gfp) {
494 			punt_bios_to_rescuer(bs);
495 			gfp_mask = saved_gfp;
496 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);
497 		}
498 
499 		if (unlikely(!bvl))
500 			goto err_free;
501 
502 		bio->bi_flags |= idx << BVEC_POOL_OFFSET;
503 	} else if (nr_iovecs) {
504 		bvl = bio->bi_inline_vecs;
505 	}
506 
507 	bio->bi_pool = bs;
508 	bio->bi_max_vecs = nr_iovecs;
509 	bio->bi_io_vec = bvl;
510 	return bio;
511 
512 err_free:
513 	mempool_free(p, bs->bio_pool);
514 	return NULL;
515 }
516 EXPORT_SYMBOL(bio_alloc_bioset);
517 
518 void zero_fill_bio(struct bio *bio)
519 {
520 	unsigned long flags;
521 	struct bio_vec bv;
522 	struct bvec_iter iter;
523 
524 	bio_for_each_segment(bv, bio, iter) {
525 		char *data = bvec_kmap_irq(&bv, &flags);
526 		memset(data, 0, bv.bv_len);
527 		flush_dcache_page(bv.bv_page);
528 		bvec_kunmap_irq(data, &flags);
529 	}
530 }
531 EXPORT_SYMBOL(zero_fill_bio);
532 
533 /**
534  * bio_put - release a reference to a bio
535  * @bio:   bio to release reference to
536  *
537  * Description:
538  *   Put a reference to a &struct bio, either one you have gotten with
539  *   bio_alloc, bio_get or bio_clone. The last put of a bio will free it.
540  **/
541 void bio_put(struct bio *bio)
542 {
543 	if (!bio_flagged(bio, BIO_REFFED))
544 		bio_free(bio);
545 	else {
546 		BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
547 
548 		/*
549 		 * last put frees it
550 		 */
551 		if (atomic_dec_and_test(&bio->__bi_cnt))
552 			bio_free(bio);
553 	}
554 }
555 EXPORT_SYMBOL(bio_put);
556 
557 inline int bio_phys_segments(struct request_queue *q, struct bio *bio)
558 {
559 	if (unlikely(!bio_flagged(bio, BIO_SEG_VALID)))
560 		blk_recount_segments(q, bio);
561 
562 	return bio->bi_phys_segments;
563 }
564 EXPORT_SYMBOL(bio_phys_segments);
565 
566 /**
567  * 	__bio_clone_fast - clone a bio that shares the original bio's biovec
568  * 	@bio: destination bio
569  * 	@bio_src: bio to clone
570  *
571  *	Clone a &bio. Caller will own the returned bio, but not
572  *	the actual data it points to. Reference count of returned
573  * 	bio will be one.
574  *
575  * 	Caller must ensure that @bio_src is not freed before @bio.
576  */
577 void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
578 {
579 	BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
580 
581 	/*
582 	 * most users will be overriding ->bi_bdev with a new target,
583 	 * so we don't set nor calculate new physical/hw segment counts here
584 	 */
585 	bio->bi_bdev = bio_src->bi_bdev;
586 	bio_set_flag(bio, BIO_CLONED);
587 	bio->bi_opf = bio_src->bi_opf;
588 	bio->bi_iter = bio_src->bi_iter;
589 	bio->bi_io_vec = bio_src->bi_io_vec;
590 
591 	bio_clone_blkcg_association(bio, bio_src);
592 }
593 EXPORT_SYMBOL(__bio_clone_fast);
594 
595 /**
596  *	bio_clone_fast - clone a bio that shares the original bio's biovec
597  *	@bio: bio to clone
598  *	@gfp_mask: allocation priority
599  *	@bs: bio_set to allocate from
600  *
601  * 	Like __bio_clone_fast, only also allocates the returned bio
602  */
603 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
604 {
605 	struct bio *b;
606 
607 	b = bio_alloc_bioset(gfp_mask, 0, bs);
608 	if (!b)
609 		return NULL;
610 
611 	__bio_clone_fast(b, bio);
612 
613 	if (bio_integrity(bio)) {
614 		int ret;
615 
616 		ret = bio_integrity_clone(b, bio, gfp_mask);
617 
618 		if (ret < 0) {
619 			bio_put(b);
620 			return NULL;
621 		}
622 	}
623 
624 	return b;
625 }
626 EXPORT_SYMBOL(bio_clone_fast);
627 
628 /**
629  * 	bio_clone_bioset - clone a bio
630  * 	@bio_src: bio to clone
631  *	@gfp_mask: allocation priority
632  *	@bs: bio_set to allocate from
633  *
634  *	Clone bio. Caller will own the returned bio, but not the actual data it
635  *	points to. Reference count of returned bio will be one.
636  */
637 struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask,
638 			     struct bio_set *bs)
639 {
640 	struct bvec_iter iter;
641 	struct bio_vec bv;
642 	struct bio *bio;
643 
644 	/*
645 	 * Pre immutable biovecs, __bio_clone() used to just do a memcpy from
646 	 * bio_src->bi_io_vec to bio->bi_io_vec.
647 	 *
648 	 * We can't do that anymore, because:
649 	 *
650 	 *  - The point of cloning the biovec is to produce a bio with a biovec
651 	 *    the caller can modify: bi_idx and bi_bvec_done should be 0.
652 	 *
653 	 *  - The original bio could've had more than BIO_MAX_PAGES biovecs; if
654 	 *    we tried to clone the whole thing bio_alloc_bioset() would fail.
655 	 *    But the clone should succeed as long as the number of biovecs we
656 	 *    actually need to allocate is fewer than BIO_MAX_PAGES.
657 	 *
658 	 *  - Lastly, bi_vcnt should not be looked at or relied upon by code
659 	 *    that does not own the bio - reason being drivers don't use it for
660 	 *    iterating over the biovec anymore, so expecting it to be kept up
661 	 *    to date (i.e. for clones that share the parent biovec) is just
662 	 *    asking for trouble and would force extra work on
663 	 *    __bio_clone_fast() anyways.
664 	 */
665 
666 	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);
667 	if (!bio)
668 		return NULL;
669 	bio->bi_bdev		= bio_src->bi_bdev;
670 	bio->bi_opf		= bio_src->bi_opf;
671 	bio->bi_iter.bi_sector	= bio_src->bi_iter.bi_sector;
672 	bio->bi_iter.bi_size	= bio_src->bi_iter.bi_size;
673 
674 	switch (bio_op(bio)) {
675 	case REQ_OP_DISCARD:
676 	case REQ_OP_SECURE_ERASE:
677 		break;
678 	case REQ_OP_WRITE_SAME:
679 		bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0];
680 		break;
681 	default:
682 		bio_for_each_segment(bv, bio_src, iter)
683 			bio->bi_io_vec[bio->bi_vcnt++] = bv;
684 		break;
685 	}
686 
687 	if (bio_integrity(bio_src)) {
688 		int ret;
689 
690 		ret = bio_integrity_clone(bio, bio_src, gfp_mask);
691 		if (ret < 0) {
692 			bio_put(bio);
693 			return NULL;
694 		}
695 	}
696 
697 	bio_clone_blkcg_association(bio, bio_src);
698 
699 	return bio;
700 }
701 EXPORT_SYMBOL(bio_clone_bioset);
702 
703 /**
704  *	bio_add_pc_page	-	attempt to add page to bio
705  *	@q: the target queue
706  *	@bio: destination bio
707  *	@page: page to add
708  *	@len: vec entry length
709  *	@offset: vec entry offset
710  *
711  *	Attempt to add a page to the bio_vec maplist. This can fail for a
712  *	number of reasons, such as the bio being full or target block device
713  *	limitations. The target block device must allow bio's up to PAGE_SIZE,
714  *	so it is always possible to add a single page to an empty bio.
715  *
716  *	This should only be used by REQ_PC bios.
717  */
718 int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page
719 		    *page, unsigned int len, unsigned int offset)
720 {
721 	int retried_segments = 0;
722 	struct bio_vec *bvec;
723 
724 	/*
725 	 * cloned bio must not modify vec list
726 	 */
727 	if (unlikely(bio_flagged(bio, BIO_CLONED)))
728 		return 0;
729 
730 	if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
731 		return 0;
732 
733 	/*
734 	 * For filesystems with a blocksize smaller than the pagesize
735 	 * we will often be called with the same page as last time and
736 	 * a consecutive offset.  Optimize this special case.
737 	 */
738 	if (bio->bi_vcnt > 0) {
739 		struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1];
740 
741 		if (page == prev->bv_page &&
742 		    offset == prev->bv_offset + prev->bv_len) {
743 			prev->bv_len += len;
744 			bio->bi_iter.bi_size += len;
745 			goto done;
746 		}
747 
748 		/*
749 		 * If the queue doesn't support SG gaps and adding this
750 		 * offset would create a gap, disallow it.
751 		 */
752 		if (bvec_gap_to_prev(q, prev, offset))
753 			return 0;
754 	}
755 
756 	if (bio->bi_vcnt >= bio->bi_max_vecs)
757 		return 0;
758 
759 	/*
760 	 * setup the new entry, we might clear it again later if we
761 	 * cannot add the page
762 	 */
763 	bvec = &bio->bi_io_vec[bio->bi_vcnt];
764 	bvec->bv_page = page;
765 	bvec->bv_len = len;
766 	bvec->bv_offset = offset;
767 	bio->bi_vcnt++;
768 	bio->bi_phys_segments++;
769 	bio->bi_iter.bi_size += len;
770 
771 	/*
772 	 * Perform a recount if the number of segments is greater
773 	 * than queue_max_segments(q).
774 	 */
775 
776 	while (bio->bi_phys_segments > queue_max_segments(q)) {
777 
778 		if (retried_segments)
779 			goto failed;
780 
781 		retried_segments = 1;
782 		blk_recount_segments(q, bio);
783 	}
784 
785 	/* If we may be able to merge these biovecs, force a recount */
786 	if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec)))
787 		bio_clear_flag(bio, BIO_SEG_VALID);
788 
789  done:
790 	return len;
791 
792  failed:
793 	bvec->bv_page = NULL;
794 	bvec->bv_len = 0;
795 	bvec->bv_offset = 0;
796 	bio->bi_vcnt--;
797 	bio->bi_iter.bi_size -= len;
798 	blk_recount_segments(q, bio);
799 	return 0;
800 }
801 EXPORT_SYMBOL(bio_add_pc_page);
802 
803 /**
804  *	bio_add_page	-	attempt to add page to bio
805  *	@bio: destination bio
806  *	@page: page to add
807  *	@len: vec entry length
808  *	@offset: vec entry offset
809  *
810  *	Attempt to add a page to the bio_vec maplist. This will only fail
811  *	if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
812  */
813 int bio_add_page(struct bio *bio, struct page *page,
814 		 unsigned int len, unsigned int offset)
815 {
816 	struct bio_vec *bv;
817 
818 	/*
819 	 * cloned bio must not modify vec list
820 	 */
821 	if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
822 		return 0;
823 
824 	/*
825 	 * For filesystems with a blocksize smaller than the pagesize
826 	 * we will often be called with the same page as last time and
827 	 * a consecutive offset.  Optimize this special case.
828 	 */
829 	if (bio->bi_vcnt > 0) {
830 		bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
831 
832 		if (page == bv->bv_page &&
833 		    offset == bv->bv_offset + bv->bv_len) {
834 			bv->bv_len += len;
835 			goto done;
836 		}
837 	}
838 
839 	if (bio->bi_vcnt >= bio->bi_max_vecs)
840 		return 0;
841 
842 	bv		= &bio->bi_io_vec[bio->bi_vcnt];
843 	bv->bv_page	= page;
844 	bv->bv_len	= len;
845 	bv->bv_offset	= offset;
846 
847 	bio->bi_vcnt++;
848 done:
849 	bio->bi_iter.bi_size += len;
850 	return len;
851 }
852 EXPORT_SYMBOL(bio_add_page);
853 
854 /**
855  * bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
856  * @bio: bio to add pages to
857  * @iter: iov iterator describing the region to be mapped
858  *
859  * Pins as many pages from *iter and appends them to @bio's bvec array. The
860  * pages will have to be released using put_page() when done.
861  */
862 int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
863 {
864 	unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
865 	struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
866 	struct page **pages = (struct page **)bv;
867 	size_t offset, diff;
868 	ssize_t size;
869 
870 	size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
871 	if (unlikely(size <= 0))
872 		return size ? size : -EFAULT;
873 	nr_pages = (size + offset + PAGE_SIZE - 1) / PAGE_SIZE;
874 
875 	/*
876 	 * Deep magic below:  We need to walk the pinned pages backwards
877 	 * because we are abusing the space allocated for the bio_vecs
878 	 * for the page array.  Because the bio_vecs are larger than the
879 	 * page pointers by definition this will always work.  But it also
880 	 * means we can't use bio_add_page, so any changes to it's semantics
881 	 * need to be reflected here as well.
882 	 */
883 	bio->bi_iter.bi_size += size;
884 	bio->bi_vcnt += nr_pages;
885 
886 	diff = (nr_pages * PAGE_SIZE - offset) - size;
887 	while (nr_pages--) {
888 		bv[nr_pages].bv_page = pages[nr_pages];
889 		bv[nr_pages].bv_len = PAGE_SIZE;
890 		bv[nr_pages].bv_offset = 0;
891 	}
892 
893 	bv[0].bv_offset += offset;
894 	bv[0].bv_len -= offset;
895 	if (diff)
896 		bv[bio->bi_vcnt - 1].bv_len -= diff;
897 
898 	iov_iter_advance(iter, size);
899 	return 0;
900 }
901 EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages);
902 
903 struct submit_bio_ret {
904 	struct completion event;
905 	int error;
906 };
907 
908 static void submit_bio_wait_endio(struct bio *bio)
909 {
910 	struct submit_bio_ret *ret = bio->bi_private;
911 
912 	ret->error = bio->bi_error;
913 	complete(&ret->event);
914 }
915 
916 /**
917  * submit_bio_wait - submit a bio, and wait until it completes
918  * @bio: The &struct bio which describes the I/O
919  *
920  * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
921  * bio_endio() on failure.
922  */
923 int submit_bio_wait(struct bio *bio)
924 {
925 	struct submit_bio_ret ret;
926 
927 	init_completion(&ret.event);
928 	bio->bi_private = &ret;
929 	bio->bi_end_io = submit_bio_wait_endio;
930 	bio->bi_opf |= REQ_SYNC;
931 	submit_bio(bio);
932 	wait_for_completion_io(&ret.event);
933 
934 	return ret.error;
935 }
936 EXPORT_SYMBOL(submit_bio_wait);
937 
938 /**
939  * bio_advance - increment/complete a bio by some number of bytes
940  * @bio:	bio to advance
941  * @bytes:	number of bytes to complete
942  *
943  * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
944  * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
945  * be updated on the last bvec as well.
946  *
947  * @bio will then represent the remaining, uncompleted portion of the io.
948  */
949 void bio_advance(struct bio *bio, unsigned bytes)
950 {
951 	if (bio_integrity(bio))
952 		bio_integrity_advance(bio, bytes);
953 
954 	bio_advance_iter(bio, &bio->bi_iter, bytes);
955 }
956 EXPORT_SYMBOL(bio_advance);
957 
958 /**
959  * bio_alloc_pages - allocates a single page for each bvec in a bio
960  * @bio: bio to allocate pages for
961  * @gfp_mask: flags for allocation
962  *
963  * Allocates pages up to @bio->bi_vcnt.
964  *
965  * Returns 0 on success, -ENOMEM on failure. On failure, any allocated pages are
966  * freed.
967  */
968 int bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
969 {
970 	int i;
971 	struct bio_vec *bv;
972 
973 	bio_for_each_segment_all(bv, bio, i) {
974 		bv->bv_page = alloc_page(gfp_mask);
975 		if (!bv->bv_page) {
976 			while (--bv >= bio->bi_io_vec)
977 				__free_page(bv->bv_page);
978 			return -ENOMEM;
979 		}
980 	}
981 
982 	return 0;
983 }
984 EXPORT_SYMBOL(bio_alloc_pages);
985 
986 /**
987  * bio_copy_data - copy contents of data buffers from one chain of bios to
988  * another
989  * @src: source bio list
990  * @dst: destination bio list
991  *
992  * If @src and @dst are single bios, bi_next must be NULL - otherwise, treats
993  * @src and @dst as linked lists of bios.
994  *
995  * Stops when it reaches the end of either @src or @dst - that is, copies
996  * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
997  */
998 void bio_copy_data(struct bio *dst, struct bio *src)
999 {
1000 	struct bvec_iter src_iter, dst_iter;
1001 	struct bio_vec src_bv, dst_bv;
1002 	void *src_p, *dst_p;
1003 	unsigned bytes;
1004 
1005 	src_iter = src->bi_iter;
1006 	dst_iter = dst->bi_iter;
1007 
1008 	while (1) {
1009 		if (!src_iter.bi_size) {
1010 			src = src->bi_next;
1011 			if (!src)
1012 				break;
1013 
1014 			src_iter = src->bi_iter;
1015 		}
1016 
1017 		if (!dst_iter.bi_size) {
1018 			dst = dst->bi_next;
1019 			if (!dst)
1020 				break;
1021 
1022 			dst_iter = dst->bi_iter;
1023 		}
1024 
1025 		src_bv = bio_iter_iovec(src, src_iter);
1026 		dst_bv = bio_iter_iovec(dst, dst_iter);
1027 
1028 		bytes = min(src_bv.bv_len, dst_bv.bv_len);
1029 
1030 		src_p = kmap_atomic(src_bv.bv_page);
1031 		dst_p = kmap_atomic(dst_bv.bv_page);
1032 
1033 		memcpy(dst_p + dst_bv.bv_offset,
1034 		       src_p + src_bv.bv_offset,
1035 		       bytes);
1036 
1037 		kunmap_atomic(dst_p);
1038 		kunmap_atomic(src_p);
1039 
1040 		bio_advance_iter(src, &src_iter, bytes);
1041 		bio_advance_iter(dst, &dst_iter, bytes);
1042 	}
1043 }
1044 EXPORT_SYMBOL(bio_copy_data);
1045 
1046 struct bio_map_data {
1047 	int is_our_pages;
1048 	struct iov_iter iter;
1049 	struct iovec iov[];
1050 };
1051 
1052 static struct bio_map_data *bio_alloc_map_data(unsigned int iov_count,
1053 					       gfp_t gfp_mask)
1054 {
1055 	if (iov_count > UIO_MAXIOV)
1056 		return NULL;
1057 
1058 	return kmalloc(sizeof(struct bio_map_data) +
1059 		       sizeof(struct iovec) * iov_count, gfp_mask);
1060 }
1061 
1062 /**
1063  * bio_copy_from_iter - copy all pages from iov_iter to bio
1064  * @bio: The &struct bio which describes the I/O as destination
1065  * @iter: iov_iter as source
1066  *
1067  * Copy all pages from iov_iter to bio.
1068  * Returns 0 on success, or error on failure.
1069  */
1070 static int bio_copy_from_iter(struct bio *bio, struct iov_iter iter)
1071 {
1072 	int i;
1073 	struct bio_vec *bvec;
1074 
1075 	bio_for_each_segment_all(bvec, bio, i) {
1076 		ssize_t ret;
1077 
1078 		ret = copy_page_from_iter(bvec->bv_page,
1079 					  bvec->bv_offset,
1080 					  bvec->bv_len,
1081 					  &iter);
1082 
1083 		if (!iov_iter_count(&iter))
1084 			break;
1085 
1086 		if (ret < bvec->bv_len)
1087 			return -EFAULT;
1088 	}
1089 
1090 	return 0;
1091 }
1092 
1093 /**
1094  * bio_copy_to_iter - copy all pages from bio to iov_iter
1095  * @bio: The &struct bio which describes the I/O as source
1096  * @iter: iov_iter as destination
1097  *
1098  * Copy all pages from bio to iov_iter.
1099  * Returns 0 on success, or error on failure.
1100  */
1101 static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
1102 {
1103 	int i;
1104 	struct bio_vec *bvec;
1105 
1106 	bio_for_each_segment_all(bvec, bio, i) {
1107 		ssize_t ret;
1108 
1109 		ret = copy_page_to_iter(bvec->bv_page,
1110 					bvec->bv_offset,
1111 					bvec->bv_len,
1112 					&iter);
1113 
1114 		if (!iov_iter_count(&iter))
1115 			break;
1116 
1117 		if (ret < bvec->bv_len)
1118 			return -EFAULT;
1119 	}
1120 
1121 	return 0;
1122 }
1123 
1124 void bio_free_pages(struct bio *bio)
1125 {
1126 	struct bio_vec *bvec;
1127 	int i;
1128 
1129 	bio_for_each_segment_all(bvec, bio, i)
1130 		__free_page(bvec->bv_page);
1131 }
1132 EXPORT_SYMBOL(bio_free_pages);
1133 
1134 /**
1135  *	bio_uncopy_user	-	finish previously mapped bio
1136  *	@bio: bio being terminated
1137  *
1138  *	Free pages allocated from bio_copy_user_iov() and write back data
1139  *	to user space in case of a read.
1140  */
1141 int bio_uncopy_user(struct bio *bio)
1142 {
1143 	struct bio_map_data *bmd = bio->bi_private;
1144 	int ret = 0;
1145 
1146 	if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
1147 		/*
1148 		 * if we're in a workqueue, the request is orphaned, so
1149 		 * don't copy into a random user address space, just free
1150 		 * and return -EINTR so user space doesn't expect any data.
1151 		 */
1152 		if (!current->mm)
1153 			ret = -EINTR;
1154 		else if (bio_data_dir(bio) == READ)
1155 			ret = bio_copy_to_iter(bio, bmd->iter);
1156 		if (bmd->is_our_pages)
1157 			bio_free_pages(bio);
1158 	}
1159 	kfree(bmd);
1160 	bio_put(bio);
1161 	return ret;
1162 }
1163 
1164 /**
1165  *	bio_copy_user_iov	-	copy user data to bio
1166  *	@q:		destination block queue
1167  *	@map_data:	pointer to the rq_map_data holding pages (if necessary)
1168  *	@iter:		iovec iterator
1169  *	@gfp_mask:	memory allocation flags
1170  *
1171  *	Prepares and returns a bio for indirect user io, bouncing data
1172  *	to/from kernel pages as necessary. Must be paired with
1173  *	call bio_uncopy_user() on io completion.
1174  */
1175 struct bio *bio_copy_user_iov(struct request_queue *q,
1176 			      struct rq_map_data *map_data,
1177 			      const struct iov_iter *iter,
1178 			      gfp_t gfp_mask)
1179 {
1180 	struct bio_map_data *bmd;
1181 	struct page *page;
1182 	struct bio *bio;
1183 	int i, ret;
1184 	int nr_pages = 0;
1185 	unsigned int len = iter->count;
1186 	unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
1187 
1188 	for (i = 0; i < iter->nr_segs; i++) {
1189 		unsigned long uaddr;
1190 		unsigned long end;
1191 		unsigned long start;
1192 
1193 		uaddr = (unsigned long) iter->iov[i].iov_base;
1194 		end = (uaddr + iter->iov[i].iov_len + PAGE_SIZE - 1)
1195 			>> PAGE_SHIFT;
1196 		start = uaddr >> PAGE_SHIFT;
1197 
1198 		/*
1199 		 * Overflow, abort
1200 		 */
1201 		if (end < start)
1202 			return ERR_PTR(-EINVAL);
1203 
1204 		nr_pages += end - start;
1205 	}
1206 
1207 	if (offset)
1208 		nr_pages++;
1209 
1210 	bmd = bio_alloc_map_data(iter->nr_segs, gfp_mask);
1211 	if (!bmd)
1212 		return ERR_PTR(-ENOMEM);
1213 
1214 	/*
1215 	 * We need to do a deep copy of the iov_iter including the iovecs.
1216 	 * The caller provided iov might point to an on-stack or otherwise
1217 	 * shortlived one.
1218 	 */
1219 	bmd->is_our_pages = map_data ? 0 : 1;
1220 	memcpy(bmd->iov, iter->iov, sizeof(struct iovec) * iter->nr_segs);
1221 	iov_iter_init(&bmd->iter, iter->type, bmd->iov,
1222 			iter->nr_segs, iter->count);
1223 
1224 	ret = -ENOMEM;
1225 	bio = bio_kmalloc(gfp_mask, nr_pages);
1226 	if (!bio)
1227 		goto out_bmd;
1228 
1229 	if (iter->type & WRITE)
1230 		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1231 
1232 	ret = 0;
1233 
1234 	if (map_data) {
1235 		nr_pages = 1 << map_data->page_order;
1236 		i = map_data->offset / PAGE_SIZE;
1237 	}
1238 	while (len) {
1239 		unsigned int bytes = PAGE_SIZE;
1240 
1241 		bytes -= offset;
1242 
1243 		if (bytes > len)
1244 			bytes = len;
1245 
1246 		if (map_data) {
1247 			if (i == map_data->nr_entries * nr_pages) {
1248 				ret = -ENOMEM;
1249 				break;
1250 			}
1251 
1252 			page = map_data->pages[i / nr_pages];
1253 			page += (i % nr_pages);
1254 
1255 			i++;
1256 		} else {
1257 			page = alloc_page(q->bounce_gfp | gfp_mask);
1258 			if (!page) {
1259 				ret = -ENOMEM;
1260 				break;
1261 			}
1262 		}
1263 
1264 		if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes)
1265 			break;
1266 
1267 		len -= bytes;
1268 		offset = 0;
1269 	}
1270 
1271 	if (ret)
1272 		goto cleanup;
1273 
1274 	/*
1275 	 * success
1276 	 */
1277 	if (((iter->type & WRITE) && (!map_data || !map_data->null_mapped)) ||
1278 	    (map_data && map_data->from_user)) {
1279 		ret = bio_copy_from_iter(bio, *iter);
1280 		if (ret)
1281 			goto cleanup;
1282 	}
1283 
1284 	bio->bi_private = bmd;
1285 	return bio;
1286 cleanup:
1287 	if (!map_data)
1288 		bio_free_pages(bio);
1289 	bio_put(bio);
1290 out_bmd:
1291 	kfree(bmd);
1292 	return ERR_PTR(ret);
1293 }
1294 
1295 /**
1296  *	bio_map_user_iov - map user iovec into bio
1297  *	@q:		the struct request_queue for the bio
1298  *	@iter:		iovec iterator
1299  *	@gfp_mask:	memory allocation flags
1300  *
1301  *	Map the user space address into a bio suitable for io to a block
1302  *	device. Returns an error pointer in case of error.
1303  */
1304 struct bio *bio_map_user_iov(struct request_queue *q,
1305 			     const struct iov_iter *iter,
1306 			     gfp_t gfp_mask)
1307 {
1308 	int j;
1309 	int nr_pages = 0;
1310 	struct page **pages;
1311 	struct bio *bio;
1312 	int cur_page = 0;
1313 	int ret, offset;
1314 	struct iov_iter i;
1315 	struct iovec iov;
1316 
1317 	iov_for_each(iov, i, *iter) {
1318 		unsigned long uaddr = (unsigned long) iov.iov_base;
1319 		unsigned long len = iov.iov_len;
1320 		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1321 		unsigned long start = uaddr >> PAGE_SHIFT;
1322 
1323 		/*
1324 		 * Overflow, abort
1325 		 */
1326 		if (end < start)
1327 			return ERR_PTR(-EINVAL);
1328 
1329 		nr_pages += end - start;
1330 		/*
1331 		 * buffer must be aligned to at least logical block size for now
1332 		 */
1333 		if (uaddr & queue_dma_alignment(q))
1334 			return ERR_PTR(-EINVAL);
1335 	}
1336 
1337 	if (!nr_pages)
1338 		return ERR_PTR(-EINVAL);
1339 
1340 	bio = bio_kmalloc(gfp_mask, nr_pages);
1341 	if (!bio)
1342 		return ERR_PTR(-ENOMEM);
1343 
1344 	ret = -ENOMEM;
1345 	pages = kcalloc(nr_pages, sizeof(struct page *), gfp_mask);
1346 	if (!pages)
1347 		goto out;
1348 
1349 	iov_for_each(iov, i, *iter) {
1350 		unsigned long uaddr = (unsigned long) iov.iov_base;
1351 		unsigned long len = iov.iov_len;
1352 		unsigned long end = (uaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1353 		unsigned long start = uaddr >> PAGE_SHIFT;
1354 		const int local_nr_pages = end - start;
1355 		const int page_limit = cur_page + local_nr_pages;
1356 
1357 		ret = get_user_pages_fast(uaddr, local_nr_pages,
1358 				(iter->type & WRITE) != WRITE,
1359 				&pages[cur_page]);
1360 		if (ret < local_nr_pages) {
1361 			ret = -EFAULT;
1362 			goto out_unmap;
1363 		}
1364 
1365 		offset = offset_in_page(uaddr);
1366 		for (j = cur_page; j < page_limit; j++) {
1367 			unsigned int bytes = PAGE_SIZE - offset;
1368 
1369 			if (len <= 0)
1370 				break;
1371 
1372 			if (bytes > len)
1373 				bytes = len;
1374 
1375 			/*
1376 			 * sorry...
1377 			 */
1378 			if (bio_add_pc_page(q, bio, pages[j], bytes, offset) <
1379 					    bytes)
1380 				break;
1381 
1382 			len -= bytes;
1383 			offset = 0;
1384 		}
1385 
1386 		cur_page = j;
1387 		/*
1388 		 * release the pages we didn't map into the bio, if any
1389 		 */
1390 		while (j < page_limit)
1391 			put_page(pages[j++]);
1392 	}
1393 
1394 	kfree(pages);
1395 
1396 	/*
1397 	 * set data direction, and check if mapped pages need bouncing
1398 	 */
1399 	if (iter->type & WRITE)
1400 		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1401 
1402 	bio_set_flag(bio, BIO_USER_MAPPED);
1403 
1404 	/*
1405 	 * subtle -- if __bio_map_user() ended up bouncing a bio,
1406 	 * it would normally disappear when its bi_end_io is run.
1407 	 * however, we need it for the unmap, so grab an extra
1408 	 * reference to it
1409 	 */
1410 	bio_get(bio);
1411 	return bio;
1412 
1413  out_unmap:
1414 	for (j = 0; j < nr_pages; j++) {
1415 		if (!pages[j])
1416 			break;
1417 		put_page(pages[j]);
1418 	}
1419  out:
1420 	kfree(pages);
1421 	bio_put(bio);
1422 	return ERR_PTR(ret);
1423 }
1424 
1425 static void __bio_unmap_user(struct bio *bio)
1426 {
1427 	struct bio_vec *bvec;
1428 	int i;
1429 
1430 	/*
1431 	 * make sure we dirty pages we wrote to
1432 	 */
1433 	bio_for_each_segment_all(bvec, bio, i) {
1434 		if (bio_data_dir(bio) == READ)
1435 			set_page_dirty_lock(bvec->bv_page);
1436 
1437 		put_page(bvec->bv_page);
1438 	}
1439 
1440 	bio_put(bio);
1441 }
1442 
1443 /**
1444  *	bio_unmap_user	-	unmap a bio
1445  *	@bio:		the bio being unmapped
1446  *
1447  *	Unmap a bio previously mapped by bio_map_user(). Must be called with
1448  *	a process context.
1449  *
1450  *	bio_unmap_user() may sleep.
1451  */
1452 void bio_unmap_user(struct bio *bio)
1453 {
1454 	__bio_unmap_user(bio);
1455 	bio_put(bio);
1456 }
1457 
1458 static void bio_map_kern_endio(struct bio *bio)
1459 {
1460 	bio_put(bio);
1461 }
1462 
1463 /**
1464  *	bio_map_kern	-	map kernel address into bio
1465  *	@q: the struct request_queue for the bio
1466  *	@data: pointer to buffer to map
1467  *	@len: length in bytes
1468  *	@gfp_mask: allocation flags for bio allocation
1469  *
1470  *	Map the kernel address into a bio suitable for io to a block
1471  *	device. Returns an error pointer in case of error.
1472  */
1473 struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1474 			 gfp_t gfp_mask)
1475 {
1476 	unsigned long kaddr = (unsigned long)data;
1477 	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1478 	unsigned long start = kaddr >> PAGE_SHIFT;
1479 	const int nr_pages = end - start;
1480 	int offset, i;
1481 	struct bio *bio;
1482 
1483 	bio = bio_kmalloc(gfp_mask, nr_pages);
1484 	if (!bio)
1485 		return ERR_PTR(-ENOMEM);
1486 
1487 	offset = offset_in_page(kaddr);
1488 	for (i = 0; i < nr_pages; i++) {
1489 		unsigned int bytes = PAGE_SIZE - offset;
1490 
1491 		if (len <= 0)
1492 			break;
1493 
1494 		if (bytes > len)
1495 			bytes = len;
1496 
1497 		if (bio_add_pc_page(q, bio, virt_to_page(data), bytes,
1498 				    offset) < bytes) {
1499 			/* we don't support partial mappings */
1500 			bio_put(bio);
1501 			return ERR_PTR(-EINVAL);
1502 		}
1503 
1504 		data += bytes;
1505 		len -= bytes;
1506 		offset = 0;
1507 	}
1508 
1509 	bio->bi_end_io = bio_map_kern_endio;
1510 	return bio;
1511 }
1512 EXPORT_SYMBOL(bio_map_kern);
1513 
1514 static void bio_copy_kern_endio(struct bio *bio)
1515 {
1516 	bio_free_pages(bio);
1517 	bio_put(bio);
1518 }
1519 
1520 static void bio_copy_kern_endio_read(struct bio *bio)
1521 {
1522 	char *p = bio->bi_private;
1523 	struct bio_vec *bvec;
1524 	int i;
1525 
1526 	bio_for_each_segment_all(bvec, bio, i) {
1527 		memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
1528 		p += bvec->bv_len;
1529 	}
1530 
1531 	bio_copy_kern_endio(bio);
1532 }
1533 
1534 /**
1535  *	bio_copy_kern	-	copy kernel address into bio
1536  *	@q: the struct request_queue for the bio
1537  *	@data: pointer to buffer to copy
1538  *	@len: length in bytes
1539  *	@gfp_mask: allocation flags for bio and page allocation
1540  *	@reading: data direction is READ
1541  *
1542  *	copy the kernel address into a bio suitable for io to a block
1543  *	device. Returns an error pointer in case of error.
1544  */
1545 struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1546 			  gfp_t gfp_mask, int reading)
1547 {
1548 	unsigned long kaddr = (unsigned long)data;
1549 	unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1550 	unsigned long start = kaddr >> PAGE_SHIFT;
1551 	struct bio *bio;
1552 	void *p = data;
1553 	int nr_pages = 0;
1554 
1555 	/*
1556 	 * Overflow, abort
1557 	 */
1558 	if (end < start)
1559 		return ERR_PTR(-EINVAL);
1560 
1561 	nr_pages = end - start;
1562 	bio = bio_kmalloc(gfp_mask, nr_pages);
1563 	if (!bio)
1564 		return ERR_PTR(-ENOMEM);
1565 
1566 	while (len) {
1567 		struct page *page;
1568 		unsigned int bytes = PAGE_SIZE;
1569 
1570 		if (bytes > len)
1571 			bytes = len;
1572 
1573 		page = alloc_page(q->bounce_gfp | gfp_mask);
1574 		if (!page)
1575 			goto cleanup;
1576 
1577 		if (!reading)
1578 			memcpy(page_address(page), p, bytes);
1579 
1580 		if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
1581 			break;
1582 
1583 		len -= bytes;
1584 		p += bytes;
1585 	}
1586 
1587 	if (reading) {
1588 		bio->bi_end_io = bio_copy_kern_endio_read;
1589 		bio->bi_private = data;
1590 	} else {
1591 		bio->bi_end_io = bio_copy_kern_endio;
1592 		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
1593 	}
1594 
1595 	return bio;
1596 
1597 cleanup:
1598 	bio_free_pages(bio);
1599 	bio_put(bio);
1600 	return ERR_PTR(-ENOMEM);
1601 }
1602 
1603 /*
1604  * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1605  * for performing direct-IO in BIOs.
1606  *
1607  * The problem is that we cannot run set_page_dirty() from interrupt context
1608  * because the required locks are not interrupt-safe.  So what we can do is to
1609  * mark the pages dirty _before_ performing IO.  And in interrupt context,
1610  * check that the pages are still dirty.   If so, fine.  If not, redirty them
1611  * in process context.
1612  *
1613  * We special-case compound pages here: normally this means reads into hugetlb
1614  * pages.  The logic in here doesn't really work right for compound pages
1615  * because the VM does not uniformly chase down the head page in all cases.
1616  * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1617  * handle them at all.  So we skip compound pages here at an early stage.
1618  *
1619  * Note that this code is very hard to test under normal circumstances because
1620  * direct-io pins the pages with get_user_pages().  This makes
1621  * is_page_cache_freeable return false, and the VM will not clean the pages.
1622  * But other code (eg, flusher threads) could clean the pages if they are mapped
1623  * pagecache.
1624  *
1625  * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1626  * deferred bio dirtying paths.
1627  */
1628 
1629 /*
1630  * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1631  */
1632 void bio_set_pages_dirty(struct bio *bio)
1633 {
1634 	struct bio_vec *bvec;
1635 	int i;
1636 
1637 	bio_for_each_segment_all(bvec, bio, i) {
1638 		struct page *page = bvec->bv_page;
1639 
1640 		if (page && !PageCompound(page))
1641 			set_page_dirty_lock(page);
1642 	}
1643 }
1644 
1645 static void bio_release_pages(struct bio *bio)
1646 {
1647 	struct bio_vec *bvec;
1648 	int i;
1649 
1650 	bio_for_each_segment_all(bvec, bio, i) {
1651 		struct page *page = bvec->bv_page;
1652 
1653 		if (page)
1654 			put_page(page);
1655 	}
1656 }
1657 
1658 /*
1659  * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1660  * If they are, then fine.  If, however, some pages are clean then they must
1661  * have been written out during the direct-IO read.  So we take another ref on
1662  * the BIO and the offending pages and re-dirty the pages in process context.
1663  *
1664  * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1665  * here on.  It will run one put_page() against each page and will run one
1666  * bio_put() against the BIO.
1667  */
1668 
1669 static void bio_dirty_fn(struct work_struct *work);
1670 
1671 static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1672 static DEFINE_SPINLOCK(bio_dirty_lock);
1673 static struct bio *bio_dirty_list;
1674 
1675 /*
1676  * This runs in process context
1677  */
1678 static void bio_dirty_fn(struct work_struct *work)
1679 {
1680 	unsigned long flags;
1681 	struct bio *bio;
1682 
1683 	spin_lock_irqsave(&bio_dirty_lock, flags);
1684 	bio = bio_dirty_list;
1685 	bio_dirty_list = NULL;
1686 	spin_unlock_irqrestore(&bio_dirty_lock, flags);
1687 
1688 	while (bio) {
1689 		struct bio *next = bio->bi_private;
1690 
1691 		bio_set_pages_dirty(bio);
1692 		bio_release_pages(bio);
1693 		bio_put(bio);
1694 		bio = next;
1695 	}
1696 }
1697 
1698 void bio_check_pages_dirty(struct bio *bio)
1699 {
1700 	struct bio_vec *bvec;
1701 	int nr_clean_pages = 0;
1702 	int i;
1703 
1704 	bio_for_each_segment_all(bvec, bio, i) {
1705 		struct page *page = bvec->bv_page;
1706 
1707 		if (PageDirty(page) || PageCompound(page)) {
1708 			put_page(page);
1709 			bvec->bv_page = NULL;
1710 		} else {
1711 			nr_clean_pages++;
1712 		}
1713 	}
1714 
1715 	if (nr_clean_pages) {
1716 		unsigned long flags;
1717 
1718 		spin_lock_irqsave(&bio_dirty_lock, flags);
1719 		bio->bi_private = bio_dirty_list;
1720 		bio_dirty_list = bio;
1721 		spin_unlock_irqrestore(&bio_dirty_lock, flags);
1722 		schedule_work(&bio_dirty_work);
1723 	} else {
1724 		bio_put(bio);
1725 	}
1726 }
1727 
1728 void generic_start_io_acct(int rw, unsigned long sectors,
1729 			   struct hd_struct *part)
1730 {
1731 	int cpu = part_stat_lock();
1732 
1733 	part_round_stats(cpu, part);
1734 	part_stat_inc(cpu, part, ios[rw]);
1735 	part_stat_add(cpu, part, sectors[rw], sectors);
1736 	part_inc_in_flight(part, rw);
1737 
1738 	part_stat_unlock();
1739 }
1740 EXPORT_SYMBOL(generic_start_io_acct);
1741 
1742 void generic_end_io_acct(int rw, struct hd_struct *part,
1743 			 unsigned long start_time)
1744 {
1745 	unsigned long duration = jiffies - start_time;
1746 	int cpu = part_stat_lock();
1747 
1748 	part_stat_add(cpu, part, ticks[rw], duration);
1749 	part_round_stats(cpu, part);
1750 	part_dec_in_flight(part, rw);
1751 
1752 	part_stat_unlock();
1753 }
1754 EXPORT_SYMBOL(generic_end_io_acct);
1755 
1756 #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
1757 void bio_flush_dcache_pages(struct bio *bi)
1758 {
1759 	struct bio_vec bvec;
1760 	struct bvec_iter iter;
1761 
1762 	bio_for_each_segment(bvec, bi, iter)
1763 		flush_dcache_page(bvec.bv_page);
1764 }
1765 EXPORT_SYMBOL(bio_flush_dcache_pages);
1766 #endif
1767 
1768 static inline bool bio_remaining_done(struct bio *bio)
1769 {
1770 	/*
1771 	 * If we're not chaining, then ->__bi_remaining is always 1 and
1772 	 * we always end io on the first invocation.
1773 	 */
1774 	if (!bio_flagged(bio, BIO_CHAIN))
1775 		return true;
1776 
1777 	BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1778 
1779 	if (atomic_dec_and_test(&bio->__bi_remaining)) {
1780 		bio_clear_flag(bio, BIO_CHAIN);
1781 		return true;
1782 	}
1783 
1784 	return false;
1785 }
1786 
1787 /**
1788  * bio_endio - end I/O on a bio
1789  * @bio:	bio
1790  *
1791  * Description:
1792  *   bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1793  *   way to end I/O on a bio. No one should call bi_end_io() directly on a
1794  *   bio unless they own it and thus know that it has an end_io function.
1795  **/
1796 void bio_endio(struct bio *bio)
1797 {
1798 again:
1799 	if (!bio_remaining_done(bio))
1800 		return;
1801 
1802 	/*
1803 	 * Need to have a real endio function for chained bios, otherwise
1804 	 * various corner cases will break (like stacking block devices that
1805 	 * save/restore bi_end_io) - however, we want to avoid unbounded
1806 	 * recursion and blowing the stack. Tail call optimization would
1807 	 * handle this, but compiling with frame pointers also disables
1808 	 * gcc's sibling call optimization.
1809 	 */
1810 	if (bio->bi_end_io == bio_chain_endio) {
1811 		bio = __bio_chain_endio(bio);
1812 		goto again;
1813 	}
1814 
1815 	if (bio->bi_end_io)
1816 		bio->bi_end_io(bio);
1817 }
1818 EXPORT_SYMBOL(bio_endio);
1819 
1820 /**
1821  * bio_split - split a bio
1822  * @bio:	bio to split
1823  * @sectors:	number of sectors to split from the front of @bio
1824  * @gfp:	gfp mask
1825  * @bs:		bio set to allocate from
1826  *
1827  * Allocates and returns a new bio which represents @sectors from the start of
1828  * @bio, and updates @bio to represent the remaining sectors.
1829  *
1830  * Unless this is a discard request the newly allocated bio will point
1831  * to @bio's bi_io_vec; it is the caller's responsibility to ensure that
1832  * @bio is not freed before the split.
1833  */
1834 struct bio *bio_split(struct bio *bio, int sectors,
1835 		      gfp_t gfp, struct bio_set *bs)
1836 {
1837 	struct bio *split = NULL;
1838 
1839 	BUG_ON(sectors <= 0);
1840 	BUG_ON(sectors >= bio_sectors(bio));
1841 
1842 	/*
1843 	 * Discards need a mutable bio_vec to accommodate the payload
1844 	 * required by the DSM TRIM and UNMAP commands.
1845 	 */
1846 	if (bio_op(bio) == REQ_OP_DISCARD || bio_op(bio) == REQ_OP_SECURE_ERASE)
1847 		split = bio_clone_bioset(bio, gfp, bs);
1848 	else
1849 		split = bio_clone_fast(bio, gfp, bs);
1850 
1851 	if (!split)
1852 		return NULL;
1853 
1854 	split->bi_iter.bi_size = sectors << 9;
1855 
1856 	if (bio_integrity(split))
1857 		bio_integrity_trim(split, 0, sectors);
1858 
1859 	bio_advance(bio, split->bi_iter.bi_size);
1860 
1861 	return split;
1862 }
1863 EXPORT_SYMBOL(bio_split);
1864 
1865 /**
1866  * bio_trim - trim a bio
1867  * @bio:	bio to trim
1868  * @offset:	number of sectors to trim from the front of @bio
1869  * @size:	size we want to trim @bio to, in sectors
1870  */
1871 void bio_trim(struct bio *bio, int offset, int size)
1872 {
1873 	/* 'bio' is a cloned bio which we need to trim to match
1874 	 * the given offset and size.
1875 	 */
1876 
1877 	size <<= 9;
1878 	if (offset == 0 && size == bio->bi_iter.bi_size)
1879 		return;
1880 
1881 	bio_clear_flag(bio, BIO_SEG_VALID);
1882 
1883 	bio_advance(bio, offset << 9);
1884 
1885 	bio->bi_iter.bi_size = size;
1886 }
1887 EXPORT_SYMBOL_GPL(bio_trim);
1888 
1889 /*
1890  * create memory pools for biovec's in a bio_set.
1891  * use the global biovec slabs created for general use.
1892  */
1893 mempool_t *biovec_create_pool(int pool_entries)
1894 {
1895 	struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
1896 
1897 	return mempool_create_slab_pool(pool_entries, bp->slab);
1898 }
1899 
1900 void bioset_free(struct bio_set *bs)
1901 {
1902 	if (bs->rescue_workqueue)
1903 		destroy_workqueue(bs->rescue_workqueue);
1904 
1905 	if (bs->bio_pool)
1906 		mempool_destroy(bs->bio_pool);
1907 
1908 	if (bs->bvec_pool)
1909 		mempool_destroy(bs->bvec_pool);
1910 
1911 	bioset_integrity_free(bs);
1912 	bio_put_slab(bs);
1913 
1914 	kfree(bs);
1915 }
1916 EXPORT_SYMBOL(bioset_free);
1917 
1918 static struct bio_set *__bioset_create(unsigned int pool_size,
1919 				       unsigned int front_pad,
1920 				       bool create_bvec_pool)
1921 {
1922 	unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1923 	struct bio_set *bs;
1924 
1925 	bs = kzalloc(sizeof(*bs), GFP_KERNEL);
1926 	if (!bs)
1927 		return NULL;
1928 
1929 	bs->front_pad = front_pad;
1930 
1931 	spin_lock_init(&bs->rescue_lock);
1932 	bio_list_init(&bs->rescue_list);
1933 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1934 
1935 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1936 	if (!bs->bio_slab) {
1937 		kfree(bs);
1938 		return NULL;
1939 	}
1940 
1941 	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);
1942 	if (!bs->bio_pool)
1943 		goto bad;
1944 
1945 	if (create_bvec_pool) {
1946 		bs->bvec_pool = biovec_create_pool(pool_size);
1947 		if (!bs->bvec_pool)
1948 			goto bad;
1949 	}
1950 
1951 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1952 	if (!bs->rescue_workqueue)
1953 		goto bad;
1954 
1955 	return bs;
1956 bad:
1957 	bioset_free(bs);
1958 	return NULL;
1959 }
1960 
1961 /**
1962  * bioset_create  - Create a bio_set
1963  * @pool_size:	Number of bio and bio_vecs to cache in the mempool
1964  * @front_pad:	Number of bytes to allocate in front of the returned bio
1965  *
1966  * Description:
1967  *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1968  *    to ask for a number of bytes to be allocated in front of the bio.
1969  *    Front pad allocation is useful for embedding the bio inside
1970  *    another structure, to avoid allocating extra data to go with the bio.
1971  *    Note that the bio must be embedded at the END of that structure always,
1972  *    or things will break badly.
1973  */
1974 struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad)
1975 {
1976 	return __bioset_create(pool_size, front_pad, true);
1977 }
1978 EXPORT_SYMBOL(bioset_create);
1979 
1980 /**
1981  * bioset_create_nobvec  - Create a bio_set without bio_vec mempool
1982  * @pool_size:	Number of bio to cache in the mempool
1983  * @front_pad:	Number of bytes to allocate in front of the returned bio
1984  *
1985  * Description:
1986  *    Same functionality as bioset_create() except that mempool is not
1987  *    created for bio_vecs. Saving some memory for bio_clone_fast() users.
1988  */
1989 struct bio_set *bioset_create_nobvec(unsigned int pool_size, unsigned int front_pad)
1990 {
1991 	return __bioset_create(pool_size, front_pad, false);
1992 }
1993 EXPORT_SYMBOL(bioset_create_nobvec);
1994 
1995 #ifdef CONFIG_BLK_CGROUP
1996 
1997 /**
1998  * bio_associate_blkcg - associate a bio with the specified blkcg
1999  * @bio: target bio
2000  * @blkcg_css: css of the blkcg to associate
2001  *
2002  * Associate @bio with the blkcg specified by @blkcg_css.  Block layer will
2003  * treat @bio as if it were issued by a task which belongs to the blkcg.
2004  *
2005  * This function takes an extra reference of @blkcg_css which will be put
2006  * when @bio is released.  The caller must own @bio and is responsible for
2007  * synchronizing calls to this function.
2008  */
2009 int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css)
2010 {
2011 	if (unlikely(bio->bi_css))
2012 		return -EBUSY;
2013 	css_get(blkcg_css);
2014 	bio->bi_css = blkcg_css;
2015 	return 0;
2016 }
2017 EXPORT_SYMBOL_GPL(bio_associate_blkcg);
2018 
2019 /**
2020  * bio_associate_current - associate a bio with %current
2021  * @bio: target bio
2022  *
2023  * Associate @bio with %current if it hasn't been associated yet.  Block
2024  * layer will treat @bio as if it were issued by %current no matter which
2025  * task actually issues it.
2026  *
2027  * This function takes an extra reference of @task's io_context and blkcg
2028  * which will be put when @bio is released.  The caller must own @bio,
2029  * ensure %current->io_context exists, and is responsible for synchronizing
2030  * calls to this function.
2031  */
2032 int bio_associate_current(struct bio *bio)
2033 {
2034 	struct io_context *ioc;
2035 
2036 	if (bio->bi_css)
2037 		return -EBUSY;
2038 
2039 	ioc = current->io_context;
2040 	if (!ioc)
2041 		return -ENOENT;
2042 
2043 	get_io_context_active(ioc);
2044 	bio->bi_ioc = ioc;
2045 	bio->bi_css = task_get_css(current, io_cgrp_id);
2046 	return 0;
2047 }
2048 EXPORT_SYMBOL_GPL(bio_associate_current);
2049 
2050 /**
2051  * bio_disassociate_task - undo bio_associate_current()
2052  * @bio: target bio
2053  */
2054 void bio_disassociate_task(struct bio *bio)
2055 {
2056 	if (bio->bi_ioc) {
2057 		put_io_context(bio->bi_ioc);
2058 		bio->bi_ioc = NULL;
2059 	}
2060 	if (bio->bi_css) {
2061 		css_put(bio->bi_css);
2062 		bio->bi_css = NULL;
2063 	}
2064 }
2065 
2066 /**
2067  * bio_clone_blkcg_association - clone blkcg association from src to dst bio
2068  * @dst: destination bio
2069  * @src: source bio
2070  */
2071 void bio_clone_blkcg_association(struct bio *dst, struct bio *src)
2072 {
2073 	if (src->bi_css)
2074 		WARN_ON(bio_associate_blkcg(dst, src->bi_css));
2075 }
2076 
2077 #endif /* CONFIG_BLK_CGROUP */
2078 
2079 static void __init biovec_init_slabs(void)
2080 {
2081 	int i;
2082 
2083 	for (i = 0; i < BVEC_POOL_NR; i++) {
2084 		int size;
2085 		struct biovec_slab *bvs = bvec_slabs + i;
2086 
2087 		if (bvs->nr_vecs <= BIO_INLINE_VECS) {
2088 			bvs->slab = NULL;
2089 			continue;
2090 		}
2091 
2092 		size = bvs->nr_vecs * sizeof(struct bio_vec);
2093 		bvs->slab = kmem_cache_create(bvs->name, size, 0,
2094                                 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2095 	}
2096 }
2097 
2098 static int __init init_bio(void)
2099 {
2100 	bio_slab_max = 2;
2101 	bio_slab_nr = 0;
2102 	bio_slabs = kzalloc(bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL);
2103 	if (!bio_slabs)
2104 		panic("bio: can't allocate bios\n");
2105 
2106 	bio_integrity_init();
2107 	biovec_init_slabs();
2108 
2109 	fs_bio_set = bioset_create(BIO_POOL_SIZE, 0);
2110 	if (!fs_bio_set)
2111 		panic("bio: can't allocate bios\n");
2112 
2113 	if (bioset_integrity_create(fs_bio_set, BIO_POOL_SIZE))
2114 		panic("bio: can't create integrity pool\n");
2115 
2116 	return 0;
2117 }
2118 subsys_initcall(init_bio);
2119