Lines Matching refs:bio
40 static void bio_csum(struct bio *bio, struct bkey *k) in bio_csum() argument
46 bio_for_each_segment(bv, bio, iter) { in bio_csum()
111 struct bio *bio = op->bio; in bch_data_invalidate() local
114 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); in bch_data_invalidate()
116 while (bio_sectors(bio)) { in bch_data_invalidate()
117 unsigned int sectors = min(bio_sectors(bio), in bch_data_invalidate()
123 bio->bi_iter.bi_sector += sectors; in bch_data_invalidate()
124 bio->bi_iter.bi_size -= sectors << 9; in bch_data_invalidate()
128 bio->bi_iter.bi_sector, in bch_data_invalidate()
134 bio_put(bio); in bch_data_invalidate()
169 static void bch_data_insert_endio(struct bio *bio) in bch_data_insert_endio() argument
171 struct closure *cl = bio->bi_private; in bch_data_insert_endio()
174 if (bio->bi_status) { in bch_data_insert_endio()
177 op->status = bio->bi_status; in bch_data_insert_endio()
184 bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache"); in bch_data_insert_endio()
190 struct bio *bio = op->bio, *n; in bch_data_insert_start() local
195 if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0) in bch_data_insert_start()
202 bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA); in bch_data_insert_start()
220 SET_KEY_OFFSET(k, bio->bi_iter.bi_sector); in bch_data_insert_start()
222 if (!bch_alloc_sectors(op->c, k, bio_sectors(bio), in bch_data_insert_start()
227 n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split); in bch_data_insert_start()
249 } while (n != bio); in bch_data_insert_start()
279 bio_put(bio); in bch_data_insert_start()
312 trace_bcache_write(op->c, op->inode, op->bio, in bch_data_insert()
316 bio_get(op->bio); in bch_data_insert()
363 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) in check_should_bypass() argument
373 (bio_op(bio) == REQ_OP_DISCARD)) in check_should_bypass()
378 op_is_write(bio_op(bio)))) in check_should_bypass()
391 if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) { in check_should_bypass()
392 if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) && in check_should_bypass()
397 if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) || in check_should_bypass()
398 bio_sectors(bio) & (c->cache->sb.block_size - 1)) { in check_should_bypass()
416 hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash) in check_should_bypass()
417 if (i->last == bio->bi_iter.bi_sector && in check_should_bypass()
426 if (i->sequential + bio->bi_iter.bi_size > i->sequential) in check_should_bypass()
427 i->sequential += bio->bi_iter.bi_size; in check_should_bypass()
429 i->last = bio_end_sector(bio); in check_should_bypass()
444 trace_bcache_bypass_sequential(bio); in check_should_bypass()
449 trace_bcache_bypass_congested(bio); in check_should_bypass()
454 bch_rescale_priorities(c, bio_sectors(bio)); in check_should_bypass()
457 bch_mark_sectors_bypassed(c, dc, bio_sectors(bio)); in check_should_bypass()
467 struct bbio bio; member
468 struct bio *orig_bio;
469 struct bio *cache_miss;
485 static void bch_cache_read_endio(struct bio *bio) in bch_cache_read_endio() argument
487 struct bbio *b = container_of(bio, struct bbio, bio); in bch_cache_read_endio()
488 struct closure *cl = bio->bi_private; in bch_cache_read_endio()
498 if (bio->bi_status) in bch_cache_read_endio()
499 s->iop.status = bio->bi_status; in bch_cache_read_endio()
506 bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache"); in bch_cache_read_endio()
516 struct bio *n, *bio = &s->bio.bio; in cache_lookup_fn() local
520 if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0) in cache_lookup_fn()
524 KEY_START(k) > bio->bi_iter.bi_sector) { in cache_lookup_fn()
525 unsigned int bio_sectors = bio_sectors(bio); in cache_lookup_fn()
528 KEY_START(k) - bio->bi_iter.bi_sector) in cache_lookup_fn()
530 int ret = s->d->cache_miss(b, s, bio, sectors); in cache_lookup_fn()
550 n = bio_next_split(bio, min_t(uint64_t, INT_MAX, in cache_lookup_fn()
551 KEY_OFFSET(k) - bio->bi_iter.bi_sector), in cache_lookup_fn()
554 bio_key = &container_of(n, struct bbio, bio)->key; in cache_lookup_fn()
575 return n == bio ? MAP_DONE : MAP_CONTINUE; in cache_lookup_fn()
581 struct bio *bio = &s->bio.bio; in cache_lookup() local
588 &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0), in cache_lookup()
621 static void request_endio(struct bio *bio) in request_endio() argument
623 struct closure *cl = bio->bi_private; in request_endio()
625 if (bio->bi_status) { in request_endio()
628 s->iop.status = bio->bi_status; in request_endio()
633 bio_put(bio); in request_endio()
637 static void backing_request_endio(struct bio *bio) in backing_request_endio() argument
639 struct closure *cl = bio->bi_private; in backing_request_endio()
641 if (bio->bi_status) { in backing_request_endio()
653 bio->bi_opf & REQ_PREFLUSH)) { in backing_request_endio()
655 dc->bdev, bio->bi_status); in backing_request_endio()
658 s->iop.status = bio->bi_status; in backing_request_endio()
662 bch_count_backing_io_errors(dc, bio); in backing_request_endio()
665 bio_put(bio); in backing_request_endio()
683 struct bio *orig_bio, in do_bio_hook()
686 struct bio *bio = &s->bio.bio; in do_bio_hook() local
688 bio_init_clone(orig_bio->bi_bdev, bio, orig_bio, GFP_NOIO); in do_bio_hook()
695 bio->bi_end_io = end_io_fn; in do_bio_hook()
696 bio->bi_private = &s->cl; in do_bio_hook()
698 bio_cnt_set(bio, 3); in do_bio_hook()
707 if (s->iop.bio) in search_free()
708 bio_put(s->iop.bio); in search_free()
715 static inline struct search *search_alloc(struct bio *bio, in search_alloc() argument
724 do_bio_hook(s, bio, request_endio); in search_alloc()
727 s->orig_bio = bio; in search_alloc()
732 s->write = op_is_write(bio_op(bio)); in search_alloc()
738 s->iop.bio = NULL; in search_alloc()
744 s->iop.flush_journal = op_is_flush(bio->bi_opf); in search_alloc()
770 if (s->iop.bio) in cached_dev_read_error_done()
771 bio_free_pages(s->iop.bio); in cached_dev_read_error_done()
779 struct bio *bio = &s->bio.bio; in cached_dev_read_error() local
798 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_read_error()
812 if (s->iop.bio) in cached_dev_cache_miss_done()
813 bio_free_pages(s->iop.bio); in cached_dev_cache_miss_done()
832 if (s->iop.bio) { in cached_dev_read_done()
833 bio_reset(s->iop.bio, s->cache_miss->bi_bdev, REQ_OP_READ); in cached_dev_read_done()
834 s->iop.bio->bi_iter.bi_sector = in cached_dev_read_done()
836 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9; in cached_dev_read_done()
837 bio_clone_blkg_association(s->iop.bio, s->cache_miss); in cached_dev_read_done()
838 bch_bio_map(s->iop.bio, NULL); in cached_dev_read_done()
840 bio_copy_data(s->cache_miss, s->iop.bio); in cached_dev_read_done()
852 if (s->iop.bio && in cached_dev_read_done()
872 else if (s->iop.bio || verify(dc)) in cached_dev_read_done_bh()
879 struct bio *bio, unsigned int sectors) in cached_dev_cache_miss() argument
883 struct bio *miss, *cache_bio; in cached_dev_cache_miss()
889 miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split); in cached_dev_cache_miss()
890 ret = miss == bio ? MAP_DONE : MAP_CONTINUE; in cached_dev_cache_miss()
897 s->insert_bio_sectors = min3(size_limit, sectors, bio_sectors(bio)); in cached_dev_cache_miss()
900 bio->bi_iter.bi_sector + s->insert_bio_sectors, in cached_dev_cache_miss()
909 miss = bio_next_split(bio, s->insert_bio_sectors, GFP_NOIO, in cached_dev_cache_miss()
913 ret = miss == bio ? MAP_DONE : -EINTR; in cached_dev_cache_miss()
932 s->iop.bio = cache_bio; in cached_dev_cache_miss()
970 struct bio *bio = &s->bio.bio; in cached_dev_write() local
971 struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0); in cached_dev_write()
972 struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0); in cached_dev_write()
993 if (bio_op(bio) == REQ_OP_DISCARD) in cached_dev_write()
1004 s->iop.bio = s->orig_bio; in cached_dev_write()
1005 bio_get(s->iop.bio); in cached_dev_write()
1007 if (bio_op(bio) == REQ_OP_DISCARD && in cached_dev_write()
1012 bio->bi_end_io = backing_request_endio; in cached_dev_write()
1013 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1017 s->iop.bio = bio; in cached_dev_write()
1019 if (bio->bi_opf & REQ_PREFLUSH) { in cached_dev_write()
1024 struct bio *flush; in cached_dev_write()
1026 flush = bio_alloc_bioset(bio->bi_bdev, 0, in cached_dev_write()
1039 s->iop.bio = bio_alloc_clone(bio->bi_bdev, bio, GFP_NOIO, in cached_dev_write()
1042 bio->bi_end_io = backing_request_endio; in cached_dev_write()
1043 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_write()
1054 struct bio *bio = &s->bio.bio; in cached_dev_nodata() local
1060 bio->bi_end_io = backing_request_endio; in cached_dev_nodata()
1061 closure_bio_submit(s->iop.c, bio, cl); in cached_dev_nodata()
1074 static void detached_dev_end_io(struct bio *bio) in detached_dev_end_io() argument
1078 ddip = bio->bi_private; in detached_dev_end_io()
1079 bio->bi_end_io = ddip->bi_end_io; in detached_dev_end_io()
1080 bio->bi_private = ddip->bi_private; in detached_dev_end_io()
1083 bio_end_io_acct_remapped(bio, ddip->start_time, ddip->orig_bdev); in detached_dev_end_io()
1085 if (bio->bi_status) { in detached_dev_end_io()
1089 bch_count_backing_io_errors(dc, bio); in detached_dev_end_io()
1093 bio->bi_end_io(bio); in detached_dev_end_io()
1096 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio, in detached_dev_do_request() argument
1109 bio->bi_status = BLK_STS_RESOURCE; in detached_dev_do_request()
1110 bio->bi_end_io(bio); in detached_dev_do_request()
1118 ddip->bi_end_io = bio->bi_end_io; in detached_dev_do_request()
1119 ddip->bi_private = bio->bi_private; in detached_dev_do_request()
1120 bio->bi_end_io = detached_dev_end_io; in detached_dev_do_request()
1121 bio->bi_private = ddip; in detached_dev_do_request()
1123 if ((bio_op(bio) == REQ_OP_DISCARD) && in detached_dev_do_request()
1125 bio->bi_end_io(bio); in detached_dev_do_request()
1127 submit_bio_noacct(bio); in detached_dev_do_request()
1170 void cached_dev_submit_bio(struct bio *bio) in cached_dev_submit_bio() argument
1173 struct block_device *orig_bdev = bio->bi_bdev; in cached_dev_submit_bio()
1177 int rw = bio_data_dir(bio); in cached_dev_submit_bio()
1181 bio->bi_status = BLK_STS_IOERR; in cached_dev_submit_bio()
1182 bio_endio(bio); in cached_dev_submit_bio()
1201 start_time = bio_start_io_acct(bio); in cached_dev_submit_bio()
1203 bio_set_dev(bio, dc->bdev); in cached_dev_submit_bio()
1204 bio->bi_iter.bi_sector += dc->sb.data_offset; in cached_dev_submit_bio()
1207 s = search_alloc(bio, d, orig_bdev, start_time); in cached_dev_submit_bio()
1208 trace_bcache_request_start(s->d, bio); in cached_dev_submit_bio()
1210 if (!bio->bi_iter.bi_size) { in cached_dev_submit_bio()
1219 s->iop.bypass = check_should_bypass(dc, bio); in cached_dev_submit_bio()
1228 detached_dev_do_request(d, bio, orig_bdev, start_time); in cached_dev_submit_bio()
1252 struct bio *bio, unsigned int sectors) in flash_dev_cache_miss() argument
1254 unsigned int bytes = min(sectors, bio_sectors(bio)) << 9; in flash_dev_cache_miss()
1256 swap(bio->bi_iter.bi_size, bytes); in flash_dev_cache_miss()
1257 zero_fill_bio(bio); in flash_dev_cache_miss()
1258 swap(bio->bi_iter.bi_size, bytes); in flash_dev_cache_miss()
1260 bio_advance(bio, bytes); in flash_dev_cache_miss()
1262 if (!bio->bi_iter.bi_size) in flash_dev_cache_miss()
1278 void flash_dev_submit_bio(struct bio *bio) in flash_dev_submit_bio() argument
1282 struct bcache_device *d = bio->bi_bdev->bd_disk->private_data; in flash_dev_submit_bio()
1285 bio->bi_status = BLK_STS_IOERR; in flash_dev_submit_bio()
1286 bio_endio(bio); in flash_dev_submit_bio()
1290 s = search_alloc(bio, d, bio->bi_bdev, bio_start_io_acct(bio)); in flash_dev_submit_bio()
1292 bio = &s->bio.bio; in flash_dev_submit_bio()
1294 trace_bcache_request_start(s->d, bio); in flash_dev_submit_bio()
1296 if (!bio->bi_iter.bi_size) { in flash_dev_submit_bio()
1304 } else if (bio_data_dir(bio)) { in flash_dev_submit_bio()
1306 &KEY(d->id, bio->bi_iter.bi_sector, 0), in flash_dev_submit_bio()
1307 &KEY(d->id, bio_end_sector(bio), 0)); in flash_dev_submit_bio()
1309 s->iop.bypass = (bio_op(bio) == REQ_OP_DISCARD) != 0; in flash_dev_submit_bio()
1311 s->iop.bio = bio; in flash_dev_submit_bio()