28 static void bch_data_insert_start(struct closure *cl);
58 static void bch_data_insert_keys(struct closure *cl)  in bch_data_insert_keys()  argument
60 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_keys()
67 					  op->flush_journal ? cl : NULL);  in bch_data_insert_keys()
82 		continue_at(cl, bch_data_insert_start, op->wq);  in bch_data_insert_keys()
87 	closure_return(cl);  in bch_data_insert_keys()
108 static void bch_data_invalidate(struct closure *cl)  in bch_data_invalidate()  argument
110 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_invalidate()
136 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_invalidate()
139 static void bch_data_insert_error(struct closure *cl)  in bch_data_insert_error()  argument
141 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_error()
166 	bch_data_insert_keys(cl);  in bch_data_insert_error()
171 	struct closure *cl = bio->bi_private;  in bch_data_insert_endio()  local
172 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_endio()
179 			set_closure_fn(cl, bch_data_insert_error, op->wq);  in bch_data_insert_endio()
181 			set_closure_fn(cl, NULL, NULL);  in bch_data_insert_endio()
187 static void bch_data_insert_start(struct closure *cl)  in bch_data_insert_start()  argument
189 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert_start()
193 		return bch_data_invalidate(cl);  in bch_data_insert_start()
213 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
230 		n->bi_private	= cl;  in bch_data_insert_start()
252 	continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
272 		return bch_data_invalidate(cl);  in bch_data_insert_start()
282 			continue_at(cl, bch_data_insert_keys, op->wq);  in bch_data_insert_start()
284 			closure_return(cl);  in bch_data_insert_start()
308 void bch_data_insert(struct closure *cl)  in bch_data_insert()  argument
310 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);  in bch_data_insert()
317 	bch_data_insert_start(cl);  in bch_data_insert()
465 	struct closure		cl;  member
488 	struct closure *cl = bio->bi_private;  in bch_cache_read_endio()  local
489 	struct search *s = container_of(cl, struct search, cl);  in bch_cache_read_endio()
561 	n->bi_private	= &s->cl;  in cache_lookup_fn()
578 static void cache_lookup(struct closure *cl)  in cache_lookup()  argument
580 	struct search *s = container_of(cl, struct search, iop.cl);  in cache_lookup()
591 		continue_at(cl, cache_lookup, bcache_wq);  in cache_lookup()
616 	closure_return(cl);  in cache_lookup()
623 	struct closure *cl = bio->bi_private;  in request_endio()  local
626 		struct search *s = container_of(cl, struct search, cl);  in request_endio()
634 	closure_put(cl);  in request_endio()
639 	struct closure *cl = bio->bi_private;  in backing_request_endio()  local
642 		struct search *s = container_of(cl, struct search, cl);  in backing_request_endio()
666 	closure_put(cl);  in backing_request_endio()
696 	bio->bi_private		= &s->cl;  in do_bio_hook()
701 static void search_free(struct closure *cl)  in search_free()  argument
703 	struct search *s = container_of(cl, struct search, cl);  in search_free()
711 	closure_debug_destroy(cl);  in search_free()
723 	closure_init(&s->cl, NULL);  in search_alloc()
752 static void cached_dev_bio_complete(struct closure *cl)  in cached_dev_bio_complete()  argument
754 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_bio_complete()
758 	search_free(cl);  in cached_dev_bio_complete()
763 static void cached_dev_read_error_done(struct closure *cl)  in cached_dev_read_error_done()  argument
765 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error_done()
773 	cached_dev_bio_complete(cl);  in cached_dev_read_error_done()
776 static void cached_dev_read_error(struct closure *cl)  in cached_dev_read_error()  argument
778 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_error()
798 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_read_error()
801 	continue_at(cl, cached_dev_read_error_done, NULL);  in cached_dev_read_error()
804 static void cached_dev_cache_miss_done(struct closure *cl)  in cached_dev_cache_miss_done()  argument
806 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_cache_miss_done()
815 	cached_dev_bio_complete(cl);  in cached_dev_cache_miss_done()
816 	closure_put(&d->cl);  in cached_dev_cache_miss_done()
819 static void cached_dev_read_done(struct closure *cl)  in cached_dev_read_done()  argument
821 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done()
849 	closure_get(&dc->disk.cl);  in cached_dev_read_done()
855 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_read_done()
858 	continue_at(cl, cached_dev_cache_miss_done, NULL);  in cached_dev_read_done()
861 static void cached_dev_read_done_bh(struct closure *cl)  in cached_dev_read_done_bh()  argument
863 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_read_done_bh()
871 		continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);  in cached_dev_read_done_bh()
873 		continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);  in cached_dev_read_done_bh()
875 		continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);  in cached_dev_read_done_bh()
925 	cache_bio->bi_private	= &s->cl;  in cached_dev_cache_miss()
935 	closure_bio_submit(s->iop.c, cache_bio, &s->cl);  in cached_dev_cache_miss()
942 	miss->bi_private	= &s->cl;  in cached_dev_cache_miss()
944 	closure_bio_submit(s->iop.c, miss, &s->cl);  in cached_dev_cache_miss()
950 	struct closure *cl = &s->cl;  in cached_dev_read()  local
952 	closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in cached_dev_read()
953 	continue_at(cl, cached_dev_read_done_bh, NULL);  in cached_dev_read()
958 static void cached_dev_write_complete(struct closure *cl)  in cached_dev_write_complete()  argument
960 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_write_complete()
964 	cached_dev_bio_complete(cl);  in cached_dev_write_complete()
969 	struct closure *cl = &s->cl;  in cached_dev_write()  local
1013 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1034 			flush->bi_private = cl;  in cached_dev_write()
1036 			closure_bio_submit(s->iop.c, flush, cl);  in cached_dev_write()
1043 		closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_write()
1047 	closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in cached_dev_write()
1048 	continue_at(cl, cached_dev_write_complete, NULL);  in cached_dev_write()
1051 static void cached_dev_nodata(struct closure *cl)  in cached_dev_nodata()  argument
1053 	struct search *s = container_of(cl, struct search, cl);  in cached_dev_nodata()
1057 		bch_journal_meta(s->iop.c, cl);  in cached_dev_nodata()
1061 	closure_bio_submit(s->iop.c, bio, cl);  in cached_dev_nodata()
1063 	continue_at(cl, cached_dev_bio_complete, NULL);  in cached_dev_nodata()
1215 			continue_at_nobarrier(&s->cl,  in cached_dev_submit_bio()
1268 static void flash_dev_nodata(struct closure *cl)  in flash_dev_nodata()  argument
1270 	struct search *s = container_of(cl, struct search, cl);  in flash_dev_nodata()
1273 		bch_journal_meta(s->iop.c, cl);  in flash_dev_nodata()
1275 	continue_at(cl, search_free, NULL);  in flash_dev_nodata()
1281 	struct closure *cl;  in flash_dev_submit_bio()  local
1291 	cl = &s->cl;  in flash_dev_submit_bio()
1300 		continue_at_nobarrier(&s->cl,  in flash_dev_submit_bio()
1313 		closure_call(&s->iop.cl, bch_data_insert, NULL, cl);  in flash_dev_submit_bio()
1315 		closure_call(&s->iop.cl, cache_lookup, NULL, cl);  in flash_dev_submit_bio()
1318 	continue_at(cl, search_free, NULL);  in flash_dev_submit_bio()