21 	struct request *orig, *clone;  member
78 static void end_clone_bio(struct bio *clone)  in end_clone_bio()  argument
81 		container_of(clone, struct dm_rq_clone_bio_info, clone);  in end_clone_bio()
84 	blk_status_t error = clone->bi_status;  in end_clone_bio()
85 	bool is_last = !clone->bi_next;  in end_clone_bio()
87 	bio_put(clone);  in end_clone_bio()
92 		 * Once error occurred, just let clone->end_io() handle  in end_clone_bio()
115 	 * the original request before the clone, and break the ordering.  in end_clone_bio()
153  * Complete the clone and the original request.
154  * Must be called without clone's queue lock held,
157 static void dm_end_request(struct request *clone, blk_status_t error)  in dm_end_request()  argument
159 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_end_request()
163 	blk_rq_unprep_clone(clone);  in dm_end_request()
164 	tio->ti->type->release_clone_rq(clone, NULL);  in dm_end_request()
195 	if (tio->clone) {  in dm_requeue_original_request()
196 		blk_rq_unprep_clone(tio->clone);  in dm_requeue_original_request()
197 		tio->ti->type->release_clone_rq(tio->clone, NULL);  in dm_requeue_original_request()
204 static void dm_done(struct request *clone, blk_status_t error, bool mapped)  in dm_done()  argument
207 	struct dm_rq_target_io *tio = clone->end_io_data;  in dm_done()
214 			r = rq_end_io(tio->ti, clone, error, &tio->info);  in dm_done()
218 		if (req_op(clone) == REQ_OP_DISCARD &&  in dm_done()
219 		    !clone->q->limits.max_discard_sectors)  in dm_done()
221 		else if (req_op(clone) == REQ_OP_WRITE_ZEROES &&  in dm_done()
222 			 !clone->q->limits.max_write_zeroes_sectors)  in dm_done()
229 		dm_end_request(clone, error);  in dm_done()
255 	struct request *clone = tio->clone;  in dm_softirq_done()  local
257 	if (!clone) {  in dm_softirq_done()
269 	dm_done(clone, tio->error, mapped);  in dm_softirq_done()
273  * Complete the clone and the original request with the error status
286  * Complete the not-mapped clone and the original request with the error status
297 static enum rq_end_io_ret end_clone_request(struct request *clone,  in end_clone_request()  argument
300 	struct dm_rq_target_io *tio = clone->end_io_data;  in end_clone_request()
311 		container_of(bio, struct dm_rq_clone_bio_info, clone);  in dm_rq_bio_constructor()
320 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
325 	r = blk_rq_prep_clone(clone, rq, &tio->md->mempools->bs, gfp_mask,  in setup_clone()
330 	clone->end_io = end_clone_request;  in setup_clone()
331 	clone->end_io_data = tio;  in setup_clone()
333 	tio->clone = clone;  in setup_clone()
343 	tio->clone = NULL;  in init_tio()
368 	struct request *clone = NULL;  in map_request()  local
371 	r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
377 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
379 			ti->type->release_clone_rq(clone, &tio->info);  in map_request()
384 		trace_block_rq_remap(clone, disk_devt(dm_disk(md)),  in map_request()
386 		ret = blk_insert_cloned_request(clone);  in map_request()
392 			blk_rq_unprep_clone(clone);  in map_request()
393 			blk_mq_cleanup_rq(clone);  in map_request()
394 			tio->ti->type->release_clone_rq(clone, &tio->info);  in map_request()
395 			tio->clone = NULL;  in map_request()
398 			/* must complete clone in terms of original request */  in map_request()