336 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);  in lo_complete_rq()  local
339 	if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||  in lo_complete_rq()
341 		if (cmd->ret < 0)  in lo_complete_rq()
342 			ret = errno_to_blk_status(cmd->ret);  in lo_complete_rq()
350 	if (cmd->ret) {  in lo_complete_rq()
351 		blk_update_request(rq, BLK_STS_OK, cmd->ret);  in lo_complete_rq()
352 		cmd->ret = 0;  in lo_complete_rq()
355 		if (cmd->use_aio) {  in lo_complete_rq()
369 static void lo_rw_aio_do_completion(struct loop_cmd *cmd)  in lo_rw_aio_do_completion()  argument
371 	struct request *rq = blk_mq_rq_from_pdu(cmd);  in lo_rw_aio_do_completion()
373 	if (!atomic_dec_and_test(&cmd->ref))  in lo_rw_aio_do_completion()
375 	kfree(cmd->bvec);  in lo_rw_aio_do_completion()
376 	cmd->bvec = NULL;  in lo_rw_aio_do_completion()
383 	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);  in lo_rw_aio_complete()  local
385 	cmd->ret = ret;  in lo_rw_aio_complete()
386 	lo_rw_aio_do_completion(cmd);  in lo_rw_aio_complete()
389 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,  in lo_rw_aio()  argument
395 	struct request *rq = blk_mq_rq_from_pdu(cmd);  in lo_rw_aio()
412 		cmd->bvec = bvec;  in lo_rw_aio()
424 		bvec = cmd->bvec;  in lo_rw_aio()
435 	atomic_set(&cmd->ref, 2);  in lo_rw_aio()
440 	cmd->iocb.ki_pos = pos;  in lo_rw_aio()
441 	cmd->iocb.ki_filp = file;  in lo_rw_aio()
442 	cmd->iocb.ki_complete = lo_rw_aio_complete;  in lo_rw_aio()
443 	cmd->iocb.ki_flags = IOCB_DIRECT;  in lo_rw_aio()
444 	cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);  in lo_rw_aio()
447 		ret = call_write_iter(file, &cmd->iocb, &iter);  in lo_rw_aio()
449 		ret = call_read_iter(file, &cmd->iocb, &iter);  in lo_rw_aio()
451 	lo_rw_aio_do_completion(cmd);  in lo_rw_aio()
454 		lo_rw_aio_complete(&cmd->iocb, ret);  in lo_rw_aio()
460 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);  in do_req_filebacked()  local
487 		if (cmd->use_aio)  in do_req_filebacked()
488 			return lo_rw_aio(lo, cmd, pos, ITER_SOURCE);  in do_req_filebacked()
492 		if (cmd->use_aio)  in do_req_filebacked()
493 			return lo_rw_aio(lo, cmd, pos, ITER_DEST);  in do_req_filebacked()
826 static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)  in loop_queue_work()  argument
835 	if (queue_on_root_worker(cmd->blkcg_css))  in loop_queue_work()
843 		if (cur_worker->blkcg_css == cmd->blkcg_css) {  in loop_queue_work()
846 		} else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) {  in loop_queue_work()
861 		cmd->blkcg_css = NULL;  in loop_queue_work()
862 		if (cmd->memcg_css)  in loop_queue_work()
863 			css_put(cmd->memcg_css);  in loop_queue_work()
864 		cmd->memcg_css = NULL;  in loop_queue_work()
868 	worker->blkcg_css = cmd->blkcg_css;  in loop_queue_work()
891 	list_add_tail(&cmd->list_entry, cmd_list);  in loop_queue_work()
1501 static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,  in lo_simple_ioctl()  argument
1509 	switch (cmd) {  in lo_simple_ioctl()
1527 	unsigned int cmd, unsigned long arg)  in lo_ioctl()  argument
1533 	switch (cmd) {  in lo_ioctl()
1580 		err = lo_simple_ioctl(lo, cmd, arg);  in lo_ioctl()
1688 			   unsigned int cmd, unsigned long arg)  in lo_compat_ioctl()  argument
1693 	switch(cmd) {  in lo_compat_ioctl()
1713 		err = lo_ioctl(bdev, mode, cmd, arg);  in lo_compat_ioctl()
1843 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);  in loop_queue_rq()  local
1855 		cmd->use_aio = false;  in loop_queue_rq()
1858 		cmd->use_aio = lo->use_dio;  in loop_queue_rq()
1863 	cmd->blkcg_css = NULL;  in loop_queue_rq()
1864 	cmd->memcg_css = NULL;  in loop_queue_rq()
1867 		cmd->blkcg_css = bio_blkcg_css(rq->bio);  in loop_queue_rq()
1869 		if (cmd->blkcg_css) {  in loop_queue_rq()
1870 			cmd->memcg_css =  in loop_queue_rq()
1871 				cgroup_get_e_css(cmd->blkcg_css->cgroup,  in loop_queue_rq()
1877 	loop_queue_work(lo, cmd);  in loop_queue_rq()
1882 static void loop_handle_cmd(struct loop_cmd *cmd)  in loop_handle_cmd()  argument
1884 	struct cgroup_subsys_state *cmd_blkcg_css = cmd->blkcg_css;  in loop_handle_cmd()
1885 	struct cgroup_subsys_state *cmd_memcg_css = cmd->memcg_css;  in loop_handle_cmd()
1886 	struct request *rq = blk_mq_rq_from_pdu(cmd);  in loop_handle_cmd()
1891 	const bool use_aio = cmd->use_aio;  in loop_handle_cmd()
1923 			cmd->ret = ret;  in loop_handle_cmd()
1925 			cmd->ret = ret ? -EIO : 0;  in loop_handle_cmd()
1935 	struct loop_cmd *cmd;  in loop_process_work()  local
1940 		cmd = container_of(  in loop_process_work()
1945 		loop_handle_cmd(cmd);  in loop_process_work()
2192 static long loop_control_ioctl(struct file *file, unsigned int cmd,  in loop_control_ioctl()  argument
2195 	switch (cmd) {  in loop_control_ioctl()