370 		struct vhost_scsi_cmd *cmd = container_of(se_cmd,  in vhost_scsi_release_cmd()  local
372 		struct vhost_scsi_virtqueue *svq =  container_of(cmd->tvc_vq,  in vhost_scsi_release_cmd()
375 		llist_add(&cmd->tvc_completion_list, &svq->completion_list);  in vhost_scsi_release_cmd()
523 static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)  in vhost_scsi_copy_sgl_to_iov()  argument
525 	struct iov_iter *iter = &cmd->saved_iter;  in vhost_scsi_copy_sgl_to_iov()
526 	struct scatterlist *sg = cmd->tvc_sgl;  in vhost_scsi_copy_sgl_to_iov()
531 	for (i = 0; i < cmd->tvc_sgl_count; i++) {  in vhost_scsi_copy_sgl_to_iov()
555 	struct vhost_scsi_cmd *cmd, *t;  in vhost_scsi_complete_cmd_work()  local
563 	llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {  in vhost_scsi_complete_cmd_work()
564 		se_cmd = &cmd->tvc_se_cmd;  in vhost_scsi_complete_cmd_work()
567 			cmd, se_cmd->residual_count, se_cmd->scsi_status);  in vhost_scsi_complete_cmd_work()
570 		if (cmd->saved_iter_addr && vhost_scsi_copy_sgl_to_iov(cmd)) {  in vhost_scsi_complete_cmd_work()
573 			v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq,  in vhost_scsi_complete_cmd_work()
577 			v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,  in vhost_scsi_complete_cmd_work()
579 			memcpy(v_rsp.sense, cmd->tvc_sense_buf,  in vhost_scsi_complete_cmd_work()
583 		iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iov,  in vhost_scsi_complete_cmd_work()
584 			      cmd->tvc_in_iovs, sizeof(v_rsp));  in vhost_scsi_complete_cmd_work()
589 			vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);  in vhost_scsi_complete_cmd_work()
607 	struct vhost_scsi_cmd *cmd;  in vhost_scsi_get_cmd()  local
626 	cmd = &svq->scsi_cmds[tag];  in vhost_scsi_get_cmd()
627 	sg = cmd->tvc_sgl;  in vhost_scsi_get_cmd()
628 	prot_sg = cmd->tvc_prot_sgl;  in vhost_scsi_get_cmd()
629 	pages = cmd->tvc_upages;  in vhost_scsi_get_cmd()
630 	tvc_resp_iov = cmd->tvc_resp_iov;  in vhost_scsi_get_cmd()
631 	memset(cmd, 0, sizeof(*cmd));  in vhost_scsi_get_cmd()
632 	cmd->tvc_sgl = sg;  in vhost_scsi_get_cmd()
633 	cmd->tvc_prot_sgl = prot_sg;  in vhost_scsi_get_cmd()
634 	cmd->tvc_upages = pages;  in vhost_scsi_get_cmd()
635 	cmd->tvc_se_cmd.map_tag = tag;  in vhost_scsi_get_cmd()
636 	cmd->tvc_tag = scsi_tag;  in vhost_scsi_get_cmd()
637 	cmd->tvc_lun = lun;  in vhost_scsi_get_cmd()
638 	cmd->tvc_task_attr = task_attr;  in vhost_scsi_get_cmd()
639 	cmd->tvc_exp_data_len = exp_data_len;  in vhost_scsi_get_cmd()
640 	cmd->tvc_data_direction = data_direction;  in vhost_scsi_get_cmd()
641 	cmd->tvc_nexus = tv_nexus;  in vhost_scsi_get_cmd()
642 	cmd->inflight = vhost_scsi_get_inflight(vq);  in vhost_scsi_get_cmd()
643 	cmd->tvc_resp_iov = tvc_resp_iov;  in vhost_scsi_get_cmd()
645 	memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);  in vhost_scsi_get_cmd()
647 	return cmd;  in vhost_scsi_get_cmd()
656 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,  in vhost_scsi_map_to_sgl()  argument
661 	struct page **pages = cmd->tvc_upages;  in vhost_scsi_map_to_sgl()
692 		    cmd->tvc_sgl_count > BIO_MAX_VECS) {  in vhost_scsi_map_to_sgl()
743 vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,  in vhost_scsi_copy_iov_to_sgl()  argument
751 	if (cmd->tvc_data_direction == DMA_FROM_DEVICE) {  in vhost_scsi_copy_iov_to_sgl()
752 		cmd->saved_iter_addr = dup_iter(&cmd->saved_iter, iter,  in vhost_scsi_copy_iov_to_sgl()
754 		if (!cmd->saved_iter_addr)  in vhost_scsi_copy_iov_to_sgl()
768 		if (cmd->tvc_data_direction == DMA_TO_DEVICE &&  in vhost_scsi_copy_iov_to_sgl()
775 	cmd->copied_iov = 1;  in vhost_scsi_copy_iov_to_sgl()
784 	kfree(cmd->saved_iter_addr);  in vhost_scsi_copy_iov_to_sgl()
789 vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,  in vhost_scsi_map_iov_to_sgl()  argument
797 		ret = vhost_scsi_map_to_sgl(cmd, iter, sg, is_prot);  in vhost_scsi_map_iov_to_sgl()
821 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,  in vhost_scsi_mapal()  argument
833 		sg_init_table(cmd->tvc_prot_sgl, sgl_count);  in vhost_scsi_mapal()
834 		cmd->tvc_prot_sgl_count = sgl_count;  in vhost_scsi_mapal()
836 			 cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);  in vhost_scsi_mapal()
838 		ret = vhost_scsi_map_iov_to_sgl(cmd, prot_iter,  in vhost_scsi_mapal()
839 						cmd->tvc_prot_sgl,  in vhost_scsi_mapal()
840 						cmd->tvc_prot_sgl_count, true);  in vhost_scsi_mapal()
842 			cmd->tvc_prot_sgl_count = 0;  in vhost_scsi_mapal()
851 	sg_init_table(cmd->tvc_sgl, sgl_count);  in vhost_scsi_mapal()
852 	cmd->tvc_sgl_count = sgl_count;  in vhost_scsi_mapal()
854 		  cmd->tvc_sgl, cmd->tvc_sgl_count);  in vhost_scsi_mapal()
856 	ret = vhost_scsi_map_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,  in vhost_scsi_mapal()
857 					cmd->tvc_sgl_count, false);  in vhost_scsi_mapal()
859 		sg_init_table(cmd->tvc_sgl, cmd->tvc_sgl_count);  in vhost_scsi_mapal()
860 		ret = vhost_scsi_copy_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,  in vhost_scsi_mapal()
861 						 cmd->tvc_sgl_count);  in vhost_scsi_mapal()
865 		cmd->tvc_sgl_count = 0;  in vhost_scsi_mapal()
888 static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)  in vhost_scsi_target_queue_cmd()  argument
890 	struct se_cmd *se_cmd = &cmd->tvc_se_cmd;  in vhost_scsi_target_queue_cmd()
895 	if (cmd->tvc_sgl_count) {  in vhost_scsi_target_queue_cmd()
896 		sg_ptr = cmd->tvc_sgl;  in vhost_scsi_target_queue_cmd()
898 		if (cmd->tvc_prot_sgl_count)  in vhost_scsi_target_queue_cmd()
899 			sg_prot_ptr = cmd->tvc_prot_sgl;  in vhost_scsi_target_queue_cmd()
905 	tv_nexus = cmd->tvc_nexus;  in vhost_scsi_target_queue_cmd()
908 	target_init_cmd(se_cmd, tv_nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],  in vhost_scsi_target_queue_cmd()
909 			cmd->tvc_lun, cmd->tvc_exp_data_len,  in vhost_scsi_target_queue_cmd()
910 			vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),  in vhost_scsi_target_queue_cmd()
911 			cmd->tvc_data_direction, TARGET_SCF_ACK_KREF);  in vhost_scsi_target_queue_cmd()
913 	if (target_submit_prep(se_cmd, cmd->tvc_cdb, sg_ptr,  in vhost_scsi_target_queue_cmd()
914 			       cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,  in vhost_scsi_target_queue_cmd()
915 			       cmd->tvc_prot_sgl_count, GFP_KERNEL))  in vhost_scsi_target_queue_cmd()
1053 	struct vhost_scsi_cmd *cmd;  in vhost_scsi_handle_vq()  local
1204 		cmd = vhost_scsi_get_cmd(vq, tpg, cdb, tag, lun, task_attr,  in vhost_scsi_handle_vq()
1207 		if (IS_ERR(cmd)) {  in vhost_scsi_handle_vq()
1209 			       PTR_ERR(cmd));  in vhost_scsi_handle_vq()
1212 		cmd->tvc_vhost = vs;  in vhost_scsi_handle_vq()
1213 		cmd->tvc_vq = vq;  in vhost_scsi_handle_vq()
1215 			cmd->tvc_resp_iov[i] = vq->iov[vc.out + i];  in vhost_scsi_handle_vq()
1216 		cmd->tvc_in_iovs = vc.in;  in vhost_scsi_handle_vq()
1219 			 cmd->tvc_cdb[0], cmd->tvc_lun);  in vhost_scsi_handle_vq()
1221 			 " %d\n", cmd, exp_data_len, prot_bytes, data_direction);  in vhost_scsi_handle_vq()
1224 			if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,  in vhost_scsi_handle_vq()
1228 				vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);  in vhost_scsi_handle_vq()
1237 		cmd->tvc_vq_desc = vc.head;  in vhost_scsi_handle_vq()
1238 		vhost_scsi_target_queue_cmd(cmd);  in vhost_scsi_handle_vq()