706 	struct ffs_io_data *io_data = req->context;  in ffs_epfile_io_complete()  local
709 		io_data->status = req->status;  in ffs_epfile_io_complete()
711 		io_data->status = req->actual;  in ffs_epfile_io_complete()
713 	complete(&io_data->done);  in ffs_epfile_io_complete()
796 static inline void *ffs_alloc_buffer(struct ffs_io_data *io_data,  in ffs_alloc_buffer()  argument
799 	if (io_data->use_sg)  in ffs_alloc_buffer()
800 		return ffs_build_sg_list(&io_data->sgt, data_len);  in ffs_alloc_buffer()
805 static inline void ffs_free_buffer(struct ffs_io_data *io_data)  in ffs_free_buffer()  argument
807 	if (!io_data->buf)  in ffs_free_buffer()
810 	if (io_data->use_sg) {  in ffs_free_buffer()
811 		sg_free_table(&io_data->sgt);  in ffs_free_buffer()
812 		vfree(io_data->buf);  in ffs_free_buffer()
814 		kfree(io_data->buf);  in ffs_free_buffer()
820 	struct ffs_io_data *io_data = container_of(work, struct ffs_io_data,  in ffs_user_copy_worker()  local
822 	int ret = io_data->status;  in ffs_user_copy_worker()
823 	bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD;  in ffs_user_copy_worker()
826 	if (io_data->read && ret > 0) {  in ffs_user_copy_worker()
827 		kthread_use_mm(io_data->mm);  in ffs_user_copy_worker()
828 		ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data);  in ffs_user_copy_worker()
829 		kthread_unuse_mm(io_data->mm);  in ffs_user_copy_worker()
832 	io_data->kiocb->ki_complete(io_data->kiocb, ret);  in ffs_user_copy_worker()
834 	if (io_data->ffs->ffs_eventfd && !kiocb_has_eventfd)  in ffs_user_copy_worker()
835 		eventfd_signal(io_data->ffs->ffs_eventfd, 1);  in ffs_user_copy_worker()
837 	spin_lock_irqsave(&io_data->ffs->eps_lock, flags);  in ffs_user_copy_worker()
838 	usb_ep_free_request(io_data->ep, io_data->req);  in ffs_user_copy_worker()
839 	io_data->req = NULL;  in ffs_user_copy_worker()
840 	spin_unlock_irqrestore(&io_data->ffs->eps_lock, flags);  in ffs_user_copy_worker()
842 	if (io_data->read)  in ffs_user_copy_worker()
843 		kfree(io_data->to_free);  in ffs_user_copy_worker()
844 	ffs_free_buffer(io_data);  in ffs_user_copy_worker()
845 	kfree(io_data);  in ffs_user_copy_worker()
851 	struct ffs_io_data *io_data = req->context;  in ffs_epfile_async_io_complete()  local
852 	struct ffs_data *ffs = io_data->ffs;  in ffs_epfile_async_io_complete()
854 	io_data->status = req->status ? req->status : req->actual;  in ffs_epfile_async_io_complete()
856 	INIT_WORK(&io_data->work, ffs_user_copy_worker);  in ffs_epfile_async_io_complete()
857 	queue_work(ffs->io_completion_wq, &io_data->work);  in ffs_epfile_async_io_complete()
942 static ssize_t ffs_epfile_io(struct file *file, struct ffs_io_data *io_data)  in ffs_epfile_io()  argument
968 	halt = (!io_data->read == !epfile->in);  in ffs_epfile_io()
988 		if (!io_data->aio && io_data->read) {  in ffs_epfile_io()
989 			ret = __ffs_epfile_read_buffered(epfile, &io_data->data);  in ffs_epfile_io()
1007 		data_len = iov_iter_count(&io_data->data);  in ffs_epfile_io()
1012 		if (io_data->read)  in ffs_epfile_io()
1015 		io_data->use_sg = gadget->sg_supported && data_len > PAGE_SIZE;  in ffs_epfile_io()
1018 		data = ffs_alloc_buffer(io_data, data_len);  in ffs_epfile_io()
1023 		if (!io_data->read &&  in ffs_epfile_io()
1024 		    !copy_from_iter_full(data, data_len, &io_data->data)) {  in ffs_epfile_io()
1053 	} else if (!io_data->aio) {  in ffs_epfile_io()
1057 		if (io_data->use_sg) {  in ffs_epfile_io()
1059 			req->sg	= io_data->sgt.sgl;  in ffs_epfile_io()
1060 			req->num_sgs = io_data->sgt.nents;  in ffs_epfile_io()
1067 		io_data->buf = data;  in ffs_epfile_io()
1069 		init_completion(&io_data->done);  in ffs_epfile_io()
1070 		req->context  = io_data;  in ffs_epfile_io()
1079 		if (wait_for_completion_interruptible(&io_data->done)) {  in ffs_epfile_io()
1093 			wait_for_completion(&io_data->done);  in ffs_epfile_io()
1094 			interrupted = io_data->status < 0;  in ffs_epfile_io()
1099 		else if (io_data->read && io_data->status > 0)  in ffs_epfile_io()
1100 			ret = __ffs_epfile_read_data(epfile, data, io_data->status,  in ffs_epfile_io()
1101 						     &io_data->data);  in ffs_epfile_io()
1103 			ret = io_data->status;  in ffs_epfile_io()
1108 		if (io_data->use_sg) {  in ffs_epfile_io()
1110 			req->sg	= io_data->sgt.sgl;  in ffs_epfile_io()
1111 			req->num_sgs = io_data->sgt.nents;  in ffs_epfile_io()
1118 		io_data->buf = data;  in ffs_epfile_io()
1119 		io_data->ep = ep->ep;  in ffs_epfile_io()
1120 		io_data->req = req;  in ffs_epfile_io()
1121 		io_data->ffs = epfile->ffs;  in ffs_epfile_io()
1123 		req->context  = io_data;  in ffs_epfile_io()
1128 			io_data->req = NULL;  in ffs_epfile_io()
1147 		ffs_free_buffer(io_data);  in ffs_epfile_io()
1167 	struct ffs_io_data *io_data = kiocb->private;  in ffs_aio_cancel()  local
1174 	if (io_data && io_data->ep && io_data->req)  in ffs_aio_cancel()
1175 		value = usb_ep_dequeue(io_data->ep, io_data->req);  in ffs_aio_cancel()
1186 	struct ffs_io_data io_data, *p = &io_data;  in ffs_epfile_write_iter()  local
1190 		p = kzalloc(sizeof(io_data), GFP_KERNEL);  in ffs_epfile_write_iter()
1221 	struct ffs_io_data io_data, *p = &io_data;  in ffs_epfile_read_iter()  local
1225 		p = kzalloc(sizeof(io_data), GFP_KERNEL);  in ffs_epfile_read_iter()