463 	struct smb_direct_sendmsg *msg;  in smb_direct_alloc_sendmsg()  local
465 	msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL);  in smb_direct_alloc_sendmsg()
466 	if (!msg)  in smb_direct_alloc_sendmsg()
468 	msg->transport = t;  in smb_direct_alloc_sendmsg()
469 	INIT_LIST_HEAD(&msg->list);  in smb_direct_alloc_sendmsg()
470 	msg->num_sge = 0;  in smb_direct_alloc_sendmsg()
471 	return msg;  in smb_direct_alloc_sendmsg()
475 				    struct smb_direct_sendmsg *msg)  in smb_direct_free_sendmsg()  argument
479 	if (msg->num_sge > 0) {  in smb_direct_free_sendmsg()
481 				    msg->sge[0].addr, msg->sge[0].length,  in smb_direct_free_sendmsg()
483 		for (i = 1; i < msg->num_sge; i++)  in smb_direct_free_sendmsg()
485 					  msg->sge[i].addr, msg->sge[i].length,  in smb_direct_free_sendmsg()
488 	mempool_free(msg, t->sendmsg_mempool);  in smb_direct_free_sendmsg()
1125 			struct smb_direct_sendmsg *msg)  in post_sendmsg()  argument
1129 	for (i = 0; i < msg->num_sge; i++)  in post_sendmsg()
1131 					      msg->sge[i].addr, msg->sge[i].length,  in post_sendmsg()
1134 	msg->cqe.done = send_done;  in post_sendmsg()
1135 	msg->wr.opcode = IB_WR_SEND;  in post_sendmsg()
1136 	msg->wr.sg_list = &msg->sge[0];  in post_sendmsg()
1137 	msg->wr.num_sge = msg->num_sge;  in post_sendmsg()
1138 	msg->wr.next = NULL;  in post_sendmsg()
1141 		msg->wr.wr_cqe = NULL;  in post_sendmsg()
1142 		msg->wr.send_flags = 0;  in post_sendmsg()
1149 			last->wr.next = &msg->wr;  in post_sendmsg()
1151 		list_add_tail(&msg->list, &send_ctx->msg_list);  in post_sendmsg()
1156 	msg->wr.wr_cqe = &msg->cqe;  in post_sendmsg()
1157 	msg->wr.send_flags = IB_SEND_SIGNALED;  in post_sendmsg()
1158 	return smb_direct_post_send(t, &msg->wr);  in post_sendmsg()
1167 	struct smb_direct_sendmsg *msg;  in smb_direct_post_send_data()  local
1180 				       &msg);  in smb_direct_post_send_data()
1199 		} else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) {  in smb_direct_post_send_data()
1208 			sge = &msg->sge[msg->num_sge];  in smb_direct_post_send_data()
1212 			msg->num_sge++;  in smb_direct_post_send_data()
1216 	ret = post_sendmsg(t, send_ctx, msg);  in smb_direct_post_send_data()
1221 	smb_direct_free_sendmsg(t, msg);  in smb_direct_post_send_data()
1317 					struct smb_direct_rdma_rw_msg *msg,  in smb_direct_free_rdma_rw_msg()  argument
1320 	rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port,  in smb_direct_free_rdma_rw_msg()
1321 			    msg->sgt.sgl, msg->sgt.nents, dir);  in smb_direct_free_rdma_rw_msg()
1322 	sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);  in smb_direct_free_rdma_rw_msg()
1323 	kfree(msg);  in smb_direct_free_rdma_rw_msg()
1329 	struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe,  in read_write_done()  local
1331 	struct smb_direct_transport *t = msg->t;  in read_write_done()
1334 		msg->status = -EIO;  in read_write_done()
1341 	complete(msg->completion);  in read_write_done()
1360 	struct smb_direct_rdma_rw_msg *msg, *next_msg;  in smb_direct_rdma_xmit()  local
1408 		msg = kzalloc(offsetof(struct smb_direct_rdma_rw_msg, sg_list) +  in smb_direct_rdma_xmit()
1410 		if (!msg) {  in smb_direct_rdma_xmit()
1417 		msg->t = t;  in smb_direct_rdma_xmit()
1418 		msg->cqe.done = is_read ? read_done : write_done;  in smb_direct_rdma_xmit()
1419 		msg->completion = &completion;  in smb_direct_rdma_xmit()
1421 		msg->sgt.sgl = &msg->sg_list[0];  in smb_direct_rdma_xmit()
1422 		ret = sg_alloc_table_chained(&msg->sgt,  in smb_direct_rdma_xmit()
1424 					     msg->sg_list, SG_CHUNK_SIZE);  in smb_direct_rdma_xmit()
1426 			kfree(msg);  in smb_direct_rdma_xmit()
1432 				  msg->sgt.sgl, msg->sgt.orig_nents);  in smb_direct_rdma_xmit()
1434 			sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);  in smb_direct_rdma_xmit()
1435 			kfree(msg);  in smb_direct_rdma_xmit()
1439 		ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port,  in smb_direct_rdma_xmit()
1440 				       msg->sgt.sgl,  in smb_direct_rdma_xmit()
1448 			sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE);  in smb_direct_rdma_xmit()
1449 			kfree(msg);  in smb_direct_rdma_xmit()
1453 		list_add_tail(&msg->list, &msg_list);  in smb_direct_rdma_xmit()
1459 	list_for_each_entry_reverse(msg, &msg_list, list) {  in smb_direct_rdma_xmit()
1460 		first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port,  in smb_direct_rdma_xmit()
1461 					   &msg->cqe, first_wr);  in smb_direct_rdma_xmit()
1470 	msg = list_last_entry(&msg_list, struct smb_direct_rdma_rw_msg, list);  in smb_direct_rdma_xmit()
1472 	ret = msg->status;  in smb_direct_rdma_xmit()
1474 	list_for_each_entry_safe(msg, next_msg, &msg_list, list) {  in smb_direct_rdma_xmit()
1475 		list_del(&msg->list);  in smb_direct_rdma_xmit()
1476 		smb_direct_free_rdma_rw_msg(t, msg,  in smb_direct_rdma_xmit()