1b9be6f18SBernard Metzler // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2b9be6f18SBernard Metzler
3b9be6f18SBernard Metzler /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4b9be6f18SBernard Metzler /* Copyright (c) 2008-2019, IBM Corporation */
5b9be6f18SBernard Metzler
6b9be6f18SBernard Metzler #include <linux/errno.h>
7b9be6f18SBernard Metzler #include <linux/types.h>
8b9be6f18SBernard Metzler #include <linux/net.h>
9b9be6f18SBernard Metzler #include <linux/scatterlist.h>
10b9be6f18SBernard Metzler #include <linux/highmem.h>
11b9be6f18SBernard Metzler #include <net/tcp.h>
12b9be6f18SBernard Metzler
13b9be6f18SBernard Metzler #include <rdma/iw_cm.h>
14b9be6f18SBernard Metzler #include <rdma/ib_verbs.h>
15b9be6f18SBernard Metzler #include <rdma/ib_user_verbs.h>
16b9be6f18SBernard Metzler
17b9be6f18SBernard Metzler #include "siw.h"
18b9be6f18SBernard Metzler #include "siw_verbs.h"
19b9be6f18SBernard Metzler #include "siw_mem.h"
20b9be6f18SBernard Metzler
21b9be6f18SBernard Metzler #define MAX_HDR_INLINE \
22b9be6f18SBernard Metzler (((uint32_t)(sizeof(struct siw_rreq_pkt) - \
23b9be6f18SBernard Metzler sizeof(struct iwarp_send))) & 0xF8)
24b9be6f18SBernard Metzler
siw_get_pblpage(struct siw_mem * mem,u64 addr,int * idx)25b9be6f18SBernard Metzler static struct page *siw_get_pblpage(struct siw_mem *mem, u64 addr, int *idx)
26b9be6f18SBernard Metzler {
27b9be6f18SBernard Metzler struct siw_pbl *pbl = mem->pbl;
28b9be6f18SBernard Metzler u64 offset = addr - mem->va;
29c536277eSBernard Metzler dma_addr_t paddr = siw_pbl_get_buffer(pbl, offset, NULL, idx);
30b9be6f18SBernard Metzler
31b9be6f18SBernard Metzler if (paddr)
328d7c7c0eSJason Gunthorpe return ib_virt_dma_to_page(paddr);
33b9be6f18SBernard Metzler
34b9be6f18SBernard Metzler return NULL;
35b9be6f18SBernard Metzler }
36b9be6f18SBernard Metzler
37b9be6f18SBernard Metzler /*
38b9be6f18SBernard Metzler * Copy short payload at provided destination payload address
39b9be6f18SBernard Metzler */
siw_try_1seg(struct siw_iwarp_tx * c_tx,void * paddr)40c536277eSBernard Metzler static int siw_try_1seg(struct siw_iwarp_tx *c_tx, void *paddr)
41b9be6f18SBernard Metzler {
42b9be6f18SBernard Metzler struct siw_wqe *wqe = &c_tx->wqe_active;
43b9be6f18SBernard Metzler struct siw_sge *sge = &wqe->sqe.sge[0];
44b9be6f18SBernard Metzler u32 bytes = sge->length;
45b9be6f18SBernard Metzler
46b9be6f18SBernard Metzler if (bytes > MAX_HDR_INLINE || wqe->sqe.num_sge != 1)
47b9be6f18SBernard Metzler return MAX_HDR_INLINE + 1;
48b9be6f18SBernard Metzler
49b9be6f18SBernard Metzler if (!bytes)
50b9be6f18SBernard Metzler return 0;
51b9be6f18SBernard Metzler
52b9be6f18SBernard Metzler if (tx_flags(wqe) & SIW_WQE_INLINE) {
53c536277eSBernard Metzler memcpy(paddr, &wqe->sqe.sge[1], bytes);
54b9be6f18SBernard Metzler } else {
55b9be6f18SBernard Metzler struct siw_mem *mem = wqe->mem[0];
56b9be6f18SBernard Metzler
57b9be6f18SBernard Metzler if (!mem->mem_obj) {
58b9be6f18SBernard Metzler /* Kernel client using kva */
598d7c7c0eSJason Gunthorpe memcpy(paddr, ib_virt_dma_to_ptr(sge->laddr), bytes);
60b9be6f18SBernard Metzler } else if (c_tx->in_syscall) {
61c536277eSBernard Metzler if (copy_from_user(paddr, u64_to_user_ptr(sge->laddr),
62b9be6f18SBernard Metzler bytes))
63b9be6f18SBernard Metzler return -EFAULT;
64b9be6f18SBernard Metzler } else {
65b9be6f18SBernard Metzler unsigned int off = sge->laddr & ~PAGE_MASK;
66b9be6f18SBernard Metzler struct page *p;
67b9be6f18SBernard Metzler char *buffer;
68b9be6f18SBernard Metzler int pbl_idx = 0;
69b9be6f18SBernard Metzler
70b9be6f18SBernard Metzler if (!mem->is_pbl)
71b9be6f18SBernard Metzler p = siw_get_upage(mem->umem, sge->laddr);
72b9be6f18SBernard Metzler else
73b9be6f18SBernard Metzler p = siw_get_pblpage(mem, sge->laddr, &pbl_idx);
74b9be6f18SBernard Metzler
75b9be6f18SBernard Metzler if (unlikely(!p))
76b9be6f18SBernard Metzler return -EFAULT;
77b9be6f18SBernard Metzler
781ec50dd1SIra Weiny buffer = kmap_local_page(p);
79b9be6f18SBernard Metzler
80b9be6f18SBernard Metzler if (likely(PAGE_SIZE - off >= bytes)) {
81c536277eSBernard Metzler memcpy(paddr, buffer + off, bytes);
82b9be6f18SBernard Metzler } else {
83b9be6f18SBernard Metzler unsigned long part = bytes - (PAGE_SIZE - off);
84b9be6f18SBernard Metzler
85c536277eSBernard Metzler memcpy(paddr, buffer + off, part);
861ec50dd1SIra Weiny kunmap_local(buffer);
87b9be6f18SBernard Metzler
88b9be6f18SBernard Metzler if (!mem->is_pbl)
89b9be6f18SBernard Metzler p = siw_get_upage(mem->umem,
90b9be6f18SBernard Metzler sge->laddr + part);
91b9be6f18SBernard Metzler else
92b9be6f18SBernard Metzler p = siw_get_pblpage(mem,
93b9be6f18SBernard Metzler sge->laddr + part,
94b9be6f18SBernard Metzler &pbl_idx);
95b9be6f18SBernard Metzler if (unlikely(!p))
96b9be6f18SBernard Metzler return -EFAULT;
97b9be6f18SBernard Metzler
981ec50dd1SIra Weiny buffer = kmap_local_page(p);
994db8fd49SBernard Metzler memcpy(paddr + part, buffer, bytes - part);
100b9be6f18SBernard Metzler }
1011ec50dd1SIra Weiny kunmap_local(buffer);
102b9be6f18SBernard Metzler }
103b9be6f18SBernard Metzler }
104b9be6f18SBernard Metzler return (int)bytes;
105b9be6f18SBernard Metzler }
106b9be6f18SBernard Metzler
107b9be6f18SBernard Metzler #define PKT_FRAGMENTED 1
108b9be6f18SBernard Metzler #define PKT_COMPLETE 0
109b9be6f18SBernard Metzler
110b9be6f18SBernard Metzler /*
111b9be6f18SBernard Metzler * siw_qp_prepare_tx()
112b9be6f18SBernard Metzler *
113b9be6f18SBernard Metzler * Prepare tx state for sending out one fpdu. Builds complete pkt
114b9be6f18SBernard Metzler * if no user data or only immediate data are present.
115b9be6f18SBernard Metzler *
116b9be6f18SBernard Metzler * returns PKT_COMPLETE if complete pkt built, PKT_FRAGMENTED otherwise.
117b9be6f18SBernard Metzler */
siw_qp_prepare_tx(struct siw_iwarp_tx * c_tx)118b9be6f18SBernard Metzler static int siw_qp_prepare_tx(struct siw_iwarp_tx *c_tx)
119b9be6f18SBernard Metzler {
120b9be6f18SBernard Metzler struct siw_wqe *wqe = &c_tx->wqe_active;
121b9be6f18SBernard Metzler char *crc = NULL;
122b9be6f18SBernard Metzler int data = 0;
123b9be6f18SBernard Metzler
124b9be6f18SBernard Metzler switch (tx_type(wqe)) {
125b9be6f18SBernard Metzler case SIW_OP_READ:
126b9be6f18SBernard Metzler case SIW_OP_READ_LOCAL_INV:
127b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl,
128b9be6f18SBernard Metzler &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
129b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
130b9be6f18SBernard Metzler
131b9be6f18SBernard Metzler c_tx->pkt.rreq.rsvd = 0;
132b9be6f18SBernard Metzler c_tx->pkt.rreq.ddp_qn = htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
133b9be6f18SBernard Metzler c_tx->pkt.rreq.ddp_msn =
134b9be6f18SBernard Metzler htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ]);
135b9be6f18SBernard Metzler c_tx->pkt.rreq.ddp_mo = 0;
136b9be6f18SBernard Metzler c_tx->pkt.rreq.sink_stag = htonl(wqe->sqe.sge[0].lkey);
137b9be6f18SBernard Metzler c_tx->pkt.rreq.sink_to =
138b9be6f18SBernard Metzler cpu_to_be64(wqe->sqe.sge[0].laddr);
139b9be6f18SBernard Metzler c_tx->pkt.rreq.source_stag = htonl(wqe->sqe.rkey);
140b9be6f18SBernard Metzler c_tx->pkt.rreq.source_to = cpu_to_be64(wqe->sqe.raddr);
141b9be6f18SBernard Metzler c_tx->pkt.rreq.read_size = htonl(wqe->sqe.sge[0].length);
142b9be6f18SBernard Metzler
143b9be6f18SBernard Metzler c_tx->ctrl_len = sizeof(struct iwarp_rdma_rreq);
144b9be6f18SBernard Metzler crc = (char *)&c_tx->pkt.rreq_pkt.crc;
145b9be6f18SBernard Metzler break;
146b9be6f18SBernard Metzler
147b9be6f18SBernard Metzler case SIW_OP_SEND:
148b9be6f18SBernard Metzler if (tx_flags(wqe) & SIW_WQE_SOLICITED)
149b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl,
150b9be6f18SBernard Metzler &iwarp_pktinfo[RDMAP_SEND_SE].ctrl,
151b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
152b9be6f18SBernard Metzler else
153b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_SEND].ctrl,
154b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
155b9be6f18SBernard Metzler
156b9be6f18SBernard Metzler c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;
157b9be6f18SBernard Metzler c_tx->pkt.send.ddp_msn =
158b9be6f18SBernard Metzler htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);
159b9be6f18SBernard Metzler c_tx->pkt.send.ddp_mo = 0;
160b9be6f18SBernard Metzler
161b9be6f18SBernard Metzler c_tx->pkt.send_inv.inval_stag = 0;
162b9be6f18SBernard Metzler
163b9be6f18SBernard Metzler c_tx->ctrl_len = sizeof(struct iwarp_send);
164b9be6f18SBernard Metzler
165b9be6f18SBernard Metzler crc = (char *)&c_tx->pkt.send_pkt.crc;
166c536277eSBernard Metzler data = siw_try_1seg(c_tx, crc);
167b9be6f18SBernard Metzler break;
168b9be6f18SBernard Metzler
169b9be6f18SBernard Metzler case SIW_OP_SEND_REMOTE_INV:
170b9be6f18SBernard Metzler if (tx_flags(wqe) & SIW_WQE_SOLICITED)
171b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl,
172b9be6f18SBernard Metzler &iwarp_pktinfo[RDMAP_SEND_SE_INVAL].ctrl,
173b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
174b9be6f18SBernard Metzler else
175b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl,
176b9be6f18SBernard Metzler &iwarp_pktinfo[RDMAP_SEND_INVAL].ctrl,
177b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
178b9be6f18SBernard Metzler
179b9be6f18SBernard Metzler c_tx->pkt.send.ddp_qn = RDMAP_UNTAGGED_QN_SEND;
180b9be6f18SBernard Metzler c_tx->pkt.send.ddp_msn =
181b9be6f18SBernard Metzler htonl(++c_tx->ddp_msn[RDMAP_UNTAGGED_QN_SEND]);
182b9be6f18SBernard Metzler c_tx->pkt.send.ddp_mo = 0;
183b9be6f18SBernard Metzler
184b9be6f18SBernard Metzler c_tx->pkt.send_inv.inval_stag = cpu_to_be32(wqe->sqe.rkey);
185b9be6f18SBernard Metzler
186b9be6f18SBernard Metzler c_tx->ctrl_len = sizeof(struct iwarp_send_inv);
187b9be6f18SBernard Metzler
188b9be6f18SBernard Metzler crc = (char *)&c_tx->pkt.send_pkt.crc;
189c536277eSBernard Metzler data = siw_try_1seg(c_tx, crc);
190b9be6f18SBernard Metzler break;
191b9be6f18SBernard Metzler
192b9be6f18SBernard Metzler case SIW_OP_WRITE:
193b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl, &iwarp_pktinfo[RDMAP_RDMA_WRITE].ctrl,
194b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
195b9be6f18SBernard Metzler
196b9be6f18SBernard Metzler c_tx->pkt.rwrite.sink_stag = htonl(wqe->sqe.rkey);
197b9be6f18SBernard Metzler c_tx->pkt.rwrite.sink_to = cpu_to_be64(wqe->sqe.raddr);
198b9be6f18SBernard Metzler c_tx->ctrl_len = sizeof(struct iwarp_rdma_write);
199b9be6f18SBernard Metzler
200b9be6f18SBernard Metzler crc = (char *)&c_tx->pkt.write_pkt.crc;
201c536277eSBernard Metzler data = siw_try_1seg(c_tx, crc);
202b9be6f18SBernard Metzler break;
203b9be6f18SBernard Metzler
204b9be6f18SBernard Metzler case SIW_OP_READ_RESPONSE:
205b9be6f18SBernard Metzler memcpy(&c_tx->pkt.ctrl,
206b9be6f18SBernard Metzler &iwarp_pktinfo[RDMAP_RDMA_READ_RESP].ctrl,
207b9be6f18SBernard Metzler sizeof(struct iwarp_ctrl));
208b9be6f18SBernard Metzler
209b9be6f18SBernard Metzler /* NBO */
210b9be6f18SBernard Metzler c_tx->pkt.rresp.sink_stag = cpu_to_be32(wqe->sqe.rkey);
211b9be6f18SBernard Metzler c_tx->pkt.rresp.sink_to = cpu_to_be64(wqe->sqe.raddr);
212b9be6f18SBernard Metzler
213b9be6f18SBernard Metzler c_tx->ctrl_len = sizeof(struct iwarp_rdma_rresp);
214b9be6f18SBernard Metzler
215b9be6f18SBernard Metzler crc = (char *)&c_tx->pkt.write_pkt.crc;
216c536277eSBernard Metzler data = siw_try_1seg(c_tx, crc);
217b9be6f18SBernard Metzler break;
218b9be6f18SBernard Metzler
219b9be6f18SBernard Metzler default:
220b9be6f18SBernard Metzler siw_dbg_qp(tx_qp(c_tx), "stale wqe type %d\n", tx_type(wqe));
221b9be6f18SBernard Metzler return -EOPNOTSUPP;
222b9be6f18SBernard Metzler }
223b9be6f18SBernard Metzler if (unlikely(data < 0))
224b9be6f18SBernard Metzler return data;
225b9be6f18SBernard Metzler
226b9be6f18SBernard Metzler c_tx->ctrl_sent = 0;
227b9be6f18SBernard Metzler
228b9be6f18SBernard Metzler if (data <= MAX_HDR_INLINE) {
229b9be6f18SBernard Metzler if (data) {
230b9be6f18SBernard Metzler wqe->processed = data;
231b9be6f18SBernard Metzler
232b9be6f18SBernard Metzler c_tx->pkt.ctrl.mpa_len =
233b9be6f18SBernard Metzler htons(c_tx->ctrl_len + data - MPA_HDR_SIZE);
234b9be6f18SBernard Metzler
235b9be6f18SBernard Metzler /* Add pad, if needed */
236b9be6f18SBernard Metzler data += -(int)data & 0x3;
237b9be6f18SBernard Metzler /* advance CRC location after payload */
238b9be6f18SBernard Metzler crc += data;
239b9be6f18SBernard Metzler c_tx->ctrl_len += data;
240b9be6f18SBernard Metzler
241b9be6f18SBernard Metzler if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))
242b9be6f18SBernard Metzler c_tx->pkt.c_untagged.ddp_mo = 0;
243b9be6f18SBernard Metzler else
244b9be6f18SBernard Metzler c_tx->pkt.c_tagged.ddp_to =
245b9be6f18SBernard Metzler cpu_to_be64(wqe->sqe.raddr);
246b9be6f18SBernard Metzler }
247b9be6f18SBernard Metzler
248b9be6f18SBernard Metzler *(u32 *)crc = 0;
249b9be6f18SBernard Metzler /*
250b9be6f18SBernard Metzler * Do complete CRC if enabled and short packet
251b9be6f18SBernard Metzler */
252b9be6f18SBernard Metzler if (c_tx->mpa_crc_hd) {
253b9be6f18SBernard Metzler crypto_shash_init(c_tx->mpa_crc_hd);
254b9be6f18SBernard Metzler if (crypto_shash_update(c_tx->mpa_crc_hd,
255b9be6f18SBernard Metzler (u8 *)&c_tx->pkt,
256b9be6f18SBernard Metzler c_tx->ctrl_len))
257b9be6f18SBernard Metzler return -EINVAL;
258b9be6f18SBernard Metzler crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)crc);
259b9be6f18SBernard Metzler }
260b9be6f18SBernard Metzler c_tx->ctrl_len += MPA_CRC_SIZE;
261b9be6f18SBernard Metzler
262b9be6f18SBernard Metzler return PKT_COMPLETE;
263b9be6f18SBernard Metzler }
264b9be6f18SBernard Metzler c_tx->ctrl_len += MPA_CRC_SIZE;
265b9be6f18SBernard Metzler c_tx->sge_idx = 0;
266b9be6f18SBernard Metzler c_tx->sge_off = 0;
267b9be6f18SBernard Metzler c_tx->pbl_idx = 0;
268b9be6f18SBernard Metzler
269b9be6f18SBernard Metzler /*
270b9be6f18SBernard Metzler * Allow direct sending out of user buffer if WR is non signalled
271b9be6f18SBernard Metzler * and payload is over threshold.
272b9be6f18SBernard Metzler * Per RDMA verbs, the application should not change the send buffer
273b9be6f18SBernard Metzler * until the work completed. In iWarp, work completion is only
274b9be6f18SBernard Metzler * local delivery to TCP. TCP may reuse the buffer for
275b9be6f18SBernard Metzler * retransmission. Changing unsent data also breaks the CRC,
276b9be6f18SBernard Metzler * if applied.
277b9be6f18SBernard Metzler */
278b9be6f18SBernard Metzler if (c_tx->zcopy_tx && wqe->bytes >= SENDPAGE_THRESH &&
279b9be6f18SBernard Metzler !(tx_flags(wqe) & SIW_WQE_SIGNALLED))
280b9be6f18SBernard Metzler c_tx->use_sendpage = 1;
281b9be6f18SBernard Metzler else
282b9be6f18SBernard Metzler c_tx->use_sendpage = 0;
283b9be6f18SBernard Metzler
284b9be6f18SBernard Metzler return PKT_FRAGMENTED;
285b9be6f18SBernard Metzler }
286b9be6f18SBernard Metzler
287b9be6f18SBernard Metzler /*
288b9be6f18SBernard Metzler * Send out one complete control type FPDU, or header of FPDU carrying
289b9be6f18SBernard Metzler * data. Used for fixed sized packets like Read.Requests or zero length
290b9be6f18SBernard Metzler * SENDs, WRITEs, READ.Responses, or header only.
291b9be6f18SBernard Metzler */
siw_tx_ctrl(struct siw_iwarp_tx * c_tx,struct socket * s,int flags)292b9be6f18SBernard Metzler static int siw_tx_ctrl(struct siw_iwarp_tx *c_tx, struct socket *s,
293b9be6f18SBernard Metzler int flags)
294b9be6f18SBernard Metzler {
295b9be6f18SBernard Metzler struct msghdr msg = { .msg_flags = flags };
296b9be6f18SBernard Metzler struct kvec iov = { .iov_base =
297b9be6f18SBernard Metzler (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent,
298b9be6f18SBernard Metzler .iov_len = c_tx->ctrl_len - c_tx->ctrl_sent };
299b9be6f18SBernard Metzler
300b9be6f18SBernard Metzler int rv = kernel_sendmsg(s, &msg, &iov, 1,
301b9be6f18SBernard Metzler c_tx->ctrl_len - c_tx->ctrl_sent);
302b9be6f18SBernard Metzler
303b9be6f18SBernard Metzler if (rv >= 0) {
304b9be6f18SBernard Metzler c_tx->ctrl_sent += rv;
305b9be6f18SBernard Metzler
306b9be6f18SBernard Metzler if (c_tx->ctrl_sent == c_tx->ctrl_len)
307b9be6f18SBernard Metzler rv = 0;
308b9be6f18SBernard Metzler else
309b9be6f18SBernard Metzler rv = -EAGAIN;
310b9be6f18SBernard Metzler }
311b9be6f18SBernard Metzler return rv;
312b9be6f18SBernard Metzler }
313b9be6f18SBernard Metzler
314b9be6f18SBernard Metzler /*
315c2ff29e9SDavid Howells * 0copy TCP transmit interface: Use MSG_SPLICE_PAGES.
316b9be6f18SBernard Metzler *
317b9be6f18SBernard Metzler * Using sendpage to push page by page appears to be less efficient
318b9be6f18SBernard Metzler * than using sendmsg, even if data are copied.
319b9be6f18SBernard Metzler *
320b9be6f18SBernard Metzler * A general performance limitation might be the extra four bytes
321b9be6f18SBernard Metzler * trailer checksum segment to be pushed after user data.
322b9be6f18SBernard Metzler */
siw_tcp_sendpages(struct socket * s,struct page ** page,int offset,size_t size)323b9be6f18SBernard Metzler static int siw_tcp_sendpages(struct socket *s, struct page **page, int offset,
324b9be6f18SBernard Metzler size_t size)
325b9be6f18SBernard Metzler {
326c2ff29e9SDavid Howells struct bio_vec bvec;
327c2ff29e9SDavid Howells struct msghdr msg = {
328f8dd95b2SDavid Howells .msg_flags = (MSG_MORE | MSG_DONTWAIT | MSG_SPLICE_PAGES),
329c2ff29e9SDavid Howells };
330b9be6f18SBernard Metzler struct sock *sk = s->sk;
331c2ff29e9SDavid Howells int i = 0, rv = 0, sent = 0;
332b9be6f18SBernard Metzler
333b9be6f18SBernard Metzler while (size) {
334b9be6f18SBernard Metzler size_t bytes = min_t(size_t, PAGE_SIZE - offset, size);
335b9be6f18SBernard Metzler
336b9be6f18SBernard Metzler if (size + offset <= PAGE_SIZE)
337f8dd95b2SDavid Howells msg.msg_flags &= ~MSG_MORE;
338b9be6f18SBernard Metzler
339b9be6f18SBernard Metzler tcp_rate_check_app_limited(sk);
340c2ff29e9SDavid Howells bvec_set_page(&bvec, page[i], bytes, offset);
341c2ff29e9SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size);
342c2ff29e9SDavid Howells
343b9be6f18SBernard Metzler try_page_again:
344b9be6f18SBernard Metzler lock_sock(sk);
345c2ff29e9SDavid Howells rv = tcp_sendmsg_locked(sk, &msg, size);
346b9be6f18SBernard Metzler release_sock(sk);
347b9be6f18SBernard Metzler
348b9be6f18SBernard Metzler if (rv > 0) {
349b9be6f18SBernard Metzler size -= rv;
350b9be6f18SBernard Metzler sent += rv;
351b9be6f18SBernard Metzler if (rv != bytes) {
352b9be6f18SBernard Metzler offset += rv;
353b9be6f18SBernard Metzler bytes -= rv;
354b9be6f18SBernard Metzler goto try_page_again;
355b9be6f18SBernard Metzler }
356b9be6f18SBernard Metzler offset = 0;
357b9be6f18SBernard Metzler } else {
358b9be6f18SBernard Metzler if (rv == -EAGAIN || rv == 0)
359b9be6f18SBernard Metzler break;
360b9be6f18SBernard Metzler return rv;
361b9be6f18SBernard Metzler }
362b9be6f18SBernard Metzler i++;
363b9be6f18SBernard Metzler }
364b9be6f18SBernard Metzler return sent;
365b9be6f18SBernard Metzler }
366b9be6f18SBernard Metzler
367b9be6f18SBernard Metzler /*
368b9be6f18SBernard Metzler * siw_0copy_tx()
369b9be6f18SBernard Metzler *
370b9be6f18SBernard Metzler * Pushes list of pages to TCP socket. If pages from multiple
371b9be6f18SBernard Metzler * SGE's, all referenced pages of each SGE are pushed in one
372b9be6f18SBernard Metzler * shot.
373b9be6f18SBernard Metzler */
siw_0copy_tx(struct socket * s,struct page ** page,struct siw_sge * sge,unsigned int offset,unsigned int size)374b9be6f18SBernard Metzler static int siw_0copy_tx(struct socket *s, struct page **page,
375b9be6f18SBernard Metzler struct siw_sge *sge, unsigned int offset,
376b9be6f18SBernard Metzler unsigned int size)
377b9be6f18SBernard Metzler {
378b9be6f18SBernard Metzler int i = 0, sent = 0, rv;
379b9be6f18SBernard Metzler int sge_bytes = min(sge->length - offset, size);
380b9be6f18SBernard Metzler
381b9be6f18SBernard Metzler offset = (sge->laddr + offset) & ~PAGE_MASK;
382b9be6f18SBernard Metzler
383b9be6f18SBernard Metzler while (sent != size) {
384b9be6f18SBernard Metzler rv = siw_tcp_sendpages(s, &page[i], offset, sge_bytes);
385b9be6f18SBernard Metzler if (rv >= 0) {
386b9be6f18SBernard Metzler sent += rv;
387b9be6f18SBernard Metzler if (size == sent || sge_bytes > rv)
388b9be6f18SBernard Metzler break;
389b9be6f18SBernard Metzler
390b9be6f18SBernard Metzler i += PAGE_ALIGN(sge_bytes + offset) >> PAGE_SHIFT;
391b9be6f18SBernard Metzler sge++;
392b9be6f18SBernard Metzler sge_bytes = min(sge->length, size - sent);
393b9be6f18SBernard Metzler offset = sge->laddr & ~PAGE_MASK;
394b9be6f18SBernard Metzler } else {
395b9be6f18SBernard Metzler sent = rv;
396b9be6f18SBernard Metzler break;
397b9be6f18SBernard Metzler }
398b9be6f18SBernard Metzler }
399b9be6f18SBernard Metzler return sent;
400b9be6f18SBernard Metzler }
401b9be6f18SBernard Metzler
402b9be6f18SBernard Metzler #define MAX_TRAILER (MPA_CRC_SIZE + 4)
403b9be6f18SBernard Metzler
siw_unmap_pages(struct kvec * iov,unsigned long kmap_mask,int len)4049d649d59SIra Weiny static void siw_unmap_pages(struct kvec *iov, unsigned long kmap_mask, int len)
405b9be6f18SBernard Metzler {
4069d649d59SIra Weiny int i;
4079d649d59SIra Weiny
4089d649d59SIra Weiny /*
4099d649d59SIra Weiny * Work backwards through the array to honor the kmap_local_page()
4109d649d59SIra Weiny * ordering requirements.
4119d649d59SIra Weiny */
4129d649d59SIra Weiny for (i = (len-1); i >= 0; i--) {
4139d649d59SIra Weiny if (kmap_mask & BIT(i)) {
4149d649d59SIra Weiny unsigned long addr = (unsigned long)iov[i].iov_base;
4159d649d59SIra Weiny
4169d649d59SIra Weiny kunmap_local((void *)(addr & PAGE_MASK));
4179d649d59SIra Weiny }
418b9be6f18SBernard Metzler }
419b9be6f18SBernard Metzler }
420b9be6f18SBernard Metzler
421b9be6f18SBernard Metzler /*
422b9be6f18SBernard Metzler * siw_tx_hdt() tries to push a complete packet to TCP where all
423b9be6f18SBernard Metzler * packet fragments are referenced by the elements of one iovec.
424b9be6f18SBernard Metzler * For the data portion, each involved page must be referenced by
425b9be6f18SBernard Metzler * one extra element. All sge's data can be non-aligned to page
426b9be6f18SBernard Metzler * boundaries. Two more elements are referencing iWARP header
427b9be6f18SBernard Metzler * and trailer:
428b9be6f18SBernard Metzler * MAX_ARRAY = 64KB/PAGE_SIZE + 1 + (2 * (SIW_MAX_SGE - 1) + HDR + TRL
429b9be6f18SBernard Metzler */
430b9be6f18SBernard Metzler #define MAX_ARRAY ((0xffff / PAGE_SIZE) + 1 + (2 * (SIW_MAX_SGE - 1) + 2))
431b9be6f18SBernard Metzler
432b9be6f18SBernard Metzler /*
433b9be6f18SBernard Metzler * Write out iov referencing hdr, data and trailer of current FPDU.
434b9be6f18SBernard Metzler * Update transmit state dependent on write return status
435b9be6f18SBernard Metzler */
siw_tx_hdt(struct siw_iwarp_tx * c_tx,struct socket * s)436b9be6f18SBernard Metzler static int siw_tx_hdt(struct siw_iwarp_tx *c_tx, struct socket *s)
437b9be6f18SBernard Metzler {
438b9be6f18SBernard Metzler struct siw_wqe *wqe = &c_tx->wqe_active;
439b9be6f18SBernard Metzler struct siw_sge *sge = &wqe->sqe.sge[c_tx->sge_idx];
440b9be6f18SBernard Metzler struct kvec iov[MAX_ARRAY];
441b9be6f18SBernard Metzler struct page *page_array[MAX_ARRAY];
442b9be6f18SBernard Metzler struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
443b9be6f18SBernard Metzler
444b9be6f18SBernard Metzler int seg = 0, do_crc = c_tx->do_crc, is_kva = 0, rv;
445b9be6f18SBernard Metzler unsigned int data_len = c_tx->bytes_unsent, hdr_len = 0, trl_len = 0,
446b9be6f18SBernard Metzler sge_off = c_tx->sge_off, sge_idx = c_tx->sge_idx,
447b9be6f18SBernard Metzler pbl_idx = c_tx->pbl_idx;
448fab4f97eSBernard Metzler unsigned long kmap_mask = 0L;
449b9be6f18SBernard Metzler
450b9be6f18SBernard Metzler if (c_tx->state == SIW_SEND_HDR) {
451b9be6f18SBernard Metzler if (c_tx->use_sendpage) {
452b9be6f18SBernard Metzler rv = siw_tx_ctrl(c_tx, s, MSG_DONTWAIT | MSG_MORE);
453b9be6f18SBernard Metzler if (rv)
454b9be6f18SBernard Metzler goto done;
455b9be6f18SBernard Metzler
456b9be6f18SBernard Metzler c_tx->state = SIW_SEND_DATA;
457b9be6f18SBernard Metzler } else {
458b9be6f18SBernard Metzler iov[0].iov_base =
459b9be6f18SBernard Metzler (char *)&c_tx->pkt.ctrl + c_tx->ctrl_sent;
460b9be6f18SBernard Metzler iov[0].iov_len = hdr_len =
461b9be6f18SBernard Metzler c_tx->ctrl_len - c_tx->ctrl_sent;
462b9be6f18SBernard Metzler seg = 1;
463b9be6f18SBernard Metzler }
464b9be6f18SBernard Metzler }
465b9be6f18SBernard Metzler
466b9be6f18SBernard Metzler wqe->processed += data_len;
467b9be6f18SBernard Metzler
468b9be6f18SBernard Metzler while (data_len) { /* walk the list of SGE's */
469b9be6f18SBernard Metzler unsigned int sge_len = min(sge->length - sge_off, data_len);
470b9be6f18SBernard Metzler unsigned int fp_off = (sge->laddr + sge_off) & ~PAGE_MASK;
471b9be6f18SBernard Metzler struct siw_mem *mem;
472b9be6f18SBernard Metzler
473b9be6f18SBernard Metzler if (!(tx_flags(wqe) & SIW_WQE_INLINE)) {
474b9be6f18SBernard Metzler mem = wqe->mem[sge_idx];
475fab4f97eSBernard Metzler is_kva = mem->mem_obj == NULL ? 1 : 0;
476b9be6f18SBernard Metzler } else {
477b9be6f18SBernard Metzler is_kva = 1;
478b9be6f18SBernard Metzler }
479b9be6f18SBernard Metzler if (is_kva && !c_tx->use_sendpage) {
480b9be6f18SBernard Metzler /*
481b9be6f18SBernard Metzler * tx from kernel virtual address: either inline data
482b9be6f18SBernard Metzler * or memory region with assigned kernel buffer
483b9be6f18SBernard Metzler */
484c536277eSBernard Metzler iov[seg].iov_base =
4858d7c7c0eSJason Gunthorpe ib_virt_dma_to_ptr(sge->laddr + sge_off);
486b9be6f18SBernard Metzler iov[seg].iov_len = sge_len;
487b9be6f18SBernard Metzler
488b9be6f18SBernard Metzler if (do_crc)
489b9be6f18SBernard Metzler crypto_shash_update(c_tx->mpa_crc_hd,
490b9be6f18SBernard Metzler iov[seg].iov_base,
491b9be6f18SBernard Metzler sge_len);
492b9be6f18SBernard Metzler sge_off += sge_len;
493b9be6f18SBernard Metzler data_len -= sge_len;
494b9be6f18SBernard Metzler seg++;
495b9be6f18SBernard Metzler goto sge_done;
496b9be6f18SBernard Metzler }
497b9be6f18SBernard Metzler
498b9be6f18SBernard Metzler while (sge_len) {
499b9be6f18SBernard Metzler size_t plen = min((int)PAGE_SIZE - fp_off, sge_len);
5001ec50dd1SIra Weiny void *kaddr;
501b9be6f18SBernard Metzler
502b9be6f18SBernard Metzler if (!is_kva) {
503b9be6f18SBernard Metzler struct page *p;
504b9be6f18SBernard Metzler
505b9be6f18SBernard Metzler if (mem->is_pbl)
506b9be6f18SBernard Metzler p = siw_get_pblpage(
507b9be6f18SBernard Metzler mem, sge->laddr + sge_off,
508b9be6f18SBernard Metzler &pbl_idx);
509b9be6f18SBernard Metzler else
510b9be6f18SBernard Metzler p = siw_get_upage(mem->umem,
511b9be6f18SBernard Metzler sge->laddr + sge_off);
512b9be6f18SBernard Metzler if (unlikely(!p)) {
5139d649d59SIra Weiny siw_unmap_pages(iov, kmap_mask, seg);
514b9be6f18SBernard Metzler wqe->processed -= c_tx->bytes_unsent;
515b9be6f18SBernard Metzler rv = -EFAULT;
516b9be6f18SBernard Metzler goto done_crc;
517b9be6f18SBernard Metzler }
518b9be6f18SBernard Metzler page_array[seg] = p;
519b9be6f18SBernard Metzler
520b9be6f18SBernard Metzler if (!c_tx->use_sendpage) {
5219d649d59SIra Weiny void *kaddr = kmap_local_page(p);
522fab4f97eSBernard Metzler
523fab4f97eSBernard Metzler /* Remember for later kunmap() */
524fab4f97eSBernard Metzler kmap_mask |= BIT(seg);
5259d649d59SIra Weiny iov[seg].iov_base = kaddr + fp_off;
5269d649d59SIra Weiny iov[seg].iov_len = plen;
527fab4f97eSBernard Metzler
528b9be6f18SBernard Metzler if (do_crc)
529b9be6f18SBernard Metzler crypto_shash_update(
530b9be6f18SBernard Metzler c_tx->mpa_crc_hd,
531b9be6f18SBernard Metzler iov[seg].iov_base,
532b9be6f18SBernard Metzler plen);
5330404bd62SBernard Metzler } else if (do_crc) {
5341ec50dd1SIra Weiny kaddr = kmap_local_page(p);
5350404bd62SBernard Metzler crypto_shash_update(c_tx->mpa_crc_hd,
5361ec50dd1SIra Weiny kaddr + fp_off,
537b9be6f18SBernard Metzler plen);
5381ec50dd1SIra Weiny kunmap_local(kaddr);
5390404bd62SBernard Metzler }
540b9be6f18SBernard Metzler } else {
5410d1b756aSLinus Walleij /*
5420d1b756aSLinus Walleij * Cast to an uintptr_t to preserve all 64 bits
5430d1b756aSLinus Walleij * in sge->laddr.
5440d1b756aSLinus Walleij */
5458d7c7c0eSJason Gunthorpe u64 va = sge->laddr + sge_off;
546b9be6f18SBernard Metzler
5478d7c7c0eSJason Gunthorpe page_array[seg] = ib_virt_dma_to_page(va);
548b9be6f18SBernard Metzler if (do_crc)
549b9be6f18SBernard Metzler crypto_shash_update(
550b9be6f18SBernard Metzler c_tx->mpa_crc_hd,
5518d7c7c0eSJason Gunthorpe ib_virt_dma_to_ptr(va),
552b9be6f18SBernard Metzler plen);
553b9be6f18SBernard Metzler }
554b9be6f18SBernard Metzler
555b9be6f18SBernard Metzler sge_len -= plen;
556b9be6f18SBernard Metzler sge_off += plen;
557b9be6f18SBernard Metzler data_len -= plen;
558b9be6f18SBernard Metzler fp_off = 0;
559b9be6f18SBernard Metzler
560271bfcfbSDaniil Dulov if (++seg >= (int)MAX_ARRAY) {
561b9be6f18SBernard Metzler siw_dbg_qp(tx_qp(c_tx), "to many fragments\n");
5629d649d59SIra Weiny siw_unmap_pages(iov, kmap_mask, seg-1);
563b9be6f18SBernard Metzler wqe->processed -= c_tx->bytes_unsent;
564b9be6f18SBernard Metzler rv = -EMSGSIZE;
565b9be6f18SBernard Metzler goto done_crc;
566b9be6f18SBernard Metzler }
567b9be6f18SBernard Metzler }
568b9be6f18SBernard Metzler sge_done:
569b9be6f18SBernard Metzler /* Update SGE variables at end of SGE */
570b9be6f18SBernard Metzler if (sge_off == sge->length &&
571b9be6f18SBernard Metzler (data_len != 0 || wqe->processed < wqe->bytes)) {
572b9be6f18SBernard Metzler sge_idx++;
573b9be6f18SBernard Metzler sge++;
574b9be6f18SBernard Metzler sge_off = 0;
575b9be6f18SBernard Metzler }
576b9be6f18SBernard Metzler }
577b9be6f18SBernard Metzler /* trailer */
578b9be6f18SBernard Metzler if (likely(c_tx->state != SIW_SEND_TRAILER)) {
579b9be6f18SBernard Metzler iov[seg].iov_base = &c_tx->trailer.pad[4 - c_tx->pad];
580b9be6f18SBernard Metzler iov[seg].iov_len = trl_len = MAX_TRAILER - (4 - c_tx->pad);
581b9be6f18SBernard Metzler } else {
582b9be6f18SBernard Metzler iov[seg].iov_base = &c_tx->trailer.pad[c_tx->ctrl_sent];
583b9be6f18SBernard Metzler iov[seg].iov_len = trl_len = MAX_TRAILER - c_tx->ctrl_sent;
584b9be6f18SBernard Metzler }
585b9be6f18SBernard Metzler
586b9be6f18SBernard Metzler if (c_tx->pad) {
587b9be6f18SBernard Metzler *(u32 *)c_tx->trailer.pad = 0;
588b9be6f18SBernard Metzler if (do_crc)
589b9be6f18SBernard Metzler crypto_shash_update(c_tx->mpa_crc_hd,
590b9be6f18SBernard Metzler (u8 *)&c_tx->trailer.crc - c_tx->pad,
591b9be6f18SBernard Metzler c_tx->pad);
592b9be6f18SBernard Metzler }
593b9be6f18SBernard Metzler if (!c_tx->mpa_crc_hd)
594b9be6f18SBernard Metzler c_tx->trailer.crc = 0;
595b9be6f18SBernard Metzler else if (do_crc)
596b9be6f18SBernard Metzler crypto_shash_final(c_tx->mpa_crc_hd, (u8 *)&c_tx->trailer.crc);
597b9be6f18SBernard Metzler
598b9be6f18SBernard Metzler data_len = c_tx->bytes_unsent;
599b9be6f18SBernard Metzler
600b9be6f18SBernard Metzler if (c_tx->use_sendpage) {
601b9be6f18SBernard Metzler rv = siw_0copy_tx(s, page_array, &wqe->sqe.sge[c_tx->sge_idx],
602b9be6f18SBernard Metzler c_tx->sge_off, data_len);
603b9be6f18SBernard Metzler if (rv == data_len) {
604b9be6f18SBernard Metzler rv = kernel_sendmsg(s, &msg, &iov[seg], 1, trl_len);
605b9be6f18SBernard Metzler if (rv > 0)
606b9be6f18SBernard Metzler rv += data_len;
607b9be6f18SBernard Metzler else
608b9be6f18SBernard Metzler rv = data_len;
609b9be6f18SBernard Metzler }
610b9be6f18SBernard Metzler } else {
611b9be6f18SBernard Metzler rv = kernel_sendmsg(s, &msg, iov, seg + 1,
612b9be6f18SBernard Metzler hdr_len + data_len + trl_len);
6139d649d59SIra Weiny siw_unmap_pages(iov, kmap_mask, seg);
614b9be6f18SBernard Metzler }
615b9be6f18SBernard Metzler if (rv < (int)hdr_len) {
616b9be6f18SBernard Metzler /* Not even complete hdr pushed or negative rv */
617b9be6f18SBernard Metzler wqe->processed -= data_len;
618b9be6f18SBernard Metzler if (rv >= 0) {
619b9be6f18SBernard Metzler c_tx->ctrl_sent += rv;
620b9be6f18SBernard Metzler rv = -EAGAIN;
621b9be6f18SBernard Metzler }
622b9be6f18SBernard Metzler goto done_crc;
623b9be6f18SBernard Metzler }
624b9be6f18SBernard Metzler rv -= hdr_len;
625b9be6f18SBernard Metzler
626b9be6f18SBernard Metzler if (rv >= (int)data_len) {
627b9be6f18SBernard Metzler /* all user data pushed to TCP or no data to push */
628b9be6f18SBernard Metzler if (data_len > 0 && wqe->processed < wqe->bytes) {
629b9be6f18SBernard Metzler /* Save the current state for next tx */
630b9be6f18SBernard Metzler c_tx->sge_idx = sge_idx;
631b9be6f18SBernard Metzler c_tx->sge_off = sge_off;
632b9be6f18SBernard Metzler c_tx->pbl_idx = pbl_idx;
633b9be6f18SBernard Metzler }
634b9be6f18SBernard Metzler rv -= data_len;
635b9be6f18SBernard Metzler
636b9be6f18SBernard Metzler if (rv == trl_len) /* all pushed */
637b9be6f18SBernard Metzler rv = 0;
638b9be6f18SBernard Metzler else {
639b9be6f18SBernard Metzler c_tx->state = SIW_SEND_TRAILER;
640b9be6f18SBernard Metzler c_tx->ctrl_len = MAX_TRAILER;
641b9be6f18SBernard Metzler c_tx->ctrl_sent = rv + 4 - c_tx->pad;
642b9be6f18SBernard Metzler c_tx->bytes_unsent = 0;
643b9be6f18SBernard Metzler rv = -EAGAIN;
644b9be6f18SBernard Metzler }
645b9be6f18SBernard Metzler
646b9be6f18SBernard Metzler } else if (data_len > 0) {
647b9be6f18SBernard Metzler /* Maybe some user data pushed to TCP */
648b9be6f18SBernard Metzler c_tx->state = SIW_SEND_DATA;
649b9be6f18SBernard Metzler wqe->processed -= data_len - rv;
650b9be6f18SBernard Metzler
651b9be6f18SBernard Metzler if (rv) {
652b9be6f18SBernard Metzler /*
653b9be6f18SBernard Metzler * Some bytes out. Recompute tx state based
654b9be6f18SBernard Metzler * on old state and bytes pushed
655b9be6f18SBernard Metzler */
656b9be6f18SBernard Metzler unsigned int sge_unsent;
657b9be6f18SBernard Metzler
658b9be6f18SBernard Metzler c_tx->bytes_unsent -= rv;
659b9be6f18SBernard Metzler sge = &wqe->sqe.sge[c_tx->sge_idx];
660b9be6f18SBernard Metzler sge_unsent = sge->length - c_tx->sge_off;
661b9be6f18SBernard Metzler
662b9be6f18SBernard Metzler while (sge_unsent <= rv) {
663b9be6f18SBernard Metzler rv -= sge_unsent;
664b9be6f18SBernard Metzler c_tx->sge_idx++;
665b9be6f18SBernard Metzler c_tx->sge_off = 0;
666b9be6f18SBernard Metzler sge++;
667b9be6f18SBernard Metzler sge_unsent = sge->length;
668b9be6f18SBernard Metzler }
669b9be6f18SBernard Metzler c_tx->sge_off += rv;
670b9be6f18SBernard Metzler }
671b9be6f18SBernard Metzler rv = -EAGAIN;
672b9be6f18SBernard Metzler }
673b9be6f18SBernard Metzler done_crc:
674b9be6f18SBernard Metzler c_tx->do_crc = 0;
675b9be6f18SBernard Metzler done:
676b9be6f18SBernard Metzler return rv;
677b9be6f18SBernard Metzler }
678b9be6f18SBernard Metzler
siw_update_tcpseg(struct siw_iwarp_tx * c_tx,struct socket * s)679b9be6f18SBernard Metzler static void siw_update_tcpseg(struct siw_iwarp_tx *c_tx,
680b9be6f18SBernard Metzler struct socket *s)
681b9be6f18SBernard Metzler {
682b9be6f18SBernard Metzler struct tcp_sock *tp = tcp_sk(s->sk);
683b9be6f18SBernard Metzler
684b9be6f18SBernard Metzler if (tp->gso_segs) {
685b9be6f18SBernard Metzler if (c_tx->gso_seg_limit == 0)
686b9be6f18SBernard Metzler c_tx->tcp_seglen = tp->mss_cache * tp->gso_segs;
687b9be6f18SBernard Metzler else
688b9be6f18SBernard Metzler c_tx->tcp_seglen =
689b9be6f18SBernard Metzler tp->mss_cache *
690b9be6f18SBernard Metzler min_t(u16, c_tx->gso_seg_limit, tp->gso_segs);
691b9be6f18SBernard Metzler } else {
692b9be6f18SBernard Metzler c_tx->tcp_seglen = tp->mss_cache;
693b9be6f18SBernard Metzler }
694b9be6f18SBernard Metzler /* Loopback may give odd numbers */
695b9be6f18SBernard Metzler c_tx->tcp_seglen &= 0xfffffff8;
696b9be6f18SBernard Metzler }
697b9be6f18SBernard Metzler
698b9be6f18SBernard Metzler /*
699b9be6f18SBernard Metzler * siw_prepare_fpdu()
700b9be6f18SBernard Metzler *
701b9be6f18SBernard Metzler * Prepares transmit context to send out one FPDU if FPDU will contain
702b9be6f18SBernard Metzler * user data and user data are not immediate data.
703b9be6f18SBernard Metzler * Computes maximum FPDU length to fill up TCP MSS if possible.
704b9be6f18SBernard Metzler *
705b9be6f18SBernard Metzler * @qp: QP from which to transmit
706b9be6f18SBernard Metzler * @wqe: Current WQE causing transmission
707b9be6f18SBernard Metzler *
708b9be6f18SBernard Metzler * TODO: Take into account real available sendspace on socket
709b9be6f18SBernard Metzler * to avoid header misalignment due to send pausing within
710b9be6f18SBernard Metzler * fpdu transmission
711b9be6f18SBernard Metzler */
siw_prepare_fpdu(struct siw_qp * qp,struct siw_wqe * wqe)712b9be6f18SBernard Metzler static void siw_prepare_fpdu(struct siw_qp *qp, struct siw_wqe *wqe)
713b9be6f18SBernard Metzler {
714b9be6f18SBernard Metzler struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
715b9be6f18SBernard Metzler int data_len;
716b9be6f18SBernard Metzler
717b9be6f18SBernard Metzler c_tx->ctrl_len =
718b9be6f18SBernard Metzler iwarp_pktinfo[__rdmap_get_opcode(&c_tx->pkt.ctrl)].hdr_len;
719b9be6f18SBernard Metzler c_tx->ctrl_sent = 0;
720b9be6f18SBernard Metzler
721b9be6f18SBernard Metzler /*
722b9be6f18SBernard Metzler * Update target buffer offset if any
723b9be6f18SBernard Metzler */
724b9be6f18SBernard Metzler if (!(c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED))
725b9be6f18SBernard Metzler /* Untagged message */
726b9be6f18SBernard Metzler c_tx->pkt.c_untagged.ddp_mo = cpu_to_be32(wqe->processed);
727b9be6f18SBernard Metzler else /* Tagged message */
728b9be6f18SBernard Metzler c_tx->pkt.c_tagged.ddp_to =
729b9be6f18SBernard Metzler cpu_to_be64(wqe->sqe.raddr + wqe->processed);
730b9be6f18SBernard Metzler
731b9be6f18SBernard Metzler data_len = wqe->bytes - wqe->processed;
732b9be6f18SBernard Metzler if (data_len + c_tx->ctrl_len + MPA_CRC_SIZE > c_tx->tcp_seglen) {
733b9be6f18SBernard Metzler /* Trim DDP payload to fit into current TCP segment */
734b9be6f18SBernard Metzler data_len = c_tx->tcp_seglen - (c_tx->ctrl_len + MPA_CRC_SIZE);
735b9be6f18SBernard Metzler c_tx->pkt.ctrl.ddp_rdmap_ctrl &= ~DDP_FLAG_LAST;
736b9be6f18SBernard Metzler c_tx->pad = 0;
737b9be6f18SBernard Metzler } else {
738b9be6f18SBernard Metzler c_tx->pkt.ctrl.ddp_rdmap_ctrl |= DDP_FLAG_LAST;
739b9be6f18SBernard Metzler c_tx->pad = -data_len & 0x3;
740b9be6f18SBernard Metzler }
741b9be6f18SBernard Metzler c_tx->bytes_unsent = data_len;
742b9be6f18SBernard Metzler
743b9be6f18SBernard Metzler c_tx->pkt.ctrl.mpa_len =
744b9be6f18SBernard Metzler htons(c_tx->ctrl_len + data_len - MPA_HDR_SIZE);
745b9be6f18SBernard Metzler
746b9be6f18SBernard Metzler /*
747b9be6f18SBernard Metzler * Init MPA CRC computation
748b9be6f18SBernard Metzler */
749b9be6f18SBernard Metzler if (c_tx->mpa_crc_hd) {
750b9be6f18SBernard Metzler crypto_shash_init(c_tx->mpa_crc_hd);
751b9be6f18SBernard Metzler crypto_shash_update(c_tx->mpa_crc_hd, (u8 *)&c_tx->pkt,
752b9be6f18SBernard Metzler c_tx->ctrl_len);
753b9be6f18SBernard Metzler c_tx->do_crc = 1;
754b9be6f18SBernard Metzler }
755b9be6f18SBernard Metzler }
756b9be6f18SBernard Metzler
757b9be6f18SBernard Metzler /*
758b9be6f18SBernard Metzler * siw_check_sgl_tx()
759b9be6f18SBernard Metzler *
760b9be6f18SBernard Metzler * Check permissions for a list of SGE's (SGL).
761b9be6f18SBernard Metzler * A successful check will have all memory referenced
762b9be6f18SBernard Metzler * for transmission resolved and assigned to the WQE.
763b9be6f18SBernard Metzler *
764b9be6f18SBernard Metzler * @pd: Protection Domain SGL should belong to
765b9be6f18SBernard Metzler * @wqe: WQE to be checked
766b9be6f18SBernard Metzler * @perms: requested access permissions
767b9be6f18SBernard Metzler *
768b9be6f18SBernard Metzler */
769b9be6f18SBernard Metzler
siw_check_sgl_tx(struct ib_pd * pd,struct siw_wqe * wqe,enum ib_access_flags perms)770b9be6f18SBernard Metzler static int siw_check_sgl_tx(struct ib_pd *pd, struct siw_wqe *wqe,
771b9be6f18SBernard Metzler enum ib_access_flags perms)
772b9be6f18SBernard Metzler {
773b9be6f18SBernard Metzler struct siw_sge *sge = &wqe->sqe.sge[0];
774b9be6f18SBernard Metzler int i, len, num_sge = wqe->sqe.num_sge;
775b9be6f18SBernard Metzler
776b9be6f18SBernard Metzler if (unlikely(num_sge > SIW_MAX_SGE))
777b9be6f18SBernard Metzler return -EINVAL;
778b9be6f18SBernard Metzler
779b9be6f18SBernard Metzler for (i = 0, len = 0; num_sge; num_sge--, i++, sge++) {
780b9be6f18SBernard Metzler /*
781b9be6f18SBernard Metzler * rdma verbs: do not check stag for a zero length sge
782b9be6f18SBernard Metzler */
783b9be6f18SBernard Metzler if (sge->length) {
784b9be6f18SBernard Metzler int rv = siw_check_sge(pd, sge, &wqe->mem[i], perms, 0,
785b9be6f18SBernard Metzler sge->length);
786b9be6f18SBernard Metzler
787b9be6f18SBernard Metzler if (unlikely(rv != E_ACCESS_OK))
788b9be6f18SBernard Metzler return rv;
789b9be6f18SBernard Metzler }
790b9be6f18SBernard Metzler len += sge->length;
791b9be6f18SBernard Metzler }
792b9be6f18SBernard Metzler return len;
793b9be6f18SBernard Metzler }
794b9be6f18SBernard Metzler
795b9be6f18SBernard Metzler /*
796b9be6f18SBernard Metzler * siw_qp_sq_proc_tx()
797b9be6f18SBernard Metzler *
798b9be6f18SBernard Metzler * Process one WQE which needs transmission on the wire.
799b9be6f18SBernard Metzler */
siw_qp_sq_proc_tx(struct siw_qp * qp,struct siw_wqe * wqe)800b9be6f18SBernard Metzler static int siw_qp_sq_proc_tx(struct siw_qp *qp, struct siw_wqe *wqe)
801b9be6f18SBernard Metzler {
802b9be6f18SBernard Metzler struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
803b9be6f18SBernard Metzler struct socket *s = qp->attrs.sk;
804b9be6f18SBernard Metzler int rv = 0, burst_len = qp->tx_ctx.burst;
805b9be6f18SBernard Metzler enum rdmap_ecode ecode = RDMAP_ECODE_CATASTROPHIC_STREAM;
806b9be6f18SBernard Metzler
807b9be6f18SBernard Metzler if (unlikely(wqe->wr_status == SIW_WR_IDLE))
808b9be6f18SBernard Metzler return 0;
809b9be6f18SBernard Metzler
810b9be6f18SBernard Metzler if (!burst_len)
811b9be6f18SBernard Metzler burst_len = SQ_USER_MAXBURST;
812b9be6f18SBernard Metzler
813b9be6f18SBernard Metzler if (wqe->wr_status == SIW_WR_QUEUED) {
814b9be6f18SBernard Metzler if (!(wqe->sqe.flags & SIW_WQE_INLINE)) {
815b9be6f18SBernard Metzler if (tx_type(wqe) == SIW_OP_READ_RESPONSE)
816b9be6f18SBernard Metzler wqe->sqe.num_sge = 1;
817b9be6f18SBernard Metzler
818b9be6f18SBernard Metzler if (tx_type(wqe) != SIW_OP_READ &&
819b9be6f18SBernard Metzler tx_type(wqe) != SIW_OP_READ_LOCAL_INV) {
820b9be6f18SBernard Metzler /*
821b9be6f18SBernard Metzler * Reference memory to be tx'd w/o checking
822b9be6f18SBernard Metzler * access for LOCAL_READ permission, since
823b9be6f18SBernard Metzler * not defined in RDMA core.
824b9be6f18SBernard Metzler */
825b9be6f18SBernard Metzler rv = siw_check_sgl_tx(qp->pd, wqe, 0);
826b9be6f18SBernard Metzler if (rv < 0) {
827b9be6f18SBernard Metzler if (tx_type(wqe) ==
828b9be6f18SBernard Metzler SIW_OP_READ_RESPONSE)
829b9be6f18SBernard Metzler ecode = siw_rdmap_error(-rv);
830b9be6f18SBernard Metzler rv = -EINVAL;
831b9be6f18SBernard Metzler goto tx_error;
832b9be6f18SBernard Metzler }
833b9be6f18SBernard Metzler wqe->bytes = rv;
834b9be6f18SBernard Metzler } else {
835b9be6f18SBernard Metzler wqe->bytes = 0;
836b9be6f18SBernard Metzler }
837b9be6f18SBernard Metzler } else {
838b9be6f18SBernard Metzler wqe->bytes = wqe->sqe.sge[0].length;
83958fb0b56SBernard Metzler if (!rdma_is_kernel_res(&qp->base_qp.res)) {
840b9be6f18SBernard Metzler if (wqe->bytes > SIW_MAX_INLINE) {
841b9be6f18SBernard Metzler rv = -EINVAL;
842b9be6f18SBernard Metzler goto tx_error;
843b9be6f18SBernard Metzler }
844c536277eSBernard Metzler wqe->sqe.sge[0].laddr =
845c536277eSBernard Metzler (u64)(uintptr_t)&wqe->sqe.sge[1];
846b9be6f18SBernard Metzler }
847b9be6f18SBernard Metzler }
848b9be6f18SBernard Metzler wqe->wr_status = SIW_WR_INPROGRESS;
849b9be6f18SBernard Metzler wqe->processed = 0;
850b9be6f18SBernard Metzler
851b9be6f18SBernard Metzler siw_update_tcpseg(c_tx, s);
852b9be6f18SBernard Metzler
853b9be6f18SBernard Metzler rv = siw_qp_prepare_tx(c_tx);
854b9be6f18SBernard Metzler if (rv == PKT_FRAGMENTED) {
855b9be6f18SBernard Metzler c_tx->state = SIW_SEND_HDR;
856b9be6f18SBernard Metzler siw_prepare_fpdu(qp, wqe);
857b9be6f18SBernard Metzler } else if (rv == PKT_COMPLETE) {
858b9be6f18SBernard Metzler c_tx->state = SIW_SEND_SHORT_FPDU;
859b9be6f18SBernard Metzler } else {
860b9be6f18SBernard Metzler goto tx_error;
861b9be6f18SBernard Metzler }
862b9be6f18SBernard Metzler }
863b9be6f18SBernard Metzler
864b9be6f18SBernard Metzler next_segment:
865b9be6f18SBernard Metzler siw_dbg_qp(qp, "wr type %d, state %d, data %u, sent %u, id %llx\n",
866b9be6f18SBernard Metzler tx_type(wqe), wqe->wr_status, wqe->bytes, wqe->processed,
867b9be6f18SBernard Metzler wqe->sqe.id);
868b9be6f18SBernard Metzler
869b9be6f18SBernard Metzler if (--burst_len == 0) {
870b9be6f18SBernard Metzler rv = -EINPROGRESS;
871b9be6f18SBernard Metzler goto tx_done;
872b9be6f18SBernard Metzler }
873b9be6f18SBernard Metzler if (c_tx->state == SIW_SEND_SHORT_FPDU) {
874b9be6f18SBernard Metzler enum siw_opcode tx_type = tx_type(wqe);
875b9be6f18SBernard Metzler unsigned int msg_flags;
876b9be6f18SBernard Metzler
877b9be6f18SBernard Metzler if (siw_sq_empty(qp) || !siw_tcp_nagle || burst_len == 1)
878b9be6f18SBernard Metzler /*
879b9be6f18SBernard Metzler * End current TCP segment, if SQ runs empty,
880b9be6f18SBernard Metzler * or siw_tcp_nagle is not set, or we bail out
881b9be6f18SBernard Metzler * soon due to no burst credit left.
882b9be6f18SBernard Metzler */
883b9be6f18SBernard Metzler msg_flags = MSG_DONTWAIT;
884b9be6f18SBernard Metzler else
885b9be6f18SBernard Metzler msg_flags = MSG_DONTWAIT | MSG_MORE;
886b9be6f18SBernard Metzler
887b9be6f18SBernard Metzler rv = siw_tx_ctrl(c_tx, s, msg_flags);
888b9be6f18SBernard Metzler
889b9be6f18SBernard Metzler if (!rv && tx_type != SIW_OP_READ &&
890b9be6f18SBernard Metzler tx_type != SIW_OP_READ_LOCAL_INV)
891b9be6f18SBernard Metzler wqe->processed = wqe->bytes;
892b9be6f18SBernard Metzler
893b9be6f18SBernard Metzler goto tx_done;
894b9be6f18SBernard Metzler
895b9be6f18SBernard Metzler } else {
896b9be6f18SBernard Metzler rv = siw_tx_hdt(c_tx, s);
897b9be6f18SBernard Metzler }
898b9be6f18SBernard Metzler if (!rv) {
899b9be6f18SBernard Metzler /*
900b9be6f18SBernard Metzler * One segment sent. Processing completed if last
901b9be6f18SBernard Metzler * segment, Do next segment otherwise.
902b9be6f18SBernard Metzler */
903b9be6f18SBernard Metzler if (unlikely(c_tx->tx_suspend)) {
904b9be6f18SBernard Metzler /*
905b9be6f18SBernard Metzler * Verbs, 6.4.: Try stopping sending after a full
906b9be6f18SBernard Metzler * DDP segment if the connection goes down
907b9be6f18SBernard Metzler * (== peer halfclose)
908b9be6f18SBernard Metzler */
909b9be6f18SBernard Metzler rv = -ECONNABORTED;
910b9be6f18SBernard Metzler goto tx_done;
911b9be6f18SBernard Metzler }
912b9be6f18SBernard Metzler if (c_tx->pkt.ctrl.ddp_rdmap_ctrl & DDP_FLAG_LAST) {
913b9be6f18SBernard Metzler siw_dbg_qp(qp, "WQE completed\n");
914b9be6f18SBernard Metzler goto tx_done;
915b9be6f18SBernard Metzler }
916b9be6f18SBernard Metzler c_tx->state = SIW_SEND_HDR;
917b9be6f18SBernard Metzler
918b9be6f18SBernard Metzler siw_update_tcpseg(c_tx, s);
919b9be6f18SBernard Metzler
920b9be6f18SBernard Metzler siw_prepare_fpdu(qp, wqe);
921b9be6f18SBernard Metzler goto next_segment;
922b9be6f18SBernard Metzler }
923b9be6f18SBernard Metzler tx_done:
924b9be6f18SBernard Metzler qp->tx_ctx.burst = burst_len;
925b9be6f18SBernard Metzler return rv;
926b9be6f18SBernard Metzler
927b9be6f18SBernard Metzler tx_error:
928b9be6f18SBernard Metzler if (ecode != RDMAP_ECODE_CATASTROPHIC_STREAM)
929b9be6f18SBernard Metzler siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
930b9be6f18SBernard Metzler RDMAP_ETYPE_REMOTE_PROTECTION, ecode, 1);
931b9be6f18SBernard Metzler else
932b9be6f18SBernard Metzler siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
933b9be6f18SBernard Metzler RDMAP_ETYPE_CATASTROPHIC,
934b9be6f18SBernard Metzler RDMAP_ECODE_UNSPECIFIED, 1);
935b9be6f18SBernard Metzler return rv;
936b9be6f18SBernard Metzler }
937b9be6f18SBernard Metzler
siw_fastreg_mr(struct ib_pd * pd,struct siw_sqe * sqe)938b9be6f18SBernard Metzler static int siw_fastreg_mr(struct ib_pd *pd, struct siw_sqe *sqe)
939b9be6f18SBernard Metzler {
940c536277eSBernard Metzler struct ib_mr *base_mr = (struct ib_mr *)(uintptr_t)sqe->base_mr;
941b9be6f18SBernard Metzler struct siw_device *sdev = to_siw_dev(pd->device);
9426e051971SJason Gunthorpe struct siw_mem *mem;
943b9be6f18SBernard Metzler int rv = 0;
944b9be6f18SBernard Metzler
945b9be6f18SBernard Metzler siw_dbg_pd(pd, "STag 0x%08x\n", sqe->rkey);
946b9be6f18SBernard Metzler
9476e051971SJason Gunthorpe if (unlikely(!base_mr)) {
948b9be6f18SBernard Metzler pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);
949b9be6f18SBernard Metzler return -EINVAL;
950b9be6f18SBernard Metzler }
9516e051971SJason Gunthorpe
952b9be6f18SBernard Metzler if (unlikely(base_mr->rkey >> 8 != sqe->rkey >> 8)) {
953b9be6f18SBernard Metzler pr_warn("siw: fastreg: STag 0x%08x: bad MR\n", sqe->rkey);
9546e051971SJason Gunthorpe return -EINVAL;
955b9be6f18SBernard Metzler }
9566e051971SJason Gunthorpe
9576e051971SJason Gunthorpe mem = siw_mem_id2obj(sdev, sqe->rkey >> 8);
9586e051971SJason Gunthorpe if (unlikely(!mem)) {
9596e051971SJason Gunthorpe pr_warn("siw: fastreg: STag 0x%08x unknown\n", sqe->rkey);
9606e051971SJason Gunthorpe return -EINVAL;
9616e051971SJason Gunthorpe }
9626e051971SJason Gunthorpe
963b9be6f18SBernard Metzler if (unlikely(mem->pd != pd)) {
964b9be6f18SBernard Metzler pr_warn("siw: fastreg: PD mismatch\n");
965b9be6f18SBernard Metzler rv = -EINVAL;
966b9be6f18SBernard Metzler goto out;
967b9be6f18SBernard Metzler }
968b9be6f18SBernard Metzler if (unlikely(mem->stag_valid)) {
969b9be6f18SBernard Metzler pr_warn("siw: fastreg: STag 0x%08x already valid\n", sqe->rkey);
970b9be6f18SBernard Metzler rv = -EINVAL;
971b9be6f18SBernard Metzler goto out;
972b9be6f18SBernard Metzler }
973b9be6f18SBernard Metzler /* Refresh STag since user may have changed key part */
974b9be6f18SBernard Metzler mem->stag = sqe->rkey;
975b9be6f18SBernard Metzler mem->perms = sqe->access;
976b9be6f18SBernard Metzler
977c536277eSBernard Metzler siw_dbg_mem(mem, "STag 0x%08x now valid\n", sqe->rkey);
978b9be6f18SBernard Metzler mem->va = base_mr->iova;
979b9be6f18SBernard Metzler mem->stag_valid = 1;
980b9be6f18SBernard Metzler out:
981b9be6f18SBernard Metzler siw_mem_put(mem);
982b9be6f18SBernard Metzler return rv;
983b9be6f18SBernard Metzler }
984b9be6f18SBernard Metzler
siw_qp_sq_proc_local(struct siw_qp * qp,struct siw_wqe * wqe)985b9be6f18SBernard Metzler static int siw_qp_sq_proc_local(struct siw_qp *qp, struct siw_wqe *wqe)
986b9be6f18SBernard Metzler {
987b9be6f18SBernard Metzler int rv;
988b9be6f18SBernard Metzler
989b9be6f18SBernard Metzler switch (tx_type(wqe)) {
990b9be6f18SBernard Metzler case SIW_OP_REG_MR:
991b9be6f18SBernard Metzler rv = siw_fastreg_mr(qp->pd, &wqe->sqe);
992b9be6f18SBernard Metzler break;
993b9be6f18SBernard Metzler
994b9be6f18SBernard Metzler case SIW_OP_INVAL_STAG:
995b9be6f18SBernard Metzler rv = siw_invalidate_stag(qp->pd, wqe->sqe.rkey);
996b9be6f18SBernard Metzler break;
997b9be6f18SBernard Metzler
998b9be6f18SBernard Metzler default:
999b9be6f18SBernard Metzler rv = -EINVAL;
1000b9be6f18SBernard Metzler }
1001b9be6f18SBernard Metzler return rv;
1002b9be6f18SBernard Metzler }
1003b9be6f18SBernard Metzler
1004b9be6f18SBernard Metzler /*
1005b9be6f18SBernard Metzler * siw_qp_sq_process()
1006b9be6f18SBernard Metzler *
1007b9be6f18SBernard Metzler * Core TX path routine for RDMAP/DDP/MPA using a TCP kernel socket.
1008b9be6f18SBernard Metzler * Sends RDMAP payload for the current SQ WR @wqe of @qp in one or more
1009b9be6f18SBernard Metzler * MPA FPDUs, each containing a DDP segment.
1010b9be6f18SBernard Metzler *
1011b9be6f18SBernard Metzler * SQ processing may occur in user context as a result of posting
1012b9be6f18SBernard Metzler * new WQE's or from siw_sq_work_handler() context. Processing in
1013b9be6f18SBernard Metzler * user context is limited to non-kernel verbs users.
1014b9be6f18SBernard Metzler *
1015b9be6f18SBernard Metzler * SQ processing may get paused anytime, possibly in the middle of a WR
1016b9be6f18SBernard Metzler * or FPDU, if insufficient send space is available. SQ processing
1017b9be6f18SBernard Metzler * gets resumed from siw_sq_work_handler(), if send space becomes
1018b9be6f18SBernard Metzler * available again.
1019b9be6f18SBernard Metzler *
1020b9be6f18SBernard Metzler * Must be called with the QP state read-locked.
1021b9be6f18SBernard Metzler *
1022b9be6f18SBernard Metzler * Note:
1023b9be6f18SBernard Metzler * An outbound RREQ can be satisfied by the corresponding RRESP
1024b9be6f18SBernard Metzler * _before_ it gets assigned to the ORQ. This happens regularly
1025b9be6f18SBernard Metzler * in RDMA READ via loopback case. Since both outbound RREQ and
1026b9be6f18SBernard Metzler * inbound RRESP can be handled by the same CPU, locking the ORQ
1027b9be6f18SBernard Metzler * is dead-lock prone and thus not an option. With that, the
1028b9be6f18SBernard Metzler * RREQ gets assigned to the ORQ _before_ being sent - see
1029b9be6f18SBernard Metzler * siw_activate_tx() - and pulled back in case of send failure.
1030b9be6f18SBernard Metzler */
siw_qp_sq_process(struct siw_qp * qp)1031b9be6f18SBernard Metzler int siw_qp_sq_process(struct siw_qp *qp)
1032b9be6f18SBernard Metzler {
1033b9be6f18SBernard Metzler struct siw_wqe *wqe = tx_wqe(qp);
1034b9be6f18SBernard Metzler enum siw_opcode tx_type;
1035b9be6f18SBernard Metzler unsigned long flags;
1036b9be6f18SBernard Metzler int rv = 0;
1037b9be6f18SBernard Metzler
1038b9be6f18SBernard Metzler siw_dbg_qp(qp, "enter for type %d\n", tx_type(wqe));
1039b9be6f18SBernard Metzler
1040b9be6f18SBernard Metzler next_wqe:
1041b9be6f18SBernard Metzler /*
1042b9be6f18SBernard Metzler * Stop QP processing if SQ state changed
1043b9be6f18SBernard Metzler */
1044b9be6f18SBernard Metzler if (unlikely(qp->tx_ctx.tx_suspend)) {
1045b9be6f18SBernard Metzler siw_dbg_qp(qp, "tx suspended\n");
1046b9be6f18SBernard Metzler goto done;
1047b9be6f18SBernard Metzler }
1048b9be6f18SBernard Metzler tx_type = tx_type(wqe);
1049b9be6f18SBernard Metzler
1050b9be6f18SBernard Metzler if (tx_type <= SIW_OP_READ_RESPONSE)
1051b9be6f18SBernard Metzler rv = siw_qp_sq_proc_tx(qp, wqe);
1052b9be6f18SBernard Metzler else
1053b9be6f18SBernard Metzler rv = siw_qp_sq_proc_local(qp, wqe);
1054b9be6f18SBernard Metzler
1055b9be6f18SBernard Metzler if (!rv) {
1056b9be6f18SBernard Metzler /*
1057b9be6f18SBernard Metzler * WQE processing done
1058b9be6f18SBernard Metzler */
1059b9be6f18SBernard Metzler switch (tx_type) {
1060b9be6f18SBernard Metzler case SIW_OP_SEND:
1061b9be6f18SBernard Metzler case SIW_OP_SEND_REMOTE_INV:
1062b9be6f18SBernard Metzler case SIW_OP_WRITE:
1063b9be6f18SBernard Metzler siw_wqe_put_mem(wqe, tx_type);
1064df561f66SGustavo A. R. Silva fallthrough;
1065cea743f2SGustavo A. R. Silva
1066b9be6f18SBernard Metzler case SIW_OP_INVAL_STAG:
1067b9be6f18SBernard Metzler case SIW_OP_REG_MR:
1068b9be6f18SBernard Metzler if (tx_flags(wqe) & SIW_WQE_SIGNALLED)
1069b9be6f18SBernard Metzler siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1070b9be6f18SBernard Metzler SIW_WC_SUCCESS);
1071b9be6f18SBernard Metzler break;
1072b9be6f18SBernard Metzler
1073b9be6f18SBernard Metzler case SIW_OP_READ:
1074b9be6f18SBernard Metzler case SIW_OP_READ_LOCAL_INV:
1075b9be6f18SBernard Metzler /*
1076b9be6f18SBernard Metzler * already enqueued to ORQ queue
1077b9be6f18SBernard Metzler */
1078b9be6f18SBernard Metzler break;
1079b9be6f18SBernard Metzler
1080b9be6f18SBernard Metzler case SIW_OP_READ_RESPONSE:
1081b9be6f18SBernard Metzler siw_wqe_put_mem(wqe, tx_type);
1082b9be6f18SBernard Metzler break;
1083b9be6f18SBernard Metzler
1084b9be6f18SBernard Metzler default:
1085b9be6f18SBernard Metzler WARN(1, "undefined WQE type %d\n", tx_type);
1086b9be6f18SBernard Metzler rv = -EINVAL;
1087b9be6f18SBernard Metzler goto done;
1088b9be6f18SBernard Metzler }
1089b9be6f18SBernard Metzler
1090b9be6f18SBernard Metzler spin_lock_irqsave(&qp->sq_lock, flags);
1091b9be6f18SBernard Metzler wqe->wr_status = SIW_WR_IDLE;
1092b9be6f18SBernard Metzler rv = siw_activate_tx(qp);
1093b9be6f18SBernard Metzler spin_unlock_irqrestore(&qp->sq_lock, flags);
1094b9be6f18SBernard Metzler
1095b9be6f18SBernard Metzler if (rv <= 0)
1096b9be6f18SBernard Metzler goto done;
1097b9be6f18SBernard Metzler
1098b9be6f18SBernard Metzler goto next_wqe;
1099b9be6f18SBernard Metzler
1100b9be6f18SBernard Metzler } else if (rv == -EAGAIN) {
1101b9be6f18SBernard Metzler siw_dbg_qp(qp, "sq paused: hd/tr %d of %d, data %d\n",
1102b9be6f18SBernard Metzler qp->tx_ctx.ctrl_sent, qp->tx_ctx.ctrl_len,
1103b9be6f18SBernard Metzler qp->tx_ctx.bytes_unsent);
1104b9be6f18SBernard Metzler rv = 0;
1105b9be6f18SBernard Metzler goto done;
1106b9be6f18SBernard Metzler } else if (rv == -EINPROGRESS) {
1107b9be6f18SBernard Metzler rv = siw_sq_start(qp);
1108b9be6f18SBernard Metzler goto done;
1109b9be6f18SBernard Metzler } else {
1110b9be6f18SBernard Metzler /*
1111b9be6f18SBernard Metzler * WQE processing failed.
1112b9be6f18SBernard Metzler * Verbs 8.3.2:
1113b9be6f18SBernard Metzler * o It turns any WQE into a signalled WQE.
1114b9be6f18SBernard Metzler * o Local catastrophic error must be surfaced
1115b9be6f18SBernard Metzler * o QP must be moved into Terminate state: done by code
1116b9be6f18SBernard Metzler * doing socket state change processing
1117b9be6f18SBernard Metzler *
1118b9be6f18SBernard Metzler * o TODO: Termination message must be sent.
1119b9be6f18SBernard Metzler * o TODO: Implement more precise work completion errors,
1120b9be6f18SBernard Metzler * see enum ib_wc_status in ib_verbs.h
1121b9be6f18SBernard Metzler */
1122b9be6f18SBernard Metzler siw_dbg_qp(qp, "wqe type %d processing failed: %d\n",
1123b9be6f18SBernard Metzler tx_type(wqe), rv);
1124b9be6f18SBernard Metzler
1125b9be6f18SBernard Metzler spin_lock_irqsave(&qp->sq_lock, flags);
1126b9be6f18SBernard Metzler /*
1127b9be6f18SBernard Metzler * RREQ may have already been completed by inbound RRESP!
1128b9be6f18SBernard Metzler */
1129661f3859SBernard Metzler if ((tx_type == SIW_OP_READ ||
1130661f3859SBernard Metzler tx_type == SIW_OP_READ_LOCAL_INV) && qp->attrs.orq_size) {
1131b9be6f18SBernard Metzler /* Cleanup pending entry in ORQ */
1132b9be6f18SBernard Metzler qp->orq_put--;
1133b9be6f18SBernard Metzler qp->orq[qp->orq_put % qp->attrs.orq_size].flags = 0;
1134b9be6f18SBernard Metzler }
1135b9be6f18SBernard Metzler spin_unlock_irqrestore(&qp->sq_lock, flags);
1136b9be6f18SBernard Metzler /*
1137b9be6f18SBernard Metzler * immediately suspends further TX processing
1138b9be6f18SBernard Metzler */
1139b9be6f18SBernard Metzler if (!qp->tx_ctx.tx_suspend)
1140b9be6f18SBernard Metzler siw_qp_cm_drop(qp, 0);
1141b9be6f18SBernard Metzler
1142b9be6f18SBernard Metzler switch (tx_type) {
1143b9be6f18SBernard Metzler case SIW_OP_SEND:
1144b9be6f18SBernard Metzler case SIW_OP_SEND_REMOTE_INV:
1145b9be6f18SBernard Metzler case SIW_OP_SEND_WITH_IMM:
1146b9be6f18SBernard Metzler case SIW_OP_WRITE:
1147b9be6f18SBernard Metzler case SIW_OP_READ:
1148b9be6f18SBernard Metzler case SIW_OP_READ_LOCAL_INV:
1149b9be6f18SBernard Metzler siw_wqe_put_mem(wqe, tx_type);
1150df561f66SGustavo A. R. Silva fallthrough;
1151cea743f2SGustavo A. R. Silva
1152b9be6f18SBernard Metzler case SIW_OP_INVAL_STAG:
1153b9be6f18SBernard Metzler case SIW_OP_REG_MR:
1154b9be6f18SBernard Metzler siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1155b9be6f18SBernard Metzler SIW_WC_LOC_QP_OP_ERR);
1156b9be6f18SBernard Metzler
1157b9be6f18SBernard Metzler siw_qp_event(qp, IB_EVENT_QP_FATAL);
1158b9be6f18SBernard Metzler
1159b9be6f18SBernard Metzler break;
1160b9be6f18SBernard Metzler
1161b9be6f18SBernard Metzler case SIW_OP_READ_RESPONSE:
1162b9be6f18SBernard Metzler siw_dbg_qp(qp, "proc. read.response failed: %d\n", rv);
1163b9be6f18SBernard Metzler
1164b9be6f18SBernard Metzler siw_qp_event(qp, IB_EVENT_QP_REQ_ERR);
1165b9be6f18SBernard Metzler
1166b9be6f18SBernard Metzler siw_wqe_put_mem(wqe, SIW_OP_READ_RESPONSE);
1167b9be6f18SBernard Metzler
1168b9be6f18SBernard Metzler break;
1169b9be6f18SBernard Metzler
1170b9be6f18SBernard Metzler default:
1171b9be6f18SBernard Metzler WARN(1, "undefined WQE type %d\n", tx_type);
1172b9be6f18SBernard Metzler rv = -EINVAL;
1173b9be6f18SBernard Metzler }
1174b9be6f18SBernard Metzler wqe->wr_status = SIW_WR_IDLE;
1175b9be6f18SBernard Metzler }
1176b9be6f18SBernard Metzler done:
1177b9be6f18SBernard Metzler return rv;
1178b9be6f18SBernard Metzler }
1179b9be6f18SBernard Metzler
siw_sq_resume(struct siw_qp * qp)1180b9be6f18SBernard Metzler static void siw_sq_resume(struct siw_qp *qp)
1181b9be6f18SBernard Metzler {
1182b9be6f18SBernard Metzler if (down_read_trylock(&qp->state_lock)) {
1183b9be6f18SBernard Metzler if (likely(qp->attrs.state == SIW_QP_STATE_RTS &&
1184b9be6f18SBernard Metzler !qp->tx_ctx.tx_suspend)) {
1185b9be6f18SBernard Metzler int rv = siw_qp_sq_process(qp);
1186b9be6f18SBernard Metzler
1187b9be6f18SBernard Metzler up_read(&qp->state_lock);
1188b9be6f18SBernard Metzler
1189b9be6f18SBernard Metzler if (unlikely(rv < 0)) {
1190b9be6f18SBernard Metzler siw_dbg_qp(qp, "SQ task failed: err %d\n", rv);
1191b9be6f18SBernard Metzler
1192b9be6f18SBernard Metzler if (!qp->tx_ctx.tx_suspend)
1193b9be6f18SBernard Metzler siw_qp_cm_drop(qp, 0);
1194b9be6f18SBernard Metzler }
1195b9be6f18SBernard Metzler } else {
1196b9be6f18SBernard Metzler up_read(&qp->state_lock);
1197b9be6f18SBernard Metzler }
1198b9be6f18SBernard Metzler } else {
1199b9be6f18SBernard Metzler siw_dbg_qp(qp, "Resume SQ while QP locked\n");
1200b9be6f18SBernard Metzler }
1201b9be6f18SBernard Metzler siw_qp_put(qp);
1202b9be6f18SBernard Metzler }
1203b9be6f18SBernard Metzler
1204b9be6f18SBernard Metzler struct tx_task_t {
1205b9be6f18SBernard Metzler struct llist_head active;
1206b9be6f18SBernard Metzler wait_queue_head_t waiting;
1207b9be6f18SBernard Metzler };
1208b9be6f18SBernard Metzler
12094c7d6dcdSJason Gunthorpe static DEFINE_PER_CPU(struct tx_task_t, siw_tx_task_g);
1210b9be6f18SBernard Metzler
siw_create_tx_threads(void)121191f36237SBernard Metzler int siw_create_tx_threads(void)
1212b9be6f18SBernard Metzler {
121391f36237SBernard Metzler int cpu, assigned = 0;
121491f36237SBernard Metzler
121591f36237SBernard Metzler for_each_online_cpu(cpu) {
121691f36237SBernard Metzler struct tx_task_t *tx_task;
121791f36237SBernard Metzler
121891f36237SBernard Metzler /* Skip HT cores */
121991f36237SBernard Metzler if (cpu % cpumask_weight(topology_sibling_cpumask(cpu)))
122091f36237SBernard Metzler continue;
122191f36237SBernard Metzler
122291f36237SBernard Metzler tx_task = &per_cpu(siw_tx_task_g, cpu);
122391f36237SBernard Metzler init_llist_head(&tx_task->active);
122491f36237SBernard Metzler init_waitqueue_head(&tx_task->waiting);
122591f36237SBernard Metzler
122691f36237SBernard Metzler siw_tx_thread[cpu] =
122791f36237SBernard Metzler kthread_run_on_cpu(siw_run_sq,
122891f36237SBernard Metzler (unsigned long *)(long)cpu,
122991f36237SBernard Metzler cpu, "siw_tx/%u");
123091f36237SBernard Metzler if (IS_ERR(siw_tx_thread[cpu])) {
123191f36237SBernard Metzler siw_tx_thread[cpu] = NULL;
123291f36237SBernard Metzler continue;
123391f36237SBernard Metzler }
123491f36237SBernard Metzler assigned++;
123591f36237SBernard Metzler }
123691f36237SBernard Metzler return assigned;
123791f36237SBernard Metzler }
123891f36237SBernard Metzler
siw_stop_tx_threads(void)123991f36237SBernard Metzler void siw_stop_tx_threads(void)
124091f36237SBernard Metzler {
124191f36237SBernard Metzler int cpu;
124291f36237SBernard Metzler
124391f36237SBernard Metzler for_each_possible_cpu(cpu) {
124491f36237SBernard Metzler if (siw_tx_thread[cpu]) {
124591f36237SBernard Metzler kthread_stop(siw_tx_thread[cpu]);
124691f36237SBernard Metzler wake_up(&per_cpu(siw_tx_task_g, cpu).waiting);
124791f36237SBernard Metzler siw_tx_thread[cpu] = NULL;
124891f36237SBernard Metzler }
124991f36237SBernard Metzler }
1250b9be6f18SBernard Metzler }
1251b9be6f18SBernard Metzler
siw_run_sq(void * data)1252b9be6f18SBernard Metzler int siw_run_sq(void *data)
1253b9be6f18SBernard Metzler {
1254b9be6f18SBernard Metzler const int nr_cpu = (unsigned int)(long)data;
1255b9be6f18SBernard Metzler struct llist_node *active;
1256b9be6f18SBernard Metzler struct siw_qp *qp;
12574c7d6dcdSJason Gunthorpe struct tx_task_t *tx_task = &per_cpu(siw_tx_task_g, nr_cpu);
1258b9be6f18SBernard Metzler
1259b9be6f18SBernard Metzler while (1) {
1260b9be6f18SBernard Metzler struct llist_node *fifo_list = NULL;
1261b9be6f18SBernard Metzler
1262b9be6f18SBernard Metzler wait_event_interruptible(tx_task->waiting,
1263b9be6f18SBernard Metzler !llist_empty(&tx_task->active) ||
1264b9be6f18SBernard Metzler kthread_should_stop());
1265b9be6f18SBernard Metzler
1266b9be6f18SBernard Metzler if (kthread_should_stop())
1267b9be6f18SBernard Metzler break;
1268b9be6f18SBernard Metzler
1269b9be6f18SBernard Metzler active = llist_del_all(&tx_task->active);
1270b9be6f18SBernard Metzler /*
1271b9be6f18SBernard Metzler * llist_del_all returns a list with newest entry first.
1272b9be6f18SBernard Metzler * Re-order list for fairness among QP's.
1273b9be6f18SBernard Metzler */
1274*9dfccb6dSGuoqing Jiang fifo_list = llist_reverse_order(active);
1275b9be6f18SBernard Metzler while (fifo_list) {
1276b9be6f18SBernard Metzler qp = container_of(fifo_list, struct siw_qp, tx_list);
1277b9be6f18SBernard Metzler fifo_list = llist_next(fifo_list);
1278b9be6f18SBernard Metzler qp->tx_list.next = NULL;
1279b9be6f18SBernard Metzler
1280b9be6f18SBernard Metzler siw_sq_resume(qp);
1281b9be6f18SBernard Metzler }
1282b9be6f18SBernard Metzler }
1283b9be6f18SBernard Metzler active = llist_del_all(&tx_task->active);
1284b9be6f18SBernard Metzler if (active) {
1285b9be6f18SBernard Metzler llist_for_each_entry(qp, active, tx_list) {
1286b9be6f18SBernard Metzler qp->tx_list.next = NULL;
1287b9be6f18SBernard Metzler siw_sq_resume(qp);
1288b9be6f18SBernard Metzler }
1289b9be6f18SBernard Metzler }
1290b9be6f18SBernard Metzler return 0;
1291b9be6f18SBernard Metzler }
1292b9be6f18SBernard Metzler
siw_sq_start(struct siw_qp * qp)1293b9be6f18SBernard Metzler int siw_sq_start(struct siw_qp *qp)
1294b9be6f18SBernard Metzler {
1295b9be6f18SBernard Metzler if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
1296b9be6f18SBernard Metzler return 0;
1297b9be6f18SBernard Metzler
1298b9be6f18SBernard Metzler if (unlikely(!cpu_online(qp->tx_cpu))) {
1299b9be6f18SBernard Metzler siw_put_tx_cpu(qp->tx_cpu);
1300b9be6f18SBernard Metzler qp->tx_cpu = siw_get_tx_cpu(qp->sdev);
1301b9be6f18SBernard Metzler if (qp->tx_cpu < 0) {
1302b9be6f18SBernard Metzler pr_warn("siw: no tx cpu available\n");
1303b9be6f18SBernard Metzler
1304b9be6f18SBernard Metzler return -EIO;
1305b9be6f18SBernard Metzler }
1306b9be6f18SBernard Metzler }
1307b9be6f18SBernard Metzler siw_qp_get(qp);
1308b9be6f18SBernard Metzler
13094c7d6dcdSJason Gunthorpe llist_add(&qp->tx_list, &per_cpu(siw_tx_task_g, qp->tx_cpu).active);
1310b9be6f18SBernard Metzler
13114c7d6dcdSJason Gunthorpe wake_up(&per_cpu(siw_tx_task_g, qp->tx_cpu).waiting);
1312b9be6f18SBernard Metzler
1313b9be6f18SBernard Metzler return 0;
1314b9be6f18SBernard Metzler }
1315