1 /*
2  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *	- Redistributions of source code must retain the above
16  *	  copyright notice, this list of conditions and the following
17  *	  disclaimer.
18  *
19  *	- Redistributions in binary form must reproduce the above
20  *	  copyright notice, this list of conditions and the following
21  *	  disclaimer in the documentation and/or other materials
22  *	  provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/skbuff.h>
35 #include <crypto/hash.h>
36 
37 #include "rxe.h"
38 #include "rxe_loc.h"
39 #include "rxe_queue.h"
40 
41 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
42 		       u32 opcode);
43 
44 static inline void retry_first_write_send(struct rxe_qp *qp,
45 					  struct rxe_send_wqe *wqe,
46 					  unsigned int mask, int npsn)
47 {
48 	int i;
49 
50 	for (i = 0; i < npsn; i++) {
51 		int to_send = (wqe->dma.resid > qp->mtu) ?
52 				qp->mtu : wqe->dma.resid;
53 
54 		qp->req.opcode = next_opcode(qp, wqe,
55 					     wqe->wr.opcode);
56 
57 		if (wqe->wr.send_flags & IB_SEND_INLINE) {
58 			wqe->dma.resid -= to_send;
59 			wqe->dma.sge_offset += to_send;
60 		} else {
61 			advance_dma_data(&wqe->dma, to_send);
62 		}
63 		if (mask & WR_WRITE_MASK)
64 			wqe->iova += qp->mtu;
65 	}
66 }
67 
68 static void req_retry(struct rxe_qp *qp)
69 {
70 	struct rxe_send_wqe *wqe;
71 	unsigned int wqe_index;
72 	unsigned int mask;
73 	int npsn;
74 	int first = 1;
75 
76 	wqe = queue_head(qp->sq.queue);
77 	npsn = (qp->comp.psn - wqe->first_psn) & BTH_PSN_MASK;
78 
79 	qp->req.wqe_index	= consumer_index(qp->sq.queue);
80 	qp->req.psn		= qp->comp.psn;
81 	qp->req.opcode		= -1;
82 
83 	for (wqe_index = consumer_index(qp->sq.queue);
84 		wqe_index != producer_index(qp->sq.queue);
85 		wqe_index = next_index(qp->sq.queue, wqe_index)) {
86 		wqe = addr_from_index(qp->sq.queue, wqe_index);
87 		mask = wr_opcode_mask(wqe->wr.opcode, qp);
88 
89 		if (wqe->state == wqe_state_posted)
90 			break;
91 
92 		if (wqe->state == wqe_state_done)
93 			continue;
94 
95 		wqe->iova = (mask & WR_ATOMIC_MASK) ?
96 			     wqe->wr.wr.atomic.remote_addr :
97 			     (mask & WR_READ_OR_WRITE_MASK) ?
98 			     wqe->wr.wr.rdma.remote_addr :
99 			     0;
100 
101 		if (!first || (mask & WR_READ_MASK) == 0) {
102 			wqe->dma.resid = wqe->dma.length;
103 			wqe->dma.cur_sge = 0;
104 			wqe->dma.sge_offset = 0;
105 		}
106 
107 		if (first) {
108 			first = 0;
109 
110 			if (mask & WR_WRITE_OR_SEND_MASK)
111 				retry_first_write_send(qp, wqe, mask, npsn);
112 
113 			if (mask & WR_READ_MASK)
114 				wqe->iova += npsn * qp->mtu;
115 		}
116 
117 		wqe->state = wqe_state_posted;
118 	}
119 }
120 
121 void rnr_nak_timer(struct timer_list *t)
122 {
123 	struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer);
124 
125 	pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
126 	rxe_run_task(&qp->req.task, 1);
127 }
128 
129 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
130 {
131 	struct rxe_send_wqe *wqe = queue_head(qp->sq.queue);
132 	unsigned long flags;
133 
134 	if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
135 		/* check to see if we are drained;
136 		 * state_lock used by requester and completer
137 		 */
138 		spin_lock_irqsave(&qp->state_lock, flags);
139 		do {
140 			if (qp->req.state != QP_STATE_DRAIN) {
141 				/* comp just finished */
142 				spin_unlock_irqrestore(&qp->state_lock,
143 						       flags);
144 				break;
145 			}
146 
147 			if (wqe && ((qp->req.wqe_index !=
148 				consumer_index(qp->sq.queue)) ||
149 				(wqe->state != wqe_state_posted))) {
150 				/* comp not done yet */
151 				spin_unlock_irqrestore(&qp->state_lock,
152 						       flags);
153 				break;
154 			}
155 
156 			qp->req.state = QP_STATE_DRAINED;
157 			spin_unlock_irqrestore(&qp->state_lock, flags);
158 
159 			if (qp->ibqp.event_handler) {
160 				struct ib_event ev;
161 
162 				ev.device = qp->ibqp.device;
163 				ev.element.qp = &qp->ibqp;
164 				ev.event = IB_EVENT_SQ_DRAINED;
165 				qp->ibqp.event_handler(&ev,
166 					qp->ibqp.qp_context);
167 			}
168 		} while (0);
169 	}
170 
171 	if (qp->req.wqe_index == producer_index(qp->sq.queue))
172 		return NULL;
173 
174 	wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index);
175 
176 	if (unlikely((qp->req.state == QP_STATE_DRAIN ||
177 		      qp->req.state == QP_STATE_DRAINED) &&
178 		     (wqe->state != wqe_state_processing)))
179 		return NULL;
180 
181 	if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) &&
182 		     (qp->req.wqe_index != consumer_index(qp->sq.queue)))) {
183 		qp->req.wait_fence = 1;
184 		return NULL;
185 	}
186 
187 	wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
188 	return wqe;
189 }
190 
191 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)
192 {
193 	switch (opcode) {
194 	case IB_WR_RDMA_WRITE:
195 		if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
196 		    qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
197 			return fits ?
198 				IB_OPCODE_RC_RDMA_WRITE_LAST :
199 				IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
200 		else
201 			return fits ?
202 				IB_OPCODE_RC_RDMA_WRITE_ONLY :
203 				IB_OPCODE_RC_RDMA_WRITE_FIRST;
204 
205 	case IB_WR_RDMA_WRITE_WITH_IMM:
206 		if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
207 		    qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
208 			return fits ?
209 				IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
210 				IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
211 		else
212 			return fits ?
213 				IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
214 				IB_OPCODE_RC_RDMA_WRITE_FIRST;
215 
216 	case IB_WR_SEND:
217 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
218 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
219 			return fits ?
220 				IB_OPCODE_RC_SEND_LAST :
221 				IB_OPCODE_RC_SEND_MIDDLE;
222 		else
223 			return fits ?
224 				IB_OPCODE_RC_SEND_ONLY :
225 				IB_OPCODE_RC_SEND_FIRST;
226 
227 	case IB_WR_SEND_WITH_IMM:
228 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
229 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
230 			return fits ?
231 				IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE :
232 				IB_OPCODE_RC_SEND_MIDDLE;
233 		else
234 			return fits ?
235 				IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
236 				IB_OPCODE_RC_SEND_FIRST;
237 
238 	case IB_WR_RDMA_READ:
239 		return IB_OPCODE_RC_RDMA_READ_REQUEST;
240 
241 	case IB_WR_ATOMIC_CMP_AND_SWP:
242 		return IB_OPCODE_RC_COMPARE_SWAP;
243 
244 	case IB_WR_ATOMIC_FETCH_AND_ADD:
245 		return IB_OPCODE_RC_FETCH_ADD;
246 
247 	case IB_WR_SEND_WITH_INV:
248 		if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
249 		    qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
250 			return fits ? IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE :
251 				IB_OPCODE_RC_SEND_MIDDLE;
252 		else
253 			return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
254 				IB_OPCODE_RC_SEND_FIRST;
255 	case IB_WR_REG_MR:
256 	case IB_WR_LOCAL_INV:
257 		return opcode;
258 	}
259 
260 	return -EINVAL;
261 }
262 
263 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)
264 {
265 	switch (opcode) {
266 	case IB_WR_RDMA_WRITE:
267 		if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
268 		    qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
269 			return fits ?
270 				IB_OPCODE_UC_RDMA_WRITE_LAST :
271 				IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
272 		else
273 			return fits ?
274 				IB_OPCODE_UC_RDMA_WRITE_ONLY :
275 				IB_OPCODE_UC_RDMA_WRITE_FIRST;
276 
277 	case IB_WR_RDMA_WRITE_WITH_IMM:
278 		if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
279 		    qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
280 			return fits ?
281 				IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
282 				IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
283 		else
284 			return fits ?
285 				IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
286 				IB_OPCODE_UC_RDMA_WRITE_FIRST;
287 
288 	case IB_WR_SEND:
289 		if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
290 		    qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
291 			return fits ?
292 				IB_OPCODE_UC_SEND_LAST :
293 				IB_OPCODE_UC_SEND_MIDDLE;
294 		else
295 			return fits ?
296 				IB_OPCODE_UC_SEND_ONLY :
297 				IB_OPCODE_UC_SEND_FIRST;
298 
299 	case IB_WR_SEND_WITH_IMM:
300 		if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
301 		    qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
302 			return fits ?
303 				IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE :
304 				IB_OPCODE_UC_SEND_MIDDLE;
305 		else
306 			return fits ?
307 				IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE :
308 				IB_OPCODE_UC_SEND_FIRST;
309 	}
310 
311 	return -EINVAL;
312 }
313 
314 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
315 		       u32 opcode)
316 {
317 	int fits = (wqe->dma.resid <= qp->mtu);
318 
319 	switch (qp_type(qp)) {
320 	case IB_QPT_RC:
321 		return next_opcode_rc(qp, opcode, fits);
322 
323 	case IB_QPT_UC:
324 		return next_opcode_uc(qp, opcode, fits);
325 
326 	case IB_QPT_SMI:
327 	case IB_QPT_UD:
328 	case IB_QPT_GSI:
329 		switch (opcode) {
330 		case IB_WR_SEND:
331 			return IB_OPCODE_UD_SEND_ONLY;
332 
333 		case IB_WR_SEND_WITH_IMM:
334 			return IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
335 		}
336 		break;
337 
338 	default:
339 		break;
340 	}
341 
342 	return -EINVAL;
343 }
344 
345 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
346 {
347 	int depth;
348 
349 	if (wqe->has_rd_atomic)
350 		return 0;
351 
352 	qp->req.need_rd_atomic = 1;
353 	depth = atomic_dec_return(&qp->req.rd_atomic);
354 
355 	if (depth >= 0) {
356 		qp->req.need_rd_atomic = 0;
357 		wqe->has_rd_atomic = 1;
358 		return 0;
359 	}
360 
361 	atomic_inc(&qp->req.rd_atomic);
362 	return -EAGAIN;
363 }
364 
365 static inline int get_mtu(struct rxe_qp *qp)
366 {
367 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
368 
369 	if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
370 		return qp->mtu;
371 
372 	return rxe->port.mtu_cap;
373 }
374 
375 static struct sk_buff *init_req_packet(struct rxe_qp *qp,
376 				       struct rxe_send_wqe *wqe,
377 				       int opcode, int payload,
378 				       struct rxe_pkt_info *pkt)
379 {
380 	struct rxe_dev		*rxe = to_rdev(qp->ibqp.device);
381 	struct rxe_port		*port = &rxe->port;
382 	struct sk_buff		*skb;
383 	struct rxe_send_wr	*ibwr = &wqe->wr;
384 	struct rxe_av		*av;
385 	int			pad = (-payload) & 0x3;
386 	int			paylen;
387 	int			solicited;
388 	u16			pkey;
389 	u32			qp_num;
390 	int			ack_req;
391 
392 	/* length from start of bth to end of icrc */
393 	paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
394 
395 	/* pkt->hdr, rxe, port_num and mask are initialized in ifc
396 	 * layer
397 	 */
398 	pkt->opcode	= opcode;
399 	pkt->qp		= qp;
400 	pkt->psn	= qp->req.psn;
401 	pkt->mask	= rxe_opcode[opcode].mask;
402 	pkt->paylen	= paylen;
403 	pkt->offset	= 0;
404 	pkt->wqe	= wqe;
405 
406 	/* init skb */
407 	av = rxe_get_av(pkt);
408 	skb = rxe_init_packet(rxe, av, paylen, pkt);
409 	if (unlikely(!skb))
410 		return NULL;
411 
412 	/* init bth */
413 	solicited = (ibwr->send_flags & IB_SEND_SOLICITED) &&
414 			(pkt->mask & RXE_END_MASK) &&
415 			((pkt->mask & (RXE_SEND_MASK)) ||
416 			(pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
417 			(RXE_WRITE_MASK | RXE_IMMDT_MASK));
418 
419 	pkey = (qp_type(qp) == IB_QPT_GSI) ?
420 		 port->pkey_tbl[ibwr->wr.ud.pkey_index] :
421 		 port->pkey_tbl[qp->attr.pkey_index];
422 
423 	qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
424 					 qp->attr.dest_qp_num;
425 
426 	ack_req = ((pkt->mask & RXE_END_MASK) ||
427 		(qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK));
428 	if (ack_req)
429 		qp->req.noack_pkts = 0;
430 
431 	bth_init(pkt, pkt->opcode, solicited, 0, pad, pkey, qp_num,
432 		 ack_req, pkt->psn);
433 
434 	/* init optional headers */
435 	if (pkt->mask & RXE_RETH_MASK) {
436 		reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
437 		reth_set_va(pkt, wqe->iova);
438 		reth_set_len(pkt, wqe->dma.length);
439 	}
440 
441 	if (pkt->mask & RXE_IMMDT_MASK)
442 		immdt_set_imm(pkt, ibwr->ex.imm_data);
443 
444 	if (pkt->mask & RXE_IETH_MASK)
445 		ieth_set_rkey(pkt, ibwr->ex.invalidate_rkey);
446 
447 	if (pkt->mask & RXE_ATMETH_MASK) {
448 		atmeth_set_va(pkt, wqe->iova);
449 		if (opcode == IB_OPCODE_RC_COMPARE_SWAP ||
450 		    opcode == IB_OPCODE_RD_COMPARE_SWAP) {
451 			atmeth_set_swap_add(pkt, ibwr->wr.atomic.swap);
452 			atmeth_set_comp(pkt, ibwr->wr.atomic.compare_add);
453 		} else {
454 			atmeth_set_swap_add(pkt, ibwr->wr.atomic.compare_add);
455 		}
456 		atmeth_set_rkey(pkt, ibwr->wr.atomic.rkey);
457 	}
458 
459 	if (pkt->mask & RXE_DETH_MASK) {
460 		if (qp->ibqp.qp_num == 1)
461 			deth_set_qkey(pkt, GSI_QKEY);
462 		else
463 			deth_set_qkey(pkt, ibwr->wr.ud.remote_qkey);
464 		deth_set_sqp(pkt, qp->ibqp.qp_num);
465 	}
466 
467 	return skb;
468 }
469 
470 static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
471 		       struct rxe_pkt_info *pkt, struct sk_buff *skb,
472 		       int paylen)
473 {
474 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
475 	u32 crc = 0;
476 	u32 *p;
477 	int err;
478 
479 	err = rxe_prepare(rxe, pkt, skb, &crc);
480 	if (err)
481 		return err;
482 
483 	if (pkt->mask & RXE_WRITE_OR_SEND) {
484 		if (wqe->wr.send_flags & IB_SEND_INLINE) {
485 			u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
486 
487 			crc = rxe_crc32(rxe, crc, tmp, paylen);
488 			memcpy(payload_addr(pkt), tmp, paylen);
489 
490 			wqe->dma.resid -= paylen;
491 			wqe->dma.sge_offset += paylen;
492 		} else {
493 			err = copy_data(qp->pd, 0, &wqe->dma,
494 					payload_addr(pkt), paylen,
495 					from_mem_obj,
496 					&crc);
497 			if (err)
498 				return err;
499 		}
500 	}
501 	p = payload_addr(pkt) + paylen + bth_pad(pkt);
502 
503 	*p = ~crc;
504 
505 	return 0;
506 }
507 
508 static void update_wqe_state(struct rxe_qp *qp,
509 		struct rxe_send_wqe *wqe,
510 		struct rxe_pkt_info *pkt)
511 {
512 	if (pkt->mask & RXE_END_MASK) {
513 		if (qp_type(qp) == IB_QPT_RC)
514 			wqe->state = wqe_state_pending;
515 	} else {
516 		wqe->state = wqe_state_processing;
517 	}
518 }
519 
520 static void update_wqe_psn(struct rxe_qp *qp,
521 			   struct rxe_send_wqe *wqe,
522 			   struct rxe_pkt_info *pkt,
523 			   int payload)
524 {
525 	/* number of packets left to send including current one */
526 	int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
527 
528 	/* handle zero length packet case */
529 	if (num_pkt == 0)
530 		num_pkt = 1;
531 
532 	if (pkt->mask & RXE_START_MASK) {
533 		wqe->first_psn = qp->req.psn;
534 		wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK;
535 	}
536 
537 	if (pkt->mask & RXE_READ_MASK)
538 		qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;
539 	else
540 		qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
541 }
542 
543 static void save_state(struct rxe_send_wqe *wqe,
544 		       struct rxe_qp *qp,
545 		       struct rxe_send_wqe *rollback_wqe,
546 		       u32 *rollback_psn)
547 {
548 	rollback_wqe->state     = wqe->state;
549 	rollback_wqe->first_psn = wqe->first_psn;
550 	rollback_wqe->last_psn  = wqe->last_psn;
551 	*rollback_psn		= qp->req.psn;
552 }
553 
554 static void rollback_state(struct rxe_send_wqe *wqe,
555 			   struct rxe_qp *qp,
556 			   struct rxe_send_wqe *rollback_wqe,
557 			   u32 rollback_psn)
558 {
559 	wqe->state     = rollback_wqe->state;
560 	wqe->first_psn = rollback_wqe->first_psn;
561 	wqe->last_psn  = rollback_wqe->last_psn;
562 	qp->req.psn    = rollback_psn;
563 }
564 
565 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
566 			 struct rxe_pkt_info *pkt, int payload)
567 {
568 	qp->req.opcode = pkt->opcode;
569 
570 	if (pkt->mask & RXE_END_MASK)
571 		qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index);
572 
573 	qp->need_req_skb = 0;
574 
575 	if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer))
576 		mod_timer(&qp->retrans_timer,
577 			  jiffies + qp->qp_timeout_jiffies);
578 }
579 
580 int rxe_requester(void *arg)
581 {
582 	struct rxe_qp *qp = (struct rxe_qp *)arg;
583 	struct rxe_pkt_info pkt;
584 	struct sk_buff *skb;
585 	struct rxe_send_wqe *wqe;
586 	enum rxe_hdr_mask mask;
587 	int payload;
588 	int mtu;
589 	int opcode;
590 	int ret;
591 	struct rxe_send_wqe rollback_wqe;
592 	u32 rollback_psn;
593 
594 	rxe_add_ref(qp);
595 
596 next_wqe:
597 	if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
598 		goto exit;
599 
600 	if (unlikely(qp->req.state == QP_STATE_RESET)) {
601 		qp->req.wqe_index = consumer_index(qp->sq.queue);
602 		qp->req.opcode = -1;
603 		qp->req.need_rd_atomic = 0;
604 		qp->req.wait_psn = 0;
605 		qp->req.need_retry = 0;
606 		goto exit;
607 	}
608 
609 	if (unlikely(qp->req.need_retry)) {
610 		req_retry(qp);
611 		qp->req.need_retry = 0;
612 	}
613 
614 	wqe = req_next_wqe(qp);
615 	if (unlikely(!wqe))
616 		goto exit;
617 
618 	if (wqe->mask & WR_REG_MASK) {
619 		if (wqe->wr.opcode == IB_WR_LOCAL_INV) {
620 			struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
621 			struct rxe_mem *rmr;
622 
623 			rmr = rxe_pool_get_index(&rxe->mr_pool,
624 						 wqe->wr.ex.invalidate_rkey >> 8);
625 			if (!rmr) {
626 				pr_err("No mr for key %#x\n",
627 				       wqe->wr.ex.invalidate_rkey);
628 				wqe->state = wqe_state_error;
629 				wqe->status = IB_WC_MW_BIND_ERR;
630 				goto exit;
631 			}
632 			rmr->state = RXE_MEM_STATE_FREE;
633 			rxe_drop_ref(rmr);
634 			wqe->state = wqe_state_done;
635 			wqe->status = IB_WC_SUCCESS;
636 		} else if (wqe->wr.opcode == IB_WR_REG_MR) {
637 			struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr);
638 
639 			rmr->state = RXE_MEM_STATE_VALID;
640 			rmr->access = wqe->wr.wr.reg.access;
641 			rmr->lkey = wqe->wr.wr.reg.key;
642 			rmr->rkey = wqe->wr.wr.reg.key;
643 			wqe->state = wqe_state_done;
644 			wqe->status = IB_WC_SUCCESS;
645 		} else {
646 			goto exit;
647 		}
648 		if ((wqe->wr.send_flags & IB_SEND_SIGNALED) ||
649 		    qp->sq_sig_type == IB_SIGNAL_ALL_WR)
650 			rxe_run_task(&qp->comp.task, 1);
651 		qp->req.wqe_index = next_index(qp->sq.queue,
652 						qp->req.wqe_index);
653 		goto next_wqe;
654 	}
655 
656 	if (unlikely(qp_type(qp) == IB_QPT_RC &&
657 		     qp->req.psn > (qp->comp.psn + RXE_MAX_UNACKED_PSNS))) {
658 		qp->req.wait_psn = 1;
659 		goto exit;
660 	}
661 
662 	/* Limit the number of inflight SKBs per QP */
663 	if (unlikely(atomic_read(&qp->skb_out) >
664 		     RXE_INFLIGHT_SKBS_PER_QP_HIGH)) {
665 		qp->need_req_skb = 1;
666 		goto exit;
667 	}
668 
669 	opcode = next_opcode(qp, wqe, wqe->wr.opcode);
670 	if (unlikely(opcode < 0)) {
671 		wqe->status = IB_WC_LOC_QP_OP_ERR;
672 		goto exit;
673 	}
674 
675 	mask = rxe_opcode[opcode].mask;
676 	if (unlikely(mask & RXE_READ_OR_ATOMIC)) {
677 		if (check_init_depth(qp, wqe))
678 			goto exit;
679 	}
680 
681 	mtu = get_mtu(qp);
682 	payload = (mask & RXE_WRITE_OR_SEND) ? wqe->dma.resid : 0;
683 	if (payload > mtu) {
684 		if (qp_type(qp) == IB_QPT_UD) {
685 			/* C10-93.1.1: If the total sum of all the buffer lengths specified for a
686 			 * UD message exceeds the MTU of the port as returned by QueryHCA, the CI
687 			 * shall not emit any packets for this message. Further, the CI shall not
688 			 * generate an error due to this condition.
689 			 */
690 
691 			/* fake a successful UD send */
692 			wqe->first_psn = qp->req.psn;
693 			wqe->last_psn = qp->req.psn;
694 			qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
695 			qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;
696 			qp->req.wqe_index = next_index(qp->sq.queue,
697 						       qp->req.wqe_index);
698 			wqe->state = wqe_state_done;
699 			wqe->status = IB_WC_SUCCESS;
700 			__rxe_do_task(&qp->comp.task);
701 			rxe_drop_ref(qp);
702 			return 0;
703 		}
704 		payload = mtu;
705 	}
706 
707 	skb = init_req_packet(qp, wqe, opcode, payload, &pkt);
708 	if (unlikely(!skb)) {
709 		pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
710 		goto err;
711 	}
712 
713 	if (fill_packet(qp, wqe, &pkt, skb, payload)) {
714 		pr_debug("qp#%d Error during fill packet\n", qp_num(qp));
715 		kfree_skb(skb);
716 		goto err;
717 	}
718 
719 	/*
720 	 * To prevent a race on wqe access between requester and completer,
721 	 * wqe members state and psn need to be set before calling
722 	 * rxe_xmit_packet().
723 	 * Otherwise, completer might initiate an unjustified retry flow.
724 	 */
725 	save_state(wqe, qp, &rollback_wqe, &rollback_psn);
726 	update_wqe_state(qp, wqe, &pkt);
727 	update_wqe_psn(qp, wqe, &pkt, payload);
728 	ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb);
729 	if (ret) {
730 		qp->need_req_skb = 1;
731 
732 		rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
733 
734 		if (ret == -EAGAIN) {
735 			rxe_run_task(&qp->req.task, 1);
736 			goto exit;
737 		}
738 
739 		goto err;
740 	}
741 
742 	update_state(qp, wqe, &pkt, payload);
743 
744 	goto next_wqe;
745 
746 err:
747 	wqe->status = IB_WC_LOC_PROT_ERR;
748 	wqe->state = wqe_state_error;
749 	__rxe_do_task(&qp->comp.task);
750 
751 exit:
752 	rxe_drop_ref(qp);
753 	return -EAGAIN;
754 }
755