xref: /openbmc/linux/drivers/infiniband/hw/hfi1/qp.c (revision 5d0e4d78)
1 /*
2  * Copyright(c) 2015, 2016 Intel Corporation.
3  *
4  * This file is provided under a dual BSD/GPLv2 license.  When using or
5  * redistributing this file, you may do so under either license.
6  *
7  * GPL LICENSE SUMMARY
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of version 2 of the GNU General Public License as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  * BSD LICENSE
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions
22  * are met:
23  *
24  *  - Redistributions of source code must retain the above copyright
25  *    notice, this list of conditions and the following disclaimer.
26  *  - Redistributions in binary form must reproduce the above copyright
27  *    notice, this list of conditions and the following disclaimer in
28  *    the documentation and/or other materials provided with the
29  *    distribution.
30  *  - Neither the name of Intel Corporation nor the names of its
31  *    contributors may be used to endorse or promote products derived
32  *    from this software without specific prior written permission.
33  *
34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  *
46  */
47 
48 #include <linux/err.h>
49 #include <linux/vmalloc.h>
50 #include <linux/hash.h>
51 #include <linux/module.h>
52 #include <linux/seq_file.h>
53 #include <rdma/rdma_vt.h>
54 #include <rdma/rdmavt_qp.h>
55 #include <rdma/ib_verbs.h>
56 
57 #include "hfi.h"
58 #include "qp.h"
59 #include "trace.h"
60 #include "verbs_txreq.h"
61 
62 unsigned int hfi1_qp_table_size = 256;
63 module_param_named(qp_table_size, hfi1_qp_table_size, uint, S_IRUGO);
64 MODULE_PARM_DESC(qp_table_size, "QP table size");
65 
66 static void flush_tx_list(struct rvt_qp *qp);
67 static int iowait_sleep(
68 	struct sdma_engine *sde,
69 	struct iowait *wait,
70 	struct sdma_txreq *stx,
71 	unsigned seq);
72 static void iowait_wakeup(struct iowait *wait, int reason);
73 static void iowait_sdma_drained(struct iowait *wait);
74 static void qp_pio_drain(struct rvt_qp *qp);
75 
76 static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
77 			      struct rvt_qpn_map *map, unsigned off)
78 {
79 	return (map - qpt->map) * RVT_BITS_PER_PAGE + off;
80 }
81 
82 const struct rvt_operation_params hfi1_post_parms[RVT_OPERATION_MAX] = {
83 [IB_WR_RDMA_WRITE] = {
84 	.length = sizeof(struct ib_rdma_wr),
85 	.qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
86 },
87 
88 [IB_WR_RDMA_READ] = {
89 	.length = sizeof(struct ib_rdma_wr),
90 	.qpt_support = BIT(IB_QPT_RC),
91 	.flags = RVT_OPERATION_ATOMIC,
92 },
93 
94 [IB_WR_ATOMIC_CMP_AND_SWP] = {
95 	.length = sizeof(struct ib_atomic_wr),
96 	.qpt_support = BIT(IB_QPT_RC),
97 	.flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
98 },
99 
100 [IB_WR_ATOMIC_FETCH_AND_ADD] = {
101 	.length = sizeof(struct ib_atomic_wr),
102 	.qpt_support = BIT(IB_QPT_RC),
103 	.flags = RVT_OPERATION_ATOMIC | RVT_OPERATION_ATOMIC_SGE,
104 },
105 
106 [IB_WR_RDMA_WRITE_WITH_IMM] = {
107 	.length = sizeof(struct ib_rdma_wr),
108 	.qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
109 },
110 
111 [IB_WR_SEND] = {
112 	.length = sizeof(struct ib_send_wr),
113 	.qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
114 		       BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
115 },
116 
117 [IB_WR_SEND_WITH_IMM] = {
118 	.length = sizeof(struct ib_send_wr),
119 	.qpt_support = BIT(IB_QPT_UD) | BIT(IB_QPT_SMI) | BIT(IB_QPT_GSI) |
120 		       BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
121 },
122 
123 [IB_WR_REG_MR] = {
124 	.length = sizeof(struct ib_reg_wr),
125 	.qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
126 	.flags = RVT_OPERATION_LOCAL,
127 },
128 
129 [IB_WR_LOCAL_INV] = {
130 	.length = sizeof(struct ib_send_wr),
131 	.qpt_support = BIT(IB_QPT_UC) | BIT(IB_QPT_RC),
132 	.flags = RVT_OPERATION_LOCAL,
133 },
134 
135 [IB_WR_SEND_WITH_INV] = {
136 	.length = sizeof(struct ib_send_wr),
137 	.qpt_support = BIT(IB_QPT_RC),
138 },
139 
140 };
141 
142 static void flush_tx_list(struct rvt_qp *qp)
143 {
144 	struct hfi1_qp_priv *priv = qp->priv;
145 
146 	while (!list_empty(&priv->s_iowait.tx_head)) {
147 		struct sdma_txreq *tx;
148 
149 		tx = list_first_entry(
150 			&priv->s_iowait.tx_head,
151 			struct sdma_txreq,
152 			list);
153 		list_del_init(&tx->list);
154 		hfi1_put_txreq(
155 			container_of(tx, struct verbs_txreq, txreq));
156 	}
157 }
158 
159 static void flush_iowait(struct rvt_qp *qp)
160 {
161 	struct hfi1_qp_priv *priv = qp->priv;
162 	unsigned long flags;
163 	seqlock_t *lock = priv->s_iowait.lock;
164 
165 	if (!lock)
166 		return;
167 	write_seqlock_irqsave(lock, flags);
168 	if (!list_empty(&priv->s_iowait.list)) {
169 		list_del_init(&priv->s_iowait.list);
170 		priv->s_iowait.lock = NULL;
171 		rvt_put_qp(qp);
172 	}
173 	write_sequnlock_irqrestore(lock, flags);
174 }
175 
176 static inline int opa_mtu_enum_to_int(int mtu)
177 {
178 	switch (mtu) {
179 	case OPA_MTU_8192:  return 8192;
180 	case OPA_MTU_10240: return 10240;
181 	default:            return -1;
182 	}
183 }
184 
185 /**
186  * This function is what we would push to the core layer if we wanted to be a
187  * "first class citizen".  Instead we hide this here and rely on Verbs ULPs
188  * to blindly pass the MTU enum value from the PathRecord to us.
189  */
190 static inline int verbs_mtu_enum_to_int(struct ib_device *dev, enum ib_mtu mtu)
191 {
192 	int val;
193 
194 	/* Constraining 10KB packets to 8KB packets */
195 	if (mtu == (enum ib_mtu)OPA_MTU_10240)
196 		mtu = OPA_MTU_8192;
197 	val = opa_mtu_enum_to_int((int)mtu);
198 	if (val > 0)
199 		return val;
200 	return ib_mtu_enum_to_int(mtu);
201 }
202 
203 int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
204 			 int attr_mask, struct ib_udata *udata)
205 {
206 	struct ib_qp *ibqp = &qp->ibqp;
207 	struct hfi1_ibdev *dev = to_idev(ibqp->device);
208 	struct hfi1_devdata *dd = dd_from_dev(dev);
209 	u8 sc;
210 
211 	if (attr_mask & IB_QP_AV) {
212 		sc = ah_to_sc(ibqp->device, &attr->ah_attr);
213 		if (sc == 0xf)
214 			return -EINVAL;
215 
216 		if (!qp_to_sdma_engine(qp, sc) &&
217 		    dd->flags & HFI1_HAS_SEND_DMA)
218 			return -EINVAL;
219 
220 		if (!qp_to_send_context(qp, sc))
221 			return -EINVAL;
222 	}
223 
224 	if (attr_mask & IB_QP_ALT_PATH) {
225 		sc = ah_to_sc(ibqp->device, &attr->alt_ah_attr);
226 		if (sc == 0xf)
227 			return -EINVAL;
228 
229 		if (!qp_to_sdma_engine(qp, sc) &&
230 		    dd->flags & HFI1_HAS_SEND_DMA)
231 			return -EINVAL;
232 
233 		if (!qp_to_send_context(qp, sc))
234 			return -EINVAL;
235 	}
236 
237 	return 0;
238 }
239 
240 void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr,
241 		    int attr_mask, struct ib_udata *udata)
242 {
243 	struct ib_qp *ibqp = &qp->ibqp;
244 	struct hfi1_qp_priv *priv = qp->priv;
245 
246 	if (attr_mask & IB_QP_AV) {
247 		priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
248 		priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
249 		priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
250 	}
251 
252 	if (attr_mask & IB_QP_PATH_MIG_STATE &&
253 	    attr->path_mig_state == IB_MIG_MIGRATED &&
254 	    qp->s_mig_state == IB_MIG_ARMED) {
255 		qp->s_flags |= RVT_S_AHG_CLEAR;
256 		priv->s_sc = ah_to_sc(ibqp->device, &qp->remote_ah_attr);
257 		priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
258 		priv->s_sendcontext = qp_to_send_context(qp, priv->s_sc);
259 	}
260 }
261 
262 /**
263  * hfi1_check_send_wqe - validate wqe
264  * @qp - The qp
265  * @wqe - The built wqe
266  *
267  * validate wqe.  This is called
268  * prior to inserting the wqe into
269  * the ring but after the wqe has been
270  * setup.
271  *
272  * Returns 0 on success, -EINVAL on failure
273  *
274  */
275 int hfi1_check_send_wqe(struct rvt_qp *qp,
276 			struct rvt_swqe *wqe)
277 {
278 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
279 	struct rvt_ah *ah;
280 
281 	switch (qp->ibqp.qp_type) {
282 	case IB_QPT_RC:
283 	case IB_QPT_UC:
284 		if (wqe->length > 0x80000000U)
285 			return -EINVAL;
286 		break;
287 	case IB_QPT_SMI:
288 		ah = ibah_to_rvtah(wqe->ud_wr.ah);
289 		if (wqe->length > (1 << ah->log_pmtu))
290 			return -EINVAL;
291 		break;
292 	case IB_QPT_GSI:
293 	case IB_QPT_UD:
294 		ah = ibah_to_rvtah(wqe->ud_wr.ah);
295 		if (wqe->length > (1 << ah->log_pmtu))
296 			return -EINVAL;
297 		if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf)
298 			return -EINVAL;
299 	default:
300 		break;
301 	}
302 	return wqe->length <= piothreshold;
303 }
304 
305 /**
306  * _hfi1_schedule_send - schedule progress
307  * @qp: the QP
308  *
309  * This schedules qp progress w/o regard to the s_flags.
310  *
311  * It is only used in the post send, which doesn't hold
312  * the s_lock.
313  */
314 void _hfi1_schedule_send(struct rvt_qp *qp)
315 {
316 	struct hfi1_qp_priv *priv = qp->priv;
317 	struct hfi1_ibport *ibp =
318 		to_iport(qp->ibqp.device, qp->port_num);
319 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
320 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
321 
322 	iowait_schedule(&priv->s_iowait, ppd->hfi1_wq,
323 			priv->s_sde ?
324 			priv->s_sde->cpu :
325 			cpumask_first(cpumask_of_node(dd->node)));
326 }
327 
328 static void qp_pio_drain(struct rvt_qp *qp)
329 {
330 	struct hfi1_ibdev *dev;
331 	struct hfi1_qp_priv *priv = qp->priv;
332 
333 	if (!priv->s_sendcontext)
334 		return;
335 	dev = to_idev(qp->ibqp.device);
336 	while (iowait_pio_pending(&priv->s_iowait)) {
337 		write_seqlock_irq(&dev->iowait_lock);
338 		hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 1);
339 		write_sequnlock_irq(&dev->iowait_lock);
340 		iowait_pio_drain(&priv->s_iowait);
341 		write_seqlock_irq(&dev->iowait_lock);
342 		hfi1_sc_wantpiobuf_intr(priv->s_sendcontext, 0);
343 		write_sequnlock_irq(&dev->iowait_lock);
344 	}
345 }
346 
347 /**
348  * hfi1_schedule_send - schedule progress
349  * @qp: the QP
350  *
351  * This schedules qp progress and caller should hold
352  * the s_lock.
353  */
354 void hfi1_schedule_send(struct rvt_qp *qp)
355 {
356 	lockdep_assert_held(&qp->s_lock);
357 	if (hfi1_send_ok(qp))
358 		_hfi1_schedule_send(qp);
359 }
360 
361 void hfi1_qp_wakeup(struct rvt_qp *qp, u32 flag)
362 {
363 	unsigned long flags;
364 
365 	spin_lock_irqsave(&qp->s_lock, flags);
366 	if (qp->s_flags & flag) {
367 		qp->s_flags &= ~flag;
368 		trace_hfi1_qpwakeup(qp, flag);
369 		hfi1_schedule_send(qp);
370 	}
371 	spin_unlock_irqrestore(&qp->s_lock, flags);
372 	/* Notify hfi1_destroy_qp() if it is waiting. */
373 	rvt_put_qp(qp);
374 }
375 
376 static int iowait_sleep(
377 	struct sdma_engine *sde,
378 	struct iowait *wait,
379 	struct sdma_txreq *stx,
380 	unsigned seq)
381 {
382 	struct verbs_txreq *tx = container_of(stx, struct verbs_txreq, txreq);
383 	struct rvt_qp *qp;
384 	struct hfi1_qp_priv *priv;
385 	unsigned long flags;
386 	int ret = 0;
387 	struct hfi1_ibdev *dev;
388 
389 	qp = tx->qp;
390 	priv = qp->priv;
391 
392 	spin_lock_irqsave(&qp->s_lock, flags);
393 	if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
394 		/*
395 		 * If we couldn't queue the DMA request, save the info
396 		 * and try again later rather than destroying the
397 		 * buffer and undoing the side effects of the copy.
398 		 */
399 		/* Make a common routine? */
400 		dev = &sde->dd->verbs_dev;
401 		list_add_tail(&stx->list, &wait->tx_head);
402 		write_seqlock(&dev->iowait_lock);
403 		if (sdma_progress(sde, seq, stx))
404 			goto eagain;
405 		if (list_empty(&priv->s_iowait.list)) {
406 			struct hfi1_ibport *ibp =
407 				to_iport(qp->ibqp.device, qp->port_num);
408 
409 			ibp->rvp.n_dmawait++;
410 			qp->s_flags |= RVT_S_WAIT_DMA_DESC;
411 			list_add_tail(&priv->s_iowait.list, &sde->dmawait);
412 			priv->s_iowait.lock = &dev->iowait_lock;
413 			trace_hfi1_qpsleep(qp, RVT_S_WAIT_DMA_DESC);
414 			rvt_get_qp(qp);
415 		}
416 		write_sequnlock(&dev->iowait_lock);
417 		qp->s_flags &= ~RVT_S_BUSY;
418 		spin_unlock_irqrestore(&qp->s_lock, flags);
419 		ret = -EBUSY;
420 	} else {
421 		spin_unlock_irqrestore(&qp->s_lock, flags);
422 		hfi1_put_txreq(tx);
423 	}
424 	return ret;
425 eagain:
426 	write_sequnlock(&dev->iowait_lock);
427 	spin_unlock_irqrestore(&qp->s_lock, flags);
428 	list_del_init(&stx->list);
429 	return -EAGAIN;
430 }
431 
432 static void iowait_wakeup(struct iowait *wait, int reason)
433 {
434 	struct rvt_qp *qp = iowait_to_qp(wait);
435 
436 	WARN_ON(reason != SDMA_AVAIL_REASON);
437 	hfi1_qp_wakeup(qp, RVT_S_WAIT_DMA_DESC);
438 }
439 
440 static void iowait_sdma_drained(struct iowait *wait)
441 {
442 	struct rvt_qp *qp = iowait_to_qp(wait);
443 	unsigned long flags;
444 
445 	/*
446 	 * This happens when the send engine notes
447 	 * a QP in the error state and cannot
448 	 * do the flush work until that QP's
449 	 * sdma work has finished.
450 	 */
451 	spin_lock_irqsave(&qp->s_lock, flags);
452 	if (qp->s_flags & RVT_S_WAIT_DMA) {
453 		qp->s_flags &= ~RVT_S_WAIT_DMA;
454 		hfi1_schedule_send(qp);
455 	}
456 	spin_unlock_irqrestore(&qp->s_lock, flags);
457 }
458 
459 /**
460  *
461  * qp_to_sdma_engine - map a qp to a send engine
462  * @qp: the QP
463  * @sc5: the 5 bit sc
464  *
465  * Return:
466  * A send engine for the qp or NULL for SMI type qp.
467  */
468 struct sdma_engine *qp_to_sdma_engine(struct rvt_qp *qp, u8 sc5)
469 {
470 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
471 	struct sdma_engine *sde;
472 
473 	if (!(dd->flags & HFI1_HAS_SEND_DMA))
474 		return NULL;
475 	switch (qp->ibqp.qp_type) {
476 	case IB_QPT_SMI:
477 		return NULL;
478 	default:
479 		break;
480 	}
481 	sde = sdma_select_engine_sc(dd, qp->ibqp.qp_num >> dd->qos_shift, sc5);
482 	return sde;
483 }
484 
485 /*
486  * qp_to_send_context - map a qp to a send context
487  * @qp: the QP
488  * @sc5: the 5 bit sc
489  *
490  * Return:
491  * A send context for the qp
492  */
493 struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5)
494 {
495 	struct hfi1_devdata *dd = dd_from_ibdev(qp->ibqp.device);
496 
497 	switch (qp->ibqp.qp_type) {
498 	case IB_QPT_SMI:
499 		/* SMA packets to VL15 */
500 		return dd->vld[15].sc;
501 	default:
502 		break;
503 	}
504 
505 	return pio_select_send_context_sc(dd, qp->ibqp.qp_num >> dd->qos_shift,
506 					  sc5);
507 }
508 
509 struct qp_iter {
510 	struct hfi1_ibdev *dev;
511 	struct rvt_qp *qp;
512 	int specials;
513 	int n;
514 };
515 
516 struct qp_iter *qp_iter_init(struct hfi1_ibdev *dev)
517 {
518 	struct qp_iter *iter;
519 
520 	iter = kzalloc(sizeof(*iter), GFP_KERNEL);
521 	if (!iter)
522 		return NULL;
523 
524 	iter->dev = dev;
525 	iter->specials = dev->rdi.ibdev.phys_port_cnt * 2;
526 
527 	return iter;
528 }
529 
530 int qp_iter_next(struct qp_iter *iter)
531 {
532 	struct hfi1_ibdev *dev = iter->dev;
533 	int n = iter->n;
534 	int ret = 1;
535 	struct rvt_qp *pqp = iter->qp;
536 	struct rvt_qp *qp;
537 
538 	/*
539 	 * The approach is to consider the special qps
540 	 * as an additional table entries before the
541 	 * real hash table.  Since the qp code sets
542 	 * the qp->next hash link to NULL, this works just fine.
543 	 *
544 	 * iter->specials is 2 * # ports
545 	 *
546 	 * n = 0..iter->specials is the special qp indices
547 	 *
548 	 * n = iter->specials..dev->rdi.qp_dev->qp_table_size+iter->specials are
549 	 * the potential hash bucket entries
550 	 *
551 	 */
552 	for (; n <  dev->rdi.qp_dev->qp_table_size + iter->specials; n++) {
553 		if (pqp) {
554 			qp = rcu_dereference(pqp->next);
555 		} else {
556 			if (n < iter->specials) {
557 				struct hfi1_pportdata *ppd;
558 				struct hfi1_ibport *ibp;
559 				int pidx;
560 
561 				pidx = n % dev->rdi.ibdev.phys_port_cnt;
562 				ppd = &dd_from_dev(dev)->pport[pidx];
563 				ibp = &ppd->ibport_data;
564 
565 				if (!(n & 1))
566 					qp = rcu_dereference(ibp->rvp.qp[0]);
567 				else
568 					qp = rcu_dereference(ibp->rvp.qp[1]);
569 			} else {
570 				qp = rcu_dereference(
571 					dev->rdi.qp_dev->qp_table[
572 						(n - iter->specials)]);
573 			}
574 		}
575 		pqp = qp;
576 		if (qp) {
577 			iter->qp = qp;
578 			iter->n = n;
579 			return 0;
580 		}
581 	}
582 	return ret;
583 }
584 
585 static const char * const qp_type_str[] = {
586 	"SMI", "GSI", "RC", "UC", "UD",
587 };
588 
589 static int qp_idle(struct rvt_qp *qp)
590 {
591 	return
592 		qp->s_last == qp->s_acked &&
593 		qp->s_acked == qp->s_cur &&
594 		qp->s_cur == qp->s_tail &&
595 		qp->s_tail == qp->s_head;
596 }
597 
598 void qp_iter_print(struct seq_file *s, struct qp_iter *iter)
599 {
600 	struct rvt_swqe *wqe;
601 	struct rvt_qp *qp = iter->qp;
602 	struct hfi1_qp_priv *priv = qp->priv;
603 	struct sdma_engine *sde;
604 	struct send_context *send_context;
605 
606 	sde = qp_to_sdma_engine(qp, priv->s_sc);
607 	wqe = rvt_get_swqe_ptr(qp, qp->s_last);
608 	send_context = qp_to_send_context(qp, priv->s_sc);
609 	seq_printf(s,
610 		   "N %d %s QP %x R %u %s %u %u %u f=%x %u %u %u %u %u %u SPSN %x %x %x %x %x RPSN %x (%u %u %u %u %u %u %u) RQP %x LID %x SL %u MTU %u %u %u %u %u SDE %p,%u SC %p,%u SCQ %u %u PID %d\n",
611 		   iter->n,
612 		   qp_idle(qp) ? "I" : "B",
613 		   qp->ibqp.qp_num,
614 		   atomic_read(&qp->refcount),
615 		   qp_type_str[qp->ibqp.qp_type],
616 		   qp->state,
617 		   wqe ? wqe->wr.opcode : 0,
618 		   qp->s_hdrwords,
619 		   qp->s_flags,
620 		   iowait_sdma_pending(&priv->s_iowait),
621 		   iowait_pio_pending(&priv->s_iowait),
622 		   !list_empty(&priv->s_iowait.list),
623 		   qp->timeout,
624 		   wqe ? wqe->ssn : 0,
625 		   qp->s_lsn,
626 		   qp->s_last_psn,
627 		   qp->s_psn, qp->s_next_psn,
628 		   qp->s_sending_psn, qp->s_sending_hpsn,
629 		   qp->r_psn,
630 		   qp->s_last, qp->s_acked, qp->s_cur,
631 		   qp->s_tail, qp->s_head, qp->s_size,
632 		   qp->s_avail,
633 		   qp->remote_qpn,
634 		   rdma_ah_get_dlid(&qp->remote_ah_attr),
635 		   rdma_ah_get_sl(&qp->remote_ah_attr),
636 		   qp->pmtu,
637 		   qp->s_retry,
638 		   qp->s_retry_cnt,
639 		   qp->s_rnr_retry_cnt,
640 		   qp->s_rnr_retry,
641 		   sde,
642 		   sde ? sde->this_idx : 0,
643 		   send_context,
644 		   send_context ? send_context->sw_index : 0,
645 		   ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head,
646 		   ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail,
647 		   qp->pid);
648 }
649 
650 void *qp_priv_alloc(struct rvt_dev_info *rdi, struct rvt_qp *qp)
651 {
652 	struct hfi1_qp_priv *priv;
653 
654 	priv = kzalloc_node(sizeof(*priv), GFP_KERNEL, rdi->dparms.node);
655 	if (!priv)
656 		return ERR_PTR(-ENOMEM);
657 
658 	priv->owner = qp;
659 
660 	priv->s_ahg = kzalloc_node(sizeof(*priv->s_ahg), GFP_KERNEL,
661 				   rdi->dparms.node);
662 	if (!priv->s_ahg) {
663 		kfree(priv);
664 		return ERR_PTR(-ENOMEM);
665 	}
666 	iowait_init(
667 		&priv->s_iowait,
668 		1,
669 		_hfi1_do_send,
670 		iowait_sleep,
671 		iowait_wakeup,
672 		iowait_sdma_drained);
673 	return priv;
674 }
675 
676 void qp_priv_free(struct rvt_dev_info *rdi, struct rvt_qp *qp)
677 {
678 	struct hfi1_qp_priv *priv = qp->priv;
679 
680 	kfree(priv->s_ahg);
681 	kfree(priv);
682 }
683 
684 unsigned free_all_qps(struct rvt_dev_info *rdi)
685 {
686 	struct hfi1_ibdev *verbs_dev = container_of(rdi,
687 						    struct hfi1_ibdev,
688 						    rdi);
689 	struct hfi1_devdata *dd = container_of(verbs_dev,
690 					       struct hfi1_devdata,
691 					       verbs_dev);
692 	int n;
693 	unsigned qp_inuse = 0;
694 
695 	for (n = 0; n < dd->num_pports; n++) {
696 		struct hfi1_ibport *ibp = &dd->pport[n].ibport_data;
697 
698 		rcu_read_lock();
699 		if (rcu_dereference(ibp->rvp.qp[0]))
700 			qp_inuse++;
701 		if (rcu_dereference(ibp->rvp.qp[1]))
702 			qp_inuse++;
703 		rcu_read_unlock();
704 	}
705 
706 	return qp_inuse;
707 }
708 
709 void flush_qp_waiters(struct rvt_qp *qp)
710 {
711 	lockdep_assert_held(&qp->s_lock);
712 	flush_iowait(qp);
713 }
714 
715 void stop_send_queue(struct rvt_qp *qp)
716 {
717 	struct hfi1_qp_priv *priv = qp->priv;
718 
719 	cancel_work_sync(&priv->s_iowait.iowork);
720 }
721 
722 void quiesce_qp(struct rvt_qp *qp)
723 {
724 	struct hfi1_qp_priv *priv = qp->priv;
725 
726 	iowait_sdma_drain(&priv->s_iowait);
727 	qp_pio_drain(qp);
728 	flush_tx_list(qp);
729 }
730 
731 void notify_qp_reset(struct rvt_qp *qp)
732 {
733 	qp->r_adefered = 0;
734 	clear_ahg(qp);
735 }
736 
737 /*
738  * Switch to alternate path.
739  * The QP s_lock should be held and interrupts disabled.
740  */
741 void hfi1_migrate_qp(struct rvt_qp *qp)
742 {
743 	struct hfi1_qp_priv *priv = qp->priv;
744 	struct ib_event ev;
745 
746 	qp->s_mig_state = IB_MIG_MIGRATED;
747 	qp->remote_ah_attr = qp->alt_ah_attr;
748 	qp->port_num = rdma_ah_get_port_num(&qp->alt_ah_attr);
749 	qp->s_pkey_index = qp->s_alt_pkey_index;
750 	qp->s_flags |= RVT_S_AHG_CLEAR;
751 	priv->s_sc = ah_to_sc(qp->ibqp.device, &qp->remote_ah_attr);
752 	priv->s_sde = qp_to_sdma_engine(qp, priv->s_sc);
753 
754 	ev.device = qp->ibqp.device;
755 	ev.element.qp = &qp->ibqp;
756 	ev.event = IB_EVENT_PATH_MIG;
757 	qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
758 }
759 
760 int mtu_to_path_mtu(u32 mtu)
761 {
762 	return mtu_to_enum(mtu, OPA_MTU_8192);
763 }
764 
765 u32 mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu)
766 {
767 	u32 mtu;
768 	struct hfi1_ibdev *verbs_dev = container_of(rdi,
769 						    struct hfi1_ibdev,
770 						    rdi);
771 	struct hfi1_devdata *dd = container_of(verbs_dev,
772 					       struct hfi1_devdata,
773 					       verbs_dev);
774 	struct hfi1_ibport *ibp;
775 	u8 sc, vl;
776 
777 	ibp = &dd->pport[qp->port_num - 1].ibport_data;
778 	sc = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
779 	vl = sc_to_vlt(dd, sc);
780 
781 	mtu = verbs_mtu_enum_to_int(qp->ibqp.device, pmtu);
782 	if (vl < PER_VL_SEND_CONTEXTS)
783 		mtu = min_t(u32, mtu, dd->vld[vl].mtu);
784 	return mtu;
785 }
786 
787 int get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp,
788 		       struct ib_qp_attr *attr)
789 {
790 	int mtu, pidx = qp->port_num - 1;
791 	struct hfi1_ibdev *verbs_dev = container_of(rdi,
792 						    struct hfi1_ibdev,
793 						    rdi);
794 	struct hfi1_devdata *dd = container_of(verbs_dev,
795 					       struct hfi1_devdata,
796 					       verbs_dev);
797 	mtu = verbs_mtu_enum_to_int(qp->ibqp.device, attr->path_mtu);
798 	if (mtu == -1)
799 		return -1; /* values less than 0 are error */
800 
801 	if (mtu > dd->pport[pidx].ibmtu)
802 		return mtu_to_enum(dd->pport[pidx].ibmtu, IB_MTU_2048);
803 	else
804 		return attr->path_mtu;
805 }
806 
807 void notify_error_qp(struct rvt_qp *qp)
808 {
809 	struct hfi1_qp_priv *priv = qp->priv;
810 	seqlock_t *lock = priv->s_iowait.lock;
811 
812 	if (lock) {
813 		write_seqlock(lock);
814 		if (!list_empty(&priv->s_iowait.list) &&
815 		    !(qp->s_flags & RVT_S_BUSY)) {
816 			qp->s_flags &= ~RVT_S_ANY_WAIT_IO;
817 			list_del_init(&priv->s_iowait.list);
818 			priv->s_iowait.lock = NULL;
819 			rvt_put_qp(qp);
820 		}
821 		write_sequnlock(lock);
822 	}
823 
824 	if (!(qp->s_flags & RVT_S_BUSY)) {
825 		qp->s_hdrwords = 0;
826 		if (qp->s_rdma_mr) {
827 			rvt_put_mr(qp->s_rdma_mr);
828 			qp->s_rdma_mr = NULL;
829 		}
830 		flush_tx_list(qp);
831 	}
832 }
833 
834 /**
835  * hfi1_error_port_qps - put a port's RC/UC qps into error state
836  * @ibp: the ibport.
837  * @sl: the service level.
838  *
839  * This function places all RC/UC qps with a given service level into error
840  * state. It is generally called to force upper lay apps to abandon stale qps
841  * after an sl->sc mapping change.
842  */
843 void hfi1_error_port_qps(struct hfi1_ibport *ibp, u8 sl)
844 {
845 	struct rvt_qp *qp = NULL;
846 	struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
847 	struct hfi1_ibdev *dev = &ppd->dd->verbs_dev;
848 	int n;
849 	int lastwqe;
850 	struct ib_event ev;
851 
852 	rcu_read_lock();
853 
854 	/* Deal only with RC/UC qps that use the given SL. */
855 	for (n = 0; n < dev->rdi.qp_dev->qp_table_size; n++) {
856 		for (qp = rcu_dereference(dev->rdi.qp_dev->qp_table[n]); qp;
857 			qp = rcu_dereference(qp->next)) {
858 			if (qp->port_num == ppd->port &&
859 			    (qp->ibqp.qp_type == IB_QPT_UC ||
860 			     qp->ibqp.qp_type == IB_QPT_RC) &&
861 			    rdma_ah_get_sl(&qp->remote_ah_attr) == sl &&
862 			    (ib_rvt_state_ops[qp->state] &
863 			     RVT_POST_SEND_OK)) {
864 				spin_lock_irq(&qp->r_lock);
865 				spin_lock(&qp->s_hlock);
866 				spin_lock(&qp->s_lock);
867 				lastwqe = rvt_error_qp(qp,
868 						       IB_WC_WR_FLUSH_ERR);
869 				spin_unlock(&qp->s_lock);
870 				spin_unlock(&qp->s_hlock);
871 				spin_unlock_irq(&qp->r_lock);
872 				if (lastwqe) {
873 					ev.device = qp->ibqp.device;
874 					ev.element.qp = &qp->ibqp;
875 					ev.event =
876 						IB_EVENT_QP_LAST_WQE_REACHED;
877 					qp->ibqp.event_handler(&ev,
878 						qp->ibqp.qp_context);
879 				}
880 			}
881 		}
882 	}
883 
884 	rcu_read_unlock();
885 }
886