1*145eba1aSCai Huoqing // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2f48ad614SDennis Dalessandro /*
3b697d7d8SMichael J. Ruhl * Copyright(c) 2016 - 2018 Intel Corporation.
4f48ad614SDennis Dalessandro */
5f48ad614SDennis Dalessandro
6f48ad614SDennis Dalessandro #include "hfi.h"
7f48ad614SDennis Dalessandro #include "verbs_txreq.h"
8f48ad614SDennis Dalessandro #include "qp.h"
9f48ad614SDennis Dalessandro #include "trace.h"
10f48ad614SDennis Dalessandro
11f48ad614SDennis Dalessandro #define TXREQ_LEN 24
12f48ad614SDennis Dalessandro
hfi1_put_txreq(struct verbs_txreq * tx)13f48ad614SDennis Dalessandro void hfi1_put_txreq(struct verbs_txreq *tx)
14f48ad614SDennis Dalessandro {
15f48ad614SDennis Dalessandro struct hfi1_ibdev *dev;
16f48ad614SDennis Dalessandro struct rvt_qp *qp;
17f48ad614SDennis Dalessandro unsigned long flags;
18f48ad614SDennis Dalessandro unsigned int seq;
19f48ad614SDennis Dalessandro struct hfi1_qp_priv *priv;
20f48ad614SDennis Dalessandro
21f48ad614SDennis Dalessandro qp = tx->qp;
22f48ad614SDennis Dalessandro dev = to_idev(qp->ibqp.device);
23f48ad614SDennis Dalessandro
24f48ad614SDennis Dalessandro if (tx->mr)
25f48ad614SDennis Dalessandro rvt_put_mr(tx->mr);
26f48ad614SDennis Dalessandro
27f48ad614SDennis Dalessandro sdma_txclean(dd_from_dev(dev), &tx->txreq);
28f48ad614SDennis Dalessandro
29f48ad614SDennis Dalessandro /* Free verbs_txreq and return to slab cache */
30f48ad614SDennis Dalessandro kmem_cache_free(dev->verbs_txreq_cache, tx);
31f48ad614SDennis Dalessandro
32f48ad614SDennis Dalessandro do {
334e045572SMike Marciniszyn seq = read_seqbegin(&dev->txwait_lock);
34f48ad614SDennis Dalessandro if (!list_empty(&dev->txwait)) {
35f48ad614SDennis Dalessandro struct iowait *wait;
36f48ad614SDennis Dalessandro
374e045572SMike Marciniszyn write_seqlock_irqsave(&dev->txwait_lock, flags);
38f48ad614SDennis Dalessandro wait = list_first_entry(&dev->txwait, struct iowait,
39f48ad614SDennis Dalessandro list);
40f48ad614SDennis Dalessandro qp = iowait_to_qp(wait);
41f48ad614SDennis Dalessandro priv = qp->priv;
42f48ad614SDennis Dalessandro list_del_init(&priv->s_iowait.list);
43f48ad614SDennis Dalessandro /* refcount held until actual wake up */
444e045572SMike Marciniszyn write_sequnlock_irqrestore(&dev->txwait_lock, flags);
45f48ad614SDennis Dalessandro hfi1_qp_wakeup(qp, RVT_S_WAIT_TX);
46f48ad614SDennis Dalessandro break;
47f48ad614SDennis Dalessandro }
484e045572SMike Marciniszyn } while (read_seqretry(&dev->txwait_lock, seq));
49f48ad614SDennis Dalessandro }
50f48ad614SDennis Dalessandro
__get_txreq(struct hfi1_ibdev * dev,struct rvt_qp * qp)51f48ad614SDennis Dalessandro struct verbs_txreq *__get_txreq(struct hfi1_ibdev *dev,
52f48ad614SDennis Dalessandro struct rvt_qp *qp)
532aee309dSMike Marciniszyn __must_hold(&qp->s_lock)
54f48ad614SDennis Dalessandro {
55b697d7d8SMichael J. Ruhl struct verbs_txreq *tx = NULL;
56f48ad614SDennis Dalessandro
574e045572SMike Marciniszyn write_seqlock(&dev->txwait_lock);
58f48ad614SDennis Dalessandro if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
59f48ad614SDennis Dalessandro struct hfi1_qp_priv *priv;
60f48ad614SDennis Dalessandro
613230f4a8SMike Marciniszyn tx = kmem_cache_alloc(dev->verbs_txreq_cache, VERBS_TXREQ_GFP);
62f48ad614SDennis Dalessandro if (tx)
63f48ad614SDennis Dalessandro goto out;
64f48ad614SDennis Dalessandro priv = qp->priv;
65f48ad614SDennis Dalessandro if (list_empty(&priv->s_iowait.list)) {
66f48ad614SDennis Dalessandro dev->n_txwait++;
67f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_TX;
68f48ad614SDennis Dalessandro list_add_tail(&priv->s_iowait.list, &dev->txwait);
694e045572SMike Marciniszyn priv->s_iowait.lock = &dev->txwait_lock;
70f48ad614SDennis Dalessandro trace_hfi1_qpsleep(qp, RVT_S_WAIT_TX);
714d6f85c3SMike Marciniszyn rvt_get_qp(qp);
72f48ad614SDennis Dalessandro }
73f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_BUSY;
74f48ad614SDennis Dalessandro }
75f48ad614SDennis Dalessandro out:
764e045572SMike Marciniszyn write_sequnlock(&dev->txwait_lock);
77f48ad614SDennis Dalessandro return tx;
78f48ad614SDennis Dalessandro }
79f48ad614SDennis Dalessandro
verbs_txreq_init(struct hfi1_ibdev * dev)80f48ad614SDennis Dalessandro int verbs_txreq_init(struct hfi1_ibdev *dev)
81f48ad614SDennis Dalessandro {
82f48ad614SDennis Dalessandro char buf[TXREQ_LEN];
83f48ad614SDennis Dalessandro struct hfi1_devdata *dd = dd_from_dev(dev);
84f48ad614SDennis Dalessandro
85f48ad614SDennis Dalessandro snprintf(buf, sizeof(buf), "hfi1_%u_vtxreq_cache", dd->unit);
86f48ad614SDennis Dalessandro dev->verbs_txreq_cache = kmem_cache_create(buf,
87f48ad614SDennis Dalessandro sizeof(struct verbs_txreq),
88f48ad614SDennis Dalessandro 0, SLAB_HWCACHE_ALIGN,
897956371eSMichael J. Ruhl NULL);
90f48ad614SDennis Dalessandro if (!dev->verbs_txreq_cache)
91f48ad614SDennis Dalessandro return -ENOMEM;
92f48ad614SDennis Dalessandro return 0;
93f48ad614SDennis Dalessandro }
94f48ad614SDennis Dalessandro
verbs_txreq_exit(struct hfi1_ibdev * dev)95f48ad614SDennis Dalessandro void verbs_txreq_exit(struct hfi1_ibdev *dev)
96f48ad614SDennis Dalessandro {
97f48ad614SDennis Dalessandro kmem_cache_destroy(dev->verbs_txreq_cache);
98f48ad614SDennis Dalessandro dev->verbs_txreq_cache = NULL;
99f48ad614SDennis Dalessandro }
100