1225c7b1fSRoland Dreier /*
2225c7b1fSRoland Dreier * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
351a379d0SJack Morgenstein * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4225c7b1fSRoland Dreier *
5225c7b1fSRoland Dreier * This software is available to you under a choice of one of two
6225c7b1fSRoland Dreier * licenses. You may choose to be licensed under the terms of the GNU
7225c7b1fSRoland Dreier * General Public License (GPL) Version 2, available from the file
8225c7b1fSRoland Dreier * COPYING in the main directory of this source tree, or the
9225c7b1fSRoland Dreier * OpenIB.org BSD license below:
10225c7b1fSRoland Dreier *
11225c7b1fSRoland Dreier * Redistribution and use in source and binary forms, with or
12225c7b1fSRoland Dreier * without modification, are permitted provided that the following
13225c7b1fSRoland Dreier * conditions are met:
14225c7b1fSRoland Dreier *
15225c7b1fSRoland Dreier * - Redistributions of source code must retain the above
16225c7b1fSRoland Dreier * copyright notice, this list of conditions and the following
17225c7b1fSRoland Dreier * disclaimer.
18225c7b1fSRoland Dreier *
19225c7b1fSRoland Dreier * - Redistributions in binary form must reproduce the above
20225c7b1fSRoland Dreier * copyright notice, this list of conditions and the following
21225c7b1fSRoland Dreier * disclaimer in the documentation and/or other materials
22225c7b1fSRoland Dreier * provided with the distribution.
23225c7b1fSRoland Dreier *
24225c7b1fSRoland Dreier * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25225c7b1fSRoland Dreier * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26225c7b1fSRoland Dreier * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27225c7b1fSRoland Dreier * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28225c7b1fSRoland Dreier * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29225c7b1fSRoland Dreier * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30225c7b1fSRoland Dreier * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31225c7b1fSRoland Dreier * SOFTWARE.
32225c7b1fSRoland Dreier */
33225c7b1fSRoland Dreier
34225c7b1fSRoland Dreier #include <linux/mlx4/qp.h>
35225c7b1fSRoland Dreier #include <linux/mlx4/srq.h>
365a0e3ad6STejun Heo #include <linux/slab.h>
37225c7b1fSRoland Dreier
38225c7b1fSRoland Dreier #include "mlx4_ib.h"
399ce28a20SLeon Romanovsky #include <rdma/mlx4-abi.h>
4089944450SShamir Rabinovitch #include <rdma/uverbs_ioctl.h>
41225c7b1fSRoland Dreier
get_wqe(struct mlx4_ib_srq * srq,int n)42225c7b1fSRoland Dreier static void *get_wqe(struct mlx4_ib_srq *srq, int n)
43225c7b1fSRoland Dreier {
441c69fc2aSRoland Dreier return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
45225c7b1fSRoland Dreier }
46225c7b1fSRoland Dreier
mlx4_ib_srq_event(struct mlx4_srq * srq,enum mlx4_event type)47225c7b1fSRoland Dreier static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
48225c7b1fSRoland Dreier {
49225c7b1fSRoland Dreier struct ib_event event;
50225c7b1fSRoland Dreier struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
51225c7b1fSRoland Dreier
52225c7b1fSRoland Dreier if (ibsrq->event_handler) {
53225c7b1fSRoland Dreier event.device = ibsrq->device;
54225c7b1fSRoland Dreier event.element.srq = ibsrq;
55225c7b1fSRoland Dreier switch (type) {
56225c7b1fSRoland Dreier case MLX4_EVENT_TYPE_SRQ_LIMIT:
57225c7b1fSRoland Dreier event.event = IB_EVENT_SRQ_LIMIT_REACHED;
58225c7b1fSRoland Dreier break;
59225c7b1fSRoland Dreier case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
60225c7b1fSRoland Dreier event.event = IB_EVENT_SRQ_ERR;
61225c7b1fSRoland Dreier break;
62225c7b1fSRoland Dreier default:
63987c8f8fSShlomo Pongratz pr_warn("Unexpected event type %d "
64225c7b1fSRoland Dreier "on SRQ %06x\n", type, srq->srqn);
65225c7b1fSRoland Dreier return;
66225c7b1fSRoland Dreier }
67225c7b1fSRoland Dreier
68225c7b1fSRoland Dreier ibsrq->event_handler(&event, ibsrq->srq_context);
69225c7b1fSRoland Dreier }
70225c7b1fSRoland Dreier }
71225c7b1fSRoland Dreier
mlx4_ib_create_srq(struct ib_srq * ib_srq,struct ib_srq_init_attr * init_attr,struct ib_udata * udata)7268e326deSLeon Romanovsky int mlx4_ib_create_srq(struct ib_srq *ib_srq,
73225c7b1fSRoland Dreier struct ib_srq_init_attr *init_attr,
74225c7b1fSRoland Dreier struct ib_udata *udata)
75225c7b1fSRoland Dreier {
7668e326deSLeon Romanovsky struct mlx4_ib_dev *dev = to_mdev(ib_srq->device);
7789944450SShamir Rabinovitch struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
7889944450SShamir Rabinovitch udata, struct mlx4_ib_ucontext, ibucontext);
7968e326deSLeon Romanovsky struct mlx4_ib_srq *srq = to_msrq(ib_srq);
80225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next;
814c425588SJack Morgenstein struct mlx4_wqe_data_seg *scatter;
8218abd5eaSSean Hefty u32 cqn;
8318abd5eaSSean Hefty u16 xrcdn;
84225c7b1fSRoland Dreier int desc_size;
85225c7b1fSRoland Dreier int buf_size;
86225c7b1fSRoland Dreier int err;
87225c7b1fSRoland Dreier int i;
88225c7b1fSRoland Dreier
89*652caba5SJason Gunthorpe if (init_attr->srq_type != IB_SRQT_BASIC &&
90*652caba5SJason Gunthorpe init_attr->srq_type != IB_SRQT_XRC)
91*652caba5SJason Gunthorpe return -EOPNOTSUPP;
92*652caba5SJason Gunthorpe
93225c7b1fSRoland Dreier /* Sanity check SRQ size before proceeding */
94225c7b1fSRoland Dreier if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes ||
95225c7b1fSRoland Dreier init_attr->attr.max_sge > dev->dev->caps.max_srq_sge)
9668e326deSLeon Romanovsky return -EINVAL;
97225c7b1fSRoland Dreier
98225c7b1fSRoland Dreier mutex_init(&srq->mutex);
99225c7b1fSRoland Dreier spin_lock_init(&srq->lock);
100225c7b1fSRoland Dreier srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1);
101225c7b1fSRoland Dreier srq->msrq.max_gs = init_attr->attr.max_sge;
102225c7b1fSRoland Dreier
103225c7b1fSRoland Dreier desc_size = max(32UL,
104225c7b1fSRoland Dreier roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) +
105225c7b1fSRoland Dreier srq->msrq.max_gs *
106225c7b1fSRoland Dreier sizeof (struct mlx4_wqe_data_seg)));
107225c7b1fSRoland Dreier srq->msrq.wqe_shift = ilog2(desc_size);
108225c7b1fSRoland Dreier
109225c7b1fSRoland Dreier buf_size = srq->msrq.max * desc_size;
110225c7b1fSRoland Dreier
111e00b64f7SShamir Rabinovitch if (udata) {
112225c7b1fSRoland Dreier struct mlx4_ib_create_srq ucmd;
113225c7b1fSRoland Dreier
11468e326deSLeon Romanovsky if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
11568e326deSLeon Romanovsky return -EFAULT;
116225c7b1fSRoland Dreier
117c320e527SMoni Shoua srq->umem =
118c320e527SMoni Shoua ib_umem_get(ib_srq->device, ucmd.buf_addr, buf_size, 0);
11968e326deSLeon Romanovsky if (IS_ERR(srq->umem))
12068e326deSLeon Romanovsky return PTR_ERR(srq->umem);
121225c7b1fSRoland Dreier
12281655d3cSJason Gunthorpe err = mlx4_mtt_init(
12381655d3cSJason Gunthorpe dev->dev, ib_umem_num_dma_blocks(srq->umem, PAGE_SIZE),
124d2183c6fSJason Gunthorpe PAGE_SHIFT, &srq->mtt);
125225c7b1fSRoland Dreier if (err)
126225c7b1fSRoland Dreier goto err_buf;
127225c7b1fSRoland Dreier
128225c7b1fSRoland Dreier err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem);
129225c7b1fSRoland Dreier if (err)
130225c7b1fSRoland Dreier goto err_mtt;
131225c7b1fSRoland Dreier
132ff23dfa1SShamir Rabinovitch err = mlx4_ib_db_map_user(udata, ucmd.db_addr, &srq->db);
133225c7b1fSRoland Dreier if (err)
134225c7b1fSRoland Dreier goto err_mtt;
135225c7b1fSRoland Dreier } else {
1368900b894SLeon Romanovsky err = mlx4_db_alloc(dev->dev, &srq->db, 0);
137225c7b1fSRoland Dreier if (err)
13868e326deSLeon Romanovsky return err;
139225c7b1fSRoland Dreier
140225c7b1fSRoland Dreier *srq->db.db = 0;
141225c7b1fSRoland Dreier
1428900b894SLeon Romanovsky if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2,
1438900b894SLeon Romanovsky &srq->buf)) {
144225c7b1fSRoland Dreier err = -ENOMEM;
145225c7b1fSRoland Dreier goto err_db;
146225c7b1fSRoland Dreier }
147225c7b1fSRoland Dreier
148225c7b1fSRoland Dreier srq->head = 0;
149225c7b1fSRoland Dreier srq->tail = srq->msrq.max - 1;
150225c7b1fSRoland Dreier srq->wqe_ctr = 0;
151225c7b1fSRoland Dreier
152225c7b1fSRoland Dreier for (i = 0; i < srq->msrq.max; ++i) {
153225c7b1fSRoland Dreier next = get_wqe(srq, i);
154225c7b1fSRoland Dreier next->next_wqe_index =
155225c7b1fSRoland Dreier cpu_to_be16((i + 1) & (srq->msrq.max - 1));
1564c425588SJack Morgenstein
1574c425588SJack Morgenstein for (scatter = (void *) (next + 1);
1584c425588SJack Morgenstein (void *) scatter < (void *) next + desc_size;
1594c425588SJack Morgenstein ++scatter)
1604c425588SJack Morgenstein scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY);
161225c7b1fSRoland Dreier }
162225c7b1fSRoland Dreier
163225c7b1fSRoland Dreier err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift,
164225c7b1fSRoland Dreier &srq->mtt);
165225c7b1fSRoland Dreier if (err)
166225c7b1fSRoland Dreier goto err_buf;
167225c7b1fSRoland Dreier
1688900b894SLeon Romanovsky err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf);
169225c7b1fSRoland Dreier if (err)
170225c7b1fSRoland Dreier goto err_mtt;
171225c7b1fSRoland Dreier
172e9105cdeSLi Dongyang srq->wrid = kvmalloc_array(srq->msrq.max,
173e9105cdeSLi Dongyang sizeof(u64), GFP_KERNEL);
1740ef2f05cSWengang Wang if (!srq->wrid) {
175225c7b1fSRoland Dreier err = -ENOMEM;
176225c7b1fSRoland Dreier goto err_mtt;
177225c7b1fSRoland Dreier }
178225c7b1fSRoland Dreier }
179225c7b1fSRoland Dreier
1801a56ff6dSArtemy Kovalyov cqn = ib_srq_has_cq(init_attr->srq_type) ?
1811a56ff6dSArtemy Kovalyov to_mcq(init_attr->ext.cq)->mcq.cqn : 0;
18218abd5eaSSean Hefty xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
18318abd5eaSSean Hefty to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn :
18418abd5eaSSean Hefty (u16) dev->dev->caps.reserved_xrcds;
18568e326deSLeon Romanovsky err = mlx4_srq_alloc(dev->dev, to_mpd(ib_srq->pd)->pdn, cqn, xrcdn,
18668e326deSLeon Romanovsky &srq->mtt, srq->db.dma, &srq->msrq);
187225c7b1fSRoland Dreier if (err)
188225c7b1fSRoland Dreier goto err_wrid;
189225c7b1fSRoland Dreier
190225c7b1fSRoland Dreier srq->msrq.event = mlx4_ib_srq_event;
19118abd5eaSSean Hefty srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
192225c7b1fSRoland Dreier
193e00b64f7SShamir Rabinovitch if (udata)
194225c7b1fSRoland Dreier if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
195225c7b1fSRoland Dreier err = -EFAULT;
196225c7b1fSRoland Dreier goto err_wrid;
197225c7b1fSRoland Dreier }
198225c7b1fSRoland Dreier
199225c7b1fSRoland Dreier init_attr->attr.max_wr = srq->msrq.max - 1;
200225c7b1fSRoland Dreier
20168e326deSLeon Romanovsky return 0;
202225c7b1fSRoland Dreier
203225c7b1fSRoland Dreier err_wrid:
204e00b64f7SShamir Rabinovitch if (udata)
20589944450SShamir Rabinovitch mlx4_ib_db_unmap_user(ucontext, &srq->db);
206225c7b1fSRoland Dreier else
2070ef2f05cSWengang Wang kvfree(srq->wrid);
208225c7b1fSRoland Dreier
209225c7b1fSRoland Dreier err_mtt:
210225c7b1fSRoland Dreier mlx4_mtt_cleanup(dev->dev, &srq->mtt);
211225c7b1fSRoland Dreier
212225c7b1fSRoland Dreier err_buf:
213836a0fbbSLeon Romanovsky if (!srq->umem)
214225c7b1fSRoland Dreier mlx4_buf_free(dev->dev, buf_size, &srq->buf);
215836a0fbbSLeon Romanovsky ib_umem_release(srq->umem);
216225c7b1fSRoland Dreier
217225c7b1fSRoland Dreier err_db:
218e00b64f7SShamir Rabinovitch if (!udata)
2196296883cSYevgeny Petrilin mlx4_db_free(dev->dev, &srq->db);
220225c7b1fSRoland Dreier
22168e326deSLeon Romanovsky return err;
222225c7b1fSRoland Dreier }
223225c7b1fSRoland Dreier
mlx4_ib_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)224225c7b1fSRoland Dreier int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
225225c7b1fSRoland Dreier enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
226225c7b1fSRoland Dreier {
227225c7b1fSRoland Dreier struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
228225c7b1fSRoland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq);
229225c7b1fSRoland Dreier int ret;
230225c7b1fSRoland Dreier
231225c7b1fSRoland Dreier /* We don't support resizing SRQs (yet?) */
232225c7b1fSRoland Dreier if (attr_mask & IB_SRQ_MAX_WR)
233225c7b1fSRoland Dreier return -EINVAL;
234225c7b1fSRoland Dreier
235225c7b1fSRoland Dreier if (attr_mask & IB_SRQ_LIMIT) {
236225c7b1fSRoland Dreier if (attr->srq_limit >= srq->msrq.max)
237225c7b1fSRoland Dreier return -EINVAL;
238225c7b1fSRoland Dreier
239225c7b1fSRoland Dreier mutex_lock(&srq->mutex);
240225c7b1fSRoland Dreier ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit);
241225c7b1fSRoland Dreier mutex_unlock(&srq->mutex);
242225c7b1fSRoland Dreier
243225c7b1fSRoland Dreier if (ret)
244225c7b1fSRoland Dreier return ret;
245225c7b1fSRoland Dreier }
246225c7b1fSRoland Dreier
247225c7b1fSRoland Dreier return 0;
248225c7b1fSRoland Dreier }
249225c7b1fSRoland Dreier
mlx4_ib_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr)25065541cb7SJack Morgenstein int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
25165541cb7SJack Morgenstein {
25265541cb7SJack Morgenstein struct mlx4_ib_dev *dev = to_mdev(ibsrq->device);
25365541cb7SJack Morgenstein struct mlx4_ib_srq *srq = to_msrq(ibsrq);
25465541cb7SJack Morgenstein int ret;
25565541cb7SJack Morgenstein int limit_watermark;
25665541cb7SJack Morgenstein
25765541cb7SJack Morgenstein ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark);
25865541cb7SJack Morgenstein if (ret)
25965541cb7SJack Morgenstein return ret;
26065541cb7SJack Morgenstein
261d7dc3ccbSRoland Dreier srq_attr->srq_limit = limit_watermark;
26265541cb7SJack Morgenstein srq_attr->max_wr = srq->msrq.max - 1;
26365541cb7SJack Morgenstein srq_attr->max_sge = srq->msrq.max_gs;
26465541cb7SJack Morgenstein
26565541cb7SJack Morgenstein return 0;
26665541cb7SJack Morgenstein }
26765541cb7SJack Morgenstein
mlx4_ib_destroy_srq(struct ib_srq * srq,struct ib_udata * udata)268119181d1SLeon Romanovsky int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
269225c7b1fSRoland Dreier {
270225c7b1fSRoland Dreier struct mlx4_ib_dev *dev = to_mdev(srq->device);
271225c7b1fSRoland Dreier struct mlx4_ib_srq *msrq = to_msrq(srq);
272225c7b1fSRoland Dreier
273225c7b1fSRoland Dreier mlx4_srq_free(dev->dev, &msrq->msrq);
274225c7b1fSRoland Dreier mlx4_mtt_cleanup(dev->dev, &msrq->mtt);
275225c7b1fSRoland Dreier
276bdeacabdSShamir Rabinovitch if (udata) {
277bdeacabdSShamir Rabinovitch mlx4_ib_db_unmap_user(
278bdeacabdSShamir Rabinovitch rdma_udata_to_drv_context(
279bdeacabdSShamir Rabinovitch udata,
280bdeacabdSShamir Rabinovitch struct mlx4_ib_ucontext,
281bdeacabdSShamir Rabinovitch ibucontext),
282bdeacabdSShamir Rabinovitch &msrq->db);
283225c7b1fSRoland Dreier } else {
284df417667SWengang Wang kvfree(msrq->wrid);
285225c7b1fSRoland Dreier mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
286225c7b1fSRoland Dreier &msrq->buf);
2876296883cSYevgeny Petrilin mlx4_db_free(dev->dev, &msrq->db);
288225c7b1fSRoland Dreier }
289836a0fbbSLeon Romanovsky ib_umem_release(msrq->umem);
290119181d1SLeon Romanovsky return 0;
291225c7b1fSRoland Dreier }
292225c7b1fSRoland Dreier
mlx4_ib_free_srq_wqe(struct mlx4_ib_srq * srq,int wqe_index)293225c7b1fSRoland Dreier void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)
294225c7b1fSRoland Dreier {
295225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next;
296225c7b1fSRoland Dreier
297225c7b1fSRoland Dreier /* always called with interrupts disabled. */
298225c7b1fSRoland Dreier spin_lock(&srq->lock);
299225c7b1fSRoland Dreier
300225c7b1fSRoland Dreier next = get_wqe(srq, srq->tail);
301225c7b1fSRoland Dreier next->next_wqe_index = cpu_to_be16(wqe_index);
302225c7b1fSRoland Dreier srq->tail = wqe_index;
303225c7b1fSRoland Dreier
304225c7b1fSRoland Dreier spin_unlock(&srq->lock);
305225c7b1fSRoland Dreier }
306225c7b1fSRoland Dreier
mlx4_ib_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)307d34ac5cdSBart Van Assche int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
308d34ac5cdSBart Van Assche const struct ib_recv_wr **bad_wr)
309225c7b1fSRoland Dreier {
310225c7b1fSRoland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq);
311225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next;
312225c7b1fSRoland Dreier struct mlx4_wqe_data_seg *scat;
313225c7b1fSRoland Dreier unsigned long flags;
314225c7b1fSRoland Dreier int err = 0;
315225c7b1fSRoland Dreier int nreq;
316225c7b1fSRoland Dreier int i;
31735f05dabSYishai Hadas struct mlx4_ib_dev *mdev = to_mdev(ibsrq->device);
318225c7b1fSRoland Dreier
319225c7b1fSRoland Dreier spin_lock_irqsave(&srq->lock, flags);
32035f05dabSYishai Hadas if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) {
32135f05dabSYishai Hadas err = -EIO;
32235f05dabSYishai Hadas *bad_wr = wr;
32335f05dabSYishai Hadas goto out;
32435f05dabSYishai Hadas }
325225c7b1fSRoland Dreier
326225c7b1fSRoland Dreier for (nreq = 0; wr; ++nreq, wr = wr->next) {
327225c7b1fSRoland Dreier if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
328225c7b1fSRoland Dreier err = -EINVAL;
329225c7b1fSRoland Dreier *bad_wr = wr;
330225c7b1fSRoland Dreier break;
331225c7b1fSRoland Dreier }
332225c7b1fSRoland Dreier
33356a8c8b6SRoland Dreier if (unlikely(srq->head == srq->tail)) {
33456a8c8b6SRoland Dreier err = -ENOMEM;
33556a8c8b6SRoland Dreier *bad_wr = wr;
33656a8c8b6SRoland Dreier break;
33756a8c8b6SRoland Dreier }
33856a8c8b6SRoland Dreier
339225c7b1fSRoland Dreier srq->wrid[srq->head] = wr->wr_id;
340225c7b1fSRoland Dreier
341225c7b1fSRoland Dreier next = get_wqe(srq, srq->head);
342225c7b1fSRoland Dreier srq->head = be16_to_cpu(next->next_wqe_index);
343225c7b1fSRoland Dreier scat = (struct mlx4_wqe_data_seg *) (next + 1);
344225c7b1fSRoland Dreier
345225c7b1fSRoland Dreier for (i = 0; i < wr->num_sge; ++i) {
346225c7b1fSRoland Dreier scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
347225c7b1fSRoland Dreier scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey);
348225c7b1fSRoland Dreier scat[i].addr = cpu_to_be64(wr->sg_list[i].addr);
349225c7b1fSRoland Dreier }
350225c7b1fSRoland Dreier
351225c7b1fSRoland Dreier if (i < srq->msrq.max_gs) {
352225c7b1fSRoland Dreier scat[i].byte_count = 0;
353225c7b1fSRoland Dreier scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY);
354225c7b1fSRoland Dreier scat[i].addr = 0;
355225c7b1fSRoland Dreier }
356225c7b1fSRoland Dreier }
357225c7b1fSRoland Dreier
358225c7b1fSRoland Dreier if (likely(nreq)) {
359225c7b1fSRoland Dreier srq->wqe_ctr += nreq;
360225c7b1fSRoland Dreier
361225c7b1fSRoland Dreier /*
362225c7b1fSRoland Dreier * Make sure that descriptors are written before
363225c7b1fSRoland Dreier * doorbell record.
364225c7b1fSRoland Dreier */
365225c7b1fSRoland Dreier wmb();
366225c7b1fSRoland Dreier
367225c7b1fSRoland Dreier *srq->db.db = cpu_to_be32(srq->wqe_ctr);
368225c7b1fSRoland Dreier }
36935f05dabSYishai Hadas out:
370225c7b1fSRoland Dreier
371225c7b1fSRoland Dreier spin_unlock_irqrestore(&srq->lock, flags);
372225c7b1fSRoland Dreier
373225c7b1fSRoland Dreier return err;
374225c7b1fSRoland Dreier }
375