1225c7b1fSRoland Dreier /* 2225c7b1fSRoland Dreier * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. 351a379d0SJack Morgenstein * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4225c7b1fSRoland Dreier * 5225c7b1fSRoland Dreier * This software is available to you under a choice of one of two 6225c7b1fSRoland Dreier * licenses. You may choose to be licensed under the terms of the GNU 7225c7b1fSRoland Dreier * General Public License (GPL) Version 2, available from the file 8225c7b1fSRoland Dreier * COPYING in the main directory of this source tree, or the 9225c7b1fSRoland Dreier * OpenIB.org BSD license below: 10225c7b1fSRoland Dreier * 11225c7b1fSRoland Dreier * Redistribution and use in source and binary forms, with or 12225c7b1fSRoland Dreier * without modification, are permitted provided that the following 13225c7b1fSRoland Dreier * conditions are met: 14225c7b1fSRoland Dreier * 15225c7b1fSRoland Dreier * - Redistributions of source code must retain the above 16225c7b1fSRoland Dreier * copyright notice, this list of conditions and the following 17225c7b1fSRoland Dreier * disclaimer. 18225c7b1fSRoland Dreier * 19225c7b1fSRoland Dreier * - Redistributions in binary form must reproduce the above 20225c7b1fSRoland Dreier * copyright notice, this list of conditions and the following 21225c7b1fSRoland Dreier * disclaimer in the documentation and/or other materials 22225c7b1fSRoland Dreier * provided with the distribution. 23225c7b1fSRoland Dreier * 24225c7b1fSRoland Dreier * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25225c7b1fSRoland Dreier * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26225c7b1fSRoland Dreier * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27225c7b1fSRoland Dreier * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28225c7b1fSRoland Dreier * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29225c7b1fSRoland Dreier * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30225c7b1fSRoland Dreier * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31225c7b1fSRoland Dreier * SOFTWARE. 32225c7b1fSRoland Dreier */ 33225c7b1fSRoland Dreier 34225c7b1fSRoland Dreier #include <linux/mlx4/qp.h> 35225c7b1fSRoland Dreier #include <linux/mlx4/srq.h> 365a0e3ad6STejun Heo #include <linux/slab.h> 37225c7b1fSRoland Dreier 38225c7b1fSRoland Dreier #include "mlx4_ib.h" 399ce28a20SLeon Romanovsky #include <rdma/mlx4-abi.h> 4089944450SShamir Rabinovitch #include <rdma/uverbs_ioctl.h> 41225c7b1fSRoland Dreier 42225c7b1fSRoland Dreier static void *get_wqe(struct mlx4_ib_srq *srq, int n) 43225c7b1fSRoland Dreier { 441c69fc2aSRoland Dreier return mlx4_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); 45225c7b1fSRoland Dreier } 46225c7b1fSRoland Dreier 47225c7b1fSRoland Dreier static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type) 48225c7b1fSRoland Dreier { 49225c7b1fSRoland Dreier struct ib_event event; 50225c7b1fSRoland Dreier struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 51225c7b1fSRoland Dreier 52225c7b1fSRoland Dreier if (ibsrq->event_handler) { 53225c7b1fSRoland Dreier event.device = ibsrq->device; 54225c7b1fSRoland Dreier event.element.srq = ibsrq; 55225c7b1fSRoland Dreier switch (type) { 56225c7b1fSRoland Dreier case MLX4_EVENT_TYPE_SRQ_LIMIT: 57225c7b1fSRoland Dreier event.event = IB_EVENT_SRQ_LIMIT_REACHED; 58225c7b1fSRoland Dreier break; 59225c7b1fSRoland Dreier case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR: 60225c7b1fSRoland Dreier event.event = IB_EVENT_SRQ_ERR; 61225c7b1fSRoland Dreier break; 62225c7b1fSRoland Dreier default: 63987c8f8fSShlomo Pongratz pr_warn("Unexpected event type %d " 64225c7b1fSRoland Dreier "on SRQ %06x\n", type, srq->srqn); 65225c7b1fSRoland Dreier return; 66225c7b1fSRoland Dreier } 67225c7b1fSRoland Dreier 68225c7b1fSRoland Dreier ibsrq->event_handler(&event, ibsrq->srq_context); 69225c7b1fSRoland Dreier } 70225c7b1fSRoland Dreier } 71225c7b1fSRoland Dreier 72225c7b1fSRoland Dreier struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd, 73225c7b1fSRoland Dreier struct ib_srq_init_attr *init_attr, 74225c7b1fSRoland Dreier struct ib_udata *udata) 75225c7b1fSRoland Dreier { 76225c7b1fSRoland Dreier struct mlx4_ib_dev *dev = to_mdev(pd->device); 7789944450SShamir Rabinovitch struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context( 7889944450SShamir Rabinovitch udata, struct mlx4_ib_ucontext, ibucontext); 79225c7b1fSRoland Dreier struct mlx4_ib_srq *srq; 80225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next; 814c425588SJack Morgenstein struct mlx4_wqe_data_seg *scatter; 8218abd5eaSSean Hefty u32 cqn; 8318abd5eaSSean Hefty u16 xrcdn; 84225c7b1fSRoland Dreier int desc_size; 85225c7b1fSRoland Dreier int buf_size; 86225c7b1fSRoland Dreier int err; 87225c7b1fSRoland Dreier int i; 88225c7b1fSRoland Dreier 89225c7b1fSRoland Dreier /* Sanity check SRQ size before proceeding */ 90225c7b1fSRoland Dreier if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes || 91225c7b1fSRoland Dreier init_attr->attr.max_sge > dev->dev->caps.max_srq_sge) 92225c7b1fSRoland Dreier return ERR_PTR(-EINVAL); 93225c7b1fSRoland Dreier 94225c7b1fSRoland Dreier srq = kmalloc(sizeof *srq, GFP_KERNEL); 95225c7b1fSRoland Dreier if (!srq) 96225c7b1fSRoland Dreier return ERR_PTR(-ENOMEM); 97225c7b1fSRoland Dreier 98225c7b1fSRoland Dreier mutex_init(&srq->mutex); 99225c7b1fSRoland Dreier spin_lock_init(&srq->lock); 100225c7b1fSRoland Dreier srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 101225c7b1fSRoland Dreier srq->msrq.max_gs = init_attr->attr.max_sge; 102225c7b1fSRoland Dreier 103225c7b1fSRoland Dreier desc_size = max(32UL, 104225c7b1fSRoland Dreier roundup_pow_of_two(sizeof (struct mlx4_wqe_srq_next_seg) + 105225c7b1fSRoland Dreier srq->msrq.max_gs * 106225c7b1fSRoland Dreier sizeof (struct mlx4_wqe_data_seg))); 107225c7b1fSRoland Dreier srq->msrq.wqe_shift = ilog2(desc_size); 108225c7b1fSRoland Dreier 109225c7b1fSRoland Dreier buf_size = srq->msrq.max * desc_size; 110225c7b1fSRoland Dreier 111e00b64f7SShamir Rabinovitch if (udata) { 112225c7b1fSRoland Dreier struct mlx4_ib_create_srq ucmd; 113225c7b1fSRoland Dreier 114225c7b1fSRoland Dreier if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { 115225c7b1fSRoland Dreier err = -EFAULT; 116225c7b1fSRoland Dreier goto err_srq; 117225c7b1fSRoland Dreier } 118225c7b1fSRoland Dreier 119b0ea0fa5SJason Gunthorpe srq->umem = ib_umem_get(udata, ucmd.buf_addr, buf_size, 0, 0); 120225c7b1fSRoland Dreier if (IS_ERR(srq->umem)) { 121225c7b1fSRoland Dreier err = PTR_ERR(srq->umem); 122225c7b1fSRoland Dreier goto err_srq; 123225c7b1fSRoland Dreier } 124225c7b1fSRoland Dreier 125225c7b1fSRoland Dreier err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), 1263e7e1193SArtemy Kovalyov srq->umem->page_shift, &srq->mtt); 127225c7b1fSRoland Dreier if (err) 128225c7b1fSRoland Dreier goto err_buf; 129225c7b1fSRoland Dreier 130225c7b1fSRoland Dreier err = mlx4_ib_umem_write_mtt(dev, &srq->mtt, srq->umem); 131225c7b1fSRoland Dreier if (err) 132225c7b1fSRoland Dreier goto err_mtt; 133225c7b1fSRoland Dreier 13489944450SShamir Rabinovitch err = mlx4_ib_db_map_user(ucontext, udata, ucmd.db_addr, 13589944450SShamir Rabinovitch &srq->db); 136225c7b1fSRoland Dreier if (err) 137225c7b1fSRoland Dreier goto err_mtt; 138225c7b1fSRoland Dreier } else { 1398900b894SLeon Romanovsky err = mlx4_db_alloc(dev->dev, &srq->db, 0); 140225c7b1fSRoland Dreier if (err) 141225c7b1fSRoland Dreier goto err_srq; 142225c7b1fSRoland Dreier 143225c7b1fSRoland Dreier *srq->db.db = 0; 144225c7b1fSRoland Dreier 1458900b894SLeon Romanovsky if (mlx4_buf_alloc(dev->dev, buf_size, PAGE_SIZE * 2, 1468900b894SLeon Romanovsky &srq->buf)) { 147225c7b1fSRoland Dreier err = -ENOMEM; 148225c7b1fSRoland Dreier goto err_db; 149225c7b1fSRoland Dreier } 150225c7b1fSRoland Dreier 151225c7b1fSRoland Dreier srq->head = 0; 152225c7b1fSRoland Dreier srq->tail = srq->msrq.max - 1; 153225c7b1fSRoland Dreier srq->wqe_ctr = 0; 154225c7b1fSRoland Dreier 155225c7b1fSRoland Dreier for (i = 0; i < srq->msrq.max; ++i) { 156225c7b1fSRoland Dreier next = get_wqe(srq, i); 157225c7b1fSRoland Dreier next->next_wqe_index = 158225c7b1fSRoland Dreier cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 1594c425588SJack Morgenstein 1604c425588SJack Morgenstein for (scatter = (void *) (next + 1); 1614c425588SJack Morgenstein (void *) scatter < (void *) next + desc_size; 1624c425588SJack Morgenstein ++scatter) 1634c425588SJack Morgenstein scatter->lkey = cpu_to_be32(MLX4_INVALID_LKEY); 164225c7b1fSRoland Dreier } 165225c7b1fSRoland Dreier 166225c7b1fSRoland Dreier err = mlx4_mtt_init(dev->dev, srq->buf.npages, srq->buf.page_shift, 167225c7b1fSRoland Dreier &srq->mtt); 168225c7b1fSRoland Dreier if (err) 169225c7b1fSRoland Dreier goto err_buf; 170225c7b1fSRoland Dreier 1718900b894SLeon Romanovsky err = mlx4_buf_write_mtt(dev->dev, &srq->mtt, &srq->buf); 172225c7b1fSRoland Dreier if (err) 173225c7b1fSRoland Dreier goto err_mtt; 174225c7b1fSRoland Dreier 175e9105cdeSLi Dongyang srq->wrid = kvmalloc_array(srq->msrq.max, 176e9105cdeSLi Dongyang sizeof(u64), GFP_KERNEL); 1770ef2f05cSWengang Wang if (!srq->wrid) { 178225c7b1fSRoland Dreier err = -ENOMEM; 179225c7b1fSRoland Dreier goto err_mtt; 180225c7b1fSRoland Dreier } 181225c7b1fSRoland Dreier } 182225c7b1fSRoland Dreier 1831a56ff6dSArtemy Kovalyov cqn = ib_srq_has_cq(init_attr->srq_type) ? 1841a56ff6dSArtemy Kovalyov to_mcq(init_attr->ext.cq)->mcq.cqn : 0; 18518abd5eaSSean Hefty xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ? 18618abd5eaSSean Hefty to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn : 18718abd5eaSSean Hefty (u16) dev->dev->caps.reserved_xrcds; 18818abd5eaSSean Hefty err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt, 189225c7b1fSRoland Dreier srq->db.dma, &srq->msrq); 190225c7b1fSRoland Dreier if (err) 191225c7b1fSRoland Dreier goto err_wrid; 192225c7b1fSRoland Dreier 193225c7b1fSRoland Dreier srq->msrq.event = mlx4_ib_srq_event; 19418abd5eaSSean Hefty srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 195225c7b1fSRoland Dreier 196e00b64f7SShamir Rabinovitch if (udata) 197225c7b1fSRoland Dreier if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) { 198225c7b1fSRoland Dreier err = -EFAULT; 199225c7b1fSRoland Dreier goto err_wrid; 200225c7b1fSRoland Dreier } 201225c7b1fSRoland Dreier 202225c7b1fSRoland Dreier init_attr->attr.max_wr = srq->msrq.max - 1; 203225c7b1fSRoland Dreier 204225c7b1fSRoland Dreier return &srq->ibsrq; 205225c7b1fSRoland Dreier 206225c7b1fSRoland Dreier err_wrid: 207e00b64f7SShamir Rabinovitch if (udata) 20889944450SShamir Rabinovitch mlx4_ib_db_unmap_user(ucontext, &srq->db); 209225c7b1fSRoland Dreier else 2100ef2f05cSWengang Wang kvfree(srq->wrid); 211225c7b1fSRoland Dreier 212225c7b1fSRoland Dreier err_mtt: 213225c7b1fSRoland Dreier mlx4_mtt_cleanup(dev->dev, &srq->mtt); 214225c7b1fSRoland Dreier 215225c7b1fSRoland Dreier err_buf: 216e00b64f7SShamir Rabinovitch if (srq->umem) 217225c7b1fSRoland Dreier ib_umem_release(srq->umem); 218225c7b1fSRoland Dreier else 219225c7b1fSRoland Dreier mlx4_buf_free(dev->dev, buf_size, &srq->buf); 220225c7b1fSRoland Dreier 221225c7b1fSRoland Dreier err_db: 222e00b64f7SShamir Rabinovitch if (!udata) 2236296883cSYevgeny Petrilin mlx4_db_free(dev->dev, &srq->db); 224225c7b1fSRoland Dreier 225225c7b1fSRoland Dreier err_srq: 226225c7b1fSRoland Dreier kfree(srq); 227225c7b1fSRoland Dreier 228225c7b1fSRoland Dreier return ERR_PTR(err); 229225c7b1fSRoland Dreier } 230225c7b1fSRoland Dreier 231225c7b1fSRoland Dreier int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 232225c7b1fSRoland Dreier enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 233225c7b1fSRoland Dreier { 234225c7b1fSRoland Dreier struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); 235225c7b1fSRoland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq); 236225c7b1fSRoland Dreier int ret; 237225c7b1fSRoland Dreier 238225c7b1fSRoland Dreier /* We don't support resizing SRQs (yet?) */ 239225c7b1fSRoland Dreier if (attr_mask & IB_SRQ_MAX_WR) 240225c7b1fSRoland Dreier return -EINVAL; 241225c7b1fSRoland Dreier 242225c7b1fSRoland Dreier if (attr_mask & IB_SRQ_LIMIT) { 243225c7b1fSRoland Dreier if (attr->srq_limit >= srq->msrq.max) 244225c7b1fSRoland Dreier return -EINVAL; 245225c7b1fSRoland Dreier 246225c7b1fSRoland Dreier mutex_lock(&srq->mutex); 247225c7b1fSRoland Dreier ret = mlx4_srq_arm(dev->dev, &srq->msrq, attr->srq_limit); 248225c7b1fSRoland Dreier mutex_unlock(&srq->mutex); 249225c7b1fSRoland Dreier 250225c7b1fSRoland Dreier if (ret) 251225c7b1fSRoland Dreier return ret; 252225c7b1fSRoland Dreier } 253225c7b1fSRoland Dreier 254225c7b1fSRoland Dreier return 0; 255225c7b1fSRoland Dreier } 256225c7b1fSRoland Dreier 25765541cb7SJack Morgenstein int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 25865541cb7SJack Morgenstein { 25965541cb7SJack Morgenstein struct mlx4_ib_dev *dev = to_mdev(ibsrq->device); 26065541cb7SJack Morgenstein struct mlx4_ib_srq *srq = to_msrq(ibsrq); 26165541cb7SJack Morgenstein int ret; 26265541cb7SJack Morgenstein int limit_watermark; 26365541cb7SJack Morgenstein 26465541cb7SJack Morgenstein ret = mlx4_srq_query(dev->dev, &srq->msrq, &limit_watermark); 26565541cb7SJack Morgenstein if (ret) 26665541cb7SJack Morgenstein return ret; 26765541cb7SJack Morgenstein 268d7dc3ccbSRoland Dreier srq_attr->srq_limit = limit_watermark; 26965541cb7SJack Morgenstein srq_attr->max_wr = srq->msrq.max - 1; 27065541cb7SJack Morgenstein srq_attr->max_sge = srq->msrq.max_gs; 27165541cb7SJack Morgenstein 27265541cb7SJack Morgenstein return 0; 27365541cb7SJack Morgenstein } 27465541cb7SJack Morgenstein 275c4367a26SShamir Rabinovitch int mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 276225c7b1fSRoland Dreier { 277225c7b1fSRoland Dreier struct mlx4_ib_dev *dev = to_mdev(srq->device); 278225c7b1fSRoland Dreier struct mlx4_ib_srq *msrq = to_msrq(srq); 279225c7b1fSRoland Dreier 280225c7b1fSRoland Dreier mlx4_srq_free(dev->dev, &msrq->msrq); 281225c7b1fSRoland Dreier mlx4_mtt_cleanup(dev->dev, &msrq->mtt); 282225c7b1fSRoland Dreier 283225c7b1fSRoland Dreier if (srq->uobject) { 284225c7b1fSRoland Dreier mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 285225c7b1fSRoland Dreier ib_umem_release(msrq->umem); 286225c7b1fSRoland Dreier } else { 287df417667SWengang Wang kvfree(msrq->wrid); 288225c7b1fSRoland Dreier mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, 289225c7b1fSRoland Dreier &msrq->buf); 2906296883cSYevgeny Petrilin mlx4_db_free(dev->dev, &msrq->db); 291225c7b1fSRoland Dreier } 292225c7b1fSRoland Dreier 293225c7b1fSRoland Dreier kfree(msrq); 294225c7b1fSRoland Dreier 295225c7b1fSRoland Dreier return 0; 296225c7b1fSRoland Dreier } 297225c7b1fSRoland Dreier 298225c7b1fSRoland Dreier void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) 299225c7b1fSRoland Dreier { 300225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next; 301225c7b1fSRoland Dreier 302225c7b1fSRoland Dreier /* always called with interrupts disabled. */ 303225c7b1fSRoland Dreier spin_lock(&srq->lock); 304225c7b1fSRoland Dreier 305225c7b1fSRoland Dreier next = get_wqe(srq, srq->tail); 306225c7b1fSRoland Dreier next->next_wqe_index = cpu_to_be16(wqe_index); 307225c7b1fSRoland Dreier srq->tail = wqe_index; 308225c7b1fSRoland Dreier 309225c7b1fSRoland Dreier spin_unlock(&srq->lock); 310225c7b1fSRoland Dreier } 311225c7b1fSRoland Dreier 312d34ac5cdSBart Van Assche int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 313d34ac5cdSBart Van Assche const struct ib_recv_wr **bad_wr) 314225c7b1fSRoland Dreier { 315225c7b1fSRoland Dreier struct mlx4_ib_srq *srq = to_msrq(ibsrq); 316225c7b1fSRoland Dreier struct mlx4_wqe_srq_next_seg *next; 317225c7b1fSRoland Dreier struct mlx4_wqe_data_seg *scat; 318225c7b1fSRoland Dreier unsigned long flags; 319225c7b1fSRoland Dreier int err = 0; 320225c7b1fSRoland Dreier int nreq; 321225c7b1fSRoland Dreier int i; 32235f05dabSYishai Hadas struct mlx4_ib_dev *mdev = to_mdev(ibsrq->device); 323225c7b1fSRoland Dreier 324225c7b1fSRoland Dreier spin_lock_irqsave(&srq->lock, flags); 32535f05dabSYishai Hadas if (mdev->dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) { 32635f05dabSYishai Hadas err = -EIO; 32735f05dabSYishai Hadas *bad_wr = wr; 32835f05dabSYishai Hadas nreq = 0; 32935f05dabSYishai Hadas goto out; 33035f05dabSYishai Hadas } 331225c7b1fSRoland Dreier 332225c7b1fSRoland Dreier for (nreq = 0; wr; ++nreq, wr = wr->next) { 333225c7b1fSRoland Dreier if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 334225c7b1fSRoland Dreier err = -EINVAL; 335225c7b1fSRoland Dreier *bad_wr = wr; 336225c7b1fSRoland Dreier break; 337225c7b1fSRoland Dreier } 338225c7b1fSRoland Dreier 33956a8c8b6SRoland Dreier if (unlikely(srq->head == srq->tail)) { 34056a8c8b6SRoland Dreier err = -ENOMEM; 34156a8c8b6SRoland Dreier *bad_wr = wr; 34256a8c8b6SRoland Dreier break; 34356a8c8b6SRoland Dreier } 34456a8c8b6SRoland Dreier 345225c7b1fSRoland Dreier srq->wrid[srq->head] = wr->wr_id; 346225c7b1fSRoland Dreier 347225c7b1fSRoland Dreier next = get_wqe(srq, srq->head); 348225c7b1fSRoland Dreier srq->head = be16_to_cpu(next->next_wqe_index); 349225c7b1fSRoland Dreier scat = (struct mlx4_wqe_data_seg *) (next + 1); 350225c7b1fSRoland Dreier 351225c7b1fSRoland Dreier for (i = 0; i < wr->num_sge; ++i) { 352225c7b1fSRoland Dreier scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 353225c7b1fSRoland Dreier scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 354225c7b1fSRoland Dreier scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 355225c7b1fSRoland Dreier } 356225c7b1fSRoland Dreier 357225c7b1fSRoland Dreier if (i < srq->msrq.max_gs) { 358225c7b1fSRoland Dreier scat[i].byte_count = 0; 359225c7b1fSRoland Dreier scat[i].lkey = cpu_to_be32(MLX4_INVALID_LKEY); 360225c7b1fSRoland Dreier scat[i].addr = 0; 361225c7b1fSRoland Dreier } 362225c7b1fSRoland Dreier } 363225c7b1fSRoland Dreier 364225c7b1fSRoland Dreier if (likely(nreq)) { 365225c7b1fSRoland Dreier srq->wqe_ctr += nreq; 366225c7b1fSRoland Dreier 367225c7b1fSRoland Dreier /* 368225c7b1fSRoland Dreier * Make sure that descriptors are written before 369225c7b1fSRoland Dreier * doorbell record. 370225c7b1fSRoland Dreier */ 371225c7b1fSRoland Dreier wmb(); 372225c7b1fSRoland Dreier 373225c7b1fSRoland Dreier *srq->db.db = cpu_to_be32(srq->wqe_ctr); 374225c7b1fSRoland Dreier } 37535f05dabSYishai Hadas out: 376225c7b1fSRoland Dreier 377225c7b1fSRoland Dreier spin_unlock_irqrestore(&srq->lock, flags); 378225c7b1fSRoland Dreier 379225c7b1fSRoland Dreier return err; 380225c7b1fSRoland Dreier } 381