16cd0014aSLeon Romanovsky // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2e126ba97SEli Cohen /* 36cd0014aSLeon Romanovsky * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4e126ba97SEli Cohen */ 5e126ba97SEli Cohen 6e126ba97SEli Cohen #include <linux/module.h> 7e126ba97SEli Cohen #include <linux/mlx5/qp.h> 8e126ba97SEli Cohen #include <linux/slab.h> 9e126ba97SEli Cohen #include <rdma/ib_umem.h> 1043bc8893SYann Droneaud #include <rdma/ib_user_verbs.h> 11e126ba97SEli Cohen #include "mlx5_ib.h" 12f02d0d6eSLeon Romanovsky #include "srq.h" 13e126ba97SEli Cohen 14e126ba97SEli Cohen static void *get_wqe(struct mlx5_ib_srq *srq, int n) 15e126ba97SEli Cohen { 1620e5a59bSGuy Levi return mlx5_frag_buf_get_wqe(&srq->fbc, n); 17e126ba97SEli Cohen } 18e126ba97SEli Cohen 19e126ba97SEli Cohen static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 20e126ba97SEli Cohen { 21e126ba97SEli Cohen struct ib_event event; 22e126ba97SEli Cohen struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 23e126ba97SEli Cohen 24e126ba97SEli Cohen if (ibsrq->event_handler) { 25e126ba97SEli Cohen event.device = ibsrq->device; 26e126ba97SEli Cohen event.element.srq = ibsrq; 27e126ba97SEli Cohen switch (type) { 28e126ba97SEli Cohen case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 29e126ba97SEli Cohen event.event = IB_EVENT_SRQ_LIMIT_REACHED; 30e126ba97SEli Cohen break; 31e126ba97SEli Cohen case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 32e126ba97SEli Cohen event.event = IB_EVENT_SRQ_ERR; 33e126ba97SEli Cohen break; 34e126ba97SEli Cohen default: 35e126ba97SEli Cohen pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 36e126ba97SEli Cohen type, srq->srqn); 37e126ba97SEli Cohen return; 38e126ba97SEli Cohen } 39e126ba97SEli Cohen 40e126ba97SEli Cohen ibsrq->event_handler(&event, ibsrq->srq_context); 41e126ba97SEli Cohen } 42e126ba97SEli Cohen } 43e126ba97SEli Cohen 44e126ba97SEli Cohen static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 45af1ba291SArtemy Kovalyov struct mlx5_srq_attr *in, 46af1ba291SArtemy Kovalyov struct ib_udata *udata, int buf_size) 47e126ba97SEli Cohen { 48e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(pd->device); 49cfb5e088SHaggai Abramovsky struct mlx5_ib_create_srq ucmd = {}; 5043bc8893SYann Droneaud size_t ucmdlen; 51e126ba97SEli Cohen int err; 52e126ba97SEli Cohen int npages; 53e126ba97SEli Cohen int page_shift; 54e126ba97SEli Cohen int ncont; 55e126ba97SEli Cohen u32 offset; 56cfb5e088SHaggai Abramovsky u32 uidx = MLX5_IB_DEFAULT_UIDX; 57e126ba97SEli Cohen 583d943c9dSMajd Dibbiny ucmdlen = min(udata->inlen, sizeof(ucmd)); 5943bc8893SYann Droneaud 6043bc8893SYann Droneaud if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 61e126ba97SEli Cohen mlx5_ib_dbg(dev, "failed copy udata\n"); 62e126ba97SEli Cohen return -EFAULT; 63e126ba97SEli Cohen } 6443bc8893SYann Droneaud 65cfb5e088SHaggai Abramovsky if (ucmd.reserved0 || ucmd.reserved1) 6643bc8893SYann Droneaud return -EINVAL; 6743bc8893SYann Droneaud 683d943c9dSMajd Dibbiny if (udata->inlen > sizeof(ucmd) && 69cfb5e088SHaggai Abramovsky !ib_is_udata_cleared(udata, sizeof(ucmd), 703d943c9dSMajd Dibbiny udata->inlen - sizeof(ucmd))) 71cfb5e088SHaggai Abramovsky return -EINVAL; 72cfb5e088SHaggai Abramovsky 733fd3307eSArtemy Kovalyov if (in->type != IB_SRQT_BASIC) { 74cfb5e088SHaggai Abramovsky err = get_srq_user_index(to_mucontext(pd->uobject->context), 75cfb5e088SHaggai Abramovsky &ucmd, udata->inlen, &uidx); 76cfb5e088SHaggai Abramovsky if (err) 77cfb5e088SHaggai Abramovsky return err; 7885d9691cSMajd Dibbiny } 79cfb5e088SHaggai Abramovsky 80e126ba97SEli Cohen srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 81e126ba97SEli Cohen 82e126ba97SEli Cohen srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 83e126ba97SEli Cohen 0, 0); 84e126ba97SEli Cohen if (IS_ERR(srq->umem)) { 85e126ba97SEli Cohen mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 86e126ba97SEli Cohen err = PTR_ERR(srq->umem); 87e126ba97SEli Cohen return err; 88e126ba97SEli Cohen } 89e126ba97SEli Cohen 90762f899aSMajd Dibbiny mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, 91e126ba97SEli Cohen &page_shift, &ncont, NULL); 92e126ba97SEli Cohen err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, 93e126ba97SEli Cohen &offset); 94e126ba97SEli Cohen if (err) { 95e126ba97SEli Cohen mlx5_ib_warn(dev, "bad offset\n"); 96e126ba97SEli Cohen goto err_umem; 97e126ba97SEli Cohen } 98e126ba97SEli Cohen 99778e1cddSKees Cook in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL); 100af1ba291SArtemy Kovalyov if (!in->pas) { 101e126ba97SEli Cohen err = -ENOMEM; 102e126ba97SEli Cohen goto err_umem; 103e126ba97SEli Cohen } 104e126ba97SEli Cohen 105af1ba291SArtemy Kovalyov mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); 106e126ba97SEli Cohen 107e126ba97SEli Cohen err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context), 108e126ba97SEli Cohen ucmd.db_addr, &srq->db); 109e126ba97SEli Cohen if (err) { 110e126ba97SEli Cohen mlx5_ib_dbg(dev, "map doorbell failed\n"); 111e126ba97SEli Cohen goto err_in; 112e126ba97SEli Cohen } 113e126ba97SEli Cohen 114af1ba291SArtemy Kovalyov in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 115af1ba291SArtemy Kovalyov in->page_offset = offset; 1165aa3771dSYishai Hadas in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 117af1ba291SArtemy Kovalyov if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 1183fd3307eSArtemy Kovalyov in->type != IB_SRQT_BASIC) 119af1ba291SArtemy Kovalyov in->user_index = uidx; 120cfb5e088SHaggai Abramovsky 121e126ba97SEli Cohen return 0; 122e126ba97SEli Cohen 123e126ba97SEli Cohen err_in: 124af1ba291SArtemy Kovalyov kvfree(in->pas); 125e126ba97SEli Cohen 126e126ba97SEli Cohen err_umem: 127e126ba97SEli Cohen ib_umem_release(srq->umem); 128e126ba97SEli Cohen 129e126ba97SEli Cohen return err; 130e126ba97SEli Cohen } 131e126ba97SEli Cohen 132e126ba97SEli Cohen static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 133af1ba291SArtemy Kovalyov struct mlx5_srq_attr *in, int buf_size) 134e126ba97SEli Cohen { 135e126ba97SEli Cohen int err; 136e126ba97SEli Cohen int i; 137e126ba97SEli Cohen struct mlx5_wqe_srq_next_seg *next; 138e126ba97SEli Cohen 1399603b61dSJack Morgenstein err = mlx5_db_alloc(dev->mdev, &srq->db); 140e126ba97SEli Cohen if (err) { 141e126ba97SEli Cohen mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 142e126ba97SEli Cohen return err; 143e126ba97SEli Cohen } 144e126ba97SEli Cohen 14520e5a59bSGuy Levi if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, 14620e5a59bSGuy Levi dev->mdev->priv.numa_node)) { 147e126ba97SEli Cohen mlx5_ib_dbg(dev, "buf alloc failed\n"); 148e126ba97SEli Cohen err = -ENOMEM; 149e126ba97SEli Cohen goto err_db; 150e126ba97SEli Cohen } 151e126ba97SEli Cohen 15220e5a59bSGuy Levi mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), 15320e5a59bSGuy Levi &srq->fbc); 15420e5a59bSGuy Levi 155e126ba97SEli Cohen srq->head = 0; 156e126ba97SEli Cohen srq->tail = srq->msrq.max - 1; 157e126ba97SEli Cohen srq->wqe_ctr = 0; 158e126ba97SEli Cohen 159e126ba97SEli Cohen for (i = 0; i < srq->msrq.max; i++) { 160e126ba97SEli Cohen next = get_wqe(srq, i); 161e126ba97SEli Cohen next->next_wqe_index = 162e126ba97SEli Cohen cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 163e126ba97SEli Cohen } 164e126ba97SEli Cohen 1650fd27a88SLeon Romanovsky mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 166778e1cddSKees Cook in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 167af1ba291SArtemy Kovalyov if (!in->pas) { 168e126ba97SEli Cohen err = -ENOMEM; 169e126ba97SEli Cohen goto err_buf; 170e126ba97SEli Cohen } 17120e5a59bSGuy Levi mlx5_fill_page_frag_array(&srq->buf, in->pas); 172e126ba97SEli Cohen 173b5883008SLi Dongyang srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 174e126ba97SEli Cohen if (!srq->wrid) { 175e126ba97SEli Cohen err = -ENOMEM; 176e126ba97SEli Cohen goto err_in; 177e126ba97SEli Cohen } 178c48d386bSLeon Romanovsky srq->wq_sig = 0; 179e126ba97SEli Cohen 1800fd27a88SLeon Romanovsky in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 181af1ba291SArtemy Kovalyov if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 1823fd3307eSArtemy Kovalyov in->type != IB_SRQT_BASIC) 183af1ba291SArtemy Kovalyov in->user_index = MLX5_IB_DEFAULT_UIDX; 184cfb5e088SHaggai Abramovsky 185e126ba97SEli Cohen return 0; 186e126ba97SEli Cohen 187e126ba97SEli Cohen err_in: 188af1ba291SArtemy Kovalyov kvfree(in->pas); 189e126ba97SEli Cohen 190e126ba97SEli Cohen err_buf: 19120e5a59bSGuy Levi mlx5_frag_buf_free(dev->mdev, &srq->buf); 192e126ba97SEli Cohen 193e126ba97SEli Cohen err_db: 1949603b61dSJack Morgenstein mlx5_db_free(dev->mdev, &srq->db); 195e126ba97SEli Cohen return err; 196e126ba97SEli Cohen } 197e126ba97SEli Cohen 198e126ba97SEli Cohen static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq) 199e126ba97SEli Cohen { 200e126ba97SEli Cohen mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); 201e126ba97SEli Cohen ib_umem_release(srq->umem); 202e126ba97SEli Cohen } 203e126ba97SEli Cohen 204e126ba97SEli Cohen 205e126ba97SEli Cohen static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 206e126ba97SEli Cohen { 207b5883008SLi Dongyang kvfree(srq->wrid); 20820e5a59bSGuy Levi mlx5_frag_buf_free(dev->mdev, &srq->buf); 2099603b61dSJack Morgenstein mlx5_db_free(dev->mdev, &srq->db); 210e126ba97SEli Cohen } 211e126ba97SEli Cohen 212e126ba97SEli Cohen struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, 213e126ba97SEli Cohen struct ib_srq_init_attr *init_attr, 214e126ba97SEli Cohen struct ib_udata *udata) 215e126ba97SEli Cohen { 216e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(pd->device); 217e126ba97SEli Cohen struct mlx5_ib_srq *srq; 218c2b37f76SBoris Pismenny size_t desc_size; 219c2b37f76SBoris Pismenny size_t buf_size; 220e126ba97SEli Cohen int err; 221af1ba291SArtemy Kovalyov struct mlx5_srq_attr in = {0}; 222938fe83cSSaeed Mahameed __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 223e126ba97SEli Cohen 224e126ba97SEli Cohen /* Sanity check SRQ size before proceeding */ 225938fe83cSSaeed Mahameed if (init_attr->attr.max_wr >= max_srq_wqes) { 226e126ba97SEli Cohen mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 227e126ba97SEli Cohen init_attr->attr.max_wr, 228938fe83cSSaeed Mahameed max_srq_wqes); 229e126ba97SEli Cohen return ERR_PTR(-EINVAL); 230e126ba97SEli Cohen } 231e126ba97SEli Cohen 232e126ba97SEli Cohen srq = kmalloc(sizeof(*srq), GFP_KERNEL); 233e126ba97SEli Cohen if (!srq) 234e126ba97SEli Cohen return ERR_PTR(-ENOMEM); 235e126ba97SEli Cohen 236e126ba97SEli Cohen mutex_init(&srq->mutex); 237e126ba97SEli Cohen spin_lock_init(&srq->lock); 238e126ba97SEli Cohen srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 239e126ba97SEli Cohen srq->msrq.max_gs = init_attr->attr.max_sge; 240e126ba97SEli Cohen 241e126ba97SEli Cohen desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 242e126ba97SEli Cohen srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 243d63c4673SKamal Heib if (desc_size == 0 || srq->msrq.max_gs > desc_size) { 244d63c4673SKamal Heib err = -EINVAL; 245d63c4673SKamal Heib goto err_srq; 246d63c4673SKamal Heib } 247e126ba97SEli Cohen desc_size = roundup_pow_of_two(desc_size); 248c2b37f76SBoris Pismenny desc_size = max_t(size_t, 32, desc_size); 249d63c4673SKamal Heib if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) { 250d63c4673SKamal Heib err = -EINVAL; 251d63c4673SKamal Heib goto err_srq; 252d63c4673SKamal Heib } 253e126ba97SEli Cohen srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 254e126ba97SEli Cohen sizeof(struct mlx5_wqe_data_seg); 255e126ba97SEli Cohen srq->msrq.wqe_shift = ilog2(desc_size); 256e126ba97SEli Cohen buf_size = srq->msrq.max * desc_size; 257d63c4673SKamal Heib if (buf_size < desc_size) { 258d63c4673SKamal Heib err = -EINVAL; 259d63c4673SKamal Heib goto err_srq; 260d63c4673SKamal Heib } 261c73b7911SMaor Gottlieb in.type = init_attr->srq_type; 262e126ba97SEli Cohen 263e00b64f7SShamir Rabinovitch if (udata) 264af1ba291SArtemy Kovalyov err = create_srq_user(pd, srq, &in, udata, buf_size); 265e126ba97SEli Cohen else 266af1ba291SArtemy Kovalyov err = create_srq_kernel(dev, srq, &in, buf_size); 267e126ba97SEli Cohen 268e126ba97SEli Cohen if (err) { 269e126ba97SEli Cohen mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 270e00b64f7SShamir Rabinovitch udata ? "user" : "kernel", err); 271e126ba97SEli Cohen goto err_srq; 272e126ba97SEli Cohen } 273e126ba97SEli Cohen 274af1ba291SArtemy Kovalyov in.log_size = ilog2(srq->msrq.max); 275af1ba291SArtemy Kovalyov in.wqe_shift = srq->msrq.wqe_shift - 4; 276af1ba291SArtemy Kovalyov if (srq->wq_sig) 277af1ba291SArtemy Kovalyov in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 2781a56ff6dSArtemy Kovalyov 2791a56ff6dSArtemy Kovalyov if (init_attr->srq_type == IB_SRQT_XRC) 280af1ba291SArtemy Kovalyov in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 2811a56ff6dSArtemy Kovalyov else 282af1ba291SArtemy Kovalyov in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; 2831a56ff6dSArtemy Kovalyov 2843fd3307eSArtemy Kovalyov if (init_attr->srq_type == IB_SRQT_TM) { 2853fd3307eSArtemy Kovalyov in.tm_log_list_size = 2863fd3307eSArtemy Kovalyov ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 2873fd3307eSArtemy Kovalyov if (in.tm_log_list_size > 2883fd3307eSArtemy Kovalyov MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 2893fd3307eSArtemy Kovalyov mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 2903fd3307eSArtemy Kovalyov err = -EINVAL; 2913fd3307eSArtemy Kovalyov goto err_usr_kern_srq; 2923fd3307eSArtemy Kovalyov } 2933fd3307eSArtemy Kovalyov in.flags |= MLX5_SRQ_FLAG_RNDV; 2943fd3307eSArtemy Kovalyov } 2953fd3307eSArtemy Kovalyov 2961a56ff6dSArtemy Kovalyov if (ib_srq_has_cq(init_attr->srq_type)) 2971a56ff6dSArtemy Kovalyov in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 2981a56ff6dSArtemy Kovalyov else 299af1ba291SArtemy Kovalyov in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 300e126ba97SEli Cohen 301af1ba291SArtemy Kovalyov in.pd = to_mpd(pd)->pdn; 302af1ba291SArtemy Kovalyov in.db_record = srq->db.dma; 303b4990804SLeon Romanovsky err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 304af1ba291SArtemy Kovalyov kvfree(in.pas); 305e126ba97SEli Cohen if (err) { 306e126ba97SEli Cohen mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 30756e1ab0fSMoshe Lazer goto err_usr_kern_srq; 308e126ba97SEli Cohen } 309e126ba97SEli Cohen 310e126ba97SEli Cohen mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 311e126ba97SEli Cohen 312e126ba97SEli Cohen srq->msrq.event = mlx5_ib_srq_event; 313e126ba97SEli Cohen srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 314e126ba97SEli Cohen 315e00b64f7SShamir Rabinovitch if (udata) 316e126ba97SEli Cohen if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) { 317e126ba97SEli Cohen mlx5_ib_dbg(dev, "copy to user failed\n"); 318e126ba97SEli Cohen err = -EFAULT; 319e126ba97SEli Cohen goto err_core; 320e126ba97SEli Cohen } 321e126ba97SEli Cohen 322e126ba97SEli Cohen init_attr->attr.max_wr = srq->msrq.max - 1; 323e126ba97SEli Cohen 324e126ba97SEli Cohen return &srq->ibsrq; 325e126ba97SEli Cohen 326e126ba97SEli Cohen err_core: 327b4990804SLeon Romanovsky mlx5_cmd_destroy_srq(dev, &srq->msrq); 32856e1ab0fSMoshe Lazer 32956e1ab0fSMoshe Lazer err_usr_kern_srq: 330e00b64f7SShamir Rabinovitch if (udata) 331e126ba97SEli Cohen destroy_srq_user(pd, srq); 332e126ba97SEli Cohen else 333e126ba97SEli Cohen destroy_srq_kernel(dev, srq); 334e126ba97SEli Cohen 335e126ba97SEli Cohen err_srq: 336e126ba97SEli Cohen kfree(srq); 337e126ba97SEli Cohen 338e126ba97SEli Cohen return ERR_PTR(err); 339e126ba97SEli Cohen } 340e126ba97SEli Cohen 341e126ba97SEli Cohen int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 342e126ba97SEli Cohen enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 343e126ba97SEli Cohen { 344e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 345e126ba97SEli Cohen struct mlx5_ib_srq *srq = to_msrq(ibsrq); 346e126ba97SEli Cohen int ret; 347e126ba97SEli Cohen 348e126ba97SEli Cohen /* We don't support resizing SRQs yet */ 349e126ba97SEli Cohen if (attr_mask & IB_SRQ_MAX_WR) 350e126ba97SEli Cohen return -EINVAL; 351e126ba97SEli Cohen 352e126ba97SEli Cohen if (attr_mask & IB_SRQ_LIMIT) { 353e126ba97SEli Cohen if (attr->srq_limit >= srq->msrq.max) 354e126ba97SEli Cohen return -EINVAL; 355e126ba97SEli Cohen 356e126ba97SEli Cohen mutex_lock(&srq->mutex); 357b4990804SLeon Romanovsky ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 358e126ba97SEli Cohen mutex_unlock(&srq->mutex); 359e126ba97SEli Cohen 360e126ba97SEli Cohen if (ret) 361e126ba97SEli Cohen return ret; 362e126ba97SEli Cohen } 363e126ba97SEli Cohen 364e126ba97SEli Cohen return 0; 365e126ba97SEli Cohen } 366e126ba97SEli Cohen 367e126ba97SEli Cohen int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 368e126ba97SEli Cohen { 369e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 370e126ba97SEli Cohen struct mlx5_ib_srq *srq = to_msrq(ibsrq); 371e126ba97SEli Cohen int ret; 372af1ba291SArtemy Kovalyov struct mlx5_srq_attr *out; 373e126ba97SEli Cohen 374e126ba97SEli Cohen out = kzalloc(sizeof(*out), GFP_KERNEL); 375e126ba97SEli Cohen if (!out) 376e126ba97SEli Cohen return -ENOMEM; 377e126ba97SEli Cohen 378b4990804SLeon Romanovsky ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 379e126ba97SEli Cohen if (ret) 380e126ba97SEli Cohen goto out_box; 381e126ba97SEli Cohen 382af1ba291SArtemy Kovalyov srq_attr->srq_limit = out->lwm; 383e126ba97SEli Cohen srq_attr->max_wr = srq->msrq.max - 1; 384e126ba97SEli Cohen srq_attr->max_sge = srq->msrq.max_gs; 385e126ba97SEli Cohen 386e126ba97SEli Cohen out_box: 387e126ba97SEli Cohen kfree(out); 388e126ba97SEli Cohen return ret; 389e126ba97SEli Cohen } 390e126ba97SEli Cohen 391e126ba97SEli Cohen int mlx5_ib_destroy_srq(struct ib_srq *srq) 392e126ba97SEli Cohen { 393e126ba97SEli Cohen struct mlx5_ib_dev *dev = to_mdev(srq->device); 394e126ba97SEli Cohen struct mlx5_ib_srq *msrq = to_msrq(srq); 395e126ba97SEli Cohen 396b4990804SLeon Romanovsky mlx5_cmd_destroy_srq(dev, &msrq->msrq); 397e126ba97SEli Cohen 398e126ba97SEli Cohen if (srq->uobject) { 399e126ba97SEli Cohen mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 400e126ba97SEli Cohen ib_umem_release(msrq->umem); 401e126ba97SEli Cohen } else { 4021faacf82SEli Cohen destroy_srq_kernel(dev, msrq); 403e126ba97SEli Cohen } 404e126ba97SEli Cohen 405e126ba97SEli Cohen kfree(srq); 406e126ba97SEli Cohen return 0; 407e126ba97SEli Cohen } 408e126ba97SEli Cohen 409e126ba97SEli Cohen void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 410e126ba97SEli Cohen { 411e126ba97SEli Cohen struct mlx5_wqe_srq_next_seg *next; 412e126ba97SEli Cohen 413e126ba97SEli Cohen /* always called with interrupts disabled. */ 414e126ba97SEli Cohen spin_lock(&srq->lock); 415e126ba97SEli Cohen 416e126ba97SEli Cohen next = get_wqe(srq, srq->tail); 417e126ba97SEli Cohen next->next_wqe_index = cpu_to_be16(wqe_index); 418e126ba97SEli Cohen srq->tail = wqe_index; 419e126ba97SEli Cohen 420e126ba97SEli Cohen spin_unlock(&srq->lock); 421e126ba97SEli Cohen } 422e126ba97SEli Cohen 423d34ac5cdSBart Van Assche int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 424d34ac5cdSBart Van Assche const struct ib_recv_wr **bad_wr) 425e126ba97SEli Cohen { 426e126ba97SEli Cohen struct mlx5_ib_srq *srq = to_msrq(ibsrq); 427e126ba97SEli Cohen struct mlx5_wqe_srq_next_seg *next; 428e126ba97SEli Cohen struct mlx5_wqe_data_seg *scat; 42989ea94a7SMaor Gottlieb struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 43089ea94a7SMaor Gottlieb struct mlx5_core_dev *mdev = dev->mdev; 431e126ba97SEli Cohen unsigned long flags; 432e126ba97SEli Cohen int err = 0; 433e126ba97SEli Cohen int nreq; 434e126ba97SEli Cohen int i; 435e126ba97SEli Cohen 436e126ba97SEli Cohen spin_lock_irqsave(&srq->lock, flags); 437e126ba97SEli Cohen 43889ea94a7SMaor Gottlieb if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 43989ea94a7SMaor Gottlieb err = -EIO; 44089ea94a7SMaor Gottlieb *bad_wr = wr; 44189ea94a7SMaor Gottlieb goto out; 44289ea94a7SMaor Gottlieb } 44389ea94a7SMaor Gottlieb 444e126ba97SEli Cohen for (nreq = 0; wr; nreq++, wr = wr->next) { 445e126ba97SEli Cohen if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 446e126ba97SEli Cohen err = -EINVAL; 447e126ba97SEli Cohen *bad_wr = wr; 448e126ba97SEli Cohen break; 449e126ba97SEli Cohen } 450e126ba97SEli Cohen 451e126ba97SEli Cohen if (unlikely(srq->head == srq->tail)) { 452e126ba97SEli Cohen err = -ENOMEM; 453e126ba97SEli Cohen *bad_wr = wr; 454e126ba97SEli Cohen break; 455e126ba97SEli Cohen } 456e126ba97SEli Cohen 457e126ba97SEli Cohen srq->wrid[srq->head] = wr->wr_id; 458e126ba97SEli Cohen 459e126ba97SEli Cohen next = get_wqe(srq, srq->head); 460e126ba97SEli Cohen srq->head = be16_to_cpu(next->next_wqe_index); 461e126ba97SEli Cohen scat = (struct mlx5_wqe_data_seg *)(next + 1); 462e126ba97SEli Cohen 463e126ba97SEli Cohen for (i = 0; i < wr->num_sge; i++) { 464e126ba97SEli Cohen scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 465e126ba97SEli Cohen scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 466e126ba97SEli Cohen scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 467e126ba97SEli Cohen } 468e126ba97SEli Cohen 469e126ba97SEli Cohen if (i < srq->msrq.max_avail_gather) { 470e126ba97SEli Cohen scat[i].byte_count = 0; 471e126ba97SEli Cohen scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 472e126ba97SEli Cohen scat[i].addr = 0; 473e126ba97SEli Cohen } 474e126ba97SEli Cohen } 475e126ba97SEli Cohen 476e126ba97SEli Cohen if (likely(nreq)) { 477e126ba97SEli Cohen srq->wqe_ctr += nreq; 478e126ba97SEli Cohen 479e126ba97SEli Cohen /* Make sure that descriptors are written before 480e126ba97SEli Cohen * doorbell record. 481e126ba97SEli Cohen */ 482e126ba97SEli Cohen wmb(); 483e126ba97SEli Cohen 484e126ba97SEli Cohen *srq->db.db = cpu_to_be32(srq->wqe_ctr); 485e126ba97SEli Cohen } 48689ea94a7SMaor Gottlieb out: 487e126ba97SEli Cohen spin_unlock_irqrestore(&srq->lock, flags); 488e126ba97SEli Cohen 489e126ba97SEli Cohen return err; 490e126ba97SEli Cohen } 491