1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 /* 3 * Copyright(c) 2016 Intel Corporation. 4 */ 5 6 #include <linux/err.h> 7 #include <linux/slab.h> 8 #include <linux/vmalloc.h> 9 #include <rdma/uverbs_ioctl.h> 10 11 #include "srq.h" 12 #include "vt.h" 13 #include "qp.h" 14 /** 15 * rvt_driver_srq_init - init srq resources on a per driver basis 16 * @rdi: rvt dev structure 17 * 18 * Do any initialization needed when a driver registers with rdmavt. 19 */ 20 void rvt_driver_srq_init(struct rvt_dev_info *rdi) 21 { 22 spin_lock_init(&rdi->n_srqs_lock); 23 rdi->n_srqs_allocated = 0; 24 } 25 26 /** 27 * rvt_create_srq - create a shared receive queue 28 * @ibsrq: the protection domain of the SRQ to create 29 * @srq_init_attr: the attributes of the SRQ 30 * @udata: data from libibverbs when creating a user SRQ 31 * 32 * Return: 0 on success 33 */ 34 int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr, 35 struct ib_udata *udata) 36 { 37 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 38 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 39 u32 sz; 40 int ret; 41 42 if (srq_init_attr->srq_type != IB_SRQT_BASIC) 43 return -EOPNOTSUPP; 44 45 if (srq_init_attr->attr.max_sge == 0 || 46 srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge || 47 srq_init_attr->attr.max_wr == 0 || 48 srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr) 49 return -EINVAL; 50 51 /* 52 * Need to use vmalloc() if we want to support large #s of entries. 53 */ 54 srq->rq.size = srq_init_attr->attr.max_wr + 1; 55 srq->rq.max_sge = srq_init_attr->attr.max_sge; 56 sz = sizeof(struct ib_sge) * srq->rq.max_sge + 57 sizeof(struct rvt_rwqe); 58 if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz, 59 dev->dparms.node, udata)) { 60 ret = -ENOMEM; 61 goto bail_srq; 62 } 63 64 /* 65 * Return the address of the RWQ as the offset to mmap. 66 * See rvt_mmap() for details. 67 */ 68 if (udata && udata->outlen >= sizeof(__u64)) { 69 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz; 70 71 srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq); 72 if (IS_ERR(srq->ip)) { 73 ret = PTR_ERR(srq->ip); 74 goto bail_wq; 75 } 76 77 ret = ib_copy_to_udata(udata, &srq->ip->offset, 78 sizeof(srq->ip->offset)); 79 if (ret) 80 goto bail_ip; 81 } 82 83 /* 84 * ib_create_srq() will initialize srq->ibsrq. 85 */ 86 spin_lock_init(&srq->rq.lock); 87 srq->limit = srq_init_attr->attr.srq_limit; 88 89 spin_lock(&dev->n_srqs_lock); 90 if (dev->n_srqs_allocated == dev->dparms.props.max_srq) { 91 spin_unlock(&dev->n_srqs_lock); 92 ret = -ENOMEM; 93 goto bail_ip; 94 } 95 96 dev->n_srqs_allocated++; 97 spin_unlock(&dev->n_srqs_lock); 98 99 if (srq->ip) { 100 spin_lock_irq(&dev->pending_lock); 101 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); 102 spin_unlock_irq(&dev->pending_lock); 103 } 104 105 return 0; 106 107 bail_ip: 108 kfree(srq->ip); 109 bail_wq: 110 rvt_free_rq(&srq->rq); 111 bail_srq: 112 return ret; 113 } 114 115 /** 116 * rvt_modify_srq - modify a shared receive queue 117 * @ibsrq: the SRQ to modify 118 * @attr: the new attributes of the SRQ 119 * @attr_mask: indicates which attributes to modify 120 * @udata: user data for libibverbs.so 121 * 122 * Return: 0 on success 123 */ 124 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 125 enum ib_srq_attr_mask attr_mask, 126 struct ib_udata *udata) 127 { 128 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 129 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 130 struct rvt_rq tmp_rq = {}; 131 int ret = 0; 132 133 if (attr_mask & IB_SRQ_MAX_WR) { 134 struct rvt_krwq *okwq = NULL; 135 struct rvt_rwq *owq = NULL; 136 struct rvt_rwqe *p; 137 u32 sz, size, n, head, tail; 138 139 /* Check that the requested sizes are below the limits. */ 140 if ((attr->max_wr > dev->dparms.props.max_srq_wr) || 141 ((attr_mask & IB_SRQ_LIMIT) ? 142 attr->srq_limit : srq->limit) > attr->max_wr) 143 return -EINVAL; 144 sz = sizeof(struct rvt_rwqe) + 145 srq->rq.max_sge * sizeof(struct ib_sge); 146 size = attr->max_wr + 1; 147 if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node, 148 udata)) 149 return -ENOMEM; 150 /* Check that we can write the offset to mmap. */ 151 if (udata && udata->inlen >= sizeof(__u64)) { 152 __u64 offset_addr; 153 __u64 offset = 0; 154 155 ret = ib_copy_from_udata(&offset_addr, udata, 156 sizeof(offset_addr)); 157 if (ret) 158 goto bail_free; 159 udata->outbuf = (void __user *) 160 (unsigned long)offset_addr; 161 ret = ib_copy_to_udata(udata, &offset, 162 sizeof(offset)); 163 if (ret) 164 goto bail_free; 165 } 166 167 spin_lock_irq(&srq->rq.kwq->c_lock); 168 /* 169 * validate head and tail pointer values and compute 170 * the number of remaining WQEs. 171 */ 172 if (udata) { 173 owq = srq->rq.wq; 174 head = RDMA_READ_UAPI_ATOMIC(owq->head); 175 tail = RDMA_READ_UAPI_ATOMIC(owq->tail); 176 } else { 177 okwq = srq->rq.kwq; 178 head = okwq->head; 179 tail = okwq->tail; 180 } 181 if (head >= srq->rq.size || tail >= srq->rq.size) { 182 ret = -EINVAL; 183 goto bail_unlock; 184 } 185 n = head; 186 if (n < tail) 187 n += srq->rq.size - tail; 188 else 189 n -= tail; 190 if (size <= n) { 191 ret = -EINVAL; 192 goto bail_unlock; 193 } 194 n = 0; 195 p = tmp_rq.kwq->curr_wq; 196 while (tail != head) { 197 struct rvt_rwqe *wqe; 198 int i; 199 200 wqe = rvt_get_rwqe_ptr(&srq->rq, tail); 201 p->wr_id = wqe->wr_id; 202 p->num_sge = wqe->num_sge; 203 for (i = 0; i < wqe->num_sge; i++) 204 p->sg_list[i] = wqe->sg_list[i]; 205 n++; 206 p = (struct rvt_rwqe *)((char *)p + sz); 207 if (++tail >= srq->rq.size) 208 tail = 0; 209 } 210 srq->rq.kwq = tmp_rq.kwq; 211 if (udata) { 212 srq->rq.wq = tmp_rq.wq; 213 RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n); 214 RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0); 215 } else { 216 tmp_rq.kwq->head = n; 217 tmp_rq.kwq->tail = 0; 218 } 219 srq->rq.size = size; 220 if (attr_mask & IB_SRQ_LIMIT) 221 srq->limit = attr->srq_limit; 222 spin_unlock_irq(&srq->rq.kwq->c_lock); 223 224 vfree(owq); 225 kvfree(okwq); 226 227 if (srq->ip) { 228 struct rvt_mmap_info *ip = srq->ip; 229 struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device); 230 u32 s = sizeof(struct rvt_rwq) + size * sz; 231 232 rvt_update_mmap_info(dev, ip, s, tmp_rq.wq); 233 234 /* 235 * Return the offset to mmap. 236 * See rvt_mmap() for details. 237 */ 238 if (udata && udata->inlen >= sizeof(__u64)) { 239 ret = ib_copy_to_udata(udata, &ip->offset, 240 sizeof(ip->offset)); 241 if (ret) 242 return ret; 243 } 244 245 /* 246 * Put user mapping info onto the pending list 247 * unless it already is on the list. 248 */ 249 spin_lock_irq(&dev->pending_lock); 250 if (list_empty(&ip->pending_mmaps)) 251 list_add(&ip->pending_mmaps, 252 &dev->pending_mmaps); 253 spin_unlock_irq(&dev->pending_lock); 254 } 255 } else if (attr_mask & IB_SRQ_LIMIT) { 256 spin_lock_irq(&srq->rq.kwq->c_lock); 257 if (attr->srq_limit >= srq->rq.size) 258 ret = -EINVAL; 259 else 260 srq->limit = attr->srq_limit; 261 spin_unlock_irq(&srq->rq.kwq->c_lock); 262 } 263 return ret; 264 265 bail_unlock: 266 spin_unlock_irq(&srq->rq.kwq->c_lock); 267 bail_free: 268 rvt_free_rq(&tmp_rq); 269 return ret; 270 } 271 272 /** 273 * rvt_query_srq - query srq data 274 * @ibsrq: srq to query 275 * @attr: return info in attr 276 * 277 * Return: always 0 278 */ 279 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 280 { 281 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 282 283 attr->max_wr = srq->rq.size - 1; 284 attr->max_sge = srq->rq.max_sge; 285 attr->srq_limit = srq->limit; 286 return 0; 287 } 288 289 /** 290 * rvt_destroy_srq - destory an srq 291 * @ibsrq: srq object to destroy 292 * @udata: user data for libibverbs.so 293 */ 294 int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 295 { 296 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 297 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 298 299 spin_lock(&dev->n_srqs_lock); 300 dev->n_srqs_allocated--; 301 spin_unlock(&dev->n_srqs_lock); 302 if (srq->ip) 303 kref_put(&srq->ip->ref, rvt_release_mmap_info); 304 kvfree(srq->rq.kwq); 305 return 0; 306 } 307