1 /* 2 * Copyright(c) 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/err.h> 49 #include <linux/slab.h> 50 #include <linux/vmalloc.h> 51 52 #include "srq.h" 53 #include "vt.h" 54 55 /** 56 * rvt_driver_srq_init - init srq resources on a per driver basis 57 * @rdi: rvt dev structure 58 * 59 * Do any initialization needed when a driver registers with rdmavt. 60 */ 61 void rvt_driver_srq_init(struct rvt_dev_info *rdi) 62 { 63 spin_lock_init(&rdi->n_srqs_lock); 64 rdi->n_srqs_allocated = 0; 65 } 66 67 /** 68 * rvt_create_srq - create a shared receive queue 69 * @ibpd: the protection domain of the SRQ to create 70 * @srq_init_attr: the attributes of the SRQ 71 * @udata: data from libibverbs when creating a user SRQ 72 * 73 * Return: Allocated srq object 74 */ 75 struct ib_srq *rvt_create_srq(struct ib_pd *ibpd, 76 struct ib_srq_init_attr *srq_init_attr, 77 struct ib_udata *udata) 78 { 79 struct rvt_dev_info *dev = ib_to_rvt(ibpd->device); 80 struct rvt_srq *srq; 81 u32 sz; 82 struct ib_srq *ret; 83 84 if (srq_init_attr->srq_type != IB_SRQT_BASIC) 85 return ERR_PTR(-ENOSYS); 86 87 if (srq_init_attr->attr.max_sge == 0 || 88 srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge || 89 srq_init_attr->attr.max_wr == 0 || 90 srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr) 91 return ERR_PTR(-EINVAL); 92 93 srq = kmalloc(sizeof(*srq), GFP_KERNEL); 94 if (!srq) 95 return ERR_PTR(-ENOMEM); 96 97 /* 98 * Need to use vmalloc() if we want to support large #s of entries. 99 */ 100 srq->rq.size = srq_init_attr->attr.max_wr + 1; 101 srq->rq.max_sge = srq_init_attr->attr.max_sge; 102 sz = sizeof(struct ib_sge) * srq->rq.max_sge + 103 sizeof(struct rvt_rwqe); 104 srq->rq.wq = vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz); 105 if (!srq->rq.wq) { 106 ret = ERR_PTR(-ENOMEM); 107 goto bail_srq; 108 } 109 110 /* 111 * Return the address of the RWQ as the offset to mmap. 112 * See rvt_mmap() for details. 113 */ 114 if (udata && udata->outlen >= sizeof(__u64)) { 115 int err; 116 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz; 117 118 srq->ip = 119 rvt_create_mmap_info(dev, s, ibpd->uobject->context, 120 srq->rq.wq); 121 if (!srq->ip) { 122 ret = ERR_PTR(-ENOMEM); 123 goto bail_wq; 124 } 125 126 err = ib_copy_to_udata(udata, &srq->ip->offset, 127 sizeof(srq->ip->offset)); 128 if (err) { 129 ret = ERR_PTR(err); 130 goto bail_ip; 131 } 132 } else { 133 srq->ip = NULL; 134 } 135 136 /* 137 * ib_create_srq() will initialize srq->ibsrq. 138 */ 139 spin_lock_init(&srq->rq.lock); 140 srq->rq.wq->head = 0; 141 srq->rq.wq->tail = 0; 142 srq->limit = srq_init_attr->attr.srq_limit; 143 144 spin_lock(&dev->n_srqs_lock); 145 if (dev->n_srqs_allocated == dev->dparms.props.max_srq) { 146 spin_unlock(&dev->n_srqs_lock); 147 ret = ERR_PTR(-ENOMEM); 148 goto bail_ip; 149 } 150 151 dev->n_srqs_allocated++; 152 spin_unlock(&dev->n_srqs_lock); 153 154 if (srq->ip) { 155 spin_lock_irq(&dev->pending_lock); 156 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); 157 spin_unlock_irq(&dev->pending_lock); 158 } 159 160 return &srq->ibsrq; 161 162 bail_ip: 163 kfree(srq->ip); 164 bail_wq: 165 vfree(srq->rq.wq); 166 bail_srq: 167 kfree(srq); 168 return ret; 169 } 170 171 /** 172 * rvt_modify_srq - modify a shared receive queue 173 * @ibsrq: the SRQ to modify 174 * @attr: the new attributes of the SRQ 175 * @attr_mask: indicates which attributes to modify 176 * @udata: user data for libibverbs.so 177 * 178 * Return: 0 on success 179 */ 180 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 181 enum ib_srq_attr_mask attr_mask, 182 struct ib_udata *udata) 183 { 184 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 185 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 186 struct rvt_rwq *wq; 187 int ret = 0; 188 189 if (attr_mask & IB_SRQ_MAX_WR) { 190 struct rvt_rwq *owq; 191 struct rvt_rwqe *p; 192 u32 sz, size, n, head, tail; 193 194 /* Check that the requested sizes are below the limits. */ 195 if ((attr->max_wr > dev->dparms.props.max_srq_wr) || 196 ((attr_mask & IB_SRQ_LIMIT) ? 197 attr->srq_limit : srq->limit) > attr->max_wr) 198 return -EINVAL; 199 200 sz = sizeof(struct rvt_rwqe) + 201 srq->rq.max_sge * sizeof(struct ib_sge); 202 size = attr->max_wr + 1; 203 wq = vmalloc_user(sizeof(struct rvt_rwq) + size * sz); 204 if (!wq) 205 return -ENOMEM; 206 207 /* Check that we can write the offset to mmap. */ 208 if (udata && udata->inlen >= sizeof(__u64)) { 209 __u64 offset_addr; 210 __u64 offset = 0; 211 212 ret = ib_copy_from_udata(&offset_addr, udata, 213 sizeof(offset_addr)); 214 if (ret) 215 goto bail_free; 216 udata->outbuf = (void __user *) 217 (unsigned long)offset_addr; 218 ret = ib_copy_to_udata(udata, &offset, 219 sizeof(offset)); 220 if (ret) 221 goto bail_free; 222 } 223 224 spin_lock_irq(&srq->rq.lock); 225 /* 226 * validate head and tail pointer values and compute 227 * the number of remaining WQEs. 228 */ 229 owq = srq->rq.wq; 230 head = owq->head; 231 tail = owq->tail; 232 if (head >= srq->rq.size || tail >= srq->rq.size) { 233 ret = -EINVAL; 234 goto bail_unlock; 235 } 236 n = head; 237 if (n < tail) 238 n += srq->rq.size - tail; 239 else 240 n -= tail; 241 if (size <= n) { 242 ret = -EINVAL; 243 goto bail_unlock; 244 } 245 n = 0; 246 p = wq->wq; 247 while (tail != head) { 248 struct rvt_rwqe *wqe; 249 int i; 250 251 wqe = rvt_get_rwqe_ptr(&srq->rq, tail); 252 p->wr_id = wqe->wr_id; 253 p->num_sge = wqe->num_sge; 254 for (i = 0; i < wqe->num_sge; i++) 255 p->sg_list[i] = wqe->sg_list[i]; 256 n++; 257 p = (struct rvt_rwqe *)((char *)p + sz); 258 if (++tail >= srq->rq.size) 259 tail = 0; 260 } 261 srq->rq.wq = wq; 262 srq->rq.size = size; 263 wq->head = n; 264 wq->tail = 0; 265 if (attr_mask & IB_SRQ_LIMIT) 266 srq->limit = attr->srq_limit; 267 spin_unlock_irq(&srq->rq.lock); 268 269 vfree(owq); 270 271 if (srq->ip) { 272 struct rvt_mmap_info *ip = srq->ip; 273 struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device); 274 u32 s = sizeof(struct rvt_rwq) + size * sz; 275 276 rvt_update_mmap_info(dev, ip, s, wq); 277 278 /* 279 * Return the offset to mmap. 280 * See rvt_mmap() for details. 281 */ 282 if (udata && udata->inlen >= sizeof(__u64)) { 283 ret = ib_copy_to_udata(udata, &ip->offset, 284 sizeof(ip->offset)); 285 if (ret) 286 return ret; 287 } 288 289 /* 290 * Put user mapping info onto the pending list 291 * unless it already is on the list. 292 */ 293 spin_lock_irq(&dev->pending_lock); 294 if (list_empty(&ip->pending_mmaps)) 295 list_add(&ip->pending_mmaps, 296 &dev->pending_mmaps); 297 spin_unlock_irq(&dev->pending_lock); 298 } 299 } else if (attr_mask & IB_SRQ_LIMIT) { 300 spin_lock_irq(&srq->rq.lock); 301 if (attr->srq_limit >= srq->rq.size) 302 ret = -EINVAL; 303 else 304 srq->limit = attr->srq_limit; 305 spin_unlock_irq(&srq->rq.lock); 306 } 307 return ret; 308 309 bail_unlock: 310 spin_unlock_irq(&srq->rq.lock); 311 bail_free: 312 vfree(wq); 313 return ret; 314 } 315 316 /** rvt_query_srq - query srq data 317 * @ibsrq: srq to query 318 * @attr: return info in attr 319 * 320 * Return: always 0 321 */ 322 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 323 { 324 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 325 326 attr->max_wr = srq->rq.size - 1; 327 attr->max_sge = srq->rq.max_sge; 328 attr->srq_limit = srq->limit; 329 return 0; 330 } 331 332 /** 333 * rvt_destroy_srq - destory an srq 334 * @ibsrq: srq object to destroy 335 * 336 * Return always 0 337 */ 338 int rvt_destroy_srq(struct ib_srq *ibsrq) 339 { 340 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 341 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 342 343 spin_lock(&dev->n_srqs_lock); 344 dev->n_srqs_allocated--; 345 spin_unlock(&dev->n_srqs_lock); 346 if (srq->ip) 347 kref_put(&srq->ip->ref, rvt_release_mmap_info); 348 else 349 vfree(srq->rq.wq); 350 kfree(srq); 351 352 return 0; 353 } 354