1 /* 2 * Copyright(c) 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 #include <linux/err.h> 49 #include <linux/slab.h> 50 #include <linux/vmalloc.h> 51 #include <rdma/uverbs_ioctl.h> 52 53 #include "srq.h" 54 #include "vt.h" 55 56 /** 57 * rvt_driver_srq_init - init srq resources on a per driver basis 58 * @rdi: rvt dev structure 59 * 60 * Do any initialization needed when a driver registers with rdmavt. 61 */ 62 void rvt_driver_srq_init(struct rvt_dev_info *rdi) 63 { 64 spin_lock_init(&rdi->n_srqs_lock); 65 rdi->n_srqs_allocated = 0; 66 } 67 68 /** 69 * rvt_create_srq - create a shared receive queue 70 * @ibpd: the protection domain of the SRQ to create 71 * @srq_init_attr: the attributes of the SRQ 72 * @udata: data from libibverbs when creating a user SRQ 73 * 74 * Return: 0 on success 75 */ 76 int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr, 77 struct ib_udata *udata) 78 { 79 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 80 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 81 u32 sz; 82 int ret; 83 84 if (srq_init_attr->srq_type != IB_SRQT_BASIC) 85 return -EOPNOTSUPP; 86 87 if (srq_init_attr->attr.max_sge == 0 || 88 srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge || 89 srq_init_attr->attr.max_wr == 0 || 90 srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr) 91 return -EINVAL; 92 93 /* 94 * Need to use vmalloc() if we want to support large #s of entries. 95 */ 96 srq->rq.size = srq_init_attr->attr.max_wr + 1; 97 srq->rq.max_sge = srq_init_attr->attr.max_sge; 98 sz = sizeof(struct ib_sge) * srq->rq.max_sge + 99 sizeof(struct rvt_rwqe); 100 srq->rq.wq = udata ? 101 vmalloc_user(sizeof(struct rvt_rwq) + srq->rq.size * sz) : 102 vzalloc_node(sizeof(struct rvt_rwq) + srq->rq.size * sz, 103 dev->dparms.node); 104 if (!srq->rq.wq) { 105 ret = -ENOMEM; 106 goto bail_srq; 107 } 108 109 /* 110 * Return the address of the RWQ as the offset to mmap. 111 * See rvt_mmap() for details. 112 */ 113 if (udata && udata->outlen >= sizeof(__u64)) { 114 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz; 115 116 srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq); 117 if (!srq->ip) { 118 ret = -ENOMEM; 119 goto bail_wq; 120 } 121 122 ret = ib_copy_to_udata(udata, &srq->ip->offset, 123 sizeof(srq->ip->offset)); 124 if (ret) 125 goto bail_ip; 126 } 127 128 /* 129 * ib_create_srq() will initialize srq->ibsrq. 130 */ 131 spin_lock_init(&srq->rq.lock); 132 srq->limit = srq_init_attr->attr.srq_limit; 133 134 spin_lock(&dev->n_srqs_lock); 135 if (dev->n_srqs_allocated == dev->dparms.props.max_srq) { 136 spin_unlock(&dev->n_srqs_lock); 137 ret = -ENOMEM; 138 goto bail_ip; 139 } 140 141 dev->n_srqs_allocated++; 142 spin_unlock(&dev->n_srqs_lock); 143 144 if (srq->ip) { 145 spin_lock_irq(&dev->pending_lock); 146 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps); 147 spin_unlock_irq(&dev->pending_lock); 148 } 149 150 return 0; 151 152 bail_ip: 153 kfree(srq->ip); 154 bail_wq: 155 vfree(srq->rq.wq); 156 bail_srq: 157 return ret; 158 } 159 160 /** 161 * rvt_modify_srq - modify a shared receive queue 162 * @ibsrq: the SRQ to modify 163 * @attr: the new attributes of the SRQ 164 * @attr_mask: indicates which attributes to modify 165 * @udata: user data for libibverbs.so 166 * 167 * Return: 0 on success 168 */ 169 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 170 enum ib_srq_attr_mask attr_mask, 171 struct ib_udata *udata) 172 { 173 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 174 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 175 struct rvt_rwq *wq; 176 int ret = 0; 177 178 if (attr_mask & IB_SRQ_MAX_WR) { 179 struct rvt_rwq *owq; 180 struct rvt_rwqe *p; 181 u32 sz, size, n, head, tail; 182 183 /* Check that the requested sizes are below the limits. */ 184 if ((attr->max_wr > dev->dparms.props.max_srq_wr) || 185 ((attr_mask & IB_SRQ_LIMIT) ? 186 attr->srq_limit : srq->limit) > attr->max_wr) 187 return -EINVAL; 188 189 sz = sizeof(struct rvt_rwqe) + 190 srq->rq.max_sge * sizeof(struct ib_sge); 191 size = attr->max_wr + 1; 192 wq = udata ? 193 vmalloc_user(sizeof(struct rvt_rwq) + size * sz) : 194 vzalloc_node(sizeof(struct rvt_rwq) + size * sz, 195 dev->dparms.node); 196 if (!wq) 197 return -ENOMEM; 198 199 /* Check that we can write the offset to mmap. */ 200 if (udata && udata->inlen >= sizeof(__u64)) { 201 __u64 offset_addr; 202 __u64 offset = 0; 203 204 ret = ib_copy_from_udata(&offset_addr, udata, 205 sizeof(offset_addr)); 206 if (ret) 207 goto bail_free; 208 udata->outbuf = (void __user *) 209 (unsigned long)offset_addr; 210 ret = ib_copy_to_udata(udata, &offset, 211 sizeof(offset)); 212 if (ret) 213 goto bail_free; 214 } 215 216 spin_lock_irq(&srq->rq.lock); 217 /* 218 * validate head and tail pointer values and compute 219 * the number of remaining WQEs. 220 */ 221 owq = srq->rq.wq; 222 head = owq->head; 223 tail = owq->tail; 224 if (head >= srq->rq.size || tail >= srq->rq.size) { 225 ret = -EINVAL; 226 goto bail_unlock; 227 } 228 n = head; 229 if (n < tail) 230 n += srq->rq.size - tail; 231 else 232 n -= tail; 233 if (size <= n) { 234 ret = -EINVAL; 235 goto bail_unlock; 236 } 237 n = 0; 238 p = wq->wq; 239 while (tail != head) { 240 struct rvt_rwqe *wqe; 241 int i; 242 243 wqe = rvt_get_rwqe_ptr(&srq->rq, tail); 244 p->wr_id = wqe->wr_id; 245 p->num_sge = wqe->num_sge; 246 for (i = 0; i < wqe->num_sge; i++) 247 p->sg_list[i] = wqe->sg_list[i]; 248 n++; 249 p = (struct rvt_rwqe *)((char *)p + sz); 250 if (++tail >= srq->rq.size) 251 tail = 0; 252 } 253 srq->rq.wq = wq; 254 srq->rq.size = size; 255 wq->head = n; 256 wq->tail = 0; 257 if (attr_mask & IB_SRQ_LIMIT) 258 srq->limit = attr->srq_limit; 259 spin_unlock_irq(&srq->rq.lock); 260 261 vfree(owq); 262 263 if (srq->ip) { 264 struct rvt_mmap_info *ip = srq->ip; 265 struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device); 266 u32 s = sizeof(struct rvt_rwq) + size * sz; 267 268 rvt_update_mmap_info(dev, ip, s, wq); 269 270 /* 271 * Return the offset to mmap. 272 * See rvt_mmap() for details. 273 */ 274 if (udata && udata->inlen >= sizeof(__u64)) { 275 ret = ib_copy_to_udata(udata, &ip->offset, 276 sizeof(ip->offset)); 277 if (ret) 278 return ret; 279 } 280 281 /* 282 * Put user mapping info onto the pending list 283 * unless it already is on the list. 284 */ 285 spin_lock_irq(&dev->pending_lock); 286 if (list_empty(&ip->pending_mmaps)) 287 list_add(&ip->pending_mmaps, 288 &dev->pending_mmaps); 289 spin_unlock_irq(&dev->pending_lock); 290 } 291 } else if (attr_mask & IB_SRQ_LIMIT) { 292 spin_lock_irq(&srq->rq.lock); 293 if (attr->srq_limit >= srq->rq.size) 294 ret = -EINVAL; 295 else 296 srq->limit = attr->srq_limit; 297 spin_unlock_irq(&srq->rq.lock); 298 } 299 return ret; 300 301 bail_unlock: 302 spin_unlock_irq(&srq->rq.lock); 303 bail_free: 304 vfree(wq); 305 return ret; 306 } 307 308 /** rvt_query_srq - query srq data 309 * @ibsrq: srq to query 310 * @attr: return info in attr 311 * 312 * Return: always 0 313 */ 314 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 315 { 316 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 317 318 attr->max_wr = srq->rq.size - 1; 319 attr->max_sge = srq->rq.max_sge; 320 attr->srq_limit = srq->limit; 321 return 0; 322 } 323 324 /** 325 * rvt_destroy_srq - destory an srq 326 * @ibsrq: srq object to destroy 327 * 328 */ 329 void rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 330 { 331 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq); 332 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device); 333 334 spin_lock(&dev->n_srqs_lock); 335 dev->n_srqs_allocated--; 336 spin_unlock(&dev->n_srqs_lock); 337 if (srq->ip) 338 kref_put(&srq->ip->ref, rvt_release_mmap_info); 339 else 340 vfree(srq->rq.wq); 341 } 342