1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 /*
3 * Copyright(c) 2016 Intel Corporation.
4 */
5
6 #include <linux/err.h>
7 #include <linux/slab.h>
8 #include <linux/vmalloc.h>
9 #include <rdma/uverbs_ioctl.h>
10
11 #include "srq.h"
12 #include "vt.h"
13 #include "qp.h"
14 /**
15 * rvt_driver_srq_init - init srq resources on a per driver basis
16 * @rdi: rvt dev structure
17 *
18 * Do any initialization needed when a driver registers with rdmavt.
19 */
rvt_driver_srq_init(struct rvt_dev_info * rdi)20 void rvt_driver_srq_init(struct rvt_dev_info *rdi)
21 {
22 spin_lock_init(&rdi->n_srqs_lock);
23 rdi->n_srqs_allocated = 0;
24 }
25
26 /**
27 * rvt_create_srq - create a shared receive queue
28 * @ibsrq: the protection domain of the SRQ to create
29 * @srq_init_attr: the attributes of the SRQ
30 * @udata: data from libibverbs when creating a user SRQ
31 *
32 * Return: 0 on success
33 */
rvt_create_srq(struct ib_srq * ibsrq,struct ib_srq_init_attr * srq_init_attr,struct ib_udata * udata)34 int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
35 struct ib_udata *udata)
36 {
37 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
38 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
39 u32 sz;
40 int ret;
41
42 if (srq_init_attr->srq_type != IB_SRQT_BASIC)
43 return -EOPNOTSUPP;
44
45 if (srq_init_attr->attr.max_sge == 0 ||
46 srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge ||
47 srq_init_attr->attr.max_wr == 0 ||
48 srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
49 return -EINVAL;
50
51 /*
52 * Need to use vmalloc() if we want to support large #s of entries.
53 */
54 srq->rq.size = srq_init_attr->attr.max_wr + 1;
55 srq->rq.max_sge = srq_init_attr->attr.max_sge;
56 sz = sizeof(struct ib_sge) * srq->rq.max_sge +
57 sizeof(struct rvt_rwqe);
58 if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz,
59 dev->dparms.node, udata)) {
60 ret = -ENOMEM;
61 goto bail_srq;
62 }
63
64 /*
65 * Return the address of the RWQ as the offset to mmap.
66 * See rvt_mmap() for details.
67 */
68 if (udata && udata->outlen >= sizeof(__u64)) {
69 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz;
70
71 srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq);
72 if (IS_ERR(srq->ip)) {
73 ret = PTR_ERR(srq->ip);
74 goto bail_wq;
75 }
76
77 ret = ib_copy_to_udata(udata, &srq->ip->offset,
78 sizeof(srq->ip->offset));
79 if (ret)
80 goto bail_ip;
81 }
82
83 /*
84 * ib_create_srq() will initialize srq->ibsrq.
85 */
86 spin_lock_init(&srq->rq.lock);
87 srq->limit = srq_init_attr->attr.srq_limit;
88
89 spin_lock(&dev->n_srqs_lock);
90 if (dev->n_srqs_allocated == dev->dparms.props.max_srq) {
91 spin_unlock(&dev->n_srqs_lock);
92 ret = -ENOMEM;
93 goto bail_ip;
94 }
95
96 dev->n_srqs_allocated++;
97 spin_unlock(&dev->n_srqs_lock);
98
99 if (srq->ip) {
100 spin_lock_irq(&dev->pending_lock);
101 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps);
102 spin_unlock_irq(&dev->pending_lock);
103 }
104
105 return 0;
106
107 bail_ip:
108 kfree(srq->ip);
109 bail_wq:
110 rvt_free_rq(&srq->rq);
111 bail_srq:
112 return ret;
113 }
114
115 /**
116 * rvt_modify_srq - modify a shared receive queue
117 * @ibsrq: the SRQ to modify
118 * @attr: the new attributes of the SRQ
119 * @attr_mask: indicates which attributes to modify
120 * @udata: user data for libibverbs.so
121 *
122 * Return: 0 on success
123 */
rvt_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)124 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
125 enum ib_srq_attr_mask attr_mask,
126 struct ib_udata *udata)
127 {
128 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
129 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
130 struct rvt_rq tmp_rq = {};
131 int ret = 0;
132
133 if (attr_mask & IB_SRQ_MAX_WR) {
134 struct rvt_krwq *okwq = NULL;
135 struct rvt_rwq *owq = NULL;
136 struct rvt_rwqe *p;
137 u32 sz, size, n, head, tail;
138
139 /* Check that the requested sizes are below the limits. */
140 if ((attr->max_wr > dev->dparms.props.max_srq_wr) ||
141 ((attr_mask & IB_SRQ_LIMIT) ?
142 attr->srq_limit : srq->limit) > attr->max_wr)
143 return -EINVAL;
144 sz = sizeof(struct rvt_rwqe) +
145 srq->rq.max_sge * sizeof(struct ib_sge);
146 size = attr->max_wr + 1;
147 if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node,
148 udata))
149 return -ENOMEM;
150 /* Check that we can write the offset to mmap. */
151 if (udata && udata->inlen >= sizeof(__u64)) {
152 __u64 offset_addr;
153 __u64 offset = 0;
154
155 ret = ib_copy_from_udata(&offset_addr, udata,
156 sizeof(offset_addr));
157 if (ret)
158 goto bail_free;
159 udata->outbuf = (void __user *)
160 (unsigned long)offset_addr;
161 ret = ib_copy_to_udata(udata, &offset,
162 sizeof(offset));
163 if (ret)
164 goto bail_free;
165 }
166
167 spin_lock_irq(&srq->rq.kwq->c_lock);
168 /*
169 * validate head and tail pointer values and compute
170 * the number of remaining WQEs.
171 */
172 if (udata) {
173 owq = srq->rq.wq;
174 head = RDMA_READ_UAPI_ATOMIC(owq->head);
175 tail = RDMA_READ_UAPI_ATOMIC(owq->tail);
176 } else {
177 okwq = srq->rq.kwq;
178 head = okwq->head;
179 tail = okwq->tail;
180 }
181 if (head >= srq->rq.size || tail >= srq->rq.size) {
182 ret = -EINVAL;
183 goto bail_unlock;
184 }
185 n = head;
186 if (n < tail)
187 n += srq->rq.size - tail;
188 else
189 n -= tail;
190 if (size <= n) {
191 ret = -EINVAL;
192 goto bail_unlock;
193 }
194 n = 0;
195 p = tmp_rq.kwq->curr_wq;
196 while (tail != head) {
197 struct rvt_rwqe *wqe;
198 int i;
199
200 wqe = rvt_get_rwqe_ptr(&srq->rq, tail);
201 p->wr_id = wqe->wr_id;
202 p->num_sge = wqe->num_sge;
203 for (i = 0; i < wqe->num_sge; i++)
204 p->sg_list[i] = wqe->sg_list[i];
205 n++;
206 p = (struct rvt_rwqe *)((char *)p + sz);
207 if (++tail >= srq->rq.size)
208 tail = 0;
209 }
210 srq->rq.kwq = tmp_rq.kwq;
211 if (udata) {
212 srq->rq.wq = tmp_rq.wq;
213 RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n);
214 RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0);
215 } else {
216 tmp_rq.kwq->head = n;
217 tmp_rq.kwq->tail = 0;
218 }
219 srq->rq.size = size;
220 if (attr_mask & IB_SRQ_LIMIT)
221 srq->limit = attr->srq_limit;
222 spin_unlock_irq(&srq->rq.kwq->c_lock);
223
224 vfree(owq);
225 kvfree(okwq);
226
227 if (srq->ip) {
228 struct rvt_mmap_info *ip = srq->ip;
229 struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device);
230 u32 s = sizeof(struct rvt_rwq) + size * sz;
231
232 rvt_update_mmap_info(dev, ip, s, tmp_rq.wq);
233
234 /*
235 * Return the offset to mmap.
236 * See rvt_mmap() for details.
237 */
238 if (udata && udata->inlen >= sizeof(__u64)) {
239 ret = ib_copy_to_udata(udata, &ip->offset,
240 sizeof(ip->offset));
241 if (ret)
242 return ret;
243 }
244
245 /*
246 * Put user mapping info onto the pending list
247 * unless it already is on the list.
248 */
249 spin_lock_irq(&dev->pending_lock);
250 if (list_empty(&ip->pending_mmaps))
251 list_add(&ip->pending_mmaps,
252 &dev->pending_mmaps);
253 spin_unlock_irq(&dev->pending_lock);
254 }
255 } else if (attr_mask & IB_SRQ_LIMIT) {
256 spin_lock_irq(&srq->rq.kwq->c_lock);
257 if (attr->srq_limit >= srq->rq.size)
258 ret = -EINVAL;
259 else
260 srq->limit = attr->srq_limit;
261 spin_unlock_irq(&srq->rq.kwq->c_lock);
262 }
263 return ret;
264
265 bail_unlock:
266 spin_unlock_irq(&srq->rq.kwq->c_lock);
267 bail_free:
268 rvt_free_rq(&tmp_rq);
269 return ret;
270 }
271
272 /**
273 * rvt_query_srq - query srq data
274 * @ibsrq: srq to query
275 * @attr: return info in attr
276 *
277 * Return: always 0
278 */
rvt_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr)279 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
280 {
281 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
282
283 attr->max_wr = srq->rq.size - 1;
284 attr->max_sge = srq->rq.max_sge;
285 attr->srq_limit = srq->limit;
286 return 0;
287 }
288
289 /**
290 * rvt_destroy_srq - destory an srq
291 * @ibsrq: srq object to destroy
292 * @udata: user data for libibverbs.so
293 */
rvt_destroy_srq(struct ib_srq * ibsrq,struct ib_udata * udata)294 int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
295 {
296 struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
297 struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
298
299 spin_lock(&dev->n_srqs_lock);
300 dev->n_srqs_allocated--;
301 spin_unlock(&dev->n_srqs_lock);
302 if (srq->ip)
303 kref_put(&srq->ip->ref, rvt_release_mmap_info);
304 kvfree(srq->rq.kwq);
305 return 0;
306 }
307