xref: /openbmc/linux/drivers/infiniband/hw/mlx5/srq.c (revision 5d0e4d78)
1 /*
2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/module.h>
34 #include <linux/mlx5/qp.h>
35 #include <linux/mlx5/srq.h>
36 #include <linux/slab.h>
37 #include <rdma/ib_umem.h>
38 #include <rdma/ib_user_verbs.h>
39 
40 #include "mlx5_ib.h"
41 
42 /* not supported currently */
43 static int srq_signature;
44 
45 static void *get_wqe(struct mlx5_ib_srq *srq, int n)
46 {
47 	return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
48 }
49 
50 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type)
51 {
52 	struct ib_event event;
53 	struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq;
54 
55 	if (ibsrq->event_handler) {
56 		event.device      = ibsrq->device;
57 		event.element.srq = ibsrq;
58 		switch (type) {
59 		case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
60 			event.event = IB_EVENT_SRQ_LIMIT_REACHED;
61 			break;
62 		case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
63 			event.event = IB_EVENT_SRQ_ERR;
64 			break;
65 		default:
66 			pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n",
67 				type, srq->srqn);
68 			return;
69 		}
70 
71 		ibsrq->event_handler(&event, ibsrq->srq_context);
72 	}
73 }
74 
75 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
76 			   struct mlx5_srq_attr *in,
77 			   struct ib_udata *udata, int buf_size)
78 {
79 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
80 	struct mlx5_ib_create_srq ucmd = {};
81 	size_t ucmdlen;
82 	int err;
83 	int npages;
84 	int page_shift;
85 	int ncont;
86 	u32 offset;
87 	u32 uidx = MLX5_IB_DEFAULT_UIDX;
88 
89 	ucmdlen = min(udata->inlen, sizeof(ucmd));
90 
91 	if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) {
92 		mlx5_ib_dbg(dev, "failed copy udata\n");
93 		return -EFAULT;
94 	}
95 
96 	if (ucmd.reserved0 || ucmd.reserved1)
97 		return -EINVAL;
98 
99 	if (udata->inlen > sizeof(ucmd) &&
100 	    !ib_is_udata_cleared(udata, sizeof(ucmd),
101 				 udata->inlen - sizeof(ucmd)))
102 		return -EINVAL;
103 
104 	if (in->type == IB_SRQT_XRC) {
105 		err = get_srq_user_index(to_mucontext(pd->uobject->context),
106 					 &ucmd, udata->inlen, &uidx);
107 		if (err)
108 			return err;
109 	}
110 
111 	srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
112 
113 	srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
114 				0, 0);
115 	if (IS_ERR(srq->umem)) {
116 		mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);
117 		err = PTR_ERR(srq->umem);
118 		return err;
119 	}
120 
121 	mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages,
122 			   &page_shift, &ncont, NULL);
123 	err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift,
124 				     &offset);
125 	if (err) {
126 		mlx5_ib_warn(dev, "bad offset\n");
127 		goto err_umem;
128 	}
129 
130 	in->pas = kvzalloc(sizeof(*in->pas) * ncont, GFP_KERNEL);
131 	if (!in->pas) {
132 		err = -ENOMEM;
133 		goto err_umem;
134 	}
135 
136 	mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0);
137 
138 	err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context),
139 				  ucmd.db_addr, &srq->db);
140 	if (err) {
141 		mlx5_ib_dbg(dev, "map doorbell failed\n");
142 		goto err_in;
143 	}
144 
145 	in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
146 	in->page_offset = offset;
147 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
148 	    in->type == IB_SRQT_XRC)
149 		in->user_index = uidx;
150 
151 	return 0;
152 
153 err_in:
154 	kvfree(in->pas);
155 
156 err_umem:
157 	ib_umem_release(srq->umem);
158 
159 	return err;
160 }
161 
162 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
163 			     struct mlx5_srq_attr *in, int buf_size)
164 {
165 	int err;
166 	int i;
167 	struct mlx5_wqe_srq_next_seg *next;
168 
169 	err = mlx5_db_alloc(dev->mdev, &srq->db);
170 	if (err) {
171 		mlx5_ib_warn(dev, "alloc dbell rec failed\n");
172 		return err;
173 	}
174 
175 	if (mlx5_buf_alloc(dev->mdev, buf_size, &srq->buf)) {
176 		mlx5_ib_dbg(dev, "buf alloc failed\n");
177 		err = -ENOMEM;
178 		goto err_db;
179 	}
180 
181 	srq->head    = 0;
182 	srq->tail    = srq->msrq.max - 1;
183 	srq->wqe_ctr = 0;
184 
185 	for (i = 0; i < srq->msrq.max; i++) {
186 		next = get_wqe(srq, i);
187 		next->next_wqe_index =
188 			cpu_to_be16((i + 1) & (srq->msrq.max - 1));
189 	}
190 
191 	mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift);
192 	in->pas = kvzalloc(sizeof(*in->pas) * srq->buf.npages, GFP_KERNEL);
193 	if (!in->pas) {
194 		err = -ENOMEM;
195 		goto err_buf;
196 	}
197 	mlx5_fill_page_array(&srq->buf, in->pas);
198 
199 	srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);
200 	if (!srq->wrid) {
201 		err = -ENOMEM;
202 		goto err_in;
203 	}
204 	srq->wq_sig = !!srq_signature;
205 
206 	in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
207 	if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
208 	    in->type == IB_SRQT_XRC)
209 		in->user_index = MLX5_IB_DEFAULT_UIDX;
210 
211 	return 0;
212 
213 err_in:
214 	kvfree(in->pas);
215 
216 err_buf:
217 	mlx5_buf_free(dev->mdev, &srq->buf);
218 
219 err_db:
220 	mlx5_db_free(dev->mdev, &srq->db);
221 	return err;
222 }
223 
224 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
225 {
226 	mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
227 	ib_umem_release(srq->umem);
228 }
229 
230 
231 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
232 {
233 	kfree(srq->wrid);
234 	mlx5_buf_free(dev->mdev, &srq->buf);
235 	mlx5_db_free(dev->mdev, &srq->db);
236 }
237 
238 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
239 				  struct ib_srq_init_attr *init_attr,
240 				  struct ib_udata *udata)
241 {
242 	struct mlx5_ib_dev *dev = to_mdev(pd->device);
243 	struct mlx5_ib_srq *srq;
244 	int desc_size;
245 	int buf_size;
246 	int err;
247 	struct mlx5_srq_attr in = {0};
248 	__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
249 
250 	/* Sanity check SRQ size before proceeding */
251 	if (init_attr->attr.max_wr >= max_srq_wqes) {
252 		mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
253 			    init_attr->attr.max_wr,
254 			    max_srq_wqes);
255 		return ERR_PTR(-EINVAL);
256 	}
257 
258 	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
259 	if (!srq)
260 		return ERR_PTR(-ENOMEM);
261 
262 	mutex_init(&srq->mutex);
263 	spin_lock_init(&srq->lock);
264 	srq->msrq.max    = roundup_pow_of_two(init_attr->attr.max_wr + 1);
265 	srq->msrq.max_gs = init_attr->attr.max_sge;
266 
267 	desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
268 		    srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
269 	desc_size = roundup_pow_of_two(desc_size);
270 	desc_size = max_t(int, 32, desc_size);
271 	srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
272 		sizeof(struct mlx5_wqe_data_seg);
273 	srq->msrq.wqe_shift = ilog2(desc_size);
274 	buf_size = srq->msrq.max * desc_size;
275 	mlx5_ib_dbg(dev, "desc_size 0x%x, req wr 0x%x, srq size 0x%x, max_gs 0x%x, max_avail_gather 0x%x\n",
276 		    desc_size, init_attr->attr.max_wr, srq->msrq.max, srq->msrq.max_gs,
277 		    srq->msrq.max_avail_gather);
278 	in.type = init_attr->srq_type;
279 
280 	if (pd->uobject)
281 		err = create_srq_user(pd, srq, &in, udata, buf_size);
282 	else
283 		err = create_srq_kernel(dev, srq, &in, buf_size);
284 
285 	if (err) {
286 		mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
287 			     pd->uobject ? "user" : "kernel", err);
288 		goto err_srq;
289 	}
290 
291 	in.log_size = ilog2(srq->msrq.max);
292 	in.wqe_shift = srq->msrq.wqe_shift - 4;
293 	if (srq->wq_sig)
294 		in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
295 	if (init_attr->srq_type == IB_SRQT_XRC) {
296 		in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
297 		in.cqn = to_mcq(init_attr->ext.xrc.cq)->mcq.cqn;
298 	} else if (init_attr->srq_type == IB_SRQT_BASIC) {
299 		in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
300 		in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
301 	}
302 
303 	in.pd = to_mpd(pd)->pdn;
304 	in.db_record = srq->db.dma;
305 	err = mlx5_core_create_srq(dev->mdev, &srq->msrq, &in);
306 	kvfree(in.pas);
307 	if (err) {
308 		mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err);
309 		goto err_usr_kern_srq;
310 	}
311 
312 	mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn);
313 
314 	srq->msrq.event = mlx5_ib_srq_event;
315 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
316 
317 	if (pd->uobject)
318 		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
319 			mlx5_ib_dbg(dev, "copy to user failed\n");
320 			err = -EFAULT;
321 			goto err_core;
322 		}
323 
324 	init_attr->attr.max_wr = srq->msrq.max - 1;
325 
326 	return &srq->ibsrq;
327 
328 err_core:
329 	mlx5_core_destroy_srq(dev->mdev, &srq->msrq);
330 
331 err_usr_kern_srq:
332 	if (pd->uobject)
333 		destroy_srq_user(pd, srq);
334 	else
335 		destroy_srq_kernel(dev, srq);
336 
337 err_srq:
338 	kfree(srq);
339 
340 	return ERR_PTR(err);
341 }
342 
343 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
344 		       enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
345 {
346 	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
347 	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
348 	int ret;
349 
350 	/* We don't support resizing SRQs yet */
351 	if (attr_mask & IB_SRQ_MAX_WR)
352 		return -EINVAL;
353 
354 	if (attr_mask & IB_SRQ_LIMIT) {
355 		if (attr->srq_limit >= srq->msrq.max)
356 			return -EINVAL;
357 
358 		mutex_lock(&srq->mutex);
359 		ret = mlx5_core_arm_srq(dev->mdev, &srq->msrq, attr->srq_limit, 1);
360 		mutex_unlock(&srq->mutex);
361 
362 		if (ret)
363 			return ret;
364 	}
365 
366 	return 0;
367 }
368 
369 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
370 {
371 	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
372 	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
373 	int ret;
374 	struct mlx5_srq_attr *out;
375 
376 	out = kzalloc(sizeof(*out), GFP_KERNEL);
377 	if (!out)
378 		return -ENOMEM;
379 
380 	ret = mlx5_core_query_srq(dev->mdev, &srq->msrq, out);
381 	if (ret)
382 		goto out_box;
383 
384 	srq_attr->srq_limit = out->lwm;
385 	srq_attr->max_wr    = srq->msrq.max - 1;
386 	srq_attr->max_sge   = srq->msrq.max_gs;
387 
388 out_box:
389 	kfree(out);
390 	return ret;
391 }
392 
393 int mlx5_ib_destroy_srq(struct ib_srq *srq)
394 {
395 	struct mlx5_ib_dev *dev = to_mdev(srq->device);
396 	struct mlx5_ib_srq *msrq = to_msrq(srq);
397 
398 	mlx5_core_destroy_srq(dev->mdev, &msrq->msrq);
399 
400 	if (srq->uobject) {
401 		mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
402 		ib_umem_release(msrq->umem);
403 	} else {
404 		destroy_srq_kernel(dev, msrq);
405 	}
406 
407 	kfree(srq);
408 	return 0;
409 }
410 
411 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)
412 {
413 	struct mlx5_wqe_srq_next_seg *next;
414 
415 	/* always called with interrupts disabled. */
416 	spin_lock(&srq->lock);
417 
418 	next = get_wqe(srq, srq->tail);
419 	next->next_wqe_index = cpu_to_be16(wqe_index);
420 	srq->tail = wqe_index;
421 
422 	spin_unlock(&srq->lock);
423 }
424 
425 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
426 			  struct ib_recv_wr **bad_wr)
427 {
428 	struct mlx5_ib_srq *srq = to_msrq(ibsrq);
429 	struct mlx5_wqe_srq_next_seg *next;
430 	struct mlx5_wqe_data_seg *scat;
431 	struct mlx5_ib_dev *dev = to_mdev(ibsrq->device);
432 	struct mlx5_core_dev *mdev = dev->mdev;
433 	unsigned long flags;
434 	int err = 0;
435 	int nreq;
436 	int i;
437 
438 	spin_lock_irqsave(&srq->lock, flags);
439 
440 	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
441 		err = -EIO;
442 		*bad_wr = wr;
443 		goto out;
444 	}
445 
446 	for (nreq = 0; wr; nreq++, wr = wr->next) {
447 		if (unlikely(wr->num_sge > srq->msrq.max_gs)) {
448 			err = -EINVAL;
449 			*bad_wr = wr;
450 			break;
451 		}
452 
453 		if (unlikely(srq->head == srq->tail)) {
454 			err = -ENOMEM;
455 			*bad_wr = wr;
456 			break;
457 		}
458 
459 		srq->wrid[srq->head] = wr->wr_id;
460 
461 		next      = get_wqe(srq, srq->head);
462 		srq->head = be16_to_cpu(next->next_wqe_index);
463 		scat      = (struct mlx5_wqe_data_seg *)(next + 1);
464 
465 		for (i = 0; i < wr->num_sge; i++) {
466 			scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length);
467 			scat[i].lkey       = cpu_to_be32(wr->sg_list[i].lkey);
468 			scat[i].addr       = cpu_to_be64(wr->sg_list[i].addr);
469 		}
470 
471 		if (i < srq->msrq.max_avail_gather) {
472 			scat[i].byte_count = 0;
473 			scat[i].lkey       = cpu_to_be32(MLX5_INVALID_LKEY);
474 			scat[i].addr       = 0;
475 		}
476 	}
477 
478 	if (likely(nreq)) {
479 		srq->wqe_ctr += nreq;
480 
481 		/* Make sure that descriptors are written before
482 		 * doorbell record.
483 		 */
484 		wmb();
485 
486 		*srq->db.db = cpu_to_be32(srq->wqe_ctr);
487 	}
488 out:
489 	spin_unlock_irqrestore(&srq->lock, flags);
490 
491 	return err;
492 }
493