1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.
4  */
5 
6 #include <linux/kernel.h>
7 #include <linux/mlx5/driver.h>
8 #include "mlx5_ib.h"
9 #include "srq.h"
10 #include "qp.h"
11 
12 static int get_pas_size(struct mlx5_srq_attr *in)
13 {
14 	u32 log_page_size = in->log_page_size + 12;
15 	u32 log_srq_size  = in->log_size;
16 	u32 log_rq_stride = in->wqe_shift;
17 	u32 page_offset   = in->page_offset;
18 	u32 po_quanta	  = 1 << (log_page_size - 6);
19 	u32 rq_sz	  = 1 << (log_srq_size + 4 + log_rq_stride);
20 	u32 page_size	  = 1 << log_page_size;
21 	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
22 	u32 rq_num_pas    = DIV_ROUND_UP(rq_sz_po, page_size);
23 
24 	return rq_num_pas * sizeof(u64);
25 }
26 
27 static void set_wq(void *wq, struct mlx5_srq_attr *in)
28 {
29 	MLX5_SET(wq,   wq, wq_signature,  !!(in->flags
30 		 & MLX5_SRQ_FLAG_WQ_SIG));
31 	MLX5_SET(wq,   wq, log_wq_pg_sz,  in->log_page_size);
32 	MLX5_SET(wq,   wq, log_wq_stride, in->wqe_shift + 4);
33 	MLX5_SET(wq,   wq, log_wq_sz,     in->log_size);
34 	MLX5_SET(wq,   wq, page_offset,   in->page_offset);
35 	MLX5_SET(wq,   wq, lwm,		  in->lwm);
36 	MLX5_SET(wq,   wq, pd,		  in->pd);
37 	MLX5_SET64(wq, wq, dbr_addr,	  in->db_record);
38 }
39 
40 static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
41 {
42 	MLX5_SET(srqc,   srqc, wq_signature,  !!(in->flags
43 		 & MLX5_SRQ_FLAG_WQ_SIG));
44 	MLX5_SET(srqc,   srqc, log_page_size, in->log_page_size);
45 	MLX5_SET(srqc,   srqc, log_rq_stride, in->wqe_shift);
46 	MLX5_SET(srqc,   srqc, log_srq_size,  in->log_size);
47 	MLX5_SET(srqc,   srqc, page_offset,   in->page_offset);
48 	MLX5_SET(srqc,	 srqc, lwm,	      in->lwm);
49 	MLX5_SET(srqc,	 srqc, pd,	      in->pd);
50 	MLX5_SET64(srqc, srqc, dbr_addr,      in->db_record);
51 	MLX5_SET(srqc,	 srqc, xrcd,	      in->xrcd);
52 	MLX5_SET(srqc,	 srqc, cqn,	      in->cqn);
53 }
54 
55 static void get_wq(void *wq, struct mlx5_srq_attr *in)
56 {
57 	if (MLX5_GET(wq, wq, wq_signature))
58 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
59 	in->log_page_size = MLX5_GET(wq,   wq, log_wq_pg_sz);
60 	in->wqe_shift	  = MLX5_GET(wq,   wq, log_wq_stride) - 4;
61 	in->log_size	  = MLX5_GET(wq,   wq, log_wq_sz);
62 	in->page_offset   = MLX5_GET(wq,   wq, page_offset);
63 	in->lwm		  = MLX5_GET(wq,   wq, lwm);
64 	in->pd		  = MLX5_GET(wq,   wq, pd);
65 	in->db_record	  = MLX5_GET64(wq, wq, dbr_addr);
66 }
67 
68 static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
69 {
70 	if (MLX5_GET(srqc, srqc, wq_signature))
71 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
72 	in->log_page_size = MLX5_GET(srqc,   srqc, log_page_size);
73 	in->wqe_shift	  = MLX5_GET(srqc,   srqc, log_rq_stride);
74 	in->log_size	  = MLX5_GET(srqc,   srqc, log_srq_size);
75 	in->page_offset   = MLX5_GET(srqc,   srqc, page_offset);
76 	in->lwm		  = MLX5_GET(srqc,   srqc, lwm);
77 	in->pd		  = MLX5_GET(srqc,   srqc, pd);
78 	in->db_record	  = MLX5_GET64(srqc, srqc, dbr_addr);
79 }
80 
81 struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn)
82 {
83 	struct mlx5_srq_table *table = &dev->srq_table;
84 	struct mlx5_core_srq *srq;
85 
86 	xa_lock_irq(&table->array);
87 	srq = xa_load(&table->array, srqn);
88 	if (srq)
89 		refcount_inc(&srq->common.refcount);
90 	xa_unlock_irq(&table->array);
91 
92 	return srq;
93 }
94 
95 static int __set_srq_page_size(struct mlx5_srq_attr *in,
96 			       unsigned long page_size)
97 {
98 	if (!page_size)
99 		return -EINVAL;
100 	in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT;
101 
102 	if (WARN_ON(get_pas_size(in) !=
103 		    ib_umem_num_dma_blocks(in->umem, page_size) * sizeof(u64)))
104 		return -EINVAL;
105 	return 0;
106 }
107 
108 #define set_srq_page_size(in, typ, log_pgsz_fld)                               \
109 	__set_srq_page_size(in, mlx5_umem_find_best_quantized_pgoff(           \
110 					(in)->umem, typ, log_pgsz_fld,         \
111 					MLX5_ADAPTER_PAGE_SHIFT, page_offset,  \
112 					64, &(in)->page_offset))
113 
114 static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
115 			  struct mlx5_srq_attr *in)
116 {
117 	u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
118 	void *create_in;
119 	void *srqc;
120 	void *pas;
121 	int pas_size;
122 	int inlen;
123 	int err;
124 
125 	if (in->umem) {
126 		err = set_srq_page_size(in, srqc, log_page_size);
127 		if (err)
128 			return err;
129 	}
130 
131 	pas_size  = get_pas_size(in);
132 	inlen	  = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
133 	create_in = kvzalloc(inlen, GFP_KERNEL);
134 	if (!create_in)
135 		return -ENOMEM;
136 
137 	MLX5_SET(create_srq_in, create_in, uid, in->uid);
138 	srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
139 	pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
140 
141 	set_srqc(srqc, in);
142 	if (in->umem)
143 		mlx5_ib_populate_pas(
144 			in->umem,
145 			1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
146 			pas, 0);
147 	else
148 		memcpy(pas, in->pas, pas_size);
149 
150 	MLX5_SET(create_srq_in, create_in, opcode,
151 		 MLX5_CMD_OP_CREATE_SRQ);
152 
153 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
154 			    sizeof(create_out));
155 	kvfree(create_in);
156 	if (!err) {
157 		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
158 		srq->uid = in->uid;
159 	}
160 
161 	return err;
162 }
163 
164 static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
165 {
166 	u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {};
167 
168 	MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ);
169 	MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);
170 	MLX5_SET(destroy_srq_in, in, uid, srq->uid);
171 
172 	return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in);
173 }
174 
175 static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
176 		       u16 lwm, int is_srq)
177 {
178 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
179 
180 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
181 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
182 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
183 	MLX5_SET(arm_rq_in, in, lwm, lwm);
184 	MLX5_SET(arm_rq_in, in, uid, srq->uid);
185 
186 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
187 }
188 
189 static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
190 			 struct mlx5_srq_attr *out)
191 {
192 	u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {};
193 	u32 *srq_out;
194 	void *srqc;
195 	int err;
196 
197 	srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);
198 	if (!srq_out)
199 		return -ENOMEM;
200 
201 	MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ);
202 	MLX5_SET(query_srq_in, in, srqn, srq->srqn);
203 	err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out);
204 	if (err)
205 		goto out;
206 
207 	srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
208 	get_srqc(srqc, out);
209 	if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
210 		out->flags |= MLX5_SRQ_FLAG_ERR;
211 out:
212 	kvfree(srq_out);
213 	return err;
214 }
215 
216 static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev,
217 			      struct mlx5_core_srq *srq,
218 			      struct mlx5_srq_attr *in)
219 {
220 	u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
221 	void *create_in;
222 	void *xrc_srqc;
223 	void *pas;
224 	int pas_size;
225 	int inlen;
226 	int err;
227 
228 	if (in->umem) {
229 		err = set_srq_page_size(in, xrc_srqc, log_page_size);
230 		if (err)
231 			return err;
232 	}
233 
234 	pas_size  = get_pas_size(in);
235 	inlen	  = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
236 	create_in = kvzalloc(inlen, GFP_KERNEL);
237 	if (!create_in)
238 		return -ENOMEM;
239 
240 	MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);
241 	xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,
242 				xrc_srq_context_entry);
243 	pas	 = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
244 
245 	set_srqc(xrc_srqc, in);
246 	MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
247 	if (in->umem)
248 		mlx5_ib_populate_pas(
249 			in->umem,
250 			1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
251 			pas, 0);
252 	else
253 		memcpy(pas, in->pas, pas_size);
254 	MLX5_SET(create_xrc_srq_in, create_in, opcode,
255 		 MLX5_CMD_OP_CREATE_XRC_SRQ);
256 
257 	memset(create_out, 0, sizeof(create_out));
258 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
259 			    sizeof(create_out));
260 	if (err)
261 		goto out;
262 
263 	srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);
264 	srq->uid = in->uid;
265 out:
266 	kvfree(create_in);
267 	return err;
268 }
269 
270 static int destroy_xrc_srq_cmd(struct mlx5_ib_dev *dev,
271 			       struct mlx5_core_srq *srq)
272 {
273 	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {};
274 
275 	MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ);
276 	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);
277 	MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);
278 
279 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in);
280 }
281 
282 static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
283 			   u16 lwm)
284 {
285 	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {};
286 
287 	MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
288 	MLX5_SET(arm_xrc_srq_in, in, op_mod,
289 		 MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
290 	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);
291 	MLX5_SET(arm_xrc_srq_in, in, lwm, lwm);
292 	MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);
293 
294 	return  mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in);
295 }
296 
297 static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev,
298 			     struct mlx5_core_srq *srq,
299 			     struct mlx5_srq_attr *out)
300 {
301 	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {};
302 	u32 *xrcsrq_out;
303 	void *xrc_srqc;
304 	int err;
305 
306 	xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
307 	if (!xrcsrq_out)
308 		return -ENOMEM;
309 
310 	MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);
311 	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);
312 
313 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out);
314 	if (err)
315 		goto out;
316 
317 	xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
318 				xrc_srq_context_entry);
319 	get_srqc(xrc_srqc, out);
320 	if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
321 		out->flags |= MLX5_SRQ_FLAG_ERR;
322 
323 out:
324 	kvfree(xrcsrq_out);
325 	return err;
326 }
327 
328 static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
329 			  struct mlx5_srq_attr *in)
330 {
331 	void *create_out = NULL;
332 	void *create_in = NULL;
333 	void *rmpc;
334 	void *wq;
335 	void *pas;
336 	int pas_size;
337 	int outlen;
338 	int inlen;
339 	int err;
340 
341 	if (in->umem) {
342 		err = set_srq_page_size(in, wq, log_wq_pg_sz);
343 		if (err)
344 			return err;
345 	}
346 
347 	pas_size = get_pas_size(in);
348 	inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
349 	outlen = MLX5_ST_SZ_BYTES(create_rmp_out);
350 	create_in = kvzalloc(inlen, GFP_KERNEL);
351 	create_out = kvzalloc(outlen, GFP_KERNEL);
352 	if (!create_in || !create_out) {
353 		err = -ENOMEM;
354 		goto out;
355 	}
356 
357 	rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
358 	wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
359 
360 	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
361 	MLX5_SET(create_rmp_in, create_in, uid, in->uid);
362 	pas = MLX5_ADDR_OF(rmpc, rmpc, wq.pas);
363 
364 	set_wq(wq, in);
365 	if (in->umem)
366 		mlx5_ib_populate_pas(
367 			in->umem,
368 			1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
369 			pas, 0);
370 	else
371 		memcpy(pas, in->pas, pas_size);
372 
373 	MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);
374 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);
375 	if (!err) {
376 		srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn);
377 		srq->uid = in->uid;
378 	}
379 
380 out:
381 	kvfree(create_in);
382 	kvfree(create_out);
383 	return err;
384 }
385 
386 static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
387 {
388 	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
389 
390 	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
391 	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
392 	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
393 	return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in);
394 }
395 
396 static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
397 		       u16 lwm)
398 {
399 	void *out = NULL;
400 	void *in = NULL;
401 	void *rmpc;
402 	void *wq;
403 	void *bitmask;
404 	int outlen;
405 	int inlen;
406 	int err;
407 
408 	inlen = MLX5_ST_SZ_BYTES(modify_rmp_in);
409 	outlen = MLX5_ST_SZ_BYTES(modify_rmp_out);
410 
411 	in = kvzalloc(inlen, GFP_KERNEL);
412 	out = kvzalloc(outlen, GFP_KERNEL);
413 	if (!in || !out) {
414 		err = -ENOMEM;
415 		goto out;
416 	}
417 
418 	rmpc =	  MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);
419 	bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);
420 	wq   =	  MLX5_ADDR_OF(rmpc,	        rmpc, wq);
421 
422 	MLX5_SET(modify_rmp_in, in,	 rmp_state, MLX5_RMPC_STATE_RDY);
423 	MLX5_SET(modify_rmp_in, in,	 rmpn,      srq->srqn);
424 	MLX5_SET(modify_rmp_in, in, uid, srq->uid);
425 	MLX5_SET(wq,		wq,	 lwm,	    lwm);
426 	MLX5_SET(rmp_bitmask,	bitmask, lwm,	    1);
427 	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
428 	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
429 
430 	err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out);
431 
432 out:
433 	kvfree(in);
434 	kvfree(out);
435 	return err;
436 }
437 
438 static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
439 			 struct mlx5_srq_attr *out)
440 {
441 	u32 *rmp_out = NULL;
442 	u32 *rmp_in = NULL;
443 	void *rmpc;
444 	int outlen;
445 	int inlen;
446 	int err;
447 
448 	outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
449 	inlen = MLX5_ST_SZ_BYTES(query_rmp_in);
450 
451 	rmp_out = kvzalloc(outlen, GFP_KERNEL);
452 	rmp_in = kvzalloc(inlen, GFP_KERNEL);
453 	if (!rmp_out || !rmp_in) {
454 		err = -ENOMEM;
455 		goto out;
456 	}
457 
458 	MLX5_SET(query_rmp_in, rmp_in, opcode, MLX5_CMD_OP_QUERY_RMP);
459 	MLX5_SET(query_rmp_in, rmp_in, rmpn,   srq->srqn);
460 	err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out);
461 	if (err)
462 		goto out;
463 
464 	rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
465 	get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
466 	if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
467 		out->flags |= MLX5_SRQ_FLAG_ERR;
468 
469 out:
470 	kvfree(rmp_out);
471 	kvfree(rmp_in);
472 	return err;
473 }
474 
475 static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
476 			  struct mlx5_srq_attr *in)
477 {
478 	u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
479 	void *create_in;
480 	void *xrqc;
481 	void *wq;
482 	void *pas;
483 	int pas_size;
484 	int inlen;
485 	int err;
486 
487 	if (in->umem) {
488 		err = set_srq_page_size(in, wq, log_wq_pg_sz);
489 		if (err)
490 			return err;
491 	}
492 
493 	pas_size = get_pas_size(in);
494 	inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
495 	create_in = kvzalloc(inlen, GFP_KERNEL);
496 	if (!create_in)
497 		return -ENOMEM;
498 
499 	xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
500 	wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
501 	pas = MLX5_ADDR_OF(xrqc, xrqc, wq.pas);
502 
503 	set_wq(wq, in);
504 	if (in->umem)
505 		mlx5_ib_populate_pas(
506 			in->umem,
507 			1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
508 			pas, 0);
509 	else
510 		memcpy(pas, in->pas, pas_size);
511 
512 	if (in->type == IB_SRQT_TM) {
513 		MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
514 		if (in->flags & MLX5_SRQ_FLAG_RNDV)
515 			MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
516 		MLX5_SET(xrqc, xrqc,
517 			 tag_matching_topology_context.log_matching_list_sz,
518 			 in->tm_log_list_size);
519 	}
520 	MLX5_SET(xrqc, xrqc, user_index, in->user_index);
521 	MLX5_SET(xrqc, xrqc, cqn, in->cqn);
522 	MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
523 	MLX5_SET(create_xrq_in, create_in, uid, in->uid);
524 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
525 			    sizeof(create_out));
526 	kvfree(create_in);
527 	if (!err) {
528 		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
529 		srq->uid = in->uid;
530 	}
531 
532 	return err;
533 }
534 
535 static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
536 {
537 	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {};
538 
539 	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
540 	MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
541 	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
542 
543 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in);
544 }
545 
546 static int arm_xrq_cmd(struct mlx5_ib_dev *dev,
547 		       struct mlx5_core_srq *srq,
548 		       u16 lwm)
549 {
550 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
551 
552 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
553 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
554 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
555 	MLX5_SET(arm_rq_in, in, lwm, lwm);
556 	MLX5_SET(arm_rq_in, in, uid, srq->uid);
557 
558 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
559 }
560 
561 static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
562 			 struct mlx5_srq_attr *out)
563 {
564 	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {};
565 	u32 *xrq_out;
566 	int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
567 	void *xrqc;
568 	int err;
569 
570 	xrq_out = kvzalloc(outlen, GFP_KERNEL);
571 	if (!xrq_out)
572 		return -ENOMEM;
573 
574 	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
575 	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
576 
577 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out);
578 	if (err)
579 		goto out;
580 
581 	xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
582 	get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
583 	if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
584 		out->flags |= MLX5_SRQ_FLAG_ERR;
585 	out->tm_next_tag =
586 		MLX5_GET(xrqc, xrqc,
587 			 tag_matching_topology_context.append_next_index);
588 	out->tm_hw_phase_cnt =
589 		MLX5_GET(xrqc, xrqc,
590 			 tag_matching_topology_context.hw_phase_cnt);
591 	out->tm_sw_phase_cnt =
592 		MLX5_GET(xrqc, xrqc,
593 			 tag_matching_topology_context.sw_phase_cnt);
594 
595 out:
596 	kvfree(xrq_out);
597 	return err;
598 }
599 
600 static int create_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
601 			    struct mlx5_srq_attr *in)
602 {
603 	if (!dev->mdev->issi)
604 		return create_srq_cmd(dev, srq, in);
605 	switch (srq->common.res) {
606 	case MLX5_RES_XSRQ:
607 		return create_xrc_srq_cmd(dev, srq, in);
608 	case MLX5_RES_XRQ:
609 		return create_xrq_cmd(dev, srq, in);
610 	default:
611 		return create_rmp_cmd(dev, srq, in);
612 	}
613 }
614 
615 static int destroy_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
616 {
617 	if (!dev->mdev->issi)
618 		return destroy_srq_cmd(dev, srq);
619 	switch (srq->common.res) {
620 	case MLX5_RES_XSRQ:
621 		return destroy_xrc_srq_cmd(dev, srq);
622 	case MLX5_RES_XRQ:
623 		return destroy_xrq_cmd(dev, srq);
624 	default:
625 		return destroy_rmp_cmd(dev, srq);
626 	}
627 }
628 
629 int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
630 			struct mlx5_srq_attr *in)
631 {
632 	struct mlx5_srq_table *table = &dev->srq_table;
633 	int err;
634 
635 	switch (in->type) {
636 	case IB_SRQT_XRC:
637 		srq->common.res = MLX5_RES_XSRQ;
638 		break;
639 	case IB_SRQT_TM:
640 		srq->common.res = MLX5_RES_XRQ;
641 		break;
642 	default:
643 		srq->common.res = MLX5_RES_SRQ;
644 	}
645 
646 	err = create_srq_split(dev, srq, in);
647 	if (err)
648 		return err;
649 
650 	refcount_set(&srq->common.refcount, 1);
651 	init_completion(&srq->common.free);
652 
653 	err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));
654 	if (err)
655 		goto err_destroy_srq_split;
656 
657 	return 0;
658 
659 err_destroy_srq_split:
660 	destroy_srq_split(dev, srq);
661 
662 	return err;
663 }
664 
665 int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
666 {
667 	struct mlx5_srq_table *table = &dev->srq_table;
668 	struct mlx5_core_srq *tmp;
669 	int err;
670 
671 	/* Delete entry, but leave index occupied */
672 	tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0);
673 	if (WARN_ON(tmp != srq))
674 		return xa_err(tmp) ?: -EINVAL;
675 
676 	err = destroy_srq_split(dev, srq);
677 	if (err) {
678 		/*
679 		 * We don't need to check returned result for an error,
680 		 * because  we are storing in pre-allocated space xarray
681 		 * entry and it can't fail at this stage.
682 		 */
683 		xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0);
684 		return err;
685 	}
686 	xa_erase_irq(&table->array, srq->srqn);
687 
688 	mlx5_core_res_put(&srq->common);
689 	wait_for_completion(&srq->common.free);
690 	return 0;
691 }
692 
693 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
694 		       struct mlx5_srq_attr *out)
695 {
696 	if (!dev->mdev->issi)
697 		return query_srq_cmd(dev, srq, out);
698 	switch (srq->common.res) {
699 	case MLX5_RES_XSRQ:
700 		return query_xrc_srq_cmd(dev, srq, out);
701 	case MLX5_RES_XRQ:
702 		return query_xrq_cmd(dev, srq, out);
703 	default:
704 		return query_rmp_cmd(dev, srq, out);
705 	}
706 }
707 
708 int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
709 		     u16 lwm, int is_srq)
710 {
711 	if (!dev->mdev->issi)
712 		return arm_srq_cmd(dev, srq, lwm, is_srq);
713 	switch (srq->common.res) {
714 	case MLX5_RES_XSRQ:
715 		return arm_xrc_srq_cmd(dev, srq, lwm);
716 	case MLX5_RES_XRQ:
717 		return arm_xrq_cmd(dev, srq, lwm);
718 	default:
719 		return arm_rmp_cmd(dev, srq, lwm);
720 	}
721 }
722 
723 static int srq_event_notifier(struct notifier_block *nb,
724 			      unsigned long type, void *data)
725 {
726 	struct mlx5_srq_table *table;
727 	struct mlx5_core_srq *srq;
728 	struct mlx5_eqe *eqe;
729 	u32 srqn;
730 
731 	if (type != MLX5_EVENT_TYPE_SRQ_CATAS_ERROR &&
732 	    type != MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)
733 		return NOTIFY_DONE;
734 
735 	table = container_of(nb, struct mlx5_srq_table, nb);
736 
737 	eqe = data;
738 	srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
739 
740 	xa_lock(&table->array);
741 	srq = xa_load(&table->array, srqn);
742 	if (srq)
743 		refcount_inc(&srq->common.refcount);
744 	xa_unlock(&table->array);
745 
746 	if (!srq)
747 		return NOTIFY_OK;
748 
749 	srq->event(srq, eqe->type);
750 
751 	mlx5_core_res_put(&srq->common);
752 
753 	return NOTIFY_OK;
754 }
755 
756 int mlx5_init_srq_table(struct mlx5_ib_dev *dev)
757 {
758 	struct mlx5_srq_table *table = &dev->srq_table;
759 
760 	memset(table, 0, sizeof(*table));
761 	xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ);
762 
763 	table->nb.notifier_call = srq_event_notifier;
764 	mlx5_notifier_register(dev->mdev, &table->nb);
765 
766 	return 0;
767 }
768 
769 void mlx5_cleanup_srq_table(struct mlx5_ib_dev *dev)
770 {
771 	struct mlx5_srq_table *table = &dev->srq_table;
772 
773 	mlx5_notifier_unregister(dev->mdev, &table->nb);
774 }
775