1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2018, Mellanox Technologies inc.  All rights reserved.
4  */
5 
6 #include <linux/kernel.h>
7 #include <linux/mlx5/driver.h>
8 #include "mlx5_ib.h"
9 #include "srq.h"
10 #include "qp.h"
11 
12 static int get_pas_size(struct mlx5_srq_attr *in)
13 {
14 	u32 log_page_size = in->log_page_size + 12;
15 	u32 log_srq_size  = in->log_size;
16 	u32 log_rq_stride = in->wqe_shift;
17 	u32 page_offset   = in->page_offset;
18 	u32 po_quanta	  = 1 << (log_page_size - 6);
19 	u32 rq_sz	  = 1 << (log_srq_size + 4 + log_rq_stride);
20 	u32 page_size	  = 1 << log_page_size;
21 	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
22 	u32 rq_num_pas    = DIV_ROUND_UP(rq_sz_po, page_size);
23 
24 	return rq_num_pas * sizeof(u64);
25 }
26 
27 static void set_wq(void *wq, struct mlx5_srq_attr *in)
28 {
29 	MLX5_SET(wq,   wq, wq_signature,  !!(in->flags
30 		 & MLX5_SRQ_FLAG_WQ_SIG));
31 	MLX5_SET(wq,   wq, log_wq_pg_sz,  in->log_page_size);
32 	MLX5_SET(wq,   wq, log_wq_stride, in->wqe_shift + 4);
33 	MLX5_SET(wq,   wq, log_wq_sz,     in->log_size);
34 	MLX5_SET(wq,   wq, page_offset,   in->page_offset);
35 	MLX5_SET(wq,   wq, lwm,		  in->lwm);
36 	MLX5_SET(wq,   wq, pd,		  in->pd);
37 	MLX5_SET64(wq, wq, dbr_addr,	  in->db_record);
38 }
39 
40 static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
41 {
42 	MLX5_SET(srqc,   srqc, wq_signature,  !!(in->flags
43 		 & MLX5_SRQ_FLAG_WQ_SIG));
44 	MLX5_SET(srqc,   srqc, log_page_size, in->log_page_size);
45 	MLX5_SET(srqc,   srqc, log_rq_stride, in->wqe_shift);
46 	MLX5_SET(srqc,   srqc, log_srq_size,  in->log_size);
47 	MLX5_SET(srqc,   srqc, page_offset,   in->page_offset);
48 	MLX5_SET(srqc,	 srqc, lwm,	      in->lwm);
49 	MLX5_SET(srqc,	 srqc, pd,	      in->pd);
50 	MLX5_SET64(srqc, srqc, dbr_addr,      in->db_record);
51 	MLX5_SET(srqc,	 srqc, xrcd,	      in->xrcd);
52 	MLX5_SET(srqc,	 srqc, cqn,	      in->cqn);
53 }
54 
55 static void get_wq(void *wq, struct mlx5_srq_attr *in)
56 {
57 	if (MLX5_GET(wq, wq, wq_signature))
58 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
59 	in->log_page_size = MLX5_GET(wq,   wq, log_wq_pg_sz);
60 	in->wqe_shift	  = MLX5_GET(wq,   wq, log_wq_stride) - 4;
61 	in->log_size	  = MLX5_GET(wq,   wq, log_wq_sz);
62 	in->page_offset   = MLX5_GET(wq,   wq, page_offset);
63 	in->lwm		  = MLX5_GET(wq,   wq, lwm);
64 	in->pd		  = MLX5_GET(wq,   wq, pd);
65 	in->db_record	  = MLX5_GET64(wq, wq, dbr_addr);
66 }
67 
68 static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
69 {
70 	if (MLX5_GET(srqc, srqc, wq_signature))
71 		in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
72 	in->log_page_size = MLX5_GET(srqc,   srqc, log_page_size);
73 	in->wqe_shift	  = MLX5_GET(srqc,   srqc, log_rq_stride);
74 	in->log_size	  = MLX5_GET(srqc,   srqc, log_srq_size);
75 	in->page_offset   = MLX5_GET(srqc,   srqc, page_offset);
76 	in->lwm		  = MLX5_GET(srqc,   srqc, lwm);
77 	in->pd		  = MLX5_GET(srqc,   srqc, pd);
78 	in->db_record	  = MLX5_GET64(srqc, srqc, dbr_addr);
79 }
80 
81 struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn)
82 {
83 	struct mlx5_srq_table *table = &dev->srq_table;
84 	struct mlx5_core_srq *srq;
85 
86 	xa_lock(&table->array);
87 	srq = xa_load(&table->array, srqn);
88 	if (srq)
89 		refcount_inc(&srq->common.refcount);
90 	xa_unlock(&table->array);
91 
92 	return srq;
93 }
94 
95 static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
96 			  struct mlx5_srq_attr *in)
97 {
98 	u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
99 	void *create_in;
100 	void *srqc;
101 	void *pas;
102 	int pas_size;
103 	int inlen;
104 	int err;
105 
106 	pas_size  = get_pas_size(in);
107 	inlen	  = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
108 	create_in = kvzalloc(inlen, GFP_KERNEL);
109 	if (!create_in)
110 		return -ENOMEM;
111 
112 	MLX5_SET(create_srq_in, create_in, uid, in->uid);
113 	srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
114 	pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
115 
116 	set_srqc(srqc, in);
117 	memcpy(pas, in->pas, pas_size);
118 
119 	MLX5_SET(create_srq_in, create_in, opcode,
120 		 MLX5_CMD_OP_CREATE_SRQ);
121 
122 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
123 			    sizeof(create_out));
124 	kvfree(create_in);
125 	if (!err) {
126 		srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
127 		srq->uid = in->uid;
128 	}
129 
130 	return err;
131 }
132 
133 static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
134 {
135 	u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {};
136 
137 	MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ);
138 	MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);
139 	MLX5_SET(destroy_srq_in, in, uid, srq->uid);
140 
141 	return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in);
142 }
143 
144 static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
145 		       u16 lwm, int is_srq)
146 {
147 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
148 
149 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
150 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
151 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
152 	MLX5_SET(arm_rq_in, in, lwm, lwm);
153 	MLX5_SET(arm_rq_in, in, uid, srq->uid);
154 
155 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
156 }
157 
158 static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
159 			 struct mlx5_srq_attr *out)
160 {
161 	u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {};
162 	u32 *srq_out;
163 	void *srqc;
164 	int err;
165 
166 	srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);
167 	if (!srq_out)
168 		return -ENOMEM;
169 
170 	MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ);
171 	MLX5_SET(query_srq_in, in, srqn, srq->srqn);
172 	err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out);
173 	if (err)
174 		goto out;
175 
176 	srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
177 	get_srqc(srqc, out);
178 	if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
179 		out->flags |= MLX5_SRQ_FLAG_ERR;
180 out:
181 	kvfree(srq_out);
182 	return err;
183 }
184 
185 static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev,
186 			      struct mlx5_core_srq *srq,
187 			      struct mlx5_srq_attr *in)
188 {
189 	u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
190 	void *create_in;
191 	void *xrc_srqc;
192 	void *pas;
193 	int pas_size;
194 	int inlen;
195 	int err;
196 
197 	pas_size  = get_pas_size(in);
198 	inlen	  = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
199 	create_in = kvzalloc(inlen, GFP_KERNEL);
200 	if (!create_in)
201 		return -ENOMEM;
202 
203 	MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);
204 	xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,
205 				xrc_srq_context_entry);
206 	pas	 = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
207 
208 	set_srqc(xrc_srqc, in);
209 	MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
210 	memcpy(pas, in->pas, pas_size);
211 	MLX5_SET(create_xrc_srq_in, create_in, opcode,
212 		 MLX5_CMD_OP_CREATE_XRC_SRQ);
213 
214 	memset(create_out, 0, sizeof(create_out));
215 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
216 			    sizeof(create_out));
217 	if (err)
218 		goto out;
219 
220 	srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);
221 	srq->uid = in->uid;
222 out:
223 	kvfree(create_in);
224 	return err;
225 }
226 
227 static int destroy_xrc_srq_cmd(struct mlx5_ib_dev *dev,
228 			       struct mlx5_core_srq *srq)
229 {
230 	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {};
231 
232 	MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ);
233 	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);
234 	MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);
235 
236 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in);
237 }
238 
239 static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
240 			   u16 lwm)
241 {
242 	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {};
243 
244 	MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
245 	MLX5_SET(arm_xrc_srq_in, in, op_mod,
246 		 MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
247 	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);
248 	MLX5_SET(arm_xrc_srq_in, in, lwm, lwm);
249 	MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);
250 
251 	return  mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in);
252 }
253 
254 static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev,
255 			     struct mlx5_core_srq *srq,
256 			     struct mlx5_srq_attr *out)
257 {
258 	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {};
259 	u32 *xrcsrq_out;
260 	void *xrc_srqc;
261 	int err;
262 
263 	xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
264 	if (!xrcsrq_out)
265 		return -ENOMEM;
266 
267 	MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);
268 	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);
269 
270 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out);
271 	if (err)
272 		goto out;
273 
274 	xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
275 				xrc_srq_context_entry);
276 	get_srqc(xrc_srqc, out);
277 	if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
278 		out->flags |= MLX5_SRQ_FLAG_ERR;
279 
280 out:
281 	kvfree(xrcsrq_out);
282 	return err;
283 }
284 
285 static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
286 			  struct mlx5_srq_attr *in)
287 {
288 	void *create_out = NULL;
289 	void *create_in = NULL;
290 	void *rmpc;
291 	void *wq;
292 	int pas_size;
293 	int outlen;
294 	int inlen;
295 	int err;
296 
297 	pas_size = get_pas_size(in);
298 	inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
299 	outlen = MLX5_ST_SZ_BYTES(create_rmp_out);
300 	create_in = kvzalloc(inlen, GFP_KERNEL);
301 	create_out = kvzalloc(outlen, GFP_KERNEL);
302 	if (!create_in || !create_out) {
303 		err = -ENOMEM;
304 		goto out;
305 	}
306 
307 	rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
308 	wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
309 
310 	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
311 	MLX5_SET(create_rmp_in, create_in, uid, in->uid);
312 	set_wq(wq, in);
313 	memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
314 
315 	MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);
316 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);
317 	if (!err) {
318 		srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn);
319 		srq->uid = in->uid;
320 	}
321 
322 out:
323 	kvfree(create_in);
324 	kvfree(create_out);
325 	return err;
326 }
327 
328 static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
329 {
330 	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
331 
332 	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
333 	MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
334 	MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
335 	return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in);
336 }
337 
338 static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
339 		       u16 lwm)
340 {
341 	void *out = NULL;
342 	void *in = NULL;
343 	void *rmpc;
344 	void *wq;
345 	void *bitmask;
346 	int outlen;
347 	int inlen;
348 	int err;
349 
350 	inlen = MLX5_ST_SZ_BYTES(modify_rmp_in);
351 	outlen = MLX5_ST_SZ_BYTES(modify_rmp_out);
352 
353 	in = kvzalloc(inlen, GFP_KERNEL);
354 	out = kvzalloc(outlen, GFP_KERNEL);
355 	if (!in || !out) {
356 		err = -ENOMEM;
357 		goto out;
358 	}
359 
360 	rmpc =	  MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);
361 	bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);
362 	wq   =	  MLX5_ADDR_OF(rmpc,	        rmpc, wq);
363 
364 	MLX5_SET(modify_rmp_in, in,	 rmp_state, MLX5_RMPC_STATE_RDY);
365 	MLX5_SET(modify_rmp_in, in,	 rmpn,      srq->srqn);
366 	MLX5_SET(modify_rmp_in, in, uid, srq->uid);
367 	MLX5_SET(wq,		wq,	 lwm,	    lwm);
368 	MLX5_SET(rmp_bitmask,	bitmask, lwm,	    1);
369 	MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
370 	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
371 
372 	err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out);
373 
374 out:
375 	kvfree(in);
376 	kvfree(out);
377 	return err;
378 }
379 
380 static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
381 			 struct mlx5_srq_attr *out)
382 {
383 	u32 *rmp_out = NULL;
384 	u32 *rmp_in = NULL;
385 	void *rmpc;
386 	int outlen;
387 	int inlen;
388 	int err;
389 
390 	outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
391 	inlen = MLX5_ST_SZ_BYTES(query_rmp_in);
392 
393 	rmp_out = kvzalloc(outlen, GFP_KERNEL);
394 	rmp_in = kvzalloc(inlen, GFP_KERNEL);
395 	if (!rmp_out || !rmp_in) {
396 		err = -ENOMEM;
397 		goto out;
398 	}
399 
400 	MLX5_SET(query_rmp_in, rmp_in, opcode, MLX5_CMD_OP_QUERY_RMP);
401 	MLX5_SET(query_rmp_in, rmp_in, rmpn,   srq->srqn);
402 	err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out);
403 	if (err)
404 		goto out;
405 
406 	rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
407 	get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
408 	if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
409 		out->flags |= MLX5_SRQ_FLAG_ERR;
410 
411 out:
412 	kvfree(rmp_out);
413 	kvfree(rmp_in);
414 	return err;
415 }
416 
417 static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
418 			  struct mlx5_srq_attr *in)
419 {
420 	u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
421 	void *create_in;
422 	void *xrqc;
423 	void *wq;
424 	int pas_size;
425 	int inlen;
426 	int err;
427 
428 	pas_size = get_pas_size(in);
429 	inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
430 	create_in = kvzalloc(inlen, GFP_KERNEL);
431 	if (!create_in)
432 		return -ENOMEM;
433 
434 	xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
435 	wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
436 
437 	set_wq(wq, in);
438 	memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size);
439 
440 	if (in->type == IB_SRQT_TM) {
441 		MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
442 		if (in->flags & MLX5_SRQ_FLAG_RNDV)
443 			MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
444 		MLX5_SET(xrqc, xrqc,
445 			 tag_matching_topology_context.log_matching_list_sz,
446 			 in->tm_log_list_size);
447 	}
448 	MLX5_SET(xrqc, xrqc, user_index, in->user_index);
449 	MLX5_SET(xrqc, xrqc, cqn, in->cqn);
450 	MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
451 	MLX5_SET(create_xrq_in, create_in, uid, in->uid);
452 	err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
453 			    sizeof(create_out));
454 	kvfree(create_in);
455 	if (!err) {
456 		srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
457 		srq->uid = in->uid;
458 	}
459 
460 	return err;
461 }
462 
463 static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
464 {
465 	u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {};
466 
467 	MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
468 	MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
469 	MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
470 
471 	return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in);
472 }
473 
474 static int arm_xrq_cmd(struct mlx5_ib_dev *dev,
475 		       struct mlx5_core_srq *srq,
476 		       u16 lwm)
477 {
478 	u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
479 
480 	MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
481 	MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
482 	MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
483 	MLX5_SET(arm_rq_in, in, lwm, lwm);
484 	MLX5_SET(arm_rq_in, in, uid, srq->uid);
485 
486 	return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
487 }
488 
489 static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
490 			 struct mlx5_srq_attr *out)
491 {
492 	u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {};
493 	u32 *xrq_out;
494 	int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
495 	void *xrqc;
496 	int err;
497 
498 	xrq_out = kvzalloc(outlen, GFP_KERNEL);
499 	if (!xrq_out)
500 		return -ENOMEM;
501 
502 	MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
503 	MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
504 
505 	err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out);
506 	if (err)
507 		goto out;
508 
509 	xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
510 	get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
511 	if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
512 		out->flags |= MLX5_SRQ_FLAG_ERR;
513 	out->tm_next_tag =
514 		MLX5_GET(xrqc, xrqc,
515 			 tag_matching_topology_context.append_next_index);
516 	out->tm_hw_phase_cnt =
517 		MLX5_GET(xrqc, xrqc,
518 			 tag_matching_topology_context.hw_phase_cnt);
519 	out->tm_sw_phase_cnt =
520 		MLX5_GET(xrqc, xrqc,
521 			 tag_matching_topology_context.sw_phase_cnt);
522 
523 out:
524 	kvfree(xrq_out);
525 	return err;
526 }
527 
528 static int create_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
529 			    struct mlx5_srq_attr *in)
530 {
531 	if (!dev->mdev->issi)
532 		return create_srq_cmd(dev, srq, in);
533 	switch (srq->common.res) {
534 	case MLX5_RES_XSRQ:
535 		return create_xrc_srq_cmd(dev, srq, in);
536 	case MLX5_RES_XRQ:
537 		return create_xrq_cmd(dev, srq, in);
538 	default:
539 		return create_rmp_cmd(dev, srq, in);
540 	}
541 }
542 
543 static int destroy_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
544 {
545 	if (!dev->mdev->issi)
546 		return destroy_srq_cmd(dev, srq);
547 	switch (srq->common.res) {
548 	case MLX5_RES_XSRQ:
549 		return destroy_xrc_srq_cmd(dev, srq);
550 	case MLX5_RES_XRQ:
551 		return destroy_xrq_cmd(dev, srq);
552 	default:
553 		return destroy_rmp_cmd(dev, srq);
554 	}
555 }
556 
557 int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
558 			struct mlx5_srq_attr *in)
559 {
560 	struct mlx5_srq_table *table = &dev->srq_table;
561 	int err;
562 
563 	switch (in->type) {
564 	case IB_SRQT_XRC:
565 		srq->common.res = MLX5_RES_XSRQ;
566 		break;
567 	case IB_SRQT_TM:
568 		srq->common.res = MLX5_RES_XRQ;
569 		break;
570 	default:
571 		srq->common.res = MLX5_RES_SRQ;
572 	}
573 
574 	err = create_srq_split(dev, srq, in);
575 	if (err)
576 		return err;
577 
578 	refcount_set(&srq->common.refcount, 1);
579 	init_completion(&srq->common.free);
580 
581 	err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));
582 	if (err)
583 		goto err_destroy_srq_split;
584 
585 	return 0;
586 
587 err_destroy_srq_split:
588 	destroy_srq_split(dev, srq);
589 
590 	return err;
591 }
592 
593 void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
594 {
595 	struct mlx5_srq_table *table = &dev->srq_table;
596 	struct mlx5_core_srq *tmp;
597 	int err;
598 
599 	tmp = xa_erase_irq(&table->array, srq->srqn);
600 	if (!tmp || tmp != srq)
601 		return;
602 
603 	err = destroy_srq_split(dev, srq);
604 	if (err)
605 		return;
606 
607 	mlx5_core_res_put(&srq->common);
608 	wait_for_completion(&srq->common.free);
609 }
610 
611 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
612 		       struct mlx5_srq_attr *out)
613 {
614 	if (!dev->mdev->issi)
615 		return query_srq_cmd(dev, srq, out);
616 	switch (srq->common.res) {
617 	case MLX5_RES_XSRQ:
618 		return query_xrc_srq_cmd(dev, srq, out);
619 	case MLX5_RES_XRQ:
620 		return query_xrq_cmd(dev, srq, out);
621 	default:
622 		return query_rmp_cmd(dev, srq, out);
623 	}
624 }
625 
626 int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
627 		     u16 lwm, int is_srq)
628 {
629 	if (!dev->mdev->issi)
630 		return arm_srq_cmd(dev, srq, lwm, is_srq);
631 	switch (srq->common.res) {
632 	case MLX5_RES_XSRQ:
633 		return arm_xrc_srq_cmd(dev, srq, lwm);
634 	case MLX5_RES_XRQ:
635 		return arm_xrq_cmd(dev, srq, lwm);
636 	default:
637 		return arm_rmp_cmd(dev, srq, lwm);
638 	}
639 }
640 
641 static int srq_event_notifier(struct notifier_block *nb,
642 			      unsigned long type, void *data)
643 {
644 	struct mlx5_srq_table *table;
645 	struct mlx5_core_srq *srq;
646 	struct mlx5_eqe *eqe;
647 	u32 srqn;
648 
649 	if (type != MLX5_EVENT_TYPE_SRQ_CATAS_ERROR &&
650 	    type != MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)
651 		return NOTIFY_DONE;
652 
653 	table = container_of(nb, struct mlx5_srq_table, nb);
654 
655 	eqe = data;
656 	srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
657 
658 	xa_lock(&table->array);
659 	srq = xa_load(&table->array, srqn);
660 	if (srq)
661 		refcount_inc(&srq->common.refcount);
662 	xa_unlock(&table->array);
663 
664 	if (!srq)
665 		return NOTIFY_OK;
666 
667 	srq->event(srq, eqe->type);
668 
669 	mlx5_core_res_put(&srq->common);
670 
671 	return NOTIFY_OK;
672 }
673 
674 int mlx5_init_srq_table(struct mlx5_ib_dev *dev)
675 {
676 	struct mlx5_srq_table *table = &dev->srq_table;
677 
678 	memset(table, 0, sizeof(*table));
679 	xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ);
680 
681 	table->nb.notifier_call = srq_event_notifier;
682 	mlx5_notifier_register(dev->mdev, &table->nb);
683 
684 	return 0;
685 }
686 
687 void mlx5_cleanup_srq_table(struct mlx5_ib_dev *dev)
688 {
689 	struct mlx5_srq_table *table = &dev->srq_table;
690 
691 	mlx5_notifier_unregister(dev->mdev, &table->nb);
692 }
693