1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/mlx5/driver.h> 8 #include <linux/mlx5/cmd.h> 9 #include "mlx5_ib.h" 10 #include "srq.h" 11 12 static int get_pas_size(struct mlx5_srq_attr *in) 13 { 14 u32 log_page_size = in->log_page_size + 12; 15 u32 log_srq_size = in->log_size; 16 u32 log_rq_stride = in->wqe_shift; 17 u32 page_offset = in->page_offset; 18 u32 po_quanta = 1 << (log_page_size - 6); 19 u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride); 20 u32 page_size = 1 << log_page_size; 21 u32 rq_sz_po = rq_sz + (page_offset * po_quanta); 22 u32 rq_num_pas = DIV_ROUND_UP(rq_sz_po, page_size); 23 24 return rq_num_pas * sizeof(u64); 25 } 26 27 static void set_wq(void *wq, struct mlx5_srq_attr *in) 28 { 29 MLX5_SET(wq, wq, wq_signature, !!(in->flags 30 & MLX5_SRQ_FLAG_WQ_SIG)); 31 MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size); 32 MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4); 33 MLX5_SET(wq, wq, log_wq_sz, in->log_size); 34 MLX5_SET(wq, wq, page_offset, in->page_offset); 35 MLX5_SET(wq, wq, lwm, in->lwm); 36 MLX5_SET(wq, wq, pd, in->pd); 37 MLX5_SET64(wq, wq, dbr_addr, in->db_record); 38 } 39 40 static void set_srqc(void *srqc, struct mlx5_srq_attr *in) 41 { 42 MLX5_SET(srqc, srqc, wq_signature, !!(in->flags 43 & MLX5_SRQ_FLAG_WQ_SIG)); 44 MLX5_SET(srqc, srqc, log_page_size, in->log_page_size); 45 MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift); 46 MLX5_SET(srqc, srqc, log_srq_size, in->log_size); 47 MLX5_SET(srqc, srqc, page_offset, in->page_offset); 48 MLX5_SET(srqc, srqc, lwm, in->lwm); 49 MLX5_SET(srqc, srqc, pd, in->pd); 50 MLX5_SET64(srqc, srqc, dbr_addr, in->db_record); 51 MLX5_SET(srqc, srqc, xrcd, in->xrcd); 52 MLX5_SET(srqc, srqc, cqn, in->cqn); 53 } 54 55 static void get_wq(void *wq, struct mlx5_srq_attr *in) 56 { 57 if (MLX5_GET(wq, wq, wq_signature)) 58 in->flags &= MLX5_SRQ_FLAG_WQ_SIG; 59 in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz); 60 in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4; 61 in->log_size = MLX5_GET(wq, wq, log_wq_sz); 62 in->page_offset = MLX5_GET(wq, wq, page_offset); 63 in->lwm = MLX5_GET(wq, wq, lwm); 64 in->pd = MLX5_GET(wq, wq, pd); 65 in->db_record = MLX5_GET64(wq, wq, dbr_addr); 66 } 67 68 static void get_srqc(void *srqc, struct mlx5_srq_attr *in) 69 { 70 if (MLX5_GET(srqc, srqc, wq_signature)) 71 in->flags &= MLX5_SRQ_FLAG_WQ_SIG; 72 in->log_page_size = MLX5_GET(srqc, srqc, log_page_size); 73 in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride); 74 in->log_size = MLX5_GET(srqc, srqc, log_srq_size); 75 in->page_offset = MLX5_GET(srqc, srqc, page_offset); 76 in->lwm = MLX5_GET(srqc, srqc, lwm); 77 in->pd = MLX5_GET(srqc, srqc, pd); 78 in->db_record = MLX5_GET64(srqc, srqc, dbr_addr); 79 } 80 81 struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn) 82 { 83 struct mlx5_srq_table *table = &dev->srq_table; 84 struct mlx5_core_srq *srq; 85 86 xa_lock(&table->array); 87 srq = xa_load(&table->array, srqn); 88 if (srq) 89 atomic_inc(&srq->common.refcount); 90 xa_unlock(&table->array); 91 92 return srq; 93 } 94 95 static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 96 struct mlx5_srq_attr *in) 97 { 98 u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0}; 99 void *create_in; 100 void *srqc; 101 void *pas; 102 int pas_size; 103 int inlen; 104 int err; 105 106 pas_size = get_pas_size(in); 107 inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size; 108 create_in = kvzalloc(inlen, GFP_KERNEL); 109 if (!create_in) 110 return -ENOMEM; 111 112 MLX5_SET(create_srq_in, create_in, uid, in->uid); 113 srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry); 114 pas = MLX5_ADDR_OF(create_srq_in, create_in, pas); 115 116 set_srqc(srqc, in); 117 memcpy(pas, in->pas, pas_size); 118 119 MLX5_SET(create_srq_in, create_in, opcode, 120 MLX5_CMD_OP_CREATE_SRQ); 121 122 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, 123 sizeof(create_out)); 124 kvfree(create_in); 125 if (!err) { 126 srq->srqn = MLX5_GET(create_srq_out, create_out, srqn); 127 srq->uid = in->uid; 128 } 129 130 return err; 131 } 132 133 static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) 134 { 135 u32 srq_in[MLX5_ST_SZ_DW(destroy_srq_in)] = {0}; 136 u32 srq_out[MLX5_ST_SZ_DW(destroy_srq_out)] = {0}; 137 138 MLX5_SET(destroy_srq_in, srq_in, opcode, 139 MLX5_CMD_OP_DESTROY_SRQ); 140 MLX5_SET(destroy_srq_in, srq_in, srqn, srq->srqn); 141 MLX5_SET(destroy_srq_in, srq_in, uid, srq->uid); 142 143 return mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out, 144 sizeof(srq_out)); 145 } 146 147 static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 148 u16 lwm, int is_srq) 149 { 150 u32 srq_in[MLX5_ST_SZ_DW(arm_rq_in)] = {0}; 151 u32 srq_out[MLX5_ST_SZ_DW(arm_rq_out)] = {0}; 152 153 MLX5_SET(arm_rq_in, srq_in, opcode, MLX5_CMD_OP_ARM_RQ); 154 MLX5_SET(arm_rq_in, srq_in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ); 155 MLX5_SET(arm_rq_in, srq_in, srq_number, srq->srqn); 156 MLX5_SET(arm_rq_in, srq_in, lwm, lwm); 157 MLX5_SET(arm_rq_in, srq_in, uid, srq->uid); 158 159 return mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out, 160 sizeof(srq_out)); 161 } 162 163 static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 164 struct mlx5_srq_attr *out) 165 { 166 u32 srq_in[MLX5_ST_SZ_DW(query_srq_in)] = {0}; 167 u32 *srq_out; 168 void *srqc; 169 int err; 170 171 srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL); 172 if (!srq_out) 173 return -ENOMEM; 174 175 MLX5_SET(query_srq_in, srq_in, opcode, 176 MLX5_CMD_OP_QUERY_SRQ); 177 MLX5_SET(query_srq_in, srq_in, srqn, srq->srqn); 178 err = mlx5_cmd_exec(dev->mdev, srq_in, sizeof(srq_in), srq_out, 179 MLX5_ST_SZ_BYTES(query_srq_out)); 180 if (err) 181 goto out; 182 183 srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry); 184 get_srqc(srqc, out); 185 if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD) 186 out->flags |= MLX5_SRQ_FLAG_ERR; 187 out: 188 kvfree(srq_out); 189 return err; 190 } 191 192 static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev, 193 struct mlx5_core_srq *srq, 194 struct mlx5_srq_attr *in) 195 { 196 u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)]; 197 void *create_in; 198 void *xrc_srqc; 199 void *pas; 200 int pas_size; 201 int inlen; 202 int err; 203 204 pas_size = get_pas_size(in); 205 inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size; 206 create_in = kvzalloc(inlen, GFP_KERNEL); 207 if (!create_in) 208 return -ENOMEM; 209 210 MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid); 211 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in, 212 xrc_srq_context_entry); 213 pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas); 214 215 set_srqc(xrc_srqc, in); 216 MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index); 217 memcpy(pas, in->pas, pas_size); 218 MLX5_SET(create_xrc_srq_in, create_in, opcode, 219 MLX5_CMD_OP_CREATE_XRC_SRQ); 220 221 memset(create_out, 0, sizeof(create_out)); 222 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, 223 sizeof(create_out)); 224 if (err) 225 goto out; 226 227 srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn); 228 srq->uid = in->uid; 229 out: 230 kvfree(create_in); 231 return err; 232 } 233 234 static int destroy_xrc_srq_cmd(struct mlx5_ib_dev *dev, 235 struct mlx5_core_srq *srq) 236 { 237 u32 xrcsrq_in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {0}; 238 u32 xrcsrq_out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)] = {0}; 239 240 MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, opcode, 241 MLX5_CMD_OP_DESTROY_XRC_SRQ); 242 MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn); 243 MLX5_SET(destroy_xrc_srq_in, xrcsrq_in, uid, srq->uid); 244 245 return mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in), 246 xrcsrq_out, sizeof(xrcsrq_out)); 247 } 248 249 static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 250 u16 lwm) 251 { 252 u32 xrcsrq_in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {0}; 253 u32 xrcsrq_out[MLX5_ST_SZ_DW(arm_xrc_srq_out)] = {0}; 254 255 MLX5_SET(arm_xrc_srq_in, xrcsrq_in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ); 256 MLX5_SET(arm_xrc_srq_in, xrcsrq_in, op_mod, MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ); 257 MLX5_SET(arm_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn); 258 MLX5_SET(arm_xrc_srq_in, xrcsrq_in, lwm, lwm); 259 MLX5_SET(arm_xrc_srq_in, xrcsrq_in, uid, srq->uid); 260 261 return mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in), 262 xrcsrq_out, sizeof(xrcsrq_out)); 263 } 264 265 static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev, 266 struct mlx5_core_srq *srq, 267 struct mlx5_srq_attr *out) 268 { 269 u32 xrcsrq_in[MLX5_ST_SZ_DW(query_xrc_srq_in)]; 270 u32 *xrcsrq_out; 271 void *xrc_srqc; 272 int err; 273 274 xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL); 275 if (!xrcsrq_out) 276 return -ENOMEM; 277 memset(xrcsrq_in, 0, sizeof(xrcsrq_in)); 278 279 MLX5_SET(query_xrc_srq_in, xrcsrq_in, opcode, 280 MLX5_CMD_OP_QUERY_XRC_SRQ); 281 MLX5_SET(query_xrc_srq_in, xrcsrq_in, xrc_srqn, srq->srqn); 282 283 err = mlx5_cmd_exec(dev->mdev, xrcsrq_in, sizeof(xrcsrq_in), 284 xrcsrq_out, MLX5_ST_SZ_BYTES(query_xrc_srq_out)); 285 if (err) 286 goto out; 287 288 xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out, 289 xrc_srq_context_entry); 290 get_srqc(xrc_srqc, out); 291 if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD) 292 out->flags |= MLX5_SRQ_FLAG_ERR; 293 294 out: 295 kvfree(xrcsrq_out); 296 return err; 297 } 298 299 static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 300 struct mlx5_srq_attr *in) 301 { 302 void *create_out = NULL; 303 void *create_in = NULL; 304 void *rmpc; 305 void *wq; 306 int pas_size; 307 int outlen; 308 int inlen; 309 int err; 310 311 pas_size = get_pas_size(in); 312 inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size; 313 outlen = MLX5_ST_SZ_BYTES(create_rmp_out); 314 create_in = kvzalloc(inlen, GFP_KERNEL); 315 create_out = kvzalloc(outlen, GFP_KERNEL); 316 if (!create_in || !create_out) { 317 err = -ENOMEM; 318 goto out; 319 } 320 321 rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx); 322 wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 323 324 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY); 325 MLX5_SET(create_rmp_in, create_in, uid, in->uid); 326 set_wq(wq, in); 327 memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size); 328 329 MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP); 330 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen); 331 if (!err) { 332 srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn); 333 srq->uid = in->uid; 334 } 335 336 out: 337 kvfree(create_in); 338 kvfree(create_out); 339 return err; 340 } 341 342 static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) 343 { 344 u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {}; 345 u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)] = {}; 346 347 MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP); 348 MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn); 349 MLX5_SET(destroy_rmp_in, in, uid, srq->uid); 350 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 351 } 352 353 static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 354 u16 lwm) 355 { 356 void *out = NULL; 357 void *in = NULL; 358 void *rmpc; 359 void *wq; 360 void *bitmask; 361 int outlen; 362 int inlen; 363 int err; 364 365 inlen = MLX5_ST_SZ_BYTES(modify_rmp_in); 366 outlen = MLX5_ST_SZ_BYTES(modify_rmp_out); 367 368 in = kvzalloc(inlen, GFP_KERNEL); 369 out = kvzalloc(outlen, GFP_KERNEL); 370 if (!in || !out) { 371 err = -ENOMEM; 372 goto out; 373 } 374 375 rmpc = MLX5_ADDR_OF(modify_rmp_in, in, ctx); 376 bitmask = MLX5_ADDR_OF(modify_rmp_in, in, bitmask); 377 wq = MLX5_ADDR_OF(rmpc, rmpc, wq); 378 379 MLX5_SET(modify_rmp_in, in, rmp_state, MLX5_RMPC_STATE_RDY); 380 MLX5_SET(modify_rmp_in, in, rmpn, srq->srqn); 381 MLX5_SET(modify_rmp_in, in, uid, srq->uid); 382 MLX5_SET(wq, wq, lwm, lwm); 383 MLX5_SET(rmp_bitmask, bitmask, lwm, 1); 384 MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY); 385 MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP); 386 387 err = mlx5_cmd_exec(dev->mdev, in, inlen, out, outlen); 388 389 out: 390 kvfree(in); 391 kvfree(out); 392 return err; 393 } 394 395 static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 396 struct mlx5_srq_attr *out) 397 { 398 u32 *rmp_out = NULL; 399 u32 *rmp_in = NULL; 400 void *rmpc; 401 int outlen; 402 int inlen; 403 int err; 404 405 outlen = MLX5_ST_SZ_BYTES(query_rmp_out); 406 inlen = MLX5_ST_SZ_BYTES(query_rmp_in); 407 408 rmp_out = kvzalloc(outlen, GFP_KERNEL); 409 rmp_in = kvzalloc(inlen, GFP_KERNEL); 410 if (!rmp_out || !rmp_in) { 411 err = -ENOMEM; 412 goto out; 413 } 414 415 MLX5_SET(query_rmp_in, rmp_in, opcode, MLX5_CMD_OP_QUERY_RMP); 416 MLX5_SET(query_rmp_in, rmp_in, rmpn, srq->srqn); 417 err = mlx5_cmd_exec(dev->mdev, rmp_in, inlen, rmp_out, outlen); 418 if (err) 419 goto out; 420 421 rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context); 422 get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out); 423 if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY) 424 out->flags |= MLX5_SRQ_FLAG_ERR; 425 426 out: 427 kvfree(rmp_out); 428 kvfree(rmp_in); 429 return err; 430 } 431 432 static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 433 struct mlx5_srq_attr *in) 434 { 435 u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0}; 436 void *create_in; 437 void *xrqc; 438 void *wq; 439 int pas_size; 440 int inlen; 441 int err; 442 443 pas_size = get_pas_size(in); 444 inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size; 445 create_in = kvzalloc(inlen, GFP_KERNEL); 446 if (!create_in) 447 return -ENOMEM; 448 449 xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context); 450 wq = MLX5_ADDR_OF(xrqc, xrqc, wq); 451 452 set_wq(wq, in); 453 memcpy(MLX5_ADDR_OF(xrqc, xrqc, wq.pas), in->pas, pas_size); 454 455 if (in->type == IB_SRQT_TM) { 456 MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING); 457 if (in->flags & MLX5_SRQ_FLAG_RNDV) 458 MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV); 459 MLX5_SET(xrqc, xrqc, 460 tag_matching_topology_context.log_matching_list_sz, 461 in->tm_log_list_size); 462 } 463 MLX5_SET(xrqc, xrqc, user_index, in->user_index); 464 MLX5_SET(xrqc, xrqc, cqn, in->cqn); 465 MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ); 466 MLX5_SET(create_xrq_in, create_in, uid, in->uid); 467 err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, 468 sizeof(create_out)); 469 kvfree(create_in); 470 if (!err) { 471 srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn); 472 srq->uid = in->uid; 473 } 474 475 return err; 476 } 477 478 static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) 479 { 480 u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {0}; 481 u32 out[MLX5_ST_SZ_DW(destroy_xrq_out)] = {0}; 482 483 MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ); 484 MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn); 485 MLX5_SET(destroy_xrq_in, in, uid, srq->uid); 486 487 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 488 } 489 490 static int arm_xrq_cmd(struct mlx5_ib_dev *dev, 491 struct mlx5_core_srq *srq, 492 u16 lwm) 493 { 494 u32 out[MLX5_ST_SZ_DW(arm_rq_out)] = {0}; 495 u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {0}; 496 497 MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ); 498 MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ); 499 MLX5_SET(arm_rq_in, in, srq_number, srq->srqn); 500 MLX5_SET(arm_rq_in, in, lwm, lwm); 501 MLX5_SET(arm_rq_in, in, uid, srq->uid); 502 503 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 504 } 505 506 static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 507 struct mlx5_srq_attr *out) 508 { 509 u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {0}; 510 u32 *xrq_out; 511 int outlen = MLX5_ST_SZ_BYTES(query_xrq_out); 512 void *xrqc; 513 int err; 514 515 xrq_out = kvzalloc(outlen, GFP_KERNEL); 516 if (!xrq_out) 517 return -ENOMEM; 518 519 MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ); 520 MLX5_SET(query_xrq_in, in, xrqn, srq->srqn); 521 522 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), xrq_out, outlen); 523 if (err) 524 goto out; 525 526 xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context); 527 get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out); 528 if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD) 529 out->flags |= MLX5_SRQ_FLAG_ERR; 530 out->tm_next_tag = 531 MLX5_GET(xrqc, xrqc, 532 tag_matching_topology_context.append_next_index); 533 out->tm_hw_phase_cnt = 534 MLX5_GET(xrqc, xrqc, 535 tag_matching_topology_context.hw_phase_cnt); 536 out->tm_sw_phase_cnt = 537 MLX5_GET(xrqc, xrqc, 538 tag_matching_topology_context.sw_phase_cnt); 539 540 out: 541 kvfree(xrq_out); 542 return err; 543 } 544 545 static int create_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 546 struct mlx5_srq_attr *in) 547 { 548 if (!dev->mdev->issi) 549 return create_srq_cmd(dev, srq, in); 550 switch (srq->common.res) { 551 case MLX5_RES_XSRQ: 552 return create_xrc_srq_cmd(dev, srq, in); 553 case MLX5_RES_XRQ: 554 return create_xrq_cmd(dev, srq, in); 555 default: 556 return create_rmp_cmd(dev, srq, in); 557 } 558 } 559 560 static int destroy_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) 561 { 562 if (!dev->mdev->issi) 563 return destroy_srq_cmd(dev, srq); 564 switch (srq->common.res) { 565 case MLX5_RES_XSRQ: 566 return destroy_xrc_srq_cmd(dev, srq); 567 case MLX5_RES_XRQ: 568 return destroy_xrq_cmd(dev, srq); 569 default: 570 return destroy_rmp_cmd(dev, srq); 571 } 572 } 573 574 int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 575 struct mlx5_srq_attr *in) 576 { 577 struct mlx5_srq_table *table = &dev->srq_table; 578 int err; 579 580 switch (in->type) { 581 case IB_SRQT_XRC: 582 srq->common.res = MLX5_RES_XSRQ; 583 break; 584 case IB_SRQT_TM: 585 srq->common.res = MLX5_RES_XRQ; 586 break; 587 default: 588 srq->common.res = MLX5_RES_SRQ; 589 } 590 591 err = create_srq_split(dev, srq, in); 592 if (err) 593 return err; 594 595 atomic_set(&srq->common.refcount, 1); 596 init_completion(&srq->common.free); 597 598 err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL)); 599 if (err) 600 goto err_destroy_srq_split; 601 602 return 0; 603 604 err_destroy_srq_split: 605 destroy_srq_split(dev, srq); 606 607 return err; 608 } 609 610 void mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq) 611 { 612 struct mlx5_srq_table *table = &dev->srq_table; 613 struct mlx5_core_srq *tmp; 614 int err; 615 616 tmp = xa_erase_irq(&table->array, srq->srqn); 617 if (!tmp || tmp != srq) 618 return; 619 620 err = destroy_srq_split(dev, srq); 621 if (err) 622 return; 623 624 mlx5_core_res_put(&srq->common); 625 wait_for_completion(&srq->common.free); 626 } 627 628 int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 629 struct mlx5_srq_attr *out) 630 { 631 if (!dev->mdev->issi) 632 return query_srq_cmd(dev, srq, out); 633 switch (srq->common.res) { 634 case MLX5_RES_XSRQ: 635 return query_xrc_srq_cmd(dev, srq, out); 636 case MLX5_RES_XRQ: 637 return query_xrq_cmd(dev, srq, out); 638 default: 639 return query_rmp_cmd(dev, srq, out); 640 } 641 } 642 643 int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq, 644 u16 lwm, int is_srq) 645 { 646 if (!dev->mdev->issi) 647 return arm_srq_cmd(dev, srq, lwm, is_srq); 648 switch (srq->common.res) { 649 case MLX5_RES_XSRQ: 650 return arm_xrc_srq_cmd(dev, srq, lwm); 651 case MLX5_RES_XRQ: 652 return arm_xrq_cmd(dev, srq, lwm); 653 default: 654 return arm_rmp_cmd(dev, srq, lwm); 655 } 656 } 657 658 static int srq_event_notifier(struct notifier_block *nb, 659 unsigned long type, void *data) 660 { 661 struct mlx5_srq_table *table; 662 struct mlx5_core_srq *srq; 663 struct mlx5_eqe *eqe; 664 u32 srqn; 665 666 if (type != MLX5_EVENT_TYPE_SRQ_CATAS_ERROR && 667 type != MLX5_EVENT_TYPE_SRQ_RQ_LIMIT) 668 return NOTIFY_DONE; 669 670 table = container_of(nb, struct mlx5_srq_table, nb); 671 672 eqe = data; 673 srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; 674 675 xa_lock(&table->array); 676 srq = xa_load(&table->array, srqn); 677 if (srq) 678 atomic_inc(&srq->common.refcount); 679 xa_unlock(&table->array); 680 681 if (!srq) 682 return NOTIFY_OK; 683 684 srq->event(srq, eqe->type); 685 686 mlx5_core_res_put(&srq->common); 687 688 return NOTIFY_OK; 689 } 690 691 int mlx5_init_srq_table(struct mlx5_ib_dev *dev) 692 { 693 struct mlx5_srq_table *table = &dev->srq_table; 694 695 memset(table, 0, sizeof(*table)); 696 xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ); 697 698 table->nb.notifier_call = srq_event_notifier; 699 mlx5_notifier_register(dev->mdev, &table->nb); 700 701 return 0; 702 } 703 704 void mlx5_cleanup_srq_table(struct mlx5_ib_dev *dev) 705 { 706 struct mlx5_srq_table *table = &dev->srq_table; 707 708 mlx5_notifier_unregister(dev->mdev, &table->nb); 709 } 710