1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/mlx5/qp.h> 7 #include <linux/slab.h> 8 #include <rdma/ib_umem.h> 9 #include <rdma/ib_user_verbs.h> 10 #include "mlx5_ib.h" 11 #include "srq.h" 12 13 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 14 { 15 return mlx5_frag_buf_get_wqe(&srq->fbc, n); 16 } 17 18 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 19 { 20 struct ib_event event; 21 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 22 23 if (ibsrq->event_handler) { 24 event.device = ibsrq->device; 25 event.element.srq = ibsrq; 26 switch (type) { 27 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 28 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 29 break; 30 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 31 event.event = IB_EVENT_SRQ_ERR; 32 break; 33 default: 34 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 35 type, srq->srqn); 36 return; 37 } 38 39 ibsrq->event_handler(&event, ibsrq->srq_context); 40 } 41 } 42 43 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 44 struct mlx5_srq_attr *in, 45 struct ib_udata *udata, int buf_size) 46 { 47 struct mlx5_ib_dev *dev = to_mdev(pd->device); 48 struct mlx5_ib_create_srq ucmd = {}; 49 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context( 50 udata, struct mlx5_ib_ucontext, ibucontext); 51 size_t ucmdlen; 52 int err; 53 u32 uidx = MLX5_IB_DEFAULT_UIDX; 54 55 ucmdlen = min(udata->inlen, sizeof(ucmd)); 56 57 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 58 mlx5_ib_dbg(dev, "failed copy udata\n"); 59 return -EFAULT; 60 } 61 62 if (ucmd.reserved0 || ucmd.reserved1) 63 return -EINVAL; 64 65 if (udata->inlen > sizeof(ucmd) && 66 !ib_is_udata_cleared(udata, sizeof(ucmd), 67 udata->inlen - sizeof(ucmd))) 68 return -EINVAL; 69 70 if (in->type != IB_SRQT_BASIC) { 71 err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx); 72 if (err) 73 return err; 74 } 75 76 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 77 78 srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); 79 if (IS_ERR(srq->umem)) { 80 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 81 err = PTR_ERR(srq->umem); 82 return err; 83 } 84 in->umem = srq->umem; 85 86 err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); 87 if (err) { 88 mlx5_ib_dbg(dev, "map doorbell failed\n"); 89 goto err_umem; 90 } 91 92 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 93 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 94 in->type != IB_SRQT_BASIC) 95 in->user_index = uidx; 96 97 return 0; 98 99 err_umem: 100 ib_umem_release(srq->umem); 101 102 return err; 103 } 104 105 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 106 struct mlx5_srq_attr *in, int buf_size) 107 { 108 int err; 109 int i; 110 struct mlx5_wqe_srq_next_seg *next; 111 112 err = mlx5_db_alloc(dev->mdev, &srq->db); 113 if (err) { 114 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 115 return err; 116 } 117 118 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, 119 dev->mdev->priv.numa_node)) { 120 mlx5_ib_dbg(dev, "buf alloc failed\n"); 121 err = -ENOMEM; 122 goto err_db; 123 } 124 125 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), 126 &srq->fbc); 127 128 srq->head = 0; 129 srq->tail = srq->msrq.max - 1; 130 srq->wqe_ctr = 0; 131 132 for (i = 0; i < srq->msrq.max; i++) { 133 next = get_wqe(srq, i); 134 next->next_wqe_index = 135 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 136 } 137 138 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 139 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 140 if (!in->pas) { 141 err = -ENOMEM; 142 goto err_buf; 143 } 144 mlx5_fill_page_frag_array(&srq->buf, in->pas); 145 146 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 147 if (!srq->wrid) { 148 err = -ENOMEM; 149 goto err_in; 150 } 151 srq->wq_sig = 0; 152 153 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 154 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 155 in->type != IB_SRQT_BASIC) 156 in->user_index = MLX5_IB_DEFAULT_UIDX; 157 158 return 0; 159 160 err_in: 161 kvfree(in->pas); 162 163 err_buf: 164 mlx5_frag_buf_free(dev->mdev, &srq->buf); 165 166 err_db: 167 mlx5_db_free(dev->mdev, &srq->db); 168 return err; 169 } 170 171 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 172 struct ib_udata *udata) 173 { 174 mlx5_ib_db_unmap_user( 175 rdma_udata_to_drv_context( 176 udata, 177 struct mlx5_ib_ucontext, 178 ibucontext), 179 &srq->db); 180 ib_umem_release(srq->umem); 181 } 182 183 184 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 185 { 186 kvfree(srq->wrid); 187 mlx5_frag_buf_free(dev->mdev, &srq->buf); 188 mlx5_db_free(dev->mdev, &srq->db); 189 } 190 191 int mlx5_ib_create_srq(struct ib_srq *ib_srq, 192 struct ib_srq_init_attr *init_attr, 193 struct ib_udata *udata) 194 { 195 struct mlx5_ib_dev *dev = to_mdev(ib_srq->device); 196 struct mlx5_ib_srq *srq = to_msrq(ib_srq); 197 size_t desc_size; 198 size_t buf_size; 199 int err; 200 struct mlx5_srq_attr in = {}; 201 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 202 203 if (init_attr->srq_type != IB_SRQT_BASIC && 204 init_attr->srq_type != IB_SRQT_XRC && 205 init_attr->srq_type != IB_SRQT_TM) 206 return -EOPNOTSUPP; 207 208 /* Sanity check SRQ size before proceeding */ 209 if (init_attr->attr.max_wr >= max_srq_wqes) { 210 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 211 init_attr->attr.max_wr, 212 max_srq_wqes); 213 return -EINVAL; 214 } 215 216 mutex_init(&srq->mutex); 217 spin_lock_init(&srq->lock); 218 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 219 srq->msrq.max_gs = init_attr->attr.max_sge; 220 221 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 222 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 223 if (desc_size == 0 || srq->msrq.max_gs > desc_size) 224 return -EINVAL; 225 226 desc_size = roundup_pow_of_two(desc_size); 227 desc_size = max_t(size_t, 32, desc_size); 228 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) 229 return -EINVAL; 230 231 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 232 sizeof(struct mlx5_wqe_data_seg); 233 srq->msrq.wqe_shift = ilog2(desc_size); 234 buf_size = srq->msrq.max * desc_size; 235 if (buf_size < desc_size) 236 return -EINVAL; 237 238 in.type = init_attr->srq_type; 239 240 if (udata) 241 err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size); 242 else 243 err = create_srq_kernel(dev, srq, &in, buf_size); 244 245 if (err) { 246 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 247 udata ? "user" : "kernel", err); 248 return err; 249 } 250 251 in.log_size = ilog2(srq->msrq.max); 252 in.wqe_shift = srq->msrq.wqe_shift - 4; 253 if (srq->wq_sig) 254 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 255 256 if (init_attr->srq_type == IB_SRQT_XRC && init_attr->ext.xrc.xrcd) 257 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 258 else 259 in.xrcd = dev->devr.xrcdn0; 260 261 if (init_attr->srq_type == IB_SRQT_TM) { 262 in.tm_log_list_size = 263 ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 264 if (in.tm_log_list_size > 265 MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 266 mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 267 err = -EINVAL; 268 goto err_usr_kern_srq; 269 } 270 in.flags |= MLX5_SRQ_FLAG_RNDV; 271 } 272 273 if (ib_srq_has_cq(init_attr->srq_type)) 274 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 275 else 276 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 277 278 in.pd = to_mpd(ib_srq->pd)->pdn; 279 in.db_record = srq->db.dma; 280 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 281 kvfree(in.pas); 282 if (err) { 283 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 284 goto err_usr_kern_srq; 285 } 286 287 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 288 289 srq->msrq.event = mlx5_ib_srq_event; 290 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 291 292 if (udata) { 293 struct mlx5_ib_create_srq_resp resp = { 294 .srqn = srq->msrq.srqn, 295 }; 296 297 if (ib_copy_to_udata(udata, &resp, min(udata->outlen, 298 sizeof(resp)))) { 299 mlx5_ib_dbg(dev, "copy to user failed\n"); 300 err = -EFAULT; 301 goto err_core; 302 } 303 } 304 305 init_attr->attr.max_wr = srq->msrq.max - 1; 306 307 return 0; 308 309 err_core: 310 mlx5_cmd_destroy_srq(dev, &srq->msrq); 311 312 err_usr_kern_srq: 313 if (udata) 314 destroy_srq_user(ib_srq->pd, srq, udata); 315 else 316 destroy_srq_kernel(dev, srq); 317 318 return err; 319 } 320 321 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 322 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 323 { 324 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 325 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 326 int ret; 327 328 /* We don't support resizing SRQs yet */ 329 if (attr_mask & IB_SRQ_MAX_WR) 330 return -EINVAL; 331 332 if (attr_mask & IB_SRQ_LIMIT) { 333 if (attr->srq_limit >= srq->msrq.max) 334 return -EINVAL; 335 336 mutex_lock(&srq->mutex); 337 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 338 mutex_unlock(&srq->mutex); 339 340 if (ret) 341 return ret; 342 } 343 344 return 0; 345 } 346 347 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 348 { 349 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 350 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 351 int ret; 352 struct mlx5_srq_attr *out; 353 354 out = kzalloc(sizeof(*out), GFP_KERNEL); 355 if (!out) 356 return -ENOMEM; 357 358 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 359 if (ret) 360 goto out_box; 361 362 srq_attr->srq_limit = out->lwm; 363 srq_attr->max_wr = srq->msrq.max - 1; 364 srq_attr->max_sge = srq->msrq.max_gs; 365 366 out_box: 367 kfree(out); 368 return ret; 369 } 370 371 int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) 372 { 373 struct mlx5_ib_dev *dev = to_mdev(srq->device); 374 struct mlx5_ib_srq *msrq = to_msrq(srq); 375 int ret; 376 377 ret = mlx5_cmd_destroy_srq(dev, &msrq->msrq); 378 if (ret) 379 return ret; 380 381 if (udata) 382 destroy_srq_user(srq->pd, msrq, udata); 383 else 384 destroy_srq_kernel(dev, msrq); 385 return 0; 386 } 387 388 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 389 { 390 struct mlx5_wqe_srq_next_seg *next; 391 392 /* always called with interrupts disabled. */ 393 spin_lock(&srq->lock); 394 395 next = get_wqe(srq, srq->tail); 396 next->next_wqe_index = cpu_to_be16(wqe_index); 397 srq->tail = wqe_index; 398 399 spin_unlock(&srq->lock); 400 } 401 402 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 403 const struct ib_recv_wr **bad_wr) 404 { 405 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 406 struct mlx5_wqe_srq_next_seg *next; 407 struct mlx5_wqe_data_seg *scat; 408 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 409 struct mlx5_core_dev *mdev = dev->mdev; 410 unsigned long flags; 411 int err = 0; 412 int nreq; 413 int i; 414 415 spin_lock_irqsave(&srq->lock, flags); 416 417 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 418 err = -EIO; 419 *bad_wr = wr; 420 goto out; 421 } 422 423 for (nreq = 0; wr; nreq++, wr = wr->next) { 424 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 425 err = -EINVAL; 426 *bad_wr = wr; 427 break; 428 } 429 430 if (unlikely(srq->head == srq->tail)) { 431 err = -ENOMEM; 432 *bad_wr = wr; 433 break; 434 } 435 436 srq->wrid[srq->head] = wr->wr_id; 437 438 next = get_wqe(srq, srq->head); 439 srq->head = be16_to_cpu(next->next_wqe_index); 440 scat = (struct mlx5_wqe_data_seg *)(next + 1); 441 442 for (i = 0; i < wr->num_sge; i++) { 443 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 444 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 445 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 446 } 447 448 if (i < srq->msrq.max_avail_gather) { 449 scat[i].byte_count = 0; 450 scat[i].lkey = dev->mkeys.terminate_scatter_list_mkey; 451 scat[i].addr = 0; 452 } 453 } 454 455 if (likely(nreq)) { 456 srq->wqe_ctr += nreq; 457 458 /* Make sure that descriptors are written before 459 * doorbell record. 460 */ 461 wmb(); 462 463 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 464 } 465 out: 466 spin_unlock_irqrestore(&srq->lock, flags); 467 468 return err; 469 } 470