1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/module.h> 7 #include <linux/mlx5/qp.h> 8 #include <linux/slab.h> 9 #include <rdma/ib_umem.h> 10 #include <rdma/ib_user_verbs.h> 11 #include "mlx5_ib.h" 12 #include "srq.h" 13 14 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 15 { 16 return mlx5_frag_buf_get_wqe(&srq->fbc, n); 17 } 18 19 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 20 { 21 struct ib_event event; 22 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 23 24 if (ibsrq->event_handler) { 25 event.device = ibsrq->device; 26 event.element.srq = ibsrq; 27 switch (type) { 28 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 29 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 30 break; 31 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 32 event.event = IB_EVENT_SRQ_ERR; 33 break; 34 default: 35 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 36 type, srq->srqn); 37 return; 38 } 39 40 ibsrq->event_handler(&event, ibsrq->srq_context); 41 } 42 } 43 44 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 45 struct mlx5_srq_attr *in, 46 struct ib_udata *udata, int buf_size) 47 { 48 struct mlx5_ib_dev *dev = to_mdev(pd->device); 49 struct mlx5_ib_create_srq ucmd = {}; 50 struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context( 51 udata, struct mlx5_ib_ucontext, ibucontext); 52 size_t ucmdlen; 53 int err; 54 int npages; 55 int page_shift; 56 int ncont; 57 u32 offset; 58 u32 uidx = MLX5_IB_DEFAULT_UIDX; 59 60 ucmdlen = min(udata->inlen, sizeof(ucmd)); 61 62 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 63 mlx5_ib_dbg(dev, "failed copy udata\n"); 64 return -EFAULT; 65 } 66 67 if (ucmd.reserved0 || ucmd.reserved1) 68 return -EINVAL; 69 70 if (udata->inlen > sizeof(ucmd) && 71 !ib_is_udata_cleared(udata, sizeof(ucmd), 72 udata->inlen - sizeof(ucmd))) 73 return -EINVAL; 74 75 if (in->type != IB_SRQT_BASIC) { 76 err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx); 77 if (err) 78 return err; 79 } 80 81 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 82 83 srq->umem = ib_umem_get(udata, ucmd.buf_addr, buf_size, 0, 0); 84 if (IS_ERR(srq->umem)) { 85 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 86 err = PTR_ERR(srq->umem); 87 return err; 88 } 89 90 mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, 91 &page_shift, &ncont, NULL); 92 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, 93 &offset); 94 if (err) { 95 mlx5_ib_warn(dev, "bad offset\n"); 96 goto err_umem; 97 } 98 99 in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL); 100 if (!in->pas) { 101 err = -ENOMEM; 102 goto err_umem; 103 } 104 105 mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); 106 107 err = mlx5_ib_db_map_user(ucontext, udata, ucmd.db_addr, &srq->db); 108 if (err) { 109 mlx5_ib_dbg(dev, "map doorbell failed\n"); 110 goto err_in; 111 } 112 113 in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 114 in->page_offset = offset; 115 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 116 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 117 in->type != IB_SRQT_BASIC) 118 in->user_index = uidx; 119 120 return 0; 121 122 err_in: 123 kvfree(in->pas); 124 125 err_umem: 126 ib_umem_release(srq->umem); 127 128 return err; 129 } 130 131 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 132 struct mlx5_srq_attr *in, int buf_size) 133 { 134 int err; 135 int i; 136 struct mlx5_wqe_srq_next_seg *next; 137 138 err = mlx5_db_alloc(dev->mdev, &srq->db); 139 if (err) { 140 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 141 return err; 142 } 143 144 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, 145 dev->mdev->priv.numa_node)) { 146 mlx5_ib_dbg(dev, "buf alloc failed\n"); 147 err = -ENOMEM; 148 goto err_db; 149 } 150 151 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), 152 &srq->fbc); 153 154 srq->head = 0; 155 srq->tail = srq->msrq.max - 1; 156 srq->wqe_ctr = 0; 157 158 for (i = 0; i < srq->msrq.max; i++) { 159 next = get_wqe(srq, i); 160 next->next_wqe_index = 161 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 162 } 163 164 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 165 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 166 if (!in->pas) { 167 err = -ENOMEM; 168 goto err_buf; 169 } 170 mlx5_fill_page_frag_array(&srq->buf, in->pas); 171 172 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 173 if (!srq->wrid) { 174 err = -ENOMEM; 175 goto err_in; 176 } 177 srq->wq_sig = 0; 178 179 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 180 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 181 in->type != IB_SRQT_BASIC) 182 in->user_index = MLX5_IB_DEFAULT_UIDX; 183 184 return 0; 185 186 err_in: 187 kvfree(in->pas); 188 189 err_buf: 190 mlx5_frag_buf_free(dev->mdev, &srq->buf); 191 192 err_db: 193 mlx5_db_free(dev->mdev, &srq->db); 194 return err; 195 } 196 197 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq) 198 { 199 mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); 200 ib_umem_release(srq->umem); 201 } 202 203 204 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 205 { 206 kvfree(srq->wrid); 207 mlx5_frag_buf_free(dev->mdev, &srq->buf); 208 mlx5_db_free(dev->mdev, &srq->db); 209 } 210 211 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, 212 struct ib_srq_init_attr *init_attr, 213 struct ib_udata *udata) 214 { 215 struct mlx5_ib_dev *dev = to_mdev(pd->device); 216 struct mlx5_ib_srq *srq; 217 size_t desc_size; 218 size_t buf_size; 219 int err; 220 struct mlx5_srq_attr in = {0}; 221 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 222 223 /* Sanity check SRQ size before proceeding */ 224 if (init_attr->attr.max_wr >= max_srq_wqes) { 225 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 226 init_attr->attr.max_wr, 227 max_srq_wqes); 228 return ERR_PTR(-EINVAL); 229 } 230 231 srq = kmalloc(sizeof(*srq), GFP_KERNEL); 232 if (!srq) 233 return ERR_PTR(-ENOMEM); 234 235 mutex_init(&srq->mutex); 236 spin_lock_init(&srq->lock); 237 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 238 srq->msrq.max_gs = init_attr->attr.max_sge; 239 240 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 241 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 242 if (desc_size == 0 || srq->msrq.max_gs > desc_size) { 243 err = -EINVAL; 244 goto err_srq; 245 } 246 desc_size = roundup_pow_of_two(desc_size); 247 desc_size = max_t(size_t, 32, desc_size); 248 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) { 249 err = -EINVAL; 250 goto err_srq; 251 } 252 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 253 sizeof(struct mlx5_wqe_data_seg); 254 srq->msrq.wqe_shift = ilog2(desc_size); 255 buf_size = srq->msrq.max * desc_size; 256 if (buf_size < desc_size) { 257 err = -EINVAL; 258 goto err_srq; 259 } 260 in.type = init_attr->srq_type; 261 262 if (udata) 263 err = create_srq_user(pd, srq, &in, udata, buf_size); 264 else 265 err = create_srq_kernel(dev, srq, &in, buf_size); 266 267 if (err) { 268 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 269 udata ? "user" : "kernel", err); 270 goto err_srq; 271 } 272 273 in.log_size = ilog2(srq->msrq.max); 274 in.wqe_shift = srq->msrq.wqe_shift - 4; 275 if (srq->wq_sig) 276 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 277 278 if (init_attr->srq_type == IB_SRQT_XRC) 279 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 280 else 281 in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; 282 283 if (init_attr->srq_type == IB_SRQT_TM) { 284 in.tm_log_list_size = 285 ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 286 if (in.tm_log_list_size > 287 MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 288 mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 289 err = -EINVAL; 290 goto err_usr_kern_srq; 291 } 292 in.flags |= MLX5_SRQ_FLAG_RNDV; 293 } 294 295 if (ib_srq_has_cq(init_attr->srq_type)) 296 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 297 else 298 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 299 300 in.pd = to_mpd(pd)->pdn; 301 in.db_record = srq->db.dma; 302 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 303 kvfree(in.pas); 304 if (err) { 305 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 306 goto err_usr_kern_srq; 307 } 308 309 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 310 311 srq->msrq.event = mlx5_ib_srq_event; 312 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 313 314 if (udata) 315 if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) { 316 mlx5_ib_dbg(dev, "copy to user failed\n"); 317 err = -EFAULT; 318 goto err_core; 319 } 320 321 init_attr->attr.max_wr = srq->msrq.max - 1; 322 323 return &srq->ibsrq; 324 325 err_core: 326 mlx5_cmd_destroy_srq(dev, &srq->msrq); 327 328 err_usr_kern_srq: 329 if (udata) 330 destroy_srq_user(pd, srq); 331 else 332 destroy_srq_kernel(dev, srq); 333 334 err_srq: 335 kfree(srq); 336 337 return ERR_PTR(err); 338 } 339 340 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 341 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 342 { 343 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 344 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 345 int ret; 346 347 /* We don't support resizing SRQs yet */ 348 if (attr_mask & IB_SRQ_MAX_WR) 349 return -EINVAL; 350 351 if (attr_mask & IB_SRQ_LIMIT) { 352 if (attr->srq_limit >= srq->msrq.max) 353 return -EINVAL; 354 355 mutex_lock(&srq->mutex); 356 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 357 mutex_unlock(&srq->mutex); 358 359 if (ret) 360 return ret; 361 } 362 363 return 0; 364 } 365 366 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 367 { 368 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 369 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 370 int ret; 371 struct mlx5_srq_attr *out; 372 373 out = kzalloc(sizeof(*out), GFP_KERNEL); 374 if (!out) 375 return -ENOMEM; 376 377 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 378 if (ret) 379 goto out_box; 380 381 srq_attr->srq_limit = out->lwm; 382 srq_attr->max_wr = srq->msrq.max - 1; 383 srq_attr->max_sge = srq->msrq.max_gs; 384 385 out_box: 386 kfree(out); 387 return ret; 388 } 389 390 int mlx5_ib_destroy_srq(struct ib_srq *srq) 391 { 392 struct mlx5_ib_dev *dev = to_mdev(srq->device); 393 struct mlx5_ib_srq *msrq = to_msrq(srq); 394 395 mlx5_cmd_destroy_srq(dev, &msrq->msrq); 396 397 if (srq->uobject) { 398 mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 399 ib_umem_release(msrq->umem); 400 } else { 401 destroy_srq_kernel(dev, msrq); 402 } 403 404 kfree(srq); 405 return 0; 406 } 407 408 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 409 { 410 struct mlx5_wqe_srq_next_seg *next; 411 412 /* always called with interrupts disabled. */ 413 spin_lock(&srq->lock); 414 415 next = get_wqe(srq, srq->tail); 416 next->next_wqe_index = cpu_to_be16(wqe_index); 417 srq->tail = wqe_index; 418 419 spin_unlock(&srq->lock); 420 } 421 422 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 423 const struct ib_recv_wr **bad_wr) 424 { 425 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 426 struct mlx5_wqe_srq_next_seg *next; 427 struct mlx5_wqe_data_seg *scat; 428 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 429 struct mlx5_core_dev *mdev = dev->mdev; 430 unsigned long flags; 431 int err = 0; 432 int nreq; 433 int i; 434 435 spin_lock_irqsave(&srq->lock, flags); 436 437 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 438 err = -EIO; 439 *bad_wr = wr; 440 goto out; 441 } 442 443 for (nreq = 0; wr; nreq++, wr = wr->next) { 444 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 445 err = -EINVAL; 446 *bad_wr = wr; 447 break; 448 } 449 450 if (unlikely(srq->head == srq->tail)) { 451 err = -ENOMEM; 452 *bad_wr = wr; 453 break; 454 } 455 456 srq->wrid[srq->head] = wr->wr_id; 457 458 next = get_wqe(srq, srq->head); 459 srq->head = be16_to_cpu(next->next_wqe_index); 460 scat = (struct mlx5_wqe_data_seg *)(next + 1); 461 462 for (i = 0; i < wr->num_sge; i++) { 463 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 464 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 465 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 466 } 467 468 if (i < srq->msrq.max_avail_gather) { 469 scat[i].byte_count = 0; 470 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 471 scat[i].addr = 0; 472 } 473 } 474 475 if (likely(nreq)) { 476 srq->wqe_ctr += nreq; 477 478 /* Make sure that descriptors are written before 479 * doorbell record. 480 */ 481 wmb(); 482 483 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 484 } 485 out: 486 spin_unlock_irqrestore(&srq->lock, flags); 487 488 return err; 489 } 490