1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/module.h> 7 #include <linux/mlx5/qp.h> 8 #include <linux/slab.h> 9 #include <rdma/ib_umem.h> 10 #include <rdma/ib_user_verbs.h> 11 #include "mlx5_ib.h" 12 #include "srq.h" 13 14 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 15 { 16 return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift); 17 } 18 19 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 20 { 21 struct ib_event event; 22 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 23 24 if (ibsrq->event_handler) { 25 event.device = ibsrq->device; 26 event.element.srq = ibsrq; 27 switch (type) { 28 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 29 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 30 break; 31 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 32 event.event = IB_EVENT_SRQ_ERR; 33 break; 34 default: 35 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 36 type, srq->srqn); 37 return; 38 } 39 40 ibsrq->event_handler(&event, ibsrq->srq_context); 41 } 42 } 43 44 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 45 struct mlx5_srq_attr *in, 46 struct ib_udata *udata, int buf_size) 47 { 48 struct mlx5_ib_dev *dev = to_mdev(pd->device); 49 struct mlx5_ib_create_srq ucmd = {}; 50 size_t ucmdlen; 51 int err; 52 int npages; 53 int page_shift; 54 int ncont; 55 u32 offset; 56 u32 uidx = MLX5_IB_DEFAULT_UIDX; 57 58 ucmdlen = min(udata->inlen, sizeof(ucmd)); 59 60 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 61 mlx5_ib_dbg(dev, "failed copy udata\n"); 62 return -EFAULT; 63 } 64 65 if (ucmd.reserved0 || ucmd.reserved1) 66 return -EINVAL; 67 68 if (udata->inlen > sizeof(ucmd) && 69 !ib_is_udata_cleared(udata, sizeof(ucmd), 70 udata->inlen - sizeof(ucmd))) 71 return -EINVAL; 72 73 if (in->type != IB_SRQT_BASIC) { 74 err = get_srq_user_index(to_mucontext(pd->uobject->context), 75 &ucmd, udata->inlen, &uidx); 76 if (err) 77 return err; 78 } 79 80 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 81 82 srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 83 0, 0); 84 if (IS_ERR(srq->umem)) { 85 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 86 err = PTR_ERR(srq->umem); 87 return err; 88 } 89 90 mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, 91 &page_shift, &ncont, NULL); 92 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, 93 &offset); 94 if (err) { 95 mlx5_ib_warn(dev, "bad offset\n"); 96 goto err_umem; 97 } 98 99 in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL); 100 if (!in->pas) { 101 err = -ENOMEM; 102 goto err_umem; 103 } 104 105 mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); 106 107 err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context), 108 ucmd.db_addr, &srq->db); 109 if (err) { 110 mlx5_ib_dbg(dev, "map doorbell failed\n"); 111 goto err_in; 112 } 113 114 in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 115 in->page_offset = offset; 116 in->uid = to_mpd(pd)->uid; 117 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 118 in->type != IB_SRQT_BASIC) 119 in->user_index = uidx; 120 121 return 0; 122 123 err_in: 124 kvfree(in->pas); 125 126 err_umem: 127 ib_umem_release(srq->umem); 128 129 return err; 130 } 131 132 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 133 struct mlx5_srq_attr *in, int buf_size) 134 { 135 int err; 136 int i; 137 struct mlx5_wqe_srq_next_seg *next; 138 139 err = mlx5_db_alloc(dev->mdev, &srq->db); 140 if (err) { 141 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 142 return err; 143 } 144 145 if (mlx5_buf_alloc(dev->mdev, buf_size, &srq->buf)) { 146 mlx5_ib_dbg(dev, "buf alloc failed\n"); 147 err = -ENOMEM; 148 goto err_db; 149 } 150 151 srq->head = 0; 152 srq->tail = srq->msrq.max - 1; 153 srq->wqe_ctr = 0; 154 155 for (i = 0; i < srq->msrq.max; i++) { 156 next = get_wqe(srq, i); 157 next->next_wqe_index = 158 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 159 } 160 161 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 162 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 163 if (!in->pas) { 164 err = -ENOMEM; 165 goto err_buf; 166 } 167 mlx5_fill_page_array(&srq->buf, in->pas); 168 169 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 170 if (!srq->wrid) { 171 err = -ENOMEM; 172 goto err_in; 173 } 174 srq->wq_sig = 0; 175 176 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 177 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 178 in->type != IB_SRQT_BASIC) 179 in->user_index = MLX5_IB_DEFAULT_UIDX; 180 181 return 0; 182 183 err_in: 184 kvfree(in->pas); 185 186 err_buf: 187 mlx5_buf_free(dev->mdev, &srq->buf); 188 189 err_db: 190 mlx5_db_free(dev->mdev, &srq->db); 191 return err; 192 } 193 194 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq) 195 { 196 mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); 197 ib_umem_release(srq->umem); 198 } 199 200 201 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 202 { 203 kvfree(srq->wrid); 204 mlx5_buf_free(dev->mdev, &srq->buf); 205 mlx5_db_free(dev->mdev, &srq->db); 206 } 207 208 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, 209 struct ib_srq_init_attr *init_attr, 210 struct ib_udata *udata) 211 { 212 struct mlx5_ib_dev *dev = to_mdev(pd->device); 213 struct mlx5_ib_srq *srq; 214 size_t desc_size; 215 size_t buf_size; 216 int err; 217 struct mlx5_srq_attr in = {0}; 218 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 219 220 /* Sanity check SRQ size before proceeding */ 221 if (init_attr->attr.max_wr >= max_srq_wqes) { 222 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 223 init_attr->attr.max_wr, 224 max_srq_wqes); 225 return ERR_PTR(-EINVAL); 226 } 227 228 srq = kmalloc(sizeof(*srq), GFP_KERNEL); 229 if (!srq) 230 return ERR_PTR(-ENOMEM); 231 232 mutex_init(&srq->mutex); 233 spin_lock_init(&srq->lock); 234 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 235 srq->msrq.max_gs = init_attr->attr.max_sge; 236 237 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 238 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 239 if (desc_size == 0 || srq->msrq.max_gs > desc_size) { 240 err = -EINVAL; 241 goto err_srq; 242 } 243 desc_size = roundup_pow_of_two(desc_size); 244 desc_size = max_t(size_t, 32, desc_size); 245 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) { 246 err = -EINVAL; 247 goto err_srq; 248 } 249 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 250 sizeof(struct mlx5_wqe_data_seg); 251 srq->msrq.wqe_shift = ilog2(desc_size); 252 buf_size = srq->msrq.max * desc_size; 253 if (buf_size < desc_size) { 254 err = -EINVAL; 255 goto err_srq; 256 } 257 in.type = init_attr->srq_type; 258 259 if (pd->uobject) 260 err = create_srq_user(pd, srq, &in, udata, buf_size); 261 else 262 err = create_srq_kernel(dev, srq, &in, buf_size); 263 264 if (err) { 265 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 266 pd->uobject ? "user" : "kernel", err); 267 goto err_srq; 268 } 269 270 in.log_size = ilog2(srq->msrq.max); 271 in.wqe_shift = srq->msrq.wqe_shift - 4; 272 if (srq->wq_sig) 273 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 274 275 if (init_attr->srq_type == IB_SRQT_XRC) 276 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 277 else 278 in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; 279 280 if (init_attr->srq_type == IB_SRQT_TM) { 281 in.tm_log_list_size = 282 ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 283 if (in.tm_log_list_size > 284 MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 285 mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 286 err = -EINVAL; 287 goto err_usr_kern_srq; 288 } 289 in.flags |= MLX5_SRQ_FLAG_RNDV; 290 } 291 292 if (ib_srq_has_cq(init_attr->srq_type)) 293 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 294 else 295 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 296 297 in.pd = to_mpd(pd)->pdn; 298 in.db_record = srq->db.dma; 299 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 300 kvfree(in.pas); 301 if (err) { 302 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 303 goto err_usr_kern_srq; 304 } 305 306 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 307 308 srq->msrq.event = mlx5_ib_srq_event; 309 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 310 311 if (pd->uobject) 312 if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) { 313 mlx5_ib_dbg(dev, "copy to user failed\n"); 314 err = -EFAULT; 315 goto err_core; 316 } 317 318 init_attr->attr.max_wr = srq->msrq.max - 1; 319 320 return &srq->ibsrq; 321 322 err_core: 323 mlx5_cmd_destroy_srq(dev, &srq->msrq); 324 325 err_usr_kern_srq: 326 if (pd->uobject) 327 destroy_srq_user(pd, srq); 328 else 329 destroy_srq_kernel(dev, srq); 330 331 err_srq: 332 kfree(srq); 333 334 return ERR_PTR(err); 335 } 336 337 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 338 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 339 { 340 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 341 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 342 int ret; 343 344 /* We don't support resizing SRQs yet */ 345 if (attr_mask & IB_SRQ_MAX_WR) 346 return -EINVAL; 347 348 if (attr_mask & IB_SRQ_LIMIT) { 349 if (attr->srq_limit >= srq->msrq.max) 350 return -EINVAL; 351 352 mutex_lock(&srq->mutex); 353 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 354 mutex_unlock(&srq->mutex); 355 356 if (ret) 357 return ret; 358 } 359 360 return 0; 361 } 362 363 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 364 { 365 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 366 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 367 int ret; 368 struct mlx5_srq_attr *out; 369 370 out = kzalloc(sizeof(*out), GFP_KERNEL); 371 if (!out) 372 return -ENOMEM; 373 374 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 375 if (ret) 376 goto out_box; 377 378 srq_attr->srq_limit = out->lwm; 379 srq_attr->max_wr = srq->msrq.max - 1; 380 srq_attr->max_sge = srq->msrq.max_gs; 381 382 out_box: 383 kfree(out); 384 return ret; 385 } 386 387 int mlx5_ib_destroy_srq(struct ib_srq *srq) 388 { 389 struct mlx5_ib_dev *dev = to_mdev(srq->device); 390 struct mlx5_ib_srq *msrq = to_msrq(srq); 391 392 mlx5_cmd_destroy_srq(dev, &msrq->msrq); 393 394 if (srq->uobject) { 395 mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 396 ib_umem_release(msrq->umem); 397 } else { 398 destroy_srq_kernel(dev, msrq); 399 } 400 401 kfree(srq); 402 return 0; 403 } 404 405 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 406 { 407 struct mlx5_wqe_srq_next_seg *next; 408 409 /* always called with interrupts disabled. */ 410 spin_lock(&srq->lock); 411 412 next = get_wqe(srq, srq->tail); 413 next->next_wqe_index = cpu_to_be16(wqe_index); 414 srq->tail = wqe_index; 415 416 spin_unlock(&srq->lock); 417 } 418 419 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 420 const struct ib_recv_wr **bad_wr) 421 { 422 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 423 struct mlx5_wqe_srq_next_seg *next; 424 struct mlx5_wqe_data_seg *scat; 425 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 426 struct mlx5_core_dev *mdev = dev->mdev; 427 unsigned long flags; 428 int err = 0; 429 int nreq; 430 int i; 431 432 spin_lock_irqsave(&srq->lock, flags); 433 434 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 435 err = -EIO; 436 *bad_wr = wr; 437 goto out; 438 } 439 440 for (nreq = 0; wr; nreq++, wr = wr->next) { 441 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 442 err = -EINVAL; 443 *bad_wr = wr; 444 break; 445 } 446 447 if (unlikely(srq->head == srq->tail)) { 448 err = -ENOMEM; 449 *bad_wr = wr; 450 break; 451 } 452 453 srq->wrid[srq->head] = wr->wr_id; 454 455 next = get_wqe(srq, srq->head); 456 srq->head = be16_to_cpu(next->next_wqe_index); 457 scat = (struct mlx5_wqe_data_seg *)(next + 1); 458 459 for (i = 0; i < wr->num_sge; i++) { 460 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 461 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 462 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 463 } 464 465 if (i < srq->msrq.max_avail_gather) { 466 scat[i].byte_count = 0; 467 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 468 scat[i].addr = 0; 469 } 470 } 471 472 if (likely(nreq)) { 473 srq->wqe_ctr += nreq; 474 475 /* Make sure that descriptors are written before 476 * doorbell record. 477 */ 478 wmb(); 479 480 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 481 } 482 out: 483 spin_unlock_irqrestore(&srq->lock, flags); 484 485 return err; 486 } 487