1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include <linux/module.h> 7 #include <linux/mlx5/qp.h> 8 #include <linux/slab.h> 9 #include <rdma/ib_umem.h> 10 #include <rdma/ib_user_verbs.h> 11 #include "mlx5_ib.h" 12 #include "srq.h" 13 14 static void *get_wqe(struct mlx5_ib_srq *srq, int n) 15 { 16 return mlx5_frag_buf_get_wqe(&srq->fbc, n); 17 } 18 19 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) 20 { 21 struct ib_event event; 22 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; 23 24 if (ibsrq->event_handler) { 25 event.device = ibsrq->device; 26 event.element.srq = ibsrq; 27 switch (type) { 28 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: 29 event.event = IB_EVENT_SRQ_LIMIT_REACHED; 30 break; 31 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: 32 event.event = IB_EVENT_SRQ_ERR; 33 break; 34 default: 35 pr_warn("mlx5_ib: Unexpected event type %d on SRQ %06x\n", 36 type, srq->srqn); 37 return; 38 } 39 40 ibsrq->event_handler(&event, ibsrq->srq_context); 41 } 42 } 43 44 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, 45 struct mlx5_srq_attr *in, 46 struct ib_udata *udata, int buf_size) 47 { 48 struct mlx5_ib_dev *dev = to_mdev(pd->device); 49 struct mlx5_ib_create_srq ucmd = {}; 50 size_t ucmdlen; 51 int err; 52 int npages; 53 int page_shift; 54 int ncont; 55 u32 offset; 56 u32 uidx = MLX5_IB_DEFAULT_UIDX; 57 58 ucmdlen = min(udata->inlen, sizeof(ucmd)); 59 60 if (ib_copy_from_udata(&ucmd, udata, ucmdlen)) { 61 mlx5_ib_dbg(dev, "failed copy udata\n"); 62 return -EFAULT; 63 } 64 65 if (ucmd.reserved0 || ucmd.reserved1) 66 return -EINVAL; 67 68 if (udata->inlen > sizeof(ucmd) && 69 !ib_is_udata_cleared(udata, sizeof(ucmd), 70 udata->inlen - sizeof(ucmd))) 71 return -EINVAL; 72 73 if (in->type != IB_SRQT_BASIC) { 74 err = get_srq_user_index(to_mucontext(pd->uobject->context), 75 &ucmd, udata->inlen, &uidx); 76 if (err) 77 return err; 78 } 79 80 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); 81 82 srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size, 83 0, 0); 84 if (IS_ERR(srq->umem)) { 85 mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size); 86 err = PTR_ERR(srq->umem); 87 return err; 88 } 89 90 mlx5_ib_cont_pages(srq->umem, ucmd.buf_addr, 0, &npages, 91 &page_shift, &ncont, NULL); 92 err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, 93 &offset); 94 if (err) { 95 mlx5_ib_warn(dev, "bad offset\n"); 96 goto err_umem; 97 } 98 99 in->pas = kvcalloc(ncont, sizeof(*in->pas), GFP_KERNEL); 100 if (!in->pas) { 101 err = -ENOMEM; 102 goto err_umem; 103 } 104 105 mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0); 106 107 err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context), 108 ucmd.db_addr, &srq->db); 109 if (err) { 110 mlx5_ib_dbg(dev, "map doorbell failed\n"); 111 goto err_in; 112 } 113 114 in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT; 115 in->page_offset = offset; 116 in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0; 117 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 118 in->type != IB_SRQT_BASIC) 119 in->user_index = uidx; 120 121 return 0; 122 123 err_in: 124 kvfree(in->pas); 125 126 err_umem: 127 ib_umem_release(srq->umem); 128 129 return err; 130 } 131 132 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, 133 struct mlx5_srq_attr *in, int buf_size) 134 { 135 int err; 136 int i; 137 struct mlx5_wqe_srq_next_seg *next; 138 139 err = mlx5_db_alloc(dev->mdev, &srq->db); 140 if (err) { 141 mlx5_ib_warn(dev, "alloc dbell rec failed\n"); 142 return err; 143 } 144 145 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, 146 dev->mdev->priv.numa_node)) { 147 mlx5_ib_dbg(dev, "buf alloc failed\n"); 148 err = -ENOMEM; 149 goto err_db; 150 } 151 152 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), 153 &srq->fbc); 154 155 srq->head = 0; 156 srq->tail = srq->msrq.max - 1; 157 srq->wqe_ctr = 0; 158 159 for (i = 0; i < srq->msrq.max; i++) { 160 next = get_wqe(srq, i); 161 next->next_wqe_index = 162 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); 163 } 164 165 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); 166 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); 167 if (!in->pas) { 168 err = -ENOMEM; 169 goto err_buf; 170 } 171 mlx5_fill_page_frag_array(&srq->buf, in->pas); 172 173 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); 174 if (!srq->wrid) { 175 err = -ENOMEM; 176 goto err_in; 177 } 178 srq->wq_sig = 0; 179 180 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; 181 if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 && 182 in->type != IB_SRQT_BASIC) 183 in->user_index = MLX5_IB_DEFAULT_UIDX; 184 185 return 0; 186 187 err_in: 188 kvfree(in->pas); 189 190 err_buf: 191 mlx5_frag_buf_free(dev->mdev, &srq->buf); 192 193 err_db: 194 mlx5_db_free(dev->mdev, &srq->db); 195 return err; 196 } 197 198 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq) 199 { 200 mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db); 201 ib_umem_release(srq->umem); 202 } 203 204 205 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) 206 { 207 kvfree(srq->wrid); 208 mlx5_frag_buf_free(dev->mdev, &srq->buf); 209 mlx5_db_free(dev->mdev, &srq->db); 210 } 211 212 struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd, 213 struct ib_srq_init_attr *init_attr, 214 struct ib_udata *udata) 215 { 216 struct mlx5_ib_dev *dev = to_mdev(pd->device); 217 struct mlx5_ib_srq *srq; 218 size_t desc_size; 219 size_t buf_size; 220 int err; 221 struct mlx5_srq_attr in = {0}; 222 __u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz); 223 224 /* Sanity check SRQ size before proceeding */ 225 if (init_attr->attr.max_wr >= max_srq_wqes) { 226 mlx5_ib_dbg(dev, "max_wr %d, cap %d\n", 227 init_attr->attr.max_wr, 228 max_srq_wqes); 229 return ERR_PTR(-EINVAL); 230 } 231 232 srq = kmalloc(sizeof(*srq), GFP_KERNEL); 233 if (!srq) 234 return ERR_PTR(-ENOMEM); 235 236 mutex_init(&srq->mutex); 237 spin_lock_init(&srq->lock); 238 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); 239 srq->msrq.max_gs = init_attr->attr.max_sge; 240 241 desc_size = sizeof(struct mlx5_wqe_srq_next_seg) + 242 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); 243 if (desc_size == 0 || srq->msrq.max_gs > desc_size) { 244 err = -EINVAL; 245 goto err_srq; 246 } 247 desc_size = roundup_pow_of_two(desc_size); 248 desc_size = max_t(size_t, 32, desc_size); 249 if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) { 250 err = -EINVAL; 251 goto err_srq; 252 } 253 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / 254 sizeof(struct mlx5_wqe_data_seg); 255 srq->msrq.wqe_shift = ilog2(desc_size); 256 buf_size = srq->msrq.max * desc_size; 257 if (buf_size < desc_size) { 258 err = -EINVAL; 259 goto err_srq; 260 } 261 in.type = init_attr->srq_type; 262 263 if (udata) 264 err = create_srq_user(pd, srq, &in, udata, buf_size); 265 else 266 err = create_srq_kernel(dev, srq, &in, buf_size); 267 268 if (err) { 269 mlx5_ib_warn(dev, "create srq %s failed, err %d\n", 270 udata ? "user" : "kernel", err); 271 goto err_srq; 272 } 273 274 in.log_size = ilog2(srq->msrq.max); 275 in.wqe_shift = srq->msrq.wqe_shift - 4; 276 if (srq->wq_sig) 277 in.flags |= MLX5_SRQ_FLAG_WQ_SIG; 278 279 if (init_attr->srq_type == IB_SRQT_XRC) 280 in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn; 281 else 282 in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn; 283 284 if (init_attr->srq_type == IB_SRQT_TM) { 285 in.tm_log_list_size = 286 ilog2(init_attr->ext.tag_matching.max_num_tags) + 1; 287 if (in.tm_log_list_size > 288 MLX5_CAP_GEN(dev->mdev, log_tag_matching_list_sz)) { 289 mlx5_ib_dbg(dev, "TM SRQ max_num_tags exceeding limit\n"); 290 err = -EINVAL; 291 goto err_usr_kern_srq; 292 } 293 in.flags |= MLX5_SRQ_FLAG_RNDV; 294 } 295 296 if (ib_srq_has_cq(init_attr->srq_type)) 297 in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn; 298 else 299 in.cqn = to_mcq(dev->devr.c0)->mcq.cqn; 300 301 in.pd = to_mpd(pd)->pdn; 302 in.db_record = srq->db.dma; 303 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); 304 kvfree(in.pas); 305 if (err) { 306 mlx5_ib_dbg(dev, "create SRQ failed, err %d\n", err); 307 goto err_usr_kern_srq; 308 } 309 310 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); 311 312 srq->msrq.event = mlx5_ib_srq_event; 313 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; 314 315 if (udata) 316 if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) { 317 mlx5_ib_dbg(dev, "copy to user failed\n"); 318 err = -EFAULT; 319 goto err_core; 320 } 321 322 init_attr->attr.max_wr = srq->msrq.max - 1; 323 324 return &srq->ibsrq; 325 326 err_core: 327 mlx5_cmd_destroy_srq(dev, &srq->msrq); 328 329 err_usr_kern_srq: 330 if (udata) 331 destroy_srq_user(pd, srq); 332 else 333 destroy_srq_kernel(dev, srq); 334 335 err_srq: 336 kfree(srq); 337 338 return ERR_PTR(err); 339 } 340 341 int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 342 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 343 { 344 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 345 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 346 int ret; 347 348 /* We don't support resizing SRQs yet */ 349 if (attr_mask & IB_SRQ_MAX_WR) 350 return -EINVAL; 351 352 if (attr_mask & IB_SRQ_LIMIT) { 353 if (attr->srq_limit >= srq->msrq.max) 354 return -EINVAL; 355 356 mutex_lock(&srq->mutex); 357 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); 358 mutex_unlock(&srq->mutex); 359 360 if (ret) 361 return ret; 362 } 363 364 return 0; 365 } 366 367 int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 368 { 369 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 370 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 371 int ret; 372 struct mlx5_srq_attr *out; 373 374 out = kzalloc(sizeof(*out), GFP_KERNEL); 375 if (!out) 376 return -ENOMEM; 377 378 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); 379 if (ret) 380 goto out_box; 381 382 srq_attr->srq_limit = out->lwm; 383 srq_attr->max_wr = srq->msrq.max - 1; 384 srq_attr->max_sge = srq->msrq.max_gs; 385 386 out_box: 387 kfree(out); 388 return ret; 389 } 390 391 int mlx5_ib_destroy_srq(struct ib_srq *srq) 392 { 393 struct mlx5_ib_dev *dev = to_mdev(srq->device); 394 struct mlx5_ib_srq *msrq = to_msrq(srq); 395 396 mlx5_cmd_destroy_srq(dev, &msrq->msrq); 397 398 if (srq->uobject) { 399 mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db); 400 ib_umem_release(msrq->umem); 401 } else { 402 destroy_srq_kernel(dev, msrq); 403 } 404 405 kfree(srq); 406 return 0; 407 } 408 409 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) 410 { 411 struct mlx5_wqe_srq_next_seg *next; 412 413 /* always called with interrupts disabled. */ 414 spin_lock(&srq->lock); 415 416 next = get_wqe(srq, srq->tail); 417 next->next_wqe_index = cpu_to_be16(wqe_index); 418 srq->tail = wqe_index; 419 420 spin_unlock(&srq->lock); 421 } 422 423 int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 424 const struct ib_recv_wr **bad_wr) 425 { 426 struct mlx5_ib_srq *srq = to_msrq(ibsrq); 427 struct mlx5_wqe_srq_next_seg *next; 428 struct mlx5_wqe_data_seg *scat; 429 struct mlx5_ib_dev *dev = to_mdev(ibsrq->device); 430 struct mlx5_core_dev *mdev = dev->mdev; 431 unsigned long flags; 432 int err = 0; 433 int nreq; 434 int i; 435 436 spin_lock_irqsave(&srq->lock, flags); 437 438 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 439 err = -EIO; 440 *bad_wr = wr; 441 goto out; 442 } 443 444 for (nreq = 0; wr; nreq++, wr = wr->next) { 445 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { 446 err = -EINVAL; 447 *bad_wr = wr; 448 break; 449 } 450 451 if (unlikely(srq->head == srq->tail)) { 452 err = -ENOMEM; 453 *bad_wr = wr; 454 break; 455 } 456 457 srq->wrid[srq->head] = wr->wr_id; 458 459 next = get_wqe(srq, srq->head); 460 srq->head = be16_to_cpu(next->next_wqe_index); 461 scat = (struct mlx5_wqe_data_seg *)(next + 1); 462 463 for (i = 0; i < wr->num_sge; i++) { 464 scat[i].byte_count = cpu_to_be32(wr->sg_list[i].length); 465 scat[i].lkey = cpu_to_be32(wr->sg_list[i].lkey); 466 scat[i].addr = cpu_to_be64(wr->sg_list[i].addr); 467 } 468 469 if (i < srq->msrq.max_avail_gather) { 470 scat[i].byte_count = 0; 471 scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 472 scat[i].addr = 0; 473 } 474 } 475 476 if (likely(nreq)) { 477 srq->wqe_ctr += nreq; 478 479 /* Make sure that descriptors are written before 480 * doorbell record. 481 */ 482 wmb(); 483 484 *srq->db.db = cpu_to_be32(srq->wqe_ctr); 485 } 486 out: 487 spin_unlock_irqrestore(&srq->lock, flags); 488 489 return err; 490 } 491