1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 138 139 attr->max_send_sge = qattr->max_sge; 140 attr->max_recv_sge = qattr->max_sge; 141 attr->max_sge_rd = qattr->max_sge; 142 attr->max_cq = qattr->max_cq; 143 attr->max_cqe = qattr->max_cqe; 144 attr->max_mr = qattr->max_mr; 145 attr->max_mw = qattr->max_mw; 146 attr->max_pd = qattr->max_pd; 147 attr->atomic_cap = dev->atomic_cap; 148 attr->max_fmr = qattr->max_fmr; 149 attr->max_map_per_fmr = 16; 150 attr->max_qp_init_rd_atom = 151 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 152 attr->max_qp_rd_atom = 153 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 154 attr->max_qp_init_rd_atom); 155 156 attr->max_srq = qattr->max_srq; 157 attr->max_srq_sge = qattr->max_srq_sge; 158 attr->max_srq_wr = qattr->max_srq_wr; 159 160 attr->local_ca_ack_delay = qattr->dev_ack_delay; 161 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 162 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 163 attr->max_ah = qattr->max_ah; 164 165 return 0; 166 } 167 168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 169 u8 *ib_width) 170 { 171 switch (speed) { 172 case 1000: 173 *ib_speed = IB_SPEED_SDR; 174 *ib_width = IB_WIDTH_1X; 175 break; 176 case 10000: 177 *ib_speed = IB_SPEED_QDR; 178 *ib_width = IB_WIDTH_1X; 179 break; 180 181 case 20000: 182 *ib_speed = IB_SPEED_DDR; 183 *ib_width = IB_WIDTH_4X; 184 break; 185 186 case 25000: 187 *ib_speed = IB_SPEED_EDR; 188 *ib_width = IB_WIDTH_1X; 189 break; 190 191 case 40000: 192 *ib_speed = IB_SPEED_QDR; 193 *ib_width = IB_WIDTH_4X; 194 break; 195 196 case 50000: 197 *ib_speed = IB_SPEED_HDR; 198 *ib_width = IB_WIDTH_1X; 199 break; 200 201 case 100000: 202 *ib_speed = IB_SPEED_EDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 default: 207 /* Unsupported */ 208 *ib_speed = IB_SPEED_SDR; 209 *ib_width = IB_WIDTH_1X; 210 } 211 } 212 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 214 { 215 struct qedr_dev *dev; 216 struct qed_rdma_port *rdma_port; 217 218 dev = get_qedr_dev(ibdev); 219 220 if (!dev->rdma_ctx) { 221 DP_ERR(dev, "rdma_ctx is NULL\n"); 222 return -EINVAL; 223 } 224 225 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 226 227 /* *attr being zeroed by the caller, avoid zeroing it here */ 228 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 229 attr->state = IB_PORT_ACTIVE; 230 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 231 } else { 232 attr->state = IB_PORT_DOWN; 233 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 234 } 235 attr->max_mtu = IB_MTU_4096; 236 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 237 attr->lid = 0; 238 attr->lmc = 0; 239 attr->sm_lid = 0; 240 attr->sm_sl = 0; 241 attr->ip_gids = true; 242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 243 attr->gid_tbl_len = 1; 244 attr->pkey_tbl_len = 1; 245 } else { 246 attr->gid_tbl_len = QEDR_MAX_SGID; 247 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 248 } 249 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 250 attr->qkey_viol_cntr = 0; 251 get_link_speed_and_width(rdma_port->link_speed, 252 &attr->active_speed, &attr->active_width); 253 attr->max_msg_sz = rdma_port->max_msg_size; 254 attr->max_vl_num = 4; 255 256 return 0; 257 } 258 259 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 260 { 261 struct ib_device *ibdev = uctx->device; 262 int rc; 263 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 264 struct qedr_alloc_ucontext_resp uresp = {}; 265 struct qedr_alloc_ucontext_req ureq = {}; 266 struct qedr_dev *dev = get_qedr_dev(ibdev); 267 struct qed_rdma_add_user_out_params oparams; 268 struct qedr_user_mmap_entry *entry; 269 270 if (!udata) 271 return -EFAULT; 272 273 if (udata->inlen) { 274 rc = ib_copy_from_udata(&ureq, udata, 275 min(sizeof(ureq), udata->inlen)); 276 if (rc) { 277 DP_ERR(dev, "Problem copying data from user space\n"); 278 return -EFAULT; 279 } 280 281 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 282 } 283 284 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 285 if (rc) { 286 DP_ERR(dev, 287 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 288 rc); 289 return rc; 290 } 291 292 ctx->dpi = oparams.dpi; 293 ctx->dpi_addr = oparams.dpi_addr; 294 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 295 ctx->dpi_size = oparams.dpi_size; 296 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 297 if (!entry) { 298 rc = -ENOMEM; 299 goto err; 300 } 301 302 entry->io_address = ctx->dpi_phys_addr; 303 entry->length = ctx->dpi_size; 304 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 305 entry->dpi = ctx->dpi; 306 entry->dev = dev; 307 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 308 ctx->dpi_size); 309 if (rc) { 310 kfree(entry); 311 goto err; 312 } 313 ctx->db_mmap_entry = &entry->rdma_entry; 314 315 if (!dev->user_dpm_enabled) 316 uresp.dpm_flags = 0; 317 else if (rdma_protocol_iwarp(&dev->ibdev, 1)) 318 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY; 319 else 320 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED | 321 QEDR_DPM_TYPE_ROCE_LEGACY; 322 323 uresp.dpm_flags |= QEDR_DPM_SIZES_SET; 324 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE; 325 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE; 326 327 uresp.wids_enabled = 1; 328 uresp.wid_count = oparams.wid_count; 329 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 330 uresp.db_size = ctx->dpi_size; 331 uresp.max_send_wr = dev->attr.max_sqe; 332 uresp.max_recv_wr = dev->attr.max_rqe; 333 uresp.max_srq_wr = dev->attr.max_srq_wr; 334 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 335 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 336 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 337 uresp.max_cqes = QEDR_MAX_CQES; 338 339 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 340 if (rc) 341 goto err; 342 343 ctx->dev = dev; 344 345 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 346 &ctx->ibucontext); 347 return 0; 348 349 err: 350 if (!ctx->db_mmap_entry) 351 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 352 else 353 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 354 355 return rc; 356 } 357 358 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 359 { 360 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 361 362 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 363 uctx); 364 365 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 366 } 367 368 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 369 { 370 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 371 struct qedr_dev *dev = entry->dev; 372 373 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 374 free_page((unsigned long)entry->address); 375 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 376 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 377 378 kfree(entry); 379 } 380 381 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 382 { 383 struct ib_device *dev = ucontext->device; 384 size_t length = vma->vm_end - vma->vm_start; 385 struct rdma_user_mmap_entry *rdma_entry; 386 struct qedr_user_mmap_entry *entry; 387 int rc = 0; 388 u64 pfn; 389 390 ibdev_dbg(dev, 391 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 392 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 393 394 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 395 if (!rdma_entry) { 396 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 397 vma->vm_pgoff); 398 return -EINVAL; 399 } 400 entry = get_qedr_mmap_entry(rdma_entry); 401 ibdev_dbg(dev, 402 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 403 entry->io_address, length, entry->mmap_flag); 404 405 switch (entry->mmap_flag) { 406 case QEDR_USER_MMAP_IO_WC: 407 pfn = entry->io_address >> PAGE_SHIFT; 408 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 409 pgprot_writecombine(vma->vm_page_prot), 410 rdma_entry); 411 break; 412 case QEDR_USER_MMAP_PHYS_PAGE: 413 rc = vm_insert_page(vma, vma->vm_start, 414 virt_to_page(entry->address)); 415 break; 416 default: 417 rc = -EINVAL; 418 } 419 420 if (rc) 421 ibdev_dbg(dev, 422 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 423 entry->io_address, length, entry->mmap_flag, rc); 424 425 rdma_user_mmap_entry_put(rdma_entry); 426 return rc; 427 } 428 429 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 430 { 431 struct ib_device *ibdev = ibpd->device; 432 struct qedr_dev *dev = get_qedr_dev(ibdev); 433 struct qedr_pd *pd = get_qedr_pd(ibpd); 434 u16 pd_id; 435 int rc; 436 437 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 438 udata ? "User Lib" : "Kernel"); 439 440 if (!dev->rdma_ctx) { 441 DP_ERR(dev, "invalid RDMA context\n"); 442 return -EINVAL; 443 } 444 445 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 446 if (rc) 447 return rc; 448 449 pd->pd_id = pd_id; 450 451 if (udata) { 452 struct qedr_alloc_pd_uresp uresp = { 453 .pd_id = pd_id, 454 }; 455 struct qedr_ucontext *context = rdma_udata_to_drv_context( 456 udata, struct qedr_ucontext, ibucontext); 457 458 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 459 if (rc) { 460 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 461 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 462 return rc; 463 } 464 465 pd->uctx = context; 466 pd->uctx->pd = pd; 467 } 468 469 return 0; 470 } 471 472 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 473 { 474 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 475 struct qedr_pd *pd = get_qedr_pd(ibpd); 476 477 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 478 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 479 } 480 481 static void qedr_free_pbl(struct qedr_dev *dev, 482 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 483 { 484 struct pci_dev *pdev = dev->pdev; 485 int i; 486 487 for (i = 0; i < pbl_info->num_pbls; i++) { 488 if (!pbl[i].va) 489 continue; 490 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 491 pbl[i].va, pbl[i].pa); 492 } 493 494 kfree(pbl); 495 } 496 497 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 498 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 499 500 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 501 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 502 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 503 504 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 505 struct qedr_pbl_info *pbl_info, 506 gfp_t flags) 507 { 508 struct pci_dev *pdev = dev->pdev; 509 struct qedr_pbl *pbl_table; 510 dma_addr_t *pbl_main_tbl; 511 dma_addr_t pa; 512 void *va; 513 int i; 514 515 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 516 if (!pbl_table) 517 return ERR_PTR(-ENOMEM); 518 519 for (i = 0; i < pbl_info->num_pbls; i++) { 520 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 521 flags); 522 if (!va) 523 goto err; 524 525 pbl_table[i].va = va; 526 pbl_table[i].pa = pa; 527 } 528 529 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 530 * the first one with physical pointers to all of the rest 531 */ 532 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 533 for (i = 0; i < pbl_info->num_pbls - 1; i++) 534 pbl_main_tbl[i] = pbl_table[i + 1].pa; 535 536 return pbl_table; 537 538 err: 539 for (i--; i >= 0; i--) 540 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 541 pbl_table[i].va, pbl_table[i].pa); 542 543 qedr_free_pbl(dev, pbl_info, pbl_table); 544 545 return ERR_PTR(-ENOMEM); 546 } 547 548 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 549 struct qedr_pbl_info *pbl_info, 550 u32 num_pbes, int two_layer_capable) 551 { 552 u32 pbl_capacity; 553 u32 pbl_size; 554 u32 num_pbls; 555 556 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 557 if (num_pbes > MAX_PBES_TWO_LAYER) { 558 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 559 num_pbes); 560 return -EINVAL; 561 } 562 563 /* calculate required pbl page size */ 564 pbl_size = MIN_FW_PBL_PAGE_SIZE; 565 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 566 NUM_PBES_ON_PAGE(pbl_size); 567 568 while (pbl_capacity < num_pbes) { 569 pbl_size *= 2; 570 pbl_capacity = pbl_size / sizeof(u64); 571 pbl_capacity = pbl_capacity * pbl_capacity; 572 } 573 574 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 575 num_pbls++; /* One for the layer0 ( points to the pbls) */ 576 pbl_info->two_layered = true; 577 } else { 578 /* One layered PBL */ 579 num_pbls = 1; 580 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 581 roundup_pow_of_two((num_pbes * sizeof(u64)))); 582 pbl_info->two_layered = false; 583 } 584 585 pbl_info->num_pbls = num_pbls; 586 pbl_info->pbl_size = pbl_size; 587 pbl_info->num_pbes = num_pbes; 588 589 DP_DEBUG(dev, QEDR_MSG_MR, 590 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 591 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 592 593 return 0; 594 } 595 596 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 597 struct qedr_pbl *pbl, 598 struct qedr_pbl_info *pbl_info, u32 pg_shift) 599 { 600 int pbe_cnt, total_num_pbes = 0; 601 u32 fw_pg_cnt, fw_pg_per_umem_pg; 602 struct qedr_pbl *pbl_tbl; 603 struct sg_dma_page_iter sg_iter; 604 struct regpair *pbe; 605 u64 pg_addr; 606 607 if (!pbl_info->num_pbes) 608 return; 609 610 /* If we have a two layered pbl, the first pbl points to the rest 611 * of the pbls and the first entry lays on the second pbl in the table 612 */ 613 if (pbl_info->two_layered) 614 pbl_tbl = &pbl[1]; 615 else 616 pbl_tbl = pbl; 617 618 pbe = (struct regpair *)pbl_tbl->va; 619 if (!pbe) { 620 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 621 return; 622 } 623 624 pbe_cnt = 0; 625 626 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); 627 628 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 629 pg_addr = sg_page_iter_dma_address(&sg_iter); 630 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 631 pbe->lo = cpu_to_le32(pg_addr); 632 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 633 634 pg_addr += BIT(pg_shift); 635 pbe_cnt++; 636 total_num_pbes++; 637 pbe++; 638 639 if (total_num_pbes == pbl_info->num_pbes) 640 return; 641 642 /* If the given pbl is full storing the pbes, 643 * move to next pbl. 644 */ 645 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 646 pbl_tbl++; 647 pbe = (struct regpair *)pbl_tbl->va; 648 pbe_cnt = 0; 649 } 650 651 fw_pg_cnt++; 652 } 653 } 654 } 655 656 static int qedr_db_recovery_add(struct qedr_dev *dev, 657 void __iomem *db_addr, 658 void *db_data, 659 enum qed_db_rec_width db_width, 660 enum qed_db_rec_space db_space) 661 { 662 if (!db_data) { 663 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 664 return 0; 665 } 666 667 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 668 db_width, db_space); 669 } 670 671 static void qedr_db_recovery_del(struct qedr_dev *dev, 672 void __iomem *db_addr, 673 void *db_data) 674 { 675 if (!db_data) { 676 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 677 return; 678 } 679 680 /* Ignore return code as there is not much we can do about it. Error 681 * log will be printed inside. 682 */ 683 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 684 } 685 686 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 687 struct qedr_cq *cq, struct ib_udata *udata, 688 u32 db_offset) 689 { 690 struct qedr_create_cq_uresp uresp; 691 int rc; 692 693 memset(&uresp, 0, sizeof(uresp)); 694 695 uresp.db_offset = db_offset; 696 uresp.icid = cq->icid; 697 if (cq->q.db_mmap_entry) 698 uresp.db_rec_addr = 699 rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 700 701 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 702 if (rc) 703 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 704 705 return rc; 706 } 707 708 static void consume_cqe(struct qedr_cq *cq) 709 { 710 if (cq->latest_cqe == cq->toggle_cqe) 711 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 712 713 cq->latest_cqe = qed_chain_consume(&cq->pbl); 714 } 715 716 static inline int qedr_align_cq_entries(int entries) 717 { 718 u64 size, aligned_size; 719 720 /* We allocate an extra entry that we don't report to the FW. */ 721 size = (entries + 1) * QEDR_CQE_SIZE; 722 aligned_size = ALIGN(size, PAGE_SIZE); 723 724 return aligned_size / QEDR_CQE_SIZE; 725 } 726 727 static int qedr_init_user_db_rec(struct ib_udata *udata, 728 struct qedr_dev *dev, struct qedr_userq *q, 729 bool requires_db_rec) 730 { 731 struct qedr_ucontext *uctx = 732 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 733 ibucontext); 734 struct qedr_user_mmap_entry *entry; 735 int rc; 736 737 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 738 if (requires_db_rec == 0 || !uctx->db_rec) 739 return 0; 740 741 /* Allocate a page for doorbell recovery, add to mmap */ 742 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 743 if (!q->db_rec_data) { 744 DP_ERR(dev, "get_zeroed_page failed\n"); 745 return -ENOMEM; 746 } 747 748 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 749 if (!entry) 750 goto err_free_db_data; 751 752 entry->address = q->db_rec_data; 753 entry->length = PAGE_SIZE; 754 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 755 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 756 &entry->rdma_entry, 757 PAGE_SIZE); 758 if (rc) 759 goto err_free_entry; 760 761 q->db_mmap_entry = &entry->rdma_entry; 762 763 return 0; 764 765 err_free_entry: 766 kfree(entry); 767 768 err_free_db_data: 769 free_page((unsigned long)q->db_rec_data); 770 q->db_rec_data = NULL; 771 return -ENOMEM; 772 } 773 774 static inline int qedr_init_user_queue(struct ib_udata *udata, 775 struct qedr_dev *dev, 776 struct qedr_userq *q, u64 buf_addr, 777 size_t buf_len, bool requires_db_rec, 778 int access, 779 int alloc_and_init) 780 { 781 u32 fw_pages; 782 int rc; 783 784 q->buf_addr = buf_addr; 785 q->buf_len = buf_len; 786 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access); 787 if (IS_ERR(q->umem)) { 788 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 789 PTR_ERR(q->umem)); 790 return PTR_ERR(q->umem); 791 } 792 793 fw_pages = ib_umem_page_count(q->umem) << 794 (PAGE_SHIFT - FW_PAGE_SHIFT); 795 796 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 797 if (rc) 798 goto err0; 799 800 if (alloc_and_init) { 801 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 802 if (IS_ERR(q->pbl_tbl)) { 803 rc = PTR_ERR(q->pbl_tbl); 804 goto err0; 805 } 806 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 807 FW_PAGE_SHIFT); 808 } else { 809 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 810 if (!q->pbl_tbl) { 811 rc = -ENOMEM; 812 goto err0; 813 } 814 } 815 816 /* mmap the user address used to store doorbell data for recovery */ 817 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 818 819 err0: 820 ib_umem_release(q->umem); 821 q->umem = NULL; 822 823 return rc; 824 } 825 826 static inline void qedr_init_cq_params(struct qedr_cq *cq, 827 struct qedr_ucontext *ctx, 828 struct qedr_dev *dev, int vector, 829 int chain_entries, int page_cnt, 830 u64 pbl_ptr, 831 struct qed_rdma_create_cq_in_params 832 *params) 833 { 834 memset(params, 0, sizeof(*params)); 835 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 836 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 837 params->cnq_id = vector; 838 params->cq_size = chain_entries - 1; 839 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 840 params->pbl_num_pages = page_cnt; 841 params->pbl_ptr = pbl_ptr; 842 params->pbl_two_level = 0; 843 } 844 845 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 846 { 847 cq->db.data.agg_flags = flags; 848 cq->db.data.value = cpu_to_le32(cons); 849 writeq(cq->db.raw, cq->db_addr); 850 } 851 852 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 853 { 854 struct qedr_cq *cq = get_qedr_cq(ibcq); 855 unsigned long sflags; 856 struct qedr_dev *dev; 857 858 dev = get_qedr_dev(ibcq->device); 859 860 if (cq->destroyed) { 861 DP_ERR(dev, 862 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 863 cq, cq->icid); 864 return -EINVAL; 865 } 866 867 868 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 869 return 0; 870 871 spin_lock_irqsave(&cq->cq_lock, sflags); 872 873 cq->arm_flags = 0; 874 875 if (flags & IB_CQ_SOLICITED) 876 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 877 878 if (flags & IB_CQ_NEXT_COMP) 879 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 880 881 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 882 883 spin_unlock_irqrestore(&cq->cq_lock, sflags); 884 885 return 0; 886 } 887 888 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 889 struct ib_udata *udata) 890 { 891 struct ib_device *ibdev = ibcq->device; 892 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 893 udata, struct qedr_ucontext, ibucontext); 894 struct qed_rdma_destroy_cq_out_params destroy_oparams; 895 struct qed_rdma_destroy_cq_in_params destroy_iparams; 896 struct qedr_dev *dev = get_qedr_dev(ibdev); 897 struct qed_rdma_create_cq_in_params params; 898 struct qedr_create_cq_ureq ureq = {}; 899 int vector = attr->comp_vector; 900 int entries = attr->cqe; 901 struct qedr_cq *cq = get_qedr_cq(ibcq); 902 int chain_entries; 903 u32 db_offset; 904 int page_cnt; 905 u64 pbl_ptr; 906 u16 icid; 907 int rc; 908 909 DP_DEBUG(dev, QEDR_MSG_INIT, 910 "create_cq: called from %s. entries=%d, vector=%d\n", 911 udata ? "User Lib" : "Kernel", entries, vector); 912 913 if (entries > QEDR_MAX_CQES) { 914 DP_ERR(dev, 915 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 916 entries, QEDR_MAX_CQES); 917 return -EINVAL; 918 } 919 920 chain_entries = qedr_align_cq_entries(entries); 921 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 922 923 /* calc db offset. user will add DPI base, kernel will add db addr */ 924 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 925 926 if (udata) { 927 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 928 udata->inlen))) { 929 DP_ERR(dev, 930 "create cq: problem copying data from user space\n"); 931 goto err0; 932 } 933 934 if (!ureq.len) { 935 DP_ERR(dev, 936 "create cq: cannot create a cq with 0 entries\n"); 937 goto err0; 938 } 939 940 cq->cq_type = QEDR_CQ_TYPE_USER; 941 942 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 943 ureq.len, true, IB_ACCESS_LOCAL_WRITE, 944 1); 945 if (rc) 946 goto err0; 947 948 pbl_ptr = cq->q.pbl_tbl->pa; 949 page_cnt = cq->q.pbl_info.num_pbes; 950 951 cq->ibcq.cqe = chain_entries; 952 cq->q.db_addr = ctx->dpi_addr + db_offset; 953 } else { 954 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 955 956 rc = dev->ops->common->chain_alloc(dev->cdev, 957 QED_CHAIN_USE_TO_CONSUME, 958 QED_CHAIN_MODE_PBL, 959 QED_CHAIN_CNT_TYPE_U32, 960 chain_entries, 961 sizeof(union rdma_cqe), 962 &cq->pbl, NULL); 963 if (rc) 964 goto err0; 965 966 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 967 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 968 cq->ibcq.cqe = cq->pbl.capacity; 969 } 970 971 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 972 pbl_ptr, ¶ms); 973 974 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 975 if (rc) 976 goto err1; 977 978 cq->icid = icid; 979 cq->sig = QEDR_CQ_MAGIC_NUMBER; 980 spin_lock_init(&cq->cq_lock); 981 982 if (udata) { 983 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 984 if (rc) 985 goto err2; 986 987 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 988 &cq->q.db_rec_data->db_data, 989 DB_REC_WIDTH_64B, 990 DB_REC_USER); 991 if (rc) 992 goto err2; 993 994 } else { 995 /* Generate doorbell address. */ 996 cq->db.data.icid = cq->icid; 997 cq->db_addr = dev->db_addr + db_offset; 998 cq->db.data.params = DB_AGG_CMD_SET << 999 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 1000 1001 /* point to the very last element, passing it we will toggle */ 1002 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 1003 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 1004 cq->latest_cqe = NULL; 1005 consume_cqe(cq); 1006 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 1007 1008 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 1009 DB_REC_WIDTH_64B, DB_REC_KERNEL); 1010 if (rc) 1011 goto err2; 1012 } 1013 1014 DP_DEBUG(dev, QEDR_MSG_CQ, 1015 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1016 cq->icid, cq, params.cq_size); 1017 1018 return 0; 1019 1020 err2: 1021 destroy_iparams.icid = cq->icid; 1022 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1023 &destroy_oparams); 1024 err1: 1025 if (udata) { 1026 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1027 ib_umem_release(cq->q.umem); 1028 if (cq->q.db_mmap_entry) 1029 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1030 } else { 1031 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1032 } 1033 err0: 1034 return -EINVAL; 1035 } 1036 1037 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1038 { 1039 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1040 struct qedr_cq *cq = get_qedr_cq(ibcq); 1041 1042 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1043 1044 return 0; 1045 } 1046 1047 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1048 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1049 1050 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1051 { 1052 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1053 struct qed_rdma_destroy_cq_out_params oparams; 1054 struct qed_rdma_destroy_cq_in_params iparams; 1055 struct qedr_cq *cq = get_qedr_cq(ibcq); 1056 int iter; 1057 1058 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1059 1060 cq->destroyed = 1; 1061 1062 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1063 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1064 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1065 return; 1066 } 1067 1068 iparams.icid = cq->icid; 1069 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1070 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1071 1072 if (udata) { 1073 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1074 ib_umem_release(cq->q.umem); 1075 1076 if (cq->q.db_rec_data) { 1077 qedr_db_recovery_del(dev, cq->q.db_addr, 1078 &cq->q.db_rec_data->db_data); 1079 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1080 } 1081 } else { 1082 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1083 } 1084 1085 /* We don't want the IRQ handler to handle a non-existing CQ so we 1086 * wait until all CNQ interrupts, if any, are received. This will always 1087 * happen and will always happen very fast. If not, then a serious error 1088 * has occured. That is why we can use a long delay. 1089 * We spin for a short time so we don’t lose time on context switching 1090 * in case all the completions are handled in that span. Otherwise 1091 * we sleep for a while and check again. Since the CNQ may be 1092 * associated with (only) the current CPU we use msleep to allow the 1093 * current CPU to be freed. 1094 * The CNQ notification is increased in qedr_irq_handler(). 1095 */ 1096 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1097 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1098 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1099 iter--; 1100 } 1101 1102 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1103 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1104 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1105 iter--; 1106 } 1107 1108 /* Note that we don't need to have explicit code to wait for the 1109 * completion of the event handler because it is invoked from the EQ. 1110 * Since the destroy CQ ramrod has also been received on the EQ we can 1111 * be certain that there's no event handler in process. 1112 */ 1113 } 1114 1115 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1116 struct ib_qp_attr *attr, 1117 int attr_mask, 1118 struct qed_rdma_modify_qp_in_params 1119 *qp_params) 1120 { 1121 const struct ib_gid_attr *gid_attr; 1122 enum rdma_network_type nw_type; 1123 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1124 u32 ipv4_addr; 1125 int ret; 1126 int i; 1127 1128 gid_attr = grh->sgid_attr; 1129 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1130 if (ret) 1131 return ret; 1132 1133 nw_type = rdma_gid_attr_network_type(gid_attr); 1134 switch (nw_type) { 1135 case RDMA_NETWORK_IPV6: 1136 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1137 sizeof(qp_params->sgid)); 1138 memcpy(&qp_params->dgid.bytes[0], 1139 &grh->dgid, 1140 sizeof(qp_params->dgid)); 1141 qp_params->roce_mode = ROCE_V2_IPV6; 1142 SET_FIELD(qp_params->modify_flags, 1143 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1144 break; 1145 case RDMA_NETWORK_IB: 1146 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1147 sizeof(qp_params->sgid)); 1148 memcpy(&qp_params->dgid.bytes[0], 1149 &grh->dgid, 1150 sizeof(qp_params->dgid)); 1151 qp_params->roce_mode = ROCE_V1; 1152 break; 1153 case RDMA_NETWORK_IPV4: 1154 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1155 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1156 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1157 qp_params->sgid.ipv4_addr = ipv4_addr; 1158 ipv4_addr = 1159 qedr_get_ipv4_from_gid(grh->dgid.raw); 1160 qp_params->dgid.ipv4_addr = ipv4_addr; 1161 SET_FIELD(qp_params->modify_flags, 1162 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1163 qp_params->roce_mode = ROCE_V2_IPV4; 1164 break; 1165 } 1166 1167 for (i = 0; i < 4; i++) { 1168 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1169 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1170 } 1171 1172 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1173 qp_params->vlan_id = 0; 1174 1175 return 0; 1176 } 1177 1178 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1179 struct ib_qp_init_attr *attrs, 1180 struct ib_udata *udata) 1181 { 1182 struct qedr_device_attr *qattr = &dev->attr; 1183 1184 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1185 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1186 DP_DEBUG(dev, QEDR_MSG_QP, 1187 "create qp: unsupported qp type=0x%x requested\n", 1188 attrs->qp_type); 1189 return -EINVAL; 1190 } 1191 1192 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1193 DP_ERR(dev, 1194 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1195 attrs->cap.max_send_wr, qattr->max_sqe); 1196 return -EINVAL; 1197 } 1198 1199 if (attrs->cap.max_inline_data > qattr->max_inline) { 1200 DP_ERR(dev, 1201 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1202 attrs->cap.max_inline_data, qattr->max_inline); 1203 return -EINVAL; 1204 } 1205 1206 if (attrs->cap.max_send_sge > qattr->max_sge) { 1207 DP_ERR(dev, 1208 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1209 attrs->cap.max_send_sge, qattr->max_sge); 1210 return -EINVAL; 1211 } 1212 1213 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1214 DP_ERR(dev, 1215 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1216 attrs->cap.max_recv_sge, qattr->max_sge); 1217 return -EINVAL; 1218 } 1219 1220 /* Unprivileged user space cannot create special QP */ 1221 if (udata && attrs->qp_type == IB_QPT_GSI) { 1222 DP_ERR(dev, 1223 "create qp: userspace can't create special QPs of type=0x%x\n", 1224 attrs->qp_type); 1225 return -EINVAL; 1226 } 1227 1228 return 0; 1229 } 1230 1231 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1232 struct qedr_srq *srq, struct ib_udata *udata) 1233 { 1234 struct qedr_create_srq_uresp uresp = {}; 1235 int rc; 1236 1237 uresp.srq_id = srq->srq_id; 1238 1239 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1240 if (rc) 1241 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1242 1243 return rc; 1244 } 1245 1246 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1247 struct qedr_create_qp_uresp *uresp, 1248 struct qedr_qp *qp) 1249 { 1250 /* iWARP requires two doorbells per RQ. */ 1251 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1252 uresp->rq_db_offset = 1253 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1254 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1255 } else { 1256 uresp->rq_db_offset = 1257 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1258 } 1259 1260 uresp->rq_icid = qp->icid; 1261 if (qp->urq.db_mmap_entry) 1262 uresp->rq_db_rec_addr = 1263 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1264 } 1265 1266 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1267 struct qedr_create_qp_uresp *uresp, 1268 struct qedr_qp *qp) 1269 { 1270 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1271 1272 /* iWARP uses the same cid for rq and sq */ 1273 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1274 uresp->sq_icid = qp->icid; 1275 else 1276 uresp->sq_icid = qp->icid + 1; 1277 1278 if (qp->usq.db_mmap_entry) 1279 uresp->sq_db_rec_addr = 1280 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1281 } 1282 1283 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1284 struct qedr_qp *qp, struct ib_udata *udata, 1285 struct qedr_create_qp_uresp *uresp) 1286 { 1287 int rc; 1288 1289 memset(uresp, 0, sizeof(*uresp)); 1290 qedr_copy_sq_uresp(dev, uresp, qp); 1291 qedr_copy_rq_uresp(dev, uresp, qp); 1292 1293 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1294 uresp->qp_id = qp->qp_id; 1295 1296 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1297 if (rc) 1298 DP_ERR(dev, 1299 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1300 qp->icid); 1301 1302 return rc; 1303 } 1304 1305 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1306 struct qedr_qp *qp, 1307 struct qedr_pd *pd, 1308 struct ib_qp_init_attr *attrs) 1309 { 1310 spin_lock_init(&qp->q_lock); 1311 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1312 kref_init(&qp->refcnt); 1313 init_completion(&qp->iwarp_cm_comp); 1314 } 1315 qp->pd = pd; 1316 qp->qp_type = attrs->qp_type; 1317 qp->max_inline_data = attrs->cap.max_inline_data; 1318 qp->sq.max_sges = attrs->cap.max_send_sge; 1319 qp->state = QED_ROCE_QP_STATE_RESET; 1320 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1321 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1322 qp->dev = dev; 1323 1324 if (attrs->srq) { 1325 qp->srq = get_qedr_srq(attrs->srq); 1326 } else { 1327 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1328 qp->rq.max_sges = attrs->cap.max_recv_sge; 1329 DP_DEBUG(dev, QEDR_MSG_QP, 1330 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1331 qp->rq.max_sges, qp->rq_cq->icid); 1332 } 1333 1334 DP_DEBUG(dev, QEDR_MSG_QP, 1335 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1336 pd->pd_id, qp->qp_type, qp->max_inline_data, 1337 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1338 DP_DEBUG(dev, QEDR_MSG_QP, 1339 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1340 qp->sq.max_sges, qp->sq_cq->icid); 1341 } 1342 1343 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1344 { 1345 int rc; 1346 1347 qp->sq.db = dev->db_addr + 1348 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1349 qp->sq.db_data.data.icid = qp->icid + 1; 1350 rc = qedr_db_recovery_add(dev, qp->sq.db, 1351 &qp->sq.db_data, 1352 DB_REC_WIDTH_32B, 1353 DB_REC_KERNEL); 1354 if (rc) 1355 return rc; 1356 1357 if (!qp->srq) { 1358 qp->rq.db = dev->db_addr + 1359 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1360 qp->rq.db_data.data.icid = qp->icid; 1361 1362 rc = qedr_db_recovery_add(dev, qp->rq.db, 1363 &qp->rq.db_data, 1364 DB_REC_WIDTH_32B, 1365 DB_REC_KERNEL); 1366 if (rc) 1367 qedr_db_recovery_del(dev, qp->sq.db, 1368 &qp->sq.db_data); 1369 } 1370 1371 return rc; 1372 } 1373 1374 static int qedr_check_srq_params(struct qedr_dev *dev, 1375 struct ib_srq_init_attr *attrs, 1376 struct ib_udata *udata) 1377 { 1378 struct qedr_device_attr *qattr = &dev->attr; 1379 1380 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1381 DP_ERR(dev, 1382 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1383 attrs->attr.max_wr, qattr->max_srq_wr); 1384 return -EINVAL; 1385 } 1386 1387 if (attrs->attr.max_sge > qattr->max_sge) { 1388 DP_ERR(dev, 1389 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1390 attrs->attr.max_sge, qattr->max_sge); 1391 return -EINVAL; 1392 } 1393 1394 return 0; 1395 } 1396 1397 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1398 { 1399 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1400 ib_umem_release(srq->usrq.umem); 1401 ib_umem_release(srq->prod_umem); 1402 } 1403 1404 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1405 { 1406 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1407 struct qedr_dev *dev = srq->dev; 1408 1409 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1410 1411 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1412 hw_srq->virt_prod_pair_addr, 1413 hw_srq->phy_prod_pair_addr); 1414 } 1415 1416 static int qedr_init_srq_user_params(struct ib_udata *udata, 1417 struct qedr_srq *srq, 1418 struct qedr_create_srq_ureq *ureq, 1419 int access) 1420 { 1421 struct scatterlist *sg; 1422 int rc; 1423 1424 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1425 ureq->srq_len, false, access, 1); 1426 if (rc) 1427 return rc; 1428 1429 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr, 1430 sizeof(struct rdma_srq_producers), access); 1431 if (IS_ERR(srq->prod_umem)) { 1432 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1433 ib_umem_release(srq->usrq.umem); 1434 DP_ERR(srq->dev, 1435 "create srq: failed ib_umem_get for producer, got %ld\n", 1436 PTR_ERR(srq->prod_umem)); 1437 return PTR_ERR(srq->prod_umem); 1438 } 1439 1440 sg = srq->prod_umem->sg_head.sgl; 1441 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1442 1443 return 0; 1444 } 1445 1446 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1447 struct qedr_dev *dev, 1448 struct ib_srq_init_attr *init_attr) 1449 { 1450 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1451 dma_addr_t phy_prod_pair_addr; 1452 u32 num_elems; 1453 void *va; 1454 int rc; 1455 1456 va = dma_alloc_coherent(&dev->pdev->dev, 1457 sizeof(struct rdma_srq_producers), 1458 &phy_prod_pair_addr, GFP_KERNEL); 1459 if (!va) { 1460 DP_ERR(dev, 1461 "create srq: failed to allocate dma memory for producer\n"); 1462 return -ENOMEM; 1463 } 1464 1465 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1466 hw_srq->virt_prod_pair_addr = va; 1467 1468 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1469 rc = dev->ops->common->chain_alloc(dev->cdev, 1470 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1471 QED_CHAIN_MODE_PBL, 1472 QED_CHAIN_CNT_TYPE_U32, 1473 num_elems, 1474 QEDR_SRQ_WQE_ELEM_SIZE, 1475 &hw_srq->pbl, NULL); 1476 if (rc) 1477 goto err0; 1478 1479 hw_srq->num_elems = num_elems; 1480 1481 return 0; 1482 1483 err0: 1484 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1485 va, phy_prod_pair_addr); 1486 return rc; 1487 } 1488 1489 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1490 struct ib_udata *udata) 1491 { 1492 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1493 struct qed_rdma_create_srq_in_params in_params = {}; 1494 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1495 struct qed_rdma_create_srq_out_params out_params; 1496 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1497 struct qedr_create_srq_ureq ureq = {}; 1498 u64 pbl_base_addr, phy_prod_pair_addr; 1499 struct qedr_srq_hwq_info *hw_srq; 1500 u32 page_cnt, page_size; 1501 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1502 int rc = 0; 1503 1504 DP_DEBUG(dev, QEDR_MSG_QP, 1505 "create SRQ called from %s (pd %p)\n", 1506 (udata) ? "User lib" : "kernel", pd); 1507 1508 rc = qedr_check_srq_params(dev, init_attr, udata); 1509 if (rc) 1510 return -EINVAL; 1511 1512 srq->dev = dev; 1513 hw_srq = &srq->hw_srq; 1514 spin_lock_init(&srq->lock); 1515 1516 hw_srq->max_wr = init_attr->attr.max_wr; 1517 hw_srq->max_sges = init_attr->attr.max_sge; 1518 1519 if (udata) { 1520 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1521 udata->inlen))) { 1522 DP_ERR(dev, 1523 "create srq: problem copying data from user space\n"); 1524 goto err0; 1525 } 1526 1527 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0); 1528 if (rc) 1529 goto err0; 1530 1531 page_cnt = srq->usrq.pbl_info.num_pbes; 1532 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1533 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1534 page_size = PAGE_SIZE; 1535 } else { 1536 struct qed_chain *pbl; 1537 1538 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1539 if (rc) 1540 goto err0; 1541 1542 pbl = &hw_srq->pbl; 1543 page_cnt = qed_chain_get_page_cnt(pbl); 1544 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1545 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1546 page_size = QED_CHAIN_PAGE_SIZE; 1547 } 1548 1549 in_params.pd_id = pd->pd_id; 1550 in_params.pbl_base_addr = pbl_base_addr; 1551 in_params.prod_pair_addr = phy_prod_pair_addr; 1552 in_params.num_pages = page_cnt; 1553 in_params.page_size = page_size; 1554 1555 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1556 if (rc) 1557 goto err1; 1558 1559 srq->srq_id = out_params.srq_id; 1560 1561 if (udata) { 1562 rc = qedr_copy_srq_uresp(dev, srq, udata); 1563 if (rc) 1564 goto err2; 1565 } 1566 1567 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1568 if (rc) 1569 goto err2; 1570 1571 DP_DEBUG(dev, QEDR_MSG_SRQ, 1572 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1573 return 0; 1574 1575 err2: 1576 destroy_in_params.srq_id = srq->srq_id; 1577 1578 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1579 err1: 1580 if (udata) 1581 qedr_free_srq_user_params(srq); 1582 else 1583 qedr_free_srq_kernel_params(srq); 1584 err0: 1585 return -EFAULT; 1586 } 1587 1588 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1589 { 1590 struct qed_rdma_destroy_srq_in_params in_params = {}; 1591 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1592 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1593 1594 xa_erase_irq(&dev->srqs, srq->srq_id); 1595 in_params.srq_id = srq->srq_id; 1596 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1597 1598 if (ibsrq->uobject) 1599 qedr_free_srq_user_params(srq); 1600 else 1601 qedr_free_srq_kernel_params(srq); 1602 1603 DP_DEBUG(dev, QEDR_MSG_SRQ, 1604 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1605 srq->srq_id); 1606 } 1607 1608 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1609 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1610 { 1611 struct qed_rdma_modify_srq_in_params in_params = {}; 1612 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1613 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1614 int rc; 1615 1616 if (attr_mask & IB_SRQ_MAX_WR) { 1617 DP_ERR(dev, 1618 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1619 attr_mask, srq); 1620 return -EINVAL; 1621 } 1622 1623 if (attr_mask & IB_SRQ_LIMIT) { 1624 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1625 DP_ERR(dev, 1626 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1627 attr->srq_limit, srq->hw_srq.max_wr); 1628 return -EINVAL; 1629 } 1630 1631 in_params.srq_id = srq->srq_id; 1632 in_params.wqe_limit = attr->srq_limit; 1633 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1634 if (rc) 1635 return rc; 1636 } 1637 1638 srq->srq_limit = attr->srq_limit; 1639 1640 DP_DEBUG(dev, QEDR_MSG_SRQ, 1641 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1642 1643 return 0; 1644 } 1645 1646 static inline void 1647 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1648 struct qedr_pd *pd, 1649 struct qedr_qp *qp, 1650 struct ib_qp_init_attr *attrs, 1651 bool fmr_and_reserved_lkey, 1652 struct qed_rdma_create_qp_in_params *params) 1653 { 1654 /* QP handle to be written in an async event */ 1655 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1656 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1657 1658 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1659 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1660 params->pd = pd->pd_id; 1661 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1662 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1663 params->stats_queue = 0; 1664 params->srq_id = 0; 1665 params->use_srq = false; 1666 1667 if (!qp->srq) { 1668 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1669 1670 } else { 1671 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1672 params->srq_id = qp->srq->srq_id; 1673 params->use_srq = true; 1674 } 1675 } 1676 1677 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1678 { 1679 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1680 "qp=%p. " 1681 "sq_addr=0x%llx, " 1682 "sq_len=%zd, " 1683 "rq_addr=0x%llx, " 1684 "rq_len=%zd" 1685 "\n", 1686 qp, 1687 qp->usq.buf_addr, 1688 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1689 } 1690 1691 static inline void 1692 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1693 struct qedr_qp *qp, 1694 struct qed_rdma_create_qp_out_params *out_params) 1695 { 1696 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1697 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1698 1699 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1700 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1701 if (!qp->srq) { 1702 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1703 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1704 } 1705 1706 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1707 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1708 } 1709 1710 static void qedr_cleanup_user(struct qedr_dev *dev, 1711 struct qedr_ucontext *ctx, 1712 struct qedr_qp *qp) 1713 { 1714 ib_umem_release(qp->usq.umem); 1715 qp->usq.umem = NULL; 1716 1717 ib_umem_release(qp->urq.umem); 1718 qp->urq.umem = NULL; 1719 1720 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1721 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1722 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1723 } else { 1724 kfree(qp->usq.pbl_tbl); 1725 kfree(qp->urq.pbl_tbl); 1726 } 1727 1728 if (qp->usq.db_rec_data) { 1729 qedr_db_recovery_del(dev, qp->usq.db_addr, 1730 &qp->usq.db_rec_data->db_data); 1731 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1732 } 1733 1734 if (qp->urq.db_rec_data) { 1735 qedr_db_recovery_del(dev, qp->urq.db_addr, 1736 &qp->urq.db_rec_data->db_data); 1737 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1738 } 1739 1740 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1741 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1742 &qp->urq.db_rec_db2_data); 1743 } 1744 1745 static int qedr_create_user_qp(struct qedr_dev *dev, 1746 struct qedr_qp *qp, 1747 struct ib_pd *ibpd, 1748 struct ib_udata *udata, 1749 struct ib_qp_init_attr *attrs) 1750 { 1751 struct qed_rdma_create_qp_in_params in_params; 1752 struct qed_rdma_create_qp_out_params out_params; 1753 struct qedr_pd *pd = get_qedr_pd(ibpd); 1754 struct qedr_create_qp_uresp uresp; 1755 struct qedr_ucontext *ctx = NULL; 1756 struct qedr_create_qp_ureq ureq; 1757 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1758 int rc = -EINVAL; 1759 1760 qp->create_type = QEDR_QP_CREATE_USER; 1761 memset(&ureq, 0, sizeof(ureq)); 1762 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen)); 1763 if (rc) { 1764 DP_ERR(dev, "Problem copying data from user space\n"); 1765 return rc; 1766 } 1767 1768 /* SQ - read access only (0) */ 1769 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1770 ureq.sq_len, true, 0, alloc_and_init); 1771 if (rc) 1772 return rc; 1773 1774 if (!qp->srq) { 1775 /* RQ - read access only (0) */ 1776 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1777 ureq.rq_len, true, 0, alloc_and_init); 1778 if (rc) 1779 return rc; 1780 } 1781 1782 memset(&in_params, 0, sizeof(in_params)); 1783 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1784 in_params.qp_handle_lo = ureq.qp_handle_lo; 1785 in_params.qp_handle_hi = ureq.qp_handle_hi; 1786 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1787 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1788 if (!qp->srq) { 1789 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1790 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1791 } 1792 1793 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1794 &in_params, &out_params); 1795 1796 if (!qp->qed_qp) { 1797 rc = -ENOMEM; 1798 goto err1; 1799 } 1800 1801 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1802 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1803 1804 qp->qp_id = out_params.qp_id; 1805 qp->icid = out_params.icid; 1806 1807 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1808 if (rc) 1809 goto err; 1810 1811 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1812 ctx = pd->uctx; 1813 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1814 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1815 1816 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1817 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1818 1819 /* calculate the db_rec_db2 data since it is constant so no 1820 * need to reflect from user 1821 */ 1822 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1823 qp->urq.db_rec_db2_data.data.value = 1824 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1825 } 1826 1827 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1828 &qp->usq.db_rec_data->db_data, 1829 DB_REC_WIDTH_32B, 1830 DB_REC_USER); 1831 if (rc) 1832 goto err; 1833 1834 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1835 &qp->urq.db_rec_data->db_data, 1836 DB_REC_WIDTH_32B, 1837 DB_REC_USER); 1838 if (rc) 1839 goto err; 1840 1841 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1842 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1843 &qp->urq.db_rec_db2_data, 1844 DB_REC_WIDTH_32B, 1845 DB_REC_USER); 1846 if (rc) 1847 goto err; 1848 } 1849 qedr_qp_user_print(dev, qp); 1850 1851 return rc; 1852 err: 1853 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1854 if (rc) 1855 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1856 1857 err1: 1858 qedr_cleanup_user(dev, ctx, qp); 1859 return rc; 1860 } 1861 1862 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1863 { 1864 int rc; 1865 1866 qp->sq.db = dev->db_addr + 1867 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1868 qp->sq.db_data.data.icid = qp->icid; 1869 1870 rc = qedr_db_recovery_add(dev, qp->sq.db, 1871 &qp->sq.db_data, 1872 DB_REC_WIDTH_32B, 1873 DB_REC_KERNEL); 1874 if (rc) 1875 return rc; 1876 1877 qp->rq.db = dev->db_addr + 1878 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1879 qp->rq.db_data.data.icid = qp->icid; 1880 qp->rq.iwarp_db2 = dev->db_addr + 1881 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1882 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1883 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1884 1885 rc = qedr_db_recovery_add(dev, qp->rq.db, 1886 &qp->rq.db_data, 1887 DB_REC_WIDTH_32B, 1888 DB_REC_KERNEL); 1889 if (rc) 1890 return rc; 1891 1892 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 1893 &qp->rq.iwarp_db2_data, 1894 DB_REC_WIDTH_32B, 1895 DB_REC_KERNEL); 1896 return rc; 1897 } 1898 1899 static int 1900 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1901 struct qedr_qp *qp, 1902 struct qed_rdma_create_qp_in_params *in_params, 1903 u32 n_sq_elems, u32 n_rq_elems) 1904 { 1905 struct qed_rdma_create_qp_out_params out_params; 1906 int rc; 1907 1908 rc = dev->ops->common->chain_alloc(dev->cdev, 1909 QED_CHAIN_USE_TO_PRODUCE, 1910 QED_CHAIN_MODE_PBL, 1911 QED_CHAIN_CNT_TYPE_U32, 1912 n_sq_elems, 1913 QEDR_SQE_ELEMENT_SIZE, 1914 &qp->sq.pbl, NULL); 1915 1916 if (rc) 1917 return rc; 1918 1919 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1920 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1921 1922 rc = dev->ops->common->chain_alloc(dev->cdev, 1923 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1924 QED_CHAIN_MODE_PBL, 1925 QED_CHAIN_CNT_TYPE_U32, 1926 n_rq_elems, 1927 QEDR_RQE_ELEMENT_SIZE, 1928 &qp->rq.pbl, NULL); 1929 if (rc) 1930 return rc; 1931 1932 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1933 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1934 1935 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1936 in_params, &out_params); 1937 1938 if (!qp->qed_qp) 1939 return -EINVAL; 1940 1941 qp->qp_id = out_params.qp_id; 1942 qp->icid = out_params.icid; 1943 1944 return qedr_set_roce_db_info(dev, qp); 1945 } 1946 1947 static int 1948 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1949 struct qedr_qp *qp, 1950 struct qed_rdma_create_qp_in_params *in_params, 1951 u32 n_sq_elems, u32 n_rq_elems) 1952 { 1953 struct qed_rdma_create_qp_out_params out_params; 1954 struct qed_chain_ext_pbl ext_pbl; 1955 int rc; 1956 1957 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1958 QEDR_SQE_ELEMENT_SIZE, 1959 QED_CHAIN_MODE_PBL); 1960 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1961 QEDR_RQE_ELEMENT_SIZE, 1962 QED_CHAIN_MODE_PBL); 1963 1964 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1965 in_params, &out_params); 1966 1967 if (!qp->qed_qp) 1968 return -EINVAL; 1969 1970 /* Now we allocate the chain */ 1971 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1972 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1973 1974 rc = dev->ops->common->chain_alloc(dev->cdev, 1975 QED_CHAIN_USE_TO_PRODUCE, 1976 QED_CHAIN_MODE_PBL, 1977 QED_CHAIN_CNT_TYPE_U32, 1978 n_sq_elems, 1979 QEDR_SQE_ELEMENT_SIZE, 1980 &qp->sq.pbl, &ext_pbl); 1981 1982 if (rc) 1983 goto err; 1984 1985 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1986 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1987 1988 rc = dev->ops->common->chain_alloc(dev->cdev, 1989 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1990 QED_CHAIN_MODE_PBL, 1991 QED_CHAIN_CNT_TYPE_U32, 1992 n_rq_elems, 1993 QEDR_RQE_ELEMENT_SIZE, 1994 &qp->rq.pbl, &ext_pbl); 1995 1996 if (rc) 1997 goto err; 1998 1999 qp->qp_id = out_params.qp_id; 2000 qp->icid = out_params.icid; 2001 2002 return qedr_set_iwarp_db_info(dev, qp); 2003 2004 err: 2005 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2006 2007 return rc; 2008 } 2009 2010 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 2011 { 2012 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2013 kfree(qp->wqe_wr_id); 2014 2015 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2016 kfree(qp->rqe_wr_id); 2017 2018 /* GSI qp is not registered to db mechanism so no need to delete */ 2019 if (qp->qp_type == IB_QPT_GSI) 2020 return; 2021 2022 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2023 2024 if (!qp->srq) { 2025 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2026 2027 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2028 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2029 &qp->rq.iwarp_db2_data); 2030 } 2031 } 2032 2033 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2034 struct qedr_qp *qp, 2035 struct ib_pd *ibpd, 2036 struct ib_qp_init_attr *attrs) 2037 { 2038 struct qed_rdma_create_qp_in_params in_params; 2039 struct qedr_pd *pd = get_qedr_pd(ibpd); 2040 int rc = -EINVAL; 2041 u32 n_rq_elems; 2042 u32 n_sq_elems; 2043 u32 n_sq_entries; 2044 2045 memset(&in_params, 0, sizeof(in_params)); 2046 qp->create_type = QEDR_QP_CREATE_KERNEL; 2047 2048 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2049 * the ring. The ring should allow at least a single WR, even if the 2050 * user requested none, due to allocation issues. 2051 * We should add an extra WR since the prod and cons indices of 2052 * wqe_wr_id are managed in such a way that the WQ is considered full 2053 * when (prod+1)%max_wr==cons. We currently don't do that because we 2054 * double the number of entries due an iSER issue that pushes far more 2055 * WRs than indicated. If we decline its ib_post_send() then we get 2056 * error prints in the dmesg we'd like to avoid. 2057 */ 2058 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2059 dev->attr.max_sqe); 2060 2061 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 2062 GFP_KERNEL); 2063 if (!qp->wqe_wr_id) { 2064 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2065 return -ENOMEM; 2066 } 2067 2068 /* QP handle to be written in CQE */ 2069 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2070 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2071 2072 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2073 * the ring. There ring should allow at least a single WR, even if the 2074 * user requested none, due to allocation issues. 2075 */ 2076 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2077 2078 /* Allocate driver internal RQ array */ 2079 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 2080 GFP_KERNEL); 2081 if (!qp->rqe_wr_id) { 2082 DP_ERR(dev, 2083 "create qp: failed RQ shadow memory allocation\n"); 2084 kfree(qp->wqe_wr_id); 2085 return -ENOMEM; 2086 } 2087 2088 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2089 2090 n_sq_entries = attrs->cap.max_send_wr; 2091 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2092 n_sq_entries = max_t(u32, n_sq_entries, 1); 2093 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2094 2095 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2096 2097 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2098 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2099 n_sq_elems, n_rq_elems); 2100 else 2101 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2102 n_sq_elems, n_rq_elems); 2103 if (rc) 2104 qedr_cleanup_kernel(dev, qp); 2105 2106 return rc; 2107 } 2108 2109 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 2110 struct ib_qp_init_attr *attrs, 2111 struct ib_udata *udata) 2112 { 2113 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2114 struct qedr_pd *pd = get_qedr_pd(ibpd); 2115 struct qedr_qp *qp; 2116 struct ib_qp *ibqp; 2117 int rc = 0; 2118 2119 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2120 udata ? "user library" : "kernel", pd); 2121 2122 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2123 if (rc) 2124 return ERR_PTR(rc); 2125 2126 DP_DEBUG(dev, QEDR_MSG_QP, 2127 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2128 udata ? "user library" : "kernel", attrs->event_handler, pd, 2129 get_qedr_cq(attrs->send_cq), 2130 get_qedr_cq(attrs->send_cq)->icid, 2131 get_qedr_cq(attrs->recv_cq), 2132 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2133 2134 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2135 if (!qp) { 2136 DP_ERR(dev, "create qp: failed allocating memory\n"); 2137 return ERR_PTR(-ENOMEM); 2138 } 2139 2140 qedr_set_common_qp_params(dev, qp, pd, attrs); 2141 2142 if (attrs->qp_type == IB_QPT_GSI) { 2143 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 2144 if (IS_ERR(ibqp)) 2145 kfree(qp); 2146 return ibqp; 2147 } 2148 2149 if (udata) 2150 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2151 else 2152 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2153 2154 if (rc) 2155 goto err; 2156 2157 qp->ibqp.qp_num = qp->qp_id; 2158 2159 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2160 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2161 if (rc) 2162 goto err; 2163 } 2164 2165 return &qp->ibqp; 2166 2167 err: 2168 kfree(qp); 2169 2170 return ERR_PTR(-EFAULT); 2171 } 2172 2173 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2174 { 2175 switch (qp_state) { 2176 case QED_ROCE_QP_STATE_RESET: 2177 return IB_QPS_RESET; 2178 case QED_ROCE_QP_STATE_INIT: 2179 return IB_QPS_INIT; 2180 case QED_ROCE_QP_STATE_RTR: 2181 return IB_QPS_RTR; 2182 case QED_ROCE_QP_STATE_RTS: 2183 return IB_QPS_RTS; 2184 case QED_ROCE_QP_STATE_SQD: 2185 return IB_QPS_SQD; 2186 case QED_ROCE_QP_STATE_ERR: 2187 return IB_QPS_ERR; 2188 case QED_ROCE_QP_STATE_SQE: 2189 return IB_QPS_SQE; 2190 } 2191 return IB_QPS_ERR; 2192 } 2193 2194 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2195 enum ib_qp_state qp_state) 2196 { 2197 switch (qp_state) { 2198 case IB_QPS_RESET: 2199 return QED_ROCE_QP_STATE_RESET; 2200 case IB_QPS_INIT: 2201 return QED_ROCE_QP_STATE_INIT; 2202 case IB_QPS_RTR: 2203 return QED_ROCE_QP_STATE_RTR; 2204 case IB_QPS_RTS: 2205 return QED_ROCE_QP_STATE_RTS; 2206 case IB_QPS_SQD: 2207 return QED_ROCE_QP_STATE_SQD; 2208 case IB_QPS_ERR: 2209 return QED_ROCE_QP_STATE_ERR; 2210 default: 2211 return QED_ROCE_QP_STATE_ERR; 2212 } 2213 } 2214 2215 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2216 { 2217 qed_chain_reset(&qph->pbl); 2218 qph->prod = 0; 2219 qph->cons = 0; 2220 qph->wqe_cons = 0; 2221 qph->db_data.data.value = cpu_to_le16(0); 2222 } 2223 2224 static int qedr_update_qp_state(struct qedr_dev *dev, 2225 struct qedr_qp *qp, 2226 enum qed_roce_qp_state cur_state, 2227 enum qed_roce_qp_state new_state) 2228 { 2229 int status = 0; 2230 2231 if (new_state == cur_state) 2232 return 0; 2233 2234 switch (cur_state) { 2235 case QED_ROCE_QP_STATE_RESET: 2236 switch (new_state) { 2237 case QED_ROCE_QP_STATE_INIT: 2238 qp->prev_wqe_size = 0; 2239 qedr_reset_qp_hwq_info(&qp->sq); 2240 qedr_reset_qp_hwq_info(&qp->rq); 2241 break; 2242 default: 2243 status = -EINVAL; 2244 break; 2245 } 2246 break; 2247 case QED_ROCE_QP_STATE_INIT: 2248 switch (new_state) { 2249 case QED_ROCE_QP_STATE_RTR: 2250 /* Update doorbell (in case post_recv was 2251 * done before move to RTR) 2252 */ 2253 2254 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2255 writel(qp->rq.db_data.raw, qp->rq.db); 2256 } 2257 break; 2258 case QED_ROCE_QP_STATE_ERR: 2259 break; 2260 default: 2261 /* Invalid state change. */ 2262 status = -EINVAL; 2263 break; 2264 } 2265 break; 2266 case QED_ROCE_QP_STATE_RTR: 2267 /* RTR->XXX */ 2268 switch (new_state) { 2269 case QED_ROCE_QP_STATE_RTS: 2270 break; 2271 case QED_ROCE_QP_STATE_ERR: 2272 break; 2273 default: 2274 /* Invalid state change. */ 2275 status = -EINVAL; 2276 break; 2277 } 2278 break; 2279 case QED_ROCE_QP_STATE_RTS: 2280 /* RTS->XXX */ 2281 switch (new_state) { 2282 case QED_ROCE_QP_STATE_SQD: 2283 break; 2284 case QED_ROCE_QP_STATE_ERR: 2285 break; 2286 default: 2287 /* Invalid state change. */ 2288 status = -EINVAL; 2289 break; 2290 } 2291 break; 2292 case QED_ROCE_QP_STATE_SQD: 2293 /* SQD->XXX */ 2294 switch (new_state) { 2295 case QED_ROCE_QP_STATE_RTS: 2296 case QED_ROCE_QP_STATE_ERR: 2297 break; 2298 default: 2299 /* Invalid state change. */ 2300 status = -EINVAL; 2301 break; 2302 } 2303 break; 2304 case QED_ROCE_QP_STATE_ERR: 2305 /* ERR->XXX */ 2306 switch (new_state) { 2307 case QED_ROCE_QP_STATE_RESET: 2308 if ((qp->rq.prod != qp->rq.cons) || 2309 (qp->sq.prod != qp->sq.cons)) { 2310 DP_NOTICE(dev, 2311 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2312 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2313 qp->sq.cons); 2314 status = -EINVAL; 2315 } 2316 break; 2317 default: 2318 status = -EINVAL; 2319 break; 2320 } 2321 break; 2322 default: 2323 status = -EINVAL; 2324 break; 2325 } 2326 2327 return status; 2328 } 2329 2330 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2331 int attr_mask, struct ib_udata *udata) 2332 { 2333 struct qedr_qp *qp = get_qedr_qp(ibqp); 2334 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2335 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2336 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2337 enum ib_qp_state old_qp_state, new_qp_state; 2338 enum qed_roce_qp_state cur_state; 2339 int rc = 0; 2340 2341 DP_DEBUG(dev, QEDR_MSG_QP, 2342 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2343 attr->qp_state); 2344 2345 old_qp_state = qedr_get_ibqp_state(qp->state); 2346 if (attr_mask & IB_QP_STATE) 2347 new_qp_state = attr->qp_state; 2348 else 2349 new_qp_state = old_qp_state; 2350 2351 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2352 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2353 ibqp->qp_type, attr_mask)) { 2354 DP_ERR(dev, 2355 "modify qp: invalid attribute mask=0x%x specified for\n" 2356 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2357 attr_mask, qp->qp_id, ibqp->qp_type, 2358 old_qp_state, new_qp_state); 2359 rc = -EINVAL; 2360 goto err; 2361 } 2362 } 2363 2364 /* Translate the masks... */ 2365 if (attr_mask & IB_QP_STATE) { 2366 SET_FIELD(qp_params.modify_flags, 2367 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2368 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2369 } 2370 2371 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2372 qp_params.sqd_async = true; 2373 2374 if (attr_mask & IB_QP_PKEY_INDEX) { 2375 SET_FIELD(qp_params.modify_flags, 2376 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2377 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2378 rc = -EINVAL; 2379 goto err; 2380 } 2381 2382 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2383 } 2384 2385 if (attr_mask & IB_QP_QKEY) 2386 qp->qkey = attr->qkey; 2387 2388 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2389 SET_FIELD(qp_params.modify_flags, 2390 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2391 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2392 IB_ACCESS_REMOTE_READ; 2393 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2394 IB_ACCESS_REMOTE_WRITE; 2395 qp_params.incoming_atomic_en = attr->qp_access_flags & 2396 IB_ACCESS_REMOTE_ATOMIC; 2397 } 2398 2399 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2400 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2401 return -EINVAL; 2402 2403 if (attr_mask & IB_QP_PATH_MTU) { 2404 if (attr->path_mtu < IB_MTU_256 || 2405 attr->path_mtu > IB_MTU_4096) { 2406 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2407 rc = -EINVAL; 2408 goto err; 2409 } 2410 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2411 ib_mtu_enum_to_int(iboe_get_mtu 2412 (dev->ndev->mtu))); 2413 } 2414 2415 if (!qp->mtu) { 2416 qp->mtu = 2417 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2418 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2419 } 2420 2421 SET_FIELD(qp_params.modify_flags, 2422 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2423 2424 qp_params.traffic_class_tos = grh->traffic_class; 2425 qp_params.flow_label = grh->flow_label; 2426 qp_params.hop_limit_ttl = grh->hop_limit; 2427 2428 qp->sgid_idx = grh->sgid_index; 2429 2430 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2431 if (rc) { 2432 DP_ERR(dev, 2433 "modify qp: problems with GID index %d (rc=%d)\n", 2434 grh->sgid_index, rc); 2435 return rc; 2436 } 2437 2438 rc = qedr_get_dmac(dev, &attr->ah_attr, 2439 qp_params.remote_mac_addr); 2440 if (rc) 2441 return rc; 2442 2443 qp_params.use_local_mac = true; 2444 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2445 2446 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2447 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2448 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2449 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2450 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2451 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2452 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2453 qp_params.remote_mac_addr); 2454 2455 qp_params.mtu = qp->mtu; 2456 qp_params.lb_indication = false; 2457 } 2458 2459 if (!qp_params.mtu) { 2460 /* Stay with current MTU */ 2461 if (qp->mtu) 2462 qp_params.mtu = qp->mtu; 2463 else 2464 qp_params.mtu = 2465 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2466 } 2467 2468 if (attr_mask & IB_QP_TIMEOUT) { 2469 SET_FIELD(qp_params.modify_flags, 2470 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2471 2472 /* The received timeout value is an exponent used like this: 2473 * "12.7.34 LOCAL ACK TIMEOUT 2474 * Value representing the transport (ACK) timeout for use by 2475 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2476 * The FW expects timeout in msec so we need to divide the usec 2477 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2478 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2479 * The value of zero means infinite so we use a 'max_t' to make 2480 * sure that sub 1 msec values will be configured as 1 msec. 2481 */ 2482 if (attr->timeout) 2483 qp_params.ack_timeout = 2484 1 << max_t(int, attr->timeout - 8, 0); 2485 else 2486 qp_params.ack_timeout = 0; 2487 } 2488 2489 if (attr_mask & IB_QP_RETRY_CNT) { 2490 SET_FIELD(qp_params.modify_flags, 2491 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2492 qp_params.retry_cnt = attr->retry_cnt; 2493 } 2494 2495 if (attr_mask & IB_QP_RNR_RETRY) { 2496 SET_FIELD(qp_params.modify_flags, 2497 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2498 qp_params.rnr_retry_cnt = attr->rnr_retry; 2499 } 2500 2501 if (attr_mask & IB_QP_RQ_PSN) { 2502 SET_FIELD(qp_params.modify_flags, 2503 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2504 qp_params.rq_psn = attr->rq_psn; 2505 qp->rq_psn = attr->rq_psn; 2506 } 2507 2508 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2509 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2510 rc = -EINVAL; 2511 DP_ERR(dev, 2512 "unsupported max_rd_atomic=%d, supported=%d\n", 2513 attr->max_rd_atomic, 2514 dev->attr.max_qp_req_rd_atomic_resc); 2515 goto err; 2516 } 2517 2518 SET_FIELD(qp_params.modify_flags, 2519 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2520 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2521 } 2522 2523 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2524 SET_FIELD(qp_params.modify_flags, 2525 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2526 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2527 } 2528 2529 if (attr_mask & IB_QP_SQ_PSN) { 2530 SET_FIELD(qp_params.modify_flags, 2531 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2532 qp_params.sq_psn = attr->sq_psn; 2533 qp->sq_psn = attr->sq_psn; 2534 } 2535 2536 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2537 if (attr->max_dest_rd_atomic > 2538 dev->attr.max_qp_resp_rd_atomic_resc) { 2539 DP_ERR(dev, 2540 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2541 attr->max_dest_rd_atomic, 2542 dev->attr.max_qp_resp_rd_atomic_resc); 2543 2544 rc = -EINVAL; 2545 goto err; 2546 } 2547 2548 SET_FIELD(qp_params.modify_flags, 2549 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2550 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2551 } 2552 2553 if (attr_mask & IB_QP_DEST_QPN) { 2554 SET_FIELD(qp_params.modify_flags, 2555 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2556 2557 qp_params.dest_qp = attr->dest_qp_num; 2558 qp->dest_qp_num = attr->dest_qp_num; 2559 } 2560 2561 cur_state = qp->state; 2562 2563 /* Update the QP state before the actual ramrod to prevent a race with 2564 * fast path. Modifying the QP state to error will cause the device to 2565 * flush the CQEs and while polling the flushed CQEs will considered as 2566 * a potential issue if the QP isn't in error state. 2567 */ 2568 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2569 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2570 qp->state = QED_ROCE_QP_STATE_ERR; 2571 2572 if (qp->qp_type != IB_QPT_GSI) 2573 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2574 qp->qed_qp, &qp_params); 2575 2576 if (attr_mask & IB_QP_STATE) { 2577 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2578 rc = qedr_update_qp_state(dev, qp, cur_state, 2579 qp_params.new_state); 2580 qp->state = qp_params.new_state; 2581 } 2582 2583 err: 2584 return rc; 2585 } 2586 2587 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2588 { 2589 int ib_qp_acc_flags = 0; 2590 2591 if (params->incoming_rdma_write_en) 2592 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2593 if (params->incoming_rdma_read_en) 2594 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2595 if (params->incoming_atomic_en) 2596 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2597 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2598 return ib_qp_acc_flags; 2599 } 2600 2601 int qedr_query_qp(struct ib_qp *ibqp, 2602 struct ib_qp_attr *qp_attr, 2603 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2604 { 2605 struct qed_rdma_query_qp_out_params params; 2606 struct qedr_qp *qp = get_qedr_qp(ibqp); 2607 struct qedr_dev *dev = qp->dev; 2608 int rc = 0; 2609 2610 memset(¶ms, 0, sizeof(params)); 2611 2612 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2613 if (rc) 2614 goto err; 2615 2616 memset(qp_attr, 0, sizeof(*qp_attr)); 2617 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2618 2619 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2620 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2621 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2622 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2623 qp_attr->rq_psn = params.rq_psn; 2624 qp_attr->sq_psn = params.sq_psn; 2625 qp_attr->dest_qp_num = params.dest_qp; 2626 2627 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2628 2629 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2630 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2631 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2632 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2633 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2634 qp_init_attr->cap = qp_attr->cap; 2635 2636 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2637 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2638 params.flow_label, qp->sgid_idx, 2639 params.hop_limit_ttl, params.traffic_class_tos); 2640 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2641 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2642 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2643 qp_attr->timeout = params.timeout; 2644 qp_attr->rnr_retry = params.rnr_retry; 2645 qp_attr->retry_cnt = params.retry_cnt; 2646 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2647 qp_attr->pkey_index = params.pkey_index; 2648 qp_attr->port_num = 1; 2649 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2650 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2651 qp_attr->alt_pkey_index = 0; 2652 qp_attr->alt_port_num = 0; 2653 qp_attr->alt_timeout = 0; 2654 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2655 2656 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2657 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2658 qp_attr->max_rd_atomic = params.max_rd_atomic; 2659 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2660 2661 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2662 qp_attr->cap.max_inline_data); 2663 2664 err: 2665 return rc; 2666 } 2667 2668 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2669 struct ib_udata *udata) 2670 { 2671 struct qedr_ucontext *ctx = 2672 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2673 ibucontext); 2674 int rc; 2675 2676 if (qp->qp_type != IB_QPT_GSI) { 2677 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2678 if (rc) 2679 return rc; 2680 } 2681 2682 if (qp->create_type == QEDR_QP_CREATE_USER) 2683 qedr_cleanup_user(dev, ctx, qp); 2684 else 2685 qedr_cleanup_kernel(dev, qp); 2686 2687 return 0; 2688 } 2689 2690 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2691 { 2692 struct qedr_qp *qp = get_qedr_qp(ibqp); 2693 struct qedr_dev *dev = qp->dev; 2694 struct ib_qp_attr attr; 2695 int attr_mask = 0; 2696 2697 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2698 qp, qp->qp_type); 2699 2700 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2701 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2702 (qp->state != QED_ROCE_QP_STATE_ERR) && 2703 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2704 2705 attr.qp_state = IB_QPS_ERR; 2706 attr_mask |= IB_QP_STATE; 2707 2708 /* Change the QP state to ERROR */ 2709 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2710 } 2711 } else { 2712 /* If connection establishment started the WAIT_FOR_CONNECT 2713 * bit will be on and we need to Wait for the establishment 2714 * to complete before destroying the qp. 2715 */ 2716 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2717 &qp->iwarp_cm_flags)) 2718 wait_for_completion(&qp->iwarp_cm_comp); 2719 2720 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2721 * bit will be on, and we need to wait for the disconnect to 2722 * complete before continuing. We can use the same completion, 2723 * iwarp_cm_comp, since this is the only place that waits for 2724 * this completion and it is sequential. In addition, 2725 * disconnect can't occur before the connection is fully 2726 * established, therefore if WAIT_FOR_DISCONNECT is on it 2727 * means WAIT_FOR_CONNECT is also on and the completion for 2728 * CONNECT already occurred. 2729 */ 2730 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2731 &qp->iwarp_cm_flags)) 2732 wait_for_completion(&qp->iwarp_cm_comp); 2733 } 2734 2735 if (qp->qp_type == IB_QPT_GSI) 2736 qedr_destroy_gsi_qp(dev); 2737 2738 /* We need to remove the entry from the xarray before we release the 2739 * qp_id to avoid a race of the qp_id being reallocated and failing 2740 * on xa_insert 2741 */ 2742 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2743 xa_erase(&dev->qps, qp->qp_id); 2744 2745 qedr_free_qp_resources(dev, qp, udata); 2746 2747 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2748 qedr_iw_qp_rem_ref(&qp->ibqp); 2749 2750 return 0; 2751 } 2752 2753 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags, 2754 struct ib_udata *udata) 2755 { 2756 struct qedr_ah *ah = get_qedr_ah(ibah); 2757 2758 rdma_copy_ah_attr(&ah->attr, attr); 2759 2760 return 0; 2761 } 2762 2763 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2764 { 2765 struct qedr_ah *ah = get_qedr_ah(ibah); 2766 2767 rdma_destroy_ah_attr(&ah->attr); 2768 } 2769 2770 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2771 { 2772 struct qedr_pbl *pbl, *tmp; 2773 2774 if (info->pbl_table) 2775 list_add_tail(&info->pbl_table->list_entry, 2776 &info->free_pbl_list); 2777 2778 if (!list_empty(&info->inuse_pbl_list)) 2779 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2780 2781 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2782 list_del(&pbl->list_entry); 2783 qedr_free_pbl(dev, &info->pbl_info, pbl); 2784 } 2785 } 2786 2787 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2788 size_t page_list_len, bool two_layered) 2789 { 2790 struct qedr_pbl *tmp; 2791 int rc; 2792 2793 INIT_LIST_HEAD(&info->free_pbl_list); 2794 INIT_LIST_HEAD(&info->inuse_pbl_list); 2795 2796 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2797 page_list_len, two_layered); 2798 if (rc) 2799 goto done; 2800 2801 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2802 if (IS_ERR(info->pbl_table)) { 2803 rc = PTR_ERR(info->pbl_table); 2804 goto done; 2805 } 2806 2807 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2808 &info->pbl_table->pa); 2809 2810 /* in usual case we use 2 PBLs, so we add one to free 2811 * list and allocating another one 2812 */ 2813 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2814 if (IS_ERR(tmp)) { 2815 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2816 goto done; 2817 } 2818 2819 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2820 2821 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2822 2823 done: 2824 if (rc) 2825 free_mr_info(dev, info); 2826 2827 return rc; 2828 } 2829 2830 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2831 u64 usr_addr, int acc, struct ib_udata *udata) 2832 { 2833 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2834 struct qedr_mr *mr; 2835 struct qedr_pd *pd; 2836 int rc = -ENOMEM; 2837 2838 pd = get_qedr_pd(ibpd); 2839 DP_DEBUG(dev, QEDR_MSG_MR, 2840 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2841 pd->pd_id, start, len, usr_addr, acc); 2842 2843 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2844 return ERR_PTR(-EINVAL); 2845 2846 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2847 if (!mr) 2848 return ERR_PTR(rc); 2849 2850 mr->type = QEDR_MR_USER; 2851 2852 mr->umem = ib_umem_get(ibpd->device, start, len, acc); 2853 if (IS_ERR(mr->umem)) { 2854 rc = -EFAULT; 2855 goto err0; 2856 } 2857 2858 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2859 if (rc) 2860 goto err1; 2861 2862 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2863 &mr->info.pbl_info, PAGE_SHIFT); 2864 2865 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2866 if (rc) { 2867 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2868 goto err1; 2869 } 2870 2871 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2872 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2873 mr->hw_mr.key = 0; 2874 mr->hw_mr.pd = pd->pd_id; 2875 mr->hw_mr.local_read = 1; 2876 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2877 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2878 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2879 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2880 mr->hw_mr.mw_bind = false; 2881 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2882 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2883 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2884 mr->hw_mr.page_size_log = PAGE_SHIFT; 2885 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2886 mr->hw_mr.length = len; 2887 mr->hw_mr.vaddr = usr_addr; 2888 mr->hw_mr.zbva = false; 2889 mr->hw_mr.phy_mr = false; 2890 mr->hw_mr.dma_mr = false; 2891 2892 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2893 if (rc) { 2894 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2895 goto err2; 2896 } 2897 2898 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2899 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2900 mr->hw_mr.remote_atomic) 2901 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2902 2903 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2904 mr->ibmr.lkey); 2905 return &mr->ibmr; 2906 2907 err2: 2908 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2909 err1: 2910 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2911 err0: 2912 kfree(mr); 2913 return ERR_PTR(rc); 2914 } 2915 2916 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 2917 { 2918 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2919 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2920 int rc = 0; 2921 2922 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2923 if (rc) 2924 return rc; 2925 2926 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2927 2928 if (mr->type != QEDR_MR_DMA) 2929 free_mr_info(dev, &mr->info); 2930 2931 /* it could be user registered memory. */ 2932 ib_umem_release(mr->umem); 2933 2934 kfree(mr); 2935 2936 return rc; 2937 } 2938 2939 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2940 int max_page_list_len) 2941 { 2942 struct qedr_pd *pd = get_qedr_pd(ibpd); 2943 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2944 struct qedr_mr *mr; 2945 int rc = -ENOMEM; 2946 2947 DP_DEBUG(dev, QEDR_MSG_MR, 2948 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2949 max_page_list_len); 2950 2951 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2952 if (!mr) 2953 return ERR_PTR(rc); 2954 2955 mr->dev = dev; 2956 mr->type = QEDR_MR_FRMR; 2957 2958 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2959 if (rc) 2960 goto err0; 2961 2962 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2963 if (rc) { 2964 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2965 goto err0; 2966 } 2967 2968 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2969 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2970 mr->hw_mr.key = 0; 2971 mr->hw_mr.pd = pd->pd_id; 2972 mr->hw_mr.local_read = 1; 2973 mr->hw_mr.local_write = 0; 2974 mr->hw_mr.remote_read = 0; 2975 mr->hw_mr.remote_write = 0; 2976 mr->hw_mr.remote_atomic = 0; 2977 mr->hw_mr.mw_bind = false; 2978 mr->hw_mr.pbl_ptr = 0; 2979 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2980 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2981 mr->hw_mr.fbo = 0; 2982 mr->hw_mr.length = 0; 2983 mr->hw_mr.vaddr = 0; 2984 mr->hw_mr.zbva = false; 2985 mr->hw_mr.phy_mr = true; 2986 mr->hw_mr.dma_mr = false; 2987 2988 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2989 if (rc) { 2990 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2991 goto err1; 2992 } 2993 2994 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2995 mr->ibmr.rkey = mr->ibmr.lkey; 2996 2997 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2998 return mr; 2999 3000 err1: 3001 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3002 err0: 3003 kfree(mr); 3004 return ERR_PTR(rc); 3005 } 3006 3007 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 3008 u32 max_num_sg, struct ib_udata *udata) 3009 { 3010 struct qedr_mr *mr; 3011 3012 if (mr_type != IB_MR_TYPE_MEM_REG) 3013 return ERR_PTR(-EINVAL); 3014 3015 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3016 3017 if (IS_ERR(mr)) 3018 return ERR_PTR(-EINVAL); 3019 3020 return &mr->ibmr; 3021 } 3022 3023 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3024 { 3025 struct qedr_mr *mr = get_qedr_mr(ibmr); 3026 struct qedr_pbl *pbl_table; 3027 struct regpair *pbe; 3028 u32 pbes_in_page; 3029 3030 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3031 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3032 return -ENOMEM; 3033 } 3034 3035 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3036 mr->npages, addr); 3037 3038 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3039 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3040 pbe = (struct regpair *)pbl_table->va; 3041 pbe += mr->npages % pbes_in_page; 3042 pbe->lo = cpu_to_le32((u32)addr); 3043 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3044 3045 mr->npages++; 3046 3047 return 0; 3048 } 3049 3050 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3051 { 3052 int work = info->completed - info->completed_handled - 1; 3053 3054 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3055 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3056 struct qedr_pbl *pbl; 3057 3058 /* Free all the page list that are possible to be freed 3059 * (all the ones that were invalidated), under the assumption 3060 * that if an FMR was completed successfully that means that 3061 * if there was an invalidate operation before it also ended 3062 */ 3063 pbl = list_first_entry(&info->inuse_pbl_list, 3064 struct qedr_pbl, list_entry); 3065 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3066 info->completed_handled++; 3067 } 3068 } 3069 3070 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3071 int sg_nents, unsigned int *sg_offset) 3072 { 3073 struct qedr_mr *mr = get_qedr_mr(ibmr); 3074 3075 mr->npages = 0; 3076 3077 handle_completed_mrs(mr->dev, &mr->info); 3078 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3079 } 3080 3081 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3082 { 3083 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3084 struct qedr_pd *pd = get_qedr_pd(ibpd); 3085 struct qedr_mr *mr; 3086 int rc; 3087 3088 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3089 if (!mr) 3090 return ERR_PTR(-ENOMEM); 3091 3092 mr->type = QEDR_MR_DMA; 3093 3094 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3095 if (rc) { 3096 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 3097 goto err1; 3098 } 3099 3100 /* index only, 18 bit long, lkey = itid << 8 | key */ 3101 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3102 mr->hw_mr.pd = pd->pd_id; 3103 mr->hw_mr.local_read = 1; 3104 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3105 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3106 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3107 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3108 mr->hw_mr.dma_mr = true; 3109 3110 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3111 if (rc) { 3112 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3113 goto err2; 3114 } 3115 3116 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3117 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3118 mr->hw_mr.remote_atomic) 3119 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3120 3121 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3122 return &mr->ibmr; 3123 3124 err2: 3125 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3126 err1: 3127 kfree(mr); 3128 return ERR_PTR(rc); 3129 } 3130 3131 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3132 { 3133 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3134 } 3135 3136 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3137 { 3138 int i, len = 0; 3139 3140 for (i = 0; i < num_sge; i++) 3141 len += sg_list[i].length; 3142 3143 return len; 3144 } 3145 3146 static void swap_wqe_data64(u64 *p) 3147 { 3148 int i; 3149 3150 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3151 *p = cpu_to_be64(cpu_to_le64(*p)); 3152 } 3153 3154 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3155 struct qedr_qp *qp, u8 *wqe_size, 3156 const struct ib_send_wr *wr, 3157 const struct ib_send_wr **bad_wr, 3158 u8 *bits, u8 bit) 3159 { 3160 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3161 char *seg_prt, *wqe; 3162 int i, seg_siz; 3163 3164 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3165 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3166 *bad_wr = wr; 3167 return 0; 3168 } 3169 3170 if (!data_size) 3171 return data_size; 3172 3173 *bits |= bit; 3174 3175 seg_prt = NULL; 3176 wqe = NULL; 3177 seg_siz = 0; 3178 3179 /* Copy data inline */ 3180 for (i = 0; i < wr->num_sge; i++) { 3181 u32 len = wr->sg_list[i].length; 3182 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3183 3184 while (len > 0) { 3185 u32 cur; 3186 3187 /* New segment required */ 3188 if (!seg_siz) { 3189 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3190 seg_prt = wqe; 3191 seg_siz = sizeof(struct rdma_sq_common_wqe); 3192 (*wqe_size)++; 3193 } 3194 3195 /* Calculate currently allowed length */ 3196 cur = min_t(u32, len, seg_siz); 3197 memcpy(seg_prt, src, cur); 3198 3199 /* Update segment variables */ 3200 seg_prt += cur; 3201 seg_siz -= cur; 3202 3203 /* Update sge variables */ 3204 src += cur; 3205 len -= cur; 3206 3207 /* Swap fully-completed segments */ 3208 if (!seg_siz) 3209 swap_wqe_data64((u64 *)wqe); 3210 } 3211 } 3212 3213 /* swap last not completed segment */ 3214 if (seg_siz) 3215 swap_wqe_data64((u64 *)wqe); 3216 3217 return data_size; 3218 } 3219 3220 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3221 do { \ 3222 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3223 (sge)->length = cpu_to_le32(vlength); \ 3224 (sge)->flags = cpu_to_le32(vflags); \ 3225 } while (0) 3226 3227 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3228 do { \ 3229 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3230 (hdr)->num_sges = num_sge; \ 3231 } while (0) 3232 3233 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3234 do { \ 3235 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3236 (sge)->length = cpu_to_le32(vlength); \ 3237 (sge)->l_key = cpu_to_le32(vlkey); \ 3238 } while (0) 3239 3240 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3241 const struct ib_send_wr *wr) 3242 { 3243 u32 data_size = 0; 3244 int i; 3245 3246 for (i = 0; i < wr->num_sge; i++) { 3247 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3248 3249 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3250 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3251 sge->length = cpu_to_le32(wr->sg_list[i].length); 3252 data_size += wr->sg_list[i].length; 3253 } 3254 3255 if (wqe_size) 3256 *wqe_size += wr->num_sge; 3257 3258 return data_size; 3259 } 3260 3261 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3262 struct qedr_qp *qp, 3263 struct rdma_sq_rdma_wqe_1st *rwqe, 3264 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3265 const struct ib_send_wr *wr, 3266 const struct ib_send_wr **bad_wr) 3267 { 3268 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3269 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3270 3271 if (wr->send_flags & IB_SEND_INLINE && 3272 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3273 wr->opcode == IB_WR_RDMA_WRITE)) { 3274 u8 flags = 0; 3275 3276 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3277 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3278 bad_wr, &rwqe->flags, flags); 3279 } 3280 3281 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3282 } 3283 3284 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3285 struct qedr_qp *qp, 3286 struct rdma_sq_send_wqe_1st *swqe, 3287 struct rdma_sq_send_wqe_2st *swqe2, 3288 const struct ib_send_wr *wr, 3289 const struct ib_send_wr **bad_wr) 3290 { 3291 memset(swqe2, 0, sizeof(*swqe2)); 3292 if (wr->send_flags & IB_SEND_INLINE) { 3293 u8 flags = 0; 3294 3295 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3296 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3297 bad_wr, &swqe->flags, flags); 3298 } 3299 3300 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3301 } 3302 3303 static int qedr_prepare_reg(struct qedr_qp *qp, 3304 struct rdma_sq_fmr_wqe_1st *fwqe1, 3305 const struct ib_reg_wr *wr) 3306 { 3307 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3308 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3309 3310 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3311 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3312 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3313 fwqe1->l_key = wr->key; 3314 3315 fwqe2->access_ctrl = 0; 3316 3317 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3318 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3319 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3320 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3321 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3322 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3323 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3324 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3325 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3326 fwqe2->fmr_ctrl = 0; 3327 3328 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3329 ilog2(mr->ibmr.page_size) - 12); 3330 3331 fwqe2->length_hi = 0; 3332 fwqe2->length_lo = mr->ibmr.length; 3333 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3334 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3335 3336 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3337 3338 return 0; 3339 } 3340 3341 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3342 { 3343 switch (opcode) { 3344 case IB_WR_RDMA_WRITE: 3345 case IB_WR_RDMA_WRITE_WITH_IMM: 3346 return IB_WC_RDMA_WRITE; 3347 case IB_WR_SEND_WITH_IMM: 3348 case IB_WR_SEND: 3349 case IB_WR_SEND_WITH_INV: 3350 return IB_WC_SEND; 3351 case IB_WR_RDMA_READ: 3352 case IB_WR_RDMA_READ_WITH_INV: 3353 return IB_WC_RDMA_READ; 3354 case IB_WR_ATOMIC_CMP_AND_SWP: 3355 return IB_WC_COMP_SWAP; 3356 case IB_WR_ATOMIC_FETCH_AND_ADD: 3357 return IB_WC_FETCH_ADD; 3358 case IB_WR_REG_MR: 3359 return IB_WC_REG_MR; 3360 case IB_WR_LOCAL_INV: 3361 return IB_WC_LOCAL_INV; 3362 default: 3363 return IB_WC_SEND; 3364 } 3365 } 3366 3367 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3368 const struct ib_send_wr *wr) 3369 { 3370 int wq_is_full, err_wr, pbl_is_full; 3371 struct qedr_dev *dev = qp->dev; 3372 3373 /* prevent SQ overflow and/or processing of a bad WR */ 3374 err_wr = wr->num_sge > qp->sq.max_sges; 3375 wq_is_full = qedr_wq_is_full(&qp->sq); 3376 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3377 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3378 if (wq_is_full || err_wr || pbl_is_full) { 3379 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3380 DP_ERR(dev, 3381 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3382 qp); 3383 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3384 } 3385 3386 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3387 DP_ERR(dev, 3388 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3389 qp); 3390 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3391 } 3392 3393 if (pbl_is_full && 3394 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3395 DP_ERR(dev, 3396 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3397 qp); 3398 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3399 } 3400 return false; 3401 } 3402 return true; 3403 } 3404 3405 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3406 const struct ib_send_wr **bad_wr) 3407 { 3408 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3409 struct qedr_qp *qp = get_qedr_qp(ibqp); 3410 struct rdma_sq_atomic_wqe_1st *awqe1; 3411 struct rdma_sq_atomic_wqe_2nd *awqe2; 3412 struct rdma_sq_atomic_wqe_3rd *awqe3; 3413 struct rdma_sq_send_wqe_2st *swqe2; 3414 struct rdma_sq_local_inv_wqe *iwqe; 3415 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3416 struct rdma_sq_send_wqe_1st *swqe; 3417 struct rdma_sq_rdma_wqe_1st *rwqe; 3418 struct rdma_sq_fmr_wqe_1st *fwqe1; 3419 struct rdma_sq_common_wqe *wqe; 3420 u32 length; 3421 int rc = 0; 3422 bool comp; 3423 3424 if (!qedr_can_post_send(qp, wr)) { 3425 *bad_wr = wr; 3426 return -ENOMEM; 3427 } 3428 3429 wqe = qed_chain_produce(&qp->sq.pbl); 3430 qp->wqe_wr_id[qp->sq.prod].signaled = 3431 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3432 3433 wqe->flags = 0; 3434 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3435 !!(wr->send_flags & IB_SEND_SOLICITED)); 3436 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3437 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3438 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3439 !!(wr->send_flags & IB_SEND_FENCE)); 3440 wqe->prev_wqe_size = qp->prev_wqe_size; 3441 3442 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3443 3444 switch (wr->opcode) { 3445 case IB_WR_SEND_WITH_IMM: 3446 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3447 rc = -EINVAL; 3448 *bad_wr = wr; 3449 break; 3450 } 3451 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3452 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3453 swqe->wqe_size = 2; 3454 swqe2 = qed_chain_produce(&qp->sq.pbl); 3455 3456 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3457 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3458 wr, bad_wr); 3459 swqe->length = cpu_to_le32(length); 3460 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3461 qp->prev_wqe_size = swqe->wqe_size; 3462 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3463 break; 3464 case IB_WR_SEND: 3465 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3466 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3467 3468 swqe->wqe_size = 2; 3469 swqe2 = qed_chain_produce(&qp->sq.pbl); 3470 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3471 wr, bad_wr); 3472 swqe->length = cpu_to_le32(length); 3473 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3474 qp->prev_wqe_size = swqe->wqe_size; 3475 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3476 break; 3477 case IB_WR_SEND_WITH_INV: 3478 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3479 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3480 swqe2 = qed_chain_produce(&qp->sq.pbl); 3481 swqe->wqe_size = 2; 3482 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3483 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3484 wr, bad_wr); 3485 swqe->length = cpu_to_le32(length); 3486 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3487 qp->prev_wqe_size = swqe->wqe_size; 3488 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3489 break; 3490 3491 case IB_WR_RDMA_WRITE_WITH_IMM: 3492 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3493 rc = -EINVAL; 3494 *bad_wr = wr; 3495 break; 3496 } 3497 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3498 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3499 3500 rwqe->wqe_size = 2; 3501 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3502 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3503 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3504 wr, bad_wr); 3505 rwqe->length = cpu_to_le32(length); 3506 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3507 qp->prev_wqe_size = rwqe->wqe_size; 3508 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3509 break; 3510 case IB_WR_RDMA_WRITE: 3511 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3512 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3513 3514 rwqe->wqe_size = 2; 3515 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3516 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3517 wr, bad_wr); 3518 rwqe->length = cpu_to_le32(length); 3519 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3520 qp->prev_wqe_size = rwqe->wqe_size; 3521 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3522 break; 3523 case IB_WR_RDMA_READ_WITH_INV: 3524 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3525 /* fallthrough -- same is identical to RDMA READ */ 3526 3527 case IB_WR_RDMA_READ: 3528 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3529 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3530 3531 rwqe->wqe_size = 2; 3532 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3533 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3534 wr, bad_wr); 3535 rwqe->length = cpu_to_le32(length); 3536 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3537 qp->prev_wqe_size = rwqe->wqe_size; 3538 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3539 break; 3540 3541 case IB_WR_ATOMIC_CMP_AND_SWP: 3542 case IB_WR_ATOMIC_FETCH_AND_ADD: 3543 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3544 awqe1->wqe_size = 4; 3545 3546 awqe2 = qed_chain_produce(&qp->sq.pbl); 3547 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3548 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3549 3550 awqe3 = qed_chain_produce(&qp->sq.pbl); 3551 3552 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3553 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3554 DMA_REGPAIR_LE(awqe3->swap_data, 3555 atomic_wr(wr)->compare_add); 3556 } else { 3557 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3558 DMA_REGPAIR_LE(awqe3->swap_data, 3559 atomic_wr(wr)->swap); 3560 DMA_REGPAIR_LE(awqe3->cmp_data, 3561 atomic_wr(wr)->compare_add); 3562 } 3563 3564 qedr_prepare_sq_sges(qp, NULL, wr); 3565 3566 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3567 qp->prev_wqe_size = awqe1->wqe_size; 3568 break; 3569 3570 case IB_WR_LOCAL_INV: 3571 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3572 iwqe->wqe_size = 1; 3573 3574 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3575 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3576 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3577 qp->prev_wqe_size = iwqe->wqe_size; 3578 break; 3579 case IB_WR_REG_MR: 3580 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3581 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3582 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3583 fwqe1->wqe_size = 2; 3584 3585 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3586 if (rc) { 3587 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3588 *bad_wr = wr; 3589 break; 3590 } 3591 3592 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3593 qp->prev_wqe_size = fwqe1->wqe_size; 3594 break; 3595 default: 3596 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3597 rc = -EINVAL; 3598 *bad_wr = wr; 3599 break; 3600 } 3601 3602 if (*bad_wr) { 3603 u16 value; 3604 3605 /* Restore prod to its position before 3606 * this WR was processed 3607 */ 3608 value = le16_to_cpu(qp->sq.db_data.data.value); 3609 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3610 3611 /* Restore prev_wqe_size */ 3612 qp->prev_wqe_size = wqe->prev_wqe_size; 3613 rc = -EINVAL; 3614 DP_ERR(dev, "POST SEND FAILED\n"); 3615 } 3616 3617 return rc; 3618 } 3619 3620 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3621 const struct ib_send_wr **bad_wr) 3622 { 3623 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3624 struct qedr_qp *qp = get_qedr_qp(ibqp); 3625 unsigned long flags; 3626 int rc = 0; 3627 3628 *bad_wr = NULL; 3629 3630 if (qp->qp_type == IB_QPT_GSI) 3631 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3632 3633 spin_lock_irqsave(&qp->q_lock, flags); 3634 3635 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3636 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3637 (qp->state != QED_ROCE_QP_STATE_ERR) && 3638 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3639 spin_unlock_irqrestore(&qp->q_lock, flags); 3640 *bad_wr = wr; 3641 DP_DEBUG(dev, QEDR_MSG_CQ, 3642 "QP in wrong state! QP icid=0x%x state %d\n", 3643 qp->icid, qp->state); 3644 return -EINVAL; 3645 } 3646 } 3647 3648 while (wr) { 3649 rc = __qedr_post_send(ibqp, wr, bad_wr); 3650 if (rc) 3651 break; 3652 3653 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3654 3655 qedr_inc_sw_prod(&qp->sq); 3656 3657 qp->sq.db_data.data.value++; 3658 3659 wr = wr->next; 3660 } 3661 3662 /* Trigger doorbell 3663 * If there was a failure in the first WR then it will be triggered in 3664 * vane. However this is not harmful (as long as the producer value is 3665 * unchanged). For performance reasons we avoid checking for this 3666 * redundant doorbell. 3667 * 3668 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3669 * soon as we give the doorbell, we could get a completion 3670 * for this wr, therefore we need to make sure that the 3671 * memory is updated before giving the doorbell. 3672 * During qedr_poll_cq, rmb is called before accessing the 3673 * cqe. This covers for the smp_rmb as well. 3674 */ 3675 smp_wmb(); 3676 writel(qp->sq.db_data.raw, qp->sq.db); 3677 3678 spin_unlock_irqrestore(&qp->q_lock, flags); 3679 3680 return rc; 3681 } 3682 3683 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3684 { 3685 u32 used; 3686 3687 /* Calculate number of elements used based on producer 3688 * count and consumer count and subtract it from max 3689 * work request supported so that we get elements left. 3690 */ 3691 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 3692 3693 return hw_srq->max_wr - used; 3694 } 3695 3696 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3697 const struct ib_recv_wr **bad_wr) 3698 { 3699 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3700 struct qedr_srq_hwq_info *hw_srq; 3701 struct qedr_dev *dev = srq->dev; 3702 struct qed_chain *pbl; 3703 unsigned long flags; 3704 int status = 0; 3705 u32 num_sge; 3706 u32 offset; 3707 3708 spin_lock_irqsave(&srq->lock, flags); 3709 3710 hw_srq = &srq->hw_srq; 3711 pbl = &srq->hw_srq.pbl; 3712 while (wr) { 3713 struct rdma_srq_wqe_header *hdr; 3714 int i; 3715 3716 if (!qedr_srq_elem_left(hw_srq) || 3717 wr->num_sge > srq->hw_srq.max_sges) { 3718 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3719 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 3720 wr->num_sge, srq->hw_srq.max_sges); 3721 status = -ENOMEM; 3722 *bad_wr = wr; 3723 break; 3724 } 3725 3726 hdr = qed_chain_produce(pbl); 3727 num_sge = wr->num_sge; 3728 /* Set number of sge and work request id in header */ 3729 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3730 3731 srq->hw_srq.wr_prod_cnt++; 3732 hw_srq->wqe_prod++; 3733 hw_srq->sge_prod++; 3734 3735 DP_DEBUG(dev, QEDR_MSG_SRQ, 3736 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3737 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3738 3739 for (i = 0; i < wr->num_sge; i++) { 3740 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3741 3742 /* Set SGE length, lkey and address */ 3743 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3744 wr->sg_list[i].length, wr->sg_list[i].lkey); 3745 3746 DP_DEBUG(dev, QEDR_MSG_SRQ, 3747 "[%d]: len %d key %x addr %x:%x\n", 3748 i, srq_sge->length, srq_sge->l_key, 3749 srq_sge->addr.hi, srq_sge->addr.lo); 3750 hw_srq->sge_prod++; 3751 } 3752 3753 /* Flush WQE and SGE information before 3754 * updating producer. 3755 */ 3756 wmb(); 3757 3758 /* SRQ producer is 8 bytes. Need to update SGE producer index 3759 * in first 4 bytes and need to update WQE producer in 3760 * next 4 bytes. 3761 */ 3762 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; 3763 offset = offsetof(struct rdma_srq_producers, wqe_prod); 3764 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 3765 hw_srq->wqe_prod; 3766 3767 /* Flush producer after updating it. */ 3768 wmb(); 3769 wr = wr->next; 3770 } 3771 3772 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3773 qed_chain_get_elem_left(pbl)); 3774 spin_unlock_irqrestore(&srq->lock, flags); 3775 3776 return status; 3777 } 3778 3779 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3780 const struct ib_recv_wr **bad_wr) 3781 { 3782 struct qedr_qp *qp = get_qedr_qp(ibqp); 3783 struct qedr_dev *dev = qp->dev; 3784 unsigned long flags; 3785 int status = 0; 3786 3787 if (qp->qp_type == IB_QPT_GSI) 3788 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3789 3790 spin_lock_irqsave(&qp->q_lock, flags); 3791 3792 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3793 spin_unlock_irqrestore(&qp->q_lock, flags); 3794 *bad_wr = wr; 3795 return -EINVAL; 3796 } 3797 3798 while (wr) { 3799 int i; 3800 3801 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3802 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3803 wr->num_sge > qp->rq.max_sges) { 3804 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3805 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3806 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3807 qp->rq.max_sges); 3808 status = -ENOMEM; 3809 *bad_wr = wr; 3810 break; 3811 } 3812 for (i = 0; i < wr->num_sge; i++) { 3813 u32 flags = 0; 3814 struct rdma_rq_sge *rqe = 3815 qed_chain_produce(&qp->rq.pbl); 3816 3817 /* First one must include the number 3818 * of SGE in the list 3819 */ 3820 if (!i) 3821 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3822 wr->num_sge); 3823 3824 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3825 wr->sg_list[i].lkey); 3826 3827 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3828 wr->sg_list[i].length, flags); 3829 } 3830 3831 /* Special case of no sges. FW requires between 1-4 sges... 3832 * in this case we need to post 1 sge with length zero. this is 3833 * because rdma write with immediate consumes an RQ. 3834 */ 3835 if (!wr->num_sge) { 3836 u32 flags = 0; 3837 struct rdma_rq_sge *rqe = 3838 qed_chain_produce(&qp->rq.pbl); 3839 3840 /* First one must include the number 3841 * of SGE in the list 3842 */ 3843 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3844 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3845 3846 RQ_SGE_SET(rqe, 0, 0, flags); 3847 i = 1; 3848 } 3849 3850 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3851 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3852 3853 qedr_inc_sw_prod(&qp->rq); 3854 3855 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3856 * soon as we give the doorbell, we could get a completion 3857 * for this wr, therefore we need to make sure that the 3858 * memory is update before giving the doorbell. 3859 * During qedr_poll_cq, rmb is called before accessing the 3860 * cqe. This covers for the smp_rmb as well. 3861 */ 3862 smp_wmb(); 3863 3864 qp->rq.db_data.data.value++; 3865 3866 writel(qp->rq.db_data.raw, qp->rq.db); 3867 3868 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3869 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3870 } 3871 3872 wr = wr->next; 3873 } 3874 3875 spin_unlock_irqrestore(&qp->q_lock, flags); 3876 3877 return status; 3878 } 3879 3880 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3881 { 3882 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3883 3884 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3885 cq->pbl_toggle; 3886 } 3887 3888 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3889 { 3890 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3891 struct qedr_qp *qp; 3892 3893 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3894 resp_cqe->qp_handle.lo, 3895 u64); 3896 return qp; 3897 } 3898 3899 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3900 { 3901 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3902 3903 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3904 } 3905 3906 /* Return latest CQE (needs processing) */ 3907 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3908 { 3909 return cq->latest_cqe; 3910 } 3911 3912 /* In fmr we need to increase the number of fmr completed counter for the fmr 3913 * algorithm determining whether we can free a pbl or not. 3914 * we need to perform this whether the work request was signaled or not. for 3915 * this purpose we call this function from the condition that checks if a wr 3916 * should be skipped, to make sure we don't miss it ( possibly this fmr 3917 * operation was not signalted) 3918 */ 3919 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3920 { 3921 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3922 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3923 } 3924 3925 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3926 struct qedr_cq *cq, int num_entries, 3927 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3928 int force) 3929 { 3930 u16 cnt = 0; 3931 3932 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3933 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3934 qedr_chk_if_fmr(qp); 3935 /* skip WC */ 3936 goto next_cqe; 3937 } 3938 3939 /* fill WC */ 3940 wc->status = status; 3941 wc->vendor_err = 0; 3942 wc->wc_flags = 0; 3943 wc->src_qp = qp->id; 3944 wc->qp = &qp->ibqp; 3945 3946 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3947 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3948 3949 switch (wc->opcode) { 3950 case IB_WC_RDMA_WRITE: 3951 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3952 break; 3953 case IB_WC_COMP_SWAP: 3954 case IB_WC_FETCH_ADD: 3955 wc->byte_len = 8; 3956 break; 3957 case IB_WC_REG_MR: 3958 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3959 break; 3960 case IB_WC_RDMA_READ: 3961 case IB_WC_SEND: 3962 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3963 break; 3964 default: 3965 break; 3966 } 3967 3968 num_entries--; 3969 wc++; 3970 cnt++; 3971 next_cqe: 3972 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3973 qed_chain_consume(&qp->sq.pbl); 3974 qedr_inc_sw_cons(&qp->sq); 3975 } 3976 3977 return cnt; 3978 } 3979 3980 static int qedr_poll_cq_req(struct qedr_dev *dev, 3981 struct qedr_qp *qp, struct qedr_cq *cq, 3982 int num_entries, struct ib_wc *wc, 3983 struct rdma_cqe_requester *req) 3984 { 3985 int cnt = 0; 3986 3987 switch (req->status) { 3988 case RDMA_CQE_REQ_STS_OK: 3989 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3990 IB_WC_SUCCESS, 0); 3991 break; 3992 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3993 if (qp->state != QED_ROCE_QP_STATE_ERR) 3994 DP_DEBUG(dev, QEDR_MSG_CQ, 3995 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3996 cq->icid, qp->icid); 3997 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3998 IB_WC_WR_FLUSH_ERR, 1); 3999 break; 4000 default: 4001 /* process all WQE before the cosumer */ 4002 qp->state = QED_ROCE_QP_STATE_ERR; 4003 cnt = process_req(dev, qp, cq, num_entries, wc, 4004 req->sq_cons - 1, IB_WC_SUCCESS, 0); 4005 wc += cnt; 4006 /* if we have extra WC fill it with actual error info */ 4007 if (cnt < num_entries) { 4008 enum ib_wc_status wc_status; 4009 4010 switch (req->status) { 4011 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4012 DP_ERR(dev, 4013 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4014 cq->icid, qp->icid); 4015 wc_status = IB_WC_BAD_RESP_ERR; 4016 break; 4017 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4018 DP_ERR(dev, 4019 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4020 cq->icid, qp->icid); 4021 wc_status = IB_WC_LOC_LEN_ERR; 4022 break; 4023 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4024 DP_ERR(dev, 4025 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4026 cq->icid, qp->icid); 4027 wc_status = IB_WC_LOC_QP_OP_ERR; 4028 break; 4029 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4030 DP_ERR(dev, 4031 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4032 cq->icid, qp->icid); 4033 wc_status = IB_WC_LOC_PROT_ERR; 4034 break; 4035 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4036 DP_ERR(dev, 4037 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4038 cq->icid, qp->icid); 4039 wc_status = IB_WC_MW_BIND_ERR; 4040 break; 4041 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4042 DP_ERR(dev, 4043 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4044 cq->icid, qp->icid); 4045 wc_status = IB_WC_REM_INV_REQ_ERR; 4046 break; 4047 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4048 DP_ERR(dev, 4049 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4050 cq->icid, qp->icid); 4051 wc_status = IB_WC_REM_ACCESS_ERR; 4052 break; 4053 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4054 DP_ERR(dev, 4055 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4056 cq->icid, qp->icid); 4057 wc_status = IB_WC_REM_OP_ERR; 4058 break; 4059 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4060 DP_ERR(dev, 4061 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4062 cq->icid, qp->icid); 4063 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4064 break; 4065 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4066 DP_ERR(dev, 4067 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4068 cq->icid, qp->icid); 4069 wc_status = IB_WC_RETRY_EXC_ERR; 4070 break; 4071 default: 4072 DP_ERR(dev, 4073 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4074 cq->icid, qp->icid); 4075 wc_status = IB_WC_GENERAL_ERR; 4076 } 4077 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4078 wc_status, 1); 4079 } 4080 } 4081 4082 return cnt; 4083 } 4084 4085 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4086 { 4087 switch (status) { 4088 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4089 return IB_WC_LOC_ACCESS_ERR; 4090 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4091 return IB_WC_LOC_LEN_ERR; 4092 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4093 return IB_WC_LOC_QP_OP_ERR; 4094 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4095 return IB_WC_LOC_PROT_ERR; 4096 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4097 return IB_WC_MW_BIND_ERR; 4098 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4099 return IB_WC_REM_INV_RD_REQ_ERR; 4100 case RDMA_CQE_RESP_STS_OK: 4101 return IB_WC_SUCCESS; 4102 default: 4103 return IB_WC_GENERAL_ERR; 4104 } 4105 } 4106 4107 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4108 struct ib_wc *wc) 4109 { 4110 wc->status = IB_WC_SUCCESS; 4111 wc->byte_len = le32_to_cpu(resp->length); 4112 4113 if (resp->flags & QEDR_RESP_IMM) { 4114 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4115 wc->wc_flags |= IB_WC_WITH_IMM; 4116 4117 if (resp->flags & QEDR_RESP_RDMA) 4118 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4119 4120 if (resp->flags & QEDR_RESP_INV) 4121 return -EINVAL; 4122 4123 } else if (resp->flags & QEDR_RESP_INV) { 4124 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4125 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4126 4127 if (resp->flags & QEDR_RESP_RDMA) 4128 return -EINVAL; 4129 4130 } else if (resp->flags & QEDR_RESP_RDMA) { 4131 return -EINVAL; 4132 } 4133 4134 return 0; 4135 } 4136 4137 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4138 struct qedr_cq *cq, struct ib_wc *wc, 4139 struct rdma_cqe_responder *resp, u64 wr_id) 4140 { 4141 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4142 wc->opcode = IB_WC_RECV; 4143 wc->wc_flags = 0; 4144 4145 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4146 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4147 DP_ERR(dev, 4148 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4149 cq, cq->icid, resp->flags); 4150 4151 } else { 4152 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4153 if (wc->status == IB_WC_GENERAL_ERR) 4154 DP_ERR(dev, 4155 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4156 cq, cq->icid, resp->status); 4157 } 4158 4159 /* Fill the rest of the WC */ 4160 wc->vendor_err = 0; 4161 wc->src_qp = qp->id; 4162 wc->qp = &qp->ibqp; 4163 wc->wr_id = wr_id; 4164 } 4165 4166 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4167 struct qedr_cq *cq, struct ib_wc *wc, 4168 struct rdma_cqe_responder *resp) 4169 { 4170 struct qedr_srq *srq = qp->srq; 4171 u64 wr_id; 4172 4173 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4174 le32_to_cpu(resp->srq_wr_id.lo), u64); 4175 4176 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4177 wc->status = IB_WC_WR_FLUSH_ERR; 4178 wc->vendor_err = 0; 4179 wc->wr_id = wr_id; 4180 wc->byte_len = 0; 4181 wc->src_qp = qp->id; 4182 wc->qp = &qp->ibqp; 4183 wc->wr_id = wr_id; 4184 } else { 4185 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4186 } 4187 srq->hw_srq.wr_cons_cnt++; 4188 4189 return 1; 4190 } 4191 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4192 struct qedr_cq *cq, struct ib_wc *wc, 4193 struct rdma_cqe_responder *resp) 4194 { 4195 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4196 4197 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4198 4199 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4200 qed_chain_consume(&qp->rq.pbl); 4201 qedr_inc_sw_cons(&qp->rq); 4202 4203 return 1; 4204 } 4205 4206 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4207 int num_entries, struct ib_wc *wc, u16 hw_cons) 4208 { 4209 u16 cnt = 0; 4210 4211 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4212 /* fill WC */ 4213 wc->status = IB_WC_WR_FLUSH_ERR; 4214 wc->vendor_err = 0; 4215 wc->wc_flags = 0; 4216 wc->src_qp = qp->id; 4217 wc->byte_len = 0; 4218 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4219 wc->qp = &qp->ibqp; 4220 num_entries--; 4221 wc++; 4222 cnt++; 4223 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4224 qed_chain_consume(&qp->rq.pbl); 4225 qedr_inc_sw_cons(&qp->rq); 4226 } 4227 4228 return cnt; 4229 } 4230 4231 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4232 struct rdma_cqe_responder *resp, int *update) 4233 { 4234 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4235 consume_cqe(cq); 4236 *update |= 1; 4237 } 4238 } 4239 4240 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4241 struct qedr_cq *cq, int num_entries, 4242 struct ib_wc *wc, 4243 struct rdma_cqe_responder *resp) 4244 { 4245 int cnt; 4246 4247 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4248 consume_cqe(cq); 4249 4250 return cnt; 4251 } 4252 4253 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4254 struct qedr_cq *cq, int num_entries, 4255 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4256 int *update) 4257 { 4258 int cnt; 4259 4260 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4261 cnt = process_resp_flush(qp, cq, num_entries, wc, 4262 resp->rq_cons_or_srq_id); 4263 try_consume_resp_cqe(cq, qp, resp, update); 4264 } else { 4265 cnt = process_resp_one(dev, qp, cq, wc, resp); 4266 consume_cqe(cq); 4267 *update |= 1; 4268 } 4269 4270 return cnt; 4271 } 4272 4273 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4274 struct rdma_cqe_requester *req, int *update) 4275 { 4276 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4277 consume_cqe(cq); 4278 *update |= 1; 4279 } 4280 } 4281 4282 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4283 { 4284 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4285 struct qedr_cq *cq = get_qedr_cq(ibcq); 4286 union rdma_cqe *cqe; 4287 u32 old_cons, new_cons; 4288 unsigned long flags; 4289 int update = 0; 4290 int done = 0; 4291 4292 if (cq->destroyed) { 4293 DP_ERR(dev, 4294 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4295 cq, cq->icid); 4296 return 0; 4297 } 4298 4299 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4300 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4301 4302 spin_lock_irqsave(&cq->cq_lock, flags); 4303 cqe = cq->latest_cqe; 4304 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4305 while (num_entries && is_valid_cqe(cq, cqe)) { 4306 struct qedr_qp *qp; 4307 int cnt = 0; 4308 4309 /* prevent speculative reads of any field of CQE */ 4310 rmb(); 4311 4312 qp = cqe_get_qp(cqe); 4313 if (!qp) { 4314 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4315 break; 4316 } 4317 4318 wc->qp = &qp->ibqp; 4319 4320 switch (cqe_get_type(cqe)) { 4321 case RDMA_CQE_TYPE_REQUESTER: 4322 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4323 &cqe->req); 4324 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4325 break; 4326 case RDMA_CQE_TYPE_RESPONDER_RQ: 4327 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4328 &cqe->resp, &update); 4329 break; 4330 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4331 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4332 wc, &cqe->resp); 4333 update = 1; 4334 break; 4335 case RDMA_CQE_TYPE_INVALID: 4336 default: 4337 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4338 cqe_get_type(cqe)); 4339 } 4340 num_entries -= cnt; 4341 wc += cnt; 4342 done += cnt; 4343 4344 cqe = get_cqe(cq); 4345 } 4346 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4347 4348 cq->cq_cons += new_cons - old_cons; 4349 4350 if (update) 4351 /* doorbell notifies abount latest VALID entry, 4352 * but chain already point to the next INVALID one 4353 */ 4354 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4355 4356 spin_unlock_irqrestore(&cq->cq_lock, flags); 4357 return done; 4358 } 4359 4360 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4361 u8 port_num, const struct ib_wc *in_wc, 4362 const struct ib_grh *in_grh, const struct ib_mad *in, 4363 struct ib_mad *out_mad, size_t *out_mad_size, 4364 u16 *out_mad_pkey_index) 4365 { 4366 return IB_MAD_RESULT_SUCCESS; 4367 } 4368