1 /* QLogic qedr NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/dma-mapping.h> 33 #include <linux/crc32.h> 34 #include <net/ip.h> 35 #include <net/ipv6.h> 36 #include <net/udp.h> 37 #include <linux/iommu.h> 38 39 #include <rdma/ib_verbs.h> 40 #include <rdma/ib_user_verbs.h> 41 #include <rdma/iw_cm.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_addr.h> 44 #include <rdma/ib_cache.h> 45 #include <rdma/uverbs_ioctl.h> 46 47 #include <linux/qed/common_hsi.h> 48 #include "qedr_hsi_rdma.h" 49 #include <linux/qed/qed_if.h> 50 #include "qedr.h" 51 #include "verbs.h" 52 #include <rdma/qedr-abi.h> 53 #include "qedr_roce_cm.h" 54 #include "qedr_iw_cm.h" 55 56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm) 57 #define RDMA_MAX_SGE_PER_SRQ (4) 58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) 59 60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) 61 62 enum { 63 QEDR_USER_MMAP_IO_WC = 0, 64 QEDR_USER_MMAP_PHYS_PAGE, 65 }; 66 67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src, 68 size_t len) 69 { 70 size_t min_len = min_t(size_t, len, udata->outlen); 71 72 return ib_copy_to_udata(udata, src, min_len); 73 } 74 75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) 76 { 77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN) 78 return -EINVAL; 79 80 *pkey = QEDR_ROCE_PKEY_DEFAULT; 81 return 0; 82 } 83 84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port, 85 int index, union ib_gid *sgid) 86 { 87 struct qedr_dev *dev = get_qedr_dev(ibdev); 88 89 memset(sgid->raw, 0, sizeof(sgid->raw)); 90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr); 91 92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index, 93 sgid->global.interface_id, sgid->global.subnet_prefix); 94 95 return 0; 96 } 97 98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) 99 { 100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 101 struct qedr_device_attr *qattr = &dev->attr; 102 struct qedr_srq *srq = get_qedr_srq(ibsrq); 103 104 srq_attr->srq_limit = srq->srq_limit; 105 srq_attr->max_wr = qattr->max_srq_wr; 106 srq_attr->max_sge = qattr->max_sge; 107 108 return 0; 109 } 110 111 int qedr_query_device(struct ib_device *ibdev, 112 struct ib_device_attr *attr, struct ib_udata *udata) 113 { 114 struct qedr_dev *dev = get_qedr_dev(ibdev); 115 struct qedr_device_attr *qattr = &dev->attr; 116 117 if (!dev->rdma_ctx) { 118 DP_ERR(dev, 119 "qedr_query_device called with invalid params rdma_ctx=%p\n", 120 dev->rdma_ctx); 121 return -EINVAL; 122 } 123 124 memset(attr, 0, sizeof(*attr)); 125 126 attr->fw_ver = qattr->fw_ver; 127 attr->sys_image_guid = qattr->sys_image_guid; 128 attr->max_mr_size = qattr->max_mr_size; 129 attr->page_size_cap = qattr->page_size_caps; 130 attr->vendor_id = qattr->vendor_id; 131 attr->vendor_part_id = qattr->vendor_part_id; 132 attr->hw_ver = qattr->hw_ver; 133 attr->max_qp = qattr->max_qp; 134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe); 135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | 136 IB_DEVICE_RC_RNR_NAK_GEN | 137 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; 138 139 attr->max_send_sge = qattr->max_sge; 140 attr->max_recv_sge = qattr->max_sge; 141 attr->max_sge_rd = qattr->max_sge; 142 attr->max_cq = qattr->max_cq; 143 attr->max_cqe = qattr->max_cqe; 144 attr->max_mr = qattr->max_mr; 145 attr->max_mw = qattr->max_mw; 146 attr->max_pd = qattr->max_pd; 147 attr->atomic_cap = dev->atomic_cap; 148 attr->max_fmr = qattr->max_fmr; 149 attr->max_map_per_fmr = 16; 150 attr->max_qp_init_rd_atom = 151 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); 152 attr->max_qp_rd_atom = 153 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), 154 attr->max_qp_init_rd_atom); 155 156 attr->max_srq = qattr->max_srq; 157 attr->max_srq_sge = qattr->max_srq_sge; 158 attr->max_srq_wr = qattr->max_srq_wr; 159 160 attr->local_ca_ack_delay = qattr->dev_ack_delay; 161 attr->max_fast_reg_page_list_len = qattr->max_mr / 8; 162 attr->max_pkeys = QEDR_ROCE_PKEY_MAX; 163 attr->max_ah = qattr->max_ah; 164 165 return 0; 166 } 167 168 static inline void get_link_speed_and_width(int speed, u8 *ib_speed, 169 u8 *ib_width) 170 { 171 switch (speed) { 172 case 1000: 173 *ib_speed = IB_SPEED_SDR; 174 *ib_width = IB_WIDTH_1X; 175 break; 176 case 10000: 177 *ib_speed = IB_SPEED_QDR; 178 *ib_width = IB_WIDTH_1X; 179 break; 180 181 case 20000: 182 *ib_speed = IB_SPEED_DDR; 183 *ib_width = IB_WIDTH_4X; 184 break; 185 186 case 25000: 187 *ib_speed = IB_SPEED_EDR; 188 *ib_width = IB_WIDTH_1X; 189 break; 190 191 case 40000: 192 *ib_speed = IB_SPEED_QDR; 193 *ib_width = IB_WIDTH_4X; 194 break; 195 196 case 50000: 197 *ib_speed = IB_SPEED_HDR; 198 *ib_width = IB_WIDTH_1X; 199 break; 200 201 case 100000: 202 *ib_speed = IB_SPEED_EDR; 203 *ib_width = IB_WIDTH_4X; 204 break; 205 206 default: 207 /* Unsupported */ 208 *ib_speed = IB_SPEED_SDR; 209 *ib_width = IB_WIDTH_1X; 210 } 211 } 212 213 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr) 214 { 215 struct qedr_dev *dev; 216 struct qed_rdma_port *rdma_port; 217 218 dev = get_qedr_dev(ibdev); 219 220 if (!dev->rdma_ctx) { 221 DP_ERR(dev, "rdma_ctx is NULL\n"); 222 return -EINVAL; 223 } 224 225 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx); 226 227 /* *attr being zeroed by the caller, avoid zeroing it here */ 228 if (rdma_port->port_state == QED_RDMA_PORT_UP) { 229 attr->state = IB_PORT_ACTIVE; 230 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 231 } else { 232 attr->state = IB_PORT_DOWN; 233 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 234 } 235 attr->max_mtu = IB_MTU_4096; 236 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu); 237 attr->lid = 0; 238 attr->lmc = 0; 239 attr->sm_lid = 0; 240 attr->sm_sl = 0; 241 attr->ip_gids = true; 242 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 243 attr->gid_tbl_len = 1; 244 attr->pkey_tbl_len = 1; 245 } else { 246 attr->gid_tbl_len = QEDR_MAX_SGID; 247 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN; 248 } 249 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; 250 attr->qkey_viol_cntr = 0; 251 get_link_speed_and_width(rdma_port->link_speed, 252 &attr->active_speed, &attr->active_width); 253 attr->max_msg_sz = rdma_port->max_msg_size; 254 attr->max_vl_num = 4; 255 256 return 0; 257 } 258 259 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) 260 { 261 struct ib_device *ibdev = uctx->device; 262 int rc; 263 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx); 264 struct qedr_alloc_ucontext_resp uresp = {}; 265 struct qedr_alloc_ucontext_req ureq = {}; 266 struct qedr_dev *dev = get_qedr_dev(ibdev); 267 struct qed_rdma_add_user_out_params oparams; 268 struct qedr_user_mmap_entry *entry; 269 270 if (!udata) 271 return -EFAULT; 272 273 if (udata->inlen) { 274 rc = ib_copy_from_udata(&ureq, udata, 275 min(sizeof(ureq), udata->inlen)); 276 if (rc) { 277 DP_ERR(dev, "Problem copying data from user space\n"); 278 return -EFAULT; 279 } 280 281 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC); 282 } 283 284 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams); 285 if (rc) { 286 DP_ERR(dev, 287 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n", 288 rc); 289 return rc; 290 } 291 292 ctx->dpi = oparams.dpi; 293 ctx->dpi_addr = oparams.dpi_addr; 294 ctx->dpi_phys_addr = oparams.dpi_phys_addr; 295 ctx->dpi_size = oparams.dpi_size; 296 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 297 if (!entry) { 298 rc = -ENOMEM; 299 goto err; 300 } 301 302 entry->io_address = ctx->dpi_phys_addr; 303 entry->length = ctx->dpi_size; 304 entry->mmap_flag = QEDR_USER_MMAP_IO_WC; 305 entry->dpi = ctx->dpi; 306 entry->dev = dev; 307 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry, 308 ctx->dpi_size); 309 if (rc) { 310 kfree(entry); 311 goto err; 312 } 313 ctx->db_mmap_entry = &entry->rdma_entry; 314 315 uresp.dpm_enabled = dev->user_dpm_enabled; 316 uresp.wids_enabled = 1; 317 uresp.wid_count = oparams.wid_count; 318 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry); 319 uresp.db_size = ctx->dpi_size; 320 uresp.max_send_wr = dev->attr.max_sqe; 321 uresp.max_recv_wr = dev->attr.max_rqe; 322 uresp.max_srq_wr = dev->attr.max_srq_wr; 323 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE; 324 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE; 325 uresp.sges_per_srq_wr = dev->attr.max_srq_sge; 326 uresp.max_cqes = QEDR_MAX_CQES; 327 328 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 329 if (rc) 330 goto err; 331 332 ctx->dev = dev; 333 334 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n", 335 &ctx->ibucontext); 336 return 0; 337 338 err: 339 if (!ctx->db_mmap_entry) 340 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi); 341 else 342 rdma_user_mmap_entry_remove(ctx->db_mmap_entry); 343 344 return rc; 345 } 346 347 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx) 348 { 349 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx); 350 351 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n", 352 uctx); 353 354 rdma_user_mmap_entry_remove(uctx->db_mmap_entry); 355 } 356 357 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 358 { 359 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry); 360 struct qedr_dev *dev = entry->dev; 361 362 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE) 363 free_page((unsigned long)entry->address); 364 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC) 365 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi); 366 367 kfree(entry); 368 } 369 370 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma) 371 { 372 struct ib_device *dev = ucontext->device; 373 size_t length = vma->vm_end - vma->vm_start; 374 struct rdma_user_mmap_entry *rdma_entry; 375 struct qedr_user_mmap_entry *entry; 376 int rc = 0; 377 u64 pfn; 378 379 ibdev_dbg(dev, 380 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n", 381 vma->vm_start, vma->vm_end, length, vma->vm_pgoff); 382 383 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma); 384 if (!rdma_entry) { 385 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n", 386 vma->vm_pgoff); 387 return -EINVAL; 388 } 389 entry = get_qedr_mmap_entry(rdma_entry); 390 ibdev_dbg(dev, 391 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n", 392 entry->io_address, length, entry->mmap_flag); 393 394 switch (entry->mmap_flag) { 395 case QEDR_USER_MMAP_IO_WC: 396 pfn = entry->io_address >> PAGE_SHIFT; 397 rc = rdma_user_mmap_io(ucontext, vma, pfn, length, 398 pgprot_writecombine(vma->vm_page_prot), 399 rdma_entry); 400 break; 401 case QEDR_USER_MMAP_PHYS_PAGE: 402 rc = vm_insert_page(vma, vma->vm_start, 403 virt_to_page(entry->address)); 404 break; 405 default: 406 rc = -EINVAL; 407 } 408 409 if (rc) 410 ibdev_dbg(dev, 411 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n", 412 entry->io_address, length, entry->mmap_flag, rc); 413 414 rdma_user_mmap_entry_put(rdma_entry); 415 return rc; 416 } 417 418 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 419 { 420 struct ib_device *ibdev = ibpd->device; 421 struct qedr_dev *dev = get_qedr_dev(ibdev); 422 struct qedr_pd *pd = get_qedr_pd(ibpd); 423 u16 pd_id; 424 int rc; 425 426 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n", 427 udata ? "User Lib" : "Kernel"); 428 429 if (!dev->rdma_ctx) { 430 DP_ERR(dev, "invalid RDMA context\n"); 431 return -EINVAL; 432 } 433 434 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id); 435 if (rc) 436 return rc; 437 438 pd->pd_id = pd_id; 439 440 if (udata) { 441 struct qedr_alloc_pd_uresp uresp = { 442 .pd_id = pd_id, 443 }; 444 struct qedr_ucontext *context = rdma_udata_to_drv_context( 445 udata, struct qedr_ucontext, ibucontext); 446 447 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 448 if (rc) { 449 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id); 450 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id); 451 return rc; 452 } 453 454 pd->uctx = context; 455 pd->uctx->pd = pd; 456 } 457 458 return 0; 459 } 460 461 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 462 { 463 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 464 struct qedr_pd *pd = get_qedr_pd(ibpd); 465 466 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id); 467 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id); 468 } 469 470 static void qedr_free_pbl(struct qedr_dev *dev, 471 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl) 472 { 473 struct pci_dev *pdev = dev->pdev; 474 int i; 475 476 for (i = 0; i < pbl_info->num_pbls; i++) { 477 if (!pbl[i].va) 478 continue; 479 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 480 pbl[i].va, pbl[i].pa); 481 } 482 483 kfree(pbl); 484 } 485 486 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024) 487 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024) 488 489 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) 490 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) 491 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE) 492 493 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev, 494 struct qedr_pbl_info *pbl_info, 495 gfp_t flags) 496 { 497 struct pci_dev *pdev = dev->pdev; 498 struct qedr_pbl *pbl_table; 499 dma_addr_t *pbl_main_tbl; 500 dma_addr_t pa; 501 void *va; 502 int i; 503 504 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags); 505 if (!pbl_table) 506 return ERR_PTR(-ENOMEM); 507 508 for (i = 0; i < pbl_info->num_pbls; i++) { 509 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa, 510 flags); 511 if (!va) 512 goto err; 513 514 pbl_table[i].va = va; 515 pbl_table[i].pa = pa; 516 } 517 518 /* Two-Layer PBLs, if we have more than one pbl we need to initialize 519 * the first one with physical pointers to all of the rest 520 */ 521 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; 522 for (i = 0; i < pbl_info->num_pbls - 1; i++) 523 pbl_main_tbl[i] = pbl_table[i + 1].pa; 524 525 return pbl_table; 526 527 err: 528 for (i--; i >= 0; i--) 529 dma_free_coherent(&pdev->dev, pbl_info->pbl_size, 530 pbl_table[i].va, pbl_table[i].pa); 531 532 qedr_free_pbl(dev, pbl_info, pbl_table); 533 534 return ERR_PTR(-ENOMEM); 535 } 536 537 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev, 538 struct qedr_pbl_info *pbl_info, 539 u32 num_pbes, int two_layer_capable) 540 { 541 u32 pbl_capacity; 542 u32 pbl_size; 543 u32 num_pbls; 544 545 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { 546 if (num_pbes > MAX_PBES_TWO_LAYER) { 547 DP_ERR(dev, "prepare pbl table: too many pages %d\n", 548 num_pbes); 549 return -EINVAL; 550 } 551 552 /* calculate required pbl page size */ 553 pbl_size = MIN_FW_PBL_PAGE_SIZE; 554 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * 555 NUM_PBES_ON_PAGE(pbl_size); 556 557 while (pbl_capacity < num_pbes) { 558 pbl_size *= 2; 559 pbl_capacity = pbl_size / sizeof(u64); 560 pbl_capacity = pbl_capacity * pbl_capacity; 561 } 562 563 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); 564 num_pbls++; /* One for the layer0 ( points to the pbls) */ 565 pbl_info->two_layered = true; 566 } else { 567 /* One layered PBL */ 568 num_pbls = 1; 569 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, 570 roundup_pow_of_two((num_pbes * sizeof(u64)))); 571 pbl_info->two_layered = false; 572 } 573 574 pbl_info->num_pbls = num_pbls; 575 pbl_info->pbl_size = pbl_size; 576 pbl_info->num_pbes = num_pbes; 577 578 DP_DEBUG(dev, QEDR_MSG_MR, 579 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n", 580 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); 581 582 return 0; 583 } 584 585 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem, 586 struct qedr_pbl *pbl, 587 struct qedr_pbl_info *pbl_info, u32 pg_shift) 588 { 589 int pbe_cnt, total_num_pbes = 0; 590 u32 fw_pg_cnt, fw_pg_per_umem_pg; 591 struct qedr_pbl *pbl_tbl; 592 struct sg_dma_page_iter sg_iter; 593 struct regpair *pbe; 594 u64 pg_addr; 595 596 if (!pbl_info->num_pbes) 597 return; 598 599 /* If we have a two layered pbl, the first pbl points to the rest 600 * of the pbls and the first entry lays on the second pbl in the table 601 */ 602 if (pbl_info->two_layered) 603 pbl_tbl = &pbl[1]; 604 else 605 pbl_tbl = pbl; 606 607 pbe = (struct regpair *)pbl_tbl->va; 608 if (!pbe) { 609 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n"); 610 return; 611 } 612 613 pbe_cnt = 0; 614 615 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift); 616 617 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { 618 pg_addr = sg_page_iter_dma_address(&sg_iter); 619 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) { 620 pbe->lo = cpu_to_le32(pg_addr); 621 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr)); 622 623 pg_addr += BIT(pg_shift); 624 pbe_cnt++; 625 total_num_pbes++; 626 pbe++; 627 628 if (total_num_pbes == pbl_info->num_pbes) 629 return; 630 631 /* If the given pbl is full storing the pbes, 632 * move to next pbl. 633 */ 634 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { 635 pbl_tbl++; 636 pbe = (struct regpair *)pbl_tbl->va; 637 pbe_cnt = 0; 638 } 639 640 fw_pg_cnt++; 641 } 642 } 643 } 644 645 static int qedr_db_recovery_add(struct qedr_dev *dev, 646 void __iomem *db_addr, 647 void *db_data, 648 enum qed_db_rec_width db_width, 649 enum qed_db_rec_space db_space) 650 { 651 if (!db_data) { 652 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 653 return 0; 654 } 655 656 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data, 657 db_width, db_space); 658 } 659 660 static void qedr_db_recovery_del(struct qedr_dev *dev, 661 void __iomem *db_addr, 662 void *db_data) 663 { 664 if (!db_data) { 665 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n"); 666 return; 667 } 668 669 /* Ignore return code as there is not much we can do about it. Error 670 * log will be printed inside. 671 */ 672 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data); 673 } 674 675 static int qedr_copy_cq_uresp(struct qedr_dev *dev, 676 struct qedr_cq *cq, struct ib_udata *udata, 677 u32 db_offset) 678 { 679 struct qedr_create_cq_uresp uresp; 680 int rc; 681 682 memset(&uresp, 0, sizeof(uresp)); 683 684 uresp.db_offset = db_offset; 685 uresp.icid = cq->icid; 686 uresp.db_rec_addr = rdma_user_mmap_get_offset(cq->q.db_mmap_entry); 687 688 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 689 if (rc) 690 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid); 691 692 return rc; 693 } 694 695 static void consume_cqe(struct qedr_cq *cq) 696 { 697 if (cq->latest_cqe == cq->toggle_cqe) 698 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 699 700 cq->latest_cqe = qed_chain_consume(&cq->pbl); 701 } 702 703 static inline int qedr_align_cq_entries(int entries) 704 { 705 u64 size, aligned_size; 706 707 /* We allocate an extra entry that we don't report to the FW. */ 708 size = (entries + 1) * QEDR_CQE_SIZE; 709 aligned_size = ALIGN(size, PAGE_SIZE); 710 711 return aligned_size / QEDR_CQE_SIZE; 712 } 713 714 static int qedr_init_user_db_rec(struct ib_udata *udata, 715 struct qedr_dev *dev, struct qedr_userq *q, 716 bool requires_db_rec) 717 { 718 struct qedr_ucontext *uctx = 719 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 720 ibucontext); 721 struct qedr_user_mmap_entry *entry; 722 int rc; 723 724 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */ 725 if (requires_db_rec == 0 || !uctx->db_rec) 726 return 0; 727 728 /* Allocate a page for doorbell recovery, add to mmap */ 729 q->db_rec_data = (void *)get_zeroed_page(GFP_USER); 730 if (!q->db_rec_data) { 731 DP_ERR(dev, "get_zeroed_page failed\n"); 732 return -ENOMEM; 733 } 734 735 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 736 if (!entry) 737 goto err_free_db_data; 738 739 entry->address = q->db_rec_data; 740 entry->length = PAGE_SIZE; 741 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE; 742 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext, 743 &entry->rdma_entry, 744 PAGE_SIZE); 745 if (rc) 746 goto err_free_entry; 747 748 q->db_mmap_entry = &entry->rdma_entry; 749 750 return 0; 751 752 err_free_entry: 753 kfree(entry); 754 755 err_free_db_data: 756 free_page((unsigned long)q->db_rec_data); 757 q->db_rec_data = NULL; 758 return -ENOMEM; 759 } 760 761 static inline int qedr_init_user_queue(struct ib_udata *udata, 762 struct qedr_dev *dev, 763 struct qedr_userq *q, u64 buf_addr, 764 size_t buf_len, bool requires_db_rec, 765 int access, int dmasync, 766 int alloc_and_init) 767 { 768 u32 fw_pages; 769 int rc; 770 771 q->buf_addr = buf_addr; 772 q->buf_len = buf_len; 773 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync); 774 if (IS_ERR(q->umem)) { 775 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n", 776 PTR_ERR(q->umem)); 777 return PTR_ERR(q->umem); 778 } 779 780 fw_pages = ib_umem_page_count(q->umem) << 781 (PAGE_SHIFT - FW_PAGE_SHIFT); 782 783 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0); 784 if (rc) 785 goto err0; 786 787 if (alloc_and_init) { 788 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); 789 if (IS_ERR(q->pbl_tbl)) { 790 rc = PTR_ERR(q->pbl_tbl); 791 goto err0; 792 } 793 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info, 794 FW_PAGE_SHIFT); 795 } else { 796 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); 797 if (!q->pbl_tbl) { 798 rc = -ENOMEM; 799 goto err0; 800 } 801 } 802 803 /* mmap the user address used to store doorbell data for recovery */ 804 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec); 805 806 err0: 807 ib_umem_release(q->umem); 808 q->umem = NULL; 809 810 return rc; 811 } 812 813 static inline void qedr_init_cq_params(struct qedr_cq *cq, 814 struct qedr_ucontext *ctx, 815 struct qedr_dev *dev, int vector, 816 int chain_entries, int page_cnt, 817 u64 pbl_ptr, 818 struct qed_rdma_create_cq_in_params 819 *params) 820 { 821 memset(params, 0, sizeof(*params)); 822 params->cq_handle_hi = upper_32_bits((uintptr_t)cq); 823 params->cq_handle_lo = lower_32_bits((uintptr_t)cq); 824 params->cnq_id = vector; 825 params->cq_size = chain_entries - 1; 826 params->dpi = (ctx) ? ctx->dpi : dev->dpi; 827 params->pbl_num_pages = page_cnt; 828 params->pbl_ptr = pbl_ptr; 829 params->pbl_two_level = 0; 830 } 831 832 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags) 833 { 834 cq->db.data.agg_flags = flags; 835 cq->db.data.value = cpu_to_le32(cons); 836 writeq(cq->db.raw, cq->db_addr); 837 } 838 839 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 840 { 841 struct qedr_cq *cq = get_qedr_cq(ibcq); 842 unsigned long sflags; 843 struct qedr_dev *dev; 844 845 dev = get_qedr_dev(ibcq->device); 846 847 if (cq->destroyed) { 848 DP_ERR(dev, 849 "warning: arm was invoked after destroy for cq %p (icid=%d)\n", 850 cq, cq->icid); 851 return -EINVAL; 852 } 853 854 855 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 856 return 0; 857 858 spin_lock_irqsave(&cq->cq_lock, sflags); 859 860 cq->arm_flags = 0; 861 862 if (flags & IB_CQ_SOLICITED) 863 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; 864 865 if (flags & IB_CQ_NEXT_COMP) 866 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; 867 868 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 869 870 spin_unlock_irqrestore(&cq->cq_lock, sflags); 871 872 return 0; 873 } 874 875 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 876 struct ib_udata *udata) 877 { 878 struct ib_device *ibdev = ibcq->device; 879 struct qedr_ucontext *ctx = rdma_udata_to_drv_context( 880 udata, struct qedr_ucontext, ibucontext); 881 struct qed_rdma_destroy_cq_out_params destroy_oparams; 882 struct qed_rdma_destroy_cq_in_params destroy_iparams; 883 struct qedr_dev *dev = get_qedr_dev(ibdev); 884 struct qed_rdma_create_cq_in_params params; 885 struct qedr_create_cq_ureq ureq = {}; 886 int vector = attr->comp_vector; 887 int entries = attr->cqe; 888 struct qedr_cq *cq = get_qedr_cq(ibcq); 889 int chain_entries; 890 u32 db_offset; 891 int page_cnt; 892 u64 pbl_ptr; 893 u16 icid; 894 int rc; 895 896 DP_DEBUG(dev, QEDR_MSG_INIT, 897 "create_cq: called from %s. entries=%d, vector=%d\n", 898 udata ? "User Lib" : "Kernel", entries, vector); 899 900 if (entries > QEDR_MAX_CQES) { 901 DP_ERR(dev, 902 "create cq: the number of entries %d is too high. Must be equal or below %d.\n", 903 entries, QEDR_MAX_CQES); 904 return -EINVAL; 905 } 906 907 chain_entries = qedr_align_cq_entries(entries); 908 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); 909 910 /* calc db offset. user will add DPI base, kernel will add db addr */ 911 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); 912 913 if (udata) { 914 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 915 udata->inlen))) { 916 DP_ERR(dev, 917 "create cq: problem copying data from user space\n"); 918 goto err0; 919 } 920 921 if (!ureq.len) { 922 DP_ERR(dev, 923 "create cq: cannot create a cq with 0 entries\n"); 924 goto err0; 925 } 926 927 cq->cq_type = QEDR_CQ_TYPE_USER; 928 929 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr, 930 ureq.len, true, 931 IB_ACCESS_LOCAL_WRITE, 932 1, 1); 933 if (rc) 934 goto err0; 935 936 pbl_ptr = cq->q.pbl_tbl->pa; 937 page_cnt = cq->q.pbl_info.num_pbes; 938 939 cq->ibcq.cqe = chain_entries; 940 cq->q.db_addr = ctx->dpi_addr + db_offset; 941 } else { 942 cq->cq_type = QEDR_CQ_TYPE_KERNEL; 943 944 rc = dev->ops->common->chain_alloc(dev->cdev, 945 QED_CHAIN_USE_TO_CONSUME, 946 QED_CHAIN_MODE_PBL, 947 QED_CHAIN_CNT_TYPE_U32, 948 chain_entries, 949 sizeof(union rdma_cqe), 950 &cq->pbl, NULL); 951 if (rc) 952 goto err0; 953 954 page_cnt = qed_chain_get_page_cnt(&cq->pbl); 955 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl); 956 cq->ibcq.cqe = cq->pbl.capacity; 957 } 958 959 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt, 960 pbl_ptr, ¶ms); 961 962 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); 963 if (rc) 964 goto err1; 965 966 cq->icid = icid; 967 cq->sig = QEDR_CQ_MAGIC_NUMBER; 968 spin_lock_init(&cq->cq_lock); 969 970 if (udata) { 971 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset); 972 if (rc) 973 goto err2; 974 975 rc = qedr_db_recovery_add(dev, cq->q.db_addr, 976 &cq->q.db_rec_data->db_data, 977 DB_REC_WIDTH_64B, 978 DB_REC_USER); 979 if (rc) 980 goto err2; 981 982 } else { 983 /* Generate doorbell address. */ 984 cq->db.data.icid = cq->icid; 985 cq->db_addr = dev->db_addr + db_offset; 986 cq->db.data.params = DB_AGG_CMD_SET << 987 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; 988 989 /* point to the very last element, passing it we will toggle */ 990 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl); 991 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK; 992 cq->latest_cqe = NULL; 993 consume_cqe(cq); 994 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 995 996 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data, 997 DB_REC_WIDTH_64B, DB_REC_KERNEL); 998 if (rc) 999 goto err2; 1000 } 1001 1002 DP_DEBUG(dev, QEDR_MSG_CQ, 1003 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", 1004 cq->icid, cq, params.cq_size); 1005 1006 return 0; 1007 1008 err2: 1009 destroy_iparams.icid = cq->icid; 1010 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, 1011 &destroy_oparams); 1012 err1: 1013 if (udata) { 1014 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1015 ib_umem_release(cq->q.umem); 1016 if (ctx) 1017 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1018 } else { 1019 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1020 } 1021 err0: 1022 return -EINVAL; 1023 } 1024 1025 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) 1026 { 1027 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1028 struct qedr_cq *cq = get_qedr_cq(ibcq); 1029 1030 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq); 1031 1032 return 0; 1033 } 1034 1035 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) 1036 #define QEDR_DESTROY_CQ_ITER_DURATION (10) 1037 1038 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1039 { 1040 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 1041 struct qed_rdma_destroy_cq_out_params oparams; 1042 struct qed_rdma_destroy_cq_in_params iparams; 1043 struct qedr_cq *cq = get_qedr_cq(ibcq); 1044 int iter; 1045 1046 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); 1047 1048 cq->destroyed = 1; 1049 1050 /* GSIs CQs are handled by driver, so they don't exist in the FW */ 1051 if (cq->cq_type == QEDR_CQ_TYPE_GSI) { 1052 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1053 return; 1054 } 1055 1056 iparams.icid = cq->icid; 1057 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); 1058 dev->ops->common->chain_free(dev->cdev, &cq->pbl); 1059 1060 if (udata) { 1061 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); 1062 ib_umem_release(cq->q.umem); 1063 1064 if (cq->q.db_rec_data) { 1065 qedr_db_recovery_del(dev, cq->q.db_addr, 1066 &cq->q.db_rec_data->db_data); 1067 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry); 1068 } 1069 } else { 1070 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data); 1071 } 1072 1073 /* We don't want the IRQ handler to handle a non-existing CQ so we 1074 * wait until all CNQ interrupts, if any, are received. This will always 1075 * happen and will always happen very fast. If not, then a serious error 1076 * has occured. That is why we can use a long delay. 1077 * We spin for a short time so we don’t lose time on context switching 1078 * in case all the completions are handled in that span. Otherwise 1079 * we sleep for a while and check again. Since the CNQ may be 1080 * associated with (only) the current CPU we use msleep to allow the 1081 * current CPU to be freed. 1082 * The CNQ notification is increased in qedr_irq_handler(). 1083 */ 1084 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1085 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1086 udelay(QEDR_DESTROY_CQ_ITER_DURATION); 1087 iter--; 1088 } 1089 1090 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS; 1091 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) { 1092 msleep(QEDR_DESTROY_CQ_ITER_DURATION); 1093 iter--; 1094 } 1095 1096 /* Note that we don't need to have explicit code to wait for the 1097 * completion of the event handler because it is invoked from the EQ. 1098 * Since the destroy CQ ramrod has also been received on the EQ we can 1099 * be certain that there's no event handler in process. 1100 */ 1101 } 1102 1103 static inline int get_gid_info_from_table(struct ib_qp *ibqp, 1104 struct ib_qp_attr *attr, 1105 int attr_mask, 1106 struct qed_rdma_modify_qp_in_params 1107 *qp_params) 1108 { 1109 const struct ib_gid_attr *gid_attr; 1110 enum rdma_network_type nw_type; 1111 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 1112 u32 ipv4_addr; 1113 int ret; 1114 int i; 1115 1116 gid_attr = grh->sgid_attr; 1117 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL); 1118 if (ret) 1119 return ret; 1120 1121 nw_type = rdma_gid_attr_network_type(gid_attr); 1122 switch (nw_type) { 1123 case RDMA_NETWORK_IPV6: 1124 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1125 sizeof(qp_params->sgid)); 1126 memcpy(&qp_params->dgid.bytes[0], 1127 &grh->dgid, 1128 sizeof(qp_params->dgid)); 1129 qp_params->roce_mode = ROCE_V2_IPV6; 1130 SET_FIELD(qp_params->modify_flags, 1131 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1132 break; 1133 case RDMA_NETWORK_IB: 1134 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0], 1135 sizeof(qp_params->sgid)); 1136 memcpy(&qp_params->dgid.bytes[0], 1137 &grh->dgid, 1138 sizeof(qp_params->dgid)); 1139 qp_params->roce_mode = ROCE_V1; 1140 break; 1141 case RDMA_NETWORK_IPV4: 1142 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid)); 1143 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid)); 1144 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw); 1145 qp_params->sgid.ipv4_addr = ipv4_addr; 1146 ipv4_addr = 1147 qedr_get_ipv4_from_gid(grh->dgid.raw); 1148 qp_params->dgid.ipv4_addr = ipv4_addr; 1149 SET_FIELD(qp_params->modify_flags, 1150 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1); 1151 qp_params->roce_mode = ROCE_V2_IPV4; 1152 break; 1153 } 1154 1155 for (i = 0; i < 4; i++) { 1156 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); 1157 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); 1158 } 1159 1160 if (qp_params->vlan_id >= VLAN_CFI_MASK) 1161 qp_params->vlan_id = 0; 1162 1163 return 0; 1164 } 1165 1166 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev, 1167 struct ib_qp_init_attr *attrs, 1168 struct ib_udata *udata) 1169 { 1170 struct qedr_device_attr *qattr = &dev->attr; 1171 1172 /* QP0... attrs->qp_type == IB_QPT_GSI */ 1173 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { 1174 DP_DEBUG(dev, QEDR_MSG_QP, 1175 "create qp: unsupported qp type=0x%x requested\n", 1176 attrs->qp_type); 1177 return -EINVAL; 1178 } 1179 1180 if (attrs->cap.max_send_wr > qattr->max_sqe) { 1181 DP_ERR(dev, 1182 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n", 1183 attrs->cap.max_send_wr, qattr->max_sqe); 1184 return -EINVAL; 1185 } 1186 1187 if (attrs->cap.max_inline_data > qattr->max_inline) { 1188 DP_ERR(dev, 1189 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n", 1190 attrs->cap.max_inline_data, qattr->max_inline); 1191 return -EINVAL; 1192 } 1193 1194 if (attrs->cap.max_send_sge > qattr->max_sge) { 1195 DP_ERR(dev, 1196 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n", 1197 attrs->cap.max_send_sge, qattr->max_sge); 1198 return -EINVAL; 1199 } 1200 1201 if (attrs->cap.max_recv_sge > qattr->max_sge) { 1202 DP_ERR(dev, 1203 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n", 1204 attrs->cap.max_recv_sge, qattr->max_sge); 1205 return -EINVAL; 1206 } 1207 1208 /* Unprivileged user space cannot create special QP */ 1209 if (udata && attrs->qp_type == IB_QPT_GSI) { 1210 DP_ERR(dev, 1211 "create qp: userspace can't create special QPs of type=0x%x\n", 1212 attrs->qp_type); 1213 return -EINVAL; 1214 } 1215 1216 return 0; 1217 } 1218 1219 static int qedr_copy_srq_uresp(struct qedr_dev *dev, 1220 struct qedr_srq *srq, struct ib_udata *udata) 1221 { 1222 struct qedr_create_srq_uresp uresp = {}; 1223 int rc; 1224 1225 uresp.srq_id = srq->srq_id; 1226 1227 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1228 if (rc) 1229 DP_ERR(dev, "create srq: problem copying data to user space\n"); 1230 1231 return rc; 1232 } 1233 1234 static void qedr_copy_rq_uresp(struct qedr_dev *dev, 1235 struct qedr_create_qp_uresp *uresp, 1236 struct qedr_qp *qp) 1237 { 1238 /* iWARP requires two doorbells per RQ. */ 1239 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1240 uresp->rq_db_offset = 1241 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1242 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1243 } else { 1244 uresp->rq_db_offset = 1245 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1246 } 1247 1248 uresp->rq_icid = qp->icid; 1249 uresp->rq_db_rec_addr = rdma_user_mmap_get_offset(qp->urq.db_mmap_entry); 1250 } 1251 1252 static void qedr_copy_sq_uresp(struct qedr_dev *dev, 1253 struct qedr_create_qp_uresp *uresp, 1254 struct qedr_qp *qp) 1255 { 1256 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1257 1258 /* iWARP uses the same cid for rq and sq */ 1259 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1260 uresp->sq_icid = qp->icid; 1261 else 1262 uresp->sq_icid = qp->icid + 1; 1263 1264 uresp->sq_db_rec_addr = 1265 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry); 1266 } 1267 1268 static int qedr_copy_qp_uresp(struct qedr_dev *dev, 1269 struct qedr_qp *qp, struct ib_udata *udata, 1270 struct qedr_create_qp_uresp *uresp) 1271 { 1272 int rc; 1273 1274 memset(uresp, 0, sizeof(*uresp)); 1275 qedr_copy_sq_uresp(dev, uresp, qp); 1276 qedr_copy_rq_uresp(dev, uresp, qp); 1277 1278 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; 1279 uresp->qp_id = qp->qp_id; 1280 1281 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp)); 1282 if (rc) 1283 DP_ERR(dev, 1284 "create qp: failed a copy to user space with qp icid=0x%x.\n", 1285 qp->icid); 1286 1287 return rc; 1288 } 1289 1290 static void qedr_set_common_qp_params(struct qedr_dev *dev, 1291 struct qedr_qp *qp, 1292 struct qedr_pd *pd, 1293 struct ib_qp_init_attr *attrs) 1294 { 1295 spin_lock_init(&qp->q_lock); 1296 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1297 kref_init(&qp->refcnt); 1298 init_completion(&qp->iwarp_cm_comp); 1299 } 1300 qp->pd = pd; 1301 qp->qp_type = attrs->qp_type; 1302 qp->max_inline_data = attrs->cap.max_inline_data; 1303 qp->sq.max_sges = attrs->cap.max_send_sge; 1304 qp->state = QED_ROCE_QP_STATE_RESET; 1305 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; 1306 qp->sq_cq = get_qedr_cq(attrs->send_cq); 1307 qp->dev = dev; 1308 1309 if (attrs->srq) { 1310 qp->srq = get_qedr_srq(attrs->srq); 1311 } else { 1312 qp->rq_cq = get_qedr_cq(attrs->recv_cq); 1313 qp->rq.max_sges = attrs->cap.max_recv_sge; 1314 DP_DEBUG(dev, QEDR_MSG_QP, 1315 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", 1316 qp->rq.max_sges, qp->rq_cq->icid); 1317 } 1318 1319 DP_DEBUG(dev, QEDR_MSG_QP, 1320 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n", 1321 pd->pd_id, qp->qp_type, qp->max_inline_data, 1322 qp->state, qp->signaled, (attrs->srq) ? 1 : 0); 1323 DP_DEBUG(dev, QEDR_MSG_QP, 1324 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", 1325 qp->sq.max_sges, qp->sq_cq->icid); 1326 } 1327 1328 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1329 { 1330 int rc; 1331 1332 qp->sq.db = dev->db_addr + 1333 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1334 qp->sq.db_data.data.icid = qp->icid + 1; 1335 rc = qedr_db_recovery_add(dev, qp->sq.db, 1336 &qp->sq.db_data, 1337 DB_REC_WIDTH_32B, 1338 DB_REC_KERNEL); 1339 if (rc) 1340 return rc; 1341 1342 if (!qp->srq) { 1343 qp->rq.db = dev->db_addr + 1344 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); 1345 qp->rq.db_data.data.icid = qp->icid; 1346 1347 rc = qedr_db_recovery_add(dev, qp->rq.db, 1348 &qp->rq.db_data, 1349 DB_REC_WIDTH_32B, 1350 DB_REC_KERNEL); 1351 if (rc) 1352 qedr_db_recovery_del(dev, qp->sq.db, 1353 &qp->sq.db_data); 1354 } 1355 1356 return rc; 1357 } 1358 1359 static int qedr_check_srq_params(struct qedr_dev *dev, 1360 struct ib_srq_init_attr *attrs, 1361 struct ib_udata *udata) 1362 { 1363 struct qedr_device_attr *qattr = &dev->attr; 1364 1365 if (attrs->attr.max_wr > qattr->max_srq_wr) { 1366 DP_ERR(dev, 1367 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n", 1368 attrs->attr.max_wr, qattr->max_srq_wr); 1369 return -EINVAL; 1370 } 1371 1372 if (attrs->attr.max_sge > qattr->max_sge) { 1373 DP_ERR(dev, 1374 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", 1375 attrs->attr.max_sge, qattr->max_sge); 1376 return -EINVAL; 1377 } 1378 1379 return 0; 1380 } 1381 1382 static void qedr_free_srq_user_params(struct qedr_srq *srq) 1383 { 1384 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1385 ib_umem_release(srq->usrq.umem); 1386 ib_umem_release(srq->prod_umem); 1387 } 1388 1389 static void qedr_free_srq_kernel_params(struct qedr_srq *srq) 1390 { 1391 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1392 struct qedr_dev *dev = srq->dev; 1393 1394 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl); 1395 1396 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1397 hw_srq->virt_prod_pair_addr, 1398 hw_srq->phy_prod_pair_addr); 1399 } 1400 1401 static int qedr_init_srq_user_params(struct ib_udata *udata, 1402 struct qedr_srq *srq, 1403 struct qedr_create_srq_ureq *ureq, 1404 int access, int dmasync) 1405 { 1406 struct scatterlist *sg; 1407 int rc; 1408 1409 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr, 1410 ureq->srq_len, false, access, dmasync, 1); 1411 if (rc) 1412 return rc; 1413 1414 srq->prod_umem = 1415 ib_umem_get(udata, ureq->prod_pair_addr, 1416 sizeof(struct rdma_srq_producers), access, dmasync); 1417 if (IS_ERR(srq->prod_umem)) { 1418 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); 1419 ib_umem_release(srq->usrq.umem); 1420 DP_ERR(srq->dev, 1421 "create srq: failed ib_umem_get for producer, got %ld\n", 1422 PTR_ERR(srq->prod_umem)); 1423 return PTR_ERR(srq->prod_umem); 1424 } 1425 1426 sg = srq->prod_umem->sg_head.sgl; 1427 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); 1428 1429 return 0; 1430 } 1431 1432 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq, 1433 struct qedr_dev *dev, 1434 struct ib_srq_init_attr *init_attr) 1435 { 1436 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq; 1437 dma_addr_t phy_prod_pair_addr; 1438 u32 num_elems; 1439 void *va; 1440 int rc; 1441 1442 va = dma_alloc_coherent(&dev->pdev->dev, 1443 sizeof(struct rdma_srq_producers), 1444 &phy_prod_pair_addr, GFP_KERNEL); 1445 if (!va) { 1446 DP_ERR(dev, 1447 "create srq: failed to allocate dma memory for producer\n"); 1448 return -ENOMEM; 1449 } 1450 1451 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; 1452 hw_srq->virt_prod_pair_addr = va; 1453 1454 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE; 1455 rc = dev->ops->common->chain_alloc(dev->cdev, 1456 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1457 QED_CHAIN_MODE_PBL, 1458 QED_CHAIN_CNT_TYPE_U32, 1459 num_elems, 1460 QEDR_SRQ_WQE_ELEM_SIZE, 1461 &hw_srq->pbl, NULL); 1462 if (rc) 1463 goto err0; 1464 1465 hw_srq->num_elems = num_elems; 1466 1467 return 0; 1468 1469 err0: 1470 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers), 1471 va, phy_prod_pair_addr); 1472 return rc; 1473 } 1474 1475 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, 1476 struct ib_udata *udata) 1477 { 1478 struct qed_rdma_destroy_srq_in_params destroy_in_params; 1479 struct qed_rdma_create_srq_in_params in_params = {}; 1480 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1481 struct qed_rdma_create_srq_out_params out_params; 1482 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd); 1483 struct qedr_create_srq_ureq ureq = {}; 1484 u64 pbl_base_addr, phy_prod_pair_addr; 1485 struct qedr_srq_hwq_info *hw_srq; 1486 u32 page_cnt, page_size; 1487 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1488 int rc = 0; 1489 1490 DP_DEBUG(dev, QEDR_MSG_QP, 1491 "create SRQ called from %s (pd %p)\n", 1492 (udata) ? "User lib" : "kernel", pd); 1493 1494 rc = qedr_check_srq_params(dev, init_attr, udata); 1495 if (rc) 1496 return -EINVAL; 1497 1498 srq->dev = dev; 1499 hw_srq = &srq->hw_srq; 1500 spin_lock_init(&srq->lock); 1501 1502 hw_srq->max_wr = init_attr->attr.max_wr; 1503 hw_srq->max_sges = init_attr->attr.max_sge; 1504 1505 if (udata) { 1506 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), 1507 udata->inlen))) { 1508 DP_ERR(dev, 1509 "create srq: problem copying data from user space\n"); 1510 goto err0; 1511 } 1512 1513 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0); 1514 if (rc) 1515 goto err0; 1516 1517 page_cnt = srq->usrq.pbl_info.num_pbes; 1518 pbl_base_addr = srq->usrq.pbl_tbl->pa; 1519 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1520 page_size = PAGE_SIZE; 1521 } else { 1522 struct qed_chain *pbl; 1523 1524 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr); 1525 if (rc) 1526 goto err0; 1527 1528 pbl = &hw_srq->pbl; 1529 page_cnt = qed_chain_get_page_cnt(pbl); 1530 pbl_base_addr = qed_chain_get_pbl_phys(pbl); 1531 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; 1532 page_size = QED_CHAIN_PAGE_SIZE; 1533 } 1534 1535 in_params.pd_id = pd->pd_id; 1536 in_params.pbl_base_addr = pbl_base_addr; 1537 in_params.prod_pair_addr = phy_prod_pair_addr; 1538 in_params.num_pages = page_cnt; 1539 in_params.page_size = page_size; 1540 1541 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); 1542 if (rc) 1543 goto err1; 1544 1545 srq->srq_id = out_params.srq_id; 1546 1547 if (udata) { 1548 rc = qedr_copy_srq_uresp(dev, srq, udata); 1549 if (rc) 1550 goto err2; 1551 } 1552 1553 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL); 1554 if (rc) 1555 goto err2; 1556 1557 DP_DEBUG(dev, QEDR_MSG_SRQ, 1558 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id); 1559 return 0; 1560 1561 err2: 1562 destroy_in_params.srq_id = srq->srq_id; 1563 1564 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); 1565 err1: 1566 if (udata) 1567 qedr_free_srq_user_params(srq); 1568 else 1569 qedr_free_srq_kernel_params(srq); 1570 err0: 1571 return -EFAULT; 1572 } 1573 1574 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 1575 { 1576 struct qed_rdma_destroy_srq_in_params in_params = {}; 1577 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1578 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1579 1580 xa_erase_irq(&dev->srqs, srq->srq_id); 1581 in_params.srq_id = srq->srq_id; 1582 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params); 1583 1584 if (ibsrq->uobject) 1585 qedr_free_srq_user_params(srq); 1586 else 1587 qedr_free_srq_kernel_params(srq); 1588 1589 DP_DEBUG(dev, QEDR_MSG_SRQ, 1590 "destroy srq: destroyed srq with srq_id=0x%0x\n", 1591 srq->srq_id); 1592 } 1593 1594 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 1595 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) 1596 { 1597 struct qed_rdma_modify_srq_in_params in_params = {}; 1598 struct qedr_dev *dev = get_qedr_dev(ibsrq->device); 1599 struct qedr_srq *srq = get_qedr_srq(ibsrq); 1600 int rc; 1601 1602 if (attr_mask & IB_SRQ_MAX_WR) { 1603 DP_ERR(dev, 1604 "modify srq: invalid attribute mask=0x%x specified for %p\n", 1605 attr_mask, srq); 1606 return -EINVAL; 1607 } 1608 1609 if (attr_mask & IB_SRQ_LIMIT) { 1610 if (attr->srq_limit >= srq->hw_srq.max_wr) { 1611 DP_ERR(dev, 1612 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n", 1613 attr->srq_limit, srq->hw_srq.max_wr); 1614 return -EINVAL; 1615 } 1616 1617 in_params.srq_id = srq->srq_id; 1618 in_params.wqe_limit = attr->srq_limit; 1619 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params); 1620 if (rc) 1621 return rc; 1622 } 1623 1624 srq->srq_limit = attr->srq_limit; 1625 1626 DP_DEBUG(dev, QEDR_MSG_SRQ, 1627 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id); 1628 1629 return 0; 1630 } 1631 1632 static inline void 1633 qedr_init_common_qp_in_params(struct qedr_dev *dev, 1634 struct qedr_pd *pd, 1635 struct qedr_qp *qp, 1636 struct ib_qp_init_attr *attrs, 1637 bool fmr_and_reserved_lkey, 1638 struct qed_rdma_create_qp_in_params *params) 1639 { 1640 /* QP handle to be written in an async event */ 1641 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp); 1642 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp); 1643 1644 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); 1645 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; 1646 params->pd = pd->pd_id; 1647 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; 1648 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid; 1649 params->stats_queue = 0; 1650 params->srq_id = 0; 1651 params->use_srq = false; 1652 1653 if (!qp->srq) { 1654 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1655 1656 } else { 1657 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid; 1658 params->srq_id = qp->srq->srq_id; 1659 params->use_srq = true; 1660 } 1661 } 1662 1663 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp) 1664 { 1665 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. " 1666 "qp=%p. " 1667 "sq_addr=0x%llx, " 1668 "sq_len=%zd, " 1669 "rq_addr=0x%llx, " 1670 "rq_len=%zd" 1671 "\n", 1672 qp, 1673 qp->usq.buf_addr, 1674 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); 1675 } 1676 1677 static inline void 1678 qedr_iwarp_populate_user_qp(struct qedr_dev *dev, 1679 struct qedr_qp *qp, 1680 struct qed_rdma_create_qp_out_params *out_params) 1681 { 1682 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; 1683 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; 1684 1685 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, 1686 &qp->usq.pbl_info, FW_PAGE_SHIFT); 1687 if (!qp->srq) { 1688 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; 1689 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; 1690 } 1691 1692 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, 1693 &qp->urq.pbl_info, FW_PAGE_SHIFT); 1694 } 1695 1696 static void qedr_cleanup_user(struct qedr_dev *dev, 1697 struct qedr_ucontext *ctx, 1698 struct qedr_qp *qp) 1699 { 1700 ib_umem_release(qp->usq.umem); 1701 qp->usq.umem = NULL; 1702 1703 ib_umem_release(qp->urq.umem); 1704 qp->urq.umem = NULL; 1705 1706 if (rdma_protocol_roce(&dev->ibdev, 1)) { 1707 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl); 1708 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl); 1709 } else { 1710 kfree(qp->usq.pbl_tbl); 1711 kfree(qp->urq.pbl_tbl); 1712 } 1713 1714 if (qp->usq.db_rec_data) { 1715 qedr_db_recovery_del(dev, qp->usq.db_addr, 1716 &qp->usq.db_rec_data->db_data); 1717 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry); 1718 } 1719 1720 if (qp->urq.db_rec_data) { 1721 qedr_db_recovery_del(dev, qp->urq.db_addr, 1722 &qp->urq.db_rec_data->db_data); 1723 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry); 1724 } 1725 1726 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1727 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr, 1728 &qp->urq.db_rec_db2_data); 1729 } 1730 1731 static int qedr_create_user_qp(struct qedr_dev *dev, 1732 struct qedr_qp *qp, 1733 struct ib_pd *ibpd, 1734 struct ib_udata *udata, 1735 struct ib_qp_init_attr *attrs) 1736 { 1737 struct qed_rdma_create_qp_in_params in_params; 1738 struct qed_rdma_create_qp_out_params out_params; 1739 struct qedr_pd *pd = get_qedr_pd(ibpd); 1740 struct qedr_create_qp_uresp uresp; 1741 struct qedr_ucontext *ctx = NULL; 1742 struct qedr_create_qp_ureq ureq; 1743 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1); 1744 int rc = -EINVAL; 1745 1746 qp->create_type = QEDR_QP_CREATE_USER; 1747 memset(&ureq, 0, sizeof(ureq)); 1748 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen)); 1749 if (rc) { 1750 DP_ERR(dev, "Problem copying data from user space\n"); 1751 return rc; 1752 } 1753 1754 /* SQ - read access only (0), dma sync not required (0) */ 1755 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr, 1756 ureq.sq_len, true, 0, 0, 1757 alloc_and_init); 1758 if (rc) 1759 return rc; 1760 1761 if (!qp->srq) { 1762 /* RQ - read access only (0), dma sync not required (0) */ 1763 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr, 1764 ureq.rq_len, true, 1765 0, 0, alloc_and_init); 1766 if (rc) 1767 return rc; 1768 } 1769 1770 memset(&in_params, 0, sizeof(in_params)); 1771 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); 1772 in_params.qp_handle_lo = ureq.qp_handle_lo; 1773 in_params.qp_handle_hi = ureq.qp_handle_hi; 1774 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; 1775 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; 1776 if (!qp->srq) { 1777 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; 1778 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; 1779 } 1780 1781 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1782 &in_params, &out_params); 1783 1784 if (!qp->qed_qp) { 1785 rc = -ENOMEM; 1786 goto err1; 1787 } 1788 1789 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 1790 qedr_iwarp_populate_user_qp(dev, qp, &out_params); 1791 1792 qp->qp_id = out_params.qp_id; 1793 qp->icid = out_params.icid; 1794 1795 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp); 1796 if (rc) 1797 goto err; 1798 1799 /* db offset was calculated in copy_qp_uresp, now set in the user q */ 1800 ctx = pd->uctx; 1801 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset; 1802 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset; 1803 1804 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1805 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset; 1806 1807 /* calculate the db_rec_db2 data since it is constant so no 1808 * need to reflect from user 1809 */ 1810 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid); 1811 qp->urq.db_rec_db2_data.data.value = 1812 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD); 1813 } 1814 1815 rc = qedr_db_recovery_add(dev, qp->usq.db_addr, 1816 &qp->usq.db_rec_data->db_data, 1817 DB_REC_WIDTH_32B, 1818 DB_REC_USER); 1819 if (rc) 1820 goto err; 1821 1822 rc = qedr_db_recovery_add(dev, qp->urq.db_addr, 1823 &qp->urq.db_rec_data->db_data, 1824 DB_REC_WIDTH_32B, 1825 DB_REC_USER); 1826 if (rc) 1827 goto err; 1828 1829 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 1830 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr, 1831 &qp->urq.db_rec_db2_data, 1832 DB_REC_WIDTH_32B, 1833 DB_REC_USER); 1834 if (rc) 1835 goto err; 1836 } 1837 qedr_qp_user_print(dev, qp); 1838 1839 return rc; 1840 err: 1841 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1842 if (rc) 1843 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc); 1844 1845 err1: 1846 qedr_cleanup_user(dev, ctx, qp); 1847 return rc; 1848 } 1849 1850 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp) 1851 { 1852 int rc; 1853 1854 qp->sq.db = dev->db_addr + 1855 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); 1856 qp->sq.db_data.data.icid = qp->icid; 1857 1858 rc = qedr_db_recovery_add(dev, qp->sq.db, 1859 &qp->sq.db_data, 1860 DB_REC_WIDTH_32B, 1861 DB_REC_KERNEL); 1862 if (rc) 1863 return rc; 1864 1865 qp->rq.db = dev->db_addr + 1866 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); 1867 qp->rq.db_data.data.icid = qp->icid; 1868 qp->rq.iwarp_db2 = dev->db_addr + 1869 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); 1870 qp->rq.iwarp_db2_data.data.icid = qp->icid; 1871 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; 1872 1873 rc = qedr_db_recovery_add(dev, qp->rq.db, 1874 &qp->rq.db_data, 1875 DB_REC_WIDTH_32B, 1876 DB_REC_KERNEL); 1877 if (rc) 1878 return rc; 1879 1880 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2, 1881 &qp->rq.iwarp_db2_data, 1882 DB_REC_WIDTH_32B, 1883 DB_REC_KERNEL); 1884 return rc; 1885 } 1886 1887 static int 1888 qedr_roce_create_kernel_qp(struct qedr_dev *dev, 1889 struct qedr_qp *qp, 1890 struct qed_rdma_create_qp_in_params *in_params, 1891 u32 n_sq_elems, u32 n_rq_elems) 1892 { 1893 struct qed_rdma_create_qp_out_params out_params; 1894 int rc; 1895 1896 rc = dev->ops->common->chain_alloc(dev->cdev, 1897 QED_CHAIN_USE_TO_PRODUCE, 1898 QED_CHAIN_MODE_PBL, 1899 QED_CHAIN_CNT_TYPE_U32, 1900 n_sq_elems, 1901 QEDR_SQE_ELEMENT_SIZE, 1902 &qp->sq.pbl, NULL); 1903 1904 if (rc) 1905 return rc; 1906 1907 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl); 1908 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl); 1909 1910 rc = dev->ops->common->chain_alloc(dev->cdev, 1911 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1912 QED_CHAIN_MODE_PBL, 1913 QED_CHAIN_CNT_TYPE_U32, 1914 n_rq_elems, 1915 QEDR_RQE_ELEMENT_SIZE, 1916 &qp->rq.pbl, NULL); 1917 if (rc) 1918 return rc; 1919 1920 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl); 1921 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl); 1922 1923 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1924 in_params, &out_params); 1925 1926 if (!qp->qed_qp) 1927 return -EINVAL; 1928 1929 qp->qp_id = out_params.qp_id; 1930 qp->icid = out_params.icid; 1931 1932 return qedr_set_roce_db_info(dev, qp); 1933 } 1934 1935 static int 1936 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev, 1937 struct qedr_qp *qp, 1938 struct qed_rdma_create_qp_in_params *in_params, 1939 u32 n_sq_elems, u32 n_rq_elems) 1940 { 1941 struct qed_rdma_create_qp_out_params out_params; 1942 struct qed_chain_ext_pbl ext_pbl; 1943 int rc; 1944 1945 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems, 1946 QEDR_SQE_ELEMENT_SIZE, 1947 QED_CHAIN_MODE_PBL); 1948 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems, 1949 QEDR_RQE_ELEMENT_SIZE, 1950 QED_CHAIN_MODE_PBL); 1951 1952 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx, 1953 in_params, &out_params); 1954 1955 if (!qp->qed_qp) 1956 return -EINVAL; 1957 1958 /* Now we allocate the chain */ 1959 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; 1960 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; 1961 1962 rc = dev->ops->common->chain_alloc(dev->cdev, 1963 QED_CHAIN_USE_TO_PRODUCE, 1964 QED_CHAIN_MODE_PBL, 1965 QED_CHAIN_CNT_TYPE_U32, 1966 n_sq_elems, 1967 QEDR_SQE_ELEMENT_SIZE, 1968 &qp->sq.pbl, &ext_pbl); 1969 1970 if (rc) 1971 goto err; 1972 1973 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; 1974 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; 1975 1976 rc = dev->ops->common->chain_alloc(dev->cdev, 1977 QED_CHAIN_USE_TO_CONSUME_PRODUCE, 1978 QED_CHAIN_MODE_PBL, 1979 QED_CHAIN_CNT_TYPE_U32, 1980 n_rq_elems, 1981 QEDR_RQE_ELEMENT_SIZE, 1982 &qp->rq.pbl, &ext_pbl); 1983 1984 if (rc) 1985 goto err; 1986 1987 qp->qp_id = out_params.qp_id; 1988 qp->icid = out_params.icid; 1989 1990 return qedr_set_iwarp_db_info(dev, qp); 1991 1992 err: 1993 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 1994 1995 return rc; 1996 } 1997 1998 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp) 1999 { 2000 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl); 2001 kfree(qp->wqe_wr_id); 2002 2003 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl); 2004 kfree(qp->rqe_wr_id); 2005 2006 /* GSI qp is not registered to db mechanism so no need to delete */ 2007 if (qp->qp_type == IB_QPT_GSI) 2008 return; 2009 2010 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data); 2011 2012 if (!qp->srq) { 2013 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data); 2014 2015 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2016 qedr_db_recovery_del(dev, qp->rq.iwarp_db2, 2017 &qp->rq.iwarp_db2_data); 2018 } 2019 } 2020 2021 static int qedr_create_kernel_qp(struct qedr_dev *dev, 2022 struct qedr_qp *qp, 2023 struct ib_pd *ibpd, 2024 struct ib_qp_init_attr *attrs) 2025 { 2026 struct qed_rdma_create_qp_in_params in_params; 2027 struct qedr_pd *pd = get_qedr_pd(ibpd); 2028 int rc = -EINVAL; 2029 u32 n_rq_elems; 2030 u32 n_sq_elems; 2031 u32 n_sq_entries; 2032 2033 memset(&in_params, 0, sizeof(in_params)); 2034 qp->create_type = QEDR_QP_CREATE_KERNEL; 2035 2036 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in 2037 * the ring. The ring should allow at least a single WR, even if the 2038 * user requested none, due to allocation issues. 2039 * We should add an extra WR since the prod and cons indices of 2040 * wqe_wr_id are managed in such a way that the WQ is considered full 2041 * when (prod+1)%max_wr==cons. We currently don't do that because we 2042 * double the number of entries due an iSER issue that pushes far more 2043 * WRs than indicated. If we decline its ib_post_send() then we get 2044 * error prints in the dmesg we'd like to avoid. 2045 */ 2046 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, 2047 dev->attr.max_sqe); 2048 2049 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), 2050 GFP_KERNEL); 2051 if (!qp->wqe_wr_id) { 2052 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n"); 2053 return -ENOMEM; 2054 } 2055 2056 /* QP handle to be written in CQE */ 2057 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp); 2058 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp); 2059 2060 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in 2061 * the ring. There ring should allow at least a single WR, even if the 2062 * user requested none, due to allocation issues. 2063 */ 2064 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1); 2065 2066 /* Allocate driver internal RQ array */ 2067 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), 2068 GFP_KERNEL); 2069 if (!qp->rqe_wr_id) { 2070 DP_ERR(dev, 2071 "create qp: failed RQ shadow memory allocation\n"); 2072 kfree(qp->wqe_wr_id); 2073 return -ENOMEM; 2074 } 2075 2076 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); 2077 2078 n_sq_entries = attrs->cap.max_send_wr; 2079 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe); 2080 n_sq_entries = max_t(u32, n_sq_entries, 1); 2081 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE; 2082 2083 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE; 2084 2085 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2086 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params, 2087 n_sq_elems, n_rq_elems); 2088 else 2089 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params, 2090 n_sq_elems, n_rq_elems); 2091 if (rc) 2092 qedr_cleanup_kernel(dev, qp); 2093 2094 return rc; 2095 } 2096 2097 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd, 2098 struct ib_qp_init_attr *attrs, 2099 struct ib_udata *udata) 2100 { 2101 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2102 struct qedr_pd *pd = get_qedr_pd(ibpd); 2103 struct qedr_qp *qp; 2104 struct ib_qp *ibqp; 2105 int rc = 0; 2106 2107 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n", 2108 udata ? "user library" : "kernel", pd); 2109 2110 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata); 2111 if (rc) 2112 return ERR_PTR(rc); 2113 2114 DP_DEBUG(dev, QEDR_MSG_QP, 2115 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", 2116 udata ? "user library" : "kernel", attrs->event_handler, pd, 2117 get_qedr_cq(attrs->send_cq), 2118 get_qedr_cq(attrs->send_cq)->icid, 2119 get_qedr_cq(attrs->recv_cq), 2120 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0); 2121 2122 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2123 if (!qp) { 2124 DP_ERR(dev, "create qp: failed allocating memory\n"); 2125 return ERR_PTR(-ENOMEM); 2126 } 2127 2128 qedr_set_common_qp_params(dev, qp, pd, attrs); 2129 2130 if (attrs->qp_type == IB_QPT_GSI) { 2131 ibqp = qedr_create_gsi_qp(dev, attrs, qp); 2132 if (IS_ERR(ibqp)) 2133 kfree(qp); 2134 return ibqp; 2135 } 2136 2137 if (udata) 2138 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs); 2139 else 2140 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs); 2141 2142 if (rc) 2143 goto err; 2144 2145 qp->ibqp.qp_num = qp->qp_id; 2146 2147 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 2148 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL); 2149 if (rc) 2150 goto err; 2151 } 2152 2153 return &qp->ibqp; 2154 2155 err: 2156 kfree(qp); 2157 2158 return ERR_PTR(-EFAULT); 2159 } 2160 2161 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state) 2162 { 2163 switch (qp_state) { 2164 case QED_ROCE_QP_STATE_RESET: 2165 return IB_QPS_RESET; 2166 case QED_ROCE_QP_STATE_INIT: 2167 return IB_QPS_INIT; 2168 case QED_ROCE_QP_STATE_RTR: 2169 return IB_QPS_RTR; 2170 case QED_ROCE_QP_STATE_RTS: 2171 return IB_QPS_RTS; 2172 case QED_ROCE_QP_STATE_SQD: 2173 return IB_QPS_SQD; 2174 case QED_ROCE_QP_STATE_ERR: 2175 return IB_QPS_ERR; 2176 case QED_ROCE_QP_STATE_SQE: 2177 return IB_QPS_SQE; 2178 } 2179 return IB_QPS_ERR; 2180 } 2181 2182 static enum qed_roce_qp_state qedr_get_state_from_ibqp( 2183 enum ib_qp_state qp_state) 2184 { 2185 switch (qp_state) { 2186 case IB_QPS_RESET: 2187 return QED_ROCE_QP_STATE_RESET; 2188 case IB_QPS_INIT: 2189 return QED_ROCE_QP_STATE_INIT; 2190 case IB_QPS_RTR: 2191 return QED_ROCE_QP_STATE_RTR; 2192 case IB_QPS_RTS: 2193 return QED_ROCE_QP_STATE_RTS; 2194 case IB_QPS_SQD: 2195 return QED_ROCE_QP_STATE_SQD; 2196 case IB_QPS_ERR: 2197 return QED_ROCE_QP_STATE_ERR; 2198 default: 2199 return QED_ROCE_QP_STATE_ERR; 2200 } 2201 } 2202 2203 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph) 2204 { 2205 qed_chain_reset(&qph->pbl); 2206 qph->prod = 0; 2207 qph->cons = 0; 2208 qph->wqe_cons = 0; 2209 qph->db_data.data.value = cpu_to_le16(0); 2210 } 2211 2212 static int qedr_update_qp_state(struct qedr_dev *dev, 2213 struct qedr_qp *qp, 2214 enum qed_roce_qp_state cur_state, 2215 enum qed_roce_qp_state new_state) 2216 { 2217 int status = 0; 2218 2219 if (new_state == cur_state) 2220 return 0; 2221 2222 switch (cur_state) { 2223 case QED_ROCE_QP_STATE_RESET: 2224 switch (new_state) { 2225 case QED_ROCE_QP_STATE_INIT: 2226 qp->prev_wqe_size = 0; 2227 qedr_reset_qp_hwq_info(&qp->sq); 2228 qedr_reset_qp_hwq_info(&qp->rq); 2229 break; 2230 default: 2231 status = -EINVAL; 2232 break; 2233 } 2234 break; 2235 case QED_ROCE_QP_STATE_INIT: 2236 switch (new_state) { 2237 case QED_ROCE_QP_STATE_RTR: 2238 /* Update doorbell (in case post_recv was 2239 * done before move to RTR) 2240 */ 2241 2242 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2243 writel(qp->rq.db_data.raw, qp->rq.db); 2244 } 2245 break; 2246 case QED_ROCE_QP_STATE_ERR: 2247 break; 2248 default: 2249 /* Invalid state change. */ 2250 status = -EINVAL; 2251 break; 2252 } 2253 break; 2254 case QED_ROCE_QP_STATE_RTR: 2255 /* RTR->XXX */ 2256 switch (new_state) { 2257 case QED_ROCE_QP_STATE_RTS: 2258 break; 2259 case QED_ROCE_QP_STATE_ERR: 2260 break; 2261 default: 2262 /* Invalid state change. */ 2263 status = -EINVAL; 2264 break; 2265 } 2266 break; 2267 case QED_ROCE_QP_STATE_RTS: 2268 /* RTS->XXX */ 2269 switch (new_state) { 2270 case QED_ROCE_QP_STATE_SQD: 2271 break; 2272 case QED_ROCE_QP_STATE_ERR: 2273 break; 2274 default: 2275 /* Invalid state change. */ 2276 status = -EINVAL; 2277 break; 2278 } 2279 break; 2280 case QED_ROCE_QP_STATE_SQD: 2281 /* SQD->XXX */ 2282 switch (new_state) { 2283 case QED_ROCE_QP_STATE_RTS: 2284 case QED_ROCE_QP_STATE_ERR: 2285 break; 2286 default: 2287 /* Invalid state change. */ 2288 status = -EINVAL; 2289 break; 2290 } 2291 break; 2292 case QED_ROCE_QP_STATE_ERR: 2293 /* ERR->XXX */ 2294 switch (new_state) { 2295 case QED_ROCE_QP_STATE_RESET: 2296 if ((qp->rq.prod != qp->rq.cons) || 2297 (qp->sq.prod != qp->sq.cons)) { 2298 DP_NOTICE(dev, 2299 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n", 2300 qp->rq.prod, qp->rq.cons, qp->sq.prod, 2301 qp->sq.cons); 2302 status = -EINVAL; 2303 } 2304 break; 2305 default: 2306 status = -EINVAL; 2307 break; 2308 } 2309 break; 2310 default: 2311 status = -EINVAL; 2312 break; 2313 } 2314 2315 return status; 2316 } 2317 2318 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 2319 int attr_mask, struct ib_udata *udata) 2320 { 2321 struct qedr_qp *qp = get_qedr_qp(ibqp); 2322 struct qed_rdma_modify_qp_in_params qp_params = { 0 }; 2323 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev); 2324 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr); 2325 enum ib_qp_state old_qp_state, new_qp_state; 2326 enum qed_roce_qp_state cur_state; 2327 int rc = 0; 2328 2329 DP_DEBUG(dev, QEDR_MSG_QP, 2330 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask, 2331 attr->qp_state); 2332 2333 old_qp_state = qedr_get_ibqp_state(qp->state); 2334 if (attr_mask & IB_QP_STATE) 2335 new_qp_state = attr->qp_state; 2336 else 2337 new_qp_state = old_qp_state; 2338 2339 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2340 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, 2341 ibqp->qp_type, attr_mask)) { 2342 DP_ERR(dev, 2343 "modify qp: invalid attribute mask=0x%x specified for\n" 2344 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n", 2345 attr_mask, qp->qp_id, ibqp->qp_type, 2346 old_qp_state, new_qp_state); 2347 rc = -EINVAL; 2348 goto err; 2349 } 2350 } 2351 2352 /* Translate the masks... */ 2353 if (attr_mask & IB_QP_STATE) { 2354 SET_FIELD(qp_params.modify_flags, 2355 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); 2356 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state); 2357 } 2358 2359 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) 2360 qp_params.sqd_async = true; 2361 2362 if (attr_mask & IB_QP_PKEY_INDEX) { 2363 SET_FIELD(qp_params.modify_flags, 2364 QED_ROCE_MODIFY_QP_VALID_PKEY, 1); 2365 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) { 2366 rc = -EINVAL; 2367 goto err; 2368 } 2369 2370 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT; 2371 } 2372 2373 if (attr_mask & IB_QP_QKEY) 2374 qp->qkey = attr->qkey; 2375 2376 if (attr_mask & IB_QP_ACCESS_FLAGS) { 2377 SET_FIELD(qp_params.modify_flags, 2378 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); 2379 qp_params.incoming_rdma_read_en = attr->qp_access_flags & 2380 IB_ACCESS_REMOTE_READ; 2381 qp_params.incoming_rdma_write_en = attr->qp_access_flags & 2382 IB_ACCESS_REMOTE_WRITE; 2383 qp_params.incoming_atomic_en = attr->qp_access_flags & 2384 IB_ACCESS_REMOTE_ATOMIC; 2385 } 2386 2387 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { 2388 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2389 return -EINVAL; 2390 2391 if (attr_mask & IB_QP_PATH_MTU) { 2392 if (attr->path_mtu < IB_MTU_256 || 2393 attr->path_mtu > IB_MTU_4096) { 2394 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n"); 2395 rc = -EINVAL; 2396 goto err; 2397 } 2398 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), 2399 ib_mtu_enum_to_int(iboe_get_mtu 2400 (dev->ndev->mtu))); 2401 } 2402 2403 if (!qp->mtu) { 2404 qp->mtu = 2405 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2406 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu); 2407 } 2408 2409 SET_FIELD(qp_params.modify_flags, 2410 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); 2411 2412 qp_params.traffic_class_tos = grh->traffic_class; 2413 qp_params.flow_label = grh->flow_label; 2414 qp_params.hop_limit_ttl = grh->hop_limit; 2415 2416 qp->sgid_idx = grh->sgid_index; 2417 2418 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params); 2419 if (rc) { 2420 DP_ERR(dev, 2421 "modify qp: problems with GID index %d (rc=%d)\n", 2422 grh->sgid_index, rc); 2423 return rc; 2424 } 2425 2426 rc = qedr_get_dmac(dev, &attr->ah_attr, 2427 qp_params.remote_mac_addr); 2428 if (rc) 2429 return rc; 2430 2431 qp_params.use_local_mac = true; 2432 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr); 2433 2434 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n", 2435 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], 2436 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); 2437 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n", 2438 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], 2439 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); 2440 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n", 2441 qp_params.remote_mac_addr); 2442 2443 qp_params.mtu = qp->mtu; 2444 qp_params.lb_indication = false; 2445 } 2446 2447 if (!qp_params.mtu) { 2448 /* Stay with current MTU */ 2449 if (qp->mtu) 2450 qp_params.mtu = qp->mtu; 2451 else 2452 qp_params.mtu = 2453 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu)); 2454 } 2455 2456 if (attr_mask & IB_QP_TIMEOUT) { 2457 SET_FIELD(qp_params.modify_flags, 2458 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); 2459 2460 /* The received timeout value is an exponent used like this: 2461 * "12.7.34 LOCAL ACK TIMEOUT 2462 * Value representing the transport (ACK) timeout for use by 2463 * the remote, expressed as: 4.096 * 2^timeout [usec]" 2464 * The FW expects timeout in msec so we need to divide the usec 2465 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2, 2466 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8). 2467 * The value of zero means infinite so we use a 'max_t' to make 2468 * sure that sub 1 msec values will be configured as 1 msec. 2469 */ 2470 if (attr->timeout) 2471 qp_params.ack_timeout = 2472 1 << max_t(int, attr->timeout - 8, 0); 2473 else 2474 qp_params.ack_timeout = 0; 2475 } 2476 2477 if (attr_mask & IB_QP_RETRY_CNT) { 2478 SET_FIELD(qp_params.modify_flags, 2479 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); 2480 qp_params.retry_cnt = attr->retry_cnt; 2481 } 2482 2483 if (attr_mask & IB_QP_RNR_RETRY) { 2484 SET_FIELD(qp_params.modify_flags, 2485 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); 2486 qp_params.rnr_retry_cnt = attr->rnr_retry; 2487 } 2488 2489 if (attr_mask & IB_QP_RQ_PSN) { 2490 SET_FIELD(qp_params.modify_flags, 2491 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); 2492 qp_params.rq_psn = attr->rq_psn; 2493 qp->rq_psn = attr->rq_psn; 2494 } 2495 2496 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 2497 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) { 2498 rc = -EINVAL; 2499 DP_ERR(dev, 2500 "unsupported max_rd_atomic=%d, supported=%d\n", 2501 attr->max_rd_atomic, 2502 dev->attr.max_qp_req_rd_atomic_resc); 2503 goto err; 2504 } 2505 2506 SET_FIELD(qp_params.modify_flags, 2507 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); 2508 qp_params.max_rd_atomic_req = attr->max_rd_atomic; 2509 } 2510 2511 if (attr_mask & IB_QP_MIN_RNR_TIMER) { 2512 SET_FIELD(qp_params.modify_flags, 2513 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); 2514 qp_params.min_rnr_nak_timer = attr->min_rnr_timer; 2515 } 2516 2517 if (attr_mask & IB_QP_SQ_PSN) { 2518 SET_FIELD(qp_params.modify_flags, 2519 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); 2520 qp_params.sq_psn = attr->sq_psn; 2521 qp->sq_psn = attr->sq_psn; 2522 } 2523 2524 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 2525 if (attr->max_dest_rd_atomic > 2526 dev->attr.max_qp_resp_rd_atomic_resc) { 2527 DP_ERR(dev, 2528 "unsupported max_dest_rd_atomic=%d, supported=%d\n", 2529 attr->max_dest_rd_atomic, 2530 dev->attr.max_qp_resp_rd_atomic_resc); 2531 2532 rc = -EINVAL; 2533 goto err; 2534 } 2535 2536 SET_FIELD(qp_params.modify_flags, 2537 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); 2538 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; 2539 } 2540 2541 if (attr_mask & IB_QP_DEST_QPN) { 2542 SET_FIELD(qp_params.modify_flags, 2543 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1); 2544 2545 qp_params.dest_qp = attr->dest_qp_num; 2546 qp->dest_qp_num = attr->dest_qp_num; 2547 } 2548 2549 cur_state = qp->state; 2550 2551 /* Update the QP state before the actual ramrod to prevent a race with 2552 * fast path. Modifying the QP state to error will cause the device to 2553 * flush the CQEs and while polling the flushed CQEs will considered as 2554 * a potential issue if the QP isn't in error state. 2555 */ 2556 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI && 2557 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR) 2558 qp->state = QED_ROCE_QP_STATE_ERR; 2559 2560 if (qp->qp_type != IB_QPT_GSI) 2561 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx, 2562 qp->qed_qp, &qp_params); 2563 2564 if (attr_mask & IB_QP_STATE) { 2565 if ((qp->qp_type != IB_QPT_GSI) && (!udata)) 2566 rc = qedr_update_qp_state(dev, qp, cur_state, 2567 qp_params.new_state); 2568 qp->state = qp_params.new_state; 2569 } 2570 2571 err: 2572 return rc; 2573 } 2574 2575 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params) 2576 { 2577 int ib_qp_acc_flags = 0; 2578 2579 if (params->incoming_rdma_write_en) 2580 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; 2581 if (params->incoming_rdma_read_en) 2582 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; 2583 if (params->incoming_atomic_en) 2584 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; 2585 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; 2586 return ib_qp_acc_flags; 2587 } 2588 2589 int qedr_query_qp(struct ib_qp *ibqp, 2590 struct ib_qp_attr *qp_attr, 2591 int attr_mask, struct ib_qp_init_attr *qp_init_attr) 2592 { 2593 struct qed_rdma_query_qp_out_params params; 2594 struct qedr_qp *qp = get_qedr_qp(ibqp); 2595 struct qedr_dev *dev = qp->dev; 2596 int rc = 0; 2597 2598 memset(¶ms, 0, sizeof(params)); 2599 2600 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms); 2601 if (rc) 2602 goto err; 2603 2604 memset(qp_attr, 0, sizeof(*qp_attr)); 2605 memset(qp_init_attr, 0, sizeof(*qp_init_attr)); 2606 2607 qp_attr->qp_state = qedr_get_ibqp_state(params.state); 2608 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state); 2609 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu); 2610 qp_attr->path_mig_state = IB_MIG_MIGRATED; 2611 qp_attr->rq_psn = params.rq_psn; 2612 qp_attr->sq_psn = params.sq_psn; 2613 qp_attr->dest_qp_num = params.dest_qp; 2614 2615 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms); 2616 2617 qp_attr->cap.max_send_wr = qp->sq.max_wr; 2618 qp_attr->cap.max_recv_wr = qp->rq.max_wr; 2619 qp_attr->cap.max_send_sge = qp->sq.max_sges; 2620 qp_attr->cap.max_recv_sge = qp->rq.max_sges; 2621 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; 2622 qp_init_attr->cap = qp_attr->cap; 2623 2624 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE; 2625 rdma_ah_set_grh(&qp_attr->ah_attr, NULL, 2626 params.flow_label, qp->sgid_idx, 2627 params.hop_limit_ttl, params.traffic_class_tos); 2628 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]); 2629 rdma_ah_set_port_num(&qp_attr->ah_attr, 1); 2630 rdma_ah_set_sl(&qp_attr->ah_attr, 0); 2631 qp_attr->timeout = params.timeout; 2632 qp_attr->rnr_retry = params.rnr_retry; 2633 qp_attr->retry_cnt = params.retry_cnt; 2634 qp_attr->min_rnr_timer = params.min_rnr_nak_timer; 2635 qp_attr->pkey_index = params.pkey_index; 2636 qp_attr->port_num = 1; 2637 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0); 2638 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0); 2639 qp_attr->alt_pkey_index = 0; 2640 qp_attr->alt_port_num = 0; 2641 qp_attr->alt_timeout = 0; 2642 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); 2643 2644 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0; 2645 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; 2646 qp_attr->max_rd_atomic = params.max_rd_atomic; 2647 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0; 2648 2649 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n", 2650 qp_attr->cap.max_inline_data); 2651 2652 err: 2653 return rc; 2654 } 2655 2656 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp, 2657 struct ib_udata *udata) 2658 { 2659 struct qedr_ucontext *ctx = 2660 rdma_udata_to_drv_context(udata, struct qedr_ucontext, 2661 ibucontext); 2662 int rc; 2663 2664 if (qp->qp_type != IB_QPT_GSI) { 2665 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp); 2666 if (rc) 2667 return rc; 2668 } 2669 2670 if (qp->create_type == QEDR_QP_CREATE_USER) 2671 qedr_cleanup_user(dev, ctx, qp); 2672 else 2673 qedr_cleanup_kernel(dev, qp); 2674 2675 return 0; 2676 } 2677 2678 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 2679 { 2680 struct qedr_qp *qp = get_qedr_qp(ibqp); 2681 struct qedr_dev *dev = qp->dev; 2682 struct ib_qp_attr attr; 2683 int attr_mask = 0; 2684 2685 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n", 2686 qp, qp->qp_type); 2687 2688 if (rdma_protocol_roce(&dev->ibdev, 1)) { 2689 if ((qp->state != QED_ROCE_QP_STATE_RESET) && 2690 (qp->state != QED_ROCE_QP_STATE_ERR) && 2691 (qp->state != QED_ROCE_QP_STATE_INIT)) { 2692 2693 attr.qp_state = IB_QPS_ERR; 2694 attr_mask |= IB_QP_STATE; 2695 2696 /* Change the QP state to ERROR */ 2697 qedr_modify_qp(ibqp, &attr, attr_mask, NULL); 2698 } 2699 } else { 2700 /* If connection establishment started the WAIT_FOR_CONNECT 2701 * bit will be on and we need to Wait for the establishment 2702 * to complete before destroying the qp. 2703 */ 2704 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT, 2705 &qp->iwarp_cm_flags)) 2706 wait_for_completion(&qp->iwarp_cm_comp); 2707 2708 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT 2709 * bit will be on, and we need to wait for the disconnect to 2710 * complete before continuing. We can use the same completion, 2711 * iwarp_cm_comp, since this is the only place that waits for 2712 * this completion and it is sequential. In addition, 2713 * disconnect can't occur before the connection is fully 2714 * established, therefore if WAIT_FOR_DISCONNECT is on it 2715 * means WAIT_FOR_CONNECT is also on and the completion for 2716 * CONNECT already occurred. 2717 */ 2718 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT, 2719 &qp->iwarp_cm_flags)) 2720 wait_for_completion(&qp->iwarp_cm_comp); 2721 } 2722 2723 if (qp->qp_type == IB_QPT_GSI) 2724 qedr_destroy_gsi_qp(dev); 2725 2726 /* We need to remove the entry from the xarray before we release the 2727 * qp_id to avoid a race of the qp_id being reallocated and failing 2728 * on xa_insert 2729 */ 2730 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2731 xa_erase(&dev->qps, qp->qp_id); 2732 2733 qedr_free_qp_resources(dev, qp, udata); 2734 2735 if (rdma_protocol_iwarp(&dev->ibdev, 1)) 2736 qedr_iw_qp_rem_ref(&qp->ibqp); 2737 2738 return 0; 2739 } 2740 2741 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags, 2742 struct ib_udata *udata) 2743 { 2744 struct qedr_ah *ah = get_qedr_ah(ibah); 2745 2746 rdma_copy_ah_attr(&ah->attr, attr); 2747 2748 return 0; 2749 } 2750 2751 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags) 2752 { 2753 struct qedr_ah *ah = get_qedr_ah(ibah); 2754 2755 rdma_destroy_ah_attr(&ah->attr); 2756 } 2757 2758 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info) 2759 { 2760 struct qedr_pbl *pbl, *tmp; 2761 2762 if (info->pbl_table) 2763 list_add_tail(&info->pbl_table->list_entry, 2764 &info->free_pbl_list); 2765 2766 if (!list_empty(&info->inuse_pbl_list)) 2767 list_splice(&info->inuse_pbl_list, &info->free_pbl_list); 2768 2769 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { 2770 list_del(&pbl->list_entry); 2771 qedr_free_pbl(dev, &info->pbl_info, pbl); 2772 } 2773 } 2774 2775 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info, 2776 size_t page_list_len, bool two_layered) 2777 { 2778 struct qedr_pbl *tmp; 2779 int rc; 2780 2781 INIT_LIST_HEAD(&info->free_pbl_list); 2782 INIT_LIST_HEAD(&info->inuse_pbl_list); 2783 2784 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info, 2785 page_list_len, two_layered); 2786 if (rc) 2787 goto done; 2788 2789 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2790 if (IS_ERR(info->pbl_table)) { 2791 rc = PTR_ERR(info->pbl_table); 2792 goto done; 2793 } 2794 2795 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n", 2796 &info->pbl_table->pa); 2797 2798 /* in usual case we use 2 PBLs, so we add one to free 2799 * list and allocating another one 2800 */ 2801 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); 2802 if (IS_ERR(tmp)) { 2803 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n"); 2804 goto done; 2805 } 2806 2807 list_add_tail(&tmp->list_entry, &info->free_pbl_list); 2808 2809 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa); 2810 2811 done: 2812 if (rc) 2813 free_mr_info(dev, info); 2814 2815 return rc; 2816 } 2817 2818 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 2819 u64 usr_addr, int acc, struct ib_udata *udata) 2820 { 2821 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2822 struct qedr_mr *mr; 2823 struct qedr_pd *pd; 2824 int rc = -ENOMEM; 2825 2826 pd = get_qedr_pd(ibpd); 2827 DP_DEBUG(dev, QEDR_MSG_MR, 2828 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", 2829 pd->pd_id, start, len, usr_addr, acc); 2830 2831 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) 2832 return ERR_PTR(-EINVAL); 2833 2834 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2835 if (!mr) 2836 return ERR_PTR(rc); 2837 2838 mr->type = QEDR_MR_USER; 2839 2840 mr->umem = ib_umem_get(udata, start, len, acc, 0); 2841 if (IS_ERR(mr->umem)) { 2842 rc = -EFAULT; 2843 goto err0; 2844 } 2845 2846 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); 2847 if (rc) 2848 goto err1; 2849 2850 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table, 2851 &mr->info.pbl_info, PAGE_SHIFT); 2852 2853 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2854 if (rc) { 2855 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2856 goto err1; 2857 } 2858 2859 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2860 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 2861 mr->hw_mr.key = 0; 2862 mr->hw_mr.pd = pd->pd_id; 2863 mr->hw_mr.local_read = 1; 2864 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 2865 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 2866 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 2867 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 2868 mr->hw_mr.mw_bind = false; 2869 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; 2870 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2871 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2872 mr->hw_mr.page_size_log = PAGE_SHIFT; 2873 mr->hw_mr.fbo = ib_umem_offset(mr->umem); 2874 mr->hw_mr.length = len; 2875 mr->hw_mr.vaddr = usr_addr; 2876 mr->hw_mr.zbva = false; 2877 mr->hw_mr.phy_mr = false; 2878 mr->hw_mr.dma_mr = false; 2879 2880 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2881 if (rc) { 2882 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2883 goto err2; 2884 } 2885 2886 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2887 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 2888 mr->hw_mr.remote_atomic) 2889 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2890 2891 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n", 2892 mr->ibmr.lkey); 2893 return &mr->ibmr; 2894 2895 err2: 2896 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2897 err1: 2898 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); 2899 err0: 2900 kfree(mr); 2901 return ERR_PTR(rc); 2902 } 2903 2904 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 2905 { 2906 struct qedr_mr *mr = get_qedr_mr(ib_mr); 2907 struct qedr_dev *dev = get_qedr_dev(ib_mr->device); 2908 int rc = 0; 2909 2910 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid); 2911 if (rc) 2912 return rc; 2913 2914 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2915 2916 if (mr->type != QEDR_MR_DMA) 2917 free_mr_info(dev, &mr->info); 2918 2919 /* it could be user registered memory. */ 2920 ib_umem_release(mr->umem); 2921 2922 kfree(mr); 2923 2924 return rc; 2925 } 2926 2927 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd, 2928 int max_page_list_len) 2929 { 2930 struct qedr_pd *pd = get_qedr_pd(ibpd); 2931 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 2932 struct qedr_mr *mr; 2933 int rc = -ENOMEM; 2934 2935 DP_DEBUG(dev, QEDR_MSG_MR, 2936 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id, 2937 max_page_list_len); 2938 2939 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2940 if (!mr) 2941 return ERR_PTR(rc); 2942 2943 mr->dev = dev; 2944 mr->type = QEDR_MR_FRMR; 2945 2946 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1); 2947 if (rc) 2948 goto err0; 2949 2950 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 2951 if (rc) { 2952 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 2953 goto err0; 2954 } 2955 2956 /* Index only, 18 bit long, lkey = itid << 8 | key */ 2957 mr->hw_mr.tid_type = QED_RDMA_TID_FMR; 2958 mr->hw_mr.key = 0; 2959 mr->hw_mr.pd = pd->pd_id; 2960 mr->hw_mr.local_read = 1; 2961 mr->hw_mr.local_write = 0; 2962 mr->hw_mr.remote_read = 0; 2963 mr->hw_mr.remote_write = 0; 2964 mr->hw_mr.remote_atomic = 0; 2965 mr->hw_mr.mw_bind = false; 2966 mr->hw_mr.pbl_ptr = 0; 2967 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; 2968 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); 2969 mr->hw_mr.fbo = 0; 2970 mr->hw_mr.length = 0; 2971 mr->hw_mr.vaddr = 0; 2972 mr->hw_mr.zbva = false; 2973 mr->hw_mr.phy_mr = true; 2974 mr->hw_mr.dma_mr = false; 2975 2976 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 2977 if (rc) { 2978 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 2979 goto err1; 2980 } 2981 2982 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 2983 mr->ibmr.rkey = mr->ibmr.lkey; 2984 2985 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey); 2986 return mr; 2987 2988 err1: 2989 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 2990 err0: 2991 kfree(mr); 2992 return ERR_PTR(rc); 2993 } 2994 2995 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 2996 u32 max_num_sg, struct ib_udata *udata) 2997 { 2998 struct qedr_mr *mr; 2999 3000 if (mr_type != IB_MR_TYPE_MEM_REG) 3001 return ERR_PTR(-EINVAL); 3002 3003 mr = __qedr_alloc_mr(ibpd, max_num_sg); 3004 3005 if (IS_ERR(mr)) 3006 return ERR_PTR(-EINVAL); 3007 3008 return &mr->ibmr; 3009 } 3010 3011 static int qedr_set_page(struct ib_mr *ibmr, u64 addr) 3012 { 3013 struct qedr_mr *mr = get_qedr_mr(ibmr); 3014 struct qedr_pbl *pbl_table; 3015 struct regpair *pbe; 3016 u32 pbes_in_page; 3017 3018 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { 3019 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages); 3020 return -ENOMEM; 3021 } 3022 3023 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n", 3024 mr->npages, addr); 3025 3026 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); 3027 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); 3028 pbe = (struct regpair *)pbl_table->va; 3029 pbe += mr->npages % pbes_in_page; 3030 pbe->lo = cpu_to_le32((u32)addr); 3031 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); 3032 3033 mr->npages++; 3034 3035 return 0; 3036 } 3037 3038 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info) 3039 { 3040 int work = info->completed - info->completed_handled - 1; 3041 3042 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work); 3043 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { 3044 struct qedr_pbl *pbl; 3045 3046 /* Free all the page list that are possible to be freed 3047 * (all the ones that were invalidated), under the assumption 3048 * that if an FMR was completed successfully that means that 3049 * if there was an invalidate operation before it also ended 3050 */ 3051 pbl = list_first_entry(&info->inuse_pbl_list, 3052 struct qedr_pbl, list_entry); 3053 list_move_tail(&pbl->list_entry, &info->free_pbl_list); 3054 info->completed_handled++; 3055 } 3056 } 3057 3058 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 3059 int sg_nents, unsigned int *sg_offset) 3060 { 3061 struct qedr_mr *mr = get_qedr_mr(ibmr); 3062 3063 mr->npages = 0; 3064 3065 handle_completed_mrs(mr->dev, &mr->info); 3066 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page); 3067 } 3068 3069 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc) 3070 { 3071 struct qedr_dev *dev = get_qedr_dev(ibpd->device); 3072 struct qedr_pd *pd = get_qedr_pd(ibpd); 3073 struct qedr_mr *mr; 3074 int rc; 3075 3076 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 3077 if (!mr) 3078 return ERR_PTR(-ENOMEM); 3079 3080 mr->type = QEDR_MR_DMA; 3081 3082 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); 3083 if (rc) { 3084 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc); 3085 goto err1; 3086 } 3087 3088 /* index only, 18 bit long, lkey = itid << 8 | key */ 3089 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR; 3090 mr->hw_mr.pd = pd->pd_id; 3091 mr->hw_mr.local_read = 1; 3092 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; 3093 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; 3094 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; 3095 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; 3096 mr->hw_mr.dma_mr = true; 3097 3098 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); 3099 if (rc) { 3100 DP_ERR(dev, "roce register tid returned an error %d\n", rc); 3101 goto err2; 3102 } 3103 3104 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3105 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || 3106 mr->hw_mr.remote_atomic) 3107 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; 3108 3109 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey); 3110 return &mr->ibmr; 3111 3112 err2: 3113 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); 3114 err1: 3115 kfree(mr); 3116 return ERR_PTR(rc); 3117 } 3118 3119 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq) 3120 { 3121 return (((wq->prod + 1) % wq->max_wr) == wq->cons); 3122 } 3123 3124 static int sge_data_len(struct ib_sge *sg_list, int num_sge) 3125 { 3126 int i, len = 0; 3127 3128 for (i = 0; i < num_sge; i++) 3129 len += sg_list[i].length; 3130 3131 return len; 3132 } 3133 3134 static void swap_wqe_data64(u64 *p) 3135 { 3136 int i; 3137 3138 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) 3139 *p = cpu_to_be64(cpu_to_le64(*p)); 3140 } 3141 3142 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev, 3143 struct qedr_qp *qp, u8 *wqe_size, 3144 const struct ib_send_wr *wr, 3145 const struct ib_send_wr **bad_wr, 3146 u8 *bits, u8 bit) 3147 { 3148 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); 3149 char *seg_prt, *wqe; 3150 int i, seg_siz; 3151 3152 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { 3153 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size); 3154 *bad_wr = wr; 3155 return 0; 3156 } 3157 3158 if (!data_size) 3159 return data_size; 3160 3161 *bits |= bit; 3162 3163 seg_prt = NULL; 3164 wqe = NULL; 3165 seg_siz = 0; 3166 3167 /* Copy data inline */ 3168 for (i = 0; i < wr->num_sge; i++) { 3169 u32 len = wr->sg_list[i].length; 3170 void *src = (void *)(uintptr_t)wr->sg_list[i].addr; 3171 3172 while (len > 0) { 3173 u32 cur; 3174 3175 /* New segment required */ 3176 if (!seg_siz) { 3177 wqe = (char *)qed_chain_produce(&qp->sq.pbl); 3178 seg_prt = wqe; 3179 seg_siz = sizeof(struct rdma_sq_common_wqe); 3180 (*wqe_size)++; 3181 } 3182 3183 /* Calculate currently allowed length */ 3184 cur = min_t(u32, len, seg_siz); 3185 memcpy(seg_prt, src, cur); 3186 3187 /* Update segment variables */ 3188 seg_prt += cur; 3189 seg_siz -= cur; 3190 3191 /* Update sge variables */ 3192 src += cur; 3193 len -= cur; 3194 3195 /* Swap fully-completed segments */ 3196 if (!seg_siz) 3197 swap_wqe_data64((u64 *)wqe); 3198 } 3199 } 3200 3201 /* swap last not completed segment */ 3202 if (seg_siz) 3203 swap_wqe_data64((u64 *)wqe); 3204 3205 return data_size; 3206 } 3207 3208 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ 3209 do { \ 3210 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3211 (sge)->length = cpu_to_le32(vlength); \ 3212 (sge)->flags = cpu_to_le32(vflags); \ 3213 } while (0) 3214 3215 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ 3216 do { \ 3217 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \ 3218 (hdr)->num_sges = num_sge; \ 3219 } while (0) 3220 3221 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ 3222 do { \ 3223 DMA_REGPAIR_LE(sge->addr, vaddr); \ 3224 (sge)->length = cpu_to_le32(vlength); \ 3225 (sge)->l_key = cpu_to_le32(vlkey); \ 3226 } while (0) 3227 3228 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size, 3229 const struct ib_send_wr *wr) 3230 { 3231 u32 data_size = 0; 3232 int i; 3233 3234 for (i = 0; i < wr->num_sge; i++) { 3235 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl); 3236 3237 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr); 3238 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); 3239 sge->length = cpu_to_le32(wr->sg_list[i].length); 3240 data_size += wr->sg_list[i].length; 3241 } 3242 3243 if (wqe_size) 3244 *wqe_size += wr->num_sge; 3245 3246 return data_size; 3247 } 3248 3249 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev, 3250 struct qedr_qp *qp, 3251 struct rdma_sq_rdma_wqe_1st *rwqe, 3252 struct rdma_sq_rdma_wqe_2nd *rwqe2, 3253 const struct ib_send_wr *wr, 3254 const struct ib_send_wr **bad_wr) 3255 { 3256 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); 3257 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr); 3258 3259 if (wr->send_flags & IB_SEND_INLINE && 3260 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM || 3261 wr->opcode == IB_WR_RDMA_WRITE)) { 3262 u8 flags = 0; 3263 3264 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); 3265 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, 3266 bad_wr, &rwqe->flags, flags); 3267 } 3268 3269 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr); 3270 } 3271 3272 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev, 3273 struct qedr_qp *qp, 3274 struct rdma_sq_send_wqe_1st *swqe, 3275 struct rdma_sq_send_wqe_2st *swqe2, 3276 const struct ib_send_wr *wr, 3277 const struct ib_send_wr **bad_wr) 3278 { 3279 memset(swqe2, 0, sizeof(*swqe2)); 3280 if (wr->send_flags & IB_SEND_INLINE) { 3281 u8 flags = 0; 3282 3283 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); 3284 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, 3285 bad_wr, &swqe->flags, flags); 3286 } 3287 3288 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr); 3289 } 3290 3291 static int qedr_prepare_reg(struct qedr_qp *qp, 3292 struct rdma_sq_fmr_wqe_1st *fwqe1, 3293 const struct ib_reg_wr *wr) 3294 { 3295 struct qedr_mr *mr = get_qedr_mr(wr->mr); 3296 struct rdma_sq_fmr_wqe_2nd *fwqe2; 3297 3298 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl); 3299 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); 3300 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); 3301 fwqe1->l_key = wr->key; 3302 3303 fwqe2->access_ctrl = 0; 3304 3305 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, 3306 !!(wr->access & IB_ACCESS_REMOTE_READ)); 3307 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, 3308 !!(wr->access & IB_ACCESS_REMOTE_WRITE)); 3309 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, 3310 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); 3311 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); 3312 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, 3313 !!(wr->access & IB_ACCESS_LOCAL_WRITE)); 3314 fwqe2->fmr_ctrl = 0; 3315 3316 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, 3317 ilog2(mr->ibmr.page_size) - 12); 3318 3319 fwqe2->length_hi = 0; 3320 fwqe2->length_lo = mr->ibmr.length; 3321 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); 3322 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); 3323 3324 qp->wqe_wr_id[qp->sq.prod].mr = mr; 3325 3326 return 0; 3327 } 3328 3329 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode) 3330 { 3331 switch (opcode) { 3332 case IB_WR_RDMA_WRITE: 3333 case IB_WR_RDMA_WRITE_WITH_IMM: 3334 return IB_WC_RDMA_WRITE; 3335 case IB_WR_SEND_WITH_IMM: 3336 case IB_WR_SEND: 3337 case IB_WR_SEND_WITH_INV: 3338 return IB_WC_SEND; 3339 case IB_WR_RDMA_READ: 3340 case IB_WR_RDMA_READ_WITH_INV: 3341 return IB_WC_RDMA_READ; 3342 case IB_WR_ATOMIC_CMP_AND_SWP: 3343 return IB_WC_COMP_SWAP; 3344 case IB_WR_ATOMIC_FETCH_AND_ADD: 3345 return IB_WC_FETCH_ADD; 3346 case IB_WR_REG_MR: 3347 return IB_WC_REG_MR; 3348 case IB_WR_LOCAL_INV: 3349 return IB_WC_LOCAL_INV; 3350 default: 3351 return IB_WC_SEND; 3352 } 3353 } 3354 3355 static inline bool qedr_can_post_send(struct qedr_qp *qp, 3356 const struct ib_send_wr *wr) 3357 { 3358 int wq_is_full, err_wr, pbl_is_full; 3359 struct qedr_dev *dev = qp->dev; 3360 3361 /* prevent SQ overflow and/or processing of a bad WR */ 3362 err_wr = wr->num_sge > qp->sq.max_sges; 3363 wq_is_full = qedr_wq_is_full(&qp->sq); 3364 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) < 3365 QEDR_MAX_SQE_ELEMENTS_PER_SQE; 3366 if (wq_is_full || err_wr || pbl_is_full) { 3367 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) { 3368 DP_ERR(dev, 3369 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n", 3370 qp); 3371 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL; 3372 } 3373 3374 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) { 3375 DP_ERR(dev, 3376 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n", 3377 qp); 3378 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR; 3379 } 3380 3381 if (pbl_is_full && 3382 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) { 3383 DP_ERR(dev, 3384 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n", 3385 qp); 3386 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL; 3387 } 3388 return false; 3389 } 3390 return true; 3391 } 3392 3393 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3394 const struct ib_send_wr **bad_wr) 3395 { 3396 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3397 struct qedr_qp *qp = get_qedr_qp(ibqp); 3398 struct rdma_sq_atomic_wqe_1st *awqe1; 3399 struct rdma_sq_atomic_wqe_2nd *awqe2; 3400 struct rdma_sq_atomic_wqe_3rd *awqe3; 3401 struct rdma_sq_send_wqe_2st *swqe2; 3402 struct rdma_sq_local_inv_wqe *iwqe; 3403 struct rdma_sq_rdma_wqe_2nd *rwqe2; 3404 struct rdma_sq_send_wqe_1st *swqe; 3405 struct rdma_sq_rdma_wqe_1st *rwqe; 3406 struct rdma_sq_fmr_wqe_1st *fwqe1; 3407 struct rdma_sq_common_wqe *wqe; 3408 u32 length; 3409 int rc = 0; 3410 bool comp; 3411 3412 if (!qedr_can_post_send(qp, wr)) { 3413 *bad_wr = wr; 3414 return -ENOMEM; 3415 } 3416 3417 wqe = qed_chain_produce(&qp->sq.pbl); 3418 qp->wqe_wr_id[qp->sq.prod].signaled = 3419 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; 3420 3421 wqe->flags = 0; 3422 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, 3423 !!(wr->send_flags & IB_SEND_SOLICITED)); 3424 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled; 3425 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); 3426 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, 3427 !!(wr->send_flags & IB_SEND_FENCE)); 3428 wqe->prev_wqe_size = qp->prev_wqe_size; 3429 3430 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode); 3431 3432 switch (wr->opcode) { 3433 case IB_WR_SEND_WITH_IMM: 3434 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3435 rc = -EINVAL; 3436 *bad_wr = wr; 3437 break; 3438 } 3439 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; 3440 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3441 swqe->wqe_size = 2; 3442 swqe2 = qed_chain_produce(&qp->sq.pbl); 3443 3444 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data)); 3445 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3446 wr, bad_wr); 3447 swqe->length = cpu_to_le32(length); 3448 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3449 qp->prev_wqe_size = swqe->wqe_size; 3450 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3451 break; 3452 case IB_WR_SEND: 3453 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; 3454 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3455 3456 swqe->wqe_size = 2; 3457 swqe2 = qed_chain_produce(&qp->sq.pbl); 3458 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3459 wr, bad_wr); 3460 swqe->length = cpu_to_le32(length); 3461 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3462 qp->prev_wqe_size = swqe->wqe_size; 3463 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3464 break; 3465 case IB_WR_SEND_WITH_INV: 3466 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; 3467 swqe = (struct rdma_sq_send_wqe_1st *)wqe; 3468 swqe2 = qed_chain_produce(&qp->sq.pbl); 3469 swqe->wqe_size = 2; 3470 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); 3471 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2, 3472 wr, bad_wr); 3473 swqe->length = cpu_to_le32(length); 3474 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; 3475 qp->prev_wqe_size = swqe->wqe_size; 3476 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; 3477 break; 3478 3479 case IB_WR_RDMA_WRITE_WITH_IMM: 3480 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) { 3481 rc = -EINVAL; 3482 *bad_wr = wr; 3483 break; 3484 } 3485 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; 3486 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3487 3488 rwqe->wqe_size = 2; 3489 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); 3490 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3491 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3492 wr, bad_wr); 3493 rwqe->length = cpu_to_le32(length); 3494 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3495 qp->prev_wqe_size = rwqe->wqe_size; 3496 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3497 break; 3498 case IB_WR_RDMA_WRITE: 3499 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; 3500 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3501 3502 rwqe->wqe_size = 2; 3503 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3504 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3505 wr, bad_wr); 3506 rwqe->length = cpu_to_le32(length); 3507 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3508 qp->prev_wqe_size = rwqe->wqe_size; 3509 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3510 break; 3511 case IB_WR_RDMA_READ_WITH_INV: 3512 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1); 3513 /* fallthrough -- same is identical to RDMA READ */ 3514 3515 case IB_WR_RDMA_READ: 3516 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; 3517 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; 3518 3519 rwqe->wqe_size = 2; 3520 rwqe2 = qed_chain_produce(&qp->sq.pbl); 3521 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, 3522 wr, bad_wr); 3523 rwqe->length = cpu_to_le32(length); 3524 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; 3525 qp->prev_wqe_size = rwqe->wqe_size; 3526 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; 3527 break; 3528 3529 case IB_WR_ATOMIC_CMP_AND_SWP: 3530 case IB_WR_ATOMIC_FETCH_AND_ADD: 3531 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe; 3532 awqe1->wqe_size = 4; 3533 3534 awqe2 = qed_chain_produce(&qp->sq.pbl); 3535 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr); 3536 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); 3537 3538 awqe3 = qed_chain_produce(&qp->sq.pbl); 3539 3540 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 3541 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; 3542 DMA_REGPAIR_LE(awqe3->swap_data, 3543 atomic_wr(wr)->compare_add); 3544 } else { 3545 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; 3546 DMA_REGPAIR_LE(awqe3->swap_data, 3547 atomic_wr(wr)->swap); 3548 DMA_REGPAIR_LE(awqe3->cmp_data, 3549 atomic_wr(wr)->compare_add); 3550 } 3551 3552 qedr_prepare_sq_sges(qp, NULL, wr); 3553 3554 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size; 3555 qp->prev_wqe_size = awqe1->wqe_size; 3556 break; 3557 3558 case IB_WR_LOCAL_INV: 3559 iwqe = (struct rdma_sq_local_inv_wqe *)wqe; 3560 iwqe->wqe_size = 1; 3561 3562 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; 3563 iwqe->inv_l_key = wr->ex.invalidate_rkey; 3564 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size; 3565 qp->prev_wqe_size = iwqe->wqe_size; 3566 break; 3567 case IB_WR_REG_MR: 3568 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n"); 3569 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; 3570 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; 3571 fwqe1->wqe_size = 2; 3572 3573 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr)); 3574 if (rc) { 3575 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc); 3576 *bad_wr = wr; 3577 break; 3578 } 3579 3580 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; 3581 qp->prev_wqe_size = fwqe1->wqe_size; 3582 break; 3583 default: 3584 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode); 3585 rc = -EINVAL; 3586 *bad_wr = wr; 3587 break; 3588 } 3589 3590 if (*bad_wr) { 3591 u16 value; 3592 3593 /* Restore prod to its position before 3594 * this WR was processed 3595 */ 3596 value = le16_to_cpu(qp->sq.db_data.data.value); 3597 qed_chain_set_prod(&qp->sq.pbl, value, wqe); 3598 3599 /* Restore prev_wqe_size */ 3600 qp->prev_wqe_size = wqe->prev_wqe_size; 3601 rc = -EINVAL; 3602 DP_ERR(dev, "POST SEND FAILED\n"); 3603 } 3604 3605 return rc; 3606 } 3607 3608 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 3609 const struct ib_send_wr **bad_wr) 3610 { 3611 struct qedr_dev *dev = get_qedr_dev(ibqp->device); 3612 struct qedr_qp *qp = get_qedr_qp(ibqp); 3613 unsigned long flags; 3614 int rc = 0; 3615 3616 *bad_wr = NULL; 3617 3618 if (qp->qp_type == IB_QPT_GSI) 3619 return qedr_gsi_post_send(ibqp, wr, bad_wr); 3620 3621 spin_lock_irqsave(&qp->q_lock, flags); 3622 3623 if (rdma_protocol_roce(&dev->ibdev, 1)) { 3624 if ((qp->state != QED_ROCE_QP_STATE_RTS) && 3625 (qp->state != QED_ROCE_QP_STATE_ERR) && 3626 (qp->state != QED_ROCE_QP_STATE_SQD)) { 3627 spin_unlock_irqrestore(&qp->q_lock, flags); 3628 *bad_wr = wr; 3629 DP_DEBUG(dev, QEDR_MSG_CQ, 3630 "QP in wrong state! QP icid=0x%x state %d\n", 3631 qp->icid, qp->state); 3632 return -EINVAL; 3633 } 3634 } 3635 3636 while (wr) { 3637 rc = __qedr_post_send(ibqp, wr, bad_wr); 3638 if (rc) 3639 break; 3640 3641 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; 3642 3643 qedr_inc_sw_prod(&qp->sq); 3644 3645 qp->sq.db_data.data.value++; 3646 3647 wr = wr->next; 3648 } 3649 3650 /* Trigger doorbell 3651 * If there was a failure in the first WR then it will be triggered in 3652 * vane. However this is not harmful (as long as the producer value is 3653 * unchanged). For performance reasons we avoid checking for this 3654 * redundant doorbell. 3655 * 3656 * qp->wqe_wr_id is accessed during qedr_poll_cq, as 3657 * soon as we give the doorbell, we could get a completion 3658 * for this wr, therefore we need to make sure that the 3659 * memory is updated before giving the doorbell. 3660 * During qedr_poll_cq, rmb is called before accessing the 3661 * cqe. This covers for the smp_rmb as well. 3662 */ 3663 smp_wmb(); 3664 writel(qp->sq.db_data.raw, qp->sq.db); 3665 3666 spin_unlock_irqrestore(&qp->q_lock, flags); 3667 3668 return rc; 3669 } 3670 3671 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq) 3672 { 3673 u32 used; 3674 3675 /* Calculate number of elements used based on producer 3676 * count and consumer count and subtract it from max 3677 * work request supported so that we get elements left. 3678 */ 3679 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; 3680 3681 return hw_srq->max_wr - used; 3682 } 3683 3684 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 3685 const struct ib_recv_wr **bad_wr) 3686 { 3687 struct qedr_srq *srq = get_qedr_srq(ibsrq); 3688 struct qedr_srq_hwq_info *hw_srq; 3689 struct qedr_dev *dev = srq->dev; 3690 struct qed_chain *pbl; 3691 unsigned long flags; 3692 int status = 0; 3693 u32 num_sge; 3694 u32 offset; 3695 3696 spin_lock_irqsave(&srq->lock, flags); 3697 3698 hw_srq = &srq->hw_srq; 3699 pbl = &srq->hw_srq.pbl; 3700 while (wr) { 3701 struct rdma_srq_wqe_header *hdr; 3702 int i; 3703 3704 if (!qedr_srq_elem_left(hw_srq) || 3705 wr->num_sge > srq->hw_srq.max_sges) { 3706 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n", 3707 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, 3708 wr->num_sge, srq->hw_srq.max_sges); 3709 status = -ENOMEM; 3710 *bad_wr = wr; 3711 break; 3712 } 3713 3714 hdr = qed_chain_produce(pbl); 3715 num_sge = wr->num_sge; 3716 /* Set number of sge and work request id in header */ 3717 SRQ_HDR_SET(hdr, wr->wr_id, num_sge); 3718 3719 srq->hw_srq.wr_prod_cnt++; 3720 hw_srq->wqe_prod++; 3721 hw_srq->sge_prod++; 3722 3723 DP_DEBUG(dev, QEDR_MSG_SRQ, 3724 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n", 3725 wr->num_sge, hw_srq->wqe_prod, wr->wr_id); 3726 3727 for (i = 0; i < wr->num_sge; i++) { 3728 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl); 3729 3730 /* Set SGE length, lkey and address */ 3731 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, 3732 wr->sg_list[i].length, wr->sg_list[i].lkey); 3733 3734 DP_DEBUG(dev, QEDR_MSG_SRQ, 3735 "[%d]: len %d key %x addr %x:%x\n", 3736 i, srq_sge->length, srq_sge->l_key, 3737 srq_sge->addr.hi, srq_sge->addr.lo); 3738 hw_srq->sge_prod++; 3739 } 3740 3741 /* Flush WQE and SGE information before 3742 * updating producer. 3743 */ 3744 wmb(); 3745 3746 /* SRQ producer is 8 bytes. Need to update SGE producer index 3747 * in first 4 bytes and need to update WQE producer in 3748 * next 4 bytes. 3749 */ 3750 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod; 3751 offset = offsetof(struct rdma_srq_producers, wqe_prod); 3752 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = 3753 hw_srq->wqe_prod; 3754 3755 /* Flush producer after updating it. */ 3756 wmb(); 3757 wr = wr->next; 3758 } 3759 3760 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n", 3761 qed_chain_get_elem_left(pbl)); 3762 spin_unlock_irqrestore(&srq->lock, flags); 3763 3764 return status; 3765 } 3766 3767 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 3768 const struct ib_recv_wr **bad_wr) 3769 { 3770 struct qedr_qp *qp = get_qedr_qp(ibqp); 3771 struct qedr_dev *dev = qp->dev; 3772 unsigned long flags; 3773 int status = 0; 3774 3775 if (qp->qp_type == IB_QPT_GSI) 3776 return qedr_gsi_post_recv(ibqp, wr, bad_wr); 3777 3778 spin_lock_irqsave(&qp->q_lock, flags); 3779 3780 if (qp->state == QED_ROCE_QP_STATE_RESET) { 3781 spin_unlock_irqrestore(&qp->q_lock, flags); 3782 *bad_wr = wr; 3783 return -EINVAL; 3784 } 3785 3786 while (wr) { 3787 int i; 3788 3789 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) < 3790 QEDR_MAX_RQE_ELEMENTS_PER_RQE || 3791 wr->num_sge > qp->rq.max_sges) { 3792 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n", 3793 qed_chain_get_elem_left_u32(&qp->rq.pbl), 3794 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge, 3795 qp->rq.max_sges); 3796 status = -ENOMEM; 3797 *bad_wr = wr; 3798 break; 3799 } 3800 for (i = 0; i < wr->num_sge; i++) { 3801 u32 flags = 0; 3802 struct rdma_rq_sge *rqe = 3803 qed_chain_produce(&qp->rq.pbl); 3804 3805 /* First one must include the number 3806 * of SGE in the list 3807 */ 3808 if (!i) 3809 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 3810 wr->num_sge); 3811 3812 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 3813 wr->sg_list[i].lkey); 3814 3815 RQ_SGE_SET(rqe, wr->sg_list[i].addr, 3816 wr->sg_list[i].length, flags); 3817 } 3818 3819 /* Special case of no sges. FW requires between 1-4 sges... 3820 * in this case we need to post 1 sge with length zero. this is 3821 * because rdma write with immediate consumes an RQ. 3822 */ 3823 if (!wr->num_sge) { 3824 u32 flags = 0; 3825 struct rdma_rq_sge *rqe = 3826 qed_chain_produce(&qp->rq.pbl); 3827 3828 /* First one must include the number 3829 * of SGE in the list 3830 */ 3831 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0); 3832 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); 3833 3834 RQ_SGE_SET(rqe, 0, 0, flags); 3835 i = 1; 3836 } 3837 3838 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; 3839 qp->rqe_wr_id[qp->rq.prod].wqe_size = i; 3840 3841 qedr_inc_sw_prod(&qp->rq); 3842 3843 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as 3844 * soon as we give the doorbell, we could get a completion 3845 * for this wr, therefore we need to make sure that the 3846 * memory is update before giving the doorbell. 3847 * During qedr_poll_cq, rmb is called before accessing the 3848 * cqe. This covers for the smp_rmb as well. 3849 */ 3850 smp_wmb(); 3851 3852 qp->rq.db_data.data.value++; 3853 3854 writel(qp->rq.db_data.raw, qp->rq.db); 3855 3856 if (rdma_protocol_iwarp(&dev->ibdev, 1)) { 3857 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); 3858 } 3859 3860 wr = wr->next; 3861 } 3862 3863 spin_unlock_irqrestore(&qp->q_lock, flags); 3864 3865 return status; 3866 } 3867 3868 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe) 3869 { 3870 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3871 3872 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) == 3873 cq->pbl_toggle; 3874 } 3875 3876 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe) 3877 { 3878 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3879 struct qedr_qp *qp; 3880 3881 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi, 3882 resp_cqe->qp_handle.lo, 3883 u64); 3884 return qp; 3885 } 3886 3887 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe) 3888 { 3889 struct rdma_cqe_requester *resp_cqe = &cqe->req; 3890 3891 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); 3892 } 3893 3894 /* Return latest CQE (needs processing) */ 3895 static union rdma_cqe *get_cqe(struct qedr_cq *cq) 3896 { 3897 return cq->latest_cqe; 3898 } 3899 3900 /* In fmr we need to increase the number of fmr completed counter for the fmr 3901 * algorithm determining whether we can free a pbl or not. 3902 * we need to perform this whether the work request was signaled or not. for 3903 * this purpose we call this function from the condition that checks if a wr 3904 * should be skipped, to make sure we don't miss it ( possibly this fmr 3905 * operation was not signalted) 3906 */ 3907 static inline void qedr_chk_if_fmr(struct qedr_qp *qp) 3908 { 3909 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) 3910 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3911 } 3912 3913 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp, 3914 struct qedr_cq *cq, int num_entries, 3915 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, 3916 int force) 3917 { 3918 u16 cnt = 0; 3919 3920 while (num_entries && qp->sq.wqe_cons != hw_cons) { 3921 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { 3922 qedr_chk_if_fmr(qp); 3923 /* skip WC */ 3924 goto next_cqe; 3925 } 3926 3927 /* fill WC */ 3928 wc->status = status; 3929 wc->vendor_err = 0; 3930 wc->wc_flags = 0; 3931 wc->src_qp = qp->id; 3932 wc->qp = &qp->ibqp; 3933 3934 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; 3935 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; 3936 3937 switch (wc->opcode) { 3938 case IB_WC_RDMA_WRITE: 3939 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3940 break; 3941 case IB_WC_COMP_SWAP: 3942 case IB_WC_FETCH_ADD: 3943 wc->byte_len = 8; 3944 break; 3945 case IB_WC_REG_MR: 3946 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; 3947 break; 3948 case IB_WC_RDMA_READ: 3949 case IB_WC_SEND: 3950 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; 3951 break; 3952 default: 3953 break; 3954 } 3955 3956 num_entries--; 3957 wc++; 3958 cnt++; 3959 next_cqe: 3960 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) 3961 qed_chain_consume(&qp->sq.pbl); 3962 qedr_inc_sw_cons(&qp->sq); 3963 } 3964 3965 return cnt; 3966 } 3967 3968 static int qedr_poll_cq_req(struct qedr_dev *dev, 3969 struct qedr_qp *qp, struct qedr_cq *cq, 3970 int num_entries, struct ib_wc *wc, 3971 struct rdma_cqe_requester *req) 3972 { 3973 int cnt = 0; 3974 3975 switch (req->status) { 3976 case RDMA_CQE_REQ_STS_OK: 3977 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3978 IB_WC_SUCCESS, 0); 3979 break; 3980 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: 3981 if (qp->state != QED_ROCE_QP_STATE_ERR) 3982 DP_DEBUG(dev, QEDR_MSG_CQ, 3983 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n", 3984 cq->icid, qp->icid); 3985 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, 3986 IB_WC_WR_FLUSH_ERR, 1); 3987 break; 3988 default: 3989 /* process all WQE before the cosumer */ 3990 qp->state = QED_ROCE_QP_STATE_ERR; 3991 cnt = process_req(dev, qp, cq, num_entries, wc, 3992 req->sq_cons - 1, IB_WC_SUCCESS, 0); 3993 wc += cnt; 3994 /* if we have extra WC fill it with actual error info */ 3995 if (cnt < num_entries) { 3996 enum ib_wc_status wc_status; 3997 3998 switch (req->status) { 3999 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: 4000 DP_ERR(dev, 4001 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4002 cq->icid, qp->icid); 4003 wc_status = IB_WC_BAD_RESP_ERR; 4004 break; 4005 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: 4006 DP_ERR(dev, 4007 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4008 cq->icid, qp->icid); 4009 wc_status = IB_WC_LOC_LEN_ERR; 4010 break; 4011 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: 4012 DP_ERR(dev, 4013 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4014 cq->icid, qp->icid); 4015 wc_status = IB_WC_LOC_QP_OP_ERR; 4016 break; 4017 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: 4018 DP_ERR(dev, 4019 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4020 cq->icid, qp->icid); 4021 wc_status = IB_WC_LOC_PROT_ERR; 4022 break; 4023 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: 4024 DP_ERR(dev, 4025 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4026 cq->icid, qp->icid); 4027 wc_status = IB_WC_MW_BIND_ERR; 4028 break; 4029 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: 4030 DP_ERR(dev, 4031 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4032 cq->icid, qp->icid); 4033 wc_status = IB_WC_REM_INV_REQ_ERR; 4034 break; 4035 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: 4036 DP_ERR(dev, 4037 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4038 cq->icid, qp->icid); 4039 wc_status = IB_WC_REM_ACCESS_ERR; 4040 break; 4041 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: 4042 DP_ERR(dev, 4043 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4044 cq->icid, qp->icid); 4045 wc_status = IB_WC_REM_OP_ERR; 4046 break; 4047 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: 4048 DP_ERR(dev, 4049 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4050 cq->icid, qp->icid); 4051 wc_status = IB_WC_RNR_RETRY_EXC_ERR; 4052 break; 4053 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: 4054 DP_ERR(dev, 4055 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4056 cq->icid, qp->icid); 4057 wc_status = IB_WC_RETRY_EXC_ERR; 4058 break; 4059 default: 4060 DP_ERR(dev, 4061 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n", 4062 cq->icid, qp->icid); 4063 wc_status = IB_WC_GENERAL_ERR; 4064 } 4065 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, 4066 wc_status, 1); 4067 } 4068 } 4069 4070 return cnt; 4071 } 4072 4073 static inline int qedr_cqe_resp_status_to_ib(u8 status) 4074 { 4075 switch (status) { 4076 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: 4077 return IB_WC_LOC_ACCESS_ERR; 4078 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: 4079 return IB_WC_LOC_LEN_ERR; 4080 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: 4081 return IB_WC_LOC_QP_OP_ERR; 4082 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: 4083 return IB_WC_LOC_PROT_ERR; 4084 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: 4085 return IB_WC_MW_BIND_ERR; 4086 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: 4087 return IB_WC_REM_INV_RD_REQ_ERR; 4088 case RDMA_CQE_RESP_STS_OK: 4089 return IB_WC_SUCCESS; 4090 default: 4091 return IB_WC_GENERAL_ERR; 4092 } 4093 } 4094 4095 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp, 4096 struct ib_wc *wc) 4097 { 4098 wc->status = IB_WC_SUCCESS; 4099 wc->byte_len = le32_to_cpu(resp->length); 4100 4101 if (resp->flags & QEDR_RESP_IMM) { 4102 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key)); 4103 wc->wc_flags |= IB_WC_WITH_IMM; 4104 4105 if (resp->flags & QEDR_RESP_RDMA) 4106 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; 4107 4108 if (resp->flags & QEDR_RESP_INV) 4109 return -EINVAL; 4110 4111 } else if (resp->flags & QEDR_RESP_INV) { 4112 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); 4113 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 4114 4115 if (resp->flags & QEDR_RESP_RDMA) 4116 return -EINVAL; 4117 4118 } else if (resp->flags & QEDR_RESP_RDMA) { 4119 return -EINVAL; 4120 } 4121 4122 return 0; 4123 } 4124 4125 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4126 struct qedr_cq *cq, struct ib_wc *wc, 4127 struct rdma_cqe_responder *resp, u64 wr_id) 4128 { 4129 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */ 4130 wc->opcode = IB_WC_RECV; 4131 wc->wc_flags = 0; 4132 4133 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) { 4134 if (qedr_set_ok_cqe_resp_wc(resp, wc)) 4135 DP_ERR(dev, 4136 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n", 4137 cq, cq->icid, resp->flags); 4138 4139 } else { 4140 wc->status = qedr_cqe_resp_status_to_ib(resp->status); 4141 if (wc->status == IB_WC_GENERAL_ERR) 4142 DP_ERR(dev, 4143 "CQ %p (icid=%d) contains an invalid CQE status %d\n", 4144 cq, cq->icid, resp->status); 4145 } 4146 4147 /* Fill the rest of the WC */ 4148 wc->vendor_err = 0; 4149 wc->src_qp = qp->id; 4150 wc->qp = &qp->ibqp; 4151 wc->wr_id = wr_id; 4152 } 4153 4154 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4155 struct qedr_cq *cq, struct ib_wc *wc, 4156 struct rdma_cqe_responder *resp) 4157 { 4158 struct qedr_srq *srq = qp->srq; 4159 u64 wr_id; 4160 4161 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi), 4162 le32_to_cpu(resp->srq_wr_id.lo), u64); 4163 4164 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4165 wc->status = IB_WC_WR_FLUSH_ERR; 4166 wc->vendor_err = 0; 4167 wc->wr_id = wr_id; 4168 wc->byte_len = 0; 4169 wc->src_qp = qp->id; 4170 wc->qp = &qp->ibqp; 4171 wc->wr_id = wr_id; 4172 } else { 4173 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4174 } 4175 srq->hw_srq.wr_cons_cnt++; 4176 4177 return 1; 4178 } 4179 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp, 4180 struct qedr_cq *cq, struct ib_wc *wc, 4181 struct rdma_cqe_responder *resp) 4182 { 4183 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4184 4185 __process_resp_one(dev, qp, cq, wc, resp, wr_id); 4186 4187 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4188 qed_chain_consume(&qp->rq.pbl); 4189 qedr_inc_sw_cons(&qp->rq); 4190 4191 return 1; 4192 } 4193 4194 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq, 4195 int num_entries, struct ib_wc *wc, u16 hw_cons) 4196 { 4197 u16 cnt = 0; 4198 4199 while (num_entries && qp->rq.wqe_cons != hw_cons) { 4200 /* fill WC */ 4201 wc->status = IB_WC_WR_FLUSH_ERR; 4202 wc->vendor_err = 0; 4203 wc->wc_flags = 0; 4204 wc->src_qp = qp->id; 4205 wc->byte_len = 0; 4206 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; 4207 wc->qp = &qp->ibqp; 4208 num_entries--; 4209 wc++; 4210 cnt++; 4211 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) 4212 qed_chain_consume(&qp->rq.pbl); 4213 qedr_inc_sw_cons(&qp->rq); 4214 } 4215 4216 return cnt; 4217 } 4218 4219 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4220 struct rdma_cqe_responder *resp, int *update) 4221 { 4222 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) { 4223 consume_cqe(cq); 4224 *update |= 1; 4225 } 4226 } 4227 4228 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp, 4229 struct qedr_cq *cq, int num_entries, 4230 struct ib_wc *wc, 4231 struct rdma_cqe_responder *resp) 4232 { 4233 int cnt; 4234 4235 cnt = process_resp_one_srq(dev, qp, cq, wc, resp); 4236 consume_cqe(cq); 4237 4238 return cnt; 4239 } 4240 4241 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp, 4242 struct qedr_cq *cq, int num_entries, 4243 struct ib_wc *wc, struct rdma_cqe_responder *resp, 4244 int *update) 4245 { 4246 int cnt; 4247 4248 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { 4249 cnt = process_resp_flush(qp, cq, num_entries, wc, 4250 resp->rq_cons_or_srq_id); 4251 try_consume_resp_cqe(cq, qp, resp, update); 4252 } else { 4253 cnt = process_resp_one(dev, qp, cq, wc, resp); 4254 consume_cqe(cq); 4255 *update |= 1; 4256 } 4257 4258 return cnt; 4259 } 4260 4261 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp, 4262 struct rdma_cqe_requester *req, int *update) 4263 { 4264 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { 4265 consume_cqe(cq); 4266 *update |= 1; 4267 } 4268 } 4269 4270 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 4271 { 4272 struct qedr_dev *dev = get_qedr_dev(ibcq->device); 4273 struct qedr_cq *cq = get_qedr_cq(ibcq); 4274 union rdma_cqe *cqe; 4275 u32 old_cons, new_cons; 4276 unsigned long flags; 4277 int update = 0; 4278 int done = 0; 4279 4280 if (cq->destroyed) { 4281 DP_ERR(dev, 4282 "warning: poll was invoked after destroy for cq %p (icid=%d)\n", 4283 cq, cq->icid); 4284 return 0; 4285 } 4286 4287 if (cq->cq_type == QEDR_CQ_TYPE_GSI) 4288 return qedr_gsi_poll_cq(ibcq, num_entries, wc); 4289 4290 spin_lock_irqsave(&cq->cq_lock, flags); 4291 cqe = cq->latest_cqe; 4292 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4293 while (num_entries && is_valid_cqe(cq, cqe)) { 4294 struct qedr_qp *qp; 4295 int cnt = 0; 4296 4297 /* prevent speculative reads of any field of CQE */ 4298 rmb(); 4299 4300 qp = cqe_get_qp(cqe); 4301 if (!qp) { 4302 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe); 4303 break; 4304 } 4305 4306 wc->qp = &qp->ibqp; 4307 4308 switch (cqe_get_type(cqe)) { 4309 case RDMA_CQE_TYPE_REQUESTER: 4310 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc, 4311 &cqe->req); 4312 try_consume_req_cqe(cq, qp, &cqe->req, &update); 4313 break; 4314 case RDMA_CQE_TYPE_RESPONDER_RQ: 4315 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc, 4316 &cqe->resp, &update); 4317 break; 4318 case RDMA_CQE_TYPE_RESPONDER_SRQ: 4319 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries, 4320 wc, &cqe->resp); 4321 update = 1; 4322 break; 4323 case RDMA_CQE_TYPE_INVALID: 4324 default: 4325 DP_ERR(dev, "Error: invalid CQE type = %d\n", 4326 cqe_get_type(cqe)); 4327 } 4328 num_entries -= cnt; 4329 wc += cnt; 4330 done += cnt; 4331 4332 cqe = get_cqe(cq); 4333 } 4334 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl); 4335 4336 cq->cq_cons += new_cons - old_cons; 4337 4338 if (update) 4339 /* doorbell notifies abount latest VALID entry, 4340 * but chain already point to the next INVALID one 4341 */ 4342 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags); 4343 4344 spin_unlock_irqrestore(&cq->cq_lock, flags); 4345 return done; 4346 } 4347 4348 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags, 4349 u8 port_num, const struct ib_wc *in_wc, 4350 const struct ib_grh *in_grh, const struct ib_mad *in, 4351 struct ib_mad *out_mad, size_t *out_mad_size, 4352 u16 *out_mad_pkey_index) 4353 { 4354 return IB_MAD_RESULT_SUCCESS; 4355 } 4356