1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 20 if (udata->inlen || udata->outlen) 21 return -EINVAL; 22 23 memset(props, 0, sizeof(*props)); 24 ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); 25 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 26 irdma_fw_minor_ver(&rf->sc_dev); 27 props->device_cap_flags = iwdev->device_cap_flags; 28 props->vendor_id = pcidev->vendor; 29 props->vendor_part_id = pcidev->device; 30 31 props->hw_ver = rf->pcidev->revision; 32 props->page_size_cap = SZ_4K | SZ_2M | SZ_1G; 33 props->max_mr_size = hw_attrs->max_mr_size; 34 props->max_qp = rf->max_qp - rf->used_qps; 35 props->max_qp_wr = hw_attrs->max_qp_wr; 36 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 37 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 38 props->max_cq = rf->max_cq - rf->used_cqs; 39 props->max_cqe = rf->max_cqe; 40 props->max_mr = rf->max_mr - rf->used_mrs; 41 props->max_mw = props->max_mr; 42 props->max_pd = rf->max_pd - rf->used_pds; 43 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 44 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 45 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 46 if (rdma_protocol_roce(ibdev, 1)) 47 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 48 props->max_ah = rf->max_ah; 49 props->max_mcast_grp = rf->max_mcg; 50 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 51 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 52 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 53 #define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff 54 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) 55 props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK; 56 57 return 0; 58 } 59 60 /** 61 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed 62 * @link_speed: netdev phy link speed 63 * @active_speed: IB port speed 64 * @active_width: IB port width 65 */ 66 static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed, 67 u8 *active_width) 68 { 69 if (link_speed <= SPEED_1000) { 70 *active_width = IB_WIDTH_1X; 71 *active_speed = IB_SPEED_SDR; 72 } else if (link_speed <= SPEED_10000) { 73 *active_width = IB_WIDTH_1X; 74 *active_speed = IB_SPEED_FDR10; 75 } else if (link_speed <= SPEED_20000) { 76 *active_width = IB_WIDTH_4X; 77 *active_speed = IB_SPEED_DDR; 78 } else if (link_speed <= SPEED_25000) { 79 *active_width = IB_WIDTH_1X; 80 *active_speed = IB_SPEED_EDR; 81 } else if (link_speed <= SPEED_40000) { 82 *active_width = IB_WIDTH_4X; 83 *active_speed = IB_SPEED_FDR10; 84 } else { 85 *active_width = IB_WIDTH_4X; 86 *active_speed = IB_SPEED_EDR; 87 } 88 } 89 90 /** 91 * irdma_query_port - get port attributes 92 * @ibdev: device pointer from stack 93 * @port: port number for query 94 * @props: returning device attributes 95 */ 96 static int irdma_query_port(struct ib_device *ibdev, u32 port, 97 struct ib_port_attr *props) 98 { 99 struct irdma_device *iwdev = to_iwdev(ibdev); 100 struct net_device *netdev = iwdev->netdev; 101 102 /* no need to zero out pros here. done by caller */ 103 104 props->max_mtu = IB_MTU_4096; 105 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 106 props->lid = 1; 107 props->lmc = 0; 108 props->sm_lid = 0; 109 props->sm_sl = 0; 110 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 111 props->state = IB_PORT_ACTIVE; 112 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 113 } else { 114 props->state = IB_PORT_DOWN; 115 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 116 } 117 irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, 118 &props->active_width); 119 120 if (rdma_protocol_roce(ibdev, 1)) { 121 props->gid_tbl_len = 32; 122 props->ip_gids = true; 123 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 124 } else { 125 props->gid_tbl_len = 1; 126 } 127 props->qkey_viol_cntr = 0; 128 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 129 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 130 131 return 0; 132 } 133 134 /** 135 * irdma_disassociate_ucontext - Disassociate user context 136 * @context: ib user context 137 */ 138 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 139 { 140 } 141 142 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 143 struct vm_area_struct *vma) 144 { 145 u64 pfn; 146 147 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 148 return -EINVAL; 149 150 vma->vm_private_data = ucontext; 151 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 152 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 153 154 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 155 pgprot_noncached(vma->vm_page_prot), NULL); 156 } 157 158 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 159 { 160 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 161 162 kfree(entry); 163 } 164 165 static struct rdma_user_mmap_entry* 166 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 167 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 168 { 169 struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 170 int ret; 171 172 if (!entry) 173 return NULL; 174 175 entry->bar_offset = bar_offset; 176 entry->mmap_flag = mmap_flag; 177 178 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 179 &entry->rdma_entry, PAGE_SIZE); 180 if (ret) { 181 kfree(entry); 182 return NULL; 183 } 184 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 185 186 return &entry->rdma_entry; 187 } 188 189 /** 190 * irdma_mmap - user memory map 191 * @context: context created during alloc 192 * @vma: kernel info for user memory map 193 */ 194 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 195 { 196 struct rdma_user_mmap_entry *rdma_entry; 197 struct irdma_user_mmap_entry *entry; 198 struct irdma_ucontext *ucontext; 199 u64 pfn; 200 int ret; 201 202 ucontext = to_ucontext(context); 203 204 /* Legacy support for libi40iw with hard-coded mmap key */ 205 if (ucontext->legacy_mode) 206 return irdma_mmap_legacy(ucontext, vma); 207 208 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 209 if (!rdma_entry) { 210 ibdev_dbg(&ucontext->iwdev->ibdev, 211 "VERBS: pgoff[0x%lx] does not have valid entry\n", 212 vma->vm_pgoff); 213 return -EINVAL; 214 } 215 216 entry = to_irdma_mmap_entry(rdma_entry); 217 ibdev_dbg(&ucontext->iwdev->ibdev, 218 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 219 entry->bar_offset, entry->mmap_flag); 220 221 pfn = (entry->bar_offset + 222 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 223 224 switch (entry->mmap_flag) { 225 case IRDMA_MMAP_IO_NC: 226 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 227 pgprot_noncached(vma->vm_page_prot), 228 rdma_entry); 229 break; 230 case IRDMA_MMAP_IO_WC: 231 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 232 pgprot_writecombine(vma->vm_page_prot), 233 rdma_entry); 234 break; 235 default: 236 ret = -EINVAL; 237 } 238 239 if (ret) 240 ibdev_dbg(&ucontext->iwdev->ibdev, 241 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 242 entry->bar_offset, entry->mmap_flag, ret); 243 rdma_user_mmap_entry_put(rdma_entry); 244 245 return ret; 246 } 247 248 /** 249 * irdma_alloc_push_page - allocate a push page for qp 250 * @iwqp: qp pointer 251 */ 252 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 253 { 254 struct irdma_cqp_request *cqp_request; 255 struct cqp_cmds_info *cqp_info; 256 struct irdma_device *iwdev = iwqp->iwdev; 257 struct irdma_sc_qp *qp = &iwqp->sc_qp; 258 enum irdma_status_code status; 259 260 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 261 if (!cqp_request) 262 return; 263 264 cqp_info = &cqp_request->info; 265 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 266 cqp_info->post_sq = 1; 267 cqp_info->in.u.manage_push_page.info.push_idx = 0; 268 cqp_info->in.u.manage_push_page.info.qs_handle = 269 qp->vsi->qos[qp->user_pri].qs_handle; 270 cqp_info->in.u.manage_push_page.info.free_page = 0; 271 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 272 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 273 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 274 275 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 276 if (!status && cqp_request->compl_info.op_ret_val < 277 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 278 qp->push_idx = cqp_request->compl_info.op_ret_val; 279 qp->push_offset = 0; 280 } 281 282 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 283 } 284 285 /** 286 * irdma_alloc_ucontext - Allocate the user context data structure 287 * @uctx: uverbs context pointer 288 * @udata: user data 289 * 290 * This keeps track of all objects associated with a particular 291 * user-mode client. 292 */ 293 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 294 struct ib_udata *udata) 295 { 296 struct ib_device *ibdev = uctx->device; 297 struct irdma_device *iwdev = to_iwdev(ibdev); 298 struct irdma_alloc_ucontext_req req; 299 struct irdma_alloc_ucontext_resp uresp = {}; 300 struct irdma_ucontext *ucontext = to_ucontext(uctx); 301 struct irdma_uk_attrs *uk_attrs; 302 303 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 304 return -EINVAL; 305 306 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 307 goto ver_error; 308 309 ucontext->iwdev = iwdev; 310 ucontext->abi_ver = req.userspace_ver; 311 312 uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 313 /* GEN_1 legacy support with libi40iw */ 314 if (udata->outlen < sizeof(uresp)) { 315 if (uk_attrs->hw_rev != IRDMA_GEN_1) 316 return -EOPNOTSUPP; 317 318 ucontext->legacy_mode = true; 319 uresp.max_qps = iwdev->rf->max_qp; 320 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 321 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 322 uresp.kernel_ver = req.userspace_ver; 323 if (ib_copy_to_udata(udata, &uresp, 324 min(sizeof(uresp), udata->outlen))) 325 return -EFAULT; 326 } else { 327 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 328 329 ucontext->db_mmap_entry = 330 irdma_user_mmap_entry_insert(ucontext, bar_off, 331 IRDMA_MMAP_IO_NC, 332 &uresp.db_mmap_key); 333 if (!ucontext->db_mmap_entry) 334 return -ENOMEM; 335 336 uresp.kernel_ver = IRDMA_ABI_VER; 337 uresp.feature_flags = uk_attrs->feature_flags; 338 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 339 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 340 uresp.max_hw_inline = uk_attrs->max_hw_inline; 341 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 342 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 343 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 344 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 345 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 346 uresp.hw_rev = uk_attrs->hw_rev; 347 if (ib_copy_to_udata(udata, &uresp, 348 min(sizeof(uresp), udata->outlen))) { 349 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 350 return -EFAULT; 351 } 352 } 353 354 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 355 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 356 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 357 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 358 359 return 0; 360 361 ver_error: 362 ibdev_err(&iwdev->ibdev, 363 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 364 req.userspace_ver, IRDMA_ABI_VER); 365 return -EINVAL; 366 } 367 368 /** 369 * irdma_dealloc_ucontext - deallocate the user context data structure 370 * @context: user context created during alloc 371 */ 372 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 373 { 374 struct irdma_ucontext *ucontext = to_ucontext(context); 375 376 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 377 } 378 379 /** 380 * irdma_alloc_pd - allocate protection domain 381 * @pd: PD pointer 382 * @udata: user data 383 */ 384 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 385 { 386 struct irdma_pd *iwpd = to_iwpd(pd); 387 struct irdma_device *iwdev = to_iwdev(pd->device); 388 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 389 struct irdma_pci_f *rf = iwdev->rf; 390 struct irdma_alloc_pd_resp uresp = {}; 391 struct irdma_sc_pd *sc_pd; 392 u32 pd_id = 0; 393 int err; 394 395 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 396 &rf->next_pd); 397 if (err) 398 return err; 399 400 sc_pd = &iwpd->sc_pd; 401 if (udata) { 402 struct irdma_ucontext *ucontext = 403 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 404 ibucontext); 405 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 406 uresp.pd_id = pd_id; 407 if (ib_copy_to_udata(udata, &uresp, 408 min(sizeof(uresp), udata->outlen))) { 409 err = -EFAULT; 410 goto error; 411 } 412 } else { 413 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 414 } 415 416 return 0; 417 error: 418 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 419 420 return err; 421 } 422 423 /** 424 * irdma_dealloc_pd - deallocate pd 425 * @ibpd: ptr of pd to be deallocated 426 * @udata: user data 427 */ 428 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 429 { 430 struct irdma_pd *iwpd = to_iwpd(ibpd); 431 struct irdma_device *iwdev = to_iwdev(ibpd->device); 432 433 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 434 435 return 0; 436 } 437 438 /** 439 * irdma_get_pbl - Retrieve pbl from a list given a virtual 440 * address 441 * @va: user virtual address 442 * @pbl_list: pbl list to search in (QP's or CQ's) 443 */ 444 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 445 struct list_head *pbl_list) 446 { 447 struct irdma_pbl *iwpbl; 448 449 list_for_each_entry (iwpbl, pbl_list, list) { 450 if (iwpbl->user_base == va) { 451 list_del(&iwpbl->list); 452 iwpbl->on_list = false; 453 return iwpbl; 454 } 455 } 456 457 return NULL; 458 } 459 460 /** 461 * irdma_clean_cqes - clean cq entries for qp 462 * @iwqp: qp ptr (user or kernel) 463 * @iwcq: cq ptr 464 */ 465 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 466 { 467 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 468 unsigned long flags; 469 470 spin_lock_irqsave(&iwcq->lock, flags); 471 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 472 spin_unlock_irqrestore(&iwcq->lock, flags); 473 } 474 475 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 476 { 477 if (iwqp->push_db_mmap_entry) { 478 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 479 iwqp->push_db_mmap_entry = NULL; 480 } 481 if (iwqp->push_wqe_mmap_entry) { 482 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 483 iwqp->push_wqe_mmap_entry = NULL; 484 } 485 } 486 487 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 488 struct irdma_qp *iwqp, 489 u64 *push_wqe_mmap_key, 490 u64 *push_db_mmap_key) 491 { 492 struct irdma_device *iwdev = ucontext->iwdev; 493 u64 rsvd, bar_off; 494 495 rsvd = IRDMA_PF_BAR_RSVD; 496 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 497 /* skip over db page */ 498 bar_off += IRDMA_HW_PAGE_SIZE; 499 /* push wqe page */ 500 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 501 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 502 bar_off, IRDMA_MMAP_IO_WC, 503 push_wqe_mmap_key); 504 if (!iwqp->push_wqe_mmap_entry) 505 return -ENOMEM; 506 507 /* push doorbell page */ 508 bar_off += IRDMA_HW_PAGE_SIZE; 509 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 510 bar_off, IRDMA_MMAP_IO_NC, 511 push_db_mmap_key); 512 if (!iwqp->push_db_mmap_entry) { 513 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 514 return -ENOMEM; 515 } 516 517 return 0; 518 } 519 520 /** 521 * irdma_destroy_qp - destroy qp 522 * @ibqp: qp's ib pointer also to get to device's qp address 523 * @udata: user data 524 */ 525 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 526 { 527 struct irdma_qp *iwqp = to_iwqp(ibqp); 528 struct irdma_device *iwdev = iwqp->iwdev; 529 530 iwqp->sc_qp.qp_uk.destroy_pending = true; 531 532 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 533 irdma_modify_qp_to_err(&iwqp->sc_qp); 534 535 irdma_qp_rem_ref(&iwqp->ibqp); 536 wait_for_completion(&iwqp->free_qp); 537 irdma_free_lsmm_rsrc(iwqp); 538 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 539 540 if (!iwqp->user_mode) { 541 if (iwqp->iwscq) { 542 irdma_clean_cqes(iwqp, iwqp->iwscq); 543 if (iwqp->iwrcq != iwqp->iwscq) 544 irdma_clean_cqes(iwqp, iwqp->iwrcq); 545 } 546 } 547 irdma_remove_push_mmap_entries(iwqp); 548 irdma_free_qp_rsrc(iwqp); 549 550 return 0; 551 } 552 553 /** 554 * irdma_setup_virt_qp - setup for allocation of virtual qp 555 * @iwdev: irdma device 556 * @iwqp: qp ptr 557 * @init_info: initialize info to return 558 */ 559 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 560 struct irdma_qp *iwqp, 561 struct irdma_qp_init_info *init_info) 562 { 563 struct irdma_pbl *iwpbl = iwqp->iwpbl; 564 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 565 566 iwqp->page = qpmr->sq_page; 567 init_info->shadow_area_pa = qpmr->shadow; 568 if (iwpbl->pbl_allocated) { 569 init_info->virtual_map = true; 570 init_info->sq_pa = qpmr->sq_pbl.idx; 571 init_info->rq_pa = qpmr->rq_pbl.idx; 572 } else { 573 init_info->sq_pa = qpmr->sq_pbl.addr; 574 init_info->rq_pa = qpmr->rq_pbl.addr; 575 } 576 } 577 578 /** 579 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 580 * @iwdev: iwarp device 581 * @iwqp: qp ptr (user or kernel) 582 * @info: initialize info to return 583 * @init_attr: Initial QP create attributes 584 */ 585 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 586 struct irdma_qp *iwqp, 587 struct irdma_qp_init_info *info, 588 struct ib_qp_init_attr *init_attr) 589 { 590 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 591 u32 sqdepth, rqdepth; 592 u8 sqshift, rqshift; 593 u32 size; 594 enum irdma_status_code status; 595 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 596 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 597 598 irdma_get_wqe_shift(uk_attrs, 599 uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 : 600 ukinfo->max_sq_frag_cnt, 601 ukinfo->max_inline_data, &sqshift); 602 status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift, 603 &sqdepth); 604 if (status) 605 return -ENOMEM; 606 607 if (uk_attrs->hw_rev == IRDMA_GEN_1) 608 rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; 609 else 610 irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0, 611 &rqshift); 612 613 status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift, 614 &rqdepth); 615 if (status) 616 return -ENOMEM; 617 618 iwqp->kqp.sq_wrid_mem = 619 kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); 620 if (!iwqp->kqp.sq_wrid_mem) 621 return -ENOMEM; 622 623 iwqp->kqp.rq_wrid_mem = 624 kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); 625 if (!iwqp->kqp.rq_wrid_mem) { 626 kfree(iwqp->kqp.sq_wrid_mem); 627 iwqp->kqp.sq_wrid_mem = NULL; 628 return -ENOMEM; 629 } 630 631 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 632 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 633 634 size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE; 635 size += (IRDMA_SHADOW_AREA_SIZE << 3); 636 637 mem->size = ALIGN(size, 256); 638 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 639 &mem->pa, GFP_KERNEL); 640 if (!mem->va) { 641 kfree(iwqp->kqp.sq_wrid_mem); 642 iwqp->kqp.sq_wrid_mem = NULL; 643 kfree(iwqp->kqp.rq_wrid_mem); 644 iwqp->kqp.rq_wrid_mem = NULL; 645 return -ENOMEM; 646 } 647 648 ukinfo->sq = mem->va; 649 info->sq_pa = mem->pa; 650 ukinfo->rq = &ukinfo->sq[sqdepth]; 651 info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE); 652 ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; 653 info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE); 654 ukinfo->sq_size = sqdepth >> sqshift; 655 ukinfo->rq_size = rqdepth >> rqshift; 656 ukinfo->qp_id = iwqp->ibqp.qp_num; 657 658 init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift; 659 init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift; 660 661 return 0; 662 } 663 664 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 665 { 666 struct irdma_pci_f *rf = iwqp->iwdev->rf; 667 struct irdma_cqp_request *cqp_request; 668 struct cqp_cmds_info *cqp_info; 669 struct irdma_create_qp_info *qp_info; 670 enum irdma_status_code status; 671 672 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 673 if (!cqp_request) 674 return -ENOMEM; 675 676 cqp_info = &cqp_request->info; 677 qp_info = &cqp_request->info.in.u.qp_create.info; 678 memset(qp_info, 0, sizeof(*qp_info)); 679 qp_info->mac_valid = true; 680 qp_info->cq_num_valid = true; 681 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 682 683 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 684 cqp_info->post_sq = 1; 685 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 686 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 687 status = irdma_handle_cqp_op(rf, cqp_request); 688 irdma_put_cqp_request(&rf->cqp, cqp_request); 689 690 return status ? -ENOMEM : 0; 691 } 692 693 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 694 struct irdma_qp_host_ctx_info *ctx_info) 695 { 696 struct irdma_device *iwdev = iwqp->iwdev; 697 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 698 struct irdma_roce_offload_info *roce_info; 699 struct irdma_udp_offload_info *udp_info; 700 701 udp_info = &iwqp->udp_info; 702 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 703 udp_info->cwnd = iwdev->roce_cwnd; 704 udp_info->rexmit_thresh = 2; 705 udp_info->rnr_nak_thresh = 2; 706 udp_info->src_port = 0xc000; 707 udp_info->dst_port = ROCE_V2_UDP_DPORT; 708 roce_info = &iwqp->roce_info; 709 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 710 711 roce_info->rd_en = true; 712 roce_info->wr_rdresp_en = true; 713 roce_info->bind_en = true; 714 roce_info->dcqcn_en = false; 715 roce_info->rtomin = 5; 716 717 roce_info->ack_credits = iwdev->roce_ackcreds; 718 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 719 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 720 721 if (!iwqp->user_mode) { 722 roce_info->priv_mode_en = true; 723 roce_info->fast_reg_en = true; 724 roce_info->udprivcq_en = true; 725 } 726 roce_info->roce_tver = 0; 727 728 ctx_info->roce_info = &iwqp->roce_info; 729 ctx_info->udp_info = &iwqp->udp_info; 730 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 731 } 732 733 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 734 struct irdma_qp_host_ctx_info *ctx_info) 735 { 736 struct irdma_device *iwdev = iwqp->iwdev; 737 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 738 struct irdma_iwarp_offload_info *iwarp_info; 739 740 iwarp_info = &iwqp->iwarp_info; 741 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 742 iwarp_info->rd_en = true; 743 iwarp_info->wr_rdresp_en = true; 744 iwarp_info->bind_en = true; 745 iwarp_info->ecn_en = true; 746 iwarp_info->rtomin = 5; 747 748 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 749 iwarp_info->ib_rd_en = true; 750 if (!iwqp->user_mode) { 751 iwarp_info->priv_mode_en = true; 752 iwarp_info->fast_reg_en = true; 753 } 754 iwarp_info->ddp_ver = 1; 755 iwarp_info->rdmap_ver = 1; 756 757 ctx_info->iwarp_info = &iwqp->iwarp_info; 758 ctx_info->iwarp_info_valid = true; 759 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 760 ctx_info->iwarp_info_valid = false; 761 } 762 763 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 764 struct irdma_device *iwdev) 765 { 766 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 767 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 768 769 if (init_attr->create_flags) 770 return -EOPNOTSUPP; 771 772 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 773 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 774 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) 775 return -EINVAL; 776 777 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 778 if (init_attr->qp_type != IB_QPT_RC && 779 init_attr->qp_type != IB_QPT_UD && 780 init_attr->qp_type != IB_QPT_GSI) 781 return -EOPNOTSUPP; 782 } else { 783 if (init_attr->qp_type != IB_QPT_RC) 784 return -EOPNOTSUPP; 785 } 786 787 return 0; 788 } 789 790 /** 791 * irdma_create_qp - create qp 792 * @ibqp: ptr of qp 793 * @init_attr: attributes for qp 794 * @udata: user data for create qp 795 */ 796 static int irdma_create_qp(struct ib_qp *ibqp, 797 struct ib_qp_init_attr *init_attr, 798 struct ib_udata *udata) 799 { 800 struct ib_pd *ibpd = ibqp->pd; 801 struct irdma_pd *iwpd = to_iwpd(ibpd); 802 struct irdma_device *iwdev = to_iwdev(ibpd->device); 803 struct irdma_pci_f *rf = iwdev->rf; 804 struct irdma_qp *iwqp = to_iwqp(ibqp); 805 struct irdma_create_qp_req req; 806 struct irdma_create_qp_resp uresp = {}; 807 u32 qp_num = 0; 808 enum irdma_status_code ret; 809 int err_code; 810 int sq_size; 811 int rq_size; 812 struct irdma_sc_qp *qp; 813 struct irdma_sc_dev *dev = &rf->sc_dev; 814 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 815 struct irdma_qp_init_info init_info = {}; 816 struct irdma_qp_host_ctx_info *ctx_info; 817 unsigned long flags; 818 819 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 820 if (err_code) 821 return err_code; 822 823 sq_size = init_attr->cap.max_send_wr; 824 rq_size = init_attr->cap.max_recv_wr; 825 826 init_info.vsi = &iwdev->vsi; 827 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 828 init_info.qp_uk_init_info.sq_size = sq_size; 829 init_info.qp_uk_init_info.rq_size = rq_size; 830 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 831 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 832 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 833 834 qp = &iwqp->sc_qp; 835 qp->qp_uk.back_qp = iwqp; 836 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 837 838 iwqp->iwdev = iwdev; 839 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 840 256); 841 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 842 iwqp->q2_ctx_mem.size, 843 &iwqp->q2_ctx_mem.pa, 844 GFP_KERNEL); 845 if (!iwqp->q2_ctx_mem.va) 846 return -ENOMEM; 847 848 init_info.q2 = iwqp->q2_ctx_mem.va; 849 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 850 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 851 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 852 853 if (init_attr->qp_type == IB_QPT_GSI) 854 qp_num = 1; 855 else 856 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 857 &qp_num, &rf->next_qp); 858 if (err_code) 859 goto error; 860 861 iwqp->iwpd = iwpd; 862 iwqp->ibqp.qp_num = qp_num; 863 qp = &iwqp->sc_qp; 864 iwqp->iwscq = to_iwcq(init_attr->send_cq); 865 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 866 iwqp->host_ctx.va = init_info.host_ctx; 867 iwqp->host_ctx.pa = init_info.host_ctx_pa; 868 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 869 870 init_info.pd = &iwpd->sc_pd; 871 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 872 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 873 init_info.qp_uk_init_info.first_sq_wq = 1; 874 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 875 init_waitqueue_head(&iwqp->waitq); 876 init_waitqueue_head(&iwqp->mod_qp_waitq); 877 878 if (udata) { 879 err_code = ib_copy_from_udata(&req, udata, 880 min(sizeof(req), udata->inlen)); 881 if (err_code) { 882 ibdev_dbg(&iwdev->ibdev, 883 "VERBS: ib_copy_from_data fail\n"); 884 goto error; 885 } 886 887 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 888 iwqp->user_mode = 1; 889 if (req.user_wqe_bufs) { 890 struct irdma_ucontext *ucontext = 891 rdma_udata_to_drv_context(udata, 892 struct irdma_ucontext, 893 ibucontext); 894 895 init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 896 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 897 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 898 &ucontext->qp_reg_mem_list); 899 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 900 901 if (!iwqp->iwpbl) { 902 err_code = -ENODATA; 903 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 904 goto error; 905 } 906 } 907 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 908 irdma_setup_virt_qp(iwdev, iwqp, &init_info); 909 } else { 910 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 911 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 912 } 913 914 if (err_code) { 915 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 916 goto error; 917 } 918 919 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 920 if (init_attr->qp_type == IB_QPT_RC) { 921 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 922 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 923 IRDMA_WRITE_WITH_IMM | 924 IRDMA_ROCE; 925 } else { 926 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 927 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 928 IRDMA_ROCE; 929 } 930 } else { 931 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 932 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 933 } 934 935 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 936 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 937 938 ret = irdma_sc_qp_init(qp, &init_info); 939 if (ret) { 940 err_code = -EPROTO; 941 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 942 goto error; 943 } 944 945 ctx_info = &iwqp->ctx_info; 946 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 947 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 948 949 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 950 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 951 else 952 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 953 954 err_code = irdma_cqp_create_qp_cmd(iwqp); 955 if (err_code) 956 goto error; 957 958 refcount_set(&iwqp->refcnt, 1); 959 spin_lock_init(&iwqp->lock); 960 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 961 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 962 rf->qp_table[qp_num] = iwqp; 963 iwqp->max_send_wr = sq_size; 964 iwqp->max_recv_wr = rq_size; 965 966 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 967 if (dev->ws_add(&iwdev->vsi, 0)) { 968 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 969 err_code = -EINVAL; 970 goto error; 971 } 972 973 irdma_qp_add_qos(&iwqp->sc_qp); 974 } 975 976 if (udata) { 977 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 978 if (udata->outlen < sizeof(uresp)) { 979 uresp.lsmm = 1; 980 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 981 } else { 982 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 983 uresp.lsmm = 1; 984 } 985 uresp.actual_sq_size = sq_size; 986 uresp.actual_rq_size = rq_size; 987 uresp.qp_id = qp_num; 988 uresp.qp_caps = qp->qp_uk.qp_caps; 989 990 err_code = ib_copy_to_udata(udata, &uresp, 991 min(sizeof(uresp), udata->outlen)); 992 if (err_code) { 993 ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); 994 irdma_destroy_qp(&iwqp->ibqp, udata); 995 return err_code; 996 } 997 } 998 999 init_completion(&iwqp->free_qp); 1000 return 0; 1001 1002 error: 1003 irdma_free_qp_rsrc(iwqp); 1004 return err_code; 1005 } 1006 1007 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1008 { 1009 int acc_flags = 0; 1010 1011 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1012 if (iwqp->roce_info.wr_rdresp_en) { 1013 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1014 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1015 } 1016 if (iwqp->roce_info.rd_en) 1017 acc_flags |= IB_ACCESS_REMOTE_READ; 1018 if (iwqp->roce_info.bind_en) 1019 acc_flags |= IB_ACCESS_MW_BIND; 1020 } else { 1021 if (iwqp->iwarp_info.wr_rdresp_en) { 1022 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1023 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1024 } 1025 if (iwqp->iwarp_info.rd_en) 1026 acc_flags |= IB_ACCESS_REMOTE_READ; 1027 if (iwqp->iwarp_info.bind_en) 1028 acc_flags |= IB_ACCESS_MW_BIND; 1029 } 1030 return acc_flags; 1031 } 1032 1033 /** 1034 * irdma_query_qp - query qp attributes 1035 * @ibqp: qp pointer 1036 * @attr: attributes pointer 1037 * @attr_mask: Not used 1038 * @init_attr: qp attributes to return 1039 */ 1040 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1041 int attr_mask, struct ib_qp_init_attr *init_attr) 1042 { 1043 struct irdma_qp *iwqp = to_iwqp(ibqp); 1044 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1045 1046 memset(attr, 0, sizeof(*attr)); 1047 memset(init_attr, 0, sizeof(*init_attr)); 1048 1049 attr->qp_state = iwqp->ibqp_state; 1050 attr->cur_qp_state = iwqp->ibqp_state; 1051 attr->cap.max_send_wr = iwqp->max_send_wr; 1052 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1053 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1054 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1055 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1056 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1057 attr->port_num = 1; 1058 if (rdma_protocol_roce(ibqp->device, 1)) { 1059 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1060 attr->qkey = iwqp->roce_info.qkey; 1061 attr->rq_psn = iwqp->udp_info.epsn; 1062 attr->sq_psn = iwqp->udp_info.psn_nxt; 1063 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1064 attr->pkey_index = iwqp->roce_info.p_key; 1065 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1066 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1067 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1068 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1069 } 1070 1071 init_attr->event_handler = iwqp->ibqp.event_handler; 1072 init_attr->qp_context = iwqp->ibqp.qp_context; 1073 init_attr->send_cq = iwqp->ibqp.send_cq; 1074 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1075 init_attr->cap = attr->cap; 1076 1077 return 0; 1078 } 1079 1080 /** 1081 * irdma_query_pkey - Query partition key 1082 * @ibdev: device pointer from stack 1083 * @port: port number 1084 * @index: index of pkey 1085 * @pkey: pointer to store the pkey 1086 */ 1087 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1088 u16 *pkey) 1089 { 1090 if (index >= IRDMA_PKEY_TBL_SZ) 1091 return -EINVAL; 1092 1093 *pkey = IRDMA_DEFAULT_PKEY; 1094 return 0; 1095 } 1096 1097 /** 1098 * irdma_modify_qp_roce - modify qp request 1099 * @ibqp: qp's pointer for modify 1100 * @attr: access attributes 1101 * @attr_mask: state mask 1102 * @udata: user data 1103 */ 1104 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1105 int attr_mask, struct ib_udata *udata) 1106 { 1107 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1108 struct irdma_qp *iwqp = to_iwqp(ibqp); 1109 struct irdma_device *iwdev = iwqp->iwdev; 1110 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1111 struct irdma_qp_host_ctx_info *ctx_info; 1112 struct irdma_roce_offload_info *roce_info; 1113 struct irdma_udp_offload_info *udp_info; 1114 struct irdma_modify_qp_info info = {}; 1115 struct irdma_modify_qp_resp uresp = {}; 1116 struct irdma_modify_qp_req ureq = {}; 1117 unsigned long flags; 1118 u8 issue_modify_qp = 0; 1119 int ret = 0; 1120 1121 ctx_info = &iwqp->ctx_info; 1122 roce_info = &iwqp->roce_info; 1123 udp_info = &iwqp->udp_info; 1124 1125 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1126 return -EOPNOTSUPP; 1127 1128 if (attr_mask & IB_QP_DEST_QPN) 1129 roce_info->dest_qp = attr->dest_qp_num; 1130 1131 if (attr_mask & IB_QP_PKEY_INDEX) { 1132 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1133 &roce_info->p_key); 1134 if (ret) 1135 return ret; 1136 } 1137 1138 if (attr_mask & IB_QP_QKEY) 1139 roce_info->qkey = attr->qkey; 1140 1141 if (attr_mask & IB_QP_PATH_MTU) 1142 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1143 1144 if (attr_mask & IB_QP_SQ_PSN) { 1145 udp_info->psn_nxt = attr->sq_psn; 1146 udp_info->lsn = 0xffff; 1147 udp_info->psn_una = attr->sq_psn; 1148 udp_info->psn_max = attr->sq_psn; 1149 } 1150 1151 if (attr_mask & IB_QP_RQ_PSN) 1152 udp_info->epsn = attr->rq_psn; 1153 1154 if (attr_mask & IB_QP_RNR_RETRY) 1155 udp_info->rnr_nak_thresh = attr->rnr_retry; 1156 1157 if (attr_mask & IB_QP_RETRY_CNT) 1158 udp_info->rexmit_thresh = attr->retry_cnt; 1159 1160 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1161 1162 if (attr_mask & IB_QP_AV) { 1163 struct irdma_av *av = &iwqp->roce_ah.av; 1164 const struct ib_gid_attr *sgid_attr; 1165 u16 vlan_id = VLAN_N_VID; 1166 u32 local_ip[4]; 1167 1168 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1169 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1170 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1171 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1172 udp_info->tos = attr->ah_attr.grh.traffic_class; 1173 irdma_qp_rem_qos(&iwqp->sc_qp); 1174 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1175 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1176 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1177 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1178 return -ENOMEM; 1179 irdma_qp_add_qos(&iwqp->sc_qp); 1180 } 1181 sgid_attr = attr->ah_attr.grh.sgid_attr; 1182 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1183 ctx_info->roce_info->mac_addr); 1184 if (ret) 1185 return ret; 1186 1187 if (vlan_id >= VLAN_N_VID && iwdev->dcb) 1188 vlan_id = 0; 1189 if (vlan_id < VLAN_N_VID) { 1190 udp_info->insert_vlan_tag = true; 1191 udp_info->vlan_tag = vlan_id | 1192 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1193 } else { 1194 udp_info->insert_vlan_tag = false; 1195 } 1196 1197 av->attrs = attr->ah_attr; 1198 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1199 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1200 if (av->sgid_addr.saddr.sa_family == AF_INET6) { 1201 __be32 *daddr = 1202 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1203 __be32 *saddr = 1204 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1205 1206 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1207 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1208 1209 udp_info->ipv4 = false; 1210 irdma_copy_ip_ntohl(local_ip, daddr); 1211 1212 udp_info->arp_idx = irdma_arp_table(iwdev->rf, 1213 &local_ip[0], 1214 false, NULL, 1215 IRDMA_ARP_RESOLVE); 1216 } else { 1217 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1218 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1219 1220 local_ip[0] = ntohl(daddr); 1221 1222 udp_info->ipv4 = true; 1223 udp_info->dest_ip_addr[0] = 0; 1224 udp_info->dest_ip_addr[1] = 0; 1225 udp_info->dest_ip_addr[2] = 0; 1226 udp_info->dest_ip_addr[3] = local_ip[0]; 1227 1228 udp_info->local_ipaddr[0] = 0; 1229 udp_info->local_ipaddr[1] = 0; 1230 udp_info->local_ipaddr[2] = 0; 1231 udp_info->local_ipaddr[3] = ntohl(saddr); 1232 } 1233 udp_info->arp_idx = 1234 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1235 attr->ah_attr.roce.dmac); 1236 } 1237 1238 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1239 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1240 ibdev_err(&iwdev->ibdev, 1241 "rd_atomic = %d, above max_hw_ord=%d\n", 1242 attr->max_rd_atomic, 1243 dev->hw_attrs.max_hw_ord); 1244 return -EINVAL; 1245 } 1246 if (attr->max_rd_atomic) 1247 roce_info->ord_size = attr->max_rd_atomic; 1248 info.ord_valid = true; 1249 } 1250 1251 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1252 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1253 ibdev_err(&iwdev->ibdev, 1254 "rd_atomic = %d, above max_hw_ird=%d\n", 1255 attr->max_rd_atomic, 1256 dev->hw_attrs.max_hw_ird); 1257 return -EINVAL; 1258 } 1259 if (attr->max_dest_rd_atomic) 1260 roce_info->ird_size = attr->max_dest_rd_atomic; 1261 } 1262 1263 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1264 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1265 roce_info->wr_rdresp_en = true; 1266 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1267 roce_info->wr_rdresp_en = true; 1268 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1269 roce_info->rd_en = true; 1270 } 1271 1272 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1273 1274 ibdev_dbg(&iwdev->ibdev, 1275 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1276 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1277 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1278 1279 spin_lock_irqsave(&iwqp->lock, flags); 1280 if (attr_mask & IB_QP_STATE) { 1281 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1282 iwqp->ibqp.qp_type, attr_mask)) { 1283 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1284 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1285 attr->qp_state); 1286 ret = -EINVAL; 1287 goto exit; 1288 } 1289 info.curr_iwarp_state = iwqp->iwarp_state; 1290 1291 switch (attr->qp_state) { 1292 case IB_QPS_INIT: 1293 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1294 ret = -EINVAL; 1295 goto exit; 1296 } 1297 1298 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1299 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1300 issue_modify_qp = 1; 1301 } 1302 break; 1303 case IB_QPS_RTR: 1304 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1305 ret = -EINVAL; 1306 goto exit; 1307 } 1308 info.arp_cache_idx_valid = true; 1309 info.cq_num_valid = true; 1310 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1311 issue_modify_qp = 1; 1312 break; 1313 case IB_QPS_RTS: 1314 if (iwqp->ibqp_state < IB_QPS_RTR || 1315 iwqp->ibqp_state == IB_QPS_ERR) { 1316 ret = -EINVAL; 1317 goto exit; 1318 } 1319 1320 info.arp_cache_idx_valid = true; 1321 info.cq_num_valid = true; 1322 info.ord_valid = true; 1323 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1324 issue_modify_qp = 1; 1325 if (iwdev->push_mode && udata && 1326 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1327 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1328 spin_unlock_irqrestore(&iwqp->lock, flags); 1329 irdma_alloc_push_page(iwqp); 1330 spin_lock_irqsave(&iwqp->lock, flags); 1331 } 1332 break; 1333 case IB_QPS_SQD: 1334 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1335 goto exit; 1336 1337 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1338 ret = -EINVAL; 1339 goto exit; 1340 } 1341 1342 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1343 issue_modify_qp = 1; 1344 break; 1345 case IB_QPS_SQE: 1346 case IB_QPS_ERR: 1347 case IB_QPS_RESET: 1348 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { 1349 spin_unlock_irqrestore(&iwqp->lock, flags); 1350 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1351 irdma_hw_modify_qp(iwdev, iwqp, &info, true); 1352 spin_lock_irqsave(&iwqp->lock, flags); 1353 } 1354 1355 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1356 spin_unlock_irqrestore(&iwqp->lock, flags); 1357 if (udata) { 1358 if (ib_copy_from_udata(&ureq, udata, 1359 min(sizeof(ureq), udata->inlen))) 1360 return -EINVAL; 1361 1362 irdma_flush_wqes(iwqp, 1363 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1364 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1365 IRDMA_REFLUSH); 1366 } 1367 return 0; 1368 } 1369 1370 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1371 issue_modify_qp = 1; 1372 break; 1373 default: 1374 ret = -EINVAL; 1375 goto exit; 1376 } 1377 1378 iwqp->ibqp_state = attr->qp_state; 1379 } 1380 1381 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1382 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1383 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1384 spin_unlock_irqrestore(&iwqp->lock, flags); 1385 1386 if (attr_mask & IB_QP_STATE) { 1387 if (issue_modify_qp) { 1388 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1389 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1390 return -EINVAL; 1391 spin_lock_irqsave(&iwqp->lock, flags); 1392 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1393 iwqp->iwarp_state = info.next_iwarp_state; 1394 iwqp->ibqp_state = attr->qp_state; 1395 } 1396 if (iwqp->ibqp_state > IB_QPS_RTS && 1397 !iwqp->flush_issued) { 1398 iwqp->flush_issued = 1; 1399 spin_unlock_irqrestore(&iwqp->lock, flags); 1400 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1401 IRDMA_FLUSH_RQ | 1402 IRDMA_FLUSH_WAIT); 1403 } else { 1404 spin_unlock_irqrestore(&iwqp->lock, flags); 1405 } 1406 } else { 1407 iwqp->ibqp_state = attr->qp_state; 1408 } 1409 if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1410 struct irdma_ucontext *ucontext; 1411 1412 ucontext = rdma_udata_to_drv_context(udata, 1413 struct irdma_ucontext, ibucontext); 1414 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1415 !iwqp->push_wqe_mmap_entry && 1416 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1417 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1418 uresp.push_valid = 1; 1419 uresp.push_offset = iwqp->sc_qp.push_offset; 1420 } 1421 ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1422 udata->outlen)); 1423 if (ret) { 1424 irdma_remove_push_mmap_entries(iwqp); 1425 ibdev_dbg(&iwdev->ibdev, 1426 "VERBS: copy_to_udata failed\n"); 1427 return ret; 1428 } 1429 } 1430 } 1431 1432 return 0; 1433 exit: 1434 spin_unlock_irqrestore(&iwqp->lock, flags); 1435 1436 return ret; 1437 } 1438 1439 /** 1440 * irdma_modify_qp - modify qp request 1441 * @ibqp: qp's pointer for modify 1442 * @attr: access attributes 1443 * @attr_mask: state mask 1444 * @udata: user data 1445 */ 1446 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1447 struct ib_udata *udata) 1448 { 1449 struct irdma_qp *iwqp = to_iwqp(ibqp); 1450 struct irdma_device *iwdev = iwqp->iwdev; 1451 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1452 struct irdma_qp_host_ctx_info *ctx_info; 1453 struct irdma_tcp_offload_info *tcp_info; 1454 struct irdma_iwarp_offload_info *offload_info; 1455 struct irdma_modify_qp_info info = {}; 1456 struct irdma_modify_qp_resp uresp = {}; 1457 struct irdma_modify_qp_req ureq = {}; 1458 u8 issue_modify_qp = 0; 1459 u8 dont_wait = 0; 1460 int err; 1461 unsigned long flags; 1462 1463 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1464 return -EOPNOTSUPP; 1465 1466 ctx_info = &iwqp->ctx_info; 1467 offload_info = &iwqp->iwarp_info; 1468 tcp_info = &iwqp->tcp_info; 1469 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1470 ibdev_dbg(&iwdev->ibdev, 1471 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1472 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1473 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1474 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1475 1476 spin_lock_irqsave(&iwqp->lock, flags); 1477 if (attr_mask & IB_QP_STATE) { 1478 info.curr_iwarp_state = iwqp->iwarp_state; 1479 switch (attr->qp_state) { 1480 case IB_QPS_INIT: 1481 case IB_QPS_RTR: 1482 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1483 err = -EINVAL; 1484 goto exit; 1485 } 1486 1487 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1488 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1489 issue_modify_qp = 1; 1490 } 1491 if (iwdev->push_mode && udata && 1492 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1493 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1494 spin_unlock_irqrestore(&iwqp->lock, flags); 1495 irdma_alloc_push_page(iwqp); 1496 spin_lock_irqsave(&iwqp->lock, flags); 1497 } 1498 break; 1499 case IB_QPS_RTS: 1500 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1501 !iwqp->cm_id) { 1502 err = -EINVAL; 1503 goto exit; 1504 } 1505 1506 issue_modify_qp = 1; 1507 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1508 iwqp->hte_added = 1; 1509 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1510 info.tcp_ctx_valid = true; 1511 info.ord_valid = true; 1512 info.arp_cache_idx_valid = true; 1513 info.cq_num_valid = true; 1514 break; 1515 case IB_QPS_SQD: 1516 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1517 err = 0; 1518 goto exit; 1519 } 1520 1521 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1522 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1523 err = 0; 1524 goto exit; 1525 } 1526 1527 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1528 err = -EINVAL; 1529 goto exit; 1530 } 1531 1532 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1533 issue_modify_qp = 1; 1534 break; 1535 case IB_QPS_SQE: 1536 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1537 err = -EINVAL; 1538 goto exit; 1539 } 1540 1541 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1542 issue_modify_qp = 1; 1543 break; 1544 case IB_QPS_ERR: 1545 case IB_QPS_RESET: 1546 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1547 spin_unlock_irqrestore(&iwqp->lock, flags); 1548 if (udata) { 1549 if (ib_copy_from_udata(&ureq, udata, 1550 min(sizeof(ureq), udata->inlen))) 1551 return -EINVAL; 1552 1553 irdma_flush_wqes(iwqp, 1554 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1555 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1556 IRDMA_REFLUSH); 1557 } 1558 return 0; 1559 } 1560 1561 if (iwqp->sc_qp.term_flags) { 1562 spin_unlock_irqrestore(&iwqp->lock, flags); 1563 irdma_terminate_del_timer(&iwqp->sc_qp); 1564 spin_lock_irqsave(&iwqp->lock, flags); 1565 } 1566 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1567 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1568 iwdev->iw_status && 1569 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1570 info.reset_tcp_conn = true; 1571 else 1572 dont_wait = 1; 1573 1574 issue_modify_qp = 1; 1575 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1576 break; 1577 default: 1578 err = -EINVAL; 1579 goto exit; 1580 } 1581 1582 iwqp->ibqp_state = attr->qp_state; 1583 } 1584 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1585 ctx_info->iwarp_info_valid = true; 1586 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1587 offload_info->wr_rdresp_en = true; 1588 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1589 offload_info->wr_rdresp_en = true; 1590 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1591 offload_info->rd_en = true; 1592 } 1593 1594 if (ctx_info->iwarp_info_valid) { 1595 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1596 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1597 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1598 } 1599 spin_unlock_irqrestore(&iwqp->lock, flags); 1600 1601 if (attr_mask & IB_QP_STATE) { 1602 if (issue_modify_qp) { 1603 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1604 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1605 return -EINVAL; 1606 } 1607 1608 spin_lock_irqsave(&iwqp->lock, flags); 1609 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1610 iwqp->iwarp_state = info.next_iwarp_state; 1611 iwqp->ibqp_state = attr->qp_state; 1612 } 1613 spin_unlock_irqrestore(&iwqp->lock, flags); 1614 } 1615 1616 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1617 if (dont_wait) { 1618 if (iwqp->cm_id && iwqp->hw_tcp_state) { 1619 spin_lock_irqsave(&iwqp->lock, flags); 1620 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1621 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1622 spin_unlock_irqrestore(&iwqp->lock, flags); 1623 irdma_cm_disconn(iwqp); 1624 } 1625 } else { 1626 int close_timer_started; 1627 1628 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1629 1630 if (iwqp->cm_node) { 1631 refcount_inc(&iwqp->cm_node->refcnt); 1632 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1633 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1634 if (iwqp->cm_id && close_timer_started == 1) 1635 irdma_schedule_cm_timer(iwqp->cm_node, 1636 (struct irdma_puda_buf *)iwqp, 1637 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1638 1639 irdma_rem_ref_cm_node(iwqp->cm_node); 1640 } else { 1641 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1642 } 1643 } 1644 } 1645 if (attr_mask & IB_QP_STATE && udata && 1646 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1647 struct irdma_ucontext *ucontext; 1648 1649 ucontext = rdma_udata_to_drv_context(udata, 1650 struct irdma_ucontext, ibucontext); 1651 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1652 !iwqp->push_wqe_mmap_entry && 1653 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1654 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1655 uresp.push_valid = 1; 1656 uresp.push_offset = iwqp->sc_qp.push_offset; 1657 } 1658 1659 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1660 udata->outlen)); 1661 if (err) { 1662 irdma_remove_push_mmap_entries(iwqp); 1663 ibdev_dbg(&iwdev->ibdev, 1664 "VERBS: copy_to_udata failed\n"); 1665 return err; 1666 } 1667 } 1668 1669 return 0; 1670 exit: 1671 spin_unlock_irqrestore(&iwqp->lock, flags); 1672 1673 return err; 1674 } 1675 1676 /** 1677 * irdma_cq_free_rsrc - free up resources for cq 1678 * @rf: RDMA PCI function 1679 * @iwcq: cq ptr 1680 */ 1681 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1682 { 1683 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1684 1685 if (!iwcq->user_mode) { 1686 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1687 iwcq->kmem.va, iwcq->kmem.pa); 1688 iwcq->kmem.va = NULL; 1689 dma_free_coherent(rf->sc_dev.hw->device, 1690 iwcq->kmem_shadow.size, 1691 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1692 iwcq->kmem_shadow.va = NULL; 1693 } 1694 1695 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1696 } 1697 1698 /** 1699 * irdma_free_cqbuf - worker to free a cq buffer 1700 * @work: provides access to the cq buffer to free 1701 */ 1702 static void irdma_free_cqbuf(struct work_struct *work) 1703 { 1704 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1705 1706 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1707 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1708 cq_buf->kmem_buf.va = NULL; 1709 kfree(cq_buf); 1710 } 1711 1712 /** 1713 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1714 * @iwcq: cq which owns the resize_list 1715 * @iwdev: irdma device 1716 * @lcqe_buf: the buffer where the last cqe is received 1717 */ 1718 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1719 struct irdma_device *iwdev, 1720 struct irdma_cq_buf *lcqe_buf) 1721 { 1722 struct list_head *tmp_node, *list_node; 1723 struct irdma_cq_buf *cq_buf; 1724 int cnt = 0; 1725 1726 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1727 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1728 if (cq_buf == lcqe_buf) 1729 return cnt; 1730 1731 list_del(&cq_buf->list); 1732 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1733 cnt++; 1734 } 1735 1736 return cnt; 1737 } 1738 1739 /** 1740 * irdma_destroy_cq - destroy cq 1741 * @ib_cq: cq pointer 1742 * @udata: user data 1743 */ 1744 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1745 { 1746 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1747 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1748 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1749 struct irdma_sc_dev *dev = cq->dev; 1750 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1751 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1752 unsigned long flags; 1753 1754 spin_lock_irqsave(&iwcq->lock, flags); 1755 if (!list_empty(&iwcq->resize_list)) 1756 irdma_process_resize_list(iwcq, iwdev, NULL); 1757 spin_unlock_irqrestore(&iwcq->lock, flags); 1758 1759 irdma_cq_wq_destroy(iwdev->rf, cq); 1760 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1761 1762 spin_lock_irqsave(&iwceq->ce_lock, flags); 1763 irdma_sc_cleanup_ceqes(cq, ceq); 1764 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1765 1766 return 0; 1767 } 1768 1769 /** 1770 * irdma_resize_cq - resize cq 1771 * @ibcq: cq to be resized 1772 * @entries: desired cq size 1773 * @udata: user data 1774 */ 1775 static int irdma_resize_cq(struct ib_cq *ibcq, int entries, 1776 struct ib_udata *udata) 1777 { 1778 struct irdma_cq *iwcq = to_iwcq(ibcq); 1779 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 1780 struct irdma_cqp_request *cqp_request; 1781 struct cqp_cmds_info *cqp_info; 1782 struct irdma_modify_cq_info *m_info; 1783 struct irdma_modify_cq_info info = {}; 1784 struct irdma_dma_mem kmem_buf; 1785 struct irdma_cq_mr *cqmr_buf; 1786 struct irdma_pbl *iwpbl_buf; 1787 struct irdma_device *iwdev; 1788 struct irdma_pci_f *rf; 1789 struct irdma_cq_buf *cq_buf = NULL; 1790 enum irdma_status_code status = 0; 1791 unsigned long flags; 1792 int ret; 1793 1794 iwdev = to_iwdev(ibcq->device); 1795 rf = iwdev->rf; 1796 1797 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1798 IRDMA_FEATURE_CQ_RESIZE)) 1799 return -EOPNOTSUPP; 1800 1801 if (entries > rf->max_cqe) 1802 return -EINVAL; 1803 1804 if (!iwcq->user_mode) { 1805 entries++; 1806 if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1807 entries *= 2; 1808 } 1809 1810 info.cq_size = max(entries, 4); 1811 1812 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 1813 return 0; 1814 1815 if (udata) { 1816 struct irdma_resize_cq_req req = {}; 1817 struct irdma_ucontext *ucontext = 1818 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1819 ibucontext); 1820 1821 /* CQ resize not supported with legacy GEN_1 libi40iw */ 1822 if (ucontext->legacy_mode) 1823 return -EOPNOTSUPP; 1824 1825 if (ib_copy_from_udata(&req, udata, 1826 min(sizeof(req), udata->inlen))) 1827 return -EINVAL; 1828 1829 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1830 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 1831 &ucontext->cq_reg_mem_list); 1832 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1833 1834 if (!iwpbl_buf) 1835 return -ENOMEM; 1836 1837 cqmr_buf = &iwpbl_buf->cq_mr; 1838 if (iwpbl_buf->pbl_allocated) { 1839 info.virtual_map = true; 1840 info.pbl_chunk_size = 1; 1841 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 1842 } else { 1843 info.cq_pa = cqmr_buf->cq_pbl.addr; 1844 } 1845 } else { 1846 /* Kmode CQ resize */ 1847 int rsize; 1848 1849 rsize = info.cq_size * sizeof(struct irdma_cqe); 1850 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 1851 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 1852 kmem_buf.size, &kmem_buf.pa, 1853 GFP_KERNEL); 1854 if (!kmem_buf.va) 1855 return -ENOMEM; 1856 1857 info.cq_base = kmem_buf.va; 1858 info.cq_pa = kmem_buf.pa; 1859 cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); 1860 if (!cq_buf) { 1861 ret = -ENOMEM; 1862 goto error; 1863 } 1864 } 1865 1866 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1867 if (!cqp_request) { 1868 ret = -ENOMEM; 1869 goto error; 1870 } 1871 1872 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 1873 info.cq_resize = true; 1874 1875 cqp_info = &cqp_request->info; 1876 m_info = &cqp_info->in.u.cq_modify.info; 1877 memcpy(m_info, &info, sizeof(*m_info)); 1878 1879 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 1880 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 1881 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 1882 cqp_info->post_sq = 1; 1883 status = irdma_handle_cqp_op(rf, cqp_request); 1884 irdma_put_cqp_request(&rf->cqp, cqp_request); 1885 if (status) { 1886 ret = -EPROTO; 1887 goto error; 1888 } 1889 1890 spin_lock_irqsave(&iwcq->lock, flags); 1891 if (cq_buf) { 1892 cq_buf->kmem_buf = iwcq->kmem; 1893 cq_buf->hw = dev->hw; 1894 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 1895 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 1896 list_add_tail(&cq_buf->list, &iwcq->resize_list); 1897 iwcq->kmem = kmem_buf; 1898 } 1899 1900 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 1901 ibcq->cqe = info.cq_size - 1; 1902 spin_unlock_irqrestore(&iwcq->lock, flags); 1903 1904 return 0; 1905 error: 1906 if (!udata) { 1907 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 1908 kmem_buf.pa); 1909 kmem_buf.va = NULL; 1910 } 1911 kfree(cq_buf); 1912 1913 return ret; 1914 } 1915 1916 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 1917 { 1918 /* GEN1 does not support CQ create flags */ 1919 if (hw_rev == IRDMA_GEN_1) 1920 return flags ? -EOPNOTSUPP : 0; 1921 1922 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 1923 } 1924 1925 /** 1926 * irdma_create_cq - create cq 1927 * @ibcq: CQ allocated 1928 * @attr: attributes for cq 1929 * @udata: user data 1930 */ 1931 static int irdma_create_cq(struct ib_cq *ibcq, 1932 const struct ib_cq_init_attr *attr, 1933 struct ib_udata *udata) 1934 { 1935 struct ib_device *ibdev = ibcq->device; 1936 struct irdma_device *iwdev = to_iwdev(ibdev); 1937 struct irdma_pci_f *rf = iwdev->rf; 1938 struct irdma_cq *iwcq = to_iwcq(ibcq); 1939 u32 cq_num = 0; 1940 struct irdma_sc_cq *cq; 1941 struct irdma_sc_dev *dev = &rf->sc_dev; 1942 struct irdma_cq_init_info info = {}; 1943 enum irdma_status_code status; 1944 struct irdma_cqp_request *cqp_request; 1945 struct cqp_cmds_info *cqp_info; 1946 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 1947 unsigned long flags; 1948 int err_code; 1949 int entries = attr->cqe; 1950 1951 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 1952 if (err_code) 1953 return err_code; 1954 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 1955 &rf->next_cq); 1956 if (err_code) 1957 return err_code; 1958 1959 cq = &iwcq->sc_cq; 1960 cq->back_cq = iwcq; 1961 spin_lock_init(&iwcq->lock); 1962 INIT_LIST_HEAD(&iwcq->resize_list); 1963 info.dev = dev; 1964 ukinfo->cq_size = max(entries, 4); 1965 ukinfo->cq_id = cq_num; 1966 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 1967 if (attr->comp_vector < rf->ceqs_count) 1968 info.ceq_id = attr->comp_vector; 1969 info.ceq_id_valid = true; 1970 info.ceqe_mask = 1; 1971 info.type = IRDMA_CQ_TYPE_IWARP; 1972 info.vsi = &iwdev->vsi; 1973 1974 if (udata) { 1975 struct irdma_ucontext *ucontext; 1976 struct irdma_create_cq_req req = {}; 1977 struct irdma_cq_mr *cqmr; 1978 struct irdma_pbl *iwpbl; 1979 struct irdma_pbl *iwpbl_shadow; 1980 struct irdma_cq_mr *cqmr_shadow; 1981 1982 iwcq->user_mode = true; 1983 ucontext = 1984 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1985 ibucontext); 1986 if (ib_copy_from_udata(&req, udata, 1987 min(sizeof(req), udata->inlen))) { 1988 err_code = -EFAULT; 1989 goto cq_free_rsrc; 1990 } 1991 1992 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1993 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 1994 &ucontext->cq_reg_mem_list); 1995 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1996 if (!iwpbl) { 1997 err_code = -EPROTO; 1998 goto cq_free_rsrc; 1999 } 2000 2001 iwcq->iwpbl = iwpbl; 2002 iwcq->cq_mem_size = 0; 2003 cqmr = &iwpbl->cq_mr; 2004 2005 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2006 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 2007 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2008 iwpbl_shadow = irdma_get_pbl( 2009 (unsigned long)req.user_shadow_area, 2010 &ucontext->cq_reg_mem_list); 2011 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2012 2013 if (!iwpbl_shadow) { 2014 err_code = -EPROTO; 2015 goto cq_free_rsrc; 2016 } 2017 iwcq->iwpbl_shadow = iwpbl_shadow; 2018 cqmr_shadow = &iwpbl_shadow->cq_mr; 2019 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2020 cqmr->split = true; 2021 } else { 2022 info.shadow_area_pa = cqmr->shadow; 2023 } 2024 if (iwpbl->pbl_allocated) { 2025 info.virtual_map = true; 2026 info.pbl_chunk_size = 1; 2027 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2028 } else { 2029 info.cq_base_pa = cqmr->cq_pbl.addr; 2030 } 2031 } else { 2032 /* Kmode allocations */ 2033 int rsize; 2034 2035 if (entries < 1 || entries > rf->max_cqe) { 2036 err_code = -EINVAL; 2037 goto cq_free_rsrc; 2038 } 2039 2040 entries++; 2041 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2042 entries *= 2; 2043 ukinfo->cq_size = entries; 2044 2045 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2046 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2047 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2048 iwcq->kmem.size, 2049 &iwcq->kmem.pa, GFP_KERNEL); 2050 if (!iwcq->kmem.va) { 2051 err_code = -ENOMEM; 2052 goto cq_free_rsrc; 2053 } 2054 2055 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2056 64); 2057 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2058 iwcq->kmem_shadow.size, 2059 &iwcq->kmem_shadow.pa, 2060 GFP_KERNEL); 2061 if (!iwcq->kmem_shadow.va) { 2062 err_code = -ENOMEM; 2063 goto cq_free_rsrc; 2064 } 2065 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2066 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2067 ukinfo->cq_base = iwcq->kmem.va; 2068 info.cq_base_pa = iwcq->kmem.pa; 2069 } 2070 2071 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2072 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2073 (u32)IRDMA_MAX_CQ_READ_THRESH); 2074 2075 if (irdma_sc_cq_init(cq, &info)) { 2076 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2077 err_code = -EPROTO; 2078 goto cq_free_rsrc; 2079 } 2080 2081 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2082 if (!cqp_request) { 2083 err_code = -ENOMEM; 2084 goto cq_free_rsrc; 2085 } 2086 2087 cqp_info = &cqp_request->info; 2088 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2089 cqp_info->post_sq = 1; 2090 cqp_info->in.u.cq_create.cq = cq; 2091 cqp_info->in.u.cq_create.check_overflow = true; 2092 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2093 status = irdma_handle_cqp_op(rf, cqp_request); 2094 irdma_put_cqp_request(&rf->cqp, cqp_request); 2095 if (status) { 2096 err_code = -ENOMEM; 2097 goto cq_free_rsrc; 2098 } 2099 2100 if (udata) { 2101 struct irdma_create_cq_resp resp = {}; 2102 2103 resp.cq_id = info.cq_uk_init_info.cq_id; 2104 resp.cq_size = info.cq_uk_init_info.cq_size; 2105 if (ib_copy_to_udata(udata, &resp, 2106 min(sizeof(resp), udata->outlen))) { 2107 ibdev_dbg(&iwdev->ibdev, 2108 "VERBS: copy to user data\n"); 2109 err_code = -EPROTO; 2110 goto cq_destroy; 2111 } 2112 } 2113 return 0; 2114 cq_destroy: 2115 irdma_cq_wq_destroy(rf, cq); 2116 cq_free_rsrc: 2117 irdma_cq_free_rsrc(rf, iwcq); 2118 2119 return err_code; 2120 } 2121 2122 /** 2123 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2124 * @access: IB access flags 2125 */ 2126 static inline u16 irdma_get_mr_access(int access) 2127 { 2128 u16 hw_access = 0; 2129 2130 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2131 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2132 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2133 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2134 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2135 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2136 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2137 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2138 hw_access |= (access & IB_ZERO_BASED) ? 2139 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2140 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2141 2142 return hw_access; 2143 } 2144 2145 /** 2146 * irdma_free_stag - free stag resource 2147 * @iwdev: irdma device 2148 * @stag: stag to free 2149 */ 2150 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2151 { 2152 u32 stag_idx; 2153 2154 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2155 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2156 } 2157 2158 /** 2159 * irdma_create_stag - create random stag 2160 * @iwdev: irdma device 2161 */ 2162 static u32 irdma_create_stag(struct irdma_device *iwdev) 2163 { 2164 u32 stag = 0; 2165 u32 stag_index = 0; 2166 u32 next_stag_index; 2167 u32 driver_key; 2168 u32 random; 2169 u8 consumer_key; 2170 int ret; 2171 2172 get_random_bytes(&random, sizeof(random)); 2173 consumer_key = (u8)random; 2174 2175 driver_key = random & ~iwdev->rf->mr_stagmask; 2176 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2177 next_stag_index %= iwdev->rf->max_mr; 2178 2179 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2180 iwdev->rf->max_mr, &stag_index, 2181 &next_stag_index); 2182 if (ret) 2183 return stag; 2184 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2185 stag |= driver_key; 2186 stag += (u32)consumer_key; 2187 2188 return stag; 2189 } 2190 2191 /** 2192 * irdma_next_pbl_addr - Get next pbl address 2193 * @pbl: pointer to a pble 2194 * @pinfo: info pointer 2195 * @idx: index 2196 */ 2197 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2198 u32 *idx) 2199 { 2200 *idx += 1; 2201 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2202 return ++pbl; 2203 *idx = 0; 2204 (*pinfo)++; 2205 2206 return (*pinfo)->addr; 2207 } 2208 2209 /** 2210 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2211 * @iwmr: iwmr for IB's user page addresses 2212 * @pbl: ple pointer to save 1 level or 0 level pble 2213 * @level: indicated level 0, 1 or 2 2214 */ 2215 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2216 enum irdma_pble_level level) 2217 { 2218 struct ib_umem *region = iwmr->region; 2219 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2220 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2221 struct irdma_pble_info *pinfo; 2222 struct ib_block_iter biter; 2223 u32 idx = 0; 2224 u32 pbl_cnt = 0; 2225 2226 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2227 2228 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2229 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2230 2231 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2232 *pbl = rdma_block_iter_dma_address(&biter); 2233 if (++pbl_cnt == palloc->total_cnt) 2234 break; 2235 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2236 } 2237 } 2238 2239 /** 2240 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2241 * @arr: lvl1 pbl array 2242 * @npages: page count 2243 * @pg_size: page size 2244 * 2245 */ 2246 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2247 { 2248 u32 pg_idx; 2249 2250 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2251 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2252 return false; 2253 } 2254 2255 return true; 2256 } 2257 2258 /** 2259 * irdma_check_mr_contiguous - check if MR is physically contiguous 2260 * @palloc: pbl allocation struct 2261 * @pg_size: page size 2262 */ 2263 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2264 u32 pg_size) 2265 { 2266 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2267 struct irdma_pble_info *leaf = lvl2->leaf; 2268 u64 *arr = NULL; 2269 u64 *start_addr = NULL; 2270 int i; 2271 bool ret; 2272 2273 if (palloc->level == PBLE_LEVEL_1) { 2274 arr = palloc->level1.addr; 2275 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2276 pg_size); 2277 return ret; 2278 } 2279 2280 start_addr = leaf->addr; 2281 2282 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2283 arr = leaf->addr; 2284 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2285 return false; 2286 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2287 if (!ret) 2288 return false; 2289 } 2290 2291 return true; 2292 } 2293 2294 /** 2295 * irdma_setup_pbles - copy user pg address to pble's 2296 * @rf: RDMA PCI function 2297 * @iwmr: mr pointer for this memory registration 2298 * @use_pbles: flag if to use pble's 2299 */ 2300 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2301 bool use_pbles) 2302 { 2303 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2304 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2305 struct irdma_pble_info *pinfo; 2306 u64 *pbl; 2307 enum irdma_status_code status; 2308 enum irdma_pble_level level = PBLE_LEVEL_1; 2309 2310 if (use_pbles) { 2311 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2312 false); 2313 if (status) 2314 return -ENOMEM; 2315 2316 iwpbl->pbl_allocated = true; 2317 level = palloc->level; 2318 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2319 palloc->level2.leaf; 2320 pbl = pinfo->addr; 2321 } else { 2322 pbl = iwmr->pgaddrmem; 2323 } 2324 2325 irdma_copy_user_pgaddrs(iwmr, pbl, level); 2326 2327 if (use_pbles) 2328 iwmr->pgaddrmem[0] = *pbl; 2329 2330 return 0; 2331 } 2332 2333 /** 2334 * irdma_handle_q_mem - handle memory for qp and cq 2335 * @iwdev: irdma device 2336 * @req: information for q memory management 2337 * @iwpbl: pble struct 2338 * @use_pbles: flag to use pble 2339 */ 2340 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2341 struct irdma_mem_reg_req *req, 2342 struct irdma_pbl *iwpbl, bool use_pbles) 2343 { 2344 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2345 struct irdma_mr *iwmr = iwpbl->iwmr; 2346 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2347 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2348 struct irdma_hmc_pble *hmc_p; 2349 u64 *arr = iwmr->pgaddrmem; 2350 u32 pg_size, total; 2351 int err = 0; 2352 bool ret = true; 2353 2354 pg_size = iwmr->page_size; 2355 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2356 if (err) 2357 return err; 2358 2359 if (use_pbles && palloc->level != PBLE_LEVEL_1) { 2360 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2361 iwpbl->pbl_allocated = false; 2362 return -ENOMEM; 2363 } 2364 2365 if (use_pbles) 2366 arr = palloc->level1.addr; 2367 2368 switch (iwmr->type) { 2369 case IRDMA_MEMREG_TYPE_QP: 2370 total = req->sq_pages + req->rq_pages; 2371 hmc_p = &qpmr->sq_pbl; 2372 qpmr->shadow = (dma_addr_t)arr[total]; 2373 2374 if (use_pbles) { 2375 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2376 pg_size); 2377 if (ret) 2378 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2379 req->rq_pages, 2380 pg_size); 2381 } 2382 2383 if (!ret) { 2384 hmc_p->idx = palloc->level1.idx; 2385 hmc_p = &qpmr->rq_pbl; 2386 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2387 } else { 2388 hmc_p->addr = arr[0]; 2389 hmc_p = &qpmr->rq_pbl; 2390 hmc_p->addr = arr[req->sq_pages]; 2391 } 2392 break; 2393 case IRDMA_MEMREG_TYPE_CQ: 2394 hmc_p = &cqmr->cq_pbl; 2395 2396 if (!cqmr->split) 2397 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2398 2399 if (use_pbles) 2400 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2401 pg_size); 2402 2403 if (!ret) 2404 hmc_p->idx = palloc->level1.idx; 2405 else 2406 hmc_p->addr = arr[0]; 2407 break; 2408 default: 2409 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2410 err = -EINVAL; 2411 } 2412 2413 if (use_pbles && ret) { 2414 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2415 iwpbl->pbl_allocated = false; 2416 } 2417 2418 return err; 2419 } 2420 2421 /** 2422 * irdma_hw_alloc_mw - create the hw memory window 2423 * @iwdev: irdma device 2424 * @iwmr: pointer to memory window info 2425 */ 2426 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2427 { 2428 struct irdma_mw_alloc_info *info; 2429 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2430 struct irdma_cqp_request *cqp_request; 2431 struct cqp_cmds_info *cqp_info; 2432 enum irdma_status_code status; 2433 2434 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2435 if (!cqp_request) 2436 return -ENOMEM; 2437 2438 cqp_info = &cqp_request->info; 2439 info = &cqp_info->in.u.mw_alloc.info; 2440 memset(info, 0, sizeof(*info)); 2441 if (iwmr->ibmw.type == IB_MW_TYPE_1) 2442 info->mw_wide = true; 2443 2444 info->page_size = PAGE_SIZE; 2445 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2446 info->pd_id = iwpd->sc_pd.pd_id; 2447 info->remote_access = true; 2448 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 2449 cqp_info->post_sq = 1; 2450 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 2451 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 2452 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2453 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2454 2455 return status ? -ENOMEM : 0; 2456 } 2457 2458 /** 2459 * irdma_alloc_mw - Allocate memory window 2460 * @ibmw: Memory Window 2461 * @udata: user data pointer 2462 */ 2463 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2464 { 2465 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2466 struct irdma_mr *iwmr = to_iwmw(ibmw); 2467 int err_code; 2468 u32 stag; 2469 2470 stag = irdma_create_stag(iwdev); 2471 if (!stag) 2472 return -ENOMEM; 2473 2474 iwmr->stag = stag; 2475 ibmw->rkey = stag; 2476 2477 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 2478 if (err_code) { 2479 irdma_free_stag(iwdev, stag); 2480 return err_code; 2481 } 2482 2483 return 0; 2484 } 2485 2486 /** 2487 * irdma_dealloc_mw - Dealloc memory window 2488 * @ibmw: memory window structure. 2489 */ 2490 static int irdma_dealloc_mw(struct ib_mw *ibmw) 2491 { 2492 struct ib_pd *ibpd = ibmw->pd; 2493 struct irdma_pd *iwpd = to_iwpd(ibpd); 2494 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 2495 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2496 struct irdma_cqp_request *cqp_request; 2497 struct cqp_cmds_info *cqp_info; 2498 struct irdma_dealloc_stag_info *info; 2499 2500 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2501 if (!cqp_request) 2502 return -ENOMEM; 2503 2504 cqp_info = &cqp_request->info; 2505 info = &cqp_info->in.u.dealloc_stag.info; 2506 memset(info, 0, sizeof(*info)); 2507 info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; 2508 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 2509 info->mr = false; 2510 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 2511 cqp_info->post_sq = 1; 2512 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 2513 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 2514 irdma_handle_cqp_op(iwdev->rf, cqp_request); 2515 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2516 irdma_free_stag(iwdev, iwmr->stag); 2517 2518 return 0; 2519 } 2520 2521 /** 2522 * irdma_hw_alloc_stag - cqp command to allocate stag 2523 * @iwdev: irdma device 2524 * @iwmr: irdma mr pointer 2525 */ 2526 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 2527 struct irdma_mr *iwmr) 2528 { 2529 struct irdma_allocate_stag_info *info; 2530 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2531 enum irdma_status_code status; 2532 int err = 0; 2533 struct irdma_cqp_request *cqp_request; 2534 struct cqp_cmds_info *cqp_info; 2535 2536 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2537 if (!cqp_request) 2538 return -ENOMEM; 2539 2540 cqp_info = &cqp_request->info; 2541 info = &cqp_info->in.u.alloc_stag.info; 2542 memset(info, 0, sizeof(*info)); 2543 info->page_size = PAGE_SIZE; 2544 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2545 info->pd_id = iwpd->sc_pd.pd_id; 2546 info->total_len = iwmr->len; 2547 info->remote_access = true; 2548 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 2549 cqp_info->post_sq = 1; 2550 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 2551 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 2552 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2553 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2554 if (status) 2555 err = -ENOMEM; 2556 2557 return err; 2558 } 2559 2560 /** 2561 * irdma_alloc_mr - register stag for fast memory registration 2562 * @pd: ibpd pointer 2563 * @mr_type: memory for stag registrion 2564 * @max_num_sg: man number of pages 2565 */ 2566 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2567 u32 max_num_sg) 2568 { 2569 struct irdma_device *iwdev = to_iwdev(pd->device); 2570 struct irdma_pble_alloc *palloc; 2571 struct irdma_pbl *iwpbl; 2572 struct irdma_mr *iwmr; 2573 enum irdma_status_code status; 2574 u32 stag; 2575 int err_code = -ENOMEM; 2576 2577 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2578 if (!iwmr) 2579 return ERR_PTR(-ENOMEM); 2580 2581 stag = irdma_create_stag(iwdev); 2582 if (!stag) { 2583 err_code = -ENOMEM; 2584 goto err; 2585 } 2586 2587 iwmr->stag = stag; 2588 iwmr->ibmr.rkey = stag; 2589 iwmr->ibmr.lkey = stag; 2590 iwmr->ibmr.pd = pd; 2591 iwmr->ibmr.device = pd->device; 2592 iwpbl = &iwmr->iwpbl; 2593 iwpbl->iwmr = iwmr; 2594 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2595 palloc = &iwpbl->pble_alloc; 2596 iwmr->page_cnt = max_num_sg; 2597 status = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 2598 true); 2599 if (status) 2600 goto err_get_pble; 2601 2602 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 2603 if (err_code) 2604 goto err_alloc_stag; 2605 2606 iwpbl->pbl_allocated = true; 2607 2608 return &iwmr->ibmr; 2609 err_alloc_stag: 2610 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2611 err_get_pble: 2612 irdma_free_stag(iwdev, stag); 2613 err: 2614 kfree(iwmr); 2615 2616 return ERR_PTR(err_code); 2617 } 2618 2619 /** 2620 * irdma_set_page - populate pbl list for fmr 2621 * @ibmr: ib mem to access iwarp mr pointer 2622 * @addr: page dma address fro pbl list 2623 */ 2624 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 2625 { 2626 struct irdma_mr *iwmr = to_iwmr(ibmr); 2627 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2628 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2629 u64 *pbl; 2630 2631 if (unlikely(iwmr->npages == iwmr->page_cnt)) 2632 return -ENOMEM; 2633 2634 pbl = palloc->level1.addr; 2635 pbl[iwmr->npages++] = addr; 2636 2637 return 0; 2638 } 2639 2640 /** 2641 * irdma_map_mr_sg - map of sg list for fmr 2642 * @ibmr: ib mem to access iwarp mr pointer 2643 * @sg: scatter gather list 2644 * @sg_nents: number of sg pages 2645 * @sg_offset: scatter gather list for fmr 2646 */ 2647 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2648 int sg_nents, unsigned int *sg_offset) 2649 { 2650 struct irdma_mr *iwmr = to_iwmr(ibmr); 2651 2652 iwmr->npages = 0; 2653 2654 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 2655 } 2656 2657 /** 2658 * irdma_hwreg_mr - send cqp command for memory registration 2659 * @iwdev: irdma device 2660 * @iwmr: irdma mr pointer 2661 * @access: access for MR 2662 */ 2663 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 2664 u16 access) 2665 { 2666 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2667 struct irdma_reg_ns_stag_info *stag_info; 2668 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2669 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2670 enum irdma_status_code status; 2671 int err = 0; 2672 struct irdma_cqp_request *cqp_request; 2673 struct cqp_cmds_info *cqp_info; 2674 2675 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2676 if (!cqp_request) 2677 return -ENOMEM; 2678 2679 cqp_info = &cqp_request->info; 2680 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 2681 memset(stag_info, 0, sizeof(*stag_info)); 2682 stag_info->va = iwpbl->user_base; 2683 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2684 stag_info->stag_key = (u8)iwmr->stag; 2685 stag_info->total_len = iwmr->len; 2686 stag_info->access_rights = irdma_get_mr_access(access); 2687 stag_info->pd_id = iwpd->sc_pd.pd_id; 2688 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 2689 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 2690 else 2691 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 2692 stag_info->page_size = iwmr->page_size; 2693 2694 if (iwpbl->pbl_allocated) { 2695 if (palloc->level == PBLE_LEVEL_1) { 2696 stag_info->first_pm_pbl_index = palloc->level1.idx; 2697 stag_info->chunk_size = 1; 2698 } else { 2699 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 2700 stag_info->chunk_size = 3; 2701 } 2702 } else { 2703 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 2704 } 2705 2706 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 2707 cqp_info->post_sq = 1; 2708 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 2709 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 2710 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2711 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2712 if (status) 2713 err = -ENOMEM; 2714 2715 return err; 2716 } 2717 2718 /** 2719 * irdma_reg_user_mr - Register a user memory region 2720 * @pd: ptr of pd 2721 * @start: virtual start address 2722 * @len: length of mr 2723 * @virt: virtual address 2724 * @access: access of mr 2725 * @udata: user data 2726 */ 2727 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 2728 u64 virt, int access, 2729 struct ib_udata *udata) 2730 { 2731 struct irdma_device *iwdev = to_iwdev(pd->device); 2732 struct irdma_ucontext *ucontext; 2733 struct irdma_pble_alloc *palloc; 2734 struct irdma_pbl *iwpbl; 2735 struct irdma_mr *iwmr; 2736 struct ib_umem *region; 2737 struct irdma_mem_reg_req req; 2738 u32 total, stag = 0; 2739 u8 shadow_pgcnt = 1; 2740 bool use_pbles = false; 2741 unsigned long flags; 2742 int err = -EINVAL; 2743 int ret; 2744 2745 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2746 return ERR_PTR(-EINVAL); 2747 2748 region = ib_umem_get(pd->device, start, len, access); 2749 2750 if (IS_ERR(region)) { 2751 ibdev_dbg(&iwdev->ibdev, 2752 "VERBS: Failed to create ib_umem region\n"); 2753 return (struct ib_mr *)region; 2754 } 2755 2756 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 2757 ib_umem_release(region); 2758 return ERR_PTR(-EFAULT); 2759 } 2760 2761 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2762 if (!iwmr) { 2763 ib_umem_release(region); 2764 return ERR_PTR(-ENOMEM); 2765 } 2766 2767 iwpbl = &iwmr->iwpbl; 2768 iwpbl->iwmr = iwmr; 2769 iwmr->region = region; 2770 iwmr->ibmr.pd = pd; 2771 iwmr->ibmr.device = pd->device; 2772 iwmr->ibmr.iova = virt; 2773 iwmr->page_size = PAGE_SIZE; 2774 2775 if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { 2776 iwmr->page_size = ib_umem_find_best_pgsz(region, 2777 SZ_4K | SZ_2M | SZ_1G, 2778 virt); 2779 if (unlikely(!iwmr->page_size)) { 2780 kfree(iwmr); 2781 ib_umem_release(region); 2782 return ERR_PTR(-EOPNOTSUPP); 2783 } 2784 } 2785 iwmr->len = region->length; 2786 iwpbl->user_base = virt; 2787 palloc = &iwpbl->pble_alloc; 2788 iwmr->type = req.reg_type; 2789 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2790 2791 switch (req.reg_type) { 2792 case IRDMA_MEMREG_TYPE_QP: 2793 total = req.sq_pages + req.rq_pages + shadow_pgcnt; 2794 if (total > iwmr->page_cnt) { 2795 err = -EINVAL; 2796 goto error; 2797 } 2798 total = req.sq_pages + req.rq_pages; 2799 use_pbles = (total > 2); 2800 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2801 if (err) 2802 goto error; 2803 2804 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2805 ibucontext); 2806 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2807 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2808 iwpbl->on_list = true; 2809 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2810 break; 2811 case IRDMA_MEMREG_TYPE_CQ: 2812 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2813 shadow_pgcnt = 0; 2814 total = req.cq_pages + shadow_pgcnt; 2815 if (total > iwmr->page_cnt) { 2816 err = -EINVAL; 2817 goto error; 2818 } 2819 2820 use_pbles = (req.cq_pages > 1); 2821 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2822 if (err) 2823 goto error; 2824 2825 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2826 ibucontext); 2827 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2828 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 2829 iwpbl->on_list = true; 2830 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2831 break; 2832 case IRDMA_MEMREG_TYPE_MEM: 2833 use_pbles = (iwmr->page_cnt != 1); 2834 2835 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2836 if (err) 2837 goto error; 2838 2839 if (use_pbles) { 2840 ret = irdma_check_mr_contiguous(palloc, 2841 iwmr->page_size); 2842 if (ret) { 2843 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2844 iwpbl->pbl_allocated = false; 2845 } 2846 } 2847 2848 stag = irdma_create_stag(iwdev); 2849 if (!stag) { 2850 err = -ENOMEM; 2851 goto error; 2852 } 2853 2854 iwmr->stag = stag; 2855 iwmr->ibmr.rkey = stag; 2856 iwmr->ibmr.lkey = stag; 2857 err = irdma_hwreg_mr(iwdev, iwmr, access); 2858 if (err) { 2859 irdma_free_stag(iwdev, stag); 2860 goto error; 2861 } 2862 2863 break; 2864 default: 2865 goto error; 2866 } 2867 2868 iwmr->type = req.reg_type; 2869 2870 return &iwmr->ibmr; 2871 2872 error: 2873 if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 2874 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2875 ib_umem_release(region); 2876 kfree(iwmr); 2877 2878 return ERR_PTR(err); 2879 } 2880 2881 /** 2882 * irdma_reg_phys_mr - register kernel physical memory 2883 * @pd: ibpd pointer 2884 * @addr: physical address of memory to register 2885 * @size: size of memory to register 2886 * @access: Access rights 2887 * @iova_start: start of virtual address for physical buffers 2888 */ 2889 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 2890 u64 *iova_start) 2891 { 2892 struct irdma_device *iwdev = to_iwdev(pd->device); 2893 struct irdma_pbl *iwpbl; 2894 struct irdma_mr *iwmr; 2895 enum irdma_status_code status; 2896 u32 stag; 2897 int ret; 2898 2899 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2900 if (!iwmr) 2901 return ERR_PTR(-ENOMEM); 2902 2903 iwmr->ibmr.pd = pd; 2904 iwmr->ibmr.device = pd->device; 2905 iwpbl = &iwmr->iwpbl; 2906 iwpbl->iwmr = iwmr; 2907 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2908 iwpbl->user_base = *iova_start; 2909 stag = irdma_create_stag(iwdev); 2910 if (!stag) { 2911 ret = -ENOMEM; 2912 goto err; 2913 } 2914 2915 iwmr->stag = stag; 2916 iwmr->ibmr.iova = *iova_start; 2917 iwmr->ibmr.rkey = stag; 2918 iwmr->ibmr.lkey = stag; 2919 iwmr->page_cnt = 1; 2920 iwmr->pgaddrmem[0] = addr; 2921 iwmr->len = size; 2922 iwmr->page_size = SZ_4K; 2923 status = irdma_hwreg_mr(iwdev, iwmr, access); 2924 if (status) { 2925 irdma_free_stag(iwdev, stag); 2926 ret = -ENOMEM; 2927 goto err; 2928 } 2929 2930 return &iwmr->ibmr; 2931 2932 err: 2933 kfree(iwmr); 2934 2935 return ERR_PTR(ret); 2936 } 2937 2938 /** 2939 * irdma_get_dma_mr - register physical mem 2940 * @pd: ptr of pd 2941 * @acc: access for memory 2942 */ 2943 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 2944 { 2945 u64 kva = 0; 2946 2947 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); 2948 } 2949 2950 /** 2951 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 2952 * @iwmr: iwmr for IB's user page addresses 2953 * @ucontext: ptr to user context 2954 */ 2955 static void irdma_del_memlist(struct irdma_mr *iwmr, 2956 struct irdma_ucontext *ucontext) 2957 { 2958 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2959 unsigned long flags; 2960 2961 switch (iwmr->type) { 2962 case IRDMA_MEMREG_TYPE_CQ: 2963 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2964 if (iwpbl->on_list) { 2965 iwpbl->on_list = false; 2966 list_del(&iwpbl->list); 2967 } 2968 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2969 break; 2970 case IRDMA_MEMREG_TYPE_QP: 2971 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2972 if (iwpbl->on_list) { 2973 iwpbl->on_list = false; 2974 list_del(&iwpbl->list); 2975 } 2976 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2977 break; 2978 default: 2979 break; 2980 } 2981 } 2982 2983 /** 2984 * irdma_dereg_mr - deregister mr 2985 * @ib_mr: mr ptr for dereg 2986 * @udata: user data 2987 */ 2988 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 2989 { 2990 struct ib_pd *ibpd = ib_mr->pd; 2991 struct irdma_pd *iwpd = to_iwpd(ibpd); 2992 struct irdma_mr *iwmr = to_iwmr(ib_mr); 2993 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 2994 struct irdma_dealloc_stag_info *info; 2995 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2996 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2997 struct irdma_cqp_request *cqp_request; 2998 struct cqp_cmds_info *cqp_info; 2999 3000 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3001 if (iwmr->region) { 3002 struct irdma_ucontext *ucontext; 3003 3004 ucontext = rdma_udata_to_drv_context(udata, 3005 struct irdma_ucontext, 3006 ibucontext); 3007 irdma_del_memlist(iwmr, ucontext); 3008 } 3009 goto done; 3010 } 3011 3012 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3013 if (!cqp_request) 3014 return -ENOMEM; 3015 3016 cqp_info = &cqp_request->info; 3017 info = &cqp_info->in.u.dealloc_stag.info; 3018 memset(info, 0, sizeof(*info)); 3019 info->pd_id = iwpd->sc_pd.pd_id & 0x00007fff; 3020 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3021 info->mr = true; 3022 if (iwpbl->pbl_allocated) 3023 info->dealloc_pbl = true; 3024 3025 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3026 cqp_info->post_sq = 1; 3027 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3028 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3029 irdma_handle_cqp_op(iwdev->rf, cqp_request); 3030 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3031 irdma_free_stag(iwdev, iwmr->stag); 3032 done: 3033 if (iwpbl->pbl_allocated) 3034 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3035 ib_umem_release(iwmr->region); 3036 kfree(iwmr); 3037 3038 return 0; 3039 } 3040 3041 /** 3042 * irdma_post_send - kernel application wr 3043 * @ibqp: qp ptr for wr 3044 * @ib_wr: work request ptr 3045 * @bad_wr: return of bad wr if err 3046 */ 3047 static int irdma_post_send(struct ib_qp *ibqp, 3048 const struct ib_send_wr *ib_wr, 3049 const struct ib_send_wr **bad_wr) 3050 { 3051 struct irdma_qp *iwqp; 3052 struct irdma_qp_uk *ukqp; 3053 struct irdma_sc_dev *dev; 3054 struct irdma_post_sq_info info; 3055 enum irdma_status_code ret; 3056 int err = 0; 3057 unsigned long flags; 3058 bool inv_stag; 3059 struct irdma_ah *ah; 3060 bool reflush = false; 3061 3062 iwqp = to_iwqp(ibqp); 3063 ukqp = &iwqp->sc_qp.qp_uk; 3064 dev = &iwqp->iwdev->rf->sc_dev; 3065 3066 spin_lock_irqsave(&iwqp->lock, flags); 3067 if (iwqp->flush_issued && ukqp->sq_flush_complete) 3068 reflush = true; 3069 while (ib_wr) { 3070 memset(&info, 0, sizeof(info)); 3071 inv_stag = false; 3072 info.wr_id = (ib_wr->wr_id); 3073 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 3074 info.signaled = true; 3075 if (ib_wr->send_flags & IB_SEND_FENCE) 3076 info.read_fence = true; 3077 switch (ib_wr->opcode) { 3078 case IB_WR_SEND_WITH_IMM: 3079 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 3080 info.imm_data_valid = true; 3081 info.imm_data = ntohl(ib_wr->ex.imm_data); 3082 } else { 3083 err = -EINVAL; 3084 break; 3085 } 3086 fallthrough; 3087 case IB_WR_SEND: 3088 case IB_WR_SEND_WITH_INV: 3089 if (ib_wr->opcode == IB_WR_SEND || 3090 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 3091 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3092 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 3093 else 3094 info.op_type = IRDMA_OP_TYPE_SEND; 3095 } else { 3096 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3097 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 3098 else 3099 info.op_type = IRDMA_OP_TYPE_SEND_INV; 3100 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 3101 } 3102 3103 if (ib_wr->send_flags & IB_SEND_INLINE) { 3104 info.op.inline_send.data = (void *)(unsigned long) 3105 ib_wr->sg_list[0].addr; 3106 info.op.inline_send.len = ib_wr->sg_list[0].length; 3107 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3108 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3109 ah = to_iwah(ud_wr(ib_wr)->ah); 3110 info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx; 3111 info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey; 3112 info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3113 } 3114 ret = irdma_uk_inline_send(ukqp, &info, false); 3115 } else { 3116 info.op.send.num_sges = ib_wr->num_sge; 3117 info.op.send.sg_list = ib_wr->sg_list; 3118 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3119 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3120 ah = to_iwah(ud_wr(ib_wr)->ah); 3121 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 3122 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 3123 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3124 } 3125 ret = irdma_uk_send(ukqp, &info, false); 3126 } 3127 3128 if (ret) { 3129 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3130 err = -ENOMEM; 3131 else 3132 err = -EINVAL; 3133 } 3134 break; 3135 case IB_WR_RDMA_WRITE_WITH_IMM: 3136 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 3137 info.imm_data_valid = true; 3138 info.imm_data = ntohl(ib_wr->ex.imm_data); 3139 } else { 3140 err = -EINVAL; 3141 break; 3142 } 3143 fallthrough; 3144 case IB_WR_RDMA_WRITE: 3145 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3146 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 3147 else 3148 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 3149 3150 if (ib_wr->send_flags & IB_SEND_INLINE) { 3151 info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr; 3152 info.op.inline_rdma_write.len = 3153 ib_wr->sg_list[0].length; 3154 info.op.inline_rdma_write.rem_addr.addr = 3155 rdma_wr(ib_wr)->remote_addr; 3156 info.op.inline_rdma_write.rem_addr.lkey = 3157 rdma_wr(ib_wr)->rkey; 3158 ret = irdma_uk_inline_rdma_write(ukqp, &info, false); 3159 } else { 3160 info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; 3161 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 3162 info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3163 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3164 ret = irdma_uk_rdma_write(ukqp, &info, false); 3165 } 3166 3167 if (ret) { 3168 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3169 err = -ENOMEM; 3170 else 3171 err = -EINVAL; 3172 } 3173 break; 3174 case IB_WR_RDMA_READ_WITH_INV: 3175 inv_stag = true; 3176 fallthrough; 3177 case IB_WR_RDMA_READ: 3178 if (ib_wr->num_sge > 3179 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 3180 err = -EINVAL; 3181 break; 3182 } 3183 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 3184 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3185 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3186 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 3187 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 3188 3189 ret = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 3190 if (ret) { 3191 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3192 err = -ENOMEM; 3193 else 3194 err = -EINVAL; 3195 } 3196 break; 3197 case IB_WR_LOCAL_INV: 3198 info.op_type = IRDMA_OP_TYPE_INV_STAG; 3199 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 3200 ret = irdma_uk_stag_local_invalidate(ukqp, &info, true); 3201 if (ret) 3202 err = -ENOMEM; 3203 break; 3204 case IB_WR_REG_MR: { 3205 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 3206 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 3207 struct irdma_fast_reg_stag_info stag_info = {}; 3208 3209 stag_info.signaled = info.signaled; 3210 stag_info.read_fence = info.read_fence; 3211 stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); 3212 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 3213 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 3214 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 3215 stag_info.wr_id = ib_wr->wr_id; 3216 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3217 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 3218 stag_info.total_len = iwmr->ibmr.length; 3219 stag_info.reg_addr_pa = *palloc->level1.addr; 3220 stag_info.first_pm_pbl_index = palloc->level1.idx; 3221 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 3222 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 3223 stag_info.chunk_size = 1; 3224 ret = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 3225 true); 3226 if (ret) 3227 err = -ENOMEM; 3228 break; 3229 } 3230 default: 3231 err = -EINVAL; 3232 ibdev_dbg(&iwqp->iwdev->ibdev, 3233 "VERBS: upost_send bad opcode = 0x%x\n", 3234 ib_wr->opcode); 3235 break; 3236 } 3237 3238 if (err) 3239 break; 3240 ib_wr = ib_wr->next; 3241 } 3242 3243 if (!iwqp->flush_issued && iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) { 3244 irdma_uk_qp_post_wr(ukqp); 3245 spin_unlock_irqrestore(&iwqp->lock, flags); 3246 } else if (reflush) { 3247 ukqp->sq_flush_complete = false; 3248 spin_unlock_irqrestore(&iwqp->lock, flags); 3249 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_REFLUSH); 3250 } else { 3251 spin_unlock_irqrestore(&iwqp->lock, flags); 3252 } 3253 if (err) 3254 *bad_wr = ib_wr; 3255 3256 return err; 3257 } 3258 3259 /** 3260 * irdma_post_recv - post receive wr for kernel application 3261 * @ibqp: ib qp pointer 3262 * @ib_wr: work request for receive 3263 * @bad_wr: bad wr caused an error 3264 */ 3265 static int irdma_post_recv(struct ib_qp *ibqp, 3266 const struct ib_recv_wr *ib_wr, 3267 const struct ib_recv_wr **bad_wr) 3268 { 3269 struct irdma_qp *iwqp; 3270 struct irdma_qp_uk *ukqp; 3271 struct irdma_post_rq_info post_recv = {}; 3272 enum irdma_status_code ret = 0; 3273 unsigned long flags; 3274 int err = 0; 3275 bool reflush = false; 3276 3277 iwqp = to_iwqp(ibqp); 3278 ukqp = &iwqp->sc_qp.qp_uk; 3279 3280 spin_lock_irqsave(&iwqp->lock, flags); 3281 if (iwqp->flush_issued && ukqp->rq_flush_complete) 3282 reflush = true; 3283 while (ib_wr) { 3284 post_recv.num_sges = ib_wr->num_sge; 3285 post_recv.wr_id = ib_wr->wr_id; 3286 post_recv.sg_list = ib_wr->sg_list; 3287 ret = irdma_uk_post_receive(ukqp, &post_recv); 3288 if (ret) { 3289 ibdev_dbg(&iwqp->iwdev->ibdev, 3290 "VERBS: post_recv err %d\n", ret); 3291 if (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) 3292 err = -ENOMEM; 3293 else 3294 err = -EINVAL; 3295 goto out; 3296 } 3297 3298 ib_wr = ib_wr->next; 3299 } 3300 3301 out: 3302 if (reflush) { 3303 ukqp->rq_flush_complete = false; 3304 spin_unlock_irqrestore(&iwqp->lock, flags); 3305 irdma_flush_wqes(iwqp, IRDMA_FLUSH_RQ | IRDMA_REFLUSH); 3306 } else { 3307 spin_unlock_irqrestore(&iwqp->lock, flags); 3308 } 3309 3310 if (err) 3311 *bad_wr = ib_wr; 3312 3313 return err; 3314 } 3315 3316 /** 3317 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 3318 * @opcode: iwarp flush code 3319 */ 3320 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 3321 { 3322 switch (opcode) { 3323 case FLUSH_PROT_ERR: 3324 return IB_WC_LOC_PROT_ERR; 3325 case FLUSH_REM_ACCESS_ERR: 3326 return IB_WC_REM_ACCESS_ERR; 3327 case FLUSH_LOC_QP_OP_ERR: 3328 return IB_WC_LOC_QP_OP_ERR; 3329 case FLUSH_REM_OP_ERR: 3330 return IB_WC_REM_OP_ERR; 3331 case FLUSH_LOC_LEN_ERR: 3332 return IB_WC_LOC_LEN_ERR; 3333 case FLUSH_GENERAL_ERR: 3334 return IB_WC_WR_FLUSH_ERR; 3335 case FLUSH_RETRY_EXC_ERR: 3336 return IB_WC_RETRY_EXC_ERR; 3337 case FLUSH_MW_BIND_ERR: 3338 return IB_WC_MW_BIND_ERR; 3339 case FLUSH_FATAL_ERR: 3340 default: 3341 return IB_WC_FATAL_ERR; 3342 } 3343 } 3344 3345 /** 3346 * irdma_process_cqe - process cqe info 3347 * @entry: processed cqe 3348 * @cq_poll_info: cqe info 3349 */ 3350 static void irdma_process_cqe(struct ib_wc *entry, 3351 struct irdma_cq_poll_info *cq_poll_info) 3352 { 3353 struct irdma_qp *iwqp; 3354 struct irdma_sc_qp *qp; 3355 3356 entry->wc_flags = 0; 3357 entry->pkey_index = 0; 3358 entry->wr_id = cq_poll_info->wr_id; 3359 3360 qp = cq_poll_info->qp_handle; 3361 iwqp = qp->qp_uk.back_qp; 3362 entry->qp = qp->qp_uk.back_qp; 3363 3364 if (cq_poll_info->error) { 3365 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 3366 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 3367 3368 entry->vendor_err = cq_poll_info->major_err << 16 | 3369 cq_poll_info->minor_err; 3370 } else { 3371 entry->status = IB_WC_SUCCESS; 3372 if (cq_poll_info->imm_valid) { 3373 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 3374 entry->wc_flags |= IB_WC_WITH_IMM; 3375 } 3376 if (cq_poll_info->ud_smac_valid) { 3377 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 3378 entry->wc_flags |= IB_WC_WITH_SMAC; 3379 } 3380 3381 if (cq_poll_info->ud_vlan_valid) { 3382 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 3383 3384 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 3385 if (vlan) { 3386 entry->vlan_id = vlan; 3387 entry->wc_flags |= IB_WC_WITH_VLAN; 3388 } 3389 } else { 3390 entry->sl = 0; 3391 } 3392 } 3393 3394 switch (cq_poll_info->op_type) { 3395 case IRDMA_OP_TYPE_RDMA_WRITE: 3396 case IRDMA_OP_TYPE_RDMA_WRITE_SOL: 3397 entry->opcode = IB_WC_RDMA_WRITE; 3398 break; 3399 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: 3400 case IRDMA_OP_TYPE_RDMA_READ: 3401 entry->opcode = IB_WC_RDMA_READ; 3402 break; 3403 case IRDMA_OP_TYPE_SEND_INV: 3404 case IRDMA_OP_TYPE_SEND_SOL: 3405 case IRDMA_OP_TYPE_SEND_SOL_INV: 3406 case IRDMA_OP_TYPE_SEND: 3407 entry->opcode = IB_WC_SEND; 3408 break; 3409 case IRDMA_OP_TYPE_FAST_REG_NSMR: 3410 entry->opcode = IB_WC_REG_MR; 3411 break; 3412 case IRDMA_OP_TYPE_INV_STAG: 3413 entry->opcode = IB_WC_LOCAL_INV; 3414 break; 3415 case IRDMA_OP_TYPE_REC_IMM: 3416 case IRDMA_OP_TYPE_REC: 3417 entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ? 3418 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; 3419 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 3420 cq_poll_info->stag_invalid_set) { 3421 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 3422 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 3423 } 3424 break; 3425 default: 3426 ibdev_err(&iwqp->iwdev->ibdev, 3427 "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); 3428 entry->status = IB_WC_GENERAL_ERR; 3429 return; 3430 } 3431 3432 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 3433 entry->src_qp = cq_poll_info->ud_src_qpn; 3434 entry->slid = 0; 3435 entry->wc_flags |= 3436 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 3437 entry->network_hdr_type = cq_poll_info->ipv4 ? 3438 RDMA_NETWORK_IPV4 : 3439 RDMA_NETWORK_IPV6; 3440 } else { 3441 entry->src_qp = cq_poll_info->qp_id; 3442 } 3443 3444 entry->byte_len = cq_poll_info->bytes_xfered; 3445 } 3446 3447 /** 3448 * irdma_poll_one - poll one entry of the CQ 3449 * @ukcq: ukcq to poll 3450 * @cur_cqe: current CQE info to be filled in 3451 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 3452 * 3453 * Returns the internal irdma device error code or 0 on success 3454 */ 3455 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 3456 struct irdma_cq_poll_info *cur_cqe, 3457 struct ib_wc *entry) 3458 { 3459 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 3460 3461 if (ret) 3462 return ret; 3463 3464 irdma_process_cqe(entry, cur_cqe); 3465 3466 return 0; 3467 } 3468 3469 /** 3470 * __irdma_poll_cq - poll cq for completion (kernel apps) 3471 * @iwcq: cq to poll 3472 * @num_entries: number of entries to poll 3473 * @entry: wr of a completed entry 3474 */ 3475 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 3476 { 3477 struct list_head *tmp_node, *list_node; 3478 struct irdma_cq_buf *last_buf = NULL; 3479 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 3480 struct irdma_cq_buf *cq_buf; 3481 enum irdma_status_code ret; 3482 struct irdma_device *iwdev; 3483 struct irdma_cq_uk *ukcq; 3484 bool cq_new_cqe = false; 3485 int resized_bufs = 0; 3486 int npolled = 0; 3487 3488 iwdev = to_iwdev(iwcq->ibcq.device); 3489 ukcq = &iwcq->sc_cq.cq_uk; 3490 3491 /* go through the list of previously resized CQ buffers */ 3492 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 3493 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 3494 while (npolled < num_entries) { 3495 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 3496 if (!ret) { 3497 ++npolled; 3498 cq_new_cqe = true; 3499 continue; 3500 } 3501 if (ret == IRDMA_ERR_Q_EMPTY) 3502 break; 3503 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3504 if (ret == IRDMA_ERR_Q_DESTROYED) { 3505 cq_new_cqe = true; 3506 continue; 3507 } 3508 goto error; 3509 } 3510 3511 /* save the resized CQ buffer which received the last cqe */ 3512 if (cq_new_cqe) 3513 last_buf = cq_buf; 3514 cq_new_cqe = false; 3515 } 3516 3517 /* check the current CQ for new cqes */ 3518 while (npolled < num_entries) { 3519 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 3520 if (!ret) { 3521 ++npolled; 3522 cq_new_cqe = true; 3523 continue; 3524 } 3525 3526 if (ret == IRDMA_ERR_Q_EMPTY) 3527 break; 3528 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3529 if (ret == IRDMA_ERR_Q_DESTROYED) { 3530 cq_new_cqe = true; 3531 continue; 3532 } 3533 goto error; 3534 } 3535 3536 if (cq_new_cqe) 3537 /* all previous CQ resizes are complete */ 3538 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 3539 else if (last_buf) 3540 /* only CQ resizes up to the last_buf are complete */ 3541 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 3542 if (resized_bufs) 3543 /* report to the HW the number of complete CQ resizes */ 3544 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 3545 3546 return npolled; 3547 error: 3548 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 3549 __func__, ret); 3550 3551 return -EINVAL; 3552 } 3553 3554 /** 3555 * irdma_poll_cq - poll cq for completion (kernel apps) 3556 * @ibcq: cq to poll 3557 * @num_entries: number of entries to poll 3558 * @entry: wr of a completed entry 3559 */ 3560 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 3561 struct ib_wc *entry) 3562 { 3563 struct irdma_cq *iwcq; 3564 unsigned long flags; 3565 int ret; 3566 3567 iwcq = to_iwcq(ibcq); 3568 3569 spin_lock_irqsave(&iwcq->lock, flags); 3570 ret = __irdma_poll_cq(iwcq, num_entries, entry); 3571 spin_unlock_irqrestore(&iwcq->lock, flags); 3572 3573 return ret; 3574 } 3575 3576 /** 3577 * irdma_req_notify_cq - arm cq kernel application 3578 * @ibcq: cq to arm 3579 * @notify_flags: notofication flags 3580 */ 3581 static int irdma_req_notify_cq(struct ib_cq *ibcq, 3582 enum ib_cq_notify_flags notify_flags) 3583 { 3584 struct irdma_cq *iwcq; 3585 struct irdma_cq_uk *ukcq; 3586 unsigned long flags; 3587 enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT; 3588 3589 iwcq = to_iwcq(ibcq); 3590 ukcq = &iwcq->sc_cq.cq_uk; 3591 if (notify_flags == IB_CQ_SOLICITED) 3592 cq_notify = IRDMA_CQ_COMPL_SOLICITED; 3593 3594 spin_lock_irqsave(&iwcq->lock, flags); 3595 irdma_uk_cq_request_notification(ukcq, cq_notify); 3596 spin_unlock_irqrestore(&iwcq->lock, flags); 3597 3598 return 0; 3599 } 3600 3601 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 3602 struct ib_port_immutable *immutable) 3603 { 3604 struct ib_port_attr attr; 3605 int err; 3606 3607 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3608 err = ib_query_port(ibdev, port_num, &attr); 3609 if (err) 3610 return err; 3611 3612 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3613 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3614 immutable->gid_tbl_len = attr.gid_tbl_len; 3615 3616 return 0; 3617 } 3618 3619 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 3620 struct ib_port_immutable *immutable) 3621 { 3622 struct ib_port_attr attr; 3623 int err; 3624 3625 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 3626 err = ib_query_port(ibdev, port_num, &attr); 3627 if (err) 3628 return err; 3629 immutable->gid_tbl_len = attr.gid_tbl_len; 3630 3631 return 0; 3632 } 3633 3634 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 3635 /* 32bit names */ 3636 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 3637 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 3638 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 3639 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 3640 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 3641 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 3642 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 3643 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs", 3644 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors", 3645 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors", 3646 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 3647 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 3648 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 3649 3650 /* 64bit names */ 3651 [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3652 "ip4InOctets", 3653 [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3654 "ip4InPkts", 3655 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3656 "ip4InReasmRqd", 3657 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3658 "ip4InMcastOctets", 3659 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3660 "ip4InMcastPkts", 3661 [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3662 "ip4OutOctets", 3663 [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3664 "ip4OutPkts", 3665 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3666 "ip4OutSegRqd", 3667 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3668 "ip4OutMcastOctets", 3669 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3670 "ip4OutMcastPkts", 3671 [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3672 "ip6InOctets", 3673 [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3674 "ip6InPkts", 3675 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3676 "ip6InReasmRqd", 3677 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3678 "ip6InMcastOctets", 3679 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3680 "ip6InMcastPkts", 3681 [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3682 "ip6OutOctets", 3683 [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3684 "ip6OutPkts", 3685 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3686 "ip6OutSegRqd", 3687 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3688 "ip6OutMcastOctets", 3689 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3690 "ip6OutMcastPkts", 3691 [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3692 "tcpInSegs", 3693 [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32].name = 3694 "tcpOutSegs", 3695 [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3696 "iwInRdmaReads", 3697 [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3698 "iwInRdmaSends", 3699 [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3700 "iwInRdmaWrites", 3701 [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3702 "iwOutRdmaReads", 3703 [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3704 "iwOutRdmaSends", 3705 [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3706 "iwOutRdmaWrites", 3707 [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32].name = 3708 "iwRdmaBnd", 3709 [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32].name = 3710 "iwRdmaInv", 3711 [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3712 "RxUDP", 3713 [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3714 "TxUDP", 3715 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] 3716 .name = "RxECNMrkd", 3717 }; 3718 3719 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 3720 { 3721 struct irdma_device *iwdev = to_iwdev(dev); 3722 3723 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 3724 irdma_fw_major_ver(&iwdev->rf->sc_dev), 3725 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 3726 } 3727 3728 /** 3729 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 3730 * @ibdev: device pointer from stack 3731 * @port_num: port number 3732 */ 3733 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 3734 u32 port_num) 3735 { 3736 int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 + 3737 IRDMA_HW_STAT_INDEX_MAX_64; 3738 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 3739 3740 BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_descs) != 3741 (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64)); 3742 3743 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 3744 lifespan); 3745 } 3746 3747 /** 3748 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 3749 * @ibdev: device pointer from stack 3750 * @stats: stats pointer from stack 3751 * @port_num: port number 3752 * @index: which hw counter the stack is requesting we update 3753 */ 3754 static int irdma_get_hw_stats(struct ib_device *ibdev, 3755 struct rdma_hw_stats *stats, u32 port_num, 3756 int index) 3757 { 3758 struct irdma_device *iwdev = to_iwdev(ibdev); 3759 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 3760 3761 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 3762 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 3763 else 3764 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 3765 3766 memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); 3767 3768 return stats->num_counters; 3769 } 3770 3771 /** 3772 * irdma_query_gid - Query port GID 3773 * @ibdev: device pointer from stack 3774 * @port: port number 3775 * @index: Entry index 3776 * @gid: Global ID 3777 */ 3778 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 3779 union ib_gid *gid) 3780 { 3781 struct irdma_device *iwdev = to_iwdev(ibdev); 3782 3783 memset(gid->raw, 0, sizeof(gid->raw)); 3784 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 3785 3786 return 0; 3787 } 3788 3789 /** 3790 * mcast_list_add - Add a new mcast item to list 3791 * @rf: RDMA PCI function 3792 * @new_elem: pointer to element to add 3793 */ 3794 static void mcast_list_add(struct irdma_pci_f *rf, 3795 struct mc_table_list *new_elem) 3796 { 3797 list_add(&new_elem->list, &rf->mc_qht_list.list); 3798 } 3799 3800 /** 3801 * mcast_list_del - Remove an mcast item from list 3802 * @mc_qht_elem: pointer to mcast table list element 3803 */ 3804 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 3805 { 3806 if (mc_qht_elem) 3807 list_del(&mc_qht_elem->list); 3808 } 3809 3810 /** 3811 * mcast_list_lookup_ip - Search mcast list for address 3812 * @rf: RDMA PCI function 3813 * @ip_mcast: pointer to mcast IP address 3814 */ 3815 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 3816 u32 *ip_mcast) 3817 { 3818 struct mc_table_list *mc_qht_el; 3819 struct list_head *pos, *q; 3820 3821 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 3822 mc_qht_el = list_entry(pos, struct mc_table_list, list); 3823 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 3824 sizeof(mc_qht_el->mc_info.dest_ip))) 3825 return mc_qht_el; 3826 } 3827 3828 return NULL; 3829 } 3830 3831 /** 3832 * irdma_mcast_cqp_op - perform a mcast cqp operation 3833 * @iwdev: irdma device 3834 * @mc_grp_ctx: mcast group info 3835 * @op: operation 3836 * 3837 * returns error status 3838 */ 3839 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 3840 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 3841 { 3842 struct cqp_cmds_info *cqp_info; 3843 struct irdma_cqp_request *cqp_request; 3844 enum irdma_status_code status; 3845 3846 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3847 if (!cqp_request) 3848 return -ENOMEM; 3849 3850 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 3851 cqp_info = &cqp_request->info; 3852 cqp_info->cqp_cmd = op; 3853 cqp_info->post_sq = 1; 3854 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 3855 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 3856 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3857 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3858 if (status) 3859 return -ENOMEM; 3860 3861 return 0; 3862 } 3863 3864 /** 3865 * irdma_mcast_mac - Get the multicast MAC for an IP address 3866 * @ip_addr: IPv4 or IPv6 address 3867 * @mac: pointer to result MAC address 3868 * @ipv4: flag indicating IPv4 or IPv6 3869 * 3870 */ 3871 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 3872 { 3873 u8 *ip = (u8 *)ip_addr; 3874 3875 if (ipv4) { 3876 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 3877 0x00, 0x00}; 3878 3879 mac4[3] = ip[2] & 0x7F; 3880 mac4[4] = ip[1]; 3881 mac4[5] = ip[0]; 3882 ether_addr_copy(mac, mac4); 3883 } else { 3884 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 3885 0x00, 0x00}; 3886 3887 mac6[2] = ip[3]; 3888 mac6[3] = ip[2]; 3889 mac6[4] = ip[1]; 3890 mac6[5] = ip[0]; 3891 ether_addr_copy(mac, mac6); 3892 } 3893 } 3894 3895 /** 3896 * irdma_attach_mcast - attach a qp to a multicast group 3897 * @ibqp: ptr to qp 3898 * @ibgid: pointer to global ID 3899 * @lid: local ID 3900 * 3901 * returns error status 3902 */ 3903 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 3904 { 3905 struct irdma_qp *iwqp = to_iwqp(ibqp); 3906 struct irdma_device *iwdev = iwqp->iwdev; 3907 struct irdma_pci_f *rf = iwdev->rf; 3908 struct mc_table_list *mc_qht_elem; 3909 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 3910 unsigned long flags; 3911 u32 ip_addr[4] = {}; 3912 u32 mgn; 3913 u32 no_mgs; 3914 int ret = 0; 3915 bool ipv4; 3916 u16 vlan_id; 3917 union { 3918 struct sockaddr saddr; 3919 struct sockaddr_in saddr_in; 3920 struct sockaddr_in6 saddr_in6; 3921 } sgid_addr; 3922 unsigned char dmac[ETH_ALEN]; 3923 3924 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 3925 3926 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 3927 irdma_copy_ip_ntohl(ip_addr, 3928 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 3929 irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); 3930 ipv4 = false; 3931 ibdev_dbg(&iwdev->ibdev, 3932 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 3933 ip_addr); 3934 irdma_mcast_mac(ip_addr, dmac, false); 3935 } else { 3936 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 3937 ipv4 = true; 3938 vlan_id = irdma_get_vlan_ipv4(ip_addr); 3939 irdma_mcast_mac(ip_addr, dmac, true); 3940 ibdev_dbg(&iwdev->ibdev, 3941 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 3942 ibqp->qp_num, ip_addr, dmac); 3943 } 3944 3945 spin_lock_irqsave(&rf->qh_list_lock, flags); 3946 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 3947 if (!mc_qht_elem) { 3948 struct irdma_dma_mem *dma_mem_mc; 3949 3950 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3951 mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); 3952 if (!mc_qht_elem) 3953 return -ENOMEM; 3954 3955 mc_qht_elem->mc_info.ipv4_valid = ipv4; 3956 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 3957 sizeof(mc_qht_elem->mc_info.dest_ip)); 3958 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 3959 &mgn, &rf->next_mcg); 3960 if (ret) { 3961 kfree(mc_qht_elem); 3962 return -ENOMEM; 3963 } 3964 3965 mc_qht_elem->mc_info.mgn = mgn; 3966 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 3967 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 3968 IRDMA_HW_PAGE_SIZE); 3969 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 3970 dma_mem_mc->size, 3971 &dma_mem_mc->pa, 3972 GFP_KERNEL); 3973 if (!dma_mem_mc->va) { 3974 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 3975 kfree(mc_qht_elem); 3976 return -ENOMEM; 3977 } 3978 3979 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 3980 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 3981 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 3982 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 3983 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 3984 if (vlan_id < VLAN_N_VID) 3985 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 3986 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id; 3987 mc_qht_elem->mc_grp_ctx.qs_handle = 3988 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 3989 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 3990 3991 spin_lock_irqsave(&rf->qh_list_lock, flags); 3992 mcast_list_add(rf, mc_qht_elem); 3993 } else { 3994 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 3995 IRDMA_MAX_MGS_PER_CTX) { 3996 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3997 return -ENOMEM; 3998 } 3999 } 4000 4001 mcg_info.qp_id = iwqp->ibqp.qp_num; 4002 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 4003 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4004 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4005 4006 /* Only if there is a change do we need to modify or create */ 4007 if (!no_mgs) { 4008 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4009 IRDMA_OP_MC_CREATE); 4010 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4011 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4012 IRDMA_OP_MC_MODIFY); 4013 } else { 4014 return 0; 4015 } 4016 4017 if (ret) 4018 goto error; 4019 4020 return 0; 4021 4022 error: 4023 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4024 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4025 mcast_list_del(mc_qht_elem); 4026 dma_free_coherent(rf->hw.device, 4027 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4028 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4029 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4030 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4031 irdma_free_rsrc(rf, rf->allocated_mcgs, 4032 mc_qht_elem->mc_grp_ctx.mg_id); 4033 kfree(mc_qht_elem); 4034 } 4035 4036 return ret; 4037 } 4038 4039 /** 4040 * irdma_detach_mcast - detach a qp from a multicast group 4041 * @ibqp: ptr to qp 4042 * @ibgid: pointer to global ID 4043 * @lid: local ID 4044 * 4045 * returns error status 4046 */ 4047 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4048 { 4049 struct irdma_qp *iwqp = to_iwqp(ibqp); 4050 struct irdma_device *iwdev = iwqp->iwdev; 4051 struct irdma_pci_f *rf = iwdev->rf; 4052 u32 ip_addr[4] = {}; 4053 struct mc_table_list *mc_qht_elem; 4054 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4055 int ret; 4056 unsigned long flags; 4057 union { 4058 struct sockaddr saddr; 4059 struct sockaddr_in saddr_in; 4060 struct sockaddr_in6 saddr_in6; 4061 } sgid_addr; 4062 4063 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4064 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 4065 irdma_copy_ip_ntohl(ip_addr, 4066 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4067 else 4068 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4069 4070 spin_lock_irqsave(&rf->qh_list_lock, flags); 4071 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4072 if (!mc_qht_elem) { 4073 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4074 ibdev_dbg(&iwdev->ibdev, 4075 "VERBS: address not found MCG\n"); 4076 return 0; 4077 } 4078 4079 mcg_info.qp_id = iwqp->ibqp.qp_num; 4080 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4081 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4082 mcast_list_del(mc_qht_elem); 4083 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4084 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4085 IRDMA_OP_MC_DESTROY); 4086 if (ret) { 4087 ibdev_dbg(&iwdev->ibdev, 4088 "VERBS: failed MC_DESTROY MCG\n"); 4089 spin_lock_irqsave(&rf->qh_list_lock, flags); 4090 mcast_list_add(rf, mc_qht_elem); 4091 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4092 return -EAGAIN; 4093 } 4094 4095 dma_free_coherent(rf->hw.device, 4096 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4097 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4098 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4099 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4100 irdma_free_rsrc(rf, rf->allocated_mcgs, 4101 mc_qht_elem->mc_grp_ctx.mg_id); 4102 kfree(mc_qht_elem); 4103 } else { 4104 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4105 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4106 IRDMA_OP_MC_MODIFY); 4107 if (ret) { 4108 ibdev_dbg(&iwdev->ibdev, 4109 "VERBS: failed Modify MCG\n"); 4110 return ret; 4111 } 4112 } 4113 4114 return 0; 4115 } 4116 4117 /** 4118 * irdma_create_ah - create address handle 4119 * @ibah: address handle 4120 * @attr: address handle attributes 4121 * @udata: User data 4122 * 4123 * returns 0 on success, error otherwise 4124 */ 4125 static int irdma_create_ah(struct ib_ah *ibah, 4126 struct rdma_ah_init_attr *attr, 4127 struct ib_udata *udata) 4128 { 4129 struct irdma_pd *pd = to_iwpd(ibah->pd); 4130 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4131 struct rdma_ah_attr *ah_attr = attr->ah_attr; 4132 const struct ib_gid_attr *sgid_attr; 4133 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4134 struct irdma_pci_f *rf = iwdev->rf; 4135 struct irdma_sc_ah *sc_ah; 4136 u32 ah_id = 0; 4137 struct irdma_ah_info *ah_info; 4138 struct irdma_create_ah_resp uresp; 4139 union { 4140 struct sockaddr saddr; 4141 struct sockaddr_in saddr_in; 4142 struct sockaddr_in6 saddr_in6; 4143 } sgid_addr, dgid_addr; 4144 int err; 4145 u8 dmac[ETH_ALEN]; 4146 4147 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah_id, 4148 &rf->next_ah); 4149 if (err) 4150 return err; 4151 4152 ah->pd = pd; 4153 sc_ah = &ah->sc_ah; 4154 sc_ah->ah_info.ah_idx = ah_id; 4155 sc_ah->ah_info.vsi = &iwdev->vsi; 4156 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 4157 ah->sgid_index = ah_attr->grh.sgid_index; 4158 sgid_attr = ah_attr->grh.sgid_attr; 4159 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 4160 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 4161 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 4162 ah->av.attrs = *ah_attr; 4163 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 4164 ah->av.sgid_addr.saddr = sgid_addr.saddr; 4165 ah->av.dgid_addr.saddr = dgid_addr.saddr; 4166 ah_info = &sc_ah->ah_info; 4167 ah_info->ah_idx = ah_id; 4168 ah_info->pd_idx = pd->sc_pd.pd_id; 4169 if (ah_attr->ah_flags & IB_AH_GRH) { 4170 ah_info->flow_label = ah_attr->grh.flow_label; 4171 ah_info->hop_ttl = ah_attr->grh.hop_limit; 4172 ah_info->tc_tos = ah_attr->grh.traffic_class; 4173 } 4174 4175 ether_addr_copy(dmac, ah_attr->roce.dmac); 4176 if (rdma_gid_attr_network_type(sgid_attr) == RDMA_NETWORK_IPV4) { 4177 ah_info->ipv4_valid = true; 4178 ah_info->dest_ip_addr[0] = 4179 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 4180 ah_info->src_ip_addr[0] = 4181 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4182 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 4183 ah_info->dest_ip_addr[0]); 4184 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 4185 ah_info->do_lpbk = true; 4186 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 4187 } 4188 } else { 4189 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 4190 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4191 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 4192 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4193 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 4194 ah_info->dest_ip_addr); 4195 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 4196 ah_info->do_lpbk = true; 4197 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 4198 } 4199 } 4200 4201 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 4202 ah_info->mac_addr); 4203 if (err) 4204 goto error; 4205 4206 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 4207 ah_info->ipv4_valid, dmac); 4208 4209 if (ah_info->dst_arpindex == -1) { 4210 err = -EINVAL; 4211 goto error; 4212 } 4213 4214 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb) 4215 ah_info->vlan_tag = 0; 4216 4217 if (ah_info->vlan_tag < VLAN_N_VID) { 4218 ah_info->insert_vlan_tag = true; 4219 ah_info->vlan_tag |= 4220 rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; 4221 } 4222 4223 err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, 4224 attr->flags & RDMA_CREATE_AH_SLEEPABLE, 4225 irdma_gsi_ud_qp_ah_cb, sc_ah); 4226 4227 if (err) { 4228 ibdev_dbg(&iwdev->ibdev, 4229 "VERBS: CQP-OP Create AH fail"); 4230 goto error; 4231 } 4232 4233 if (!(attr->flags & RDMA_CREATE_AH_SLEEPABLE)) { 4234 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 4235 4236 do { 4237 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 4238 mdelay(1); 4239 } while (!sc_ah->ah_info.ah_valid && --cnt); 4240 4241 if (!cnt) { 4242 ibdev_dbg(&iwdev->ibdev, 4243 "VERBS: CQP create AH timed out"); 4244 err = -ETIMEDOUT; 4245 goto error; 4246 } 4247 } 4248 4249 if (udata) { 4250 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 4251 err = ib_copy_to_udata(udata, &uresp, 4252 min(sizeof(uresp), udata->outlen)); 4253 } 4254 return 0; 4255 4256 error: 4257 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id); 4258 4259 return err; 4260 } 4261 4262 /** 4263 * irdma_destroy_ah - Destroy address handle 4264 * @ibah: pointer to address handle 4265 * @ah_flags: flags for sleepable 4266 */ 4267 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 4268 { 4269 struct irdma_device *iwdev = to_iwdev(ibah->device); 4270 struct irdma_ah *ah = to_iwah(ibah); 4271 4272 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 4273 false, NULL, ah); 4274 4275 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 4276 ah->sc_ah.ah_info.ah_idx); 4277 4278 return 0; 4279 } 4280 4281 /** 4282 * irdma_query_ah - Query address handle 4283 * @ibah: pointer to address handle 4284 * @ah_attr: address handle attributes 4285 */ 4286 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 4287 { 4288 struct irdma_ah *ah = to_iwah(ibah); 4289 4290 memset(ah_attr, 0, sizeof(*ah_attr)); 4291 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 4292 ah_attr->ah_flags = IB_AH_GRH; 4293 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 4294 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 4295 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 4296 ah_attr->grh.sgid_index = ah->sgid_index; 4297 ah_attr->grh.sgid_index = ah->sgid_index; 4298 memcpy(&ah_attr->grh.dgid, &ah->dgid, 4299 sizeof(ah_attr->grh.dgid)); 4300 } 4301 4302 return 0; 4303 } 4304 4305 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 4306 u32 port_num) 4307 { 4308 return IB_LINK_LAYER_ETHERNET; 4309 } 4310 4311 static __be64 irdma_mac_to_guid(struct net_device *ndev) 4312 { 4313 const unsigned char *mac = ndev->dev_addr; 4314 __be64 guid; 4315 unsigned char *dst = (unsigned char *)&guid; 4316 4317 dst[0] = mac[0] ^ 2; 4318 dst[1] = mac[1]; 4319 dst[2] = mac[2]; 4320 dst[3] = 0xff; 4321 dst[4] = 0xfe; 4322 dst[5] = mac[3]; 4323 dst[6] = mac[4]; 4324 dst[7] = mac[5]; 4325 4326 return guid; 4327 } 4328 4329 static const struct ib_device_ops irdma_roce_dev_ops = { 4330 .attach_mcast = irdma_attach_mcast, 4331 .create_ah = irdma_create_ah, 4332 .create_user_ah = irdma_create_ah, 4333 .destroy_ah = irdma_destroy_ah, 4334 .detach_mcast = irdma_detach_mcast, 4335 .get_link_layer = irdma_get_link_layer, 4336 .get_port_immutable = irdma_roce_port_immutable, 4337 .modify_qp = irdma_modify_qp_roce, 4338 .query_ah = irdma_query_ah, 4339 .query_pkey = irdma_query_pkey, 4340 }; 4341 4342 static const struct ib_device_ops irdma_iw_dev_ops = { 4343 .modify_qp = irdma_modify_qp, 4344 .get_port_immutable = irdma_iw_port_immutable, 4345 .query_gid = irdma_query_gid, 4346 }; 4347 4348 static const struct ib_device_ops irdma_dev_ops = { 4349 .owner = THIS_MODULE, 4350 .driver_id = RDMA_DRIVER_IRDMA, 4351 .uverbs_abi_ver = IRDMA_ABI_VER, 4352 4353 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 4354 .alloc_mr = irdma_alloc_mr, 4355 .alloc_mw = irdma_alloc_mw, 4356 .alloc_pd = irdma_alloc_pd, 4357 .alloc_ucontext = irdma_alloc_ucontext, 4358 .create_cq = irdma_create_cq, 4359 .create_qp = irdma_create_qp, 4360 .dealloc_driver = irdma_ib_dealloc_device, 4361 .dealloc_mw = irdma_dealloc_mw, 4362 .dealloc_pd = irdma_dealloc_pd, 4363 .dealloc_ucontext = irdma_dealloc_ucontext, 4364 .dereg_mr = irdma_dereg_mr, 4365 .destroy_cq = irdma_destroy_cq, 4366 .destroy_qp = irdma_destroy_qp, 4367 .disassociate_ucontext = irdma_disassociate_ucontext, 4368 .get_dev_fw_str = irdma_get_dev_fw_str, 4369 .get_dma_mr = irdma_get_dma_mr, 4370 .get_hw_stats = irdma_get_hw_stats, 4371 .map_mr_sg = irdma_map_mr_sg, 4372 .mmap = irdma_mmap, 4373 .mmap_free = irdma_mmap_free, 4374 .poll_cq = irdma_poll_cq, 4375 .post_recv = irdma_post_recv, 4376 .post_send = irdma_post_send, 4377 .query_device = irdma_query_device, 4378 .query_port = irdma_query_port, 4379 .query_qp = irdma_query_qp, 4380 .reg_user_mr = irdma_reg_user_mr, 4381 .req_notify_cq = irdma_req_notify_cq, 4382 .resize_cq = irdma_resize_cq, 4383 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 4384 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 4385 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 4386 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 4387 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 4388 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 4389 }; 4390 4391 /** 4392 * irdma_init_roce_device - initialization of roce rdma device 4393 * @iwdev: irdma device 4394 */ 4395 static void irdma_init_roce_device(struct irdma_device *iwdev) 4396 { 4397 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 4398 iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); 4399 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 4400 } 4401 4402 /** 4403 * irdma_init_iw_device - initialization of iwarp rdma device 4404 * @iwdev: irdma device 4405 */ 4406 static int irdma_init_iw_device(struct irdma_device *iwdev) 4407 { 4408 struct net_device *netdev = iwdev->netdev; 4409 4410 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 4411 ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, netdev->dev_addr); 4412 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; 4413 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; 4414 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; 4415 iwdev->ibdev.ops.iw_connect = irdma_connect; 4416 iwdev->ibdev.ops.iw_accept = irdma_accept; 4417 iwdev->ibdev.ops.iw_reject = irdma_reject; 4418 iwdev->ibdev.ops.iw_create_listen = irdma_create_listen; 4419 iwdev->ibdev.ops.iw_destroy_listen = irdma_destroy_listen; 4420 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 4421 sizeof(iwdev->ibdev.iw_ifname)); 4422 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 4423 4424 return 0; 4425 } 4426 4427 /** 4428 * irdma_init_rdma_device - initialization of rdma device 4429 * @iwdev: irdma device 4430 */ 4431 static int irdma_init_rdma_device(struct irdma_device *iwdev) 4432 { 4433 struct pci_dev *pcidev = iwdev->rf->pcidev; 4434 int ret; 4435 4436 if (iwdev->roce_mode) { 4437 irdma_init_roce_device(iwdev); 4438 } else { 4439 ret = irdma_init_iw_device(iwdev); 4440 if (ret) 4441 return ret; 4442 } 4443 iwdev->ibdev.phys_port_cnt = 1; 4444 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 4445 iwdev->ibdev.dev.parent = &pcidev->dev; 4446 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 4447 4448 return 0; 4449 } 4450 4451 /** 4452 * irdma_port_ibevent - indicate port event 4453 * @iwdev: irdma device 4454 */ 4455 void irdma_port_ibevent(struct irdma_device *iwdev) 4456 { 4457 struct ib_event event; 4458 4459 event.device = &iwdev->ibdev; 4460 event.element.port_num = 1; 4461 event.event = 4462 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 4463 ib_dispatch_event(&event); 4464 } 4465 4466 /** 4467 * irdma_ib_unregister_device - unregister rdma device from IB 4468 * core 4469 * @iwdev: irdma device 4470 */ 4471 void irdma_ib_unregister_device(struct irdma_device *iwdev) 4472 { 4473 iwdev->iw_status = 0; 4474 irdma_port_ibevent(iwdev); 4475 ib_unregister_device(&iwdev->ibdev); 4476 } 4477 4478 /** 4479 * irdma_ib_register_device - register irdma device to IB core 4480 * @iwdev: irdma device 4481 */ 4482 int irdma_ib_register_device(struct irdma_device *iwdev) 4483 { 4484 int ret; 4485 4486 ret = irdma_init_rdma_device(iwdev); 4487 if (ret) 4488 return ret; 4489 4490 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 4491 if (ret) 4492 goto error; 4493 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 4494 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 4495 if (ret) 4496 goto error; 4497 4498 iwdev->iw_status = 1; 4499 irdma_port_ibevent(iwdev); 4500 4501 return 0; 4502 4503 error: 4504 if (ret) 4505 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 4506 4507 return ret; 4508 } 4509 4510 /** 4511 * irdma_ib_dealloc_device 4512 * @ibdev: ib device 4513 * 4514 * callback from ibdev dealloc_driver to deallocate resources 4515 * unber irdma device 4516 */ 4517 void irdma_ib_dealloc_device(struct ib_device *ibdev) 4518 { 4519 struct irdma_device *iwdev = to_iwdev(ibdev); 4520 4521 irdma_rt_deinit_hw(iwdev); 4522 irdma_ctrl_deinit_hw(iwdev->rf); 4523 kfree(iwdev->rf); 4524 } 4525