1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "main.h" 4 5 /** 6 * irdma_query_device - get device attributes 7 * @ibdev: device pointer from stack 8 * @props: returning device attributes 9 * @udata: user data 10 */ 11 static int irdma_query_device(struct ib_device *ibdev, 12 struct ib_device_attr *props, 13 struct ib_udata *udata) 14 { 15 struct irdma_device *iwdev = to_iwdev(ibdev); 16 struct irdma_pci_f *rf = iwdev->rf; 17 struct pci_dev *pcidev = iwdev->rf->pcidev; 18 struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs; 19 20 if (udata->inlen || udata->outlen) 21 return -EINVAL; 22 23 memset(props, 0, sizeof(*props)); 24 addrconf_addr_eui48((u8 *)&props->sys_image_guid, 25 iwdev->netdev->dev_addr); 26 props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | 27 irdma_fw_minor_ver(&rf->sc_dev); 28 props->device_cap_flags = IB_DEVICE_MEM_WINDOW | 29 IB_DEVICE_MEM_MGT_EXTENSIONS; 30 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 31 props->vendor_id = pcidev->vendor; 32 props->vendor_part_id = pcidev->device; 33 34 props->hw_ver = rf->pcidev->revision; 35 props->page_size_cap = hw_attrs->page_size_cap; 36 props->max_mr_size = hw_attrs->max_mr_size; 37 props->max_qp = rf->max_qp - rf->used_qps; 38 props->max_qp_wr = hw_attrs->max_qp_wr; 39 props->max_send_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 40 props->max_recv_sge = hw_attrs->uk_attrs.max_hw_wq_frags; 41 props->max_cq = rf->max_cq - rf->used_cqs; 42 props->max_cqe = rf->max_cqe - 1; 43 props->max_mr = rf->max_mr - rf->used_mrs; 44 props->max_mw = props->max_mr; 45 props->max_pd = rf->max_pd - rf->used_pds; 46 props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges; 47 props->max_qp_rd_atom = hw_attrs->max_hw_ird; 48 props->max_qp_init_rd_atom = hw_attrs->max_hw_ord; 49 if (rdma_protocol_roce(ibdev, 1)) { 50 props->device_cap_flags |= IB_DEVICE_RC_RNR_NAK_GEN; 51 props->max_pkeys = IRDMA_PKEY_TBL_SZ; 52 } 53 54 props->max_ah = rf->max_ah; 55 props->max_mcast_grp = rf->max_mcg; 56 props->max_mcast_qp_attach = IRDMA_MAX_MGS_PER_CTX; 57 props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX; 58 props->max_fast_reg_page_list_len = IRDMA_MAX_PAGES_PER_FMR; 59 #define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff 60 if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2) 61 props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK; 62 63 return 0; 64 } 65 66 /** 67 * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed 68 * @link_speed: netdev phy link speed 69 * @active_speed: IB port speed 70 * @active_width: IB port width 71 */ 72 static void irdma_get_eth_speed_and_width(u32 link_speed, u16 *active_speed, 73 u8 *active_width) 74 { 75 if (link_speed <= SPEED_1000) { 76 *active_width = IB_WIDTH_1X; 77 *active_speed = IB_SPEED_SDR; 78 } else if (link_speed <= SPEED_10000) { 79 *active_width = IB_WIDTH_1X; 80 *active_speed = IB_SPEED_FDR10; 81 } else if (link_speed <= SPEED_20000) { 82 *active_width = IB_WIDTH_4X; 83 *active_speed = IB_SPEED_DDR; 84 } else if (link_speed <= SPEED_25000) { 85 *active_width = IB_WIDTH_1X; 86 *active_speed = IB_SPEED_EDR; 87 } else if (link_speed <= SPEED_40000) { 88 *active_width = IB_WIDTH_4X; 89 *active_speed = IB_SPEED_FDR10; 90 } else { 91 *active_width = IB_WIDTH_4X; 92 *active_speed = IB_SPEED_EDR; 93 } 94 } 95 96 /** 97 * irdma_query_port - get port attributes 98 * @ibdev: device pointer from stack 99 * @port: port number for query 100 * @props: returning device attributes 101 */ 102 static int irdma_query_port(struct ib_device *ibdev, u32 port, 103 struct ib_port_attr *props) 104 { 105 struct irdma_device *iwdev = to_iwdev(ibdev); 106 struct net_device *netdev = iwdev->netdev; 107 108 /* no need to zero out pros here. done by caller */ 109 110 props->max_mtu = IB_MTU_4096; 111 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); 112 props->lid = 1; 113 props->lmc = 0; 114 props->sm_lid = 0; 115 props->sm_sl = 0; 116 if (netif_carrier_ok(netdev) && netif_running(netdev)) { 117 props->state = IB_PORT_ACTIVE; 118 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 119 } else { 120 props->state = IB_PORT_DOWN; 121 props->phys_state = IB_PORT_PHYS_STATE_DISABLED; 122 } 123 irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, 124 &props->active_width); 125 126 if (rdma_protocol_roce(ibdev, 1)) { 127 props->gid_tbl_len = 32; 128 props->ip_gids = true; 129 props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; 130 } else { 131 props->gid_tbl_len = 1; 132 } 133 props->qkey_viol_cntr = 0; 134 props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_REINIT_SUP; 135 props->max_msg_sz = iwdev->rf->sc_dev.hw_attrs.max_hw_outbound_msg_size; 136 137 return 0; 138 } 139 140 /** 141 * irdma_disassociate_ucontext - Disassociate user context 142 * @context: ib user context 143 */ 144 static void irdma_disassociate_ucontext(struct ib_ucontext *context) 145 { 146 } 147 148 static int irdma_mmap_legacy(struct irdma_ucontext *ucontext, 149 struct vm_area_struct *vma) 150 { 151 u64 pfn; 152 153 if (vma->vm_pgoff || vma->vm_end - vma->vm_start != PAGE_SIZE) 154 return -EINVAL; 155 156 vma->vm_private_data = ucontext; 157 pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] + 158 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 159 160 return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE, 161 pgprot_noncached(vma->vm_page_prot), NULL); 162 } 163 164 static void irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 165 { 166 struct irdma_user_mmap_entry *entry = to_irdma_mmap_entry(rdma_entry); 167 168 kfree(entry); 169 } 170 171 static struct rdma_user_mmap_entry* 172 irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset, 173 enum irdma_mmap_flag mmap_flag, u64 *mmap_offset) 174 { 175 struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL); 176 int ret; 177 178 if (!entry) 179 return NULL; 180 181 entry->bar_offset = bar_offset; 182 entry->mmap_flag = mmap_flag; 183 184 ret = rdma_user_mmap_entry_insert(&ucontext->ibucontext, 185 &entry->rdma_entry, PAGE_SIZE); 186 if (ret) { 187 kfree(entry); 188 return NULL; 189 } 190 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 191 192 return &entry->rdma_entry; 193 } 194 195 /** 196 * irdma_mmap - user memory map 197 * @context: context created during alloc 198 * @vma: kernel info for user memory map 199 */ 200 static int irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) 201 { 202 struct rdma_user_mmap_entry *rdma_entry; 203 struct irdma_user_mmap_entry *entry; 204 struct irdma_ucontext *ucontext; 205 u64 pfn; 206 int ret; 207 208 ucontext = to_ucontext(context); 209 210 /* Legacy support for libi40iw with hard-coded mmap key */ 211 if (ucontext->legacy_mode) 212 return irdma_mmap_legacy(ucontext, vma); 213 214 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); 215 if (!rdma_entry) { 216 ibdev_dbg(&ucontext->iwdev->ibdev, 217 "VERBS: pgoff[0x%lx] does not have valid entry\n", 218 vma->vm_pgoff); 219 return -EINVAL; 220 } 221 222 entry = to_irdma_mmap_entry(rdma_entry); 223 ibdev_dbg(&ucontext->iwdev->ibdev, 224 "VERBS: bar_offset [0x%llx] mmap_flag [%d]\n", 225 entry->bar_offset, entry->mmap_flag); 226 227 pfn = (entry->bar_offset + 228 pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; 229 230 switch (entry->mmap_flag) { 231 case IRDMA_MMAP_IO_NC: 232 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 233 pgprot_noncached(vma->vm_page_prot), 234 rdma_entry); 235 break; 236 case IRDMA_MMAP_IO_WC: 237 ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE, 238 pgprot_writecombine(vma->vm_page_prot), 239 rdma_entry); 240 break; 241 default: 242 ret = -EINVAL; 243 } 244 245 if (ret) 246 ibdev_dbg(&ucontext->iwdev->ibdev, 247 "VERBS: bar_offset [0x%llx] mmap_flag[%d] err[%d]\n", 248 entry->bar_offset, entry->mmap_flag, ret); 249 rdma_user_mmap_entry_put(rdma_entry); 250 251 return ret; 252 } 253 254 /** 255 * irdma_alloc_push_page - allocate a push page for qp 256 * @iwqp: qp pointer 257 */ 258 static void irdma_alloc_push_page(struct irdma_qp *iwqp) 259 { 260 struct irdma_cqp_request *cqp_request; 261 struct cqp_cmds_info *cqp_info; 262 struct irdma_device *iwdev = iwqp->iwdev; 263 struct irdma_sc_qp *qp = &iwqp->sc_qp; 264 int status; 265 266 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 267 if (!cqp_request) 268 return; 269 270 cqp_info = &cqp_request->info; 271 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE; 272 cqp_info->post_sq = 1; 273 cqp_info->in.u.manage_push_page.info.push_idx = 0; 274 cqp_info->in.u.manage_push_page.info.qs_handle = 275 qp->vsi->qos[qp->user_pri].qs_handle; 276 cqp_info->in.u.manage_push_page.info.free_page = 0; 277 cqp_info->in.u.manage_push_page.info.push_page_type = 0; 278 cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp; 279 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request; 280 281 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 282 if (!status && cqp_request->compl_info.op_ret_val < 283 iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) { 284 qp->push_idx = cqp_request->compl_info.op_ret_val; 285 qp->push_offset = 0; 286 } 287 288 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 289 } 290 291 /** 292 * irdma_alloc_ucontext - Allocate the user context data structure 293 * @uctx: uverbs context pointer 294 * @udata: user data 295 * 296 * This keeps track of all objects associated with a particular 297 * user-mode client. 298 */ 299 static int irdma_alloc_ucontext(struct ib_ucontext *uctx, 300 struct ib_udata *udata) 301 { 302 struct ib_device *ibdev = uctx->device; 303 struct irdma_device *iwdev = to_iwdev(ibdev); 304 struct irdma_alloc_ucontext_req req; 305 struct irdma_alloc_ucontext_resp uresp = {}; 306 struct irdma_ucontext *ucontext = to_ucontext(uctx); 307 struct irdma_uk_attrs *uk_attrs; 308 309 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) 310 return -EINVAL; 311 312 if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER) 313 goto ver_error; 314 315 ucontext->iwdev = iwdev; 316 ucontext->abi_ver = req.userspace_ver; 317 318 uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 319 /* GEN_1 legacy support with libi40iw */ 320 if (udata->outlen < sizeof(uresp)) { 321 if (uk_attrs->hw_rev != IRDMA_GEN_1) 322 return -EOPNOTSUPP; 323 324 ucontext->legacy_mode = true; 325 uresp.max_qps = iwdev->rf->max_qp; 326 uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds; 327 uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2; 328 uresp.kernel_ver = req.userspace_ver; 329 if (ib_copy_to_udata(udata, &uresp, 330 min(sizeof(uresp), udata->outlen))) 331 return -EFAULT; 332 } else { 333 u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 334 335 ucontext->db_mmap_entry = 336 irdma_user_mmap_entry_insert(ucontext, bar_off, 337 IRDMA_MMAP_IO_NC, 338 &uresp.db_mmap_key); 339 if (!ucontext->db_mmap_entry) 340 return -ENOMEM; 341 342 uresp.kernel_ver = IRDMA_ABI_VER; 343 uresp.feature_flags = uk_attrs->feature_flags; 344 uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags; 345 uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges; 346 uresp.max_hw_inline = uk_attrs->max_hw_inline; 347 uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta; 348 uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta; 349 uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk; 350 uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; 351 uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; 352 uresp.hw_rev = uk_attrs->hw_rev; 353 if (ib_copy_to_udata(udata, &uresp, 354 min(sizeof(uresp), udata->outlen))) { 355 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 356 return -EFAULT; 357 } 358 } 359 360 INIT_LIST_HEAD(&ucontext->cq_reg_mem_list); 361 spin_lock_init(&ucontext->cq_reg_mem_list_lock); 362 INIT_LIST_HEAD(&ucontext->qp_reg_mem_list); 363 spin_lock_init(&ucontext->qp_reg_mem_list_lock); 364 365 return 0; 366 367 ver_error: 368 ibdev_err(&iwdev->ibdev, 369 "Invalid userspace driver version detected. Detected version %d, should be %d\n", 370 req.userspace_ver, IRDMA_ABI_VER); 371 return -EINVAL; 372 } 373 374 /** 375 * irdma_dealloc_ucontext - deallocate the user context data structure 376 * @context: user context created during alloc 377 */ 378 static void irdma_dealloc_ucontext(struct ib_ucontext *context) 379 { 380 struct irdma_ucontext *ucontext = to_ucontext(context); 381 382 rdma_user_mmap_entry_remove(ucontext->db_mmap_entry); 383 } 384 385 /** 386 * irdma_alloc_pd - allocate protection domain 387 * @pd: PD pointer 388 * @udata: user data 389 */ 390 static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) 391 { 392 struct irdma_pd *iwpd = to_iwpd(pd); 393 struct irdma_device *iwdev = to_iwdev(pd->device); 394 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 395 struct irdma_pci_f *rf = iwdev->rf; 396 struct irdma_alloc_pd_resp uresp = {}; 397 struct irdma_sc_pd *sc_pd; 398 u32 pd_id = 0; 399 int err; 400 401 err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id, 402 &rf->next_pd); 403 if (err) 404 return err; 405 406 sc_pd = &iwpd->sc_pd; 407 if (udata) { 408 struct irdma_ucontext *ucontext = 409 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 410 ibucontext); 411 irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); 412 uresp.pd_id = pd_id; 413 if (ib_copy_to_udata(udata, &uresp, 414 min(sizeof(uresp), udata->outlen))) { 415 err = -EFAULT; 416 goto error; 417 } 418 } else { 419 irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); 420 } 421 422 return 0; 423 error: 424 irdma_free_rsrc(rf, rf->allocated_pds, pd_id); 425 426 return err; 427 } 428 429 /** 430 * irdma_dealloc_pd - deallocate pd 431 * @ibpd: ptr of pd to be deallocated 432 * @udata: user data 433 */ 434 static int irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 435 { 436 struct irdma_pd *iwpd = to_iwpd(ibpd); 437 struct irdma_device *iwdev = to_iwdev(ibpd->device); 438 439 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id); 440 441 return 0; 442 } 443 444 /** 445 * irdma_get_pbl - Retrieve pbl from a list given a virtual 446 * address 447 * @va: user virtual address 448 * @pbl_list: pbl list to search in (QP's or CQ's) 449 */ 450 static struct irdma_pbl *irdma_get_pbl(unsigned long va, 451 struct list_head *pbl_list) 452 { 453 struct irdma_pbl *iwpbl; 454 455 list_for_each_entry (iwpbl, pbl_list, list) { 456 if (iwpbl->user_base == va) { 457 list_del(&iwpbl->list); 458 iwpbl->on_list = false; 459 return iwpbl; 460 } 461 } 462 463 return NULL; 464 } 465 466 /** 467 * irdma_clean_cqes - clean cq entries for qp 468 * @iwqp: qp ptr (user or kernel) 469 * @iwcq: cq ptr 470 */ 471 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq) 472 { 473 struct irdma_cq_uk *ukcq = &iwcq->sc_cq.cq_uk; 474 unsigned long flags; 475 476 spin_lock_irqsave(&iwcq->lock, flags); 477 irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq); 478 spin_unlock_irqrestore(&iwcq->lock, flags); 479 } 480 481 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp) 482 { 483 if (iwqp->push_db_mmap_entry) { 484 rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry); 485 iwqp->push_db_mmap_entry = NULL; 486 } 487 if (iwqp->push_wqe_mmap_entry) { 488 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 489 iwqp->push_wqe_mmap_entry = NULL; 490 } 491 } 492 493 static int irdma_setup_push_mmap_entries(struct irdma_ucontext *ucontext, 494 struct irdma_qp *iwqp, 495 u64 *push_wqe_mmap_key, 496 u64 *push_db_mmap_key) 497 { 498 struct irdma_device *iwdev = ucontext->iwdev; 499 u64 rsvd, bar_off; 500 501 rsvd = IRDMA_PF_BAR_RSVD; 502 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; 503 /* skip over db page */ 504 bar_off += IRDMA_HW_PAGE_SIZE; 505 /* push wqe page */ 506 bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE; 507 iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 508 bar_off, IRDMA_MMAP_IO_WC, 509 push_wqe_mmap_key); 510 if (!iwqp->push_wqe_mmap_entry) 511 return -ENOMEM; 512 513 /* push doorbell page */ 514 bar_off += IRDMA_HW_PAGE_SIZE; 515 iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext, 516 bar_off, IRDMA_MMAP_IO_NC, 517 push_db_mmap_key); 518 if (!iwqp->push_db_mmap_entry) { 519 rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry); 520 return -ENOMEM; 521 } 522 523 return 0; 524 } 525 526 /** 527 * irdma_destroy_qp - destroy qp 528 * @ibqp: qp's ib pointer also to get to device's qp address 529 * @udata: user data 530 */ 531 static int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 532 { 533 struct irdma_qp *iwqp = to_iwqp(ibqp); 534 struct irdma_device *iwdev = iwqp->iwdev; 535 536 iwqp->sc_qp.qp_uk.destroy_pending = true; 537 538 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) 539 irdma_modify_qp_to_err(&iwqp->sc_qp); 540 541 if (!iwqp->user_mode) 542 cancel_delayed_work_sync(&iwqp->dwork_flush); 543 544 irdma_qp_rem_ref(&iwqp->ibqp); 545 wait_for_completion(&iwqp->free_qp); 546 irdma_free_lsmm_rsrc(iwqp); 547 irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp); 548 549 if (!iwqp->user_mode) { 550 if (iwqp->iwscq) { 551 irdma_clean_cqes(iwqp, iwqp->iwscq); 552 if (iwqp->iwrcq != iwqp->iwscq) 553 irdma_clean_cqes(iwqp, iwqp->iwrcq); 554 } 555 } 556 irdma_remove_push_mmap_entries(iwqp); 557 irdma_free_qp_rsrc(iwqp); 558 559 return 0; 560 } 561 562 /** 563 * irdma_setup_virt_qp - setup for allocation of virtual qp 564 * @iwdev: irdma device 565 * @iwqp: qp ptr 566 * @init_info: initialize info to return 567 */ 568 static void irdma_setup_virt_qp(struct irdma_device *iwdev, 569 struct irdma_qp *iwqp, 570 struct irdma_qp_init_info *init_info) 571 { 572 struct irdma_pbl *iwpbl = iwqp->iwpbl; 573 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 574 575 iwqp->page = qpmr->sq_page; 576 init_info->shadow_area_pa = qpmr->shadow; 577 if (iwpbl->pbl_allocated) { 578 init_info->virtual_map = true; 579 init_info->sq_pa = qpmr->sq_pbl.idx; 580 init_info->rq_pa = qpmr->rq_pbl.idx; 581 } else { 582 init_info->sq_pa = qpmr->sq_pbl.addr; 583 init_info->rq_pa = qpmr->rq_pbl.addr; 584 } 585 } 586 587 /** 588 * irdma_setup_kmode_qp - setup initialization for kernel mode qp 589 * @iwdev: iwarp device 590 * @iwqp: qp ptr (user or kernel) 591 * @info: initialize info to return 592 * @init_attr: Initial QP create attributes 593 */ 594 static int irdma_setup_kmode_qp(struct irdma_device *iwdev, 595 struct irdma_qp *iwqp, 596 struct irdma_qp_init_info *info, 597 struct ib_qp_init_attr *init_attr) 598 { 599 struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem; 600 u32 sqdepth, rqdepth; 601 u8 sqshift, rqshift; 602 u32 size; 603 int status; 604 struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; 605 struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; 606 607 irdma_get_wqe_shift(uk_attrs, 608 uk_attrs->hw_rev >= IRDMA_GEN_2 ? ukinfo->max_sq_frag_cnt + 1 : 609 ukinfo->max_sq_frag_cnt, 610 ukinfo->max_inline_data, &sqshift); 611 status = irdma_get_sqdepth(uk_attrs, ukinfo->sq_size, sqshift, 612 &sqdepth); 613 if (status) 614 return status; 615 616 if (uk_attrs->hw_rev == IRDMA_GEN_1) 617 rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; 618 else 619 irdma_get_wqe_shift(uk_attrs, ukinfo->max_rq_frag_cnt, 0, 620 &rqshift); 621 622 status = irdma_get_rqdepth(uk_attrs, ukinfo->rq_size, rqshift, 623 &rqdepth); 624 if (status) 625 return status; 626 627 iwqp->kqp.sq_wrid_mem = 628 kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL); 629 if (!iwqp->kqp.sq_wrid_mem) 630 return -ENOMEM; 631 632 iwqp->kqp.rq_wrid_mem = 633 kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL); 634 if (!iwqp->kqp.rq_wrid_mem) { 635 kfree(iwqp->kqp.sq_wrid_mem); 636 iwqp->kqp.sq_wrid_mem = NULL; 637 return -ENOMEM; 638 } 639 640 ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem; 641 ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem; 642 643 size = (sqdepth + rqdepth) * IRDMA_QP_WQE_MIN_SIZE; 644 size += (IRDMA_SHADOW_AREA_SIZE << 3); 645 646 mem->size = ALIGN(size, 256); 647 mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size, 648 &mem->pa, GFP_KERNEL); 649 if (!mem->va) { 650 kfree(iwqp->kqp.sq_wrid_mem); 651 iwqp->kqp.sq_wrid_mem = NULL; 652 kfree(iwqp->kqp.rq_wrid_mem); 653 iwqp->kqp.rq_wrid_mem = NULL; 654 return -ENOMEM; 655 } 656 657 ukinfo->sq = mem->va; 658 info->sq_pa = mem->pa; 659 ukinfo->rq = &ukinfo->sq[sqdepth]; 660 info->rq_pa = info->sq_pa + (sqdepth * IRDMA_QP_WQE_MIN_SIZE); 661 ukinfo->shadow_area = ukinfo->rq[rqdepth].elem; 662 info->shadow_area_pa = info->rq_pa + (rqdepth * IRDMA_QP_WQE_MIN_SIZE); 663 ukinfo->sq_size = sqdepth >> sqshift; 664 ukinfo->rq_size = rqdepth >> rqshift; 665 ukinfo->qp_id = iwqp->ibqp.qp_num; 666 667 init_attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift; 668 init_attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift; 669 670 return 0; 671 } 672 673 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp) 674 { 675 struct irdma_pci_f *rf = iwqp->iwdev->rf; 676 struct irdma_cqp_request *cqp_request; 677 struct cqp_cmds_info *cqp_info; 678 struct irdma_create_qp_info *qp_info; 679 int status; 680 681 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 682 if (!cqp_request) 683 return -ENOMEM; 684 685 cqp_info = &cqp_request->info; 686 qp_info = &cqp_request->info.in.u.qp_create.info; 687 memset(qp_info, 0, sizeof(*qp_info)); 688 qp_info->mac_valid = true; 689 qp_info->cq_num_valid = true; 690 qp_info->next_iwarp_state = IRDMA_QP_STATE_IDLE; 691 692 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE; 693 cqp_info->post_sq = 1; 694 cqp_info->in.u.qp_create.qp = &iwqp->sc_qp; 695 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request; 696 status = irdma_handle_cqp_op(rf, cqp_request); 697 irdma_put_cqp_request(&rf->cqp, cqp_request); 698 699 return status; 700 } 701 702 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 703 struct irdma_qp_host_ctx_info *ctx_info) 704 { 705 struct irdma_device *iwdev = iwqp->iwdev; 706 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 707 struct irdma_roce_offload_info *roce_info; 708 struct irdma_udp_offload_info *udp_info; 709 710 udp_info = &iwqp->udp_info; 711 udp_info->snd_mss = ib_mtu_enum_to_int(ib_mtu_int_to_enum(iwdev->vsi.mtu)); 712 udp_info->cwnd = iwdev->roce_cwnd; 713 udp_info->rexmit_thresh = 2; 714 udp_info->rnr_nak_thresh = 2; 715 udp_info->src_port = 0xc000; 716 udp_info->dst_port = ROCE_V2_UDP_DPORT; 717 roce_info = &iwqp->roce_info; 718 ether_addr_copy(roce_info->mac_addr, iwdev->netdev->dev_addr); 719 720 roce_info->rd_en = true; 721 roce_info->wr_rdresp_en = true; 722 roce_info->bind_en = true; 723 roce_info->dcqcn_en = false; 724 roce_info->rtomin = 5; 725 726 roce_info->ack_credits = iwdev->roce_ackcreds; 727 roce_info->ird_size = dev->hw_attrs.max_hw_ird; 728 roce_info->ord_size = dev->hw_attrs.max_hw_ord; 729 730 if (!iwqp->user_mode) { 731 roce_info->priv_mode_en = true; 732 roce_info->fast_reg_en = true; 733 roce_info->udprivcq_en = true; 734 } 735 roce_info->roce_tver = 0; 736 737 ctx_info->roce_info = &iwqp->roce_info; 738 ctx_info->udp_info = &iwqp->udp_info; 739 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 740 } 741 742 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp, 743 struct irdma_qp_host_ctx_info *ctx_info) 744 { 745 struct irdma_device *iwdev = iwqp->iwdev; 746 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 747 struct irdma_iwarp_offload_info *iwarp_info; 748 749 iwarp_info = &iwqp->iwarp_info; 750 ether_addr_copy(iwarp_info->mac_addr, iwdev->netdev->dev_addr); 751 iwarp_info->rd_en = true; 752 iwarp_info->wr_rdresp_en = true; 753 iwarp_info->bind_en = true; 754 iwarp_info->ecn_en = true; 755 iwarp_info->rtomin = 5; 756 757 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 758 iwarp_info->ib_rd_en = true; 759 if (!iwqp->user_mode) { 760 iwarp_info->priv_mode_en = true; 761 iwarp_info->fast_reg_en = true; 762 } 763 iwarp_info->ddp_ver = 1; 764 iwarp_info->rdmap_ver = 1; 765 766 ctx_info->iwarp_info = &iwqp->iwarp_info; 767 ctx_info->iwarp_info_valid = true; 768 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 769 ctx_info->iwarp_info_valid = false; 770 } 771 772 static int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr, 773 struct irdma_device *iwdev) 774 { 775 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 776 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 777 778 if (init_attr->create_flags) 779 return -EOPNOTSUPP; 780 781 if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline || 782 init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags || 783 init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags) 784 return -EINVAL; 785 786 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 787 if (init_attr->qp_type != IB_QPT_RC && 788 init_attr->qp_type != IB_QPT_UD && 789 init_attr->qp_type != IB_QPT_GSI) 790 return -EOPNOTSUPP; 791 } else { 792 if (init_attr->qp_type != IB_QPT_RC) 793 return -EOPNOTSUPP; 794 } 795 796 return 0; 797 } 798 799 static void irdma_flush_worker(struct work_struct *work) 800 { 801 struct delayed_work *dwork = to_delayed_work(work); 802 struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush); 803 804 irdma_generate_flush_completions(iwqp); 805 } 806 807 /** 808 * irdma_create_qp - create qp 809 * @ibqp: ptr of qp 810 * @init_attr: attributes for qp 811 * @udata: user data for create qp 812 */ 813 static int irdma_create_qp(struct ib_qp *ibqp, 814 struct ib_qp_init_attr *init_attr, 815 struct ib_udata *udata) 816 { 817 struct ib_pd *ibpd = ibqp->pd; 818 struct irdma_pd *iwpd = to_iwpd(ibpd); 819 struct irdma_device *iwdev = to_iwdev(ibpd->device); 820 struct irdma_pci_f *rf = iwdev->rf; 821 struct irdma_qp *iwqp = to_iwqp(ibqp); 822 struct irdma_create_qp_req req; 823 struct irdma_create_qp_resp uresp = {}; 824 u32 qp_num = 0; 825 int err_code; 826 int sq_size; 827 int rq_size; 828 struct irdma_sc_qp *qp; 829 struct irdma_sc_dev *dev = &rf->sc_dev; 830 struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; 831 struct irdma_qp_init_info init_info = {}; 832 struct irdma_qp_host_ctx_info *ctx_info; 833 unsigned long flags; 834 835 err_code = irdma_validate_qp_attrs(init_attr, iwdev); 836 if (err_code) 837 return err_code; 838 839 sq_size = init_attr->cap.max_send_wr; 840 rq_size = init_attr->cap.max_recv_wr; 841 842 init_info.vsi = &iwdev->vsi; 843 init_info.qp_uk_init_info.uk_attrs = uk_attrs; 844 init_info.qp_uk_init_info.sq_size = sq_size; 845 init_info.qp_uk_init_info.rq_size = rq_size; 846 init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; 847 init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; 848 init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; 849 850 qp = &iwqp->sc_qp; 851 qp->qp_uk.back_qp = iwqp; 852 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX; 853 854 iwqp->iwdev = iwdev; 855 iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE, 856 256); 857 iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device, 858 iwqp->q2_ctx_mem.size, 859 &iwqp->q2_ctx_mem.pa, 860 GFP_KERNEL); 861 if (!iwqp->q2_ctx_mem.va) 862 return -ENOMEM; 863 864 init_info.q2 = iwqp->q2_ctx_mem.va; 865 init_info.q2_pa = iwqp->q2_ctx_mem.pa; 866 init_info.host_ctx = (__le64 *)(init_info.q2 + IRDMA_Q2_BUF_SIZE); 867 init_info.host_ctx_pa = init_info.q2_pa + IRDMA_Q2_BUF_SIZE; 868 869 if (init_attr->qp_type == IB_QPT_GSI) 870 qp_num = 1; 871 else 872 err_code = irdma_alloc_rsrc(rf, rf->allocated_qps, rf->max_qp, 873 &qp_num, &rf->next_qp); 874 if (err_code) 875 goto error; 876 877 iwqp->iwpd = iwpd; 878 iwqp->ibqp.qp_num = qp_num; 879 qp = &iwqp->sc_qp; 880 iwqp->iwscq = to_iwcq(init_attr->send_cq); 881 iwqp->iwrcq = to_iwcq(init_attr->recv_cq); 882 iwqp->host_ctx.va = init_info.host_ctx; 883 iwqp->host_ctx.pa = init_info.host_ctx_pa; 884 iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE; 885 886 init_info.pd = &iwpd->sc_pd; 887 init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num; 888 if (!rdma_protocol_roce(&iwdev->ibdev, 1)) 889 init_info.qp_uk_init_info.first_sq_wq = 1; 890 iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp; 891 init_waitqueue_head(&iwqp->waitq); 892 init_waitqueue_head(&iwqp->mod_qp_waitq); 893 894 if (udata) { 895 err_code = ib_copy_from_udata(&req, udata, 896 min(sizeof(req), udata->inlen)); 897 if (err_code) { 898 ibdev_dbg(&iwdev->ibdev, 899 "VERBS: ib_copy_from_data fail\n"); 900 goto error; 901 } 902 903 iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; 904 iwqp->user_mode = 1; 905 if (req.user_wqe_bufs) { 906 struct irdma_ucontext *ucontext = 907 rdma_udata_to_drv_context(udata, 908 struct irdma_ucontext, 909 ibucontext); 910 911 init_info.qp_uk_init_info.legacy_mode = ucontext->legacy_mode; 912 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 913 iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, 914 &ucontext->qp_reg_mem_list); 915 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 916 917 if (!iwqp->iwpbl) { 918 err_code = -ENODATA; 919 ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); 920 goto error; 921 } 922 } 923 init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; 924 irdma_setup_virt_qp(iwdev, iwqp, &init_info); 925 } else { 926 INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker); 927 init_info.qp_uk_init_info.abi_ver = IRDMA_ABI_VER; 928 err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr); 929 } 930 931 if (err_code) { 932 ibdev_dbg(&iwdev->ibdev, "VERBS: setup qp failed\n"); 933 goto error; 934 } 935 936 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 937 if (init_attr->qp_type == IB_QPT_RC) { 938 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_RC; 939 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 940 IRDMA_WRITE_WITH_IMM | 941 IRDMA_ROCE; 942 } else { 943 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_ROCE_UD; 944 init_info.qp_uk_init_info.qp_caps = IRDMA_SEND_WITH_IMM | 945 IRDMA_ROCE; 946 } 947 } else { 948 init_info.qp_uk_init_info.type = IRDMA_QP_TYPE_IWARP; 949 init_info.qp_uk_init_info.qp_caps = IRDMA_WRITE_WITH_IMM; 950 } 951 952 if (dev->hw_attrs.uk_attrs.hw_rev > IRDMA_GEN_1) 953 init_info.qp_uk_init_info.qp_caps |= IRDMA_PUSH_MODE; 954 955 err_code = irdma_sc_qp_init(qp, &init_info); 956 if (err_code) { 957 ibdev_dbg(&iwdev->ibdev, "VERBS: qp_init fail\n"); 958 goto error; 959 } 960 961 ctx_info = &iwqp->ctx_info; 962 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 963 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 964 965 if (rdma_protocol_roce(&iwdev->ibdev, 1)) 966 irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info); 967 else 968 irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info); 969 970 err_code = irdma_cqp_create_qp_cmd(iwqp); 971 if (err_code) 972 goto error; 973 974 refcount_set(&iwqp->refcnt, 1); 975 spin_lock_init(&iwqp->lock); 976 spin_lock_init(&iwqp->sc_qp.pfpdu.lock); 977 iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0; 978 rf->qp_table[qp_num] = iwqp; 979 iwqp->max_send_wr = sq_size; 980 iwqp->max_recv_wr = rq_size; 981 982 if (rdma_protocol_roce(&iwdev->ibdev, 1)) { 983 if (dev->ws_add(&iwdev->vsi, 0)) { 984 irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp); 985 err_code = -EINVAL; 986 goto error; 987 } 988 989 irdma_qp_add_qos(&iwqp->sc_qp); 990 } 991 992 if (udata) { 993 /* GEN_1 legacy support with libi40iw does not have expanded uresp struct */ 994 if (udata->outlen < sizeof(uresp)) { 995 uresp.lsmm = 1; 996 uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1; 997 } else { 998 if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) 999 uresp.lsmm = 1; 1000 } 1001 uresp.actual_sq_size = sq_size; 1002 uresp.actual_rq_size = rq_size; 1003 uresp.qp_id = qp_num; 1004 uresp.qp_caps = qp->qp_uk.qp_caps; 1005 1006 err_code = ib_copy_to_udata(udata, &uresp, 1007 min(sizeof(uresp), udata->outlen)); 1008 if (err_code) { 1009 ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n"); 1010 irdma_destroy_qp(&iwqp->ibqp, udata); 1011 return err_code; 1012 } 1013 } 1014 1015 init_completion(&iwqp->free_qp); 1016 return 0; 1017 1018 error: 1019 irdma_free_qp_rsrc(iwqp); 1020 return err_code; 1021 } 1022 1023 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp) 1024 { 1025 int acc_flags = 0; 1026 1027 if (rdma_protocol_roce(iwqp->ibqp.device, 1)) { 1028 if (iwqp->roce_info.wr_rdresp_en) { 1029 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1030 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1031 } 1032 if (iwqp->roce_info.rd_en) 1033 acc_flags |= IB_ACCESS_REMOTE_READ; 1034 if (iwqp->roce_info.bind_en) 1035 acc_flags |= IB_ACCESS_MW_BIND; 1036 } else { 1037 if (iwqp->iwarp_info.wr_rdresp_en) { 1038 acc_flags |= IB_ACCESS_LOCAL_WRITE; 1039 acc_flags |= IB_ACCESS_REMOTE_WRITE; 1040 } 1041 if (iwqp->iwarp_info.rd_en) 1042 acc_flags |= IB_ACCESS_REMOTE_READ; 1043 if (iwqp->iwarp_info.bind_en) 1044 acc_flags |= IB_ACCESS_MW_BIND; 1045 } 1046 return acc_flags; 1047 } 1048 1049 /** 1050 * irdma_query_qp - query qp attributes 1051 * @ibqp: qp pointer 1052 * @attr: attributes pointer 1053 * @attr_mask: Not used 1054 * @init_attr: qp attributes to return 1055 */ 1056 static int irdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1057 int attr_mask, struct ib_qp_init_attr *init_attr) 1058 { 1059 struct irdma_qp *iwqp = to_iwqp(ibqp); 1060 struct irdma_sc_qp *qp = &iwqp->sc_qp; 1061 1062 memset(attr, 0, sizeof(*attr)); 1063 memset(init_attr, 0, sizeof(*init_attr)); 1064 1065 attr->qp_state = iwqp->ibqp_state; 1066 attr->cur_qp_state = iwqp->ibqp_state; 1067 attr->cap.max_send_wr = iwqp->max_send_wr; 1068 attr->cap.max_recv_wr = iwqp->max_recv_wr; 1069 attr->cap.max_inline_data = qp->qp_uk.max_inline_data; 1070 attr->cap.max_send_sge = qp->qp_uk.max_sq_frag_cnt; 1071 attr->cap.max_recv_sge = qp->qp_uk.max_rq_frag_cnt; 1072 attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp); 1073 attr->port_num = 1; 1074 if (rdma_protocol_roce(ibqp->device, 1)) { 1075 attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss); 1076 attr->qkey = iwqp->roce_info.qkey; 1077 attr->rq_psn = iwqp->udp_info.epsn; 1078 attr->sq_psn = iwqp->udp_info.psn_nxt; 1079 attr->dest_qp_num = iwqp->roce_info.dest_qp; 1080 attr->pkey_index = iwqp->roce_info.p_key; 1081 attr->retry_cnt = iwqp->udp_info.rexmit_thresh; 1082 attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh; 1083 attr->max_rd_atomic = iwqp->roce_info.ord_size; 1084 attr->max_dest_rd_atomic = iwqp->roce_info.ird_size; 1085 } 1086 1087 init_attr->event_handler = iwqp->ibqp.event_handler; 1088 init_attr->qp_context = iwqp->ibqp.qp_context; 1089 init_attr->send_cq = iwqp->ibqp.send_cq; 1090 init_attr->recv_cq = iwqp->ibqp.recv_cq; 1091 init_attr->cap = attr->cap; 1092 1093 return 0; 1094 } 1095 1096 /** 1097 * irdma_query_pkey - Query partition key 1098 * @ibdev: device pointer from stack 1099 * @port: port number 1100 * @index: index of pkey 1101 * @pkey: pointer to store the pkey 1102 */ 1103 static int irdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, 1104 u16 *pkey) 1105 { 1106 if (index >= IRDMA_PKEY_TBL_SZ) 1107 return -EINVAL; 1108 1109 *pkey = IRDMA_DEFAULT_PKEY; 1110 return 0; 1111 } 1112 1113 /** 1114 * irdma_modify_qp_roce - modify qp request 1115 * @ibqp: qp's pointer for modify 1116 * @attr: access attributes 1117 * @attr_mask: state mask 1118 * @udata: user data 1119 */ 1120 int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr, 1121 int attr_mask, struct ib_udata *udata) 1122 { 1123 struct irdma_pd *iwpd = to_iwpd(ibqp->pd); 1124 struct irdma_qp *iwqp = to_iwqp(ibqp); 1125 struct irdma_device *iwdev = iwqp->iwdev; 1126 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1127 struct irdma_qp_host_ctx_info *ctx_info; 1128 struct irdma_roce_offload_info *roce_info; 1129 struct irdma_udp_offload_info *udp_info; 1130 struct irdma_modify_qp_info info = {}; 1131 struct irdma_modify_qp_resp uresp = {}; 1132 struct irdma_modify_qp_req ureq = {}; 1133 unsigned long flags; 1134 u8 issue_modify_qp = 0; 1135 int ret = 0; 1136 1137 ctx_info = &iwqp->ctx_info; 1138 roce_info = &iwqp->roce_info; 1139 udp_info = &iwqp->udp_info; 1140 1141 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1142 return -EOPNOTSUPP; 1143 1144 if (attr_mask & IB_QP_DEST_QPN) 1145 roce_info->dest_qp = attr->dest_qp_num; 1146 1147 if (attr_mask & IB_QP_PKEY_INDEX) { 1148 ret = irdma_query_pkey(ibqp->device, 0, attr->pkey_index, 1149 &roce_info->p_key); 1150 if (ret) 1151 return ret; 1152 } 1153 1154 if (attr_mask & IB_QP_QKEY) 1155 roce_info->qkey = attr->qkey; 1156 1157 if (attr_mask & IB_QP_PATH_MTU) 1158 udp_info->snd_mss = ib_mtu_enum_to_int(attr->path_mtu); 1159 1160 if (attr_mask & IB_QP_SQ_PSN) { 1161 udp_info->psn_nxt = attr->sq_psn; 1162 udp_info->lsn = 0xffff; 1163 udp_info->psn_una = attr->sq_psn; 1164 udp_info->psn_max = attr->sq_psn; 1165 } 1166 1167 if (attr_mask & IB_QP_RQ_PSN) 1168 udp_info->epsn = attr->rq_psn; 1169 1170 if (attr_mask & IB_QP_RNR_RETRY) 1171 udp_info->rnr_nak_thresh = attr->rnr_retry; 1172 1173 if (attr_mask & IB_QP_RETRY_CNT) 1174 udp_info->rexmit_thresh = attr->retry_cnt; 1175 1176 ctx_info->roce_info->pd_id = iwpd->sc_pd.pd_id; 1177 1178 if (attr_mask & IB_QP_AV) { 1179 struct irdma_av *av = &iwqp->roce_ah.av; 1180 const struct ib_gid_attr *sgid_attr; 1181 u16 vlan_id = VLAN_N_VID; 1182 u32 local_ip[4]; 1183 1184 memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah)); 1185 if (attr->ah_attr.ah_flags & IB_AH_GRH) { 1186 udp_info->ttl = attr->ah_attr.grh.hop_limit; 1187 udp_info->flow_label = attr->ah_attr.grh.flow_label; 1188 udp_info->tos = attr->ah_attr.grh.traffic_class; 1189 udp_info->src_port = 1190 rdma_get_udp_sport(udp_info->flow_label, 1191 ibqp->qp_num, 1192 roce_info->dest_qp); 1193 irdma_qp_rem_qos(&iwqp->sc_qp); 1194 dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri); 1195 ctx_info->user_pri = rt_tos2priority(udp_info->tos); 1196 iwqp->sc_qp.user_pri = ctx_info->user_pri; 1197 if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri)) 1198 return -ENOMEM; 1199 irdma_qp_add_qos(&iwqp->sc_qp); 1200 } 1201 sgid_attr = attr->ah_attr.grh.sgid_attr; 1202 ret = rdma_read_gid_l2_fields(sgid_attr, &vlan_id, 1203 ctx_info->roce_info->mac_addr); 1204 if (ret) 1205 return ret; 1206 1207 if (vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode) 1208 vlan_id = 0; 1209 if (vlan_id < VLAN_N_VID) { 1210 udp_info->insert_vlan_tag = true; 1211 udp_info->vlan_tag = vlan_id | 1212 ctx_info->user_pri << VLAN_PRIO_SHIFT; 1213 } else { 1214 udp_info->insert_vlan_tag = false; 1215 } 1216 1217 av->attrs = attr->ah_attr; 1218 rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid_attr->gid); 1219 rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); 1220 if (av->net_type == RDMA_NETWORK_IPV6) { 1221 __be32 *daddr = 1222 av->dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1223 __be32 *saddr = 1224 av->sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32; 1225 1226 irdma_copy_ip_ntohl(&udp_info->dest_ip_addr[0], daddr); 1227 irdma_copy_ip_ntohl(&udp_info->local_ipaddr[0], saddr); 1228 1229 udp_info->ipv4 = false; 1230 irdma_copy_ip_ntohl(local_ip, daddr); 1231 1232 udp_info->arp_idx = irdma_arp_table(iwdev->rf, 1233 &local_ip[0], 1234 false, NULL, 1235 IRDMA_ARP_RESOLVE); 1236 } else if (av->net_type == RDMA_NETWORK_IPV4) { 1237 __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; 1238 __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; 1239 1240 local_ip[0] = ntohl(daddr); 1241 1242 udp_info->ipv4 = true; 1243 udp_info->dest_ip_addr[0] = 0; 1244 udp_info->dest_ip_addr[1] = 0; 1245 udp_info->dest_ip_addr[2] = 0; 1246 udp_info->dest_ip_addr[3] = local_ip[0]; 1247 1248 udp_info->local_ipaddr[0] = 0; 1249 udp_info->local_ipaddr[1] = 0; 1250 udp_info->local_ipaddr[2] = 0; 1251 udp_info->local_ipaddr[3] = ntohl(saddr); 1252 } 1253 udp_info->arp_idx = 1254 irdma_add_arp(iwdev->rf, local_ip, udp_info->ipv4, 1255 attr->ah_attr.roce.dmac); 1256 } 1257 1258 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { 1259 if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { 1260 ibdev_err(&iwdev->ibdev, 1261 "rd_atomic = %d, above max_hw_ord=%d\n", 1262 attr->max_rd_atomic, 1263 dev->hw_attrs.max_hw_ord); 1264 return -EINVAL; 1265 } 1266 if (attr->max_rd_atomic) 1267 roce_info->ord_size = attr->max_rd_atomic; 1268 info.ord_valid = true; 1269 } 1270 1271 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { 1272 if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { 1273 ibdev_err(&iwdev->ibdev, 1274 "rd_atomic = %d, above max_hw_ird=%d\n", 1275 attr->max_rd_atomic, 1276 dev->hw_attrs.max_hw_ird); 1277 return -EINVAL; 1278 } 1279 if (attr->max_dest_rd_atomic) 1280 roce_info->ird_size = attr->max_dest_rd_atomic; 1281 } 1282 1283 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1284 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1285 roce_info->wr_rdresp_en = true; 1286 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1287 roce_info->wr_rdresp_en = true; 1288 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1289 roce_info->rd_en = true; 1290 } 1291 1292 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1293 1294 ibdev_dbg(&iwdev->ibdev, 1295 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", 1296 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1297 iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); 1298 1299 spin_lock_irqsave(&iwqp->lock, flags); 1300 if (attr_mask & IB_QP_STATE) { 1301 if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, 1302 iwqp->ibqp.qp_type, attr_mask)) { 1303 ibdev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", 1304 iwqp->ibqp.qp_num, iwqp->ibqp_state, 1305 attr->qp_state); 1306 ret = -EINVAL; 1307 goto exit; 1308 } 1309 info.curr_iwarp_state = iwqp->iwarp_state; 1310 1311 switch (attr->qp_state) { 1312 case IB_QPS_INIT: 1313 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1314 ret = -EINVAL; 1315 goto exit; 1316 } 1317 1318 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1319 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1320 issue_modify_qp = 1; 1321 } 1322 break; 1323 case IB_QPS_RTR: 1324 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1325 ret = -EINVAL; 1326 goto exit; 1327 } 1328 info.arp_cache_idx_valid = true; 1329 info.cq_num_valid = true; 1330 info.next_iwarp_state = IRDMA_QP_STATE_RTR; 1331 issue_modify_qp = 1; 1332 break; 1333 case IB_QPS_RTS: 1334 if (iwqp->ibqp_state < IB_QPS_RTR || 1335 iwqp->ibqp_state == IB_QPS_ERR) { 1336 ret = -EINVAL; 1337 goto exit; 1338 } 1339 1340 info.arp_cache_idx_valid = true; 1341 info.cq_num_valid = true; 1342 info.ord_valid = true; 1343 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1344 issue_modify_qp = 1; 1345 if (iwdev->push_mode && udata && 1346 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1347 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1348 spin_unlock_irqrestore(&iwqp->lock, flags); 1349 irdma_alloc_push_page(iwqp); 1350 spin_lock_irqsave(&iwqp->lock, flags); 1351 } 1352 break; 1353 case IB_QPS_SQD: 1354 if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD) 1355 goto exit; 1356 1357 if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) { 1358 ret = -EINVAL; 1359 goto exit; 1360 } 1361 1362 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1363 issue_modify_qp = 1; 1364 break; 1365 case IB_QPS_SQE: 1366 case IB_QPS_ERR: 1367 case IB_QPS_RESET: 1368 if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { 1369 spin_unlock_irqrestore(&iwqp->lock, flags); 1370 info.next_iwarp_state = IRDMA_QP_STATE_SQD; 1371 irdma_hw_modify_qp(iwdev, iwqp, &info, true); 1372 spin_lock_irqsave(&iwqp->lock, flags); 1373 } 1374 1375 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1376 spin_unlock_irqrestore(&iwqp->lock, flags); 1377 if (udata) { 1378 if (ib_copy_from_udata(&ureq, udata, 1379 min(sizeof(ureq), udata->inlen))) 1380 return -EINVAL; 1381 1382 irdma_flush_wqes(iwqp, 1383 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1384 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1385 IRDMA_REFLUSH); 1386 } 1387 return 0; 1388 } 1389 1390 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1391 issue_modify_qp = 1; 1392 break; 1393 default: 1394 ret = -EINVAL; 1395 goto exit; 1396 } 1397 1398 iwqp->ibqp_state = attr->qp_state; 1399 } 1400 1401 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1402 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1403 irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1404 spin_unlock_irqrestore(&iwqp->lock, flags); 1405 1406 if (attr_mask & IB_QP_STATE) { 1407 if (issue_modify_qp) { 1408 ctx_info->rem_endpoint_idx = udp_info->arp_idx; 1409 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1410 return -EINVAL; 1411 spin_lock_irqsave(&iwqp->lock, flags); 1412 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1413 iwqp->iwarp_state = info.next_iwarp_state; 1414 iwqp->ibqp_state = attr->qp_state; 1415 } 1416 if (iwqp->ibqp_state > IB_QPS_RTS && 1417 !iwqp->flush_issued) { 1418 spin_unlock_irqrestore(&iwqp->lock, flags); 1419 irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | 1420 IRDMA_FLUSH_RQ | 1421 IRDMA_FLUSH_WAIT); 1422 iwqp->flush_issued = 1; 1423 } else { 1424 spin_unlock_irqrestore(&iwqp->lock, flags); 1425 } 1426 } else { 1427 iwqp->ibqp_state = attr->qp_state; 1428 } 1429 if (udata && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1430 struct irdma_ucontext *ucontext; 1431 1432 ucontext = rdma_udata_to_drv_context(udata, 1433 struct irdma_ucontext, ibucontext); 1434 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1435 !iwqp->push_wqe_mmap_entry && 1436 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1437 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1438 uresp.push_valid = 1; 1439 uresp.push_offset = iwqp->sc_qp.push_offset; 1440 } 1441 ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1442 udata->outlen)); 1443 if (ret) { 1444 irdma_remove_push_mmap_entries(iwqp); 1445 ibdev_dbg(&iwdev->ibdev, 1446 "VERBS: copy_to_udata failed\n"); 1447 return ret; 1448 } 1449 } 1450 } 1451 1452 return 0; 1453 exit: 1454 spin_unlock_irqrestore(&iwqp->lock, flags); 1455 1456 return ret; 1457 } 1458 1459 /** 1460 * irdma_modify_qp - modify qp request 1461 * @ibqp: qp's pointer for modify 1462 * @attr: access attributes 1463 * @attr_mask: state mask 1464 * @udata: user data 1465 */ 1466 int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1467 struct ib_udata *udata) 1468 { 1469 struct irdma_qp *iwqp = to_iwqp(ibqp); 1470 struct irdma_device *iwdev = iwqp->iwdev; 1471 struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; 1472 struct irdma_qp_host_ctx_info *ctx_info; 1473 struct irdma_tcp_offload_info *tcp_info; 1474 struct irdma_iwarp_offload_info *offload_info; 1475 struct irdma_modify_qp_info info = {}; 1476 struct irdma_modify_qp_resp uresp = {}; 1477 struct irdma_modify_qp_req ureq = {}; 1478 u8 issue_modify_qp = 0; 1479 u8 dont_wait = 0; 1480 int err; 1481 unsigned long flags; 1482 1483 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1484 return -EOPNOTSUPP; 1485 1486 ctx_info = &iwqp->ctx_info; 1487 offload_info = &iwqp->iwarp_info; 1488 tcp_info = &iwqp->tcp_info; 1489 wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); 1490 ibdev_dbg(&iwdev->ibdev, 1491 "VERBS: caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", 1492 __builtin_return_address(0), ibqp->qp_num, attr->qp_state, 1493 iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, 1494 iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); 1495 1496 spin_lock_irqsave(&iwqp->lock, flags); 1497 if (attr_mask & IB_QP_STATE) { 1498 info.curr_iwarp_state = iwqp->iwarp_state; 1499 switch (attr->qp_state) { 1500 case IB_QPS_INIT: 1501 case IB_QPS_RTR: 1502 if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) { 1503 err = -EINVAL; 1504 goto exit; 1505 } 1506 1507 if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) { 1508 info.next_iwarp_state = IRDMA_QP_STATE_IDLE; 1509 issue_modify_qp = 1; 1510 } 1511 if (iwdev->push_mode && udata && 1512 iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX && 1513 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1514 spin_unlock_irqrestore(&iwqp->lock, flags); 1515 irdma_alloc_push_page(iwqp); 1516 spin_lock_irqsave(&iwqp->lock, flags); 1517 } 1518 break; 1519 case IB_QPS_RTS: 1520 if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS || 1521 !iwqp->cm_id) { 1522 err = -EINVAL; 1523 goto exit; 1524 } 1525 1526 issue_modify_qp = 1; 1527 iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED; 1528 iwqp->hte_added = 1; 1529 info.next_iwarp_state = IRDMA_QP_STATE_RTS; 1530 info.tcp_ctx_valid = true; 1531 info.ord_valid = true; 1532 info.arp_cache_idx_valid = true; 1533 info.cq_num_valid = true; 1534 break; 1535 case IB_QPS_SQD: 1536 if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) { 1537 err = 0; 1538 goto exit; 1539 } 1540 1541 if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING || 1542 iwqp->iwarp_state < IRDMA_QP_STATE_RTS) { 1543 err = 0; 1544 goto exit; 1545 } 1546 1547 if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) { 1548 err = -EINVAL; 1549 goto exit; 1550 } 1551 1552 info.next_iwarp_state = IRDMA_QP_STATE_CLOSING; 1553 issue_modify_qp = 1; 1554 break; 1555 case IB_QPS_SQE: 1556 if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) { 1557 err = -EINVAL; 1558 goto exit; 1559 } 1560 1561 info.next_iwarp_state = IRDMA_QP_STATE_TERMINATE; 1562 issue_modify_qp = 1; 1563 break; 1564 case IB_QPS_ERR: 1565 case IB_QPS_RESET: 1566 if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { 1567 spin_unlock_irqrestore(&iwqp->lock, flags); 1568 if (udata) { 1569 if (ib_copy_from_udata(&ureq, udata, 1570 min(sizeof(ureq), udata->inlen))) 1571 return -EINVAL; 1572 1573 irdma_flush_wqes(iwqp, 1574 (ureq.sq_flush ? IRDMA_FLUSH_SQ : 0) | 1575 (ureq.rq_flush ? IRDMA_FLUSH_RQ : 0) | 1576 IRDMA_REFLUSH); 1577 } 1578 return 0; 1579 } 1580 1581 if (iwqp->sc_qp.term_flags) { 1582 spin_unlock_irqrestore(&iwqp->lock, flags); 1583 irdma_terminate_del_timer(&iwqp->sc_qp); 1584 spin_lock_irqsave(&iwqp->lock, flags); 1585 } 1586 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1587 if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED && 1588 iwdev->iw_status && 1589 iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT) 1590 info.reset_tcp_conn = true; 1591 else 1592 dont_wait = 1; 1593 1594 issue_modify_qp = 1; 1595 info.next_iwarp_state = IRDMA_QP_STATE_ERROR; 1596 break; 1597 default: 1598 err = -EINVAL; 1599 goto exit; 1600 } 1601 1602 iwqp->ibqp_state = attr->qp_state; 1603 } 1604 if (attr_mask & IB_QP_ACCESS_FLAGS) { 1605 ctx_info->iwarp_info_valid = true; 1606 if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) 1607 offload_info->wr_rdresp_en = true; 1608 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) 1609 offload_info->wr_rdresp_en = true; 1610 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) 1611 offload_info->rd_en = true; 1612 } 1613 1614 if (ctx_info->iwarp_info_valid) { 1615 ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id; 1616 ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id; 1617 irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info); 1618 } 1619 spin_unlock_irqrestore(&iwqp->lock, flags); 1620 1621 if (attr_mask & IB_QP_STATE) { 1622 if (issue_modify_qp) { 1623 ctx_info->rem_endpoint_idx = tcp_info->arp_idx; 1624 if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) 1625 return -EINVAL; 1626 } 1627 1628 spin_lock_irqsave(&iwqp->lock, flags); 1629 if (iwqp->iwarp_state == info.curr_iwarp_state) { 1630 iwqp->iwarp_state = info.next_iwarp_state; 1631 iwqp->ibqp_state = attr->qp_state; 1632 } 1633 spin_unlock_irqrestore(&iwqp->lock, flags); 1634 } 1635 1636 if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) { 1637 if (dont_wait) { 1638 if (iwqp->hw_tcp_state) { 1639 spin_lock_irqsave(&iwqp->lock, flags); 1640 iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED; 1641 iwqp->last_aeq = IRDMA_AE_RESET_SENT; 1642 spin_unlock_irqrestore(&iwqp->lock, flags); 1643 } 1644 irdma_cm_disconn(iwqp); 1645 } else { 1646 int close_timer_started; 1647 1648 spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags); 1649 1650 if (iwqp->cm_node) { 1651 refcount_inc(&iwqp->cm_node->refcnt); 1652 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1653 close_timer_started = atomic_inc_return(&iwqp->close_timer_started); 1654 if (iwqp->cm_id && close_timer_started == 1) 1655 irdma_schedule_cm_timer(iwqp->cm_node, 1656 (struct irdma_puda_buf *)iwqp, 1657 IRDMA_TIMER_TYPE_CLOSE, 1, 0); 1658 1659 irdma_rem_ref_cm_node(iwqp->cm_node); 1660 } else { 1661 spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags); 1662 } 1663 } 1664 } 1665 if (attr_mask & IB_QP_STATE && udata && 1666 dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { 1667 struct irdma_ucontext *ucontext; 1668 1669 ucontext = rdma_udata_to_drv_context(udata, 1670 struct irdma_ucontext, ibucontext); 1671 if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX && 1672 !iwqp->push_wqe_mmap_entry && 1673 !irdma_setup_push_mmap_entries(ucontext, iwqp, 1674 &uresp.push_wqe_mmap_key, &uresp.push_db_mmap_key)) { 1675 uresp.push_valid = 1; 1676 uresp.push_offset = iwqp->sc_qp.push_offset; 1677 } 1678 1679 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), 1680 udata->outlen)); 1681 if (err) { 1682 irdma_remove_push_mmap_entries(iwqp); 1683 ibdev_dbg(&iwdev->ibdev, 1684 "VERBS: copy_to_udata failed\n"); 1685 return err; 1686 } 1687 } 1688 1689 return 0; 1690 exit: 1691 spin_unlock_irqrestore(&iwqp->lock, flags); 1692 1693 return err; 1694 } 1695 1696 /** 1697 * irdma_cq_free_rsrc - free up resources for cq 1698 * @rf: RDMA PCI function 1699 * @iwcq: cq ptr 1700 */ 1701 static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) 1702 { 1703 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1704 1705 if (!iwcq->user_mode) { 1706 dma_free_coherent(rf->sc_dev.hw->device, iwcq->kmem.size, 1707 iwcq->kmem.va, iwcq->kmem.pa); 1708 iwcq->kmem.va = NULL; 1709 dma_free_coherent(rf->sc_dev.hw->device, 1710 iwcq->kmem_shadow.size, 1711 iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); 1712 iwcq->kmem_shadow.va = NULL; 1713 } 1714 1715 irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); 1716 } 1717 1718 /** 1719 * irdma_free_cqbuf - worker to free a cq buffer 1720 * @work: provides access to the cq buffer to free 1721 */ 1722 static void irdma_free_cqbuf(struct work_struct *work) 1723 { 1724 struct irdma_cq_buf *cq_buf = container_of(work, struct irdma_cq_buf, work); 1725 1726 dma_free_coherent(cq_buf->hw->device, cq_buf->kmem_buf.size, 1727 cq_buf->kmem_buf.va, cq_buf->kmem_buf.pa); 1728 cq_buf->kmem_buf.va = NULL; 1729 kfree(cq_buf); 1730 } 1731 1732 /** 1733 * irdma_process_resize_list - remove resized cq buffers from the resize_list 1734 * @iwcq: cq which owns the resize_list 1735 * @iwdev: irdma device 1736 * @lcqe_buf: the buffer where the last cqe is received 1737 */ 1738 static int irdma_process_resize_list(struct irdma_cq *iwcq, 1739 struct irdma_device *iwdev, 1740 struct irdma_cq_buf *lcqe_buf) 1741 { 1742 struct list_head *tmp_node, *list_node; 1743 struct irdma_cq_buf *cq_buf; 1744 int cnt = 0; 1745 1746 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 1747 cq_buf = list_entry(list_node, struct irdma_cq_buf, list); 1748 if (cq_buf == lcqe_buf) 1749 return cnt; 1750 1751 list_del(&cq_buf->list); 1752 queue_work(iwdev->cleanup_wq, &cq_buf->work); 1753 cnt++; 1754 } 1755 1756 return cnt; 1757 } 1758 1759 /** 1760 * irdma_destroy_cq - destroy cq 1761 * @ib_cq: cq pointer 1762 * @udata: user data 1763 */ 1764 static int irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) 1765 { 1766 struct irdma_device *iwdev = to_iwdev(ib_cq->device); 1767 struct irdma_cq *iwcq = to_iwcq(ib_cq); 1768 struct irdma_sc_cq *cq = &iwcq->sc_cq; 1769 struct irdma_sc_dev *dev = cq->dev; 1770 struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id]; 1771 struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq); 1772 unsigned long flags; 1773 1774 spin_lock_irqsave(&iwcq->lock, flags); 1775 if (!list_empty(&iwcq->cmpl_generated)) 1776 irdma_remove_cmpls_list(iwcq); 1777 if (!list_empty(&iwcq->resize_list)) 1778 irdma_process_resize_list(iwcq, iwdev, NULL); 1779 spin_unlock_irqrestore(&iwcq->lock, flags); 1780 1781 irdma_cq_wq_destroy(iwdev->rf, cq); 1782 1783 spin_lock_irqsave(&iwceq->ce_lock, flags); 1784 irdma_sc_cleanup_ceqes(cq, ceq); 1785 spin_unlock_irqrestore(&iwceq->ce_lock, flags); 1786 irdma_cq_free_rsrc(iwdev->rf, iwcq); 1787 1788 return 0; 1789 } 1790 1791 /** 1792 * irdma_resize_cq - resize cq 1793 * @ibcq: cq to be resized 1794 * @entries: desired cq size 1795 * @udata: user data 1796 */ 1797 static int irdma_resize_cq(struct ib_cq *ibcq, int entries, 1798 struct ib_udata *udata) 1799 { 1800 struct irdma_cq *iwcq = to_iwcq(ibcq); 1801 struct irdma_sc_dev *dev = iwcq->sc_cq.dev; 1802 struct irdma_cqp_request *cqp_request; 1803 struct cqp_cmds_info *cqp_info; 1804 struct irdma_modify_cq_info *m_info; 1805 struct irdma_modify_cq_info info = {}; 1806 struct irdma_dma_mem kmem_buf; 1807 struct irdma_cq_mr *cqmr_buf; 1808 struct irdma_pbl *iwpbl_buf; 1809 struct irdma_device *iwdev; 1810 struct irdma_pci_f *rf; 1811 struct irdma_cq_buf *cq_buf = NULL; 1812 unsigned long flags; 1813 int ret; 1814 1815 iwdev = to_iwdev(ibcq->device); 1816 rf = iwdev->rf; 1817 1818 if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 1819 IRDMA_FEATURE_CQ_RESIZE)) 1820 return -EOPNOTSUPP; 1821 1822 if (entries > rf->max_cqe) 1823 return -EINVAL; 1824 1825 if (!iwcq->user_mode) { 1826 entries++; 1827 if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 1828 entries *= 2; 1829 } 1830 1831 info.cq_size = max(entries, 4); 1832 1833 if (info.cq_size == iwcq->sc_cq.cq_uk.cq_size - 1) 1834 return 0; 1835 1836 if (udata) { 1837 struct irdma_resize_cq_req req = {}; 1838 struct irdma_ucontext *ucontext = 1839 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 1840 ibucontext); 1841 1842 /* CQ resize not supported with legacy GEN_1 libi40iw */ 1843 if (ucontext->legacy_mode) 1844 return -EOPNOTSUPP; 1845 1846 if (ib_copy_from_udata(&req, udata, 1847 min(sizeof(req), udata->inlen))) 1848 return -EINVAL; 1849 1850 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 1851 iwpbl_buf = irdma_get_pbl((unsigned long)req.user_cq_buffer, 1852 &ucontext->cq_reg_mem_list); 1853 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 1854 1855 if (!iwpbl_buf) 1856 return -ENOMEM; 1857 1858 cqmr_buf = &iwpbl_buf->cq_mr; 1859 if (iwpbl_buf->pbl_allocated) { 1860 info.virtual_map = true; 1861 info.pbl_chunk_size = 1; 1862 info.first_pm_pbl_idx = cqmr_buf->cq_pbl.idx; 1863 } else { 1864 info.cq_pa = cqmr_buf->cq_pbl.addr; 1865 } 1866 } else { 1867 /* Kmode CQ resize */ 1868 int rsize; 1869 1870 rsize = info.cq_size * sizeof(struct irdma_cqe); 1871 kmem_buf.size = ALIGN(round_up(rsize, 256), 256); 1872 kmem_buf.va = dma_alloc_coherent(dev->hw->device, 1873 kmem_buf.size, &kmem_buf.pa, 1874 GFP_KERNEL); 1875 if (!kmem_buf.va) 1876 return -ENOMEM; 1877 1878 info.cq_base = kmem_buf.va; 1879 info.cq_pa = kmem_buf.pa; 1880 cq_buf = kzalloc(sizeof(*cq_buf), GFP_KERNEL); 1881 if (!cq_buf) { 1882 ret = -ENOMEM; 1883 goto error; 1884 } 1885 } 1886 1887 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 1888 if (!cqp_request) { 1889 ret = -ENOMEM; 1890 goto error; 1891 } 1892 1893 info.shadow_read_threshold = iwcq->sc_cq.shadow_read_threshold; 1894 info.cq_resize = true; 1895 1896 cqp_info = &cqp_request->info; 1897 m_info = &cqp_info->in.u.cq_modify.info; 1898 memcpy(m_info, &info, sizeof(*m_info)); 1899 1900 cqp_info->cqp_cmd = IRDMA_OP_CQ_MODIFY; 1901 cqp_info->in.u.cq_modify.cq = &iwcq->sc_cq; 1902 cqp_info->in.u.cq_modify.scratch = (uintptr_t)cqp_request; 1903 cqp_info->post_sq = 1; 1904 ret = irdma_handle_cqp_op(rf, cqp_request); 1905 irdma_put_cqp_request(&rf->cqp, cqp_request); 1906 if (ret) 1907 goto error; 1908 1909 spin_lock_irqsave(&iwcq->lock, flags); 1910 if (cq_buf) { 1911 cq_buf->kmem_buf = iwcq->kmem; 1912 cq_buf->hw = dev->hw; 1913 memcpy(&cq_buf->cq_uk, &iwcq->sc_cq.cq_uk, sizeof(cq_buf->cq_uk)); 1914 INIT_WORK(&cq_buf->work, irdma_free_cqbuf); 1915 list_add_tail(&cq_buf->list, &iwcq->resize_list); 1916 iwcq->kmem = kmem_buf; 1917 } 1918 1919 irdma_sc_cq_resize(&iwcq->sc_cq, &info); 1920 ibcq->cqe = info.cq_size - 1; 1921 spin_unlock_irqrestore(&iwcq->lock, flags); 1922 1923 return 0; 1924 error: 1925 if (!udata) { 1926 dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, 1927 kmem_buf.pa); 1928 kmem_buf.va = NULL; 1929 } 1930 kfree(cq_buf); 1931 1932 return ret; 1933 } 1934 1935 static inline int cq_validate_flags(u32 flags, u8 hw_rev) 1936 { 1937 /* GEN1 does not support CQ create flags */ 1938 if (hw_rev == IRDMA_GEN_1) 1939 return flags ? -EOPNOTSUPP : 0; 1940 1941 return flags & ~IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION ? -EOPNOTSUPP : 0; 1942 } 1943 1944 /** 1945 * irdma_create_cq - create cq 1946 * @ibcq: CQ allocated 1947 * @attr: attributes for cq 1948 * @udata: user data 1949 */ 1950 static int irdma_create_cq(struct ib_cq *ibcq, 1951 const struct ib_cq_init_attr *attr, 1952 struct ib_udata *udata) 1953 { 1954 struct ib_device *ibdev = ibcq->device; 1955 struct irdma_device *iwdev = to_iwdev(ibdev); 1956 struct irdma_pci_f *rf = iwdev->rf; 1957 struct irdma_cq *iwcq = to_iwcq(ibcq); 1958 u32 cq_num = 0; 1959 struct irdma_sc_cq *cq; 1960 struct irdma_sc_dev *dev = &rf->sc_dev; 1961 struct irdma_cq_init_info info = {}; 1962 struct irdma_cqp_request *cqp_request; 1963 struct cqp_cmds_info *cqp_info; 1964 struct irdma_cq_uk_init_info *ukinfo = &info.cq_uk_init_info; 1965 unsigned long flags; 1966 int err_code; 1967 int entries = attr->cqe; 1968 1969 err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev); 1970 if (err_code) 1971 return err_code; 1972 err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num, 1973 &rf->next_cq); 1974 if (err_code) 1975 return err_code; 1976 1977 cq = &iwcq->sc_cq; 1978 cq->back_cq = iwcq; 1979 spin_lock_init(&iwcq->lock); 1980 INIT_LIST_HEAD(&iwcq->resize_list); 1981 INIT_LIST_HEAD(&iwcq->cmpl_generated); 1982 info.dev = dev; 1983 ukinfo->cq_size = max(entries, 4); 1984 ukinfo->cq_id = cq_num; 1985 iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; 1986 if (attr->comp_vector < rf->ceqs_count) 1987 info.ceq_id = attr->comp_vector; 1988 info.ceq_id_valid = true; 1989 info.ceqe_mask = 1; 1990 info.type = IRDMA_CQ_TYPE_IWARP; 1991 info.vsi = &iwdev->vsi; 1992 1993 if (udata) { 1994 struct irdma_ucontext *ucontext; 1995 struct irdma_create_cq_req req = {}; 1996 struct irdma_cq_mr *cqmr; 1997 struct irdma_pbl *iwpbl; 1998 struct irdma_pbl *iwpbl_shadow; 1999 struct irdma_cq_mr *cqmr_shadow; 2000 2001 iwcq->user_mode = true; 2002 ucontext = 2003 rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2004 ibucontext); 2005 if (ib_copy_from_udata(&req, udata, 2006 min(sizeof(req), udata->inlen))) { 2007 err_code = -EFAULT; 2008 goto cq_free_rsrc; 2009 } 2010 2011 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2012 iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, 2013 &ucontext->cq_reg_mem_list); 2014 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2015 if (!iwpbl) { 2016 err_code = -EPROTO; 2017 goto cq_free_rsrc; 2018 } 2019 2020 iwcq->iwpbl = iwpbl; 2021 iwcq->cq_mem_size = 0; 2022 cqmr = &iwpbl->cq_mr; 2023 2024 if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & 2025 IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { 2026 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2027 iwpbl_shadow = irdma_get_pbl( 2028 (unsigned long)req.user_shadow_area, 2029 &ucontext->cq_reg_mem_list); 2030 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2031 2032 if (!iwpbl_shadow) { 2033 err_code = -EPROTO; 2034 goto cq_free_rsrc; 2035 } 2036 iwcq->iwpbl_shadow = iwpbl_shadow; 2037 cqmr_shadow = &iwpbl_shadow->cq_mr; 2038 info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; 2039 cqmr->split = true; 2040 } else { 2041 info.shadow_area_pa = cqmr->shadow; 2042 } 2043 if (iwpbl->pbl_allocated) { 2044 info.virtual_map = true; 2045 info.pbl_chunk_size = 1; 2046 info.first_pm_pbl_idx = cqmr->cq_pbl.idx; 2047 } else { 2048 info.cq_base_pa = cqmr->cq_pbl.addr; 2049 } 2050 } else { 2051 /* Kmode allocations */ 2052 int rsize; 2053 2054 if (entries < 1 || entries > rf->max_cqe) { 2055 err_code = -EINVAL; 2056 goto cq_free_rsrc; 2057 } 2058 2059 entries++; 2060 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2061 entries *= 2; 2062 ukinfo->cq_size = entries; 2063 2064 rsize = info.cq_uk_init_info.cq_size * sizeof(struct irdma_cqe); 2065 iwcq->kmem.size = ALIGN(round_up(rsize, 256), 256); 2066 iwcq->kmem.va = dma_alloc_coherent(dev->hw->device, 2067 iwcq->kmem.size, 2068 &iwcq->kmem.pa, GFP_KERNEL); 2069 if (!iwcq->kmem.va) { 2070 err_code = -ENOMEM; 2071 goto cq_free_rsrc; 2072 } 2073 2074 iwcq->kmem_shadow.size = ALIGN(IRDMA_SHADOW_AREA_SIZE << 3, 2075 64); 2076 iwcq->kmem_shadow.va = dma_alloc_coherent(dev->hw->device, 2077 iwcq->kmem_shadow.size, 2078 &iwcq->kmem_shadow.pa, 2079 GFP_KERNEL); 2080 if (!iwcq->kmem_shadow.va) { 2081 err_code = -ENOMEM; 2082 goto cq_free_rsrc; 2083 } 2084 info.shadow_area_pa = iwcq->kmem_shadow.pa; 2085 ukinfo->shadow_area = iwcq->kmem_shadow.va; 2086 ukinfo->cq_base = iwcq->kmem.va; 2087 info.cq_base_pa = iwcq->kmem.pa; 2088 } 2089 2090 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) 2091 info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, 2092 (u32)IRDMA_MAX_CQ_READ_THRESH); 2093 2094 if (irdma_sc_cq_init(cq, &info)) { 2095 ibdev_dbg(&iwdev->ibdev, "VERBS: init cq fail\n"); 2096 err_code = -EPROTO; 2097 goto cq_free_rsrc; 2098 } 2099 2100 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); 2101 if (!cqp_request) { 2102 err_code = -ENOMEM; 2103 goto cq_free_rsrc; 2104 } 2105 2106 cqp_info = &cqp_request->info; 2107 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; 2108 cqp_info->post_sq = 1; 2109 cqp_info->in.u.cq_create.cq = cq; 2110 cqp_info->in.u.cq_create.check_overflow = true; 2111 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request; 2112 err_code = irdma_handle_cqp_op(rf, cqp_request); 2113 irdma_put_cqp_request(&rf->cqp, cqp_request); 2114 if (err_code) 2115 goto cq_free_rsrc; 2116 2117 if (udata) { 2118 struct irdma_create_cq_resp resp = {}; 2119 2120 resp.cq_id = info.cq_uk_init_info.cq_id; 2121 resp.cq_size = info.cq_uk_init_info.cq_size; 2122 if (ib_copy_to_udata(udata, &resp, 2123 min(sizeof(resp), udata->outlen))) { 2124 ibdev_dbg(&iwdev->ibdev, 2125 "VERBS: copy to user data\n"); 2126 err_code = -EPROTO; 2127 goto cq_destroy; 2128 } 2129 } 2130 return 0; 2131 cq_destroy: 2132 irdma_cq_wq_destroy(rf, cq); 2133 cq_free_rsrc: 2134 irdma_cq_free_rsrc(rf, iwcq); 2135 2136 return err_code; 2137 } 2138 2139 /** 2140 * irdma_get_mr_access - get hw MR access permissions from IB access flags 2141 * @access: IB access flags 2142 */ 2143 static inline u16 irdma_get_mr_access(int access) 2144 { 2145 u16 hw_access = 0; 2146 2147 hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ? 2148 IRDMA_ACCESS_FLAGS_LOCALWRITE : 0; 2149 hw_access |= (access & IB_ACCESS_REMOTE_WRITE) ? 2150 IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0; 2151 hw_access |= (access & IB_ACCESS_REMOTE_READ) ? 2152 IRDMA_ACCESS_FLAGS_REMOTEREAD : 0; 2153 hw_access |= (access & IB_ACCESS_MW_BIND) ? 2154 IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0; 2155 hw_access |= (access & IB_ZERO_BASED) ? 2156 IRDMA_ACCESS_FLAGS_ZERO_BASED : 0; 2157 hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD; 2158 2159 return hw_access; 2160 } 2161 2162 /** 2163 * irdma_free_stag - free stag resource 2164 * @iwdev: irdma device 2165 * @stag: stag to free 2166 */ 2167 static void irdma_free_stag(struct irdma_device *iwdev, u32 stag) 2168 { 2169 u32 stag_idx; 2170 2171 stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S; 2172 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx); 2173 } 2174 2175 /** 2176 * irdma_create_stag - create random stag 2177 * @iwdev: irdma device 2178 */ 2179 static u32 irdma_create_stag(struct irdma_device *iwdev) 2180 { 2181 u32 stag = 0; 2182 u32 stag_index = 0; 2183 u32 next_stag_index; 2184 u32 driver_key; 2185 u32 random; 2186 u8 consumer_key; 2187 int ret; 2188 2189 get_random_bytes(&random, sizeof(random)); 2190 consumer_key = (u8)random; 2191 2192 driver_key = random & ~iwdev->rf->mr_stagmask; 2193 next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8; 2194 next_stag_index %= iwdev->rf->max_mr; 2195 2196 ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, 2197 iwdev->rf->max_mr, &stag_index, 2198 &next_stag_index); 2199 if (ret) 2200 return stag; 2201 stag = stag_index << IRDMA_CQPSQ_STAG_IDX_S; 2202 stag |= driver_key; 2203 stag += (u32)consumer_key; 2204 2205 return stag; 2206 } 2207 2208 /** 2209 * irdma_next_pbl_addr - Get next pbl address 2210 * @pbl: pointer to a pble 2211 * @pinfo: info pointer 2212 * @idx: index 2213 */ 2214 static inline u64 *irdma_next_pbl_addr(u64 *pbl, struct irdma_pble_info **pinfo, 2215 u32 *idx) 2216 { 2217 *idx += 1; 2218 if (!(*pinfo) || *idx != (*pinfo)->cnt) 2219 return ++pbl; 2220 *idx = 0; 2221 (*pinfo)++; 2222 2223 return (*pinfo)->addr; 2224 } 2225 2226 /** 2227 * irdma_copy_user_pgaddrs - copy user page address to pble's os locally 2228 * @iwmr: iwmr for IB's user page addresses 2229 * @pbl: ple pointer to save 1 level or 0 level pble 2230 * @level: indicated level 0, 1 or 2 2231 */ 2232 static void irdma_copy_user_pgaddrs(struct irdma_mr *iwmr, u64 *pbl, 2233 enum irdma_pble_level level) 2234 { 2235 struct ib_umem *region = iwmr->region; 2236 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2237 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2238 struct irdma_pble_info *pinfo; 2239 struct ib_block_iter biter; 2240 u32 idx = 0; 2241 u32 pbl_cnt = 0; 2242 2243 pinfo = (level == PBLE_LEVEL_1) ? NULL : palloc->level2.leaf; 2244 2245 if (iwmr->type == IRDMA_MEMREG_TYPE_QP) 2246 iwpbl->qp_mr.sq_page = sg_page(region->sgt_append.sgt.sgl); 2247 2248 rdma_umem_for_each_dma_block(region, &biter, iwmr->page_size) { 2249 *pbl = rdma_block_iter_dma_address(&biter); 2250 if (++pbl_cnt == palloc->total_cnt) 2251 break; 2252 pbl = irdma_next_pbl_addr(pbl, &pinfo, &idx); 2253 } 2254 } 2255 2256 /** 2257 * irdma_check_mem_contiguous - check if pbls stored in arr are contiguous 2258 * @arr: lvl1 pbl array 2259 * @npages: page count 2260 * @pg_size: page size 2261 * 2262 */ 2263 static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) 2264 { 2265 u32 pg_idx; 2266 2267 for (pg_idx = 0; pg_idx < npages; pg_idx++) { 2268 if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) 2269 return false; 2270 } 2271 2272 return true; 2273 } 2274 2275 /** 2276 * irdma_check_mr_contiguous - check if MR is physically contiguous 2277 * @palloc: pbl allocation struct 2278 * @pg_size: page size 2279 */ 2280 static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, 2281 u32 pg_size) 2282 { 2283 struct irdma_pble_level2 *lvl2 = &palloc->level2; 2284 struct irdma_pble_info *leaf = lvl2->leaf; 2285 u64 *arr = NULL; 2286 u64 *start_addr = NULL; 2287 int i; 2288 bool ret; 2289 2290 if (palloc->level == PBLE_LEVEL_1) { 2291 arr = palloc->level1.addr; 2292 ret = irdma_check_mem_contiguous(arr, palloc->total_cnt, 2293 pg_size); 2294 return ret; 2295 } 2296 2297 start_addr = leaf->addr; 2298 2299 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 2300 arr = leaf->addr; 2301 if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) 2302 return false; 2303 ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); 2304 if (!ret) 2305 return false; 2306 } 2307 2308 return true; 2309 } 2310 2311 /** 2312 * irdma_setup_pbles - copy user pg address to pble's 2313 * @rf: RDMA PCI function 2314 * @iwmr: mr pointer for this memory registration 2315 * @use_pbles: flag if to use pble's 2316 */ 2317 static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, 2318 bool use_pbles) 2319 { 2320 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2321 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2322 struct irdma_pble_info *pinfo; 2323 u64 *pbl; 2324 int status; 2325 enum irdma_pble_level level = PBLE_LEVEL_1; 2326 2327 if (use_pbles) { 2328 status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, 2329 false); 2330 if (status) 2331 return status; 2332 2333 iwpbl->pbl_allocated = true; 2334 level = palloc->level; 2335 pinfo = (level == PBLE_LEVEL_1) ? &palloc->level1 : 2336 palloc->level2.leaf; 2337 pbl = pinfo->addr; 2338 } else { 2339 pbl = iwmr->pgaddrmem; 2340 } 2341 2342 irdma_copy_user_pgaddrs(iwmr, pbl, level); 2343 2344 if (use_pbles) 2345 iwmr->pgaddrmem[0] = *pbl; 2346 2347 return 0; 2348 } 2349 2350 /** 2351 * irdma_handle_q_mem - handle memory for qp and cq 2352 * @iwdev: irdma device 2353 * @req: information for q memory management 2354 * @iwpbl: pble struct 2355 * @use_pbles: flag to use pble 2356 */ 2357 static int irdma_handle_q_mem(struct irdma_device *iwdev, 2358 struct irdma_mem_reg_req *req, 2359 struct irdma_pbl *iwpbl, bool use_pbles) 2360 { 2361 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2362 struct irdma_mr *iwmr = iwpbl->iwmr; 2363 struct irdma_qp_mr *qpmr = &iwpbl->qp_mr; 2364 struct irdma_cq_mr *cqmr = &iwpbl->cq_mr; 2365 struct irdma_hmc_pble *hmc_p; 2366 u64 *arr = iwmr->pgaddrmem; 2367 u32 pg_size, total; 2368 int err = 0; 2369 bool ret = true; 2370 2371 pg_size = iwmr->page_size; 2372 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2373 if (err) 2374 return err; 2375 2376 if (use_pbles && palloc->level != PBLE_LEVEL_1) { 2377 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2378 iwpbl->pbl_allocated = false; 2379 return -ENOMEM; 2380 } 2381 2382 if (use_pbles) 2383 arr = palloc->level1.addr; 2384 2385 switch (iwmr->type) { 2386 case IRDMA_MEMREG_TYPE_QP: 2387 total = req->sq_pages + req->rq_pages; 2388 hmc_p = &qpmr->sq_pbl; 2389 qpmr->shadow = (dma_addr_t)arr[total]; 2390 2391 if (use_pbles) { 2392 ret = irdma_check_mem_contiguous(arr, req->sq_pages, 2393 pg_size); 2394 if (ret) 2395 ret = irdma_check_mem_contiguous(&arr[req->sq_pages], 2396 req->rq_pages, 2397 pg_size); 2398 } 2399 2400 if (!ret) { 2401 hmc_p->idx = palloc->level1.idx; 2402 hmc_p = &qpmr->rq_pbl; 2403 hmc_p->idx = palloc->level1.idx + req->sq_pages; 2404 } else { 2405 hmc_p->addr = arr[0]; 2406 hmc_p = &qpmr->rq_pbl; 2407 hmc_p->addr = arr[req->sq_pages]; 2408 } 2409 break; 2410 case IRDMA_MEMREG_TYPE_CQ: 2411 hmc_p = &cqmr->cq_pbl; 2412 2413 if (!cqmr->split) 2414 cqmr->shadow = (dma_addr_t)arr[req->cq_pages]; 2415 2416 if (use_pbles) 2417 ret = irdma_check_mem_contiguous(arr, req->cq_pages, 2418 pg_size); 2419 2420 if (!ret) 2421 hmc_p->idx = palloc->level1.idx; 2422 else 2423 hmc_p->addr = arr[0]; 2424 break; 2425 default: 2426 ibdev_dbg(&iwdev->ibdev, "VERBS: MR type error\n"); 2427 err = -EINVAL; 2428 } 2429 2430 if (use_pbles && ret) { 2431 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2432 iwpbl->pbl_allocated = false; 2433 } 2434 2435 return err; 2436 } 2437 2438 /** 2439 * irdma_hw_alloc_mw - create the hw memory window 2440 * @iwdev: irdma device 2441 * @iwmr: pointer to memory window info 2442 */ 2443 static int irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr) 2444 { 2445 struct irdma_mw_alloc_info *info; 2446 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2447 struct irdma_cqp_request *cqp_request; 2448 struct cqp_cmds_info *cqp_info; 2449 int status; 2450 2451 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2452 if (!cqp_request) 2453 return -ENOMEM; 2454 2455 cqp_info = &cqp_request->info; 2456 info = &cqp_info->in.u.mw_alloc.info; 2457 memset(info, 0, sizeof(*info)); 2458 if (iwmr->ibmw.type == IB_MW_TYPE_1) 2459 info->mw_wide = true; 2460 2461 info->page_size = PAGE_SIZE; 2462 info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2463 info->pd_id = iwpd->sc_pd.pd_id; 2464 info->remote_access = true; 2465 cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC; 2466 cqp_info->post_sq = 1; 2467 cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev; 2468 cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request; 2469 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2470 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2471 2472 return status; 2473 } 2474 2475 /** 2476 * irdma_alloc_mw - Allocate memory window 2477 * @ibmw: Memory Window 2478 * @udata: user data pointer 2479 */ 2480 static int irdma_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2481 { 2482 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2483 struct irdma_mr *iwmr = to_iwmw(ibmw); 2484 int err_code; 2485 u32 stag; 2486 2487 stag = irdma_create_stag(iwdev); 2488 if (!stag) 2489 return -ENOMEM; 2490 2491 iwmr->stag = stag; 2492 ibmw->rkey = stag; 2493 2494 err_code = irdma_hw_alloc_mw(iwdev, iwmr); 2495 if (err_code) { 2496 irdma_free_stag(iwdev, stag); 2497 return err_code; 2498 } 2499 2500 return 0; 2501 } 2502 2503 /** 2504 * irdma_dealloc_mw - Dealloc memory window 2505 * @ibmw: memory window structure. 2506 */ 2507 static int irdma_dealloc_mw(struct ib_mw *ibmw) 2508 { 2509 struct ib_pd *ibpd = ibmw->pd; 2510 struct irdma_pd *iwpd = to_iwpd(ibpd); 2511 struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw); 2512 struct irdma_device *iwdev = to_iwdev(ibmw->device); 2513 struct irdma_cqp_request *cqp_request; 2514 struct cqp_cmds_info *cqp_info; 2515 struct irdma_dealloc_stag_info *info; 2516 2517 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2518 if (!cqp_request) 2519 return -ENOMEM; 2520 2521 cqp_info = &cqp_request->info; 2522 info = &cqp_info->in.u.dealloc_stag.info; 2523 memset(info, 0, sizeof(*info)); 2524 info->pd_id = iwpd->sc_pd.pd_id; 2525 info->stag_idx = ibmw->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 2526 info->mr = false; 2527 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 2528 cqp_info->post_sq = 1; 2529 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 2530 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 2531 irdma_handle_cqp_op(iwdev->rf, cqp_request); 2532 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2533 irdma_free_stag(iwdev, iwmr->stag); 2534 2535 return 0; 2536 } 2537 2538 /** 2539 * irdma_hw_alloc_stag - cqp command to allocate stag 2540 * @iwdev: irdma device 2541 * @iwmr: irdma mr pointer 2542 */ 2543 static int irdma_hw_alloc_stag(struct irdma_device *iwdev, 2544 struct irdma_mr *iwmr) 2545 { 2546 struct irdma_allocate_stag_info *info; 2547 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2548 int status; 2549 struct irdma_cqp_request *cqp_request; 2550 struct cqp_cmds_info *cqp_info; 2551 2552 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2553 if (!cqp_request) 2554 return -ENOMEM; 2555 2556 cqp_info = &cqp_request->info; 2557 info = &cqp_info->in.u.alloc_stag.info; 2558 memset(info, 0, sizeof(*info)); 2559 info->page_size = PAGE_SIZE; 2560 info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2561 info->pd_id = iwpd->sc_pd.pd_id; 2562 info->total_len = iwmr->len; 2563 info->remote_access = true; 2564 cqp_info->cqp_cmd = IRDMA_OP_ALLOC_STAG; 2565 cqp_info->post_sq = 1; 2566 cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev; 2567 cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; 2568 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2569 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2570 2571 return status; 2572 } 2573 2574 /** 2575 * irdma_alloc_mr - register stag for fast memory registration 2576 * @pd: ibpd pointer 2577 * @mr_type: memory for stag registrion 2578 * @max_num_sg: man number of pages 2579 */ 2580 static struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2581 u32 max_num_sg) 2582 { 2583 struct irdma_device *iwdev = to_iwdev(pd->device); 2584 struct irdma_pble_alloc *palloc; 2585 struct irdma_pbl *iwpbl; 2586 struct irdma_mr *iwmr; 2587 u32 stag; 2588 int err_code; 2589 2590 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2591 if (!iwmr) 2592 return ERR_PTR(-ENOMEM); 2593 2594 stag = irdma_create_stag(iwdev); 2595 if (!stag) { 2596 err_code = -ENOMEM; 2597 goto err; 2598 } 2599 2600 iwmr->stag = stag; 2601 iwmr->ibmr.rkey = stag; 2602 iwmr->ibmr.lkey = stag; 2603 iwmr->ibmr.pd = pd; 2604 iwmr->ibmr.device = pd->device; 2605 iwpbl = &iwmr->iwpbl; 2606 iwpbl->iwmr = iwmr; 2607 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2608 palloc = &iwpbl->pble_alloc; 2609 iwmr->page_cnt = max_num_sg; 2610 err_code = irdma_get_pble(iwdev->rf->pble_rsrc, palloc, iwmr->page_cnt, 2611 false); 2612 if (err_code) 2613 goto err_get_pble; 2614 2615 err_code = irdma_hw_alloc_stag(iwdev, iwmr); 2616 if (err_code) 2617 goto err_alloc_stag; 2618 2619 iwpbl->pbl_allocated = true; 2620 2621 return &iwmr->ibmr; 2622 err_alloc_stag: 2623 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2624 err_get_pble: 2625 irdma_free_stag(iwdev, stag); 2626 err: 2627 kfree(iwmr); 2628 2629 return ERR_PTR(err_code); 2630 } 2631 2632 /** 2633 * irdma_set_page - populate pbl list for fmr 2634 * @ibmr: ib mem to access iwarp mr pointer 2635 * @addr: page dma address fro pbl list 2636 */ 2637 static int irdma_set_page(struct ib_mr *ibmr, u64 addr) 2638 { 2639 struct irdma_mr *iwmr = to_iwmr(ibmr); 2640 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2641 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2642 u64 *pbl; 2643 2644 if (unlikely(iwmr->npages == iwmr->page_cnt)) 2645 return -ENOMEM; 2646 2647 if (palloc->level == PBLE_LEVEL_2) { 2648 struct irdma_pble_info *palloc_info = 2649 palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT); 2650 2651 palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr; 2652 } else { 2653 pbl = palloc->level1.addr; 2654 pbl[iwmr->npages] = addr; 2655 } 2656 iwmr->npages++; 2657 2658 return 0; 2659 } 2660 2661 /** 2662 * irdma_map_mr_sg - map of sg list for fmr 2663 * @ibmr: ib mem to access iwarp mr pointer 2664 * @sg: scatter gather list 2665 * @sg_nents: number of sg pages 2666 * @sg_offset: scatter gather list for fmr 2667 */ 2668 static int irdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, 2669 int sg_nents, unsigned int *sg_offset) 2670 { 2671 struct irdma_mr *iwmr = to_iwmr(ibmr); 2672 2673 iwmr->npages = 0; 2674 2675 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, irdma_set_page); 2676 } 2677 2678 /** 2679 * irdma_hwreg_mr - send cqp command for memory registration 2680 * @iwdev: irdma device 2681 * @iwmr: irdma mr pointer 2682 * @access: access for MR 2683 */ 2684 static int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr, 2685 u16 access) 2686 { 2687 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2688 struct irdma_reg_ns_stag_info *stag_info; 2689 struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd); 2690 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 2691 struct irdma_cqp_request *cqp_request; 2692 struct cqp_cmds_info *cqp_info; 2693 int ret; 2694 2695 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 2696 if (!cqp_request) 2697 return -ENOMEM; 2698 2699 cqp_info = &cqp_request->info; 2700 stag_info = &cqp_info->in.u.mr_reg_non_shared.info; 2701 memset(stag_info, 0, sizeof(*stag_info)); 2702 stag_info->va = iwpbl->user_base; 2703 stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S; 2704 stag_info->stag_key = (u8)iwmr->stag; 2705 stag_info->total_len = iwmr->len; 2706 stag_info->access_rights = irdma_get_mr_access(access); 2707 stag_info->pd_id = iwpd->sc_pd.pd_id; 2708 if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED) 2709 stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED; 2710 else 2711 stag_info->addr_type = IRDMA_ADDR_TYPE_VA_BASED; 2712 stag_info->page_size = iwmr->page_size; 2713 2714 if (iwpbl->pbl_allocated) { 2715 if (palloc->level == PBLE_LEVEL_1) { 2716 stag_info->first_pm_pbl_index = palloc->level1.idx; 2717 stag_info->chunk_size = 1; 2718 } else { 2719 stag_info->first_pm_pbl_index = palloc->level2.root.idx; 2720 stag_info->chunk_size = 3; 2721 } 2722 } else { 2723 stag_info->reg_addr_pa = iwmr->pgaddrmem[0]; 2724 } 2725 2726 cqp_info->cqp_cmd = IRDMA_OP_MR_REG_NON_SHARED; 2727 cqp_info->post_sq = 1; 2728 cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev; 2729 cqp_info->in.u.mr_reg_non_shared.scratch = (uintptr_t)cqp_request; 2730 ret = irdma_handle_cqp_op(iwdev->rf, cqp_request); 2731 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 2732 2733 return ret; 2734 } 2735 2736 /** 2737 * irdma_reg_user_mr - Register a user memory region 2738 * @pd: ptr of pd 2739 * @start: virtual start address 2740 * @len: length of mr 2741 * @virt: virtual address 2742 * @access: access of mr 2743 * @udata: user data 2744 */ 2745 static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len, 2746 u64 virt, int access, 2747 struct ib_udata *udata) 2748 { 2749 struct irdma_device *iwdev = to_iwdev(pd->device); 2750 struct irdma_ucontext *ucontext; 2751 struct irdma_pble_alloc *palloc; 2752 struct irdma_pbl *iwpbl; 2753 struct irdma_mr *iwmr; 2754 struct ib_umem *region; 2755 struct irdma_mem_reg_req req; 2756 u32 total, stag = 0; 2757 u8 shadow_pgcnt = 1; 2758 bool use_pbles = false; 2759 unsigned long flags; 2760 int err = -EINVAL; 2761 int ret; 2762 2763 if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) 2764 return ERR_PTR(-EINVAL); 2765 2766 region = ib_umem_get(pd->device, start, len, access); 2767 2768 if (IS_ERR(region)) { 2769 ibdev_dbg(&iwdev->ibdev, 2770 "VERBS: Failed to create ib_umem region\n"); 2771 return (struct ib_mr *)region; 2772 } 2773 2774 if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) { 2775 ib_umem_release(region); 2776 return ERR_PTR(-EFAULT); 2777 } 2778 2779 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2780 if (!iwmr) { 2781 ib_umem_release(region); 2782 return ERR_PTR(-ENOMEM); 2783 } 2784 2785 iwpbl = &iwmr->iwpbl; 2786 iwpbl->iwmr = iwmr; 2787 iwmr->region = region; 2788 iwmr->ibmr.pd = pd; 2789 iwmr->ibmr.device = pd->device; 2790 iwmr->ibmr.iova = virt; 2791 iwmr->page_size = PAGE_SIZE; 2792 2793 if (req.reg_type == IRDMA_MEMREG_TYPE_MEM) { 2794 iwmr->page_size = ib_umem_find_best_pgsz(region, 2795 iwdev->rf->sc_dev.hw_attrs.page_size_cap, 2796 virt); 2797 if (unlikely(!iwmr->page_size)) { 2798 kfree(iwmr); 2799 ib_umem_release(region); 2800 return ERR_PTR(-EOPNOTSUPP); 2801 } 2802 } 2803 iwmr->len = region->length; 2804 iwpbl->user_base = virt; 2805 palloc = &iwpbl->pble_alloc; 2806 iwmr->type = req.reg_type; 2807 iwmr->page_cnt = ib_umem_num_dma_blocks(region, iwmr->page_size); 2808 2809 switch (req.reg_type) { 2810 case IRDMA_MEMREG_TYPE_QP: 2811 total = req.sq_pages + req.rq_pages + shadow_pgcnt; 2812 if (total > iwmr->page_cnt) { 2813 err = -EINVAL; 2814 goto error; 2815 } 2816 total = req.sq_pages + req.rq_pages; 2817 use_pbles = (total > 2); 2818 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2819 if (err) 2820 goto error; 2821 2822 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2823 ibucontext); 2824 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2825 list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); 2826 iwpbl->on_list = true; 2827 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2828 break; 2829 case IRDMA_MEMREG_TYPE_CQ: 2830 if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) 2831 shadow_pgcnt = 0; 2832 total = req.cq_pages + shadow_pgcnt; 2833 if (total > iwmr->page_cnt) { 2834 err = -EINVAL; 2835 goto error; 2836 } 2837 2838 use_pbles = (req.cq_pages > 1); 2839 err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); 2840 if (err) 2841 goto error; 2842 2843 ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, 2844 ibucontext); 2845 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2846 list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); 2847 iwpbl->on_list = true; 2848 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2849 break; 2850 case IRDMA_MEMREG_TYPE_MEM: 2851 use_pbles = (iwmr->page_cnt != 1); 2852 2853 err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles); 2854 if (err) 2855 goto error; 2856 2857 if (use_pbles) { 2858 ret = irdma_check_mr_contiguous(palloc, 2859 iwmr->page_size); 2860 if (ret) { 2861 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2862 iwpbl->pbl_allocated = false; 2863 } 2864 } 2865 2866 stag = irdma_create_stag(iwdev); 2867 if (!stag) { 2868 err = -ENOMEM; 2869 goto error; 2870 } 2871 2872 iwmr->stag = stag; 2873 iwmr->ibmr.rkey = stag; 2874 iwmr->ibmr.lkey = stag; 2875 err = irdma_hwreg_mr(iwdev, iwmr, access); 2876 if (err) { 2877 irdma_free_stag(iwdev, stag); 2878 goto error; 2879 } 2880 2881 break; 2882 default: 2883 goto error; 2884 } 2885 2886 iwmr->type = req.reg_type; 2887 2888 return &iwmr->ibmr; 2889 2890 error: 2891 if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) 2892 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 2893 ib_umem_release(region); 2894 kfree(iwmr); 2895 2896 return ERR_PTR(err); 2897 } 2898 2899 /** 2900 * irdma_reg_phys_mr - register kernel physical memory 2901 * @pd: ibpd pointer 2902 * @addr: physical address of memory to register 2903 * @size: size of memory to register 2904 * @access: Access rights 2905 * @iova_start: start of virtual address for physical buffers 2906 */ 2907 struct ib_mr *irdma_reg_phys_mr(struct ib_pd *pd, u64 addr, u64 size, int access, 2908 u64 *iova_start) 2909 { 2910 struct irdma_device *iwdev = to_iwdev(pd->device); 2911 struct irdma_pbl *iwpbl; 2912 struct irdma_mr *iwmr; 2913 u32 stag; 2914 int ret; 2915 2916 iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); 2917 if (!iwmr) 2918 return ERR_PTR(-ENOMEM); 2919 2920 iwmr->ibmr.pd = pd; 2921 iwmr->ibmr.device = pd->device; 2922 iwpbl = &iwmr->iwpbl; 2923 iwpbl->iwmr = iwmr; 2924 iwmr->type = IRDMA_MEMREG_TYPE_MEM; 2925 iwpbl->user_base = *iova_start; 2926 stag = irdma_create_stag(iwdev); 2927 if (!stag) { 2928 ret = -ENOMEM; 2929 goto err; 2930 } 2931 2932 iwmr->stag = stag; 2933 iwmr->ibmr.iova = *iova_start; 2934 iwmr->ibmr.rkey = stag; 2935 iwmr->ibmr.lkey = stag; 2936 iwmr->page_cnt = 1; 2937 iwmr->pgaddrmem[0] = addr; 2938 iwmr->len = size; 2939 iwmr->page_size = SZ_4K; 2940 ret = irdma_hwreg_mr(iwdev, iwmr, access); 2941 if (ret) { 2942 irdma_free_stag(iwdev, stag); 2943 goto err; 2944 } 2945 2946 return &iwmr->ibmr; 2947 2948 err: 2949 kfree(iwmr); 2950 2951 return ERR_PTR(ret); 2952 } 2953 2954 /** 2955 * irdma_get_dma_mr - register physical mem 2956 * @pd: ptr of pd 2957 * @acc: access for memory 2958 */ 2959 static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) 2960 { 2961 u64 kva = 0; 2962 2963 return irdma_reg_phys_mr(pd, 0, 0, acc, &kva); 2964 } 2965 2966 /** 2967 * irdma_del_memlist - Deleting pbl list entries for CQ/QP 2968 * @iwmr: iwmr for IB's user page addresses 2969 * @ucontext: ptr to user context 2970 */ 2971 static void irdma_del_memlist(struct irdma_mr *iwmr, 2972 struct irdma_ucontext *ucontext) 2973 { 2974 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 2975 unsigned long flags; 2976 2977 switch (iwmr->type) { 2978 case IRDMA_MEMREG_TYPE_CQ: 2979 spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); 2980 if (iwpbl->on_list) { 2981 iwpbl->on_list = false; 2982 list_del(&iwpbl->list); 2983 } 2984 spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); 2985 break; 2986 case IRDMA_MEMREG_TYPE_QP: 2987 spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); 2988 if (iwpbl->on_list) { 2989 iwpbl->on_list = false; 2990 list_del(&iwpbl->list); 2991 } 2992 spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); 2993 break; 2994 default: 2995 break; 2996 } 2997 } 2998 2999 /** 3000 * irdma_dereg_mr - deregister mr 3001 * @ib_mr: mr ptr for dereg 3002 * @udata: user data 3003 */ 3004 static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) 3005 { 3006 struct ib_pd *ibpd = ib_mr->pd; 3007 struct irdma_pd *iwpd = to_iwpd(ibpd); 3008 struct irdma_mr *iwmr = to_iwmr(ib_mr); 3009 struct irdma_device *iwdev = to_iwdev(ib_mr->device); 3010 struct irdma_dealloc_stag_info *info; 3011 struct irdma_pbl *iwpbl = &iwmr->iwpbl; 3012 struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; 3013 struct irdma_cqp_request *cqp_request; 3014 struct cqp_cmds_info *cqp_info; 3015 int status; 3016 3017 if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) { 3018 if (iwmr->region) { 3019 struct irdma_ucontext *ucontext; 3020 3021 ucontext = rdma_udata_to_drv_context(udata, 3022 struct irdma_ucontext, 3023 ibucontext); 3024 irdma_del_memlist(iwmr, ucontext); 3025 } 3026 goto done; 3027 } 3028 3029 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3030 if (!cqp_request) 3031 return -ENOMEM; 3032 3033 cqp_info = &cqp_request->info; 3034 info = &cqp_info->in.u.dealloc_stag.info; 3035 memset(info, 0, sizeof(*info)); 3036 info->pd_id = iwpd->sc_pd.pd_id; 3037 info->stag_idx = ib_mr->rkey >> IRDMA_CQPSQ_STAG_IDX_S; 3038 info->mr = true; 3039 if (iwpbl->pbl_allocated) 3040 info->dealloc_pbl = true; 3041 3042 cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG; 3043 cqp_info->post_sq = 1; 3044 cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev; 3045 cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request; 3046 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3047 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3048 if (status) 3049 return status; 3050 3051 irdma_free_stag(iwdev, iwmr->stag); 3052 done: 3053 if (iwpbl->pbl_allocated) 3054 irdma_free_pble(iwdev->rf->pble_rsrc, palloc); 3055 ib_umem_release(iwmr->region); 3056 kfree(iwmr); 3057 3058 return 0; 3059 } 3060 3061 /** 3062 * irdma_post_send - kernel application wr 3063 * @ibqp: qp ptr for wr 3064 * @ib_wr: work request ptr 3065 * @bad_wr: return of bad wr if err 3066 */ 3067 static int irdma_post_send(struct ib_qp *ibqp, 3068 const struct ib_send_wr *ib_wr, 3069 const struct ib_send_wr **bad_wr) 3070 { 3071 struct irdma_qp *iwqp; 3072 struct irdma_qp_uk *ukqp; 3073 struct irdma_sc_dev *dev; 3074 struct irdma_post_sq_info info; 3075 int err = 0; 3076 unsigned long flags; 3077 bool inv_stag; 3078 struct irdma_ah *ah; 3079 3080 iwqp = to_iwqp(ibqp); 3081 ukqp = &iwqp->sc_qp.qp_uk; 3082 dev = &iwqp->iwdev->rf->sc_dev; 3083 3084 spin_lock_irqsave(&iwqp->lock, flags); 3085 while (ib_wr) { 3086 memset(&info, 0, sizeof(info)); 3087 inv_stag = false; 3088 info.wr_id = (ib_wr->wr_id); 3089 if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) 3090 info.signaled = true; 3091 if (ib_wr->send_flags & IB_SEND_FENCE) 3092 info.read_fence = true; 3093 switch (ib_wr->opcode) { 3094 case IB_WR_SEND_WITH_IMM: 3095 if (ukqp->qp_caps & IRDMA_SEND_WITH_IMM) { 3096 info.imm_data_valid = true; 3097 info.imm_data = ntohl(ib_wr->ex.imm_data); 3098 } else { 3099 err = -EINVAL; 3100 break; 3101 } 3102 fallthrough; 3103 case IB_WR_SEND: 3104 case IB_WR_SEND_WITH_INV: 3105 if (ib_wr->opcode == IB_WR_SEND || 3106 ib_wr->opcode == IB_WR_SEND_WITH_IMM) { 3107 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3108 info.op_type = IRDMA_OP_TYPE_SEND_SOL; 3109 else 3110 info.op_type = IRDMA_OP_TYPE_SEND; 3111 } else { 3112 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3113 info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV; 3114 else 3115 info.op_type = IRDMA_OP_TYPE_SEND_INV; 3116 info.stag_to_inv = ib_wr->ex.invalidate_rkey; 3117 } 3118 3119 if (ib_wr->send_flags & IB_SEND_INLINE) { 3120 info.op.inline_send.data = (void *)(unsigned long) 3121 ib_wr->sg_list[0].addr; 3122 info.op.inline_send.len = ib_wr->sg_list[0].length; 3123 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3124 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3125 ah = to_iwah(ud_wr(ib_wr)->ah); 3126 info.op.inline_send.ah_id = ah->sc_ah.ah_info.ah_idx; 3127 info.op.inline_send.qkey = ud_wr(ib_wr)->remote_qkey; 3128 info.op.inline_send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3129 } 3130 err = irdma_uk_inline_send(ukqp, &info, false); 3131 } else { 3132 info.op.send.num_sges = ib_wr->num_sge; 3133 info.op.send.sg_list = ib_wr->sg_list; 3134 if (iwqp->ibqp.qp_type == IB_QPT_UD || 3135 iwqp->ibqp.qp_type == IB_QPT_GSI) { 3136 ah = to_iwah(ud_wr(ib_wr)->ah); 3137 info.op.send.ah_id = ah->sc_ah.ah_info.ah_idx; 3138 info.op.send.qkey = ud_wr(ib_wr)->remote_qkey; 3139 info.op.send.dest_qp = ud_wr(ib_wr)->remote_qpn; 3140 } 3141 err = irdma_uk_send(ukqp, &info, false); 3142 } 3143 break; 3144 case IB_WR_RDMA_WRITE_WITH_IMM: 3145 if (ukqp->qp_caps & IRDMA_WRITE_WITH_IMM) { 3146 info.imm_data_valid = true; 3147 info.imm_data = ntohl(ib_wr->ex.imm_data); 3148 } else { 3149 err = -EINVAL; 3150 break; 3151 } 3152 fallthrough; 3153 case IB_WR_RDMA_WRITE: 3154 if (ib_wr->send_flags & IB_SEND_SOLICITED) 3155 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL; 3156 else 3157 info.op_type = IRDMA_OP_TYPE_RDMA_WRITE; 3158 3159 if (ib_wr->send_flags & IB_SEND_INLINE) { 3160 info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr; 3161 info.op.inline_rdma_write.len = 3162 ib_wr->sg_list[0].length; 3163 info.op.inline_rdma_write.rem_addr.addr = 3164 rdma_wr(ib_wr)->remote_addr; 3165 info.op.inline_rdma_write.rem_addr.lkey = 3166 rdma_wr(ib_wr)->rkey; 3167 err = irdma_uk_inline_rdma_write(ukqp, &info, false); 3168 } else { 3169 info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list; 3170 info.op.rdma_write.num_lo_sges = ib_wr->num_sge; 3171 info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3172 info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3173 err = irdma_uk_rdma_write(ukqp, &info, false); 3174 } 3175 break; 3176 case IB_WR_RDMA_READ_WITH_INV: 3177 inv_stag = true; 3178 fallthrough; 3179 case IB_WR_RDMA_READ: 3180 if (ib_wr->num_sge > 3181 dev->hw_attrs.uk_attrs.max_hw_read_sges) { 3182 err = -EINVAL; 3183 break; 3184 } 3185 info.op_type = IRDMA_OP_TYPE_RDMA_READ; 3186 info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr; 3187 info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey; 3188 info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list; 3189 info.op.rdma_read.num_lo_sges = ib_wr->num_sge; 3190 err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false); 3191 break; 3192 case IB_WR_LOCAL_INV: 3193 info.op_type = IRDMA_OP_TYPE_INV_STAG; 3194 info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; 3195 err = irdma_uk_stag_local_invalidate(ukqp, &info, true); 3196 break; 3197 case IB_WR_REG_MR: { 3198 struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); 3199 struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; 3200 struct irdma_fast_reg_stag_info stag_info = {}; 3201 3202 stag_info.signaled = info.signaled; 3203 stag_info.read_fence = info.read_fence; 3204 stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access); 3205 stag_info.stag_key = reg_wr(ib_wr)->key & 0xff; 3206 stag_info.stag_idx = reg_wr(ib_wr)->key >> 8; 3207 stag_info.page_size = reg_wr(ib_wr)->mr->page_size; 3208 stag_info.wr_id = ib_wr->wr_id; 3209 stag_info.addr_type = IRDMA_ADDR_TYPE_VA_BASED; 3210 stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova; 3211 stag_info.total_len = iwmr->ibmr.length; 3212 stag_info.reg_addr_pa = *palloc->level1.addr; 3213 stag_info.first_pm_pbl_index = palloc->level1.idx; 3214 stag_info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; 3215 if (iwmr->npages > IRDMA_MIN_PAGES_PER_FMR) 3216 stag_info.chunk_size = 1; 3217 err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info, 3218 true); 3219 break; 3220 } 3221 default: 3222 err = -EINVAL; 3223 ibdev_dbg(&iwqp->iwdev->ibdev, 3224 "VERBS: upost_send bad opcode = 0x%x\n", 3225 ib_wr->opcode); 3226 break; 3227 } 3228 3229 if (err) 3230 break; 3231 ib_wr = ib_wr->next; 3232 } 3233 3234 if (!iwqp->flush_issued) { 3235 if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS) 3236 irdma_uk_qp_post_wr(ukqp); 3237 spin_unlock_irqrestore(&iwqp->lock, flags); 3238 } else { 3239 spin_unlock_irqrestore(&iwqp->lock, flags); 3240 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 3241 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 3242 } 3243 if (err) 3244 *bad_wr = ib_wr; 3245 3246 return err; 3247 } 3248 3249 /** 3250 * irdma_post_recv - post receive wr for kernel application 3251 * @ibqp: ib qp pointer 3252 * @ib_wr: work request for receive 3253 * @bad_wr: bad wr caused an error 3254 */ 3255 static int irdma_post_recv(struct ib_qp *ibqp, 3256 const struct ib_recv_wr *ib_wr, 3257 const struct ib_recv_wr **bad_wr) 3258 { 3259 struct irdma_qp *iwqp; 3260 struct irdma_qp_uk *ukqp; 3261 struct irdma_post_rq_info post_recv = {}; 3262 unsigned long flags; 3263 int err = 0; 3264 3265 iwqp = to_iwqp(ibqp); 3266 ukqp = &iwqp->sc_qp.qp_uk; 3267 3268 spin_lock_irqsave(&iwqp->lock, flags); 3269 while (ib_wr) { 3270 post_recv.num_sges = ib_wr->num_sge; 3271 post_recv.wr_id = ib_wr->wr_id; 3272 post_recv.sg_list = ib_wr->sg_list; 3273 err = irdma_uk_post_receive(ukqp, &post_recv); 3274 if (err) { 3275 ibdev_dbg(&iwqp->iwdev->ibdev, 3276 "VERBS: post_recv err %d\n", err); 3277 goto out; 3278 } 3279 3280 ib_wr = ib_wr->next; 3281 } 3282 3283 out: 3284 spin_unlock_irqrestore(&iwqp->lock, flags); 3285 if (iwqp->flush_issued) 3286 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, 3287 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)); 3288 3289 if (err) 3290 *bad_wr = ib_wr; 3291 3292 return err; 3293 } 3294 3295 /** 3296 * irdma_flush_err_to_ib_wc_status - return change flush error code to IB status 3297 * @opcode: iwarp flush code 3298 */ 3299 static enum ib_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode) 3300 { 3301 switch (opcode) { 3302 case FLUSH_PROT_ERR: 3303 return IB_WC_LOC_PROT_ERR; 3304 case FLUSH_REM_ACCESS_ERR: 3305 return IB_WC_REM_ACCESS_ERR; 3306 case FLUSH_LOC_QP_OP_ERR: 3307 return IB_WC_LOC_QP_OP_ERR; 3308 case FLUSH_REM_OP_ERR: 3309 return IB_WC_REM_OP_ERR; 3310 case FLUSH_LOC_LEN_ERR: 3311 return IB_WC_LOC_LEN_ERR; 3312 case FLUSH_GENERAL_ERR: 3313 return IB_WC_WR_FLUSH_ERR; 3314 case FLUSH_RETRY_EXC_ERR: 3315 return IB_WC_RETRY_EXC_ERR; 3316 case FLUSH_MW_BIND_ERR: 3317 return IB_WC_MW_BIND_ERR; 3318 case FLUSH_FATAL_ERR: 3319 default: 3320 return IB_WC_FATAL_ERR; 3321 } 3322 } 3323 3324 /** 3325 * irdma_process_cqe - process cqe info 3326 * @entry: processed cqe 3327 * @cq_poll_info: cqe info 3328 */ 3329 static void irdma_process_cqe(struct ib_wc *entry, 3330 struct irdma_cq_poll_info *cq_poll_info) 3331 { 3332 struct irdma_qp *iwqp; 3333 struct irdma_sc_qp *qp; 3334 3335 entry->wc_flags = 0; 3336 entry->pkey_index = 0; 3337 entry->wr_id = cq_poll_info->wr_id; 3338 3339 qp = cq_poll_info->qp_handle; 3340 iwqp = qp->qp_uk.back_qp; 3341 entry->qp = qp->qp_uk.back_qp; 3342 3343 if (cq_poll_info->error) { 3344 entry->status = (cq_poll_info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ? 3345 irdma_flush_err_to_ib_wc_status(cq_poll_info->minor_err) : IB_WC_GENERAL_ERR; 3346 3347 entry->vendor_err = cq_poll_info->major_err << 16 | 3348 cq_poll_info->minor_err; 3349 } else { 3350 entry->status = IB_WC_SUCCESS; 3351 if (cq_poll_info->imm_valid) { 3352 entry->ex.imm_data = htonl(cq_poll_info->imm_data); 3353 entry->wc_flags |= IB_WC_WITH_IMM; 3354 } 3355 if (cq_poll_info->ud_smac_valid) { 3356 ether_addr_copy(entry->smac, cq_poll_info->ud_smac); 3357 entry->wc_flags |= IB_WC_WITH_SMAC; 3358 } 3359 3360 if (cq_poll_info->ud_vlan_valid) { 3361 u16 vlan = cq_poll_info->ud_vlan & VLAN_VID_MASK; 3362 3363 entry->sl = cq_poll_info->ud_vlan >> VLAN_PRIO_SHIFT; 3364 if (vlan) { 3365 entry->vlan_id = vlan; 3366 entry->wc_flags |= IB_WC_WITH_VLAN; 3367 } 3368 } else { 3369 entry->sl = 0; 3370 } 3371 } 3372 3373 switch (cq_poll_info->op_type) { 3374 case IRDMA_OP_TYPE_RDMA_WRITE: 3375 case IRDMA_OP_TYPE_RDMA_WRITE_SOL: 3376 entry->opcode = IB_WC_RDMA_WRITE; 3377 break; 3378 case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: 3379 case IRDMA_OP_TYPE_RDMA_READ: 3380 entry->opcode = IB_WC_RDMA_READ; 3381 break; 3382 case IRDMA_OP_TYPE_SEND_INV: 3383 case IRDMA_OP_TYPE_SEND_SOL: 3384 case IRDMA_OP_TYPE_SEND_SOL_INV: 3385 case IRDMA_OP_TYPE_SEND: 3386 entry->opcode = IB_WC_SEND; 3387 break; 3388 case IRDMA_OP_TYPE_FAST_REG_NSMR: 3389 entry->opcode = IB_WC_REG_MR; 3390 break; 3391 case IRDMA_OP_TYPE_INV_STAG: 3392 entry->opcode = IB_WC_LOCAL_INV; 3393 break; 3394 case IRDMA_OP_TYPE_REC_IMM: 3395 case IRDMA_OP_TYPE_REC: 3396 entry->opcode = cq_poll_info->op_type == IRDMA_OP_TYPE_REC_IMM ? 3397 IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; 3398 if (qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_UD && 3399 cq_poll_info->stag_invalid_set) { 3400 entry->ex.invalidate_rkey = cq_poll_info->inv_stag; 3401 entry->wc_flags |= IB_WC_WITH_INVALIDATE; 3402 } 3403 break; 3404 default: 3405 ibdev_err(&iwqp->iwdev->ibdev, 3406 "Invalid opcode = %d in CQE\n", cq_poll_info->op_type); 3407 entry->status = IB_WC_GENERAL_ERR; 3408 return; 3409 } 3410 3411 if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) { 3412 entry->src_qp = cq_poll_info->ud_src_qpn; 3413 entry->slid = 0; 3414 entry->wc_flags |= 3415 (IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE); 3416 entry->network_hdr_type = cq_poll_info->ipv4 ? 3417 RDMA_NETWORK_IPV4 : 3418 RDMA_NETWORK_IPV6; 3419 } else { 3420 entry->src_qp = cq_poll_info->qp_id; 3421 } 3422 3423 entry->byte_len = cq_poll_info->bytes_xfered; 3424 } 3425 3426 /** 3427 * irdma_poll_one - poll one entry of the CQ 3428 * @ukcq: ukcq to poll 3429 * @cur_cqe: current CQE info to be filled in 3430 * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ 3431 * 3432 * Returns the internal irdma device error code or 0 on success 3433 */ 3434 static inline int irdma_poll_one(struct irdma_cq_uk *ukcq, 3435 struct irdma_cq_poll_info *cur_cqe, 3436 struct ib_wc *entry) 3437 { 3438 int ret = irdma_uk_cq_poll_cmpl(ukcq, cur_cqe); 3439 3440 if (ret) 3441 return ret; 3442 3443 irdma_process_cqe(entry, cur_cqe); 3444 3445 return 0; 3446 } 3447 3448 /** 3449 * __irdma_poll_cq - poll cq for completion (kernel apps) 3450 * @iwcq: cq to poll 3451 * @num_entries: number of entries to poll 3452 * @entry: wr of a completed entry 3453 */ 3454 static int __irdma_poll_cq(struct irdma_cq *iwcq, int num_entries, struct ib_wc *entry) 3455 { 3456 struct list_head *tmp_node, *list_node; 3457 struct irdma_cq_buf *last_buf = NULL; 3458 struct irdma_cq_poll_info *cur_cqe = &iwcq->cur_cqe; 3459 struct irdma_cq_buf *cq_buf; 3460 int ret; 3461 struct irdma_device *iwdev; 3462 struct irdma_cq_uk *ukcq; 3463 bool cq_new_cqe = false; 3464 int resized_bufs = 0; 3465 int npolled = 0; 3466 3467 iwdev = to_iwdev(iwcq->ibcq.device); 3468 ukcq = &iwcq->sc_cq.cq_uk; 3469 3470 /* go through the list of previously resized CQ buffers */ 3471 list_for_each_safe(list_node, tmp_node, &iwcq->resize_list) { 3472 cq_buf = container_of(list_node, struct irdma_cq_buf, list); 3473 while (npolled < num_entries) { 3474 ret = irdma_poll_one(&cq_buf->cq_uk, cur_cqe, entry + npolled); 3475 if (!ret) { 3476 ++npolled; 3477 cq_new_cqe = true; 3478 continue; 3479 } 3480 if (ret == -ENOENT) 3481 break; 3482 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3483 if (ret == -EFAULT) { 3484 cq_new_cqe = true; 3485 continue; 3486 } 3487 goto error; 3488 } 3489 3490 /* save the resized CQ buffer which received the last cqe */ 3491 if (cq_new_cqe) 3492 last_buf = cq_buf; 3493 cq_new_cqe = false; 3494 } 3495 3496 /* check the current CQ for new cqes */ 3497 while (npolled < num_entries) { 3498 ret = irdma_poll_one(ukcq, cur_cqe, entry + npolled); 3499 if (ret == -ENOENT) { 3500 ret = irdma_generated_cmpls(iwcq, cur_cqe); 3501 if (!ret) 3502 irdma_process_cqe(entry + npolled, cur_cqe); 3503 } 3504 if (!ret) { 3505 ++npolled; 3506 cq_new_cqe = true; 3507 continue; 3508 } 3509 3510 if (ret == -ENOENT) 3511 break; 3512 /* QP using the CQ is destroyed. Skip reporting this CQE */ 3513 if (ret == -EFAULT) { 3514 cq_new_cqe = true; 3515 continue; 3516 } 3517 goto error; 3518 } 3519 3520 if (cq_new_cqe) 3521 /* all previous CQ resizes are complete */ 3522 resized_bufs = irdma_process_resize_list(iwcq, iwdev, NULL); 3523 else if (last_buf) 3524 /* only CQ resizes up to the last_buf are complete */ 3525 resized_bufs = irdma_process_resize_list(iwcq, iwdev, last_buf); 3526 if (resized_bufs) 3527 /* report to the HW the number of complete CQ resizes */ 3528 irdma_uk_cq_set_resized_cnt(ukcq, resized_bufs); 3529 3530 return npolled; 3531 error: 3532 ibdev_dbg(&iwdev->ibdev, "%s: Error polling CQ, irdma_err: %d\n", 3533 __func__, ret); 3534 3535 return ret; 3536 } 3537 3538 /** 3539 * irdma_poll_cq - poll cq for completion (kernel apps) 3540 * @ibcq: cq to poll 3541 * @num_entries: number of entries to poll 3542 * @entry: wr of a completed entry 3543 */ 3544 static int irdma_poll_cq(struct ib_cq *ibcq, int num_entries, 3545 struct ib_wc *entry) 3546 { 3547 struct irdma_cq *iwcq; 3548 unsigned long flags; 3549 int ret; 3550 3551 iwcq = to_iwcq(ibcq); 3552 3553 spin_lock_irqsave(&iwcq->lock, flags); 3554 ret = __irdma_poll_cq(iwcq, num_entries, entry); 3555 spin_unlock_irqrestore(&iwcq->lock, flags); 3556 3557 return ret; 3558 } 3559 3560 /** 3561 * irdma_req_notify_cq - arm cq kernel application 3562 * @ibcq: cq to arm 3563 * @notify_flags: notofication flags 3564 */ 3565 static int irdma_req_notify_cq(struct ib_cq *ibcq, 3566 enum ib_cq_notify_flags notify_flags) 3567 { 3568 struct irdma_cq *iwcq; 3569 struct irdma_cq_uk *ukcq; 3570 unsigned long flags; 3571 enum irdma_cmpl_notify cq_notify; 3572 bool promo_event = false; 3573 int ret = 0; 3574 3575 cq_notify = notify_flags == IB_CQ_SOLICITED ? 3576 IRDMA_CQ_COMPL_SOLICITED : IRDMA_CQ_COMPL_EVENT; 3577 iwcq = to_iwcq(ibcq); 3578 ukcq = &iwcq->sc_cq.cq_uk; 3579 3580 spin_lock_irqsave(&iwcq->lock, flags); 3581 /* Only promote to arm the CQ for any event if the last arm event was solicited. */ 3582 if (iwcq->last_notify == IRDMA_CQ_COMPL_SOLICITED && notify_flags != IB_CQ_SOLICITED) 3583 promo_event = true; 3584 3585 if (!atomic_cmpxchg(&iwcq->armed, 0, 1) || promo_event) { 3586 iwcq->last_notify = cq_notify; 3587 irdma_uk_cq_request_notification(ukcq, cq_notify); 3588 } 3589 3590 if ((notify_flags & IB_CQ_REPORT_MISSED_EVENTS) && 3591 (!irdma_cq_empty(iwcq) || !list_empty(&iwcq->cmpl_generated))) 3592 ret = 1; 3593 spin_unlock_irqrestore(&iwcq->lock, flags); 3594 3595 return ret; 3596 } 3597 3598 static int irdma_roce_port_immutable(struct ib_device *ibdev, u32 port_num, 3599 struct ib_port_immutable *immutable) 3600 { 3601 struct ib_port_attr attr; 3602 int err; 3603 3604 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 3605 err = ib_query_port(ibdev, port_num, &attr); 3606 if (err) 3607 return err; 3608 3609 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 3610 immutable->pkey_tbl_len = attr.pkey_tbl_len; 3611 immutable->gid_tbl_len = attr.gid_tbl_len; 3612 3613 return 0; 3614 } 3615 3616 static int irdma_iw_port_immutable(struct ib_device *ibdev, u32 port_num, 3617 struct ib_port_immutable *immutable) 3618 { 3619 struct ib_port_attr attr; 3620 int err; 3621 3622 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 3623 err = ib_query_port(ibdev, port_num, &attr); 3624 if (err) 3625 return err; 3626 immutable->gid_tbl_len = attr.gid_tbl_len; 3627 3628 return 0; 3629 } 3630 3631 static const struct rdma_stat_desc irdma_hw_stat_descs[] = { 3632 /* 32bit names */ 3633 [IRDMA_HW_STAT_INDEX_RXVLANERR].name = "rxVlanErrors", 3634 [IRDMA_HW_STAT_INDEX_IP4RXDISCARD].name = "ip4InDiscards", 3635 [IRDMA_HW_STAT_INDEX_IP4RXTRUNC].name = "ip4InTruncatedPkts", 3636 [IRDMA_HW_STAT_INDEX_IP4TXNOROUTE].name = "ip4OutNoRoutes", 3637 [IRDMA_HW_STAT_INDEX_IP6RXDISCARD].name = "ip6InDiscards", 3638 [IRDMA_HW_STAT_INDEX_IP6RXTRUNC].name = "ip6InTruncatedPkts", 3639 [IRDMA_HW_STAT_INDEX_IP6TXNOROUTE].name = "ip6OutNoRoutes", 3640 [IRDMA_HW_STAT_INDEX_TCPRTXSEG].name = "tcpRetransSegs", 3641 [IRDMA_HW_STAT_INDEX_TCPRXOPTERR].name = "tcpInOptErrors", 3642 [IRDMA_HW_STAT_INDEX_TCPRXPROTOERR].name = "tcpInProtoErrors", 3643 [IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED].name = "cnpHandled", 3644 [IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED].name = "cnpIgnored", 3645 [IRDMA_HW_STAT_INDEX_TXNPCNPSENT].name = "cnpSent", 3646 3647 /* 64bit names */ 3648 [IRDMA_HW_STAT_INDEX_IP4RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3649 "ip4InOctets", 3650 [IRDMA_HW_STAT_INDEX_IP4RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3651 "ip4InPkts", 3652 [IRDMA_HW_STAT_INDEX_IP4RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3653 "ip4InReasmRqd", 3654 [IRDMA_HW_STAT_INDEX_IP4RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3655 "ip4InMcastOctets", 3656 [IRDMA_HW_STAT_INDEX_IP4RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3657 "ip4InMcastPkts", 3658 [IRDMA_HW_STAT_INDEX_IP4TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3659 "ip4OutOctets", 3660 [IRDMA_HW_STAT_INDEX_IP4TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3661 "ip4OutPkts", 3662 [IRDMA_HW_STAT_INDEX_IP4TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3663 "ip4OutSegRqd", 3664 [IRDMA_HW_STAT_INDEX_IP4TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3665 "ip4OutMcastOctets", 3666 [IRDMA_HW_STAT_INDEX_IP4TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3667 "ip4OutMcastPkts", 3668 [IRDMA_HW_STAT_INDEX_IP6RXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3669 "ip6InOctets", 3670 [IRDMA_HW_STAT_INDEX_IP6RXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3671 "ip6InPkts", 3672 [IRDMA_HW_STAT_INDEX_IP6RXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3673 "ip6InReasmRqd", 3674 [IRDMA_HW_STAT_INDEX_IP6RXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3675 "ip6InMcastOctets", 3676 [IRDMA_HW_STAT_INDEX_IP6RXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3677 "ip6InMcastPkts", 3678 [IRDMA_HW_STAT_INDEX_IP6TXOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3679 "ip6OutOctets", 3680 [IRDMA_HW_STAT_INDEX_IP6TXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3681 "ip6OutPkts", 3682 [IRDMA_HW_STAT_INDEX_IP6TXFRAGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3683 "ip6OutSegRqd", 3684 [IRDMA_HW_STAT_INDEX_IP6TXMCOCTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3685 "ip6OutMcastOctets", 3686 [IRDMA_HW_STAT_INDEX_IP6TXMCPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3687 "ip6OutMcastPkts", 3688 [IRDMA_HW_STAT_INDEX_TCPRXSEGS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3689 "tcpInSegs", 3690 [IRDMA_HW_STAT_INDEX_TCPTXSEG + IRDMA_HW_STAT_INDEX_MAX_32].name = 3691 "tcpOutSegs", 3692 [IRDMA_HW_STAT_INDEX_RDMARXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3693 "iwInRdmaReads", 3694 [IRDMA_HW_STAT_INDEX_RDMARXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3695 "iwInRdmaSends", 3696 [IRDMA_HW_STAT_INDEX_RDMARXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3697 "iwInRdmaWrites", 3698 [IRDMA_HW_STAT_INDEX_RDMATXRDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3699 "iwOutRdmaReads", 3700 [IRDMA_HW_STAT_INDEX_RDMATXSNDS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3701 "iwOutRdmaSends", 3702 [IRDMA_HW_STAT_INDEX_RDMATXWRS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3703 "iwOutRdmaWrites", 3704 [IRDMA_HW_STAT_INDEX_RDMAVBND + IRDMA_HW_STAT_INDEX_MAX_32].name = 3705 "iwRdmaBnd", 3706 [IRDMA_HW_STAT_INDEX_RDMAVINV + IRDMA_HW_STAT_INDEX_MAX_32].name = 3707 "iwRdmaInv", 3708 [IRDMA_HW_STAT_INDEX_UDPRXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3709 "RxUDP", 3710 [IRDMA_HW_STAT_INDEX_UDPTXPKTS + IRDMA_HW_STAT_INDEX_MAX_32].name = 3711 "TxUDP", 3712 [IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS + IRDMA_HW_STAT_INDEX_MAX_32] 3713 .name = "RxECNMrkd", 3714 }; 3715 3716 static void irdma_get_dev_fw_str(struct ib_device *dev, char *str) 3717 { 3718 struct irdma_device *iwdev = to_iwdev(dev); 3719 3720 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", 3721 irdma_fw_major_ver(&iwdev->rf->sc_dev), 3722 irdma_fw_minor_ver(&iwdev->rf->sc_dev)); 3723 } 3724 3725 /** 3726 * irdma_alloc_hw_port_stats - Allocate a hw stats structure 3727 * @ibdev: device pointer from stack 3728 * @port_num: port number 3729 */ 3730 static struct rdma_hw_stats *irdma_alloc_hw_port_stats(struct ib_device *ibdev, 3731 u32 port_num) 3732 { 3733 int num_counters = IRDMA_HW_STAT_INDEX_MAX_32 + 3734 IRDMA_HW_STAT_INDEX_MAX_64; 3735 unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; 3736 3737 BUILD_BUG_ON(ARRAY_SIZE(irdma_hw_stat_descs) != 3738 (IRDMA_HW_STAT_INDEX_MAX_32 + IRDMA_HW_STAT_INDEX_MAX_64)); 3739 3740 return rdma_alloc_hw_stats_struct(irdma_hw_stat_descs, num_counters, 3741 lifespan); 3742 } 3743 3744 /** 3745 * irdma_get_hw_stats - Populates the rdma_hw_stats structure 3746 * @ibdev: device pointer from stack 3747 * @stats: stats pointer from stack 3748 * @port_num: port number 3749 * @index: which hw counter the stack is requesting we update 3750 */ 3751 static int irdma_get_hw_stats(struct ib_device *ibdev, 3752 struct rdma_hw_stats *stats, u32 port_num, 3753 int index) 3754 { 3755 struct irdma_device *iwdev = to_iwdev(ibdev); 3756 struct irdma_dev_hw_stats *hw_stats = &iwdev->vsi.pestat->hw_stats; 3757 3758 if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) 3759 irdma_cqp_gather_stats_cmd(&iwdev->rf->sc_dev, iwdev->vsi.pestat, true); 3760 else 3761 irdma_cqp_gather_stats_gen1(&iwdev->rf->sc_dev, iwdev->vsi.pestat); 3762 3763 memcpy(&stats->value[0], hw_stats, sizeof(*hw_stats)); 3764 3765 return stats->num_counters; 3766 } 3767 3768 /** 3769 * irdma_query_gid - Query port GID 3770 * @ibdev: device pointer from stack 3771 * @port: port number 3772 * @index: Entry index 3773 * @gid: Global ID 3774 */ 3775 static int irdma_query_gid(struct ib_device *ibdev, u32 port, int index, 3776 union ib_gid *gid) 3777 { 3778 struct irdma_device *iwdev = to_iwdev(ibdev); 3779 3780 memset(gid->raw, 0, sizeof(gid->raw)); 3781 ether_addr_copy(gid->raw, iwdev->netdev->dev_addr); 3782 3783 return 0; 3784 } 3785 3786 /** 3787 * mcast_list_add - Add a new mcast item to list 3788 * @rf: RDMA PCI function 3789 * @new_elem: pointer to element to add 3790 */ 3791 static void mcast_list_add(struct irdma_pci_f *rf, 3792 struct mc_table_list *new_elem) 3793 { 3794 list_add(&new_elem->list, &rf->mc_qht_list.list); 3795 } 3796 3797 /** 3798 * mcast_list_del - Remove an mcast item from list 3799 * @mc_qht_elem: pointer to mcast table list element 3800 */ 3801 static void mcast_list_del(struct mc_table_list *mc_qht_elem) 3802 { 3803 if (mc_qht_elem) 3804 list_del(&mc_qht_elem->list); 3805 } 3806 3807 /** 3808 * mcast_list_lookup_ip - Search mcast list for address 3809 * @rf: RDMA PCI function 3810 * @ip_mcast: pointer to mcast IP address 3811 */ 3812 static struct mc_table_list *mcast_list_lookup_ip(struct irdma_pci_f *rf, 3813 u32 *ip_mcast) 3814 { 3815 struct mc_table_list *mc_qht_el; 3816 struct list_head *pos, *q; 3817 3818 list_for_each_safe (pos, q, &rf->mc_qht_list.list) { 3819 mc_qht_el = list_entry(pos, struct mc_table_list, list); 3820 if (!memcmp(mc_qht_el->mc_info.dest_ip, ip_mcast, 3821 sizeof(mc_qht_el->mc_info.dest_ip))) 3822 return mc_qht_el; 3823 } 3824 3825 return NULL; 3826 } 3827 3828 /** 3829 * irdma_mcast_cqp_op - perform a mcast cqp operation 3830 * @iwdev: irdma device 3831 * @mc_grp_ctx: mcast group info 3832 * @op: operation 3833 * 3834 * returns error status 3835 */ 3836 static int irdma_mcast_cqp_op(struct irdma_device *iwdev, 3837 struct irdma_mcast_grp_info *mc_grp_ctx, u8 op) 3838 { 3839 struct cqp_cmds_info *cqp_info; 3840 struct irdma_cqp_request *cqp_request; 3841 int status; 3842 3843 cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true); 3844 if (!cqp_request) 3845 return -ENOMEM; 3846 3847 cqp_request->info.in.u.mc_create.info = *mc_grp_ctx; 3848 cqp_info = &cqp_request->info; 3849 cqp_info->cqp_cmd = op; 3850 cqp_info->post_sq = 1; 3851 cqp_info->in.u.mc_create.scratch = (uintptr_t)cqp_request; 3852 cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp; 3853 status = irdma_handle_cqp_op(iwdev->rf, cqp_request); 3854 irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); 3855 3856 return status; 3857 } 3858 3859 /** 3860 * irdma_mcast_mac - Get the multicast MAC for an IP address 3861 * @ip_addr: IPv4 or IPv6 address 3862 * @mac: pointer to result MAC address 3863 * @ipv4: flag indicating IPv4 or IPv6 3864 * 3865 */ 3866 void irdma_mcast_mac(u32 *ip_addr, u8 *mac, bool ipv4) 3867 { 3868 u8 *ip = (u8 *)ip_addr; 3869 3870 if (ipv4) { 3871 unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, 0x00, 3872 0x00, 0x00}; 3873 3874 mac4[3] = ip[2] & 0x7F; 3875 mac4[4] = ip[1]; 3876 mac4[5] = ip[0]; 3877 ether_addr_copy(mac, mac4); 3878 } else { 3879 unsigned char mac6[ETH_ALEN] = {0x33, 0x33, 0x00, 0x00, 3880 0x00, 0x00}; 3881 3882 mac6[2] = ip[3]; 3883 mac6[3] = ip[2]; 3884 mac6[4] = ip[1]; 3885 mac6[5] = ip[0]; 3886 ether_addr_copy(mac, mac6); 3887 } 3888 } 3889 3890 /** 3891 * irdma_attach_mcast - attach a qp to a multicast group 3892 * @ibqp: ptr to qp 3893 * @ibgid: pointer to global ID 3894 * @lid: local ID 3895 * 3896 * returns error status 3897 */ 3898 static int irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 3899 { 3900 struct irdma_qp *iwqp = to_iwqp(ibqp); 3901 struct irdma_device *iwdev = iwqp->iwdev; 3902 struct irdma_pci_f *rf = iwdev->rf; 3903 struct mc_table_list *mc_qht_elem; 3904 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 3905 unsigned long flags; 3906 u32 ip_addr[4] = {}; 3907 u32 mgn; 3908 u32 no_mgs; 3909 int ret = 0; 3910 bool ipv4; 3911 u16 vlan_id; 3912 union irdma_sockaddr sgid_addr; 3913 unsigned char dmac[ETH_ALEN]; 3914 3915 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 3916 3917 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) { 3918 irdma_copy_ip_ntohl(ip_addr, 3919 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 3920 irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); 3921 ipv4 = false; 3922 ibdev_dbg(&iwdev->ibdev, 3923 "VERBS: qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, 3924 ip_addr); 3925 irdma_mcast_mac(ip_addr, dmac, false); 3926 } else { 3927 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 3928 ipv4 = true; 3929 vlan_id = irdma_get_vlan_ipv4(ip_addr); 3930 irdma_mcast_mac(ip_addr, dmac, true); 3931 ibdev_dbg(&iwdev->ibdev, 3932 "VERBS: qp_id=%d, IP4address=%pI4, MAC=%pM\n", 3933 ibqp->qp_num, ip_addr, dmac); 3934 } 3935 3936 spin_lock_irqsave(&rf->qh_list_lock, flags); 3937 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 3938 if (!mc_qht_elem) { 3939 struct irdma_dma_mem *dma_mem_mc; 3940 3941 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3942 mc_qht_elem = kzalloc(sizeof(*mc_qht_elem), GFP_KERNEL); 3943 if (!mc_qht_elem) 3944 return -ENOMEM; 3945 3946 mc_qht_elem->mc_info.ipv4_valid = ipv4; 3947 memcpy(mc_qht_elem->mc_info.dest_ip, ip_addr, 3948 sizeof(mc_qht_elem->mc_info.dest_ip)); 3949 ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg, 3950 &mgn, &rf->next_mcg); 3951 if (ret) { 3952 kfree(mc_qht_elem); 3953 return -ENOMEM; 3954 } 3955 3956 mc_qht_elem->mc_info.mgn = mgn; 3957 dma_mem_mc = &mc_qht_elem->mc_grp_ctx.dma_mem_mc; 3958 dma_mem_mc->size = ALIGN(sizeof(u64) * IRDMA_MAX_MGS_PER_CTX, 3959 IRDMA_HW_PAGE_SIZE); 3960 dma_mem_mc->va = dma_alloc_coherent(rf->hw.device, 3961 dma_mem_mc->size, 3962 &dma_mem_mc->pa, 3963 GFP_KERNEL); 3964 if (!dma_mem_mc->va) { 3965 irdma_free_rsrc(rf, rf->allocated_mcgs, mgn); 3966 kfree(mc_qht_elem); 3967 return -ENOMEM; 3968 } 3969 3970 mc_qht_elem->mc_grp_ctx.mg_id = (u16)mgn; 3971 memcpy(mc_qht_elem->mc_grp_ctx.dest_ip_addr, ip_addr, 3972 sizeof(mc_qht_elem->mc_grp_ctx.dest_ip_addr)); 3973 mc_qht_elem->mc_grp_ctx.ipv4_valid = ipv4; 3974 mc_qht_elem->mc_grp_ctx.vlan_id = vlan_id; 3975 if (vlan_id < VLAN_N_VID) 3976 mc_qht_elem->mc_grp_ctx.vlan_valid = true; 3977 mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->vsi.fcn_id; 3978 mc_qht_elem->mc_grp_ctx.qs_handle = 3979 iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle; 3980 ether_addr_copy(mc_qht_elem->mc_grp_ctx.dest_mac_addr, dmac); 3981 3982 spin_lock_irqsave(&rf->qh_list_lock, flags); 3983 mcast_list_add(rf, mc_qht_elem); 3984 } else { 3985 if (mc_qht_elem->mc_grp_ctx.no_of_mgs == 3986 IRDMA_MAX_MGS_PER_CTX) { 3987 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3988 return -ENOMEM; 3989 } 3990 } 3991 3992 mcg_info.qp_id = iwqp->ibqp.qp_num; 3993 no_mgs = mc_qht_elem->mc_grp_ctx.no_of_mgs; 3994 irdma_sc_add_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 3995 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 3996 3997 /* Only if there is a change do we need to modify or create */ 3998 if (!no_mgs) { 3999 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4000 IRDMA_OP_MC_CREATE); 4001 } else if (no_mgs != mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4002 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4003 IRDMA_OP_MC_MODIFY); 4004 } else { 4005 return 0; 4006 } 4007 4008 if (ret) 4009 goto error; 4010 4011 return 0; 4012 4013 error: 4014 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4015 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4016 mcast_list_del(mc_qht_elem); 4017 dma_free_coherent(rf->hw.device, 4018 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4019 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4020 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4021 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4022 irdma_free_rsrc(rf, rf->allocated_mcgs, 4023 mc_qht_elem->mc_grp_ctx.mg_id); 4024 kfree(mc_qht_elem); 4025 } 4026 4027 return ret; 4028 } 4029 4030 /** 4031 * irdma_detach_mcast - detach a qp from a multicast group 4032 * @ibqp: ptr to qp 4033 * @ibgid: pointer to global ID 4034 * @lid: local ID 4035 * 4036 * returns error status 4037 */ 4038 static int irdma_detach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid) 4039 { 4040 struct irdma_qp *iwqp = to_iwqp(ibqp); 4041 struct irdma_device *iwdev = iwqp->iwdev; 4042 struct irdma_pci_f *rf = iwdev->rf; 4043 u32 ip_addr[4] = {}; 4044 struct mc_table_list *mc_qht_elem; 4045 struct irdma_mcast_grp_ctx_entry_info mcg_info = {}; 4046 int ret; 4047 unsigned long flags; 4048 union irdma_sockaddr sgid_addr; 4049 4050 rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); 4051 if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) 4052 irdma_copy_ip_ntohl(ip_addr, 4053 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4054 else 4055 ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4056 4057 spin_lock_irqsave(&rf->qh_list_lock, flags); 4058 mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); 4059 if (!mc_qht_elem) { 4060 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4061 ibdev_dbg(&iwdev->ibdev, 4062 "VERBS: address not found MCG\n"); 4063 return 0; 4064 } 4065 4066 mcg_info.qp_id = iwqp->ibqp.qp_num; 4067 irdma_sc_del_mcast_grp(&mc_qht_elem->mc_grp_ctx, &mcg_info); 4068 if (!mc_qht_elem->mc_grp_ctx.no_of_mgs) { 4069 mcast_list_del(mc_qht_elem); 4070 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4071 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4072 IRDMA_OP_MC_DESTROY); 4073 if (ret) { 4074 ibdev_dbg(&iwdev->ibdev, 4075 "VERBS: failed MC_DESTROY MCG\n"); 4076 spin_lock_irqsave(&rf->qh_list_lock, flags); 4077 mcast_list_add(rf, mc_qht_elem); 4078 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4079 return -EAGAIN; 4080 } 4081 4082 dma_free_coherent(rf->hw.device, 4083 mc_qht_elem->mc_grp_ctx.dma_mem_mc.size, 4084 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va, 4085 mc_qht_elem->mc_grp_ctx.dma_mem_mc.pa); 4086 mc_qht_elem->mc_grp_ctx.dma_mem_mc.va = NULL; 4087 irdma_free_rsrc(rf, rf->allocated_mcgs, 4088 mc_qht_elem->mc_grp_ctx.mg_id); 4089 kfree(mc_qht_elem); 4090 } else { 4091 spin_unlock_irqrestore(&rf->qh_list_lock, flags); 4092 ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, 4093 IRDMA_OP_MC_MODIFY); 4094 if (ret) { 4095 ibdev_dbg(&iwdev->ibdev, 4096 "VERBS: failed Modify MCG\n"); 4097 return ret; 4098 } 4099 } 4100 4101 return 0; 4102 } 4103 4104 static int irdma_create_hw_ah(struct irdma_device *iwdev, struct irdma_ah *ah, bool sleep) 4105 { 4106 struct irdma_pci_f *rf = iwdev->rf; 4107 int err; 4108 4109 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah, &ah->sc_ah.ah_info.ah_idx, 4110 &rf->next_ah); 4111 if (err) 4112 return err; 4113 4114 err = irdma_ah_cqp_op(rf, &ah->sc_ah, IRDMA_OP_AH_CREATE, sleep, 4115 irdma_gsi_ud_qp_ah_cb, &ah->sc_ah); 4116 4117 if (err) { 4118 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP-OP Create AH fail"); 4119 goto err_ah_create; 4120 } 4121 4122 if (!sleep) { 4123 int cnt = CQP_COMPL_WAIT_TIME_MS * CQP_TIMEOUT_THRESHOLD; 4124 4125 do { 4126 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); 4127 mdelay(1); 4128 } while (!ah->sc_ah.ah_info.ah_valid && --cnt); 4129 4130 if (!cnt) { 4131 ibdev_dbg(&iwdev->ibdev, "VERBS: CQP create AH timed out"); 4132 err = -ETIMEDOUT; 4133 goto err_ah_create; 4134 } 4135 } 4136 return 0; 4137 4138 err_ah_create: 4139 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah->sc_ah.ah_info.ah_idx); 4140 4141 return err; 4142 } 4143 4144 static int irdma_setup_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr) 4145 { 4146 struct irdma_pd *pd = to_iwpd(ibah->pd); 4147 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4148 struct rdma_ah_attr *ah_attr = attr->ah_attr; 4149 const struct ib_gid_attr *sgid_attr; 4150 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4151 struct irdma_pci_f *rf = iwdev->rf; 4152 struct irdma_sc_ah *sc_ah; 4153 struct irdma_ah_info *ah_info; 4154 union irdma_sockaddr sgid_addr, dgid_addr; 4155 int err; 4156 u8 dmac[ETH_ALEN]; 4157 4158 ah->pd = pd; 4159 sc_ah = &ah->sc_ah; 4160 sc_ah->ah_info.vsi = &iwdev->vsi; 4161 irdma_sc_init_ah(&rf->sc_dev, sc_ah); 4162 ah->sgid_index = ah_attr->grh.sgid_index; 4163 sgid_attr = ah_attr->grh.sgid_attr; 4164 memcpy(&ah->dgid, &ah_attr->grh.dgid, sizeof(ah->dgid)); 4165 rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid_attr->gid); 4166 rdma_gid2ip((struct sockaddr *)&dgid_addr, &ah_attr->grh.dgid); 4167 ah->av.attrs = *ah_attr; 4168 ah->av.net_type = rdma_gid_attr_network_type(sgid_attr); 4169 ah_info = &sc_ah->ah_info; 4170 ah_info->pd_idx = pd->sc_pd.pd_id; 4171 if (ah_attr->ah_flags & IB_AH_GRH) { 4172 ah_info->flow_label = ah_attr->grh.flow_label; 4173 ah_info->hop_ttl = ah_attr->grh.hop_limit; 4174 ah_info->tc_tos = ah_attr->grh.traffic_class; 4175 } 4176 4177 ether_addr_copy(dmac, ah_attr->roce.dmac); 4178 if (ah->av.net_type == RDMA_NETWORK_IPV4) { 4179 ah_info->ipv4_valid = true; 4180 ah_info->dest_ip_addr[0] = 4181 ntohl(dgid_addr.saddr_in.sin_addr.s_addr); 4182 ah_info->src_ip_addr[0] = 4183 ntohl(sgid_addr.saddr_in.sin_addr.s_addr); 4184 ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], 4185 ah_info->dest_ip_addr[0]); 4186 if (ipv4_is_multicast(dgid_addr.saddr_in.sin_addr.s_addr)) { 4187 ah_info->do_lpbk = true; 4188 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, true); 4189 } 4190 } else { 4191 irdma_copy_ip_ntohl(ah_info->dest_ip_addr, 4192 dgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4193 irdma_copy_ip_ntohl(ah_info->src_ip_addr, 4194 sgid_addr.saddr_in6.sin6_addr.in6_u.u6_addr32); 4195 ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, 4196 ah_info->dest_ip_addr); 4197 if (rdma_is_multicast_addr(&dgid_addr.saddr_in6.sin6_addr)) { 4198 ah_info->do_lpbk = true; 4199 irdma_mcast_mac(ah_info->dest_ip_addr, dmac, false); 4200 } 4201 } 4202 4203 err = rdma_read_gid_l2_fields(sgid_attr, &ah_info->vlan_tag, 4204 ah_info->mac_addr); 4205 if (err) 4206 return err; 4207 4208 ah_info->dst_arpindex = irdma_add_arp(iwdev->rf, ah_info->dest_ip_addr, 4209 ah_info->ipv4_valid, dmac); 4210 4211 if (ah_info->dst_arpindex == -1) 4212 return -EINVAL; 4213 4214 if (ah_info->vlan_tag >= VLAN_N_VID && iwdev->dcb_vlan_mode) 4215 ah_info->vlan_tag = 0; 4216 4217 if (ah_info->vlan_tag < VLAN_N_VID) { 4218 ah_info->insert_vlan_tag = true; 4219 ah_info->vlan_tag |= 4220 rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; 4221 } 4222 4223 return 0; 4224 } 4225 4226 /** 4227 * irdma_ah_exists - Check for existing identical AH 4228 * @iwdev: irdma device 4229 * @new_ah: AH to check for 4230 * 4231 * returns true if AH is found, false if not found. 4232 */ 4233 static bool irdma_ah_exists(struct irdma_device *iwdev, 4234 struct irdma_ah *new_ah) 4235 { 4236 struct irdma_ah *ah; 4237 u32 key = new_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 4238 new_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 4239 new_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 4240 new_ah->sc_ah.ah_info.dest_ip_addr[3]; 4241 4242 hash_for_each_possible(iwdev->ah_hash_tbl, ah, list, key) { 4243 /* Set ah_valid and ah_id the same so memcmp can work */ 4244 new_ah->sc_ah.ah_info.ah_idx = ah->sc_ah.ah_info.ah_idx; 4245 new_ah->sc_ah.ah_info.ah_valid = ah->sc_ah.ah_info.ah_valid; 4246 if (!memcmp(&ah->sc_ah.ah_info, &new_ah->sc_ah.ah_info, 4247 sizeof(ah->sc_ah.ah_info))) { 4248 refcount_inc(&ah->refcnt); 4249 new_ah->parent_ah = ah; 4250 return true; 4251 } 4252 } 4253 4254 return false; 4255 } 4256 4257 /** 4258 * irdma_destroy_ah - Destroy address handle 4259 * @ibah: pointer to address handle 4260 * @ah_flags: flags for sleepable 4261 */ 4262 static int irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags) 4263 { 4264 struct irdma_device *iwdev = to_iwdev(ibah->device); 4265 struct irdma_ah *ah = to_iwah(ibah); 4266 4267 if ((ah_flags & RDMA_DESTROY_AH_SLEEPABLE) && ah->parent_ah) { 4268 mutex_lock(&iwdev->ah_tbl_lock); 4269 if (!refcount_dec_and_test(&ah->parent_ah->refcnt)) { 4270 mutex_unlock(&iwdev->ah_tbl_lock); 4271 return 0; 4272 } 4273 hash_del(&ah->parent_ah->list); 4274 kfree(ah->parent_ah); 4275 mutex_unlock(&iwdev->ah_tbl_lock); 4276 } 4277 4278 irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, 4279 false, NULL, ah); 4280 4281 irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, 4282 ah->sc_ah.ah_info.ah_idx); 4283 4284 return 0; 4285 } 4286 4287 /** 4288 * irdma_create_user_ah - create user address handle 4289 * @ibah: address handle 4290 * @attr: address handle attributes 4291 * @udata: User data 4292 * 4293 * returns 0 on success, error otherwise 4294 */ 4295 static int irdma_create_user_ah(struct ib_ah *ibah, 4296 struct rdma_ah_init_attr *attr, 4297 struct ib_udata *udata) 4298 { 4299 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4300 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4301 struct irdma_create_ah_resp uresp; 4302 struct irdma_ah *parent_ah; 4303 int err; 4304 4305 err = irdma_setup_ah(ibah, attr); 4306 if (err) 4307 return err; 4308 mutex_lock(&iwdev->ah_tbl_lock); 4309 if (!irdma_ah_exists(iwdev, ah)) { 4310 err = irdma_create_hw_ah(iwdev, ah, true); 4311 if (err) { 4312 mutex_unlock(&iwdev->ah_tbl_lock); 4313 return err; 4314 } 4315 /* Add new AH to list */ 4316 parent_ah = kmemdup(ah, sizeof(*ah), GFP_KERNEL); 4317 if (parent_ah) { 4318 u32 key = parent_ah->sc_ah.ah_info.dest_ip_addr[0] ^ 4319 parent_ah->sc_ah.ah_info.dest_ip_addr[1] ^ 4320 parent_ah->sc_ah.ah_info.dest_ip_addr[2] ^ 4321 parent_ah->sc_ah.ah_info.dest_ip_addr[3]; 4322 4323 ah->parent_ah = parent_ah; 4324 hash_add(iwdev->ah_hash_tbl, &parent_ah->list, key); 4325 refcount_set(&parent_ah->refcnt, 1); 4326 } 4327 } 4328 mutex_unlock(&iwdev->ah_tbl_lock); 4329 4330 uresp.ah_id = ah->sc_ah.ah_info.ah_idx; 4331 err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); 4332 if (err) 4333 irdma_destroy_ah(ibah, attr->flags); 4334 4335 return err; 4336 } 4337 4338 /** 4339 * irdma_create_ah - create address handle 4340 * @ibah: address handle 4341 * @attr: address handle attributes 4342 * @udata: NULL 4343 * 4344 * returns 0 on success, error otherwise 4345 */ 4346 static int irdma_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *attr, 4347 struct ib_udata *udata) 4348 { 4349 struct irdma_ah *ah = container_of(ibah, struct irdma_ah, ibah); 4350 struct irdma_device *iwdev = to_iwdev(ibah->pd->device); 4351 int err; 4352 4353 err = irdma_setup_ah(ibah, attr); 4354 if (err) 4355 return err; 4356 err = irdma_create_hw_ah(iwdev, ah, attr->flags & RDMA_CREATE_AH_SLEEPABLE); 4357 4358 return err; 4359 } 4360 4361 /** 4362 * irdma_query_ah - Query address handle 4363 * @ibah: pointer to address handle 4364 * @ah_attr: address handle attributes 4365 */ 4366 static int irdma_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr) 4367 { 4368 struct irdma_ah *ah = to_iwah(ibah); 4369 4370 memset(ah_attr, 0, sizeof(*ah_attr)); 4371 if (ah->av.attrs.ah_flags & IB_AH_GRH) { 4372 ah_attr->ah_flags = IB_AH_GRH; 4373 ah_attr->grh.flow_label = ah->sc_ah.ah_info.flow_label; 4374 ah_attr->grh.traffic_class = ah->sc_ah.ah_info.tc_tos; 4375 ah_attr->grh.hop_limit = ah->sc_ah.ah_info.hop_ttl; 4376 ah_attr->grh.sgid_index = ah->sgid_index; 4377 ah_attr->grh.sgid_index = ah->sgid_index; 4378 memcpy(&ah_attr->grh.dgid, &ah->dgid, 4379 sizeof(ah_attr->grh.dgid)); 4380 } 4381 4382 return 0; 4383 } 4384 4385 static enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, 4386 u32 port_num) 4387 { 4388 return IB_LINK_LAYER_ETHERNET; 4389 } 4390 4391 static const struct ib_device_ops irdma_roce_dev_ops = { 4392 .attach_mcast = irdma_attach_mcast, 4393 .create_ah = irdma_create_ah, 4394 .create_user_ah = irdma_create_user_ah, 4395 .destroy_ah = irdma_destroy_ah, 4396 .detach_mcast = irdma_detach_mcast, 4397 .get_link_layer = irdma_get_link_layer, 4398 .get_port_immutable = irdma_roce_port_immutable, 4399 .modify_qp = irdma_modify_qp_roce, 4400 .query_ah = irdma_query_ah, 4401 .query_pkey = irdma_query_pkey, 4402 }; 4403 4404 static const struct ib_device_ops irdma_iw_dev_ops = { 4405 .modify_qp = irdma_modify_qp, 4406 .get_port_immutable = irdma_iw_port_immutable, 4407 .query_gid = irdma_query_gid, 4408 }; 4409 4410 static const struct ib_device_ops irdma_dev_ops = { 4411 .owner = THIS_MODULE, 4412 .driver_id = RDMA_DRIVER_IRDMA, 4413 .uverbs_abi_ver = IRDMA_ABI_VER, 4414 4415 .alloc_hw_port_stats = irdma_alloc_hw_port_stats, 4416 .alloc_mr = irdma_alloc_mr, 4417 .alloc_mw = irdma_alloc_mw, 4418 .alloc_pd = irdma_alloc_pd, 4419 .alloc_ucontext = irdma_alloc_ucontext, 4420 .create_cq = irdma_create_cq, 4421 .create_qp = irdma_create_qp, 4422 .dealloc_driver = irdma_ib_dealloc_device, 4423 .dealloc_mw = irdma_dealloc_mw, 4424 .dealloc_pd = irdma_dealloc_pd, 4425 .dealloc_ucontext = irdma_dealloc_ucontext, 4426 .dereg_mr = irdma_dereg_mr, 4427 .destroy_cq = irdma_destroy_cq, 4428 .destroy_qp = irdma_destroy_qp, 4429 .disassociate_ucontext = irdma_disassociate_ucontext, 4430 .get_dev_fw_str = irdma_get_dev_fw_str, 4431 .get_dma_mr = irdma_get_dma_mr, 4432 .get_hw_stats = irdma_get_hw_stats, 4433 .map_mr_sg = irdma_map_mr_sg, 4434 .mmap = irdma_mmap, 4435 .mmap_free = irdma_mmap_free, 4436 .poll_cq = irdma_poll_cq, 4437 .post_recv = irdma_post_recv, 4438 .post_send = irdma_post_send, 4439 .query_device = irdma_query_device, 4440 .query_port = irdma_query_port, 4441 .query_qp = irdma_query_qp, 4442 .reg_user_mr = irdma_reg_user_mr, 4443 .req_notify_cq = irdma_req_notify_cq, 4444 .resize_cq = irdma_resize_cq, 4445 INIT_RDMA_OBJ_SIZE(ib_pd, irdma_pd, ibpd), 4446 INIT_RDMA_OBJ_SIZE(ib_ucontext, irdma_ucontext, ibucontext), 4447 INIT_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah), 4448 INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq), 4449 INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw), 4450 INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp), 4451 }; 4452 4453 /** 4454 * irdma_init_roce_device - initialization of roce rdma device 4455 * @iwdev: irdma device 4456 */ 4457 static void irdma_init_roce_device(struct irdma_device *iwdev) 4458 { 4459 iwdev->ibdev.node_type = RDMA_NODE_IB_CA; 4460 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 4461 iwdev->netdev->dev_addr); 4462 ib_set_device_ops(&iwdev->ibdev, &irdma_roce_dev_ops); 4463 } 4464 4465 /** 4466 * irdma_init_iw_device - initialization of iwarp rdma device 4467 * @iwdev: irdma device 4468 */ 4469 static int irdma_init_iw_device(struct irdma_device *iwdev) 4470 { 4471 struct net_device *netdev = iwdev->netdev; 4472 4473 iwdev->ibdev.node_type = RDMA_NODE_RNIC; 4474 addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, 4475 netdev->dev_addr); 4476 iwdev->ibdev.ops.iw_add_ref = irdma_qp_add_ref; 4477 iwdev->ibdev.ops.iw_rem_ref = irdma_qp_rem_ref; 4478 iwdev->ibdev.ops.iw_get_qp = irdma_get_qp; 4479 iwdev->ibdev.ops.iw_connect = irdma_connect; 4480 iwdev->ibdev.ops.iw_accept = irdma_accept; 4481 iwdev->ibdev.ops.iw_reject = irdma_reject; 4482 iwdev->ibdev.ops.iw_create_listen = irdma_create_listen; 4483 iwdev->ibdev.ops.iw_destroy_listen = irdma_destroy_listen; 4484 memcpy(iwdev->ibdev.iw_ifname, netdev->name, 4485 sizeof(iwdev->ibdev.iw_ifname)); 4486 ib_set_device_ops(&iwdev->ibdev, &irdma_iw_dev_ops); 4487 4488 return 0; 4489 } 4490 4491 /** 4492 * irdma_init_rdma_device - initialization of rdma device 4493 * @iwdev: irdma device 4494 */ 4495 static int irdma_init_rdma_device(struct irdma_device *iwdev) 4496 { 4497 struct pci_dev *pcidev = iwdev->rf->pcidev; 4498 int ret; 4499 4500 if (iwdev->roce_mode) { 4501 irdma_init_roce_device(iwdev); 4502 } else { 4503 ret = irdma_init_iw_device(iwdev); 4504 if (ret) 4505 return ret; 4506 } 4507 iwdev->ibdev.phys_port_cnt = 1; 4508 iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; 4509 iwdev->ibdev.dev.parent = &pcidev->dev; 4510 ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops); 4511 4512 return 0; 4513 } 4514 4515 /** 4516 * irdma_port_ibevent - indicate port event 4517 * @iwdev: irdma device 4518 */ 4519 void irdma_port_ibevent(struct irdma_device *iwdev) 4520 { 4521 struct ib_event event; 4522 4523 event.device = &iwdev->ibdev; 4524 event.element.port_num = 1; 4525 event.event = 4526 iwdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; 4527 ib_dispatch_event(&event); 4528 } 4529 4530 /** 4531 * irdma_ib_unregister_device - unregister rdma device from IB 4532 * core 4533 * @iwdev: irdma device 4534 */ 4535 void irdma_ib_unregister_device(struct irdma_device *iwdev) 4536 { 4537 iwdev->iw_status = 0; 4538 irdma_port_ibevent(iwdev); 4539 ib_unregister_device(&iwdev->ibdev); 4540 } 4541 4542 /** 4543 * irdma_ib_register_device - register irdma device to IB core 4544 * @iwdev: irdma device 4545 */ 4546 int irdma_ib_register_device(struct irdma_device *iwdev) 4547 { 4548 int ret; 4549 4550 ret = irdma_init_rdma_device(iwdev); 4551 if (ret) 4552 return ret; 4553 4554 ret = ib_device_set_netdev(&iwdev->ibdev, iwdev->netdev, 1); 4555 if (ret) 4556 goto error; 4557 dma_set_max_seg_size(iwdev->rf->hw.device, UINT_MAX); 4558 ret = ib_register_device(&iwdev->ibdev, "irdma%d", iwdev->rf->hw.device); 4559 if (ret) 4560 goto error; 4561 4562 iwdev->iw_status = 1; 4563 irdma_port_ibevent(iwdev); 4564 4565 return 0; 4566 4567 error: 4568 if (ret) 4569 ibdev_dbg(&iwdev->ibdev, "VERBS: Register RDMA device fail\n"); 4570 4571 return ret; 4572 } 4573 4574 /** 4575 * irdma_ib_dealloc_device 4576 * @ibdev: ib device 4577 * 4578 * callback from ibdev dealloc_driver to deallocate resources 4579 * unber irdma device 4580 */ 4581 void irdma_ib_dealloc_device(struct ib_device *ibdev) 4582 { 4583 struct irdma_device *iwdev = to_iwdev(ibdev); 4584 4585 irdma_rt_deinit_hw(iwdev); 4586 irdma_ctrl_deinit_hw(iwdev->rf); 4587 kfree(iwdev->rf); 4588 } 4589