1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Copyright (c) 2008-2019, IBM Corporation */ 9 10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ 11 12 #include <linux/errno.h> 13 #include <linux/pci.h> 14 #include <linux/types.h> 15 #include <linux/uaccess.h> 16 #include <linux/vmalloc.h> 17 #include <net/addrconf.h> 18 #include <rdma/erdma-abi.h> 19 #include <rdma/ib_umem.h> 20 #include <rdma/ib_user_verbs.h> 21 #include <rdma/ib_verbs.h> 22 #include <rdma/uverbs_ioctl.h> 23 24 #include "erdma.h" 25 #include "erdma_cm.h" 26 #include "erdma_hw.h" 27 #include "erdma_verbs.h" 28 29 static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp) 30 { 31 struct erdma_cmdq_create_qp_req req; 32 struct erdma_pd *pd = to_epd(qp->ibqp.pd); 33 struct erdma_uqp *user_qp; 34 u64 resp0, resp1; 35 int err; 36 37 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 38 CMDQ_OPCODE_CREATE_QP); 39 40 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK, 41 ilog2(qp->attrs.sq_size)) | 42 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp)); 43 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK, 44 ilog2(qp->attrs.rq_size)) | 45 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn); 46 47 if (rdma_is_kernel_res(&qp->ibqp.res)) { 48 u32 pgsz_range = ilog2(SZ_1M) - PAGE_SHIFT; 49 50 req.sq_cqn_mtt_cfg = 51 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 52 pgsz_range) | 53 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 54 req.rq_cqn_mtt_cfg = 55 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 56 pgsz_range) | 57 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 58 59 req.sq_mtt_cfg = 60 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | 61 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | 62 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 63 ERDMA_MR_INLINE_MTT); 64 req.rq_mtt_cfg = req.sq_mtt_cfg; 65 66 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 67 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 68 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + 69 (qp->attrs.sq_size << SQEBB_SHIFT); 70 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + 71 (qp->attrs.rq_size << RQE_SHIFT); 72 } else { 73 user_qp = &qp->user_qp; 74 req.sq_cqn_mtt_cfg = FIELD_PREP( 75 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 76 ilog2(user_qp->sq_mtt.page_size) - PAGE_SHIFT); 77 req.sq_cqn_mtt_cfg |= 78 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 79 80 req.rq_cqn_mtt_cfg = FIELD_PREP( 81 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 82 ilog2(user_qp->rq_mtt.page_size) - PAGE_SHIFT); 83 req.rq_cqn_mtt_cfg |= 84 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 85 86 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset; 87 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 88 user_qp->sq_mtt.mtt_nents) | 89 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 90 user_qp->sq_mtt.mtt_type); 91 92 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset; 93 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 94 user_qp->rq_mtt.mtt_nents) | 95 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 96 user_qp->rq_mtt.mtt_type); 97 98 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0]; 99 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0]; 100 101 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; 102 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; 103 } 104 105 err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), &resp0, 106 &resp1); 107 if (!err) 108 qp->attrs.cookie = 109 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0); 110 111 return err; 112 } 113 114 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 115 { 116 struct erdma_cmdq_reg_mr_req req; 117 struct erdma_pd *pd = to_epd(mr->ibmr.pd); 118 u64 *phy_addr; 119 int i; 120 121 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR); 122 123 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) | 124 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) | 125 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8); 126 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) | 127 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) | 128 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) | 129 FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0); 130 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK, 131 ilog2(mr->mem.page_size)) | 132 FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) | 133 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt); 134 135 if (mr->type == ERDMA_MR_TYPE_DMA) 136 goto post_cmd; 137 138 if (mr->type == ERDMA_MR_TYPE_NORMAL) { 139 req.start_va = mr->mem.va; 140 req.size = mr->mem.len; 141 } 142 143 if (mr->type == ERDMA_MR_TYPE_FRMR || 144 mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) { 145 phy_addr = req.phy_addr; 146 *phy_addr = mr->mem.mtt_entry[0]; 147 } else { 148 phy_addr = req.phy_addr; 149 for (i = 0; i < mr->mem.mtt_nents; i++) 150 *phy_addr++ = mr->mem.mtt_entry[i]; 151 } 152 153 post_cmd: 154 return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, 155 NULL); 156 } 157 158 static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq) 159 { 160 struct erdma_cmdq_create_cq_req req; 161 u32 page_size; 162 struct erdma_mem *mtt; 163 164 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 165 CMDQ_OPCODE_CREATE_CQ); 166 167 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) | 168 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth)); 169 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); 170 171 if (rdma_is_kernel_res(&cq->ibcq.res)) { 172 page_size = SZ_32M; 173 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 174 ilog2(page_size) - PAGE_SHIFT); 175 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); 176 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); 177 178 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | 179 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 180 ERDMA_MR_INLINE_MTT); 181 182 req.first_page_offset = 0; 183 req.cq_db_info_addr = 184 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); 185 } else { 186 mtt = &cq->user_cq.qbuf_mtt; 187 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 188 ilog2(mtt->page_size) - PAGE_SHIFT); 189 if (mtt->mtt_nents == 1) { 190 req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf); 191 req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf); 192 } else { 193 req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]); 194 req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]); 195 } 196 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 197 mtt->mtt_nents); 198 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 199 mtt->mtt_type); 200 201 req.first_page_offset = mtt->page_offset; 202 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; 203 } 204 205 return erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, 206 NULL); 207 } 208 209 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) 210 { 211 int idx; 212 unsigned long flags; 213 214 spin_lock_irqsave(&res_cb->lock, flags); 215 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, 216 res_cb->next_alloc_idx); 217 if (idx == res_cb->max_cap) { 218 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap); 219 if (idx == res_cb->max_cap) { 220 res_cb->next_alloc_idx = 1; 221 spin_unlock_irqrestore(&res_cb->lock, flags); 222 return -ENOSPC; 223 } 224 } 225 226 set_bit(idx, res_cb->bitmap); 227 res_cb->next_alloc_idx = idx + 1; 228 spin_unlock_irqrestore(&res_cb->lock, flags); 229 230 return idx; 231 } 232 233 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx) 234 { 235 unsigned long flags; 236 u32 used; 237 238 spin_lock_irqsave(&res_cb->lock, flags); 239 used = __test_and_clear_bit(idx, res_cb->bitmap); 240 spin_unlock_irqrestore(&res_cb->lock, flags); 241 WARN_ON(!used); 242 } 243 244 static struct rdma_user_mmap_entry * 245 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, 246 u32 size, u8 mmap_flag, u64 *mmap_offset) 247 { 248 struct erdma_user_mmap_entry *entry = 249 kzalloc(sizeof(*entry), GFP_KERNEL); 250 int ret; 251 252 if (!entry) 253 return NULL; 254 255 entry->address = (u64)address; 256 entry->mmap_flag = mmap_flag; 257 258 size = PAGE_ALIGN(size); 259 260 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry, 261 size); 262 if (ret) { 263 kfree(entry); 264 return NULL; 265 } 266 267 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 268 269 return &entry->rdma_entry; 270 } 271 272 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 273 struct ib_udata *unused) 274 { 275 struct erdma_dev *dev = to_edev(ibdev); 276 277 memset(attr, 0, sizeof(*attr)); 278 279 attr->max_mr_size = dev->attrs.max_mr_size; 280 attr->vendor_id = PCI_VENDOR_ID_ALIBABA; 281 attr->vendor_part_id = dev->pdev->device; 282 attr->hw_ver = dev->pdev->revision; 283 attr->max_qp = dev->attrs.max_qp - 1; 284 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr); 285 attr->max_qp_rd_atom = dev->attrs.max_ord; 286 attr->max_qp_init_rd_atom = dev->attrs.max_ird; 287 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird; 288 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 289 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 290 ibdev->local_dma_lkey = dev->attrs.local_dma_key; 291 attr->max_send_sge = dev->attrs.max_send_sge; 292 attr->max_recv_sge = dev->attrs.max_recv_sge; 293 attr->max_sge_rd = dev->attrs.max_sge_rd; 294 attr->max_cq = dev->attrs.max_cq - 1; 295 attr->max_cqe = dev->attrs.max_cqe; 296 attr->max_mr = dev->attrs.max_mr; 297 attr->max_pd = dev->attrs.max_pd; 298 attr->max_mw = dev->attrs.max_mw; 299 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA; 300 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT; 301 attr->fw_ver = dev->attrs.fw_version; 302 303 if (dev->netdev) 304 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 305 dev->netdev->dev_addr); 306 307 return 0; 308 } 309 310 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, 311 union ib_gid *gid) 312 { 313 struct erdma_dev *dev = to_edev(ibdev); 314 315 memset(gid, 0, sizeof(*gid)); 316 ether_addr_copy(gid->raw, dev->attrs.peer_addr); 317 318 return 0; 319 } 320 321 int erdma_query_port(struct ib_device *ibdev, u32 port, 322 struct ib_port_attr *attr) 323 { 324 struct erdma_dev *dev = to_edev(ibdev); 325 struct net_device *ndev = dev->netdev; 326 327 memset(attr, 0, sizeof(*attr)); 328 329 attr->gid_tbl_len = 1; 330 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; 331 attr->max_msg_sz = -1; 332 333 if (!ndev) 334 goto out; 335 336 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width); 337 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu); 338 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 339 if (netif_running(ndev) && netif_carrier_ok(ndev)) 340 dev->state = IB_PORT_ACTIVE; 341 else 342 dev->state = IB_PORT_DOWN; 343 attr->state = dev->state; 344 345 out: 346 if (dev->state == IB_PORT_ACTIVE) 347 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 348 else 349 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 350 351 return 0; 352 } 353 354 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port, 355 struct ib_port_immutable *port_immutable) 356 { 357 port_immutable->gid_tbl_len = 1; 358 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 359 360 return 0; 361 } 362 363 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 364 { 365 struct erdma_pd *pd = to_epd(ibpd); 366 struct erdma_dev *dev = to_edev(ibpd->device); 367 int pdn; 368 369 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]); 370 if (pdn < 0) 371 return pdn; 372 373 pd->pdn = pdn; 374 375 return 0; 376 } 377 378 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 379 { 380 struct erdma_pd *pd = to_epd(ibpd); 381 struct erdma_dev *dev = to_edev(ibpd->device); 382 383 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn); 384 385 return 0; 386 } 387 388 static int erdma_qp_validate_cap(struct erdma_dev *dev, 389 struct ib_qp_init_attr *attrs) 390 { 391 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) || 392 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) || 393 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) || 394 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) || 395 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) || 396 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) { 397 return -EINVAL; 398 } 399 400 return 0; 401 } 402 403 static int erdma_qp_validate_attr(struct erdma_dev *dev, 404 struct ib_qp_init_attr *attrs) 405 { 406 if (attrs->qp_type != IB_QPT_RC) 407 return -EOPNOTSUPP; 408 409 if (attrs->srq) 410 return -EOPNOTSUPP; 411 412 if (!attrs->send_cq || !attrs->recv_cq) 413 return -EOPNOTSUPP; 414 415 return 0; 416 } 417 418 static void free_kernel_qp(struct erdma_qp *qp) 419 { 420 struct erdma_dev *dev = qp->dev; 421 422 vfree(qp->kern_qp.swr_tbl); 423 vfree(qp->kern_qp.rwr_tbl); 424 425 if (qp->kern_qp.sq_buf) 426 dma_free_coherent( 427 &dev->pdev->dev, 428 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 429 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 430 431 if (qp->kern_qp.rq_buf) 432 dma_free_coherent( 433 &dev->pdev->dev, 434 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 435 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 436 } 437 438 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, 439 struct ib_qp_init_attr *attrs) 440 { 441 struct erdma_kqp *kqp = &qp->kern_qp; 442 int size; 443 444 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) 445 kqp->sig_all = 1; 446 447 kqp->sq_pi = 0; 448 kqp->sq_ci = 0; 449 kqp->rq_pi = 0; 450 kqp->rq_ci = 0; 451 kqp->hw_sq_db = 452 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); 453 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; 454 455 kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64)); 456 kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64)); 457 if (!kqp->swr_tbl || !kqp->rwr_tbl) 458 goto err_out; 459 460 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 461 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 462 &kqp->sq_buf_dma_addr, GFP_KERNEL); 463 if (!kqp->sq_buf) 464 goto err_out; 465 466 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 467 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 468 &kqp->rq_buf_dma_addr, GFP_KERNEL); 469 if (!kqp->rq_buf) 470 goto err_out; 471 472 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); 473 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); 474 475 return 0; 476 477 err_out: 478 free_kernel_qp(qp); 479 return -ENOMEM; 480 } 481 482 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, 483 u64 start, u64 len, int access, u64 virt, 484 unsigned long req_page_size, u8 force_indirect_mtt) 485 { 486 struct ib_block_iter biter; 487 uint64_t *phy_addr = NULL; 488 int ret = 0; 489 490 mem->umem = ib_umem_get(&dev->ibdev, start, len, access); 491 if (IS_ERR(mem->umem)) { 492 ret = PTR_ERR(mem->umem); 493 mem->umem = NULL; 494 return ret; 495 } 496 497 mem->va = virt; 498 mem->len = len; 499 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); 500 mem->page_offset = start & (mem->page_size - 1); 501 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); 502 mem->page_cnt = mem->mtt_nents; 503 504 if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES || 505 force_indirect_mtt) { 506 mem->mtt_type = ERDMA_MR_INDIRECT_MTT; 507 mem->mtt_buf = 508 alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL); 509 if (!mem->mtt_buf) { 510 ret = -ENOMEM; 511 goto error_ret; 512 } 513 phy_addr = mem->mtt_buf; 514 } else { 515 mem->mtt_type = ERDMA_MR_INLINE_MTT; 516 phy_addr = mem->mtt_entry; 517 } 518 519 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) { 520 *phy_addr = rdma_block_iter_dma_address(&biter); 521 phy_addr++; 522 } 523 524 if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) { 525 mem->mtt_entry[0] = 526 dma_map_single(&dev->pdev->dev, mem->mtt_buf, 527 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 528 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) { 529 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 530 mem->mtt_buf = NULL; 531 ret = -ENOMEM; 532 goto error_ret; 533 } 534 } 535 536 return 0; 537 538 error_ret: 539 if (mem->umem) { 540 ib_umem_release(mem->umem); 541 mem->umem = NULL; 542 } 543 544 return ret; 545 } 546 547 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) 548 { 549 if (mem->mtt_buf) { 550 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0], 551 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 552 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 553 } 554 555 if (mem->umem) { 556 ib_umem_release(mem->umem); 557 mem->umem = NULL; 558 } 559 } 560 561 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, 562 u64 dbrecords_va, 563 struct erdma_user_dbrecords_page **dbr_page, 564 dma_addr_t *dma_addr) 565 { 566 struct erdma_user_dbrecords_page *page = NULL; 567 int rv = 0; 568 569 mutex_lock(&ctx->dbrecords_page_mutex); 570 571 list_for_each_entry(page, &ctx->dbrecords_page_list, list) 572 if (page->va == (dbrecords_va & PAGE_MASK)) 573 goto found; 574 575 page = kmalloc(sizeof(*page), GFP_KERNEL); 576 if (!page) { 577 rv = -ENOMEM; 578 goto out; 579 } 580 581 page->va = (dbrecords_va & PAGE_MASK); 582 page->refcnt = 0; 583 584 page->umem = ib_umem_get(ctx->ibucontext.device, 585 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); 586 if (IS_ERR(page->umem)) { 587 rv = PTR_ERR(page->umem); 588 kfree(page); 589 goto out; 590 } 591 592 list_add(&page->list, &ctx->dbrecords_page_list); 593 594 found: 595 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) + 596 (dbrecords_va & ~PAGE_MASK); 597 *dbr_page = page; 598 page->refcnt++; 599 600 out: 601 mutex_unlock(&ctx->dbrecords_page_mutex); 602 return rv; 603 } 604 605 static void 606 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, 607 struct erdma_user_dbrecords_page **dbr_page) 608 { 609 if (!ctx || !(*dbr_page)) 610 return; 611 612 mutex_lock(&ctx->dbrecords_page_mutex); 613 if (--(*dbr_page)->refcnt == 0) { 614 list_del(&(*dbr_page)->list); 615 ib_umem_release((*dbr_page)->umem); 616 kfree(*dbr_page); 617 } 618 619 *dbr_page = NULL; 620 mutex_unlock(&ctx->dbrecords_page_mutex); 621 } 622 623 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, 624 u64 va, u32 len, u64 db_info_va) 625 { 626 dma_addr_t db_info_dma_addr; 627 u32 rq_offset; 628 int ret; 629 630 if (len < (PAGE_ALIGN(qp->attrs.sq_size * SQEBB_SIZE) + 631 qp->attrs.rq_size * RQE_SIZE)) 632 return -EINVAL; 633 634 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va, 635 qp->attrs.sq_size << SQEBB_SHIFT, 0, va, 636 (SZ_1M - SZ_4K), 1); 637 if (ret) 638 return ret; 639 640 rq_offset = PAGE_ALIGN(qp->attrs.sq_size << SQEBB_SHIFT); 641 qp->user_qp.rq_offset = rq_offset; 642 643 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset, 644 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, 645 (SZ_1M - SZ_4K), 1); 646 if (ret) 647 goto put_sq_mtt; 648 649 ret = erdma_map_user_dbrecords(uctx, db_info_va, 650 &qp->user_qp.user_dbr_page, 651 &db_info_dma_addr); 652 if (ret) 653 goto put_rq_mtt; 654 655 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; 656 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; 657 658 return 0; 659 660 put_rq_mtt: 661 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 662 663 put_sq_mtt: 664 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 665 666 return ret; 667 } 668 669 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) 670 { 671 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 672 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 673 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); 674 } 675 676 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 677 struct ib_udata *udata) 678 { 679 struct erdma_qp *qp = to_eqp(ibqp); 680 struct erdma_dev *dev = to_edev(ibqp->device); 681 struct erdma_ucontext *uctx = rdma_udata_to_drv_context( 682 udata, struct erdma_ucontext, ibucontext); 683 struct erdma_ureq_create_qp ureq; 684 struct erdma_uresp_create_qp uresp; 685 int ret; 686 687 ret = erdma_qp_validate_cap(dev, attrs); 688 if (ret) 689 goto err_out; 690 691 ret = erdma_qp_validate_attr(dev, attrs); 692 if (ret) 693 goto err_out; 694 695 qp->scq = to_ecq(attrs->send_cq); 696 qp->rcq = to_ecq(attrs->recv_cq); 697 qp->dev = dev; 698 qp->attrs.cc = dev->attrs.cc; 699 700 init_rwsem(&qp->state_lock); 701 kref_init(&qp->ref); 702 init_completion(&qp->safe_free); 703 704 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, 705 XA_LIMIT(1, dev->attrs.max_qp - 1), 706 &dev->next_alloc_qpn, GFP_KERNEL); 707 if (ret < 0) { 708 ret = -ENOMEM; 709 goto err_out; 710 } 711 712 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr * 713 ERDMA_MAX_WQEBB_PER_SQE); 714 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr); 715 716 if (uctx) { 717 ret = ib_copy_from_udata(&ureq, udata, 718 min(sizeof(ureq), udata->inlen)); 719 if (ret) 720 goto err_out_xa; 721 722 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len, 723 ureq.db_record_va); 724 if (ret) 725 goto err_out_xa; 726 727 memset(&uresp, 0, sizeof(uresp)); 728 729 uresp.num_sqe = qp->attrs.sq_size; 730 uresp.num_rqe = qp->attrs.rq_size; 731 uresp.qp_id = QP_ID(qp); 732 uresp.rq_offset = qp->user_qp.rq_offset; 733 734 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 735 if (ret) 736 goto err_out_cmd; 737 } else { 738 init_kernel_qp(dev, qp, attrs); 739 } 740 741 qp->attrs.max_send_sge = attrs->cap.max_send_sge; 742 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge; 743 qp->attrs.state = ERDMA_QP_STATE_IDLE; 744 745 ret = create_qp_cmd(dev, qp); 746 if (ret) 747 goto err_out_cmd; 748 749 spin_lock_init(&qp->lock); 750 751 return 0; 752 753 err_out_cmd: 754 if (uctx) 755 free_user_qp(qp, uctx); 756 else 757 free_kernel_qp(qp); 758 err_out_xa: 759 xa_erase(&dev->qp_xa, QP_ID(qp)); 760 err_out: 761 return ret; 762 } 763 764 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag) 765 { 766 int stag_idx; 767 768 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]); 769 if (stag_idx < 0) 770 return stag_idx; 771 772 /* For now, we always let key field be zero. */ 773 *stag = (stag_idx << 8); 774 775 return 0; 776 } 777 778 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc) 779 { 780 struct erdma_dev *dev = to_edev(ibpd->device); 781 struct erdma_mr *mr; 782 u32 stag; 783 int ret; 784 785 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 786 if (!mr) 787 return ERR_PTR(-ENOMEM); 788 789 ret = erdma_create_stag(dev, &stag); 790 if (ret) 791 goto out_free; 792 793 mr->type = ERDMA_MR_TYPE_DMA; 794 795 mr->ibmr.lkey = stag; 796 mr->ibmr.rkey = stag; 797 mr->ibmr.pd = ibpd; 798 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc); 799 ret = regmr_cmd(dev, mr); 800 if (ret) 801 goto out_remove_stag; 802 803 return &mr->ibmr; 804 805 out_remove_stag: 806 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 807 mr->ibmr.lkey >> 8); 808 809 out_free: 810 kfree(mr); 811 812 return ERR_PTR(ret); 813 } 814 815 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 816 u32 max_num_sg) 817 { 818 struct erdma_mr *mr; 819 struct erdma_dev *dev = to_edev(ibpd->device); 820 int ret; 821 u32 stag; 822 823 if (mr_type != IB_MR_TYPE_MEM_REG) 824 return ERR_PTR(-EOPNOTSUPP); 825 826 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT) 827 return ERR_PTR(-EINVAL); 828 829 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 830 if (!mr) 831 return ERR_PTR(-ENOMEM); 832 833 ret = erdma_create_stag(dev, &stag); 834 if (ret) 835 goto out_free; 836 837 mr->type = ERDMA_MR_TYPE_FRMR; 838 839 mr->ibmr.lkey = stag; 840 mr->ibmr.rkey = stag; 841 mr->ibmr.pd = ibpd; 842 /* update it in FRMR. */ 843 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR | 844 ERDMA_MR_ACC_RW; 845 846 mr->mem.page_size = PAGE_SIZE; /* update it later. */ 847 mr->mem.page_cnt = max_num_sg; 848 mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT; 849 mr->mem.mtt_buf = 850 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL); 851 if (!mr->mem.mtt_buf) { 852 ret = -ENOMEM; 853 goto out_remove_stag; 854 } 855 856 mr->mem.mtt_entry[0] = 857 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf, 858 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 859 if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) { 860 ret = -ENOMEM; 861 goto out_free_mtt; 862 } 863 864 ret = regmr_cmd(dev, mr); 865 if (ret) 866 goto out_dma_unmap; 867 868 return &mr->ibmr; 869 870 out_dma_unmap: 871 dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0], 872 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 873 out_free_mtt: 874 free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt)); 875 876 out_remove_stag: 877 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 878 mr->ibmr.lkey >> 8); 879 880 out_free: 881 kfree(mr); 882 883 return ERR_PTR(ret); 884 } 885 886 static int erdma_set_page(struct ib_mr *ibmr, u64 addr) 887 { 888 struct erdma_mr *mr = to_emr(ibmr); 889 890 if (mr->mem.mtt_nents >= mr->mem.page_cnt) 891 return -1; 892 893 *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr; 894 mr->mem.mtt_nents++; 895 896 return 0; 897 } 898 899 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 900 unsigned int *sg_offset) 901 { 902 struct erdma_mr *mr = to_emr(ibmr); 903 int num; 904 905 mr->mem.mtt_nents = 0; 906 907 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset, 908 erdma_set_page); 909 910 return num; 911 } 912 913 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 914 u64 virt, int access, struct ib_udata *udata) 915 { 916 struct erdma_mr *mr = NULL; 917 struct erdma_dev *dev = to_edev(ibpd->device); 918 u32 stag; 919 int ret; 920 921 if (!len || len > dev->attrs.max_mr_size) 922 return ERR_PTR(-EINVAL); 923 924 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 925 if (!mr) 926 return ERR_PTR(-ENOMEM); 927 928 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, 929 SZ_2G - SZ_4K, 0); 930 if (ret) 931 goto err_out_free; 932 933 ret = erdma_create_stag(dev, &stag); 934 if (ret) 935 goto err_out_put_mtt; 936 937 mr->ibmr.lkey = mr->ibmr.rkey = stag; 938 mr->ibmr.pd = ibpd; 939 mr->mem.va = virt; 940 mr->mem.len = len; 941 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); 942 mr->valid = 1; 943 mr->type = ERDMA_MR_TYPE_NORMAL; 944 945 ret = regmr_cmd(dev, mr); 946 if (ret) 947 goto err_out_mr; 948 949 return &mr->ibmr; 950 951 err_out_mr: 952 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 953 mr->ibmr.lkey >> 8); 954 955 err_out_put_mtt: 956 put_mtt_entries(dev, &mr->mem); 957 958 err_out_free: 959 kfree(mr); 960 961 return ERR_PTR(ret); 962 } 963 964 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 965 { 966 struct erdma_mr *mr; 967 struct erdma_dev *dev = to_edev(ibmr->device); 968 struct erdma_cmdq_dereg_mr_req req; 969 int ret; 970 971 mr = to_emr(ibmr); 972 973 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 974 CMDQ_OPCODE_DEREG_MR); 975 976 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | 977 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); 978 979 ret = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, 980 NULL); 981 if (ret) 982 return ret; 983 984 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); 985 986 put_mtt_entries(dev, &mr->mem); 987 988 kfree(mr); 989 return 0; 990 } 991 992 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 993 { 994 struct erdma_cq *cq = to_ecq(ibcq); 995 struct erdma_dev *dev = to_edev(ibcq->device); 996 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 997 udata, struct erdma_ucontext, ibucontext); 998 int err; 999 struct erdma_cmdq_destroy_cq_req req; 1000 1001 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1002 CMDQ_OPCODE_DESTROY_CQ); 1003 req.cqn = cq->cqn; 1004 1005 err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, 1006 NULL); 1007 if (err) 1008 return err; 1009 1010 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1011 dma_free_coherent(&dev->pdev->dev, 1012 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1013 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1014 } else { 1015 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1016 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1017 } 1018 1019 xa_erase(&dev->cq_xa, cq->cqn); 1020 1021 return 0; 1022 } 1023 1024 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1025 { 1026 struct erdma_qp *qp = to_eqp(ibqp); 1027 struct erdma_dev *dev = to_edev(ibqp->device); 1028 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1029 udata, struct erdma_ucontext, ibucontext); 1030 struct erdma_qp_attrs qp_attrs; 1031 int err; 1032 struct erdma_cmdq_destroy_qp_req req; 1033 1034 down_write(&qp->state_lock); 1035 qp_attrs.state = ERDMA_QP_STATE_ERROR; 1036 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE); 1037 up_write(&qp->state_lock); 1038 1039 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1040 CMDQ_OPCODE_DESTROY_QP); 1041 req.qpn = QP_ID(qp); 1042 1043 err = erdma_post_cmd_wait(&dev->cmdq, (u64 *)&req, sizeof(req), NULL, 1044 NULL); 1045 if (err) 1046 return err; 1047 1048 erdma_qp_put(qp); 1049 wait_for_completion(&qp->safe_free); 1050 1051 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1052 vfree(qp->kern_qp.swr_tbl); 1053 vfree(qp->kern_qp.rwr_tbl); 1054 dma_free_coherent( 1055 &dev->pdev->dev, 1056 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 1057 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 1058 dma_free_coherent( 1059 &dev->pdev->dev, 1060 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 1061 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 1062 } else { 1063 put_mtt_entries(dev, &qp->user_qp.sq_mtt); 1064 put_mtt_entries(dev, &qp->user_qp.rq_mtt); 1065 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); 1066 } 1067 1068 if (qp->cep) 1069 erdma_cep_put(qp->cep); 1070 xa_erase(&dev->qp_xa, QP_ID(qp)); 1071 1072 return 0; 1073 } 1074 1075 void erdma_qp_get_ref(struct ib_qp *ibqp) 1076 { 1077 erdma_qp_get(to_eqp(ibqp)); 1078 } 1079 1080 void erdma_qp_put_ref(struct ib_qp *ibqp) 1081 { 1082 erdma_qp_put(to_eqp(ibqp)); 1083 } 1084 1085 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) 1086 { 1087 struct rdma_user_mmap_entry *rdma_entry; 1088 struct erdma_user_mmap_entry *entry; 1089 pgprot_t prot; 1090 int err; 1091 1092 rdma_entry = rdma_user_mmap_entry_get(ctx, vma); 1093 if (!rdma_entry) 1094 return -EINVAL; 1095 1096 entry = to_emmap(rdma_entry); 1097 1098 switch (entry->mmap_flag) { 1099 case ERDMA_MMAP_IO_NC: 1100 /* map doorbell. */ 1101 prot = pgprot_device(vma->vm_page_prot); 1102 break; 1103 default: 1104 return -EINVAL; 1105 } 1106 1107 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1108 prot, rdma_entry); 1109 1110 rdma_user_mmap_entry_put(rdma_entry); 1111 return err; 1112 } 1113 1114 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1115 { 1116 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry); 1117 1118 kfree(entry); 1119 } 1120 1121 #define ERDMA_SDB_PAGE 0 1122 #define ERDMA_SDB_ENTRY 1 1123 #define ERDMA_SDB_SHARED 2 1124 1125 static void alloc_db_resources(struct erdma_dev *dev, 1126 struct erdma_ucontext *ctx) 1127 { 1128 u32 bitmap_idx; 1129 struct erdma_devattr *attrs = &dev->attrs; 1130 1131 if (attrs->disable_dwqe) 1132 goto alloc_normal_db; 1133 1134 /* Try to alloc independent SDB page. */ 1135 spin_lock(&dev->db_bitmap_lock); 1136 bitmap_idx = find_first_zero_bit(dev->sdb_page, attrs->dwqe_pages); 1137 if (bitmap_idx != attrs->dwqe_pages) { 1138 set_bit(bitmap_idx, dev->sdb_page); 1139 spin_unlock(&dev->db_bitmap_lock); 1140 1141 ctx->sdb_type = ERDMA_SDB_PAGE; 1142 ctx->sdb_idx = bitmap_idx; 1143 ctx->sdb_page_idx = bitmap_idx; 1144 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET + 1145 (bitmap_idx << PAGE_SHIFT); 1146 ctx->sdb_page_off = 0; 1147 1148 return; 1149 } 1150 1151 bitmap_idx = find_first_zero_bit(dev->sdb_entry, attrs->dwqe_entries); 1152 if (bitmap_idx != attrs->dwqe_entries) { 1153 set_bit(bitmap_idx, dev->sdb_entry); 1154 spin_unlock(&dev->db_bitmap_lock); 1155 1156 ctx->sdb_type = ERDMA_SDB_ENTRY; 1157 ctx->sdb_idx = bitmap_idx; 1158 ctx->sdb_page_idx = attrs->dwqe_pages + 1159 bitmap_idx / ERDMA_DWQE_TYPE1_CNT_PER_PAGE; 1160 ctx->sdb_page_off = bitmap_idx % ERDMA_DWQE_TYPE1_CNT_PER_PAGE; 1161 1162 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET + 1163 (ctx->sdb_page_idx << PAGE_SHIFT); 1164 1165 return; 1166 } 1167 1168 spin_unlock(&dev->db_bitmap_lock); 1169 1170 alloc_normal_db: 1171 ctx->sdb_type = ERDMA_SDB_SHARED; 1172 ctx->sdb_idx = 0; 1173 ctx->sdb_page_idx = ERDMA_SDB_SHARED_PAGE_INDEX; 1174 ctx->sdb_page_off = 0; 1175 1176 ctx->sdb = dev->func_bar_addr + (ctx->sdb_page_idx << PAGE_SHIFT); 1177 } 1178 1179 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx) 1180 { 1181 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry); 1182 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry); 1183 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry); 1184 } 1185 1186 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 1187 { 1188 struct erdma_ucontext *ctx = to_ectx(ibctx); 1189 struct erdma_dev *dev = to_edev(ibctx->device); 1190 int ret; 1191 struct erdma_uresp_alloc_ctx uresp = {}; 1192 1193 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) { 1194 ret = -ENOMEM; 1195 goto err_out; 1196 } 1197 1198 INIT_LIST_HEAD(&ctx->dbrecords_page_list); 1199 mutex_init(&ctx->dbrecords_page_mutex); 1200 1201 alloc_db_resources(dev, ctx); 1202 1203 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET; 1204 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET; 1205 1206 if (udata->outlen < sizeof(uresp)) { 1207 ret = -EINVAL; 1208 goto err_out; 1209 } 1210 1211 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert( 1212 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb); 1213 if (!ctx->sq_db_mmap_entry) { 1214 ret = -ENOMEM; 1215 goto err_out; 1216 } 1217 1218 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert( 1219 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb); 1220 if (!ctx->rq_db_mmap_entry) { 1221 ret = -EINVAL; 1222 goto err_out; 1223 } 1224 1225 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert( 1226 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb); 1227 if (!ctx->cq_db_mmap_entry) { 1228 ret = -EINVAL; 1229 goto err_out; 1230 } 1231 1232 uresp.dev_id = dev->pdev->device; 1233 uresp.sdb_type = ctx->sdb_type; 1234 uresp.sdb_offset = ctx->sdb_page_off; 1235 1236 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1237 if (ret) 1238 goto err_out; 1239 1240 return 0; 1241 1242 err_out: 1243 erdma_uctx_user_mmap_entries_remove(ctx); 1244 atomic_dec(&dev->num_ctx); 1245 return ret; 1246 } 1247 1248 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx) 1249 { 1250 struct erdma_ucontext *ctx = to_ectx(ibctx); 1251 struct erdma_dev *dev = to_edev(ibctx->device); 1252 1253 spin_lock(&dev->db_bitmap_lock); 1254 if (ctx->sdb_type == ERDMA_SDB_PAGE) 1255 clear_bit(ctx->sdb_idx, dev->sdb_page); 1256 else if (ctx->sdb_type == ERDMA_SDB_ENTRY) 1257 clear_bit(ctx->sdb_idx, dev->sdb_entry); 1258 1259 erdma_uctx_user_mmap_entries_remove(ctx); 1260 1261 spin_unlock(&dev->db_bitmap_lock); 1262 1263 atomic_dec(&dev->num_ctx); 1264 } 1265 1266 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = { 1267 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE, 1268 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE, 1269 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR, 1270 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS, 1271 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING, 1272 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE, 1273 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR 1274 }; 1275 1276 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1277 struct ib_udata *udata) 1278 { 1279 struct erdma_qp_attrs new_attrs; 1280 enum erdma_qp_attr_mask erdma_attr_mask = 0; 1281 struct erdma_qp *qp = to_eqp(ibqp); 1282 int ret = 0; 1283 1284 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1285 return -EOPNOTSUPP; 1286 1287 memset(&new_attrs, 0, sizeof(new_attrs)); 1288 1289 if (attr_mask & IB_QP_STATE) { 1290 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state]; 1291 1292 erdma_attr_mask |= ERDMA_QP_ATTR_STATE; 1293 } 1294 1295 down_write(&qp->state_lock); 1296 1297 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask); 1298 1299 up_write(&qp->state_lock); 1300 1301 return ret; 1302 } 1303 1304 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 1305 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1306 { 1307 struct erdma_qp *qp; 1308 struct erdma_dev *dev; 1309 1310 if (ibqp && qp_attr && qp_init_attr) { 1311 qp = to_eqp(ibqp); 1312 dev = to_edev(ibqp->device); 1313 } else { 1314 return -EINVAL; 1315 } 1316 1317 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1318 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1319 1320 qp_attr->cap.max_send_wr = qp->attrs.sq_size; 1321 qp_attr->cap.max_recv_wr = qp->attrs.rq_size; 1322 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge; 1323 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge; 1324 1325 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu); 1326 qp_attr->max_rd_atomic = qp->attrs.irq_size; 1327 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; 1328 1329 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | 1330 IB_ACCESS_REMOTE_WRITE | 1331 IB_ACCESS_REMOTE_READ; 1332 1333 qp_init_attr->cap = qp_attr->cap; 1334 1335 return 0; 1336 } 1337 1338 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, 1339 struct erdma_ureq_create_cq *ureq) 1340 { 1341 int ret; 1342 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1343 1344 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va, 1345 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, 1346 1); 1347 if (ret) 1348 return ret; 1349 1350 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, 1351 &cq->user_cq.user_dbr_page, 1352 &cq->user_cq.db_info_dma_addr); 1353 if (ret) 1354 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1355 1356 return ret; 1357 } 1358 1359 static int erdma_init_kernel_cq(struct erdma_cq *cq) 1360 { 1361 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1362 1363 cq->kern_cq.qbuf = 1364 dma_alloc_coherent(&dev->pdev->dev, 1365 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1366 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1367 if (!cq->kern_cq.qbuf) 1368 return -ENOMEM; 1369 1370 cq->kern_cq.db_record = 1371 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); 1372 spin_lock_init(&cq->kern_cq.lock); 1373 /* use default cqdb addr */ 1374 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1375 1376 return 0; 1377 } 1378 1379 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1380 struct ib_udata *udata) 1381 { 1382 struct erdma_cq *cq = to_ecq(ibcq); 1383 struct erdma_dev *dev = to_edev(ibcq->device); 1384 unsigned int depth = attr->cqe; 1385 int ret; 1386 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1387 udata, struct erdma_ucontext, ibucontext); 1388 1389 if (depth > dev->attrs.max_cqe) 1390 return -EINVAL; 1391 1392 depth = roundup_pow_of_two(depth); 1393 cq->ibcq.cqe = depth; 1394 cq->depth = depth; 1395 cq->assoc_eqn = attr->comp_vector + 1; 1396 1397 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, 1398 XA_LIMIT(1, dev->attrs.max_cq - 1), 1399 &dev->next_alloc_cqn, GFP_KERNEL); 1400 if (ret < 0) 1401 return ret; 1402 1403 if (!rdma_is_kernel_res(&ibcq->res)) { 1404 struct erdma_ureq_create_cq ureq; 1405 struct erdma_uresp_create_cq uresp; 1406 1407 ret = ib_copy_from_udata(&ureq, udata, 1408 min(udata->inlen, sizeof(ureq))); 1409 if (ret) 1410 goto err_out_xa; 1411 1412 ret = erdma_init_user_cq(ctx, cq, &ureq); 1413 if (ret) 1414 goto err_out_xa; 1415 1416 uresp.cq_id = cq->cqn; 1417 uresp.num_cqe = depth; 1418 1419 ret = ib_copy_to_udata(udata, &uresp, 1420 min(sizeof(uresp), udata->outlen)); 1421 if (ret) 1422 goto err_free_res; 1423 } else { 1424 ret = erdma_init_kernel_cq(cq); 1425 if (ret) 1426 goto err_out_xa; 1427 } 1428 1429 ret = create_cq_cmd(dev, cq); 1430 if (ret) 1431 goto err_free_res; 1432 1433 return 0; 1434 1435 err_free_res: 1436 if (!rdma_is_kernel_res(&ibcq->res)) { 1437 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1438 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1439 } else { 1440 dma_free_coherent(&dev->pdev->dev, 1441 WARPPED_BUFSIZE(depth << CQE_SHIFT), 1442 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1443 } 1444 1445 err_out_xa: 1446 xa_erase(&dev->cq_xa, cq->cqn); 1447 1448 return ret; 1449 } 1450 1451 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) 1452 { 1453 struct ib_event event; 1454 1455 event.device = &dev->ibdev; 1456 event.element.port_num = 1; 1457 event.event = reason; 1458 1459 ib_dispatch_event(&event); 1460 } 1461