1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Copyright (c) 2008-2019, IBM Corporation */ 9 10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ 11 12 #include <linux/vmalloc.h> 13 #include <net/addrconf.h> 14 #include <rdma/erdma-abi.h> 15 #include <rdma/ib_umem.h> 16 #include <rdma/uverbs_ioctl.h> 17 18 #include "erdma.h" 19 #include "erdma_cm.h" 20 #include "erdma_verbs.h" 21 22 static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp) 23 { 24 struct erdma_cmdq_create_qp_req req; 25 struct erdma_pd *pd = to_epd(qp->ibqp.pd); 26 struct erdma_uqp *user_qp; 27 u64 resp0, resp1; 28 int err; 29 30 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 31 CMDQ_OPCODE_CREATE_QP); 32 33 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK, 34 ilog2(qp->attrs.sq_size)) | 35 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp)); 36 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK, 37 ilog2(qp->attrs.rq_size)) | 38 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn); 39 40 if (rdma_is_kernel_res(&qp->ibqp.res)) { 41 u32 pgsz_range = ilog2(SZ_1M) - PAGE_SHIFT; 42 43 req.sq_cqn_mtt_cfg = 44 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 45 pgsz_range) | 46 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 47 req.rq_cqn_mtt_cfg = 48 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 49 pgsz_range) | 50 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 51 52 req.sq_mtt_cfg = 53 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | 54 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | 55 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 56 ERDMA_MR_INLINE_MTT); 57 req.rq_mtt_cfg = req.sq_mtt_cfg; 58 59 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 60 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 61 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + 62 (qp->attrs.sq_size << SQEBB_SHIFT); 63 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + 64 (qp->attrs.rq_size << RQE_SHIFT); 65 } else { 66 user_qp = &qp->user_qp; 67 req.sq_cqn_mtt_cfg = FIELD_PREP( 68 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 69 ilog2(user_qp->sq_mtt.page_size) - PAGE_SHIFT); 70 req.sq_cqn_mtt_cfg |= 71 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 72 73 req.rq_cqn_mtt_cfg = FIELD_PREP( 74 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 75 ilog2(user_qp->rq_mtt.page_size) - PAGE_SHIFT); 76 req.rq_cqn_mtt_cfg |= 77 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 78 79 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset; 80 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 81 user_qp->sq_mtt.mtt_nents) | 82 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 83 user_qp->sq_mtt.mtt_type); 84 85 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset; 86 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 87 user_qp->rq_mtt.mtt_nents) | 88 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 89 user_qp->rq_mtt.mtt_type); 90 91 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0]; 92 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0]; 93 94 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; 95 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; 96 } 97 98 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, 99 &resp1); 100 if (!err) 101 qp->attrs.cookie = 102 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0); 103 104 return err; 105 } 106 107 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 108 { 109 struct erdma_cmdq_reg_mr_req req; 110 struct erdma_pd *pd = to_epd(mr->ibmr.pd); 111 u64 *phy_addr; 112 int i; 113 114 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR); 115 116 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) | 117 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) | 118 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8); 119 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) | 120 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) | 121 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access) | 122 FIELD_PREP(ERDMA_CMD_REGMR_ACC_MODE_MASK, 0); 123 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK, 124 ilog2(mr->mem.page_size)) | 125 FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) | 126 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt); 127 128 if (mr->type == ERDMA_MR_TYPE_DMA) 129 goto post_cmd; 130 131 if (mr->type == ERDMA_MR_TYPE_NORMAL) { 132 req.start_va = mr->mem.va; 133 req.size = mr->mem.len; 134 } 135 136 if (mr->type == ERDMA_MR_TYPE_FRMR || 137 mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) { 138 phy_addr = req.phy_addr; 139 *phy_addr = mr->mem.mtt_entry[0]; 140 } else { 141 phy_addr = req.phy_addr; 142 for (i = 0; i < mr->mem.mtt_nents; i++) 143 *phy_addr++ = mr->mem.mtt_entry[i]; 144 } 145 146 post_cmd: 147 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 148 } 149 150 static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq) 151 { 152 struct erdma_cmdq_create_cq_req req; 153 u32 page_size; 154 struct erdma_mem *mtt; 155 156 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 157 CMDQ_OPCODE_CREATE_CQ); 158 159 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) | 160 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth)); 161 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); 162 163 if (rdma_is_kernel_res(&cq->ibcq.res)) { 164 page_size = SZ_32M; 165 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 166 ilog2(page_size) - PAGE_SHIFT); 167 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); 168 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); 169 170 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | 171 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 172 ERDMA_MR_INLINE_MTT); 173 174 req.first_page_offset = 0; 175 req.cq_db_info_addr = 176 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); 177 } else { 178 mtt = &cq->user_cq.qbuf_mtt; 179 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 180 ilog2(mtt->page_size) - PAGE_SHIFT); 181 if (mtt->mtt_nents == 1) { 182 req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf); 183 req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf); 184 } else { 185 req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]); 186 req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]); 187 } 188 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 189 mtt->mtt_nents); 190 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 191 mtt->mtt_type); 192 193 req.first_page_offset = mtt->page_offset; 194 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; 195 } 196 197 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 198 } 199 200 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) 201 { 202 int idx; 203 unsigned long flags; 204 205 spin_lock_irqsave(&res_cb->lock, flags); 206 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, 207 res_cb->next_alloc_idx); 208 if (idx == res_cb->max_cap) { 209 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap); 210 if (idx == res_cb->max_cap) { 211 res_cb->next_alloc_idx = 1; 212 spin_unlock_irqrestore(&res_cb->lock, flags); 213 return -ENOSPC; 214 } 215 } 216 217 set_bit(idx, res_cb->bitmap); 218 res_cb->next_alloc_idx = idx + 1; 219 spin_unlock_irqrestore(&res_cb->lock, flags); 220 221 return idx; 222 } 223 224 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx) 225 { 226 unsigned long flags; 227 u32 used; 228 229 spin_lock_irqsave(&res_cb->lock, flags); 230 used = __test_and_clear_bit(idx, res_cb->bitmap); 231 spin_unlock_irqrestore(&res_cb->lock, flags); 232 WARN_ON(!used); 233 } 234 235 static struct rdma_user_mmap_entry * 236 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, 237 u32 size, u8 mmap_flag, u64 *mmap_offset) 238 { 239 struct erdma_user_mmap_entry *entry = 240 kzalloc(sizeof(*entry), GFP_KERNEL); 241 int ret; 242 243 if (!entry) 244 return NULL; 245 246 entry->address = (u64)address; 247 entry->mmap_flag = mmap_flag; 248 249 size = PAGE_ALIGN(size); 250 251 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry, 252 size); 253 if (ret) { 254 kfree(entry); 255 return NULL; 256 } 257 258 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 259 260 return &entry->rdma_entry; 261 } 262 263 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 264 struct ib_udata *unused) 265 { 266 struct erdma_dev *dev = to_edev(ibdev); 267 268 memset(attr, 0, sizeof(*attr)); 269 270 attr->max_mr_size = dev->attrs.max_mr_size; 271 attr->vendor_id = PCI_VENDOR_ID_ALIBABA; 272 attr->vendor_part_id = dev->pdev->device; 273 attr->hw_ver = dev->pdev->revision; 274 attr->max_qp = dev->attrs.max_qp - 1; 275 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr); 276 attr->max_qp_rd_atom = dev->attrs.max_ord; 277 attr->max_qp_init_rd_atom = dev->attrs.max_ird; 278 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird; 279 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 280 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 281 ibdev->local_dma_lkey = dev->attrs.local_dma_key; 282 attr->max_send_sge = dev->attrs.max_send_sge; 283 attr->max_recv_sge = dev->attrs.max_recv_sge; 284 attr->max_sge_rd = dev->attrs.max_sge_rd; 285 attr->max_cq = dev->attrs.max_cq - 1; 286 attr->max_cqe = dev->attrs.max_cqe; 287 attr->max_mr = dev->attrs.max_mr; 288 attr->max_pd = dev->attrs.max_pd; 289 attr->max_mw = dev->attrs.max_mw; 290 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA; 291 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT; 292 attr->fw_ver = dev->attrs.fw_version; 293 294 if (dev->netdev) 295 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 296 dev->netdev->dev_addr); 297 298 return 0; 299 } 300 301 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, 302 union ib_gid *gid) 303 { 304 struct erdma_dev *dev = to_edev(ibdev); 305 306 memset(gid, 0, sizeof(*gid)); 307 ether_addr_copy(gid->raw, dev->attrs.peer_addr); 308 309 return 0; 310 } 311 312 int erdma_query_port(struct ib_device *ibdev, u32 port, 313 struct ib_port_attr *attr) 314 { 315 struct erdma_dev *dev = to_edev(ibdev); 316 struct net_device *ndev = dev->netdev; 317 318 memset(attr, 0, sizeof(*attr)); 319 320 attr->gid_tbl_len = 1; 321 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; 322 attr->max_msg_sz = -1; 323 324 if (!ndev) 325 goto out; 326 327 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width); 328 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu); 329 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 330 if (netif_running(ndev) && netif_carrier_ok(ndev)) 331 dev->state = IB_PORT_ACTIVE; 332 else 333 dev->state = IB_PORT_DOWN; 334 attr->state = dev->state; 335 336 out: 337 if (dev->state == IB_PORT_ACTIVE) 338 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 339 else 340 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 341 342 return 0; 343 } 344 345 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port, 346 struct ib_port_immutable *port_immutable) 347 { 348 port_immutable->gid_tbl_len = 1; 349 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 350 351 return 0; 352 } 353 354 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 355 { 356 struct erdma_pd *pd = to_epd(ibpd); 357 struct erdma_dev *dev = to_edev(ibpd->device); 358 int pdn; 359 360 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]); 361 if (pdn < 0) 362 return pdn; 363 364 pd->pdn = pdn; 365 366 return 0; 367 } 368 369 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 370 { 371 struct erdma_pd *pd = to_epd(ibpd); 372 struct erdma_dev *dev = to_edev(ibpd->device); 373 374 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn); 375 376 return 0; 377 } 378 379 static int erdma_qp_validate_cap(struct erdma_dev *dev, 380 struct ib_qp_init_attr *attrs) 381 { 382 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) || 383 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) || 384 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) || 385 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) || 386 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) || 387 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) { 388 return -EINVAL; 389 } 390 391 return 0; 392 } 393 394 static int erdma_qp_validate_attr(struct erdma_dev *dev, 395 struct ib_qp_init_attr *attrs) 396 { 397 if (attrs->qp_type != IB_QPT_RC) 398 return -EOPNOTSUPP; 399 400 if (attrs->srq) 401 return -EOPNOTSUPP; 402 403 if (!attrs->send_cq || !attrs->recv_cq) 404 return -EOPNOTSUPP; 405 406 return 0; 407 } 408 409 static void free_kernel_qp(struct erdma_qp *qp) 410 { 411 struct erdma_dev *dev = qp->dev; 412 413 vfree(qp->kern_qp.swr_tbl); 414 vfree(qp->kern_qp.rwr_tbl); 415 416 if (qp->kern_qp.sq_buf) 417 dma_free_coherent( 418 &dev->pdev->dev, 419 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 420 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 421 422 if (qp->kern_qp.rq_buf) 423 dma_free_coherent( 424 &dev->pdev->dev, 425 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 426 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 427 } 428 429 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, 430 struct ib_qp_init_attr *attrs) 431 { 432 struct erdma_kqp *kqp = &qp->kern_qp; 433 int size; 434 435 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) 436 kqp->sig_all = 1; 437 438 kqp->sq_pi = 0; 439 kqp->sq_ci = 0; 440 kqp->rq_pi = 0; 441 kqp->rq_ci = 0; 442 kqp->hw_sq_db = 443 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); 444 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; 445 446 kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64)); 447 kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64)); 448 if (!kqp->swr_tbl || !kqp->rwr_tbl) 449 goto err_out; 450 451 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 452 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 453 &kqp->sq_buf_dma_addr, GFP_KERNEL); 454 if (!kqp->sq_buf) 455 goto err_out; 456 457 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 458 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 459 &kqp->rq_buf_dma_addr, GFP_KERNEL); 460 if (!kqp->rq_buf) 461 goto err_out; 462 463 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); 464 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); 465 466 return 0; 467 468 err_out: 469 free_kernel_qp(qp); 470 return -ENOMEM; 471 } 472 473 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, 474 u64 start, u64 len, int access, u64 virt, 475 unsigned long req_page_size, u8 force_indirect_mtt) 476 { 477 struct ib_block_iter biter; 478 uint64_t *phy_addr = NULL; 479 int ret = 0; 480 481 mem->umem = ib_umem_get(&dev->ibdev, start, len, access); 482 if (IS_ERR(mem->umem)) { 483 ret = PTR_ERR(mem->umem); 484 mem->umem = NULL; 485 return ret; 486 } 487 488 mem->va = virt; 489 mem->len = len; 490 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); 491 mem->page_offset = start & (mem->page_size - 1); 492 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); 493 mem->page_cnt = mem->mtt_nents; 494 495 if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES || 496 force_indirect_mtt) { 497 mem->mtt_type = ERDMA_MR_INDIRECT_MTT; 498 mem->mtt_buf = 499 alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL); 500 if (!mem->mtt_buf) { 501 ret = -ENOMEM; 502 goto error_ret; 503 } 504 phy_addr = mem->mtt_buf; 505 } else { 506 mem->mtt_type = ERDMA_MR_INLINE_MTT; 507 phy_addr = mem->mtt_entry; 508 } 509 510 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) { 511 *phy_addr = rdma_block_iter_dma_address(&biter); 512 phy_addr++; 513 } 514 515 if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) { 516 mem->mtt_entry[0] = 517 dma_map_single(&dev->pdev->dev, mem->mtt_buf, 518 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 519 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) { 520 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 521 mem->mtt_buf = NULL; 522 ret = -ENOMEM; 523 goto error_ret; 524 } 525 } 526 527 return 0; 528 529 error_ret: 530 if (mem->umem) { 531 ib_umem_release(mem->umem); 532 mem->umem = NULL; 533 } 534 535 return ret; 536 } 537 538 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) 539 { 540 if (mem->mtt_buf) { 541 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0], 542 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 543 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 544 } 545 546 if (mem->umem) { 547 ib_umem_release(mem->umem); 548 mem->umem = NULL; 549 } 550 } 551 552 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, 553 u64 dbrecords_va, 554 struct erdma_user_dbrecords_page **dbr_page, 555 dma_addr_t *dma_addr) 556 { 557 struct erdma_user_dbrecords_page *page = NULL; 558 int rv = 0; 559 560 mutex_lock(&ctx->dbrecords_page_mutex); 561 562 list_for_each_entry(page, &ctx->dbrecords_page_list, list) 563 if (page->va == (dbrecords_va & PAGE_MASK)) 564 goto found; 565 566 page = kmalloc(sizeof(*page), GFP_KERNEL); 567 if (!page) { 568 rv = -ENOMEM; 569 goto out; 570 } 571 572 page->va = (dbrecords_va & PAGE_MASK); 573 page->refcnt = 0; 574 575 page->umem = ib_umem_get(ctx->ibucontext.device, 576 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); 577 if (IS_ERR(page->umem)) { 578 rv = PTR_ERR(page->umem); 579 kfree(page); 580 goto out; 581 } 582 583 list_add(&page->list, &ctx->dbrecords_page_list); 584 585 found: 586 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) + 587 (dbrecords_va & ~PAGE_MASK); 588 *dbr_page = page; 589 page->refcnt++; 590 591 out: 592 mutex_unlock(&ctx->dbrecords_page_mutex); 593 return rv; 594 } 595 596 static void 597 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, 598 struct erdma_user_dbrecords_page **dbr_page) 599 { 600 if (!ctx || !(*dbr_page)) 601 return; 602 603 mutex_lock(&ctx->dbrecords_page_mutex); 604 if (--(*dbr_page)->refcnt == 0) { 605 list_del(&(*dbr_page)->list); 606 ib_umem_release((*dbr_page)->umem); 607 kfree(*dbr_page); 608 } 609 610 *dbr_page = NULL; 611 mutex_unlock(&ctx->dbrecords_page_mutex); 612 } 613 614 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, 615 u64 va, u32 len, u64 db_info_va) 616 { 617 dma_addr_t db_info_dma_addr; 618 u32 rq_offset; 619 int ret; 620 621 if (len < (PAGE_ALIGN(qp->attrs.sq_size * SQEBB_SIZE) + 622 qp->attrs.rq_size * RQE_SIZE)) 623 return -EINVAL; 624 625 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va, 626 qp->attrs.sq_size << SQEBB_SHIFT, 0, va, 627 (SZ_1M - SZ_4K), 1); 628 if (ret) 629 return ret; 630 631 rq_offset = PAGE_ALIGN(qp->attrs.sq_size << SQEBB_SHIFT); 632 qp->user_qp.rq_offset = rq_offset; 633 634 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset, 635 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, 636 (SZ_1M - SZ_4K), 1); 637 if (ret) 638 goto put_sq_mtt; 639 640 ret = erdma_map_user_dbrecords(uctx, db_info_va, 641 &qp->user_qp.user_dbr_page, 642 &db_info_dma_addr); 643 if (ret) 644 goto put_rq_mtt; 645 646 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; 647 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; 648 649 return 0; 650 651 put_rq_mtt: 652 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 653 654 put_sq_mtt: 655 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 656 657 return ret; 658 } 659 660 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) 661 { 662 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 663 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 664 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); 665 } 666 667 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 668 struct ib_udata *udata) 669 { 670 struct erdma_qp *qp = to_eqp(ibqp); 671 struct erdma_dev *dev = to_edev(ibqp->device); 672 struct erdma_ucontext *uctx = rdma_udata_to_drv_context( 673 udata, struct erdma_ucontext, ibucontext); 674 struct erdma_ureq_create_qp ureq; 675 struct erdma_uresp_create_qp uresp; 676 int ret; 677 678 ret = erdma_qp_validate_cap(dev, attrs); 679 if (ret) 680 goto err_out; 681 682 ret = erdma_qp_validate_attr(dev, attrs); 683 if (ret) 684 goto err_out; 685 686 qp->scq = to_ecq(attrs->send_cq); 687 qp->rcq = to_ecq(attrs->recv_cq); 688 qp->dev = dev; 689 qp->attrs.cc = dev->attrs.cc; 690 691 init_rwsem(&qp->state_lock); 692 kref_init(&qp->ref); 693 init_completion(&qp->safe_free); 694 695 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, 696 XA_LIMIT(1, dev->attrs.max_qp - 1), 697 &dev->next_alloc_qpn, GFP_KERNEL); 698 if (ret < 0) { 699 ret = -ENOMEM; 700 goto err_out; 701 } 702 703 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr * 704 ERDMA_MAX_WQEBB_PER_SQE); 705 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr); 706 707 if (uctx) { 708 ret = ib_copy_from_udata(&ureq, udata, 709 min(sizeof(ureq), udata->inlen)); 710 if (ret) 711 goto err_out_xa; 712 713 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len, 714 ureq.db_record_va); 715 if (ret) 716 goto err_out_xa; 717 718 memset(&uresp, 0, sizeof(uresp)); 719 720 uresp.num_sqe = qp->attrs.sq_size; 721 uresp.num_rqe = qp->attrs.rq_size; 722 uresp.qp_id = QP_ID(qp); 723 uresp.rq_offset = qp->user_qp.rq_offset; 724 725 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 726 if (ret) 727 goto err_out_cmd; 728 } else { 729 init_kernel_qp(dev, qp, attrs); 730 } 731 732 qp->attrs.max_send_sge = attrs->cap.max_send_sge; 733 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge; 734 qp->attrs.state = ERDMA_QP_STATE_IDLE; 735 736 ret = create_qp_cmd(dev, qp); 737 if (ret) 738 goto err_out_cmd; 739 740 spin_lock_init(&qp->lock); 741 742 return 0; 743 744 err_out_cmd: 745 if (uctx) 746 free_user_qp(qp, uctx); 747 else 748 free_kernel_qp(qp); 749 err_out_xa: 750 xa_erase(&dev->qp_xa, QP_ID(qp)); 751 err_out: 752 return ret; 753 } 754 755 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag) 756 { 757 int stag_idx; 758 759 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]); 760 if (stag_idx < 0) 761 return stag_idx; 762 763 /* For now, we always let key field be zero. */ 764 *stag = (stag_idx << 8); 765 766 return 0; 767 } 768 769 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc) 770 { 771 struct erdma_dev *dev = to_edev(ibpd->device); 772 struct erdma_mr *mr; 773 u32 stag; 774 int ret; 775 776 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 777 if (!mr) 778 return ERR_PTR(-ENOMEM); 779 780 ret = erdma_create_stag(dev, &stag); 781 if (ret) 782 goto out_free; 783 784 mr->type = ERDMA_MR_TYPE_DMA; 785 786 mr->ibmr.lkey = stag; 787 mr->ibmr.rkey = stag; 788 mr->ibmr.pd = ibpd; 789 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc); 790 ret = regmr_cmd(dev, mr); 791 if (ret) 792 goto out_remove_stag; 793 794 return &mr->ibmr; 795 796 out_remove_stag: 797 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 798 mr->ibmr.lkey >> 8); 799 800 out_free: 801 kfree(mr); 802 803 return ERR_PTR(ret); 804 } 805 806 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 807 u32 max_num_sg) 808 { 809 struct erdma_mr *mr; 810 struct erdma_dev *dev = to_edev(ibpd->device); 811 int ret; 812 u32 stag; 813 814 if (mr_type != IB_MR_TYPE_MEM_REG) 815 return ERR_PTR(-EOPNOTSUPP); 816 817 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT) 818 return ERR_PTR(-EINVAL); 819 820 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 821 if (!mr) 822 return ERR_PTR(-ENOMEM); 823 824 ret = erdma_create_stag(dev, &stag); 825 if (ret) 826 goto out_free; 827 828 mr->type = ERDMA_MR_TYPE_FRMR; 829 830 mr->ibmr.lkey = stag; 831 mr->ibmr.rkey = stag; 832 mr->ibmr.pd = ibpd; 833 /* update it in FRMR. */ 834 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR | 835 ERDMA_MR_ACC_RW; 836 837 mr->mem.page_size = PAGE_SIZE; /* update it later. */ 838 mr->mem.page_cnt = max_num_sg; 839 mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT; 840 mr->mem.mtt_buf = 841 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL); 842 if (!mr->mem.mtt_buf) { 843 ret = -ENOMEM; 844 goto out_remove_stag; 845 } 846 847 mr->mem.mtt_entry[0] = 848 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf, 849 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 850 if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) { 851 ret = -ENOMEM; 852 goto out_free_mtt; 853 } 854 855 ret = regmr_cmd(dev, mr); 856 if (ret) 857 goto out_dma_unmap; 858 859 return &mr->ibmr; 860 861 out_dma_unmap: 862 dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0], 863 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 864 out_free_mtt: 865 free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt)); 866 867 out_remove_stag: 868 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 869 mr->ibmr.lkey >> 8); 870 871 out_free: 872 kfree(mr); 873 874 return ERR_PTR(ret); 875 } 876 877 static int erdma_set_page(struct ib_mr *ibmr, u64 addr) 878 { 879 struct erdma_mr *mr = to_emr(ibmr); 880 881 if (mr->mem.mtt_nents >= mr->mem.page_cnt) 882 return -1; 883 884 *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr; 885 mr->mem.mtt_nents++; 886 887 return 0; 888 } 889 890 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 891 unsigned int *sg_offset) 892 { 893 struct erdma_mr *mr = to_emr(ibmr); 894 int num; 895 896 mr->mem.mtt_nents = 0; 897 898 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset, 899 erdma_set_page); 900 901 return num; 902 } 903 904 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 905 u64 virt, int access, struct ib_udata *udata) 906 { 907 struct erdma_mr *mr = NULL; 908 struct erdma_dev *dev = to_edev(ibpd->device); 909 u32 stag; 910 int ret; 911 912 if (!len || len > dev->attrs.max_mr_size) 913 return ERR_PTR(-EINVAL); 914 915 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 916 if (!mr) 917 return ERR_PTR(-ENOMEM); 918 919 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, 920 SZ_2G - SZ_4K, 0); 921 if (ret) 922 goto err_out_free; 923 924 ret = erdma_create_stag(dev, &stag); 925 if (ret) 926 goto err_out_put_mtt; 927 928 mr->ibmr.lkey = mr->ibmr.rkey = stag; 929 mr->ibmr.pd = ibpd; 930 mr->mem.va = virt; 931 mr->mem.len = len; 932 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); 933 mr->valid = 1; 934 mr->type = ERDMA_MR_TYPE_NORMAL; 935 936 ret = regmr_cmd(dev, mr); 937 if (ret) 938 goto err_out_mr; 939 940 return &mr->ibmr; 941 942 err_out_mr: 943 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 944 mr->ibmr.lkey >> 8); 945 946 err_out_put_mtt: 947 put_mtt_entries(dev, &mr->mem); 948 949 err_out_free: 950 kfree(mr); 951 952 return ERR_PTR(ret); 953 } 954 955 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 956 { 957 struct erdma_mr *mr; 958 struct erdma_dev *dev = to_edev(ibmr->device); 959 struct erdma_cmdq_dereg_mr_req req; 960 int ret; 961 962 mr = to_emr(ibmr); 963 964 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 965 CMDQ_OPCODE_DEREG_MR); 966 967 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | 968 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); 969 970 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 971 if (ret) 972 return ret; 973 974 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); 975 976 put_mtt_entries(dev, &mr->mem); 977 978 kfree(mr); 979 return 0; 980 } 981 982 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 983 { 984 struct erdma_cq *cq = to_ecq(ibcq); 985 struct erdma_dev *dev = to_edev(ibcq->device); 986 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 987 udata, struct erdma_ucontext, ibucontext); 988 int err; 989 struct erdma_cmdq_destroy_cq_req req; 990 991 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 992 CMDQ_OPCODE_DESTROY_CQ); 993 req.cqn = cq->cqn; 994 995 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 996 if (err) 997 return err; 998 999 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1000 dma_free_coherent(&dev->pdev->dev, 1001 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1002 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1003 } else { 1004 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1005 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1006 } 1007 1008 xa_erase(&dev->cq_xa, cq->cqn); 1009 1010 return 0; 1011 } 1012 1013 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1014 { 1015 struct erdma_qp *qp = to_eqp(ibqp); 1016 struct erdma_dev *dev = to_edev(ibqp->device); 1017 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1018 udata, struct erdma_ucontext, ibucontext); 1019 struct erdma_qp_attrs qp_attrs; 1020 int err; 1021 struct erdma_cmdq_destroy_qp_req req; 1022 1023 down_write(&qp->state_lock); 1024 qp_attrs.state = ERDMA_QP_STATE_ERROR; 1025 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE); 1026 up_write(&qp->state_lock); 1027 1028 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1029 CMDQ_OPCODE_DESTROY_QP); 1030 req.qpn = QP_ID(qp); 1031 1032 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1033 if (err) 1034 return err; 1035 1036 erdma_qp_put(qp); 1037 wait_for_completion(&qp->safe_free); 1038 1039 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1040 vfree(qp->kern_qp.swr_tbl); 1041 vfree(qp->kern_qp.rwr_tbl); 1042 dma_free_coherent( 1043 &dev->pdev->dev, 1044 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 1045 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 1046 dma_free_coherent( 1047 &dev->pdev->dev, 1048 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 1049 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 1050 } else { 1051 put_mtt_entries(dev, &qp->user_qp.sq_mtt); 1052 put_mtt_entries(dev, &qp->user_qp.rq_mtt); 1053 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); 1054 } 1055 1056 if (qp->cep) 1057 erdma_cep_put(qp->cep); 1058 xa_erase(&dev->qp_xa, QP_ID(qp)); 1059 1060 return 0; 1061 } 1062 1063 void erdma_qp_get_ref(struct ib_qp *ibqp) 1064 { 1065 erdma_qp_get(to_eqp(ibqp)); 1066 } 1067 1068 void erdma_qp_put_ref(struct ib_qp *ibqp) 1069 { 1070 erdma_qp_put(to_eqp(ibqp)); 1071 } 1072 1073 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) 1074 { 1075 struct rdma_user_mmap_entry *rdma_entry; 1076 struct erdma_user_mmap_entry *entry; 1077 pgprot_t prot; 1078 int err; 1079 1080 rdma_entry = rdma_user_mmap_entry_get(ctx, vma); 1081 if (!rdma_entry) 1082 return -EINVAL; 1083 1084 entry = to_emmap(rdma_entry); 1085 1086 switch (entry->mmap_flag) { 1087 case ERDMA_MMAP_IO_NC: 1088 /* map doorbell. */ 1089 prot = pgprot_device(vma->vm_page_prot); 1090 break; 1091 default: 1092 return -EINVAL; 1093 } 1094 1095 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1096 prot, rdma_entry); 1097 1098 rdma_user_mmap_entry_put(rdma_entry); 1099 return err; 1100 } 1101 1102 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1103 { 1104 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry); 1105 1106 kfree(entry); 1107 } 1108 1109 #define ERDMA_SDB_PAGE 0 1110 #define ERDMA_SDB_ENTRY 1 1111 #define ERDMA_SDB_SHARED 2 1112 1113 static void alloc_db_resources(struct erdma_dev *dev, 1114 struct erdma_ucontext *ctx) 1115 { 1116 u32 bitmap_idx; 1117 struct erdma_devattr *attrs = &dev->attrs; 1118 1119 if (attrs->disable_dwqe) 1120 goto alloc_normal_db; 1121 1122 /* Try to alloc independent SDB page. */ 1123 spin_lock(&dev->db_bitmap_lock); 1124 bitmap_idx = find_first_zero_bit(dev->sdb_page, attrs->dwqe_pages); 1125 if (bitmap_idx != attrs->dwqe_pages) { 1126 set_bit(bitmap_idx, dev->sdb_page); 1127 spin_unlock(&dev->db_bitmap_lock); 1128 1129 ctx->sdb_type = ERDMA_SDB_PAGE; 1130 ctx->sdb_idx = bitmap_idx; 1131 ctx->sdb_page_idx = bitmap_idx; 1132 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET + 1133 (bitmap_idx << PAGE_SHIFT); 1134 ctx->sdb_page_off = 0; 1135 1136 return; 1137 } 1138 1139 bitmap_idx = find_first_zero_bit(dev->sdb_entry, attrs->dwqe_entries); 1140 if (bitmap_idx != attrs->dwqe_entries) { 1141 set_bit(bitmap_idx, dev->sdb_entry); 1142 spin_unlock(&dev->db_bitmap_lock); 1143 1144 ctx->sdb_type = ERDMA_SDB_ENTRY; 1145 ctx->sdb_idx = bitmap_idx; 1146 ctx->sdb_page_idx = attrs->dwqe_pages + 1147 bitmap_idx / ERDMA_DWQE_TYPE1_CNT_PER_PAGE; 1148 ctx->sdb_page_off = bitmap_idx % ERDMA_DWQE_TYPE1_CNT_PER_PAGE; 1149 1150 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET + 1151 (ctx->sdb_page_idx << PAGE_SHIFT); 1152 1153 return; 1154 } 1155 1156 spin_unlock(&dev->db_bitmap_lock); 1157 1158 alloc_normal_db: 1159 ctx->sdb_type = ERDMA_SDB_SHARED; 1160 ctx->sdb_idx = 0; 1161 ctx->sdb_page_idx = ERDMA_SDB_SHARED_PAGE_INDEX; 1162 ctx->sdb_page_off = 0; 1163 1164 ctx->sdb = dev->func_bar_addr + (ctx->sdb_page_idx << PAGE_SHIFT); 1165 } 1166 1167 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx) 1168 { 1169 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry); 1170 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry); 1171 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry); 1172 } 1173 1174 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 1175 { 1176 struct erdma_ucontext *ctx = to_ectx(ibctx); 1177 struct erdma_dev *dev = to_edev(ibctx->device); 1178 int ret; 1179 struct erdma_uresp_alloc_ctx uresp = {}; 1180 1181 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) { 1182 ret = -ENOMEM; 1183 goto err_out; 1184 } 1185 1186 INIT_LIST_HEAD(&ctx->dbrecords_page_list); 1187 mutex_init(&ctx->dbrecords_page_mutex); 1188 1189 alloc_db_resources(dev, ctx); 1190 1191 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET; 1192 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET; 1193 1194 if (udata->outlen < sizeof(uresp)) { 1195 ret = -EINVAL; 1196 goto err_out; 1197 } 1198 1199 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert( 1200 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb); 1201 if (!ctx->sq_db_mmap_entry) { 1202 ret = -ENOMEM; 1203 goto err_out; 1204 } 1205 1206 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert( 1207 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb); 1208 if (!ctx->rq_db_mmap_entry) { 1209 ret = -EINVAL; 1210 goto err_out; 1211 } 1212 1213 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert( 1214 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb); 1215 if (!ctx->cq_db_mmap_entry) { 1216 ret = -EINVAL; 1217 goto err_out; 1218 } 1219 1220 uresp.dev_id = dev->pdev->device; 1221 uresp.sdb_type = ctx->sdb_type; 1222 uresp.sdb_offset = ctx->sdb_page_off; 1223 1224 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1225 if (ret) 1226 goto err_out; 1227 1228 return 0; 1229 1230 err_out: 1231 erdma_uctx_user_mmap_entries_remove(ctx); 1232 atomic_dec(&dev->num_ctx); 1233 return ret; 1234 } 1235 1236 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx) 1237 { 1238 struct erdma_ucontext *ctx = to_ectx(ibctx); 1239 struct erdma_dev *dev = to_edev(ibctx->device); 1240 1241 spin_lock(&dev->db_bitmap_lock); 1242 if (ctx->sdb_type == ERDMA_SDB_PAGE) 1243 clear_bit(ctx->sdb_idx, dev->sdb_page); 1244 else if (ctx->sdb_type == ERDMA_SDB_ENTRY) 1245 clear_bit(ctx->sdb_idx, dev->sdb_entry); 1246 1247 erdma_uctx_user_mmap_entries_remove(ctx); 1248 1249 spin_unlock(&dev->db_bitmap_lock); 1250 1251 atomic_dec(&dev->num_ctx); 1252 } 1253 1254 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = { 1255 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE, 1256 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE, 1257 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR, 1258 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS, 1259 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING, 1260 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE, 1261 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR 1262 }; 1263 1264 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1265 struct ib_udata *udata) 1266 { 1267 struct erdma_qp_attrs new_attrs; 1268 enum erdma_qp_attr_mask erdma_attr_mask = 0; 1269 struct erdma_qp *qp = to_eqp(ibqp); 1270 int ret = 0; 1271 1272 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1273 return -EOPNOTSUPP; 1274 1275 memset(&new_attrs, 0, sizeof(new_attrs)); 1276 1277 if (attr_mask & IB_QP_STATE) { 1278 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state]; 1279 1280 erdma_attr_mask |= ERDMA_QP_ATTR_STATE; 1281 } 1282 1283 down_write(&qp->state_lock); 1284 1285 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask); 1286 1287 up_write(&qp->state_lock); 1288 1289 return ret; 1290 } 1291 1292 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 1293 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1294 { 1295 struct erdma_qp *qp; 1296 struct erdma_dev *dev; 1297 1298 if (ibqp && qp_attr && qp_init_attr) { 1299 qp = to_eqp(ibqp); 1300 dev = to_edev(ibqp->device); 1301 } else { 1302 return -EINVAL; 1303 } 1304 1305 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1306 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1307 1308 qp_attr->cap.max_send_wr = qp->attrs.sq_size; 1309 qp_attr->cap.max_recv_wr = qp->attrs.rq_size; 1310 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge; 1311 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge; 1312 1313 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu); 1314 qp_attr->max_rd_atomic = qp->attrs.irq_size; 1315 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; 1316 1317 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | 1318 IB_ACCESS_REMOTE_WRITE | 1319 IB_ACCESS_REMOTE_READ; 1320 1321 qp_init_attr->cap = qp_attr->cap; 1322 1323 return 0; 1324 } 1325 1326 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, 1327 struct erdma_ureq_create_cq *ureq) 1328 { 1329 int ret; 1330 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1331 1332 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va, 1333 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, 1334 1); 1335 if (ret) 1336 return ret; 1337 1338 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, 1339 &cq->user_cq.user_dbr_page, 1340 &cq->user_cq.db_info_dma_addr); 1341 if (ret) 1342 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1343 1344 return ret; 1345 } 1346 1347 static int erdma_init_kernel_cq(struct erdma_cq *cq) 1348 { 1349 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1350 1351 cq->kern_cq.qbuf = 1352 dma_alloc_coherent(&dev->pdev->dev, 1353 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1354 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1355 if (!cq->kern_cq.qbuf) 1356 return -ENOMEM; 1357 1358 cq->kern_cq.db_record = 1359 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); 1360 spin_lock_init(&cq->kern_cq.lock); 1361 /* use default cqdb addr */ 1362 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1363 1364 return 0; 1365 } 1366 1367 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1368 struct ib_udata *udata) 1369 { 1370 struct erdma_cq *cq = to_ecq(ibcq); 1371 struct erdma_dev *dev = to_edev(ibcq->device); 1372 unsigned int depth = attr->cqe; 1373 int ret; 1374 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1375 udata, struct erdma_ucontext, ibucontext); 1376 1377 if (depth > dev->attrs.max_cqe) 1378 return -EINVAL; 1379 1380 depth = roundup_pow_of_two(depth); 1381 cq->ibcq.cqe = depth; 1382 cq->depth = depth; 1383 cq->assoc_eqn = attr->comp_vector + 1; 1384 1385 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, 1386 XA_LIMIT(1, dev->attrs.max_cq - 1), 1387 &dev->next_alloc_cqn, GFP_KERNEL); 1388 if (ret < 0) 1389 return ret; 1390 1391 if (!rdma_is_kernel_res(&ibcq->res)) { 1392 struct erdma_ureq_create_cq ureq; 1393 struct erdma_uresp_create_cq uresp; 1394 1395 ret = ib_copy_from_udata(&ureq, udata, 1396 min(udata->inlen, sizeof(ureq))); 1397 if (ret) 1398 goto err_out_xa; 1399 1400 ret = erdma_init_user_cq(ctx, cq, &ureq); 1401 if (ret) 1402 goto err_out_xa; 1403 1404 uresp.cq_id = cq->cqn; 1405 uresp.num_cqe = depth; 1406 1407 ret = ib_copy_to_udata(udata, &uresp, 1408 min(sizeof(uresp), udata->outlen)); 1409 if (ret) 1410 goto err_free_res; 1411 } else { 1412 ret = erdma_init_kernel_cq(cq); 1413 if (ret) 1414 goto err_out_xa; 1415 } 1416 1417 ret = create_cq_cmd(dev, cq); 1418 if (ret) 1419 goto err_free_res; 1420 1421 return 0; 1422 1423 err_free_res: 1424 if (!rdma_is_kernel_res(&ibcq->res)) { 1425 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1426 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1427 } else { 1428 dma_free_coherent(&dev->pdev->dev, 1429 WARPPED_BUFSIZE(depth << CQE_SHIFT), 1430 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1431 } 1432 1433 err_out_xa: 1434 xa_erase(&dev->cq_xa, cq->cqn); 1435 1436 return ret; 1437 } 1438 1439 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu) 1440 { 1441 struct erdma_cmdq_config_mtu_req req; 1442 1443 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1444 CMDQ_OPCODE_CONF_MTU); 1445 req.mtu = mtu; 1446 1447 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1448 } 1449 1450 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) 1451 { 1452 struct ib_event event; 1453 1454 event.device = &dev->ibdev; 1455 event.element.port_num = 1; 1456 event.event = reason; 1457 1458 ib_dispatch_event(&event); 1459 } 1460