1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Copyright (c) 2008-2019, IBM Corporation */ 9 10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */ 11 12 #include <linux/vmalloc.h> 13 #include <net/addrconf.h> 14 #include <rdma/erdma-abi.h> 15 #include <rdma/ib_umem.h> 16 #include <rdma/uverbs_ioctl.h> 17 18 #include "erdma.h" 19 #include "erdma_cm.h" 20 #include "erdma_verbs.h" 21 22 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp) 23 { 24 struct erdma_dev *dev = to_edev(qp->ibqp.device); 25 struct erdma_pd *pd = to_epd(qp->ibqp.pd); 26 struct erdma_cmdq_create_qp_req req; 27 struct erdma_uqp *user_qp; 28 u64 resp0, resp1; 29 int err; 30 31 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 32 CMDQ_OPCODE_CREATE_QP); 33 34 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK, 35 ilog2(qp->attrs.sq_size)) | 36 FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp)); 37 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK, 38 ilog2(qp->attrs.rq_size)) | 39 FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn); 40 41 if (rdma_is_kernel_res(&qp->ibqp.res)) { 42 u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT; 43 44 req.sq_cqn_mtt_cfg = 45 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 46 pgsz_range) | 47 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 48 req.rq_cqn_mtt_cfg = 49 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 50 pgsz_range) | 51 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 52 53 req.sq_mtt_cfg = 54 FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) | 55 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) | 56 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 57 ERDMA_MR_INLINE_MTT); 58 req.rq_mtt_cfg = req.sq_mtt_cfg; 59 60 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 61 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 62 req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + 63 (qp->attrs.sq_size << SQEBB_SHIFT); 64 req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + 65 (qp->attrs.rq_size << RQE_SHIFT); 66 } else { 67 user_qp = &qp->user_qp; 68 req.sq_cqn_mtt_cfg = FIELD_PREP( 69 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 70 ilog2(user_qp->sq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT); 71 req.sq_cqn_mtt_cfg |= 72 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn); 73 74 req.rq_cqn_mtt_cfg = FIELD_PREP( 75 ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK, 76 ilog2(user_qp->rq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT); 77 req.rq_cqn_mtt_cfg |= 78 FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn); 79 80 req.sq_mtt_cfg = user_qp->sq_mtt.page_offset; 81 req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 82 user_qp->sq_mtt.mtt_nents) | 83 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 84 user_qp->sq_mtt.mtt_type); 85 86 req.rq_mtt_cfg = user_qp->rq_mtt.page_offset; 87 req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 88 user_qp->rq_mtt.mtt_nents) | 89 FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_TYPE_MASK, 90 user_qp->rq_mtt.mtt_type); 91 92 req.sq_buf_addr = user_qp->sq_mtt.mtt_entry[0]; 93 req.rq_buf_addr = user_qp->rq_mtt.mtt_entry[0]; 94 95 req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr; 96 req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr; 97 98 if (uctx->ext_db.enable) { 99 req.sq_cqn_mtt_cfg |= 100 FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1); 101 req.db_cfg = 102 FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK, 103 uctx->ext_db.sdb_off) | 104 FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK, 105 uctx->ext_db.rdb_off); 106 } 107 } 108 109 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0, 110 &resp1); 111 if (!err) 112 qp->attrs.cookie = 113 FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0); 114 115 return err; 116 } 117 118 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr) 119 { 120 struct erdma_cmdq_reg_mr_req req; 121 struct erdma_pd *pd = to_epd(mr->ibmr.pd); 122 u64 *phy_addr; 123 int i; 124 125 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR); 126 127 req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) | 128 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) | 129 FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8); 130 req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) | 131 FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) | 132 FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access); 133 req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK, 134 ilog2(mr->mem.page_size)) | 135 FIELD_PREP(ERDMA_CMD_REGMR_MTT_TYPE_MASK, mr->mem.mtt_type) | 136 FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt); 137 138 if (mr->type == ERDMA_MR_TYPE_DMA) 139 goto post_cmd; 140 141 if (mr->type == ERDMA_MR_TYPE_NORMAL) { 142 req.start_va = mr->mem.va; 143 req.size = mr->mem.len; 144 } 145 146 if (mr->type == ERDMA_MR_TYPE_FRMR || 147 mr->mem.mtt_type == ERDMA_MR_INDIRECT_MTT) { 148 phy_addr = req.phy_addr; 149 *phy_addr = mr->mem.mtt_entry[0]; 150 } else { 151 phy_addr = req.phy_addr; 152 for (i = 0; i < mr->mem.mtt_nents; i++) 153 *phy_addr++ = mr->mem.mtt_entry[i]; 154 } 155 156 post_cmd: 157 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 158 } 159 160 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq) 161 { 162 struct erdma_dev *dev = to_edev(cq->ibcq.device); 163 struct erdma_cmdq_create_cq_req req; 164 struct erdma_mem *mtt; 165 u32 page_size; 166 167 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 168 CMDQ_OPCODE_CREATE_CQ); 169 170 req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) | 171 FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth)); 172 req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn); 173 174 if (rdma_is_kernel_res(&cq->ibcq.res)) { 175 page_size = SZ_32M; 176 req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 177 ilog2(page_size) - ERDMA_HW_PAGE_SHIFT); 178 req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr); 179 req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr); 180 181 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) | 182 FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 183 ERDMA_MR_INLINE_MTT); 184 185 req.first_page_offset = 0; 186 req.cq_db_info_addr = 187 cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); 188 } else { 189 mtt = &cq->user_cq.qbuf_mtt; 190 req.cfg0 |= 191 FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK, 192 ilog2(mtt->page_size) - ERDMA_HW_PAGE_SHIFT); 193 if (mtt->mtt_nents == 1) { 194 req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf); 195 req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf); 196 } else { 197 req.qbuf_addr_l = lower_32_bits(mtt->mtt_entry[0]); 198 req.qbuf_addr_h = upper_32_bits(mtt->mtt_entry[0]); 199 } 200 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 201 mtt->mtt_nents); 202 req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_TYPE_MASK, 203 mtt->mtt_type); 204 205 req.first_page_offset = mtt->page_offset; 206 req.cq_db_info_addr = cq->user_cq.db_info_dma_addr; 207 208 if (uctx->ext_db.enable) { 209 req.cfg1 |= FIELD_PREP( 210 ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1); 211 req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK, 212 uctx->ext_db.cdb_off); 213 } 214 } 215 216 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 217 } 218 219 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb) 220 { 221 int idx; 222 unsigned long flags; 223 224 spin_lock_irqsave(&res_cb->lock, flags); 225 idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap, 226 res_cb->next_alloc_idx); 227 if (idx == res_cb->max_cap) { 228 idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap); 229 if (idx == res_cb->max_cap) { 230 res_cb->next_alloc_idx = 1; 231 spin_unlock_irqrestore(&res_cb->lock, flags); 232 return -ENOSPC; 233 } 234 } 235 236 set_bit(idx, res_cb->bitmap); 237 res_cb->next_alloc_idx = idx + 1; 238 spin_unlock_irqrestore(&res_cb->lock, flags); 239 240 return idx; 241 } 242 243 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx) 244 { 245 unsigned long flags; 246 u32 used; 247 248 spin_lock_irqsave(&res_cb->lock, flags); 249 used = __test_and_clear_bit(idx, res_cb->bitmap); 250 spin_unlock_irqrestore(&res_cb->lock, flags); 251 WARN_ON(!used); 252 } 253 254 static struct rdma_user_mmap_entry * 255 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address, 256 u32 size, u8 mmap_flag, u64 *mmap_offset) 257 { 258 struct erdma_user_mmap_entry *entry = 259 kzalloc(sizeof(*entry), GFP_KERNEL); 260 int ret; 261 262 if (!entry) 263 return NULL; 264 265 entry->address = (u64)address; 266 entry->mmap_flag = mmap_flag; 267 268 size = PAGE_ALIGN(size); 269 270 ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry, 271 size); 272 if (ret) { 273 kfree(entry); 274 return NULL; 275 } 276 277 *mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry); 278 279 return &entry->rdma_entry; 280 } 281 282 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, 283 struct ib_udata *unused) 284 { 285 struct erdma_dev *dev = to_edev(ibdev); 286 287 memset(attr, 0, sizeof(*attr)); 288 289 attr->max_mr_size = dev->attrs.max_mr_size; 290 attr->vendor_id = PCI_VENDOR_ID_ALIBABA; 291 attr->vendor_part_id = dev->pdev->device; 292 attr->hw_ver = dev->pdev->revision; 293 attr->max_qp = dev->attrs.max_qp - 1; 294 attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr); 295 attr->max_qp_rd_atom = dev->attrs.max_ord; 296 attr->max_qp_init_rd_atom = dev->attrs.max_ird; 297 attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird; 298 attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS; 299 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY; 300 ibdev->local_dma_lkey = dev->attrs.local_dma_key; 301 attr->max_send_sge = dev->attrs.max_send_sge; 302 attr->max_recv_sge = dev->attrs.max_recv_sge; 303 attr->max_sge_rd = dev->attrs.max_sge_rd; 304 attr->max_cq = dev->attrs.max_cq - 1; 305 attr->max_cqe = dev->attrs.max_cqe; 306 attr->max_mr = dev->attrs.max_mr; 307 attr->max_pd = dev->attrs.max_pd; 308 attr->max_mw = dev->attrs.max_mw; 309 attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA; 310 attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT; 311 312 if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC) 313 attr->atomic_cap = IB_ATOMIC_GLOB; 314 315 attr->fw_ver = dev->attrs.fw_version; 316 317 if (dev->netdev) 318 addrconf_addr_eui48((u8 *)&attr->sys_image_guid, 319 dev->netdev->dev_addr); 320 321 return 0; 322 } 323 324 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx, 325 union ib_gid *gid) 326 { 327 struct erdma_dev *dev = to_edev(ibdev); 328 329 memset(gid, 0, sizeof(*gid)); 330 ether_addr_copy(gid->raw, dev->attrs.peer_addr); 331 332 return 0; 333 } 334 335 int erdma_query_port(struct ib_device *ibdev, u32 port, 336 struct ib_port_attr *attr) 337 { 338 struct erdma_dev *dev = to_edev(ibdev); 339 struct net_device *ndev = dev->netdev; 340 341 memset(attr, 0, sizeof(*attr)); 342 343 attr->gid_tbl_len = 1; 344 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP; 345 attr->max_msg_sz = -1; 346 347 if (!ndev) 348 goto out; 349 350 ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width); 351 attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu); 352 attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu); 353 if (netif_running(ndev) && netif_carrier_ok(ndev)) 354 dev->state = IB_PORT_ACTIVE; 355 else 356 dev->state = IB_PORT_DOWN; 357 attr->state = dev->state; 358 359 out: 360 if (dev->state == IB_PORT_ACTIVE) 361 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 362 else 363 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 364 365 return 0; 366 } 367 368 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port, 369 struct ib_port_immutable *port_immutable) 370 { 371 port_immutable->gid_tbl_len = 1; 372 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; 373 374 return 0; 375 } 376 377 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 378 { 379 struct erdma_pd *pd = to_epd(ibpd); 380 struct erdma_dev *dev = to_edev(ibpd->device); 381 int pdn; 382 383 pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]); 384 if (pdn < 0) 385 return pdn; 386 387 pd->pdn = pdn; 388 389 return 0; 390 } 391 392 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 393 { 394 struct erdma_pd *pd = to_epd(ibpd); 395 struct erdma_dev *dev = to_edev(ibpd->device); 396 397 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn); 398 399 return 0; 400 } 401 402 static void erdma_flush_worker(struct work_struct *work) 403 { 404 struct delayed_work *dwork = to_delayed_work(work); 405 struct erdma_qp *qp = 406 container_of(dwork, struct erdma_qp, reflush_dwork); 407 struct erdma_cmdq_reflush_req req; 408 409 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 410 CMDQ_OPCODE_REFLUSH); 411 req.qpn = QP_ID(qp); 412 req.sq_pi = qp->kern_qp.sq_pi; 413 req.rq_pi = qp->kern_qp.rq_pi; 414 erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL); 415 } 416 417 static int erdma_qp_validate_cap(struct erdma_dev *dev, 418 struct ib_qp_init_attr *attrs) 419 { 420 if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) || 421 (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) || 422 (attrs->cap.max_send_sge > dev->attrs.max_send_sge) || 423 (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) || 424 (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) || 425 !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) { 426 return -EINVAL; 427 } 428 429 return 0; 430 } 431 432 static int erdma_qp_validate_attr(struct erdma_dev *dev, 433 struct ib_qp_init_attr *attrs) 434 { 435 if (attrs->qp_type != IB_QPT_RC) 436 return -EOPNOTSUPP; 437 438 if (attrs->srq) 439 return -EOPNOTSUPP; 440 441 if (!attrs->send_cq || !attrs->recv_cq) 442 return -EOPNOTSUPP; 443 444 return 0; 445 } 446 447 static void free_kernel_qp(struct erdma_qp *qp) 448 { 449 struct erdma_dev *dev = qp->dev; 450 451 vfree(qp->kern_qp.swr_tbl); 452 vfree(qp->kern_qp.rwr_tbl); 453 454 if (qp->kern_qp.sq_buf) 455 dma_free_coherent( 456 &dev->pdev->dev, 457 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 458 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 459 460 if (qp->kern_qp.rq_buf) 461 dma_free_coherent( 462 &dev->pdev->dev, 463 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 464 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 465 } 466 467 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, 468 struct ib_qp_init_attr *attrs) 469 { 470 struct erdma_kqp *kqp = &qp->kern_qp; 471 int size; 472 473 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) 474 kqp->sig_all = 1; 475 476 kqp->sq_pi = 0; 477 kqp->sq_ci = 0; 478 kqp->rq_pi = 0; 479 kqp->rq_ci = 0; 480 kqp->hw_sq_db = 481 dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT); 482 kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET; 483 484 kqp->swr_tbl = vmalloc(qp->attrs.sq_size * sizeof(u64)); 485 kqp->rwr_tbl = vmalloc(qp->attrs.rq_size * sizeof(u64)); 486 if (!kqp->swr_tbl || !kqp->rwr_tbl) 487 goto err_out; 488 489 size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 490 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 491 &kqp->sq_buf_dma_addr, GFP_KERNEL); 492 if (!kqp->sq_buf) 493 goto err_out; 494 495 size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 496 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 497 &kqp->rq_buf_dma_addr, GFP_KERNEL); 498 if (!kqp->rq_buf) 499 goto err_out; 500 501 kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); 502 kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); 503 504 return 0; 505 506 err_out: 507 free_kernel_qp(qp); 508 return -ENOMEM; 509 } 510 511 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem, 512 u64 start, u64 len, int access, u64 virt, 513 unsigned long req_page_size, u8 force_indirect_mtt) 514 { 515 struct ib_block_iter biter; 516 uint64_t *phy_addr = NULL; 517 int ret = 0; 518 519 mem->umem = ib_umem_get(&dev->ibdev, start, len, access); 520 if (IS_ERR(mem->umem)) { 521 ret = PTR_ERR(mem->umem); 522 mem->umem = NULL; 523 return ret; 524 } 525 526 mem->va = virt; 527 mem->len = len; 528 mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt); 529 mem->page_offset = start & (mem->page_size - 1); 530 mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size); 531 mem->page_cnt = mem->mtt_nents; 532 533 if (mem->page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES || 534 force_indirect_mtt) { 535 mem->mtt_type = ERDMA_MR_INDIRECT_MTT; 536 mem->mtt_buf = 537 alloc_pages_exact(MTT_SIZE(mem->page_cnt), GFP_KERNEL); 538 if (!mem->mtt_buf) { 539 ret = -ENOMEM; 540 goto error_ret; 541 } 542 phy_addr = mem->mtt_buf; 543 } else { 544 mem->mtt_type = ERDMA_MR_INLINE_MTT; 545 phy_addr = mem->mtt_entry; 546 } 547 548 rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size) { 549 *phy_addr = rdma_block_iter_dma_address(&biter); 550 phy_addr++; 551 } 552 553 if (mem->mtt_type == ERDMA_MR_INDIRECT_MTT) { 554 mem->mtt_entry[0] = 555 dma_map_single(&dev->pdev->dev, mem->mtt_buf, 556 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 557 if (dma_mapping_error(&dev->pdev->dev, mem->mtt_entry[0])) { 558 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 559 mem->mtt_buf = NULL; 560 ret = -ENOMEM; 561 goto error_ret; 562 } 563 } 564 565 return 0; 566 567 error_ret: 568 if (mem->umem) { 569 ib_umem_release(mem->umem); 570 mem->umem = NULL; 571 } 572 573 return ret; 574 } 575 576 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem) 577 { 578 if (mem->mtt_buf) { 579 dma_unmap_single(&dev->pdev->dev, mem->mtt_entry[0], 580 MTT_SIZE(mem->page_cnt), DMA_TO_DEVICE); 581 free_pages_exact(mem->mtt_buf, MTT_SIZE(mem->page_cnt)); 582 } 583 584 if (mem->umem) { 585 ib_umem_release(mem->umem); 586 mem->umem = NULL; 587 } 588 } 589 590 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx, 591 u64 dbrecords_va, 592 struct erdma_user_dbrecords_page **dbr_page, 593 dma_addr_t *dma_addr) 594 { 595 struct erdma_user_dbrecords_page *page = NULL; 596 int rv = 0; 597 598 mutex_lock(&ctx->dbrecords_page_mutex); 599 600 list_for_each_entry(page, &ctx->dbrecords_page_list, list) 601 if (page->va == (dbrecords_va & PAGE_MASK)) 602 goto found; 603 604 page = kmalloc(sizeof(*page), GFP_KERNEL); 605 if (!page) { 606 rv = -ENOMEM; 607 goto out; 608 } 609 610 page->va = (dbrecords_va & PAGE_MASK); 611 page->refcnt = 0; 612 613 page->umem = ib_umem_get(ctx->ibucontext.device, 614 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0); 615 if (IS_ERR(page->umem)) { 616 rv = PTR_ERR(page->umem); 617 kfree(page); 618 goto out; 619 } 620 621 list_add(&page->list, &ctx->dbrecords_page_list); 622 623 found: 624 *dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) + 625 (dbrecords_va & ~PAGE_MASK); 626 *dbr_page = page; 627 page->refcnt++; 628 629 out: 630 mutex_unlock(&ctx->dbrecords_page_mutex); 631 return rv; 632 } 633 634 static void 635 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx, 636 struct erdma_user_dbrecords_page **dbr_page) 637 { 638 if (!ctx || !(*dbr_page)) 639 return; 640 641 mutex_lock(&ctx->dbrecords_page_mutex); 642 if (--(*dbr_page)->refcnt == 0) { 643 list_del(&(*dbr_page)->list); 644 ib_umem_release((*dbr_page)->umem); 645 kfree(*dbr_page); 646 } 647 648 *dbr_page = NULL; 649 mutex_unlock(&ctx->dbrecords_page_mutex); 650 } 651 652 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx, 653 u64 va, u32 len, u64 db_info_va) 654 { 655 dma_addr_t db_info_dma_addr; 656 u32 rq_offset; 657 int ret; 658 659 if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) + 660 qp->attrs.rq_size * RQE_SIZE)) 661 return -EINVAL; 662 663 ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mtt, va, 664 qp->attrs.sq_size << SQEBB_SHIFT, 0, va, 665 (SZ_1M - SZ_4K), 1); 666 if (ret) 667 return ret; 668 669 rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE); 670 qp->user_qp.rq_offset = rq_offset; 671 672 ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset, 673 qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset, 674 (SZ_1M - SZ_4K), 1); 675 if (ret) 676 goto put_sq_mtt; 677 678 ret = erdma_map_user_dbrecords(uctx, db_info_va, 679 &qp->user_qp.user_dbr_page, 680 &db_info_dma_addr); 681 if (ret) 682 goto put_rq_mtt; 683 684 qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr; 685 qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE; 686 687 return 0; 688 689 put_rq_mtt: 690 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 691 692 put_sq_mtt: 693 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 694 695 return ret; 696 } 697 698 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx) 699 { 700 put_mtt_entries(qp->dev, &qp->user_qp.sq_mtt); 701 put_mtt_entries(qp->dev, &qp->user_qp.rq_mtt); 702 erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page); 703 } 704 705 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, 706 struct ib_udata *udata) 707 { 708 struct erdma_qp *qp = to_eqp(ibqp); 709 struct erdma_dev *dev = to_edev(ibqp->device); 710 struct erdma_ucontext *uctx = rdma_udata_to_drv_context( 711 udata, struct erdma_ucontext, ibucontext); 712 struct erdma_ureq_create_qp ureq; 713 struct erdma_uresp_create_qp uresp; 714 int ret; 715 716 ret = erdma_qp_validate_cap(dev, attrs); 717 if (ret) 718 goto err_out; 719 720 ret = erdma_qp_validate_attr(dev, attrs); 721 if (ret) 722 goto err_out; 723 724 qp->scq = to_ecq(attrs->send_cq); 725 qp->rcq = to_ecq(attrs->recv_cq); 726 qp->dev = dev; 727 qp->attrs.cc = dev->attrs.cc; 728 729 init_rwsem(&qp->state_lock); 730 kref_init(&qp->ref); 731 init_completion(&qp->safe_free); 732 733 ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp, 734 XA_LIMIT(1, dev->attrs.max_qp - 1), 735 &dev->next_alloc_qpn, GFP_KERNEL); 736 if (ret < 0) { 737 ret = -ENOMEM; 738 goto err_out; 739 } 740 741 qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr * 742 ERDMA_MAX_WQEBB_PER_SQE); 743 qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr); 744 745 if (uctx) { 746 ret = ib_copy_from_udata(&ureq, udata, 747 min(sizeof(ureq), udata->inlen)); 748 if (ret) 749 goto err_out_xa; 750 751 ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len, 752 ureq.db_record_va); 753 if (ret) 754 goto err_out_xa; 755 756 memset(&uresp, 0, sizeof(uresp)); 757 758 uresp.num_sqe = qp->attrs.sq_size; 759 uresp.num_rqe = qp->attrs.rq_size; 760 uresp.qp_id = QP_ID(qp); 761 uresp.rq_offset = qp->user_qp.rq_offset; 762 763 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 764 if (ret) 765 goto err_out_cmd; 766 } else { 767 init_kernel_qp(dev, qp, attrs); 768 } 769 770 qp->attrs.max_send_sge = attrs->cap.max_send_sge; 771 qp->attrs.max_recv_sge = attrs->cap.max_recv_sge; 772 qp->attrs.state = ERDMA_QP_STATE_IDLE; 773 INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker); 774 775 ret = create_qp_cmd(uctx, qp); 776 if (ret) 777 goto err_out_cmd; 778 779 spin_lock_init(&qp->lock); 780 781 return 0; 782 783 err_out_cmd: 784 if (uctx) 785 free_user_qp(qp, uctx); 786 else 787 free_kernel_qp(qp); 788 err_out_xa: 789 xa_erase(&dev->qp_xa, QP_ID(qp)); 790 err_out: 791 return ret; 792 } 793 794 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag) 795 { 796 int stag_idx; 797 798 stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]); 799 if (stag_idx < 0) 800 return stag_idx; 801 802 /* For now, we always let key field be zero. */ 803 *stag = (stag_idx << 8); 804 805 return 0; 806 } 807 808 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc) 809 { 810 struct erdma_dev *dev = to_edev(ibpd->device); 811 struct erdma_mr *mr; 812 u32 stag; 813 int ret; 814 815 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 816 if (!mr) 817 return ERR_PTR(-ENOMEM); 818 819 ret = erdma_create_stag(dev, &stag); 820 if (ret) 821 goto out_free; 822 823 mr->type = ERDMA_MR_TYPE_DMA; 824 825 mr->ibmr.lkey = stag; 826 mr->ibmr.rkey = stag; 827 mr->ibmr.pd = ibpd; 828 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc); 829 ret = regmr_cmd(dev, mr); 830 if (ret) 831 goto out_remove_stag; 832 833 return &mr->ibmr; 834 835 out_remove_stag: 836 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 837 mr->ibmr.lkey >> 8); 838 839 out_free: 840 kfree(mr); 841 842 return ERR_PTR(ret); 843 } 844 845 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 846 u32 max_num_sg) 847 { 848 struct erdma_mr *mr; 849 struct erdma_dev *dev = to_edev(ibpd->device); 850 int ret; 851 u32 stag; 852 853 if (mr_type != IB_MR_TYPE_MEM_REG) 854 return ERR_PTR(-EOPNOTSUPP); 855 856 if (max_num_sg > ERDMA_MR_MAX_MTT_CNT) 857 return ERR_PTR(-EINVAL); 858 859 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 860 if (!mr) 861 return ERR_PTR(-ENOMEM); 862 863 ret = erdma_create_stag(dev, &stag); 864 if (ret) 865 goto out_free; 866 867 mr->type = ERDMA_MR_TYPE_FRMR; 868 869 mr->ibmr.lkey = stag; 870 mr->ibmr.rkey = stag; 871 mr->ibmr.pd = ibpd; 872 /* update it in FRMR. */ 873 mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR | 874 ERDMA_MR_ACC_RW; 875 876 mr->mem.page_size = PAGE_SIZE; /* update it later. */ 877 mr->mem.page_cnt = max_num_sg; 878 mr->mem.mtt_type = ERDMA_MR_INDIRECT_MTT; 879 mr->mem.mtt_buf = 880 alloc_pages_exact(MTT_SIZE(mr->mem.page_cnt), GFP_KERNEL); 881 if (!mr->mem.mtt_buf) { 882 ret = -ENOMEM; 883 goto out_remove_stag; 884 } 885 886 mr->mem.mtt_entry[0] = 887 dma_map_single(&dev->pdev->dev, mr->mem.mtt_buf, 888 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 889 if (dma_mapping_error(&dev->pdev->dev, mr->mem.mtt_entry[0])) { 890 ret = -ENOMEM; 891 goto out_free_mtt; 892 } 893 894 ret = regmr_cmd(dev, mr); 895 if (ret) 896 goto out_dma_unmap; 897 898 return &mr->ibmr; 899 900 out_dma_unmap: 901 dma_unmap_single(&dev->pdev->dev, mr->mem.mtt_entry[0], 902 MTT_SIZE(mr->mem.page_cnt), DMA_TO_DEVICE); 903 out_free_mtt: 904 free_pages_exact(mr->mem.mtt_buf, MTT_SIZE(mr->mem.page_cnt)); 905 906 out_remove_stag: 907 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 908 mr->ibmr.lkey >> 8); 909 910 out_free: 911 kfree(mr); 912 913 return ERR_PTR(ret); 914 } 915 916 static int erdma_set_page(struct ib_mr *ibmr, u64 addr) 917 { 918 struct erdma_mr *mr = to_emr(ibmr); 919 920 if (mr->mem.mtt_nents >= mr->mem.page_cnt) 921 return -1; 922 923 *((u64 *)mr->mem.mtt_buf + mr->mem.mtt_nents) = addr; 924 mr->mem.mtt_nents++; 925 926 return 0; 927 } 928 929 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 930 unsigned int *sg_offset) 931 { 932 struct erdma_mr *mr = to_emr(ibmr); 933 int num; 934 935 mr->mem.mtt_nents = 0; 936 937 num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset, 938 erdma_set_page); 939 940 return num; 941 } 942 943 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, 944 u64 virt, int access, struct ib_udata *udata) 945 { 946 struct erdma_mr *mr = NULL; 947 struct erdma_dev *dev = to_edev(ibpd->device); 948 u32 stag; 949 int ret; 950 951 if (!len || len > dev->attrs.max_mr_size) 952 return ERR_PTR(-EINVAL); 953 954 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 955 if (!mr) 956 return ERR_PTR(-ENOMEM); 957 958 ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt, 959 SZ_2G - SZ_4K, 0); 960 if (ret) 961 goto err_out_free; 962 963 ret = erdma_create_stag(dev, &stag); 964 if (ret) 965 goto err_out_put_mtt; 966 967 mr->ibmr.lkey = mr->ibmr.rkey = stag; 968 mr->ibmr.pd = ibpd; 969 mr->mem.va = virt; 970 mr->mem.len = len; 971 mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access); 972 mr->valid = 1; 973 mr->type = ERDMA_MR_TYPE_NORMAL; 974 975 ret = regmr_cmd(dev, mr); 976 if (ret) 977 goto err_out_mr; 978 979 return &mr->ibmr; 980 981 err_out_mr: 982 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], 983 mr->ibmr.lkey >> 8); 984 985 err_out_put_mtt: 986 put_mtt_entries(dev, &mr->mem); 987 988 err_out_free: 989 kfree(mr); 990 991 return ERR_PTR(ret); 992 } 993 994 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 995 { 996 struct erdma_mr *mr; 997 struct erdma_dev *dev = to_edev(ibmr->device); 998 struct erdma_cmdq_dereg_mr_req req; 999 int ret; 1000 1001 mr = to_emr(ibmr); 1002 1003 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1004 CMDQ_OPCODE_DEREG_MR); 1005 1006 req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) | 1007 FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF); 1008 1009 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1010 if (ret) 1011 return ret; 1012 1013 erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); 1014 1015 put_mtt_entries(dev, &mr->mem); 1016 1017 kfree(mr); 1018 return 0; 1019 } 1020 1021 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1022 { 1023 struct erdma_cq *cq = to_ecq(ibcq); 1024 struct erdma_dev *dev = to_edev(ibcq->device); 1025 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1026 udata, struct erdma_ucontext, ibucontext); 1027 int err; 1028 struct erdma_cmdq_destroy_cq_req req; 1029 1030 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1031 CMDQ_OPCODE_DESTROY_CQ); 1032 req.cqn = cq->cqn; 1033 1034 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1035 if (err) 1036 return err; 1037 1038 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1039 dma_free_coherent(&dev->pdev->dev, 1040 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1041 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1042 } else { 1043 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1044 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1045 } 1046 1047 xa_erase(&dev->cq_xa, cq->cqn); 1048 1049 return 0; 1050 } 1051 1052 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 1053 { 1054 struct erdma_qp *qp = to_eqp(ibqp); 1055 struct erdma_dev *dev = to_edev(ibqp->device); 1056 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1057 udata, struct erdma_ucontext, ibucontext); 1058 struct erdma_qp_attrs qp_attrs; 1059 int err; 1060 struct erdma_cmdq_destroy_qp_req req; 1061 1062 down_write(&qp->state_lock); 1063 qp_attrs.state = ERDMA_QP_STATE_ERROR; 1064 erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE); 1065 up_write(&qp->state_lock); 1066 1067 cancel_delayed_work_sync(&qp->reflush_dwork); 1068 1069 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, 1070 CMDQ_OPCODE_DESTROY_QP); 1071 req.qpn = QP_ID(qp); 1072 1073 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1074 if (err) 1075 return err; 1076 1077 erdma_qp_put(qp); 1078 wait_for_completion(&qp->safe_free); 1079 1080 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1081 vfree(qp->kern_qp.swr_tbl); 1082 vfree(qp->kern_qp.rwr_tbl); 1083 dma_free_coherent( 1084 &dev->pdev->dev, 1085 WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 1086 qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 1087 dma_free_coherent( 1088 &dev->pdev->dev, 1089 WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 1090 qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 1091 } else { 1092 put_mtt_entries(dev, &qp->user_qp.sq_mtt); 1093 put_mtt_entries(dev, &qp->user_qp.rq_mtt); 1094 erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page); 1095 } 1096 1097 if (qp->cep) 1098 erdma_cep_put(qp->cep); 1099 xa_erase(&dev->qp_xa, QP_ID(qp)); 1100 1101 return 0; 1102 } 1103 1104 void erdma_qp_get_ref(struct ib_qp *ibqp) 1105 { 1106 erdma_qp_get(to_eqp(ibqp)); 1107 } 1108 1109 void erdma_qp_put_ref(struct ib_qp *ibqp) 1110 { 1111 erdma_qp_put(to_eqp(ibqp)); 1112 } 1113 1114 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma) 1115 { 1116 struct rdma_user_mmap_entry *rdma_entry; 1117 struct erdma_user_mmap_entry *entry; 1118 pgprot_t prot; 1119 int err; 1120 1121 rdma_entry = rdma_user_mmap_entry_get(ctx, vma); 1122 if (!rdma_entry) 1123 return -EINVAL; 1124 1125 entry = to_emmap(rdma_entry); 1126 1127 switch (entry->mmap_flag) { 1128 case ERDMA_MMAP_IO_NC: 1129 /* map doorbell. */ 1130 prot = pgprot_device(vma->vm_page_prot); 1131 break; 1132 default: 1133 err = -EINVAL; 1134 goto put_entry; 1135 } 1136 1137 err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE, 1138 prot, rdma_entry); 1139 1140 put_entry: 1141 rdma_user_mmap_entry_put(rdma_entry); 1142 return err; 1143 } 1144 1145 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry) 1146 { 1147 struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry); 1148 1149 kfree(entry); 1150 } 1151 1152 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx, 1153 bool ext_db_en) 1154 { 1155 struct erdma_cmdq_ext_db_req req = {}; 1156 u64 val0, val1; 1157 int ret; 1158 1159 /* 1160 * CAP_SYS_RAWIO is required if hardware does not support extend 1161 * doorbell mechanism. 1162 */ 1163 if (!ext_db_en && !capable(CAP_SYS_RAWIO)) 1164 return -EPERM; 1165 1166 if (!ext_db_en) { 1167 ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET; 1168 ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET; 1169 ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET; 1170 return 0; 1171 } 1172 1173 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1174 CMDQ_OPCODE_ALLOC_DB); 1175 1176 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1177 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1178 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1179 1180 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1); 1181 if (ret) 1182 return ret; 1183 1184 ctx->ext_db.enable = true; 1185 ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB); 1186 ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB); 1187 ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB); 1188 1189 ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT); 1190 ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT); 1191 ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT); 1192 1193 return 0; 1194 } 1195 1196 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx) 1197 { 1198 struct erdma_cmdq_ext_db_req req = {}; 1199 int ret; 1200 1201 if (!ctx->ext_db.enable) 1202 return; 1203 1204 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1205 CMDQ_OPCODE_FREE_DB); 1206 1207 req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) | 1208 FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) | 1209 FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1); 1210 1211 req.sdb_off = ctx->ext_db.sdb_off; 1212 req.rdb_off = ctx->ext_db.rdb_off; 1213 req.cdb_off = ctx->ext_db.cdb_off; 1214 1215 ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1216 if (ret) 1217 ibdev_err_ratelimited(&dev->ibdev, 1218 "free db resources failed %d", ret); 1219 } 1220 1221 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx) 1222 { 1223 rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry); 1224 rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry); 1225 rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry); 1226 } 1227 1228 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata) 1229 { 1230 struct erdma_ucontext *ctx = to_ectx(ibctx); 1231 struct erdma_dev *dev = to_edev(ibctx->device); 1232 int ret; 1233 struct erdma_uresp_alloc_ctx uresp = {}; 1234 1235 if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) { 1236 ret = -ENOMEM; 1237 goto err_out; 1238 } 1239 1240 if (udata->outlen < sizeof(uresp)) { 1241 ret = -EINVAL; 1242 goto err_out; 1243 } 1244 1245 INIT_LIST_HEAD(&ctx->dbrecords_page_list); 1246 mutex_init(&ctx->dbrecords_page_mutex); 1247 1248 ret = alloc_db_resources(dev, ctx, 1249 !!(dev->attrs.cap_flags & 1250 ERDMA_DEV_CAP_FLAGS_EXTEND_DB)); 1251 if (ret) 1252 goto err_out; 1253 1254 ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert( 1255 ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb); 1256 if (!ctx->sq_db_mmap_entry) { 1257 ret = -ENOMEM; 1258 goto err_free_ext_db; 1259 } 1260 1261 ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert( 1262 ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb); 1263 if (!ctx->rq_db_mmap_entry) { 1264 ret = -EINVAL; 1265 goto err_put_mmap_entries; 1266 } 1267 1268 ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert( 1269 ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb); 1270 if (!ctx->cq_db_mmap_entry) { 1271 ret = -EINVAL; 1272 goto err_put_mmap_entries; 1273 } 1274 1275 uresp.dev_id = dev->pdev->device; 1276 1277 ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); 1278 if (ret) 1279 goto err_put_mmap_entries; 1280 1281 return 0; 1282 1283 err_put_mmap_entries: 1284 erdma_uctx_user_mmap_entries_remove(ctx); 1285 1286 err_free_ext_db: 1287 free_db_resources(dev, ctx); 1288 1289 err_out: 1290 atomic_dec(&dev->num_ctx); 1291 return ret; 1292 } 1293 1294 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx) 1295 { 1296 struct erdma_dev *dev = to_edev(ibctx->device); 1297 struct erdma_ucontext *ctx = to_ectx(ibctx); 1298 1299 erdma_uctx_user_mmap_entries_remove(ctx); 1300 free_db_resources(dev, ctx); 1301 atomic_dec(&dev->num_ctx); 1302 } 1303 1304 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = { 1305 [IB_QPS_RESET] = ERDMA_QP_STATE_IDLE, 1306 [IB_QPS_INIT] = ERDMA_QP_STATE_IDLE, 1307 [IB_QPS_RTR] = ERDMA_QP_STATE_RTR, 1308 [IB_QPS_RTS] = ERDMA_QP_STATE_RTS, 1309 [IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING, 1310 [IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE, 1311 [IB_QPS_ERR] = ERDMA_QP_STATE_ERROR 1312 }; 1313 1314 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, 1315 struct ib_udata *udata) 1316 { 1317 struct erdma_qp_attrs new_attrs; 1318 enum erdma_qp_attr_mask erdma_attr_mask = 0; 1319 struct erdma_qp *qp = to_eqp(ibqp); 1320 int ret = 0; 1321 1322 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS) 1323 return -EOPNOTSUPP; 1324 1325 memset(&new_attrs, 0, sizeof(new_attrs)); 1326 1327 if (attr_mask & IB_QP_STATE) { 1328 new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state]; 1329 1330 erdma_attr_mask |= ERDMA_QP_ATTR_STATE; 1331 } 1332 1333 down_write(&qp->state_lock); 1334 1335 ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask); 1336 1337 up_write(&qp->state_lock); 1338 1339 return ret; 1340 } 1341 1342 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, 1343 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) 1344 { 1345 struct erdma_qp *qp; 1346 struct erdma_dev *dev; 1347 1348 if (ibqp && qp_attr && qp_init_attr) { 1349 qp = to_eqp(ibqp); 1350 dev = to_edev(ibqp->device); 1351 } else { 1352 return -EINVAL; 1353 } 1354 1355 qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1356 qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE; 1357 1358 qp_attr->cap.max_send_wr = qp->attrs.sq_size; 1359 qp_attr->cap.max_recv_wr = qp->attrs.rq_size; 1360 qp_attr->cap.max_send_sge = qp->attrs.max_send_sge; 1361 qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge; 1362 1363 qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu); 1364 qp_attr->max_rd_atomic = qp->attrs.irq_size; 1365 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size; 1366 1367 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | 1368 IB_ACCESS_REMOTE_WRITE | 1369 IB_ACCESS_REMOTE_READ; 1370 1371 qp_init_attr->cap = qp_attr->cap; 1372 1373 return 0; 1374 } 1375 1376 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq, 1377 struct erdma_ureq_create_cq *ureq) 1378 { 1379 int ret; 1380 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1381 1382 ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mtt, ureq->qbuf_va, 1383 ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K, 1384 1); 1385 if (ret) 1386 return ret; 1387 1388 ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va, 1389 &cq->user_cq.user_dbr_page, 1390 &cq->user_cq.db_info_dma_addr); 1391 if (ret) 1392 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1393 1394 return ret; 1395 } 1396 1397 static int erdma_init_kernel_cq(struct erdma_cq *cq) 1398 { 1399 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1400 1401 cq->kern_cq.qbuf = 1402 dma_alloc_coherent(&dev->pdev->dev, 1403 WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1404 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1405 if (!cq->kern_cq.qbuf) 1406 return -ENOMEM; 1407 1408 cq->kern_cq.db_record = 1409 (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); 1410 spin_lock_init(&cq->kern_cq.lock); 1411 /* use default cqdb addr */ 1412 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1413 1414 return 0; 1415 } 1416 1417 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1418 struct ib_udata *udata) 1419 { 1420 struct erdma_cq *cq = to_ecq(ibcq); 1421 struct erdma_dev *dev = to_edev(ibcq->device); 1422 unsigned int depth = attr->cqe; 1423 int ret; 1424 struct erdma_ucontext *ctx = rdma_udata_to_drv_context( 1425 udata, struct erdma_ucontext, ibucontext); 1426 1427 if (depth > dev->attrs.max_cqe) 1428 return -EINVAL; 1429 1430 depth = roundup_pow_of_two(depth); 1431 cq->ibcq.cqe = depth; 1432 cq->depth = depth; 1433 cq->assoc_eqn = attr->comp_vector + 1; 1434 1435 ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, 1436 XA_LIMIT(1, dev->attrs.max_cq - 1), 1437 &dev->next_alloc_cqn, GFP_KERNEL); 1438 if (ret < 0) 1439 return ret; 1440 1441 if (!rdma_is_kernel_res(&ibcq->res)) { 1442 struct erdma_ureq_create_cq ureq; 1443 struct erdma_uresp_create_cq uresp; 1444 1445 ret = ib_copy_from_udata(&ureq, udata, 1446 min(udata->inlen, sizeof(ureq))); 1447 if (ret) 1448 goto err_out_xa; 1449 1450 ret = erdma_init_user_cq(ctx, cq, &ureq); 1451 if (ret) 1452 goto err_out_xa; 1453 1454 uresp.cq_id = cq->cqn; 1455 uresp.num_cqe = depth; 1456 1457 ret = ib_copy_to_udata(udata, &uresp, 1458 min(sizeof(uresp), udata->outlen)); 1459 if (ret) 1460 goto err_free_res; 1461 } else { 1462 ret = erdma_init_kernel_cq(cq); 1463 if (ret) 1464 goto err_out_xa; 1465 } 1466 1467 ret = create_cq_cmd(ctx, cq); 1468 if (ret) 1469 goto err_free_res; 1470 1471 return 0; 1472 1473 err_free_res: 1474 if (!rdma_is_kernel_res(&ibcq->res)) { 1475 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1476 put_mtt_entries(dev, &cq->user_cq.qbuf_mtt); 1477 } else { 1478 dma_free_coherent(&dev->pdev->dev, 1479 WARPPED_BUFSIZE(depth << CQE_SHIFT), 1480 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1481 } 1482 1483 err_out_xa: 1484 xa_erase(&dev->cq_xa, cq->cqn); 1485 1486 return ret; 1487 } 1488 1489 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu) 1490 { 1491 struct erdma_cmdq_config_mtu_req req; 1492 1493 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 1494 CMDQ_OPCODE_CONF_MTU); 1495 req.mtu = mtu; 1496 1497 erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 1498 } 1499 1500 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason) 1501 { 1502 struct ib_event event; 1503 1504 event.device = &dev->ibdev; 1505 event.element.port_num = 1; 1506 event.event = reason; 1507 1508 ib_dispatch_event(&event); 1509 } 1510