1 // SPDX-License-Identifier: GPL-2.0
2 
3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4 /*          Kai Shen <kaishen@linux.alibaba.com> */
5 /* Copyright (c) 2020-2022, Alibaba Group. */
6 
7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
8 /* Copyright (c) 2008-2019, IBM Corporation */
9 
10 /* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. */
11 
12 #include <linux/vmalloc.h>
13 #include <net/addrconf.h>
14 #include <rdma/erdma-abi.h>
15 #include <rdma/ib_umem.h>
16 #include <rdma/uverbs_ioctl.h>
17 
18 #include "erdma.h"
19 #include "erdma_cm.h"
20 #include "erdma_verbs.h"
21 
22 static void assemble_qbuf_mtt_for_cmd(struct erdma_mem *mem, u32 *cfg,
23 				      u64 *addr0, u64 *addr1)
24 {
25 	struct erdma_mtt *mtt = mem->mtt;
26 
27 	if (mem->mtt_nents > ERDMA_MAX_INLINE_MTT_ENTRIES) {
28 		*addr0 = mtt->buf_dma;
29 		*cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
30 				   ERDMA_MR_MTT_1LEVEL);
31 	} else {
32 		*addr0 = mtt->buf[0];
33 		memcpy(addr1, mtt->buf + 1, MTT_SIZE(mem->mtt_nents - 1));
34 		*cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
35 				   ERDMA_MR_MTT_0LEVEL);
36 	}
37 }
38 
39 static int create_qp_cmd(struct erdma_ucontext *uctx, struct erdma_qp *qp)
40 {
41 	struct erdma_dev *dev = to_edev(qp->ibqp.device);
42 	struct erdma_pd *pd = to_epd(qp->ibqp.pd);
43 	struct erdma_cmdq_create_qp_req req;
44 	struct erdma_uqp *user_qp;
45 	u64 resp0, resp1;
46 	int err;
47 
48 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
49 				CMDQ_OPCODE_CREATE_QP);
50 
51 	req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_QP_SQ_DEPTH_MASK,
52 			      ilog2(qp->attrs.sq_size)) |
53 		   FIELD_PREP(ERDMA_CMD_CREATE_QP_QPN_MASK, QP_ID(qp));
54 	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_QP_RQ_DEPTH_MASK,
55 			      ilog2(qp->attrs.rq_size)) |
56 		   FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
57 
58 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
59 		u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
60 
61 		req.sq_cqn_mtt_cfg =
62 			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
63 				   pgsz_range) |
64 			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
65 		req.rq_cqn_mtt_cfg =
66 			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
67 				   pgsz_range) |
68 			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
69 
70 		req.sq_mtt_cfg =
71 			FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_OFFSET_MASK, 0) |
72 			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK, 1) |
73 			FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_LEVEL_MASK,
74 				   ERDMA_MR_MTT_0LEVEL);
75 		req.rq_mtt_cfg = req.sq_mtt_cfg;
76 
77 		req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr;
78 		req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr;
79 		req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr +
80 					  (qp->attrs.sq_size << SQEBB_SHIFT);
81 		req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr +
82 					  (qp->attrs.rq_size << RQE_SHIFT);
83 	} else {
84 		user_qp = &qp->user_qp;
85 		req.sq_cqn_mtt_cfg = FIELD_PREP(
86 			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
87 			ilog2(user_qp->sq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
88 		req.sq_cqn_mtt_cfg |=
89 			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
90 
91 		req.rq_cqn_mtt_cfg = FIELD_PREP(
92 			ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
93 			ilog2(user_qp->rq_mem.page_size) - ERDMA_HW_PAGE_SHIFT);
94 		req.rq_cqn_mtt_cfg |=
95 			FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
96 
97 		req.sq_mtt_cfg = user_qp->sq_mem.page_offset;
98 		req.sq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
99 					     user_qp->sq_mem.mtt_nents);
100 
101 		req.rq_mtt_cfg = user_qp->rq_mem.page_offset;
102 		req.rq_mtt_cfg |= FIELD_PREP(ERDMA_CMD_CREATE_QP_MTT_CNT_MASK,
103 					     user_qp->rq_mem.mtt_nents);
104 
105 		assemble_qbuf_mtt_for_cmd(&user_qp->sq_mem, &req.sq_mtt_cfg,
106 					  &req.sq_buf_addr, req.sq_mtt_entry);
107 		assemble_qbuf_mtt_for_cmd(&user_qp->rq_mem, &req.rq_mtt_cfg,
108 					  &req.rq_buf_addr, req.rq_mtt_entry);
109 
110 		req.sq_db_info_dma_addr = user_qp->sq_db_info_dma_addr;
111 		req.rq_db_info_dma_addr = user_qp->rq_db_info_dma_addr;
112 
113 		if (uctx->ext_db.enable) {
114 			req.sq_cqn_mtt_cfg |=
115 				FIELD_PREP(ERDMA_CMD_CREATE_QP_DB_CFG_MASK, 1);
116 			req.db_cfg =
117 				FIELD_PREP(ERDMA_CMD_CREATE_QP_SQDB_CFG_MASK,
118 					   uctx->ext_db.sdb_off) |
119 				FIELD_PREP(ERDMA_CMD_CREATE_QP_RQDB_CFG_MASK,
120 					   uctx->ext_db.rdb_off);
121 		}
122 	}
123 
124 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &resp0,
125 				  &resp1);
126 	if (!err)
127 		qp->attrs.cookie =
128 			FIELD_GET(ERDMA_CMDQ_CREATE_QP_RESP_COOKIE_MASK, resp0);
129 
130 	return err;
131 }
132 
133 static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
134 {
135 	struct erdma_pd *pd = to_epd(mr->ibmr.pd);
136 	u32 mtt_level = ERDMA_MR_MTT_0LEVEL;
137 	struct erdma_cmdq_reg_mr_req req;
138 
139 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA, CMDQ_OPCODE_REG_MR);
140 
141 	if (mr->type == ERDMA_MR_TYPE_FRMR ||
142 	    mr->mem.page_cnt > ERDMA_MAX_INLINE_MTT_ENTRIES) {
143 		if (mr->mem.mtt->continuous) {
144 			req.phy_addr[0] = mr->mem.mtt->buf_dma;
145 			mtt_level = ERDMA_MR_MTT_1LEVEL;
146 		} else {
147 			req.phy_addr[0] = sg_dma_address(mr->mem.mtt->sglist);
148 			mtt_level = mr->mem.mtt->level;
149 		}
150 	} else if (mr->type != ERDMA_MR_TYPE_DMA) {
151 		memcpy(req.phy_addr, mr->mem.mtt->buf,
152 		       MTT_SIZE(mr->mem.page_cnt));
153 	}
154 
155 	req.cfg0 = FIELD_PREP(ERDMA_CMD_MR_VALID_MASK, mr->valid) |
156 		   FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF) |
157 		   FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8);
158 	req.cfg1 = FIELD_PREP(ERDMA_CMD_REGMR_PD_MASK, pd->pdn) |
159 		   FIELD_PREP(ERDMA_CMD_REGMR_TYPE_MASK, mr->type) |
160 		   FIELD_PREP(ERDMA_CMD_REGMR_RIGHT_MASK, mr->access);
161 	req.cfg2 = FIELD_PREP(ERDMA_CMD_REGMR_PAGESIZE_MASK,
162 			      ilog2(mr->mem.page_size)) |
163 		   FIELD_PREP(ERDMA_CMD_REGMR_MTT_LEVEL_MASK, mtt_level) |
164 		   FIELD_PREP(ERDMA_CMD_REGMR_MTT_CNT_MASK, mr->mem.page_cnt);
165 
166 	if (mr->type == ERDMA_MR_TYPE_DMA)
167 		goto post_cmd;
168 
169 	if (mr->type == ERDMA_MR_TYPE_NORMAL) {
170 		req.start_va = mr->mem.va;
171 		req.size = mr->mem.len;
172 	}
173 
174 	if (!mr->mem.mtt->continuous && mr->mem.mtt->level > 1) {
175 		req.cfg0 |= FIELD_PREP(ERDMA_CMD_MR_VERSION_MASK, 1);
176 		req.cfg2 |= FIELD_PREP(ERDMA_CMD_REGMR_MTT_PAGESIZE_MASK,
177 				       PAGE_SHIFT - ERDMA_HW_PAGE_SHIFT);
178 		req.size_h = upper_32_bits(mr->mem.len);
179 		req.mtt_cnt_h = mr->mem.page_cnt >> 20;
180 	}
181 
182 post_cmd:
183 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
184 }
185 
186 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
187 {
188 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
189 	struct erdma_cmdq_create_cq_req req;
190 	struct erdma_mem *mem;
191 	u32 page_size;
192 
193 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
194 				CMDQ_OPCODE_CREATE_CQ);
195 
196 	req.cfg0 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_CQN_MASK, cq->cqn) |
197 		   FIELD_PREP(ERDMA_CMD_CREATE_CQ_DEPTH_MASK, ilog2(cq->depth));
198 	req.cfg1 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_EQN_MASK, cq->assoc_eqn);
199 
200 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
201 		page_size = SZ_32M;
202 		req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
203 				       ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
204 		req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
205 		req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
206 
207 		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK, 1) |
208 			    FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
209 				       ERDMA_MR_MTT_0LEVEL);
210 
211 		req.first_page_offset = 0;
212 		req.cq_db_info_addr =
213 			cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
214 	} else {
215 		mem = &cq->user_cq.qbuf_mem;
216 		req.cfg0 |=
217 			FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
218 				   ilog2(mem->page_size) - ERDMA_HW_PAGE_SHIFT);
219 		if (mem->mtt_nents == 1) {
220 			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf[0]);
221 			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf[0]);
222 			req.cfg1 |=
223 				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
224 					   ERDMA_MR_MTT_0LEVEL);
225 		} else {
226 			req.qbuf_addr_l = lower_32_bits(mem->mtt->buf_dma);
227 			req.qbuf_addr_h = upper_32_bits(mem->mtt->buf_dma);
228 			req.cfg1 |=
229 				FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_LEVEL_MASK,
230 					   ERDMA_MR_MTT_1LEVEL);
231 		}
232 		req.cfg1 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_MTT_CNT_MASK,
233 				       mem->mtt_nents);
234 
235 		req.first_page_offset = mem->page_offset;
236 		req.cq_db_info_addr = cq->user_cq.db_info_dma_addr;
237 
238 		if (uctx->ext_db.enable) {
239 			req.cfg1 |= FIELD_PREP(
240 				ERDMA_CMD_CREATE_CQ_MTT_DB_CFG_MASK, 1);
241 			req.cfg2 = FIELD_PREP(ERDMA_CMD_CREATE_CQ_DB_CFG_MASK,
242 					      uctx->ext_db.cdb_off);
243 		}
244 	}
245 
246 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
247 }
248 
249 static int erdma_alloc_idx(struct erdma_resource_cb *res_cb)
250 {
251 	int idx;
252 	unsigned long flags;
253 
254 	spin_lock_irqsave(&res_cb->lock, flags);
255 	idx = find_next_zero_bit(res_cb->bitmap, res_cb->max_cap,
256 				 res_cb->next_alloc_idx);
257 	if (idx == res_cb->max_cap) {
258 		idx = find_first_zero_bit(res_cb->bitmap, res_cb->max_cap);
259 		if (idx == res_cb->max_cap) {
260 			res_cb->next_alloc_idx = 1;
261 			spin_unlock_irqrestore(&res_cb->lock, flags);
262 			return -ENOSPC;
263 		}
264 	}
265 
266 	set_bit(idx, res_cb->bitmap);
267 	res_cb->next_alloc_idx = idx + 1;
268 	spin_unlock_irqrestore(&res_cb->lock, flags);
269 
270 	return idx;
271 }
272 
273 static inline void erdma_free_idx(struct erdma_resource_cb *res_cb, u32 idx)
274 {
275 	unsigned long flags;
276 	u32 used;
277 
278 	spin_lock_irqsave(&res_cb->lock, flags);
279 	used = __test_and_clear_bit(idx, res_cb->bitmap);
280 	spin_unlock_irqrestore(&res_cb->lock, flags);
281 	WARN_ON(!used);
282 }
283 
284 static struct rdma_user_mmap_entry *
285 erdma_user_mmap_entry_insert(struct erdma_ucontext *uctx, void *address,
286 			     u32 size, u8 mmap_flag, u64 *mmap_offset)
287 {
288 	struct erdma_user_mmap_entry *entry =
289 		kzalloc(sizeof(*entry), GFP_KERNEL);
290 	int ret;
291 
292 	if (!entry)
293 		return NULL;
294 
295 	entry->address = (u64)address;
296 	entry->mmap_flag = mmap_flag;
297 
298 	size = PAGE_ALIGN(size);
299 
300 	ret = rdma_user_mmap_entry_insert(&uctx->ibucontext, &entry->rdma_entry,
301 					  size);
302 	if (ret) {
303 		kfree(entry);
304 		return NULL;
305 	}
306 
307 	*mmap_offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
308 
309 	return &entry->rdma_entry;
310 }
311 
312 int erdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
313 		       struct ib_udata *unused)
314 {
315 	struct erdma_dev *dev = to_edev(ibdev);
316 
317 	memset(attr, 0, sizeof(*attr));
318 
319 	attr->max_mr_size = dev->attrs.max_mr_size;
320 	attr->vendor_id = PCI_VENDOR_ID_ALIBABA;
321 	attr->vendor_part_id = dev->pdev->device;
322 	attr->hw_ver = dev->pdev->revision;
323 	attr->max_qp = dev->attrs.max_qp - 1;
324 	attr->max_qp_wr = min(dev->attrs.max_send_wr, dev->attrs.max_recv_wr);
325 	attr->max_qp_rd_atom = dev->attrs.max_ord;
326 	attr->max_qp_init_rd_atom = dev->attrs.max_ird;
327 	attr->max_res_rd_atom = dev->attrs.max_qp * dev->attrs.max_ird;
328 	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
329 	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
330 	ibdev->local_dma_lkey = dev->attrs.local_dma_key;
331 	attr->max_send_sge = dev->attrs.max_send_sge;
332 	attr->max_recv_sge = dev->attrs.max_recv_sge;
333 	attr->max_sge_rd = dev->attrs.max_sge_rd;
334 	attr->max_cq = dev->attrs.max_cq - 1;
335 	attr->max_cqe = dev->attrs.max_cqe;
336 	attr->max_mr = dev->attrs.max_mr;
337 	attr->max_pd = dev->attrs.max_pd;
338 	attr->max_mw = dev->attrs.max_mw;
339 	attr->max_fast_reg_page_list_len = ERDMA_MAX_FRMR_PA;
340 	attr->page_size_cap = ERDMA_PAGE_SIZE_SUPPORT;
341 
342 	if (dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_ATOMIC)
343 		attr->atomic_cap = IB_ATOMIC_GLOB;
344 
345 	attr->fw_ver = dev->attrs.fw_version;
346 
347 	if (dev->netdev)
348 		addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
349 				    dev->netdev->dev_addr);
350 
351 	return 0;
352 }
353 
354 int erdma_query_gid(struct ib_device *ibdev, u32 port, int idx,
355 		    union ib_gid *gid)
356 {
357 	struct erdma_dev *dev = to_edev(ibdev);
358 
359 	memset(gid, 0, sizeof(*gid));
360 	ether_addr_copy(gid->raw, dev->attrs.peer_addr);
361 
362 	return 0;
363 }
364 
365 int erdma_query_port(struct ib_device *ibdev, u32 port,
366 		     struct ib_port_attr *attr)
367 {
368 	struct erdma_dev *dev = to_edev(ibdev);
369 	struct net_device *ndev = dev->netdev;
370 
371 	memset(attr, 0, sizeof(*attr));
372 
373 	attr->gid_tbl_len = 1;
374 	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
375 	attr->max_msg_sz = -1;
376 
377 	if (!ndev)
378 		goto out;
379 
380 	ib_get_eth_speed(ibdev, port, &attr->active_speed, &attr->active_width);
381 	attr->max_mtu = ib_mtu_int_to_enum(ndev->mtu);
382 	attr->active_mtu = ib_mtu_int_to_enum(ndev->mtu);
383 	if (netif_running(ndev) && netif_carrier_ok(ndev))
384 		dev->state = IB_PORT_ACTIVE;
385 	else
386 		dev->state = IB_PORT_DOWN;
387 	attr->state = dev->state;
388 
389 out:
390 	if (dev->state == IB_PORT_ACTIVE)
391 		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
392 	else
393 		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
394 
395 	return 0;
396 }
397 
398 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
399 			     struct ib_port_immutable *port_immutable)
400 {
401 	port_immutable->gid_tbl_len = 1;
402 	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
403 
404 	return 0;
405 }
406 
407 int erdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
408 {
409 	struct erdma_pd *pd = to_epd(ibpd);
410 	struct erdma_dev *dev = to_edev(ibpd->device);
411 	int pdn;
412 
413 	pdn = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_PD]);
414 	if (pdn < 0)
415 		return pdn;
416 
417 	pd->pdn = pdn;
418 
419 	return 0;
420 }
421 
422 int erdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
423 {
424 	struct erdma_pd *pd = to_epd(ibpd);
425 	struct erdma_dev *dev = to_edev(ibpd->device);
426 
427 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_PD], pd->pdn);
428 
429 	return 0;
430 }
431 
432 static void erdma_flush_worker(struct work_struct *work)
433 {
434 	struct delayed_work *dwork = to_delayed_work(work);
435 	struct erdma_qp *qp =
436 		container_of(dwork, struct erdma_qp, reflush_dwork);
437 	struct erdma_cmdq_reflush_req req;
438 
439 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
440 				CMDQ_OPCODE_REFLUSH);
441 	req.qpn = QP_ID(qp);
442 	req.sq_pi = qp->kern_qp.sq_pi;
443 	req.rq_pi = qp->kern_qp.rq_pi;
444 	erdma_post_cmd_wait(&qp->dev->cmdq, &req, sizeof(req), NULL, NULL);
445 }
446 
447 static int erdma_qp_validate_cap(struct erdma_dev *dev,
448 				 struct ib_qp_init_attr *attrs)
449 {
450 	if ((attrs->cap.max_send_wr > dev->attrs.max_send_wr) ||
451 	    (attrs->cap.max_recv_wr > dev->attrs.max_recv_wr) ||
452 	    (attrs->cap.max_send_sge > dev->attrs.max_send_sge) ||
453 	    (attrs->cap.max_recv_sge > dev->attrs.max_recv_sge) ||
454 	    (attrs->cap.max_inline_data > ERDMA_MAX_INLINE) ||
455 	    !attrs->cap.max_send_wr || !attrs->cap.max_recv_wr) {
456 		return -EINVAL;
457 	}
458 
459 	return 0;
460 }
461 
462 static int erdma_qp_validate_attr(struct erdma_dev *dev,
463 				  struct ib_qp_init_attr *attrs)
464 {
465 	if (attrs->qp_type != IB_QPT_RC)
466 		return -EOPNOTSUPP;
467 
468 	if (attrs->srq)
469 		return -EOPNOTSUPP;
470 
471 	if (!attrs->send_cq || !attrs->recv_cq)
472 		return -EOPNOTSUPP;
473 
474 	return 0;
475 }
476 
477 static void free_kernel_qp(struct erdma_qp *qp)
478 {
479 	struct erdma_dev *dev = qp->dev;
480 
481 	vfree(qp->kern_qp.swr_tbl);
482 	vfree(qp->kern_qp.rwr_tbl);
483 
484 	if (qp->kern_qp.sq_buf)
485 		dma_free_coherent(
486 			&dev->pdev->dev,
487 			WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
488 			qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
489 
490 	if (qp->kern_qp.rq_buf)
491 		dma_free_coherent(
492 			&dev->pdev->dev,
493 			WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
494 			qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
495 }
496 
497 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp,
498 			  struct ib_qp_init_attr *attrs)
499 {
500 	struct erdma_kqp *kqp = &qp->kern_qp;
501 	int size;
502 
503 	if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
504 		kqp->sig_all = 1;
505 
506 	kqp->sq_pi = 0;
507 	kqp->sq_ci = 0;
508 	kqp->rq_pi = 0;
509 	kqp->rq_ci = 0;
510 	kqp->hw_sq_db =
511 		dev->func_bar + (ERDMA_SDB_SHARED_PAGE_INDEX << PAGE_SHIFT);
512 	kqp->hw_rq_db = dev->func_bar + ERDMA_BAR_RQDB_SPACE_OFFSET;
513 
514 	kqp->swr_tbl = vmalloc_array(qp->attrs.sq_size, sizeof(u64));
515 	kqp->rwr_tbl = vmalloc_array(qp->attrs.rq_size, sizeof(u64));
516 	if (!kqp->swr_tbl || !kqp->rwr_tbl)
517 		goto err_out;
518 
519 	size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
520 	kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
521 					 &kqp->sq_buf_dma_addr, GFP_KERNEL);
522 	if (!kqp->sq_buf)
523 		goto err_out;
524 
525 	size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE;
526 	kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size,
527 					 &kqp->rq_buf_dma_addr, GFP_KERNEL);
528 	if (!kqp->rq_buf)
529 		goto err_out;
530 
531 	kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT);
532 	kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT);
533 
534 	return 0;
535 
536 err_out:
537 	free_kernel_qp(qp);
538 	return -ENOMEM;
539 }
540 
541 static void erdma_fill_bottom_mtt(struct erdma_dev *dev, struct erdma_mem *mem)
542 {
543 	struct erdma_mtt *mtt = mem->mtt;
544 	struct ib_block_iter biter;
545 	u32 idx = 0;
546 
547 	while (mtt->low_level)
548 		mtt = mtt->low_level;
549 
550 	rdma_umem_for_each_dma_block(mem->umem, &biter, mem->page_size)
551 		mtt->buf[idx++] = rdma_block_iter_dma_address(&biter);
552 }
553 
554 static struct erdma_mtt *erdma_create_cont_mtt(struct erdma_dev *dev,
555 					       size_t size)
556 {
557 	struct erdma_mtt *mtt;
558 
559 	mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
560 	if (!mtt)
561 		return ERR_PTR(-ENOMEM);
562 
563 	mtt->size = size;
564 	mtt->buf = kzalloc(mtt->size, GFP_KERNEL);
565 	if (!mtt->buf)
566 		goto err_free_mtt;
567 
568 	mtt->continuous = true;
569 	mtt->buf_dma = dma_map_single(&dev->pdev->dev, mtt->buf, mtt->size,
570 				      DMA_TO_DEVICE);
571 	if (dma_mapping_error(&dev->pdev->dev, mtt->buf_dma))
572 		goto err_free_mtt_buf;
573 
574 	return mtt;
575 
576 err_free_mtt_buf:
577 	kfree(mtt->buf);
578 
579 err_free_mtt:
580 	kfree(mtt);
581 
582 	return ERR_PTR(-ENOMEM);
583 }
584 
585 static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
586 				     struct erdma_mtt *mtt)
587 {
588 	dma_unmap_sg(&dev->pdev->dev, mtt->sglist, mtt->nsg, DMA_TO_DEVICE);
589 	vfree(mtt->sglist);
590 }
591 
592 static void erdma_destroy_scatter_mtt(struct erdma_dev *dev,
593 				      struct erdma_mtt *mtt)
594 {
595 	erdma_destroy_mtt_buf_sg(dev, mtt);
596 	vfree(mtt->buf);
597 	kfree(mtt);
598 }
599 
600 static void erdma_init_middle_mtt(struct erdma_mtt *mtt,
601 				  struct erdma_mtt *low_mtt)
602 {
603 	struct scatterlist *sg;
604 	u32 idx = 0, i;
605 
606 	for_each_sg(low_mtt->sglist, sg, low_mtt->nsg, i)
607 		mtt->buf[idx++] = sg_dma_address(sg);
608 }
609 
610 static int erdma_create_mtt_buf_sg(struct erdma_dev *dev, struct erdma_mtt *mtt)
611 {
612 	struct scatterlist *sglist;
613 	void *buf = mtt->buf;
614 	u32 npages, i, nsg;
615 	struct page *pg;
616 
617 	/* Failed if buf is not page aligned */
618 	if ((uintptr_t)buf & ~PAGE_MASK)
619 		return -EINVAL;
620 
621 	npages = DIV_ROUND_UP(mtt->size, PAGE_SIZE);
622 	sglist = vzalloc(npages * sizeof(*sglist));
623 	if (!sglist)
624 		return -ENOMEM;
625 
626 	sg_init_table(sglist, npages);
627 	for (i = 0; i < npages; i++) {
628 		pg = vmalloc_to_page(buf);
629 		if (!pg)
630 			goto err;
631 		sg_set_page(&sglist[i], pg, PAGE_SIZE, 0);
632 		buf += PAGE_SIZE;
633 	}
634 
635 	nsg = dma_map_sg(&dev->pdev->dev, sglist, npages, DMA_TO_DEVICE);
636 	if (!nsg)
637 		goto err;
638 
639 	mtt->sglist = sglist;
640 	mtt->nsg = nsg;
641 
642 	return 0;
643 err:
644 	vfree(sglist);
645 
646 	return -ENOMEM;
647 }
648 
649 static struct erdma_mtt *erdma_create_scatter_mtt(struct erdma_dev *dev,
650 						  size_t size)
651 {
652 	struct erdma_mtt *mtt;
653 	int ret = -ENOMEM;
654 
655 	mtt = kzalloc(sizeof(*mtt), GFP_KERNEL);
656 	if (!mtt)
657 		return ERR_PTR(-ENOMEM);
658 
659 	mtt->size = ALIGN(size, PAGE_SIZE);
660 	mtt->buf = vzalloc(mtt->size);
661 	mtt->continuous = false;
662 	if (!mtt->buf)
663 		goto err_free_mtt;
664 
665 	ret = erdma_create_mtt_buf_sg(dev, mtt);
666 	if (ret)
667 		goto err_free_mtt_buf;
668 
669 	ibdev_dbg(&dev->ibdev, "create scatter mtt, size:%lu, nsg:%u\n",
670 		  mtt->size, mtt->nsg);
671 
672 	return mtt;
673 
674 err_free_mtt_buf:
675 	vfree(mtt->buf);
676 
677 err_free_mtt:
678 	kfree(mtt);
679 
680 	return ERR_PTR(ret);
681 }
682 
683 static struct erdma_mtt *erdma_create_mtt(struct erdma_dev *dev, size_t size,
684 					  bool force_continuous)
685 {
686 	struct erdma_mtt *mtt, *tmp_mtt;
687 	int ret, level = 0;
688 
689 	ibdev_dbg(&dev->ibdev, "create_mtt, size:%lu, force cont:%d\n", size,
690 		  force_continuous);
691 
692 	if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_MTT_VA))
693 		force_continuous = true;
694 
695 	if (force_continuous)
696 		return erdma_create_cont_mtt(dev, size);
697 
698 	mtt = erdma_create_scatter_mtt(dev, size);
699 	if (IS_ERR(mtt))
700 		return mtt;
701 	level = 1;
702 
703 	/* convergence the mtt table. */
704 	while (mtt->nsg != 1 && level <= 3) {
705 		tmp_mtt = erdma_create_scatter_mtt(dev, MTT_SIZE(mtt->nsg));
706 		if (IS_ERR(tmp_mtt)) {
707 			ret = PTR_ERR(tmp_mtt);
708 			goto err_free_mtt;
709 		}
710 		erdma_init_middle_mtt(tmp_mtt, mtt);
711 		tmp_mtt->low_level = mtt;
712 		mtt = tmp_mtt;
713 		level++;
714 	}
715 
716 	if (level > 3) {
717 		ret = -ENOMEM;
718 		goto err_free_mtt;
719 	}
720 
721 	mtt->level = level;
722 	ibdev_dbg(&dev->ibdev, "top mtt: level:%d, dma_addr 0x%llx\n",
723 		  mtt->level, mtt->sglist[0].dma_address);
724 
725 	return mtt;
726 err_free_mtt:
727 	while (mtt) {
728 		tmp_mtt = mtt->low_level;
729 		erdma_destroy_scatter_mtt(dev, mtt);
730 		mtt = tmp_mtt;
731 	}
732 
733 	return ERR_PTR(ret);
734 }
735 
736 static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
737 {
738 	struct erdma_mtt *tmp_mtt;
739 
740 	if (mtt->continuous) {
741 		dma_unmap_single(&dev->pdev->dev, mtt->buf_dma, mtt->size,
742 				 DMA_TO_DEVICE);
743 		kfree(mtt->buf);
744 		kfree(mtt);
745 	} else {
746 		while (mtt) {
747 			tmp_mtt = mtt->low_level;
748 			erdma_destroy_scatter_mtt(dev, mtt);
749 			mtt = tmp_mtt;
750 		}
751 	}
752 }
753 
754 static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
755 			   u64 start, u64 len, int access, u64 virt,
756 			   unsigned long req_page_size, bool force_continuous)
757 {
758 	int ret = 0;
759 
760 	mem->umem = ib_umem_get(&dev->ibdev, start, len, access);
761 	if (IS_ERR(mem->umem)) {
762 		ret = PTR_ERR(mem->umem);
763 		mem->umem = NULL;
764 		return ret;
765 	}
766 
767 	mem->va = virt;
768 	mem->len = len;
769 	mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
770 	mem->page_offset = start & (mem->page_size - 1);
771 	mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
772 	mem->page_cnt = mem->mtt_nents;
773 	mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
774 				    force_continuous);
775 	if (IS_ERR(mem->mtt)) {
776 		ret = PTR_ERR(mem->mtt);
777 		goto error_ret;
778 	}
779 
780 	erdma_fill_bottom_mtt(dev, mem);
781 
782 	return 0;
783 
784 error_ret:
785 	if (mem->umem) {
786 		ib_umem_release(mem->umem);
787 		mem->umem = NULL;
788 	}
789 
790 	return ret;
791 }
792 
793 static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
794 {
795 	if (mem->mtt)
796 		erdma_destroy_mtt(dev, mem->mtt);
797 
798 	if (mem->umem) {
799 		ib_umem_release(mem->umem);
800 		mem->umem = NULL;
801 	}
802 }
803 
804 static int erdma_map_user_dbrecords(struct erdma_ucontext *ctx,
805 				    u64 dbrecords_va,
806 				    struct erdma_user_dbrecords_page **dbr_page,
807 				    dma_addr_t *dma_addr)
808 {
809 	struct erdma_user_dbrecords_page *page = NULL;
810 	int rv = 0;
811 
812 	mutex_lock(&ctx->dbrecords_page_mutex);
813 
814 	list_for_each_entry(page, &ctx->dbrecords_page_list, list)
815 		if (page->va == (dbrecords_va & PAGE_MASK))
816 			goto found;
817 
818 	page = kmalloc(sizeof(*page), GFP_KERNEL);
819 	if (!page) {
820 		rv = -ENOMEM;
821 		goto out;
822 	}
823 
824 	page->va = (dbrecords_va & PAGE_MASK);
825 	page->refcnt = 0;
826 
827 	page->umem = ib_umem_get(ctx->ibucontext.device,
828 				 dbrecords_va & PAGE_MASK, PAGE_SIZE, 0);
829 	if (IS_ERR(page->umem)) {
830 		rv = PTR_ERR(page->umem);
831 		kfree(page);
832 		goto out;
833 	}
834 
835 	list_add(&page->list, &ctx->dbrecords_page_list);
836 
837 found:
838 	*dma_addr = sg_dma_address(page->umem->sgt_append.sgt.sgl) +
839 		    (dbrecords_va & ~PAGE_MASK);
840 	*dbr_page = page;
841 	page->refcnt++;
842 
843 out:
844 	mutex_unlock(&ctx->dbrecords_page_mutex);
845 	return rv;
846 }
847 
848 static void
849 erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
850 			   struct erdma_user_dbrecords_page **dbr_page)
851 {
852 	if (!ctx || !(*dbr_page))
853 		return;
854 
855 	mutex_lock(&ctx->dbrecords_page_mutex);
856 	if (--(*dbr_page)->refcnt == 0) {
857 		list_del(&(*dbr_page)->list);
858 		ib_umem_release((*dbr_page)->umem);
859 		kfree(*dbr_page);
860 	}
861 
862 	*dbr_page = NULL;
863 	mutex_unlock(&ctx->dbrecords_page_mutex);
864 }
865 
866 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
867 			u64 va, u32 len, u64 db_info_va)
868 {
869 	dma_addr_t db_info_dma_addr;
870 	u32 rq_offset;
871 	int ret;
872 
873 	if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
874 		   qp->attrs.rq_size * RQE_SIZE))
875 		return -EINVAL;
876 
877 	ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
878 			      qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
879 			      (SZ_1M - SZ_4K), true);
880 	if (ret)
881 		return ret;
882 
883 	rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
884 	qp->user_qp.rq_offset = rq_offset;
885 
886 	ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
887 			      qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
888 			      (SZ_1M - SZ_4K), true);
889 	if (ret)
890 		goto put_sq_mtt;
891 
892 	ret = erdma_map_user_dbrecords(uctx, db_info_va,
893 				       &qp->user_qp.user_dbr_page,
894 				       &db_info_dma_addr);
895 	if (ret)
896 		goto put_rq_mtt;
897 
898 	qp->user_qp.sq_db_info_dma_addr = db_info_dma_addr;
899 	qp->user_qp.rq_db_info_dma_addr = db_info_dma_addr + ERDMA_DB_SIZE;
900 
901 	return 0;
902 
903 put_rq_mtt:
904 	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
905 
906 put_sq_mtt:
907 	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
908 
909 	return ret;
910 }
911 
912 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
913 {
914 	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
915 	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
916 	erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
917 }
918 
919 int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
920 		    struct ib_udata *udata)
921 {
922 	struct erdma_qp *qp = to_eqp(ibqp);
923 	struct erdma_dev *dev = to_edev(ibqp->device);
924 	struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
925 		udata, struct erdma_ucontext, ibucontext);
926 	struct erdma_ureq_create_qp ureq;
927 	struct erdma_uresp_create_qp uresp;
928 	int ret;
929 
930 	ret = erdma_qp_validate_cap(dev, attrs);
931 	if (ret)
932 		goto err_out;
933 
934 	ret = erdma_qp_validate_attr(dev, attrs);
935 	if (ret)
936 		goto err_out;
937 
938 	qp->scq = to_ecq(attrs->send_cq);
939 	qp->rcq = to_ecq(attrs->recv_cq);
940 	qp->dev = dev;
941 	qp->attrs.cc = dev->attrs.cc;
942 
943 	init_rwsem(&qp->state_lock);
944 	kref_init(&qp->ref);
945 	init_completion(&qp->safe_free);
946 
947 	ret = xa_alloc_cyclic(&dev->qp_xa, &qp->ibqp.qp_num, qp,
948 			      XA_LIMIT(1, dev->attrs.max_qp - 1),
949 			      &dev->next_alloc_qpn, GFP_KERNEL);
950 	if (ret < 0) {
951 		ret = -ENOMEM;
952 		goto err_out;
953 	}
954 
955 	qp->attrs.sq_size = roundup_pow_of_two(attrs->cap.max_send_wr *
956 					       ERDMA_MAX_WQEBB_PER_SQE);
957 	qp->attrs.rq_size = roundup_pow_of_two(attrs->cap.max_recv_wr);
958 
959 	if (uctx) {
960 		ret = ib_copy_from_udata(&ureq, udata,
961 					 min(sizeof(ureq), udata->inlen));
962 		if (ret)
963 			goto err_out_xa;
964 
965 		ret = init_user_qp(qp, uctx, ureq.qbuf_va, ureq.qbuf_len,
966 				   ureq.db_record_va);
967 		if (ret)
968 			goto err_out_xa;
969 
970 		memset(&uresp, 0, sizeof(uresp));
971 
972 		uresp.num_sqe = qp->attrs.sq_size;
973 		uresp.num_rqe = qp->attrs.rq_size;
974 		uresp.qp_id = QP_ID(qp);
975 		uresp.rq_offset = qp->user_qp.rq_offset;
976 
977 		ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
978 		if (ret)
979 			goto err_out_cmd;
980 	} else {
981 		init_kernel_qp(dev, qp, attrs);
982 	}
983 
984 	qp->attrs.max_send_sge = attrs->cap.max_send_sge;
985 	qp->attrs.max_recv_sge = attrs->cap.max_recv_sge;
986 	qp->attrs.state = ERDMA_QP_STATE_IDLE;
987 	INIT_DELAYED_WORK(&qp->reflush_dwork, erdma_flush_worker);
988 
989 	ret = create_qp_cmd(uctx, qp);
990 	if (ret)
991 		goto err_out_cmd;
992 
993 	spin_lock_init(&qp->lock);
994 
995 	return 0;
996 
997 err_out_cmd:
998 	if (uctx)
999 		free_user_qp(qp, uctx);
1000 	else
1001 		free_kernel_qp(qp);
1002 err_out_xa:
1003 	xa_erase(&dev->qp_xa, QP_ID(qp));
1004 err_out:
1005 	return ret;
1006 }
1007 
1008 static int erdma_create_stag(struct erdma_dev *dev, u32 *stag)
1009 {
1010 	int stag_idx;
1011 
1012 	stag_idx = erdma_alloc_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX]);
1013 	if (stag_idx < 0)
1014 		return stag_idx;
1015 
1016 	/* For now, we always let key field be zero. */
1017 	*stag = (stag_idx << 8);
1018 
1019 	return 0;
1020 }
1021 
1022 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int acc)
1023 {
1024 	struct erdma_dev *dev = to_edev(ibpd->device);
1025 	struct erdma_mr *mr;
1026 	u32 stag;
1027 	int ret;
1028 
1029 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1030 	if (!mr)
1031 		return ERR_PTR(-ENOMEM);
1032 
1033 	ret = erdma_create_stag(dev, &stag);
1034 	if (ret)
1035 		goto out_free;
1036 
1037 	mr->type = ERDMA_MR_TYPE_DMA;
1038 
1039 	mr->ibmr.lkey = stag;
1040 	mr->ibmr.rkey = stag;
1041 	mr->ibmr.pd = ibpd;
1042 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(acc);
1043 	ret = regmr_cmd(dev, mr);
1044 	if (ret)
1045 		goto out_remove_stag;
1046 
1047 	return &mr->ibmr;
1048 
1049 out_remove_stag:
1050 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1051 		       mr->ibmr.lkey >> 8);
1052 
1053 out_free:
1054 	kfree(mr);
1055 
1056 	return ERR_PTR(ret);
1057 }
1058 
1059 struct ib_mr *erdma_ib_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
1060 				u32 max_num_sg)
1061 {
1062 	struct erdma_mr *mr;
1063 	struct erdma_dev *dev = to_edev(ibpd->device);
1064 	int ret;
1065 	u32 stag;
1066 
1067 	if (mr_type != IB_MR_TYPE_MEM_REG)
1068 		return ERR_PTR(-EOPNOTSUPP);
1069 
1070 	if (max_num_sg > ERDMA_MR_MAX_MTT_CNT)
1071 		return ERR_PTR(-EINVAL);
1072 
1073 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1074 	if (!mr)
1075 		return ERR_PTR(-ENOMEM);
1076 
1077 	ret = erdma_create_stag(dev, &stag);
1078 	if (ret)
1079 		goto out_free;
1080 
1081 	mr->type = ERDMA_MR_TYPE_FRMR;
1082 
1083 	mr->ibmr.lkey = stag;
1084 	mr->ibmr.rkey = stag;
1085 	mr->ibmr.pd = ibpd;
1086 	/* update it in FRMR. */
1087 	mr->access = ERDMA_MR_ACC_LR | ERDMA_MR_ACC_LW | ERDMA_MR_ACC_RR |
1088 		     ERDMA_MR_ACC_RW;
1089 
1090 	mr->mem.page_size = PAGE_SIZE; /* update it later. */
1091 	mr->mem.page_cnt = max_num_sg;
1092 	mr->mem.mtt = erdma_create_mtt(dev, MTT_SIZE(max_num_sg), true);
1093 	if (IS_ERR(mr->mem.mtt)) {
1094 		ret = PTR_ERR(mr->mem.mtt);
1095 		goto out_remove_stag;
1096 	}
1097 
1098 	ret = regmr_cmd(dev, mr);
1099 	if (ret)
1100 		goto out_destroy_mtt;
1101 
1102 	return &mr->ibmr;
1103 
1104 out_destroy_mtt:
1105 	erdma_destroy_mtt(dev, mr->mem.mtt);
1106 
1107 out_remove_stag:
1108 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1109 		       mr->ibmr.lkey >> 8);
1110 
1111 out_free:
1112 	kfree(mr);
1113 
1114 	return ERR_PTR(ret);
1115 }
1116 
1117 static int erdma_set_page(struct ib_mr *ibmr, u64 addr)
1118 {
1119 	struct erdma_mr *mr = to_emr(ibmr);
1120 
1121 	if (mr->mem.mtt_nents >= mr->mem.page_cnt)
1122 		return -1;
1123 
1124 	mr->mem.mtt->buf[mr->mem.mtt_nents] = addr;
1125 	mr->mem.mtt_nents++;
1126 
1127 	return 0;
1128 }
1129 
1130 int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1131 		    unsigned int *sg_offset)
1132 {
1133 	struct erdma_mr *mr = to_emr(ibmr);
1134 	int num;
1135 
1136 	mr->mem.mtt_nents = 0;
1137 
1138 	num = ib_sg_to_pages(&mr->ibmr, sg, sg_nents, sg_offset,
1139 			     erdma_set_page);
1140 
1141 	return num;
1142 }
1143 
1144 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1145 				u64 virt, int access, struct ib_udata *udata)
1146 {
1147 	struct erdma_mr *mr = NULL;
1148 	struct erdma_dev *dev = to_edev(ibpd->device);
1149 	u32 stag;
1150 	int ret;
1151 
1152 	if (!len || len > dev->attrs.max_mr_size)
1153 		return ERR_PTR(-EINVAL);
1154 
1155 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1156 	if (!mr)
1157 		return ERR_PTR(-ENOMEM);
1158 
1159 	ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
1160 			      SZ_2G - SZ_4K, false);
1161 	if (ret)
1162 		goto err_out_free;
1163 
1164 	ret = erdma_create_stag(dev, &stag);
1165 	if (ret)
1166 		goto err_out_put_mtt;
1167 
1168 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
1169 	mr->ibmr.pd = ibpd;
1170 	mr->mem.va = virt;
1171 	mr->mem.len = len;
1172 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
1173 	mr->valid = 1;
1174 	mr->type = ERDMA_MR_TYPE_NORMAL;
1175 
1176 	ret = regmr_cmd(dev, mr);
1177 	if (ret)
1178 		goto err_out_mr;
1179 
1180 	return &mr->ibmr;
1181 
1182 err_out_mr:
1183 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
1184 		       mr->ibmr.lkey >> 8);
1185 
1186 err_out_put_mtt:
1187 	put_mtt_entries(dev, &mr->mem);
1188 
1189 err_out_free:
1190 	kfree(mr);
1191 
1192 	return ERR_PTR(ret);
1193 }
1194 
1195 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1196 {
1197 	struct erdma_mr *mr;
1198 	struct erdma_dev *dev = to_edev(ibmr->device);
1199 	struct erdma_cmdq_dereg_mr_req req;
1200 	int ret;
1201 
1202 	mr = to_emr(ibmr);
1203 
1204 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1205 				CMDQ_OPCODE_DEREG_MR);
1206 
1207 	req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
1208 		  FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
1209 
1210 	ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1211 	if (ret)
1212 		return ret;
1213 
1214 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
1215 
1216 	put_mtt_entries(dev, &mr->mem);
1217 
1218 	kfree(mr);
1219 	return 0;
1220 }
1221 
1222 int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1223 {
1224 	struct erdma_cq *cq = to_ecq(ibcq);
1225 	struct erdma_dev *dev = to_edev(ibcq->device);
1226 	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1227 		udata, struct erdma_ucontext, ibucontext);
1228 	int err;
1229 	struct erdma_cmdq_destroy_cq_req req;
1230 
1231 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1232 				CMDQ_OPCODE_DESTROY_CQ);
1233 	req.cqn = cq->cqn;
1234 
1235 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1236 	if (err)
1237 		return err;
1238 
1239 	if (rdma_is_kernel_res(&cq->ibcq.res)) {
1240 		dma_free_coherent(&dev->pdev->dev,
1241 				  WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1242 				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1243 	} else {
1244 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1245 		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1246 	}
1247 
1248 	xa_erase(&dev->cq_xa, cq->cqn);
1249 
1250 	return 0;
1251 }
1252 
1253 int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
1254 {
1255 	struct erdma_qp *qp = to_eqp(ibqp);
1256 	struct erdma_dev *dev = to_edev(ibqp->device);
1257 	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1258 		udata, struct erdma_ucontext, ibucontext);
1259 	struct erdma_qp_attrs qp_attrs;
1260 	int err;
1261 	struct erdma_cmdq_destroy_qp_req req;
1262 
1263 	down_write(&qp->state_lock);
1264 	qp_attrs.state = ERDMA_QP_STATE_ERROR;
1265 	erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
1266 	up_write(&qp->state_lock);
1267 
1268 	cancel_delayed_work_sync(&qp->reflush_dwork);
1269 
1270 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
1271 				CMDQ_OPCODE_DESTROY_QP);
1272 	req.qpn = QP_ID(qp);
1273 
1274 	err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1275 	if (err)
1276 		return err;
1277 
1278 	erdma_qp_put(qp);
1279 	wait_for_completion(&qp->safe_free);
1280 
1281 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
1282 		vfree(qp->kern_qp.swr_tbl);
1283 		vfree(qp->kern_qp.rwr_tbl);
1284 		dma_free_coherent(
1285 			&dev->pdev->dev,
1286 			WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT),
1287 			qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr);
1288 		dma_free_coherent(
1289 			&dev->pdev->dev,
1290 			WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT),
1291 			qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr);
1292 	} else {
1293 		put_mtt_entries(dev, &qp->user_qp.sq_mem);
1294 		put_mtt_entries(dev, &qp->user_qp.rq_mem);
1295 		erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
1296 	}
1297 
1298 	if (qp->cep)
1299 		erdma_cep_put(qp->cep);
1300 	xa_erase(&dev->qp_xa, QP_ID(qp));
1301 
1302 	return 0;
1303 }
1304 
1305 void erdma_qp_get_ref(struct ib_qp *ibqp)
1306 {
1307 	erdma_qp_get(to_eqp(ibqp));
1308 }
1309 
1310 void erdma_qp_put_ref(struct ib_qp *ibqp)
1311 {
1312 	erdma_qp_put(to_eqp(ibqp));
1313 }
1314 
1315 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
1316 {
1317 	struct rdma_user_mmap_entry *rdma_entry;
1318 	struct erdma_user_mmap_entry *entry;
1319 	pgprot_t prot;
1320 	int err;
1321 
1322 	rdma_entry = rdma_user_mmap_entry_get(ctx, vma);
1323 	if (!rdma_entry)
1324 		return -EINVAL;
1325 
1326 	entry = to_emmap(rdma_entry);
1327 
1328 	switch (entry->mmap_flag) {
1329 	case ERDMA_MMAP_IO_NC:
1330 		/* map doorbell. */
1331 		prot = pgprot_device(vma->vm_page_prot);
1332 		break;
1333 	default:
1334 		err = -EINVAL;
1335 		goto put_entry;
1336 	}
1337 
1338 	err = rdma_user_mmap_io(ctx, vma, PFN_DOWN(entry->address), PAGE_SIZE,
1339 				prot, rdma_entry);
1340 
1341 put_entry:
1342 	rdma_user_mmap_entry_put(rdma_entry);
1343 	return err;
1344 }
1345 
1346 void erdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
1347 {
1348 	struct erdma_user_mmap_entry *entry = to_emmap(rdma_entry);
1349 
1350 	kfree(entry);
1351 }
1352 
1353 static int alloc_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx,
1354 			      bool ext_db_en)
1355 {
1356 	struct erdma_cmdq_ext_db_req req = {};
1357 	u64 val0, val1;
1358 	int ret;
1359 
1360 	/*
1361 	 * CAP_SYS_RAWIO is required if hardware does not support extend
1362 	 * doorbell mechanism.
1363 	 */
1364 	if (!ext_db_en && !capable(CAP_SYS_RAWIO))
1365 		return -EPERM;
1366 
1367 	if (!ext_db_en) {
1368 		ctx->sdb = dev->func_bar_addr + ERDMA_BAR_SQDB_SPACE_OFFSET;
1369 		ctx->rdb = dev->func_bar_addr + ERDMA_BAR_RQDB_SPACE_OFFSET;
1370 		ctx->cdb = dev->func_bar_addr + ERDMA_BAR_CQDB_SPACE_OFFSET;
1371 		return 0;
1372 	}
1373 
1374 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1375 				CMDQ_OPCODE_ALLOC_DB);
1376 
1377 	req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1378 		  FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1379 		  FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1380 
1381 	ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), &val0, &val1);
1382 	if (ret)
1383 		return ret;
1384 
1385 	ctx->ext_db.enable = true;
1386 	ctx->ext_db.sdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_SDB);
1387 	ctx->ext_db.rdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_RDB);
1388 	ctx->ext_db.cdb_off = ERDMA_GET(val0, ALLOC_DB_RESP_CDB);
1389 
1390 	ctx->sdb = dev->func_bar_addr + (ctx->ext_db.sdb_off << PAGE_SHIFT);
1391 	ctx->cdb = dev->func_bar_addr + (ctx->ext_db.rdb_off << PAGE_SHIFT);
1392 	ctx->rdb = dev->func_bar_addr + (ctx->ext_db.cdb_off << PAGE_SHIFT);
1393 
1394 	return 0;
1395 }
1396 
1397 static void free_db_resources(struct erdma_dev *dev, struct erdma_ucontext *ctx)
1398 {
1399 	struct erdma_cmdq_ext_db_req req = {};
1400 	int ret;
1401 
1402 	if (!ctx->ext_db.enable)
1403 		return;
1404 
1405 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1406 				CMDQ_OPCODE_FREE_DB);
1407 
1408 	req.cfg = FIELD_PREP(ERDMA_CMD_EXT_DB_CQ_EN_MASK, 1) |
1409 		  FIELD_PREP(ERDMA_CMD_EXT_DB_RQ_EN_MASK, 1) |
1410 		  FIELD_PREP(ERDMA_CMD_EXT_DB_SQ_EN_MASK, 1);
1411 
1412 	req.sdb_off = ctx->ext_db.sdb_off;
1413 	req.rdb_off = ctx->ext_db.rdb_off;
1414 	req.cdb_off = ctx->ext_db.cdb_off;
1415 
1416 	ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1417 	if (ret)
1418 		ibdev_err_ratelimited(&dev->ibdev,
1419 				      "free db resources failed %d", ret);
1420 }
1421 
1422 static void erdma_uctx_user_mmap_entries_remove(struct erdma_ucontext *uctx)
1423 {
1424 	rdma_user_mmap_entry_remove(uctx->sq_db_mmap_entry);
1425 	rdma_user_mmap_entry_remove(uctx->rq_db_mmap_entry);
1426 	rdma_user_mmap_entry_remove(uctx->cq_db_mmap_entry);
1427 }
1428 
1429 int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
1430 {
1431 	struct erdma_ucontext *ctx = to_ectx(ibctx);
1432 	struct erdma_dev *dev = to_edev(ibctx->device);
1433 	int ret;
1434 	struct erdma_uresp_alloc_ctx uresp = {};
1435 
1436 	if (atomic_inc_return(&dev->num_ctx) > ERDMA_MAX_CONTEXT) {
1437 		ret = -ENOMEM;
1438 		goto err_out;
1439 	}
1440 
1441 	if (udata->outlen < sizeof(uresp)) {
1442 		ret = -EINVAL;
1443 		goto err_out;
1444 	}
1445 
1446 	INIT_LIST_HEAD(&ctx->dbrecords_page_list);
1447 	mutex_init(&ctx->dbrecords_page_mutex);
1448 
1449 	ret = alloc_db_resources(dev, ctx,
1450 				 !!(dev->attrs.cap_flags &
1451 				    ERDMA_DEV_CAP_FLAGS_EXTEND_DB));
1452 	if (ret)
1453 		goto err_out;
1454 
1455 	ctx->sq_db_mmap_entry = erdma_user_mmap_entry_insert(
1456 		ctx, (void *)ctx->sdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.sdb);
1457 	if (!ctx->sq_db_mmap_entry) {
1458 		ret = -ENOMEM;
1459 		goto err_free_ext_db;
1460 	}
1461 
1462 	ctx->rq_db_mmap_entry = erdma_user_mmap_entry_insert(
1463 		ctx, (void *)ctx->rdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.rdb);
1464 	if (!ctx->rq_db_mmap_entry) {
1465 		ret = -EINVAL;
1466 		goto err_put_mmap_entries;
1467 	}
1468 
1469 	ctx->cq_db_mmap_entry = erdma_user_mmap_entry_insert(
1470 		ctx, (void *)ctx->cdb, PAGE_SIZE, ERDMA_MMAP_IO_NC, &uresp.cdb);
1471 	if (!ctx->cq_db_mmap_entry) {
1472 		ret = -EINVAL;
1473 		goto err_put_mmap_entries;
1474 	}
1475 
1476 	uresp.dev_id = dev->pdev->device;
1477 
1478 	ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1479 	if (ret)
1480 		goto err_put_mmap_entries;
1481 
1482 	return 0;
1483 
1484 err_put_mmap_entries:
1485 	erdma_uctx_user_mmap_entries_remove(ctx);
1486 
1487 err_free_ext_db:
1488 	free_db_resources(dev, ctx);
1489 
1490 err_out:
1491 	atomic_dec(&dev->num_ctx);
1492 	return ret;
1493 }
1494 
1495 void erdma_dealloc_ucontext(struct ib_ucontext *ibctx)
1496 {
1497 	struct erdma_dev *dev = to_edev(ibctx->device);
1498 	struct erdma_ucontext *ctx = to_ectx(ibctx);
1499 
1500 	erdma_uctx_user_mmap_entries_remove(ctx);
1501 	free_db_resources(dev, ctx);
1502 	atomic_dec(&dev->num_ctx);
1503 }
1504 
1505 static int ib_qp_state_to_erdma_qp_state[IB_QPS_ERR + 1] = {
1506 	[IB_QPS_RESET] = ERDMA_QP_STATE_IDLE,
1507 	[IB_QPS_INIT] = ERDMA_QP_STATE_IDLE,
1508 	[IB_QPS_RTR] = ERDMA_QP_STATE_RTR,
1509 	[IB_QPS_RTS] = ERDMA_QP_STATE_RTS,
1510 	[IB_QPS_SQD] = ERDMA_QP_STATE_CLOSING,
1511 	[IB_QPS_SQE] = ERDMA_QP_STATE_TERMINATE,
1512 	[IB_QPS_ERR] = ERDMA_QP_STATE_ERROR
1513 };
1514 
1515 int erdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
1516 		    struct ib_udata *udata)
1517 {
1518 	struct erdma_qp_attrs new_attrs;
1519 	enum erdma_qp_attr_mask erdma_attr_mask = 0;
1520 	struct erdma_qp *qp = to_eqp(ibqp);
1521 	int ret = 0;
1522 
1523 	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
1524 		return -EOPNOTSUPP;
1525 
1526 	memset(&new_attrs, 0, sizeof(new_attrs));
1527 
1528 	if (attr_mask & IB_QP_STATE) {
1529 		new_attrs.state = ib_qp_state_to_erdma_qp_state[attr->qp_state];
1530 
1531 		erdma_attr_mask |= ERDMA_QP_ATTR_STATE;
1532 	}
1533 
1534 	down_write(&qp->state_lock);
1535 
1536 	ret = erdma_modify_qp_internal(qp, &new_attrs, erdma_attr_mask);
1537 
1538 	up_write(&qp->state_lock);
1539 
1540 	return ret;
1541 }
1542 
1543 int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
1544 		   int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1545 {
1546 	struct erdma_qp *qp;
1547 	struct erdma_dev *dev;
1548 
1549 	if (ibqp && qp_attr && qp_init_attr) {
1550 		qp = to_eqp(ibqp);
1551 		dev = to_edev(ibqp->device);
1552 	} else {
1553 		return -EINVAL;
1554 	}
1555 
1556 	qp_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1557 	qp_init_attr->cap.max_inline_data = ERDMA_MAX_INLINE;
1558 
1559 	qp_attr->cap.max_send_wr = qp->attrs.sq_size;
1560 	qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
1561 	qp_attr->cap.max_send_sge = qp->attrs.max_send_sge;
1562 	qp_attr->cap.max_recv_sge = qp->attrs.max_recv_sge;
1563 
1564 	qp_attr->path_mtu = ib_mtu_int_to_enum(dev->netdev->mtu);
1565 	qp_attr->max_rd_atomic = qp->attrs.irq_size;
1566 	qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
1567 
1568 	qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
1569 				   IB_ACCESS_REMOTE_WRITE |
1570 				   IB_ACCESS_REMOTE_READ;
1571 
1572 	qp_init_attr->cap = qp_attr->cap;
1573 
1574 	return 0;
1575 }
1576 
1577 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
1578 			      struct erdma_ureq_create_cq *ureq)
1579 {
1580 	int ret;
1581 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
1582 
1583 	ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
1584 			      ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
1585 			      true);
1586 	if (ret)
1587 		return ret;
1588 
1589 	ret = erdma_map_user_dbrecords(ctx, ureq->db_record_va,
1590 				       &cq->user_cq.user_dbr_page,
1591 				       &cq->user_cq.db_info_dma_addr);
1592 	if (ret)
1593 		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1594 
1595 	return ret;
1596 }
1597 
1598 static int erdma_init_kernel_cq(struct erdma_cq *cq)
1599 {
1600 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
1601 
1602 	cq->kern_cq.qbuf =
1603 		dma_alloc_coherent(&dev->pdev->dev,
1604 				   WARPPED_BUFSIZE(cq->depth << CQE_SHIFT),
1605 				   &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL);
1606 	if (!cq->kern_cq.qbuf)
1607 		return -ENOMEM;
1608 
1609 	cq->kern_cq.db_record =
1610 		(u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT));
1611 	spin_lock_init(&cq->kern_cq.lock);
1612 	/* use default cqdb addr */
1613 	cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET;
1614 
1615 	return 0;
1616 }
1617 
1618 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1619 		    struct ib_udata *udata)
1620 {
1621 	struct erdma_cq *cq = to_ecq(ibcq);
1622 	struct erdma_dev *dev = to_edev(ibcq->device);
1623 	unsigned int depth = attr->cqe;
1624 	int ret;
1625 	struct erdma_ucontext *ctx = rdma_udata_to_drv_context(
1626 		udata, struct erdma_ucontext, ibucontext);
1627 
1628 	if (depth > dev->attrs.max_cqe)
1629 		return -EINVAL;
1630 
1631 	depth = roundup_pow_of_two(depth);
1632 	cq->ibcq.cqe = depth;
1633 	cq->depth = depth;
1634 	cq->assoc_eqn = attr->comp_vector + 1;
1635 
1636 	ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq,
1637 			      XA_LIMIT(1, dev->attrs.max_cq - 1),
1638 			      &dev->next_alloc_cqn, GFP_KERNEL);
1639 	if (ret < 0)
1640 		return ret;
1641 
1642 	if (!rdma_is_kernel_res(&ibcq->res)) {
1643 		struct erdma_ureq_create_cq ureq;
1644 		struct erdma_uresp_create_cq uresp;
1645 
1646 		ret = ib_copy_from_udata(&ureq, udata,
1647 					 min(udata->inlen, sizeof(ureq)));
1648 		if (ret)
1649 			goto err_out_xa;
1650 
1651 		ret = erdma_init_user_cq(ctx, cq, &ureq);
1652 		if (ret)
1653 			goto err_out_xa;
1654 
1655 		uresp.cq_id = cq->cqn;
1656 		uresp.num_cqe = depth;
1657 
1658 		ret = ib_copy_to_udata(udata, &uresp,
1659 				       min(sizeof(uresp), udata->outlen));
1660 		if (ret)
1661 			goto err_free_res;
1662 	} else {
1663 		ret = erdma_init_kernel_cq(cq);
1664 		if (ret)
1665 			goto err_out_xa;
1666 	}
1667 
1668 	ret = create_cq_cmd(ctx, cq);
1669 	if (ret)
1670 		goto err_free_res;
1671 
1672 	return 0;
1673 
1674 err_free_res:
1675 	if (!rdma_is_kernel_res(&ibcq->res)) {
1676 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
1677 		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
1678 	} else {
1679 		dma_free_coherent(&dev->pdev->dev,
1680 				  WARPPED_BUFSIZE(depth << CQE_SHIFT),
1681 				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
1682 	}
1683 
1684 err_out_xa:
1685 	xa_erase(&dev->cq_xa, cq->cqn);
1686 
1687 	return ret;
1688 }
1689 
1690 void erdma_set_mtu(struct erdma_dev *dev, u32 mtu)
1691 {
1692 	struct erdma_cmdq_config_mtu_req req;
1693 
1694 	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
1695 				CMDQ_OPCODE_CONF_MTU);
1696 	req.mtu = mtu;
1697 
1698 	erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
1699 }
1700 
1701 void erdma_port_event(struct erdma_dev *dev, enum ib_event_type reason)
1702 {
1703 	struct ib_event event;
1704 
1705 	event.device = &dev->ibdev;
1706 	event.element.port_num = 1;
1707 	event.event = reason;
1708 
1709 	ib_dispatch_event(&event);
1710 }
1711