xref: /openbmc/linux/drivers/infiniband/hw/qedr/verbs.c (revision 95175dda)
1 /* QLogic qedr NIC Driver
2  * Copyright (c) 2015-2016  QLogic Corporation
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and /or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38 
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45 #include <rdma/uverbs_ioctl.h>
46 
47 #include <linux/qed/common_hsi.h>
48 #include "qedr_hsi_rdma.h"
49 #include <linux/qed/qed_if.h>
50 #include "qedr.h"
51 #include "verbs.h"
52 #include <rdma/qedr-abi.h>
53 #include "qedr_roce_cm.h"
54 #include "qedr_iw_cm.h"
55 
56 #define QEDR_SRQ_WQE_ELEM_SIZE	sizeof(union rdma_srq_elm)
57 #define	RDMA_MAX_SGE_PER_SRQ	(4)
58 #define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1)
59 
60 #define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61 
62 enum {
63 	QEDR_USER_MMAP_IO_WC = 0,
64 	QEDR_USER_MMAP_PHYS_PAGE,
65 };
66 
qedr_ib_copy_to_udata(struct ib_udata * udata,void * src,size_t len)67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 					size_t len)
69 {
70 	size_t min_len = min_t(size_t, len, udata->outlen);
71 
72 	return ib_copy_to_udata(udata, src, min_len);
73 }
74 
qedr_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)75 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
76 {
77 	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
78 		return -EINVAL;
79 
80 	*pkey = QEDR_ROCE_PKEY_DEFAULT;
81 	return 0;
82 }
83 
qedr_iw_query_gid(struct ib_device * ibdev,u32 port,int index,union ib_gid * sgid)84 int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
85 		      int index, union ib_gid *sgid)
86 {
87 	struct qedr_dev *dev = get_qedr_dev(ibdev);
88 
89 	memset(sgid->raw, 0, sizeof(sgid->raw));
90 	ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91 
92 	DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 		 sgid->global.interface_id, sgid->global.subnet_prefix);
94 
95 	return 0;
96 }
97 
qedr_query_srq(struct ib_srq * ibsrq,struct ib_srq_attr * srq_attr)98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99 {
100 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 	struct qedr_device_attr *qattr = &dev->attr;
102 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
103 
104 	srq_attr->srq_limit = srq->srq_limit;
105 	srq_attr->max_wr = qattr->max_srq_wr;
106 	srq_attr->max_sge = qattr->max_sge;
107 
108 	return 0;
109 }
110 
qedr_query_device(struct ib_device * ibdev,struct ib_device_attr * attr,struct ib_udata * udata)111 int qedr_query_device(struct ib_device *ibdev,
112 		      struct ib_device_attr *attr, struct ib_udata *udata)
113 {
114 	struct qedr_dev *dev = get_qedr_dev(ibdev);
115 	struct qedr_device_attr *qattr = &dev->attr;
116 
117 	if (!dev->rdma_ctx) {
118 		DP_ERR(dev,
119 		       "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 		       dev->rdma_ctx);
121 		return -EINVAL;
122 	}
123 
124 	memset(attr, 0, sizeof(*attr));
125 
126 	attr->fw_ver = qattr->fw_ver;
127 	attr->sys_image_guid = qattr->sys_image_guid;
128 	attr->max_mr_size = qattr->max_mr_size;
129 	attr->page_size_cap = qattr->page_size_caps;
130 	attr->vendor_id = qattr->vendor_id;
131 	attr->vendor_part_id = qattr->vendor_part_id;
132 	attr->hw_ver = qattr->hw_ver;
133 	attr->max_qp = qattr->max_qp;
134 	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 	    IB_DEVICE_RC_RNR_NAK_GEN |
137 	    IB_DEVICE_MEM_MGT_EXTENSIONS;
138 	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
139 
140 	if (!rdma_protocol_iwarp(&dev->ibdev, 1))
141 		attr->device_cap_flags |= IB_DEVICE_XRC;
142 	attr->max_send_sge = qattr->max_sge;
143 	attr->max_recv_sge = qattr->max_sge;
144 	attr->max_sge_rd = qattr->max_sge;
145 	attr->max_cq = qattr->max_cq;
146 	attr->max_cqe = qattr->max_cqe;
147 	attr->max_mr = qattr->max_mr;
148 	attr->max_mw = qattr->max_mw;
149 	attr->max_pd = qattr->max_pd;
150 	attr->atomic_cap = dev->atomic_cap;
151 	attr->max_qp_init_rd_atom =
152 	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
153 	attr->max_qp_rd_atom =
154 	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
155 		attr->max_qp_init_rd_atom);
156 
157 	attr->max_srq = qattr->max_srq;
158 	attr->max_srq_sge = qattr->max_srq_sge;
159 	attr->max_srq_wr = qattr->max_srq_wr;
160 
161 	attr->local_ca_ack_delay = qattr->dev_ack_delay;
162 	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
163 	attr->max_pkeys = qattr->max_pkey;
164 	attr->max_ah = qattr->max_ah;
165 
166 	return 0;
167 }
168 
get_link_speed_and_width(int speed,u16 * ib_speed,u8 * ib_width)169 static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
170 					    u8 *ib_width)
171 {
172 	switch (speed) {
173 	case 1000:
174 		*ib_speed = IB_SPEED_SDR;
175 		*ib_width = IB_WIDTH_1X;
176 		break;
177 	case 10000:
178 		*ib_speed = IB_SPEED_QDR;
179 		*ib_width = IB_WIDTH_1X;
180 		break;
181 
182 	case 20000:
183 		*ib_speed = IB_SPEED_DDR;
184 		*ib_width = IB_WIDTH_4X;
185 		break;
186 
187 	case 25000:
188 		*ib_speed = IB_SPEED_EDR;
189 		*ib_width = IB_WIDTH_1X;
190 		break;
191 
192 	case 40000:
193 		*ib_speed = IB_SPEED_QDR;
194 		*ib_width = IB_WIDTH_4X;
195 		break;
196 
197 	case 50000:
198 		*ib_speed = IB_SPEED_HDR;
199 		*ib_width = IB_WIDTH_1X;
200 		break;
201 
202 	case 100000:
203 		*ib_speed = IB_SPEED_EDR;
204 		*ib_width = IB_WIDTH_4X;
205 		break;
206 
207 	default:
208 		/* Unsupported */
209 		*ib_speed = IB_SPEED_SDR;
210 		*ib_width = IB_WIDTH_1X;
211 	}
212 }
213 
qedr_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * attr)214 int qedr_query_port(struct ib_device *ibdev, u32 port,
215 		    struct ib_port_attr *attr)
216 {
217 	struct qedr_dev *dev;
218 	struct qed_rdma_port *rdma_port;
219 
220 	dev = get_qedr_dev(ibdev);
221 
222 	if (!dev->rdma_ctx) {
223 		DP_ERR(dev, "rdma_ctx is NULL\n");
224 		return -EINVAL;
225 	}
226 
227 	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
228 
229 	/* *attr being zeroed by the caller, avoid zeroing it here */
230 	if (rdma_port->port_state == QED_RDMA_PORT_UP) {
231 		attr->state = IB_PORT_ACTIVE;
232 		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
233 	} else {
234 		attr->state = IB_PORT_DOWN;
235 		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
236 	}
237 	attr->max_mtu = IB_MTU_4096;
238 	attr->lid = 0;
239 	attr->lmc = 0;
240 	attr->sm_lid = 0;
241 	attr->sm_sl = 0;
242 	attr->ip_gids = true;
243 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
244 		attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
245 		attr->gid_tbl_len = 1;
246 	} else {
247 		attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
248 		attr->gid_tbl_len = QEDR_MAX_SGID;
249 		attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
250 	}
251 	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
252 	attr->qkey_viol_cntr = 0;
253 	get_link_speed_and_width(rdma_port->link_speed,
254 				 &attr->active_speed, &attr->active_width);
255 	attr->max_msg_sz = rdma_port->max_msg_size;
256 	attr->max_vl_num = 4;
257 
258 	return 0;
259 }
260 
qedr_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)261 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
262 {
263 	struct ib_device *ibdev = uctx->device;
264 	int rc;
265 	struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
266 	struct qedr_alloc_ucontext_resp uresp = {};
267 	struct qedr_alloc_ucontext_req ureq = {};
268 	struct qedr_dev *dev = get_qedr_dev(ibdev);
269 	struct qed_rdma_add_user_out_params oparams;
270 	struct qedr_user_mmap_entry *entry;
271 
272 	if (!udata)
273 		return -EFAULT;
274 
275 	if (udata->inlen) {
276 		rc = ib_copy_from_udata(&ureq, udata,
277 					min(sizeof(ureq), udata->inlen));
278 		if (rc) {
279 			DP_ERR(dev, "Problem copying data from user space\n");
280 			return -EFAULT;
281 		}
282 		ctx->edpm_mode = !!(ureq.context_flags &
283 				    QEDR_ALLOC_UCTX_EDPM_MODE);
284 		ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
285 	}
286 
287 	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
288 	if (rc) {
289 		DP_ERR(dev,
290 		       "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
291 		       rc);
292 		return rc;
293 	}
294 
295 	ctx->dpi = oparams.dpi;
296 	ctx->dpi_addr = oparams.dpi_addr;
297 	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
298 	ctx->dpi_size = oparams.dpi_size;
299 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
300 	if (!entry) {
301 		rc = -ENOMEM;
302 		goto err;
303 	}
304 
305 	entry->io_address = ctx->dpi_phys_addr;
306 	entry->length = ctx->dpi_size;
307 	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
308 	entry->dpi = ctx->dpi;
309 	entry->dev = dev;
310 	rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
311 					 ctx->dpi_size);
312 	if (rc) {
313 		kfree(entry);
314 		goto err;
315 	}
316 	ctx->db_mmap_entry = &entry->rdma_entry;
317 
318 	if (!dev->user_dpm_enabled)
319 		uresp.dpm_flags = 0;
320 	else if (rdma_protocol_iwarp(&dev->ibdev, 1))
321 		uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
322 	else
323 		uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
324 				  QEDR_DPM_TYPE_ROCE_LEGACY |
325 				  QEDR_DPM_TYPE_ROCE_EDPM_MODE;
326 
327 	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
328 		uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
329 		uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
330 		uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
331 		uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
332 	}
333 
334 	uresp.wids_enabled = 1;
335 	uresp.wid_count = oparams.wid_count;
336 	uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
337 	uresp.db_size = ctx->dpi_size;
338 	uresp.max_send_wr = dev->attr.max_sqe;
339 	uresp.max_recv_wr = dev->attr.max_rqe;
340 	uresp.max_srq_wr = dev->attr.max_srq_wr;
341 	uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
342 	uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
343 	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
344 	uresp.max_cqes = QEDR_MAX_CQES;
345 
346 	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
347 	if (rc)
348 		goto err;
349 
350 	ctx->dev = dev;
351 
352 	DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
353 		 &ctx->ibucontext);
354 	return 0;
355 
356 err:
357 	if (!ctx->db_mmap_entry)
358 		dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
359 	else
360 		rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
361 
362 	return rc;
363 }
364 
qedr_dealloc_ucontext(struct ib_ucontext * ibctx)365 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
366 {
367 	struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
368 
369 	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
370 		 uctx);
371 
372 	rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
373 }
374 
qedr_mmap_free(struct rdma_user_mmap_entry * rdma_entry)375 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
376 {
377 	struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
378 	struct qedr_dev *dev = entry->dev;
379 
380 	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
381 		free_page((unsigned long)entry->address);
382 	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
383 		dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
384 
385 	kfree(entry);
386 }
387 
qedr_mmap(struct ib_ucontext * ucontext,struct vm_area_struct * vma)388 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
389 {
390 	struct ib_device *dev = ucontext->device;
391 	size_t length = vma->vm_end - vma->vm_start;
392 	struct rdma_user_mmap_entry *rdma_entry;
393 	struct qedr_user_mmap_entry *entry;
394 	int rc = 0;
395 	u64 pfn;
396 
397 	ibdev_dbg(dev,
398 		  "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
399 		  vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
400 
401 	rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
402 	if (!rdma_entry) {
403 		ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
404 			  vma->vm_pgoff);
405 		return -EINVAL;
406 	}
407 	entry = get_qedr_mmap_entry(rdma_entry);
408 	ibdev_dbg(dev,
409 		  "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
410 		  entry->io_address, length, entry->mmap_flag);
411 
412 	switch (entry->mmap_flag) {
413 	case QEDR_USER_MMAP_IO_WC:
414 		pfn = entry->io_address >> PAGE_SHIFT;
415 		rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
416 				       pgprot_writecombine(vma->vm_page_prot),
417 				       rdma_entry);
418 		break;
419 	case QEDR_USER_MMAP_PHYS_PAGE:
420 		rc = vm_insert_page(vma, vma->vm_start,
421 				    virt_to_page(entry->address));
422 		break;
423 	default:
424 		rc = -EINVAL;
425 	}
426 
427 	if (rc)
428 		ibdev_dbg(dev,
429 			  "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
430 			  entry->io_address, length, entry->mmap_flag, rc);
431 
432 	rdma_user_mmap_entry_put(rdma_entry);
433 	return rc;
434 }
435 
qedr_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)436 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
437 {
438 	struct ib_device *ibdev = ibpd->device;
439 	struct qedr_dev *dev = get_qedr_dev(ibdev);
440 	struct qedr_pd *pd = get_qedr_pd(ibpd);
441 	u16 pd_id;
442 	int rc;
443 
444 	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
445 		 udata ? "User Lib" : "Kernel");
446 
447 	if (!dev->rdma_ctx) {
448 		DP_ERR(dev, "invalid RDMA context\n");
449 		return -EINVAL;
450 	}
451 
452 	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
453 	if (rc)
454 		return rc;
455 
456 	pd->pd_id = pd_id;
457 
458 	if (udata) {
459 		struct qedr_alloc_pd_uresp uresp = {
460 			.pd_id = pd_id,
461 		};
462 		struct qedr_ucontext *context = rdma_udata_to_drv_context(
463 			udata, struct qedr_ucontext, ibucontext);
464 
465 		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
466 		if (rc) {
467 			DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
468 			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
469 			return rc;
470 		}
471 
472 		pd->uctx = context;
473 		pd->uctx->pd = pd;
474 	}
475 
476 	return 0;
477 }
478 
qedr_dealloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)479 int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
480 {
481 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
482 	struct qedr_pd *pd = get_qedr_pd(ibpd);
483 
484 	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
485 	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
486 	return 0;
487 }
488 
489 
qedr_alloc_xrcd(struct ib_xrcd * ibxrcd,struct ib_udata * udata)490 int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
491 {
492 	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
493 	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
494 
495 	return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
496 }
497 
qedr_dealloc_xrcd(struct ib_xrcd * ibxrcd,struct ib_udata * udata)498 int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
499 {
500 	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
501 	u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
502 
503 	dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
504 	return 0;
505 }
qedr_free_pbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,struct qedr_pbl * pbl)506 static void qedr_free_pbl(struct qedr_dev *dev,
507 			  struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
508 {
509 	struct pci_dev *pdev = dev->pdev;
510 	int i;
511 
512 	for (i = 0; i < pbl_info->num_pbls; i++) {
513 		if (!pbl[i].va)
514 			continue;
515 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
516 				  pbl[i].va, pbl[i].pa);
517 	}
518 
519 	kfree(pbl);
520 }
521 
522 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
523 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
524 
525 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
526 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
527 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
528 
qedr_alloc_pbl_tbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,gfp_t flags)529 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
530 					   struct qedr_pbl_info *pbl_info,
531 					   gfp_t flags)
532 {
533 	struct pci_dev *pdev = dev->pdev;
534 	struct qedr_pbl *pbl_table;
535 	dma_addr_t *pbl_main_tbl;
536 	dma_addr_t pa;
537 	void *va;
538 	int i;
539 
540 	pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
541 	if (!pbl_table)
542 		return ERR_PTR(-ENOMEM);
543 
544 	for (i = 0; i < pbl_info->num_pbls; i++) {
545 		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
546 					flags);
547 		if (!va)
548 			goto err;
549 
550 		pbl_table[i].va = va;
551 		pbl_table[i].pa = pa;
552 	}
553 
554 	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
555 	 * the first one with physical pointers to all of the rest
556 	 */
557 	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
558 	for (i = 0; i < pbl_info->num_pbls - 1; i++)
559 		pbl_main_tbl[i] = pbl_table[i + 1].pa;
560 
561 	return pbl_table;
562 
563 err:
564 	for (i--; i >= 0; i--)
565 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
566 				  pbl_table[i].va, pbl_table[i].pa);
567 
568 	qedr_free_pbl(dev, pbl_info, pbl_table);
569 
570 	return ERR_PTR(-ENOMEM);
571 }
572 
qedr_prepare_pbl_tbl(struct qedr_dev * dev,struct qedr_pbl_info * pbl_info,u32 num_pbes,int two_layer_capable)573 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
574 				struct qedr_pbl_info *pbl_info,
575 				u32 num_pbes, int two_layer_capable)
576 {
577 	u32 pbl_capacity;
578 	u32 pbl_size;
579 	u32 num_pbls;
580 
581 	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
582 		if (num_pbes > MAX_PBES_TWO_LAYER) {
583 			DP_ERR(dev, "prepare pbl table: too many pages %d\n",
584 			       num_pbes);
585 			return -EINVAL;
586 		}
587 
588 		/* calculate required pbl page size */
589 		pbl_size = MIN_FW_PBL_PAGE_SIZE;
590 		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
591 			       NUM_PBES_ON_PAGE(pbl_size);
592 
593 		while (pbl_capacity < num_pbes) {
594 			pbl_size *= 2;
595 			pbl_capacity = pbl_size / sizeof(u64);
596 			pbl_capacity = pbl_capacity * pbl_capacity;
597 		}
598 
599 		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
600 		num_pbls++;	/* One for the layer0 ( points to the pbls) */
601 		pbl_info->two_layered = true;
602 	} else {
603 		/* One layered PBL */
604 		num_pbls = 1;
605 		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
606 				 roundup_pow_of_two((num_pbes * sizeof(u64))));
607 		pbl_info->two_layered = false;
608 	}
609 
610 	pbl_info->num_pbls = num_pbls;
611 	pbl_info->pbl_size = pbl_size;
612 	pbl_info->num_pbes = num_pbes;
613 
614 	DP_DEBUG(dev, QEDR_MSG_MR,
615 		 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
616 		 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
617 
618 	return 0;
619 }
620 
qedr_populate_pbls(struct qedr_dev * dev,struct ib_umem * umem,struct qedr_pbl * pbl,struct qedr_pbl_info * pbl_info,u32 pg_shift)621 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
622 			       struct qedr_pbl *pbl,
623 			       struct qedr_pbl_info *pbl_info, u32 pg_shift)
624 {
625 	int pbe_cnt, total_num_pbes = 0;
626 	struct qedr_pbl *pbl_tbl;
627 	struct ib_block_iter biter;
628 	struct regpair *pbe;
629 
630 	if (!pbl_info->num_pbes)
631 		return;
632 
633 	/* If we have a two layered pbl, the first pbl points to the rest
634 	 * of the pbls and the first entry lays on the second pbl in the table
635 	 */
636 	if (pbl_info->two_layered)
637 		pbl_tbl = &pbl[1];
638 	else
639 		pbl_tbl = pbl;
640 
641 	pbe = (struct regpair *)pbl_tbl->va;
642 	if (!pbe) {
643 		DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
644 		return;
645 	}
646 
647 	pbe_cnt = 0;
648 
649 	rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
650 		u64 pg_addr = rdma_block_iter_dma_address(&biter);
651 
652 		pbe->lo = cpu_to_le32(pg_addr);
653 		pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
654 
655 		pbe_cnt++;
656 		total_num_pbes++;
657 		pbe++;
658 
659 		if (total_num_pbes == pbl_info->num_pbes)
660 			return;
661 
662 		/* If the given pbl is full storing the pbes, move to next pbl.
663 		 */
664 		if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
665 			pbl_tbl++;
666 			pbe = (struct regpair *)pbl_tbl->va;
667 			pbe_cnt = 0;
668 		}
669 	}
670 }
671 
qedr_db_recovery_add(struct qedr_dev * dev,void __iomem * db_addr,void * db_data,enum qed_db_rec_width db_width,enum qed_db_rec_space db_space)672 static int qedr_db_recovery_add(struct qedr_dev *dev,
673 				void __iomem *db_addr,
674 				void *db_data,
675 				enum qed_db_rec_width db_width,
676 				enum qed_db_rec_space db_space)
677 {
678 	if (!db_data) {
679 		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
680 		return 0;
681 	}
682 
683 	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
684 						 db_width, db_space);
685 }
686 
qedr_db_recovery_del(struct qedr_dev * dev,void __iomem * db_addr,void * db_data)687 static void qedr_db_recovery_del(struct qedr_dev *dev,
688 				 void __iomem *db_addr,
689 				 void *db_data)
690 {
691 	if (!db_data) {
692 		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
693 		return;
694 	}
695 
696 	/* Ignore return code as there is not much we can do about it. Error
697 	 * log will be printed inside.
698 	 */
699 	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
700 }
701 
qedr_copy_cq_uresp(struct qedr_dev * dev,struct qedr_cq * cq,struct ib_udata * udata,u32 db_offset)702 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
703 			      struct qedr_cq *cq, struct ib_udata *udata,
704 			      u32 db_offset)
705 {
706 	struct qedr_create_cq_uresp uresp;
707 	int rc;
708 
709 	memset(&uresp, 0, sizeof(uresp));
710 
711 	uresp.db_offset = db_offset;
712 	uresp.icid = cq->icid;
713 	if (cq->q.db_mmap_entry)
714 		uresp.db_rec_addr =
715 			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
716 
717 	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
718 	if (rc)
719 		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
720 
721 	return rc;
722 }
723 
consume_cqe(struct qedr_cq * cq)724 static void consume_cqe(struct qedr_cq *cq)
725 {
726 	if (cq->latest_cqe == cq->toggle_cqe)
727 		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
728 
729 	cq->latest_cqe = qed_chain_consume(&cq->pbl);
730 }
731 
qedr_align_cq_entries(int entries)732 static inline int qedr_align_cq_entries(int entries)
733 {
734 	u64 size, aligned_size;
735 
736 	/* We allocate an extra entry that we don't report to the FW. */
737 	size = (entries + 1) * QEDR_CQE_SIZE;
738 	aligned_size = ALIGN(size, PAGE_SIZE);
739 
740 	return aligned_size / QEDR_CQE_SIZE;
741 }
742 
qedr_init_user_db_rec(struct ib_udata * udata,struct qedr_dev * dev,struct qedr_userq * q,bool requires_db_rec)743 static int qedr_init_user_db_rec(struct ib_udata *udata,
744 				 struct qedr_dev *dev, struct qedr_userq *q,
745 				 bool requires_db_rec)
746 {
747 	struct qedr_ucontext *uctx =
748 		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
749 					  ibucontext);
750 	struct qedr_user_mmap_entry *entry;
751 	int rc;
752 
753 	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
754 	if (requires_db_rec == 0 || !uctx->db_rec)
755 		return 0;
756 
757 	/* Allocate a page for doorbell recovery, add to mmap */
758 	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
759 	if (!q->db_rec_data) {
760 		DP_ERR(dev, "get_zeroed_page failed\n");
761 		return -ENOMEM;
762 	}
763 
764 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
765 	if (!entry)
766 		goto err_free_db_data;
767 
768 	entry->address = q->db_rec_data;
769 	entry->length = PAGE_SIZE;
770 	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
771 	rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
772 					 &entry->rdma_entry,
773 					 PAGE_SIZE);
774 	if (rc)
775 		goto err_free_entry;
776 
777 	q->db_mmap_entry = &entry->rdma_entry;
778 
779 	return 0;
780 
781 err_free_entry:
782 	kfree(entry);
783 
784 err_free_db_data:
785 	free_page((unsigned long)q->db_rec_data);
786 	q->db_rec_data = NULL;
787 	return -ENOMEM;
788 }
789 
qedr_init_user_queue(struct ib_udata * udata,struct qedr_dev * dev,struct qedr_userq * q,u64 buf_addr,size_t buf_len,bool requires_db_rec,int access,int alloc_and_init)790 static inline int qedr_init_user_queue(struct ib_udata *udata,
791 				       struct qedr_dev *dev,
792 				       struct qedr_userq *q, u64 buf_addr,
793 				       size_t buf_len, bool requires_db_rec,
794 				       int access,
795 				       int alloc_and_init)
796 {
797 	u32 fw_pages;
798 	int rc;
799 
800 	q->buf_addr = buf_addr;
801 	q->buf_len = buf_len;
802 	q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
803 	if (IS_ERR(q->umem)) {
804 		DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
805 		       PTR_ERR(q->umem));
806 		return PTR_ERR(q->umem);
807 	}
808 
809 	fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
810 	rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
811 	if (rc)
812 		goto err0;
813 
814 	if (alloc_and_init) {
815 		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
816 		if (IS_ERR(q->pbl_tbl)) {
817 			rc = PTR_ERR(q->pbl_tbl);
818 			goto err0;
819 		}
820 		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
821 				   FW_PAGE_SHIFT);
822 	} else {
823 		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
824 		if (!q->pbl_tbl) {
825 			rc = -ENOMEM;
826 			goto err0;
827 		}
828 	}
829 
830 	/* mmap the user address used to store doorbell data for recovery */
831 	return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
832 
833 err0:
834 	ib_umem_release(q->umem);
835 	q->umem = NULL;
836 
837 	return rc;
838 }
839 
qedr_init_cq_params(struct qedr_cq * cq,struct qedr_ucontext * ctx,struct qedr_dev * dev,int vector,int chain_entries,int page_cnt,u64 pbl_ptr,struct qed_rdma_create_cq_in_params * params)840 static inline void qedr_init_cq_params(struct qedr_cq *cq,
841 				       struct qedr_ucontext *ctx,
842 				       struct qedr_dev *dev, int vector,
843 				       int chain_entries, int page_cnt,
844 				       u64 pbl_ptr,
845 				       struct qed_rdma_create_cq_in_params
846 				       *params)
847 {
848 	memset(params, 0, sizeof(*params));
849 	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
850 	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
851 	params->cnq_id = vector;
852 	params->cq_size = chain_entries - 1;
853 	params->dpi = (ctx) ? ctx->dpi : dev->dpi;
854 	params->pbl_num_pages = page_cnt;
855 	params->pbl_ptr = pbl_ptr;
856 	params->pbl_two_level = 0;
857 }
858 
doorbell_cq(struct qedr_cq * cq,u32 cons,u8 flags)859 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
860 {
861 	cq->db.data.agg_flags = flags;
862 	cq->db.data.value = cpu_to_le32(cons);
863 	writeq(cq->db.raw, cq->db_addr);
864 }
865 
qedr_arm_cq(struct ib_cq * ibcq,enum ib_cq_notify_flags flags)866 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
867 {
868 	struct qedr_cq *cq = get_qedr_cq(ibcq);
869 	unsigned long sflags;
870 	struct qedr_dev *dev;
871 
872 	dev = get_qedr_dev(ibcq->device);
873 
874 	if (cq->destroyed) {
875 		DP_ERR(dev,
876 		       "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
877 		       cq, cq->icid);
878 		return -EINVAL;
879 	}
880 
881 
882 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
883 		return 0;
884 
885 	spin_lock_irqsave(&cq->cq_lock, sflags);
886 
887 	cq->arm_flags = 0;
888 
889 	if (flags & IB_CQ_SOLICITED)
890 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
891 
892 	if (flags & IB_CQ_NEXT_COMP)
893 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
894 
895 	doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
896 
897 	spin_unlock_irqrestore(&cq->cq_lock, sflags);
898 
899 	return 0;
900 }
901 
qedr_create_cq(struct ib_cq * ibcq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)902 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
903 		   struct ib_udata *udata)
904 {
905 	struct ib_device *ibdev = ibcq->device;
906 	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
907 		udata, struct qedr_ucontext, ibucontext);
908 	struct qed_rdma_destroy_cq_out_params destroy_oparams;
909 	struct qed_rdma_destroy_cq_in_params destroy_iparams;
910 	struct qed_chain_init_params chain_params = {
911 		.mode		= QED_CHAIN_MODE_PBL,
912 		.intended_use	= QED_CHAIN_USE_TO_CONSUME,
913 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
914 		.elem_size	= sizeof(union rdma_cqe),
915 	};
916 	struct qedr_dev *dev = get_qedr_dev(ibdev);
917 	struct qed_rdma_create_cq_in_params params;
918 	struct qedr_create_cq_ureq ureq = {};
919 	int vector = attr->comp_vector;
920 	int entries = attr->cqe;
921 	struct qedr_cq *cq = get_qedr_cq(ibcq);
922 	int chain_entries;
923 	u32 db_offset;
924 	int page_cnt;
925 	u64 pbl_ptr;
926 	u16 icid;
927 	int rc;
928 
929 	DP_DEBUG(dev, QEDR_MSG_INIT,
930 		 "create_cq: called from %s. entries=%d, vector=%d\n",
931 		 udata ? "User Lib" : "Kernel", entries, vector);
932 
933 	if (attr->flags)
934 		return -EOPNOTSUPP;
935 
936 	if (entries > QEDR_MAX_CQES) {
937 		DP_ERR(dev,
938 		       "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
939 		       entries, QEDR_MAX_CQES);
940 		return -EINVAL;
941 	}
942 
943 	chain_entries = qedr_align_cq_entries(entries);
944 	chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
945 	chain_params.num_elems = chain_entries;
946 
947 	/* calc db offset. user will add DPI base, kernel will add db addr */
948 	db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
949 
950 	if (udata) {
951 		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
952 							 udata->inlen))) {
953 			DP_ERR(dev,
954 			       "create cq: problem copying data from user space\n");
955 			goto err0;
956 		}
957 
958 		if (!ureq.len) {
959 			DP_ERR(dev,
960 			       "create cq: cannot create a cq with 0 entries\n");
961 			goto err0;
962 		}
963 
964 		cq->cq_type = QEDR_CQ_TYPE_USER;
965 
966 		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
967 					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
968 					  1);
969 		if (rc)
970 			goto err0;
971 
972 		pbl_ptr = cq->q.pbl_tbl->pa;
973 		page_cnt = cq->q.pbl_info.num_pbes;
974 
975 		cq->ibcq.cqe = chain_entries;
976 		cq->q.db_addr = ctx->dpi_addr + db_offset;
977 	} else {
978 		cq->cq_type = QEDR_CQ_TYPE_KERNEL;
979 
980 		rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
981 						   &chain_params);
982 		if (rc)
983 			goto err0;
984 
985 		page_cnt = qed_chain_get_page_cnt(&cq->pbl);
986 		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
987 		cq->ibcq.cqe = cq->pbl.capacity;
988 	}
989 
990 	qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
991 			    pbl_ptr, &params);
992 
993 	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
994 	if (rc)
995 		goto err1;
996 
997 	cq->icid = icid;
998 	cq->sig = QEDR_CQ_MAGIC_NUMBER;
999 	spin_lock_init(&cq->cq_lock);
1000 
1001 	if (udata) {
1002 		rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003 		if (rc)
1004 			goto err2;
1005 
1006 		rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007 					  &cq->q.db_rec_data->db_data,
1008 					  DB_REC_WIDTH_64B,
1009 					  DB_REC_USER);
1010 		if (rc)
1011 			goto err2;
1012 
1013 	} else {
1014 		/* Generate doorbell address. */
1015 		cq->db.data.icid = cq->icid;
1016 		cq->db_addr = dev->db_addr + db_offset;
1017 		cq->db.data.params = DB_AGG_CMD_MAX <<
1018 		    RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019 
1020 		/* point to the very last element, passing it we will toggle */
1021 		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022 		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023 		cq->latest_cqe = NULL;
1024 		consume_cqe(cq);
1025 		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1026 
1027 		rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028 					  DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029 		if (rc)
1030 			goto err2;
1031 	}
1032 
1033 	DP_DEBUG(dev, QEDR_MSG_CQ,
1034 		 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035 		 cq->icid, cq, params.cq_size);
1036 
1037 	return 0;
1038 
1039 err2:
1040 	destroy_iparams.icid = cq->icid;
1041 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042 				  &destroy_oparams);
1043 err1:
1044 	if (udata) {
1045 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1046 		ib_umem_release(cq->q.umem);
1047 		if (cq->q.db_mmap_entry)
1048 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049 	} else {
1050 		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051 	}
1052 err0:
1053 	return -EINVAL;
1054 }
1055 
1056 #define QEDR_DESTROY_CQ_MAX_ITERATIONS		(10)
1057 #define QEDR_DESTROY_CQ_ITER_DURATION		(10)
1058 
qedr_destroy_cq(struct ib_cq * ibcq,struct ib_udata * udata)1059 int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060 {
1061 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062 	struct qed_rdma_destroy_cq_out_params oparams;
1063 	struct qed_rdma_destroy_cq_in_params iparams;
1064 	struct qedr_cq *cq = get_qedr_cq(ibcq);
1065 	int iter;
1066 
1067 	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068 
1069 	cq->destroyed = 1;
1070 
1071 	/* GSIs CQs are handled by driver, so they don't exist in the FW */
1072 	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1074 		return 0;
1075 	}
1076 
1077 	iparams.icid = cq->icid;
1078 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079 	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080 
1081 	if (udata) {
1082 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083 		ib_umem_release(cq->q.umem);
1084 
1085 		if (cq->q.db_rec_data) {
1086 			qedr_db_recovery_del(dev, cq->q.db_addr,
1087 					     &cq->q.db_rec_data->db_data);
1088 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089 		}
1090 	} else {
1091 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1092 	}
1093 
1094 	/* We don't want the IRQ handler to handle a non-existing CQ so we
1095 	 * wait until all CNQ interrupts, if any, are received. This will always
1096 	 * happen and will always happen very fast. If not, then a serious error
1097 	 * has occured. That is why we can use a long delay.
1098 	 * We spin for a short time so we don’t lose time on context switching
1099 	 * in case all the completions are handled in that span. Otherwise
1100 	 * we sleep for a while and check again. Since the CNQ may be
1101 	 * associated with (only) the current CPU we use msleep to allow the
1102 	 * current CPU to be freed.
1103 	 * The CNQ notification is increased in qedr_irq_handler().
1104 	 */
1105 	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107 		udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108 		iter--;
1109 	}
1110 
1111 	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113 		msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114 		iter--;
1115 	}
1116 
1117 	/* Note that we don't need to have explicit code to wait for the
1118 	 * completion of the event handler because it is invoked from the EQ.
1119 	 * Since the destroy CQ ramrod has also been received on the EQ we can
1120 	 * be certain that there's no event handler in process.
1121 	 */
1122 	return 0;
1123 }
1124 
get_gid_info_from_table(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct qed_rdma_modify_qp_in_params * qp_params)1125 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126 					  struct ib_qp_attr *attr,
1127 					  int attr_mask,
1128 					  struct qed_rdma_modify_qp_in_params
1129 					  *qp_params)
1130 {
1131 	const struct ib_gid_attr *gid_attr;
1132 	enum rdma_network_type nw_type;
1133 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1134 	u32 ipv4_addr;
1135 	int ret;
1136 	int i;
1137 
1138 	gid_attr = grh->sgid_attr;
1139 	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140 	if (ret)
1141 		return ret;
1142 
1143 	nw_type = rdma_gid_attr_network_type(gid_attr);
1144 	switch (nw_type) {
1145 	case RDMA_NETWORK_IPV6:
1146 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147 		       sizeof(qp_params->sgid));
1148 		memcpy(&qp_params->dgid.bytes[0],
1149 		       &grh->dgid,
1150 		       sizeof(qp_params->dgid));
1151 		qp_params->roce_mode = ROCE_V2_IPV6;
1152 		SET_FIELD(qp_params->modify_flags,
1153 			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154 		break;
1155 	case RDMA_NETWORK_ROCE_V1:
1156 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157 		       sizeof(qp_params->sgid));
1158 		memcpy(&qp_params->dgid.bytes[0],
1159 		       &grh->dgid,
1160 		       sizeof(qp_params->dgid));
1161 		qp_params->roce_mode = ROCE_V1;
1162 		break;
1163 	case RDMA_NETWORK_IPV4:
1164 		memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165 		memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166 		ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1167 		qp_params->sgid.ipv4_addr = ipv4_addr;
1168 		ipv4_addr =
1169 		    qedr_get_ipv4_from_gid(grh->dgid.raw);
1170 		qp_params->dgid.ipv4_addr = ipv4_addr;
1171 		SET_FIELD(qp_params->modify_flags,
1172 			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173 		qp_params->roce_mode = ROCE_V2_IPV4;
1174 		break;
1175 	default:
1176 		return -EINVAL;
1177 	}
1178 
1179 	for (i = 0; i < 4; i++) {
1180 		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181 		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182 	}
1183 
1184 	if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185 		qp_params->vlan_id = 0;
1186 
1187 	return 0;
1188 }
1189 
qedr_check_qp_attrs(struct ib_pd * ibpd,struct qedr_dev * dev,struct ib_qp_init_attr * attrs,struct ib_udata * udata)1190 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191 			       struct ib_qp_init_attr *attrs,
1192 			       struct ib_udata *udata)
1193 {
1194 	struct qedr_device_attr *qattr = &dev->attr;
1195 
1196 	/* QP0... attrs->qp_type == IB_QPT_GSI */
1197 	if (attrs->qp_type != IB_QPT_RC &&
1198 	    attrs->qp_type != IB_QPT_GSI &&
1199 	    attrs->qp_type != IB_QPT_XRC_INI &&
1200 	    attrs->qp_type != IB_QPT_XRC_TGT) {
1201 		DP_DEBUG(dev, QEDR_MSG_QP,
1202 			 "create qp: unsupported qp type=0x%x requested\n",
1203 			 attrs->qp_type);
1204 		return -EOPNOTSUPP;
1205 	}
1206 
1207 	if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208 		DP_ERR(dev,
1209 		       "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210 		       attrs->cap.max_send_wr, qattr->max_sqe);
1211 		return -EINVAL;
1212 	}
1213 
1214 	if (attrs->cap.max_inline_data > qattr->max_inline) {
1215 		DP_ERR(dev,
1216 		       "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217 		       attrs->cap.max_inline_data, qattr->max_inline);
1218 		return -EINVAL;
1219 	}
1220 
1221 	if (attrs->cap.max_send_sge > qattr->max_sge) {
1222 		DP_ERR(dev,
1223 		       "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224 		       attrs->cap.max_send_sge, qattr->max_sge);
1225 		return -EINVAL;
1226 	}
1227 
1228 	if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229 		DP_ERR(dev,
1230 		       "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231 		       attrs->cap.max_recv_sge, qattr->max_sge);
1232 		return -EINVAL;
1233 	}
1234 
1235 	/* verify consumer QPs are not trying to use GSI QP's CQ.
1236 	 * TGT QP isn't associated with RQ/SQ
1237 	 */
1238 	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239 	    (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240 	    (attrs->qp_type != IB_QPT_XRC_INI)) {
1241 		struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242 		struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243 
1244 		if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245 		    (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246 			DP_ERR(dev,
1247 			       "create qp: consumer QP cannot use GSI CQs.\n");
1248 			return -EINVAL;
1249 		}
1250 	}
1251 
1252 	return 0;
1253 }
1254 
qedr_copy_srq_uresp(struct qedr_dev * dev,struct qedr_srq * srq,struct ib_udata * udata)1255 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256 			       struct qedr_srq *srq, struct ib_udata *udata)
1257 {
1258 	struct qedr_create_srq_uresp uresp = {};
1259 	int rc;
1260 
1261 	uresp.srq_id = srq->srq_id;
1262 
1263 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264 	if (rc)
1265 		DP_ERR(dev, "create srq: problem copying data to user space\n");
1266 
1267 	return rc;
1268 }
1269 
qedr_copy_rq_uresp(struct qedr_dev * dev,struct qedr_create_qp_uresp * uresp,struct qedr_qp * qp)1270 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271 			       struct qedr_create_qp_uresp *uresp,
1272 			       struct qedr_qp *qp)
1273 {
1274 	/* iWARP requires two doorbells per RQ. */
1275 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276 		uresp->rq_db_offset =
1277 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278 		uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279 	} else {
1280 		uresp->rq_db_offset =
1281 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282 	}
1283 
1284 	uresp->rq_icid = qp->icid;
1285 	if (qp->urq.db_mmap_entry)
1286 		uresp->rq_db_rec_addr =
1287 			rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1288 }
1289 
qedr_copy_sq_uresp(struct qedr_dev * dev,struct qedr_create_qp_uresp * uresp,struct qedr_qp * qp)1290 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291 			       struct qedr_create_qp_uresp *uresp,
1292 			       struct qedr_qp *qp)
1293 {
1294 	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295 
1296 	/* iWARP uses the same cid for rq and sq */
1297 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298 		uresp->sq_icid = qp->icid;
1299 	else
1300 		uresp->sq_icid = qp->icid + 1;
1301 
1302 	if (qp->usq.db_mmap_entry)
1303 		uresp->sq_db_rec_addr =
1304 			rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1305 }
1306 
qedr_copy_qp_uresp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_udata * udata,struct qedr_create_qp_uresp * uresp)1307 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308 			      struct qedr_qp *qp, struct ib_udata *udata,
1309 			      struct qedr_create_qp_uresp *uresp)
1310 {
1311 	int rc;
1312 
1313 	memset(uresp, 0, sizeof(*uresp));
1314 
1315 	if (qedr_qp_has_sq(qp))
1316 		qedr_copy_sq_uresp(dev, uresp, qp);
1317 
1318 	if (qedr_qp_has_rq(qp))
1319 		qedr_copy_rq_uresp(dev, uresp, qp);
1320 
1321 	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322 	uresp->qp_id = qp->qp_id;
1323 
1324 	rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1325 	if (rc)
1326 		DP_ERR(dev,
1327 		       "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328 		       qp->icid);
1329 
1330 	return rc;
1331 }
1332 
qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info * qph)1333 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334 {
1335 	qed_chain_reset(&qph->pbl);
1336 	qph->prod = 0;
1337 	qph->cons = 0;
1338 	qph->wqe_cons = 0;
1339 	qph->db_data.data.value = cpu_to_le16(0);
1340 }
1341 
qedr_set_common_qp_params(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_pd * pd,struct ib_qp_init_attr * attrs)1342 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343 				      struct qedr_qp *qp,
1344 				      struct qedr_pd *pd,
1345 				      struct ib_qp_init_attr *attrs)
1346 {
1347 	spin_lock_init(&qp->q_lock);
1348 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349 		kref_init(&qp->refcnt);
1350 		init_completion(&qp->iwarp_cm_comp);
1351 		init_completion(&qp->qp_rel_comp);
1352 	}
1353 
1354 	qp->pd = pd;
1355 	qp->qp_type = attrs->qp_type;
1356 	qp->max_inline_data = attrs->cap.max_inline_data;
1357 	qp->state = QED_ROCE_QP_STATE_RESET;
1358 
1359 	qp->prev_wqe_size = 0;
1360 
1361 	qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1362 	qp->dev = dev;
1363 	if (qedr_qp_has_sq(qp)) {
1364 		qedr_reset_qp_hwq_info(&qp->sq);
1365 		qp->sq.max_sges = attrs->cap.max_send_sge;
1366 		qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367 		DP_DEBUG(dev, QEDR_MSG_QP,
1368 			 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369 			 qp->sq.max_sges, qp->sq_cq->icid);
1370 	}
1371 
1372 	if (attrs->srq)
1373 		qp->srq = get_qedr_srq(attrs->srq);
1374 
1375 	if (qedr_qp_has_rq(qp)) {
1376 		qedr_reset_qp_hwq_info(&qp->rq);
1377 		qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378 		qp->rq.max_sges = attrs->cap.max_recv_sge;
1379 		DP_DEBUG(dev, QEDR_MSG_QP,
1380 			 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381 			 qp->rq.max_sges, qp->rq_cq->icid);
1382 	}
1383 
1384 	DP_DEBUG(dev, QEDR_MSG_QP,
1385 		 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386 		 pd->pd_id, qp->qp_type, qp->max_inline_data,
1387 		 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388 	DP_DEBUG(dev, QEDR_MSG_QP,
1389 		 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390 		 qp->sq.max_sges, qp->sq_cq->icid);
1391 }
1392 
qedr_set_roce_db_info(struct qedr_dev * dev,struct qedr_qp * qp)1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394 {
1395 	int rc = 0;
1396 
1397 	if (qedr_qp_has_sq(qp)) {
1398 		qp->sq.db = dev->db_addr +
1399 			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400 		qp->sq.db_data.data.icid = qp->icid + 1;
1401 		rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402 					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403 		if (rc)
1404 			return rc;
1405 	}
1406 
1407 	if (qedr_qp_has_rq(qp)) {
1408 		qp->rq.db = dev->db_addr +
1409 			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410 		qp->rq.db_data.data.icid = qp->icid;
1411 		rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412 					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413 		if (rc && qedr_qp_has_sq(qp))
1414 			qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1415 	}
1416 
1417 	return rc;
1418 }
1419 
qedr_check_srq_params(struct qedr_dev * dev,struct ib_srq_init_attr * attrs,struct ib_udata * udata)1420 static int qedr_check_srq_params(struct qedr_dev *dev,
1421 				 struct ib_srq_init_attr *attrs,
1422 				 struct ib_udata *udata)
1423 {
1424 	struct qedr_device_attr *qattr = &dev->attr;
1425 
1426 	if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427 		DP_ERR(dev,
1428 		       "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429 		       attrs->attr.max_wr, qattr->max_srq_wr);
1430 		return -EINVAL;
1431 	}
1432 
1433 	if (attrs->attr.max_sge > qattr->max_sge) {
1434 		DP_ERR(dev,
1435 		       "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436 		       attrs->attr.max_sge, qattr->max_sge);
1437 	}
1438 
1439 	if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440 		DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441 		return -EINVAL;
1442 	}
1443 
1444 	return 0;
1445 }
1446 
qedr_free_srq_user_params(struct qedr_srq * srq)1447 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448 {
1449 	qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450 	ib_umem_release(srq->usrq.umem);
1451 	ib_umem_release(srq->prod_umem);
1452 }
1453 
qedr_free_srq_kernel_params(struct qedr_srq * srq)1454 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455 {
1456 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457 	struct qedr_dev *dev = srq->dev;
1458 
1459 	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460 
1461 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462 			  hw_srq->virt_prod_pair_addr,
1463 			  hw_srq->phy_prod_pair_addr);
1464 }
1465 
qedr_init_srq_user_params(struct ib_udata * udata,struct qedr_srq * srq,struct qedr_create_srq_ureq * ureq,int access)1466 static int qedr_init_srq_user_params(struct ib_udata *udata,
1467 				     struct qedr_srq *srq,
1468 				     struct qedr_create_srq_ureq *ureq,
1469 				     int access)
1470 {
1471 	struct scatterlist *sg;
1472 	int rc;
1473 
1474 	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1475 				  ureq->srq_len, false, access, 1);
1476 	if (rc)
1477 		return rc;
1478 
1479 	srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480 				     sizeof(struct rdma_srq_producers), access);
1481 	if (IS_ERR(srq->prod_umem)) {
1482 		qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483 		ib_umem_release(srq->usrq.umem);
1484 		DP_ERR(srq->dev,
1485 		       "create srq: failed ib_umem_get for producer, got %ld\n",
1486 		       PTR_ERR(srq->prod_umem));
1487 		return PTR_ERR(srq->prod_umem);
1488 	}
1489 
1490 	sg = srq->prod_umem->sgt_append.sgt.sgl;
1491 	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492 
1493 	return 0;
1494 }
1495 
qedr_alloc_srq_kernel_params(struct qedr_srq * srq,struct qedr_dev * dev,struct ib_srq_init_attr * init_attr)1496 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497 					struct qedr_dev *dev,
1498 					struct ib_srq_init_attr *init_attr)
1499 {
1500 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501 	struct qed_chain_init_params params = {
1502 		.mode		= QED_CHAIN_MODE_PBL,
1503 		.intended_use	= QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1505 		.elem_size	= QEDR_SRQ_WQE_ELEM_SIZE,
1506 	};
1507 	dma_addr_t phy_prod_pair_addr;
1508 	u32 num_elems;
1509 	void *va;
1510 	int rc;
1511 
1512 	va = dma_alloc_coherent(&dev->pdev->dev,
1513 				sizeof(struct rdma_srq_producers),
1514 				&phy_prod_pair_addr, GFP_KERNEL);
1515 	if (!va) {
1516 		DP_ERR(dev,
1517 		       "create srq: failed to allocate dma memory for producer\n");
1518 		return -ENOMEM;
1519 	}
1520 
1521 	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522 	hw_srq->virt_prod_pair_addr = va;
1523 
1524 	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525 	params.num_elems = num_elems;
1526 
1527 	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1528 	if (rc)
1529 		goto err0;
1530 
1531 	hw_srq->num_elems = num_elems;
1532 
1533 	return 0;
1534 
1535 err0:
1536 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537 			  va, phy_prod_pair_addr);
1538 	return rc;
1539 }
1540 
qedr_create_srq(struct ib_srq * ibsrq,struct ib_srq_init_attr * init_attr,struct ib_udata * udata)1541 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542 		    struct ib_udata *udata)
1543 {
1544 	struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545 	struct qed_rdma_create_srq_in_params in_params = {};
1546 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1547 	struct qed_rdma_create_srq_out_params out_params;
1548 	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1549 	struct qedr_create_srq_ureq ureq = {};
1550 	u64 pbl_base_addr, phy_prod_pair_addr;
1551 	struct qedr_srq_hwq_info *hw_srq;
1552 	u32 page_cnt, page_size;
1553 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554 	int rc = 0;
1555 
1556 	DP_DEBUG(dev, QEDR_MSG_QP,
1557 		 "create SRQ called from %s (pd %p)\n",
1558 		 (udata) ? "User lib" : "kernel", pd);
1559 
1560 	if (init_attr->srq_type != IB_SRQT_BASIC &&
1561 	    init_attr->srq_type != IB_SRQT_XRC)
1562 		return -EOPNOTSUPP;
1563 
1564 	rc = qedr_check_srq_params(dev, init_attr, udata);
1565 	if (rc)
1566 		return -EINVAL;
1567 
1568 	srq->dev = dev;
1569 	srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570 	hw_srq = &srq->hw_srq;
1571 	spin_lock_init(&srq->lock);
1572 
1573 	hw_srq->max_wr = init_attr->attr.max_wr;
1574 	hw_srq->max_sges = init_attr->attr.max_sge;
1575 
1576 	if (udata) {
1577 		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578 							 udata->inlen))) {
1579 			DP_ERR(dev,
1580 			       "create srq: problem copying data from user space\n");
1581 			goto err0;
1582 		}
1583 
1584 		rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1585 		if (rc)
1586 			goto err0;
1587 
1588 		page_cnt = srq->usrq.pbl_info.num_pbes;
1589 		pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591 		page_size = PAGE_SIZE;
1592 	} else {
1593 		struct qed_chain *pbl;
1594 
1595 		rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596 		if (rc)
1597 			goto err0;
1598 
1599 		pbl = &hw_srq->pbl;
1600 		page_cnt = qed_chain_get_page_cnt(pbl);
1601 		pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603 		page_size = QED_CHAIN_PAGE_SIZE;
1604 	}
1605 
1606 	in_params.pd_id = pd->pd_id;
1607 	in_params.pbl_base_addr = pbl_base_addr;
1608 	in_params.prod_pair_addr = phy_prod_pair_addr;
1609 	in_params.num_pages = page_cnt;
1610 	in_params.page_size = page_size;
1611 	if (srq->is_xrc) {
1612 		struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613 		struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614 
1615 		in_params.is_xrc = 1;
1616 		in_params.xrcd_id = xrcd->xrcd_id;
1617 		in_params.cq_cid = cq->icid;
1618 	}
1619 
1620 	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621 	if (rc)
1622 		goto err1;
1623 
1624 	srq->srq_id = out_params.srq_id;
1625 
1626 	if (udata) {
1627 		rc = qedr_copy_srq_uresp(dev, srq, udata);
1628 		if (rc)
1629 			goto err2;
1630 	}
1631 
1632 	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1633 	if (rc)
1634 		goto err2;
1635 
1636 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1637 		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638 	return 0;
1639 
1640 err2:
1641 	destroy_in_params.srq_id = srq->srq_id;
1642 
1643 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644 err1:
1645 	if (udata)
1646 		qedr_free_srq_user_params(srq);
1647 	else
1648 		qedr_free_srq_kernel_params(srq);
1649 err0:
1650 	return -EFAULT;
1651 }
1652 
qedr_destroy_srq(struct ib_srq * ibsrq,struct ib_udata * udata)1653 int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654 {
1655 	struct qed_rdma_destroy_srq_in_params in_params = {};
1656 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658 
1659 	xa_erase_irq(&dev->srqs, srq->srq_id);
1660 	in_params.srq_id = srq->srq_id;
1661 	in_params.is_xrc = srq->is_xrc;
1662 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663 
1664 	if (ibsrq->uobject)
1665 		qedr_free_srq_user_params(srq);
1666 	else
1667 		qedr_free_srq_kernel_params(srq);
1668 
1669 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1670 		 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671 		 srq->srq_id);
1672 	return 0;
1673 }
1674 
qedr_modify_srq(struct ib_srq * ibsrq,struct ib_srq_attr * attr,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1675 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676 		    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677 {
1678 	struct qed_rdma_modify_srq_in_params in_params = {};
1679 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681 	int rc;
1682 
1683 	if (attr_mask & IB_SRQ_MAX_WR) {
1684 		DP_ERR(dev,
1685 		       "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686 		       attr_mask, srq);
1687 		return -EINVAL;
1688 	}
1689 
1690 	if (attr_mask & IB_SRQ_LIMIT) {
1691 		if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692 			DP_ERR(dev,
1693 			       "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694 			       attr->srq_limit, srq->hw_srq.max_wr);
1695 			return -EINVAL;
1696 		}
1697 
1698 		in_params.srq_id = srq->srq_id;
1699 		in_params.wqe_limit = attr->srq_limit;
1700 		rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701 		if (rc)
1702 			return rc;
1703 	}
1704 
1705 	srq->srq_limit = attr->srq_limit;
1706 
1707 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1708 		 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709 
1710 	return 0;
1711 }
1712 
qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)1713 static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714 {
1715 	switch (ib_qp_type) {
1716 	case IB_QPT_RC:
1717 		return QED_RDMA_QP_TYPE_RC;
1718 	case IB_QPT_XRC_INI:
1719 		return QED_RDMA_QP_TYPE_XRC_INI;
1720 	case IB_QPT_XRC_TGT:
1721 		return QED_RDMA_QP_TYPE_XRC_TGT;
1722 	default:
1723 		return QED_RDMA_QP_TYPE_INVAL;
1724 	}
1725 }
1726 
1727 static inline void
qedr_init_common_qp_in_params(struct qedr_dev * dev,struct qedr_pd * pd,struct qedr_qp * qp,struct ib_qp_init_attr * attrs,bool fmr_and_reserved_lkey,struct qed_rdma_create_qp_in_params * params)1728 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729 			      struct qedr_pd *pd,
1730 			      struct qedr_qp *qp,
1731 			      struct ib_qp_init_attr *attrs,
1732 			      bool fmr_and_reserved_lkey,
1733 			      struct qed_rdma_create_qp_in_params *params)
1734 {
1735 	/* QP handle to be written in an async event */
1736 	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737 	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738 
1739 	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740 	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741 	params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1742 	params->stats_queue = 0;
1743 
1744 	if (pd) {
1745 		params->pd = pd->pd_id;
1746 		params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747 	}
1748 
1749 	if (qedr_qp_has_sq(qp))
1750 		params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751 
1752 	if (qedr_qp_has_rq(qp))
1753 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754 
1755 	if (qedr_qp_has_srq(qp)) {
1756 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757 		params->srq_id = qp->srq->srq_id;
1758 		params->use_srq = true;
1759 	} else {
1760 		params->srq_id = 0;
1761 		params->use_srq = false;
1762 	}
1763 }
1764 
qedr_qp_user_print(struct qedr_dev * dev,struct qedr_qp * qp)1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766 {
1767 	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768 		 "qp=%p. "
1769 		 "sq_addr=0x%llx, "
1770 		 "sq_len=%zd, "
1771 		 "rq_addr=0x%llx, "
1772 		 "rq_len=%zd"
1773 		 "\n",
1774 		 qp,
1775 		 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776 		 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777 		 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778 		 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779 }
1780 
1781 static inline void
qedr_iwarp_populate_user_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_out_params * out_params)1782 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783 			    struct qedr_qp *qp,
1784 			    struct qed_rdma_create_qp_out_params *out_params)
1785 {
1786 	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787 	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788 
1789 	qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790 			   &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791 	if (!qp->srq) {
1792 		qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793 		qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794 	}
1795 
1796 	qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797 			   &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798 }
1799 
qedr_cleanup_user(struct qedr_dev * dev,struct qedr_ucontext * ctx,struct qedr_qp * qp)1800 static void qedr_cleanup_user(struct qedr_dev *dev,
1801 			      struct qedr_ucontext *ctx,
1802 			      struct qedr_qp *qp)
1803 {
1804 	if (qedr_qp_has_sq(qp)) {
1805 		ib_umem_release(qp->usq.umem);
1806 		qp->usq.umem = NULL;
1807 	}
1808 
1809 	if (qedr_qp_has_rq(qp)) {
1810 		ib_umem_release(qp->urq.umem);
1811 		qp->urq.umem = NULL;
1812 	}
1813 
1814 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815 		qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816 		qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817 	} else {
1818 		kfree(qp->usq.pbl_tbl);
1819 		kfree(qp->urq.pbl_tbl);
1820 	}
1821 
1822 	if (qp->usq.db_rec_data) {
1823 		qedr_db_recovery_del(dev, qp->usq.db_addr,
1824 				     &qp->usq.db_rec_data->db_data);
1825 		rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826 	}
1827 
1828 	if (qp->urq.db_rec_data) {
1829 		qedr_db_recovery_del(dev, qp->urq.db_addr,
1830 				     &qp->urq.db_rec_data->db_data);
1831 		rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832 	}
1833 
1834 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835 		qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836 				     &qp->urq.db_rec_db2_data);
1837 }
1838 
qedr_create_user_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_pd * ibpd,struct ib_udata * udata,struct ib_qp_init_attr * attrs)1839 static int qedr_create_user_qp(struct qedr_dev *dev,
1840 			       struct qedr_qp *qp,
1841 			       struct ib_pd *ibpd,
1842 			       struct ib_udata *udata,
1843 			       struct ib_qp_init_attr *attrs)
1844 {
1845 	struct qed_rdma_create_qp_in_params in_params;
1846 	struct qed_rdma_create_qp_out_params out_params;
1847 	struct qedr_create_qp_uresp uresp = {};
1848 	struct qedr_create_qp_ureq ureq = {};
1849 	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1850 	struct qedr_ucontext *ctx = NULL;
1851 	struct qedr_pd *pd = NULL;
1852 	int rc = 0;
1853 
1854 	qp->create_type = QEDR_QP_CREATE_USER;
1855 
1856 	if (ibpd) {
1857 		pd = get_qedr_pd(ibpd);
1858 		ctx = pd->uctx;
1859 	}
1860 
1861 	if (udata) {
1862 		rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863 					udata->inlen));
1864 		if (rc) {
1865 			DP_ERR(dev, "Problem copying data from user space\n");
1866 			return rc;
1867 		}
1868 	}
1869 
1870 	if (qedr_qp_has_sq(qp)) {
1871 		/* SQ - read access only (0) */
1872 		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873 					  ureq.sq_len, true, 0, alloc_and_init);
1874 		if (rc)
1875 			return rc;
1876 	}
1877 
1878 	if (qedr_qp_has_rq(qp)) {
1879 		/* RQ - read access only (0) */
1880 		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1881 					  ureq.rq_len, true, 0, alloc_and_init);
1882 		if (rc) {
1883 			ib_umem_release(qp->usq.umem);
1884 			qp->usq.umem = NULL;
1885 			if (rdma_protocol_roce(&dev->ibdev, 1)) {
1886 				qedr_free_pbl(dev, &qp->usq.pbl_info,
1887 					      qp->usq.pbl_tbl);
1888 			} else {
1889 				kfree(qp->usq.pbl_tbl);
1890 			}
1891 			return rc;
1892 		}
1893 	}
1894 
1895 	memset(&in_params, 0, sizeof(in_params));
1896 	qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1897 	in_params.qp_handle_lo = ureq.qp_handle_lo;
1898 	in_params.qp_handle_hi = ureq.qp_handle_hi;
1899 
1900 	if (qp->qp_type == IB_QPT_XRC_TGT) {
1901 		struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1902 
1903 		in_params.xrcd_id = xrcd->xrcd_id;
1904 		in_params.qp_handle_lo = qp->qp_id;
1905 		in_params.use_srq = 1;
1906 	}
1907 
1908 	if (qedr_qp_has_sq(qp)) {
1909 		in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1910 		in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1911 	}
1912 
1913 	if (qedr_qp_has_rq(qp)) {
1914 		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1915 		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1916 	}
1917 
1918 	if (ctx)
1919 		SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1920 
1921 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1922 					      &in_params, &out_params);
1923 
1924 	if (!qp->qed_qp) {
1925 		rc = -ENOMEM;
1926 		goto err1;
1927 	}
1928 
1929 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1930 		qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1931 
1932 	qp->qp_id = out_params.qp_id;
1933 	qp->icid = out_params.icid;
1934 
1935 	if (udata) {
1936 		rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1937 		if (rc)
1938 			goto err;
1939 	}
1940 
1941 	/* db offset was calculated in copy_qp_uresp, now set in the user q */
1942 	if (qedr_qp_has_sq(qp)) {
1943 		qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1944 		qp->sq.max_wr = attrs->cap.max_send_wr;
1945 		rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1946 					  &qp->usq.db_rec_data->db_data,
1947 					  DB_REC_WIDTH_32B,
1948 					  DB_REC_USER);
1949 		if (rc)
1950 			goto err;
1951 	}
1952 
1953 	if (qedr_qp_has_rq(qp)) {
1954 		qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1955 		qp->rq.max_wr = attrs->cap.max_recv_wr;
1956 		rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1957 					  &qp->urq.db_rec_data->db_data,
1958 					  DB_REC_WIDTH_32B,
1959 					  DB_REC_USER);
1960 		if (rc)
1961 			goto err;
1962 	}
1963 
1964 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1965 		qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1966 
1967 		/* calculate the db_rec_db2 data since it is constant so no
1968 		 * need to reflect from user
1969 		 */
1970 		qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1971 		qp->urq.db_rec_db2_data.data.value =
1972 			cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1973 
1974 		rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1975 					  &qp->urq.db_rec_db2_data,
1976 					  DB_REC_WIDTH_32B,
1977 					  DB_REC_USER);
1978 		if (rc)
1979 			goto err;
1980 	}
1981 	qedr_qp_user_print(dev, qp);
1982 	return rc;
1983 err:
1984 	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1985 	if (rc)
1986 		DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1987 
1988 err1:
1989 	qedr_cleanup_user(dev, ctx, qp);
1990 	return rc;
1991 }
1992 
qedr_set_iwarp_db_info(struct qedr_dev * dev,struct qedr_qp * qp)1993 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1994 {
1995 	int rc;
1996 
1997 	qp->sq.db = dev->db_addr +
1998 	    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1999 	qp->sq.db_data.data.icid = qp->icid;
2000 
2001 	rc = qedr_db_recovery_add(dev, qp->sq.db,
2002 				  &qp->sq.db_data,
2003 				  DB_REC_WIDTH_32B,
2004 				  DB_REC_KERNEL);
2005 	if (rc)
2006 		return rc;
2007 
2008 	qp->rq.db = dev->db_addr +
2009 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2010 	qp->rq.db_data.data.icid = qp->icid;
2011 	qp->rq.iwarp_db2 = dev->db_addr +
2012 			   DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2013 	qp->rq.iwarp_db2_data.data.icid = qp->icid;
2014 	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2015 
2016 	rc = qedr_db_recovery_add(dev, qp->rq.db,
2017 				  &qp->rq.db_data,
2018 				  DB_REC_WIDTH_32B,
2019 				  DB_REC_KERNEL);
2020 	if (rc)
2021 		return rc;
2022 
2023 	rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2024 				  &qp->rq.iwarp_db2_data,
2025 				  DB_REC_WIDTH_32B,
2026 				  DB_REC_KERNEL);
2027 	return rc;
2028 }
2029 
2030 static int
qedr_roce_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_in_params * in_params,u32 n_sq_elems,u32 n_rq_elems)2031 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2032 			   struct qedr_qp *qp,
2033 			   struct qed_rdma_create_qp_in_params *in_params,
2034 			   u32 n_sq_elems, u32 n_rq_elems)
2035 {
2036 	struct qed_rdma_create_qp_out_params out_params;
2037 	struct qed_chain_init_params params = {
2038 		.mode		= QED_CHAIN_MODE_PBL,
2039 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2040 	};
2041 	int rc;
2042 
2043 	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2044 	params.num_elems = n_sq_elems;
2045 	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2046 
2047 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2048 	if (rc)
2049 		return rc;
2050 
2051 	in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2052 	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2053 
2054 	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2055 	params.num_elems = n_rq_elems;
2056 	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2057 
2058 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2059 	if (rc)
2060 		return rc;
2061 
2062 	in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2063 	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2064 
2065 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2066 					      in_params, &out_params);
2067 
2068 	if (!qp->qed_qp)
2069 		return -EINVAL;
2070 
2071 	qp->qp_id = out_params.qp_id;
2072 	qp->icid = out_params.icid;
2073 
2074 	return qedr_set_roce_db_info(dev, qp);
2075 }
2076 
2077 static int
qedr_iwarp_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct qed_rdma_create_qp_in_params * in_params,u32 n_sq_elems,u32 n_rq_elems)2078 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2079 			    struct qedr_qp *qp,
2080 			    struct qed_rdma_create_qp_in_params *in_params,
2081 			    u32 n_sq_elems, u32 n_rq_elems)
2082 {
2083 	struct qed_rdma_create_qp_out_params out_params;
2084 	struct qed_chain_init_params params = {
2085 		.mode		= QED_CHAIN_MODE_PBL,
2086 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2087 	};
2088 	int rc;
2089 
2090 	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2091 						     QEDR_SQE_ELEMENT_SIZE,
2092 						     QED_CHAIN_PAGE_SIZE,
2093 						     QED_CHAIN_MODE_PBL);
2094 	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2095 						     QEDR_RQE_ELEMENT_SIZE,
2096 						     QED_CHAIN_PAGE_SIZE,
2097 						     QED_CHAIN_MODE_PBL);
2098 
2099 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2100 					      in_params, &out_params);
2101 
2102 	if (!qp->qed_qp)
2103 		return -EINVAL;
2104 
2105 	/* Now we allocate the chain */
2106 
2107 	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2108 	params.num_elems = n_sq_elems;
2109 	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2110 	params.ext_pbl_virt = out_params.sq_pbl_virt;
2111 	params.ext_pbl_phys = out_params.sq_pbl_phys;
2112 
2113 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2114 	if (rc)
2115 		goto err;
2116 
2117 	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2118 	params.num_elems = n_rq_elems;
2119 	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2120 	params.ext_pbl_virt = out_params.rq_pbl_virt;
2121 	params.ext_pbl_phys = out_params.rq_pbl_phys;
2122 
2123 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2124 	if (rc)
2125 		goto err;
2126 
2127 	qp->qp_id = out_params.qp_id;
2128 	qp->icid = out_params.icid;
2129 
2130 	return qedr_set_iwarp_db_info(dev, qp);
2131 
2132 err:
2133 	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2134 
2135 	return rc;
2136 }
2137 
qedr_cleanup_kernel(struct qedr_dev * dev,struct qedr_qp * qp)2138 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2139 {
2140 	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2141 	kfree(qp->wqe_wr_id);
2142 
2143 	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2144 	kfree(qp->rqe_wr_id);
2145 
2146 	/* GSI qp is not registered to db mechanism so no need to delete */
2147 	if (qp->qp_type == IB_QPT_GSI)
2148 		return;
2149 
2150 	qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2151 
2152 	if (!qp->srq) {
2153 		qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2154 
2155 		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2156 			qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2157 					     &qp->rq.iwarp_db2_data);
2158 	}
2159 }
2160 
qedr_create_kernel_qp(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_pd * ibpd,struct ib_qp_init_attr * attrs)2161 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2162 				 struct qedr_qp *qp,
2163 				 struct ib_pd *ibpd,
2164 				 struct ib_qp_init_attr *attrs)
2165 {
2166 	struct qed_rdma_create_qp_in_params in_params;
2167 	struct qedr_pd *pd = get_qedr_pd(ibpd);
2168 	int rc = -EINVAL;
2169 	u32 n_rq_elems;
2170 	u32 n_sq_elems;
2171 	u32 n_sq_entries;
2172 
2173 	memset(&in_params, 0, sizeof(in_params));
2174 	qp->create_type = QEDR_QP_CREATE_KERNEL;
2175 
2176 	/* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2177 	 * the ring. The ring should allow at least a single WR, even if the
2178 	 * user requested none, due to allocation issues.
2179 	 * We should add an extra WR since the prod and cons indices of
2180 	 * wqe_wr_id are managed in such a way that the WQ is considered full
2181 	 * when (prod+1)%max_wr==cons. We currently don't do that because we
2182 	 * double the number of entries due an iSER issue that pushes far more
2183 	 * WRs than indicated. If we decline its ib_post_send() then we get
2184 	 * error prints in the dmesg we'd like to avoid.
2185 	 */
2186 	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2187 			      dev->attr.max_sqe);
2188 
2189 	qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2190 				GFP_KERNEL);
2191 	if (!qp->wqe_wr_id) {
2192 		DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2193 		return -ENOMEM;
2194 	}
2195 
2196 	/* QP handle to be written in CQE */
2197 	in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2198 	in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2199 
2200 	/* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2201 	 * the ring. There ring should allow at least a single WR, even if the
2202 	 * user requested none, due to allocation issues.
2203 	 */
2204 	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2205 
2206 	/* Allocate driver internal RQ array */
2207 	qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2208 				GFP_KERNEL);
2209 	if (!qp->rqe_wr_id) {
2210 		DP_ERR(dev,
2211 		       "create qp: failed RQ shadow memory allocation\n");
2212 		kfree(qp->wqe_wr_id);
2213 		return -ENOMEM;
2214 	}
2215 
2216 	qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2217 
2218 	n_sq_entries = attrs->cap.max_send_wr;
2219 	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2220 	n_sq_entries = max_t(u32, n_sq_entries, 1);
2221 	n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2222 
2223 	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2224 
2225 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2226 		rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2227 						 n_sq_elems, n_rq_elems);
2228 	else
2229 		rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2230 						n_sq_elems, n_rq_elems);
2231 	if (rc)
2232 		qedr_cleanup_kernel(dev, qp);
2233 
2234 	return rc;
2235 }
2236 
qedr_free_qp_resources(struct qedr_dev * dev,struct qedr_qp * qp,struct ib_udata * udata)2237 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2238 				  struct ib_udata *udata)
2239 {
2240 	struct qedr_ucontext *ctx =
2241 		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2242 					  ibucontext);
2243 	int rc;
2244 
2245 	if (qp->qp_type != IB_QPT_GSI) {
2246 		rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2247 		if (rc)
2248 			return rc;
2249 	}
2250 
2251 	if (qp->create_type == QEDR_QP_CREATE_USER)
2252 		qedr_cleanup_user(dev, ctx, qp);
2253 	else
2254 		qedr_cleanup_kernel(dev, qp);
2255 
2256 	return 0;
2257 }
2258 
qedr_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)2259 int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2260 		   struct ib_udata *udata)
2261 {
2262 	struct qedr_xrcd *xrcd = NULL;
2263 	struct ib_pd *ibpd = ibqp->pd;
2264 	struct qedr_pd *pd = get_qedr_pd(ibpd);
2265 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2266 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2267 	int rc = 0;
2268 
2269 	if (attrs->create_flags)
2270 		return -EOPNOTSUPP;
2271 
2272 	if (attrs->qp_type == IB_QPT_XRC_TGT)
2273 		xrcd = get_qedr_xrcd(attrs->xrcd);
2274 	else
2275 		pd = get_qedr_pd(ibpd);
2276 
2277 	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2278 		 udata ? "user library" : "kernel", pd);
2279 
2280 	rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2281 	if (rc)
2282 		return rc;
2283 
2284 	DP_DEBUG(dev, QEDR_MSG_QP,
2285 		 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2286 		 udata ? "user library" : "kernel", attrs->event_handler, pd,
2287 		 get_qedr_cq(attrs->send_cq),
2288 		 get_qedr_cq(attrs->send_cq)->icid,
2289 		 get_qedr_cq(attrs->recv_cq),
2290 		 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2291 
2292 	qedr_set_common_qp_params(dev, qp, pd, attrs);
2293 
2294 	if (attrs->qp_type == IB_QPT_GSI)
2295 		return qedr_create_gsi_qp(dev, attrs, qp);
2296 
2297 	if (udata || xrcd)
2298 		rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2299 	else
2300 		rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2301 
2302 	if (rc)
2303 		return rc;
2304 
2305 	qp->ibqp.qp_num = qp->qp_id;
2306 
2307 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2308 		rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2309 		if (rc)
2310 			goto out_free_qp_resources;
2311 	}
2312 
2313 	return 0;
2314 
2315 out_free_qp_resources:
2316 	qedr_free_qp_resources(dev, qp, udata);
2317 	return -EFAULT;
2318 }
2319 
qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)2320 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2321 {
2322 	switch (qp_state) {
2323 	case QED_ROCE_QP_STATE_RESET:
2324 		return IB_QPS_RESET;
2325 	case QED_ROCE_QP_STATE_INIT:
2326 		return IB_QPS_INIT;
2327 	case QED_ROCE_QP_STATE_RTR:
2328 		return IB_QPS_RTR;
2329 	case QED_ROCE_QP_STATE_RTS:
2330 		return IB_QPS_RTS;
2331 	case QED_ROCE_QP_STATE_SQD:
2332 		return IB_QPS_SQD;
2333 	case QED_ROCE_QP_STATE_ERR:
2334 		return IB_QPS_ERR;
2335 	case QED_ROCE_QP_STATE_SQE:
2336 		return IB_QPS_SQE;
2337 	}
2338 	return IB_QPS_ERR;
2339 }
2340 
qedr_get_state_from_ibqp(enum ib_qp_state qp_state)2341 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2342 					enum ib_qp_state qp_state)
2343 {
2344 	switch (qp_state) {
2345 	case IB_QPS_RESET:
2346 		return QED_ROCE_QP_STATE_RESET;
2347 	case IB_QPS_INIT:
2348 		return QED_ROCE_QP_STATE_INIT;
2349 	case IB_QPS_RTR:
2350 		return QED_ROCE_QP_STATE_RTR;
2351 	case IB_QPS_RTS:
2352 		return QED_ROCE_QP_STATE_RTS;
2353 	case IB_QPS_SQD:
2354 		return QED_ROCE_QP_STATE_SQD;
2355 	case IB_QPS_ERR:
2356 		return QED_ROCE_QP_STATE_ERR;
2357 	default:
2358 		return QED_ROCE_QP_STATE_ERR;
2359 	}
2360 }
2361 
qedr_update_qp_state(struct qedr_dev * dev,struct qedr_qp * qp,enum qed_roce_qp_state cur_state,enum qed_roce_qp_state new_state)2362 static int qedr_update_qp_state(struct qedr_dev *dev,
2363 				struct qedr_qp *qp,
2364 				enum qed_roce_qp_state cur_state,
2365 				enum qed_roce_qp_state new_state)
2366 {
2367 	int status = 0;
2368 
2369 	if (new_state == cur_state)
2370 		return 0;
2371 
2372 	switch (cur_state) {
2373 	case QED_ROCE_QP_STATE_RESET:
2374 		switch (new_state) {
2375 		case QED_ROCE_QP_STATE_INIT:
2376 			break;
2377 		default:
2378 			status = -EINVAL;
2379 			break;
2380 		}
2381 		break;
2382 	case QED_ROCE_QP_STATE_INIT:
2383 		switch (new_state) {
2384 		case QED_ROCE_QP_STATE_RTR:
2385 			/* Update doorbell (in case post_recv was
2386 			 * done before move to RTR)
2387 			 */
2388 
2389 			if (rdma_protocol_roce(&dev->ibdev, 1)) {
2390 				writel(qp->rq.db_data.raw, qp->rq.db);
2391 			}
2392 			break;
2393 		case QED_ROCE_QP_STATE_ERR:
2394 			break;
2395 		default:
2396 			/* Invalid state change. */
2397 			status = -EINVAL;
2398 			break;
2399 		}
2400 		break;
2401 	case QED_ROCE_QP_STATE_RTR:
2402 		/* RTR->XXX */
2403 		switch (new_state) {
2404 		case QED_ROCE_QP_STATE_RTS:
2405 			break;
2406 		case QED_ROCE_QP_STATE_ERR:
2407 			break;
2408 		default:
2409 			/* Invalid state change. */
2410 			status = -EINVAL;
2411 			break;
2412 		}
2413 		break;
2414 	case QED_ROCE_QP_STATE_RTS:
2415 		/* RTS->XXX */
2416 		switch (new_state) {
2417 		case QED_ROCE_QP_STATE_SQD:
2418 			break;
2419 		case QED_ROCE_QP_STATE_ERR:
2420 			break;
2421 		default:
2422 			/* Invalid state change. */
2423 			status = -EINVAL;
2424 			break;
2425 		}
2426 		break;
2427 	case QED_ROCE_QP_STATE_SQD:
2428 		/* SQD->XXX */
2429 		switch (new_state) {
2430 		case QED_ROCE_QP_STATE_RTS:
2431 		case QED_ROCE_QP_STATE_ERR:
2432 			break;
2433 		default:
2434 			/* Invalid state change. */
2435 			status = -EINVAL;
2436 			break;
2437 		}
2438 		break;
2439 	case QED_ROCE_QP_STATE_ERR:
2440 		/* ERR->XXX */
2441 		switch (new_state) {
2442 		case QED_ROCE_QP_STATE_RESET:
2443 			if ((qp->rq.prod != qp->rq.cons) ||
2444 			    (qp->sq.prod != qp->sq.cons)) {
2445 				DP_NOTICE(dev,
2446 					  "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2447 					  qp->rq.prod, qp->rq.cons, qp->sq.prod,
2448 					  qp->sq.cons);
2449 				status = -EINVAL;
2450 			}
2451 			break;
2452 		default:
2453 			status = -EINVAL;
2454 			break;
2455 		}
2456 		break;
2457 	default:
2458 		status = -EINVAL;
2459 		break;
2460 	}
2461 
2462 	return status;
2463 }
2464 
qedr_modify_qp(struct ib_qp * ibqp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)2465 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2466 		   int attr_mask, struct ib_udata *udata)
2467 {
2468 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2469 	struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2470 	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2471 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2472 	enum ib_qp_state old_qp_state, new_qp_state;
2473 	enum qed_roce_qp_state cur_state;
2474 	int rc = 0;
2475 
2476 	DP_DEBUG(dev, QEDR_MSG_QP,
2477 		 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2478 		 attr->qp_state);
2479 
2480 	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2481 		return -EOPNOTSUPP;
2482 
2483 	old_qp_state = qedr_get_ibqp_state(qp->state);
2484 	if (attr_mask & IB_QP_STATE)
2485 		new_qp_state = attr->qp_state;
2486 	else
2487 		new_qp_state = old_qp_state;
2488 
2489 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2490 		if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2491 					ibqp->qp_type, attr_mask)) {
2492 			DP_ERR(dev,
2493 			       "modify qp: invalid attribute mask=0x%x specified for\n"
2494 			       "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2495 			       attr_mask, qp->qp_id, ibqp->qp_type,
2496 			       old_qp_state, new_qp_state);
2497 			rc = -EINVAL;
2498 			goto err;
2499 		}
2500 	}
2501 
2502 	/* Translate the masks... */
2503 	if (attr_mask & IB_QP_STATE) {
2504 		SET_FIELD(qp_params.modify_flags,
2505 			  QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2506 		qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2507 	}
2508 
2509 	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2510 		qp_params.sqd_async = true;
2511 
2512 	if (attr_mask & IB_QP_PKEY_INDEX) {
2513 		SET_FIELD(qp_params.modify_flags,
2514 			  QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2515 		if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2516 			rc = -EINVAL;
2517 			goto err;
2518 		}
2519 
2520 		qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2521 	}
2522 
2523 	if (attr_mask & IB_QP_QKEY)
2524 		qp->qkey = attr->qkey;
2525 
2526 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2527 		SET_FIELD(qp_params.modify_flags,
2528 			  QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2529 		qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2530 						  IB_ACCESS_REMOTE_READ;
2531 		qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2532 						   IB_ACCESS_REMOTE_WRITE;
2533 		qp_params.incoming_atomic_en = attr->qp_access_flags &
2534 					       IB_ACCESS_REMOTE_ATOMIC;
2535 	}
2536 
2537 	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2538 		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2539 			return -EINVAL;
2540 
2541 		if (attr_mask & IB_QP_PATH_MTU) {
2542 			if (attr->path_mtu < IB_MTU_256 ||
2543 			    attr->path_mtu > IB_MTU_4096) {
2544 				pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2545 				rc = -EINVAL;
2546 				goto err;
2547 			}
2548 			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2549 				      ib_mtu_enum_to_int(iboe_get_mtu
2550 							 (dev->ndev->mtu)));
2551 		}
2552 
2553 		if (!qp->mtu) {
2554 			qp->mtu =
2555 			ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2556 			pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2557 		}
2558 
2559 		SET_FIELD(qp_params.modify_flags,
2560 			  QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2561 
2562 		qp_params.traffic_class_tos = grh->traffic_class;
2563 		qp_params.flow_label = grh->flow_label;
2564 		qp_params.hop_limit_ttl = grh->hop_limit;
2565 
2566 		qp->sgid_idx = grh->sgid_index;
2567 
2568 		rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2569 		if (rc) {
2570 			DP_ERR(dev,
2571 			       "modify qp: problems with GID index %d (rc=%d)\n",
2572 			       grh->sgid_index, rc);
2573 			return rc;
2574 		}
2575 
2576 		rc = qedr_get_dmac(dev, &attr->ah_attr,
2577 				   qp_params.remote_mac_addr);
2578 		if (rc)
2579 			return rc;
2580 
2581 		qp_params.use_local_mac = true;
2582 		ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2583 
2584 		DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2585 			 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2586 			 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2587 		DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2588 			 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2589 			 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2590 		DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2591 			 qp_params.remote_mac_addr);
2592 
2593 		qp_params.mtu = qp->mtu;
2594 		qp_params.lb_indication = false;
2595 	}
2596 
2597 	if (!qp_params.mtu) {
2598 		/* Stay with current MTU */
2599 		if (qp->mtu)
2600 			qp_params.mtu = qp->mtu;
2601 		else
2602 			qp_params.mtu =
2603 			    ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2604 	}
2605 
2606 	if (attr_mask & IB_QP_TIMEOUT) {
2607 		SET_FIELD(qp_params.modify_flags,
2608 			  QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2609 
2610 		/* The received timeout value is an exponent used like this:
2611 		 *    "12.7.34 LOCAL ACK TIMEOUT
2612 		 *    Value representing the transport (ACK) timeout for use by
2613 		 *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2614 		 * The FW expects timeout in msec so we need to divide the usec
2615 		 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2616 		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2617 		 * The value of zero means infinite so we use a 'max_t' to make
2618 		 * sure that sub 1 msec values will be configured as 1 msec.
2619 		 */
2620 		if (attr->timeout)
2621 			qp_params.ack_timeout =
2622 					1 << max_t(int, attr->timeout - 8, 0);
2623 		else
2624 			qp_params.ack_timeout = 0;
2625 
2626 		qp->timeout = attr->timeout;
2627 	}
2628 
2629 	if (attr_mask & IB_QP_RETRY_CNT) {
2630 		SET_FIELD(qp_params.modify_flags,
2631 			  QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2632 		qp_params.retry_cnt = attr->retry_cnt;
2633 	}
2634 
2635 	if (attr_mask & IB_QP_RNR_RETRY) {
2636 		SET_FIELD(qp_params.modify_flags,
2637 			  QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2638 		qp_params.rnr_retry_cnt = attr->rnr_retry;
2639 	}
2640 
2641 	if (attr_mask & IB_QP_RQ_PSN) {
2642 		SET_FIELD(qp_params.modify_flags,
2643 			  QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2644 		qp_params.rq_psn = attr->rq_psn;
2645 		qp->rq_psn = attr->rq_psn;
2646 	}
2647 
2648 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2649 		if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2650 			rc = -EINVAL;
2651 			DP_ERR(dev,
2652 			       "unsupported max_rd_atomic=%d, supported=%d\n",
2653 			       attr->max_rd_atomic,
2654 			       dev->attr.max_qp_req_rd_atomic_resc);
2655 			goto err;
2656 		}
2657 
2658 		SET_FIELD(qp_params.modify_flags,
2659 			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2660 		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2661 	}
2662 
2663 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2664 		SET_FIELD(qp_params.modify_flags,
2665 			  QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2666 		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2667 	}
2668 
2669 	if (attr_mask & IB_QP_SQ_PSN) {
2670 		SET_FIELD(qp_params.modify_flags,
2671 			  QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2672 		qp_params.sq_psn = attr->sq_psn;
2673 		qp->sq_psn = attr->sq_psn;
2674 	}
2675 
2676 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2677 		if (attr->max_dest_rd_atomic >
2678 		    dev->attr.max_qp_resp_rd_atomic_resc) {
2679 			DP_ERR(dev,
2680 			       "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2681 			       attr->max_dest_rd_atomic,
2682 			       dev->attr.max_qp_resp_rd_atomic_resc);
2683 
2684 			rc = -EINVAL;
2685 			goto err;
2686 		}
2687 
2688 		SET_FIELD(qp_params.modify_flags,
2689 			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2690 		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2691 	}
2692 
2693 	if (attr_mask & IB_QP_DEST_QPN) {
2694 		SET_FIELD(qp_params.modify_flags,
2695 			  QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2696 
2697 		qp_params.dest_qp = attr->dest_qp_num;
2698 		qp->dest_qp_num = attr->dest_qp_num;
2699 	}
2700 
2701 	cur_state = qp->state;
2702 
2703 	/* Update the QP state before the actual ramrod to prevent a race with
2704 	 * fast path. Modifying the QP state to error will cause the device to
2705 	 * flush the CQEs and while polling the flushed CQEs will considered as
2706 	 * a potential issue if the QP isn't in error state.
2707 	 */
2708 	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2709 	    !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2710 		qp->state = QED_ROCE_QP_STATE_ERR;
2711 
2712 	if (qp->qp_type != IB_QPT_GSI)
2713 		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2714 					      qp->qed_qp, &qp_params);
2715 
2716 	if (attr_mask & IB_QP_STATE) {
2717 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2718 			rc = qedr_update_qp_state(dev, qp, cur_state,
2719 						  qp_params.new_state);
2720 		qp->state = qp_params.new_state;
2721 	}
2722 
2723 err:
2724 	return rc;
2725 }
2726 
qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params * params)2727 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2728 {
2729 	int ib_qp_acc_flags = 0;
2730 
2731 	if (params->incoming_rdma_write_en)
2732 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2733 	if (params->incoming_rdma_read_en)
2734 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2735 	if (params->incoming_atomic_en)
2736 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2737 	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2738 	return ib_qp_acc_flags;
2739 }
2740 
qedr_query_qp(struct ib_qp * ibqp,struct ib_qp_attr * qp_attr,int attr_mask,struct ib_qp_init_attr * qp_init_attr)2741 int qedr_query_qp(struct ib_qp *ibqp,
2742 		  struct ib_qp_attr *qp_attr,
2743 		  int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2744 {
2745 	struct qed_rdma_query_qp_out_params params;
2746 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2747 	struct qedr_dev *dev = qp->dev;
2748 	int rc = 0;
2749 
2750 	memset(&params, 0, sizeof(params));
2751 	memset(qp_attr, 0, sizeof(*qp_attr));
2752 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2753 
2754 	if (qp->qp_type != IB_QPT_GSI) {
2755 		rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2756 		if (rc)
2757 			goto err;
2758 		qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2759 	} else {
2760 		qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2761 	}
2762 
2763 	qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2764 	qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2765 	qp_attr->path_mig_state = IB_MIG_MIGRATED;
2766 	qp_attr->rq_psn = params.rq_psn;
2767 	qp_attr->sq_psn = params.sq_psn;
2768 	qp_attr->dest_qp_num = params.dest_qp;
2769 
2770 	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2771 
2772 	qp_attr->cap.max_send_wr = qp->sq.max_wr;
2773 	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2774 	qp_attr->cap.max_send_sge = qp->sq.max_sges;
2775 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2776 	qp_attr->cap.max_inline_data = dev->attr.max_inline;
2777 	qp_init_attr->cap = qp_attr->cap;
2778 
2779 	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2780 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2781 			params.flow_label, qp->sgid_idx,
2782 			params.hop_limit_ttl, params.traffic_class_tos);
2783 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2784 	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2785 	rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2786 	qp_attr->timeout = qp->timeout;
2787 	qp_attr->rnr_retry = params.rnr_retry;
2788 	qp_attr->retry_cnt = params.retry_cnt;
2789 	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2790 	qp_attr->pkey_index = params.pkey_index;
2791 	qp_attr->port_num = 1;
2792 	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2793 	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2794 	qp_attr->alt_pkey_index = 0;
2795 	qp_attr->alt_port_num = 0;
2796 	qp_attr->alt_timeout = 0;
2797 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2798 
2799 	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2800 	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2801 	qp_attr->max_rd_atomic = params.max_rd_atomic;
2802 	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2803 
2804 	DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2805 		 qp_attr->cap.max_inline_data);
2806 
2807 err:
2808 	return rc;
2809 }
2810 
qedr_destroy_qp(struct ib_qp * ibqp,struct ib_udata * udata)2811 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2812 {
2813 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2814 	struct qedr_dev *dev = qp->dev;
2815 	struct ib_qp_attr attr;
2816 	int attr_mask = 0;
2817 
2818 	DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2819 		 qp, qp->qp_type);
2820 
2821 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2822 		if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2823 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
2824 		    (qp->state != QED_ROCE_QP_STATE_INIT)) {
2825 
2826 			attr.qp_state = IB_QPS_ERR;
2827 			attr_mask |= IB_QP_STATE;
2828 
2829 			/* Change the QP state to ERROR */
2830 			qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2831 		}
2832 	} else {
2833 		/* If connection establishment started the WAIT_FOR_CONNECT
2834 		 * bit will be on and we need to Wait for the establishment
2835 		 * to complete before destroying the qp.
2836 		 */
2837 		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2838 				     &qp->iwarp_cm_flags))
2839 			wait_for_completion(&qp->iwarp_cm_comp);
2840 
2841 		/* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2842 		 * bit will be on, and we need to wait for the disconnect to
2843 		 * complete before continuing. We can use the same completion,
2844 		 * iwarp_cm_comp, since this is the only place that waits for
2845 		 * this completion and it is sequential. In addition,
2846 		 * disconnect can't occur before the connection is fully
2847 		 * established, therefore if WAIT_FOR_DISCONNECT is on it
2848 		 * means WAIT_FOR_CONNECT is also on and the completion for
2849 		 * CONNECT already occurred.
2850 		 */
2851 		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2852 				     &qp->iwarp_cm_flags))
2853 			wait_for_completion(&qp->iwarp_cm_comp);
2854 	}
2855 
2856 	if (qp->qp_type == IB_QPT_GSI)
2857 		qedr_destroy_gsi_qp(dev);
2858 
2859 	/* We need to remove the entry from the xarray before we release the
2860 	 * qp_id to avoid a race of the qp_id being reallocated and failing
2861 	 * on xa_insert
2862 	 */
2863 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2864 		xa_erase(&dev->qps, qp->qp_id);
2865 
2866 	qedr_free_qp_resources(dev, qp, udata);
2867 
2868 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2869 		qedr_iw_qp_rem_ref(&qp->ibqp);
2870 		wait_for_completion(&qp->qp_rel_comp);
2871 	}
2872 
2873 	return 0;
2874 }
2875 
qedr_create_ah(struct ib_ah * ibah,struct rdma_ah_init_attr * init_attr,struct ib_udata * udata)2876 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2877 		   struct ib_udata *udata)
2878 {
2879 	struct qedr_ah *ah = get_qedr_ah(ibah);
2880 
2881 	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2882 
2883 	return 0;
2884 }
2885 
qedr_destroy_ah(struct ib_ah * ibah,u32 flags)2886 int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2887 {
2888 	struct qedr_ah *ah = get_qedr_ah(ibah);
2889 
2890 	rdma_destroy_ah_attr(&ah->attr);
2891 	return 0;
2892 }
2893 
free_mr_info(struct qedr_dev * dev,struct mr_info * info)2894 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2895 {
2896 	struct qedr_pbl *pbl, *tmp;
2897 
2898 	if (info->pbl_table)
2899 		list_add_tail(&info->pbl_table->list_entry,
2900 			      &info->free_pbl_list);
2901 
2902 	if (!list_empty(&info->inuse_pbl_list))
2903 		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2904 
2905 	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2906 		list_del(&pbl->list_entry);
2907 		qedr_free_pbl(dev, &info->pbl_info, pbl);
2908 	}
2909 }
2910 
init_mr_info(struct qedr_dev * dev,struct mr_info * info,size_t page_list_len,bool two_layered)2911 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2912 			size_t page_list_len, bool two_layered)
2913 {
2914 	struct qedr_pbl *tmp;
2915 	int rc;
2916 
2917 	INIT_LIST_HEAD(&info->free_pbl_list);
2918 	INIT_LIST_HEAD(&info->inuse_pbl_list);
2919 
2920 	rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2921 				  page_list_len, two_layered);
2922 	if (rc)
2923 		goto done;
2924 
2925 	info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2926 	if (IS_ERR(info->pbl_table)) {
2927 		rc = PTR_ERR(info->pbl_table);
2928 		goto done;
2929 	}
2930 
2931 	DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2932 		 &info->pbl_table->pa);
2933 
2934 	/* in usual case we use 2 PBLs, so we add one to free
2935 	 * list and allocating another one
2936 	 */
2937 	tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2938 	if (IS_ERR(tmp)) {
2939 		DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2940 		goto done;
2941 	}
2942 
2943 	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2944 
2945 	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2946 
2947 done:
2948 	if (rc)
2949 		free_mr_info(dev, info);
2950 
2951 	return rc;
2952 }
2953 
qedr_reg_user_mr(struct ib_pd * ibpd,u64 start,u64 len,u64 usr_addr,int acc,struct ib_udata * udata)2954 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2955 			       u64 usr_addr, int acc, struct ib_udata *udata)
2956 {
2957 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2958 	struct qedr_mr *mr;
2959 	struct qedr_pd *pd;
2960 	int rc = -ENOMEM;
2961 
2962 	pd = get_qedr_pd(ibpd);
2963 	DP_DEBUG(dev, QEDR_MSG_MR,
2964 		 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2965 		 pd->pd_id, start, len, usr_addr, acc);
2966 
2967 	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2968 		return ERR_PTR(-EINVAL);
2969 
2970 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2971 	if (!mr)
2972 		return ERR_PTR(rc);
2973 
2974 	mr->type = QEDR_MR_USER;
2975 
2976 	mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2977 	if (IS_ERR(mr->umem)) {
2978 		rc = -EFAULT;
2979 		goto err0;
2980 	}
2981 
2982 	rc = init_mr_info(dev, &mr->info,
2983 			  ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2984 	if (rc)
2985 		goto err1;
2986 
2987 	qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2988 			   &mr->info.pbl_info, PAGE_SHIFT);
2989 
2990 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2991 	if (rc) {
2992 		if (rc == -EINVAL)
2993 			DP_ERR(dev, "Out of MR resources\n");
2994 		else
2995 			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2996 
2997 		goto err1;
2998 	}
2999 
3000 	/* Index only, 18 bit long, lkey = itid << 8 | key */
3001 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3002 	mr->hw_mr.key = 0;
3003 	mr->hw_mr.pd = pd->pd_id;
3004 	mr->hw_mr.local_read = 1;
3005 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3006 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3007 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3008 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3009 	mr->hw_mr.mw_bind = false;
3010 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3011 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3012 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3013 	mr->hw_mr.page_size_log = PAGE_SHIFT;
3014 	mr->hw_mr.length = len;
3015 	mr->hw_mr.vaddr = usr_addr;
3016 	mr->hw_mr.phy_mr = false;
3017 	mr->hw_mr.dma_mr = false;
3018 
3019 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3020 	if (rc) {
3021 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3022 		goto err2;
3023 	}
3024 
3025 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3026 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3027 	    mr->hw_mr.remote_atomic)
3028 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3029 
3030 	DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3031 		 mr->ibmr.lkey);
3032 	return &mr->ibmr;
3033 
3034 err2:
3035 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3036 err1:
3037 	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3038 err0:
3039 	kfree(mr);
3040 	return ERR_PTR(rc);
3041 }
3042 
qedr_dereg_mr(struct ib_mr * ib_mr,struct ib_udata * udata)3043 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3044 {
3045 	struct qedr_mr *mr = get_qedr_mr(ib_mr);
3046 	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3047 	int rc = 0;
3048 
3049 	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3050 	if (rc)
3051 		return rc;
3052 
3053 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3054 
3055 	if (mr->type != QEDR_MR_DMA)
3056 		free_mr_info(dev, &mr->info);
3057 
3058 	/* it could be user registered memory. */
3059 	ib_umem_release(mr->umem);
3060 
3061 	kfree(mr);
3062 
3063 	return rc;
3064 }
3065 
__qedr_alloc_mr(struct ib_pd * ibpd,int max_page_list_len)3066 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3067 				       int max_page_list_len)
3068 {
3069 	struct qedr_pd *pd = get_qedr_pd(ibpd);
3070 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3071 	struct qedr_mr *mr;
3072 	int rc = -ENOMEM;
3073 
3074 	DP_DEBUG(dev, QEDR_MSG_MR,
3075 		 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3076 		 max_page_list_len);
3077 
3078 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3079 	if (!mr)
3080 		return ERR_PTR(rc);
3081 
3082 	mr->dev = dev;
3083 	mr->type = QEDR_MR_FRMR;
3084 
3085 	rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3086 	if (rc)
3087 		goto err0;
3088 
3089 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3090 	if (rc) {
3091 		if (rc == -EINVAL)
3092 			DP_ERR(dev, "Out of MR resources\n");
3093 		else
3094 			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3095 
3096 		goto err1;
3097 	}
3098 
3099 	/* Index only, 18 bit long, lkey = itid << 8 | key */
3100 	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3101 	mr->hw_mr.key = 0;
3102 	mr->hw_mr.pd = pd->pd_id;
3103 	mr->hw_mr.local_read = 1;
3104 	mr->hw_mr.local_write = 0;
3105 	mr->hw_mr.remote_read = 0;
3106 	mr->hw_mr.remote_write = 0;
3107 	mr->hw_mr.remote_atomic = 0;
3108 	mr->hw_mr.mw_bind = false;
3109 	mr->hw_mr.pbl_ptr = 0;
3110 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3111 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3112 	mr->hw_mr.length = 0;
3113 	mr->hw_mr.vaddr = 0;
3114 	mr->hw_mr.phy_mr = true;
3115 	mr->hw_mr.dma_mr = false;
3116 
3117 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3118 	if (rc) {
3119 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3120 		goto err2;
3121 	}
3122 
3123 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3124 	mr->ibmr.rkey = mr->ibmr.lkey;
3125 
3126 	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3127 	return mr;
3128 
3129 err2:
3130 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3131 err1:
3132 	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3133 err0:
3134 	kfree(mr);
3135 	return ERR_PTR(rc);
3136 }
3137 
qedr_alloc_mr(struct ib_pd * ibpd,enum ib_mr_type mr_type,u32 max_num_sg)3138 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3139 			    u32 max_num_sg)
3140 {
3141 	struct qedr_mr *mr;
3142 
3143 	if (mr_type != IB_MR_TYPE_MEM_REG)
3144 		return ERR_PTR(-EINVAL);
3145 
3146 	mr = __qedr_alloc_mr(ibpd, max_num_sg);
3147 
3148 	if (IS_ERR(mr))
3149 		return ERR_PTR(-EINVAL);
3150 
3151 	return &mr->ibmr;
3152 }
3153 
qedr_set_page(struct ib_mr * ibmr,u64 addr)3154 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3155 {
3156 	struct qedr_mr *mr = get_qedr_mr(ibmr);
3157 	struct qedr_pbl *pbl_table;
3158 	struct regpair *pbe;
3159 	u32 pbes_in_page;
3160 
3161 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3162 		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3163 		return -ENOMEM;
3164 	}
3165 
3166 	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3167 		 mr->npages, addr);
3168 
3169 	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3170 	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3171 	pbe = (struct regpair *)pbl_table->va;
3172 	pbe +=  mr->npages % pbes_in_page;
3173 	pbe->lo = cpu_to_le32((u32)addr);
3174 	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3175 
3176 	mr->npages++;
3177 
3178 	return 0;
3179 }
3180 
handle_completed_mrs(struct qedr_dev * dev,struct mr_info * info)3181 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3182 {
3183 	int work = info->completed - info->completed_handled - 1;
3184 
3185 	DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3186 	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3187 		struct qedr_pbl *pbl;
3188 
3189 		/* Free all the page list that are possible to be freed
3190 		 * (all the ones that were invalidated), under the assumption
3191 		 * that if an FMR was completed successfully that means that
3192 		 * if there was an invalidate operation before it also ended
3193 		 */
3194 		pbl = list_first_entry(&info->inuse_pbl_list,
3195 				       struct qedr_pbl, list_entry);
3196 		list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3197 		info->completed_handled++;
3198 	}
3199 }
3200 
qedr_map_mr_sg(struct ib_mr * ibmr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset)3201 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3202 		   int sg_nents, unsigned int *sg_offset)
3203 {
3204 	struct qedr_mr *mr = get_qedr_mr(ibmr);
3205 
3206 	mr->npages = 0;
3207 
3208 	handle_completed_mrs(mr->dev, &mr->info);
3209 	return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3210 }
3211 
qedr_get_dma_mr(struct ib_pd * ibpd,int acc)3212 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3213 {
3214 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3215 	struct qedr_pd *pd = get_qedr_pd(ibpd);
3216 	struct qedr_mr *mr;
3217 	int rc;
3218 
3219 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3220 	if (!mr)
3221 		return ERR_PTR(-ENOMEM);
3222 
3223 	mr->type = QEDR_MR_DMA;
3224 
3225 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3226 	if (rc) {
3227 		if (rc == -EINVAL)
3228 			DP_ERR(dev, "Out of MR resources\n");
3229 		else
3230 			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3231 
3232 		goto err1;
3233 	}
3234 
3235 	/* index only, 18 bit long, lkey = itid << 8 | key */
3236 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3237 	mr->hw_mr.pd = pd->pd_id;
3238 	mr->hw_mr.local_read = 1;
3239 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3240 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3241 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3242 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3243 	mr->hw_mr.dma_mr = true;
3244 
3245 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3246 	if (rc) {
3247 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3248 		goto err2;
3249 	}
3250 
3251 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3252 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3253 	    mr->hw_mr.remote_atomic)
3254 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3255 
3256 	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3257 	return &mr->ibmr;
3258 
3259 err2:
3260 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3261 err1:
3262 	kfree(mr);
3263 	return ERR_PTR(rc);
3264 }
3265 
qedr_wq_is_full(struct qedr_qp_hwq_info * wq)3266 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3267 {
3268 	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3269 }
3270 
sge_data_len(struct ib_sge * sg_list,int num_sge)3271 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3272 {
3273 	int i, len = 0;
3274 
3275 	for (i = 0; i < num_sge; i++)
3276 		len += sg_list[i].length;
3277 
3278 	return len;
3279 }
3280 
swap_wqe_data64(u64 * p)3281 static void swap_wqe_data64(u64 *p)
3282 {
3283 	int i;
3284 
3285 	for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3286 		*p = cpu_to_be64(cpu_to_le64(*p));
3287 }
3288 
qedr_prepare_sq_inline_data(struct qedr_dev * dev,struct qedr_qp * qp,u8 * wqe_size,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr,u8 * bits,u8 bit)3289 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3290 				       struct qedr_qp *qp, u8 *wqe_size,
3291 				       const struct ib_send_wr *wr,
3292 				       const struct ib_send_wr **bad_wr,
3293 				       u8 *bits, u8 bit)
3294 {
3295 	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3296 	char *seg_prt, *wqe;
3297 	int i, seg_siz;
3298 
3299 	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3300 		DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3301 		*bad_wr = wr;
3302 		return 0;
3303 	}
3304 
3305 	if (!data_size)
3306 		return data_size;
3307 
3308 	*bits |= bit;
3309 
3310 	seg_prt = NULL;
3311 	wqe = NULL;
3312 	seg_siz = 0;
3313 
3314 	/* Copy data inline */
3315 	for (i = 0; i < wr->num_sge; i++) {
3316 		u32 len = wr->sg_list[i].length;
3317 		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3318 
3319 		while (len > 0) {
3320 			u32 cur;
3321 
3322 			/* New segment required */
3323 			if (!seg_siz) {
3324 				wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3325 				seg_prt = wqe;
3326 				seg_siz = sizeof(struct rdma_sq_common_wqe);
3327 				(*wqe_size)++;
3328 			}
3329 
3330 			/* Calculate currently allowed length */
3331 			cur = min_t(u32, len, seg_siz);
3332 			memcpy(seg_prt, src, cur);
3333 
3334 			/* Update segment variables */
3335 			seg_prt += cur;
3336 			seg_siz -= cur;
3337 
3338 			/* Update sge variables */
3339 			src += cur;
3340 			len -= cur;
3341 
3342 			/* Swap fully-completed segments */
3343 			if (!seg_siz)
3344 				swap_wqe_data64((u64 *)wqe);
3345 		}
3346 	}
3347 
3348 	/* swap last not completed segment */
3349 	if (seg_siz)
3350 		swap_wqe_data64((u64 *)wqe);
3351 
3352 	return data_size;
3353 }
3354 
3355 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
3356 	do {							\
3357 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3358 		(sge)->length = cpu_to_le32(vlength);		\
3359 		(sge)->flags = cpu_to_le32(vflags);		\
3360 	} while (0)
3361 
3362 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
3363 	do {							\
3364 		DMA_REGPAIR_LE(hdr->wr_id, vwr_id);		\
3365 		(hdr)->num_sges = num_sge;			\
3366 	} while (0)
3367 
3368 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
3369 	do {							\
3370 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3371 		(sge)->length = cpu_to_le32(vlength);		\
3372 		(sge)->l_key = cpu_to_le32(vlkey);		\
3373 	} while (0)
3374 
qedr_prepare_sq_sges(struct qedr_qp * qp,u8 * wqe_size,const struct ib_send_wr * wr)3375 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3376 				const struct ib_send_wr *wr)
3377 {
3378 	u32 data_size = 0;
3379 	int i;
3380 
3381 	for (i = 0; i < wr->num_sge; i++) {
3382 		struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3383 
3384 		DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3385 		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3386 		sge->length = cpu_to_le32(wr->sg_list[i].length);
3387 		data_size += wr->sg_list[i].length;
3388 	}
3389 
3390 	if (wqe_size)
3391 		*wqe_size += wr->num_sge;
3392 
3393 	return data_size;
3394 }
3395 
qedr_prepare_sq_rdma_data(struct qedr_dev * dev,struct qedr_qp * qp,struct rdma_sq_rdma_wqe_1st * rwqe,struct rdma_sq_rdma_wqe_2nd * rwqe2,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3396 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3397 				     struct qedr_qp *qp,
3398 				     struct rdma_sq_rdma_wqe_1st *rwqe,
3399 				     struct rdma_sq_rdma_wqe_2nd *rwqe2,
3400 				     const struct ib_send_wr *wr,
3401 				     const struct ib_send_wr **bad_wr)
3402 {
3403 	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3404 	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3405 
3406 	if (wr->send_flags & IB_SEND_INLINE &&
3407 	    (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3408 	     wr->opcode == IB_WR_RDMA_WRITE)) {
3409 		u8 flags = 0;
3410 
3411 		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3412 		return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3413 						   bad_wr, &rwqe->flags, flags);
3414 	}
3415 
3416 	return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3417 }
3418 
qedr_prepare_sq_send_data(struct qedr_dev * dev,struct qedr_qp * qp,struct rdma_sq_send_wqe_1st * swqe,struct rdma_sq_send_wqe_2st * swqe2,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3419 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3420 				     struct qedr_qp *qp,
3421 				     struct rdma_sq_send_wqe_1st *swqe,
3422 				     struct rdma_sq_send_wqe_2st *swqe2,
3423 				     const struct ib_send_wr *wr,
3424 				     const struct ib_send_wr **bad_wr)
3425 {
3426 	memset(swqe2, 0, sizeof(*swqe2));
3427 	if (wr->send_flags & IB_SEND_INLINE) {
3428 		u8 flags = 0;
3429 
3430 		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3431 		return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3432 						   bad_wr, &swqe->flags, flags);
3433 	}
3434 
3435 	return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3436 }
3437 
qedr_prepare_reg(struct qedr_qp * qp,struct rdma_sq_fmr_wqe_1st * fwqe1,const struct ib_reg_wr * wr)3438 static int qedr_prepare_reg(struct qedr_qp *qp,
3439 			    struct rdma_sq_fmr_wqe_1st *fwqe1,
3440 			    const struct ib_reg_wr *wr)
3441 {
3442 	struct qedr_mr *mr = get_qedr_mr(wr->mr);
3443 	struct rdma_sq_fmr_wqe_2nd *fwqe2;
3444 
3445 	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3446 	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3447 	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3448 	fwqe1->l_key = wr->key;
3449 
3450 	fwqe2->access_ctrl = 0;
3451 
3452 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3453 		   !!(wr->access & IB_ACCESS_REMOTE_READ));
3454 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3455 		   !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3456 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3457 		   !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3458 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3459 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3460 		   !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3461 	fwqe2->fmr_ctrl = 0;
3462 
3463 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3464 		   ilog2(mr->ibmr.page_size) - 12);
3465 
3466 	fwqe2->length_hi = 0;
3467 	fwqe2->length_lo = mr->ibmr.length;
3468 	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3469 	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3470 
3471 	qp->wqe_wr_id[qp->sq.prod].mr = mr;
3472 
3473 	return 0;
3474 }
3475 
qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)3476 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3477 {
3478 	switch (opcode) {
3479 	case IB_WR_RDMA_WRITE:
3480 	case IB_WR_RDMA_WRITE_WITH_IMM:
3481 		return IB_WC_RDMA_WRITE;
3482 	case IB_WR_SEND_WITH_IMM:
3483 	case IB_WR_SEND:
3484 	case IB_WR_SEND_WITH_INV:
3485 		return IB_WC_SEND;
3486 	case IB_WR_RDMA_READ:
3487 	case IB_WR_RDMA_READ_WITH_INV:
3488 		return IB_WC_RDMA_READ;
3489 	case IB_WR_ATOMIC_CMP_AND_SWP:
3490 		return IB_WC_COMP_SWAP;
3491 	case IB_WR_ATOMIC_FETCH_AND_ADD:
3492 		return IB_WC_FETCH_ADD;
3493 	case IB_WR_REG_MR:
3494 		return IB_WC_REG_MR;
3495 	case IB_WR_LOCAL_INV:
3496 		return IB_WC_LOCAL_INV;
3497 	default:
3498 		return IB_WC_SEND;
3499 	}
3500 }
3501 
qedr_can_post_send(struct qedr_qp * qp,const struct ib_send_wr * wr)3502 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3503 				      const struct ib_send_wr *wr)
3504 {
3505 	int wq_is_full, err_wr, pbl_is_full;
3506 	struct qedr_dev *dev = qp->dev;
3507 
3508 	/* prevent SQ overflow and/or processing of a bad WR */
3509 	err_wr = wr->num_sge > qp->sq.max_sges;
3510 	wq_is_full = qedr_wq_is_full(&qp->sq);
3511 	pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3512 		      QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3513 	if (wq_is_full || err_wr || pbl_is_full) {
3514 		if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3515 			DP_ERR(dev,
3516 			       "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3517 			       qp);
3518 			qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3519 		}
3520 
3521 		if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3522 			DP_ERR(dev,
3523 			       "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3524 			       qp);
3525 			qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3526 		}
3527 
3528 		if (pbl_is_full &&
3529 		    !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3530 			DP_ERR(dev,
3531 			       "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3532 			       qp);
3533 			qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3534 		}
3535 		return false;
3536 	}
3537 	return true;
3538 }
3539 
__qedr_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3540 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3541 			    const struct ib_send_wr **bad_wr)
3542 {
3543 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3544 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3545 	struct rdma_sq_atomic_wqe_1st *awqe1;
3546 	struct rdma_sq_atomic_wqe_2nd *awqe2;
3547 	struct rdma_sq_atomic_wqe_3rd *awqe3;
3548 	struct rdma_sq_send_wqe_2st *swqe2;
3549 	struct rdma_sq_local_inv_wqe *iwqe;
3550 	struct rdma_sq_rdma_wqe_2nd *rwqe2;
3551 	struct rdma_sq_send_wqe_1st *swqe;
3552 	struct rdma_sq_rdma_wqe_1st *rwqe;
3553 	struct rdma_sq_fmr_wqe_1st *fwqe1;
3554 	struct rdma_sq_common_wqe *wqe;
3555 	u32 length;
3556 	int rc = 0;
3557 	bool comp;
3558 
3559 	if (!qedr_can_post_send(qp, wr)) {
3560 		*bad_wr = wr;
3561 		return -ENOMEM;
3562 	}
3563 
3564 	wqe = qed_chain_produce(&qp->sq.pbl);
3565 	qp->wqe_wr_id[qp->sq.prod].signaled =
3566 		!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3567 
3568 	wqe->flags = 0;
3569 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3570 		   !!(wr->send_flags & IB_SEND_SOLICITED));
3571 	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3572 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3573 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3574 		   !!(wr->send_flags & IB_SEND_FENCE));
3575 	wqe->prev_wqe_size = qp->prev_wqe_size;
3576 
3577 	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3578 
3579 	switch (wr->opcode) {
3580 	case IB_WR_SEND_WITH_IMM:
3581 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3582 			rc = -EINVAL;
3583 			*bad_wr = wr;
3584 			break;
3585 		}
3586 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3587 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3588 		swqe->wqe_size = 2;
3589 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3590 
3591 		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3592 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3593 						   wr, bad_wr);
3594 		swqe->length = cpu_to_le32(length);
3595 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3596 		qp->prev_wqe_size = swqe->wqe_size;
3597 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3598 		break;
3599 	case IB_WR_SEND:
3600 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3601 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3602 
3603 		swqe->wqe_size = 2;
3604 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3605 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3606 						   wr, bad_wr);
3607 		swqe->length = cpu_to_le32(length);
3608 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3609 		qp->prev_wqe_size = swqe->wqe_size;
3610 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3611 		break;
3612 	case IB_WR_SEND_WITH_INV:
3613 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3614 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3615 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3616 		swqe->wqe_size = 2;
3617 		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3618 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3619 						   wr, bad_wr);
3620 		swqe->length = cpu_to_le32(length);
3621 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3622 		qp->prev_wqe_size = swqe->wqe_size;
3623 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3624 		break;
3625 
3626 	case IB_WR_RDMA_WRITE_WITH_IMM:
3627 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3628 			rc = -EINVAL;
3629 			*bad_wr = wr;
3630 			break;
3631 		}
3632 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3633 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3634 
3635 		rwqe->wqe_size = 2;
3636 		rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3637 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3638 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3639 						   wr, bad_wr);
3640 		rwqe->length = cpu_to_le32(length);
3641 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3642 		qp->prev_wqe_size = rwqe->wqe_size;
3643 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3644 		break;
3645 	case IB_WR_RDMA_WRITE:
3646 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3647 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3648 
3649 		rwqe->wqe_size = 2;
3650 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3651 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3652 						   wr, bad_wr);
3653 		rwqe->length = cpu_to_le32(length);
3654 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3655 		qp->prev_wqe_size = rwqe->wqe_size;
3656 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3657 		break;
3658 	case IB_WR_RDMA_READ_WITH_INV:
3659 		SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3660 		fallthrough;	/* same is identical to RDMA READ */
3661 
3662 	case IB_WR_RDMA_READ:
3663 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3664 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3665 
3666 		rwqe->wqe_size = 2;
3667 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3668 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3669 						   wr, bad_wr);
3670 		rwqe->length = cpu_to_le32(length);
3671 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3672 		qp->prev_wqe_size = rwqe->wqe_size;
3673 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3674 		break;
3675 
3676 	case IB_WR_ATOMIC_CMP_AND_SWP:
3677 	case IB_WR_ATOMIC_FETCH_AND_ADD:
3678 		awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3679 		awqe1->wqe_size = 4;
3680 
3681 		awqe2 = qed_chain_produce(&qp->sq.pbl);
3682 		DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3683 		awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3684 
3685 		awqe3 = qed_chain_produce(&qp->sq.pbl);
3686 
3687 		if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3688 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3689 			DMA_REGPAIR_LE(awqe3->swap_data,
3690 				       atomic_wr(wr)->compare_add);
3691 		} else {
3692 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3693 			DMA_REGPAIR_LE(awqe3->swap_data,
3694 				       atomic_wr(wr)->swap);
3695 			DMA_REGPAIR_LE(awqe3->cmp_data,
3696 				       atomic_wr(wr)->compare_add);
3697 		}
3698 
3699 		qedr_prepare_sq_sges(qp, NULL, wr);
3700 
3701 		qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3702 		qp->prev_wqe_size = awqe1->wqe_size;
3703 		break;
3704 
3705 	case IB_WR_LOCAL_INV:
3706 		iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3707 		iwqe->wqe_size = 1;
3708 
3709 		iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3710 		iwqe->inv_l_key = wr->ex.invalidate_rkey;
3711 		qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3712 		qp->prev_wqe_size = iwqe->wqe_size;
3713 		break;
3714 	case IB_WR_REG_MR:
3715 		DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3716 		wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3717 		fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3718 		fwqe1->wqe_size = 2;
3719 
3720 		rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3721 		if (rc) {
3722 			DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3723 			*bad_wr = wr;
3724 			break;
3725 		}
3726 
3727 		qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3728 		qp->prev_wqe_size = fwqe1->wqe_size;
3729 		break;
3730 	default:
3731 		DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3732 		rc = -EINVAL;
3733 		*bad_wr = wr;
3734 		break;
3735 	}
3736 
3737 	if (*bad_wr) {
3738 		u16 value;
3739 
3740 		/* Restore prod to its position before
3741 		 * this WR was processed
3742 		 */
3743 		value = le16_to_cpu(qp->sq.db_data.data.value);
3744 		qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3745 
3746 		/* Restore prev_wqe_size */
3747 		qp->prev_wqe_size = wqe->prev_wqe_size;
3748 		rc = -EINVAL;
3749 		DP_ERR(dev, "POST SEND FAILED\n");
3750 	}
3751 
3752 	return rc;
3753 }
3754 
qedr_post_send(struct ib_qp * ibqp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)3755 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3756 		   const struct ib_send_wr **bad_wr)
3757 {
3758 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3759 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3760 	unsigned long flags;
3761 	int rc = 0;
3762 
3763 	*bad_wr = NULL;
3764 
3765 	if (qp->qp_type == IB_QPT_GSI)
3766 		return qedr_gsi_post_send(ibqp, wr, bad_wr);
3767 
3768 	spin_lock_irqsave(&qp->q_lock, flags);
3769 
3770 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
3771 		if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3772 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
3773 		    (qp->state != QED_ROCE_QP_STATE_SQD)) {
3774 			spin_unlock_irqrestore(&qp->q_lock, flags);
3775 			*bad_wr = wr;
3776 			DP_DEBUG(dev, QEDR_MSG_CQ,
3777 				 "QP in wrong state! QP icid=0x%x state %d\n",
3778 				 qp->icid, qp->state);
3779 			return -EINVAL;
3780 		}
3781 	}
3782 
3783 	while (wr) {
3784 		rc = __qedr_post_send(ibqp, wr, bad_wr);
3785 		if (rc)
3786 			break;
3787 
3788 		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3789 
3790 		qedr_inc_sw_prod(&qp->sq);
3791 
3792 		qp->sq.db_data.data.value++;
3793 
3794 		wr = wr->next;
3795 	}
3796 
3797 	/* Trigger doorbell
3798 	 * If there was a failure in the first WR then it will be triggered in
3799 	 * vane. However this is not harmful (as long as the producer value is
3800 	 * unchanged). For performance reasons we avoid checking for this
3801 	 * redundant doorbell.
3802 	 *
3803 	 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3804 	 * soon as we give the doorbell, we could get a completion
3805 	 * for this wr, therefore we need to make sure that the
3806 	 * memory is updated before giving the doorbell.
3807 	 * During qedr_poll_cq, rmb is called before accessing the
3808 	 * cqe. This covers for the smp_rmb as well.
3809 	 */
3810 	smp_wmb();
3811 	writel(qp->sq.db_data.raw, qp->sq.db);
3812 
3813 	spin_unlock_irqrestore(&qp->q_lock, flags);
3814 
3815 	return rc;
3816 }
3817 
qedr_srq_elem_left(struct qedr_srq_hwq_info * hw_srq)3818 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3819 {
3820 	u32 used;
3821 
3822 	/* Calculate number of elements used based on producer
3823 	 * count and consumer count and subtract it from max
3824 	 * work request supported so that we get elements left.
3825 	 */
3826 	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3827 
3828 	return hw_srq->max_wr - used;
3829 }
3830 
qedr_post_srq_recv(struct ib_srq * ibsrq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3831 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3832 		       const struct ib_recv_wr **bad_wr)
3833 {
3834 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
3835 	struct qedr_srq_hwq_info *hw_srq;
3836 	struct qedr_dev *dev = srq->dev;
3837 	struct qed_chain *pbl;
3838 	unsigned long flags;
3839 	int status = 0;
3840 	u32 num_sge;
3841 
3842 	spin_lock_irqsave(&srq->lock, flags);
3843 
3844 	hw_srq = &srq->hw_srq;
3845 	pbl = &srq->hw_srq.pbl;
3846 	while (wr) {
3847 		struct rdma_srq_wqe_header *hdr;
3848 		int i;
3849 
3850 		if (!qedr_srq_elem_left(hw_srq) ||
3851 		    wr->num_sge > srq->hw_srq.max_sges) {
3852 			DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3853 			       hw_srq->wr_prod_cnt,
3854 			       atomic_read(&hw_srq->wr_cons_cnt),
3855 			       wr->num_sge, srq->hw_srq.max_sges);
3856 			status = -ENOMEM;
3857 			*bad_wr = wr;
3858 			break;
3859 		}
3860 
3861 		hdr = qed_chain_produce(pbl);
3862 		num_sge = wr->num_sge;
3863 		/* Set number of sge and work request id in header */
3864 		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3865 
3866 		srq->hw_srq.wr_prod_cnt++;
3867 		hw_srq->wqe_prod++;
3868 		hw_srq->sge_prod++;
3869 
3870 		DP_DEBUG(dev, QEDR_MSG_SRQ,
3871 			 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3872 			 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3873 
3874 		for (i = 0; i < wr->num_sge; i++) {
3875 			struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3876 
3877 			/* Set SGE length, lkey and address */
3878 			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3879 				    wr->sg_list[i].length, wr->sg_list[i].lkey);
3880 
3881 			DP_DEBUG(dev, QEDR_MSG_SRQ,
3882 				 "[%d]: len %d key %x addr %x:%x\n",
3883 				 i, srq_sge->length, srq_sge->l_key,
3884 				 srq_sge->addr.hi, srq_sge->addr.lo);
3885 			hw_srq->sge_prod++;
3886 		}
3887 
3888 		/* Update WQE and SGE information before
3889 		 * updating producer.
3890 		 */
3891 		dma_wmb();
3892 
3893 		/* SRQ producer is 8 bytes. Need to update SGE producer index
3894 		 * in first 4 bytes and need to update WQE producer in
3895 		 * next 4 bytes.
3896 		 */
3897 		srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3898 		/* Make sure sge producer is updated first */
3899 		dma_wmb();
3900 		srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3901 
3902 		wr = wr->next;
3903 	}
3904 
3905 	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3906 		 qed_chain_get_elem_left(pbl));
3907 	spin_unlock_irqrestore(&srq->lock, flags);
3908 
3909 	return status;
3910 }
3911 
qedr_post_recv(struct ib_qp * ibqp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)3912 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3913 		   const struct ib_recv_wr **bad_wr)
3914 {
3915 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3916 	struct qedr_dev *dev = qp->dev;
3917 	unsigned long flags;
3918 	int status = 0;
3919 
3920 	if (qp->qp_type == IB_QPT_GSI)
3921 		return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3922 
3923 	spin_lock_irqsave(&qp->q_lock, flags);
3924 
3925 	while (wr) {
3926 		int i;
3927 
3928 		if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3929 		    QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3930 		    wr->num_sge > qp->rq.max_sges) {
3931 			DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3932 			       qed_chain_get_elem_left_u32(&qp->rq.pbl),
3933 			       QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3934 			       qp->rq.max_sges);
3935 			status = -ENOMEM;
3936 			*bad_wr = wr;
3937 			break;
3938 		}
3939 		for (i = 0; i < wr->num_sge; i++) {
3940 			u32 flags = 0;
3941 			struct rdma_rq_sge *rqe =
3942 			    qed_chain_produce(&qp->rq.pbl);
3943 
3944 			/* First one must include the number
3945 			 * of SGE in the list
3946 			 */
3947 			if (!i)
3948 				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3949 					  wr->num_sge);
3950 
3951 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3952 				  wr->sg_list[i].lkey);
3953 
3954 			RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3955 				   wr->sg_list[i].length, flags);
3956 		}
3957 
3958 		/* Special case of no sges. FW requires between 1-4 sges...
3959 		 * in this case we need to post 1 sge with length zero. this is
3960 		 * because rdma write with immediate consumes an RQ.
3961 		 */
3962 		if (!wr->num_sge) {
3963 			u32 flags = 0;
3964 			struct rdma_rq_sge *rqe =
3965 			    qed_chain_produce(&qp->rq.pbl);
3966 
3967 			/* First one must include the number
3968 			 * of SGE in the list
3969 			 */
3970 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3971 			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3972 
3973 			RQ_SGE_SET(rqe, 0, 0, flags);
3974 			i = 1;
3975 		}
3976 
3977 		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3978 		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3979 
3980 		qedr_inc_sw_prod(&qp->rq);
3981 
3982 		/* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3983 		 * soon as we give the doorbell, we could get a completion
3984 		 * for this wr, therefore we need to make sure that the
3985 		 * memory is update before giving the doorbell.
3986 		 * During qedr_poll_cq, rmb is called before accessing the
3987 		 * cqe. This covers for the smp_rmb as well.
3988 		 */
3989 		smp_wmb();
3990 
3991 		qp->rq.db_data.data.value++;
3992 
3993 		writel(qp->rq.db_data.raw, qp->rq.db);
3994 
3995 		if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3996 			writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3997 		}
3998 
3999 		wr = wr->next;
4000 	}
4001 
4002 	spin_unlock_irqrestore(&qp->q_lock, flags);
4003 
4004 	return status;
4005 }
4006 
is_valid_cqe(struct qedr_cq * cq,union rdma_cqe * cqe)4007 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4008 {
4009 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4010 
4011 	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4012 		cq->pbl_toggle;
4013 }
4014 
cqe_get_qp(union rdma_cqe * cqe)4015 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4016 {
4017 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4018 	struct qedr_qp *qp;
4019 
4020 	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4021 						   resp_cqe->qp_handle.lo,
4022 						   u64);
4023 	return qp;
4024 }
4025 
cqe_get_type(union rdma_cqe * cqe)4026 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4027 {
4028 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4029 
4030 	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4031 }
4032 
4033 /* Return latest CQE (needs processing) */
get_cqe(struct qedr_cq * cq)4034 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4035 {
4036 	return cq->latest_cqe;
4037 }
4038 
4039 /* In fmr we need to increase the number of fmr completed counter for the fmr
4040  * algorithm determining whether we can free a pbl or not.
4041  * we need to perform this whether the work request was signaled or not. for
4042  * this purpose we call this function from the condition that checks if a wr
4043  * should be skipped, to make sure we don't miss it ( possibly this fmr
4044  * operation was not signalted)
4045  */
qedr_chk_if_fmr(struct qedr_qp * qp)4046 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4047 {
4048 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4049 		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4050 }
4051 
process_req(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,u16 hw_cons,enum ib_wc_status status,int force)4052 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4053 		       struct qedr_cq *cq, int num_entries,
4054 		       struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4055 		       int force)
4056 {
4057 	u16 cnt = 0;
4058 
4059 	while (num_entries && qp->sq.wqe_cons != hw_cons) {
4060 		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4061 			qedr_chk_if_fmr(qp);
4062 			/* skip WC */
4063 			goto next_cqe;
4064 		}
4065 
4066 		/* fill WC */
4067 		wc->status = status;
4068 		wc->vendor_err = 0;
4069 		wc->wc_flags = 0;
4070 		wc->src_qp = qp->id;
4071 		wc->qp = &qp->ibqp;
4072 
4073 		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4074 		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4075 
4076 		switch (wc->opcode) {
4077 		case IB_WC_RDMA_WRITE:
4078 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4079 			break;
4080 		case IB_WC_COMP_SWAP:
4081 		case IB_WC_FETCH_ADD:
4082 			wc->byte_len = 8;
4083 			break;
4084 		case IB_WC_REG_MR:
4085 			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4086 			break;
4087 		case IB_WC_RDMA_READ:
4088 		case IB_WC_SEND:
4089 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4090 			break;
4091 		default:
4092 			break;
4093 		}
4094 
4095 		num_entries--;
4096 		wc++;
4097 		cnt++;
4098 next_cqe:
4099 		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4100 			qed_chain_consume(&qp->sq.pbl);
4101 		qedr_inc_sw_cons(&qp->sq);
4102 	}
4103 
4104 	return cnt;
4105 }
4106 
qedr_poll_cq_req(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_requester * req)4107 static int qedr_poll_cq_req(struct qedr_dev *dev,
4108 			    struct qedr_qp *qp, struct qedr_cq *cq,
4109 			    int num_entries, struct ib_wc *wc,
4110 			    struct rdma_cqe_requester *req)
4111 {
4112 	int cnt = 0;
4113 
4114 	switch (req->status) {
4115 	case RDMA_CQE_REQ_STS_OK:
4116 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4117 				  IB_WC_SUCCESS, 0);
4118 		break;
4119 	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4120 		if (qp->state != QED_ROCE_QP_STATE_ERR)
4121 			DP_DEBUG(dev, QEDR_MSG_CQ,
4122 				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4123 				 cq->icid, qp->icid);
4124 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4125 				  IB_WC_WR_FLUSH_ERR, 1);
4126 		break;
4127 	default:
4128 		/* process all WQE before the cosumer */
4129 		qp->state = QED_ROCE_QP_STATE_ERR;
4130 		cnt = process_req(dev, qp, cq, num_entries, wc,
4131 				  req->sq_cons - 1, IB_WC_SUCCESS, 0);
4132 		wc += cnt;
4133 		/* if we have extra WC fill it with actual error info */
4134 		if (cnt < num_entries) {
4135 			enum ib_wc_status wc_status;
4136 
4137 			switch (req->status) {
4138 			case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4139 				DP_ERR(dev,
4140 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4141 				       cq->icid, qp->icid);
4142 				wc_status = IB_WC_BAD_RESP_ERR;
4143 				break;
4144 			case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4145 				DP_ERR(dev,
4146 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4147 				       cq->icid, qp->icid);
4148 				wc_status = IB_WC_LOC_LEN_ERR;
4149 				break;
4150 			case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4151 				DP_ERR(dev,
4152 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4153 				       cq->icid, qp->icid);
4154 				wc_status = IB_WC_LOC_QP_OP_ERR;
4155 				break;
4156 			case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4157 				DP_ERR(dev,
4158 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4159 				       cq->icid, qp->icid);
4160 				wc_status = IB_WC_LOC_PROT_ERR;
4161 				break;
4162 			case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4163 				DP_ERR(dev,
4164 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4165 				       cq->icid, qp->icid);
4166 				wc_status = IB_WC_MW_BIND_ERR;
4167 				break;
4168 			case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4169 				DP_ERR(dev,
4170 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4171 				       cq->icid, qp->icid);
4172 				wc_status = IB_WC_REM_INV_REQ_ERR;
4173 				break;
4174 			case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4175 				DP_ERR(dev,
4176 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4177 				       cq->icid, qp->icid);
4178 				wc_status = IB_WC_REM_ACCESS_ERR;
4179 				break;
4180 			case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4181 				DP_ERR(dev,
4182 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4183 				       cq->icid, qp->icid);
4184 				wc_status = IB_WC_REM_OP_ERR;
4185 				break;
4186 			case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4187 				DP_ERR(dev,
4188 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4189 				       cq->icid, qp->icid);
4190 				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4191 				break;
4192 			case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4193 				DP_ERR(dev,
4194 				       "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4195 				       cq->icid, qp->icid);
4196 				wc_status = IB_WC_RETRY_EXC_ERR;
4197 				break;
4198 			default:
4199 				DP_ERR(dev,
4200 				       "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4201 				       cq->icid, qp->icid);
4202 				wc_status = IB_WC_GENERAL_ERR;
4203 			}
4204 			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4205 					   wc_status, 1);
4206 		}
4207 	}
4208 
4209 	return cnt;
4210 }
4211 
qedr_cqe_resp_status_to_ib(u8 status)4212 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4213 {
4214 	switch (status) {
4215 	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4216 		return IB_WC_LOC_ACCESS_ERR;
4217 	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4218 		return IB_WC_LOC_LEN_ERR;
4219 	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4220 		return IB_WC_LOC_QP_OP_ERR;
4221 	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4222 		return IB_WC_LOC_PROT_ERR;
4223 	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4224 		return IB_WC_MW_BIND_ERR;
4225 	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4226 		return IB_WC_REM_INV_RD_REQ_ERR;
4227 	case RDMA_CQE_RESP_STS_OK:
4228 		return IB_WC_SUCCESS;
4229 	default:
4230 		return IB_WC_GENERAL_ERR;
4231 	}
4232 }
4233 
qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder * resp,struct ib_wc * wc)4234 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4235 					  struct ib_wc *wc)
4236 {
4237 	wc->status = IB_WC_SUCCESS;
4238 	wc->byte_len = le32_to_cpu(resp->length);
4239 
4240 	if (resp->flags & QEDR_RESP_IMM) {
4241 		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4242 		wc->wc_flags |= IB_WC_WITH_IMM;
4243 
4244 		if (resp->flags & QEDR_RESP_RDMA)
4245 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4246 
4247 		if (resp->flags & QEDR_RESP_INV)
4248 			return -EINVAL;
4249 
4250 	} else if (resp->flags & QEDR_RESP_INV) {
4251 		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4252 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4253 
4254 		if (resp->flags & QEDR_RESP_RDMA)
4255 			return -EINVAL;
4256 
4257 	} else if (resp->flags & QEDR_RESP_RDMA) {
4258 		return -EINVAL;
4259 	}
4260 
4261 	return 0;
4262 }
4263 
__process_resp_one(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp,u64 wr_id)4264 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4265 			       struct qedr_cq *cq, struct ib_wc *wc,
4266 			       struct rdma_cqe_responder *resp, u64 wr_id)
4267 {
4268 	/* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4269 	wc->opcode = IB_WC_RECV;
4270 	wc->wc_flags = 0;
4271 
4272 	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4273 		if (qedr_set_ok_cqe_resp_wc(resp, wc))
4274 			DP_ERR(dev,
4275 			       "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4276 			       cq, cq->icid, resp->flags);
4277 
4278 	} else {
4279 		wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4280 		if (wc->status == IB_WC_GENERAL_ERR)
4281 			DP_ERR(dev,
4282 			       "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4283 			       cq, cq->icid, resp->status);
4284 	}
4285 
4286 	/* Fill the rest of the WC */
4287 	wc->vendor_err = 0;
4288 	wc->src_qp = qp->id;
4289 	wc->qp = &qp->ibqp;
4290 	wc->wr_id = wr_id;
4291 }
4292 
process_resp_one_srq(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp)4293 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4294 				struct qedr_cq *cq, struct ib_wc *wc,
4295 				struct rdma_cqe_responder *resp)
4296 {
4297 	struct qedr_srq *srq = qp->srq;
4298 	u64 wr_id;
4299 
4300 	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4301 			 le32_to_cpu(resp->srq_wr_id.lo), u64);
4302 
4303 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4304 		wc->status = IB_WC_WR_FLUSH_ERR;
4305 		wc->vendor_err = 0;
4306 		wc->wr_id = wr_id;
4307 		wc->byte_len = 0;
4308 		wc->src_qp = qp->id;
4309 		wc->qp = &qp->ibqp;
4310 		wc->wr_id = wr_id;
4311 	} else {
4312 		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4313 	}
4314 	atomic_inc(&srq->hw_srq.wr_cons_cnt);
4315 
4316 	return 1;
4317 }
process_resp_one(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,struct ib_wc * wc,struct rdma_cqe_responder * resp)4318 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4319 			    struct qedr_cq *cq, struct ib_wc *wc,
4320 			    struct rdma_cqe_responder *resp)
4321 {
4322 	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4323 
4324 	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4325 
4326 	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4327 		qed_chain_consume(&qp->rq.pbl);
4328 	qedr_inc_sw_cons(&qp->rq);
4329 
4330 	return 1;
4331 }
4332 
process_resp_flush(struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,u16 hw_cons)4333 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4334 			      int num_entries, struct ib_wc *wc, u16 hw_cons)
4335 {
4336 	u16 cnt = 0;
4337 
4338 	while (num_entries && qp->rq.wqe_cons != hw_cons) {
4339 		/* fill WC */
4340 		wc->status = IB_WC_WR_FLUSH_ERR;
4341 		wc->vendor_err = 0;
4342 		wc->wc_flags = 0;
4343 		wc->src_qp = qp->id;
4344 		wc->byte_len = 0;
4345 		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4346 		wc->qp = &qp->ibqp;
4347 		num_entries--;
4348 		wc++;
4349 		cnt++;
4350 		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4351 			qed_chain_consume(&qp->rq.pbl);
4352 		qedr_inc_sw_cons(&qp->rq);
4353 	}
4354 
4355 	return cnt;
4356 }
4357 
try_consume_resp_cqe(struct qedr_cq * cq,struct qedr_qp * qp,struct rdma_cqe_responder * resp,int * update)4358 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4359 				 struct rdma_cqe_responder *resp, int *update)
4360 {
4361 	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4362 		consume_cqe(cq);
4363 		*update |= 1;
4364 	}
4365 }
4366 
qedr_poll_cq_resp_srq(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_responder * resp)4367 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4368 				 struct qedr_cq *cq, int num_entries,
4369 				 struct ib_wc *wc,
4370 				 struct rdma_cqe_responder *resp)
4371 {
4372 	int cnt;
4373 
4374 	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4375 	consume_cqe(cq);
4376 
4377 	return cnt;
4378 }
4379 
qedr_poll_cq_resp(struct qedr_dev * dev,struct qedr_qp * qp,struct qedr_cq * cq,int num_entries,struct ib_wc * wc,struct rdma_cqe_responder * resp,int * update)4380 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4381 			     struct qedr_cq *cq, int num_entries,
4382 			     struct ib_wc *wc, struct rdma_cqe_responder *resp,
4383 			     int *update)
4384 {
4385 	int cnt;
4386 
4387 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4388 		cnt = process_resp_flush(qp, cq, num_entries, wc,
4389 					 resp->rq_cons_or_srq_id);
4390 		try_consume_resp_cqe(cq, qp, resp, update);
4391 	} else {
4392 		cnt = process_resp_one(dev, qp, cq, wc, resp);
4393 		consume_cqe(cq);
4394 		*update |= 1;
4395 	}
4396 
4397 	return cnt;
4398 }
4399 
try_consume_req_cqe(struct qedr_cq * cq,struct qedr_qp * qp,struct rdma_cqe_requester * req,int * update)4400 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4401 				struct rdma_cqe_requester *req, int *update)
4402 {
4403 	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4404 		consume_cqe(cq);
4405 		*update |= 1;
4406 	}
4407 }
4408 
qedr_poll_cq(struct ib_cq * ibcq,int num_entries,struct ib_wc * wc)4409 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4410 {
4411 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4412 	struct qedr_cq *cq = get_qedr_cq(ibcq);
4413 	union rdma_cqe *cqe;
4414 	u32 old_cons, new_cons;
4415 	unsigned long flags;
4416 	int update = 0;
4417 	int done = 0;
4418 
4419 	if (cq->destroyed) {
4420 		DP_ERR(dev,
4421 		       "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4422 		       cq, cq->icid);
4423 		return 0;
4424 	}
4425 
4426 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4427 		return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4428 
4429 	spin_lock_irqsave(&cq->cq_lock, flags);
4430 	cqe = cq->latest_cqe;
4431 	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4432 	while (num_entries && is_valid_cqe(cq, cqe)) {
4433 		struct qedr_qp *qp;
4434 		int cnt = 0;
4435 
4436 		/* prevent speculative reads of any field of CQE */
4437 		rmb();
4438 
4439 		qp = cqe_get_qp(cqe);
4440 		if (!qp) {
4441 			WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4442 			break;
4443 		}
4444 
4445 		wc->qp = &qp->ibqp;
4446 
4447 		switch (cqe_get_type(cqe)) {
4448 		case RDMA_CQE_TYPE_REQUESTER:
4449 			cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4450 					       &cqe->req);
4451 			try_consume_req_cqe(cq, qp, &cqe->req, &update);
4452 			break;
4453 		case RDMA_CQE_TYPE_RESPONDER_RQ:
4454 			cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4455 						&cqe->resp, &update);
4456 			break;
4457 		case RDMA_CQE_TYPE_RESPONDER_SRQ:
4458 			cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4459 						    wc, &cqe->resp);
4460 			update = 1;
4461 			break;
4462 		case RDMA_CQE_TYPE_INVALID:
4463 		default:
4464 			DP_ERR(dev, "Error: invalid CQE type = %d\n",
4465 			       cqe_get_type(cqe));
4466 		}
4467 		num_entries -= cnt;
4468 		wc += cnt;
4469 		done += cnt;
4470 
4471 		cqe = get_cqe(cq);
4472 	}
4473 	new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4474 
4475 	cq->cq_cons += new_cons - old_cons;
4476 
4477 	if (update)
4478 		/* doorbell notifies abount latest VALID entry,
4479 		 * but chain already point to the next INVALID one
4480 		 */
4481 		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4482 
4483 	spin_unlock_irqrestore(&cq->cq_lock, flags);
4484 	return done;
4485 }
4486 
qedr_process_mad(struct ib_device * ibdev,int process_mad_flags,u32 port_num,const struct ib_wc * in_wc,const struct ib_grh * in_grh,const struct ib_mad * in,struct ib_mad * out_mad,size_t * out_mad_size,u16 * out_mad_pkey_index)4487 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4488 		     u32 port_num, const struct ib_wc *in_wc,
4489 		     const struct ib_grh *in_grh, const struct ib_mad *in,
4490 		     struct ib_mad *out_mad, size_t *out_mad_size,
4491 		     u16 *out_mad_pkey_index)
4492 {
4493 	return IB_MAD_RESULT_SUCCESS;
4494 }
4495