xref: /openbmc/linux/drivers/infiniband/hw/qedr/verbs.c (revision f97769fd)
1 /* QLogic qedr NIC Driver
2  * Copyright (c) 2015-2016  QLogic Corporation
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and /or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38 
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45 #include <rdma/uverbs_ioctl.h>
46 
47 #include <linux/qed/common_hsi.h>
48 #include "qedr_hsi_rdma.h"
49 #include <linux/qed/qed_if.h>
50 #include "qedr.h"
51 #include "verbs.h"
52 #include <rdma/qedr-abi.h>
53 #include "qedr_roce_cm.h"
54 #include "qedr_iw_cm.h"
55 
56 #define QEDR_SRQ_WQE_ELEM_SIZE	sizeof(union rdma_srq_elm)
57 #define	RDMA_MAX_SGE_PER_SRQ	(4)
58 #define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1)
59 
60 #define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61 
62 enum {
63 	QEDR_USER_MMAP_IO_WC = 0,
64 	QEDR_USER_MMAP_PHYS_PAGE,
65 };
66 
67 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 					size_t len)
69 {
70 	size_t min_len = min_t(size_t, len, udata->outlen);
71 
72 	return ib_copy_to_udata(udata, src, min_len);
73 }
74 
75 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
76 {
77 	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
78 		return -EINVAL;
79 
80 	*pkey = QEDR_ROCE_PKEY_DEFAULT;
81 	return 0;
82 }
83 
84 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
85 		      int index, union ib_gid *sgid)
86 {
87 	struct qedr_dev *dev = get_qedr_dev(ibdev);
88 
89 	memset(sgid->raw, 0, sizeof(sgid->raw));
90 	ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91 
92 	DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 		 sgid->global.interface_id, sgid->global.subnet_prefix);
94 
95 	return 0;
96 }
97 
98 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99 {
100 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 	struct qedr_device_attr *qattr = &dev->attr;
102 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
103 
104 	srq_attr->srq_limit = srq->srq_limit;
105 	srq_attr->max_wr = qattr->max_srq_wr;
106 	srq_attr->max_sge = qattr->max_sge;
107 
108 	return 0;
109 }
110 
111 int qedr_query_device(struct ib_device *ibdev,
112 		      struct ib_device_attr *attr, struct ib_udata *udata)
113 {
114 	struct qedr_dev *dev = get_qedr_dev(ibdev);
115 	struct qedr_device_attr *qattr = &dev->attr;
116 
117 	if (!dev->rdma_ctx) {
118 		DP_ERR(dev,
119 		       "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 		       dev->rdma_ctx);
121 		return -EINVAL;
122 	}
123 
124 	memset(attr, 0, sizeof(*attr));
125 
126 	attr->fw_ver = qattr->fw_ver;
127 	attr->sys_image_guid = qattr->sys_image_guid;
128 	attr->max_mr_size = qattr->max_mr_size;
129 	attr->page_size_cap = qattr->page_size_caps;
130 	attr->vendor_id = qattr->vendor_id;
131 	attr->vendor_part_id = qattr->vendor_part_id;
132 	attr->hw_ver = qattr->hw_ver;
133 	attr->max_qp = qattr->max_qp;
134 	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 	    IB_DEVICE_RC_RNR_NAK_GEN |
137 	    IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
138 
139 	attr->max_send_sge = qattr->max_sge;
140 	attr->max_recv_sge = qattr->max_sge;
141 	attr->max_sge_rd = qattr->max_sge;
142 	attr->max_cq = qattr->max_cq;
143 	attr->max_cqe = qattr->max_cqe;
144 	attr->max_mr = qattr->max_mr;
145 	attr->max_mw = qattr->max_mw;
146 	attr->max_pd = qattr->max_pd;
147 	attr->atomic_cap = dev->atomic_cap;
148 	attr->max_qp_init_rd_atom =
149 	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
150 	attr->max_qp_rd_atom =
151 	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
152 		attr->max_qp_init_rd_atom);
153 
154 	attr->max_srq = qattr->max_srq;
155 	attr->max_srq_sge = qattr->max_srq_sge;
156 	attr->max_srq_wr = qattr->max_srq_wr;
157 
158 	attr->local_ca_ack_delay = qattr->dev_ack_delay;
159 	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
160 	attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
161 	attr->max_ah = qattr->max_ah;
162 
163 	return 0;
164 }
165 
166 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
167 					    u8 *ib_width)
168 {
169 	switch (speed) {
170 	case 1000:
171 		*ib_speed = IB_SPEED_SDR;
172 		*ib_width = IB_WIDTH_1X;
173 		break;
174 	case 10000:
175 		*ib_speed = IB_SPEED_QDR;
176 		*ib_width = IB_WIDTH_1X;
177 		break;
178 
179 	case 20000:
180 		*ib_speed = IB_SPEED_DDR;
181 		*ib_width = IB_WIDTH_4X;
182 		break;
183 
184 	case 25000:
185 		*ib_speed = IB_SPEED_EDR;
186 		*ib_width = IB_WIDTH_1X;
187 		break;
188 
189 	case 40000:
190 		*ib_speed = IB_SPEED_QDR;
191 		*ib_width = IB_WIDTH_4X;
192 		break;
193 
194 	case 50000:
195 		*ib_speed = IB_SPEED_HDR;
196 		*ib_width = IB_WIDTH_1X;
197 		break;
198 
199 	case 100000:
200 		*ib_speed = IB_SPEED_EDR;
201 		*ib_width = IB_WIDTH_4X;
202 		break;
203 
204 	default:
205 		/* Unsupported */
206 		*ib_speed = IB_SPEED_SDR;
207 		*ib_width = IB_WIDTH_1X;
208 	}
209 }
210 
211 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
212 {
213 	struct qedr_dev *dev;
214 	struct qed_rdma_port *rdma_port;
215 
216 	dev = get_qedr_dev(ibdev);
217 
218 	if (!dev->rdma_ctx) {
219 		DP_ERR(dev, "rdma_ctx is NULL\n");
220 		return -EINVAL;
221 	}
222 
223 	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
224 
225 	/* *attr being zeroed by the caller, avoid zeroing it here */
226 	if (rdma_port->port_state == QED_RDMA_PORT_UP) {
227 		attr->state = IB_PORT_ACTIVE;
228 		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
229 	} else {
230 		attr->state = IB_PORT_DOWN;
231 		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
232 	}
233 	attr->max_mtu = IB_MTU_4096;
234 	attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
235 	attr->lid = 0;
236 	attr->lmc = 0;
237 	attr->sm_lid = 0;
238 	attr->sm_sl = 0;
239 	attr->ip_gids = true;
240 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
241 		attr->gid_tbl_len = 1;
242 	} else {
243 		attr->gid_tbl_len = QEDR_MAX_SGID;
244 		attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
245 	}
246 	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
247 	attr->qkey_viol_cntr = 0;
248 	get_link_speed_and_width(rdma_port->link_speed,
249 				 &attr->active_speed, &attr->active_width);
250 	attr->max_msg_sz = rdma_port->max_msg_size;
251 	attr->max_vl_num = 4;
252 
253 	return 0;
254 }
255 
256 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
257 {
258 	struct ib_device *ibdev = uctx->device;
259 	int rc;
260 	struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
261 	struct qedr_alloc_ucontext_resp uresp = {};
262 	struct qedr_alloc_ucontext_req ureq = {};
263 	struct qedr_dev *dev = get_qedr_dev(ibdev);
264 	struct qed_rdma_add_user_out_params oparams;
265 	struct qedr_user_mmap_entry *entry;
266 
267 	if (!udata)
268 		return -EFAULT;
269 
270 	if (udata->inlen) {
271 		rc = ib_copy_from_udata(&ureq, udata,
272 					min(sizeof(ureq), udata->inlen));
273 		if (rc) {
274 			DP_ERR(dev, "Problem copying data from user space\n");
275 			return -EFAULT;
276 		}
277 		ctx->edpm_mode = !!(ureq.context_flags &
278 				    QEDR_ALLOC_UCTX_EDPM_MODE);
279 		ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
280 	}
281 
282 	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
283 	if (rc) {
284 		DP_ERR(dev,
285 		       "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
286 		       rc);
287 		return rc;
288 	}
289 
290 	ctx->dpi = oparams.dpi;
291 	ctx->dpi_addr = oparams.dpi_addr;
292 	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
293 	ctx->dpi_size = oparams.dpi_size;
294 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
295 	if (!entry) {
296 		rc = -ENOMEM;
297 		goto err;
298 	}
299 
300 	entry->io_address = ctx->dpi_phys_addr;
301 	entry->length = ctx->dpi_size;
302 	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
303 	entry->dpi = ctx->dpi;
304 	entry->dev = dev;
305 	rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
306 					 ctx->dpi_size);
307 	if (rc) {
308 		kfree(entry);
309 		goto err;
310 	}
311 	ctx->db_mmap_entry = &entry->rdma_entry;
312 
313 	if (!dev->user_dpm_enabled)
314 		uresp.dpm_flags = 0;
315 	else if (rdma_protocol_iwarp(&dev->ibdev, 1))
316 		uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
317 	else
318 		uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
319 				  QEDR_DPM_TYPE_ROCE_LEGACY |
320 				  QEDR_DPM_TYPE_ROCE_EDPM_MODE;
321 
322 	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
323 		uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
324 		uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
325 		uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
326 		uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
327 	}
328 
329 	uresp.wids_enabled = 1;
330 	uresp.wid_count = oparams.wid_count;
331 	uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
332 	uresp.db_size = ctx->dpi_size;
333 	uresp.max_send_wr = dev->attr.max_sqe;
334 	uresp.max_recv_wr = dev->attr.max_rqe;
335 	uresp.max_srq_wr = dev->attr.max_srq_wr;
336 	uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
337 	uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
338 	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
339 	uresp.max_cqes = QEDR_MAX_CQES;
340 
341 	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
342 	if (rc)
343 		goto err;
344 
345 	ctx->dev = dev;
346 
347 	DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
348 		 &ctx->ibucontext);
349 	return 0;
350 
351 err:
352 	if (!ctx->db_mmap_entry)
353 		dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
354 	else
355 		rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
356 
357 	return rc;
358 }
359 
360 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
361 {
362 	struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
363 
364 	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
365 		 uctx);
366 
367 	rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
368 }
369 
370 void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
371 {
372 	struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
373 	struct qedr_dev *dev = entry->dev;
374 
375 	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
376 		free_page((unsigned long)entry->address);
377 	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
378 		dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
379 
380 	kfree(entry);
381 }
382 
383 int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
384 {
385 	struct ib_device *dev = ucontext->device;
386 	size_t length = vma->vm_end - vma->vm_start;
387 	struct rdma_user_mmap_entry *rdma_entry;
388 	struct qedr_user_mmap_entry *entry;
389 	int rc = 0;
390 	u64 pfn;
391 
392 	ibdev_dbg(dev,
393 		  "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
394 		  vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
395 
396 	rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
397 	if (!rdma_entry) {
398 		ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
399 			  vma->vm_pgoff);
400 		return -EINVAL;
401 	}
402 	entry = get_qedr_mmap_entry(rdma_entry);
403 	ibdev_dbg(dev,
404 		  "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
405 		  entry->io_address, length, entry->mmap_flag);
406 
407 	switch (entry->mmap_flag) {
408 	case QEDR_USER_MMAP_IO_WC:
409 		pfn = entry->io_address >> PAGE_SHIFT;
410 		rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
411 				       pgprot_writecombine(vma->vm_page_prot),
412 				       rdma_entry);
413 		break;
414 	case QEDR_USER_MMAP_PHYS_PAGE:
415 		rc = vm_insert_page(vma, vma->vm_start,
416 				    virt_to_page(entry->address));
417 		break;
418 	default:
419 		rc = -EINVAL;
420 	}
421 
422 	if (rc)
423 		ibdev_dbg(dev,
424 			  "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
425 			  entry->io_address, length, entry->mmap_flag, rc);
426 
427 	rdma_user_mmap_entry_put(rdma_entry);
428 	return rc;
429 }
430 
431 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
432 {
433 	struct ib_device *ibdev = ibpd->device;
434 	struct qedr_dev *dev = get_qedr_dev(ibdev);
435 	struct qedr_pd *pd = get_qedr_pd(ibpd);
436 	u16 pd_id;
437 	int rc;
438 
439 	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
440 		 udata ? "User Lib" : "Kernel");
441 
442 	if (!dev->rdma_ctx) {
443 		DP_ERR(dev, "invalid RDMA context\n");
444 		return -EINVAL;
445 	}
446 
447 	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
448 	if (rc)
449 		return rc;
450 
451 	pd->pd_id = pd_id;
452 
453 	if (udata) {
454 		struct qedr_alloc_pd_uresp uresp = {
455 			.pd_id = pd_id,
456 		};
457 		struct qedr_ucontext *context = rdma_udata_to_drv_context(
458 			udata, struct qedr_ucontext, ibucontext);
459 
460 		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
461 		if (rc) {
462 			DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
463 			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
464 			return rc;
465 		}
466 
467 		pd->uctx = context;
468 		pd->uctx->pd = pd;
469 	}
470 
471 	return 0;
472 }
473 
474 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
475 {
476 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
477 	struct qedr_pd *pd = get_qedr_pd(ibpd);
478 
479 	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
480 	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
481 }
482 
483 static void qedr_free_pbl(struct qedr_dev *dev,
484 			  struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
485 {
486 	struct pci_dev *pdev = dev->pdev;
487 	int i;
488 
489 	for (i = 0; i < pbl_info->num_pbls; i++) {
490 		if (!pbl[i].va)
491 			continue;
492 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
493 				  pbl[i].va, pbl[i].pa);
494 	}
495 
496 	kfree(pbl);
497 }
498 
499 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
500 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
501 
502 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
503 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
504 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
505 
506 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
507 					   struct qedr_pbl_info *pbl_info,
508 					   gfp_t flags)
509 {
510 	struct pci_dev *pdev = dev->pdev;
511 	struct qedr_pbl *pbl_table;
512 	dma_addr_t *pbl_main_tbl;
513 	dma_addr_t pa;
514 	void *va;
515 	int i;
516 
517 	pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
518 	if (!pbl_table)
519 		return ERR_PTR(-ENOMEM);
520 
521 	for (i = 0; i < pbl_info->num_pbls; i++) {
522 		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
523 					flags);
524 		if (!va)
525 			goto err;
526 
527 		pbl_table[i].va = va;
528 		pbl_table[i].pa = pa;
529 	}
530 
531 	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
532 	 * the first one with physical pointers to all of the rest
533 	 */
534 	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
535 	for (i = 0; i < pbl_info->num_pbls - 1; i++)
536 		pbl_main_tbl[i] = pbl_table[i + 1].pa;
537 
538 	return pbl_table;
539 
540 err:
541 	for (i--; i >= 0; i--)
542 		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
543 				  pbl_table[i].va, pbl_table[i].pa);
544 
545 	qedr_free_pbl(dev, pbl_info, pbl_table);
546 
547 	return ERR_PTR(-ENOMEM);
548 }
549 
550 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
551 				struct qedr_pbl_info *pbl_info,
552 				u32 num_pbes, int two_layer_capable)
553 {
554 	u32 pbl_capacity;
555 	u32 pbl_size;
556 	u32 num_pbls;
557 
558 	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
559 		if (num_pbes > MAX_PBES_TWO_LAYER) {
560 			DP_ERR(dev, "prepare pbl table: too many pages %d\n",
561 			       num_pbes);
562 			return -EINVAL;
563 		}
564 
565 		/* calculate required pbl page size */
566 		pbl_size = MIN_FW_PBL_PAGE_SIZE;
567 		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
568 			       NUM_PBES_ON_PAGE(pbl_size);
569 
570 		while (pbl_capacity < num_pbes) {
571 			pbl_size *= 2;
572 			pbl_capacity = pbl_size / sizeof(u64);
573 			pbl_capacity = pbl_capacity * pbl_capacity;
574 		}
575 
576 		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
577 		num_pbls++;	/* One for the layer0 ( points to the pbls) */
578 		pbl_info->two_layered = true;
579 	} else {
580 		/* One layered PBL */
581 		num_pbls = 1;
582 		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
583 				 roundup_pow_of_two((num_pbes * sizeof(u64))));
584 		pbl_info->two_layered = false;
585 	}
586 
587 	pbl_info->num_pbls = num_pbls;
588 	pbl_info->pbl_size = pbl_size;
589 	pbl_info->num_pbes = num_pbes;
590 
591 	DP_DEBUG(dev, QEDR_MSG_MR,
592 		 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
593 		 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
594 
595 	return 0;
596 }
597 
598 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
599 			       struct qedr_pbl *pbl,
600 			       struct qedr_pbl_info *pbl_info, u32 pg_shift)
601 {
602 	int pbe_cnt, total_num_pbes = 0;
603 	u32 fw_pg_cnt, fw_pg_per_umem_pg;
604 	struct qedr_pbl *pbl_tbl;
605 	struct sg_dma_page_iter sg_iter;
606 	struct regpair *pbe;
607 	u64 pg_addr;
608 
609 	if (!pbl_info->num_pbes)
610 		return;
611 
612 	/* If we have a two layered pbl, the first pbl points to the rest
613 	 * of the pbls and the first entry lays on the second pbl in the table
614 	 */
615 	if (pbl_info->two_layered)
616 		pbl_tbl = &pbl[1];
617 	else
618 		pbl_tbl = pbl;
619 
620 	pbe = (struct regpair *)pbl_tbl->va;
621 	if (!pbe) {
622 		DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
623 		return;
624 	}
625 
626 	pbe_cnt = 0;
627 
628 	fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
629 
630 	for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
631 		pg_addr = sg_page_iter_dma_address(&sg_iter);
632 		for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
633 			pbe->lo = cpu_to_le32(pg_addr);
634 			pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
635 
636 			pg_addr += BIT(pg_shift);
637 			pbe_cnt++;
638 			total_num_pbes++;
639 			pbe++;
640 
641 			if (total_num_pbes == pbl_info->num_pbes)
642 				return;
643 
644 			/* If the given pbl is full storing the pbes,
645 			 * move to next pbl.
646 			 */
647 			if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
648 				pbl_tbl++;
649 				pbe = (struct regpair *)pbl_tbl->va;
650 				pbe_cnt = 0;
651 			}
652 
653 			fw_pg_cnt++;
654 		}
655 	}
656 }
657 
658 static int qedr_db_recovery_add(struct qedr_dev *dev,
659 				void __iomem *db_addr,
660 				void *db_data,
661 				enum qed_db_rec_width db_width,
662 				enum qed_db_rec_space db_space)
663 {
664 	if (!db_data) {
665 		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
666 		return 0;
667 	}
668 
669 	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
670 						 db_width, db_space);
671 }
672 
673 static void qedr_db_recovery_del(struct qedr_dev *dev,
674 				 void __iomem *db_addr,
675 				 void *db_data)
676 {
677 	if (!db_data) {
678 		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
679 		return;
680 	}
681 
682 	/* Ignore return code as there is not much we can do about it. Error
683 	 * log will be printed inside.
684 	 */
685 	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
686 }
687 
688 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
689 			      struct qedr_cq *cq, struct ib_udata *udata,
690 			      u32 db_offset)
691 {
692 	struct qedr_create_cq_uresp uresp;
693 	int rc;
694 
695 	memset(&uresp, 0, sizeof(uresp));
696 
697 	uresp.db_offset = db_offset;
698 	uresp.icid = cq->icid;
699 	if (cq->q.db_mmap_entry)
700 		uresp.db_rec_addr =
701 			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
702 
703 	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
704 	if (rc)
705 		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
706 
707 	return rc;
708 }
709 
710 static void consume_cqe(struct qedr_cq *cq)
711 {
712 	if (cq->latest_cqe == cq->toggle_cqe)
713 		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
714 
715 	cq->latest_cqe = qed_chain_consume(&cq->pbl);
716 }
717 
718 static inline int qedr_align_cq_entries(int entries)
719 {
720 	u64 size, aligned_size;
721 
722 	/* We allocate an extra entry that we don't report to the FW. */
723 	size = (entries + 1) * QEDR_CQE_SIZE;
724 	aligned_size = ALIGN(size, PAGE_SIZE);
725 
726 	return aligned_size / QEDR_CQE_SIZE;
727 }
728 
729 static int qedr_init_user_db_rec(struct ib_udata *udata,
730 				 struct qedr_dev *dev, struct qedr_userq *q,
731 				 bool requires_db_rec)
732 {
733 	struct qedr_ucontext *uctx =
734 		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
735 					  ibucontext);
736 	struct qedr_user_mmap_entry *entry;
737 	int rc;
738 
739 	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
740 	if (requires_db_rec == 0 || !uctx->db_rec)
741 		return 0;
742 
743 	/* Allocate a page for doorbell recovery, add to mmap */
744 	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
745 	if (!q->db_rec_data) {
746 		DP_ERR(dev, "get_zeroed_page failed\n");
747 		return -ENOMEM;
748 	}
749 
750 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
751 	if (!entry)
752 		goto err_free_db_data;
753 
754 	entry->address = q->db_rec_data;
755 	entry->length = PAGE_SIZE;
756 	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
757 	rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
758 					 &entry->rdma_entry,
759 					 PAGE_SIZE);
760 	if (rc)
761 		goto err_free_entry;
762 
763 	q->db_mmap_entry = &entry->rdma_entry;
764 
765 	return 0;
766 
767 err_free_entry:
768 	kfree(entry);
769 
770 err_free_db_data:
771 	free_page((unsigned long)q->db_rec_data);
772 	q->db_rec_data = NULL;
773 	return -ENOMEM;
774 }
775 
776 static inline int qedr_init_user_queue(struct ib_udata *udata,
777 				       struct qedr_dev *dev,
778 				       struct qedr_userq *q, u64 buf_addr,
779 				       size_t buf_len, bool requires_db_rec,
780 				       int access,
781 				       int alloc_and_init)
782 {
783 	u32 fw_pages;
784 	int rc;
785 
786 	q->buf_addr = buf_addr;
787 	q->buf_len = buf_len;
788 	q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
789 	if (IS_ERR(q->umem)) {
790 		DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
791 		       PTR_ERR(q->umem));
792 		return PTR_ERR(q->umem);
793 	}
794 
795 	fw_pages = ib_umem_page_count(q->umem) <<
796 	    (PAGE_SHIFT - FW_PAGE_SHIFT);
797 
798 	rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
799 	if (rc)
800 		goto err0;
801 
802 	if (alloc_and_init) {
803 		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
804 		if (IS_ERR(q->pbl_tbl)) {
805 			rc = PTR_ERR(q->pbl_tbl);
806 			goto err0;
807 		}
808 		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
809 				   FW_PAGE_SHIFT);
810 	} else {
811 		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
812 		if (!q->pbl_tbl) {
813 			rc = -ENOMEM;
814 			goto err0;
815 		}
816 	}
817 
818 	/* mmap the user address used to store doorbell data for recovery */
819 	return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
820 
821 err0:
822 	ib_umem_release(q->umem);
823 	q->umem = NULL;
824 
825 	return rc;
826 }
827 
828 static inline void qedr_init_cq_params(struct qedr_cq *cq,
829 				       struct qedr_ucontext *ctx,
830 				       struct qedr_dev *dev, int vector,
831 				       int chain_entries, int page_cnt,
832 				       u64 pbl_ptr,
833 				       struct qed_rdma_create_cq_in_params
834 				       *params)
835 {
836 	memset(params, 0, sizeof(*params));
837 	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
838 	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
839 	params->cnq_id = vector;
840 	params->cq_size = chain_entries - 1;
841 	params->dpi = (ctx) ? ctx->dpi : dev->dpi;
842 	params->pbl_num_pages = page_cnt;
843 	params->pbl_ptr = pbl_ptr;
844 	params->pbl_two_level = 0;
845 }
846 
847 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
848 {
849 	cq->db.data.agg_flags = flags;
850 	cq->db.data.value = cpu_to_le32(cons);
851 	writeq(cq->db.raw, cq->db_addr);
852 }
853 
854 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
855 {
856 	struct qedr_cq *cq = get_qedr_cq(ibcq);
857 	unsigned long sflags;
858 	struct qedr_dev *dev;
859 
860 	dev = get_qedr_dev(ibcq->device);
861 
862 	if (cq->destroyed) {
863 		DP_ERR(dev,
864 		       "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
865 		       cq, cq->icid);
866 		return -EINVAL;
867 	}
868 
869 
870 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
871 		return 0;
872 
873 	spin_lock_irqsave(&cq->cq_lock, sflags);
874 
875 	cq->arm_flags = 0;
876 
877 	if (flags & IB_CQ_SOLICITED)
878 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
879 
880 	if (flags & IB_CQ_NEXT_COMP)
881 		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
882 
883 	doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
884 
885 	spin_unlock_irqrestore(&cq->cq_lock, sflags);
886 
887 	return 0;
888 }
889 
890 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
891 		   struct ib_udata *udata)
892 {
893 	struct ib_device *ibdev = ibcq->device;
894 	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
895 		udata, struct qedr_ucontext, ibucontext);
896 	struct qed_rdma_destroy_cq_out_params destroy_oparams;
897 	struct qed_rdma_destroy_cq_in_params destroy_iparams;
898 	struct qed_chain_init_params chain_params = {
899 		.mode		= QED_CHAIN_MODE_PBL,
900 		.intended_use	= QED_CHAIN_USE_TO_CONSUME,
901 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
902 		.elem_size	= sizeof(union rdma_cqe),
903 	};
904 	struct qedr_dev *dev = get_qedr_dev(ibdev);
905 	struct qed_rdma_create_cq_in_params params;
906 	struct qedr_create_cq_ureq ureq = {};
907 	int vector = attr->comp_vector;
908 	int entries = attr->cqe;
909 	struct qedr_cq *cq = get_qedr_cq(ibcq);
910 	int chain_entries;
911 	u32 db_offset;
912 	int page_cnt;
913 	u64 pbl_ptr;
914 	u16 icid;
915 	int rc;
916 
917 	DP_DEBUG(dev, QEDR_MSG_INIT,
918 		 "create_cq: called from %s. entries=%d, vector=%d\n",
919 		 udata ? "User Lib" : "Kernel", entries, vector);
920 
921 	if (entries > QEDR_MAX_CQES) {
922 		DP_ERR(dev,
923 		       "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
924 		       entries, QEDR_MAX_CQES);
925 		return -EINVAL;
926 	}
927 
928 	chain_entries = qedr_align_cq_entries(entries);
929 	chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
930 	chain_params.num_elems = chain_entries;
931 
932 	/* calc db offset. user will add DPI base, kernel will add db addr */
933 	db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
934 
935 	if (udata) {
936 		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
937 							 udata->inlen))) {
938 			DP_ERR(dev,
939 			       "create cq: problem copying data from user space\n");
940 			goto err0;
941 		}
942 
943 		if (!ureq.len) {
944 			DP_ERR(dev,
945 			       "create cq: cannot create a cq with 0 entries\n");
946 			goto err0;
947 		}
948 
949 		cq->cq_type = QEDR_CQ_TYPE_USER;
950 
951 		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
952 					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
953 					  1);
954 		if (rc)
955 			goto err0;
956 
957 		pbl_ptr = cq->q.pbl_tbl->pa;
958 		page_cnt = cq->q.pbl_info.num_pbes;
959 
960 		cq->ibcq.cqe = chain_entries;
961 		cq->q.db_addr = ctx->dpi_addr + db_offset;
962 	} else {
963 		cq->cq_type = QEDR_CQ_TYPE_KERNEL;
964 
965 		rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
966 						   &chain_params);
967 		if (rc)
968 			goto err0;
969 
970 		page_cnt = qed_chain_get_page_cnt(&cq->pbl);
971 		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
972 		cq->ibcq.cqe = cq->pbl.capacity;
973 	}
974 
975 	qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
976 			    pbl_ptr, &params);
977 
978 	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
979 	if (rc)
980 		goto err1;
981 
982 	cq->icid = icid;
983 	cq->sig = QEDR_CQ_MAGIC_NUMBER;
984 	spin_lock_init(&cq->cq_lock);
985 
986 	if (udata) {
987 		rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
988 		if (rc)
989 			goto err2;
990 
991 		rc = qedr_db_recovery_add(dev, cq->q.db_addr,
992 					  &cq->q.db_rec_data->db_data,
993 					  DB_REC_WIDTH_64B,
994 					  DB_REC_USER);
995 		if (rc)
996 			goto err2;
997 
998 	} else {
999 		/* Generate doorbell address. */
1000 		cq->db.data.icid = cq->icid;
1001 		cq->db_addr = dev->db_addr + db_offset;
1002 		cq->db.data.params = DB_AGG_CMD_SET <<
1003 		    RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1004 
1005 		/* point to the very last element, passing it we will toggle */
1006 		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1007 		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1008 		cq->latest_cqe = NULL;
1009 		consume_cqe(cq);
1010 		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1011 
1012 		rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1013 					  DB_REC_WIDTH_64B, DB_REC_KERNEL);
1014 		if (rc)
1015 			goto err2;
1016 	}
1017 
1018 	DP_DEBUG(dev, QEDR_MSG_CQ,
1019 		 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1020 		 cq->icid, cq, params.cq_size);
1021 
1022 	return 0;
1023 
1024 err2:
1025 	destroy_iparams.icid = cq->icid;
1026 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1027 				  &destroy_oparams);
1028 err1:
1029 	if (udata) {
1030 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1031 		ib_umem_release(cq->q.umem);
1032 		if (cq->q.db_mmap_entry)
1033 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1034 	} else {
1035 		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1036 	}
1037 err0:
1038 	return -EINVAL;
1039 }
1040 
1041 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
1042 {
1043 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1044 	struct qedr_cq *cq = get_qedr_cq(ibcq);
1045 
1046 	DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1047 
1048 	return 0;
1049 }
1050 
1051 #define QEDR_DESTROY_CQ_MAX_ITERATIONS		(10)
1052 #define QEDR_DESTROY_CQ_ITER_DURATION		(10)
1053 
1054 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1055 {
1056 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1057 	struct qed_rdma_destroy_cq_out_params oparams;
1058 	struct qed_rdma_destroy_cq_in_params iparams;
1059 	struct qedr_cq *cq = get_qedr_cq(ibcq);
1060 	int iter;
1061 
1062 	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1063 
1064 	cq->destroyed = 1;
1065 
1066 	/* GSIs CQs are handled by driver, so they don't exist in the FW */
1067 	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1068 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1069 		return;
1070 	}
1071 
1072 	iparams.icid = cq->icid;
1073 	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1074 	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1075 
1076 	if (udata) {
1077 		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1078 		ib_umem_release(cq->q.umem);
1079 
1080 		if (cq->q.db_rec_data) {
1081 			qedr_db_recovery_del(dev, cq->q.db_addr,
1082 					     &cq->q.db_rec_data->db_data);
1083 			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1084 		}
1085 	} else {
1086 		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1087 	}
1088 
1089 	/* We don't want the IRQ handler to handle a non-existing CQ so we
1090 	 * wait until all CNQ interrupts, if any, are received. This will always
1091 	 * happen and will always happen very fast. If not, then a serious error
1092 	 * has occured. That is why we can use a long delay.
1093 	 * We spin for a short time so we don’t lose time on context switching
1094 	 * in case all the completions are handled in that span. Otherwise
1095 	 * we sleep for a while and check again. Since the CNQ may be
1096 	 * associated with (only) the current CPU we use msleep to allow the
1097 	 * current CPU to be freed.
1098 	 * The CNQ notification is increased in qedr_irq_handler().
1099 	 */
1100 	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1101 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1102 		udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1103 		iter--;
1104 	}
1105 
1106 	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1107 	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1108 		msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1109 		iter--;
1110 	}
1111 
1112 	/* Note that we don't need to have explicit code to wait for the
1113 	 * completion of the event handler because it is invoked from the EQ.
1114 	 * Since the destroy CQ ramrod has also been received on the EQ we can
1115 	 * be certain that there's no event handler in process.
1116 	 */
1117 }
1118 
1119 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1120 					  struct ib_qp_attr *attr,
1121 					  int attr_mask,
1122 					  struct qed_rdma_modify_qp_in_params
1123 					  *qp_params)
1124 {
1125 	const struct ib_gid_attr *gid_attr;
1126 	enum rdma_network_type nw_type;
1127 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1128 	u32 ipv4_addr;
1129 	int ret;
1130 	int i;
1131 
1132 	gid_attr = grh->sgid_attr;
1133 	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1134 	if (ret)
1135 		return ret;
1136 
1137 	nw_type = rdma_gid_attr_network_type(gid_attr);
1138 	switch (nw_type) {
1139 	case RDMA_NETWORK_IPV6:
1140 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1141 		       sizeof(qp_params->sgid));
1142 		memcpy(&qp_params->dgid.bytes[0],
1143 		       &grh->dgid,
1144 		       sizeof(qp_params->dgid));
1145 		qp_params->roce_mode = ROCE_V2_IPV6;
1146 		SET_FIELD(qp_params->modify_flags,
1147 			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1148 		break;
1149 	case RDMA_NETWORK_IB:
1150 		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1151 		       sizeof(qp_params->sgid));
1152 		memcpy(&qp_params->dgid.bytes[0],
1153 		       &grh->dgid,
1154 		       sizeof(qp_params->dgid));
1155 		qp_params->roce_mode = ROCE_V1;
1156 		break;
1157 	case RDMA_NETWORK_IPV4:
1158 		memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1159 		memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1160 		ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1161 		qp_params->sgid.ipv4_addr = ipv4_addr;
1162 		ipv4_addr =
1163 		    qedr_get_ipv4_from_gid(grh->dgid.raw);
1164 		qp_params->dgid.ipv4_addr = ipv4_addr;
1165 		SET_FIELD(qp_params->modify_flags,
1166 			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1167 		qp_params->roce_mode = ROCE_V2_IPV4;
1168 		break;
1169 	}
1170 
1171 	for (i = 0; i < 4; i++) {
1172 		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1173 		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1174 	}
1175 
1176 	if (qp_params->vlan_id >= VLAN_CFI_MASK)
1177 		qp_params->vlan_id = 0;
1178 
1179 	return 0;
1180 }
1181 
1182 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1183 			       struct ib_qp_init_attr *attrs,
1184 			       struct ib_udata *udata)
1185 {
1186 	struct qedr_device_attr *qattr = &dev->attr;
1187 
1188 	/* QP0... attrs->qp_type == IB_QPT_GSI */
1189 	if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1190 		DP_DEBUG(dev, QEDR_MSG_QP,
1191 			 "create qp: unsupported qp type=0x%x requested\n",
1192 			 attrs->qp_type);
1193 		return -EOPNOTSUPP;
1194 	}
1195 
1196 	if (attrs->cap.max_send_wr > qattr->max_sqe) {
1197 		DP_ERR(dev,
1198 		       "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1199 		       attrs->cap.max_send_wr, qattr->max_sqe);
1200 		return -EINVAL;
1201 	}
1202 
1203 	if (attrs->cap.max_inline_data > qattr->max_inline) {
1204 		DP_ERR(dev,
1205 		       "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1206 		       attrs->cap.max_inline_data, qattr->max_inline);
1207 		return -EINVAL;
1208 	}
1209 
1210 	if (attrs->cap.max_send_sge > qattr->max_sge) {
1211 		DP_ERR(dev,
1212 		       "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1213 		       attrs->cap.max_send_sge, qattr->max_sge);
1214 		return -EINVAL;
1215 	}
1216 
1217 	if (attrs->cap.max_recv_sge > qattr->max_sge) {
1218 		DP_ERR(dev,
1219 		       "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1220 		       attrs->cap.max_recv_sge, qattr->max_sge);
1221 		return -EINVAL;
1222 	}
1223 
1224 	/* Unprivileged user space cannot create special QP */
1225 	if (udata && attrs->qp_type == IB_QPT_GSI) {
1226 		DP_ERR(dev,
1227 		       "create qp: userspace can't create special QPs of type=0x%x\n",
1228 		       attrs->qp_type);
1229 		return -EINVAL;
1230 	}
1231 
1232 	return 0;
1233 }
1234 
1235 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1236 			       struct qedr_srq *srq, struct ib_udata *udata)
1237 {
1238 	struct qedr_create_srq_uresp uresp = {};
1239 	int rc;
1240 
1241 	uresp.srq_id = srq->srq_id;
1242 
1243 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1244 	if (rc)
1245 		DP_ERR(dev, "create srq: problem copying data to user space\n");
1246 
1247 	return rc;
1248 }
1249 
1250 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1251 			      struct qedr_create_qp_uresp *uresp,
1252 			      struct qedr_qp *qp)
1253 {
1254 	/* iWARP requires two doorbells per RQ. */
1255 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1256 		uresp->rq_db_offset =
1257 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1258 		uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1259 	} else {
1260 		uresp->rq_db_offset =
1261 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1262 	}
1263 
1264 	uresp->rq_icid = qp->icid;
1265 	if (qp->urq.db_mmap_entry)
1266 		uresp->rq_db_rec_addr =
1267 			rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1268 }
1269 
1270 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1271 			       struct qedr_create_qp_uresp *uresp,
1272 			       struct qedr_qp *qp)
1273 {
1274 	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1275 
1276 	/* iWARP uses the same cid for rq and sq */
1277 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1278 		uresp->sq_icid = qp->icid;
1279 	else
1280 		uresp->sq_icid = qp->icid + 1;
1281 
1282 	if (qp->usq.db_mmap_entry)
1283 		uresp->sq_db_rec_addr =
1284 			rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1285 }
1286 
1287 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1288 			      struct qedr_qp *qp, struct ib_udata *udata,
1289 			      struct qedr_create_qp_uresp *uresp)
1290 {
1291 	int rc;
1292 
1293 	memset(uresp, 0, sizeof(*uresp));
1294 	qedr_copy_sq_uresp(dev, uresp, qp);
1295 	qedr_copy_rq_uresp(dev, uresp, qp);
1296 
1297 	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1298 	uresp->qp_id = qp->qp_id;
1299 
1300 	rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1301 	if (rc)
1302 		DP_ERR(dev,
1303 		       "create qp: failed a copy to user space with qp icid=0x%x.\n",
1304 		       qp->icid);
1305 
1306 	return rc;
1307 }
1308 
1309 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1310 				      struct qedr_qp *qp,
1311 				      struct qedr_pd *pd,
1312 				      struct ib_qp_init_attr *attrs)
1313 {
1314 	spin_lock_init(&qp->q_lock);
1315 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1316 		kref_init(&qp->refcnt);
1317 		init_completion(&qp->iwarp_cm_comp);
1318 	}
1319 	qp->pd = pd;
1320 	qp->qp_type = attrs->qp_type;
1321 	qp->max_inline_data = attrs->cap.max_inline_data;
1322 	qp->sq.max_sges = attrs->cap.max_send_sge;
1323 	qp->state = QED_ROCE_QP_STATE_RESET;
1324 	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1325 	qp->sq_cq = get_qedr_cq(attrs->send_cq);
1326 	qp->dev = dev;
1327 
1328 	if (attrs->srq) {
1329 		qp->srq = get_qedr_srq(attrs->srq);
1330 	} else {
1331 		qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1332 		qp->rq.max_sges = attrs->cap.max_recv_sge;
1333 		DP_DEBUG(dev, QEDR_MSG_QP,
1334 			 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1335 			 qp->rq.max_sges, qp->rq_cq->icid);
1336 	}
1337 
1338 	DP_DEBUG(dev, QEDR_MSG_QP,
1339 		 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1340 		 pd->pd_id, qp->qp_type, qp->max_inline_data,
1341 		 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1342 	DP_DEBUG(dev, QEDR_MSG_QP,
1343 		 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1344 		 qp->sq.max_sges, qp->sq_cq->icid);
1345 }
1346 
1347 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1348 {
1349 	int rc;
1350 
1351 	qp->sq.db = dev->db_addr +
1352 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1353 	qp->sq.db_data.data.icid = qp->icid + 1;
1354 	rc = qedr_db_recovery_add(dev, qp->sq.db,
1355 				  &qp->sq.db_data,
1356 				  DB_REC_WIDTH_32B,
1357 				  DB_REC_KERNEL);
1358 	if (rc)
1359 		return rc;
1360 
1361 	if (!qp->srq) {
1362 		qp->rq.db = dev->db_addr +
1363 			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1364 		qp->rq.db_data.data.icid = qp->icid;
1365 
1366 		rc = qedr_db_recovery_add(dev, qp->rq.db,
1367 					  &qp->rq.db_data,
1368 					  DB_REC_WIDTH_32B,
1369 					  DB_REC_KERNEL);
1370 		if (rc)
1371 			qedr_db_recovery_del(dev, qp->sq.db,
1372 					     &qp->sq.db_data);
1373 	}
1374 
1375 	return rc;
1376 }
1377 
1378 static int qedr_check_srq_params(struct qedr_dev *dev,
1379 				 struct ib_srq_init_attr *attrs,
1380 				 struct ib_udata *udata)
1381 {
1382 	struct qedr_device_attr *qattr = &dev->attr;
1383 
1384 	if (attrs->attr.max_wr > qattr->max_srq_wr) {
1385 		DP_ERR(dev,
1386 		       "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1387 		       attrs->attr.max_wr, qattr->max_srq_wr);
1388 		return -EINVAL;
1389 	}
1390 
1391 	if (attrs->attr.max_sge > qattr->max_sge) {
1392 		DP_ERR(dev,
1393 		       "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1394 		       attrs->attr.max_sge, qattr->max_sge);
1395 		return -EINVAL;
1396 	}
1397 
1398 	return 0;
1399 }
1400 
1401 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1402 {
1403 	qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1404 	ib_umem_release(srq->usrq.umem);
1405 	ib_umem_release(srq->prod_umem);
1406 }
1407 
1408 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1409 {
1410 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1411 	struct qedr_dev *dev = srq->dev;
1412 
1413 	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1414 
1415 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1416 			  hw_srq->virt_prod_pair_addr,
1417 			  hw_srq->phy_prod_pair_addr);
1418 }
1419 
1420 static int qedr_init_srq_user_params(struct ib_udata *udata,
1421 				     struct qedr_srq *srq,
1422 				     struct qedr_create_srq_ureq *ureq,
1423 				     int access)
1424 {
1425 	struct scatterlist *sg;
1426 	int rc;
1427 
1428 	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1429 				  ureq->srq_len, false, access, 1);
1430 	if (rc)
1431 		return rc;
1432 
1433 	srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1434 				     sizeof(struct rdma_srq_producers), access);
1435 	if (IS_ERR(srq->prod_umem)) {
1436 		qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1437 		ib_umem_release(srq->usrq.umem);
1438 		DP_ERR(srq->dev,
1439 		       "create srq: failed ib_umem_get for producer, got %ld\n",
1440 		       PTR_ERR(srq->prod_umem));
1441 		return PTR_ERR(srq->prod_umem);
1442 	}
1443 
1444 	sg = srq->prod_umem->sg_head.sgl;
1445 	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1446 
1447 	return 0;
1448 }
1449 
1450 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1451 					struct qedr_dev *dev,
1452 					struct ib_srq_init_attr *init_attr)
1453 {
1454 	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1455 	struct qed_chain_init_params params = {
1456 		.mode		= QED_CHAIN_MODE_PBL,
1457 		.intended_use	= QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1458 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1459 		.elem_size	= QEDR_SRQ_WQE_ELEM_SIZE,
1460 	};
1461 	dma_addr_t phy_prod_pair_addr;
1462 	u32 num_elems;
1463 	void *va;
1464 	int rc;
1465 
1466 	va = dma_alloc_coherent(&dev->pdev->dev,
1467 				sizeof(struct rdma_srq_producers),
1468 				&phy_prod_pair_addr, GFP_KERNEL);
1469 	if (!va) {
1470 		DP_ERR(dev,
1471 		       "create srq: failed to allocate dma memory for producer\n");
1472 		return -ENOMEM;
1473 	}
1474 
1475 	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1476 	hw_srq->virt_prod_pair_addr = va;
1477 
1478 	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1479 	params.num_elems = num_elems;
1480 
1481 	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1482 	if (rc)
1483 		goto err0;
1484 
1485 	hw_srq->num_elems = num_elems;
1486 
1487 	return 0;
1488 
1489 err0:
1490 	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1491 			  va, phy_prod_pair_addr);
1492 	return rc;
1493 }
1494 
1495 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1496 		    struct ib_udata *udata)
1497 {
1498 	struct qed_rdma_destroy_srq_in_params destroy_in_params;
1499 	struct qed_rdma_create_srq_in_params in_params = {};
1500 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1501 	struct qed_rdma_create_srq_out_params out_params;
1502 	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1503 	struct qedr_create_srq_ureq ureq = {};
1504 	u64 pbl_base_addr, phy_prod_pair_addr;
1505 	struct qedr_srq_hwq_info *hw_srq;
1506 	u32 page_cnt, page_size;
1507 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1508 	int rc = 0;
1509 
1510 	DP_DEBUG(dev, QEDR_MSG_QP,
1511 		 "create SRQ called from %s (pd %p)\n",
1512 		 (udata) ? "User lib" : "kernel", pd);
1513 
1514 	rc = qedr_check_srq_params(dev, init_attr, udata);
1515 	if (rc)
1516 		return -EINVAL;
1517 
1518 	srq->dev = dev;
1519 	hw_srq = &srq->hw_srq;
1520 	spin_lock_init(&srq->lock);
1521 
1522 	hw_srq->max_wr = init_attr->attr.max_wr;
1523 	hw_srq->max_sges = init_attr->attr.max_sge;
1524 
1525 	if (udata) {
1526 		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1527 							 udata->inlen))) {
1528 			DP_ERR(dev,
1529 			       "create srq: problem copying data from user space\n");
1530 			goto err0;
1531 		}
1532 
1533 		rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1534 		if (rc)
1535 			goto err0;
1536 
1537 		page_cnt = srq->usrq.pbl_info.num_pbes;
1538 		pbl_base_addr = srq->usrq.pbl_tbl->pa;
1539 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1540 		page_size = PAGE_SIZE;
1541 	} else {
1542 		struct qed_chain *pbl;
1543 
1544 		rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1545 		if (rc)
1546 			goto err0;
1547 
1548 		pbl = &hw_srq->pbl;
1549 		page_cnt = qed_chain_get_page_cnt(pbl);
1550 		pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1551 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1552 		page_size = QED_CHAIN_PAGE_SIZE;
1553 	}
1554 
1555 	in_params.pd_id = pd->pd_id;
1556 	in_params.pbl_base_addr = pbl_base_addr;
1557 	in_params.prod_pair_addr = phy_prod_pair_addr;
1558 	in_params.num_pages = page_cnt;
1559 	in_params.page_size = page_size;
1560 
1561 	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1562 	if (rc)
1563 		goto err1;
1564 
1565 	srq->srq_id = out_params.srq_id;
1566 
1567 	if (udata) {
1568 		rc = qedr_copy_srq_uresp(dev, srq, udata);
1569 		if (rc)
1570 			goto err2;
1571 	}
1572 
1573 	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1574 	if (rc)
1575 		goto err2;
1576 
1577 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1578 		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1579 	return 0;
1580 
1581 err2:
1582 	destroy_in_params.srq_id = srq->srq_id;
1583 
1584 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1585 err1:
1586 	if (udata)
1587 		qedr_free_srq_user_params(srq);
1588 	else
1589 		qedr_free_srq_kernel_params(srq);
1590 err0:
1591 	return -EFAULT;
1592 }
1593 
1594 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1595 {
1596 	struct qed_rdma_destroy_srq_in_params in_params = {};
1597 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1598 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1599 
1600 	xa_erase_irq(&dev->srqs, srq->srq_id);
1601 	in_params.srq_id = srq->srq_id;
1602 	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1603 
1604 	if (ibsrq->uobject)
1605 		qedr_free_srq_user_params(srq);
1606 	else
1607 		qedr_free_srq_kernel_params(srq);
1608 
1609 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1610 		 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1611 		 srq->srq_id);
1612 }
1613 
1614 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1615 		    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1616 {
1617 	struct qed_rdma_modify_srq_in_params in_params = {};
1618 	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1619 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1620 	int rc;
1621 
1622 	if (attr_mask & IB_SRQ_MAX_WR) {
1623 		DP_ERR(dev,
1624 		       "modify srq: invalid attribute mask=0x%x specified for %p\n",
1625 		       attr_mask, srq);
1626 		return -EINVAL;
1627 	}
1628 
1629 	if (attr_mask & IB_SRQ_LIMIT) {
1630 		if (attr->srq_limit >= srq->hw_srq.max_wr) {
1631 			DP_ERR(dev,
1632 			       "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1633 			       attr->srq_limit, srq->hw_srq.max_wr);
1634 			return -EINVAL;
1635 		}
1636 
1637 		in_params.srq_id = srq->srq_id;
1638 		in_params.wqe_limit = attr->srq_limit;
1639 		rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1640 		if (rc)
1641 			return rc;
1642 	}
1643 
1644 	srq->srq_limit = attr->srq_limit;
1645 
1646 	DP_DEBUG(dev, QEDR_MSG_SRQ,
1647 		 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1648 
1649 	return 0;
1650 }
1651 
1652 static inline void
1653 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1654 			      struct qedr_pd *pd,
1655 			      struct qedr_qp *qp,
1656 			      struct ib_qp_init_attr *attrs,
1657 			      bool fmr_and_reserved_lkey,
1658 			      struct qed_rdma_create_qp_in_params *params)
1659 {
1660 	/* QP handle to be written in an async event */
1661 	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1662 	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1663 
1664 	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1665 	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1666 	params->pd = pd->pd_id;
1667 	params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1668 	params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1669 	params->stats_queue = 0;
1670 	params->srq_id = 0;
1671 	params->use_srq = false;
1672 
1673 	if (!qp->srq) {
1674 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1675 
1676 	} else {
1677 		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1678 		params->srq_id = qp->srq->srq_id;
1679 		params->use_srq = true;
1680 	}
1681 }
1682 
1683 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1684 {
1685 	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1686 		 "qp=%p. "
1687 		 "sq_addr=0x%llx, "
1688 		 "sq_len=%zd, "
1689 		 "rq_addr=0x%llx, "
1690 		 "rq_len=%zd"
1691 		 "\n",
1692 		 qp,
1693 		 qp->usq.buf_addr,
1694 		 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1695 }
1696 
1697 static inline void
1698 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1699 			    struct qedr_qp *qp,
1700 			    struct qed_rdma_create_qp_out_params *out_params)
1701 {
1702 	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1703 	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1704 
1705 	qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1706 			   &qp->usq.pbl_info, FW_PAGE_SHIFT);
1707 	if (!qp->srq) {
1708 		qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1709 		qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1710 	}
1711 
1712 	qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1713 			   &qp->urq.pbl_info, FW_PAGE_SHIFT);
1714 }
1715 
1716 static void qedr_cleanup_user(struct qedr_dev *dev,
1717 			      struct qedr_ucontext *ctx,
1718 			      struct qedr_qp *qp)
1719 {
1720 	ib_umem_release(qp->usq.umem);
1721 	qp->usq.umem = NULL;
1722 
1723 	ib_umem_release(qp->urq.umem);
1724 	qp->urq.umem = NULL;
1725 
1726 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
1727 		qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1728 		qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1729 	} else {
1730 		kfree(qp->usq.pbl_tbl);
1731 		kfree(qp->urq.pbl_tbl);
1732 	}
1733 
1734 	if (qp->usq.db_rec_data) {
1735 		qedr_db_recovery_del(dev, qp->usq.db_addr,
1736 				     &qp->usq.db_rec_data->db_data);
1737 		rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1738 	}
1739 
1740 	if (qp->urq.db_rec_data) {
1741 		qedr_db_recovery_del(dev, qp->urq.db_addr,
1742 				     &qp->urq.db_rec_data->db_data);
1743 		rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1744 	}
1745 
1746 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1747 		qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1748 				     &qp->urq.db_rec_db2_data);
1749 }
1750 
1751 static int qedr_create_user_qp(struct qedr_dev *dev,
1752 			       struct qedr_qp *qp,
1753 			       struct ib_pd *ibpd,
1754 			       struct ib_udata *udata,
1755 			       struct ib_qp_init_attr *attrs)
1756 {
1757 	struct qed_rdma_create_qp_in_params in_params;
1758 	struct qed_rdma_create_qp_out_params out_params;
1759 	struct qedr_pd *pd = get_qedr_pd(ibpd);
1760 	struct qedr_create_qp_uresp uresp;
1761 	struct qedr_ucontext *ctx = pd ? pd->uctx : NULL;
1762 	struct qedr_create_qp_ureq ureq;
1763 	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1764 	int rc = -EINVAL;
1765 
1766 	qp->create_type = QEDR_QP_CREATE_USER;
1767 	memset(&ureq, 0, sizeof(ureq));
1768 	rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen));
1769 	if (rc) {
1770 		DP_ERR(dev, "Problem copying data from user space\n");
1771 		return rc;
1772 	}
1773 
1774 	/* SQ - read access only (0) */
1775 	rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1776 				  ureq.sq_len, true, 0, alloc_and_init);
1777 	if (rc)
1778 		return rc;
1779 
1780 	if (!qp->srq) {
1781 		/* RQ - read access only (0) */
1782 		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1783 					  ureq.rq_len, true, 0, alloc_and_init);
1784 		if (rc)
1785 			return rc;
1786 	}
1787 
1788 	memset(&in_params, 0, sizeof(in_params));
1789 	qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1790 	in_params.qp_handle_lo = ureq.qp_handle_lo;
1791 	in_params.qp_handle_hi = ureq.qp_handle_hi;
1792 	in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1793 	in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1794 	if (!qp->srq) {
1795 		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1796 		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1797 	}
1798 
1799 	if (ctx)
1800 		SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1801 
1802 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1803 					      &in_params, &out_params);
1804 
1805 	if (!qp->qed_qp) {
1806 		rc = -ENOMEM;
1807 		goto err1;
1808 	}
1809 
1810 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1811 		qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1812 
1813 	qp->qp_id = out_params.qp_id;
1814 	qp->icid = out_params.icid;
1815 
1816 	rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1817 	if (rc)
1818 		goto err;
1819 
1820 	/* db offset was calculated in copy_qp_uresp, now set in the user q */
1821 	ctx = pd->uctx;
1822 	qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1823 	qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1824 
1825 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1826 		qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1827 
1828 		/* calculate the db_rec_db2 data since it is constant so no
1829 		 *  need to reflect from user
1830 		 */
1831 		qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1832 		qp->urq.db_rec_db2_data.data.value =
1833 			cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1834 	}
1835 
1836 	rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1837 				  &qp->usq.db_rec_data->db_data,
1838 				  DB_REC_WIDTH_32B,
1839 				  DB_REC_USER);
1840 	if (rc)
1841 		goto err;
1842 
1843 	rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1844 				  &qp->urq.db_rec_data->db_data,
1845 				  DB_REC_WIDTH_32B,
1846 				  DB_REC_USER);
1847 	if (rc)
1848 		goto err;
1849 
1850 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1851 		rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1852 					  &qp->urq.db_rec_db2_data,
1853 					  DB_REC_WIDTH_32B,
1854 					  DB_REC_USER);
1855 		if (rc)
1856 			goto err;
1857 	}
1858 	qedr_qp_user_print(dev, qp);
1859 
1860 	return rc;
1861 err:
1862 	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1863 	if (rc)
1864 		DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1865 
1866 err1:
1867 	qedr_cleanup_user(dev, ctx, qp);
1868 	return rc;
1869 }
1870 
1871 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1872 {
1873 	int rc;
1874 
1875 	qp->sq.db = dev->db_addr +
1876 	    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1877 	qp->sq.db_data.data.icid = qp->icid;
1878 
1879 	rc = qedr_db_recovery_add(dev, qp->sq.db,
1880 				  &qp->sq.db_data,
1881 				  DB_REC_WIDTH_32B,
1882 				  DB_REC_KERNEL);
1883 	if (rc)
1884 		return rc;
1885 
1886 	qp->rq.db = dev->db_addr +
1887 		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1888 	qp->rq.db_data.data.icid = qp->icid;
1889 	qp->rq.iwarp_db2 = dev->db_addr +
1890 			   DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1891 	qp->rq.iwarp_db2_data.data.icid = qp->icid;
1892 	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1893 
1894 	rc = qedr_db_recovery_add(dev, qp->rq.db,
1895 				  &qp->rq.db_data,
1896 				  DB_REC_WIDTH_32B,
1897 				  DB_REC_KERNEL);
1898 	if (rc)
1899 		return rc;
1900 
1901 	rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
1902 				  &qp->rq.iwarp_db2_data,
1903 				  DB_REC_WIDTH_32B,
1904 				  DB_REC_KERNEL);
1905 	return rc;
1906 }
1907 
1908 static int
1909 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1910 			   struct qedr_qp *qp,
1911 			   struct qed_rdma_create_qp_in_params *in_params,
1912 			   u32 n_sq_elems, u32 n_rq_elems)
1913 {
1914 	struct qed_rdma_create_qp_out_params out_params;
1915 	struct qed_chain_init_params params = {
1916 		.mode		= QED_CHAIN_MODE_PBL,
1917 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1918 	};
1919 	int rc;
1920 
1921 	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1922 	params.num_elems = n_sq_elems;
1923 	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
1924 
1925 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
1926 	if (rc)
1927 		return rc;
1928 
1929 	in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1930 	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1931 
1932 	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1933 	params.num_elems = n_rq_elems;
1934 	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1935 
1936 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
1937 	if (rc)
1938 		return rc;
1939 
1940 	in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1941 	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1942 
1943 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1944 					      in_params, &out_params);
1945 
1946 	if (!qp->qed_qp)
1947 		return -EINVAL;
1948 
1949 	qp->qp_id = out_params.qp_id;
1950 	qp->icid = out_params.icid;
1951 
1952 	return qedr_set_roce_db_info(dev, qp);
1953 }
1954 
1955 static int
1956 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1957 			    struct qedr_qp *qp,
1958 			    struct qed_rdma_create_qp_in_params *in_params,
1959 			    u32 n_sq_elems, u32 n_rq_elems)
1960 {
1961 	struct qed_rdma_create_qp_out_params out_params;
1962 	struct qed_chain_init_params params = {
1963 		.mode		= QED_CHAIN_MODE_PBL,
1964 		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1965 	};
1966 	int rc;
1967 
1968 	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1969 						     QEDR_SQE_ELEMENT_SIZE,
1970 						     QED_CHAIN_PAGE_SIZE,
1971 						     QED_CHAIN_MODE_PBL);
1972 	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1973 						     QEDR_RQE_ELEMENT_SIZE,
1974 						     QED_CHAIN_PAGE_SIZE,
1975 						     QED_CHAIN_MODE_PBL);
1976 
1977 	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1978 					      in_params, &out_params);
1979 
1980 	if (!qp->qed_qp)
1981 		return -EINVAL;
1982 
1983 	/* Now we allocate the chain */
1984 
1985 	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
1986 	params.num_elems = n_sq_elems;
1987 	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
1988 	params.ext_pbl_virt = out_params.sq_pbl_virt;
1989 	params.ext_pbl_phys = out_params.sq_pbl_phys;
1990 
1991 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
1992 	if (rc)
1993 		goto err;
1994 
1995 	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
1996 	params.num_elems = n_rq_elems;
1997 	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
1998 	params.ext_pbl_virt = out_params.rq_pbl_virt;
1999 	params.ext_pbl_phys = out_params.rq_pbl_phys;
2000 
2001 	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2002 	if (rc)
2003 		goto err;
2004 
2005 	qp->qp_id = out_params.qp_id;
2006 	qp->icid = out_params.icid;
2007 
2008 	return qedr_set_iwarp_db_info(dev, qp);
2009 
2010 err:
2011 	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2012 
2013 	return rc;
2014 }
2015 
2016 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2017 {
2018 	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2019 	kfree(qp->wqe_wr_id);
2020 
2021 	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2022 	kfree(qp->rqe_wr_id);
2023 
2024 	/* GSI qp is not registered to db mechanism so no need to delete */
2025 	if (qp->qp_type == IB_QPT_GSI)
2026 		return;
2027 
2028 	qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2029 
2030 	if (!qp->srq) {
2031 		qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2032 
2033 		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2034 			qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2035 					     &qp->rq.iwarp_db2_data);
2036 	}
2037 }
2038 
2039 static int qedr_create_kernel_qp(struct qedr_dev *dev,
2040 				 struct qedr_qp *qp,
2041 				 struct ib_pd *ibpd,
2042 				 struct ib_qp_init_attr *attrs)
2043 {
2044 	struct qed_rdma_create_qp_in_params in_params;
2045 	struct qedr_pd *pd = get_qedr_pd(ibpd);
2046 	int rc = -EINVAL;
2047 	u32 n_rq_elems;
2048 	u32 n_sq_elems;
2049 	u32 n_sq_entries;
2050 
2051 	memset(&in_params, 0, sizeof(in_params));
2052 	qp->create_type = QEDR_QP_CREATE_KERNEL;
2053 
2054 	/* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2055 	 * the ring. The ring should allow at least a single WR, even if the
2056 	 * user requested none, due to allocation issues.
2057 	 * We should add an extra WR since the prod and cons indices of
2058 	 * wqe_wr_id are managed in such a way that the WQ is considered full
2059 	 * when (prod+1)%max_wr==cons. We currently don't do that because we
2060 	 * double the number of entries due an iSER issue that pushes far more
2061 	 * WRs than indicated. If we decline its ib_post_send() then we get
2062 	 * error prints in the dmesg we'd like to avoid.
2063 	 */
2064 	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2065 			      dev->attr.max_sqe);
2066 
2067 	qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2068 				GFP_KERNEL);
2069 	if (!qp->wqe_wr_id) {
2070 		DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2071 		return -ENOMEM;
2072 	}
2073 
2074 	/* QP handle to be written in CQE */
2075 	in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2076 	in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2077 
2078 	/* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2079 	 * the ring. There ring should allow at least a single WR, even if the
2080 	 * user requested none, due to allocation issues.
2081 	 */
2082 	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2083 
2084 	/* Allocate driver internal RQ array */
2085 	qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2086 				GFP_KERNEL);
2087 	if (!qp->rqe_wr_id) {
2088 		DP_ERR(dev,
2089 		       "create qp: failed RQ shadow memory allocation\n");
2090 		kfree(qp->wqe_wr_id);
2091 		return -ENOMEM;
2092 	}
2093 
2094 	qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2095 
2096 	n_sq_entries = attrs->cap.max_send_wr;
2097 	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2098 	n_sq_entries = max_t(u32, n_sq_entries, 1);
2099 	n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2100 
2101 	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2102 
2103 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2104 		rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2105 						 n_sq_elems, n_rq_elems);
2106 	else
2107 		rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2108 						n_sq_elems, n_rq_elems);
2109 	if (rc)
2110 		qedr_cleanup_kernel(dev, qp);
2111 
2112 	return rc;
2113 }
2114 
2115 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
2116 			     struct ib_qp_init_attr *attrs,
2117 			     struct ib_udata *udata)
2118 {
2119 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2120 	struct qedr_pd *pd = get_qedr_pd(ibpd);
2121 	struct qedr_qp *qp;
2122 	struct ib_qp *ibqp;
2123 	int rc = 0;
2124 
2125 	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2126 		 udata ? "user library" : "kernel", pd);
2127 
2128 	rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2129 	if (rc)
2130 		return ERR_PTR(rc);
2131 
2132 	DP_DEBUG(dev, QEDR_MSG_QP,
2133 		 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2134 		 udata ? "user library" : "kernel", attrs->event_handler, pd,
2135 		 get_qedr_cq(attrs->send_cq),
2136 		 get_qedr_cq(attrs->send_cq)->icid,
2137 		 get_qedr_cq(attrs->recv_cq),
2138 		 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2139 
2140 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2141 	if (!qp) {
2142 		DP_ERR(dev, "create qp: failed allocating memory\n");
2143 		return ERR_PTR(-ENOMEM);
2144 	}
2145 
2146 	qedr_set_common_qp_params(dev, qp, pd, attrs);
2147 
2148 	if (attrs->qp_type == IB_QPT_GSI) {
2149 		ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2150 		if (IS_ERR(ibqp))
2151 			kfree(qp);
2152 		return ibqp;
2153 	}
2154 
2155 	if (udata)
2156 		rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2157 	else
2158 		rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2159 
2160 	if (rc)
2161 		goto err;
2162 
2163 	qp->ibqp.qp_num = qp->qp_id;
2164 
2165 	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2166 		rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2167 		if (rc)
2168 			goto err;
2169 	}
2170 
2171 	return &qp->ibqp;
2172 
2173 err:
2174 	kfree(qp);
2175 
2176 	return ERR_PTR(-EFAULT);
2177 }
2178 
2179 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2180 {
2181 	switch (qp_state) {
2182 	case QED_ROCE_QP_STATE_RESET:
2183 		return IB_QPS_RESET;
2184 	case QED_ROCE_QP_STATE_INIT:
2185 		return IB_QPS_INIT;
2186 	case QED_ROCE_QP_STATE_RTR:
2187 		return IB_QPS_RTR;
2188 	case QED_ROCE_QP_STATE_RTS:
2189 		return IB_QPS_RTS;
2190 	case QED_ROCE_QP_STATE_SQD:
2191 		return IB_QPS_SQD;
2192 	case QED_ROCE_QP_STATE_ERR:
2193 		return IB_QPS_ERR;
2194 	case QED_ROCE_QP_STATE_SQE:
2195 		return IB_QPS_SQE;
2196 	}
2197 	return IB_QPS_ERR;
2198 }
2199 
2200 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2201 					enum ib_qp_state qp_state)
2202 {
2203 	switch (qp_state) {
2204 	case IB_QPS_RESET:
2205 		return QED_ROCE_QP_STATE_RESET;
2206 	case IB_QPS_INIT:
2207 		return QED_ROCE_QP_STATE_INIT;
2208 	case IB_QPS_RTR:
2209 		return QED_ROCE_QP_STATE_RTR;
2210 	case IB_QPS_RTS:
2211 		return QED_ROCE_QP_STATE_RTS;
2212 	case IB_QPS_SQD:
2213 		return QED_ROCE_QP_STATE_SQD;
2214 	case IB_QPS_ERR:
2215 		return QED_ROCE_QP_STATE_ERR;
2216 	default:
2217 		return QED_ROCE_QP_STATE_ERR;
2218 	}
2219 }
2220 
2221 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2222 {
2223 	qed_chain_reset(&qph->pbl);
2224 	qph->prod = 0;
2225 	qph->cons = 0;
2226 	qph->wqe_cons = 0;
2227 	qph->db_data.data.value = cpu_to_le16(0);
2228 }
2229 
2230 static int qedr_update_qp_state(struct qedr_dev *dev,
2231 				struct qedr_qp *qp,
2232 				enum qed_roce_qp_state cur_state,
2233 				enum qed_roce_qp_state new_state)
2234 {
2235 	int status = 0;
2236 
2237 	if (new_state == cur_state)
2238 		return 0;
2239 
2240 	switch (cur_state) {
2241 	case QED_ROCE_QP_STATE_RESET:
2242 		switch (new_state) {
2243 		case QED_ROCE_QP_STATE_INIT:
2244 			qp->prev_wqe_size = 0;
2245 			qedr_reset_qp_hwq_info(&qp->sq);
2246 			qedr_reset_qp_hwq_info(&qp->rq);
2247 			break;
2248 		default:
2249 			status = -EINVAL;
2250 			break;
2251 		}
2252 		break;
2253 	case QED_ROCE_QP_STATE_INIT:
2254 		switch (new_state) {
2255 		case QED_ROCE_QP_STATE_RTR:
2256 			/* Update doorbell (in case post_recv was
2257 			 * done before move to RTR)
2258 			 */
2259 
2260 			if (rdma_protocol_roce(&dev->ibdev, 1)) {
2261 				writel(qp->rq.db_data.raw, qp->rq.db);
2262 			}
2263 			break;
2264 		case QED_ROCE_QP_STATE_ERR:
2265 			break;
2266 		default:
2267 			/* Invalid state change. */
2268 			status = -EINVAL;
2269 			break;
2270 		}
2271 		break;
2272 	case QED_ROCE_QP_STATE_RTR:
2273 		/* RTR->XXX */
2274 		switch (new_state) {
2275 		case QED_ROCE_QP_STATE_RTS:
2276 			break;
2277 		case QED_ROCE_QP_STATE_ERR:
2278 			break;
2279 		default:
2280 			/* Invalid state change. */
2281 			status = -EINVAL;
2282 			break;
2283 		}
2284 		break;
2285 	case QED_ROCE_QP_STATE_RTS:
2286 		/* RTS->XXX */
2287 		switch (new_state) {
2288 		case QED_ROCE_QP_STATE_SQD:
2289 			break;
2290 		case QED_ROCE_QP_STATE_ERR:
2291 			break;
2292 		default:
2293 			/* Invalid state change. */
2294 			status = -EINVAL;
2295 			break;
2296 		}
2297 		break;
2298 	case QED_ROCE_QP_STATE_SQD:
2299 		/* SQD->XXX */
2300 		switch (new_state) {
2301 		case QED_ROCE_QP_STATE_RTS:
2302 		case QED_ROCE_QP_STATE_ERR:
2303 			break;
2304 		default:
2305 			/* Invalid state change. */
2306 			status = -EINVAL;
2307 			break;
2308 		}
2309 		break;
2310 	case QED_ROCE_QP_STATE_ERR:
2311 		/* ERR->XXX */
2312 		switch (new_state) {
2313 		case QED_ROCE_QP_STATE_RESET:
2314 			if ((qp->rq.prod != qp->rq.cons) ||
2315 			    (qp->sq.prod != qp->sq.cons)) {
2316 				DP_NOTICE(dev,
2317 					  "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2318 					  qp->rq.prod, qp->rq.cons, qp->sq.prod,
2319 					  qp->sq.cons);
2320 				status = -EINVAL;
2321 			}
2322 			break;
2323 		default:
2324 			status = -EINVAL;
2325 			break;
2326 		}
2327 		break;
2328 	default:
2329 		status = -EINVAL;
2330 		break;
2331 	}
2332 
2333 	return status;
2334 }
2335 
2336 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2337 		   int attr_mask, struct ib_udata *udata)
2338 {
2339 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2340 	struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2341 	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2342 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2343 	enum ib_qp_state old_qp_state, new_qp_state;
2344 	enum qed_roce_qp_state cur_state;
2345 	int rc = 0;
2346 
2347 	DP_DEBUG(dev, QEDR_MSG_QP,
2348 		 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2349 		 attr->qp_state);
2350 
2351 	old_qp_state = qedr_get_ibqp_state(qp->state);
2352 	if (attr_mask & IB_QP_STATE)
2353 		new_qp_state = attr->qp_state;
2354 	else
2355 		new_qp_state = old_qp_state;
2356 
2357 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2358 		if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2359 					ibqp->qp_type, attr_mask)) {
2360 			DP_ERR(dev,
2361 			       "modify qp: invalid attribute mask=0x%x specified for\n"
2362 			       "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2363 			       attr_mask, qp->qp_id, ibqp->qp_type,
2364 			       old_qp_state, new_qp_state);
2365 			rc = -EINVAL;
2366 			goto err;
2367 		}
2368 	}
2369 
2370 	/* Translate the masks... */
2371 	if (attr_mask & IB_QP_STATE) {
2372 		SET_FIELD(qp_params.modify_flags,
2373 			  QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2374 		qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2375 	}
2376 
2377 	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2378 		qp_params.sqd_async = true;
2379 
2380 	if (attr_mask & IB_QP_PKEY_INDEX) {
2381 		SET_FIELD(qp_params.modify_flags,
2382 			  QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2383 		if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2384 			rc = -EINVAL;
2385 			goto err;
2386 		}
2387 
2388 		qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2389 	}
2390 
2391 	if (attr_mask & IB_QP_QKEY)
2392 		qp->qkey = attr->qkey;
2393 
2394 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2395 		SET_FIELD(qp_params.modify_flags,
2396 			  QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2397 		qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2398 						  IB_ACCESS_REMOTE_READ;
2399 		qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2400 						   IB_ACCESS_REMOTE_WRITE;
2401 		qp_params.incoming_atomic_en = attr->qp_access_flags &
2402 					       IB_ACCESS_REMOTE_ATOMIC;
2403 	}
2404 
2405 	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2406 		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2407 			return -EINVAL;
2408 
2409 		if (attr_mask & IB_QP_PATH_MTU) {
2410 			if (attr->path_mtu < IB_MTU_256 ||
2411 			    attr->path_mtu > IB_MTU_4096) {
2412 				pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2413 				rc = -EINVAL;
2414 				goto err;
2415 			}
2416 			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2417 				      ib_mtu_enum_to_int(iboe_get_mtu
2418 							 (dev->ndev->mtu)));
2419 		}
2420 
2421 		if (!qp->mtu) {
2422 			qp->mtu =
2423 			ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2424 			pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2425 		}
2426 
2427 		SET_FIELD(qp_params.modify_flags,
2428 			  QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2429 
2430 		qp_params.traffic_class_tos = grh->traffic_class;
2431 		qp_params.flow_label = grh->flow_label;
2432 		qp_params.hop_limit_ttl = grh->hop_limit;
2433 
2434 		qp->sgid_idx = grh->sgid_index;
2435 
2436 		rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2437 		if (rc) {
2438 			DP_ERR(dev,
2439 			       "modify qp: problems with GID index %d (rc=%d)\n",
2440 			       grh->sgid_index, rc);
2441 			return rc;
2442 		}
2443 
2444 		rc = qedr_get_dmac(dev, &attr->ah_attr,
2445 				   qp_params.remote_mac_addr);
2446 		if (rc)
2447 			return rc;
2448 
2449 		qp_params.use_local_mac = true;
2450 		ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2451 
2452 		DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2453 			 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2454 			 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2455 		DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2456 			 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2457 			 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2458 		DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2459 			 qp_params.remote_mac_addr);
2460 
2461 		qp_params.mtu = qp->mtu;
2462 		qp_params.lb_indication = false;
2463 	}
2464 
2465 	if (!qp_params.mtu) {
2466 		/* Stay with current MTU */
2467 		if (qp->mtu)
2468 			qp_params.mtu = qp->mtu;
2469 		else
2470 			qp_params.mtu =
2471 			    ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2472 	}
2473 
2474 	if (attr_mask & IB_QP_TIMEOUT) {
2475 		SET_FIELD(qp_params.modify_flags,
2476 			  QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2477 
2478 		/* The received timeout value is an exponent used like this:
2479 		 *    "12.7.34 LOCAL ACK TIMEOUT
2480 		 *    Value representing the transport (ACK) timeout for use by
2481 		 *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2482 		 * The FW expects timeout in msec so we need to divide the usec
2483 		 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2484 		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2485 		 * The value of zero means infinite so we use a 'max_t' to make
2486 		 * sure that sub 1 msec values will be configured as 1 msec.
2487 		 */
2488 		if (attr->timeout)
2489 			qp_params.ack_timeout =
2490 					1 << max_t(int, attr->timeout - 8, 0);
2491 		else
2492 			qp_params.ack_timeout = 0;
2493 	}
2494 
2495 	if (attr_mask & IB_QP_RETRY_CNT) {
2496 		SET_FIELD(qp_params.modify_flags,
2497 			  QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2498 		qp_params.retry_cnt = attr->retry_cnt;
2499 	}
2500 
2501 	if (attr_mask & IB_QP_RNR_RETRY) {
2502 		SET_FIELD(qp_params.modify_flags,
2503 			  QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2504 		qp_params.rnr_retry_cnt = attr->rnr_retry;
2505 	}
2506 
2507 	if (attr_mask & IB_QP_RQ_PSN) {
2508 		SET_FIELD(qp_params.modify_flags,
2509 			  QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2510 		qp_params.rq_psn = attr->rq_psn;
2511 		qp->rq_psn = attr->rq_psn;
2512 	}
2513 
2514 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2515 		if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2516 			rc = -EINVAL;
2517 			DP_ERR(dev,
2518 			       "unsupported max_rd_atomic=%d, supported=%d\n",
2519 			       attr->max_rd_atomic,
2520 			       dev->attr.max_qp_req_rd_atomic_resc);
2521 			goto err;
2522 		}
2523 
2524 		SET_FIELD(qp_params.modify_flags,
2525 			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2526 		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2527 	}
2528 
2529 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2530 		SET_FIELD(qp_params.modify_flags,
2531 			  QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2532 		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2533 	}
2534 
2535 	if (attr_mask & IB_QP_SQ_PSN) {
2536 		SET_FIELD(qp_params.modify_flags,
2537 			  QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2538 		qp_params.sq_psn = attr->sq_psn;
2539 		qp->sq_psn = attr->sq_psn;
2540 	}
2541 
2542 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2543 		if (attr->max_dest_rd_atomic >
2544 		    dev->attr.max_qp_resp_rd_atomic_resc) {
2545 			DP_ERR(dev,
2546 			       "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2547 			       attr->max_dest_rd_atomic,
2548 			       dev->attr.max_qp_resp_rd_atomic_resc);
2549 
2550 			rc = -EINVAL;
2551 			goto err;
2552 		}
2553 
2554 		SET_FIELD(qp_params.modify_flags,
2555 			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2556 		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2557 	}
2558 
2559 	if (attr_mask & IB_QP_DEST_QPN) {
2560 		SET_FIELD(qp_params.modify_flags,
2561 			  QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2562 
2563 		qp_params.dest_qp = attr->dest_qp_num;
2564 		qp->dest_qp_num = attr->dest_qp_num;
2565 	}
2566 
2567 	cur_state = qp->state;
2568 
2569 	/* Update the QP state before the actual ramrod to prevent a race with
2570 	 * fast path. Modifying the QP state to error will cause the device to
2571 	 * flush the CQEs and while polling the flushed CQEs will considered as
2572 	 * a potential issue if the QP isn't in error state.
2573 	 */
2574 	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2575 	    !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2576 		qp->state = QED_ROCE_QP_STATE_ERR;
2577 
2578 	if (qp->qp_type != IB_QPT_GSI)
2579 		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2580 					      qp->qed_qp, &qp_params);
2581 
2582 	if (attr_mask & IB_QP_STATE) {
2583 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2584 			rc = qedr_update_qp_state(dev, qp, cur_state,
2585 						  qp_params.new_state);
2586 		qp->state = qp_params.new_state;
2587 	}
2588 
2589 err:
2590 	return rc;
2591 }
2592 
2593 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2594 {
2595 	int ib_qp_acc_flags = 0;
2596 
2597 	if (params->incoming_rdma_write_en)
2598 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2599 	if (params->incoming_rdma_read_en)
2600 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2601 	if (params->incoming_atomic_en)
2602 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2603 	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2604 	return ib_qp_acc_flags;
2605 }
2606 
2607 int qedr_query_qp(struct ib_qp *ibqp,
2608 		  struct ib_qp_attr *qp_attr,
2609 		  int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2610 {
2611 	struct qed_rdma_query_qp_out_params params;
2612 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2613 	struct qedr_dev *dev = qp->dev;
2614 	int rc = 0;
2615 
2616 	memset(&params, 0, sizeof(params));
2617 
2618 	rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2619 	if (rc)
2620 		goto err;
2621 
2622 	memset(qp_attr, 0, sizeof(*qp_attr));
2623 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2624 
2625 	qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2626 	qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2627 	qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2628 	qp_attr->path_mig_state = IB_MIG_MIGRATED;
2629 	qp_attr->rq_psn = params.rq_psn;
2630 	qp_attr->sq_psn = params.sq_psn;
2631 	qp_attr->dest_qp_num = params.dest_qp;
2632 
2633 	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2634 
2635 	qp_attr->cap.max_send_wr = qp->sq.max_wr;
2636 	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2637 	qp_attr->cap.max_send_sge = qp->sq.max_sges;
2638 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2639 	qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2640 	qp_init_attr->cap = qp_attr->cap;
2641 
2642 	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2643 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2644 			params.flow_label, qp->sgid_idx,
2645 			params.hop_limit_ttl, params.traffic_class_tos);
2646 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2647 	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2648 	rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2649 	qp_attr->timeout = params.timeout;
2650 	qp_attr->rnr_retry = params.rnr_retry;
2651 	qp_attr->retry_cnt = params.retry_cnt;
2652 	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2653 	qp_attr->pkey_index = params.pkey_index;
2654 	qp_attr->port_num = 1;
2655 	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2656 	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2657 	qp_attr->alt_pkey_index = 0;
2658 	qp_attr->alt_port_num = 0;
2659 	qp_attr->alt_timeout = 0;
2660 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2661 
2662 	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2663 	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2664 	qp_attr->max_rd_atomic = params.max_rd_atomic;
2665 	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2666 
2667 	DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2668 		 qp_attr->cap.max_inline_data);
2669 
2670 err:
2671 	return rc;
2672 }
2673 
2674 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2675 				  struct ib_udata *udata)
2676 {
2677 	struct qedr_ucontext *ctx =
2678 		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2679 					  ibucontext);
2680 	int rc;
2681 
2682 	if (qp->qp_type != IB_QPT_GSI) {
2683 		rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2684 		if (rc)
2685 			return rc;
2686 	}
2687 
2688 	if (qp->create_type == QEDR_QP_CREATE_USER)
2689 		qedr_cleanup_user(dev, ctx, qp);
2690 	else
2691 		qedr_cleanup_kernel(dev, qp);
2692 
2693 	return 0;
2694 }
2695 
2696 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2697 {
2698 	struct qedr_qp *qp = get_qedr_qp(ibqp);
2699 	struct qedr_dev *dev = qp->dev;
2700 	struct ib_qp_attr attr;
2701 	int attr_mask = 0;
2702 
2703 	DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2704 		 qp, qp->qp_type);
2705 
2706 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2707 		if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2708 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
2709 		    (qp->state != QED_ROCE_QP_STATE_INIT)) {
2710 
2711 			attr.qp_state = IB_QPS_ERR;
2712 			attr_mask |= IB_QP_STATE;
2713 
2714 			/* Change the QP state to ERROR */
2715 			qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2716 		}
2717 	} else {
2718 		/* If connection establishment started the WAIT_FOR_CONNECT
2719 		 * bit will be on and we need to Wait for the establishment
2720 		 * to complete before destroying the qp.
2721 		 */
2722 		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2723 				     &qp->iwarp_cm_flags))
2724 			wait_for_completion(&qp->iwarp_cm_comp);
2725 
2726 		/* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2727 		 * bit will be on, and we need to wait for the disconnect to
2728 		 * complete before continuing. We can use the same completion,
2729 		 * iwarp_cm_comp, since this is the only place that waits for
2730 		 * this completion and it is sequential. In addition,
2731 		 * disconnect can't occur before the connection is fully
2732 		 * established, therefore if WAIT_FOR_DISCONNECT is on it
2733 		 * means WAIT_FOR_CONNECT is also on and the completion for
2734 		 * CONNECT already occurred.
2735 		 */
2736 		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2737 				     &qp->iwarp_cm_flags))
2738 			wait_for_completion(&qp->iwarp_cm_comp);
2739 	}
2740 
2741 	if (qp->qp_type == IB_QPT_GSI)
2742 		qedr_destroy_gsi_qp(dev);
2743 
2744 	/* We need to remove the entry from the xarray before we release the
2745 	 * qp_id to avoid a race of the qp_id being reallocated and failing
2746 	 * on xa_insert
2747 	 */
2748 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2749 		xa_erase(&dev->qps, qp->qp_id);
2750 
2751 	qedr_free_qp_resources(dev, qp, udata);
2752 
2753 	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2754 		qedr_iw_qp_rem_ref(&qp->ibqp);
2755 
2756 	return 0;
2757 }
2758 
2759 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2760 		   struct ib_udata *udata)
2761 {
2762 	struct qedr_ah *ah = get_qedr_ah(ibah);
2763 
2764 	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2765 
2766 	return 0;
2767 }
2768 
2769 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2770 {
2771 	struct qedr_ah *ah = get_qedr_ah(ibah);
2772 
2773 	rdma_destroy_ah_attr(&ah->attr);
2774 }
2775 
2776 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2777 {
2778 	struct qedr_pbl *pbl, *tmp;
2779 
2780 	if (info->pbl_table)
2781 		list_add_tail(&info->pbl_table->list_entry,
2782 			      &info->free_pbl_list);
2783 
2784 	if (!list_empty(&info->inuse_pbl_list))
2785 		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2786 
2787 	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2788 		list_del(&pbl->list_entry);
2789 		qedr_free_pbl(dev, &info->pbl_info, pbl);
2790 	}
2791 }
2792 
2793 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2794 			size_t page_list_len, bool two_layered)
2795 {
2796 	struct qedr_pbl *tmp;
2797 	int rc;
2798 
2799 	INIT_LIST_HEAD(&info->free_pbl_list);
2800 	INIT_LIST_HEAD(&info->inuse_pbl_list);
2801 
2802 	rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2803 				  page_list_len, two_layered);
2804 	if (rc)
2805 		goto done;
2806 
2807 	info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2808 	if (IS_ERR(info->pbl_table)) {
2809 		rc = PTR_ERR(info->pbl_table);
2810 		goto done;
2811 	}
2812 
2813 	DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2814 		 &info->pbl_table->pa);
2815 
2816 	/* in usual case we use 2 PBLs, so we add one to free
2817 	 * list and allocating another one
2818 	 */
2819 	tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2820 	if (IS_ERR(tmp)) {
2821 		DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2822 		goto done;
2823 	}
2824 
2825 	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2826 
2827 	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2828 
2829 done:
2830 	if (rc)
2831 		free_mr_info(dev, info);
2832 
2833 	return rc;
2834 }
2835 
2836 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2837 			       u64 usr_addr, int acc, struct ib_udata *udata)
2838 {
2839 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2840 	struct qedr_mr *mr;
2841 	struct qedr_pd *pd;
2842 	int rc = -ENOMEM;
2843 
2844 	pd = get_qedr_pd(ibpd);
2845 	DP_DEBUG(dev, QEDR_MSG_MR,
2846 		 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2847 		 pd->pd_id, start, len, usr_addr, acc);
2848 
2849 	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2850 		return ERR_PTR(-EINVAL);
2851 
2852 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2853 	if (!mr)
2854 		return ERR_PTR(rc);
2855 
2856 	mr->type = QEDR_MR_USER;
2857 
2858 	mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2859 	if (IS_ERR(mr->umem)) {
2860 		rc = -EFAULT;
2861 		goto err0;
2862 	}
2863 
2864 	rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2865 	if (rc)
2866 		goto err1;
2867 
2868 	qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2869 			   &mr->info.pbl_info, PAGE_SHIFT);
2870 
2871 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2872 	if (rc) {
2873 		DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2874 		goto err1;
2875 	}
2876 
2877 	/* Index only, 18 bit long, lkey = itid << 8 | key */
2878 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2879 	mr->hw_mr.key = 0;
2880 	mr->hw_mr.pd = pd->pd_id;
2881 	mr->hw_mr.local_read = 1;
2882 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2883 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2884 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2885 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2886 	mr->hw_mr.mw_bind = false;
2887 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2888 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2889 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2890 	mr->hw_mr.page_size_log = PAGE_SHIFT;
2891 	mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2892 	mr->hw_mr.length = len;
2893 	mr->hw_mr.vaddr = usr_addr;
2894 	mr->hw_mr.zbva = false;
2895 	mr->hw_mr.phy_mr = false;
2896 	mr->hw_mr.dma_mr = false;
2897 
2898 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2899 	if (rc) {
2900 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2901 		goto err2;
2902 	}
2903 
2904 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2905 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2906 	    mr->hw_mr.remote_atomic)
2907 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2908 
2909 	DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2910 		 mr->ibmr.lkey);
2911 	return &mr->ibmr;
2912 
2913 err2:
2914 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2915 err1:
2916 	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2917 err0:
2918 	kfree(mr);
2919 	return ERR_PTR(rc);
2920 }
2921 
2922 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2923 {
2924 	struct qedr_mr *mr = get_qedr_mr(ib_mr);
2925 	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2926 	int rc = 0;
2927 
2928 	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2929 	if (rc)
2930 		return rc;
2931 
2932 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2933 
2934 	if (mr->type != QEDR_MR_DMA)
2935 		free_mr_info(dev, &mr->info);
2936 
2937 	/* it could be user registered memory. */
2938 	ib_umem_release(mr->umem);
2939 
2940 	kfree(mr);
2941 
2942 	return rc;
2943 }
2944 
2945 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2946 				       int max_page_list_len)
2947 {
2948 	struct qedr_pd *pd = get_qedr_pd(ibpd);
2949 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2950 	struct qedr_mr *mr;
2951 	int rc = -ENOMEM;
2952 
2953 	DP_DEBUG(dev, QEDR_MSG_MR,
2954 		 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2955 		 max_page_list_len);
2956 
2957 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2958 	if (!mr)
2959 		return ERR_PTR(rc);
2960 
2961 	mr->dev = dev;
2962 	mr->type = QEDR_MR_FRMR;
2963 
2964 	rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2965 	if (rc)
2966 		goto err0;
2967 
2968 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2969 	if (rc) {
2970 		DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2971 		goto err0;
2972 	}
2973 
2974 	/* Index only, 18 bit long, lkey = itid << 8 | key */
2975 	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2976 	mr->hw_mr.key = 0;
2977 	mr->hw_mr.pd = pd->pd_id;
2978 	mr->hw_mr.local_read = 1;
2979 	mr->hw_mr.local_write = 0;
2980 	mr->hw_mr.remote_read = 0;
2981 	mr->hw_mr.remote_write = 0;
2982 	mr->hw_mr.remote_atomic = 0;
2983 	mr->hw_mr.mw_bind = false;
2984 	mr->hw_mr.pbl_ptr = 0;
2985 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2986 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2987 	mr->hw_mr.fbo = 0;
2988 	mr->hw_mr.length = 0;
2989 	mr->hw_mr.vaddr = 0;
2990 	mr->hw_mr.zbva = false;
2991 	mr->hw_mr.phy_mr = true;
2992 	mr->hw_mr.dma_mr = false;
2993 
2994 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2995 	if (rc) {
2996 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2997 		goto err1;
2998 	}
2999 
3000 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3001 	mr->ibmr.rkey = mr->ibmr.lkey;
3002 
3003 	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3004 	return mr;
3005 
3006 err1:
3007 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3008 err0:
3009 	kfree(mr);
3010 	return ERR_PTR(rc);
3011 }
3012 
3013 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3014 			    u32 max_num_sg)
3015 {
3016 	struct qedr_mr *mr;
3017 
3018 	if (mr_type != IB_MR_TYPE_MEM_REG)
3019 		return ERR_PTR(-EINVAL);
3020 
3021 	mr = __qedr_alloc_mr(ibpd, max_num_sg);
3022 
3023 	if (IS_ERR(mr))
3024 		return ERR_PTR(-EINVAL);
3025 
3026 	return &mr->ibmr;
3027 }
3028 
3029 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3030 {
3031 	struct qedr_mr *mr = get_qedr_mr(ibmr);
3032 	struct qedr_pbl *pbl_table;
3033 	struct regpair *pbe;
3034 	u32 pbes_in_page;
3035 
3036 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3037 		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3038 		return -ENOMEM;
3039 	}
3040 
3041 	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3042 		 mr->npages, addr);
3043 
3044 	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3045 	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3046 	pbe = (struct regpair *)pbl_table->va;
3047 	pbe +=  mr->npages % pbes_in_page;
3048 	pbe->lo = cpu_to_le32((u32)addr);
3049 	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3050 
3051 	mr->npages++;
3052 
3053 	return 0;
3054 }
3055 
3056 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3057 {
3058 	int work = info->completed - info->completed_handled - 1;
3059 
3060 	DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3061 	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3062 		struct qedr_pbl *pbl;
3063 
3064 		/* Free all the page list that are possible to be freed
3065 		 * (all the ones that were invalidated), under the assumption
3066 		 * that if an FMR was completed successfully that means that
3067 		 * if there was an invalidate operation before it also ended
3068 		 */
3069 		pbl = list_first_entry(&info->inuse_pbl_list,
3070 				       struct qedr_pbl, list_entry);
3071 		list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3072 		info->completed_handled++;
3073 	}
3074 }
3075 
3076 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3077 		   int sg_nents, unsigned int *sg_offset)
3078 {
3079 	struct qedr_mr *mr = get_qedr_mr(ibmr);
3080 
3081 	mr->npages = 0;
3082 
3083 	handle_completed_mrs(mr->dev, &mr->info);
3084 	return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3085 }
3086 
3087 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3088 {
3089 	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3090 	struct qedr_pd *pd = get_qedr_pd(ibpd);
3091 	struct qedr_mr *mr;
3092 	int rc;
3093 
3094 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3095 	if (!mr)
3096 		return ERR_PTR(-ENOMEM);
3097 
3098 	mr->type = QEDR_MR_DMA;
3099 
3100 	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3101 	if (rc) {
3102 		DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
3103 		goto err1;
3104 	}
3105 
3106 	/* index only, 18 bit long, lkey = itid << 8 | key */
3107 	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3108 	mr->hw_mr.pd = pd->pd_id;
3109 	mr->hw_mr.local_read = 1;
3110 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3111 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3112 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3113 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3114 	mr->hw_mr.dma_mr = true;
3115 
3116 	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3117 	if (rc) {
3118 		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3119 		goto err2;
3120 	}
3121 
3122 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3123 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3124 	    mr->hw_mr.remote_atomic)
3125 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3126 
3127 	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3128 	return &mr->ibmr;
3129 
3130 err2:
3131 	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3132 err1:
3133 	kfree(mr);
3134 	return ERR_PTR(rc);
3135 }
3136 
3137 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3138 {
3139 	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3140 }
3141 
3142 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3143 {
3144 	int i, len = 0;
3145 
3146 	for (i = 0; i < num_sge; i++)
3147 		len += sg_list[i].length;
3148 
3149 	return len;
3150 }
3151 
3152 static void swap_wqe_data64(u64 *p)
3153 {
3154 	int i;
3155 
3156 	for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3157 		*p = cpu_to_be64(cpu_to_le64(*p));
3158 }
3159 
3160 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3161 				       struct qedr_qp *qp, u8 *wqe_size,
3162 				       const struct ib_send_wr *wr,
3163 				       const struct ib_send_wr **bad_wr,
3164 				       u8 *bits, u8 bit)
3165 {
3166 	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3167 	char *seg_prt, *wqe;
3168 	int i, seg_siz;
3169 
3170 	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3171 		DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3172 		*bad_wr = wr;
3173 		return 0;
3174 	}
3175 
3176 	if (!data_size)
3177 		return data_size;
3178 
3179 	*bits |= bit;
3180 
3181 	seg_prt = NULL;
3182 	wqe = NULL;
3183 	seg_siz = 0;
3184 
3185 	/* Copy data inline */
3186 	for (i = 0; i < wr->num_sge; i++) {
3187 		u32 len = wr->sg_list[i].length;
3188 		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3189 
3190 		while (len > 0) {
3191 			u32 cur;
3192 
3193 			/* New segment required */
3194 			if (!seg_siz) {
3195 				wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3196 				seg_prt = wqe;
3197 				seg_siz = sizeof(struct rdma_sq_common_wqe);
3198 				(*wqe_size)++;
3199 			}
3200 
3201 			/* Calculate currently allowed length */
3202 			cur = min_t(u32, len, seg_siz);
3203 			memcpy(seg_prt, src, cur);
3204 
3205 			/* Update segment variables */
3206 			seg_prt += cur;
3207 			seg_siz -= cur;
3208 
3209 			/* Update sge variables */
3210 			src += cur;
3211 			len -= cur;
3212 
3213 			/* Swap fully-completed segments */
3214 			if (!seg_siz)
3215 				swap_wqe_data64((u64 *)wqe);
3216 		}
3217 	}
3218 
3219 	/* swap last not completed segment */
3220 	if (seg_siz)
3221 		swap_wqe_data64((u64 *)wqe);
3222 
3223 	return data_size;
3224 }
3225 
3226 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
3227 	do {							\
3228 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3229 		(sge)->length = cpu_to_le32(vlength);		\
3230 		(sge)->flags = cpu_to_le32(vflags);		\
3231 	} while (0)
3232 
3233 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
3234 	do {							\
3235 		DMA_REGPAIR_LE(hdr->wr_id, vwr_id);		\
3236 		(hdr)->num_sges = num_sge;			\
3237 	} while (0)
3238 
3239 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
3240 	do {							\
3241 		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3242 		(sge)->length = cpu_to_le32(vlength);		\
3243 		(sge)->l_key = cpu_to_le32(vlkey);		\
3244 	} while (0)
3245 
3246 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3247 				const struct ib_send_wr *wr)
3248 {
3249 	u32 data_size = 0;
3250 	int i;
3251 
3252 	for (i = 0; i < wr->num_sge; i++) {
3253 		struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3254 
3255 		DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3256 		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3257 		sge->length = cpu_to_le32(wr->sg_list[i].length);
3258 		data_size += wr->sg_list[i].length;
3259 	}
3260 
3261 	if (wqe_size)
3262 		*wqe_size += wr->num_sge;
3263 
3264 	return data_size;
3265 }
3266 
3267 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3268 				     struct qedr_qp *qp,
3269 				     struct rdma_sq_rdma_wqe_1st *rwqe,
3270 				     struct rdma_sq_rdma_wqe_2nd *rwqe2,
3271 				     const struct ib_send_wr *wr,
3272 				     const struct ib_send_wr **bad_wr)
3273 {
3274 	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3275 	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3276 
3277 	if (wr->send_flags & IB_SEND_INLINE &&
3278 	    (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3279 	     wr->opcode == IB_WR_RDMA_WRITE)) {
3280 		u8 flags = 0;
3281 
3282 		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3283 		return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3284 						   bad_wr, &rwqe->flags, flags);
3285 	}
3286 
3287 	return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3288 }
3289 
3290 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3291 				     struct qedr_qp *qp,
3292 				     struct rdma_sq_send_wqe_1st *swqe,
3293 				     struct rdma_sq_send_wqe_2st *swqe2,
3294 				     const struct ib_send_wr *wr,
3295 				     const struct ib_send_wr **bad_wr)
3296 {
3297 	memset(swqe2, 0, sizeof(*swqe2));
3298 	if (wr->send_flags & IB_SEND_INLINE) {
3299 		u8 flags = 0;
3300 
3301 		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3302 		return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3303 						   bad_wr, &swqe->flags, flags);
3304 	}
3305 
3306 	return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3307 }
3308 
3309 static int qedr_prepare_reg(struct qedr_qp *qp,
3310 			    struct rdma_sq_fmr_wqe_1st *fwqe1,
3311 			    const struct ib_reg_wr *wr)
3312 {
3313 	struct qedr_mr *mr = get_qedr_mr(wr->mr);
3314 	struct rdma_sq_fmr_wqe_2nd *fwqe2;
3315 
3316 	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3317 	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3318 	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3319 	fwqe1->l_key = wr->key;
3320 
3321 	fwqe2->access_ctrl = 0;
3322 
3323 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3324 		   !!(wr->access & IB_ACCESS_REMOTE_READ));
3325 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3326 		   !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3327 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3328 		   !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3329 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3330 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3331 		   !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3332 	fwqe2->fmr_ctrl = 0;
3333 
3334 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3335 		   ilog2(mr->ibmr.page_size) - 12);
3336 
3337 	fwqe2->length_hi = 0;
3338 	fwqe2->length_lo = mr->ibmr.length;
3339 	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3340 	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3341 
3342 	qp->wqe_wr_id[qp->sq.prod].mr = mr;
3343 
3344 	return 0;
3345 }
3346 
3347 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3348 {
3349 	switch (opcode) {
3350 	case IB_WR_RDMA_WRITE:
3351 	case IB_WR_RDMA_WRITE_WITH_IMM:
3352 		return IB_WC_RDMA_WRITE;
3353 	case IB_WR_SEND_WITH_IMM:
3354 	case IB_WR_SEND:
3355 	case IB_WR_SEND_WITH_INV:
3356 		return IB_WC_SEND;
3357 	case IB_WR_RDMA_READ:
3358 	case IB_WR_RDMA_READ_WITH_INV:
3359 		return IB_WC_RDMA_READ;
3360 	case IB_WR_ATOMIC_CMP_AND_SWP:
3361 		return IB_WC_COMP_SWAP;
3362 	case IB_WR_ATOMIC_FETCH_AND_ADD:
3363 		return IB_WC_FETCH_ADD;
3364 	case IB_WR_REG_MR:
3365 		return IB_WC_REG_MR;
3366 	case IB_WR_LOCAL_INV:
3367 		return IB_WC_LOCAL_INV;
3368 	default:
3369 		return IB_WC_SEND;
3370 	}
3371 }
3372 
3373 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3374 				      const struct ib_send_wr *wr)
3375 {
3376 	int wq_is_full, err_wr, pbl_is_full;
3377 	struct qedr_dev *dev = qp->dev;
3378 
3379 	/* prevent SQ overflow and/or processing of a bad WR */
3380 	err_wr = wr->num_sge > qp->sq.max_sges;
3381 	wq_is_full = qedr_wq_is_full(&qp->sq);
3382 	pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3383 		      QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3384 	if (wq_is_full || err_wr || pbl_is_full) {
3385 		if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3386 			DP_ERR(dev,
3387 			       "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3388 			       qp);
3389 			qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3390 		}
3391 
3392 		if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3393 			DP_ERR(dev,
3394 			       "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3395 			       qp);
3396 			qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3397 		}
3398 
3399 		if (pbl_is_full &&
3400 		    !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3401 			DP_ERR(dev,
3402 			       "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3403 			       qp);
3404 			qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3405 		}
3406 		return false;
3407 	}
3408 	return true;
3409 }
3410 
3411 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3412 			    const struct ib_send_wr **bad_wr)
3413 {
3414 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3415 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3416 	struct rdma_sq_atomic_wqe_1st *awqe1;
3417 	struct rdma_sq_atomic_wqe_2nd *awqe2;
3418 	struct rdma_sq_atomic_wqe_3rd *awqe3;
3419 	struct rdma_sq_send_wqe_2st *swqe2;
3420 	struct rdma_sq_local_inv_wqe *iwqe;
3421 	struct rdma_sq_rdma_wqe_2nd *rwqe2;
3422 	struct rdma_sq_send_wqe_1st *swqe;
3423 	struct rdma_sq_rdma_wqe_1st *rwqe;
3424 	struct rdma_sq_fmr_wqe_1st *fwqe1;
3425 	struct rdma_sq_common_wqe *wqe;
3426 	u32 length;
3427 	int rc = 0;
3428 	bool comp;
3429 
3430 	if (!qedr_can_post_send(qp, wr)) {
3431 		*bad_wr = wr;
3432 		return -ENOMEM;
3433 	}
3434 
3435 	wqe = qed_chain_produce(&qp->sq.pbl);
3436 	qp->wqe_wr_id[qp->sq.prod].signaled =
3437 		!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3438 
3439 	wqe->flags = 0;
3440 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3441 		   !!(wr->send_flags & IB_SEND_SOLICITED));
3442 	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3443 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3444 	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3445 		   !!(wr->send_flags & IB_SEND_FENCE));
3446 	wqe->prev_wqe_size = qp->prev_wqe_size;
3447 
3448 	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3449 
3450 	switch (wr->opcode) {
3451 	case IB_WR_SEND_WITH_IMM:
3452 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3453 			rc = -EINVAL;
3454 			*bad_wr = wr;
3455 			break;
3456 		}
3457 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3458 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3459 		swqe->wqe_size = 2;
3460 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3461 
3462 		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3463 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3464 						   wr, bad_wr);
3465 		swqe->length = cpu_to_le32(length);
3466 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3467 		qp->prev_wqe_size = swqe->wqe_size;
3468 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3469 		break;
3470 	case IB_WR_SEND:
3471 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3472 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3473 
3474 		swqe->wqe_size = 2;
3475 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3476 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3477 						   wr, bad_wr);
3478 		swqe->length = cpu_to_le32(length);
3479 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3480 		qp->prev_wqe_size = swqe->wqe_size;
3481 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3482 		break;
3483 	case IB_WR_SEND_WITH_INV:
3484 		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3485 		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3486 		swqe2 = qed_chain_produce(&qp->sq.pbl);
3487 		swqe->wqe_size = 2;
3488 		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3489 		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3490 						   wr, bad_wr);
3491 		swqe->length = cpu_to_le32(length);
3492 		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3493 		qp->prev_wqe_size = swqe->wqe_size;
3494 		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3495 		break;
3496 
3497 	case IB_WR_RDMA_WRITE_WITH_IMM:
3498 		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3499 			rc = -EINVAL;
3500 			*bad_wr = wr;
3501 			break;
3502 		}
3503 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3504 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3505 
3506 		rwqe->wqe_size = 2;
3507 		rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3508 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3509 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3510 						   wr, bad_wr);
3511 		rwqe->length = cpu_to_le32(length);
3512 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3513 		qp->prev_wqe_size = rwqe->wqe_size;
3514 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3515 		break;
3516 	case IB_WR_RDMA_WRITE:
3517 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3518 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3519 
3520 		rwqe->wqe_size = 2;
3521 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3522 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3523 						   wr, bad_wr);
3524 		rwqe->length = cpu_to_le32(length);
3525 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3526 		qp->prev_wqe_size = rwqe->wqe_size;
3527 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3528 		break;
3529 	case IB_WR_RDMA_READ_WITH_INV:
3530 		SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3531 		fallthrough;	/* same is identical to RDMA READ */
3532 
3533 	case IB_WR_RDMA_READ:
3534 		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3535 		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3536 
3537 		rwqe->wqe_size = 2;
3538 		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3539 		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3540 						   wr, bad_wr);
3541 		rwqe->length = cpu_to_le32(length);
3542 		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3543 		qp->prev_wqe_size = rwqe->wqe_size;
3544 		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3545 		break;
3546 
3547 	case IB_WR_ATOMIC_CMP_AND_SWP:
3548 	case IB_WR_ATOMIC_FETCH_AND_ADD:
3549 		awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3550 		awqe1->wqe_size = 4;
3551 
3552 		awqe2 = qed_chain_produce(&qp->sq.pbl);
3553 		DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3554 		awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3555 
3556 		awqe3 = qed_chain_produce(&qp->sq.pbl);
3557 
3558 		if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3559 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3560 			DMA_REGPAIR_LE(awqe3->swap_data,
3561 				       atomic_wr(wr)->compare_add);
3562 		} else {
3563 			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3564 			DMA_REGPAIR_LE(awqe3->swap_data,
3565 				       atomic_wr(wr)->swap);
3566 			DMA_REGPAIR_LE(awqe3->cmp_data,
3567 				       atomic_wr(wr)->compare_add);
3568 		}
3569 
3570 		qedr_prepare_sq_sges(qp, NULL, wr);
3571 
3572 		qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3573 		qp->prev_wqe_size = awqe1->wqe_size;
3574 		break;
3575 
3576 	case IB_WR_LOCAL_INV:
3577 		iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3578 		iwqe->wqe_size = 1;
3579 
3580 		iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3581 		iwqe->inv_l_key = wr->ex.invalidate_rkey;
3582 		qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3583 		qp->prev_wqe_size = iwqe->wqe_size;
3584 		break;
3585 	case IB_WR_REG_MR:
3586 		DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3587 		wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3588 		fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3589 		fwqe1->wqe_size = 2;
3590 
3591 		rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3592 		if (rc) {
3593 			DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3594 			*bad_wr = wr;
3595 			break;
3596 		}
3597 
3598 		qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3599 		qp->prev_wqe_size = fwqe1->wqe_size;
3600 		break;
3601 	default:
3602 		DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3603 		rc = -EINVAL;
3604 		*bad_wr = wr;
3605 		break;
3606 	}
3607 
3608 	if (*bad_wr) {
3609 		u16 value;
3610 
3611 		/* Restore prod to its position before
3612 		 * this WR was processed
3613 		 */
3614 		value = le16_to_cpu(qp->sq.db_data.data.value);
3615 		qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3616 
3617 		/* Restore prev_wqe_size */
3618 		qp->prev_wqe_size = wqe->prev_wqe_size;
3619 		rc = -EINVAL;
3620 		DP_ERR(dev, "POST SEND FAILED\n");
3621 	}
3622 
3623 	return rc;
3624 }
3625 
3626 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3627 		   const struct ib_send_wr **bad_wr)
3628 {
3629 	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3630 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3631 	unsigned long flags;
3632 	int rc = 0;
3633 
3634 	*bad_wr = NULL;
3635 
3636 	if (qp->qp_type == IB_QPT_GSI)
3637 		return qedr_gsi_post_send(ibqp, wr, bad_wr);
3638 
3639 	spin_lock_irqsave(&qp->q_lock, flags);
3640 
3641 	if (rdma_protocol_roce(&dev->ibdev, 1)) {
3642 		if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3643 		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
3644 		    (qp->state != QED_ROCE_QP_STATE_SQD)) {
3645 			spin_unlock_irqrestore(&qp->q_lock, flags);
3646 			*bad_wr = wr;
3647 			DP_DEBUG(dev, QEDR_MSG_CQ,
3648 				 "QP in wrong state! QP icid=0x%x state %d\n",
3649 				 qp->icid, qp->state);
3650 			return -EINVAL;
3651 		}
3652 	}
3653 
3654 	while (wr) {
3655 		rc = __qedr_post_send(ibqp, wr, bad_wr);
3656 		if (rc)
3657 			break;
3658 
3659 		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3660 
3661 		qedr_inc_sw_prod(&qp->sq);
3662 
3663 		qp->sq.db_data.data.value++;
3664 
3665 		wr = wr->next;
3666 	}
3667 
3668 	/* Trigger doorbell
3669 	 * If there was a failure in the first WR then it will be triggered in
3670 	 * vane. However this is not harmful (as long as the producer value is
3671 	 * unchanged). For performance reasons we avoid checking for this
3672 	 * redundant doorbell.
3673 	 *
3674 	 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3675 	 * soon as we give the doorbell, we could get a completion
3676 	 * for this wr, therefore we need to make sure that the
3677 	 * memory is updated before giving the doorbell.
3678 	 * During qedr_poll_cq, rmb is called before accessing the
3679 	 * cqe. This covers for the smp_rmb as well.
3680 	 */
3681 	smp_wmb();
3682 	writel(qp->sq.db_data.raw, qp->sq.db);
3683 
3684 	spin_unlock_irqrestore(&qp->q_lock, flags);
3685 
3686 	return rc;
3687 }
3688 
3689 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3690 {
3691 	u32 used;
3692 
3693 	/* Calculate number of elements used based on producer
3694 	 * count and consumer count and subtract it from max
3695 	 * work request supported so that we get elements left.
3696 	 */
3697 	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3698 
3699 	return hw_srq->max_wr - used;
3700 }
3701 
3702 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3703 		       const struct ib_recv_wr **bad_wr)
3704 {
3705 	struct qedr_srq *srq = get_qedr_srq(ibsrq);
3706 	struct qedr_srq_hwq_info *hw_srq;
3707 	struct qedr_dev *dev = srq->dev;
3708 	struct qed_chain *pbl;
3709 	unsigned long flags;
3710 	int status = 0;
3711 	u32 num_sge;
3712 
3713 	spin_lock_irqsave(&srq->lock, flags);
3714 
3715 	hw_srq = &srq->hw_srq;
3716 	pbl = &srq->hw_srq.pbl;
3717 	while (wr) {
3718 		struct rdma_srq_wqe_header *hdr;
3719 		int i;
3720 
3721 		if (!qedr_srq_elem_left(hw_srq) ||
3722 		    wr->num_sge > srq->hw_srq.max_sges) {
3723 			DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3724 			       hw_srq->wr_prod_cnt,
3725 			       atomic_read(&hw_srq->wr_cons_cnt),
3726 			       wr->num_sge, srq->hw_srq.max_sges);
3727 			status = -ENOMEM;
3728 			*bad_wr = wr;
3729 			break;
3730 		}
3731 
3732 		hdr = qed_chain_produce(pbl);
3733 		num_sge = wr->num_sge;
3734 		/* Set number of sge and work request id in header */
3735 		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3736 
3737 		srq->hw_srq.wr_prod_cnt++;
3738 		hw_srq->wqe_prod++;
3739 		hw_srq->sge_prod++;
3740 
3741 		DP_DEBUG(dev, QEDR_MSG_SRQ,
3742 			 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3743 			 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3744 
3745 		for (i = 0; i < wr->num_sge; i++) {
3746 			struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3747 
3748 			/* Set SGE length, lkey and address */
3749 			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3750 				    wr->sg_list[i].length, wr->sg_list[i].lkey);
3751 
3752 			DP_DEBUG(dev, QEDR_MSG_SRQ,
3753 				 "[%d]: len %d key %x addr %x:%x\n",
3754 				 i, srq_sge->length, srq_sge->l_key,
3755 				 srq_sge->addr.hi, srq_sge->addr.lo);
3756 			hw_srq->sge_prod++;
3757 		}
3758 
3759 		/* Update WQE and SGE information before
3760 		 * updating producer.
3761 		 */
3762 		dma_wmb();
3763 
3764 		/* SRQ producer is 8 bytes. Need to update SGE producer index
3765 		 * in first 4 bytes and need to update WQE producer in
3766 		 * next 4 bytes.
3767 		 */
3768 		srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3769 		/* Make sure sge producer is updated first */
3770 		dma_wmb();
3771 		srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3772 
3773 		wr = wr->next;
3774 	}
3775 
3776 	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3777 		 qed_chain_get_elem_left(pbl));
3778 	spin_unlock_irqrestore(&srq->lock, flags);
3779 
3780 	return status;
3781 }
3782 
3783 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3784 		   const struct ib_recv_wr **bad_wr)
3785 {
3786 	struct qedr_qp *qp = get_qedr_qp(ibqp);
3787 	struct qedr_dev *dev = qp->dev;
3788 	unsigned long flags;
3789 	int status = 0;
3790 
3791 	if (qp->qp_type == IB_QPT_GSI)
3792 		return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3793 
3794 	spin_lock_irqsave(&qp->q_lock, flags);
3795 
3796 	if (qp->state == QED_ROCE_QP_STATE_RESET) {
3797 		spin_unlock_irqrestore(&qp->q_lock, flags);
3798 		*bad_wr = wr;
3799 		return -EINVAL;
3800 	}
3801 
3802 	while (wr) {
3803 		int i;
3804 
3805 		if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3806 		    QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3807 		    wr->num_sge > qp->rq.max_sges) {
3808 			DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3809 			       qed_chain_get_elem_left_u32(&qp->rq.pbl),
3810 			       QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3811 			       qp->rq.max_sges);
3812 			status = -ENOMEM;
3813 			*bad_wr = wr;
3814 			break;
3815 		}
3816 		for (i = 0; i < wr->num_sge; i++) {
3817 			u32 flags = 0;
3818 			struct rdma_rq_sge *rqe =
3819 			    qed_chain_produce(&qp->rq.pbl);
3820 
3821 			/* First one must include the number
3822 			 * of SGE in the list
3823 			 */
3824 			if (!i)
3825 				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3826 					  wr->num_sge);
3827 
3828 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3829 				  wr->sg_list[i].lkey);
3830 
3831 			RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3832 				   wr->sg_list[i].length, flags);
3833 		}
3834 
3835 		/* Special case of no sges. FW requires between 1-4 sges...
3836 		 * in this case we need to post 1 sge with length zero. this is
3837 		 * because rdma write with immediate consumes an RQ.
3838 		 */
3839 		if (!wr->num_sge) {
3840 			u32 flags = 0;
3841 			struct rdma_rq_sge *rqe =
3842 			    qed_chain_produce(&qp->rq.pbl);
3843 
3844 			/* First one must include the number
3845 			 * of SGE in the list
3846 			 */
3847 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3848 			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3849 
3850 			RQ_SGE_SET(rqe, 0, 0, flags);
3851 			i = 1;
3852 		}
3853 
3854 		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3855 		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3856 
3857 		qedr_inc_sw_prod(&qp->rq);
3858 
3859 		/* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3860 		 * soon as we give the doorbell, we could get a completion
3861 		 * for this wr, therefore we need to make sure that the
3862 		 * memory is update before giving the doorbell.
3863 		 * During qedr_poll_cq, rmb is called before accessing the
3864 		 * cqe. This covers for the smp_rmb as well.
3865 		 */
3866 		smp_wmb();
3867 
3868 		qp->rq.db_data.data.value++;
3869 
3870 		writel(qp->rq.db_data.raw, qp->rq.db);
3871 
3872 		if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3873 			writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3874 		}
3875 
3876 		wr = wr->next;
3877 	}
3878 
3879 	spin_unlock_irqrestore(&qp->q_lock, flags);
3880 
3881 	return status;
3882 }
3883 
3884 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3885 {
3886 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
3887 
3888 	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3889 		cq->pbl_toggle;
3890 }
3891 
3892 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3893 {
3894 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
3895 	struct qedr_qp *qp;
3896 
3897 	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3898 						   resp_cqe->qp_handle.lo,
3899 						   u64);
3900 	return qp;
3901 }
3902 
3903 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3904 {
3905 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
3906 
3907 	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3908 }
3909 
3910 /* Return latest CQE (needs processing) */
3911 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3912 {
3913 	return cq->latest_cqe;
3914 }
3915 
3916 /* In fmr we need to increase the number of fmr completed counter for the fmr
3917  * algorithm determining whether we can free a pbl or not.
3918  * we need to perform this whether the work request was signaled or not. for
3919  * this purpose we call this function from the condition that checks if a wr
3920  * should be skipped, to make sure we don't miss it ( possibly this fmr
3921  * operation was not signalted)
3922  */
3923 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3924 {
3925 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3926 		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3927 }
3928 
3929 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3930 		       struct qedr_cq *cq, int num_entries,
3931 		       struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3932 		       int force)
3933 {
3934 	u16 cnt = 0;
3935 
3936 	while (num_entries && qp->sq.wqe_cons != hw_cons) {
3937 		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3938 			qedr_chk_if_fmr(qp);
3939 			/* skip WC */
3940 			goto next_cqe;
3941 		}
3942 
3943 		/* fill WC */
3944 		wc->status = status;
3945 		wc->vendor_err = 0;
3946 		wc->wc_flags = 0;
3947 		wc->src_qp = qp->id;
3948 		wc->qp = &qp->ibqp;
3949 
3950 		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3951 		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3952 
3953 		switch (wc->opcode) {
3954 		case IB_WC_RDMA_WRITE:
3955 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3956 			break;
3957 		case IB_WC_COMP_SWAP:
3958 		case IB_WC_FETCH_ADD:
3959 			wc->byte_len = 8;
3960 			break;
3961 		case IB_WC_REG_MR:
3962 			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3963 			break;
3964 		case IB_WC_RDMA_READ:
3965 		case IB_WC_SEND:
3966 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3967 			break;
3968 		default:
3969 			break;
3970 		}
3971 
3972 		num_entries--;
3973 		wc++;
3974 		cnt++;
3975 next_cqe:
3976 		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3977 			qed_chain_consume(&qp->sq.pbl);
3978 		qedr_inc_sw_cons(&qp->sq);
3979 	}
3980 
3981 	return cnt;
3982 }
3983 
3984 static int qedr_poll_cq_req(struct qedr_dev *dev,
3985 			    struct qedr_qp *qp, struct qedr_cq *cq,
3986 			    int num_entries, struct ib_wc *wc,
3987 			    struct rdma_cqe_requester *req)
3988 {
3989 	int cnt = 0;
3990 
3991 	switch (req->status) {
3992 	case RDMA_CQE_REQ_STS_OK:
3993 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3994 				  IB_WC_SUCCESS, 0);
3995 		break;
3996 	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3997 		if (qp->state != QED_ROCE_QP_STATE_ERR)
3998 			DP_DEBUG(dev, QEDR_MSG_CQ,
3999 				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4000 				 cq->icid, qp->icid);
4001 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4002 				  IB_WC_WR_FLUSH_ERR, 1);
4003 		break;
4004 	default:
4005 		/* process all WQE before the cosumer */
4006 		qp->state = QED_ROCE_QP_STATE_ERR;
4007 		cnt = process_req(dev, qp, cq, num_entries, wc,
4008 				  req->sq_cons - 1, IB_WC_SUCCESS, 0);
4009 		wc += cnt;
4010 		/* if we have extra WC fill it with actual error info */
4011 		if (cnt < num_entries) {
4012 			enum ib_wc_status wc_status;
4013 
4014 			switch (req->status) {
4015 			case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4016 				DP_ERR(dev,
4017 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4018 				       cq->icid, qp->icid);
4019 				wc_status = IB_WC_BAD_RESP_ERR;
4020 				break;
4021 			case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4022 				DP_ERR(dev,
4023 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4024 				       cq->icid, qp->icid);
4025 				wc_status = IB_WC_LOC_LEN_ERR;
4026 				break;
4027 			case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4028 				DP_ERR(dev,
4029 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4030 				       cq->icid, qp->icid);
4031 				wc_status = IB_WC_LOC_QP_OP_ERR;
4032 				break;
4033 			case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4034 				DP_ERR(dev,
4035 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4036 				       cq->icid, qp->icid);
4037 				wc_status = IB_WC_LOC_PROT_ERR;
4038 				break;
4039 			case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4040 				DP_ERR(dev,
4041 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4042 				       cq->icid, qp->icid);
4043 				wc_status = IB_WC_MW_BIND_ERR;
4044 				break;
4045 			case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4046 				DP_ERR(dev,
4047 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4048 				       cq->icid, qp->icid);
4049 				wc_status = IB_WC_REM_INV_REQ_ERR;
4050 				break;
4051 			case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4052 				DP_ERR(dev,
4053 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4054 				       cq->icid, qp->icid);
4055 				wc_status = IB_WC_REM_ACCESS_ERR;
4056 				break;
4057 			case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4058 				DP_ERR(dev,
4059 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4060 				       cq->icid, qp->icid);
4061 				wc_status = IB_WC_REM_OP_ERR;
4062 				break;
4063 			case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4064 				DP_ERR(dev,
4065 				       "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4066 				       cq->icid, qp->icid);
4067 				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4068 				break;
4069 			case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4070 				DP_ERR(dev,
4071 				       "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4072 				       cq->icid, qp->icid);
4073 				wc_status = IB_WC_RETRY_EXC_ERR;
4074 				break;
4075 			default:
4076 				DP_ERR(dev,
4077 				       "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4078 				       cq->icid, qp->icid);
4079 				wc_status = IB_WC_GENERAL_ERR;
4080 			}
4081 			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4082 					   wc_status, 1);
4083 		}
4084 	}
4085 
4086 	return cnt;
4087 }
4088 
4089 static inline int qedr_cqe_resp_status_to_ib(u8 status)
4090 {
4091 	switch (status) {
4092 	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4093 		return IB_WC_LOC_ACCESS_ERR;
4094 	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4095 		return IB_WC_LOC_LEN_ERR;
4096 	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4097 		return IB_WC_LOC_QP_OP_ERR;
4098 	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4099 		return IB_WC_LOC_PROT_ERR;
4100 	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4101 		return IB_WC_MW_BIND_ERR;
4102 	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4103 		return IB_WC_REM_INV_RD_REQ_ERR;
4104 	case RDMA_CQE_RESP_STS_OK:
4105 		return IB_WC_SUCCESS;
4106 	default:
4107 		return IB_WC_GENERAL_ERR;
4108 	}
4109 }
4110 
4111 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4112 					  struct ib_wc *wc)
4113 {
4114 	wc->status = IB_WC_SUCCESS;
4115 	wc->byte_len = le32_to_cpu(resp->length);
4116 
4117 	if (resp->flags & QEDR_RESP_IMM) {
4118 		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4119 		wc->wc_flags |= IB_WC_WITH_IMM;
4120 
4121 		if (resp->flags & QEDR_RESP_RDMA)
4122 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4123 
4124 		if (resp->flags & QEDR_RESP_INV)
4125 			return -EINVAL;
4126 
4127 	} else if (resp->flags & QEDR_RESP_INV) {
4128 		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4129 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4130 
4131 		if (resp->flags & QEDR_RESP_RDMA)
4132 			return -EINVAL;
4133 
4134 	} else if (resp->flags & QEDR_RESP_RDMA) {
4135 		return -EINVAL;
4136 	}
4137 
4138 	return 0;
4139 }
4140 
4141 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4142 			       struct qedr_cq *cq, struct ib_wc *wc,
4143 			       struct rdma_cqe_responder *resp, u64 wr_id)
4144 {
4145 	/* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4146 	wc->opcode = IB_WC_RECV;
4147 	wc->wc_flags = 0;
4148 
4149 	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4150 		if (qedr_set_ok_cqe_resp_wc(resp, wc))
4151 			DP_ERR(dev,
4152 			       "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4153 			       cq, cq->icid, resp->flags);
4154 
4155 	} else {
4156 		wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4157 		if (wc->status == IB_WC_GENERAL_ERR)
4158 			DP_ERR(dev,
4159 			       "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4160 			       cq, cq->icid, resp->status);
4161 	}
4162 
4163 	/* Fill the rest of the WC */
4164 	wc->vendor_err = 0;
4165 	wc->src_qp = qp->id;
4166 	wc->qp = &qp->ibqp;
4167 	wc->wr_id = wr_id;
4168 }
4169 
4170 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4171 				struct qedr_cq *cq, struct ib_wc *wc,
4172 				struct rdma_cqe_responder *resp)
4173 {
4174 	struct qedr_srq *srq = qp->srq;
4175 	u64 wr_id;
4176 
4177 	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4178 			 le32_to_cpu(resp->srq_wr_id.lo), u64);
4179 
4180 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4181 		wc->status = IB_WC_WR_FLUSH_ERR;
4182 		wc->vendor_err = 0;
4183 		wc->wr_id = wr_id;
4184 		wc->byte_len = 0;
4185 		wc->src_qp = qp->id;
4186 		wc->qp = &qp->ibqp;
4187 		wc->wr_id = wr_id;
4188 	} else {
4189 		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4190 	}
4191 	atomic_inc(&srq->hw_srq.wr_cons_cnt);
4192 
4193 	return 1;
4194 }
4195 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4196 			    struct qedr_cq *cq, struct ib_wc *wc,
4197 			    struct rdma_cqe_responder *resp)
4198 {
4199 	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4200 
4201 	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4202 
4203 	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4204 		qed_chain_consume(&qp->rq.pbl);
4205 	qedr_inc_sw_cons(&qp->rq);
4206 
4207 	return 1;
4208 }
4209 
4210 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4211 			      int num_entries, struct ib_wc *wc, u16 hw_cons)
4212 {
4213 	u16 cnt = 0;
4214 
4215 	while (num_entries && qp->rq.wqe_cons != hw_cons) {
4216 		/* fill WC */
4217 		wc->status = IB_WC_WR_FLUSH_ERR;
4218 		wc->vendor_err = 0;
4219 		wc->wc_flags = 0;
4220 		wc->src_qp = qp->id;
4221 		wc->byte_len = 0;
4222 		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4223 		wc->qp = &qp->ibqp;
4224 		num_entries--;
4225 		wc++;
4226 		cnt++;
4227 		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4228 			qed_chain_consume(&qp->rq.pbl);
4229 		qedr_inc_sw_cons(&qp->rq);
4230 	}
4231 
4232 	return cnt;
4233 }
4234 
4235 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4236 				 struct rdma_cqe_responder *resp, int *update)
4237 {
4238 	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4239 		consume_cqe(cq);
4240 		*update |= 1;
4241 	}
4242 }
4243 
4244 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4245 				 struct qedr_cq *cq, int num_entries,
4246 				 struct ib_wc *wc,
4247 				 struct rdma_cqe_responder *resp)
4248 {
4249 	int cnt;
4250 
4251 	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4252 	consume_cqe(cq);
4253 
4254 	return cnt;
4255 }
4256 
4257 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4258 			     struct qedr_cq *cq, int num_entries,
4259 			     struct ib_wc *wc, struct rdma_cqe_responder *resp,
4260 			     int *update)
4261 {
4262 	int cnt;
4263 
4264 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4265 		cnt = process_resp_flush(qp, cq, num_entries, wc,
4266 					 resp->rq_cons_or_srq_id);
4267 		try_consume_resp_cqe(cq, qp, resp, update);
4268 	} else {
4269 		cnt = process_resp_one(dev, qp, cq, wc, resp);
4270 		consume_cqe(cq);
4271 		*update |= 1;
4272 	}
4273 
4274 	return cnt;
4275 }
4276 
4277 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4278 				struct rdma_cqe_requester *req, int *update)
4279 {
4280 	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4281 		consume_cqe(cq);
4282 		*update |= 1;
4283 	}
4284 }
4285 
4286 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4287 {
4288 	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4289 	struct qedr_cq *cq = get_qedr_cq(ibcq);
4290 	union rdma_cqe *cqe;
4291 	u32 old_cons, new_cons;
4292 	unsigned long flags;
4293 	int update = 0;
4294 	int done = 0;
4295 
4296 	if (cq->destroyed) {
4297 		DP_ERR(dev,
4298 		       "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4299 		       cq, cq->icid);
4300 		return 0;
4301 	}
4302 
4303 	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4304 		return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4305 
4306 	spin_lock_irqsave(&cq->cq_lock, flags);
4307 	cqe = cq->latest_cqe;
4308 	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4309 	while (num_entries && is_valid_cqe(cq, cqe)) {
4310 		struct qedr_qp *qp;
4311 		int cnt = 0;
4312 
4313 		/* prevent speculative reads of any field of CQE */
4314 		rmb();
4315 
4316 		qp = cqe_get_qp(cqe);
4317 		if (!qp) {
4318 			WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4319 			break;
4320 		}
4321 
4322 		wc->qp = &qp->ibqp;
4323 
4324 		switch (cqe_get_type(cqe)) {
4325 		case RDMA_CQE_TYPE_REQUESTER:
4326 			cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4327 					       &cqe->req);
4328 			try_consume_req_cqe(cq, qp, &cqe->req, &update);
4329 			break;
4330 		case RDMA_CQE_TYPE_RESPONDER_RQ:
4331 			cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4332 						&cqe->resp, &update);
4333 			break;
4334 		case RDMA_CQE_TYPE_RESPONDER_SRQ:
4335 			cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4336 						    wc, &cqe->resp);
4337 			update = 1;
4338 			break;
4339 		case RDMA_CQE_TYPE_INVALID:
4340 		default:
4341 			DP_ERR(dev, "Error: invalid CQE type = %d\n",
4342 			       cqe_get_type(cqe));
4343 		}
4344 		num_entries -= cnt;
4345 		wc += cnt;
4346 		done += cnt;
4347 
4348 		cqe = get_cqe(cq);
4349 	}
4350 	new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4351 
4352 	cq->cq_cons += new_cons - old_cons;
4353 
4354 	if (update)
4355 		/* doorbell notifies abount latest VALID entry,
4356 		 * but chain already point to the next INVALID one
4357 		 */
4358 		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4359 
4360 	spin_unlock_irqrestore(&cq->cq_lock, flags);
4361 	return done;
4362 }
4363 
4364 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4365 		     u8 port_num, const struct ib_wc *in_wc,
4366 		     const struct ib_grh *in_grh, const struct ib_mad *in,
4367 		     struct ib_mad *out_mad, size_t *out_mad_size,
4368 		     u16 *out_mad_pkey_index)
4369 {
4370 	return IB_MAD_RESULT_SUCCESS;
4371 }
4372