1 /*
2  * Broadcom NetXtreme-E RoCE driver.
3  *
4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * BSD license below:
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in
21  *    the documentation and/or other materials provided with the
22  *    distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Description: IB Verbs interpreter
37  */
38 
39 #include <linux/interrupt.h>
40 #include <linux/types.h>
41 #include <linux/pci.h>
42 #include <linux/netdevice.h>
43 #include <linux/if_ether.h>
44 
45 #include <rdma/ib_verbs.h>
46 #include <rdma/ib_user_verbs.h>
47 #include <rdma/ib_umem.h>
48 #include <rdma/ib_addr.h>
49 #include <rdma/ib_mad.h>
50 #include <rdma/ib_cache.h>
51 
52 #include "bnxt_ulp.h"
53 
54 #include "roce_hsi.h"
55 #include "qplib_res.h"
56 #include "qplib_sp.h"
57 #include "qplib_fp.h"
58 #include "qplib_rcfw.h"
59 
60 #include "bnxt_re.h"
61 #include "ib_verbs.h"
62 #include <rdma/bnxt_re-abi.h>
63 
64 static int bnxt_re_build_sgl(struct ib_sge *ib_sg_list,
65 			     struct bnxt_qplib_sge *sg_list, int num)
66 {
67 	int i, total = 0;
68 
69 	for (i = 0; i < num; i++) {
70 		sg_list[i].addr = ib_sg_list[i].addr;
71 		sg_list[i].lkey = ib_sg_list[i].lkey;
72 		sg_list[i].size = ib_sg_list[i].length;
73 		total += sg_list[i].size;
74 	}
75 	return total;
76 }
77 
78 /* Device */
79 struct net_device *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num)
80 {
81 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
82 	struct net_device *netdev = NULL;
83 
84 	rcu_read_lock();
85 	if (rdev)
86 		netdev = rdev->netdev;
87 	if (netdev)
88 		dev_hold(netdev);
89 
90 	rcu_read_unlock();
91 	return netdev;
92 }
93 
94 int bnxt_re_query_device(struct ib_device *ibdev,
95 			 struct ib_device_attr *ib_attr,
96 			 struct ib_udata *udata)
97 {
98 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
99 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
100 
101 	memset(ib_attr, 0, sizeof(*ib_attr));
102 
103 	ib_attr->fw_ver = (u64)(unsigned long)(dev_attr->fw_ver);
104 	bnxt_qplib_get_guid(rdev->netdev->dev_addr,
105 			    (u8 *)&ib_attr->sys_image_guid);
106 	ib_attr->max_mr_size = ~0ull;
107 	ib_attr->page_size_cap = BNXT_RE_PAGE_SIZE_4K | BNXT_RE_PAGE_SIZE_8K |
108 				 BNXT_RE_PAGE_SIZE_64K | BNXT_RE_PAGE_SIZE_2M |
109 				 BNXT_RE_PAGE_SIZE_8M | BNXT_RE_PAGE_SIZE_1G;
110 
111 	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
112 	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
113 	ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
114 	ib_attr->max_qp = dev_attr->max_qp;
115 	ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
116 	ib_attr->device_cap_flags =
117 				    IB_DEVICE_CURR_QP_STATE_MOD
118 				    | IB_DEVICE_RC_RNR_NAK_GEN
119 				    | IB_DEVICE_SHUTDOWN_PORT
120 				    | IB_DEVICE_SYS_IMAGE_GUID
121 				    | IB_DEVICE_LOCAL_DMA_LKEY
122 				    | IB_DEVICE_RESIZE_MAX_WR
123 				    | IB_DEVICE_PORT_ACTIVE_EVENT
124 				    | IB_DEVICE_N_NOTIFY_CQ
125 				    | IB_DEVICE_MEM_WINDOW
126 				    | IB_DEVICE_MEM_WINDOW_TYPE_2B
127 				    | IB_DEVICE_MEM_MGT_EXTENSIONS;
128 	ib_attr->max_sge = dev_attr->max_qp_sges;
129 	ib_attr->max_sge_rd = dev_attr->max_qp_sges;
130 	ib_attr->max_cq = dev_attr->max_cq;
131 	ib_attr->max_cqe = dev_attr->max_cq_wqes;
132 	ib_attr->max_mr = dev_attr->max_mr;
133 	ib_attr->max_pd = dev_attr->max_pd;
134 	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
135 	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_rd_atom;
136 	ib_attr->atomic_cap = IB_ATOMIC_HCA;
137 	ib_attr->masked_atomic_cap = IB_ATOMIC_HCA;
138 
139 	ib_attr->max_ee_rd_atom = 0;
140 	ib_attr->max_res_rd_atom = 0;
141 	ib_attr->max_ee_init_rd_atom = 0;
142 	ib_attr->max_ee = 0;
143 	ib_attr->max_rdd = 0;
144 	ib_attr->max_mw = dev_attr->max_mw;
145 	ib_attr->max_raw_ipv6_qp = 0;
146 	ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
147 	ib_attr->max_mcast_grp = 0;
148 	ib_attr->max_mcast_qp_attach = 0;
149 	ib_attr->max_total_mcast_qp_attach = 0;
150 	ib_attr->max_ah = dev_attr->max_ah;
151 
152 	ib_attr->max_fmr = dev_attr->max_fmr;
153 	ib_attr->max_map_per_fmr = 1;	/* ? */
154 
155 	ib_attr->max_srq = dev_attr->max_srq;
156 	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
157 	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
158 
159 	ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
160 
161 	ib_attr->max_pkeys = 1;
162 	ib_attr->local_ca_ack_delay = 0;
163 	return 0;
164 }
165 
166 int bnxt_re_modify_device(struct ib_device *ibdev,
167 			  int device_modify_mask,
168 			  struct ib_device_modify *device_modify)
169 {
170 	switch (device_modify_mask) {
171 	case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
172 		/* Modify the GUID requires the modification of the GID table */
173 		/* GUID should be made as READ-ONLY */
174 		break;
175 	case IB_DEVICE_MODIFY_NODE_DESC:
176 		/* Node Desc should be made as READ-ONLY */
177 		break;
178 	default:
179 		break;
180 	}
181 	return 0;
182 }
183 
184 static void __to_ib_speed_width(struct net_device *netdev, u8 *speed, u8 *width)
185 {
186 	struct ethtool_link_ksettings lksettings;
187 	u32 espeed;
188 
189 	if (netdev->ethtool_ops && netdev->ethtool_ops->get_link_ksettings) {
190 		memset(&lksettings, 0, sizeof(lksettings));
191 		rtnl_lock();
192 		netdev->ethtool_ops->get_link_ksettings(netdev, &lksettings);
193 		rtnl_unlock();
194 		espeed = lksettings.base.speed;
195 	} else {
196 		espeed = SPEED_UNKNOWN;
197 	}
198 	switch (espeed) {
199 	case SPEED_1000:
200 		*speed = IB_SPEED_SDR;
201 		*width = IB_WIDTH_1X;
202 		break;
203 	case SPEED_10000:
204 		*speed = IB_SPEED_QDR;
205 		*width = IB_WIDTH_1X;
206 		break;
207 	case SPEED_20000:
208 		*speed = IB_SPEED_DDR;
209 		*width = IB_WIDTH_4X;
210 		break;
211 	case SPEED_25000:
212 		*speed = IB_SPEED_EDR;
213 		*width = IB_WIDTH_1X;
214 		break;
215 	case SPEED_40000:
216 		*speed = IB_SPEED_QDR;
217 		*width = IB_WIDTH_4X;
218 		break;
219 	case SPEED_50000:
220 		break;
221 	default:
222 		*speed = IB_SPEED_SDR;
223 		*width = IB_WIDTH_1X;
224 		break;
225 	}
226 }
227 
228 /* Port */
229 int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
230 		       struct ib_port_attr *port_attr)
231 {
232 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
233 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
234 
235 	memset(port_attr, 0, sizeof(*port_attr));
236 
237 	if (netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev)) {
238 		port_attr->state = IB_PORT_ACTIVE;
239 		port_attr->phys_state = 5;
240 	} else {
241 		port_attr->state = IB_PORT_DOWN;
242 		port_attr->phys_state = 3;
243 	}
244 	port_attr->max_mtu = IB_MTU_4096;
245 	port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
246 	port_attr->gid_tbl_len = dev_attr->max_sgid;
247 	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
248 				    IB_PORT_DEVICE_MGMT_SUP |
249 				    IB_PORT_VENDOR_CLASS_SUP |
250 				    IB_PORT_IP_BASED_GIDS;
251 
252 	/* Max MSG size set to 2G for now */
253 	port_attr->max_msg_sz = 0x80000000;
254 	port_attr->bad_pkey_cntr = 0;
255 	port_attr->qkey_viol_cntr = 0;
256 	port_attr->pkey_tbl_len = dev_attr->max_pkey;
257 	port_attr->lid = 0;
258 	port_attr->sm_lid = 0;
259 	port_attr->lmc = 0;
260 	port_attr->max_vl_num = 4;
261 	port_attr->sm_sl = 0;
262 	port_attr->subnet_timeout = 0;
263 	port_attr->init_type_reply = 0;
264 	/* call the underlying netdev's ethtool hooks to query speed settings
265 	 * for which we acquire rtnl_lock _only_ if it's registered with
266 	 * IB stack to avoid race in the NETDEV_UNREG path
267 	 */
268 	if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
269 		__to_ib_speed_width(rdev->netdev, &port_attr->active_speed,
270 				    &port_attr->active_width);
271 	return 0;
272 }
273 
274 int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
275 			int port_modify_mask,
276 			struct ib_port_modify *port_modify)
277 {
278 	switch (port_modify_mask) {
279 	case IB_PORT_SHUTDOWN:
280 		break;
281 	case IB_PORT_INIT_TYPE:
282 		break;
283 	case IB_PORT_RESET_QKEY_CNTR:
284 		break;
285 	default:
286 		break;
287 	}
288 	return 0;
289 }
290 
291 int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
292 			       struct ib_port_immutable *immutable)
293 {
294 	struct ib_port_attr port_attr;
295 
296 	if (bnxt_re_query_port(ibdev, port_num, &port_attr))
297 		return -EINVAL;
298 
299 	immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
300 	immutable->gid_tbl_len = port_attr.gid_tbl_len;
301 	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
302 	immutable->core_cap_flags |= RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP;
303 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
304 	return 0;
305 }
306 
307 int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
308 		       u16 index, u16 *pkey)
309 {
310 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
311 
312 	/* Ignore port_num */
313 
314 	memset(pkey, 0, sizeof(*pkey));
315 	return bnxt_qplib_get_pkey(&rdev->qplib_res,
316 				   &rdev->qplib_res.pkey_tbl, index, pkey);
317 }
318 
319 int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
320 		      int index, union ib_gid *gid)
321 {
322 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
323 	int rc = 0;
324 
325 	/* Ignore port_num */
326 	memset(gid, 0, sizeof(*gid));
327 	rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
328 				 &rdev->qplib_res.sgid_tbl, index,
329 				 (struct bnxt_qplib_gid *)gid);
330 	return rc;
331 }
332 
333 int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
334 		    unsigned int index, void **context)
335 {
336 	int rc = 0;
337 	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
338 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
339 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
340 
341 	/* Delete the entry from the hardware */
342 	ctx = *context;
343 	if (!ctx)
344 		return -EINVAL;
345 
346 	if (sgid_tbl && sgid_tbl->active) {
347 		if (ctx->idx >= sgid_tbl->max)
348 			return -EINVAL;
349 		ctx->refcnt--;
350 		if (!ctx->refcnt) {
351 			rc = bnxt_qplib_del_sgid
352 					(sgid_tbl,
353 					 &sgid_tbl->tbl[ctx->idx], true);
354 			if (rc)
355 				dev_err(rdev_to_dev(rdev),
356 					"Failed to remove GID: %#x", rc);
357 			ctx_tbl = sgid_tbl->ctx;
358 			ctx_tbl[ctx->idx] = NULL;
359 			kfree(ctx);
360 		}
361 	} else {
362 		return -EINVAL;
363 	}
364 	return rc;
365 }
366 
367 int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
368 		    unsigned int index, const union ib_gid *gid,
369 		    const struct ib_gid_attr *attr, void **context)
370 {
371 	int rc;
372 	u32 tbl_idx = 0;
373 	u16 vlan_id = 0xFFFF;
374 	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
375 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
376 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
377 
378 	if ((attr->ndev) && is_vlan_dev(attr->ndev))
379 		vlan_id = vlan_dev_vlan_id(attr->ndev);
380 
381 	rc = bnxt_qplib_add_sgid(sgid_tbl, (struct bnxt_qplib_gid *)gid,
382 				 rdev->qplib_res.netdev->dev_addr,
383 				 vlan_id, true, &tbl_idx);
384 	if (rc == -EALREADY) {
385 		ctx_tbl = sgid_tbl->ctx;
386 		ctx_tbl[tbl_idx]->refcnt++;
387 		*context = ctx_tbl[tbl_idx];
388 		return 0;
389 	}
390 
391 	if (rc < 0) {
392 		dev_err(rdev_to_dev(rdev), "Failed to add GID: %#x", rc);
393 		return rc;
394 	}
395 
396 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
397 	if (!ctx)
398 		return -ENOMEM;
399 	ctx_tbl = sgid_tbl->ctx;
400 	ctx->idx = tbl_idx;
401 	ctx->refcnt = 1;
402 	ctx_tbl[tbl_idx] = ctx;
403 
404 	return rc;
405 }
406 
407 enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
408 					    u8 port_num)
409 {
410 	return IB_LINK_LAYER_ETHERNET;
411 }
412 
413 /* Protection Domains */
414 int bnxt_re_dealloc_pd(struct ib_pd *ib_pd)
415 {
416 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
417 	struct bnxt_re_dev *rdev = pd->rdev;
418 	int rc;
419 
420 	if (ib_pd->uobject && pd->dpi.dbr) {
421 		struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
422 		struct bnxt_re_ucontext *ucntx;
423 
424 		/* Free DPI only if this is the first PD allocated by the
425 		 * application and mark the context dpi as NULL
426 		 */
427 		ucntx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
428 
429 		rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
430 					    &rdev->qplib_res.dpi_tbl,
431 					    &pd->dpi);
432 		if (rc)
433 			dev_err(rdev_to_dev(rdev), "Failed to deallocate HW DPI");
434 			/* Don't fail, continue*/
435 		ucntx->dpi = NULL;
436 	}
437 
438 	rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
439 				   &rdev->qplib_res.pd_tbl,
440 				   &pd->qplib_pd);
441 	if (rc) {
442 		dev_err(rdev_to_dev(rdev), "Failed to deallocate HW PD");
443 		return rc;
444 	}
445 
446 	kfree(pd);
447 	return 0;
448 }
449 
450 struct ib_pd *bnxt_re_alloc_pd(struct ib_device *ibdev,
451 			       struct ib_ucontext *ucontext,
452 			       struct ib_udata *udata)
453 {
454 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
455 	struct bnxt_re_ucontext *ucntx = container_of(ucontext,
456 						      struct bnxt_re_ucontext,
457 						      ib_uctx);
458 	struct bnxt_re_pd *pd;
459 	int rc;
460 
461 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
462 	if (!pd)
463 		return ERR_PTR(-ENOMEM);
464 
465 	pd->rdev = rdev;
466 	if (bnxt_qplib_alloc_pd(&rdev->qplib_res.pd_tbl, &pd->qplib_pd)) {
467 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW PD");
468 		rc = -ENOMEM;
469 		goto fail;
470 	}
471 
472 	if (udata) {
473 		struct bnxt_re_pd_resp resp;
474 
475 		if (!ucntx->dpi) {
476 			/* Allocate DPI in alloc_pd to avoid failing of
477 			 * ibv_devinfo and family of application when DPIs
478 			 * are depleted.
479 			 */
480 			if (bnxt_qplib_alloc_dpi(&rdev->qplib_res.dpi_tbl,
481 						 &pd->dpi, ucntx)) {
482 				rc = -ENOMEM;
483 				goto dbfail;
484 			}
485 			ucntx->dpi = &pd->dpi;
486 		}
487 
488 		resp.pdid = pd->qplib_pd.id;
489 		/* Still allow mapping this DBR to the new user PD. */
490 		resp.dpi = ucntx->dpi->dpi;
491 		resp.dbr = (u64)ucntx->dpi->umdbr;
492 
493 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
494 		if (rc) {
495 			dev_err(rdev_to_dev(rdev),
496 				"Failed to copy user response\n");
497 			goto dbfail;
498 		}
499 	}
500 
501 	return &pd->ib_pd;
502 dbfail:
503 	(void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
504 				    &pd->qplib_pd);
505 fail:
506 	kfree(pd);
507 	return ERR_PTR(rc);
508 }
509 
510 /* Address Handles */
511 int bnxt_re_destroy_ah(struct ib_ah *ib_ah)
512 {
513 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
514 	struct bnxt_re_dev *rdev = ah->rdev;
515 	int rc;
516 
517 	rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah);
518 	if (rc) {
519 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW AH");
520 		return rc;
521 	}
522 	kfree(ah);
523 	return 0;
524 }
525 
526 struct ib_ah *bnxt_re_create_ah(struct ib_pd *ib_pd,
527 				struct rdma_ah_attr *ah_attr,
528 				struct ib_udata *udata)
529 {
530 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
531 	struct bnxt_re_dev *rdev = pd->rdev;
532 	struct bnxt_re_ah *ah;
533 	const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
534 	int rc;
535 	u16 vlan_tag;
536 	u8 nw_type;
537 
538 	struct ib_gid_attr sgid_attr;
539 
540 	if (!(rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH)) {
541 		dev_err(rdev_to_dev(rdev), "Failed to alloc AH: GRH not set");
542 		return ERR_PTR(-EINVAL);
543 	}
544 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
545 	if (!ah)
546 		return ERR_PTR(-ENOMEM);
547 
548 	ah->rdev = rdev;
549 	ah->qplib_ah.pd = &pd->qplib_pd;
550 
551 	/* Supply the configuration for the HW */
552 	memcpy(ah->qplib_ah.dgid.data, grh->dgid.raw,
553 	       sizeof(union ib_gid));
554 	/*
555 	 * If RoCE V2 is enabled, stack will have two entries for
556 	 * each GID entry. Avoiding this duplicte entry in HW. Dividing
557 	 * the GID index by 2 for RoCE V2
558 	 */
559 	ah->qplib_ah.sgid_index = grh->sgid_index / 2;
560 	ah->qplib_ah.host_sgid_index = grh->sgid_index;
561 	ah->qplib_ah.traffic_class = grh->traffic_class;
562 	ah->qplib_ah.flow_label = grh->flow_label;
563 	ah->qplib_ah.hop_limit = grh->hop_limit;
564 	ah->qplib_ah.sl = rdma_ah_get_sl(ah_attr);
565 	if (ib_pd->uobject &&
566 	    !rdma_is_multicast_addr((struct in6_addr *)
567 				    grh->dgid.raw) &&
568 	    !rdma_link_local_addr((struct in6_addr *)
569 				  grh->dgid.raw)) {
570 		union ib_gid sgid;
571 
572 		rc = ib_get_cached_gid(&rdev->ibdev, 1,
573 				       grh->sgid_index, &sgid,
574 				       &sgid_attr);
575 		if (rc) {
576 			dev_err(rdev_to_dev(rdev),
577 				"Failed to query gid at index %d",
578 				grh->sgid_index);
579 			goto fail;
580 		}
581 		if (sgid_attr.ndev) {
582 			if (is_vlan_dev(sgid_attr.ndev))
583 				vlan_tag = vlan_dev_vlan_id(sgid_attr.ndev);
584 			dev_put(sgid_attr.ndev);
585 		}
586 		/* Get network header type for this GID */
587 		nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
588 		switch (nw_type) {
589 		case RDMA_NETWORK_IPV4:
590 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
591 			break;
592 		case RDMA_NETWORK_IPV6:
593 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
594 			break;
595 		default:
596 			ah->qplib_ah.nw_type = CMDQ_CREATE_AH_TYPE_V1;
597 			break;
598 		}
599 		rc = rdma_addr_find_l2_eth_by_grh(&sgid, &grh->dgid,
600 						  ah_attr->roce.dmac, &vlan_tag,
601 						  &sgid_attr.ndev->ifindex,
602 						  NULL);
603 		if (rc) {
604 			dev_err(rdev_to_dev(rdev), "Failed to get dmac\n");
605 			goto fail;
606 		}
607 	}
608 
609 	memcpy(ah->qplib_ah.dmac, ah_attr->roce.dmac, ETH_ALEN);
610 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
611 	if (rc) {
612 		dev_err(rdev_to_dev(rdev), "Failed to allocate HW AH");
613 		goto fail;
614 	}
615 
616 	/* Write AVID to shared page. */
617 	if (ib_pd->uobject) {
618 		struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
619 		struct bnxt_re_ucontext *uctx;
620 		unsigned long flag;
621 		u32 *wrptr;
622 
623 		uctx = container_of(ib_uctx, struct bnxt_re_ucontext, ib_uctx);
624 		spin_lock_irqsave(&uctx->sh_lock, flag);
625 		wrptr = (u32 *)(uctx->shpg + BNXT_RE_AVID_OFFT);
626 		*wrptr = ah->qplib_ah.id;
627 		wmb(); /* make sure cache is updated. */
628 		spin_unlock_irqrestore(&uctx->sh_lock, flag);
629 	}
630 
631 	return &ah->ib_ah;
632 
633 fail:
634 	kfree(ah);
635 	return ERR_PTR(rc);
636 }
637 
638 int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
639 {
640 	return 0;
641 }
642 
643 int bnxt_re_query_ah(struct ib_ah *ib_ah, struct rdma_ah_attr *ah_attr)
644 {
645 	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ib_ah);
646 
647 	ah_attr->type = ib_ah->type;
648 	rdma_ah_set_sl(ah_attr, ah->qplib_ah.sl);
649 	memcpy(ah_attr->roce.dmac, ah->qplib_ah.dmac, ETH_ALEN);
650 	rdma_ah_set_grh(ah_attr, NULL, 0,
651 			ah->qplib_ah.host_sgid_index,
652 			0, ah->qplib_ah.traffic_class);
653 	rdma_ah_set_dgid_raw(ah_attr, ah->qplib_ah.dgid.data);
654 	rdma_ah_set_port_num(ah_attr, 1);
655 	rdma_ah_set_static_rate(ah_attr, 0);
656 	return 0;
657 }
658 
659 /* Queue Pairs */
660 int bnxt_re_destroy_qp(struct ib_qp *ib_qp)
661 {
662 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
663 	struct bnxt_re_dev *rdev = qp->rdev;
664 	int rc;
665 
666 	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
667 	if (rc) {
668 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW QP");
669 		return rc;
670 	}
671 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp) {
672 		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
673 					   &rdev->sqp_ah->qplib_ah);
674 		if (rc) {
675 			dev_err(rdev_to_dev(rdev),
676 				"Failed to destroy HW AH for shadow QP");
677 			return rc;
678 		}
679 
680 		rc = bnxt_qplib_destroy_qp(&rdev->qplib_res,
681 					   &rdev->qp1_sqp->qplib_qp);
682 		if (rc) {
683 			dev_err(rdev_to_dev(rdev),
684 				"Failed to destroy Shadow QP");
685 			return rc;
686 		}
687 		mutex_lock(&rdev->qp_lock);
688 		list_del(&rdev->qp1_sqp->list);
689 		atomic_dec(&rdev->qp_count);
690 		mutex_unlock(&rdev->qp_lock);
691 
692 		kfree(rdev->sqp_ah);
693 		kfree(rdev->qp1_sqp);
694 	}
695 
696 	if (!IS_ERR_OR_NULL(qp->rumem))
697 		ib_umem_release(qp->rumem);
698 	if (!IS_ERR_OR_NULL(qp->sumem))
699 		ib_umem_release(qp->sumem);
700 
701 	mutex_lock(&rdev->qp_lock);
702 	list_del(&qp->list);
703 	atomic_dec(&rdev->qp_count);
704 	mutex_unlock(&rdev->qp_lock);
705 	kfree(qp);
706 	return 0;
707 }
708 
709 static u8 __from_ib_qp_type(enum ib_qp_type type)
710 {
711 	switch (type) {
712 	case IB_QPT_GSI:
713 		return CMDQ_CREATE_QP1_TYPE_GSI;
714 	case IB_QPT_RC:
715 		return CMDQ_CREATE_QP_TYPE_RC;
716 	case IB_QPT_UD:
717 		return CMDQ_CREATE_QP_TYPE_UD;
718 	default:
719 		return IB_QPT_MAX;
720 	}
721 }
722 
723 static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd,
724 				struct bnxt_re_qp *qp, struct ib_udata *udata)
725 {
726 	struct bnxt_re_qp_req ureq;
727 	struct bnxt_qplib_qp *qplib_qp = &qp->qplib_qp;
728 	struct ib_umem *umem;
729 	int bytes = 0;
730 	struct ib_ucontext *context = pd->ib_pd.uobject->context;
731 	struct bnxt_re_ucontext *cntx = container_of(context,
732 						     struct bnxt_re_ucontext,
733 						     ib_uctx);
734 	if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))
735 		return -EFAULT;
736 
737 	bytes = (qplib_qp->sq.max_wqe * BNXT_QPLIB_MAX_SQE_ENTRY_SIZE);
738 	/* Consider mapping PSN search memory only for RC QPs. */
739 	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC)
740 		bytes += (qplib_qp->sq.max_wqe * sizeof(struct sq_psn_search));
741 	bytes = PAGE_ALIGN(bytes);
742 	umem = ib_umem_get(context, ureq.qpsva, bytes,
743 			   IB_ACCESS_LOCAL_WRITE, 1);
744 	if (IS_ERR(umem))
745 		return PTR_ERR(umem);
746 
747 	qp->sumem = umem;
748 	qplib_qp->sq.sglist = umem->sg_head.sgl;
749 	qplib_qp->sq.nmap = umem->nmap;
750 	qplib_qp->qp_handle = ureq.qp_handle;
751 
752 	if (!qp->qplib_qp.srq) {
753 		bytes = (qplib_qp->rq.max_wqe * BNXT_QPLIB_MAX_RQE_ENTRY_SIZE);
754 		bytes = PAGE_ALIGN(bytes);
755 		umem = ib_umem_get(context, ureq.qprva, bytes,
756 				   IB_ACCESS_LOCAL_WRITE, 1);
757 		if (IS_ERR(umem))
758 			goto rqfail;
759 		qp->rumem = umem;
760 		qplib_qp->rq.sglist = umem->sg_head.sgl;
761 		qplib_qp->rq.nmap = umem->nmap;
762 	}
763 
764 	qplib_qp->dpi = cntx->dpi;
765 	return 0;
766 rqfail:
767 	ib_umem_release(qp->sumem);
768 	qp->sumem = NULL;
769 	qplib_qp->sq.sglist = NULL;
770 	qplib_qp->sq.nmap = 0;
771 
772 	return PTR_ERR(umem);
773 }
774 
775 static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah
776 				(struct bnxt_re_pd *pd,
777 				 struct bnxt_qplib_res *qp1_res,
778 				 struct bnxt_qplib_qp *qp1_qp)
779 {
780 	struct bnxt_re_dev *rdev = pd->rdev;
781 	struct bnxt_re_ah *ah;
782 	union ib_gid sgid;
783 	int rc;
784 
785 	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
786 	if (!ah)
787 		return NULL;
788 
789 	memset(ah, 0, sizeof(*ah));
790 	ah->rdev = rdev;
791 	ah->qplib_ah.pd = &pd->qplib_pd;
792 
793 	rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
794 	if (rc)
795 		goto fail;
796 
797 	/* supply the dgid data same as sgid */
798 	memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
799 	       sizeof(union ib_gid));
800 	ah->qplib_ah.sgid_index = 0;
801 
802 	ah->qplib_ah.traffic_class = 0;
803 	ah->qplib_ah.flow_label = 0;
804 	ah->qplib_ah.hop_limit = 1;
805 	ah->qplib_ah.sl = 0;
806 	/* Have DMAC same as SMAC */
807 	ether_addr_copy(ah->qplib_ah.dmac, rdev->netdev->dev_addr);
808 
809 	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah);
810 	if (rc) {
811 		dev_err(rdev_to_dev(rdev),
812 			"Failed to allocate HW AH for Shadow QP");
813 		goto fail;
814 	}
815 
816 	return ah;
817 
818 fail:
819 	kfree(ah);
820 	return NULL;
821 }
822 
823 static struct bnxt_re_qp *bnxt_re_create_shadow_qp
824 				(struct bnxt_re_pd *pd,
825 				 struct bnxt_qplib_res *qp1_res,
826 				 struct bnxt_qplib_qp *qp1_qp)
827 {
828 	struct bnxt_re_dev *rdev = pd->rdev;
829 	struct bnxt_re_qp *qp;
830 	int rc;
831 
832 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
833 	if (!qp)
834 		return NULL;
835 
836 	memset(qp, 0, sizeof(*qp));
837 	qp->rdev = rdev;
838 
839 	/* Initialize the shadow QP structure from the QP1 values */
840 	ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
841 
842 	qp->qplib_qp.pd = &pd->qplib_pd;
843 	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
844 	qp->qplib_qp.type = IB_QPT_UD;
845 
846 	qp->qplib_qp.max_inline_data = 0;
847 	qp->qplib_qp.sig_type = true;
848 
849 	/* Shadow QP SQ depth should be same as QP1 RQ depth */
850 	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
851 	qp->qplib_qp.sq.max_sge = 2;
852 
853 	qp->qplib_qp.scq = qp1_qp->scq;
854 	qp->qplib_qp.rcq = qp1_qp->rcq;
855 
856 	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
857 	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
858 
859 	qp->qplib_qp.mtu = qp1_qp->mtu;
860 
861 	qp->qplib_qp.sq_hdr_buf_size = 0;
862 	qp->qplib_qp.rq_hdr_buf_size = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
863 	qp->qplib_qp.dpi = &rdev->dpi_privileged;
864 
865 	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
866 	if (rc)
867 		goto fail;
868 
869 	rdev->sqp_id = qp->qplib_qp.id;
870 
871 	spin_lock_init(&qp->sq_lock);
872 	INIT_LIST_HEAD(&qp->list);
873 	mutex_lock(&rdev->qp_lock);
874 	list_add_tail(&qp->list, &rdev->qp_list);
875 	atomic_inc(&rdev->qp_count);
876 	mutex_unlock(&rdev->qp_lock);
877 	return qp;
878 fail:
879 	kfree(qp);
880 	return NULL;
881 }
882 
883 struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd,
884 				struct ib_qp_init_attr *qp_init_attr,
885 				struct ib_udata *udata)
886 {
887 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
888 	struct bnxt_re_dev *rdev = pd->rdev;
889 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
890 	struct bnxt_re_qp *qp;
891 	struct bnxt_re_cq *cq;
892 	int rc, entries;
893 
894 	if ((qp_init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
895 	    (qp_init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
896 	    (qp_init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
897 	    (qp_init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
898 	    (qp_init_attr->cap.max_inline_data > dev_attr->max_inline_data))
899 		return ERR_PTR(-EINVAL);
900 
901 	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
902 	if (!qp)
903 		return ERR_PTR(-ENOMEM);
904 
905 	qp->rdev = rdev;
906 	ether_addr_copy(qp->qplib_qp.smac, rdev->netdev->dev_addr);
907 	qp->qplib_qp.pd = &pd->qplib_pd;
908 	qp->qplib_qp.qp_handle = (u64)(unsigned long)(&qp->qplib_qp);
909 	qp->qplib_qp.type = __from_ib_qp_type(qp_init_attr->qp_type);
910 	if (qp->qplib_qp.type == IB_QPT_MAX) {
911 		dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported",
912 			qp->qplib_qp.type);
913 		rc = -EINVAL;
914 		goto fail;
915 	}
916 	qp->qplib_qp.max_inline_data = qp_init_attr->cap.max_inline_data;
917 	qp->qplib_qp.sig_type = ((qp_init_attr->sq_sig_type ==
918 				  IB_SIGNAL_ALL_WR) ? true : false);
919 
920 	entries = roundup_pow_of_two(qp_init_attr->cap.max_send_wr + 1);
921 	qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
922 					dev_attr->max_qp_wqes + 1);
923 
924 	qp->qplib_qp.sq.max_sge = qp_init_attr->cap.max_send_sge;
925 	if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
926 		qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
927 
928 	if (qp_init_attr->send_cq) {
929 		cq = container_of(qp_init_attr->send_cq, struct bnxt_re_cq,
930 				  ib_cq);
931 		if (!cq) {
932 			dev_err(rdev_to_dev(rdev), "Send CQ not found");
933 			rc = -EINVAL;
934 			goto fail;
935 		}
936 		qp->qplib_qp.scq = &cq->qplib_cq;
937 	}
938 
939 	if (qp_init_attr->recv_cq) {
940 		cq = container_of(qp_init_attr->recv_cq, struct bnxt_re_cq,
941 				  ib_cq);
942 		if (!cq) {
943 			dev_err(rdev_to_dev(rdev), "Receive CQ not found");
944 			rc = -EINVAL;
945 			goto fail;
946 		}
947 		qp->qplib_qp.rcq = &cq->qplib_cq;
948 	}
949 
950 	if (qp_init_attr->srq) {
951 		dev_err(rdev_to_dev(rdev), "SRQ not supported");
952 		rc = -ENOTSUPP;
953 		goto fail;
954 	} else {
955 		/* Allocate 1 more than what's provided so posting max doesn't
956 		 * mean empty
957 		 */
958 		entries = roundup_pow_of_two(qp_init_attr->cap.max_recv_wr + 1);
959 		qp->qplib_qp.rq.max_wqe = min_t(u32, entries,
960 						dev_attr->max_qp_wqes + 1);
961 
962 		qp->qplib_qp.rq.max_sge = qp_init_attr->cap.max_recv_sge;
963 		if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
964 			qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
965 	}
966 
967 	qp->qplib_qp.mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->mtu));
968 
969 	if (qp_init_attr->qp_type == IB_QPT_GSI) {
970 		qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
971 		if (qp->qplib_qp.rq.max_sge > dev_attr->max_qp_sges)
972 			qp->qplib_qp.rq.max_sge = dev_attr->max_qp_sges;
973 		qp->qplib_qp.sq.max_sge++;
974 		if (qp->qplib_qp.sq.max_sge > dev_attr->max_qp_sges)
975 			qp->qplib_qp.sq.max_sge = dev_attr->max_qp_sges;
976 
977 		qp->qplib_qp.rq_hdr_buf_size =
978 					BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
979 
980 		qp->qplib_qp.sq_hdr_buf_size =
981 					BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2;
982 		qp->qplib_qp.dpi = &rdev->dpi_privileged;
983 		rc = bnxt_qplib_create_qp1(&rdev->qplib_res, &qp->qplib_qp);
984 		if (rc) {
985 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP1");
986 			goto fail;
987 		}
988 		/* Create a shadow QP to handle the QP1 traffic */
989 		rdev->qp1_sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res,
990 							 &qp->qplib_qp);
991 		if (!rdev->qp1_sqp) {
992 			rc = -EINVAL;
993 			dev_err(rdev_to_dev(rdev),
994 				"Failed to create Shadow QP for QP1");
995 			goto qp_destroy;
996 		}
997 		rdev->sqp_ah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
998 							   &qp->qplib_qp);
999 		if (!rdev->sqp_ah) {
1000 			bnxt_qplib_destroy_qp(&rdev->qplib_res,
1001 					      &rdev->qp1_sqp->qplib_qp);
1002 			rc = -EINVAL;
1003 			dev_err(rdev_to_dev(rdev),
1004 				"Failed to create AH entry for ShadowQP");
1005 			goto qp_destroy;
1006 		}
1007 
1008 	} else {
1009 		qp->qplib_qp.max_rd_atomic = dev_attr->max_qp_rd_atom;
1010 		qp->qplib_qp.max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
1011 		if (udata) {
1012 			rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
1013 			if (rc)
1014 				goto fail;
1015 		} else {
1016 			qp->qplib_qp.dpi = &rdev->dpi_privileged;
1017 		}
1018 
1019 		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
1020 		if (rc) {
1021 			dev_err(rdev_to_dev(rdev), "Failed to create HW QP");
1022 			goto fail;
1023 		}
1024 	}
1025 
1026 	qp->ib_qp.qp_num = qp->qplib_qp.id;
1027 	spin_lock_init(&qp->sq_lock);
1028 
1029 	if (udata) {
1030 		struct bnxt_re_qp_resp resp;
1031 
1032 		resp.qpid = qp->ib_qp.qp_num;
1033 		resp.rsvd = 0;
1034 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
1035 		if (rc) {
1036 			dev_err(rdev_to_dev(rdev), "Failed to copy QP udata");
1037 			goto qp_destroy;
1038 		}
1039 	}
1040 	INIT_LIST_HEAD(&qp->list);
1041 	mutex_lock(&rdev->qp_lock);
1042 	list_add_tail(&qp->list, &rdev->qp_list);
1043 	atomic_inc(&rdev->qp_count);
1044 	mutex_unlock(&rdev->qp_lock);
1045 
1046 	return &qp->ib_qp;
1047 qp_destroy:
1048 	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
1049 fail:
1050 	kfree(qp);
1051 	return ERR_PTR(rc);
1052 }
1053 
1054 static u8 __from_ib_qp_state(enum ib_qp_state state)
1055 {
1056 	switch (state) {
1057 	case IB_QPS_RESET:
1058 		return CMDQ_MODIFY_QP_NEW_STATE_RESET;
1059 	case IB_QPS_INIT:
1060 		return CMDQ_MODIFY_QP_NEW_STATE_INIT;
1061 	case IB_QPS_RTR:
1062 		return CMDQ_MODIFY_QP_NEW_STATE_RTR;
1063 	case IB_QPS_RTS:
1064 		return CMDQ_MODIFY_QP_NEW_STATE_RTS;
1065 	case IB_QPS_SQD:
1066 		return CMDQ_MODIFY_QP_NEW_STATE_SQD;
1067 	case IB_QPS_SQE:
1068 		return CMDQ_MODIFY_QP_NEW_STATE_SQE;
1069 	case IB_QPS_ERR:
1070 	default:
1071 		return CMDQ_MODIFY_QP_NEW_STATE_ERR;
1072 	}
1073 }
1074 
1075 static enum ib_qp_state __to_ib_qp_state(u8 state)
1076 {
1077 	switch (state) {
1078 	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
1079 		return IB_QPS_RESET;
1080 	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
1081 		return IB_QPS_INIT;
1082 	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
1083 		return IB_QPS_RTR;
1084 	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
1085 		return IB_QPS_RTS;
1086 	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
1087 		return IB_QPS_SQD;
1088 	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
1089 		return IB_QPS_SQE;
1090 	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
1091 	default:
1092 		return IB_QPS_ERR;
1093 	}
1094 }
1095 
1096 static u32 __from_ib_mtu(enum ib_mtu mtu)
1097 {
1098 	switch (mtu) {
1099 	case IB_MTU_256:
1100 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
1101 	case IB_MTU_512:
1102 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
1103 	case IB_MTU_1024:
1104 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
1105 	case IB_MTU_2048:
1106 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1107 	case IB_MTU_4096:
1108 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
1109 	default:
1110 		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
1111 	}
1112 }
1113 
1114 static enum ib_mtu __to_ib_mtu(u32 mtu)
1115 {
1116 	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
1117 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
1118 		return IB_MTU_256;
1119 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
1120 		return IB_MTU_512;
1121 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
1122 		return IB_MTU_1024;
1123 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
1124 		return IB_MTU_2048;
1125 	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
1126 		return IB_MTU_4096;
1127 	default:
1128 		return IB_MTU_2048;
1129 	}
1130 }
1131 
1132 static int __from_ib_access_flags(int iflags)
1133 {
1134 	int qflags = 0;
1135 
1136 	if (iflags & IB_ACCESS_LOCAL_WRITE)
1137 		qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1138 	if (iflags & IB_ACCESS_REMOTE_READ)
1139 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
1140 	if (iflags & IB_ACCESS_REMOTE_WRITE)
1141 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
1142 	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
1143 		qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
1144 	if (iflags & IB_ACCESS_MW_BIND)
1145 		qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
1146 	if (iflags & IB_ZERO_BASED)
1147 		qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
1148 	if (iflags & IB_ACCESS_ON_DEMAND)
1149 		qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
1150 	return qflags;
1151 };
1152 
1153 static enum ib_access_flags __to_ib_access_flags(int qflags)
1154 {
1155 	enum ib_access_flags iflags = 0;
1156 
1157 	if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
1158 		iflags |= IB_ACCESS_LOCAL_WRITE;
1159 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
1160 		iflags |= IB_ACCESS_REMOTE_WRITE;
1161 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
1162 		iflags |= IB_ACCESS_REMOTE_READ;
1163 	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
1164 		iflags |= IB_ACCESS_REMOTE_ATOMIC;
1165 	if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
1166 		iflags |= IB_ACCESS_MW_BIND;
1167 	if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
1168 		iflags |= IB_ZERO_BASED;
1169 	if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
1170 		iflags |= IB_ACCESS_ON_DEMAND;
1171 	return iflags;
1172 };
1173 
1174 static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
1175 				    struct bnxt_re_qp *qp1_qp,
1176 				    int qp_attr_mask)
1177 {
1178 	struct bnxt_re_qp *qp = rdev->qp1_sqp;
1179 	int rc = 0;
1180 
1181 	if (qp_attr_mask & IB_QP_STATE) {
1182 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1183 		qp->qplib_qp.state = qp1_qp->qplib_qp.state;
1184 	}
1185 	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1186 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1187 		qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
1188 	}
1189 
1190 	if (qp_attr_mask & IB_QP_QKEY) {
1191 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1192 		/* Using a Random  QKEY */
1193 		qp->qplib_qp.qkey = 0x81818181;
1194 	}
1195 	if (qp_attr_mask & IB_QP_SQ_PSN) {
1196 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1197 		qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
1198 	}
1199 
1200 	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1201 	if (rc)
1202 		dev_err(rdev_to_dev(rdev),
1203 			"Failed to modify Shadow QP for QP1");
1204 	return rc;
1205 }
1206 
1207 int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1208 		      int qp_attr_mask, struct ib_udata *udata)
1209 {
1210 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1211 	struct bnxt_re_dev *rdev = qp->rdev;
1212 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
1213 	enum ib_qp_state curr_qp_state, new_qp_state;
1214 	int rc, entries;
1215 	int status;
1216 	union ib_gid sgid;
1217 	struct ib_gid_attr sgid_attr;
1218 	u8 nw_type;
1219 
1220 	qp->qplib_qp.modify_flags = 0;
1221 	if (qp_attr_mask & IB_QP_STATE) {
1222 		curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
1223 		new_qp_state = qp_attr->qp_state;
1224 		if (!ib_modify_qp_is_ok(curr_qp_state, new_qp_state,
1225 					ib_qp->qp_type, qp_attr_mask,
1226 					IB_LINK_LAYER_ETHERNET)) {
1227 			dev_err(rdev_to_dev(rdev),
1228 				"Invalid attribute mask: %#x specified ",
1229 				qp_attr_mask);
1230 			dev_err(rdev_to_dev(rdev),
1231 				"for qpn: %#x type: %#x",
1232 				ib_qp->qp_num, ib_qp->qp_type);
1233 			dev_err(rdev_to_dev(rdev),
1234 				"curr_qp_state=0x%x, new_qp_state=0x%x\n",
1235 				curr_qp_state, new_qp_state);
1236 			return -EINVAL;
1237 		}
1238 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
1239 		qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
1240 	}
1241 	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
1242 		qp->qplib_qp.modify_flags |=
1243 				CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
1244 		qp->qplib_qp.en_sqd_async_notify = true;
1245 	}
1246 	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
1247 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
1248 		qp->qplib_qp.access =
1249 			__from_ib_access_flags(qp_attr->qp_access_flags);
1250 		/* LOCAL_WRITE access must be set to allow RC receive */
1251 		qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
1252 	}
1253 	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
1254 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
1255 		qp->qplib_qp.pkey_index = qp_attr->pkey_index;
1256 	}
1257 	if (qp_attr_mask & IB_QP_QKEY) {
1258 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
1259 		qp->qplib_qp.qkey = qp_attr->qkey;
1260 	}
1261 	if (qp_attr_mask & IB_QP_AV) {
1262 		const struct ib_global_route *grh =
1263 			rdma_ah_read_grh(&qp_attr->ah_attr);
1264 
1265 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
1266 				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
1267 				     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
1268 				     CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
1269 				     CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
1270 				     CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
1271 				     CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
1272 		memcpy(qp->qplib_qp.ah.dgid.data, grh->dgid.raw,
1273 		       sizeof(qp->qplib_qp.ah.dgid.data));
1274 		qp->qplib_qp.ah.flow_label = grh->flow_label;
1275 		/* If RoCE V2 is enabled, stack will have two entries for
1276 		 * each GID entry. Avoiding this duplicte entry in HW. Dividing
1277 		 * the GID index by 2 for RoCE V2
1278 		 */
1279 		qp->qplib_qp.ah.sgid_index = grh->sgid_index / 2;
1280 		qp->qplib_qp.ah.host_sgid_index = grh->sgid_index;
1281 		qp->qplib_qp.ah.hop_limit = grh->hop_limit;
1282 		qp->qplib_qp.ah.traffic_class = grh->traffic_class;
1283 		qp->qplib_qp.ah.sl = rdma_ah_get_sl(&qp_attr->ah_attr);
1284 		ether_addr_copy(qp->qplib_qp.ah.dmac,
1285 				qp_attr->ah_attr.roce.dmac);
1286 
1287 		status = ib_get_cached_gid(&rdev->ibdev, 1,
1288 					   grh->sgid_index,
1289 					   &sgid, &sgid_attr);
1290 		if (!status && sgid_attr.ndev) {
1291 			memcpy(qp->qplib_qp.smac, sgid_attr.ndev->dev_addr,
1292 			       ETH_ALEN);
1293 			dev_put(sgid_attr.ndev);
1294 			nw_type = ib_gid_to_network_type(sgid_attr.gid_type,
1295 							 &sgid);
1296 			switch (nw_type) {
1297 			case RDMA_NETWORK_IPV4:
1298 				qp->qplib_qp.nw_type =
1299 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
1300 				break;
1301 			case RDMA_NETWORK_IPV6:
1302 				qp->qplib_qp.nw_type =
1303 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
1304 				break;
1305 			default:
1306 				qp->qplib_qp.nw_type =
1307 					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
1308 				break;
1309 			}
1310 		}
1311 	}
1312 
1313 	if (qp_attr_mask & IB_QP_PATH_MTU) {
1314 		qp->qplib_qp.modify_flags |=
1315 				CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1316 		qp->qplib_qp.path_mtu = __from_ib_mtu(qp_attr->path_mtu);
1317 	} else if (qp_attr->qp_state == IB_QPS_RTR) {
1318 		qp->qplib_qp.modify_flags |=
1319 			CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
1320 		qp->qplib_qp.path_mtu =
1321 			__from_ib_mtu(iboe_get_mtu(rdev->netdev->mtu));
1322 	}
1323 
1324 	if (qp_attr_mask & IB_QP_TIMEOUT) {
1325 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
1326 		qp->qplib_qp.timeout = qp_attr->timeout;
1327 	}
1328 	if (qp_attr_mask & IB_QP_RETRY_CNT) {
1329 		qp->qplib_qp.modify_flags |=
1330 				CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
1331 		qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
1332 	}
1333 	if (qp_attr_mask & IB_QP_RNR_RETRY) {
1334 		qp->qplib_qp.modify_flags |=
1335 				CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
1336 		qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
1337 	}
1338 	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
1339 		qp->qplib_qp.modify_flags |=
1340 				CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
1341 		qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
1342 	}
1343 	if (qp_attr_mask & IB_QP_RQ_PSN) {
1344 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
1345 		qp->qplib_qp.rq.psn = qp_attr->rq_psn;
1346 	}
1347 	if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1348 		qp->qplib_qp.modify_flags |=
1349 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
1350 		qp->qplib_qp.max_rd_atomic = qp_attr->max_rd_atomic;
1351 	}
1352 	if (qp_attr_mask & IB_QP_SQ_PSN) {
1353 		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
1354 		qp->qplib_qp.sq.psn = qp_attr->sq_psn;
1355 	}
1356 	if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1357 		qp->qplib_qp.modify_flags |=
1358 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
1359 		qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
1360 	}
1361 	if (qp_attr_mask & IB_QP_CAP) {
1362 		qp->qplib_qp.modify_flags |=
1363 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
1364 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
1365 				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
1366 				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
1367 				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
1368 		if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
1369 		    (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
1370 		    (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
1371 		    (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
1372 		    (qp_attr->cap.max_inline_data >=
1373 						dev_attr->max_inline_data)) {
1374 			dev_err(rdev_to_dev(rdev),
1375 				"Create QP failed - max exceeded");
1376 			return -EINVAL;
1377 		}
1378 		entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
1379 		qp->qplib_qp.sq.max_wqe = min_t(u32, entries,
1380 						dev_attr->max_qp_wqes + 1);
1381 		qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
1382 		if (qp->qplib_qp.rq.max_wqe) {
1383 			entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
1384 			qp->qplib_qp.rq.max_wqe =
1385 				min_t(u32, entries, dev_attr->max_qp_wqes + 1);
1386 			qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
1387 		} else {
1388 			/* SRQ was used prior, just ignore the RQ caps */
1389 		}
1390 	}
1391 	if (qp_attr_mask & IB_QP_DEST_QPN) {
1392 		qp->qplib_qp.modify_flags |=
1393 				CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
1394 		qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
1395 	}
1396 	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
1397 	if (rc) {
1398 		dev_err(rdev_to_dev(rdev), "Failed to modify HW QP");
1399 		return rc;
1400 	}
1401 	if (ib_qp->qp_type == IB_QPT_GSI && rdev->qp1_sqp)
1402 		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
1403 	return rc;
1404 }
1405 
1406 int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
1407 		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
1408 {
1409 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1410 	struct bnxt_re_dev *rdev = qp->rdev;
1411 	struct bnxt_qplib_qp qplib_qp;
1412 	int rc;
1413 
1414 	memset(&qplib_qp, 0, sizeof(struct bnxt_qplib_qp));
1415 	qplib_qp.id = qp->qplib_qp.id;
1416 	qplib_qp.ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
1417 
1418 	rc = bnxt_qplib_query_qp(&rdev->qplib_res, &qplib_qp);
1419 	if (rc) {
1420 		dev_err(rdev_to_dev(rdev), "Failed to query HW QP");
1421 		return rc;
1422 	}
1423 	qp_attr->qp_state = __to_ib_qp_state(qplib_qp.state);
1424 	qp_attr->en_sqd_async_notify = qplib_qp.en_sqd_async_notify ? 1 : 0;
1425 	qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp.access);
1426 	qp_attr->pkey_index = qplib_qp.pkey_index;
1427 	qp_attr->qkey = qplib_qp.qkey;
1428 	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
1429 	rdma_ah_set_grh(&qp_attr->ah_attr, NULL, qplib_qp.ah.flow_label,
1430 			qplib_qp.ah.host_sgid_index,
1431 			qplib_qp.ah.hop_limit,
1432 			qplib_qp.ah.traffic_class);
1433 	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, qplib_qp.ah.dgid.data);
1434 	rdma_ah_set_sl(&qp_attr->ah_attr, qplib_qp.ah.sl);
1435 	ether_addr_copy(qp_attr->ah_attr.roce.dmac, qplib_qp.ah.dmac);
1436 	qp_attr->path_mtu = __to_ib_mtu(qplib_qp.path_mtu);
1437 	qp_attr->timeout = qplib_qp.timeout;
1438 	qp_attr->retry_cnt = qplib_qp.retry_cnt;
1439 	qp_attr->rnr_retry = qplib_qp.rnr_retry;
1440 	qp_attr->min_rnr_timer = qplib_qp.min_rnr_timer;
1441 	qp_attr->rq_psn = qplib_qp.rq.psn;
1442 	qp_attr->max_rd_atomic = qplib_qp.max_rd_atomic;
1443 	qp_attr->sq_psn = qplib_qp.sq.psn;
1444 	qp_attr->max_dest_rd_atomic = qplib_qp.max_dest_rd_atomic;
1445 	qp_init_attr->sq_sig_type = qplib_qp.sig_type ? IB_SIGNAL_ALL_WR :
1446 							IB_SIGNAL_REQ_WR;
1447 	qp_attr->dest_qp_num = qplib_qp.dest_qpn;
1448 
1449 	qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
1450 	qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
1451 	qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
1452 	qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
1453 	qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
1454 	qp_init_attr->cap = qp_attr->cap;
1455 
1456 	return 0;
1457 }
1458 
1459 /* Routine for sending QP1 packets for RoCE V1 an V2
1460  */
1461 static int bnxt_re_build_qp1_send_v2(struct bnxt_re_qp *qp,
1462 				     struct ib_send_wr *wr,
1463 				     struct bnxt_qplib_swqe *wqe,
1464 				     int payload_size)
1465 {
1466 	struct ib_device *ibdev = &qp->rdev->ibdev;
1467 	struct bnxt_re_ah *ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah,
1468 					     ib_ah);
1469 	struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
1470 	struct bnxt_qplib_sge sge;
1471 	union ib_gid sgid;
1472 	u8 nw_type;
1473 	u16 ether_type;
1474 	struct ib_gid_attr sgid_attr;
1475 	union ib_gid dgid;
1476 	bool is_eth = false;
1477 	bool is_vlan = false;
1478 	bool is_grh = false;
1479 	bool is_udp = false;
1480 	u8 ip_version = 0;
1481 	u16 vlan_id = 0xFFFF;
1482 	void *buf;
1483 	int i, rc = 0, size;
1484 
1485 	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
1486 
1487 	rc = ib_get_cached_gid(ibdev, 1,
1488 			       qplib_ah->host_sgid_index, &sgid,
1489 			       &sgid_attr);
1490 	if (rc) {
1491 		dev_err(rdev_to_dev(qp->rdev),
1492 			"Failed to query gid at index %d",
1493 			qplib_ah->host_sgid_index);
1494 		return rc;
1495 	}
1496 	if (sgid_attr.ndev) {
1497 		if (is_vlan_dev(sgid_attr.ndev))
1498 			vlan_id = vlan_dev_vlan_id(sgid_attr.ndev);
1499 		dev_put(sgid_attr.ndev);
1500 	}
1501 	/* Get network header type for this GID */
1502 	nw_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
1503 	switch (nw_type) {
1504 	case RDMA_NETWORK_IPV4:
1505 		nw_type = BNXT_RE_ROCEV2_IPV4_PACKET;
1506 		break;
1507 	case RDMA_NETWORK_IPV6:
1508 		nw_type = BNXT_RE_ROCEV2_IPV6_PACKET;
1509 		break;
1510 	default:
1511 		nw_type = BNXT_RE_ROCE_V1_PACKET;
1512 		break;
1513 	}
1514 	memcpy(&dgid.raw, &qplib_ah->dgid, 16);
1515 	is_udp = sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP;
1516 	if (is_udp) {
1517 		if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) {
1518 			ip_version = 4;
1519 			ether_type = ETH_P_IP;
1520 		} else {
1521 			ip_version = 6;
1522 			ether_type = ETH_P_IPV6;
1523 		}
1524 		is_grh = false;
1525 	} else {
1526 		ether_type = ETH_P_IBOE;
1527 		is_grh = true;
1528 	}
1529 
1530 	is_eth = true;
1531 	is_vlan = (vlan_id && (vlan_id < 0x1000)) ? true : false;
1532 
1533 	ib_ud_header_init(payload_size, !is_eth, is_eth, is_vlan, is_grh,
1534 			  ip_version, is_udp, 0, &qp->qp1_hdr);
1535 
1536 	/* ETH */
1537 	ether_addr_copy(qp->qp1_hdr.eth.dmac_h, ah->qplib_ah.dmac);
1538 	ether_addr_copy(qp->qp1_hdr.eth.smac_h, qp->qplib_qp.smac);
1539 
1540 	/* For vlan, check the sgid for vlan existence */
1541 
1542 	if (!is_vlan) {
1543 		qp->qp1_hdr.eth.type = cpu_to_be16(ether_type);
1544 	} else {
1545 		qp->qp1_hdr.vlan.type = cpu_to_be16(ether_type);
1546 		qp->qp1_hdr.vlan.tag = cpu_to_be16(vlan_id);
1547 	}
1548 
1549 	if (is_grh || (ip_version == 6)) {
1550 		memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
1551 		memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
1552 		       sizeof(sgid));
1553 		qp->qp1_hdr.grh.hop_limit     = qplib_ah->hop_limit;
1554 	}
1555 
1556 	if (ip_version == 4) {
1557 		qp->qp1_hdr.ip4.tos = 0;
1558 		qp->qp1_hdr.ip4.id = 0;
1559 		qp->qp1_hdr.ip4.frag_off = htons(IP_DF);
1560 		qp->qp1_hdr.ip4.ttl = qplib_ah->hop_limit;
1561 
1562 		memcpy(&qp->qp1_hdr.ip4.saddr, sgid.raw + 12, 4);
1563 		memcpy(&qp->qp1_hdr.ip4.daddr, qplib_ah->dgid.data + 12, 4);
1564 		qp->qp1_hdr.ip4.check = ib_ud_ip4_csum(&qp->qp1_hdr);
1565 	}
1566 
1567 	if (is_udp) {
1568 		qp->qp1_hdr.udp.dport = htons(ROCE_V2_UDP_DPORT);
1569 		qp->qp1_hdr.udp.sport = htons(0x8CD1);
1570 		qp->qp1_hdr.udp.csum = 0;
1571 	}
1572 
1573 	/* BTH */
1574 	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
1575 		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1576 		qp->qp1_hdr.immediate_present = 1;
1577 	} else {
1578 		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
1579 	}
1580 	if (wr->send_flags & IB_SEND_SOLICITED)
1581 		qp->qp1_hdr.bth.solicited_event = 1;
1582 	/* pad_count */
1583 	qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
1584 
1585 	/* P_key for QP1 is for all members */
1586 	qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
1587 	qp->qp1_hdr.bth.destination_qpn = IB_QP1;
1588 	qp->qp1_hdr.bth.ack_req = 0;
1589 	qp->send_psn++;
1590 	qp->send_psn &= BTH_PSN_MASK;
1591 	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
1592 	/* DETH */
1593 	/* Use the priviledged Q_Key for QP1 */
1594 	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
1595 	qp->qp1_hdr.deth.source_qpn = IB_QP1;
1596 
1597 	/* Pack the QP1 to the transmit buffer */
1598 	buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
1599 	if (buf) {
1600 		size = ib_ud_header_pack(&qp->qp1_hdr, buf);
1601 		for (i = wqe->num_sge; i; i--) {
1602 			wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
1603 			wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
1604 			wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
1605 		}
1606 
1607 		/*
1608 		 * Max Header buf size for IPV6 RoCE V2 is 86,
1609 		 * which is same as the QP1 SQ header buffer.
1610 		 * Header buf size for IPV4 RoCE V2 can be 66.
1611 		 * ETH(14) + VLAN(4)+ IP(20) + UDP (8) + BTH(20).
1612 		 * Subtract 20 bytes from QP1 SQ header buf size
1613 		 */
1614 		if (is_udp && ip_version == 4)
1615 			sge.size -= 20;
1616 		/*
1617 		 * Max Header buf size for RoCE V1 is 78.
1618 		 * ETH(14) + VLAN(4) + GRH(40) + BTH(20).
1619 		 * Subtract 8 bytes from QP1 SQ header buf size
1620 		 */
1621 		if (!is_udp)
1622 			sge.size -= 8;
1623 
1624 		/* Subtract 4 bytes for non vlan packets */
1625 		if (!is_vlan)
1626 			sge.size -= 4;
1627 
1628 		wqe->sg_list[0].addr = sge.addr;
1629 		wqe->sg_list[0].lkey = sge.lkey;
1630 		wqe->sg_list[0].size = sge.size;
1631 		wqe->num_sge++;
1632 
1633 	} else {
1634 		dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!");
1635 		rc = -ENOMEM;
1636 	}
1637 	return rc;
1638 }
1639 
1640 /* For the MAD layer, it only provides the recv SGE the size of
1641  * ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
1642  * nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
1643  * receive packet (334 bytes) with no VLAN and then copy the GRH
1644  * and the MAD datagram out to the provided SGE.
1645  */
1646 static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
1647 					    struct ib_recv_wr *wr,
1648 					    struct bnxt_qplib_swqe *wqe,
1649 					    int payload_size)
1650 {
1651 	struct bnxt_qplib_sge ref, sge;
1652 	u32 rq_prod_index;
1653 	struct bnxt_re_sqp_entries *sqp_entry;
1654 
1655 	rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
1656 
1657 	if (!bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge))
1658 		return -ENOMEM;
1659 
1660 	/* Create 1 SGE to receive the entire
1661 	 * ethernet packet
1662 	 */
1663 	/* Save the reference from ULP */
1664 	ref.addr = wqe->sg_list[0].addr;
1665 	ref.lkey = wqe->sg_list[0].lkey;
1666 	ref.size = wqe->sg_list[0].size;
1667 
1668 	sqp_entry = &qp->rdev->sqp_tbl[rq_prod_index];
1669 
1670 	/* SGE 1 */
1671 	wqe->sg_list[0].addr = sge.addr;
1672 	wqe->sg_list[0].lkey = sge.lkey;
1673 	wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
1674 	sge.size -= wqe->sg_list[0].size;
1675 
1676 	sqp_entry->sge.addr = ref.addr;
1677 	sqp_entry->sge.lkey = ref.lkey;
1678 	sqp_entry->sge.size = ref.size;
1679 	/* Store the wrid for reporting completion */
1680 	sqp_entry->wrid = wqe->wr_id;
1681 	/* change the wqe->wrid to table index */
1682 	wqe->wr_id = rq_prod_index;
1683 	return 0;
1684 }
1685 
1686 static int is_ud_qp(struct bnxt_re_qp *qp)
1687 {
1688 	return qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD;
1689 }
1690 
1691 static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
1692 				  struct ib_send_wr *wr,
1693 				  struct bnxt_qplib_swqe *wqe)
1694 {
1695 	struct bnxt_re_ah *ah = NULL;
1696 
1697 	if (is_ud_qp(qp)) {
1698 		ah = container_of(ud_wr(wr)->ah, struct bnxt_re_ah, ib_ah);
1699 		wqe->send.q_key = ud_wr(wr)->remote_qkey;
1700 		wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
1701 		wqe->send.avid = ah->qplib_ah.id;
1702 	}
1703 	switch (wr->opcode) {
1704 	case IB_WR_SEND:
1705 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
1706 		break;
1707 	case IB_WR_SEND_WITH_IMM:
1708 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
1709 		wqe->send.imm_data = wr->ex.imm_data;
1710 		break;
1711 	case IB_WR_SEND_WITH_INV:
1712 		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
1713 		wqe->send.inv_key = wr->ex.invalidate_rkey;
1714 		break;
1715 	default:
1716 		return -EINVAL;
1717 	}
1718 	if (wr->send_flags & IB_SEND_SIGNALED)
1719 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1720 	if (wr->send_flags & IB_SEND_FENCE)
1721 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1722 	if (wr->send_flags & IB_SEND_SOLICITED)
1723 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1724 	if (wr->send_flags & IB_SEND_INLINE)
1725 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1726 
1727 	return 0;
1728 }
1729 
1730 static int bnxt_re_build_rdma_wqe(struct ib_send_wr *wr,
1731 				  struct bnxt_qplib_swqe *wqe)
1732 {
1733 	switch (wr->opcode) {
1734 	case IB_WR_RDMA_WRITE:
1735 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
1736 		break;
1737 	case IB_WR_RDMA_WRITE_WITH_IMM:
1738 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
1739 		wqe->rdma.imm_data = wr->ex.imm_data;
1740 		break;
1741 	case IB_WR_RDMA_READ:
1742 		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
1743 		wqe->rdma.inv_key = wr->ex.invalidate_rkey;
1744 		break;
1745 	default:
1746 		return -EINVAL;
1747 	}
1748 	wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
1749 	wqe->rdma.r_key = rdma_wr(wr)->rkey;
1750 	if (wr->send_flags & IB_SEND_SIGNALED)
1751 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1752 	if (wr->send_flags & IB_SEND_FENCE)
1753 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1754 	if (wr->send_flags & IB_SEND_SOLICITED)
1755 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1756 	if (wr->send_flags & IB_SEND_INLINE)
1757 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
1758 
1759 	return 0;
1760 }
1761 
1762 static int bnxt_re_build_atomic_wqe(struct ib_send_wr *wr,
1763 				    struct bnxt_qplib_swqe *wqe)
1764 {
1765 	switch (wr->opcode) {
1766 	case IB_WR_ATOMIC_CMP_AND_SWP:
1767 		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
1768 		wqe->atomic.swap_data = atomic_wr(wr)->swap;
1769 		break;
1770 	case IB_WR_ATOMIC_FETCH_AND_ADD:
1771 		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
1772 		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
1773 		break;
1774 	default:
1775 		return -EINVAL;
1776 	}
1777 	wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
1778 	wqe->atomic.r_key = atomic_wr(wr)->rkey;
1779 	if (wr->send_flags & IB_SEND_SIGNALED)
1780 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1781 	if (wr->send_flags & IB_SEND_FENCE)
1782 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1783 	if (wr->send_flags & IB_SEND_SOLICITED)
1784 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1785 	return 0;
1786 }
1787 
1788 static int bnxt_re_build_inv_wqe(struct ib_send_wr *wr,
1789 				 struct bnxt_qplib_swqe *wqe)
1790 {
1791 	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
1792 	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
1793 
1794 	if (wr->send_flags & IB_SEND_SIGNALED)
1795 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1796 	if (wr->send_flags & IB_SEND_FENCE)
1797 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1798 	if (wr->send_flags & IB_SEND_SOLICITED)
1799 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
1800 
1801 	return 0;
1802 }
1803 
1804 static int bnxt_re_build_reg_wqe(struct ib_reg_wr *wr,
1805 				 struct bnxt_qplib_swqe *wqe)
1806 {
1807 	struct bnxt_re_mr *mr = container_of(wr->mr, struct bnxt_re_mr, ib_mr);
1808 	struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
1809 	int access = wr->access;
1810 
1811 	wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
1812 	wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
1813 	wqe->frmr.page_list = mr->pages;
1814 	wqe->frmr.page_list_len = mr->npages;
1815 	wqe->frmr.levels = qplib_frpl->hwq.level + 1;
1816 	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
1817 
1818 	if (wr->wr.send_flags & IB_SEND_FENCE)
1819 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
1820 	if (wr->wr.send_flags & IB_SEND_SIGNALED)
1821 		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
1822 
1823 	if (access & IB_ACCESS_LOCAL_WRITE)
1824 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
1825 	if (access & IB_ACCESS_REMOTE_READ)
1826 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
1827 	if (access & IB_ACCESS_REMOTE_WRITE)
1828 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
1829 	if (access & IB_ACCESS_REMOTE_ATOMIC)
1830 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
1831 	if (access & IB_ACCESS_MW_BIND)
1832 		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
1833 
1834 	wqe->frmr.l_key = wr->key;
1835 	wqe->frmr.length = wr->mr->length;
1836 	wqe->frmr.pbl_pg_sz_log = (wr->mr->page_size >> PAGE_SHIFT_4K) - 1;
1837 	wqe->frmr.va = wr->mr->iova;
1838 	return 0;
1839 }
1840 
1841 static int bnxt_re_copy_inline_data(struct bnxt_re_dev *rdev,
1842 				    struct ib_send_wr *wr,
1843 				    struct bnxt_qplib_swqe *wqe)
1844 {
1845 	/*  Copy the inline data to the data  field */
1846 	u8 *in_data;
1847 	u32 i, sge_len;
1848 	void *sge_addr;
1849 
1850 	in_data = wqe->inline_data;
1851 	for (i = 0; i < wr->num_sge; i++) {
1852 		sge_addr = (void *)(unsigned long)
1853 				wr->sg_list[i].addr;
1854 		sge_len = wr->sg_list[i].length;
1855 
1856 		if ((sge_len + wqe->inline_len) >
1857 		    BNXT_QPLIB_SWQE_MAX_INLINE_LENGTH) {
1858 			dev_err(rdev_to_dev(rdev),
1859 				"Inline data size requested > supported value");
1860 			return -EINVAL;
1861 		}
1862 		sge_len = wr->sg_list[i].length;
1863 
1864 		memcpy(in_data, sge_addr, sge_len);
1865 		in_data += wr->sg_list[i].length;
1866 		wqe->inline_len += wr->sg_list[i].length;
1867 	}
1868 	return wqe->inline_len;
1869 }
1870 
1871 static int bnxt_re_copy_wr_payload(struct bnxt_re_dev *rdev,
1872 				   struct ib_send_wr *wr,
1873 				   struct bnxt_qplib_swqe *wqe)
1874 {
1875 	int payload_sz = 0;
1876 
1877 	if (wr->send_flags & IB_SEND_INLINE)
1878 		payload_sz = bnxt_re_copy_inline_data(rdev, wr, wqe);
1879 	else
1880 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe->sg_list,
1881 					       wqe->num_sge);
1882 
1883 	return payload_sz;
1884 }
1885 
1886 static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
1887 				       struct bnxt_re_qp *qp,
1888 				struct ib_send_wr *wr)
1889 {
1890 	struct bnxt_qplib_swqe wqe;
1891 	int rc = 0, payload_sz = 0;
1892 	unsigned long flags;
1893 
1894 	spin_lock_irqsave(&qp->sq_lock, flags);
1895 	memset(&wqe, 0, sizeof(wqe));
1896 	while (wr) {
1897 		/* House keeping */
1898 		memset(&wqe, 0, sizeof(wqe));
1899 
1900 		/* Common */
1901 		wqe.num_sge = wr->num_sge;
1902 		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1903 			dev_err(rdev_to_dev(rdev),
1904 				"Limit exceeded for Send SGEs");
1905 			rc = -EINVAL;
1906 			goto bad;
1907 		}
1908 
1909 		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1910 		if (payload_sz < 0) {
1911 			rc = -EINVAL;
1912 			goto bad;
1913 		}
1914 		wqe.wr_id = wr->wr_id;
1915 
1916 		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
1917 
1918 		rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1919 		if (!rc)
1920 			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
1921 bad:
1922 		if (rc) {
1923 			dev_err(rdev_to_dev(rdev),
1924 				"Post send failed opcode = %#x rc = %d",
1925 				wr->opcode, rc);
1926 			break;
1927 		}
1928 		wr = wr->next;
1929 	}
1930 	bnxt_qplib_post_send_db(&qp->qplib_qp);
1931 	spin_unlock_irqrestore(&qp->sq_lock, flags);
1932 	return rc;
1933 }
1934 
1935 int bnxt_re_post_send(struct ib_qp *ib_qp, struct ib_send_wr *wr,
1936 		      struct ib_send_wr **bad_wr)
1937 {
1938 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
1939 	struct bnxt_qplib_swqe wqe;
1940 	int rc = 0, payload_sz = 0;
1941 	unsigned long flags;
1942 
1943 	spin_lock_irqsave(&qp->sq_lock, flags);
1944 	while (wr) {
1945 		/* House keeping */
1946 		memset(&wqe, 0, sizeof(wqe));
1947 
1948 		/* Common */
1949 		wqe.num_sge = wr->num_sge;
1950 		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
1951 			dev_err(rdev_to_dev(qp->rdev),
1952 				"Limit exceeded for Send SGEs");
1953 			rc = -EINVAL;
1954 			goto bad;
1955 		}
1956 
1957 		payload_sz = bnxt_re_copy_wr_payload(qp->rdev, wr, &wqe);
1958 		if (payload_sz < 0) {
1959 			rc = -EINVAL;
1960 			goto bad;
1961 		}
1962 		wqe.wr_id = wr->wr_id;
1963 
1964 		switch (wr->opcode) {
1965 		case IB_WR_SEND:
1966 		case IB_WR_SEND_WITH_IMM:
1967 			if (ib_qp->qp_type == IB_QPT_GSI) {
1968 				rc = bnxt_re_build_qp1_send_v2(qp, wr, &wqe,
1969 							       payload_sz);
1970 				if (rc)
1971 					goto bad;
1972 				wqe.rawqp1.lflags |=
1973 					SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
1974 			}
1975 			switch (wr->send_flags) {
1976 			case IB_SEND_IP_CSUM:
1977 				wqe.rawqp1.lflags |=
1978 					SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
1979 				break;
1980 			default:
1981 				break;
1982 			}
1983 			/* Fall thru to build the wqe */
1984 		case IB_WR_SEND_WITH_INV:
1985 			rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
1986 			break;
1987 		case IB_WR_RDMA_WRITE:
1988 		case IB_WR_RDMA_WRITE_WITH_IMM:
1989 		case IB_WR_RDMA_READ:
1990 			rc = bnxt_re_build_rdma_wqe(wr, &wqe);
1991 			break;
1992 		case IB_WR_ATOMIC_CMP_AND_SWP:
1993 		case IB_WR_ATOMIC_FETCH_AND_ADD:
1994 			rc = bnxt_re_build_atomic_wqe(wr, &wqe);
1995 			break;
1996 		case IB_WR_RDMA_READ_WITH_INV:
1997 			dev_err(rdev_to_dev(qp->rdev),
1998 				"RDMA Read with Invalidate is not supported");
1999 			rc = -EINVAL;
2000 			goto bad;
2001 		case IB_WR_LOCAL_INV:
2002 			rc = bnxt_re_build_inv_wqe(wr, &wqe);
2003 			break;
2004 		case IB_WR_REG_MR:
2005 			rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
2006 			break;
2007 		default:
2008 			/* Unsupported WRs */
2009 			dev_err(rdev_to_dev(qp->rdev),
2010 				"WR (%#x) is not supported", wr->opcode);
2011 			rc = -EINVAL;
2012 			goto bad;
2013 		}
2014 		if (!rc)
2015 			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
2016 bad:
2017 		if (rc) {
2018 			dev_err(rdev_to_dev(qp->rdev),
2019 				"post_send failed op:%#x qps = %#x rc = %d\n",
2020 				wr->opcode, qp->qplib_qp.state, rc);
2021 			*bad_wr = wr;
2022 			break;
2023 		}
2024 		wr = wr->next;
2025 	}
2026 	bnxt_qplib_post_send_db(&qp->qplib_qp);
2027 	spin_unlock_irqrestore(&qp->sq_lock, flags);
2028 
2029 	return rc;
2030 }
2031 
2032 static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
2033 				       struct bnxt_re_qp *qp,
2034 				       struct ib_recv_wr *wr)
2035 {
2036 	struct bnxt_qplib_swqe wqe;
2037 	int rc = 0, payload_sz = 0;
2038 
2039 	memset(&wqe, 0, sizeof(wqe));
2040 	while (wr) {
2041 		/* House keeping */
2042 		memset(&wqe, 0, sizeof(wqe));
2043 
2044 		/* Common */
2045 		wqe.num_sge = wr->num_sge;
2046 		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2047 			dev_err(rdev_to_dev(rdev),
2048 				"Limit exceeded for Receive SGEs");
2049 			rc = -EINVAL;
2050 			break;
2051 		}
2052 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2053 					       wr->num_sge);
2054 		wqe.wr_id = wr->wr_id;
2055 		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2056 
2057 		rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2058 		if (rc)
2059 			break;
2060 
2061 		wr = wr->next;
2062 	}
2063 	if (!rc)
2064 		bnxt_qplib_post_recv_db(&qp->qplib_qp);
2065 	return rc;
2066 }
2067 
2068 int bnxt_re_post_recv(struct ib_qp *ib_qp, struct ib_recv_wr *wr,
2069 		      struct ib_recv_wr **bad_wr)
2070 {
2071 	struct bnxt_re_qp *qp = container_of(ib_qp, struct bnxt_re_qp, ib_qp);
2072 	struct bnxt_qplib_swqe wqe;
2073 	int rc = 0, payload_sz = 0;
2074 
2075 	while (wr) {
2076 		/* House keeping */
2077 		memset(&wqe, 0, sizeof(wqe));
2078 
2079 		/* Common */
2080 		wqe.num_sge = wr->num_sge;
2081 		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
2082 			dev_err(rdev_to_dev(qp->rdev),
2083 				"Limit exceeded for Receive SGEs");
2084 			rc = -EINVAL;
2085 			*bad_wr = wr;
2086 			break;
2087 		}
2088 
2089 		payload_sz = bnxt_re_build_sgl(wr->sg_list, wqe.sg_list,
2090 					       wr->num_sge);
2091 		wqe.wr_id = wr->wr_id;
2092 		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
2093 
2094 		if (ib_qp->qp_type == IB_QPT_GSI)
2095 			rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, &wqe,
2096 							      payload_sz);
2097 		if (!rc)
2098 			rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
2099 		if (rc) {
2100 			*bad_wr = wr;
2101 			break;
2102 		}
2103 		wr = wr->next;
2104 	}
2105 	bnxt_qplib_post_recv_db(&qp->qplib_qp);
2106 	return rc;
2107 }
2108 
2109 /* Completion Queues */
2110 int bnxt_re_destroy_cq(struct ib_cq *ib_cq)
2111 {
2112 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2113 	struct bnxt_re_dev *rdev = cq->rdev;
2114 	int rc;
2115 
2116 	rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2117 	if (rc) {
2118 		dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ");
2119 		return rc;
2120 	}
2121 	if (!IS_ERR_OR_NULL(cq->umem))
2122 		ib_umem_release(cq->umem);
2123 
2124 	if (cq) {
2125 		kfree(cq->cql);
2126 		kfree(cq);
2127 	}
2128 	atomic_dec(&rdev->cq_count);
2129 	rdev->nq.budget--;
2130 	return 0;
2131 }
2132 
2133 struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev,
2134 				const struct ib_cq_init_attr *attr,
2135 				struct ib_ucontext *context,
2136 				struct ib_udata *udata)
2137 {
2138 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
2139 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
2140 	struct bnxt_re_cq *cq = NULL;
2141 	int rc, entries;
2142 	int cqe = attr->cqe;
2143 
2144 	/* Validate CQ fields */
2145 	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
2146 		dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded");
2147 		return ERR_PTR(-EINVAL);
2148 	}
2149 	cq = kzalloc(sizeof(*cq), GFP_KERNEL);
2150 	if (!cq)
2151 		return ERR_PTR(-ENOMEM);
2152 
2153 	cq->rdev = rdev;
2154 	cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq);
2155 
2156 	entries = roundup_pow_of_two(cqe + 1);
2157 	if (entries > dev_attr->max_cq_wqes + 1)
2158 		entries = dev_attr->max_cq_wqes + 1;
2159 
2160 	if (context) {
2161 		struct bnxt_re_cq_req req;
2162 		struct bnxt_re_ucontext *uctx = container_of
2163 						(context,
2164 						 struct bnxt_re_ucontext,
2165 						 ib_uctx);
2166 		if (ib_copy_from_udata(&req, udata, sizeof(req))) {
2167 			rc = -EFAULT;
2168 			goto fail;
2169 		}
2170 
2171 		cq->umem = ib_umem_get(context, req.cq_va,
2172 				       entries * sizeof(struct cq_base),
2173 				       IB_ACCESS_LOCAL_WRITE, 1);
2174 		if (IS_ERR(cq->umem)) {
2175 			rc = PTR_ERR(cq->umem);
2176 			goto fail;
2177 		}
2178 		cq->qplib_cq.sghead = cq->umem->sg_head.sgl;
2179 		cq->qplib_cq.nmap = cq->umem->nmap;
2180 		cq->qplib_cq.dpi = uctx->dpi;
2181 	} else {
2182 		cq->max_cql = min_t(u32, entries, MAX_CQL_PER_POLL);
2183 		cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
2184 				  GFP_KERNEL);
2185 		if (!cq->cql) {
2186 			rc = -ENOMEM;
2187 			goto fail;
2188 		}
2189 
2190 		cq->qplib_cq.dpi = &rdev->dpi_privileged;
2191 		cq->qplib_cq.sghead = NULL;
2192 		cq->qplib_cq.nmap = 0;
2193 	}
2194 	cq->qplib_cq.max_wqe = entries;
2195 	cq->qplib_cq.cnq_hw_ring_id = rdev->nq.ring_id;
2196 
2197 	rc = bnxt_qplib_create_cq(&rdev->qplib_res, &cq->qplib_cq);
2198 	if (rc) {
2199 		dev_err(rdev_to_dev(rdev), "Failed to create HW CQ");
2200 		goto fail;
2201 	}
2202 
2203 	cq->ib_cq.cqe = entries;
2204 	cq->cq_period = cq->qplib_cq.period;
2205 	rdev->nq.budget++;
2206 
2207 	atomic_inc(&rdev->cq_count);
2208 
2209 	if (context) {
2210 		struct bnxt_re_cq_resp resp;
2211 
2212 		resp.cqid = cq->qplib_cq.id;
2213 		resp.tail = cq->qplib_cq.hwq.cons;
2214 		resp.phase = cq->qplib_cq.period;
2215 		resp.rsvd = 0;
2216 		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
2217 		if (rc) {
2218 			dev_err(rdev_to_dev(rdev), "Failed to copy CQ udata");
2219 			bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
2220 			goto c2fail;
2221 		}
2222 	}
2223 
2224 	return &cq->ib_cq;
2225 
2226 c2fail:
2227 	if (context)
2228 		ib_umem_release(cq->umem);
2229 fail:
2230 	kfree(cq->cql);
2231 	kfree(cq);
2232 	return ERR_PTR(rc);
2233 }
2234 
2235 static u8 __req_to_ib_wc_status(u8 qstatus)
2236 {
2237 	switch (qstatus) {
2238 	case CQ_REQ_STATUS_OK:
2239 		return IB_WC_SUCCESS;
2240 	case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
2241 		return IB_WC_BAD_RESP_ERR;
2242 	case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
2243 		return IB_WC_LOC_LEN_ERR;
2244 	case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
2245 		return IB_WC_LOC_QP_OP_ERR;
2246 	case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
2247 		return IB_WC_LOC_PROT_ERR;
2248 	case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
2249 		return IB_WC_GENERAL_ERR;
2250 	case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
2251 		return IB_WC_REM_INV_REQ_ERR;
2252 	case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
2253 		return IB_WC_REM_ACCESS_ERR;
2254 	case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
2255 		return IB_WC_REM_OP_ERR;
2256 	case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
2257 		return IB_WC_RNR_RETRY_EXC_ERR;
2258 	case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
2259 		return IB_WC_RETRY_EXC_ERR;
2260 	case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
2261 		return IB_WC_WR_FLUSH_ERR;
2262 	default:
2263 		return IB_WC_GENERAL_ERR;
2264 	}
2265 	return 0;
2266 }
2267 
2268 static u8 __rawqp1_to_ib_wc_status(u8 qstatus)
2269 {
2270 	switch (qstatus) {
2271 	case CQ_RES_RAWETH_QP1_STATUS_OK:
2272 		return IB_WC_SUCCESS;
2273 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
2274 		return IB_WC_LOC_ACCESS_ERR;
2275 	case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
2276 		return IB_WC_LOC_LEN_ERR;
2277 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
2278 		return IB_WC_LOC_PROT_ERR;
2279 	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
2280 		return IB_WC_LOC_QP_OP_ERR;
2281 	case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
2282 		return IB_WC_GENERAL_ERR;
2283 	case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
2284 		return IB_WC_WR_FLUSH_ERR;
2285 	case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
2286 		return IB_WC_WR_FLUSH_ERR;
2287 	default:
2288 		return IB_WC_GENERAL_ERR;
2289 	}
2290 }
2291 
2292 static u8 __rc_to_ib_wc_status(u8 qstatus)
2293 {
2294 	switch (qstatus) {
2295 	case CQ_RES_RC_STATUS_OK:
2296 		return IB_WC_SUCCESS;
2297 	case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
2298 		return IB_WC_LOC_ACCESS_ERR;
2299 	case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
2300 		return IB_WC_LOC_LEN_ERR;
2301 	case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
2302 		return IB_WC_LOC_PROT_ERR;
2303 	case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
2304 		return IB_WC_LOC_QP_OP_ERR;
2305 	case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
2306 		return IB_WC_GENERAL_ERR;
2307 	case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
2308 		return IB_WC_REM_INV_REQ_ERR;
2309 	case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
2310 		return IB_WC_WR_FLUSH_ERR;
2311 	case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
2312 		return IB_WC_WR_FLUSH_ERR;
2313 	default:
2314 		return IB_WC_GENERAL_ERR;
2315 	}
2316 }
2317 
2318 static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
2319 {
2320 	switch (cqe->type) {
2321 	case BNXT_QPLIB_SWQE_TYPE_SEND:
2322 		wc->opcode = IB_WC_SEND;
2323 		break;
2324 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
2325 		wc->opcode = IB_WC_SEND;
2326 		wc->wc_flags |= IB_WC_WITH_IMM;
2327 		break;
2328 	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
2329 		wc->opcode = IB_WC_SEND;
2330 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2331 		break;
2332 	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
2333 		wc->opcode = IB_WC_RDMA_WRITE;
2334 		break;
2335 	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
2336 		wc->opcode = IB_WC_RDMA_WRITE;
2337 		wc->wc_flags |= IB_WC_WITH_IMM;
2338 		break;
2339 	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
2340 		wc->opcode = IB_WC_RDMA_READ;
2341 		break;
2342 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
2343 		wc->opcode = IB_WC_COMP_SWAP;
2344 		break;
2345 	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
2346 		wc->opcode = IB_WC_FETCH_ADD;
2347 		break;
2348 	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
2349 		wc->opcode = IB_WC_LOCAL_INV;
2350 		break;
2351 	case BNXT_QPLIB_SWQE_TYPE_REG_MR:
2352 		wc->opcode = IB_WC_REG_MR;
2353 		break;
2354 	default:
2355 		wc->opcode = IB_WC_SEND;
2356 		break;
2357 	}
2358 
2359 	wc->status = __req_to_ib_wc_status(cqe->status);
2360 }
2361 
2362 static int bnxt_re_check_packet_type(u16 raweth_qp1_flags,
2363 				     u16 raweth_qp1_flags2)
2364 {
2365 	bool is_udp = false, is_ipv6 = false, is_ipv4 = false;
2366 
2367 	/* raweth_qp1_flags Bit 9-6 indicates itype */
2368 	if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2369 	    != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
2370 		return -1;
2371 
2372 	if (raweth_qp1_flags2 &
2373 	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
2374 	    raweth_qp1_flags2 &
2375 	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
2376 		is_udp = true;
2377 		/* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
2378 		(raweth_qp1_flags2 &
2379 		 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
2380 			(is_ipv6 = true) : (is_ipv4 = true);
2381 		return ((is_ipv6) ?
2382 			 BNXT_RE_ROCEV2_IPV6_PACKET :
2383 			 BNXT_RE_ROCEV2_IPV4_PACKET);
2384 	} else {
2385 		return BNXT_RE_ROCE_V1_PACKET;
2386 	}
2387 }
2388 
2389 static int bnxt_re_to_ib_nw_type(int nw_type)
2390 {
2391 	u8 nw_hdr_type = 0xFF;
2392 
2393 	switch (nw_type) {
2394 	case BNXT_RE_ROCE_V1_PACKET:
2395 		nw_hdr_type = RDMA_NETWORK_ROCE_V1;
2396 		break;
2397 	case BNXT_RE_ROCEV2_IPV4_PACKET:
2398 		nw_hdr_type = RDMA_NETWORK_IPV4;
2399 		break;
2400 	case BNXT_RE_ROCEV2_IPV6_PACKET:
2401 		nw_hdr_type = RDMA_NETWORK_IPV6;
2402 		break;
2403 	}
2404 	return nw_hdr_type;
2405 }
2406 
2407 static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
2408 				       void *rq_hdr_buf)
2409 {
2410 	u8 *tmp_buf = NULL;
2411 	struct ethhdr *eth_hdr;
2412 	u16 eth_type;
2413 	bool rc = false;
2414 
2415 	tmp_buf = (u8 *)rq_hdr_buf;
2416 	/*
2417 	 * If dest mac is not same as I/F mac, this could be a
2418 	 * loopback address or multicast address, check whether
2419 	 * it is a loopback packet
2420 	 */
2421 	if (!ether_addr_equal(tmp_buf, rdev->netdev->dev_addr)) {
2422 		tmp_buf += 4;
2423 		/* Check the  ether type */
2424 		eth_hdr = (struct ethhdr *)tmp_buf;
2425 		eth_type = ntohs(eth_hdr->h_proto);
2426 		switch (eth_type) {
2427 		case ETH_P_IBOE:
2428 			rc = true;
2429 			break;
2430 		case ETH_P_IP:
2431 		case ETH_P_IPV6: {
2432 			u32 len;
2433 			struct udphdr *udp_hdr;
2434 
2435 			len = (eth_type == ETH_P_IP ? sizeof(struct iphdr) :
2436 						      sizeof(struct ipv6hdr));
2437 			tmp_buf += sizeof(struct ethhdr) + len;
2438 			udp_hdr = (struct udphdr *)tmp_buf;
2439 			if (ntohs(udp_hdr->dest) ==
2440 				    ROCE_V2_UDP_DPORT)
2441 				rc = true;
2442 			break;
2443 			}
2444 		default:
2445 			break;
2446 		}
2447 	}
2448 
2449 	return rc;
2450 }
2451 
2452 static int bnxt_re_process_raw_qp_pkt_rx(struct bnxt_re_qp *qp1_qp,
2453 					 struct bnxt_qplib_cqe *cqe)
2454 {
2455 	struct bnxt_re_dev *rdev = qp1_qp->rdev;
2456 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2457 	struct bnxt_re_qp *qp = rdev->qp1_sqp;
2458 	struct ib_send_wr *swr;
2459 	struct ib_ud_wr udwr;
2460 	struct ib_recv_wr rwr;
2461 	int pkt_type = 0;
2462 	u32 tbl_idx;
2463 	void *rq_hdr_buf;
2464 	dma_addr_t rq_hdr_buf_map;
2465 	dma_addr_t shrq_hdr_buf_map;
2466 	u32 offset = 0;
2467 	u32 skip_bytes = 0;
2468 	struct ib_sge s_sge[2];
2469 	struct ib_sge r_sge[2];
2470 	int rc;
2471 
2472 	memset(&udwr, 0, sizeof(udwr));
2473 	memset(&rwr, 0, sizeof(rwr));
2474 	memset(&s_sge, 0, sizeof(s_sge));
2475 	memset(&r_sge, 0, sizeof(r_sge));
2476 
2477 	swr = &udwr.wr;
2478 	tbl_idx = cqe->wr_id;
2479 
2480 	rq_hdr_buf = qp1_qp->qplib_qp.rq_hdr_buf +
2481 			(tbl_idx * qp1_qp->qplib_qp.rq_hdr_buf_size);
2482 	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp1_qp->qplib_qp,
2483 							  tbl_idx);
2484 
2485 	/* Shadow QP header buffer */
2486 	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&qp->qplib_qp,
2487 							    tbl_idx);
2488 	sqp_entry = &rdev->sqp_tbl[tbl_idx];
2489 
2490 	/* Store this cqe */
2491 	memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
2492 	sqp_entry->qp1_qp = qp1_qp;
2493 
2494 	/* Find packet type from the cqe */
2495 
2496 	pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
2497 					     cqe->raweth_qp1_flags2);
2498 	if (pkt_type < 0) {
2499 		dev_err(rdev_to_dev(rdev), "Invalid packet\n");
2500 		return -EINVAL;
2501 	}
2502 
2503 	/* Adjust the offset for the user buffer and post in the rq */
2504 
2505 	if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
2506 		offset = 20;
2507 
2508 	/*
2509 	 * QP1 loopback packet has 4 bytes of internal header before
2510 	 * ether header. Skip these four bytes.
2511 	 */
2512 	if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
2513 		skip_bytes = 4;
2514 
2515 	/* First send SGE . Skip the ether header*/
2516 	s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
2517 			+ skip_bytes;
2518 	s_sge[0].lkey = 0xFFFFFFFF;
2519 	s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
2520 				BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
2521 
2522 	/* Second Send SGE */
2523 	s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
2524 			BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
2525 	if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
2526 		s_sge[1].addr += 8;
2527 	s_sge[1].lkey = 0xFFFFFFFF;
2528 	s_sge[1].length = 256;
2529 
2530 	/* First recv SGE */
2531 
2532 	r_sge[0].addr = shrq_hdr_buf_map;
2533 	r_sge[0].lkey = 0xFFFFFFFF;
2534 	r_sge[0].length = 40;
2535 
2536 	r_sge[1].addr = sqp_entry->sge.addr + offset;
2537 	r_sge[1].lkey = sqp_entry->sge.lkey;
2538 	r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
2539 
2540 	/* Create receive work request */
2541 	rwr.num_sge = 2;
2542 	rwr.sg_list = r_sge;
2543 	rwr.wr_id = tbl_idx;
2544 	rwr.next = NULL;
2545 
2546 	rc = bnxt_re_post_recv_shadow_qp(rdev, qp, &rwr);
2547 	if (rc) {
2548 		dev_err(rdev_to_dev(rdev),
2549 			"Failed to post Rx buffers to shadow QP");
2550 		return -ENOMEM;
2551 	}
2552 
2553 	swr->num_sge = 2;
2554 	swr->sg_list = s_sge;
2555 	swr->wr_id = tbl_idx;
2556 	swr->opcode = IB_WR_SEND;
2557 	swr->next = NULL;
2558 
2559 	udwr.ah = &rdev->sqp_ah->ib_ah;
2560 	udwr.remote_qpn = rdev->qp1_sqp->qplib_qp.id;
2561 	udwr.remote_qkey = rdev->qp1_sqp->qplib_qp.qkey;
2562 
2563 	/* post data received  in the send queue */
2564 	rc = bnxt_re_post_send_shadow_qp(rdev, qp, swr);
2565 
2566 	return 0;
2567 }
2568 
2569 static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
2570 					  struct bnxt_qplib_cqe *cqe)
2571 {
2572 	wc->opcode = IB_WC_RECV;
2573 	wc->status = __rawqp1_to_ib_wc_status(cqe->status);
2574 	wc->wc_flags |= IB_WC_GRH;
2575 }
2576 
2577 static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
2578 				      struct bnxt_qplib_cqe *cqe)
2579 {
2580 	wc->opcode = IB_WC_RECV;
2581 	wc->status = __rc_to_ib_wc_status(cqe->status);
2582 
2583 	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2584 		wc->wc_flags |= IB_WC_WITH_IMM;
2585 	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2586 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2587 	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2588 	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2589 		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2590 }
2591 
2592 static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *qp,
2593 					     struct ib_wc *wc,
2594 					     struct bnxt_qplib_cqe *cqe)
2595 {
2596 	u32 tbl_idx;
2597 	struct bnxt_re_dev *rdev = qp->rdev;
2598 	struct bnxt_re_qp *qp1_qp = NULL;
2599 	struct bnxt_qplib_cqe *orig_cqe = NULL;
2600 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2601 	int nw_type;
2602 
2603 	tbl_idx = cqe->wr_id;
2604 
2605 	sqp_entry = &rdev->sqp_tbl[tbl_idx];
2606 	qp1_qp = sqp_entry->qp1_qp;
2607 	orig_cqe = &sqp_entry->cqe;
2608 
2609 	wc->wr_id = sqp_entry->wrid;
2610 	wc->byte_len = orig_cqe->length;
2611 	wc->qp = &qp1_qp->ib_qp;
2612 
2613 	wc->ex.imm_data = orig_cqe->immdata;
2614 	wc->src_qp = orig_cqe->src_qp;
2615 	memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
2616 	wc->port_num = 1;
2617 	wc->vendor_err = orig_cqe->status;
2618 
2619 	wc->opcode = IB_WC_RECV;
2620 	wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
2621 	wc->wc_flags |= IB_WC_GRH;
2622 
2623 	nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
2624 					    orig_cqe->raweth_qp1_flags2);
2625 	if (nw_type >= 0) {
2626 		wc->network_hdr_type = bnxt_re_to_ib_nw_type(nw_type);
2627 		wc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;
2628 	}
2629 }
2630 
2631 static void bnxt_re_process_res_ud_wc(struct ib_wc *wc,
2632 				      struct bnxt_qplib_cqe *cqe)
2633 {
2634 	wc->opcode = IB_WC_RECV;
2635 	wc->status = __rc_to_ib_wc_status(cqe->status);
2636 
2637 	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
2638 		wc->wc_flags |= IB_WC_WITH_IMM;
2639 	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
2640 		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
2641 	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
2642 	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
2643 		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
2644 }
2645 
2646 int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
2647 {
2648 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2649 	struct bnxt_re_qp *qp;
2650 	struct bnxt_qplib_cqe *cqe;
2651 	int i, ncqe, budget;
2652 	u32 tbl_idx;
2653 	struct bnxt_re_sqp_entries *sqp_entry = NULL;
2654 	unsigned long flags;
2655 
2656 	spin_lock_irqsave(&cq->cq_lock, flags);
2657 	budget = min_t(u32, num_entries, cq->max_cql);
2658 	if (!cq->cql) {
2659 		dev_err(rdev_to_dev(cq->rdev), "POLL CQ : no CQL to use");
2660 		goto exit;
2661 	}
2662 	cqe = &cq->cql[0];
2663 	while (budget) {
2664 		ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget);
2665 		if (!ncqe)
2666 			break;
2667 
2668 		for (i = 0; i < ncqe; i++, cqe++) {
2669 			/* Transcribe each qplib_wqe back to ib_wc */
2670 			memset(wc, 0, sizeof(*wc));
2671 
2672 			wc->wr_id = cqe->wr_id;
2673 			wc->byte_len = cqe->length;
2674 			qp = container_of
2675 				((struct bnxt_qplib_qp *)
2676 				 (unsigned long)(cqe->qp_handle),
2677 				 struct bnxt_re_qp, qplib_qp);
2678 			if (!qp) {
2679 				dev_err(rdev_to_dev(cq->rdev),
2680 					"POLL CQ : bad QP handle");
2681 				continue;
2682 			}
2683 			wc->qp = &qp->ib_qp;
2684 			wc->ex.imm_data = cqe->immdata;
2685 			wc->src_qp = cqe->src_qp;
2686 			memcpy(wc->smac, cqe->smac, ETH_ALEN);
2687 			wc->port_num = 1;
2688 			wc->vendor_err = cqe->status;
2689 
2690 			switch (cqe->opcode) {
2691 			case CQ_BASE_CQE_TYPE_REQ:
2692 				if (qp->qplib_qp.id ==
2693 				    qp->rdev->qp1_sqp->qplib_qp.id) {
2694 					/* Handle this completion with
2695 					 * the stored completion
2696 					 */
2697 					memset(wc, 0, sizeof(*wc));
2698 					continue;
2699 				}
2700 				bnxt_re_process_req_wc(wc, cqe);
2701 				break;
2702 			case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
2703 				if (!cqe->status) {
2704 					int rc = 0;
2705 
2706 					rc = bnxt_re_process_raw_qp_pkt_rx
2707 								(qp, cqe);
2708 					if (!rc) {
2709 						memset(wc, 0, sizeof(*wc));
2710 						continue;
2711 					}
2712 					cqe->status = -1;
2713 				}
2714 				/* Errors need not be looped back.
2715 				 * But change the wr_id to the one
2716 				 * stored in the table
2717 				 */
2718 				tbl_idx = cqe->wr_id;
2719 				sqp_entry = &cq->rdev->sqp_tbl[tbl_idx];
2720 				wc->wr_id = sqp_entry->wrid;
2721 				bnxt_re_process_res_rawqp1_wc(wc, cqe);
2722 				break;
2723 			case CQ_BASE_CQE_TYPE_RES_RC:
2724 				bnxt_re_process_res_rc_wc(wc, cqe);
2725 				break;
2726 			case CQ_BASE_CQE_TYPE_RES_UD:
2727 				if (qp->qplib_qp.id ==
2728 				    qp->rdev->qp1_sqp->qplib_qp.id) {
2729 					/* Handle this completion with
2730 					 * the stored completion
2731 					 */
2732 					if (cqe->status) {
2733 						continue;
2734 					} else {
2735 						bnxt_re_process_res_shadow_qp_wc
2736 								(qp, wc, cqe);
2737 						break;
2738 					}
2739 				}
2740 				bnxt_re_process_res_ud_wc(wc, cqe);
2741 				break;
2742 			default:
2743 				dev_err(rdev_to_dev(cq->rdev),
2744 					"POLL CQ : type 0x%x not handled",
2745 					cqe->opcode);
2746 				continue;
2747 			}
2748 			wc++;
2749 			budget--;
2750 		}
2751 	}
2752 exit:
2753 	spin_unlock_irqrestore(&cq->cq_lock, flags);
2754 	return num_entries - budget;
2755 }
2756 
2757 int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
2758 			  enum ib_cq_notify_flags ib_cqn_flags)
2759 {
2760 	struct bnxt_re_cq *cq = container_of(ib_cq, struct bnxt_re_cq, ib_cq);
2761 	int type = 0;
2762 
2763 	/* Trigger on the very next completion */
2764 	if (ib_cqn_flags & IB_CQ_NEXT_COMP)
2765 		type = DBR_DBR_TYPE_CQ_ARMALL;
2766 	/* Trigger on the next solicited completion */
2767 	else if (ib_cqn_flags & IB_CQ_SOLICITED)
2768 		type = DBR_DBR_TYPE_CQ_ARMSE;
2769 
2770 	bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
2771 
2772 	return 0;
2773 }
2774 
2775 /* Memory Regions */
2776 struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
2777 {
2778 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2779 	struct bnxt_re_dev *rdev = pd->rdev;
2780 	struct bnxt_re_mr *mr;
2781 	u64 pbl = 0;
2782 	int rc;
2783 
2784 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2785 	if (!mr)
2786 		return ERR_PTR(-ENOMEM);
2787 
2788 	mr->rdev = rdev;
2789 	mr->qplib_mr.pd = &pd->qplib_pd;
2790 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
2791 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2792 
2793 	/* Allocate and register 0 as the address */
2794 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2795 	if (rc)
2796 		goto fail;
2797 
2798 	mr->qplib_mr.hwq.level = PBL_LVL_MAX;
2799 	mr->qplib_mr.total_size = -1; /* Infinte length */
2800 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, &pbl, 0, false);
2801 	if (rc)
2802 		goto fail_mr;
2803 
2804 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
2805 	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
2806 			       IB_ACCESS_REMOTE_ATOMIC))
2807 		mr->ib_mr.rkey = mr->ib_mr.lkey;
2808 	atomic_inc(&rdev->mr_count);
2809 
2810 	return &mr->ib_mr;
2811 
2812 fail_mr:
2813 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2814 fail:
2815 	kfree(mr);
2816 	return ERR_PTR(rc);
2817 }
2818 
2819 int bnxt_re_dereg_mr(struct ib_mr *ib_mr)
2820 {
2821 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2822 	struct bnxt_re_dev *rdev = mr->rdev;
2823 	int rc;
2824 
2825 	if (mr->npages && mr->pages) {
2826 		rc = bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
2827 							&mr->qplib_frpl);
2828 		kfree(mr->pages);
2829 		mr->npages = 0;
2830 		mr->pages = NULL;
2831 	}
2832 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2833 
2834 	if (!IS_ERR_OR_NULL(mr->ib_umem))
2835 		ib_umem_release(mr->ib_umem);
2836 
2837 	kfree(mr);
2838 	atomic_dec(&rdev->mr_count);
2839 	return rc;
2840 }
2841 
2842 static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
2843 {
2844 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2845 
2846 	if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
2847 		return -ENOMEM;
2848 
2849 	mr->pages[mr->npages++] = addr;
2850 	return 0;
2851 }
2852 
2853 int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents,
2854 		      unsigned int *sg_offset)
2855 {
2856 	struct bnxt_re_mr *mr = container_of(ib_mr, struct bnxt_re_mr, ib_mr);
2857 
2858 	mr->npages = 0;
2859 	return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page);
2860 }
2861 
2862 struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
2863 			       u32 max_num_sg)
2864 {
2865 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2866 	struct bnxt_re_dev *rdev = pd->rdev;
2867 	struct bnxt_re_mr *mr = NULL;
2868 	int rc;
2869 
2870 	if (type != IB_MR_TYPE_MEM_REG) {
2871 		dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported", type);
2872 		return ERR_PTR(-EINVAL);
2873 	}
2874 	if (max_num_sg > MAX_PBL_LVL_1_PGS)
2875 		return ERR_PTR(-EINVAL);
2876 
2877 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2878 	if (!mr)
2879 		return ERR_PTR(-ENOMEM);
2880 
2881 	mr->rdev = rdev;
2882 	mr->qplib_mr.pd = &pd->qplib_pd;
2883 	mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
2884 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2885 
2886 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
2887 	if (rc)
2888 		goto fail;
2889 
2890 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
2891 	mr->ib_mr.rkey = mr->ib_mr.lkey;
2892 
2893 	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
2894 	if (!mr->pages) {
2895 		rc = -ENOMEM;
2896 		goto fail;
2897 	}
2898 	rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
2899 						 &mr->qplib_frpl, max_num_sg);
2900 	if (rc) {
2901 		dev_err(rdev_to_dev(rdev),
2902 			"Failed to allocate HW FR page list");
2903 		goto fail_mr;
2904 	}
2905 
2906 	atomic_inc(&rdev->mr_count);
2907 	return &mr->ib_mr;
2908 
2909 fail_mr:
2910 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
2911 fail:
2912 	kfree(mr->pages);
2913 	kfree(mr);
2914 	return ERR_PTR(rc);
2915 }
2916 
2917 /* Fast Memory Regions */
2918 struct ib_fmr *bnxt_re_alloc_fmr(struct ib_pd *ib_pd, int mr_access_flags,
2919 				 struct ib_fmr_attr *fmr_attr)
2920 {
2921 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
2922 	struct bnxt_re_dev *rdev = pd->rdev;
2923 	struct bnxt_re_fmr *fmr;
2924 	int rc;
2925 
2926 	if (fmr_attr->max_pages > MAX_PBL_LVL_2_PGS ||
2927 	    fmr_attr->max_maps > rdev->dev_attr.max_map_per_fmr) {
2928 		dev_err(rdev_to_dev(rdev), "Allocate FMR exceeded Max limit");
2929 		return ERR_PTR(-ENOMEM);
2930 	}
2931 	fmr = kzalloc(sizeof(*fmr), GFP_KERNEL);
2932 	if (!fmr)
2933 		return ERR_PTR(-ENOMEM);
2934 
2935 	fmr->rdev = rdev;
2936 	fmr->qplib_fmr.pd = &pd->qplib_pd;
2937 	fmr->qplib_fmr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
2938 
2939 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
2940 	if (rc)
2941 		goto fail;
2942 
2943 	fmr->qplib_fmr.flags = __from_ib_access_flags(mr_access_flags);
2944 	fmr->ib_fmr.lkey = fmr->qplib_fmr.lkey;
2945 	fmr->ib_fmr.rkey = fmr->ib_fmr.lkey;
2946 
2947 	atomic_inc(&rdev->mr_count);
2948 	return &fmr->ib_fmr;
2949 fail:
2950 	kfree(fmr);
2951 	return ERR_PTR(rc);
2952 }
2953 
2954 int bnxt_re_map_phys_fmr(struct ib_fmr *ib_fmr, u64 *page_list, int list_len,
2955 			 u64 iova)
2956 {
2957 	struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2958 					     ib_fmr);
2959 	struct bnxt_re_dev *rdev = fmr->rdev;
2960 	int rc;
2961 
2962 	fmr->qplib_fmr.va = iova;
2963 	fmr->qplib_fmr.total_size = list_len * PAGE_SIZE;
2964 
2965 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &fmr->qplib_fmr, page_list,
2966 			       list_len, true);
2967 	if (rc)
2968 		dev_err(rdev_to_dev(rdev), "Failed to map FMR for lkey = 0x%x!",
2969 			fmr->ib_fmr.lkey);
2970 	return rc;
2971 }
2972 
2973 int bnxt_re_unmap_fmr(struct list_head *fmr_list)
2974 {
2975 	struct bnxt_re_dev *rdev;
2976 	struct bnxt_re_fmr *fmr;
2977 	struct ib_fmr *ib_fmr;
2978 	int rc = 0;
2979 
2980 	/* Validate each FMRs inside the fmr_list */
2981 	list_for_each_entry(ib_fmr, fmr_list, list) {
2982 		fmr = container_of(ib_fmr, struct bnxt_re_fmr, ib_fmr);
2983 		rdev = fmr->rdev;
2984 
2985 		if (rdev) {
2986 			rc = bnxt_qplib_dereg_mrw(&rdev->qplib_res,
2987 						  &fmr->qplib_fmr, true);
2988 			if (rc)
2989 				break;
2990 		}
2991 	}
2992 	return rc;
2993 }
2994 
2995 int bnxt_re_dealloc_fmr(struct ib_fmr *ib_fmr)
2996 {
2997 	struct bnxt_re_fmr *fmr = container_of(ib_fmr, struct bnxt_re_fmr,
2998 					       ib_fmr);
2999 	struct bnxt_re_dev *rdev = fmr->rdev;
3000 	int rc;
3001 
3002 	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &fmr->qplib_fmr);
3003 	if (rc)
3004 		dev_err(rdev_to_dev(rdev), "Failed to free FMR");
3005 
3006 	kfree(fmr);
3007 	atomic_dec(&rdev->mr_count);
3008 	return rc;
3009 }
3010 
3011 /* uverbs */
3012 struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
3013 				  u64 virt_addr, int mr_access_flags,
3014 				  struct ib_udata *udata)
3015 {
3016 	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
3017 	struct bnxt_re_dev *rdev = pd->rdev;
3018 	struct bnxt_re_mr *mr;
3019 	struct ib_umem *umem;
3020 	u64 *pbl_tbl, *pbl_tbl_orig;
3021 	int i, umem_pgs, pages, rc;
3022 	struct scatterlist *sg;
3023 	int entry;
3024 
3025 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3026 	if (!mr)
3027 		return ERR_PTR(-ENOMEM);
3028 
3029 	mr->rdev = rdev;
3030 	mr->qplib_mr.pd = &pd->qplib_pd;
3031 	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
3032 	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
3033 
3034 	umem = ib_umem_get(ib_pd->uobject->context, start, length,
3035 			   mr_access_flags, 0);
3036 	if (IS_ERR(umem)) {
3037 		dev_err(rdev_to_dev(rdev), "Failed to get umem");
3038 		rc = -EFAULT;
3039 		goto free_mr;
3040 	}
3041 	mr->ib_umem = umem;
3042 
3043 	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
3044 	if (rc) {
3045 		dev_err(rdev_to_dev(rdev), "Failed to allocate MR");
3046 		goto release_umem;
3047 	}
3048 	/* The fixed portion of the rkey is the same as the lkey */
3049 	mr->ib_mr.rkey = mr->qplib_mr.rkey;
3050 
3051 	mr->qplib_mr.va = virt_addr;
3052 	umem_pgs = ib_umem_page_count(umem);
3053 	if (!umem_pgs) {
3054 		dev_err(rdev_to_dev(rdev), "umem is invalid!");
3055 		rc = -EINVAL;
3056 		goto free_mrw;
3057 	}
3058 	mr->qplib_mr.total_size = length;
3059 
3060 	pbl_tbl = kcalloc(umem_pgs, sizeof(u64 *), GFP_KERNEL);
3061 	if (!pbl_tbl) {
3062 		rc = -EINVAL;
3063 		goto free_mrw;
3064 	}
3065 	pbl_tbl_orig = pbl_tbl;
3066 
3067 	if (umem->hugetlb) {
3068 		dev_err(rdev_to_dev(rdev), "umem hugetlb not supported!");
3069 		rc = -EFAULT;
3070 		goto fail;
3071 	}
3072 
3073 	if (umem->page_shift != PAGE_SHIFT) {
3074 		dev_err(rdev_to_dev(rdev), "umem page shift unsupported!");
3075 		rc = -EFAULT;
3076 		goto fail;
3077 	}
3078 	/* Map umem buf ptrs to the PBL */
3079 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
3080 		pages = sg_dma_len(sg) >> umem->page_shift;
3081 		for (i = 0; i < pages; i++, pbl_tbl++)
3082 			*pbl_tbl = sg_dma_address(sg) + (i << umem->page_shift);
3083 	}
3084 	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mr->qplib_mr, pbl_tbl_orig,
3085 			       umem_pgs, false);
3086 	if (rc) {
3087 		dev_err(rdev_to_dev(rdev), "Failed to register user MR");
3088 		goto fail;
3089 	}
3090 
3091 	kfree(pbl_tbl_orig);
3092 
3093 	mr->ib_mr.lkey = mr->qplib_mr.lkey;
3094 	mr->ib_mr.rkey = mr->qplib_mr.lkey;
3095 	atomic_inc(&rdev->mr_count);
3096 
3097 	return &mr->ib_mr;
3098 fail:
3099 	kfree(pbl_tbl_orig);
3100 free_mrw:
3101 	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
3102 release_umem:
3103 	ib_umem_release(umem);
3104 free_mr:
3105 	kfree(mr);
3106 	return ERR_PTR(rc);
3107 }
3108 
3109 struct ib_ucontext *bnxt_re_alloc_ucontext(struct ib_device *ibdev,
3110 					   struct ib_udata *udata)
3111 {
3112 	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
3113 	struct bnxt_re_uctx_resp resp;
3114 	struct bnxt_re_ucontext *uctx;
3115 	struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr;
3116 	int rc;
3117 
3118 	dev_dbg(rdev_to_dev(rdev), "ABI version requested %d",
3119 		ibdev->uverbs_abi_ver);
3120 
3121 	if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
3122 		dev_dbg(rdev_to_dev(rdev), " is different from the device %d ",
3123 			BNXT_RE_ABI_VERSION);
3124 		return ERR_PTR(-EPERM);
3125 	}
3126 
3127 	uctx = kzalloc(sizeof(*uctx), GFP_KERNEL);
3128 	if (!uctx)
3129 		return ERR_PTR(-ENOMEM);
3130 
3131 	uctx->rdev = rdev;
3132 
3133 	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
3134 	if (!uctx->shpg) {
3135 		rc = -ENOMEM;
3136 		goto fail;
3137 	}
3138 	spin_lock_init(&uctx->sh_lock);
3139 
3140 	resp.dev_id = rdev->en_dev->pdev->devfn; /*Temp, Use idr_alloc instead*/
3141 	resp.max_qp = rdev->qplib_ctx.qpc_count;
3142 	resp.pg_size = PAGE_SIZE;
3143 	resp.cqe_sz = sizeof(struct cq_base);
3144 	resp.max_cqd = dev_attr->max_cq_wqes;
3145 	resp.rsvd    = 0;
3146 
3147 	rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
3148 	if (rc) {
3149 		dev_err(rdev_to_dev(rdev), "Failed to copy user context");
3150 		rc = -EFAULT;
3151 		goto cfail;
3152 	}
3153 
3154 	return &uctx->ib_uctx;
3155 cfail:
3156 	free_page((unsigned long)uctx->shpg);
3157 	uctx->shpg = NULL;
3158 fail:
3159 	kfree(uctx);
3160 	return ERR_PTR(rc);
3161 }
3162 
3163 int bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
3164 {
3165 	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3166 						   struct bnxt_re_ucontext,
3167 						   ib_uctx);
3168 	if (uctx->shpg)
3169 		free_page((unsigned long)uctx->shpg);
3170 	kfree(uctx);
3171 	return 0;
3172 }
3173 
3174 /* Helper function to mmap the virtual memory from user app */
3175 int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
3176 {
3177 	struct bnxt_re_ucontext *uctx = container_of(ib_uctx,
3178 						   struct bnxt_re_ucontext,
3179 						   ib_uctx);
3180 	struct bnxt_re_dev *rdev = uctx->rdev;
3181 	u64 pfn;
3182 
3183 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
3184 		return -EINVAL;
3185 
3186 	if (vma->vm_pgoff) {
3187 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
3188 		if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
3189 				       PAGE_SIZE, vma->vm_page_prot)) {
3190 			dev_err(rdev_to_dev(rdev), "Failed to map DPI");
3191 			return -EAGAIN;
3192 		}
3193 	} else {
3194 		pfn = virt_to_phys(uctx->shpg) >> PAGE_SHIFT;
3195 		if (remap_pfn_range(vma, vma->vm_start,
3196 				    pfn, PAGE_SIZE, vma->vm_page_prot)) {
3197 			dev_err(rdev_to_dev(rdev),
3198 				"Failed to map shared page");
3199 			return -EAGAIN;
3200 		}
3201 	}
3202 
3203 	return 0;
3204 }
3205