1 /*
2  * Broadcom NetXtreme-E RoCE driver.
3  *
4  * Copyright (c) 2016 - 2017, Broadcom. All rights reserved.  The term
5  * Broadcom refers to Broadcom Limited and/or its subsidiaries.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * BSD license below:
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in
21  *    the documentation and/or other materials provided with the
22  *    distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
31  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
32  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
33  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
34  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  *
36  * Description: Main component of the bnxt_re driver
37  */
38 
39 #include <linux/module.h>
40 #include <linux/netdevice.h>
41 #include <linux/ethtool.h>
42 #include <linux/mutex.h>
43 #include <linux/list.h>
44 #include <linux/rculist.h>
45 #include <linux/spinlock.h>
46 #include <linux/pci.h>
47 #include <net/dcbnl.h>
48 #include <net/ipv6.h>
49 #include <net/addrconf.h>
50 #include <linux/if_ether.h>
51 #include <linux/auxiliary_bus.h>
52 
53 #include <rdma/ib_verbs.h>
54 #include <rdma/ib_user_verbs.h>
55 #include <rdma/ib_umem.h>
56 #include <rdma/ib_addr.h>
57 
58 #include "bnxt_ulp.h"
59 #include "roce_hsi.h"
60 #include "qplib_res.h"
61 #include "qplib_sp.h"
62 #include "qplib_fp.h"
63 #include "qplib_rcfw.h"
64 #include "bnxt_re.h"
65 #include "ib_verbs.h"
66 #include <rdma/bnxt_re-abi.h>
67 #include "bnxt.h"
68 #include "hw_counters.h"
69 
70 static char version[] =
71 		BNXT_RE_DESC "\n";
72 
73 MODULE_AUTHOR("Eddie Wai <eddie.wai@broadcom.com>");
74 MODULE_DESCRIPTION(BNXT_RE_DESC);
75 MODULE_LICENSE("Dual BSD/GPL");
76 
77 /* globals */
78 static DEFINE_MUTEX(bnxt_re_mutex);
79 
80 static void bnxt_re_stop_irq(void *handle);
81 static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev);
82 static int bnxt_re_netdev_event(struct notifier_block *notifier,
83 				unsigned long event, void *ptr);
84 static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev);
85 static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev);
86 static int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
87 
88 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
89 			     u32 *offset);
90 static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
91 {
92 	struct bnxt_qplib_chip_ctx *cctx;
93 	struct bnxt_en_dev *en_dev;
94 	struct bnxt_qplib_res *res;
95 	u32 l2db_len = 0;
96 	u32 offset = 0;
97 	u32 barlen;
98 	int rc;
99 
100 	res = &rdev->qplib_res;
101 	en_dev = rdev->en_dev;
102 	cctx = rdev->chip_ctx;
103 
104 	/* Issue qcfg */
105 	rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset);
106 	if (rc)
107 		dev_info(rdev_to_dev(rdev),
108 			 "Couldn't get DB bar size, Low latency framework is disabled\n");
109 	/* set register offsets for both UC and WC */
110 	res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :
111 						 BNXT_QPLIB_DBR_PF_DB_OFFSET;
112 	res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset;
113 
114 	/* If WC mapping is disabled by L2 driver then en_dev->l2_db_size
115 	 * is equal to the DB-Bar actual size. This indicates that L2
116 	 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping
117 	 * in such cases and DB-push will be disabled.
118 	 */
119 	barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION);
120 	if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) {
121 		res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
122 		dev_info(rdev_to_dev(rdev),  "Low latency framework is enabled\n");
123 	}
124 }
125 
126 static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
127 {
128 	struct bnxt_qplib_chip_ctx *cctx;
129 
130 	cctx = rdev->chip_ctx;
131 	cctx->modes.wqe_mode = bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
132 			       mode : BNXT_QPLIB_WQE_MODE_STATIC;
133 	if (bnxt_re_hwrm_qcaps(rdev))
134 		dev_err(rdev_to_dev(rdev),
135 			"Failed to query hwrm qcaps\n");
136 }
137 
138 static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
139 {
140 	struct bnxt_qplib_chip_ctx *chip_ctx;
141 
142 	if (!rdev->chip_ctx)
143 		return;
144 	chip_ctx = rdev->chip_ctx;
145 	rdev->chip_ctx = NULL;
146 	rdev->rcfw.res = NULL;
147 	rdev->qplib_res.cctx = NULL;
148 	rdev->qplib_res.pdev = NULL;
149 	rdev->qplib_res.netdev = NULL;
150 	kfree(chip_ctx);
151 }
152 
153 static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
154 {
155 	struct bnxt_qplib_chip_ctx *chip_ctx;
156 	struct bnxt_en_dev *en_dev;
157 	int rc;
158 
159 	en_dev = rdev->en_dev;
160 
161 	chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
162 	if (!chip_ctx)
163 		return -ENOMEM;
164 	chip_ctx->chip_num = en_dev->chip_num;
165 	chip_ctx->hw_stats_size = en_dev->hw_ring_stats_size;
166 
167 	rdev->chip_ctx = chip_ctx;
168 	/* rest members to follow eventually */
169 
170 	rdev->qplib_res.cctx = rdev->chip_ctx;
171 	rdev->rcfw.res = &rdev->qplib_res;
172 	rdev->qplib_res.dattr = &rdev->dev_attr;
173 	rdev->qplib_res.is_vf = BNXT_EN_VF(en_dev);
174 
175 	bnxt_re_set_drv_mode(rdev, wqe_mode);
176 
177 	bnxt_re_set_db_offset(rdev);
178 	rc = bnxt_qplib_map_db_bar(&rdev->qplib_res);
179 	if (rc)
180 		return rc;
181 
182 	if (bnxt_qplib_determine_atomics(en_dev->pdev))
183 		ibdev_info(&rdev->ibdev,
184 			   "platform doesn't support global atomics.");
185 	return 0;
186 }
187 
188 /* SR-IOV helper functions */
189 
190 static void bnxt_re_get_sriov_func_type(struct bnxt_re_dev *rdev)
191 {
192 	if (BNXT_EN_VF(rdev->en_dev))
193 		rdev->is_virtfn = 1;
194 }
195 
196 /* Set the maximum number of each resource that the driver actually wants
197  * to allocate. This may be up to the maximum number the firmware has
198  * reserved for the function. The driver may choose to allocate fewer
199  * resources than the firmware maximum.
200  */
201 static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
202 {
203 	struct bnxt_qplib_dev_attr *attr;
204 	struct bnxt_qplib_ctx *ctx;
205 	int i;
206 
207 	attr = &rdev->dev_attr;
208 	ctx = &rdev->qplib_ctx;
209 
210 	ctx->qpc_count = min_t(u32, BNXT_RE_MAX_QPC_COUNT,
211 			       attr->max_qp);
212 	ctx->mrw_count = BNXT_RE_MAX_MRW_COUNT_256K;
213 	/* Use max_mr from fw since max_mrw does not get set */
214 	ctx->mrw_count = min_t(u32, ctx->mrw_count, attr->max_mr);
215 	ctx->srqc_count = min_t(u32, BNXT_RE_MAX_SRQC_COUNT,
216 				attr->max_srq);
217 	ctx->cq_count = min_t(u32, BNXT_RE_MAX_CQ_COUNT, attr->max_cq);
218 	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))
219 		for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
220 			rdev->qplib_ctx.tqm_ctx.qcount[i] =
221 			rdev->dev_attr.tqm_alloc_reqs[i];
222 }
223 
224 static void bnxt_re_limit_vf_res(struct bnxt_qplib_ctx *qplib_ctx, u32 num_vf)
225 {
226 	struct bnxt_qplib_vf_res *vf_res;
227 	u32 mrws = 0;
228 	u32 vf_pct;
229 	u32 nvfs;
230 
231 	vf_res = &qplib_ctx->vf_res;
232 	/*
233 	 * Reserve a set of resources for the PF. Divide the remaining
234 	 * resources among the VFs
235 	 */
236 	vf_pct = 100 - BNXT_RE_PCT_RSVD_FOR_PF;
237 	nvfs = num_vf;
238 	num_vf = 100 * num_vf;
239 	vf_res->max_qp_per_vf = (qplib_ctx->qpc_count * vf_pct) / num_vf;
240 	vf_res->max_srq_per_vf = (qplib_ctx->srqc_count * vf_pct) / num_vf;
241 	vf_res->max_cq_per_vf = (qplib_ctx->cq_count * vf_pct) / num_vf;
242 	/*
243 	 * The driver allows many more MRs than other resources. If the
244 	 * firmware does also, then reserve a fixed amount for the PF and
245 	 * divide the rest among VFs. VFs may use many MRs for NFS
246 	 * mounts, ISER, NVME applications, etc. If the firmware severely
247 	 * restricts the number of MRs, then let PF have half and divide
248 	 * the rest among VFs, as for the other resource types.
249 	 */
250 	if (qplib_ctx->mrw_count < BNXT_RE_MAX_MRW_COUNT_64K) {
251 		mrws = qplib_ctx->mrw_count * vf_pct;
252 		nvfs = num_vf;
253 	} else {
254 		mrws = qplib_ctx->mrw_count - BNXT_RE_RESVD_MR_FOR_PF;
255 	}
256 	vf_res->max_mrw_per_vf = (mrws / nvfs);
257 	vf_res->max_gid_per_vf = BNXT_RE_MAX_GID_PER_VF;
258 }
259 
260 static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
261 {
262 	u32 num_vfs;
263 
264 	memset(&rdev->qplib_ctx.vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
265 	bnxt_re_limit_pf_res(rdev);
266 
267 	num_vfs =  bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
268 			BNXT_RE_GEN_P5_MAX_VF : rdev->num_vfs;
269 	if (num_vfs)
270 		bnxt_re_limit_vf_res(&rdev->qplib_ctx, num_vfs);
271 }
272 
273 static void bnxt_re_vf_res_config(struct bnxt_re_dev *rdev)
274 {
275 
276 	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
277 		return;
278 	rdev->num_vfs = pci_sriov_get_totalvfs(rdev->en_dev->pdev);
279 	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx)) {
280 		bnxt_re_set_resource_limits(rdev);
281 		bnxt_qplib_set_func_resources(&rdev->qplib_res, &rdev->rcfw,
282 					      &rdev->qplib_ctx);
283 	}
284 }
285 
286 static void bnxt_re_shutdown(struct auxiliary_device *adev)
287 {
288 	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
289 
290 	if (!rdev)
291 		return;
292 	ib_unregister_device(&rdev->ibdev);
293 	bnxt_re_dev_uninit(rdev);
294 }
295 
296 static void bnxt_re_stop_irq(void *handle)
297 {
298 	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
299 	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
300 	struct bnxt_qplib_nq *nq;
301 	int indx;
302 
303 	for (indx = BNXT_RE_NQ_IDX; indx < rdev->num_msix; indx++) {
304 		nq = &rdev->nq[indx - 1];
305 		bnxt_qplib_nq_stop_irq(nq, false);
306 	}
307 
308 	bnxt_qplib_rcfw_stop_irq(rcfw, false);
309 }
310 
311 static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
312 {
313 	struct bnxt_re_dev *rdev = (struct bnxt_re_dev *)handle;
314 	struct bnxt_msix_entry *msix_ent = rdev->en_dev->msix_entries;
315 	struct bnxt_qplib_rcfw *rcfw = &rdev->rcfw;
316 	struct bnxt_qplib_nq *nq;
317 	int indx, rc;
318 
319 	if (!ent) {
320 		/* Not setting the f/w timeout bit in rcfw.
321 		 * During the driver unload the first command
322 		 * to f/w will timeout and that will set the
323 		 * timeout bit.
324 		 */
325 		ibdev_err(&rdev->ibdev, "Failed to re-start IRQs\n");
326 		return;
327 	}
328 
329 	/* Vectors may change after restart, so update with new vectors
330 	 * in device sctructure.
331 	 */
332 	for (indx = 0; indx < rdev->num_msix; indx++)
333 		rdev->en_dev->msix_entries[indx].vector = ent[indx].vector;
334 
335 	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
336 				       false);
337 	if (rc) {
338 		ibdev_warn(&rdev->ibdev, "Failed to reinit CREQ\n");
339 		return;
340 	}
341 	for (indx = BNXT_RE_NQ_IDX ; indx < rdev->num_msix; indx++) {
342 		nq = &rdev->nq[indx - 1];
343 		rc = bnxt_qplib_nq_start_irq(nq, indx - 1,
344 					     msix_ent[indx].vector, false);
345 		if (rc) {
346 			ibdev_warn(&rdev->ibdev, "Failed to reinit NQ index %d\n",
347 				   indx - 1);
348 			return;
349 		}
350 	}
351 }
352 
353 static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
354 	.ulp_irq_stop = bnxt_re_stop_irq,
355 	.ulp_irq_restart = bnxt_re_start_irq
356 };
357 
358 /* RoCE -> Net driver */
359 
360 static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
361 {
362 	struct bnxt_en_dev *en_dev;
363 	int rc;
364 
365 	en_dev = rdev->en_dev;
366 
367 	rc = bnxt_register_dev(en_dev, &bnxt_re_ulp_ops, rdev);
368 	if (!rc)
369 		rdev->qplib_res.pdev = rdev->en_dev->pdev;
370 	return rc;
371 }
372 
373 static void bnxt_re_init_hwrm_hdr(struct input *hdr, u16 opcd)
374 {
375 	hdr->req_type = cpu_to_le16(opcd);
376 	hdr->cmpl_ring = cpu_to_le16(-1);
377 	hdr->target_id = cpu_to_le16(-1);
378 }
379 
380 static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
381 				int msg_len, void *resp, int resp_max_len,
382 				int timeout)
383 {
384 	fw_msg->msg = msg;
385 	fw_msg->msg_len = msg_len;
386 	fw_msg->resp = resp;
387 	fw_msg->resp_max_len = resp_max_len;
388 	fw_msg->timeout = timeout;
389 }
390 
391 /* Query device config using common hwrm */
392 static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
393 			     u32 *offset)
394 {
395 	struct bnxt_en_dev *en_dev = rdev->en_dev;
396 	struct hwrm_func_qcfg_output resp = {0};
397 	struct hwrm_func_qcfg_input req = {0};
398 	struct bnxt_fw_msg fw_msg = {};
399 	int rc;
400 
401 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_FUNC_QCFG);
402 	req.fid = cpu_to_le16(0xffff);
403 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
404 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
405 	rc = bnxt_send_msg(en_dev, &fw_msg);
406 	if (!rc) {
407 		*db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);
408 		*offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);
409 	}
410 	return rc;
411 }
412 
413 /* Query function capabilities using common hwrm */
414 int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)
415 {
416 	struct bnxt_en_dev *en_dev = rdev->en_dev;
417 	struct hwrm_func_qcaps_output resp = {};
418 	struct hwrm_func_qcaps_input req = {};
419 	struct bnxt_qplib_chip_ctx *cctx;
420 	struct bnxt_fw_msg fw_msg = {};
421 	int rc;
422 
423 	cctx = rdev->chip_ctx;
424 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_FUNC_QCAPS);
425 	req.fid = cpu_to_le16(0xffff);
426 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
427 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
428 
429 	rc = bnxt_send_msg(en_dev, &fw_msg);
430 	if (rc)
431 		return rc;
432 	cctx->modes.db_push = le32_to_cpu(resp.flags) & FUNC_QCAPS_RESP_FLAGS_WCB_PUSH_MODE;
433 
434 	cctx->modes.dbr_pacing =
435 		le32_to_cpu(resp.flags_ext2) &
436 		FUNC_QCAPS_RESP_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED;
437 	return 0;
438 }
439 
440 static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)
441 {
442 	struct hwrm_func_dbr_pacing_qcfg_output resp = {};
443 	struct hwrm_func_dbr_pacing_qcfg_input req = {};
444 	struct bnxt_en_dev *en_dev = rdev->en_dev;
445 	struct bnxt_qplib_chip_ctx *cctx;
446 	struct bnxt_fw_msg fw_msg = {};
447 	int rc;
448 
449 	cctx = rdev->chip_ctx;
450 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_FUNC_DBR_PACING_QCFG);
451 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
452 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
453 	rc = bnxt_send_msg(en_dev, &fw_msg);
454 	if (rc)
455 		return rc;
456 
457 	if ((le32_to_cpu(resp.dbr_stat_db_fifo_reg) &
458 	    FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==
459 		FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)
460 		cctx->dbr_stat_db_fifo =
461 			le32_to_cpu(resp.dbr_stat_db_fifo_reg) &
462 			~FUNC_DBR_PACING_QCFG_RESP_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
463 	return 0;
464 }
465 
466 /* Update the pacing tunable parameters to the default values */
467 static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)
468 {
469 	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
470 
471 	pacing_data->do_pacing = rdev->pacing.dbr_def_do_pacing;
472 	pacing_data->pacing_th = rdev->pacing.pacing_algo_th;
473 	pacing_data->alarm_th =
474 		pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;
475 }
476 
477 static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)
478 {
479 	u32 read_val, fifo_occup;
480 
481 	/* loop shouldn't run infintely as the occupancy usually goes
482 	 * below pacing algo threshold as soon as pacing kicks in.
483 	 */
484 	while (1) {
485 		read_val = readl(rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off);
486 		fifo_occup = BNXT_RE_MAX_FIFO_DEPTH -
487 			((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >>
488 			 BNXT_RE_DB_FIFO_ROOM_SHIFT);
489 		/* Fifo occupancy cannot be greater the MAX FIFO depth */
490 		if (fifo_occup > BNXT_RE_MAX_FIFO_DEPTH)
491 			break;
492 
493 		if (fifo_occup < rdev->qplib_res.pacing_data->pacing_th)
494 			break;
495 	}
496 }
497 
498 static void bnxt_re_db_fifo_check(struct work_struct *work)
499 {
500 	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
501 			dbq_fifo_check_work);
502 	struct bnxt_qplib_db_pacing_data *pacing_data;
503 	u32 pacing_save;
504 
505 	if (!mutex_trylock(&rdev->pacing.dbq_lock))
506 		return;
507 	pacing_data = rdev->qplib_res.pacing_data;
508 	pacing_save = rdev->pacing.do_pacing_save;
509 	__wait_for_fifo_occupancy_below_th(rdev);
510 	cancel_delayed_work_sync(&rdev->dbq_pacing_work);
511 	if (pacing_save > rdev->pacing.dbr_def_do_pacing) {
512 		/* Double the do_pacing value during the congestion */
513 		pacing_save = pacing_save << 1;
514 	} else {
515 		/*
516 		 * when a new congestion is detected increase the do_pacing
517 		 * by 8 times. And also increase the pacing_th by 4 times. The
518 		 * reason to increase pacing_th is to give more space for the
519 		 * queue to oscillate down without getting empty, but also more
520 		 * room for the queue to increase without causing another alarm.
521 		 */
522 		pacing_save = pacing_save << 3;
523 		pacing_data->pacing_th = rdev->pacing.pacing_algo_th * 4;
524 	}
525 
526 	if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING)
527 		pacing_save = BNXT_RE_MAX_DBR_DO_PACING;
528 
529 	pacing_data->do_pacing = pacing_save;
530 	rdev->pacing.do_pacing_save = pacing_data->do_pacing;
531 	pacing_data->alarm_th =
532 		pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE;
533 	schedule_delayed_work(&rdev->dbq_pacing_work,
534 			      msecs_to_jiffies(rdev->pacing.dbq_pacing_time));
535 	rdev->stats.pacing.alerts++;
536 	mutex_unlock(&rdev->pacing.dbq_lock);
537 }
538 
539 static void bnxt_re_pacing_timer_exp(struct work_struct *work)
540 {
541 	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
542 			dbq_pacing_work.work);
543 	struct bnxt_qplib_db_pacing_data *pacing_data;
544 	u32 read_val, fifo_occup;
545 
546 	if (!mutex_trylock(&rdev->pacing.dbq_lock))
547 		return;
548 
549 	pacing_data = rdev->qplib_res.pacing_data;
550 	read_val = readl(rdev->en_dev->bar0 + rdev->pacing.dbr_db_fifo_reg_off);
551 	fifo_occup = BNXT_RE_MAX_FIFO_DEPTH -
552 		((read_val & BNXT_RE_DB_FIFO_ROOM_MASK) >>
553 		 BNXT_RE_DB_FIFO_ROOM_SHIFT);
554 
555 	if (fifo_occup > pacing_data->pacing_th)
556 		goto restart_timer;
557 
558 	/*
559 	 * Instead of immediately going back to the default do_pacing
560 	 * reduce it by 1/8 times and restart the timer.
561 	 */
562 	pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3);
563 	pacing_data->do_pacing = max_t(u32, rdev->pacing.dbr_def_do_pacing, pacing_data->do_pacing);
564 	if (pacing_data->do_pacing <= rdev->pacing.dbr_def_do_pacing) {
565 		bnxt_re_set_default_pacing_data(rdev);
566 		rdev->stats.pacing.complete++;
567 		goto dbq_unlock;
568 	}
569 
570 restart_timer:
571 	schedule_delayed_work(&rdev->dbq_pacing_work,
572 			      msecs_to_jiffies(rdev->pacing.dbq_pacing_time));
573 	rdev->stats.pacing.resched++;
574 dbq_unlock:
575 	rdev->pacing.do_pacing_save = pacing_data->do_pacing;
576 	mutex_unlock(&rdev->pacing.dbq_lock);
577 }
578 
579 void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)
580 {
581 	struct bnxt_qplib_db_pacing_data *pacing_data;
582 
583 	if (!rdev->pacing.dbr_pacing)
584 		return;
585 	mutex_lock(&rdev->pacing.dbq_lock);
586 	pacing_data = rdev->qplib_res.pacing_data;
587 
588 	/*
589 	 * Increase the alarm_th to max so that other user lib instances do not
590 	 * keep alerting the driver.
591 	 */
592 	pacing_data->alarm_th = BNXT_RE_MAX_FIFO_DEPTH;
593 	pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING;
594 	cancel_work_sync(&rdev->dbq_fifo_check_work);
595 	schedule_work(&rdev->dbq_fifo_check_work);
596 	mutex_unlock(&rdev->pacing.dbq_lock);
597 }
598 
599 static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)
600 {
601 	if (bnxt_re_hwrm_dbr_pacing_qcfg(rdev))
602 		return -EIO;
603 
604 	/* Allocate a page for app use */
605 	rdev->pacing.dbr_page = (void *)__get_free_page(GFP_KERNEL);
606 	if (!rdev->pacing.dbr_page)
607 		return -ENOMEM;
608 
609 	memset((u8 *)rdev->pacing.dbr_page, 0, PAGE_SIZE);
610 	rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->pacing.dbr_page;
611 
612 	/* MAP HW window 2 for reading db fifo depth */
613 	writel(rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK,
614 	       rdev->en_dev->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
615 	rdev->pacing.dbr_db_fifo_reg_off =
616 		(rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) +
617 		 BNXT_RE_GRC_FIFO_REG_BASE;
618 	rdev->pacing.dbr_bar_addr =
619 		pci_resource_start(rdev->qplib_res.pdev, 0) + rdev->pacing.dbr_db_fifo_reg_off;
620 
621 	rdev->pacing.pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;
622 	rdev->pacing.dbq_pacing_time = BNXT_RE_DBR_PACING_TIME;
623 	rdev->pacing.dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;
624 	rdev->pacing.do_pacing_save = rdev->pacing.dbr_def_do_pacing;
625 	rdev->qplib_res.pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH;
626 	rdev->qplib_res.pacing_data->fifo_room_mask = BNXT_RE_DB_FIFO_ROOM_MASK;
627 	rdev->qplib_res.pacing_data->fifo_room_shift = BNXT_RE_DB_FIFO_ROOM_SHIFT;
628 	rdev->qplib_res.pacing_data->grc_reg_offset = rdev->pacing.dbr_db_fifo_reg_off;
629 	bnxt_re_set_default_pacing_data(rdev);
630 	/* Initialize worker for DBR Pacing */
631 	INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check);
632 	INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp);
633 	return 0;
634 }
635 
636 static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)
637 {
638 	cancel_work_sync(&rdev->dbq_fifo_check_work);
639 	cancel_delayed_work_sync(&rdev->dbq_pacing_work);
640 	if (rdev->pacing.dbr_page)
641 		free_page((u64)rdev->pacing.dbr_page);
642 
643 	rdev->pacing.dbr_page = NULL;
644 	rdev->pacing.dbr_pacing = false;
645 }
646 
647 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
648 				 u16 fw_ring_id, int type)
649 {
650 	struct bnxt_en_dev *en_dev;
651 	struct hwrm_ring_free_input req = {};
652 	struct hwrm_ring_free_output resp;
653 	struct bnxt_fw_msg fw_msg = {};
654 	int rc = -EINVAL;
655 
656 	if (!rdev)
657 		return rc;
658 
659 	en_dev = rdev->en_dev;
660 
661 	if (!en_dev)
662 		return rc;
663 
664 	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
665 		return 0;
666 
667 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_RING_FREE);
668 	req.ring_type = type;
669 	req.ring_id = cpu_to_le16(fw_ring_id);
670 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
671 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
672 	rc = bnxt_send_msg(en_dev, &fw_msg);
673 	if (rc)
674 		ibdev_err(&rdev->ibdev, "Failed to free HW ring:%d :%#x",
675 			  req.ring_id, rc);
676 	return rc;
677 }
678 
679 static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
680 				  struct bnxt_re_ring_attr *ring_attr,
681 				  u16 *fw_ring_id)
682 {
683 	struct bnxt_en_dev *en_dev = rdev->en_dev;
684 	struct hwrm_ring_alloc_input req = {};
685 	struct hwrm_ring_alloc_output resp;
686 	struct bnxt_fw_msg fw_msg = {};
687 	int rc = -EINVAL;
688 
689 	if (!en_dev)
690 		return rc;
691 
692 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_RING_ALLOC);
693 	req.enables = 0;
694 	req.page_tbl_addr =  cpu_to_le64(ring_attr->dma_arr[0]);
695 	if (ring_attr->pages > 1) {
696 		/* Page size is in log2 units */
697 		req.page_size = BNXT_PAGE_SHIFT;
698 		req.page_tbl_depth = 1;
699 	}
700 	req.fbo = 0;
701 	/* Association of ring index with doorbell index and MSIX number */
702 	req.logical_id = cpu_to_le16(ring_attr->lrid);
703 	req.length = cpu_to_le32(ring_attr->depth + 1);
704 	req.ring_type = ring_attr->type;
705 	req.int_mode = ring_attr->mode;
706 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
707 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
708 	rc = bnxt_send_msg(en_dev, &fw_msg);
709 	if (!rc)
710 		*fw_ring_id = le16_to_cpu(resp.ring_id);
711 
712 	return rc;
713 }
714 
715 static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
716 				      u32 fw_stats_ctx_id)
717 {
718 	struct bnxt_en_dev *en_dev = rdev->en_dev;
719 	struct hwrm_stat_ctx_free_input req = {};
720 	struct hwrm_stat_ctx_free_output resp = {};
721 	struct bnxt_fw_msg fw_msg = {};
722 	int rc = -EINVAL;
723 
724 	if (!en_dev)
725 		return rc;
726 
727 	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
728 		return 0;
729 
730 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_FREE);
731 	req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
732 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
733 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
734 	rc = bnxt_send_msg(en_dev, &fw_msg);
735 	if (rc)
736 		ibdev_err(&rdev->ibdev, "Failed to free HW stats context %#x",
737 			  rc);
738 
739 	return rc;
740 }
741 
742 static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
743 				       dma_addr_t dma_map,
744 				       u32 *fw_stats_ctx_id)
745 {
746 	struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
747 	struct hwrm_stat_ctx_alloc_output resp = {};
748 	struct hwrm_stat_ctx_alloc_input req = {};
749 	struct bnxt_en_dev *en_dev = rdev->en_dev;
750 	struct bnxt_fw_msg fw_msg = {};
751 	int rc = -EINVAL;
752 
753 	*fw_stats_ctx_id = INVALID_STATS_CTX_ID;
754 
755 	if (!en_dev)
756 		return rc;
757 
758 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_STAT_CTX_ALLOC);
759 	req.update_period_ms = cpu_to_le32(1000);
760 	req.stats_dma_addr = cpu_to_le64(dma_map);
761 	req.stats_dma_length = cpu_to_le16(chip_ctx->hw_stats_size);
762 	req.stat_ctx_flags = STAT_CTX_ALLOC_REQ_STAT_CTX_FLAGS_ROCE;
763 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
764 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
765 	rc = bnxt_send_msg(en_dev, &fw_msg);
766 	if (!rc)
767 		*fw_stats_ctx_id = le32_to_cpu(resp.stat_ctx_id);
768 
769 	return rc;
770 }
771 
772 static void bnxt_re_disassociate_ucontext(struct ib_ucontext *ibcontext)
773 {
774 }
775 
776 /* Device */
777 
778 static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
779 {
780 	struct ib_device *ibdev =
781 		ib_device_get_by_netdev(netdev, RDMA_DRIVER_BNXT_RE);
782 	if (!ibdev)
783 		return NULL;
784 
785 	return container_of(ibdev, struct bnxt_re_dev, ibdev);
786 }
787 
788 static ssize_t hw_rev_show(struct device *device, struct device_attribute *attr,
789 			   char *buf)
790 {
791 	struct bnxt_re_dev *rdev =
792 		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
793 
794 	return sysfs_emit(buf, "0x%x\n", rdev->en_dev->pdev->vendor);
795 }
796 static DEVICE_ATTR_RO(hw_rev);
797 
798 static ssize_t hca_type_show(struct device *device,
799 			     struct device_attribute *attr, char *buf)
800 {
801 	struct bnxt_re_dev *rdev =
802 		rdma_device_to_drv_device(device, struct bnxt_re_dev, ibdev);
803 
804 	return sysfs_emit(buf, "%s\n", rdev->ibdev.node_desc);
805 }
806 static DEVICE_ATTR_RO(hca_type);
807 
808 static struct attribute *bnxt_re_attributes[] = {
809 	&dev_attr_hw_rev.attr,
810 	&dev_attr_hca_type.attr,
811 	NULL
812 };
813 
814 static const struct attribute_group bnxt_re_dev_attr_group = {
815 	.attrs = bnxt_re_attributes,
816 };
817 
818 static const struct ib_device_ops bnxt_re_dev_ops = {
819 	.owner = THIS_MODULE,
820 	.driver_id = RDMA_DRIVER_BNXT_RE,
821 	.uverbs_abi_ver = BNXT_RE_ABI_VERSION,
822 
823 	.add_gid = bnxt_re_add_gid,
824 	.alloc_hw_port_stats = bnxt_re_ib_alloc_hw_port_stats,
825 	.alloc_mr = bnxt_re_alloc_mr,
826 	.alloc_pd = bnxt_re_alloc_pd,
827 	.alloc_ucontext = bnxt_re_alloc_ucontext,
828 	.create_ah = bnxt_re_create_ah,
829 	.create_cq = bnxt_re_create_cq,
830 	.create_qp = bnxt_re_create_qp,
831 	.create_srq = bnxt_re_create_srq,
832 	.create_user_ah = bnxt_re_create_ah,
833 	.dealloc_pd = bnxt_re_dealloc_pd,
834 	.dealloc_ucontext = bnxt_re_dealloc_ucontext,
835 	.del_gid = bnxt_re_del_gid,
836 	.dereg_mr = bnxt_re_dereg_mr,
837 	.destroy_ah = bnxt_re_destroy_ah,
838 	.destroy_cq = bnxt_re_destroy_cq,
839 	.destroy_qp = bnxt_re_destroy_qp,
840 	.destroy_srq = bnxt_re_destroy_srq,
841 	.device_group = &bnxt_re_dev_attr_group,
842 	.disassociate_ucontext = bnxt_re_disassociate_ucontext,
843 	.get_dev_fw_str = bnxt_re_query_fw_str,
844 	.get_dma_mr = bnxt_re_get_dma_mr,
845 	.get_hw_stats = bnxt_re_ib_get_hw_stats,
846 	.get_link_layer = bnxt_re_get_link_layer,
847 	.get_port_immutable = bnxt_re_get_port_immutable,
848 	.map_mr_sg = bnxt_re_map_mr_sg,
849 	.mmap = bnxt_re_mmap,
850 	.mmap_free = bnxt_re_mmap_free,
851 	.modify_qp = bnxt_re_modify_qp,
852 	.modify_srq = bnxt_re_modify_srq,
853 	.poll_cq = bnxt_re_poll_cq,
854 	.post_recv = bnxt_re_post_recv,
855 	.post_send = bnxt_re_post_send,
856 	.post_srq_recv = bnxt_re_post_srq_recv,
857 	.query_ah = bnxt_re_query_ah,
858 	.query_device = bnxt_re_query_device,
859 	.query_pkey = bnxt_re_query_pkey,
860 	.query_port = bnxt_re_query_port,
861 	.query_qp = bnxt_re_query_qp,
862 	.query_srq = bnxt_re_query_srq,
863 	.reg_user_mr = bnxt_re_reg_user_mr,
864 	.reg_user_mr_dmabuf = bnxt_re_reg_user_mr_dmabuf,
865 	.req_notify_cq = bnxt_re_req_notify_cq,
866 	.resize_cq = bnxt_re_resize_cq,
867 	INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah),
868 	INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq),
869 	INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd),
870 	INIT_RDMA_OBJ_SIZE(ib_qp, bnxt_re_qp, ib_qp),
871 	INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq),
872 	INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx),
873 };
874 
875 static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
876 {
877 	struct ib_device *ibdev = &rdev->ibdev;
878 	int ret;
879 
880 	/* ib device init */
881 	ibdev->node_type = RDMA_NODE_IB_CA;
882 	strscpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
883 		strlen(BNXT_RE_DESC) + 5);
884 	ibdev->phys_port_cnt = 1;
885 
886 	addrconf_addr_eui48((u8 *)&ibdev->node_guid, rdev->netdev->dev_addr);
887 
888 	ibdev->num_comp_vectors	= rdev->num_msix - 1;
889 	ibdev->dev.parent = &rdev->en_dev->pdev->dev;
890 	ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
891 
892 	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
893 		ibdev->driver_def = bnxt_re_uapi_defs;
894 
895 	ib_set_device_ops(ibdev, &bnxt_re_dev_ops);
896 	ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1);
897 	if (ret)
898 		return ret;
899 
900 	dma_set_max_seg_size(&rdev->en_dev->pdev->dev, UINT_MAX);
901 	ibdev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ);
902 	return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev);
903 }
904 
905 static struct bnxt_re_dev *bnxt_re_dev_add(struct bnxt_aux_priv *aux_priv,
906 					   struct bnxt_en_dev *en_dev)
907 {
908 	struct bnxt_re_dev *rdev;
909 
910 	/* Allocate bnxt_re_dev instance here */
911 	rdev = ib_alloc_device(bnxt_re_dev, ibdev);
912 	if (!rdev) {
913 		ibdev_err(NULL, "%s: bnxt_re_dev allocation failure!",
914 			  ROCE_DRV_MODULE_NAME);
915 		return NULL;
916 	}
917 	/* Default values */
918 	rdev->nb.notifier_call = NULL;
919 	rdev->netdev = en_dev->net;
920 	rdev->en_dev = en_dev;
921 	rdev->id = rdev->en_dev->pdev->devfn;
922 	INIT_LIST_HEAD(&rdev->qp_list);
923 	mutex_init(&rdev->qp_lock);
924 	mutex_init(&rdev->pacing.dbq_lock);
925 	atomic_set(&rdev->stats.res.qp_count, 0);
926 	atomic_set(&rdev->stats.res.cq_count, 0);
927 	atomic_set(&rdev->stats.res.srq_count, 0);
928 	atomic_set(&rdev->stats.res.mr_count, 0);
929 	atomic_set(&rdev->stats.res.mw_count, 0);
930 	atomic_set(&rdev->stats.res.ah_count, 0);
931 	atomic_set(&rdev->stats.res.pd_count, 0);
932 	rdev->cosq[0] = 0xFFFF;
933 	rdev->cosq[1] = 0xFFFF;
934 
935 	return rdev;
936 }
937 
938 static int bnxt_re_handle_unaffi_async_event(struct creq_func_event
939 					     *unaffi_async)
940 {
941 	switch (unaffi_async->event) {
942 	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
943 		break;
944 	case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
945 		break;
946 	case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
947 		break;
948 	case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
949 		break;
950 	case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
951 		break;
952 	case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
953 		break;
954 	case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
955 		break;
956 	case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
957 		break;
958 	case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
959 		break;
960 	case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
961 		break;
962 	case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
963 		break;
964 	default:
965 		return -EINVAL;
966 	}
967 	return 0;
968 }
969 
970 static int bnxt_re_handle_qp_async_event(struct creq_qp_event *qp_event,
971 					 struct bnxt_re_qp *qp)
972 {
973 	struct ib_event event = {};
974 	unsigned int flags;
975 
976 	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &&
977 	    rdma_is_kernel_res(&qp->ib_qp.res)) {
978 		flags = bnxt_re_lock_cqs(qp);
979 		bnxt_qplib_add_flush_qp(&qp->qplib_qp);
980 		bnxt_re_unlock_cqs(qp, flags);
981 	}
982 
983 	if (qp->qplib_qp.srq) {
984 		event.device = &qp->rdev->ibdev;
985 		event.element.qp = &qp->ib_qp;
986 		event.event = IB_EVENT_QP_LAST_WQE_REACHED;
987 	}
988 
989 	if (event.device && qp->ib_qp.event_handler)
990 		qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
991 
992 	return 0;
993 }
994 
995 static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
996 					   void *obj)
997 {
998 	int rc = 0;
999 	u8 event;
1000 
1001 	if (!obj)
1002 		return rc; /* QP was already dead, still return success */
1003 
1004 	event = affi_async->event;
1005 	if (event == CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION) {
1006 		struct bnxt_qplib_qp *lib_qp = obj;
1007 		struct bnxt_re_qp *qp = container_of(lib_qp, struct bnxt_re_qp,
1008 						     qplib_qp);
1009 		rc = bnxt_re_handle_qp_async_event(affi_async, qp);
1010 	}
1011 	return rc;
1012 }
1013 
1014 static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
1015 			       void *aeqe, void *obj)
1016 {
1017 	struct creq_qp_event *affi_async;
1018 	struct creq_func_event *unaffi_async;
1019 	u8 type;
1020 	int rc;
1021 
1022 	type = ((struct creq_base *)aeqe)->type;
1023 	if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
1024 		unaffi_async = aeqe;
1025 		rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
1026 	} else {
1027 		affi_async = aeqe;
1028 		rc = bnxt_re_handle_affi_async_event(affi_async, obj);
1029 	}
1030 
1031 	return rc;
1032 }
1033 
1034 static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
1035 				struct bnxt_qplib_srq *handle, u8 event)
1036 {
1037 	struct bnxt_re_srq *srq = container_of(handle, struct bnxt_re_srq,
1038 					       qplib_srq);
1039 	struct ib_event ib_event;
1040 
1041 	ib_event.device = &srq->rdev->ibdev;
1042 	ib_event.element.srq = &srq->ib_srq;
1043 	if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
1044 		ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
1045 	else
1046 		ib_event.event = IB_EVENT_SRQ_ERR;
1047 
1048 	if (srq->ib_srq.event_handler) {
1049 		/* Lock event_handler? */
1050 		(*srq->ib_srq.event_handler)(&ib_event,
1051 					     srq->ib_srq.srq_context);
1052 	}
1053 	return 0;
1054 }
1055 
1056 static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
1057 			       struct bnxt_qplib_cq *handle)
1058 {
1059 	struct bnxt_re_cq *cq = container_of(handle, struct bnxt_re_cq,
1060 					     qplib_cq);
1061 
1062 	if (cq->ib_cq.comp_handler) {
1063 		/* Lock comp_handler? */
1064 		(*cq->ib_cq.comp_handler)(&cq->ib_cq, cq->ib_cq.cq_context);
1065 	}
1066 
1067 	return 0;
1068 }
1069 
1070 #define BNXT_RE_GEN_P5_PF_NQ_DB		0x10000
1071 #define BNXT_RE_GEN_P5_VF_NQ_DB		0x4000
1072 static u32 bnxt_re_get_nqdb_offset(struct bnxt_re_dev *rdev, u16 indx)
1073 {
1074 	return bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx) ?
1075 		(rdev->is_virtfn ? BNXT_RE_GEN_P5_VF_NQ_DB :
1076 				   BNXT_RE_GEN_P5_PF_NQ_DB) :
1077 				   rdev->en_dev->msix_entries[indx].db_offset;
1078 }
1079 
1080 static void bnxt_re_cleanup_res(struct bnxt_re_dev *rdev)
1081 {
1082 	int i;
1083 
1084 	for (i = 1; i < rdev->num_msix; i++)
1085 		bnxt_qplib_disable_nq(&rdev->nq[i - 1]);
1086 
1087 	if (rdev->qplib_res.rcfw)
1088 		bnxt_qplib_cleanup_res(&rdev->qplib_res);
1089 }
1090 
1091 static int bnxt_re_init_res(struct bnxt_re_dev *rdev)
1092 {
1093 	int num_vec_enabled = 0;
1094 	int rc = 0, i;
1095 	u32 db_offt;
1096 
1097 	bnxt_qplib_init_res(&rdev->qplib_res);
1098 
1099 	for (i = 1; i < rdev->num_msix ; i++) {
1100 		db_offt = bnxt_re_get_nqdb_offset(rdev, i);
1101 		rc = bnxt_qplib_enable_nq(rdev->en_dev->pdev, &rdev->nq[i - 1],
1102 					  i - 1, rdev->en_dev->msix_entries[i].vector,
1103 					  db_offt, &bnxt_re_cqn_handler,
1104 					  &bnxt_re_srqn_handler);
1105 		if (rc) {
1106 			ibdev_err(&rdev->ibdev,
1107 				  "Failed to enable NQ with rc = 0x%x", rc);
1108 			goto fail;
1109 		}
1110 		num_vec_enabled++;
1111 	}
1112 	return 0;
1113 fail:
1114 	for (i = num_vec_enabled; i >= 0; i--)
1115 		bnxt_qplib_disable_nq(&rdev->nq[i]);
1116 	return rc;
1117 }
1118 
1119 static void bnxt_re_free_nq_res(struct bnxt_re_dev *rdev)
1120 {
1121 	u8 type;
1122 	int i;
1123 
1124 	for (i = 0; i < rdev->num_msix - 1; i++) {
1125 		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1126 		bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
1127 		bnxt_qplib_free_nq(&rdev->nq[i]);
1128 		rdev->nq[i].res = NULL;
1129 	}
1130 }
1131 
1132 static void bnxt_re_free_res(struct bnxt_re_dev *rdev)
1133 {
1134 	bnxt_re_free_nq_res(rdev);
1135 
1136 	if (rdev->qplib_res.dpi_tbl.max) {
1137 		bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
1138 				       &rdev->dpi_privileged);
1139 	}
1140 	if (rdev->qplib_res.rcfw) {
1141 		bnxt_qplib_free_res(&rdev->qplib_res);
1142 		rdev->qplib_res.rcfw = NULL;
1143 	}
1144 }
1145 
1146 static int bnxt_re_alloc_res(struct bnxt_re_dev *rdev)
1147 {
1148 	struct bnxt_re_ring_attr rattr = {};
1149 	int num_vec_created = 0;
1150 	int rc, i;
1151 	u8 type;
1152 
1153 	/* Configure and allocate resources for qplib */
1154 	rdev->qplib_res.rcfw = &rdev->rcfw;
1155 	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
1156 	if (rc)
1157 		goto fail;
1158 
1159 	rc = bnxt_qplib_alloc_res(&rdev->qplib_res, rdev->en_dev->pdev,
1160 				  rdev->netdev, &rdev->dev_attr);
1161 	if (rc)
1162 		goto fail;
1163 
1164 	rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res,
1165 				  &rdev->dpi_privileged,
1166 				  rdev, BNXT_QPLIB_DPI_TYPE_KERNEL);
1167 	if (rc)
1168 		goto dealloc_res;
1169 
1170 	for (i = 0; i < rdev->num_msix - 1; i++) {
1171 		struct bnxt_qplib_nq *nq;
1172 
1173 		nq = &rdev->nq[i];
1174 		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
1175 		rc = bnxt_qplib_alloc_nq(&rdev->qplib_res, &rdev->nq[i]);
1176 		if (rc) {
1177 			ibdev_err(&rdev->ibdev, "Alloc Failed NQ%d rc:%#x",
1178 				  i, rc);
1179 			goto free_nq;
1180 		}
1181 		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1182 		rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
1183 		rattr.pages = nq->hwq.pbl[rdev->nq[i].hwq.level].pg_count;
1184 		rattr.type = type;
1185 		rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
1186 		rattr.depth = BNXT_QPLIB_NQE_MAX_CNT - 1;
1187 		rattr.lrid = rdev->en_dev->msix_entries[i + 1].ring_idx;
1188 		rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
1189 		if (rc) {
1190 			ibdev_err(&rdev->ibdev,
1191 				  "Failed to allocate NQ fw id with rc = 0x%x",
1192 				  rc);
1193 			bnxt_qplib_free_nq(&rdev->nq[i]);
1194 			goto free_nq;
1195 		}
1196 		num_vec_created++;
1197 	}
1198 	return 0;
1199 free_nq:
1200 	for (i = num_vec_created - 1; i >= 0; i--) {
1201 		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1202 		bnxt_re_net_ring_free(rdev, rdev->nq[i].ring_id, type);
1203 		bnxt_qplib_free_nq(&rdev->nq[i]);
1204 	}
1205 	bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
1206 			       &rdev->dpi_privileged);
1207 dealloc_res:
1208 	bnxt_qplib_free_res(&rdev->qplib_res);
1209 
1210 fail:
1211 	rdev->qplib_res.rcfw = NULL;
1212 	return rc;
1213 }
1214 
1215 static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
1216 				   u8 port_num, enum ib_event_type event)
1217 {
1218 	struct ib_event ib_event;
1219 
1220 	ib_event.device = ibdev;
1221 	if (qp) {
1222 		ib_event.element.qp = qp;
1223 		ib_event.event = event;
1224 		if (qp->event_handler)
1225 			qp->event_handler(&ib_event, qp->qp_context);
1226 
1227 	} else {
1228 		ib_event.element.port_num = port_num;
1229 		ib_event.event = event;
1230 		ib_dispatch_event(&ib_event);
1231 	}
1232 }
1233 
1234 static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
1235 					struct bnxt_re_qp *qp)
1236 {
1237 	return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
1238 	       (qp == rdev->gsi_ctx.gsi_sqp);
1239 }
1240 
1241 static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
1242 {
1243 	int mask = IB_QP_STATE;
1244 	struct ib_qp_attr qp_attr;
1245 	struct bnxt_re_qp *qp;
1246 
1247 	qp_attr.qp_state = IB_QPS_ERR;
1248 	mutex_lock(&rdev->qp_lock);
1249 	list_for_each_entry(qp, &rdev->qp_list, list) {
1250 		/* Modify the state of all QPs except QP1/Shadow QP */
1251 		if (!bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
1252 			if (qp->qplib_qp.state !=
1253 			    CMDQ_MODIFY_QP_NEW_STATE_RESET &&
1254 			    qp->qplib_qp.state !=
1255 			    CMDQ_MODIFY_QP_NEW_STATE_ERR) {
1256 				bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
1257 						       1, IB_EVENT_QP_FATAL);
1258 				bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask,
1259 						  NULL);
1260 			}
1261 		}
1262 	}
1263 	mutex_unlock(&rdev->qp_lock);
1264 }
1265 
1266 static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
1267 {
1268 	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
1269 	struct bnxt_qplib_gid gid;
1270 	u16 gid_idx, index;
1271 	int rc = 0;
1272 
1273 	if (!ib_device_try_get(&rdev->ibdev))
1274 		return 0;
1275 
1276 	for (index = 0; index < sgid_tbl->active; index++) {
1277 		gid_idx = sgid_tbl->hw_id[index];
1278 
1279 		if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
1280 			    sizeof(bnxt_qplib_gid_zero)))
1281 			continue;
1282 		/* need to modify the VLAN enable setting of non VLAN GID only
1283 		 * as setting is done for VLAN GID while adding GID
1284 		 */
1285 		if (sgid_tbl->vlan[index])
1286 			continue;
1287 
1288 		memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
1289 
1290 		rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
1291 					    rdev->qplib_res.netdev->dev_addr);
1292 	}
1293 
1294 	ib_device_put(&rdev->ibdev);
1295 	return rc;
1296 }
1297 
1298 static u32 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev)
1299 {
1300 	u32 prio_map = 0, tmp_map = 0;
1301 	struct net_device *netdev;
1302 	struct dcb_app app = {};
1303 
1304 	netdev = rdev->netdev;
1305 
1306 	app.selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE;
1307 	app.protocol = ETH_P_IBOE;
1308 	tmp_map = dcb_ieee_getapp_mask(netdev, &app);
1309 	prio_map = tmp_map;
1310 
1311 	app.selector = IEEE_8021QAZ_APP_SEL_DGRAM;
1312 	app.protocol = ROCE_V2_UDP_DPORT;
1313 	tmp_map = dcb_ieee_getapp_mask(netdev, &app);
1314 	prio_map |= tmp_map;
1315 
1316 	return prio_map;
1317 }
1318 
1319 static int bnxt_re_setup_qos(struct bnxt_re_dev *rdev)
1320 {
1321 	u8 prio_map = 0;
1322 
1323 	/* Get priority for roce */
1324 	prio_map = bnxt_re_get_priority_mask(rdev);
1325 
1326 	if (prio_map == rdev->cur_prio_map)
1327 		return 0;
1328 	rdev->cur_prio_map = prio_map;
1329 	/* Actual priorities are not programmed as they are already
1330 	 * done by L2 driver; just enable or disable priority vlan tagging
1331 	 */
1332 	if ((prio_map == 0 && rdev->qplib_res.prio) ||
1333 	    (prio_map != 0 && !rdev->qplib_res.prio)) {
1334 		rdev->qplib_res.prio = prio_map;
1335 		bnxt_re_update_gid(rdev);
1336 	}
1337 
1338 	return 0;
1339 }
1340 
1341 static void bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
1342 {
1343 	struct bnxt_en_dev *en_dev = rdev->en_dev;
1344 	struct hwrm_ver_get_output resp = {};
1345 	struct hwrm_ver_get_input req = {};
1346 	struct bnxt_qplib_chip_ctx *cctx;
1347 	struct bnxt_fw_msg fw_msg = {};
1348 	int rc;
1349 
1350 	bnxt_re_init_hwrm_hdr((void *)&req, HWRM_VER_GET);
1351 	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
1352 	req.hwrm_intf_min = HWRM_VERSION_MINOR;
1353 	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
1354 	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
1355 			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
1356 	rc = bnxt_send_msg(en_dev, &fw_msg);
1357 	if (rc) {
1358 		ibdev_err(&rdev->ibdev, "Failed to query HW version, rc = 0x%x",
1359 			  rc);
1360 		return;
1361 	}
1362 
1363 	cctx = rdev->chip_ctx;
1364 	cctx->hwrm_intf_ver =
1365 		(u64)le16_to_cpu(resp.hwrm_intf_major) << 48 |
1366 		(u64)le16_to_cpu(resp.hwrm_intf_minor) << 32 |
1367 		(u64)le16_to_cpu(resp.hwrm_intf_build) << 16 |
1368 		le16_to_cpu(resp.hwrm_intf_patch);
1369 
1370 	cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);
1371 
1372 	if (!cctx->hwrm_cmd_max_timeout)
1373 		cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;
1374 }
1375 
1376 static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
1377 {
1378 	int rc;
1379 	u32 event;
1380 
1381 	/* Register ib dev */
1382 	rc = bnxt_re_register_ib(rdev);
1383 	if (rc) {
1384 		pr_err("Failed to register with IB: %#x\n", rc);
1385 		return rc;
1386 	}
1387 	dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");
1388 	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
1389 
1390 	event = netif_running(rdev->netdev) && netif_carrier_ok(rdev->netdev) ?
1391 		IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
1392 
1393 	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, event);
1394 
1395 	return rc;
1396 }
1397 
1398 static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev)
1399 {
1400 	u8 type;
1401 	int rc;
1402 
1403 	if (test_and_clear_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags))
1404 		cancel_delayed_work_sync(&rdev->worker);
1405 
1406 	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED,
1407 			       &rdev->flags))
1408 		bnxt_re_cleanup_res(rdev);
1409 	if (test_and_clear_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags))
1410 		bnxt_re_free_res(rdev);
1411 
1412 	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) {
1413 		rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
1414 		if (rc)
1415 			ibdev_warn(&rdev->ibdev,
1416 				   "Failed to deinitialize RCFW: %#x", rc);
1417 		bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
1418 		bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
1419 		bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1420 		type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1421 		bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
1422 		bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1423 	}
1424 
1425 	rdev->num_msix = 0;
1426 
1427 	if (rdev->pacing.dbr_pacing)
1428 		bnxt_re_deinitialize_dbr_pacing(rdev);
1429 
1430 	bnxt_re_destroy_chip_ctx(rdev);
1431 	if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags))
1432 		bnxt_unregister_dev(rdev->en_dev);
1433 }
1434 
1435 /* worker thread for polling periodic events. Now used for QoS programming*/
1436 static void bnxt_re_worker(struct work_struct *work)
1437 {
1438 	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
1439 						worker.work);
1440 
1441 	bnxt_re_setup_qos(rdev);
1442 	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1443 }
1444 
1445 static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
1446 {
1447 	struct bnxt_re_ring_attr rattr = {};
1448 	struct bnxt_qplib_creq_ctx *creq;
1449 	u32 db_offt;
1450 	int vid;
1451 	u8 type;
1452 	int rc;
1453 
1454 	/* Registered a new RoCE device instance to netdev */
1455 	rc = bnxt_re_register_netdev(rdev);
1456 	if (rc) {
1457 		ibdev_err(&rdev->ibdev,
1458 			  "Failed to register with netedev: %#x\n", rc);
1459 		return -EINVAL;
1460 	}
1461 	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
1462 
1463 	rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
1464 	if (rc) {
1465 		bnxt_unregister_dev(rdev->en_dev);
1466 		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
1467 		ibdev_err(&rdev->ibdev, "Failed to get chip context\n");
1468 		return -EINVAL;
1469 	}
1470 
1471 	/* Check whether VF or PF */
1472 	bnxt_re_get_sriov_func_type(rdev);
1473 
1474 	if (!rdev->en_dev->ulp_tbl->msix_requested) {
1475 		ibdev_err(&rdev->ibdev,
1476 			  "Failed to get MSI-X vectors: %#x\n", rc);
1477 		rc = -EINVAL;
1478 		goto fail;
1479 	}
1480 	ibdev_dbg(&rdev->ibdev, "Got %d MSI-X vectors\n",
1481 		  rdev->en_dev->ulp_tbl->msix_requested);
1482 	rdev->num_msix = rdev->en_dev->ulp_tbl->msix_requested;
1483 
1484 	bnxt_re_query_hwrm_intf_version(rdev);
1485 
1486 	/* Establish RCFW Communication Channel to initialize the context
1487 	 * memory for the function and all child VFs
1488 	 */
1489 	rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res, &rdev->rcfw,
1490 					   &rdev->qplib_ctx,
1491 					   BNXT_RE_MAX_QPC_COUNT);
1492 	if (rc) {
1493 		ibdev_err(&rdev->ibdev,
1494 			  "Failed to allocate RCFW Channel: %#x\n", rc);
1495 		goto fail;
1496 	}
1497 
1498 	type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1499 	creq = &rdev->rcfw.creq;
1500 	rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
1501 	rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
1502 	rattr.type = type;
1503 	rattr.mode = RING_ALLOC_REQ_INT_MODE_MSIX;
1504 	rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
1505 	rattr.lrid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
1506 	rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
1507 	if (rc) {
1508 		ibdev_err(&rdev->ibdev, "Failed to allocate CREQ: %#x\n", rc);
1509 		goto free_rcfw;
1510 	}
1511 	db_offt = bnxt_re_get_nqdb_offset(rdev, BNXT_RE_AEQ_IDX);
1512 	vid = rdev->en_dev->msix_entries[BNXT_RE_AEQ_IDX].vector;
1513 	rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw,
1514 					    vid, db_offt,
1515 					    &bnxt_re_aeq_handler);
1516 	if (rc) {
1517 		ibdev_err(&rdev->ibdev, "Failed to enable RCFW channel: %#x\n",
1518 			  rc);
1519 		goto free_ring;
1520 	}
1521 
1522 	if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) {
1523 		rc = bnxt_re_initialize_dbr_pacing(rdev);
1524 		if (!rc) {
1525 			rdev->pacing.dbr_pacing = true;
1526 		} else {
1527 			ibdev_err(&rdev->ibdev,
1528 				  "DBR pacing disabled with error : %d\n", rc);
1529 			rdev->pacing.dbr_pacing = false;
1530 		}
1531 	}
1532 	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw, &rdev->dev_attr);
1533 	if (rc)
1534 		goto disable_rcfw;
1535 
1536 	bnxt_re_set_resource_limits(rdev);
1537 
1538 	rc = bnxt_qplib_alloc_ctx(&rdev->qplib_res, &rdev->qplib_ctx, 0,
1539 				  bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx));
1540 	if (rc) {
1541 		ibdev_err(&rdev->ibdev,
1542 			  "Failed to allocate QPLIB context: %#x\n", rc);
1543 		goto disable_rcfw;
1544 	}
1545 	rc = bnxt_re_net_stats_ctx_alloc(rdev,
1546 					 rdev->qplib_ctx.stats.dma_map,
1547 					 &rdev->qplib_ctx.stats.fw_id);
1548 	if (rc) {
1549 		ibdev_err(&rdev->ibdev,
1550 			  "Failed to allocate stats context: %#x\n", rc);
1551 		goto free_ctx;
1552 	}
1553 
1554 	rc = bnxt_qplib_init_rcfw(&rdev->rcfw, &rdev->qplib_ctx,
1555 				  rdev->is_virtfn);
1556 	if (rc) {
1557 		ibdev_err(&rdev->ibdev,
1558 			  "Failed to initialize RCFW: %#x\n", rc);
1559 		goto free_sctx;
1560 	}
1561 	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
1562 
1563 	/* Resources based on the 'new' device caps */
1564 	rc = bnxt_re_alloc_res(rdev);
1565 	if (rc) {
1566 		ibdev_err(&rdev->ibdev,
1567 			  "Failed to allocate resources: %#x\n", rc);
1568 		goto fail;
1569 	}
1570 	set_bit(BNXT_RE_FLAG_RESOURCES_ALLOCATED, &rdev->flags);
1571 	rc = bnxt_re_init_res(rdev);
1572 	if (rc) {
1573 		ibdev_err(&rdev->ibdev,
1574 			  "Failed to initialize resources: %#x\n", rc);
1575 		goto fail;
1576 	}
1577 
1578 	set_bit(BNXT_RE_FLAG_RESOURCES_INITIALIZED, &rdev->flags);
1579 
1580 	if (!rdev->is_virtfn) {
1581 		rc = bnxt_re_setup_qos(rdev);
1582 		if (rc)
1583 			ibdev_info(&rdev->ibdev,
1584 				   "RoCE priority not yet configured\n");
1585 
1586 		INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
1587 		set_bit(BNXT_RE_FLAG_QOS_WORK_REG, &rdev->flags);
1588 		schedule_delayed_work(&rdev->worker, msecs_to_jiffies(30000));
1589 		/*
1590 		 * Use the total VF count since the actual VF count may not be
1591 		 * available at this point.
1592 		 */
1593 		bnxt_re_vf_res_config(rdev);
1594 	}
1595 
1596 	return 0;
1597 free_sctx:
1598 	bnxt_re_net_stats_ctx_free(rdev, rdev->qplib_ctx.stats.fw_id);
1599 free_ctx:
1600 	bnxt_qplib_free_ctx(&rdev->qplib_res, &rdev->qplib_ctx);
1601 disable_rcfw:
1602 	bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
1603 free_ring:
1604 	type = bnxt_qplib_get_ring_type(rdev->chip_ctx);
1605 	bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id, type);
1606 free_rcfw:
1607 	bnxt_qplib_free_rcfw_channel(&rdev->rcfw);
1608 fail:
1609 	bnxt_re_dev_uninit(rdev);
1610 
1611 	return rc;
1612 }
1613 
1614 static int bnxt_re_add_device(struct auxiliary_device *adev, u8 wqe_mode)
1615 {
1616 	struct bnxt_aux_priv *aux_priv =
1617 		container_of(adev, struct bnxt_aux_priv, aux_dev);
1618 	struct bnxt_en_dev *en_dev;
1619 	struct bnxt_re_dev *rdev;
1620 	int rc;
1621 
1622 	/* en_dev should never be NULL as long as adev and aux_dev are valid. */
1623 	en_dev = aux_priv->edev;
1624 
1625 	rdev = bnxt_re_dev_add(aux_priv, en_dev);
1626 	if (!rdev || !rdev_to_dev(rdev)) {
1627 		rc = -ENOMEM;
1628 		goto exit;
1629 	}
1630 
1631 	rc = bnxt_re_dev_init(rdev, wqe_mode);
1632 	if (rc)
1633 		goto re_dev_dealloc;
1634 
1635 	rc = bnxt_re_ib_init(rdev);
1636 	if (rc) {
1637 		pr_err("Failed to register with IB: %s",
1638 			aux_priv->aux_dev.name);
1639 		goto re_dev_uninit;
1640 	}
1641 	auxiliary_set_drvdata(adev, rdev);
1642 
1643 	return 0;
1644 
1645 re_dev_uninit:
1646 	bnxt_re_dev_uninit(rdev);
1647 re_dev_dealloc:
1648 	ib_dealloc_device(&rdev->ibdev);
1649 exit:
1650 	return rc;
1651 }
1652 
1653 static void bnxt_re_setup_cc(struct bnxt_re_dev *rdev, bool enable)
1654 {
1655 	struct bnxt_qplib_cc_param cc_param = {};
1656 
1657 	/* Do not enable congestion control on VFs */
1658 	if (rdev->is_virtfn)
1659 		return;
1660 
1661 	/* Currently enabling only for GenP5 adapters */
1662 	if (!bnxt_qplib_is_chip_gen_p5(rdev->chip_ctx))
1663 		return;
1664 
1665 	if (enable) {
1666 		cc_param.enable  = 1;
1667 		cc_param.cc_mode = CMDQ_MODIFY_ROCE_CC_CC_MODE_PROBABILISTIC_CC_MODE;
1668 	}
1669 
1670 	cc_param.mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
1671 			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
1672 			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
1673 
1674 	if (bnxt_qplib_modify_cc(&rdev->qplib_res, &cc_param))
1675 		ibdev_err(&rdev->ibdev, "Failed to setup CC enable = %d\n", enable);
1676 }
1677 
1678 /*
1679  * "Notifier chain callback can be invoked for the same chain from
1680  * different CPUs at the same time".
1681  *
1682  * For cases when the netdev is already present, our call to the
1683  * register_netdevice_notifier() will actually get the rtnl_lock()
1684  * before sending NETDEV_REGISTER and (if up) NETDEV_UP
1685  * events.
1686  *
1687  * But for cases when the netdev is not already present, the notifier
1688  * chain is subjected to be invoked from different CPUs simultaneously.
1689  *
1690  * This is protected by the netdev_mutex.
1691  */
1692 static int bnxt_re_netdev_event(struct notifier_block *notifier,
1693 				unsigned long event, void *ptr)
1694 {
1695 	struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);
1696 	struct bnxt_re_dev *rdev;
1697 
1698 	real_dev = rdma_vlan_dev_real_dev(netdev);
1699 	if (!real_dev)
1700 		real_dev = netdev;
1701 
1702 	if (real_dev != netdev)
1703 		goto exit;
1704 
1705 	rdev = bnxt_re_from_netdev(real_dev);
1706 	if (!rdev)
1707 		return NOTIFY_DONE;
1708 
1709 
1710 	switch (event) {
1711 	case NETDEV_UP:
1712 	case NETDEV_DOWN:
1713 	case NETDEV_CHANGE:
1714 		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
1715 					netif_carrier_ok(real_dev) ?
1716 					IB_EVENT_PORT_ACTIVE :
1717 					IB_EVENT_PORT_ERR);
1718 		break;
1719 	default:
1720 		break;
1721 	}
1722 	ib_device_put(&rdev->ibdev);
1723 exit:
1724 	return NOTIFY_DONE;
1725 }
1726 
1727 #define BNXT_ADEV_NAME "bnxt_en"
1728 
1729 static void bnxt_re_remove(struct auxiliary_device *adev)
1730 {
1731 	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
1732 
1733 	if (!rdev)
1734 		return;
1735 
1736 	mutex_lock(&bnxt_re_mutex);
1737 	if (rdev->nb.notifier_call) {
1738 		unregister_netdevice_notifier(&rdev->nb);
1739 		rdev->nb.notifier_call = NULL;
1740 	} else {
1741 		/* If notifier is null, we should have already done a
1742 		 * clean up before coming here.
1743 		 */
1744 		goto skip_remove;
1745 	}
1746 	bnxt_re_setup_cc(rdev, false);
1747 	ib_unregister_device(&rdev->ibdev);
1748 	bnxt_re_dev_uninit(rdev);
1749 	ib_dealloc_device(&rdev->ibdev);
1750 skip_remove:
1751 	mutex_unlock(&bnxt_re_mutex);
1752 }
1753 
1754 static int bnxt_re_probe(struct auxiliary_device *adev,
1755 			 const struct auxiliary_device_id *id)
1756 {
1757 	struct bnxt_re_dev *rdev;
1758 	int rc;
1759 
1760 	mutex_lock(&bnxt_re_mutex);
1761 	rc = bnxt_re_add_device(adev, BNXT_QPLIB_WQE_MODE_STATIC);
1762 	if (rc) {
1763 		mutex_unlock(&bnxt_re_mutex);
1764 		return rc;
1765 	}
1766 
1767 	rdev = auxiliary_get_drvdata(adev);
1768 
1769 	rdev->nb.notifier_call = bnxt_re_netdev_event;
1770 	rc = register_netdevice_notifier(&rdev->nb);
1771 	if (rc) {
1772 		rdev->nb.notifier_call = NULL;
1773 		pr_err("%s: Cannot register to netdevice_notifier",
1774 		       ROCE_DRV_MODULE_NAME);
1775 		goto err;
1776 	}
1777 
1778 	bnxt_re_setup_cc(rdev, true);
1779 	mutex_unlock(&bnxt_re_mutex);
1780 	return 0;
1781 
1782 err:
1783 	mutex_unlock(&bnxt_re_mutex);
1784 	bnxt_re_remove(adev);
1785 
1786 	return rc;
1787 }
1788 
1789 static int bnxt_re_suspend(struct auxiliary_device *adev, pm_message_t state)
1790 {
1791 	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
1792 
1793 	if (!rdev)
1794 		return 0;
1795 
1796 	mutex_lock(&bnxt_re_mutex);
1797 	/* L2 driver may invoke this callback during device error/crash or device
1798 	 * reset. Current RoCE driver doesn't recover the device in case of
1799 	 * error. Handle the error by dispatching fatal events to all qps
1800 	 * ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
1801 	 * L2 driver want to modify the MSIx table.
1802 	 */
1803 
1804 	ibdev_info(&rdev->ibdev, "Handle device suspend call");
1805 	/* Check the current device state from bnxt_en_dev and move the
1806 	 * device to detached state if FW_FATAL_COND is set.
1807 	 * This prevents more commands to HW during clean-up,
1808 	 * in case the device is already in error.
1809 	 */
1810 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &rdev->en_dev->en_state))
1811 		set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
1812 
1813 	bnxt_re_dev_stop(rdev);
1814 	bnxt_re_stop_irq(rdev);
1815 	/* Move the device states to detached and  avoid sending any more
1816 	 * commands to HW
1817 	 */
1818 	set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
1819 	set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
1820 	wake_up_all(&rdev->rcfw.cmdq.waitq);
1821 	mutex_unlock(&bnxt_re_mutex);
1822 
1823 	return 0;
1824 }
1825 
1826 static int bnxt_re_resume(struct auxiliary_device *adev)
1827 {
1828 	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
1829 
1830 	if (!rdev)
1831 		return 0;
1832 
1833 	mutex_lock(&bnxt_re_mutex);
1834 	/* L2 driver may invoke this callback during device recovery, resume.
1835 	 * reset. Current RoCE driver doesn't recover the device in case of
1836 	 * error. Handle the error by dispatching fatal events to all qps
1837 	 * ie. by calling bnxt_re_dev_stop and release the MSIx vectors as
1838 	 * L2 driver want to modify the MSIx table.
1839 	 */
1840 
1841 	ibdev_info(&rdev->ibdev, "Handle device resume call");
1842 	mutex_unlock(&bnxt_re_mutex);
1843 
1844 	return 0;
1845 }
1846 
1847 static const struct auxiliary_device_id bnxt_re_id_table[] = {
1848 	{ .name = BNXT_ADEV_NAME ".rdma", },
1849 	{},
1850 };
1851 
1852 MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
1853 
1854 static struct auxiliary_driver bnxt_re_driver = {
1855 	.name = "rdma",
1856 	.probe = bnxt_re_probe,
1857 	.remove = bnxt_re_remove,
1858 	.shutdown = bnxt_re_shutdown,
1859 	.suspend = bnxt_re_suspend,
1860 	.resume = bnxt_re_resume,
1861 	.id_table = bnxt_re_id_table,
1862 };
1863 
1864 static int __init bnxt_re_mod_init(void)
1865 {
1866 	int rc;
1867 
1868 	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
1869 	rc = auxiliary_driver_register(&bnxt_re_driver);
1870 	if (rc) {
1871 		pr_err("%s: Failed to register auxiliary driver\n",
1872 			ROCE_DRV_MODULE_NAME);
1873 		return rc;
1874 	}
1875 	return 0;
1876 }
1877 
1878 static void __exit bnxt_re_mod_exit(void)
1879 {
1880 	auxiliary_driver_unregister(&bnxt_re_driver);
1881 }
1882 
1883 module_init(bnxt_re_mod_init);
1884 module_exit(bnxt_re_mod_exit);
1885