1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2019 Broadcom Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  */
10 
11 #include <linux/module.h>
12 
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/gro.h>
41 #include <net/ip.h>
42 #include <net/tcp.h>
43 #include <net/udp.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 #include <net/udp_tunnel.h>
47 #include <linux/workqueue.h>
48 #include <linux/prefetch.h>
49 #include <linux/cache.h>
50 #include <linux/log2.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool/helpers.h>
58 #include <linux/align.h>
59 #include <net/netdev_queues.h>
60 
61 #include "bnxt_hsi.h"
62 #include "bnxt.h"
63 #include "bnxt_hwrm.h"
64 #include "bnxt_ulp.h"
65 #include "bnxt_sriov.h"
66 #include "bnxt_ethtool.h"
67 #include "bnxt_dcb.h"
68 #include "bnxt_xdp.h"
69 #include "bnxt_ptp.h"
70 #include "bnxt_vfr.h"
71 #include "bnxt_tc.h"
72 #include "bnxt_devlink.h"
73 #include "bnxt_debugfs.h"
74 
75 #define BNXT_TX_TIMEOUT		(5 * HZ)
76 #define BNXT_DEF_MSG_ENABLE	(NETIF_MSG_DRV | NETIF_MSG_HW | \
77 				 NETIF_MSG_TX_ERR)
78 
79 MODULE_LICENSE("GPL");
80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
81 
82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
84 #define BNXT_RX_COPY_THRESH 256
85 
86 #define BNXT_TX_PUSH_THRESH 164
87 
88 /* indexed by enum board_idx */
89 static const struct {
90 	char *name;
91 } board_info[] = {
92 	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
93 	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
94 	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
95 	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
96 	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
97 	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
98 	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
99 	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
100 	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
101 	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
102 	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
103 	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
104 	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
105 	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
106 	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
107 	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
108 	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
109 	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
110 	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
111 	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
112 	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
113 	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
114 	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
115 	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
116 	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
117 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
118 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
119 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
120 	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
121 	[BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 	[BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
123 	[BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
124 	[BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
125 	[BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
126 	[BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
127 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
128 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
129 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
130 	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
131 	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
132 	[NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
133 	[NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
134 	[NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
135 	[NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
136 	[NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
137 };
138 
139 static const struct pci_device_id bnxt_pci_tbl[] = {
140 	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
141 	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
142 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
143 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
144 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
145 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
146 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
147 	{ PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
148 	{ PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
149 	{ PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
150 	{ PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
151 	{ PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
152 	{ PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
153 	{ PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
154 	{ PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
155 	{ PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
156 	{ PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
157 	{ PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
158 	{ PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
159 	{ PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
160 	{ PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
161 	{ PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
162 	{ PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
163 	{ PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
164 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
165 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
166 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
167 	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
168 	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
169 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
170 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
171 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
172 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
173 	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
174 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
175 	{ PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
176 	{ PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
177 	{ PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
178 	{ PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
179 	{ PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
180 	{ PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
181 	{ PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
182 	{ PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
183 	{ PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
184 	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
185 	{ PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
186 #ifdef CONFIG_BNXT_SRIOV
187 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
188 	{ PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
189 	{ PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
190 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
191 	{ PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
192 	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
193 	{ PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
194 	{ PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
195 	{ PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
196 	{ PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
197 	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
198 	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
199 	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
200 	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
201 	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
202 	{ PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
203 	{ PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
204 	{ PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
205 	{ PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
206 	{ PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
207 	{ PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
208 #endif
209 	{ 0 }
210 };
211 
212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
213 
214 static const u16 bnxt_vf_req_snif[] = {
215 	HWRM_FUNC_CFG,
216 	HWRM_FUNC_VF_CFG,
217 	HWRM_PORT_PHY_QCFG,
218 	HWRM_CFA_L2_FILTER_ALLOC,
219 };
220 
221 static const u16 bnxt_async_events_arr[] = {
222 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
223 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
224 	ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
225 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
226 	ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
227 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
228 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
229 	ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
230 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
231 	ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
232 	ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
233 	ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
234 	ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
235 	ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
236 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
237 	ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
238 };
239 
240 static struct workqueue_struct *bnxt_pf_wq;
241 
242 static bool bnxt_vf_pciid(enum board_idx idx)
243 {
244 	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
245 		idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
246 		idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
247 		idx == NETXTREME_E_P5_VF_HV);
248 }
249 
250 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
251 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
252 #define DB_CP_IRQ_DIS_FLAGS	(DB_KEY_CP | DB_IRQ_DIS)
253 
254 #define BNXT_CP_DB_IRQ_DIS(db)						\
255 		writel(DB_CP_IRQ_DIS_FLAGS, db)
256 
257 #define BNXT_DB_CQ(db, idx)						\
258 	writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
259 
260 #define BNXT_DB_NQ_P5(db, idx)						\
261 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx),	\
262 		    (db)->doorbell)
263 
264 #define BNXT_DB_CQ_ARM(db, idx)						\
265 	writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
266 
267 #define BNXT_DB_NQ_ARM_P5(db, idx)					\
268 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\
269 		    (db)->doorbell)
270 
271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
272 {
273 	if (bp->flags & BNXT_FLAG_CHIP_P5)
274 		BNXT_DB_NQ_P5(db, idx);
275 	else
276 		BNXT_DB_CQ(db, idx);
277 }
278 
279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
280 {
281 	if (bp->flags & BNXT_FLAG_CHIP_P5)
282 		BNXT_DB_NQ_ARM_P5(db, idx);
283 	else
284 		BNXT_DB_CQ_ARM(db, idx);
285 }
286 
287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
288 {
289 	if (bp->flags & BNXT_FLAG_CHIP_P5)
290 		bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
291 			    RING_CMP(idx), db->doorbell);
292 	else
293 		BNXT_DB_CQ(db, idx);
294 }
295 
296 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
297 {
298 	if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
299 		return;
300 
301 	if (BNXT_PF(bp))
302 		queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
303 	else
304 		schedule_delayed_work(&bp->fw_reset_task, delay);
305 }
306 
307 static void __bnxt_queue_sp_work(struct bnxt *bp)
308 {
309 	if (BNXT_PF(bp))
310 		queue_work(bnxt_pf_wq, &bp->sp_task);
311 	else
312 		schedule_work(&bp->sp_task);
313 }
314 
315 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
316 {
317 	set_bit(event, &bp->sp_event);
318 	__bnxt_queue_sp_work(bp);
319 }
320 
321 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
322 {
323 	if (!rxr->bnapi->in_reset) {
324 		rxr->bnapi->in_reset = true;
325 		if (bp->flags & BNXT_FLAG_CHIP_P5)
326 			set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
327 		else
328 			set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
329 		__bnxt_queue_sp_work(bp);
330 	}
331 	rxr->rx_next_cons = 0xffff;
332 }
333 
334 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
335 			  int idx)
336 {
337 	struct bnxt_napi *bnapi = txr->bnapi;
338 
339 	if (bnapi->tx_fault)
340 		return;
341 
342 	netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)",
343 		   txr->txq_index, bnapi->tx_pkts,
344 		   txr->tx_cons, txr->tx_prod, idx);
345 	WARN_ON_ONCE(1);
346 	bnapi->tx_fault = 1;
347 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
348 }
349 
350 const u16 bnxt_lhint_arr[] = {
351 	TX_BD_FLAGS_LHINT_512_AND_SMALLER,
352 	TX_BD_FLAGS_LHINT_512_TO_1023,
353 	TX_BD_FLAGS_LHINT_1024_TO_2047,
354 	TX_BD_FLAGS_LHINT_1024_TO_2047,
355 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
356 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
357 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
358 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
359 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
360 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
361 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
362 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
363 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
364 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
365 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
366 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
367 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
368 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
369 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
370 };
371 
372 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
373 {
374 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
375 
376 	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
377 		return 0;
378 
379 	return md_dst->u.port_info.port_id;
380 }
381 
382 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
383 			     u16 prod)
384 {
385 	bnxt_db_write(bp, &txr->tx_db, prod);
386 	txr->kick_pending = 0;
387 }
388 
389 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
390 {
391 	struct bnxt *bp = netdev_priv(dev);
392 	struct tx_bd *txbd;
393 	struct tx_bd_ext *txbd1;
394 	struct netdev_queue *txq;
395 	int i;
396 	dma_addr_t mapping;
397 	unsigned int length, pad = 0;
398 	u32 len, free_size, vlan_tag_flags, cfa_action, flags;
399 	u16 prod, last_frag;
400 	struct pci_dev *pdev = bp->pdev;
401 	struct bnxt_tx_ring_info *txr;
402 	struct bnxt_sw_tx_bd *tx_buf;
403 	__le32 lflags = 0;
404 
405 	i = skb_get_queue_mapping(skb);
406 	if (unlikely(i >= bp->tx_nr_rings)) {
407 		dev_kfree_skb_any(skb);
408 		dev_core_stats_tx_dropped_inc(dev);
409 		return NETDEV_TX_OK;
410 	}
411 
412 	txq = netdev_get_tx_queue(dev, i);
413 	txr = &bp->tx_ring[bp->tx_ring_map[i]];
414 	prod = txr->tx_prod;
415 
416 	free_size = bnxt_tx_avail(bp, txr);
417 	if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
418 		/* We must have raced with NAPI cleanup */
419 		if (net_ratelimit() && txr->kick_pending)
420 			netif_warn(bp, tx_err, dev,
421 				   "bnxt: ring busy w/ flush pending!\n");
422 		if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
423 					bp->tx_wake_thresh))
424 			return NETDEV_TX_BUSY;
425 	}
426 
427 	if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
428 		goto tx_free;
429 
430 	length = skb->len;
431 	len = skb_headlen(skb);
432 	last_frag = skb_shinfo(skb)->nr_frags;
433 
434 	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
435 
436 	txbd->tx_bd_opaque = prod;
437 
438 	tx_buf = &txr->tx_buf_ring[prod];
439 	tx_buf->skb = skb;
440 	tx_buf->nr_frags = last_frag;
441 
442 	vlan_tag_flags = 0;
443 	cfa_action = bnxt_xmit_get_cfa_action(skb);
444 	if (skb_vlan_tag_present(skb)) {
445 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
446 				 skb_vlan_tag_get(skb);
447 		/* Currently supports 8021Q, 8021AD vlan offloads
448 		 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
449 		 */
450 		if (skb->vlan_proto == htons(ETH_P_8021Q))
451 			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
452 	}
453 
454 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
455 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
456 
457 		if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
458 		    atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
459 			if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
460 					    &ptp->tx_hdr_off)) {
461 				if (vlan_tag_flags)
462 					ptp->tx_hdr_off += VLAN_HLEN;
463 				lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
464 				skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
465 			} else {
466 				atomic_inc(&bp->ptp_cfg->tx_avail);
467 			}
468 		}
469 	}
470 
471 	if (unlikely(skb->no_fcs))
472 		lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
473 
474 	if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
475 	    !lflags) {
476 		struct tx_push_buffer *tx_push_buf = txr->tx_push;
477 		struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
478 		struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
479 		void __iomem *db = txr->tx_db.doorbell;
480 		void *pdata = tx_push_buf->data;
481 		u64 *end;
482 		int j, push_len;
483 
484 		/* Set COAL_NOW to be ready quickly for the next push */
485 		tx_push->tx_bd_len_flags_type =
486 			cpu_to_le32((length << TX_BD_LEN_SHIFT) |
487 					TX_BD_TYPE_LONG_TX_BD |
488 					TX_BD_FLAGS_LHINT_512_AND_SMALLER |
489 					TX_BD_FLAGS_COAL_NOW |
490 					TX_BD_FLAGS_PACKET_END |
491 					(2 << TX_BD_FLAGS_BD_CNT_SHIFT));
492 
493 		if (skb->ip_summed == CHECKSUM_PARTIAL)
494 			tx_push1->tx_bd_hsize_lflags =
495 					cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
496 		else
497 			tx_push1->tx_bd_hsize_lflags = 0;
498 
499 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
500 		tx_push1->tx_bd_cfa_action =
501 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
502 
503 		end = pdata + length;
504 		end = PTR_ALIGN(end, 8) - 1;
505 		*end = 0;
506 
507 		skb_copy_from_linear_data(skb, pdata, len);
508 		pdata += len;
509 		for (j = 0; j < last_frag; j++) {
510 			skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
511 			void *fptr;
512 
513 			fptr = skb_frag_address_safe(frag);
514 			if (!fptr)
515 				goto normal_tx;
516 
517 			memcpy(pdata, fptr, skb_frag_size(frag));
518 			pdata += skb_frag_size(frag);
519 		}
520 
521 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
522 		txbd->tx_bd_haddr = txr->data_mapping;
523 		prod = NEXT_TX(prod);
524 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
525 		memcpy(txbd, tx_push1, sizeof(*txbd));
526 		prod = NEXT_TX(prod);
527 		tx_push->doorbell =
528 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
529 		WRITE_ONCE(txr->tx_prod, prod);
530 
531 		tx_buf->is_push = 1;
532 		netdev_tx_sent_queue(txq, skb->len);
533 		wmb();	/* Sync is_push and byte queue before pushing data */
534 
535 		push_len = (length + sizeof(*tx_push) + 7) / 8;
536 		if (push_len > 16) {
537 			__iowrite64_copy(db, tx_push_buf, 16);
538 			__iowrite32_copy(db + 4, tx_push_buf + 1,
539 					 (push_len - 16) << 1);
540 		} else {
541 			__iowrite64_copy(db, tx_push_buf, push_len);
542 		}
543 
544 		goto tx_done;
545 	}
546 
547 normal_tx:
548 	if (length < BNXT_MIN_PKT_SIZE) {
549 		pad = BNXT_MIN_PKT_SIZE - length;
550 		if (skb_pad(skb, pad))
551 			/* SKB already freed. */
552 			goto tx_kick_pending;
553 		length = BNXT_MIN_PKT_SIZE;
554 	}
555 
556 	mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
557 
558 	if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
559 		goto tx_free;
560 
561 	dma_unmap_addr_set(tx_buf, mapping, mapping);
562 	flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
563 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
564 
565 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
566 
567 	prod = NEXT_TX(prod);
568 	txbd1 = (struct tx_bd_ext *)
569 		&txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
570 
571 	txbd1->tx_bd_hsize_lflags = lflags;
572 	if (skb_is_gso(skb)) {
573 		u32 hdr_len;
574 
575 		if (skb->encapsulation)
576 			hdr_len = skb_inner_tcp_all_headers(skb);
577 		else
578 			hdr_len = skb_tcp_all_headers(skb);
579 
580 		txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
581 					TX_BD_FLAGS_T_IPID |
582 					(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
583 		length = skb_shinfo(skb)->gso_size;
584 		txbd1->tx_bd_mss = cpu_to_le32(length);
585 		length += hdr_len;
586 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
587 		txbd1->tx_bd_hsize_lflags |=
588 			cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
589 		txbd1->tx_bd_mss = 0;
590 	}
591 
592 	length >>= 9;
593 	if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
594 		dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
595 				     skb->len);
596 		i = 0;
597 		goto tx_dma_error;
598 	}
599 	flags |= bnxt_lhint_arr[length];
600 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
601 
602 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
603 	txbd1->tx_bd_cfa_action =
604 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
605 	for (i = 0; i < last_frag; i++) {
606 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
607 
608 		prod = NEXT_TX(prod);
609 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
610 
611 		len = skb_frag_size(frag);
612 		mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
613 					   DMA_TO_DEVICE);
614 
615 		if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
616 			goto tx_dma_error;
617 
618 		tx_buf = &txr->tx_buf_ring[prod];
619 		dma_unmap_addr_set(tx_buf, mapping, mapping);
620 
621 		txbd->tx_bd_haddr = cpu_to_le64(mapping);
622 
623 		flags = len << TX_BD_LEN_SHIFT;
624 		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
625 	}
626 
627 	flags &= ~TX_BD_LEN;
628 	txbd->tx_bd_len_flags_type =
629 		cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
630 			    TX_BD_FLAGS_PACKET_END);
631 
632 	netdev_tx_sent_queue(txq, skb->len);
633 
634 	skb_tx_timestamp(skb);
635 
636 	/* Sync BD data before updating doorbell */
637 	wmb();
638 
639 	prod = NEXT_TX(prod);
640 	WRITE_ONCE(txr->tx_prod, prod);
641 
642 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
643 		bnxt_txr_db_kick(bp, txr, prod);
644 	else
645 		txr->kick_pending = 1;
646 
647 tx_done:
648 
649 	if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
650 		if (netdev_xmit_more() && !tx_buf->is_push)
651 			bnxt_txr_db_kick(bp, txr, prod);
652 
653 		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
654 				   bp->tx_wake_thresh);
655 	}
656 	return NETDEV_TX_OK;
657 
658 tx_dma_error:
659 	last_frag = i;
660 
661 	/* start back at beginning and unmap skb */
662 	prod = txr->tx_prod;
663 	tx_buf = &txr->tx_buf_ring[prod];
664 	dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
665 			 skb_headlen(skb), DMA_TO_DEVICE);
666 	prod = NEXT_TX(prod);
667 
668 	/* unmap remaining mapped pages */
669 	for (i = 0; i < last_frag; i++) {
670 		prod = NEXT_TX(prod);
671 		tx_buf = &txr->tx_buf_ring[prod];
672 		dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
673 			       skb_frag_size(&skb_shinfo(skb)->frags[i]),
674 			       DMA_TO_DEVICE);
675 	}
676 
677 tx_free:
678 	dev_kfree_skb_any(skb);
679 tx_kick_pending:
680 	if (BNXT_TX_PTP_IS_SET(lflags))
681 		atomic_inc(&bp->ptp_cfg->tx_avail);
682 	if (txr->kick_pending)
683 		bnxt_txr_db_kick(bp, txr, txr->tx_prod);
684 	txr->tx_buf_ring[txr->tx_prod].skb = NULL;
685 	dev_core_stats_tx_dropped_inc(dev);
686 	return NETDEV_TX_OK;
687 }
688 
689 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
690 {
691 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
692 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
693 	u16 cons = txr->tx_cons;
694 	struct pci_dev *pdev = bp->pdev;
695 	int nr_pkts = bnapi->tx_pkts;
696 	int i;
697 	unsigned int tx_bytes = 0;
698 
699 	for (i = 0; i < nr_pkts; i++) {
700 		struct bnxt_sw_tx_bd *tx_buf;
701 		struct sk_buff *skb;
702 		int j, last;
703 
704 		tx_buf = &txr->tx_buf_ring[cons];
705 		cons = NEXT_TX(cons);
706 		skb = tx_buf->skb;
707 		tx_buf->skb = NULL;
708 
709 		if (unlikely(!skb)) {
710 			bnxt_sched_reset_txr(bp, txr, i);
711 			return;
712 		}
713 
714 		tx_bytes += skb->len;
715 
716 		if (tx_buf->is_push) {
717 			tx_buf->is_push = 0;
718 			goto next_tx_int;
719 		}
720 
721 		dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
722 				 skb_headlen(skb), DMA_TO_DEVICE);
723 		last = tx_buf->nr_frags;
724 
725 		for (j = 0; j < last; j++) {
726 			cons = NEXT_TX(cons);
727 			tx_buf = &txr->tx_buf_ring[cons];
728 			dma_unmap_page(
729 				&pdev->dev,
730 				dma_unmap_addr(tx_buf, mapping),
731 				skb_frag_size(&skb_shinfo(skb)->frags[j]),
732 				DMA_TO_DEVICE);
733 		}
734 		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
735 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
736 				/* PTP worker takes ownership of the skb */
737 				if (!bnxt_get_tx_ts_p5(bp, skb))
738 					skb = NULL;
739 				else
740 					atomic_inc(&bp->ptp_cfg->tx_avail);
741 			}
742 		}
743 
744 next_tx_int:
745 		cons = NEXT_TX(cons);
746 
747 		dev_consume_skb_any(skb);
748 	}
749 
750 	bnapi->tx_pkts = 0;
751 	WRITE_ONCE(txr->tx_cons, cons);
752 
753 	__netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
754 				   bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
755 				   READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
756 }
757 
758 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
759 					 struct bnxt_rx_ring_info *rxr,
760 					 unsigned int *offset,
761 					 gfp_t gfp)
762 {
763 	struct page *page;
764 
765 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
766 		page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
767 						BNXT_RX_PAGE_SIZE);
768 	} else {
769 		page = page_pool_dev_alloc_pages(rxr->page_pool);
770 		*offset = 0;
771 	}
772 	if (!page)
773 		return NULL;
774 
775 	*mapping = page_pool_get_dma_addr(page) + *offset;
776 	return page;
777 }
778 
779 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
780 				       gfp_t gfp)
781 {
782 	u8 *data;
783 	struct pci_dev *pdev = bp->pdev;
784 
785 	if (gfp == GFP_ATOMIC)
786 		data = napi_alloc_frag(bp->rx_buf_size);
787 	else
788 		data = netdev_alloc_frag(bp->rx_buf_size);
789 	if (!data)
790 		return NULL;
791 
792 	*mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
793 					bp->rx_buf_use_size, bp->rx_dir,
794 					DMA_ATTR_WEAK_ORDERING);
795 
796 	if (dma_mapping_error(&pdev->dev, *mapping)) {
797 		skb_free_frag(data);
798 		data = NULL;
799 	}
800 	return data;
801 }
802 
803 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
804 		       u16 prod, gfp_t gfp)
805 {
806 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
807 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
808 	dma_addr_t mapping;
809 
810 	if (BNXT_RX_PAGE_MODE(bp)) {
811 		unsigned int offset;
812 		struct page *page =
813 			__bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
814 
815 		if (!page)
816 			return -ENOMEM;
817 
818 		mapping += bp->rx_dma_offset;
819 		rx_buf->data = page;
820 		rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
821 	} else {
822 		u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
823 
824 		if (!data)
825 			return -ENOMEM;
826 
827 		rx_buf->data = data;
828 		rx_buf->data_ptr = data + bp->rx_offset;
829 	}
830 	rx_buf->mapping = mapping;
831 
832 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
833 	return 0;
834 }
835 
836 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
837 {
838 	u16 prod = rxr->rx_prod;
839 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
840 	struct rx_bd *cons_bd, *prod_bd;
841 
842 	prod_rx_buf = &rxr->rx_buf_ring[prod];
843 	cons_rx_buf = &rxr->rx_buf_ring[cons];
844 
845 	prod_rx_buf->data = data;
846 	prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
847 
848 	prod_rx_buf->mapping = cons_rx_buf->mapping;
849 
850 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
851 	cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
852 
853 	prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
854 }
855 
856 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
857 {
858 	u16 next, max = rxr->rx_agg_bmap_size;
859 
860 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
861 	if (next >= max)
862 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
863 	return next;
864 }
865 
866 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
867 				     struct bnxt_rx_ring_info *rxr,
868 				     u16 prod, gfp_t gfp)
869 {
870 	struct rx_bd *rxbd =
871 		&rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
872 	struct bnxt_sw_rx_agg_bd *rx_agg_buf;
873 	struct page *page;
874 	dma_addr_t mapping;
875 	u16 sw_prod = rxr->rx_sw_agg_prod;
876 	unsigned int offset = 0;
877 
878 	page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
879 
880 	if (!page)
881 		return -ENOMEM;
882 
883 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
884 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
885 
886 	__set_bit(sw_prod, rxr->rx_agg_bmap);
887 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
888 	rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
889 
890 	rx_agg_buf->page = page;
891 	rx_agg_buf->offset = offset;
892 	rx_agg_buf->mapping = mapping;
893 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
894 	rxbd->rx_bd_opaque = sw_prod;
895 	return 0;
896 }
897 
898 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
899 				       struct bnxt_cp_ring_info *cpr,
900 				       u16 cp_cons, u16 curr)
901 {
902 	struct rx_agg_cmp *agg;
903 
904 	cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
905 	agg = (struct rx_agg_cmp *)
906 		&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
907 	return agg;
908 }
909 
910 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
911 					      struct bnxt_rx_ring_info *rxr,
912 					      u16 agg_id, u16 curr)
913 {
914 	struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
915 
916 	return &tpa_info->agg_arr[curr];
917 }
918 
919 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
920 				   u16 start, u32 agg_bufs, bool tpa)
921 {
922 	struct bnxt_napi *bnapi = cpr->bnapi;
923 	struct bnxt *bp = bnapi->bp;
924 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
925 	u16 prod = rxr->rx_agg_prod;
926 	u16 sw_prod = rxr->rx_sw_agg_prod;
927 	bool p5_tpa = false;
928 	u32 i;
929 
930 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
931 		p5_tpa = true;
932 
933 	for (i = 0; i < agg_bufs; i++) {
934 		u16 cons;
935 		struct rx_agg_cmp *agg;
936 		struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
937 		struct rx_bd *prod_bd;
938 		struct page *page;
939 
940 		if (p5_tpa)
941 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
942 		else
943 			agg = bnxt_get_agg(bp, cpr, idx, start + i);
944 		cons = agg->rx_agg_cmp_opaque;
945 		__clear_bit(cons, rxr->rx_agg_bmap);
946 
947 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
948 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
949 
950 		__set_bit(sw_prod, rxr->rx_agg_bmap);
951 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
952 		cons_rx_buf = &rxr->rx_agg_ring[cons];
953 
954 		/* It is possible for sw_prod to be equal to cons, so
955 		 * set cons_rx_buf->page to NULL first.
956 		 */
957 		page = cons_rx_buf->page;
958 		cons_rx_buf->page = NULL;
959 		prod_rx_buf->page = page;
960 		prod_rx_buf->offset = cons_rx_buf->offset;
961 
962 		prod_rx_buf->mapping = cons_rx_buf->mapping;
963 
964 		prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
965 
966 		prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
967 		prod_bd->rx_bd_opaque = sw_prod;
968 
969 		prod = NEXT_RX_AGG(prod);
970 		sw_prod = NEXT_RX_AGG(sw_prod);
971 	}
972 	rxr->rx_agg_prod = prod;
973 	rxr->rx_sw_agg_prod = sw_prod;
974 }
975 
976 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
977 					      struct bnxt_rx_ring_info *rxr,
978 					      u16 cons, void *data, u8 *data_ptr,
979 					      dma_addr_t dma_addr,
980 					      unsigned int offset_and_len)
981 {
982 	unsigned int len = offset_and_len & 0xffff;
983 	struct page *page = data;
984 	u16 prod = rxr->rx_prod;
985 	struct sk_buff *skb;
986 	int err;
987 
988 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
989 	if (unlikely(err)) {
990 		bnxt_reuse_rx_data(rxr, cons, data);
991 		return NULL;
992 	}
993 	dma_addr -= bp->rx_dma_offset;
994 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
995 				bp->rx_dir);
996 	skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
997 	if (!skb) {
998 		page_pool_recycle_direct(rxr->page_pool, page);
999 		return NULL;
1000 	}
1001 	skb_mark_for_recycle(skb);
1002 	skb_reserve(skb, bp->rx_offset);
1003 	__skb_put(skb, len);
1004 
1005 	return skb;
1006 }
1007 
1008 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1009 					struct bnxt_rx_ring_info *rxr,
1010 					u16 cons, void *data, u8 *data_ptr,
1011 					dma_addr_t dma_addr,
1012 					unsigned int offset_and_len)
1013 {
1014 	unsigned int payload = offset_and_len >> 16;
1015 	unsigned int len = offset_and_len & 0xffff;
1016 	skb_frag_t *frag;
1017 	struct page *page = data;
1018 	u16 prod = rxr->rx_prod;
1019 	struct sk_buff *skb;
1020 	int off, err;
1021 
1022 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1023 	if (unlikely(err)) {
1024 		bnxt_reuse_rx_data(rxr, cons, data);
1025 		return NULL;
1026 	}
1027 	dma_addr -= bp->rx_dma_offset;
1028 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1029 				bp->rx_dir);
1030 
1031 	if (unlikely(!payload))
1032 		payload = eth_get_headlen(bp->dev, data_ptr, len);
1033 
1034 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1035 	if (!skb) {
1036 		page_pool_recycle_direct(rxr->page_pool, page);
1037 		return NULL;
1038 	}
1039 
1040 	skb_mark_for_recycle(skb);
1041 	off = (void *)data_ptr - page_address(page);
1042 	skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1043 	memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1044 	       payload + NET_IP_ALIGN);
1045 
1046 	frag = &skb_shinfo(skb)->frags[0];
1047 	skb_frag_size_sub(frag, payload);
1048 	skb_frag_off_add(frag, payload);
1049 	skb->data_len -= payload;
1050 	skb->tail += payload;
1051 
1052 	return skb;
1053 }
1054 
1055 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1056 				   struct bnxt_rx_ring_info *rxr, u16 cons,
1057 				   void *data, u8 *data_ptr,
1058 				   dma_addr_t dma_addr,
1059 				   unsigned int offset_and_len)
1060 {
1061 	u16 prod = rxr->rx_prod;
1062 	struct sk_buff *skb;
1063 	int err;
1064 
1065 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1066 	if (unlikely(err)) {
1067 		bnxt_reuse_rx_data(rxr, cons, data);
1068 		return NULL;
1069 	}
1070 
1071 	skb = napi_build_skb(data, bp->rx_buf_size);
1072 	dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1073 			       bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1074 	if (!skb) {
1075 		skb_free_frag(data);
1076 		return NULL;
1077 	}
1078 
1079 	skb_reserve(skb, bp->rx_offset);
1080 	skb_put(skb, offset_and_len & 0xffff);
1081 	return skb;
1082 }
1083 
1084 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1085 			       struct bnxt_cp_ring_info *cpr,
1086 			       struct skb_shared_info *shinfo,
1087 			       u16 idx, u32 agg_bufs, bool tpa,
1088 			       struct xdp_buff *xdp)
1089 {
1090 	struct bnxt_napi *bnapi = cpr->bnapi;
1091 	struct pci_dev *pdev = bp->pdev;
1092 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1093 	u16 prod = rxr->rx_agg_prod;
1094 	u32 i, total_frag_len = 0;
1095 	bool p5_tpa = false;
1096 
1097 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1098 		p5_tpa = true;
1099 
1100 	for (i = 0; i < agg_bufs; i++) {
1101 		skb_frag_t *frag = &shinfo->frags[i];
1102 		u16 cons, frag_len;
1103 		struct rx_agg_cmp *agg;
1104 		struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1105 		struct page *page;
1106 		dma_addr_t mapping;
1107 
1108 		if (p5_tpa)
1109 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1110 		else
1111 			agg = bnxt_get_agg(bp, cpr, idx, i);
1112 		cons = agg->rx_agg_cmp_opaque;
1113 		frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1114 			    RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1115 
1116 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1117 		skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1118 					cons_rx_buf->offset, frag_len);
1119 		shinfo->nr_frags = i + 1;
1120 		__clear_bit(cons, rxr->rx_agg_bmap);
1121 
1122 		/* It is possible for bnxt_alloc_rx_page() to allocate
1123 		 * a sw_prod index that equals the cons index, so we
1124 		 * need to clear the cons entry now.
1125 		 */
1126 		mapping = cons_rx_buf->mapping;
1127 		page = cons_rx_buf->page;
1128 		cons_rx_buf->page = NULL;
1129 
1130 		if (xdp && page_is_pfmemalloc(page))
1131 			xdp_buff_set_frag_pfmemalloc(xdp);
1132 
1133 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1134 			--shinfo->nr_frags;
1135 			cons_rx_buf->page = page;
1136 
1137 			/* Update prod since possibly some pages have been
1138 			 * allocated already.
1139 			 */
1140 			rxr->rx_agg_prod = prod;
1141 			bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1142 			return 0;
1143 		}
1144 
1145 		dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1146 					bp->rx_dir);
1147 
1148 		total_frag_len += frag_len;
1149 		prod = NEXT_RX_AGG(prod);
1150 	}
1151 	rxr->rx_agg_prod = prod;
1152 	return total_frag_len;
1153 }
1154 
1155 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1156 					     struct bnxt_cp_ring_info *cpr,
1157 					     struct sk_buff *skb, u16 idx,
1158 					     u32 agg_bufs, bool tpa)
1159 {
1160 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1161 	u32 total_frag_len = 0;
1162 
1163 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1164 					     agg_bufs, tpa, NULL);
1165 	if (!total_frag_len) {
1166 		skb_mark_for_recycle(skb);
1167 		dev_kfree_skb(skb);
1168 		return NULL;
1169 	}
1170 
1171 	skb->data_len += total_frag_len;
1172 	skb->len += total_frag_len;
1173 	skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1174 	return skb;
1175 }
1176 
1177 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1178 				 struct bnxt_cp_ring_info *cpr,
1179 				 struct xdp_buff *xdp, u16 idx,
1180 				 u32 agg_bufs, bool tpa)
1181 {
1182 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1183 	u32 total_frag_len = 0;
1184 
1185 	if (!xdp_buff_has_frags(xdp))
1186 		shinfo->nr_frags = 0;
1187 
1188 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1189 					     idx, agg_bufs, tpa, xdp);
1190 	if (total_frag_len) {
1191 		xdp_buff_set_frags_flag(xdp);
1192 		shinfo->nr_frags = agg_bufs;
1193 		shinfo->xdp_frags_size = total_frag_len;
1194 	}
1195 	return total_frag_len;
1196 }
1197 
1198 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1199 			       u8 agg_bufs, u32 *raw_cons)
1200 {
1201 	u16 last;
1202 	struct rx_agg_cmp *agg;
1203 
1204 	*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1205 	last = RING_CMP(*raw_cons);
1206 	agg = (struct rx_agg_cmp *)
1207 		&cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1208 	return RX_AGG_CMP_VALID(agg, *raw_cons);
1209 }
1210 
1211 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1212 					    unsigned int len,
1213 					    dma_addr_t mapping)
1214 {
1215 	struct bnxt *bp = bnapi->bp;
1216 	struct pci_dev *pdev = bp->pdev;
1217 	struct sk_buff *skb;
1218 
1219 	skb = napi_alloc_skb(&bnapi->napi, len);
1220 	if (!skb)
1221 		return NULL;
1222 
1223 	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1224 				bp->rx_dir);
1225 
1226 	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1227 	       len + NET_IP_ALIGN);
1228 
1229 	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1230 				   bp->rx_dir);
1231 
1232 	skb_put(skb, len);
1233 	return skb;
1234 }
1235 
1236 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1237 			   u32 *raw_cons, void *cmp)
1238 {
1239 	struct rx_cmp *rxcmp = cmp;
1240 	u32 tmp_raw_cons = *raw_cons;
1241 	u8 cmp_type, agg_bufs = 0;
1242 
1243 	cmp_type = RX_CMP_TYPE(rxcmp);
1244 
1245 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1246 		agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1247 			    RX_CMP_AGG_BUFS) >>
1248 			   RX_CMP_AGG_BUFS_SHIFT;
1249 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1250 		struct rx_tpa_end_cmp *tpa_end = cmp;
1251 
1252 		if (bp->flags & BNXT_FLAG_CHIP_P5)
1253 			return 0;
1254 
1255 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1256 	}
1257 
1258 	if (agg_bufs) {
1259 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1260 			return -EBUSY;
1261 	}
1262 	*raw_cons = tmp_raw_cons;
1263 	return 0;
1264 }
1265 
1266 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1267 {
1268 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1269 	u16 idx = agg_id & MAX_TPA_P5_MASK;
1270 
1271 	if (test_bit(idx, map->agg_idx_bmap))
1272 		idx = find_first_zero_bit(map->agg_idx_bmap,
1273 					  BNXT_AGG_IDX_BMAP_SIZE);
1274 	__set_bit(idx, map->agg_idx_bmap);
1275 	map->agg_id_tbl[agg_id] = idx;
1276 	return idx;
1277 }
1278 
1279 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1280 {
1281 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1282 
1283 	__clear_bit(idx, map->agg_idx_bmap);
1284 }
1285 
1286 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1287 {
1288 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1289 
1290 	return map->agg_id_tbl[agg_id];
1291 }
1292 
1293 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1294 			   struct rx_tpa_start_cmp *tpa_start,
1295 			   struct rx_tpa_start_cmp_ext *tpa_start1)
1296 {
1297 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1298 	struct bnxt_tpa_info *tpa_info;
1299 	u16 cons, prod, agg_id;
1300 	struct rx_bd *prod_bd;
1301 	dma_addr_t mapping;
1302 
1303 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1304 		agg_id = TPA_START_AGG_ID_P5(tpa_start);
1305 		agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1306 	} else {
1307 		agg_id = TPA_START_AGG_ID(tpa_start);
1308 	}
1309 	cons = tpa_start->rx_tpa_start_cmp_opaque;
1310 	prod = rxr->rx_prod;
1311 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1312 	prod_rx_buf = &rxr->rx_buf_ring[prod];
1313 	tpa_info = &rxr->rx_tpa[agg_id];
1314 
1315 	if (unlikely(cons != rxr->rx_next_cons ||
1316 		     TPA_START_ERROR(tpa_start))) {
1317 		netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1318 			    cons, rxr->rx_next_cons,
1319 			    TPA_START_ERROR_CODE(tpa_start1));
1320 		bnxt_sched_reset_rxr(bp, rxr);
1321 		return;
1322 	}
1323 	/* Store cfa_code in tpa_info to use in tpa_end
1324 	 * completion processing.
1325 	 */
1326 	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1327 	prod_rx_buf->data = tpa_info->data;
1328 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
1329 
1330 	mapping = tpa_info->mapping;
1331 	prod_rx_buf->mapping = mapping;
1332 
1333 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1334 
1335 	prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1336 
1337 	tpa_info->data = cons_rx_buf->data;
1338 	tpa_info->data_ptr = cons_rx_buf->data_ptr;
1339 	cons_rx_buf->data = NULL;
1340 	tpa_info->mapping = cons_rx_buf->mapping;
1341 
1342 	tpa_info->len =
1343 		le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1344 				RX_TPA_START_CMP_LEN_SHIFT;
1345 	if (likely(TPA_START_HASH_VALID(tpa_start))) {
1346 		u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1347 
1348 		tpa_info->hash_type = PKT_HASH_TYPE_L4;
1349 		tpa_info->gso_type = SKB_GSO_TCPV4;
1350 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1351 		if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1352 			tpa_info->gso_type = SKB_GSO_TCPV6;
1353 		tpa_info->rss_hash =
1354 			le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1355 	} else {
1356 		tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1357 		tpa_info->gso_type = 0;
1358 		netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1359 	}
1360 	tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1361 	tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1362 	tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1363 	tpa_info->agg_count = 0;
1364 
1365 	rxr->rx_prod = NEXT_RX(prod);
1366 	cons = NEXT_RX(cons);
1367 	rxr->rx_next_cons = NEXT_RX(cons);
1368 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1369 
1370 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1371 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1372 	cons_rx_buf->data = NULL;
1373 }
1374 
1375 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1376 {
1377 	if (agg_bufs)
1378 		bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1379 }
1380 
1381 #ifdef CONFIG_INET
1382 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1383 {
1384 	struct udphdr *uh = NULL;
1385 
1386 	if (ip_proto == htons(ETH_P_IP)) {
1387 		struct iphdr *iph = (struct iphdr *)skb->data;
1388 
1389 		if (iph->protocol == IPPROTO_UDP)
1390 			uh = (struct udphdr *)(iph + 1);
1391 	} else {
1392 		struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1393 
1394 		if (iph->nexthdr == IPPROTO_UDP)
1395 			uh = (struct udphdr *)(iph + 1);
1396 	}
1397 	if (uh) {
1398 		if (uh->check)
1399 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1400 		else
1401 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1402 	}
1403 }
1404 #endif
1405 
1406 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1407 					   int payload_off, int tcp_ts,
1408 					   struct sk_buff *skb)
1409 {
1410 #ifdef CONFIG_INET
1411 	struct tcphdr *th;
1412 	int len, nw_off;
1413 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1414 	u32 hdr_info = tpa_info->hdr_info;
1415 	bool loopback = false;
1416 
1417 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1418 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1419 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1420 
1421 	/* If the packet is an internal loopback packet, the offsets will
1422 	 * have an extra 4 bytes.
1423 	 */
1424 	if (inner_mac_off == 4) {
1425 		loopback = true;
1426 	} else if (inner_mac_off > 4) {
1427 		__be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1428 					    ETH_HLEN - 2));
1429 
1430 		/* We only support inner iPv4/ipv6.  If we don't see the
1431 		 * correct protocol ID, it must be a loopback packet where
1432 		 * the offsets are off by 4.
1433 		 */
1434 		if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1435 			loopback = true;
1436 	}
1437 	if (loopback) {
1438 		/* internal loopback packet, subtract all offsets by 4 */
1439 		inner_ip_off -= 4;
1440 		inner_mac_off -= 4;
1441 		outer_ip_off -= 4;
1442 	}
1443 
1444 	nw_off = inner_ip_off - ETH_HLEN;
1445 	skb_set_network_header(skb, nw_off);
1446 	if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1447 		struct ipv6hdr *iph = ipv6_hdr(skb);
1448 
1449 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1450 		len = skb->len - skb_transport_offset(skb);
1451 		th = tcp_hdr(skb);
1452 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1453 	} else {
1454 		struct iphdr *iph = ip_hdr(skb);
1455 
1456 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1457 		len = skb->len - skb_transport_offset(skb);
1458 		th = tcp_hdr(skb);
1459 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1460 	}
1461 
1462 	if (inner_mac_off) { /* tunnel */
1463 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1464 					    ETH_HLEN - 2));
1465 
1466 		bnxt_gro_tunnel(skb, proto);
1467 	}
1468 #endif
1469 	return skb;
1470 }
1471 
1472 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1473 					   int payload_off, int tcp_ts,
1474 					   struct sk_buff *skb)
1475 {
1476 #ifdef CONFIG_INET
1477 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1478 	u32 hdr_info = tpa_info->hdr_info;
1479 	int iphdr_len, nw_off;
1480 
1481 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1482 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1483 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1484 
1485 	nw_off = inner_ip_off - ETH_HLEN;
1486 	skb_set_network_header(skb, nw_off);
1487 	iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1488 		     sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1489 	skb_set_transport_header(skb, nw_off + iphdr_len);
1490 
1491 	if (inner_mac_off) { /* tunnel */
1492 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1493 					    ETH_HLEN - 2));
1494 
1495 		bnxt_gro_tunnel(skb, proto);
1496 	}
1497 #endif
1498 	return skb;
1499 }
1500 
1501 #define BNXT_IPV4_HDR_SIZE	(sizeof(struct iphdr) + sizeof(struct tcphdr))
1502 #define BNXT_IPV6_HDR_SIZE	(sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1503 
1504 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1505 					   int payload_off, int tcp_ts,
1506 					   struct sk_buff *skb)
1507 {
1508 #ifdef CONFIG_INET
1509 	struct tcphdr *th;
1510 	int len, nw_off, tcp_opt_len = 0;
1511 
1512 	if (tcp_ts)
1513 		tcp_opt_len = 12;
1514 
1515 	if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1516 		struct iphdr *iph;
1517 
1518 		nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1519 			 ETH_HLEN;
1520 		skb_set_network_header(skb, nw_off);
1521 		iph = ip_hdr(skb);
1522 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1523 		len = skb->len - skb_transport_offset(skb);
1524 		th = tcp_hdr(skb);
1525 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1526 	} else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1527 		struct ipv6hdr *iph;
1528 
1529 		nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1530 			 ETH_HLEN;
1531 		skb_set_network_header(skb, nw_off);
1532 		iph = ipv6_hdr(skb);
1533 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1534 		len = skb->len - skb_transport_offset(skb);
1535 		th = tcp_hdr(skb);
1536 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1537 	} else {
1538 		dev_kfree_skb_any(skb);
1539 		return NULL;
1540 	}
1541 
1542 	if (nw_off) /* tunnel */
1543 		bnxt_gro_tunnel(skb, skb->protocol);
1544 #endif
1545 	return skb;
1546 }
1547 
1548 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1549 					   struct bnxt_tpa_info *tpa_info,
1550 					   struct rx_tpa_end_cmp *tpa_end,
1551 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1552 					   struct sk_buff *skb)
1553 {
1554 #ifdef CONFIG_INET
1555 	int payload_off;
1556 	u16 segs;
1557 
1558 	segs = TPA_END_TPA_SEGS(tpa_end);
1559 	if (segs == 1)
1560 		return skb;
1561 
1562 	NAPI_GRO_CB(skb)->count = segs;
1563 	skb_shinfo(skb)->gso_size =
1564 		le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1565 	skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1566 	if (bp->flags & BNXT_FLAG_CHIP_P5)
1567 		payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1568 	else
1569 		payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1570 	skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1571 	if (likely(skb))
1572 		tcp_gro_complete(skb);
1573 #endif
1574 	return skb;
1575 }
1576 
1577 /* Given the cfa_code of a received packet determine which
1578  * netdev (vf-rep or PF) the packet is destined to.
1579  */
1580 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1581 {
1582 	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1583 
1584 	/* if vf-rep dev is NULL, the must belongs to the PF */
1585 	return dev ? dev : bp->dev;
1586 }
1587 
1588 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1589 					   struct bnxt_cp_ring_info *cpr,
1590 					   u32 *raw_cons,
1591 					   struct rx_tpa_end_cmp *tpa_end,
1592 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1593 					   u8 *event)
1594 {
1595 	struct bnxt_napi *bnapi = cpr->bnapi;
1596 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1597 	u8 *data_ptr, agg_bufs;
1598 	unsigned int len;
1599 	struct bnxt_tpa_info *tpa_info;
1600 	dma_addr_t mapping;
1601 	struct sk_buff *skb;
1602 	u16 idx = 0, agg_id;
1603 	void *data;
1604 	bool gro;
1605 
1606 	if (unlikely(bnapi->in_reset)) {
1607 		int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1608 
1609 		if (rc < 0)
1610 			return ERR_PTR(-EBUSY);
1611 		return NULL;
1612 	}
1613 
1614 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1615 		agg_id = TPA_END_AGG_ID_P5(tpa_end);
1616 		agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1617 		agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1618 		tpa_info = &rxr->rx_tpa[agg_id];
1619 		if (unlikely(agg_bufs != tpa_info->agg_count)) {
1620 			netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1621 				    agg_bufs, tpa_info->agg_count);
1622 			agg_bufs = tpa_info->agg_count;
1623 		}
1624 		tpa_info->agg_count = 0;
1625 		*event |= BNXT_AGG_EVENT;
1626 		bnxt_free_agg_idx(rxr, agg_id);
1627 		idx = agg_id;
1628 		gro = !!(bp->flags & BNXT_FLAG_GRO);
1629 	} else {
1630 		agg_id = TPA_END_AGG_ID(tpa_end);
1631 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1632 		tpa_info = &rxr->rx_tpa[agg_id];
1633 		idx = RING_CMP(*raw_cons);
1634 		if (agg_bufs) {
1635 			if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1636 				return ERR_PTR(-EBUSY);
1637 
1638 			*event |= BNXT_AGG_EVENT;
1639 			idx = NEXT_CMP(idx);
1640 		}
1641 		gro = !!TPA_END_GRO(tpa_end);
1642 	}
1643 	data = tpa_info->data;
1644 	data_ptr = tpa_info->data_ptr;
1645 	prefetch(data_ptr);
1646 	len = tpa_info->len;
1647 	mapping = tpa_info->mapping;
1648 
1649 	if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1650 		bnxt_abort_tpa(cpr, idx, agg_bufs);
1651 		if (agg_bufs > MAX_SKB_FRAGS)
1652 			netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1653 				    agg_bufs, (int)MAX_SKB_FRAGS);
1654 		return NULL;
1655 	}
1656 
1657 	if (len <= bp->rx_copy_thresh) {
1658 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1659 		if (!skb) {
1660 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1661 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1662 			return NULL;
1663 		}
1664 	} else {
1665 		u8 *new_data;
1666 		dma_addr_t new_mapping;
1667 
1668 		new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1669 		if (!new_data) {
1670 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1671 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1672 			return NULL;
1673 		}
1674 
1675 		tpa_info->data = new_data;
1676 		tpa_info->data_ptr = new_data + bp->rx_offset;
1677 		tpa_info->mapping = new_mapping;
1678 
1679 		skb = napi_build_skb(data, bp->rx_buf_size);
1680 		dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1681 				       bp->rx_buf_use_size, bp->rx_dir,
1682 				       DMA_ATTR_WEAK_ORDERING);
1683 
1684 		if (!skb) {
1685 			skb_free_frag(data);
1686 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1687 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1688 			return NULL;
1689 		}
1690 		skb_reserve(skb, bp->rx_offset);
1691 		skb_put(skb, len);
1692 	}
1693 
1694 	if (agg_bufs) {
1695 		skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1696 		if (!skb) {
1697 			/* Page reuse already handled by bnxt_rx_pages(). */
1698 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1699 			return NULL;
1700 		}
1701 	}
1702 
1703 	skb->protocol =
1704 		eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1705 
1706 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1707 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1708 
1709 	if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1710 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1711 		__be16 vlan_proto = htons(tpa_info->metadata >>
1712 					  RX_CMP_FLAGS2_METADATA_TPID_SFT);
1713 		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1714 
1715 		if (eth_type_vlan(vlan_proto)) {
1716 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1717 		} else {
1718 			dev_kfree_skb(skb);
1719 			return NULL;
1720 		}
1721 	}
1722 
1723 	skb_checksum_none_assert(skb);
1724 	if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1725 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1726 		skb->csum_level =
1727 			(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1728 	}
1729 
1730 	if (gro)
1731 		skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1732 
1733 	return skb;
1734 }
1735 
1736 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1737 			 struct rx_agg_cmp *rx_agg)
1738 {
1739 	u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1740 	struct bnxt_tpa_info *tpa_info;
1741 
1742 	agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1743 	tpa_info = &rxr->rx_tpa[agg_id];
1744 	BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1745 	tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1746 }
1747 
1748 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1749 			     struct sk_buff *skb)
1750 {
1751 	skb_mark_for_recycle(skb);
1752 
1753 	if (skb->dev != bp->dev) {
1754 		/* this packet belongs to a vf-rep */
1755 		bnxt_vf_rep_rx(bp, skb);
1756 		return;
1757 	}
1758 	skb_record_rx_queue(skb, bnapi->index);
1759 	napi_gro_receive(&bnapi->napi, skb);
1760 }
1761 
1762 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1763 			     struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1764 {
1765 	u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1766 
1767 	if (BNXT_PTP_RX_TS_VALID(flags))
1768 		goto ts_valid;
1769 	if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1770 		return false;
1771 
1772 ts_valid:
1773 	*cmpl_ts = ts;
1774 	return true;
1775 }
1776 
1777 /* returns the following:
1778  * 1       - 1 packet successfully received
1779  * 0       - successful TPA_START, packet not completed yet
1780  * -EBUSY  - completion ring does not have all the agg buffers yet
1781  * -ENOMEM - packet aborted due to out of memory
1782  * -EIO    - packet aborted due to hw error indicated in BD
1783  */
1784 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1785 		       u32 *raw_cons, u8 *event)
1786 {
1787 	struct bnxt_napi *bnapi = cpr->bnapi;
1788 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1789 	struct net_device *dev = bp->dev;
1790 	struct rx_cmp *rxcmp;
1791 	struct rx_cmp_ext *rxcmp1;
1792 	u32 tmp_raw_cons = *raw_cons;
1793 	u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1794 	struct bnxt_sw_rx_bd *rx_buf;
1795 	unsigned int len;
1796 	u8 *data_ptr, agg_bufs, cmp_type;
1797 	bool xdp_active = false;
1798 	dma_addr_t dma_addr;
1799 	struct sk_buff *skb;
1800 	struct xdp_buff xdp;
1801 	u32 flags, misc;
1802 	u32 cmpl_ts;
1803 	void *data;
1804 	int rc = 0;
1805 
1806 	rxcmp = (struct rx_cmp *)
1807 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1808 
1809 	cmp_type = RX_CMP_TYPE(rxcmp);
1810 
1811 	if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1812 		bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1813 		goto next_rx_no_prod_no_len;
1814 	}
1815 
1816 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1817 	cp_cons = RING_CMP(tmp_raw_cons);
1818 	rxcmp1 = (struct rx_cmp_ext *)
1819 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1820 
1821 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1822 		return -EBUSY;
1823 
1824 	/* The valid test of the entry must be done first before
1825 	 * reading any further.
1826 	 */
1827 	dma_rmb();
1828 	prod = rxr->rx_prod;
1829 
1830 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1831 		bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1832 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
1833 
1834 		*event |= BNXT_RX_EVENT;
1835 		goto next_rx_no_prod_no_len;
1836 
1837 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1838 		skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1839 				   (struct rx_tpa_end_cmp *)rxcmp,
1840 				   (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1841 
1842 		if (IS_ERR(skb))
1843 			return -EBUSY;
1844 
1845 		rc = -ENOMEM;
1846 		if (likely(skb)) {
1847 			bnxt_deliver_skb(bp, bnapi, skb);
1848 			rc = 1;
1849 		}
1850 		*event |= BNXT_RX_EVENT;
1851 		goto next_rx_no_prod_no_len;
1852 	}
1853 
1854 	cons = rxcmp->rx_cmp_opaque;
1855 	if (unlikely(cons != rxr->rx_next_cons)) {
1856 		int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1857 
1858 		/* 0xffff is forced error, don't print it */
1859 		if (rxr->rx_next_cons != 0xffff)
1860 			netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1861 				    cons, rxr->rx_next_cons);
1862 		bnxt_sched_reset_rxr(bp, rxr);
1863 		if (rc1)
1864 			return rc1;
1865 		goto next_rx_no_prod_no_len;
1866 	}
1867 	rx_buf = &rxr->rx_buf_ring[cons];
1868 	data = rx_buf->data;
1869 	data_ptr = rx_buf->data_ptr;
1870 	prefetch(data_ptr);
1871 
1872 	misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1873 	agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1874 
1875 	if (agg_bufs) {
1876 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1877 			return -EBUSY;
1878 
1879 		cp_cons = NEXT_CMP(cp_cons);
1880 		*event |= BNXT_AGG_EVENT;
1881 	}
1882 	*event |= BNXT_RX_EVENT;
1883 
1884 	rx_buf->data = NULL;
1885 	if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1886 		u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1887 
1888 		bnxt_reuse_rx_data(rxr, cons, data);
1889 		if (agg_bufs)
1890 			bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1891 					       false);
1892 
1893 		rc = -EIO;
1894 		if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1895 			bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1896 			if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1897 			    !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1898 				netdev_warn_once(bp->dev, "RX buffer error %x\n",
1899 						 rx_err);
1900 				bnxt_sched_reset_rxr(bp, rxr);
1901 			}
1902 		}
1903 		goto next_rx_no_len;
1904 	}
1905 
1906 	flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1907 	len = flags >> RX_CMP_LEN_SHIFT;
1908 	dma_addr = rx_buf->mapping;
1909 
1910 	if (bnxt_xdp_attached(bp, rxr)) {
1911 		bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
1912 		if (agg_bufs) {
1913 			u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
1914 							     cp_cons, agg_bufs,
1915 							     false);
1916 			if (!frag_len)
1917 				goto oom_next_rx;
1918 		}
1919 		xdp_active = true;
1920 	}
1921 
1922 	if (xdp_active) {
1923 		if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
1924 			rc = 1;
1925 			goto next_rx;
1926 		}
1927 	}
1928 
1929 	if (len <= bp->rx_copy_thresh) {
1930 		skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1931 		bnxt_reuse_rx_data(rxr, cons, data);
1932 		if (!skb) {
1933 			if (agg_bufs) {
1934 				if (!xdp_active)
1935 					bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1936 							       agg_bufs, false);
1937 				else
1938 					bnxt_xdp_buff_frags_free(rxr, &xdp);
1939 			}
1940 			goto oom_next_rx;
1941 		}
1942 	} else {
1943 		u32 payload;
1944 
1945 		if (rx_buf->data_ptr == data_ptr)
1946 			payload = misc & RX_CMP_PAYLOAD_OFFSET;
1947 		else
1948 			payload = 0;
1949 		skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1950 				      payload | len);
1951 		if (!skb)
1952 			goto oom_next_rx;
1953 	}
1954 
1955 	if (agg_bufs) {
1956 		if (!xdp_active) {
1957 			skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
1958 			if (!skb)
1959 				goto oom_next_rx;
1960 		} else {
1961 			skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
1962 			if (!skb) {
1963 				/* we should be able to free the old skb here */
1964 				bnxt_xdp_buff_frags_free(rxr, &xdp);
1965 				goto oom_next_rx;
1966 			}
1967 		}
1968 	}
1969 
1970 	if (RX_CMP_HASH_VALID(rxcmp)) {
1971 		u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1972 		enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1973 
1974 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1975 		if (hash_type != 1 && hash_type != 3)
1976 			type = PKT_HASH_TYPE_L3;
1977 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1978 	}
1979 
1980 	cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1981 	skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1982 
1983 	if ((rxcmp1->rx_cmp_flags2 &
1984 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1985 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1986 		u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1987 		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1988 		__be16 vlan_proto = htons(meta_data >>
1989 					  RX_CMP_FLAGS2_METADATA_TPID_SFT);
1990 
1991 		if (eth_type_vlan(vlan_proto)) {
1992 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1993 		} else {
1994 			dev_kfree_skb(skb);
1995 			goto next_rx;
1996 		}
1997 	}
1998 
1999 	skb_checksum_none_assert(skb);
2000 	if (RX_CMP_L4_CS_OK(rxcmp1)) {
2001 		if (dev->features & NETIF_F_RXCSUM) {
2002 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2003 			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2004 		}
2005 	} else {
2006 		if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2007 			if (dev->features & NETIF_F_RXCSUM)
2008 				bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2009 		}
2010 	}
2011 
2012 	if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2013 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
2014 			u64 ns, ts;
2015 
2016 			if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2017 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2018 
2019 				spin_lock_bh(&ptp->ptp_lock);
2020 				ns = timecounter_cyc2time(&ptp->tc, ts);
2021 				spin_unlock_bh(&ptp->ptp_lock);
2022 				memset(skb_hwtstamps(skb), 0,
2023 				       sizeof(*skb_hwtstamps(skb)));
2024 				skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2025 			}
2026 		}
2027 	}
2028 	bnxt_deliver_skb(bp, bnapi, skb);
2029 	rc = 1;
2030 
2031 next_rx:
2032 	cpr->rx_packets += 1;
2033 	cpr->rx_bytes += len;
2034 
2035 next_rx_no_len:
2036 	rxr->rx_prod = NEXT_RX(prod);
2037 	rxr->rx_next_cons = NEXT_RX(cons);
2038 
2039 next_rx_no_prod_no_len:
2040 	*raw_cons = tmp_raw_cons;
2041 
2042 	return rc;
2043 
2044 oom_next_rx:
2045 	cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
2046 	rc = -ENOMEM;
2047 	goto next_rx;
2048 }
2049 
2050 /* In netpoll mode, if we are using a combined completion ring, we need to
2051  * discard the rx packets and recycle the buffers.
2052  */
2053 static int bnxt_force_rx_discard(struct bnxt *bp,
2054 				 struct bnxt_cp_ring_info *cpr,
2055 				 u32 *raw_cons, u8 *event)
2056 {
2057 	u32 tmp_raw_cons = *raw_cons;
2058 	struct rx_cmp_ext *rxcmp1;
2059 	struct rx_cmp *rxcmp;
2060 	u16 cp_cons;
2061 	u8 cmp_type;
2062 	int rc;
2063 
2064 	cp_cons = RING_CMP(tmp_raw_cons);
2065 	rxcmp = (struct rx_cmp *)
2066 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2067 
2068 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2069 	cp_cons = RING_CMP(tmp_raw_cons);
2070 	rxcmp1 = (struct rx_cmp_ext *)
2071 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2072 
2073 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2074 		return -EBUSY;
2075 
2076 	/* The valid test of the entry must be done first before
2077 	 * reading any further.
2078 	 */
2079 	dma_rmb();
2080 	cmp_type = RX_CMP_TYPE(rxcmp);
2081 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2082 		rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2083 			cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2084 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2085 		struct rx_tpa_end_cmp_ext *tpa_end1;
2086 
2087 		tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2088 		tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2089 			cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2090 	}
2091 	rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2092 	if (rc && rc != -EBUSY)
2093 		cpr->bnapi->cp_ring.sw_stats.rx.rx_netpoll_discards += 1;
2094 	return rc;
2095 }
2096 
2097 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2098 {
2099 	struct bnxt_fw_health *fw_health = bp->fw_health;
2100 	u32 reg = fw_health->regs[reg_idx];
2101 	u32 reg_type, reg_off, val = 0;
2102 
2103 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2104 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2105 	switch (reg_type) {
2106 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
2107 		pci_read_config_dword(bp->pdev, reg_off, &val);
2108 		break;
2109 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
2110 		reg_off = fw_health->mapped_regs[reg_idx];
2111 		fallthrough;
2112 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2113 		val = readl(bp->bar0 + reg_off);
2114 		break;
2115 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2116 		val = readl(bp->bar1 + reg_off);
2117 		break;
2118 	}
2119 	if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2120 		val &= fw_health->fw_reset_inprog_reg_mask;
2121 	return val;
2122 }
2123 
2124 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2125 {
2126 	int i;
2127 
2128 	for (i = 0; i < bp->rx_nr_rings; i++) {
2129 		u16 grp_idx = bp->rx_ring[i].bnapi->index;
2130 		struct bnxt_ring_grp_info *grp_info;
2131 
2132 		grp_info = &bp->grp_info[grp_idx];
2133 		if (grp_info->agg_fw_ring_id == ring_id)
2134 			return grp_idx;
2135 	}
2136 	return INVALID_HW_RING_ID;
2137 }
2138 
2139 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2140 {
2141 	u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2142 
2143 	switch (err_type) {
2144 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2145 		netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2146 			   BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2147 		break;
2148 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2149 		netdev_warn(bp->dev, "Pause Storm detected!\n");
2150 		break;
2151 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2152 		netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2153 		break;
2154 	default:
2155 		netdev_err(bp->dev, "FW reported unknown error type %u\n",
2156 			   err_type);
2157 		break;
2158 	}
2159 }
2160 
2161 #define BNXT_GET_EVENT_PORT(data)	\
2162 	((data) &			\
2163 	 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2164 
2165 #define BNXT_EVENT_RING_TYPE(data2)	\
2166 	((data2) &			\
2167 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2168 
2169 #define BNXT_EVENT_RING_TYPE_RX(data2)	\
2170 	(BNXT_EVENT_RING_TYPE(data2) ==	\
2171 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2172 
2173 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)	\
2174 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2175 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2176 
2177 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)	\
2178 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2179 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2180 
2181 #define BNXT_PHC_BITS	48
2182 
2183 static int bnxt_async_event_process(struct bnxt *bp,
2184 				    struct hwrm_async_event_cmpl *cmpl)
2185 {
2186 	u16 event_id = le16_to_cpu(cmpl->event_id);
2187 	u32 data1 = le32_to_cpu(cmpl->event_data1);
2188 	u32 data2 = le32_to_cpu(cmpl->event_data2);
2189 
2190 	netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2191 		   event_id, data1, data2);
2192 
2193 	/* TODO CHIMP_FW: Define event id's for link change, error etc */
2194 	switch (event_id) {
2195 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2196 		struct bnxt_link_info *link_info = &bp->link_info;
2197 
2198 		if (BNXT_VF(bp))
2199 			goto async_event_process_exit;
2200 
2201 		/* print unsupported speed warning in forced speed mode only */
2202 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2203 		    (data1 & 0x20000)) {
2204 			u16 fw_speed = link_info->force_link_speed;
2205 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2206 
2207 			if (speed != SPEED_UNKNOWN)
2208 				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2209 					    speed);
2210 		}
2211 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2212 	}
2213 		fallthrough;
2214 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2215 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2216 		set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2217 		fallthrough;
2218 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2219 		set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2220 		break;
2221 	case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2222 		set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2223 		break;
2224 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2225 		u16 port_id = BNXT_GET_EVENT_PORT(data1);
2226 
2227 		if (BNXT_VF(bp))
2228 			break;
2229 
2230 		if (bp->pf.port_id != port_id)
2231 			break;
2232 
2233 		set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2234 		break;
2235 	}
2236 	case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2237 		if (BNXT_PF(bp))
2238 			goto async_event_process_exit;
2239 		set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2240 		break;
2241 	case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2242 		char *type_str = "Solicited";
2243 
2244 		if (!bp->fw_health)
2245 			goto async_event_process_exit;
2246 
2247 		bp->fw_reset_timestamp = jiffies;
2248 		bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2249 		if (!bp->fw_reset_min_dsecs)
2250 			bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2251 		bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2252 		if (!bp->fw_reset_max_dsecs)
2253 			bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2254 		if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2255 			set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2256 		} else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2257 			type_str = "Fatal";
2258 			bp->fw_health->fatalities++;
2259 			set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2260 		} else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2261 			   EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2262 			type_str = "Non-fatal";
2263 			bp->fw_health->survivals++;
2264 			set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2265 		}
2266 		netif_warn(bp, hw, bp->dev,
2267 			   "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2268 			   type_str, data1, data2,
2269 			   bp->fw_reset_min_dsecs * 100,
2270 			   bp->fw_reset_max_dsecs * 100);
2271 		set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2272 		break;
2273 	}
2274 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2275 		struct bnxt_fw_health *fw_health = bp->fw_health;
2276 		char *status_desc = "healthy";
2277 		u32 status;
2278 
2279 		if (!fw_health)
2280 			goto async_event_process_exit;
2281 
2282 		if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2283 			fw_health->enabled = false;
2284 			netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2285 			break;
2286 		}
2287 		fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2288 		fw_health->tmr_multiplier =
2289 			DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2290 				     bp->current_interval * 10);
2291 		fw_health->tmr_counter = fw_health->tmr_multiplier;
2292 		if (!fw_health->enabled)
2293 			fw_health->last_fw_heartbeat =
2294 				bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2295 		fw_health->last_fw_reset_cnt =
2296 			bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2297 		status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2298 		if (status != BNXT_FW_STATUS_HEALTHY)
2299 			status_desc = "unhealthy";
2300 		netif_info(bp, drv, bp->dev,
2301 			   "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2302 			   fw_health->primary ? "primary" : "backup", status,
2303 			   status_desc, fw_health->last_fw_reset_cnt);
2304 		if (!fw_health->enabled) {
2305 			/* Make sure tmr_counter is set and visible to
2306 			 * bnxt_health_check() before setting enabled to true.
2307 			 */
2308 			smp_wmb();
2309 			fw_health->enabled = true;
2310 		}
2311 		goto async_event_process_exit;
2312 	}
2313 	case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2314 		netif_notice(bp, hw, bp->dev,
2315 			     "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2316 			     data1, data2);
2317 		goto async_event_process_exit;
2318 	case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2319 		struct bnxt_rx_ring_info *rxr;
2320 		u16 grp_idx;
2321 
2322 		if (bp->flags & BNXT_FLAG_CHIP_P5)
2323 			goto async_event_process_exit;
2324 
2325 		netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2326 			    BNXT_EVENT_RING_TYPE(data2), data1);
2327 		if (!BNXT_EVENT_RING_TYPE_RX(data2))
2328 			goto async_event_process_exit;
2329 
2330 		grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2331 		if (grp_idx == INVALID_HW_RING_ID) {
2332 			netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2333 				    data1);
2334 			goto async_event_process_exit;
2335 		}
2336 		rxr = bp->bnapi[grp_idx]->rx_ring;
2337 		bnxt_sched_reset_rxr(bp, rxr);
2338 		goto async_event_process_exit;
2339 	}
2340 	case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2341 		struct bnxt_fw_health *fw_health = bp->fw_health;
2342 
2343 		netif_notice(bp, hw, bp->dev,
2344 			     "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2345 			     data1, data2);
2346 		if (fw_health) {
2347 			fw_health->echo_req_data1 = data1;
2348 			fw_health->echo_req_data2 = data2;
2349 			set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2350 			break;
2351 		}
2352 		goto async_event_process_exit;
2353 	}
2354 	case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2355 		bnxt_ptp_pps_event(bp, data1, data2);
2356 		goto async_event_process_exit;
2357 	}
2358 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2359 		bnxt_event_error_report(bp, data1, data2);
2360 		goto async_event_process_exit;
2361 	}
2362 	case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2363 		switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2364 		case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2365 			if (BNXT_PTP_USE_RTC(bp)) {
2366 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2367 				u64 ns;
2368 
2369 				if (!ptp)
2370 					goto async_event_process_exit;
2371 
2372 				spin_lock_bh(&ptp->ptp_lock);
2373 				bnxt_ptp_update_current_time(bp);
2374 				ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2375 				       BNXT_PHC_BITS) | ptp->current_time);
2376 				bnxt_ptp_rtc_timecounter_init(ptp, ns);
2377 				spin_unlock_bh(&ptp->ptp_lock);
2378 			}
2379 			break;
2380 		}
2381 		goto async_event_process_exit;
2382 	}
2383 	case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2384 		u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2385 
2386 		hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2387 		goto async_event_process_exit;
2388 	}
2389 	default:
2390 		goto async_event_process_exit;
2391 	}
2392 	__bnxt_queue_sp_work(bp);
2393 async_event_process_exit:
2394 	return 0;
2395 }
2396 
2397 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2398 {
2399 	u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2400 	struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2401 	struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2402 				(struct hwrm_fwd_req_cmpl *)txcmp;
2403 
2404 	switch (cmpl_type) {
2405 	case CMPL_BASE_TYPE_HWRM_DONE:
2406 		seq_id = le16_to_cpu(h_cmpl->sequence_id);
2407 		hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2408 		break;
2409 
2410 	case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2411 		vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2412 
2413 		if ((vf_id < bp->pf.first_vf_id) ||
2414 		    (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2415 			netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2416 				   vf_id);
2417 			return -EINVAL;
2418 		}
2419 
2420 		set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2421 		bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2422 		break;
2423 
2424 	case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2425 		bnxt_async_event_process(bp,
2426 					 (struct hwrm_async_event_cmpl *)txcmp);
2427 		break;
2428 
2429 	default:
2430 		break;
2431 	}
2432 
2433 	return 0;
2434 }
2435 
2436 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2437 {
2438 	struct bnxt_napi *bnapi = dev_instance;
2439 	struct bnxt *bp = bnapi->bp;
2440 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2441 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2442 
2443 	cpr->event_ctr++;
2444 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2445 	napi_schedule(&bnapi->napi);
2446 	return IRQ_HANDLED;
2447 }
2448 
2449 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2450 {
2451 	u32 raw_cons = cpr->cp_raw_cons;
2452 	u16 cons = RING_CMP(raw_cons);
2453 	struct tx_cmp *txcmp;
2454 
2455 	txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2456 
2457 	return TX_CMP_VALID(txcmp, raw_cons);
2458 }
2459 
2460 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2461 {
2462 	struct bnxt_napi *bnapi = dev_instance;
2463 	struct bnxt *bp = bnapi->bp;
2464 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2466 	u32 int_status;
2467 
2468 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2469 
2470 	if (!bnxt_has_work(bp, cpr)) {
2471 		int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2472 		/* return if erroneous interrupt */
2473 		if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2474 			return IRQ_NONE;
2475 	}
2476 
2477 	/* disable ring IRQ */
2478 	BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2479 
2480 	/* Return here if interrupt is shared and is disabled. */
2481 	if (unlikely(atomic_read(&bp->intr_sem) != 0))
2482 		return IRQ_HANDLED;
2483 
2484 	napi_schedule(&bnapi->napi);
2485 	return IRQ_HANDLED;
2486 }
2487 
2488 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2489 			    int budget)
2490 {
2491 	struct bnxt_napi *bnapi = cpr->bnapi;
2492 	u32 raw_cons = cpr->cp_raw_cons;
2493 	u32 cons;
2494 	int tx_pkts = 0;
2495 	int rx_pkts = 0;
2496 	u8 event = 0;
2497 	struct tx_cmp *txcmp;
2498 
2499 	cpr->has_more_work = 0;
2500 	cpr->had_work_done = 1;
2501 	while (1) {
2502 		int rc;
2503 
2504 		cons = RING_CMP(raw_cons);
2505 		txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2506 
2507 		if (!TX_CMP_VALID(txcmp, raw_cons))
2508 			break;
2509 
2510 		/* The valid test of the entry must be done first before
2511 		 * reading any further.
2512 		 */
2513 		dma_rmb();
2514 		if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2515 			tx_pkts++;
2516 			/* return full budget so NAPI will complete. */
2517 			if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
2518 				rx_pkts = budget;
2519 				raw_cons = NEXT_RAW_CMP(raw_cons);
2520 				if (budget)
2521 					cpr->has_more_work = 1;
2522 				break;
2523 			}
2524 		} else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2525 			if (likely(budget))
2526 				rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2527 			else
2528 				rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2529 							   &event);
2530 			if (likely(rc >= 0))
2531 				rx_pkts += rc;
2532 			/* Increment rx_pkts when rc is -ENOMEM to count towards
2533 			 * the NAPI budget.  Otherwise, we may potentially loop
2534 			 * here forever if we consistently cannot allocate
2535 			 * buffers.
2536 			 */
2537 			else if (rc == -ENOMEM && budget)
2538 				rx_pkts++;
2539 			else if (rc == -EBUSY)	/* partial completion */
2540 				break;
2541 		} else if (unlikely((TX_CMP_TYPE(txcmp) ==
2542 				     CMPL_BASE_TYPE_HWRM_DONE) ||
2543 				    (TX_CMP_TYPE(txcmp) ==
2544 				     CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2545 				    (TX_CMP_TYPE(txcmp) ==
2546 				     CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2547 			bnxt_hwrm_handler(bp, txcmp);
2548 		}
2549 		raw_cons = NEXT_RAW_CMP(raw_cons);
2550 
2551 		if (rx_pkts && rx_pkts == budget) {
2552 			cpr->has_more_work = 1;
2553 			break;
2554 		}
2555 	}
2556 
2557 	if (event & BNXT_REDIRECT_EVENT)
2558 		xdp_do_flush();
2559 
2560 	if (event & BNXT_TX_EVENT) {
2561 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2562 		u16 prod = txr->tx_prod;
2563 
2564 		/* Sync BD data before updating doorbell */
2565 		wmb();
2566 
2567 		bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2568 	}
2569 
2570 	cpr->cp_raw_cons = raw_cons;
2571 	bnapi->tx_pkts += tx_pkts;
2572 	bnapi->events |= event;
2573 	return rx_pkts;
2574 }
2575 
2576 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2577 				  int budget)
2578 {
2579 	if (bnapi->tx_pkts && !bnapi->tx_fault)
2580 		bnapi->tx_int(bp, bnapi, budget);
2581 
2582 	if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2583 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2584 
2585 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2586 	}
2587 	if (bnapi->events & BNXT_AGG_EVENT) {
2588 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2589 
2590 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2591 	}
2592 	bnapi->events = 0;
2593 }
2594 
2595 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2596 			  int budget)
2597 {
2598 	struct bnxt_napi *bnapi = cpr->bnapi;
2599 	int rx_pkts;
2600 
2601 	rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2602 
2603 	/* ACK completion ring before freeing tx ring and producing new
2604 	 * buffers in rx/agg rings to prevent overflowing the completion
2605 	 * ring.
2606 	 */
2607 	bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2608 
2609 	__bnxt_poll_work_done(bp, bnapi, budget);
2610 	return rx_pkts;
2611 }
2612 
2613 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2614 {
2615 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2616 	struct bnxt *bp = bnapi->bp;
2617 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2618 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2619 	struct tx_cmp *txcmp;
2620 	struct rx_cmp_ext *rxcmp1;
2621 	u32 cp_cons, tmp_raw_cons;
2622 	u32 raw_cons = cpr->cp_raw_cons;
2623 	bool flush_xdp = false;
2624 	u32 rx_pkts = 0;
2625 	u8 event = 0;
2626 
2627 	while (1) {
2628 		int rc;
2629 
2630 		cp_cons = RING_CMP(raw_cons);
2631 		txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2632 
2633 		if (!TX_CMP_VALID(txcmp, raw_cons))
2634 			break;
2635 
2636 		/* The valid test of the entry must be done first before
2637 		 * reading any further.
2638 		 */
2639 		dma_rmb();
2640 		if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2641 			tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2642 			cp_cons = RING_CMP(tmp_raw_cons);
2643 			rxcmp1 = (struct rx_cmp_ext *)
2644 			  &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2645 
2646 			if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2647 				break;
2648 
2649 			/* force an error to recycle the buffer */
2650 			rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2651 				cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2652 
2653 			rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2654 			if (likely(rc == -EIO) && budget)
2655 				rx_pkts++;
2656 			else if (rc == -EBUSY)	/* partial completion */
2657 				break;
2658 			if (event & BNXT_REDIRECT_EVENT)
2659 				flush_xdp = true;
2660 		} else if (unlikely(TX_CMP_TYPE(txcmp) ==
2661 				    CMPL_BASE_TYPE_HWRM_DONE)) {
2662 			bnxt_hwrm_handler(bp, txcmp);
2663 		} else {
2664 			netdev_err(bp->dev,
2665 				   "Invalid completion received on special ring\n");
2666 		}
2667 		raw_cons = NEXT_RAW_CMP(raw_cons);
2668 
2669 		if (rx_pkts == budget)
2670 			break;
2671 	}
2672 
2673 	cpr->cp_raw_cons = raw_cons;
2674 	BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2675 	bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2676 
2677 	if (event & BNXT_AGG_EVENT)
2678 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2679 	if (flush_xdp)
2680 		xdp_do_flush();
2681 
2682 	if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2683 		napi_complete_done(napi, rx_pkts);
2684 		BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2685 	}
2686 	return rx_pkts;
2687 }
2688 
2689 static int bnxt_poll(struct napi_struct *napi, int budget)
2690 {
2691 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2692 	struct bnxt *bp = bnapi->bp;
2693 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2694 	int work_done = 0;
2695 
2696 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2697 		napi_complete(napi);
2698 		return 0;
2699 	}
2700 	while (1) {
2701 		work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2702 
2703 		if (work_done >= budget) {
2704 			if (!budget)
2705 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2706 			break;
2707 		}
2708 
2709 		if (!bnxt_has_work(bp, cpr)) {
2710 			if (napi_complete_done(napi, work_done))
2711 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2712 			break;
2713 		}
2714 	}
2715 	if (bp->flags & BNXT_FLAG_DIM) {
2716 		struct dim_sample dim_sample = {};
2717 
2718 		dim_update_sample(cpr->event_ctr,
2719 				  cpr->rx_packets,
2720 				  cpr->rx_bytes,
2721 				  &dim_sample);
2722 		net_dim(&cpr->dim, dim_sample);
2723 	}
2724 	return work_done;
2725 }
2726 
2727 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2728 {
2729 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2730 	int i, work_done = 0;
2731 
2732 	for (i = 0; i < 2; i++) {
2733 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2734 
2735 		if (cpr2) {
2736 			work_done += __bnxt_poll_work(bp, cpr2,
2737 						      budget - work_done);
2738 			cpr->has_more_work |= cpr2->has_more_work;
2739 		}
2740 	}
2741 	return work_done;
2742 }
2743 
2744 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2745 				 u64 dbr_type, int budget)
2746 {
2747 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2748 	int i;
2749 
2750 	for (i = 0; i < 2; i++) {
2751 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2752 		struct bnxt_db_info *db;
2753 
2754 		if (cpr2 && cpr2->had_work_done) {
2755 			db = &cpr2->cp_db;
2756 			bnxt_writeq(bp, db->db_key64 | dbr_type |
2757 				    RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2758 			cpr2->had_work_done = 0;
2759 		}
2760 	}
2761 	__bnxt_poll_work_done(bp, bnapi, budget);
2762 }
2763 
2764 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2765 {
2766 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2767 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2768 	struct bnxt_cp_ring_info *cpr_rx;
2769 	u32 raw_cons = cpr->cp_raw_cons;
2770 	struct bnxt *bp = bnapi->bp;
2771 	struct nqe_cn *nqcmp;
2772 	int work_done = 0;
2773 	u32 cons;
2774 
2775 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2776 		napi_complete(napi);
2777 		return 0;
2778 	}
2779 	if (cpr->has_more_work) {
2780 		cpr->has_more_work = 0;
2781 		work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2782 	}
2783 	while (1) {
2784 		cons = RING_CMP(raw_cons);
2785 		nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2786 
2787 		if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2788 			if (cpr->has_more_work)
2789 				break;
2790 
2791 			__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2792 					     budget);
2793 			cpr->cp_raw_cons = raw_cons;
2794 			if (napi_complete_done(napi, work_done))
2795 				BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2796 						  cpr->cp_raw_cons);
2797 			goto poll_done;
2798 		}
2799 
2800 		/* The valid test of the entry must be done first before
2801 		 * reading any further.
2802 		 */
2803 		dma_rmb();
2804 
2805 		if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2806 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2807 			struct bnxt_cp_ring_info *cpr2;
2808 
2809 			/* No more budget for RX work */
2810 			if (budget && work_done >= budget && idx == BNXT_RX_HDL)
2811 				break;
2812 
2813 			cpr2 = cpr->cp_ring_arr[idx];
2814 			work_done += __bnxt_poll_work(bp, cpr2,
2815 						      budget - work_done);
2816 			cpr->has_more_work |= cpr2->has_more_work;
2817 		} else {
2818 			bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2819 		}
2820 		raw_cons = NEXT_RAW_CMP(raw_cons);
2821 	}
2822 	__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
2823 	if (raw_cons != cpr->cp_raw_cons) {
2824 		cpr->cp_raw_cons = raw_cons;
2825 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2826 	}
2827 poll_done:
2828 	cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
2829 	if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
2830 		struct dim_sample dim_sample = {};
2831 
2832 		dim_update_sample(cpr->event_ctr,
2833 				  cpr_rx->rx_packets,
2834 				  cpr_rx->rx_bytes,
2835 				  &dim_sample);
2836 		net_dim(&cpr->dim, dim_sample);
2837 	}
2838 	return work_done;
2839 }
2840 
2841 static void bnxt_free_tx_skbs(struct bnxt *bp)
2842 {
2843 	int i, max_idx;
2844 	struct pci_dev *pdev = bp->pdev;
2845 
2846 	if (!bp->tx_ring)
2847 		return;
2848 
2849 	max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2850 	for (i = 0; i < bp->tx_nr_rings; i++) {
2851 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2852 		int j;
2853 
2854 		if (!txr->tx_buf_ring)
2855 			continue;
2856 
2857 		for (j = 0; j < max_idx;) {
2858 			struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2859 			struct sk_buff *skb;
2860 			int k, last;
2861 
2862 			if (i < bp->tx_nr_rings_xdp &&
2863 			    tx_buf->action == XDP_REDIRECT) {
2864 				dma_unmap_single(&pdev->dev,
2865 					dma_unmap_addr(tx_buf, mapping),
2866 					dma_unmap_len(tx_buf, len),
2867 					DMA_TO_DEVICE);
2868 				xdp_return_frame(tx_buf->xdpf);
2869 				tx_buf->action = 0;
2870 				tx_buf->xdpf = NULL;
2871 				j++;
2872 				continue;
2873 			}
2874 
2875 			skb = tx_buf->skb;
2876 			if (!skb) {
2877 				j++;
2878 				continue;
2879 			}
2880 
2881 			tx_buf->skb = NULL;
2882 
2883 			if (tx_buf->is_push) {
2884 				dev_kfree_skb(skb);
2885 				j += 2;
2886 				continue;
2887 			}
2888 
2889 			dma_unmap_single(&pdev->dev,
2890 					 dma_unmap_addr(tx_buf, mapping),
2891 					 skb_headlen(skb),
2892 					 DMA_TO_DEVICE);
2893 
2894 			last = tx_buf->nr_frags;
2895 			j += 2;
2896 			for (k = 0; k < last; k++, j++) {
2897 				int ring_idx = j & bp->tx_ring_mask;
2898 				skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2899 
2900 				tx_buf = &txr->tx_buf_ring[ring_idx];
2901 				dma_unmap_page(
2902 					&pdev->dev,
2903 					dma_unmap_addr(tx_buf, mapping),
2904 					skb_frag_size(frag), DMA_TO_DEVICE);
2905 			}
2906 			dev_kfree_skb(skb);
2907 		}
2908 		netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2909 	}
2910 }
2911 
2912 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2913 {
2914 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2915 	struct pci_dev *pdev = bp->pdev;
2916 	struct bnxt_tpa_idx_map *map;
2917 	int i, max_idx, max_agg_idx;
2918 
2919 	max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2920 	max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2921 	if (!rxr->rx_tpa)
2922 		goto skip_rx_tpa_free;
2923 
2924 	for (i = 0; i < bp->max_tpa; i++) {
2925 		struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2926 		u8 *data = tpa_info->data;
2927 
2928 		if (!data)
2929 			continue;
2930 
2931 		dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2932 				       bp->rx_buf_use_size, bp->rx_dir,
2933 				       DMA_ATTR_WEAK_ORDERING);
2934 
2935 		tpa_info->data = NULL;
2936 
2937 		skb_free_frag(data);
2938 	}
2939 
2940 skip_rx_tpa_free:
2941 	if (!rxr->rx_buf_ring)
2942 		goto skip_rx_buf_free;
2943 
2944 	for (i = 0; i < max_idx; i++) {
2945 		struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2946 		dma_addr_t mapping = rx_buf->mapping;
2947 		void *data = rx_buf->data;
2948 
2949 		if (!data)
2950 			continue;
2951 
2952 		rx_buf->data = NULL;
2953 		if (BNXT_RX_PAGE_MODE(bp)) {
2954 			page_pool_recycle_direct(rxr->page_pool, data);
2955 		} else {
2956 			dma_unmap_single_attrs(&pdev->dev, mapping,
2957 					       bp->rx_buf_use_size, bp->rx_dir,
2958 					       DMA_ATTR_WEAK_ORDERING);
2959 			skb_free_frag(data);
2960 		}
2961 	}
2962 
2963 skip_rx_buf_free:
2964 	if (!rxr->rx_agg_ring)
2965 		goto skip_rx_agg_free;
2966 
2967 	for (i = 0; i < max_agg_idx; i++) {
2968 		struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2969 		struct page *page = rx_agg_buf->page;
2970 
2971 		if (!page)
2972 			continue;
2973 
2974 		rx_agg_buf->page = NULL;
2975 		__clear_bit(i, rxr->rx_agg_bmap);
2976 
2977 		page_pool_recycle_direct(rxr->page_pool, page);
2978 	}
2979 
2980 skip_rx_agg_free:
2981 	map = rxr->rx_tpa_idx_map;
2982 	if (map)
2983 		memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2984 }
2985 
2986 static void bnxt_free_rx_skbs(struct bnxt *bp)
2987 {
2988 	int i;
2989 
2990 	if (!bp->rx_ring)
2991 		return;
2992 
2993 	for (i = 0; i < bp->rx_nr_rings; i++)
2994 		bnxt_free_one_rx_ring_skbs(bp, i);
2995 }
2996 
2997 static void bnxt_free_skbs(struct bnxt *bp)
2998 {
2999 	bnxt_free_tx_skbs(bp);
3000 	bnxt_free_rx_skbs(bp);
3001 }
3002 
3003 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
3004 {
3005 	u8 init_val = mem_init->init_val;
3006 	u16 offset = mem_init->offset;
3007 	u8 *p2 = p;
3008 	int i;
3009 
3010 	if (!init_val)
3011 		return;
3012 	if (offset == BNXT_MEM_INVALID_OFFSET) {
3013 		memset(p, init_val, len);
3014 		return;
3015 	}
3016 	for (i = 0; i < len; i += mem_init->size)
3017 		*(p2 + i + offset) = init_val;
3018 }
3019 
3020 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3021 {
3022 	struct pci_dev *pdev = bp->pdev;
3023 	int i;
3024 
3025 	if (!rmem->pg_arr)
3026 		goto skip_pages;
3027 
3028 	for (i = 0; i < rmem->nr_pages; i++) {
3029 		if (!rmem->pg_arr[i])
3030 			continue;
3031 
3032 		dma_free_coherent(&pdev->dev, rmem->page_size,
3033 				  rmem->pg_arr[i], rmem->dma_arr[i]);
3034 
3035 		rmem->pg_arr[i] = NULL;
3036 	}
3037 skip_pages:
3038 	if (rmem->pg_tbl) {
3039 		size_t pg_tbl_size = rmem->nr_pages * 8;
3040 
3041 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3042 			pg_tbl_size = rmem->page_size;
3043 		dma_free_coherent(&pdev->dev, pg_tbl_size,
3044 				  rmem->pg_tbl, rmem->pg_tbl_map);
3045 		rmem->pg_tbl = NULL;
3046 	}
3047 	if (rmem->vmem_size && *rmem->vmem) {
3048 		vfree(*rmem->vmem);
3049 		*rmem->vmem = NULL;
3050 	}
3051 }
3052 
3053 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3054 {
3055 	struct pci_dev *pdev = bp->pdev;
3056 	u64 valid_bit = 0;
3057 	int i;
3058 
3059 	if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3060 		valid_bit = PTU_PTE_VALID;
3061 	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3062 		size_t pg_tbl_size = rmem->nr_pages * 8;
3063 
3064 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3065 			pg_tbl_size = rmem->page_size;
3066 		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3067 						  &rmem->pg_tbl_map,
3068 						  GFP_KERNEL);
3069 		if (!rmem->pg_tbl)
3070 			return -ENOMEM;
3071 	}
3072 
3073 	for (i = 0; i < rmem->nr_pages; i++) {
3074 		u64 extra_bits = valid_bit;
3075 
3076 		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3077 						     rmem->page_size,
3078 						     &rmem->dma_arr[i],
3079 						     GFP_KERNEL);
3080 		if (!rmem->pg_arr[i])
3081 			return -ENOMEM;
3082 
3083 		if (rmem->mem_init)
3084 			bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
3085 					  rmem->page_size);
3086 		if (rmem->nr_pages > 1 || rmem->depth > 0) {
3087 			if (i == rmem->nr_pages - 2 &&
3088 			    (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3089 				extra_bits |= PTU_PTE_NEXT_TO_LAST;
3090 			else if (i == rmem->nr_pages - 1 &&
3091 				 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3092 				extra_bits |= PTU_PTE_LAST;
3093 			rmem->pg_tbl[i] =
3094 				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3095 		}
3096 	}
3097 
3098 	if (rmem->vmem_size) {
3099 		*rmem->vmem = vzalloc(rmem->vmem_size);
3100 		if (!(*rmem->vmem))
3101 			return -ENOMEM;
3102 	}
3103 	return 0;
3104 }
3105 
3106 static void bnxt_free_tpa_info(struct bnxt *bp)
3107 {
3108 	int i, j;
3109 
3110 	for (i = 0; i < bp->rx_nr_rings; i++) {
3111 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3112 
3113 		kfree(rxr->rx_tpa_idx_map);
3114 		rxr->rx_tpa_idx_map = NULL;
3115 		if (rxr->rx_tpa) {
3116 			for (j = 0; j < bp->max_tpa; j++) {
3117 				kfree(rxr->rx_tpa[j].agg_arr);
3118 				rxr->rx_tpa[j].agg_arr = NULL;
3119 			}
3120 		}
3121 		kfree(rxr->rx_tpa);
3122 		rxr->rx_tpa = NULL;
3123 	}
3124 }
3125 
3126 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3127 {
3128 	int i, j;
3129 
3130 	bp->max_tpa = MAX_TPA;
3131 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
3132 		if (!bp->max_tpa_v2)
3133 			return 0;
3134 		bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3135 	}
3136 
3137 	for (i = 0; i < bp->rx_nr_rings; i++) {
3138 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3139 		struct rx_agg_cmp *agg;
3140 
3141 		rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3142 				      GFP_KERNEL);
3143 		if (!rxr->rx_tpa)
3144 			return -ENOMEM;
3145 
3146 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3147 			continue;
3148 		for (j = 0; j < bp->max_tpa; j++) {
3149 			agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3150 			if (!agg)
3151 				return -ENOMEM;
3152 			rxr->rx_tpa[j].agg_arr = agg;
3153 		}
3154 		rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3155 					      GFP_KERNEL);
3156 		if (!rxr->rx_tpa_idx_map)
3157 			return -ENOMEM;
3158 	}
3159 	return 0;
3160 }
3161 
3162 static void bnxt_free_rx_rings(struct bnxt *bp)
3163 {
3164 	int i;
3165 
3166 	if (!bp->rx_ring)
3167 		return;
3168 
3169 	bnxt_free_tpa_info(bp);
3170 	for (i = 0; i < bp->rx_nr_rings; i++) {
3171 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3172 		struct bnxt_ring_struct *ring;
3173 
3174 		if (rxr->xdp_prog)
3175 			bpf_prog_put(rxr->xdp_prog);
3176 
3177 		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3178 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3179 
3180 		page_pool_destroy(rxr->page_pool);
3181 		rxr->page_pool = NULL;
3182 
3183 		kfree(rxr->rx_agg_bmap);
3184 		rxr->rx_agg_bmap = NULL;
3185 
3186 		ring = &rxr->rx_ring_struct;
3187 		bnxt_free_ring(bp, &ring->ring_mem);
3188 
3189 		ring = &rxr->rx_agg_ring_struct;
3190 		bnxt_free_ring(bp, &ring->ring_mem);
3191 	}
3192 }
3193 
3194 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3195 				   struct bnxt_rx_ring_info *rxr)
3196 {
3197 	struct page_pool_params pp = { 0 };
3198 
3199 	pp.pool_size = bp->rx_agg_ring_size;
3200 	if (BNXT_RX_PAGE_MODE(bp))
3201 		pp.pool_size += bp->rx_ring_size;
3202 	pp.nid = dev_to_node(&bp->pdev->dev);
3203 	pp.napi = &rxr->bnapi->napi;
3204 	pp.dev = &bp->pdev->dev;
3205 	pp.dma_dir = bp->rx_dir;
3206 	pp.max_len = PAGE_SIZE;
3207 	pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3208 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE)
3209 		pp.flags |= PP_FLAG_PAGE_FRAG;
3210 
3211 	rxr->page_pool = page_pool_create(&pp);
3212 	if (IS_ERR(rxr->page_pool)) {
3213 		int err = PTR_ERR(rxr->page_pool);
3214 
3215 		rxr->page_pool = NULL;
3216 		return err;
3217 	}
3218 	return 0;
3219 }
3220 
3221 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3222 {
3223 	int i, rc = 0, agg_rings = 0;
3224 
3225 	if (!bp->rx_ring)
3226 		return -ENOMEM;
3227 
3228 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
3229 		agg_rings = 1;
3230 
3231 	for (i = 0; i < bp->rx_nr_rings; i++) {
3232 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3233 		struct bnxt_ring_struct *ring;
3234 
3235 		ring = &rxr->rx_ring_struct;
3236 
3237 		rc = bnxt_alloc_rx_page_pool(bp, rxr);
3238 		if (rc)
3239 			return rc;
3240 
3241 		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3242 		if (rc < 0)
3243 			return rc;
3244 
3245 		rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3246 						MEM_TYPE_PAGE_POOL,
3247 						rxr->page_pool);
3248 		if (rc) {
3249 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3250 			return rc;
3251 		}
3252 
3253 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3254 		if (rc)
3255 			return rc;
3256 
3257 		ring->grp_idx = i;
3258 		if (agg_rings) {
3259 			u16 mem_size;
3260 
3261 			ring = &rxr->rx_agg_ring_struct;
3262 			rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3263 			if (rc)
3264 				return rc;
3265 
3266 			ring->grp_idx = i;
3267 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3268 			mem_size = rxr->rx_agg_bmap_size / 8;
3269 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3270 			if (!rxr->rx_agg_bmap)
3271 				return -ENOMEM;
3272 		}
3273 	}
3274 	if (bp->flags & BNXT_FLAG_TPA)
3275 		rc = bnxt_alloc_tpa_info(bp);
3276 	return rc;
3277 }
3278 
3279 static void bnxt_free_tx_rings(struct bnxt *bp)
3280 {
3281 	int i;
3282 	struct pci_dev *pdev = bp->pdev;
3283 
3284 	if (!bp->tx_ring)
3285 		return;
3286 
3287 	for (i = 0; i < bp->tx_nr_rings; i++) {
3288 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3289 		struct bnxt_ring_struct *ring;
3290 
3291 		if (txr->tx_push) {
3292 			dma_free_coherent(&pdev->dev, bp->tx_push_size,
3293 					  txr->tx_push, txr->tx_push_mapping);
3294 			txr->tx_push = NULL;
3295 		}
3296 
3297 		ring = &txr->tx_ring_struct;
3298 
3299 		bnxt_free_ring(bp, &ring->ring_mem);
3300 	}
3301 }
3302 
3303 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3304 {
3305 	int i, j, rc;
3306 	struct pci_dev *pdev = bp->pdev;
3307 
3308 	bp->tx_push_size = 0;
3309 	if (bp->tx_push_thresh) {
3310 		int push_size;
3311 
3312 		push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3313 					bp->tx_push_thresh);
3314 
3315 		if (push_size > 256) {
3316 			push_size = 0;
3317 			bp->tx_push_thresh = 0;
3318 		}
3319 
3320 		bp->tx_push_size = push_size;
3321 	}
3322 
3323 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3324 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3325 		struct bnxt_ring_struct *ring;
3326 		u8 qidx;
3327 
3328 		ring = &txr->tx_ring_struct;
3329 
3330 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3331 		if (rc)
3332 			return rc;
3333 
3334 		ring->grp_idx = txr->bnapi->index;
3335 		if (bp->tx_push_size) {
3336 			dma_addr_t mapping;
3337 
3338 			/* One pre-allocated DMA buffer to backup
3339 			 * TX push operation
3340 			 */
3341 			txr->tx_push = dma_alloc_coherent(&pdev->dev,
3342 						bp->tx_push_size,
3343 						&txr->tx_push_mapping,
3344 						GFP_KERNEL);
3345 
3346 			if (!txr->tx_push)
3347 				return -ENOMEM;
3348 
3349 			mapping = txr->tx_push_mapping +
3350 				sizeof(struct tx_push_bd);
3351 			txr->data_mapping = cpu_to_le64(mapping);
3352 		}
3353 		qidx = bp->tc_to_qidx[j];
3354 		ring->queue_id = bp->q_info[qidx].queue_id;
3355 		spin_lock_init(&txr->xdp_tx_lock);
3356 		if (i < bp->tx_nr_rings_xdp)
3357 			continue;
3358 		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3359 			j++;
3360 	}
3361 	return 0;
3362 }
3363 
3364 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3365 {
3366 	struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3367 
3368 	kfree(cpr->cp_desc_ring);
3369 	cpr->cp_desc_ring = NULL;
3370 	ring->ring_mem.pg_arr = NULL;
3371 	kfree(cpr->cp_desc_mapping);
3372 	cpr->cp_desc_mapping = NULL;
3373 	ring->ring_mem.dma_arr = NULL;
3374 }
3375 
3376 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3377 {
3378 	cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3379 	if (!cpr->cp_desc_ring)
3380 		return -ENOMEM;
3381 	cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3382 				       GFP_KERNEL);
3383 	if (!cpr->cp_desc_mapping)
3384 		return -ENOMEM;
3385 	return 0;
3386 }
3387 
3388 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3389 {
3390 	int i;
3391 
3392 	if (!bp->bnapi)
3393 		return;
3394 	for (i = 0; i < bp->cp_nr_rings; i++) {
3395 		struct bnxt_napi *bnapi = bp->bnapi[i];
3396 
3397 		if (!bnapi)
3398 			continue;
3399 		bnxt_free_cp_arrays(&bnapi->cp_ring);
3400 	}
3401 }
3402 
3403 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3404 {
3405 	int i, n = bp->cp_nr_pages;
3406 
3407 	for (i = 0; i < bp->cp_nr_rings; i++) {
3408 		struct bnxt_napi *bnapi = bp->bnapi[i];
3409 		int rc;
3410 
3411 		if (!bnapi)
3412 			continue;
3413 		rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3414 		if (rc)
3415 			return rc;
3416 	}
3417 	return 0;
3418 }
3419 
3420 static void bnxt_free_cp_rings(struct bnxt *bp)
3421 {
3422 	int i;
3423 
3424 	if (!bp->bnapi)
3425 		return;
3426 
3427 	for (i = 0; i < bp->cp_nr_rings; i++) {
3428 		struct bnxt_napi *bnapi = bp->bnapi[i];
3429 		struct bnxt_cp_ring_info *cpr;
3430 		struct bnxt_ring_struct *ring;
3431 		int j;
3432 
3433 		if (!bnapi)
3434 			continue;
3435 
3436 		cpr = &bnapi->cp_ring;
3437 		ring = &cpr->cp_ring_struct;
3438 
3439 		bnxt_free_ring(bp, &ring->ring_mem);
3440 
3441 		for (j = 0; j < 2; j++) {
3442 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3443 
3444 			if (cpr2) {
3445 				ring = &cpr2->cp_ring_struct;
3446 				bnxt_free_ring(bp, &ring->ring_mem);
3447 				bnxt_free_cp_arrays(cpr2);
3448 				kfree(cpr2);
3449 				cpr->cp_ring_arr[j] = NULL;
3450 			}
3451 		}
3452 	}
3453 }
3454 
3455 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3456 {
3457 	struct bnxt_ring_mem_info *rmem;
3458 	struct bnxt_ring_struct *ring;
3459 	struct bnxt_cp_ring_info *cpr;
3460 	int rc;
3461 
3462 	cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3463 	if (!cpr)
3464 		return NULL;
3465 
3466 	rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3467 	if (rc) {
3468 		bnxt_free_cp_arrays(cpr);
3469 		kfree(cpr);
3470 		return NULL;
3471 	}
3472 	ring = &cpr->cp_ring_struct;
3473 	rmem = &ring->ring_mem;
3474 	rmem->nr_pages = bp->cp_nr_pages;
3475 	rmem->page_size = HW_CMPD_RING_SIZE;
3476 	rmem->pg_arr = (void **)cpr->cp_desc_ring;
3477 	rmem->dma_arr = cpr->cp_desc_mapping;
3478 	rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3479 	rc = bnxt_alloc_ring(bp, rmem);
3480 	if (rc) {
3481 		bnxt_free_ring(bp, rmem);
3482 		bnxt_free_cp_arrays(cpr);
3483 		kfree(cpr);
3484 		cpr = NULL;
3485 	}
3486 	return cpr;
3487 }
3488 
3489 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3490 {
3491 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3492 	int i, rc, ulp_base_vec, ulp_msix;
3493 
3494 	ulp_msix = bnxt_get_ulp_msix_num(bp);
3495 	ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3496 	for (i = 0; i < bp->cp_nr_rings; i++) {
3497 		struct bnxt_napi *bnapi = bp->bnapi[i];
3498 		struct bnxt_cp_ring_info *cpr;
3499 		struct bnxt_ring_struct *ring;
3500 
3501 		if (!bnapi)
3502 			continue;
3503 
3504 		cpr = &bnapi->cp_ring;
3505 		cpr->bnapi = bnapi;
3506 		ring = &cpr->cp_ring_struct;
3507 
3508 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3509 		if (rc)
3510 			return rc;
3511 
3512 		if (ulp_msix && i >= ulp_base_vec)
3513 			ring->map_idx = i + ulp_msix;
3514 		else
3515 			ring->map_idx = i;
3516 
3517 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3518 			continue;
3519 
3520 		if (i < bp->rx_nr_rings) {
3521 			struct bnxt_cp_ring_info *cpr2 =
3522 				bnxt_alloc_cp_sub_ring(bp);
3523 
3524 			cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3525 			if (!cpr2)
3526 				return -ENOMEM;
3527 			cpr2->bnapi = bnapi;
3528 		}
3529 		if ((sh && i < bp->tx_nr_rings) ||
3530 		    (!sh && i >= bp->rx_nr_rings)) {
3531 			struct bnxt_cp_ring_info *cpr2 =
3532 				bnxt_alloc_cp_sub_ring(bp);
3533 
3534 			cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3535 			if (!cpr2)
3536 				return -ENOMEM;
3537 			cpr2->bnapi = bnapi;
3538 		}
3539 	}
3540 	return 0;
3541 }
3542 
3543 static void bnxt_init_ring_struct(struct bnxt *bp)
3544 {
3545 	int i;
3546 
3547 	for (i = 0; i < bp->cp_nr_rings; i++) {
3548 		struct bnxt_napi *bnapi = bp->bnapi[i];
3549 		struct bnxt_ring_mem_info *rmem;
3550 		struct bnxt_cp_ring_info *cpr;
3551 		struct bnxt_rx_ring_info *rxr;
3552 		struct bnxt_tx_ring_info *txr;
3553 		struct bnxt_ring_struct *ring;
3554 
3555 		if (!bnapi)
3556 			continue;
3557 
3558 		cpr = &bnapi->cp_ring;
3559 		ring = &cpr->cp_ring_struct;
3560 		rmem = &ring->ring_mem;
3561 		rmem->nr_pages = bp->cp_nr_pages;
3562 		rmem->page_size = HW_CMPD_RING_SIZE;
3563 		rmem->pg_arr = (void **)cpr->cp_desc_ring;
3564 		rmem->dma_arr = cpr->cp_desc_mapping;
3565 		rmem->vmem_size = 0;
3566 
3567 		rxr = bnapi->rx_ring;
3568 		if (!rxr)
3569 			goto skip_rx;
3570 
3571 		ring = &rxr->rx_ring_struct;
3572 		rmem = &ring->ring_mem;
3573 		rmem->nr_pages = bp->rx_nr_pages;
3574 		rmem->page_size = HW_RXBD_RING_SIZE;
3575 		rmem->pg_arr = (void **)rxr->rx_desc_ring;
3576 		rmem->dma_arr = rxr->rx_desc_mapping;
3577 		rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3578 		rmem->vmem = (void **)&rxr->rx_buf_ring;
3579 
3580 		ring = &rxr->rx_agg_ring_struct;
3581 		rmem = &ring->ring_mem;
3582 		rmem->nr_pages = bp->rx_agg_nr_pages;
3583 		rmem->page_size = HW_RXBD_RING_SIZE;
3584 		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3585 		rmem->dma_arr = rxr->rx_agg_desc_mapping;
3586 		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3587 		rmem->vmem = (void **)&rxr->rx_agg_ring;
3588 
3589 skip_rx:
3590 		txr = bnapi->tx_ring;
3591 		if (!txr)
3592 			continue;
3593 
3594 		ring = &txr->tx_ring_struct;
3595 		rmem = &ring->ring_mem;
3596 		rmem->nr_pages = bp->tx_nr_pages;
3597 		rmem->page_size = HW_RXBD_RING_SIZE;
3598 		rmem->pg_arr = (void **)txr->tx_desc_ring;
3599 		rmem->dma_arr = txr->tx_desc_mapping;
3600 		rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3601 		rmem->vmem = (void **)&txr->tx_buf_ring;
3602 	}
3603 }
3604 
3605 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3606 {
3607 	int i;
3608 	u32 prod;
3609 	struct rx_bd **rx_buf_ring;
3610 
3611 	rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3612 	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3613 		int j;
3614 		struct rx_bd *rxbd;
3615 
3616 		rxbd = rx_buf_ring[i];
3617 		if (!rxbd)
3618 			continue;
3619 
3620 		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3621 			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3622 			rxbd->rx_bd_opaque = prod;
3623 		}
3624 	}
3625 }
3626 
3627 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3628 {
3629 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3630 	struct net_device *dev = bp->dev;
3631 	u32 prod;
3632 	int i;
3633 
3634 	prod = rxr->rx_prod;
3635 	for (i = 0; i < bp->rx_ring_size; i++) {
3636 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3637 			netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3638 				    ring_nr, i, bp->rx_ring_size);
3639 			break;
3640 		}
3641 		prod = NEXT_RX(prod);
3642 	}
3643 	rxr->rx_prod = prod;
3644 
3645 	if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3646 		return 0;
3647 
3648 	prod = rxr->rx_agg_prod;
3649 	for (i = 0; i < bp->rx_agg_ring_size; i++) {
3650 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3651 			netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3652 				    ring_nr, i, bp->rx_ring_size);
3653 			break;
3654 		}
3655 		prod = NEXT_RX_AGG(prod);
3656 	}
3657 	rxr->rx_agg_prod = prod;
3658 
3659 	if (rxr->rx_tpa) {
3660 		dma_addr_t mapping;
3661 		u8 *data;
3662 
3663 		for (i = 0; i < bp->max_tpa; i++) {
3664 			data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
3665 			if (!data)
3666 				return -ENOMEM;
3667 
3668 			rxr->rx_tpa[i].data = data;
3669 			rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3670 			rxr->rx_tpa[i].mapping = mapping;
3671 		}
3672 	}
3673 	return 0;
3674 }
3675 
3676 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3677 {
3678 	struct bnxt_rx_ring_info *rxr;
3679 	struct bnxt_ring_struct *ring;
3680 	u32 type;
3681 
3682 	type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3683 		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3684 
3685 	if (NET_IP_ALIGN == 2)
3686 		type |= RX_BD_FLAGS_SOP;
3687 
3688 	rxr = &bp->rx_ring[ring_nr];
3689 	ring = &rxr->rx_ring_struct;
3690 	bnxt_init_rxbd_pages(ring, type);
3691 
3692 	if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3693 		bpf_prog_add(bp->xdp_prog, 1);
3694 		rxr->xdp_prog = bp->xdp_prog;
3695 	}
3696 	ring->fw_ring_id = INVALID_HW_RING_ID;
3697 
3698 	ring = &rxr->rx_agg_ring_struct;
3699 	ring->fw_ring_id = INVALID_HW_RING_ID;
3700 
3701 	if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3702 		type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3703 			RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3704 
3705 		bnxt_init_rxbd_pages(ring, type);
3706 	}
3707 
3708 	return bnxt_alloc_one_rx_ring(bp, ring_nr);
3709 }
3710 
3711 static void bnxt_init_cp_rings(struct bnxt *bp)
3712 {
3713 	int i, j;
3714 
3715 	for (i = 0; i < bp->cp_nr_rings; i++) {
3716 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3717 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3718 
3719 		ring->fw_ring_id = INVALID_HW_RING_ID;
3720 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3721 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3722 		for (j = 0; j < 2; j++) {
3723 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3724 
3725 			if (!cpr2)
3726 				continue;
3727 
3728 			ring = &cpr2->cp_ring_struct;
3729 			ring->fw_ring_id = INVALID_HW_RING_ID;
3730 			cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3731 			cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3732 		}
3733 	}
3734 }
3735 
3736 static int bnxt_init_rx_rings(struct bnxt *bp)
3737 {
3738 	int i, rc = 0;
3739 
3740 	if (BNXT_RX_PAGE_MODE(bp)) {
3741 		bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3742 		bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3743 	} else {
3744 		bp->rx_offset = BNXT_RX_OFFSET;
3745 		bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3746 	}
3747 
3748 	for (i = 0; i < bp->rx_nr_rings; i++) {
3749 		rc = bnxt_init_one_rx_ring(bp, i);
3750 		if (rc)
3751 			break;
3752 	}
3753 
3754 	return rc;
3755 }
3756 
3757 static int bnxt_init_tx_rings(struct bnxt *bp)
3758 {
3759 	u16 i;
3760 
3761 	bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3762 				   BNXT_MIN_TX_DESC_CNT);
3763 
3764 	for (i = 0; i < bp->tx_nr_rings; i++) {
3765 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3766 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3767 
3768 		ring->fw_ring_id = INVALID_HW_RING_ID;
3769 	}
3770 
3771 	return 0;
3772 }
3773 
3774 static void bnxt_free_ring_grps(struct bnxt *bp)
3775 {
3776 	kfree(bp->grp_info);
3777 	bp->grp_info = NULL;
3778 }
3779 
3780 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3781 {
3782 	int i;
3783 
3784 	if (irq_re_init) {
3785 		bp->grp_info = kcalloc(bp->cp_nr_rings,
3786 				       sizeof(struct bnxt_ring_grp_info),
3787 				       GFP_KERNEL);
3788 		if (!bp->grp_info)
3789 			return -ENOMEM;
3790 	}
3791 	for (i = 0; i < bp->cp_nr_rings; i++) {
3792 		if (irq_re_init)
3793 			bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3794 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3795 		bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3796 		bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3797 		bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3798 	}
3799 	return 0;
3800 }
3801 
3802 static void bnxt_free_vnics(struct bnxt *bp)
3803 {
3804 	kfree(bp->vnic_info);
3805 	bp->vnic_info = NULL;
3806 	bp->nr_vnics = 0;
3807 }
3808 
3809 static int bnxt_alloc_vnics(struct bnxt *bp)
3810 {
3811 	int num_vnics = 1;
3812 
3813 #ifdef CONFIG_RFS_ACCEL
3814 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3815 		num_vnics += bp->rx_nr_rings;
3816 #endif
3817 
3818 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3819 		num_vnics++;
3820 
3821 	bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3822 				GFP_KERNEL);
3823 	if (!bp->vnic_info)
3824 		return -ENOMEM;
3825 
3826 	bp->nr_vnics = num_vnics;
3827 	return 0;
3828 }
3829 
3830 static void bnxt_init_vnics(struct bnxt *bp)
3831 {
3832 	int i;
3833 
3834 	for (i = 0; i < bp->nr_vnics; i++) {
3835 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3836 		int j;
3837 
3838 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
3839 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3840 			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3841 
3842 		vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3843 
3844 		if (bp->vnic_info[i].rss_hash_key) {
3845 			if (i == 0)
3846 				get_random_bytes(vnic->rss_hash_key,
3847 					      HW_HASH_KEY_SIZE);
3848 			else
3849 				memcpy(vnic->rss_hash_key,
3850 				       bp->vnic_info[0].rss_hash_key,
3851 				       HW_HASH_KEY_SIZE);
3852 		}
3853 	}
3854 }
3855 
3856 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3857 {
3858 	int pages;
3859 
3860 	pages = ring_size / desc_per_pg;
3861 
3862 	if (!pages)
3863 		return 1;
3864 
3865 	pages++;
3866 
3867 	while (pages & (pages - 1))
3868 		pages++;
3869 
3870 	return pages;
3871 }
3872 
3873 void bnxt_set_tpa_flags(struct bnxt *bp)
3874 {
3875 	bp->flags &= ~BNXT_FLAG_TPA;
3876 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3877 		return;
3878 	if (bp->dev->features & NETIF_F_LRO)
3879 		bp->flags |= BNXT_FLAG_LRO;
3880 	else if (bp->dev->features & NETIF_F_GRO_HW)
3881 		bp->flags |= BNXT_FLAG_GRO;
3882 }
3883 
3884 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3885  * be set on entry.
3886  */
3887 void bnxt_set_ring_params(struct bnxt *bp)
3888 {
3889 	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3890 	u32 agg_factor = 0, agg_ring_size = 0;
3891 
3892 	/* 8 for CRC and VLAN */
3893 	rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3894 
3895 	rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
3896 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3897 
3898 	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3899 	ring_size = bp->rx_ring_size;
3900 	bp->rx_agg_ring_size = 0;
3901 	bp->rx_agg_nr_pages = 0;
3902 
3903 	if (bp->flags & BNXT_FLAG_TPA)
3904 		agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3905 
3906 	bp->flags &= ~BNXT_FLAG_JUMBO;
3907 	if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3908 		u32 jumbo_factor;
3909 
3910 		bp->flags |= BNXT_FLAG_JUMBO;
3911 		jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3912 		if (jumbo_factor > agg_factor)
3913 			agg_factor = jumbo_factor;
3914 	}
3915 	if (agg_factor) {
3916 		if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
3917 			ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
3918 			netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
3919 				    bp->rx_ring_size, ring_size);
3920 			bp->rx_ring_size = ring_size;
3921 		}
3922 		agg_ring_size = ring_size * agg_factor;
3923 
3924 		bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3925 							RX_DESC_CNT);
3926 		if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3927 			u32 tmp = agg_ring_size;
3928 
3929 			bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3930 			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3931 			netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3932 				    tmp, agg_ring_size);
3933 		}
3934 		bp->rx_agg_ring_size = agg_ring_size;
3935 		bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3936 
3937 		if (BNXT_RX_PAGE_MODE(bp)) {
3938 			rx_space = PAGE_SIZE;
3939 			rx_size = PAGE_SIZE -
3940 				  ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
3941 				  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3942 		} else {
3943 			rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3944 			rx_space = rx_size + NET_SKB_PAD +
3945 				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3946 		}
3947 	}
3948 
3949 	bp->rx_buf_use_size = rx_size;
3950 	bp->rx_buf_size = rx_space;
3951 
3952 	bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3953 	bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3954 
3955 	ring_size = bp->tx_ring_size;
3956 	bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3957 	bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3958 
3959 	max_rx_cmpl = bp->rx_ring_size;
3960 	/* MAX TPA needs to be added because TPA_START completions are
3961 	 * immediately recycled, so the TPA completions are not bound by
3962 	 * the RX ring size.
3963 	 */
3964 	if (bp->flags & BNXT_FLAG_TPA)
3965 		max_rx_cmpl += bp->max_tpa;
3966 	/* RX and TPA completions are 32-byte, all others are 16-byte */
3967 	ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3968 	bp->cp_ring_size = ring_size;
3969 
3970 	bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3971 	if (bp->cp_nr_pages > MAX_CP_PAGES) {
3972 		bp->cp_nr_pages = MAX_CP_PAGES;
3973 		bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3974 		netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3975 			    ring_size, bp->cp_ring_size);
3976 	}
3977 	bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3978 	bp->cp_ring_mask = bp->cp_bit - 1;
3979 }
3980 
3981 /* Changing allocation mode of RX rings.
3982  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3983  */
3984 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3985 {
3986 	struct net_device *dev = bp->dev;
3987 
3988 	if (page_mode) {
3989 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3990 		bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
3991 
3992 		if (bp->xdp_prog->aux->xdp_has_frags)
3993 			dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
3994 		else
3995 			dev->max_mtu =
3996 				min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3997 		if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
3998 			bp->flags |= BNXT_FLAG_JUMBO;
3999 			bp->rx_skb_func = bnxt_rx_multi_page_skb;
4000 		} else {
4001 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4002 			bp->rx_skb_func = bnxt_rx_page_skb;
4003 		}
4004 		bp->rx_dir = DMA_BIDIRECTIONAL;
4005 		/* Disable LRO or GRO_HW */
4006 		netdev_update_features(dev);
4007 	} else {
4008 		dev->max_mtu = bp->max_mtu;
4009 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4010 		bp->rx_dir = DMA_FROM_DEVICE;
4011 		bp->rx_skb_func = bnxt_rx_skb;
4012 	}
4013 	return 0;
4014 }
4015 
4016 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4017 {
4018 	int i;
4019 	struct bnxt_vnic_info *vnic;
4020 	struct pci_dev *pdev = bp->pdev;
4021 
4022 	if (!bp->vnic_info)
4023 		return;
4024 
4025 	for (i = 0; i < bp->nr_vnics; i++) {
4026 		vnic = &bp->vnic_info[i];
4027 
4028 		kfree(vnic->fw_grp_ids);
4029 		vnic->fw_grp_ids = NULL;
4030 
4031 		kfree(vnic->uc_list);
4032 		vnic->uc_list = NULL;
4033 
4034 		if (vnic->mc_list) {
4035 			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4036 					  vnic->mc_list, vnic->mc_list_mapping);
4037 			vnic->mc_list = NULL;
4038 		}
4039 
4040 		if (vnic->rss_table) {
4041 			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4042 					  vnic->rss_table,
4043 					  vnic->rss_table_dma_addr);
4044 			vnic->rss_table = NULL;
4045 		}
4046 
4047 		vnic->rss_hash_key = NULL;
4048 		vnic->flags = 0;
4049 	}
4050 }
4051 
4052 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4053 {
4054 	int i, rc = 0, size;
4055 	struct bnxt_vnic_info *vnic;
4056 	struct pci_dev *pdev = bp->pdev;
4057 	int max_rings;
4058 
4059 	for (i = 0; i < bp->nr_vnics; i++) {
4060 		vnic = &bp->vnic_info[i];
4061 
4062 		if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4063 			int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4064 
4065 			if (mem_size > 0) {
4066 				vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4067 				if (!vnic->uc_list) {
4068 					rc = -ENOMEM;
4069 					goto out;
4070 				}
4071 			}
4072 		}
4073 
4074 		if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4075 			vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4076 			vnic->mc_list =
4077 				dma_alloc_coherent(&pdev->dev,
4078 						   vnic->mc_list_size,
4079 						   &vnic->mc_list_mapping,
4080 						   GFP_KERNEL);
4081 			if (!vnic->mc_list) {
4082 				rc = -ENOMEM;
4083 				goto out;
4084 			}
4085 		}
4086 
4087 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4088 			goto vnic_skip_grps;
4089 
4090 		if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4091 			max_rings = bp->rx_nr_rings;
4092 		else
4093 			max_rings = 1;
4094 
4095 		vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4096 		if (!vnic->fw_grp_ids) {
4097 			rc = -ENOMEM;
4098 			goto out;
4099 		}
4100 vnic_skip_grps:
4101 		if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
4102 		    !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4103 			continue;
4104 
4105 		/* Allocate rss table and hash key */
4106 		size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4107 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4108 			size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4109 
4110 		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4111 		vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4112 						     vnic->rss_table_size,
4113 						     &vnic->rss_table_dma_addr,
4114 						     GFP_KERNEL);
4115 		if (!vnic->rss_table) {
4116 			rc = -ENOMEM;
4117 			goto out;
4118 		}
4119 
4120 		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4121 		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4122 	}
4123 	return 0;
4124 
4125 out:
4126 	return rc;
4127 }
4128 
4129 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4130 {
4131 	struct bnxt_hwrm_wait_token *token;
4132 
4133 	dma_pool_destroy(bp->hwrm_dma_pool);
4134 	bp->hwrm_dma_pool = NULL;
4135 
4136 	rcu_read_lock();
4137 	hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4138 		WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4139 	rcu_read_unlock();
4140 }
4141 
4142 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4143 {
4144 	bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4145 					    BNXT_HWRM_DMA_SIZE,
4146 					    BNXT_HWRM_DMA_ALIGN, 0);
4147 	if (!bp->hwrm_dma_pool)
4148 		return -ENOMEM;
4149 
4150 	INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4151 
4152 	return 0;
4153 }
4154 
4155 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4156 {
4157 	kfree(stats->hw_masks);
4158 	stats->hw_masks = NULL;
4159 	kfree(stats->sw_stats);
4160 	stats->sw_stats = NULL;
4161 	if (stats->hw_stats) {
4162 		dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4163 				  stats->hw_stats_map);
4164 		stats->hw_stats = NULL;
4165 	}
4166 }
4167 
4168 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4169 				bool alloc_masks)
4170 {
4171 	stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4172 					     &stats->hw_stats_map, GFP_KERNEL);
4173 	if (!stats->hw_stats)
4174 		return -ENOMEM;
4175 
4176 	stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4177 	if (!stats->sw_stats)
4178 		goto stats_mem_err;
4179 
4180 	if (alloc_masks) {
4181 		stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4182 		if (!stats->hw_masks)
4183 			goto stats_mem_err;
4184 	}
4185 	return 0;
4186 
4187 stats_mem_err:
4188 	bnxt_free_stats_mem(bp, stats);
4189 	return -ENOMEM;
4190 }
4191 
4192 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4193 {
4194 	int i;
4195 
4196 	for (i = 0; i < count; i++)
4197 		mask_arr[i] = mask;
4198 }
4199 
4200 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4201 {
4202 	int i;
4203 
4204 	for (i = 0; i < count; i++)
4205 		mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4206 }
4207 
4208 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4209 				    struct bnxt_stats_mem *stats)
4210 {
4211 	struct hwrm_func_qstats_ext_output *resp;
4212 	struct hwrm_func_qstats_ext_input *req;
4213 	__le64 *hw_masks;
4214 	int rc;
4215 
4216 	if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4217 	    !(bp->flags & BNXT_FLAG_CHIP_P5))
4218 		return -EOPNOTSUPP;
4219 
4220 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4221 	if (rc)
4222 		return rc;
4223 
4224 	req->fid = cpu_to_le16(0xffff);
4225 	req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4226 
4227 	resp = hwrm_req_hold(bp, req);
4228 	rc = hwrm_req_send(bp, req);
4229 	if (!rc) {
4230 		hw_masks = &resp->rx_ucast_pkts;
4231 		bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4232 	}
4233 	hwrm_req_drop(bp, req);
4234 	return rc;
4235 }
4236 
4237 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4238 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4239 
4240 static void bnxt_init_stats(struct bnxt *bp)
4241 {
4242 	struct bnxt_napi *bnapi = bp->bnapi[0];
4243 	struct bnxt_cp_ring_info *cpr;
4244 	struct bnxt_stats_mem *stats;
4245 	__le64 *rx_stats, *tx_stats;
4246 	int rc, rx_count, tx_count;
4247 	u64 *rx_masks, *tx_masks;
4248 	u64 mask;
4249 	u8 flags;
4250 
4251 	cpr = &bnapi->cp_ring;
4252 	stats = &cpr->stats;
4253 	rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4254 	if (rc) {
4255 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4256 			mask = (1ULL << 48) - 1;
4257 		else
4258 			mask = -1ULL;
4259 		bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4260 	}
4261 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
4262 		stats = &bp->port_stats;
4263 		rx_stats = stats->hw_stats;
4264 		rx_masks = stats->hw_masks;
4265 		rx_count = sizeof(struct rx_port_stats) / 8;
4266 		tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4267 		tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4268 		tx_count = sizeof(struct tx_port_stats) / 8;
4269 
4270 		flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4271 		rc = bnxt_hwrm_port_qstats(bp, flags);
4272 		if (rc) {
4273 			mask = (1ULL << 40) - 1;
4274 
4275 			bnxt_fill_masks(rx_masks, mask, rx_count);
4276 			bnxt_fill_masks(tx_masks, mask, tx_count);
4277 		} else {
4278 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4279 			bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4280 			bnxt_hwrm_port_qstats(bp, 0);
4281 		}
4282 	}
4283 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4284 		stats = &bp->rx_port_stats_ext;
4285 		rx_stats = stats->hw_stats;
4286 		rx_masks = stats->hw_masks;
4287 		rx_count = sizeof(struct rx_port_stats_ext) / 8;
4288 		stats = &bp->tx_port_stats_ext;
4289 		tx_stats = stats->hw_stats;
4290 		tx_masks = stats->hw_masks;
4291 		tx_count = sizeof(struct tx_port_stats_ext) / 8;
4292 
4293 		flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4294 		rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4295 		if (rc) {
4296 			mask = (1ULL << 40) - 1;
4297 
4298 			bnxt_fill_masks(rx_masks, mask, rx_count);
4299 			if (tx_stats)
4300 				bnxt_fill_masks(tx_masks, mask, tx_count);
4301 		} else {
4302 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4303 			if (tx_stats)
4304 				bnxt_copy_hw_masks(tx_masks, tx_stats,
4305 						   tx_count);
4306 			bnxt_hwrm_port_qstats_ext(bp, 0);
4307 		}
4308 	}
4309 }
4310 
4311 static void bnxt_free_port_stats(struct bnxt *bp)
4312 {
4313 	bp->flags &= ~BNXT_FLAG_PORT_STATS;
4314 	bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4315 
4316 	bnxt_free_stats_mem(bp, &bp->port_stats);
4317 	bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4318 	bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4319 }
4320 
4321 static void bnxt_free_ring_stats(struct bnxt *bp)
4322 {
4323 	int i;
4324 
4325 	if (!bp->bnapi)
4326 		return;
4327 
4328 	for (i = 0; i < bp->cp_nr_rings; i++) {
4329 		struct bnxt_napi *bnapi = bp->bnapi[i];
4330 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4331 
4332 		bnxt_free_stats_mem(bp, &cpr->stats);
4333 	}
4334 }
4335 
4336 static int bnxt_alloc_stats(struct bnxt *bp)
4337 {
4338 	u32 size, i;
4339 	int rc;
4340 
4341 	size = bp->hw_ring_stats_size;
4342 
4343 	for (i = 0; i < bp->cp_nr_rings; i++) {
4344 		struct bnxt_napi *bnapi = bp->bnapi[i];
4345 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4346 
4347 		cpr->stats.len = size;
4348 		rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4349 		if (rc)
4350 			return rc;
4351 
4352 		cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4353 	}
4354 
4355 	if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4356 		return 0;
4357 
4358 	if (bp->port_stats.hw_stats)
4359 		goto alloc_ext_stats;
4360 
4361 	bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4362 	rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4363 	if (rc)
4364 		return rc;
4365 
4366 	bp->flags |= BNXT_FLAG_PORT_STATS;
4367 
4368 alloc_ext_stats:
4369 	/* Display extended statistics only if FW supports it */
4370 	if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4371 		if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4372 			return 0;
4373 
4374 	if (bp->rx_port_stats_ext.hw_stats)
4375 		goto alloc_tx_ext_stats;
4376 
4377 	bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4378 	rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4379 	/* Extended stats are optional */
4380 	if (rc)
4381 		return 0;
4382 
4383 alloc_tx_ext_stats:
4384 	if (bp->tx_port_stats_ext.hw_stats)
4385 		return 0;
4386 
4387 	if (bp->hwrm_spec_code >= 0x10902 ||
4388 	    (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4389 		bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4390 		rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4391 		/* Extended stats are optional */
4392 		if (rc)
4393 			return 0;
4394 	}
4395 	bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4396 	return 0;
4397 }
4398 
4399 static void bnxt_clear_ring_indices(struct bnxt *bp)
4400 {
4401 	int i;
4402 
4403 	if (!bp->bnapi)
4404 		return;
4405 
4406 	for (i = 0; i < bp->cp_nr_rings; i++) {
4407 		struct bnxt_napi *bnapi = bp->bnapi[i];
4408 		struct bnxt_cp_ring_info *cpr;
4409 		struct bnxt_rx_ring_info *rxr;
4410 		struct bnxt_tx_ring_info *txr;
4411 
4412 		if (!bnapi)
4413 			continue;
4414 
4415 		cpr = &bnapi->cp_ring;
4416 		cpr->cp_raw_cons = 0;
4417 
4418 		txr = bnapi->tx_ring;
4419 		if (txr) {
4420 			txr->tx_prod = 0;
4421 			txr->tx_cons = 0;
4422 		}
4423 
4424 		rxr = bnapi->rx_ring;
4425 		if (rxr) {
4426 			rxr->rx_prod = 0;
4427 			rxr->rx_agg_prod = 0;
4428 			rxr->rx_sw_agg_prod = 0;
4429 			rxr->rx_next_cons = 0;
4430 		}
4431 	}
4432 }
4433 
4434 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4435 {
4436 #ifdef CONFIG_RFS_ACCEL
4437 	int i;
4438 
4439 	/* Under rtnl_lock and all our NAPIs have been disabled.  It's
4440 	 * safe to delete the hash table.
4441 	 */
4442 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4443 		struct hlist_head *head;
4444 		struct hlist_node *tmp;
4445 		struct bnxt_ntuple_filter *fltr;
4446 
4447 		head = &bp->ntp_fltr_hash_tbl[i];
4448 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4449 			hlist_del(&fltr->hash);
4450 			kfree(fltr);
4451 		}
4452 	}
4453 	if (irq_reinit) {
4454 		bitmap_free(bp->ntp_fltr_bmap);
4455 		bp->ntp_fltr_bmap = NULL;
4456 	}
4457 	bp->ntp_fltr_count = 0;
4458 #endif
4459 }
4460 
4461 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4462 {
4463 #ifdef CONFIG_RFS_ACCEL
4464 	int i, rc = 0;
4465 
4466 	if (!(bp->flags & BNXT_FLAG_RFS))
4467 		return 0;
4468 
4469 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4470 		INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4471 
4472 	bp->ntp_fltr_count = 0;
4473 	bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL);
4474 
4475 	if (!bp->ntp_fltr_bmap)
4476 		rc = -ENOMEM;
4477 
4478 	return rc;
4479 #else
4480 	return 0;
4481 #endif
4482 }
4483 
4484 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4485 {
4486 	bnxt_free_vnic_attributes(bp);
4487 	bnxt_free_tx_rings(bp);
4488 	bnxt_free_rx_rings(bp);
4489 	bnxt_free_cp_rings(bp);
4490 	bnxt_free_all_cp_arrays(bp);
4491 	bnxt_free_ntp_fltrs(bp, irq_re_init);
4492 	if (irq_re_init) {
4493 		bnxt_free_ring_stats(bp);
4494 		if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4495 		    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4496 			bnxt_free_port_stats(bp);
4497 		bnxt_free_ring_grps(bp);
4498 		bnxt_free_vnics(bp);
4499 		kfree(bp->tx_ring_map);
4500 		bp->tx_ring_map = NULL;
4501 		kfree(bp->tx_ring);
4502 		bp->tx_ring = NULL;
4503 		kfree(bp->rx_ring);
4504 		bp->rx_ring = NULL;
4505 		kfree(bp->bnapi);
4506 		bp->bnapi = NULL;
4507 	} else {
4508 		bnxt_clear_ring_indices(bp);
4509 	}
4510 }
4511 
4512 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4513 {
4514 	int i, j, rc, size, arr_size;
4515 	void *bnapi;
4516 
4517 	if (irq_re_init) {
4518 		/* Allocate bnapi mem pointer array and mem block for
4519 		 * all queues
4520 		 */
4521 		arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4522 				bp->cp_nr_rings);
4523 		size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4524 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4525 		if (!bnapi)
4526 			return -ENOMEM;
4527 
4528 		bp->bnapi = bnapi;
4529 		bnapi += arr_size;
4530 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4531 			bp->bnapi[i] = bnapi;
4532 			bp->bnapi[i]->index = i;
4533 			bp->bnapi[i]->bp = bp;
4534 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4535 				struct bnxt_cp_ring_info *cpr =
4536 					&bp->bnapi[i]->cp_ring;
4537 
4538 				cpr->cp_ring_struct.ring_mem.flags =
4539 					BNXT_RMEM_RING_PTE_FLAG;
4540 			}
4541 		}
4542 
4543 		bp->rx_ring = kcalloc(bp->rx_nr_rings,
4544 				      sizeof(struct bnxt_rx_ring_info),
4545 				      GFP_KERNEL);
4546 		if (!bp->rx_ring)
4547 			return -ENOMEM;
4548 
4549 		for (i = 0; i < bp->rx_nr_rings; i++) {
4550 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4551 
4552 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4553 				rxr->rx_ring_struct.ring_mem.flags =
4554 					BNXT_RMEM_RING_PTE_FLAG;
4555 				rxr->rx_agg_ring_struct.ring_mem.flags =
4556 					BNXT_RMEM_RING_PTE_FLAG;
4557 			}
4558 			rxr->bnapi = bp->bnapi[i];
4559 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4560 		}
4561 
4562 		bp->tx_ring = kcalloc(bp->tx_nr_rings,
4563 				      sizeof(struct bnxt_tx_ring_info),
4564 				      GFP_KERNEL);
4565 		if (!bp->tx_ring)
4566 			return -ENOMEM;
4567 
4568 		bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4569 					  GFP_KERNEL);
4570 
4571 		if (!bp->tx_ring_map)
4572 			return -ENOMEM;
4573 
4574 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4575 			j = 0;
4576 		else
4577 			j = bp->rx_nr_rings;
4578 
4579 		for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4580 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4581 
4582 			if (bp->flags & BNXT_FLAG_CHIP_P5)
4583 				txr->tx_ring_struct.ring_mem.flags =
4584 					BNXT_RMEM_RING_PTE_FLAG;
4585 			txr->bnapi = bp->bnapi[j];
4586 			bp->bnapi[j]->tx_ring = txr;
4587 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4588 			if (i >= bp->tx_nr_rings_xdp) {
4589 				txr->txq_index = i - bp->tx_nr_rings_xdp;
4590 				bp->bnapi[j]->tx_int = bnxt_tx_int;
4591 			} else {
4592 				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4593 				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4594 			}
4595 		}
4596 
4597 		rc = bnxt_alloc_stats(bp);
4598 		if (rc)
4599 			goto alloc_mem_err;
4600 		bnxt_init_stats(bp);
4601 
4602 		rc = bnxt_alloc_ntp_fltrs(bp);
4603 		if (rc)
4604 			goto alloc_mem_err;
4605 
4606 		rc = bnxt_alloc_vnics(bp);
4607 		if (rc)
4608 			goto alloc_mem_err;
4609 	}
4610 
4611 	rc = bnxt_alloc_all_cp_arrays(bp);
4612 	if (rc)
4613 		goto alloc_mem_err;
4614 
4615 	bnxt_init_ring_struct(bp);
4616 
4617 	rc = bnxt_alloc_rx_rings(bp);
4618 	if (rc)
4619 		goto alloc_mem_err;
4620 
4621 	rc = bnxt_alloc_tx_rings(bp);
4622 	if (rc)
4623 		goto alloc_mem_err;
4624 
4625 	rc = bnxt_alloc_cp_rings(bp);
4626 	if (rc)
4627 		goto alloc_mem_err;
4628 
4629 	bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4630 				  BNXT_VNIC_UCAST_FLAG;
4631 	rc = bnxt_alloc_vnic_attributes(bp);
4632 	if (rc)
4633 		goto alloc_mem_err;
4634 	return 0;
4635 
4636 alloc_mem_err:
4637 	bnxt_free_mem(bp, true);
4638 	return rc;
4639 }
4640 
4641 static void bnxt_disable_int(struct bnxt *bp)
4642 {
4643 	int i;
4644 
4645 	if (!bp->bnapi)
4646 		return;
4647 
4648 	for (i = 0; i < bp->cp_nr_rings; i++) {
4649 		struct bnxt_napi *bnapi = bp->bnapi[i];
4650 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4651 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4652 
4653 		if (ring->fw_ring_id != INVALID_HW_RING_ID)
4654 			bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4655 	}
4656 }
4657 
4658 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4659 {
4660 	struct bnxt_napi *bnapi = bp->bnapi[n];
4661 	struct bnxt_cp_ring_info *cpr;
4662 
4663 	cpr = &bnapi->cp_ring;
4664 	return cpr->cp_ring_struct.map_idx;
4665 }
4666 
4667 static void bnxt_disable_int_sync(struct bnxt *bp)
4668 {
4669 	int i;
4670 
4671 	if (!bp->irq_tbl)
4672 		return;
4673 
4674 	atomic_inc(&bp->intr_sem);
4675 
4676 	bnxt_disable_int(bp);
4677 	for (i = 0; i < bp->cp_nr_rings; i++) {
4678 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4679 
4680 		synchronize_irq(bp->irq_tbl[map_idx].vector);
4681 	}
4682 }
4683 
4684 static void bnxt_enable_int(struct bnxt *bp)
4685 {
4686 	int i;
4687 
4688 	atomic_set(&bp->intr_sem, 0);
4689 	for (i = 0; i < bp->cp_nr_rings; i++) {
4690 		struct bnxt_napi *bnapi = bp->bnapi[i];
4691 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4692 
4693 		bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4694 	}
4695 }
4696 
4697 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4698 			    bool async_only)
4699 {
4700 	DECLARE_BITMAP(async_events_bmap, 256);
4701 	u32 *events = (u32 *)async_events_bmap;
4702 	struct hwrm_func_drv_rgtr_output *resp;
4703 	struct hwrm_func_drv_rgtr_input *req;
4704 	u32 flags;
4705 	int rc, i;
4706 
4707 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
4708 	if (rc)
4709 		return rc;
4710 
4711 	req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4712 				   FUNC_DRV_RGTR_REQ_ENABLES_VER |
4713 				   FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4714 
4715 	req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4716 	flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4717 	if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4718 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4719 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4720 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4721 			 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4722 	req->flags = cpu_to_le32(flags);
4723 	req->ver_maj_8b = DRV_VER_MAJ;
4724 	req->ver_min_8b = DRV_VER_MIN;
4725 	req->ver_upd_8b = DRV_VER_UPD;
4726 	req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
4727 	req->ver_min = cpu_to_le16(DRV_VER_MIN);
4728 	req->ver_upd = cpu_to_le16(DRV_VER_UPD);
4729 
4730 	if (BNXT_PF(bp)) {
4731 		u32 data[8];
4732 		int i;
4733 
4734 		memset(data, 0, sizeof(data));
4735 		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4736 			u16 cmd = bnxt_vf_req_snif[i];
4737 			unsigned int bit, idx;
4738 
4739 			idx = cmd / 32;
4740 			bit = cmd % 32;
4741 			data[idx] |= 1 << bit;
4742 		}
4743 
4744 		for (i = 0; i < 8; i++)
4745 			req->vf_req_fwd[i] = cpu_to_le32(data[i]);
4746 
4747 		req->enables |=
4748 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4749 	}
4750 
4751 	if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4752 		req->flags |= cpu_to_le32(
4753 			FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4754 
4755 	memset(async_events_bmap, 0, sizeof(async_events_bmap));
4756 	for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4757 		u16 event_id = bnxt_async_events_arr[i];
4758 
4759 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4760 		    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4761 			continue;
4762 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
4763 		    !bp->ptp_cfg)
4764 			continue;
4765 		__set_bit(bnxt_async_events_arr[i], async_events_bmap);
4766 	}
4767 	if (bmap && bmap_size) {
4768 		for (i = 0; i < bmap_size; i++) {
4769 			if (test_bit(i, bmap))
4770 				__set_bit(i, async_events_bmap);
4771 		}
4772 	}
4773 	for (i = 0; i < 8; i++)
4774 		req->async_event_fwd[i] |= cpu_to_le32(events[i]);
4775 
4776 	if (async_only)
4777 		req->enables =
4778 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4779 
4780 	resp = hwrm_req_hold(bp, req);
4781 	rc = hwrm_req_send(bp, req);
4782 	if (!rc) {
4783 		set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4784 		if (resp->flags &
4785 		    cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4786 			bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4787 	}
4788 	hwrm_req_drop(bp, req);
4789 	return rc;
4790 }
4791 
4792 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4793 {
4794 	struct hwrm_func_drv_unrgtr_input *req;
4795 	int rc;
4796 
4797 	if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4798 		return 0;
4799 
4800 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
4801 	if (rc)
4802 		return rc;
4803 	return hwrm_req_send(bp, req);
4804 }
4805 
4806 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4807 {
4808 	struct hwrm_tunnel_dst_port_free_input *req;
4809 	int rc;
4810 
4811 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
4812 	    bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
4813 		return 0;
4814 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
4815 	    bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
4816 		return 0;
4817 
4818 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
4819 	if (rc)
4820 		return rc;
4821 
4822 	req->tunnel_type = tunnel_type;
4823 
4824 	switch (tunnel_type) {
4825 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4826 		req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4827 		bp->vxlan_port = 0;
4828 		bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4829 		break;
4830 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4831 		req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4832 		bp->nge_port = 0;
4833 		bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4834 		break;
4835 	default:
4836 		break;
4837 	}
4838 
4839 	rc = hwrm_req_send(bp, req);
4840 	if (rc)
4841 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4842 			   rc);
4843 	return rc;
4844 }
4845 
4846 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4847 					   u8 tunnel_type)
4848 {
4849 	struct hwrm_tunnel_dst_port_alloc_output *resp;
4850 	struct hwrm_tunnel_dst_port_alloc_input *req;
4851 	int rc;
4852 
4853 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
4854 	if (rc)
4855 		return rc;
4856 
4857 	req->tunnel_type = tunnel_type;
4858 	req->tunnel_dst_port_val = port;
4859 
4860 	resp = hwrm_req_hold(bp, req);
4861 	rc = hwrm_req_send(bp, req);
4862 	if (rc) {
4863 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4864 			   rc);
4865 		goto err_out;
4866 	}
4867 
4868 	switch (tunnel_type) {
4869 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4870 		bp->vxlan_port = port;
4871 		bp->vxlan_fw_dst_port_id =
4872 			le16_to_cpu(resp->tunnel_dst_port_id);
4873 		break;
4874 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4875 		bp->nge_port = port;
4876 		bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4877 		break;
4878 	default:
4879 		break;
4880 	}
4881 
4882 err_out:
4883 	hwrm_req_drop(bp, req);
4884 	return rc;
4885 }
4886 
4887 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4888 {
4889 	struct hwrm_cfa_l2_set_rx_mask_input *req;
4890 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4891 	int rc;
4892 
4893 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
4894 	if (rc)
4895 		return rc;
4896 
4897 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4898 	if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
4899 		req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4900 		req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4901 	}
4902 	req->mask = cpu_to_le32(vnic->rx_mask);
4903 	return hwrm_req_send_silent(bp, req);
4904 }
4905 
4906 #ifdef CONFIG_RFS_ACCEL
4907 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4908 					    struct bnxt_ntuple_filter *fltr)
4909 {
4910 	struct hwrm_cfa_ntuple_filter_free_input *req;
4911 	int rc;
4912 
4913 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
4914 	if (rc)
4915 		return rc;
4916 
4917 	req->ntuple_filter_id = fltr->filter_id;
4918 	return hwrm_req_send(bp, req);
4919 }
4920 
4921 #define BNXT_NTP_FLTR_FLAGS					\
4922 	(CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |	\
4923 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |	\
4924 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |	\
4925 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |	\
4926 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |	\
4927 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |	\
4928 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |	\
4929 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |	\
4930 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |	\
4931 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |		\
4932 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |	\
4933 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |		\
4934 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |	\
4935 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4936 
4937 #define BNXT_NTP_TUNNEL_FLTR_FLAG				\
4938 		CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4939 
4940 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4941 					     struct bnxt_ntuple_filter *fltr)
4942 {
4943 	struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4944 	struct hwrm_cfa_ntuple_filter_alloc_input *req;
4945 	struct flow_keys *keys = &fltr->fkeys;
4946 	struct bnxt_vnic_info *vnic;
4947 	u32 flags = 0;
4948 	int rc;
4949 
4950 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
4951 	if (rc)
4952 		return rc;
4953 
4954 	req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4955 
4956 	if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4957 		flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4958 		req->dst_id = cpu_to_le16(fltr->rxq);
4959 	} else {
4960 		vnic = &bp->vnic_info[fltr->rxq + 1];
4961 		req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
4962 	}
4963 	req->flags = cpu_to_le32(flags);
4964 	req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4965 
4966 	req->ethertype = htons(ETH_P_IP);
4967 	memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4968 	req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4969 	req->ip_protocol = keys->basic.ip_proto;
4970 
4971 	if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4972 		int i;
4973 
4974 		req->ethertype = htons(ETH_P_IPV6);
4975 		req->ip_addr_type =
4976 			CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4977 		*(struct in6_addr *)&req->src_ipaddr[0] =
4978 			keys->addrs.v6addrs.src;
4979 		*(struct in6_addr *)&req->dst_ipaddr[0] =
4980 			keys->addrs.v6addrs.dst;
4981 		for (i = 0; i < 4; i++) {
4982 			req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4983 			req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4984 		}
4985 	} else {
4986 		req->src_ipaddr[0] = keys->addrs.v4addrs.src;
4987 		req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4988 		req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4989 		req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4990 	}
4991 	if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4992 		req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4993 		req->tunnel_type =
4994 			CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4995 	}
4996 
4997 	req->src_port = keys->ports.src;
4998 	req->src_port_mask = cpu_to_be16(0xffff);
4999 	req->dst_port = keys->ports.dst;
5000 	req->dst_port_mask = cpu_to_be16(0xffff);
5001 
5002 	resp = hwrm_req_hold(bp, req);
5003 	rc = hwrm_req_send(bp, req);
5004 	if (!rc)
5005 		fltr->filter_id = resp->ntuple_filter_id;
5006 	hwrm_req_drop(bp, req);
5007 	return rc;
5008 }
5009 #endif
5010 
5011 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5012 				     const u8 *mac_addr)
5013 {
5014 	struct hwrm_cfa_l2_filter_alloc_output *resp;
5015 	struct hwrm_cfa_l2_filter_alloc_input *req;
5016 	int rc;
5017 
5018 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5019 	if (rc)
5020 		return rc;
5021 
5022 	req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5023 	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5024 		req->flags |=
5025 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5026 	req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
5027 	req->enables =
5028 		cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5029 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5030 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5031 	memcpy(req->l2_addr, mac_addr, ETH_ALEN);
5032 	req->l2_addr_mask[0] = 0xff;
5033 	req->l2_addr_mask[1] = 0xff;
5034 	req->l2_addr_mask[2] = 0xff;
5035 	req->l2_addr_mask[3] = 0xff;
5036 	req->l2_addr_mask[4] = 0xff;
5037 	req->l2_addr_mask[5] = 0xff;
5038 
5039 	resp = hwrm_req_hold(bp, req);
5040 	rc = hwrm_req_send(bp, req);
5041 	if (!rc)
5042 		bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
5043 							resp->l2_filter_id;
5044 	hwrm_req_drop(bp, req);
5045 	return rc;
5046 }
5047 
5048 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5049 {
5050 	struct hwrm_cfa_l2_filter_free_input *req;
5051 	u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5052 	int rc;
5053 
5054 	/* Any associated ntuple filters will also be cleared by firmware. */
5055 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5056 	if (rc)
5057 		return rc;
5058 	hwrm_req_hold(bp, req);
5059 	for (i = 0; i < num_of_vnics; i++) {
5060 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5061 
5062 		for (j = 0; j < vnic->uc_filter_count; j++) {
5063 			req->l2_filter_id = vnic->fw_l2_filter_id[j];
5064 
5065 			rc = hwrm_req_send(bp, req);
5066 		}
5067 		vnic->uc_filter_count = 0;
5068 	}
5069 	hwrm_req_drop(bp, req);
5070 	return rc;
5071 }
5072 
5073 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5074 {
5075 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5076 	u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5077 	struct hwrm_vnic_tpa_cfg_input *req;
5078 	int rc;
5079 
5080 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5081 		return 0;
5082 
5083 	rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5084 	if (rc)
5085 		return rc;
5086 
5087 	if (tpa_flags) {
5088 		u16 mss = bp->dev->mtu - 40;
5089 		u32 nsegs, n, segs = 0, flags;
5090 
5091 		flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5092 			VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5093 			VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5094 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5095 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5096 		if (tpa_flags & BNXT_FLAG_GRO)
5097 			flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5098 
5099 		req->flags = cpu_to_le32(flags);
5100 
5101 		req->enables =
5102 			cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5103 				    VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5104 				    VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5105 
5106 		/* Number of segs are log2 units, and first packet is not
5107 		 * included as part of this units.
5108 		 */
5109 		if (mss <= BNXT_RX_PAGE_SIZE) {
5110 			n = BNXT_RX_PAGE_SIZE / mss;
5111 			nsegs = (MAX_SKB_FRAGS - 1) * n;
5112 		} else {
5113 			n = mss / BNXT_RX_PAGE_SIZE;
5114 			if (mss & (BNXT_RX_PAGE_SIZE - 1))
5115 				n++;
5116 			nsegs = (MAX_SKB_FRAGS - n) / n;
5117 		}
5118 
5119 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5120 			segs = MAX_TPA_SEGS_P5;
5121 			max_aggs = bp->max_tpa;
5122 		} else {
5123 			segs = ilog2(nsegs);
5124 		}
5125 		req->max_agg_segs = cpu_to_le16(segs);
5126 		req->max_aggs = cpu_to_le16(max_aggs);
5127 
5128 		req->min_agg_len = cpu_to_le32(512);
5129 	}
5130 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5131 
5132 	return hwrm_req_send(bp, req);
5133 }
5134 
5135 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5136 {
5137 	struct bnxt_ring_grp_info *grp_info;
5138 
5139 	grp_info = &bp->grp_info[ring->grp_idx];
5140 	return grp_info->cp_fw_ring_id;
5141 }
5142 
5143 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5144 {
5145 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5146 		struct bnxt_napi *bnapi = rxr->bnapi;
5147 		struct bnxt_cp_ring_info *cpr;
5148 
5149 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5150 		return cpr->cp_ring_struct.fw_ring_id;
5151 	} else {
5152 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5153 	}
5154 }
5155 
5156 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5157 {
5158 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5159 		struct bnxt_napi *bnapi = txr->bnapi;
5160 		struct bnxt_cp_ring_info *cpr;
5161 
5162 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5163 		return cpr->cp_ring_struct.fw_ring_id;
5164 	} else {
5165 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5166 	}
5167 }
5168 
5169 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5170 {
5171 	int entries;
5172 
5173 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5174 		entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5175 	else
5176 		entries = HW_HASH_INDEX_SIZE;
5177 
5178 	bp->rss_indir_tbl_entries = entries;
5179 	bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5180 					  GFP_KERNEL);
5181 	if (!bp->rss_indir_tbl)
5182 		return -ENOMEM;
5183 	return 0;
5184 }
5185 
5186 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5187 {
5188 	u16 max_rings, max_entries, pad, i;
5189 
5190 	if (!bp->rx_nr_rings)
5191 		return;
5192 
5193 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5194 		max_rings = bp->rx_nr_rings - 1;
5195 	else
5196 		max_rings = bp->rx_nr_rings;
5197 
5198 	max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5199 
5200 	for (i = 0; i < max_entries; i++)
5201 		bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5202 
5203 	pad = bp->rss_indir_tbl_entries - max_entries;
5204 	if (pad)
5205 		memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5206 }
5207 
5208 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5209 {
5210 	u16 i, tbl_size, max_ring = 0;
5211 
5212 	if (!bp->rss_indir_tbl)
5213 		return 0;
5214 
5215 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5216 	for (i = 0; i < tbl_size; i++)
5217 		max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5218 	return max_ring;
5219 }
5220 
5221 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5222 {
5223 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5224 		return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5225 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5226 		return 2;
5227 	return 1;
5228 }
5229 
5230 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5231 {
5232 	bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5233 	u16 i, j;
5234 
5235 	/* Fill the RSS indirection table with ring group ids */
5236 	for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5237 		if (!no_rss)
5238 			j = bp->rss_indir_tbl[i];
5239 		vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5240 	}
5241 }
5242 
5243 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5244 				    struct bnxt_vnic_info *vnic)
5245 {
5246 	__le16 *ring_tbl = vnic->rss_table;
5247 	struct bnxt_rx_ring_info *rxr;
5248 	u16 tbl_size, i;
5249 
5250 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5251 
5252 	for (i = 0; i < tbl_size; i++) {
5253 		u16 ring_id, j;
5254 
5255 		j = bp->rss_indir_tbl[i];
5256 		rxr = &bp->rx_ring[j];
5257 
5258 		ring_id = rxr->rx_ring_struct.fw_ring_id;
5259 		*ring_tbl++ = cpu_to_le16(ring_id);
5260 		ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5261 		*ring_tbl++ = cpu_to_le16(ring_id);
5262 	}
5263 }
5264 
5265 static void
5266 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5267 			 struct bnxt_vnic_info *vnic)
5268 {
5269 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5270 		bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5271 	else
5272 		bnxt_fill_hw_rss_tbl(bp, vnic);
5273 
5274 	if (bp->rss_hash_delta) {
5275 		req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5276 		if (bp->rss_hash_cfg & bp->rss_hash_delta)
5277 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5278 		else
5279 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5280 	} else {
5281 		req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5282 	}
5283 	req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5284 	req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5285 	req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5286 }
5287 
5288 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5289 {
5290 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5291 	struct hwrm_vnic_rss_cfg_input *req;
5292 	int rc;
5293 
5294 	if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5295 	    vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5296 		return 0;
5297 
5298 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5299 	if (rc)
5300 		return rc;
5301 
5302 	if (set_rss)
5303 		__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5304 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5305 	return hwrm_req_send(bp, req);
5306 }
5307 
5308 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5309 {
5310 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5311 	struct hwrm_vnic_rss_cfg_input *req;
5312 	dma_addr_t ring_tbl_map;
5313 	u32 i, nr_ctxs;
5314 	int rc;
5315 
5316 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5317 	if (rc)
5318 		return rc;
5319 
5320 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5321 	if (!set_rss)
5322 		return hwrm_req_send(bp, req);
5323 
5324 	__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5325 	ring_tbl_map = vnic->rss_table_dma_addr;
5326 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5327 
5328 	hwrm_req_hold(bp, req);
5329 	for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5330 		req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5331 		req->ring_table_pair_index = i;
5332 		req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5333 		rc = hwrm_req_send(bp, req);
5334 		if (rc)
5335 			goto exit;
5336 	}
5337 
5338 exit:
5339 	hwrm_req_drop(bp, req);
5340 	return rc;
5341 }
5342 
5343 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
5344 {
5345 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5346 	struct hwrm_vnic_rss_qcfg_output *resp;
5347 	struct hwrm_vnic_rss_qcfg_input *req;
5348 
5349 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
5350 		return;
5351 
5352 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5353 	/* all contexts configured to same hash_type, zero always exists */
5354 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5355 	resp = hwrm_req_hold(bp, req);
5356 	if (!hwrm_req_send(bp, req)) {
5357 		bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
5358 		bp->rss_hash_delta = 0;
5359 	}
5360 	hwrm_req_drop(bp, req);
5361 }
5362 
5363 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5364 {
5365 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5366 	struct hwrm_vnic_plcmodes_cfg_input *req;
5367 	int rc;
5368 
5369 	rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
5370 	if (rc)
5371 		return rc;
5372 
5373 	req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
5374 	req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
5375 
5376 	if (BNXT_RX_PAGE_MODE(bp)) {
5377 		req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
5378 	} else {
5379 		req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5380 					  VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5381 		req->enables |=
5382 			cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5383 		req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5384 		req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5385 	}
5386 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5387 	return hwrm_req_send(bp, req);
5388 }
5389 
5390 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5391 					u16 ctx_idx)
5392 {
5393 	struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
5394 
5395 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
5396 		return;
5397 
5398 	req->rss_cos_lb_ctx_id =
5399 		cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5400 
5401 	hwrm_req_send(bp, req);
5402 	bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5403 }
5404 
5405 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5406 {
5407 	int i, j;
5408 
5409 	for (i = 0; i < bp->nr_vnics; i++) {
5410 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5411 
5412 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5413 			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5414 				bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5415 		}
5416 	}
5417 	bp->rsscos_nr_ctxs = 0;
5418 }
5419 
5420 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5421 {
5422 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
5423 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
5424 	int rc;
5425 
5426 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
5427 	if (rc)
5428 		return rc;
5429 
5430 	resp = hwrm_req_hold(bp, req);
5431 	rc = hwrm_req_send(bp, req);
5432 	if (!rc)
5433 		bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5434 			le16_to_cpu(resp->rss_cos_lb_ctx_id);
5435 	hwrm_req_drop(bp, req);
5436 
5437 	return rc;
5438 }
5439 
5440 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5441 {
5442 	if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5443 		return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5444 	return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5445 }
5446 
5447 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5448 {
5449 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5450 	struct hwrm_vnic_cfg_input *req;
5451 	unsigned int ring = 0, grp_idx;
5452 	u16 def_vlan = 0;
5453 	int rc;
5454 
5455 	rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
5456 	if (rc)
5457 		return rc;
5458 
5459 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5460 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5461 
5462 		req->default_rx_ring_id =
5463 			cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5464 		req->default_cmpl_ring_id =
5465 			cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5466 		req->enables =
5467 			cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5468 				    VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5469 		goto vnic_mru;
5470 	}
5471 	req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5472 	/* Only RSS support for now TBD: COS & LB */
5473 	if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5474 		req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5475 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5476 					   VNIC_CFG_REQ_ENABLES_MRU);
5477 	} else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5478 		req->rss_rule =
5479 			cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5480 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5481 					   VNIC_CFG_REQ_ENABLES_MRU);
5482 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5483 	} else {
5484 		req->rss_rule = cpu_to_le16(0xffff);
5485 	}
5486 
5487 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5488 	    (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5489 		req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5490 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5491 	} else {
5492 		req->cos_rule = cpu_to_le16(0xffff);
5493 	}
5494 
5495 	if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5496 		ring = 0;
5497 	else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5498 		ring = vnic_id - 1;
5499 	else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5500 		ring = bp->rx_nr_rings - 1;
5501 
5502 	grp_idx = bp->rx_ring[ring].bnapi->index;
5503 	req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5504 	req->lb_rule = cpu_to_le16(0xffff);
5505 vnic_mru:
5506 	req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5507 
5508 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5509 #ifdef CONFIG_BNXT_SRIOV
5510 	if (BNXT_VF(bp))
5511 		def_vlan = bp->vf.vlan;
5512 #endif
5513 	if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5514 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5515 	if (!vnic_id && bnxt_ulp_registered(bp->edev))
5516 		req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5517 
5518 	return hwrm_req_send(bp, req);
5519 }
5520 
5521 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5522 {
5523 	if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5524 		struct hwrm_vnic_free_input *req;
5525 
5526 		if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
5527 			return;
5528 
5529 		req->vnic_id =
5530 			cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5531 
5532 		hwrm_req_send(bp, req);
5533 		bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5534 	}
5535 }
5536 
5537 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5538 {
5539 	u16 i;
5540 
5541 	for (i = 0; i < bp->nr_vnics; i++)
5542 		bnxt_hwrm_vnic_free_one(bp, i);
5543 }
5544 
5545 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5546 				unsigned int start_rx_ring_idx,
5547 				unsigned int nr_rings)
5548 {
5549 	unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5550 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5551 	struct hwrm_vnic_alloc_output *resp;
5552 	struct hwrm_vnic_alloc_input *req;
5553 	int rc;
5554 
5555 	rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
5556 	if (rc)
5557 		return rc;
5558 
5559 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5560 		goto vnic_no_ring_grps;
5561 
5562 	/* map ring groups to this vnic */
5563 	for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5564 		grp_idx = bp->rx_ring[i].bnapi->index;
5565 		if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5566 			netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5567 				   j, nr_rings);
5568 			break;
5569 		}
5570 		vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5571 	}
5572 
5573 vnic_no_ring_grps:
5574 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5575 		vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5576 	if (vnic_id == 0)
5577 		req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5578 
5579 	resp = hwrm_req_hold(bp, req);
5580 	rc = hwrm_req_send(bp, req);
5581 	if (!rc)
5582 		vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5583 	hwrm_req_drop(bp, req);
5584 	return rc;
5585 }
5586 
5587 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5588 {
5589 	struct hwrm_vnic_qcaps_output *resp;
5590 	struct hwrm_vnic_qcaps_input *req;
5591 	int rc;
5592 
5593 	bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5594 	bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5595 	if (bp->hwrm_spec_code < 0x10600)
5596 		return 0;
5597 
5598 	rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
5599 	if (rc)
5600 		return rc;
5601 
5602 	resp = hwrm_req_hold(bp, req);
5603 	rc = hwrm_req_send(bp, req);
5604 	if (!rc) {
5605 		u32 flags = le32_to_cpu(resp->flags);
5606 
5607 		if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5608 		    (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5609 			bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5610 		if (flags &
5611 		    VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5612 			bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5613 
5614 		/* Older P5 fw before EXT_HW_STATS support did not set
5615 		 * VLAN_STRIP_CAP properly.
5616 		 */
5617 		if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5618 		    (BNXT_CHIP_P5_THOR(bp) &&
5619 		     !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5620 			bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5621 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
5622 			bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA;
5623 		bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5624 		if (bp->max_tpa_v2) {
5625 			if (BNXT_CHIP_P5_THOR(bp))
5626 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5627 			else
5628 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5629 		}
5630 	}
5631 	hwrm_req_drop(bp, req);
5632 	return rc;
5633 }
5634 
5635 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5636 {
5637 	struct hwrm_ring_grp_alloc_output *resp;
5638 	struct hwrm_ring_grp_alloc_input *req;
5639 	int rc;
5640 	u16 i;
5641 
5642 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5643 		return 0;
5644 
5645 	rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
5646 	if (rc)
5647 		return rc;
5648 
5649 	resp = hwrm_req_hold(bp, req);
5650 	for (i = 0; i < bp->rx_nr_rings; i++) {
5651 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5652 
5653 		req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5654 		req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5655 		req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5656 		req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5657 
5658 		rc = hwrm_req_send(bp, req);
5659 
5660 		if (rc)
5661 			break;
5662 
5663 		bp->grp_info[grp_idx].fw_grp_id =
5664 			le32_to_cpu(resp->ring_group_id);
5665 	}
5666 	hwrm_req_drop(bp, req);
5667 	return rc;
5668 }
5669 
5670 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5671 {
5672 	struct hwrm_ring_grp_free_input *req;
5673 	u16 i;
5674 
5675 	if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5676 		return;
5677 
5678 	if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
5679 		return;
5680 
5681 	hwrm_req_hold(bp, req);
5682 	for (i = 0; i < bp->cp_nr_rings; i++) {
5683 		if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5684 			continue;
5685 		req->ring_group_id =
5686 			cpu_to_le32(bp->grp_info[i].fw_grp_id);
5687 
5688 		hwrm_req_send(bp, req);
5689 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5690 	}
5691 	hwrm_req_drop(bp, req);
5692 }
5693 
5694 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5695 				    struct bnxt_ring_struct *ring,
5696 				    u32 ring_type, u32 map_index)
5697 {
5698 	struct hwrm_ring_alloc_output *resp;
5699 	struct hwrm_ring_alloc_input *req;
5700 	struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5701 	struct bnxt_ring_grp_info *grp_info;
5702 	int rc, err = 0;
5703 	u16 ring_id;
5704 
5705 	rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
5706 	if (rc)
5707 		goto exit;
5708 
5709 	req->enables = 0;
5710 	if (rmem->nr_pages > 1) {
5711 		req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5712 		/* Page size is in log2 units */
5713 		req->page_size = BNXT_PAGE_SHIFT;
5714 		req->page_tbl_depth = 1;
5715 	} else {
5716 		req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5717 	}
5718 	req->fbo = 0;
5719 	/* Association of ring index with doorbell index and MSIX number */
5720 	req->logical_id = cpu_to_le16(map_index);
5721 
5722 	switch (ring_type) {
5723 	case HWRM_RING_ALLOC_TX: {
5724 		struct bnxt_tx_ring_info *txr;
5725 
5726 		txr = container_of(ring, struct bnxt_tx_ring_info,
5727 				   tx_ring_struct);
5728 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5729 		/* Association of transmit ring with completion ring */
5730 		grp_info = &bp->grp_info[ring->grp_idx];
5731 		req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5732 		req->length = cpu_to_le32(bp->tx_ring_mask + 1);
5733 		req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5734 		req->queue_id = cpu_to_le16(ring->queue_id);
5735 		break;
5736 	}
5737 	case HWRM_RING_ALLOC_RX:
5738 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5739 		req->length = cpu_to_le32(bp->rx_ring_mask + 1);
5740 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5741 			u16 flags = 0;
5742 
5743 			/* Association of rx ring with stats context */
5744 			grp_info = &bp->grp_info[ring->grp_idx];
5745 			req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5746 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5747 			req->enables |= cpu_to_le32(
5748 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5749 			if (NET_IP_ALIGN == 2)
5750 				flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5751 			req->flags = cpu_to_le16(flags);
5752 		}
5753 		break;
5754 	case HWRM_RING_ALLOC_AGG:
5755 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5756 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5757 			/* Association of agg ring with rx ring */
5758 			grp_info = &bp->grp_info[ring->grp_idx];
5759 			req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5760 			req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5761 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5762 			req->enables |= cpu_to_le32(
5763 				RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5764 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5765 		} else {
5766 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5767 		}
5768 		req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5769 		break;
5770 	case HWRM_RING_ALLOC_CMPL:
5771 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5772 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5773 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5774 			/* Association of cp ring with nq */
5775 			grp_info = &bp->grp_info[map_index];
5776 			req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5777 			req->cq_handle = cpu_to_le64(ring->handle);
5778 			req->enables |= cpu_to_le32(
5779 				RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5780 		} else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5781 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5782 		}
5783 		break;
5784 	case HWRM_RING_ALLOC_NQ:
5785 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5786 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5787 		if (bp->flags & BNXT_FLAG_USING_MSIX)
5788 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5789 		break;
5790 	default:
5791 		netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5792 			   ring_type);
5793 		return -1;
5794 	}
5795 
5796 	resp = hwrm_req_hold(bp, req);
5797 	rc = hwrm_req_send(bp, req);
5798 	err = le16_to_cpu(resp->error_code);
5799 	ring_id = le16_to_cpu(resp->ring_id);
5800 	hwrm_req_drop(bp, req);
5801 
5802 exit:
5803 	if (rc || err) {
5804 		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5805 			   ring_type, rc, err);
5806 		return -EIO;
5807 	}
5808 	ring->fw_ring_id = ring_id;
5809 	return rc;
5810 }
5811 
5812 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5813 {
5814 	int rc;
5815 
5816 	if (BNXT_PF(bp)) {
5817 		struct hwrm_func_cfg_input *req;
5818 
5819 		rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
5820 		if (rc)
5821 			return rc;
5822 
5823 		req->fid = cpu_to_le16(0xffff);
5824 		req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5825 		req->async_event_cr = cpu_to_le16(idx);
5826 		return hwrm_req_send(bp, req);
5827 	} else {
5828 		struct hwrm_func_vf_cfg_input *req;
5829 
5830 		rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
5831 		if (rc)
5832 			return rc;
5833 
5834 		req->enables =
5835 			cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5836 		req->async_event_cr = cpu_to_le16(idx);
5837 		return hwrm_req_send(bp, req);
5838 	}
5839 }
5840 
5841 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5842 			u32 map_idx, u32 xid)
5843 {
5844 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5845 		if (BNXT_PF(bp))
5846 			db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5847 		else
5848 			db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5849 		switch (ring_type) {
5850 		case HWRM_RING_ALLOC_TX:
5851 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5852 			break;
5853 		case HWRM_RING_ALLOC_RX:
5854 		case HWRM_RING_ALLOC_AGG:
5855 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5856 			break;
5857 		case HWRM_RING_ALLOC_CMPL:
5858 			db->db_key64 = DBR_PATH_L2;
5859 			break;
5860 		case HWRM_RING_ALLOC_NQ:
5861 			db->db_key64 = DBR_PATH_L2;
5862 			break;
5863 		}
5864 		db->db_key64 |= (u64)xid << DBR_XID_SFT;
5865 	} else {
5866 		db->doorbell = bp->bar1 + map_idx * 0x80;
5867 		switch (ring_type) {
5868 		case HWRM_RING_ALLOC_TX:
5869 			db->db_key32 = DB_KEY_TX;
5870 			break;
5871 		case HWRM_RING_ALLOC_RX:
5872 		case HWRM_RING_ALLOC_AGG:
5873 			db->db_key32 = DB_KEY_RX;
5874 			break;
5875 		case HWRM_RING_ALLOC_CMPL:
5876 			db->db_key32 = DB_KEY_CP;
5877 			break;
5878 		}
5879 	}
5880 }
5881 
5882 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5883 {
5884 	bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5885 	int i, rc = 0;
5886 	u32 type;
5887 
5888 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5889 		type = HWRM_RING_ALLOC_NQ;
5890 	else
5891 		type = HWRM_RING_ALLOC_CMPL;
5892 	for (i = 0; i < bp->cp_nr_rings; i++) {
5893 		struct bnxt_napi *bnapi = bp->bnapi[i];
5894 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5895 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5896 		u32 map_idx = ring->map_idx;
5897 		unsigned int vector;
5898 
5899 		vector = bp->irq_tbl[map_idx].vector;
5900 		disable_irq_nosync(vector);
5901 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5902 		if (rc) {
5903 			enable_irq(vector);
5904 			goto err_out;
5905 		}
5906 		bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5907 		bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5908 		enable_irq(vector);
5909 		bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5910 
5911 		if (!i) {
5912 			rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5913 			if (rc)
5914 				netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5915 		}
5916 	}
5917 
5918 	type = HWRM_RING_ALLOC_TX;
5919 	for (i = 0; i < bp->tx_nr_rings; i++) {
5920 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5921 		struct bnxt_ring_struct *ring;
5922 		u32 map_idx;
5923 
5924 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5925 			struct bnxt_napi *bnapi = txr->bnapi;
5926 			struct bnxt_cp_ring_info *cpr, *cpr2;
5927 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5928 
5929 			cpr = &bnapi->cp_ring;
5930 			cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5931 			ring = &cpr2->cp_ring_struct;
5932 			ring->handle = BNXT_TX_HDL;
5933 			map_idx = bnapi->index;
5934 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5935 			if (rc)
5936 				goto err_out;
5937 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5938 				    ring->fw_ring_id);
5939 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5940 		}
5941 		ring = &txr->tx_ring_struct;
5942 		map_idx = i;
5943 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5944 		if (rc)
5945 			goto err_out;
5946 		bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5947 	}
5948 
5949 	type = HWRM_RING_ALLOC_RX;
5950 	for (i = 0; i < bp->rx_nr_rings; i++) {
5951 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5952 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5953 		struct bnxt_napi *bnapi = rxr->bnapi;
5954 		u32 map_idx = bnapi->index;
5955 
5956 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5957 		if (rc)
5958 			goto err_out;
5959 		bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5960 		/* If we have agg rings, post agg buffers first. */
5961 		if (!agg_rings)
5962 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5963 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5964 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5965 			struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5966 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5967 			struct bnxt_cp_ring_info *cpr2;
5968 
5969 			cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5970 			ring = &cpr2->cp_ring_struct;
5971 			ring->handle = BNXT_RX_HDL;
5972 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5973 			if (rc)
5974 				goto err_out;
5975 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5976 				    ring->fw_ring_id);
5977 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5978 		}
5979 	}
5980 
5981 	if (agg_rings) {
5982 		type = HWRM_RING_ALLOC_AGG;
5983 		for (i = 0; i < bp->rx_nr_rings; i++) {
5984 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5985 			struct bnxt_ring_struct *ring =
5986 						&rxr->rx_agg_ring_struct;
5987 			u32 grp_idx = ring->grp_idx;
5988 			u32 map_idx = grp_idx + bp->rx_nr_rings;
5989 
5990 			rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5991 			if (rc)
5992 				goto err_out;
5993 
5994 			bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5995 				    ring->fw_ring_id);
5996 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5997 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5998 			bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5999 		}
6000 	}
6001 err_out:
6002 	return rc;
6003 }
6004 
6005 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6006 				   struct bnxt_ring_struct *ring,
6007 				   u32 ring_type, int cmpl_ring_id)
6008 {
6009 	struct hwrm_ring_free_output *resp;
6010 	struct hwrm_ring_free_input *req;
6011 	u16 error_code = 0;
6012 	int rc;
6013 
6014 	if (BNXT_NO_FW_ACCESS(bp))
6015 		return 0;
6016 
6017 	rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6018 	if (rc)
6019 		goto exit;
6020 
6021 	req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6022 	req->ring_type = ring_type;
6023 	req->ring_id = cpu_to_le16(ring->fw_ring_id);
6024 
6025 	resp = hwrm_req_hold(bp, req);
6026 	rc = hwrm_req_send(bp, req);
6027 	error_code = le16_to_cpu(resp->error_code);
6028 	hwrm_req_drop(bp, req);
6029 exit:
6030 	if (rc || error_code) {
6031 		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6032 			   ring_type, rc, error_code);
6033 		return -EIO;
6034 	}
6035 	return 0;
6036 }
6037 
6038 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6039 {
6040 	u32 type;
6041 	int i;
6042 
6043 	if (!bp->bnapi)
6044 		return;
6045 
6046 	for (i = 0; i < bp->tx_nr_rings; i++) {
6047 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6048 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6049 
6050 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6051 			u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6052 
6053 			hwrm_ring_free_send_msg(bp, ring,
6054 						RING_FREE_REQ_RING_TYPE_TX,
6055 						close_path ? cmpl_ring_id :
6056 						INVALID_HW_RING_ID);
6057 			ring->fw_ring_id = INVALID_HW_RING_ID;
6058 		}
6059 	}
6060 
6061 	for (i = 0; i < bp->rx_nr_rings; i++) {
6062 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6063 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6064 		u32 grp_idx = rxr->bnapi->index;
6065 
6066 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6067 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6068 
6069 			hwrm_ring_free_send_msg(bp, ring,
6070 						RING_FREE_REQ_RING_TYPE_RX,
6071 						close_path ? cmpl_ring_id :
6072 						INVALID_HW_RING_ID);
6073 			ring->fw_ring_id = INVALID_HW_RING_ID;
6074 			bp->grp_info[grp_idx].rx_fw_ring_id =
6075 				INVALID_HW_RING_ID;
6076 		}
6077 	}
6078 
6079 	if (bp->flags & BNXT_FLAG_CHIP_P5)
6080 		type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6081 	else
6082 		type = RING_FREE_REQ_RING_TYPE_RX;
6083 	for (i = 0; i < bp->rx_nr_rings; i++) {
6084 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6085 		struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6086 		u32 grp_idx = rxr->bnapi->index;
6087 
6088 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6089 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6090 
6091 			hwrm_ring_free_send_msg(bp, ring, type,
6092 						close_path ? cmpl_ring_id :
6093 						INVALID_HW_RING_ID);
6094 			ring->fw_ring_id = INVALID_HW_RING_ID;
6095 			bp->grp_info[grp_idx].agg_fw_ring_id =
6096 				INVALID_HW_RING_ID;
6097 		}
6098 	}
6099 
6100 	/* The completion rings are about to be freed.  After that the
6101 	 * IRQ doorbell will not work anymore.  So we need to disable
6102 	 * IRQ here.
6103 	 */
6104 	bnxt_disable_int_sync(bp);
6105 
6106 	if (bp->flags & BNXT_FLAG_CHIP_P5)
6107 		type = RING_FREE_REQ_RING_TYPE_NQ;
6108 	else
6109 		type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6110 	for (i = 0; i < bp->cp_nr_rings; i++) {
6111 		struct bnxt_napi *bnapi = bp->bnapi[i];
6112 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6113 		struct bnxt_ring_struct *ring;
6114 		int j;
6115 
6116 		for (j = 0; j < 2; j++) {
6117 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
6118 
6119 			if (cpr2) {
6120 				ring = &cpr2->cp_ring_struct;
6121 				if (ring->fw_ring_id == INVALID_HW_RING_ID)
6122 					continue;
6123 				hwrm_ring_free_send_msg(bp, ring,
6124 					RING_FREE_REQ_RING_TYPE_L2_CMPL,
6125 					INVALID_HW_RING_ID);
6126 				ring->fw_ring_id = INVALID_HW_RING_ID;
6127 			}
6128 		}
6129 		ring = &cpr->cp_ring_struct;
6130 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6131 			hwrm_ring_free_send_msg(bp, ring, type,
6132 						INVALID_HW_RING_ID);
6133 			ring->fw_ring_id = INVALID_HW_RING_ID;
6134 			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6135 		}
6136 	}
6137 }
6138 
6139 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6140 			   bool shared);
6141 
6142 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6143 {
6144 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6145 	struct hwrm_func_qcfg_output *resp;
6146 	struct hwrm_func_qcfg_input *req;
6147 	int rc;
6148 
6149 	if (bp->hwrm_spec_code < 0x10601)
6150 		return 0;
6151 
6152 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6153 	if (rc)
6154 		return rc;
6155 
6156 	req->fid = cpu_to_le16(0xffff);
6157 	resp = hwrm_req_hold(bp, req);
6158 	rc = hwrm_req_send(bp, req);
6159 	if (rc) {
6160 		hwrm_req_drop(bp, req);
6161 		return rc;
6162 	}
6163 
6164 	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6165 	if (BNXT_NEW_RM(bp)) {
6166 		u16 cp, stats;
6167 
6168 		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6169 		hw_resc->resv_hw_ring_grps =
6170 			le32_to_cpu(resp->alloc_hw_ring_grps);
6171 		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6172 		cp = le16_to_cpu(resp->alloc_cmpl_rings);
6173 		stats = le16_to_cpu(resp->alloc_stat_ctx);
6174 		hw_resc->resv_irqs = cp;
6175 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6176 			int rx = hw_resc->resv_rx_rings;
6177 			int tx = hw_resc->resv_tx_rings;
6178 
6179 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
6180 				rx >>= 1;
6181 			if (cp < (rx + tx)) {
6182 				bnxt_trim_rings(bp, &rx, &tx, cp, false);
6183 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
6184 					rx <<= 1;
6185 				hw_resc->resv_rx_rings = rx;
6186 				hw_resc->resv_tx_rings = tx;
6187 			}
6188 			hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6189 			hw_resc->resv_hw_ring_grps = rx;
6190 		}
6191 		hw_resc->resv_cp_rings = cp;
6192 		hw_resc->resv_stat_ctxs = stats;
6193 	}
6194 	hwrm_req_drop(bp, req);
6195 	return 0;
6196 }
6197 
6198 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6199 {
6200 	struct hwrm_func_qcfg_output *resp;
6201 	struct hwrm_func_qcfg_input *req;
6202 	int rc;
6203 
6204 	if (bp->hwrm_spec_code < 0x10601)
6205 		return 0;
6206 
6207 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6208 	if (rc)
6209 		return rc;
6210 
6211 	req->fid = cpu_to_le16(fid);
6212 	resp = hwrm_req_hold(bp, req);
6213 	rc = hwrm_req_send(bp, req);
6214 	if (!rc)
6215 		*tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6216 
6217 	hwrm_req_drop(bp, req);
6218 	return rc;
6219 }
6220 
6221 static bool bnxt_rfs_supported(struct bnxt *bp);
6222 
6223 static struct hwrm_func_cfg_input *
6224 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6225 			     int ring_grps, int cp_rings, int stats, int vnics)
6226 {
6227 	struct hwrm_func_cfg_input *req;
6228 	u32 enables = 0;
6229 
6230 	if (hwrm_req_init(bp, req, HWRM_FUNC_CFG))
6231 		return NULL;
6232 
6233 	req->fid = cpu_to_le16(0xffff);
6234 	enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6235 	req->num_tx_rings = cpu_to_le16(tx_rings);
6236 	if (BNXT_NEW_RM(bp)) {
6237 		enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6238 		enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6239 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6240 			enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6241 			enables |= tx_rings + ring_grps ?
6242 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6243 			enables |= rx_rings ?
6244 				FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6245 		} else {
6246 			enables |= cp_rings ?
6247 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6248 			enables |= ring_grps ?
6249 				   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6250 				   FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6251 		}
6252 		enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6253 
6254 		req->num_rx_rings = cpu_to_le16(rx_rings);
6255 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6256 			req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6257 			req->num_msix = cpu_to_le16(cp_rings);
6258 			req->num_rsscos_ctxs =
6259 				cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6260 		} else {
6261 			req->num_cmpl_rings = cpu_to_le16(cp_rings);
6262 			req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6263 			req->num_rsscos_ctxs = cpu_to_le16(1);
6264 			if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6265 			    bnxt_rfs_supported(bp))
6266 				req->num_rsscos_ctxs =
6267 					cpu_to_le16(ring_grps + 1);
6268 		}
6269 		req->num_stat_ctxs = cpu_to_le16(stats);
6270 		req->num_vnics = cpu_to_le16(vnics);
6271 	}
6272 	req->enables = cpu_to_le32(enables);
6273 	return req;
6274 }
6275 
6276 static struct hwrm_func_vf_cfg_input *
6277 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6278 			     int ring_grps, int cp_rings, int stats, int vnics)
6279 {
6280 	struct hwrm_func_vf_cfg_input *req;
6281 	u32 enables = 0;
6282 
6283 	if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
6284 		return NULL;
6285 
6286 	enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6287 	enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6288 			      FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6289 	enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6290 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6291 		enables |= tx_rings + ring_grps ?
6292 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6293 	} else {
6294 		enables |= cp_rings ?
6295 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6296 		enables |= ring_grps ?
6297 			   FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6298 	}
6299 	enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6300 	enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6301 
6302 	req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6303 	req->num_tx_rings = cpu_to_le16(tx_rings);
6304 	req->num_rx_rings = cpu_to_le16(rx_rings);
6305 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6306 		req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6307 		req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6308 	} else {
6309 		req->num_cmpl_rings = cpu_to_le16(cp_rings);
6310 		req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6311 		req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6312 	}
6313 	req->num_stat_ctxs = cpu_to_le16(stats);
6314 	req->num_vnics = cpu_to_le16(vnics);
6315 
6316 	req->enables = cpu_to_le32(enables);
6317 	return req;
6318 }
6319 
6320 static int
6321 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6322 			   int ring_grps, int cp_rings, int stats, int vnics)
6323 {
6324 	struct hwrm_func_cfg_input *req;
6325 	int rc;
6326 
6327 	req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6328 					   cp_rings, stats, vnics);
6329 	if (!req)
6330 		return -ENOMEM;
6331 
6332 	if (!req->enables) {
6333 		hwrm_req_drop(bp, req);
6334 		return 0;
6335 	}
6336 
6337 	rc = hwrm_req_send(bp, req);
6338 	if (rc)
6339 		return rc;
6340 
6341 	if (bp->hwrm_spec_code < 0x10601)
6342 		bp->hw_resc.resv_tx_rings = tx_rings;
6343 
6344 	return bnxt_hwrm_get_rings(bp);
6345 }
6346 
6347 static int
6348 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6349 			   int ring_grps, int cp_rings, int stats, int vnics)
6350 {
6351 	struct hwrm_func_vf_cfg_input *req;
6352 	int rc;
6353 
6354 	if (!BNXT_NEW_RM(bp)) {
6355 		bp->hw_resc.resv_tx_rings = tx_rings;
6356 		return 0;
6357 	}
6358 
6359 	req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6360 					   cp_rings, stats, vnics);
6361 	if (!req)
6362 		return -ENOMEM;
6363 
6364 	rc = hwrm_req_send(bp, req);
6365 	if (rc)
6366 		return rc;
6367 
6368 	return bnxt_hwrm_get_rings(bp);
6369 }
6370 
6371 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6372 				   int cp, int stat, int vnic)
6373 {
6374 	if (BNXT_PF(bp))
6375 		return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6376 						  vnic);
6377 	else
6378 		return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6379 						  vnic);
6380 }
6381 
6382 int bnxt_nq_rings_in_use(struct bnxt *bp)
6383 {
6384 	int cp = bp->cp_nr_rings;
6385 	int ulp_msix, ulp_base;
6386 
6387 	ulp_msix = bnxt_get_ulp_msix_num(bp);
6388 	if (ulp_msix) {
6389 		ulp_base = bnxt_get_ulp_msix_base(bp);
6390 		cp += ulp_msix;
6391 		if ((ulp_base + ulp_msix) > cp)
6392 			cp = ulp_base + ulp_msix;
6393 	}
6394 	return cp;
6395 }
6396 
6397 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6398 {
6399 	int cp;
6400 
6401 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6402 		return bnxt_nq_rings_in_use(bp);
6403 
6404 	cp = bp->tx_nr_rings + bp->rx_nr_rings;
6405 	return cp;
6406 }
6407 
6408 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6409 {
6410 	int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6411 	int cp = bp->cp_nr_rings;
6412 
6413 	if (!ulp_stat)
6414 		return cp;
6415 
6416 	if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6417 		return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6418 
6419 	return cp + ulp_stat;
6420 }
6421 
6422 /* Check if a default RSS map needs to be setup.  This function is only
6423  * used on older firmware that does not require reserving RX rings.
6424  */
6425 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6426 {
6427 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6428 
6429 	/* The RSS map is valid for RX rings set to resv_rx_rings */
6430 	if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6431 		hw_resc->resv_rx_rings = bp->rx_nr_rings;
6432 		if (!netif_is_rxfh_configured(bp->dev))
6433 			bnxt_set_dflt_rss_indir_tbl(bp);
6434 	}
6435 }
6436 
6437 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6438 {
6439 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6440 	int cp = bnxt_cp_rings_in_use(bp);
6441 	int nq = bnxt_nq_rings_in_use(bp);
6442 	int rx = bp->rx_nr_rings, stat;
6443 	int vnic = 1, grp = rx;
6444 
6445 	if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6446 	    bp->hwrm_spec_code >= 0x10601)
6447 		return true;
6448 
6449 	/* Old firmware does not need RX ring reservations but we still
6450 	 * need to setup a default RSS map when needed.  With new firmware
6451 	 * we go through RX ring reservations first and then set up the
6452 	 * RSS map for the successfully reserved RX rings when needed.
6453 	 */
6454 	if (!BNXT_NEW_RM(bp)) {
6455 		bnxt_check_rss_tbl_no_rmgr(bp);
6456 		return false;
6457 	}
6458 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6459 		vnic = rx + 1;
6460 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6461 		rx <<= 1;
6462 	stat = bnxt_get_func_stat_ctxs(bp);
6463 	if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6464 	    hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6465 	    (hw_resc->resv_hw_ring_grps != grp &&
6466 	     !(bp->flags & BNXT_FLAG_CHIP_P5)))
6467 		return true;
6468 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6469 	    hw_resc->resv_irqs != nq)
6470 		return true;
6471 	return false;
6472 }
6473 
6474 static int __bnxt_reserve_rings(struct bnxt *bp)
6475 {
6476 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6477 	int cp = bnxt_nq_rings_in_use(bp);
6478 	int tx = bp->tx_nr_rings;
6479 	int rx = bp->rx_nr_rings;
6480 	int grp, rx_rings, rc;
6481 	int vnic = 1, stat;
6482 	bool sh = false;
6483 
6484 	if (!bnxt_need_reserve_rings(bp))
6485 		return 0;
6486 
6487 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6488 		sh = true;
6489 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6490 		vnic = rx + 1;
6491 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6492 		rx <<= 1;
6493 	grp = bp->rx_nr_rings;
6494 	stat = bnxt_get_func_stat_ctxs(bp);
6495 
6496 	rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6497 	if (rc)
6498 		return rc;
6499 
6500 	tx = hw_resc->resv_tx_rings;
6501 	if (BNXT_NEW_RM(bp)) {
6502 		rx = hw_resc->resv_rx_rings;
6503 		cp = hw_resc->resv_irqs;
6504 		grp = hw_resc->resv_hw_ring_grps;
6505 		vnic = hw_resc->resv_vnics;
6506 		stat = hw_resc->resv_stat_ctxs;
6507 	}
6508 
6509 	rx_rings = rx;
6510 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6511 		if (rx >= 2) {
6512 			rx_rings = rx >> 1;
6513 		} else {
6514 			if (netif_running(bp->dev))
6515 				return -ENOMEM;
6516 
6517 			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6518 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6519 			bp->dev->hw_features &= ~NETIF_F_LRO;
6520 			bp->dev->features &= ~NETIF_F_LRO;
6521 			bnxt_set_ring_params(bp);
6522 		}
6523 	}
6524 	rx_rings = min_t(int, rx_rings, grp);
6525 	cp = min_t(int, cp, bp->cp_nr_rings);
6526 	if (stat > bnxt_get_ulp_stat_ctxs(bp))
6527 		stat -= bnxt_get_ulp_stat_ctxs(bp);
6528 	cp = min_t(int, cp, stat);
6529 	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6530 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6531 		rx = rx_rings << 1;
6532 	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6533 	bp->tx_nr_rings = tx;
6534 
6535 	/* If we cannot reserve all the RX rings, reset the RSS map only
6536 	 * if absolutely necessary
6537 	 */
6538 	if (rx_rings != bp->rx_nr_rings) {
6539 		netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6540 			    rx_rings, bp->rx_nr_rings);
6541 		if (netif_is_rxfh_configured(bp->dev) &&
6542 		    (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6543 		     bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6544 		     bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6545 			netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6546 			bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6547 		}
6548 	}
6549 	bp->rx_nr_rings = rx_rings;
6550 	bp->cp_nr_rings = cp;
6551 
6552 	if (!tx || !rx || !cp || !grp || !vnic || !stat)
6553 		return -ENOMEM;
6554 
6555 	if (!netif_is_rxfh_configured(bp->dev))
6556 		bnxt_set_dflt_rss_indir_tbl(bp);
6557 
6558 	return rc;
6559 }
6560 
6561 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6562 				    int ring_grps, int cp_rings, int stats,
6563 				    int vnics)
6564 {
6565 	struct hwrm_func_vf_cfg_input *req;
6566 	u32 flags;
6567 
6568 	if (!BNXT_NEW_RM(bp))
6569 		return 0;
6570 
6571 	req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6572 					   cp_rings, stats, vnics);
6573 	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6574 		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6575 		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6576 		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6577 		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6578 		FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6579 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6580 		flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6581 
6582 	req->flags = cpu_to_le32(flags);
6583 	return hwrm_req_send_silent(bp, req);
6584 }
6585 
6586 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6587 				    int ring_grps, int cp_rings, int stats,
6588 				    int vnics)
6589 {
6590 	struct hwrm_func_cfg_input *req;
6591 	u32 flags;
6592 
6593 	req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6594 					   cp_rings, stats, vnics);
6595 	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6596 	if (BNXT_NEW_RM(bp)) {
6597 		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6598 			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6599 			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6600 			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6601 		if (bp->flags & BNXT_FLAG_CHIP_P5)
6602 			flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6603 				 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6604 		else
6605 			flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6606 	}
6607 
6608 	req->flags = cpu_to_le32(flags);
6609 	return hwrm_req_send_silent(bp, req);
6610 }
6611 
6612 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6613 				 int ring_grps, int cp_rings, int stats,
6614 				 int vnics)
6615 {
6616 	if (bp->hwrm_spec_code < 0x10801)
6617 		return 0;
6618 
6619 	if (BNXT_PF(bp))
6620 		return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6621 						ring_grps, cp_rings, stats,
6622 						vnics);
6623 
6624 	return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6625 					cp_rings, stats, vnics);
6626 }
6627 
6628 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6629 {
6630 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6631 	struct hwrm_ring_aggint_qcaps_output *resp;
6632 	struct hwrm_ring_aggint_qcaps_input *req;
6633 	int rc;
6634 
6635 	coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6636 	coal_cap->num_cmpl_dma_aggr_max = 63;
6637 	coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6638 	coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6639 	coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6640 	coal_cap->int_lat_tmr_min_max = 65535;
6641 	coal_cap->int_lat_tmr_max_max = 65535;
6642 	coal_cap->num_cmpl_aggr_int_max = 65535;
6643 	coal_cap->timer_units = 80;
6644 
6645 	if (bp->hwrm_spec_code < 0x10902)
6646 		return;
6647 
6648 	if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
6649 		return;
6650 
6651 	resp = hwrm_req_hold(bp, req);
6652 	rc = hwrm_req_send_silent(bp, req);
6653 	if (!rc) {
6654 		coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6655 		coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6656 		coal_cap->num_cmpl_dma_aggr_max =
6657 			le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6658 		coal_cap->num_cmpl_dma_aggr_during_int_max =
6659 			le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6660 		coal_cap->cmpl_aggr_dma_tmr_max =
6661 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6662 		coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6663 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6664 		coal_cap->int_lat_tmr_min_max =
6665 			le16_to_cpu(resp->int_lat_tmr_min_max);
6666 		coal_cap->int_lat_tmr_max_max =
6667 			le16_to_cpu(resp->int_lat_tmr_max_max);
6668 		coal_cap->num_cmpl_aggr_int_max =
6669 			le16_to_cpu(resp->num_cmpl_aggr_int_max);
6670 		coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6671 	}
6672 	hwrm_req_drop(bp, req);
6673 }
6674 
6675 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6676 {
6677 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6678 
6679 	return usec * 1000 / coal_cap->timer_units;
6680 }
6681 
6682 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6683 	struct bnxt_coal *hw_coal,
6684 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6685 {
6686 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6687 	u16 val, tmr, max, flags = hw_coal->flags;
6688 	u32 cmpl_params = coal_cap->cmpl_params;
6689 
6690 	max = hw_coal->bufs_per_record * 128;
6691 	if (hw_coal->budget)
6692 		max = hw_coal->bufs_per_record * hw_coal->budget;
6693 	max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6694 
6695 	val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6696 	req->num_cmpl_aggr_int = cpu_to_le16(val);
6697 
6698 	val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6699 	req->num_cmpl_dma_aggr = cpu_to_le16(val);
6700 
6701 	val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6702 		      coal_cap->num_cmpl_dma_aggr_during_int_max);
6703 	req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6704 
6705 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6706 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6707 	req->int_lat_tmr_max = cpu_to_le16(tmr);
6708 
6709 	/* min timer set to 1/2 of interrupt timer */
6710 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6711 		val = tmr / 2;
6712 		val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6713 		req->int_lat_tmr_min = cpu_to_le16(val);
6714 		req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6715 	}
6716 
6717 	/* buf timer set to 1/4 of interrupt timer */
6718 	val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6719 	req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6720 
6721 	if (cmpl_params &
6722 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6723 		tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6724 		val = clamp_t(u16, tmr, 1,
6725 			      coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6726 		req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6727 		req->enables |=
6728 			cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6729 	}
6730 
6731 	if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6732 	    hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6733 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6734 	req->flags = cpu_to_le16(flags);
6735 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6736 }
6737 
6738 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6739 				   struct bnxt_coal *hw_coal)
6740 {
6741 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
6742 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6743 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6744 	u32 nq_params = coal_cap->nq_params;
6745 	u16 tmr;
6746 	int rc;
6747 
6748 	if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6749 		return 0;
6750 
6751 	rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6752 	if (rc)
6753 		return rc;
6754 
6755 	req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6756 	req->flags =
6757 		cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6758 
6759 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6760 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6761 	req->int_lat_tmr_min = cpu_to_le16(tmr);
6762 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6763 	return hwrm_req_send(bp, req);
6764 }
6765 
6766 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6767 {
6768 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
6769 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6770 	struct bnxt_coal coal;
6771 	int rc;
6772 
6773 	/* Tick values in micro seconds.
6774 	 * 1 coal_buf x bufs_per_record = 1 completion record.
6775 	 */
6776 	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6777 
6778 	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6779 	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6780 
6781 	if (!bnapi->rx_ring)
6782 		return -ENODEV;
6783 
6784 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6785 	if (rc)
6786 		return rc;
6787 
6788 	bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
6789 
6790 	req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6791 
6792 	return hwrm_req_send(bp, req_rx);
6793 }
6794 
6795 int bnxt_hwrm_set_coal(struct bnxt *bp)
6796 {
6797 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
6798 							   *req;
6799 	int i, rc;
6800 
6801 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6802 	if (rc)
6803 		return rc;
6804 
6805 	rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6806 	if (rc) {
6807 		hwrm_req_drop(bp, req_rx);
6808 		return rc;
6809 	}
6810 
6811 	bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
6812 	bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
6813 
6814 	hwrm_req_hold(bp, req_rx);
6815 	hwrm_req_hold(bp, req_tx);
6816 	for (i = 0; i < bp->cp_nr_rings; i++) {
6817 		struct bnxt_napi *bnapi = bp->bnapi[i];
6818 		struct bnxt_coal *hw_coal;
6819 		u16 ring_id;
6820 
6821 		req = req_rx;
6822 		if (!bnapi->rx_ring) {
6823 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6824 			req = req_tx;
6825 		} else {
6826 			ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6827 		}
6828 		req->ring_id = cpu_to_le16(ring_id);
6829 
6830 		rc = hwrm_req_send(bp, req);
6831 		if (rc)
6832 			break;
6833 
6834 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6835 			continue;
6836 
6837 		if (bnapi->rx_ring && bnapi->tx_ring) {
6838 			req = req_tx;
6839 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6840 			req->ring_id = cpu_to_le16(ring_id);
6841 			rc = hwrm_req_send(bp, req);
6842 			if (rc)
6843 				break;
6844 		}
6845 		if (bnapi->rx_ring)
6846 			hw_coal = &bp->rx_coal;
6847 		else
6848 			hw_coal = &bp->tx_coal;
6849 		__bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6850 	}
6851 	hwrm_req_drop(bp, req_rx);
6852 	hwrm_req_drop(bp, req_tx);
6853 	return rc;
6854 }
6855 
6856 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6857 {
6858 	struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
6859 	struct hwrm_stat_ctx_free_input *req;
6860 	int i;
6861 
6862 	if (!bp->bnapi)
6863 		return;
6864 
6865 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6866 		return;
6867 
6868 	if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
6869 		return;
6870 	if (BNXT_FW_MAJ(bp) <= 20) {
6871 		if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
6872 			hwrm_req_drop(bp, req);
6873 			return;
6874 		}
6875 		hwrm_req_hold(bp, req0);
6876 	}
6877 	hwrm_req_hold(bp, req);
6878 	for (i = 0; i < bp->cp_nr_rings; i++) {
6879 		struct bnxt_napi *bnapi = bp->bnapi[i];
6880 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6881 
6882 		if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6883 			req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6884 			if (req0) {
6885 				req0->stat_ctx_id = req->stat_ctx_id;
6886 				hwrm_req_send(bp, req0);
6887 			}
6888 			hwrm_req_send(bp, req);
6889 
6890 			cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6891 		}
6892 	}
6893 	hwrm_req_drop(bp, req);
6894 	if (req0)
6895 		hwrm_req_drop(bp, req0);
6896 }
6897 
6898 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6899 {
6900 	struct hwrm_stat_ctx_alloc_output *resp;
6901 	struct hwrm_stat_ctx_alloc_input *req;
6902 	int rc, i;
6903 
6904 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6905 		return 0;
6906 
6907 	rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
6908 	if (rc)
6909 		return rc;
6910 
6911 	req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6912 	req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6913 
6914 	resp = hwrm_req_hold(bp, req);
6915 	for (i = 0; i < bp->cp_nr_rings; i++) {
6916 		struct bnxt_napi *bnapi = bp->bnapi[i];
6917 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6918 
6919 		req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6920 
6921 		rc = hwrm_req_send(bp, req);
6922 		if (rc)
6923 			break;
6924 
6925 		cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6926 
6927 		bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6928 	}
6929 	hwrm_req_drop(bp, req);
6930 	return rc;
6931 }
6932 
6933 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6934 {
6935 	struct hwrm_func_qcfg_output *resp;
6936 	struct hwrm_func_qcfg_input *req;
6937 	u32 min_db_offset = 0;
6938 	u16 flags;
6939 	int rc;
6940 
6941 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6942 	if (rc)
6943 		return rc;
6944 
6945 	req->fid = cpu_to_le16(0xffff);
6946 	resp = hwrm_req_hold(bp, req);
6947 	rc = hwrm_req_send(bp, req);
6948 	if (rc)
6949 		goto func_qcfg_exit;
6950 
6951 #ifdef CONFIG_BNXT_SRIOV
6952 	if (BNXT_VF(bp)) {
6953 		struct bnxt_vf_info *vf = &bp->vf;
6954 
6955 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6956 	} else {
6957 		bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6958 	}
6959 #endif
6960 	flags = le16_to_cpu(resp->flags);
6961 	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6962 		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6963 		bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6964 		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6965 			bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6966 	}
6967 	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6968 		bp->flags |= BNXT_FLAG_MULTI_HOST;
6969 
6970 	if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6971 		bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6972 
6973 	switch (resp->port_partition_type) {
6974 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6975 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6976 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6977 		bp->port_partition_type = resp->port_partition_type;
6978 		break;
6979 	}
6980 	if (bp->hwrm_spec_code < 0x10707 ||
6981 	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6982 		bp->br_mode = BRIDGE_MODE_VEB;
6983 	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6984 		bp->br_mode = BRIDGE_MODE_VEPA;
6985 	else
6986 		bp->br_mode = BRIDGE_MODE_UNDEF;
6987 
6988 	bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6989 	if (!bp->max_mtu)
6990 		bp->max_mtu = BNXT_MAX_MTU;
6991 
6992 	if (bp->db_size)
6993 		goto func_qcfg_exit;
6994 
6995 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6996 		if (BNXT_PF(bp))
6997 			min_db_offset = DB_PF_OFFSET_P5;
6998 		else
6999 			min_db_offset = DB_VF_OFFSET_P5;
7000 	}
7001 	bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7002 				 1024);
7003 	if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7004 	    bp->db_size <= min_db_offset)
7005 		bp->db_size = pci_resource_len(bp->pdev, 2);
7006 
7007 func_qcfg_exit:
7008 	hwrm_req_drop(bp, req);
7009 	return rc;
7010 }
7011 
7012 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
7013 			struct hwrm_func_backing_store_qcaps_output *resp)
7014 {
7015 	struct bnxt_mem_init *mem_init;
7016 	u16 init_mask;
7017 	u8 init_val;
7018 	u8 *offset;
7019 	int i;
7020 
7021 	init_val = resp->ctx_kind_initializer;
7022 	init_mask = le16_to_cpu(resp->ctx_init_mask);
7023 	offset = &resp->qp_init_offset;
7024 	mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7025 	for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
7026 		mem_init->init_val = init_val;
7027 		mem_init->offset = BNXT_MEM_INVALID_OFFSET;
7028 		if (!init_mask)
7029 			continue;
7030 		if (i == BNXT_CTX_MEM_INIT_STAT)
7031 			offset = &resp->stat_init_offset;
7032 		if (init_mask & (1 << i))
7033 			mem_init->offset = *offset * 4;
7034 		else
7035 			mem_init->init_val = 0;
7036 	}
7037 	ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
7038 	ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
7039 	ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
7040 	ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
7041 	ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
7042 	ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
7043 }
7044 
7045 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7046 {
7047 	struct hwrm_func_backing_store_qcaps_output *resp;
7048 	struct hwrm_func_backing_store_qcaps_input *req;
7049 	int rc;
7050 
7051 	if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7052 		return 0;
7053 
7054 	rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7055 	if (rc)
7056 		return rc;
7057 
7058 	resp = hwrm_req_hold(bp, req);
7059 	rc = hwrm_req_send_silent(bp, req);
7060 	if (!rc) {
7061 		struct bnxt_ctx_pg_info *ctx_pg;
7062 		struct bnxt_ctx_mem_info *ctx;
7063 		int i, tqm_rings;
7064 
7065 		ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7066 		if (!ctx) {
7067 			rc = -ENOMEM;
7068 			goto ctx_err;
7069 		}
7070 		ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
7071 		ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7072 		ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7073 		ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
7074 		ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7075 		ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
7076 		ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
7077 		ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7078 		ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
7079 		ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
7080 		ctx->vnic_max_vnic_entries =
7081 			le16_to_cpu(resp->vnic_max_vnic_entries);
7082 		ctx->vnic_max_ring_table_entries =
7083 			le16_to_cpu(resp->vnic_max_ring_table_entries);
7084 		ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
7085 		ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
7086 		ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
7087 		ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
7088 		ctx->tqm_min_entries_per_ring =
7089 			le32_to_cpu(resp->tqm_min_entries_per_ring);
7090 		ctx->tqm_max_entries_per_ring =
7091 			le32_to_cpu(resp->tqm_max_entries_per_ring);
7092 		ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
7093 		if (!ctx->tqm_entries_multiple)
7094 			ctx->tqm_entries_multiple = 1;
7095 		ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
7096 		ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
7097 		ctx->mrav_num_entries_units =
7098 			le16_to_cpu(resp->mrav_num_entries_units);
7099 		ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
7100 		ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
7101 
7102 		bnxt_init_ctx_initializer(ctx, resp);
7103 
7104 		ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7105 		if (!ctx->tqm_fp_rings_count)
7106 			ctx->tqm_fp_rings_count = bp->max_q;
7107 		else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7108 			ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7109 
7110 		tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
7111 		ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
7112 		if (!ctx_pg) {
7113 			kfree(ctx);
7114 			rc = -ENOMEM;
7115 			goto ctx_err;
7116 		}
7117 		for (i = 0; i < tqm_rings; i++, ctx_pg++)
7118 			ctx->tqm_mem[i] = ctx_pg;
7119 		bp->ctx = ctx;
7120 	} else {
7121 		rc = 0;
7122 	}
7123 ctx_err:
7124 	hwrm_req_drop(bp, req);
7125 	return rc;
7126 }
7127 
7128 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
7129 				  __le64 *pg_dir)
7130 {
7131 	if (!rmem->nr_pages)
7132 		return;
7133 
7134 	BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
7135 	if (rmem->depth >= 1) {
7136 		if (rmem->depth == 2)
7137 			*pg_attr |= 2;
7138 		else
7139 			*pg_attr |= 1;
7140 		*pg_dir = cpu_to_le64(rmem->pg_tbl_map);
7141 	} else {
7142 		*pg_dir = cpu_to_le64(rmem->dma_arr[0]);
7143 	}
7144 }
7145 
7146 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
7147 	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
7148 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
7149 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
7150 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
7151 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
7152 
7153 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
7154 {
7155 	struct hwrm_func_backing_store_cfg_input *req;
7156 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
7157 	struct bnxt_ctx_pg_info *ctx_pg;
7158 	void **__req = (void **)&req;
7159 	u32 req_len = sizeof(*req);
7160 	__le32 *num_entries;
7161 	__le64 *pg_dir;
7162 	u32 flags = 0;
7163 	u8 *pg_attr;
7164 	u32 ena;
7165 	int rc;
7166 	int i;
7167 
7168 	if (!ctx)
7169 		return 0;
7170 
7171 	if (req_len > bp->hwrm_max_ext_req_len)
7172 		req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
7173 	rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
7174 	if (rc)
7175 		return rc;
7176 
7177 	req->enables = cpu_to_le32(enables);
7178 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
7179 		ctx_pg = &ctx->qp_mem;
7180 		req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
7181 		req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
7182 		req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
7183 		req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
7184 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7185 				      &req->qpc_pg_size_qpc_lvl,
7186 				      &req->qpc_page_dir);
7187 	}
7188 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
7189 		ctx_pg = &ctx->srq_mem;
7190 		req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
7191 		req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
7192 		req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
7193 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7194 				      &req->srq_pg_size_srq_lvl,
7195 				      &req->srq_page_dir);
7196 	}
7197 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7198 		ctx_pg = &ctx->cq_mem;
7199 		req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
7200 		req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7201 		req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7202 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7203 				      &req->cq_pg_size_cq_lvl,
7204 				      &req->cq_page_dir);
7205 	}
7206 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7207 		ctx_pg = &ctx->vnic_mem;
7208 		req->vnic_num_vnic_entries =
7209 			cpu_to_le16(ctx->vnic_max_vnic_entries);
7210 		req->vnic_num_ring_table_entries =
7211 			cpu_to_le16(ctx->vnic_max_ring_table_entries);
7212 		req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7213 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7214 				      &req->vnic_pg_size_vnic_lvl,
7215 				      &req->vnic_page_dir);
7216 	}
7217 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7218 		ctx_pg = &ctx->stat_mem;
7219 		req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7220 		req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7221 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7222 				      &req->stat_pg_size_stat_lvl,
7223 				      &req->stat_page_dir);
7224 	}
7225 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7226 		ctx_pg = &ctx->mrav_mem;
7227 		req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7228 		if (ctx->mrav_num_entries_units)
7229 			flags |=
7230 			FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7231 		req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7232 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7233 				      &req->mrav_pg_size_mrav_lvl,
7234 				      &req->mrav_page_dir);
7235 	}
7236 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7237 		ctx_pg = &ctx->tim_mem;
7238 		req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
7239 		req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7240 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7241 				      &req->tim_pg_size_tim_lvl,
7242 				      &req->tim_page_dir);
7243 	}
7244 	for (i = 0, num_entries = &req->tqm_sp_num_entries,
7245 	     pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
7246 	     pg_dir = &req->tqm_sp_page_dir,
7247 	     ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7248 	     i < BNXT_MAX_TQM_RINGS;
7249 	     i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7250 		if (!(enables & ena))
7251 			continue;
7252 
7253 		req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7254 		ctx_pg = ctx->tqm_mem[i];
7255 		*num_entries = cpu_to_le32(ctx_pg->entries);
7256 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7257 	}
7258 	req->flags = cpu_to_le32(flags);
7259 	return hwrm_req_send(bp, req);
7260 }
7261 
7262 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7263 				  struct bnxt_ctx_pg_info *ctx_pg)
7264 {
7265 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7266 
7267 	rmem->page_size = BNXT_PAGE_SIZE;
7268 	rmem->pg_arr = ctx_pg->ctx_pg_arr;
7269 	rmem->dma_arr = ctx_pg->ctx_dma_arr;
7270 	rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7271 	if (rmem->depth >= 1)
7272 		rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7273 	return bnxt_alloc_ring(bp, rmem);
7274 }
7275 
7276 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7277 				  struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7278 				  u8 depth, struct bnxt_mem_init *mem_init)
7279 {
7280 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7281 	int rc;
7282 
7283 	if (!mem_size)
7284 		return -EINVAL;
7285 
7286 	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7287 	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7288 		ctx_pg->nr_pages = 0;
7289 		return -EINVAL;
7290 	}
7291 	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7292 		int nr_tbls, i;
7293 
7294 		rmem->depth = 2;
7295 		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7296 					     GFP_KERNEL);
7297 		if (!ctx_pg->ctx_pg_tbl)
7298 			return -ENOMEM;
7299 		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7300 		rmem->nr_pages = nr_tbls;
7301 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7302 		if (rc)
7303 			return rc;
7304 		for (i = 0; i < nr_tbls; i++) {
7305 			struct bnxt_ctx_pg_info *pg_tbl;
7306 
7307 			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7308 			if (!pg_tbl)
7309 				return -ENOMEM;
7310 			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7311 			rmem = &pg_tbl->ring_mem;
7312 			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7313 			rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7314 			rmem->depth = 1;
7315 			rmem->nr_pages = MAX_CTX_PAGES;
7316 			rmem->mem_init = mem_init;
7317 			if (i == (nr_tbls - 1)) {
7318 				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7319 
7320 				if (rem)
7321 					rmem->nr_pages = rem;
7322 			}
7323 			rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7324 			if (rc)
7325 				break;
7326 		}
7327 	} else {
7328 		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7329 		if (rmem->nr_pages > 1 || depth)
7330 			rmem->depth = 1;
7331 		rmem->mem_init = mem_init;
7332 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7333 	}
7334 	return rc;
7335 }
7336 
7337 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7338 				  struct bnxt_ctx_pg_info *ctx_pg)
7339 {
7340 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7341 
7342 	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7343 	    ctx_pg->ctx_pg_tbl) {
7344 		int i, nr_tbls = rmem->nr_pages;
7345 
7346 		for (i = 0; i < nr_tbls; i++) {
7347 			struct bnxt_ctx_pg_info *pg_tbl;
7348 			struct bnxt_ring_mem_info *rmem2;
7349 
7350 			pg_tbl = ctx_pg->ctx_pg_tbl[i];
7351 			if (!pg_tbl)
7352 				continue;
7353 			rmem2 = &pg_tbl->ring_mem;
7354 			bnxt_free_ring(bp, rmem2);
7355 			ctx_pg->ctx_pg_arr[i] = NULL;
7356 			kfree(pg_tbl);
7357 			ctx_pg->ctx_pg_tbl[i] = NULL;
7358 		}
7359 		kfree(ctx_pg->ctx_pg_tbl);
7360 		ctx_pg->ctx_pg_tbl = NULL;
7361 	}
7362 	bnxt_free_ring(bp, rmem);
7363 	ctx_pg->nr_pages = 0;
7364 }
7365 
7366 void bnxt_free_ctx_mem(struct bnxt *bp)
7367 {
7368 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
7369 	int i;
7370 
7371 	if (!ctx)
7372 		return;
7373 
7374 	if (ctx->tqm_mem[0]) {
7375 		for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7376 			bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7377 		kfree(ctx->tqm_mem[0]);
7378 		ctx->tqm_mem[0] = NULL;
7379 	}
7380 
7381 	bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7382 	bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7383 	bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7384 	bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7385 	bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7386 	bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7387 	bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7388 	ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7389 }
7390 
7391 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7392 {
7393 	struct bnxt_ctx_pg_info *ctx_pg;
7394 	struct bnxt_ctx_mem_info *ctx;
7395 	struct bnxt_mem_init *init;
7396 	u32 mem_size, ena, entries;
7397 	u32 entries_sp, min;
7398 	u32 num_mr, num_ah;
7399 	u32 extra_srqs = 0;
7400 	u32 extra_qps = 0;
7401 	u8 pg_lvl = 1;
7402 	int i, rc;
7403 
7404 	rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7405 	if (rc) {
7406 		netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7407 			   rc);
7408 		return rc;
7409 	}
7410 	ctx = bp->ctx;
7411 	if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7412 		return 0;
7413 
7414 	if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7415 		pg_lvl = 2;
7416 		extra_qps = 65536;
7417 		extra_srqs = 8192;
7418 	}
7419 
7420 	ctx_pg = &ctx->qp_mem;
7421 	ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7422 			  extra_qps;
7423 	if (ctx->qp_entry_size) {
7424 		mem_size = ctx->qp_entry_size * ctx_pg->entries;
7425 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7426 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7427 		if (rc)
7428 			return rc;
7429 	}
7430 
7431 	ctx_pg = &ctx->srq_mem;
7432 	ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7433 	if (ctx->srq_entry_size) {
7434 		mem_size = ctx->srq_entry_size * ctx_pg->entries;
7435 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7436 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7437 		if (rc)
7438 			return rc;
7439 	}
7440 
7441 	ctx_pg = &ctx->cq_mem;
7442 	ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7443 	if (ctx->cq_entry_size) {
7444 		mem_size = ctx->cq_entry_size * ctx_pg->entries;
7445 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7446 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7447 		if (rc)
7448 			return rc;
7449 	}
7450 
7451 	ctx_pg = &ctx->vnic_mem;
7452 	ctx_pg->entries = ctx->vnic_max_vnic_entries +
7453 			  ctx->vnic_max_ring_table_entries;
7454 	if (ctx->vnic_entry_size) {
7455 		mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7456 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7457 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7458 		if (rc)
7459 			return rc;
7460 	}
7461 
7462 	ctx_pg = &ctx->stat_mem;
7463 	ctx_pg->entries = ctx->stat_max_entries;
7464 	if (ctx->stat_entry_size) {
7465 		mem_size = ctx->stat_entry_size * ctx_pg->entries;
7466 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7467 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7468 		if (rc)
7469 			return rc;
7470 	}
7471 
7472 	ena = 0;
7473 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7474 		goto skip_rdma;
7475 
7476 	ctx_pg = &ctx->mrav_mem;
7477 	/* 128K extra is needed to accommodate static AH context
7478 	 * allocation by f/w.
7479 	 */
7480 	num_mr = 1024 * 256;
7481 	num_ah = 1024 * 128;
7482 	ctx_pg->entries = num_mr + num_ah;
7483 	if (ctx->mrav_entry_size) {
7484 		mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7485 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7486 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7487 		if (rc)
7488 			return rc;
7489 	}
7490 	ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7491 	if (ctx->mrav_num_entries_units)
7492 		ctx_pg->entries =
7493 			((num_mr / ctx->mrav_num_entries_units) << 16) |
7494 			 (num_ah / ctx->mrav_num_entries_units);
7495 
7496 	ctx_pg = &ctx->tim_mem;
7497 	ctx_pg->entries = ctx->qp_mem.entries;
7498 	if (ctx->tim_entry_size) {
7499 		mem_size = ctx->tim_entry_size * ctx_pg->entries;
7500 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7501 		if (rc)
7502 			return rc;
7503 	}
7504 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7505 
7506 skip_rdma:
7507 	min = ctx->tqm_min_entries_per_ring;
7508 	entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7509 		     2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7510 	entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7511 	entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7512 	entries = roundup(entries, ctx->tqm_entries_multiple);
7513 	entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7514 	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7515 		ctx_pg = ctx->tqm_mem[i];
7516 		ctx_pg->entries = i ? entries : entries_sp;
7517 		if (ctx->tqm_entry_size) {
7518 			mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7519 			rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7520 						    NULL);
7521 			if (rc)
7522 				return rc;
7523 		}
7524 		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7525 	}
7526 	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7527 	rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7528 	if (rc) {
7529 		netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7530 			   rc);
7531 		return rc;
7532 	}
7533 	ctx->flags |= BNXT_CTX_FLAG_INITED;
7534 	return 0;
7535 }
7536 
7537 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7538 {
7539 	struct hwrm_func_resource_qcaps_output *resp;
7540 	struct hwrm_func_resource_qcaps_input *req;
7541 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7542 	int rc;
7543 
7544 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
7545 	if (rc)
7546 		return rc;
7547 
7548 	req->fid = cpu_to_le16(0xffff);
7549 	resp = hwrm_req_hold(bp, req);
7550 	rc = hwrm_req_send_silent(bp, req);
7551 	if (rc)
7552 		goto hwrm_func_resc_qcaps_exit;
7553 
7554 	hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7555 	if (!all)
7556 		goto hwrm_func_resc_qcaps_exit;
7557 
7558 	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7559 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7560 	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7561 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7562 	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7563 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7564 	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7565 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7566 	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7567 	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7568 	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7569 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7570 	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7571 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7572 	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7573 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7574 
7575 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
7576 		u16 max_msix = le16_to_cpu(resp->max_msix);
7577 
7578 		hw_resc->max_nqs = max_msix;
7579 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7580 	}
7581 
7582 	if (BNXT_PF(bp)) {
7583 		struct bnxt_pf_info *pf = &bp->pf;
7584 
7585 		pf->vf_resv_strategy =
7586 			le16_to_cpu(resp->vf_reservation_strategy);
7587 		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7588 			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7589 	}
7590 hwrm_func_resc_qcaps_exit:
7591 	hwrm_req_drop(bp, req);
7592 	return rc;
7593 }
7594 
7595 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7596 {
7597 	struct hwrm_port_mac_ptp_qcfg_output *resp;
7598 	struct hwrm_port_mac_ptp_qcfg_input *req;
7599 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7600 	bool phc_cfg;
7601 	u8 flags;
7602 	int rc;
7603 
7604 	if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) {
7605 		rc = -ENODEV;
7606 		goto no_ptp;
7607 	}
7608 
7609 	rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
7610 	if (rc)
7611 		goto no_ptp;
7612 
7613 	req->port_id = cpu_to_le16(bp->pf.port_id);
7614 	resp = hwrm_req_hold(bp, req);
7615 	rc = hwrm_req_send(bp, req);
7616 	if (rc)
7617 		goto exit;
7618 
7619 	flags = resp->flags;
7620 	if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7621 		rc = -ENODEV;
7622 		goto exit;
7623 	}
7624 	if (!ptp) {
7625 		ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7626 		if (!ptp) {
7627 			rc = -ENOMEM;
7628 			goto exit;
7629 		}
7630 		ptp->bp = bp;
7631 		bp->ptp_cfg = ptp;
7632 	}
7633 	if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7634 		ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7635 		ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7636 	} else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7637 		ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7638 		ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7639 	} else {
7640 		rc = -ENODEV;
7641 		goto exit;
7642 	}
7643 	phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
7644 	rc = bnxt_ptp_init(bp, phc_cfg);
7645 	if (rc)
7646 		netdev_warn(bp->dev, "PTP initialization failed.\n");
7647 exit:
7648 	hwrm_req_drop(bp, req);
7649 	if (!rc)
7650 		return 0;
7651 
7652 no_ptp:
7653 	bnxt_ptp_clear(bp);
7654 	kfree(ptp);
7655 	bp->ptp_cfg = NULL;
7656 	return rc;
7657 }
7658 
7659 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7660 {
7661 	struct hwrm_func_qcaps_output *resp;
7662 	struct hwrm_func_qcaps_input *req;
7663 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7664 	u32 flags, flags_ext, flags_ext2;
7665 	int rc;
7666 
7667 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
7668 	if (rc)
7669 		return rc;
7670 
7671 	req->fid = cpu_to_le16(0xffff);
7672 	resp = hwrm_req_hold(bp, req);
7673 	rc = hwrm_req_send(bp, req);
7674 	if (rc)
7675 		goto hwrm_func_qcaps_exit;
7676 
7677 	flags = le32_to_cpu(resp->flags);
7678 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7679 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7680 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7681 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7682 	if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7683 		bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7684 	if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7685 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7686 	if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7687 		bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7688 	if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7689 		bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7690 	if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7691 		bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7692 	if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7693 		bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7694 	if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
7695 		bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
7696 
7697 	flags_ext = le32_to_cpu(resp->flags_ext);
7698 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7699 		bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7700 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
7701 		bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
7702 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
7703 		bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
7704 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
7705 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
7706 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
7707 		bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
7708 
7709 	flags_ext2 = le32_to_cpu(resp->flags_ext2);
7710 	if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
7711 		bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
7712 
7713 	bp->tx_push_thresh = 0;
7714 	if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7715 	    BNXT_FW_MAJ(bp) > 217)
7716 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7717 
7718 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7719 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7720 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7721 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7722 	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7723 	if (!hw_resc->max_hw_ring_grps)
7724 		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7725 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7726 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7727 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7728 
7729 	if (BNXT_PF(bp)) {
7730 		struct bnxt_pf_info *pf = &bp->pf;
7731 
7732 		pf->fw_fid = le16_to_cpu(resp->fid);
7733 		pf->port_id = le16_to_cpu(resp->port_id);
7734 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7735 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7736 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
7737 		pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7738 		pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7739 		pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7740 		pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7741 		pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7742 		pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7743 		bp->flags &= ~BNXT_FLAG_WOL_CAP;
7744 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7745 			bp->flags |= BNXT_FLAG_WOL_CAP;
7746 		if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
7747 			bp->fw_cap |= BNXT_FW_CAP_PTP;
7748 		} else {
7749 			bnxt_ptp_clear(bp);
7750 			kfree(bp->ptp_cfg);
7751 			bp->ptp_cfg = NULL;
7752 		}
7753 	} else {
7754 #ifdef CONFIG_BNXT_SRIOV
7755 		struct bnxt_vf_info *vf = &bp->vf;
7756 
7757 		vf->fw_fid = le16_to_cpu(resp->fid);
7758 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7759 #endif
7760 	}
7761 
7762 hwrm_func_qcaps_exit:
7763 	hwrm_req_drop(bp, req);
7764 	return rc;
7765 }
7766 
7767 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
7768 {
7769 	struct hwrm_dbg_qcaps_output *resp;
7770 	struct hwrm_dbg_qcaps_input *req;
7771 	int rc;
7772 
7773 	bp->fw_dbg_cap = 0;
7774 	if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
7775 		return;
7776 
7777 	rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
7778 	if (rc)
7779 		return;
7780 
7781 	req->fid = cpu_to_le16(0xffff);
7782 	resp = hwrm_req_hold(bp, req);
7783 	rc = hwrm_req_send(bp, req);
7784 	if (rc)
7785 		goto hwrm_dbg_qcaps_exit;
7786 
7787 	bp->fw_dbg_cap = le32_to_cpu(resp->flags);
7788 
7789 hwrm_dbg_qcaps_exit:
7790 	hwrm_req_drop(bp, req);
7791 }
7792 
7793 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7794 
7795 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7796 {
7797 	int rc;
7798 
7799 	rc = __bnxt_hwrm_func_qcaps(bp);
7800 	if (rc)
7801 		return rc;
7802 
7803 	bnxt_hwrm_dbg_qcaps(bp);
7804 
7805 	rc = bnxt_hwrm_queue_qportcfg(bp);
7806 	if (rc) {
7807 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7808 		return rc;
7809 	}
7810 	if (bp->hwrm_spec_code >= 0x10803) {
7811 		rc = bnxt_alloc_ctx_mem(bp);
7812 		if (rc)
7813 			return rc;
7814 		rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7815 		if (!rc)
7816 			bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7817 	}
7818 	return 0;
7819 }
7820 
7821 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7822 {
7823 	struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7824 	struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
7825 	u32 flags;
7826 	int rc;
7827 
7828 	if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7829 		return 0;
7830 
7831 	rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
7832 	if (rc)
7833 		return rc;
7834 
7835 	resp = hwrm_req_hold(bp, req);
7836 	rc = hwrm_req_send(bp, req);
7837 	if (rc)
7838 		goto hwrm_cfa_adv_qcaps_exit;
7839 
7840 	flags = le32_to_cpu(resp->flags);
7841 	if (flags &
7842 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7843 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7844 
7845 hwrm_cfa_adv_qcaps_exit:
7846 	hwrm_req_drop(bp, req);
7847 	return rc;
7848 }
7849 
7850 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7851 {
7852 	if (bp->fw_health)
7853 		return 0;
7854 
7855 	bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7856 	if (!bp->fw_health)
7857 		return -ENOMEM;
7858 
7859 	mutex_init(&bp->fw_health->lock);
7860 	return 0;
7861 }
7862 
7863 static int bnxt_alloc_fw_health(struct bnxt *bp)
7864 {
7865 	int rc;
7866 
7867 	if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7868 	    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7869 		return 0;
7870 
7871 	rc = __bnxt_alloc_fw_health(bp);
7872 	if (rc) {
7873 		bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7874 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7875 		return rc;
7876 	}
7877 
7878 	return 0;
7879 }
7880 
7881 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7882 {
7883 	writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7884 					 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7885 					 BNXT_FW_HEALTH_WIN_MAP_OFF);
7886 }
7887 
7888 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7889 {
7890 	struct bnxt_fw_health *fw_health = bp->fw_health;
7891 	u32 reg_type;
7892 
7893 	if (!fw_health)
7894 		return;
7895 
7896 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7897 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7898 		fw_health->status_reliable = false;
7899 
7900 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
7901 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7902 		fw_health->resets_reliable = false;
7903 }
7904 
7905 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7906 {
7907 	void __iomem *hs;
7908 	u32 status_loc;
7909 	u32 reg_type;
7910 	u32 sig;
7911 
7912 	if (bp->fw_health)
7913 		bp->fw_health->status_reliable = false;
7914 
7915 	__bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7916 	hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7917 
7918 	sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7919 	if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7920 		if (!bp->chip_num) {
7921 			__bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7922 			bp->chip_num = readl(bp->bar0 +
7923 					     BNXT_FW_HEALTH_WIN_BASE +
7924 					     BNXT_GRC_REG_CHIP_NUM);
7925 		}
7926 		if (!BNXT_CHIP_P5(bp))
7927 			return;
7928 
7929 		status_loc = BNXT_GRC_REG_STATUS_P5 |
7930 			     BNXT_FW_HEALTH_REG_TYPE_BAR0;
7931 	} else {
7932 		status_loc = readl(hs + offsetof(struct hcomm_status,
7933 						 fw_status_loc));
7934 	}
7935 
7936 	if (__bnxt_alloc_fw_health(bp)) {
7937 		netdev_warn(bp->dev, "no memory for firmware status checks\n");
7938 		return;
7939 	}
7940 
7941 	bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7942 	reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7943 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7944 		__bnxt_map_fw_health_reg(bp, status_loc);
7945 		bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7946 			BNXT_FW_HEALTH_WIN_OFF(status_loc);
7947 	}
7948 
7949 	bp->fw_health->status_reliable = true;
7950 }
7951 
7952 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7953 {
7954 	struct bnxt_fw_health *fw_health = bp->fw_health;
7955 	u32 reg_base = 0xffffffff;
7956 	int i;
7957 
7958 	bp->fw_health->status_reliable = false;
7959 	bp->fw_health->resets_reliable = false;
7960 	/* Only pre-map the monitoring GRC registers using window 3 */
7961 	for (i = 0; i < 4; i++) {
7962 		u32 reg = fw_health->regs[i];
7963 
7964 		if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7965 			continue;
7966 		if (reg_base == 0xffffffff)
7967 			reg_base = reg & BNXT_GRC_BASE_MASK;
7968 		if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7969 			return -ERANGE;
7970 		fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7971 	}
7972 	bp->fw_health->status_reliable = true;
7973 	bp->fw_health->resets_reliable = true;
7974 	if (reg_base == 0xffffffff)
7975 		return 0;
7976 
7977 	__bnxt_map_fw_health_reg(bp, reg_base);
7978 	return 0;
7979 }
7980 
7981 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
7982 {
7983 	if (!bp->fw_health)
7984 		return;
7985 
7986 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
7987 		bp->fw_health->status_reliable = true;
7988 		bp->fw_health->resets_reliable = true;
7989 	} else {
7990 		bnxt_try_map_fw_health_reg(bp);
7991 	}
7992 }
7993 
7994 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7995 {
7996 	struct bnxt_fw_health *fw_health = bp->fw_health;
7997 	struct hwrm_error_recovery_qcfg_output *resp;
7998 	struct hwrm_error_recovery_qcfg_input *req;
7999 	int rc, i;
8000 
8001 	if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8002 		return 0;
8003 
8004 	rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8005 	if (rc)
8006 		return rc;
8007 
8008 	resp = hwrm_req_hold(bp, req);
8009 	rc = hwrm_req_send(bp, req);
8010 	if (rc)
8011 		goto err_recovery_out;
8012 	fw_health->flags = le32_to_cpu(resp->flags);
8013 	if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8014 	    !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8015 		rc = -EINVAL;
8016 		goto err_recovery_out;
8017 	}
8018 	fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
8019 	fw_health->master_func_wait_dsecs =
8020 		le32_to_cpu(resp->master_func_wait_period);
8021 	fw_health->normal_func_wait_dsecs =
8022 		le32_to_cpu(resp->normal_func_wait_period);
8023 	fw_health->post_reset_wait_dsecs =
8024 		le32_to_cpu(resp->master_func_wait_period_after_reset);
8025 	fw_health->post_reset_max_wait_dsecs =
8026 		le32_to_cpu(resp->max_bailout_time_after_reset);
8027 	fw_health->regs[BNXT_FW_HEALTH_REG] =
8028 		le32_to_cpu(resp->fw_health_status_reg);
8029 	fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
8030 		le32_to_cpu(resp->fw_heartbeat_reg);
8031 	fw_health->regs[BNXT_FW_RESET_CNT_REG] =
8032 		le32_to_cpu(resp->fw_reset_cnt_reg);
8033 	fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
8034 		le32_to_cpu(resp->reset_inprogress_reg);
8035 	fw_health->fw_reset_inprog_reg_mask =
8036 		le32_to_cpu(resp->reset_inprogress_reg_mask);
8037 	fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
8038 	if (fw_health->fw_reset_seq_cnt >= 16) {
8039 		rc = -EINVAL;
8040 		goto err_recovery_out;
8041 	}
8042 	for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
8043 		fw_health->fw_reset_seq_regs[i] =
8044 			le32_to_cpu(resp->reset_reg[i]);
8045 		fw_health->fw_reset_seq_vals[i] =
8046 			le32_to_cpu(resp->reset_reg_val[i]);
8047 		fw_health->fw_reset_seq_delay_msec[i] =
8048 			resp->delay_after_reset[i];
8049 	}
8050 err_recovery_out:
8051 	hwrm_req_drop(bp, req);
8052 	if (!rc)
8053 		rc = bnxt_map_fw_health_regs(bp);
8054 	if (rc)
8055 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8056 	return rc;
8057 }
8058 
8059 static int bnxt_hwrm_func_reset(struct bnxt *bp)
8060 {
8061 	struct hwrm_func_reset_input *req;
8062 	int rc;
8063 
8064 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
8065 	if (rc)
8066 		return rc;
8067 
8068 	req->enables = 0;
8069 	hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
8070 	return hwrm_req_send(bp, req);
8071 }
8072 
8073 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
8074 {
8075 	struct hwrm_nvm_get_dev_info_output nvm_info;
8076 
8077 	if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
8078 		snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
8079 			 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
8080 			 nvm_info.nvm_cfg_ver_upd);
8081 }
8082 
8083 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
8084 {
8085 	struct hwrm_queue_qportcfg_output *resp;
8086 	struct hwrm_queue_qportcfg_input *req;
8087 	u8 i, j, *qptr;
8088 	bool no_rdma;
8089 	int rc = 0;
8090 
8091 	rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
8092 	if (rc)
8093 		return rc;
8094 
8095 	resp = hwrm_req_hold(bp, req);
8096 	rc = hwrm_req_send(bp, req);
8097 	if (rc)
8098 		goto qportcfg_exit;
8099 
8100 	if (!resp->max_configurable_queues) {
8101 		rc = -EINVAL;
8102 		goto qportcfg_exit;
8103 	}
8104 	bp->max_tc = resp->max_configurable_queues;
8105 	bp->max_lltc = resp->max_configurable_lossless_queues;
8106 	if (bp->max_tc > BNXT_MAX_QUEUE)
8107 		bp->max_tc = BNXT_MAX_QUEUE;
8108 
8109 	no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
8110 	qptr = &resp->queue_id0;
8111 	for (i = 0, j = 0; i < bp->max_tc; i++) {
8112 		bp->q_info[j].queue_id = *qptr;
8113 		bp->q_ids[i] = *qptr++;
8114 		bp->q_info[j].queue_profile = *qptr++;
8115 		bp->tc_to_qidx[j] = j;
8116 		if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
8117 		    (no_rdma && BNXT_PF(bp)))
8118 			j++;
8119 	}
8120 	bp->max_q = bp->max_tc;
8121 	bp->max_tc = max_t(u8, j, 1);
8122 
8123 	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
8124 		bp->max_tc = 1;
8125 
8126 	if (bp->max_lltc > bp->max_tc)
8127 		bp->max_lltc = bp->max_tc;
8128 
8129 qportcfg_exit:
8130 	hwrm_req_drop(bp, req);
8131 	return rc;
8132 }
8133 
8134 static int bnxt_hwrm_poll(struct bnxt *bp)
8135 {
8136 	struct hwrm_ver_get_input *req;
8137 	int rc;
8138 
8139 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8140 	if (rc)
8141 		return rc;
8142 
8143 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8144 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
8145 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8146 
8147 	hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
8148 	rc = hwrm_req_send(bp, req);
8149 	return rc;
8150 }
8151 
8152 static int bnxt_hwrm_ver_get(struct bnxt *bp)
8153 {
8154 	struct hwrm_ver_get_output *resp;
8155 	struct hwrm_ver_get_input *req;
8156 	u16 fw_maj, fw_min, fw_bld, fw_rsv;
8157 	u32 dev_caps_cfg, hwrm_ver;
8158 	int rc, len;
8159 
8160 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8161 	if (rc)
8162 		return rc;
8163 
8164 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
8165 	bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
8166 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8167 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
8168 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8169 
8170 	resp = hwrm_req_hold(bp, req);
8171 	rc = hwrm_req_send(bp, req);
8172 	if (rc)
8173 		goto hwrm_ver_get_exit;
8174 
8175 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
8176 
8177 	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
8178 			     resp->hwrm_intf_min_8b << 8 |
8179 			     resp->hwrm_intf_upd_8b;
8180 	if (resp->hwrm_intf_maj_8b < 1) {
8181 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
8182 			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8183 			    resp->hwrm_intf_upd_8b);
8184 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
8185 	}
8186 
8187 	hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
8188 			HWRM_VERSION_UPDATE;
8189 
8190 	if (bp->hwrm_spec_code > hwrm_ver)
8191 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8192 			 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
8193 			 HWRM_VERSION_UPDATE);
8194 	else
8195 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8196 			 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8197 			 resp->hwrm_intf_upd_8b);
8198 
8199 	fw_maj = le16_to_cpu(resp->hwrm_fw_major);
8200 	if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
8201 		fw_min = le16_to_cpu(resp->hwrm_fw_minor);
8202 		fw_bld = le16_to_cpu(resp->hwrm_fw_build);
8203 		fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
8204 		len = FW_VER_STR_LEN;
8205 	} else {
8206 		fw_maj = resp->hwrm_fw_maj_8b;
8207 		fw_min = resp->hwrm_fw_min_8b;
8208 		fw_bld = resp->hwrm_fw_bld_8b;
8209 		fw_rsv = resp->hwrm_fw_rsvd_8b;
8210 		len = BC_HWRM_STR_LEN;
8211 	}
8212 	bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
8213 	snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
8214 		 fw_rsv);
8215 
8216 	if (strlen(resp->active_pkg_name)) {
8217 		int fw_ver_len = strlen(bp->fw_ver_str);
8218 
8219 		snprintf(bp->fw_ver_str + fw_ver_len,
8220 			 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
8221 			 resp->active_pkg_name);
8222 		bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
8223 	}
8224 
8225 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
8226 	if (!bp->hwrm_cmd_timeout)
8227 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
8228 	bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
8229 	if (!bp->hwrm_cmd_max_timeout)
8230 		bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
8231 	else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
8232 		netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
8233 			    bp->hwrm_cmd_max_timeout / 1000);
8234 
8235 	if (resp->hwrm_intf_maj_8b >= 1) {
8236 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
8237 		bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
8238 	}
8239 	if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
8240 		bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
8241 
8242 	bp->chip_num = le16_to_cpu(resp->chip_num);
8243 	bp->chip_rev = resp->chip_rev;
8244 	if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
8245 	    !resp->chip_metal)
8246 		bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
8247 
8248 	dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
8249 	if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
8250 	    (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
8251 		bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
8252 
8253 	if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
8254 		bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
8255 
8256 	if (dev_caps_cfg &
8257 	    VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
8258 		bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
8259 
8260 	if (dev_caps_cfg &
8261 	    VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
8262 		bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
8263 
8264 	if (dev_caps_cfg &
8265 	    VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
8266 		bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
8267 
8268 hwrm_ver_get_exit:
8269 	hwrm_req_drop(bp, req);
8270 	return rc;
8271 }
8272 
8273 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
8274 {
8275 	struct hwrm_fw_set_time_input *req;
8276 	struct tm tm;
8277 	time64_t now = ktime_get_real_seconds();
8278 	int rc;
8279 
8280 	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
8281 	    bp->hwrm_spec_code < 0x10400)
8282 		return -EOPNOTSUPP;
8283 
8284 	time64_to_tm(now, 0, &tm);
8285 	rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
8286 	if (rc)
8287 		return rc;
8288 
8289 	req->year = cpu_to_le16(1900 + tm.tm_year);
8290 	req->month = 1 + tm.tm_mon;
8291 	req->day = tm.tm_mday;
8292 	req->hour = tm.tm_hour;
8293 	req->minute = tm.tm_min;
8294 	req->second = tm.tm_sec;
8295 	return hwrm_req_send(bp, req);
8296 }
8297 
8298 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
8299 {
8300 	u64 sw_tmp;
8301 
8302 	hw &= mask;
8303 	sw_tmp = (*sw & ~mask) | hw;
8304 	if (hw < (*sw & mask))
8305 		sw_tmp += mask + 1;
8306 	WRITE_ONCE(*sw, sw_tmp);
8307 }
8308 
8309 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8310 				    int count, bool ignore_zero)
8311 {
8312 	int i;
8313 
8314 	for (i = 0; i < count; i++) {
8315 		u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8316 
8317 		if (ignore_zero && !hw)
8318 			continue;
8319 
8320 		if (masks[i] == -1ULL)
8321 			sw_stats[i] = hw;
8322 		else
8323 			bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8324 	}
8325 }
8326 
8327 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8328 {
8329 	if (!stats->hw_stats)
8330 		return;
8331 
8332 	__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8333 				stats->hw_masks, stats->len / 8, false);
8334 }
8335 
8336 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8337 {
8338 	struct bnxt_stats_mem *ring0_stats;
8339 	bool ignore_zero = false;
8340 	int i;
8341 
8342 	/* Chip bug.  Counter intermittently becomes 0. */
8343 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8344 		ignore_zero = true;
8345 
8346 	for (i = 0; i < bp->cp_nr_rings; i++) {
8347 		struct bnxt_napi *bnapi = bp->bnapi[i];
8348 		struct bnxt_cp_ring_info *cpr;
8349 		struct bnxt_stats_mem *stats;
8350 
8351 		cpr = &bnapi->cp_ring;
8352 		stats = &cpr->stats;
8353 		if (!i)
8354 			ring0_stats = stats;
8355 		__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8356 					ring0_stats->hw_masks,
8357 					ring0_stats->len / 8, ignore_zero);
8358 	}
8359 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
8360 		struct bnxt_stats_mem *stats = &bp->port_stats;
8361 		__le64 *hw_stats = stats->hw_stats;
8362 		u64 *sw_stats = stats->sw_stats;
8363 		u64 *masks = stats->hw_masks;
8364 		int cnt;
8365 
8366 		cnt = sizeof(struct rx_port_stats) / 8;
8367 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8368 
8369 		hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8370 		sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8371 		masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8372 		cnt = sizeof(struct tx_port_stats) / 8;
8373 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8374 	}
8375 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8376 		bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8377 		bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8378 	}
8379 }
8380 
8381 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8382 {
8383 	struct hwrm_port_qstats_input *req;
8384 	struct bnxt_pf_info *pf = &bp->pf;
8385 	int rc;
8386 
8387 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8388 		return 0;
8389 
8390 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8391 		return -EOPNOTSUPP;
8392 
8393 	rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
8394 	if (rc)
8395 		return rc;
8396 
8397 	req->flags = flags;
8398 	req->port_id = cpu_to_le16(pf->port_id);
8399 	req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8400 					    BNXT_TX_PORT_STATS_BYTE_OFFSET);
8401 	req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8402 	return hwrm_req_send(bp, req);
8403 }
8404 
8405 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8406 {
8407 	struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
8408 	struct hwrm_queue_pri2cos_qcfg_input *req_qc;
8409 	struct hwrm_port_qstats_ext_output *resp_qs;
8410 	struct hwrm_port_qstats_ext_input *req_qs;
8411 	struct bnxt_pf_info *pf = &bp->pf;
8412 	u32 tx_stat_size;
8413 	int rc;
8414 
8415 	if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8416 		return 0;
8417 
8418 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8419 		return -EOPNOTSUPP;
8420 
8421 	rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
8422 	if (rc)
8423 		return rc;
8424 
8425 	req_qs->flags = flags;
8426 	req_qs->port_id = cpu_to_le16(pf->port_id);
8427 	req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8428 	req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8429 	tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8430 		       sizeof(struct tx_port_stats_ext) : 0;
8431 	req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
8432 	req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8433 	resp_qs = hwrm_req_hold(bp, req_qs);
8434 	rc = hwrm_req_send(bp, req_qs);
8435 	if (!rc) {
8436 		bp->fw_rx_stats_ext_size =
8437 			le16_to_cpu(resp_qs->rx_stat_size) / 8;
8438 		if (BNXT_FW_MAJ(bp) < 220 &&
8439 		    bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
8440 			bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
8441 
8442 		bp->fw_tx_stats_ext_size = tx_stat_size ?
8443 			le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
8444 	} else {
8445 		bp->fw_rx_stats_ext_size = 0;
8446 		bp->fw_tx_stats_ext_size = 0;
8447 	}
8448 	hwrm_req_drop(bp, req_qs);
8449 
8450 	if (flags)
8451 		return rc;
8452 
8453 	if (bp->fw_tx_stats_ext_size <=
8454 	    offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8455 		bp->pri2cos_valid = 0;
8456 		return rc;
8457 	}
8458 
8459 	rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
8460 	if (rc)
8461 		return rc;
8462 
8463 	req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8464 
8465 	resp_qc = hwrm_req_hold(bp, req_qc);
8466 	rc = hwrm_req_send(bp, req_qc);
8467 	if (!rc) {
8468 		u8 *pri2cos;
8469 		int i, j;
8470 
8471 		pri2cos = &resp_qc->pri0_cos_queue_id;
8472 		for (i = 0; i < 8; i++) {
8473 			u8 queue_id = pri2cos[i];
8474 			u8 queue_idx;
8475 
8476 			/* Per port queue IDs start from 0, 10, 20, etc */
8477 			queue_idx = queue_id % 10;
8478 			if (queue_idx > BNXT_MAX_QUEUE) {
8479 				bp->pri2cos_valid = false;
8480 				hwrm_req_drop(bp, req_qc);
8481 				return rc;
8482 			}
8483 			for (j = 0; j < bp->max_q; j++) {
8484 				if (bp->q_ids[j] == queue_id)
8485 					bp->pri2cos_idx[i] = queue_idx;
8486 			}
8487 		}
8488 		bp->pri2cos_valid = true;
8489 	}
8490 	hwrm_req_drop(bp, req_qc);
8491 
8492 	return rc;
8493 }
8494 
8495 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8496 {
8497 	bnxt_hwrm_tunnel_dst_port_free(bp,
8498 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8499 	bnxt_hwrm_tunnel_dst_port_free(bp,
8500 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8501 }
8502 
8503 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8504 {
8505 	int rc, i;
8506 	u32 tpa_flags = 0;
8507 
8508 	if (set_tpa)
8509 		tpa_flags = bp->flags & BNXT_FLAG_TPA;
8510 	else if (BNXT_NO_FW_ACCESS(bp))
8511 		return 0;
8512 	for (i = 0; i < bp->nr_vnics; i++) {
8513 		rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8514 		if (rc) {
8515 			netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8516 				   i, rc);
8517 			return rc;
8518 		}
8519 	}
8520 	return 0;
8521 }
8522 
8523 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8524 {
8525 	int i;
8526 
8527 	for (i = 0; i < bp->nr_vnics; i++)
8528 		bnxt_hwrm_vnic_set_rss(bp, i, false);
8529 }
8530 
8531 static void bnxt_clear_vnic(struct bnxt *bp)
8532 {
8533 	if (!bp->vnic_info)
8534 		return;
8535 
8536 	bnxt_hwrm_clear_vnic_filter(bp);
8537 	if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8538 		/* clear all RSS setting before free vnic ctx */
8539 		bnxt_hwrm_clear_vnic_rss(bp);
8540 		bnxt_hwrm_vnic_ctx_free(bp);
8541 	}
8542 	/* before free the vnic, undo the vnic tpa settings */
8543 	if (bp->flags & BNXT_FLAG_TPA)
8544 		bnxt_set_tpa(bp, false);
8545 	bnxt_hwrm_vnic_free(bp);
8546 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8547 		bnxt_hwrm_vnic_ctx_free(bp);
8548 }
8549 
8550 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8551 				    bool irq_re_init)
8552 {
8553 	bnxt_clear_vnic(bp);
8554 	bnxt_hwrm_ring_free(bp, close_path);
8555 	bnxt_hwrm_ring_grp_free(bp);
8556 	if (irq_re_init) {
8557 		bnxt_hwrm_stat_ctx_free(bp);
8558 		bnxt_hwrm_free_tunnel_ports(bp);
8559 	}
8560 }
8561 
8562 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8563 {
8564 	struct hwrm_func_cfg_input *req;
8565 	u8 evb_mode;
8566 	int rc;
8567 
8568 	if (br_mode == BRIDGE_MODE_VEB)
8569 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8570 	else if (br_mode == BRIDGE_MODE_VEPA)
8571 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8572 	else
8573 		return -EINVAL;
8574 
8575 	rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8576 	if (rc)
8577 		return rc;
8578 
8579 	req->fid = cpu_to_le16(0xffff);
8580 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8581 	req->evb_mode = evb_mode;
8582 	return hwrm_req_send(bp, req);
8583 }
8584 
8585 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8586 {
8587 	struct hwrm_func_cfg_input *req;
8588 	int rc;
8589 
8590 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8591 		return 0;
8592 
8593 	rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8594 	if (rc)
8595 		return rc;
8596 
8597 	req->fid = cpu_to_le16(0xffff);
8598 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8599 	req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8600 	if (size == 128)
8601 		req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8602 
8603 	return hwrm_req_send(bp, req);
8604 }
8605 
8606 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8607 {
8608 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8609 	int rc;
8610 
8611 	if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8612 		goto skip_rss_ctx;
8613 
8614 	/* allocate context for vnic */
8615 	rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8616 	if (rc) {
8617 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8618 			   vnic_id, rc);
8619 		goto vnic_setup_err;
8620 	}
8621 	bp->rsscos_nr_ctxs++;
8622 
8623 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8624 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8625 		if (rc) {
8626 			netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8627 				   vnic_id, rc);
8628 			goto vnic_setup_err;
8629 		}
8630 		bp->rsscos_nr_ctxs++;
8631 	}
8632 
8633 skip_rss_ctx:
8634 	/* configure default vnic, ring grp */
8635 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8636 	if (rc) {
8637 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8638 			   vnic_id, rc);
8639 		goto vnic_setup_err;
8640 	}
8641 
8642 	/* Enable RSS hashing on vnic */
8643 	rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8644 	if (rc) {
8645 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8646 			   vnic_id, rc);
8647 		goto vnic_setup_err;
8648 	}
8649 
8650 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8651 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8652 		if (rc) {
8653 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8654 				   vnic_id, rc);
8655 		}
8656 	}
8657 
8658 vnic_setup_err:
8659 	return rc;
8660 }
8661 
8662 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8663 {
8664 	int rc, i, nr_ctxs;
8665 
8666 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8667 	for (i = 0; i < nr_ctxs; i++) {
8668 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8669 		if (rc) {
8670 			netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8671 				   vnic_id, i, rc);
8672 			break;
8673 		}
8674 		bp->rsscos_nr_ctxs++;
8675 	}
8676 	if (i < nr_ctxs)
8677 		return -ENOMEM;
8678 
8679 	rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8680 	if (rc) {
8681 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8682 			   vnic_id, rc);
8683 		return rc;
8684 	}
8685 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8686 	if (rc) {
8687 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8688 			   vnic_id, rc);
8689 		return rc;
8690 	}
8691 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8692 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8693 		if (rc) {
8694 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8695 				   vnic_id, rc);
8696 		}
8697 	}
8698 	return rc;
8699 }
8700 
8701 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8702 {
8703 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8704 		return __bnxt_setup_vnic_p5(bp, vnic_id);
8705 	else
8706 		return __bnxt_setup_vnic(bp, vnic_id);
8707 }
8708 
8709 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8710 {
8711 #ifdef CONFIG_RFS_ACCEL
8712 	int i, rc = 0;
8713 
8714 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8715 		return 0;
8716 
8717 	for (i = 0; i < bp->rx_nr_rings; i++) {
8718 		struct bnxt_vnic_info *vnic;
8719 		u16 vnic_id = i + 1;
8720 		u16 ring_id = i;
8721 
8722 		if (vnic_id >= bp->nr_vnics)
8723 			break;
8724 
8725 		vnic = &bp->vnic_info[vnic_id];
8726 		vnic->flags |= BNXT_VNIC_RFS_FLAG;
8727 		if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8728 			vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8729 		rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8730 		if (rc) {
8731 			netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8732 				   vnic_id, rc);
8733 			break;
8734 		}
8735 		rc = bnxt_setup_vnic(bp, vnic_id);
8736 		if (rc)
8737 			break;
8738 	}
8739 	return rc;
8740 #else
8741 	return 0;
8742 #endif
8743 }
8744 
8745 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8746 static bool bnxt_promisc_ok(struct bnxt *bp)
8747 {
8748 #ifdef CONFIG_BNXT_SRIOV
8749 	if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8750 		return false;
8751 #endif
8752 	return true;
8753 }
8754 
8755 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8756 {
8757 	unsigned int rc = 0;
8758 
8759 	rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8760 	if (rc) {
8761 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8762 			   rc);
8763 		return rc;
8764 	}
8765 
8766 	rc = bnxt_hwrm_vnic_cfg(bp, 1);
8767 	if (rc) {
8768 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8769 			   rc);
8770 		return rc;
8771 	}
8772 	return rc;
8773 }
8774 
8775 static int bnxt_cfg_rx_mode(struct bnxt *);
8776 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8777 
8778 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8779 {
8780 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8781 	int rc = 0;
8782 	unsigned int rx_nr_rings = bp->rx_nr_rings;
8783 
8784 	if (irq_re_init) {
8785 		rc = bnxt_hwrm_stat_ctx_alloc(bp);
8786 		if (rc) {
8787 			netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8788 				   rc);
8789 			goto err_out;
8790 		}
8791 	}
8792 
8793 	rc = bnxt_hwrm_ring_alloc(bp);
8794 	if (rc) {
8795 		netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8796 		goto err_out;
8797 	}
8798 
8799 	rc = bnxt_hwrm_ring_grp_alloc(bp);
8800 	if (rc) {
8801 		netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8802 		goto err_out;
8803 	}
8804 
8805 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8806 		rx_nr_rings--;
8807 
8808 	/* default vnic 0 */
8809 	rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8810 	if (rc) {
8811 		netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8812 		goto err_out;
8813 	}
8814 
8815 	if (BNXT_VF(bp))
8816 		bnxt_hwrm_func_qcfg(bp);
8817 
8818 	rc = bnxt_setup_vnic(bp, 0);
8819 	if (rc)
8820 		goto err_out;
8821 	if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
8822 		bnxt_hwrm_update_rss_hash_cfg(bp);
8823 
8824 	if (bp->flags & BNXT_FLAG_RFS) {
8825 		rc = bnxt_alloc_rfs_vnics(bp);
8826 		if (rc)
8827 			goto err_out;
8828 	}
8829 
8830 	if (bp->flags & BNXT_FLAG_TPA) {
8831 		rc = bnxt_set_tpa(bp, true);
8832 		if (rc)
8833 			goto err_out;
8834 	}
8835 
8836 	if (BNXT_VF(bp))
8837 		bnxt_update_vf_mac(bp);
8838 
8839 	/* Filter for default vnic 0 */
8840 	rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8841 	if (rc) {
8842 		if (BNXT_VF(bp) && rc == -ENODEV)
8843 			netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
8844 		else
8845 			netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8846 		goto err_out;
8847 	}
8848 	vnic->uc_filter_count = 1;
8849 
8850 	vnic->rx_mask = 0;
8851 	if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
8852 		goto skip_rx_mask;
8853 
8854 	if (bp->dev->flags & IFF_BROADCAST)
8855 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8856 
8857 	if (bp->dev->flags & IFF_PROMISC)
8858 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8859 
8860 	if (bp->dev->flags & IFF_ALLMULTI) {
8861 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8862 		vnic->mc_list_count = 0;
8863 	} else if (bp->dev->flags & IFF_MULTICAST) {
8864 		u32 mask = 0;
8865 
8866 		bnxt_mc_list_updated(bp, &mask);
8867 		vnic->rx_mask |= mask;
8868 	}
8869 
8870 	rc = bnxt_cfg_rx_mode(bp);
8871 	if (rc)
8872 		goto err_out;
8873 
8874 skip_rx_mask:
8875 	rc = bnxt_hwrm_set_coal(bp);
8876 	if (rc)
8877 		netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8878 				rc);
8879 
8880 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8881 		rc = bnxt_setup_nitroa0_vnic(bp);
8882 		if (rc)
8883 			netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8884 				   rc);
8885 	}
8886 
8887 	if (BNXT_VF(bp)) {
8888 		bnxt_hwrm_func_qcfg(bp);
8889 		netdev_update_features(bp->dev);
8890 	}
8891 
8892 	return 0;
8893 
8894 err_out:
8895 	bnxt_hwrm_resource_free(bp, 0, true);
8896 
8897 	return rc;
8898 }
8899 
8900 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8901 {
8902 	bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8903 	return 0;
8904 }
8905 
8906 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8907 {
8908 	bnxt_init_cp_rings(bp);
8909 	bnxt_init_rx_rings(bp);
8910 	bnxt_init_tx_rings(bp);
8911 	bnxt_init_ring_grps(bp, irq_re_init);
8912 	bnxt_init_vnics(bp);
8913 
8914 	return bnxt_init_chip(bp, irq_re_init);
8915 }
8916 
8917 static int bnxt_set_real_num_queues(struct bnxt *bp)
8918 {
8919 	int rc;
8920 	struct net_device *dev = bp->dev;
8921 
8922 	rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8923 					  bp->tx_nr_rings_xdp);
8924 	if (rc)
8925 		return rc;
8926 
8927 	rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8928 	if (rc)
8929 		return rc;
8930 
8931 #ifdef CONFIG_RFS_ACCEL
8932 	if (bp->flags & BNXT_FLAG_RFS)
8933 		dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8934 #endif
8935 
8936 	return rc;
8937 }
8938 
8939 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8940 			   bool shared)
8941 {
8942 	int _rx = *rx, _tx = *tx;
8943 
8944 	if (shared) {
8945 		*rx = min_t(int, _rx, max);
8946 		*tx = min_t(int, _tx, max);
8947 	} else {
8948 		if (max < 2)
8949 			return -ENOMEM;
8950 
8951 		while (_rx + _tx > max) {
8952 			if (_rx > _tx && _rx > 1)
8953 				_rx--;
8954 			else if (_tx > 1)
8955 				_tx--;
8956 		}
8957 		*rx = _rx;
8958 		*tx = _tx;
8959 	}
8960 	return 0;
8961 }
8962 
8963 static void bnxt_setup_msix(struct bnxt *bp)
8964 {
8965 	const int len = sizeof(bp->irq_tbl[0].name);
8966 	struct net_device *dev = bp->dev;
8967 	int tcs, i;
8968 
8969 	tcs = netdev_get_num_tc(dev);
8970 	if (tcs) {
8971 		int i, off, count;
8972 
8973 		for (i = 0; i < tcs; i++) {
8974 			count = bp->tx_nr_rings_per_tc;
8975 			off = i * count;
8976 			netdev_set_tc_queue(dev, i, count, off);
8977 		}
8978 	}
8979 
8980 	for (i = 0; i < bp->cp_nr_rings; i++) {
8981 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8982 		char *attr;
8983 
8984 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8985 			attr = "TxRx";
8986 		else if (i < bp->rx_nr_rings)
8987 			attr = "rx";
8988 		else
8989 			attr = "tx";
8990 
8991 		snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8992 			 attr, i);
8993 		bp->irq_tbl[map_idx].handler = bnxt_msix;
8994 	}
8995 }
8996 
8997 static void bnxt_setup_inta(struct bnxt *bp)
8998 {
8999 	const int len = sizeof(bp->irq_tbl[0].name);
9000 
9001 	if (netdev_get_num_tc(bp->dev))
9002 		netdev_reset_tc(bp->dev);
9003 
9004 	snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
9005 		 0);
9006 	bp->irq_tbl[0].handler = bnxt_inta;
9007 }
9008 
9009 static int bnxt_init_int_mode(struct bnxt *bp);
9010 
9011 static int bnxt_setup_int_mode(struct bnxt *bp)
9012 {
9013 	int rc;
9014 
9015 	if (!bp->irq_tbl) {
9016 		rc = bnxt_init_int_mode(bp);
9017 		if (rc || !bp->irq_tbl)
9018 			return rc ?: -ENODEV;
9019 	}
9020 
9021 	if (bp->flags & BNXT_FLAG_USING_MSIX)
9022 		bnxt_setup_msix(bp);
9023 	else
9024 		bnxt_setup_inta(bp);
9025 
9026 	rc = bnxt_set_real_num_queues(bp);
9027 	return rc;
9028 }
9029 
9030 #ifdef CONFIG_RFS_ACCEL
9031 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
9032 {
9033 	return bp->hw_resc.max_rsscos_ctxs;
9034 }
9035 
9036 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
9037 {
9038 	return bp->hw_resc.max_vnics;
9039 }
9040 #endif
9041 
9042 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
9043 {
9044 	return bp->hw_resc.max_stat_ctxs;
9045 }
9046 
9047 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
9048 {
9049 	return bp->hw_resc.max_cp_rings;
9050 }
9051 
9052 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
9053 {
9054 	unsigned int cp = bp->hw_resc.max_cp_rings;
9055 
9056 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9057 		cp -= bnxt_get_ulp_msix_num(bp);
9058 
9059 	return cp;
9060 }
9061 
9062 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
9063 {
9064 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9065 
9066 	if (bp->flags & BNXT_FLAG_CHIP_P5)
9067 		return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
9068 
9069 	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
9070 }
9071 
9072 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
9073 {
9074 	bp->hw_resc.max_irqs = max_irqs;
9075 }
9076 
9077 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
9078 {
9079 	unsigned int cp;
9080 
9081 	cp = bnxt_get_max_func_cp_rings_for_en(bp);
9082 	if (bp->flags & BNXT_FLAG_CHIP_P5)
9083 		return cp - bp->rx_nr_rings - bp->tx_nr_rings;
9084 	else
9085 		return cp - bp->cp_nr_rings;
9086 }
9087 
9088 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
9089 {
9090 	return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
9091 }
9092 
9093 int bnxt_get_avail_msix(struct bnxt *bp, int num)
9094 {
9095 	int max_cp = bnxt_get_max_func_cp_rings(bp);
9096 	int max_irq = bnxt_get_max_func_irqs(bp);
9097 	int total_req = bp->cp_nr_rings + num;
9098 	int max_idx, avail_msix;
9099 
9100 	max_idx = bp->total_irqs;
9101 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9102 		max_idx = min_t(int, bp->total_irqs, max_cp);
9103 	avail_msix = max_idx - bp->cp_nr_rings;
9104 	if (!BNXT_NEW_RM(bp) || avail_msix >= num)
9105 		return avail_msix;
9106 
9107 	if (max_irq < total_req) {
9108 		num = max_irq - bp->cp_nr_rings;
9109 		if (num <= 0)
9110 			return 0;
9111 	}
9112 	return num;
9113 }
9114 
9115 static int bnxt_get_num_msix(struct bnxt *bp)
9116 {
9117 	if (!BNXT_NEW_RM(bp))
9118 		return bnxt_get_max_func_irqs(bp);
9119 
9120 	return bnxt_nq_rings_in_use(bp);
9121 }
9122 
9123 static int bnxt_init_msix(struct bnxt *bp)
9124 {
9125 	int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
9126 	struct msix_entry *msix_ent;
9127 
9128 	total_vecs = bnxt_get_num_msix(bp);
9129 	max = bnxt_get_max_func_irqs(bp);
9130 	if (total_vecs > max)
9131 		total_vecs = max;
9132 
9133 	if (!total_vecs)
9134 		return 0;
9135 
9136 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
9137 	if (!msix_ent)
9138 		return -ENOMEM;
9139 
9140 	for (i = 0; i < total_vecs; i++) {
9141 		msix_ent[i].entry = i;
9142 		msix_ent[i].vector = 0;
9143 	}
9144 
9145 	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
9146 		min = 2;
9147 
9148 	total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
9149 	ulp_msix = bnxt_get_ulp_msix_num(bp);
9150 	if (total_vecs < 0 || total_vecs < ulp_msix) {
9151 		rc = -ENODEV;
9152 		goto msix_setup_exit;
9153 	}
9154 
9155 	bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
9156 	if (bp->irq_tbl) {
9157 		for (i = 0; i < total_vecs; i++)
9158 			bp->irq_tbl[i].vector = msix_ent[i].vector;
9159 
9160 		bp->total_irqs = total_vecs;
9161 		/* Trim rings based upon num of vectors allocated */
9162 		rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
9163 				     total_vecs - ulp_msix, min == 1);
9164 		if (rc)
9165 			goto msix_setup_exit;
9166 
9167 		bp->cp_nr_rings = (min == 1) ?
9168 				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9169 				  bp->tx_nr_rings + bp->rx_nr_rings;
9170 
9171 	} else {
9172 		rc = -ENOMEM;
9173 		goto msix_setup_exit;
9174 	}
9175 	bp->flags |= BNXT_FLAG_USING_MSIX;
9176 	kfree(msix_ent);
9177 	return 0;
9178 
9179 msix_setup_exit:
9180 	netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
9181 	kfree(bp->irq_tbl);
9182 	bp->irq_tbl = NULL;
9183 	pci_disable_msix(bp->pdev);
9184 	kfree(msix_ent);
9185 	return rc;
9186 }
9187 
9188 static int bnxt_init_inta(struct bnxt *bp)
9189 {
9190 	bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
9191 	if (!bp->irq_tbl)
9192 		return -ENOMEM;
9193 
9194 	bp->total_irqs = 1;
9195 	bp->rx_nr_rings = 1;
9196 	bp->tx_nr_rings = 1;
9197 	bp->cp_nr_rings = 1;
9198 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
9199 	bp->irq_tbl[0].vector = bp->pdev->irq;
9200 	return 0;
9201 }
9202 
9203 static int bnxt_init_int_mode(struct bnxt *bp)
9204 {
9205 	int rc = -ENODEV;
9206 
9207 	if (bp->flags & BNXT_FLAG_MSIX_CAP)
9208 		rc = bnxt_init_msix(bp);
9209 
9210 	if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
9211 		/* fallback to INTA */
9212 		rc = bnxt_init_inta(bp);
9213 	}
9214 	return rc;
9215 }
9216 
9217 static void bnxt_clear_int_mode(struct bnxt *bp)
9218 {
9219 	if (bp->flags & BNXT_FLAG_USING_MSIX)
9220 		pci_disable_msix(bp->pdev);
9221 
9222 	kfree(bp->irq_tbl);
9223 	bp->irq_tbl = NULL;
9224 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
9225 }
9226 
9227 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
9228 {
9229 	int tcs = netdev_get_num_tc(bp->dev);
9230 	bool irq_cleared = false;
9231 	int rc;
9232 
9233 	if (!bnxt_need_reserve_rings(bp))
9234 		return 0;
9235 
9236 	if (irq_re_init && BNXT_NEW_RM(bp) &&
9237 	    bnxt_get_num_msix(bp) != bp->total_irqs) {
9238 		bnxt_ulp_irq_stop(bp);
9239 		bnxt_clear_int_mode(bp);
9240 		irq_cleared = true;
9241 	}
9242 	rc = __bnxt_reserve_rings(bp);
9243 	if (irq_cleared) {
9244 		if (!rc)
9245 			rc = bnxt_init_int_mode(bp);
9246 		bnxt_ulp_irq_restart(bp, rc);
9247 	}
9248 	if (rc) {
9249 		netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
9250 		return rc;
9251 	}
9252 	if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
9253 		    bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
9254 		netdev_err(bp->dev, "tx ring reservation failure\n");
9255 		netdev_reset_tc(bp->dev);
9256 		if (bp->tx_nr_rings_xdp)
9257 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
9258 		else
9259 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9260 		return -ENOMEM;
9261 	}
9262 	return 0;
9263 }
9264 
9265 static void bnxt_free_irq(struct bnxt *bp)
9266 {
9267 	struct bnxt_irq *irq;
9268 	int i;
9269 
9270 #ifdef CONFIG_RFS_ACCEL
9271 	free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
9272 	bp->dev->rx_cpu_rmap = NULL;
9273 #endif
9274 	if (!bp->irq_tbl || !bp->bnapi)
9275 		return;
9276 
9277 	for (i = 0; i < bp->cp_nr_rings; i++) {
9278 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9279 
9280 		irq = &bp->irq_tbl[map_idx];
9281 		if (irq->requested) {
9282 			if (irq->have_cpumask) {
9283 				irq_set_affinity_hint(irq->vector, NULL);
9284 				free_cpumask_var(irq->cpu_mask);
9285 				irq->have_cpumask = 0;
9286 			}
9287 			free_irq(irq->vector, bp->bnapi[i]);
9288 		}
9289 
9290 		irq->requested = 0;
9291 	}
9292 }
9293 
9294 static int bnxt_request_irq(struct bnxt *bp)
9295 {
9296 	int i, j, rc = 0;
9297 	unsigned long flags = 0;
9298 #ifdef CONFIG_RFS_ACCEL
9299 	struct cpu_rmap *rmap;
9300 #endif
9301 
9302 	rc = bnxt_setup_int_mode(bp);
9303 	if (rc) {
9304 		netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
9305 			   rc);
9306 		return rc;
9307 	}
9308 #ifdef CONFIG_RFS_ACCEL
9309 	rmap = bp->dev->rx_cpu_rmap;
9310 #endif
9311 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
9312 		flags = IRQF_SHARED;
9313 
9314 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
9315 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9316 		struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
9317 
9318 #ifdef CONFIG_RFS_ACCEL
9319 		if (rmap && bp->bnapi[i]->rx_ring) {
9320 			rc = irq_cpu_rmap_add(rmap, irq->vector);
9321 			if (rc)
9322 				netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
9323 					    j);
9324 			j++;
9325 		}
9326 #endif
9327 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
9328 				 bp->bnapi[i]);
9329 		if (rc)
9330 			break;
9331 
9332 		irq->requested = 1;
9333 
9334 		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
9335 			int numa_node = dev_to_node(&bp->pdev->dev);
9336 
9337 			irq->have_cpumask = 1;
9338 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
9339 					irq->cpu_mask);
9340 			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
9341 			if (rc) {
9342 				netdev_warn(bp->dev,
9343 					    "Set affinity failed, IRQ = %d\n",
9344 					    irq->vector);
9345 				break;
9346 			}
9347 		}
9348 	}
9349 	return rc;
9350 }
9351 
9352 static void bnxt_del_napi(struct bnxt *bp)
9353 {
9354 	int i;
9355 
9356 	if (!bp->bnapi)
9357 		return;
9358 
9359 	for (i = 0; i < bp->cp_nr_rings; i++) {
9360 		struct bnxt_napi *bnapi = bp->bnapi[i];
9361 
9362 		__netif_napi_del(&bnapi->napi);
9363 	}
9364 	/* We called __netif_napi_del(), we need
9365 	 * to respect an RCU grace period before freeing napi structures.
9366 	 */
9367 	synchronize_net();
9368 }
9369 
9370 static void bnxt_init_napi(struct bnxt *bp)
9371 {
9372 	int i;
9373 	unsigned int cp_nr_rings = bp->cp_nr_rings;
9374 	struct bnxt_napi *bnapi;
9375 
9376 	if (bp->flags & BNXT_FLAG_USING_MSIX) {
9377 		int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9378 
9379 		if (bp->flags & BNXT_FLAG_CHIP_P5)
9380 			poll_fn = bnxt_poll_p5;
9381 		else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9382 			cp_nr_rings--;
9383 		for (i = 0; i < cp_nr_rings; i++) {
9384 			bnapi = bp->bnapi[i];
9385 			netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
9386 		}
9387 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9388 			bnapi = bp->bnapi[cp_nr_rings];
9389 			netif_napi_add(bp->dev, &bnapi->napi,
9390 				       bnxt_poll_nitroa0);
9391 		}
9392 	} else {
9393 		bnapi = bp->bnapi[0];
9394 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
9395 	}
9396 }
9397 
9398 static void bnxt_disable_napi(struct bnxt *bp)
9399 {
9400 	int i;
9401 
9402 	if (!bp->bnapi ||
9403 	    test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9404 		return;
9405 
9406 	for (i = 0; i < bp->cp_nr_rings; i++) {
9407 		struct bnxt_napi *bnapi = bp->bnapi[i];
9408 		struct bnxt_cp_ring_info *cpr;
9409 
9410 		cpr = &bnapi->cp_ring;
9411 		if (bnapi->tx_fault)
9412 			cpr->sw_stats.tx.tx_resets++;
9413 		if (bnapi->in_reset)
9414 			cpr->sw_stats.rx.rx_resets++;
9415 		napi_disable(&bnapi->napi);
9416 		if (bnapi->rx_ring)
9417 			cancel_work_sync(&cpr->dim.work);
9418 	}
9419 }
9420 
9421 static void bnxt_enable_napi(struct bnxt *bp)
9422 {
9423 	int i;
9424 
9425 	clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9426 	for (i = 0; i < bp->cp_nr_rings; i++) {
9427 		struct bnxt_napi *bnapi = bp->bnapi[i];
9428 		struct bnxt_cp_ring_info *cpr;
9429 
9430 		bnapi->tx_fault = 0;
9431 
9432 		cpr = &bnapi->cp_ring;
9433 		bnapi->in_reset = false;
9434 
9435 		bnapi->tx_pkts = 0;
9436 
9437 		if (bnapi->rx_ring) {
9438 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9439 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9440 		}
9441 		napi_enable(&bnapi->napi);
9442 	}
9443 }
9444 
9445 void bnxt_tx_disable(struct bnxt *bp)
9446 {
9447 	int i;
9448 	struct bnxt_tx_ring_info *txr;
9449 
9450 	if (bp->tx_ring) {
9451 		for (i = 0; i < bp->tx_nr_rings; i++) {
9452 			txr = &bp->tx_ring[i];
9453 			WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
9454 		}
9455 	}
9456 	/* Make sure napi polls see @dev_state change */
9457 	synchronize_net();
9458 	/* Drop carrier first to prevent TX timeout */
9459 	netif_carrier_off(bp->dev);
9460 	/* Stop all TX queues */
9461 	netif_tx_disable(bp->dev);
9462 }
9463 
9464 void bnxt_tx_enable(struct bnxt *bp)
9465 {
9466 	int i;
9467 	struct bnxt_tx_ring_info *txr;
9468 
9469 	for (i = 0; i < bp->tx_nr_rings; i++) {
9470 		txr = &bp->tx_ring[i];
9471 		WRITE_ONCE(txr->dev_state, 0);
9472 	}
9473 	/* Make sure napi polls see @dev_state change */
9474 	synchronize_net();
9475 	netif_tx_wake_all_queues(bp->dev);
9476 	if (BNXT_LINK_IS_UP(bp))
9477 		netif_carrier_on(bp->dev);
9478 }
9479 
9480 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9481 {
9482 	u8 active_fec = link_info->active_fec_sig_mode &
9483 			PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9484 
9485 	switch (active_fec) {
9486 	default:
9487 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9488 		return "None";
9489 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9490 		return "Clause 74 BaseR";
9491 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9492 		return "Clause 91 RS(528,514)";
9493 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9494 		return "Clause 91 RS544_1XN";
9495 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9496 		return "Clause 91 RS(544,514)";
9497 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9498 		return "Clause 91 RS272_1XN";
9499 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9500 		return "Clause 91 RS(272,257)";
9501 	}
9502 }
9503 
9504 void bnxt_report_link(struct bnxt *bp)
9505 {
9506 	if (BNXT_LINK_IS_UP(bp)) {
9507 		const char *signal = "";
9508 		const char *flow_ctrl;
9509 		const char *duplex;
9510 		u32 speed;
9511 		u16 fec;
9512 
9513 		netif_carrier_on(bp->dev);
9514 		speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9515 		if (speed == SPEED_UNKNOWN) {
9516 			netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9517 			return;
9518 		}
9519 		if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9520 			duplex = "full";
9521 		else
9522 			duplex = "half";
9523 		if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9524 			flow_ctrl = "ON - receive & transmit";
9525 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9526 			flow_ctrl = "ON - transmit";
9527 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9528 			flow_ctrl = "ON - receive";
9529 		else
9530 			flow_ctrl = "none";
9531 		if (bp->link_info.phy_qcfg_resp.option_flags &
9532 		    PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9533 			u8 sig_mode = bp->link_info.active_fec_sig_mode &
9534 				      PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9535 			switch (sig_mode) {
9536 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9537 				signal = "(NRZ) ";
9538 				break;
9539 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9540 				signal = "(PAM4) ";
9541 				break;
9542 			default:
9543 				break;
9544 			}
9545 		}
9546 		netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9547 			    speed, signal, duplex, flow_ctrl);
9548 		if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9549 			netdev_info(bp->dev, "EEE is %s\n",
9550 				    bp->eee.eee_active ? "active" :
9551 							 "not active");
9552 		fec = bp->link_info.fec_cfg;
9553 		if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9554 			netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9555 				    (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9556 				    bnxt_report_fec(&bp->link_info));
9557 	} else {
9558 		netif_carrier_off(bp->dev);
9559 		netdev_err(bp->dev, "NIC Link is Down\n");
9560 	}
9561 }
9562 
9563 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9564 {
9565 	if (!resp->supported_speeds_auto_mode &&
9566 	    !resp->supported_speeds_force_mode &&
9567 	    !resp->supported_pam4_speeds_auto_mode &&
9568 	    !resp->supported_pam4_speeds_force_mode)
9569 		return true;
9570 	return false;
9571 }
9572 
9573 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9574 {
9575 	struct bnxt_link_info *link_info = &bp->link_info;
9576 	struct hwrm_port_phy_qcaps_output *resp;
9577 	struct hwrm_port_phy_qcaps_input *req;
9578 	int rc = 0;
9579 
9580 	if (bp->hwrm_spec_code < 0x10201)
9581 		return 0;
9582 
9583 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
9584 	if (rc)
9585 		return rc;
9586 
9587 	resp = hwrm_req_hold(bp, req);
9588 	rc = hwrm_req_send(bp, req);
9589 	if (rc)
9590 		goto hwrm_phy_qcaps_exit;
9591 
9592 	bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
9593 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9594 		struct ethtool_eee *eee = &bp->eee;
9595 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9596 
9597 		eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9598 		bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9599 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9600 		bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9601 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9602 	}
9603 
9604 	if (bp->hwrm_spec_code >= 0x10a01) {
9605 		if (bnxt_phy_qcaps_no_speed(resp)) {
9606 			link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9607 			netdev_warn(bp->dev, "Ethernet link disabled\n");
9608 		} else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9609 			link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9610 			netdev_info(bp->dev, "Ethernet link enabled\n");
9611 			/* Phy re-enabled, reprobe the speeds */
9612 			link_info->support_auto_speeds = 0;
9613 			link_info->support_pam4_auto_speeds = 0;
9614 		}
9615 	}
9616 	if (resp->supported_speeds_auto_mode)
9617 		link_info->support_auto_speeds =
9618 			le16_to_cpu(resp->supported_speeds_auto_mode);
9619 	if (resp->supported_pam4_speeds_auto_mode)
9620 		link_info->support_pam4_auto_speeds =
9621 			le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9622 
9623 	bp->port_count = resp->port_cnt;
9624 
9625 hwrm_phy_qcaps_exit:
9626 	hwrm_req_drop(bp, req);
9627 	return rc;
9628 }
9629 
9630 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9631 {
9632 	u16 diff = advertising ^ supported;
9633 
9634 	return ((supported | diff) != supported);
9635 }
9636 
9637 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9638 {
9639 	struct bnxt_link_info *link_info = &bp->link_info;
9640 	struct hwrm_port_phy_qcfg_output *resp;
9641 	struct hwrm_port_phy_qcfg_input *req;
9642 	u8 link_state = link_info->link_state;
9643 	bool support_changed = false;
9644 	int rc;
9645 
9646 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
9647 	if (rc)
9648 		return rc;
9649 
9650 	resp = hwrm_req_hold(bp, req);
9651 	rc = hwrm_req_send(bp, req);
9652 	if (rc) {
9653 		hwrm_req_drop(bp, req);
9654 		if (BNXT_VF(bp) && rc == -ENODEV) {
9655 			netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
9656 			rc = 0;
9657 		}
9658 		return rc;
9659 	}
9660 
9661 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9662 	link_info->phy_link_status = resp->link;
9663 	link_info->duplex = resp->duplex_cfg;
9664 	if (bp->hwrm_spec_code >= 0x10800)
9665 		link_info->duplex = resp->duplex_state;
9666 	link_info->pause = resp->pause;
9667 	link_info->auto_mode = resp->auto_mode;
9668 	link_info->auto_pause_setting = resp->auto_pause;
9669 	link_info->lp_pause = resp->link_partner_adv_pause;
9670 	link_info->force_pause_setting = resp->force_pause;
9671 	link_info->duplex_setting = resp->duplex_cfg;
9672 	if (link_info->phy_link_status == BNXT_LINK_LINK)
9673 		link_info->link_speed = le16_to_cpu(resp->link_speed);
9674 	else
9675 		link_info->link_speed = 0;
9676 	link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9677 	link_info->force_pam4_link_speed =
9678 		le16_to_cpu(resp->force_pam4_link_speed);
9679 	link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9680 	link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9681 	link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9682 	link_info->auto_pam4_link_speeds =
9683 		le16_to_cpu(resp->auto_pam4_link_speed_mask);
9684 	link_info->lp_auto_link_speeds =
9685 		le16_to_cpu(resp->link_partner_adv_speeds);
9686 	link_info->lp_auto_pam4_link_speeds =
9687 		resp->link_partner_pam4_adv_speeds;
9688 	link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9689 	link_info->phy_ver[0] = resp->phy_maj;
9690 	link_info->phy_ver[1] = resp->phy_min;
9691 	link_info->phy_ver[2] = resp->phy_bld;
9692 	link_info->media_type = resp->media_type;
9693 	link_info->phy_type = resp->phy_type;
9694 	link_info->transceiver = resp->xcvr_pkg_type;
9695 	link_info->phy_addr = resp->eee_config_phy_addr &
9696 			      PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9697 	link_info->module_status = resp->module_status;
9698 
9699 	if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9700 		struct ethtool_eee *eee = &bp->eee;
9701 		u16 fw_speeds;
9702 
9703 		eee->eee_active = 0;
9704 		if (resp->eee_config_phy_addr &
9705 		    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9706 			eee->eee_active = 1;
9707 			fw_speeds = le16_to_cpu(
9708 				resp->link_partner_adv_eee_link_speed_mask);
9709 			eee->lp_advertised =
9710 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9711 		}
9712 
9713 		/* Pull initial EEE config */
9714 		if (!chng_link_state) {
9715 			if (resp->eee_config_phy_addr &
9716 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9717 				eee->eee_enabled = 1;
9718 
9719 			fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9720 			eee->advertised =
9721 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9722 
9723 			if (resp->eee_config_phy_addr &
9724 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9725 				__le32 tmr;
9726 
9727 				eee->tx_lpi_enabled = 1;
9728 				tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9729 				eee->tx_lpi_timer = le32_to_cpu(tmr) &
9730 					PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9731 			}
9732 		}
9733 	}
9734 
9735 	link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9736 	if (bp->hwrm_spec_code >= 0x10504) {
9737 		link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9738 		link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9739 	}
9740 	/* TODO: need to add more logic to report VF link */
9741 	if (chng_link_state) {
9742 		if (link_info->phy_link_status == BNXT_LINK_LINK)
9743 			link_info->link_state = BNXT_LINK_STATE_UP;
9744 		else
9745 			link_info->link_state = BNXT_LINK_STATE_DOWN;
9746 		if (link_state != link_info->link_state)
9747 			bnxt_report_link(bp);
9748 	} else {
9749 		/* always link down if not require to update link state */
9750 		link_info->link_state = BNXT_LINK_STATE_DOWN;
9751 	}
9752 	hwrm_req_drop(bp, req);
9753 
9754 	if (!BNXT_PHY_CFG_ABLE(bp))
9755 		return 0;
9756 
9757 	/* Check if any advertised speeds are no longer supported. The caller
9758 	 * holds the link_lock mutex, so we can modify link_info settings.
9759 	 */
9760 	if (bnxt_support_dropped(link_info->advertising,
9761 				 link_info->support_auto_speeds)) {
9762 		link_info->advertising = link_info->support_auto_speeds;
9763 		support_changed = true;
9764 	}
9765 	if (bnxt_support_dropped(link_info->advertising_pam4,
9766 				 link_info->support_pam4_auto_speeds)) {
9767 		link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9768 		support_changed = true;
9769 	}
9770 	if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9771 		bnxt_hwrm_set_link_setting(bp, true, false);
9772 	return 0;
9773 }
9774 
9775 static void bnxt_get_port_module_status(struct bnxt *bp)
9776 {
9777 	struct bnxt_link_info *link_info = &bp->link_info;
9778 	struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9779 	u8 module_status;
9780 
9781 	if (bnxt_update_link(bp, true))
9782 		return;
9783 
9784 	module_status = link_info->module_status;
9785 	switch (module_status) {
9786 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9787 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9788 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9789 		netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9790 			    bp->pf.port_id);
9791 		if (bp->hwrm_spec_code >= 0x10201) {
9792 			netdev_warn(bp->dev, "Module part number %s\n",
9793 				    resp->phy_vendor_partnumber);
9794 		}
9795 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9796 			netdev_warn(bp->dev, "TX is disabled\n");
9797 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9798 			netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9799 	}
9800 }
9801 
9802 static void
9803 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9804 {
9805 	if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9806 		if (bp->hwrm_spec_code >= 0x10201)
9807 			req->auto_pause =
9808 				PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9809 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9810 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9811 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9812 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9813 		req->enables |=
9814 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9815 	} else {
9816 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9817 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9818 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9819 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9820 		req->enables |=
9821 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9822 		if (bp->hwrm_spec_code >= 0x10201) {
9823 			req->auto_pause = req->force_pause;
9824 			req->enables |= cpu_to_le32(
9825 				PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9826 		}
9827 	}
9828 }
9829 
9830 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9831 {
9832 	if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9833 		req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9834 		if (bp->link_info.advertising) {
9835 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9836 			req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9837 		}
9838 		if (bp->link_info.advertising_pam4) {
9839 			req->enables |=
9840 				cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9841 			req->auto_link_pam4_speed_mask =
9842 				cpu_to_le16(bp->link_info.advertising_pam4);
9843 		}
9844 		req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9845 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9846 	} else {
9847 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9848 		if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9849 			req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9850 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9851 		} else {
9852 			req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9853 		}
9854 	}
9855 
9856 	/* tell chimp that the setting takes effect immediately */
9857 	req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9858 }
9859 
9860 int bnxt_hwrm_set_pause(struct bnxt *bp)
9861 {
9862 	struct hwrm_port_phy_cfg_input *req;
9863 	int rc;
9864 
9865 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9866 	if (rc)
9867 		return rc;
9868 
9869 	bnxt_hwrm_set_pause_common(bp, req);
9870 
9871 	if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9872 	    bp->link_info.force_link_chng)
9873 		bnxt_hwrm_set_link_common(bp, req);
9874 
9875 	rc = hwrm_req_send(bp, req);
9876 	if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9877 		/* since changing of pause setting doesn't trigger any link
9878 		 * change event, the driver needs to update the current pause
9879 		 * result upon successfully return of the phy_cfg command
9880 		 */
9881 		bp->link_info.pause =
9882 		bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9883 		bp->link_info.auto_pause_setting = 0;
9884 		if (!bp->link_info.force_link_chng)
9885 			bnxt_report_link(bp);
9886 	}
9887 	bp->link_info.force_link_chng = false;
9888 	return rc;
9889 }
9890 
9891 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9892 			      struct hwrm_port_phy_cfg_input *req)
9893 {
9894 	struct ethtool_eee *eee = &bp->eee;
9895 
9896 	if (eee->eee_enabled) {
9897 		u16 eee_speeds;
9898 		u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9899 
9900 		if (eee->tx_lpi_enabled)
9901 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9902 		else
9903 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9904 
9905 		req->flags |= cpu_to_le32(flags);
9906 		eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9907 		req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9908 		req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9909 	} else {
9910 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9911 	}
9912 }
9913 
9914 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9915 {
9916 	struct hwrm_port_phy_cfg_input *req;
9917 	int rc;
9918 
9919 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9920 	if (rc)
9921 		return rc;
9922 
9923 	if (set_pause)
9924 		bnxt_hwrm_set_pause_common(bp, req);
9925 
9926 	bnxt_hwrm_set_link_common(bp, req);
9927 
9928 	if (set_eee)
9929 		bnxt_hwrm_set_eee(bp, req);
9930 	return hwrm_req_send(bp, req);
9931 }
9932 
9933 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9934 {
9935 	struct hwrm_port_phy_cfg_input *req;
9936 	int rc;
9937 
9938 	if (!BNXT_SINGLE_PF(bp))
9939 		return 0;
9940 
9941 	if (pci_num_vf(bp->pdev) &&
9942 	    !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9943 		return 0;
9944 
9945 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9946 	if (rc)
9947 		return rc;
9948 
9949 	req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9950 	rc = hwrm_req_send(bp, req);
9951 	if (!rc) {
9952 		mutex_lock(&bp->link_lock);
9953 		/* Device is not obliged link down in certain scenarios, even
9954 		 * when forced. Setting the state unknown is consistent with
9955 		 * driver startup and will force link state to be reported
9956 		 * during subsequent open based on PORT_PHY_QCFG.
9957 		 */
9958 		bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
9959 		mutex_unlock(&bp->link_lock);
9960 	}
9961 	return rc;
9962 }
9963 
9964 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9965 {
9966 #ifdef CONFIG_TEE_BNXT_FW
9967 	int rc = tee_bnxt_fw_load();
9968 
9969 	if (rc)
9970 		netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9971 
9972 	return rc;
9973 #else
9974 	netdev_err(bp->dev, "OP-TEE not supported\n");
9975 	return -ENODEV;
9976 #endif
9977 }
9978 
9979 static int bnxt_try_recover_fw(struct bnxt *bp)
9980 {
9981 	if (bp->fw_health && bp->fw_health->status_reliable) {
9982 		int retry = 0, rc;
9983 		u32 sts;
9984 
9985 		do {
9986 			sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9987 			rc = bnxt_hwrm_poll(bp);
9988 			if (!BNXT_FW_IS_BOOTING(sts) &&
9989 			    !BNXT_FW_IS_RECOVERING(sts))
9990 				break;
9991 			retry++;
9992 		} while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9993 
9994 		if (!BNXT_FW_IS_HEALTHY(sts)) {
9995 			netdev_err(bp->dev,
9996 				   "Firmware not responding, status: 0x%x\n",
9997 				   sts);
9998 			rc = -ENODEV;
9999 		}
10000 		if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
10001 			netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
10002 			return bnxt_fw_reset_via_optee(bp);
10003 		}
10004 		return rc;
10005 	}
10006 
10007 	return -ENODEV;
10008 }
10009 
10010 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
10011 {
10012 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10013 
10014 	if (!BNXT_NEW_RM(bp))
10015 		return; /* no resource reservations required */
10016 
10017 	hw_resc->resv_cp_rings = 0;
10018 	hw_resc->resv_stat_ctxs = 0;
10019 	hw_resc->resv_irqs = 0;
10020 	hw_resc->resv_tx_rings = 0;
10021 	hw_resc->resv_rx_rings = 0;
10022 	hw_resc->resv_hw_ring_grps = 0;
10023 	hw_resc->resv_vnics = 0;
10024 	if (!fw_reset) {
10025 		bp->tx_nr_rings = 0;
10026 		bp->rx_nr_rings = 0;
10027 	}
10028 }
10029 
10030 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
10031 {
10032 	int rc;
10033 
10034 	if (!BNXT_NEW_RM(bp))
10035 		return 0; /* no resource reservations required */
10036 
10037 	rc = bnxt_hwrm_func_resc_qcaps(bp, true);
10038 	if (rc)
10039 		netdev_err(bp->dev, "resc_qcaps failed\n");
10040 
10041 	bnxt_clear_reservations(bp, fw_reset);
10042 
10043 	return rc;
10044 }
10045 
10046 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
10047 {
10048 	struct hwrm_func_drv_if_change_output *resp;
10049 	struct hwrm_func_drv_if_change_input *req;
10050 	bool fw_reset = !bp->irq_tbl;
10051 	bool resc_reinit = false;
10052 	int rc, retry = 0;
10053 	u32 flags = 0;
10054 
10055 	if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
10056 		return 0;
10057 
10058 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
10059 	if (rc)
10060 		return rc;
10061 
10062 	if (up)
10063 		req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
10064 	resp = hwrm_req_hold(bp, req);
10065 
10066 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
10067 	while (retry < BNXT_FW_IF_RETRY) {
10068 		rc = hwrm_req_send(bp, req);
10069 		if (rc != -EAGAIN)
10070 			break;
10071 
10072 		msleep(50);
10073 		retry++;
10074 	}
10075 
10076 	if (rc == -EAGAIN) {
10077 		hwrm_req_drop(bp, req);
10078 		return rc;
10079 	} else if (!rc) {
10080 		flags = le32_to_cpu(resp->flags);
10081 	} else if (up) {
10082 		rc = bnxt_try_recover_fw(bp);
10083 		fw_reset = true;
10084 	}
10085 	hwrm_req_drop(bp, req);
10086 	if (rc)
10087 		return rc;
10088 
10089 	if (!up) {
10090 		bnxt_inv_fw_health_reg(bp);
10091 		return 0;
10092 	}
10093 
10094 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
10095 		resc_reinit = true;
10096 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
10097 	    test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
10098 		fw_reset = true;
10099 	else
10100 		bnxt_remap_fw_health_regs(bp);
10101 
10102 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
10103 		netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
10104 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10105 		return -ENODEV;
10106 	}
10107 	if (resc_reinit || fw_reset) {
10108 		if (fw_reset) {
10109 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10110 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10111 				bnxt_ulp_stop(bp);
10112 			bnxt_free_ctx_mem(bp);
10113 			kfree(bp->ctx);
10114 			bp->ctx = NULL;
10115 			bnxt_dcb_free(bp);
10116 			rc = bnxt_fw_init_one(bp);
10117 			if (rc) {
10118 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10119 				set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10120 				return rc;
10121 			}
10122 			bnxt_clear_int_mode(bp);
10123 			rc = bnxt_init_int_mode(bp);
10124 			if (rc) {
10125 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10126 				netdev_err(bp->dev, "init int mode failed\n");
10127 				return rc;
10128 			}
10129 		}
10130 		rc = bnxt_cancel_reservations(bp, fw_reset);
10131 	}
10132 	return rc;
10133 }
10134 
10135 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
10136 {
10137 	struct hwrm_port_led_qcaps_output *resp;
10138 	struct hwrm_port_led_qcaps_input *req;
10139 	struct bnxt_pf_info *pf = &bp->pf;
10140 	int rc;
10141 
10142 	bp->num_leds = 0;
10143 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
10144 		return 0;
10145 
10146 	rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
10147 	if (rc)
10148 		return rc;
10149 
10150 	req->port_id = cpu_to_le16(pf->port_id);
10151 	resp = hwrm_req_hold(bp, req);
10152 	rc = hwrm_req_send(bp, req);
10153 	if (rc) {
10154 		hwrm_req_drop(bp, req);
10155 		return rc;
10156 	}
10157 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
10158 		int i;
10159 
10160 		bp->num_leds = resp->num_leds;
10161 		memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
10162 						 bp->num_leds);
10163 		for (i = 0; i < bp->num_leds; i++) {
10164 			struct bnxt_led_info *led = &bp->leds[i];
10165 			__le16 caps = led->led_state_caps;
10166 
10167 			if (!led->led_group_id ||
10168 			    !BNXT_LED_ALT_BLINK_CAP(caps)) {
10169 				bp->num_leds = 0;
10170 				break;
10171 			}
10172 		}
10173 	}
10174 	hwrm_req_drop(bp, req);
10175 	return 0;
10176 }
10177 
10178 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
10179 {
10180 	struct hwrm_wol_filter_alloc_output *resp;
10181 	struct hwrm_wol_filter_alloc_input *req;
10182 	int rc;
10183 
10184 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
10185 	if (rc)
10186 		return rc;
10187 
10188 	req->port_id = cpu_to_le16(bp->pf.port_id);
10189 	req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
10190 	req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
10191 	memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
10192 
10193 	resp = hwrm_req_hold(bp, req);
10194 	rc = hwrm_req_send(bp, req);
10195 	if (!rc)
10196 		bp->wol_filter_id = resp->wol_filter_id;
10197 	hwrm_req_drop(bp, req);
10198 	return rc;
10199 }
10200 
10201 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
10202 {
10203 	struct hwrm_wol_filter_free_input *req;
10204 	int rc;
10205 
10206 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
10207 	if (rc)
10208 		return rc;
10209 
10210 	req->port_id = cpu_to_le16(bp->pf.port_id);
10211 	req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
10212 	req->wol_filter_id = bp->wol_filter_id;
10213 
10214 	return hwrm_req_send(bp, req);
10215 }
10216 
10217 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
10218 {
10219 	struct hwrm_wol_filter_qcfg_output *resp;
10220 	struct hwrm_wol_filter_qcfg_input *req;
10221 	u16 next_handle = 0;
10222 	int rc;
10223 
10224 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
10225 	if (rc)
10226 		return rc;
10227 
10228 	req->port_id = cpu_to_le16(bp->pf.port_id);
10229 	req->handle = cpu_to_le16(handle);
10230 	resp = hwrm_req_hold(bp, req);
10231 	rc = hwrm_req_send(bp, req);
10232 	if (!rc) {
10233 		next_handle = le16_to_cpu(resp->next_handle);
10234 		if (next_handle != 0) {
10235 			if (resp->wol_type ==
10236 			    WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
10237 				bp->wol = 1;
10238 				bp->wol_filter_id = resp->wol_filter_id;
10239 			}
10240 		}
10241 	}
10242 	hwrm_req_drop(bp, req);
10243 	return next_handle;
10244 }
10245 
10246 static void bnxt_get_wol_settings(struct bnxt *bp)
10247 {
10248 	u16 handle = 0;
10249 
10250 	bp->wol = 0;
10251 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
10252 		return;
10253 
10254 	do {
10255 		handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
10256 	} while (handle && handle != 0xffff);
10257 }
10258 
10259 #ifdef CONFIG_BNXT_HWMON
10260 static ssize_t bnxt_show_temp(struct device *dev,
10261 			      struct device_attribute *devattr, char *buf)
10262 {
10263 	struct hwrm_temp_monitor_query_output *resp;
10264 	struct hwrm_temp_monitor_query_input *req;
10265 	struct bnxt *bp = dev_get_drvdata(dev);
10266 	u32 len = 0;
10267 	int rc;
10268 
10269 	rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10270 	if (rc)
10271 		return rc;
10272 	resp = hwrm_req_hold(bp, req);
10273 	rc = hwrm_req_send(bp, req);
10274 	if (!rc)
10275 		len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
10276 	hwrm_req_drop(bp, req);
10277 	if (rc)
10278 		return rc;
10279 	return len;
10280 }
10281 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
10282 
10283 static struct attribute *bnxt_attrs[] = {
10284 	&sensor_dev_attr_temp1_input.dev_attr.attr,
10285 	NULL
10286 };
10287 ATTRIBUTE_GROUPS(bnxt);
10288 
10289 static void bnxt_hwmon_close(struct bnxt *bp)
10290 {
10291 	if (bp->hwmon_dev) {
10292 		hwmon_device_unregister(bp->hwmon_dev);
10293 		bp->hwmon_dev = NULL;
10294 	}
10295 }
10296 
10297 static void bnxt_hwmon_open(struct bnxt *bp)
10298 {
10299 	struct hwrm_temp_monitor_query_input *req;
10300 	struct pci_dev *pdev = bp->pdev;
10301 	int rc;
10302 
10303 	rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10304 	if (!rc)
10305 		rc = hwrm_req_send_silent(bp, req);
10306 	if (rc == -EACCES || rc == -EOPNOTSUPP) {
10307 		bnxt_hwmon_close(bp);
10308 		return;
10309 	}
10310 
10311 	if (bp->hwmon_dev)
10312 		return;
10313 
10314 	bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
10315 							  DRV_MODULE_NAME, bp,
10316 							  bnxt_groups);
10317 	if (IS_ERR(bp->hwmon_dev)) {
10318 		bp->hwmon_dev = NULL;
10319 		dev_warn(&pdev->dev, "Cannot register hwmon device\n");
10320 	}
10321 }
10322 #else
10323 static void bnxt_hwmon_close(struct bnxt *bp)
10324 {
10325 }
10326 
10327 static void bnxt_hwmon_open(struct bnxt *bp)
10328 {
10329 }
10330 #endif
10331 
10332 static bool bnxt_eee_config_ok(struct bnxt *bp)
10333 {
10334 	struct ethtool_eee *eee = &bp->eee;
10335 	struct bnxt_link_info *link_info = &bp->link_info;
10336 
10337 	if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
10338 		return true;
10339 
10340 	if (eee->eee_enabled) {
10341 		u32 advertising =
10342 			_bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
10343 
10344 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10345 			eee->eee_enabled = 0;
10346 			return false;
10347 		}
10348 		if (eee->advertised & ~advertising) {
10349 			eee->advertised = advertising & eee->supported;
10350 			return false;
10351 		}
10352 	}
10353 	return true;
10354 }
10355 
10356 static int bnxt_update_phy_setting(struct bnxt *bp)
10357 {
10358 	int rc;
10359 	bool update_link = false;
10360 	bool update_pause = false;
10361 	bool update_eee = false;
10362 	struct bnxt_link_info *link_info = &bp->link_info;
10363 
10364 	rc = bnxt_update_link(bp, true);
10365 	if (rc) {
10366 		netdev_err(bp->dev, "failed to update link (rc: %x)\n",
10367 			   rc);
10368 		return rc;
10369 	}
10370 	if (!BNXT_SINGLE_PF(bp))
10371 		return 0;
10372 
10373 	if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10374 	    (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
10375 	    link_info->req_flow_ctrl)
10376 		update_pause = true;
10377 	if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10378 	    link_info->force_pause_setting != link_info->req_flow_ctrl)
10379 		update_pause = true;
10380 	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10381 		if (BNXT_AUTO_MODE(link_info->auto_mode))
10382 			update_link = true;
10383 		if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
10384 		    link_info->req_link_speed != link_info->force_link_speed)
10385 			update_link = true;
10386 		else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
10387 			 link_info->req_link_speed != link_info->force_pam4_link_speed)
10388 			update_link = true;
10389 		if (link_info->req_duplex != link_info->duplex_setting)
10390 			update_link = true;
10391 	} else {
10392 		if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
10393 			update_link = true;
10394 		if (link_info->advertising != link_info->auto_link_speeds ||
10395 		    link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
10396 			update_link = true;
10397 	}
10398 
10399 	/* The last close may have shutdown the link, so need to call
10400 	 * PHY_CFG to bring it back up.
10401 	 */
10402 	if (!BNXT_LINK_IS_UP(bp))
10403 		update_link = true;
10404 
10405 	if (!bnxt_eee_config_ok(bp))
10406 		update_eee = true;
10407 
10408 	if (update_link)
10409 		rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
10410 	else if (update_pause)
10411 		rc = bnxt_hwrm_set_pause(bp);
10412 	if (rc) {
10413 		netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
10414 			   rc);
10415 		return rc;
10416 	}
10417 
10418 	return rc;
10419 }
10420 
10421 /* Common routine to pre-map certain register block to different GRC window.
10422  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10423  * in PF and 3 windows in VF that can be customized to map in different
10424  * register blocks.
10425  */
10426 static void bnxt_preset_reg_win(struct bnxt *bp)
10427 {
10428 	if (BNXT_PF(bp)) {
10429 		/* CAG registers map to GRC window #4 */
10430 		writel(BNXT_CAG_REG_BASE,
10431 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10432 	}
10433 }
10434 
10435 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10436 
10437 static int bnxt_reinit_after_abort(struct bnxt *bp)
10438 {
10439 	int rc;
10440 
10441 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10442 		return -EBUSY;
10443 
10444 	if (bp->dev->reg_state == NETREG_UNREGISTERED)
10445 		return -ENODEV;
10446 
10447 	rc = bnxt_fw_init_one(bp);
10448 	if (!rc) {
10449 		bnxt_clear_int_mode(bp);
10450 		rc = bnxt_init_int_mode(bp);
10451 		if (!rc) {
10452 			clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10453 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10454 		}
10455 	}
10456 	return rc;
10457 }
10458 
10459 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10460 {
10461 	int rc = 0;
10462 
10463 	bnxt_preset_reg_win(bp);
10464 	netif_carrier_off(bp->dev);
10465 	if (irq_re_init) {
10466 		/* Reserve rings now if none were reserved at driver probe. */
10467 		rc = bnxt_init_dflt_ring_mode(bp);
10468 		if (rc) {
10469 			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10470 			return rc;
10471 		}
10472 	}
10473 	rc = bnxt_reserve_rings(bp, irq_re_init);
10474 	if (rc)
10475 		return rc;
10476 	if ((bp->flags & BNXT_FLAG_RFS) &&
10477 	    !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10478 		/* disable RFS if falling back to INTA */
10479 		bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10480 		bp->flags &= ~BNXT_FLAG_RFS;
10481 	}
10482 
10483 	rc = bnxt_alloc_mem(bp, irq_re_init);
10484 	if (rc) {
10485 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10486 		goto open_err_free_mem;
10487 	}
10488 
10489 	if (irq_re_init) {
10490 		bnxt_init_napi(bp);
10491 		rc = bnxt_request_irq(bp);
10492 		if (rc) {
10493 			netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10494 			goto open_err_irq;
10495 		}
10496 	}
10497 
10498 	rc = bnxt_init_nic(bp, irq_re_init);
10499 	if (rc) {
10500 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10501 		goto open_err_irq;
10502 	}
10503 
10504 	bnxt_enable_napi(bp);
10505 	bnxt_debug_dev_init(bp);
10506 
10507 	if (link_re_init) {
10508 		mutex_lock(&bp->link_lock);
10509 		rc = bnxt_update_phy_setting(bp);
10510 		mutex_unlock(&bp->link_lock);
10511 		if (rc) {
10512 			netdev_warn(bp->dev, "failed to update phy settings\n");
10513 			if (BNXT_SINGLE_PF(bp)) {
10514 				bp->link_info.phy_retry = true;
10515 				bp->link_info.phy_retry_expires =
10516 					jiffies + 5 * HZ;
10517 			}
10518 		}
10519 	}
10520 
10521 	if (irq_re_init)
10522 		udp_tunnel_nic_reset_ntf(bp->dev);
10523 
10524 	if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
10525 		if (!static_key_enabled(&bnxt_xdp_locking_key))
10526 			static_branch_enable(&bnxt_xdp_locking_key);
10527 	} else if (static_key_enabled(&bnxt_xdp_locking_key)) {
10528 		static_branch_disable(&bnxt_xdp_locking_key);
10529 	}
10530 	set_bit(BNXT_STATE_OPEN, &bp->state);
10531 	bnxt_enable_int(bp);
10532 	/* Enable TX queues */
10533 	bnxt_tx_enable(bp);
10534 	mod_timer(&bp->timer, jiffies + bp->current_interval);
10535 	/* Poll link status and check for SFP+ module status */
10536 	mutex_lock(&bp->link_lock);
10537 	bnxt_get_port_module_status(bp);
10538 	mutex_unlock(&bp->link_lock);
10539 
10540 	/* VF-reps may need to be re-opened after the PF is re-opened */
10541 	if (BNXT_PF(bp))
10542 		bnxt_vf_reps_open(bp);
10543 	if (bp->ptp_cfg)
10544 		atomic_set(&bp->ptp_cfg->tx_avail, BNXT_MAX_TX_TS);
10545 	bnxt_ptp_init_rtc(bp, true);
10546 	bnxt_ptp_cfg_tstamp_filters(bp);
10547 	return 0;
10548 
10549 open_err_irq:
10550 	bnxt_del_napi(bp);
10551 
10552 open_err_free_mem:
10553 	bnxt_free_skbs(bp);
10554 	bnxt_free_irq(bp);
10555 	bnxt_free_mem(bp, true);
10556 	return rc;
10557 }
10558 
10559 /* rtnl_lock held */
10560 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10561 {
10562 	int rc = 0;
10563 
10564 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10565 		rc = -EIO;
10566 	if (!rc)
10567 		rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10568 	if (rc) {
10569 		netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10570 		dev_close(bp->dev);
10571 	}
10572 	return rc;
10573 }
10574 
10575 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10576  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10577  * self tests.
10578  */
10579 int bnxt_half_open_nic(struct bnxt *bp)
10580 {
10581 	int rc = 0;
10582 
10583 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10584 		netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
10585 		rc = -ENODEV;
10586 		goto half_open_err;
10587 	}
10588 
10589 	rc = bnxt_alloc_mem(bp, true);
10590 	if (rc) {
10591 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10592 		goto half_open_err;
10593 	}
10594 	bnxt_init_napi(bp);
10595 	set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10596 	rc = bnxt_init_nic(bp, true);
10597 	if (rc) {
10598 		clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10599 		bnxt_del_napi(bp);
10600 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10601 		goto half_open_err;
10602 	}
10603 	return 0;
10604 
10605 half_open_err:
10606 	bnxt_free_skbs(bp);
10607 	bnxt_free_mem(bp, true);
10608 	dev_close(bp->dev);
10609 	return rc;
10610 }
10611 
10612 /* rtnl_lock held, this call can only be made after a previous successful
10613  * call to bnxt_half_open_nic().
10614  */
10615 void bnxt_half_close_nic(struct bnxt *bp)
10616 {
10617 	bnxt_hwrm_resource_free(bp, false, true);
10618 	bnxt_del_napi(bp);
10619 	bnxt_free_skbs(bp);
10620 	bnxt_free_mem(bp, true);
10621 	clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10622 }
10623 
10624 void bnxt_reenable_sriov(struct bnxt *bp)
10625 {
10626 	if (BNXT_PF(bp)) {
10627 		struct bnxt_pf_info *pf = &bp->pf;
10628 		int n = pf->active_vfs;
10629 
10630 		if (n)
10631 			bnxt_cfg_hw_sriov(bp, &n, true);
10632 	}
10633 }
10634 
10635 static int bnxt_open(struct net_device *dev)
10636 {
10637 	struct bnxt *bp = netdev_priv(dev);
10638 	int rc;
10639 
10640 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10641 		rc = bnxt_reinit_after_abort(bp);
10642 		if (rc) {
10643 			if (rc == -EBUSY)
10644 				netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10645 			else
10646 				netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10647 			return -ENODEV;
10648 		}
10649 	}
10650 
10651 	rc = bnxt_hwrm_if_change(bp, true);
10652 	if (rc)
10653 		return rc;
10654 
10655 	rc = __bnxt_open_nic(bp, true, true);
10656 	if (rc) {
10657 		bnxt_hwrm_if_change(bp, false);
10658 	} else {
10659 		if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10660 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10661 				bnxt_ulp_start(bp, 0);
10662 				bnxt_reenable_sriov(bp);
10663 			}
10664 		}
10665 		bnxt_hwmon_open(bp);
10666 	}
10667 
10668 	return rc;
10669 }
10670 
10671 static bool bnxt_drv_busy(struct bnxt *bp)
10672 {
10673 	return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10674 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
10675 }
10676 
10677 static void bnxt_get_ring_stats(struct bnxt *bp,
10678 				struct rtnl_link_stats64 *stats);
10679 
10680 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10681 			     bool link_re_init)
10682 {
10683 	/* Close the VF-reps before closing PF */
10684 	if (BNXT_PF(bp))
10685 		bnxt_vf_reps_close(bp);
10686 
10687 	/* Change device state to avoid TX queue wake up's */
10688 	bnxt_tx_disable(bp);
10689 
10690 	clear_bit(BNXT_STATE_OPEN, &bp->state);
10691 	smp_mb__after_atomic();
10692 	while (bnxt_drv_busy(bp))
10693 		msleep(20);
10694 
10695 	/* Flush rings and disable interrupts */
10696 	bnxt_shutdown_nic(bp, irq_re_init);
10697 
10698 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10699 
10700 	bnxt_debug_dev_exit(bp);
10701 	bnxt_disable_napi(bp);
10702 	del_timer_sync(&bp->timer);
10703 	bnxt_free_skbs(bp);
10704 
10705 	/* Save ring stats before shutdown */
10706 	if (bp->bnapi && irq_re_init) {
10707 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10708 		bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
10709 	}
10710 	if (irq_re_init) {
10711 		bnxt_free_irq(bp);
10712 		bnxt_del_napi(bp);
10713 	}
10714 	bnxt_free_mem(bp, irq_re_init);
10715 }
10716 
10717 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10718 {
10719 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10720 		/* If we get here, it means firmware reset is in progress
10721 		 * while we are trying to close.  We can safely proceed with
10722 		 * the close because we are holding rtnl_lock().  Some firmware
10723 		 * messages may fail as we proceed to close.  We set the
10724 		 * ABORT_ERR flag here so that the FW reset thread will later
10725 		 * abort when it gets the rtnl_lock() and sees the flag.
10726 		 */
10727 		netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10728 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10729 	}
10730 
10731 #ifdef CONFIG_BNXT_SRIOV
10732 	if (bp->sriov_cfg) {
10733 		int rc;
10734 
10735 		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10736 						      !bp->sriov_cfg,
10737 						      BNXT_SRIOV_CFG_WAIT_TMO);
10738 		if (!rc)
10739 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
10740 		else if (rc < 0)
10741 			netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
10742 	}
10743 #endif
10744 	__bnxt_close_nic(bp, irq_re_init, link_re_init);
10745 }
10746 
10747 static int bnxt_close(struct net_device *dev)
10748 {
10749 	struct bnxt *bp = netdev_priv(dev);
10750 
10751 	bnxt_hwmon_close(bp);
10752 	bnxt_close_nic(bp, true, true);
10753 	bnxt_hwrm_shutdown_link(bp);
10754 	bnxt_hwrm_if_change(bp, false);
10755 	return 0;
10756 }
10757 
10758 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10759 				   u16 *val)
10760 {
10761 	struct hwrm_port_phy_mdio_read_output *resp;
10762 	struct hwrm_port_phy_mdio_read_input *req;
10763 	int rc;
10764 
10765 	if (bp->hwrm_spec_code < 0x10a00)
10766 		return -EOPNOTSUPP;
10767 
10768 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
10769 	if (rc)
10770 		return rc;
10771 
10772 	req->port_id = cpu_to_le16(bp->pf.port_id);
10773 	req->phy_addr = phy_addr;
10774 	req->reg_addr = cpu_to_le16(reg & 0x1f);
10775 	if (mdio_phy_id_is_c45(phy_addr)) {
10776 		req->cl45_mdio = 1;
10777 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
10778 		req->dev_addr = mdio_phy_id_devad(phy_addr);
10779 		req->reg_addr = cpu_to_le16(reg);
10780 	}
10781 
10782 	resp = hwrm_req_hold(bp, req);
10783 	rc = hwrm_req_send(bp, req);
10784 	if (!rc)
10785 		*val = le16_to_cpu(resp->reg_data);
10786 	hwrm_req_drop(bp, req);
10787 	return rc;
10788 }
10789 
10790 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10791 				    u16 val)
10792 {
10793 	struct hwrm_port_phy_mdio_write_input *req;
10794 	int rc;
10795 
10796 	if (bp->hwrm_spec_code < 0x10a00)
10797 		return -EOPNOTSUPP;
10798 
10799 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
10800 	if (rc)
10801 		return rc;
10802 
10803 	req->port_id = cpu_to_le16(bp->pf.port_id);
10804 	req->phy_addr = phy_addr;
10805 	req->reg_addr = cpu_to_le16(reg & 0x1f);
10806 	if (mdio_phy_id_is_c45(phy_addr)) {
10807 		req->cl45_mdio = 1;
10808 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
10809 		req->dev_addr = mdio_phy_id_devad(phy_addr);
10810 		req->reg_addr = cpu_to_le16(reg);
10811 	}
10812 	req->reg_data = cpu_to_le16(val);
10813 
10814 	return hwrm_req_send(bp, req);
10815 }
10816 
10817 /* rtnl_lock held */
10818 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10819 {
10820 	struct mii_ioctl_data *mdio = if_mii(ifr);
10821 	struct bnxt *bp = netdev_priv(dev);
10822 	int rc;
10823 
10824 	switch (cmd) {
10825 	case SIOCGMIIPHY:
10826 		mdio->phy_id = bp->link_info.phy_addr;
10827 
10828 		fallthrough;
10829 	case SIOCGMIIREG: {
10830 		u16 mii_regval = 0;
10831 
10832 		if (!netif_running(dev))
10833 			return -EAGAIN;
10834 
10835 		rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10836 					     &mii_regval);
10837 		mdio->val_out = mii_regval;
10838 		return rc;
10839 	}
10840 
10841 	case SIOCSMIIREG:
10842 		if (!netif_running(dev))
10843 			return -EAGAIN;
10844 
10845 		return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10846 						mdio->val_in);
10847 
10848 	case SIOCSHWTSTAMP:
10849 		return bnxt_hwtstamp_set(dev, ifr);
10850 
10851 	case SIOCGHWTSTAMP:
10852 		return bnxt_hwtstamp_get(dev, ifr);
10853 
10854 	default:
10855 		/* do nothing */
10856 		break;
10857 	}
10858 	return -EOPNOTSUPP;
10859 }
10860 
10861 static void bnxt_get_ring_stats(struct bnxt *bp,
10862 				struct rtnl_link_stats64 *stats)
10863 {
10864 	int i;
10865 
10866 	for (i = 0; i < bp->cp_nr_rings; i++) {
10867 		struct bnxt_napi *bnapi = bp->bnapi[i];
10868 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10869 		u64 *sw = cpr->stats.sw_stats;
10870 
10871 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10872 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10873 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10874 
10875 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10876 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10877 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10878 
10879 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10880 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10881 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10882 
10883 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10884 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10885 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10886 
10887 		stats->rx_missed_errors +=
10888 			BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10889 
10890 		stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10891 
10892 		stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10893 
10894 		stats->rx_dropped +=
10895 			cpr->sw_stats.rx.rx_netpoll_discards +
10896 			cpr->sw_stats.rx.rx_oom_discards;
10897 	}
10898 }
10899 
10900 static void bnxt_add_prev_stats(struct bnxt *bp,
10901 				struct rtnl_link_stats64 *stats)
10902 {
10903 	struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10904 
10905 	stats->rx_packets += prev_stats->rx_packets;
10906 	stats->tx_packets += prev_stats->tx_packets;
10907 	stats->rx_bytes += prev_stats->rx_bytes;
10908 	stats->tx_bytes += prev_stats->tx_bytes;
10909 	stats->rx_missed_errors += prev_stats->rx_missed_errors;
10910 	stats->multicast += prev_stats->multicast;
10911 	stats->rx_dropped += prev_stats->rx_dropped;
10912 	stats->tx_dropped += prev_stats->tx_dropped;
10913 }
10914 
10915 static void
10916 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10917 {
10918 	struct bnxt *bp = netdev_priv(dev);
10919 
10920 	set_bit(BNXT_STATE_READ_STATS, &bp->state);
10921 	/* Make sure bnxt_close_nic() sees that we are reading stats before
10922 	 * we check the BNXT_STATE_OPEN flag.
10923 	 */
10924 	smp_mb__after_atomic();
10925 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10926 		clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10927 		*stats = bp->net_stats_prev;
10928 		return;
10929 	}
10930 
10931 	bnxt_get_ring_stats(bp, stats);
10932 	bnxt_add_prev_stats(bp, stats);
10933 
10934 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
10935 		u64 *rx = bp->port_stats.sw_stats;
10936 		u64 *tx = bp->port_stats.sw_stats +
10937 			  BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10938 
10939 		stats->rx_crc_errors =
10940 			BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10941 		stats->rx_frame_errors =
10942 			BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10943 		stats->rx_length_errors =
10944 			BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10945 			BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10946 			BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10947 		stats->rx_errors =
10948 			BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10949 			BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10950 		stats->collisions =
10951 			BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10952 		stats->tx_fifo_errors =
10953 			BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10954 		stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10955 	}
10956 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10957 }
10958 
10959 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
10960 					struct bnxt_total_ring_err_stats *stats,
10961 					struct bnxt_cp_ring_info *cpr)
10962 {
10963 	struct bnxt_sw_stats *sw_stats = &cpr->sw_stats;
10964 	u64 *hw_stats = cpr->stats.sw_stats;
10965 
10966 	stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
10967 	stats->rx_total_resets += sw_stats->rx.rx_resets;
10968 	stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
10969 	stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
10970 	stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
10971 	stats->rx_total_ring_discards +=
10972 		BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
10973 	stats->tx_total_resets += sw_stats->tx.tx_resets;
10974 	stats->tx_total_ring_discards +=
10975 		BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
10976 	stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
10977 }
10978 
10979 void bnxt_get_ring_err_stats(struct bnxt *bp,
10980 			     struct bnxt_total_ring_err_stats *stats)
10981 {
10982 	int i;
10983 
10984 	for (i = 0; i < bp->cp_nr_rings; i++)
10985 		bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
10986 }
10987 
10988 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10989 {
10990 	struct net_device *dev = bp->dev;
10991 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10992 	struct netdev_hw_addr *ha;
10993 	u8 *haddr;
10994 	int mc_count = 0;
10995 	bool update = false;
10996 	int off = 0;
10997 
10998 	netdev_for_each_mc_addr(ha, dev) {
10999 		if (mc_count >= BNXT_MAX_MC_ADDRS) {
11000 			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11001 			vnic->mc_list_count = 0;
11002 			return false;
11003 		}
11004 		haddr = ha->addr;
11005 		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
11006 			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
11007 			update = true;
11008 		}
11009 		off += ETH_ALEN;
11010 		mc_count++;
11011 	}
11012 	if (mc_count)
11013 		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11014 
11015 	if (mc_count != vnic->mc_list_count) {
11016 		vnic->mc_list_count = mc_count;
11017 		update = true;
11018 	}
11019 	return update;
11020 }
11021 
11022 static bool bnxt_uc_list_updated(struct bnxt *bp)
11023 {
11024 	struct net_device *dev = bp->dev;
11025 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11026 	struct netdev_hw_addr *ha;
11027 	int off = 0;
11028 
11029 	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
11030 		return true;
11031 
11032 	netdev_for_each_uc_addr(ha, dev) {
11033 		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
11034 			return true;
11035 
11036 		off += ETH_ALEN;
11037 	}
11038 	return false;
11039 }
11040 
11041 static void bnxt_set_rx_mode(struct net_device *dev)
11042 {
11043 	struct bnxt *bp = netdev_priv(dev);
11044 	struct bnxt_vnic_info *vnic;
11045 	bool mc_update = false;
11046 	bool uc_update;
11047 	u32 mask;
11048 
11049 	if (!test_bit(BNXT_STATE_OPEN, &bp->state))
11050 		return;
11051 
11052 	vnic = &bp->vnic_info[0];
11053 	mask = vnic->rx_mask;
11054 	mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
11055 		  CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
11056 		  CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
11057 		  CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
11058 
11059 	if (dev->flags & IFF_PROMISC)
11060 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11061 
11062 	uc_update = bnxt_uc_list_updated(bp);
11063 
11064 	if (dev->flags & IFF_BROADCAST)
11065 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
11066 	if (dev->flags & IFF_ALLMULTI) {
11067 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11068 		vnic->mc_list_count = 0;
11069 	} else if (dev->flags & IFF_MULTICAST) {
11070 		mc_update = bnxt_mc_list_updated(bp, &mask);
11071 	}
11072 
11073 	if (mask != vnic->rx_mask || uc_update || mc_update) {
11074 		vnic->rx_mask = mask;
11075 
11076 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
11077 	}
11078 }
11079 
11080 static int bnxt_cfg_rx_mode(struct bnxt *bp)
11081 {
11082 	struct net_device *dev = bp->dev;
11083 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11084 	struct hwrm_cfa_l2_filter_free_input *req;
11085 	struct netdev_hw_addr *ha;
11086 	int i, off = 0, rc;
11087 	bool uc_update;
11088 
11089 	netif_addr_lock_bh(dev);
11090 	uc_update = bnxt_uc_list_updated(bp);
11091 	netif_addr_unlock_bh(dev);
11092 
11093 	if (!uc_update)
11094 		goto skip_uc;
11095 
11096 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
11097 	if (rc)
11098 		return rc;
11099 	hwrm_req_hold(bp, req);
11100 	for (i = 1; i < vnic->uc_filter_count; i++) {
11101 		req->l2_filter_id = vnic->fw_l2_filter_id[i];
11102 
11103 		rc = hwrm_req_send(bp, req);
11104 	}
11105 	hwrm_req_drop(bp, req);
11106 
11107 	vnic->uc_filter_count = 1;
11108 
11109 	netif_addr_lock_bh(dev);
11110 	if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
11111 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11112 	} else {
11113 		netdev_for_each_uc_addr(ha, dev) {
11114 			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
11115 			off += ETH_ALEN;
11116 			vnic->uc_filter_count++;
11117 		}
11118 	}
11119 	netif_addr_unlock_bh(dev);
11120 
11121 	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
11122 		rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
11123 		if (rc) {
11124 			if (BNXT_VF(bp) && rc == -ENODEV) {
11125 				if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11126 					netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
11127 				else
11128 					netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
11129 				rc = 0;
11130 			} else {
11131 				netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
11132 			}
11133 			vnic->uc_filter_count = i;
11134 			return rc;
11135 		}
11136 	}
11137 	if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11138 		netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
11139 
11140 skip_uc:
11141 	if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
11142 	    !bnxt_promisc_ok(bp))
11143 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11144 	rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11145 	if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
11146 		netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
11147 			    rc);
11148 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11149 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11150 		vnic->mc_list_count = 0;
11151 		rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11152 	}
11153 	if (rc)
11154 		netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
11155 			   rc);
11156 
11157 	return rc;
11158 }
11159 
11160 static bool bnxt_can_reserve_rings(struct bnxt *bp)
11161 {
11162 #ifdef CONFIG_BNXT_SRIOV
11163 	if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
11164 		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11165 
11166 		/* No minimum rings were provisioned by the PF.  Don't
11167 		 * reserve rings by default when device is down.
11168 		 */
11169 		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
11170 			return true;
11171 
11172 		if (!netif_running(bp->dev))
11173 			return false;
11174 	}
11175 #endif
11176 	return true;
11177 }
11178 
11179 /* If the chip and firmware supports RFS */
11180 static bool bnxt_rfs_supported(struct bnxt *bp)
11181 {
11182 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
11183 		if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
11184 			return true;
11185 		return false;
11186 	}
11187 	/* 212 firmware is broken for aRFS */
11188 	if (BNXT_FW_MAJ(bp) == 212)
11189 		return false;
11190 	if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
11191 		return true;
11192 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11193 		return true;
11194 	return false;
11195 }
11196 
11197 /* If runtime conditions support RFS */
11198 static bool bnxt_rfs_capable(struct bnxt *bp)
11199 {
11200 #ifdef CONFIG_RFS_ACCEL
11201 	int vnics, max_vnics, max_rss_ctxs;
11202 
11203 	if (bp->flags & BNXT_FLAG_CHIP_P5)
11204 		return bnxt_rfs_supported(bp);
11205 	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
11206 		return false;
11207 
11208 	vnics = 1 + bp->rx_nr_rings;
11209 	max_vnics = bnxt_get_max_func_vnics(bp);
11210 	max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
11211 
11212 	/* RSS contexts not a limiting factor */
11213 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11214 		max_rss_ctxs = max_vnics;
11215 	if (vnics > max_vnics || vnics > max_rss_ctxs) {
11216 		if (bp->rx_nr_rings > 1)
11217 			netdev_warn(bp->dev,
11218 				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
11219 				    min(max_rss_ctxs - 1, max_vnics - 1));
11220 		return false;
11221 	}
11222 
11223 	if (!BNXT_NEW_RM(bp))
11224 		return true;
11225 
11226 	if (vnics == bp->hw_resc.resv_vnics)
11227 		return true;
11228 
11229 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
11230 	if (vnics <= bp->hw_resc.resv_vnics)
11231 		return true;
11232 
11233 	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
11234 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
11235 	return false;
11236 #else
11237 	return false;
11238 #endif
11239 }
11240 
11241 static netdev_features_t bnxt_fix_features(struct net_device *dev,
11242 					   netdev_features_t features)
11243 {
11244 	struct bnxt *bp = netdev_priv(dev);
11245 	netdev_features_t vlan_features;
11246 
11247 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
11248 		features &= ~NETIF_F_NTUPLE;
11249 
11250 	if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
11251 		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11252 
11253 	if (!(features & NETIF_F_GRO))
11254 		features &= ~NETIF_F_GRO_HW;
11255 
11256 	if (features & NETIF_F_GRO_HW)
11257 		features &= ~NETIF_F_LRO;
11258 
11259 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
11260 	 * turned on or off together.
11261 	 */
11262 	vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
11263 	if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
11264 		if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11265 			features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11266 		else if (vlan_features)
11267 			features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
11268 	}
11269 #ifdef CONFIG_BNXT_SRIOV
11270 	if (BNXT_VF(bp) && bp->vf.vlan)
11271 		features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11272 #endif
11273 	return features;
11274 }
11275 
11276 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
11277 {
11278 	struct bnxt *bp = netdev_priv(dev);
11279 	u32 flags = bp->flags;
11280 	u32 changes;
11281 	int rc = 0;
11282 	bool re_init = false;
11283 	bool update_tpa = false;
11284 
11285 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
11286 	if (features & NETIF_F_GRO_HW)
11287 		flags |= BNXT_FLAG_GRO;
11288 	else if (features & NETIF_F_LRO)
11289 		flags |= BNXT_FLAG_LRO;
11290 
11291 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11292 		flags &= ~BNXT_FLAG_TPA;
11293 
11294 	if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11295 		flags |= BNXT_FLAG_STRIP_VLAN;
11296 
11297 	if (features & NETIF_F_NTUPLE)
11298 		flags |= BNXT_FLAG_RFS;
11299 
11300 	changes = flags ^ bp->flags;
11301 	if (changes & BNXT_FLAG_TPA) {
11302 		update_tpa = true;
11303 		if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
11304 		    (flags & BNXT_FLAG_TPA) == 0 ||
11305 		    (bp->flags & BNXT_FLAG_CHIP_P5))
11306 			re_init = true;
11307 	}
11308 
11309 	if (changes & ~BNXT_FLAG_TPA)
11310 		re_init = true;
11311 
11312 	if (flags != bp->flags) {
11313 		u32 old_flags = bp->flags;
11314 
11315 		if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11316 			bp->flags = flags;
11317 			if (update_tpa)
11318 				bnxt_set_ring_params(bp);
11319 			return rc;
11320 		}
11321 
11322 		if (re_init) {
11323 			bnxt_close_nic(bp, false, false);
11324 			bp->flags = flags;
11325 			if (update_tpa)
11326 				bnxt_set_ring_params(bp);
11327 
11328 			return bnxt_open_nic(bp, false, false);
11329 		}
11330 		if (update_tpa) {
11331 			bp->flags = flags;
11332 			rc = bnxt_set_tpa(bp,
11333 					  (flags & BNXT_FLAG_TPA) ?
11334 					  true : false);
11335 			if (rc)
11336 				bp->flags = old_flags;
11337 		}
11338 	}
11339 	return rc;
11340 }
11341 
11342 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
11343 			      u8 **nextp)
11344 {
11345 	struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
11346 	struct hop_jumbo_hdr *jhdr;
11347 	int hdr_count = 0;
11348 	u8 *nexthdr;
11349 	int start;
11350 
11351 	/* Check that there are at most 2 IPv6 extension headers, no
11352 	 * fragment header, and each is <= 64 bytes.
11353 	 */
11354 	start = nw_off + sizeof(*ip6h);
11355 	nexthdr = &ip6h->nexthdr;
11356 	while (ipv6_ext_hdr(*nexthdr)) {
11357 		struct ipv6_opt_hdr *hp;
11358 		int hdrlen;
11359 
11360 		if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
11361 		    *nexthdr == NEXTHDR_FRAGMENT)
11362 			return false;
11363 		hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
11364 					  skb_headlen(skb), NULL);
11365 		if (!hp)
11366 			return false;
11367 		if (*nexthdr == NEXTHDR_AUTH)
11368 			hdrlen = ipv6_authlen(hp);
11369 		else
11370 			hdrlen = ipv6_optlen(hp);
11371 
11372 		if (hdrlen > 64)
11373 			return false;
11374 
11375 		/* The ext header may be a hop-by-hop header inserted for
11376 		 * big TCP purposes. This will be removed before sending
11377 		 * from NIC, so do not count it.
11378 		 */
11379 		if (*nexthdr == NEXTHDR_HOP) {
11380 			if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
11381 				goto increment_hdr;
11382 
11383 			jhdr = (struct hop_jumbo_hdr *)hp;
11384 			if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
11385 			    jhdr->nexthdr != IPPROTO_TCP)
11386 				goto increment_hdr;
11387 
11388 			goto next_hdr;
11389 		}
11390 increment_hdr:
11391 		hdr_count++;
11392 next_hdr:
11393 		nexthdr = &hp->nexthdr;
11394 		start += hdrlen;
11395 	}
11396 	if (nextp) {
11397 		/* Caller will check inner protocol */
11398 		if (skb->encapsulation) {
11399 			*nextp = nexthdr;
11400 			return true;
11401 		}
11402 		*nextp = NULL;
11403 	}
11404 	/* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
11405 	return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
11406 }
11407 
11408 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
11409 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
11410 {
11411 	struct udphdr *uh = udp_hdr(skb);
11412 	__be16 udp_port = uh->dest;
11413 
11414 	if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
11415 		return false;
11416 	if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
11417 		struct ethhdr *eh = inner_eth_hdr(skb);
11418 
11419 		switch (eh->h_proto) {
11420 		case htons(ETH_P_IP):
11421 			return true;
11422 		case htons(ETH_P_IPV6):
11423 			return bnxt_exthdr_check(bp, skb,
11424 						 skb_inner_network_offset(skb),
11425 						 NULL);
11426 		}
11427 	}
11428 	return false;
11429 }
11430 
11431 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
11432 {
11433 	switch (l4_proto) {
11434 	case IPPROTO_UDP:
11435 		return bnxt_udp_tunl_check(bp, skb);
11436 	case IPPROTO_IPIP:
11437 		return true;
11438 	case IPPROTO_GRE: {
11439 		switch (skb->inner_protocol) {
11440 		default:
11441 			return false;
11442 		case htons(ETH_P_IP):
11443 			return true;
11444 		case htons(ETH_P_IPV6):
11445 			fallthrough;
11446 		}
11447 	}
11448 	case IPPROTO_IPV6:
11449 		/* Check ext headers of inner ipv6 */
11450 		return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
11451 					 NULL);
11452 	}
11453 	return false;
11454 }
11455 
11456 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
11457 					     struct net_device *dev,
11458 					     netdev_features_t features)
11459 {
11460 	struct bnxt *bp = netdev_priv(dev);
11461 	u8 *l4_proto;
11462 
11463 	features = vlan_features_check(skb, features);
11464 	switch (vlan_get_protocol(skb)) {
11465 	case htons(ETH_P_IP):
11466 		if (!skb->encapsulation)
11467 			return features;
11468 		l4_proto = &ip_hdr(skb)->protocol;
11469 		if (bnxt_tunl_check(bp, skb, *l4_proto))
11470 			return features;
11471 		break;
11472 	case htons(ETH_P_IPV6):
11473 		if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
11474 				       &l4_proto))
11475 			break;
11476 		if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
11477 			return features;
11478 		break;
11479 	}
11480 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11481 }
11482 
11483 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
11484 			 u32 *reg_buf)
11485 {
11486 	struct hwrm_dbg_read_direct_output *resp;
11487 	struct hwrm_dbg_read_direct_input *req;
11488 	__le32 *dbg_reg_buf;
11489 	dma_addr_t mapping;
11490 	int rc, i;
11491 
11492 	rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
11493 	if (rc)
11494 		return rc;
11495 
11496 	dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
11497 					 &mapping);
11498 	if (!dbg_reg_buf) {
11499 		rc = -ENOMEM;
11500 		goto dbg_rd_reg_exit;
11501 	}
11502 
11503 	req->host_dest_addr = cpu_to_le64(mapping);
11504 
11505 	resp = hwrm_req_hold(bp, req);
11506 	req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
11507 	req->read_len32 = cpu_to_le32(num_words);
11508 
11509 	rc = hwrm_req_send(bp, req);
11510 	if (rc || resp->error_code) {
11511 		rc = -EIO;
11512 		goto dbg_rd_reg_exit;
11513 	}
11514 	for (i = 0; i < num_words; i++)
11515 		reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
11516 
11517 dbg_rd_reg_exit:
11518 	hwrm_req_drop(bp, req);
11519 	return rc;
11520 }
11521 
11522 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
11523 				       u32 ring_id, u32 *prod, u32 *cons)
11524 {
11525 	struct hwrm_dbg_ring_info_get_output *resp;
11526 	struct hwrm_dbg_ring_info_get_input *req;
11527 	int rc;
11528 
11529 	rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
11530 	if (rc)
11531 		return rc;
11532 
11533 	req->ring_type = ring_type;
11534 	req->fw_ring_id = cpu_to_le32(ring_id);
11535 	resp = hwrm_req_hold(bp, req);
11536 	rc = hwrm_req_send(bp, req);
11537 	if (!rc) {
11538 		*prod = le32_to_cpu(resp->producer_index);
11539 		*cons = le32_to_cpu(resp->consumer_index);
11540 	}
11541 	hwrm_req_drop(bp, req);
11542 	return rc;
11543 }
11544 
11545 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11546 {
11547 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11548 	int i = bnapi->index;
11549 
11550 	if (!txr)
11551 		return;
11552 
11553 	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11554 		    i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11555 		    txr->tx_cons);
11556 }
11557 
11558 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11559 {
11560 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11561 	int i = bnapi->index;
11562 
11563 	if (!rxr)
11564 		return;
11565 
11566 	netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11567 		    i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11568 		    rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11569 		    rxr->rx_sw_agg_prod);
11570 }
11571 
11572 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11573 {
11574 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11575 	int i = bnapi->index;
11576 
11577 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11578 		    i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11579 }
11580 
11581 static void bnxt_dbg_dump_states(struct bnxt *bp)
11582 {
11583 	int i;
11584 	struct bnxt_napi *bnapi;
11585 
11586 	for (i = 0; i < bp->cp_nr_rings; i++) {
11587 		bnapi = bp->bnapi[i];
11588 		if (netif_msg_drv(bp)) {
11589 			bnxt_dump_tx_sw_state(bnapi);
11590 			bnxt_dump_rx_sw_state(bnapi);
11591 			bnxt_dump_cp_sw_state(bnapi);
11592 		}
11593 	}
11594 }
11595 
11596 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11597 {
11598 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11599 	struct hwrm_ring_reset_input *req;
11600 	struct bnxt_napi *bnapi = rxr->bnapi;
11601 	struct bnxt_cp_ring_info *cpr;
11602 	u16 cp_ring_id;
11603 	int rc;
11604 
11605 	rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
11606 	if (rc)
11607 		return rc;
11608 
11609 	cpr = &bnapi->cp_ring;
11610 	cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11611 	req->cmpl_ring = cpu_to_le16(cp_ring_id);
11612 	req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11613 	req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11614 	return hwrm_req_send_silent(bp, req);
11615 }
11616 
11617 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11618 {
11619 	if (!silent)
11620 		bnxt_dbg_dump_states(bp);
11621 	if (netif_running(bp->dev)) {
11622 		int rc;
11623 
11624 		if (silent) {
11625 			bnxt_close_nic(bp, false, false);
11626 			bnxt_open_nic(bp, false, false);
11627 		} else {
11628 			bnxt_ulp_stop(bp);
11629 			bnxt_close_nic(bp, true, false);
11630 			rc = bnxt_open_nic(bp, true, false);
11631 			bnxt_ulp_start(bp, rc);
11632 		}
11633 	}
11634 }
11635 
11636 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11637 {
11638 	struct bnxt *bp = netdev_priv(dev);
11639 
11640 	netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11641 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
11642 }
11643 
11644 static void bnxt_fw_health_check(struct bnxt *bp)
11645 {
11646 	struct bnxt_fw_health *fw_health = bp->fw_health;
11647 	struct pci_dev *pdev = bp->pdev;
11648 	u32 val;
11649 
11650 	if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11651 		return;
11652 
11653 	/* Make sure it is enabled before checking the tmr_counter. */
11654 	smp_rmb();
11655 	if (fw_health->tmr_counter) {
11656 		fw_health->tmr_counter--;
11657 		return;
11658 	}
11659 
11660 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11661 	if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
11662 		fw_health->arrests++;
11663 		goto fw_reset;
11664 	}
11665 
11666 	fw_health->last_fw_heartbeat = val;
11667 
11668 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11669 	if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
11670 		fw_health->discoveries++;
11671 		goto fw_reset;
11672 	}
11673 
11674 	fw_health->tmr_counter = fw_health->tmr_multiplier;
11675 	return;
11676 
11677 fw_reset:
11678 	bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
11679 }
11680 
11681 static void bnxt_timer(struct timer_list *t)
11682 {
11683 	struct bnxt *bp = from_timer(bp, t, timer);
11684 	struct net_device *dev = bp->dev;
11685 
11686 	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11687 		return;
11688 
11689 	if (atomic_read(&bp->intr_sem) != 0)
11690 		goto bnxt_restart_timer;
11691 
11692 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11693 		bnxt_fw_health_check(bp);
11694 
11695 	if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
11696 		bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
11697 
11698 	if (bnxt_tc_flower_enabled(bp))
11699 		bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
11700 
11701 #ifdef CONFIG_RFS_ACCEL
11702 	if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
11703 		bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
11704 #endif /*CONFIG_RFS_ACCEL*/
11705 
11706 	if (bp->link_info.phy_retry) {
11707 		if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11708 			bp->link_info.phy_retry = false;
11709 			netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11710 		} else {
11711 			bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
11712 		}
11713 	}
11714 
11715 	if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11716 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
11717 
11718 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11719 	    netif_carrier_ok(dev))
11720 		bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
11721 
11722 bnxt_restart_timer:
11723 	mod_timer(&bp->timer, jiffies + bp->current_interval);
11724 }
11725 
11726 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11727 {
11728 	/* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11729 	 * set.  If the device is being closed, bnxt_close() may be holding
11730 	 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11731 	 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11732 	 */
11733 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11734 	rtnl_lock();
11735 }
11736 
11737 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11738 {
11739 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11740 	rtnl_unlock();
11741 }
11742 
11743 /* Only called from bnxt_sp_task() */
11744 static void bnxt_reset(struct bnxt *bp, bool silent)
11745 {
11746 	bnxt_rtnl_lock_sp(bp);
11747 	if (test_bit(BNXT_STATE_OPEN, &bp->state))
11748 		bnxt_reset_task(bp, silent);
11749 	bnxt_rtnl_unlock_sp(bp);
11750 }
11751 
11752 /* Only called from bnxt_sp_task() */
11753 static void bnxt_rx_ring_reset(struct bnxt *bp)
11754 {
11755 	int i;
11756 
11757 	bnxt_rtnl_lock_sp(bp);
11758 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11759 		bnxt_rtnl_unlock_sp(bp);
11760 		return;
11761 	}
11762 	/* Disable and flush TPA before resetting the RX ring */
11763 	if (bp->flags & BNXT_FLAG_TPA)
11764 		bnxt_set_tpa(bp, false);
11765 	for (i = 0; i < bp->rx_nr_rings; i++) {
11766 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11767 		struct bnxt_cp_ring_info *cpr;
11768 		int rc;
11769 
11770 		if (!rxr->bnapi->in_reset)
11771 			continue;
11772 
11773 		rc = bnxt_hwrm_rx_ring_reset(bp, i);
11774 		if (rc) {
11775 			if (rc == -EINVAL || rc == -EOPNOTSUPP)
11776 				netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11777 			else
11778 				netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11779 					    rc);
11780 			bnxt_reset_task(bp, true);
11781 			break;
11782 		}
11783 		bnxt_free_one_rx_ring_skbs(bp, i);
11784 		rxr->rx_prod = 0;
11785 		rxr->rx_agg_prod = 0;
11786 		rxr->rx_sw_agg_prod = 0;
11787 		rxr->rx_next_cons = 0;
11788 		rxr->bnapi->in_reset = false;
11789 		bnxt_alloc_one_rx_ring(bp, i);
11790 		cpr = &rxr->bnapi->cp_ring;
11791 		cpr->sw_stats.rx.rx_resets++;
11792 		if (bp->flags & BNXT_FLAG_AGG_RINGS)
11793 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11794 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11795 	}
11796 	if (bp->flags & BNXT_FLAG_TPA)
11797 		bnxt_set_tpa(bp, true);
11798 	bnxt_rtnl_unlock_sp(bp);
11799 }
11800 
11801 static void bnxt_fw_fatal_close(struct bnxt *bp)
11802 {
11803 	bnxt_tx_disable(bp);
11804 	bnxt_disable_napi(bp);
11805 	bnxt_disable_int_sync(bp);
11806 	bnxt_free_irq(bp);
11807 	bnxt_clear_int_mode(bp);
11808 	pci_disable_device(bp->pdev);
11809 }
11810 
11811 static void bnxt_fw_reset_close(struct bnxt *bp)
11812 {
11813 	bnxt_ulp_stop(bp);
11814 	/* When firmware is in fatal state, quiesce device and disable
11815 	 * bus master to prevent any potential bad DMAs before freeing
11816 	 * kernel memory.
11817 	 */
11818 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11819 		u16 val = 0;
11820 
11821 		pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11822 		if (val == 0xffff)
11823 			bp->fw_reset_min_dsecs = 0;
11824 		bnxt_fw_fatal_close(bp);
11825 	}
11826 	__bnxt_close_nic(bp, true, false);
11827 	bnxt_vf_reps_free(bp);
11828 	bnxt_clear_int_mode(bp);
11829 	bnxt_hwrm_func_drv_unrgtr(bp);
11830 	if (pci_is_enabled(bp->pdev))
11831 		pci_disable_device(bp->pdev);
11832 	bnxt_free_ctx_mem(bp);
11833 	kfree(bp->ctx);
11834 	bp->ctx = NULL;
11835 }
11836 
11837 static bool is_bnxt_fw_ok(struct bnxt *bp)
11838 {
11839 	struct bnxt_fw_health *fw_health = bp->fw_health;
11840 	bool no_heartbeat = false, has_reset = false;
11841 	u32 val;
11842 
11843 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11844 	if (val == fw_health->last_fw_heartbeat)
11845 		no_heartbeat = true;
11846 
11847 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11848 	if (val != fw_health->last_fw_reset_cnt)
11849 		has_reset = true;
11850 
11851 	if (!no_heartbeat && has_reset)
11852 		return true;
11853 
11854 	return false;
11855 }
11856 
11857 /* rtnl_lock is acquired before calling this function */
11858 static void bnxt_force_fw_reset(struct bnxt *bp)
11859 {
11860 	struct bnxt_fw_health *fw_health = bp->fw_health;
11861 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11862 	u32 wait_dsecs;
11863 
11864 	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11865 	    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11866 		return;
11867 
11868 	if (ptp) {
11869 		spin_lock_bh(&ptp->ptp_lock);
11870 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11871 		spin_unlock_bh(&ptp->ptp_lock);
11872 	} else {
11873 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11874 	}
11875 	bnxt_fw_reset_close(bp);
11876 	wait_dsecs = fw_health->master_func_wait_dsecs;
11877 	if (fw_health->primary) {
11878 		if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11879 			wait_dsecs = 0;
11880 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11881 	} else {
11882 		bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11883 		wait_dsecs = fw_health->normal_func_wait_dsecs;
11884 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11885 	}
11886 
11887 	bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11888 	bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11889 	bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11890 }
11891 
11892 void bnxt_fw_exception(struct bnxt *bp)
11893 {
11894 	netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11895 	set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11896 	bnxt_rtnl_lock_sp(bp);
11897 	bnxt_force_fw_reset(bp);
11898 	bnxt_rtnl_unlock_sp(bp);
11899 }
11900 
11901 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11902  * < 0 on error.
11903  */
11904 static int bnxt_get_registered_vfs(struct bnxt *bp)
11905 {
11906 #ifdef CONFIG_BNXT_SRIOV
11907 	int rc;
11908 
11909 	if (!BNXT_PF(bp))
11910 		return 0;
11911 
11912 	rc = bnxt_hwrm_func_qcfg(bp);
11913 	if (rc) {
11914 		netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11915 		return rc;
11916 	}
11917 	if (bp->pf.registered_vfs)
11918 		return bp->pf.registered_vfs;
11919 	if (bp->sriov_cfg)
11920 		return 1;
11921 #endif
11922 	return 0;
11923 }
11924 
11925 void bnxt_fw_reset(struct bnxt *bp)
11926 {
11927 	bnxt_rtnl_lock_sp(bp);
11928 	if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11929 	    !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11930 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11931 		int n = 0, tmo;
11932 
11933 		if (ptp) {
11934 			spin_lock_bh(&ptp->ptp_lock);
11935 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11936 			spin_unlock_bh(&ptp->ptp_lock);
11937 		} else {
11938 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11939 		}
11940 		if (bp->pf.active_vfs &&
11941 		    !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11942 			n = bnxt_get_registered_vfs(bp);
11943 		if (n < 0) {
11944 			netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11945 				   n);
11946 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11947 			dev_close(bp->dev);
11948 			goto fw_reset_exit;
11949 		} else if (n > 0) {
11950 			u16 vf_tmo_dsecs = n * 10;
11951 
11952 			if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11953 				bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11954 			bp->fw_reset_state =
11955 				BNXT_FW_RESET_STATE_POLL_VF;
11956 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11957 			goto fw_reset_exit;
11958 		}
11959 		bnxt_fw_reset_close(bp);
11960 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11961 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11962 			tmo = HZ / 10;
11963 		} else {
11964 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11965 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
11966 		}
11967 		bnxt_queue_fw_reset_work(bp, tmo);
11968 	}
11969 fw_reset_exit:
11970 	bnxt_rtnl_unlock_sp(bp);
11971 }
11972 
11973 static void bnxt_chk_missed_irq(struct bnxt *bp)
11974 {
11975 	int i;
11976 
11977 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11978 		return;
11979 
11980 	for (i = 0; i < bp->cp_nr_rings; i++) {
11981 		struct bnxt_napi *bnapi = bp->bnapi[i];
11982 		struct bnxt_cp_ring_info *cpr;
11983 		u32 fw_ring_id;
11984 		int j;
11985 
11986 		if (!bnapi)
11987 			continue;
11988 
11989 		cpr = &bnapi->cp_ring;
11990 		for (j = 0; j < 2; j++) {
11991 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11992 			u32 val[2];
11993 
11994 			if (!cpr2 || cpr2->has_more_work ||
11995 			    !bnxt_has_work(bp, cpr2))
11996 				continue;
11997 
11998 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
11999 				cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
12000 				continue;
12001 			}
12002 			fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
12003 			bnxt_dbg_hwrm_ring_info_get(bp,
12004 				DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
12005 				fw_ring_id, &val[0], &val[1]);
12006 			cpr->sw_stats.cmn.missed_irqs++;
12007 		}
12008 	}
12009 }
12010 
12011 static void bnxt_cfg_ntp_filters(struct bnxt *);
12012 
12013 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
12014 {
12015 	struct bnxt_link_info *link_info = &bp->link_info;
12016 
12017 	if (BNXT_AUTO_MODE(link_info->auto_mode)) {
12018 		link_info->autoneg = BNXT_AUTONEG_SPEED;
12019 		if (bp->hwrm_spec_code >= 0x10201) {
12020 			if (link_info->auto_pause_setting &
12021 			    PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
12022 				link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12023 		} else {
12024 			link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12025 		}
12026 		link_info->advertising = link_info->auto_link_speeds;
12027 		link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
12028 	} else {
12029 		link_info->req_link_speed = link_info->force_link_speed;
12030 		link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
12031 		if (link_info->force_pam4_link_speed) {
12032 			link_info->req_link_speed =
12033 				link_info->force_pam4_link_speed;
12034 			link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
12035 		}
12036 		link_info->req_duplex = link_info->duplex_setting;
12037 	}
12038 	if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
12039 		link_info->req_flow_ctrl =
12040 			link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
12041 	else
12042 		link_info->req_flow_ctrl = link_info->force_pause_setting;
12043 }
12044 
12045 static void bnxt_fw_echo_reply(struct bnxt *bp)
12046 {
12047 	struct bnxt_fw_health *fw_health = bp->fw_health;
12048 	struct hwrm_func_echo_response_input *req;
12049 	int rc;
12050 
12051 	rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
12052 	if (rc)
12053 		return;
12054 	req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
12055 	req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
12056 	hwrm_req_send(bp, req);
12057 }
12058 
12059 static void bnxt_sp_task(struct work_struct *work)
12060 {
12061 	struct bnxt *bp = container_of(work, struct bnxt, sp_task);
12062 
12063 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12064 	smp_mb__after_atomic();
12065 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12066 		clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12067 		return;
12068 	}
12069 
12070 	if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
12071 		bnxt_cfg_rx_mode(bp);
12072 
12073 	if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
12074 		bnxt_cfg_ntp_filters(bp);
12075 	if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
12076 		bnxt_hwrm_exec_fwd_req(bp);
12077 	if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
12078 		netdev_info(bp->dev, "Receive PF driver unload event!\n");
12079 	if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
12080 		bnxt_hwrm_port_qstats(bp, 0);
12081 		bnxt_hwrm_port_qstats_ext(bp, 0);
12082 		bnxt_accumulate_all_stats(bp);
12083 	}
12084 
12085 	if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
12086 		int rc;
12087 
12088 		mutex_lock(&bp->link_lock);
12089 		if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
12090 				       &bp->sp_event))
12091 			bnxt_hwrm_phy_qcaps(bp);
12092 
12093 		rc = bnxt_update_link(bp, true);
12094 		if (rc)
12095 			netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
12096 				   rc);
12097 
12098 		if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
12099 				       &bp->sp_event))
12100 			bnxt_init_ethtool_link_settings(bp);
12101 		mutex_unlock(&bp->link_lock);
12102 	}
12103 	if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
12104 		int rc;
12105 
12106 		mutex_lock(&bp->link_lock);
12107 		rc = bnxt_update_phy_setting(bp);
12108 		mutex_unlock(&bp->link_lock);
12109 		if (rc) {
12110 			netdev_warn(bp->dev, "update phy settings retry failed\n");
12111 		} else {
12112 			bp->link_info.phy_retry = false;
12113 			netdev_info(bp->dev, "update phy settings retry succeeded\n");
12114 		}
12115 	}
12116 	if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
12117 		mutex_lock(&bp->link_lock);
12118 		bnxt_get_port_module_status(bp);
12119 		mutex_unlock(&bp->link_lock);
12120 	}
12121 
12122 	if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
12123 		bnxt_tc_flow_stats_work(bp);
12124 
12125 	if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
12126 		bnxt_chk_missed_irq(bp);
12127 
12128 	if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
12129 		bnxt_fw_echo_reply(bp);
12130 
12131 	/* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
12132 	 * must be the last functions to be called before exiting.
12133 	 */
12134 	if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
12135 		bnxt_reset(bp, false);
12136 
12137 	if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
12138 		bnxt_reset(bp, true);
12139 
12140 	if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
12141 		bnxt_rx_ring_reset(bp);
12142 
12143 	if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
12144 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
12145 		    test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
12146 			bnxt_devlink_health_fw_report(bp);
12147 		else
12148 			bnxt_fw_reset(bp);
12149 	}
12150 
12151 	if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
12152 		if (!is_bnxt_fw_ok(bp))
12153 			bnxt_devlink_health_fw_report(bp);
12154 	}
12155 
12156 	smp_mb__before_atomic();
12157 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12158 }
12159 
12160 /* Under rtnl_lock */
12161 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
12162 		     int tx_xdp)
12163 {
12164 	int max_rx, max_tx, tx_sets = 1;
12165 	int tx_rings_needed, stats;
12166 	int rx_rings = rx;
12167 	int cp, vnics, rc;
12168 
12169 	if (tcs)
12170 		tx_sets = tcs;
12171 
12172 	rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
12173 	if (rc)
12174 		return rc;
12175 
12176 	if (max_rx < rx)
12177 		return -ENOMEM;
12178 
12179 	tx_rings_needed = tx * tx_sets + tx_xdp;
12180 	if (max_tx < tx_rings_needed)
12181 		return -ENOMEM;
12182 
12183 	vnics = 1;
12184 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
12185 		vnics += rx_rings;
12186 
12187 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
12188 		rx_rings <<= 1;
12189 	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
12190 	stats = cp;
12191 	if (BNXT_NEW_RM(bp)) {
12192 		cp += bnxt_get_ulp_msix_num(bp);
12193 		stats += bnxt_get_ulp_stat_ctxs(bp);
12194 	}
12195 	return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
12196 				     stats, vnics);
12197 }
12198 
12199 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
12200 {
12201 	if (bp->bar2) {
12202 		pci_iounmap(pdev, bp->bar2);
12203 		bp->bar2 = NULL;
12204 	}
12205 
12206 	if (bp->bar1) {
12207 		pci_iounmap(pdev, bp->bar1);
12208 		bp->bar1 = NULL;
12209 	}
12210 
12211 	if (bp->bar0) {
12212 		pci_iounmap(pdev, bp->bar0);
12213 		bp->bar0 = NULL;
12214 	}
12215 }
12216 
12217 static void bnxt_cleanup_pci(struct bnxt *bp)
12218 {
12219 	bnxt_unmap_bars(bp, bp->pdev);
12220 	pci_release_regions(bp->pdev);
12221 	if (pci_is_enabled(bp->pdev))
12222 		pci_disable_device(bp->pdev);
12223 }
12224 
12225 static void bnxt_init_dflt_coal(struct bnxt *bp)
12226 {
12227 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
12228 	struct bnxt_coal *coal;
12229 	u16 flags = 0;
12230 
12231 	if (coal_cap->cmpl_params &
12232 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
12233 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
12234 
12235 	/* Tick values in micro seconds.
12236 	 * 1 coal_buf x bufs_per_record = 1 completion record.
12237 	 */
12238 	coal = &bp->rx_coal;
12239 	coal->coal_ticks = 10;
12240 	coal->coal_bufs = 30;
12241 	coal->coal_ticks_irq = 1;
12242 	coal->coal_bufs_irq = 2;
12243 	coal->idle_thresh = 50;
12244 	coal->bufs_per_record = 2;
12245 	coal->budget = 64;		/* NAPI budget */
12246 	coal->flags = flags;
12247 
12248 	coal = &bp->tx_coal;
12249 	coal->coal_ticks = 28;
12250 	coal->coal_bufs = 30;
12251 	coal->coal_ticks_irq = 2;
12252 	coal->coal_bufs_irq = 2;
12253 	coal->bufs_per_record = 1;
12254 	coal->flags = flags;
12255 
12256 	bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
12257 }
12258 
12259 static int bnxt_fw_init_one_p1(struct bnxt *bp)
12260 {
12261 	int rc;
12262 
12263 	bp->fw_cap = 0;
12264 	rc = bnxt_hwrm_ver_get(bp);
12265 	/* FW may be unresponsive after FLR. FLR must complete within 100 msec
12266 	 * so wait before continuing with recovery.
12267 	 */
12268 	if (rc)
12269 		msleep(100);
12270 	bnxt_try_map_fw_health_reg(bp);
12271 	if (rc) {
12272 		rc = bnxt_try_recover_fw(bp);
12273 		if (rc)
12274 			return rc;
12275 		rc = bnxt_hwrm_ver_get(bp);
12276 		if (rc)
12277 			return rc;
12278 	}
12279 
12280 	bnxt_nvm_cfg_ver_get(bp);
12281 
12282 	rc = bnxt_hwrm_func_reset(bp);
12283 	if (rc)
12284 		return -ENODEV;
12285 
12286 	bnxt_hwrm_fw_set_time(bp);
12287 	return 0;
12288 }
12289 
12290 static int bnxt_fw_init_one_p2(struct bnxt *bp)
12291 {
12292 	int rc;
12293 
12294 	/* Get the MAX capabilities for this function */
12295 	rc = bnxt_hwrm_func_qcaps(bp);
12296 	if (rc) {
12297 		netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
12298 			   rc);
12299 		return -ENODEV;
12300 	}
12301 
12302 	rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
12303 	if (rc)
12304 		netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
12305 			    rc);
12306 
12307 	if (bnxt_alloc_fw_health(bp)) {
12308 		netdev_warn(bp->dev, "no memory for firmware error recovery\n");
12309 	} else {
12310 		rc = bnxt_hwrm_error_recovery_qcfg(bp);
12311 		if (rc)
12312 			netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
12313 				    rc);
12314 	}
12315 
12316 	rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
12317 	if (rc)
12318 		return -ENODEV;
12319 
12320 	bnxt_hwrm_func_qcfg(bp);
12321 	bnxt_hwrm_vnic_qcaps(bp);
12322 	bnxt_hwrm_port_led_qcaps(bp);
12323 	bnxt_ethtool_init(bp);
12324 	if (bp->fw_cap & BNXT_FW_CAP_PTP)
12325 		__bnxt_hwrm_ptp_qcfg(bp);
12326 	bnxt_dcb_init(bp);
12327 	return 0;
12328 }
12329 
12330 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
12331 {
12332 	bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
12333 	bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
12334 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
12335 			   VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
12336 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
12337 	if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
12338 		bp->rss_hash_delta = bp->rss_hash_cfg;
12339 	if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
12340 		bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
12341 		bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
12342 				    VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
12343 	}
12344 }
12345 
12346 static void bnxt_set_dflt_rfs(struct bnxt *bp)
12347 {
12348 	struct net_device *dev = bp->dev;
12349 
12350 	dev->hw_features &= ~NETIF_F_NTUPLE;
12351 	dev->features &= ~NETIF_F_NTUPLE;
12352 	bp->flags &= ~BNXT_FLAG_RFS;
12353 	if (bnxt_rfs_supported(bp)) {
12354 		dev->hw_features |= NETIF_F_NTUPLE;
12355 		if (bnxt_rfs_capable(bp)) {
12356 			bp->flags |= BNXT_FLAG_RFS;
12357 			dev->features |= NETIF_F_NTUPLE;
12358 		}
12359 	}
12360 }
12361 
12362 static void bnxt_fw_init_one_p3(struct bnxt *bp)
12363 {
12364 	struct pci_dev *pdev = bp->pdev;
12365 
12366 	bnxt_set_dflt_rss_hash_type(bp);
12367 	bnxt_set_dflt_rfs(bp);
12368 
12369 	bnxt_get_wol_settings(bp);
12370 	if (bp->flags & BNXT_FLAG_WOL_CAP)
12371 		device_set_wakeup_enable(&pdev->dev, bp->wol);
12372 	else
12373 		device_set_wakeup_capable(&pdev->dev, false);
12374 
12375 	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
12376 	bnxt_hwrm_coal_params_qcaps(bp);
12377 }
12378 
12379 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
12380 
12381 int bnxt_fw_init_one(struct bnxt *bp)
12382 {
12383 	int rc;
12384 
12385 	rc = bnxt_fw_init_one_p1(bp);
12386 	if (rc) {
12387 		netdev_err(bp->dev, "Firmware init phase 1 failed\n");
12388 		return rc;
12389 	}
12390 	rc = bnxt_fw_init_one_p2(bp);
12391 	if (rc) {
12392 		netdev_err(bp->dev, "Firmware init phase 2 failed\n");
12393 		return rc;
12394 	}
12395 	rc = bnxt_probe_phy(bp, false);
12396 	if (rc)
12397 		return rc;
12398 	rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
12399 	if (rc)
12400 		return rc;
12401 
12402 	bnxt_fw_init_one_p3(bp);
12403 	return 0;
12404 }
12405 
12406 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
12407 {
12408 	struct bnxt_fw_health *fw_health = bp->fw_health;
12409 	u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
12410 	u32 val = fw_health->fw_reset_seq_vals[reg_idx];
12411 	u32 reg_type, reg_off, delay_msecs;
12412 
12413 	delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
12414 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
12415 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
12416 	switch (reg_type) {
12417 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
12418 		pci_write_config_dword(bp->pdev, reg_off, val);
12419 		break;
12420 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
12421 		writel(reg_off & BNXT_GRC_BASE_MASK,
12422 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
12423 		reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
12424 		fallthrough;
12425 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
12426 		writel(val, bp->bar0 + reg_off);
12427 		break;
12428 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
12429 		writel(val, bp->bar1 + reg_off);
12430 		break;
12431 	}
12432 	if (delay_msecs) {
12433 		pci_read_config_dword(bp->pdev, 0, &val);
12434 		msleep(delay_msecs);
12435 	}
12436 }
12437 
12438 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
12439 {
12440 	struct hwrm_func_qcfg_output *resp;
12441 	struct hwrm_func_qcfg_input *req;
12442 	bool result = true; /* firmware will enforce if unknown */
12443 
12444 	if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
12445 		return result;
12446 
12447 	if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
12448 		return result;
12449 
12450 	req->fid = cpu_to_le16(0xffff);
12451 	resp = hwrm_req_hold(bp, req);
12452 	if (!hwrm_req_send(bp, req))
12453 		result = !!(le16_to_cpu(resp->flags) &
12454 			    FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
12455 	hwrm_req_drop(bp, req);
12456 	return result;
12457 }
12458 
12459 static void bnxt_reset_all(struct bnxt *bp)
12460 {
12461 	struct bnxt_fw_health *fw_health = bp->fw_health;
12462 	int i, rc;
12463 
12464 	if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12465 		bnxt_fw_reset_via_optee(bp);
12466 		bp->fw_reset_timestamp = jiffies;
12467 		return;
12468 	}
12469 
12470 	if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
12471 		for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
12472 			bnxt_fw_reset_writel(bp, i);
12473 	} else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
12474 		struct hwrm_fw_reset_input *req;
12475 
12476 		rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
12477 		if (!rc) {
12478 			req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
12479 			req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
12480 			req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
12481 			req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
12482 			rc = hwrm_req_send(bp, req);
12483 		}
12484 		if (rc != -ENODEV)
12485 			netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
12486 	}
12487 	bp->fw_reset_timestamp = jiffies;
12488 }
12489 
12490 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
12491 {
12492 	return time_after(jiffies, bp->fw_reset_timestamp +
12493 			  (bp->fw_reset_max_dsecs * HZ / 10));
12494 }
12495 
12496 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
12497 {
12498 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12499 	if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
12500 		bnxt_ulp_start(bp, rc);
12501 		bnxt_dl_health_fw_status_update(bp, false);
12502 	}
12503 	bp->fw_reset_state = 0;
12504 	dev_close(bp->dev);
12505 }
12506 
12507 static void bnxt_fw_reset_task(struct work_struct *work)
12508 {
12509 	struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
12510 	int rc = 0;
12511 
12512 	if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12513 		netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
12514 		return;
12515 	}
12516 
12517 	switch (bp->fw_reset_state) {
12518 	case BNXT_FW_RESET_STATE_POLL_VF: {
12519 		int n = bnxt_get_registered_vfs(bp);
12520 		int tmo;
12521 
12522 		if (n < 0) {
12523 			netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
12524 				   n, jiffies_to_msecs(jiffies -
12525 				   bp->fw_reset_timestamp));
12526 			goto fw_reset_abort;
12527 		} else if (n > 0) {
12528 			if (bnxt_fw_reset_timeout(bp)) {
12529 				clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12530 				bp->fw_reset_state = 0;
12531 				netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
12532 					   n);
12533 				return;
12534 			}
12535 			bnxt_queue_fw_reset_work(bp, HZ / 10);
12536 			return;
12537 		}
12538 		bp->fw_reset_timestamp = jiffies;
12539 		rtnl_lock();
12540 		if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12541 			bnxt_fw_reset_abort(bp, rc);
12542 			rtnl_unlock();
12543 			return;
12544 		}
12545 		bnxt_fw_reset_close(bp);
12546 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12547 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12548 			tmo = HZ / 10;
12549 		} else {
12550 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12551 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
12552 		}
12553 		rtnl_unlock();
12554 		bnxt_queue_fw_reset_work(bp, tmo);
12555 		return;
12556 	}
12557 	case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
12558 		u32 val;
12559 
12560 		val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12561 		if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
12562 		    !bnxt_fw_reset_timeout(bp)) {
12563 			bnxt_queue_fw_reset_work(bp, HZ / 5);
12564 			return;
12565 		}
12566 
12567 		if (!bp->fw_health->primary) {
12568 			u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
12569 
12570 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12571 			bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12572 			return;
12573 		}
12574 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12575 	}
12576 		fallthrough;
12577 	case BNXT_FW_RESET_STATE_RESET_FW:
12578 		bnxt_reset_all(bp);
12579 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12580 		bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
12581 		return;
12582 	case BNXT_FW_RESET_STATE_ENABLE_DEV:
12583 		bnxt_inv_fw_health_reg(bp);
12584 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
12585 		    !bp->fw_reset_min_dsecs) {
12586 			u16 val;
12587 
12588 			pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12589 			if (val == 0xffff) {
12590 				if (bnxt_fw_reset_timeout(bp)) {
12591 					netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
12592 					rc = -ETIMEDOUT;
12593 					goto fw_reset_abort;
12594 				}
12595 				bnxt_queue_fw_reset_work(bp, HZ / 1000);
12596 				return;
12597 			}
12598 		}
12599 		clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12600 		clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
12601 		if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
12602 		    !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
12603 			bnxt_dl_remote_reload(bp);
12604 		if (pci_enable_device(bp->pdev)) {
12605 			netdev_err(bp->dev, "Cannot re-enable PCI device\n");
12606 			rc = -ENODEV;
12607 			goto fw_reset_abort;
12608 		}
12609 		pci_set_master(bp->pdev);
12610 		bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12611 		fallthrough;
12612 	case BNXT_FW_RESET_STATE_POLL_FW:
12613 		bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12614 		rc = bnxt_hwrm_poll(bp);
12615 		if (rc) {
12616 			if (bnxt_fw_reset_timeout(bp)) {
12617 				netdev_err(bp->dev, "Firmware reset aborted\n");
12618 				goto fw_reset_abort_status;
12619 			}
12620 			bnxt_queue_fw_reset_work(bp, HZ / 5);
12621 			return;
12622 		}
12623 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12624 		bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12625 		fallthrough;
12626 	case BNXT_FW_RESET_STATE_OPENING:
12627 		while (!rtnl_trylock()) {
12628 			bnxt_queue_fw_reset_work(bp, HZ / 10);
12629 			return;
12630 		}
12631 		rc = bnxt_open(bp->dev);
12632 		if (rc) {
12633 			netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
12634 			bnxt_fw_reset_abort(bp, rc);
12635 			rtnl_unlock();
12636 			return;
12637 		}
12638 
12639 		if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
12640 		    bp->fw_health->enabled) {
12641 			bp->fw_health->last_fw_reset_cnt =
12642 				bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12643 		}
12644 		bp->fw_reset_state = 0;
12645 		/* Make sure fw_reset_state is 0 before clearing the flag */
12646 		smp_mb__before_atomic();
12647 		clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12648 		bnxt_ulp_start(bp, 0);
12649 		bnxt_reenable_sriov(bp);
12650 		bnxt_vf_reps_alloc(bp);
12651 		bnxt_vf_reps_open(bp);
12652 		bnxt_ptp_reapply_pps(bp);
12653 		clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
12654 		if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
12655 			bnxt_dl_health_fw_recovery_done(bp);
12656 			bnxt_dl_health_fw_status_update(bp, true);
12657 		}
12658 		rtnl_unlock();
12659 		break;
12660 	}
12661 	return;
12662 
12663 fw_reset_abort_status:
12664 	if (bp->fw_health->status_reliable ||
12665 	    (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12666 		u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12667 
12668 		netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12669 	}
12670 fw_reset_abort:
12671 	rtnl_lock();
12672 	bnxt_fw_reset_abort(bp, rc);
12673 	rtnl_unlock();
12674 }
12675 
12676 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12677 {
12678 	int rc;
12679 	struct bnxt *bp = netdev_priv(dev);
12680 
12681 	SET_NETDEV_DEV(dev, &pdev->dev);
12682 
12683 	/* enable device (incl. PCI PM wakeup), and bus-mastering */
12684 	rc = pci_enable_device(pdev);
12685 	if (rc) {
12686 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12687 		goto init_err;
12688 	}
12689 
12690 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12691 		dev_err(&pdev->dev,
12692 			"Cannot find PCI device base address, aborting\n");
12693 		rc = -ENODEV;
12694 		goto init_err_disable;
12695 	}
12696 
12697 	rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12698 	if (rc) {
12699 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12700 		goto init_err_disable;
12701 	}
12702 
12703 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12704 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12705 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12706 		rc = -EIO;
12707 		goto init_err_release;
12708 	}
12709 
12710 	pci_set_master(pdev);
12711 
12712 	bp->dev = dev;
12713 	bp->pdev = pdev;
12714 
12715 	/* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12716 	 * determines the BAR size.
12717 	 */
12718 	bp->bar0 = pci_ioremap_bar(pdev, 0);
12719 	if (!bp->bar0) {
12720 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12721 		rc = -ENOMEM;
12722 		goto init_err_release;
12723 	}
12724 
12725 	bp->bar2 = pci_ioremap_bar(pdev, 4);
12726 	if (!bp->bar2) {
12727 		dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12728 		rc = -ENOMEM;
12729 		goto init_err_release;
12730 	}
12731 
12732 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
12733 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12734 
12735 	spin_lock_init(&bp->ntp_fltr_lock);
12736 #if BITS_PER_LONG == 32
12737 	spin_lock_init(&bp->db_lock);
12738 #endif
12739 
12740 	bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12741 	bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12742 
12743 	timer_setup(&bp->timer, bnxt_timer, 0);
12744 	bp->current_interval = BNXT_TIMER_INTERVAL;
12745 
12746 	bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12747 	bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12748 
12749 	clear_bit(BNXT_STATE_OPEN, &bp->state);
12750 	return 0;
12751 
12752 init_err_release:
12753 	bnxt_unmap_bars(bp, pdev);
12754 	pci_release_regions(pdev);
12755 
12756 init_err_disable:
12757 	pci_disable_device(pdev);
12758 
12759 init_err:
12760 	return rc;
12761 }
12762 
12763 /* rtnl_lock held */
12764 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12765 {
12766 	struct sockaddr *addr = p;
12767 	struct bnxt *bp = netdev_priv(dev);
12768 	int rc = 0;
12769 
12770 	if (!is_valid_ether_addr(addr->sa_data))
12771 		return -EADDRNOTAVAIL;
12772 
12773 	if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12774 		return 0;
12775 
12776 	rc = bnxt_approve_mac(bp, addr->sa_data, true);
12777 	if (rc)
12778 		return rc;
12779 
12780 	eth_hw_addr_set(dev, addr->sa_data);
12781 	if (netif_running(dev)) {
12782 		bnxt_close_nic(bp, false, false);
12783 		rc = bnxt_open_nic(bp, false, false);
12784 	}
12785 
12786 	return rc;
12787 }
12788 
12789 /* rtnl_lock held */
12790 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12791 {
12792 	struct bnxt *bp = netdev_priv(dev);
12793 
12794 	if (netif_running(dev))
12795 		bnxt_close_nic(bp, true, false);
12796 
12797 	dev->mtu = new_mtu;
12798 	bnxt_set_ring_params(bp);
12799 
12800 	if (netif_running(dev))
12801 		return bnxt_open_nic(bp, true, false);
12802 
12803 	return 0;
12804 }
12805 
12806 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12807 {
12808 	struct bnxt *bp = netdev_priv(dev);
12809 	bool sh = false;
12810 	int rc;
12811 
12812 	if (tc > bp->max_tc) {
12813 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12814 			   tc, bp->max_tc);
12815 		return -EINVAL;
12816 	}
12817 
12818 	if (netdev_get_num_tc(dev) == tc)
12819 		return 0;
12820 
12821 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12822 		sh = true;
12823 
12824 	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12825 			      sh, tc, bp->tx_nr_rings_xdp);
12826 	if (rc)
12827 		return rc;
12828 
12829 	/* Needs to close the device and do hw resource re-allocations */
12830 	if (netif_running(bp->dev))
12831 		bnxt_close_nic(bp, true, false);
12832 
12833 	if (tc) {
12834 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12835 		netdev_set_num_tc(dev, tc);
12836 	} else {
12837 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12838 		netdev_reset_tc(dev);
12839 	}
12840 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12841 	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12842 			       bp->tx_nr_rings + bp->rx_nr_rings;
12843 
12844 	if (netif_running(bp->dev))
12845 		return bnxt_open_nic(bp, true, false);
12846 
12847 	return 0;
12848 }
12849 
12850 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12851 				  void *cb_priv)
12852 {
12853 	struct bnxt *bp = cb_priv;
12854 
12855 	if (!bnxt_tc_flower_enabled(bp) ||
12856 	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12857 		return -EOPNOTSUPP;
12858 
12859 	switch (type) {
12860 	case TC_SETUP_CLSFLOWER:
12861 		return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12862 	default:
12863 		return -EOPNOTSUPP;
12864 	}
12865 }
12866 
12867 LIST_HEAD(bnxt_block_cb_list);
12868 
12869 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12870 			 void *type_data)
12871 {
12872 	struct bnxt *bp = netdev_priv(dev);
12873 
12874 	switch (type) {
12875 	case TC_SETUP_BLOCK:
12876 		return flow_block_cb_setup_simple(type_data,
12877 						  &bnxt_block_cb_list,
12878 						  bnxt_setup_tc_block_cb,
12879 						  bp, bp, true);
12880 	case TC_SETUP_QDISC_MQPRIO: {
12881 		struct tc_mqprio_qopt *mqprio = type_data;
12882 
12883 		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12884 
12885 		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12886 	}
12887 	default:
12888 		return -EOPNOTSUPP;
12889 	}
12890 }
12891 
12892 #ifdef CONFIG_RFS_ACCEL
12893 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12894 			    struct bnxt_ntuple_filter *f2)
12895 {
12896 	struct flow_keys *keys1 = &f1->fkeys;
12897 	struct flow_keys *keys2 = &f2->fkeys;
12898 
12899 	if (keys1->basic.n_proto != keys2->basic.n_proto ||
12900 	    keys1->basic.ip_proto != keys2->basic.ip_proto)
12901 		return false;
12902 
12903 	if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12904 		if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12905 		    keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12906 			return false;
12907 	} else {
12908 		if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12909 			   sizeof(keys1->addrs.v6addrs.src)) ||
12910 		    memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12911 			   sizeof(keys1->addrs.v6addrs.dst)))
12912 			return false;
12913 	}
12914 
12915 	if (keys1->ports.ports == keys2->ports.ports &&
12916 	    keys1->control.flags == keys2->control.flags &&
12917 	    ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12918 	    ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12919 		return true;
12920 
12921 	return false;
12922 }
12923 
12924 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12925 			      u16 rxq_index, u32 flow_id)
12926 {
12927 	struct bnxt *bp = netdev_priv(dev);
12928 	struct bnxt_ntuple_filter *fltr, *new_fltr;
12929 	struct flow_keys *fkeys;
12930 	struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12931 	int rc = 0, idx, bit_id, l2_idx = 0;
12932 	struct hlist_head *head;
12933 	u32 flags;
12934 
12935 	if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12936 		struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12937 		int off = 0, j;
12938 
12939 		netif_addr_lock_bh(dev);
12940 		for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12941 			if (ether_addr_equal(eth->h_dest,
12942 					     vnic->uc_list + off)) {
12943 				l2_idx = j + 1;
12944 				break;
12945 			}
12946 		}
12947 		netif_addr_unlock_bh(dev);
12948 		if (!l2_idx)
12949 			return -EINVAL;
12950 	}
12951 	new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12952 	if (!new_fltr)
12953 		return -ENOMEM;
12954 
12955 	fkeys = &new_fltr->fkeys;
12956 	if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12957 		rc = -EPROTONOSUPPORT;
12958 		goto err_free;
12959 	}
12960 
12961 	if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12962 	     fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12963 	    ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12964 	     (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12965 		rc = -EPROTONOSUPPORT;
12966 		goto err_free;
12967 	}
12968 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12969 	    bp->hwrm_spec_code < 0x10601) {
12970 		rc = -EPROTONOSUPPORT;
12971 		goto err_free;
12972 	}
12973 	flags = fkeys->control.flags;
12974 	if (((flags & FLOW_DIS_ENCAPSULATION) &&
12975 	     bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12976 		rc = -EPROTONOSUPPORT;
12977 		goto err_free;
12978 	}
12979 
12980 	memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12981 	memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12982 
12983 	idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12984 	head = &bp->ntp_fltr_hash_tbl[idx];
12985 	rcu_read_lock();
12986 	hlist_for_each_entry_rcu(fltr, head, hash) {
12987 		if (bnxt_fltr_match(fltr, new_fltr)) {
12988 			rc = fltr->sw_id;
12989 			rcu_read_unlock();
12990 			goto err_free;
12991 		}
12992 	}
12993 	rcu_read_unlock();
12994 
12995 	spin_lock_bh(&bp->ntp_fltr_lock);
12996 	bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12997 					 BNXT_NTP_FLTR_MAX_FLTR, 0);
12998 	if (bit_id < 0) {
12999 		spin_unlock_bh(&bp->ntp_fltr_lock);
13000 		rc = -ENOMEM;
13001 		goto err_free;
13002 	}
13003 
13004 	new_fltr->sw_id = (u16)bit_id;
13005 	new_fltr->flow_id = flow_id;
13006 	new_fltr->l2_fltr_idx = l2_idx;
13007 	new_fltr->rxq = rxq_index;
13008 	hlist_add_head_rcu(&new_fltr->hash, head);
13009 	bp->ntp_fltr_count++;
13010 	spin_unlock_bh(&bp->ntp_fltr_lock);
13011 
13012 	bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
13013 
13014 	return new_fltr->sw_id;
13015 
13016 err_free:
13017 	kfree(new_fltr);
13018 	return rc;
13019 }
13020 
13021 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13022 {
13023 	int i;
13024 
13025 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
13026 		struct hlist_head *head;
13027 		struct hlist_node *tmp;
13028 		struct bnxt_ntuple_filter *fltr;
13029 		int rc;
13030 
13031 		head = &bp->ntp_fltr_hash_tbl[i];
13032 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
13033 			bool del = false;
13034 
13035 			if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
13036 				if (rps_may_expire_flow(bp->dev, fltr->rxq,
13037 							fltr->flow_id,
13038 							fltr->sw_id)) {
13039 					bnxt_hwrm_cfa_ntuple_filter_free(bp,
13040 									 fltr);
13041 					del = true;
13042 				}
13043 			} else {
13044 				rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
13045 								       fltr);
13046 				if (rc)
13047 					del = true;
13048 				else
13049 					set_bit(BNXT_FLTR_VALID, &fltr->state);
13050 			}
13051 
13052 			if (del) {
13053 				spin_lock_bh(&bp->ntp_fltr_lock);
13054 				hlist_del_rcu(&fltr->hash);
13055 				bp->ntp_fltr_count--;
13056 				spin_unlock_bh(&bp->ntp_fltr_lock);
13057 				synchronize_rcu();
13058 				clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
13059 				kfree(fltr);
13060 			}
13061 		}
13062 	}
13063 }
13064 
13065 #else
13066 
13067 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13068 {
13069 }
13070 
13071 #endif /* CONFIG_RFS_ACCEL */
13072 
13073 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
13074 				    unsigned int entry, struct udp_tunnel_info *ti)
13075 {
13076 	struct bnxt *bp = netdev_priv(netdev);
13077 	unsigned int cmd;
13078 
13079 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
13080 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13081 	else
13082 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13083 
13084 	return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
13085 }
13086 
13087 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
13088 				      unsigned int entry, struct udp_tunnel_info *ti)
13089 {
13090 	struct bnxt *bp = netdev_priv(netdev);
13091 	unsigned int cmd;
13092 
13093 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
13094 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13095 	else
13096 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13097 
13098 	return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
13099 }
13100 
13101 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
13102 	.set_port	= bnxt_udp_tunnel_set_port,
13103 	.unset_port	= bnxt_udp_tunnel_unset_port,
13104 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
13105 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
13106 	.tables		= {
13107 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
13108 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
13109 	},
13110 };
13111 
13112 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13113 			       struct net_device *dev, u32 filter_mask,
13114 			       int nlflags)
13115 {
13116 	struct bnxt *bp = netdev_priv(dev);
13117 
13118 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
13119 				       nlflags, filter_mask, NULL);
13120 }
13121 
13122 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
13123 			       u16 flags, struct netlink_ext_ack *extack)
13124 {
13125 	struct bnxt *bp = netdev_priv(dev);
13126 	struct nlattr *attr, *br_spec;
13127 	int rem, rc = 0;
13128 
13129 	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
13130 		return -EOPNOTSUPP;
13131 
13132 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13133 	if (!br_spec)
13134 		return -EINVAL;
13135 
13136 	nla_for_each_nested(attr, br_spec, rem) {
13137 		u16 mode;
13138 
13139 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
13140 			continue;
13141 
13142 		mode = nla_get_u16(attr);
13143 		if (mode == bp->br_mode)
13144 			break;
13145 
13146 		rc = bnxt_hwrm_set_br_mode(bp, mode);
13147 		if (!rc)
13148 			bp->br_mode = mode;
13149 		break;
13150 	}
13151 	return rc;
13152 }
13153 
13154 int bnxt_get_port_parent_id(struct net_device *dev,
13155 			    struct netdev_phys_item_id *ppid)
13156 {
13157 	struct bnxt *bp = netdev_priv(dev);
13158 
13159 	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
13160 		return -EOPNOTSUPP;
13161 
13162 	/* The PF and it's VF-reps only support the switchdev framework */
13163 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
13164 		return -EOPNOTSUPP;
13165 
13166 	ppid->id_len = sizeof(bp->dsn);
13167 	memcpy(ppid->id, bp->dsn, ppid->id_len);
13168 
13169 	return 0;
13170 }
13171 
13172 static const struct net_device_ops bnxt_netdev_ops = {
13173 	.ndo_open		= bnxt_open,
13174 	.ndo_start_xmit		= bnxt_start_xmit,
13175 	.ndo_stop		= bnxt_close,
13176 	.ndo_get_stats64	= bnxt_get_stats64,
13177 	.ndo_set_rx_mode	= bnxt_set_rx_mode,
13178 	.ndo_eth_ioctl		= bnxt_ioctl,
13179 	.ndo_validate_addr	= eth_validate_addr,
13180 	.ndo_set_mac_address	= bnxt_change_mac_addr,
13181 	.ndo_change_mtu		= bnxt_change_mtu,
13182 	.ndo_fix_features	= bnxt_fix_features,
13183 	.ndo_set_features	= bnxt_set_features,
13184 	.ndo_features_check	= bnxt_features_check,
13185 	.ndo_tx_timeout		= bnxt_tx_timeout,
13186 #ifdef CONFIG_BNXT_SRIOV
13187 	.ndo_get_vf_config	= bnxt_get_vf_config,
13188 	.ndo_set_vf_mac		= bnxt_set_vf_mac,
13189 	.ndo_set_vf_vlan	= bnxt_set_vf_vlan,
13190 	.ndo_set_vf_rate	= bnxt_set_vf_bw,
13191 	.ndo_set_vf_link_state	= bnxt_set_vf_link_state,
13192 	.ndo_set_vf_spoofchk	= bnxt_set_vf_spoofchk,
13193 	.ndo_set_vf_trust	= bnxt_set_vf_trust,
13194 #endif
13195 	.ndo_setup_tc           = bnxt_setup_tc,
13196 #ifdef CONFIG_RFS_ACCEL
13197 	.ndo_rx_flow_steer	= bnxt_rx_flow_steer,
13198 #endif
13199 	.ndo_bpf		= bnxt_xdp,
13200 	.ndo_xdp_xmit		= bnxt_xdp_xmit,
13201 	.ndo_bridge_getlink	= bnxt_bridge_getlink,
13202 	.ndo_bridge_setlink	= bnxt_bridge_setlink,
13203 };
13204 
13205 static void bnxt_remove_one(struct pci_dev *pdev)
13206 {
13207 	struct net_device *dev = pci_get_drvdata(pdev);
13208 	struct bnxt *bp = netdev_priv(dev);
13209 
13210 	if (BNXT_PF(bp))
13211 		bnxt_sriov_disable(bp);
13212 
13213 	bnxt_rdma_aux_device_uninit(bp);
13214 
13215 	bnxt_ptp_clear(bp);
13216 	unregister_netdev(dev);
13217 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13218 	/* Flush any pending tasks */
13219 	cancel_work_sync(&bp->sp_task);
13220 	cancel_delayed_work_sync(&bp->fw_reset_task);
13221 	bp->sp_event = 0;
13222 
13223 	bnxt_dl_fw_reporters_destroy(bp);
13224 	bnxt_dl_unregister(bp);
13225 	bnxt_shutdown_tc(bp);
13226 
13227 	bnxt_clear_int_mode(bp);
13228 	bnxt_hwrm_func_drv_unrgtr(bp);
13229 	bnxt_free_hwrm_resources(bp);
13230 	bnxt_ethtool_free(bp);
13231 	bnxt_dcb_free(bp);
13232 	kfree(bp->ptp_cfg);
13233 	bp->ptp_cfg = NULL;
13234 	kfree(bp->fw_health);
13235 	bp->fw_health = NULL;
13236 	bnxt_cleanup_pci(bp);
13237 	bnxt_free_ctx_mem(bp);
13238 	kfree(bp->ctx);
13239 	bp->ctx = NULL;
13240 	kfree(bp->rss_indir_tbl);
13241 	bp->rss_indir_tbl = NULL;
13242 	bnxt_free_port_stats(bp);
13243 	free_netdev(dev);
13244 }
13245 
13246 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
13247 {
13248 	int rc = 0;
13249 	struct bnxt_link_info *link_info = &bp->link_info;
13250 
13251 	bp->phy_flags = 0;
13252 	rc = bnxt_hwrm_phy_qcaps(bp);
13253 	if (rc) {
13254 		netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
13255 			   rc);
13256 		return rc;
13257 	}
13258 	if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
13259 		bp->dev->priv_flags |= IFF_SUPP_NOFCS;
13260 	else
13261 		bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
13262 	if (!fw_dflt)
13263 		return 0;
13264 
13265 	mutex_lock(&bp->link_lock);
13266 	rc = bnxt_update_link(bp, false);
13267 	if (rc) {
13268 		mutex_unlock(&bp->link_lock);
13269 		netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
13270 			   rc);
13271 		return rc;
13272 	}
13273 
13274 	/* Older firmware does not have supported_auto_speeds, so assume
13275 	 * that all supported speeds can be autonegotiated.
13276 	 */
13277 	if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
13278 		link_info->support_auto_speeds = link_info->support_speeds;
13279 
13280 	bnxt_init_ethtool_link_settings(bp);
13281 	mutex_unlock(&bp->link_lock);
13282 	return 0;
13283 }
13284 
13285 static int bnxt_get_max_irq(struct pci_dev *pdev)
13286 {
13287 	u16 ctrl;
13288 
13289 	if (!pdev->msix_cap)
13290 		return 1;
13291 
13292 	pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
13293 	return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
13294 }
13295 
13296 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13297 				int *max_cp)
13298 {
13299 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13300 	int max_ring_grps = 0, max_irq;
13301 
13302 	*max_tx = hw_resc->max_tx_rings;
13303 	*max_rx = hw_resc->max_rx_rings;
13304 	*max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
13305 	max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
13306 			bnxt_get_ulp_msix_num(bp),
13307 			hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
13308 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
13309 		*max_cp = min_t(int, *max_cp, max_irq);
13310 	max_ring_grps = hw_resc->max_hw_ring_grps;
13311 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
13312 		*max_cp -= 1;
13313 		*max_rx -= 2;
13314 	}
13315 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
13316 		*max_rx >>= 1;
13317 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
13318 		bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
13319 		/* On P5 chips, max_cp output param should be available NQs */
13320 		*max_cp = max_irq;
13321 	}
13322 	*max_rx = min_t(int, *max_rx, max_ring_grps);
13323 }
13324 
13325 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
13326 {
13327 	int rx, tx, cp;
13328 
13329 	_bnxt_get_max_rings(bp, &rx, &tx, &cp);
13330 	*max_rx = rx;
13331 	*max_tx = tx;
13332 	if (!rx || !tx || !cp)
13333 		return -ENOMEM;
13334 
13335 	return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
13336 }
13337 
13338 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13339 			       bool shared)
13340 {
13341 	int rc;
13342 
13343 	rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13344 	if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
13345 		/* Not enough rings, try disabling agg rings. */
13346 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
13347 		rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13348 		if (rc) {
13349 			/* set BNXT_FLAG_AGG_RINGS back for consistency */
13350 			bp->flags |= BNXT_FLAG_AGG_RINGS;
13351 			return rc;
13352 		}
13353 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
13354 		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13355 		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13356 		bnxt_set_ring_params(bp);
13357 	}
13358 
13359 	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
13360 		int max_cp, max_stat, max_irq;
13361 
13362 		/* Reserve minimum resources for RoCE */
13363 		max_cp = bnxt_get_max_func_cp_rings(bp);
13364 		max_stat = bnxt_get_max_func_stat_ctxs(bp);
13365 		max_irq = bnxt_get_max_func_irqs(bp);
13366 		if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
13367 		    max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
13368 		    max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
13369 			return 0;
13370 
13371 		max_cp -= BNXT_MIN_ROCE_CP_RINGS;
13372 		max_irq -= BNXT_MIN_ROCE_CP_RINGS;
13373 		max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
13374 		max_cp = min_t(int, max_cp, max_irq);
13375 		max_cp = min_t(int, max_cp, max_stat);
13376 		rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
13377 		if (rc)
13378 			rc = 0;
13379 	}
13380 	return rc;
13381 }
13382 
13383 /* In initial default shared ring setting, each shared ring must have a
13384  * RX/TX ring pair.
13385  */
13386 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
13387 {
13388 	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
13389 	bp->rx_nr_rings = bp->cp_nr_rings;
13390 	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
13391 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13392 }
13393 
13394 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
13395 {
13396 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
13397 
13398 	if (!bnxt_can_reserve_rings(bp))
13399 		return 0;
13400 
13401 	if (sh)
13402 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
13403 	dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
13404 	/* Reduce default rings on multi-port cards so that total default
13405 	 * rings do not exceed CPU count.
13406 	 */
13407 	if (bp->port_count > 1) {
13408 		int max_rings =
13409 			max_t(int, num_online_cpus() / bp->port_count, 1);
13410 
13411 		dflt_rings = min_t(int, dflt_rings, max_rings);
13412 	}
13413 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
13414 	if (rc)
13415 		return rc;
13416 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
13417 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
13418 	if (sh)
13419 		bnxt_trim_dflt_sh_rings(bp);
13420 	else
13421 		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
13422 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13423 
13424 	rc = __bnxt_reserve_rings(bp);
13425 	if (rc && rc != -ENODEV)
13426 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
13427 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13428 	if (sh)
13429 		bnxt_trim_dflt_sh_rings(bp);
13430 
13431 	/* Rings may have been trimmed, re-reserve the trimmed rings. */
13432 	if (bnxt_need_reserve_rings(bp)) {
13433 		rc = __bnxt_reserve_rings(bp);
13434 		if (rc && rc != -ENODEV)
13435 			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
13436 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13437 	}
13438 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
13439 		bp->rx_nr_rings++;
13440 		bp->cp_nr_rings++;
13441 	}
13442 	if (rc) {
13443 		bp->tx_nr_rings = 0;
13444 		bp->rx_nr_rings = 0;
13445 	}
13446 	return rc;
13447 }
13448 
13449 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
13450 {
13451 	int rc;
13452 
13453 	if (bp->tx_nr_rings)
13454 		return 0;
13455 
13456 	bnxt_ulp_irq_stop(bp);
13457 	bnxt_clear_int_mode(bp);
13458 	rc = bnxt_set_dflt_rings(bp, true);
13459 	if (rc) {
13460 		if (BNXT_VF(bp) && rc == -ENODEV)
13461 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13462 		else
13463 			netdev_err(bp->dev, "Not enough rings available.\n");
13464 		goto init_dflt_ring_err;
13465 	}
13466 	rc = bnxt_init_int_mode(bp);
13467 	if (rc)
13468 		goto init_dflt_ring_err;
13469 
13470 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13471 
13472 	bnxt_set_dflt_rfs(bp);
13473 
13474 init_dflt_ring_err:
13475 	bnxt_ulp_irq_restart(bp, rc);
13476 	return rc;
13477 }
13478 
13479 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
13480 {
13481 	int rc;
13482 
13483 	ASSERT_RTNL();
13484 	bnxt_hwrm_func_qcaps(bp);
13485 
13486 	if (netif_running(bp->dev))
13487 		__bnxt_close_nic(bp, true, false);
13488 
13489 	bnxt_ulp_irq_stop(bp);
13490 	bnxt_clear_int_mode(bp);
13491 	rc = bnxt_init_int_mode(bp);
13492 	bnxt_ulp_irq_restart(bp, rc);
13493 
13494 	if (netif_running(bp->dev)) {
13495 		if (rc)
13496 			dev_close(bp->dev);
13497 		else
13498 			rc = bnxt_open_nic(bp, true, false);
13499 	}
13500 
13501 	return rc;
13502 }
13503 
13504 static int bnxt_init_mac_addr(struct bnxt *bp)
13505 {
13506 	int rc = 0;
13507 
13508 	if (BNXT_PF(bp)) {
13509 		eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
13510 	} else {
13511 #ifdef CONFIG_BNXT_SRIOV
13512 		struct bnxt_vf_info *vf = &bp->vf;
13513 		bool strict_approval = true;
13514 
13515 		if (is_valid_ether_addr(vf->mac_addr)) {
13516 			/* overwrite netdev dev_addr with admin VF MAC */
13517 			eth_hw_addr_set(bp->dev, vf->mac_addr);
13518 			/* Older PF driver or firmware may not approve this
13519 			 * correctly.
13520 			 */
13521 			strict_approval = false;
13522 		} else {
13523 			eth_hw_addr_random(bp->dev);
13524 		}
13525 		rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
13526 #endif
13527 	}
13528 	return rc;
13529 }
13530 
13531 static void bnxt_vpd_read_info(struct bnxt *bp)
13532 {
13533 	struct pci_dev *pdev = bp->pdev;
13534 	unsigned int vpd_size, kw_len;
13535 	int pos, size;
13536 	u8 *vpd_data;
13537 
13538 	vpd_data = pci_vpd_alloc(pdev, &vpd_size);
13539 	if (IS_ERR(vpd_data)) {
13540 		pci_warn(pdev, "Unable to read VPD\n");
13541 		return;
13542 	}
13543 
13544 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13545 					   PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
13546 	if (pos < 0)
13547 		goto read_sn;
13548 
13549 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13550 	memcpy(bp->board_partno, &vpd_data[pos], size);
13551 
13552 read_sn:
13553 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13554 					   PCI_VPD_RO_KEYWORD_SERIALNO,
13555 					   &kw_len);
13556 	if (pos < 0)
13557 		goto exit;
13558 
13559 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13560 	memcpy(bp->board_serialno, &vpd_data[pos], size);
13561 exit:
13562 	kfree(vpd_data);
13563 }
13564 
13565 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
13566 {
13567 	struct pci_dev *pdev = bp->pdev;
13568 	u64 qword;
13569 
13570 	qword = pci_get_dsn(pdev);
13571 	if (!qword) {
13572 		netdev_info(bp->dev, "Unable to read adapter's DSN\n");
13573 		return -EOPNOTSUPP;
13574 	}
13575 
13576 	put_unaligned_le64(qword, dsn);
13577 
13578 	bp->flags |= BNXT_FLAG_DSN_VALID;
13579 	return 0;
13580 }
13581 
13582 static int bnxt_map_db_bar(struct bnxt *bp)
13583 {
13584 	if (!bp->db_size)
13585 		return -ENODEV;
13586 	bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13587 	if (!bp->bar1)
13588 		return -ENOMEM;
13589 	return 0;
13590 }
13591 
13592 void bnxt_print_device_info(struct bnxt *bp)
13593 {
13594 	netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
13595 		    board_info[bp->board_idx].name,
13596 		    (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
13597 
13598 	pcie_print_link_status(bp->pdev);
13599 }
13600 
13601 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13602 {
13603 	struct net_device *dev;
13604 	struct bnxt *bp;
13605 	int rc, max_irqs;
13606 
13607 	if (pci_is_bridge(pdev))
13608 		return -ENODEV;
13609 
13610 	/* Clear any pending DMA transactions from crash kernel
13611 	 * while loading driver in capture kernel.
13612 	 */
13613 	if (is_kdump_kernel()) {
13614 		pci_clear_master(pdev);
13615 		pcie_flr(pdev);
13616 	}
13617 
13618 	max_irqs = bnxt_get_max_irq(pdev);
13619 	dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13620 	if (!dev)
13621 		return -ENOMEM;
13622 
13623 	bp = netdev_priv(dev);
13624 	bp->board_idx = ent->driver_data;
13625 	bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13626 	bnxt_set_max_func_irqs(bp, max_irqs);
13627 
13628 	if (bnxt_vf_pciid(bp->board_idx))
13629 		bp->flags |= BNXT_FLAG_VF;
13630 
13631 	/* No devlink port registration in case of a VF */
13632 	if (BNXT_PF(bp))
13633 		SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
13634 
13635 	if (pdev->msix_cap)
13636 		bp->flags |= BNXT_FLAG_MSIX_CAP;
13637 
13638 	rc = bnxt_init_board(pdev, dev);
13639 	if (rc < 0)
13640 		goto init_err_free;
13641 
13642 	dev->netdev_ops = &bnxt_netdev_ops;
13643 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13644 	dev->ethtool_ops = &bnxt_ethtool_ops;
13645 	pci_set_drvdata(pdev, dev);
13646 
13647 	rc = bnxt_alloc_hwrm_resources(bp);
13648 	if (rc)
13649 		goto init_err_pci_clean;
13650 
13651 	mutex_init(&bp->hwrm_cmd_lock);
13652 	mutex_init(&bp->link_lock);
13653 
13654 	rc = bnxt_fw_init_one_p1(bp);
13655 	if (rc)
13656 		goto init_err_pci_clean;
13657 
13658 	if (BNXT_PF(bp))
13659 		bnxt_vpd_read_info(bp);
13660 
13661 	if (BNXT_CHIP_P5(bp)) {
13662 		bp->flags |= BNXT_FLAG_CHIP_P5;
13663 		if (BNXT_CHIP_SR2(bp))
13664 			bp->flags |= BNXT_FLAG_CHIP_SR2;
13665 	}
13666 
13667 	rc = bnxt_alloc_rss_indir_tbl(bp);
13668 	if (rc)
13669 		goto init_err_pci_clean;
13670 
13671 	rc = bnxt_fw_init_one_p2(bp);
13672 	if (rc)
13673 		goto init_err_pci_clean;
13674 
13675 	rc = bnxt_map_db_bar(bp);
13676 	if (rc) {
13677 		dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13678 			rc);
13679 		goto init_err_pci_clean;
13680 	}
13681 
13682 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13683 			   NETIF_F_TSO | NETIF_F_TSO6 |
13684 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13685 			   NETIF_F_GSO_IPXIP4 |
13686 			   NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13687 			   NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13688 			   NETIF_F_RXCSUM | NETIF_F_GRO;
13689 
13690 	if (BNXT_SUPPORTS_TPA(bp))
13691 		dev->hw_features |= NETIF_F_LRO;
13692 
13693 	dev->hw_enc_features =
13694 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13695 			NETIF_F_TSO | NETIF_F_TSO6 |
13696 			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13697 			NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13698 			NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13699 	dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13700 
13701 	dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13702 				    NETIF_F_GSO_GRE_CSUM;
13703 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13704 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13705 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13706 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13707 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13708 	if (BNXT_SUPPORTS_TPA(bp))
13709 		dev->hw_features |= NETIF_F_GRO_HW;
13710 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13711 	if (dev->features & NETIF_F_GRO_HW)
13712 		dev->features &= ~NETIF_F_LRO;
13713 	dev->priv_flags |= IFF_UNICAST_FLT;
13714 
13715 	netif_set_tso_max_size(dev, GSO_MAX_SIZE);
13716 
13717 	dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
13718 			    NETDEV_XDP_ACT_RX_SG;
13719 
13720 #ifdef CONFIG_BNXT_SRIOV
13721 	init_waitqueue_head(&bp->sriov_cfg_wait);
13722 #endif
13723 	if (BNXT_SUPPORTS_TPA(bp)) {
13724 		bp->gro_func = bnxt_gro_func_5730x;
13725 		if (BNXT_CHIP_P4(bp))
13726 			bp->gro_func = bnxt_gro_func_5731x;
13727 		else if (BNXT_CHIP_P5(bp))
13728 			bp->gro_func = bnxt_gro_func_5750x;
13729 	}
13730 	if (!BNXT_CHIP_P4_PLUS(bp))
13731 		bp->flags |= BNXT_FLAG_DOUBLE_DB;
13732 
13733 	rc = bnxt_init_mac_addr(bp);
13734 	if (rc) {
13735 		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13736 		rc = -EADDRNOTAVAIL;
13737 		goto init_err_pci_clean;
13738 	}
13739 
13740 	if (BNXT_PF(bp)) {
13741 		/* Read the adapter's DSN to use as the eswitch switch_id */
13742 		rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13743 	}
13744 
13745 	/* MTU range: 60 - FW defined max */
13746 	dev->min_mtu = ETH_ZLEN;
13747 	dev->max_mtu = bp->max_mtu;
13748 
13749 	rc = bnxt_probe_phy(bp, true);
13750 	if (rc)
13751 		goto init_err_pci_clean;
13752 
13753 	bnxt_set_rx_skb_mode(bp, false);
13754 	bnxt_set_tpa_flags(bp);
13755 	bnxt_set_ring_params(bp);
13756 	rc = bnxt_set_dflt_rings(bp, true);
13757 	if (rc) {
13758 		if (BNXT_VF(bp) && rc == -ENODEV) {
13759 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13760 		} else {
13761 			netdev_err(bp->dev, "Not enough rings available.\n");
13762 			rc = -ENOMEM;
13763 		}
13764 		goto init_err_pci_clean;
13765 	}
13766 
13767 	bnxt_fw_init_one_p3(bp);
13768 
13769 	bnxt_init_dflt_coal(bp);
13770 
13771 	if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13772 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
13773 
13774 	rc = bnxt_init_int_mode(bp);
13775 	if (rc)
13776 		goto init_err_pci_clean;
13777 
13778 	/* No TC has been set yet and rings may have been trimmed due to
13779 	 * limited MSIX, so we re-initialize the TX rings per TC.
13780 	 */
13781 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13782 
13783 	if (BNXT_PF(bp)) {
13784 		if (!bnxt_pf_wq) {
13785 			bnxt_pf_wq =
13786 				create_singlethread_workqueue("bnxt_pf_wq");
13787 			if (!bnxt_pf_wq) {
13788 				dev_err(&pdev->dev, "Unable to create workqueue.\n");
13789 				rc = -ENOMEM;
13790 				goto init_err_pci_clean;
13791 			}
13792 		}
13793 		rc = bnxt_init_tc(bp);
13794 		if (rc)
13795 			netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13796 				   rc);
13797 	}
13798 
13799 	bnxt_inv_fw_health_reg(bp);
13800 	rc = bnxt_dl_register(bp);
13801 	if (rc)
13802 		goto init_err_dl;
13803 
13804 	rc = register_netdev(dev);
13805 	if (rc)
13806 		goto init_err_cleanup;
13807 
13808 	bnxt_dl_fw_reporters_create(bp);
13809 
13810 	bnxt_rdma_aux_device_init(bp);
13811 
13812 	bnxt_print_device_info(bp);
13813 
13814 	pci_save_state(pdev);
13815 
13816 	return 0;
13817 init_err_cleanup:
13818 	bnxt_dl_unregister(bp);
13819 init_err_dl:
13820 	bnxt_shutdown_tc(bp);
13821 	bnxt_clear_int_mode(bp);
13822 
13823 init_err_pci_clean:
13824 	bnxt_hwrm_func_drv_unrgtr(bp);
13825 	bnxt_free_hwrm_resources(bp);
13826 	bnxt_ethtool_free(bp);
13827 	bnxt_ptp_clear(bp);
13828 	kfree(bp->ptp_cfg);
13829 	bp->ptp_cfg = NULL;
13830 	kfree(bp->fw_health);
13831 	bp->fw_health = NULL;
13832 	bnxt_cleanup_pci(bp);
13833 	bnxt_free_ctx_mem(bp);
13834 	kfree(bp->ctx);
13835 	bp->ctx = NULL;
13836 	kfree(bp->rss_indir_tbl);
13837 	bp->rss_indir_tbl = NULL;
13838 
13839 init_err_free:
13840 	free_netdev(dev);
13841 	return rc;
13842 }
13843 
13844 static void bnxt_shutdown(struct pci_dev *pdev)
13845 {
13846 	struct net_device *dev = pci_get_drvdata(pdev);
13847 	struct bnxt *bp;
13848 
13849 	if (!dev)
13850 		return;
13851 
13852 	rtnl_lock();
13853 	bp = netdev_priv(dev);
13854 	if (!bp)
13855 		goto shutdown_exit;
13856 
13857 	if (netif_running(dev))
13858 		dev_close(dev);
13859 
13860 	bnxt_clear_int_mode(bp);
13861 	pci_disable_device(pdev);
13862 
13863 	if (system_state == SYSTEM_POWER_OFF) {
13864 		pci_wake_from_d3(pdev, bp->wol);
13865 		pci_set_power_state(pdev, PCI_D3hot);
13866 	}
13867 
13868 shutdown_exit:
13869 	rtnl_unlock();
13870 }
13871 
13872 #ifdef CONFIG_PM_SLEEP
13873 static int bnxt_suspend(struct device *device)
13874 {
13875 	struct net_device *dev = dev_get_drvdata(device);
13876 	struct bnxt *bp = netdev_priv(dev);
13877 	int rc = 0;
13878 
13879 	rtnl_lock();
13880 	bnxt_ulp_stop(bp);
13881 	if (netif_running(dev)) {
13882 		netif_device_detach(dev);
13883 		rc = bnxt_close(dev);
13884 	}
13885 	bnxt_hwrm_func_drv_unrgtr(bp);
13886 	pci_disable_device(bp->pdev);
13887 	bnxt_free_ctx_mem(bp);
13888 	kfree(bp->ctx);
13889 	bp->ctx = NULL;
13890 	rtnl_unlock();
13891 	return rc;
13892 }
13893 
13894 static int bnxt_resume(struct device *device)
13895 {
13896 	struct net_device *dev = dev_get_drvdata(device);
13897 	struct bnxt *bp = netdev_priv(dev);
13898 	int rc = 0;
13899 
13900 	rtnl_lock();
13901 	rc = pci_enable_device(bp->pdev);
13902 	if (rc) {
13903 		netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13904 			   rc);
13905 		goto resume_exit;
13906 	}
13907 	pci_set_master(bp->pdev);
13908 	if (bnxt_hwrm_ver_get(bp)) {
13909 		rc = -ENODEV;
13910 		goto resume_exit;
13911 	}
13912 	rc = bnxt_hwrm_func_reset(bp);
13913 	if (rc) {
13914 		rc = -EBUSY;
13915 		goto resume_exit;
13916 	}
13917 
13918 	rc = bnxt_hwrm_func_qcaps(bp);
13919 	if (rc)
13920 		goto resume_exit;
13921 
13922 	bnxt_clear_reservations(bp, true);
13923 
13924 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13925 		rc = -ENODEV;
13926 		goto resume_exit;
13927 	}
13928 
13929 	bnxt_get_wol_settings(bp);
13930 	if (netif_running(dev)) {
13931 		rc = bnxt_open(dev);
13932 		if (!rc)
13933 			netif_device_attach(dev);
13934 	}
13935 
13936 resume_exit:
13937 	bnxt_ulp_start(bp, rc);
13938 	if (!rc)
13939 		bnxt_reenable_sriov(bp);
13940 	rtnl_unlock();
13941 	return rc;
13942 }
13943 
13944 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13945 #define BNXT_PM_OPS (&bnxt_pm_ops)
13946 
13947 #else
13948 
13949 #define BNXT_PM_OPS NULL
13950 
13951 #endif /* CONFIG_PM_SLEEP */
13952 
13953 /**
13954  * bnxt_io_error_detected - called when PCI error is detected
13955  * @pdev: Pointer to PCI device
13956  * @state: The current pci connection state
13957  *
13958  * This function is called after a PCI bus error affecting
13959  * this device has been detected.
13960  */
13961 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13962 					       pci_channel_state_t state)
13963 {
13964 	struct net_device *netdev = pci_get_drvdata(pdev);
13965 	struct bnxt *bp = netdev_priv(netdev);
13966 	bool abort = false;
13967 
13968 	netdev_info(netdev, "PCI I/O error detected\n");
13969 
13970 	rtnl_lock();
13971 	netif_device_detach(netdev);
13972 
13973 	bnxt_ulp_stop(bp);
13974 
13975 	if (test_and_set_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13976 		netdev_err(bp->dev, "Firmware reset already in progress\n");
13977 		abort = true;
13978 	}
13979 
13980 	if (abort || state == pci_channel_io_perm_failure) {
13981 		rtnl_unlock();
13982 		return PCI_ERS_RESULT_DISCONNECT;
13983 	}
13984 
13985 	/* Link is not reliable anymore if state is pci_channel_io_frozen
13986 	 * so we disable bus master to prevent any potential bad DMAs before
13987 	 * freeing kernel memory.
13988 	 */
13989 	if (state == pci_channel_io_frozen) {
13990 		set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13991 		bnxt_fw_fatal_close(bp);
13992 	}
13993 
13994 	if (netif_running(netdev))
13995 		__bnxt_close_nic(bp, true, true);
13996 
13997 	if (pci_is_enabled(pdev))
13998 		pci_disable_device(pdev);
13999 	bnxt_free_ctx_mem(bp);
14000 	kfree(bp->ctx);
14001 	bp->ctx = NULL;
14002 	rtnl_unlock();
14003 
14004 	/* Request a slot slot reset. */
14005 	return PCI_ERS_RESULT_NEED_RESET;
14006 }
14007 
14008 /**
14009  * bnxt_io_slot_reset - called after the pci bus has been reset.
14010  * @pdev: Pointer to PCI device
14011  *
14012  * Restart the card from scratch, as if from a cold-boot.
14013  * At this point, the card has exprienced a hard reset,
14014  * followed by fixups by BIOS, and has its config space
14015  * set up identically to what it was at cold boot.
14016  */
14017 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
14018 {
14019 	pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
14020 	struct net_device *netdev = pci_get_drvdata(pdev);
14021 	struct bnxt *bp = netdev_priv(netdev);
14022 	int retry = 0;
14023 	int err = 0;
14024 	int off;
14025 
14026 	netdev_info(bp->dev, "PCI Slot Reset\n");
14027 
14028 	rtnl_lock();
14029 
14030 	if (pci_enable_device(pdev)) {
14031 		dev_err(&pdev->dev,
14032 			"Cannot re-enable PCI device after reset.\n");
14033 	} else {
14034 		pci_set_master(pdev);
14035 		/* Upon fatal error, our device internal logic that latches to
14036 		 * BAR value is getting reset and will restore only upon
14037 		 * rewritting the BARs.
14038 		 *
14039 		 * As pci_restore_state() does not re-write the BARs if the
14040 		 * value is same as saved value earlier, driver needs to
14041 		 * write the BARs to 0 to force restore, in case of fatal error.
14042 		 */
14043 		if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
14044 				       &bp->state)) {
14045 			for (off = PCI_BASE_ADDRESS_0;
14046 			     off <= PCI_BASE_ADDRESS_5; off += 4)
14047 				pci_write_config_dword(bp->pdev, off, 0);
14048 		}
14049 		pci_restore_state(pdev);
14050 		pci_save_state(pdev);
14051 
14052 		bnxt_inv_fw_health_reg(bp);
14053 		bnxt_try_map_fw_health_reg(bp);
14054 
14055 		/* In some PCIe AER scenarios, firmware may take up to
14056 		 * 10 seconds to become ready in the worst case.
14057 		 */
14058 		do {
14059 			err = bnxt_try_recover_fw(bp);
14060 			if (!err)
14061 				break;
14062 			retry++;
14063 		} while (retry < BNXT_FW_SLOT_RESET_RETRY);
14064 
14065 		if (err) {
14066 			dev_err(&pdev->dev, "Firmware not ready\n");
14067 			goto reset_exit;
14068 		}
14069 
14070 		err = bnxt_hwrm_func_reset(bp);
14071 		if (!err)
14072 			result = PCI_ERS_RESULT_RECOVERED;
14073 
14074 		bnxt_ulp_irq_stop(bp);
14075 		bnxt_clear_int_mode(bp);
14076 		err = bnxt_init_int_mode(bp);
14077 		bnxt_ulp_irq_restart(bp, err);
14078 	}
14079 
14080 reset_exit:
14081 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14082 	bnxt_clear_reservations(bp, true);
14083 	rtnl_unlock();
14084 
14085 	return result;
14086 }
14087 
14088 /**
14089  * bnxt_io_resume - called when traffic can start flowing again.
14090  * @pdev: Pointer to PCI device
14091  *
14092  * This callback is called when the error recovery driver tells
14093  * us that its OK to resume normal operation.
14094  */
14095 static void bnxt_io_resume(struct pci_dev *pdev)
14096 {
14097 	struct net_device *netdev = pci_get_drvdata(pdev);
14098 	struct bnxt *bp = netdev_priv(netdev);
14099 	int err;
14100 
14101 	netdev_info(bp->dev, "PCI Slot Resume\n");
14102 	rtnl_lock();
14103 
14104 	err = bnxt_hwrm_func_qcaps(bp);
14105 	if (!err && netif_running(netdev))
14106 		err = bnxt_open(netdev);
14107 
14108 	bnxt_ulp_start(bp, err);
14109 	if (!err) {
14110 		bnxt_reenable_sriov(bp);
14111 		netif_device_attach(netdev);
14112 	}
14113 
14114 	rtnl_unlock();
14115 }
14116 
14117 static const struct pci_error_handlers bnxt_err_handler = {
14118 	.error_detected	= bnxt_io_error_detected,
14119 	.slot_reset	= bnxt_io_slot_reset,
14120 	.resume		= bnxt_io_resume
14121 };
14122 
14123 static struct pci_driver bnxt_pci_driver = {
14124 	.name		= DRV_MODULE_NAME,
14125 	.id_table	= bnxt_pci_tbl,
14126 	.probe		= bnxt_init_one,
14127 	.remove		= bnxt_remove_one,
14128 	.shutdown	= bnxt_shutdown,
14129 	.driver.pm	= BNXT_PM_OPS,
14130 	.err_handler	= &bnxt_err_handler,
14131 #if defined(CONFIG_BNXT_SRIOV)
14132 	.sriov_configure = bnxt_sriov_configure,
14133 #endif
14134 };
14135 
14136 static int __init bnxt_init(void)
14137 {
14138 	int err;
14139 
14140 	bnxt_debug_init();
14141 	err = pci_register_driver(&bnxt_pci_driver);
14142 	if (err) {
14143 		bnxt_debug_exit();
14144 		return err;
14145 	}
14146 
14147 	return 0;
14148 }
14149 
14150 static void __exit bnxt_exit(void)
14151 {
14152 	pci_unregister_driver(&bnxt_pci_driver);
14153 	if (bnxt_pf_wq)
14154 		destroy_workqueue(bnxt_pf_wq);
14155 	bnxt_debug_exit();
14156 }
14157 
14158 module_init(bnxt_init);
14159 module_exit(bnxt_exit);
14160