1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2019 Broadcom Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  */
10 
11 #include <linux/module.h>
12 
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/gro.h>
41 #include <net/ip.h>
42 #include <net/tcp.h>
43 #include <net/udp.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46 #include <net/udp_tunnel.h>
47 #include <linux/workqueue.h>
48 #include <linux/prefetch.h>
49 #include <linux/cache.h>
50 #include <linux/log2.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool/helpers.h>
58 #include <linux/align.h>
59 #include <net/netdev_queues.h>
60 
61 #include "bnxt_hsi.h"
62 #include "bnxt.h"
63 #include "bnxt_hwrm.h"
64 #include "bnxt_ulp.h"
65 #include "bnxt_sriov.h"
66 #include "bnxt_ethtool.h"
67 #include "bnxt_dcb.h"
68 #include "bnxt_xdp.h"
69 #include "bnxt_ptp.h"
70 #include "bnxt_vfr.h"
71 #include "bnxt_tc.h"
72 #include "bnxt_devlink.h"
73 #include "bnxt_debugfs.h"
74 
75 #define BNXT_TX_TIMEOUT		(5 * HZ)
76 #define BNXT_DEF_MSG_ENABLE	(NETIF_MSG_DRV | NETIF_MSG_HW | \
77 				 NETIF_MSG_TX_ERR)
78 
79 MODULE_LICENSE("GPL");
80 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
81 
82 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
83 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
84 #define BNXT_RX_COPY_THRESH 256
85 
86 #define BNXT_TX_PUSH_THRESH 164
87 
88 /* indexed by enum board_idx */
89 static const struct {
90 	char *name;
91 } board_info[] = {
92 	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
93 	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
94 	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
95 	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
96 	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
97 	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
98 	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
99 	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
100 	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
101 	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
102 	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
103 	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
104 	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
105 	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
106 	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
107 	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
108 	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
109 	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
110 	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
111 	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
112 	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
113 	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
114 	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
115 	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
116 	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
117 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
118 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
119 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
120 	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
121 	[BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
122 	[BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
123 	[BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
124 	[BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
125 	[BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
126 	[BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
127 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
128 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
129 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
130 	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
131 	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
132 	[NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
133 	[NETXTREME_C_VF_HV] = { "Broadcom NetXtreme-C Virtual Function for Hyper-V" },
134 	[NETXTREME_E_VF_HV] = { "Broadcom NetXtreme-E Virtual Function for Hyper-V" },
135 	[NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
136 	[NETXTREME_E_P5_VF_HV] = { "Broadcom BCM5750X NetXtreme-E Virtual Function for Hyper-V" },
137 };
138 
139 static const struct pci_device_id bnxt_pci_tbl[] = {
140 	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
141 	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
142 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
143 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
144 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
145 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
146 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
147 	{ PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
148 	{ PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
149 	{ PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
150 	{ PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
151 	{ PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
152 	{ PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
153 	{ PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
154 	{ PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
155 	{ PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
156 	{ PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
157 	{ PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
158 	{ PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
159 	{ PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
160 	{ PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
161 	{ PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
162 	{ PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
163 	{ PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
164 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
165 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
166 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
167 	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
168 	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
169 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
170 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
171 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
172 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
173 	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
174 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
175 	{ PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
176 	{ PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
177 	{ PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
178 	{ PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57502_NPAR },
179 	{ PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
180 	{ PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57508_NPAR },
181 	{ PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57502_NPAR },
182 	{ PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
183 	{ PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57508_NPAR },
184 	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
185 	{ PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
186 #ifdef CONFIG_BNXT_SRIOV
187 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
188 	{ PCI_VDEVICE(BROADCOM, 0x1607), .driver_data = NETXTREME_E_VF_HV },
189 	{ PCI_VDEVICE(BROADCOM, 0x1608), .driver_data = NETXTREME_E_VF_HV },
190 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
191 	{ PCI_VDEVICE(BROADCOM, 0x16bd), .driver_data = NETXTREME_E_VF_HV },
192 	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
193 	{ PCI_VDEVICE(BROADCOM, 0x16c2), .driver_data = NETXTREME_C_VF_HV },
194 	{ PCI_VDEVICE(BROADCOM, 0x16c3), .driver_data = NETXTREME_C_VF_HV },
195 	{ PCI_VDEVICE(BROADCOM, 0x16c4), .driver_data = NETXTREME_E_VF_HV },
196 	{ PCI_VDEVICE(BROADCOM, 0x16c5), .driver_data = NETXTREME_E_VF_HV },
197 	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
198 	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
199 	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
200 	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
201 	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
202 	{ PCI_VDEVICE(BROADCOM, 0x16e6), .driver_data = NETXTREME_C_VF_HV },
203 	{ PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
204 	{ PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
205 	{ PCI_VDEVICE(BROADCOM, 0x1808), .driver_data = NETXTREME_E_P5_VF_HV },
206 	{ PCI_VDEVICE(BROADCOM, 0x1809), .driver_data = NETXTREME_E_P5_VF_HV },
207 	{ PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
208 #endif
209 	{ 0 }
210 };
211 
212 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
213 
214 static const u16 bnxt_vf_req_snif[] = {
215 	HWRM_FUNC_CFG,
216 	HWRM_FUNC_VF_CFG,
217 	HWRM_PORT_PHY_QCFG,
218 	HWRM_CFA_L2_FILTER_ALLOC,
219 };
220 
221 static const u16 bnxt_async_events_arr[] = {
222 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
223 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
224 	ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
225 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
226 	ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
227 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
228 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
229 	ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
230 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
231 	ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION,
232 	ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE,
233 	ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
234 	ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST,
235 	ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP,
236 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT,
237 	ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE,
238 };
239 
240 static struct workqueue_struct *bnxt_pf_wq;
241 
242 static bool bnxt_vf_pciid(enum board_idx idx)
243 {
244 	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
245 		idx == NETXTREME_S_VF || idx == NETXTREME_C_VF_HV ||
246 		idx == NETXTREME_E_VF_HV || idx == NETXTREME_E_P5_VF ||
247 		idx == NETXTREME_E_P5_VF_HV);
248 }
249 
250 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
251 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
252 #define DB_CP_IRQ_DIS_FLAGS	(DB_KEY_CP | DB_IRQ_DIS)
253 
254 #define BNXT_CP_DB_IRQ_DIS(db)						\
255 		writel(DB_CP_IRQ_DIS_FLAGS, db)
256 
257 #define BNXT_DB_CQ(db, idx)						\
258 	writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
259 
260 #define BNXT_DB_NQ_P5(db, idx)						\
261 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx),	\
262 		    (db)->doorbell)
263 
264 #define BNXT_DB_CQ_ARM(db, idx)						\
265 	writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
266 
267 #define BNXT_DB_NQ_ARM_P5(db, idx)					\
268 	bnxt_writeq(bp, (db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx),\
269 		    (db)->doorbell)
270 
271 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
272 {
273 	if (bp->flags & BNXT_FLAG_CHIP_P5)
274 		BNXT_DB_NQ_P5(db, idx);
275 	else
276 		BNXT_DB_CQ(db, idx);
277 }
278 
279 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
280 {
281 	if (bp->flags & BNXT_FLAG_CHIP_P5)
282 		BNXT_DB_NQ_ARM_P5(db, idx);
283 	else
284 		BNXT_DB_CQ_ARM(db, idx);
285 }
286 
287 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
288 {
289 	if (bp->flags & BNXT_FLAG_CHIP_P5)
290 		bnxt_writeq(bp, db->db_key64 | DBR_TYPE_CQ_ARMALL |
291 			    RING_CMP(idx), db->doorbell);
292 	else
293 		BNXT_DB_CQ(db, idx);
294 }
295 
296 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
297 {
298 	if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
299 		return;
300 
301 	if (BNXT_PF(bp))
302 		queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
303 	else
304 		schedule_delayed_work(&bp->fw_reset_task, delay);
305 }
306 
307 static void __bnxt_queue_sp_work(struct bnxt *bp)
308 {
309 	if (BNXT_PF(bp))
310 		queue_work(bnxt_pf_wq, &bp->sp_task);
311 	else
312 		schedule_work(&bp->sp_task);
313 }
314 
315 static void bnxt_queue_sp_work(struct bnxt *bp, unsigned int event)
316 {
317 	set_bit(event, &bp->sp_event);
318 	__bnxt_queue_sp_work(bp);
319 }
320 
321 static void bnxt_sched_reset_rxr(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
322 {
323 	if (!rxr->bnapi->in_reset) {
324 		rxr->bnapi->in_reset = true;
325 		if (bp->flags & BNXT_FLAG_CHIP_P5)
326 			set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
327 		else
328 			set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
329 		__bnxt_queue_sp_work(bp);
330 	}
331 	rxr->rx_next_cons = 0xffff;
332 }
333 
334 void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
335 			  int idx)
336 {
337 	struct bnxt_napi *bnapi = txr->bnapi;
338 
339 	if (bnapi->tx_fault)
340 		return;
341 
342 	netdev_err(bp->dev, "Invalid Tx completion (ring:%d tx_pkts:%d cons:%u prod:%u i:%d)",
343 		   txr->txq_index, bnapi->tx_pkts,
344 		   txr->tx_cons, txr->tx_prod, idx);
345 	WARN_ON_ONCE(1);
346 	bnapi->tx_fault = 1;
347 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
348 }
349 
350 const u16 bnxt_lhint_arr[] = {
351 	TX_BD_FLAGS_LHINT_512_AND_SMALLER,
352 	TX_BD_FLAGS_LHINT_512_TO_1023,
353 	TX_BD_FLAGS_LHINT_1024_TO_2047,
354 	TX_BD_FLAGS_LHINT_1024_TO_2047,
355 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
356 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
357 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
358 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
359 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
360 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
361 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
362 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
363 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
364 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
365 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
366 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
367 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
368 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
369 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
370 };
371 
372 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
373 {
374 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
375 
376 	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
377 		return 0;
378 
379 	return md_dst->u.port_info.port_id;
380 }
381 
382 static void bnxt_txr_db_kick(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
383 			     u16 prod)
384 {
385 	bnxt_db_write(bp, &txr->tx_db, prod);
386 	txr->kick_pending = 0;
387 }
388 
389 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
390 {
391 	struct bnxt *bp = netdev_priv(dev);
392 	struct tx_bd *txbd;
393 	struct tx_bd_ext *txbd1;
394 	struct netdev_queue *txq;
395 	int i;
396 	dma_addr_t mapping;
397 	unsigned int length, pad = 0;
398 	u32 len, free_size, vlan_tag_flags, cfa_action, flags;
399 	u16 prod, last_frag;
400 	struct pci_dev *pdev = bp->pdev;
401 	struct bnxt_tx_ring_info *txr;
402 	struct bnxt_sw_tx_bd *tx_buf;
403 	__le32 lflags = 0;
404 
405 	i = skb_get_queue_mapping(skb);
406 	if (unlikely(i >= bp->tx_nr_rings)) {
407 		dev_kfree_skb_any(skb);
408 		dev_core_stats_tx_dropped_inc(dev);
409 		return NETDEV_TX_OK;
410 	}
411 
412 	txq = netdev_get_tx_queue(dev, i);
413 	txr = &bp->tx_ring[bp->tx_ring_map[i]];
414 	prod = txr->tx_prod;
415 
416 	free_size = bnxt_tx_avail(bp, txr);
417 	if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
418 		/* We must have raced with NAPI cleanup */
419 		if (net_ratelimit() && txr->kick_pending)
420 			netif_warn(bp, tx_err, dev,
421 				   "bnxt: ring busy w/ flush pending!\n");
422 		if (!netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
423 					bp->tx_wake_thresh))
424 			return NETDEV_TX_BUSY;
425 	}
426 
427 	if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
428 		goto tx_free;
429 
430 	length = skb->len;
431 	len = skb_headlen(skb);
432 	last_frag = skb_shinfo(skb)->nr_frags;
433 
434 	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
435 
436 	txbd->tx_bd_opaque = prod;
437 
438 	tx_buf = &txr->tx_buf_ring[prod];
439 	tx_buf->skb = skb;
440 	tx_buf->nr_frags = last_frag;
441 
442 	vlan_tag_flags = 0;
443 	cfa_action = bnxt_xmit_get_cfa_action(skb);
444 	if (skb_vlan_tag_present(skb)) {
445 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
446 				 skb_vlan_tag_get(skb);
447 		/* Currently supports 8021Q, 8021AD vlan offloads
448 		 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
449 		 */
450 		if (skb->vlan_proto == htons(ETH_P_8021Q))
451 			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
452 	}
453 
454 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
455 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
456 
457 		if (ptp && ptp->tx_tstamp_en && !skb_is_gso(skb) &&
458 		    atomic_dec_if_positive(&ptp->tx_avail) >= 0) {
459 			if (!bnxt_ptp_parse(skb, &ptp->tx_seqid,
460 					    &ptp->tx_hdr_off)) {
461 				if (vlan_tag_flags)
462 					ptp->tx_hdr_off += VLAN_HLEN;
463 				lflags |= cpu_to_le32(TX_BD_FLAGS_STAMP);
464 				skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
465 			} else {
466 				atomic_inc(&bp->ptp_cfg->tx_avail);
467 			}
468 		}
469 	}
470 
471 	if (unlikely(skb->no_fcs))
472 		lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
473 
474 	if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh &&
475 	    !lflags) {
476 		struct tx_push_buffer *tx_push_buf = txr->tx_push;
477 		struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
478 		struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
479 		void __iomem *db = txr->tx_db.doorbell;
480 		void *pdata = tx_push_buf->data;
481 		u64 *end;
482 		int j, push_len;
483 
484 		/* Set COAL_NOW to be ready quickly for the next push */
485 		tx_push->tx_bd_len_flags_type =
486 			cpu_to_le32((length << TX_BD_LEN_SHIFT) |
487 					TX_BD_TYPE_LONG_TX_BD |
488 					TX_BD_FLAGS_LHINT_512_AND_SMALLER |
489 					TX_BD_FLAGS_COAL_NOW |
490 					TX_BD_FLAGS_PACKET_END |
491 					(2 << TX_BD_FLAGS_BD_CNT_SHIFT));
492 
493 		if (skb->ip_summed == CHECKSUM_PARTIAL)
494 			tx_push1->tx_bd_hsize_lflags =
495 					cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
496 		else
497 			tx_push1->tx_bd_hsize_lflags = 0;
498 
499 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
500 		tx_push1->tx_bd_cfa_action =
501 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
502 
503 		end = pdata + length;
504 		end = PTR_ALIGN(end, 8) - 1;
505 		*end = 0;
506 
507 		skb_copy_from_linear_data(skb, pdata, len);
508 		pdata += len;
509 		for (j = 0; j < last_frag; j++) {
510 			skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
511 			void *fptr;
512 
513 			fptr = skb_frag_address_safe(frag);
514 			if (!fptr)
515 				goto normal_tx;
516 
517 			memcpy(pdata, fptr, skb_frag_size(frag));
518 			pdata += skb_frag_size(frag);
519 		}
520 
521 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
522 		txbd->tx_bd_haddr = txr->data_mapping;
523 		prod = NEXT_TX(prod);
524 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
525 		memcpy(txbd, tx_push1, sizeof(*txbd));
526 		prod = NEXT_TX(prod);
527 		tx_push->doorbell =
528 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
529 		WRITE_ONCE(txr->tx_prod, prod);
530 
531 		tx_buf->is_push = 1;
532 		netdev_tx_sent_queue(txq, skb->len);
533 		wmb();	/* Sync is_push and byte queue before pushing data */
534 
535 		push_len = (length + sizeof(*tx_push) + 7) / 8;
536 		if (push_len > 16) {
537 			__iowrite64_copy(db, tx_push_buf, 16);
538 			__iowrite32_copy(db + 4, tx_push_buf + 1,
539 					 (push_len - 16) << 1);
540 		} else {
541 			__iowrite64_copy(db, tx_push_buf, push_len);
542 		}
543 
544 		goto tx_done;
545 	}
546 
547 normal_tx:
548 	if (length < BNXT_MIN_PKT_SIZE) {
549 		pad = BNXT_MIN_PKT_SIZE - length;
550 		if (skb_pad(skb, pad))
551 			/* SKB already freed. */
552 			goto tx_kick_pending;
553 		length = BNXT_MIN_PKT_SIZE;
554 	}
555 
556 	mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
557 
558 	if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
559 		goto tx_free;
560 
561 	dma_unmap_addr_set(tx_buf, mapping, mapping);
562 	flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
563 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
564 
565 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
566 
567 	prod = NEXT_TX(prod);
568 	txbd1 = (struct tx_bd_ext *)
569 		&txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
570 
571 	txbd1->tx_bd_hsize_lflags = lflags;
572 	if (skb_is_gso(skb)) {
573 		u32 hdr_len;
574 
575 		if (skb->encapsulation)
576 			hdr_len = skb_inner_tcp_all_headers(skb);
577 		else
578 			hdr_len = skb_tcp_all_headers(skb);
579 
580 		txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_LSO |
581 					TX_BD_FLAGS_T_IPID |
582 					(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
583 		length = skb_shinfo(skb)->gso_size;
584 		txbd1->tx_bd_mss = cpu_to_le32(length);
585 		length += hdr_len;
586 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
587 		txbd1->tx_bd_hsize_lflags |=
588 			cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
589 		txbd1->tx_bd_mss = 0;
590 	}
591 
592 	length >>= 9;
593 	if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
594 		dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
595 				     skb->len);
596 		i = 0;
597 		goto tx_dma_error;
598 	}
599 	flags |= bnxt_lhint_arr[length];
600 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
601 
602 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
603 	txbd1->tx_bd_cfa_action =
604 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
605 	for (i = 0; i < last_frag; i++) {
606 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
607 
608 		prod = NEXT_TX(prod);
609 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
610 
611 		len = skb_frag_size(frag);
612 		mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
613 					   DMA_TO_DEVICE);
614 
615 		if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
616 			goto tx_dma_error;
617 
618 		tx_buf = &txr->tx_buf_ring[prod];
619 		dma_unmap_addr_set(tx_buf, mapping, mapping);
620 
621 		txbd->tx_bd_haddr = cpu_to_le64(mapping);
622 
623 		flags = len << TX_BD_LEN_SHIFT;
624 		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
625 	}
626 
627 	flags &= ~TX_BD_LEN;
628 	txbd->tx_bd_len_flags_type =
629 		cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
630 			    TX_BD_FLAGS_PACKET_END);
631 
632 	netdev_tx_sent_queue(txq, skb->len);
633 
634 	skb_tx_timestamp(skb);
635 
636 	/* Sync BD data before updating doorbell */
637 	wmb();
638 
639 	prod = NEXT_TX(prod);
640 	WRITE_ONCE(txr->tx_prod, prod);
641 
642 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
643 		bnxt_txr_db_kick(bp, txr, prod);
644 	else
645 		txr->kick_pending = 1;
646 
647 tx_done:
648 
649 	if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
650 		if (netdev_xmit_more() && !tx_buf->is_push)
651 			bnxt_txr_db_kick(bp, txr, prod);
652 
653 		netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr),
654 				   bp->tx_wake_thresh);
655 	}
656 	return NETDEV_TX_OK;
657 
658 tx_dma_error:
659 	if (BNXT_TX_PTP_IS_SET(lflags))
660 		atomic_inc(&bp->ptp_cfg->tx_avail);
661 
662 	last_frag = i;
663 
664 	/* start back at beginning and unmap skb */
665 	prod = txr->tx_prod;
666 	tx_buf = &txr->tx_buf_ring[prod];
667 	dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
668 			 skb_headlen(skb), DMA_TO_DEVICE);
669 	prod = NEXT_TX(prod);
670 
671 	/* unmap remaining mapped pages */
672 	for (i = 0; i < last_frag; i++) {
673 		prod = NEXT_TX(prod);
674 		tx_buf = &txr->tx_buf_ring[prod];
675 		dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
676 			       skb_frag_size(&skb_shinfo(skb)->frags[i]),
677 			       DMA_TO_DEVICE);
678 	}
679 
680 tx_free:
681 	dev_kfree_skb_any(skb);
682 tx_kick_pending:
683 	if (txr->kick_pending)
684 		bnxt_txr_db_kick(bp, txr, txr->tx_prod);
685 	txr->tx_buf_ring[txr->tx_prod].skb = NULL;
686 	dev_core_stats_tx_dropped_inc(dev);
687 	return NETDEV_TX_OK;
688 }
689 
690 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
691 {
692 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
693 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
694 	u16 cons = txr->tx_cons;
695 	struct pci_dev *pdev = bp->pdev;
696 	int nr_pkts = bnapi->tx_pkts;
697 	int i;
698 	unsigned int tx_bytes = 0;
699 
700 	for (i = 0; i < nr_pkts; i++) {
701 		struct bnxt_sw_tx_bd *tx_buf;
702 		struct sk_buff *skb;
703 		int j, last;
704 
705 		tx_buf = &txr->tx_buf_ring[cons];
706 		cons = NEXT_TX(cons);
707 		skb = tx_buf->skb;
708 		tx_buf->skb = NULL;
709 
710 		if (unlikely(!skb)) {
711 			bnxt_sched_reset_txr(bp, txr, i);
712 			return;
713 		}
714 
715 		tx_bytes += skb->len;
716 
717 		if (tx_buf->is_push) {
718 			tx_buf->is_push = 0;
719 			goto next_tx_int;
720 		}
721 
722 		dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
723 				 skb_headlen(skb), DMA_TO_DEVICE);
724 		last = tx_buf->nr_frags;
725 
726 		for (j = 0; j < last; j++) {
727 			cons = NEXT_TX(cons);
728 			tx_buf = &txr->tx_buf_ring[cons];
729 			dma_unmap_page(
730 				&pdev->dev,
731 				dma_unmap_addr(tx_buf, mapping),
732 				skb_frag_size(&skb_shinfo(skb)->frags[j]),
733 				DMA_TO_DEVICE);
734 		}
735 		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) {
736 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
737 				/* PTP worker takes ownership of the skb */
738 				if (!bnxt_get_tx_ts_p5(bp, skb))
739 					skb = NULL;
740 				else
741 					atomic_inc(&bp->ptp_cfg->tx_avail);
742 			}
743 		}
744 
745 next_tx_int:
746 		cons = NEXT_TX(cons);
747 
748 		dev_consume_skb_any(skb);
749 	}
750 
751 	bnapi->tx_pkts = 0;
752 	WRITE_ONCE(txr->tx_cons, cons);
753 
754 	__netif_txq_completed_wake(txq, nr_pkts, tx_bytes,
755 				   bnxt_tx_avail(bp, txr), bp->tx_wake_thresh,
756 				   READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING);
757 }
758 
759 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
760 					 struct bnxt_rx_ring_info *rxr,
761 					 unsigned int *offset,
762 					 gfp_t gfp)
763 {
764 	struct page *page;
765 
766 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
767 		page = page_pool_dev_alloc_frag(rxr->page_pool, offset,
768 						BNXT_RX_PAGE_SIZE);
769 	} else {
770 		page = page_pool_dev_alloc_pages(rxr->page_pool);
771 		*offset = 0;
772 	}
773 	if (!page)
774 		return NULL;
775 
776 	*mapping = page_pool_get_dma_addr(page) + *offset;
777 	return page;
778 }
779 
780 static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping,
781 				       gfp_t gfp)
782 {
783 	u8 *data;
784 	struct pci_dev *pdev = bp->pdev;
785 
786 	if (gfp == GFP_ATOMIC)
787 		data = napi_alloc_frag(bp->rx_buf_size);
788 	else
789 		data = netdev_alloc_frag(bp->rx_buf_size);
790 	if (!data)
791 		return NULL;
792 
793 	*mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
794 					bp->rx_buf_use_size, bp->rx_dir,
795 					DMA_ATTR_WEAK_ORDERING);
796 
797 	if (dma_mapping_error(&pdev->dev, *mapping)) {
798 		skb_free_frag(data);
799 		data = NULL;
800 	}
801 	return data;
802 }
803 
804 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
805 		       u16 prod, gfp_t gfp)
806 {
807 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
808 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
809 	dma_addr_t mapping;
810 
811 	if (BNXT_RX_PAGE_MODE(bp)) {
812 		unsigned int offset;
813 		struct page *page =
814 			__bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
815 
816 		if (!page)
817 			return -ENOMEM;
818 
819 		mapping += bp->rx_dma_offset;
820 		rx_buf->data = page;
821 		rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset;
822 	} else {
823 		u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, gfp);
824 
825 		if (!data)
826 			return -ENOMEM;
827 
828 		rx_buf->data = data;
829 		rx_buf->data_ptr = data + bp->rx_offset;
830 	}
831 	rx_buf->mapping = mapping;
832 
833 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
834 	return 0;
835 }
836 
837 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
838 {
839 	u16 prod = rxr->rx_prod;
840 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
841 	struct rx_bd *cons_bd, *prod_bd;
842 
843 	prod_rx_buf = &rxr->rx_buf_ring[prod];
844 	cons_rx_buf = &rxr->rx_buf_ring[cons];
845 
846 	prod_rx_buf->data = data;
847 	prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
848 
849 	prod_rx_buf->mapping = cons_rx_buf->mapping;
850 
851 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
852 	cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
853 
854 	prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
855 }
856 
857 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
858 {
859 	u16 next, max = rxr->rx_agg_bmap_size;
860 
861 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
862 	if (next >= max)
863 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
864 	return next;
865 }
866 
867 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
868 				     struct bnxt_rx_ring_info *rxr,
869 				     u16 prod, gfp_t gfp)
870 {
871 	struct rx_bd *rxbd =
872 		&rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
873 	struct bnxt_sw_rx_agg_bd *rx_agg_buf;
874 	struct page *page;
875 	dma_addr_t mapping;
876 	u16 sw_prod = rxr->rx_sw_agg_prod;
877 	unsigned int offset = 0;
878 
879 	page = __bnxt_alloc_rx_page(bp, &mapping, rxr, &offset, gfp);
880 
881 	if (!page)
882 		return -ENOMEM;
883 
884 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
885 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
886 
887 	__set_bit(sw_prod, rxr->rx_agg_bmap);
888 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
889 	rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
890 
891 	rx_agg_buf->page = page;
892 	rx_agg_buf->offset = offset;
893 	rx_agg_buf->mapping = mapping;
894 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
895 	rxbd->rx_bd_opaque = sw_prod;
896 	return 0;
897 }
898 
899 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
900 				       struct bnxt_cp_ring_info *cpr,
901 				       u16 cp_cons, u16 curr)
902 {
903 	struct rx_agg_cmp *agg;
904 
905 	cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
906 	agg = (struct rx_agg_cmp *)
907 		&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
908 	return agg;
909 }
910 
911 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
912 					      struct bnxt_rx_ring_info *rxr,
913 					      u16 agg_id, u16 curr)
914 {
915 	struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
916 
917 	return &tpa_info->agg_arr[curr];
918 }
919 
920 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
921 				   u16 start, u32 agg_bufs, bool tpa)
922 {
923 	struct bnxt_napi *bnapi = cpr->bnapi;
924 	struct bnxt *bp = bnapi->bp;
925 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
926 	u16 prod = rxr->rx_agg_prod;
927 	u16 sw_prod = rxr->rx_sw_agg_prod;
928 	bool p5_tpa = false;
929 	u32 i;
930 
931 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
932 		p5_tpa = true;
933 
934 	for (i = 0; i < agg_bufs; i++) {
935 		u16 cons;
936 		struct rx_agg_cmp *agg;
937 		struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
938 		struct rx_bd *prod_bd;
939 		struct page *page;
940 
941 		if (p5_tpa)
942 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
943 		else
944 			agg = bnxt_get_agg(bp, cpr, idx, start + i);
945 		cons = agg->rx_agg_cmp_opaque;
946 		__clear_bit(cons, rxr->rx_agg_bmap);
947 
948 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
949 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
950 
951 		__set_bit(sw_prod, rxr->rx_agg_bmap);
952 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
953 		cons_rx_buf = &rxr->rx_agg_ring[cons];
954 
955 		/* It is possible for sw_prod to be equal to cons, so
956 		 * set cons_rx_buf->page to NULL first.
957 		 */
958 		page = cons_rx_buf->page;
959 		cons_rx_buf->page = NULL;
960 		prod_rx_buf->page = page;
961 		prod_rx_buf->offset = cons_rx_buf->offset;
962 
963 		prod_rx_buf->mapping = cons_rx_buf->mapping;
964 
965 		prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
966 
967 		prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
968 		prod_bd->rx_bd_opaque = sw_prod;
969 
970 		prod = NEXT_RX_AGG(prod);
971 		sw_prod = NEXT_RX_AGG(sw_prod);
972 	}
973 	rxr->rx_agg_prod = prod;
974 	rxr->rx_sw_agg_prod = sw_prod;
975 }
976 
977 static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp,
978 					      struct bnxt_rx_ring_info *rxr,
979 					      u16 cons, void *data, u8 *data_ptr,
980 					      dma_addr_t dma_addr,
981 					      unsigned int offset_and_len)
982 {
983 	unsigned int len = offset_and_len & 0xffff;
984 	struct page *page = data;
985 	u16 prod = rxr->rx_prod;
986 	struct sk_buff *skb;
987 	int err;
988 
989 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
990 	if (unlikely(err)) {
991 		bnxt_reuse_rx_data(rxr, cons, data);
992 		return NULL;
993 	}
994 	dma_addr -= bp->rx_dma_offset;
995 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
996 				bp->rx_dir);
997 	skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE);
998 	if (!skb) {
999 		page_pool_recycle_direct(rxr->page_pool, page);
1000 		return NULL;
1001 	}
1002 	skb_mark_for_recycle(skb);
1003 	skb_reserve(skb, bp->rx_offset);
1004 	__skb_put(skb, len);
1005 
1006 	return skb;
1007 }
1008 
1009 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
1010 					struct bnxt_rx_ring_info *rxr,
1011 					u16 cons, void *data, u8 *data_ptr,
1012 					dma_addr_t dma_addr,
1013 					unsigned int offset_and_len)
1014 {
1015 	unsigned int payload = offset_and_len >> 16;
1016 	unsigned int len = offset_and_len & 0xffff;
1017 	skb_frag_t *frag;
1018 	struct page *page = data;
1019 	u16 prod = rxr->rx_prod;
1020 	struct sk_buff *skb;
1021 	int off, err;
1022 
1023 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1024 	if (unlikely(err)) {
1025 		bnxt_reuse_rx_data(rxr, cons, data);
1026 		return NULL;
1027 	}
1028 	dma_addr -= bp->rx_dma_offset;
1029 	dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE,
1030 				bp->rx_dir);
1031 
1032 	if (unlikely(!payload))
1033 		payload = eth_get_headlen(bp->dev, data_ptr, len);
1034 
1035 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
1036 	if (!skb) {
1037 		page_pool_recycle_direct(rxr->page_pool, page);
1038 		return NULL;
1039 	}
1040 
1041 	skb_mark_for_recycle(skb);
1042 	off = (void *)data_ptr - page_address(page);
1043 	skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE);
1044 	memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
1045 	       payload + NET_IP_ALIGN);
1046 
1047 	frag = &skb_shinfo(skb)->frags[0];
1048 	skb_frag_size_sub(frag, payload);
1049 	skb_frag_off_add(frag, payload);
1050 	skb->data_len -= payload;
1051 	skb->tail += payload;
1052 
1053 	return skb;
1054 }
1055 
1056 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
1057 				   struct bnxt_rx_ring_info *rxr, u16 cons,
1058 				   void *data, u8 *data_ptr,
1059 				   dma_addr_t dma_addr,
1060 				   unsigned int offset_and_len)
1061 {
1062 	u16 prod = rxr->rx_prod;
1063 	struct sk_buff *skb;
1064 	int err;
1065 
1066 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
1067 	if (unlikely(err)) {
1068 		bnxt_reuse_rx_data(rxr, cons, data);
1069 		return NULL;
1070 	}
1071 
1072 	skb = napi_build_skb(data, bp->rx_buf_size);
1073 	dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
1074 			       bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
1075 	if (!skb) {
1076 		skb_free_frag(data);
1077 		return NULL;
1078 	}
1079 
1080 	skb_reserve(skb, bp->rx_offset);
1081 	skb_put(skb, offset_and_len & 0xffff);
1082 	return skb;
1083 }
1084 
1085 static u32 __bnxt_rx_agg_pages(struct bnxt *bp,
1086 			       struct bnxt_cp_ring_info *cpr,
1087 			       struct skb_shared_info *shinfo,
1088 			       u16 idx, u32 agg_bufs, bool tpa,
1089 			       struct xdp_buff *xdp)
1090 {
1091 	struct bnxt_napi *bnapi = cpr->bnapi;
1092 	struct pci_dev *pdev = bp->pdev;
1093 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1094 	u16 prod = rxr->rx_agg_prod;
1095 	u32 i, total_frag_len = 0;
1096 	bool p5_tpa = false;
1097 
1098 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1099 		p5_tpa = true;
1100 
1101 	for (i = 0; i < agg_bufs; i++) {
1102 		skb_frag_t *frag = &shinfo->frags[i];
1103 		u16 cons, frag_len;
1104 		struct rx_agg_cmp *agg;
1105 		struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1106 		struct page *page;
1107 		dma_addr_t mapping;
1108 
1109 		if (p5_tpa)
1110 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1111 		else
1112 			agg = bnxt_get_agg(bp, cpr, idx, i);
1113 		cons = agg->rx_agg_cmp_opaque;
1114 		frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1115 			    RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1116 
1117 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1118 		skb_frag_fill_page_desc(frag, cons_rx_buf->page,
1119 					cons_rx_buf->offset, frag_len);
1120 		shinfo->nr_frags = i + 1;
1121 		__clear_bit(cons, rxr->rx_agg_bmap);
1122 
1123 		/* It is possible for bnxt_alloc_rx_page() to allocate
1124 		 * a sw_prod index that equals the cons index, so we
1125 		 * need to clear the cons entry now.
1126 		 */
1127 		mapping = cons_rx_buf->mapping;
1128 		page = cons_rx_buf->page;
1129 		cons_rx_buf->page = NULL;
1130 
1131 		if (xdp && page_is_pfmemalloc(page))
1132 			xdp_buff_set_frag_pfmemalloc(xdp);
1133 
1134 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1135 			--shinfo->nr_frags;
1136 			cons_rx_buf->page = page;
1137 
1138 			/* Update prod since possibly some pages have been
1139 			 * allocated already.
1140 			 */
1141 			rxr->rx_agg_prod = prod;
1142 			bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1143 			return 0;
1144 		}
1145 
1146 		dma_sync_single_for_cpu(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1147 					bp->rx_dir);
1148 
1149 		total_frag_len += frag_len;
1150 		prod = NEXT_RX_AGG(prod);
1151 	}
1152 	rxr->rx_agg_prod = prod;
1153 	return total_frag_len;
1154 }
1155 
1156 static struct sk_buff *bnxt_rx_agg_pages_skb(struct bnxt *bp,
1157 					     struct bnxt_cp_ring_info *cpr,
1158 					     struct sk_buff *skb, u16 idx,
1159 					     u32 agg_bufs, bool tpa)
1160 {
1161 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1162 	u32 total_frag_len = 0;
1163 
1164 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo, idx,
1165 					     agg_bufs, tpa, NULL);
1166 	if (!total_frag_len) {
1167 		skb_mark_for_recycle(skb);
1168 		dev_kfree_skb(skb);
1169 		return NULL;
1170 	}
1171 
1172 	skb->data_len += total_frag_len;
1173 	skb->len += total_frag_len;
1174 	skb->truesize += BNXT_RX_PAGE_SIZE * agg_bufs;
1175 	return skb;
1176 }
1177 
1178 static u32 bnxt_rx_agg_pages_xdp(struct bnxt *bp,
1179 				 struct bnxt_cp_ring_info *cpr,
1180 				 struct xdp_buff *xdp, u16 idx,
1181 				 u32 agg_bufs, bool tpa)
1182 {
1183 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp);
1184 	u32 total_frag_len = 0;
1185 
1186 	if (!xdp_buff_has_frags(xdp))
1187 		shinfo->nr_frags = 0;
1188 
1189 	total_frag_len = __bnxt_rx_agg_pages(bp, cpr, shinfo,
1190 					     idx, agg_bufs, tpa, xdp);
1191 	if (total_frag_len) {
1192 		xdp_buff_set_frags_flag(xdp);
1193 		shinfo->nr_frags = agg_bufs;
1194 		shinfo->xdp_frags_size = total_frag_len;
1195 	}
1196 	return total_frag_len;
1197 }
1198 
1199 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1200 			       u8 agg_bufs, u32 *raw_cons)
1201 {
1202 	u16 last;
1203 	struct rx_agg_cmp *agg;
1204 
1205 	*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1206 	last = RING_CMP(*raw_cons);
1207 	agg = (struct rx_agg_cmp *)
1208 		&cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1209 	return RX_AGG_CMP_VALID(agg, *raw_cons);
1210 }
1211 
1212 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1213 					    unsigned int len,
1214 					    dma_addr_t mapping)
1215 {
1216 	struct bnxt *bp = bnapi->bp;
1217 	struct pci_dev *pdev = bp->pdev;
1218 	struct sk_buff *skb;
1219 
1220 	skb = napi_alloc_skb(&bnapi->napi, len);
1221 	if (!skb)
1222 		return NULL;
1223 
1224 	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1225 				bp->rx_dir);
1226 
1227 	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1228 	       len + NET_IP_ALIGN);
1229 
1230 	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1231 				   bp->rx_dir);
1232 
1233 	skb_put(skb, len);
1234 	return skb;
1235 }
1236 
1237 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1238 			   u32 *raw_cons, void *cmp)
1239 {
1240 	struct rx_cmp *rxcmp = cmp;
1241 	u32 tmp_raw_cons = *raw_cons;
1242 	u8 cmp_type, agg_bufs = 0;
1243 
1244 	cmp_type = RX_CMP_TYPE(rxcmp);
1245 
1246 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1247 		agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1248 			    RX_CMP_AGG_BUFS) >>
1249 			   RX_CMP_AGG_BUFS_SHIFT;
1250 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1251 		struct rx_tpa_end_cmp *tpa_end = cmp;
1252 
1253 		if (bp->flags & BNXT_FLAG_CHIP_P5)
1254 			return 0;
1255 
1256 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1257 	}
1258 
1259 	if (agg_bufs) {
1260 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1261 			return -EBUSY;
1262 	}
1263 	*raw_cons = tmp_raw_cons;
1264 	return 0;
1265 }
1266 
1267 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1268 {
1269 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1270 	u16 idx = agg_id & MAX_TPA_P5_MASK;
1271 
1272 	if (test_bit(idx, map->agg_idx_bmap))
1273 		idx = find_first_zero_bit(map->agg_idx_bmap,
1274 					  BNXT_AGG_IDX_BMAP_SIZE);
1275 	__set_bit(idx, map->agg_idx_bmap);
1276 	map->agg_id_tbl[agg_id] = idx;
1277 	return idx;
1278 }
1279 
1280 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1281 {
1282 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1283 
1284 	__clear_bit(idx, map->agg_idx_bmap);
1285 }
1286 
1287 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1288 {
1289 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1290 
1291 	return map->agg_id_tbl[agg_id];
1292 }
1293 
1294 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1295 			   struct rx_tpa_start_cmp *tpa_start,
1296 			   struct rx_tpa_start_cmp_ext *tpa_start1)
1297 {
1298 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1299 	struct bnxt_tpa_info *tpa_info;
1300 	u16 cons, prod, agg_id;
1301 	struct rx_bd *prod_bd;
1302 	dma_addr_t mapping;
1303 
1304 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1305 		agg_id = TPA_START_AGG_ID_P5(tpa_start);
1306 		agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1307 	} else {
1308 		agg_id = TPA_START_AGG_ID(tpa_start);
1309 	}
1310 	cons = tpa_start->rx_tpa_start_cmp_opaque;
1311 	prod = rxr->rx_prod;
1312 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1313 	prod_rx_buf = &rxr->rx_buf_ring[prod];
1314 	tpa_info = &rxr->rx_tpa[agg_id];
1315 
1316 	if (unlikely(cons != rxr->rx_next_cons ||
1317 		     TPA_START_ERROR(tpa_start))) {
1318 		netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1319 			    cons, rxr->rx_next_cons,
1320 			    TPA_START_ERROR_CODE(tpa_start1));
1321 		bnxt_sched_reset_rxr(bp, rxr);
1322 		return;
1323 	}
1324 	/* Store cfa_code in tpa_info to use in tpa_end
1325 	 * completion processing.
1326 	 */
1327 	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1328 	prod_rx_buf->data = tpa_info->data;
1329 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
1330 
1331 	mapping = tpa_info->mapping;
1332 	prod_rx_buf->mapping = mapping;
1333 
1334 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1335 
1336 	prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1337 
1338 	tpa_info->data = cons_rx_buf->data;
1339 	tpa_info->data_ptr = cons_rx_buf->data_ptr;
1340 	cons_rx_buf->data = NULL;
1341 	tpa_info->mapping = cons_rx_buf->mapping;
1342 
1343 	tpa_info->len =
1344 		le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1345 				RX_TPA_START_CMP_LEN_SHIFT;
1346 	if (likely(TPA_START_HASH_VALID(tpa_start))) {
1347 		u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1348 
1349 		tpa_info->hash_type = PKT_HASH_TYPE_L4;
1350 		tpa_info->gso_type = SKB_GSO_TCPV4;
1351 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1352 		if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1353 			tpa_info->gso_type = SKB_GSO_TCPV6;
1354 		tpa_info->rss_hash =
1355 			le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1356 	} else {
1357 		tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1358 		tpa_info->gso_type = 0;
1359 		netif_warn(bp, rx_err, bp->dev, "TPA packet without valid hash\n");
1360 	}
1361 	tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1362 	tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1363 	tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1364 	tpa_info->agg_count = 0;
1365 
1366 	rxr->rx_prod = NEXT_RX(prod);
1367 	cons = NEXT_RX(cons);
1368 	rxr->rx_next_cons = NEXT_RX(cons);
1369 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1370 
1371 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1372 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1373 	cons_rx_buf->data = NULL;
1374 }
1375 
1376 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1377 {
1378 	if (agg_bufs)
1379 		bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1380 }
1381 
1382 #ifdef CONFIG_INET
1383 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1384 {
1385 	struct udphdr *uh = NULL;
1386 
1387 	if (ip_proto == htons(ETH_P_IP)) {
1388 		struct iphdr *iph = (struct iphdr *)skb->data;
1389 
1390 		if (iph->protocol == IPPROTO_UDP)
1391 			uh = (struct udphdr *)(iph + 1);
1392 	} else {
1393 		struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1394 
1395 		if (iph->nexthdr == IPPROTO_UDP)
1396 			uh = (struct udphdr *)(iph + 1);
1397 	}
1398 	if (uh) {
1399 		if (uh->check)
1400 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1401 		else
1402 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1403 	}
1404 }
1405 #endif
1406 
1407 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1408 					   int payload_off, int tcp_ts,
1409 					   struct sk_buff *skb)
1410 {
1411 #ifdef CONFIG_INET
1412 	struct tcphdr *th;
1413 	int len, nw_off;
1414 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1415 	u32 hdr_info = tpa_info->hdr_info;
1416 	bool loopback = false;
1417 
1418 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1419 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1420 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1421 
1422 	/* If the packet is an internal loopback packet, the offsets will
1423 	 * have an extra 4 bytes.
1424 	 */
1425 	if (inner_mac_off == 4) {
1426 		loopback = true;
1427 	} else if (inner_mac_off > 4) {
1428 		__be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1429 					    ETH_HLEN - 2));
1430 
1431 		/* We only support inner iPv4/ipv6.  If we don't see the
1432 		 * correct protocol ID, it must be a loopback packet where
1433 		 * the offsets are off by 4.
1434 		 */
1435 		if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1436 			loopback = true;
1437 	}
1438 	if (loopback) {
1439 		/* internal loopback packet, subtract all offsets by 4 */
1440 		inner_ip_off -= 4;
1441 		inner_mac_off -= 4;
1442 		outer_ip_off -= 4;
1443 	}
1444 
1445 	nw_off = inner_ip_off - ETH_HLEN;
1446 	skb_set_network_header(skb, nw_off);
1447 	if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1448 		struct ipv6hdr *iph = ipv6_hdr(skb);
1449 
1450 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1451 		len = skb->len - skb_transport_offset(skb);
1452 		th = tcp_hdr(skb);
1453 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1454 	} else {
1455 		struct iphdr *iph = ip_hdr(skb);
1456 
1457 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1458 		len = skb->len - skb_transport_offset(skb);
1459 		th = tcp_hdr(skb);
1460 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1461 	}
1462 
1463 	if (inner_mac_off) { /* tunnel */
1464 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1465 					    ETH_HLEN - 2));
1466 
1467 		bnxt_gro_tunnel(skb, proto);
1468 	}
1469 #endif
1470 	return skb;
1471 }
1472 
1473 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1474 					   int payload_off, int tcp_ts,
1475 					   struct sk_buff *skb)
1476 {
1477 #ifdef CONFIG_INET
1478 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1479 	u32 hdr_info = tpa_info->hdr_info;
1480 	int iphdr_len, nw_off;
1481 
1482 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1483 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1484 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1485 
1486 	nw_off = inner_ip_off - ETH_HLEN;
1487 	skb_set_network_header(skb, nw_off);
1488 	iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1489 		     sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1490 	skb_set_transport_header(skb, nw_off + iphdr_len);
1491 
1492 	if (inner_mac_off) { /* tunnel */
1493 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1494 					    ETH_HLEN - 2));
1495 
1496 		bnxt_gro_tunnel(skb, proto);
1497 	}
1498 #endif
1499 	return skb;
1500 }
1501 
1502 #define BNXT_IPV4_HDR_SIZE	(sizeof(struct iphdr) + sizeof(struct tcphdr))
1503 #define BNXT_IPV6_HDR_SIZE	(sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1504 
1505 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1506 					   int payload_off, int tcp_ts,
1507 					   struct sk_buff *skb)
1508 {
1509 #ifdef CONFIG_INET
1510 	struct tcphdr *th;
1511 	int len, nw_off, tcp_opt_len = 0;
1512 
1513 	if (tcp_ts)
1514 		tcp_opt_len = 12;
1515 
1516 	if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1517 		struct iphdr *iph;
1518 
1519 		nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1520 			 ETH_HLEN;
1521 		skb_set_network_header(skb, nw_off);
1522 		iph = ip_hdr(skb);
1523 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1524 		len = skb->len - skb_transport_offset(skb);
1525 		th = tcp_hdr(skb);
1526 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1527 	} else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1528 		struct ipv6hdr *iph;
1529 
1530 		nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1531 			 ETH_HLEN;
1532 		skb_set_network_header(skb, nw_off);
1533 		iph = ipv6_hdr(skb);
1534 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1535 		len = skb->len - skb_transport_offset(skb);
1536 		th = tcp_hdr(skb);
1537 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1538 	} else {
1539 		dev_kfree_skb_any(skb);
1540 		return NULL;
1541 	}
1542 
1543 	if (nw_off) /* tunnel */
1544 		bnxt_gro_tunnel(skb, skb->protocol);
1545 #endif
1546 	return skb;
1547 }
1548 
1549 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1550 					   struct bnxt_tpa_info *tpa_info,
1551 					   struct rx_tpa_end_cmp *tpa_end,
1552 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1553 					   struct sk_buff *skb)
1554 {
1555 #ifdef CONFIG_INET
1556 	int payload_off;
1557 	u16 segs;
1558 
1559 	segs = TPA_END_TPA_SEGS(tpa_end);
1560 	if (segs == 1)
1561 		return skb;
1562 
1563 	NAPI_GRO_CB(skb)->count = segs;
1564 	skb_shinfo(skb)->gso_size =
1565 		le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1566 	skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1567 	if (bp->flags & BNXT_FLAG_CHIP_P5)
1568 		payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1569 	else
1570 		payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1571 	skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1572 	if (likely(skb))
1573 		tcp_gro_complete(skb);
1574 #endif
1575 	return skb;
1576 }
1577 
1578 /* Given the cfa_code of a received packet determine which
1579  * netdev (vf-rep or PF) the packet is destined to.
1580  */
1581 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1582 {
1583 	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1584 
1585 	/* if vf-rep dev is NULL, the must belongs to the PF */
1586 	return dev ? dev : bp->dev;
1587 }
1588 
1589 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1590 					   struct bnxt_cp_ring_info *cpr,
1591 					   u32 *raw_cons,
1592 					   struct rx_tpa_end_cmp *tpa_end,
1593 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1594 					   u8 *event)
1595 {
1596 	struct bnxt_napi *bnapi = cpr->bnapi;
1597 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1598 	u8 *data_ptr, agg_bufs;
1599 	unsigned int len;
1600 	struct bnxt_tpa_info *tpa_info;
1601 	dma_addr_t mapping;
1602 	struct sk_buff *skb;
1603 	u16 idx = 0, agg_id;
1604 	void *data;
1605 	bool gro;
1606 
1607 	if (unlikely(bnapi->in_reset)) {
1608 		int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1609 
1610 		if (rc < 0)
1611 			return ERR_PTR(-EBUSY);
1612 		return NULL;
1613 	}
1614 
1615 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1616 		agg_id = TPA_END_AGG_ID_P5(tpa_end);
1617 		agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1618 		agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1619 		tpa_info = &rxr->rx_tpa[agg_id];
1620 		if (unlikely(agg_bufs != tpa_info->agg_count)) {
1621 			netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1622 				    agg_bufs, tpa_info->agg_count);
1623 			agg_bufs = tpa_info->agg_count;
1624 		}
1625 		tpa_info->agg_count = 0;
1626 		*event |= BNXT_AGG_EVENT;
1627 		bnxt_free_agg_idx(rxr, agg_id);
1628 		idx = agg_id;
1629 		gro = !!(bp->flags & BNXT_FLAG_GRO);
1630 	} else {
1631 		agg_id = TPA_END_AGG_ID(tpa_end);
1632 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1633 		tpa_info = &rxr->rx_tpa[agg_id];
1634 		idx = RING_CMP(*raw_cons);
1635 		if (agg_bufs) {
1636 			if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1637 				return ERR_PTR(-EBUSY);
1638 
1639 			*event |= BNXT_AGG_EVENT;
1640 			idx = NEXT_CMP(idx);
1641 		}
1642 		gro = !!TPA_END_GRO(tpa_end);
1643 	}
1644 	data = tpa_info->data;
1645 	data_ptr = tpa_info->data_ptr;
1646 	prefetch(data_ptr);
1647 	len = tpa_info->len;
1648 	mapping = tpa_info->mapping;
1649 
1650 	if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1651 		bnxt_abort_tpa(cpr, idx, agg_bufs);
1652 		if (agg_bufs > MAX_SKB_FRAGS)
1653 			netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1654 				    agg_bufs, (int)MAX_SKB_FRAGS);
1655 		return NULL;
1656 	}
1657 
1658 	if (len <= bp->rx_copy_thresh) {
1659 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1660 		if (!skb) {
1661 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1662 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1663 			return NULL;
1664 		}
1665 	} else {
1666 		u8 *new_data;
1667 		dma_addr_t new_mapping;
1668 
1669 		new_data = __bnxt_alloc_rx_frag(bp, &new_mapping, GFP_ATOMIC);
1670 		if (!new_data) {
1671 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1672 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1673 			return NULL;
1674 		}
1675 
1676 		tpa_info->data = new_data;
1677 		tpa_info->data_ptr = new_data + bp->rx_offset;
1678 		tpa_info->mapping = new_mapping;
1679 
1680 		skb = napi_build_skb(data, bp->rx_buf_size);
1681 		dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1682 				       bp->rx_buf_use_size, bp->rx_dir,
1683 				       DMA_ATTR_WEAK_ORDERING);
1684 
1685 		if (!skb) {
1686 			skb_free_frag(data);
1687 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1688 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1689 			return NULL;
1690 		}
1691 		skb_reserve(skb, bp->rx_offset);
1692 		skb_put(skb, len);
1693 	}
1694 
1695 	if (agg_bufs) {
1696 		skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, idx, agg_bufs, true);
1697 		if (!skb) {
1698 			/* Page reuse already handled by bnxt_rx_pages(). */
1699 			cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
1700 			return NULL;
1701 		}
1702 	}
1703 
1704 	skb->protocol =
1705 		eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1706 
1707 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1708 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1709 
1710 	if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1711 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1712 		__be16 vlan_proto = htons(tpa_info->metadata >>
1713 					  RX_CMP_FLAGS2_METADATA_TPID_SFT);
1714 		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1715 
1716 		if (eth_type_vlan(vlan_proto)) {
1717 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1718 		} else {
1719 			dev_kfree_skb(skb);
1720 			return NULL;
1721 		}
1722 	}
1723 
1724 	skb_checksum_none_assert(skb);
1725 	if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1726 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1727 		skb->csum_level =
1728 			(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1729 	}
1730 
1731 	if (gro)
1732 		skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1733 
1734 	return skb;
1735 }
1736 
1737 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1738 			 struct rx_agg_cmp *rx_agg)
1739 {
1740 	u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1741 	struct bnxt_tpa_info *tpa_info;
1742 
1743 	agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1744 	tpa_info = &rxr->rx_tpa[agg_id];
1745 	BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1746 	tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1747 }
1748 
1749 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1750 			     struct sk_buff *skb)
1751 {
1752 	skb_mark_for_recycle(skb);
1753 
1754 	if (skb->dev != bp->dev) {
1755 		/* this packet belongs to a vf-rep */
1756 		bnxt_vf_rep_rx(bp, skb);
1757 		return;
1758 	}
1759 	skb_record_rx_queue(skb, bnapi->index);
1760 	napi_gro_receive(&bnapi->napi, skb);
1761 }
1762 
1763 static bool bnxt_rx_ts_valid(struct bnxt *bp, u32 flags,
1764 			     struct rx_cmp_ext *rxcmp1, u32 *cmpl_ts)
1765 {
1766 	u32 ts = le32_to_cpu(rxcmp1->rx_cmp_timestamp);
1767 
1768 	if (BNXT_PTP_RX_TS_VALID(flags))
1769 		goto ts_valid;
1770 	if (!bp->ptp_all_rx_tstamp || !ts || !BNXT_ALL_RX_TS_VALID(flags))
1771 		return false;
1772 
1773 ts_valid:
1774 	*cmpl_ts = ts;
1775 	return true;
1776 }
1777 
1778 /* returns the following:
1779  * 1       - 1 packet successfully received
1780  * 0       - successful TPA_START, packet not completed yet
1781  * -EBUSY  - completion ring does not have all the agg buffers yet
1782  * -ENOMEM - packet aborted due to out of memory
1783  * -EIO    - packet aborted due to hw error indicated in BD
1784  */
1785 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1786 		       u32 *raw_cons, u8 *event)
1787 {
1788 	struct bnxt_napi *bnapi = cpr->bnapi;
1789 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1790 	struct net_device *dev = bp->dev;
1791 	struct rx_cmp *rxcmp;
1792 	struct rx_cmp_ext *rxcmp1;
1793 	u32 tmp_raw_cons = *raw_cons;
1794 	u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1795 	struct bnxt_sw_rx_bd *rx_buf;
1796 	unsigned int len;
1797 	u8 *data_ptr, agg_bufs, cmp_type;
1798 	bool xdp_active = false;
1799 	dma_addr_t dma_addr;
1800 	struct sk_buff *skb;
1801 	struct xdp_buff xdp;
1802 	u32 flags, misc;
1803 	u32 cmpl_ts;
1804 	void *data;
1805 	int rc = 0;
1806 
1807 	rxcmp = (struct rx_cmp *)
1808 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1809 
1810 	cmp_type = RX_CMP_TYPE(rxcmp);
1811 
1812 	if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1813 		bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1814 		goto next_rx_no_prod_no_len;
1815 	}
1816 
1817 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1818 	cp_cons = RING_CMP(tmp_raw_cons);
1819 	rxcmp1 = (struct rx_cmp_ext *)
1820 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1821 
1822 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1823 		return -EBUSY;
1824 
1825 	/* The valid test of the entry must be done first before
1826 	 * reading any further.
1827 	 */
1828 	dma_rmb();
1829 	prod = rxr->rx_prod;
1830 
1831 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1832 		bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1833 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
1834 
1835 		*event |= BNXT_RX_EVENT;
1836 		goto next_rx_no_prod_no_len;
1837 
1838 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1839 		skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1840 				   (struct rx_tpa_end_cmp *)rxcmp,
1841 				   (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1842 
1843 		if (IS_ERR(skb))
1844 			return -EBUSY;
1845 
1846 		rc = -ENOMEM;
1847 		if (likely(skb)) {
1848 			bnxt_deliver_skb(bp, bnapi, skb);
1849 			rc = 1;
1850 		}
1851 		*event |= BNXT_RX_EVENT;
1852 		goto next_rx_no_prod_no_len;
1853 	}
1854 
1855 	cons = rxcmp->rx_cmp_opaque;
1856 	if (unlikely(cons != rxr->rx_next_cons)) {
1857 		int rc1 = bnxt_discard_rx(bp, cpr, &tmp_raw_cons, rxcmp);
1858 
1859 		/* 0xffff is forced error, don't print it */
1860 		if (rxr->rx_next_cons != 0xffff)
1861 			netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1862 				    cons, rxr->rx_next_cons);
1863 		bnxt_sched_reset_rxr(bp, rxr);
1864 		if (rc1)
1865 			return rc1;
1866 		goto next_rx_no_prod_no_len;
1867 	}
1868 	rx_buf = &rxr->rx_buf_ring[cons];
1869 	data = rx_buf->data;
1870 	data_ptr = rx_buf->data_ptr;
1871 	prefetch(data_ptr);
1872 
1873 	misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1874 	agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1875 
1876 	if (agg_bufs) {
1877 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1878 			return -EBUSY;
1879 
1880 		cp_cons = NEXT_CMP(cp_cons);
1881 		*event |= BNXT_AGG_EVENT;
1882 	}
1883 	*event |= BNXT_RX_EVENT;
1884 
1885 	rx_buf->data = NULL;
1886 	if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1887 		u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1888 
1889 		bnxt_reuse_rx_data(rxr, cons, data);
1890 		if (agg_bufs)
1891 			bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1892 					       false);
1893 
1894 		rc = -EIO;
1895 		if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1896 			bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1897 			if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1898 			    !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1899 				netdev_warn_once(bp->dev, "RX buffer error %x\n",
1900 						 rx_err);
1901 				bnxt_sched_reset_rxr(bp, rxr);
1902 			}
1903 		}
1904 		goto next_rx_no_len;
1905 	}
1906 
1907 	flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
1908 	len = flags >> RX_CMP_LEN_SHIFT;
1909 	dma_addr = rx_buf->mapping;
1910 
1911 	if (bnxt_xdp_attached(bp, rxr)) {
1912 		bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp);
1913 		if (agg_bufs) {
1914 			u32 frag_len = bnxt_rx_agg_pages_xdp(bp, cpr, &xdp,
1915 							     cp_cons, agg_bufs,
1916 							     false);
1917 			if (!frag_len)
1918 				goto oom_next_rx;
1919 		}
1920 		xdp_active = true;
1921 	}
1922 
1923 	if (xdp_active) {
1924 		if (bnxt_rx_xdp(bp, rxr, cons, xdp, data, &data_ptr, &len, event)) {
1925 			rc = 1;
1926 			goto next_rx;
1927 		}
1928 	}
1929 
1930 	if (len <= bp->rx_copy_thresh) {
1931 		skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1932 		bnxt_reuse_rx_data(rxr, cons, data);
1933 		if (!skb) {
1934 			if (agg_bufs) {
1935 				if (!xdp_active)
1936 					bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1937 							       agg_bufs, false);
1938 				else
1939 					bnxt_xdp_buff_frags_free(rxr, &xdp);
1940 			}
1941 			goto oom_next_rx;
1942 		}
1943 	} else {
1944 		u32 payload;
1945 
1946 		if (rx_buf->data_ptr == data_ptr)
1947 			payload = misc & RX_CMP_PAYLOAD_OFFSET;
1948 		else
1949 			payload = 0;
1950 		skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1951 				      payload | len);
1952 		if (!skb)
1953 			goto oom_next_rx;
1954 	}
1955 
1956 	if (agg_bufs) {
1957 		if (!xdp_active) {
1958 			skb = bnxt_rx_agg_pages_skb(bp, cpr, skb, cp_cons, agg_bufs, false);
1959 			if (!skb)
1960 				goto oom_next_rx;
1961 		} else {
1962 			skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr->page_pool, &xdp, rxcmp1);
1963 			if (!skb) {
1964 				/* we should be able to free the old skb here */
1965 				bnxt_xdp_buff_frags_free(rxr, &xdp);
1966 				goto oom_next_rx;
1967 			}
1968 		}
1969 	}
1970 
1971 	if (RX_CMP_HASH_VALID(rxcmp)) {
1972 		u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1973 		enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1974 
1975 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1976 		if (hash_type != 1 && hash_type != 3)
1977 			type = PKT_HASH_TYPE_L3;
1978 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1979 	}
1980 
1981 	cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1982 	skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1983 
1984 	if ((rxcmp1->rx_cmp_flags2 &
1985 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1986 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1987 		u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1988 		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1989 		__be16 vlan_proto = htons(meta_data >>
1990 					  RX_CMP_FLAGS2_METADATA_TPID_SFT);
1991 
1992 		if (eth_type_vlan(vlan_proto)) {
1993 			__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
1994 		} else {
1995 			dev_kfree_skb(skb);
1996 			goto next_rx;
1997 		}
1998 	}
1999 
2000 	skb_checksum_none_assert(skb);
2001 	if (RX_CMP_L4_CS_OK(rxcmp1)) {
2002 		if (dev->features & NETIF_F_RXCSUM) {
2003 			skb->ip_summed = CHECKSUM_UNNECESSARY;
2004 			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
2005 		}
2006 	} else {
2007 		if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
2008 			if (dev->features & NETIF_F_RXCSUM)
2009 				bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
2010 		}
2011 	}
2012 
2013 	if (bnxt_rx_ts_valid(bp, flags, rxcmp1, &cmpl_ts)) {
2014 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
2015 			u64 ns, ts;
2016 
2017 			if (!bnxt_get_rx_ts_p5(bp, &ts, cmpl_ts)) {
2018 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2019 
2020 				spin_lock_bh(&ptp->ptp_lock);
2021 				ns = timecounter_cyc2time(&ptp->tc, ts);
2022 				spin_unlock_bh(&ptp->ptp_lock);
2023 				memset(skb_hwtstamps(skb), 0,
2024 				       sizeof(*skb_hwtstamps(skb)));
2025 				skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ns);
2026 			}
2027 		}
2028 	}
2029 	bnxt_deliver_skb(bp, bnapi, skb);
2030 	rc = 1;
2031 
2032 next_rx:
2033 	cpr->rx_packets += 1;
2034 	cpr->rx_bytes += len;
2035 
2036 next_rx_no_len:
2037 	rxr->rx_prod = NEXT_RX(prod);
2038 	rxr->rx_next_cons = NEXT_RX(cons);
2039 
2040 next_rx_no_prod_no_len:
2041 	*raw_cons = tmp_raw_cons;
2042 
2043 	return rc;
2044 
2045 oom_next_rx:
2046 	cpr->bnapi->cp_ring.sw_stats.rx.rx_oom_discards += 1;
2047 	rc = -ENOMEM;
2048 	goto next_rx;
2049 }
2050 
2051 /* In netpoll mode, if we are using a combined completion ring, we need to
2052  * discard the rx packets and recycle the buffers.
2053  */
2054 static int bnxt_force_rx_discard(struct bnxt *bp,
2055 				 struct bnxt_cp_ring_info *cpr,
2056 				 u32 *raw_cons, u8 *event)
2057 {
2058 	u32 tmp_raw_cons = *raw_cons;
2059 	struct rx_cmp_ext *rxcmp1;
2060 	struct rx_cmp *rxcmp;
2061 	u16 cp_cons;
2062 	u8 cmp_type;
2063 	int rc;
2064 
2065 	cp_cons = RING_CMP(tmp_raw_cons);
2066 	rxcmp = (struct rx_cmp *)
2067 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2068 
2069 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
2070 	cp_cons = RING_CMP(tmp_raw_cons);
2071 	rxcmp1 = (struct rx_cmp_ext *)
2072 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2073 
2074 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2075 		return -EBUSY;
2076 
2077 	/* The valid test of the entry must be done first before
2078 	 * reading any further.
2079 	 */
2080 	dma_rmb();
2081 	cmp_type = RX_CMP_TYPE(rxcmp);
2082 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
2083 		rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2084 			cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2085 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
2086 		struct rx_tpa_end_cmp_ext *tpa_end1;
2087 
2088 		tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
2089 		tpa_end1->rx_tpa_end_cmp_errors_v2 |=
2090 			cpu_to_le32(RX_TPA_END_CMP_ERRORS);
2091 	}
2092 	rc = bnxt_rx_pkt(bp, cpr, raw_cons, event);
2093 	if (rc && rc != -EBUSY)
2094 		cpr->bnapi->cp_ring.sw_stats.rx.rx_netpoll_discards += 1;
2095 	return rc;
2096 }
2097 
2098 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
2099 {
2100 	struct bnxt_fw_health *fw_health = bp->fw_health;
2101 	u32 reg = fw_health->regs[reg_idx];
2102 	u32 reg_type, reg_off, val = 0;
2103 
2104 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
2105 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
2106 	switch (reg_type) {
2107 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
2108 		pci_read_config_dword(bp->pdev, reg_off, &val);
2109 		break;
2110 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
2111 		reg_off = fw_health->mapped_regs[reg_idx];
2112 		fallthrough;
2113 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
2114 		val = readl(bp->bar0 + reg_off);
2115 		break;
2116 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
2117 		val = readl(bp->bar1 + reg_off);
2118 		break;
2119 	}
2120 	if (reg_idx == BNXT_FW_RESET_INPROG_REG)
2121 		val &= fw_health->fw_reset_inprog_reg_mask;
2122 	return val;
2123 }
2124 
2125 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
2126 {
2127 	int i;
2128 
2129 	for (i = 0; i < bp->rx_nr_rings; i++) {
2130 		u16 grp_idx = bp->rx_ring[i].bnapi->index;
2131 		struct bnxt_ring_grp_info *grp_info;
2132 
2133 		grp_info = &bp->grp_info[grp_idx];
2134 		if (grp_info->agg_fw_ring_id == ring_id)
2135 			return grp_idx;
2136 	}
2137 	return INVALID_HW_RING_ID;
2138 }
2139 
2140 static void bnxt_event_error_report(struct bnxt *bp, u32 data1, u32 data2)
2141 {
2142 	u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1);
2143 
2144 	switch (err_type) {
2145 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL:
2146 		netdev_err(bp->dev, "1PPS: Received invalid signal on pin%lu from the external source. Please fix the signal and reconfigure the pin\n",
2147 			   BNXT_EVENT_INVALID_SIGNAL_DATA(data2));
2148 		break;
2149 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM:
2150 		netdev_warn(bp->dev, "Pause Storm detected!\n");
2151 		break;
2152 	case ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD:
2153 		netdev_warn(bp->dev, "One or more MMIO doorbells dropped by the device!\n");
2154 		break;
2155 	default:
2156 		netdev_err(bp->dev, "FW reported unknown error type %u\n",
2157 			   err_type);
2158 		break;
2159 	}
2160 }
2161 
2162 #define BNXT_GET_EVENT_PORT(data)	\
2163 	((data) &			\
2164 	 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
2165 
2166 #define BNXT_EVENT_RING_TYPE(data2)	\
2167 	((data2) &			\
2168 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
2169 
2170 #define BNXT_EVENT_RING_TYPE_RX(data2)	\
2171 	(BNXT_EVENT_RING_TYPE(data2) ==	\
2172 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
2173 
2174 #define BNXT_EVENT_PHC_EVENT_TYPE(data1)	\
2175 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_MASK) >>\
2176 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_SFT)
2177 
2178 #define BNXT_EVENT_PHC_RTC_UPDATE(data1)	\
2179 	(((data1) & ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_MASK) >>\
2180 	 ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_PHC_TIME_MSB_SFT)
2181 
2182 #define BNXT_PHC_BITS	48
2183 
2184 static int bnxt_async_event_process(struct bnxt *bp,
2185 				    struct hwrm_async_event_cmpl *cmpl)
2186 {
2187 	u16 event_id = le16_to_cpu(cmpl->event_id);
2188 	u32 data1 = le32_to_cpu(cmpl->event_data1);
2189 	u32 data2 = le32_to_cpu(cmpl->event_data2);
2190 
2191 	netdev_dbg(bp->dev, "hwrm event 0x%x {0x%x, 0x%x}\n",
2192 		   event_id, data1, data2);
2193 
2194 	/* TODO CHIMP_FW: Define event id's for link change, error etc */
2195 	switch (event_id) {
2196 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
2197 		struct bnxt_link_info *link_info = &bp->link_info;
2198 
2199 		if (BNXT_VF(bp))
2200 			goto async_event_process_exit;
2201 
2202 		/* print unsupported speed warning in forced speed mode only */
2203 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
2204 		    (data1 & 0x20000)) {
2205 			u16 fw_speed = link_info->force_link_speed;
2206 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
2207 
2208 			if (speed != SPEED_UNKNOWN)
2209 				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
2210 					    speed);
2211 		}
2212 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
2213 	}
2214 		fallthrough;
2215 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
2216 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
2217 		set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2218 		fallthrough;
2219 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2220 		set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2221 		break;
2222 	case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2223 		set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2224 		break;
2225 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2226 		u16 port_id = BNXT_GET_EVENT_PORT(data1);
2227 
2228 		if (BNXT_VF(bp))
2229 			break;
2230 
2231 		if (bp->pf.port_id != port_id)
2232 			break;
2233 
2234 		set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2235 		break;
2236 	}
2237 	case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2238 		if (BNXT_PF(bp))
2239 			goto async_event_process_exit;
2240 		set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2241 		break;
2242 	case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2243 		char *type_str = "Solicited";
2244 
2245 		if (!bp->fw_health)
2246 			goto async_event_process_exit;
2247 
2248 		bp->fw_reset_timestamp = jiffies;
2249 		bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2250 		if (!bp->fw_reset_min_dsecs)
2251 			bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2252 		bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2253 		if (!bp->fw_reset_max_dsecs)
2254 			bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2255 		if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) {
2256 			set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state);
2257 		} else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2258 			type_str = "Fatal";
2259 			bp->fw_health->fatalities++;
2260 			set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2261 		} else if (data2 && BNXT_FW_STATUS_HEALTHY !=
2262 			   EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) {
2263 			type_str = "Non-fatal";
2264 			bp->fw_health->survivals++;
2265 			set_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
2266 		}
2267 		netif_warn(bp, hw, bp->dev,
2268 			   "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n",
2269 			   type_str, data1, data2,
2270 			   bp->fw_reset_min_dsecs * 100,
2271 			   bp->fw_reset_max_dsecs * 100);
2272 		set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2273 		break;
2274 	}
2275 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2276 		struct bnxt_fw_health *fw_health = bp->fw_health;
2277 		char *status_desc = "healthy";
2278 		u32 status;
2279 
2280 		if (!fw_health)
2281 			goto async_event_process_exit;
2282 
2283 		if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) {
2284 			fw_health->enabled = false;
2285 			netif_info(bp, drv, bp->dev, "Driver recovery watchdog is disabled\n");
2286 			break;
2287 		}
2288 		fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2289 		fw_health->tmr_multiplier =
2290 			DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2291 				     bp->current_interval * 10);
2292 		fw_health->tmr_counter = fw_health->tmr_multiplier;
2293 		if (!fw_health->enabled)
2294 			fw_health->last_fw_heartbeat =
2295 				bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2296 		fw_health->last_fw_reset_cnt =
2297 			bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2298 		status = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
2299 		if (status != BNXT_FW_STATUS_HEALTHY)
2300 			status_desc = "unhealthy";
2301 		netif_info(bp, drv, bp->dev,
2302 			   "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n",
2303 			   fw_health->primary ? "primary" : "backup", status,
2304 			   status_desc, fw_health->last_fw_reset_cnt);
2305 		if (!fw_health->enabled) {
2306 			/* Make sure tmr_counter is set and visible to
2307 			 * bnxt_health_check() before setting enabled to true.
2308 			 */
2309 			smp_wmb();
2310 			fw_health->enabled = true;
2311 		}
2312 		goto async_event_process_exit;
2313 	}
2314 	case ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION:
2315 		netif_notice(bp, hw, bp->dev,
2316 			     "Received firmware debug notification, data1: 0x%x, data2: 0x%x\n",
2317 			     data1, data2);
2318 		goto async_event_process_exit;
2319 	case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2320 		struct bnxt_rx_ring_info *rxr;
2321 		u16 grp_idx;
2322 
2323 		if (bp->flags & BNXT_FLAG_CHIP_P5)
2324 			goto async_event_process_exit;
2325 
2326 		netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2327 			    BNXT_EVENT_RING_TYPE(data2), data1);
2328 		if (!BNXT_EVENT_RING_TYPE_RX(data2))
2329 			goto async_event_process_exit;
2330 
2331 		grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2332 		if (grp_idx == INVALID_HW_RING_ID) {
2333 			netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2334 				    data1);
2335 			goto async_event_process_exit;
2336 		}
2337 		rxr = bp->bnapi[grp_idx]->rx_ring;
2338 		bnxt_sched_reset_rxr(bp, rxr);
2339 		goto async_event_process_exit;
2340 	}
2341 	case ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST: {
2342 		struct bnxt_fw_health *fw_health = bp->fw_health;
2343 
2344 		netif_notice(bp, hw, bp->dev,
2345 			     "Received firmware echo request, data1: 0x%x, data2: 0x%x\n",
2346 			     data1, data2);
2347 		if (fw_health) {
2348 			fw_health->echo_req_data1 = data1;
2349 			fw_health->echo_req_data2 = data2;
2350 			set_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event);
2351 			break;
2352 		}
2353 		goto async_event_process_exit;
2354 	}
2355 	case ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP: {
2356 		bnxt_ptp_pps_event(bp, data1, data2);
2357 		goto async_event_process_exit;
2358 	}
2359 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: {
2360 		bnxt_event_error_report(bp, data1, data2);
2361 		goto async_event_process_exit;
2362 	}
2363 	case ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE: {
2364 		switch (BNXT_EVENT_PHC_EVENT_TYPE(data1)) {
2365 		case ASYNC_EVENT_CMPL_PHC_UPDATE_EVENT_DATA1_FLAGS_PHC_RTC_UPDATE:
2366 			if (BNXT_PTP_USE_RTC(bp)) {
2367 				struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
2368 				u64 ns;
2369 
2370 				if (!ptp)
2371 					goto async_event_process_exit;
2372 
2373 				spin_lock_bh(&ptp->ptp_lock);
2374 				bnxt_ptp_update_current_time(bp);
2375 				ns = (((u64)BNXT_EVENT_PHC_RTC_UPDATE(data1) <<
2376 				       BNXT_PHC_BITS) | ptp->current_time);
2377 				bnxt_ptp_rtc_timecounter_init(ptp, ns);
2378 				spin_unlock_bh(&ptp->ptp_lock);
2379 			}
2380 			break;
2381 		}
2382 		goto async_event_process_exit;
2383 	}
2384 	case ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE: {
2385 		u16 seq_id = le32_to_cpu(cmpl->event_data2) & 0xffff;
2386 
2387 		hwrm_update_token(bp, seq_id, BNXT_HWRM_DEFERRED);
2388 		goto async_event_process_exit;
2389 	}
2390 	default:
2391 		goto async_event_process_exit;
2392 	}
2393 	__bnxt_queue_sp_work(bp);
2394 async_event_process_exit:
2395 	return 0;
2396 }
2397 
2398 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2399 {
2400 	u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2401 	struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2402 	struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2403 				(struct hwrm_fwd_req_cmpl *)txcmp;
2404 
2405 	switch (cmpl_type) {
2406 	case CMPL_BASE_TYPE_HWRM_DONE:
2407 		seq_id = le16_to_cpu(h_cmpl->sequence_id);
2408 		hwrm_update_token(bp, seq_id, BNXT_HWRM_COMPLETE);
2409 		break;
2410 
2411 	case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2412 		vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2413 
2414 		if ((vf_id < bp->pf.first_vf_id) ||
2415 		    (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2416 			netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2417 				   vf_id);
2418 			return -EINVAL;
2419 		}
2420 
2421 		set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2422 		bnxt_queue_sp_work(bp, BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT);
2423 		break;
2424 
2425 	case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2426 		bnxt_async_event_process(bp,
2427 					 (struct hwrm_async_event_cmpl *)txcmp);
2428 		break;
2429 
2430 	default:
2431 		break;
2432 	}
2433 
2434 	return 0;
2435 }
2436 
2437 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2438 {
2439 	struct bnxt_napi *bnapi = dev_instance;
2440 	struct bnxt *bp = bnapi->bp;
2441 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2442 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2443 
2444 	cpr->event_ctr++;
2445 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2446 	napi_schedule(&bnapi->napi);
2447 	return IRQ_HANDLED;
2448 }
2449 
2450 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2451 {
2452 	u32 raw_cons = cpr->cp_raw_cons;
2453 	u16 cons = RING_CMP(raw_cons);
2454 	struct tx_cmp *txcmp;
2455 
2456 	txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2457 
2458 	return TX_CMP_VALID(txcmp, raw_cons);
2459 }
2460 
2461 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2462 {
2463 	struct bnxt_napi *bnapi = dev_instance;
2464 	struct bnxt *bp = bnapi->bp;
2465 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2466 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2467 	u32 int_status;
2468 
2469 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2470 
2471 	if (!bnxt_has_work(bp, cpr)) {
2472 		int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2473 		/* return if erroneous interrupt */
2474 		if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2475 			return IRQ_NONE;
2476 	}
2477 
2478 	/* disable ring IRQ */
2479 	BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2480 
2481 	/* Return here if interrupt is shared and is disabled. */
2482 	if (unlikely(atomic_read(&bp->intr_sem) != 0))
2483 		return IRQ_HANDLED;
2484 
2485 	napi_schedule(&bnapi->napi);
2486 	return IRQ_HANDLED;
2487 }
2488 
2489 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2490 			    int budget)
2491 {
2492 	struct bnxt_napi *bnapi = cpr->bnapi;
2493 	u32 raw_cons = cpr->cp_raw_cons;
2494 	u32 cons;
2495 	int tx_pkts = 0;
2496 	int rx_pkts = 0;
2497 	u8 event = 0;
2498 	struct tx_cmp *txcmp;
2499 
2500 	cpr->has_more_work = 0;
2501 	cpr->had_work_done = 1;
2502 	while (1) {
2503 		int rc;
2504 
2505 		cons = RING_CMP(raw_cons);
2506 		txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2507 
2508 		if (!TX_CMP_VALID(txcmp, raw_cons))
2509 			break;
2510 
2511 		/* The valid test of the entry must be done first before
2512 		 * reading any further.
2513 		 */
2514 		dma_rmb();
2515 		if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2516 			tx_pkts++;
2517 			/* return full budget so NAPI will complete. */
2518 			if (unlikely(tx_pkts >= bp->tx_wake_thresh)) {
2519 				rx_pkts = budget;
2520 				raw_cons = NEXT_RAW_CMP(raw_cons);
2521 				if (budget)
2522 					cpr->has_more_work = 1;
2523 				break;
2524 			}
2525 		} else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2526 			if (likely(budget))
2527 				rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2528 			else
2529 				rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2530 							   &event);
2531 			if (likely(rc >= 0))
2532 				rx_pkts += rc;
2533 			/* Increment rx_pkts when rc is -ENOMEM to count towards
2534 			 * the NAPI budget.  Otherwise, we may potentially loop
2535 			 * here forever if we consistently cannot allocate
2536 			 * buffers.
2537 			 */
2538 			else if (rc == -ENOMEM && budget)
2539 				rx_pkts++;
2540 			else if (rc == -EBUSY)	/* partial completion */
2541 				break;
2542 		} else if (unlikely((TX_CMP_TYPE(txcmp) ==
2543 				     CMPL_BASE_TYPE_HWRM_DONE) ||
2544 				    (TX_CMP_TYPE(txcmp) ==
2545 				     CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2546 				    (TX_CMP_TYPE(txcmp) ==
2547 				     CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2548 			bnxt_hwrm_handler(bp, txcmp);
2549 		}
2550 		raw_cons = NEXT_RAW_CMP(raw_cons);
2551 
2552 		if (rx_pkts && rx_pkts == budget) {
2553 			cpr->has_more_work = 1;
2554 			break;
2555 		}
2556 	}
2557 
2558 	if (event & BNXT_REDIRECT_EVENT)
2559 		xdp_do_flush();
2560 
2561 	if (event & BNXT_TX_EVENT) {
2562 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2563 		u16 prod = txr->tx_prod;
2564 
2565 		/* Sync BD data before updating doorbell */
2566 		wmb();
2567 
2568 		bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2569 	}
2570 
2571 	cpr->cp_raw_cons = raw_cons;
2572 	bnapi->tx_pkts += tx_pkts;
2573 	bnapi->events |= event;
2574 	return rx_pkts;
2575 }
2576 
2577 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2578 				  int budget)
2579 {
2580 	if (bnapi->tx_pkts && !bnapi->tx_fault)
2581 		bnapi->tx_int(bp, bnapi, budget);
2582 
2583 	if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2584 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2585 
2586 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2587 	}
2588 	if (bnapi->events & BNXT_AGG_EVENT) {
2589 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2590 
2591 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2592 	}
2593 	bnapi->events = 0;
2594 }
2595 
2596 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2597 			  int budget)
2598 {
2599 	struct bnxt_napi *bnapi = cpr->bnapi;
2600 	int rx_pkts;
2601 
2602 	rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2603 
2604 	/* ACK completion ring before freeing tx ring and producing new
2605 	 * buffers in rx/agg rings to prevent overflowing the completion
2606 	 * ring.
2607 	 */
2608 	bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2609 
2610 	__bnxt_poll_work_done(bp, bnapi, budget);
2611 	return rx_pkts;
2612 }
2613 
2614 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2615 {
2616 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2617 	struct bnxt *bp = bnapi->bp;
2618 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2619 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2620 	struct tx_cmp *txcmp;
2621 	struct rx_cmp_ext *rxcmp1;
2622 	u32 cp_cons, tmp_raw_cons;
2623 	u32 raw_cons = cpr->cp_raw_cons;
2624 	bool flush_xdp = false;
2625 	u32 rx_pkts = 0;
2626 	u8 event = 0;
2627 
2628 	while (1) {
2629 		int rc;
2630 
2631 		cp_cons = RING_CMP(raw_cons);
2632 		txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2633 
2634 		if (!TX_CMP_VALID(txcmp, raw_cons))
2635 			break;
2636 
2637 		/* The valid test of the entry must be done first before
2638 		 * reading any further.
2639 		 */
2640 		dma_rmb();
2641 		if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2642 			tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2643 			cp_cons = RING_CMP(tmp_raw_cons);
2644 			rxcmp1 = (struct rx_cmp_ext *)
2645 			  &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2646 
2647 			if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2648 				break;
2649 
2650 			/* force an error to recycle the buffer */
2651 			rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2652 				cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2653 
2654 			rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2655 			if (likely(rc == -EIO) && budget)
2656 				rx_pkts++;
2657 			else if (rc == -EBUSY)	/* partial completion */
2658 				break;
2659 			if (event & BNXT_REDIRECT_EVENT)
2660 				flush_xdp = true;
2661 		} else if (unlikely(TX_CMP_TYPE(txcmp) ==
2662 				    CMPL_BASE_TYPE_HWRM_DONE)) {
2663 			bnxt_hwrm_handler(bp, txcmp);
2664 		} else {
2665 			netdev_err(bp->dev,
2666 				   "Invalid completion received on special ring\n");
2667 		}
2668 		raw_cons = NEXT_RAW_CMP(raw_cons);
2669 
2670 		if (rx_pkts == budget)
2671 			break;
2672 	}
2673 
2674 	cpr->cp_raw_cons = raw_cons;
2675 	BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2676 	bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2677 
2678 	if (event & BNXT_AGG_EVENT)
2679 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2680 	if (flush_xdp)
2681 		xdp_do_flush();
2682 
2683 	if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2684 		napi_complete_done(napi, rx_pkts);
2685 		BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2686 	}
2687 	return rx_pkts;
2688 }
2689 
2690 static int bnxt_poll(struct napi_struct *napi, int budget)
2691 {
2692 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2693 	struct bnxt *bp = bnapi->bp;
2694 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2695 	int work_done = 0;
2696 
2697 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2698 		napi_complete(napi);
2699 		return 0;
2700 	}
2701 	while (1) {
2702 		work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2703 
2704 		if (work_done >= budget) {
2705 			if (!budget)
2706 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2707 			break;
2708 		}
2709 
2710 		if (!bnxt_has_work(bp, cpr)) {
2711 			if (napi_complete_done(napi, work_done))
2712 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2713 			break;
2714 		}
2715 	}
2716 	if (bp->flags & BNXT_FLAG_DIM) {
2717 		struct dim_sample dim_sample = {};
2718 
2719 		dim_update_sample(cpr->event_ctr,
2720 				  cpr->rx_packets,
2721 				  cpr->rx_bytes,
2722 				  &dim_sample);
2723 		net_dim(&cpr->dim, dim_sample);
2724 	}
2725 	return work_done;
2726 }
2727 
2728 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2729 {
2730 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2731 	int i, work_done = 0;
2732 
2733 	for (i = 0; i < 2; i++) {
2734 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2735 
2736 		if (cpr2) {
2737 			work_done += __bnxt_poll_work(bp, cpr2,
2738 						      budget - work_done);
2739 			cpr->has_more_work |= cpr2->has_more_work;
2740 		}
2741 	}
2742 	return work_done;
2743 }
2744 
2745 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2746 				 u64 dbr_type, int budget)
2747 {
2748 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2749 	int i;
2750 
2751 	for (i = 0; i < 2; i++) {
2752 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2753 		struct bnxt_db_info *db;
2754 
2755 		if (cpr2 && cpr2->had_work_done) {
2756 			db = &cpr2->cp_db;
2757 			bnxt_writeq(bp, db->db_key64 | dbr_type |
2758 				    RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2759 			cpr2->had_work_done = 0;
2760 		}
2761 	}
2762 	__bnxt_poll_work_done(bp, bnapi, budget);
2763 }
2764 
2765 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2766 {
2767 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2768 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2769 	struct bnxt_cp_ring_info *cpr_rx;
2770 	u32 raw_cons = cpr->cp_raw_cons;
2771 	struct bnxt *bp = bnapi->bp;
2772 	struct nqe_cn *nqcmp;
2773 	int work_done = 0;
2774 	u32 cons;
2775 
2776 	if (unlikely(test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))) {
2777 		napi_complete(napi);
2778 		return 0;
2779 	}
2780 	if (cpr->has_more_work) {
2781 		cpr->has_more_work = 0;
2782 		work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2783 	}
2784 	while (1) {
2785 		cons = RING_CMP(raw_cons);
2786 		nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2787 
2788 		if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2789 			if (cpr->has_more_work)
2790 				break;
2791 
2792 			__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL,
2793 					     budget);
2794 			cpr->cp_raw_cons = raw_cons;
2795 			if (napi_complete_done(napi, work_done))
2796 				BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2797 						  cpr->cp_raw_cons);
2798 			goto poll_done;
2799 		}
2800 
2801 		/* The valid test of the entry must be done first before
2802 		 * reading any further.
2803 		 */
2804 		dma_rmb();
2805 
2806 		if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2807 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2808 			struct bnxt_cp_ring_info *cpr2;
2809 
2810 			/* No more budget for RX work */
2811 			if (budget && work_done >= budget && idx == BNXT_RX_HDL)
2812 				break;
2813 
2814 			cpr2 = cpr->cp_ring_arr[idx];
2815 			work_done += __bnxt_poll_work(bp, cpr2,
2816 						      budget - work_done);
2817 			cpr->has_more_work |= cpr2->has_more_work;
2818 		} else {
2819 			bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2820 		}
2821 		raw_cons = NEXT_RAW_CMP(raw_cons);
2822 	}
2823 	__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ, budget);
2824 	if (raw_cons != cpr->cp_raw_cons) {
2825 		cpr->cp_raw_cons = raw_cons;
2826 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2827 	}
2828 poll_done:
2829 	cpr_rx = cpr->cp_ring_arr[BNXT_RX_HDL];
2830 	if (cpr_rx && (bp->flags & BNXT_FLAG_DIM)) {
2831 		struct dim_sample dim_sample = {};
2832 
2833 		dim_update_sample(cpr->event_ctr,
2834 				  cpr_rx->rx_packets,
2835 				  cpr_rx->rx_bytes,
2836 				  &dim_sample);
2837 		net_dim(&cpr->dim, dim_sample);
2838 	}
2839 	return work_done;
2840 }
2841 
2842 static void bnxt_free_tx_skbs(struct bnxt *bp)
2843 {
2844 	int i, max_idx;
2845 	struct pci_dev *pdev = bp->pdev;
2846 
2847 	if (!bp->tx_ring)
2848 		return;
2849 
2850 	max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2851 	for (i = 0; i < bp->tx_nr_rings; i++) {
2852 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2853 		int j;
2854 
2855 		if (!txr->tx_buf_ring)
2856 			continue;
2857 
2858 		for (j = 0; j < max_idx;) {
2859 			struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2860 			struct sk_buff *skb;
2861 			int k, last;
2862 
2863 			if (i < bp->tx_nr_rings_xdp &&
2864 			    tx_buf->action == XDP_REDIRECT) {
2865 				dma_unmap_single(&pdev->dev,
2866 					dma_unmap_addr(tx_buf, mapping),
2867 					dma_unmap_len(tx_buf, len),
2868 					DMA_TO_DEVICE);
2869 				xdp_return_frame(tx_buf->xdpf);
2870 				tx_buf->action = 0;
2871 				tx_buf->xdpf = NULL;
2872 				j++;
2873 				continue;
2874 			}
2875 
2876 			skb = tx_buf->skb;
2877 			if (!skb) {
2878 				j++;
2879 				continue;
2880 			}
2881 
2882 			tx_buf->skb = NULL;
2883 
2884 			if (tx_buf->is_push) {
2885 				dev_kfree_skb(skb);
2886 				j += 2;
2887 				continue;
2888 			}
2889 
2890 			dma_unmap_single(&pdev->dev,
2891 					 dma_unmap_addr(tx_buf, mapping),
2892 					 skb_headlen(skb),
2893 					 DMA_TO_DEVICE);
2894 
2895 			last = tx_buf->nr_frags;
2896 			j += 2;
2897 			for (k = 0; k < last; k++, j++) {
2898 				int ring_idx = j & bp->tx_ring_mask;
2899 				skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2900 
2901 				tx_buf = &txr->tx_buf_ring[ring_idx];
2902 				dma_unmap_page(
2903 					&pdev->dev,
2904 					dma_unmap_addr(tx_buf, mapping),
2905 					skb_frag_size(frag), DMA_TO_DEVICE);
2906 			}
2907 			dev_kfree_skb(skb);
2908 		}
2909 		netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2910 	}
2911 }
2912 
2913 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2914 {
2915 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2916 	struct pci_dev *pdev = bp->pdev;
2917 	struct bnxt_tpa_idx_map *map;
2918 	int i, max_idx, max_agg_idx;
2919 
2920 	max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2921 	max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2922 	if (!rxr->rx_tpa)
2923 		goto skip_rx_tpa_free;
2924 
2925 	for (i = 0; i < bp->max_tpa; i++) {
2926 		struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2927 		u8 *data = tpa_info->data;
2928 
2929 		if (!data)
2930 			continue;
2931 
2932 		dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2933 				       bp->rx_buf_use_size, bp->rx_dir,
2934 				       DMA_ATTR_WEAK_ORDERING);
2935 
2936 		tpa_info->data = NULL;
2937 
2938 		skb_free_frag(data);
2939 	}
2940 
2941 skip_rx_tpa_free:
2942 	if (!rxr->rx_buf_ring)
2943 		goto skip_rx_buf_free;
2944 
2945 	for (i = 0; i < max_idx; i++) {
2946 		struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2947 		dma_addr_t mapping = rx_buf->mapping;
2948 		void *data = rx_buf->data;
2949 
2950 		if (!data)
2951 			continue;
2952 
2953 		rx_buf->data = NULL;
2954 		if (BNXT_RX_PAGE_MODE(bp)) {
2955 			page_pool_recycle_direct(rxr->page_pool, data);
2956 		} else {
2957 			dma_unmap_single_attrs(&pdev->dev, mapping,
2958 					       bp->rx_buf_use_size, bp->rx_dir,
2959 					       DMA_ATTR_WEAK_ORDERING);
2960 			skb_free_frag(data);
2961 		}
2962 	}
2963 
2964 skip_rx_buf_free:
2965 	if (!rxr->rx_agg_ring)
2966 		goto skip_rx_agg_free;
2967 
2968 	for (i = 0; i < max_agg_idx; i++) {
2969 		struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2970 		struct page *page = rx_agg_buf->page;
2971 
2972 		if (!page)
2973 			continue;
2974 
2975 		rx_agg_buf->page = NULL;
2976 		__clear_bit(i, rxr->rx_agg_bmap);
2977 
2978 		page_pool_recycle_direct(rxr->page_pool, page);
2979 	}
2980 
2981 skip_rx_agg_free:
2982 	map = rxr->rx_tpa_idx_map;
2983 	if (map)
2984 		memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2985 }
2986 
2987 static void bnxt_free_rx_skbs(struct bnxt *bp)
2988 {
2989 	int i;
2990 
2991 	if (!bp->rx_ring)
2992 		return;
2993 
2994 	for (i = 0; i < bp->rx_nr_rings; i++)
2995 		bnxt_free_one_rx_ring_skbs(bp, i);
2996 }
2997 
2998 static void bnxt_free_skbs(struct bnxt *bp)
2999 {
3000 	bnxt_free_tx_skbs(bp);
3001 	bnxt_free_rx_skbs(bp);
3002 }
3003 
3004 static void bnxt_init_ctx_mem(struct bnxt_mem_init *mem_init, void *p, int len)
3005 {
3006 	u8 init_val = mem_init->init_val;
3007 	u16 offset = mem_init->offset;
3008 	u8 *p2 = p;
3009 	int i;
3010 
3011 	if (!init_val)
3012 		return;
3013 	if (offset == BNXT_MEM_INVALID_OFFSET) {
3014 		memset(p, init_val, len);
3015 		return;
3016 	}
3017 	for (i = 0; i < len; i += mem_init->size)
3018 		*(p2 + i + offset) = init_val;
3019 }
3020 
3021 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3022 {
3023 	struct pci_dev *pdev = bp->pdev;
3024 	int i;
3025 
3026 	if (!rmem->pg_arr)
3027 		goto skip_pages;
3028 
3029 	for (i = 0; i < rmem->nr_pages; i++) {
3030 		if (!rmem->pg_arr[i])
3031 			continue;
3032 
3033 		dma_free_coherent(&pdev->dev, rmem->page_size,
3034 				  rmem->pg_arr[i], rmem->dma_arr[i]);
3035 
3036 		rmem->pg_arr[i] = NULL;
3037 	}
3038 skip_pages:
3039 	if (rmem->pg_tbl) {
3040 		size_t pg_tbl_size = rmem->nr_pages * 8;
3041 
3042 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3043 			pg_tbl_size = rmem->page_size;
3044 		dma_free_coherent(&pdev->dev, pg_tbl_size,
3045 				  rmem->pg_tbl, rmem->pg_tbl_map);
3046 		rmem->pg_tbl = NULL;
3047 	}
3048 	if (rmem->vmem_size && *rmem->vmem) {
3049 		vfree(*rmem->vmem);
3050 		*rmem->vmem = NULL;
3051 	}
3052 }
3053 
3054 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
3055 {
3056 	struct pci_dev *pdev = bp->pdev;
3057 	u64 valid_bit = 0;
3058 	int i;
3059 
3060 	if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
3061 		valid_bit = PTU_PTE_VALID;
3062 	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
3063 		size_t pg_tbl_size = rmem->nr_pages * 8;
3064 
3065 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
3066 			pg_tbl_size = rmem->page_size;
3067 		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
3068 						  &rmem->pg_tbl_map,
3069 						  GFP_KERNEL);
3070 		if (!rmem->pg_tbl)
3071 			return -ENOMEM;
3072 	}
3073 
3074 	for (i = 0; i < rmem->nr_pages; i++) {
3075 		u64 extra_bits = valid_bit;
3076 
3077 		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
3078 						     rmem->page_size,
3079 						     &rmem->dma_arr[i],
3080 						     GFP_KERNEL);
3081 		if (!rmem->pg_arr[i])
3082 			return -ENOMEM;
3083 
3084 		if (rmem->mem_init)
3085 			bnxt_init_ctx_mem(rmem->mem_init, rmem->pg_arr[i],
3086 					  rmem->page_size);
3087 		if (rmem->nr_pages > 1 || rmem->depth > 0) {
3088 			if (i == rmem->nr_pages - 2 &&
3089 			    (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3090 				extra_bits |= PTU_PTE_NEXT_TO_LAST;
3091 			else if (i == rmem->nr_pages - 1 &&
3092 				 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
3093 				extra_bits |= PTU_PTE_LAST;
3094 			rmem->pg_tbl[i] =
3095 				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
3096 		}
3097 	}
3098 
3099 	if (rmem->vmem_size) {
3100 		*rmem->vmem = vzalloc(rmem->vmem_size);
3101 		if (!(*rmem->vmem))
3102 			return -ENOMEM;
3103 	}
3104 	return 0;
3105 }
3106 
3107 static void bnxt_free_tpa_info(struct bnxt *bp)
3108 {
3109 	int i, j;
3110 
3111 	for (i = 0; i < bp->rx_nr_rings; i++) {
3112 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3113 
3114 		kfree(rxr->rx_tpa_idx_map);
3115 		rxr->rx_tpa_idx_map = NULL;
3116 		if (rxr->rx_tpa) {
3117 			for (j = 0; j < bp->max_tpa; j++) {
3118 				kfree(rxr->rx_tpa[j].agg_arr);
3119 				rxr->rx_tpa[j].agg_arr = NULL;
3120 			}
3121 		}
3122 		kfree(rxr->rx_tpa);
3123 		rxr->rx_tpa = NULL;
3124 	}
3125 }
3126 
3127 static int bnxt_alloc_tpa_info(struct bnxt *bp)
3128 {
3129 	int i, j;
3130 
3131 	bp->max_tpa = MAX_TPA;
3132 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
3133 		if (!bp->max_tpa_v2)
3134 			return 0;
3135 		bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
3136 	}
3137 
3138 	for (i = 0; i < bp->rx_nr_rings; i++) {
3139 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3140 		struct rx_agg_cmp *agg;
3141 
3142 		rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
3143 				      GFP_KERNEL);
3144 		if (!rxr->rx_tpa)
3145 			return -ENOMEM;
3146 
3147 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3148 			continue;
3149 		for (j = 0; j < bp->max_tpa; j++) {
3150 			agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
3151 			if (!agg)
3152 				return -ENOMEM;
3153 			rxr->rx_tpa[j].agg_arr = agg;
3154 		}
3155 		rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
3156 					      GFP_KERNEL);
3157 		if (!rxr->rx_tpa_idx_map)
3158 			return -ENOMEM;
3159 	}
3160 	return 0;
3161 }
3162 
3163 static void bnxt_free_rx_rings(struct bnxt *bp)
3164 {
3165 	int i;
3166 
3167 	if (!bp->rx_ring)
3168 		return;
3169 
3170 	bnxt_free_tpa_info(bp);
3171 	for (i = 0; i < bp->rx_nr_rings; i++) {
3172 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3173 		struct bnxt_ring_struct *ring;
3174 
3175 		if (rxr->xdp_prog)
3176 			bpf_prog_put(rxr->xdp_prog);
3177 
3178 		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
3179 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3180 
3181 		page_pool_destroy(rxr->page_pool);
3182 		rxr->page_pool = NULL;
3183 
3184 		kfree(rxr->rx_agg_bmap);
3185 		rxr->rx_agg_bmap = NULL;
3186 
3187 		ring = &rxr->rx_ring_struct;
3188 		bnxt_free_ring(bp, &ring->ring_mem);
3189 
3190 		ring = &rxr->rx_agg_ring_struct;
3191 		bnxt_free_ring(bp, &ring->ring_mem);
3192 	}
3193 }
3194 
3195 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
3196 				   struct bnxt_rx_ring_info *rxr)
3197 {
3198 	struct page_pool_params pp = { 0 };
3199 
3200 	pp.pool_size = bp->rx_agg_ring_size;
3201 	if (BNXT_RX_PAGE_MODE(bp))
3202 		pp.pool_size += bp->rx_ring_size;
3203 	pp.nid = dev_to_node(&bp->pdev->dev);
3204 	pp.napi = &rxr->bnapi->napi;
3205 	pp.dev = &bp->pdev->dev;
3206 	pp.dma_dir = bp->rx_dir;
3207 	pp.max_len = PAGE_SIZE;
3208 	pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
3209 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE)
3210 		pp.flags |= PP_FLAG_PAGE_FRAG;
3211 
3212 	rxr->page_pool = page_pool_create(&pp);
3213 	if (IS_ERR(rxr->page_pool)) {
3214 		int err = PTR_ERR(rxr->page_pool);
3215 
3216 		rxr->page_pool = NULL;
3217 		return err;
3218 	}
3219 	return 0;
3220 }
3221 
3222 static int bnxt_alloc_rx_rings(struct bnxt *bp)
3223 {
3224 	int i, rc = 0, agg_rings = 0;
3225 
3226 	if (!bp->rx_ring)
3227 		return -ENOMEM;
3228 
3229 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
3230 		agg_rings = 1;
3231 
3232 	for (i = 0; i < bp->rx_nr_rings; i++) {
3233 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
3234 		struct bnxt_ring_struct *ring;
3235 
3236 		ring = &rxr->rx_ring_struct;
3237 
3238 		rc = bnxt_alloc_rx_page_pool(bp, rxr);
3239 		if (rc)
3240 			return rc;
3241 
3242 		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0);
3243 		if (rc < 0)
3244 			return rc;
3245 
3246 		rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
3247 						MEM_TYPE_PAGE_POOL,
3248 						rxr->page_pool);
3249 		if (rc) {
3250 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
3251 			return rc;
3252 		}
3253 
3254 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3255 		if (rc)
3256 			return rc;
3257 
3258 		ring->grp_idx = i;
3259 		if (agg_rings) {
3260 			u16 mem_size;
3261 
3262 			ring = &rxr->rx_agg_ring_struct;
3263 			rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3264 			if (rc)
3265 				return rc;
3266 
3267 			ring->grp_idx = i;
3268 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
3269 			mem_size = rxr->rx_agg_bmap_size / 8;
3270 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
3271 			if (!rxr->rx_agg_bmap)
3272 				return -ENOMEM;
3273 		}
3274 	}
3275 	if (bp->flags & BNXT_FLAG_TPA)
3276 		rc = bnxt_alloc_tpa_info(bp);
3277 	return rc;
3278 }
3279 
3280 static void bnxt_free_tx_rings(struct bnxt *bp)
3281 {
3282 	int i;
3283 	struct pci_dev *pdev = bp->pdev;
3284 
3285 	if (!bp->tx_ring)
3286 		return;
3287 
3288 	for (i = 0; i < bp->tx_nr_rings; i++) {
3289 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3290 		struct bnxt_ring_struct *ring;
3291 
3292 		if (txr->tx_push) {
3293 			dma_free_coherent(&pdev->dev, bp->tx_push_size,
3294 					  txr->tx_push, txr->tx_push_mapping);
3295 			txr->tx_push = NULL;
3296 		}
3297 
3298 		ring = &txr->tx_ring_struct;
3299 
3300 		bnxt_free_ring(bp, &ring->ring_mem);
3301 	}
3302 }
3303 
3304 static int bnxt_alloc_tx_rings(struct bnxt *bp)
3305 {
3306 	int i, j, rc;
3307 	struct pci_dev *pdev = bp->pdev;
3308 
3309 	bp->tx_push_size = 0;
3310 	if (bp->tx_push_thresh) {
3311 		int push_size;
3312 
3313 		push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
3314 					bp->tx_push_thresh);
3315 
3316 		if (push_size > 256) {
3317 			push_size = 0;
3318 			bp->tx_push_thresh = 0;
3319 		}
3320 
3321 		bp->tx_push_size = push_size;
3322 	}
3323 
3324 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
3325 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3326 		struct bnxt_ring_struct *ring;
3327 		u8 qidx;
3328 
3329 		ring = &txr->tx_ring_struct;
3330 
3331 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3332 		if (rc)
3333 			return rc;
3334 
3335 		ring->grp_idx = txr->bnapi->index;
3336 		if (bp->tx_push_size) {
3337 			dma_addr_t mapping;
3338 
3339 			/* One pre-allocated DMA buffer to backup
3340 			 * TX push operation
3341 			 */
3342 			txr->tx_push = dma_alloc_coherent(&pdev->dev,
3343 						bp->tx_push_size,
3344 						&txr->tx_push_mapping,
3345 						GFP_KERNEL);
3346 
3347 			if (!txr->tx_push)
3348 				return -ENOMEM;
3349 
3350 			mapping = txr->tx_push_mapping +
3351 				sizeof(struct tx_push_bd);
3352 			txr->data_mapping = cpu_to_le64(mapping);
3353 		}
3354 		qidx = bp->tc_to_qidx[j];
3355 		ring->queue_id = bp->q_info[qidx].queue_id;
3356 		spin_lock_init(&txr->xdp_tx_lock);
3357 		if (i < bp->tx_nr_rings_xdp)
3358 			continue;
3359 		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3360 			j++;
3361 	}
3362 	return 0;
3363 }
3364 
3365 static void bnxt_free_cp_arrays(struct bnxt_cp_ring_info *cpr)
3366 {
3367 	struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3368 
3369 	kfree(cpr->cp_desc_ring);
3370 	cpr->cp_desc_ring = NULL;
3371 	ring->ring_mem.pg_arr = NULL;
3372 	kfree(cpr->cp_desc_mapping);
3373 	cpr->cp_desc_mapping = NULL;
3374 	ring->ring_mem.dma_arr = NULL;
3375 }
3376 
3377 static int bnxt_alloc_cp_arrays(struct bnxt_cp_ring_info *cpr, int n)
3378 {
3379 	cpr->cp_desc_ring = kcalloc(n, sizeof(*cpr->cp_desc_ring), GFP_KERNEL);
3380 	if (!cpr->cp_desc_ring)
3381 		return -ENOMEM;
3382 	cpr->cp_desc_mapping = kcalloc(n, sizeof(*cpr->cp_desc_mapping),
3383 				       GFP_KERNEL);
3384 	if (!cpr->cp_desc_mapping)
3385 		return -ENOMEM;
3386 	return 0;
3387 }
3388 
3389 static void bnxt_free_all_cp_arrays(struct bnxt *bp)
3390 {
3391 	int i;
3392 
3393 	if (!bp->bnapi)
3394 		return;
3395 	for (i = 0; i < bp->cp_nr_rings; i++) {
3396 		struct bnxt_napi *bnapi = bp->bnapi[i];
3397 
3398 		if (!bnapi)
3399 			continue;
3400 		bnxt_free_cp_arrays(&bnapi->cp_ring);
3401 	}
3402 }
3403 
3404 static int bnxt_alloc_all_cp_arrays(struct bnxt *bp)
3405 {
3406 	int i, n = bp->cp_nr_pages;
3407 
3408 	for (i = 0; i < bp->cp_nr_rings; i++) {
3409 		struct bnxt_napi *bnapi = bp->bnapi[i];
3410 		int rc;
3411 
3412 		if (!bnapi)
3413 			continue;
3414 		rc = bnxt_alloc_cp_arrays(&bnapi->cp_ring, n);
3415 		if (rc)
3416 			return rc;
3417 	}
3418 	return 0;
3419 }
3420 
3421 static void bnxt_free_cp_rings(struct bnxt *bp)
3422 {
3423 	int i;
3424 
3425 	if (!bp->bnapi)
3426 		return;
3427 
3428 	for (i = 0; i < bp->cp_nr_rings; i++) {
3429 		struct bnxt_napi *bnapi = bp->bnapi[i];
3430 		struct bnxt_cp_ring_info *cpr;
3431 		struct bnxt_ring_struct *ring;
3432 		int j;
3433 
3434 		if (!bnapi)
3435 			continue;
3436 
3437 		cpr = &bnapi->cp_ring;
3438 		ring = &cpr->cp_ring_struct;
3439 
3440 		bnxt_free_ring(bp, &ring->ring_mem);
3441 
3442 		for (j = 0; j < 2; j++) {
3443 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3444 
3445 			if (cpr2) {
3446 				ring = &cpr2->cp_ring_struct;
3447 				bnxt_free_ring(bp, &ring->ring_mem);
3448 				bnxt_free_cp_arrays(cpr2);
3449 				kfree(cpr2);
3450 				cpr->cp_ring_arr[j] = NULL;
3451 			}
3452 		}
3453 	}
3454 }
3455 
3456 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3457 {
3458 	struct bnxt_ring_mem_info *rmem;
3459 	struct bnxt_ring_struct *ring;
3460 	struct bnxt_cp_ring_info *cpr;
3461 	int rc;
3462 
3463 	cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3464 	if (!cpr)
3465 		return NULL;
3466 
3467 	rc = bnxt_alloc_cp_arrays(cpr, bp->cp_nr_pages);
3468 	if (rc) {
3469 		bnxt_free_cp_arrays(cpr);
3470 		kfree(cpr);
3471 		return NULL;
3472 	}
3473 	ring = &cpr->cp_ring_struct;
3474 	rmem = &ring->ring_mem;
3475 	rmem->nr_pages = bp->cp_nr_pages;
3476 	rmem->page_size = HW_CMPD_RING_SIZE;
3477 	rmem->pg_arr = (void **)cpr->cp_desc_ring;
3478 	rmem->dma_arr = cpr->cp_desc_mapping;
3479 	rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3480 	rc = bnxt_alloc_ring(bp, rmem);
3481 	if (rc) {
3482 		bnxt_free_ring(bp, rmem);
3483 		bnxt_free_cp_arrays(cpr);
3484 		kfree(cpr);
3485 		cpr = NULL;
3486 	}
3487 	return cpr;
3488 }
3489 
3490 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3491 {
3492 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3493 	int i, rc, ulp_base_vec, ulp_msix;
3494 
3495 	ulp_msix = bnxt_get_ulp_msix_num(bp);
3496 	ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3497 	for (i = 0; i < bp->cp_nr_rings; i++) {
3498 		struct bnxt_napi *bnapi = bp->bnapi[i];
3499 		struct bnxt_cp_ring_info *cpr;
3500 		struct bnxt_ring_struct *ring;
3501 
3502 		if (!bnapi)
3503 			continue;
3504 
3505 		cpr = &bnapi->cp_ring;
3506 		cpr->bnapi = bnapi;
3507 		ring = &cpr->cp_ring_struct;
3508 
3509 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3510 		if (rc)
3511 			return rc;
3512 
3513 		if (ulp_msix && i >= ulp_base_vec)
3514 			ring->map_idx = i + ulp_msix;
3515 		else
3516 			ring->map_idx = i;
3517 
3518 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3519 			continue;
3520 
3521 		if (i < bp->rx_nr_rings) {
3522 			struct bnxt_cp_ring_info *cpr2 =
3523 				bnxt_alloc_cp_sub_ring(bp);
3524 
3525 			cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3526 			if (!cpr2)
3527 				return -ENOMEM;
3528 			cpr2->bnapi = bnapi;
3529 		}
3530 		if ((sh && i < bp->tx_nr_rings) ||
3531 		    (!sh && i >= bp->rx_nr_rings)) {
3532 			struct bnxt_cp_ring_info *cpr2 =
3533 				bnxt_alloc_cp_sub_ring(bp);
3534 
3535 			cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3536 			if (!cpr2)
3537 				return -ENOMEM;
3538 			cpr2->bnapi = bnapi;
3539 		}
3540 	}
3541 	return 0;
3542 }
3543 
3544 static void bnxt_init_ring_struct(struct bnxt *bp)
3545 {
3546 	int i;
3547 
3548 	for (i = 0; i < bp->cp_nr_rings; i++) {
3549 		struct bnxt_napi *bnapi = bp->bnapi[i];
3550 		struct bnxt_ring_mem_info *rmem;
3551 		struct bnxt_cp_ring_info *cpr;
3552 		struct bnxt_rx_ring_info *rxr;
3553 		struct bnxt_tx_ring_info *txr;
3554 		struct bnxt_ring_struct *ring;
3555 
3556 		if (!bnapi)
3557 			continue;
3558 
3559 		cpr = &bnapi->cp_ring;
3560 		ring = &cpr->cp_ring_struct;
3561 		rmem = &ring->ring_mem;
3562 		rmem->nr_pages = bp->cp_nr_pages;
3563 		rmem->page_size = HW_CMPD_RING_SIZE;
3564 		rmem->pg_arr = (void **)cpr->cp_desc_ring;
3565 		rmem->dma_arr = cpr->cp_desc_mapping;
3566 		rmem->vmem_size = 0;
3567 
3568 		rxr = bnapi->rx_ring;
3569 		if (!rxr)
3570 			goto skip_rx;
3571 
3572 		ring = &rxr->rx_ring_struct;
3573 		rmem = &ring->ring_mem;
3574 		rmem->nr_pages = bp->rx_nr_pages;
3575 		rmem->page_size = HW_RXBD_RING_SIZE;
3576 		rmem->pg_arr = (void **)rxr->rx_desc_ring;
3577 		rmem->dma_arr = rxr->rx_desc_mapping;
3578 		rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3579 		rmem->vmem = (void **)&rxr->rx_buf_ring;
3580 
3581 		ring = &rxr->rx_agg_ring_struct;
3582 		rmem = &ring->ring_mem;
3583 		rmem->nr_pages = bp->rx_agg_nr_pages;
3584 		rmem->page_size = HW_RXBD_RING_SIZE;
3585 		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3586 		rmem->dma_arr = rxr->rx_agg_desc_mapping;
3587 		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3588 		rmem->vmem = (void **)&rxr->rx_agg_ring;
3589 
3590 skip_rx:
3591 		txr = bnapi->tx_ring;
3592 		if (!txr)
3593 			continue;
3594 
3595 		ring = &txr->tx_ring_struct;
3596 		rmem = &ring->ring_mem;
3597 		rmem->nr_pages = bp->tx_nr_pages;
3598 		rmem->page_size = HW_RXBD_RING_SIZE;
3599 		rmem->pg_arr = (void **)txr->tx_desc_ring;
3600 		rmem->dma_arr = txr->tx_desc_mapping;
3601 		rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3602 		rmem->vmem = (void **)&txr->tx_buf_ring;
3603 	}
3604 }
3605 
3606 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3607 {
3608 	int i;
3609 	u32 prod;
3610 	struct rx_bd **rx_buf_ring;
3611 
3612 	rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3613 	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3614 		int j;
3615 		struct rx_bd *rxbd;
3616 
3617 		rxbd = rx_buf_ring[i];
3618 		if (!rxbd)
3619 			continue;
3620 
3621 		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3622 			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3623 			rxbd->rx_bd_opaque = prod;
3624 		}
3625 	}
3626 }
3627 
3628 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3629 {
3630 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3631 	struct net_device *dev = bp->dev;
3632 	u32 prod;
3633 	int i;
3634 
3635 	prod = rxr->rx_prod;
3636 	for (i = 0; i < bp->rx_ring_size; i++) {
3637 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3638 			netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3639 				    ring_nr, i, bp->rx_ring_size);
3640 			break;
3641 		}
3642 		prod = NEXT_RX(prod);
3643 	}
3644 	rxr->rx_prod = prod;
3645 
3646 	if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3647 		return 0;
3648 
3649 	prod = rxr->rx_agg_prod;
3650 	for (i = 0; i < bp->rx_agg_ring_size; i++) {
3651 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3652 			netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3653 				    ring_nr, i, bp->rx_ring_size);
3654 			break;
3655 		}
3656 		prod = NEXT_RX_AGG(prod);
3657 	}
3658 	rxr->rx_agg_prod = prod;
3659 
3660 	if (rxr->rx_tpa) {
3661 		dma_addr_t mapping;
3662 		u8 *data;
3663 
3664 		for (i = 0; i < bp->max_tpa; i++) {
3665 			data = __bnxt_alloc_rx_frag(bp, &mapping, GFP_KERNEL);
3666 			if (!data)
3667 				return -ENOMEM;
3668 
3669 			rxr->rx_tpa[i].data = data;
3670 			rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3671 			rxr->rx_tpa[i].mapping = mapping;
3672 		}
3673 	}
3674 	return 0;
3675 }
3676 
3677 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3678 {
3679 	struct bnxt_rx_ring_info *rxr;
3680 	struct bnxt_ring_struct *ring;
3681 	u32 type;
3682 
3683 	type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3684 		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3685 
3686 	if (NET_IP_ALIGN == 2)
3687 		type |= RX_BD_FLAGS_SOP;
3688 
3689 	rxr = &bp->rx_ring[ring_nr];
3690 	ring = &rxr->rx_ring_struct;
3691 	bnxt_init_rxbd_pages(ring, type);
3692 
3693 	if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3694 		bpf_prog_add(bp->xdp_prog, 1);
3695 		rxr->xdp_prog = bp->xdp_prog;
3696 	}
3697 	ring->fw_ring_id = INVALID_HW_RING_ID;
3698 
3699 	ring = &rxr->rx_agg_ring_struct;
3700 	ring->fw_ring_id = INVALID_HW_RING_ID;
3701 
3702 	if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3703 		type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3704 			RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3705 
3706 		bnxt_init_rxbd_pages(ring, type);
3707 	}
3708 
3709 	return bnxt_alloc_one_rx_ring(bp, ring_nr);
3710 }
3711 
3712 static void bnxt_init_cp_rings(struct bnxt *bp)
3713 {
3714 	int i, j;
3715 
3716 	for (i = 0; i < bp->cp_nr_rings; i++) {
3717 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3718 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3719 
3720 		ring->fw_ring_id = INVALID_HW_RING_ID;
3721 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3722 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3723 		for (j = 0; j < 2; j++) {
3724 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3725 
3726 			if (!cpr2)
3727 				continue;
3728 
3729 			ring = &cpr2->cp_ring_struct;
3730 			ring->fw_ring_id = INVALID_HW_RING_ID;
3731 			cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3732 			cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3733 		}
3734 	}
3735 }
3736 
3737 static int bnxt_init_rx_rings(struct bnxt *bp)
3738 {
3739 	int i, rc = 0;
3740 
3741 	if (BNXT_RX_PAGE_MODE(bp)) {
3742 		bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3743 		bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3744 	} else {
3745 		bp->rx_offset = BNXT_RX_OFFSET;
3746 		bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3747 	}
3748 
3749 	for (i = 0; i < bp->rx_nr_rings; i++) {
3750 		rc = bnxt_init_one_rx_ring(bp, i);
3751 		if (rc)
3752 			break;
3753 	}
3754 
3755 	return rc;
3756 }
3757 
3758 static int bnxt_init_tx_rings(struct bnxt *bp)
3759 {
3760 	u16 i;
3761 
3762 	bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3763 				   BNXT_MIN_TX_DESC_CNT);
3764 
3765 	for (i = 0; i < bp->tx_nr_rings; i++) {
3766 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3767 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3768 
3769 		ring->fw_ring_id = INVALID_HW_RING_ID;
3770 	}
3771 
3772 	return 0;
3773 }
3774 
3775 static void bnxt_free_ring_grps(struct bnxt *bp)
3776 {
3777 	kfree(bp->grp_info);
3778 	bp->grp_info = NULL;
3779 }
3780 
3781 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3782 {
3783 	int i;
3784 
3785 	if (irq_re_init) {
3786 		bp->grp_info = kcalloc(bp->cp_nr_rings,
3787 				       sizeof(struct bnxt_ring_grp_info),
3788 				       GFP_KERNEL);
3789 		if (!bp->grp_info)
3790 			return -ENOMEM;
3791 	}
3792 	for (i = 0; i < bp->cp_nr_rings; i++) {
3793 		if (irq_re_init)
3794 			bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3795 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3796 		bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3797 		bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3798 		bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3799 	}
3800 	return 0;
3801 }
3802 
3803 static void bnxt_free_vnics(struct bnxt *bp)
3804 {
3805 	kfree(bp->vnic_info);
3806 	bp->vnic_info = NULL;
3807 	bp->nr_vnics = 0;
3808 }
3809 
3810 static int bnxt_alloc_vnics(struct bnxt *bp)
3811 {
3812 	int num_vnics = 1;
3813 
3814 #ifdef CONFIG_RFS_ACCEL
3815 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3816 		num_vnics += bp->rx_nr_rings;
3817 #endif
3818 
3819 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3820 		num_vnics++;
3821 
3822 	bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3823 				GFP_KERNEL);
3824 	if (!bp->vnic_info)
3825 		return -ENOMEM;
3826 
3827 	bp->nr_vnics = num_vnics;
3828 	return 0;
3829 }
3830 
3831 static void bnxt_init_vnics(struct bnxt *bp)
3832 {
3833 	int i;
3834 
3835 	for (i = 0; i < bp->nr_vnics; i++) {
3836 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3837 		int j;
3838 
3839 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
3840 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3841 			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3842 
3843 		vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3844 
3845 		if (bp->vnic_info[i].rss_hash_key) {
3846 			if (i == 0)
3847 				get_random_bytes(vnic->rss_hash_key,
3848 					      HW_HASH_KEY_SIZE);
3849 			else
3850 				memcpy(vnic->rss_hash_key,
3851 				       bp->vnic_info[0].rss_hash_key,
3852 				       HW_HASH_KEY_SIZE);
3853 		}
3854 	}
3855 }
3856 
3857 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3858 {
3859 	int pages;
3860 
3861 	pages = ring_size / desc_per_pg;
3862 
3863 	if (!pages)
3864 		return 1;
3865 
3866 	pages++;
3867 
3868 	while (pages & (pages - 1))
3869 		pages++;
3870 
3871 	return pages;
3872 }
3873 
3874 void bnxt_set_tpa_flags(struct bnxt *bp)
3875 {
3876 	bp->flags &= ~BNXT_FLAG_TPA;
3877 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3878 		return;
3879 	if (bp->dev->features & NETIF_F_LRO)
3880 		bp->flags |= BNXT_FLAG_LRO;
3881 	else if (bp->dev->features & NETIF_F_GRO_HW)
3882 		bp->flags |= BNXT_FLAG_GRO;
3883 }
3884 
3885 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3886  * be set on entry.
3887  */
3888 void bnxt_set_ring_params(struct bnxt *bp)
3889 {
3890 	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3891 	u32 agg_factor = 0, agg_ring_size = 0;
3892 
3893 	/* 8 for CRC and VLAN */
3894 	rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3895 
3896 	rx_space = rx_size + ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) +
3897 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3898 
3899 	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3900 	ring_size = bp->rx_ring_size;
3901 	bp->rx_agg_ring_size = 0;
3902 	bp->rx_agg_nr_pages = 0;
3903 
3904 	if (bp->flags & BNXT_FLAG_TPA)
3905 		agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3906 
3907 	bp->flags &= ~BNXT_FLAG_JUMBO;
3908 	if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3909 		u32 jumbo_factor;
3910 
3911 		bp->flags |= BNXT_FLAG_JUMBO;
3912 		jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3913 		if (jumbo_factor > agg_factor)
3914 			agg_factor = jumbo_factor;
3915 	}
3916 	if (agg_factor) {
3917 		if (ring_size > BNXT_MAX_RX_DESC_CNT_JUM_ENA) {
3918 			ring_size = BNXT_MAX_RX_DESC_CNT_JUM_ENA;
3919 			netdev_warn(bp->dev, "RX ring size reduced from %d to %d because the jumbo ring is now enabled\n",
3920 				    bp->rx_ring_size, ring_size);
3921 			bp->rx_ring_size = ring_size;
3922 		}
3923 		agg_ring_size = ring_size * agg_factor;
3924 
3925 		bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3926 							RX_DESC_CNT);
3927 		if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3928 			u32 tmp = agg_ring_size;
3929 
3930 			bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3931 			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3932 			netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3933 				    tmp, agg_ring_size);
3934 		}
3935 		bp->rx_agg_ring_size = agg_ring_size;
3936 		bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3937 
3938 		if (BNXT_RX_PAGE_MODE(bp)) {
3939 			rx_space = PAGE_SIZE;
3940 			rx_size = PAGE_SIZE -
3941 				  ALIGN(max(NET_SKB_PAD, XDP_PACKET_HEADROOM), 8) -
3942 				  SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3943 		} else {
3944 			rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3945 			rx_space = rx_size + NET_SKB_PAD +
3946 				SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3947 		}
3948 	}
3949 
3950 	bp->rx_buf_use_size = rx_size;
3951 	bp->rx_buf_size = rx_space;
3952 
3953 	bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3954 	bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3955 
3956 	ring_size = bp->tx_ring_size;
3957 	bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3958 	bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3959 
3960 	max_rx_cmpl = bp->rx_ring_size;
3961 	/* MAX TPA needs to be added because TPA_START completions are
3962 	 * immediately recycled, so the TPA completions are not bound by
3963 	 * the RX ring size.
3964 	 */
3965 	if (bp->flags & BNXT_FLAG_TPA)
3966 		max_rx_cmpl += bp->max_tpa;
3967 	/* RX and TPA completions are 32-byte, all others are 16-byte */
3968 	ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3969 	bp->cp_ring_size = ring_size;
3970 
3971 	bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3972 	if (bp->cp_nr_pages > MAX_CP_PAGES) {
3973 		bp->cp_nr_pages = MAX_CP_PAGES;
3974 		bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3975 		netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3976 			    ring_size, bp->cp_ring_size);
3977 	}
3978 	bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3979 	bp->cp_ring_mask = bp->cp_bit - 1;
3980 }
3981 
3982 /* Changing allocation mode of RX rings.
3983  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3984  */
3985 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3986 {
3987 	struct net_device *dev = bp->dev;
3988 
3989 	if (page_mode) {
3990 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3991 		bp->flags |= BNXT_FLAG_RX_PAGE_MODE;
3992 
3993 		if (bp->xdp_prog->aux->xdp_has_frags)
3994 			dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_MTU);
3995 		else
3996 			dev->max_mtu =
3997 				min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3998 		if (dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
3999 			bp->flags |= BNXT_FLAG_JUMBO;
4000 			bp->rx_skb_func = bnxt_rx_multi_page_skb;
4001 		} else {
4002 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
4003 			bp->rx_skb_func = bnxt_rx_page_skb;
4004 		}
4005 		bp->rx_dir = DMA_BIDIRECTIONAL;
4006 		/* Disable LRO or GRO_HW */
4007 		netdev_update_features(dev);
4008 	} else {
4009 		dev->max_mtu = bp->max_mtu;
4010 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
4011 		bp->rx_dir = DMA_FROM_DEVICE;
4012 		bp->rx_skb_func = bnxt_rx_skb;
4013 	}
4014 	return 0;
4015 }
4016 
4017 static void bnxt_free_vnic_attributes(struct bnxt *bp)
4018 {
4019 	int i;
4020 	struct bnxt_vnic_info *vnic;
4021 	struct pci_dev *pdev = bp->pdev;
4022 
4023 	if (!bp->vnic_info)
4024 		return;
4025 
4026 	for (i = 0; i < bp->nr_vnics; i++) {
4027 		vnic = &bp->vnic_info[i];
4028 
4029 		kfree(vnic->fw_grp_ids);
4030 		vnic->fw_grp_ids = NULL;
4031 
4032 		kfree(vnic->uc_list);
4033 		vnic->uc_list = NULL;
4034 
4035 		if (vnic->mc_list) {
4036 			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
4037 					  vnic->mc_list, vnic->mc_list_mapping);
4038 			vnic->mc_list = NULL;
4039 		}
4040 
4041 		if (vnic->rss_table) {
4042 			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
4043 					  vnic->rss_table,
4044 					  vnic->rss_table_dma_addr);
4045 			vnic->rss_table = NULL;
4046 		}
4047 
4048 		vnic->rss_hash_key = NULL;
4049 		vnic->flags = 0;
4050 	}
4051 }
4052 
4053 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
4054 {
4055 	int i, rc = 0, size;
4056 	struct bnxt_vnic_info *vnic;
4057 	struct pci_dev *pdev = bp->pdev;
4058 	int max_rings;
4059 
4060 	for (i = 0; i < bp->nr_vnics; i++) {
4061 		vnic = &bp->vnic_info[i];
4062 
4063 		if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
4064 			int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
4065 
4066 			if (mem_size > 0) {
4067 				vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
4068 				if (!vnic->uc_list) {
4069 					rc = -ENOMEM;
4070 					goto out;
4071 				}
4072 			}
4073 		}
4074 
4075 		if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
4076 			vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
4077 			vnic->mc_list =
4078 				dma_alloc_coherent(&pdev->dev,
4079 						   vnic->mc_list_size,
4080 						   &vnic->mc_list_mapping,
4081 						   GFP_KERNEL);
4082 			if (!vnic->mc_list) {
4083 				rc = -ENOMEM;
4084 				goto out;
4085 			}
4086 		}
4087 
4088 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4089 			goto vnic_skip_grps;
4090 
4091 		if (vnic->flags & BNXT_VNIC_RSS_FLAG)
4092 			max_rings = bp->rx_nr_rings;
4093 		else
4094 			max_rings = 1;
4095 
4096 		vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
4097 		if (!vnic->fw_grp_ids) {
4098 			rc = -ENOMEM;
4099 			goto out;
4100 		}
4101 vnic_skip_grps:
4102 		if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
4103 		    !(vnic->flags & BNXT_VNIC_RSS_FLAG))
4104 			continue;
4105 
4106 		/* Allocate rss table and hash key */
4107 		size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
4108 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4109 			size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
4110 
4111 		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
4112 		vnic->rss_table = dma_alloc_coherent(&pdev->dev,
4113 						     vnic->rss_table_size,
4114 						     &vnic->rss_table_dma_addr,
4115 						     GFP_KERNEL);
4116 		if (!vnic->rss_table) {
4117 			rc = -ENOMEM;
4118 			goto out;
4119 		}
4120 
4121 		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
4122 		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
4123 	}
4124 	return 0;
4125 
4126 out:
4127 	return rc;
4128 }
4129 
4130 static void bnxt_free_hwrm_resources(struct bnxt *bp)
4131 {
4132 	struct bnxt_hwrm_wait_token *token;
4133 
4134 	dma_pool_destroy(bp->hwrm_dma_pool);
4135 	bp->hwrm_dma_pool = NULL;
4136 
4137 	rcu_read_lock();
4138 	hlist_for_each_entry_rcu(token, &bp->hwrm_pending_list, node)
4139 		WRITE_ONCE(token->state, BNXT_HWRM_CANCELLED);
4140 	rcu_read_unlock();
4141 }
4142 
4143 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
4144 {
4145 	bp->hwrm_dma_pool = dma_pool_create("bnxt_hwrm", &bp->pdev->dev,
4146 					    BNXT_HWRM_DMA_SIZE,
4147 					    BNXT_HWRM_DMA_ALIGN, 0);
4148 	if (!bp->hwrm_dma_pool)
4149 		return -ENOMEM;
4150 
4151 	INIT_HLIST_HEAD(&bp->hwrm_pending_list);
4152 
4153 	return 0;
4154 }
4155 
4156 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
4157 {
4158 	kfree(stats->hw_masks);
4159 	stats->hw_masks = NULL;
4160 	kfree(stats->sw_stats);
4161 	stats->sw_stats = NULL;
4162 	if (stats->hw_stats) {
4163 		dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
4164 				  stats->hw_stats_map);
4165 		stats->hw_stats = NULL;
4166 	}
4167 }
4168 
4169 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
4170 				bool alloc_masks)
4171 {
4172 	stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
4173 					     &stats->hw_stats_map, GFP_KERNEL);
4174 	if (!stats->hw_stats)
4175 		return -ENOMEM;
4176 
4177 	stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
4178 	if (!stats->sw_stats)
4179 		goto stats_mem_err;
4180 
4181 	if (alloc_masks) {
4182 		stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
4183 		if (!stats->hw_masks)
4184 			goto stats_mem_err;
4185 	}
4186 	return 0;
4187 
4188 stats_mem_err:
4189 	bnxt_free_stats_mem(bp, stats);
4190 	return -ENOMEM;
4191 }
4192 
4193 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
4194 {
4195 	int i;
4196 
4197 	for (i = 0; i < count; i++)
4198 		mask_arr[i] = mask;
4199 }
4200 
4201 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
4202 {
4203 	int i;
4204 
4205 	for (i = 0; i < count; i++)
4206 		mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
4207 }
4208 
4209 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
4210 				    struct bnxt_stats_mem *stats)
4211 {
4212 	struct hwrm_func_qstats_ext_output *resp;
4213 	struct hwrm_func_qstats_ext_input *req;
4214 	__le64 *hw_masks;
4215 	int rc;
4216 
4217 	if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
4218 	    !(bp->flags & BNXT_FLAG_CHIP_P5))
4219 		return -EOPNOTSUPP;
4220 
4221 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QSTATS_EXT);
4222 	if (rc)
4223 		return rc;
4224 
4225 	req->fid = cpu_to_le16(0xffff);
4226 	req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4227 
4228 	resp = hwrm_req_hold(bp, req);
4229 	rc = hwrm_req_send(bp, req);
4230 	if (!rc) {
4231 		hw_masks = &resp->rx_ucast_pkts;
4232 		bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
4233 	}
4234 	hwrm_req_drop(bp, req);
4235 	return rc;
4236 }
4237 
4238 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
4239 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
4240 
4241 static void bnxt_init_stats(struct bnxt *bp)
4242 {
4243 	struct bnxt_napi *bnapi = bp->bnapi[0];
4244 	struct bnxt_cp_ring_info *cpr;
4245 	struct bnxt_stats_mem *stats;
4246 	__le64 *rx_stats, *tx_stats;
4247 	int rc, rx_count, tx_count;
4248 	u64 *rx_masks, *tx_masks;
4249 	u64 mask;
4250 	u8 flags;
4251 
4252 	cpr = &bnapi->cp_ring;
4253 	stats = &cpr->stats;
4254 	rc = bnxt_hwrm_func_qstat_ext(bp, stats);
4255 	if (rc) {
4256 		if (bp->flags & BNXT_FLAG_CHIP_P5)
4257 			mask = (1ULL << 48) - 1;
4258 		else
4259 			mask = -1ULL;
4260 		bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
4261 	}
4262 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
4263 		stats = &bp->port_stats;
4264 		rx_stats = stats->hw_stats;
4265 		rx_masks = stats->hw_masks;
4266 		rx_count = sizeof(struct rx_port_stats) / 8;
4267 		tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4268 		tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
4269 		tx_count = sizeof(struct tx_port_stats) / 8;
4270 
4271 		flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
4272 		rc = bnxt_hwrm_port_qstats(bp, flags);
4273 		if (rc) {
4274 			mask = (1ULL << 40) - 1;
4275 
4276 			bnxt_fill_masks(rx_masks, mask, rx_count);
4277 			bnxt_fill_masks(tx_masks, mask, tx_count);
4278 		} else {
4279 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4280 			bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
4281 			bnxt_hwrm_port_qstats(bp, 0);
4282 		}
4283 	}
4284 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
4285 		stats = &bp->rx_port_stats_ext;
4286 		rx_stats = stats->hw_stats;
4287 		rx_masks = stats->hw_masks;
4288 		rx_count = sizeof(struct rx_port_stats_ext) / 8;
4289 		stats = &bp->tx_port_stats_ext;
4290 		tx_stats = stats->hw_stats;
4291 		tx_masks = stats->hw_masks;
4292 		tx_count = sizeof(struct tx_port_stats_ext) / 8;
4293 
4294 		flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
4295 		rc = bnxt_hwrm_port_qstats_ext(bp, flags);
4296 		if (rc) {
4297 			mask = (1ULL << 40) - 1;
4298 
4299 			bnxt_fill_masks(rx_masks, mask, rx_count);
4300 			if (tx_stats)
4301 				bnxt_fill_masks(tx_masks, mask, tx_count);
4302 		} else {
4303 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
4304 			if (tx_stats)
4305 				bnxt_copy_hw_masks(tx_masks, tx_stats,
4306 						   tx_count);
4307 			bnxt_hwrm_port_qstats_ext(bp, 0);
4308 		}
4309 	}
4310 }
4311 
4312 static void bnxt_free_port_stats(struct bnxt *bp)
4313 {
4314 	bp->flags &= ~BNXT_FLAG_PORT_STATS;
4315 	bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
4316 
4317 	bnxt_free_stats_mem(bp, &bp->port_stats);
4318 	bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
4319 	bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
4320 }
4321 
4322 static void bnxt_free_ring_stats(struct bnxt *bp)
4323 {
4324 	int i;
4325 
4326 	if (!bp->bnapi)
4327 		return;
4328 
4329 	for (i = 0; i < bp->cp_nr_rings; i++) {
4330 		struct bnxt_napi *bnapi = bp->bnapi[i];
4331 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4332 
4333 		bnxt_free_stats_mem(bp, &cpr->stats);
4334 	}
4335 }
4336 
4337 static int bnxt_alloc_stats(struct bnxt *bp)
4338 {
4339 	u32 size, i;
4340 	int rc;
4341 
4342 	size = bp->hw_ring_stats_size;
4343 
4344 	for (i = 0; i < bp->cp_nr_rings; i++) {
4345 		struct bnxt_napi *bnapi = bp->bnapi[i];
4346 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4347 
4348 		cpr->stats.len = size;
4349 		rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
4350 		if (rc)
4351 			return rc;
4352 
4353 		cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
4354 	}
4355 
4356 	if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
4357 		return 0;
4358 
4359 	if (bp->port_stats.hw_stats)
4360 		goto alloc_ext_stats;
4361 
4362 	bp->port_stats.len = BNXT_PORT_STATS_SIZE;
4363 	rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
4364 	if (rc)
4365 		return rc;
4366 
4367 	bp->flags |= BNXT_FLAG_PORT_STATS;
4368 
4369 alloc_ext_stats:
4370 	/* Display extended statistics only if FW supports it */
4371 	if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
4372 		if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
4373 			return 0;
4374 
4375 	if (bp->rx_port_stats_ext.hw_stats)
4376 		goto alloc_tx_ext_stats;
4377 
4378 	bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
4379 	rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
4380 	/* Extended stats are optional */
4381 	if (rc)
4382 		return 0;
4383 
4384 alloc_tx_ext_stats:
4385 	if (bp->tx_port_stats_ext.hw_stats)
4386 		return 0;
4387 
4388 	if (bp->hwrm_spec_code >= 0x10902 ||
4389 	    (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
4390 		bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
4391 		rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
4392 		/* Extended stats are optional */
4393 		if (rc)
4394 			return 0;
4395 	}
4396 	bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4397 	return 0;
4398 }
4399 
4400 static void bnxt_clear_ring_indices(struct bnxt *bp)
4401 {
4402 	int i;
4403 
4404 	if (!bp->bnapi)
4405 		return;
4406 
4407 	for (i = 0; i < bp->cp_nr_rings; i++) {
4408 		struct bnxt_napi *bnapi = bp->bnapi[i];
4409 		struct bnxt_cp_ring_info *cpr;
4410 		struct bnxt_rx_ring_info *rxr;
4411 		struct bnxt_tx_ring_info *txr;
4412 
4413 		if (!bnapi)
4414 			continue;
4415 
4416 		cpr = &bnapi->cp_ring;
4417 		cpr->cp_raw_cons = 0;
4418 
4419 		txr = bnapi->tx_ring;
4420 		if (txr) {
4421 			txr->tx_prod = 0;
4422 			txr->tx_cons = 0;
4423 		}
4424 
4425 		rxr = bnapi->rx_ring;
4426 		if (rxr) {
4427 			rxr->rx_prod = 0;
4428 			rxr->rx_agg_prod = 0;
4429 			rxr->rx_sw_agg_prod = 0;
4430 			rxr->rx_next_cons = 0;
4431 		}
4432 	}
4433 }
4434 
4435 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4436 {
4437 #ifdef CONFIG_RFS_ACCEL
4438 	int i;
4439 
4440 	/* Under rtnl_lock and all our NAPIs have been disabled.  It's
4441 	 * safe to delete the hash table.
4442 	 */
4443 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4444 		struct hlist_head *head;
4445 		struct hlist_node *tmp;
4446 		struct bnxt_ntuple_filter *fltr;
4447 
4448 		head = &bp->ntp_fltr_hash_tbl[i];
4449 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4450 			hlist_del(&fltr->hash);
4451 			kfree(fltr);
4452 		}
4453 	}
4454 	if (irq_reinit) {
4455 		bitmap_free(bp->ntp_fltr_bmap);
4456 		bp->ntp_fltr_bmap = NULL;
4457 	}
4458 	bp->ntp_fltr_count = 0;
4459 #endif
4460 }
4461 
4462 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4463 {
4464 #ifdef CONFIG_RFS_ACCEL
4465 	int i, rc = 0;
4466 
4467 	if (!(bp->flags & BNXT_FLAG_RFS))
4468 		return 0;
4469 
4470 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4471 		INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4472 
4473 	bp->ntp_fltr_count = 0;
4474 	bp->ntp_fltr_bmap = bitmap_zalloc(BNXT_NTP_FLTR_MAX_FLTR, GFP_KERNEL);
4475 
4476 	if (!bp->ntp_fltr_bmap)
4477 		rc = -ENOMEM;
4478 
4479 	return rc;
4480 #else
4481 	return 0;
4482 #endif
4483 }
4484 
4485 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4486 {
4487 	bnxt_free_vnic_attributes(bp);
4488 	bnxt_free_tx_rings(bp);
4489 	bnxt_free_rx_rings(bp);
4490 	bnxt_free_cp_rings(bp);
4491 	bnxt_free_all_cp_arrays(bp);
4492 	bnxt_free_ntp_fltrs(bp, irq_re_init);
4493 	if (irq_re_init) {
4494 		bnxt_free_ring_stats(bp);
4495 		if (!(bp->phy_flags & BNXT_PHY_FL_PORT_STATS_NO_RESET) ||
4496 		    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
4497 			bnxt_free_port_stats(bp);
4498 		bnxt_free_ring_grps(bp);
4499 		bnxt_free_vnics(bp);
4500 		kfree(bp->tx_ring_map);
4501 		bp->tx_ring_map = NULL;
4502 		kfree(bp->tx_ring);
4503 		bp->tx_ring = NULL;
4504 		kfree(bp->rx_ring);
4505 		bp->rx_ring = NULL;
4506 		kfree(bp->bnapi);
4507 		bp->bnapi = NULL;
4508 	} else {
4509 		bnxt_clear_ring_indices(bp);
4510 	}
4511 }
4512 
4513 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4514 {
4515 	int i, j, rc, size, arr_size;
4516 	void *bnapi;
4517 
4518 	if (irq_re_init) {
4519 		/* Allocate bnapi mem pointer array and mem block for
4520 		 * all queues
4521 		 */
4522 		arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4523 				bp->cp_nr_rings);
4524 		size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4525 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4526 		if (!bnapi)
4527 			return -ENOMEM;
4528 
4529 		bp->bnapi = bnapi;
4530 		bnapi += arr_size;
4531 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4532 			bp->bnapi[i] = bnapi;
4533 			bp->bnapi[i]->index = i;
4534 			bp->bnapi[i]->bp = bp;
4535 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4536 				struct bnxt_cp_ring_info *cpr =
4537 					&bp->bnapi[i]->cp_ring;
4538 
4539 				cpr->cp_ring_struct.ring_mem.flags =
4540 					BNXT_RMEM_RING_PTE_FLAG;
4541 			}
4542 		}
4543 
4544 		bp->rx_ring = kcalloc(bp->rx_nr_rings,
4545 				      sizeof(struct bnxt_rx_ring_info),
4546 				      GFP_KERNEL);
4547 		if (!bp->rx_ring)
4548 			return -ENOMEM;
4549 
4550 		for (i = 0; i < bp->rx_nr_rings; i++) {
4551 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4552 
4553 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4554 				rxr->rx_ring_struct.ring_mem.flags =
4555 					BNXT_RMEM_RING_PTE_FLAG;
4556 				rxr->rx_agg_ring_struct.ring_mem.flags =
4557 					BNXT_RMEM_RING_PTE_FLAG;
4558 			}
4559 			rxr->bnapi = bp->bnapi[i];
4560 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4561 		}
4562 
4563 		bp->tx_ring = kcalloc(bp->tx_nr_rings,
4564 				      sizeof(struct bnxt_tx_ring_info),
4565 				      GFP_KERNEL);
4566 		if (!bp->tx_ring)
4567 			return -ENOMEM;
4568 
4569 		bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4570 					  GFP_KERNEL);
4571 
4572 		if (!bp->tx_ring_map)
4573 			return -ENOMEM;
4574 
4575 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4576 			j = 0;
4577 		else
4578 			j = bp->rx_nr_rings;
4579 
4580 		for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4581 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4582 
4583 			if (bp->flags & BNXT_FLAG_CHIP_P5)
4584 				txr->tx_ring_struct.ring_mem.flags =
4585 					BNXT_RMEM_RING_PTE_FLAG;
4586 			txr->bnapi = bp->bnapi[j];
4587 			bp->bnapi[j]->tx_ring = txr;
4588 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4589 			if (i >= bp->tx_nr_rings_xdp) {
4590 				txr->txq_index = i - bp->tx_nr_rings_xdp;
4591 				bp->bnapi[j]->tx_int = bnxt_tx_int;
4592 			} else {
4593 				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4594 				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4595 			}
4596 		}
4597 
4598 		rc = bnxt_alloc_stats(bp);
4599 		if (rc)
4600 			goto alloc_mem_err;
4601 		bnxt_init_stats(bp);
4602 
4603 		rc = bnxt_alloc_ntp_fltrs(bp);
4604 		if (rc)
4605 			goto alloc_mem_err;
4606 
4607 		rc = bnxt_alloc_vnics(bp);
4608 		if (rc)
4609 			goto alloc_mem_err;
4610 	}
4611 
4612 	rc = bnxt_alloc_all_cp_arrays(bp);
4613 	if (rc)
4614 		goto alloc_mem_err;
4615 
4616 	bnxt_init_ring_struct(bp);
4617 
4618 	rc = bnxt_alloc_rx_rings(bp);
4619 	if (rc)
4620 		goto alloc_mem_err;
4621 
4622 	rc = bnxt_alloc_tx_rings(bp);
4623 	if (rc)
4624 		goto alloc_mem_err;
4625 
4626 	rc = bnxt_alloc_cp_rings(bp);
4627 	if (rc)
4628 		goto alloc_mem_err;
4629 
4630 	bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4631 				  BNXT_VNIC_UCAST_FLAG;
4632 	rc = bnxt_alloc_vnic_attributes(bp);
4633 	if (rc)
4634 		goto alloc_mem_err;
4635 	return 0;
4636 
4637 alloc_mem_err:
4638 	bnxt_free_mem(bp, true);
4639 	return rc;
4640 }
4641 
4642 static void bnxt_disable_int(struct bnxt *bp)
4643 {
4644 	int i;
4645 
4646 	if (!bp->bnapi)
4647 		return;
4648 
4649 	for (i = 0; i < bp->cp_nr_rings; i++) {
4650 		struct bnxt_napi *bnapi = bp->bnapi[i];
4651 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4652 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4653 
4654 		if (ring->fw_ring_id != INVALID_HW_RING_ID)
4655 			bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4656 	}
4657 }
4658 
4659 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4660 {
4661 	struct bnxt_napi *bnapi = bp->bnapi[n];
4662 	struct bnxt_cp_ring_info *cpr;
4663 
4664 	cpr = &bnapi->cp_ring;
4665 	return cpr->cp_ring_struct.map_idx;
4666 }
4667 
4668 static void bnxt_disable_int_sync(struct bnxt *bp)
4669 {
4670 	int i;
4671 
4672 	if (!bp->irq_tbl)
4673 		return;
4674 
4675 	atomic_inc(&bp->intr_sem);
4676 
4677 	bnxt_disable_int(bp);
4678 	for (i = 0; i < bp->cp_nr_rings; i++) {
4679 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4680 
4681 		synchronize_irq(bp->irq_tbl[map_idx].vector);
4682 	}
4683 }
4684 
4685 static void bnxt_enable_int(struct bnxt *bp)
4686 {
4687 	int i;
4688 
4689 	atomic_set(&bp->intr_sem, 0);
4690 	for (i = 0; i < bp->cp_nr_rings; i++) {
4691 		struct bnxt_napi *bnapi = bp->bnapi[i];
4692 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4693 
4694 		bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4695 	}
4696 }
4697 
4698 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4699 			    bool async_only)
4700 {
4701 	DECLARE_BITMAP(async_events_bmap, 256);
4702 	u32 *events = (u32 *)async_events_bmap;
4703 	struct hwrm_func_drv_rgtr_output *resp;
4704 	struct hwrm_func_drv_rgtr_input *req;
4705 	u32 flags;
4706 	int rc, i;
4707 
4708 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_RGTR);
4709 	if (rc)
4710 		return rc;
4711 
4712 	req->enables = cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4713 				   FUNC_DRV_RGTR_REQ_ENABLES_VER |
4714 				   FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4715 
4716 	req->os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4717 	flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4718 	if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4719 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4720 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4721 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4722 			 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4723 	req->flags = cpu_to_le32(flags);
4724 	req->ver_maj_8b = DRV_VER_MAJ;
4725 	req->ver_min_8b = DRV_VER_MIN;
4726 	req->ver_upd_8b = DRV_VER_UPD;
4727 	req->ver_maj = cpu_to_le16(DRV_VER_MAJ);
4728 	req->ver_min = cpu_to_le16(DRV_VER_MIN);
4729 	req->ver_upd = cpu_to_le16(DRV_VER_UPD);
4730 
4731 	if (BNXT_PF(bp)) {
4732 		u32 data[8];
4733 		int i;
4734 
4735 		memset(data, 0, sizeof(data));
4736 		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4737 			u16 cmd = bnxt_vf_req_snif[i];
4738 			unsigned int bit, idx;
4739 
4740 			idx = cmd / 32;
4741 			bit = cmd % 32;
4742 			data[idx] |= 1 << bit;
4743 		}
4744 
4745 		for (i = 0; i < 8; i++)
4746 			req->vf_req_fwd[i] = cpu_to_le32(data[i]);
4747 
4748 		req->enables |=
4749 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4750 	}
4751 
4752 	if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4753 		req->flags |= cpu_to_le32(
4754 			FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4755 
4756 	memset(async_events_bmap, 0, sizeof(async_events_bmap));
4757 	for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4758 		u16 event_id = bnxt_async_events_arr[i];
4759 
4760 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4761 		    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4762 			continue;
4763 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE &&
4764 		    !bp->ptp_cfg)
4765 			continue;
4766 		__set_bit(bnxt_async_events_arr[i], async_events_bmap);
4767 	}
4768 	if (bmap && bmap_size) {
4769 		for (i = 0; i < bmap_size; i++) {
4770 			if (test_bit(i, bmap))
4771 				__set_bit(i, async_events_bmap);
4772 		}
4773 	}
4774 	for (i = 0; i < 8; i++)
4775 		req->async_event_fwd[i] |= cpu_to_le32(events[i]);
4776 
4777 	if (async_only)
4778 		req->enables =
4779 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4780 
4781 	resp = hwrm_req_hold(bp, req);
4782 	rc = hwrm_req_send(bp, req);
4783 	if (!rc) {
4784 		set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4785 		if (resp->flags &
4786 		    cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4787 			bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4788 	}
4789 	hwrm_req_drop(bp, req);
4790 	return rc;
4791 }
4792 
4793 int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4794 {
4795 	struct hwrm_func_drv_unrgtr_input *req;
4796 	int rc;
4797 
4798 	if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4799 		return 0;
4800 
4801 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_UNRGTR);
4802 	if (rc)
4803 		return rc;
4804 	return hwrm_req_send(bp, req);
4805 }
4806 
4807 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4808 {
4809 	struct hwrm_tunnel_dst_port_free_input *req;
4810 	int rc;
4811 
4812 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN &&
4813 	    bp->vxlan_fw_dst_port_id == INVALID_HW_RING_ID)
4814 		return 0;
4815 	if (tunnel_type == TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE &&
4816 	    bp->nge_fw_dst_port_id == INVALID_HW_RING_ID)
4817 		return 0;
4818 
4819 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_FREE);
4820 	if (rc)
4821 		return rc;
4822 
4823 	req->tunnel_type = tunnel_type;
4824 
4825 	switch (tunnel_type) {
4826 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4827 		req->tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4828 		bp->vxlan_port = 0;
4829 		bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4830 		break;
4831 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4832 		req->tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4833 		bp->nge_port = 0;
4834 		bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4835 		break;
4836 	default:
4837 		break;
4838 	}
4839 
4840 	rc = hwrm_req_send(bp, req);
4841 	if (rc)
4842 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4843 			   rc);
4844 	return rc;
4845 }
4846 
4847 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4848 					   u8 tunnel_type)
4849 {
4850 	struct hwrm_tunnel_dst_port_alloc_output *resp;
4851 	struct hwrm_tunnel_dst_port_alloc_input *req;
4852 	int rc;
4853 
4854 	rc = hwrm_req_init(bp, req, HWRM_TUNNEL_DST_PORT_ALLOC);
4855 	if (rc)
4856 		return rc;
4857 
4858 	req->tunnel_type = tunnel_type;
4859 	req->tunnel_dst_port_val = port;
4860 
4861 	resp = hwrm_req_hold(bp, req);
4862 	rc = hwrm_req_send(bp, req);
4863 	if (rc) {
4864 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4865 			   rc);
4866 		goto err_out;
4867 	}
4868 
4869 	switch (tunnel_type) {
4870 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4871 		bp->vxlan_port = port;
4872 		bp->vxlan_fw_dst_port_id =
4873 			le16_to_cpu(resp->tunnel_dst_port_id);
4874 		break;
4875 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4876 		bp->nge_port = port;
4877 		bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4878 		break;
4879 	default:
4880 		break;
4881 	}
4882 
4883 err_out:
4884 	hwrm_req_drop(bp, req);
4885 	return rc;
4886 }
4887 
4888 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4889 {
4890 	struct hwrm_cfa_l2_set_rx_mask_input *req;
4891 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4892 	int rc;
4893 
4894 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_SET_RX_MASK);
4895 	if (rc)
4896 		return rc;
4897 
4898 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4899 	if (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST) {
4900 		req->num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4901 		req->mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4902 	}
4903 	req->mask = cpu_to_le32(vnic->rx_mask);
4904 	return hwrm_req_send_silent(bp, req);
4905 }
4906 
4907 #ifdef CONFIG_RFS_ACCEL
4908 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4909 					    struct bnxt_ntuple_filter *fltr)
4910 {
4911 	struct hwrm_cfa_ntuple_filter_free_input *req;
4912 	int rc;
4913 
4914 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_FREE);
4915 	if (rc)
4916 		return rc;
4917 
4918 	req->ntuple_filter_id = fltr->filter_id;
4919 	return hwrm_req_send(bp, req);
4920 }
4921 
4922 #define BNXT_NTP_FLTR_FLAGS					\
4923 	(CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |	\
4924 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |	\
4925 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |	\
4926 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |	\
4927 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |	\
4928 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |	\
4929 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |	\
4930 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |	\
4931 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |	\
4932 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |		\
4933 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |	\
4934 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |		\
4935 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |	\
4936 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4937 
4938 #define BNXT_NTP_TUNNEL_FLTR_FLAG				\
4939 		CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4940 
4941 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4942 					     struct bnxt_ntuple_filter *fltr)
4943 {
4944 	struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4945 	struct hwrm_cfa_ntuple_filter_alloc_input *req;
4946 	struct flow_keys *keys = &fltr->fkeys;
4947 	struct bnxt_vnic_info *vnic;
4948 	u32 flags = 0;
4949 	int rc;
4950 
4951 	rc = hwrm_req_init(bp, req, HWRM_CFA_NTUPLE_FILTER_ALLOC);
4952 	if (rc)
4953 		return rc;
4954 
4955 	req->l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4956 
4957 	if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4958 		flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4959 		req->dst_id = cpu_to_le16(fltr->rxq);
4960 	} else {
4961 		vnic = &bp->vnic_info[fltr->rxq + 1];
4962 		req->dst_id = cpu_to_le16(vnic->fw_vnic_id);
4963 	}
4964 	req->flags = cpu_to_le32(flags);
4965 	req->enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4966 
4967 	req->ethertype = htons(ETH_P_IP);
4968 	memcpy(req->src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4969 	req->ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4970 	req->ip_protocol = keys->basic.ip_proto;
4971 
4972 	if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4973 		int i;
4974 
4975 		req->ethertype = htons(ETH_P_IPV6);
4976 		req->ip_addr_type =
4977 			CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4978 		*(struct in6_addr *)&req->src_ipaddr[0] =
4979 			keys->addrs.v6addrs.src;
4980 		*(struct in6_addr *)&req->dst_ipaddr[0] =
4981 			keys->addrs.v6addrs.dst;
4982 		for (i = 0; i < 4; i++) {
4983 			req->src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4984 			req->dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4985 		}
4986 	} else {
4987 		req->src_ipaddr[0] = keys->addrs.v4addrs.src;
4988 		req->src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4989 		req->dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4990 		req->dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4991 	}
4992 	if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4993 		req->enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4994 		req->tunnel_type =
4995 			CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4996 	}
4997 
4998 	req->src_port = keys->ports.src;
4999 	req->src_port_mask = cpu_to_be16(0xffff);
5000 	req->dst_port = keys->ports.dst;
5001 	req->dst_port_mask = cpu_to_be16(0xffff);
5002 
5003 	resp = hwrm_req_hold(bp, req);
5004 	rc = hwrm_req_send(bp, req);
5005 	if (!rc)
5006 		fltr->filter_id = resp->ntuple_filter_id;
5007 	hwrm_req_drop(bp, req);
5008 	return rc;
5009 }
5010 #endif
5011 
5012 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
5013 				     const u8 *mac_addr)
5014 {
5015 	struct hwrm_cfa_l2_filter_alloc_output *resp;
5016 	struct hwrm_cfa_l2_filter_alloc_input *req;
5017 	int rc;
5018 
5019 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_ALLOC);
5020 	if (rc)
5021 		return rc;
5022 
5023 	req->flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
5024 	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
5025 		req->flags |=
5026 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
5027 	req->dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
5028 	req->enables =
5029 		cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
5030 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
5031 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
5032 	memcpy(req->l2_addr, mac_addr, ETH_ALEN);
5033 	req->l2_addr_mask[0] = 0xff;
5034 	req->l2_addr_mask[1] = 0xff;
5035 	req->l2_addr_mask[2] = 0xff;
5036 	req->l2_addr_mask[3] = 0xff;
5037 	req->l2_addr_mask[4] = 0xff;
5038 	req->l2_addr_mask[5] = 0xff;
5039 
5040 	resp = hwrm_req_hold(bp, req);
5041 	rc = hwrm_req_send(bp, req);
5042 	if (!rc)
5043 		bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
5044 							resp->l2_filter_id;
5045 	hwrm_req_drop(bp, req);
5046 	return rc;
5047 }
5048 
5049 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
5050 {
5051 	struct hwrm_cfa_l2_filter_free_input *req;
5052 	u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
5053 	int rc;
5054 
5055 	/* Any associated ntuple filters will also be cleared by firmware. */
5056 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
5057 	if (rc)
5058 		return rc;
5059 	hwrm_req_hold(bp, req);
5060 	for (i = 0; i < num_of_vnics; i++) {
5061 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5062 
5063 		for (j = 0; j < vnic->uc_filter_count; j++) {
5064 			req->l2_filter_id = vnic->fw_l2_filter_id[j];
5065 
5066 			rc = hwrm_req_send(bp, req);
5067 		}
5068 		vnic->uc_filter_count = 0;
5069 	}
5070 	hwrm_req_drop(bp, req);
5071 	return rc;
5072 }
5073 
5074 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
5075 {
5076 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5077 	u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
5078 	struct hwrm_vnic_tpa_cfg_input *req;
5079 	int rc;
5080 
5081 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
5082 		return 0;
5083 
5084 	rc = hwrm_req_init(bp, req, HWRM_VNIC_TPA_CFG);
5085 	if (rc)
5086 		return rc;
5087 
5088 	if (tpa_flags) {
5089 		u16 mss = bp->dev->mtu - 40;
5090 		u32 nsegs, n, segs = 0, flags;
5091 
5092 		flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
5093 			VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
5094 			VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
5095 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
5096 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
5097 		if (tpa_flags & BNXT_FLAG_GRO)
5098 			flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
5099 
5100 		req->flags = cpu_to_le32(flags);
5101 
5102 		req->enables =
5103 			cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
5104 				    VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
5105 				    VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
5106 
5107 		/* Number of segs are log2 units, and first packet is not
5108 		 * included as part of this units.
5109 		 */
5110 		if (mss <= BNXT_RX_PAGE_SIZE) {
5111 			n = BNXT_RX_PAGE_SIZE / mss;
5112 			nsegs = (MAX_SKB_FRAGS - 1) * n;
5113 		} else {
5114 			n = mss / BNXT_RX_PAGE_SIZE;
5115 			if (mss & (BNXT_RX_PAGE_SIZE - 1))
5116 				n++;
5117 			nsegs = (MAX_SKB_FRAGS - n) / n;
5118 		}
5119 
5120 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5121 			segs = MAX_TPA_SEGS_P5;
5122 			max_aggs = bp->max_tpa;
5123 		} else {
5124 			segs = ilog2(nsegs);
5125 		}
5126 		req->max_agg_segs = cpu_to_le16(segs);
5127 		req->max_aggs = cpu_to_le16(max_aggs);
5128 
5129 		req->min_agg_len = cpu_to_le32(512);
5130 	}
5131 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5132 
5133 	return hwrm_req_send(bp, req);
5134 }
5135 
5136 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
5137 {
5138 	struct bnxt_ring_grp_info *grp_info;
5139 
5140 	grp_info = &bp->grp_info[ring->grp_idx];
5141 	return grp_info->cp_fw_ring_id;
5142 }
5143 
5144 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
5145 {
5146 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5147 		struct bnxt_napi *bnapi = rxr->bnapi;
5148 		struct bnxt_cp_ring_info *cpr;
5149 
5150 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
5151 		return cpr->cp_ring_struct.fw_ring_id;
5152 	} else {
5153 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
5154 	}
5155 }
5156 
5157 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
5158 {
5159 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5160 		struct bnxt_napi *bnapi = txr->bnapi;
5161 		struct bnxt_cp_ring_info *cpr;
5162 
5163 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
5164 		return cpr->cp_ring_struct.fw_ring_id;
5165 	} else {
5166 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
5167 	}
5168 }
5169 
5170 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5171 {
5172 	int entries;
5173 
5174 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5175 		entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5176 	else
5177 		entries = HW_HASH_INDEX_SIZE;
5178 
5179 	bp->rss_indir_tbl_entries = entries;
5180 	bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5181 					  GFP_KERNEL);
5182 	if (!bp->rss_indir_tbl)
5183 		return -ENOMEM;
5184 	return 0;
5185 }
5186 
5187 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5188 {
5189 	u16 max_rings, max_entries, pad, i;
5190 
5191 	if (!bp->rx_nr_rings)
5192 		return;
5193 
5194 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5195 		max_rings = bp->rx_nr_rings - 1;
5196 	else
5197 		max_rings = bp->rx_nr_rings;
5198 
5199 	max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5200 
5201 	for (i = 0; i < max_entries; i++)
5202 		bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5203 
5204 	pad = bp->rss_indir_tbl_entries - max_entries;
5205 	if (pad)
5206 		memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5207 }
5208 
5209 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5210 {
5211 	u16 i, tbl_size, max_ring = 0;
5212 
5213 	if (!bp->rss_indir_tbl)
5214 		return 0;
5215 
5216 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5217 	for (i = 0; i < tbl_size; i++)
5218 		max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5219 	return max_ring;
5220 }
5221 
5222 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5223 {
5224 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5225 		return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5226 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5227 		return 2;
5228 	return 1;
5229 }
5230 
5231 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5232 {
5233 	bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5234 	u16 i, j;
5235 
5236 	/* Fill the RSS indirection table with ring group ids */
5237 	for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5238 		if (!no_rss)
5239 			j = bp->rss_indir_tbl[i];
5240 		vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5241 	}
5242 }
5243 
5244 static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5245 				    struct bnxt_vnic_info *vnic)
5246 {
5247 	__le16 *ring_tbl = vnic->rss_table;
5248 	struct bnxt_rx_ring_info *rxr;
5249 	u16 tbl_size, i;
5250 
5251 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5252 
5253 	for (i = 0; i < tbl_size; i++) {
5254 		u16 ring_id, j;
5255 
5256 		j = bp->rss_indir_tbl[i];
5257 		rxr = &bp->rx_ring[j];
5258 
5259 		ring_id = rxr->rx_ring_struct.fw_ring_id;
5260 		*ring_tbl++ = cpu_to_le16(ring_id);
5261 		ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5262 		*ring_tbl++ = cpu_to_le16(ring_id);
5263 	}
5264 }
5265 
5266 static void
5267 __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req,
5268 			 struct bnxt_vnic_info *vnic)
5269 {
5270 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5271 		bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5272 	else
5273 		bnxt_fill_hw_rss_tbl(bp, vnic);
5274 
5275 	if (bp->rss_hash_delta) {
5276 		req->hash_type = cpu_to_le32(bp->rss_hash_delta);
5277 		if (bp->rss_hash_cfg & bp->rss_hash_delta)
5278 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_INCLUDE;
5279 		else
5280 			req->flags |= VNIC_RSS_CFG_REQ_FLAGS_HASH_TYPE_EXCLUDE;
5281 	} else {
5282 		req->hash_type = cpu_to_le32(bp->rss_hash_cfg);
5283 	}
5284 	req->hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5285 	req->ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5286 	req->hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5287 }
5288 
5289 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5290 {
5291 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5292 	struct hwrm_vnic_rss_cfg_input *req;
5293 	int rc;
5294 
5295 	if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5296 	    vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5297 		return 0;
5298 
5299 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5300 	if (rc)
5301 		return rc;
5302 
5303 	if (set_rss)
5304 		__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5305 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5306 	return hwrm_req_send(bp, req);
5307 }
5308 
5309 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5310 {
5311 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5312 	struct hwrm_vnic_rss_cfg_input *req;
5313 	dma_addr_t ring_tbl_map;
5314 	u32 i, nr_ctxs;
5315 	int rc;
5316 
5317 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_CFG);
5318 	if (rc)
5319 		return rc;
5320 
5321 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5322 	if (!set_rss)
5323 		return hwrm_req_send(bp, req);
5324 
5325 	__bnxt_hwrm_vnic_set_rss(bp, req, vnic);
5326 	ring_tbl_map = vnic->rss_table_dma_addr;
5327 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5328 
5329 	hwrm_req_hold(bp, req);
5330 	for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5331 		req->ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5332 		req->ring_table_pair_index = i;
5333 		req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5334 		rc = hwrm_req_send(bp, req);
5335 		if (rc)
5336 			goto exit;
5337 	}
5338 
5339 exit:
5340 	hwrm_req_drop(bp, req);
5341 	return rc;
5342 }
5343 
5344 static void bnxt_hwrm_update_rss_hash_cfg(struct bnxt *bp)
5345 {
5346 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
5347 	struct hwrm_vnic_rss_qcfg_output *resp;
5348 	struct hwrm_vnic_rss_qcfg_input *req;
5349 
5350 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_QCFG))
5351 		return;
5352 
5353 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5354 	/* all contexts configured to same hash_type, zero always exists */
5355 	req->rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5356 	resp = hwrm_req_hold(bp, req);
5357 	if (!hwrm_req_send(bp, req)) {
5358 		bp->rss_hash_cfg = le32_to_cpu(resp->hash_type) ?: bp->rss_hash_cfg;
5359 		bp->rss_hash_delta = 0;
5360 	}
5361 	hwrm_req_drop(bp, req);
5362 }
5363 
5364 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5365 {
5366 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5367 	struct hwrm_vnic_plcmodes_cfg_input *req;
5368 	int rc;
5369 
5370 	rc = hwrm_req_init(bp, req, HWRM_VNIC_PLCMODES_CFG);
5371 	if (rc)
5372 		return rc;
5373 
5374 	req->flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT);
5375 	req->enables = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID);
5376 
5377 	if (BNXT_RX_PAGE_MODE(bp)) {
5378 		req->jumbo_thresh = cpu_to_le16(bp->rx_buf_use_size);
5379 	} else {
5380 		req->flags |= cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5381 					  VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5382 		req->enables |=
5383 			cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5384 		req->jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5385 		req->hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5386 	}
5387 	req->vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5388 	return hwrm_req_send(bp, req);
5389 }
5390 
5391 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5392 					u16 ctx_idx)
5393 {
5394 	struct hwrm_vnic_rss_cos_lb_ctx_free_input *req;
5395 
5396 	if (hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_FREE))
5397 		return;
5398 
5399 	req->rss_cos_lb_ctx_id =
5400 		cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5401 
5402 	hwrm_req_send(bp, req);
5403 	bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5404 }
5405 
5406 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5407 {
5408 	int i, j;
5409 
5410 	for (i = 0; i < bp->nr_vnics; i++) {
5411 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5412 
5413 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5414 			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5415 				bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5416 		}
5417 	}
5418 	bp->rsscos_nr_ctxs = 0;
5419 }
5420 
5421 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5422 {
5423 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp;
5424 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_input *req;
5425 	int rc;
5426 
5427 	rc = hwrm_req_init(bp, req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC);
5428 	if (rc)
5429 		return rc;
5430 
5431 	resp = hwrm_req_hold(bp, req);
5432 	rc = hwrm_req_send(bp, req);
5433 	if (!rc)
5434 		bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5435 			le16_to_cpu(resp->rss_cos_lb_ctx_id);
5436 	hwrm_req_drop(bp, req);
5437 
5438 	return rc;
5439 }
5440 
5441 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5442 {
5443 	if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5444 		return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5445 	return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5446 }
5447 
5448 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5449 {
5450 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5451 	struct hwrm_vnic_cfg_input *req;
5452 	unsigned int ring = 0, grp_idx;
5453 	u16 def_vlan = 0;
5454 	int rc;
5455 
5456 	rc = hwrm_req_init(bp, req, HWRM_VNIC_CFG);
5457 	if (rc)
5458 		return rc;
5459 
5460 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5461 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5462 
5463 		req->default_rx_ring_id =
5464 			cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5465 		req->default_cmpl_ring_id =
5466 			cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5467 		req->enables =
5468 			cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5469 				    VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5470 		goto vnic_mru;
5471 	}
5472 	req->enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5473 	/* Only RSS support for now TBD: COS & LB */
5474 	if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5475 		req->rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5476 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5477 					   VNIC_CFG_REQ_ENABLES_MRU);
5478 	} else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5479 		req->rss_rule =
5480 			cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5481 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5482 					   VNIC_CFG_REQ_ENABLES_MRU);
5483 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5484 	} else {
5485 		req->rss_rule = cpu_to_le16(0xffff);
5486 	}
5487 
5488 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5489 	    (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5490 		req->cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5491 		req->enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5492 	} else {
5493 		req->cos_rule = cpu_to_le16(0xffff);
5494 	}
5495 
5496 	if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5497 		ring = 0;
5498 	else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5499 		ring = vnic_id - 1;
5500 	else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5501 		ring = bp->rx_nr_rings - 1;
5502 
5503 	grp_idx = bp->rx_ring[ring].bnapi->index;
5504 	req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5505 	req->lb_rule = cpu_to_le16(0xffff);
5506 vnic_mru:
5507 	req->mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5508 
5509 	req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5510 #ifdef CONFIG_BNXT_SRIOV
5511 	if (BNXT_VF(bp))
5512 		def_vlan = bp->vf.vlan;
5513 #endif
5514 	if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5515 		req->flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5516 	if (!vnic_id && bnxt_ulp_registered(bp->edev))
5517 		req->flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5518 
5519 	return hwrm_req_send(bp, req);
5520 }
5521 
5522 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5523 {
5524 	if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5525 		struct hwrm_vnic_free_input *req;
5526 
5527 		if (hwrm_req_init(bp, req, HWRM_VNIC_FREE))
5528 			return;
5529 
5530 		req->vnic_id =
5531 			cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5532 
5533 		hwrm_req_send(bp, req);
5534 		bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5535 	}
5536 }
5537 
5538 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5539 {
5540 	u16 i;
5541 
5542 	for (i = 0; i < bp->nr_vnics; i++)
5543 		bnxt_hwrm_vnic_free_one(bp, i);
5544 }
5545 
5546 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5547 				unsigned int start_rx_ring_idx,
5548 				unsigned int nr_rings)
5549 {
5550 	unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5551 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5552 	struct hwrm_vnic_alloc_output *resp;
5553 	struct hwrm_vnic_alloc_input *req;
5554 	int rc;
5555 
5556 	rc = hwrm_req_init(bp, req, HWRM_VNIC_ALLOC);
5557 	if (rc)
5558 		return rc;
5559 
5560 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5561 		goto vnic_no_ring_grps;
5562 
5563 	/* map ring groups to this vnic */
5564 	for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5565 		grp_idx = bp->rx_ring[i].bnapi->index;
5566 		if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5567 			netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5568 				   j, nr_rings);
5569 			break;
5570 		}
5571 		vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5572 	}
5573 
5574 vnic_no_ring_grps:
5575 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5576 		vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5577 	if (vnic_id == 0)
5578 		req->flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5579 
5580 	resp = hwrm_req_hold(bp, req);
5581 	rc = hwrm_req_send(bp, req);
5582 	if (!rc)
5583 		vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5584 	hwrm_req_drop(bp, req);
5585 	return rc;
5586 }
5587 
5588 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5589 {
5590 	struct hwrm_vnic_qcaps_output *resp;
5591 	struct hwrm_vnic_qcaps_input *req;
5592 	int rc;
5593 
5594 	bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5595 	bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5596 	if (bp->hwrm_spec_code < 0x10600)
5597 		return 0;
5598 
5599 	rc = hwrm_req_init(bp, req, HWRM_VNIC_QCAPS);
5600 	if (rc)
5601 		return rc;
5602 
5603 	resp = hwrm_req_hold(bp, req);
5604 	rc = hwrm_req_send(bp, req);
5605 	if (!rc) {
5606 		u32 flags = le32_to_cpu(resp->flags);
5607 
5608 		if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5609 		    (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5610 			bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5611 		if (flags &
5612 		    VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5613 			bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5614 
5615 		/* Older P5 fw before EXT_HW_STATS support did not set
5616 		 * VLAN_STRIP_CAP properly.
5617 		 */
5618 		if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5619 		    (BNXT_CHIP_P5_THOR(bp) &&
5620 		     !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5621 			bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5622 		if (flags & VNIC_QCAPS_RESP_FLAGS_RSS_HASH_TYPE_DELTA_CAP)
5623 			bp->fw_cap |= BNXT_FW_CAP_RSS_HASH_TYPE_DELTA;
5624 		bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5625 		if (bp->max_tpa_v2) {
5626 			if (BNXT_CHIP_P5_THOR(bp))
5627 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5628 			else
5629 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5630 		}
5631 	}
5632 	hwrm_req_drop(bp, req);
5633 	return rc;
5634 }
5635 
5636 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5637 {
5638 	struct hwrm_ring_grp_alloc_output *resp;
5639 	struct hwrm_ring_grp_alloc_input *req;
5640 	int rc;
5641 	u16 i;
5642 
5643 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5644 		return 0;
5645 
5646 	rc = hwrm_req_init(bp, req, HWRM_RING_GRP_ALLOC);
5647 	if (rc)
5648 		return rc;
5649 
5650 	resp = hwrm_req_hold(bp, req);
5651 	for (i = 0; i < bp->rx_nr_rings; i++) {
5652 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5653 
5654 		req->cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5655 		req->rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5656 		req->ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5657 		req->sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5658 
5659 		rc = hwrm_req_send(bp, req);
5660 
5661 		if (rc)
5662 			break;
5663 
5664 		bp->grp_info[grp_idx].fw_grp_id =
5665 			le32_to_cpu(resp->ring_group_id);
5666 	}
5667 	hwrm_req_drop(bp, req);
5668 	return rc;
5669 }
5670 
5671 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5672 {
5673 	struct hwrm_ring_grp_free_input *req;
5674 	u16 i;
5675 
5676 	if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5677 		return;
5678 
5679 	if (hwrm_req_init(bp, req, HWRM_RING_GRP_FREE))
5680 		return;
5681 
5682 	hwrm_req_hold(bp, req);
5683 	for (i = 0; i < bp->cp_nr_rings; i++) {
5684 		if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5685 			continue;
5686 		req->ring_group_id =
5687 			cpu_to_le32(bp->grp_info[i].fw_grp_id);
5688 
5689 		hwrm_req_send(bp, req);
5690 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5691 	}
5692 	hwrm_req_drop(bp, req);
5693 }
5694 
5695 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5696 				    struct bnxt_ring_struct *ring,
5697 				    u32 ring_type, u32 map_index)
5698 {
5699 	struct hwrm_ring_alloc_output *resp;
5700 	struct hwrm_ring_alloc_input *req;
5701 	struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5702 	struct bnxt_ring_grp_info *grp_info;
5703 	int rc, err = 0;
5704 	u16 ring_id;
5705 
5706 	rc = hwrm_req_init(bp, req, HWRM_RING_ALLOC);
5707 	if (rc)
5708 		goto exit;
5709 
5710 	req->enables = 0;
5711 	if (rmem->nr_pages > 1) {
5712 		req->page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5713 		/* Page size is in log2 units */
5714 		req->page_size = BNXT_PAGE_SHIFT;
5715 		req->page_tbl_depth = 1;
5716 	} else {
5717 		req->page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5718 	}
5719 	req->fbo = 0;
5720 	/* Association of ring index with doorbell index and MSIX number */
5721 	req->logical_id = cpu_to_le16(map_index);
5722 
5723 	switch (ring_type) {
5724 	case HWRM_RING_ALLOC_TX: {
5725 		struct bnxt_tx_ring_info *txr;
5726 
5727 		txr = container_of(ring, struct bnxt_tx_ring_info,
5728 				   tx_ring_struct);
5729 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5730 		/* Association of transmit ring with completion ring */
5731 		grp_info = &bp->grp_info[ring->grp_idx];
5732 		req->cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5733 		req->length = cpu_to_le32(bp->tx_ring_mask + 1);
5734 		req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5735 		req->queue_id = cpu_to_le16(ring->queue_id);
5736 		break;
5737 	}
5738 	case HWRM_RING_ALLOC_RX:
5739 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5740 		req->length = cpu_to_le32(bp->rx_ring_mask + 1);
5741 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5742 			u16 flags = 0;
5743 
5744 			/* Association of rx ring with stats context */
5745 			grp_info = &bp->grp_info[ring->grp_idx];
5746 			req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5747 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5748 			req->enables |= cpu_to_le32(
5749 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5750 			if (NET_IP_ALIGN == 2)
5751 				flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5752 			req->flags = cpu_to_le16(flags);
5753 		}
5754 		break;
5755 	case HWRM_RING_ALLOC_AGG:
5756 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5757 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5758 			/* Association of agg ring with rx ring */
5759 			grp_info = &bp->grp_info[ring->grp_idx];
5760 			req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5761 			req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5762 			req->stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5763 			req->enables |= cpu_to_le32(
5764 				RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5765 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5766 		} else {
5767 			req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5768 		}
5769 		req->length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5770 		break;
5771 	case HWRM_RING_ALLOC_CMPL:
5772 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5773 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5774 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5775 			/* Association of cp ring with nq */
5776 			grp_info = &bp->grp_info[map_index];
5777 			req->nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5778 			req->cq_handle = cpu_to_le64(ring->handle);
5779 			req->enables |= cpu_to_le32(
5780 				RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5781 		} else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5782 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5783 		}
5784 		break;
5785 	case HWRM_RING_ALLOC_NQ:
5786 		req->ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5787 		req->length = cpu_to_le32(bp->cp_ring_mask + 1);
5788 		if (bp->flags & BNXT_FLAG_USING_MSIX)
5789 			req->int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5790 		break;
5791 	default:
5792 		netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5793 			   ring_type);
5794 		return -1;
5795 	}
5796 
5797 	resp = hwrm_req_hold(bp, req);
5798 	rc = hwrm_req_send(bp, req);
5799 	err = le16_to_cpu(resp->error_code);
5800 	ring_id = le16_to_cpu(resp->ring_id);
5801 	hwrm_req_drop(bp, req);
5802 
5803 exit:
5804 	if (rc || err) {
5805 		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5806 			   ring_type, rc, err);
5807 		return -EIO;
5808 	}
5809 	ring->fw_ring_id = ring_id;
5810 	return rc;
5811 }
5812 
5813 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5814 {
5815 	int rc;
5816 
5817 	if (BNXT_PF(bp)) {
5818 		struct hwrm_func_cfg_input *req;
5819 
5820 		rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
5821 		if (rc)
5822 			return rc;
5823 
5824 		req->fid = cpu_to_le16(0xffff);
5825 		req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5826 		req->async_event_cr = cpu_to_le16(idx);
5827 		return hwrm_req_send(bp, req);
5828 	} else {
5829 		struct hwrm_func_vf_cfg_input *req;
5830 
5831 		rc = hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG);
5832 		if (rc)
5833 			return rc;
5834 
5835 		req->enables =
5836 			cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5837 		req->async_event_cr = cpu_to_le16(idx);
5838 		return hwrm_req_send(bp, req);
5839 	}
5840 }
5841 
5842 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5843 			u32 map_idx, u32 xid)
5844 {
5845 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5846 		if (BNXT_PF(bp))
5847 			db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5848 		else
5849 			db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5850 		switch (ring_type) {
5851 		case HWRM_RING_ALLOC_TX:
5852 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5853 			break;
5854 		case HWRM_RING_ALLOC_RX:
5855 		case HWRM_RING_ALLOC_AGG:
5856 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5857 			break;
5858 		case HWRM_RING_ALLOC_CMPL:
5859 			db->db_key64 = DBR_PATH_L2;
5860 			break;
5861 		case HWRM_RING_ALLOC_NQ:
5862 			db->db_key64 = DBR_PATH_L2;
5863 			break;
5864 		}
5865 		db->db_key64 |= (u64)xid << DBR_XID_SFT;
5866 	} else {
5867 		db->doorbell = bp->bar1 + map_idx * 0x80;
5868 		switch (ring_type) {
5869 		case HWRM_RING_ALLOC_TX:
5870 			db->db_key32 = DB_KEY_TX;
5871 			break;
5872 		case HWRM_RING_ALLOC_RX:
5873 		case HWRM_RING_ALLOC_AGG:
5874 			db->db_key32 = DB_KEY_RX;
5875 			break;
5876 		case HWRM_RING_ALLOC_CMPL:
5877 			db->db_key32 = DB_KEY_CP;
5878 			break;
5879 		}
5880 	}
5881 }
5882 
5883 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5884 {
5885 	bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5886 	int i, rc = 0;
5887 	u32 type;
5888 
5889 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5890 		type = HWRM_RING_ALLOC_NQ;
5891 	else
5892 		type = HWRM_RING_ALLOC_CMPL;
5893 	for (i = 0; i < bp->cp_nr_rings; i++) {
5894 		struct bnxt_napi *bnapi = bp->bnapi[i];
5895 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5896 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5897 		u32 map_idx = ring->map_idx;
5898 		unsigned int vector;
5899 
5900 		vector = bp->irq_tbl[map_idx].vector;
5901 		disable_irq_nosync(vector);
5902 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5903 		if (rc) {
5904 			enable_irq(vector);
5905 			goto err_out;
5906 		}
5907 		bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5908 		bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5909 		enable_irq(vector);
5910 		bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5911 
5912 		if (!i) {
5913 			rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5914 			if (rc)
5915 				netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5916 		}
5917 	}
5918 
5919 	type = HWRM_RING_ALLOC_TX;
5920 	for (i = 0; i < bp->tx_nr_rings; i++) {
5921 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5922 		struct bnxt_ring_struct *ring;
5923 		u32 map_idx;
5924 
5925 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5926 			struct bnxt_napi *bnapi = txr->bnapi;
5927 			struct bnxt_cp_ring_info *cpr, *cpr2;
5928 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5929 
5930 			cpr = &bnapi->cp_ring;
5931 			cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5932 			ring = &cpr2->cp_ring_struct;
5933 			ring->handle = BNXT_TX_HDL;
5934 			map_idx = bnapi->index;
5935 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5936 			if (rc)
5937 				goto err_out;
5938 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5939 				    ring->fw_ring_id);
5940 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5941 		}
5942 		ring = &txr->tx_ring_struct;
5943 		map_idx = i;
5944 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5945 		if (rc)
5946 			goto err_out;
5947 		bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5948 	}
5949 
5950 	type = HWRM_RING_ALLOC_RX;
5951 	for (i = 0; i < bp->rx_nr_rings; i++) {
5952 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5953 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5954 		struct bnxt_napi *bnapi = rxr->bnapi;
5955 		u32 map_idx = bnapi->index;
5956 
5957 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5958 		if (rc)
5959 			goto err_out;
5960 		bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5961 		/* If we have agg rings, post agg buffers first. */
5962 		if (!agg_rings)
5963 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5964 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5965 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5966 			struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5967 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5968 			struct bnxt_cp_ring_info *cpr2;
5969 
5970 			cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5971 			ring = &cpr2->cp_ring_struct;
5972 			ring->handle = BNXT_RX_HDL;
5973 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5974 			if (rc)
5975 				goto err_out;
5976 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5977 				    ring->fw_ring_id);
5978 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5979 		}
5980 	}
5981 
5982 	if (agg_rings) {
5983 		type = HWRM_RING_ALLOC_AGG;
5984 		for (i = 0; i < bp->rx_nr_rings; i++) {
5985 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5986 			struct bnxt_ring_struct *ring =
5987 						&rxr->rx_agg_ring_struct;
5988 			u32 grp_idx = ring->grp_idx;
5989 			u32 map_idx = grp_idx + bp->rx_nr_rings;
5990 
5991 			rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5992 			if (rc)
5993 				goto err_out;
5994 
5995 			bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5996 				    ring->fw_ring_id);
5997 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5998 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5999 			bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
6000 		}
6001 	}
6002 err_out:
6003 	return rc;
6004 }
6005 
6006 static int hwrm_ring_free_send_msg(struct bnxt *bp,
6007 				   struct bnxt_ring_struct *ring,
6008 				   u32 ring_type, int cmpl_ring_id)
6009 {
6010 	struct hwrm_ring_free_output *resp;
6011 	struct hwrm_ring_free_input *req;
6012 	u16 error_code = 0;
6013 	int rc;
6014 
6015 	if (BNXT_NO_FW_ACCESS(bp))
6016 		return 0;
6017 
6018 	rc = hwrm_req_init(bp, req, HWRM_RING_FREE);
6019 	if (rc)
6020 		goto exit;
6021 
6022 	req->cmpl_ring = cpu_to_le16(cmpl_ring_id);
6023 	req->ring_type = ring_type;
6024 	req->ring_id = cpu_to_le16(ring->fw_ring_id);
6025 
6026 	resp = hwrm_req_hold(bp, req);
6027 	rc = hwrm_req_send(bp, req);
6028 	error_code = le16_to_cpu(resp->error_code);
6029 	hwrm_req_drop(bp, req);
6030 exit:
6031 	if (rc || error_code) {
6032 		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
6033 			   ring_type, rc, error_code);
6034 		return -EIO;
6035 	}
6036 	return 0;
6037 }
6038 
6039 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
6040 {
6041 	u32 type;
6042 	int i;
6043 
6044 	if (!bp->bnapi)
6045 		return;
6046 
6047 	for (i = 0; i < bp->tx_nr_rings; i++) {
6048 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
6049 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
6050 
6051 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6052 			u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
6053 
6054 			hwrm_ring_free_send_msg(bp, ring,
6055 						RING_FREE_REQ_RING_TYPE_TX,
6056 						close_path ? cmpl_ring_id :
6057 						INVALID_HW_RING_ID);
6058 			ring->fw_ring_id = INVALID_HW_RING_ID;
6059 		}
6060 	}
6061 
6062 	for (i = 0; i < bp->rx_nr_rings; i++) {
6063 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6064 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
6065 		u32 grp_idx = rxr->bnapi->index;
6066 
6067 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6068 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6069 
6070 			hwrm_ring_free_send_msg(bp, ring,
6071 						RING_FREE_REQ_RING_TYPE_RX,
6072 						close_path ? cmpl_ring_id :
6073 						INVALID_HW_RING_ID);
6074 			ring->fw_ring_id = INVALID_HW_RING_ID;
6075 			bp->grp_info[grp_idx].rx_fw_ring_id =
6076 				INVALID_HW_RING_ID;
6077 		}
6078 	}
6079 
6080 	if (bp->flags & BNXT_FLAG_CHIP_P5)
6081 		type = RING_FREE_REQ_RING_TYPE_RX_AGG;
6082 	else
6083 		type = RING_FREE_REQ_RING_TYPE_RX;
6084 	for (i = 0; i < bp->rx_nr_rings; i++) {
6085 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
6086 		struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
6087 		u32 grp_idx = rxr->bnapi->index;
6088 
6089 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6090 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
6091 
6092 			hwrm_ring_free_send_msg(bp, ring, type,
6093 						close_path ? cmpl_ring_id :
6094 						INVALID_HW_RING_ID);
6095 			ring->fw_ring_id = INVALID_HW_RING_ID;
6096 			bp->grp_info[grp_idx].agg_fw_ring_id =
6097 				INVALID_HW_RING_ID;
6098 		}
6099 	}
6100 
6101 	/* The completion rings are about to be freed.  After that the
6102 	 * IRQ doorbell will not work anymore.  So we need to disable
6103 	 * IRQ here.
6104 	 */
6105 	bnxt_disable_int_sync(bp);
6106 
6107 	if (bp->flags & BNXT_FLAG_CHIP_P5)
6108 		type = RING_FREE_REQ_RING_TYPE_NQ;
6109 	else
6110 		type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
6111 	for (i = 0; i < bp->cp_nr_rings; i++) {
6112 		struct bnxt_napi *bnapi = bp->bnapi[i];
6113 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6114 		struct bnxt_ring_struct *ring;
6115 		int j;
6116 
6117 		for (j = 0; j < 2; j++) {
6118 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
6119 
6120 			if (cpr2) {
6121 				ring = &cpr2->cp_ring_struct;
6122 				if (ring->fw_ring_id == INVALID_HW_RING_ID)
6123 					continue;
6124 				hwrm_ring_free_send_msg(bp, ring,
6125 					RING_FREE_REQ_RING_TYPE_L2_CMPL,
6126 					INVALID_HW_RING_ID);
6127 				ring->fw_ring_id = INVALID_HW_RING_ID;
6128 			}
6129 		}
6130 		ring = &cpr->cp_ring_struct;
6131 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
6132 			hwrm_ring_free_send_msg(bp, ring, type,
6133 						INVALID_HW_RING_ID);
6134 			ring->fw_ring_id = INVALID_HW_RING_ID;
6135 			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
6136 		}
6137 	}
6138 }
6139 
6140 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
6141 			   bool shared);
6142 
6143 static int bnxt_hwrm_get_rings(struct bnxt *bp)
6144 {
6145 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6146 	struct hwrm_func_qcfg_output *resp;
6147 	struct hwrm_func_qcfg_input *req;
6148 	int rc;
6149 
6150 	if (bp->hwrm_spec_code < 0x10601)
6151 		return 0;
6152 
6153 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6154 	if (rc)
6155 		return rc;
6156 
6157 	req->fid = cpu_to_le16(0xffff);
6158 	resp = hwrm_req_hold(bp, req);
6159 	rc = hwrm_req_send(bp, req);
6160 	if (rc) {
6161 		hwrm_req_drop(bp, req);
6162 		return rc;
6163 	}
6164 
6165 	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6166 	if (BNXT_NEW_RM(bp)) {
6167 		u16 cp, stats;
6168 
6169 		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
6170 		hw_resc->resv_hw_ring_grps =
6171 			le32_to_cpu(resp->alloc_hw_ring_grps);
6172 		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
6173 		cp = le16_to_cpu(resp->alloc_cmpl_rings);
6174 		stats = le16_to_cpu(resp->alloc_stat_ctx);
6175 		hw_resc->resv_irqs = cp;
6176 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6177 			int rx = hw_resc->resv_rx_rings;
6178 			int tx = hw_resc->resv_tx_rings;
6179 
6180 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
6181 				rx >>= 1;
6182 			if (cp < (rx + tx)) {
6183 				bnxt_trim_rings(bp, &rx, &tx, cp, false);
6184 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
6185 					rx <<= 1;
6186 				hw_resc->resv_rx_rings = rx;
6187 				hw_resc->resv_tx_rings = tx;
6188 			}
6189 			hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
6190 			hw_resc->resv_hw_ring_grps = rx;
6191 		}
6192 		hw_resc->resv_cp_rings = cp;
6193 		hw_resc->resv_stat_ctxs = stats;
6194 	}
6195 	hwrm_req_drop(bp, req);
6196 	return 0;
6197 }
6198 
6199 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
6200 {
6201 	struct hwrm_func_qcfg_output *resp;
6202 	struct hwrm_func_qcfg_input *req;
6203 	int rc;
6204 
6205 	if (bp->hwrm_spec_code < 0x10601)
6206 		return 0;
6207 
6208 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6209 	if (rc)
6210 		return rc;
6211 
6212 	req->fid = cpu_to_le16(fid);
6213 	resp = hwrm_req_hold(bp, req);
6214 	rc = hwrm_req_send(bp, req);
6215 	if (!rc)
6216 		*tx_rings = le16_to_cpu(resp->alloc_tx_rings);
6217 
6218 	hwrm_req_drop(bp, req);
6219 	return rc;
6220 }
6221 
6222 static bool bnxt_rfs_supported(struct bnxt *bp);
6223 
6224 static struct hwrm_func_cfg_input *
6225 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6226 			     int ring_grps, int cp_rings, int stats, int vnics)
6227 {
6228 	struct hwrm_func_cfg_input *req;
6229 	u32 enables = 0;
6230 
6231 	if (hwrm_req_init(bp, req, HWRM_FUNC_CFG))
6232 		return NULL;
6233 
6234 	req->fid = cpu_to_le16(0xffff);
6235 	enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6236 	req->num_tx_rings = cpu_to_le16(tx_rings);
6237 	if (BNXT_NEW_RM(bp)) {
6238 		enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
6239 		enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6240 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6241 			enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
6242 			enables |= tx_rings + ring_grps ?
6243 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6244 			enables |= rx_rings ?
6245 				FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6246 		} else {
6247 			enables |= cp_rings ?
6248 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6249 			enables |= ring_grps ?
6250 				   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6251 				   FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6252 		}
6253 		enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6254 
6255 		req->num_rx_rings = cpu_to_le16(rx_rings);
6256 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6257 			req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6258 			req->num_msix = cpu_to_le16(cp_rings);
6259 			req->num_rsscos_ctxs =
6260 				cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6261 		} else {
6262 			req->num_cmpl_rings = cpu_to_le16(cp_rings);
6263 			req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6264 			req->num_rsscos_ctxs = cpu_to_le16(1);
6265 			if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6266 			    bnxt_rfs_supported(bp))
6267 				req->num_rsscos_ctxs =
6268 					cpu_to_le16(ring_grps + 1);
6269 		}
6270 		req->num_stat_ctxs = cpu_to_le16(stats);
6271 		req->num_vnics = cpu_to_le16(vnics);
6272 	}
6273 	req->enables = cpu_to_le32(enables);
6274 	return req;
6275 }
6276 
6277 static struct hwrm_func_vf_cfg_input *
6278 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6279 			     int ring_grps, int cp_rings, int stats, int vnics)
6280 {
6281 	struct hwrm_func_vf_cfg_input *req;
6282 	u32 enables = 0;
6283 
6284 	if (hwrm_req_init(bp, req, HWRM_FUNC_VF_CFG))
6285 		return NULL;
6286 
6287 	enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6288 	enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6289 			      FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6290 	enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6291 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6292 		enables |= tx_rings + ring_grps ?
6293 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6294 	} else {
6295 		enables |= cp_rings ?
6296 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6297 		enables |= ring_grps ?
6298 			   FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6299 	}
6300 	enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6301 	enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6302 
6303 	req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6304 	req->num_tx_rings = cpu_to_le16(tx_rings);
6305 	req->num_rx_rings = cpu_to_le16(rx_rings);
6306 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6307 		req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6308 		req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6309 	} else {
6310 		req->num_cmpl_rings = cpu_to_le16(cp_rings);
6311 		req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6312 		req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6313 	}
6314 	req->num_stat_ctxs = cpu_to_le16(stats);
6315 	req->num_vnics = cpu_to_le16(vnics);
6316 
6317 	req->enables = cpu_to_le32(enables);
6318 	return req;
6319 }
6320 
6321 static int
6322 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6323 			   int ring_grps, int cp_rings, int stats, int vnics)
6324 {
6325 	struct hwrm_func_cfg_input *req;
6326 	int rc;
6327 
6328 	req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6329 					   cp_rings, stats, vnics);
6330 	if (!req)
6331 		return -ENOMEM;
6332 
6333 	if (!req->enables) {
6334 		hwrm_req_drop(bp, req);
6335 		return 0;
6336 	}
6337 
6338 	rc = hwrm_req_send(bp, req);
6339 	if (rc)
6340 		return rc;
6341 
6342 	if (bp->hwrm_spec_code < 0x10601)
6343 		bp->hw_resc.resv_tx_rings = tx_rings;
6344 
6345 	return bnxt_hwrm_get_rings(bp);
6346 }
6347 
6348 static int
6349 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6350 			   int ring_grps, int cp_rings, int stats, int vnics)
6351 {
6352 	struct hwrm_func_vf_cfg_input *req;
6353 	int rc;
6354 
6355 	if (!BNXT_NEW_RM(bp)) {
6356 		bp->hw_resc.resv_tx_rings = tx_rings;
6357 		return 0;
6358 	}
6359 
6360 	req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6361 					   cp_rings, stats, vnics);
6362 	if (!req)
6363 		return -ENOMEM;
6364 
6365 	rc = hwrm_req_send(bp, req);
6366 	if (rc)
6367 		return rc;
6368 
6369 	return bnxt_hwrm_get_rings(bp);
6370 }
6371 
6372 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6373 				   int cp, int stat, int vnic)
6374 {
6375 	if (BNXT_PF(bp))
6376 		return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6377 						  vnic);
6378 	else
6379 		return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6380 						  vnic);
6381 }
6382 
6383 int bnxt_nq_rings_in_use(struct bnxt *bp)
6384 {
6385 	int cp = bp->cp_nr_rings;
6386 	int ulp_msix, ulp_base;
6387 
6388 	ulp_msix = bnxt_get_ulp_msix_num(bp);
6389 	if (ulp_msix) {
6390 		ulp_base = bnxt_get_ulp_msix_base(bp);
6391 		cp += ulp_msix;
6392 		if ((ulp_base + ulp_msix) > cp)
6393 			cp = ulp_base + ulp_msix;
6394 	}
6395 	return cp;
6396 }
6397 
6398 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6399 {
6400 	int cp;
6401 
6402 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6403 		return bnxt_nq_rings_in_use(bp);
6404 
6405 	cp = bp->tx_nr_rings + bp->rx_nr_rings;
6406 	return cp;
6407 }
6408 
6409 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6410 {
6411 	int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6412 	int cp = bp->cp_nr_rings;
6413 
6414 	if (!ulp_stat)
6415 		return cp;
6416 
6417 	if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6418 		return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6419 
6420 	return cp + ulp_stat;
6421 }
6422 
6423 /* Check if a default RSS map needs to be setup.  This function is only
6424  * used on older firmware that does not require reserving RX rings.
6425  */
6426 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6427 {
6428 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6429 
6430 	/* The RSS map is valid for RX rings set to resv_rx_rings */
6431 	if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6432 		hw_resc->resv_rx_rings = bp->rx_nr_rings;
6433 		if (!netif_is_rxfh_configured(bp->dev))
6434 			bnxt_set_dflt_rss_indir_tbl(bp);
6435 	}
6436 }
6437 
6438 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6439 {
6440 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6441 	int cp = bnxt_cp_rings_in_use(bp);
6442 	int nq = bnxt_nq_rings_in_use(bp);
6443 	int rx = bp->rx_nr_rings, stat;
6444 	int vnic = 1, grp = rx;
6445 
6446 	if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6447 	    bp->hwrm_spec_code >= 0x10601)
6448 		return true;
6449 
6450 	/* Old firmware does not need RX ring reservations but we still
6451 	 * need to setup a default RSS map when needed.  With new firmware
6452 	 * we go through RX ring reservations first and then set up the
6453 	 * RSS map for the successfully reserved RX rings when needed.
6454 	 */
6455 	if (!BNXT_NEW_RM(bp)) {
6456 		bnxt_check_rss_tbl_no_rmgr(bp);
6457 		return false;
6458 	}
6459 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6460 		vnic = rx + 1;
6461 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6462 		rx <<= 1;
6463 	stat = bnxt_get_func_stat_ctxs(bp);
6464 	if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6465 	    hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6466 	    (hw_resc->resv_hw_ring_grps != grp &&
6467 	     !(bp->flags & BNXT_FLAG_CHIP_P5)))
6468 		return true;
6469 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6470 	    hw_resc->resv_irqs != nq)
6471 		return true;
6472 	return false;
6473 }
6474 
6475 static int __bnxt_reserve_rings(struct bnxt *bp)
6476 {
6477 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6478 	int cp = bnxt_nq_rings_in_use(bp);
6479 	int tx = bp->tx_nr_rings;
6480 	int rx = bp->rx_nr_rings;
6481 	int grp, rx_rings, rc;
6482 	int vnic = 1, stat;
6483 	bool sh = false;
6484 
6485 	if (!bnxt_need_reserve_rings(bp))
6486 		return 0;
6487 
6488 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6489 		sh = true;
6490 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6491 		vnic = rx + 1;
6492 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6493 		rx <<= 1;
6494 	grp = bp->rx_nr_rings;
6495 	stat = bnxt_get_func_stat_ctxs(bp);
6496 
6497 	rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6498 	if (rc)
6499 		return rc;
6500 
6501 	tx = hw_resc->resv_tx_rings;
6502 	if (BNXT_NEW_RM(bp)) {
6503 		rx = hw_resc->resv_rx_rings;
6504 		cp = hw_resc->resv_irqs;
6505 		grp = hw_resc->resv_hw_ring_grps;
6506 		vnic = hw_resc->resv_vnics;
6507 		stat = hw_resc->resv_stat_ctxs;
6508 	}
6509 
6510 	rx_rings = rx;
6511 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6512 		if (rx >= 2) {
6513 			rx_rings = rx >> 1;
6514 		} else {
6515 			if (netif_running(bp->dev))
6516 				return -ENOMEM;
6517 
6518 			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6519 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6520 			bp->dev->hw_features &= ~NETIF_F_LRO;
6521 			bp->dev->features &= ~NETIF_F_LRO;
6522 			bnxt_set_ring_params(bp);
6523 		}
6524 	}
6525 	rx_rings = min_t(int, rx_rings, grp);
6526 	cp = min_t(int, cp, bp->cp_nr_rings);
6527 	if (stat > bnxt_get_ulp_stat_ctxs(bp))
6528 		stat -= bnxt_get_ulp_stat_ctxs(bp);
6529 	cp = min_t(int, cp, stat);
6530 	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6531 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6532 		rx = rx_rings << 1;
6533 	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6534 	bp->tx_nr_rings = tx;
6535 
6536 	/* If we cannot reserve all the RX rings, reset the RSS map only
6537 	 * if absolutely necessary
6538 	 */
6539 	if (rx_rings != bp->rx_nr_rings) {
6540 		netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6541 			    rx_rings, bp->rx_nr_rings);
6542 		if (netif_is_rxfh_configured(bp->dev) &&
6543 		    (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6544 		     bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6545 		     bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6546 			netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6547 			bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6548 		}
6549 	}
6550 	bp->rx_nr_rings = rx_rings;
6551 	bp->cp_nr_rings = cp;
6552 
6553 	if (!tx || !rx || !cp || !grp || !vnic || !stat)
6554 		return -ENOMEM;
6555 
6556 	if (!netif_is_rxfh_configured(bp->dev))
6557 		bnxt_set_dflt_rss_indir_tbl(bp);
6558 
6559 	return rc;
6560 }
6561 
6562 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6563 				    int ring_grps, int cp_rings, int stats,
6564 				    int vnics)
6565 {
6566 	struct hwrm_func_vf_cfg_input *req;
6567 	u32 flags;
6568 
6569 	if (!BNXT_NEW_RM(bp))
6570 		return 0;
6571 
6572 	req = __bnxt_hwrm_reserve_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6573 					   cp_rings, stats, vnics);
6574 	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6575 		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6576 		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6577 		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6578 		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6579 		FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6580 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6581 		flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6582 
6583 	req->flags = cpu_to_le32(flags);
6584 	return hwrm_req_send_silent(bp, req);
6585 }
6586 
6587 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6588 				    int ring_grps, int cp_rings, int stats,
6589 				    int vnics)
6590 {
6591 	struct hwrm_func_cfg_input *req;
6592 	u32 flags;
6593 
6594 	req = __bnxt_hwrm_reserve_pf_rings(bp, tx_rings, rx_rings, ring_grps,
6595 					   cp_rings, stats, vnics);
6596 	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6597 	if (BNXT_NEW_RM(bp)) {
6598 		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6599 			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6600 			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6601 			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6602 		if (bp->flags & BNXT_FLAG_CHIP_P5)
6603 			flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6604 				 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6605 		else
6606 			flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6607 	}
6608 
6609 	req->flags = cpu_to_le32(flags);
6610 	return hwrm_req_send_silent(bp, req);
6611 }
6612 
6613 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6614 				 int ring_grps, int cp_rings, int stats,
6615 				 int vnics)
6616 {
6617 	if (bp->hwrm_spec_code < 0x10801)
6618 		return 0;
6619 
6620 	if (BNXT_PF(bp))
6621 		return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6622 						ring_grps, cp_rings, stats,
6623 						vnics);
6624 
6625 	return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6626 					cp_rings, stats, vnics);
6627 }
6628 
6629 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6630 {
6631 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6632 	struct hwrm_ring_aggint_qcaps_output *resp;
6633 	struct hwrm_ring_aggint_qcaps_input *req;
6634 	int rc;
6635 
6636 	coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6637 	coal_cap->num_cmpl_dma_aggr_max = 63;
6638 	coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6639 	coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6640 	coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6641 	coal_cap->int_lat_tmr_min_max = 65535;
6642 	coal_cap->int_lat_tmr_max_max = 65535;
6643 	coal_cap->num_cmpl_aggr_int_max = 65535;
6644 	coal_cap->timer_units = 80;
6645 
6646 	if (bp->hwrm_spec_code < 0x10902)
6647 		return;
6648 
6649 	if (hwrm_req_init(bp, req, HWRM_RING_AGGINT_QCAPS))
6650 		return;
6651 
6652 	resp = hwrm_req_hold(bp, req);
6653 	rc = hwrm_req_send_silent(bp, req);
6654 	if (!rc) {
6655 		coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6656 		coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6657 		coal_cap->num_cmpl_dma_aggr_max =
6658 			le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6659 		coal_cap->num_cmpl_dma_aggr_during_int_max =
6660 			le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6661 		coal_cap->cmpl_aggr_dma_tmr_max =
6662 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6663 		coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6664 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6665 		coal_cap->int_lat_tmr_min_max =
6666 			le16_to_cpu(resp->int_lat_tmr_min_max);
6667 		coal_cap->int_lat_tmr_max_max =
6668 			le16_to_cpu(resp->int_lat_tmr_max_max);
6669 		coal_cap->num_cmpl_aggr_int_max =
6670 			le16_to_cpu(resp->num_cmpl_aggr_int_max);
6671 		coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6672 	}
6673 	hwrm_req_drop(bp, req);
6674 }
6675 
6676 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6677 {
6678 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6679 
6680 	return usec * 1000 / coal_cap->timer_units;
6681 }
6682 
6683 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6684 	struct bnxt_coal *hw_coal,
6685 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6686 {
6687 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6688 	u16 val, tmr, max, flags = hw_coal->flags;
6689 	u32 cmpl_params = coal_cap->cmpl_params;
6690 
6691 	max = hw_coal->bufs_per_record * 128;
6692 	if (hw_coal->budget)
6693 		max = hw_coal->bufs_per_record * hw_coal->budget;
6694 	max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6695 
6696 	val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6697 	req->num_cmpl_aggr_int = cpu_to_le16(val);
6698 
6699 	val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6700 	req->num_cmpl_dma_aggr = cpu_to_le16(val);
6701 
6702 	val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6703 		      coal_cap->num_cmpl_dma_aggr_during_int_max);
6704 	req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6705 
6706 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6707 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6708 	req->int_lat_tmr_max = cpu_to_le16(tmr);
6709 
6710 	/* min timer set to 1/2 of interrupt timer */
6711 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6712 		val = tmr / 2;
6713 		val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6714 		req->int_lat_tmr_min = cpu_to_le16(val);
6715 		req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6716 	}
6717 
6718 	/* buf timer set to 1/4 of interrupt timer */
6719 	val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6720 	req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6721 
6722 	if (cmpl_params &
6723 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6724 		tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6725 		val = clamp_t(u16, tmr, 1,
6726 			      coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6727 		req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6728 		req->enables |=
6729 			cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6730 	}
6731 
6732 	if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6733 	    hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6734 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6735 	req->flags = cpu_to_le16(flags);
6736 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6737 }
6738 
6739 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6740 				   struct bnxt_coal *hw_coal)
6741 {
6742 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req;
6743 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6744 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6745 	u32 nq_params = coal_cap->nq_params;
6746 	u16 tmr;
6747 	int rc;
6748 
6749 	if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6750 		return 0;
6751 
6752 	rc = hwrm_req_init(bp, req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6753 	if (rc)
6754 		return rc;
6755 
6756 	req->ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6757 	req->flags =
6758 		cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6759 
6760 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6761 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6762 	req->int_lat_tmr_min = cpu_to_le16(tmr);
6763 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6764 	return hwrm_req_send(bp, req);
6765 }
6766 
6767 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6768 {
6769 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx;
6770 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6771 	struct bnxt_coal coal;
6772 	int rc;
6773 
6774 	/* Tick values in micro seconds.
6775 	 * 1 coal_buf x bufs_per_record = 1 completion record.
6776 	 */
6777 	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6778 
6779 	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6780 	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6781 
6782 	if (!bnapi->rx_ring)
6783 		return -ENODEV;
6784 
6785 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6786 	if (rc)
6787 		return rc;
6788 
6789 	bnxt_hwrm_set_coal_params(bp, &coal, req_rx);
6790 
6791 	req_rx->ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6792 
6793 	return hwrm_req_send(bp, req_rx);
6794 }
6795 
6796 int bnxt_hwrm_set_coal(struct bnxt *bp)
6797 {
6798 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req_rx, *req_tx,
6799 							   *req;
6800 	int i, rc;
6801 
6802 	rc = hwrm_req_init(bp, req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6803 	if (rc)
6804 		return rc;
6805 
6806 	rc = hwrm_req_init(bp, req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS);
6807 	if (rc) {
6808 		hwrm_req_drop(bp, req_rx);
6809 		return rc;
6810 	}
6811 
6812 	bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, req_rx);
6813 	bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, req_tx);
6814 
6815 	hwrm_req_hold(bp, req_rx);
6816 	hwrm_req_hold(bp, req_tx);
6817 	for (i = 0; i < bp->cp_nr_rings; i++) {
6818 		struct bnxt_napi *bnapi = bp->bnapi[i];
6819 		struct bnxt_coal *hw_coal;
6820 		u16 ring_id;
6821 
6822 		req = req_rx;
6823 		if (!bnapi->rx_ring) {
6824 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6825 			req = req_tx;
6826 		} else {
6827 			ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6828 		}
6829 		req->ring_id = cpu_to_le16(ring_id);
6830 
6831 		rc = hwrm_req_send(bp, req);
6832 		if (rc)
6833 			break;
6834 
6835 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6836 			continue;
6837 
6838 		if (bnapi->rx_ring && bnapi->tx_ring) {
6839 			req = req_tx;
6840 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6841 			req->ring_id = cpu_to_le16(ring_id);
6842 			rc = hwrm_req_send(bp, req);
6843 			if (rc)
6844 				break;
6845 		}
6846 		if (bnapi->rx_ring)
6847 			hw_coal = &bp->rx_coal;
6848 		else
6849 			hw_coal = &bp->tx_coal;
6850 		__bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6851 	}
6852 	hwrm_req_drop(bp, req_rx);
6853 	hwrm_req_drop(bp, req_tx);
6854 	return rc;
6855 }
6856 
6857 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6858 {
6859 	struct hwrm_stat_ctx_clr_stats_input *req0 = NULL;
6860 	struct hwrm_stat_ctx_free_input *req;
6861 	int i;
6862 
6863 	if (!bp->bnapi)
6864 		return;
6865 
6866 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6867 		return;
6868 
6869 	if (hwrm_req_init(bp, req, HWRM_STAT_CTX_FREE))
6870 		return;
6871 	if (BNXT_FW_MAJ(bp) <= 20) {
6872 		if (hwrm_req_init(bp, req0, HWRM_STAT_CTX_CLR_STATS)) {
6873 			hwrm_req_drop(bp, req);
6874 			return;
6875 		}
6876 		hwrm_req_hold(bp, req0);
6877 	}
6878 	hwrm_req_hold(bp, req);
6879 	for (i = 0; i < bp->cp_nr_rings; i++) {
6880 		struct bnxt_napi *bnapi = bp->bnapi[i];
6881 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6882 
6883 		if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6884 			req->stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6885 			if (req0) {
6886 				req0->stat_ctx_id = req->stat_ctx_id;
6887 				hwrm_req_send(bp, req0);
6888 			}
6889 			hwrm_req_send(bp, req);
6890 
6891 			cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6892 		}
6893 	}
6894 	hwrm_req_drop(bp, req);
6895 	if (req0)
6896 		hwrm_req_drop(bp, req0);
6897 }
6898 
6899 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6900 {
6901 	struct hwrm_stat_ctx_alloc_output *resp;
6902 	struct hwrm_stat_ctx_alloc_input *req;
6903 	int rc, i;
6904 
6905 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6906 		return 0;
6907 
6908 	rc = hwrm_req_init(bp, req, HWRM_STAT_CTX_ALLOC);
6909 	if (rc)
6910 		return rc;
6911 
6912 	req->stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6913 	req->update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6914 
6915 	resp = hwrm_req_hold(bp, req);
6916 	for (i = 0; i < bp->cp_nr_rings; i++) {
6917 		struct bnxt_napi *bnapi = bp->bnapi[i];
6918 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6919 
6920 		req->stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6921 
6922 		rc = hwrm_req_send(bp, req);
6923 		if (rc)
6924 			break;
6925 
6926 		cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6927 
6928 		bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6929 	}
6930 	hwrm_req_drop(bp, req);
6931 	return rc;
6932 }
6933 
6934 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6935 {
6936 	struct hwrm_func_qcfg_output *resp;
6937 	struct hwrm_func_qcfg_input *req;
6938 	u32 min_db_offset = 0;
6939 	u16 flags;
6940 	int rc;
6941 
6942 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCFG);
6943 	if (rc)
6944 		return rc;
6945 
6946 	req->fid = cpu_to_le16(0xffff);
6947 	resp = hwrm_req_hold(bp, req);
6948 	rc = hwrm_req_send(bp, req);
6949 	if (rc)
6950 		goto func_qcfg_exit;
6951 
6952 #ifdef CONFIG_BNXT_SRIOV
6953 	if (BNXT_VF(bp)) {
6954 		struct bnxt_vf_info *vf = &bp->vf;
6955 
6956 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6957 	} else {
6958 		bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6959 	}
6960 #endif
6961 	flags = le16_to_cpu(resp->flags);
6962 	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6963 		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6964 		bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6965 		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6966 			bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6967 	}
6968 	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6969 		bp->flags |= BNXT_FLAG_MULTI_HOST;
6970 
6971 	if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6972 		bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6973 
6974 	switch (resp->port_partition_type) {
6975 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6976 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6977 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6978 		bp->port_partition_type = resp->port_partition_type;
6979 		break;
6980 	}
6981 	if (bp->hwrm_spec_code < 0x10707 ||
6982 	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6983 		bp->br_mode = BRIDGE_MODE_VEB;
6984 	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6985 		bp->br_mode = BRIDGE_MODE_VEPA;
6986 	else
6987 		bp->br_mode = BRIDGE_MODE_UNDEF;
6988 
6989 	bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6990 	if (!bp->max_mtu)
6991 		bp->max_mtu = BNXT_MAX_MTU;
6992 
6993 	if (bp->db_size)
6994 		goto func_qcfg_exit;
6995 
6996 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6997 		if (BNXT_PF(bp))
6998 			min_db_offset = DB_PF_OFFSET_P5;
6999 		else
7000 			min_db_offset = DB_VF_OFFSET_P5;
7001 	}
7002 	bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
7003 				 1024);
7004 	if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
7005 	    bp->db_size <= min_db_offset)
7006 		bp->db_size = pci_resource_len(bp->pdev, 2);
7007 
7008 func_qcfg_exit:
7009 	hwrm_req_drop(bp, req);
7010 	return rc;
7011 }
7012 
7013 static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_info *ctx,
7014 			struct hwrm_func_backing_store_qcaps_output *resp)
7015 {
7016 	struct bnxt_mem_init *mem_init;
7017 	u16 init_mask;
7018 	u8 init_val;
7019 	u8 *offset;
7020 	int i;
7021 
7022 	init_val = resp->ctx_kind_initializer;
7023 	init_mask = le16_to_cpu(resp->ctx_init_mask);
7024 	offset = &resp->qp_init_offset;
7025 	mem_init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7026 	for (i = 0; i < BNXT_CTX_MEM_INIT_MAX; i++, mem_init++, offset++) {
7027 		mem_init->init_val = init_val;
7028 		mem_init->offset = BNXT_MEM_INVALID_OFFSET;
7029 		if (!init_mask)
7030 			continue;
7031 		if (i == BNXT_CTX_MEM_INIT_STAT)
7032 			offset = &resp->stat_init_offset;
7033 		if (init_mask & (1 << i))
7034 			mem_init->offset = *offset * 4;
7035 		else
7036 			mem_init->init_val = 0;
7037 	}
7038 	ctx->mem_init[BNXT_CTX_MEM_INIT_QP].size = ctx->qp_entry_size;
7039 	ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ].size = ctx->srq_entry_size;
7040 	ctx->mem_init[BNXT_CTX_MEM_INIT_CQ].size = ctx->cq_entry_size;
7041 	ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC].size = ctx->vnic_entry_size;
7042 	ctx->mem_init[BNXT_CTX_MEM_INIT_STAT].size = ctx->stat_entry_size;
7043 	ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV].size = ctx->mrav_entry_size;
7044 }
7045 
7046 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
7047 {
7048 	struct hwrm_func_backing_store_qcaps_output *resp;
7049 	struct hwrm_func_backing_store_qcaps_input *req;
7050 	int rc;
7051 
7052 	if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
7053 		return 0;
7054 
7055 	rc = hwrm_req_init(bp, req, HWRM_FUNC_BACKING_STORE_QCAPS);
7056 	if (rc)
7057 		return rc;
7058 
7059 	resp = hwrm_req_hold(bp, req);
7060 	rc = hwrm_req_send_silent(bp, req);
7061 	if (!rc) {
7062 		struct bnxt_ctx_pg_info *ctx_pg;
7063 		struct bnxt_ctx_mem_info *ctx;
7064 		int i, tqm_rings;
7065 
7066 		ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
7067 		if (!ctx) {
7068 			rc = -ENOMEM;
7069 			goto ctx_err;
7070 		}
7071 		ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
7072 		ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
7073 		ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
7074 		ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
7075 		ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
7076 		ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
7077 		ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
7078 		ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
7079 		ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
7080 		ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
7081 		ctx->vnic_max_vnic_entries =
7082 			le16_to_cpu(resp->vnic_max_vnic_entries);
7083 		ctx->vnic_max_ring_table_entries =
7084 			le16_to_cpu(resp->vnic_max_ring_table_entries);
7085 		ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
7086 		ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
7087 		ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
7088 		ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
7089 		ctx->tqm_min_entries_per_ring =
7090 			le32_to_cpu(resp->tqm_min_entries_per_ring);
7091 		ctx->tqm_max_entries_per_ring =
7092 			le32_to_cpu(resp->tqm_max_entries_per_ring);
7093 		ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
7094 		if (!ctx->tqm_entries_multiple)
7095 			ctx->tqm_entries_multiple = 1;
7096 		ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
7097 		ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
7098 		ctx->mrav_num_entries_units =
7099 			le16_to_cpu(resp->mrav_num_entries_units);
7100 		ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
7101 		ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
7102 
7103 		bnxt_init_ctx_initializer(ctx, resp);
7104 
7105 		ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
7106 		if (!ctx->tqm_fp_rings_count)
7107 			ctx->tqm_fp_rings_count = bp->max_q;
7108 		else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS)
7109 			ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS;
7110 
7111 		tqm_rings = ctx->tqm_fp_rings_count + BNXT_MAX_TQM_SP_RINGS;
7112 		ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
7113 		if (!ctx_pg) {
7114 			kfree(ctx);
7115 			rc = -ENOMEM;
7116 			goto ctx_err;
7117 		}
7118 		for (i = 0; i < tqm_rings; i++, ctx_pg++)
7119 			ctx->tqm_mem[i] = ctx_pg;
7120 		bp->ctx = ctx;
7121 	} else {
7122 		rc = 0;
7123 	}
7124 ctx_err:
7125 	hwrm_req_drop(bp, req);
7126 	return rc;
7127 }
7128 
7129 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
7130 				  __le64 *pg_dir)
7131 {
7132 	if (!rmem->nr_pages)
7133 		return;
7134 
7135 	BNXT_SET_CTX_PAGE_ATTR(*pg_attr);
7136 	if (rmem->depth >= 1) {
7137 		if (rmem->depth == 2)
7138 			*pg_attr |= 2;
7139 		else
7140 			*pg_attr |= 1;
7141 		*pg_dir = cpu_to_le64(rmem->pg_tbl_map);
7142 	} else {
7143 		*pg_dir = cpu_to_le64(rmem->dma_arr[0]);
7144 	}
7145 }
7146 
7147 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
7148 	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
7149 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
7150 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
7151 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
7152 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
7153 
7154 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
7155 {
7156 	struct hwrm_func_backing_store_cfg_input *req;
7157 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
7158 	struct bnxt_ctx_pg_info *ctx_pg;
7159 	void **__req = (void **)&req;
7160 	u32 req_len = sizeof(*req);
7161 	__le32 *num_entries;
7162 	__le64 *pg_dir;
7163 	u32 flags = 0;
7164 	u8 *pg_attr;
7165 	u32 ena;
7166 	int rc;
7167 	int i;
7168 
7169 	if (!ctx)
7170 		return 0;
7171 
7172 	if (req_len > bp->hwrm_max_ext_req_len)
7173 		req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN;
7174 	rc = __hwrm_req_init(bp, __req, HWRM_FUNC_BACKING_STORE_CFG, req_len);
7175 	if (rc)
7176 		return rc;
7177 
7178 	req->enables = cpu_to_le32(enables);
7179 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
7180 		ctx_pg = &ctx->qp_mem;
7181 		req->qp_num_entries = cpu_to_le32(ctx_pg->entries);
7182 		req->qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
7183 		req->qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
7184 		req->qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
7185 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7186 				      &req->qpc_pg_size_qpc_lvl,
7187 				      &req->qpc_page_dir);
7188 	}
7189 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
7190 		ctx_pg = &ctx->srq_mem;
7191 		req->srq_num_entries = cpu_to_le32(ctx_pg->entries);
7192 		req->srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
7193 		req->srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
7194 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7195 				      &req->srq_pg_size_srq_lvl,
7196 				      &req->srq_page_dir);
7197 	}
7198 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
7199 		ctx_pg = &ctx->cq_mem;
7200 		req->cq_num_entries = cpu_to_le32(ctx_pg->entries);
7201 		req->cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
7202 		req->cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
7203 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7204 				      &req->cq_pg_size_cq_lvl,
7205 				      &req->cq_page_dir);
7206 	}
7207 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
7208 		ctx_pg = &ctx->vnic_mem;
7209 		req->vnic_num_vnic_entries =
7210 			cpu_to_le16(ctx->vnic_max_vnic_entries);
7211 		req->vnic_num_ring_table_entries =
7212 			cpu_to_le16(ctx->vnic_max_ring_table_entries);
7213 		req->vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
7214 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7215 				      &req->vnic_pg_size_vnic_lvl,
7216 				      &req->vnic_page_dir);
7217 	}
7218 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
7219 		ctx_pg = &ctx->stat_mem;
7220 		req->stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
7221 		req->stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
7222 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7223 				      &req->stat_pg_size_stat_lvl,
7224 				      &req->stat_page_dir);
7225 	}
7226 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
7227 		ctx_pg = &ctx->mrav_mem;
7228 		req->mrav_num_entries = cpu_to_le32(ctx_pg->entries);
7229 		if (ctx->mrav_num_entries_units)
7230 			flags |=
7231 			FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
7232 		req->mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
7233 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7234 				      &req->mrav_pg_size_mrav_lvl,
7235 				      &req->mrav_page_dir);
7236 	}
7237 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
7238 		ctx_pg = &ctx->tim_mem;
7239 		req->tim_num_entries = cpu_to_le32(ctx_pg->entries);
7240 		req->tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
7241 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
7242 				      &req->tim_pg_size_tim_lvl,
7243 				      &req->tim_page_dir);
7244 	}
7245 	for (i = 0, num_entries = &req->tqm_sp_num_entries,
7246 	     pg_attr = &req->tqm_sp_pg_size_tqm_sp_lvl,
7247 	     pg_dir = &req->tqm_sp_page_dir,
7248 	     ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
7249 	     i < BNXT_MAX_TQM_RINGS;
7250 	     i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
7251 		if (!(enables & ena))
7252 			continue;
7253 
7254 		req->tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
7255 		ctx_pg = ctx->tqm_mem[i];
7256 		*num_entries = cpu_to_le32(ctx_pg->entries);
7257 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
7258 	}
7259 	req->flags = cpu_to_le32(flags);
7260 	return hwrm_req_send(bp, req);
7261 }
7262 
7263 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
7264 				  struct bnxt_ctx_pg_info *ctx_pg)
7265 {
7266 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7267 
7268 	rmem->page_size = BNXT_PAGE_SIZE;
7269 	rmem->pg_arr = ctx_pg->ctx_pg_arr;
7270 	rmem->dma_arr = ctx_pg->ctx_dma_arr;
7271 	rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
7272 	if (rmem->depth >= 1)
7273 		rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
7274 	return bnxt_alloc_ring(bp, rmem);
7275 }
7276 
7277 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
7278 				  struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
7279 				  u8 depth, struct bnxt_mem_init *mem_init)
7280 {
7281 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7282 	int rc;
7283 
7284 	if (!mem_size)
7285 		return -EINVAL;
7286 
7287 	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7288 	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
7289 		ctx_pg->nr_pages = 0;
7290 		return -EINVAL;
7291 	}
7292 	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
7293 		int nr_tbls, i;
7294 
7295 		rmem->depth = 2;
7296 		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
7297 					     GFP_KERNEL);
7298 		if (!ctx_pg->ctx_pg_tbl)
7299 			return -ENOMEM;
7300 		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
7301 		rmem->nr_pages = nr_tbls;
7302 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7303 		if (rc)
7304 			return rc;
7305 		for (i = 0; i < nr_tbls; i++) {
7306 			struct bnxt_ctx_pg_info *pg_tbl;
7307 
7308 			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
7309 			if (!pg_tbl)
7310 				return -ENOMEM;
7311 			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
7312 			rmem = &pg_tbl->ring_mem;
7313 			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
7314 			rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
7315 			rmem->depth = 1;
7316 			rmem->nr_pages = MAX_CTX_PAGES;
7317 			rmem->mem_init = mem_init;
7318 			if (i == (nr_tbls - 1)) {
7319 				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
7320 
7321 				if (rem)
7322 					rmem->nr_pages = rem;
7323 			}
7324 			rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7325 			if (rc)
7326 				break;
7327 		}
7328 	} else {
7329 		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7330 		if (rmem->nr_pages > 1 || depth)
7331 			rmem->depth = 1;
7332 		rmem->mem_init = mem_init;
7333 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7334 	}
7335 	return rc;
7336 }
7337 
7338 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7339 				  struct bnxt_ctx_pg_info *ctx_pg)
7340 {
7341 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7342 
7343 	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7344 	    ctx_pg->ctx_pg_tbl) {
7345 		int i, nr_tbls = rmem->nr_pages;
7346 
7347 		for (i = 0; i < nr_tbls; i++) {
7348 			struct bnxt_ctx_pg_info *pg_tbl;
7349 			struct bnxt_ring_mem_info *rmem2;
7350 
7351 			pg_tbl = ctx_pg->ctx_pg_tbl[i];
7352 			if (!pg_tbl)
7353 				continue;
7354 			rmem2 = &pg_tbl->ring_mem;
7355 			bnxt_free_ring(bp, rmem2);
7356 			ctx_pg->ctx_pg_arr[i] = NULL;
7357 			kfree(pg_tbl);
7358 			ctx_pg->ctx_pg_tbl[i] = NULL;
7359 		}
7360 		kfree(ctx_pg->ctx_pg_tbl);
7361 		ctx_pg->ctx_pg_tbl = NULL;
7362 	}
7363 	bnxt_free_ring(bp, rmem);
7364 	ctx_pg->nr_pages = 0;
7365 }
7366 
7367 void bnxt_free_ctx_mem(struct bnxt *bp)
7368 {
7369 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
7370 	int i;
7371 
7372 	if (!ctx)
7373 		return;
7374 
7375 	if (ctx->tqm_mem[0]) {
7376 		for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7377 			bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7378 		kfree(ctx->tqm_mem[0]);
7379 		ctx->tqm_mem[0] = NULL;
7380 	}
7381 
7382 	bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7383 	bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7384 	bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7385 	bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7386 	bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7387 	bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7388 	bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7389 	ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7390 }
7391 
7392 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7393 {
7394 	struct bnxt_ctx_pg_info *ctx_pg;
7395 	struct bnxt_ctx_mem_info *ctx;
7396 	struct bnxt_mem_init *init;
7397 	u32 mem_size, ena, entries;
7398 	u32 entries_sp, min;
7399 	u32 num_mr, num_ah;
7400 	u32 extra_srqs = 0;
7401 	u32 extra_qps = 0;
7402 	u8 pg_lvl = 1;
7403 	int i, rc;
7404 
7405 	rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7406 	if (rc) {
7407 		netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7408 			   rc);
7409 		return rc;
7410 	}
7411 	ctx = bp->ctx;
7412 	if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7413 		return 0;
7414 
7415 	if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7416 		pg_lvl = 2;
7417 		extra_qps = 65536;
7418 		extra_srqs = 8192;
7419 	}
7420 
7421 	ctx_pg = &ctx->qp_mem;
7422 	ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7423 			  extra_qps;
7424 	if (ctx->qp_entry_size) {
7425 		mem_size = ctx->qp_entry_size * ctx_pg->entries;
7426 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_QP];
7427 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7428 		if (rc)
7429 			return rc;
7430 	}
7431 
7432 	ctx_pg = &ctx->srq_mem;
7433 	ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7434 	if (ctx->srq_entry_size) {
7435 		mem_size = ctx->srq_entry_size * ctx_pg->entries;
7436 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_SRQ];
7437 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7438 		if (rc)
7439 			return rc;
7440 	}
7441 
7442 	ctx_pg = &ctx->cq_mem;
7443 	ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7444 	if (ctx->cq_entry_size) {
7445 		mem_size = ctx->cq_entry_size * ctx_pg->entries;
7446 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_CQ];
7447 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, init);
7448 		if (rc)
7449 			return rc;
7450 	}
7451 
7452 	ctx_pg = &ctx->vnic_mem;
7453 	ctx_pg->entries = ctx->vnic_max_vnic_entries +
7454 			  ctx->vnic_max_ring_table_entries;
7455 	if (ctx->vnic_entry_size) {
7456 		mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7457 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_VNIC];
7458 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7459 		if (rc)
7460 			return rc;
7461 	}
7462 
7463 	ctx_pg = &ctx->stat_mem;
7464 	ctx_pg->entries = ctx->stat_max_entries;
7465 	if (ctx->stat_entry_size) {
7466 		mem_size = ctx->stat_entry_size * ctx_pg->entries;
7467 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_STAT];
7468 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, init);
7469 		if (rc)
7470 			return rc;
7471 	}
7472 
7473 	ena = 0;
7474 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7475 		goto skip_rdma;
7476 
7477 	ctx_pg = &ctx->mrav_mem;
7478 	/* 128K extra is needed to accommodate static AH context
7479 	 * allocation by f/w.
7480 	 */
7481 	num_mr = 1024 * 256;
7482 	num_ah = 1024 * 128;
7483 	ctx_pg->entries = num_mr + num_ah;
7484 	if (ctx->mrav_entry_size) {
7485 		mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7486 		init = &ctx->mem_init[BNXT_CTX_MEM_INIT_MRAV];
7487 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, init);
7488 		if (rc)
7489 			return rc;
7490 	}
7491 	ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7492 	if (ctx->mrav_num_entries_units)
7493 		ctx_pg->entries =
7494 			((num_mr / ctx->mrav_num_entries_units) << 16) |
7495 			 (num_ah / ctx->mrav_num_entries_units);
7496 
7497 	ctx_pg = &ctx->tim_mem;
7498 	ctx_pg->entries = ctx->qp_mem.entries;
7499 	if (ctx->tim_entry_size) {
7500 		mem_size = ctx->tim_entry_size * ctx_pg->entries;
7501 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, NULL);
7502 		if (rc)
7503 			return rc;
7504 	}
7505 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7506 
7507 skip_rdma:
7508 	min = ctx->tqm_min_entries_per_ring;
7509 	entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7510 		     2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7511 	entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7512 	entries = ctx->qp_max_l2_entries + 2 * (extra_qps + ctx->qp_min_qp1_entries);
7513 	entries = roundup(entries, ctx->tqm_entries_multiple);
7514 	entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7515 	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7516 		ctx_pg = ctx->tqm_mem[i];
7517 		ctx_pg->entries = i ? entries : entries_sp;
7518 		if (ctx->tqm_entry_size) {
7519 			mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7520 			rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1,
7521 						    NULL);
7522 			if (rc)
7523 				return rc;
7524 		}
7525 		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7526 	}
7527 	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7528 	rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7529 	if (rc) {
7530 		netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7531 			   rc);
7532 		return rc;
7533 	}
7534 	ctx->flags |= BNXT_CTX_FLAG_INITED;
7535 	return 0;
7536 }
7537 
7538 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7539 {
7540 	struct hwrm_func_resource_qcaps_output *resp;
7541 	struct hwrm_func_resource_qcaps_input *req;
7542 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7543 	int rc;
7544 
7545 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESOURCE_QCAPS);
7546 	if (rc)
7547 		return rc;
7548 
7549 	req->fid = cpu_to_le16(0xffff);
7550 	resp = hwrm_req_hold(bp, req);
7551 	rc = hwrm_req_send_silent(bp, req);
7552 	if (rc)
7553 		goto hwrm_func_resc_qcaps_exit;
7554 
7555 	hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7556 	if (!all)
7557 		goto hwrm_func_resc_qcaps_exit;
7558 
7559 	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7560 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7561 	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7562 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7563 	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7564 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7565 	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7566 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7567 	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7568 	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7569 	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7570 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7571 	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7572 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7573 	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7574 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7575 
7576 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
7577 		u16 max_msix = le16_to_cpu(resp->max_msix);
7578 
7579 		hw_resc->max_nqs = max_msix;
7580 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7581 	}
7582 
7583 	if (BNXT_PF(bp)) {
7584 		struct bnxt_pf_info *pf = &bp->pf;
7585 
7586 		pf->vf_resv_strategy =
7587 			le16_to_cpu(resp->vf_reservation_strategy);
7588 		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7589 			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7590 	}
7591 hwrm_func_resc_qcaps_exit:
7592 	hwrm_req_drop(bp, req);
7593 	return rc;
7594 }
7595 
7596 static int __bnxt_hwrm_ptp_qcfg(struct bnxt *bp)
7597 {
7598 	struct hwrm_port_mac_ptp_qcfg_output *resp;
7599 	struct hwrm_port_mac_ptp_qcfg_input *req;
7600 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
7601 	bool phc_cfg;
7602 	u8 flags;
7603 	int rc;
7604 
7605 	if (bp->hwrm_spec_code < 0x10801 || !BNXT_CHIP_P5_THOR(bp)) {
7606 		rc = -ENODEV;
7607 		goto no_ptp;
7608 	}
7609 
7610 	rc = hwrm_req_init(bp, req, HWRM_PORT_MAC_PTP_QCFG);
7611 	if (rc)
7612 		goto no_ptp;
7613 
7614 	req->port_id = cpu_to_le16(bp->pf.port_id);
7615 	resp = hwrm_req_hold(bp, req);
7616 	rc = hwrm_req_send(bp, req);
7617 	if (rc)
7618 		goto exit;
7619 
7620 	flags = resp->flags;
7621 	if (!(flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_HWRM_ACCESS)) {
7622 		rc = -ENODEV;
7623 		goto exit;
7624 	}
7625 	if (!ptp) {
7626 		ptp = kzalloc(sizeof(*ptp), GFP_KERNEL);
7627 		if (!ptp) {
7628 			rc = -ENOMEM;
7629 			goto exit;
7630 		}
7631 		ptp->bp = bp;
7632 		bp->ptp_cfg = ptp;
7633 	}
7634 	if (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_PARTIAL_DIRECT_ACCESS_REF_CLOCK) {
7635 		ptp->refclk_regs[0] = le32_to_cpu(resp->ts_ref_clock_reg_lower);
7636 		ptp->refclk_regs[1] = le32_to_cpu(resp->ts_ref_clock_reg_upper);
7637 	} else if (bp->flags & BNXT_FLAG_CHIP_P5) {
7638 		ptp->refclk_regs[0] = BNXT_TS_REG_TIMESYNC_TS0_LOWER;
7639 		ptp->refclk_regs[1] = BNXT_TS_REG_TIMESYNC_TS0_UPPER;
7640 	} else {
7641 		rc = -ENODEV;
7642 		goto exit;
7643 	}
7644 	phc_cfg = (flags & PORT_MAC_PTP_QCFG_RESP_FLAGS_RTC_CONFIGURED) != 0;
7645 	rc = bnxt_ptp_init(bp, phc_cfg);
7646 	if (rc)
7647 		netdev_warn(bp->dev, "PTP initialization failed.\n");
7648 exit:
7649 	hwrm_req_drop(bp, req);
7650 	if (!rc)
7651 		return 0;
7652 
7653 no_ptp:
7654 	bnxt_ptp_clear(bp);
7655 	kfree(ptp);
7656 	bp->ptp_cfg = NULL;
7657 	return rc;
7658 }
7659 
7660 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7661 {
7662 	struct hwrm_func_qcaps_output *resp;
7663 	struct hwrm_func_qcaps_input *req;
7664 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7665 	u32 flags, flags_ext, flags_ext2;
7666 	int rc;
7667 
7668 	rc = hwrm_req_init(bp, req, HWRM_FUNC_QCAPS);
7669 	if (rc)
7670 		return rc;
7671 
7672 	req->fid = cpu_to_le16(0xffff);
7673 	resp = hwrm_req_hold(bp, req);
7674 	rc = hwrm_req_send(bp, req);
7675 	if (rc)
7676 		goto hwrm_func_qcaps_exit;
7677 
7678 	flags = le32_to_cpu(resp->flags);
7679 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7680 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7681 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7682 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7683 	if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7684 		bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7685 	if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7686 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7687 	if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7688 		bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7689 	if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7690 		bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7691 	if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7692 		bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7693 	if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7694 		bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7695 	if (flags & FUNC_QCAPS_RESP_FLAGS_DBG_QCAPS_CMD_SUPPORTED)
7696 		bp->fw_cap |= BNXT_FW_CAP_DBG_QCAPS;
7697 
7698 	flags_ext = le32_to_cpu(resp->flags_ext);
7699 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7700 		bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7701 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_PPS_SUPPORTED))
7702 		bp->fw_cap |= BNXT_FW_CAP_PTP_PPS;
7703 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED)
7704 		bp->fw_cap |= BNXT_FW_CAP_PTP_RTC;
7705 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_HOT_RESET_IF_SUPPORT))
7706 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF;
7707 	if (BNXT_PF(bp) && (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED))
7708 		bp->fw_cap |= BNXT_FW_CAP_LIVEPATCH;
7709 
7710 	flags_ext2 = le32_to_cpu(resp->flags_ext2);
7711 	if (flags_ext2 & FUNC_QCAPS_RESP_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED)
7712 		bp->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS;
7713 
7714 	bp->tx_push_thresh = 0;
7715 	if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7716 	    BNXT_FW_MAJ(bp) > 217)
7717 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7718 
7719 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7720 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7721 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7722 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7723 	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7724 	if (!hw_resc->max_hw_ring_grps)
7725 		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7726 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7727 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7728 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7729 
7730 	if (BNXT_PF(bp)) {
7731 		struct bnxt_pf_info *pf = &bp->pf;
7732 
7733 		pf->fw_fid = le16_to_cpu(resp->fid);
7734 		pf->port_id = le16_to_cpu(resp->port_id);
7735 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7736 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7737 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
7738 		pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7739 		pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7740 		pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7741 		pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7742 		pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7743 		pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7744 		bp->flags &= ~BNXT_FLAG_WOL_CAP;
7745 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7746 			bp->flags |= BNXT_FLAG_WOL_CAP;
7747 		if (flags & FUNC_QCAPS_RESP_FLAGS_PTP_SUPPORTED) {
7748 			bp->fw_cap |= BNXT_FW_CAP_PTP;
7749 		} else {
7750 			bnxt_ptp_clear(bp);
7751 			kfree(bp->ptp_cfg);
7752 			bp->ptp_cfg = NULL;
7753 		}
7754 	} else {
7755 #ifdef CONFIG_BNXT_SRIOV
7756 		struct bnxt_vf_info *vf = &bp->vf;
7757 
7758 		vf->fw_fid = le16_to_cpu(resp->fid);
7759 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7760 #endif
7761 	}
7762 
7763 hwrm_func_qcaps_exit:
7764 	hwrm_req_drop(bp, req);
7765 	return rc;
7766 }
7767 
7768 static void bnxt_hwrm_dbg_qcaps(struct bnxt *bp)
7769 {
7770 	struct hwrm_dbg_qcaps_output *resp;
7771 	struct hwrm_dbg_qcaps_input *req;
7772 	int rc;
7773 
7774 	bp->fw_dbg_cap = 0;
7775 	if (!(bp->fw_cap & BNXT_FW_CAP_DBG_QCAPS))
7776 		return;
7777 
7778 	rc = hwrm_req_init(bp, req, HWRM_DBG_QCAPS);
7779 	if (rc)
7780 		return;
7781 
7782 	req->fid = cpu_to_le16(0xffff);
7783 	resp = hwrm_req_hold(bp, req);
7784 	rc = hwrm_req_send(bp, req);
7785 	if (rc)
7786 		goto hwrm_dbg_qcaps_exit;
7787 
7788 	bp->fw_dbg_cap = le32_to_cpu(resp->flags);
7789 
7790 hwrm_dbg_qcaps_exit:
7791 	hwrm_req_drop(bp, req);
7792 }
7793 
7794 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7795 
7796 int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7797 {
7798 	int rc;
7799 
7800 	rc = __bnxt_hwrm_func_qcaps(bp);
7801 	if (rc)
7802 		return rc;
7803 
7804 	bnxt_hwrm_dbg_qcaps(bp);
7805 
7806 	rc = bnxt_hwrm_queue_qportcfg(bp);
7807 	if (rc) {
7808 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7809 		return rc;
7810 	}
7811 	if (bp->hwrm_spec_code >= 0x10803) {
7812 		rc = bnxt_alloc_ctx_mem(bp);
7813 		if (rc)
7814 			return rc;
7815 		rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7816 		if (!rc)
7817 			bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7818 	}
7819 	return 0;
7820 }
7821 
7822 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7823 {
7824 	struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7825 	struct hwrm_cfa_adv_flow_mgnt_qcaps_input *req;
7826 	u32 flags;
7827 	int rc;
7828 
7829 	if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7830 		return 0;
7831 
7832 	rc = hwrm_req_init(bp, req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS);
7833 	if (rc)
7834 		return rc;
7835 
7836 	resp = hwrm_req_hold(bp, req);
7837 	rc = hwrm_req_send(bp, req);
7838 	if (rc)
7839 		goto hwrm_cfa_adv_qcaps_exit;
7840 
7841 	flags = le32_to_cpu(resp->flags);
7842 	if (flags &
7843 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7844 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7845 
7846 hwrm_cfa_adv_qcaps_exit:
7847 	hwrm_req_drop(bp, req);
7848 	return rc;
7849 }
7850 
7851 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7852 {
7853 	if (bp->fw_health)
7854 		return 0;
7855 
7856 	bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7857 	if (!bp->fw_health)
7858 		return -ENOMEM;
7859 
7860 	mutex_init(&bp->fw_health->lock);
7861 	return 0;
7862 }
7863 
7864 static int bnxt_alloc_fw_health(struct bnxt *bp)
7865 {
7866 	int rc;
7867 
7868 	if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7869 	    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7870 		return 0;
7871 
7872 	rc = __bnxt_alloc_fw_health(bp);
7873 	if (rc) {
7874 		bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7875 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7876 		return rc;
7877 	}
7878 
7879 	return 0;
7880 }
7881 
7882 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7883 {
7884 	writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7885 					 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7886 					 BNXT_FW_HEALTH_WIN_MAP_OFF);
7887 }
7888 
7889 static void bnxt_inv_fw_health_reg(struct bnxt *bp)
7890 {
7891 	struct bnxt_fw_health *fw_health = bp->fw_health;
7892 	u32 reg_type;
7893 
7894 	if (!fw_health)
7895 		return;
7896 
7897 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]);
7898 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7899 		fw_health->status_reliable = false;
7900 
7901 	reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]);
7902 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC)
7903 		fw_health->resets_reliable = false;
7904 }
7905 
7906 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7907 {
7908 	void __iomem *hs;
7909 	u32 status_loc;
7910 	u32 reg_type;
7911 	u32 sig;
7912 
7913 	if (bp->fw_health)
7914 		bp->fw_health->status_reliable = false;
7915 
7916 	__bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7917 	hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7918 
7919 	sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7920 	if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7921 		if (!bp->chip_num) {
7922 			__bnxt_map_fw_health_reg(bp, BNXT_GRC_REG_BASE);
7923 			bp->chip_num = readl(bp->bar0 +
7924 					     BNXT_FW_HEALTH_WIN_BASE +
7925 					     BNXT_GRC_REG_CHIP_NUM);
7926 		}
7927 		if (!BNXT_CHIP_P5(bp))
7928 			return;
7929 
7930 		status_loc = BNXT_GRC_REG_STATUS_P5 |
7931 			     BNXT_FW_HEALTH_REG_TYPE_BAR0;
7932 	} else {
7933 		status_loc = readl(hs + offsetof(struct hcomm_status,
7934 						 fw_status_loc));
7935 	}
7936 
7937 	if (__bnxt_alloc_fw_health(bp)) {
7938 		netdev_warn(bp->dev, "no memory for firmware status checks\n");
7939 		return;
7940 	}
7941 
7942 	bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7943 	reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7944 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7945 		__bnxt_map_fw_health_reg(bp, status_loc);
7946 		bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7947 			BNXT_FW_HEALTH_WIN_OFF(status_loc);
7948 	}
7949 
7950 	bp->fw_health->status_reliable = true;
7951 }
7952 
7953 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7954 {
7955 	struct bnxt_fw_health *fw_health = bp->fw_health;
7956 	u32 reg_base = 0xffffffff;
7957 	int i;
7958 
7959 	bp->fw_health->status_reliable = false;
7960 	bp->fw_health->resets_reliable = false;
7961 	/* Only pre-map the monitoring GRC registers using window 3 */
7962 	for (i = 0; i < 4; i++) {
7963 		u32 reg = fw_health->regs[i];
7964 
7965 		if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7966 			continue;
7967 		if (reg_base == 0xffffffff)
7968 			reg_base = reg & BNXT_GRC_BASE_MASK;
7969 		if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7970 			return -ERANGE;
7971 		fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7972 	}
7973 	bp->fw_health->status_reliable = true;
7974 	bp->fw_health->resets_reliable = true;
7975 	if (reg_base == 0xffffffff)
7976 		return 0;
7977 
7978 	__bnxt_map_fw_health_reg(bp, reg_base);
7979 	return 0;
7980 }
7981 
7982 static void bnxt_remap_fw_health_regs(struct bnxt *bp)
7983 {
7984 	if (!bp->fw_health)
7985 		return;
7986 
7987 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) {
7988 		bp->fw_health->status_reliable = true;
7989 		bp->fw_health->resets_reliable = true;
7990 	} else {
7991 		bnxt_try_map_fw_health_reg(bp);
7992 	}
7993 }
7994 
7995 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7996 {
7997 	struct bnxt_fw_health *fw_health = bp->fw_health;
7998 	struct hwrm_error_recovery_qcfg_output *resp;
7999 	struct hwrm_error_recovery_qcfg_input *req;
8000 	int rc, i;
8001 
8002 	if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
8003 		return 0;
8004 
8005 	rc = hwrm_req_init(bp, req, HWRM_ERROR_RECOVERY_QCFG);
8006 	if (rc)
8007 		return rc;
8008 
8009 	resp = hwrm_req_hold(bp, req);
8010 	rc = hwrm_req_send(bp, req);
8011 	if (rc)
8012 		goto err_recovery_out;
8013 	fw_health->flags = le32_to_cpu(resp->flags);
8014 	if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
8015 	    !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
8016 		rc = -EINVAL;
8017 		goto err_recovery_out;
8018 	}
8019 	fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
8020 	fw_health->master_func_wait_dsecs =
8021 		le32_to_cpu(resp->master_func_wait_period);
8022 	fw_health->normal_func_wait_dsecs =
8023 		le32_to_cpu(resp->normal_func_wait_period);
8024 	fw_health->post_reset_wait_dsecs =
8025 		le32_to_cpu(resp->master_func_wait_period_after_reset);
8026 	fw_health->post_reset_max_wait_dsecs =
8027 		le32_to_cpu(resp->max_bailout_time_after_reset);
8028 	fw_health->regs[BNXT_FW_HEALTH_REG] =
8029 		le32_to_cpu(resp->fw_health_status_reg);
8030 	fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
8031 		le32_to_cpu(resp->fw_heartbeat_reg);
8032 	fw_health->regs[BNXT_FW_RESET_CNT_REG] =
8033 		le32_to_cpu(resp->fw_reset_cnt_reg);
8034 	fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
8035 		le32_to_cpu(resp->reset_inprogress_reg);
8036 	fw_health->fw_reset_inprog_reg_mask =
8037 		le32_to_cpu(resp->reset_inprogress_reg_mask);
8038 	fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
8039 	if (fw_health->fw_reset_seq_cnt >= 16) {
8040 		rc = -EINVAL;
8041 		goto err_recovery_out;
8042 	}
8043 	for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
8044 		fw_health->fw_reset_seq_regs[i] =
8045 			le32_to_cpu(resp->reset_reg[i]);
8046 		fw_health->fw_reset_seq_vals[i] =
8047 			le32_to_cpu(resp->reset_reg_val[i]);
8048 		fw_health->fw_reset_seq_delay_msec[i] =
8049 			resp->delay_after_reset[i];
8050 	}
8051 err_recovery_out:
8052 	hwrm_req_drop(bp, req);
8053 	if (!rc)
8054 		rc = bnxt_map_fw_health_regs(bp);
8055 	if (rc)
8056 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
8057 	return rc;
8058 }
8059 
8060 static int bnxt_hwrm_func_reset(struct bnxt *bp)
8061 {
8062 	struct hwrm_func_reset_input *req;
8063 	int rc;
8064 
8065 	rc = hwrm_req_init(bp, req, HWRM_FUNC_RESET);
8066 	if (rc)
8067 		return rc;
8068 
8069 	req->enables = 0;
8070 	hwrm_req_timeout(bp, req, HWRM_RESET_TIMEOUT);
8071 	return hwrm_req_send(bp, req);
8072 }
8073 
8074 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
8075 {
8076 	struct hwrm_nvm_get_dev_info_output nvm_info;
8077 
8078 	if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
8079 		snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
8080 			 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
8081 			 nvm_info.nvm_cfg_ver_upd);
8082 }
8083 
8084 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
8085 {
8086 	struct hwrm_queue_qportcfg_output *resp;
8087 	struct hwrm_queue_qportcfg_input *req;
8088 	u8 i, j, *qptr;
8089 	bool no_rdma;
8090 	int rc = 0;
8091 
8092 	rc = hwrm_req_init(bp, req, HWRM_QUEUE_QPORTCFG);
8093 	if (rc)
8094 		return rc;
8095 
8096 	resp = hwrm_req_hold(bp, req);
8097 	rc = hwrm_req_send(bp, req);
8098 	if (rc)
8099 		goto qportcfg_exit;
8100 
8101 	if (!resp->max_configurable_queues) {
8102 		rc = -EINVAL;
8103 		goto qportcfg_exit;
8104 	}
8105 	bp->max_tc = resp->max_configurable_queues;
8106 	bp->max_lltc = resp->max_configurable_lossless_queues;
8107 	if (bp->max_tc > BNXT_MAX_QUEUE)
8108 		bp->max_tc = BNXT_MAX_QUEUE;
8109 
8110 	no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
8111 	qptr = &resp->queue_id0;
8112 	for (i = 0, j = 0; i < bp->max_tc; i++) {
8113 		bp->q_info[j].queue_id = *qptr;
8114 		bp->q_ids[i] = *qptr++;
8115 		bp->q_info[j].queue_profile = *qptr++;
8116 		bp->tc_to_qidx[j] = j;
8117 		if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
8118 		    (no_rdma && BNXT_PF(bp)))
8119 			j++;
8120 	}
8121 	bp->max_q = bp->max_tc;
8122 	bp->max_tc = max_t(u8, j, 1);
8123 
8124 	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
8125 		bp->max_tc = 1;
8126 
8127 	if (bp->max_lltc > bp->max_tc)
8128 		bp->max_lltc = bp->max_tc;
8129 
8130 qportcfg_exit:
8131 	hwrm_req_drop(bp, req);
8132 	return rc;
8133 }
8134 
8135 static int bnxt_hwrm_poll(struct bnxt *bp)
8136 {
8137 	struct hwrm_ver_get_input *req;
8138 	int rc;
8139 
8140 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8141 	if (rc)
8142 		return rc;
8143 
8144 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8145 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
8146 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8147 
8148 	hwrm_req_flags(bp, req, BNXT_HWRM_CTX_SILENT | BNXT_HWRM_FULL_WAIT);
8149 	rc = hwrm_req_send(bp, req);
8150 	return rc;
8151 }
8152 
8153 static int bnxt_hwrm_ver_get(struct bnxt *bp)
8154 {
8155 	struct hwrm_ver_get_output *resp;
8156 	struct hwrm_ver_get_input *req;
8157 	u16 fw_maj, fw_min, fw_bld, fw_rsv;
8158 	u32 dev_caps_cfg, hwrm_ver;
8159 	int rc, len;
8160 
8161 	rc = hwrm_req_init(bp, req, HWRM_VER_GET);
8162 	if (rc)
8163 		return rc;
8164 
8165 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
8166 	bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
8167 	req->hwrm_intf_maj = HWRM_VERSION_MAJOR;
8168 	req->hwrm_intf_min = HWRM_VERSION_MINOR;
8169 	req->hwrm_intf_upd = HWRM_VERSION_UPDATE;
8170 
8171 	resp = hwrm_req_hold(bp, req);
8172 	rc = hwrm_req_send(bp, req);
8173 	if (rc)
8174 		goto hwrm_ver_get_exit;
8175 
8176 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
8177 
8178 	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
8179 			     resp->hwrm_intf_min_8b << 8 |
8180 			     resp->hwrm_intf_upd_8b;
8181 	if (resp->hwrm_intf_maj_8b < 1) {
8182 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
8183 			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8184 			    resp->hwrm_intf_upd_8b);
8185 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
8186 	}
8187 
8188 	hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
8189 			HWRM_VERSION_UPDATE;
8190 
8191 	if (bp->hwrm_spec_code > hwrm_ver)
8192 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8193 			 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
8194 			 HWRM_VERSION_UPDATE);
8195 	else
8196 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
8197 			 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
8198 			 resp->hwrm_intf_upd_8b);
8199 
8200 	fw_maj = le16_to_cpu(resp->hwrm_fw_major);
8201 	if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
8202 		fw_min = le16_to_cpu(resp->hwrm_fw_minor);
8203 		fw_bld = le16_to_cpu(resp->hwrm_fw_build);
8204 		fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
8205 		len = FW_VER_STR_LEN;
8206 	} else {
8207 		fw_maj = resp->hwrm_fw_maj_8b;
8208 		fw_min = resp->hwrm_fw_min_8b;
8209 		fw_bld = resp->hwrm_fw_bld_8b;
8210 		fw_rsv = resp->hwrm_fw_rsvd_8b;
8211 		len = BC_HWRM_STR_LEN;
8212 	}
8213 	bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
8214 	snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
8215 		 fw_rsv);
8216 
8217 	if (strlen(resp->active_pkg_name)) {
8218 		int fw_ver_len = strlen(bp->fw_ver_str);
8219 
8220 		snprintf(bp->fw_ver_str + fw_ver_len,
8221 			 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
8222 			 resp->active_pkg_name);
8223 		bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
8224 	}
8225 
8226 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
8227 	if (!bp->hwrm_cmd_timeout)
8228 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
8229 	bp->hwrm_cmd_max_timeout = le16_to_cpu(resp->max_req_timeout) * 1000;
8230 	if (!bp->hwrm_cmd_max_timeout)
8231 		bp->hwrm_cmd_max_timeout = HWRM_CMD_MAX_TIMEOUT;
8232 	else if (bp->hwrm_cmd_max_timeout > HWRM_CMD_MAX_TIMEOUT)
8233 		netdev_warn(bp->dev, "Device requests max timeout of %d seconds, may trigger hung task watchdog\n",
8234 			    bp->hwrm_cmd_max_timeout / 1000);
8235 
8236 	if (resp->hwrm_intf_maj_8b >= 1) {
8237 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
8238 		bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
8239 	}
8240 	if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
8241 		bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
8242 
8243 	bp->chip_num = le16_to_cpu(resp->chip_num);
8244 	bp->chip_rev = resp->chip_rev;
8245 	if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
8246 	    !resp->chip_metal)
8247 		bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
8248 
8249 	dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
8250 	if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
8251 	    (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
8252 		bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
8253 
8254 	if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
8255 		bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
8256 
8257 	if (dev_caps_cfg &
8258 	    VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
8259 		bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
8260 
8261 	if (dev_caps_cfg &
8262 	    VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
8263 		bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
8264 
8265 	if (dev_caps_cfg &
8266 	    VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
8267 		bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
8268 
8269 hwrm_ver_get_exit:
8270 	hwrm_req_drop(bp, req);
8271 	return rc;
8272 }
8273 
8274 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
8275 {
8276 	struct hwrm_fw_set_time_input *req;
8277 	struct tm tm;
8278 	time64_t now = ktime_get_real_seconds();
8279 	int rc;
8280 
8281 	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
8282 	    bp->hwrm_spec_code < 0x10400)
8283 		return -EOPNOTSUPP;
8284 
8285 	time64_to_tm(now, 0, &tm);
8286 	rc = hwrm_req_init(bp, req, HWRM_FW_SET_TIME);
8287 	if (rc)
8288 		return rc;
8289 
8290 	req->year = cpu_to_le16(1900 + tm.tm_year);
8291 	req->month = 1 + tm.tm_mon;
8292 	req->day = tm.tm_mday;
8293 	req->hour = tm.tm_hour;
8294 	req->minute = tm.tm_min;
8295 	req->second = tm.tm_sec;
8296 	return hwrm_req_send(bp, req);
8297 }
8298 
8299 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
8300 {
8301 	u64 sw_tmp;
8302 
8303 	hw &= mask;
8304 	sw_tmp = (*sw & ~mask) | hw;
8305 	if (hw < (*sw & mask))
8306 		sw_tmp += mask + 1;
8307 	WRITE_ONCE(*sw, sw_tmp);
8308 }
8309 
8310 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
8311 				    int count, bool ignore_zero)
8312 {
8313 	int i;
8314 
8315 	for (i = 0; i < count; i++) {
8316 		u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
8317 
8318 		if (ignore_zero && !hw)
8319 			continue;
8320 
8321 		if (masks[i] == -1ULL)
8322 			sw_stats[i] = hw;
8323 		else
8324 			bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
8325 	}
8326 }
8327 
8328 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
8329 {
8330 	if (!stats->hw_stats)
8331 		return;
8332 
8333 	__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8334 				stats->hw_masks, stats->len / 8, false);
8335 }
8336 
8337 static void bnxt_accumulate_all_stats(struct bnxt *bp)
8338 {
8339 	struct bnxt_stats_mem *ring0_stats;
8340 	bool ignore_zero = false;
8341 	int i;
8342 
8343 	/* Chip bug.  Counter intermittently becomes 0. */
8344 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8345 		ignore_zero = true;
8346 
8347 	for (i = 0; i < bp->cp_nr_rings; i++) {
8348 		struct bnxt_napi *bnapi = bp->bnapi[i];
8349 		struct bnxt_cp_ring_info *cpr;
8350 		struct bnxt_stats_mem *stats;
8351 
8352 		cpr = &bnapi->cp_ring;
8353 		stats = &cpr->stats;
8354 		if (!i)
8355 			ring0_stats = stats;
8356 		__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
8357 					ring0_stats->hw_masks,
8358 					ring0_stats->len / 8, ignore_zero);
8359 	}
8360 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
8361 		struct bnxt_stats_mem *stats = &bp->port_stats;
8362 		__le64 *hw_stats = stats->hw_stats;
8363 		u64 *sw_stats = stats->sw_stats;
8364 		u64 *masks = stats->hw_masks;
8365 		int cnt;
8366 
8367 		cnt = sizeof(struct rx_port_stats) / 8;
8368 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8369 
8370 		hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8371 		sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8372 		masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
8373 		cnt = sizeof(struct tx_port_stats) / 8;
8374 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
8375 	}
8376 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
8377 		bnxt_accumulate_stats(&bp->rx_port_stats_ext);
8378 		bnxt_accumulate_stats(&bp->tx_port_stats_ext);
8379 	}
8380 }
8381 
8382 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
8383 {
8384 	struct hwrm_port_qstats_input *req;
8385 	struct bnxt_pf_info *pf = &bp->pf;
8386 	int rc;
8387 
8388 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
8389 		return 0;
8390 
8391 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8392 		return -EOPNOTSUPP;
8393 
8394 	rc = hwrm_req_init(bp, req, HWRM_PORT_QSTATS);
8395 	if (rc)
8396 		return rc;
8397 
8398 	req->flags = flags;
8399 	req->port_id = cpu_to_le16(pf->port_id);
8400 	req->tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
8401 					    BNXT_TX_PORT_STATS_BYTE_OFFSET);
8402 	req->rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
8403 	return hwrm_req_send(bp, req);
8404 }
8405 
8406 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
8407 {
8408 	struct hwrm_queue_pri2cos_qcfg_output *resp_qc;
8409 	struct hwrm_queue_pri2cos_qcfg_input *req_qc;
8410 	struct hwrm_port_qstats_ext_output *resp_qs;
8411 	struct hwrm_port_qstats_ext_input *req_qs;
8412 	struct bnxt_pf_info *pf = &bp->pf;
8413 	u32 tx_stat_size;
8414 	int rc;
8415 
8416 	if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
8417 		return 0;
8418 
8419 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
8420 		return -EOPNOTSUPP;
8421 
8422 	rc = hwrm_req_init(bp, req_qs, HWRM_PORT_QSTATS_EXT);
8423 	if (rc)
8424 		return rc;
8425 
8426 	req_qs->flags = flags;
8427 	req_qs->port_id = cpu_to_le16(pf->port_id);
8428 	req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
8429 	req_qs->rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
8430 	tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
8431 		       sizeof(struct tx_port_stats_ext) : 0;
8432 	req_qs->tx_stat_size = cpu_to_le16(tx_stat_size);
8433 	req_qs->tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
8434 	resp_qs = hwrm_req_hold(bp, req_qs);
8435 	rc = hwrm_req_send(bp, req_qs);
8436 	if (!rc) {
8437 		bp->fw_rx_stats_ext_size =
8438 			le16_to_cpu(resp_qs->rx_stat_size) / 8;
8439 		if (BNXT_FW_MAJ(bp) < 220 &&
8440 		    bp->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY)
8441 			bp->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY;
8442 
8443 		bp->fw_tx_stats_ext_size = tx_stat_size ?
8444 			le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0;
8445 	} else {
8446 		bp->fw_rx_stats_ext_size = 0;
8447 		bp->fw_tx_stats_ext_size = 0;
8448 	}
8449 	hwrm_req_drop(bp, req_qs);
8450 
8451 	if (flags)
8452 		return rc;
8453 
8454 	if (bp->fw_tx_stats_ext_size <=
8455 	    offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
8456 		bp->pri2cos_valid = 0;
8457 		return rc;
8458 	}
8459 
8460 	rc = hwrm_req_init(bp, req_qc, HWRM_QUEUE_PRI2COS_QCFG);
8461 	if (rc)
8462 		return rc;
8463 
8464 	req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
8465 
8466 	resp_qc = hwrm_req_hold(bp, req_qc);
8467 	rc = hwrm_req_send(bp, req_qc);
8468 	if (!rc) {
8469 		u8 *pri2cos;
8470 		int i, j;
8471 
8472 		pri2cos = &resp_qc->pri0_cos_queue_id;
8473 		for (i = 0; i < 8; i++) {
8474 			u8 queue_id = pri2cos[i];
8475 			u8 queue_idx;
8476 
8477 			/* Per port queue IDs start from 0, 10, 20, etc */
8478 			queue_idx = queue_id % 10;
8479 			if (queue_idx > BNXT_MAX_QUEUE) {
8480 				bp->pri2cos_valid = false;
8481 				hwrm_req_drop(bp, req_qc);
8482 				return rc;
8483 			}
8484 			for (j = 0; j < bp->max_q; j++) {
8485 				if (bp->q_ids[j] == queue_id)
8486 					bp->pri2cos_idx[i] = queue_idx;
8487 			}
8488 		}
8489 		bp->pri2cos_valid = true;
8490 	}
8491 	hwrm_req_drop(bp, req_qc);
8492 
8493 	return rc;
8494 }
8495 
8496 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
8497 {
8498 	bnxt_hwrm_tunnel_dst_port_free(bp,
8499 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
8500 	bnxt_hwrm_tunnel_dst_port_free(bp,
8501 		TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
8502 }
8503 
8504 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
8505 {
8506 	int rc, i;
8507 	u32 tpa_flags = 0;
8508 
8509 	if (set_tpa)
8510 		tpa_flags = bp->flags & BNXT_FLAG_TPA;
8511 	else if (BNXT_NO_FW_ACCESS(bp))
8512 		return 0;
8513 	for (i = 0; i < bp->nr_vnics; i++) {
8514 		rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
8515 		if (rc) {
8516 			netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
8517 				   i, rc);
8518 			return rc;
8519 		}
8520 	}
8521 	return 0;
8522 }
8523 
8524 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
8525 {
8526 	int i;
8527 
8528 	for (i = 0; i < bp->nr_vnics; i++)
8529 		bnxt_hwrm_vnic_set_rss(bp, i, false);
8530 }
8531 
8532 static void bnxt_clear_vnic(struct bnxt *bp)
8533 {
8534 	if (!bp->vnic_info)
8535 		return;
8536 
8537 	bnxt_hwrm_clear_vnic_filter(bp);
8538 	if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
8539 		/* clear all RSS setting before free vnic ctx */
8540 		bnxt_hwrm_clear_vnic_rss(bp);
8541 		bnxt_hwrm_vnic_ctx_free(bp);
8542 	}
8543 	/* before free the vnic, undo the vnic tpa settings */
8544 	if (bp->flags & BNXT_FLAG_TPA)
8545 		bnxt_set_tpa(bp, false);
8546 	bnxt_hwrm_vnic_free(bp);
8547 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8548 		bnxt_hwrm_vnic_ctx_free(bp);
8549 }
8550 
8551 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
8552 				    bool irq_re_init)
8553 {
8554 	bnxt_clear_vnic(bp);
8555 	bnxt_hwrm_ring_free(bp, close_path);
8556 	bnxt_hwrm_ring_grp_free(bp);
8557 	if (irq_re_init) {
8558 		bnxt_hwrm_stat_ctx_free(bp);
8559 		bnxt_hwrm_free_tunnel_ports(bp);
8560 	}
8561 }
8562 
8563 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8564 {
8565 	struct hwrm_func_cfg_input *req;
8566 	u8 evb_mode;
8567 	int rc;
8568 
8569 	if (br_mode == BRIDGE_MODE_VEB)
8570 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8571 	else if (br_mode == BRIDGE_MODE_VEPA)
8572 		evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8573 	else
8574 		return -EINVAL;
8575 
8576 	rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8577 	if (rc)
8578 		return rc;
8579 
8580 	req->fid = cpu_to_le16(0xffff);
8581 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8582 	req->evb_mode = evb_mode;
8583 	return hwrm_req_send(bp, req);
8584 }
8585 
8586 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8587 {
8588 	struct hwrm_func_cfg_input *req;
8589 	int rc;
8590 
8591 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8592 		return 0;
8593 
8594 	rc = hwrm_req_init(bp, req, HWRM_FUNC_CFG);
8595 	if (rc)
8596 		return rc;
8597 
8598 	req->fid = cpu_to_le16(0xffff);
8599 	req->enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8600 	req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8601 	if (size == 128)
8602 		req->options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8603 
8604 	return hwrm_req_send(bp, req);
8605 }
8606 
8607 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8608 {
8609 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8610 	int rc;
8611 
8612 	if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8613 		goto skip_rss_ctx;
8614 
8615 	/* allocate context for vnic */
8616 	rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8617 	if (rc) {
8618 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8619 			   vnic_id, rc);
8620 		goto vnic_setup_err;
8621 	}
8622 	bp->rsscos_nr_ctxs++;
8623 
8624 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8625 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8626 		if (rc) {
8627 			netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8628 				   vnic_id, rc);
8629 			goto vnic_setup_err;
8630 		}
8631 		bp->rsscos_nr_ctxs++;
8632 	}
8633 
8634 skip_rss_ctx:
8635 	/* configure default vnic, ring grp */
8636 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8637 	if (rc) {
8638 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8639 			   vnic_id, rc);
8640 		goto vnic_setup_err;
8641 	}
8642 
8643 	/* Enable RSS hashing on vnic */
8644 	rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8645 	if (rc) {
8646 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8647 			   vnic_id, rc);
8648 		goto vnic_setup_err;
8649 	}
8650 
8651 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8652 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8653 		if (rc) {
8654 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8655 				   vnic_id, rc);
8656 		}
8657 	}
8658 
8659 vnic_setup_err:
8660 	return rc;
8661 }
8662 
8663 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8664 {
8665 	int rc, i, nr_ctxs;
8666 
8667 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8668 	for (i = 0; i < nr_ctxs; i++) {
8669 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8670 		if (rc) {
8671 			netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8672 				   vnic_id, i, rc);
8673 			break;
8674 		}
8675 		bp->rsscos_nr_ctxs++;
8676 	}
8677 	if (i < nr_ctxs)
8678 		return -ENOMEM;
8679 
8680 	rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8681 	if (rc) {
8682 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8683 			   vnic_id, rc);
8684 		return rc;
8685 	}
8686 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8687 	if (rc) {
8688 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8689 			   vnic_id, rc);
8690 		return rc;
8691 	}
8692 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8693 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8694 		if (rc) {
8695 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8696 				   vnic_id, rc);
8697 		}
8698 	}
8699 	return rc;
8700 }
8701 
8702 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8703 {
8704 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8705 		return __bnxt_setup_vnic_p5(bp, vnic_id);
8706 	else
8707 		return __bnxt_setup_vnic(bp, vnic_id);
8708 }
8709 
8710 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8711 {
8712 #ifdef CONFIG_RFS_ACCEL
8713 	int i, rc = 0;
8714 
8715 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8716 		return 0;
8717 
8718 	for (i = 0; i < bp->rx_nr_rings; i++) {
8719 		struct bnxt_vnic_info *vnic;
8720 		u16 vnic_id = i + 1;
8721 		u16 ring_id = i;
8722 
8723 		if (vnic_id >= bp->nr_vnics)
8724 			break;
8725 
8726 		vnic = &bp->vnic_info[vnic_id];
8727 		vnic->flags |= BNXT_VNIC_RFS_FLAG;
8728 		if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8729 			vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8730 		rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8731 		if (rc) {
8732 			netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8733 				   vnic_id, rc);
8734 			break;
8735 		}
8736 		rc = bnxt_setup_vnic(bp, vnic_id);
8737 		if (rc)
8738 			break;
8739 	}
8740 	return rc;
8741 #else
8742 	return 0;
8743 #endif
8744 }
8745 
8746 /* Allow PF, trusted VFs and VFs with default VLAN to be in promiscuous mode */
8747 static bool bnxt_promisc_ok(struct bnxt *bp)
8748 {
8749 #ifdef CONFIG_BNXT_SRIOV
8750 	if (BNXT_VF(bp) && !bp->vf.vlan && !bnxt_is_trusted_vf(bp, &bp->vf))
8751 		return false;
8752 #endif
8753 	return true;
8754 }
8755 
8756 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8757 {
8758 	unsigned int rc = 0;
8759 
8760 	rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8761 	if (rc) {
8762 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8763 			   rc);
8764 		return rc;
8765 	}
8766 
8767 	rc = bnxt_hwrm_vnic_cfg(bp, 1);
8768 	if (rc) {
8769 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8770 			   rc);
8771 		return rc;
8772 	}
8773 	return rc;
8774 }
8775 
8776 static int bnxt_cfg_rx_mode(struct bnxt *);
8777 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8778 
8779 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8780 {
8781 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8782 	int rc = 0;
8783 	unsigned int rx_nr_rings = bp->rx_nr_rings;
8784 
8785 	if (irq_re_init) {
8786 		rc = bnxt_hwrm_stat_ctx_alloc(bp);
8787 		if (rc) {
8788 			netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8789 				   rc);
8790 			goto err_out;
8791 		}
8792 	}
8793 
8794 	rc = bnxt_hwrm_ring_alloc(bp);
8795 	if (rc) {
8796 		netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8797 		goto err_out;
8798 	}
8799 
8800 	rc = bnxt_hwrm_ring_grp_alloc(bp);
8801 	if (rc) {
8802 		netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8803 		goto err_out;
8804 	}
8805 
8806 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8807 		rx_nr_rings--;
8808 
8809 	/* default vnic 0 */
8810 	rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8811 	if (rc) {
8812 		netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8813 		goto err_out;
8814 	}
8815 
8816 	if (BNXT_VF(bp))
8817 		bnxt_hwrm_func_qcfg(bp);
8818 
8819 	rc = bnxt_setup_vnic(bp, 0);
8820 	if (rc)
8821 		goto err_out;
8822 	if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
8823 		bnxt_hwrm_update_rss_hash_cfg(bp);
8824 
8825 	if (bp->flags & BNXT_FLAG_RFS) {
8826 		rc = bnxt_alloc_rfs_vnics(bp);
8827 		if (rc)
8828 			goto err_out;
8829 	}
8830 
8831 	if (bp->flags & BNXT_FLAG_TPA) {
8832 		rc = bnxt_set_tpa(bp, true);
8833 		if (rc)
8834 			goto err_out;
8835 	}
8836 
8837 	if (BNXT_VF(bp))
8838 		bnxt_update_vf_mac(bp);
8839 
8840 	/* Filter for default vnic 0 */
8841 	rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8842 	if (rc) {
8843 		if (BNXT_VF(bp) && rc == -ENODEV)
8844 			netdev_err(bp->dev, "Cannot configure L2 filter while PF is unavailable\n");
8845 		else
8846 			netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8847 		goto err_out;
8848 	}
8849 	vnic->uc_filter_count = 1;
8850 
8851 	vnic->rx_mask = 0;
8852 	if (test_bit(BNXT_STATE_HALF_OPEN, &bp->state))
8853 		goto skip_rx_mask;
8854 
8855 	if (bp->dev->flags & IFF_BROADCAST)
8856 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8857 
8858 	if (bp->dev->flags & IFF_PROMISC)
8859 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8860 
8861 	if (bp->dev->flags & IFF_ALLMULTI) {
8862 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8863 		vnic->mc_list_count = 0;
8864 	} else if (bp->dev->flags & IFF_MULTICAST) {
8865 		u32 mask = 0;
8866 
8867 		bnxt_mc_list_updated(bp, &mask);
8868 		vnic->rx_mask |= mask;
8869 	}
8870 
8871 	rc = bnxt_cfg_rx_mode(bp);
8872 	if (rc)
8873 		goto err_out;
8874 
8875 skip_rx_mask:
8876 	rc = bnxt_hwrm_set_coal(bp);
8877 	if (rc)
8878 		netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8879 				rc);
8880 
8881 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8882 		rc = bnxt_setup_nitroa0_vnic(bp);
8883 		if (rc)
8884 			netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8885 				   rc);
8886 	}
8887 
8888 	if (BNXT_VF(bp)) {
8889 		bnxt_hwrm_func_qcfg(bp);
8890 		netdev_update_features(bp->dev);
8891 	}
8892 
8893 	return 0;
8894 
8895 err_out:
8896 	bnxt_hwrm_resource_free(bp, 0, true);
8897 
8898 	return rc;
8899 }
8900 
8901 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8902 {
8903 	bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8904 	return 0;
8905 }
8906 
8907 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8908 {
8909 	bnxt_init_cp_rings(bp);
8910 	bnxt_init_rx_rings(bp);
8911 	bnxt_init_tx_rings(bp);
8912 	bnxt_init_ring_grps(bp, irq_re_init);
8913 	bnxt_init_vnics(bp);
8914 
8915 	return bnxt_init_chip(bp, irq_re_init);
8916 }
8917 
8918 static int bnxt_set_real_num_queues(struct bnxt *bp)
8919 {
8920 	int rc;
8921 	struct net_device *dev = bp->dev;
8922 
8923 	rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8924 					  bp->tx_nr_rings_xdp);
8925 	if (rc)
8926 		return rc;
8927 
8928 	rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8929 	if (rc)
8930 		return rc;
8931 
8932 #ifdef CONFIG_RFS_ACCEL
8933 	if (bp->flags & BNXT_FLAG_RFS)
8934 		dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8935 #endif
8936 
8937 	return rc;
8938 }
8939 
8940 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8941 			   bool shared)
8942 {
8943 	int _rx = *rx, _tx = *tx;
8944 
8945 	if (shared) {
8946 		*rx = min_t(int, _rx, max);
8947 		*tx = min_t(int, _tx, max);
8948 	} else {
8949 		if (max < 2)
8950 			return -ENOMEM;
8951 
8952 		while (_rx + _tx > max) {
8953 			if (_rx > _tx && _rx > 1)
8954 				_rx--;
8955 			else if (_tx > 1)
8956 				_tx--;
8957 		}
8958 		*rx = _rx;
8959 		*tx = _tx;
8960 	}
8961 	return 0;
8962 }
8963 
8964 static void bnxt_setup_msix(struct bnxt *bp)
8965 {
8966 	const int len = sizeof(bp->irq_tbl[0].name);
8967 	struct net_device *dev = bp->dev;
8968 	int tcs, i;
8969 
8970 	tcs = netdev_get_num_tc(dev);
8971 	if (tcs) {
8972 		int i, off, count;
8973 
8974 		for (i = 0; i < tcs; i++) {
8975 			count = bp->tx_nr_rings_per_tc;
8976 			off = i * count;
8977 			netdev_set_tc_queue(dev, i, count, off);
8978 		}
8979 	}
8980 
8981 	for (i = 0; i < bp->cp_nr_rings; i++) {
8982 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8983 		char *attr;
8984 
8985 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8986 			attr = "TxRx";
8987 		else if (i < bp->rx_nr_rings)
8988 			attr = "rx";
8989 		else
8990 			attr = "tx";
8991 
8992 		snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8993 			 attr, i);
8994 		bp->irq_tbl[map_idx].handler = bnxt_msix;
8995 	}
8996 }
8997 
8998 static void bnxt_setup_inta(struct bnxt *bp)
8999 {
9000 	const int len = sizeof(bp->irq_tbl[0].name);
9001 
9002 	if (netdev_get_num_tc(bp->dev))
9003 		netdev_reset_tc(bp->dev);
9004 
9005 	snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
9006 		 0);
9007 	bp->irq_tbl[0].handler = bnxt_inta;
9008 }
9009 
9010 static int bnxt_init_int_mode(struct bnxt *bp);
9011 
9012 static int bnxt_setup_int_mode(struct bnxt *bp)
9013 {
9014 	int rc;
9015 
9016 	if (!bp->irq_tbl) {
9017 		rc = bnxt_init_int_mode(bp);
9018 		if (rc || !bp->irq_tbl)
9019 			return rc ?: -ENODEV;
9020 	}
9021 
9022 	if (bp->flags & BNXT_FLAG_USING_MSIX)
9023 		bnxt_setup_msix(bp);
9024 	else
9025 		bnxt_setup_inta(bp);
9026 
9027 	rc = bnxt_set_real_num_queues(bp);
9028 	return rc;
9029 }
9030 
9031 #ifdef CONFIG_RFS_ACCEL
9032 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
9033 {
9034 	return bp->hw_resc.max_rsscos_ctxs;
9035 }
9036 
9037 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
9038 {
9039 	return bp->hw_resc.max_vnics;
9040 }
9041 #endif
9042 
9043 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
9044 {
9045 	return bp->hw_resc.max_stat_ctxs;
9046 }
9047 
9048 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
9049 {
9050 	return bp->hw_resc.max_cp_rings;
9051 }
9052 
9053 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
9054 {
9055 	unsigned int cp = bp->hw_resc.max_cp_rings;
9056 
9057 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9058 		cp -= bnxt_get_ulp_msix_num(bp);
9059 
9060 	return cp;
9061 }
9062 
9063 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
9064 {
9065 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9066 
9067 	if (bp->flags & BNXT_FLAG_CHIP_P5)
9068 		return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
9069 
9070 	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
9071 }
9072 
9073 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
9074 {
9075 	bp->hw_resc.max_irqs = max_irqs;
9076 }
9077 
9078 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
9079 {
9080 	unsigned int cp;
9081 
9082 	cp = bnxt_get_max_func_cp_rings_for_en(bp);
9083 	if (bp->flags & BNXT_FLAG_CHIP_P5)
9084 		return cp - bp->rx_nr_rings - bp->tx_nr_rings;
9085 	else
9086 		return cp - bp->cp_nr_rings;
9087 }
9088 
9089 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
9090 {
9091 	return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
9092 }
9093 
9094 int bnxt_get_avail_msix(struct bnxt *bp, int num)
9095 {
9096 	int max_cp = bnxt_get_max_func_cp_rings(bp);
9097 	int max_irq = bnxt_get_max_func_irqs(bp);
9098 	int total_req = bp->cp_nr_rings + num;
9099 	int max_idx, avail_msix;
9100 
9101 	max_idx = bp->total_irqs;
9102 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
9103 		max_idx = min_t(int, bp->total_irqs, max_cp);
9104 	avail_msix = max_idx - bp->cp_nr_rings;
9105 	if (!BNXT_NEW_RM(bp) || avail_msix >= num)
9106 		return avail_msix;
9107 
9108 	if (max_irq < total_req) {
9109 		num = max_irq - bp->cp_nr_rings;
9110 		if (num <= 0)
9111 			return 0;
9112 	}
9113 	return num;
9114 }
9115 
9116 static int bnxt_get_num_msix(struct bnxt *bp)
9117 {
9118 	if (!BNXT_NEW_RM(bp))
9119 		return bnxt_get_max_func_irqs(bp);
9120 
9121 	return bnxt_nq_rings_in_use(bp);
9122 }
9123 
9124 static int bnxt_init_msix(struct bnxt *bp)
9125 {
9126 	int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
9127 	struct msix_entry *msix_ent;
9128 
9129 	total_vecs = bnxt_get_num_msix(bp);
9130 	max = bnxt_get_max_func_irqs(bp);
9131 	if (total_vecs > max)
9132 		total_vecs = max;
9133 
9134 	if (!total_vecs)
9135 		return 0;
9136 
9137 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
9138 	if (!msix_ent)
9139 		return -ENOMEM;
9140 
9141 	for (i = 0; i < total_vecs; i++) {
9142 		msix_ent[i].entry = i;
9143 		msix_ent[i].vector = 0;
9144 	}
9145 
9146 	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
9147 		min = 2;
9148 
9149 	total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
9150 	ulp_msix = bnxt_get_ulp_msix_num(bp);
9151 	if (total_vecs < 0 || total_vecs < ulp_msix) {
9152 		rc = -ENODEV;
9153 		goto msix_setup_exit;
9154 	}
9155 
9156 	bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
9157 	if (bp->irq_tbl) {
9158 		for (i = 0; i < total_vecs; i++)
9159 			bp->irq_tbl[i].vector = msix_ent[i].vector;
9160 
9161 		bp->total_irqs = total_vecs;
9162 		/* Trim rings based upon num of vectors allocated */
9163 		rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
9164 				     total_vecs - ulp_msix, min == 1);
9165 		if (rc)
9166 			goto msix_setup_exit;
9167 
9168 		bp->cp_nr_rings = (min == 1) ?
9169 				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
9170 				  bp->tx_nr_rings + bp->rx_nr_rings;
9171 
9172 	} else {
9173 		rc = -ENOMEM;
9174 		goto msix_setup_exit;
9175 	}
9176 	bp->flags |= BNXT_FLAG_USING_MSIX;
9177 	kfree(msix_ent);
9178 	return 0;
9179 
9180 msix_setup_exit:
9181 	netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
9182 	kfree(bp->irq_tbl);
9183 	bp->irq_tbl = NULL;
9184 	pci_disable_msix(bp->pdev);
9185 	kfree(msix_ent);
9186 	return rc;
9187 }
9188 
9189 static int bnxt_init_inta(struct bnxt *bp)
9190 {
9191 	bp->irq_tbl = kzalloc(sizeof(struct bnxt_irq), GFP_KERNEL);
9192 	if (!bp->irq_tbl)
9193 		return -ENOMEM;
9194 
9195 	bp->total_irqs = 1;
9196 	bp->rx_nr_rings = 1;
9197 	bp->tx_nr_rings = 1;
9198 	bp->cp_nr_rings = 1;
9199 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
9200 	bp->irq_tbl[0].vector = bp->pdev->irq;
9201 	return 0;
9202 }
9203 
9204 static int bnxt_init_int_mode(struct bnxt *bp)
9205 {
9206 	int rc = -ENODEV;
9207 
9208 	if (bp->flags & BNXT_FLAG_MSIX_CAP)
9209 		rc = bnxt_init_msix(bp);
9210 
9211 	if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
9212 		/* fallback to INTA */
9213 		rc = bnxt_init_inta(bp);
9214 	}
9215 	return rc;
9216 }
9217 
9218 static void bnxt_clear_int_mode(struct bnxt *bp)
9219 {
9220 	if (bp->flags & BNXT_FLAG_USING_MSIX)
9221 		pci_disable_msix(bp->pdev);
9222 
9223 	kfree(bp->irq_tbl);
9224 	bp->irq_tbl = NULL;
9225 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
9226 }
9227 
9228 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
9229 {
9230 	int tcs = netdev_get_num_tc(bp->dev);
9231 	bool irq_cleared = false;
9232 	int rc;
9233 
9234 	if (!bnxt_need_reserve_rings(bp))
9235 		return 0;
9236 
9237 	if (irq_re_init && BNXT_NEW_RM(bp) &&
9238 	    bnxt_get_num_msix(bp) != bp->total_irqs) {
9239 		bnxt_ulp_irq_stop(bp);
9240 		bnxt_clear_int_mode(bp);
9241 		irq_cleared = true;
9242 	}
9243 	rc = __bnxt_reserve_rings(bp);
9244 	if (irq_cleared) {
9245 		if (!rc)
9246 			rc = bnxt_init_int_mode(bp);
9247 		bnxt_ulp_irq_restart(bp, rc);
9248 	}
9249 	if (rc) {
9250 		netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
9251 		return rc;
9252 	}
9253 	if (tcs && (bp->tx_nr_rings_per_tc * tcs !=
9254 		    bp->tx_nr_rings - bp->tx_nr_rings_xdp)) {
9255 		netdev_err(bp->dev, "tx ring reservation failure\n");
9256 		netdev_reset_tc(bp->dev);
9257 		if (bp->tx_nr_rings_xdp)
9258 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings_xdp;
9259 		else
9260 			bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
9261 		return -ENOMEM;
9262 	}
9263 	return 0;
9264 }
9265 
9266 static void bnxt_free_irq(struct bnxt *bp)
9267 {
9268 	struct bnxt_irq *irq;
9269 	int i;
9270 
9271 #ifdef CONFIG_RFS_ACCEL
9272 	free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
9273 	bp->dev->rx_cpu_rmap = NULL;
9274 #endif
9275 	if (!bp->irq_tbl || !bp->bnapi)
9276 		return;
9277 
9278 	for (i = 0; i < bp->cp_nr_rings; i++) {
9279 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9280 
9281 		irq = &bp->irq_tbl[map_idx];
9282 		if (irq->requested) {
9283 			if (irq->have_cpumask) {
9284 				irq_set_affinity_hint(irq->vector, NULL);
9285 				free_cpumask_var(irq->cpu_mask);
9286 				irq->have_cpumask = 0;
9287 			}
9288 			free_irq(irq->vector, bp->bnapi[i]);
9289 		}
9290 
9291 		irq->requested = 0;
9292 	}
9293 }
9294 
9295 static int bnxt_request_irq(struct bnxt *bp)
9296 {
9297 	int i, j, rc = 0;
9298 	unsigned long flags = 0;
9299 #ifdef CONFIG_RFS_ACCEL
9300 	struct cpu_rmap *rmap;
9301 #endif
9302 
9303 	rc = bnxt_setup_int_mode(bp);
9304 	if (rc) {
9305 		netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
9306 			   rc);
9307 		return rc;
9308 	}
9309 #ifdef CONFIG_RFS_ACCEL
9310 	rmap = bp->dev->rx_cpu_rmap;
9311 #endif
9312 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
9313 		flags = IRQF_SHARED;
9314 
9315 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
9316 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
9317 		struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
9318 
9319 #ifdef CONFIG_RFS_ACCEL
9320 		if (rmap && bp->bnapi[i]->rx_ring) {
9321 			rc = irq_cpu_rmap_add(rmap, irq->vector);
9322 			if (rc)
9323 				netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
9324 					    j);
9325 			j++;
9326 		}
9327 #endif
9328 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
9329 				 bp->bnapi[i]);
9330 		if (rc)
9331 			break;
9332 
9333 		irq->requested = 1;
9334 
9335 		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
9336 			int numa_node = dev_to_node(&bp->pdev->dev);
9337 
9338 			irq->have_cpumask = 1;
9339 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
9340 					irq->cpu_mask);
9341 			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
9342 			if (rc) {
9343 				netdev_warn(bp->dev,
9344 					    "Set affinity failed, IRQ = %d\n",
9345 					    irq->vector);
9346 				break;
9347 			}
9348 		}
9349 	}
9350 	return rc;
9351 }
9352 
9353 static void bnxt_del_napi(struct bnxt *bp)
9354 {
9355 	int i;
9356 
9357 	if (!bp->bnapi)
9358 		return;
9359 
9360 	for (i = 0; i < bp->cp_nr_rings; i++) {
9361 		struct bnxt_napi *bnapi = bp->bnapi[i];
9362 
9363 		__netif_napi_del(&bnapi->napi);
9364 	}
9365 	/* We called __netif_napi_del(), we need
9366 	 * to respect an RCU grace period before freeing napi structures.
9367 	 */
9368 	synchronize_net();
9369 }
9370 
9371 static void bnxt_init_napi(struct bnxt *bp)
9372 {
9373 	int i;
9374 	unsigned int cp_nr_rings = bp->cp_nr_rings;
9375 	struct bnxt_napi *bnapi;
9376 
9377 	if (bp->flags & BNXT_FLAG_USING_MSIX) {
9378 		int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
9379 
9380 		if (bp->flags & BNXT_FLAG_CHIP_P5)
9381 			poll_fn = bnxt_poll_p5;
9382 		else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
9383 			cp_nr_rings--;
9384 		for (i = 0; i < cp_nr_rings; i++) {
9385 			bnapi = bp->bnapi[i];
9386 			netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
9387 		}
9388 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
9389 			bnapi = bp->bnapi[cp_nr_rings];
9390 			netif_napi_add(bp->dev, &bnapi->napi,
9391 				       bnxt_poll_nitroa0);
9392 		}
9393 	} else {
9394 		bnapi = bp->bnapi[0];
9395 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
9396 	}
9397 }
9398 
9399 static void bnxt_disable_napi(struct bnxt *bp)
9400 {
9401 	int i;
9402 
9403 	if (!bp->bnapi ||
9404 	    test_and_set_bit(BNXT_STATE_NAPI_DISABLED, &bp->state))
9405 		return;
9406 
9407 	for (i = 0; i < bp->cp_nr_rings; i++) {
9408 		struct bnxt_napi *bnapi = bp->bnapi[i];
9409 		struct bnxt_cp_ring_info *cpr;
9410 
9411 		cpr = &bnapi->cp_ring;
9412 		if (bnapi->tx_fault)
9413 			cpr->sw_stats.tx.tx_resets++;
9414 		if (bnapi->in_reset)
9415 			cpr->sw_stats.rx.rx_resets++;
9416 		napi_disable(&bnapi->napi);
9417 		if (bnapi->rx_ring)
9418 			cancel_work_sync(&cpr->dim.work);
9419 	}
9420 }
9421 
9422 static void bnxt_enable_napi(struct bnxt *bp)
9423 {
9424 	int i;
9425 
9426 	clear_bit(BNXT_STATE_NAPI_DISABLED, &bp->state);
9427 	for (i = 0; i < bp->cp_nr_rings; i++) {
9428 		struct bnxt_napi *bnapi = bp->bnapi[i];
9429 		struct bnxt_cp_ring_info *cpr;
9430 
9431 		bnapi->tx_fault = 0;
9432 
9433 		cpr = &bnapi->cp_ring;
9434 		bnapi->in_reset = false;
9435 
9436 		bnapi->tx_pkts = 0;
9437 
9438 		if (bnapi->rx_ring) {
9439 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
9440 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
9441 		}
9442 		napi_enable(&bnapi->napi);
9443 	}
9444 }
9445 
9446 void bnxt_tx_disable(struct bnxt *bp)
9447 {
9448 	int i;
9449 	struct bnxt_tx_ring_info *txr;
9450 
9451 	if (bp->tx_ring) {
9452 		for (i = 0; i < bp->tx_nr_rings; i++) {
9453 			txr = &bp->tx_ring[i];
9454 			WRITE_ONCE(txr->dev_state, BNXT_DEV_STATE_CLOSING);
9455 		}
9456 	}
9457 	/* Make sure napi polls see @dev_state change */
9458 	synchronize_net();
9459 	/* Drop carrier first to prevent TX timeout */
9460 	netif_carrier_off(bp->dev);
9461 	/* Stop all TX queues */
9462 	netif_tx_disable(bp->dev);
9463 }
9464 
9465 void bnxt_tx_enable(struct bnxt *bp)
9466 {
9467 	int i;
9468 	struct bnxt_tx_ring_info *txr;
9469 
9470 	for (i = 0; i < bp->tx_nr_rings; i++) {
9471 		txr = &bp->tx_ring[i];
9472 		WRITE_ONCE(txr->dev_state, 0);
9473 	}
9474 	/* Make sure napi polls see @dev_state change */
9475 	synchronize_net();
9476 	netif_tx_wake_all_queues(bp->dev);
9477 	if (BNXT_LINK_IS_UP(bp))
9478 		netif_carrier_on(bp->dev);
9479 }
9480 
9481 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
9482 {
9483 	u8 active_fec = link_info->active_fec_sig_mode &
9484 			PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
9485 
9486 	switch (active_fec) {
9487 	default:
9488 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
9489 		return "None";
9490 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
9491 		return "Clause 74 BaseR";
9492 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
9493 		return "Clause 91 RS(528,514)";
9494 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
9495 		return "Clause 91 RS544_1XN";
9496 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
9497 		return "Clause 91 RS(544,514)";
9498 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
9499 		return "Clause 91 RS272_1XN";
9500 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
9501 		return "Clause 91 RS(272,257)";
9502 	}
9503 }
9504 
9505 void bnxt_report_link(struct bnxt *bp)
9506 {
9507 	if (BNXT_LINK_IS_UP(bp)) {
9508 		const char *signal = "";
9509 		const char *flow_ctrl;
9510 		const char *duplex;
9511 		u32 speed;
9512 		u16 fec;
9513 
9514 		netif_carrier_on(bp->dev);
9515 		speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
9516 		if (speed == SPEED_UNKNOWN) {
9517 			netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
9518 			return;
9519 		}
9520 		if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
9521 			duplex = "full";
9522 		else
9523 			duplex = "half";
9524 		if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
9525 			flow_ctrl = "ON - receive & transmit";
9526 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
9527 			flow_ctrl = "ON - transmit";
9528 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
9529 			flow_ctrl = "ON - receive";
9530 		else
9531 			flow_ctrl = "none";
9532 		if (bp->link_info.phy_qcfg_resp.option_flags &
9533 		    PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) {
9534 			u8 sig_mode = bp->link_info.active_fec_sig_mode &
9535 				      PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK;
9536 			switch (sig_mode) {
9537 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ:
9538 				signal = "(NRZ) ";
9539 				break;
9540 			case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4:
9541 				signal = "(PAM4) ";
9542 				break;
9543 			default:
9544 				break;
9545 			}
9546 		}
9547 		netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n",
9548 			    speed, signal, duplex, flow_ctrl);
9549 		if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP)
9550 			netdev_info(bp->dev, "EEE is %s\n",
9551 				    bp->eee.eee_active ? "active" :
9552 							 "not active");
9553 		fec = bp->link_info.fec_cfg;
9554 		if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
9555 			netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
9556 				    (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
9557 				    bnxt_report_fec(&bp->link_info));
9558 	} else {
9559 		netif_carrier_off(bp->dev);
9560 		netdev_err(bp->dev, "NIC Link is Down\n");
9561 	}
9562 }
9563 
9564 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
9565 {
9566 	if (!resp->supported_speeds_auto_mode &&
9567 	    !resp->supported_speeds_force_mode &&
9568 	    !resp->supported_pam4_speeds_auto_mode &&
9569 	    !resp->supported_pam4_speeds_force_mode)
9570 		return true;
9571 	return false;
9572 }
9573 
9574 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
9575 {
9576 	struct bnxt_link_info *link_info = &bp->link_info;
9577 	struct hwrm_port_phy_qcaps_output *resp;
9578 	struct hwrm_port_phy_qcaps_input *req;
9579 	int rc = 0;
9580 
9581 	if (bp->hwrm_spec_code < 0x10201)
9582 		return 0;
9583 
9584 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCAPS);
9585 	if (rc)
9586 		return rc;
9587 
9588 	resp = hwrm_req_hold(bp, req);
9589 	rc = hwrm_req_send(bp, req);
9590 	if (rc)
9591 		goto hwrm_phy_qcaps_exit;
9592 
9593 	bp->phy_flags = resp->flags | (le16_to_cpu(resp->flags2) << 8);
9594 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
9595 		struct ethtool_eee *eee = &bp->eee;
9596 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
9597 
9598 		eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9599 		bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
9600 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
9601 		bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
9602 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
9603 	}
9604 
9605 	if (bp->hwrm_spec_code >= 0x10a01) {
9606 		if (bnxt_phy_qcaps_no_speed(resp)) {
9607 			link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9608 			netdev_warn(bp->dev, "Ethernet link disabled\n");
9609 		} else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9610 			link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9611 			netdev_info(bp->dev, "Ethernet link enabled\n");
9612 			/* Phy re-enabled, reprobe the speeds */
9613 			link_info->support_auto_speeds = 0;
9614 			link_info->support_pam4_auto_speeds = 0;
9615 		}
9616 	}
9617 	if (resp->supported_speeds_auto_mode)
9618 		link_info->support_auto_speeds =
9619 			le16_to_cpu(resp->supported_speeds_auto_mode);
9620 	if (resp->supported_pam4_speeds_auto_mode)
9621 		link_info->support_pam4_auto_speeds =
9622 			le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9623 
9624 	bp->port_count = resp->port_cnt;
9625 
9626 hwrm_phy_qcaps_exit:
9627 	hwrm_req_drop(bp, req);
9628 	return rc;
9629 }
9630 
9631 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9632 {
9633 	u16 diff = advertising ^ supported;
9634 
9635 	return ((supported | diff) != supported);
9636 }
9637 
9638 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9639 {
9640 	struct bnxt_link_info *link_info = &bp->link_info;
9641 	struct hwrm_port_phy_qcfg_output *resp;
9642 	struct hwrm_port_phy_qcfg_input *req;
9643 	u8 link_state = link_info->link_state;
9644 	bool support_changed = false;
9645 	int rc;
9646 
9647 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_QCFG);
9648 	if (rc)
9649 		return rc;
9650 
9651 	resp = hwrm_req_hold(bp, req);
9652 	rc = hwrm_req_send(bp, req);
9653 	if (rc) {
9654 		hwrm_req_drop(bp, req);
9655 		if (BNXT_VF(bp) && rc == -ENODEV) {
9656 			netdev_warn(bp->dev, "Cannot obtain link state while PF unavailable.\n");
9657 			rc = 0;
9658 		}
9659 		return rc;
9660 	}
9661 
9662 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9663 	link_info->phy_link_status = resp->link;
9664 	link_info->duplex = resp->duplex_cfg;
9665 	if (bp->hwrm_spec_code >= 0x10800)
9666 		link_info->duplex = resp->duplex_state;
9667 	link_info->pause = resp->pause;
9668 	link_info->auto_mode = resp->auto_mode;
9669 	link_info->auto_pause_setting = resp->auto_pause;
9670 	link_info->lp_pause = resp->link_partner_adv_pause;
9671 	link_info->force_pause_setting = resp->force_pause;
9672 	link_info->duplex_setting = resp->duplex_cfg;
9673 	if (link_info->phy_link_status == BNXT_LINK_LINK)
9674 		link_info->link_speed = le16_to_cpu(resp->link_speed);
9675 	else
9676 		link_info->link_speed = 0;
9677 	link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9678 	link_info->force_pam4_link_speed =
9679 		le16_to_cpu(resp->force_pam4_link_speed);
9680 	link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9681 	link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9682 	link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9683 	link_info->auto_pam4_link_speeds =
9684 		le16_to_cpu(resp->auto_pam4_link_speed_mask);
9685 	link_info->lp_auto_link_speeds =
9686 		le16_to_cpu(resp->link_partner_adv_speeds);
9687 	link_info->lp_auto_pam4_link_speeds =
9688 		resp->link_partner_pam4_adv_speeds;
9689 	link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9690 	link_info->phy_ver[0] = resp->phy_maj;
9691 	link_info->phy_ver[1] = resp->phy_min;
9692 	link_info->phy_ver[2] = resp->phy_bld;
9693 	link_info->media_type = resp->media_type;
9694 	link_info->phy_type = resp->phy_type;
9695 	link_info->transceiver = resp->xcvr_pkg_type;
9696 	link_info->phy_addr = resp->eee_config_phy_addr &
9697 			      PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9698 	link_info->module_status = resp->module_status;
9699 
9700 	if (bp->phy_flags & BNXT_PHY_FL_EEE_CAP) {
9701 		struct ethtool_eee *eee = &bp->eee;
9702 		u16 fw_speeds;
9703 
9704 		eee->eee_active = 0;
9705 		if (resp->eee_config_phy_addr &
9706 		    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9707 			eee->eee_active = 1;
9708 			fw_speeds = le16_to_cpu(
9709 				resp->link_partner_adv_eee_link_speed_mask);
9710 			eee->lp_advertised =
9711 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9712 		}
9713 
9714 		/* Pull initial EEE config */
9715 		if (!chng_link_state) {
9716 			if (resp->eee_config_phy_addr &
9717 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9718 				eee->eee_enabled = 1;
9719 
9720 			fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9721 			eee->advertised =
9722 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9723 
9724 			if (resp->eee_config_phy_addr &
9725 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9726 				__le32 tmr;
9727 
9728 				eee->tx_lpi_enabled = 1;
9729 				tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9730 				eee->tx_lpi_timer = le32_to_cpu(tmr) &
9731 					PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9732 			}
9733 		}
9734 	}
9735 
9736 	link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9737 	if (bp->hwrm_spec_code >= 0x10504) {
9738 		link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9739 		link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9740 	}
9741 	/* TODO: need to add more logic to report VF link */
9742 	if (chng_link_state) {
9743 		if (link_info->phy_link_status == BNXT_LINK_LINK)
9744 			link_info->link_state = BNXT_LINK_STATE_UP;
9745 		else
9746 			link_info->link_state = BNXT_LINK_STATE_DOWN;
9747 		if (link_state != link_info->link_state)
9748 			bnxt_report_link(bp);
9749 	} else {
9750 		/* always link down if not require to update link state */
9751 		link_info->link_state = BNXT_LINK_STATE_DOWN;
9752 	}
9753 	hwrm_req_drop(bp, req);
9754 
9755 	if (!BNXT_PHY_CFG_ABLE(bp))
9756 		return 0;
9757 
9758 	/* Check if any advertised speeds are no longer supported. The caller
9759 	 * holds the link_lock mutex, so we can modify link_info settings.
9760 	 */
9761 	if (bnxt_support_dropped(link_info->advertising,
9762 				 link_info->support_auto_speeds)) {
9763 		link_info->advertising = link_info->support_auto_speeds;
9764 		support_changed = true;
9765 	}
9766 	if (bnxt_support_dropped(link_info->advertising_pam4,
9767 				 link_info->support_pam4_auto_speeds)) {
9768 		link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9769 		support_changed = true;
9770 	}
9771 	if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9772 		bnxt_hwrm_set_link_setting(bp, true, false);
9773 	return 0;
9774 }
9775 
9776 static void bnxt_get_port_module_status(struct bnxt *bp)
9777 {
9778 	struct bnxt_link_info *link_info = &bp->link_info;
9779 	struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9780 	u8 module_status;
9781 
9782 	if (bnxt_update_link(bp, true))
9783 		return;
9784 
9785 	module_status = link_info->module_status;
9786 	switch (module_status) {
9787 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9788 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9789 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9790 		netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9791 			    bp->pf.port_id);
9792 		if (bp->hwrm_spec_code >= 0x10201) {
9793 			netdev_warn(bp->dev, "Module part number %s\n",
9794 				    resp->phy_vendor_partnumber);
9795 		}
9796 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9797 			netdev_warn(bp->dev, "TX is disabled\n");
9798 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9799 			netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9800 	}
9801 }
9802 
9803 static void
9804 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9805 {
9806 	if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9807 		if (bp->hwrm_spec_code >= 0x10201)
9808 			req->auto_pause =
9809 				PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9810 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9811 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9812 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9813 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9814 		req->enables |=
9815 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9816 	} else {
9817 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9818 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9819 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9820 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9821 		req->enables |=
9822 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9823 		if (bp->hwrm_spec_code >= 0x10201) {
9824 			req->auto_pause = req->force_pause;
9825 			req->enables |= cpu_to_le32(
9826 				PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9827 		}
9828 	}
9829 }
9830 
9831 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9832 {
9833 	if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9834 		req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9835 		if (bp->link_info.advertising) {
9836 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9837 			req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9838 		}
9839 		if (bp->link_info.advertising_pam4) {
9840 			req->enables |=
9841 				cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9842 			req->auto_link_pam4_speed_mask =
9843 				cpu_to_le16(bp->link_info.advertising_pam4);
9844 		}
9845 		req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9846 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9847 	} else {
9848 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9849 		if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9850 			req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9851 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9852 		} else {
9853 			req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9854 		}
9855 	}
9856 
9857 	/* tell chimp that the setting takes effect immediately */
9858 	req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9859 }
9860 
9861 int bnxt_hwrm_set_pause(struct bnxt *bp)
9862 {
9863 	struct hwrm_port_phy_cfg_input *req;
9864 	int rc;
9865 
9866 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9867 	if (rc)
9868 		return rc;
9869 
9870 	bnxt_hwrm_set_pause_common(bp, req);
9871 
9872 	if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9873 	    bp->link_info.force_link_chng)
9874 		bnxt_hwrm_set_link_common(bp, req);
9875 
9876 	rc = hwrm_req_send(bp, req);
9877 	if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9878 		/* since changing of pause setting doesn't trigger any link
9879 		 * change event, the driver needs to update the current pause
9880 		 * result upon successfully return of the phy_cfg command
9881 		 */
9882 		bp->link_info.pause =
9883 		bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9884 		bp->link_info.auto_pause_setting = 0;
9885 		if (!bp->link_info.force_link_chng)
9886 			bnxt_report_link(bp);
9887 	}
9888 	bp->link_info.force_link_chng = false;
9889 	return rc;
9890 }
9891 
9892 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9893 			      struct hwrm_port_phy_cfg_input *req)
9894 {
9895 	struct ethtool_eee *eee = &bp->eee;
9896 
9897 	if (eee->eee_enabled) {
9898 		u16 eee_speeds;
9899 		u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9900 
9901 		if (eee->tx_lpi_enabled)
9902 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9903 		else
9904 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9905 
9906 		req->flags |= cpu_to_le32(flags);
9907 		eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9908 		req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9909 		req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9910 	} else {
9911 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9912 	}
9913 }
9914 
9915 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9916 {
9917 	struct hwrm_port_phy_cfg_input *req;
9918 	int rc;
9919 
9920 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9921 	if (rc)
9922 		return rc;
9923 
9924 	if (set_pause)
9925 		bnxt_hwrm_set_pause_common(bp, req);
9926 
9927 	bnxt_hwrm_set_link_common(bp, req);
9928 
9929 	if (set_eee)
9930 		bnxt_hwrm_set_eee(bp, req);
9931 	return hwrm_req_send(bp, req);
9932 }
9933 
9934 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9935 {
9936 	struct hwrm_port_phy_cfg_input *req;
9937 	int rc;
9938 
9939 	if (!BNXT_SINGLE_PF(bp))
9940 		return 0;
9941 
9942 	if (pci_num_vf(bp->pdev) &&
9943 	    !(bp->phy_flags & BNXT_PHY_FL_FW_MANAGED_LKDN))
9944 		return 0;
9945 
9946 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_CFG);
9947 	if (rc)
9948 		return rc;
9949 
9950 	req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9951 	rc = hwrm_req_send(bp, req);
9952 	if (!rc) {
9953 		mutex_lock(&bp->link_lock);
9954 		/* Device is not obliged link down in certain scenarios, even
9955 		 * when forced. Setting the state unknown is consistent with
9956 		 * driver startup and will force link state to be reported
9957 		 * during subsequent open based on PORT_PHY_QCFG.
9958 		 */
9959 		bp->link_info.link_state = BNXT_LINK_STATE_UNKNOWN;
9960 		mutex_unlock(&bp->link_lock);
9961 	}
9962 	return rc;
9963 }
9964 
9965 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
9966 {
9967 #ifdef CONFIG_TEE_BNXT_FW
9968 	int rc = tee_bnxt_fw_load();
9969 
9970 	if (rc)
9971 		netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
9972 
9973 	return rc;
9974 #else
9975 	netdev_err(bp->dev, "OP-TEE not supported\n");
9976 	return -ENODEV;
9977 #endif
9978 }
9979 
9980 static int bnxt_try_recover_fw(struct bnxt *bp)
9981 {
9982 	if (bp->fw_health && bp->fw_health->status_reliable) {
9983 		int retry = 0, rc;
9984 		u32 sts;
9985 
9986 		do {
9987 			sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
9988 			rc = bnxt_hwrm_poll(bp);
9989 			if (!BNXT_FW_IS_BOOTING(sts) &&
9990 			    !BNXT_FW_IS_RECOVERING(sts))
9991 				break;
9992 			retry++;
9993 		} while (rc == -EBUSY && retry < BNXT_FW_RETRY);
9994 
9995 		if (!BNXT_FW_IS_HEALTHY(sts)) {
9996 			netdev_err(bp->dev,
9997 				   "Firmware not responding, status: 0x%x\n",
9998 				   sts);
9999 			rc = -ENODEV;
10000 		}
10001 		if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
10002 			netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
10003 			return bnxt_fw_reset_via_optee(bp);
10004 		}
10005 		return rc;
10006 	}
10007 
10008 	return -ENODEV;
10009 }
10010 
10011 static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset)
10012 {
10013 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10014 
10015 	if (!BNXT_NEW_RM(bp))
10016 		return; /* no resource reservations required */
10017 
10018 	hw_resc->resv_cp_rings = 0;
10019 	hw_resc->resv_stat_ctxs = 0;
10020 	hw_resc->resv_irqs = 0;
10021 	hw_resc->resv_tx_rings = 0;
10022 	hw_resc->resv_rx_rings = 0;
10023 	hw_resc->resv_hw_ring_grps = 0;
10024 	hw_resc->resv_vnics = 0;
10025 	if (!fw_reset) {
10026 		bp->tx_nr_rings = 0;
10027 		bp->rx_nr_rings = 0;
10028 	}
10029 }
10030 
10031 int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset)
10032 {
10033 	int rc;
10034 
10035 	if (!BNXT_NEW_RM(bp))
10036 		return 0; /* no resource reservations required */
10037 
10038 	rc = bnxt_hwrm_func_resc_qcaps(bp, true);
10039 	if (rc)
10040 		netdev_err(bp->dev, "resc_qcaps failed\n");
10041 
10042 	bnxt_clear_reservations(bp, fw_reset);
10043 
10044 	return rc;
10045 }
10046 
10047 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
10048 {
10049 	struct hwrm_func_drv_if_change_output *resp;
10050 	struct hwrm_func_drv_if_change_input *req;
10051 	bool fw_reset = !bp->irq_tbl;
10052 	bool resc_reinit = false;
10053 	int rc, retry = 0;
10054 	u32 flags = 0;
10055 
10056 	if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
10057 		return 0;
10058 
10059 	rc = hwrm_req_init(bp, req, HWRM_FUNC_DRV_IF_CHANGE);
10060 	if (rc)
10061 		return rc;
10062 
10063 	if (up)
10064 		req->flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
10065 	resp = hwrm_req_hold(bp, req);
10066 
10067 	hwrm_req_flags(bp, req, BNXT_HWRM_FULL_WAIT);
10068 	while (retry < BNXT_FW_IF_RETRY) {
10069 		rc = hwrm_req_send(bp, req);
10070 		if (rc != -EAGAIN)
10071 			break;
10072 
10073 		msleep(50);
10074 		retry++;
10075 	}
10076 
10077 	if (rc == -EAGAIN) {
10078 		hwrm_req_drop(bp, req);
10079 		return rc;
10080 	} else if (!rc) {
10081 		flags = le32_to_cpu(resp->flags);
10082 	} else if (up) {
10083 		rc = bnxt_try_recover_fw(bp);
10084 		fw_reset = true;
10085 	}
10086 	hwrm_req_drop(bp, req);
10087 	if (rc)
10088 		return rc;
10089 
10090 	if (!up) {
10091 		bnxt_inv_fw_health_reg(bp);
10092 		return 0;
10093 	}
10094 
10095 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
10096 		resc_reinit = true;
10097 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE ||
10098 	    test_bit(BNXT_STATE_FW_RESET_DET, &bp->state))
10099 		fw_reset = true;
10100 	else
10101 		bnxt_remap_fw_health_regs(bp);
10102 
10103 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
10104 		netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
10105 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10106 		return -ENODEV;
10107 	}
10108 	if (resc_reinit || fw_reset) {
10109 		if (fw_reset) {
10110 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10111 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10112 				bnxt_ulp_stop(bp);
10113 			bnxt_free_ctx_mem(bp);
10114 			kfree(bp->ctx);
10115 			bp->ctx = NULL;
10116 			bnxt_dcb_free(bp);
10117 			rc = bnxt_fw_init_one(bp);
10118 			if (rc) {
10119 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10120 				set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10121 				return rc;
10122 			}
10123 			bnxt_clear_int_mode(bp);
10124 			rc = bnxt_init_int_mode(bp);
10125 			if (rc) {
10126 				clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10127 				netdev_err(bp->dev, "init int mode failed\n");
10128 				return rc;
10129 			}
10130 		}
10131 		rc = bnxt_cancel_reservations(bp, fw_reset);
10132 	}
10133 	return rc;
10134 }
10135 
10136 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
10137 {
10138 	struct hwrm_port_led_qcaps_output *resp;
10139 	struct hwrm_port_led_qcaps_input *req;
10140 	struct bnxt_pf_info *pf = &bp->pf;
10141 	int rc;
10142 
10143 	bp->num_leds = 0;
10144 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
10145 		return 0;
10146 
10147 	rc = hwrm_req_init(bp, req, HWRM_PORT_LED_QCAPS);
10148 	if (rc)
10149 		return rc;
10150 
10151 	req->port_id = cpu_to_le16(pf->port_id);
10152 	resp = hwrm_req_hold(bp, req);
10153 	rc = hwrm_req_send(bp, req);
10154 	if (rc) {
10155 		hwrm_req_drop(bp, req);
10156 		return rc;
10157 	}
10158 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
10159 		int i;
10160 
10161 		bp->num_leds = resp->num_leds;
10162 		memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
10163 						 bp->num_leds);
10164 		for (i = 0; i < bp->num_leds; i++) {
10165 			struct bnxt_led_info *led = &bp->leds[i];
10166 			__le16 caps = led->led_state_caps;
10167 
10168 			if (!led->led_group_id ||
10169 			    !BNXT_LED_ALT_BLINK_CAP(caps)) {
10170 				bp->num_leds = 0;
10171 				break;
10172 			}
10173 		}
10174 	}
10175 	hwrm_req_drop(bp, req);
10176 	return 0;
10177 }
10178 
10179 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
10180 {
10181 	struct hwrm_wol_filter_alloc_output *resp;
10182 	struct hwrm_wol_filter_alloc_input *req;
10183 	int rc;
10184 
10185 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_ALLOC);
10186 	if (rc)
10187 		return rc;
10188 
10189 	req->port_id = cpu_to_le16(bp->pf.port_id);
10190 	req->wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
10191 	req->enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
10192 	memcpy(req->mac_address, bp->dev->dev_addr, ETH_ALEN);
10193 
10194 	resp = hwrm_req_hold(bp, req);
10195 	rc = hwrm_req_send(bp, req);
10196 	if (!rc)
10197 		bp->wol_filter_id = resp->wol_filter_id;
10198 	hwrm_req_drop(bp, req);
10199 	return rc;
10200 }
10201 
10202 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
10203 {
10204 	struct hwrm_wol_filter_free_input *req;
10205 	int rc;
10206 
10207 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_FREE);
10208 	if (rc)
10209 		return rc;
10210 
10211 	req->port_id = cpu_to_le16(bp->pf.port_id);
10212 	req->enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
10213 	req->wol_filter_id = bp->wol_filter_id;
10214 
10215 	return hwrm_req_send(bp, req);
10216 }
10217 
10218 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
10219 {
10220 	struct hwrm_wol_filter_qcfg_output *resp;
10221 	struct hwrm_wol_filter_qcfg_input *req;
10222 	u16 next_handle = 0;
10223 	int rc;
10224 
10225 	rc = hwrm_req_init(bp, req, HWRM_WOL_FILTER_QCFG);
10226 	if (rc)
10227 		return rc;
10228 
10229 	req->port_id = cpu_to_le16(bp->pf.port_id);
10230 	req->handle = cpu_to_le16(handle);
10231 	resp = hwrm_req_hold(bp, req);
10232 	rc = hwrm_req_send(bp, req);
10233 	if (!rc) {
10234 		next_handle = le16_to_cpu(resp->next_handle);
10235 		if (next_handle != 0) {
10236 			if (resp->wol_type ==
10237 			    WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
10238 				bp->wol = 1;
10239 				bp->wol_filter_id = resp->wol_filter_id;
10240 			}
10241 		}
10242 	}
10243 	hwrm_req_drop(bp, req);
10244 	return next_handle;
10245 }
10246 
10247 static void bnxt_get_wol_settings(struct bnxt *bp)
10248 {
10249 	u16 handle = 0;
10250 
10251 	bp->wol = 0;
10252 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
10253 		return;
10254 
10255 	do {
10256 		handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
10257 	} while (handle && handle != 0xffff);
10258 }
10259 
10260 #ifdef CONFIG_BNXT_HWMON
10261 static ssize_t bnxt_show_temp(struct device *dev,
10262 			      struct device_attribute *devattr, char *buf)
10263 {
10264 	struct hwrm_temp_monitor_query_output *resp;
10265 	struct hwrm_temp_monitor_query_input *req;
10266 	struct bnxt *bp = dev_get_drvdata(dev);
10267 	u32 len = 0;
10268 	int rc;
10269 
10270 	rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10271 	if (rc)
10272 		return rc;
10273 	resp = hwrm_req_hold(bp, req);
10274 	rc = hwrm_req_send(bp, req);
10275 	if (!rc)
10276 		len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
10277 	hwrm_req_drop(bp, req);
10278 	if (rc)
10279 		return rc;
10280 	return len;
10281 }
10282 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
10283 
10284 static struct attribute *bnxt_attrs[] = {
10285 	&sensor_dev_attr_temp1_input.dev_attr.attr,
10286 	NULL
10287 };
10288 ATTRIBUTE_GROUPS(bnxt);
10289 
10290 static void bnxt_hwmon_close(struct bnxt *bp)
10291 {
10292 	if (bp->hwmon_dev) {
10293 		hwmon_device_unregister(bp->hwmon_dev);
10294 		bp->hwmon_dev = NULL;
10295 	}
10296 }
10297 
10298 static void bnxt_hwmon_open(struct bnxt *bp)
10299 {
10300 	struct hwrm_temp_monitor_query_input *req;
10301 	struct pci_dev *pdev = bp->pdev;
10302 	int rc;
10303 
10304 	rc = hwrm_req_init(bp, req, HWRM_TEMP_MONITOR_QUERY);
10305 	if (!rc)
10306 		rc = hwrm_req_send_silent(bp, req);
10307 	if (rc == -EACCES || rc == -EOPNOTSUPP) {
10308 		bnxt_hwmon_close(bp);
10309 		return;
10310 	}
10311 
10312 	if (bp->hwmon_dev)
10313 		return;
10314 
10315 	bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
10316 							  DRV_MODULE_NAME, bp,
10317 							  bnxt_groups);
10318 	if (IS_ERR(bp->hwmon_dev)) {
10319 		bp->hwmon_dev = NULL;
10320 		dev_warn(&pdev->dev, "Cannot register hwmon device\n");
10321 	}
10322 }
10323 #else
10324 static void bnxt_hwmon_close(struct bnxt *bp)
10325 {
10326 }
10327 
10328 static void bnxt_hwmon_open(struct bnxt *bp)
10329 {
10330 }
10331 #endif
10332 
10333 static bool bnxt_eee_config_ok(struct bnxt *bp)
10334 {
10335 	struct ethtool_eee *eee = &bp->eee;
10336 	struct bnxt_link_info *link_info = &bp->link_info;
10337 
10338 	if (!(bp->phy_flags & BNXT_PHY_FL_EEE_CAP))
10339 		return true;
10340 
10341 	if (eee->eee_enabled) {
10342 		u32 advertising =
10343 			_bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
10344 
10345 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10346 			eee->eee_enabled = 0;
10347 			return false;
10348 		}
10349 		if (eee->advertised & ~advertising) {
10350 			eee->advertised = advertising & eee->supported;
10351 			return false;
10352 		}
10353 	}
10354 	return true;
10355 }
10356 
10357 static int bnxt_update_phy_setting(struct bnxt *bp)
10358 {
10359 	int rc;
10360 	bool update_link = false;
10361 	bool update_pause = false;
10362 	bool update_eee = false;
10363 	struct bnxt_link_info *link_info = &bp->link_info;
10364 
10365 	rc = bnxt_update_link(bp, true);
10366 	if (rc) {
10367 		netdev_err(bp->dev, "failed to update link (rc: %x)\n",
10368 			   rc);
10369 		return rc;
10370 	}
10371 	if (!BNXT_SINGLE_PF(bp))
10372 		return 0;
10373 
10374 	if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10375 	    (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
10376 	    link_info->req_flow_ctrl)
10377 		update_pause = true;
10378 	if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
10379 	    link_info->force_pause_setting != link_info->req_flow_ctrl)
10380 		update_pause = true;
10381 	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
10382 		if (BNXT_AUTO_MODE(link_info->auto_mode))
10383 			update_link = true;
10384 		if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
10385 		    link_info->req_link_speed != link_info->force_link_speed)
10386 			update_link = true;
10387 		else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
10388 			 link_info->req_link_speed != link_info->force_pam4_link_speed)
10389 			update_link = true;
10390 		if (link_info->req_duplex != link_info->duplex_setting)
10391 			update_link = true;
10392 	} else {
10393 		if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
10394 			update_link = true;
10395 		if (link_info->advertising != link_info->auto_link_speeds ||
10396 		    link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
10397 			update_link = true;
10398 	}
10399 
10400 	/* The last close may have shutdown the link, so need to call
10401 	 * PHY_CFG to bring it back up.
10402 	 */
10403 	if (!BNXT_LINK_IS_UP(bp))
10404 		update_link = true;
10405 
10406 	if (!bnxt_eee_config_ok(bp))
10407 		update_eee = true;
10408 
10409 	if (update_link)
10410 		rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
10411 	else if (update_pause)
10412 		rc = bnxt_hwrm_set_pause(bp);
10413 	if (rc) {
10414 		netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
10415 			   rc);
10416 		return rc;
10417 	}
10418 
10419 	return rc;
10420 }
10421 
10422 /* Common routine to pre-map certain register block to different GRC window.
10423  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
10424  * in PF and 3 windows in VF that can be customized to map in different
10425  * register blocks.
10426  */
10427 static void bnxt_preset_reg_win(struct bnxt *bp)
10428 {
10429 	if (BNXT_PF(bp)) {
10430 		/* CAG registers map to GRC window #4 */
10431 		writel(BNXT_CAG_REG_BASE,
10432 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
10433 	}
10434 }
10435 
10436 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
10437 
10438 static int bnxt_reinit_after_abort(struct bnxt *bp)
10439 {
10440 	int rc;
10441 
10442 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10443 		return -EBUSY;
10444 
10445 	if (bp->dev->reg_state == NETREG_UNREGISTERED)
10446 		return -ENODEV;
10447 
10448 	rc = bnxt_fw_init_one(bp);
10449 	if (!rc) {
10450 		bnxt_clear_int_mode(bp);
10451 		rc = bnxt_init_int_mode(bp);
10452 		if (!rc) {
10453 			clear_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10454 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
10455 		}
10456 	}
10457 	return rc;
10458 }
10459 
10460 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10461 {
10462 	int rc = 0;
10463 
10464 	bnxt_preset_reg_win(bp);
10465 	netif_carrier_off(bp->dev);
10466 	if (irq_re_init) {
10467 		/* Reserve rings now if none were reserved at driver probe. */
10468 		rc = bnxt_init_dflt_ring_mode(bp);
10469 		if (rc) {
10470 			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
10471 			return rc;
10472 		}
10473 	}
10474 	rc = bnxt_reserve_rings(bp, irq_re_init);
10475 	if (rc)
10476 		return rc;
10477 	if ((bp->flags & BNXT_FLAG_RFS) &&
10478 	    !(bp->flags & BNXT_FLAG_USING_MSIX)) {
10479 		/* disable RFS if falling back to INTA */
10480 		bp->dev->hw_features &= ~NETIF_F_NTUPLE;
10481 		bp->flags &= ~BNXT_FLAG_RFS;
10482 	}
10483 
10484 	rc = bnxt_alloc_mem(bp, irq_re_init);
10485 	if (rc) {
10486 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10487 		goto open_err_free_mem;
10488 	}
10489 
10490 	if (irq_re_init) {
10491 		bnxt_init_napi(bp);
10492 		rc = bnxt_request_irq(bp);
10493 		if (rc) {
10494 			netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
10495 			goto open_err_irq;
10496 		}
10497 	}
10498 
10499 	rc = bnxt_init_nic(bp, irq_re_init);
10500 	if (rc) {
10501 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10502 		goto open_err_irq;
10503 	}
10504 
10505 	bnxt_enable_napi(bp);
10506 	bnxt_debug_dev_init(bp);
10507 
10508 	if (link_re_init) {
10509 		mutex_lock(&bp->link_lock);
10510 		rc = bnxt_update_phy_setting(bp);
10511 		mutex_unlock(&bp->link_lock);
10512 		if (rc) {
10513 			netdev_warn(bp->dev, "failed to update phy settings\n");
10514 			if (BNXT_SINGLE_PF(bp)) {
10515 				bp->link_info.phy_retry = true;
10516 				bp->link_info.phy_retry_expires =
10517 					jiffies + 5 * HZ;
10518 			}
10519 		}
10520 	}
10521 
10522 	if (irq_re_init)
10523 		udp_tunnel_nic_reset_ntf(bp->dev);
10524 
10525 	if (bp->tx_nr_rings_xdp < num_possible_cpus()) {
10526 		if (!static_key_enabled(&bnxt_xdp_locking_key))
10527 			static_branch_enable(&bnxt_xdp_locking_key);
10528 	} else if (static_key_enabled(&bnxt_xdp_locking_key)) {
10529 		static_branch_disable(&bnxt_xdp_locking_key);
10530 	}
10531 	set_bit(BNXT_STATE_OPEN, &bp->state);
10532 	bnxt_enable_int(bp);
10533 	/* Enable TX queues */
10534 	bnxt_tx_enable(bp);
10535 	mod_timer(&bp->timer, jiffies + bp->current_interval);
10536 	/* Poll link status and check for SFP+ module status */
10537 	mutex_lock(&bp->link_lock);
10538 	bnxt_get_port_module_status(bp);
10539 	mutex_unlock(&bp->link_lock);
10540 
10541 	/* VF-reps may need to be re-opened after the PF is re-opened */
10542 	if (BNXT_PF(bp))
10543 		bnxt_vf_reps_open(bp);
10544 	if (bp->ptp_cfg)
10545 		atomic_set(&bp->ptp_cfg->tx_avail, BNXT_MAX_TX_TS);
10546 	bnxt_ptp_init_rtc(bp, true);
10547 	bnxt_ptp_cfg_tstamp_filters(bp);
10548 	return 0;
10549 
10550 open_err_irq:
10551 	bnxt_del_napi(bp);
10552 
10553 open_err_free_mem:
10554 	bnxt_free_skbs(bp);
10555 	bnxt_free_irq(bp);
10556 	bnxt_free_mem(bp, true);
10557 	return rc;
10558 }
10559 
10560 /* rtnl_lock held */
10561 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10562 {
10563 	int rc = 0;
10564 
10565 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
10566 		rc = -EIO;
10567 	if (!rc)
10568 		rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
10569 	if (rc) {
10570 		netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
10571 		dev_close(bp->dev);
10572 	}
10573 	return rc;
10574 }
10575 
10576 /* rtnl_lock held, open the NIC half way by allocating all resources, but
10577  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
10578  * self tests.
10579  */
10580 int bnxt_half_open_nic(struct bnxt *bp)
10581 {
10582 	int rc = 0;
10583 
10584 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10585 		netdev_err(bp->dev, "A previous firmware reset has not completed, aborting half open\n");
10586 		rc = -ENODEV;
10587 		goto half_open_err;
10588 	}
10589 
10590 	rc = bnxt_alloc_mem(bp, true);
10591 	if (rc) {
10592 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
10593 		goto half_open_err;
10594 	}
10595 	bnxt_init_napi(bp);
10596 	set_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10597 	rc = bnxt_init_nic(bp, true);
10598 	if (rc) {
10599 		clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10600 		bnxt_del_napi(bp);
10601 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
10602 		goto half_open_err;
10603 	}
10604 	return 0;
10605 
10606 half_open_err:
10607 	bnxt_free_skbs(bp);
10608 	bnxt_free_mem(bp, true);
10609 	dev_close(bp->dev);
10610 	return rc;
10611 }
10612 
10613 /* rtnl_lock held, this call can only be made after a previous successful
10614  * call to bnxt_half_open_nic().
10615  */
10616 void bnxt_half_close_nic(struct bnxt *bp)
10617 {
10618 	bnxt_hwrm_resource_free(bp, false, true);
10619 	bnxt_del_napi(bp);
10620 	bnxt_free_skbs(bp);
10621 	bnxt_free_mem(bp, true);
10622 	clear_bit(BNXT_STATE_HALF_OPEN, &bp->state);
10623 }
10624 
10625 void bnxt_reenable_sriov(struct bnxt *bp)
10626 {
10627 	if (BNXT_PF(bp)) {
10628 		struct bnxt_pf_info *pf = &bp->pf;
10629 		int n = pf->active_vfs;
10630 
10631 		if (n)
10632 			bnxt_cfg_hw_sriov(bp, &n, true);
10633 	}
10634 }
10635 
10636 static int bnxt_open(struct net_device *dev)
10637 {
10638 	struct bnxt *bp = netdev_priv(dev);
10639 	int rc;
10640 
10641 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
10642 		rc = bnxt_reinit_after_abort(bp);
10643 		if (rc) {
10644 			if (rc == -EBUSY)
10645 				netdev_err(bp->dev, "A previous firmware reset has not completed, aborting\n");
10646 			else
10647 				netdev_err(bp->dev, "Failed to reinitialize after aborted firmware reset\n");
10648 			return -ENODEV;
10649 		}
10650 	}
10651 
10652 	rc = bnxt_hwrm_if_change(bp, true);
10653 	if (rc)
10654 		return rc;
10655 
10656 	rc = __bnxt_open_nic(bp, true, true);
10657 	if (rc) {
10658 		bnxt_hwrm_if_change(bp, false);
10659 	} else {
10660 		if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
10661 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10662 				bnxt_ulp_start(bp, 0);
10663 				bnxt_reenable_sriov(bp);
10664 			}
10665 		}
10666 		bnxt_hwmon_open(bp);
10667 	}
10668 
10669 	return rc;
10670 }
10671 
10672 static bool bnxt_drv_busy(struct bnxt *bp)
10673 {
10674 	return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
10675 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
10676 }
10677 
10678 static void bnxt_get_ring_stats(struct bnxt *bp,
10679 				struct rtnl_link_stats64 *stats);
10680 
10681 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
10682 			     bool link_re_init)
10683 {
10684 	/* Close the VF-reps before closing PF */
10685 	if (BNXT_PF(bp))
10686 		bnxt_vf_reps_close(bp);
10687 
10688 	/* Change device state to avoid TX queue wake up's */
10689 	bnxt_tx_disable(bp);
10690 
10691 	clear_bit(BNXT_STATE_OPEN, &bp->state);
10692 	smp_mb__after_atomic();
10693 	while (bnxt_drv_busy(bp))
10694 		msleep(20);
10695 
10696 	/* Flush rings and disable interrupts */
10697 	bnxt_shutdown_nic(bp, irq_re_init);
10698 
10699 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
10700 
10701 	bnxt_debug_dev_exit(bp);
10702 	bnxt_disable_napi(bp);
10703 	del_timer_sync(&bp->timer);
10704 	bnxt_free_skbs(bp);
10705 
10706 	/* Save ring stats before shutdown */
10707 	if (bp->bnapi && irq_re_init) {
10708 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
10709 		bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev);
10710 	}
10711 	if (irq_re_init) {
10712 		bnxt_free_irq(bp);
10713 		bnxt_del_napi(bp);
10714 	}
10715 	bnxt_free_mem(bp, irq_re_init);
10716 }
10717 
10718 void bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
10719 {
10720 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10721 		/* If we get here, it means firmware reset is in progress
10722 		 * while we are trying to close.  We can safely proceed with
10723 		 * the close because we are holding rtnl_lock().  Some firmware
10724 		 * messages may fail as we proceed to close.  We set the
10725 		 * ABORT_ERR flag here so that the FW reset thread will later
10726 		 * abort when it gets the rtnl_lock() and sees the flag.
10727 		 */
10728 		netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
10729 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
10730 	}
10731 
10732 #ifdef CONFIG_BNXT_SRIOV
10733 	if (bp->sriov_cfg) {
10734 		int rc;
10735 
10736 		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
10737 						      !bp->sriov_cfg,
10738 						      BNXT_SRIOV_CFG_WAIT_TMO);
10739 		if (!rc)
10740 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete, proceeding to close!\n");
10741 		else if (rc < 0)
10742 			netdev_warn(bp->dev, "SRIOV config operation interrupted, proceeding to close!\n");
10743 	}
10744 #endif
10745 	__bnxt_close_nic(bp, irq_re_init, link_re_init);
10746 }
10747 
10748 static int bnxt_close(struct net_device *dev)
10749 {
10750 	struct bnxt *bp = netdev_priv(dev);
10751 
10752 	bnxt_hwmon_close(bp);
10753 	bnxt_close_nic(bp, true, true);
10754 	bnxt_hwrm_shutdown_link(bp);
10755 	bnxt_hwrm_if_change(bp, false);
10756 	return 0;
10757 }
10758 
10759 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
10760 				   u16 *val)
10761 {
10762 	struct hwrm_port_phy_mdio_read_output *resp;
10763 	struct hwrm_port_phy_mdio_read_input *req;
10764 	int rc;
10765 
10766 	if (bp->hwrm_spec_code < 0x10a00)
10767 		return -EOPNOTSUPP;
10768 
10769 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_READ);
10770 	if (rc)
10771 		return rc;
10772 
10773 	req->port_id = cpu_to_le16(bp->pf.port_id);
10774 	req->phy_addr = phy_addr;
10775 	req->reg_addr = cpu_to_le16(reg & 0x1f);
10776 	if (mdio_phy_id_is_c45(phy_addr)) {
10777 		req->cl45_mdio = 1;
10778 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
10779 		req->dev_addr = mdio_phy_id_devad(phy_addr);
10780 		req->reg_addr = cpu_to_le16(reg);
10781 	}
10782 
10783 	resp = hwrm_req_hold(bp, req);
10784 	rc = hwrm_req_send(bp, req);
10785 	if (!rc)
10786 		*val = le16_to_cpu(resp->reg_data);
10787 	hwrm_req_drop(bp, req);
10788 	return rc;
10789 }
10790 
10791 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
10792 				    u16 val)
10793 {
10794 	struct hwrm_port_phy_mdio_write_input *req;
10795 	int rc;
10796 
10797 	if (bp->hwrm_spec_code < 0x10a00)
10798 		return -EOPNOTSUPP;
10799 
10800 	rc = hwrm_req_init(bp, req, HWRM_PORT_PHY_MDIO_WRITE);
10801 	if (rc)
10802 		return rc;
10803 
10804 	req->port_id = cpu_to_le16(bp->pf.port_id);
10805 	req->phy_addr = phy_addr;
10806 	req->reg_addr = cpu_to_le16(reg & 0x1f);
10807 	if (mdio_phy_id_is_c45(phy_addr)) {
10808 		req->cl45_mdio = 1;
10809 		req->phy_addr = mdio_phy_id_prtad(phy_addr);
10810 		req->dev_addr = mdio_phy_id_devad(phy_addr);
10811 		req->reg_addr = cpu_to_le16(reg);
10812 	}
10813 	req->reg_data = cpu_to_le16(val);
10814 
10815 	return hwrm_req_send(bp, req);
10816 }
10817 
10818 /* rtnl_lock held */
10819 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10820 {
10821 	struct mii_ioctl_data *mdio = if_mii(ifr);
10822 	struct bnxt *bp = netdev_priv(dev);
10823 	int rc;
10824 
10825 	switch (cmd) {
10826 	case SIOCGMIIPHY:
10827 		mdio->phy_id = bp->link_info.phy_addr;
10828 
10829 		fallthrough;
10830 	case SIOCGMIIREG: {
10831 		u16 mii_regval = 0;
10832 
10833 		if (!netif_running(dev))
10834 			return -EAGAIN;
10835 
10836 		rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10837 					     &mii_regval);
10838 		mdio->val_out = mii_regval;
10839 		return rc;
10840 	}
10841 
10842 	case SIOCSMIIREG:
10843 		if (!netif_running(dev))
10844 			return -EAGAIN;
10845 
10846 		return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10847 						mdio->val_in);
10848 
10849 	case SIOCSHWTSTAMP:
10850 		return bnxt_hwtstamp_set(dev, ifr);
10851 
10852 	case SIOCGHWTSTAMP:
10853 		return bnxt_hwtstamp_get(dev, ifr);
10854 
10855 	default:
10856 		/* do nothing */
10857 		break;
10858 	}
10859 	return -EOPNOTSUPP;
10860 }
10861 
10862 static void bnxt_get_ring_stats(struct bnxt *bp,
10863 				struct rtnl_link_stats64 *stats)
10864 {
10865 	int i;
10866 
10867 	for (i = 0; i < bp->cp_nr_rings; i++) {
10868 		struct bnxt_napi *bnapi = bp->bnapi[i];
10869 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10870 		u64 *sw = cpr->stats.sw_stats;
10871 
10872 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10873 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10874 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10875 
10876 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10877 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10878 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10879 
10880 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10881 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10882 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10883 
10884 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10885 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10886 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10887 
10888 		stats->rx_missed_errors +=
10889 			BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10890 
10891 		stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10892 
10893 		stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10894 
10895 		stats->rx_dropped +=
10896 			cpr->sw_stats.rx.rx_netpoll_discards +
10897 			cpr->sw_stats.rx.rx_oom_discards;
10898 	}
10899 }
10900 
10901 static void bnxt_add_prev_stats(struct bnxt *bp,
10902 				struct rtnl_link_stats64 *stats)
10903 {
10904 	struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10905 
10906 	stats->rx_packets += prev_stats->rx_packets;
10907 	stats->tx_packets += prev_stats->tx_packets;
10908 	stats->rx_bytes += prev_stats->rx_bytes;
10909 	stats->tx_bytes += prev_stats->tx_bytes;
10910 	stats->rx_missed_errors += prev_stats->rx_missed_errors;
10911 	stats->multicast += prev_stats->multicast;
10912 	stats->rx_dropped += prev_stats->rx_dropped;
10913 	stats->tx_dropped += prev_stats->tx_dropped;
10914 }
10915 
10916 static void
10917 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10918 {
10919 	struct bnxt *bp = netdev_priv(dev);
10920 
10921 	set_bit(BNXT_STATE_READ_STATS, &bp->state);
10922 	/* Make sure bnxt_close_nic() sees that we are reading stats before
10923 	 * we check the BNXT_STATE_OPEN flag.
10924 	 */
10925 	smp_mb__after_atomic();
10926 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10927 		clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10928 		*stats = bp->net_stats_prev;
10929 		return;
10930 	}
10931 
10932 	bnxt_get_ring_stats(bp, stats);
10933 	bnxt_add_prev_stats(bp, stats);
10934 
10935 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
10936 		u64 *rx = bp->port_stats.sw_stats;
10937 		u64 *tx = bp->port_stats.sw_stats +
10938 			  BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10939 
10940 		stats->rx_crc_errors =
10941 			BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10942 		stats->rx_frame_errors =
10943 			BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10944 		stats->rx_length_errors =
10945 			BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10946 			BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10947 			BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10948 		stats->rx_errors =
10949 			BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10950 			BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10951 		stats->collisions =
10952 			BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10953 		stats->tx_fifo_errors =
10954 			BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10955 		stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10956 	}
10957 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10958 }
10959 
10960 static void bnxt_get_one_ring_err_stats(struct bnxt *bp,
10961 					struct bnxt_total_ring_err_stats *stats,
10962 					struct bnxt_cp_ring_info *cpr)
10963 {
10964 	struct bnxt_sw_stats *sw_stats = &cpr->sw_stats;
10965 	u64 *hw_stats = cpr->stats.sw_stats;
10966 
10967 	stats->rx_total_l4_csum_errors += sw_stats->rx.rx_l4_csum_errors;
10968 	stats->rx_total_resets += sw_stats->rx.rx_resets;
10969 	stats->rx_total_buf_errors += sw_stats->rx.rx_buf_errors;
10970 	stats->rx_total_oom_discards += sw_stats->rx.rx_oom_discards;
10971 	stats->rx_total_netpoll_discards += sw_stats->rx.rx_netpoll_discards;
10972 	stats->rx_total_ring_discards +=
10973 		BNXT_GET_RING_STATS64(hw_stats, rx_discard_pkts);
10974 	stats->tx_total_resets += sw_stats->tx.tx_resets;
10975 	stats->tx_total_ring_discards +=
10976 		BNXT_GET_RING_STATS64(hw_stats, tx_discard_pkts);
10977 	stats->total_missed_irqs += sw_stats->cmn.missed_irqs;
10978 }
10979 
10980 void bnxt_get_ring_err_stats(struct bnxt *bp,
10981 			     struct bnxt_total_ring_err_stats *stats)
10982 {
10983 	int i;
10984 
10985 	for (i = 0; i < bp->cp_nr_rings; i++)
10986 		bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring);
10987 }
10988 
10989 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10990 {
10991 	struct net_device *dev = bp->dev;
10992 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10993 	struct netdev_hw_addr *ha;
10994 	u8 *haddr;
10995 	int mc_count = 0;
10996 	bool update = false;
10997 	int off = 0;
10998 
10999 	netdev_for_each_mc_addr(ha, dev) {
11000 		if (mc_count >= BNXT_MAX_MC_ADDRS) {
11001 			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11002 			vnic->mc_list_count = 0;
11003 			return false;
11004 		}
11005 		haddr = ha->addr;
11006 		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
11007 			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
11008 			update = true;
11009 		}
11010 		off += ETH_ALEN;
11011 		mc_count++;
11012 	}
11013 	if (mc_count)
11014 		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11015 
11016 	if (mc_count != vnic->mc_list_count) {
11017 		vnic->mc_list_count = mc_count;
11018 		update = true;
11019 	}
11020 	return update;
11021 }
11022 
11023 static bool bnxt_uc_list_updated(struct bnxt *bp)
11024 {
11025 	struct net_device *dev = bp->dev;
11026 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11027 	struct netdev_hw_addr *ha;
11028 	int off = 0;
11029 
11030 	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
11031 		return true;
11032 
11033 	netdev_for_each_uc_addr(ha, dev) {
11034 		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
11035 			return true;
11036 
11037 		off += ETH_ALEN;
11038 	}
11039 	return false;
11040 }
11041 
11042 static void bnxt_set_rx_mode(struct net_device *dev)
11043 {
11044 	struct bnxt *bp = netdev_priv(dev);
11045 	struct bnxt_vnic_info *vnic;
11046 	bool mc_update = false;
11047 	bool uc_update;
11048 	u32 mask;
11049 
11050 	if (!test_bit(BNXT_STATE_OPEN, &bp->state))
11051 		return;
11052 
11053 	vnic = &bp->vnic_info[0];
11054 	mask = vnic->rx_mask;
11055 	mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
11056 		  CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
11057 		  CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
11058 		  CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
11059 
11060 	if (dev->flags & IFF_PROMISC)
11061 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11062 
11063 	uc_update = bnxt_uc_list_updated(bp);
11064 
11065 	if (dev->flags & IFF_BROADCAST)
11066 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
11067 	if (dev->flags & IFF_ALLMULTI) {
11068 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11069 		vnic->mc_list_count = 0;
11070 	} else if (dev->flags & IFF_MULTICAST) {
11071 		mc_update = bnxt_mc_list_updated(bp, &mask);
11072 	}
11073 
11074 	if (mask != vnic->rx_mask || uc_update || mc_update) {
11075 		vnic->rx_mask = mask;
11076 
11077 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
11078 	}
11079 }
11080 
11081 static int bnxt_cfg_rx_mode(struct bnxt *bp)
11082 {
11083 	struct net_device *dev = bp->dev;
11084 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11085 	struct hwrm_cfa_l2_filter_free_input *req;
11086 	struct netdev_hw_addr *ha;
11087 	int i, off = 0, rc;
11088 	bool uc_update;
11089 
11090 	netif_addr_lock_bh(dev);
11091 	uc_update = bnxt_uc_list_updated(bp);
11092 	netif_addr_unlock_bh(dev);
11093 
11094 	if (!uc_update)
11095 		goto skip_uc;
11096 
11097 	rc = hwrm_req_init(bp, req, HWRM_CFA_L2_FILTER_FREE);
11098 	if (rc)
11099 		return rc;
11100 	hwrm_req_hold(bp, req);
11101 	for (i = 1; i < vnic->uc_filter_count; i++) {
11102 		req->l2_filter_id = vnic->fw_l2_filter_id[i];
11103 
11104 		rc = hwrm_req_send(bp, req);
11105 	}
11106 	hwrm_req_drop(bp, req);
11107 
11108 	vnic->uc_filter_count = 1;
11109 
11110 	netif_addr_lock_bh(dev);
11111 	if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
11112 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11113 	} else {
11114 		netdev_for_each_uc_addr(ha, dev) {
11115 			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
11116 			off += ETH_ALEN;
11117 			vnic->uc_filter_count++;
11118 		}
11119 	}
11120 	netif_addr_unlock_bh(dev);
11121 
11122 	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
11123 		rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
11124 		if (rc) {
11125 			if (BNXT_VF(bp) && rc == -ENODEV) {
11126 				if (!test_and_set_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11127 					netdev_warn(bp->dev, "Cannot configure L2 filters while PF is unavailable, will retry\n");
11128 				else
11129 					netdev_dbg(bp->dev, "PF still unavailable while configuring L2 filters.\n");
11130 				rc = 0;
11131 			} else {
11132 				netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
11133 			}
11134 			vnic->uc_filter_count = i;
11135 			return rc;
11136 		}
11137 	}
11138 	if (test_and_clear_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11139 		netdev_notice(bp->dev, "Retry of L2 filter configuration successful.\n");
11140 
11141 skip_uc:
11142 	if ((vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS) &&
11143 	    !bnxt_promisc_ok(bp))
11144 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
11145 	rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11146 	if (rc && (vnic->rx_mask & CFA_L2_SET_RX_MASK_REQ_MASK_MCAST)) {
11147 		netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
11148 			    rc);
11149 		vnic->rx_mask &= ~CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
11150 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
11151 		vnic->mc_list_count = 0;
11152 		rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
11153 	}
11154 	if (rc)
11155 		netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
11156 			   rc);
11157 
11158 	return rc;
11159 }
11160 
11161 static bool bnxt_can_reserve_rings(struct bnxt *bp)
11162 {
11163 #ifdef CONFIG_BNXT_SRIOV
11164 	if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
11165 		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11166 
11167 		/* No minimum rings were provisioned by the PF.  Don't
11168 		 * reserve rings by default when device is down.
11169 		 */
11170 		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
11171 			return true;
11172 
11173 		if (!netif_running(bp->dev))
11174 			return false;
11175 	}
11176 #endif
11177 	return true;
11178 }
11179 
11180 /* If the chip and firmware supports RFS */
11181 static bool bnxt_rfs_supported(struct bnxt *bp)
11182 {
11183 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
11184 		if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
11185 			return true;
11186 		return false;
11187 	}
11188 	/* 212 firmware is broken for aRFS */
11189 	if (BNXT_FW_MAJ(bp) == 212)
11190 		return false;
11191 	if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
11192 		return true;
11193 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11194 		return true;
11195 	return false;
11196 }
11197 
11198 /* If runtime conditions support RFS */
11199 static bool bnxt_rfs_capable(struct bnxt *bp)
11200 {
11201 #ifdef CONFIG_RFS_ACCEL
11202 	int vnics, max_vnics, max_rss_ctxs;
11203 
11204 	if (bp->flags & BNXT_FLAG_CHIP_P5)
11205 		return bnxt_rfs_supported(bp);
11206 	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp) || !bp->rx_nr_rings)
11207 		return false;
11208 
11209 	vnics = 1 + bp->rx_nr_rings;
11210 	max_vnics = bnxt_get_max_func_vnics(bp);
11211 	max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
11212 
11213 	/* RSS contexts not a limiting factor */
11214 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
11215 		max_rss_ctxs = max_vnics;
11216 	if (vnics > max_vnics || vnics > max_rss_ctxs) {
11217 		if (bp->rx_nr_rings > 1)
11218 			netdev_warn(bp->dev,
11219 				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
11220 				    min(max_rss_ctxs - 1, max_vnics - 1));
11221 		return false;
11222 	}
11223 
11224 	if (!BNXT_NEW_RM(bp))
11225 		return true;
11226 
11227 	if (vnics == bp->hw_resc.resv_vnics)
11228 		return true;
11229 
11230 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
11231 	if (vnics <= bp->hw_resc.resv_vnics)
11232 		return true;
11233 
11234 	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
11235 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
11236 	return false;
11237 #else
11238 	return false;
11239 #endif
11240 }
11241 
11242 static netdev_features_t bnxt_fix_features(struct net_device *dev,
11243 					   netdev_features_t features)
11244 {
11245 	struct bnxt *bp = netdev_priv(dev);
11246 	netdev_features_t vlan_features;
11247 
11248 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
11249 		features &= ~NETIF_F_NTUPLE;
11250 
11251 	if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog)
11252 		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11253 
11254 	if (!(features & NETIF_F_GRO))
11255 		features &= ~NETIF_F_GRO_HW;
11256 
11257 	if (features & NETIF_F_GRO_HW)
11258 		features &= ~NETIF_F_LRO;
11259 
11260 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
11261 	 * turned on or off together.
11262 	 */
11263 	vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
11264 	if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
11265 		if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11266 			features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11267 		else if (vlan_features)
11268 			features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
11269 	}
11270 #ifdef CONFIG_BNXT_SRIOV
11271 	if (BNXT_VF(bp) && bp->vf.vlan)
11272 		features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
11273 #endif
11274 	return features;
11275 }
11276 
11277 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
11278 {
11279 	struct bnxt *bp = netdev_priv(dev);
11280 	u32 flags = bp->flags;
11281 	u32 changes;
11282 	int rc = 0;
11283 	bool re_init = false;
11284 	bool update_tpa = false;
11285 
11286 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
11287 	if (features & NETIF_F_GRO_HW)
11288 		flags |= BNXT_FLAG_GRO;
11289 	else if (features & NETIF_F_LRO)
11290 		flags |= BNXT_FLAG_LRO;
11291 
11292 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
11293 		flags &= ~BNXT_FLAG_TPA;
11294 
11295 	if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
11296 		flags |= BNXT_FLAG_STRIP_VLAN;
11297 
11298 	if (features & NETIF_F_NTUPLE)
11299 		flags |= BNXT_FLAG_RFS;
11300 
11301 	changes = flags ^ bp->flags;
11302 	if (changes & BNXT_FLAG_TPA) {
11303 		update_tpa = true;
11304 		if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
11305 		    (flags & BNXT_FLAG_TPA) == 0 ||
11306 		    (bp->flags & BNXT_FLAG_CHIP_P5))
11307 			re_init = true;
11308 	}
11309 
11310 	if (changes & ~BNXT_FLAG_TPA)
11311 		re_init = true;
11312 
11313 	if (flags != bp->flags) {
11314 		u32 old_flags = bp->flags;
11315 
11316 		if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11317 			bp->flags = flags;
11318 			if (update_tpa)
11319 				bnxt_set_ring_params(bp);
11320 			return rc;
11321 		}
11322 
11323 		if (re_init) {
11324 			bnxt_close_nic(bp, false, false);
11325 			bp->flags = flags;
11326 			if (update_tpa)
11327 				bnxt_set_ring_params(bp);
11328 
11329 			return bnxt_open_nic(bp, false, false);
11330 		}
11331 		if (update_tpa) {
11332 			bp->flags = flags;
11333 			rc = bnxt_set_tpa(bp,
11334 					  (flags & BNXT_FLAG_TPA) ?
11335 					  true : false);
11336 			if (rc)
11337 				bp->flags = old_flags;
11338 		}
11339 	}
11340 	return rc;
11341 }
11342 
11343 static bool bnxt_exthdr_check(struct bnxt *bp, struct sk_buff *skb, int nw_off,
11344 			      u8 **nextp)
11345 {
11346 	struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + nw_off);
11347 	struct hop_jumbo_hdr *jhdr;
11348 	int hdr_count = 0;
11349 	u8 *nexthdr;
11350 	int start;
11351 
11352 	/* Check that there are at most 2 IPv6 extension headers, no
11353 	 * fragment header, and each is <= 64 bytes.
11354 	 */
11355 	start = nw_off + sizeof(*ip6h);
11356 	nexthdr = &ip6h->nexthdr;
11357 	while (ipv6_ext_hdr(*nexthdr)) {
11358 		struct ipv6_opt_hdr *hp;
11359 		int hdrlen;
11360 
11361 		if (hdr_count >= 3 || *nexthdr == NEXTHDR_NONE ||
11362 		    *nexthdr == NEXTHDR_FRAGMENT)
11363 			return false;
11364 		hp = __skb_header_pointer(NULL, start, sizeof(*hp), skb->data,
11365 					  skb_headlen(skb), NULL);
11366 		if (!hp)
11367 			return false;
11368 		if (*nexthdr == NEXTHDR_AUTH)
11369 			hdrlen = ipv6_authlen(hp);
11370 		else
11371 			hdrlen = ipv6_optlen(hp);
11372 
11373 		if (hdrlen > 64)
11374 			return false;
11375 
11376 		/* The ext header may be a hop-by-hop header inserted for
11377 		 * big TCP purposes. This will be removed before sending
11378 		 * from NIC, so do not count it.
11379 		 */
11380 		if (*nexthdr == NEXTHDR_HOP) {
11381 			if (likely(skb->len <= GRO_LEGACY_MAX_SIZE))
11382 				goto increment_hdr;
11383 
11384 			jhdr = (struct hop_jumbo_hdr *)hp;
11385 			if (jhdr->tlv_type != IPV6_TLV_JUMBO || jhdr->hdrlen != 0 ||
11386 			    jhdr->nexthdr != IPPROTO_TCP)
11387 				goto increment_hdr;
11388 
11389 			goto next_hdr;
11390 		}
11391 increment_hdr:
11392 		hdr_count++;
11393 next_hdr:
11394 		nexthdr = &hp->nexthdr;
11395 		start += hdrlen;
11396 	}
11397 	if (nextp) {
11398 		/* Caller will check inner protocol */
11399 		if (skb->encapsulation) {
11400 			*nextp = nexthdr;
11401 			return true;
11402 		}
11403 		*nextp = NULL;
11404 	}
11405 	/* Only support TCP/UDP for non-tunneled ipv6 and inner ipv6 */
11406 	return *nexthdr == IPPROTO_TCP || *nexthdr == IPPROTO_UDP;
11407 }
11408 
11409 /* For UDP, we can only handle 1 Vxlan port and 1 Geneve port. */
11410 static bool bnxt_udp_tunl_check(struct bnxt *bp, struct sk_buff *skb)
11411 {
11412 	struct udphdr *uh = udp_hdr(skb);
11413 	__be16 udp_port = uh->dest;
11414 
11415 	if (udp_port != bp->vxlan_port && udp_port != bp->nge_port)
11416 		return false;
11417 	if (skb->inner_protocol_type == ENCAP_TYPE_ETHER) {
11418 		struct ethhdr *eh = inner_eth_hdr(skb);
11419 
11420 		switch (eh->h_proto) {
11421 		case htons(ETH_P_IP):
11422 			return true;
11423 		case htons(ETH_P_IPV6):
11424 			return bnxt_exthdr_check(bp, skb,
11425 						 skb_inner_network_offset(skb),
11426 						 NULL);
11427 		}
11428 	}
11429 	return false;
11430 }
11431 
11432 static bool bnxt_tunl_check(struct bnxt *bp, struct sk_buff *skb, u8 l4_proto)
11433 {
11434 	switch (l4_proto) {
11435 	case IPPROTO_UDP:
11436 		return bnxt_udp_tunl_check(bp, skb);
11437 	case IPPROTO_IPIP:
11438 		return true;
11439 	case IPPROTO_GRE: {
11440 		switch (skb->inner_protocol) {
11441 		default:
11442 			return false;
11443 		case htons(ETH_P_IP):
11444 			return true;
11445 		case htons(ETH_P_IPV6):
11446 			fallthrough;
11447 		}
11448 	}
11449 	case IPPROTO_IPV6:
11450 		/* Check ext headers of inner ipv6 */
11451 		return bnxt_exthdr_check(bp, skb, skb_inner_network_offset(skb),
11452 					 NULL);
11453 	}
11454 	return false;
11455 }
11456 
11457 static netdev_features_t bnxt_features_check(struct sk_buff *skb,
11458 					     struct net_device *dev,
11459 					     netdev_features_t features)
11460 {
11461 	struct bnxt *bp = netdev_priv(dev);
11462 	u8 *l4_proto;
11463 
11464 	features = vlan_features_check(skb, features);
11465 	switch (vlan_get_protocol(skb)) {
11466 	case htons(ETH_P_IP):
11467 		if (!skb->encapsulation)
11468 			return features;
11469 		l4_proto = &ip_hdr(skb)->protocol;
11470 		if (bnxt_tunl_check(bp, skb, *l4_proto))
11471 			return features;
11472 		break;
11473 	case htons(ETH_P_IPV6):
11474 		if (!bnxt_exthdr_check(bp, skb, skb_network_offset(skb),
11475 				       &l4_proto))
11476 			break;
11477 		if (!l4_proto || bnxt_tunl_check(bp, skb, *l4_proto))
11478 			return features;
11479 		break;
11480 	}
11481 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
11482 }
11483 
11484 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
11485 			 u32 *reg_buf)
11486 {
11487 	struct hwrm_dbg_read_direct_output *resp;
11488 	struct hwrm_dbg_read_direct_input *req;
11489 	__le32 *dbg_reg_buf;
11490 	dma_addr_t mapping;
11491 	int rc, i;
11492 
11493 	rc = hwrm_req_init(bp, req, HWRM_DBG_READ_DIRECT);
11494 	if (rc)
11495 		return rc;
11496 
11497 	dbg_reg_buf = hwrm_req_dma_slice(bp, req, num_words * 4,
11498 					 &mapping);
11499 	if (!dbg_reg_buf) {
11500 		rc = -ENOMEM;
11501 		goto dbg_rd_reg_exit;
11502 	}
11503 
11504 	req->host_dest_addr = cpu_to_le64(mapping);
11505 
11506 	resp = hwrm_req_hold(bp, req);
11507 	req->read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
11508 	req->read_len32 = cpu_to_le32(num_words);
11509 
11510 	rc = hwrm_req_send(bp, req);
11511 	if (rc || resp->error_code) {
11512 		rc = -EIO;
11513 		goto dbg_rd_reg_exit;
11514 	}
11515 	for (i = 0; i < num_words; i++)
11516 		reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
11517 
11518 dbg_rd_reg_exit:
11519 	hwrm_req_drop(bp, req);
11520 	return rc;
11521 }
11522 
11523 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
11524 				       u32 ring_id, u32 *prod, u32 *cons)
11525 {
11526 	struct hwrm_dbg_ring_info_get_output *resp;
11527 	struct hwrm_dbg_ring_info_get_input *req;
11528 	int rc;
11529 
11530 	rc = hwrm_req_init(bp, req, HWRM_DBG_RING_INFO_GET);
11531 	if (rc)
11532 		return rc;
11533 
11534 	req->ring_type = ring_type;
11535 	req->fw_ring_id = cpu_to_le32(ring_id);
11536 	resp = hwrm_req_hold(bp, req);
11537 	rc = hwrm_req_send(bp, req);
11538 	if (!rc) {
11539 		*prod = le32_to_cpu(resp->producer_index);
11540 		*cons = le32_to_cpu(resp->consumer_index);
11541 	}
11542 	hwrm_req_drop(bp, req);
11543 	return rc;
11544 }
11545 
11546 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
11547 {
11548 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
11549 	int i = bnapi->index;
11550 
11551 	if (!txr)
11552 		return;
11553 
11554 	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
11555 		    i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
11556 		    txr->tx_cons);
11557 }
11558 
11559 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
11560 {
11561 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
11562 	int i = bnapi->index;
11563 
11564 	if (!rxr)
11565 		return;
11566 
11567 	netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
11568 		    i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
11569 		    rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
11570 		    rxr->rx_sw_agg_prod);
11571 }
11572 
11573 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
11574 {
11575 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
11576 	int i = bnapi->index;
11577 
11578 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
11579 		    i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
11580 }
11581 
11582 static void bnxt_dbg_dump_states(struct bnxt *bp)
11583 {
11584 	int i;
11585 	struct bnxt_napi *bnapi;
11586 
11587 	for (i = 0; i < bp->cp_nr_rings; i++) {
11588 		bnapi = bp->bnapi[i];
11589 		if (netif_msg_drv(bp)) {
11590 			bnxt_dump_tx_sw_state(bnapi);
11591 			bnxt_dump_rx_sw_state(bnapi);
11592 			bnxt_dump_cp_sw_state(bnapi);
11593 		}
11594 	}
11595 }
11596 
11597 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
11598 {
11599 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
11600 	struct hwrm_ring_reset_input *req;
11601 	struct bnxt_napi *bnapi = rxr->bnapi;
11602 	struct bnxt_cp_ring_info *cpr;
11603 	u16 cp_ring_id;
11604 	int rc;
11605 
11606 	rc = hwrm_req_init(bp, req, HWRM_RING_RESET);
11607 	if (rc)
11608 		return rc;
11609 
11610 	cpr = &bnapi->cp_ring;
11611 	cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
11612 	req->cmpl_ring = cpu_to_le16(cp_ring_id);
11613 	req->ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
11614 	req->ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
11615 	return hwrm_req_send_silent(bp, req);
11616 }
11617 
11618 static void bnxt_reset_task(struct bnxt *bp, bool silent)
11619 {
11620 	if (!silent)
11621 		bnxt_dbg_dump_states(bp);
11622 	if (netif_running(bp->dev)) {
11623 		int rc;
11624 
11625 		if (silent) {
11626 			bnxt_close_nic(bp, false, false);
11627 			bnxt_open_nic(bp, false, false);
11628 		} else {
11629 			bnxt_ulp_stop(bp);
11630 			bnxt_close_nic(bp, true, false);
11631 			rc = bnxt_open_nic(bp, true, false);
11632 			bnxt_ulp_start(bp, rc);
11633 		}
11634 	}
11635 }
11636 
11637 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
11638 {
11639 	struct bnxt *bp = netdev_priv(dev);
11640 
11641 	netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
11642 	bnxt_queue_sp_work(bp, BNXT_RESET_TASK_SP_EVENT);
11643 }
11644 
11645 static void bnxt_fw_health_check(struct bnxt *bp)
11646 {
11647 	struct bnxt_fw_health *fw_health = bp->fw_health;
11648 	struct pci_dev *pdev = bp->pdev;
11649 	u32 val;
11650 
11651 	if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11652 		return;
11653 
11654 	/* Make sure it is enabled before checking the tmr_counter. */
11655 	smp_rmb();
11656 	if (fw_health->tmr_counter) {
11657 		fw_health->tmr_counter--;
11658 		return;
11659 	}
11660 
11661 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11662 	if (val == fw_health->last_fw_heartbeat && pci_device_is_present(pdev)) {
11663 		fw_health->arrests++;
11664 		goto fw_reset;
11665 	}
11666 
11667 	fw_health->last_fw_heartbeat = val;
11668 
11669 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11670 	if (val != fw_health->last_fw_reset_cnt && pci_device_is_present(pdev)) {
11671 		fw_health->discoveries++;
11672 		goto fw_reset;
11673 	}
11674 
11675 	fw_health->tmr_counter = fw_health->tmr_multiplier;
11676 	return;
11677 
11678 fw_reset:
11679 	bnxt_queue_sp_work(bp, BNXT_FW_EXCEPTION_SP_EVENT);
11680 }
11681 
11682 static void bnxt_timer(struct timer_list *t)
11683 {
11684 	struct bnxt *bp = from_timer(bp, t, timer);
11685 	struct net_device *dev = bp->dev;
11686 
11687 	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
11688 		return;
11689 
11690 	if (atomic_read(&bp->intr_sem) != 0)
11691 		goto bnxt_restart_timer;
11692 
11693 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
11694 		bnxt_fw_health_check(bp);
11695 
11696 	if (BNXT_LINK_IS_UP(bp) && bp->stats_coal_ticks)
11697 		bnxt_queue_sp_work(bp, BNXT_PERIODIC_STATS_SP_EVENT);
11698 
11699 	if (bnxt_tc_flower_enabled(bp))
11700 		bnxt_queue_sp_work(bp, BNXT_FLOW_STATS_SP_EVENT);
11701 
11702 #ifdef CONFIG_RFS_ACCEL
11703 	if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count)
11704 		bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
11705 #endif /*CONFIG_RFS_ACCEL*/
11706 
11707 	if (bp->link_info.phy_retry) {
11708 		if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
11709 			bp->link_info.phy_retry = false;
11710 			netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
11711 		} else {
11712 			bnxt_queue_sp_work(bp, BNXT_UPDATE_PHY_SP_EVENT);
11713 		}
11714 	}
11715 
11716 	if (test_bit(BNXT_STATE_L2_FILTER_RETRY, &bp->state))
11717 		bnxt_queue_sp_work(bp, BNXT_RX_MASK_SP_EVENT);
11718 
11719 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
11720 	    netif_carrier_ok(dev))
11721 		bnxt_queue_sp_work(bp, BNXT_RING_COAL_NOW_SP_EVENT);
11722 
11723 bnxt_restart_timer:
11724 	mod_timer(&bp->timer, jiffies + bp->current_interval);
11725 }
11726 
11727 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
11728 {
11729 	/* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
11730 	 * set.  If the device is being closed, bnxt_close() may be holding
11731 	 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
11732 	 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
11733 	 */
11734 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11735 	rtnl_lock();
11736 }
11737 
11738 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
11739 {
11740 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11741 	rtnl_unlock();
11742 }
11743 
11744 /* Only called from bnxt_sp_task() */
11745 static void bnxt_reset(struct bnxt *bp, bool silent)
11746 {
11747 	bnxt_rtnl_lock_sp(bp);
11748 	if (test_bit(BNXT_STATE_OPEN, &bp->state))
11749 		bnxt_reset_task(bp, silent);
11750 	bnxt_rtnl_unlock_sp(bp);
11751 }
11752 
11753 /* Only called from bnxt_sp_task() */
11754 static void bnxt_rx_ring_reset(struct bnxt *bp)
11755 {
11756 	int i;
11757 
11758 	bnxt_rtnl_lock_sp(bp);
11759 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11760 		bnxt_rtnl_unlock_sp(bp);
11761 		return;
11762 	}
11763 	/* Disable and flush TPA before resetting the RX ring */
11764 	if (bp->flags & BNXT_FLAG_TPA)
11765 		bnxt_set_tpa(bp, false);
11766 	for (i = 0; i < bp->rx_nr_rings; i++) {
11767 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
11768 		struct bnxt_cp_ring_info *cpr;
11769 		int rc;
11770 
11771 		if (!rxr->bnapi->in_reset)
11772 			continue;
11773 
11774 		rc = bnxt_hwrm_rx_ring_reset(bp, i);
11775 		if (rc) {
11776 			if (rc == -EINVAL || rc == -EOPNOTSUPP)
11777 				netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
11778 			else
11779 				netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
11780 					    rc);
11781 			bnxt_reset_task(bp, true);
11782 			break;
11783 		}
11784 		bnxt_free_one_rx_ring_skbs(bp, i);
11785 		rxr->rx_prod = 0;
11786 		rxr->rx_agg_prod = 0;
11787 		rxr->rx_sw_agg_prod = 0;
11788 		rxr->rx_next_cons = 0;
11789 		rxr->bnapi->in_reset = false;
11790 		bnxt_alloc_one_rx_ring(bp, i);
11791 		cpr = &rxr->bnapi->cp_ring;
11792 		cpr->sw_stats.rx.rx_resets++;
11793 		if (bp->flags & BNXT_FLAG_AGG_RINGS)
11794 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
11795 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
11796 	}
11797 	if (bp->flags & BNXT_FLAG_TPA)
11798 		bnxt_set_tpa(bp, true);
11799 	bnxt_rtnl_unlock_sp(bp);
11800 }
11801 
11802 static void bnxt_fw_fatal_close(struct bnxt *bp)
11803 {
11804 	bnxt_tx_disable(bp);
11805 	bnxt_disable_napi(bp);
11806 	bnxt_disable_int_sync(bp);
11807 	bnxt_free_irq(bp);
11808 	bnxt_clear_int_mode(bp);
11809 	pci_disable_device(bp->pdev);
11810 }
11811 
11812 static void bnxt_fw_reset_close(struct bnxt *bp)
11813 {
11814 	bnxt_ulp_stop(bp);
11815 	/* When firmware is in fatal state, quiesce device and disable
11816 	 * bus master to prevent any potential bad DMAs before freeing
11817 	 * kernel memory.
11818 	 */
11819 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11820 		u16 val = 0;
11821 
11822 		pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
11823 		if (val == 0xffff)
11824 			bp->fw_reset_min_dsecs = 0;
11825 		bnxt_fw_fatal_close(bp);
11826 	}
11827 	__bnxt_close_nic(bp, true, false);
11828 	bnxt_vf_reps_free(bp);
11829 	bnxt_clear_int_mode(bp);
11830 	bnxt_hwrm_func_drv_unrgtr(bp);
11831 	if (pci_is_enabled(bp->pdev))
11832 		pci_disable_device(bp->pdev);
11833 	bnxt_free_ctx_mem(bp);
11834 	kfree(bp->ctx);
11835 	bp->ctx = NULL;
11836 }
11837 
11838 static bool is_bnxt_fw_ok(struct bnxt *bp)
11839 {
11840 	struct bnxt_fw_health *fw_health = bp->fw_health;
11841 	bool no_heartbeat = false, has_reset = false;
11842 	u32 val;
11843 
11844 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
11845 	if (val == fw_health->last_fw_heartbeat)
11846 		no_heartbeat = true;
11847 
11848 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
11849 	if (val != fw_health->last_fw_reset_cnt)
11850 		has_reset = true;
11851 
11852 	if (!no_heartbeat && has_reset)
11853 		return true;
11854 
11855 	return false;
11856 }
11857 
11858 /* rtnl_lock is acquired before calling this function */
11859 static void bnxt_force_fw_reset(struct bnxt *bp)
11860 {
11861 	struct bnxt_fw_health *fw_health = bp->fw_health;
11862 	struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11863 	u32 wait_dsecs;
11864 
11865 	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
11866 	    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
11867 		return;
11868 
11869 	if (ptp) {
11870 		spin_lock_bh(&ptp->ptp_lock);
11871 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11872 		spin_unlock_bh(&ptp->ptp_lock);
11873 	} else {
11874 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11875 	}
11876 	bnxt_fw_reset_close(bp);
11877 	wait_dsecs = fw_health->master_func_wait_dsecs;
11878 	if (fw_health->primary) {
11879 		if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
11880 			wait_dsecs = 0;
11881 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11882 	} else {
11883 		bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
11884 		wait_dsecs = fw_health->normal_func_wait_dsecs;
11885 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11886 	}
11887 
11888 	bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
11889 	bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
11890 	bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11891 }
11892 
11893 void bnxt_fw_exception(struct bnxt *bp)
11894 {
11895 	netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
11896 	set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11897 	bnxt_rtnl_lock_sp(bp);
11898 	bnxt_force_fw_reset(bp);
11899 	bnxt_rtnl_unlock_sp(bp);
11900 }
11901 
11902 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
11903  * < 0 on error.
11904  */
11905 static int bnxt_get_registered_vfs(struct bnxt *bp)
11906 {
11907 #ifdef CONFIG_BNXT_SRIOV
11908 	int rc;
11909 
11910 	if (!BNXT_PF(bp))
11911 		return 0;
11912 
11913 	rc = bnxt_hwrm_func_qcfg(bp);
11914 	if (rc) {
11915 		netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
11916 		return rc;
11917 	}
11918 	if (bp->pf.registered_vfs)
11919 		return bp->pf.registered_vfs;
11920 	if (bp->sriov_cfg)
11921 		return 1;
11922 #endif
11923 	return 0;
11924 }
11925 
11926 void bnxt_fw_reset(struct bnxt *bp)
11927 {
11928 	bnxt_rtnl_lock_sp(bp);
11929 	if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
11930 	    !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11931 		struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
11932 		int n = 0, tmo;
11933 
11934 		if (ptp) {
11935 			spin_lock_bh(&ptp->ptp_lock);
11936 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11937 			spin_unlock_bh(&ptp->ptp_lock);
11938 		} else {
11939 			set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11940 		}
11941 		if (bp->pf.active_vfs &&
11942 		    !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
11943 			n = bnxt_get_registered_vfs(bp);
11944 		if (n < 0) {
11945 			netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
11946 				   n);
11947 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11948 			dev_close(bp->dev);
11949 			goto fw_reset_exit;
11950 		} else if (n > 0) {
11951 			u16 vf_tmo_dsecs = n * 10;
11952 
11953 			if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
11954 				bp->fw_reset_max_dsecs = vf_tmo_dsecs;
11955 			bp->fw_reset_state =
11956 				BNXT_FW_RESET_STATE_POLL_VF;
11957 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11958 			goto fw_reset_exit;
11959 		}
11960 		bnxt_fw_reset_close(bp);
11961 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11962 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11963 			tmo = HZ / 10;
11964 		} else {
11965 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11966 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
11967 		}
11968 		bnxt_queue_fw_reset_work(bp, tmo);
11969 	}
11970 fw_reset_exit:
11971 	bnxt_rtnl_unlock_sp(bp);
11972 }
11973 
11974 static void bnxt_chk_missed_irq(struct bnxt *bp)
11975 {
11976 	int i;
11977 
11978 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11979 		return;
11980 
11981 	for (i = 0; i < bp->cp_nr_rings; i++) {
11982 		struct bnxt_napi *bnapi = bp->bnapi[i];
11983 		struct bnxt_cp_ring_info *cpr;
11984 		u32 fw_ring_id;
11985 		int j;
11986 
11987 		if (!bnapi)
11988 			continue;
11989 
11990 		cpr = &bnapi->cp_ring;
11991 		for (j = 0; j < 2; j++) {
11992 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
11993 			u32 val[2];
11994 
11995 			if (!cpr2 || cpr2->has_more_work ||
11996 			    !bnxt_has_work(bp, cpr2))
11997 				continue;
11998 
11999 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
12000 				cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
12001 				continue;
12002 			}
12003 			fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
12004 			bnxt_dbg_hwrm_ring_info_get(bp,
12005 				DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
12006 				fw_ring_id, &val[0], &val[1]);
12007 			cpr->sw_stats.cmn.missed_irqs++;
12008 		}
12009 	}
12010 }
12011 
12012 static void bnxt_cfg_ntp_filters(struct bnxt *);
12013 
12014 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
12015 {
12016 	struct bnxt_link_info *link_info = &bp->link_info;
12017 
12018 	if (BNXT_AUTO_MODE(link_info->auto_mode)) {
12019 		link_info->autoneg = BNXT_AUTONEG_SPEED;
12020 		if (bp->hwrm_spec_code >= 0x10201) {
12021 			if (link_info->auto_pause_setting &
12022 			    PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
12023 				link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12024 		} else {
12025 			link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
12026 		}
12027 		link_info->advertising = link_info->auto_link_speeds;
12028 		link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
12029 	} else {
12030 		link_info->req_link_speed = link_info->force_link_speed;
12031 		link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
12032 		if (link_info->force_pam4_link_speed) {
12033 			link_info->req_link_speed =
12034 				link_info->force_pam4_link_speed;
12035 			link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
12036 		}
12037 		link_info->req_duplex = link_info->duplex_setting;
12038 	}
12039 	if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
12040 		link_info->req_flow_ctrl =
12041 			link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
12042 	else
12043 		link_info->req_flow_ctrl = link_info->force_pause_setting;
12044 }
12045 
12046 static void bnxt_fw_echo_reply(struct bnxt *bp)
12047 {
12048 	struct bnxt_fw_health *fw_health = bp->fw_health;
12049 	struct hwrm_func_echo_response_input *req;
12050 	int rc;
12051 
12052 	rc = hwrm_req_init(bp, req, HWRM_FUNC_ECHO_RESPONSE);
12053 	if (rc)
12054 		return;
12055 	req->event_data1 = cpu_to_le32(fw_health->echo_req_data1);
12056 	req->event_data2 = cpu_to_le32(fw_health->echo_req_data2);
12057 	hwrm_req_send(bp, req);
12058 }
12059 
12060 static void bnxt_sp_task(struct work_struct *work)
12061 {
12062 	struct bnxt *bp = container_of(work, struct bnxt, sp_task);
12063 
12064 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12065 	smp_mb__after_atomic();
12066 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
12067 		clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12068 		return;
12069 	}
12070 
12071 	if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
12072 		bnxt_cfg_rx_mode(bp);
12073 
12074 	if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
12075 		bnxt_cfg_ntp_filters(bp);
12076 	if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
12077 		bnxt_hwrm_exec_fwd_req(bp);
12078 	if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
12079 		netdev_info(bp->dev, "Receive PF driver unload event!\n");
12080 	if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
12081 		bnxt_hwrm_port_qstats(bp, 0);
12082 		bnxt_hwrm_port_qstats_ext(bp, 0);
12083 		bnxt_accumulate_all_stats(bp);
12084 	}
12085 
12086 	if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
12087 		int rc;
12088 
12089 		mutex_lock(&bp->link_lock);
12090 		if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
12091 				       &bp->sp_event))
12092 			bnxt_hwrm_phy_qcaps(bp);
12093 
12094 		rc = bnxt_update_link(bp, true);
12095 		if (rc)
12096 			netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
12097 				   rc);
12098 
12099 		if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
12100 				       &bp->sp_event))
12101 			bnxt_init_ethtool_link_settings(bp);
12102 		mutex_unlock(&bp->link_lock);
12103 	}
12104 	if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
12105 		int rc;
12106 
12107 		mutex_lock(&bp->link_lock);
12108 		rc = bnxt_update_phy_setting(bp);
12109 		mutex_unlock(&bp->link_lock);
12110 		if (rc) {
12111 			netdev_warn(bp->dev, "update phy settings retry failed\n");
12112 		} else {
12113 			bp->link_info.phy_retry = false;
12114 			netdev_info(bp->dev, "update phy settings retry succeeded\n");
12115 		}
12116 	}
12117 	if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
12118 		mutex_lock(&bp->link_lock);
12119 		bnxt_get_port_module_status(bp);
12120 		mutex_unlock(&bp->link_lock);
12121 	}
12122 
12123 	if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
12124 		bnxt_tc_flow_stats_work(bp);
12125 
12126 	if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
12127 		bnxt_chk_missed_irq(bp);
12128 
12129 	if (test_and_clear_bit(BNXT_FW_ECHO_REQUEST_SP_EVENT, &bp->sp_event))
12130 		bnxt_fw_echo_reply(bp);
12131 
12132 	/* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
12133 	 * must be the last functions to be called before exiting.
12134 	 */
12135 	if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
12136 		bnxt_reset(bp, false);
12137 
12138 	if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
12139 		bnxt_reset(bp, true);
12140 
12141 	if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
12142 		bnxt_rx_ring_reset(bp);
12143 
12144 	if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) {
12145 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) ||
12146 		    test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state))
12147 			bnxt_devlink_health_fw_report(bp);
12148 		else
12149 			bnxt_fw_reset(bp);
12150 	}
12151 
12152 	if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
12153 		if (!is_bnxt_fw_ok(bp))
12154 			bnxt_devlink_health_fw_report(bp);
12155 	}
12156 
12157 	smp_mb__before_atomic();
12158 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
12159 }
12160 
12161 /* Under rtnl_lock */
12162 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
12163 		     int tx_xdp)
12164 {
12165 	int max_rx, max_tx, tx_sets = 1;
12166 	int tx_rings_needed, stats;
12167 	int rx_rings = rx;
12168 	int cp, vnics, rc;
12169 
12170 	if (tcs)
12171 		tx_sets = tcs;
12172 
12173 	rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
12174 	if (rc)
12175 		return rc;
12176 
12177 	if (max_rx < rx)
12178 		return -ENOMEM;
12179 
12180 	tx_rings_needed = tx * tx_sets + tx_xdp;
12181 	if (max_tx < tx_rings_needed)
12182 		return -ENOMEM;
12183 
12184 	vnics = 1;
12185 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
12186 		vnics += rx_rings;
12187 
12188 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
12189 		rx_rings <<= 1;
12190 	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
12191 	stats = cp;
12192 	if (BNXT_NEW_RM(bp)) {
12193 		cp += bnxt_get_ulp_msix_num(bp);
12194 		stats += bnxt_get_ulp_stat_ctxs(bp);
12195 	}
12196 	return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
12197 				     stats, vnics);
12198 }
12199 
12200 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
12201 {
12202 	if (bp->bar2) {
12203 		pci_iounmap(pdev, bp->bar2);
12204 		bp->bar2 = NULL;
12205 	}
12206 
12207 	if (bp->bar1) {
12208 		pci_iounmap(pdev, bp->bar1);
12209 		bp->bar1 = NULL;
12210 	}
12211 
12212 	if (bp->bar0) {
12213 		pci_iounmap(pdev, bp->bar0);
12214 		bp->bar0 = NULL;
12215 	}
12216 }
12217 
12218 static void bnxt_cleanup_pci(struct bnxt *bp)
12219 {
12220 	bnxt_unmap_bars(bp, bp->pdev);
12221 	pci_release_regions(bp->pdev);
12222 	if (pci_is_enabled(bp->pdev))
12223 		pci_disable_device(bp->pdev);
12224 }
12225 
12226 static void bnxt_init_dflt_coal(struct bnxt *bp)
12227 {
12228 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
12229 	struct bnxt_coal *coal;
12230 	u16 flags = 0;
12231 
12232 	if (coal_cap->cmpl_params &
12233 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
12234 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
12235 
12236 	/* Tick values in micro seconds.
12237 	 * 1 coal_buf x bufs_per_record = 1 completion record.
12238 	 */
12239 	coal = &bp->rx_coal;
12240 	coal->coal_ticks = 10;
12241 	coal->coal_bufs = 30;
12242 	coal->coal_ticks_irq = 1;
12243 	coal->coal_bufs_irq = 2;
12244 	coal->idle_thresh = 50;
12245 	coal->bufs_per_record = 2;
12246 	coal->budget = 64;		/* NAPI budget */
12247 	coal->flags = flags;
12248 
12249 	coal = &bp->tx_coal;
12250 	coal->coal_ticks = 28;
12251 	coal->coal_bufs = 30;
12252 	coal->coal_ticks_irq = 2;
12253 	coal->coal_bufs_irq = 2;
12254 	coal->bufs_per_record = 1;
12255 	coal->flags = flags;
12256 
12257 	bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
12258 }
12259 
12260 static int bnxt_fw_init_one_p1(struct bnxt *bp)
12261 {
12262 	int rc;
12263 
12264 	bp->fw_cap = 0;
12265 	rc = bnxt_hwrm_ver_get(bp);
12266 	/* FW may be unresponsive after FLR. FLR must complete within 100 msec
12267 	 * so wait before continuing with recovery.
12268 	 */
12269 	if (rc)
12270 		msleep(100);
12271 	bnxt_try_map_fw_health_reg(bp);
12272 	if (rc) {
12273 		rc = bnxt_try_recover_fw(bp);
12274 		if (rc)
12275 			return rc;
12276 		rc = bnxt_hwrm_ver_get(bp);
12277 		if (rc)
12278 			return rc;
12279 	}
12280 
12281 	bnxt_nvm_cfg_ver_get(bp);
12282 
12283 	rc = bnxt_hwrm_func_reset(bp);
12284 	if (rc)
12285 		return -ENODEV;
12286 
12287 	bnxt_hwrm_fw_set_time(bp);
12288 	return 0;
12289 }
12290 
12291 static int bnxt_fw_init_one_p2(struct bnxt *bp)
12292 {
12293 	int rc;
12294 
12295 	/* Get the MAX capabilities for this function */
12296 	rc = bnxt_hwrm_func_qcaps(bp);
12297 	if (rc) {
12298 		netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
12299 			   rc);
12300 		return -ENODEV;
12301 	}
12302 
12303 	rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
12304 	if (rc)
12305 		netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
12306 			    rc);
12307 
12308 	if (bnxt_alloc_fw_health(bp)) {
12309 		netdev_warn(bp->dev, "no memory for firmware error recovery\n");
12310 	} else {
12311 		rc = bnxt_hwrm_error_recovery_qcfg(bp);
12312 		if (rc)
12313 			netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
12314 				    rc);
12315 	}
12316 
12317 	rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
12318 	if (rc)
12319 		return -ENODEV;
12320 
12321 	bnxt_hwrm_func_qcfg(bp);
12322 	bnxt_hwrm_vnic_qcaps(bp);
12323 	bnxt_hwrm_port_led_qcaps(bp);
12324 	bnxt_ethtool_init(bp);
12325 	if (bp->fw_cap & BNXT_FW_CAP_PTP)
12326 		__bnxt_hwrm_ptp_qcfg(bp);
12327 	bnxt_dcb_init(bp);
12328 	return 0;
12329 }
12330 
12331 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
12332 {
12333 	bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
12334 	bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
12335 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
12336 			   VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
12337 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
12338 	if (bp->fw_cap & BNXT_FW_CAP_RSS_HASH_TYPE_DELTA)
12339 		bp->rss_hash_delta = bp->rss_hash_cfg;
12340 	if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
12341 		bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
12342 		bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
12343 				    VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
12344 	}
12345 }
12346 
12347 static void bnxt_set_dflt_rfs(struct bnxt *bp)
12348 {
12349 	struct net_device *dev = bp->dev;
12350 
12351 	dev->hw_features &= ~NETIF_F_NTUPLE;
12352 	dev->features &= ~NETIF_F_NTUPLE;
12353 	bp->flags &= ~BNXT_FLAG_RFS;
12354 	if (bnxt_rfs_supported(bp)) {
12355 		dev->hw_features |= NETIF_F_NTUPLE;
12356 		if (bnxt_rfs_capable(bp)) {
12357 			bp->flags |= BNXT_FLAG_RFS;
12358 			dev->features |= NETIF_F_NTUPLE;
12359 		}
12360 	}
12361 }
12362 
12363 static void bnxt_fw_init_one_p3(struct bnxt *bp)
12364 {
12365 	struct pci_dev *pdev = bp->pdev;
12366 
12367 	bnxt_set_dflt_rss_hash_type(bp);
12368 	bnxt_set_dflt_rfs(bp);
12369 
12370 	bnxt_get_wol_settings(bp);
12371 	if (bp->flags & BNXT_FLAG_WOL_CAP)
12372 		device_set_wakeup_enable(&pdev->dev, bp->wol);
12373 	else
12374 		device_set_wakeup_capable(&pdev->dev, false);
12375 
12376 	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
12377 	bnxt_hwrm_coal_params_qcaps(bp);
12378 }
12379 
12380 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt);
12381 
12382 int bnxt_fw_init_one(struct bnxt *bp)
12383 {
12384 	int rc;
12385 
12386 	rc = bnxt_fw_init_one_p1(bp);
12387 	if (rc) {
12388 		netdev_err(bp->dev, "Firmware init phase 1 failed\n");
12389 		return rc;
12390 	}
12391 	rc = bnxt_fw_init_one_p2(bp);
12392 	if (rc) {
12393 		netdev_err(bp->dev, "Firmware init phase 2 failed\n");
12394 		return rc;
12395 	}
12396 	rc = bnxt_probe_phy(bp, false);
12397 	if (rc)
12398 		return rc;
12399 	rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
12400 	if (rc)
12401 		return rc;
12402 
12403 	bnxt_fw_init_one_p3(bp);
12404 	return 0;
12405 }
12406 
12407 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
12408 {
12409 	struct bnxt_fw_health *fw_health = bp->fw_health;
12410 	u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
12411 	u32 val = fw_health->fw_reset_seq_vals[reg_idx];
12412 	u32 reg_type, reg_off, delay_msecs;
12413 
12414 	delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
12415 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
12416 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
12417 	switch (reg_type) {
12418 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
12419 		pci_write_config_dword(bp->pdev, reg_off, val);
12420 		break;
12421 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
12422 		writel(reg_off & BNXT_GRC_BASE_MASK,
12423 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
12424 		reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
12425 		fallthrough;
12426 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
12427 		writel(val, bp->bar0 + reg_off);
12428 		break;
12429 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
12430 		writel(val, bp->bar1 + reg_off);
12431 		break;
12432 	}
12433 	if (delay_msecs) {
12434 		pci_read_config_dword(bp->pdev, 0, &val);
12435 		msleep(delay_msecs);
12436 	}
12437 }
12438 
12439 bool bnxt_hwrm_reset_permitted(struct bnxt *bp)
12440 {
12441 	struct hwrm_func_qcfg_output *resp;
12442 	struct hwrm_func_qcfg_input *req;
12443 	bool result = true; /* firmware will enforce if unknown */
12444 
12445 	if (~bp->fw_cap & BNXT_FW_CAP_HOT_RESET_IF)
12446 		return result;
12447 
12448 	if (hwrm_req_init(bp, req, HWRM_FUNC_QCFG))
12449 		return result;
12450 
12451 	req->fid = cpu_to_le16(0xffff);
12452 	resp = hwrm_req_hold(bp, req);
12453 	if (!hwrm_req_send(bp, req))
12454 		result = !!(le16_to_cpu(resp->flags) &
12455 			    FUNC_QCFG_RESP_FLAGS_HOT_RESET_ALLOWED);
12456 	hwrm_req_drop(bp, req);
12457 	return result;
12458 }
12459 
12460 static void bnxt_reset_all(struct bnxt *bp)
12461 {
12462 	struct bnxt_fw_health *fw_health = bp->fw_health;
12463 	int i, rc;
12464 
12465 	if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12466 		bnxt_fw_reset_via_optee(bp);
12467 		bp->fw_reset_timestamp = jiffies;
12468 		return;
12469 	}
12470 
12471 	if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
12472 		for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
12473 			bnxt_fw_reset_writel(bp, i);
12474 	} else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
12475 		struct hwrm_fw_reset_input *req;
12476 
12477 		rc = hwrm_req_init(bp, req, HWRM_FW_RESET);
12478 		if (!rc) {
12479 			req->target_id = cpu_to_le16(HWRM_TARGET_ID_KONG);
12480 			req->embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
12481 			req->selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
12482 			req->flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
12483 			rc = hwrm_req_send(bp, req);
12484 		}
12485 		if (rc != -ENODEV)
12486 			netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
12487 	}
12488 	bp->fw_reset_timestamp = jiffies;
12489 }
12490 
12491 static bool bnxt_fw_reset_timeout(struct bnxt *bp)
12492 {
12493 	return time_after(jiffies, bp->fw_reset_timestamp +
12494 			  (bp->fw_reset_max_dsecs * HZ / 10));
12495 }
12496 
12497 static void bnxt_fw_reset_abort(struct bnxt *bp, int rc)
12498 {
12499 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12500 	if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) {
12501 		bnxt_ulp_start(bp, rc);
12502 		bnxt_dl_health_fw_status_update(bp, false);
12503 	}
12504 	bp->fw_reset_state = 0;
12505 	dev_close(bp->dev);
12506 }
12507 
12508 static void bnxt_fw_reset_task(struct work_struct *work)
12509 {
12510 	struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
12511 	int rc = 0;
12512 
12513 	if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
12514 		netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
12515 		return;
12516 	}
12517 
12518 	switch (bp->fw_reset_state) {
12519 	case BNXT_FW_RESET_STATE_POLL_VF: {
12520 		int n = bnxt_get_registered_vfs(bp);
12521 		int tmo;
12522 
12523 		if (n < 0) {
12524 			netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
12525 				   n, jiffies_to_msecs(jiffies -
12526 				   bp->fw_reset_timestamp));
12527 			goto fw_reset_abort;
12528 		} else if (n > 0) {
12529 			if (bnxt_fw_reset_timeout(bp)) {
12530 				clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12531 				bp->fw_reset_state = 0;
12532 				netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
12533 					   n);
12534 				return;
12535 			}
12536 			bnxt_queue_fw_reset_work(bp, HZ / 10);
12537 			return;
12538 		}
12539 		bp->fw_reset_timestamp = jiffies;
12540 		rtnl_lock();
12541 		if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
12542 			bnxt_fw_reset_abort(bp, rc);
12543 			rtnl_unlock();
12544 			return;
12545 		}
12546 		bnxt_fw_reset_close(bp);
12547 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
12548 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
12549 			tmo = HZ / 10;
12550 		} else {
12551 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12552 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
12553 		}
12554 		rtnl_unlock();
12555 		bnxt_queue_fw_reset_work(bp, tmo);
12556 		return;
12557 	}
12558 	case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
12559 		u32 val;
12560 
12561 		val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12562 		if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
12563 		    !bnxt_fw_reset_timeout(bp)) {
12564 			bnxt_queue_fw_reset_work(bp, HZ / 5);
12565 			return;
12566 		}
12567 
12568 		if (!bp->fw_health->primary) {
12569 			u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
12570 
12571 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12572 			bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
12573 			return;
12574 		}
12575 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
12576 	}
12577 		fallthrough;
12578 	case BNXT_FW_RESET_STATE_RESET_FW:
12579 		bnxt_reset_all(bp);
12580 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
12581 		bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
12582 		return;
12583 	case BNXT_FW_RESET_STATE_ENABLE_DEV:
12584 		bnxt_inv_fw_health_reg(bp);
12585 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) &&
12586 		    !bp->fw_reset_min_dsecs) {
12587 			u16 val;
12588 
12589 			pci_read_config_word(bp->pdev, PCI_SUBSYSTEM_ID, &val);
12590 			if (val == 0xffff) {
12591 				if (bnxt_fw_reset_timeout(bp)) {
12592 					netdev_err(bp->dev, "Firmware reset aborted, PCI config space invalid\n");
12593 					rc = -ETIMEDOUT;
12594 					goto fw_reset_abort;
12595 				}
12596 				bnxt_queue_fw_reset_work(bp, HZ / 1000);
12597 				return;
12598 			}
12599 		}
12600 		clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
12601 		clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state);
12602 		if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) &&
12603 		    !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state))
12604 			bnxt_dl_remote_reload(bp);
12605 		if (pci_enable_device(bp->pdev)) {
12606 			netdev_err(bp->dev, "Cannot re-enable PCI device\n");
12607 			rc = -ENODEV;
12608 			goto fw_reset_abort;
12609 		}
12610 		pci_set_master(bp->pdev);
12611 		bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
12612 		fallthrough;
12613 	case BNXT_FW_RESET_STATE_POLL_FW:
12614 		bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
12615 		rc = bnxt_hwrm_poll(bp);
12616 		if (rc) {
12617 			if (bnxt_fw_reset_timeout(bp)) {
12618 				netdev_err(bp->dev, "Firmware reset aborted\n");
12619 				goto fw_reset_abort_status;
12620 			}
12621 			bnxt_queue_fw_reset_work(bp, HZ / 5);
12622 			return;
12623 		}
12624 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
12625 		bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
12626 		fallthrough;
12627 	case BNXT_FW_RESET_STATE_OPENING:
12628 		while (!rtnl_trylock()) {
12629 			bnxt_queue_fw_reset_work(bp, HZ / 10);
12630 			return;
12631 		}
12632 		rc = bnxt_open(bp->dev);
12633 		if (rc) {
12634 			netdev_err(bp->dev, "bnxt_open() failed during FW reset\n");
12635 			bnxt_fw_reset_abort(bp, rc);
12636 			rtnl_unlock();
12637 			return;
12638 		}
12639 
12640 		if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) &&
12641 		    bp->fw_health->enabled) {
12642 			bp->fw_health->last_fw_reset_cnt =
12643 				bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
12644 		}
12645 		bp->fw_reset_state = 0;
12646 		/* Make sure fw_reset_state is 0 before clearing the flag */
12647 		smp_mb__before_atomic();
12648 		clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12649 		bnxt_ulp_start(bp, 0);
12650 		bnxt_reenable_sriov(bp);
12651 		bnxt_vf_reps_alloc(bp);
12652 		bnxt_vf_reps_open(bp);
12653 		bnxt_ptp_reapply_pps(bp);
12654 		clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state);
12655 		if (test_and_clear_bit(BNXT_STATE_RECOVER, &bp->state)) {
12656 			bnxt_dl_health_fw_recovery_done(bp);
12657 			bnxt_dl_health_fw_status_update(bp, true);
12658 		}
12659 		rtnl_unlock();
12660 		break;
12661 	}
12662 	return;
12663 
12664 fw_reset_abort_status:
12665 	if (bp->fw_health->status_reliable ||
12666 	    (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
12667 		u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
12668 
12669 		netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
12670 	}
12671 fw_reset_abort:
12672 	rtnl_lock();
12673 	bnxt_fw_reset_abort(bp, rc);
12674 	rtnl_unlock();
12675 }
12676 
12677 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
12678 {
12679 	int rc;
12680 	struct bnxt *bp = netdev_priv(dev);
12681 
12682 	SET_NETDEV_DEV(dev, &pdev->dev);
12683 
12684 	/* enable device (incl. PCI PM wakeup), and bus-mastering */
12685 	rc = pci_enable_device(pdev);
12686 	if (rc) {
12687 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
12688 		goto init_err;
12689 	}
12690 
12691 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
12692 		dev_err(&pdev->dev,
12693 			"Cannot find PCI device base address, aborting\n");
12694 		rc = -ENODEV;
12695 		goto init_err_disable;
12696 	}
12697 
12698 	rc = pci_request_regions(pdev, DRV_MODULE_NAME);
12699 	if (rc) {
12700 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
12701 		goto init_err_disable;
12702 	}
12703 
12704 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
12705 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
12706 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
12707 		rc = -EIO;
12708 		goto init_err_release;
12709 	}
12710 
12711 	pci_set_master(pdev);
12712 
12713 	bp->dev = dev;
12714 	bp->pdev = pdev;
12715 
12716 	/* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
12717 	 * determines the BAR size.
12718 	 */
12719 	bp->bar0 = pci_ioremap_bar(pdev, 0);
12720 	if (!bp->bar0) {
12721 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
12722 		rc = -ENOMEM;
12723 		goto init_err_release;
12724 	}
12725 
12726 	bp->bar2 = pci_ioremap_bar(pdev, 4);
12727 	if (!bp->bar2) {
12728 		dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
12729 		rc = -ENOMEM;
12730 		goto init_err_release;
12731 	}
12732 
12733 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
12734 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
12735 
12736 	spin_lock_init(&bp->ntp_fltr_lock);
12737 #if BITS_PER_LONG == 32
12738 	spin_lock_init(&bp->db_lock);
12739 #endif
12740 
12741 	bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
12742 	bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
12743 
12744 	timer_setup(&bp->timer, bnxt_timer, 0);
12745 	bp->current_interval = BNXT_TIMER_INTERVAL;
12746 
12747 	bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
12748 	bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
12749 
12750 	clear_bit(BNXT_STATE_OPEN, &bp->state);
12751 	return 0;
12752 
12753 init_err_release:
12754 	bnxt_unmap_bars(bp, pdev);
12755 	pci_release_regions(pdev);
12756 
12757 init_err_disable:
12758 	pci_disable_device(pdev);
12759 
12760 init_err:
12761 	return rc;
12762 }
12763 
12764 /* rtnl_lock held */
12765 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
12766 {
12767 	struct sockaddr *addr = p;
12768 	struct bnxt *bp = netdev_priv(dev);
12769 	int rc = 0;
12770 
12771 	if (!is_valid_ether_addr(addr->sa_data))
12772 		return -EADDRNOTAVAIL;
12773 
12774 	if (ether_addr_equal(addr->sa_data, dev->dev_addr))
12775 		return 0;
12776 
12777 	rc = bnxt_approve_mac(bp, addr->sa_data, true);
12778 	if (rc)
12779 		return rc;
12780 
12781 	eth_hw_addr_set(dev, addr->sa_data);
12782 	if (netif_running(dev)) {
12783 		bnxt_close_nic(bp, false, false);
12784 		rc = bnxt_open_nic(bp, false, false);
12785 	}
12786 
12787 	return rc;
12788 }
12789 
12790 /* rtnl_lock held */
12791 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
12792 {
12793 	struct bnxt *bp = netdev_priv(dev);
12794 
12795 	if (netif_running(dev))
12796 		bnxt_close_nic(bp, true, false);
12797 
12798 	dev->mtu = new_mtu;
12799 	bnxt_set_ring_params(bp);
12800 
12801 	if (netif_running(dev))
12802 		return bnxt_open_nic(bp, true, false);
12803 
12804 	return 0;
12805 }
12806 
12807 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
12808 {
12809 	struct bnxt *bp = netdev_priv(dev);
12810 	bool sh = false;
12811 	int rc;
12812 
12813 	if (tc > bp->max_tc) {
12814 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
12815 			   tc, bp->max_tc);
12816 		return -EINVAL;
12817 	}
12818 
12819 	if (netdev_get_num_tc(dev) == tc)
12820 		return 0;
12821 
12822 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
12823 		sh = true;
12824 
12825 	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
12826 			      sh, tc, bp->tx_nr_rings_xdp);
12827 	if (rc)
12828 		return rc;
12829 
12830 	/* Needs to close the device and do hw resource re-allocations */
12831 	if (netif_running(bp->dev))
12832 		bnxt_close_nic(bp, true, false);
12833 
12834 	if (tc) {
12835 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
12836 		netdev_set_num_tc(dev, tc);
12837 	} else {
12838 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12839 		netdev_reset_tc(dev);
12840 	}
12841 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
12842 	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
12843 			       bp->tx_nr_rings + bp->rx_nr_rings;
12844 
12845 	if (netif_running(bp->dev))
12846 		return bnxt_open_nic(bp, true, false);
12847 
12848 	return 0;
12849 }
12850 
12851 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
12852 				  void *cb_priv)
12853 {
12854 	struct bnxt *bp = cb_priv;
12855 
12856 	if (!bnxt_tc_flower_enabled(bp) ||
12857 	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
12858 		return -EOPNOTSUPP;
12859 
12860 	switch (type) {
12861 	case TC_SETUP_CLSFLOWER:
12862 		return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
12863 	default:
12864 		return -EOPNOTSUPP;
12865 	}
12866 }
12867 
12868 LIST_HEAD(bnxt_block_cb_list);
12869 
12870 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
12871 			 void *type_data)
12872 {
12873 	struct bnxt *bp = netdev_priv(dev);
12874 
12875 	switch (type) {
12876 	case TC_SETUP_BLOCK:
12877 		return flow_block_cb_setup_simple(type_data,
12878 						  &bnxt_block_cb_list,
12879 						  bnxt_setup_tc_block_cb,
12880 						  bp, bp, true);
12881 	case TC_SETUP_QDISC_MQPRIO: {
12882 		struct tc_mqprio_qopt *mqprio = type_data;
12883 
12884 		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
12885 
12886 		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
12887 	}
12888 	default:
12889 		return -EOPNOTSUPP;
12890 	}
12891 }
12892 
12893 #ifdef CONFIG_RFS_ACCEL
12894 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
12895 			    struct bnxt_ntuple_filter *f2)
12896 {
12897 	struct flow_keys *keys1 = &f1->fkeys;
12898 	struct flow_keys *keys2 = &f2->fkeys;
12899 
12900 	if (keys1->basic.n_proto != keys2->basic.n_proto ||
12901 	    keys1->basic.ip_proto != keys2->basic.ip_proto)
12902 		return false;
12903 
12904 	if (keys1->basic.n_proto == htons(ETH_P_IP)) {
12905 		if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
12906 		    keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
12907 			return false;
12908 	} else {
12909 		if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
12910 			   sizeof(keys1->addrs.v6addrs.src)) ||
12911 		    memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
12912 			   sizeof(keys1->addrs.v6addrs.dst)))
12913 			return false;
12914 	}
12915 
12916 	if (keys1->ports.ports == keys2->ports.ports &&
12917 	    keys1->control.flags == keys2->control.flags &&
12918 	    ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
12919 	    ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
12920 		return true;
12921 
12922 	return false;
12923 }
12924 
12925 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
12926 			      u16 rxq_index, u32 flow_id)
12927 {
12928 	struct bnxt *bp = netdev_priv(dev);
12929 	struct bnxt_ntuple_filter *fltr, *new_fltr;
12930 	struct flow_keys *fkeys;
12931 	struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
12932 	int rc = 0, idx, bit_id, l2_idx = 0;
12933 	struct hlist_head *head;
12934 	u32 flags;
12935 
12936 	if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
12937 		struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
12938 		int off = 0, j;
12939 
12940 		netif_addr_lock_bh(dev);
12941 		for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
12942 			if (ether_addr_equal(eth->h_dest,
12943 					     vnic->uc_list + off)) {
12944 				l2_idx = j + 1;
12945 				break;
12946 			}
12947 		}
12948 		netif_addr_unlock_bh(dev);
12949 		if (!l2_idx)
12950 			return -EINVAL;
12951 	}
12952 	new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
12953 	if (!new_fltr)
12954 		return -ENOMEM;
12955 
12956 	fkeys = &new_fltr->fkeys;
12957 	if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
12958 		rc = -EPROTONOSUPPORT;
12959 		goto err_free;
12960 	}
12961 
12962 	if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
12963 	     fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
12964 	    ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
12965 	     (fkeys->basic.ip_proto != IPPROTO_UDP))) {
12966 		rc = -EPROTONOSUPPORT;
12967 		goto err_free;
12968 	}
12969 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
12970 	    bp->hwrm_spec_code < 0x10601) {
12971 		rc = -EPROTONOSUPPORT;
12972 		goto err_free;
12973 	}
12974 	flags = fkeys->control.flags;
12975 	if (((flags & FLOW_DIS_ENCAPSULATION) &&
12976 	     bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
12977 		rc = -EPROTONOSUPPORT;
12978 		goto err_free;
12979 	}
12980 
12981 	memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
12982 	memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
12983 
12984 	idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
12985 	head = &bp->ntp_fltr_hash_tbl[idx];
12986 	rcu_read_lock();
12987 	hlist_for_each_entry_rcu(fltr, head, hash) {
12988 		if (bnxt_fltr_match(fltr, new_fltr)) {
12989 			rc = fltr->sw_id;
12990 			rcu_read_unlock();
12991 			goto err_free;
12992 		}
12993 	}
12994 	rcu_read_unlock();
12995 
12996 	spin_lock_bh(&bp->ntp_fltr_lock);
12997 	bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
12998 					 BNXT_NTP_FLTR_MAX_FLTR, 0);
12999 	if (bit_id < 0) {
13000 		spin_unlock_bh(&bp->ntp_fltr_lock);
13001 		rc = -ENOMEM;
13002 		goto err_free;
13003 	}
13004 
13005 	new_fltr->sw_id = (u16)bit_id;
13006 	new_fltr->flow_id = flow_id;
13007 	new_fltr->l2_fltr_idx = l2_idx;
13008 	new_fltr->rxq = rxq_index;
13009 	hlist_add_head_rcu(&new_fltr->hash, head);
13010 	bp->ntp_fltr_count++;
13011 	spin_unlock_bh(&bp->ntp_fltr_lock);
13012 
13013 	bnxt_queue_sp_work(bp, BNXT_RX_NTP_FLTR_SP_EVENT);
13014 
13015 	return new_fltr->sw_id;
13016 
13017 err_free:
13018 	kfree(new_fltr);
13019 	return rc;
13020 }
13021 
13022 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13023 {
13024 	int i;
13025 
13026 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
13027 		struct hlist_head *head;
13028 		struct hlist_node *tmp;
13029 		struct bnxt_ntuple_filter *fltr;
13030 		int rc;
13031 
13032 		head = &bp->ntp_fltr_hash_tbl[i];
13033 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
13034 			bool del = false;
13035 
13036 			if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
13037 				if (rps_may_expire_flow(bp->dev, fltr->rxq,
13038 							fltr->flow_id,
13039 							fltr->sw_id)) {
13040 					bnxt_hwrm_cfa_ntuple_filter_free(bp,
13041 									 fltr);
13042 					del = true;
13043 				}
13044 			} else {
13045 				rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
13046 								       fltr);
13047 				if (rc)
13048 					del = true;
13049 				else
13050 					set_bit(BNXT_FLTR_VALID, &fltr->state);
13051 			}
13052 
13053 			if (del) {
13054 				spin_lock_bh(&bp->ntp_fltr_lock);
13055 				hlist_del_rcu(&fltr->hash);
13056 				bp->ntp_fltr_count--;
13057 				spin_unlock_bh(&bp->ntp_fltr_lock);
13058 				synchronize_rcu();
13059 				clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
13060 				kfree(fltr);
13061 			}
13062 		}
13063 	}
13064 }
13065 
13066 #else
13067 
13068 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
13069 {
13070 }
13071 
13072 #endif /* CONFIG_RFS_ACCEL */
13073 
13074 static int bnxt_udp_tunnel_set_port(struct net_device *netdev, unsigned int table,
13075 				    unsigned int entry, struct udp_tunnel_info *ti)
13076 {
13077 	struct bnxt *bp = netdev_priv(netdev);
13078 	unsigned int cmd;
13079 
13080 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
13081 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13082 	else
13083 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13084 
13085 	return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti->port, cmd);
13086 }
13087 
13088 static int bnxt_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table,
13089 				      unsigned int entry, struct udp_tunnel_info *ti)
13090 {
13091 	struct bnxt *bp = netdev_priv(netdev);
13092 	unsigned int cmd;
13093 
13094 	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)
13095 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
13096 	else
13097 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
13098 
13099 	return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
13100 }
13101 
13102 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
13103 	.set_port	= bnxt_udp_tunnel_set_port,
13104 	.unset_port	= bnxt_udp_tunnel_unset_port,
13105 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
13106 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
13107 	.tables		= {
13108 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
13109 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
13110 	},
13111 };
13112 
13113 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13114 			       struct net_device *dev, u32 filter_mask,
13115 			       int nlflags)
13116 {
13117 	struct bnxt *bp = netdev_priv(dev);
13118 
13119 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
13120 				       nlflags, filter_mask, NULL);
13121 }
13122 
13123 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
13124 			       u16 flags, struct netlink_ext_ack *extack)
13125 {
13126 	struct bnxt *bp = netdev_priv(dev);
13127 	struct nlattr *attr, *br_spec;
13128 	int rem, rc = 0;
13129 
13130 	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
13131 		return -EOPNOTSUPP;
13132 
13133 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13134 	if (!br_spec)
13135 		return -EINVAL;
13136 
13137 	nla_for_each_nested(attr, br_spec, rem) {
13138 		u16 mode;
13139 
13140 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
13141 			continue;
13142 
13143 		mode = nla_get_u16(attr);
13144 		if (mode == bp->br_mode)
13145 			break;
13146 
13147 		rc = bnxt_hwrm_set_br_mode(bp, mode);
13148 		if (!rc)
13149 			bp->br_mode = mode;
13150 		break;
13151 	}
13152 	return rc;
13153 }
13154 
13155 int bnxt_get_port_parent_id(struct net_device *dev,
13156 			    struct netdev_phys_item_id *ppid)
13157 {
13158 	struct bnxt *bp = netdev_priv(dev);
13159 
13160 	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
13161 		return -EOPNOTSUPP;
13162 
13163 	/* The PF and it's VF-reps only support the switchdev framework */
13164 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
13165 		return -EOPNOTSUPP;
13166 
13167 	ppid->id_len = sizeof(bp->dsn);
13168 	memcpy(ppid->id, bp->dsn, ppid->id_len);
13169 
13170 	return 0;
13171 }
13172 
13173 static const struct net_device_ops bnxt_netdev_ops = {
13174 	.ndo_open		= bnxt_open,
13175 	.ndo_start_xmit		= bnxt_start_xmit,
13176 	.ndo_stop		= bnxt_close,
13177 	.ndo_get_stats64	= bnxt_get_stats64,
13178 	.ndo_set_rx_mode	= bnxt_set_rx_mode,
13179 	.ndo_eth_ioctl		= bnxt_ioctl,
13180 	.ndo_validate_addr	= eth_validate_addr,
13181 	.ndo_set_mac_address	= bnxt_change_mac_addr,
13182 	.ndo_change_mtu		= bnxt_change_mtu,
13183 	.ndo_fix_features	= bnxt_fix_features,
13184 	.ndo_set_features	= bnxt_set_features,
13185 	.ndo_features_check	= bnxt_features_check,
13186 	.ndo_tx_timeout		= bnxt_tx_timeout,
13187 #ifdef CONFIG_BNXT_SRIOV
13188 	.ndo_get_vf_config	= bnxt_get_vf_config,
13189 	.ndo_set_vf_mac		= bnxt_set_vf_mac,
13190 	.ndo_set_vf_vlan	= bnxt_set_vf_vlan,
13191 	.ndo_set_vf_rate	= bnxt_set_vf_bw,
13192 	.ndo_set_vf_link_state	= bnxt_set_vf_link_state,
13193 	.ndo_set_vf_spoofchk	= bnxt_set_vf_spoofchk,
13194 	.ndo_set_vf_trust	= bnxt_set_vf_trust,
13195 #endif
13196 	.ndo_setup_tc           = bnxt_setup_tc,
13197 #ifdef CONFIG_RFS_ACCEL
13198 	.ndo_rx_flow_steer	= bnxt_rx_flow_steer,
13199 #endif
13200 	.ndo_bpf		= bnxt_xdp,
13201 	.ndo_xdp_xmit		= bnxt_xdp_xmit,
13202 	.ndo_bridge_getlink	= bnxt_bridge_getlink,
13203 	.ndo_bridge_setlink	= bnxt_bridge_setlink,
13204 };
13205 
13206 static void bnxt_remove_one(struct pci_dev *pdev)
13207 {
13208 	struct net_device *dev = pci_get_drvdata(pdev);
13209 	struct bnxt *bp = netdev_priv(dev);
13210 
13211 	if (BNXT_PF(bp))
13212 		bnxt_sriov_disable(bp);
13213 
13214 	bnxt_rdma_aux_device_uninit(bp);
13215 
13216 	bnxt_ptp_clear(bp);
13217 	unregister_netdev(dev);
13218 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
13219 	/* Flush any pending tasks */
13220 	cancel_work_sync(&bp->sp_task);
13221 	cancel_delayed_work_sync(&bp->fw_reset_task);
13222 	bp->sp_event = 0;
13223 
13224 	bnxt_dl_fw_reporters_destroy(bp);
13225 	bnxt_dl_unregister(bp);
13226 	bnxt_shutdown_tc(bp);
13227 
13228 	bnxt_clear_int_mode(bp);
13229 	bnxt_hwrm_func_drv_unrgtr(bp);
13230 	bnxt_free_hwrm_resources(bp);
13231 	bnxt_ethtool_free(bp);
13232 	bnxt_dcb_free(bp);
13233 	kfree(bp->ptp_cfg);
13234 	bp->ptp_cfg = NULL;
13235 	kfree(bp->fw_health);
13236 	bp->fw_health = NULL;
13237 	bnxt_cleanup_pci(bp);
13238 	bnxt_free_ctx_mem(bp);
13239 	kfree(bp->ctx);
13240 	bp->ctx = NULL;
13241 	kfree(bp->rss_indir_tbl);
13242 	bp->rss_indir_tbl = NULL;
13243 	bnxt_free_port_stats(bp);
13244 	free_netdev(dev);
13245 }
13246 
13247 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
13248 {
13249 	int rc = 0;
13250 	struct bnxt_link_info *link_info = &bp->link_info;
13251 
13252 	bp->phy_flags = 0;
13253 	rc = bnxt_hwrm_phy_qcaps(bp);
13254 	if (rc) {
13255 		netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
13256 			   rc);
13257 		return rc;
13258 	}
13259 	if (bp->phy_flags & BNXT_PHY_FL_NO_FCS)
13260 		bp->dev->priv_flags |= IFF_SUPP_NOFCS;
13261 	else
13262 		bp->dev->priv_flags &= ~IFF_SUPP_NOFCS;
13263 	if (!fw_dflt)
13264 		return 0;
13265 
13266 	mutex_lock(&bp->link_lock);
13267 	rc = bnxt_update_link(bp, false);
13268 	if (rc) {
13269 		mutex_unlock(&bp->link_lock);
13270 		netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
13271 			   rc);
13272 		return rc;
13273 	}
13274 
13275 	/* Older firmware does not have supported_auto_speeds, so assume
13276 	 * that all supported speeds can be autonegotiated.
13277 	 */
13278 	if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
13279 		link_info->support_auto_speeds = link_info->support_speeds;
13280 
13281 	bnxt_init_ethtool_link_settings(bp);
13282 	mutex_unlock(&bp->link_lock);
13283 	return 0;
13284 }
13285 
13286 static int bnxt_get_max_irq(struct pci_dev *pdev)
13287 {
13288 	u16 ctrl;
13289 
13290 	if (!pdev->msix_cap)
13291 		return 1;
13292 
13293 	pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
13294 	return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
13295 }
13296 
13297 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13298 				int *max_cp)
13299 {
13300 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
13301 	int max_ring_grps = 0, max_irq;
13302 
13303 	*max_tx = hw_resc->max_tx_rings;
13304 	*max_rx = hw_resc->max_rx_rings;
13305 	*max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
13306 	max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
13307 			bnxt_get_ulp_msix_num(bp),
13308 			hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
13309 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
13310 		*max_cp = min_t(int, *max_cp, max_irq);
13311 	max_ring_grps = hw_resc->max_hw_ring_grps;
13312 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
13313 		*max_cp -= 1;
13314 		*max_rx -= 2;
13315 	}
13316 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
13317 		*max_rx >>= 1;
13318 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
13319 		bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
13320 		/* On P5 chips, max_cp output param should be available NQs */
13321 		*max_cp = max_irq;
13322 	}
13323 	*max_rx = min_t(int, *max_rx, max_ring_grps);
13324 }
13325 
13326 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
13327 {
13328 	int rx, tx, cp;
13329 
13330 	_bnxt_get_max_rings(bp, &rx, &tx, &cp);
13331 	*max_rx = rx;
13332 	*max_tx = tx;
13333 	if (!rx || !tx || !cp)
13334 		return -ENOMEM;
13335 
13336 	return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
13337 }
13338 
13339 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
13340 			       bool shared)
13341 {
13342 	int rc;
13343 
13344 	rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13345 	if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
13346 		/* Not enough rings, try disabling agg rings. */
13347 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
13348 		rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
13349 		if (rc) {
13350 			/* set BNXT_FLAG_AGG_RINGS back for consistency */
13351 			bp->flags |= BNXT_FLAG_AGG_RINGS;
13352 			return rc;
13353 		}
13354 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
13355 		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13356 		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
13357 		bnxt_set_ring_params(bp);
13358 	}
13359 
13360 	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
13361 		int max_cp, max_stat, max_irq;
13362 
13363 		/* Reserve minimum resources for RoCE */
13364 		max_cp = bnxt_get_max_func_cp_rings(bp);
13365 		max_stat = bnxt_get_max_func_stat_ctxs(bp);
13366 		max_irq = bnxt_get_max_func_irqs(bp);
13367 		if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
13368 		    max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
13369 		    max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
13370 			return 0;
13371 
13372 		max_cp -= BNXT_MIN_ROCE_CP_RINGS;
13373 		max_irq -= BNXT_MIN_ROCE_CP_RINGS;
13374 		max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
13375 		max_cp = min_t(int, max_cp, max_irq);
13376 		max_cp = min_t(int, max_cp, max_stat);
13377 		rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
13378 		if (rc)
13379 			rc = 0;
13380 	}
13381 	return rc;
13382 }
13383 
13384 /* In initial default shared ring setting, each shared ring must have a
13385  * RX/TX ring pair.
13386  */
13387 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
13388 {
13389 	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
13390 	bp->rx_nr_rings = bp->cp_nr_rings;
13391 	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
13392 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13393 }
13394 
13395 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
13396 {
13397 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
13398 
13399 	if (!bnxt_can_reserve_rings(bp))
13400 		return 0;
13401 
13402 	if (sh)
13403 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
13404 	dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
13405 	/* Reduce default rings on multi-port cards so that total default
13406 	 * rings do not exceed CPU count.
13407 	 */
13408 	if (bp->port_count > 1) {
13409 		int max_rings =
13410 			max_t(int, num_online_cpus() / bp->port_count, 1);
13411 
13412 		dflt_rings = min_t(int, dflt_rings, max_rings);
13413 	}
13414 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
13415 	if (rc)
13416 		return rc;
13417 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
13418 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
13419 	if (sh)
13420 		bnxt_trim_dflt_sh_rings(bp);
13421 	else
13422 		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
13423 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
13424 
13425 	rc = __bnxt_reserve_rings(bp);
13426 	if (rc && rc != -ENODEV)
13427 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
13428 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13429 	if (sh)
13430 		bnxt_trim_dflt_sh_rings(bp);
13431 
13432 	/* Rings may have been trimmed, re-reserve the trimmed rings. */
13433 	if (bnxt_need_reserve_rings(bp)) {
13434 		rc = __bnxt_reserve_rings(bp);
13435 		if (rc && rc != -ENODEV)
13436 			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
13437 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13438 	}
13439 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
13440 		bp->rx_nr_rings++;
13441 		bp->cp_nr_rings++;
13442 	}
13443 	if (rc) {
13444 		bp->tx_nr_rings = 0;
13445 		bp->rx_nr_rings = 0;
13446 	}
13447 	return rc;
13448 }
13449 
13450 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
13451 {
13452 	int rc;
13453 
13454 	if (bp->tx_nr_rings)
13455 		return 0;
13456 
13457 	bnxt_ulp_irq_stop(bp);
13458 	bnxt_clear_int_mode(bp);
13459 	rc = bnxt_set_dflt_rings(bp, true);
13460 	if (rc) {
13461 		if (BNXT_VF(bp) && rc == -ENODEV)
13462 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13463 		else
13464 			netdev_err(bp->dev, "Not enough rings available.\n");
13465 		goto init_dflt_ring_err;
13466 	}
13467 	rc = bnxt_init_int_mode(bp);
13468 	if (rc)
13469 		goto init_dflt_ring_err;
13470 
13471 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13472 
13473 	bnxt_set_dflt_rfs(bp);
13474 
13475 init_dflt_ring_err:
13476 	bnxt_ulp_irq_restart(bp, rc);
13477 	return rc;
13478 }
13479 
13480 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
13481 {
13482 	int rc;
13483 
13484 	ASSERT_RTNL();
13485 	bnxt_hwrm_func_qcaps(bp);
13486 
13487 	if (netif_running(bp->dev))
13488 		__bnxt_close_nic(bp, true, false);
13489 
13490 	bnxt_ulp_irq_stop(bp);
13491 	bnxt_clear_int_mode(bp);
13492 	rc = bnxt_init_int_mode(bp);
13493 	bnxt_ulp_irq_restart(bp, rc);
13494 
13495 	if (netif_running(bp->dev)) {
13496 		if (rc)
13497 			dev_close(bp->dev);
13498 		else
13499 			rc = bnxt_open_nic(bp, true, false);
13500 	}
13501 
13502 	return rc;
13503 }
13504 
13505 static int bnxt_init_mac_addr(struct bnxt *bp)
13506 {
13507 	int rc = 0;
13508 
13509 	if (BNXT_PF(bp)) {
13510 		eth_hw_addr_set(bp->dev, bp->pf.mac_addr);
13511 	} else {
13512 #ifdef CONFIG_BNXT_SRIOV
13513 		struct bnxt_vf_info *vf = &bp->vf;
13514 		bool strict_approval = true;
13515 
13516 		if (is_valid_ether_addr(vf->mac_addr)) {
13517 			/* overwrite netdev dev_addr with admin VF MAC */
13518 			eth_hw_addr_set(bp->dev, vf->mac_addr);
13519 			/* Older PF driver or firmware may not approve this
13520 			 * correctly.
13521 			 */
13522 			strict_approval = false;
13523 		} else {
13524 			eth_hw_addr_random(bp->dev);
13525 		}
13526 		rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
13527 #endif
13528 	}
13529 	return rc;
13530 }
13531 
13532 static void bnxt_vpd_read_info(struct bnxt *bp)
13533 {
13534 	struct pci_dev *pdev = bp->pdev;
13535 	unsigned int vpd_size, kw_len;
13536 	int pos, size;
13537 	u8 *vpd_data;
13538 
13539 	vpd_data = pci_vpd_alloc(pdev, &vpd_size);
13540 	if (IS_ERR(vpd_data)) {
13541 		pci_warn(pdev, "Unable to read VPD\n");
13542 		return;
13543 	}
13544 
13545 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13546 					   PCI_VPD_RO_KEYWORD_PARTNO, &kw_len);
13547 	if (pos < 0)
13548 		goto read_sn;
13549 
13550 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13551 	memcpy(bp->board_partno, &vpd_data[pos], size);
13552 
13553 read_sn:
13554 	pos = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size,
13555 					   PCI_VPD_RO_KEYWORD_SERIALNO,
13556 					   &kw_len);
13557 	if (pos < 0)
13558 		goto exit;
13559 
13560 	size = min_t(int, kw_len, BNXT_VPD_FLD_LEN - 1);
13561 	memcpy(bp->board_serialno, &vpd_data[pos], size);
13562 exit:
13563 	kfree(vpd_data);
13564 }
13565 
13566 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
13567 {
13568 	struct pci_dev *pdev = bp->pdev;
13569 	u64 qword;
13570 
13571 	qword = pci_get_dsn(pdev);
13572 	if (!qword) {
13573 		netdev_info(bp->dev, "Unable to read adapter's DSN\n");
13574 		return -EOPNOTSUPP;
13575 	}
13576 
13577 	put_unaligned_le64(qword, dsn);
13578 
13579 	bp->flags |= BNXT_FLAG_DSN_VALID;
13580 	return 0;
13581 }
13582 
13583 static int bnxt_map_db_bar(struct bnxt *bp)
13584 {
13585 	if (!bp->db_size)
13586 		return -ENODEV;
13587 	bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
13588 	if (!bp->bar1)
13589 		return -ENOMEM;
13590 	return 0;
13591 }
13592 
13593 void bnxt_print_device_info(struct bnxt *bp)
13594 {
13595 	netdev_info(bp->dev, "%s found at mem %lx, node addr %pM\n",
13596 		    board_info[bp->board_idx].name,
13597 		    (long)pci_resource_start(bp->pdev, 0), bp->dev->dev_addr);
13598 
13599 	pcie_print_link_status(bp->pdev);
13600 }
13601 
13602 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
13603 {
13604 	struct net_device *dev;
13605 	struct bnxt *bp;
13606 	int rc, max_irqs;
13607 
13608 	if (pci_is_bridge(pdev))
13609 		return -ENODEV;
13610 
13611 	/* Clear any pending DMA transactions from crash kernel
13612 	 * while loading driver in capture kernel.
13613 	 */
13614 	if (is_kdump_kernel()) {
13615 		pci_clear_master(pdev);
13616 		pcie_flr(pdev);
13617 	}
13618 
13619 	max_irqs = bnxt_get_max_irq(pdev);
13620 	dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
13621 	if (!dev)
13622 		return -ENOMEM;
13623 
13624 	bp = netdev_priv(dev);
13625 	bp->board_idx = ent->driver_data;
13626 	bp->msg_enable = BNXT_DEF_MSG_ENABLE;
13627 	bnxt_set_max_func_irqs(bp, max_irqs);
13628 
13629 	if (bnxt_vf_pciid(bp->board_idx))
13630 		bp->flags |= BNXT_FLAG_VF;
13631 
13632 	/* No devlink port registration in case of a VF */
13633 	if (BNXT_PF(bp))
13634 		SET_NETDEV_DEVLINK_PORT(dev, &bp->dl_port);
13635 
13636 	if (pdev->msix_cap)
13637 		bp->flags |= BNXT_FLAG_MSIX_CAP;
13638 
13639 	rc = bnxt_init_board(pdev, dev);
13640 	if (rc < 0)
13641 		goto init_err_free;
13642 
13643 	dev->netdev_ops = &bnxt_netdev_ops;
13644 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
13645 	dev->ethtool_ops = &bnxt_ethtool_ops;
13646 	pci_set_drvdata(pdev, dev);
13647 
13648 	rc = bnxt_alloc_hwrm_resources(bp);
13649 	if (rc)
13650 		goto init_err_pci_clean;
13651 
13652 	mutex_init(&bp->hwrm_cmd_lock);
13653 	mutex_init(&bp->link_lock);
13654 
13655 	rc = bnxt_fw_init_one_p1(bp);
13656 	if (rc)
13657 		goto init_err_pci_clean;
13658 
13659 	if (BNXT_PF(bp))
13660 		bnxt_vpd_read_info(bp);
13661 
13662 	if (BNXT_CHIP_P5(bp)) {
13663 		bp->flags |= BNXT_FLAG_CHIP_P5;
13664 		if (BNXT_CHIP_SR2(bp))
13665 			bp->flags |= BNXT_FLAG_CHIP_SR2;
13666 	}
13667 
13668 	rc = bnxt_alloc_rss_indir_tbl(bp);
13669 	if (rc)
13670 		goto init_err_pci_clean;
13671 
13672 	rc = bnxt_fw_init_one_p2(bp);
13673 	if (rc)
13674 		goto init_err_pci_clean;
13675 
13676 	rc = bnxt_map_db_bar(bp);
13677 	if (rc) {
13678 		dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
13679 			rc);
13680 		goto init_err_pci_clean;
13681 	}
13682 
13683 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13684 			   NETIF_F_TSO | NETIF_F_TSO6 |
13685 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13686 			   NETIF_F_GSO_IPXIP4 |
13687 			   NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13688 			   NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
13689 			   NETIF_F_RXCSUM | NETIF_F_GRO;
13690 
13691 	if (BNXT_SUPPORTS_TPA(bp))
13692 		dev->hw_features |= NETIF_F_LRO;
13693 
13694 	dev->hw_enc_features =
13695 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
13696 			NETIF_F_TSO | NETIF_F_TSO6 |
13697 			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
13698 			NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
13699 			NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
13700 	dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
13701 
13702 	dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
13703 				    NETIF_F_GSO_GRE_CSUM;
13704 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
13705 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
13706 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
13707 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
13708 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
13709 	if (BNXT_SUPPORTS_TPA(bp))
13710 		dev->hw_features |= NETIF_F_GRO_HW;
13711 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
13712 	if (dev->features & NETIF_F_GRO_HW)
13713 		dev->features &= ~NETIF_F_LRO;
13714 	dev->priv_flags |= IFF_UNICAST_FLT;
13715 
13716 	netif_set_tso_max_size(dev, GSO_MAX_SIZE);
13717 
13718 	dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
13719 			    NETDEV_XDP_ACT_RX_SG;
13720 
13721 #ifdef CONFIG_BNXT_SRIOV
13722 	init_waitqueue_head(&bp->sriov_cfg_wait);
13723 #endif
13724 	if (BNXT_SUPPORTS_TPA(bp)) {
13725 		bp->gro_func = bnxt_gro_func_5730x;
13726 		if (BNXT_CHIP_P4(bp))
13727 			bp->gro_func = bnxt_gro_func_5731x;
13728 		else if (BNXT_CHIP_P5(bp))
13729 			bp->gro_func = bnxt_gro_func_5750x;
13730 	}
13731 	if (!BNXT_CHIP_P4_PLUS(bp))
13732 		bp->flags |= BNXT_FLAG_DOUBLE_DB;
13733 
13734 	rc = bnxt_init_mac_addr(bp);
13735 	if (rc) {
13736 		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
13737 		rc = -EADDRNOTAVAIL;
13738 		goto init_err_pci_clean;
13739 	}
13740 
13741 	if (BNXT_PF(bp)) {
13742 		/* Read the adapter's DSN to use as the eswitch switch_id */
13743 		rc = bnxt_pcie_dsn_get(bp, bp->dsn);
13744 	}
13745 
13746 	/* MTU range: 60 - FW defined max */
13747 	dev->min_mtu = ETH_ZLEN;
13748 	dev->max_mtu = bp->max_mtu;
13749 
13750 	rc = bnxt_probe_phy(bp, true);
13751 	if (rc)
13752 		goto init_err_pci_clean;
13753 
13754 	bnxt_set_rx_skb_mode(bp, false);
13755 	bnxt_set_tpa_flags(bp);
13756 	bnxt_set_ring_params(bp);
13757 	rc = bnxt_set_dflt_rings(bp, true);
13758 	if (rc) {
13759 		if (BNXT_VF(bp) && rc == -ENODEV) {
13760 			netdev_err(bp->dev, "Cannot configure VF rings while PF is unavailable.\n");
13761 		} else {
13762 			netdev_err(bp->dev, "Not enough rings available.\n");
13763 			rc = -ENOMEM;
13764 		}
13765 		goto init_err_pci_clean;
13766 	}
13767 
13768 	bnxt_fw_init_one_p3(bp);
13769 
13770 	bnxt_init_dflt_coal(bp);
13771 
13772 	if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
13773 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
13774 
13775 	rc = bnxt_init_int_mode(bp);
13776 	if (rc)
13777 		goto init_err_pci_clean;
13778 
13779 	/* No TC has been set yet and rings may have been trimmed due to
13780 	 * limited MSIX, so we re-initialize the TX rings per TC.
13781 	 */
13782 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
13783 
13784 	if (BNXT_PF(bp)) {
13785 		if (!bnxt_pf_wq) {
13786 			bnxt_pf_wq =
13787 				create_singlethread_workqueue("bnxt_pf_wq");
13788 			if (!bnxt_pf_wq) {
13789 				dev_err(&pdev->dev, "Unable to create workqueue.\n");
13790 				rc = -ENOMEM;
13791 				goto init_err_pci_clean;
13792 			}
13793 		}
13794 		rc = bnxt_init_tc(bp);
13795 		if (rc)
13796 			netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
13797 				   rc);
13798 	}
13799 
13800 	bnxt_inv_fw_health_reg(bp);
13801 	rc = bnxt_dl_register(bp);
13802 	if (rc)
13803 		goto init_err_dl;
13804 
13805 	rc = register_netdev(dev);
13806 	if (rc)
13807 		goto init_err_cleanup;
13808 
13809 	bnxt_dl_fw_reporters_create(bp);
13810 
13811 	bnxt_rdma_aux_device_init(bp);
13812 
13813 	bnxt_print_device_info(bp);
13814 
13815 	pci_save_state(pdev);
13816 
13817 	return 0;
13818 init_err_cleanup:
13819 	bnxt_dl_unregister(bp);
13820 init_err_dl:
13821 	bnxt_shutdown_tc(bp);
13822 	bnxt_clear_int_mode(bp);
13823 
13824 init_err_pci_clean:
13825 	bnxt_hwrm_func_drv_unrgtr(bp);
13826 	bnxt_free_hwrm_resources(bp);
13827 	bnxt_ethtool_free(bp);
13828 	bnxt_ptp_clear(bp);
13829 	kfree(bp->ptp_cfg);
13830 	bp->ptp_cfg = NULL;
13831 	kfree(bp->fw_health);
13832 	bp->fw_health = NULL;
13833 	bnxt_cleanup_pci(bp);
13834 	bnxt_free_ctx_mem(bp);
13835 	kfree(bp->ctx);
13836 	bp->ctx = NULL;
13837 	kfree(bp->rss_indir_tbl);
13838 	bp->rss_indir_tbl = NULL;
13839 
13840 init_err_free:
13841 	free_netdev(dev);
13842 	return rc;
13843 }
13844 
13845 static void bnxt_shutdown(struct pci_dev *pdev)
13846 {
13847 	struct net_device *dev = pci_get_drvdata(pdev);
13848 	struct bnxt *bp;
13849 
13850 	if (!dev)
13851 		return;
13852 
13853 	rtnl_lock();
13854 	bp = netdev_priv(dev);
13855 	if (!bp)
13856 		goto shutdown_exit;
13857 
13858 	if (netif_running(dev))
13859 		dev_close(dev);
13860 
13861 	bnxt_clear_int_mode(bp);
13862 	pci_disable_device(pdev);
13863 
13864 	if (system_state == SYSTEM_POWER_OFF) {
13865 		pci_wake_from_d3(pdev, bp->wol);
13866 		pci_set_power_state(pdev, PCI_D3hot);
13867 	}
13868 
13869 shutdown_exit:
13870 	rtnl_unlock();
13871 }
13872 
13873 #ifdef CONFIG_PM_SLEEP
13874 static int bnxt_suspend(struct device *device)
13875 {
13876 	struct net_device *dev = dev_get_drvdata(device);
13877 	struct bnxt *bp = netdev_priv(dev);
13878 	int rc = 0;
13879 
13880 	rtnl_lock();
13881 	bnxt_ulp_stop(bp);
13882 	if (netif_running(dev)) {
13883 		netif_device_detach(dev);
13884 		rc = bnxt_close(dev);
13885 	}
13886 	bnxt_hwrm_func_drv_unrgtr(bp);
13887 	pci_disable_device(bp->pdev);
13888 	bnxt_free_ctx_mem(bp);
13889 	kfree(bp->ctx);
13890 	bp->ctx = NULL;
13891 	rtnl_unlock();
13892 	return rc;
13893 }
13894 
13895 static int bnxt_resume(struct device *device)
13896 {
13897 	struct net_device *dev = dev_get_drvdata(device);
13898 	struct bnxt *bp = netdev_priv(dev);
13899 	int rc = 0;
13900 
13901 	rtnl_lock();
13902 	rc = pci_enable_device(bp->pdev);
13903 	if (rc) {
13904 		netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
13905 			   rc);
13906 		goto resume_exit;
13907 	}
13908 	pci_set_master(bp->pdev);
13909 	if (bnxt_hwrm_ver_get(bp)) {
13910 		rc = -ENODEV;
13911 		goto resume_exit;
13912 	}
13913 	rc = bnxt_hwrm_func_reset(bp);
13914 	if (rc) {
13915 		rc = -EBUSY;
13916 		goto resume_exit;
13917 	}
13918 
13919 	rc = bnxt_hwrm_func_qcaps(bp);
13920 	if (rc)
13921 		goto resume_exit;
13922 
13923 	bnxt_clear_reservations(bp, true);
13924 
13925 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
13926 		rc = -ENODEV;
13927 		goto resume_exit;
13928 	}
13929 
13930 	bnxt_get_wol_settings(bp);
13931 	if (netif_running(dev)) {
13932 		rc = bnxt_open(dev);
13933 		if (!rc)
13934 			netif_device_attach(dev);
13935 	}
13936 
13937 resume_exit:
13938 	bnxt_ulp_start(bp, rc);
13939 	if (!rc)
13940 		bnxt_reenable_sriov(bp);
13941 	rtnl_unlock();
13942 	return rc;
13943 }
13944 
13945 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
13946 #define BNXT_PM_OPS (&bnxt_pm_ops)
13947 
13948 #else
13949 
13950 #define BNXT_PM_OPS NULL
13951 
13952 #endif /* CONFIG_PM_SLEEP */
13953 
13954 /**
13955  * bnxt_io_error_detected - called when PCI error is detected
13956  * @pdev: Pointer to PCI device
13957  * @state: The current pci connection state
13958  *
13959  * This function is called after a PCI bus error affecting
13960  * this device has been detected.
13961  */
13962 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
13963 					       pci_channel_state_t state)
13964 {
13965 	struct net_device *netdev = pci_get_drvdata(pdev);
13966 	struct bnxt *bp = netdev_priv(netdev);
13967 	bool abort = false;
13968 
13969 	netdev_info(netdev, "PCI I/O error detected\n");
13970 
13971 	rtnl_lock();
13972 	netif_device_detach(netdev);
13973 
13974 	bnxt_ulp_stop(bp);
13975 
13976 	if (test_and_set_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
13977 		netdev_err(bp->dev, "Firmware reset already in progress\n");
13978 		abort = true;
13979 	}
13980 
13981 	if (abort || state == pci_channel_io_perm_failure) {
13982 		rtnl_unlock();
13983 		return PCI_ERS_RESULT_DISCONNECT;
13984 	}
13985 
13986 	/* Link is not reliable anymore if state is pci_channel_io_frozen
13987 	 * so we disable bus master to prevent any potential bad DMAs before
13988 	 * freeing kernel memory.
13989 	 */
13990 	if (state == pci_channel_io_frozen) {
13991 		set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
13992 		bnxt_fw_fatal_close(bp);
13993 	}
13994 
13995 	if (netif_running(netdev))
13996 		__bnxt_close_nic(bp, true, true);
13997 
13998 	if (pci_is_enabled(pdev))
13999 		pci_disable_device(pdev);
14000 	bnxt_free_ctx_mem(bp);
14001 	kfree(bp->ctx);
14002 	bp->ctx = NULL;
14003 	rtnl_unlock();
14004 
14005 	/* Request a slot slot reset. */
14006 	return PCI_ERS_RESULT_NEED_RESET;
14007 }
14008 
14009 /**
14010  * bnxt_io_slot_reset - called after the pci bus has been reset.
14011  * @pdev: Pointer to PCI device
14012  *
14013  * Restart the card from scratch, as if from a cold-boot.
14014  * At this point, the card has exprienced a hard reset,
14015  * followed by fixups by BIOS, and has its config space
14016  * set up identically to what it was at cold boot.
14017  */
14018 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
14019 {
14020 	pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
14021 	struct net_device *netdev = pci_get_drvdata(pdev);
14022 	struct bnxt *bp = netdev_priv(netdev);
14023 	int retry = 0;
14024 	int err = 0;
14025 	int off;
14026 
14027 	netdev_info(bp->dev, "PCI Slot Reset\n");
14028 
14029 	rtnl_lock();
14030 
14031 	if (pci_enable_device(pdev)) {
14032 		dev_err(&pdev->dev,
14033 			"Cannot re-enable PCI device after reset.\n");
14034 	} else {
14035 		pci_set_master(pdev);
14036 		/* Upon fatal error, our device internal logic that latches to
14037 		 * BAR value is getting reset and will restore only upon
14038 		 * rewritting the BARs.
14039 		 *
14040 		 * As pci_restore_state() does not re-write the BARs if the
14041 		 * value is same as saved value earlier, driver needs to
14042 		 * write the BARs to 0 to force restore, in case of fatal error.
14043 		 */
14044 		if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
14045 				       &bp->state)) {
14046 			for (off = PCI_BASE_ADDRESS_0;
14047 			     off <= PCI_BASE_ADDRESS_5; off += 4)
14048 				pci_write_config_dword(bp->pdev, off, 0);
14049 		}
14050 		pci_restore_state(pdev);
14051 		pci_save_state(pdev);
14052 
14053 		bnxt_inv_fw_health_reg(bp);
14054 		bnxt_try_map_fw_health_reg(bp);
14055 
14056 		/* In some PCIe AER scenarios, firmware may take up to
14057 		 * 10 seconds to become ready in the worst case.
14058 		 */
14059 		do {
14060 			err = bnxt_try_recover_fw(bp);
14061 			if (!err)
14062 				break;
14063 			retry++;
14064 		} while (retry < BNXT_FW_SLOT_RESET_RETRY);
14065 
14066 		if (err) {
14067 			dev_err(&pdev->dev, "Firmware not ready\n");
14068 			goto reset_exit;
14069 		}
14070 
14071 		err = bnxt_hwrm_func_reset(bp);
14072 		if (!err)
14073 			result = PCI_ERS_RESULT_RECOVERED;
14074 
14075 		bnxt_ulp_irq_stop(bp);
14076 		bnxt_clear_int_mode(bp);
14077 		err = bnxt_init_int_mode(bp);
14078 		bnxt_ulp_irq_restart(bp, err);
14079 	}
14080 
14081 reset_exit:
14082 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
14083 	bnxt_clear_reservations(bp, true);
14084 	rtnl_unlock();
14085 
14086 	return result;
14087 }
14088 
14089 /**
14090  * bnxt_io_resume - called when traffic can start flowing again.
14091  * @pdev: Pointer to PCI device
14092  *
14093  * This callback is called when the error recovery driver tells
14094  * us that its OK to resume normal operation.
14095  */
14096 static void bnxt_io_resume(struct pci_dev *pdev)
14097 {
14098 	struct net_device *netdev = pci_get_drvdata(pdev);
14099 	struct bnxt *bp = netdev_priv(netdev);
14100 	int err;
14101 
14102 	netdev_info(bp->dev, "PCI Slot Resume\n");
14103 	rtnl_lock();
14104 
14105 	err = bnxt_hwrm_func_qcaps(bp);
14106 	if (!err && netif_running(netdev))
14107 		err = bnxt_open(netdev);
14108 
14109 	bnxt_ulp_start(bp, err);
14110 	if (!err) {
14111 		bnxt_reenable_sriov(bp);
14112 		netif_device_attach(netdev);
14113 	}
14114 
14115 	rtnl_unlock();
14116 }
14117 
14118 static const struct pci_error_handlers bnxt_err_handler = {
14119 	.error_detected	= bnxt_io_error_detected,
14120 	.slot_reset	= bnxt_io_slot_reset,
14121 	.resume		= bnxt_io_resume
14122 };
14123 
14124 static struct pci_driver bnxt_pci_driver = {
14125 	.name		= DRV_MODULE_NAME,
14126 	.id_table	= bnxt_pci_tbl,
14127 	.probe		= bnxt_init_one,
14128 	.remove		= bnxt_remove_one,
14129 	.shutdown	= bnxt_shutdown,
14130 	.driver.pm	= BNXT_PM_OPS,
14131 	.err_handler	= &bnxt_err_handler,
14132 #if defined(CONFIG_BNXT_SRIOV)
14133 	.sriov_configure = bnxt_sriov_configure,
14134 #endif
14135 };
14136 
14137 static int __init bnxt_init(void)
14138 {
14139 	int err;
14140 
14141 	bnxt_debug_init();
14142 	err = pci_register_driver(&bnxt_pci_driver);
14143 	if (err) {
14144 		bnxt_debug_exit();
14145 		return err;
14146 	}
14147 
14148 	return 0;
14149 }
14150 
14151 static void __exit bnxt_exit(void)
14152 {
14153 	pci_unregister_driver(&bnxt_pci_driver);
14154 	if (bnxt_pf_wq)
14155 		destroy_workqueue(bnxt_pf_wq);
14156 	bnxt_debug_exit();
14157 }
14158 
14159 module_init(bnxt_init);
14160 module_exit(bnxt_exit);
14161