1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2019 Broadcom Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  */
10 
11 #include <linux/module.h>
12 
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/ip.h>
41 #include <net/tcp.h>
42 #include <net/udp.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <net/udp_tunnel.h>
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool.h>
58 
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_ulp.h"
62 #include "bnxt_sriov.h"
63 #include "bnxt_ethtool.h"
64 #include "bnxt_dcb.h"
65 #include "bnxt_xdp.h"
66 #include "bnxt_vfr.h"
67 #include "bnxt_tc.h"
68 #include "bnxt_devlink.h"
69 #include "bnxt_debugfs.h"
70 
71 #define BNXT_TX_TIMEOUT		(5 * HZ)
72 #define BNXT_DEF_MSG_ENABLE	(NETIF_MSG_DRV | NETIF_MSG_HW)
73 
74 MODULE_LICENSE("GPL");
75 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
76 
77 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
78 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
79 #define BNXT_RX_COPY_THRESH 256
80 
81 #define BNXT_TX_PUSH_THRESH 164
82 
83 enum board_idx {
84 	BCM57301,
85 	BCM57302,
86 	BCM57304,
87 	BCM57417_NPAR,
88 	BCM58700,
89 	BCM57311,
90 	BCM57312,
91 	BCM57402,
92 	BCM57404,
93 	BCM57406,
94 	BCM57402_NPAR,
95 	BCM57407,
96 	BCM57412,
97 	BCM57414,
98 	BCM57416,
99 	BCM57417,
100 	BCM57412_NPAR,
101 	BCM57314,
102 	BCM57417_SFP,
103 	BCM57416_SFP,
104 	BCM57404_NPAR,
105 	BCM57406_NPAR,
106 	BCM57407_SFP,
107 	BCM57407_NPAR,
108 	BCM57414_NPAR,
109 	BCM57416_NPAR,
110 	BCM57452,
111 	BCM57454,
112 	BCM5745x_NPAR,
113 	BCM57508,
114 	BCM57504,
115 	BCM57502,
116 	BCM57508_NPAR,
117 	BCM57504_NPAR,
118 	BCM57502_NPAR,
119 	BCM58802,
120 	BCM58804,
121 	BCM58808,
122 	NETXTREME_E_VF,
123 	NETXTREME_C_VF,
124 	NETXTREME_S_VF,
125 	NETXTREME_E_P5_VF,
126 };
127 
128 /* indexed by enum above */
129 static const struct {
130 	char *name;
131 } board_info[] = {
132 	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
133 	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
134 	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
135 	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
136 	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
137 	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
138 	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
139 	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
140 	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
141 	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
142 	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
143 	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
144 	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
145 	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
146 	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
147 	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
148 	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
149 	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
150 	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
151 	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
152 	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
153 	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
154 	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
155 	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
156 	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
157 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
158 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
159 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
160 	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
161 	[BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
162 	[BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
163 	[BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
164 	[BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
165 	[BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
166 	[BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
167 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
168 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
169 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
170 	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
171 	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
172 	[NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
173 	[NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
174 };
175 
176 static const struct pci_device_id bnxt_pci_tbl[] = {
177 	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
178 	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
179 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
180 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
181 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
182 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
183 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
184 	{ PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
185 	{ PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
186 	{ PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
187 	{ PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
188 	{ PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
189 	{ PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
190 	{ PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
191 	{ PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
192 	{ PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
193 	{ PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
194 	{ PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
195 	{ PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
196 	{ PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
197 	{ PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
198 	{ PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
199 	{ PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
200 	{ PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
201 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
202 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
203 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
204 	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
205 	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
206 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
207 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
208 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
209 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
210 	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
211 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
212 	{ PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
213 	{ PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
214 	{ PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
215 	{ PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
216 	{ PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
217 	{ PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
218 	{ PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
219 	{ PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
220 	{ PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
221 	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
222 	{ PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
223 #ifdef CONFIG_BNXT_SRIOV
224 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
225 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
226 	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
227 	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
228 	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
229 	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
230 	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
231 	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
232 	{ PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
233 	{ PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
234 	{ PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
235 #endif
236 	{ 0 }
237 };
238 
239 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
240 
241 static const u16 bnxt_vf_req_snif[] = {
242 	HWRM_FUNC_CFG,
243 	HWRM_FUNC_VF_CFG,
244 	HWRM_PORT_PHY_QCFG,
245 	HWRM_CFA_L2_FILTER_ALLOC,
246 };
247 
248 static const u16 bnxt_async_events_arr[] = {
249 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
250 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
251 	ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
252 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
253 	ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
254 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
255 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
256 	ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
257 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
258 	ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG,
259 };
260 
261 static struct workqueue_struct *bnxt_pf_wq;
262 
263 static bool bnxt_vf_pciid(enum board_idx idx)
264 {
265 	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
266 		idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
267 }
268 
269 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
270 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
271 #define DB_CP_IRQ_DIS_FLAGS	(DB_KEY_CP | DB_IRQ_DIS)
272 
273 #define BNXT_CP_DB_IRQ_DIS(db)						\
274 		writel(DB_CP_IRQ_DIS_FLAGS, db)
275 
276 #define BNXT_DB_CQ(db, idx)						\
277 	writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
278 
279 #define BNXT_DB_NQ_P5(db, idx)						\
280 	writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
281 
282 #define BNXT_DB_CQ_ARM(db, idx)						\
283 	writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
284 
285 #define BNXT_DB_NQ_ARM_P5(db, idx)					\
286 	writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
287 
288 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
289 {
290 	if (bp->flags & BNXT_FLAG_CHIP_P5)
291 		BNXT_DB_NQ_P5(db, idx);
292 	else
293 		BNXT_DB_CQ(db, idx);
294 }
295 
296 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
297 {
298 	if (bp->flags & BNXT_FLAG_CHIP_P5)
299 		BNXT_DB_NQ_ARM_P5(db, idx);
300 	else
301 		BNXT_DB_CQ_ARM(db, idx);
302 }
303 
304 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
305 {
306 	if (bp->flags & BNXT_FLAG_CHIP_P5)
307 		writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
308 		       db->doorbell);
309 	else
310 		BNXT_DB_CQ(db, idx);
311 }
312 
313 const u16 bnxt_lhint_arr[] = {
314 	TX_BD_FLAGS_LHINT_512_AND_SMALLER,
315 	TX_BD_FLAGS_LHINT_512_TO_1023,
316 	TX_BD_FLAGS_LHINT_1024_TO_2047,
317 	TX_BD_FLAGS_LHINT_1024_TO_2047,
318 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
319 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
320 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
321 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
322 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
323 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
324 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
325 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
326 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
327 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
328 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
329 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
330 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
331 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
332 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
333 };
334 
335 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
336 {
337 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
338 
339 	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
340 		return 0;
341 
342 	return md_dst->u.port_info.port_id;
343 }
344 
345 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
346 {
347 	struct bnxt *bp = netdev_priv(dev);
348 	struct tx_bd *txbd;
349 	struct tx_bd_ext *txbd1;
350 	struct netdev_queue *txq;
351 	int i;
352 	dma_addr_t mapping;
353 	unsigned int length, pad = 0;
354 	u32 len, free_size, vlan_tag_flags, cfa_action, flags;
355 	u16 prod, last_frag;
356 	struct pci_dev *pdev = bp->pdev;
357 	struct bnxt_tx_ring_info *txr;
358 	struct bnxt_sw_tx_bd *tx_buf;
359 
360 	i = skb_get_queue_mapping(skb);
361 	if (unlikely(i >= bp->tx_nr_rings)) {
362 		dev_kfree_skb_any(skb);
363 		return NETDEV_TX_OK;
364 	}
365 
366 	txq = netdev_get_tx_queue(dev, i);
367 	txr = &bp->tx_ring[bp->tx_ring_map[i]];
368 	prod = txr->tx_prod;
369 
370 	free_size = bnxt_tx_avail(bp, txr);
371 	if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
372 		netif_tx_stop_queue(txq);
373 		return NETDEV_TX_BUSY;
374 	}
375 
376 	length = skb->len;
377 	len = skb_headlen(skb);
378 	last_frag = skb_shinfo(skb)->nr_frags;
379 
380 	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
381 
382 	txbd->tx_bd_opaque = prod;
383 
384 	tx_buf = &txr->tx_buf_ring[prod];
385 	tx_buf->skb = skb;
386 	tx_buf->nr_frags = last_frag;
387 
388 	vlan_tag_flags = 0;
389 	cfa_action = bnxt_xmit_get_cfa_action(skb);
390 	if (skb_vlan_tag_present(skb)) {
391 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
392 				 skb_vlan_tag_get(skb);
393 		/* Currently supports 8021Q, 8021AD vlan offloads
394 		 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
395 		 */
396 		if (skb->vlan_proto == htons(ETH_P_8021Q))
397 			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
398 	}
399 
400 	if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
401 		struct tx_push_buffer *tx_push_buf = txr->tx_push;
402 		struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
403 		struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
404 		void __iomem *db = txr->tx_db.doorbell;
405 		void *pdata = tx_push_buf->data;
406 		u64 *end;
407 		int j, push_len;
408 
409 		/* Set COAL_NOW to be ready quickly for the next push */
410 		tx_push->tx_bd_len_flags_type =
411 			cpu_to_le32((length << TX_BD_LEN_SHIFT) |
412 					TX_BD_TYPE_LONG_TX_BD |
413 					TX_BD_FLAGS_LHINT_512_AND_SMALLER |
414 					TX_BD_FLAGS_COAL_NOW |
415 					TX_BD_FLAGS_PACKET_END |
416 					(2 << TX_BD_FLAGS_BD_CNT_SHIFT));
417 
418 		if (skb->ip_summed == CHECKSUM_PARTIAL)
419 			tx_push1->tx_bd_hsize_lflags =
420 					cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
421 		else
422 			tx_push1->tx_bd_hsize_lflags = 0;
423 
424 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
425 		tx_push1->tx_bd_cfa_action =
426 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
427 
428 		end = pdata + length;
429 		end = PTR_ALIGN(end, 8) - 1;
430 		*end = 0;
431 
432 		skb_copy_from_linear_data(skb, pdata, len);
433 		pdata += len;
434 		for (j = 0; j < last_frag; j++) {
435 			skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
436 			void *fptr;
437 
438 			fptr = skb_frag_address_safe(frag);
439 			if (!fptr)
440 				goto normal_tx;
441 
442 			memcpy(pdata, fptr, skb_frag_size(frag));
443 			pdata += skb_frag_size(frag);
444 		}
445 
446 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
447 		txbd->tx_bd_haddr = txr->data_mapping;
448 		prod = NEXT_TX(prod);
449 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
450 		memcpy(txbd, tx_push1, sizeof(*txbd));
451 		prod = NEXT_TX(prod);
452 		tx_push->doorbell =
453 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
454 		txr->tx_prod = prod;
455 
456 		tx_buf->is_push = 1;
457 		netdev_tx_sent_queue(txq, skb->len);
458 		wmb();	/* Sync is_push and byte queue before pushing data */
459 
460 		push_len = (length + sizeof(*tx_push) + 7) / 8;
461 		if (push_len > 16) {
462 			__iowrite64_copy(db, tx_push_buf, 16);
463 			__iowrite32_copy(db + 4, tx_push_buf + 1,
464 					 (push_len - 16) << 1);
465 		} else {
466 			__iowrite64_copy(db, tx_push_buf, push_len);
467 		}
468 
469 		goto tx_done;
470 	}
471 
472 normal_tx:
473 	if (length < BNXT_MIN_PKT_SIZE) {
474 		pad = BNXT_MIN_PKT_SIZE - length;
475 		if (skb_pad(skb, pad)) {
476 			/* SKB already freed. */
477 			tx_buf->skb = NULL;
478 			return NETDEV_TX_OK;
479 		}
480 		length = BNXT_MIN_PKT_SIZE;
481 	}
482 
483 	mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
484 
485 	if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
486 		dev_kfree_skb_any(skb);
487 		tx_buf->skb = NULL;
488 		return NETDEV_TX_OK;
489 	}
490 
491 	dma_unmap_addr_set(tx_buf, mapping, mapping);
492 	flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
493 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
494 
495 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
496 
497 	prod = NEXT_TX(prod);
498 	txbd1 = (struct tx_bd_ext *)
499 		&txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
500 
501 	txbd1->tx_bd_hsize_lflags = 0;
502 	if (skb_is_gso(skb)) {
503 		u32 hdr_len;
504 
505 		if (skb->encapsulation)
506 			hdr_len = skb_inner_network_offset(skb) +
507 				skb_inner_network_header_len(skb) +
508 				inner_tcp_hdrlen(skb);
509 		else
510 			hdr_len = skb_transport_offset(skb) +
511 				tcp_hdrlen(skb);
512 
513 		txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
514 					TX_BD_FLAGS_T_IPID |
515 					(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
516 		length = skb_shinfo(skb)->gso_size;
517 		txbd1->tx_bd_mss = cpu_to_le32(length);
518 		length += hdr_len;
519 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
520 		txbd1->tx_bd_hsize_lflags =
521 			cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
522 		txbd1->tx_bd_mss = 0;
523 	}
524 
525 	length >>= 9;
526 	if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
527 		dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
528 				     skb->len);
529 		i = 0;
530 		goto tx_dma_error;
531 	}
532 	flags |= bnxt_lhint_arr[length];
533 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
534 
535 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
536 	txbd1->tx_bd_cfa_action =
537 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
538 	for (i = 0; i < last_frag; i++) {
539 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
540 
541 		prod = NEXT_TX(prod);
542 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
543 
544 		len = skb_frag_size(frag);
545 		mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
546 					   DMA_TO_DEVICE);
547 
548 		if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
549 			goto tx_dma_error;
550 
551 		tx_buf = &txr->tx_buf_ring[prod];
552 		dma_unmap_addr_set(tx_buf, mapping, mapping);
553 
554 		txbd->tx_bd_haddr = cpu_to_le64(mapping);
555 
556 		flags = len << TX_BD_LEN_SHIFT;
557 		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
558 	}
559 
560 	flags &= ~TX_BD_LEN;
561 	txbd->tx_bd_len_flags_type =
562 		cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
563 			    TX_BD_FLAGS_PACKET_END);
564 
565 	netdev_tx_sent_queue(txq, skb->len);
566 
567 	/* Sync BD data before updating doorbell */
568 	wmb();
569 
570 	prod = NEXT_TX(prod);
571 	txr->tx_prod = prod;
572 
573 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
574 		bnxt_db_write(bp, &txr->tx_db, prod);
575 
576 tx_done:
577 
578 	if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
579 		if (netdev_xmit_more() && !tx_buf->is_push)
580 			bnxt_db_write(bp, &txr->tx_db, prod);
581 
582 		netif_tx_stop_queue(txq);
583 
584 		/* netif_tx_stop_queue() must be done before checking
585 		 * tx index in bnxt_tx_avail() below, because in
586 		 * bnxt_tx_int(), we update tx index before checking for
587 		 * netif_tx_queue_stopped().
588 		 */
589 		smp_mb();
590 		if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
591 			netif_tx_wake_queue(txq);
592 	}
593 	return NETDEV_TX_OK;
594 
595 tx_dma_error:
596 	last_frag = i;
597 
598 	/* start back at beginning and unmap skb */
599 	prod = txr->tx_prod;
600 	tx_buf = &txr->tx_buf_ring[prod];
601 	tx_buf->skb = NULL;
602 	dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
603 			 skb_headlen(skb), PCI_DMA_TODEVICE);
604 	prod = NEXT_TX(prod);
605 
606 	/* unmap remaining mapped pages */
607 	for (i = 0; i < last_frag; i++) {
608 		prod = NEXT_TX(prod);
609 		tx_buf = &txr->tx_buf_ring[prod];
610 		dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
611 			       skb_frag_size(&skb_shinfo(skb)->frags[i]),
612 			       PCI_DMA_TODEVICE);
613 	}
614 
615 	dev_kfree_skb_any(skb);
616 	return NETDEV_TX_OK;
617 }
618 
619 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
620 {
621 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
622 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
623 	u16 cons = txr->tx_cons;
624 	struct pci_dev *pdev = bp->pdev;
625 	int i;
626 	unsigned int tx_bytes = 0;
627 
628 	for (i = 0; i < nr_pkts; i++) {
629 		struct bnxt_sw_tx_bd *tx_buf;
630 		struct sk_buff *skb;
631 		int j, last;
632 
633 		tx_buf = &txr->tx_buf_ring[cons];
634 		cons = NEXT_TX(cons);
635 		skb = tx_buf->skb;
636 		tx_buf->skb = NULL;
637 
638 		if (tx_buf->is_push) {
639 			tx_buf->is_push = 0;
640 			goto next_tx_int;
641 		}
642 
643 		dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
644 				 skb_headlen(skb), PCI_DMA_TODEVICE);
645 		last = tx_buf->nr_frags;
646 
647 		for (j = 0; j < last; j++) {
648 			cons = NEXT_TX(cons);
649 			tx_buf = &txr->tx_buf_ring[cons];
650 			dma_unmap_page(
651 				&pdev->dev,
652 				dma_unmap_addr(tx_buf, mapping),
653 				skb_frag_size(&skb_shinfo(skb)->frags[j]),
654 				PCI_DMA_TODEVICE);
655 		}
656 
657 next_tx_int:
658 		cons = NEXT_TX(cons);
659 
660 		tx_bytes += skb->len;
661 		dev_kfree_skb_any(skb);
662 	}
663 
664 	netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
665 	txr->tx_cons = cons;
666 
667 	/* Need to make the tx_cons update visible to bnxt_start_xmit()
668 	 * before checking for netif_tx_queue_stopped().  Without the
669 	 * memory barrier, there is a small possibility that bnxt_start_xmit()
670 	 * will miss it and cause the queue to be stopped forever.
671 	 */
672 	smp_mb();
673 
674 	if (unlikely(netif_tx_queue_stopped(txq)) &&
675 	    (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
676 		__netif_tx_lock(txq, smp_processor_id());
677 		if (netif_tx_queue_stopped(txq) &&
678 		    bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
679 		    txr->dev_state != BNXT_DEV_STATE_CLOSING)
680 			netif_tx_wake_queue(txq);
681 		__netif_tx_unlock(txq);
682 	}
683 }
684 
685 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
686 					 struct bnxt_rx_ring_info *rxr,
687 					 gfp_t gfp)
688 {
689 	struct device *dev = &bp->pdev->dev;
690 	struct page *page;
691 
692 	page = page_pool_dev_alloc_pages(rxr->page_pool);
693 	if (!page)
694 		return NULL;
695 
696 	*mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
697 				      DMA_ATTR_WEAK_ORDERING);
698 	if (dma_mapping_error(dev, *mapping)) {
699 		page_pool_recycle_direct(rxr->page_pool, page);
700 		return NULL;
701 	}
702 	*mapping += bp->rx_dma_offset;
703 	return page;
704 }
705 
706 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
707 				       gfp_t gfp)
708 {
709 	u8 *data;
710 	struct pci_dev *pdev = bp->pdev;
711 
712 	data = kmalloc(bp->rx_buf_size, gfp);
713 	if (!data)
714 		return NULL;
715 
716 	*mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
717 					bp->rx_buf_use_size, bp->rx_dir,
718 					DMA_ATTR_WEAK_ORDERING);
719 
720 	if (dma_mapping_error(&pdev->dev, *mapping)) {
721 		kfree(data);
722 		data = NULL;
723 	}
724 	return data;
725 }
726 
727 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
728 		       u16 prod, gfp_t gfp)
729 {
730 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
731 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
732 	dma_addr_t mapping;
733 
734 	if (BNXT_RX_PAGE_MODE(bp)) {
735 		struct page *page =
736 			__bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
737 
738 		if (!page)
739 			return -ENOMEM;
740 
741 		rx_buf->data = page;
742 		rx_buf->data_ptr = page_address(page) + bp->rx_offset;
743 	} else {
744 		u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
745 
746 		if (!data)
747 			return -ENOMEM;
748 
749 		rx_buf->data = data;
750 		rx_buf->data_ptr = data + bp->rx_offset;
751 	}
752 	rx_buf->mapping = mapping;
753 
754 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
755 	return 0;
756 }
757 
758 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
759 {
760 	u16 prod = rxr->rx_prod;
761 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
762 	struct rx_bd *cons_bd, *prod_bd;
763 
764 	prod_rx_buf = &rxr->rx_buf_ring[prod];
765 	cons_rx_buf = &rxr->rx_buf_ring[cons];
766 
767 	prod_rx_buf->data = data;
768 	prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
769 
770 	prod_rx_buf->mapping = cons_rx_buf->mapping;
771 
772 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
773 	cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
774 
775 	prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
776 }
777 
778 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
779 {
780 	u16 next, max = rxr->rx_agg_bmap_size;
781 
782 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
783 	if (next >= max)
784 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
785 	return next;
786 }
787 
788 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
789 				     struct bnxt_rx_ring_info *rxr,
790 				     u16 prod, gfp_t gfp)
791 {
792 	struct rx_bd *rxbd =
793 		&rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
794 	struct bnxt_sw_rx_agg_bd *rx_agg_buf;
795 	struct pci_dev *pdev = bp->pdev;
796 	struct page *page;
797 	dma_addr_t mapping;
798 	u16 sw_prod = rxr->rx_sw_agg_prod;
799 	unsigned int offset = 0;
800 
801 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
802 		page = rxr->rx_page;
803 		if (!page) {
804 			page = alloc_page(gfp);
805 			if (!page)
806 				return -ENOMEM;
807 			rxr->rx_page = page;
808 			rxr->rx_page_offset = 0;
809 		}
810 		offset = rxr->rx_page_offset;
811 		rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
812 		if (rxr->rx_page_offset == PAGE_SIZE)
813 			rxr->rx_page = NULL;
814 		else
815 			get_page(page);
816 	} else {
817 		page = alloc_page(gfp);
818 		if (!page)
819 			return -ENOMEM;
820 	}
821 
822 	mapping = dma_map_page_attrs(&pdev->dev, page, offset,
823 				     BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
824 				     DMA_ATTR_WEAK_ORDERING);
825 	if (dma_mapping_error(&pdev->dev, mapping)) {
826 		__free_page(page);
827 		return -EIO;
828 	}
829 
830 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
831 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
832 
833 	__set_bit(sw_prod, rxr->rx_agg_bmap);
834 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
835 	rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
836 
837 	rx_agg_buf->page = page;
838 	rx_agg_buf->offset = offset;
839 	rx_agg_buf->mapping = mapping;
840 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
841 	rxbd->rx_bd_opaque = sw_prod;
842 	return 0;
843 }
844 
845 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
846 				       struct bnxt_cp_ring_info *cpr,
847 				       u16 cp_cons, u16 curr)
848 {
849 	struct rx_agg_cmp *agg;
850 
851 	cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
852 	agg = (struct rx_agg_cmp *)
853 		&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
854 	return agg;
855 }
856 
857 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
858 					      struct bnxt_rx_ring_info *rxr,
859 					      u16 agg_id, u16 curr)
860 {
861 	struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
862 
863 	return &tpa_info->agg_arr[curr];
864 }
865 
866 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
867 				   u16 start, u32 agg_bufs, bool tpa)
868 {
869 	struct bnxt_napi *bnapi = cpr->bnapi;
870 	struct bnxt *bp = bnapi->bp;
871 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
872 	u16 prod = rxr->rx_agg_prod;
873 	u16 sw_prod = rxr->rx_sw_agg_prod;
874 	bool p5_tpa = false;
875 	u32 i;
876 
877 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
878 		p5_tpa = true;
879 
880 	for (i = 0; i < agg_bufs; i++) {
881 		u16 cons;
882 		struct rx_agg_cmp *agg;
883 		struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
884 		struct rx_bd *prod_bd;
885 		struct page *page;
886 
887 		if (p5_tpa)
888 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
889 		else
890 			agg = bnxt_get_agg(bp, cpr, idx, start + i);
891 		cons = agg->rx_agg_cmp_opaque;
892 		__clear_bit(cons, rxr->rx_agg_bmap);
893 
894 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
895 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
896 
897 		__set_bit(sw_prod, rxr->rx_agg_bmap);
898 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
899 		cons_rx_buf = &rxr->rx_agg_ring[cons];
900 
901 		/* It is possible for sw_prod to be equal to cons, so
902 		 * set cons_rx_buf->page to NULL first.
903 		 */
904 		page = cons_rx_buf->page;
905 		cons_rx_buf->page = NULL;
906 		prod_rx_buf->page = page;
907 		prod_rx_buf->offset = cons_rx_buf->offset;
908 
909 		prod_rx_buf->mapping = cons_rx_buf->mapping;
910 
911 		prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
912 
913 		prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
914 		prod_bd->rx_bd_opaque = sw_prod;
915 
916 		prod = NEXT_RX_AGG(prod);
917 		sw_prod = NEXT_RX_AGG(sw_prod);
918 	}
919 	rxr->rx_agg_prod = prod;
920 	rxr->rx_sw_agg_prod = sw_prod;
921 }
922 
923 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
924 					struct bnxt_rx_ring_info *rxr,
925 					u16 cons, void *data, u8 *data_ptr,
926 					dma_addr_t dma_addr,
927 					unsigned int offset_and_len)
928 {
929 	unsigned int payload = offset_and_len >> 16;
930 	unsigned int len = offset_and_len & 0xffff;
931 	skb_frag_t *frag;
932 	struct page *page = data;
933 	u16 prod = rxr->rx_prod;
934 	struct sk_buff *skb;
935 	int off, err;
936 
937 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
938 	if (unlikely(err)) {
939 		bnxt_reuse_rx_data(rxr, cons, data);
940 		return NULL;
941 	}
942 	dma_addr -= bp->rx_dma_offset;
943 	dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
944 			     DMA_ATTR_WEAK_ORDERING);
945 	page_pool_release_page(rxr->page_pool, page);
946 
947 	if (unlikely(!payload))
948 		payload = eth_get_headlen(bp->dev, data_ptr, len);
949 
950 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
951 	if (!skb) {
952 		__free_page(page);
953 		return NULL;
954 	}
955 
956 	off = (void *)data_ptr - page_address(page);
957 	skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
958 	memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
959 	       payload + NET_IP_ALIGN);
960 
961 	frag = &skb_shinfo(skb)->frags[0];
962 	skb_frag_size_sub(frag, payload);
963 	skb_frag_off_add(frag, payload);
964 	skb->data_len -= payload;
965 	skb->tail += payload;
966 
967 	return skb;
968 }
969 
970 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
971 				   struct bnxt_rx_ring_info *rxr, u16 cons,
972 				   void *data, u8 *data_ptr,
973 				   dma_addr_t dma_addr,
974 				   unsigned int offset_and_len)
975 {
976 	u16 prod = rxr->rx_prod;
977 	struct sk_buff *skb;
978 	int err;
979 
980 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
981 	if (unlikely(err)) {
982 		bnxt_reuse_rx_data(rxr, cons, data);
983 		return NULL;
984 	}
985 
986 	skb = build_skb(data, 0);
987 	dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
988 			       bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
989 	if (!skb) {
990 		kfree(data);
991 		return NULL;
992 	}
993 
994 	skb_reserve(skb, bp->rx_offset);
995 	skb_put(skb, offset_and_len & 0xffff);
996 	return skb;
997 }
998 
999 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
1000 				     struct bnxt_cp_ring_info *cpr,
1001 				     struct sk_buff *skb, u16 idx,
1002 				     u32 agg_bufs, bool tpa)
1003 {
1004 	struct bnxt_napi *bnapi = cpr->bnapi;
1005 	struct pci_dev *pdev = bp->pdev;
1006 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1007 	u16 prod = rxr->rx_agg_prod;
1008 	bool p5_tpa = false;
1009 	u32 i;
1010 
1011 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1012 		p5_tpa = true;
1013 
1014 	for (i = 0; i < agg_bufs; i++) {
1015 		u16 cons, frag_len;
1016 		struct rx_agg_cmp *agg;
1017 		struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1018 		struct page *page;
1019 		dma_addr_t mapping;
1020 
1021 		if (p5_tpa)
1022 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1023 		else
1024 			agg = bnxt_get_agg(bp, cpr, idx, i);
1025 		cons = agg->rx_agg_cmp_opaque;
1026 		frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1027 			    RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1028 
1029 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1030 		skb_fill_page_desc(skb, i, cons_rx_buf->page,
1031 				   cons_rx_buf->offset, frag_len);
1032 		__clear_bit(cons, rxr->rx_agg_bmap);
1033 
1034 		/* It is possible for bnxt_alloc_rx_page() to allocate
1035 		 * a sw_prod index that equals the cons index, so we
1036 		 * need to clear the cons entry now.
1037 		 */
1038 		mapping = cons_rx_buf->mapping;
1039 		page = cons_rx_buf->page;
1040 		cons_rx_buf->page = NULL;
1041 
1042 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1043 			struct skb_shared_info *shinfo;
1044 			unsigned int nr_frags;
1045 
1046 			shinfo = skb_shinfo(skb);
1047 			nr_frags = --shinfo->nr_frags;
1048 			__skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1049 
1050 			dev_kfree_skb(skb);
1051 
1052 			cons_rx_buf->page = page;
1053 
1054 			/* Update prod since possibly some pages have been
1055 			 * allocated already.
1056 			 */
1057 			rxr->rx_agg_prod = prod;
1058 			bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1059 			return NULL;
1060 		}
1061 
1062 		dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1063 				     PCI_DMA_FROMDEVICE,
1064 				     DMA_ATTR_WEAK_ORDERING);
1065 
1066 		skb->data_len += frag_len;
1067 		skb->len += frag_len;
1068 		skb->truesize += PAGE_SIZE;
1069 
1070 		prod = NEXT_RX_AGG(prod);
1071 	}
1072 	rxr->rx_agg_prod = prod;
1073 	return skb;
1074 }
1075 
1076 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1077 			       u8 agg_bufs, u32 *raw_cons)
1078 {
1079 	u16 last;
1080 	struct rx_agg_cmp *agg;
1081 
1082 	*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1083 	last = RING_CMP(*raw_cons);
1084 	agg = (struct rx_agg_cmp *)
1085 		&cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1086 	return RX_AGG_CMP_VALID(agg, *raw_cons);
1087 }
1088 
1089 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1090 					    unsigned int len,
1091 					    dma_addr_t mapping)
1092 {
1093 	struct bnxt *bp = bnapi->bp;
1094 	struct pci_dev *pdev = bp->pdev;
1095 	struct sk_buff *skb;
1096 
1097 	skb = napi_alloc_skb(&bnapi->napi, len);
1098 	if (!skb)
1099 		return NULL;
1100 
1101 	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1102 				bp->rx_dir);
1103 
1104 	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1105 	       len + NET_IP_ALIGN);
1106 
1107 	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1108 				   bp->rx_dir);
1109 
1110 	skb_put(skb, len);
1111 	return skb;
1112 }
1113 
1114 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1115 			   u32 *raw_cons, void *cmp)
1116 {
1117 	struct rx_cmp *rxcmp = cmp;
1118 	u32 tmp_raw_cons = *raw_cons;
1119 	u8 cmp_type, agg_bufs = 0;
1120 
1121 	cmp_type = RX_CMP_TYPE(rxcmp);
1122 
1123 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1124 		agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1125 			    RX_CMP_AGG_BUFS) >>
1126 			   RX_CMP_AGG_BUFS_SHIFT;
1127 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1128 		struct rx_tpa_end_cmp *tpa_end = cmp;
1129 
1130 		if (bp->flags & BNXT_FLAG_CHIP_P5)
1131 			return 0;
1132 
1133 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1134 	}
1135 
1136 	if (agg_bufs) {
1137 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1138 			return -EBUSY;
1139 	}
1140 	*raw_cons = tmp_raw_cons;
1141 	return 0;
1142 }
1143 
1144 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1145 {
1146 	if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1147 		return;
1148 
1149 	if (BNXT_PF(bp))
1150 		queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1151 	else
1152 		schedule_delayed_work(&bp->fw_reset_task, delay);
1153 }
1154 
1155 static void bnxt_queue_sp_work(struct bnxt *bp)
1156 {
1157 	if (BNXT_PF(bp))
1158 		queue_work(bnxt_pf_wq, &bp->sp_task);
1159 	else
1160 		schedule_work(&bp->sp_task);
1161 }
1162 
1163 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1164 {
1165 	if (!rxr->bnapi->in_reset) {
1166 		rxr->bnapi->in_reset = true;
1167 		if (bp->flags & BNXT_FLAG_CHIP_P5)
1168 			set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1169 		else
1170 			set_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event);
1171 		bnxt_queue_sp_work(bp);
1172 	}
1173 	rxr->rx_next_cons = 0xffff;
1174 }
1175 
1176 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1177 {
1178 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1179 	u16 idx = agg_id & MAX_TPA_P5_MASK;
1180 
1181 	if (test_bit(idx, map->agg_idx_bmap))
1182 		idx = find_first_zero_bit(map->agg_idx_bmap,
1183 					  BNXT_AGG_IDX_BMAP_SIZE);
1184 	__set_bit(idx, map->agg_idx_bmap);
1185 	map->agg_id_tbl[agg_id] = idx;
1186 	return idx;
1187 }
1188 
1189 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1190 {
1191 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1192 
1193 	__clear_bit(idx, map->agg_idx_bmap);
1194 }
1195 
1196 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1197 {
1198 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1199 
1200 	return map->agg_id_tbl[agg_id];
1201 }
1202 
1203 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1204 			   struct rx_tpa_start_cmp *tpa_start,
1205 			   struct rx_tpa_start_cmp_ext *tpa_start1)
1206 {
1207 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1208 	struct bnxt_tpa_info *tpa_info;
1209 	u16 cons, prod, agg_id;
1210 	struct rx_bd *prod_bd;
1211 	dma_addr_t mapping;
1212 
1213 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1214 		agg_id = TPA_START_AGG_ID_P5(tpa_start);
1215 		agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1216 	} else {
1217 		agg_id = TPA_START_AGG_ID(tpa_start);
1218 	}
1219 	cons = tpa_start->rx_tpa_start_cmp_opaque;
1220 	prod = rxr->rx_prod;
1221 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1222 	prod_rx_buf = &rxr->rx_buf_ring[prod];
1223 	tpa_info = &rxr->rx_tpa[agg_id];
1224 
1225 	if (unlikely(cons != rxr->rx_next_cons ||
1226 		     TPA_START_ERROR(tpa_start))) {
1227 		netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1228 			    cons, rxr->rx_next_cons,
1229 			    TPA_START_ERROR_CODE(tpa_start1));
1230 		bnxt_sched_reset(bp, rxr);
1231 		return;
1232 	}
1233 	/* Store cfa_code in tpa_info to use in tpa_end
1234 	 * completion processing.
1235 	 */
1236 	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1237 	prod_rx_buf->data = tpa_info->data;
1238 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
1239 
1240 	mapping = tpa_info->mapping;
1241 	prod_rx_buf->mapping = mapping;
1242 
1243 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1244 
1245 	prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1246 
1247 	tpa_info->data = cons_rx_buf->data;
1248 	tpa_info->data_ptr = cons_rx_buf->data_ptr;
1249 	cons_rx_buf->data = NULL;
1250 	tpa_info->mapping = cons_rx_buf->mapping;
1251 
1252 	tpa_info->len =
1253 		le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1254 				RX_TPA_START_CMP_LEN_SHIFT;
1255 	if (likely(TPA_START_HASH_VALID(tpa_start))) {
1256 		u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1257 
1258 		tpa_info->hash_type = PKT_HASH_TYPE_L4;
1259 		tpa_info->gso_type = SKB_GSO_TCPV4;
1260 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1261 		if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1262 			tpa_info->gso_type = SKB_GSO_TCPV6;
1263 		tpa_info->rss_hash =
1264 			le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1265 	} else {
1266 		tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1267 		tpa_info->gso_type = 0;
1268 		if (netif_msg_rx_err(bp))
1269 			netdev_warn(bp->dev, "TPA packet without valid hash\n");
1270 	}
1271 	tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1272 	tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1273 	tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1274 	tpa_info->agg_count = 0;
1275 
1276 	rxr->rx_prod = NEXT_RX(prod);
1277 	cons = NEXT_RX(cons);
1278 	rxr->rx_next_cons = NEXT_RX(cons);
1279 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1280 
1281 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1282 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1283 	cons_rx_buf->data = NULL;
1284 }
1285 
1286 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1287 {
1288 	if (agg_bufs)
1289 		bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1290 }
1291 
1292 #ifdef CONFIG_INET
1293 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1294 {
1295 	struct udphdr *uh = NULL;
1296 
1297 	if (ip_proto == htons(ETH_P_IP)) {
1298 		struct iphdr *iph = (struct iphdr *)skb->data;
1299 
1300 		if (iph->protocol == IPPROTO_UDP)
1301 			uh = (struct udphdr *)(iph + 1);
1302 	} else {
1303 		struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1304 
1305 		if (iph->nexthdr == IPPROTO_UDP)
1306 			uh = (struct udphdr *)(iph + 1);
1307 	}
1308 	if (uh) {
1309 		if (uh->check)
1310 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1311 		else
1312 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1313 	}
1314 }
1315 #endif
1316 
1317 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1318 					   int payload_off, int tcp_ts,
1319 					   struct sk_buff *skb)
1320 {
1321 #ifdef CONFIG_INET
1322 	struct tcphdr *th;
1323 	int len, nw_off;
1324 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1325 	u32 hdr_info = tpa_info->hdr_info;
1326 	bool loopback = false;
1327 
1328 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1329 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1330 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1331 
1332 	/* If the packet is an internal loopback packet, the offsets will
1333 	 * have an extra 4 bytes.
1334 	 */
1335 	if (inner_mac_off == 4) {
1336 		loopback = true;
1337 	} else if (inner_mac_off > 4) {
1338 		__be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1339 					    ETH_HLEN - 2));
1340 
1341 		/* We only support inner iPv4/ipv6.  If we don't see the
1342 		 * correct protocol ID, it must be a loopback packet where
1343 		 * the offsets are off by 4.
1344 		 */
1345 		if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1346 			loopback = true;
1347 	}
1348 	if (loopback) {
1349 		/* internal loopback packet, subtract all offsets by 4 */
1350 		inner_ip_off -= 4;
1351 		inner_mac_off -= 4;
1352 		outer_ip_off -= 4;
1353 	}
1354 
1355 	nw_off = inner_ip_off - ETH_HLEN;
1356 	skb_set_network_header(skb, nw_off);
1357 	if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1358 		struct ipv6hdr *iph = ipv6_hdr(skb);
1359 
1360 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1361 		len = skb->len - skb_transport_offset(skb);
1362 		th = tcp_hdr(skb);
1363 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1364 	} else {
1365 		struct iphdr *iph = ip_hdr(skb);
1366 
1367 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1368 		len = skb->len - skb_transport_offset(skb);
1369 		th = tcp_hdr(skb);
1370 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1371 	}
1372 
1373 	if (inner_mac_off) { /* tunnel */
1374 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1375 					    ETH_HLEN - 2));
1376 
1377 		bnxt_gro_tunnel(skb, proto);
1378 	}
1379 #endif
1380 	return skb;
1381 }
1382 
1383 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1384 					   int payload_off, int tcp_ts,
1385 					   struct sk_buff *skb)
1386 {
1387 #ifdef CONFIG_INET
1388 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1389 	u32 hdr_info = tpa_info->hdr_info;
1390 	int iphdr_len, nw_off;
1391 
1392 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1393 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1394 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1395 
1396 	nw_off = inner_ip_off - ETH_HLEN;
1397 	skb_set_network_header(skb, nw_off);
1398 	iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1399 		     sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1400 	skb_set_transport_header(skb, nw_off + iphdr_len);
1401 
1402 	if (inner_mac_off) { /* tunnel */
1403 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1404 					    ETH_HLEN - 2));
1405 
1406 		bnxt_gro_tunnel(skb, proto);
1407 	}
1408 #endif
1409 	return skb;
1410 }
1411 
1412 #define BNXT_IPV4_HDR_SIZE	(sizeof(struct iphdr) + sizeof(struct tcphdr))
1413 #define BNXT_IPV6_HDR_SIZE	(sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1414 
1415 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1416 					   int payload_off, int tcp_ts,
1417 					   struct sk_buff *skb)
1418 {
1419 #ifdef CONFIG_INET
1420 	struct tcphdr *th;
1421 	int len, nw_off, tcp_opt_len = 0;
1422 
1423 	if (tcp_ts)
1424 		tcp_opt_len = 12;
1425 
1426 	if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1427 		struct iphdr *iph;
1428 
1429 		nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1430 			 ETH_HLEN;
1431 		skb_set_network_header(skb, nw_off);
1432 		iph = ip_hdr(skb);
1433 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1434 		len = skb->len - skb_transport_offset(skb);
1435 		th = tcp_hdr(skb);
1436 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1437 	} else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1438 		struct ipv6hdr *iph;
1439 
1440 		nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1441 			 ETH_HLEN;
1442 		skb_set_network_header(skb, nw_off);
1443 		iph = ipv6_hdr(skb);
1444 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1445 		len = skb->len - skb_transport_offset(skb);
1446 		th = tcp_hdr(skb);
1447 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1448 	} else {
1449 		dev_kfree_skb_any(skb);
1450 		return NULL;
1451 	}
1452 
1453 	if (nw_off) /* tunnel */
1454 		bnxt_gro_tunnel(skb, skb->protocol);
1455 #endif
1456 	return skb;
1457 }
1458 
1459 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1460 					   struct bnxt_tpa_info *tpa_info,
1461 					   struct rx_tpa_end_cmp *tpa_end,
1462 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1463 					   struct sk_buff *skb)
1464 {
1465 #ifdef CONFIG_INET
1466 	int payload_off;
1467 	u16 segs;
1468 
1469 	segs = TPA_END_TPA_SEGS(tpa_end);
1470 	if (segs == 1)
1471 		return skb;
1472 
1473 	NAPI_GRO_CB(skb)->count = segs;
1474 	skb_shinfo(skb)->gso_size =
1475 		le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1476 	skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1477 	if (bp->flags & BNXT_FLAG_CHIP_P5)
1478 		payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1479 	else
1480 		payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1481 	skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1482 	if (likely(skb))
1483 		tcp_gro_complete(skb);
1484 #endif
1485 	return skb;
1486 }
1487 
1488 /* Given the cfa_code of a received packet determine which
1489  * netdev (vf-rep or PF) the packet is destined to.
1490  */
1491 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1492 {
1493 	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1494 
1495 	/* if vf-rep dev is NULL, the must belongs to the PF */
1496 	return dev ? dev : bp->dev;
1497 }
1498 
1499 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1500 					   struct bnxt_cp_ring_info *cpr,
1501 					   u32 *raw_cons,
1502 					   struct rx_tpa_end_cmp *tpa_end,
1503 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1504 					   u8 *event)
1505 {
1506 	struct bnxt_napi *bnapi = cpr->bnapi;
1507 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1508 	u8 *data_ptr, agg_bufs;
1509 	unsigned int len;
1510 	struct bnxt_tpa_info *tpa_info;
1511 	dma_addr_t mapping;
1512 	struct sk_buff *skb;
1513 	u16 idx = 0, agg_id;
1514 	void *data;
1515 	bool gro;
1516 
1517 	if (unlikely(bnapi->in_reset)) {
1518 		int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1519 
1520 		if (rc < 0)
1521 			return ERR_PTR(-EBUSY);
1522 		return NULL;
1523 	}
1524 
1525 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1526 		agg_id = TPA_END_AGG_ID_P5(tpa_end);
1527 		agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1528 		agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1529 		tpa_info = &rxr->rx_tpa[agg_id];
1530 		if (unlikely(agg_bufs != tpa_info->agg_count)) {
1531 			netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1532 				    agg_bufs, tpa_info->agg_count);
1533 			agg_bufs = tpa_info->agg_count;
1534 		}
1535 		tpa_info->agg_count = 0;
1536 		*event |= BNXT_AGG_EVENT;
1537 		bnxt_free_agg_idx(rxr, agg_id);
1538 		idx = agg_id;
1539 		gro = !!(bp->flags & BNXT_FLAG_GRO);
1540 	} else {
1541 		agg_id = TPA_END_AGG_ID(tpa_end);
1542 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1543 		tpa_info = &rxr->rx_tpa[agg_id];
1544 		idx = RING_CMP(*raw_cons);
1545 		if (agg_bufs) {
1546 			if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1547 				return ERR_PTR(-EBUSY);
1548 
1549 			*event |= BNXT_AGG_EVENT;
1550 			idx = NEXT_CMP(idx);
1551 		}
1552 		gro = !!TPA_END_GRO(tpa_end);
1553 	}
1554 	data = tpa_info->data;
1555 	data_ptr = tpa_info->data_ptr;
1556 	prefetch(data_ptr);
1557 	len = tpa_info->len;
1558 	mapping = tpa_info->mapping;
1559 
1560 	if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1561 		bnxt_abort_tpa(cpr, idx, agg_bufs);
1562 		if (agg_bufs > MAX_SKB_FRAGS)
1563 			netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1564 				    agg_bufs, (int)MAX_SKB_FRAGS);
1565 		return NULL;
1566 	}
1567 
1568 	if (len <= bp->rx_copy_thresh) {
1569 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1570 		if (!skb) {
1571 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1572 			return NULL;
1573 		}
1574 	} else {
1575 		u8 *new_data;
1576 		dma_addr_t new_mapping;
1577 
1578 		new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1579 		if (!new_data) {
1580 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1581 			return NULL;
1582 		}
1583 
1584 		tpa_info->data = new_data;
1585 		tpa_info->data_ptr = new_data + bp->rx_offset;
1586 		tpa_info->mapping = new_mapping;
1587 
1588 		skb = build_skb(data, 0);
1589 		dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1590 				       bp->rx_buf_use_size, bp->rx_dir,
1591 				       DMA_ATTR_WEAK_ORDERING);
1592 
1593 		if (!skb) {
1594 			kfree(data);
1595 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1596 			return NULL;
1597 		}
1598 		skb_reserve(skb, bp->rx_offset);
1599 		skb_put(skb, len);
1600 	}
1601 
1602 	if (agg_bufs) {
1603 		skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1604 		if (!skb) {
1605 			/* Page reuse already handled by bnxt_rx_pages(). */
1606 			return NULL;
1607 		}
1608 	}
1609 
1610 	skb->protocol =
1611 		eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1612 
1613 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1614 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1615 
1616 	if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1617 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1618 		u16 vlan_proto = tpa_info->metadata >>
1619 			RX_CMP_FLAGS2_METADATA_TPID_SFT;
1620 		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1621 
1622 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1623 	}
1624 
1625 	skb_checksum_none_assert(skb);
1626 	if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1627 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1628 		skb->csum_level =
1629 			(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1630 	}
1631 
1632 	if (gro)
1633 		skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1634 
1635 	return skb;
1636 }
1637 
1638 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1639 			 struct rx_agg_cmp *rx_agg)
1640 {
1641 	u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1642 	struct bnxt_tpa_info *tpa_info;
1643 
1644 	agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1645 	tpa_info = &rxr->rx_tpa[agg_id];
1646 	BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1647 	tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1648 }
1649 
1650 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1651 			     struct sk_buff *skb)
1652 {
1653 	if (skb->dev != bp->dev) {
1654 		/* this packet belongs to a vf-rep */
1655 		bnxt_vf_rep_rx(bp, skb);
1656 		return;
1657 	}
1658 	skb_record_rx_queue(skb, bnapi->index);
1659 	napi_gro_receive(&bnapi->napi, skb);
1660 }
1661 
1662 /* returns the following:
1663  * 1       - 1 packet successfully received
1664  * 0       - successful TPA_START, packet not completed yet
1665  * -EBUSY  - completion ring does not have all the agg buffers yet
1666  * -ENOMEM - packet aborted due to out of memory
1667  * -EIO    - packet aborted due to hw error indicated in BD
1668  */
1669 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1670 		       u32 *raw_cons, u8 *event)
1671 {
1672 	struct bnxt_napi *bnapi = cpr->bnapi;
1673 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1674 	struct net_device *dev = bp->dev;
1675 	struct rx_cmp *rxcmp;
1676 	struct rx_cmp_ext *rxcmp1;
1677 	u32 tmp_raw_cons = *raw_cons;
1678 	u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1679 	struct bnxt_sw_rx_bd *rx_buf;
1680 	unsigned int len;
1681 	u8 *data_ptr, agg_bufs, cmp_type;
1682 	dma_addr_t dma_addr;
1683 	struct sk_buff *skb;
1684 	void *data;
1685 	int rc = 0;
1686 	u32 misc;
1687 
1688 	rxcmp = (struct rx_cmp *)
1689 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1690 
1691 	cmp_type = RX_CMP_TYPE(rxcmp);
1692 
1693 	if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1694 		bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1695 		goto next_rx_no_prod_no_len;
1696 	}
1697 
1698 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1699 	cp_cons = RING_CMP(tmp_raw_cons);
1700 	rxcmp1 = (struct rx_cmp_ext *)
1701 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1702 
1703 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1704 		return -EBUSY;
1705 
1706 	prod = rxr->rx_prod;
1707 
1708 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1709 		bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1710 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
1711 
1712 		*event |= BNXT_RX_EVENT;
1713 		goto next_rx_no_prod_no_len;
1714 
1715 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1716 		skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1717 				   (struct rx_tpa_end_cmp *)rxcmp,
1718 				   (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1719 
1720 		if (IS_ERR(skb))
1721 			return -EBUSY;
1722 
1723 		rc = -ENOMEM;
1724 		if (likely(skb)) {
1725 			bnxt_deliver_skb(bp, bnapi, skb);
1726 			rc = 1;
1727 		}
1728 		*event |= BNXT_RX_EVENT;
1729 		goto next_rx_no_prod_no_len;
1730 	}
1731 
1732 	cons = rxcmp->rx_cmp_opaque;
1733 	if (unlikely(cons != rxr->rx_next_cons)) {
1734 		int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1735 
1736 		/* 0xffff is forced error, don't print it */
1737 		if (rxr->rx_next_cons != 0xffff)
1738 			netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1739 				    cons, rxr->rx_next_cons);
1740 		bnxt_sched_reset(bp, rxr);
1741 		return rc1;
1742 	}
1743 	rx_buf = &rxr->rx_buf_ring[cons];
1744 	data = rx_buf->data;
1745 	data_ptr = rx_buf->data_ptr;
1746 	prefetch(data_ptr);
1747 
1748 	misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1749 	agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1750 
1751 	if (agg_bufs) {
1752 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1753 			return -EBUSY;
1754 
1755 		cp_cons = NEXT_CMP(cp_cons);
1756 		*event |= BNXT_AGG_EVENT;
1757 	}
1758 	*event |= BNXT_RX_EVENT;
1759 
1760 	rx_buf->data = NULL;
1761 	if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1762 		u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1763 
1764 		bnxt_reuse_rx_data(rxr, cons, data);
1765 		if (agg_bufs)
1766 			bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1767 					       false);
1768 
1769 		rc = -EIO;
1770 		if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1771 			bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1772 			if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
1773 			    !(bp->fw_cap & BNXT_FW_CAP_RING_MONITOR)) {
1774 				netdev_warn_once(bp->dev, "RX buffer error %x\n",
1775 						 rx_err);
1776 				bnxt_sched_reset(bp, rxr);
1777 			}
1778 		}
1779 		goto next_rx_no_len;
1780 	}
1781 
1782 	len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1783 	dma_addr = rx_buf->mapping;
1784 
1785 	if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1786 		rc = 1;
1787 		goto next_rx;
1788 	}
1789 
1790 	if (len <= bp->rx_copy_thresh) {
1791 		skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1792 		bnxt_reuse_rx_data(rxr, cons, data);
1793 		if (!skb) {
1794 			if (agg_bufs)
1795 				bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1796 						       agg_bufs, false);
1797 			rc = -ENOMEM;
1798 			goto next_rx;
1799 		}
1800 	} else {
1801 		u32 payload;
1802 
1803 		if (rx_buf->data_ptr == data_ptr)
1804 			payload = misc & RX_CMP_PAYLOAD_OFFSET;
1805 		else
1806 			payload = 0;
1807 		skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1808 				      payload | len);
1809 		if (!skb) {
1810 			rc = -ENOMEM;
1811 			goto next_rx;
1812 		}
1813 	}
1814 
1815 	if (agg_bufs) {
1816 		skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1817 		if (!skb) {
1818 			rc = -ENOMEM;
1819 			goto next_rx;
1820 		}
1821 	}
1822 
1823 	if (RX_CMP_HASH_VALID(rxcmp)) {
1824 		u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1825 		enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1826 
1827 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1828 		if (hash_type != 1 && hash_type != 3)
1829 			type = PKT_HASH_TYPE_L3;
1830 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1831 	}
1832 
1833 	cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1834 	skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1835 
1836 	if ((rxcmp1->rx_cmp_flags2 &
1837 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1838 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1839 		u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1840 		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1841 		u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1842 
1843 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1844 	}
1845 
1846 	skb_checksum_none_assert(skb);
1847 	if (RX_CMP_L4_CS_OK(rxcmp1)) {
1848 		if (dev->features & NETIF_F_RXCSUM) {
1849 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1850 			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1851 		}
1852 	} else {
1853 		if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1854 			if (dev->features & NETIF_F_RXCSUM)
1855 				bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1856 		}
1857 	}
1858 
1859 	bnxt_deliver_skb(bp, bnapi, skb);
1860 	rc = 1;
1861 
1862 next_rx:
1863 	cpr->rx_packets += 1;
1864 	cpr->rx_bytes += len;
1865 
1866 next_rx_no_len:
1867 	rxr->rx_prod = NEXT_RX(prod);
1868 	rxr->rx_next_cons = NEXT_RX(cons);
1869 
1870 next_rx_no_prod_no_len:
1871 	*raw_cons = tmp_raw_cons;
1872 
1873 	return rc;
1874 }
1875 
1876 /* In netpoll mode, if we are using a combined completion ring, we need to
1877  * discard the rx packets and recycle the buffers.
1878  */
1879 static int bnxt_force_rx_discard(struct bnxt *bp,
1880 				 struct bnxt_cp_ring_info *cpr,
1881 				 u32 *raw_cons, u8 *event)
1882 {
1883 	u32 tmp_raw_cons = *raw_cons;
1884 	struct rx_cmp_ext *rxcmp1;
1885 	struct rx_cmp *rxcmp;
1886 	u16 cp_cons;
1887 	u8 cmp_type;
1888 
1889 	cp_cons = RING_CMP(tmp_raw_cons);
1890 	rxcmp = (struct rx_cmp *)
1891 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1892 
1893 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1894 	cp_cons = RING_CMP(tmp_raw_cons);
1895 	rxcmp1 = (struct rx_cmp_ext *)
1896 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1897 
1898 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1899 		return -EBUSY;
1900 
1901 	cmp_type = RX_CMP_TYPE(rxcmp);
1902 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1903 		rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1904 			cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1905 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1906 		struct rx_tpa_end_cmp_ext *tpa_end1;
1907 
1908 		tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1909 		tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1910 			cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1911 	}
1912 	return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1913 }
1914 
1915 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1916 {
1917 	struct bnxt_fw_health *fw_health = bp->fw_health;
1918 	u32 reg = fw_health->regs[reg_idx];
1919 	u32 reg_type, reg_off, val = 0;
1920 
1921 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1922 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1923 	switch (reg_type) {
1924 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
1925 		pci_read_config_dword(bp->pdev, reg_off, &val);
1926 		break;
1927 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
1928 		reg_off = fw_health->mapped_regs[reg_idx];
1929 		fallthrough;
1930 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1931 		val = readl(bp->bar0 + reg_off);
1932 		break;
1933 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1934 		val = readl(bp->bar1 + reg_off);
1935 		break;
1936 	}
1937 	if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1938 		val &= fw_health->fw_reset_inprog_reg_mask;
1939 	return val;
1940 }
1941 
1942 static u16 bnxt_agg_ring_id_to_grp_idx(struct bnxt *bp, u16 ring_id)
1943 {
1944 	int i;
1945 
1946 	for (i = 0; i < bp->rx_nr_rings; i++) {
1947 		u16 grp_idx = bp->rx_ring[i].bnapi->index;
1948 		struct bnxt_ring_grp_info *grp_info;
1949 
1950 		grp_info = &bp->grp_info[grp_idx];
1951 		if (grp_info->agg_fw_ring_id == ring_id)
1952 			return grp_idx;
1953 	}
1954 	return INVALID_HW_RING_ID;
1955 }
1956 
1957 #define BNXT_GET_EVENT_PORT(data)	\
1958 	((data) &			\
1959 	 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1960 
1961 #define BNXT_EVENT_RING_TYPE(data2)	\
1962 	((data2) &			\
1963 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_MASK)
1964 
1965 #define BNXT_EVENT_RING_TYPE_RX(data2)	\
1966 	(BNXT_EVENT_RING_TYPE(data2) ==	\
1967 	 ASYNC_EVENT_CMPL_RING_MONITOR_MSG_EVENT_DATA2_DISABLE_RING_TYPE_RX)
1968 
1969 static int bnxt_async_event_process(struct bnxt *bp,
1970 				    struct hwrm_async_event_cmpl *cmpl)
1971 {
1972 	u16 event_id = le16_to_cpu(cmpl->event_id);
1973 	u32 data1 = le32_to_cpu(cmpl->event_data1);
1974 	u32 data2 = le32_to_cpu(cmpl->event_data2);
1975 
1976 	/* TODO CHIMP_FW: Define event id's for link change, error etc */
1977 	switch (event_id) {
1978 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1979 		struct bnxt_link_info *link_info = &bp->link_info;
1980 
1981 		if (BNXT_VF(bp))
1982 			goto async_event_process_exit;
1983 
1984 		/* print unsupported speed warning in forced speed mode only */
1985 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1986 		    (data1 & 0x20000)) {
1987 			u16 fw_speed = link_info->force_link_speed;
1988 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1989 
1990 			if (speed != SPEED_UNKNOWN)
1991 				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1992 					    speed);
1993 		}
1994 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1995 	}
1996 		fallthrough;
1997 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1998 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1999 		set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
2000 		fallthrough;
2001 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
2002 		set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
2003 		break;
2004 	case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
2005 		set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
2006 		break;
2007 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
2008 		u16 port_id = BNXT_GET_EVENT_PORT(data1);
2009 
2010 		if (BNXT_VF(bp))
2011 			break;
2012 
2013 		if (bp->pf.port_id != port_id)
2014 			break;
2015 
2016 		set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
2017 		break;
2018 	}
2019 	case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
2020 		if (BNXT_PF(bp))
2021 			goto async_event_process_exit;
2022 		set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2023 		break;
2024 	case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
2025 		if (netif_msg_hw(bp))
2026 			netdev_warn(bp->dev, "Received RESET_NOTIFY event, data1: 0x%x, data2: 0x%x\n",
2027 				    data1, data2);
2028 		if (!bp->fw_health)
2029 			goto async_event_process_exit;
2030 
2031 		bp->fw_reset_timestamp = jiffies;
2032 		bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2033 		if (!bp->fw_reset_min_dsecs)
2034 			bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2035 		bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2036 		if (!bp->fw_reset_max_dsecs)
2037 			bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2038 		if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2039 			netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2040 			set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2041 		} else {
2042 			netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2043 				    bp->fw_reset_max_dsecs * 100);
2044 		}
2045 		set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2046 		break;
2047 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2048 		struct bnxt_fw_health *fw_health = bp->fw_health;
2049 
2050 		if (!fw_health)
2051 			goto async_event_process_exit;
2052 
2053 		fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2054 		fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2055 		if (!fw_health->enabled)
2056 			break;
2057 
2058 		if (netif_msg_drv(bp))
2059 			netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2060 				    fw_health->enabled, fw_health->master,
2061 				    bnxt_fw_health_readl(bp,
2062 							 BNXT_FW_RESET_CNT_REG),
2063 				    bnxt_fw_health_readl(bp,
2064 							 BNXT_FW_HEALTH_REG));
2065 		fw_health->tmr_multiplier =
2066 			DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2067 				     bp->current_interval * 10);
2068 		fw_health->tmr_counter = fw_health->tmr_multiplier;
2069 		fw_health->last_fw_heartbeat =
2070 			bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2071 		fw_health->last_fw_reset_cnt =
2072 			bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2073 		goto async_event_process_exit;
2074 	}
2075 	case ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG: {
2076 		struct bnxt_rx_ring_info *rxr;
2077 		u16 grp_idx;
2078 
2079 		if (bp->flags & BNXT_FLAG_CHIP_P5)
2080 			goto async_event_process_exit;
2081 
2082 		netdev_warn(bp->dev, "Ring monitor event, ring type %lu id 0x%x\n",
2083 			    BNXT_EVENT_RING_TYPE(data2), data1);
2084 		if (!BNXT_EVENT_RING_TYPE_RX(data2))
2085 			goto async_event_process_exit;
2086 
2087 		grp_idx = bnxt_agg_ring_id_to_grp_idx(bp, data1);
2088 		if (grp_idx == INVALID_HW_RING_ID) {
2089 			netdev_warn(bp->dev, "Unknown RX agg ring id 0x%x\n",
2090 				    data1);
2091 			goto async_event_process_exit;
2092 		}
2093 		rxr = bp->bnapi[grp_idx]->rx_ring;
2094 		bnxt_sched_reset(bp, rxr);
2095 		goto async_event_process_exit;
2096 	}
2097 	default:
2098 		goto async_event_process_exit;
2099 	}
2100 	bnxt_queue_sp_work(bp);
2101 async_event_process_exit:
2102 	bnxt_ulp_async_events(bp, cmpl);
2103 	return 0;
2104 }
2105 
2106 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2107 {
2108 	u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2109 	struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2110 	struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2111 				(struct hwrm_fwd_req_cmpl *)txcmp;
2112 
2113 	switch (cmpl_type) {
2114 	case CMPL_BASE_TYPE_HWRM_DONE:
2115 		seq_id = le16_to_cpu(h_cmpl->sequence_id);
2116 		if (seq_id == bp->hwrm_intr_seq_id)
2117 			bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2118 		else
2119 			netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2120 		break;
2121 
2122 	case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2123 		vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2124 
2125 		if ((vf_id < bp->pf.first_vf_id) ||
2126 		    (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2127 			netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2128 				   vf_id);
2129 			return -EINVAL;
2130 		}
2131 
2132 		set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2133 		set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2134 		bnxt_queue_sp_work(bp);
2135 		break;
2136 
2137 	case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2138 		bnxt_async_event_process(bp,
2139 					 (struct hwrm_async_event_cmpl *)txcmp);
2140 
2141 	default:
2142 		break;
2143 	}
2144 
2145 	return 0;
2146 }
2147 
2148 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2149 {
2150 	struct bnxt_napi *bnapi = dev_instance;
2151 	struct bnxt *bp = bnapi->bp;
2152 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2153 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2154 
2155 	cpr->event_ctr++;
2156 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2157 	napi_schedule(&bnapi->napi);
2158 	return IRQ_HANDLED;
2159 }
2160 
2161 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2162 {
2163 	u32 raw_cons = cpr->cp_raw_cons;
2164 	u16 cons = RING_CMP(raw_cons);
2165 	struct tx_cmp *txcmp;
2166 
2167 	txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2168 
2169 	return TX_CMP_VALID(txcmp, raw_cons);
2170 }
2171 
2172 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2173 {
2174 	struct bnxt_napi *bnapi = dev_instance;
2175 	struct bnxt *bp = bnapi->bp;
2176 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2177 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2178 	u32 int_status;
2179 
2180 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2181 
2182 	if (!bnxt_has_work(bp, cpr)) {
2183 		int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2184 		/* return if erroneous interrupt */
2185 		if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2186 			return IRQ_NONE;
2187 	}
2188 
2189 	/* disable ring IRQ */
2190 	BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2191 
2192 	/* Return here if interrupt is shared and is disabled. */
2193 	if (unlikely(atomic_read(&bp->intr_sem) != 0))
2194 		return IRQ_HANDLED;
2195 
2196 	napi_schedule(&bnapi->napi);
2197 	return IRQ_HANDLED;
2198 }
2199 
2200 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2201 			    int budget)
2202 {
2203 	struct bnxt_napi *bnapi = cpr->bnapi;
2204 	u32 raw_cons = cpr->cp_raw_cons;
2205 	u32 cons;
2206 	int tx_pkts = 0;
2207 	int rx_pkts = 0;
2208 	u8 event = 0;
2209 	struct tx_cmp *txcmp;
2210 
2211 	cpr->has_more_work = 0;
2212 	cpr->had_work_done = 1;
2213 	while (1) {
2214 		int rc;
2215 
2216 		cons = RING_CMP(raw_cons);
2217 		txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2218 
2219 		if (!TX_CMP_VALID(txcmp, raw_cons))
2220 			break;
2221 
2222 		/* The valid test of the entry must be done first before
2223 		 * reading any further.
2224 		 */
2225 		dma_rmb();
2226 		if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2227 			tx_pkts++;
2228 			/* return full budget so NAPI will complete. */
2229 			if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2230 				rx_pkts = budget;
2231 				raw_cons = NEXT_RAW_CMP(raw_cons);
2232 				if (budget)
2233 					cpr->has_more_work = 1;
2234 				break;
2235 			}
2236 		} else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2237 			if (likely(budget))
2238 				rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2239 			else
2240 				rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2241 							   &event);
2242 			if (likely(rc >= 0))
2243 				rx_pkts += rc;
2244 			/* Increment rx_pkts when rc is -ENOMEM to count towards
2245 			 * the NAPI budget.  Otherwise, we may potentially loop
2246 			 * here forever if we consistently cannot allocate
2247 			 * buffers.
2248 			 */
2249 			else if (rc == -ENOMEM && budget)
2250 				rx_pkts++;
2251 			else if (rc == -EBUSY)	/* partial completion */
2252 				break;
2253 		} else if (unlikely((TX_CMP_TYPE(txcmp) ==
2254 				     CMPL_BASE_TYPE_HWRM_DONE) ||
2255 				    (TX_CMP_TYPE(txcmp) ==
2256 				     CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2257 				    (TX_CMP_TYPE(txcmp) ==
2258 				     CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2259 			bnxt_hwrm_handler(bp, txcmp);
2260 		}
2261 		raw_cons = NEXT_RAW_CMP(raw_cons);
2262 
2263 		if (rx_pkts && rx_pkts == budget) {
2264 			cpr->has_more_work = 1;
2265 			break;
2266 		}
2267 	}
2268 
2269 	if (event & BNXT_REDIRECT_EVENT)
2270 		xdp_do_flush_map();
2271 
2272 	if (event & BNXT_TX_EVENT) {
2273 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2274 		u16 prod = txr->tx_prod;
2275 
2276 		/* Sync BD data before updating doorbell */
2277 		wmb();
2278 
2279 		bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2280 	}
2281 
2282 	cpr->cp_raw_cons = raw_cons;
2283 	bnapi->tx_pkts += tx_pkts;
2284 	bnapi->events |= event;
2285 	return rx_pkts;
2286 }
2287 
2288 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2289 {
2290 	if (bnapi->tx_pkts) {
2291 		bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2292 		bnapi->tx_pkts = 0;
2293 	}
2294 
2295 	if ((bnapi->events & BNXT_RX_EVENT) && !(bnapi->in_reset)) {
2296 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2297 
2298 		if (bnapi->events & BNXT_AGG_EVENT)
2299 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2300 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2301 	}
2302 	bnapi->events = 0;
2303 }
2304 
2305 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2306 			  int budget)
2307 {
2308 	struct bnxt_napi *bnapi = cpr->bnapi;
2309 	int rx_pkts;
2310 
2311 	rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2312 
2313 	/* ACK completion ring before freeing tx ring and producing new
2314 	 * buffers in rx/agg rings to prevent overflowing the completion
2315 	 * ring.
2316 	 */
2317 	bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2318 
2319 	__bnxt_poll_work_done(bp, bnapi);
2320 	return rx_pkts;
2321 }
2322 
2323 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2324 {
2325 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2326 	struct bnxt *bp = bnapi->bp;
2327 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2328 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2329 	struct tx_cmp *txcmp;
2330 	struct rx_cmp_ext *rxcmp1;
2331 	u32 cp_cons, tmp_raw_cons;
2332 	u32 raw_cons = cpr->cp_raw_cons;
2333 	u32 rx_pkts = 0;
2334 	u8 event = 0;
2335 
2336 	while (1) {
2337 		int rc;
2338 
2339 		cp_cons = RING_CMP(raw_cons);
2340 		txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2341 
2342 		if (!TX_CMP_VALID(txcmp, raw_cons))
2343 			break;
2344 
2345 		if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2346 			tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2347 			cp_cons = RING_CMP(tmp_raw_cons);
2348 			rxcmp1 = (struct rx_cmp_ext *)
2349 			  &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2350 
2351 			if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2352 				break;
2353 
2354 			/* force an error to recycle the buffer */
2355 			rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2356 				cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2357 
2358 			rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2359 			if (likely(rc == -EIO) && budget)
2360 				rx_pkts++;
2361 			else if (rc == -EBUSY)	/* partial completion */
2362 				break;
2363 		} else if (unlikely(TX_CMP_TYPE(txcmp) ==
2364 				    CMPL_BASE_TYPE_HWRM_DONE)) {
2365 			bnxt_hwrm_handler(bp, txcmp);
2366 		} else {
2367 			netdev_err(bp->dev,
2368 				   "Invalid completion received on special ring\n");
2369 		}
2370 		raw_cons = NEXT_RAW_CMP(raw_cons);
2371 
2372 		if (rx_pkts == budget)
2373 			break;
2374 	}
2375 
2376 	cpr->cp_raw_cons = raw_cons;
2377 	BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2378 	bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2379 
2380 	if (event & BNXT_AGG_EVENT)
2381 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2382 
2383 	if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2384 		napi_complete_done(napi, rx_pkts);
2385 		BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2386 	}
2387 	return rx_pkts;
2388 }
2389 
2390 static int bnxt_poll(struct napi_struct *napi, int budget)
2391 {
2392 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2393 	struct bnxt *bp = bnapi->bp;
2394 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2395 	int work_done = 0;
2396 
2397 	while (1) {
2398 		work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2399 
2400 		if (work_done >= budget) {
2401 			if (!budget)
2402 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2403 			break;
2404 		}
2405 
2406 		if (!bnxt_has_work(bp, cpr)) {
2407 			if (napi_complete_done(napi, work_done))
2408 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2409 			break;
2410 		}
2411 	}
2412 	if (bp->flags & BNXT_FLAG_DIM) {
2413 		struct dim_sample dim_sample = {};
2414 
2415 		dim_update_sample(cpr->event_ctr,
2416 				  cpr->rx_packets,
2417 				  cpr->rx_bytes,
2418 				  &dim_sample);
2419 		net_dim(&cpr->dim, dim_sample);
2420 	}
2421 	return work_done;
2422 }
2423 
2424 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2425 {
2426 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2427 	int i, work_done = 0;
2428 
2429 	for (i = 0; i < 2; i++) {
2430 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2431 
2432 		if (cpr2) {
2433 			work_done += __bnxt_poll_work(bp, cpr2,
2434 						      budget - work_done);
2435 			cpr->has_more_work |= cpr2->has_more_work;
2436 		}
2437 	}
2438 	return work_done;
2439 }
2440 
2441 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2442 				 u64 dbr_type)
2443 {
2444 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2445 	int i;
2446 
2447 	for (i = 0; i < 2; i++) {
2448 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2449 		struct bnxt_db_info *db;
2450 
2451 		if (cpr2 && cpr2->had_work_done) {
2452 			db = &cpr2->cp_db;
2453 			writeq(db->db_key64 | dbr_type |
2454 			       RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2455 			cpr2->had_work_done = 0;
2456 		}
2457 	}
2458 	__bnxt_poll_work_done(bp, bnapi);
2459 }
2460 
2461 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2462 {
2463 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2464 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2465 	u32 raw_cons = cpr->cp_raw_cons;
2466 	struct bnxt *bp = bnapi->bp;
2467 	struct nqe_cn *nqcmp;
2468 	int work_done = 0;
2469 	u32 cons;
2470 
2471 	if (cpr->has_more_work) {
2472 		cpr->has_more_work = 0;
2473 		work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2474 	}
2475 	while (1) {
2476 		cons = RING_CMP(raw_cons);
2477 		nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2478 
2479 		if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2480 			if (cpr->has_more_work)
2481 				break;
2482 
2483 			__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2484 			cpr->cp_raw_cons = raw_cons;
2485 			if (napi_complete_done(napi, work_done))
2486 				BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2487 						  cpr->cp_raw_cons);
2488 			return work_done;
2489 		}
2490 
2491 		/* The valid test of the entry must be done first before
2492 		 * reading any further.
2493 		 */
2494 		dma_rmb();
2495 
2496 		if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2497 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2498 			struct bnxt_cp_ring_info *cpr2;
2499 
2500 			cpr2 = cpr->cp_ring_arr[idx];
2501 			work_done += __bnxt_poll_work(bp, cpr2,
2502 						      budget - work_done);
2503 			cpr->has_more_work |= cpr2->has_more_work;
2504 		} else {
2505 			bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2506 		}
2507 		raw_cons = NEXT_RAW_CMP(raw_cons);
2508 	}
2509 	__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2510 	if (raw_cons != cpr->cp_raw_cons) {
2511 		cpr->cp_raw_cons = raw_cons;
2512 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2513 	}
2514 	return work_done;
2515 }
2516 
2517 static void bnxt_free_tx_skbs(struct bnxt *bp)
2518 {
2519 	int i, max_idx;
2520 	struct pci_dev *pdev = bp->pdev;
2521 
2522 	if (!bp->tx_ring)
2523 		return;
2524 
2525 	max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2526 	for (i = 0; i < bp->tx_nr_rings; i++) {
2527 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2528 		int j;
2529 
2530 		for (j = 0; j < max_idx;) {
2531 			struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2532 			struct sk_buff *skb;
2533 			int k, last;
2534 
2535 			if (i < bp->tx_nr_rings_xdp &&
2536 			    tx_buf->action == XDP_REDIRECT) {
2537 				dma_unmap_single(&pdev->dev,
2538 					dma_unmap_addr(tx_buf, mapping),
2539 					dma_unmap_len(tx_buf, len),
2540 					PCI_DMA_TODEVICE);
2541 				xdp_return_frame(tx_buf->xdpf);
2542 				tx_buf->action = 0;
2543 				tx_buf->xdpf = NULL;
2544 				j++;
2545 				continue;
2546 			}
2547 
2548 			skb = tx_buf->skb;
2549 			if (!skb) {
2550 				j++;
2551 				continue;
2552 			}
2553 
2554 			tx_buf->skb = NULL;
2555 
2556 			if (tx_buf->is_push) {
2557 				dev_kfree_skb(skb);
2558 				j += 2;
2559 				continue;
2560 			}
2561 
2562 			dma_unmap_single(&pdev->dev,
2563 					 dma_unmap_addr(tx_buf, mapping),
2564 					 skb_headlen(skb),
2565 					 PCI_DMA_TODEVICE);
2566 
2567 			last = tx_buf->nr_frags;
2568 			j += 2;
2569 			for (k = 0; k < last; k++, j++) {
2570 				int ring_idx = j & bp->tx_ring_mask;
2571 				skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2572 
2573 				tx_buf = &txr->tx_buf_ring[ring_idx];
2574 				dma_unmap_page(
2575 					&pdev->dev,
2576 					dma_unmap_addr(tx_buf, mapping),
2577 					skb_frag_size(frag), PCI_DMA_TODEVICE);
2578 			}
2579 			dev_kfree_skb(skb);
2580 		}
2581 		netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2582 	}
2583 }
2584 
2585 static void bnxt_free_one_rx_ring_skbs(struct bnxt *bp, int ring_nr)
2586 {
2587 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
2588 	struct pci_dev *pdev = bp->pdev;
2589 	struct bnxt_tpa_idx_map *map;
2590 	int i, max_idx, max_agg_idx;
2591 
2592 	max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2593 	max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2594 	if (!rxr->rx_tpa)
2595 		goto skip_rx_tpa_free;
2596 
2597 	for (i = 0; i < bp->max_tpa; i++) {
2598 		struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[i];
2599 		u8 *data = tpa_info->data;
2600 
2601 		if (!data)
2602 			continue;
2603 
2604 		dma_unmap_single_attrs(&pdev->dev, tpa_info->mapping,
2605 				       bp->rx_buf_use_size, bp->rx_dir,
2606 				       DMA_ATTR_WEAK_ORDERING);
2607 
2608 		tpa_info->data = NULL;
2609 
2610 		kfree(data);
2611 	}
2612 
2613 skip_rx_tpa_free:
2614 	for (i = 0; i < max_idx; i++) {
2615 		struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[i];
2616 		dma_addr_t mapping = rx_buf->mapping;
2617 		void *data = rx_buf->data;
2618 
2619 		if (!data)
2620 			continue;
2621 
2622 		rx_buf->data = NULL;
2623 		if (BNXT_RX_PAGE_MODE(bp)) {
2624 			mapping -= bp->rx_dma_offset;
2625 			dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE,
2626 					     bp->rx_dir,
2627 					     DMA_ATTR_WEAK_ORDERING);
2628 			page_pool_recycle_direct(rxr->page_pool, data);
2629 		} else {
2630 			dma_unmap_single_attrs(&pdev->dev, mapping,
2631 					       bp->rx_buf_use_size, bp->rx_dir,
2632 					       DMA_ATTR_WEAK_ORDERING);
2633 			kfree(data);
2634 		}
2635 	}
2636 	for (i = 0; i < max_agg_idx; i++) {
2637 		struct bnxt_sw_rx_agg_bd *rx_agg_buf = &rxr->rx_agg_ring[i];
2638 		struct page *page = rx_agg_buf->page;
2639 
2640 		if (!page)
2641 			continue;
2642 
2643 		dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2644 				     BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
2645 				     DMA_ATTR_WEAK_ORDERING);
2646 
2647 		rx_agg_buf->page = NULL;
2648 		__clear_bit(i, rxr->rx_agg_bmap);
2649 
2650 		__free_page(page);
2651 	}
2652 	if (rxr->rx_page) {
2653 		__free_page(rxr->rx_page);
2654 		rxr->rx_page = NULL;
2655 	}
2656 	map = rxr->rx_tpa_idx_map;
2657 	if (map)
2658 		memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2659 }
2660 
2661 static void bnxt_free_rx_skbs(struct bnxt *bp)
2662 {
2663 	int i;
2664 
2665 	if (!bp->rx_ring)
2666 		return;
2667 
2668 	for (i = 0; i < bp->rx_nr_rings; i++)
2669 		bnxt_free_one_rx_ring_skbs(bp, i);
2670 }
2671 
2672 static void bnxt_free_skbs(struct bnxt *bp)
2673 {
2674 	bnxt_free_tx_skbs(bp);
2675 	bnxt_free_rx_skbs(bp);
2676 }
2677 
2678 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2679 {
2680 	struct pci_dev *pdev = bp->pdev;
2681 	int i;
2682 
2683 	for (i = 0; i < rmem->nr_pages; i++) {
2684 		if (!rmem->pg_arr[i])
2685 			continue;
2686 
2687 		dma_free_coherent(&pdev->dev, rmem->page_size,
2688 				  rmem->pg_arr[i], rmem->dma_arr[i]);
2689 
2690 		rmem->pg_arr[i] = NULL;
2691 	}
2692 	if (rmem->pg_tbl) {
2693 		size_t pg_tbl_size = rmem->nr_pages * 8;
2694 
2695 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2696 			pg_tbl_size = rmem->page_size;
2697 		dma_free_coherent(&pdev->dev, pg_tbl_size,
2698 				  rmem->pg_tbl, rmem->pg_tbl_map);
2699 		rmem->pg_tbl = NULL;
2700 	}
2701 	if (rmem->vmem_size && *rmem->vmem) {
2702 		vfree(*rmem->vmem);
2703 		*rmem->vmem = NULL;
2704 	}
2705 }
2706 
2707 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2708 {
2709 	struct pci_dev *pdev = bp->pdev;
2710 	u64 valid_bit = 0;
2711 	int i;
2712 
2713 	if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2714 		valid_bit = PTU_PTE_VALID;
2715 	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2716 		size_t pg_tbl_size = rmem->nr_pages * 8;
2717 
2718 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2719 			pg_tbl_size = rmem->page_size;
2720 		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2721 						  &rmem->pg_tbl_map,
2722 						  GFP_KERNEL);
2723 		if (!rmem->pg_tbl)
2724 			return -ENOMEM;
2725 	}
2726 
2727 	for (i = 0; i < rmem->nr_pages; i++) {
2728 		u64 extra_bits = valid_bit;
2729 
2730 		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2731 						     rmem->page_size,
2732 						     &rmem->dma_arr[i],
2733 						     GFP_KERNEL);
2734 		if (!rmem->pg_arr[i])
2735 			return -ENOMEM;
2736 
2737 		if (rmem->init_val)
2738 			memset(rmem->pg_arr[i], rmem->init_val,
2739 			       rmem->page_size);
2740 		if (rmem->nr_pages > 1 || rmem->depth > 0) {
2741 			if (i == rmem->nr_pages - 2 &&
2742 			    (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2743 				extra_bits |= PTU_PTE_NEXT_TO_LAST;
2744 			else if (i == rmem->nr_pages - 1 &&
2745 				 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2746 				extra_bits |= PTU_PTE_LAST;
2747 			rmem->pg_tbl[i] =
2748 				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2749 		}
2750 	}
2751 
2752 	if (rmem->vmem_size) {
2753 		*rmem->vmem = vzalloc(rmem->vmem_size);
2754 		if (!(*rmem->vmem))
2755 			return -ENOMEM;
2756 	}
2757 	return 0;
2758 }
2759 
2760 static void bnxt_free_tpa_info(struct bnxt *bp)
2761 {
2762 	int i;
2763 
2764 	for (i = 0; i < bp->rx_nr_rings; i++) {
2765 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2766 
2767 		kfree(rxr->rx_tpa_idx_map);
2768 		rxr->rx_tpa_idx_map = NULL;
2769 		if (rxr->rx_tpa) {
2770 			kfree(rxr->rx_tpa[0].agg_arr);
2771 			rxr->rx_tpa[0].agg_arr = NULL;
2772 		}
2773 		kfree(rxr->rx_tpa);
2774 		rxr->rx_tpa = NULL;
2775 	}
2776 }
2777 
2778 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2779 {
2780 	int i, j, total_aggs = 0;
2781 
2782 	bp->max_tpa = MAX_TPA;
2783 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
2784 		if (!bp->max_tpa_v2)
2785 			return 0;
2786 		bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2787 		total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2788 	}
2789 
2790 	for (i = 0; i < bp->rx_nr_rings; i++) {
2791 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2792 		struct rx_agg_cmp *agg;
2793 
2794 		rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2795 				      GFP_KERNEL);
2796 		if (!rxr->rx_tpa)
2797 			return -ENOMEM;
2798 
2799 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2800 			continue;
2801 		agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2802 		rxr->rx_tpa[0].agg_arr = agg;
2803 		if (!agg)
2804 			return -ENOMEM;
2805 		for (j = 1; j < bp->max_tpa; j++)
2806 			rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2807 		rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2808 					      GFP_KERNEL);
2809 		if (!rxr->rx_tpa_idx_map)
2810 			return -ENOMEM;
2811 	}
2812 	return 0;
2813 }
2814 
2815 static void bnxt_free_rx_rings(struct bnxt *bp)
2816 {
2817 	int i;
2818 
2819 	if (!bp->rx_ring)
2820 		return;
2821 
2822 	bnxt_free_tpa_info(bp);
2823 	for (i = 0; i < bp->rx_nr_rings; i++) {
2824 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2825 		struct bnxt_ring_struct *ring;
2826 
2827 		if (rxr->xdp_prog)
2828 			bpf_prog_put(rxr->xdp_prog);
2829 
2830 		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2831 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
2832 
2833 		page_pool_destroy(rxr->page_pool);
2834 		rxr->page_pool = NULL;
2835 
2836 		kfree(rxr->rx_agg_bmap);
2837 		rxr->rx_agg_bmap = NULL;
2838 
2839 		ring = &rxr->rx_ring_struct;
2840 		bnxt_free_ring(bp, &ring->ring_mem);
2841 
2842 		ring = &rxr->rx_agg_ring_struct;
2843 		bnxt_free_ring(bp, &ring->ring_mem);
2844 	}
2845 }
2846 
2847 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2848 				   struct bnxt_rx_ring_info *rxr)
2849 {
2850 	struct page_pool_params pp = { 0 };
2851 
2852 	pp.pool_size = bp->rx_ring_size;
2853 	pp.nid = dev_to_node(&bp->pdev->dev);
2854 	pp.dev = &bp->pdev->dev;
2855 	pp.dma_dir = DMA_BIDIRECTIONAL;
2856 
2857 	rxr->page_pool = page_pool_create(&pp);
2858 	if (IS_ERR(rxr->page_pool)) {
2859 		int err = PTR_ERR(rxr->page_pool);
2860 
2861 		rxr->page_pool = NULL;
2862 		return err;
2863 	}
2864 	return 0;
2865 }
2866 
2867 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2868 {
2869 	int i, rc = 0, agg_rings = 0;
2870 
2871 	if (!bp->rx_ring)
2872 		return -ENOMEM;
2873 
2874 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
2875 		agg_rings = 1;
2876 
2877 	for (i = 0; i < bp->rx_nr_rings; i++) {
2878 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2879 		struct bnxt_ring_struct *ring;
2880 
2881 		ring = &rxr->rx_ring_struct;
2882 
2883 		rc = bnxt_alloc_rx_page_pool(bp, rxr);
2884 		if (rc)
2885 			return rc;
2886 
2887 		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2888 		if (rc < 0)
2889 			return rc;
2890 
2891 		rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2892 						MEM_TYPE_PAGE_POOL,
2893 						rxr->page_pool);
2894 		if (rc) {
2895 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
2896 			return rc;
2897 		}
2898 
2899 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2900 		if (rc)
2901 			return rc;
2902 
2903 		ring->grp_idx = i;
2904 		if (agg_rings) {
2905 			u16 mem_size;
2906 
2907 			ring = &rxr->rx_agg_ring_struct;
2908 			rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2909 			if (rc)
2910 				return rc;
2911 
2912 			ring->grp_idx = i;
2913 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2914 			mem_size = rxr->rx_agg_bmap_size / 8;
2915 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2916 			if (!rxr->rx_agg_bmap)
2917 				return -ENOMEM;
2918 		}
2919 	}
2920 	if (bp->flags & BNXT_FLAG_TPA)
2921 		rc = bnxt_alloc_tpa_info(bp);
2922 	return rc;
2923 }
2924 
2925 static void bnxt_free_tx_rings(struct bnxt *bp)
2926 {
2927 	int i;
2928 	struct pci_dev *pdev = bp->pdev;
2929 
2930 	if (!bp->tx_ring)
2931 		return;
2932 
2933 	for (i = 0; i < bp->tx_nr_rings; i++) {
2934 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2935 		struct bnxt_ring_struct *ring;
2936 
2937 		if (txr->tx_push) {
2938 			dma_free_coherent(&pdev->dev, bp->tx_push_size,
2939 					  txr->tx_push, txr->tx_push_mapping);
2940 			txr->tx_push = NULL;
2941 		}
2942 
2943 		ring = &txr->tx_ring_struct;
2944 
2945 		bnxt_free_ring(bp, &ring->ring_mem);
2946 	}
2947 }
2948 
2949 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2950 {
2951 	int i, j, rc;
2952 	struct pci_dev *pdev = bp->pdev;
2953 
2954 	bp->tx_push_size = 0;
2955 	if (bp->tx_push_thresh) {
2956 		int push_size;
2957 
2958 		push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2959 					bp->tx_push_thresh);
2960 
2961 		if (push_size > 256) {
2962 			push_size = 0;
2963 			bp->tx_push_thresh = 0;
2964 		}
2965 
2966 		bp->tx_push_size = push_size;
2967 	}
2968 
2969 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2970 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2971 		struct bnxt_ring_struct *ring;
2972 		u8 qidx;
2973 
2974 		ring = &txr->tx_ring_struct;
2975 
2976 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2977 		if (rc)
2978 			return rc;
2979 
2980 		ring->grp_idx = txr->bnapi->index;
2981 		if (bp->tx_push_size) {
2982 			dma_addr_t mapping;
2983 
2984 			/* One pre-allocated DMA buffer to backup
2985 			 * TX push operation
2986 			 */
2987 			txr->tx_push = dma_alloc_coherent(&pdev->dev,
2988 						bp->tx_push_size,
2989 						&txr->tx_push_mapping,
2990 						GFP_KERNEL);
2991 
2992 			if (!txr->tx_push)
2993 				return -ENOMEM;
2994 
2995 			mapping = txr->tx_push_mapping +
2996 				sizeof(struct tx_push_bd);
2997 			txr->data_mapping = cpu_to_le64(mapping);
2998 		}
2999 		qidx = bp->tc_to_qidx[j];
3000 		ring->queue_id = bp->q_info[qidx].queue_id;
3001 		if (i < bp->tx_nr_rings_xdp)
3002 			continue;
3003 		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
3004 			j++;
3005 	}
3006 	return 0;
3007 }
3008 
3009 static void bnxt_free_cp_rings(struct bnxt *bp)
3010 {
3011 	int i;
3012 
3013 	if (!bp->bnapi)
3014 		return;
3015 
3016 	for (i = 0; i < bp->cp_nr_rings; i++) {
3017 		struct bnxt_napi *bnapi = bp->bnapi[i];
3018 		struct bnxt_cp_ring_info *cpr;
3019 		struct bnxt_ring_struct *ring;
3020 		int j;
3021 
3022 		if (!bnapi)
3023 			continue;
3024 
3025 		cpr = &bnapi->cp_ring;
3026 		ring = &cpr->cp_ring_struct;
3027 
3028 		bnxt_free_ring(bp, &ring->ring_mem);
3029 
3030 		for (j = 0; j < 2; j++) {
3031 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3032 
3033 			if (cpr2) {
3034 				ring = &cpr2->cp_ring_struct;
3035 				bnxt_free_ring(bp, &ring->ring_mem);
3036 				kfree(cpr2);
3037 				cpr->cp_ring_arr[j] = NULL;
3038 			}
3039 		}
3040 	}
3041 }
3042 
3043 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3044 {
3045 	struct bnxt_ring_mem_info *rmem;
3046 	struct bnxt_ring_struct *ring;
3047 	struct bnxt_cp_ring_info *cpr;
3048 	int rc;
3049 
3050 	cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3051 	if (!cpr)
3052 		return NULL;
3053 
3054 	ring = &cpr->cp_ring_struct;
3055 	rmem = &ring->ring_mem;
3056 	rmem->nr_pages = bp->cp_nr_pages;
3057 	rmem->page_size = HW_CMPD_RING_SIZE;
3058 	rmem->pg_arr = (void **)cpr->cp_desc_ring;
3059 	rmem->dma_arr = cpr->cp_desc_mapping;
3060 	rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3061 	rc = bnxt_alloc_ring(bp, rmem);
3062 	if (rc) {
3063 		bnxt_free_ring(bp, rmem);
3064 		kfree(cpr);
3065 		cpr = NULL;
3066 	}
3067 	return cpr;
3068 }
3069 
3070 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3071 {
3072 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3073 	int i, rc, ulp_base_vec, ulp_msix;
3074 
3075 	ulp_msix = bnxt_get_ulp_msix_num(bp);
3076 	ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3077 	for (i = 0; i < bp->cp_nr_rings; i++) {
3078 		struct bnxt_napi *bnapi = bp->bnapi[i];
3079 		struct bnxt_cp_ring_info *cpr;
3080 		struct bnxt_ring_struct *ring;
3081 
3082 		if (!bnapi)
3083 			continue;
3084 
3085 		cpr = &bnapi->cp_ring;
3086 		cpr->bnapi = bnapi;
3087 		ring = &cpr->cp_ring_struct;
3088 
3089 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3090 		if (rc)
3091 			return rc;
3092 
3093 		if (ulp_msix && i >= ulp_base_vec)
3094 			ring->map_idx = i + ulp_msix;
3095 		else
3096 			ring->map_idx = i;
3097 
3098 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3099 			continue;
3100 
3101 		if (i < bp->rx_nr_rings) {
3102 			struct bnxt_cp_ring_info *cpr2 =
3103 				bnxt_alloc_cp_sub_ring(bp);
3104 
3105 			cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3106 			if (!cpr2)
3107 				return -ENOMEM;
3108 			cpr2->bnapi = bnapi;
3109 		}
3110 		if ((sh && i < bp->tx_nr_rings) ||
3111 		    (!sh && i >= bp->rx_nr_rings)) {
3112 			struct bnxt_cp_ring_info *cpr2 =
3113 				bnxt_alloc_cp_sub_ring(bp);
3114 
3115 			cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3116 			if (!cpr2)
3117 				return -ENOMEM;
3118 			cpr2->bnapi = bnapi;
3119 		}
3120 	}
3121 	return 0;
3122 }
3123 
3124 static void bnxt_init_ring_struct(struct bnxt *bp)
3125 {
3126 	int i;
3127 
3128 	for (i = 0; i < bp->cp_nr_rings; i++) {
3129 		struct bnxt_napi *bnapi = bp->bnapi[i];
3130 		struct bnxt_ring_mem_info *rmem;
3131 		struct bnxt_cp_ring_info *cpr;
3132 		struct bnxt_rx_ring_info *rxr;
3133 		struct bnxt_tx_ring_info *txr;
3134 		struct bnxt_ring_struct *ring;
3135 
3136 		if (!bnapi)
3137 			continue;
3138 
3139 		cpr = &bnapi->cp_ring;
3140 		ring = &cpr->cp_ring_struct;
3141 		rmem = &ring->ring_mem;
3142 		rmem->nr_pages = bp->cp_nr_pages;
3143 		rmem->page_size = HW_CMPD_RING_SIZE;
3144 		rmem->pg_arr = (void **)cpr->cp_desc_ring;
3145 		rmem->dma_arr = cpr->cp_desc_mapping;
3146 		rmem->vmem_size = 0;
3147 
3148 		rxr = bnapi->rx_ring;
3149 		if (!rxr)
3150 			goto skip_rx;
3151 
3152 		ring = &rxr->rx_ring_struct;
3153 		rmem = &ring->ring_mem;
3154 		rmem->nr_pages = bp->rx_nr_pages;
3155 		rmem->page_size = HW_RXBD_RING_SIZE;
3156 		rmem->pg_arr = (void **)rxr->rx_desc_ring;
3157 		rmem->dma_arr = rxr->rx_desc_mapping;
3158 		rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3159 		rmem->vmem = (void **)&rxr->rx_buf_ring;
3160 
3161 		ring = &rxr->rx_agg_ring_struct;
3162 		rmem = &ring->ring_mem;
3163 		rmem->nr_pages = bp->rx_agg_nr_pages;
3164 		rmem->page_size = HW_RXBD_RING_SIZE;
3165 		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3166 		rmem->dma_arr = rxr->rx_agg_desc_mapping;
3167 		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3168 		rmem->vmem = (void **)&rxr->rx_agg_ring;
3169 
3170 skip_rx:
3171 		txr = bnapi->tx_ring;
3172 		if (!txr)
3173 			continue;
3174 
3175 		ring = &txr->tx_ring_struct;
3176 		rmem = &ring->ring_mem;
3177 		rmem->nr_pages = bp->tx_nr_pages;
3178 		rmem->page_size = HW_RXBD_RING_SIZE;
3179 		rmem->pg_arr = (void **)txr->tx_desc_ring;
3180 		rmem->dma_arr = txr->tx_desc_mapping;
3181 		rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3182 		rmem->vmem = (void **)&txr->tx_buf_ring;
3183 	}
3184 }
3185 
3186 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3187 {
3188 	int i;
3189 	u32 prod;
3190 	struct rx_bd **rx_buf_ring;
3191 
3192 	rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3193 	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3194 		int j;
3195 		struct rx_bd *rxbd;
3196 
3197 		rxbd = rx_buf_ring[i];
3198 		if (!rxbd)
3199 			continue;
3200 
3201 		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3202 			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3203 			rxbd->rx_bd_opaque = prod;
3204 		}
3205 	}
3206 }
3207 
3208 static int bnxt_alloc_one_rx_ring(struct bnxt *bp, int ring_nr)
3209 {
3210 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
3211 	struct net_device *dev = bp->dev;
3212 	u32 prod;
3213 	int i;
3214 
3215 	prod = rxr->rx_prod;
3216 	for (i = 0; i < bp->rx_ring_size; i++) {
3217 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL)) {
3218 			netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3219 				    ring_nr, i, bp->rx_ring_size);
3220 			break;
3221 		}
3222 		prod = NEXT_RX(prod);
3223 	}
3224 	rxr->rx_prod = prod;
3225 
3226 	if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3227 		return 0;
3228 
3229 	prod = rxr->rx_agg_prod;
3230 	for (i = 0; i < bp->rx_agg_ring_size; i++) {
3231 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL)) {
3232 			netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3233 				    ring_nr, i, bp->rx_ring_size);
3234 			break;
3235 		}
3236 		prod = NEXT_RX_AGG(prod);
3237 	}
3238 	rxr->rx_agg_prod = prod;
3239 
3240 	if (rxr->rx_tpa) {
3241 		dma_addr_t mapping;
3242 		u8 *data;
3243 
3244 		for (i = 0; i < bp->max_tpa; i++) {
3245 			data = __bnxt_alloc_rx_data(bp, &mapping, GFP_KERNEL);
3246 			if (!data)
3247 				return -ENOMEM;
3248 
3249 			rxr->rx_tpa[i].data = data;
3250 			rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3251 			rxr->rx_tpa[i].mapping = mapping;
3252 		}
3253 	}
3254 	return 0;
3255 }
3256 
3257 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3258 {
3259 	struct bnxt_rx_ring_info *rxr;
3260 	struct bnxt_ring_struct *ring;
3261 	u32 type;
3262 
3263 	type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3264 		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3265 
3266 	if (NET_IP_ALIGN == 2)
3267 		type |= RX_BD_FLAGS_SOP;
3268 
3269 	rxr = &bp->rx_ring[ring_nr];
3270 	ring = &rxr->rx_ring_struct;
3271 	bnxt_init_rxbd_pages(ring, type);
3272 
3273 	if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3274 		bpf_prog_add(bp->xdp_prog, 1);
3275 		rxr->xdp_prog = bp->xdp_prog;
3276 	}
3277 	ring->fw_ring_id = INVALID_HW_RING_ID;
3278 
3279 	ring = &rxr->rx_agg_ring_struct;
3280 	ring->fw_ring_id = INVALID_HW_RING_ID;
3281 
3282 	if ((bp->flags & BNXT_FLAG_AGG_RINGS)) {
3283 		type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3284 			RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3285 
3286 		bnxt_init_rxbd_pages(ring, type);
3287 	}
3288 
3289 	return bnxt_alloc_one_rx_ring(bp, ring_nr);
3290 }
3291 
3292 static void bnxt_init_cp_rings(struct bnxt *bp)
3293 {
3294 	int i, j;
3295 
3296 	for (i = 0; i < bp->cp_nr_rings; i++) {
3297 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3298 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3299 
3300 		ring->fw_ring_id = INVALID_HW_RING_ID;
3301 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3302 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3303 		for (j = 0; j < 2; j++) {
3304 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3305 
3306 			if (!cpr2)
3307 				continue;
3308 
3309 			ring = &cpr2->cp_ring_struct;
3310 			ring->fw_ring_id = INVALID_HW_RING_ID;
3311 			cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3312 			cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3313 		}
3314 	}
3315 }
3316 
3317 static int bnxt_init_rx_rings(struct bnxt *bp)
3318 {
3319 	int i, rc = 0;
3320 
3321 	if (BNXT_RX_PAGE_MODE(bp)) {
3322 		bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3323 		bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3324 	} else {
3325 		bp->rx_offset = BNXT_RX_OFFSET;
3326 		bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3327 	}
3328 
3329 	for (i = 0; i < bp->rx_nr_rings; i++) {
3330 		rc = bnxt_init_one_rx_ring(bp, i);
3331 		if (rc)
3332 			break;
3333 	}
3334 
3335 	return rc;
3336 }
3337 
3338 static int bnxt_init_tx_rings(struct bnxt *bp)
3339 {
3340 	u16 i;
3341 
3342 	bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3343 				   MAX_SKB_FRAGS + 1);
3344 
3345 	for (i = 0; i < bp->tx_nr_rings; i++) {
3346 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3347 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3348 
3349 		ring->fw_ring_id = INVALID_HW_RING_ID;
3350 	}
3351 
3352 	return 0;
3353 }
3354 
3355 static void bnxt_free_ring_grps(struct bnxt *bp)
3356 {
3357 	kfree(bp->grp_info);
3358 	bp->grp_info = NULL;
3359 }
3360 
3361 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3362 {
3363 	int i;
3364 
3365 	if (irq_re_init) {
3366 		bp->grp_info = kcalloc(bp->cp_nr_rings,
3367 				       sizeof(struct bnxt_ring_grp_info),
3368 				       GFP_KERNEL);
3369 		if (!bp->grp_info)
3370 			return -ENOMEM;
3371 	}
3372 	for (i = 0; i < bp->cp_nr_rings; i++) {
3373 		if (irq_re_init)
3374 			bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3375 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3376 		bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3377 		bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3378 		bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3379 	}
3380 	return 0;
3381 }
3382 
3383 static void bnxt_free_vnics(struct bnxt *bp)
3384 {
3385 	kfree(bp->vnic_info);
3386 	bp->vnic_info = NULL;
3387 	bp->nr_vnics = 0;
3388 }
3389 
3390 static int bnxt_alloc_vnics(struct bnxt *bp)
3391 {
3392 	int num_vnics = 1;
3393 
3394 #ifdef CONFIG_RFS_ACCEL
3395 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3396 		num_vnics += bp->rx_nr_rings;
3397 #endif
3398 
3399 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3400 		num_vnics++;
3401 
3402 	bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3403 				GFP_KERNEL);
3404 	if (!bp->vnic_info)
3405 		return -ENOMEM;
3406 
3407 	bp->nr_vnics = num_vnics;
3408 	return 0;
3409 }
3410 
3411 static void bnxt_init_vnics(struct bnxt *bp)
3412 {
3413 	int i;
3414 
3415 	for (i = 0; i < bp->nr_vnics; i++) {
3416 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3417 		int j;
3418 
3419 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
3420 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3421 			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3422 
3423 		vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3424 
3425 		if (bp->vnic_info[i].rss_hash_key) {
3426 			if (i == 0)
3427 				prandom_bytes(vnic->rss_hash_key,
3428 					      HW_HASH_KEY_SIZE);
3429 			else
3430 				memcpy(vnic->rss_hash_key,
3431 				       bp->vnic_info[0].rss_hash_key,
3432 				       HW_HASH_KEY_SIZE);
3433 		}
3434 	}
3435 }
3436 
3437 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3438 {
3439 	int pages;
3440 
3441 	pages = ring_size / desc_per_pg;
3442 
3443 	if (!pages)
3444 		return 1;
3445 
3446 	pages++;
3447 
3448 	while (pages & (pages - 1))
3449 		pages++;
3450 
3451 	return pages;
3452 }
3453 
3454 void bnxt_set_tpa_flags(struct bnxt *bp)
3455 {
3456 	bp->flags &= ~BNXT_FLAG_TPA;
3457 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3458 		return;
3459 	if (bp->dev->features & NETIF_F_LRO)
3460 		bp->flags |= BNXT_FLAG_LRO;
3461 	else if (bp->dev->features & NETIF_F_GRO_HW)
3462 		bp->flags |= BNXT_FLAG_GRO;
3463 }
3464 
3465 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3466  * be set on entry.
3467  */
3468 void bnxt_set_ring_params(struct bnxt *bp)
3469 {
3470 	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3471 	u32 agg_factor = 0, agg_ring_size = 0;
3472 
3473 	/* 8 for CRC and VLAN */
3474 	rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3475 
3476 	rx_space = rx_size + NET_SKB_PAD +
3477 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3478 
3479 	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3480 	ring_size = bp->rx_ring_size;
3481 	bp->rx_agg_ring_size = 0;
3482 	bp->rx_agg_nr_pages = 0;
3483 
3484 	if (bp->flags & BNXT_FLAG_TPA)
3485 		agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3486 
3487 	bp->flags &= ~BNXT_FLAG_JUMBO;
3488 	if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3489 		u32 jumbo_factor;
3490 
3491 		bp->flags |= BNXT_FLAG_JUMBO;
3492 		jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3493 		if (jumbo_factor > agg_factor)
3494 			agg_factor = jumbo_factor;
3495 	}
3496 	agg_ring_size = ring_size * agg_factor;
3497 
3498 	if (agg_ring_size) {
3499 		bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3500 							RX_DESC_CNT);
3501 		if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3502 			u32 tmp = agg_ring_size;
3503 
3504 			bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3505 			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3506 			netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3507 				    tmp, agg_ring_size);
3508 		}
3509 		bp->rx_agg_ring_size = agg_ring_size;
3510 		bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3511 		rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3512 		rx_space = rx_size + NET_SKB_PAD +
3513 			SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3514 	}
3515 
3516 	bp->rx_buf_use_size = rx_size;
3517 	bp->rx_buf_size = rx_space;
3518 
3519 	bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3520 	bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3521 
3522 	ring_size = bp->tx_ring_size;
3523 	bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3524 	bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3525 
3526 	max_rx_cmpl = bp->rx_ring_size;
3527 	/* MAX TPA needs to be added because TPA_START completions are
3528 	 * immediately recycled, so the TPA completions are not bound by
3529 	 * the RX ring size.
3530 	 */
3531 	if (bp->flags & BNXT_FLAG_TPA)
3532 		max_rx_cmpl += bp->max_tpa;
3533 	/* RX and TPA completions are 32-byte, all others are 16-byte */
3534 	ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3535 	bp->cp_ring_size = ring_size;
3536 
3537 	bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3538 	if (bp->cp_nr_pages > MAX_CP_PAGES) {
3539 		bp->cp_nr_pages = MAX_CP_PAGES;
3540 		bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3541 		netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3542 			    ring_size, bp->cp_ring_size);
3543 	}
3544 	bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3545 	bp->cp_ring_mask = bp->cp_bit - 1;
3546 }
3547 
3548 /* Changing allocation mode of RX rings.
3549  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3550  */
3551 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3552 {
3553 	if (page_mode) {
3554 		if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3555 			return -EOPNOTSUPP;
3556 		bp->dev->max_mtu =
3557 			min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3558 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3559 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3560 		bp->rx_dir = DMA_BIDIRECTIONAL;
3561 		bp->rx_skb_func = bnxt_rx_page_skb;
3562 		/* Disable LRO or GRO_HW */
3563 		netdev_update_features(bp->dev);
3564 	} else {
3565 		bp->dev->max_mtu = bp->max_mtu;
3566 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3567 		bp->rx_dir = DMA_FROM_DEVICE;
3568 		bp->rx_skb_func = bnxt_rx_skb;
3569 	}
3570 	return 0;
3571 }
3572 
3573 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3574 {
3575 	int i;
3576 	struct bnxt_vnic_info *vnic;
3577 	struct pci_dev *pdev = bp->pdev;
3578 
3579 	if (!bp->vnic_info)
3580 		return;
3581 
3582 	for (i = 0; i < bp->nr_vnics; i++) {
3583 		vnic = &bp->vnic_info[i];
3584 
3585 		kfree(vnic->fw_grp_ids);
3586 		vnic->fw_grp_ids = NULL;
3587 
3588 		kfree(vnic->uc_list);
3589 		vnic->uc_list = NULL;
3590 
3591 		if (vnic->mc_list) {
3592 			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3593 					  vnic->mc_list, vnic->mc_list_mapping);
3594 			vnic->mc_list = NULL;
3595 		}
3596 
3597 		if (vnic->rss_table) {
3598 			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3599 					  vnic->rss_table,
3600 					  vnic->rss_table_dma_addr);
3601 			vnic->rss_table = NULL;
3602 		}
3603 
3604 		vnic->rss_hash_key = NULL;
3605 		vnic->flags = 0;
3606 	}
3607 }
3608 
3609 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3610 {
3611 	int i, rc = 0, size;
3612 	struct bnxt_vnic_info *vnic;
3613 	struct pci_dev *pdev = bp->pdev;
3614 	int max_rings;
3615 
3616 	for (i = 0; i < bp->nr_vnics; i++) {
3617 		vnic = &bp->vnic_info[i];
3618 
3619 		if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3620 			int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3621 
3622 			if (mem_size > 0) {
3623 				vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3624 				if (!vnic->uc_list) {
3625 					rc = -ENOMEM;
3626 					goto out;
3627 				}
3628 			}
3629 		}
3630 
3631 		if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3632 			vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3633 			vnic->mc_list =
3634 				dma_alloc_coherent(&pdev->dev,
3635 						   vnic->mc_list_size,
3636 						   &vnic->mc_list_mapping,
3637 						   GFP_KERNEL);
3638 			if (!vnic->mc_list) {
3639 				rc = -ENOMEM;
3640 				goto out;
3641 			}
3642 		}
3643 
3644 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3645 			goto vnic_skip_grps;
3646 
3647 		if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3648 			max_rings = bp->rx_nr_rings;
3649 		else
3650 			max_rings = 1;
3651 
3652 		vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3653 		if (!vnic->fw_grp_ids) {
3654 			rc = -ENOMEM;
3655 			goto out;
3656 		}
3657 vnic_skip_grps:
3658 		if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3659 		    !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3660 			continue;
3661 
3662 		/* Allocate rss table and hash key */
3663 		size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3664 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3665 			size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3666 
3667 		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3668 		vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3669 						     vnic->rss_table_size,
3670 						     &vnic->rss_table_dma_addr,
3671 						     GFP_KERNEL);
3672 		if (!vnic->rss_table) {
3673 			rc = -ENOMEM;
3674 			goto out;
3675 		}
3676 
3677 		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3678 		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3679 	}
3680 	return 0;
3681 
3682 out:
3683 	return rc;
3684 }
3685 
3686 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3687 {
3688 	struct pci_dev *pdev = bp->pdev;
3689 
3690 	if (bp->hwrm_cmd_resp_addr) {
3691 		dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3692 				  bp->hwrm_cmd_resp_dma_addr);
3693 		bp->hwrm_cmd_resp_addr = NULL;
3694 	}
3695 
3696 	if (bp->hwrm_cmd_kong_resp_addr) {
3697 		dma_free_coherent(&pdev->dev, PAGE_SIZE,
3698 				  bp->hwrm_cmd_kong_resp_addr,
3699 				  bp->hwrm_cmd_kong_resp_dma_addr);
3700 		bp->hwrm_cmd_kong_resp_addr = NULL;
3701 	}
3702 }
3703 
3704 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3705 {
3706 	struct pci_dev *pdev = bp->pdev;
3707 
3708 	if (bp->hwrm_cmd_kong_resp_addr)
3709 		return 0;
3710 
3711 	bp->hwrm_cmd_kong_resp_addr =
3712 		dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3713 				   &bp->hwrm_cmd_kong_resp_dma_addr,
3714 				   GFP_KERNEL);
3715 	if (!bp->hwrm_cmd_kong_resp_addr)
3716 		return -ENOMEM;
3717 
3718 	return 0;
3719 }
3720 
3721 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3722 {
3723 	struct pci_dev *pdev = bp->pdev;
3724 
3725 	bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3726 						   &bp->hwrm_cmd_resp_dma_addr,
3727 						   GFP_KERNEL);
3728 	if (!bp->hwrm_cmd_resp_addr)
3729 		return -ENOMEM;
3730 
3731 	return 0;
3732 }
3733 
3734 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3735 {
3736 	if (bp->hwrm_short_cmd_req_addr) {
3737 		struct pci_dev *pdev = bp->pdev;
3738 
3739 		dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3740 				  bp->hwrm_short_cmd_req_addr,
3741 				  bp->hwrm_short_cmd_req_dma_addr);
3742 		bp->hwrm_short_cmd_req_addr = NULL;
3743 	}
3744 }
3745 
3746 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3747 {
3748 	struct pci_dev *pdev = bp->pdev;
3749 
3750 	if (bp->hwrm_short_cmd_req_addr)
3751 		return 0;
3752 
3753 	bp->hwrm_short_cmd_req_addr =
3754 		dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3755 				   &bp->hwrm_short_cmd_req_dma_addr,
3756 				   GFP_KERNEL);
3757 	if (!bp->hwrm_short_cmd_req_addr)
3758 		return -ENOMEM;
3759 
3760 	return 0;
3761 }
3762 
3763 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3764 {
3765 	kfree(stats->hw_masks);
3766 	stats->hw_masks = NULL;
3767 	kfree(stats->sw_stats);
3768 	stats->sw_stats = NULL;
3769 	if (stats->hw_stats) {
3770 		dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3771 				  stats->hw_stats_map);
3772 		stats->hw_stats = NULL;
3773 	}
3774 }
3775 
3776 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3777 				bool alloc_masks)
3778 {
3779 	stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3780 					     &stats->hw_stats_map, GFP_KERNEL);
3781 	if (!stats->hw_stats)
3782 		return -ENOMEM;
3783 
3784 	stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3785 	if (!stats->sw_stats)
3786 		goto stats_mem_err;
3787 
3788 	if (alloc_masks) {
3789 		stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3790 		if (!stats->hw_masks)
3791 			goto stats_mem_err;
3792 	}
3793 	return 0;
3794 
3795 stats_mem_err:
3796 	bnxt_free_stats_mem(bp, stats);
3797 	return -ENOMEM;
3798 }
3799 
3800 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3801 {
3802 	int i;
3803 
3804 	for (i = 0; i < count; i++)
3805 		mask_arr[i] = mask;
3806 }
3807 
3808 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3809 {
3810 	int i;
3811 
3812 	for (i = 0; i < count; i++)
3813 		mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3814 }
3815 
3816 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3817 				    struct bnxt_stats_mem *stats)
3818 {
3819 	struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3820 	struct hwrm_func_qstats_ext_input req = {0};
3821 	__le64 *hw_masks;
3822 	int rc;
3823 
3824 	if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3825 	    !(bp->flags & BNXT_FLAG_CHIP_P5))
3826 		return -EOPNOTSUPP;
3827 
3828 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3829 	req.fid = cpu_to_le16(0xffff);
3830 	req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3831 	mutex_lock(&bp->hwrm_cmd_lock);
3832 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3833 	if (rc)
3834 		goto qstat_exit;
3835 
3836 	hw_masks = &resp->rx_ucast_pkts;
3837 	bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3838 
3839 qstat_exit:
3840 	mutex_unlock(&bp->hwrm_cmd_lock);
3841 	return rc;
3842 }
3843 
3844 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3845 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3846 
3847 static void bnxt_init_stats(struct bnxt *bp)
3848 {
3849 	struct bnxt_napi *bnapi = bp->bnapi[0];
3850 	struct bnxt_cp_ring_info *cpr;
3851 	struct bnxt_stats_mem *stats;
3852 	__le64 *rx_stats, *tx_stats;
3853 	int rc, rx_count, tx_count;
3854 	u64 *rx_masks, *tx_masks;
3855 	u64 mask;
3856 	u8 flags;
3857 
3858 	cpr = &bnapi->cp_ring;
3859 	stats = &cpr->stats;
3860 	rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3861 	if (rc) {
3862 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3863 			mask = (1ULL << 48) - 1;
3864 		else
3865 			mask = -1ULL;
3866 		bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3867 	}
3868 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
3869 		stats = &bp->port_stats;
3870 		rx_stats = stats->hw_stats;
3871 		rx_masks = stats->hw_masks;
3872 		rx_count = sizeof(struct rx_port_stats) / 8;
3873 		tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3874 		tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3875 		tx_count = sizeof(struct tx_port_stats) / 8;
3876 
3877 		flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3878 		rc = bnxt_hwrm_port_qstats(bp, flags);
3879 		if (rc) {
3880 			mask = (1ULL << 40) - 1;
3881 
3882 			bnxt_fill_masks(rx_masks, mask, rx_count);
3883 			bnxt_fill_masks(tx_masks, mask, tx_count);
3884 		} else {
3885 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3886 			bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3887 			bnxt_hwrm_port_qstats(bp, 0);
3888 		}
3889 	}
3890 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3891 		stats = &bp->rx_port_stats_ext;
3892 		rx_stats = stats->hw_stats;
3893 		rx_masks = stats->hw_masks;
3894 		rx_count = sizeof(struct rx_port_stats_ext) / 8;
3895 		stats = &bp->tx_port_stats_ext;
3896 		tx_stats = stats->hw_stats;
3897 		tx_masks = stats->hw_masks;
3898 		tx_count = sizeof(struct tx_port_stats_ext) / 8;
3899 
3900 		flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3901 		rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3902 		if (rc) {
3903 			mask = (1ULL << 40) - 1;
3904 
3905 			bnxt_fill_masks(rx_masks, mask, rx_count);
3906 			if (tx_stats)
3907 				bnxt_fill_masks(tx_masks, mask, tx_count);
3908 		} else {
3909 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3910 			if (tx_stats)
3911 				bnxt_copy_hw_masks(tx_masks, tx_stats,
3912 						   tx_count);
3913 			bnxt_hwrm_port_qstats_ext(bp, 0);
3914 		}
3915 	}
3916 }
3917 
3918 static void bnxt_free_port_stats(struct bnxt *bp)
3919 {
3920 	bp->flags &= ~BNXT_FLAG_PORT_STATS;
3921 	bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3922 
3923 	bnxt_free_stats_mem(bp, &bp->port_stats);
3924 	bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3925 	bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3926 }
3927 
3928 static void bnxt_free_ring_stats(struct bnxt *bp)
3929 {
3930 	int i;
3931 
3932 	if (!bp->bnapi)
3933 		return;
3934 
3935 	for (i = 0; i < bp->cp_nr_rings; i++) {
3936 		struct bnxt_napi *bnapi = bp->bnapi[i];
3937 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3938 
3939 		bnxt_free_stats_mem(bp, &cpr->stats);
3940 	}
3941 }
3942 
3943 static int bnxt_alloc_stats(struct bnxt *bp)
3944 {
3945 	u32 size, i;
3946 	int rc;
3947 
3948 	size = bp->hw_ring_stats_size;
3949 
3950 	for (i = 0; i < bp->cp_nr_rings; i++) {
3951 		struct bnxt_napi *bnapi = bp->bnapi[i];
3952 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3953 
3954 		cpr->stats.len = size;
3955 		rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3956 		if (rc)
3957 			return rc;
3958 
3959 		cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3960 	}
3961 
3962 	if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3963 		return 0;
3964 
3965 	if (bp->port_stats.hw_stats)
3966 		goto alloc_ext_stats;
3967 
3968 	bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3969 	rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3970 	if (rc)
3971 		return rc;
3972 
3973 	bp->flags |= BNXT_FLAG_PORT_STATS;
3974 
3975 alloc_ext_stats:
3976 	/* Display extended statistics only if FW supports it */
3977 	if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3978 		if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3979 			return 0;
3980 
3981 	if (bp->rx_port_stats_ext.hw_stats)
3982 		goto alloc_tx_ext_stats;
3983 
3984 	bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3985 	rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3986 	/* Extended stats are optional */
3987 	if (rc)
3988 		return 0;
3989 
3990 alloc_tx_ext_stats:
3991 	if (bp->tx_port_stats_ext.hw_stats)
3992 		return 0;
3993 
3994 	if (bp->hwrm_spec_code >= 0x10902 ||
3995 	    (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3996 		bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3997 		rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3998 		/* Extended stats are optional */
3999 		if (rc)
4000 			return 0;
4001 	}
4002 	bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
4003 	return 0;
4004 }
4005 
4006 static void bnxt_clear_ring_indices(struct bnxt *bp)
4007 {
4008 	int i;
4009 
4010 	if (!bp->bnapi)
4011 		return;
4012 
4013 	for (i = 0; i < bp->cp_nr_rings; i++) {
4014 		struct bnxt_napi *bnapi = bp->bnapi[i];
4015 		struct bnxt_cp_ring_info *cpr;
4016 		struct bnxt_rx_ring_info *rxr;
4017 		struct bnxt_tx_ring_info *txr;
4018 
4019 		if (!bnapi)
4020 			continue;
4021 
4022 		cpr = &bnapi->cp_ring;
4023 		cpr->cp_raw_cons = 0;
4024 
4025 		txr = bnapi->tx_ring;
4026 		if (txr) {
4027 			txr->tx_prod = 0;
4028 			txr->tx_cons = 0;
4029 		}
4030 
4031 		rxr = bnapi->rx_ring;
4032 		if (rxr) {
4033 			rxr->rx_prod = 0;
4034 			rxr->rx_agg_prod = 0;
4035 			rxr->rx_sw_agg_prod = 0;
4036 			rxr->rx_next_cons = 0;
4037 		}
4038 	}
4039 }
4040 
4041 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
4042 {
4043 #ifdef CONFIG_RFS_ACCEL
4044 	int i;
4045 
4046 	/* Under rtnl_lock and all our NAPIs have been disabled.  It's
4047 	 * safe to delete the hash table.
4048 	 */
4049 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4050 		struct hlist_head *head;
4051 		struct hlist_node *tmp;
4052 		struct bnxt_ntuple_filter *fltr;
4053 
4054 		head = &bp->ntp_fltr_hash_tbl[i];
4055 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4056 			hlist_del(&fltr->hash);
4057 			kfree(fltr);
4058 		}
4059 	}
4060 	if (irq_reinit) {
4061 		kfree(bp->ntp_fltr_bmap);
4062 		bp->ntp_fltr_bmap = NULL;
4063 	}
4064 	bp->ntp_fltr_count = 0;
4065 #endif
4066 }
4067 
4068 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4069 {
4070 #ifdef CONFIG_RFS_ACCEL
4071 	int i, rc = 0;
4072 
4073 	if (!(bp->flags & BNXT_FLAG_RFS))
4074 		return 0;
4075 
4076 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4077 		INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4078 
4079 	bp->ntp_fltr_count = 0;
4080 	bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4081 				    sizeof(long),
4082 				    GFP_KERNEL);
4083 
4084 	if (!bp->ntp_fltr_bmap)
4085 		rc = -ENOMEM;
4086 
4087 	return rc;
4088 #else
4089 	return 0;
4090 #endif
4091 }
4092 
4093 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4094 {
4095 	bnxt_free_vnic_attributes(bp);
4096 	bnxt_free_tx_rings(bp);
4097 	bnxt_free_rx_rings(bp);
4098 	bnxt_free_cp_rings(bp);
4099 	bnxt_free_ntp_fltrs(bp, irq_re_init);
4100 	if (irq_re_init) {
4101 		bnxt_free_ring_stats(bp);
4102 		if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET))
4103 			bnxt_free_port_stats(bp);
4104 		bnxt_free_ring_grps(bp);
4105 		bnxt_free_vnics(bp);
4106 		kfree(bp->tx_ring_map);
4107 		bp->tx_ring_map = NULL;
4108 		kfree(bp->tx_ring);
4109 		bp->tx_ring = NULL;
4110 		kfree(bp->rx_ring);
4111 		bp->rx_ring = NULL;
4112 		kfree(bp->bnapi);
4113 		bp->bnapi = NULL;
4114 	} else {
4115 		bnxt_clear_ring_indices(bp);
4116 	}
4117 }
4118 
4119 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4120 {
4121 	int i, j, rc, size, arr_size;
4122 	void *bnapi;
4123 
4124 	if (irq_re_init) {
4125 		/* Allocate bnapi mem pointer array and mem block for
4126 		 * all queues
4127 		 */
4128 		arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4129 				bp->cp_nr_rings);
4130 		size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4131 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4132 		if (!bnapi)
4133 			return -ENOMEM;
4134 
4135 		bp->bnapi = bnapi;
4136 		bnapi += arr_size;
4137 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4138 			bp->bnapi[i] = bnapi;
4139 			bp->bnapi[i]->index = i;
4140 			bp->bnapi[i]->bp = bp;
4141 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4142 				struct bnxt_cp_ring_info *cpr =
4143 					&bp->bnapi[i]->cp_ring;
4144 
4145 				cpr->cp_ring_struct.ring_mem.flags =
4146 					BNXT_RMEM_RING_PTE_FLAG;
4147 			}
4148 		}
4149 
4150 		bp->rx_ring = kcalloc(bp->rx_nr_rings,
4151 				      sizeof(struct bnxt_rx_ring_info),
4152 				      GFP_KERNEL);
4153 		if (!bp->rx_ring)
4154 			return -ENOMEM;
4155 
4156 		for (i = 0; i < bp->rx_nr_rings; i++) {
4157 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4158 
4159 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4160 				rxr->rx_ring_struct.ring_mem.flags =
4161 					BNXT_RMEM_RING_PTE_FLAG;
4162 				rxr->rx_agg_ring_struct.ring_mem.flags =
4163 					BNXT_RMEM_RING_PTE_FLAG;
4164 			}
4165 			rxr->bnapi = bp->bnapi[i];
4166 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4167 		}
4168 
4169 		bp->tx_ring = kcalloc(bp->tx_nr_rings,
4170 				      sizeof(struct bnxt_tx_ring_info),
4171 				      GFP_KERNEL);
4172 		if (!bp->tx_ring)
4173 			return -ENOMEM;
4174 
4175 		bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4176 					  GFP_KERNEL);
4177 
4178 		if (!bp->tx_ring_map)
4179 			return -ENOMEM;
4180 
4181 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4182 			j = 0;
4183 		else
4184 			j = bp->rx_nr_rings;
4185 
4186 		for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4187 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4188 
4189 			if (bp->flags & BNXT_FLAG_CHIP_P5)
4190 				txr->tx_ring_struct.ring_mem.flags =
4191 					BNXT_RMEM_RING_PTE_FLAG;
4192 			txr->bnapi = bp->bnapi[j];
4193 			bp->bnapi[j]->tx_ring = txr;
4194 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4195 			if (i >= bp->tx_nr_rings_xdp) {
4196 				txr->txq_index = i - bp->tx_nr_rings_xdp;
4197 				bp->bnapi[j]->tx_int = bnxt_tx_int;
4198 			} else {
4199 				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4200 				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4201 			}
4202 		}
4203 
4204 		rc = bnxt_alloc_stats(bp);
4205 		if (rc)
4206 			goto alloc_mem_err;
4207 		bnxt_init_stats(bp);
4208 
4209 		rc = bnxt_alloc_ntp_fltrs(bp);
4210 		if (rc)
4211 			goto alloc_mem_err;
4212 
4213 		rc = bnxt_alloc_vnics(bp);
4214 		if (rc)
4215 			goto alloc_mem_err;
4216 	}
4217 
4218 	bnxt_init_ring_struct(bp);
4219 
4220 	rc = bnxt_alloc_rx_rings(bp);
4221 	if (rc)
4222 		goto alloc_mem_err;
4223 
4224 	rc = bnxt_alloc_tx_rings(bp);
4225 	if (rc)
4226 		goto alloc_mem_err;
4227 
4228 	rc = bnxt_alloc_cp_rings(bp);
4229 	if (rc)
4230 		goto alloc_mem_err;
4231 
4232 	bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4233 				  BNXT_VNIC_UCAST_FLAG;
4234 	rc = bnxt_alloc_vnic_attributes(bp);
4235 	if (rc)
4236 		goto alloc_mem_err;
4237 	return 0;
4238 
4239 alloc_mem_err:
4240 	bnxt_free_mem(bp, true);
4241 	return rc;
4242 }
4243 
4244 static void bnxt_disable_int(struct bnxt *bp)
4245 {
4246 	int i;
4247 
4248 	if (!bp->bnapi)
4249 		return;
4250 
4251 	for (i = 0; i < bp->cp_nr_rings; i++) {
4252 		struct bnxt_napi *bnapi = bp->bnapi[i];
4253 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4254 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4255 
4256 		if (ring->fw_ring_id != INVALID_HW_RING_ID)
4257 			bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4258 	}
4259 }
4260 
4261 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4262 {
4263 	struct bnxt_napi *bnapi = bp->bnapi[n];
4264 	struct bnxt_cp_ring_info *cpr;
4265 
4266 	cpr = &bnapi->cp_ring;
4267 	return cpr->cp_ring_struct.map_idx;
4268 }
4269 
4270 static void bnxt_disable_int_sync(struct bnxt *bp)
4271 {
4272 	int i;
4273 
4274 	atomic_inc(&bp->intr_sem);
4275 
4276 	bnxt_disable_int(bp);
4277 	for (i = 0; i < bp->cp_nr_rings; i++) {
4278 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4279 
4280 		synchronize_irq(bp->irq_tbl[map_idx].vector);
4281 	}
4282 }
4283 
4284 static void bnxt_enable_int(struct bnxt *bp)
4285 {
4286 	int i;
4287 
4288 	atomic_set(&bp->intr_sem, 0);
4289 	for (i = 0; i < bp->cp_nr_rings; i++) {
4290 		struct bnxt_napi *bnapi = bp->bnapi[i];
4291 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4292 
4293 		bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4294 	}
4295 }
4296 
4297 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4298 			    u16 cmpl_ring, u16 target_id)
4299 {
4300 	struct input *req = request;
4301 
4302 	req->req_type = cpu_to_le16(req_type);
4303 	req->cmpl_ring = cpu_to_le16(cmpl_ring);
4304 	req->target_id = cpu_to_le16(target_id);
4305 	if (bnxt_kong_hwrm_message(bp, req))
4306 		req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4307 	else
4308 		req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4309 }
4310 
4311 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4312 {
4313 	switch (hwrm_err) {
4314 	case HWRM_ERR_CODE_SUCCESS:
4315 		return 0;
4316 	case HWRM_ERR_CODE_RESOURCE_LOCKED:
4317 		return -EROFS;
4318 	case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4319 		return -EACCES;
4320 	case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4321 		return -ENOSPC;
4322 	case HWRM_ERR_CODE_INVALID_PARAMS:
4323 	case HWRM_ERR_CODE_INVALID_FLAGS:
4324 	case HWRM_ERR_CODE_INVALID_ENABLES:
4325 	case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4326 	case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4327 		return -EINVAL;
4328 	case HWRM_ERR_CODE_NO_BUFFER:
4329 		return -ENOMEM;
4330 	case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4331 	case HWRM_ERR_CODE_BUSY:
4332 		return -EAGAIN;
4333 	case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4334 		return -EOPNOTSUPP;
4335 	default:
4336 		return -EIO;
4337 	}
4338 }
4339 
4340 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4341 				 int timeout, bool silent)
4342 {
4343 	int i, intr_process, rc, tmo_count;
4344 	struct input *req = msg;
4345 	u32 *data = msg;
4346 	u8 *valid;
4347 	u16 cp_ring_id, len = 0;
4348 	struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4349 	u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4350 	struct hwrm_short_input short_input = {0};
4351 	u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4352 	u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4353 	u16 dst = BNXT_HWRM_CHNL_CHIMP;
4354 
4355 	if (BNXT_NO_FW_ACCESS(bp) &&
4356 	    le16_to_cpu(req->req_type) != HWRM_FUNC_RESET)
4357 		return -EBUSY;
4358 
4359 	if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4360 		if (msg_len > bp->hwrm_max_ext_req_len ||
4361 		    !bp->hwrm_short_cmd_req_addr)
4362 			return -EINVAL;
4363 	}
4364 
4365 	if (bnxt_hwrm_kong_chnl(bp, req)) {
4366 		dst = BNXT_HWRM_CHNL_KONG;
4367 		bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4368 		doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4369 		resp = bp->hwrm_cmd_kong_resp_addr;
4370 	}
4371 
4372 	memset(resp, 0, PAGE_SIZE);
4373 	cp_ring_id = le16_to_cpu(req->cmpl_ring);
4374 	intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4375 
4376 	req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4377 	/* currently supports only one outstanding message */
4378 	if (intr_process)
4379 		bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4380 
4381 	if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4382 	    msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4383 		void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4384 		u16 max_msg_len;
4385 
4386 		/* Set boundary for maximum extended request length for short
4387 		 * cmd format. If passed up from device use the max supported
4388 		 * internal req length.
4389 		 */
4390 		max_msg_len = bp->hwrm_max_ext_req_len;
4391 
4392 		memcpy(short_cmd_req, req, msg_len);
4393 		if (msg_len < max_msg_len)
4394 			memset(short_cmd_req + msg_len, 0,
4395 			       max_msg_len - msg_len);
4396 
4397 		short_input.req_type = req->req_type;
4398 		short_input.signature =
4399 				cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4400 		short_input.size = cpu_to_le16(msg_len);
4401 		short_input.req_addr =
4402 			cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4403 
4404 		data = (u32 *)&short_input;
4405 		msg_len = sizeof(short_input);
4406 
4407 		/* Sync memory write before updating doorbell */
4408 		wmb();
4409 
4410 		max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4411 	}
4412 
4413 	/* Write request msg to hwrm channel */
4414 	__iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4415 
4416 	for (i = msg_len; i < max_req_len; i += 4)
4417 		writel(0, bp->bar0 + bar_offset + i);
4418 
4419 	/* Ring channel doorbell */
4420 	writel(1, bp->bar0 + doorbell_offset);
4421 
4422 	if (!pci_is_enabled(bp->pdev))
4423 		return 0;
4424 
4425 	if (!timeout)
4426 		timeout = DFLT_HWRM_CMD_TIMEOUT;
4427 	/* convert timeout to usec */
4428 	timeout *= 1000;
4429 
4430 	i = 0;
4431 	/* Short timeout for the first few iterations:
4432 	 * number of loops = number of loops for short timeout +
4433 	 * number of loops for standard timeout.
4434 	 */
4435 	tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4436 	timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4437 	tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4438 
4439 	if (intr_process) {
4440 		u16 seq_id = bp->hwrm_intr_seq_id;
4441 
4442 		/* Wait until hwrm response cmpl interrupt is processed */
4443 		while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4444 		       i++ < tmo_count) {
4445 			/* Abort the wait for completion if the FW health
4446 			 * check has failed.
4447 			 */
4448 			if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4449 				return -EBUSY;
4450 			/* on first few passes, just barely sleep */
4451 			if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4452 				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4453 					     HWRM_SHORT_MAX_TIMEOUT);
4454 			else
4455 				usleep_range(HWRM_MIN_TIMEOUT,
4456 					     HWRM_MAX_TIMEOUT);
4457 		}
4458 
4459 		if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4460 			if (!silent)
4461 				netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4462 					   le16_to_cpu(req->req_type));
4463 			return -EBUSY;
4464 		}
4465 		len = le16_to_cpu(resp->resp_len);
4466 		valid = ((u8 *)resp) + len - 1;
4467 	} else {
4468 		int j;
4469 
4470 		/* Check if response len is updated */
4471 		for (i = 0; i < tmo_count; i++) {
4472 			/* Abort the wait for completion if the FW health
4473 			 * check has failed.
4474 			 */
4475 			if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4476 				return -EBUSY;
4477 			len = le16_to_cpu(resp->resp_len);
4478 			if (len)
4479 				break;
4480 			/* on first few passes, just barely sleep */
4481 			if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4482 				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4483 					     HWRM_SHORT_MAX_TIMEOUT);
4484 			else
4485 				usleep_range(HWRM_MIN_TIMEOUT,
4486 					     HWRM_MAX_TIMEOUT);
4487 		}
4488 
4489 		if (i >= tmo_count) {
4490 			if (!silent)
4491 				netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4492 					   HWRM_TOTAL_TIMEOUT(i),
4493 					   le16_to_cpu(req->req_type),
4494 					   le16_to_cpu(req->seq_id), len);
4495 			return -EBUSY;
4496 		}
4497 
4498 		/* Last byte of resp contains valid bit */
4499 		valid = ((u8 *)resp) + len - 1;
4500 		for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4501 			/* make sure we read from updated DMA memory */
4502 			dma_rmb();
4503 			if (*valid)
4504 				break;
4505 			usleep_range(1, 5);
4506 		}
4507 
4508 		if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4509 			if (!silent)
4510 				netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4511 					   HWRM_TOTAL_TIMEOUT(i),
4512 					   le16_to_cpu(req->req_type),
4513 					   le16_to_cpu(req->seq_id), len,
4514 					   *valid);
4515 			return -EBUSY;
4516 		}
4517 	}
4518 
4519 	/* Zero valid bit for compatibility.  Valid bit in an older spec
4520 	 * may become a new field in a newer spec.  We must make sure that
4521 	 * a new field not implemented by old spec will read zero.
4522 	 */
4523 	*valid = 0;
4524 	rc = le16_to_cpu(resp->error_code);
4525 	if (rc && !silent)
4526 		netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4527 			   le16_to_cpu(resp->req_type),
4528 			   le16_to_cpu(resp->seq_id), rc);
4529 	return bnxt_hwrm_to_stderr(rc);
4530 }
4531 
4532 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4533 {
4534 	return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4535 }
4536 
4537 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4538 			      int timeout)
4539 {
4540 	return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4541 }
4542 
4543 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4544 {
4545 	int rc;
4546 
4547 	mutex_lock(&bp->hwrm_cmd_lock);
4548 	rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4549 	mutex_unlock(&bp->hwrm_cmd_lock);
4550 	return rc;
4551 }
4552 
4553 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4554 			     int timeout)
4555 {
4556 	int rc;
4557 
4558 	mutex_lock(&bp->hwrm_cmd_lock);
4559 	rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4560 	mutex_unlock(&bp->hwrm_cmd_lock);
4561 	return rc;
4562 }
4563 
4564 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4565 			    bool async_only)
4566 {
4567 	struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4568 	struct hwrm_func_drv_rgtr_input req = {0};
4569 	DECLARE_BITMAP(async_events_bmap, 256);
4570 	u32 *events = (u32 *)async_events_bmap;
4571 	u32 flags;
4572 	int rc, i;
4573 
4574 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4575 
4576 	req.enables =
4577 		cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4578 			    FUNC_DRV_RGTR_REQ_ENABLES_VER |
4579 			    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4580 
4581 	req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4582 	flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4583 	if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4584 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4585 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4586 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4587 			 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4588 	req.flags = cpu_to_le32(flags);
4589 	req.ver_maj_8b = DRV_VER_MAJ;
4590 	req.ver_min_8b = DRV_VER_MIN;
4591 	req.ver_upd_8b = DRV_VER_UPD;
4592 	req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4593 	req.ver_min = cpu_to_le16(DRV_VER_MIN);
4594 	req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4595 
4596 	if (BNXT_PF(bp)) {
4597 		u32 data[8];
4598 		int i;
4599 
4600 		memset(data, 0, sizeof(data));
4601 		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4602 			u16 cmd = bnxt_vf_req_snif[i];
4603 			unsigned int bit, idx;
4604 
4605 			idx = cmd / 32;
4606 			bit = cmd % 32;
4607 			data[idx] |= 1 << bit;
4608 		}
4609 
4610 		for (i = 0; i < 8; i++)
4611 			req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4612 
4613 		req.enables |=
4614 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4615 	}
4616 
4617 	if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4618 		req.flags |= cpu_to_le32(
4619 			FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4620 
4621 	memset(async_events_bmap, 0, sizeof(async_events_bmap));
4622 	for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4623 		u16 event_id = bnxt_async_events_arr[i];
4624 
4625 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4626 		    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4627 			continue;
4628 		__set_bit(bnxt_async_events_arr[i], async_events_bmap);
4629 	}
4630 	if (bmap && bmap_size) {
4631 		for (i = 0; i < bmap_size; i++) {
4632 			if (test_bit(i, bmap))
4633 				__set_bit(i, async_events_bmap);
4634 		}
4635 	}
4636 	for (i = 0; i < 8; i++)
4637 		req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4638 
4639 	if (async_only)
4640 		req.enables =
4641 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4642 
4643 	mutex_lock(&bp->hwrm_cmd_lock);
4644 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4645 	if (!rc) {
4646 		set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4647 		if (resp->flags &
4648 		    cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4649 			bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4650 	}
4651 	mutex_unlock(&bp->hwrm_cmd_lock);
4652 	return rc;
4653 }
4654 
4655 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4656 {
4657 	struct hwrm_func_drv_unrgtr_input req = {0};
4658 
4659 	if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4660 		return 0;
4661 
4662 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4663 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4664 }
4665 
4666 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4667 {
4668 	u32 rc = 0;
4669 	struct hwrm_tunnel_dst_port_free_input req = {0};
4670 
4671 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4672 	req.tunnel_type = tunnel_type;
4673 
4674 	switch (tunnel_type) {
4675 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4676 		req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4677 		bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4678 		break;
4679 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4680 		req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4681 		bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4682 		break;
4683 	default:
4684 		break;
4685 	}
4686 
4687 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4688 	if (rc)
4689 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4690 			   rc);
4691 	return rc;
4692 }
4693 
4694 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4695 					   u8 tunnel_type)
4696 {
4697 	u32 rc = 0;
4698 	struct hwrm_tunnel_dst_port_alloc_input req = {0};
4699 	struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4700 
4701 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4702 
4703 	req.tunnel_type = tunnel_type;
4704 	req.tunnel_dst_port_val = port;
4705 
4706 	mutex_lock(&bp->hwrm_cmd_lock);
4707 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4708 	if (rc) {
4709 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4710 			   rc);
4711 		goto err_out;
4712 	}
4713 
4714 	switch (tunnel_type) {
4715 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4716 		bp->vxlan_fw_dst_port_id =
4717 			le16_to_cpu(resp->tunnel_dst_port_id);
4718 		break;
4719 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4720 		bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4721 		break;
4722 	default:
4723 		break;
4724 	}
4725 
4726 err_out:
4727 	mutex_unlock(&bp->hwrm_cmd_lock);
4728 	return rc;
4729 }
4730 
4731 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4732 {
4733 	struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4734 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4735 
4736 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4737 	req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4738 
4739 	req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4740 	req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4741 	req.mask = cpu_to_le32(vnic->rx_mask);
4742 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4743 }
4744 
4745 #ifdef CONFIG_RFS_ACCEL
4746 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4747 					    struct bnxt_ntuple_filter *fltr)
4748 {
4749 	struct hwrm_cfa_ntuple_filter_free_input req = {0};
4750 
4751 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4752 	req.ntuple_filter_id = fltr->filter_id;
4753 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4754 }
4755 
4756 #define BNXT_NTP_FLTR_FLAGS					\
4757 	(CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |	\
4758 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |	\
4759 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |	\
4760 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |	\
4761 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |	\
4762 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |	\
4763 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |	\
4764 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |	\
4765 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |	\
4766 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |		\
4767 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |	\
4768 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |		\
4769 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |	\
4770 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4771 
4772 #define BNXT_NTP_TUNNEL_FLTR_FLAG				\
4773 		CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4774 
4775 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4776 					     struct bnxt_ntuple_filter *fltr)
4777 {
4778 	struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4779 	struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4780 	struct flow_keys *keys = &fltr->fkeys;
4781 	struct bnxt_vnic_info *vnic;
4782 	u32 flags = 0;
4783 	int rc = 0;
4784 
4785 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4786 	req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4787 
4788 	if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4789 		flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4790 		req.dst_id = cpu_to_le16(fltr->rxq);
4791 	} else {
4792 		vnic = &bp->vnic_info[fltr->rxq + 1];
4793 		req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4794 	}
4795 	req.flags = cpu_to_le32(flags);
4796 	req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4797 
4798 	req.ethertype = htons(ETH_P_IP);
4799 	memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4800 	req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4801 	req.ip_protocol = keys->basic.ip_proto;
4802 
4803 	if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4804 		int i;
4805 
4806 		req.ethertype = htons(ETH_P_IPV6);
4807 		req.ip_addr_type =
4808 			CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4809 		*(struct in6_addr *)&req.src_ipaddr[0] =
4810 			keys->addrs.v6addrs.src;
4811 		*(struct in6_addr *)&req.dst_ipaddr[0] =
4812 			keys->addrs.v6addrs.dst;
4813 		for (i = 0; i < 4; i++) {
4814 			req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4815 			req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4816 		}
4817 	} else {
4818 		req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4819 		req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4820 		req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4821 		req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4822 	}
4823 	if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4824 		req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4825 		req.tunnel_type =
4826 			CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4827 	}
4828 
4829 	req.src_port = keys->ports.src;
4830 	req.src_port_mask = cpu_to_be16(0xffff);
4831 	req.dst_port = keys->ports.dst;
4832 	req.dst_port_mask = cpu_to_be16(0xffff);
4833 
4834 	mutex_lock(&bp->hwrm_cmd_lock);
4835 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4836 	if (!rc) {
4837 		resp = bnxt_get_hwrm_resp_addr(bp, &req);
4838 		fltr->filter_id = resp->ntuple_filter_id;
4839 	}
4840 	mutex_unlock(&bp->hwrm_cmd_lock);
4841 	return rc;
4842 }
4843 #endif
4844 
4845 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4846 				     u8 *mac_addr)
4847 {
4848 	u32 rc = 0;
4849 	struct hwrm_cfa_l2_filter_alloc_input req = {0};
4850 	struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4851 
4852 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4853 	req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4854 	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4855 		req.flags |=
4856 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4857 	req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4858 	req.enables =
4859 		cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4860 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4861 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4862 	memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4863 	req.l2_addr_mask[0] = 0xff;
4864 	req.l2_addr_mask[1] = 0xff;
4865 	req.l2_addr_mask[2] = 0xff;
4866 	req.l2_addr_mask[3] = 0xff;
4867 	req.l2_addr_mask[4] = 0xff;
4868 	req.l2_addr_mask[5] = 0xff;
4869 
4870 	mutex_lock(&bp->hwrm_cmd_lock);
4871 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4872 	if (!rc)
4873 		bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4874 							resp->l2_filter_id;
4875 	mutex_unlock(&bp->hwrm_cmd_lock);
4876 	return rc;
4877 }
4878 
4879 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4880 {
4881 	u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4882 	int rc = 0;
4883 
4884 	/* Any associated ntuple filters will also be cleared by firmware. */
4885 	mutex_lock(&bp->hwrm_cmd_lock);
4886 	for (i = 0; i < num_of_vnics; i++) {
4887 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4888 
4889 		for (j = 0; j < vnic->uc_filter_count; j++) {
4890 			struct hwrm_cfa_l2_filter_free_input req = {0};
4891 
4892 			bnxt_hwrm_cmd_hdr_init(bp, &req,
4893 					       HWRM_CFA_L2_FILTER_FREE, -1, -1);
4894 
4895 			req.l2_filter_id = vnic->fw_l2_filter_id[j];
4896 
4897 			rc = _hwrm_send_message(bp, &req, sizeof(req),
4898 						HWRM_CMD_TIMEOUT);
4899 		}
4900 		vnic->uc_filter_count = 0;
4901 	}
4902 	mutex_unlock(&bp->hwrm_cmd_lock);
4903 
4904 	return rc;
4905 }
4906 
4907 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4908 {
4909 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4910 	u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4911 	struct hwrm_vnic_tpa_cfg_input req = {0};
4912 
4913 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4914 		return 0;
4915 
4916 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4917 
4918 	if (tpa_flags) {
4919 		u16 mss = bp->dev->mtu - 40;
4920 		u32 nsegs, n, segs = 0, flags;
4921 
4922 		flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4923 			VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4924 			VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4925 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4926 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4927 		if (tpa_flags & BNXT_FLAG_GRO)
4928 			flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4929 
4930 		req.flags = cpu_to_le32(flags);
4931 
4932 		req.enables =
4933 			cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4934 				    VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4935 				    VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4936 
4937 		/* Number of segs are log2 units, and first packet is not
4938 		 * included as part of this units.
4939 		 */
4940 		if (mss <= BNXT_RX_PAGE_SIZE) {
4941 			n = BNXT_RX_PAGE_SIZE / mss;
4942 			nsegs = (MAX_SKB_FRAGS - 1) * n;
4943 		} else {
4944 			n = mss / BNXT_RX_PAGE_SIZE;
4945 			if (mss & (BNXT_RX_PAGE_SIZE - 1))
4946 				n++;
4947 			nsegs = (MAX_SKB_FRAGS - n) / n;
4948 		}
4949 
4950 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
4951 			segs = MAX_TPA_SEGS_P5;
4952 			max_aggs = bp->max_tpa;
4953 		} else {
4954 			segs = ilog2(nsegs);
4955 		}
4956 		req.max_agg_segs = cpu_to_le16(segs);
4957 		req.max_aggs = cpu_to_le16(max_aggs);
4958 
4959 		req.min_agg_len = cpu_to_le32(512);
4960 	}
4961 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4962 
4963 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4964 }
4965 
4966 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4967 {
4968 	struct bnxt_ring_grp_info *grp_info;
4969 
4970 	grp_info = &bp->grp_info[ring->grp_idx];
4971 	return grp_info->cp_fw_ring_id;
4972 }
4973 
4974 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4975 {
4976 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
4977 		struct bnxt_napi *bnapi = rxr->bnapi;
4978 		struct bnxt_cp_ring_info *cpr;
4979 
4980 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4981 		return cpr->cp_ring_struct.fw_ring_id;
4982 	} else {
4983 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4984 	}
4985 }
4986 
4987 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4988 {
4989 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
4990 		struct bnxt_napi *bnapi = txr->bnapi;
4991 		struct bnxt_cp_ring_info *cpr;
4992 
4993 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4994 		return cpr->cp_ring_struct.fw_ring_id;
4995 	} else {
4996 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4997 	}
4998 }
4999 
5000 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
5001 {
5002 	int entries;
5003 
5004 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5005 		entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
5006 	else
5007 		entries = HW_HASH_INDEX_SIZE;
5008 
5009 	bp->rss_indir_tbl_entries = entries;
5010 	bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
5011 					  GFP_KERNEL);
5012 	if (!bp->rss_indir_tbl)
5013 		return -ENOMEM;
5014 	return 0;
5015 }
5016 
5017 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
5018 {
5019 	u16 max_rings, max_entries, pad, i;
5020 
5021 	if (!bp->rx_nr_rings)
5022 		return;
5023 
5024 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5025 		max_rings = bp->rx_nr_rings - 1;
5026 	else
5027 		max_rings = bp->rx_nr_rings;
5028 
5029 	max_entries = bnxt_get_rxfh_indir_size(bp->dev);
5030 
5031 	for (i = 0; i < max_entries; i++)
5032 		bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
5033 
5034 	pad = bp->rss_indir_tbl_entries - max_entries;
5035 	if (pad)
5036 		memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
5037 }
5038 
5039 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
5040 {
5041 	u16 i, tbl_size, max_ring = 0;
5042 
5043 	if (!bp->rss_indir_tbl)
5044 		return 0;
5045 
5046 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5047 	for (i = 0; i < tbl_size; i++)
5048 		max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5049 	return max_ring;
5050 }
5051 
5052 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5053 {
5054 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5055 		return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5056 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5057 		return 2;
5058 	return 1;
5059 }
5060 
5061 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5062 {
5063 	bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5064 	u16 i, j;
5065 
5066 	/* Fill the RSS indirection table with ring group ids */
5067 	for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5068 		if (!no_rss)
5069 			j = bp->rss_indir_tbl[i];
5070 		vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5071 	}
5072 }
5073 
5074 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5075 				      struct bnxt_vnic_info *vnic)
5076 {
5077 	__le16 *ring_tbl = vnic->rss_table;
5078 	struct bnxt_rx_ring_info *rxr;
5079 	u16 tbl_size, i;
5080 
5081 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5082 
5083 	for (i = 0; i < tbl_size; i++) {
5084 		u16 ring_id, j;
5085 
5086 		j = bp->rss_indir_tbl[i];
5087 		rxr = &bp->rx_ring[j];
5088 
5089 		ring_id = rxr->rx_ring_struct.fw_ring_id;
5090 		*ring_tbl++ = cpu_to_le16(ring_id);
5091 		ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5092 		*ring_tbl++ = cpu_to_le16(ring_id);
5093 	}
5094 }
5095 
5096 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5097 {
5098 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5099 		__bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5100 	else
5101 		__bnxt_fill_hw_rss_tbl(bp, vnic);
5102 }
5103 
5104 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5105 {
5106 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5107 	struct hwrm_vnic_rss_cfg_input req = {0};
5108 
5109 	if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5110 	    vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5111 		return 0;
5112 
5113 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5114 	if (set_rss) {
5115 		bnxt_fill_hw_rss_tbl(bp, vnic);
5116 		req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5117 		req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5118 		req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5119 		req.hash_key_tbl_addr =
5120 			cpu_to_le64(vnic->rss_hash_key_dma_addr);
5121 	}
5122 	req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5123 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5124 }
5125 
5126 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5127 {
5128 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5129 	struct hwrm_vnic_rss_cfg_input req = {0};
5130 	dma_addr_t ring_tbl_map;
5131 	u32 i, nr_ctxs;
5132 
5133 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5134 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5135 	if (!set_rss) {
5136 		hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5137 		return 0;
5138 	}
5139 	bnxt_fill_hw_rss_tbl(bp, vnic);
5140 	req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5141 	req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5142 	req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5143 	ring_tbl_map = vnic->rss_table_dma_addr;
5144 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5145 	for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5146 		int rc;
5147 
5148 		req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5149 		req.ring_table_pair_index = i;
5150 		req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5151 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5152 		if (rc)
5153 			return rc;
5154 	}
5155 	return 0;
5156 }
5157 
5158 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5159 {
5160 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5161 	struct hwrm_vnic_plcmodes_cfg_input req = {0};
5162 
5163 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5164 	req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5165 				VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5166 				VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5167 	req.enables =
5168 		cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5169 			    VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5170 	/* thresholds not implemented in firmware yet */
5171 	req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5172 	req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5173 	req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5174 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5175 }
5176 
5177 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5178 					u16 ctx_idx)
5179 {
5180 	struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5181 
5182 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5183 	req.rss_cos_lb_ctx_id =
5184 		cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5185 
5186 	hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5187 	bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5188 }
5189 
5190 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5191 {
5192 	int i, j;
5193 
5194 	for (i = 0; i < bp->nr_vnics; i++) {
5195 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5196 
5197 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5198 			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5199 				bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5200 		}
5201 	}
5202 	bp->rsscos_nr_ctxs = 0;
5203 }
5204 
5205 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5206 {
5207 	int rc;
5208 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5209 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5210 						bp->hwrm_cmd_resp_addr;
5211 
5212 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5213 			       -1);
5214 
5215 	mutex_lock(&bp->hwrm_cmd_lock);
5216 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5217 	if (!rc)
5218 		bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5219 			le16_to_cpu(resp->rss_cos_lb_ctx_id);
5220 	mutex_unlock(&bp->hwrm_cmd_lock);
5221 
5222 	return rc;
5223 }
5224 
5225 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5226 {
5227 	if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5228 		return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5229 	return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5230 }
5231 
5232 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5233 {
5234 	unsigned int ring = 0, grp_idx;
5235 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5236 	struct hwrm_vnic_cfg_input req = {0};
5237 	u16 def_vlan = 0;
5238 
5239 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5240 
5241 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5242 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5243 
5244 		req.default_rx_ring_id =
5245 			cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5246 		req.default_cmpl_ring_id =
5247 			cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5248 		req.enables =
5249 			cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5250 				    VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5251 		goto vnic_mru;
5252 	}
5253 	req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5254 	/* Only RSS support for now TBD: COS & LB */
5255 	if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5256 		req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5257 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5258 					   VNIC_CFG_REQ_ENABLES_MRU);
5259 	} else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5260 		req.rss_rule =
5261 			cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5262 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5263 					   VNIC_CFG_REQ_ENABLES_MRU);
5264 		req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5265 	} else {
5266 		req.rss_rule = cpu_to_le16(0xffff);
5267 	}
5268 
5269 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5270 	    (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5271 		req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5272 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5273 	} else {
5274 		req.cos_rule = cpu_to_le16(0xffff);
5275 	}
5276 
5277 	if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5278 		ring = 0;
5279 	else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5280 		ring = vnic_id - 1;
5281 	else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5282 		ring = bp->rx_nr_rings - 1;
5283 
5284 	grp_idx = bp->rx_ring[ring].bnapi->index;
5285 	req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5286 	req.lb_rule = cpu_to_le16(0xffff);
5287 vnic_mru:
5288 	req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5289 
5290 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5291 #ifdef CONFIG_BNXT_SRIOV
5292 	if (BNXT_VF(bp))
5293 		def_vlan = bp->vf.vlan;
5294 #endif
5295 	if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5296 		req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5297 	if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5298 		req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5299 
5300 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5301 }
5302 
5303 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5304 {
5305 	if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5306 		struct hwrm_vnic_free_input req = {0};
5307 
5308 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5309 		req.vnic_id =
5310 			cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5311 
5312 		hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5313 		bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5314 	}
5315 }
5316 
5317 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5318 {
5319 	u16 i;
5320 
5321 	for (i = 0; i < bp->nr_vnics; i++)
5322 		bnxt_hwrm_vnic_free_one(bp, i);
5323 }
5324 
5325 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5326 				unsigned int start_rx_ring_idx,
5327 				unsigned int nr_rings)
5328 {
5329 	int rc = 0;
5330 	unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5331 	struct hwrm_vnic_alloc_input req = {0};
5332 	struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5333 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5334 
5335 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5336 		goto vnic_no_ring_grps;
5337 
5338 	/* map ring groups to this vnic */
5339 	for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5340 		grp_idx = bp->rx_ring[i].bnapi->index;
5341 		if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5342 			netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5343 				   j, nr_rings);
5344 			break;
5345 		}
5346 		vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5347 	}
5348 
5349 vnic_no_ring_grps:
5350 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5351 		vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5352 	if (vnic_id == 0)
5353 		req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5354 
5355 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5356 
5357 	mutex_lock(&bp->hwrm_cmd_lock);
5358 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5359 	if (!rc)
5360 		vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5361 	mutex_unlock(&bp->hwrm_cmd_lock);
5362 	return rc;
5363 }
5364 
5365 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5366 {
5367 	struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5368 	struct hwrm_vnic_qcaps_input req = {0};
5369 	int rc;
5370 
5371 	bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5372 	bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5373 	if (bp->hwrm_spec_code < 0x10600)
5374 		return 0;
5375 
5376 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5377 	mutex_lock(&bp->hwrm_cmd_lock);
5378 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5379 	if (!rc) {
5380 		u32 flags = le32_to_cpu(resp->flags);
5381 
5382 		if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5383 		    (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5384 			bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5385 		if (flags &
5386 		    VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5387 			bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5388 
5389 		/* Older P5 fw before EXT_HW_STATS support did not set
5390 		 * VLAN_STRIP_CAP properly.
5391 		 */
5392 		if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5393 		    (BNXT_CHIP_P5_THOR(bp) &&
5394 		     !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5395 			bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5396 		bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5397 		if (bp->max_tpa_v2) {
5398 			if (BNXT_CHIP_P5_THOR(bp))
5399 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5;
5400 			else
5401 				bp->hw_ring_stats_size = BNXT_RING_STATS_SIZE_P5_SR2;
5402 		}
5403 	}
5404 	mutex_unlock(&bp->hwrm_cmd_lock);
5405 	return rc;
5406 }
5407 
5408 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5409 {
5410 	u16 i;
5411 	u32 rc = 0;
5412 
5413 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5414 		return 0;
5415 
5416 	mutex_lock(&bp->hwrm_cmd_lock);
5417 	for (i = 0; i < bp->rx_nr_rings; i++) {
5418 		struct hwrm_ring_grp_alloc_input req = {0};
5419 		struct hwrm_ring_grp_alloc_output *resp =
5420 					bp->hwrm_cmd_resp_addr;
5421 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5422 
5423 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5424 
5425 		req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5426 		req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5427 		req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5428 		req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5429 
5430 		rc = _hwrm_send_message(bp, &req, sizeof(req),
5431 					HWRM_CMD_TIMEOUT);
5432 		if (rc)
5433 			break;
5434 
5435 		bp->grp_info[grp_idx].fw_grp_id =
5436 			le32_to_cpu(resp->ring_group_id);
5437 	}
5438 	mutex_unlock(&bp->hwrm_cmd_lock);
5439 	return rc;
5440 }
5441 
5442 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5443 {
5444 	u16 i;
5445 	struct hwrm_ring_grp_free_input req = {0};
5446 
5447 	if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5448 		return;
5449 
5450 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5451 
5452 	mutex_lock(&bp->hwrm_cmd_lock);
5453 	for (i = 0; i < bp->cp_nr_rings; i++) {
5454 		if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5455 			continue;
5456 		req.ring_group_id =
5457 			cpu_to_le32(bp->grp_info[i].fw_grp_id);
5458 
5459 		_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5460 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5461 	}
5462 	mutex_unlock(&bp->hwrm_cmd_lock);
5463 }
5464 
5465 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5466 				    struct bnxt_ring_struct *ring,
5467 				    u32 ring_type, u32 map_index)
5468 {
5469 	int rc = 0, err = 0;
5470 	struct hwrm_ring_alloc_input req = {0};
5471 	struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5472 	struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5473 	struct bnxt_ring_grp_info *grp_info;
5474 	u16 ring_id;
5475 
5476 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5477 
5478 	req.enables = 0;
5479 	if (rmem->nr_pages > 1) {
5480 		req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5481 		/* Page size is in log2 units */
5482 		req.page_size = BNXT_PAGE_SHIFT;
5483 		req.page_tbl_depth = 1;
5484 	} else {
5485 		req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5486 	}
5487 	req.fbo = 0;
5488 	/* Association of ring index with doorbell index and MSIX number */
5489 	req.logical_id = cpu_to_le16(map_index);
5490 
5491 	switch (ring_type) {
5492 	case HWRM_RING_ALLOC_TX: {
5493 		struct bnxt_tx_ring_info *txr;
5494 
5495 		txr = container_of(ring, struct bnxt_tx_ring_info,
5496 				   tx_ring_struct);
5497 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5498 		/* Association of transmit ring with completion ring */
5499 		grp_info = &bp->grp_info[ring->grp_idx];
5500 		req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5501 		req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5502 		req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5503 		req.queue_id = cpu_to_le16(ring->queue_id);
5504 		break;
5505 	}
5506 	case HWRM_RING_ALLOC_RX:
5507 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5508 		req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5509 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5510 			u16 flags = 0;
5511 
5512 			/* Association of rx ring with stats context */
5513 			grp_info = &bp->grp_info[ring->grp_idx];
5514 			req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5515 			req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5516 			req.enables |= cpu_to_le32(
5517 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5518 			if (NET_IP_ALIGN == 2)
5519 				flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5520 			req.flags = cpu_to_le16(flags);
5521 		}
5522 		break;
5523 	case HWRM_RING_ALLOC_AGG:
5524 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5525 			req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5526 			/* Association of agg ring with rx ring */
5527 			grp_info = &bp->grp_info[ring->grp_idx];
5528 			req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5529 			req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5530 			req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5531 			req.enables |= cpu_to_le32(
5532 				RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5533 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5534 		} else {
5535 			req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5536 		}
5537 		req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5538 		break;
5539 	case HWRM_RING_ALLOC_CMPL:
5540 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5541 		req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5542 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5543 			/* Association of cp ring with nq */
5544 			grp_info = &bp->grp_info[map_index];
5545 			req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5546 			req.cq_handle = cpu_to_le64(ring->handle);
5547 			req.enables |= cpu_to_le32(
5548 				RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5549 		} else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5550 			req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5551 		}
5552 		break;
5553 	case HWRM_RING_ALLOC_NQ:
5554 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5555 		req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5556 		if (bp->flags & BNXT_FLAG_USING_MSIX)
5557 			req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5558 		break;
5559 	default:
5560 		netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5561 			   ring_type);
5562 		return -1;
5563 	}
5564 
5565 	mutex_lock(&bp->hwrm_cmd_lock);
5566 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5567 	err = le16_to_cpu(resp->error_code);
5568 	ring_id = le16_to_cpu(resp->ring_id);
5569 	mutex_unlock(&bp->hwrm_cmd_lock);
5570 
5571 	if (rc || err) {
5572 		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5573 			   ring_type, rc, err);
5574 		return -EIO;
5575 	}
5576 	ring->fw_ring_id = ring_id;
5577 	return rc;
5578 }
5579 
5580 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5581 {
5582 	int rc;
5583 
5584 	if (BNXT_PF(bp)) {
5585 		struct hwrm_func_cfg_input req = {0};
5586 
5587 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5588 		req.fid = cpu_to_le16(0xffff);
5589 		req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5590 		req.async_event_cr = cpu_to_le16(idx);
5591 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5592 	} else {
5593 		struct hwrm_func_vf_cfg_input req = {0};
5594 
5595 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5596 		req.enables =
5597 			cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5598 		req.async_event_cr = cpu_to_le16(idx);
5599 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5600 	}
5601 	return rc;
5602 }
5603 
5604 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5605 			u32 map_idx, u32 xid)
5606 {
5607 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5608 		if (BNXT_PF(bp))
5609 			db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5610 		else
5611 			db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5612 		switch (ring_type) {
5613 		case HWRM_RING_ALLOC_TX:
5614 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5615 			break;
5616 		case HWRM_RING_ALLOC_RX:
5617 		case HWRM_RING_ALLOC_AGG:
5618 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5619 			break;
5620 		case HWRM_RING_ALLOC_CMPL:
5621 			db->db_key64 = DBR_PATH_L2;
5622 			break;
5623 		case HWRM_RING_ALLOC_NQ:
5624 			db->db_key64 = DBR_PATH_L2;
5625 			break;
5626 		}
5627 		db->db_key64 |= (u64)xid << DBR_XID_SFT;
5628 	} else {
5629 		db->doorbell = bp->bar1 + map_idx * 0x80;
5630 		switch (ring_type) {
5631 		case HWRM_RING_ALLOC_TX:
5632 			db->db_key32 = DB_KEY_TX;
5633 			break;
5634 		case HWRM_RING_ALLOC_RX:
5635 		case HWRM_RING_ALLOC_AGG:
5636 			db->db_key32 = DB_KEY_RX;
5637 			break;
5638 		case HWRM_RING_ALLOC_CMPL:
5639 			db->db_key32 = DB_KEY_CP;
5640 			break;
5641 		}
5642 	}
5643 }
5644 
5645 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5646 {
5647 	bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5648 	int i, rc = 0;
5649 	u32 type;
5650 
5651 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5652 		type = HWRM_RING_ALLOC_NQ;
5653 	else
5654 		type = HWRM_RING_ALLOC_CMPL;
5655 	for (i = 0; i < bp->cp_nr_rings; i++) {
5656 		struct bnxt_napi *bnapi = bp->bnapi[i];
5657 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5658 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5659 		u32 map_idx = ring->map_idx;
5660 		unsigned int vector;
5661 
5662 		vector = bp->irq_tbl[map_idx].vector;
5663 		disable_irq_nosync(vector);
5664 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5665 		if (rc) {
5666 			enable_irq(vector);
5667 			goto err_out;
5668 		}
5669 		bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5670 		bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5671 		enable_irq(vector);
5672 		bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5673 
5674 		if (!i) {
5675 			rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5676 			if (rc)
5677 				netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5678 		}
5679 	}
5680 
5681 	type = HWRM_RING_ALLOC_TX;
5682 	for (i = 0; i < bp->tx_nr_rings; i++) {
5683 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5684 		struct bnxt_ring_struct *ring;
5685 		u32 map_idx;
5686 
5687 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5688 			struct bnxt_napi *bnapi = txr->bnapi;
5689 			struct bnxt_cp_ring_info *cpr, *cpr2;
5690 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5691 
5692 			cpr = &bnapi->cp_ring;
5693 			cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5694 			ring = &cpr2->cp_ring_struct;
5695 			ring->handle = BNXT_TX_HDL;
5696 			map_idx = bnapi->index;
5697 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5698 			if (rc)
5699 				goto err_out;
5700 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5701 				    ring->fw_ring_id);
5702 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5703 		}
5704 		ring = &txr->tx_ring_struct;
5705 		map_idx = i;
5706 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5707 		if (rc)
5708 			goto err_out;
5709 		bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5710 	}
5711 
5712 	type = HWRM_RING_ALLOC_RX;
5713 	for (i = 0; i < bp->rx_nr_rings; i++) {
5714 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5715 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5716 		struct bnxt_napi *bnapi = rxr->bnapi;
5717 		u32 map_idx = bnapi->index;
5718 
5719 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5720 		if (rc)
5721 			goto err_out;
5722 		bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5723 		/* If we have agg rings, post agg buffers first. */
5724 		if (!agg_rings)
5725 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5726 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5727 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5728 			struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5729 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5730 			struct bnxt_cp_ring_info *cpr2;
5731 
5732 			cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5733 			ring = &cpr2->cp_ring_struct;
5734 			ring->handle = BNXT_RX_HDL;
5735 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5736 			if (rc)
5737 				goto err_out;
5738 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5739 				    ring->fw_ring_id);
5740 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5741 		}
5742 	}
5743 
5744 	if (agg_rings) {
5745 		type = HWRM_RING_ALLOC_AGG;
5746 		for (i = 0; i < bp->rx_nr_rings; i++) {
5747 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5748 			struct bnxt_ring_struct *ring =
5749 						&rxr->rx_agg_ring_struct;
5750 			u32 grp_idx = ring->grp_idx;
5751 			u32 map_idx = grp_idx + bp->rx_nr_rings;
5752 
5753 			rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5754 			if (rc)
5755 				goto err_out;
5756 
5757 			bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5758 				    ring->fw_ring_id);
5759 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5760 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5761 			bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5762 		}
5763 	}
5764 err_out:
5765 	return rc;
5766 }
5767 
5768 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5769 				   struct bnxt_ring_struct *ring,
5770 				   u32 ring_type, int cmpl_ring_id)
5771 {
5772 	int rc;
5773 	struct hwrm_ring_free_input req = {0};
5774 	struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5775 	u16 error_code;
5776 
5777 	if (BNXT_NO_FW_ACCESS(bp))
5778 		return 0;
5779 
5780 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5781 	req.ring_type = ring_type;
5782 	req.ring_id = cpu_to_le16(ring->fw_ring_id);
5783 
5784 	mutex_lock(&bp->hwrm_cmd_lock);
5785 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5786 	error_code = le16_to_cpu(resp->error_code);
5787 	mutex_unlock(&bp->hwrm_cmd_lock);
5788 
5789 	if (rc || error_code) {
5790 		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5791 			   ring_type, rc, error_code);
5792 		return -EIO;
5793 	}
5794 	return 0;
5795 }
5796 
5797 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5798 {
5799 	u32 type;
5800 	int i;
5801 
5802 	if (!bp->bnapi)
5803 		return;
5804 
5805 	for (i = 0; i < bp->tx_nr_rings; i++) {
5806 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5807 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5808 
5809 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5810 			u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5811 
5812 			hwrm_ring_free_send_msg(bp, ring,
5813 						RING_FREE_REQ_RING_TYPE_TX,
5814 						close_path ? cmpl_ring_id :
5815 						INVALID_HW_RING_ID);
5816 			ring->fw_ring_id = INVALID_HW_RING_ID;
5817 		}
5818 	}
5819 
5820 	for (i = 0; i < bp->rx_nr_rings; i++) {
5821 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5822 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5823 		u32 grp_idx = rxr->bnapi->index;
5824 
5825 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5826 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5827 
5828 			hwrm_ring_free_send_msg(bp, ring,
5829 						RING_FREE_REQ_RING_TYPE_RX,
5830 						close_path ? cmpl_ring_id :
5831 						INVALID_HW_RING_ID);
5832 			ring->fw_ring_id = INVALID_HW_RING_ID;
5833 			bp->grp_info[grp_idx].rx_fw_ring_id =
5834 				INVALID_HW_RING_ID;
5835 		}
5836 	}
5837 
5838 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5839 		type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5840 	else
5841 		type = RING_FREE_REQ_RING_TYPE_RX;
5842 	for (i = 0; i < bp->rx_nr_rings; i++) {
5843 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5844 		struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5845 		u32 grp_idx = rxr->bnapi->index;
5846 
5847 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5848 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5849 
5850 			hwrm_ring_free_send_msg(bp, ring, type,
5851 						close_path ? cmpl_ring_id :
5852 						INVALID_HW_RING_ID);
5853 			ring->fw_ring_id = INVALID_HW_RING_ID;
5854 			bp->grp_info[grp_idx].agg_fw_ring_id =
5855 				INVALID_HW_RING_ID;
5856 		}
5857 	}
5858 
5859 	/* The completion rings are about to be freed.  After that the
5860 	 * IRQ doorbell will not work anymore.  So we need to disable
5861 	 * IRQ here.
5862 	 */
5863 	bnxt_disable_int_sync(bp);
5864 
5865 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5866 		type = RING_FREE_REQ_RING_TYPE_NQ;
5867 	else
5868 		type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5869 	for (i = 0; i < bp->cp_nr_rings; i++) {
5870 		struct bnxt_napi *bnapi = bp->bnapi[i];
5871 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5872 		struct bnxt_ring_struct *ring;
5873 		int j;
5874 
5875 		for (j = 0; j < 2; j++) {
5876 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5877 
5878 			if (cpr2) {
5879 				ring = &cpr2->cp_ring_struct;
5880 				if (ring->fw_ring_id == INVALID_HW_RING_ID)
5881 					continue;
5882 				hwrm_ring_free_send_msg(bp, ring,
5883 					RING_FREE_REQ_RING_TYPE_L2_CMPL,
5884 					INVALID_HW_RING_ID);
5885 				ring->fw_ring_id = INVALID_HW_RING_ID;
5886 			}
5887 		}
5888 		ring = &cpr->cp_ring_struct;
5889 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5890 			hwrm_ring_free_send_msg(bp, ring, type,
5891 						INVALID_HW_RING_ID);
5892 			ring->fw_ring_id = INVALID_HW_RING_ID;
5893 			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5894 		}
5895 	}
5896 }
5897 
5898 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5899 			   bool shared);
5900 
5901 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5902 {
5903 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5904 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5905 	struct hwrm_func_qcfg_input req = {0};
5906 	int rc;
5907 
5908 	if (bp->hwrm_spec_code < 0x10601)
5909 		return 0;
5910 
5911 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5912 	req.fid = cpu_to_le16(0xffff);
5913 	mutex_lock(&bp->hwrm_cmd_lock);
5914 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5915 	if (rc) {
5916 		mutex_unlock(&bp->hwrm_cmd_lock);
5917 		return rc;
5918 	}
5919 
5920 	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5921 	if (BNXT_NEW_RM(bp)) {
5922 		u16 cp, stats;
5923 
5924 		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5925 		hw_resc->resv_hw_ring_grps =
5926 			le32_to_cpu(resp->alloc_hw_ring_grps);
5927 		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5928 		cp = le16_to_cpu(resp->alloc_cmpl_rings);
5929 		stats = le16_to_cpu(resp->alloc_stat_ctx);
5930 		hw_resc->resv_irqs = cp;
5931 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5932 			int rx = hw_resc->resv_rx_rings;
5933 			int tx = hw_resc->resv_tx_rings;
5934 
5935 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
5936 				rx >>= 1;
5937 			if (cp < (rx + tx)) {
5938 				bnxt_trim_rings(bp, &rx, &tx, cp, false);
5939 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
5940 					rx <<= 1;
5941 				hw_resc->resv_rx_rings = rx;
5942 				hw_resc->resv_tx_rings = tx;
5943 			}
5944 			hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5945 			hw_resc->resv_hw_ring_grps = rx;
5946 		}
5947 		hw_resc->resv_cp_rings = cp;
5948 		hw_resc->resv_stat_ctxs = stats;
5949 	}
5950 	mutex_unlock(&bp->hwrm_cmd_lock);
5951 	return 0;
5952 }
5953 
5954 /* Caller must hold bp->hwrm_cmd_lock */
5955 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5956 {
5957 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5958 	struct hwrm_func_qcfg_input req = {0};
5959 	int rc;
5960 
5961 	if (bp->hwrm_spec_code < 0x10601)
5962 		return 0;
5963 
5964 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5965 	req.fid = cpu_to_le16(fid);
5966 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5967 	if (!rc)
5968 		*tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5969 
5970 	return rc;
5971 }
5972 
5973 static bool bnxt_rfs_supported(struct bnxt *bp);
5974 
5975 static void
5976 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5977 			     int tx_rings, int rx_rings, int ring_grps,
5978 			     int cp_rings, int stats, int vnics)
5979 {
5980 	u32 enables = 0;
5981 
5982 	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5983 	req->fid = cpu_to_le16(0xffff);
5984 	enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5985 	req->num_tx_rings = cpu_to_le16(tx_rings);
5986 	if (BNXT_NEW_RM(bp)) {
5987 		enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5988 		enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5989 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5990 			enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5991 			enables |= tx_rings + ring_grps ?
5992 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5993 			enables |= rx_rings ?
5994 				FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5995 		} else {
5996 			enables |= cp_rings ?
5997 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5998 			enables |= ring_grps ?
5999 				   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
6000 				   FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6001 		}
6002 		enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
6003 
6004 		req->num_rx_rings = cpu_to_le16(rx_rings);
6005 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
6006 			req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6007 			req->num_msix = cpu_to_le16(cp_rings);
6008 			req->num_rsscos_ctxs =
6009 				cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6010 		} else {
6011 			req->num_cmpl_rings = cpu_to_le16(cp_rings);
6012 			req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6013 			req->num_rsscos_ctxs = cpu_to_le16(1);
6014 			if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
6015 			    bnxt_rfs_supported(bp))
6016 				req->num_rsscos_ctxs =
6017 					cpu_to_le16(ring_grps + 1);
6018 		}
6019 		req->num_stat_ctxs = cpu_to_le16(stats);
6020 		req->num_vnics = cpu_to_le16(vnics);
6021 	}
6022 	req->enables = cpu_to_le32(enables);
6023 }
6024 
6025 static void
6026 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
6027 			     struct hwrm_func_vf_cfg_input *req, int tx_rings,
6028 			     int rx_rings, int ring_grps, int cp_rings,
6029 			     int stats, int vnics)
6030 {
6031 	u32 enables = 0;
6032 
6033 	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
6034 	enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
6035 	enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
6036 			      FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
6037 	enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
6038 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6039 		enables |= tx_rings + ring_grps ?
6040 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6041 	} else {
6042 		enables |= cp_rings ?
6043 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
6044 		enables |= ring_grps ?
6045 			   FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
6046 	}
6047 	enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
6048 	enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
6049 
6050 	req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6051 	req->num_tx_rings = cpu_to_le16(tx_rings);
6052 	req->num_rx_rings = cpu_to_le16(rx_rings);
6053 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6054 		req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6055 		req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6056 	} else {
6057 		req->num_cmpl_rings = cpu_to_le16(cp_rings);
6058 		req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6059 		req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6060 	}
6061 	req->num_stat_ctxs = cpu_to_le16(stats);
6062 	req->num_vnics = cpu_to_le16(vnics);
6063 
6064 	req->enables = cpu_to_le32(enables);
6065 }
6066 
6067 static int
6068 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6069 			   int ring_grps, int cp_rings, int stats, int vnics)
6070 {
6071 	struct hwrm_func_cfg_input req = {0};
6072 	int rc;
6073 
6074 	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6075 				     cp_rings, stats, vnics);
6076 	if (!req.enables)
6077 		return 0;
6078 
6079 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6080 	if (rc)
6081 		return rc;
6082 
6083 	if (bp->hwrm_spec_code < 0x10601)
6084 		bp->hw_resc.resv_tx_rings = tx_rings;
6085 
6086 	return bnxt_hwrm_get_rings(bp);
6087 }
6088 
6089 static int
6090 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6091 			   int ring_grps, int cp_rings, int stats, int vnics)
6092 {
6093 	struct hwrm_func_vf_cfg_input req = {0};
6094 	int rc;
6095 
6096 	if (!BNXT_NEW_RM(bp)) {
6097 		bp->hw_resc.resv_tx_rings = tx_rings;
6098 		return 0;
6099 	}
6100 
6101 	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6102 				     cp_rings, stats, vnics);
6103 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6104 	if (rc)
6105 		return rc;
6106 
6107 	return bnxt_hwrm_get_rings(bp);
6108 }
6109 
6110 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6111 				   int cp, int stat, int vnic)
6112 {
6113 	if (BNXT_PF(bp))
6114 		return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6115 						  vnic);
6116 	else
6117 		return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6118 						  vnic);
6119 }
6120 
6121 int bnxt_nq_rings_in_use(struct bnxt *bp)
6122 {
6123 	int cp = bp->cp_nr_rings;
6124 	int ulp_msix, ulp_base;
6125 
6126 	ulp_msix = bnxt_get_ulp_msix_num(bp);
6127 	if (ulp_msix) {
6128 		ulp_base = bnxt_get_ulp_msix_base(bp);
6129 		cp += ulp_msix;
6130 		if ((ulp_base + ulp_msix) > cp)
6131 			cp = ulp_base + ulp_msix;
6132 	}
6133 	return cp;
6134 }
6135 
6136 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6137 {
6138 	int cp;
6139 
6140 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6141 		return bnxt_nq_rings_in_use(bp);
6142 
6143 	cp = bp->tx_nr_rings + bp->rx_nr_rings;
6144 	return cp;
6145 }
6146 
6147 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6148 {
6149 	int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6150 	int cp = bp->cp_nr_rings;
6151 
6152 	if (!ulp_stat)
6153 		return cp;
6154 
6155 	if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6156 		return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6157 
6158 	return cp + ulp_stat;
6159 }
6160 
6161 /* Check if a default RSS map needs to be setup.  This function is only
6162  * used on older firmware that does not require reserving RX rings.
6163  */
6164 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6165 {
6166 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6167 
6168 	/* The RSS map is valid for RX rings set to resv_rx_rings */
6169 	if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6170 		hw_resc->resv_rx_rings = bp->rx_nr_rings;
6171 		if (!netif_is_rxfh_configured(bp->dev))
6172 			bnxt_set_dflt_rss_indir_tbl(bp);
6173 	}
6174 }
6175 
6176 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6177 {
6178 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6179 	int cp = bnxt_cp_rings_in_use(bp);
6180 	int nq = bnxt_nq_rings_in_use(bp);
6181 	int rx = bp->rx_nr_rings, stat;
6182 	int vnic = 1, grp = rx;
6183 
6184 	if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6185 	    bp->hwrm_spec_code >= 0x10601)
6186 		return true;
6187 
6188 	/* Old firmware does not need RX ring reservations but we still
6189 	 * need to setup a default RSS map when needed.  With new firmware
6190 	 * we go through RX ring reservations first and then set up the
6191 	 * RSS map for the successfully reserved RX rings when needed.
6192 	 */
6193 	if (!BNXT_NEW_RM(bp)) {
6194 		bnxt_check_rss_tbl_no_rmgr(bp);
6195 		return false;
6196 	}
6197 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6198 		vnic = rx + 1;
6199 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6200 		rx <<= 1;
6201 	stat = bnxt_get_func_stat_ctxs(bp);
6202 	if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6203 	    hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6204 	    (hw_resc->resv_hw_ring_grps != grp &&
6205 	     !(bp->flags & BNXT_FLAG_CHIP_P5)))
6206 		return true;
6207 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6208 	    hw_resc->resv_irqs != nq)
6209 		return true;
6210 	return false;
6211 }
6212 
6213 static int __bnxt_reserve_rings(struct bnxt *bp)
6214 {
6215 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6216 	int cp = bnxt_nq_rings_in_use(bp);
6217 	int tx = bp->tx_nr_rings;
6218 	int rx = bp->rx_nr_rings;
6219 	int grp, rx_rings, rc;
6220 	int vnic = 1, stat;
6221 	bool sh = false;
6222 
6223 	if (!bnxt_need_reserve_rings(bp))
6224 		return 0;
6225 
6226 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6227 		sh = true;
6228 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6229 		vnic = rx + 1;
6230 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6231 		rx <<= 1;
6232 	grp = bp->rx_nr_rings;
6233 	stat = bnxt_get_func_stat_ctxs(bp);
6234 
6235 	rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6236 	if (rc)
6237 		return rc;
6238 
6239 	tx = hw_resc->resv_tx_rings;
6240 	if (BNXT_NEW_RM(bp)) {
6241 		rx = hw_resc->resv_rx_rings;
6242 		cp = hw_resc->resv_irqs;
6243 		grp = hw_resc->resv_hw_ring_grps;
6244 		vnic = hw_resc->resv_vnics;
6245 		stat = hw_resc->resv_stat_ctxs;
6246 	}
6247 
6248 	rx_rings = rx;
6249 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6250 		if (rx >= 2) {
6251 			rx_rings = rx >> 1;
6252 		} else {
6253 			if (netif_running(bp->dev))
6254 				return -ENOMEM;
6255 
6256 			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6257 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6258 			bp->dev->hw_features &= ~NETIF_F_LRO;
6259 			bp->dev->features &= ~NETIF_F_LRO;
6260 			bnxt_set_ring_params(bp);
6261 		}
6262 	}
6263 	rx_rings = min_t(int, rx_rings, grp);
6264 	cp = min_t(int, cp, bp->cp_nr_rings);
6265 	if (stat > bnxt_get_ulp_stat_ctxs(bp))
6266 		stat -= bnxt_get_ulp_stat_ctxs(bp);
6267 	cp = min_t(int, cp, stat);
6268 	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6269 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6270 		rx = rx_rings << 1;
6271 	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6272 	bp->tx_nr_rings = tx;
6273 
6274 	/* If we cannot reserve all the RX rings, reset the RSS map only
6275 	 * if absolutely necessary
6276 	 */
6277 	if (rx_rings != bp->rx_nr_rings) {
6278 		netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6279 			    rx_rings, bp->rx_nr_rings);
6280 		if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6281 		    (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6282 		     bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6283 		     bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6284 			netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6285 			bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6286 		}
6287 	}
6288 	bp->rx_nr_rings = rx_rings;
6289 	bp->cp_nr_rings = cp;
6290 
6291 	if (!tx || !rx || !cp || !grp || !vnic || !stat)
6292 		return -ENOMEM;
6293 
6294 	if (!netif_is_rxfh_configured(bp->dev))
6295 		bnxt_set_dflt_rss_indir_tbl(bp);
6296 
6297 	return rc;
6298 }
6299 
6300 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6301 				    int ring_grps, int cp_rings, int stats,
6302 				    int vnics)
6303 {
6304 	struct hwrm_func_vf_cfg_input req = {0};
6305 	u32 flags;
6306 
6307 	if (!BNXT_NEW_RM(bp))
6308 		return 0;
6309 
6310 	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6311 				     cp_rings, stats, vnics);
6312 	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6313 		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6314 		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6315 		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6316 		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6317 		FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6318 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6319 		flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6320 
6321 	req.flags = cpu_to_le32(flags);
6322 	return hwrm_send_message_silent(bp, &req, sizeof(req),
6323 					HWRM_CMD_TIMEOUT);
6324 }
6325 
6326 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6327 				    int ring_grps, int cp_rings, int stats,
6328 				    int vnics)
6329 {
6330 	struct hwrm_func_cfg_input req = {0};
6331 	u32 flags;
6332 
6333 	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6334 				     cp_rings, stats, vnics);
6335 	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6336 	if (BNXT_NEW_RM(bp)) {
6337 		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6338 			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6339 			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6340 			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6341 		if (bp->flags & BNXT_FLAG_CHIP_P5)
6342 			flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6343 				 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6344 		else
6345 			flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6346 	}
6347 
6348 	req.flags = cpu_to_le32(flags);
6349 	return hwrm_send_message_silent(bp, &req, sizeof(req),
6350 					HWRM_CMD_TIMEOUT);
6351 }
6352 
6353 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6354 				 int ring_grps, int cp_rings, int stats,
6355 				 int vnics)
6356 {
6357 	if (bp->hwrm_spec_code < 0x10801)
6358 		return 0;
6359 
6360 	if (BNXT_PF(bp))
6361 		return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6362 						ring_grps, cp_rings, stats,
6363 						vnics);
6364 
6365 	return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6366 					cp_rings, stats, vnics);
6367 }
6368 
6369 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6370 {
6371 	struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6372 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6373 	struct hwrm_ring_aggint_qcaps_input req = {0};
6374 	int rc;
6375 
6376 	coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6377 	coal_cap->num_cmpl_dma_aggr_max = 63;
6378 	coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6379 	coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6380 	coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6381 	coal_cap->int_lat_tmr_min_max = 65535;
6382 	coal_cap->int_lat_tmr_max_max = 65535;
6383 	coal_cap->num_cmpl_aggr_int_max = 65535;
6384 	coal_cap->timer_units = 80;
6385 
6386 	if (bp->hwrm_spec_code < 0x10902)
6387 		return;
6388 
6389 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6390 	mutex_lock(&bp->hwrm_cmd_lock);
6391 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6392 	if (!rc) {
6393 		coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6394 		coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6395 		coal_cap->num_cmpl_dma_aggr_max =
6396 			le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6397 		coal_cap->num_cmpl_dma_aggr_during_int_max =
6398 			le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6399 		coal_cap->cmpl_aggr_dma_tmr_max =
6400 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6401 		coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6402 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6403 		coal_cap->int_lat_tmr_min_max =
6404 			le16_to_cpu(resp->int_lat_tmr_min_max);
6405 		coal_cap->int_lat_tmr_max_max =
6406 			le16_to_cpu(resp->int_lat_tmr_max_max);
6407 		coal_cap->num_cmpl_aggr_int_max =
6408 			le16_to_cpu(resp->num_cmpl_aggr_int_max);
6409 		coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6410 	}
6411 	mutex_unlock(&bp->hwrm_cmd_lock);
6412 }
6413 
6414 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6415 {
6416 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6417 
6418 	return usec * 1000 / coal_cap->timer_units;
6419 }
6420 
6421 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6422 	struct bnxt_coal *hw_coal,
6423 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6424 {
6425 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6426 	u32 cmpl_params = coal_cap->cmpl_params;
6427 	u16 val, tmr, max, flags = 0;
6428 
6429 	max = hw_coal->bufs_per_record * 128;
6430 	if (hw_coal->budget)
6431 		max = hw_coal->bufs_per_record * hw_coal->budget;
6432 	max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6433 
6434 	val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6435 	req->num_cmpl_aggr_int = cpu_to_le16(val);
6436 
6437 	val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6438 	req->num_cmpl_dma_aggr = cpu_to_le16(val);
6439 
6440 	val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6441 		      coal_cap->num_cmpl_dma_aggr_during_int_max);
6442 	req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6443 
6444 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6445 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6446 	req->int_lat_tmr_max = cpu_to_le16(tmr);
6447 
6448 	/* min timer set to 1/2 of interrupt timer */
6449 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6450 		val = tmr / 2;
6451 		val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6452 		req->int_lat_tmr_min = cpu_to_le16(val);
6453 		req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6454 	}
6455 
6456 	/* buf timer set to 1/4 of interrupt timer */
6457 	val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6458 	req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6459 
6460 	if (cmpl_params &
6461 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6462 		tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6463 		val = clamp_t(u16, tmr, 1,
6464 			      coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6465 		req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6466 		req->enables |=
6467 			cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6468 	}
6469 
6470 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6471 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6472 	if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6473 	    hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6474 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6475 	req->flags = cpu_to_le16(flags);
6476 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6477 }
6478 
6479 /* Caller holds bp->hwrm_cmd_lock */
6480 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6481 				   struct bnxt_coal *hw_coal)
6482 {
6483 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6484 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6485 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6486 	u32 nq_params = coal_cap->nq_params;
6487 	u16 tmr;
6488 
6489 	if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6490 		return 0;
6491 
6492 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6493 			       -1, -1);
6494 	req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6495 	req.flags =
6496 		cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6497 
6498 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6499 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6500 	req.int_lat_tmr_min = cpu_to_le16(tmr);
6501 	req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6502 	return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6503 }
6504 
6505 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6506 {
6507 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6508 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6509 	struct bnxt_coal coal;
6510 
6511 	/* Tick values in micro seconds.
6512 	 * 1 coal_buf x bufs_per_record = 1 completion record.
6513 	 */
6514 	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6515 
6516 	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6517 	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6518 
6519 	if (!bnapi->rx_ring)
6520 		return -ENODEV;
6521 
6522 	bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6523 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6524 
6525 	bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6526 
6527 	req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6528 
6529 	return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6530 				 HWRM_CMD_TIMEOUT);
6531 }
6532 
6533 int bnxt_hwrm_set_coal(struct bnxt *bp)
6534 {
6535 	int i, rc = 0;
6536 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6537 							   req_tx = {0}, *req;
6538 
6539 	bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6540 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6541 	bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6542 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6543 
6544 	bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6545 	bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6546 
6547 	mutex_lock(&bp->hwrm_cmd_lock);
6548 	for (i = 0; i < bp->cp_nr_rings; i++) {
6549 		struct bnxt_napi *bnapi = bp->bnapi[i];
6550 		struct bnxt_coal *hw_coal;
6551 		u16 ring_id;
6552 
6553 		req = &req_rx;
6554 		if (!bnapi->rx_ring) {
6555 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6556 			req = &req_tx;
6557 		} else {
6558 			ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6559 		}
6560 		req->ring_id = cpu_to_le16(ring_id);
6561 
6562 		rc = _hwrm_send_message(bp, req, sizeof(*req),
6563 					HWRM_CMD_TIMEOUT);
6564 		if (rc)
6565 			break;
6566 
6567 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6568 			continue;
6569 
6570 		if (bnapi->rx_ring && bnapi->tx_ring) {
6571 			req = &req_tx;
6572 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6573 			req->ring_id = cpu_to_le16(ring_id);
6574 			rc = _hwrm_send_message(bp, req, sizeof(*req),
6575 						HWRM_CMD_TIMEOUT);
6576 			if (rc)
6577 				break;
6578 		}
6579 		if (bnapi->rx_ring)
6580 			hw_coal = &bp->rx_coal;
6581 		else
6582 			hw_coal = &bp->tx_coal;
6583 		__bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6584 	}
6585 	mutex_unlock(&bp->hwrm_cmd_lock);
6586 	return rc;
6587 }
6588 
6589 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6590 {
6591 	struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6592 	struct hwrm_stat_ctx_free_input req = {0};
6593 	int i;
6594 
6595 	if (!bp->bnapi)
6596 		return;
6597 
6598 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6599 		return;
6600 
6601 	bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6602 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6603 
6604 	mutex_lock(&bp->hwrm_cmd_lock);
6605 	for (i = 0; i < bp->cp_nr_rings; i++) {
6606 		struct bnxt_napi *bnapi = bp->bnapi[i];
6607 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6608 
6609 		if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6610 			req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6611 			if (BNXT_FW_MAJ(bp) <= 20) {
6612 				req0.stat_ctx_id = req.stat_ctx_id;
6613 				_hwrm_send_message(bp, &req0, sizeof(req0),
6614 						   HWRM_CMD_TIMEOUT);
6615 			}
6616 			_hwrm_send_message(bp, &req, sizeof(req),
6617 					   HWRM_CMD_TIMEOUT);
6618 
6619 			cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6620 		}
6621 	}
6622 	mutex_unlock(&bp->hwrm_cmd_lock);
6623 }
6624 
6625 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6626 {
6627 	int rc = 0, i;
6628 	struct hwrm_stat_ctx_alloc_input req = {0};
6629 	struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6630 
6631 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6632 		return 0;
6633 
6634 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6635 
6636 	req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6637 	req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6638 
6639 	mutex_lock(&bp->hwrm_cmd_lock);
6640 	for (i = 0; i < bp->cp_nr_rings; i++) {
6641 		struct bnxt_napi *bnapi = bp->bnapi[i];
6642 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6643 
6644 		req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6645 
6646 		rc = _hwrm_send_message(bp, &req, sizeof(req),
6647 					HWRM_CMD_TIMEOUT);
6648 		if (rc)
6649 			break;
6650 
6651 		cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6652 
6653 		bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6654 	}
6655 	mutex_unlock(&bp->hwrm_cmd_lock);
6656 	return rc;
6657 }
6658 
6659 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6660 {
6661 	struct hwrm_func_qcfg_input req = {0};
6662 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6663 	u32 min_db_offset = 0;
6664 	u16 flags;
6665 	int rc;
6666 
6667 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6668 	req.fid = cpu_to_le16(0xffff);
6669 	mutex_lock(&bp->hwrm_cmd_lock);
6670 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6671 	if (rc)
6672 		goto func_qcfg_exit;
6673 
6674 #ifdef CONFIG_BNXT_SRIOV
6675 	if (BNXT_VF(bp)) {
6676 		struct bnxt_vf_info *vf = &bp->vf;
6677 
6678 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6679 	} else {
6680 		bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6681 	}
6682 #endif
6683 	flags = le16_to_cpu(resp->flags);
6684 	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6685 		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6686 		bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6687 		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6688 			bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6689 	}
6690 	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6691 		bp->flags |= BNXT_FLAG_MULTI_HOST;
6692 	if (flags & FUNC_QCFG_RESP_FLAGS_RING_MONITOR_ENABLED)
6693 		bp->fw_cap |= BNXT_FW_CAP_RING_MONITOR;
6694 
6695 	switch (resp->port_partition_type) {
6696 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6697 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6698 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6699 		bp->port_partition_type = resp->port_partition_type;
6700 		break;
6701 	}
6702 	if (bp->hwrm_spec_code < 0x10707 ||
6703 	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6704 		bp->br_mode = BRIDGE_MODE_VEB;
6705 	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6706 		bp->br_mode = BRIDGE_MODE_VEPA;
6707 	else
6708 		bp->br_mode = BRIDGE_MODE_UNDEF;
6709 
6710 	bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6711 	if (!bp->max_mtu)
6712 		bp->max_mtu = BNXT_MAX_MTU;
6713 
6714 	if (bp->db_size)
6715 		goto func_qcfg_exit;
6716 
6717 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6718 		if (BNXT_PF(bp))
6719 			min_db_offset = DB_PF_OFFSET_P5;
6720 		else
6721 			min_db_offset = DB_VF_OFFSET_P5;
6722 	}
6723 	bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6724 				 1024);
6725 	if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6726 	    bp->db_size <= min_db_offset)
6727 		bp->db_size = pci_resource_len(bp->pdev, 2);
6728 
6729 func_qcfg_exit:
6730 	mutex_unlock(&bp->hwrm_cmd_lock);
6731 	return rc;
6732 }
6733 
6734 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6735 {
6736 	struct hwrm_func_backing_store_qcaps_input req = {0};
6737 	struct hwrm_func_backing_store_qcaps_output *resp =
6738 		bp->hwrm_cmd_resp_addr;
6739 	int rc;
6740 
6741 	if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6742 		return 0;
6743 
6744 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6745 	mutex_lock(&bp->hwrm_cmd_lock);
6746 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6747 	if (!rc) {
6748 		struct bnxt_ctx_pg_info *ctx_pg;
6749 		struct bnxt_ctx_mem_info *ctx;
6750 		int i, tqm_rings;
6751 
6752 		ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6753 		if (!ctx) {
6754 			rc = -ENOMEM;
6755 			goto ctx_err;
6756 		}
6757 		ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6758 		ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6759 		ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6760 		ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6761 		ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6762 		ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6763 		ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6764 		ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6765 		ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6766 		ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6767 		ctx->vnic_max_vnic_entries =
6768 			le16_to_cpu(resp->vnic_max_vnic_entries);
6769 		ctx->vnic_max_ring_table_entries =
6770 			le16_to_cpu(resp->vnic_max_ring_table_entries);
6771 		ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6772 		ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6773 		ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6774 		ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6775 		ctx->tqm_min_entries_per_ring =
6776 			le32_to_cpu(resp->tqm_min_entries_per_ring);
6777 		ctx->tqm_max_entries_per_ring =
6778 			le32_to_cpu(resp->tqm_max_entries_per_ring);
6779 		ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6780 		if (!ctx->tqm_entries_multiple)
6781 			ctx->tqm_entries_multiple = 1;
6782 		ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6783 		ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6784 		ctx->mrav_num_entries_units =
6785 			le16_to_cpu(resp->mrav_num_entries_units);
6786 		ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6787 		ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6788 		ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6789 		ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6790 		if (!ctx->tqm_fp_rings_count)
6791 			ctx->tqm_fp_rings_count = bp->max_q;
6792 
6793 		tqm_rings = ctx->tqm_fp_rings_count + 1;
6794 		ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6795 		if (!ctx_pg) {
6796 			kfree(ctx);
6797 			rc = -ENOMEM;
6798 			goto ctx_err;
6799 		}
6800 		for (i = 0; i < tqm_rings; i++, ctx_pg++)
6801 			ctx->tqm_mem[i] = ctx_pg;
6802 		bp->ctx = ctx;
6803 	} else {
6804 		rc = 0;
6805 	}
6806 ctx_err:
6807 	mutex_unlock(&bp->hwrm_cmd_lock);
6808 	return rc;
6809 }
6810 
6811 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6812 				  __le64 *pg_dir)
6813 {
6814 	u8 pg_size = 0;
6815 
6816 	if (BNXT_PAGE_SHIFT == 13)
6817 		pg_size = 1 << 4;
6818 	else if (BNXT_PAGE_SIZE == 16)
6819 		pg_size = 2 << 4;
6820 
6821 	*pg_attr = pg_size;
6822 	if (rmem->depth >= 1) {
6823 		if (rmem->depth == 2)
6824 			*pg_attr |= 2;
6825 		else
6826 			*pg_attr |= 1;
6827 		*pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6828 	} else {
6829 		*pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6830 	}
6831 }
6832 
6833 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
6834 	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
6835 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
6836 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
6837 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
6838 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6839 
6840 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6841 {
6842 	struct hwrm_func_backing_store_cfg_input req = {0};
6843 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
6844 	struct bnxt_ctx_pg_info *ctx_pg;
6845 	__le32 *num_entries;
6846 	__le64 *pg_dir;
6847 	u32 flags = 0;
6848 	u8 *pg_attr;
6849 	u32 ena;
6850 	int i;
6851 
6852 	if (!ctx)
6853 		return 0;
6854 
6855 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6856 	req.enables = cpu_to_le32(enables);
6857 
6858 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6859 		ctx_pg = &ctx->qp_mem;
6860 		req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6861 		req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6862 		req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6863 		req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6864 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6865 				      &req.qpc_pg_size_qpc_lvl,
6866 				      &req.qpc_page_dir);
6867 	}
6868 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6869 		ctx_pg = &ctx->srq_mem;
6870 		req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6871 		req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6872 		req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6873 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6874 				      &req.srq_pg_size_srq_lvl,
6875 				      &req.srq_page_dir);
6876 	}
6877 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6878 		ctx_pg = &ctx->cq_mem;
6879 		req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6880 		req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6881 		req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6882 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6883 				      &req.cq_page_dir);
6884 	}
6885 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6886 		ctx_pg = &ctx->vnic_mem;
6887 		req.vnic_num_vnic_entries =
6888 			cpu_to_le16(ctx->vnic_max_vnic_entries);
6889 		req.vnic_num_ring_table_entries =
6890 			cpu_to_le16(ctx->vnic_max_ring_table_entries);
6891 		req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6892 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6893 				      &req.vnic_pg_size_vnic_lvl,
6894 				      &req.vnic_page_dir);
6895 	}
6896 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6897 		ctx_pg = &ctx->stat_mem;
6898 		req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6899 		req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6900 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6901 				      &req.stat_pg_size_stat_lvl,
6902 				      &req.stat_page_dir);
6903 	}
6904 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6905 		ctx_pg = &ctx->mrav_mem;
6906 		req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6907 		if (ctx->mrav_num_entries_units)
6908 			flags |=
6909 			FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6910 		req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6911 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6912 				      &req.mrav_pg_size_mrav_lvl,
6913 				      &req.mrav_page_dir);
6914 	}
6915 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6916 		ctx_pg = &ctx->tim_mem;
6917 		req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6918 		req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6919 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6920 				      &req.tim_pg_size_tim_lvl,
6921 				      &req.tim_page_dir);
6922 	}
6923 	for (i = 0, num_entries = &req.tqm_sp_num_entries,
6924 	     pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6925 	     pg_dir = &req.tqm_sp_page_dir,
6926 	     ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6927 	     i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6928 		if (!(enables & ena))
6929 			continue;
6930 
6931 		req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6932 		ctx_pg = ctx->tqm_mem[i];
6933 		*num_entries = cpu_to_le32(ctx_pg->entries);
6934 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6935 	}
6936 	req.flags = cpu_to_le32(flags);
6937 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6938 }
6939 
6940 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6941 				  struct bnxt_ctx_pg_info *ctx_pg)
6942 {
6943 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6944 
6945 	rmem->page_size = BNXT_PAGE_SIZE;
6946 	rmem->pg_arr = ctx_pg->ctx_pg_arr;
6947 	rmem->dma_arr = ctx_pg->ctx_dma_arr;
6948 	rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6949 	if (rmem->depth >= 1)
6950 		rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6951 	return bnxt_alloc_ring(bp, rmem);
6952 }
6953 
6954 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6955 				  struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6956 				  u8 depth, bool use_init_val)
6957 {
6958 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6959 	int rc;
6960 
6961 	if (!mem_size)
6962 		return -EINVAL;
6963 
6964 	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6965 	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6966 		ctx_pg->nr_pages = 0;
6967 		return -EINVAL;
6968 	}
6969 	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6970 		int nr_tbls, i;
6971 
6972 		rmem->depth = 2;
6973 		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6974 					     GFP_KERNEL);
6975 		if (!ctx_pg->ctx_pg_tbl)
6976 			return -ENOMEM;
6977 		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6978 		rmem->nr_pages = nr_tbls;
6979 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6980 		if (rc)
6981 			return rc;
6982 		for (i = 0; i < nr_tbls; i++) {
6983 			struct bnxt_ctx_pg_info *pg_tbl;
6984 
6985 			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6986 			if (!pg_tbl)
6987 				return -ENOMEM;
6988 			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6989 			rmem = &pg_tbl->ring_mem;
6990 			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6991 			rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6992 			rmem->depth = 1;
6993 			rmem->nr_pages = MAX_CTX_PAGES;
6994 			if (use_init_val)
6995 				rmem->init_val = bp->ctx->ctx_kind_initializer;
6996 			if (i == (nr_tbls - 1)) {
6997 				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6998 
6999 				if (rem)
7000 					rmem->nr_pages = rem;
7001 			}
7002 			rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
7003 			if (rc)
7004 				break;
7005 		}
7006 	} else {
7007 		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
7008 		if (rmem->nr_pages > 1 || depth)
7009 			rmem->depth = 1;
7010 		if (use_init_val)
7011 			rmem->init_val = bp->ctx->ctx_kind_initializer;
7012 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
7013 	}
7014 	return rc;
7015 }
7016 
7017 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
7018 				  struct bnxt_ctx_pg_info *ctx_pg)
7019 {
7020 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
7021 
7022 	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
7023 	    ctx_pg->ctx_pg_tbl) {
7024 		int i, nr_tbls = rmem->nr_pages;
7025 
7026 		for (i = 0; i < nr_tbls; i++) {
7027 			struct bnxt_ctx_pg_info *pg_tbl;
7028 			struct bnxt_ring_mem_info *rmem2;
7029 
7030 			pg_tbl = ctx_pg->ctx_pg_tbl[i];
7031 			if (!pg_tbl)
7032 				continue;
7033 			rmem2 = &pg_tbl->ring_mem;
7034 			bnxt_free_ring(bp, rmem2);
7035 			ctx_pg->ctx_pg_arr[i] = NULL;
7036 			kfree(pg_tbl);
7037 			ctx_pg->ctx_pg_tbl[i] = NULL;
7038 		}
7039 		kfree(ctx_pg->ctx_pg_tbl);
7040 		ctx_pg->ctx_pg_tbl = NULL;
7041 	}
7042 	bnxt_free_ring(bp, rmem);
7043 	ctx_pg->nr_pages = 0;
7044 }
7045 
7046 static void bnxt_free_ctx_mem(struct bnxt *bp)
7047 {
7048 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
7049 	int i;
7050 
7051 	if (!ctx)
7052 		return;
7053 
7054 	if (ctx->tqm_mem[0]) {
7055 		for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7056 			bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7057 		kfree(ctx->tqm_mem[0]);
7058 		ctx->tqm_mem[0] = NULL;
7059 	}
7060 
7061 	bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7062 	bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7063 	bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7064 	bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7065 	bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7066 	bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7067 	bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7068 	ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7069 }
7070 
7071 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7072 {
7073 	struct bnxt_ctx_pg_info *ctx_pg;
7074 	struct bnxt_ctx_mem_info *ctx;
7075 	u32 mem_size, ena, entries;
7076 	u32 entries_sp, min;
7077 	u32 num_mr, num_ah;
7078 	u32 extra_srqs = 0;
7079 	u32 extra_qps = 0;
7080 	u8 pg_lvl = 1;
7081 	int i, rc;
7082 
7083 	rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7084 	if (rc) {
7085 		netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7086 			   rc);
7087 		return rc;
7088 	}
7089 	ctx = bp->ctx;
7090 	if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7091 		return 0;
7092 
7093 	if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7094 		pg_lvl = 2;
7095 		extra_qps = 65536;
7096 		extra_srqs = 8192;
7097 	}
7098 
7099 	ctx_pg = &ctx->qp_mem;
7100 	ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7101 			  extra_qps;
7102 	mem_size = ctx->qp_entry_size * ctx_pg->entries;
7103 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7104 	if (rc)
7105 		return rc;
7106 
7107 	ctx_pg = &ctx->srq_mem;
7108 	ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7109 	mem_size = ctx->srq_entry_size * ctx_pg->entries;
7110 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7111 	if (rc)
7112 		return rc;
7113 
7114 	ctx_pg = &ctx->cq_mem;
7115 	ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7116 	mem_size = ctx->cq_entry_size * ctx_pg->entries;
7117 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7118 	if (rc)
7119 		return rc;
7120 
7121 	ctx_pg = &ctx->vnic_mem;
7122 	ctx_pg->entries = ctx->vnic_max_vnic_entries +
7123 			  ctx->vnic_max_ring_table_entries;
7124 	mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7125 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7126 	if (rc)
7127 		return rc;
7128 
7129 	ctx_pg = &ctx->stat_mem;
7130 	ctx_pg->entries = ctx->stat_max_entries;
7131 	mem_size = ctx->stat_entry_size * ctx_pg->entries;
7132 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7133 	if (rc)
7134 		return rc;
7135 
7136 	ena = 0;
7137 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7138 		goto skip_rdma;
7139 
7140 	ctx_pg = &ctx->mrav_mem;
7141 	/* 128K extra is needed to accommodate static AH context
7142 	 * allocation by f/w.
7143 	 */
7144 	num_mr = 1024 * 256;
7145 	num_ah = 1024 * 128;
7146 	ctx_pg->entries = num_mr + num_ah;
7147 	mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7148 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7149 	if (rc)
7150 		return rc;
7151 	ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7152 	if (ctx->mrav_num_entries_units)
7153 		ctx_pg->entries =
7154 			((num_mr / ctx->mrav_num_entries_units) << 16) |
7155 			 (num_ah / ctx->mrav_num_entries_units);
7156 
7157 	ctx_pg = &ctx->tim_mem;
7158 	ctx_pg->entries = ctx->qp_mem.entries;
7159 	mem_size = ctx->tim_entry_size * ctx_pg->entries;
7160 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7161 	if (rc)
7162 		return rc;
7163 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7164 
7165 skip_rdma:
7166 	min = ctx->tqm_min_entries_per_ring;
7167 	entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7168 		     2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7169 	entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7170 	entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7171 	entries = roundup(entries, ctx->tqm_entries_multiple);
7172 	entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7173 	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7174 		ctx_pg = ctx->tqm_mem[i];
7175 		ctx_pg->entries = i ? entries : entries_sp;
7176 		mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7177 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7178 		if (rc)
7179 			return rc;
7180 		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7181 	}
7182 	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7183 	rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7184 	if (rc) {
7185 		netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7186 			   rc);
7187 		return rc;
7188 	}
7189 	ctx->flags |= BNXT_CTX_FLAG_INITED;
7190 	return 0;
7191 }
7192 
7193 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7194 {
7195 	struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7196 	struct hwrm_func_resource_qcaps_input req = {0};
7197 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7198 	int rc;
7199 
7200 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7201 	req.fid = cpu_to_le16(0xffff);
7202 
7203 	mutex_lock(&bp->hwrm_cmd_lock);
7204 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7205 				       HWRM_CMD_TIMEOUT);
7206 	if (rc)
7207 		goto hwrm_func_resc_qcaps_exit;
7208 
7209 	hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7210 	if (!all)
7211 		goto hwrm_func_resc_qcaps_exit;
7212 
7213 	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7214 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7215 	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7216 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7217 	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7218 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7219 	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7220 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7221 	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7222 	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7223 	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7224 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7225 	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7226 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7227 	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7228 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7229 
7230 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
7231 		u16 max_msix = le16_to_cpu(resp->max_msix);
7232 
7233 		hw_resc->max_nqs = max_msix;
7234 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7235 	}
7236 
7237 	if (BNXT_PF(bp)) {
7238 		struct bnxt_pf_info *pf = &bp->pf;
7239 
7240 		pf->vf_resv_strategy =
7241 			le16_to_cpu(resp->vf_reservation_strategy);
7242 		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7243 			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7244 	}
7245 hwrm_func_resc_qcaps_exit:
7246 	mutex_unlock(&bp->hwrm_cmd_lock);
7247 	return rc;
7248 }
7249 
7250 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7251 {
7252 	int rc = 0;
7253 	struct hwrm_func_qcaps_input req = {0};
7254 	struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7255 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7256 	u32 flags, flags_ext;
7257 
7258 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7259 	req.fid = cpu_to_le16(0xffff);
7260 
7261 	mutex_lock(&bp->hwrm_cmd_lock);
7262 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7263 	if (rc)
7264 		goto hwrm_func_qcaps_exit;
7265 
7266 	flags = le32_to_cpu(resp->flags);
7267 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7268 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7269 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7270 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7271 	if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7272 		bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7273 	if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7274 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7275 	if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7276 		bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7277 	if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7278 		bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7279 	if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7280 		bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7281 	if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7282 		bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7283 
7284 	flags_ext = le32_to_cpu(resp->flags_ext);
7285 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7286 		bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7287 
7288 	bp->tx_push_thresh = 0;
7289 	if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7290 	    BNXT_FW_MAJ(bp) > 217)
7291 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7292 
7293 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7294 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7295 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7296 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7297 	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7298 	if (!hw_resc->max_hw_ring_grps)
7299 		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7300 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7301 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7302 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7303 
7304 	if (BNXT_PF(bp)) {
7305 		struct bnxt_pf_info *pf = &bp->pf;
7306 
7307 		pf->fw_fid = le16_to_cpu(resp->fid);
7308 		pf->port_id = le16_to_cpu(resp->port_id);
7309 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7310 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7311 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
7312 		pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7313 		pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7314 		pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7315 		pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7316 		pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7317 		pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7318 		bp->flags &= ~BNXT_FLAG_WOL_CAP;
7319 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7320 			bp->flags |= BNXT_FLAG_WOL_CAP;
7321 	} else {
7322 #ifdef CONFIG_BNXT_SRIOV
7323 		struct bnxt_vf_info *vf = &bp->vf;
7324 
7325 		vf->fw_fid = le16_to_cpu(resp->fid);
7326 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7327 #endif
7328 	}
7329 
7330 hwrm_func_qcaps_exit:
7331 	mutex_unlock(&bp->hwrm_cmd_lock);
7332 	return rc;
7333 }
7334 
7335 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7336 
7337 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7338 {
7339 	int rc;
7340 
7341 	rc = __bnxt_hwrm_func_qcaps(bp);
7342 	if (rc)
7343 		return rc;
7344 	rc = bnxt_hwrm_queue_qportcfg(bp);
7345 	if (rc) {
7346 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7347 		return rc;
7348 	}
7349 	if (bp->hwrm_spec_code >= 0x10803) {
7350 		rc = bnxt_alloc_ctx_mem(bp);
7351 		if (rc)
7352 			return rc;
7353 		rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7354 		if (!rc)
7355 			bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7356 	}
7357 	return 0;
7358 }
7359 
7360 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7361 {
7362 	struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7363 	struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7364 	int rc = 0;
7365 	u32 flags;
7366 
7367 	if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7368 		return 0;
7369 
7370 	resp = bp->hwrm_cmd_resp_addr;
7371 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7372 
7373 	mutex_lock(&bp->hwrm_cmd_lock);
7374 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7375 	if (rc)
7376 		goto hwrm_cfa_adv_qcaps_exit;
7377 
7378 	flags = le32_to_cpu(resp->flags);
7379 	if (flags &
7380 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7381 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7382 
7383 hwrm_cfa_adv_qcaps_exit:
7384 	mutex_unlock(&bp->hwrm_cmd_lock);
7385 	return rc;
7386 }
7387 
7388 static int __bnxt_alloc_fw_health(struct bnxt *bp)
7389 {
7390 	if (bp->fw_health)
7391 		return 0;
7392 
7393 	bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
7394 	if (!bp->fw_health)
7395 		return -ENOMEM;
7396 
7397 	return 0;
7398 }
7399 
7400 static int bnxt_alloc_fw_health(struct bnxt *bp)
7401 {
7402 	int rc;
7403 
7404 	if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
7405 	    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7406 		return 0;
7407 
7408 	rc = __bnxt_alloc_fw_health(bp);
7409 	if (rc) {
7410 		bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
7411 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7412 		return rc;
7413 	}
7414 
7415 	return 0;
7416 }
7417 
7418 static void __bnxt_map_fw_health_reg(struct bnxt *bp, u32 reg)
7419 {
7420 	writel(reg & BNXT_GRC_BASE_MASK, bp->bar0 +
7421 					 BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7422 					 BNXT_FW_HEALTH_WIN_MAP_OFF);
7423 }
7424 
7425 static void bnxt_try_map_fw_health_reg(struct bnxt *bp)
7426 {
7427 	void __iomem *hs;
7428 	u32 status_loc;
7429 	u32 reg_type;
7430 	u32 sig;
7431 
7432 	__bnxt_map_fw_health_reg(bp, HCOMM_STATUS_STRUCT_LOC);
7433 	hs = bp->bar0 + BNXT_FW_HEALTH_WIN_OFF(HCOMM_STATUS_STRUCT_LOC);
7434 
7435 	sig = readl(hs + offsetof(struct hcomm_status, sig_ver));
7436 	if ((sig & HCOMM_STATUS_SIGNATURE_MASK) != HCOMM_STATUS_SIGNATURE_VAL) {
7437 		if (bp->fw_health)
7438 			bp->fw_health->status_reliable = false;
7439 		return;
7440 	}
7441 
7442 	if (__bnxt_alloc_fw_health(bp)) {
7443 		netdev_warn(bp->dev, "no memory for firmware status checks\n");
7444 		return;
7445 	}
7446 
7447 	status_loc = readl(hs + offsetof(struct hcomm_status, fw_status_loc));
7448 	bp->fw_health->regs[BNXT_FW_HEALTH_REG] = status_loc;
7449 	reg_type = BNXT_FW_HEALTH_REG_TYPE(status_loc);
7450 	if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) {
7451 		__bnxt_map_fw_health_reg(bp, status_loc);
7452 		bp->fw_health->mapped_regs[BNXT_FW_HEALTH_REG] =
7453 			BNXT_FW_HEALTH_WIN_OFF(status_loc);
7454 	}
7455 
7456 	bp->fw_health->status_reliable = true;
7457 }
7458 
7459 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7460 {
7461 	struct bnxt_fw_health *fw_health = bp->fw_health;
7462 	u32 reg_base = 0xffffffff;
7463 	int i;
7464 
7465 	/* Only pre-map the monitoring GRC registers using window 3 */
7466 	for (i = 0; i < 4; i++) {
7467 		u32 reg = fw_health->regs[i];
7468 
7469 		if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7470 			continue;
7471 		if (reg_base == 0xffffffff)
7472 			reg_base = reg & BNXT_GRC_BASE_MASK;
7473 		if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7474 			return -ERANGE;
7475 		fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg);
7476 	}
7477 	if (reg_base == 0xffffffff)
7478 		return 0;
7479 
7480 	__bnxt_map_fw_health_reg(bp, reg_base);
7481 	return 0;
7482 }
7483 
7484 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7485 {
7486 	struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7487 	struct bnxt_fw_health *fw_health = bp->fw_health;
7488 	struct hwrm_error_recovery_qcfg_input req = {0};
7489 	int rc, i;
7490 
7491 	if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7492 		return 0;
7493 
7494 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7495 	mutex_lock(&bp->hwrm_cmd_lock);
7496 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7497 	if (rc)
7498 		goto err_recovery_out;
7499 	fw_health->flags = le32_to_cpu(resp->flags);
7500 	if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7501 	    !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7502 		rc = -EINVAL;
7503 		goto err_recovery_out;
7504 	}
7505 	fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7506 	fw_health->master_func_wait_dsecs =
7507 		le32_to_cpu(resp->master_func_wait_period);
7508 	fw_health->normal_func_wait_dsecs =
7509 		le32_to_cpu(resp->normal_func_wait_period);
7510 	fw_health->post_reset_wait_dsecs =
7511 		le32_to_cpu(resp->master_func_wait_period_after_reset);
7512 	fw_health->post_reset_max_wait_dsecs =
7513 		le32_to_cpu(resp->max_bailout_time_after_reset);
7514 	fw_health->regs[BNXT_FW_HEALTH_REG] =
7515 		le32_to_cpu(resp->fw_health_status_reg);
7516 	fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7517 		le32_to_cpu(resp->fw_heartbeat_reg);
7518 	fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7519 		le32_to_cpu(resp->fw_reset_cnt_reg);
7520 	fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7521 		le32_to_cpu(resp->reset_inprogress_reg);
7522 	fw_health->fw_reset_inprog_reg_mask =
7523 		le32_to_cpu(resp->reset_inprogress_reg_mask);
7524 	fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7525 	if (fw_health->fw_reset_seq_cnt >= 16) {
7526 		rc = -EINVAL;
7527 		goto err_recovery_out;
7528 	}
7529 	for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7530 		fw_health->fw_reset_seq_regs[i] =
7531 			le32_to_cpu(resp->reset_reg[i]);
7532 		fw_health->fw_reset_seq_vals[i] =
7533 			le32_to_cpu(resp->reset_reg_val[i]);
7534 		fw_health->fw_reset_seq_delay_msec[i] =
7535 			resp->delay_after_reset[i];
7536 	}
7537 err_recovery_out:
7538 	mutex_unlock(&bp->hwrm_cmd_lock);
7539 	if (!rc)
7540 		rc = bnxt_map_fw_health_regs(bp);
7541 	if (rc)
7542 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7543 	return rc;
7544 }
7545 
7546 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7547 {
7548 	struct hwrm_func_reset_input req = {0};
7549 
7550 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7551 	req.enables = 0;
7552 
7553 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7554 }
7555 
7556 static void bnxt_nvm_cfg_ver_get(struct bnxt *bp)
7557 {
7558 	struct hwrm_nvm_get_dev_info_output nvm_info;
7559 
7560 	if (!bnxt_hwrm_nvm_get_dev_info(bp, &nvm_info))
7561 		snprintf(bp->nvm_cfg_ver, FW_VER_STR_LEN, "%d.%d.%d",
7562 			 nvm_info.nvm_cfg_ver_maj, nvm_info.nvm_cfg_ver_min,
7563 			 nvm_info.nvm_cfg_ver_upd);
7564 }
7565 
7566 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7567 {
7568 	int rc = 0;
7569 	struct hwrm_queue_qportcfg_input req = {0};
7570 	struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7571 	u8 i, j, *qptr;
7572 	bool no_rdma;
7573 
7574 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7575 
7576 	mutex_lock(&bp->hwrm_cmd_lock);
7577 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7578 	if (rc)
7579 		goto qportcfg_exit;
7580 
7581 	if (!resp->max_configurable_queues) {
7582 		rc = -EINVAL;
7583 		goto qportcfg_exit;
7584 	}
7585 	bp->max_tc = resp->max_configurable_queues;
7586 	bp->max_lltc = resp->max_configurable_lossless_queues;
7587 	if (bp->max_tc > BNXT_MAX_QUEUE)
7588 		bp->max_tc = BNXT_MAX_QUEUE;
7589 
7590 	no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7591 	qptr = &resp->queue_id0;
7592 	for (i = 0, j = 0; i < bp->max_tc; i++) {
7593 		bp->q_info[j].queue_id = *qptr;
7594 		bp->q_ids[i] = *qptr++;
7595 		bp->q_info[j].queue_profile = *qptr++;
7596 		bp->tc_to_qidx[j] = j;
7597 		if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7598 		    (no_rdma && BNXT_PF(bp)))
7599 			j++;
7600 	}
7601 	bp->max_q = bp->max_tc;
7602 	bp->max_tc = max_t(u8, j, 1);
7603 
7604 	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7605 		bp->max_tc = 1;
7606 
7607 	if (bp->max_lltc > bp->max_tc)
7608 		bp->max_lltc = bp->max_tc;
7609 
7610 qportcfg_exit:
7611 	mutex_unlock(&bp->hwrm_cmd_lock);
7612 	return rc;
7613 }
7614 
7615 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7616 {
7617 	struct hwrm_ver_get_input req = {0};
7618 	int rc;
7619 
7620 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7621 	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7622 	req.hwrm_intf_min = HWRM_VERSION_MINOR;
7623 	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7624 
7625 	rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7626 				   silent);
7627 	return rc;
7628 }
7629 
7630 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7631 {
7632 	struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7633 	u16 fw_maj, fw_min, fw_bld, fw_rsv;
7634 	u32 dev_caps_cfg, hwrm_ver;
7635 	int rc, len;
7636 
7637 	bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7638 	mutex_lock(&bp->hwrm_cmd_lock);
7639 	rc = __bnxt_hwrm_ver_get(bp, false);
7640 	if (rc)
7641 		goto hwrm_ver_get_exit;
7642 
7643 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7644 
7645 	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7646 			     resp->hwrm_intf_min_8b << 8 |
7647 			     resp->hwrm_intf_upd_8b;
7648 	if (resp->hwrm_intf_maj_8b < 1) {
7649 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7650 			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7651 			    resp->hwrm_intf_upd_8b);
7652 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7653 	}
7654 
7655 	hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7656 			HWRM_VERSION_UPDATE;
7657 
7658 	if (bp->hwrm_spec_code > hwrm_ver)
7659 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7660 			 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7661 			 HWRM_VERSION_UPDATE);
7662 	else
7663 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7664 			 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7665 			 resp->hwrm_intf_upd_8b);
7666 
7667 	fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7668 	if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7669 		fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7670 		fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7671 		fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7672 		len = FW_VER_STR_LEN;
7673 	} else {
7674 		fw_maj = resp->hwrm_fw_maj_8b;
7675 		fw_min = resp->hwrm_fw_min_8b;
7676 		fw_bld = resp->hwrm_fw_bld_8b;
7677 		fw_rsv = resp->hwrm_fw_rsvd_8b;
7678 		len = BC_HWRM_STR_LEN;
7679 	}
7680 	bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7681 	snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7682 		 fw_rsv);
7683 
7684 	if (strlen(resp->active_pkg_name)) {
7685 		int fw_ver_len = strlen(bp->fw_ver_str);
7686 
7687 		snprintf(bp->fw_ver_str + fw_ver_len,
7688 			 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7689 			 resp->active_pkg_name);
7690 		bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7691 	}
7692 
7693 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7694 	if (!bp->hwrm_cmd_timeout)
7695 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7696 
7697 	if (resp->hwrm_intf_maj_8b >= 1) {
7698 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7699 		bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7700 	}
7701 	if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7702 		bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7703 
7704 	bp->chip_num = le16_to_cpu(resp->chip_num);
7705 	bp->chip_rev = resp->chip_rev;
7706 	if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7707 	    !resp->chip_metal)
7708 		bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7709 
7710 	dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7711 	if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7712 	    (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7713 		bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7714 
7715 	if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7716 		bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7717 
7718 	if (dev_caps_cfg &
7719 	    VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7720 		bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7721 
7722 	if (dev_caps_cfg &
7723 	    VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7724 		bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7725 
7726 	if (dev_caps_cfg &
7727 	    VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7728 		bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7729 
7730 hwrm_ver_get_exit:
7731 	mutex_unlock(&bp->hwrm_cmd_lock);
7732 	return rc;
7733 }
7734 
7735 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7736 {
7737 	struct hwrm_fw_set_time_input req = {0};
7738 	struct tm tm;
7739 	time64_t now = ktime_get_real_seconds();
7740 
7741 	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7742 	    bp->hwrm_spec_code < 0x10400)
7743 		return -EOPNOTSUPP;
7744 
7745 	time64_to_tm(now, 0, &tm);
7746 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7747 	req.year = cpu_to_le16(1900 + tm.tm_year);
7748 	req.month = 1 + tm.tm_mon;
7749 	req.day = tm.tm_mday;
7750 	req.hour = tm.tm_hour;
7751 	req.minute = tm.tm_min;
7752 	req.second = tm.tm_sec;
7753 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7754 }
7755 
7756 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7757 {
7758 	u64 sw_tmp;
7759 
7760 	sw_tmp = (*sw & ~mask) | hw;
7761 	if (hw < (*sw & mask))
7762 		sw_tmp += mask + 1;
7763 	WRITE_ONCE(*sw, sw_tmp);
7764 }
7765 
7766 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7767 				    int count, bool ignore_zero)
7768 {
7769 	int i;
7770 
7771 	for (i = 0; i < count; i++) {
7772 		u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7773 
7774 		if (ignore_zero && !hw)
7775 			continue;
7776 
7777 		if (masks[i] == -1ULL)
7778 			sw_stats[i] = hw;
7779 		else
7780 			bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7781 	}
7782 }
7783 
7784 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7785 {
7786 	if (!stats->hw_stats)
7787 		return;
7788 
7789 	__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7790 				stats->hw_masks, stats->len / 8, false);
7791 }
7792 
7793 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7794 {
7795 	struct bnxt_stats_mem *ring0_stats;
7796 	bool ignore_zero = false;
7797 	int i;
7798 
7799 	/* Chip bug.  Counter intermittently becomes 0. */
7800 	if (bp->flags & BNXT_FLAG_CHIP_P5)
7801 		ignore_zero = true;
7802 
7803 	for (i = 0; i < bp->cp_nr_rings; i++) {
7804 		struct bnxt_napi *bnapi = bp->bnapi[i];
7805 		struct bnxt_cp_ring_info *cpr;
7806 		struct bnxt_stats_mem *stats;
7807 
7808 		cpr = &bnapi->cp_ring;
7809 		stats = &cpr->stats;
7810 		if (!i)
7811 			ring0_stats = stats;
7812 		__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7813 					ring0_stats->hw_masks,
7814 					ring0_stats->len / 8, ignore_zero);
7815 	}
7816 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
7817 		struct bnxt_stats_mem *stats = &bp->port_stats;
7818 		__le64 *hw_stats = stats->hw_stats;
7819 		u64 *sw_stats = stats->sw_stats;
7820 		u64 *masks = stats->hw_masks;
7821 		int cnt;
7822 
7823 		cnt = sizeof(struct rx_port_stats) / 8;
7824 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7825 
7826 		hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7827 		sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7828 		masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7829 		cnt = sizeof(struct tx_port_stats) / 8;
7830 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7831 	}
7832 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7833 		bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7834 		bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7835 	}
7836 }
7837 
7838 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7839 {
7840 	struct bnxt_pf_info *pf = &bp->pf;
7841 	struct hwrm_port_qstats_input req = {0};
7842 
7843 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7844 		return 0;
7845 
7846 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7847 		return -EOPNOTSUPP;
7848 
7849 	req.flags = flags;
7850 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7851 	req.port_id = cpu_to_le16(pf->port_id);
7852 	req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7853 					    BNXT_TX_PORT_STATS_BYTE_OFFSET);
7854 	req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7855 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7856 }
7857 
7858 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7859 {
7860 	struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7861 	struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7862 	struct hwrm_port_qstats_ext_input req = {0};
7863 	struct bnxt_pf_info *pf = &bp->pf;
7864 	u32 tx_stat_size;
7865 	int rc;
7866 
7867 	if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7868 		return 0;
7869 
7870 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7871 		return -EOPNOTSUPP;
7872 
7873 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7874 	req.flags = flags;
7875 	req.port_id = cpu_to_le16(pf->port_id);
7876 	req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7877 	req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7878 	tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7879 		       sizeof(struct tx_port_stats_ext) : 0;
7880 	req.tx_stat_size = cpu_to_le16(tx_stat_size);
7881 	req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7882 	mutex_lock(&bp->hwrm_cmd_lock);
7883 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7884 	if (!rc) {
7885 		bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7886 		bp->fw_tx_stats_ext_size = tx_stat_size ?
7887 			le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7888 	} else {
7889 		bp->fw_rx_stats_ext_size = 0;
7890 		bp->fw_tx_stats_ext_size = 0;
7891 	}
7892 	if (flags)
7893 		goto qstats_done;
7894 
7895 	if (bp->fw_tx_stats_ext_size <=
7896 	    offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7897 		mutex_unlock(&bp->hwrm_cmd_lock);
7898 		bp->pri2cos_valid = 0;
7899 		return rc;
7900 	}
7901 
7902 	bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7903 	req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7904 
7905 	rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7906 	if (!rc) {
7907 		struct hwrm_queue_pri2cos_qcfg_output *resp2;
7908 		u8 *pri2cos;
7909 		int i, j;
7910 
7911 		resp2 = bp->hwrm_cmd_resp_addr;
7912 		pri2cos = &resp2->pri0_cos_queue_id;
7913 		for (i = 0; i < 8; i++) {
7914 			u8 queue_id = pri2cos[i];
7915 			u8 queue_idx;
7916 
7917 			/* Per port queue IDs start from 0, 10, 20, etc */
7918 			queue_idx = queue_id % 10;
7919 			if (queue_idx > BNXT_MAX_QUEUE) {
7920 				bp->pri2cos_valid = false;
7921 				goto qstats_done;
7922 			}
7923 			for (j = 0; j < bp->max_q; j++) {
7924 				if (bp->q_ids[j] == queue_id)
7925 					bp->pri2cos_idx[i] = queue_idx;
7926 			}
7927 		}
7928 		bp->pri2cos_valid = 1;
7929 	}
7930 qstats_done:
7931 	mutex_unlock(&bp->hwrm_cmd_lock);
7932 	return rc;
7933 }
7934 
7935 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7936 {
7937 	if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7938 		bnxt_hwrm_tunnel_dst_port_free(
7939 			bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7940 	if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7941 		bnxt_hwrm_tunnel_dst_port_free(
7942 			bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7943 }
7944 
7945 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7946 {
7947 	int rc, i;
7948 	u32 tpa_flags = 0;
7949 
7950 	if (set_tpa)
7951 		tpa_flags = bp->flags & BNXT_FLAG_TPA;
7952 	else if (BNXT_NO_FW_ACCESS(bp))
7953 		return 0;
7954 	for (i = 0; i < bp->nr_vnics; i++) {
7955 		rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7956 		if (rc) {
7957 			netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7958 				   i, rc);
7959 			return rc;
7960 		}
7961 	}
7962 	return 0;
7963 }
7964 
7965 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7966 {
7967 	int i;
7968 
7969 	for (i = 0; i < bp->nr_vnics; i++)
7970 		bnxt_hwrm_vnic_set_rss(bp, i, false);
7971 }
7972 
7973 static void bnxt_clear_vnic(struct bnxt *bp)
7974 {
7975 	if (!bp->vnic_info)
7976 		return;
7977 
7978 	bnxt_hwrm_clear_vnic_filter(bp);
7979 	if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7980 		/* clear all RSS setting before free vnic ctx */
7981 		bnxt_hwrm_clear_vnic_rss(bp);
7982 		bnxt_hwrm_vnic_ctx_free(bp);
7983 	}
7984 	/* before free the vnic, undo the vnic tpa settings */
7985 	if (bp->flags & BNXT_FLAG_TPA)
7986 		bnxt_set_tpa(bp, false);
7987 	bnxt_hwrm_vnic_free(bp);
7988 	if (bp->flags & BNXT_FLAG_CHIP_P5)
7989 		bnxt_hwrm_vnic_ctx_free(bp);
7990 }
7991 
7992 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7993 				    bool irq_re_init)
7994 {
7995 	bnxt_clear_vnic(bp);
7996 	bnxt_hwrm_ring_free(bp, close_path);
7997 	bnxt_hwrm_ring_grp_free(bp);
7998 	if (irq_re_init) {
7999 		bnxt_hwrm_stat_ctx_free(bp);
8000 		bnxt_hwrm_free_tunnel_ports(bp);
8001 	}
8002 }
8003 
8004 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
8005 {
8006 	struct hwrm_func_cfg_input req = {0};
8007 
8008 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8009 	req.fid = cpu_to_le16(0xffff);
8010 	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
8011 	if (br_mode == BRIDGE_MODE_VEB)
8012 		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
8013 	else if (br_mode == BRIDGE_MODE_VEPA)
8014 		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
8015 	else
8016 		return -EINVAL;
8017 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8018 }
8019 
8020 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
8021 {
8022 	struct hwrm_func_cfg_input req = {0};
8023 
8024 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
8025 		return 0;
8026 
8027 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
8028 	req.fid = cpu_to_le16(0xffff);
8029 	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
8030 	req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
8031 	if (size == 128)
8032 		req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
8033 
8034 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8035 }
8036 
8037 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8038 {
8039 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
8040 	int rc;
8041 
8042 	if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
8043 		goto skip_rss_ctx;
8044 
8045 	/* allocate context for vnic */
8046 	rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
8047 	if (rc) {
8048 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8049 			   vnic_id, rc);
8050 		goto vnic_setup_err;
8051 	}
8052 	bp->rsscos_nr_ctxs++;
8053 
8054 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8055 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
8056 		if (rc) {
8057 			netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
8058 				   vnic_id, rc);
8059 			goto vnic_setup_err;
8060 		}
8061 		bp->rsscos_nr_ctxs++;
8062 	}
8063 
8064 skip_rss_ctx:
8065 	/* configure default vnic, ring grp */
8066 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8067 	if (rc) {
8068 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8069 			   vnic_id, rc);
8070 		goto vnic_setup_err;
8071 	}
8072 
8073 	/* Enable RSS hashing on vnic */
8074 	rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
8075 	if (rc) {
8076 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
8077 			   vnic_id, rc);
8078 		goto vnic_setup_err;
8079 	}
8080 
8081 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8082 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8083 		if (rc) {
8084 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8085 				   vnic_id, rc);
8086 		}
8087 	}
8088 
8089 vnic_setup_err:
8090 	return rc;
8091 }
8092 
8093 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
8094 {
8095 	int rc, i, nr_ctxs;
8096 
8097 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
8098 	for (i = 0; i < nr_ctxs; i++) {
8099 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
8100 		if (rc) {
8101 			netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
8102 				   vnic_id, i, rc);
8103 			break;
8104 		}
8105 		bp->rsscos_nr_ctxs++;
8106 	}
8107 	if (i < nr_ctxs)
8108 		return -ENOMEM;
8109 
8110 	rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
8111 	if (rc) {
8112 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
8113 			   vnic_id, rc);
8114 		return rc;
8115 	}
8116 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
8117 	if (rc) {
8118 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
8119 			   vnic_id, rc);
8120 		return rc;
8121 	}
8122 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
8123 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
8124 		if (rc) {
8125 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
8126 				   vnic_id, rc);
8127 		}
8128 	}
8129 	return rc;
8130 }
8131 
8132 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8133 {
8134 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8135 		return __bnxt_setup_vnic_p5(bp, vnic_id);
8136 	else
8137 		return __bnxt_setup_vnic(bp, vnic_id);
8138 }
8139 
8140 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8141 {
8142 #ifdef CONFIG_RFS_ACCEL
8143 	int i, rc = 0;
8144 
8145 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8146 		return 0;
8147 
8148 	for (i = 0; i < bp->rx_nr_rings; i++) {
8149 		struct bnxt_vnic_info *vnic;
8150 		u16 vnic_id = i + 1;
8151 		u16 ring_id = i;
8152 
8153 		if (vnic_id >= bp->nr_vnics)
8154 			break;
8155 
8156 		vnic = &bp->vnic_info[vnic_id];
8157 		vnic->flags |= BNXT_VNIC_RFS_FLAG;
8158 		if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8159 			vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8160 		rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8161 		if (rc) {
8162 			netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8163 				   vnic_id, rc);
8164 			break;
8165 		}
8166 		rc = bnxt_setup_vnic(bp, vnic_id);
8167 		if (rc)
8168 			break;
8169 	}
8170 	return rc;
8171 #else
8172 	return 0;
8173 #endif
8174 }
8175 
8176 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8177 static bool bnxt_promisc_ok(struct bnxt *bp)
8178 {
8179 #ifdef CONFIG_BNXT_SRIOV
8180 	if (BNXT_VF(bp) && !bp->vf.vlan)
8181 		return false;
8182 #endif
8183 	return true;
8184 }
8185 
8186 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8187 {
8188 	unsigned int rc = 0;
8189 
8190 	rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8191 	if (rc) {
8192 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8193 			   rc);
8194 		return rc;
8195 	}
8196 
8197 	rc = bnxt_hwrm_vnic_cfg(bp, 1);
8198 	if (rc) {
8199 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8200 			   rc);
8201 		return rc;
8202 	}
8203 	return rc;
8204 }
8205 
8206 static int bnxt_cfg_rx_mode(struct bnxt *);
8207 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8208 
8209 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8210 {
8211 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8212 	int rc = 0;
8213 	unsigned int rx_nr_rings = bp->rx_nr_rings;
8214 
8215 	if (irq_re_init) {
8216 		rc = bnxt_hwrm_stat_ctx_alloc(bp);
8217 		if (rc) {
8218 			netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8219 				   rc);
8220 			goto err_out;
8221 		}
8222 	}
8223 
8224 	rc = bnxt_hwrm_ring_alloc(bp);
8225 	if (rc) {
8226 		netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8227 		goto err_out;
8228 	}
8229 
8230 	rc = bnxt_hwrm_ring_grp_alloc(bp);
8231 	if (rc) {
8232 		netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8233 		goto err_out;
8234 	}
8235 
8236 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8237 		rx_nr_rings--;
8238 
8239 	/* default vnic 0 */
8240 	rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8241 	if (rc) {
8242 		netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8243 		goto err_out;
8244 	}
8245 
8246 	rc = bnxt_setup_vnic(bp, 0);
8247 	if (rc)
8248 		goto err_out;
8249 
8250 	if (bp->flags & BNXT_FLAG_RFS) {
8251 		rc = bnxt_alloc_rfs_vnics(bp);
8252 		if (rc)
8253 			goto err_out;
8254 	}
8255 
8256 	if (bp->flags & BNXT_FLAG_TPA) {
8257 		rc = bnxt_set_tpa(bp, true);
8258 		if (rc)
8259 			goto err_out;
8260 	}
8261 
8262 	if (BNXT_VF(bp))
8263 		bnxt_update_vf_mac(bp);
8264 
8265 	/* Filter for default vnic 0 */
8266 	rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8267 	if (rc) {
8268 		netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8269 		goto err_out;
8270 	}
8271 	vnic->uc_filter_count = 1;
8272 
8273 	vnic->rx_mask = 0;
8274 	if (bp->dev->flags & IFF_BROADCAST)
8275 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8276 
8277 	if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8278 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8279 
8280 	if (bp->dev->flags & IFF_ALLMULTI) {
8281 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8282 		vnic->mc_list_count = 0;
8283 	} else {
8284 		u32 mask = 0;
8285 
8286 		bnxt_mc_list_updated(bp, &mask);
8287 		vnic->rx_mask |= mask;
8288 	}
8289 
8290 	rc = bnxt_cfg_rx_mode(bp);
8291 	if (rc)
8292 		goto err_out;
8293 
8294 	rc = bnxt_hwrm_set_coal(bp);
8295 	if (rc)
8296 		netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8297 				rc);
8298 
8299 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8300 		rc = bnxt_setup_nitroa0_vnic(bp);
8301 		if (rc)
8302 			netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8303 				   rc);
8304 	}
8305 
8306 	if (BNXT_VF(bp)) {
8307 		bnxt_hwrm_func_qcfg(bp);
8308 		netdev_update_features(bp->dev);
8309 	}
8310 
8311 	return 0;
8312 
8313 err_out:
8314 	bnxt_hwrm_resource_free(bp, 0, true);
8315 
8316 	return rc;
8317 }
8318 
8319 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8320 {
8321 	bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8322 	return 0;
8323 }
8324 
8325 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8326 {
8327 	bnxt_init_cp_rings(bp);
8328 	bnxt_init_rx_rings(bp);
8329 	bnxt_init_tx_rings(bp);
8330 	bnxt_init_ring_grps(bp, irq_re_init);
8331 	bnxt_init_vnics(bp);
8332 
8333 	return bnxt_init_chip(bp, irq_re_init);
8334 }
8335 
8336 static int bnxt_set_real_num_queues(struct bnxt *bp)
8337 {
8338 	int rc;
8339 	struct net_device *dev = bp->dev;
8340 
8341 	rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8342 					  bp->tx_nr_rings_xdp);
8343 	if (rc)
8344 		return rc;
8345 
8346 	rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8347 	if (rc)
8348 		return rc;
8349 
8350 #ifdef CONFIG_RFS_ACCEL
8351 	if (bp->flags & BNXT_FLAG_RFS)
8352 		dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8353 #endif
8354 
8355 	return rc;
8356 }
8357 
8358 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8359 			   bool shared)
8360 {
8361 	int _rx = *rx, _tx = *tx;
8362 
8363 	if (shared) {
8364 		*rx = min_t(int, _rx, max);
8365 		*tx = min_t(int, _tx, max);
8366 	} else {
8367 		if (max < 2)
8368 			return -ENOMEM;
8369 
8370 		while (_rx + _tx > max) {
8371 			if (_rx > _tx && _rx > 1)
8372 				_rx--;
8373 			else if (_tx > 1)
8374 				_tx--;
8375 		}
8376 		*rx = _rx;
8377 		*tx = _tx;
8378 	}
8379 	return 0;
8380 }
8381 
8382 static void bnxt_setup_msix(struct bnxt *bp)
8383 {
8384 	const int len = sizeof(bp->irq_tbl[0].name);
8385 	struct net_device *dev = bp->dev;
8386 	int tcs, i;
8387 
8388 	tcs = netdev_get_num_tc(dev);
8389 	if (tcs) {
8390 		int i, off, count;
8391 
8392 		for (i = 0; i < tcs; i++) {
8393 			count = bp->tx_nr_rings_per_tc;
8394 			off = i * count;
8395 			netdev_set_tc_queue(dev, i, count, off);
8396 		}
8397 	}
8398 
8399 	for (i = 0; i < bp->cp_nr_rings; i++) {
8400 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8401 		char *attr;
8402 
8403 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8404 			attr = "TxRx";
8405 		else if (i < bp->rx_nr_rings)
8406 			attr = "rx";
8407 		else
8408 			attr = "tx";
8409 
8410 		snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8411 			 attr, i);
8412 		bp->irq_tbl[map_idx].handler = bnxt_msix;
8413 	}
8414 }
8415 
8416 static void bnxt_setup_inta(struct bnxt *bp)
8417 {
8418 	const int len = sizeof(bp->irq_tbl[0].name);
8419 
8420 	if (netdev_get_num_tc(bp->dev))
8421 		netdev_reset_tc(bp->dev);
8422 
8423 	snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8424 		 0);
8425 	bp->irq_tbl[0].handler = bnxt_inta;
8426 }
8427 
8428 static int bnxt_setup_int_mode(struct bnxt *bp)
8429 {
8430 	int rc;
8431 
8432 	if (bp->flags & BNXT_FLAG_USING_MSIX)
8433 		bnxt_setup_msix(bp);
8434 	else
8435 		bnxt_setup_inta(bp);
8436 
8437 	rc = bnxt_set_real_num_queues(bp);
8438 	return rc;
8439 }
8440 
8441 #ifdef CONFIG_RFS_ACCEL
8442 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8443 {
8444 	return bp->hw_resc.max_rsscos_ctxs;
8445 }
8446 
8447 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8448 {
8449 	return bp->hw_resc.max_vnics;
8450 }
8451 #endif
8452 
8453 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8454 {
8455 	return bp->hw_resc.max_stat_ctxs;
8456 }
8457 
8458 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8459 {
8460 	return bp->hw_resc.max_cp_rings;
8461 }
8462 
8463 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8464 {
8465 	unsigned int cp = bp->hw_resc.max_cp_rings;
8466 
8467 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8468 		cp -= bnxt_get_ulp_msix_num(bp);
8469 
8470 	return cp;
8471 }
8472 
8473 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8474 {
8475 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8476 
8477 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8478 		return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8479 
8480 	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8481 }
8482 
8483 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8484 {
8485 	bp->hw_resc.max_irqs = max_irqs;
8486 }
8487 
8488 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8489 {
8490 	unsigned int cp;
8491 
8492 	cp = bnxt_get_max_func_cp_rings_for_en(bp);
8493 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8494 		return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8495 	else
8496 		return cp - bp->cp_nr_rings;
8497 }
8498 
8499 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8500 {
8501 	return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8502 }
8503 
8504 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8505 {
8506 	int max_cp = bnxt_get_max_func_cp_rings(bp);
8507 	int max_irq = bnxt_get_max_func_irqs(bp);
8508 	int total_req = bp->cp_nr_rings + num;
8509 	int max_idx, avail_msix;
8510 
8511 	max_idx = bp->total_irqs;
8512 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8513 		max_idx = min_t(int, bp->total_irqs, max_cp);
8514 	avail_msix = max_idx - bp->cp_nr_rings;
8515 	if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8516 		return avail_msix;
8517 
8518 	if (max_irq < total_req) {
8519 		num = max_irq - bp->cp_nr_rings;
8520 		if (num <= 0)
8521 			return 0;
8522 	}
8523 	return num;
8524 }
8525 
8526 static int bnxt_get_num_msix(struct bnxt *bp)
8527 {
8528 	if (!BNXT_NEW_RM(bp))
8529 		return bnxt_get_max_func_irqs(bp);
8530 
8531 	return bnxt_nq_rings_in_use(bp);
8532 }
8533 
8534 static int bnxt_init_msix(struct bnxt *bp)
8535 {
8536 	int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8537 	struct msix_entry *msix_ent;
8538 
8539 	total_vecs = bnxt_get_num_msix(bp);
8540 	max = bnxt_get_max_func_irqs(bp);
8541 	if (total_vecs > max)
8542 		total_vecs = max;
8543 
8544 	if (!total_vecs)
8545 		return 0;
8546 
8547 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8548 	if (!msix_ent)
8549 		return -ENOMEM;
8550 
8551 	for (i = 0; i < total_vecs; i++) {
8552 		msix_ent[i].entry = i;
8553 		msix_ent[i].vector = 0;
8554 	}
8555 
8556 	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8557 		min = 2;
8558 
8559 	total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8560 	ulp_msix = bnxt_get_ulp_msix_num(bp);
8561 	if (total_vecs < 0 || total_vecs < ulp_msix) {
8562 		rc = -ENODEV;
8563 		goto msix_setup_exit;
8564 	}
8565 
8566 	bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8567 	if (bp->irq_tbl) {
8568 		for (i = 0; i < total_vecs; i++)
8569 			bp->irq_tbl[i].vector = msix_ent[i].vector;
8570 
8571 		bp->total_irqs = total_vecs;
8572 		/* Trim rings based upon num of vectors allocated */
8573 		rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8574 				     total_vecs - ulp_msix, min == 1);
8575 		if (rc)
8576 			goto msix_setup_exit;
8577 
8578 		bp->cp_nr_rings = (min == 1) ?
8579 				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8580 				  bp->tx_nr_rings + bp->rx_nr_rings;
8581 
8582 	} else {
8583 		rc = -ENOMEM;
8584 		goto msix_setup_exit;
8585 	}
8586 	bp->flags |= BNXT_FLAG_USING_MSIX;
8587 	kfree(msix_ent);
8588 	return 0;
8589 
8590 msix_setup_exit:
8591 	netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8592 	kfree(bp->irq_tbl);
8593 	bp->irq_tbl = NULL;
8594 	pci_disable_msix(bp->pdev);
8595 	kfree(msix_ent);
8596 	return rc;
8597 }
8598 
8599 static int bnxt_init_inta(struct bnxt *bp)
8600 {
8601 	bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8602 	if (!bp->irq_tbl)
8603 		return -ENOMEM;
8604 
8605 	bp->total_irqs = 1;
8606 	bp->rx_nr_rings = 1;
8607 	bp->tx_nr_rings = 1;
8608 	bp->cp_nr_rings = 1;
8609 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
8610 	bp->irq_tbl[0].vector = bp->pdev->irq;
8611 	return 0;
8612 }
8613 
8614 static int bnxt_init_int_mode(struct bnxt *bp)
8615 {
8616 	int rc = 0;
8617 
8618 	if (bp->flags & BNXT_FLAG_MSIX_CAP)
8619 		rc = bnxt_init_msix(bp);
8620 
8621 	if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8622 		/* fallback to INTA */
8623 		rc = bnxt_init_inta(bp);
8624 	}
8625 	return rc;
8626 }
8627 
8628 static void bnxt_clear_int_mode(struct bnxt *bp)
8629 {
8630 	if (bp->flags & BNXT_FLAG_USING_MSIX)
8631 		pci_disable_msix(bp->pdev);
8632 
8633 	kfree(bp->irq_tbl);
8634 	bp->irq_tbl = NULL;
8635 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
8636 }
8637 
8638 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8639 {
8640 	int tcs = netdev_get_num_tc(bp->dev);
8641 	bool irq_cleared = false;
8642 	int rc;
8643 
8644 	if (!bnxt_need_reserve_rings(bp))
8645 		return 0;
8646 
8647 	if (irq_re_init && BNXT_NEW_RM(bp) &&
8648 	    bnxt_get_num_msix(bp) != bp->total_irqs) {
8649 		bnxt_ulp_irq_stop(bp);
8650 		bnxt_clear_int_mode(bp);
8651 		irq_cleared = true;
8652 	}
8653 	rc = __bnxt_reserve_rings(bp);
8654 	if (irq_cleared) {
8655 		if (!rc)
8656 			rc = bnxt_init_int_mode(bp);
8657 		bnxt_ulp_irq_restart(bp, rc);
8658 	}
8659 	if (rc) {
8660 		netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8661 		return rc;
8662 	}
8663 	if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8664 		netdev_err(bp->dev, "tx ring reservation failure\n");
8665 		netdev_reset_tc(bp->dev);
8666 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8667 		return -ENOMEM;
8668 	}
8669 	return 0;
8670 }
8671 
8672 static void bnxt_free_irq(struct bnxt *bp)
8673 {
8674 	struct bnxt_irq *irq;
8675 	int i;
8676 
8677 #ifdef CONFIG_RFS_ACCEL
8678 	free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8679 	bp->dev->rx_cpu_rmap = NULL;
8680 #endif
8681 	if (!bp->irq_tbl || !bp->bnapi)
8682 		return;
8683 
8684 	for (i = 0; i < bp->cp_nr_rings; i++) {
8685 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8686 
8687 		irq = &bp->irq_tbl[map_idx];
8688 		if (irq->requested) {
8689 			if (irq->have_cpumask) {
8690 				irq_set_affinity_hint(irq->vector, NULL);
8691 				free_cpumask_var(irq->cpu_mask);
8692 				irq->have_cpumask = 0;
8693 			}
8694 			free_irq(irq->vector, bp->bnapi[i]);
8695 		}
8696 
8697 		irq->requested = 0;
8698 	}
8699 }
8700 
8701 static int bnxt_request_irq(struct bnxt *bp)
8702 {
8703 	int i, j, rc = 0;
8704 	unsigned long flags = 0;
8705 #ifdef CONFIG_RFS_ACCEL
8706 	struct cpu_rmap *rmap;
8707 #endif
8708 
8709 	rc = bnxt_setup_int_mode(bp);
8710 	if (rc) {
8711 		netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8712 			   rc);
8713 		return rc;
8714 	}
8715 #ifdef CONFIG_RFS_ACCEL
8716 	rmap = bp->dev->rx_cpu_rmap;
8717 #endif
8718 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8719 		flags = IRQF_SHARED;
8720 
8721 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8722 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8723 		struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8724 
8725 #ifdef CONFIG_RFS_ACCEL
8726 		if (rmap && bp->bnapi[i]->rx_ring) {
8727 			rc = irq_cpu_rmap_add(rmap, irq->vector);
8728 			if (rc)
8729 				netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8730 					    j);
8731 			j++;
8732 		}
8733 #endif
8734 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8735 				 bp->bnapi[i]);
8736 		if (rc)
8737 			break;
8738 
8739 		irq->requested = 1;
8740 
8741 		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8742 			int numa_node = dev_to_node(&bp->pdev->dev);
8743 
8744 			irq->have_cpumask = 1;
8745 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8746 					irq->cpu_mask);
8747 			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8748 			if (rc) {
8749 				netdev_warn(bp->dev,
8750 					    "Set affinity failed, IRQ = %d\n",
8751 					    irq->vector);
8752 				break;
8753 			}
8754 		}
8755 	}
8756 	return rc;
8757 }
8758 
8759 static void bnxt_del_napi(struct bnxt *bp)
8760 {
8761 	int i;
8762 
8763 	if (!bp->bnapi)
8764 		return;
8765 
8766 	for (i = 0; i < bp->cp_nr_rings; i++) {
8767 		struct bnxt_napi *bnapi = bp->bnapi[i];
8768 
8769 		__netif_napi_del(&bnapi->napi);
8770 	}
8771 	/* We called __netif_napi_del(), we need
8772 	 * to respect an RCU grace period before freeing napi structures.
8773 	 */
8774 	synchronize_net();
8775 }
8776 
8777 static void bnxt_init_napi(struct bnxt *bp)
8778 {
8779 	int i;
8780 	unsigned int cp_nr_rings = bp->cp_nr_rings;
8781 	struct bnxt_napi *bnapi;
8782 
8783 	if (bp->flags & BNXT_FLAG_USING_MSIX) {
8784 		int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8785 
8786 		if (bp->flags & BNXT_FLAG_CHIP_P5)
8787 			poll_fn = bnxt_poll_p5;
8788 		else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8789 			cp_nr_rings--;
8790 		for (i = 0; i < cp_nr_rings; i++) {
8791 			bnapi = bp->bnapi[i];
8792 			netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8793 		}
8794 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8795 			bnapi = bp->bnapi[cp_nr_rings];
8796 			netif_napi_add(bp->dev, &bnapi->napi,
8797 				       bnxt_poll_nitroa0, 64);
8798 		}
8799 	} else {
8800 		bnapi = bp->bnapi[0];
8801 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8802 	}
8803 }
8804 
8805 static void bnxt_disable_napi(struct bnxt *bp)
8806 {
8807 	int i;
8808 
8809 	if (!bp->bnapi)
8810 		return;
8811 
8812 	for (i = 0; i < bp->cp_nr_rings; i++) {
8813 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8814 
8815 		if (bp->bnapi[i]->rx_ring)
8816 			cancel_work_sync(&cpr->dim.work);
8817 
8818 		napi_disable(&bp->bnapi[i]->napi);
8819 	}
8820 }
8821 
8822 static void bnxt_enable_napi(struct bnxt *bp)
8823 {
8824 	int i;
8825 
8826 	for (i = 0; i < bp->cp_nr_rings; i++) {
8827 		struct bnxt_napi *bnapi = bp->bnapi[i];
8828 		struct bnxt_cp_ring_info *cpr;
8829 
8830 		cpr = &bnapi->cp_ring;
8831 		if (bnapi->in_reset)
8832 			cpr->sw_stats.rx.rx_resets++;
8833 		bnapi->in_reset = false;
8834 
8835 		if (bnapi->rx_ring) {
8836 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8837 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8838 		}
8839 		napi_enable(&bnapi->napi);
8840 	}
8841 }
8842 
8843 void bnxt_tx_disable(struct bnxt *bp)
8844 {
8845 	int i;
8846 	struct bnxt_tx_ring_info *txr;
8847 
8848 	if (bp->tx_ring) {
8849 		for (i = 0; i < bp->tx_nr_rings; i++) {
8850 			txr = &bp->tx_ring[i];
8851 			txr->dev_state = BNXT_DEV_STATE_CLOSING;
8852 		}
8853 	}
8854 	/* Stop all TX queues */
8855 	netif_tx_disable(bp->dev);
8856 	netif_carrier_off(bp->dev);
8857 }
8858 
8859 void bnxt_tx_enable(struct bnxt *bp)
8860 {
8861 	int i;
8862 	struct bnxt_tx_ring_info *txr;
8863 
8864 	for (i = 0; i < bp->tx_nr_rings; i++) {
8865 		txr = &bp->tx_ring[i];
8866 		txr->dev_state = 0;
8867 	}
8868 	netif_tx_wake_all_queues(bp->dev);
8869 	if (bp->link_info.link_up)
8870 		netif_carrier_on(bp->dev);
8871 }
8872 
8873 static char *bnxt_report_fec(struct bnxt_link_info *link_info)
8874 {
8875 	u8 active_fec = link_info->active_fec_sig_mode &
8876 			PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK;
8877 
8878 	switch (active_fec) {
8879 	default:
8880 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE:
8881 		return "None";
8882 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE:
8883 		return "Clause 74 BaseR";
8884 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE:
8885 		return "Clause 91 RS(528,514)";
8886 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE:
8887 		return "Clause 91 RS544_1XN";
8888 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE:
8889 		return "Clause 91 RS(544,514)";
8890 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE:
8891 		return "Clause 91 RS272_1XN";
8892 	case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE:
8893 		return "Clause 91 RS(272,257)";
8894 	}
8895 }
8896 
8897 static void bnxt_report_link(struct bnxt *bp)
8898 {
8899 	if (bp->link_info.link_up) {
8900 		const char *duplex;
8901 		const char *flow_ctrl;
8902 		u32 speed;
8903 		u16 fec;
8904 
8905 		netif_carrier_on(bp->dev);
8906 		speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8907 		if (speed == SPEED_UNKNOWN) {
8908 			netdev_info(bp->dev, "NIC Link is Up, speed unknown\n");
8909 			return;
8910 		}
8911 		if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8912 			duplex = "full";
8913 		else
8914 			duplex = "half";
8915 		if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8916 			flow_ctrl = "ON - receive & transmit";
8917 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8918 			flow_ctrl = "ON - transmit";
8919 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8920 			flow_ctrl = "ON - receive";
8921 		else
8922 			flow_ctrl = "none";
8923 		netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8924 			    speed, duplex, flow_ctrl);
8925 		if (bp->flags & BNXT_FLAG_EEE_CAP)
8926 			netdev_info(bp->dev, "EEE is %s\n",
8927 				    bp->eee.eee_active ? "active" :
8928 							 "not active");
8929 		fec = bp->link_info.fec_cfg;
8930 		if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8931 			netdev_info(bp->dev, "FEC autoneg %s encoding: %s\n",
8932 				    (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8933 				    bnxt_report_fec(&bp->link_info));
8934 	} else {
8935 		netif_carrier_off(bp->dev);
8936 		netdev_err(bp->dev, "NIC Link is Down\n");
8937 	}
8938 }
8939 
8940 static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp)
8941 {
8942 	if (!resp->supported_speeds_auto_mode &&
8943 	    !resp->supported_speeds_force_mode &&
8944 	    !resp->supported_pam4_speeds_auto_mode &&
8945 	    !resp->supported_pam4_speeds_force_mode)
8946 		return true;
8947 	return false;
8948 }
8949 
8950 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8951 {
8952 	int rc = 0;
8953 	struct hwrm_port_phy_qcaps_input req = {0};
8954 	struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8955 	struct bnxt_link_info *link_info = &bp->link_info;
8956 
8957 	bp->flags &= ~BNXT_FLAG_EEE_CAP;
8958 	if (bp->test_info)
8959 		bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8960 					  BNXT_TEST_FL_AN_PHY_LPBK);
8961 	if (bp->hwrm_spec_code < 0x10201)
8962 		return 0;
8963 
8964 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8965 
8966 	mutex_lock(&bp->hwrm_cmd_lock);
8967 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8968 	if (rc)
8969 		goto hwrm_phy_qcaps_exit;
8970 
8971 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8972 		struct ethtool_eee *eee = &bp->eee;
8973 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8974 
8975 		bp->flags |= BNXT_FLAG_EEE_CAP;
8976 		eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8977 		bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8978 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8979 		bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8980 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8981 	}
8982 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8983 		if (bp->test_info)
8984 			bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8985 	}
8986 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8987 		if (bp->test_info)
8988 			bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8989 	}
8990 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8991 		if (BNXT_PF(bp))
8992 			bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8993 	}
8994 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8995 		bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8996 
8997 	if (bp->hwrm_spec_code >= 0x10a01) {
8998 		if (bnxt_phy_qcaps_no_speed(resp)) {
8999 			link_info->phy_state = BNXT_PHY_STATE_DISABLED;
9000 			netdev_warn(bp->dev, "Ethernet link disabled\n");
9001 		} else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) {
9002 			link_info->phy_state = BNXT_PHY_STATE_ENABLED;
9003 			netdev_info(bp->dev, "Ethernet link enabled\n");
9004 			/* Phy re-enabled, reprobe the speeds */
9005 			link_info->support_auto_speeds = 0;
9006 			link_info->support_pam4_auto_speeds = 0;
9007 		}
9008 	}
9009 	if (resp->supported_speeds_auto_mode)
9010 		link_info->support_auto_speeds =
9011 			le16_to_cpu(resp->supported_speeds_auto_mode);
9012 	if (resp->supported_pam4_speeds_auto_mode)
9013 		link_info->support_pam4_auto_speeds =
9014 			le16_to_cpu(resp->supported_pam4_speeds_auto_mode);
9015 
9016 	bp->port_count = resp->port_cnt;
9017 
9018 hwrm_phy_qcaps_exit:
9019 	mutex_unlock(&bp->hwrm_cmd_lock);
9020 	return rc;
9021 }
9022 
9023 static bool bnxt_support_dropped(u16 advertising, u16 supported)
9024 {
9025 	u16 diff = advertising ^ supported;
9026 
9027 	return ((supported | diff) != supported);
9028 }
9029 
9030 int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
9031 {
9032 	int rc = 0;
9033 	struct bnxt_link_info *link_info = &bp->link_info;
9034 	struct hwrm_port_phy_qcfg_input req = {0};
9035 	struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9036 	u8 link_up = link_info->link_up;
9037 	bool support_changed = false;
9038 
9039 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
9040 
9041 	mutex_lock(&bp->hwrm_cmd_lock);
9042 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9043 	if (rc) {
9044 		mutex_unlock(&bp->hwrm_cmd_lock);
9045 		return rc;
9046 	}
9047 
9048 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
9049 	link_info->phy_link_status = resp->link;
9050 	link_info->duplex = resp->duplex_cfg;
9051 	if (bp->hwrm_spec_code >= 0x10800)
9052 		link_info->duplex = resp->duplex_state;
9053 	link_info->pause = resp->pause;
9054 	link_info->auto_mode = resp->auto_mode;
9055 	link_info->auto_pause_setting = resp->auto_pause;
9056 	link_info->lp_pause = resp->link_partner_adv_pause;
9057 	link_info->force_pause_setting = resp->force_pause;
9058 	link_info->duplex_setting = resp->duplex_cfg;
9059 	if (link_info->phy_link_status == BNXT_LINK_LINK)
9060 		link_info->link_speed = le16_to_cpu(resp->link_speed);
9061 	else
9062 		link_info->link_speed = 0;
9063 	link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
9064 	link_info->force_pam4_link_speed =
9065 		le16_to_cpu(resp->force_pam4_link_speed);
9066 	link_info->support_speeds = le16_to_cpu(resp->support_speeds);
9067 	link_info->support_pam4_speeds = le16_to_cpu(resp->support_pam4_speeds);
9068 	link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
9069 	link_info->auto_pam4_link_speeds =
9070 		le16_to_cpu(resp->auto_pam4_link_speed_mask);
9071 	link_info->lp_auto_link_speeds =
9072 		le16_to_cpu(resp->link_partner_adv_speeds);
9073 	link_info->lp_auto_pam4_link_speeds =
9074 		resp->link_partner_pam4_adv_speeds;
9075 	link_info->preemphasis = le32_to_cpu(resp->preemphasis);
9076 	link_info->phy_ver[0] = resp->phy_maj;
9077 	link_info->phy_ver[1] = resp->phy_min;
9078 	link_info->phy_ver[2] = resp->phy_bld;
9079 	link_info->media_type = resp->media_type;
9080 	link_info->phy_type = resp->phy_type;
9081 	link_info->transceiver = resp->xcvr_pkg_type;
9082 	link_info->phy_addr = resp->eee_config_phy_addr &
9083 			      PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
9084 	link_info->module_status = resp->module_status;
9085 
9086 	if (bp->flags & BNXT_FLAG_EEE_CAP) {
9087 		struct ethtool_eee *eee = &bp->eee;
9088 		u16 fw_speeds;
9089 
9090 		eee->eee_active = 0;
9091 		if (resp->eee_config_phy_addr &
9092 		    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
9093 			eee->eee_active = 1;
9094 			fw_speeds = le16_to_cpu(
9095 				resp->link_partner_adv_eee_link_speed_mask);
9096 			eee->lp_advertised =
9097 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9098 		}
9099 
9100 		/* Pull initial EEE config */
9101 		if (!chng_link_state) {
9102 			if (resp->eee_config_phy_addr &
9103 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
9104 				eee->eee_enabled = 1;
9105 
9106 			fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
9107 			eee->advertised =
9108 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
9109 
9110 			if (resp->eee_config_phy_addr &
9111 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
9112 				__le32 tmr;
9113 
9114 				eee->tx_lpi_enabled = 1;
9115 				tmr = resp->xcvr_identifier_type_tx_lpi_timer;
9116 				eee->tx_lpi_timer = le32_to_cpu(tmr) &
9117 					PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
9118 			}
9119 		}
9120 	}
9121 
9122 	link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
9123 	if (bp->hwrm_spec_code >= 0x10504) {
9124 		link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
9125 		link_info->active_fec_sig_mode = resp->active_fec_signal_mode;
9126 	}
9127 	/* TODO: need to add more logic to report VF link */
9128 	if (chng_link_state) {
9129 		if (link_info->phy_link_status == BNXT_LINK_LINK)
9130 			link_info->link_up = 1;
9131 		else
9132 			link_info->link_up = 0;
9133 		if (link_up != link_info->link_up)
9134 			bnxt_report_link(bp);
9135 	} else {
9136 		/* alwasy link down if not require to update link state */
9137 		link_info->link_up = 0;
9138 	}
9139 	mutex_unlock(&bp->hwrm_cmd_lock);
9140 
9141 	if (!BNXT_PHY_CFG_ABLE(bp))
9142 		return 0;
9143 
9144 	/* Check if any advertised speeds are no longer supported. The caller
9145 	 * holds the link_lock mutex, so we can modify link_info settings.
9146 	 */
9147 	if (bnxt_support_dropped(link_info->advertising,
9148 				 link_info->support_auto_speeds)) {
9149 		link_info->advertising = link_info->support_auto_speeds;
9150 		support_changed = true;
9151 	}
9152 	if (bnxt_support_dropped(link_info->advertising_pam4,
9153 				 link_info->support_pam4_auto_speeds)) {
9154 		link_info->advertising_pam4 = link_info->support_pam4_auto_speeds;
9155 		support_changed = true;
9156 	}
9157 	if (support_changed && (link_info->autoneg & BNXT_AUTONEG_SPEED))
9158 		bnxt_hwrm_set_link_setting(bp, true, false);
9159 	return 0;
9160 }
9161 
9162 static void bnxt_get_port_module_status(struct bnxt *bp)
9163 {
9164 	struct bnxt_link_info *link_info = &bp->link_info;
9165 	struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
9166 	u8 module_status;
9167 
9168 	if (bnxt_update_link(bp, true))
9169 		return;
9170 
9171 	module_status = link_info->module_status;
9172 	switch (module_status) {
9173 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
9174 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
9175 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
9176 		netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
9177 			    bp->pf.port_id);
9178 		if (bp->hwrm_spec_code >= 0x10201) {
9179 			netdev_warn(bp->dev, "Module part number %s\n",
9180 				    resp->phy_vendor_partnumber);
9181 		}
9182 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
9183 			netdev_warn(bp->dev, "TX is disabled\n");
9184 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
9185 			netdev_warn(bp->dev, "SFP+ module is shutdown\n");
9186 	}
9187 }
9188 
9189 static void
9190 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9191 {
9192 	if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
9193 		if (bp->hwrm_spec_code >= 0x10201)
9194 			req->auto_pause =
9195 				PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
9196 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9197 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
9198 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9199 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
9200 		req->enables |=
9201 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9202 	} else {
9203 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
9204 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
9205 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
9206 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9207 		req->enables |=
9208 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9209 		if (bp->hwrm_spec_code >= 0x10201) {
9210 			req->auto_pause = req->force_pause;
9211 			req->enables |= cpu_to_le32(
9212 				PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9213 		}
9214 	}
9215 }
9216 
9217 static void bnxt_hwrm_set_link_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
9218 {
9219 	if (bp->link_info.autoneg & BNXT_AUTONEG_SPEED) {
9220 		req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9221 		if (bp->link_info.advertising) {
9222 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9223 			req->auto_link_speed_mask = cpu_to_le16(bp->link_info.advertising);
9224 		}
9225 		if (bp->link_info.advertising_pam4) {
9226 			req->enables |=
9227 				cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAM4_LINK_SPEED_MASK);
9228 			req->auto_link_pam4_speed_mask =
9229 				cpu_to_le16(bp->link_info.advertising_pam4);
9230 		}
9231 		req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9232 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9233 	} else {
9234 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9235 		if (bp->link_info.req_signal_mode == BNXT_SIG_MODE_PAM4) {
9236 			req->force_pam4_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9237 			req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAM4_LINK_SPEED);
9238 		} else {
9239 			req->force_link_speed = cpu_to_le16(bp->link_info.req_link_speed);
9240 		}
9241 	}
9242 
9243 	/* tell chimp that the setting takes effect immediately */
9244 	req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9245 }
9246 
9247 int bnxt_hwrm_set_pause(struct bnxt *bp)
9248 {
9249 	struct hwrm_port_phy_cfg_input req = {0};
9250 	int rc;
9251 
9252 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9253 	bnxt_hwrm_set_pause_common(bp, &req);
9254 
9255 	if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9256 	    bp->link_info.force_link_chng)
9257 		bnxt_hwrm_set_link_common(bp, &req);
9258 
9259 	mutex_lock(&bp->hwrm_cmd_lock);
9260 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9261 	if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9262 		/* since changing of pause setting doesn't trigger any link
9263 		 * change event, the driver needs to update the current pause
9264 		 * result upon successfully return of the phy_cfg command
9265 		 */
9266 		bp->link_info.pause =
9267 		bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9268 		bp->link_info.auto_pause_setting = 0;
9269 		if (!bp->link_info.force_link_chng)
9270 			bnxt_report_link(bp);
9271 	}
9272 	bp->link_info.force_link_chng = false;
9273 	mutex_unlock(&bp->hwrm_cmd_lock);
9274 	return rc;
9275 }
9276 
9277 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9278 			      struct hwrm_port_phy_cfg_input *req)
9279 {
9280 	struct ethtool_eee *eee = &bp->eee;
9281 
9282 	if (eee->eee_enabled) {
9283 		u16 eee_speeds;
9284 		u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9285 
9286 		if (eee->tx_lpi_enabled)
9287 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9288 		else
9289 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9290 
9291 		req->flags |= cpu_to_le32(flags);
9292 		eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9293 		req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9294 		req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9295 	} else {
9296 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9297 	}
9298 }
9299 
9300 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9301 {
9302 	struct hwrm_port_phy_cfg_input req = {0};
9303 
9304 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9305 	if (set_pause)
9306 		bnxt_hwrm_set_pause_common(bp, &req);
9307 
9308 	bnxt_hwrm_set_link_common(bp, &req);
9309 
9310 	if (set_eee)
9311 		bnxt_hwrm_set_eee(bp, &req);
9312 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9313 }
9314 
9315 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9316 {
9317 	struct hwrm_port_phy_cfg_input req = {0};
9318 
9319 	if (!BNXT_SINGLE_PF(bp))
9320 		return 0;
9321 
9322 	if (pci_num_vf(bp->pdev))
9323 		return 0;
9324 
9325 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9326 	req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9327 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9328 }
9329 
9330 static int bnxt_fw_init_one(struct bnxt *bp);
9331 
9332 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9333 {
9334 	struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9335 	struct hwrm_func_drv_if_change_input req = {0};
9336 	bool resc_reinit = false, fw_reset = false;
9337 	u32 flags = 0;
9338 	int rc;
9339 
9340 	if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9341 		return 0;
9342 
9343 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9344 	if (up)
9345 		req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9346 	mutex_lock(&bp->hwrm_cmd_lock);
9347 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9348 	if (!rc)
9349 		flags = le32_to_cpu(resp->flags);
9350 	mutex_unlock(&bp->hwrm_cmd_lock);
9351 	if (rc)
9352 		return rc;
9353 
9354 	if (!up)
9355 		return 0;
9356 
9357 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9358 		resc_reinit = true;
9359 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9360 		fw_reset = true;
9361 
9362 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9363 		netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9364 		return -ENODEV;
9365 	}
9366 	if (resc_reinit || fw_reset) {
9367 		if (fw_reset) {
9368 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9369 				bnxt_ulp_stop(bp);
9370 			bnxt_free_ctx_mem(bp);
9371 			kfree(bp->ctx);
9372 			bp->ctx = NULL;
9373 			bnxt_dcb_free(bp);
9374 			rc = bnxt_fw_init_one(bp);
9375 			if (rc) {
9376 				set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9377 				return rc;
9378 			}
9379 			bnxt_clear_int_mode(bp);
9380 			rc = bnxt_init_int_mode(bp);
9381 			if (rc) {
9382 				netdev_err(bp->dev, "init int mode failed\n");
9383 				return rc;
9384 			}
9385 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9386 		}
9387 		if (BNXT_NEW_RM(bp)) {
9388 			struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9389 
9390 			rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9391 			hw_resc->resv_cp_rings = 0;
9392 			hw_resc->resv_stat_ctxs = 0;
9393 			hw_resc->resv_irqs = 0;
9394 			hw_resc->resv_tx_rings = 0;
9395 			hw_resc->resv_rx_rings = 0;
9396 			hw_resc->resv_hw_ring_grps = 0;
9397 			hw_resc->resv_vnics = 0;
9398 			if (!fw_reset) {
9399 				bp->tx_nr_rings = 0;
9400 				bp->rx_nr_rings = 0;
9401 			}
9402 		}
9403 	}
9404 	return 0;
9405 }
9406 
9407 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9408 {
9409 	struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9410 	struct hwrm_port_led_qcaps_input req = {0};
9411 	struct bnxt_pf_info *pf = &bp->pf;
9412 	int rc;
9413 
9414 	bp->num_leds = 0;
9415 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9416 		return 0;
9417 
9418 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9419 	req.port_id = cpu_to_le16(pf->port_id);
9420 	mutex_lock(&bp->hwrm_cmd_lock);
9421 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9422 	if (rc) {
9423 		mutex_unlock(&bp->hwrm_cmd_lock);
9424 		return rc;
9425 	}
9426 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9427 		int i;
9428 
9429 		bp->num_leds = resp->num_leds;
9430 		memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9431 						 bp->num_leds);
9432 		for (i = 0; i < bp->num_leds; i++) {
9433 			struct bnxt_led_info *led = &bp->leds[i];
9434 			__le16 caps = led->led_state_caps;
9435 
9436 			if (!led->led_group_id ||
9437 			    !BNXT_LED_ALT_BLINK_CAP(caps)) {
9438 				bp->num_leds = 0;
9439 				break;
9440 			}
9441 		}
9442 	}
9443 	mutex_unlock(&bp->hwrm_cmd_lock);
9444 	return 0;
9445 }
9446 
9447 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9448 {
9449 	struct hwrm_wol_filter_alloc_input req = {0};
9450 	struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9451 	int rc;
9452 
9453 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9454 	req.port_id = cpu_to_le16(bp->pf.port_id);
9455 	req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9456 	req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9457 	memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9458 	mutex_lock(&bp->hwrm_cmd_lock);
9459 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9460 	if (!rc)
9461 		bp->wol_filter_id = resp->wol_filter_id;
9462 	mutex_unlock(&bp->hwrm_cmd_lock);
9463 	return rc;
9464 }
9465 
9466 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9467 {
9468 	struct hwrm_wol_filter_free_input req = {0};
9469 
9470 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9471 	req.port_id = cpu_to_le16(bp->pf.port_id);
9472 	req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9473 	req.wol_filter_id = bp->wol_filter_id;
9474 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9475 }
9476 
9477 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9478 {
9479 	struct hwrm_wol_filter_qcfg_input req = {0};
9480 	struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9481 	u16 next_handle = 0;
9482 	int rc;
9483 
9484 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9485 	req.port_id = cpu_to_le16(bp->pf.port_id);
9486 	req.handle = cpu_to_le16(handle);
9487 	mutex_lock(&bp->hwrm_cmd_lock);
9488 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9489 	if (!rc) {
9490 		next_handle = le16_to_cpu(resp->next_handle);
9491 		if (next_handle != 0) {
9492 			if (resp->wol_type ==
9493 			    WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9494 				bp->wol = 1;
9495 				bp->wol_filter_id = resp->wol_filter_id;
9496 			}
9497 		}
9498 	}
9499 	mutex_unlock(&bp->hwrm_cmd_lock);
9500 	return next_handle;
9501 }
9502 
9503 static void bnxt_get_wol_settings(struct bnxt *bp)
9504 {
9505 	u16 handle = 0;
9506 
9507 	bp->wol = 0;
9508 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9509 		return;
9510 
9511 	do {
9512 		handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9513 	} while (handle && handle != 0xffff);
9514 }
9515 
9516 #ifdef CONFIG_BNXT_HWMON
9517 static ssize_t bnxt_show_temp(struct device *dev,
9518 			      struct device_attribute *devattr, char *buf)
9519 {
9520 	struct hwrm_temp_monitor_query_input req = {0};
9521 	struct hwrm_temp_monitor_query_output *resp;
9522 	struct bnxt *bp = dev_get_drvdata(dev);
9523 	u32 len = 0;
9524 	int rc;
9525 
9526 	resp = bp->hwrm_cmd_resp_addr;
9527 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9528 	mutex_lock(&bp->hwrm_cmd_lock);
9529 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9530 	if (!rc)
9531 		len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9532 	mutex_unlock(&bp->hwrm_cmd_lock);
9533 	return rc ?: len;
9534 }
9535 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9536 
9537 static struct attribute *bnxt_attrs[] = {
9538 	&sensor_dev_attr_temp1_input.dev_attr.attr,
9539 	NULL
9540 };
9541 ATTRIBUTE_GROUPS(bnxt);
9542 
9543 static void bnxt_hwmon_close(struct bnxt *bp)
9544 {
9545 	if (bp->hwmon_dev) {
9546 		hwmon_device_unregister(bp->hwmon_dev);
9547 		bp->hwmon_dev = NULL;
9548 	}
9549 }
9550 
9551 static void bnxt_hwmon_open(struct bnxt *bp)
9552 {
9553 	struct hwrm_temp_monitor_query_input req = {0};
9554 	struct pci_dev *pdev = bp->pdev;
9555 	int rc;
9556 
9557 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9558 	rc = hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9559 	if (rc == -EACCES || rc == -EOPNOTSUPP) {
9560 		bnxt_hwmon_close(bp);
9561 		return;
9562 	}
9563 
9564 	if (bp->hwmon_dev)
9565 		return;
9566 
9567 	bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9568 							  DRV_MODULE_NAME, bp,
9569 							  bnxt_groups);
9570 	if (IS_ERR(bp->hwmon_dev)) {
9571 		bp->hwmon_dev = NULL;
9572 		dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9573 	}
9574 }
9575 #else
9576 static void bnxt_hwmon_close(struct bnxt *bp)
9577 {
9578 }
9579 
9580 static void bnxt_hwmon_open(struct bnxt *bp)
9581 {
9582 }
9583 #endif
9584 
9585 static bool bnxt_eee_config_ok(struct bnxt *bp)
9586 {
9587 	struct ethtool_eee *eee = &bp->eee;
9588 	struct bnxt_link_info *link_info = &bp->link_info;
9589 
9590 	if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9591 		return true;
9592 
9593 	if (eee->eee_enabled) {
9594 		u32 advertising =
9595 			_bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9596 
9597 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9598 			eee->eee_enabled = 0;
9599 			return false;
9600 		}
9601 		if (eee->advertised & ~advertising) {
9602 			eee->advertised = advertising & eee->supported;
9603 			return false;
9604 		}
9605 	}
9606 	return true;
9607 }
9608 
9609 static int bnxt_update_phy_setting(struct bnxt *bp)
9610 {
9611 	int rc;
9612 	bool update_link = false;
9613 	bool update_pause = false;
9614 	bool update_eee = false;
9615 	struct bnxt_link_info *link_info = &bp->link_info;
9616 
9617 	rc = bnxt_update_link(bp, true);
9618 	if (rc) {
9619 		netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9620 			   rc);
9621 		return rc;
9622 	}
9623 	if (!BNXT_SINGLE_PF(bp))
9624 		return 0;
9625 
9626 	if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9627 	    (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9628 	    link_info->req_flow_ctrl)
9629 		update_pause = true;
9630 	if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9631 	    link_info->force_pause_setting != link_info->req_flow_ctrl)
9632 		update_pause = true;
9633 	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9634 		if (BNXT_AUTO_MODE(link_info->auto_mode))
9635 			update_link = true;
9636 		if (link_info->req_signal_mode == BNXT_SIG_MODE_NRZ &&
9637 		    link_info->req_link_speed != link_info->force_link_speed)
9638 			update_link = true;
9639 		else if (link_info->req_signal_mode == BNXT_SIG_MODE_PAM4 &&
9640 			 link_info->req_link_speed != link_info->force_pam4_link_speed)
9641 			update_link = true;
9642 		if (link_info->req_duplex != link_info->duplex_setting)
9643 			update_link = true;
9644 	} else {
9645 		if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9646 			update_link = true;
9647 		if (link_info->advertising != link_info->auto_link_speeds ||
9648 		    link_info->advertising_pam4 != link_info->auto_pam4_link_speeds)
9649 			update_link = true;
9650 	}
9651 
9652 	/* The last close may have shutdown the link, so need to call
9653 	 * PHY_CFG to bring it back up.
9654 	 */
9655 	if (!bp->link_info.link_up)
9656 		update_link = true;
9657 
9658 	if (!bnxt_eee_config_ok(bp))
9659 		update_eee = true;
9660 
9661 	if (update_link)
9662 		rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9663 	else if (update_pause)
9664 		rc = bnxt_hwrm_set_pause(bp);
9665 	if (rc) {
9666 		netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9667 			   rc);
9668 		return rc;
9669 	}
9670 
9671 	return rc;
9672 }
9673 
9674 /* Common routine to pre-map certain register block to different GRC window.
9675  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9676  * in PF and 3 windows in VF that can be customized to map in different
9677  * register blocks.
9678  */
9679 static void bnxt_preset_reg_win(struct bnxt *bp)
9680 {
9681 	if (BNXT_PF(bp)) {
9682 		/* CAG registers map to GRC window #4 */
9683 		writel(BNXT_CAG_REG_BASE,
9684 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9685 	}
9686 }
9687 
9688 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9689 
9690 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9691 {
9692 	int rc = 0;
9693 
9694 	bnxt_preset_reg_win(bp);
9695 	netif_carrier_off(bp->dev);
9696 	if (irq_re_init) {
9697 		/* Reserve rings now if none were reserved at driver probe. */
9698 		rc = bnxt_init_dflt_ring_mode(bp);
9699 		if (rc) {
9700 			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9701 			return rc;
9702 		}
9703 	}
9704 	rc = bnxt_reserve_rings(bp, irq_re_init);
9705 	if (rc)
9706 		return rc;
9707 	if ((bp->flags & BNXT_FLAG_RFS) &&
9708 	    !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9709 		/* disable RFS if falling back to INTA */
9710 		bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9711 		bp->flags &= ~BNXT_FLAG_RFS;
9712 	}
9713 
9714 	rc = bnxt_alloc_mem(bp, irq_re_init);
9715 	if (rc) {
9716 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9717 		goto open_err_free_mem;
9718 	}
9719 
9720 	if (irq_re_init) {
9721 		bnxt_init_napi(bp);
9722 		rc = bnxt_request_irq(bp);
9723 		if (rc) {
9724 			netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9725 			goto open_err_irq;
9726 		}
9727 	}
9728 
9729 	rc = bnxt_init_nic(bp, irq_re_init);
9730 	if (rc) {
9731 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9732 		goto open_err_irq;
9733 	}
9734 
9735 	bnxt_enable_napi(bp);
9736 	bnxt_debug_dev_init(bp);
9737 
9738 	if (link_re_init) {
9739 		mutex_lock(&bp->link_lock);
9740 		rc = bnxt_update_phy_setting(bp);
9741 		mutex_unlock(&bp->link_lock);
9742 		if (rc) {
9743 			netdev_warn(bp->dev, "failed to update phy settings\n");
9744 			if (BNXT_SINGLE_PF(bp)) {
9745 				bp->link_info.phy_retry = true;
9746 				bp->link_info.phy_retry_expires =
9747 					jiffies + 5 * HZ;
9748 			}
9749 		}
9750 	}
9751 
9752 	if (irq_re_init)
9753 		udp_tunnel_nic_reset_ntf(bp->dev);
9754 
9755 	set_bit(BNXT_STATE_OPEN, &bp->state);
9756 	bnxt_enable_int(bp);
9757 	/* Enable TX queues */
9758 	bnxt_tx_enable(bp);
9759 	mod_timer(&bp->timer, jiffies + bp->current_interval);
9760 	/* Poll link status and check for SFP+ module status */
9761 	bnxt_get_port_module_status(bp);
9762 
9763 	/* VF-reps may need to be re-opened after the PF is re-opened */
9764 	if (BNXT_PF(bp))
9765 		bnxt_vf_reps_open(bp);
9766 	return 0;
9767 
9768 open_err_irq:
9769 	bnxt_del_napi(bp);
9770 
9771 open_err_free_mem:
9772 	bnxt_free_skbs(bp);
9773 	bnxt_free_irq(bp);
9774 	bnxt_free_mem(bp, true);
9775 	return rc;
9776 }
9777 
9778 /* rtnl_lock held */
9779 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9780 {
9781 	int rc = 0;
9782 
9783 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state))
9784 		rc = -EIO;
9785 	if (!rc)
9786 		rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9787 	if (rc) {
9788 		netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9789 		dev_close(bp->dev);
9790 	}
9791 	return rc;
9792 }
9793 
9794 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9795  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9796  * self tests.
9797  */
9798 int bnxt_half_open_nic(struct bnxt *bp)
9799 {
9800 	int rc = 0;
9801 
9802 	rc = bnxt_alloc_mem(bp, false);
9803 	if (rc) {
9804 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9805 		goto half_open_err;
9806 	}
9807 	rc = bnxt_init_nic(bp, false);
9808 	if (rc) {
9809 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9810 		goto half_open_err;
9811 	}
9812 	return 0;
9813 
9814 half_open_err:
9815 	bnxt_free_skbs(bp);
9816 	bnxt_free_mem(bp, false);
9817 	dev_close(bp->dev);
9818 	return rc;
9819 }
9820 
9821 /* rtnl_lock held, this call can only be made after a previous successful
9822  * call to bnxt_half_open_nic().
9823  */
9824 void bnxt_half_close_nic(struct bnxt *bp)
9825 {
9826 	bnxt_hwrm_resource_free(bp, false, false);
9827 	bnxt_free_skbs(bp);
9828 	bnxt_free_mem(bp, false);
9829 }
9830 
9831 static void bnxt_reenable_sriov(struct bnxt *bp)
9832 {
9833 	if (BNXT_PF(bp)) {
9834 		struct bnxt_pf_info *pf = &bp->pf;
9835 		int n = pf->active_vfs;
9836 
9837 		if (n)
9838 			bnxt_cfg_hw_sriov(bp, &n, true);
9839 	}
9840 }
9841 
9842 static int bnxt_open(struct net_device *dev)
9843 {
9844 	struct bnxt *bp = netdev_priv(dev);
9845 	int rc;
9846 
9847 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9848 		netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9849 		return -ENODEV;
9850 	}
9851 
9852 	rc = bnxt_hwrm_if_change(bp, true);
9853 	if (rc)
9854 		return rc;
9855 	rc = __bnxt_open_nic(bp, true, true);
9856 	if (rc) {
9857 		bnxt_hwrm_if_change(bp, false);
9858 	} else {
9859 		if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9860 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9861 				bnxt_ulp_start(bp, 0);
9862 				bnxt_reenable_sriov(bp);
9863 			}
9864 		}
9865 		bnxt_hwmon_open(bp);
9866 	}
9867 
9868 	return rc;
9869 }
9870 
9871 static bool bnxt_drv_busy(struct bnxt *bp)
9872 {
9873 	return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9874 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
9875 }
9876 
9877 static void bnxt_get_ring_stats(struct bnxt *bp,
9878 				struct rtnl_link_stats64 *stats);
9879 
9880 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9881 			     bool link_re_init)
9882 {
9883 	/* Close the VF-reps before closing PF */
9884 	if (BNXT_PF(bp))
9885 		bnxt_vf_reps_close(bp);
9886 
9887 	/* Change device state to avoid TX queue wake up's */
9888 	bnxt_tx_disable(bp);
9889 
9890 	clear_bit(BNXT_STATE_OPEN, &bp->state);
9891 	smp_mb__after_atomic();
9892 	while (bnxt_drv_busy(bp))
9893 		msleep(20);
9894 
9895 	/* Flush rings and and disable interrupts */
9896 	bnxt_shutdown_nic(bp, irq_re_init);
9897 
9898 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9899 
9900 	bnxt_debug_dev_exit(bp);
9901 	bnxt_disable_napi(bp);
9902 	del_timer_sync(&bp->timer);
9903 	bnxt_free_skbs(bp);
9904 
9905 	/* Save ring stats before shutdown */
9906 	if (bp->bnapi && irq_re_init)
9907 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9908 	if (irq_re_init) {
9909 		bnxt_free_irq(bp);
9910 		bnxt_del_napi(bp);
9911 	}
9912 	bnxt_free_mem(bp, irq_re_init);
9913 }
9914 
9915 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9916 {
9917 	int rc = 0;
9918 
9919 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9920 		/* If we get here, it means firmware reset is in progress
9921 		 * while we are trying to close.  We can safely proceed with
9922 		 * the close because we are holding rtnl_lock().  Some firmware
9923 		 * messages may fail as we proceed to close.  We set the
9924 		 * ABORT_ERR flag here so that the FW reset thread will later
9925 		 * abort when it gets the rtnl_lock() and sees the flag.
9926 		 */
9927 		netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9928 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9929 	}
9930 
9931 #ifdef CONFIG_BNXT_SRIOV
9932 	if (bp->sriov_cfg) {
9933 		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9934 						      !bp->sriov_cfg,
9935 						      BNXT_SRIOV_CFG_WAIT_TMO);
9936 		if (rc)
9937 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9938 	}
9939 #endif
9940 	__bnxt_close_nic(bp, irq_re_init, link_re_init);
9941 	return rc;
9942 }
9943 
9944 static int bnxt_close(struct net_device *dev)
9945 {
9946 	struct bnxt *bp = netdev_priv(dev);
9947 
9948 	bnxt_hwmon_close(bp);
9949 	bnxt_close_nic(bp, true, true);
9950 	bnxt_hwrm_shutdown_link(bp);
9951 	bnxt_hwrm_if_change(bp, false);
9952 	return 0;
9953 }
9954 
9955 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9956 				   u16 *val)
9957 {
9958 	struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9959 	struct hwrm_port_phy_mdio_read_input req = {0};
9960 	int rc;
9961 
9962 	if (bp->hwrm_spec_code < 0x10a00)
9963 		return -EOPNOTSUPP;
9964 
9965 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9966 	req.port_id = cpu_to_le16(bp->pf.port_id);
9967 	req.phy_addr = phy_addr;
9968 	req.reg_addr = cpu_to_le16(reg & 0x1f);
9969 	if (mdio_phy_id_is_c45(phy_addr)) {
9970 		req.cl45_mdio = 1;
9971 		req.phy_addr = mdio_phy_id_prtad(phy_addr);
9972 		req.dev_addr = mdio_phy_id_devad(phy_addr);
9973 		req.reg_addr = cpu_to_le16(reg);
9974 	}
9975 
9976 	mutex_lock(&bp->hwrm_cmd_lock);
9977 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9978 	if (!rc)
9979 		*val = le16_to_cpu(resp->reg_data);
9980 	mutex_unlock(&bp->hwrm_cmd_lock);
9981 	return rc;
9982 }
9983 
9984 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9985 				    u16 val)
9986 {
9987 	struct hwrm_port_phy_mdio_write_input req = {0};
9988 
9989 	if (bp->hwrm_spec_code < 0x10a00)
9990 		return -EOPNOTSUPP;
9991 
9992 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9993 	req.port_id = cpu_to_le16(bp->pf.port_id);
9994 	req.phy_addr = phy_addr;
9995 	req.reg_addr = cpu_to_le16(reg & 0x1f);
9996 	if (mdio_phy_id_is_c45(phy_addr)) {
9997 		req.cl45_mdio = 1;
9998 		req.phy_addr = mdio_phy_id_prtad(phy_addr);
9999 		req.dev_addr = mdio_phy_id_devad(phy_addr);
10000 		req.reg_addr = cpu_to_le16(reg);
10001 	}
10002 	req.reg_data = cpu_to_le16(val);
10003 
10004 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10005 }
10006 
10007 /* rtnl_lock held */
10008 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
10009 {
10010 	struct mii_ioctl_data *mdio = if_mii(ifr);
10011 	struct bnxt *bp = netdev_priv(dev);
10012 	int rc;
10013 
10014 	switch (cmd) {
10015 	case SIOCGMIIPHY:
10016 		mdio->phy_id = bp->link_info.phy_addr;
10017 
10018 		fallthrough;
10019 	case SIOCGMIIREG: {
10020 		u16 mii_regval = 0;
10021 
10022 		if (!netif_running(dev))
10023 			return -EAGAIN;
10024 
10025 		rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
10026 					     &mii_regval);
10027 		mdio->val_out = mii_regval;
10028 		return rc;
10029 	}
10030 
10031 	case SIOCSMIIREG:
10032 		if (!netif_running(dev))
10033 			return -EAGAIN;
10034 
10035 		return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
10036 						mdio->val_in);
10037 
10038 	default:
10039 		/* do nothing */
10040 		break;
10041 	}
10042 	return -EOPNOTSUPP;
10043 }
10044 
10045 static void bnxt_get_ring_stats(struct bnxt *bp,
10046 				struct rtnl_link_stats64 *stats)
10047 {
10048 	int i;
10049 
10050 	for (i = 0; i < bp->cp_nr_rings; i++) {
10051 		struct bnxt_napi *bnapi = bp->bnapi[i];
10052 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10053 		u64 *sw = cpr->stats.sw_stats;
10054 
10055 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
10056 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10057 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
10058 
10059 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
10060 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
10061 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
10062 
10063 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
10064 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
10065 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
10066 
10067 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
10068 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
10069 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
10070 
10071 		stats->rx_missed_errors +=
10072 			BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
10073 
10074 		stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
10075 
10076 		stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
10077 	}
10078 }
10079 
10080 static void bnxt_add_prev_stats(struct bnxt *bp,
10081 				struct rtnl_link_stats64 *stats)
10082 {
10083 	struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
10084 
10085 	stats->rx_packets += prev_stats->rx_packets;
10086 	stats->tx_packets += prev_stats->tx_packets;
10087 	stats->rx_bytes += prev_stats->rx_bytes;
10088 	stats->tx_bytes += prev_stats->tx_bytes;
10089 	stats->rx_missed_errors += prev_stats->rx_missed_errors;
10090 	stats->multicast += prev_stats->multicast;
10091 	stats->tx_dropped += prev_stats->tx_dropped;
10092 }
10093 
10094 static void
10095 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
10096 {
10097 	struct bnxt *bp = netdev_priv(dev);
10098 
10099 	set_bit(BNXT_STATE_READ_STATS, &bp->state);
10100 	/* Make sure bnxt_close_nic() sees that we are reading stats before
10101 	 * we check the BNXT_STATE_OPEN flag.
10102 	 */
10103 	smp_mb__after_atomic();
10104 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10105 		clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10106 		*stats = bp->net_stats_prev;
10107 		return;
10108 	}
10109 
10110 	bnxt_get_ring_stats(bp, stats);
10111 	bnxt_add_prev_stats(bp, stats);
10112 
10113 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
10114 		u64 *rx = bp->port_stats.sw_stats;
10115 		u64 *tx = bp->port_stats.sw_stats +
10116 			  BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
10117 
10118 		stats->rx_crc_errors =
10119 			BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
10120 		stats->rx_frame_errors =
10121 			BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
10122 		stats->rx_length_errors =
10123 			BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
10124 			BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
10125 			BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
10126 		stats->rx_errors =
10127 			BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
10128 			BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
10129 		stats->collisions =
10130 			BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
10131 		stats->tx_fifo_errors =
10132 			BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
10133 		stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
10134 	}
10135 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
10136 }
10137 
10138 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
10139 {
10140 	struct net_device *dev = bp->dev;
10141 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10142 	struct netdev_hw_addr *ha;
10143 	u8 *haddr;
10144 	int mc_count = 0;
10145 	bool update = false;
10146 	int off = 0;
10147 
10148 	netdev_for_each_mc_addr(ha, dev) {
10149 		if (mc_count >= BNXT_MAX_MC_ADDRS) {
10150 			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10151 			vnic->mc_list_count = 0;
10152 			return false;
10153 		}
10154 		haddr = ha->addr;
10155 		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
10156 			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
10157 			update = true;
10158 		}
10159 		off += ETH_ALEN;
10160 		mc_count++;
10161 	}
10162 	if (mc_count)
10163 		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
10164 
10165 	if (mc_count != vnic->mc_list_count) {
10166 		vnic->mc_list_count = mc_count;
10167 		update = true;
10168 	}
10169 	return update;
10170 }
10171 
10172 static bool bnxt_uc_list_updated(struct bnxt *bp)
10173 {
10174 	struct net_device *dev = bp->dev;
10175 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10176 	struct netdev_hw_addr *ha;
10177 	int off = 0;
10178 
10179 	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
10180 		return true;
10181 
10182 	netdev_for_each_uc_addr(ha, dev) {
10183 		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
10184 			return true;
10185 
10186 		off += ETH_ALEN;
10187 	}
10188 	return false;
10189 }
10190 
10191 static void bnxt_set_rx_mode(struct net_device *dev)
10192 {
10193 	struct bnxt *bp = netdev_priv(dev);
10194 	struct bnxt_vnic_info *vnic;
10195 	bool mc_update = false;
10196 	bool uc_update;
10197 	u32 mask;
10198 
10199 	if (!test_bit(BNXT_STATE_OPEN, &bp->state))
10200 		return;
10201 
10202 	vnic = &bp->vnic_info[0];
10203 	mask = vnic->rx_mask;
10204 	mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
10205 		  CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
10206 		  CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
10207 		  CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
10208 
10209 	if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
10210 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10211 
10212 	uc_update = bnxt_uc_list_updated(bp);
10213 
10214 	if (dev->flags & IFF_BROADCAST)
10215 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
10216 	if (dev->flags & IFF_ALLMULTI) {
10217 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10218 		vnic->mc_list_count = 0;
10219 	} else {
10220 		mc_update = bnxt_mc_list_updated(bp, &mask);
10221 	}
10222 
10223 	if (mask != vnic->rx_mask || uc_update || mc_update) {
10224 		vnic->rx_mask = mask;
10225 
10226 		set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10227 		bnxt_queue_sp_work(bp);
10228 	}
10229 }
10230 
10231 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10232 {
10233 	struct net_device *dev = bp->dev;
10234 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10235 	struct netdev_hw_addr *ha;
10236 	int i, off = 0, rc;
10237 	bool uc_update;
10238 
10239 	netif_addr_lock_bh(dev);
10240 	uc_update = bnxt_uc_list_updated(bp);
10241 	netif_addr_unlock_bh(dev);
10242 
10243 	if (!uc_update)
10244 		goto skip_uc;
10245 
10246 	mutex_lock(&bp->hwrm_cmd_lock);
10247 	for (i = 1; i < vnic->uc_filter_count; i++) {
10248 		struct hwrm_cfa_l2_filter_free_input req = {0};
10249 
10250 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10251 				       -1);
10252 
10253 		req.l2_filter_id = vnic->fw_l2_filter_id[i];
10254 
10255 		rc = _hwrm_send_message(bp, &req, sizeof(req),
10256 					HWRM_CMD_TIMEOUT);
10257 	}
10258 	mutex_unlock(&bp->hwrm_cmd_lock);
10259 
10260 	vnic->uc_filter_count = 1;
10261 
10262 	netif_addr_lock_bh(dev);
10263 	if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10264 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10265 	} else {
10266 		netdev_for_each_uc_addr(ha, dev) {
10267 			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10268 			off += ETH_ALEN;
10269 			vnic->uc_filter_count++;
10270 		}
10271 	}
10272 	netif_addr_unlock_bh(dev);
10273 
10274 	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10275 		rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10276 		if (rc) {
10277 			netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10278 				   rc);
10279 			vnic->uc_filter_count = i;
10280 			return rc;
10281 		}
10282 	}
10283 
10284 skip_uc:
10285 	rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10286 	if (rc && vnic->mc_list_count) {
10287 		netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10288 			    rc);
10289 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10290 		vnic->mc_list_count = 0;
10291 		rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10292 	}
10293 	if (rc)
10294 		netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10295 			   rc);
10296 
10297 	return rc;
10298 }
10299 
10300 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10301 {
10302 #ifdef CONFIG_BNXT_SRIOV
10303 	if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10304 		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10305 
10306 		/* No minimum rings were provisioned by the PF.  Don't
10307 		 * reserve rings by default when device is down.
10308 		 */
10309 		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10310 			return true;
10311 
10312 		if (!netif_running(bp->dev))
10313 			return false;
10314 	}
10315 #endif
10316 	return true;
10317 }
10318 
10319 /* If the chip and firmware supports RFS */
10320 static bool bnxt_rfs_supported(struct bnxt *bp)
10321 {
10322 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
10323 		if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10324 			return true;
10325 		return false;
10326 	}
10327 	if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10328 		return true;
10329 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10330 		return true;
10331 	return false;
10332 }
10333 
10334 /* If runtime conditions support RFS */
10335 static bool bnxt_rfs_capable(struct bnxt *bp)
10336 {
10337 #ifdef CONFIG_RFS_ACCEL
10338 	int vnics, max_vnics, max_rss_ctxs;
10339 
10340 	if (bp->flags & BNXT_FLAG_CHIP_P5)
10341 		return bnxt_rfs_supported(bp);
10342 	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10343 		return false;
10344 
10345 	vnics = 1 + bp->rx_nr_rings;
10346 	max_vnics = bnxt_get_max_func_vnics(bp);
10347 	max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10348 
10349 	/* RSS contexts not a limiting factor */
10350 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10351 		max_rss_ctxs = max_vnics;
10352 	if (vnics > max_vnics || vnics > max_rss_ctxs) {
10353 		if (bp->rx_nr_rings > 1)
10354 			netdev_warn(bp->dev,
10355 				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10356 				    min(max_rss_ctxs - 1, max_vnics - 1));
10357 		return false;
10358 	}
10359 
10360 	if (!BNXT_NEW_RM(bp))
10361 		return true;
10362 
10363 	if (vnics == bp->hw_resc.resv_vnics)
10364 		return true;
10365 
10366 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10367 	if (vnics <= bp->hw_resc.resv_vnics)
10368 		return true;
10369 
10370 	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10371 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10372 	return false;
10373 #else
10374 	return false;
10375 #endif
10376 }
10377 
10378 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10379 					   netdev_features_t features)
10380 {
10381 	struct bnxt *bp = netdev_priv(dev);
10382 	netdev_features_t vlan_features;
10383 
10384 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10385 		features &= ~NETIF_F_NTUPLE;
10386 
10387 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10388 		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10389 
10390 	if (!(features & NETIF_F_GRO))
10391 		features &= ~NETIF_F_GRO_HW;
10392 
10393 	if (features & NETIF_F_GRO_HW)
10394 		features &= ~NETIF_F_LRO;
10395 
10396 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
10397 	 * turned on or off together.
10398 	 */
10399 	vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10400 	if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10401 		if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10402 			features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10403 		else if (vlan_features)
10404 			features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10405 	}
10406 #ifdef CONFIG_BNXT_SRIOV
10407 	if (BNXT_VF(bp) && bp->vf.vlan)
10408 		features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10409 #endif
10410 	return features;
10411 }
10412 
10413 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10414 {
10415 	struct bnxt *bp = netdev_priv(dev);
10416 	u32 flags = bp->flags;
10417 	u32 changes;
10418 	int rc = 0;
10419 	bool re_init = false;
10420 	bool update_tpa = false;
10421 
10422 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10423 	if (features & NETIF_F_GRO_HW)
10424 		flags |= BNXT_FLAG_GRO;
10425 	else if (features & NETIF_F_LRO)
10426 		flags |= BNXT_FLAG_LRO;
10427 
10428 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10429 		flags &= ~BNXT_FLAG_TPA;
10430 
10431 	if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10432 		flags |= BNXT_FLAG_STRIP_VLAN;
10433 
10434 	if (features & NETIF_F_NTUPLE)
10435 		flags |= BNXT_FLAG_RFS;
10436 
10437 	changes = flags ^ bp->flags;
10438 	if (changes & BNXT_FLAG_TPA) {
10439 		update_tpa = true;
10440 		if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10441 		    (flags & BNXT_FLAG_TPA) == 0 ||
10442 		    (bp->flags & BNXT_FLAG_CHIP_P5))
10443 			re_init = true;
10444 	}
10445 
10446 	if (changes & ~BNXT_FLAG_TPA)
10447 		re_init = true;
10448 
10449 	if (flags != bp->flags) {
10450 		u32 old_flags = bp->flags;
10451 
10452 		if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10453 			bp->flags = flags;
10454 			if (update_tpa)
10455 				bnxt_set_ring_params(bp);
10456 			return rc;
10457 		}
10458 
10459 		if (re_init) {
10460 			bnxt_close_nic(bp, false, false);
10461 			bp->flags = flags;
10462 			if (update_tpa)
10463 				bnxt_set_ring_params(bp);
10464 
10465 			return bnxt_open_nic(bp, false, false);
10466 		}
10467 		if (update_tpa) {
10468 			bp->flags = flags;
10469 			rc = bnxt_set_tpa(bp,
10470 					  (flags & BNXT_FLAG_TPA) ?
10471 					  true : false);
10472 			if (rc)
10473 				bp->flags = old_flags;
10474 		}
10475 	}
10476 	return rc;
10477 }
10478 
10479 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10480 			 u32 *reg_buf)
10481 {
10482 	struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10483 	struct hwrm_dbg_read_direct_input req = {0};
10484 	__le32 *dbg_reg_buf;
10485 	dma_addr_t mapping;
10486 	int rc, i;
10487 
10488 	dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10489 					 &mapping, GFP_KERNEL);
10490 	if (!dbg_reg_buf)
10491 		return -ENOMEM;
10492 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10493 	req.host_dest_addr = cpu_to_le64(mapping);
10494 	req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10495 	req.read_len32 = cpu_to_le32(num_words);
10496 	mutex_lock(&bp->hwrm_cmd_lock);
10497 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10498 	if (rc || resp->error_code) {
10499 		rc = -EIO;
10500 		goto dbg_rd_reg_exit;
10501 	}
10502 	for (i = 0; i < num_words; i++)
10503 		reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10504 
10505 dbg_rd_reg_exit:
10506 	mutex_unlock(&bp->hwrm_cmd_lock);
10507 	dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10508 	return rc;
10509 }
10510 
10511 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10512 				       u32 ring_id, u32 *prod, u32 *cons)
10513 {
10514 	struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10515 	struct hwrm_dbg_ring_info_get_input req = {0};
10516 	int rc;
10517 
10518 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10519 	req.ring_type = ring_type;
10520 	req.fw_ring_id = cpu_to_le32(ring_id);
10521 	mutex_lock(&bp->hwrm_cmd_lock);
10522 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10523 	if (!rc) {
10524 		*prod = le32_to_cpu(resp->producer_index);
10525 		*cons = le32_to_cpu(resp->consumer_index);
10526 	}
10527 	mutex_unlock(&bp->hwrm_cmd_lock);
10528 	return rc;
10529 }
10530 
10531 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10532 {
10533 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10534 	int i = bnapi->index;
10535 
10536 	if (!txr)
10537 		return;
10538 
10539 	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10540 		    i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10541 		    txr->tx_cons);
10542 }
10543 
10544 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10545 {
10546 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10547 	int i = bnapi->index;
10548 
10549 	if (!rxr)
10550 		return;
10551 
10552 	netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10553 		    i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10554 		    rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10555 		    rxr->rx_sw_agg_prod);
10556 }
10557 
10558 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10559 {
10560 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10561 	int i = bnapi->index;
10562 
10563 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10564 		    i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10565 }
10566 
10567 static void bnxt_dbg_dump_states(struct bnxt *bp)
10568 {
10569 	int i;
10570 	struct bnxt_napi *bnapi;
10571 
10572 	for (i = 0; i < bp->cp_nr_rings; i++) {
10573 		bnapi = bp->bnapi[i];
10574 		if (netif_msg_drv(bp)) {
10575 			bnxt_dump_tx_sw_state(bnapi);
10576 			bnxt_dump_rx_sw_state(bnapi);
10577 			bnxt_dump_cp_sw_state(bnapi);
10578 		}
10579 	}
10580 }
10581 
10582 static int bnxt_hwrm_rx_ring_reset(struct bnxt *bp, int ring_nr)
10583 {
10584 	struct bnxt_rx_ring_info *rxr = &bp->rx_ring[ring_nr];
10585 	struct hwrm_ring_reset_input req = {0};
10586 	struct bnxt_napi *bnapi = rxr->bnapi;
10587 	struct bnxt_cp_ring_info *cpr;
10588 	u16 cp_ring_id;
10589 
10590 	cpr = &bnapi->cp_ring;
10591 	cp_ring_id = cpr->cp_ring_struct.fw_ring_id;
10592 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_RESET, cp_ring_id, -1);
10593 	req.ring_type = RING_RESET_REQ_RING_TYPE_RX_RING_GRP;
10594 	req.ring_id = cpu_to_le16(bp->grp_info[bnapi->index].fw_grp_id);
10595 	return hwrm_send_message_silent(bp, &req, sizeof(req),
10596 					HWRM_CMD_TIMEOUT);
10597 }
10598 
10599 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10600 {
10601 	if (!silent)
10602 		bnxt_dbg_dump_states(bp);
10603 	if (netif_running(bp->dev)) {
10604 		int rc;
10605 
10606 		if (silent) {
10607 			bnxt_close_nic(bp, false, false);
10608 			bnxt_open_nic(bp, false, false);
10609 		} else {
10610 			bnxt_ulp_stop(bp);
10611 			bnxt_close_nic(bp, true, false);
10612 			rc = bnxt_open_nic(bp, true, false);
10613 			bnxt_ulp_start(bp, rc);
10614 		}
10615 	}
10616 }
10617 
10618 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10619 {
10620 	struct bnxt *bp = netdev_priv(dev);
10621 
10622 	netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10623 	set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10624 	bnxt_queue_sp_work(bp);
10625 }
10626 
10627 static void bnxt_fw_health_check(struct bnxt *bp)
10628 {
10629 	struct bnxt_fw_health *fw_health = bp->fw_health;
10630 	u32 val;
10631 
10632 	if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10633 		return;
10634 
10635 	if (fw_health->tmr_counter) {
10636 		fw_health->tmr_counter--;
10637 		return;
10638 	}
10639 
10640 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10641 	if (val == fw_health->last_fw_heartbeat)
10642 		goto fw_reset;
10643 
10644 	fw_health->last_fw_heartbeat = val;
10645 
10646 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10647 	if (val != fw_health->last_fw_reset_cnt)
10648 		goto fw_reset;
10649 
10650 	fw_health->tmr_counter = fw_health->tmr_multiplier;
10651 	return;
10652 
10653 fw_reset:
10654 	set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10655 	bnxt_queue_sp_work(bp);
10656 }
10657 
10658 static void bnxt_timer(struct timer_list *t)
10659 {
10660 	struct bnxt *bp = from_timer(bp, t, timer);
10661 	struct net_device *dev = bp->dev;
10662 
10663 	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10664 		return;
10665 
10666 	if (atomic_read(&bp->intr_sem) != 0)
10667 		goto bnxt_restart_timer;
10668 
10669 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10670 		bnxt_fw_health_check(bp);
10671 
10672 	if (bp->link_info.link_up && bp->stats_coal_ticks) {
10673 		set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10674 		bnxt_queue_sp_work(bp);
10675 	}
10676 
10677 	if (bnxt_tc_flower_enabled(bp)) {
10678 		set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10679 		bnxt_queue_sp_work(bp);
10680 	}
10681 
10682 #ifdef CONFIG_RFS_ACCEL
10683 	if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10684 		set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10685 		bnxt_queue_sp_work(bp);
10686 	}
10687 #endif /*CONFIG_RFS_ACCEL*/
10688 
10689 	if (bp->link_info.phy_retry) {
10690 		if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10691 			bp->link_info.phy_retry = false;
10692 			netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10693 		} else {
10694 			set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10695 			bnxt_queue_sp_work(bp);
10696 		}
10697 	}
10698 
10699 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10700 	    netif_carrier_ok(dev)) {
10701 		set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10702 		bnxt_queue_sp_work(bp);
10703 	}
10704 bnxt_restart_timer:
10705 	mod_timer(&bp->timer, jiffies + bp->current_interval);
10706 }
10707 
10708 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10709 {
10710 	/* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10711 	 * set.  If the device is being closed, bnxt_close() may be holding
10712 	 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10713 	 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10714 	 */
10715 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10716 	rtnl_lock();
10717 }
10718 
10719 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10720 {
10721 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10722 	rtnl_unlock();
10723 }
10724 
10725 /* Only called from bnxt_sp_task() */
10726 static void bnxt_reset(struct bnxt *bp, bool silent)
10727 {
10728 	bnxt_rtnl_lock_sp(bp);
10729 	if (test_bit(BNXT_STATE_OPEN, &bp->state))
10730 		bnxt_reset_task(bp, silent);
10731 	bnxt_rtnl_unlock_sp(bp);
10732 }
10733 
10734 /* Only called from bnxt_sp_task() */
10735 static void bnxt_rx_ring_reset(struct bnxt *bp)
10736 {
10737 	int i;
10738 
10739 	bnxt_rtnl_lock_sp(bp);
10740 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10741 		bnxt_rtnl_unlock_sp(bp);
10742 		return;
10743 	}
10744 	/* Disable and flush TPA before resetting the RX ring */
10745 	if (bp->flags & BNXT_FLAG_TPA)
10746 		bnxt_set_tpa(bp, false);
10747 	for (i = 0; i < bp->rx_nr_rings; i++) {
10748 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
10749 		struct bnxt_cp_ring_info *cpr;
10750 		int rc;
10751 
10752 		if (!rxr->bnapi->in_reset)
10753 			continue;
10754 
10755 		rc = bnxt_hwrm_rx_ring_reset(bp, i);
10756 		if (rc) {
10757 			if (rc == -EINVAL || rc == -EOPNOTSUPP)
10758 				netdev_info_once(bp->dev, "RX ring reset not supported by firmware, falling back to global reset\n");
10759 			else
10760 				netdev_warn(bp->dev, "RX ring reset failed, rc = %d, falling back to global reset\n",
10761 					    rc);
10762 			bnxt_reset_task(bp, true);
10763 			break;
10764 		}
10765 		bnxt_free_one_rx_ring_skbs(bp, i);
10766 		rxr->rx_prod = 0;
10767 		rxr->rx_agg_prod = 0;
10768 		rxr->rx_sw_agg_prod = 0;
10769 		rxr->rx_next_cons = 0;
10770 		rxr->bnapi->in_reset = false;
10771 		bnxt_alloc_one_rx_ring(bp, i);
10772 		cpr = &rxr->bnapi->cp_ring;
10773 		cpr->sw_stats.rx.rx_resets++;
10774 		if (bp->flags & BNXT_FLAG_AGG_RINGS)
10775 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
10776 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
10777 	}
10778 	if (bp->flags & BNXT_FLAG_TPA)
10779 		bnxt_set_tpa(bp, true);
10780 	bnxt_rtnl_unlock_sp(bp);
10781 }
10782 
10783 static void bnxt_fw_reset_close(struct bnxt *bp)
10784 {
10785 	bnxt_ulp_stop(bp);
10786 	/* When firmware is fatal state, disable PCI device to prevent
10787 	 * any potential bad DMAs before freeing kernel memory.
10788 	 */
10789 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10790 		pci_disable_device(bp->pdev);
10791 	__bnxt_close_nic(bp, true, false);
10792 	bnxt_clear_int_mode(bp);
10793 	bnxt_hwrm_func_drv_unrgtr(bp);
10794 	if (pci_is_enabled(bp->pdev))
10795 		pci_disable_device(bp->pdev);
10796 	bnxt_free_ctx_mem(bp);
10797 	kfree(bp->ctx);
10798 	bp->ctx = NULL;
10799 }
10800 
10801 static bool is_bnxt_fw_ok(struct bnxt *bp)
10802 {
10803 	struct bnxt_fw_health *fw_health = bp->fw_health;
10804 	bool no_heartbeat = false, has_reset = false;
10805 	u32 val;
10806 
10807 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10808 	if (val == fw_health->last_fw_heartbeat)
10809 		no_heartbeat = true;
10810 
10811 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10812 	if (val != fw_health->last_fw_reset_cnt)
10813 		has_reset = true;
10814 
10815 	if (!no_heartbeat && has_reset)
10816 		return true;
10817 
10818 	return false;
10819 }
10820 
10821 /* rtnl_lock is acquired before calling this function */
10822 static void bnxt_force_fw_reset(struct bnxt *bp)
10823 {
10824 	struct bnxt_fw_health *fw_health = bp->fw_health;
10825 	u32 wait_dsecs;
10826 
10827 	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10828 	    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10829 		return;
10830 
10831 	set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10832 	bnxt_fw_reset_close(bp);
10833 	wait_dsecs = fw_health->master_func_wait_dsecs;
10834 	if (fw_health->master) {
10835 		if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10836 			wait_dsecs = 0;
10837 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10838 	} else {
10839 		bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10840 		wait_dsecs = fw_health->normal_func_wait_dsecs;
10841 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10842 	}
10843 
10844 	bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10845 	bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10846 	bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10847 }
10848 
10849 void bnxt_fw_exception(struct bnxt *bp)
10850 {
10851 	netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10852 	set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10853 	bnxt_rtnl_lock_sp(bp);
10854 	bnxt_force_fw_reset(bp);
10855 	bnxt_rtnl_unlock_sp(bp);
10856 }
10857 
10858 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10859  * < 0 on error.
10860  */
10861 static int bnxt_get_registered_vfs(struct bnxt *bp)
10862 {
10863 #ifdef CONFIG_BNXT_SRIOV
10864 	int rc;
10865 
10866 	if (!BNXT_PF(bp))
10867 		return 0;
10868 
10869 	rc = bnxt_hwrm_func_qcfg(bp);
10870 	if (rc) {
10871 		netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10872 		return rc;
10873 	}
10874 	if (bp->pf.registered_vfs)
10875 		return bp->pf.registered_vfs;
10876 	if (bp->sriov_cfg)
10877 		return 1;
10878 #endif
10879 	return 0;
10880 }
10881 
10882 void bnxt_fw_reset(struct bnxt *bp)
10883 {
10884 	bnxt_rtnl_lock_sp(bp);
10885 	if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10886 	    !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10887 		int n = 0, tmo;
10888 
10889 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10890 		if (bp->pf.active_vfs &&
10891 		    !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10892 			n = bnxt_get_registered_vfs(bp);
10893 		if (n < 0) {
10894 			netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10895 				   n);
10896 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10897 			dev_close(bp->dev);
10898 			goto fw_reset_exit;
10899 		} else if (n > 0) {
10900 			u16 vf_tmo_dsecs = n * 10;
10901 
10902 			if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10903 				bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10904 			bp->fw_reset_state =
10905 				BNXT_FW_RESET_STATE_POLL_VF;
10906 			bnxt_queue_fw_reset_work(bp, HZ / 10);
10907 			goto fw_reset_exit;
10908 		}
10909 		bnxt_fw_reset_close(bp);
10910 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10911 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10912 			tmo = HZ / 10;
10913 		} else {
10914 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10915 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
10916 		}
10917 		bnxt_queue_fw_reset_work(bp, tmo);
10918 	}
10919 fw_reset_exit:
10920 	bnxt_rtnl_unlock_sp(bp);
10921 }
10922 
10923 static void bnxt_chk_missed_irq(struct bnxt *bp)
10924 {
10925 	int i;
10926 
10927 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10928 		return;
10929 
10930 	for (i = 0; i < bp->cp_nr_rings; i++) {
10931 		struct bnxt_napi *bnapi = bp->bnapi[i];
10932 		struct bnxt_cp_ring_info *cpr;
10933 		u32 fw_ring_id;
10934 		int j;
10935 
10936 		if (!bnapi)
10937 			continue;
10938 
10939 		cpr = &bnapi->cp_ring;
10940 		for (j = 0; j < 2; j++) {
10941 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10942 			u32 val[2];
10943 
10944 			if (!cpr2 || cpr2->has_more_work ||
10945 			    !bnxt_has_work(bp, cpr2))
10946 				continue;
10947 
10948 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10949 				cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10950 				continue;
10951 			}
10952 			fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10953 			bnxt_dbg_hwrm_ring_info_get(bp,
10954 				DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10955 				fw_ring_id, &val[0], &val[1]);
10956 			cpr->sw_stats.cmn.missed_irqs++;
10957 		}
10958 	}
10959 }
10960 
10961 static void bnxt_cfg_ntp_filters(struct bnxt *);
10962 
10963 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10964 {
10965 	struct bnxt_link_info *link_info = &bp->link_info;
10966 
10967 	if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10968 		link_info->autoneg = BNXT_AUTONEG_SPEED;
10969 		if (bp->hwrm_spec_code >= 0x10201) {
10970 			if (link_info->auto_pause_setting &
10971 			    PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10972 				link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10973 		} else {
10974 			link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10975 		}
10976 		link_info->advertising = link_info->auto_link_speeds;
10977 		link_info->advertising_pam4 = link_info->auto_pam4_link_speeds;
10978 	} else {
10979 		link_info->req_link_speed = link_info->force_link_speed;
10980 		link_info->req_signal_mode = BNXT_SIG_MODE_NRZ;
10981 		if (link_info->force_pam4_link_speed) {
10982 			link_info->req_link_speed =
10983 				link_info->force_pam4_link_speed;
10984 			link_info->req_signal_mode = BNXT_SIG_MODE_PAM4;
10985 		}
10986 		link_info->req_duplex = link_info->duplex_setting;
10987 	}
10988 	if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10989 		link_info->req_flow_ctrl =
10990 			link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10991 	else
10992 		link_info->req_flow_ctrl = link_info->force_pause_setting;
10993 }
10994 
10995 static void bnxt_sp_task(struct work_struct *work)
10996 {
10997 	struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10998 
10999 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11000 	smp_mb__after_atomic();
11001 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
11002 		clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11003 		return;
11004 	}
11005 
11006 	if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
11007 		bnxt_cfg_rx_mode(bp);
11008 
11009 	if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
11010 		bnxt_cfg_ntp_filters(bp);
11011 	if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
11012 		bnxt_hwrm_exec_fwd_req(bp);
11013 	if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
11014 		bnxt_hwrm_port_qstats(bp, 0);
11015 		bnxt_hwrm_port_qstats_ext(bp, 0);
11016 		bnxt_accumulate_all_stats(bp);
11017 	}
11018 
11019 	if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
11020 		int rc;
11021 
11022 		mutex_lock(&bp->link_lock);
11023 		if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
11024 				       &bp->sp_event))
11025 			bnxt_hwrm_phy_qcaps(bp);
11026 
11027 		rc = bnxt_update_link(bp, true);
11028 		if (rc)
11029 			netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
11030 				   rc);
11031 
11032 		if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
11033 				       &bp->sp_event))
11034 			bnxt_init_ethtool_link_settings(bp);
11035 		mutex_unlock(&bp->link_lock);
11036 	}
11037 	if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
11038 		int rc;
11039 
11040 		mutex_lock(&bp->link_lock);
11041 		rc = bnxt_update_phy_setting(bp);
11042 		mutex_unlock(&bp->link_lock);
11043 		if (rc) {
11044 			netdev_warn(bp->dev, "update phy settings retry failed\n");
11045 		} else {
11046 			bp->link_info.phy_retry = false;
11047 			netdev_info(bp->dev, "update phy settings retry succeeded\n");
11048 		}
11049 	}
11050 	if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
11051 		mutex_lock(&bp->link_lock);
11052 		bnxt_get_port_module_status(bp);
11053 		mutex_unlock(&bp->link_lock);
11054 	}
11055 
11056 	if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
11057 		bnxt_tc_flow_stats_work(bp);
11058 
11059 	if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
11060 		bnxt_chk_missed_irq(bp);
11061 
11062 	/* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
11063 	 * must be the last functions to be called before exiting.
11064 	 */
11065 	if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
11066 		bnxt_reset(bp, false);
11067 
11068 	if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
11069 		bnxt_reset(bp, true);
11070 
11071 	if (test_and_clear_bit(BNXT_RST_RING_SP_EVENT, &bp->sp_event))
11072 		bnxt_rx_ring_reset(bp);
11073 
11074 	if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
11075 		bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
11076 
11077 	if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
11078 		if (!is_bnxt_fw_ok(bp))
11079 			bnxt_devlink_health_report(bp,
11080 						   BNXT_FW_EXCEPTION_SP_EVENT);
11081 	}
11082 
11083 	smp_mb__before_atomic();
11084 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
11085 }
11086 
11087 /* Under rtnl_lock */
11088 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
11089 		     int tx_xdp)
11090 {
11091 	int max_rx, max_tx, tx_sets = 1;
11092 	int tx_rings_needed, stats;
11093 	int rx_rings = rx;
11094 	int cp, vnics, rc;
11095 
11096 	if (tcs)
11097 		tx_sets = tcs;
11098 
11099 	rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
11100 	if (rc)
11101 		return rc;
11102 
11103 	if (max_rx < rx)
11104 		return -ENOMEM;
11105 
11106 	tx_rings_needed = tx * tx_sets + tx_xdp;
11107 	if (max_tx < tx_rings_needed)
11108 		return -ENOMEM;
11109 
11110 	vnics = 1;
11111 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
11112 		vnics += rx_rings;
11113 
11114 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
11115 		rx_rings <<= 1;
11116 	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
11117 	stats = cp;
11118 	if (BNXT_NEW_RM(bp)) {
11119 		cp += bnxt_get_ulp_msix_num(bp);
11120 		stats += bnxt_get_ulp_stat_ctxs(bp);
11121 	}
11122 	return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
11123 				     stats, vnics);
11124 }
11125 
11126 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
11127 {
11128 	if (bp->bar2) {
11129 		pci_iounmap(pdev, bp->bar2);
11130 		bp->bar2 = NULL;
11131 	}
11132 
11133 	if (bp->bar1) {
11134 		pci_iounmap(pdev, bp->bar1);
11135 		bp->bar1 = NULL;
11136 	}
11137 
11138 	if (bp->bar0) {
11139 		pci_iounmap(pdev, bp->bar0);
11140 		bp->bar0 = NULL;
11141 	}
11142 }
11143 
11144 static void bnxt_cleanup_pci(struct bnxt *bp)
11145 {
11146 	bnxt_unmap_bars(bp, bp->pdev);
11147 	pci_release_regions(bp->pdev);
11148 	if (pci_is_enabled(bp->pdev))
11149 		pci_disable_device(bp->pdev);
11150 }
11151 
11152 static void bnxt_init_dflt_coal(struct bnxt *bp)
11153 {
11154 	struct bnxt_coal *coal;
11155 
11156 	/* Tick values in micro seconds.
11157 	 * 1 coal_buf x bufs_per_record = 1 completion record.
11158 	 */
11159 	coal = &bp->rx_coal;
11160 	coal->coal_ticks = 10;
11161 	coal->coal_bufs = 30;
11162 	coal->coal_ticks_irq = 1;
11163 	coal->coal_bufs_irq = 2;
11164 	coal->idle_thresh = 50;
11165 	coal->bufs_per_record = 2;
11166 	coal->budget = 64;		/* NAPI budget */
11167 
11168 	coal = &bp->tx_coal;
11169 	coal->coal_ticks = 28;
11170 	coal->coal_bufs = 30;
11171 	coal->coal_ticks_irq = 2;
11172 	coal->coal_bufs_irq = 2;
11173 	coal->bufs_per_record = 1;
11174 
11175 	bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
11176 }
11177 
11178 static int bnxt_fw_reset_via_optee(struct bnxt *bp)
11179 {
11180 #ifdef CONFIG_TEE_BNXT_FW
11181 	int rc = tee_bnxt_fw_load();
11182 
11183 	if (rc)
11184 		netdev_err(bp->dev, "Failed FW reset via OP-TEE, rc=%d\n", rc);
11185 
11186 	return rc;
11187 #else
11188 	netdev_err(bp->dev, "OP-TEE not supported\n");
11189 	return -ENODEV;
11190 #endif
11191 }
11192 
11193 static int bnxt_fw_init_one_p1(struct bnxt *bp)
11194 {
11195 	int rc;
11196 
11197 	bp->fw_cap = 0;
11198 	rc = bnxt_hwrm_ver_get(bp);
11199 	bnxt_try_map_fw_health_reg(bp);
11200 	if (rc) {
11201 		if (bp->fw_health && bp->fw_health->status_reliable) {
11202 			u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11203 
11204 			netdev_err(bp->dev,
11205 				   "Firmware not responding, status: 0x%x\n",
11206 				   sts);
11207 			if (sts & FW_STATUS_REG_CRASHED_NO_MASTER) {
11208 				netdev_warn(bp->dev, "Firmware recover via OP-TEE requested\n");
11209 				rc = bnxt_fw_reset_via_optee(bp);
11210 				if (!rc)
11211 					rc = bnxt_hwrm_ver_get(bp);
11212 			}
11213 		}
11214 		if (rc)
11215 			return rc;
11216 	}
11217 
11218 	if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
11219 		rc = bnxt_alloc_kong_hwrm_resources(bp);
11220 		if (rc)
11221 			bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
11222 	}
11223 
11224 	if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
11225 	    bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
11226 		rc = bnxt_alloc_hwrm_short_cmd_req(bp);
11227 		if (rc)
11228 			return rc;
11229 	}
11230 	bnxt_nvm_cfg_ver_get(bp);
11231 
11232 	rc = bnxt_hwrm_func_reset(bp);
11233 	if (rc)
11234 		return -ENODEV;
11235 
11236 	bnxt_hwrm_fw_set_time(bp);
11237 	return 0;
11238 }
11239 
11240 static int bnxt_fw_init_one_p2(struct bnxt *bp)
11241 {
11242 	int rc;
11243 
11244 	/* Get the MAX capabilities for this function */
11245 	rc = bnxt_hwrm_func_qcaps(bp);
11246 	if (rc) {
11247 		netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
11248 			   rc);
11249 		return -ENODEV;
11250 	}
11251 
11252 	rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
11253 	if (rc)
11254 		netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
11255 			    rc);
11256 
11257 	if (bnxt_alloc_fw_health(bp)) {
11258 		netdev_warn(bp->dev, "no memory for firmware error recovery\n");
11259 	} else {
11260 		rc = bnxt_hwrm_error_recovery_qcfg(bp);
11261 		if (rc)
11262 			netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
11263 				    rc);
11264 	}
11265 
11266 	rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
11267 	if (rc)
11268 		return -ENODEV;
11269 
11270 	bnxt_hwrm_func_qcfg(bp);
11271 	bnxt_hwrm_vnic_qcaps(bp);
11272 	bnxt_hwrm_port_led_qcaps(bp);
11273 	bnxt_ethtool_init(bp);
11274 	bnxt_dcb_init(bp);
11275 	return 0;
11276 }
11277 
11278 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
11279 {
11280 	bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
11281 	bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
11282 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
11283 			   VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
11284 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
11285 	if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
11286 		bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
11287 		bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
11288 				    VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
11289 	}
11290 }
11291 
11292 static void bnxt_set_dflt_rfs(struct bnxt *bp)
11293 {
11294 	struct net_device *dev = bp->dev;
11295 
11296 	dev->hw_features &= ~NETIF_F_NTUPLE;
11297 	dev->features &= ~NETIF_F_NTUPLE;
11298 	bp->flags &= ~BNXT_FLAG_RFS;
11299 	if (bnxt_rfs_supported(bp)) {
11300 		dev->hw_features |= NETIF_F_NTUPLE;
11301 		if (bnxt_rfs_capable(bp)) {
11302 			bp->flags |= BNXT_FLAG_RFS;
11303 			dev->features |= NETIF_F_NTUPLE;
11304 		}
11305 	}
11306 }
11307 
11308 static void bnxt_fw_init_one_p3(struct bnxt *bp)
11309 {
11310 	struct pci_dev *pdev = bp->pdev;
11311 
11312 	bnxt_set_dflt_rss_hash_type(bp);
11313 	bnxt_set_dflt_rfs(bp);
11314 
11315 	bnxt_get_wol_settings(bp);
11316 	if (bp->flags & BNXT_FLAG_WOL_CAP)
11317 		device_set_wakeup_enable(&pdev->dev, bp->wol);
11318 	else
11319 		device_set_wakeup_capable(&pdev->dev, false);
11320 
11321 	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11322 	bnxt_hwrm_coal_params_qcaps(bp);
11323 }
11324 
11325 static int bnxt_fw_init_one(struct bnxt *bp)
11326 {
11327 	int rc;
11328 
11329 	rc = bnxt_fw_init_one_p1(bp);
11330 	if (rc) {
11331 		netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11332 		return rc;
11333 	}
11334 	rc = bnxt_fw_init_one_p2(bp);
11335 	if (rc) {
11336 		netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11337 		return rc;
11338 	}
11339 	rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11340 	if (rc)
11341 		return rc;
11342 
11343 	/* In case fw capabilities have changed, destroy the unneeded
11344 	 * reporters and create newly capable ones.
11345 	 */
11346 	bnxt_dl_fw_reporters_destroy(bp, false);
11347 	bnxt_dl_fw_reporters_create(bp);
11348 	bnxt_fw_init_one_p3(bp);
11349 	return 0;
11350 }
11351 
11352 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11353 {
11354 	struct bnxt_fw_health *fw_health = bp->fw_health;
11355 	u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11356 	u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11357 	u32 reg_type, reg_off, delay_msecs;
11358 
11359 	delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11360 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11361 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11362 	switch (reg_type) {
11363 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
11364 		pci_write_config_dword(bp->pdev, reg_off, val);
11365 		break;
11366 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
11367 		writel(reg_off & BNXT_GRC_BASE_MASK,
11368 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11369 		reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11370 		fallthrough;
11371 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11372 		writel(val, bp->bar0 + reg_off);
11373 		break;
11374 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11375 		writel(val, bp->bar1 + reg_off);
11376 		break;
11377 	}
11378 	if (delay_msecs) {
11379 		pci_read_config_dword(bp->pdev, 0, &val);
11380 		msleep(delay_msecs);
11381 	}
11382 }
11383 
11384 static void bnxt_reset_all(struct bnxt *bp)
11385 {
11386 	struct bnxt_fw_health *fw_health = bp->fw_health;
11387 	int i, rc;
11388 
11389 	if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11390 		bnxt_fw_reset_via_optee(bp);
11391 		bp->fw_reset_timestamp = jiffies;
11392 		return;
11393 	}
11394 
11395 	if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11396 		for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11397 			bnxt_fw_reset_writel(bp, i);
11398 	} else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11399 		struct hwrm_fw_reset_input req = {0};
11400 
11401 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11402 		req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11403 		req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11404 		req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11405 		req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11406 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11407 		if (rc)
11408 			netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11409 	}
11410 	bp->fw_reset_timestamp = jiffies;
11411 }
11412 
11413 static void bnxt_fw_reset_task(struct work_struct *work)
11414 {
11415 	struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11416 	int rc;
11417 
11418 	if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11419 		netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11420 		return;
11421 	}
11422 
11423 	switch (bp->fw_reset_state) {
11424 	case BNXT_FW_RESET_STATE_POLL_VF: {
11425 		int n = bnxt_get_registered_vfs(bp);
11426 		int tmo;
11427 
11428 		if (n < 0) {
11429 			netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11430 				   n, jiffies_to_msecs(jiffies -
11431 				   bp->fw_reset_timestamp));
11432 			goto fw_reset_abort;
11433 		} else if (n > 0) {
11434 			if (time_after(jiffies, bp->fw_reset_timestamp +
11435 				       (bp->fw_reset_max_dsecs * HZ / 10))) {
11436 				clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11437 				bp->fw_reset_state = 0;
11438 				netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11439 					   n);
11440 				return;
11441 			}
11442 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11443 			return;
11444 		}
11445 		bp->fw_reset_timestamp = jiffies;
11446 		rtnl_lock();
11447 		bnxt_fw_reset_close(bp);
11448 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11449 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11450 			tmo = HZ / 10;
11451 		} else {
11452 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11453 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
11454 		}
11455 		rtnl_unlock();
11456 		bnxt_queue_fw_reset_work(bp, tmo);
11457 		return;
11458 	}
11459 	case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11460 		u32 val;
11461 
11462 		val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11463 		if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11464 		    !time_after(jiffies, bp->fw_reset_timestamp +
11465 		    (bp->fw_reset_max_dsecs * HZ / 10))) {
11466 			bnxt_queue_fw_reset_work(bp, HZ / 5);
11467 			return;
11468 		}
11469 
11470 		if (!bp->fw_health->master) {
11471 			u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11472 
11473 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11474 			bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11475 			return;
11476 		}
11477 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11478 	}
11479 		fallthrough;
11480 	case BNXT_FW_RESET_STATE_RESET_FW:
11481 		bnxt_reset_all(bp);
11482 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11483 		bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11484 		return;
11485 	case BNXT_FW_RESET_STATE_ENABLE_DEV:
11486 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11487 			u32 val;
11488 
11489 			val = bnxt_fw_health_readl(bp,
11490 						   BNXT_FW_RESET_INPROG_REG);
11491 			if (val)
11492 				netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11493 					    val);
11494 		}
11495 		clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11496 		if (pci_enable_device(bp->pdev)) {
11497 			netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11498 			goto fw_reset_abort;
11499 		}
11500 		pci_set_master(bp->pdev);
11501 		bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11502 		fallthrough;
11503 	case BNXT_FW_RESET_STATE_POLL_FW:
11504 		bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11505 		rc = __bnxt_hwrm_ver_get(bp, true);
11506 		if (rc) {
11507 			if (time_after(jiffies, bp->fw_reset_timestamp +
11508 				       (bp->fw_reset_max_dsecs * HZ / 10))) {
11509 				netdev_err(bp->dev, "Firmware reset aborted\n");
11510 				goto fw_reset_abort_status;
11511 			}
11512 			bnxt_queue_fw_reset_work(bp, HZ / 5);
11513 			return;
11514 		}
11515 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11516 		bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11517 		fallthrough;
11518 	case BNXT_FW_RESET_STATE_OPENING:
11519 		while (!rtnl_trylock()) {
11520 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11521 			return;
11522 		}
11523 		rc = bnxt_open(bp->dev);
11524 		if (rc) {
11525 			netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11526 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11527 			dev_close(bp->dev);
11528 		}
11529 
11530 		bp->fw_reset_state = 0;
11531 		/* Make sure fw_reset_state is 0 before clearing the flag */
11532 		smp_mb__before_atomic();
11533 		clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11534 		bnxt_ulp_start(bp, rc);
11535 		if (!rc)
11536 			bnxt_reenable_sriov(bp);
11537 		bnxt_dl_health_recovery_done(bp);
11538 		bnxt_dl_health_status_update(bp, true);
11539 		rtnl_unlock();
11540 		break;
11541 	}
11542 	return;
11543 
11544 fw_reset_abort_status:
11545 	if (bp->fw_health->status_reliable ||
11546 	    (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) {
11547 		u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11548 
11549 		netdev_err(bp->dev, "fw_health_status 0x%x\n", sts);
11550 	}
11551 fw_reset_abort:
11552 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11553 	if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11554 		bnxt_dl_health_status_update(bp, false);
11555 	bp->fw_reset_state = 0;
11556 	rtnl_lock();
11557 	dev_close(bp->dev);
11558 	rtnl_unlock();
11559 }
11560 
11561 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11562 {
11563 	int rc;
11564 	struct bnxt *bp = netdev_priv(dev);
11565 
11566 	SET_NETDEV_DEV(dev, &pdev->dev);
11567 
11568 	/* enable device (incl. PCI PM wakeup), and bus-mastering */
11569 	rc = pci_enable_device(pdev);
11570 	if (rc) {
11571 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11572 		goto init_err;
11573 	}
11574 
11575 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11576 		dev_err(&pdev->dev,
11577 			"Cannot find PCI device base address, aborting\n");
11578 		rc = -ENODEV;
11579 		goto init_err_disable;
11580 	}
11581 
11582 	rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11583 	if (rc) {
11584 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11585 		goto init_err_disable;
11586 	}
11587 
11588 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11589 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11590 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11591 		goto init_err_disable;
11592 	}
11593 
11594 	pci_set_master(pdev);
11595 
11596 	bp->dev = dev;
11597 	bp->pdev = pdev;
11598 
11599 	/* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11600 	 * determines the BAR size.
11601 	 */
11602 	bp->bar0 = pci_ioremap_bar(pdev, 0);
11603 	if (!bp->bar0) {
11604 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11605 		rc = -ENOMEM;
11606 		goto init_err_release;
11607 	}
11608 
11609 	bp->bar2 = pci_ioremap_bar(pdev, 4);
11610 	if (!bp->bar2) {
11611 		dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11612 		rc = -ENOMEM;
11613 		goto init_err_release;
11614 	}
11615 
11616 	pci_enable_pcie_error_reporting(pdev);
11617 
11618 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
11619 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11620 
11621 	spin_lock_init(&bp->ntp_fltr_lock);
11622 #if BITS_PER_LONG == 32
11623 	spin_lock_init(&bp->db_lock);
11624 #endif
11625 
11626 	bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11627 	bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11628 
11629 	bnxt_init_dflt_coal(bp);
11630 
11631 	timer_setup(&bp->timer, bnxt_timer, 0);
11632 	bp->current_interval = BNXT_TIMER_INTERVAL;
11633 
11634 	bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11635 	bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11636 
11637 	clear_bit(BNXT_STATE_OPEN, &bp->state);
11638 	return 0;
11639 
11640 init_err_release:
11641 	bnxt_unmap_bars(bp, pdev);
11642 	pci_release_regions(pdev);
11643 
11644 init_err_disable:
11645 	pci_disable_device(pdev);
11646 
11647 init_err:
11648 	return rc;
11649 }
11650 
11651 /* rtnl_lock held */
11652 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11653 {
11654 	struct sockaddr *addr = p;
11655 	struct bnxt *bp = netdev_priv(dev);
11656 	int rc = 0;
11657 
11658 	if (!is_valid_ether_addr(addr->sa_data))
11659 		return -EADDRNOTAVAIL;
11660 
11661 	if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11662 		return 0;
11663 
11664 	rc = bnxt_approve_mac(bp, addr->sa_data, true);
11665 	if (rc)
11666 		return rc;
11667 
11668 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11669 	if (netif_running(dev)) {
11670 		bnxt_close_nic(bp, false, false);
11671 		rc = bnxt_open_nic(bp, false, false);
11672 	}
11673 
11674 	return rc;
11675 }
11676 
11677 /* rtnl_lock held */
11678 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11679 {
11680 	struct bnxt *bp = netdev_priv(dev);
11681 
11682 	if (netif_running(dev))
11683 		bnxt_close_nic(bp, true, false);
11684 
11685 	dev->mtu = new_mtu;
11686 	bnxt_set_ring_params(bp);
11687 
11688 	if (netif_running(dev))
11689 		return bnxt_open_nic(bp, true, false);
11690 
11691 	return 0;
11692 }
11693 
11694 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11695 {
11696 	struct bnxt *bp = netdev_priv(dev);
11697 	bool sh = false;
11698 	int rc;
11699 
11700 	if (tc > bp->max_tc) {
11701 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11702 			   tc, bp->max_tc);
11703 		return -EINVAL;
11704 	}
11705 
11706 	if (netdev_get_num_tc(dev) == tc)
11707 		return 0;
11708 
11709 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11710 		sh = true;
11711 
11712 	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11713 			      sh, tc, bp->tx_nr_rings_xdp);
11714 	if (rc)
11715 		return rc;
11716 
11717 	/* Needs to close the device and do hw resource re-allocations */
11718 	if (netif_running(bp->dev))
11719 		bnxt_close_nic(bp, true, false);
11720 
11721 	if (tc) {
11722 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11723 		netdev_set_num_tc(dev, tc);
11724 	} else {
11725 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11726 		netdev_reset_tc(dev);
11727 	}
11728 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11729 	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11730 			       bp->tx_nr_rings + bp->rx_nr_rings;
11731 
11732 	if (netif_running(bp->dev))
11733 		return bnxt_open_nic(bp, true, false);
11734 
11735 	return 0;
11736 }
11737 
11738 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11739 				  void *cb_priv)
11740 {
11741 	struct bnxt *bp = cb_priv;
11742 
11743 	if (!bnxt_tc_flower_enabled(bp) ||
11744 	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11745 		return -EOPNOTSUPP;
11746 
11747 	switch (type) {
11748 	case TC_SETUP_CLSFLOWER:
11749 		return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11750 	default:
11751 		return -EOPNOTSUPP;
11752 	}
11753 }
11754 
11755 LIST_HEAD(bnxt_block_cb_list);
11756 
11757 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11758 			 void *type_data)
11759 {
11760 	struct bnxt *bp = netdev_priv(dev);
11761 
11762 	switch (type) {
11763 	case TC_SETUP_BLOCK:
11764 		return flow_block_cb_setup_simple(type_data,
11765 						  &bnxt_block_cb_list,
11766 						  bnxt_setup_tc_block_cb,
11767 						  bp, bp, true);
11768 	case TC_SETUP_QDISC_MQPRIO: {
11769 		struct tc_mqprio_qopt *mqprio = type_data;
11770 
11771 		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11772 
11773 		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11774 	}
11775 	default:
11776 		return -EOPNOTSUPP;
11777 	}
11778 }
11779 
11780 #ifdef CONFIG_RFS_ACCEL
11781 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11782 			    struct bnxt_ntuple_filter *f2)
11783 {
11784 	struct flow_keys *keys1 = &f1->fkeys;
11785 	struct flow_keys *keys2 = &f2->fkeys;
11786 
11787 	if (keys1->basic.n_proto != keys2->basic.n_proto ||
11788 	    keys1->basic.ip_proto != keys2->basic.ip_proto)
11789 		return false;
11790 
11791 	if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11792 		if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11793 		    keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11794 			return false;
11795 	} else {
11796 		if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11797 			   sizeof(keys1->addrs.v6addrs.src)) ||
11798 		    memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11799 			   sizeof(keys1->addrs.v6addrs.dst)))
11800 			return false;
11801 	}
11802 
11803 	if (keys1->ports.ports == keys2->ports.ports &&
11804 	    keys1->control.flags == keys2->control.flags &&
11805 	    ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11806 	    ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11807 		return true;
11808 
11809 	return false;
11810 }
11811 
11812 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11813 			      u16 rxq_index, u32 flow_id)
11814 {
11815 	struct bnxt *bp = netdev_priv(dev);
11816 	struct bnxt_ntuple_filter *fltr, *new_fltr;
11817 	struct flow_keys *fkeys;
11818 	struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11819 	int rc = 0, idx, bit_id, l2_idx = 0;
11820 	struct hlist_head *head;
11821 	u32 flags;
11822 
11823 	if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11824 		struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11825 		int off = 0, j;
11826 
11827 		netif_addr_lock_bh(dev);
11828 		for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11829 			if (ether_addr_equal(eth->h_dest,
11830 					     vnic->uc_list + off)) {
11831 				l2_idx = j + 1;
11832 				break;
11833 			}
11834 		}
11835 		netif_addr_unlock_bh(dev);
11836 		if (!l2_idx)
11837 			return -EINVAL;
11838 	}
11839 	new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11840 	if (!new_fltr)
11841 		return -ENOMEM;
11842 
11843 	fkeys = &new_fltr->fkeys;
11844 	if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11845 		rc = -EPROTONOSUPPORT;
11846 		goto err_free;
11847 	}
11848 
11849 	if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11850 	     fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11851 	    ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11852 	     (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11853 		rc = -EPROTONOSUPPORT;
11854 		goto err_free;
11855 	}
11856 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11857 	    bp->hwrm_spec_code < 0x10601) {
11858 		rc = -EPROTONOSUPPORT;
11859 		goto err_free;
11860 	}
11861 	flags = fkeys->control.flags;
11862 	if (((flags & FLOW_DIS_ENCAPSULATION) &&
11863 	     bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11864 		rc = -EPROTONOSUPPORT;
11865 		goto err_free;
11866 	}
11867 
11868 	memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11869 	memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11870 
11871 	idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11872 	head = &bp->ntp_fltr_hash_tbl[idx];
11873 	rcu_read_lock();
11874 	hlist_for_each_entry_rcu(fltr, head, hash) {
11875 		if (bnxt_fltr_match(fltr, new_fltr)) {
11876 			rcu_read_unlock();
11877 			rc = 0;
11878 			goto err_free;
11879 		}
11880 	}
11881 	rcu_read_unlock();
11882 
11883 	spin_lock_bh(&bp->ntp_fltr_lock);
11884 	bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11885 					 BNXT_NTP_FLTR_MAX_FLTR, 0);
11886 	if (bit_id < 0) {
11887 		spin_unlock_bh(&bp->ntp_fltr_lock);
11888 		rc = -ENOMEM;
11889 		goto err_free;
11890 	}
11891 
11892 	new_fltr->sw_id = (u16)bit_id;
11893 	new_fltr->flow_id = flow_id;
11894 	new_fltr->l2_fltr_idx = l2_idx;
11895 	new_fltr->rxq = rxq_index;
11896 	hlist_add_head_rcu(&new_fltr->hash, head);
11897 	bp->ntp_fltr_count++;
11898 	spin_unlock_bh(&bp->ntp_fltr_lock);
11899 
11900 	set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11901 	bnxt_queue_sp_work(bp);
11902 
11903 	return new_fltr->sw_id;
11904 
11905 err_free:
11906 	kfree(new_fltr);
11907 	return rc;
11908 }
11909 
11910 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11911 {
11912 	int i;
11913 
11914 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11915 		struct hlist_head *head;
11916 		struct hlist_node *tmp;
11917 		struct bnxt_ntuple_filter *fltr;
11918 		int rc;
11919 
11920 		head = &bp->ntp_fltr_hash_tbl[i];
11921 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11922 			bool del = false;
11923 
11924 			if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11925 				if (rps_may_expire_flow(bp->dev, fltr->rxq,
11926 							fltr->flow_id,
11927 							fltr->sw_id)) {
11928 					bnxt_hwrm_cfa_ntuple_filter_free(bp,
11929 									 fltr);
11930 					del = true;
11931 				}
11932 			} else {
11933 				rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11934 								       fltr);
11935 				if (rc)
11936 					del = true;
11937 				else
11938 					set_bit(BNXT_FLTR_VALID, &fltr->state);
11939 			}
11940 
11941 			if (del) {
11942 				spin_lock_bh(&bp->ntp_fltr_lock);
11943 				hlist_del_rcu(&fltr->hash);
11944 				bp->ntp_fltr_count--;
11945 				spin_unlock_bh(&bp->ntp_fltr_lock);
11946 				synchronize_rcu();
11947 				clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11948 				kfree(fltr);
11949 			}
11950 		}
11951 	}
11952 	if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11953 		netdev_info(bp->dev, "Receive PF driver unload event!\n");
11954 }
11955 
11956 #else
11957 
11958 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11959 {
11960 }
11961 
11962 #endif /* CONFIG_RFS_ACCEL */
11963 
11964 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11965 {
11966 	struct bnxt *bp = netdev_priv(netdev);
11967 	struct udp_tunnel_info ti;
11968 	unsigned int cmd;
11969 
11970 	udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11971 	if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11972 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11973 	else
11974 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11975 
11976 	if (ti.port)
11977 		return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11978 
11979 	return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11980 }
11981 
11982 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11983 	.sync_table	= bnxt_udp_tunnel_sync,
11984 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11985 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11986 	.tables		= {
11987 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11988 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11989 	},
11990 };
11991 
11992 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11993 			       struct net_device *dev, u32 filter_mask,
11994 			       int nlflags)
11995 {
11996 	struct bnxt *bp = netdev_priv(dev);
11997 
11998 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11999 				       nlflags, filter_mask, NULL);
12000 }
12001 
12002 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
12003 			       u16 flags, struct netlink_ext_ack *extack)
12004 {
12005 	struct bnxt *bp = netdev_priv(dev);
12006 	struct nlattr *attr, *br_spec;
12007 	int rem, rc = 0;
12008 
12009 	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
12010 		return -EOPNOTSUPP;
12011 
12012 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12013 	if (!br_spec)
12014 		return -EINVAL;
12015 
12016 	nla_for_each_nested(attr, br_spec, rem) {
12017 		u16 mode;
12018 
12019 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
12020 			continue;
12021 
12022 		if (nla_len(attr) < sizeof(mode))
12023 			return -EINVAL;
12024 
12025 		mode = nla_get_u16(attr);
12026 		if (mode == bp->br_mode)
12027 			break;
12028 
12029 		rc = bnxt_hwrm_set_br_mode(bp, mode);
12030 		if (!rc)
12031 			bp->br_mode = mode;
12032 		break;
12033 	}
12034 	return rc;
12035 }
12036 
12037 int bnxt_get_port_parent_id(struct net_device *dev,
12038 			    struct netdev_phys_item_id *ppid)
12039 {
12040 	struct bnxt *bp = netdev_priv(dev);
12041 
12042 	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
12043 		return -EOPNOTSUPP;
12044 
12045 	/* The PF and it's VF-reps only support the switchdev framework */
12046 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
12047 		return -EOPNOTSUPP;
12048 
12049 	ppid->id_len = sizeof(bp->dsn);
12050 	memcpy(ppid->id, bp->dsn, ppid->id_len);
12051 
12052 	return 0;
12053 }
12054 
12055 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
12056 {
12057 	struct bnxt *bp = netdev_priv(dev);
12058 
12059 	return &bp->dl_port;
12060 }
12061 
12062 static const struct net_device_ops bnxt_netdev_ops = {
12063 	.ndo_open		= bnxt_open,
12064 	.ndo_start_xmit		= bnxt_start_xmit,
12065 	.ndo_stop		= bnxt_close,
12066 	.ndo_get_stats64	= bnxt_get_stats64,
12067 	.ndo_set_rx_mode	= bnxt_set_rx_mode,
12068 	.ndo_do_ioctl		= bnxt_ioctl,
12069 	.ndo_validate_addr	= eth_validate_addr,
12070 	.ndo_set_mac_address	= bnxt_change_mac_addr,
12071 	.ndo_change_mtu		= bnxt_change_mtu,
12072 	.ndo_fix_features	= bnxt_fix_features,
12073 	.ndo_set_features	= bnxt_set_features,
12074 	.ndo_tx_timeout		= bnxt_tx_timeout,
12075 #ifdef CONFIG_BNXT_SRIOV
12076 	.ndo_get_vf_config	= bnxt_get_vf_config,
12077 	.ndo_set_vf_mac		= bnxt_set_vf_mac,
12078 	.ndo_set_vf_vlan	= bnxt_set_vf_vlan,
12079 	.ndo_set_vf_rate	= bnxt_set_vf_bw,
12080 	.ndo_set_vf_link_state	= bnxt_set_vf_link_state,
12081 	.ndo_set_vf_spoofchk	= bnxt_set_vf_spoofchk,
12082 	.ndo_set_vf_trust	= bnxt_set_vf_trust,
12083 #endif
12084 	.ndo_setup_tc           = bnxt_setup_tc,
12085 #ifdef CONFIG_RFS_ACCEL
12086 	.ndo_rx_flow_steer	= bnxt_rx_flow_steer,
12087 #endif
12088 	.ndo_udp_tunnel_add	= udp_tunnel_nic_add_port,
12089 	.ndo_udp_tunnel_del	= udp_tunnel_nic_del_port,
12090 	.ndo_bpf		= bnxt_xdp,
12091 	.ndo_xdp_xmit		= bnxt_xdp_xmit,
12092 	.ndo_bridge_getlink	= bnxt_bridge_getlink,
12093 	.ndo_bridge_setlink	= bnxt_bridge_setlink,
12094 	.ndo_get_devlink_port	= bnxt_get_devlink_port,
12095 };
12096 
12097 static void bnxt_remove_one(struct pci_dev *pdev)
12098 {
12099 	struct net_device *dev = pci_get_drvdata(pdev);
12100 	struct bnxt *bp = netdev_priv(dev);
12101 
12102 	if (BNXT_PF(bp))
12103 		bnxt_sriov_disable(bp);
12104 
12105 	if (BNXT_PF(bp))
12106 		devlink_port_type_clear(&bp->dl_port);
12107 	pci_disable_pcie_error_reporting(pdev);
12108 	unregister_netdev(dev);
12109 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
12110 	/* Flush any pending tasks */
12111 	cancel_work_sync(&bp->sp_task);
12112 	cancel_delayed_work_sync(&bp->fw_reset_task);
12113 	bp->sp_event = 0;
12114 
12115 	bnxt_dl_fw_reporters_destroy(bp, true);
12116 	bnxt_dl_unregister(bp);
12117 	bnxt_shutdown_tc(bp);
12118 
12119 	bnxt_clear_int_mode(bp);
12120 	bnxt_hwrm_func_drv_unrgtr(bp);
12121 	bnxt_free_hwrm_resources(bp);
12122 	bnxt_free_hwrm_short_cmd_req(bp);
12123 	bnxt_ethtool_free(bp);
12124 	bnxt_dcb_free(bp);
12125 	kfree(bp->edev);
12126 	bp->edev = NULL;
12127 	kfree(bp->fw_health);
12128 	bp->fw_health = NULL;
12129 	bnxt_cleanup_pci(bp);
12130 	bnxt_free_ctx_mem(bp);
12131 	kfree(bp->ctx);
12132 	bp->ctx = NULL;
12133 	kfree(bp->rss_indir_tbl);
12134 	bp->rss_indir_tbl = NULL;
12135 	bnxt_free_port_stats(bp);
12136 	free_netdev(dev);
12137 }
12138 
12139 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
12140 {
12141 	int rc = 0;
12142 	struct bnxt_link_info *link_info = &bp->link_info;
12143 
12144 	rc = bnxt_hwrm_phy_qcaps(bp);
12145 	if (rc) {
12146 		netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
12147 			   rc);
12148 		return rc;
12149 	}
12150 	if (!fw_dflt)
12151 		return 0;
12152 
12153 	rc = bnxt_update_link(bp, false);
12154 	if (rc) {
12155 		netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
12156 			   rc);
12157 		return rc;
12158 	}
12159 
12160 	/* Older firmware does not have supported_auto_speeds, so assume
12161 	 * that all supported speeds can be autonegotiated.
12162 	 */
12163 	if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
12164 		link_info->support_auto_speeds = link_info->support_speeds;
12165 
12166 	bnxt_init_ethtool_link_settings(bp);
12167 	return 0;
12168 }
12169 
12170 static int bnxt_get_max_irq(struct pci_dev *pdev)
12171 {
12172 	u16 ctrl;
12173 
12174 	if (!pdev->msix_cap)
12175 		return 1;
12176 
12177 	pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
12178 	return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
12179 }
12180 
12181 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12182 				int *max_cp)
12183 {
12184 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
12185 	int max_ring_grps = 0, max_irq;
12186 
12187 	*max_tx = hw_resc->max_tx_rings;
12188 	*max_rx = hw_resc->max_rx_rings;
12189 	*max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
12190 	max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
12191 			bnxt_get_ulp_msix_num(bp),
12192 			hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
12193 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
12194 		*max_cp = min_t(int, *max_cp, max_irq);
12195 	max_ring_grps = hw_resc->max_hw_ring_grps;
12196 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
12197 		*max_cp -= 1;
12198 		*max_rx -= 2;
12199 	}
12200 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
12201 		*max_rx >>= 1;
12202 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
12203 		bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
12204 		/* On P5 chips, max_cp output param should be available NQs */
12205 		*max_cp = max_irq;
12206 	}
12207 	*max_rx = min_t(int, *max_rx, max_ring_grps);
12208 }
12209 
12210 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
12211 {
12212 	int rx, tx, cp;
12213 
12214 	_bnxt_get_max_rings(bp, &rx, &tx, &cp);
12215 	*max_rx = rx;
12216 	*max_tx = tx;
12217 	if (!rx || !tx || !cp)
12218 		return -ENOMEM;
12219 
12220 	return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
12221 }
12222 
12223 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
12224 			       bool shared)
12225 {
12226 	int rc;
12227 
12228 	rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12229 	if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
12230 		/* Not enough rings, try disabling agg rings. */
12231 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
12232 		rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
12233 		if (rc) {
12234 			/* set BNXT_FLAG_AGG_RINGS back for consistency */
12235 			bp->flags |= BNXT_FLAG_AGG_RINGS;
12236 			return rc;
12237 		}
12238 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
12239 		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12240 		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
12241 		bnxt_set_ring_params(bp);
12242 	}
12243 
12244 	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
12245 		int max_cp, max_stat, max_irq;
12246 
12247 		/* Reserve minimum resources for RoCE */
12248 		max_cp = bnxt_get_max_func_cp_rings(bp);
12249 		max_stat = bnxt_get_max_func_stat_ctxs(bp);
12250 		max_irq = bnxt_get_max_func_irqs(bp);
12251 		if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
12252 		    max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
12253 		    max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
12254 			return 0;
12255 
12256 		max_cp -= BNXT_MIN_ROCE_CP_RINGS;
12257 		max_irq -= BNXT_MIN_ROCE_CP_RINGS;
12258 		max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
12259 		max_cp = min_t(int, max_cp, max_irq);
12260 		max_cp = min_t(int, max_cp, max_stat);
12261 		rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
12262 		if (rc)
12263 			rc = 0;
12264 	}
12265 	return rc;
12266 }
12267 
12268 /* In initial default shared ring setting, each shared ring must have a
12269  * RX/TX ring pair.
12270  */
12271 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
12272 {
12273 	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
12274 	bp->rx_nr_rings = bp->cp_nr_rings;
12275 	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
12276 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12277 }
12278 
12279 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
12280 {
12281 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
12282 
12283 	if (!bnxt_can_reserve_rings(bp))
12284 		return 0;
12285 
12286 	if (sh)
12287 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
12288 	dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
12289 	/* Reduce default rings on multi-port cards so that total default
12290 	 * rings do not exceed CPU count.
12291 	 */
12292 	if (bp->port_count > 1) {
12293 		int max_rings =
12294 			max_t(int, num_online_cpus() / bp->port_count, 1);
12295 
12296 		dflt_rings = min_t(int, dflt_rings, max_rings);
12297 	}
12298 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
12299 	if (rc)
12300 		return rc;
12301 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
12302 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
12303 	if (sh)
12304 		bnxt_trim_dflt_sh_rings(bp);
12305 	else
12306 		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
12307 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
12308 
12309 	rc = __bnxt_reserve_rings(bp);
12310 	if (rc)
12311 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
12312 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12313 	if (sh)
12314 		bnxt_trim_dflt_sh_rings(bp);
12315 
12316 	/* Rings may have been trimmed, re-reserve the trimmed rings. */
12317 	if (bnxt_need_reserve_rings(bp)) {
12318 		rc = __bnxt_reserve_rings(bp);
12319 		if (rc)
12320 			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
12321 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12322 	}
12323 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
12324 		bp->rx_nr_rings++;
12325 		bp->cp_nr_rings++;
12326 	}
12327 	if (rc) {
12328 		bp->tx_nr_rings = 0;
12329 		bp->rx_nr_rings = 0;
12330 	}
12331 	return rc;
12332 }
12333 
12334 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12335 {
12336 	int rc;
12337 
12338 	if (bp->tx_nr_rings)
12339 		return 0;
12340 
12341 	bnxt_ulp_irq_stop(bp);
12342 	bnxt_clear_int_mode(bp);
12343 	rc = bnxt_set_dflt_rings(bp, true);
12344 	if (rc) {
12345 		netdev_err(bp->dev, "Not enough rings available.\n");
12346 		goto init_dflt_ring_err;
12347 	}
12348 	rc = bnxt_init_int_mode(bp);
12349 	if (rc)
12350 		goto init_dflt_ring_err;
12351 
12352 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12353 	if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12354 		bp->flags |= BNXT_FLAG_RFS;
12355 		bp->dev->features |= NETIF_F_NTUPLE;
12356 	}
12357 init_dflt_ring_err:
12358 	bnxt_ulp_irq_restart(bp, rc);
12359 	return rc;
12360 }
12361 
12362 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12363 {
12364 	int rc;
12365 
12366 	ASSERT_RTNL();
12367 	bnxt_hwrm_func_qcaps(bp);
12368 
12369 	if (netif_running(bp->dev))
12370 		__bnxt_close_nic(bp, true, false);
12371 
12372 	bnxt_ulp_irq_stop(bp);
12373 	bnxt_clear_int_mode(bp);
12374 	rc = bnxt_init_int_mode(bp);
12375 	bnxt_ulp_irq_restart(bp, rc);
12376 
12377 	if (netif_running(bp->dev)) {
12378 		if (rc)
12379 			dev_close(bp->dev);
12380 		else
12381 			rc = bnxt_open_nic(bp, true, false);
12382 	}
12383 
12384 	return rc;
12385 }
12386 
12387 static int bnxt_init_mac_addr(struct bnxt *bp)
12388 {
12389 	int rc = 0;
12390 
12391 	if (BNXT_PF(bp)) {
12392 		memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12393 	} else {
12394 #ifdef CONFIG_BNXT_SRIOV
12395 		struct bnxt_vf_info *vf = &bp->vf;
12396 		bool strict_approval = true;
12397 
12398 		if (is_valid_ether_addr(vf->mac_addr)) {
12399 			/* overwrite netdev dev_addr with admin VF MAC */
12400 			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12401 			/* Older PF driver or firmware may not approve this
12402 			 * correctly.
12403 			 */
12404 			strict_approval = false;
12405 		} else {
12406 			eth_hw_addr_random(bp->dev);
12407 		}
12408 		rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12409 #endif
12410 	}
12411 	return rc;
12412 }
12413 
12414 #define BNXT_VPD_LEN	512
12415 static void bnxt_vpd_read_info(struct bnxt *bp)
12416 {
12417 	struct pci_dev *pdev = bp->pdev;
12418 	int i, len, pos, ro_size, size;
12419 	ssize_t vpd_size;
12420 	u8 *vpd_data;
12421 
12422 	vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12423 	if (!vpd_data)
12424 		return;
12425 
12426 	vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12427 	if (vpd_size <= 0) {
12428 		netdev_err(bp->dev, "Unable to read VPD\n");
12429 		goto exit;
12430 	}
12431 
12432 	i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12433 	if (i < 0) {
12434 		netdev_err(bp->dev, "VPD READ-Only not found\n");
12435 		goto exit;
12436 	}
12437 
12438 	ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12439 	i += PCI_VPD_LRDT_TAG_SIZE;
12440 	if (i + ro_size > vpd_size)
12441 		goto exit;
12442 
12443 	pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12444 					PCI_VPD_RO_KEYWORD_PARTNO);
12445 	if (pos < 0)
12446 		goto read_sn;
12447 
12448 	len = pci_vpd_info_field_size(&vpd_data[pos]);
12449 	pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12450 	if (len + pos > vpd_size)
12451 		goto read_sn;
12452 
12453 	size = min(len, BNXT_VPD_FLD_LEN - 1);
12454 	memcpy(bp->board_partno, &vpd_data[pos], size);
12455 
12456 read_sn:
12457 	pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12458 					PCI_VPD_RO_KEYWORD_SERIALNO);
12459 	if (pos < 0)
12460 		goto exit;
12461 
12462 	len = pci_vpd_info_field_size(&vpd_data[pos]);
12463 	pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12464 	if (len + pos > vpd_size)
12465 		goto exit;
12466 
12467 	size = min(len, BNXT_VPD_FLD_LEN - 1);
12468 	memcpy(bp->board_serialno, &vpd_data[pos], size);
12469 exit:
12470 	kfree(vpd_data);
12471 }
12472 
12473 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12474 {
12475 	struct pci_dev *pdev = bp->pdev;
12476 	u64 qword;
12477 
12478 	qword = pci_get_dsn(pdev);
12479 	if (!qword) {
12480 		netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12481 		return -EOPNOTSUPP;
12482 	}
12483 
12484 	put_unaligned_le64(qword, dsn);
12485 
12486 	bp->flags |= BNXT_FLAG_DSN_VALID;
12487 	return 0;
12488 }
12489 
12490 static int bnxt_map_db_bar(struct bnxt *bp)
12491 {
12492 	if (!bp->db_size)
12493 		return -ENODEV;
12494 	bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12495 	if (!bp->bar1)
12496 		return -ENOMEM;
12497 	return 0;
12498 }
12499 
12500 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12501 {
12502 	struct net_device *dev;
12503 	struct bnxt *bp;
12504 	int rc, max_irqs;
12505 
12506 	if (pci_is_bridge(pdev))
12507 		return -ENODEV;
12508 
12509 	/* Clear any pending DMA transactions from crash kernel
12510 	 * while loading driver in capture kernel.
12511 	 */
12512 	if (is_kdump_kernel()) {
12513 		pci_clear_master(pdev);
12514 		pcie_flr(pdev);
12515 	}
12516 
12517 	max_irqs = bnxt_get_max_irq(pdev);
12518 	dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12519 	if (!dev)
12520 		return -ENOMEM;
12521 
12522 	bp = netdev_priv(dev);
12523 	bp->msg_enable = BNXT_DEF_MSG_ENABLE;
12524 	bnxt_set_max_func_irqs(bp, max_irqs);
12525 
12526 	if (bnxt_vf_pciid(ent->driver_data))
12527 		bp->flags |= BNXT_FLAG_VF;
12528 
12529 	if (pdev->msix_cap)
12530 		bp->flags |= BNXT_FLAG_MSIX_CAP;
12531 
12532 	rc = bnxt_init_board(pdev, dev);
12533 	if (rc < 0)
12534 		goto init_err_free;
12535 
12536 	dev->netdev_ops = &bnxt_netdev_ops;
12537 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12538 	dev->ethtool_ops = &bnxt_ethtool_ops;
12539 	pci_set_drvdata(pdev, dev);
12540 
12541 	if (BNXT_PF(bp))
12542 		bnxt_vpd_read_info(bp);
12543 
12544 	rc = bnxt_alloc_hwrm_resources(bp);
12545 	if (rc)
12546 		goto init_err_pci_clean;
12547 
12548 	mutex_init(&bp->hwrm_cmd_lock);
12549 	mutex_init(&bp->link_lock);
12550 
12551 	rc = bnxt_fw_init_one_p1(bp);
12552 	if (rc)
12553 		goto init_err_pci_clean;
12554 
12555 	if (BNXT_CHIP_P5(bp)) {
12556 		bp->flags |= BNXT_FLAG_CHIP_P5;
12557 		if (BNXT_CHIP_SR2(bp))
12558 			bp->flags |= BNXT_FLAG_CHIP_SR2;
12559 	}
12560 
12561 	rc = bnxt_alloc_rss_indir_tbl(bp);
12562 	if (rc)
12563 		goto init_err_pci_clean;
12564 
12565 	rc = bnxt_fw_init_one_p2(bp);
12566 	if (rc)
12567 		goto init_err_pci_clean;
12568 
12569 	rc = bnxt_map_db_bar(bp);
12570 	if (rc) {
12571 		dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12572 			rc);
12573 		goto init_err_pci_clean;
12574 	}
12575 
12576 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12577 			   NETIF_F_TSO | NETIF_F_TSO6 |
12578 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12579 			   NETIF_F_GSO_IPXIP4 |
12580 			   NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12581 			   NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12582 			   NETIF_F_RXCSUM | NETIF_F_GRO;
12583 
12584 	if (BNXT_SUPPORTS_TPA(bp))
12585 		dev->hw_features |= NETIF_F_LRO;
12586 
12587 	dev->hw_enc_features =
12588 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12589 			NETIF_F_TSO | NETIF_F_TSO6 |
12590 			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12591 			NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12592 			NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12593 	dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12594 
12595 	dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12596 				    NETIF_F_GSO_GRE_CSUM;
12597 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12598 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12599 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12600 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12601 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12602 	if (BNXT_SUPPORTS_TPA(bp))
12603 		dev->hw_features |= NETIF_F_GRO_HW;
12604 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12605 	if (dev->features & NETIF_F_GRO_HW)
12606 		dev->features &= ~NETIF_F_LRO;
12607 	dev->priv_flags |= IFF_UNICAST_FLT;
12608 
12609 #ifdef CONFIG_BNXT_SRIOV
12610 	init_waitqueue_head(&bp->sriov_cfg_wait);
12611 	mutex_init(&bp->sriov_lock);
12612 #endif
12613 	if (BNXT_SUPPORTS_TPA(bp)) {
12614 		bp->gro_func = bnxt_gro_func_5730x;
12615 		if (BNXT_CHIP_P4(bp))
12616 			bp->gro_func = bnxt_gro_func_5731x;
12617 		else if (BNXT_CHIP_P5(bp))
12618 			bp->gro_func = bnxt_gro_func_5750x;
12619 	}
12620 	if (!BNXT_CHIP_P4_PLUS(bp))
12621 		bp->flags |= BNXT_FLAG_DOUBLE_DB;
12622 
12623 	bp->ulp_probe = bnxt_ulp_probe;
12624 
12625 	rc = bnxt_init_mac_addr(bp);
12626 	if (rc) {
12627 		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12628 		rc = -EADDRNOTAVAIL;
12629 		goto init_err_pci_clean;
12630 	}
12631 
12632 	if (BNXT_PF(bp)) {
12633 		/* Read the adapter's DSN to use as the eswitch switch_id */
12634 		rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12635 	}
12636 
12637 	/* MTU range: 60 - FW defined max */
12638 	dev->min_mtu = ETH_ZLEN;
12639 	dev->max_mtu = bp->max_mtu;
12640 
12641 	rc = bnxt_probe_phy(bp, true);
12642 	if (rc)
12643 		goto init_err_pci_clean;
12644 
12645 	bnxt_set_rx_skb_mode(bp, false);
12646 	bnxt_set_tpa_flags(bp);
12647 	bnxt_set_ring_params(bp);
12648 	rc = bnxt_set_dflt_rings(bp, true);
12649 	if (rc) {
12650 		netdev_err(bp->dev, "Not enough rings available.\n");
12651 		rc = -ENOMEM;
12652 		goto init_err_pci_clean;
12653 	}
12654 
12655 	bnxt_fw_init_one_p3(bp);
12656 
12657 	if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12658 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
12659 
12660 	rc = bnxt_init_int_mode(bp);
12661 	if (rc)
12662 		goto init_err_pci_clean;
12663 
12664 	/* No TC has been set yet and rings may have been trimmed due to
12665 	 * limited MSIX, so we re-initialize the TX rings per TC.
12666 	 */
12667 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12668 
12669 	if (BNXT_PF(bp)) {
12670 		if (!bnxt_pf_wq) {
12671 			bnxt_pf_wq =
12672 				create_singlethread_workqueue("bnxt_pf_wq");
12673 			if (!bnxt_pf_wq) {
12674 				dev_err(&pdev->dev, "Unable to create workqueue.\n");
12675 				goto init_err_pci_clean;
12676 			}
12677 		}
12678 		rc = bnxt_init_tc(bp);
12679 		if (rc)
12680 			netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12681 				   rc);
12682 	}
12683 
12684 	bnxt_dl_register(bp);
12685 
12686 	rc = register_netdev(dev);
12687 	if (rc)
12688 		goto init_err_cleanup;
12689 
12690 	if (BNXT_PF(bp))
12691 		devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12692 	bnxt_dl_fw_reporters_create(bp);
12693 
12694 	netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12695 		    board_info[ent->driver_data].name,
12696 		    (long)pci_resource_start(pdev, 0), dev->dev_addr);
12697 	pcie_print_link_status(pdev);
12698 
12699 	pci_save_state(pdev);
12700 	return 0;
12701 
12702 init_err_cleanup:
12703 	bnxt_dl_unregister(bp);
12704 	bnxt_shutdown_tc(bp);
12705 	bnxt_clear_int_mode(bp);
12706 
12707 init_err_pci_clean:
12708 	bnxt_hwrm_func_drv_unrgtr(bp);
12709 	bnxt_free_hwrm_short_cmd_req(bp);
12710 	bnxt_free_hwrm_resources(bp);
12711 	kfree(bp->fw_health);
12712 	bp->fw_health = NULL;
12713 	bnxt_cleanup_pci(bp);
12714 	bnxt_free_ctx_mem(bp);
12715 	kfree(bp->ctx);
12716 	bp->ctx = NULL;
12717 	kfree(bp->rss_indir_tbl);
12718 	bp->rss_indir_tbl = NULL;
12719 
12720 init_err_free:
12721 	free_netdev(dev);
12722 	return rc;
12723 }
12724 
12725 static void bnxt_shutdown(struct pci_dev *pdev)
12726 {
12727 	struct net_device *dev = pci_get_drvdata(pdev);
12728 	struct bnxt *bp;
12729 
12730 	if (!dev)
12731 		return;
12732 
12733 	rtnl_lock();
12734 	bp = netdev_priv(dev);
12735 	if (!bp)
12736 		goto shutdown_exit;
12737 
12738 	if (netif_running(dev))
12739 		dev_close(dev);
12740 
12741 	bnxt_ulp_shutdown(bp);
12742 	bnxt_clear_int_mode(bp);
12743 	pci_disable_device(pdev);
12744 
12745 	if (system_state == SYSTEM_POWER_OFF) {
12746 		pci_wake_from_d3(pdev, bp->wol);
12747 		pci_set_power_state(pdev, PCI_D3hot);
12748 	}
12749 
12750 shutdown_exit:
12751 	rtnl_unlock();
12752 }
12753 
12754 #ifdef CONFIG_PM_SLEEP
12755 static int bnxt_suspend(struct device *device)
12756 {
12757 	struct net_device *dev = dev_get_drvdata(device);
12758 	struct bnxt *bp = netdev_priv(dev);
12759 	int rc = 0;
12760 
12761 	rtnl_lock();
12762 	bnxt_ulp_stop(bp);
12763 	if (netif_running(dev)) {
12764 		netif_device_detach(dev);
12765 		rc = bnxt_close(dev);
12766 	}
12767 	bnxt_hwrm_func_drv_unrgtr(bp);
12768 	pci_disable_device(bp->pdev);
12769 	bnxt_free_ctx_mem(bp);
12770 	kfree(bp->ctx);
12771 	bp->ctx = NULL;
12772 	rtnl_unlock();
12773 	return rc;
12774 }
12775 
12776 static int bnxt_resume(struct device *device)
12777 {
12778 	struct net_device *dev = dev_get_drvdata(device);
12779 	struct bnxt *bp = netdev_priv(dev);
12780 	int rc = 0;
12781 
12782 	rtnl_lock();
12783 	rc = pci_enable_device(bp->pdev);
12784 	if (rc) {
12785 		netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12786 			   rc);
12787 		goto resume_exit;
12788 	}
12789 	pci_set_master(bp->pdev);
12790 	if (bnxt_hwrm_ver_get(bp)) {
12791 		rc = -ENODEV;
12792 		goto resume_exit;
12793 	}
12794 	rc = bnxt_hwrm_func_reset(bp);
12795 	if (rc) {
12796 		rc = -EBUSY;
12797 		goto resume_exit;
12798 	}
12799 
12800 	rc = bnxt_hwrm_func_qcaps(bp);
12801 	if (rc)
12802 		goto resume_exit;
12803 
12804 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12805 		rc = -ENODEV;
12806 		goto resume_exit;
12807 	}
12808 
12809 	bnxt_get_wol_settings(bp);
12810 	if (netif_running(dev)) {
12811 		rc = bnxt_open(dev);
12812 		if (!rc)
12813 			netif_device_attach(dev);
12814 	}
12815 
12816 resume_exit:
12817 	bnxt_ulp_start(bp, rc);
12818 	if (!rc)
12819 		bnxt_reenable_sriov(bp);
12820 	rtnl_unlock();
12821 	return rc;
12822 }
12823 
12824 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12825 #define BNXT_PM_OPS (&bnxt_pm_ops)
12826 
12827 #else
12828 
12829 #define BNXT_PM_OPS NULL
12830 
12831 #endif /* CONFIG_PM_SLEEP */
12832 
12833 /**
12834  * bnxt_io_error_detected - called when PCI error is detected
12835  * @pdev: Pointer to PCI device
12836  * @state: The current pci connection state
12837  *
12838  * This function is called after a PCI bus error affecting
12839  * this device has been detected.
12840  */
12841 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12842 					       pci_channel_state_t state)
12843 {
12844 	struct net_device *netdev = pci_get_drvdata(pdev);
12845 	struct bnxt *bp = netdev_priv(netdev);
12846 
12847 	netdev_info(netdev, "PCI I/O error detected\n");
12848 
12849 	rtnl_lock();
12850 	netif_device_detach(netdev);
12851 
12852 	bnxt_ulp_stop(bp);
12853 
12854 	if (state == pci_channel_io_perm_failure) {
12855 		rtnl_unlock();
12856 		return PCI_ERS_RESULT_DISCONNECT;
12857 	}
12858 
12859 	if (state == pci_channel_io_frozen)
12860 		set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state);
12861 
12862 	if (netif_running(netdev))
12863 		bnxt_close(netdev);
12864 
12865 	pci_disable_device(pdev);
12866 	bnxt_free_ctx_mem(bp);
12867 	kfree(bp->ctx);
12868 	bp->ctx = NULL;
12869 	rtnl_unlock();
12870 
12871 	/* Request a slot slot reset. */
12872 	return PCI_ERS_RESULT_NEED_RESET;
12873 }
12874 
12875 /**
12876  * bnxt_io_slot_reset - called after the pci bus has been reset.
12877  * @pdev: Pointer to PCI device
12878  *
12879  * Restart the card from scratch, as if from a cold-boot.
12880  * At this point, the card has exprienced a hard reset,
12881  * followed by fixups by BIOS, and has its config space
12882  * set up identically to what it was at cold boot.
12883  */
12884 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12885 {
12886 	struct net_device *netdev = pci_get_drvdata(pdev);
12887 	struct bnxt *bp = netdev_priv(netdev);
12888 	int err = 0, off;
12889 	pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12890 
12891 	netdev_info(bp->dev, "PCI Slot Reset\n");
12892 
12893 	rtnl_lock();
12894 
12895 	if (pci_enable_device(pdev)) {
12896 		dev_err(&pdev->dev,
12897 			"Cannot re-enable PCI device after reset.\n");
12898 	} else {
12899 		pci_set_master(pdev);
12900 		/* Upon fatal error, our device internal logic that latches to
12901 		 * BAR value is getting reset and will restore only upon
12902 		 * rewritting the BARs.
12903 		 *
12904 		 * As pci_restore_state() does not re-write the BARs if the
12905 		 * value is same as saved value earlier, driver needs to
12906 		 * write the BARs to 0 to force restore, in case of fatal error.
12907 		 */
12908 		if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN,
12909 				       &bp->state)) {
12910 			for (off = PCI_BASE_ADDRESS_0;
12911 			     off <= PCI_BASE_ADDRESS_5; off += 4)
12912 				pci_write_config_dword(bp->pdev, off, 0);
12913 		}
12914 		pci_restore_state(pdev);
12915 		pci_save_state(pdev);
12916 
12917 		err = bnxt_hwrm_func_reset(bp);
12918 		if (!err) {
12919 			err = bnxt_hwrm_func_qcaps(bp);
12920 			if (!err && netif_running(netdev))
12921 				err = bnxt_open(netdev);
12922 		}
12923 		bnxt_ulp_start(bp, err);
12924 		if (!err) {
12925 			bnxt_reenable_sriov(bp);
12926 			result = PCI_ERS_RESULT_RECOVERED;
12927 		}
12928 	}
12929 
12930 	if (result != PCI_ERS_RESULT_RECOVERED) {
12931 		if (netif_running(netdev))
12932 			dev_close(netdev);
12933 		pci_disable_device(pdev);
12934 	}
12935 
12936 	rtnl_unlock();
12937 
12938 	return result;
12939 }
12940 
12941 /**
12942  * bnxt_io_resume - called when traffic can start flowing again.
12943  * @pdev: Pointer to PCI device
12944  *
12945  * This callback is called when the error recovery driver tells
12946  * us that its OK to resume normal operation.
12947  */
12948 static void bnxt_io_resume(struct pci_dev *pdev)
12949 {
12950 	struct net_device *netdev = pci_get_drvdata(pdev);
12951 
12952 	rtnl_lock();
12953 
12954 	netif_device_attach(netdev);
12955 
12956 	rtnl_unlock();
12957 }
12958 
12959 static const struct pci_error_handlers bnxt_err_handler = {
12960 	.error_detected	= bnxt_io_error_detected,
12961 	.slot_reset	= bnxt_io_slot_reset,
12962 	.resume		= bnxt_io_resume
12963 };
12964 
12965 static struct pci_driver bnxt_pci_driver = {
12966 	.name		= DRV_MODULE_NAME,
12967 	.id_table	= bnxt_pci_tbl,
12968 	.probe		= bnxt_init_one,
12969 	.remove		= bnxt_remove_one,
12970 	.shutdown	= bnxt_shutdown,
12971 	.driver.pm	= BNXT_PM_OPS,
12972 	.err_handler	= &bnxt_err_handler,
12973 #if defined(CONFIG_BNXT_SRIOV)
12974 	.sriov_configure = bnxt_sriov_configure,
12975 #endif
12976 };
12977 
12978 static int __init bnxt_init(void)
12979 {
12980 	bnxt_debug_init();
12981 	return pci_register_driver(&bnxt_pci_driver);
12982 }
12983 
12984 static void __exit bnxt_exit(void)
12985 {
12986 	pci_unregister_driver(&bnxt_pci_driver);
12987 	if (bnxt_pf_wq)
12988 		destroy_workqueue(bnxt_pf_wq);
12989 	bnxt_debug_exit();
12990 }
12991 
12992 module_init(bnxt_init);
12993 module_exit(bnxt_exit);
12994