1 /* Broadcom NetXtreme-C/E network driver.
2  *
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2019 Broadcom Limited
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation.
9  */
10 
11 #include <linux/module.h>
12 
13 #include <linux/stringify.h>
14 #include <linux/kernel.h>
15 #include <linux/timer.h>
16 #include <linux/errno.h>
17 #include <linux/ioport.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/interrupt.h>
21 #include <linux/pci.h>
22 #include <linux/netdevice.h>
23 #include <linux/etherdevice.h>
24 #include <linux/skbuff.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/bitops.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <asm/byteorder.h>
31 #include <asm/page.h>
32 #include <linux/time.h>
33 #include <linux/mii.h>
34 #include <linux/mdio.h>
35 #include <linux/if.h>
36 #include <linux/if_vlan.h>
37 #include <linux/if_bridge.h>
38 #include <linux/rtc.h>
39 #include <linux/bpf.h>
40 #include <net/ip.h>
41 #include <net/tcp.h>
42 #include <net/udp.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <net/udp_tunnel.h>
46 #include <linux/workqueue.h>
47 #include <linux/prefetch.h>
48 #include <linux/cache.h>
49 #include <linux/log2.h>
50 #include <linux/aer.h>
51 #include <linux/bitmap.h>
52 #include <linux/cpu_rmap.h>
53 #include <linux/cpumask.h>
54 #include <net/pkt_cls.h>
55 #include <linux/hwmon.h>
56 #include <linux/hwmon-sysfs.h>
57 #include <net/page_pool.h>
58 
59 #include "bnxt_hsi.h"
60 #include "bnxt.h"
61 #include "bnxt_ulp.h"
62 #include "bnxt_sriov.h"
63 #include "bnxt_ethtool.h"
64 #include "bnxt_dcb.h"
65 #include "bnxt_xdp.h"
66 #include "bnxt_vfr.h"
67 #include "bnxt_tc.h"
68 #include "bnxt_devlink.h"
69 #include "bnxt_debugfs.h"
70 
71 #define BNXT_TX_TIMEOUT		(5 * HZ)
72 
73 MODULE_LICENSE("GPL");
74 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
75 
76 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
77 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
78 #define BNXT_RX_COPY_THRESH 256
79 
80 #define BNXT_TX_PUSH_THRESH 164
81 
82 enum board_idx {
83 	BCM57301,
84 	BCM57302,
85 	BCM57304,
86 	BCM57417_NPAR,
87 	BCM58700,
88 	BCM57311,
89 	BCM57312,
90 	BCM57402,
91 	BCM57404,
92 	BCM57406,
93 	BCM57402_NPAR,
94 	BCM57407,
95 	BCM57412,
96 	BCM57414,
97 	BCM57416,
98 	BCM57417,
99 	BCM57412_NPAR,
100 	BCM57314,
101 	BCM57417_SFP,
102 	BCM57416_SFP,
103 	BCM57404_NPAR,
104 	BCM57406_NPAR,
105 	BCM57407_SFP,
106 	BCM57407_NPAR,
107 	BCM57414_NPAR,
108 	BCM57416_NPAR,
109 	BCM57452,
110 	BCM57454,
111 	BCM5745x_NPAR,
112 	BCM57508,
113 	BCM57504,
114 	BCM57502,
115 	BCM57508_NPAR,
116 	BCM57504_NPAR,
117 	BCM57502_NPAR,
118 	BCM58802,
119 	BCM58804,
120 	BCM58808,
121 	NETXTREME_E_VF,
122 	NETXTREME_C_VF,
123 	NETXTREME_S_VF,
124 	NETXTREME_E_P5_VF,
125 };
126 
127 /* indexed by enum above */
128 static const struct {
129 	char *name;
130 } board_info[] = {
131 	[BCM57301] = { "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet" },
132 	[BCM57302] = { "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet" },
133 	[BCM57304] = { "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
134 	[BCM57417_NPAR] = { "Broadcom BCM57417 NetXtreme-E Ethernet Partition" },
135 	[BCM58700] = { "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet" },
136 	[BCM57311] = { "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet" },
137 	[BCM57312] = { "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet" },
138 	[BCM57402] = { "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet" },
139 	[BCM57404] = { "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet" },
140 	[BCM57406] = { "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet" },
141 	[BCM57402_NPAR] = { "Broadcom BCM57402 NetXtreme-E Ethernet Partition" },
142 	[BCM57407] = { "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet" },
143 	[BCM57412] = { "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet" },
144 	[BCM57414] = { "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet" },
145 	[BCM57416] = { "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet" },
146 	[BCM57417] = { "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet" },
147 	[BCM57412_NPAR] = { "Broadcom BCM57412 NetXtreme-E Ethernet Partition" },
148 	[BCM57314] = { "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet" },
149 	[BCM57417_SFP] = { "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet" },
150 	[BCM57416_SFP] = { "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet" },
151 	[BCM57404_NPAR] = { "Broadcom BCM57404 NetXtreme-E Ethernet Partition" },
152 	[BCM57406_NPAR] = { "Broadcom BCM57406 NetXtreme-E Ethernet Partition" },
153 	[BCM57407_SFP] = { "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet" },
154 	[BCM57407_NPAR] = { "Broadcom BCM57407 NetXtreme-E Ethernet Partition" },
155 	[BCM57414_NPAR] = { "Broadcom BCM57414 NetXtreme-E Ethernet Partition" },
156 	[BCM57416_NPAR] = { "Broadcom BCM57416 NetXtreme-E Ethernet Partition" },
157 	[BCM57452] = { "Broadcom BCM57452 NetXtreme-E 10Gb/25Gb/40Gb/50Gb Ethernet" },
158 	[BCM57454] = { "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
159 	[BCM5745x_NPAR] = { "Broadcom BCM5745x NetXtreme-E Ethernet Partition" },
160 	[BCM57508] = { "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
161 	[BCM57504] = { "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet" },
162 	[BCM57502] = { "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb Ethernet" },
163 	[BCM57508_NPAR] = { "Broadcom BCM57508 NetXtreme-E Ethernet Partition" },
164 	[BCM57504_NPAR] = { "Broadcom BCM57504 NetXtreme-E Ethernet Partition" },
165 	[BCM57502_NPAR] = { "Broadcom BCM57502 NetXtreme-E Ethernet Partition" },
166 	[BCM58802] = { "Broadcom BCM58802 NetXtreme-S 10Gb/25Gb/40Gb/50Gb Ethernet" },
167 	[BCM58804] = { "Broadcom BCM58804 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
168 	[BCM58808] = { "Broadcom BCM58808 NetXtreme-S 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet" },
169 	[NETXTREME_E_VF] = { "Broadcom NetXtreme-E Ethernet Virtual Function" },
170 	[NETXTREME_C_VF] = { "Broadcom NetXtreme-C Ethernet Virtual Function" },
171 	[NETXTREME_S_VF] = { "Broadcom NetXtreme-S Ethernet Virtual Function" },
172 	[NETXTREME_E_P5_VF] = { "Broadcom BCM5750X NetXtreme-E Ethernet Virtual Function" },
173 };
174 
175 static const struct pci_device_id bnxt_pci_tbl[] = {
176 	{ PCI_VDEVICE(BROADCOM, 0x1604), .driver_data = BCM5745x_NPAR },
177 	{ PCI_VDEVICE(BROADCOM, 0x1605), .driver_data = BCM5745x_NPAR },
178 	{ PCI_VDEVICE(BROADCOM, 0x1614), .driver_data = BCM57454 },
179 	{ PCI_VDEVICE(BROADCOM, 0x16c0), .driver_data = BCM57417_NPAR },
180 	{ PCI_VDEVICE(BROADCOM, 0x16c8), .driver_data = BCM57301 },
181 	{ PCI_VDEVICE(BROADCOM, 0x16c9), .driver_data = BCM57302 },
182 	{ PCI_VDEVICE(BROADCOM, 0x16ca), .driver_data = BCM57304 },
183 	{ PCI_VDEVICE(BROADCOM, 0x16cc), .driver_data = BCM57417_NPAR },
184 	{ PCI_VDEVICE(BROADCOM, 0x16cd), .driver_data = BCM58700 },
185 	{ PCI_VDEVICE(BROADCOM, 0x16ce), .driver_data = BCM57311 },
186 	{ PCI_VDEVICE(BROADCOM, 0x16cf), .driver_data = BCM57312 },
187 	{ PCI_VDEVICE(BROADCOM, 0x16d0), .driver_data = BCM57402 },
188 	{ PCI_VDEVICE(BROADCOM, 0x16d1), .driver_data = BCM57404 },
189 	{ PCI_VDEVICE(BROADCOM, 0x16d2), .driver_data = BCM57406 },
190 	{ PCI_VDEVICE(BROADCOM, 0x16d4), .driver_data = BCM57402_NPAR },
191 	{ PCI_VDEVICE(BROADCOM, 0x16d5), .driver_data = BCM57407 },
192 	{ PCI_VDEVICE(BROADCOM, 0x16d6), .driver_data = BCM57412 },
193 	{ PCI_VDEVICE(BROADCOM, 0x16d7), .driver_data = BCM57414 },
194 	{ PCI_VDEVICE(BROADCOM, 0x16d8), .driver_data = BCM57416 },
195 	{ PCI_VDEVICE(BROADCOM, 0x16d9), .driver_data = BCM57417 },
196 	{ PCI_VDEVICE(BROADCOM, 0x16de), .driver_data = BCM57412_NPAR },
197 	{ PCI_VDEVICE(BROADCOM, 0x16df), .driver_data = BCM57314 },
198 	{ PCI_VDEVICE(BROADCOM, 0x16e2), .driver_data = BCM57417_SFP },
199 	{ PCI_VDEVICE(BROADCOM, 0x16e3), .driver_data = BCM57416_SFP },
200 	{ PCI_VDEVICE(BROADCOM, 0x16e7), .driver_data = BCM57404_NPAR },
201 	{ PCI_VDEVICE(BROADCOM, 0x16e8), .driver_data = BCM57406_NPAR },
202 	{ PCI_VDEVICE(BROADCOM, 0x16e9), .driver_data = BCM57407_SFP },
203 	{ PCI_VDEVICE(BROADCOM, 0x16ea), .driver_data = BCM57407_NPAR },
204 	{ PCI_VDEVICE(BROADCOM, 0x16eb), .driver_data = BCM57412_NPAR },
205 	{ PCI_VDEVICE(BROADCOM, 0x16ec), .driver_data = BCM57414_NPAR },
206 	{ PCI_VDEVICE(BROADCOM, 0x16ed), .driver_data = BCM57414_NPAR },
207 	{ PCI_VDEVICE(BROADCOM, 0x16ee), .driver_data = BCM57416_NPAR },
208 	{ PCI_VDEVICE(BROADCOM, 0x16ef), .driver_data = BCM57416_NPAR },
209 	{ PCI_VDEVICE(BROADCOM, 0x16f0), .driver_data = BCM58808 },
210 	{ PCI_VDEVICE(BROADCOM, 0x16f1), .driver_data = BCM57452 },
211 	{ PCI_VDEVICE(BROADCOM, 0x1750), .driver_data = BCM57508 },
212 	{ PCI_VDEVICE(BROADCOM, 0x1751), .driver_data = BCM57504 },
213 	{ PCI_VDEVICE(BROADCOM, 0x1752), .driver_data = BCM57502 },
214 	{ PCI_VDEVICE(BROADCOM, 0x1800), .driver_data = BCM57508_NPAR },
215 	{ PCI_VDEVICE(BROADCOM, 0x1801), .driver_data = BCM57504_NPAR },
216 	{ PCI_VDEVICE(BROADCOM, 0x1802), .driver_data = BCM57502_NPAR },
217 	{ PCI_VDEVICE(BROADCOM, 0x1803), .driver_data = BCM57508_NPAR },
218 	{ PCI_VDEVICE(BROADCOM, 0x1804), .driver_data = BCM57504_NPAR },
219 	{ PCI_VDEVICE(BROADCOM, 0x1805), .driver_data = BCM57502_NPAR },
220 	{ PCI_VDEVICE(BROADCOM, 0xd802), .driver_data = BCM58802 },
221 	{ PCI_VDEVICE(BROADCOM, 0xd804), .driver_data = BCM58804 },
222 #ifdef CONFIG_BNXT_SRIOV
223 	{ PCI_VDEVICE(BROADCOM, 0x1606), .driver_data = NETXTREME_E_VF },
224 	{ PCI_VDEVICE(BROADCOM, 0x1609), .driver_data = NETXTREME_E_VF },
225 	{ PCI_VDEVICE(BROADCOM, 0x16c1), .driver_data = NETXTREME_E_VF },
226 	{ PCI_VDEVICE(BROADCOM, 0x16cb), .driver_data = NETXTREME_C_VF },
227 	{ PCI_VDEVICE(BROADCOM, 0x16d3), .driver_data = NETXTREME_E_VF },
228 	{ PCI_VDEVICE(BROADCOM, 0x16dc), .driver_data = NETXTREME_E_VF },
229 	{ PCI_VDEVICE(BROADCOM, 0x16e1), .driver_data = NETXTREME_C_VF },
230 	{ PCI_VDEVICE(BROADCOM, 0x16e5), .driver_data = NETXTREME_C_VF },
231 	{ PCI_VDEVICE(BROADCOM, 0x1806), .driver_data = NETXTREME_E_P5_VF },
232 	{ PCI_VDEVICE(BROADCOM, 0x1807), .driver_data = NETXTREME_E_P5_VF },
233 	{ PCI_VDEVICE(BROADCOM, 0xd800), .driver_data = NETXTREME_S_VF },
234 #endif
235 	{ 0 }
236 };
237 
238 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
239 
240 static const u16 bnxt_vf_req_snif[] = {
241 	HWRM_FUNC_CFG,
242 	HWRM_FUNC_VF_CFG,
243 	HWRM_PORT_PHY_QCFG,
244 	HWRM_CFA_L2_FILTER_ALLOC,
245 };
246 
247 static const u16 bnxt_async_events_arr[] = {
248 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE,
249 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE,
250 	ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD,
251 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED,
252 	ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE,
253 	ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE,
254 	ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE,
255 	ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY,
256 	ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY,
257 };
258 
259 static struct workqueue_struct *bnxt_pf_wq;
260 
261 static bool bnxt_vf_pciid(enum board_idx idx)
262 {
263 	return (idx == NETXTREME_C_VF || idx == NETXTREME_E_VF ||
264 		idx == NETXTREME_S_VF || idx == NETXTREME_E_P5_VF);
265 }
266 
267 #define DB_CP_REARM_FLAGS	(DB_KEY_CP | DB_IDX_VALID)
268 #define DB_CP_FLAGS		(DB_KEY_CP | DB_IDX_VALID | DB_IRQ_DIS)
269 #define DB_CP_IRQ_DIS_FLAGS	(DB_KEY_CP | DB_IRQ_DIS)
270 
271 #define BNXT_CP_DB_IRQ_DIS(db)						\
272 		writel(DB_CP_IRQ_DIS_FLAGS, db)
273 
274 #define BNXT_DB_CQ(db, idx)						\
275 	writel(DB_CP_FLAGS | RING_CMP(idx), (db)->doorbell)
276 
277 #define BNXT_DB_NQ_P5(db, idx)						\
278 	writeq((db)->db_key64 | DBR_TYPE_NQ | RING_CMP(idx), (db)->doorbell)
279 
280 #define BNXT_DB_CQ_ARM(db, idx)						\
281 	writel(DB_CP_REARM_FLAGS | RING_CMP(idx), (db)->doorbell)
282 
283 #define BNXT_DB_NQ_ARM_P5(db, idx)					\
284 	writeq((db)->db_key64 | DBR_TYPE_NQ_ARM | RING_CMP(idx), (db)->doorbell)
285 
286 static void bnxt_db_nq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
287 {
288 	if (bp->flags & BNXT_FLAG_CHIP_P5)
289 		BNXT_DB_NQ_P5(db, idx);
290 	else
291 		BNXT_DB_CQ(db, idx);
292 }
293 
294 static void bnxt_db_nq_arm(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
295 {
296 	if (bp->flags & BNXT_FLAG_CHIP_P5)
297 		BNXT_DB_NQ_ARM_P5(db, idx);
298 	else
299 		BNXT_DB_CQ_ARM(db, idx);
300 }
301 
302 static void bnxt_db_cq(struct bnxt *bp, struct bnxt_db_info *db, u32 idx)
303 {
304 	if (bp->flags & BNXT_FLAG_CHIP_P5)
305 		writeq(db->db_key64 | DBR_TYPE_CQ_ARMALL | RING_CMP(idx),
306 		       db->doorbell);
307 	else
308 		BNXT_DB_CQ(db, idx);
309 }
310 
311 const u16 bnxt_lhint_arr[] = {
312 	TX_BD_FLAGS_LHINT_512_AND_SMALLER,
313 	TX_BD_FLAGS_LHINT_512_TO_1023,
314 	TX_BD_FLAGS_LHINT_1024_TO_2047,
315 	TX_BD_FLAGS_LHINT_1024_TO_2047,
316 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
317 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
318 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
319 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
320 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
321 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
322 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
323 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
324 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
325 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
326 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
327 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
328 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
329 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
330 	TX_BD_FLAGS_LHINT_2048_AND_LARGER,
331 };
332 
333 static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb)
334 {
335 	struct metadata_dst *md_dst = skb_metadata_dst(skb);
336 
337 	if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
338 		return 0;
339 
340 	return md_dst->u.port_info.port_id;
341 }
342 
343 static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev)
344 {
345 	struct bnxt *bp = netdev_priv(dev);
346 	struct tx_bd *txbd;
347 	struct tx_bd_ext *txbd1;
348 	struct netdev_queue *txq;
349 	int i;
350 	dma_addr_t mapping;
351 	unsigned int length, pad = 0;
352 	u32 len, free_size, vlan_tag_flags, cfa_action, flags;
353 	u16 prod, last_frag;
354 	struct pci_dev *pdev = bp->pdev;
355 	struct bnxt_tx_ring_info *txr;
356 	struct bnxt_sw_tx_bd *tx_buf;
357 
358 	i = skb_get_queue_mapping(skb);
359 	if (unlikely(i >= bp->tx_nr_rings)) {
360 		dev_kfree_skb_any(skb);
361 		return NETDEV_TX_OK;
362 	}
363 
364 	txq = netdev_get_tx_queue(dev, i);
365 	txr = &bp->tx_ring[bp->tx_ring_map[i]];
366 	prod = txr->tx_prod;
367 
368 	free_size = bnxt_tx_avail(bp, txr);
369 	if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
370 		netif_tx_stop_queue(txq);
371 		return NETDEV_TX_BUSY;
372 	}
373 
374 	length = skb->len;
375 	len = skb_headlen(skb);
376 	last_frag = skb_shinfo(skb)->nr_frags;
377 
378 	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
379 
380 	txbd->tx_bd_opaque = prod;
381 
382 	tx_buf = &txr->tx_buf_ring[prod];
383 	tx_buf->skb = skb;
384 	tx_buf->nr_frags = last_frag;
385 
386 	vlan_tag_flags = 0;
387 	cfa_action = bnxt_xmit_get_cfa_action(skb);
388 	if (skb_vlan_tag_present(skb)) {
389 		vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
390 				 skb_vlan_tag_get(skb);
391 		/* Currently supports 8021Q, 8021AD vlan offloads
392 		 * QINQ1, QINQ2, QINQ3 vlan headers are deprecated
393 		 */
394 		if (skb->vlan_proto == htons(ETH_P_8021Q))
395 			vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
396 	}
397 
398 	if (free_size == bp->tx_ring_size && length <= bp->tx_push_thresh) {
399 		struct tx_push_buffer *tx_push_buf = txr->tx_push;
400 		struct tx_push_bd *tx_push = &tx_push_buf->push_bd;
401 		struct tx_bd_ext *tx_push1 = &tx_push->txbd2;
402 		void __iomem *db = txr->tx_db.doorbell;
403 		void *pdata = tx_push_buf->data;
404 		u64 *end;
405 		int j, push_len;
406 
407 		/* Set COAL_NOW to be ready quickly for the next push */
408 		tx_push->tx_bd_len_flags_type =
409 			cpu_to_le32((length << TX_BD_LEN_SHIFT) |
410 					TX_BD_TYPE_LONG_TX_BD |
411 					TX_BD_FLAGS_LHINT_512_AND_SMALLER |
412 					TX_BD_FLAGS_COAL_NOW |
413 					TX_BD_FLAGS_PACKET_END |
414 					(2 << TX_BD_FLAGS_BD_CNT_SHIFT));
415 
416 		if (skb->ip_summed == CHECKSUM_PARTIAL)
417 			tx_push1->tx_bd_hsize_lflags =
418 					cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
419 		else
420 			tx_push1->tx_bd_hsize_lflags = 0;
421 
422 		tx_push1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
423 		tx_push1->tx_bd_cfa_action =
424 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
425 
426 		end = pdata + length;
427 		end = PTR_ALIGN(end, 8) - 1;
428 		*end = 0;
429 
430 		skb_copy_from_linear_data(skb, pdata, len);
431 		pdata += len;
432 		for (j = 0; j < last_frag; j++) {
433 			skb_frag_t *frag = &skb_shinfo(skb)->frags[j];
434 			void *fptr;
435 
436 			fptr = skb_frag_address_safe(frag);
437 			if (!fptr)
438 				goto normal_tx;
439 
440 			memcpy(pdata, fptr, skb_frag_size(frag));
441 			pdata += skb_frag_size(frag);
442 		}
443 
444 		txbd->tx_bd_len_flags_type = tx_push->tx_bd_len_flags_type;
445 		txbd->tx_bd_haddr = txr->data_mapping;
446 		prod = NEXT_TX(prod);
447 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
448 		memcpy(txbd, tx_push1, sizeof(*txbd));
449 		prod = NEXT_TX(prod);
450 		tx_push->doorbell =
451 			cpu_to_le32(DB_KEY_TX_PUSH | DB_LONG_TX_PUSH | prod);
452 		txr->tx_prod = prod;
453 
454 		tx_buf->is_push = 1;
455 		netdev_tx_sent_queue(txq, skb->len);
456 		wmb();	/* Sync is_push and byte queue before pushing data */
457 
458 		push_len = (length + sizeof(*tx_push) + 7) / 8;
459 		if (push_len > 16) {
460 			__iowrite64_copy(db, tx_push_buf, 16);
461 			__iowrite32_copy(db + 4, tx_push_buf + 1,
462 					 (push_len - 16) << 1);
463 		} else {
464 			__iowrite64_copy(db, tx_push_buf, push_len);
465 		}
466 
467 		goto tx_done;
468 	}
469 
470 normal_tx:
471 	if (length < BNXT_MIN_PKT_SIZE) {
472 		pad = BNXT_MIN_PKT_SIZE - length;
473 		if (skb_pad(skb, pad)) {
474 			/* SKB already freed. */
475 			tx_buf->skb = NULL;
476 			return NETDEV_TX_OK;
477 		}
478 		length = BNXT_MIN_PKT_SIZE;
479 	}
480 
481 	mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE);
482 
483 	if (unlikely(dma_mapping_error(&pdev->dev, mapping))) {
484 		dev_kfree_skb_any(skb);
485 		tx_buf->skb = NULL;
486 		return NETDEV_TX_OK;
487 	}
488 
489 	dma_unmap_addr_set(tx_buf, mapping, mapping);
490 	flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
491 		((last_frag + 2) << TX_BD_FLAGS_BD_CNT_SHIFT);
492 
493 	txbd->tx_bd_haddr = cpu_to_le64(mapping);
494 
495 	prod = NEXT_TX(prod);
496 	txbd1 = (struct tx_bd_ext *)
497 		&txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
498 
499 	txbd1->tx_bd_hsize_lflags = 0;
500 	if (skb_is_gso(skb)) {
501 		u32 hdr_len;
502 
503 		if (skb->encapsulation)
504 			hdr_len = skb_inner_network_offset(skb) +
505 				skb_inner_network_header_len(skb) +
506 				inner_tcp_hdrlen(skb);
507 		else
508 			hdr_len = skb_transport_offset(skb) +
509 				tcp_hdrlen(skb);
510 
511 		txbd1->tx_bd_hsize_lflags = cpu_to_le32(TX_BD_FLAGS_LSO |
512 					TX_BD_FLAGS_T_IPID |
513 					(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
514 		length = skb_shinfo(skb)->gso_size;
515 		txbd1->tx_bd_mss = cpu_to_le32(length);
516 		length += hdr_len;
517 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
518 		txbd1->tx_bd_hsize_lflags =
519 			cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
520 		txbd1->tx_bd_mss = 0;
521 	}
522 
523 	length >>= 9;
524 	if (unlikely(length >= ARRAY_SIZE(bnxt_lhint_arr))) {
525 		dev_warn_ratelimited(&pdev->dev, "Dropped oversize %d bytes TX packet.\n",
526 				     skb->len);
527 		i = 0;
528 		goto tx_dma_error;
529 	}
530 	flags |= bnxt_lhint_arr[length];
531 	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
532 
533 	txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
534 	txbd1->tx_bd_cfa_action =
535 			cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
536 	for (i = 0; i < last_frag; i++) {
537 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
538 
539 		prod = NEXT_TX(prod);
540 		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
541 
542 		len = skb_frag_size(frag);
543 		mapping = skb_frag_dma_map(&pdev->dev, frag, 0, len,
544 					   DMA_TO_DEVICE);
545 
546 		if (unlikely(dma_mapping_error(&pdev->dev, mapping)))
547 			goto tx_dma_error;
548 
549 		tx_buf = &txr->tx_buf_ring[prod];
550 		dma_unmap_addr_set(tx_buf, mapping, mapping);
551 
552 		txbd->tx_bd_haddr = cpu_to_le64(mapping);
553 
554 		flags = len << TX_BD_LEN_SHIFT;
555 		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
556 	}
557 
558 	flags &= ~TX_BD_LEN;
559 	txbd->tx_bd_len_flags_type =
560 		cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags |
561 			    TX_BD_FLAGS_PACKET_END);
562 
563 	netdev_tx_sent_queue(txq, skb->len);
564 
565 	/* Sync BD data before updating doorbell */
566 	wmb();
567 
568 	prod = NEXT_TX(prod);
569 	txr->tx_prod = prod;
570 
571 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
572 		bnxt_db_write(bp, &txr->tx_db, prod);
573 
574 tx_done:
575 
576 	if (unlikely(bnxt_tx_avail(bp, txr) <= MAX_SKB_FRAGS + 1)) {
577 		if (netdev_xmit_more() && !tx_buf->is_push)
578 			bnxt_db_write(bp, &txr->tx_db, prod);
579 
580 		netif_tx_stop_queue(txq);
581 
582 		/* netif_tx_stop_queue() must be done before checking
583 		 * tx index in bnxt_tx_avail() below, because in
584 		 * bnxt_tx_int(), we update tx index before checking for
585 		 * netif_tx_queue_stopped().
586 		 */
587 		smp_mb();
588 		if (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)
589 			netif_tx_wake_queue(txq);
590 	}
591 	return NETDEV_TX_OK;
592 
593 tx_dma_error:
594 	last_frag = i;
595 
596 	/* start back at beginning and unmap skb */
597 	prod = txr->tx_prod;
598 	tx_buf = &txr->tx_buf_ring[prod];
599 	tx_buf->skb = NULL;
600 	dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
601 			 skb_headlen(skb), PCI_DMA_TODEVICE);
602 	prod = NEXT_TX(prod);
603 
604 	/* unmap remaining mapped pages */
605 	for (i = 0; i < last_frag; i++) {
606 		prod = NEXT_TX(prod);
607 		tx_buf = &txr->tx_buf_ring[prod];
608 		dma_unmap_page(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
609 			       skb_frag_size(&skb_shinfo(skb)->frags[i]),
610 			       PCI_DMA_TODEVICE);
611 	}
612 
613 	dev_kfree_skb_any(skb);
614 	return NETDEV_TX_OK;
615 }
616 
617 static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
618 {
619 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
620 	struct netdev_queue *txq = netdev_get_tx_queue(bp->dev, txr->txq_index);
621 	u16 cons = txr->tx_cons;
622 	struct pci_dev *pdev = bp->pdev;
623 	int i;
624 	unsigned int tx_bytes = 0;
625 
626 	for (i = 0; i < nr_pkts; i++) {
627 		struct bnxt_sw_tx_bd *tx_buf;
628 		struct sk_buff *skb;
629 		int j, last;
630 
631 		tx_buf = &txr->tx_buf_ring[cons];
632 		cons = NEXT_TX(cons);
633 		skb = tx_buf->skb;
634 		tx_buf->skb = NULL;
635 
636 		if (tx_buf->is_push) {
637 			tx_buf->is_push = 0;
638 			goto next_tx_int;
639 		}
640 
641 		dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping),
642 				 skb_headlen(skb), PCI_DMA_TODEVICE);
643 		last = tx_buf->nr_frags;
644 
645 		for (j = 0; j < last; j++) {
646 			cons = NEXT_TX(cons);
647 			tx_buf = &txr->tx_buf_ring[cons];
648 			dma_unmap_page(
649 				&pdev->dev,
650 				dma_unmap_addr(tx_buf, mapping),
651 				skb_frag_size(&skb_shinfo(skb)->frags[j]),
652 				PCI_DMA_TODEVICE);
653 		}
654 
655 next_tx_int:
656 		cons = NEXT_TX(cons);
657 
658 		tx_bytes += skb->len;
659 		dev_kfree_skb_any(skb);
660 	}
661 
662 	netdev_tx_completed_queue(txq, nr_pkts, tx_bytes);
663 	txr->tx_cons = cons;
664 
665 	/* Need to make the tx_cons update visible to bnxt_start_xmit()
666 	 * before checking for netif_tx_queue_stopped().  Without the
667 	 * memory barrier, there is a small possibility that bnxt_start_xmit()
668 	 * will miss it and cause the queue to be stopped forever.
669 	 */
670 	smp_mb();
671 
672 	if (unlikely(netif_tx_queue_stopped(txq)) &&
673 	    (bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh)) {
674 		__netif_tx_lock(txq, smp_processor_id());
675 		if (netif_tx_queue_stopped(txq) &&
676 		    bnxt_tx_avail(bp, txr) > bp->tx_wake_thresh &&
677 		    txr->dev_state != BNXT_DEV_STATE_CLOSING)
678 			netif_tx_wake_queue(txq);
679 		__netif_tx_unlock(txq);
680 	}
681 }
682 
683 static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping,
684 					 struct bnxt_rx_ring_info *rxr,
685 					 gfp_t gfp)
686 {
687 	struct device *dev = &bp->pdev->dev;
688 	struct page *page;
689 
690 	page = page_pool_dev_alloc_pages(rxr->page_pool);
691 	if (!page)
692 		return NULL;
693 
694 	*mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir,
695 				      DMA_ATTR_WEAK_ORDERING);
696 	if (dma_mapping_error(dev, *mapping)) {
697 		page_pool_recycle_direct(rxr->page_pool, page);
698 		return NULL;
699 	}
700 	*mapping += bp->rx_dma_offset;
701 	return page;
702 }
703 
704 static inline u8 *__bnxt_alloc_rx_data(struct bnxt *bp, dma_addr_t *mapping,
705 				       gfp_t gfp)
706 {
707 	u8 *data;
708 	struct pci_dev *pdev = bp->pdev;
709 
710 	data = kmalloc(bp->rx_buf_size, gfp);
711 	if (!data)
712 		return NULL;
713 
714 	*mapping = dma_map_single_attrs(&pdev->dev, data + bp->rx_dma_offset,
715 					bp->rx_buf_use_size, bp->rx_dir,
716 					DMA_ATTR_WEAK_ORDERING);
717 
718 	if (dma_mapping_error(&pdev->dev, *mapping)) {
719 		kfree(data);
720 		data = NULL;
721 	}
722 	return data;
723 }
724 
725 int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
726 		       u16 prod, gfp_t gfp)
727 {
728 	struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
729 	struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[prod];
730 	dma_addr_t mapping;
731 
732 	if (BNXT_RX_PAGE_MODE(bp)) {
733 		struct page *page =
734 			__bnxt_alloc_rx_page(bp, &mapping, rxr, gfp);
735 
736 		if (!page)
737 			return -ENOMEM;
738 
739 		rx_buf->data = page;
740 		rx_buf->data_ptr = page_address(page) + bp->rx_offset;
741 	} else {
742 		u8 *data = __bnxt_alloc_rx_data(bp, &mapping, gfp);
743 
744 		if (!data)
745 			return -ENOMEM;
746 
747 		rx_buf->data = data;
748 		rx_buf->data_ptr = data + bp->rx_offset;
749 	}
750 	rx_buf->mapping = mapping;
751 
752 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
753 	return 0;
754 }
755 
756 void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data)
757 {
758 	u16 prod = rxr->rx_prod;
759 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
760 	struct rx_bd *cons_bd, *prod_bd;
761 
762 	prod_rx_buf = &rxr->rx_buf_ring[prod];
763 	cons_rx_buf = &rxr->rx_buf_ring[cons];
764 
765 	prod_rx_buf->data = data;
766 	prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
767 
768 	prod_rx_buf->mapping = cons_rx_buf->mapping;
769 
770 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
771 	cons_bd = &rxr->rx_desc_ring[RX_RING(cons)][RX_IDX(cons)];
772 
773 	prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
774 }
775 
776 static inline u16 bnxt_find_next_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
777 {
778 	u16 next, max = rxr->rx_agg_bmap_size;
779 
780 	next = find_next_zero_bit(rxr->rx_agg_bmap, max, idx);
781 	if (next >= max)
782 		next = find_first_zero_bit(rxr->rx_agg_bmap, max);
783 	return next;
784 }
785 
786 static inline int bnxt_alloc_rx_page(struct bnxt *bp,
787 				     struct bnxt_rx_ring_info *rxr,
788 				     u16 prod, gfp_t gfp)
789 {
790 	struct rx_bd *rxbd =
791 		&rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
792 	struct bnxt_sw_rx_agg_bd *rx_agg_buf;
793 	struct pci_dev *pdev = bp->pdev;
794 	struct page *page;
795 	dma_addr_t mapping;
796 	u16 sw_prod = rxr->rx_sw_agg_prod;
797 	unsigned int offset = 0;
798 
799 	if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) {
800 		page = rxr->rx_page;
801 		if (!page) {
802 			page = alloc_page(gfp);
803 			if (!page)
804 				return -ENOMEM;
805 			rxr->rx_page = page;
806 			rxr->rx_page_offset = 0;
807 		}
808 		offset = rxr->rx_page_offset;
809 		rxr->rx_page_offset += BNXT_RX_PAGE_SIZE;
810 		if (rxr->rx_page_offset == PAGE_SIZE)
811 			rxr->rx_page = NULL;
812 		else
813 			get_page(page);
814 	} else {
815 		page = alloc_page(gfp);
816 		if (!page)
817 			return -ENOMEM;
818 	}
819 
820 	mapping = dma_map_page_attrs(&pdev->dev, page, offset,
821 				     BNXT_RX_PAGE_SIZE, PCI_DMA_FROMDEVICE,
822 				     DMA_ATTR_WEAK_ORDERING);
823 	if (dma_mapping_error(&pdev->dev, mapping)) {
824 		__free_page(page);
825 		return -EIO;
826 	}
827 
828 	if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
829 		sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
830 
831 	__set_bit(sw_prod, rxr->rx_agg_bmap);
832 	rx_agg_buf = &rxr->rx_agg_ring[sw_prod];
833 	rxr->rx_sw_agg_prod = NEXT_RX_AGG(sw_prod);
834 
835 	rx_agg_buf->page = page;
836 	rx_agg_buf->offset = offset;
837 	rx_agg_buf->mapping = mapping;
838 	rxbd->rx_bd_haddr = cpu_to_le64(mapping);
839 	rxbd->rx_bd_opaque = sw_prod;
840 	return 0;
841 }
842 
843 static struct rx_agg_cmp *bnxt_get_agg(struct bnxt *bp,
844 				       struct bnxt_cp_ring_info *cpr,
845 				       u16 cp_cons, u16 curr)
846 {
847 	struct rx_agg_cmp *agg;
848 
849 	cp_cons = RING_CMP(ADV_RAW_CMP(cp_cons, curr));
850 	agg = (struct rx_agg_cmp *)
851 		&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
852 	return agg;
853 }
854 
855 static struct rx_agg_cmp *bnxt_get_tpa_agg_p5(struct bnxt *bp,
856 					      struct bnxt_rx_ring_info *rxr,
857 					      u16 agg_id, u16 curr)
858 {
859 	struct bnxt_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
860 
861 	return &tpa_info->agg_arr[curr];
862 }
863 
864 static void bnxt_reuse_rx_agg_bufs(struct bnxt_cp_ring_info *cpr, u16 idx,
865 				   u16 start, u32 agg_bufs, bool tpa)
866 {
867 	struct bnxt_napi *bnapi = cpr->bnapi;
868 	struct bnxt *bp = bnapi->bp;
869 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
870 	u16 prod = rxr->rx_agg_prod;
871 	u16 sw_prod = rxr->rx_sw_agg_prod;
872 	bool p5_tpa = false;
873 	u32 i;
874 
875 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
876 		p5_tpa = true;
877 
878 	for (i = 0; i < agg_bufs; i++) {
879 		u16 cons;
880 		struct rx_agg_cmp *agg;
881 		struct bnxt_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
882 		struct rx_bd *prod_bd;
883 		struct page *page;
884 
885 		if (p5_tpa)
886 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, start + i);
887 		else
888 			agg = bnxt_get_agg(bp, cpr, idx, start + i);
889 		cons = agg->rx_agg_cmp_opaque;
890 		__clear_bit(cons, rxr->rx_agg_bmap);
891 
892 		if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
893 			sw_prod = bnxt_find_next_agg_idx(rxr, sw_prod);
894 
895 		__set_bit(sw_prod, rxr->rx_agg_bmap);
896 		prod_rx_buf = &rxr->rx_agg_ring[sw_prod];
897 		cons_rx_buf = &rxr->rx_agg_ring[cons];
898 
899 		/* It is possible for sw_prod to be equal to cons, so
900 		 * set cons_rx_buf->page to NULL first.
901 		 */
902 		page = cons_rx_buf->page;
903 		cons_rx_buf->page = NULL;
904 		prod_rx_buf->page = page;
905 		prod_rx_buf->offset = cons_rx_buf->offset;
906 
907 		prod_rx_buf->mapping = cons_rx_buf->mapping;
908 
909 		prod_bd = &rxr->rx_agg_desc_ring[RX_RING(prod)][RX_IDX(prod)];
910 
911 		prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
912 		prod_bd->rx_bd_opaque = sw_prod;
913 
914 		prod = NEXT_RX_AGG(prod);
915 		sw_prod = NEXT_RX_AGG(sw_prod);
916 	}
917 	rxr->rx_agg_prod = prod;
918 	rxr->rx_sw_agg_prod = sw_prod;
919 }
920 
921 static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp,
922 					struct bnxt_rx_ring_info *rxr,
923 					u16 cons, void *data, u8 *data_ptr,
924 					dma_addr_t dma_addr,
925 					unsigned int offset_and_len)
926 {
927 	unsigned int payload = offset_and_len >> 16;
928 	unsigned int len = offset_and_len & 0xffff;
929 	skb_frag_t *frag;
930 	struct page *page = data;
931 	u16 prod = rxr->rx_prod;
932 	struct sk_buff *skb;
933 	int off, err;
934 
935 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
936 	if (unlikely(err)) {
937 		bnxt_reuse_rx_data(rxr, cons, data);
938 		return NULL;
939 	}
940 	dma_addr -= bp->rx_dma_offset;
941 	dma_unmap_page_attrs(&bp->pdev->dev, dma_addr, PAGE_SIZE, bp->rx_dir,
942 			     DMA_ATTR_WEAK_ORDERING);
943 	page_pool_release_page(rxr->page_pool, page);
944 
945 	if (unlikely(!payload))
946 		payload = eth_get_headlen(bp->dev, data_ptr, len);
947 
948 	skb = napi_alloc_skb(&rxr->bnapi->napi, payload);
949 	if (!skb) {
950 		__free_page(page);
951 		return NULL;
952 	}
953 
954 	off = (void *)data_ptr - page_address(page);
955 	skb_add_rx_frag(skb, 0, page, off, len, PAGE_SIZE);
956 	memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN,
957 	       payload + NET_IP_ALIGN);
958 
959 	frag = &skb_shinfo(skb)->frags[0];
960 	skb_frag_size_sub(frag, payload);
961 	skb_frag_off_add(frag, payload);
962 	skb->data_len -= payload;
963 	skb->tail += payload;
964 
965 	return skb;
966 }
967 
968 static struct sk_buff *bnxt_rx_skb(struct bnxt *bp,
969 				   struct bnxt_rx_ring_info *rxr, u16 cons,
970 				   void *data, u8 *data_ptr,
971 				   dma_addr_t dma_addr,
972 				   unsigned int offset_and_len)
973 {
974 	u16 prod = rxr->rx_prod;
975 	struct sk_buff *skb;
976 	int err;
977 
978 	err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC);
979 	if (unlikely(err)) {
980 		bnxt_reuse_rx_data(rxr, cons, data);
981 		return NULL;
982 	}
983 
984 	skb = build_skb(data, 0);
985 	dma_unmap_single_attrs(&bp->pdev->dev, dma_addr, bp->rx_buf_use_size,
986 			       bp->rx_dir, DMA_ATTR_WEAK_ORDERING);
987 	if (!skb) {
988 		kfree(data);
989 		return NULL;
990 	}
991 
992 	skb_reserve(skb, bp->rx_offset);
993 	skb_put(skb, offset_and_len & 0xffff);
994 	return skb;
995 }
996 
997 static struct sk_buff *bnxt_rx_pages(struct bnxt *bp,
998 				     struct bnxt_cp_ring_info *cpr,
999 				     struct sk_buff *skb, u16 idx,
1000 				     u32 agg_bufs, bool tpa)
1001 {
1002 	struct bnxt_napi *bnapi = cpr->bnapi;
1003 	struct pci_dev *pdev = bp->pdev;
1004 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1005 	u16 prod = rxr->rx_agg_prod;
1006 	bool p5_tpa = false;
1007 	u32 i;
1008 
1009 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && tpa)
1010 		p5_tpa = true;
1011 
1012 	for (i = 0; i < agg_bufs; i++) {
1013 		u16 cons, frag_len;
1014 		struct rx_agg_cmp *agg;
1015 		struct bnxt_sw_rx_agg_bd *cons_rx_buf;
1016 		struct page *page;
1017 		dma_addr_t mapping;
1018 
1019 		if (p5_tpa)
1020 			agg = bnxt_get_tpa_agg_p5(bp, rxr, idx, i);
1021 		else
1022 			agg = bnxt_get_agg(bp, cpr, idx, i);
1023 		cons = agg->rx_agg_cmp_opaque;
1024 		frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
1025 			    RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
1026 
1027 		cons_rx_buf = &rxr->rx_agg_ring[cons];
1028 		skb_fill_page_desc(skb, i, cons_rx_buf->page,
1029 				   cons_rx_buf->offset, frag_len);
1030 		__clear_bit(cons, rxr->rx_agg_bmap);
1031 
1032 		/* It is possible for bnxt_alloc_rx_page() to allocate
1033 		 * a sw_prod index that equals the cons index, so we
1034 		 * need to clear the cons entry now.
1035 		 */
1036 		mapping = cons_rx_buf->mapping;
1037 		page = cons_rx_buf->page;
1038 		cons_rx_buf->page = NULL;
1039 
1040 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_ATOMIC) != 0) {
1041 			struct skb_shared_info *shinfo;
1042 			unsigned int nr_frags;
1043 
1044 			shinfo = skb_shinfo(skb);
1045 			nr_frags = --shinfo->nr_frags;
1046 			__skb_frag_set_page(&shinfo->frags[nr_frags], NULL);
1047 
1048 			dev_kfree_skb(skb);
1049 
1050 			cons_rx_buf->page = page;
1051 
1052 			/* Update prod since possibly some pages have been
1053 			 * allocated already.
1054 			 */
1055 			rxr->rx_agg_prod = prod;
1056 			bnxt_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
1057 			return NULL;
1058 		}
1059 
1060 		dma_unmap_page_attrs(&pdev->dev, mapping, BNXT_RX_PAGE_SIZE,
1061 				     PCI_DMA_FROMDEVICE,
1062 				     DMA_ATTR_WEAK_ORDERING);
1063 
1064 		skb->data_len += frag_len;
1065 		skb->len += frag_len;
1066 		skb->truesize += PAGE_SIZE;
1067 
1068 		prod = NEXT_RX_AGG(prod);
1069 	}
1070 	rxr->rx_agg_prod = prod;
1071 	return skb;
1072 }
1073 
1074 static int bnxt_agg_bufs_valid(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1075 			       u8 agg_bufs, u32 *raw_cons)
1076 {
1077 	u16 last;
1078 	struct rx_agg_cmp *agg;
1079 
1080 	*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
1081 	last = RING_CMP(*raw_cons);
1082 	agg = (struct rx_agg_cmp *)
1083 		&cpr->cp_desc_ring[CP_RING(last)][CP_IDX(last)];
1084 	return RX_AGG_CMP_VALID(agg, *raw_cons);
1085 }
1086 
1087 static inline struct sk_buff *bnxt_copy_skb(struct bnxt_napi *bnapi, u8 *data,
1088 					    unsigned int len,
1089 					    dma_addr_t mapping)
1090 {
1091 	struct bnxt *bp = bnapi->bp;
1092 	struct pci_dev *pdev = bp->pdev;
1093 	struct sk_buff *skb;
1094 
1095 	skb = napi_alloc_skb(&bnapi->napi, len);
1096 	if (!skb)
1097 		return NULL;
1098 
1099 	dma_sync_single_for_cpu(&pdev->dev, mapping, bp->rx_copy_thresh,
1100 				bp->rx_dir);
1101 
1102 	memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
1103 	       len + NET_IP_ALIGN);
1104 
1105 	dma_sync_single_for_device(&pdev->dev, mapping, bp->rx_copy_thresh,
1106 				   bp->rx_dir);
1107 
1108 	skb_put(skb, len);
1109 	return skb;
1110 }
1111 
1112 static int bnxt_discard_rx(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1113 			   u32 *raw_cons, void *cmp)
1114 {
1115 	struct rx_cmp *rxcmp = cmp;
1116 	u32 tmp_raw_cons = *raw_cons;
1117 	u8 cmp_type, agg_bufs = 0;
1118 
1119 	cmp_type = RX_CMP_TYPE(rxcmp);
1120 
1121 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1122 		agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
1123 			    RX_CMP_AGG_BUFS) >>
1124 			   RX_CMP_AGG_BUFS_SHIFT;
1125 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1126 		struct rx_tpa_end_cmp *tpa_end = cmp;
1127 
1128 		if (bp->flags & BNXT_FLAG_CHIP_P5)
1129 			return 0;
1130 
1131 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1132 	}
1133 
1134 	if (agg_bufs) {
1135 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1136 			return -EBUSY;
1137 	}
1138 	*raw_cons = tmp_raw_cons;
1139 	return 0;
1140 }
1141 
1142 static void bnxt_queue_fw_reset_work(struct bnxt *bp, unsigned long delay)
1143 {
1144 	if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)))
1145 		return;
1146 
1147 	if (BNXT_PF(bp))
1148 		queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay);
1149 	else
1150 		schedule_delayed_work(&bp->fw_reset_task, delay);
1151 }
1152 
1153 static void bnxt_queue_sp_work(struct bnxt *bp)
1154 {
1155 	if (BNXT_PF(bp))
1156 		queue_work(bnxt_pf_wq, &bp->sp_task);
1157 	else
1158 		schedule_work(&bp->sp_task);
1159 }
1160 
1161 static void bnxt_cancel_sp_work(struct bnxt *bp)
1162 {
1163 	if (BNXT_PF(bp)) {
1164 		flush_workqueue(bnxt_pf_wq);
1165 	} else {
1166 		cancel_work_sync(&bp->sp_task);
1167 		cancel_delayed_work_sync(&bp->fw_reset_task);
1168 	}
1169 }
1170 
1171 static void bnxt_sched_reset(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
1172 {
1173 	if (!rxr->bnapi->in_reset) {
1174 		rxr->bnapi->in_reset = true;
1175 		set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
1176 		bnxt_queue_sp_work(bp);
1177 	}
1178 	rxr->rx_next_cons = 0xffff;
1179 }
1180 
1181 static u16 bnxt_alloc_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1182 {
1183 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1184 	u16 idx = agg_id & MAX_TPA_P5_MASK;
1185 
1186 	if (test_bit(idx, map->agg_idx_bmap))
1187 		idx = find_first_zero_bit(map->agg_idx_bmap,
1188 					  BNXT_AGG_IDX_BMAP_SIZE);
1189 	__set_bit(idx, map->agg_idx_bmap);
1190 	map->agg_id_tbl[agg_id] = idx;
1191 	return idx;
1192 }
1193 
1194 static void bnxt_free_agg_idx(struct bnxt_rx_ring_info *rxr, u16 idx)
1195 {
1196 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1197 
1198 	__clear_bit(idx, map->agg_idx_bmap);
1199 }
1200 
1201 static u16 bnxt_lookup_agg_idx(struct bnxt_rx_ring_info *rxr, u16 agg_id)
1202 {
1203 	struct bnxt_tpa_idx_map *map = rxr->rx_tpa_idx_map;
1204 
1205 	return map->agg_id_tbl[agg_id];
1206 }
1207 
1208 static void bnxt_tpa_start(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1209 			   struct rx_tpa_start_cmp *tpa_start,
1210 			   struct rx_tpa_start_cmp_ext *tpa_start1)
1211 {
1212 	struct bnxt_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
1213 	struct bnxt_tpa_info *tpa_info;
1214 	u16 cons, prod, agg_id;
1215 	struct rx_bd *prod_bd;
1216 	dma_addr_t mapping;
1217 
1218 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1219 		agg_id = TPA_START_AGG_ID_P5(tpa_start);
1220 		agg_id = bnxt_alloc_agg_idx(rxr, agg_id);
1221 	} else {
1222 		agg_id = TPA_START_AGG_ID(tpa_start);
1223 	}
1224 	cons = tpa_start->rx_tpa_start_cmp_opaque;
1225 	prod = rxr->rx_prod;
1226 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1227 	prod_rx_buf = &rxr->rx_buf_ring[prod];
1228 	tpa_info = &rxr->rx_tpa[agg_id];
1229 
1230 	if (unlikely(cons != rxr->rx_next_cons ||
1231 		     TPA_START_ERROR(tpa_start))) {
1232 		netdev_warn(bp->dev, "TPA cons %x, expected cons %x, error code %x\n",
1233 			    cons, rxr->rx_next_cons,
1234 			    TPA_START_ERROR_CODE(tpa_start1));
1235 		bnxt_sched_reset(bp, rxr);
1236 		return;
1237 	}
1238 	/* Store cfa_code in tpa_info to use in tpa_end
1239 	 * completion processing.
1240 	 */
1241 	tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
1242 	prod_rx_buf->data = tpa_info->data;
1243 	prod_rx_buf->data_ptr = tpa_info->data_ptr;
1244 
1245 	mapping = tpa_info->mapping;
1246 	prod_rx_buf->mapping = mapping;
1247 
1248 	prod_bd = &rxr->rx_desc_ring[RX_RING(prod)][RX_IDX(prod)];
1249 
1250 	prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
1251 
1252 	tpa_info->data = cons_rx_buf->data;
1253 	tpa_info->data_ptr = cons_rx_buf->data_ptr;
1254 	cons_rx_buf->data = NULL;
1255 	tpa_info->mapping = cons_rx_buf->mapping;
1256 
1257 	tpa_info->len =
1258 		le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
1259 				RX_TPA_START_CMP_LEN_SHIFT;
1260 	if (likely(TPA_START_HASH_VALID(tpa_start))) {
1261 		u32 hash_type = TPA_START_HASH_TYPE(tpa_start);
1262 
1263 		tpa_info->hash_type = PKT_HASH_TYPE_L4;
1264 		tpa_info->gso_type = SKB_GSO_TCPV4;
1265 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1266 		if (hash_type == 3 || TPA_START_IS_IPV6(tpa_start1))
1267 			tpa_info->gso_type = SKB_GSO_TCPV6;
1268 		tpa_info->rss_hash =
1269 			le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
1270 	} else {
1271 		tpa_info->hash_type = PKT_HASH_TYPE_NONE;
1272 		tpa_info->gso_type = 0;
1273 		if (netif_msg_rx_err(bp))
1274 			netdev_warn(bp->dev, "TPA packet without valid hash\n");
1275 	}
1276 	tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
1277 	tpa_info->metadata = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
1278 	tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
1279 	tpa_info->agg_count = 0;
1280 
1281 	rxr->rx_prod = NEXT_RX(prod);
1282 	cons = NEXT_RX(cons);
1283 	rxr->rx_next_cons = NEXT_RX(cons);
1284 	cons_rx_buf = &rxr->rx_buf_ring[cons];
1285 
1286 	bnxt_reuse_rx_data(rxr, cons, cons_rx_buf->data);
1287 	rxr->rx_prod = NEXT_RX(rxr->rx_prod);
1288 	cons_rx_buf->data = NULL;
1289 }
1290 
1291 static void bnxt_abort_tpa(struct bnxt_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
1292 {
1293 	if (agg_bufs)
1294 		bnxt_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
1295 }
1296 
1297 #ifdef CONFIG_INET
1298 static void bnxt_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
1299 {
1300 	struct udphdr *uh = NULL;
1301 
1302 	if (ip_proto == htons(ETH_P_IP)) {
1303 		struct iphdr *iph = (struct iphdr *)skb->data;
1304 
1305 		if (iph->protocol == IPPROTO_UDP)
1306 			uh = (struct udphdr *)(iph + 1);
1307 	} else {
1308 		struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
1309 
1310 		if (iph->nexthdr == IPPROTO_UDP)
1311 			uh = (struct udphdr *)(iph + 1);
1312 	}
1313 	if (uh) {
1314 		if (uh->check)
1315 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
1316 		else
1317 			skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
1318 	}
1319 }
1320 #endif
1321 
1322 static struct sk_buff *bnxt_gro_func_5731x(struct bnxt_tpa_info *tpa_info,
1323 					   int payload_off, int tcp_ts,
1324 					   struct sk_buff *skb)
1325 {
1326 #ifdef CONFIG_INET
1327 	struct tcphdr *th;
1328 	int len, nw_off;
1329 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1330 	u32 hdr_info = tpa_info->hdr_info;
1331 	bool loopback = false;
1332 
1333 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1334 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1335 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1336 
1337 	/* If the packet is an internal loopback packet, the offsets will
1338 	 * have an extra 4 bytes.
1339 	 */
1340 	if (inner_mac_off == 4) {
1341 		loopback = true;
1342 	} else if (inner_mac_off > 4) {
1343 		__be16 proto = *((__be16 *)(skb->data + inner_ip_off -
1344 					    ETH_HLEN - 2));
1345 
1346 		/* We only support inner iPv4/ipv6.  If we don't see the
1347 		 * correct protocol ID, it must be a loopback packet where
1348 		 * the offsets are off by 4.
1349 		 */
1350 		if (proto != htons(ETH_P_IP) && proto != htons(ETH_P_IPV6))
1351 			loopback = true;
1352 	}
1353 	if (loopback) {
1354 		/* internal loopback packet, subtract all offsets by 4 */
1355 		inner_ip_off -= 4;
1356 		inner_mac_off -= 4;
1357 		outer_ip_off -= 4;
1358 	}
1359 
1360 	nw_off = inner_ip_off - ETH_HLEN;
1361 	skb_set_network_header(skb, nw_off);
1362 	if (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) {
1363 		struct ipv6hdr *iph = ipv6_hdr(skb);
1364 
1365 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1366 		len = skb->len - skb_transport_offset(skb);
1367 		th = tcp_hdr(skb);
1368 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1369 	} else {
1370 		struct iphdr *iph = ip_hdr(skb);
1371 
1372 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1373 		len = skb->len - skb_transport_offset(skb);
1374 		th = tcp_hdr(skb);
1375 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1376 	}
1377 
1378 	if (inner_mac_off) { /* tunnel */
1379 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1380 					    ETH_HLEN - 2));
1381 
1382 		bnxt_gro_tunnel(skb, proto);
1383 	}
1384 #endif
1385 	return skb;
1386 }
1387 
1388 static struct sk_buff *bnxt_gro_func_5750x(struct bnxt_tpa_info *tpa_info,
1389 					   int payload_off, int tcp_ts,
1390 					   struct sk_buff *skb)
1391 {
1392 #ifdef CONFIG_INET
1393 	u16 outer_ip_off, inner_ip_off, inner_mac_off;
1394 	u32 hdr_info = tpa_info->hdr_info;
1395 	int iphdr_len, nw_off;
1396 
1397 	inner_ip_off = BNXT_TPA_INNER_L3_OFF(hdr_info);
1398 	inner_mac_off = BNXT_TPA_INNER_L2_OFF(hdr_info);
1399 	outer_ip_off = BNXT_TPA_OUTER_L3_OFF(hdr_info);
1400 
1401 	nw_off = inner_ip_off - ETH_HLEN;
1402 	skb_set_network_header(skb, nw_off);
1403 	iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
1404 		     sizeof(struct ipv6hdr) : sizeof(struct iphdr);
1405 	skb_set_transport_header(skb, nw_off + iphdr_len);
1406 
1407 	if (inner_mac_off) { /* tunnel */
1408 		__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
1409 					    ETH_HLEN - 2));
1410 
1411 		bnxt_gro_tunnel(skb, proto);
1412 	}
1413 #endif
1414 	return skb;
1415 }
1416 
1417 #define BNXT_IPV4_HDR_SIZE	(sizeof(struct iphdr) + sizeof(struct tcphdr))
1418 #define BNXT_IPV6_HDR_SIZE	(sizeof(struct ipv6hdr) + sizeof(struct tcphdr))
1419 
1420 static struct sk_buff *bnxt_gro_func_5730x(struct bnxt_tpa_info *tpa_info,
1421 					   int payload_off, int tcp_ts,
1422 					   struct sk_buff *skb)
1423 {
1424 #ifdef CONFIG_INET
1425 	struct tcphdr *th;
1426 	int len, nw_off, tcp_opt_len = 0;
1427 
1428 	if (tcp_ts)
1429 		tcp_opt_len = 12;
1430 
1431 	if (tpa_info->gso_type == SKB_GSO_TCPV4) {
1432 		struct iphdr *iph;
1433 
1434 		nw_off = payload_off - BNXT_IPV4_HDR_SIZE - tcp_opt_len -
1435 			 ETH_HLEN;
1436 		skb_set_network_header(skb, nw_off);
1437 		iph = ip_hdr(skb);
1438 		skb_set_transport_header(skb, nw_off + sizeof(struct iphdr));
1439 		len = skb->len - skb_transport_offset(skb);
1440 		th = tcp_hdr(skb);
1441 		th->check = ~tcp_v4_check(len, iph->saddr, iph->daddr, 0);
1442 	} else if (tpa_info->gso_type == SKB_GSO_TCPV6) {
1443 		struct ipv6hdr *iph;
1444 
1445 		nw_off = payload_off - BNXT_IPV6_HDR_SIZE - tcp_opt_len -
1446 			 ETH_HLEN;
1447 		skb_set_network_header(skb, nw_off);
1448 		iph = ipv6_hdr(skb);
1449 		skb_set_transport_header(skb, nw_off + sizeof(struct ipv6hdr));
1450 		len = skb->len - skb_transport_offset(skb);
1451 		th = tcp_hdr(skb);
1452 		th->check = ~tcp_v6_check(len, &iph->saddr, &iph->daddr, 0);
1453 	} else {
1454 		dev_kfree_skb_any(skb);
1455 		return NULL;
1456 	}
1457 
1458 	if (nw_off) /* tunnel */
1459 		bnxt_gro_tunnel(skb, skb->protocol);
1460 #endif
1461 	return skb;
1462 }
1463 
1464 static inline struct sk_buff *bnxt_gro_skb(struct bnxt *bp,
1465 					   struct bnxt_tpa_info *tpa_info,
1466 					   struct rx_tpa_end_cmp *tpa_end,
1467 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1468 					   struct sk_buff *skb)
1469 {
1470 #ifdef CONFIG_INET
1471 	int payload_off;
1472 	u16 segs;
1473 
1474 	segs = TPA_END_TPA_SEGS(tpa_end);
1475 	if (segs == 1)
1476 		return skb;
1477 
1478 	NAPI_GRO_CB(skb)->count = segs;
1479 	skb_shinfo(skb)->gso_size =
1480 		le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
1481 	skb_shinfo(skb)->gso_type = tpa_info->gso_type;
1482 	if (bp->flags & BNXT_FLAG_CHIP_P5)
1483 		payload_off = TPA_END_PAYLOAD_OFF_P5(tpa_end1);
1484 	else
1485 		payload_off = TPA_END_PAYLOAD_OFF(tpa_end);
1486 	skb = bp->gro_func(tpa_info, payload_off, TPA_END_GRO_TS(tpa_end), skb);
1487 	if (likely(skb))
1488 		tcp_gro_complete(skb);
1489 #endif
1490 	return skb;
1491 }
1492 
1493 /* Given the cfa_code of a received packet determine which
1494  * netdev (vf-rep or PF) the packet is destined to.
1495  */
1496 static struct net_device *bnxt_get_pkt_dev(struct bnxt *bp, u16 cfa_code)
1497 {
1498 	struct net_device *dev = bnxt_get_vf_rep(bp, cfa_code);
1499 
1500 	/* if vf-rep dev is NULL, the must belongs to the PF */
1501 	return dev ? dev : bp->dev;
1502 }
1503 
1504 static inline struct sk_buff *bnxt_tpa_end(struct bnxt *bp,
1505 					   struct bnxt_cp_ring_info *cpr,
1506 					   u32 *raw_cons,
1507 					   struct rx_tpa_end_cmp *tpa_end,
1508 					   struct rx_tpa_end_cmp_ext *tpa_end1,
1509 					   u8 *event)
1510 {
1511 	struct bnxt_napi *bnapi = cpr->bnapi;
1512 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1513 	u8 *data_ptr, agg_bufs;
1514 	unsigned int len;
1515 	struct bnxt_tpa_info *tpa_info;
1516 	dma_addr_t mapping;
1517 	struct sk_buff *skb;
1518 	u16 idx = 0, agg_id;
1519 	void *data;
1520 	bool gro;
1521 
1522 	if (unlikely(bnapi->in_reset)) {
1523 		int rc = bnxt_discard_rx(bp, cpr, raw_cons, tpa_end);
1524 
1525 		if (rc < 0)
1526 			return ERR_PTR(-EBUSY);
1527 		return NULL;
1528 	}
1529 
1530 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
1531 		agg_id = TPA_END_AGG_ID_P5(tpa_end);
1532 		agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1533 		agg_bufs = TPA_END_AGG_BUFS_P5(tpa_end1);
1534 		tpa_info = &rxr->rx_tpa[agg_id];
1535 		if (unlikely(agg_bufs != tpa_info->agg_count)) {
1536 			netdev_warn(bp->dev, "TPA end agg_buf %d != expected agg_bufs %d\n",
1537 				    agg_bufs, tpa_info->agg_count);
1538 			agg_bufs = tpa_info->agg_count;
1539 		}
1540 		tpa_info->agg_count = 0;
1541 		*event |= BNXT_AGG_EVENT;
1542 		bnxt_free_agg_idx(rxr, agg_id);
1543 		idx = agg_id;
1544 		gro = !!(bp->flags & BNXT_FLAG_GRO);
1545 	} else {
1546 		agg_id = TPA_END_AGG_ID(tpa_end);
1547 		agg_bufs = TPA_END_AGG_BUFS(tpa_end);
1548 		tpa_info = &rxr->rx_tpa[agg_id];
1549 		idx = RING_CMP(*raw_cons);
1550 		if (agg_bufs) {
1551 			if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, raw_cons))
1552 				return ERR_PTR(-EBUSY);
1553 
1554 			*event |= BNXT_AGG_EVENT;
1555 			idx = NEXT_CMP(idx);
1556 		}
1557 		gro = !!TPA_END_GRO(tpa_end);
1558 	}
1559 	data = tpa_info->data;
1560 	data_ptr = tpa_info->data_ptr;
1561 	prefetch(data_ptr);
1562 	len = tpa_info->len;
1563 	mapping = tpa_info->mapping;
1564 
1565 	if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
1566 		bnxt_abort_tpa(cpr, idx, agg_bufs);
1567 		if (agg_bufs > MAX_SKB_FRAGS)
1568 			netdev_warn(bp->dev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
1569 				    agg_bufs, (int)MAX_SKB_FRAGS);
1570 		return NULL;
1571 	}
1572 
1573 	if (len <= bp->rx_copy_thresh) {
1574 		skb = bnxt_copy_skb(bnapi, data_ptr, len, mapping);
1575 		if (!skb) {
1576 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1577 			return NULL;
1578 		}
1579 	} else {
1580 		u8 *new_data;
1581 		dma_addr_t new_mapping;
1582 
1583 		new_data = __bnxt_alloc_rx_data(bp, &new_mapping, GFP_ATOMIC);
1584 		if (!new_data) {
1585 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1586 			return NULL;
1587 		}
1588 
1589 		tpa_info->data = new_data;
1590 		tpa_info->data_ptr = new_data + bp->rx_offset;
1591 		tpa_info->mapping = new_mapping;
1592 
1593 		skb = build_skb(data, 0);
1594 		dma_unmap_single_attrs(&bp->pdev->dev, mapping,
1595 				       bp->rx_buf_use_size, bp->rx_dir,
1596 				       DMA_ATTR_WEAK_ORDERING);
1597 
1598 		if (!skb) {
1599 			kfree(data);
1600 			bnxt_abort_tpa(cpr, idx, agg_bufs);
1601 			return NULL;
1602 		}
1603 		skb_reserve(skb, bp->rx_offset);
1604 		skb_put(skb, len);
1605 	}
1606 
1607 	if (agg_bufs) {
1608 		skb = bnxt_rx_pages(bp, cpr, skb, idx, agg_bufs, true);
1609 		if (!skb) {
1610 			/* Page reuse already handled by bnxt_rx_pages(). */
1611 			return NULL;
1612 		}
1613 	}
1614 
1615 	skb->protocol =
1616 		eth_type_trans(skb, bnxt_get_pkt_dev(bp, tpa_info->cfa_code));
1617 
1618 	if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
1619 		skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
1620 
1621 	if ((tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) &&
1622 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1623 		u16 vlan_proto = tpa_info->metadata >>
1624 			RX_CMP_FLAGS2_METADATA_TPID_SFT;
1625 		u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1626 
1627 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1628 	}
1629 
1630 	skb_checksum_none_assert(skb);
1631 	if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
1632 		skb->ip_summed = CHECKSUM_UNNECESSARY;
1633 		skb->csum_level =
1634 			(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
1635 	}
1636 
1637 	if (gro)
1638 		skb = bnxt_gro_skb(bp, tpa_info, tpa_end, tpa_end1, skb);
1639 
1640 	return skb;
1641 }
1642 
1643 static void bnxt_tpa_agg(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
1644 			 struct rx_agg_cmp *rx_agg)
1645 {
1646 	u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
1647 	struct bnxt_tpa_info *tpa_info;
1648 
1649 	agg_id = bnxt_lookup_agg_idx(rxr, agg_id);
1650 	tpa_info = &rxr->rx_tpa[agg_id];
1651 	BUG_ON(tpa_info->agg_count >= MAX_SKB_FRAGS);
1652 	tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
1653 }
1654 
1655 static void bnxt_deliver_skb(struct bnxt *bp, struct bnxt_napi *bnapi,
1656 			     struct sk_buff *skb)
1657 {
1658 	if (skb->dev != bp->dev) {
1659 		/* this packet belongs to a vf-rep */
1660 		bnxt_vf_rep_rx(bp, skb);
1661 		return;
1662 	}
1663 	skb_record_rx_queue(skb, bnapi->index);
1664 	napi_gro_receive(&bnapi->napi, skb);
1665 }
1666 
1667 /* returns the following:
1668  * 1       - 1 packet successfully received
1669  * 0       - successful TPA_START, packet not completed yet
1670  * -EBUSY  - completion ring does not have all the agg buffers yet
1671  * -ENOMEM - packet aborted due to out of memory
1672  * -EIO    - packet aborted due to hw error indicated in BD
1673  */
1674 static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
1675 		       u32 *raw_cons, u8 *event)
1676 {
1677 	struct bnxt_napi *bnapi = cpr->bnapi;
1678 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
1679 	struct net_device *dev = bp->dev;
1680 	struct rx_cmp *rxcmp;
1681 	struct rx_cmp_ext *rxcmp1;
1682 	u32 tmp_raw_cons = *raw_cons;
1683 	u16 cfa_code, cons, prod, cp_cons = RING_CMP(tmp_raw_cons);
1684 	struct bnxt_sw_rx_bd *rx_buf;
1685 	unsigned int len;
1686 	u8 *data_ptr, agg_bufs, cmp_type;
1687 	dma_addr_t dma_addr;
1688 	struct sk_buff *skb;
1689 	void *data;
1690 	int rc = 0;
1691 	u32 misc;
1692 
1693 	rxcmp = (struct rx_cmp *)
1694 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1695 
1696 	cmp_type = RX_CMP_TYPE(rxcmp);
1697 
1698 	if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
1699 		bnxt_tpa_agg(bp, rxr, (struct rx_agg_cmp *)rxcmp);
1700 		goto next_rx_no_prod_no_len;
1701 	}
1702 
1703 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1704 	cp_cons = RING_CMP(tmp_raw_cons);
1705 	rxcmp1 = (struct rx_cmp_ext *)
1706 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1707 
1708 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1709 		return -EBUSY;
1710 
1711 	prod = rxr->rx_prod;
1712 
1713 	if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP) {
1714 		bnxt_tpa_start(bp, rxr, (struct rx_tpa_start_cmp *)rxcmp,
1715 			       (struct rx_tpa_start_cmp_ext *)rxcmp1);
1716 
1717 		*event |= BNXT_RX_EVENT;
1718 		goto next_rx_no_prod_no_len;
1719 
1720 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1721 		skb = bnxt_tpa_end(bp, cpr, &tmp_raw_cons,
1722 				   (struct rx_tpa_end_cmp *)rxcmp,
1723 				   (struct rx_tpa_end_cmp_ext *)rxcmp1, event);
1724 
1725 		if (IS_ERR(skb))
1726 			return -EBUSY;
1727 
1728 		rc = -ENOMEM;
1729 		if (likely(skb)) {
1730 			bnxt_deliver_skb(bp, bnapi, skb);
1731 			rc = 1;
1732 		}
1733 		*event |= BNXT_RX_EVENT;
1734 		goto next_rx_no_prod_no_len;
1735 	}
1736 
1737 	cons = rxcmp->rx_cmp_opaque;
1738 	if (unlikely(cons != rxr->rx_next_cons)) {
1739 		int rc1 = bnxt_discard_rx(bp, cpr, raw_cons, rxcmp);
1740 
1741 		netdev_warn(bp->dev, "RX cons %x != expected cons %x\n",
1742 			    cons, rxr->rx_next_cons);
1743 		bnxt_sched_reset(bp, rxr);
1744 		return rc1;
1745 	}
1746 	rx_buf = &rxr->rx_buf_ring[cons];
1747 	data = rx_buf->data;
1748 	data_ptr = rx_buf->data_ptr;
1749 	prefetch(data_ptr);
1750 
1751 	misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
1752 	agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
1753 
1754 	if (agg_bufs) {
1755 		if (!bnxt_agg_bufs_valid(bp, cpr, agg_bufs, &tmp_raw_cons))
1756 			return -EBUSY;
1757 
1758 		cp_cons = NEXT_CMP(cp_cons);
1759 		*event |= BNXT_AGG_EVENT;
1760 	}
1761 	*event |= BNXT_RX_EVENT;
1762 
1763 	rx_buf->data = NULL;
1764 	if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
1765 		u32 rx_err = le32_to_cpu(rxcmp1->rx_cmp_cfa_code_errors_v2);
1766 
1767 		bnxt_reuse_rx_data(rxr, cons, data);
1768 		if (agg_bufs)
1769 			bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
1770 					       false);
1771 
1772 		rc = -EIO;
1773 		if (rx_err & RX_CMPL_ERRORS_BUFFER_ERROR_MASK) {
1774 			bnapi->cp_ring.sw_stats.rx.rx_buf_errors++;
1775 			if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
1776 				netdev_warn(bp->dev, "RX buffer error %x\n",
1777 					    rx_err);
1778 				bnxt_sched_reset(bp, rxr);
1779 			}
1780 		}
1781 		goto next_rx_no_len;
1782 	}
1783 
1784 	len = le32_to_cpu(rxcmp->rx_cmp_len_flags_type) >> RX_CMP_LEN_SHIFT;
1785 	dma_addr = rx_buf->mapping;
1786 
1787 	if (bnxt_rx_xdp(bp, rxr, cons, data, &data_ptr, &len, event)) {
1788 		rc = 1;
1789 		goto next_rx;
1790 	}
1791 
1792 	if (len <= bp->rx_copy_thresh) {
1793 		skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr);
1794 		bnxt_reuse_rx_data(rxr, cons, data);
1795 		if (!skb) {
1796 			if (agg_bufs)
1797 				bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0,
1798 						       agg_bufs, false);
1799 			rc = -ENOMEM;
1800 			goto next_rx;
1801 		}
1802 	} else {
1803 		u32 payload;
1804 
1805 		if (rx_buf->data_ptr == data_ptr)
1806 			payload = misc & RX_CMP_PAYLOAD_OFFSET;
1807 		else
1808 			payload = 0;
1809 		skb = bp->rx_skb_func(bp, rxr, cons, data, data_ptr, dma_addr,
1810 				      payload | len);
1811 		if (!skb) {
1812 			rc = -ENOMEM;
1813 			goto next_rx;
1814 		}
1815 	}
1816 
1817 	if (agg_bufs) {
1818 		skb = bnxt_rx_pages(bp, cpr, skb, cp_cons, agg_bufs, false);
1819 		if (!skb) {
1820 			rc = -ENOMEM;
1821 			goto next_rx;
1822 		}
1823 	}
1824 
1825 	if (RX_CMP_HASH_VALID(rxcmp)) {
1826 		u32 hash_type = RX_CMP_HASH_TYPE(rxcmp);
1827 		enum pkt_hash_types type = PKT_HASH_TYPE_L4;
1828 
1829 		/* RSS profiles 1 and 3 with extract code 0 for inner 4-tuple */
1830 		if (hash_type != 1 && hash_type != 3)
1831 			type = PKT_HASH_TYPE_L3;
1832 		skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
1833 	}
1834 
1835 	cfa_code = RX_CMP_CFA_CODE(rxcmp1);
1836 	skb->protocol = eth_type_trans(skb, bnxt_get_pkt_dev(bp, cfa_code));
1837 
1838 	if ((rxcmp1->rx_cmp_flags2 &
1839 	     cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)) &&
1840 	    (skb->dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)) {
1841 		u32 meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
1842 		u16 vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
1843 		u16 vlan_proto = meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT;
1844 
1845 		__vlan_hwaccel_put_tag(skb, htons(vlan_proto), vtag);
1846 	}
1847 
1848 	skb_checksum_none_assert(skb);
1849 	if (RX_CMP_L4_CS_OK(rxcmp1)) {
1850 		if (dev->features & NETIF_F_RXCSUM) {
1851 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1852 			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
1853 		}
1854 	} else {
1855 		if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS) {
1856 			if (dev->features & NETIF_F_RXCSUM)
1857 				bnapi->cp_ring.sw_stats.rx.rx_l4_csum_errors++;
1858 		}
1859 	}
1860 
1861 	bnxt_deliver_skb(bp, bnapi, skb);
1862 	rc = 1;
1863 
1864 next_rx:
1865 	cpr->rx_packets += 1;
1866 	cpr->rx_bytes += len;
1867 
1868 next_rx_no_len:
1869 	rxr->rx_prod = NEXT_RX(prod);
1870 	rxr->rx_next_cons = NEXT_RX(cons);
1871 
1872 next_rx_no_prod_no_len:
1873 	*raw_cons = tmp_raw_cons;
1874 
1875 	return rc;
1876 }
1877 
1878 /* In netpoll mode, if we are using a combined completion ring, we need to
1879  * discard the rx packets and recycle the buffers.
1880  */
1881 static int bnxt_force_rx_discard(struct bnxt *bp,
1882 				 struct bnxt_cp_ring_info *cpr,
1883 				 u32 *raw_cons, u8 *event)
1884 {
1885 	u32 tmp_raw_cons = *raw_cons;
1886 	struct rx_cmp_ext *rxcmp1;
1887 	struct rx_cmp *rxcmp;
1888 	u16 cp_cons;
1889 	u8 cmp_type;
1890 
1891 	cp_cons = RING_CMP(tmp_raw_cons);
1892 	rxcmp = (struct rx_cmp *)
1893 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1894 
1895 	tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1896 	cp_cons = RING_CMP(tmp_raw_cons);
1897 	rxcmp1 = (struct rx_cmp_ext *)
1898 			&cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1899 
1900 	if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
1901 		return -EBUSY;
1902 
1903 	cmp_type = RX_CMP_TYPE(rxcmp);
1904 	if (cmp_type == CMP_TYPE_RX_L2_CMP) {
1905 		rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1906 			cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1907 	} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1908 		struct rx_tpa_end_cmp_ext *tpa_end1;
1909 
1910 		tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1911 		tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1912 			cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1913 	}
1914 	return bnxt_rx_pkt(bp, cpr, raw_cons, event);
1915 }
1916 
1917 u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx)
1918 {
1919 	struct bnxt_fw_health *fw_health = bp->fw_health;
1920 	u32 reg = fw_health->regs[reg_idx];
1921 	u32 reg_type, reg_off, val = 0;
1922 
1923 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
1924 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
1925 	switch (reg_type) {
1926 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
1927 		pci_read_config_dword(bp->pdev, reg_off, &val);
1928 		break;
1929 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
1930 		reg_off = fw_health->mapped_regs[reg_idx];
1931 		fallthrough;
1932 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
1933 		val = readl(bp->bar0 + reg_off);
1934 		break;
1935 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
1936 		val = readl(bp->bar1 + reg_off);
1937 		break;
1938 	}
1939 	if (reg_idx == BNXT_FW_RESET_INPROG_REG)
1940 		val &= fw_health->fw_reset_inprog_reg_mask;
1941 	return val;
1942 }
1943 
1944 #define BNXT_GET_EVENT_PORT(data)	\
1945 	((data) &			\
1946 	 ASYNC_EVENT_CMPL_PORT_CONN_NOT_ALLOWED_EVENT_DATA1_PORT_ID_MASK)
1947 
1948 static int bnxt_async_event_process(struct bnxt *bp,
1949 				    struct hwrm_async_event_cmpl *cmpl)
1950 {
1951 	u16 event_id = le16_to_cpu(cmpl->event_id);
1952 
1953 	/* TODO CHIMP_FW: Define event id's for link change, error etc */
1954 	switch (event_id) {
1955 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: {
1956 		u32 data1 = le32_to_cpu(cmpl->event_data1);
1957 		struct bnxt_link_info *link_info = &bp->link_info;
1958 
1959 		if (BNXT_VF(bp))
1960 			goto async_event_process_exit;
1961 
1962 		/* print unsupported speed warning in forced speed mode only */
1963 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED) &&
1964 		    (data1 & 0x20000)) {
1965 			u16 fw_speed = link_info->force_link_speed;
1966 			u32 speed = bnxt_fw_to_ethtool_speed(fw_speed);
1967 
1968 			if (speed != SPEED_UNKNOWN)
1969 				netdev_warn(bp->dev, "Link speed %d no longer supported\n",
1970 					    speed);
1971 		}
1972 		set_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT, &bp->sp_event);
1973 	}
1974 		fallthrough;
1975 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE:
1976 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE:
1977 		set_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT, &bp->sp_event);
1978 		fallthrough;
1979 	case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE:
1980 		set_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event);
1981 		break;
1982 	case ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD:
1983 		set_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event);
1984 		break;
1985 	case ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: {
1986 		u32 data1 = le32_to_cpu(cmpl->event_data1);
1987 		u16 port_id = BNXT_GET_EVENT_PORT(data1);
1988 
1989 		if (BNXT_VF(bp))
1990 			break;
1991 
1992 		if (bp->pf.port_id != port_id)
1993 			break;
1994 
1995 		set_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event);
1996 		break;
1997 	}
1998 	case ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE:
1999 		if (BNXT_PF(bp))
2000 			goto async_event_process_exit;
2001 		set_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event);
2002 		break;
2003 	case ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: {
2004 		u32 data1 = le32_to_cpu(cmpl->event_data1);
2005 
2006 		if (!bp->fw_health)
2007 			goto async_event_process_exit;
2008 
2009 		bp->fw_reset_timestamp = jiffies;
2010 		bp->fw_reset_min_dsecs = cmpl->timestamp_lo;
2011 		if (!bp->fw_reset_min_dsecs)
2012 			bp->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS;
2013 		bp->fw_reset_max_dsecs = le16_to_cpu(cmpl->timestamp_hi);
2014 		if (!bp->fw_reset_max_dsecs)
2015 			bp->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS;
2016 		if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
2017 			netdev_warn(bp->dev, "Firmware fatal reset event received\n");
2018 			set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
2019 		} else {
2020 			netdev_warn(bp->dev, "Firmware non-fatal reset event received, max wait time %d msec\n",
2021 				    bp->fw_reset_max_dsecs * 100);
2022 		}
2023 		set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event);
2024 		break;
2025 	}
2026 	case ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: {
2027 		struct bnxt_fw_health *fw_health = bp->fw_health;
2028 		u32 data1 = le32_to_cpu(cmpl->event_data1);
2029 
2030 		if (!fw_health)
2031 			goto async_event_process_exit;
2032 
2033 		fw_health->enabled = EVENT_DATA1_RECOVERY_ENABLED(data1);
2034 		fw_health->master = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1);
2035 		if (!fw_health->enabled)
2036 			break;
2037 
2038 		if (netif_msg_drv(bp))
2039 			netdev_info(bp->dev, "Error recovery info: error recovery[%d], master[%d], reset count[0x%x], health status: 0x%x\n",
2040 				    fw_health->enabled, fw_health->master,
2041 				    bnxt_fw_health_readl(bp,
2042 							 BNXT_FW_RESET_CNT_REG),
2043 				    bnxt_fw_health_readl(bp,
2044 							 BNXT_FW_HEALTH_REG));
2045 		fw_health->tmr_multiplier =
2046 			DIV_ROUND_UP(fw_health->polling_dsecs * HZ,
2047 				     bp->current_interval * 10);
2048 		fw_health->tmr_counter = fw_health->tmr_multiplier;
2049 		fw_health->last_fw_heartbeat =
2050 			bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
2051 		fw_health->last_fw_reset_cnt =
2052 			bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
2053 		goto async_event_process_exit;
2054 	}
2055 	default:
2056 		goto async_event_process_exit;
2057 	}
2058 	bnxt_queue_sp_work(bp);
2059 async_event_process_exit:
2060 	bnxt_ulp_async_events(bp, cmpl);
2061 	return 0;
2062 }
2063 
2064 static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp)
2065 {
2066 	u16 cmpl_type = TX_CMP_TYPE(txcmp), vf_id, seq_id;
2067 	struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
2068 	struct hwrm_fwd_req_cmpl *fwd_req_cmpl =
2069 				(struct hwrm_fwd_req_cmpl *)txcmp;
2070 
2071 	switch (cmpl_type) {
2072 	case CMPL_BASE_TYPE_HWRM_DONE:
2073 		seq_id = le16_to_cpu(h_cmpl->sequence_id);
2074 		if (seq_id == bp->hwrm_intr_seq_id)
2075 			bp->hwrm_intr_seq_id = (u16)~bp->hwrm_intr_seq_id;
2076 		else
2077 			netdev_err(bp->dev, "Invalid hwrm seq id %d\n", seq_id);
2078 		break;
2079 
2080 	case CMPL_BASE_TYPE_HWRM_FWD_REQ:
2081 		vf_id = le16_to_cpu(fwd_req_cmpl->source_id);
2082 
2083 		if ((vf_id < bp->pf.first_vf_id) ||
2084 		    (vf_id >= bp->pf.first_vf_id + bp->pf.active_vfs)) {
2085 			netdev_err(bp->dev, "Msg contains invalid VF id %x\n",
2086 				   vf_id);
2087 			return -EINVAL;
2088 		}
2089 
2090 		set_bit(vf_id - bp->pf.first_vf_id, bp->pf.vf_event_bmap);
2091 		set_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event);
2092 		bnxt_queue_sp_work(bp);
2093 		break;
2094 
2095 	case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
2096 		bnxt_async_event_process(bp,
2097 					 (struct hwrm_async_event_cmpl *)txcmp);
2098 
2099 	default:
2100 		break;
2101 	}
2102 
2103 	return 0;
2104 }
2105 
2106 static irqreturn_t bnxt_msix(int irq, void *dev_instance)
2107 {
2108 	struct bnxt_napi *bnapi = dev_instance;
2109 	struct bnxt *bp = bnapi->bp;
2110 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2111 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2112 
2113 	cpr->event_ctr++;
2114 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2115 	napi_schedule(&bnapi->napi);
2116 	return IRQ_HANDLED;
2117 }
2118 
2119 static inline int bnxt_has_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr)
2120 {
2121 	u32 raw_cons = cpr->cp_raw_cons;
2122 	u16 cons = RING_CMP(raw_cons);
2123 	struct tx_cmp *txcmp;
2124 
2125 	txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2126 
2127 	return TX_CMP_VALID(txcmp, raw_cons);
2128 }
2129 
2130 static irqreturn_t bnxt_inta(int irq, void *dev_instance)
2131 {
2132 	struct bnxt_napi *bnapi = dev_instance;
2133 	struct bnxt *bp = bnapi->bp;
2134 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2135 	u32 cons = RING_CMP(cpr->cp_raw_cons);
2136 	u32 int_status;
2137 
2138 	prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]);
2139 
2140 	if (!bnxt_has_work(bp, cpr)) {
2141 		int_status = readl(bp->bar0 + BNXT_CAG_REG_LEGACY_INT_STATUS);
2142 		/* return if erroneous interrupt */
2143 		if (!(int_status & (0x10000 << cpr->cp_ring_struct.fw_ring_id)))
2144 			return IRQ_NONE;
2145 	}
2146 
2147 	/* disable ring IRQ */
2148 	BNXT_CP_DB_IRQ_DIS(cpr->cp_db.doorbell);
2149 
2150 	/* Return here if interrupt is shared and is disabled. */
2151 	if (unlikely(atomic_read(&bp->intr_sem) != 0))
2152 		return IRQ_HANDLED;
2153 
2154 	napi_schedule(&bnapi->napi);
2155 	return IRQ_HANDLED;
2156 }
2157 
2158 static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2159 			    int budget)
2160 {
2161 	struct bnxt_napi *bnapi = cpr->bnapi;
2162 	u32 raw_cons = cpr->cp_raw_cons;
2163 	u32 cons;
2164 	int tx_pkts = 0;
2165 	int rx_pkts = 0;
2166 	u8 event = 0;
2167 	struct tx_cmp *txcmp;
2168 
2169 	cpr->has_more_work = 0;
2170 	cpr->had_work_done = 1;
2171 	while (1) {
2172 		int rc;
2173 
2174 		cons = RING_CMP(raw_cons);
2175 		txcmp = &cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2176 
2177 		if (!TX_CMP_VALID(txcmp, raw_cons))
2178 			break;
2179 
2180 		/* The valid test of the entry must be done first before
2181 		 * reading any further.
2182 		 */
2183 		dma_rmb();
2184 		if (TX_CMP_TYPE(txcmp) == CMP_TYPE_TX_L2_CMP) {
2185 			tx_pkts++;
2186 			/* return full budget so NAPI will complete. */
2187 			if (unlikely(tx_pkts > bp->tx_wake_thresh)) {
2188 				rx_pkts = budget;
2189 				raw_cons = NEXT_RAW_CMP(raw_cons);
2190 				if (budget)
2191 					cpr->has_more_work = 1;
2192 				break;
2193 			}
2194 		} else if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2195 			if (likely(budget))
2196 				rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2197 			else
2198 				rc = bnxt_force_rx_discard(bp, cpr, &raw_cons,
2199 							   &event);
2200 			if (likely(rc >= 0))
2201 				rx_pkts += rc;
2202 			/* Increment rx_pkts when rc is -ENOMEM to count towards
2203 			 * the NAPI budget.  Otherwise, we may potentially loop
2204 			 * here forever if we consistently cannot allocate
2205 			 * buffers.
2206 			 */
2207 			else if (rc == -ENOMEM && budget)
2208 				rx_pkts++;
2209 			else if (rc == -EBUSY)	/* partial completion */
2210 				break;
2211 		} else if (unlikely((TX_CMP_TYPE(txcmp) ==
2212 				     CMPL_BASE_TYPE_HWRM_DONE) ||
2213 				    (TX_CMP_TYPE(txcmp) ==
2214 				     CMPL_BASE_TYPE_HWRM_FWD_REQ) ||
2215 				    (TX_CMP_TYPE(txcmp) ==
2216 				     CMPL_BASE_TYPE_HWRM_ASYNC_EVENT))) {
2217 			bnxt_hwrm_handler(bp, txcmp);
2218 		}
2219 		raw_cons = NEXT_RAW_CMP(raw_cons);
2220 
2221 		if (rx_pkts && rx_pkts == budget) {
2222 			cpr->has_more_work = 1;
2223 			break;
2224 		}
2225 	}
2226 
2227 	if (event & BNXT_REDIRECT_EVENT)
2228 		xdp_do_flush_map();
2229 
2230 	if (event & BNXT_TX_EVENT) {
2231 		struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
2232 		u16 prod = txr->tx_prod;
2233 
2234 		/* Sync BD data before updating doorbell */
2235 		wmb();
2236 
2237 		bnxt_db_write_relaxed(bp, &txr->tx_db, prod);
2238 	}
2239 
2240 	cpr->cp_raw_cons = raw_cons;
2241 	bnapi->tx_pkts += tx_pkts;
2242 	bnapi->events |= event;
2243 	return rx_pkts;
2244 }
2245 
2246 static void __bnxt_poll_work_done(struct bnxt *bp, struct bnxt_napi *bnapi)
2247 {
2248 	if (bnapi->tx_pkts) {
2249 		bnapi->tx_int(bp, bnapi, bnapi->tx_pkts);
2250 		bnapi->tx_pkts = 0;
2251 	}
2252 
2253 	if (bnapi->events & BNXT_RX_EVENT) {
2254 		struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2255 
2256 		if (bnapi->events & BNXT_AGG_EVENT)
2257 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2258 		bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2259 	}
2260 	bnapi->events = 0;
2261 }
2262 
2263 static int bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr,
2264 			  int budget)
2265 {
2266 	struct bnxt_napi *bnapi = cpr->bnapi;
2267 	int rx_pkts;
2268 
2269 	rx_pkts = __bnxt_poll_work(bp, cpr, budget);
2270 
2271 	/* ACK completion ring before freeing tx ring and producing new
2272 	 * buffers in rx/agg rings to prevent overflowing the completion
2273 	 * ring.
2274 	 */
2275 	bnxt_db_cq(bp, &cpr->cp_db, cpr->cp_raw_cons);
2276 
2277 	__bnxt_poll_work_done(bp, bnapi);
2278 	return rx_pkts;
2279 }
2280 
2281 static int bnxt_poll_nitroa0(struct napi_struct *napi, int budget)
2282 {
2283 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2284 	struct bnxt *bp = bnapi->bp;
2285 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2286 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
2287 	struct tx_cmp *txcmp;
2288 	struct rx_cmp_ext *rxcmp1;
2289 	u32 cp_cons, tmp_raw_cons;
2290 	u32 raw_cons = cpr->cp_raw_cons;
2291 	u32 rx_pkts = 0;
2292 	u8 event = 0;
2293 
2294 	while (1) {
2295 		int rc;
2296 
2297 		cp_cons = RING_CMP(raw_cons);
2298 		txcmp = &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2299 
2300 		if (!TX_CMP_VALID(txcmp, raw_cons))
2301 			break;
2302 
2303 		if ((TX_CMP_TYPE(txcmp) & 0x30) == 0x10) {
2304 			tmp_raw_cons = NEXT_RAW_CMP(raw_cons);
2305 			cp_cons = RING_CMP(tmp_raw_cons);
2306 			rxcmp1 = (struct rx_cmp_ext *)
2307 			  &cpr->cp_desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
2308 
2309 			if (!RX_CMP_VALID(rxcmp1, tmp_raw_cons))
2310 				break;
2311 
2312 			/* force an error to recycle the buffer */
2313 			rxcmp1->rx_cmp_cfa_code_errors_v2 |=
2314 				cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
2315 
2316 			rc = bnxt_rx_pkt(bp, cpr, &raw_cons, &event);
2317 			if (likely(rc == -EIO) && budget)
2318 				rx_pkts++;
2319 			else if (rc == -EBUSY)	/* partial completion */
2320 				break;
2321 		} else if (unlikely(TX_CMP_TYPE(txcmp) ==
2322 				    CMPL_BASE_TYPE_HWRM_DONE)) {
2323 			bnxt_hwrm_handler(bp, txcmp);
2324 		} else {
2325 			netdev_err(bp->dev,
2326 				   "Invalid completion received on special ring\n");
2327 		}
2328 		raw_cons = NEXT_RAW_CMP(raw_cons);
2329 
2330 		if (rx_pkts == budget)
2331 			break;
2332 	}
2333 
2334 	cpr->cp_raw_cons = raw_cons;
2335 	BNXT_DB_CQ(&cpr->cp_db, cpr->cp_raw_cons);
2336 	bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
2337 
2338 	if (event & BNXT_AGG_EVENT)
2339 		bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
2340 
2341 	if (!bnxt_has_work(bp, cpr) && rx_pkts < budget) {
2342 		napi_complete_done(napi, rx_pkts);
2343 		BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2344 	}
2345 	return rx_pkts;
2346 }
2347 
2348 static int bnxt_poll(struct napi_struct *napi, int budget)
2349 {
2350 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2351 	struct bnxt *bp = bnapi->bp;
2352 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2353 	int work_done = 0;
2354 
2355 	while (1) {
2356 		work_done += bnxt_poll_work(bp, cpr, budget - work_done);
2357 
2358 		if (work_done >= budget) {
2359 			if (!budget)
2360 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2361 			break;
2362 		}
2363 
2364 		if (!bnxt_has_work(bp, cpr)) {
2365 			if (napi_complete_done(napi, work_done))
2366 				BNXT_DB_CQ_ARM(&cpr->cp_db, cpr->cp_raw_cons);
2367 			break;
2368 		}
2369 	}
2370 	if (bp->flags & BNXT_FLAG_DIM) {
2371 		struct dim_sample dim_sample = {};
2372 
2373 		dim_update_sample(cpr->event_ctr,
2374 				  cpr->rx_packets,
2375 				  cpr->rx_bytes,
2376 				  &dim_sample);
2377 		net_dim(&cpr->dim, dim_sample);
2378 	}
2379 	return work_done;
2380 }
2381 
2382 static int __bnxt_poll_cqs(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)
2383 {
2384 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2385 	int i, work_done = 0;
2386 
2387 	for (i = 0; i < 2; i++) {
2388 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2389 
2390 		if (cpr2) {
2391 			work_done += __bnxt_poll_work(bp, cpr2,
2392 						      budget - work_done);
2393 			cpr->has_more_work |= cpr2->has_more_work;
2394 		}
2395 	}
2396 	return work_done;
2397 }
2398 
2399 static void __bnxt_poll_cqs_done(struct bnxt *bp, struct bnxt_napi *bnapi,
2400 				 u64 dbr_type)
2401 {
2402 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2403 	int i;
2404 
2405 	for (i = 0; i < 2; i++) {
2406 		struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[i];
2407 		struct bnxt_db_info *db;
2408 
2409 		if (cpr2 && cpr2->had_work_done) {
2410 			db = &cpr2->cp_db;
2411 			writeq(db->db_key64 | dbr_type |
2412 			       RING_CMP(cpr2->cp_raw_cons), db->doorbell);
2413 			cpr2->had_work_done = 0;
2414 		}
2415 	}
2416 	__bnxt_poll_work_done(bp, bnapi);
2417 }
2418 
2419 static int bnxt_poll_p5(struct napi_struct *napi, int budget)
2420 {
2421 	struct bnxt_napi *bnapi = container_of(napi, struct bnxt_napi, napi);
2422 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
2423 	u32 raw_cons = cpr->cp_raw_cons;
2424 	struct bnxt *bp = bnapi->bp;
2425 	struct nqe_cn *nqcmp;
2426 	int work_done = 0;
2427 	u32 cons;
2428 
2429 	if (cpr->has_more_work) {
2430 		cpr->has_more_work = 0;
2431 		work_done = __bnxt_poll_cqs(bp, bnapi, budget);
2432 	}
2433 	while (1) {
2434 		cons = RING_CMP(raw_cons);
2435 		nqcmp = &cpr->nq_desc_ring[CP_RING(cons)][CP_IDX(cons)];
2436 
2437 		if (!NQ_CMP_VALID(nqcmp, raw_cons)) {
2438 			if (cpr->has_more_work)
2439 				break;
2440 
2441 			__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ_ARMALL);
2442 			cpr->cp_raw_cons = raw_cons;
2443 			if (napi_complete_done(napi, work_done))
2444 				BNXT_DB_NQ_ARM_P5(&cpr->cp_db,
2445 						  cpr->cp_raw_cons);
2446 			return work_done;
2447 		}
2448 
2449 		/* The valid test of the entry must be done first before
2450 		 * reading any further.
2451 		 */
2452 		dma_rmb();
2453 
2454 		if (nqcmp->type == cpu_to_le16(NQ_CN_TYPE_CQ_NOTIFICATION)) {
2455 			u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
2456 			struct bnxt_cp_ring_info *cpr2;
2457 
2458 			cpr2 = cpr->cp_ring_arr[idx];
2459 			work_done += __bnxt_poll_work(bp, cpr2,
2460 						      budget - work_done);
2461 			cpr->has_more_work |= cpr2->has_more_work;
2462 		} else {
2463 			bnxt_hwrm_handler(bp, (struct tx_cmp *)nqcmp);
2464 		}
2465 		raw_cons = NEXT_RAW_CMP(raw_cons);
2466 	}
2467 	__bnxt_poll_cqs_done(bp, bnapi, DBR_TYPE_CQ);
2468 	if (raw_cons != cpr->cp_raw_cons) {
2469 		cpr->cp_raw_cons = raw_cons;
2470 		BNXT_DB_NQ_P5(&cpr->cp_db, raw_cons);
2471 	}
2472 	return work_done;
2473 }
2474 
2475 static void bnxt_free_tx_skbs(struct bnxt *bp)
2476 {
2477 	int i, max_idx;
2478 	struct pci_dev *pdev = bp->pdev;
2479 
2480 	if (!bp->tx_ring)
2481 		return;
2482 
2483 	max_idx = bp->tx_nr_pages * TX_DESC_CNT;
2484 	for (i = 0; i < bp->tx_nr_rings; i++) {
2485 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2486 		int j;
2487 
2488 		for (j = 0; j < max_idx;) {
2489 			struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
2490 			struct sk_buff *skb;
2491 			int k, last;
2492 
2493 			if (i < bp->tx_nr_rings_xdp &&
2494 			    tx_buf->action == XDP_REDIRECT) {
2495 				dma_unmap_single(&pdev->dev,
2496 					dma_unmap_addr(tx_buf, mapping),
2497 					dma_unmap_len(tx_buf, len),
2498 					PCI_DMA_TODEVICE);
2499 				xdp_return_frame(tx_buf->xdpf);
2500 				tx_buf->action = 0;
2501 				tx_buf->xdpf = NULL;
2502 				j++;
2503 				continue;
2504 			}
2505 
2506 			skb = tx_buf->skb;
2507 			if (!skb) {
2508 				j++;
2509 				continue;
2510 			}
2511 
2512 			tx_buf->skb = NULL;
2513 
2514 			if (tx_buf->is_push) {
2515 				dev_kfree_skb(skb);
2516 				j += 2;
2517 				continue;
2518 			}
2519 
2520 			dma_unmap_single(&pdev->dev,
2521 					 dma_unmap_addr(tx_buf, mapping),
2522 					 skb_headlen(skb),
2523 					 PCI_DMA_TODEVICE);
2524 
2525 			last = tx_buf->nr_frags;
2526 			j += 2;
2527 			for (k = 0; k < last; k++, j++) {
2528 				int ring_idx = j & bp->tx_ring_mask;
2529 				skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
2530 
2531 				tx_buf = &txr->tx_buf_ring[ring_idx];
2532 				dma_unmap_page(
2533 					&pdev->dev,
2534 					dma_unmap_addr(tx_buf, mapping),
2535 					skb_frag_size(frag), PCI_DMA_TODEVICE);
2536 			}
2537 			dev_kfree_skb(skb);
2538 		}
2539 		netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, i));
2540 	}
2541 }
2542 
2543 static void bnxt_free_rx_skbs(struct bnxt *bp)
2544 {
2545 	int i, max_idx, max_agg_idx;
2546 	struct pci_dev *pdev = bp->pdev;
2547 
2548 	if (!bp->rx_ring)
2549 		return;
2550 
2551 	max_idx = bp->rx_nr_pages * RX_DESC_CNT;
2552 	max_agg_idx = bp->rx_agg_nr_pages * RX_DESC_CNT;
2553 	for (i = 0; i < bp->rx_nr_rings; i++) {
2554 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2555 		struct bnxt_tpa_idx_map *map;
2556 		int j;
2557 
2558 		if (rxr->rx_tpa) {
2559 			for (j = 0; j < bp->max_tpa; j++) {
2560 				struct bnxt_tpa_info *tpa_info =
2561 							&rxr->rx_tpa[j];
2562 				u8 *data = tpa_info->data;
2563 
2564 				if (!data)
2565 					continue;
2566 
2567 				dma_unmap_single_attrs(&pdev->dev,
2568 						       tpa_info->mapping,
2569 						       bp->rx_buf_use_size,
2570 						       bp->rx_dir,
2571 						       DMA_ATTR_WEAK_ORDERING);
2572 
2573 				tpa_info->data = NULL;
2574 
2575 				kfree(data);
2576 			}
2577 		}
2578 
2579 		for (j = 0; j < max_idx; j++) {
2580 			struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[j];
2581 			dma_addr_t mapping = rx_buf->mapping;
2582 			void *data = rx_buf->data;
2583 
2584 			if (!data)
2585 				continue;
2586 
2587 			rx_buf->data = NULL;
2588 
2589 			if (BNXT_RX_PAGE_MODE(bp)) {
2590 				mapping -= bp->rx_dma_offset;
2591 				dma_unmap_page_attrs(&pdev->dev, mapping,
2592 						     PAGE_SIZE, bp->rx_dir,
2593 						     DMA_ATTR_WEAK_ORDERING);
2594 				page_pool_recycle_direct(rxr->page_pool, data);
2595 			} else {
2596 				dma_unmap_single_attrs(&pdev->dev, mapping,
2597 						       bp->rx_buf_use_size,
2598 						       bp->rx_dir,
2599 						       DMA_ATTR_WEAK_ORDERING);
2600 				kfree(data);
2601 			}
2602 		}
2603 
2604 		for (j = 0; j < max_agg_idx; j++) {
2605 			struct bnxt_sw_rx_agg_bd *rx_agg_buf =
2606 				&rxr->rx_agg_ring[j];
2607 			struct page *page = rx_agg_buf->page;
2608 
2609 			if (!page)
2610 				continue;
2611 
2612 			dma_unmap_page_attrs(&pdev->dev, rx_agg_buf->mapping,
2613 					     BNXT_RX_PAGE_SIZE,
2614 					     PCI_DMA_FROMDEVICE,
2615 					     DMA_ATTR_WEAK_ORDERING);
2616 
2617 			rx_agg_buf->page = NULL;
2618 			__clear_bit(j, rxr->rx_agg_bmap);
2619 
2620 			__free_page(page);
2621 		}
2622 		if (rxr->rx_page) {
2623 			__free_page(rxr->rx_page);
2624 			rxr->rx_page = NULL;
2625 		}
2626 		map = rxr->rx_tpa_idx_map;
2627 		if (map)
2628 			memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
2629 	}
2630 }
2631 
2632 static void bnxt_free_skbs(struct bnxt *bp)
2633 {
2634 	bnxt_free_tx_skbs(bp);
2635 	bnxt_free_rx_skbs(bp);
2636 }
2637 
2638 static void bnxt_free_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2639 {
2640 	struct pci_dev *pdev = bp->pdev;
2641 	int i;
2642 
2643 	for (i = 0; i < rmem->nr_pages; i++) {
2644 		if (!rmem->pg_arr[i])
2645 			continue;
2646 
2647 		dma_free_coherent(&pdev->dev, rmem->page_size,
2648 				  rmem->pg_arr[i], rmem->dma_arr[i]);
2649 
2650 		rmem->pg_arr[i] = NULL;
2651 	}
2652 	if (rmem->pg_tbl) {
2653 		size_t pg_tbl_size = rmem->nr_pages * 8;
2654 
2655 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2656 			pg_tbl_size = rmem->page_size;
2657 		dma_free_coherent(&pdev->dev, pg_tbl_size,
2658 				  rmem->pg_tbl, rmem->pg_tbl_map);
2659 		rmem->pg_tbl = NULL;
2660 	}
2661 	if (rmem->vmem_size && *rmem->vmem) {
2662 		vfree(*rmem->vmem);
2663 		*rmem->vmem = NULL;
2664 	}
2665 }
2666 
2667 static int bnxt_alloc_ring(struct bnxt *bp, struct bnxt_ring_mem_info *rmem)
2668 {
2669 	struct pci_dev *pdev = bp->pdev;
2670 	u64 valid_bit = 0;
2671 	int i;
2672 
2673 	if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG))
2674 		valid_bit = PTU_PTE_VALID;
2675 	if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl) {
2676 		size_t pg_tbl_size = rmem->nr_pages * 8;
2677 
2678 		if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG)
2679 			pg_tbl_size = rmem->page_size;
2680 		rmem->pg_tbl = dma_alloc_coherent(&pdev->dev, pg_tbl_size,
2681 						  &rmem->pg_tbl_map,
2682 						  GFP_KERNEL);
2683 		if (!rmem->pg_tbl)
2684 			return -ENOMEM;
2685 	}
2686 
2687 	for (i = 0; i < rmem->nr_pages; i++) {
2688 		u64 extra_bits = valid_bit;
2689 
2690 		rmem->pg_arr[i] = dma_alloc_coherent(&pdev->dev,
2691 						     rmem->page_size,
2692 						     &rmem->dma_arr[i],
2693 						     GFP_KERNEL);
2694 		if (!rmem->pg_arr[i])
2695 			return -ENOMEM;
2696 
2697 		if (rmem->init_val)
2698 			memset(rmem->pg_arr[i], rmem->init_val,
2699 			       rmem->page_size);
2700 		if (rmem->nr_pages > 1 || rmem->depth > 0) {
2701 			if (i == rmem->nr_pages - 2 &&
2702 			    (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2703 				extra_bits |= PTU_PTE_NEXT_TO_LAST;
2704 			else if (i == rmem->nr_pages - 1 &&
2705 				 (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
2706 				extra_bits |= PTU_PTE_LAST;
2707 			rmem->pg_tbl[i] =
2708 				cpu_to_le64(rmem->dma_arr[i] | extra_bits);
2709 		}
2710 	}
2711 
2712 	if (rmem->vmem_size) {
2713 		*rmem->vmem = vzalloc(rmem->vmem_size);
2714 		if (!(*rmem->vmem))
2715 			return -ENOMEM;
2716 	}
2717 	return 0;
2718 }
2719 
2720 static void bnxt_free_tpa_info(struct bnxt *bp)
2721 {
2722 	int i;
2723 
2724 	for (i = 0; i < bp->rx_nr_rings; i++) {
2725 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2726 
2727 		kfree(rxr->rx_tpa_idx_map);
2728 		rxr->rx_tpa_idx_map = NULL;
2729 		if (rxr->rx_tpa) {
2730 			kfree(rxr->rx_tpa[0].agg_arr);
2731 			rxr->rx_tpa[0].agg_arr = NULL;
2732 		}
2733 		kfree(rxr->rx_tpa);
2734 		rxr->rx_tpa = NULL;
2735 	}
2736 }
2737 
2738 static int bnxt_alloc_tpa_info(struct bnxt *bp)
2739 {
2740 	int i, j, total_aggs = 0;
2741 
2742 	bp->max_tpa = MAX_TPA;
2743 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
2744 		if (!bp->max_tpa_v2)
2745 			return 0;
2746 		bp->max_tpa = max_t(u16, bp->max_tpa_v2, MAX_TPA_P5);
2747 		total_aggs = bp->max_tpa * MAX_SKB_FRAGS;
2748 	}
2749 
2750 	for (i = 0; i < bp->rx_nr_rings; i++) {
2751 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2752 		struct rx_agg_cmp *agg;
2753 
2754 		rxr->rx_tpa = kcalloc(bp->max_tpa, sizeof(struct bnxt_tpa_info),
2755 				      GFP_KERNEL);
2756 		if (!rxr->rx_tpa)
2757 			return -ENOMEM;
2758 
2759 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
2760 			continue;
2761 		agg = kcalloc(total_aggs, sizeof(*agg), GFP_KERNEL);
2762 		rxr->rx_tpa[0].agg_arr = agg;
2763 		if (!agg)
2764 			return -ENOMEM;
2765 		for (j = 1; j < bp->max_tpa; j++)
2766 			rxr->rx_tpa[j].agg_arr = agg + j * MAX_SKB_FRAGS;
2767 		rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
2768 					      GFP_KERNEL);
2769 		if (!rxr->rx_tpa_idx_map)
2770 			return -ENOMEM;
2771 	}
2772 	return 0;
2773 }
2774 
2775 static void bnxt_free_rx_rings(struct bnxt *bp)
2776 {
2777 	int i;
2778 
2779 	if (!bp->rx_ring)
2780 		return;
2781 
2782 	bnxt_free_tpa_info(bp);
2783 	for (i = 0; i < bp->rx_nr_rings; i++) {
2784 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2785 		struct bnxt_ring_struct *ring;
2786 
2787 		if (rxr->xdp_prog)
2788 			bpf_prog_put(rxr->xdp_prog);
2789 
2790 		if (xdp_rxq_info_is_reg(&rxr->xdp_rxq))
2791 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
2792 
2793 		page_pool_destroy(rxr->page_pool);
2794 		rxr->page_pool = NULL;
2795 
2796 		kfree(rxr->rx_agg_bmap);
2797 		rxr->rx_agg_bmap = NULL;
2798 
2799 		ring = &rxr->rx_ring_struct;
2800 		bnxt_free_ring(bp, &ring->ring_mem);
2801 
2802 		ring = &rxr->rx_agg_ring_struct;
2803 		bnxt_free_ring(bp, &ring->ring_mem);
2804 	}
2805 }
2806 
2807 static int bnxt_alloc_rx_page_pool(struct bnxt *bp,
2808 				   struct bnxt_rx_ring_info *rxr)
2809 {
2810 	struct page_pool_params pp = { 0 };
2811 
2812 	pp.pool_size = bp->rx_ring_size;
2813 	pp.nid = dev_to_node(&bp->pdev->dev);
2814 	pp.dev = &bp->pdev->dev;
2815 	pp.dma_dir = DMA_BIDIRECTIONAL;
2816 
2817 	rxr->page_pool = page_pool_create(&pp);
2818 	if (IS_ERR(rxr->page_pool)) {
2819 		int err = PTR_ERR(rxr->page_pool);
2820 
2821 		rxr->page_pool = NULL;
2822 		return err;
2823 	}
2824 	return 0;
2825 }
2826 
2827 static int bnxt_alloc_rx_rings(struct bnxt *bp)
2828 {
2829 	int i, rc = 0, agg_rings = 0;
2830 
2831 	if (!bp->rx_ring)
2832 		return -ENOMEM;
2833 
2834 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
2835 		agg_rings = 1;
2836 
2837 	for (i = 0; i < bp->rx_nr_rings; i++) {
2838 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
2839 		struct bnxt_ring_struct *ring;
2840 
2841 		ring = &rxr->rx_ring_struct;
2842 
2843 		rc = bnxt_alloc_rx_page_pool(bp, rxr);
2844 		if (rc)
2845 			return rc;
2846 
2847 		rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i);
2848 		if (rc < 0)
2849 			return rc;
2850 
2851 		rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq,
2852 						MEM_TYPE_PAGE_POOL,
2853 						rxr->page_pool);
2854 		if (rc) {
2855 			xdp_rxq_info_unreg(&rxr->xdp_rxq);
2856 			return rc;
2857 		}
2858 
2859 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2860 		if (rc)
2861 			return rc;
2862 
2863 		ring->grp_idx = i;
2864 		if (agg_rings) {
2865 			u16 mem_size;
2866 
2867 			ring = &rxr->rx_agg_ring_struct;
2868 			rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2869 			if (rc)
2870 				return rc;
2871 
2872 			ring->grp_idx = i;
2873 			rxr->rx_agg_bmap_size = bp->rx_agg_ring_mask + 1;
2874 			mem_size = rxr->rx_agg_bmap_size / 8;
2875 			rxr->rx_agg_bmap = kzalloc(mem_size, GFP_KERNEL);
2876 			if (!rxr->rx_agg_bmap)
2877 				return -ENOMEM;
2878 		}
2879 	}
2880 	if (bp->flags & BNXT_FLAG_TPA)
2881 		rc = bnxt_alloc_tpa_info(bp);
2882 	return rc;
2883 }
2884 
2885 static void bnxt_free_tx_rings(struct bnxt *bp)
2886 {
2887 	int i;
2888 	struct pci_dev *pdev = bp->pdev;
2889 
2890 	if (!bp->tx_ring)
2891 		return;
2892 
2893 	for (i = 0; i < bp->tx_nr_rings; i++) {
2894 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2895 		struct bnxt_ring_struct *ring;
2896 
2897 		if (txr->tx_push) {
2898 			dma_free_coherent(&pdev->dev, bp->tx_push_size,
2899 					  txr->tx_push, txr->tx_push_mapping);
2900 			txr->tx_push = NULL;
2901 		}
2902 
2903 		ring = &txr->tx_ring_struct;
2904 
2905 		bnxt_free_ring(bp, &ring->ring_mem);
2906 	}
2907 }
2908 
2909 static int bnxt_alloc_tx_rings(struct bnxt *bp)
2910 {
2911 	int i, j, rc;
2912 	struct pci_dev *pdev = bp->pdev;
2913 
2914 	bp->tx_push_size = 0;
2915 	if (bp->tx_push_thresh) {
2916 		int push_size;
2917 
2918 		push_size  = L1_CACHE_ALIGN(sizeof(struct tx_push_bd) +
2919 					bp->tx_push_thresh);
2920 
2921 		if (push_size > 256) {
2922 			push_size = 0;
2923 			bp->tx_push_thresh = 0;
2924 		}
2925 
2926 		bp->tx_push_size = push_size;
2927 	}
2928 
2929 	for (i = 0, j = 0; i < bp->tx_nr_rings; i++) {
2930 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
2931 		struct bnxt_ring_struct *ring;
2932 		u8 qidx;
2933 
2934 		ring = &txr->tx_ring_struct;
2935 
2936 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
2937 		if (rc)
2938 			return rc;
2939 
2940 		ring->grp_idx = txr->bnapi->index;
2941 		if (bp->tx_push_size) {
2942 			dma_addr_t mapping;
2943 
2944 			/* One pre-allocated DMA buffer to backup
2945 			 * TX push operation
2946 			 */
2947 			txr->tx_push = dma_alloc_coherent(&pdev->dev,
2948 						bp->tx_push_size,
2949 						&txr->tx_push_mapping,
2950 						GFP_KERNEL);
2951 
2952 			if (!txr->tx_push)
2953 				return -ENOMEM;
2954 
2955 			mapping = txr->tx_push_mapping +
2956 				sizeof(struct tx_push_bd);
2957 			txr->data_mapping = cpu_to_le64(mapping);
2958 		}
2959 		qidx = bp->tc_to_qidx[j];
2960 		ring->queue_id = bp->q_info[qidx].queue_id;
2961 		if (i < bp->tx_nr_rings_xdp)
2962 			continue;
2963 		if (i % bp->tx_nr_rings_per_tc == (bp->tx_nr_rings_per_tc - 1))
2964 			j++;
2965 	}
2966 	return 0;
2967 }
2968 
2969 static void bnxt_free_cp_rings(struct bnxt *bp)
2970 {
2971 	int i;
2972 
2973 	if (!bp->bnapi)
2974 		return;
2975 
2976 	for (i = 0; i < bp->cp_nr_rings; i++) {
2977 		struct bnxt_napi *bnapi = bp->bnapi[i];
2978 		struct bnxt_cp_ring_info *cpr;
2979 		struct bnxt_ring_struct *ring;
2980 		int j;
2981 
2982 		if (!bnapi)
2983 			continue;
2984 
2985 		cpr = &bnapi->cp_ring;
2986 		ring = &cpr->cp_ring_struct;
2987 
2988 		bnxt_free_ring(bp, &ring->ring_mem);
2989 
2990 		for (j = 0; j < 2; j++) {
2991 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
2992 
2993 			if (cpr2) {
2994 				ring = &cpr2->cp_ring_struct;
2995 				bnxt_free_ring(bp, &ring->ring_mem);
2996 				kfree(cpr2);
2997 				cpr->cp_ring_arr[j] = NULL;
2998 			}
2999 		}
3000 	}
3001 }
3002 
3003 static struct bnxt_cp_ring_info *bnxt_alloc_cp_sub_ring(struct bnxt *bp)
3004 {
3005 	struct bnxt_ring_mem_info *rmem;
3006 	struct bnxt_ring_struct *ring;
3007 	struct bnxt_cp_ring_info *cpr;
3008 	int rc;
3009 
3010 	cpr = kzalloc(sizeof(*cpr), GFP_KERNEL);
3011 	if (!cpr)
3012 		return NULL;
3013 
3014 	ring = &cpr->cp_ring_struct;
3015 	rmem = &ring->ring_mem;
3016 	rmem->nr_pages = bp->cp_nr_pages;
3017 	rmem->page_size = HW_CMPD_RING_SIZE;
3018 	rmem->pg_arr = (void **)cpr->cp_desc_ring;
3019 	rmem->dma_arr = cpr->cp_desc_mapping;
3020 	rmem->flags = BNXT_RMEM_RING_PTE_FLAG;
3021 	rc = bnxt_alloc_ring(bp, rmem);
3022 	if (rc) {
3023 		bnxt_free_ring(bp, rmem);
3024 		kfree(cpr);
3025 		cpr = NULL;
3026 	}
3027 	return cpr;
3028 }
3029 
3030 static int bnxt_alloc_cp_rings(struct bnxt *bp)
3031 {
3032 	bool sh = !!(bp->flags & BNXT_FLAG_SHARED_RINGS);
3033 	int i, rc, ulp_base_vec, ulp_msix;
3034 
3035 	ulp_msix = bnxt_get_ulp_msix_num(bp);
3036 	ulp_base_vec = bnxt_get_ulp_msix_base(bp);
3037 	for (i = 0; i < bp->cp_nr_rings; i++) {
3038 		struct bnxt_napi *bnapi = bp->bnapi[i];
3039 		struct bnxt_cp_ring_info *cpr;
3040 		struct bnxt_ring_struct *ring;
3041 
3042 		if (!bnapi)
3043 			continue;
3044 
3045 		cpr = &bnapi->cp_ring;
3046 		cpr->bnapi = bnapi;
3047 		ring = &cpr->cp_ring_struct;
3048 
3049 		rc = bnxt_alloc_ring(bp, &ring->ring_mem);
3050 		if (rc)
3051 			return rc;
3052 
3053 		if (ulp_msix && i >= ulp_base_vec)
3054 			ring->map_idx = i + ulp_msix;
3055 		else
3056 			ring->map_idx = i;
3057 
3058 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
3059 			continue;
3060 
3061 		if (i < bp->rx_nr_rings) {
3062 			struct bnxt_cp_ring_info *cpr2 =
3063 				bnxt_alloc_cp_sub_ring(bp);
3064 
3065 			cpr->cp_ring_arr[BNXT_RX_HDL] = cpr2;
3066 			if (!cpr2)
3067 				return -ENOMEM;
3068 			cpr2->bnapi = bnapi;
3069 		}
3070 		if ((sh && i < bp->tx_nr_rings) ||
3071 		    (!sh && i >= bp->rx_nr_rings)) {
3072 			struct bnxt_cp_ring_info *cpr2 =
3073 				bnxt_alloc_cp_sub_ring(bp);
3074 
3075 			cpr->cp_ring_arr[BNXT_TX_HDL] = cpr2;
3076 			if (!cpr2)
3077 				return -ENOMEM;
3078 			cpr2->bnapi = bnapi;
3079 		}
3080 	}
3081 	return 0;
3082 }
3083 
3084 static void bnxt_init_ring_struct(struct bnxt *bp)
3085 {
3086 	int i;
3087 
3088 	for (i = 0; i < bp->cp_nr_rings; i++) {
3089 		struct bnxt_napi *bnapi = bp->bnapi[i];
3090 		struct bnxt_ring_mem_info *rmem;
3091 		struct bnxt_cp_ring_info *cpr;
3092 		struct bnxt_rx_ring_info *rxr;
3093 		struct bnxt_tx_ring_info *txr;
3094 		struct bnxt_ring_struct *ring;
3095 
3096 		if (!bnapi)
3097 			continue;
3098 
3099 		cpr = &bnapi->cp_ring;
3100 		ring = &cpr->cp_ring_struct;
3101 		rmem = &ring->ring_mem;
3102 		rmem->nr_pages = bp->cp_nr_pages;
3103 		rmem->page_size = HW_CMPD_RING_SIZE;
3104 		rmem->pg_arr = (void **)cpr->cp_desc_ring;
3105 		rmem->dma_arr = cpr->cp_desc_mapping;
3106 		rmem->vmem_size = 0;
3107 
3108 		rxr = bnapi->rx_ring;
3109 		if (!rxr)
3110 			goto skip_rx;
3111 
3112 		ring = &rxr->rx_ring_struct;
3113 		rmem = &ring->ring_mem;
3114 		rmem->nr_pages = bp->rx_nr_pages;
3115 		rmem->page_size = HW_RXBD_RING_SIZE;
3116 		rmem->pg_arr = (void **)rxr->rx_desc_ring;
3117 		rmem->dma_arr = rxr->rx_desc_mapping;
3118 		rmem->vmem_size = SW_RXBD_RING_SIZE * bp->rx_nr_pages;
3119 		rmem->vmem = (void **)&rxr->rx_buf_ring;
3120 
3121 		ring = &rxr->rx_agg_ring_struct;
3122 		rmem = &ring->ring_mem;
3123 		rmem->nr_pages = bp->rx_agg_nr_pages;
3124 		rmem->page_size = HW_RXBD_RING_SIZE;
3125 		rmem->pg_arr = (void **)rxr->rx_agg_desc_ring;
3126 		rmem->dma_arr = rxr->rx_agg_desc_mapping;
3127 		rmem->vmem_size = SW_RXBD_AGG_RING_SIZE * bp->rx_agg_nr_pages;
3128 		rmem->vmem = (void **)&rxr->rx_agg_ring;
3129 
3130 skip_rx:
3131 		txr = bnapi->tx_ring;
3132 		if (!txr)
3133 			continue;
3134 
3135 		ring = &txr->tx_ring_struct;
3136 		rmem = &ring->ring_mem;
3137 		rmem->nr_pages = bp->tx_nr_pages;
3138 		rmem->page_size = HW_RXBD_RING_SIZE;
3139 		rmem->pg_arr = (void **)txr->tx_desc_ring;
3140 		rmem->dma_arr = txr->tx_desc_mapping;
3141 		rmem->vmem_size = SW_TXBD_RING_SIZE * bp->tx_nr_pages;
3142 		rmem->vmem = (void **)&txr->tx_buf_ring;
3143 	}
3144 }
3145 
3146 static void bnxt_init_rxbd_pages(struct bnxt_ring_struct *ring, u32 type)
3147 {
3148 	int i;
3149 	u32 prod;
3150 	struct rx_bd **rx_buf_ring;
3151 
3152 	rx_buf_ring = (struct rx_bd **)ring->ring_mem.pg_arr;
3153 	for (i = 0, prod = 0; i < ring->ring_mem.nr_pages; i++) {
3154 		int j;
3155 		struct rx_bd *rxbd;
3156 
3157 		rxbd = rx_buf_ring[i];
3158 		if (!rxbd)
3159 			continue;
3160 
3161 		for (j = 0; j < RX_DESC_CNT; j++, rxbd++, prod++) {
3162 			rxbd->rx_bd_len_flags_type = cpu_to_le32(type);
3163 			rxbd->rx_bd_opaque = prod;
3164 		}
3165 	}
3166 }
3167 
3168 static int bnxt_init_one_rx_ring(struct bnxt *bp, int ring_nr)
3169 {
3170 	struct net_device *dev = bp->dev;
3171 	struct bnxt_rx_ring_info *rxr;
3172 	struct bnxt_ring_struct *ring;
3173 	u32 prod, type;
3174 	int i;
3175 
3176 	type = (bp->rx_buf_use_size << RX_BD_LEN_SHIFT) |
3177 		RX_BD_TYPE_RX_PACKET_BD | RX_BD_FLAGS_EOP;
3178 
3179 	if (NET_IP_ALIGN == 2)
3180 		type |= RX_BD_FLAGS_SOP;
3181 
3182 	rxr = &bp->rx_ring[ring_nr];
3183 	ring = &rxr->rx_ring_struct;
3184 	bnxt_init_rxbd_pages(ring, type);
3185 
3186 	if (BNXT_RX_PAGE_MODE(bp) && bp->xdp_prog) {
3187 		bpf_prog_add(bp->xdp_prog, 1);
3188 		rxr->xdp_prog = bp->xdp_prog;
3189 	}
3190 	prod = rxr->rx_prod;
3191 	for (i = 0; i < bp->rx_ring_size; i++) {
3192 		if (bnxt_alloc_rx_data(bp, rxr, prod, GFP_KERNEL) != 0) {
3193 			netdev_warn(dev, "init'ed rx ring %d with %d/%d skbs only\n",
3194 				    ring_nr, i, bp->rx_ring_size);
3195 			break;
3196 		}
3197 		prod = NEXT_RX(prod);
3198 	}
3199 	rxr->rx_prod = prod;
3200 	ring->fw_ring_id = INVALID_HW_RING_ID;
3201 
3202 	ring = &rxr->rx_agg_ring_struct;
3203 	ring->fw_ring_id = INVALID_HW_RING_ID;
3204 
3205 	if (!(bp->flags & BNXT_FLAG_AGG_RINGS))
3206 		return 0;
3207 
3208 	type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) |
3209 		RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP;
3210 
3211 	bnxt_init_rxbd_pages(ring, type);
3212 
3213 	prod = rxr->rx_agg_prod;
3214 	for (i = 0; i < bp->rx_agg_ring_size; i++) {
3215 		if (bnxt_alloc_rx_page(bp, rxr, prod, GFP_KERNEL) != 0) {
3216 			netdev_warn(dev, "init'ed rx ring %d with %d/%d pages only\n",
3217 				    ring_nr, i, bp->rx_ring_size);
3218 			break;
3219 		}
3220 		prod = NEXT_RX_AGG(prod);
3221 	}
3222 	rxr->rx_agg_prod = prod;
3223 
3224 	if (bp->flags & BNXT_FLAG_TPA) {
3225 		if (rxr->rx_tpa) {
3226 			u8 *data;
3227 			dma_addr_t mapping;
3228 
3229 			for (i = 0; i < bp->max_tpa; i++) {
3230 				data = __bnxt_alloc_rx_data(bp, &mapping,
3231 							    GFP_KERNEL);
3232 				if (!data)
3233 					return -ENOMEM;
3234 
3235 				rxr->rx_tpa[i].data = data;
3236 				rxr->rx_tpa[i].data_ptr = data + bp->rx_offset;
3237 				rxr->rx_tpa[i].mapping = mapping;
3238 			}
3239 		} else {
3240 			netdev_err(bp->dev, "No resource allocated for LRO/GRO\n");
3241 			return -ENOMEM;
3242 		}
3243 	}
3244 
3245 	return 0;
3246 }
3247 
3248 static void bnxt_init_cp_rings(struct bnxt *bp)
3249 {
3250 	int i, j;
3251 
3252 	for (i = 0; i < bp->cp_nr_rings; i++) {
3253 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
3254 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
3255 
3256 		ring->fw_ring_id = INVALID_HW_RING_ID;
3257 		cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3258 		cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3259 		for (j = 0; j < 2; j++) {
3260 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
3261 
3262 			if (!cpr2)
3263 				continue;
3264 
3265 			ring = &cpr2->cp_ring_struct;
3266 			ring->fw_ring_id = INVALID_HW_RING_ID;
3267 			cpr2->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks;
3268 			cpr2->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs;
3269 		}
3270 	}
3271 }
3272 
3273 static int bnxt_init_rx_rings(struct bnxt *bp)
3274 {
3275 	int i, rc = 0;
3276 
3277 	if (BNXT_RX_PAGE_MODE(bp)) {
3278 		bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM;
3279 		bp->rx_dma_offset = XDP_PACKET_HEADROOM;
3280 	} else {
3281 		bp->rx_offset = BNXT_RX_OFFSET;
3282 		bp->rx_dma_offset = BNXT_RX_DMA_OFFSET;
3283 	}
3284 
3285 	for (i = 0; i < bp->rx_nr_rings; i++) {
3286 		rc = bnxt_init_one_rx_ring(bp, i);
3287 		if (rc)
3288 			break;
3289 	}
3290 
3291 	return rc;
3292 }
3293 
3294 static int bnxt_init_tx_rings(struct bnxt *bp)
3295 {
3296 	u16 i;
3297 
3298 	bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2,
3299 				   MAX_SKB_FRAGS + 1);
3300 
3301 	for (i = 0; i < bp->tx_nr_rings; i++) {
3302 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
3303 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
3304 
3305 		ring->fw_ring_id = INVALID_HW_RING_ID;
3306 	}
3307 
3308 	return 0;
3309 }
3310 
3311 static void bnxt_free_ring_grps(struct bnxt *bp)
3312 {
3313 	kfree(bp->grp_info);
3314 	bp->grp_info = NULL;
3315 }
3316 
3317 static int bnxt_init_ring_grps(struct bnxt *bp, bool irq_re_init)
3318 {
3319 	int i;
3320 
3321 	if (irq_re_init) {
3322 		bp->grp_info = kcalloc(bp->cp_nr_rings,
3323 				       sizeof(struct bnxt_ring_grp_info),
3324 				       GFP_KERNEL);
3325 		if (!bp->grp_info)
3326 			return -ENOMEM;
3327 	}
3328 	for (i = 0; i < bp->cp_nr_rings; i++) {
3329 		if (irq_re_init)
3330 			bp->grp_info[i].fw_stats_ctx = INVALID_HW_RING_ID;
3331 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
3332 		bp->grp_info[i].rx_fw_ring_id = INVALID_HW_RING_ID;
3333 		bp->grp_info[i].agg_fw_ring_id = INVALID_HW_RING_ID;
3334 		bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
3335 	}
3336 	return 0;
3337 }
3338 
3339 static void bnxt_free_vnics(struct bnxt *bp)
3340 {
3341 	kfree(bp->vnic_info);
3342 	bp->vnic_info = NULL;
3343 	bp->nr_vnics = 0;
3344 }
3345 
3346 static int bnxt_alloc_vnics(struct bnxt *bp)
3347 {
3348 	int num_vnics = 1;
3349 
3350 #ifdef CONFIG_RFS_ACCEL
3351 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
3352 		num_vnics += bp->rx_nr_rings;
3353 #endif
3354 
3355 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
3356 		num_vnics++;
3357 
3358 	bp->vnic_info = kcalloc(num_vnics, sizeof(struct bnxt_vnic_info),
3359 				GFP_KERNEL);
3360 	if (!bp->vnic_info)
3361 		return -ENOMEM;
3362 
3363 	bp->nr_vnics = num_vnics;
3364 	return 0;
3365 }
3366 
3367 static void bnxt_init_vnics(struct bnxt *bp)
3368 {
3369 	int i;
3370 
3371 	for (i = 0; i < bp->nr_vnics; i++) {
3372 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3373 		int j;
3374 
3375 		vnic->fw_vnic_id = INVALID_HW_RING_ID;
3376 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++)
3377 			vnic->fw_rss_cos_lb_ctx[j] = INVALID_HW_RING_ID;
3378 
3379 		vnic->fw_l2_ctx_id = INVALID_HW_RING_ID;
3380 
3381 		if (bp->vnic_info[i].rss_hash_key) {
3382 			if (i == 0)
3383 				prandom_bytes(vnic->rss_hash_key,
3384 					      HW_HASH_KEY_SIZE);
3385 			else
3386 				memcpy(vnic->rss_hash_key,
3387 				       bp->vnic_info[0].rss_hash_key,
3388 				       HW_HASH_KEY_SIZE);
3389 		}
3390 	}
3391 }
3392 
3393 static int bnxt_calc_nr_ring_pages(u32 ring_size, int desc_per_pg)
3394 {
3395 	int pages;
3396 
3397 	pages = ring_size / desc_per_pg;
3398 
3399 	if (!pages)
3400 		return 1;
3401 
3402 	pages++;
3403 
3404 	while (pages & (pages - 1))
3405 		pages++;
3406 
3407 	return pages;
3408 }
3409 
3410 void bnxt_set_tpa_flags(struct bnxt *bp)
3411 {
3412 	bp->flags &= ~BNXT_FLAG_TPA;
3413 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
3414 		return;
3415 	if (bp->dev->features & NETIF_F_LRO)
3416 		bp->flags |= BNXT_FLAG_LRO;
3417 	else if (bp->dev->features & NETIF_F_GRO_HW)
3418 		bp->flags |= BNXT_FLAG_GRO;
3419 }
3420 
3421 /* bp->rx_ring_size, bp->tx_ring_size, dev->mtu, BNXT_FLAG_{G|L}RO flags must
3422  * be set on entry.
3423  */
3424 void bnxt_set_ring_params(struct bnxt *bp)
3425 {
3426 	u32 ring_size, rx_size, rx_space, max_rx_cmpl;
3427 	u32 agg_factor = 0, agg_ring_size = 0;
3428 
3429 	/* 8 for CRC and VLAN */
3430 	rx_size = SKB_DATA_ALIGN(bp->dev->mtu + ETH_HLEN + NET_IP_ALIGN + 8);
3431 
3432 	rx_space = rx_size + NET_SKB_PAD +
3433 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3434 
3435 	bp->rx_copy_thresh = BNXT_RX_COPY_THRESH;
3436 	ring_size = bp->rx_ring_size;
3437 	bp->rx_agg_ring_size = 0;
3438 	bp->rx_agg_nr_pages = 0;
3439 
3440 	if (bp->flags & BNXT_FLAG_TPA)
3441 		agg_factor = min_t(u32, 4, 65536 / BNXT_RX_PAGE_SIZE);
3442 
3443 	bp->flags &= ~BNXT_FLAG_JUMBO;
3444 	if (rx_space > PAGE_SIZE && !(bp->flags & BNXT_FLAG_NO_AGG_RINGS)) {
3445 		u32 jumbo_factor;
3446 
3447 		bp->flags |= BNXT_FLAG_JUMBO;
3448 		jumbo_factor = PAGE_ALIGN(bp->dev->mtu - 40) >> PAGE_SHIFT;
3449 		if (jumbo_factor > agg_factor)
3450 			agg_factor = jumbo_factor;
3451 	}
3452 	agg_ring_size = ring_size * agg_factor;
3453 
3454 	if (agg_ring_size) {
3455 		bp->rx_agg_nr_pages = bnxt_calc_nr_ring_pages(agg_ring_size,
3456 							RX_DESC_CNT);
3457 		if (bp->rx_agg_nr_pages > MAX_RX_AGG_PAGES) {
3458 			u32 tmp = agg_ring_size;
3459 
3460 			bp->rx_agg_nr_pages = MAX_RX_AGG_PAGES;
3461 			agg_ring_size = MAX_RX_AGG_PAGES * RX_DESC_CNT - 1;
3462 			netdev_warn(bp->dev, "rx agg ring size %d reduced to %d.\n",
3463 				    tmp, agg_ring_size);
3464 		}
3465 		bp->rx_agg_ring_size = agg_ring_size;
3466 		bp->rx_agg_ring_mask = (bp->rx_agg_nr_pages * RX_DESC_CNT) - 1;
3467 		rx_size = SKB_DATA_ALIGN(BNXT_RX_COPY_THRESH + NET_IP_ALIGN);
3468 		rx_space = rx_size + NET_SKB_PAD +
3469 			SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3470 	}
3471 
3472 	bp->rx_buf_use_size = rx_size;
3473 	bp->rx_buf_size = rx_space;
3474 
3475 	bp->rx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, RX_DESC_CNT);
3476 	bp->rx_ring_mask = (bp->rx_nr_pages * RX_DESC_CNT) - 1;
3477 
3478 	ring_size = bp->tx_ring_size;
3479 	bp->tx_nr_pages = bnxt_calc_nr_ring_pages(ring_size, TX_DESC_CNT);
3480 	bp->tx_ring_mask = (bp->tx_nr_pages * TX_DESC_CNT) - 1;
3481 
3482 	max_rx_cmpl = bp->rx_ring_size;
3483 	/* MAX TPA needs to be added because TPA_START completions are
3484 	 * immediately recycled, so the TPA completions are not bound by
3485 	 * the RX ring size.
3486 	 */
3487 	if (bp->flags & BNXT_FLAG_TPA)
3488 		max_rx_cmpl += bp->max_tpa;
3489 	/* RX and TPA completions are 32-byte, all others are 16-byte */
3490 	ring_size = max_rx_cmpl * 2 + agg_ring_size + bp->tx_ring_size;
3491 	bp->cp_ring_size = ring_size;
3492 
3493 	bp->cp_nr_pages = bnxt_calc_nr_ring_pages(ring_size, CP_DESC_CNT);
3494 	if (bp->cp_nr_pages > MAX_CP_PAGES) {
3495 		bp->cp_nr_pages = MAX_CP_PAGES;
3496 		bp->cp_ring_size = MAX_CP_PAGES * CP_DESC_CNT - 1;
3497 		netdev_warn(bp->dev, "completion ring size %d reduced to %d.\n",
3498 			    ring_size, bp->cp_ring_size);
3499 	}
3500 	bp->cp_bit = bp->cp_nr_pages * CP_DESC_CNT;
3501 	bp->cp_ring_mask = bp->cp_bit - 1;
3502 }
3503 
3504 /* Changing allocation mode of RX rings.
3505  * TODO: Update when extending xdp_rxq_info to support allocation modes.
3506  */
3507 int bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode)
3508 {
3509 	if (page_mode) {
3510 		if (bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU)
3511 			return -EOPNOTSUPP;
3512 		bp->dev->max_mtu =
3513 			min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU);
3514 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
3515 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS | BNXT_FLAG_RX_PAGE_MODE;
3516 		bp->rx_dir = DMA_BIDIRECTIONAL;
3517 		bp->rx_skb_func = bnxt_rx_page_skb;
3518 		/* Disable LRO or GRO_HW */
3519 		netdev_update_features(bp->dev);
3520 	} else {
3521 		bp->dev->max_mtu = bp->max_mtu;
3522 		bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE;
3523 		bp->rx_dir = DMA_FROM_DEVICE;
3524 		bp->rx_skb_func = bnxt_rx_skb;
3525 	}
3526 	return 0;
3527 }
3528 
3529 static void bnxt_free_vnic_attributes(struct bnxt *bp)
3530 {
3531 	int i;
3532 	struct bnxt_vnic_info *vnic;
3533 	struct pci_dev *pdev = bp->pdev;
3534 
3535 	if (!bp->vnic_info)
3536 		return;
3537 
3538 	for (i = 0; i < bp->nr_vnics; i++) {
3539 		vnic = &bp->vnic_info[i];
3540 
3541 		kfree(vnic->fw_grp_ids);
3542 		vnic->fw_grp_ids = NULL;
3543 
3544 		kfree(vnic->uc_list);
3545 		vnic->uc_list = NULL;
3546 
3547 		if (vnic->mc_list) {
3548 			dma_free_coherent(&pdev->dev, vnic->mc_list_size,
3549 					  vnic->mc_list, vnic->mc_list_mapping);
3550 			vnic->mc_list = NULL;
3551 		}
3552 
3553 		if (vnic->rss_table) {
3554 			dma_free_coherent(&pdev->dev, vnic->rss_table_size,
3555 					  vnic->rss_table,
3556 					  vnic->rss_table_dma_addr);
3557 			vnic->rss_table = NULL;
3558 		}
3559 
3560 		vnic->rss_hash_key = NULL;
3561 		vnic->flags = 0;
3562 	}
3563 }
3564 
3565 static int bnxt_alloc_vnic_attributes(struct bnxt *bp)
3566 {
3567 	int i, rc = 0, size;
3568 	struct bnxt_vnic_info *vnic;
3569 	struct pci_dev *pdev = bp->pdev;
3570 	int max_rings;
3571 
3572 	for (i = 0; i < bp->nr_vnics; i++) {
3573 		vnic = &bp->vnic_info[i];
3574 
3575 		if (vnic->flags & BNXT_VNIC_UCAST_FLAG) {
3576 			int mem_size = (BNXT_MAX_UC_ADDRS - 1) * ETH_ALEN;
3577 
3578 			if (mem_size > 0) {
3579 				vnic->uc_list = kmalloc(mem_size, GFP_KERNEL);
3580 				if (!vnic->uc_list) {
3581 					rc = -ENOMEM;
3582 					goto out;
3583 				}
3584 			}
3585 		}
3586 
3587 		if (vnic->flags & BNXT_VNIC_MCAST_FLAG) {
3588 			vnic->mc_list_size = BNXT_MAX_MC_ADDRS * ETH_ALEN;
3589 			vnic->mc_list =
3590 				dma_alloc_coherent(&pdev->dev,
3591 						   vnic->mc_list_size,
3592 						   &vnic->mc_list_mapping,
3593 						   GFP_KERNEL);
3594 			if (!vnic->mc_list) {
3595 				rc = -ENOMEM;
3596 				goto out;
3597 			}
3598 		}
3599 
3600 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3601 			goto vnic_skip_grps;
3602 
3603 		if (vnic->flags & BNXT_VNIC_RSS_FLAG)
3604 			max_rings = bp->rx_nr_rings;
3605 		else
3606 			max_rings = 1;
3607 
3608 		vnic->fw_grp_ids = kcalloc(max_rings, sizeof(u16), GFP_KERNEL);
3609 		if (!vnic->fw_grp_ids) {
3610 			rc = -ENOMEM;
3611 			goto out;
3612 		}
3613 vnic_skip_grps:
3614 		if ((bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
3615 		    !(vnic->flags & BNXT_VNIC_RSS_FLAG))
3616 			continue;
3617 
3618 		/* Allocate rss table and hash key */
3619 		size = L1_CACHE_ALIGN(HW_HASH_INDEX_SIZE * sizeof(u16));
3620 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3621 			size = L1_CACHE_ALIGN(BNXT_MAX_RSS_TABLE_SIZE_P5);
3622 
3623 		vnic->rss_table_size = size + HW_HASH_KEY_SIZE;
3624 		vnic->rss_table = dma_alloc_coherent(&pdev->dev,
3625 						     vnic->rss_table_size,
3626 						     &vnic->rss_table_dma_addr,
3627 						     GFP_KERNEL);
3628 		if (!vnic->rss_table) {
3629 			rc = -ENOMEM;
3630 			goto out;
3631 		}
3632 
3633 		vnic->rss_hash_key = ((void *)vnic->rss_table) + size;
3634 		vnic->rss_hash_key_dma_addr = vnic->rss_table_dma_addr + size;
3635 	}
3636 	return 0;
3637 
3638 out:
3639 	return rc;
3640 }
3641 
3642 static void bnxt_free_hwrm_resources(struct bnxt *bp)
3643 {
3644 	struct pci_dev *pdev = bp->pdev;
3645 
3646 	if (bp->hwrm_cmd_resp_addr) {
3647 		dma_free_coherent(&pdev->dev, PAGE_SIZE, bp->hwrm_cmd_resp_addr,
3648 				  bp->hwrm_cmd_resp_dma_addr);
3649 		bp->hwrm_cmd_resp_addr = NULL;
3650 	}
3651 
3652 	if (bp->hwrm_cmd_kong_resp_addr) {
3653 		dma_free_coherent(&pdev->dev, PAGE_SIZE,
3654 				  bp->hwrm_cmd_kong_resp_addr,
3655 				  bp->hwrm_cmd_kong_resp_dma_addr);
3656 		bp->hwrm_cmd_kong_resp_addr = NULL;
3657 	}
3658 }
3659 
3660 static int bnxt_alloc_kong_hwrm_resources(struct bnxt *bp)
3661 {
3662 	struct pci_dev *pdev = bp->pdev;
3663 
3664 	if (bp->hwrm_cmd_kong_resp_addr)
3665 		return 0;
3666 
3667 	bp->hwrm_cmd_kong_resp_addr =
3668 		dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3669 				   &bp->hwrm_cmd_kong_resp_dma_addr,
3670 				   GFP_KERNEL);
3671 	if (!bp->hwrm_cmd_kong_resp_addr)
3672 		return -ENOMEM;
3673 
3674 	return 0;
3675 }
3676 
3677 static int bnxt_alloc_hwrm_resources(struct bnxt *bp)
3678 {
3679 	struct pci_dev *pdev = bp->pdev;
3680 
3681 	bp->hwrm_cmd_resp_addr = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
3682 						   &bp->hwrm_cmd_resp_dma_addr,
3683 						   GFP_KERNEL);
3684 	if (!bp->hwrm_cmd_resp_addr)
3685 		return -ENOMEM;
3686 
3687 	return 0;
3688 }
3689 
3690 static void bnxt_free_hwrm_short_cmd_req(struct bnxt *bp)
3691 {
3692 	if (bp->hwrm_short_cmd_req_addr) {
3693 		struct pci_dev *pdev = bp->pdev;
3694 
3695 		dma_free_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3696 				  bp->hwrm_short_cmd_req_addr,
3697 				  bp->hwrm_short_cmd_req_dma_addr);
3698 		bp->hwrm_short_cmd_req_addr = NULL;
3699 	}
3700 }
3701 
3702 static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt *bp)
3703 {
3704 	struct pci_dev *pdev = bp->pdev;
3705 
3706 	if (bp->hwrm_short_cmd_req_addr)
3707 		return 0;
3708 
3709 	bp->hwrm_short_cmd_req_addr =
3710 		dma_alloc_coherent(&pdev->dev, bp->hwrm_max_ext_req_len,
3711 				   &bp->hwrm_short_cmd_req_dma_addr,
3712 				   GFP_KERNEL);
3713 	if (!bp->hwrm_short_cmd_req_addr)
3714 		return -ENOMEM;
3715 
3716 	return 0;
3717 }
3718 
3719 static void bnxt_free_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats)
3720 {
3721 	kfree(stats->hw_masks);
3722 	stats->hw_masks = NULL;
3723 	kfree(stats->sw_stats);
3724 	stats->sw_stats = NULL;
3725 	if (stats->hw_stats) {
3726 		dma_free_coherent(&bp->pdev->dev, stats->len, stats->hw_stats,
3727 				  stats->hw_stats_map);
3728 		stats->hw_stats = NULL;
3729 	}
3730 }
3731 
3732 static int bnxt_alloc_stats_mem(struct bnxt *bp, struct bnxt_stats_mem *stats,
3733 				bool alloc_masks)
3734 {
3735 	stats->hw_stats = dma_alloc_coherent(&bp->pdev->dev, stats->len,
3736 					     &stats->hw_stats_map, GFP_KERNEL);
3737 	if (!stats->hw_stats)
3738 		return -ENOMEM;
3739 
3740 	stats->sw_stats = kzalloc(stats->len, GFP_KERNEL);
3741 	if (!stats->sw_stats)
3742 		goto stats_mem_err;
3743 
3744 	if (alloc_masks) {
3745 		stats->hw_masks = kzalloc(stats->len, GFP_KERNEL);
3746 		if (!stats->hw_masks)
3747 			goto stats_mem_err;
3748 	}
3749 	return 0;
3750 
3751 stats_mem_err:
3752 	bnxt_free_stats_mem(bp, stats);
3753 	return -ENOMEM;
3754 }
3755 
3756 static void bnxt_fill_masks(u64 *mask_arr, u64 mask, int count)
3757 {
3758 	int i;
3759 
3760 	for (i = 0; i < count; i++)
3761 		mask_arr[i] = mask;
3762 }
3763 
3764 static void bnxt_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count)
3765 {
3766 	int i;
3767 
3768 	for (i = 0; i < count; i++)
3769 		mask_arr[i] = le64_to_cpu(hw_mask_arr[i]);
3770 }
3771 
3772 static int bnxt_hwrm_func_qstat_ext(struct bnxt *bp,
3773 				    struct bnxt_stats_mem *stats)
3774 {
3775 	struct hwrm_func_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
3776 	struct hwrm_func_qstats_ext_input req = {0};
3777 	__le64 *hw_masks;
3778 	int rc;
3779 
3780 	if (!(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED) ||
3781 	    !(bp->flags & BNXT_FLAG_CHIP_P5))
3782 		return -EOPNOTSUPP;
3783 
3784 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QSTATS_EXT, -1, -1);
3785 	req.flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3786 	mutex_lock(&bp->hwrm_cmd_lock);
3787 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
3788 	if (rc)
3789 		goto qstat_exit;
3790 
3791 	hw_masks = &resp->rx_ucast_pkts;
3792 	bnxt_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8);
3793 
3794 qstat_exit:
3795 	mutex_unlock(&bp->hwrm_cmd_lock);
3796 	return rc;
3797 }
3798 
3799 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags);
3800 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags);
3801 
3802 static void bnxt_init_stats(struct bnxt *bp)
3803 {
3804 	struct bnxt_napi *bnapi = bp->bnapi[0];
3805 	struct bnxt_cp_ring_info *cpr;
3806 	struct bnxt_stats_mem *stats;
3807 	__le64 *rx_stats, *tx_stats;
3808 	int rc, rx_count, tx_count;
3809 	u64 *rx_masks, *tx_masks;
3810 	u64 mask;
3811 	u8 flags;
3812 
3813 	cpr = &bnapi->cp_ring;
3814 	stats = &cpr->stats;
3815 	rc = bnxt_hwrm_func_qstat_ext(bp, stats);
3816 	if (rc) {
3817 		if (bp->flags & BNXT_FLAG_CHIP_P5)
3818 			mask = (1ULL << 48) - 1;
3819 		else
3820 			mask = -1ULL;
3821 		bnxt_fill_masks(stats->hw_masks, mask, stats->len / 8);
3822 	}
3823 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
3824 		stats = &bp->port_stats;
3825 		rx_stats = stats->hw_stats;
3826 		rx_masks = stats->hw_masks;
3827 		rx_count = sizeof(struct rx_port_stats) / 8;
3828 		tx_stats = rx_stats + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3829 		tx_masks = rx_masks + BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
3830 		tx_count = sizeof(struct tx_port_stats) / 8;
3831 
3832 		flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK;
3833 		rc = bnxt_hwrm_port_qstats(bp, flags);
3834 		if (rc) {
3835 			mask = (1ULL << 40) - 1;
3836 
3837 			bnxt_fill_masks(rx_masks, mask, rx_count);
3838 			bnxt_fill_masks(tx_masks, mask, tx_count);
3839 		} else {
3840 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3841 			bnxt_copy_hw_masks(tx_masks, tx_stats, tx_count);
3842 			bnxt_hwrm_port_qstats(bp, 0);
3843 		}
3844 	}
3845 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
3846 		stats = &bp->rx_port_stats_ext;
3847 		rx_stats = stats->hw_stats;
3848 		rx_masks = stats->hw_masks;
3849 		rx_count = sizeof(struct rx_port_stats_ext) / 8;
3850 		stats = &bp->tx_port_stats_ext;
3851 		tx_stats = stats->hw_stats;
3852 		tx_masks = stats->hw_masks;
3853 		tx_count = sizeof(struct tx_port_stats_ext) / 8;
3854 
3855 		flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK;
3856 		rc = bnxt_hwrm_port_qstats_ext(bp, flags);
3857 		if (rc) {
3858 			mask = (1ULL << 40) - 1;
3859 
3860 			bnxt_fill_masks(rx_masks, mask, rx_count);
3861 			if (tx_stats)
3862 				bnxt_fill_masks(tx_masks, mask, tx_count);
3863 		} else {
3864 			bnxt_copy_hw_masks(rx_masks, rx_stats, rx_count);
3865 			if (tx_stats)
3866 				bnxt_copy_hw_masks(tx_masks, tx_stats,
3867 						   tx_count);
3868 			bnxt_hwrm_port_qstats_ext(bp, 0);
3869 		}
3870 	}
3871 }
3872 
3873 static void bnxt_free_port_stats(struct bnxt *bp)
3874 {
3875 	bp->flags &= ~BNXT_FLAG_PORT_STATS;
3876 	bp->flags &= ~BNXT_FLAG_PORT_STATS_EXT;
3877 
3878 	bnxt_free_stats_mem(bp, &bp->port_stats);
3879 	bnxt_free_stats_mem(bp, &bp->rx_port_stats_ext);
3880 	bnxt_free_stats_mem(bp, &bp->tx_port_stats_ext);
3881 }
3882 
3883 static void bnxt_free_ring_stats(struct bnxt *bp)
3884 {
3885 	int i;
3886 
3887 	if (!bp->bnapi)
3888 		return;
3889 
3890 	for (i = 0; i < bp->cp_nr_rings; i++) {
3891 		struct bnxt_napi *bnapi = bp->bnapi[i];
3892 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3893 
3894 		bnxt_free_stats_mem(bp, &cpr->stats);
3895 	}
3896 }
3897 
3898 static int bnxt_alloc_stats(struct bnxt *bp)
3899 {
3900 	u32 size, i;
3901 	int rc;
3902 
3903 	size = bp->hw_ring_stats_size;
3904 
3905 	for (i = 0; i < bp->cp_nr_rings; i++) {
3906 		struct bnxt_napi *bnapi = bp->bnapi[i];
3907 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
3908 
3909 		cpr->stats.len = size;
3910 		rc = bnxt_alloc_stats_mem(bp, &cpr->stats, !i);
3911 		if (rc)
3912 			return rc;
3913 
3914 		cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
3915 	}
3916 
3917 	if (BNXT_VF(bp) || bp->chip_num == CHIP_NUM_58700)
3918 		return 0;
3919 
3920 	if (bp->port_stats.hw_stats)
3921 		goto alloc_ext_stats;
3922 
3923 	bp->port_stats.len = BNXT_PORT_STATS_SIZE;
3924 	rc = bnxt_alloc_stats_mem(bp, &bp->port_stats, true);
3925 	if (rc)
3926 		return rc;
3927 
3928 	bp->flags |= BNXT_FLAG_PORT_STATS;
3929 
3930 alloc_ext_stats:
3931 	/* Display extended statistics only if FW supports it */
3932 	if (bp->hwrm_spec_code < 0x10804 || bp->hwrm_spec_code == 0x10900)
3933 		if (!(bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED))
3934 			return 0;
3935 
3936 	if (bp->rx_port_stats_ext.hw_stats)
3937 		goto alloc_tx_ext_stats;
3938 
3939 	bp->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext);
3940 	rc = bnxt_alloc_stats_mem(bp, &bp->rx_port_stats_ext, true);
3941 	/* Extended stats are optional */
3942 	if (rc)
3943 		return 0;
3944 
3945 alloc_tx_ext_stats:
3946 	if (bp->tx_port_stats_ext.hw_stats)
3947 		return 0;
3948 
3949 	if (bp->hwrm_spec_code >= 0x10902 ||
3950 	    (bp->fw_cap & BNXT_FW_CAP_EXT_STATS_SUPPORTED)) {
3951 		bp->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext);
3952 		rc = bnxt_alloc_stats_mem(bp, &bp->tx_port_stats_ext, true);
3953 		/* Extended stats are optional */
3954 		if (rc)
3955 			return 0;
3956 	}
3957 	bp->flags |= BNXT_FLAG_PORT_STATS_EXT;
3958 	return 0;
3959 }
3960 
3961 static void bnxt_clear_ring_indices(struct bnxt *bp)
3962 {
3963 	int i;
3964 
3965 	if (!bp->bnapi)
3966 		return;
3967 
3968 	for (i = 0; i < bp->cp_nr_rings; i++) {
3969 		struct bnxt_napi *bnapi = bp->bnapi[i];
3970 		struct bnxt_cp_ring_info *cpr;
3971 		struct bnxt_rx_ring_info *rxr;
3972 		struct bnxt_tx_ring_info *txr;
3973 
3974 		if (!bnapi)
3975 			continue;
3976 
3977 		cpr = &bnapi->cp_ring;
3978 		cpr->cp_raw_cons = 0;
3979 
3980 		txr = bnapi->tx_ring;
3981 		if (txr) {
3982 			txr->tx_prod = 0;
3983 			txr->tx_cons = 0;
3984 		}
3985 
3986 		rxr = bnapi->rx_ring;
3987 		if (rxr) {
3988 			rxr->rx_prod = 0;
3989 			rxr->rx_agg_prod = 0;
3990 			rxr->rx_sw_agg_prod = 0;
3991 			rxr->rx_next_cons = 0;
3992 		}
3993 	}
3994 }
3995 
3996 static void bnxt_free_ntp_fltrs(struct bnxt *bp, bool irq_reinit)
3997 {
3998 #ifdef CONFIG_RFS_ACCEL
3999 	int i;
4000 
4001 	/* Under rtnl_lock and all our NAPIs have been disabled.  It's
4002 	 * safe to delete the hash table.
4003 	 */
4004 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
4005 		struct hlist_head *head;
4006 		struct hlist_node *tmp;
4007 		struct bnxt_ntuple_filter *fltr;
4008 
4009 		head = &bp->ntp_fltr_hash_tbl[i];
4010 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
4011 			hlist_del(&fltr->hash);
4012 			kfree(fltr);
4013 		}
4014 	}
4015 	if (irq_reinit) {
4016 		kfree(bp->ntp_fltr_bmap);
4017 		bp->ntp_fltr_bmap = NULL;
4018 	}
4019 	bp->ntp_fltr_count = 0;
4020 #endif
4021 }
4022 
4023 static int bnxt_alloc_ntp_fltrs(struct bnxt *bp)
4024 {
4025 #ifdef CONFIG_RFS_ACCEL
4026 	int i, rc = 0;
4027 
4028 	if (!(bp->flags & BNXT_FLAG_RFS))
4029 		return 0;
4030 
4031 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++)
4032 		INIT_HLIST_HEAD(&bp->ntp_fltr_hash_tbl[i]);
4033 
4034 	bp->ntp_fltr_count = 0;
4035 	bp->ntp_fltr_bmap = kcalloc(BITS_TO_LONGS(BNXT_NTP_FLTR_MAX_FLTR),
4036 				    sizeof(long),
4037 				    GFP_KERNEL);
4038 
4039 	if (!bp->ntp_fltr_bmap)
4040 		rc = -ENOMEM;
4041 
4042 	return rc;
4043 #else
4044 	return 0;
4045 #endif
4046 }
4047 
4048 static void bnxt_free_mem(struct bnxt *bp, bool irq_re_init)
4049 {
4050 	bnxt_free_vnic_attributes(bp);
4051 	bnxt_free_tx_rings(bp);
4052 	bnxt_free_rx_rings(bp);
4053 	bnxt_free_cp_rings(bp);
4054 	bnxt_free_ntp_fltrs(bp, irq_re_init);
4055 	if (irq_re_init) {
4056 		bnxt_free_ring_stats(bp);
4057 		if (!(bp->fw_cap & BNXT_FW_CAP_PORT_STATS_NO_RESET))
4058 			bnxt_free_port_stats(bp);
4059 		bnxt_free_ring_grps(bp);
4060 		bnxt_free_vnics(bp);
4061 		kfree(bp->tx_ring_map);
4062 		bp->tx_ring_map = NULL;
4063 		kfree(bp->tx_ring);
4064 		bp->tx_ring = NULL;
4065 		kfree(bp->rx_ring);
4066 		bp->rx_ring = NULL;
4067 		kfree(bp->bnapi);
4068 		bp->bnapi = NULL;
4069 	} else {
4070 		bnxt_clear_ring_indices(bp);
4071 	}
4072 }
4073 
4074 static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init)
4075 {
4076 	int i, j, rc, size, arr_size;
4077 	void *bnapi;
4078 
4079 	if (irq_re_init) {
4080 		/* Allocate bnapi mem pointer array and mem block for
4081 		 * all queues
4082 		 */
4083 		arr_size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi *) *
4084 				bp->cp_nr_rings);
4085 		size = L1_CACHE_ALIGN(sizeof(struct bnxt_napi));
4086 		bnapi = kzalloc(arr_size + size * bp->cp_nr_rings, GFP_KERNEL);
4087 		if (!bnapi)
4088 			return -ENOMEM;
4089 
4090 		bp->bnapi = bnapi;
4091 		bnapi += arr_size;
4092 		for (i = 0; i < bp->cp_nr_rings; i++, bnapi += size) {
4093 			bp->bnapi[i] = bnapi;
4094 			bp->bnapi[i]->index = i;
4095 			bp->bnapi[i]->bp = bp;
4096 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4097 				struct bnxt_cp_ring_info *cpr =
4098 					&bp->bnapi[i]->cp_ring;
4099 
4100 				cpr->cp_ring_struct.ring_mem.flags =
4101 					BNXT_RMEM_RING_PTE_FLAG;
4102 			}
4103 		}
4104 
4105 		bp->rx_ring = kcalloc(bp->rx_nr_rings,
4106 				      sizeof(struct bnxt_rx_ring_info),
4107 				      GFP_KERNEL);
4108 		if (!bp->rx_ring)
4109 			return -ENOMEM;
4110 
4111 		for (i = 0; i < bp->rx_nr_rings; i++) {
4112 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
4113 
4114 			if (bp->flags & BNXT_FLAG_CHIP_P5) {
4115 				rxr->rx_ring_struct.ring_mem.flags =
4116 					BNXT_RMEM_RING_PTE_FLAG;
4117 				rxr->rx_agg_ring_struct.ring_mem.flags =
4118 					BNXT_RMEM_RING_PTE_FLAG;
4119 			}
4120 			rxr->bnapi = bp->bnapi[i];
4121 			bp->bnapi[i]->rx_ring = &bp->rx_ring[i];
4122 		}
4123 
4124 		bp->tx_ring = kcalloc(bp->tx_nr_rings,
4125 				      sizeof(struct bnxt_tx_ring_info),
4126 				      GFP_KERNEL);
4127 		if (!bp->tx_ring)
4128 			return -ENOMEM;
4129 
4130 		bp->tx_ring_map = kcalloc(bp->tx_nr_rings, sizeof(u16),
4131 					  GFP_KERNEL);
4132 
4133 		if (!bp->tx_ring_map)
4134 			return -ENOMEM;
4135 
4136 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
4137 			j = 0;
4138 		else
4139 			j = bp->rx_nr_rings;
4140 
4141 		for (i = 0; i < bp->tx_nr_rings; i++, j++) {
4142 			struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
4143 
4144 			if (bp->flags & BNXT_FLAG_CHIP_P5)
4145 				txr->tx_ring_struct.ring_mem.flags =
4146 					BNXT_RMEM_RING_PTE_FLAG;
4147 			txr->bnapi = bp->bnapi[j];
4148 			bp->bnapi[j]->tx_ring = txr;
4149 			bp->tx_ring_map[i] = bp->tx_nr_rings_xdp + i;
4150 			if (i >= bp->tx_nr_rings_xdp) {
4151 				txr->txq_index = i - bp->tx_nr_rings_xdp;
4152 				bp->bnapi[j]->tx_int = bnxt_tx_int;
4153 			} else {
4154 				bp->bnapi[j]->flags |= BNXT_NAPI_FLAG_XDP;
4155 				bp->bnapi[j]->tx_int = bnxt_tx_int_xdp;
4156 			}
4157 		}
4158 
4159 		rc = bnxt_alloc_stats(bp);
4160 		if (rc)
4161 			goto alloc_mem_err;
4162 		bnxt_init_stats(bp);
4163 
4164 		rc = bnxt_alloc_ntp_fltrs(bp);
4165 		if (rc)
4166 			goto alloc_mem_err;
4167 
4168 		rc = bnxt_alloc_vnics(bp);
4169 		if (rc)
4170 			goto alloc_mem_err;
4171 	}
4172 
4173 	bnxt_init_ring_struct(bp);
4174 
4175 	rc = bnxt_alloc_rx_rings(bp);
4176 	if (rc)
4177 		goto alloc_mem_err;
4178 
4179 	rc = bnxt_alloc_tx_rings(bp);
4180 	if (rc)
4181 		goto alloc_mem_err;
4182 
4183 	rc = bnxt_alloc_cp_rings(bp);
4184 	if (rc)
4185 		goto alloc_mem_err;
4186 
4187 	bp->vnic_info[0].flags |= BNXT_VNIC_RSS_FLAG | BNXT_VNIC_MCAST_FLAG |
4188 				  BNXT_VNIC_UCAST_FLAG;
4189 	rc = bnxt_alloc_vnic_attributes(bp);
4190 	if (rc)
4191 		goto alloc_mem_err;
4192 	return 0;
4193 
4194 alloc_mem_err:
4195 	bnxt_free_mem(bp, true);
4196 	return rc;
4197 }
4198 
4199 static void bnxt_disable_int(struct bnxt *bp)
4200 {
4201 	int i;
4202 
4203 	if (!bp->bnapi)
4204 		return;
4205 
4206 	for (i = 0; i < bp->cp_nr_rings; i++) {
4207 		struct bnxt_napi *bnapi = bp->bnapi[i];
4208 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4209 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
4210 
4211 		if (ring->fw_ring_id != INVALID_HW_RING_ID)
4212 			bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
4213 	}
4214 }
4215 
4216 static int bnxt_cp_num_to_irq_num(struct bnxt *bp, int n)
4217 {
4218 	struct bnxt_napi *bnapi = bp->bnapi[n];
4219 	struct bnxt_cp_ring_info *cpr;
4220 
4221 	cpr = &bnapi->cp_ring;
4222 	return cpr->cp_ring_struct.map_idx;
4223 }
4224 
4225 static void bnxt_disable_int_sync(struct bnxt *bp)
4226 {
4227 	int i;
4228 
4229 	atomic_inc(&bp->intr_sem);
4230 
4231 	bnxt_disable_int(bp);
4232 	for (i = 0; i < bp->cp_nr_rings; i++) {
4233 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
4234 
4235 		synchronize_irq(bp->irq_tbl[map_idx].vector);
4236 	}
4237 }
4238 
4239 static void bnxt_enable_int(struct bnxt *bp)
4240 {
4241 	int i;
4242 
4243 	atomic_set(&bp->intr_sem, 0);
4244 	for (i = 0; i < bp->cp_nr_rings; i++) {
4245 		struct bnxt_napi *bnapi = bp->bnapi[i];
4246 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
4247 
4248 		bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons);
4249 	}
4250 }
4251 
4252 void bnxt_hwrm_cmd_hdr_init(struct bnxt *bp, void *request, u16 req_type,
4253 			    u16 cmpl_ring, u16 target_id)
4254 {
4255 	struct input *req = request;
4256 
4257 	req->req_type = cpu_to_le16(req_type);
4258 	req->cmpl_ring = cpu_to_le16(cmpl_ring);
4259 	req->target_id = cpu_to_le16(target_id);
4260 	if (bnxt_kong_hwrm_message(bp, req))
4261 		req->resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
4262 	else
4263 		req->resp_addr = cpu_to_le64(bp->hwrm_cmd_resp_dma_addr);
4264 }
4265 
4266 static int bnxt_hwrm_to_stderr(u32 hwrm_err)
4267 {
4268 	switch (hwrm_err) {
4269 	case HWRM_ERR_CODE_SUCCESS:
4270 		return 0;
4271 	case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED:
4272 		return -EACCES;
4273 	case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR:
4274 		return -ENOSPC;
4275 	case HWRM_ERR_CODE_INVALID_PARAMS:
4276 	case HWRM_ERR_CODE_INVALID_FLAGS:
4277 	case HWRM_ERR_CODE_INVALID_ENABLES:
4278 	case HWRM_ERR_CODE_UNSUPPORTED_TLV:
4279 	case HWRM_ERR_CODE_UNSUPPORTED_OPTION_ERR:
4280 		return -EINVAL;
4281 	case HWRM_ERR_CODE_NO_BUFFER:
4282 		return -ENOMEM;
4283 	case HWRM_ERR_CODE_HOT_RESET_PROGRESS:
4284 	case HWRM_ERR_CODE_BUSY:
4285 		return -EAGAIN;
4286 	case HWRM_ERR_CODE_CMD_NOT_SUPPORTED:
4287 		return -EOPNOTSUPP;
4288 	default:
4289 		return -EIO;
4290 	}
4291 }
4292 
4293 static int bnxt_hwrm_do_send_msg(struct bnxt *bp, void *msg, u32 msg_len,
4294 				 int timeout, bool silent)
4295 {
4296 	int i, intr_process, rc, tmo_count;
4297 	struct input *req = msg;
4298 	u32 *data = msg;
4299 	u8 *valid;
4300 	u16 cp_ring_id, len = 0;
4301 	struct hwrm_err_output *resp = bp->hwrm_cmd_resp_addr;
4302 	u16 max_req_len = BNXT_HWRM_MAX_REQ_LEN;
4303 	struct hwrm_short_input short_input = {0};
4304 	u32 doorbell_offset = BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER;
4305 	u32 bar_offset = BNXT_GRCPF_REG_CHIMP_COMM;
4306 	u16 dst = BNXT_HWRM_CHNL_CHIMP;
4307 
4308 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4309 		return -EBUSY;
4310 
4311 	if (msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4312 		if (msg_len > bp->hwrm_max_ext_req_len ||
4313 		    !bp->hwrm_short_cmd_req_addr)
4314 			return -EINVAL;
4315 	}
4316 
4317 	if (bnxt_hwrm_kong_chnl(bp, req)) {
4318 		dst = BNXT_HWRM_CHNL_KONG;
4319 		bar_offset = BNXT_GRCPF_REG_KONG_COMM;
4320 		doorbell_offset = BNXT_GRCPF_REG_KONG_COMM_TRIGGER;
4321 		resp = bp->hwrm_cmd_kong_resp_addr;
4322 	}
4323 
4324 	memset(resp, 0, PAGE_SIZE);
4325 	cp_ring_id = le16_to_cpu(req->cmpl_ring);
4326 	intr_process = (cp_ring_id == INVALID_HW_RING_ID) ? 0 : 1;
4327 
4328 	req->seq_id = cpu_to_le16(bnxt_get_hwrm_seq_id(bp, dst));
4329 	/* currently supports only one outstanding message */
4330 	if (intr_process)
4331 		bp->hwrm_intr_seq_id = le16_to_cpu(req->seq_id);
4332 
4333 	if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
4334 	    msg_len > BNXT_HWRM_MAX_REQ_LEN) {
4335 		void *short_cmd_req = bp->hwrm_short_cmd_req_addr;
4336 		u16 max_msg_len;
4337 
4338 		/* Set boundary for maximum extended request length for short
4339 		 * cmd format. If passed up from device use the max supported
4340 		 * internal req length.
4341 		 */
4342 		max_msg_len = bp->hwrm_max_ext_req_len;
4343 
4344 		memcpy(short_cmd_req, req, msg_len);
4345 		if (msg_len < max_msg_len)
4346 			memset(short_cmd_req + msg_len, 0,
4347 			       max_msg_len - msg_len);
4348 
4349 		short_input.req_type = req->req_type;
4350 		short_input.signature =
4351 				cpu_to_le16(SHORT_REQ_SIGNATURE_SHORT_CMD);
4352 		short_input.size = cpu_to_le16(msg_len);
4353 		short_input.req_addr =
4354 			cpu_to_le64(bp->hwrm_short_cmd_req_dma_addr);
4355 
4356 		data = (u32 *)&short_input;
4357 		msg_len = sizeof(short_input);
4358 
4359 		/* Sync memory write before updating doorbell */
4360 		wmb();
4361 
4362 		max_req_len = BNXT_HWRM_SHORT_REQ_LEN;
4363 	}
4364 
4365 	/* Write request msg to hwrm channel */
4366 	__iowrite32_copy(bp->bar0 + bar_offset, data, msg_len / 4);
4367 
4368 	for (i = msg_len; i < max_req_len; i += 4)
4369 		writel(0, bp->bar0 + bar_offset + i);
4370 
4371 	/* Ring channel doorbell */
4372 	writel(1, bp->bar0 + doorbell_offset);
4373 
4374 	if (!pci_is_enabled(bp->pdev))
4375 		return 0;
4376 
4377 	if (!timeout)
4378 		timeout = DFLT_HWRM_CMD_TIMEOUT;
4379 	/* convert timeout to usec */
4380 	timeout *= 1000;
4381 
4382 	i = 0;
4383 	/* Short timeout for the first few iterations:
4384 	 * number of loops = number of loops for short timeout +
4385 	 * number of loops for standard timeout.
4386 	 */
4387 	tmo_count = HWRM_SHORT_TIMEOUT_COUNTER;
4388 	timeout = timeout - HWRM_SHORT_MIN_TIMEOUT * HWRM_SHORT_TIMEOUT_COUNTER;
4389 	tmo_count += DIV_ROUND_UP(timeout, HWRM_MIN_TIMEOUT);
4390 
4391 	if (intr_process) {
4392 		u16 seq_id = bp->hwrm_intr_seq_id;
4393 
4394 		/* Wait until hwrm response cmpl interrupt is processed */
4395 		while (bp->hwrm_intr_seq_id != (u16)~seq_id &&
4396 		       i++ < tmo_count) {
4397 			/* Abort the wait for completion if the FW health
4398 			 * check has failed.
4399 			 */
4400 			if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4401 				return -EBUSY;
4402 			/* on first few passes, just barely sleep */
4403 			if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4404 				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4405 					     HWRM_SHORT_MAX_TIMEOUT);
4406 			else
4407 				usleep_range(HWRM_MIN_TIMEOUT,
4408 					     HWRM_MAX_TIMEOUT);
4409 		}
4410 
4411 		if (bp->hwrm_intr_seq_id != (u16)~seq_id) {
4412 			if (!silent)
4413 				netdev_err(bp->dev, "Resp cmpl intr err msg: 0x%x\n",
4414 					   le16_to_cpu(req->req_type));
4415 			return -EBUSY;
4416 		}
4417 		len = le16_to_cpu(resp->resp_len);
4418 		valid = ((u8 *)resp) + len - 1;
4419 	} else {
4420 		int j;
4421 
4422 		/* Check if response len is updated */
4423 		for (i = 0; i < tmo_count; i++) {
4424 			/* Abort the wait for completion if the FW health
4425 			 * check has failed.
4426 			 */
4427 			if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
4428 				return -EBUSY;
4429 			len = le16_to_cpu(resp->resp_len);
4430 			if (len)
4431 				break;
4432 			/* on first few passes, just barely sleep */
4433 			if (i < HWRM_SHORT_TIMEOUT_COUNTER)
4434 				usleep_range(HWRM_SHORT_MIN_TIMEOUT,
4435 					     HWRM_SHORT_MAX_TIMEOUT);
4436 			else
4437 				usleep_range(HWRM_MIN_TIMEOUT,
4438 					     HWRM_MAX_TIMEOUT);
4439 		}
4440 
4441 		if (i >= tmo_count) {
4442 			if (!silent)
4443 				netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d\n",
4444 					   HWRM_TOTAL_TIMEOUT(i),
4445 					   le16_to_cpu(req->req_type),
4446 					   le16_to_cpu(req->seq_id), len);
4447 			return -EBUSY;
4448 		}
4449 
4450 		/* Last byte of resp contains valid bit */
4451 		valid = ((u8 *)resp) + len - 1;
4452 		for (j = 0; j < HWRM_VALID_BIT_DELAY_USEC; j++) {
4453 			/* make sure we read from updated DMA memory */
4454 			dma_rmb();
4455 			if (*valid)
4456 				break;
4457 			usleep_range(1, 5);
4458 		}
4459 
4460 		if (j >= HWRM_VALID_BIT_DELAY_USEC) {
4461 			if (!silent)
4462 				netdev_err(bp->dev, "Error (timeout: %d) msg {0x%x 0x%x} len:%d v:%d\n",
4463 					   HWRM_TOTAL_TIMEOUT(i),
4464 					   le16_to_cpu(req->req_type),
4465 					   le16_to_cpu(req->seq_id), len,
4466 					   *valid);
4467 			return -EBUSY;
4468 		}
4469 	}
4470 
4471 	/* Zero valid bit for compatibility.  Valid bit in an older spec
4472 	 * may become a new field in a newer spec.  We must make sure that
4473 	 * a new field not implemented by old spec will read zero.
4474 	 */
4475 	*valid = 0;
4476 	rc = le16_to_cpu(resp->error_code);
4477 	if (rc && !silent)
4478 		netdev_err(bp->dev, "hwrm req_type 0x%x seq id 0x%x error 0x%x\n",
4479 			   le16_to_cpu(resp->req_type),
4480 			   le16_to_cpu(resp->seq_id), rc);
4481 	return bnxt_hwrm_to_stderr(rc);
4482 }
4483 
4484 int _hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4485 {
4486 	return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, false);
4487 }
4488 
4489 int _hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4490 			      int timeout)
4491 {
4492 	return bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4493 }
4494 
4495 int hwrm_send_message(struct bnxt *bp, void *msg, u32 msg_len, int timeout)
4496 {
4497 	int rc;
4498 
4499 	mutex_lock(&bp->hwrm_cmd_lock);
4500 	rc = _hwrm_send_message(bp, msg, msg_len, timeout);
4501 	mutex_unlock(&bp->hwrm_cmd_lock);
4502 	return rc;
4503 }
4504 
4505 int hwrm_send_message_silent(struct bnxt *bp, void *msg, u32 msg_len,
4506 			     int timeout)
4507 {
4508 	int rc;
4509 
4510 	mutex_lock(&bp->hwrm_cmd_lock);
4511 	rc = bnxt_hwrm_do_send_msg(bp, msg, msg_len, timeout, true);
4512 	mutex_unlock(&bp->hwrm_cmd_lock);
4513 	return rc;
4514 }
4515 
4516 int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, int bmap_size,
4517 			    bool async_only)
4518 {
4519 	struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr;
4520 	struct hwrm_func_drv_rgtr_input req = {0};
4521 	DECLARE_BITMAP(async_events_bmap, 256);
4522 	u32 *events = (u32 *)async_events_bmap;
4523 	u32 flags;
4524 	int rc, i;
4525 
4526 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR, -1, -1);
4527 
4528 	req.enables =
4529 		cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_OS_TYPE |
4530 			    FUNC_DRV_RGTR_REQ_ENABLES_VER |
4531 			    FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4532 
4533 	req.os_type = cpu_to_le16(FUNC_DRV_RGTR_REQ_OS_TYPE_LINUX);
4534 	flags = FUNC_DRV_RGTR_REQ_FLAGS_16BIT_VER_MODE;
4535 	if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET)
4536 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_HOT_RESET_SUPPORT;
4537 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
4538 		flags |= FUNC_DRV_RGTR_REQ_FLAGS_ERROR_RECOVERY_SUPPORT |
4539 			 FUNC_DRV_RGTR_REQ_FLAGS_MASTER_SUPPORT;
4540 	req.flags = cpu_to_le32(flags);
4541 	req.ver_maj_8b = DRV_VER_MAJ;
4542 	req.ver_min_8b = DRV_VER_MIN;
4543 	req.ver_upd_8b = DRV_VER_UPD;
4544 	req.ver_maj = cpu_to_le16(DRV_VER_MAJ);
4545 	req.ver_min = cpu_to_le16(DRV_VER_MIN);
4546 	req.ver_upd = cpu_to_le16(DRV_VER_UPD);
4547 
4548 	if (BNXT_PF(bp)) {
4549 		u32 data[8];
4550 		int i;
4551 
4552 		memset(data, 0, sizeof(data));
4553 		for (i = 0; i < ARRAY_SIZE(bnxt_vf_req_snif); i++) {
4554 			u16 cmd = bnxt_vf_req_snif[i];
4555 			unsigned int bit, idx;
4556 
4557 			idx = cmd / 32;
4558 			bit = cmd % 32;
4559 			data[idx] |= 1 << bit;
4560 		}
4561 
4562 		for (i = 0; i < 8; i++)
4563 			req.vf_req_fwd[i] = cpu_to_le32(data[i]);
4564 
4565 		req.enables |=
4566 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_VF_REQ_FWD);
4567 	}
4568 
4569 	if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE)
4570 		req.flags |= cpu_to_le32(
4571 			FUNC_DRV_RGTR_REQ_FLAGS_FLOW_HANDLE_64BIT_MODE);
4572 
4573 	memset(async_events_bmap, 0, sizeof(async_events_bmap));
4574 	for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) {
4575 		u16 event_id = bnxt_async_events_arr[i];
4576 
4577 		if (event_id == ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY &&
4578 		    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
4579 			continue;
4580 		__set_bit(bnxt_async_events_arr[i], async_events_bmap);
4581 	}
4582 	if (bmap && bmap_size) {
4583 		for (i = 0; i < bmap_size; i++) {
4584 			if (test_bit(i, bmap))
4585 				__set_bit(i, async_events_bmap);
4586 		}
4587 	}
4588 	for (i = 0; i < 8; i++)
4589 		req.async_event_fwd[i] |= cpu_to_le32(events[i]);
4590 
4591 	if (async_only)
4592 		req.enables =
4593 			cpu_to_le32(FUNC_DRV_RGTR_REQ_ENABLES_ASYNC_EVENT_FWD);
4594 
4595 	mutex_lock(&bp->hwrm_cmd_lock);
4596 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4597 	if (!rc) {
4598 		set_bit(BNXT_STATE_DRV_REGISTERED, &bp->state);
4599 		if (resp->flags &
4600 		    cpu_to_le32(FUNC_DRV_RGTR_RESP_FLAGS_IF_CHANGE_SUPPORTED))
4601 			bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE;
4602 	}
4603 	mutex_unlock(&bp->hwrm_cmd_lock);
4604 	return rc;
4605 }
4606 
4607 static int bnxt_hwrm_func_drv_unrgtr(struct bnxt *bp)
4608 {
4609 	struct hwrm_func_drv_unrgtr_input req = {0};
4610 
4611 	if (!test_and_clear_bit(BNXT_STATE_DRV_REGISTERED, &bp->state))
4612 		return 0;
4613 
4614 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_UNRGTR, -1, -1);
4615 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4616 }
4617 
4618 static int bnxt_hwrm_tunnel_dst_port_free(struct bnxt *bp, u8 tunnel_type)
4619 {
4620 	u32 rc = 0;
4621 	struct hwrm_tunnel_dst_port_free_input req = {0};
4622 
4623 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_FREE, -1, -1);
4624 	req.tunnel_type = tunnel_type;
4625 
4626 	switch (tunnel_type) {
4627 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN:
4628 		req.tunnel_dst_port_id = cpu_to_le16(bp->vxlan_fw_dst_port_id);
4629 		bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
4630 		break;
4631 	case TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE:
4632 		req.tunnel_dst_port_id = cpu_to_le16(bp->nge_fw_dst_port_id);
4633 		bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
4634 		break;
4635 	default:
4636 		break;
4637 	}
4638 
4639 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4640 	if (rc)
4641 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_free failed. rc:%d\n",
4642 			   rc);
4643 	return rc;
4644 }
4645 
4646 static int bnxt_hwrm_tunnel_dst_port_alloc(struct bnxt *bp, __be16 port,
4647 					   u8 tunnel_type)
4648 {
4649 	u32 rc = 0;
4650 	struct hwrm_tunnel_dst_port_alloc_input req = {0};
4651 	struct hwrm_tunnel_dst_port_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4652 
4653 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TUNNEL_DST_PORT_ALLOC, -1, -1);
4654 
4655 	req.tunnel_type = tunnel_type;
4656 	req.tunnel_dst_port_val = port;
4657 
4658 	mutex_lock(&bp->hwrm_cmd_lock);
4659 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4660 	if (rc) {
4661 		netdev_err(bp->dev, "hwrm_tunnel_dst_port_alloc failed. rc:%d\n",
4662 			   rc);
4663 		goto err_out;
4664 	}
4665 
4666 	switch (tunnel_type) {
4667 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_VXLAN:
4668 		bp->vxlan_fw_dst_port_id =
4669 			le16_to_cpu(resp->tunnel_dst_port_id);
4670 		break;
4671 	case TUNNEL_DST_PORT_ALLOC_REQ_TUNNEL_TYPE_GENEVE:
4672 		bp->nge_fw_dst_port_id = le16_to_cpu(resp->tunnel_dst_port_id);
4673 		break;
4674 	default:
4675 		break;
4676 	}
4677 
4678 err_out:
4679 	mutex_unlock(&bp->hwrm_cmd_lock);
4680 	return rc;
4681 }
4682 
4683 static int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, u16 vnic_id)
4684 {
4685 	struct hwrm_cfa_l2_set_rx_mask_input req = {0};
4686 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4687 
4688 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_SET_RX_MASK, -1, -1);
4689 	req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
4690 
4691 	req.num_mc_entries = cpu_to_le32(vnic->mc_list_count);
4692 	req.mc_tbl_addr = cpu_to_le64(vnic->mc_list_mapping);
4693 	req.mask = cpu_to_le32(vnic->rx_mask);
4694 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4695 }
4696 
4697 #ifdef CONFIG_RFS_ACCEL
4698 static int bnxt_hwrm_cfa_ntuple_filter_free(struct bnxt *bp,
4699 					    struct bnxt_ntuple_filter *fltr)
4700 {
4701 	struct hwrm_cfa_ntuple_filter_free_input req = {0};
4702 
4703 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_FREE, -1, -1);
4704 	req.ntuple_filter_id = fltr->filter_id;
4705 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4706 }
4707 
4708 #define BNXT_NTP_FLTR_FLAGS					\
4709 	(CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID |	\
4710 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE |	\
4711 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR |	\
4712 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE |	\
4713 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |	\
4714 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK |	\
4715 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |	\
4716 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK |	\
4717 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL |	\
4718 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT |		\
4719 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK |	\
4720 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT |		\
4721 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK |	\
4722 	 CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID)
4723 
4724 #define BNXT_NTP_TUNNEL_FLTR_FLAG				\
4725 		CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE
4726 
4727 static int bnxt_hwrm_cfa_ntuple_filter_alloc(struct bnxt *bp,
4728 					     struct bnxt_ntuple_filter *fltr)
4729 {
4730 	struct hwrm_cfa_ntuple_filter_alloc_input req = {0};
4731 	struct hwrm_cfa_ntuple_filter_alloc_output *resp;
4732 	struct flow_keys *keys = &fltr->fkeys;
4733 	struct bnxt_vnic_info *vnic;
4734 	u32 flags = 0;
4735 	int rc = 0;
4736 
4737 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_NTUPLE_FILTER_ALLOC, -1, -1);
4738 	req.l2_filter_id = bp->vnic_info[0].fw_l2_filter_id[fltr->l2_fltr_idx];
4739 
4740 	if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2) {
4741 		flags = CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX;
4742 		req.dst_id = cpu_to_le16(fltr->rxq);
4743 	} else {
4744 		vnic = &bp->vnic_info[fltr->rxq + 1];
4745 		req.dst_id = cpu_to_le16(vnic->fw_vnic_id);
4746 	}
4747 	req.flags = cpu_to_le32(flags);
4748 	req.enables = cpu_to_le32(BNXT_NTP_FLTR_FLAGS);
4749 
4750 	req.ethertype = htons(ETH_P_IP);
4751 	memcpy(req.src_macaddr, fltr->src_mac_addr, ETH_ALEN);
4752 	req.ip_addr_type = CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
4753 	req.ip_protocol = keys->basic.ip_proto;
4754 
4755 	if (keys->basic.n_proto == htons(ETH_P_IPV6)) {
4756 		int i;
4757 
4758 		req.ethertype = htons(ETH_P_IPV6);
4759 		req.ip_addr_type =
4760 			CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6;
4761 		*(struct in6_addr *)&req.src_ipaddr[0] =
4762 			keys->addrs.v6addrs.src;
4763 		*(struct in6_addr *)&req.dst_ipaddr[0] =
4764 			keys->addrs.v6addrs.dst;
4765 		for (i = 0; i < 4; i++) {
4766 			req.src_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4767 			req.dst_ipaddr_mask[i] = cpu_to_be32(0xffffffff);
4768 		}
4769 	} else {
4770 		req.src_ipaddr[0] = keys->addrs.v4addrs.src;
4771 		req.src_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4772 		req.dst_ipaddr[0] = keys->addrs.v4addrs.dst;
4773 		req.dst_ipaddr_mask[0] = cpu_to_be32(0xffffffff);
4774 	}
4775 	if (keys->control.flags & FLOW_DIS_ENCAPSULATION) {
4776 		req.enables |= cpu_to_le32(BNXT_NTP_TUNNEL_FLTR_FLAG);
4777 		req.tunnel_type =
4778 			CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL;
4779 	}
4780 
4781 	req.src_port = keys->ports.src;
4782 	req.src_port_mask = cpu_to_be16(0xffff);
4783 	req.dst_port = keys->ports.dst;
4784 	req.dst_port_mask = cpu_to_be16(0xffff);
4785 
4786 	mutex_lock(&bp->hwrm_cmd_lock);
4787 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4788 	if (!rc) {
4789 		resp = bnxt_get_hwrm_resp_addr(bp, &req);
4790 		fltr->filter_id = resp->ntuple_filter_id;
4791 	}
4792 	mutex_unlock(&bp->hwrm_cmd_lock);
4793 	return rc;
4794 }
4795 #endif
4796 
4797 static int bnxt_hwrm_set_vnic_filter(struct bnxt *bp, u16 vnic_id, u16 idx,
4798 				     u8 *mac_addr)
4799 {
4800 	u32 rc = 0;
4801 	struct hwrm_cfa_l2_filter_alloc_input req = {0};
4802 	struct hwrm_cfa_l2_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
4803 
4804 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_ALLOC, -1, -1);
4805 	req.flags = cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_PATH_RX);
4806 	if (!BNXT_CHIP_TYPE_NITRO_A0(bp))
4807 		req.flags |=
4808 			cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_FLAGS_OUTERMOST);
4809 	req.dst_id = cpu_to_le16(bp->vnic_info[vnic_id].fw_vnic_id);
4810 	req.enables =
4811 		cpu_to_le32(CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR |
4812 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_DST_ID |
4813 			    CFA_L2_FILTER_ALLOC_REQ_ENABLES_L2_ADDR_MASK);
4814 	memcpy(req.l2_addr, mac_addr, ETH_ALEN);
4815 	req.l2_addr_mask[0] = 0xff;
4816 	req.l2_addr_mask[1] = 0xff;
4817 	req.l2_addr_mask[2] = 0xff;
4818 	req.l2_addr_mask[3] = 0xff;
4819 	req.l2_addr_mask[4] = 0xff;
4820 	req.l2_addr_mask[5] = 0xff;
4821 
4822 	mutex_lock(&bp->hwrm_cmd_lock);
4823 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4824 	if (!rc)
4825 		bp->vnic_info[vnic_id].fw_l2_filter_id[idx] =
4826 							resp->l2_filter_id;
4827 	mutex_unlock(&bp->hwrm_cmd_lock);
4828 	return rc;
4829 }
4830 
4831 static int bnxt_hwrm_clear_vnic_filter(struct bnxt *bp)
4832 {
4833 	u16 i, j, num_of_vnics = 1; /* only vnic 0 supported */
4834 	int rc = 0;
4835 
4836 	/* Any associated ntuple filters will also be cleared by firmware. */
4837 	mutex_lock(&bp->hwrm_cmd_lock);
4838 	for (i = 0; i < num_of_vnics; i++) {
4839 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
4840 
4841 		for (j = 0; j < vnic->uc_filter_count; j++) {
4842 			struct hwrm_cfa_l2_filter_free_input req = {0};
4843 
4844 			bnxt_hwrm_cmd_hdr_init(bp, &req,
4845 					       HWRM_CFA_L2_FILTER_FREE, -1, -1);
4846 
4847 			req.l2_filter_id = vnic->fw_l2_filter_id[j];
4848 
4849 			rc = _hwrm_send_message(bp, &req, sizeof(req),
4850 						HWRM_CMD_TIMEOUT);
4851 		}
4852 		vnic->uc_filter_count = 0;
4853 	}
4854 	mutex_unlock(&bp->hwrm_cmd_lock);
4855 
4856 	return rc;
4857 }
4858 
4859 static int bnxt_hwrm_vnic_set_tpa(struct bnxt *bp, u16 vnic_id, u32 tpa_flags)
4860 {
4861 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
4862 	u16 max_aggs = VNIC_TPA_CFG_REQ_MAX_AGGS_MAX;
4863 	struct hwrm_vnic_tpa_cfg_input req = {0};
4864 
4865 	if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
4866 		return 0;
4867 
4868 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_TPA_CFG, -1, -1);
4869 
4870 	if (tpa_flags) {
4871 		u16 mss = bp->dev->mtu - 40;
4872 		u32 nsegs, n, segs = 0, flags;
4873 
4874 		flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
4875 			VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
4876 			VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
4877 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
4878 			VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
4879 		if (tpa_flags & BNXT_FLAG_GRO)
4880 			flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
4881 
4882 		req.flags = cpu_to_le32(flags);
4883 
4884 		req.enables =
4885 			cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
4886 				    VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
4887 				    VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
4888 
4889 		/* Number of segs are log2 units, and first packet is not
4890 		 * included as part of this units.
4891 		 */
4892 		if (mss <= BNXT_RX_PAGE_SIZE) {
4893 			n = BNXT_RX_PAGE_SIZE / mss;
4894 			nsegs = (MAX_SKB_FRAGS - 1) * n;
4895 		} else {
4896 			n = mss / BNXT_RX_PAGE_SIZE;
4897 			if (mss & (BNXT_RX_PAGE_SIZE - 1))
4898 				n++;
4899 			nsegs = (MAX_SKB_FRAGS - n) / n;
4900 		}
4901 
4902 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
4903 			segs = MAX_TPA_SEGS_P5;
4904 			max_aggs = bp->max_tpa;
4905 		} else {
4906 			segs = ilog2(nsegs);
4907 		}
4908 		req.max_agg_segs = cpu_to_le16(segs);
4909 		req.max_aggs = cpu_to_le16(max_aggs);
4910 
4911 		req.min_agg_len = cpu_to_le32(512);
4912 	}
4913 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
4914 
4915 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
4916 }
4917 
4918 static u16 bnxt_cp_ring_from_grp(struct bnxt *bp, struct bnxt_ring_struct *ring)
4919 {
4920 	struct bnxt_ring_grp_info *grp_info;
4921 
4922 	grp_info = &bp->grp_info[ring->grp_idx];
4923 	return grp_info->cp_fw_ring_id;
4924 }
4925 
4926 static u16 bnxt_cp_ring_for_rx(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
4927 {
4928 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
4929 		struct bnxt_napi *bnapi = rxr->bnapi;
4930 		struct bnxt_cp_ring_info *cpr;
4931 
4932 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_RX_HDL];
4933 		return cpr->cp_ring_struct.fw_ring_id;
4934 	} else {
4935 		return bnxt_cp_ring_from_grp(bp, &rxr->rx_ring_struct);
4936 	}
4937 }
4938 
4939 static u16 bnxt_cp_ring_for_tx(struct bnxt *bp, struct bnxt_tx_ring_info *txr)
4940 {
4941 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
4942 		struct bnxt_napi *bnapi = txr->bnapi;
4943 		struct bnxt_cp_ring_info *cpr;
4944 
4945 		cpr = bnapi->cp_ring.cp_ring_arr[BNXT_TX_HDL];
4946 		return cpr->cp_ring_struct.fw_ring_id;
4947 	} else {
4948 		return bnxt_cp_ring_from_grp(bp, &txr->tx_ring_struct);
4949 	}
4950 }
4951 
4952 static int bnxt_alloc_rss_indir_tbl(struct bnxt *bp)
4953 {
4954 	int entries;
4955 
4956 	if (bp->flags & BNXT_FLAG_CHIP_P5)
4957 		entries = BNXT_MAX_RSS_TABLE_ENTRIES_P5;
4958 	else
4959 		entries = HW_HASH_INDEX_SIZE;
4960 
4961 	bp->rss_indir_tbl_entries = entries;
4962 	bp->rss_indir_tbl = kmalloc_array(entries, sizeof(*bp->rss_indir_tbl),
4963 					  GFP_KERNEL);
4964 	if (!bp->rss_indir_tbl)
4965 		return -ENOMEM;
4966 	return 0;
4967 }
4968 
4969 static void bnxt_set_dflt_rss_indir_tbl(struct bnxt *bp)
4970 {
4971 	u16 max_rings, max_entries, pad, i;
4972 
4973 	if (!bp->rx_nr_rings)
4974 		return;
4975 
4976 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
4977 		max_rings = bp->rx_nr_rings - 1;
4978 	else
4979 		max_rings = bp->rx_nr_rings;
4980 
4981 	max_entries = bnxt_get_rxfh_indir_size(bp->dev);
4982 
4983 	for (i = 0; i < max_entries; i++)
4984 		bp->rss_indir_tbl[i] = ethtool_rxfh_indir_default(i, max_rings);
4985 
4986 	pad = bp->rss_indir_tbl_entries - max_entries;
4987 	if (pad)
4988 		memset(&bp->rss_indir_tbl[i], 0, pad * sizeof(u16));
4989 }
4990 
4991 static u16 bnxt_get_max_rss_ring(struct bnxt *bp)
4992 {
4993 	u16 i, tbl_size, max_ring = 0;
4994 
4995 	if (!bp->rss_indir_tbl)
4996 		return 0;
4997 
4998 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
4999 	for (i = 0; i < tbl_size; i++)
5000 		max_ring = max(max_ring, bp->rss_indir_tbl[i]);
5001 	return max_ring;
5002 }
5003 
5004 int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings)
5005 {
5006 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5007 		return DIV_ROUND_UP(rx_rings, BNXT_RSS_TABLE_ENTRIES_P5);
5008 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
5009 		return 2;
5010 	return 1;
5011 }
5012 
5013 static void __bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5014 {
5015 	bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG);
5016 	u16 i, j;
5017 
5018 	/* Fill the RSS indirection table with ring group ids */
5019 	for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) {
5020 		if (!no_rss)
5021 			j = bp->rss_indir_tbl[i];
5022 		vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]);
5023 	}
5024 }
5025 
5026 static void __bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp,
5027 				      struct bnxt_vnic_info *vnic)
5028 {
5029 	__le16 *ring_tbl = vnic->rss_table;
5030 	struct bnxt_rx_ring_info *rxr;
5031 	u16 tbl_size, i;
5032 
5033 	tbl_size = bnxt_get_rxfh_indir_size(bp->dev);
5034 
5035 	for (i = 0; i < tbl_size; i++) {
5036 		u16 ring_id, j;
5037 
5038 		j = bp->rss_indir_tbl[i];
5039 		rxr = &bp->rx_ring[j];
5040 
5041 		ring_id = rxr->rx_ring_struct.fw_ring_id;
5042 		*ring_tbl++ = cpu_to_le16(ring_id);
5043 		ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5044 		*ring_tbl++ = cpu_to_le16(ring_id);
5045 	}
5046 }
5047 
5048 static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic)
5049 {
5050 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5051 		__bnxt_fill_hw_rss_tbl_p5(bp, vnic);
5052 	else
5053 		__bnxt_fill_hw_rss_tbl(bp, vnic);
5054 }
5055 
5056 static int bnxt_hwrm_vnic_set_rss(struct bnxt *bp, u16 vnic_id, bool set_rss)
5057 {
5058 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5059 	struct hwrm_vnic_rss_cfg_input req = {0};
5060 
5061 	if ((bp->flags & BNXT_FLAG_CHIP_P5) ||
5062 	    vnic->fw_rss_cos_lb_ctx[0] == INVALID_HW_RING_ID)
5063 		return 0;
5064 
5065 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5066 	if (set_rss) {
5067 		bnxt_fill_hw_rss_tbl(bp, vnic);
5068 		req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5069 		req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5070 		req.ring_grp_tbl_addr = cpu_to_le64(vnic->rss_table_dma_addr);
5071 		req.hash_key_tbl_addr =
5072 			cpu_to_le64(vnic->rss_hash_key_dma_addr);
5073 	}
5074 	req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5075 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5076 }
5077 
5078 static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, u16 vnic_id, bool set_rss)
5079 {
5080 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5081 	struct hwrm_vnic_rss_cfg_input req = {0};
5082 	dma_addr_t ring_tbl_map;
5083 	u32 i, nr_ctxs;
5084 
5085 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_CFG, -1, -1);
5086 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5087 	if (!set_rss) {
5088 		hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5089 		return 0;
5090 	}
5091 	bnxt_fill_hw_rss_tbl(bp, vnic);
5092 	req.hash_type = cpu_to_le32(bp->rss_hash_cfg);
5093 	req.hash_mode_flags = VNIC_RSS_CFG_REQ_HASH_MODE_FLAGS_DEFAULT;
5094 	req.hash_key_tbl_addr = cpu_to_le64(vnic->rss_hash_key_dma_addr);
5095 	ring_tbl_map = vnic->rss_table_dma_addr;
5096 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
5097 	for (i = 0; i < nr_ctxs; ring_tbl_map += BNXT_RSS_TABLE_SIZE_P5, i++) {
5098 		int rc;
5099 
5100 		req.ring_grp_tbl_addr = cpu_to_le64(ring_tbl_map);
5101 		req.ring_table_pair_index = i;
5102 		req.rss_ctx_idx = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[i]);
5103 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5104 		if (rc)
5105 			return rc;
5106 	}
5107 	return 0;
5108 }
5109 
5110 static int bnxt_hwrm_vnic_set_hds(struct bnxt *bp, u16 vnic_id)
5111 {
5112 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5113 	struct hwrm_vnic_plcmodes_cfg_input req = {0};
5114 
5115 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_PLCMODES_CFG, -1, -1);
5116 	req.flags = cpu_to_le32(VNIC_PLCMODES_CFG_REQ_FLAGS_JUMBO_PLACEMENT |
5117 				VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV4 |
5118 				VNIC_PLCMODES_CFG_REQ_FLAGS_HDS_IPV6);
5119 	req.enables =
5120 		cpu_to_le32(VNIC_PLCMODES_CFG_REQ_ENABLES_JUMBO_THRESH_VALID |
5121 			    VNIC_PLCMODES_CFG_REQ_ENABLES_HDS_THRESHOLD_VALID);
5122 	/* thresholds not implemented in firmware yet */
5123 	req.jumbo_thresh = cpu_to_le16(bp->rx_copy_thresh);
5124 	req.hds_threshold = cpu_to_le16(bp->rx_copy_thresh);
5125 	req.vnic_id = cpu_to_le32(vnic->fw_vnic_id);
5126 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5127 }
5128 
5129 static void bnxt_hwrm_vnic_ctx_free_one(struct bnxt *bp, u16 vnic_id,
5130 					u16 ctx_idx)
5131 {
5132 	struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0};
5133 
5134 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE, -1, -1);
5135 	req.rss_cos_lb_ctx_id =
5136 		cpu_to_le16(bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx]);
5137 
5138 	hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5139 	bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID;
5140 }
5141 
5142 static void bnxt_hwrm_vnic_ctx_free(struct bnxt *bp)
5143 {
5144 	int i, j;
5145 
5146 	for (i = 0; i < bp->nr_vnics; i++) {
5147 		struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
5148 
5149 		for (j = 0; j < BNXT_MAX_CTX_PER_VNIC; j++) {
5150 			if (vnic->fw_rss_cos_lb_ctx[j] != INVALID_HW_RING_ID)
5151 				bnxt_hwrm_vnic_ctx_free_one(bp, i, j);
5152 		}
5153 	}
5154 	bp->rsscos_nr_ctxs = 0;
5155 }
5156 
5157 static int bnxt_hwrm_vnic_ctx_alloc(struct bnxt *bp, u16 vnic_id, u16 ctx_idx)
5158 {
5159 	int rc;
5160 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0};
5161 	struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp =
5162 						bp->hwrm_cmd_resp_addr;
5163 
5164 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC, -1,
5165 			       -1);
5166 
5167 	mutex_lock(&bp->hwrm_cmd_lock);
5168 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5169 	if (!rc)
5170 		bp->vnic_info[vnic_id].fw_rss_cos_lb_ctx[ctx_idx] =
5171 			le16_to_cpu(resp->rss_cos_lb_ctx_id);
5172 	mutex_unlock(&bp->hwrm_cmd_lock);
5173 
5174 	return rc;
5175 }
5176 
5177 static u32 bnxt_get_roce_vnic_mode(struct bnxt *bp)
5178 {
5179 	if (bp->flags & BNXT_FLAG_ROCE_MIRROR_CAP)
5180 		return VNIC_CFG_REQ_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_MODE;
5181 	return VNIC_CFG_REQ_FLAGS_ROCE_DUAL_VNIC_MODE;
5182 }
5183 
5184 int bnxt_hwrm_vnic_cfg(struct bnxt *bp, u16 vnic_id)
5185 {
5186 	unsigned int ring = 0, grp_idx;
5187 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5188 	struct hwrm_vnic_cfg_input req = {0};
5189 	u16 def_vlan = 0;
5190 
5191 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_CFG, -1, -1);
5192 
5193 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5194 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0];
5195 
5196 		req.default_rx_ring_id =
5197 			cpu_to_le16(rxr->rx_ring_struct.fw_ring_id);
5198 		req.default_cmpl_ring_id =
5199 			cpu_to_le16(bnxt_cp_ring_for_rx(bp, rxr));
5200 		req.enables =
5201 			cpu_to_le32(VNIC_CFG_REQ_ENABLES_DEFAULT_RX_RING_ID |
5202 				    VNIC_CFG_REQ_ENABLES_DEFAULT_CMPL_RING_ID);
5203 		goto vnic_mru;
5204 	}
5205 	req.enables = cpu_to_le32(VNIC_CFG_REQ_ENABLES_DFLT_RING_GRP);
5206 	/* Only RSS support for now TBD: COS & LB */
5207 	if (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID) {
5208 		req.rss_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[0]);
5209 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5210 					   VNIC_CFG_REQ_ENABLES_MRU);
5211 	} else if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG) {
5212 		req.rss_rule =
5213 			cpu_to_le16(bp->vnic_info[0].fw_rss_cos_lb_ctx[0]);
5214 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_RSS_RULE |
5215 					   VNIC_CFG_REQ_ENABLES_MRU);
5216 		req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_RSS_DFLT_CR_MODE);
5217 	} else {
5218 		req.rss_rule = cpu_to_le16(0xffff);
5219 	}
5220 
5221 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) &&
5222 	    (vnic->fw_rss_cos_lb_ctx[0] != INVALID_HW_RING_ID)) {
5223 		req.cos_rule = cpu_to_le16(vnic->fw_rss_cos_lb_ctx[1]);
5224 		req.enables |= cpu_to_le32(VNIC_CFG_REQ_ENABLES_COS_RULE);
5225 	} else {
5226 		req.cos_rule = cpu_to_le16(0xffff);
5227 	}
5228 
5229 	if (vnic->flags & BNXT_VNIC_RSS_FLAG)
5230 		ring = 0;
5231 	else if (vnic->flags & BNXT_VNIC_RFS_FLAG)
5232 		ring = vnic_id - 1;
5233 	else if ((vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp))
5234 		ring = bp->rx_nr_rings - 1;
5235 
5236 	grp_idx = bp->rx_ring[ring].bnapi->index;
5237 	req.dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id);
5238 	req.lb_rule = cpu_to_le16(0xffff);
5239 vnic_mru:
5240 	req.mru = cpu_to_le16(bp->dev->mtu + ETH_HLEN + VLAN_HLEN);
5241 
5242 	req.vnic_id = cpu_to_le16(vnic->fw_vnic_id);
5243 #ifdef CONFIG_BNXT_SRIOV
5244 	if (BNXT_VF(bp))
5245 		def_vlan = bp->vf.vlan;
5246 #endif
5247 	if ((bp->flags & BNXT_FLAG_STRIP_VLAN) || def_vlan)
5248 		req.flags |= cpu_to_le32(VNIC_CFG_REQ_FLAGS_VLAN_STRIP_MODE);
5249 	if (!vnic_id && bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP))
5250 		req.flags |= cpu_to_le32(bnxt_get_roce_vnic_mode(bp));
5251 
5252 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5253 }
5254 
5255 static void bnxt_hwrm_vnic_free_one(struct bnxt *bp, u16 vnic_id)
5256 {
5257 	if (bp->vnic_info[vnic_id].fw_vnic_id != INVALID_HW_RING_ID) {
5258 		struct hwrm_vnic_free_input req = {0};
5259 
5260 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_FREE, -1, -1);
5261 		req.vnic_id =
5262 			cpu_to_le32(bp->vnic_info[vnic_id].fw_vnic_id);
5263 
5264 		hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5265 		bp->vnic_info[vnic_id].fw_vnic_id = INVALID_HW_RING_ID;
5266 	}
5267 }
5268 
5269 static void bnxt_hwrm_vnic_free(struct bnxt *bp)
5270 {
5271 	u16 i;
5272 
5273 	for (i = 0; i < bp->nr_vnics; i++)
5274 		bnxt_hwrm_vnic_free_one(bp, i);
5275 }
5276 
5277 static int bnxt_hwrm_vnic_alloc(struct bnxt *bp, u16 vnic_id,
5278 				unsigned int start_rx_ring_idx,
5279 				unsigned int nr_rings)
5280 {
5281 	int rc = 0;
5282 	unsigned int i, j, grp_idx, end_idx = start_rx_ring_idx + nr_rings;
5283 	struct hwrm_vnic_alloc_input req = {0};
5284 	struct hwrm_vnic_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5285 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
5286 
5287 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5288 		goto vnic_no_ring_grps;
5289 
5290 	/* map ring groups to this vnic */
5291 	for (i = start_rx_ring_idx, j = 0; i < end_idx; i++, j++) {
5292 		grp_idx = bp->rx_ring[i].bnapi->index;
5293 		if (bp->grp_info[grp_idx].fw_grp_id == INVALID_HW_RING_ID) {
5294 			netdev_err(bp->dev, "Not enough ring groups avail:%x req:%x\n",
5295 				   j, nr_rings);
5296 			break;
5297 		}
5298 		vnic->fw_grp_ids[j] = bp->grp_info[grp_idx].fw_grp_id;
5299 	}
5300 
5301 vnic_no_ring_grps:
5302 	for (i = 0; i < BNXT_MAX_CTX_PER_VNIC; i++)
5303 		vnic->fw_rss_cos_lb_ctx[i] = INVALID_HW_RING_ID;
5304 	if (vnic_id == 0)
5305 		req.flags = cpu_to_le32(VNIC_ALLOC_REQ_FLAGS_DEFAULT);
5306 
5307 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_ALLOC, -1, -1);
5308 
5309 	mutex_lock(&bp->hwrm_cmd_lock);
5310 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5311 	if (!rc)
5312 		vnic->fw_vnic_id = le32_to_cpu(resp->vnic_id);
5313 	mutex_unlock(&bp->hwrm_cmd_lock);
5314 	return rc;
5315 }
5316 
5317 static int bnxt_hwrm_vnic_qcaps(struct bnxt *bp)
5318 {
5319 	struct hwrm_vnic_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
5320 	struct hwrm_vnic_qcaps_input req = {0};
5321 	int rc;
5322 
5323 	bp->hw_ring_stats_size = sizeof(struct ctx_hw_stats);
5324 	bp->flags &= ~(BNXT_FLAG_NEW_RSS_CAP | BNXT_FLAG_ROCE_MIRROR_CAP);
5325 	if (bp->hwrm_spec_code < 0x10600)
5326 		return 0;
5327 
5328 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VNIC_QCAPS, -1, -1);
5329 	mutex_lock(&bp->hwrm_cmd_lock);
5330 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5331 	if (!rc) {
5332 		u32 flags = le32_to_cpu(resp->flags);
5333 
5334 		if (!(bp->flags & BNXT_FLAG_CHIP_P5) &&
5335 		    (flags & VNIC_QCAPS_RESP_FLAGS_RSS_DFLT_CR_CAP))
5336 			bp->flags |= BNXT_FLAG_NEW_RSS_CAP;
5337 		if (flags &
5338 		    VNIC_QCAPS_RESP_FLAGS_ROCE_MIRRORING_CAPABLE_VNIC_CAP)
5339 			bp->flags |= BNXT_FLAG_ROCE_MIRROR_CAP;
5340 
5341 		/* Older P5 fw before EXT_HW_STATS support did not set
5342 		 * VLAN_STRIP_CAP properly.
5343 		 */
5344 		if ((flags & VNIC_QCAPS_RESP_FLAGS_VLAN_STRIP_CAP) ||
5345 		    ((bp->flags & BNXT_FLAG_CHIP_P5) &&
5346 		     !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED)))
5347 			bp->fw_cap |= BNXT_FW_CAP_VLAN_RX_STRIP;
5348 		bp->max_tpa_v2 = le16_to_cpu(resp->max_aggs_supported);
5349 		if (bp->max_tpa_v2)
5350 			bp->hw_ring_stats_size =
5351 				sizeof(struct ctx_hw_stats_ext);
5352 	}
5353 	mutex_unlock(&bp->hwrm_cmd_lock);
5354 	return rc;
5355 }
5356 
5357 static int bnxt_hwrm_ring_grp_alloc(struct bnxt *bp)
5358 {
5359 	u16 i;
5360 	u32 rc = 0;
5361 
5362 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5363 		return 0;
5364 
5365 	mutex_lock(&bp->hwrm_cmd_lock);
5366 	for (i = 0; i < bp->rx_nr_rings; i++) {
5367 		struct hwrm_ring_grp_alloc_input req = {0};
5368 		struct hwrm_ring_grp_alloc_output *resp =
5369 					bp->hwrm_cmd_resp_addr;
5370 		unsigned int grp_idx = bp->rx_ring[i].bnapi->index;
5371 
5372 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_ALLOC, -1, -1);
5373 
5374 		req.cr = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id);
5375 		req.rr = cpu_to_le16(bp->grp_info[grp_idx].rx_fw_ring_id);
5376 		req.ar = cpu_to_le16(bp->grp_info[grp_idx].agg_fw_ring_id);
5377 		req.sc = cpu_to_le16(bp->grp_info[grp_idx].fw_stats_ctx);
5378 
5379 		rc = _hwrm_send_message(bp, &req, sizeof(req),
5380 					HWRM_CMD_TIMEOUT);
5381 		if (rc)
5382 			break;
5383 
5384 		bp->grp_info[grp_idx].fw_grp_id =
5385 			le32_to_cpu(resp->ring_group_id);
5386 	}
5387 	mutex_unlock(&bp->hwrm_cmd_lock);
5388 	return rc;
5389 }
5390 
5391 static void bnxt_hwrm_ring_grp_free(struct bnxt *bp)
5392 {
5393 	u16 i;
5394 	struct hwrm_ring_grp_free_input req = {0};
5395 
5396 	if (!bp->grp_info || (bp->flags & BNXT_FLAG_CHIP_P5))
5397 		return;
5398 
5399 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_GRP_FREE, -1, -1);
5400 
5401 	mutex_lock(&bp->hwrm_cmd_lock);
5402 	for (i = 0; i < bp->cp_nr_rings; i++) {
5403 		if (bp->grp_info[i].fw_grp_id == INVALID_HW_RING_ID)
5404 			continue;
5405 		req.ring_group_id =
5406 			cpu_to_le32(bp->grp_info[i].fw_grp_id);
5407 
5408 		_hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5409 		bp->grp_info[i].fw_grp_id = INVALID_HW_RING_ID;
5410 	}
5411 	mutex_unlock(&bp->hwrm_cmd_lock);
5412 }
5413 
5414 static int hwrm_ring_alloc_send_msg(struct bnxt *bp,
5415 				    struct bnxt_ring_struct *ring,
5416 				    u32 ring_type, u32 map_index)
5417 {
5418 	int rc = 0, err = 0;
5419 	struct hwrm_ring_alloc_input req = {0};
5420 	struct hwrm_ring_alloc_output *resp = bp->hwrm_cmd_resp_addr;
5421 	struct bnxt_ring_mem_info *rmem = &ring->ring_mem;
5422 	struct bnxt_ring_grp_info *grp_info;
5423 	u16 ring_id;
5424 
5425 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_ALLOC, -1, -1);
5426 
5427 	req.enables = 0;
5428 	if (rmem->nr_pages > 1) {
5429 		req.page_tbl_addr = cpu_to_le64(rmem->pg_tbl_map);
5430 		/* Page size is in log2 units */
5431 		req.page_size = BNXT_PAGE_SHIFT;
5432 		req.page_tbl_depth = 1;
5433 	} else {
5434 		req.page_tbl_addr =  cpu_to_le64(rmem->dma_arr[0]);
5435 	}
5436 	req.fbo = 0;
5437 	/* Association of ring index with doorbell index and MSIX number */
5438 	req.logical_id = cpu_to_le16(map_index);
5439 
5440 	switch (ring_type) {
5441 	case HWRM_RING_ALLOC_TX: {
5442 		struct bnxt_tx_ring_info *txr;
5443 
5444 		txr = container_of(ring, struct bnxt_tx_ring_info,
5445 				   tx_ring_struct);
5446 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_TX;
5447 		/* Association of transmit ring with completion ring */
5448 		grp_info = &bp->grp_info[ring->grp_idx];
5449 		req.cmpl_ring_id = cpu_to_le16(bnxt_cp_ring_for_tx(bp, txr));
5450 		req.length = cpu_to_le32(bp->tx_ring_mask + 1);
5451 		req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5452 		req.queue_id = cpu_to_le16(ring->queue_id);
5453 		break;
5454 	}
5455 	case HWRM_RING_ALLOC_RX:
5456 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5457 		req.length = cpu_to_le32(bp->rx_ring_mask + 1);
5458 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5459 			u16 flags = 0;
5460 
5461 			/* Association of rx ring with stats context */
5462 			grp_info = &bp->grp_info[ring->grp_idx];
5463 			req.rx_buf_size = cpu_to_le16(bp->rx_buf_use_size);
5464 			req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5465 			req.enables |= cpu_to_le32(
5466 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5467 			if (NET_IP_ALIGN == 2)
5468 				flags = RING_ALLOC_REQ_FLAGS_RX_SOP_PAD;
5469 			req.flags = cpu_to_le16(flags);
5470 		}
5471 		break;
5472 	case HWRM_RING_ALLOC_AGG:
5473 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5474 			req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG;
5475 			/* Association of agg ring with rx ring */
5476 			grp_info = &bp->grp_info[ring->grp_idx];
5477 			req.rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id);
5478 			req.rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE);
5479 			req.stat_ctx_id = cpu_to_le32(grp_info->fw_stats_ctx);
5480 			req.enables |= cpu_to_le32(
5481 				RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID |
5482 				RING_ALLOC_REQ_ENABLES_RX_BUF_SIZE_VALID);
5483 		} else {
5484 			req.ring_type = RING_ALLOC_REQ_RING_TYPE_RX;
5485 		}
5486 		req.length = cpu_to_le32(bp->rx_agg_ring_mask + 1);
5487 		break;
5488 	case HWRM_RING_ALLOC_CMPL:
5489 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL;
5490 		req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5491 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5492 			/* Association of cp ring with nq */
5493 			grp_info = &bp->grp_info[map_index];
5494 			req.nq_ring_id = cpu_to_le16(grp_info->cp_fw_ring_id);
5495 			req.cq_handle = cpu_to_le64(ring->handle);
5496 			req.enables |= cpu_to_le32(
5497 				RING_ALLOC_REQ_ENABLES_NQ_RING_ID_VALID);
5498 		} else if (bp->flags & BNXT_FLAG_USING_MSIX) {
5499 			req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5500 		}
5501 		break;
5502 	case HWRM_RING_ALLOC_NQ:
5503 		req.ring_type = RING_ALLOC_REQ_RING_TYPE_NQ;
5504 		req.length = cpu_to_le32(bp->cp_ring_mask + 1);
5505 		if (bp->flags & BNXT_FLAG_USING_MSIX)
5506 			req.int_mode = RING_ALLOC_REQ_INT_MODE_MSIX;
5507 		break;
5508 	default:
5509 		netdev_err(bp->dev, "hwrm alloc invalid ring type %d\n",
5510 			   ring_type);
5511 		return -1;
5512 	}
5513 
5514 	mutex_lock(&bp->hwrm_cmd_lock);
5515 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5516 	err = le16_to_cpu(resp->error_code);
5517 	ring_id = le16_to_cpu(resp->ring_id);
5518 	mutex_unlock(&bp->hwrm_cmd_lock);
5519 
5520 	if (rc || err) {
5521 		netdev_err(bp->dev, "hwrm_ring_alloc type %d failed. rc:%x err:%x\n",
5522 			   ring_type, rc, err);
5523 		return -EIO;
5524 	}
5525 	ring->fw_ring_id = ring_id;
5526 	return rc;
5527 }
5528 
5529 static int bnxt_hwrm_set_async_event_cr(struct bnxt *bp, int idx)
5530 {
5531 	int rc;
5532 
5533 	if (BNXT_PF(bp)) {
5534 		struct hwrm_func_cfg_input req = {0};
5535 
5536 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
5537 		req.fid = cpu_to_le16(0xffff);
5538 		req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5539 		req.async_event_cr = cpu_to_le16(idx);
5540 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5541 	} else {
5542 		struct hwrm_func_vf_cfg_input req = {0};
5543 
5544 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_VF_CFG, -1, -1);
5545 		req.enables =
5546 			cpu_to_le32(FUNC_VF_CFG_REQ_ENABLES_ASYNC_EVENT_CR);
5547 		req.async_event_cr = cpu_to_le16(idx);
5548 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5549 	}
5550 	return rc;
5551 }
5552 
5553 static void bnxt_set_db(struct bnxt *bp, struct bnxt_db_info *db, u32 ring_type,
5554 			u32 map_idx, u32 xid)
5555 {
5556 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5557 		if (BNXT_PF(bp))
5558 			db->doorbell = bp->bar1 + DB_PF_OFFSET_P5;
5559 		else
5560 			db->doorbell = bp->bar1 + DB_VF_OFFSET_P5;
5561 		switch (ring_type) {
5562 		case HWRM_RING_ALLOC_TX:
5563 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SQ;
5564 			break;
5565 		case HWRM_RING_ALLOC_RX:
5566 		case HWRM_RING_ALLOC_AGG:
5567 			db->db_key64 = DBR_PATH_L2 | DBR_TYPE_SRQ;
5568 			break;
5569 		case HWRM_RING_ALLOC_CMPL:
5570 			db->db_key64 = DBR_PATH_L2;
5571 			break;
5572 		case HWRM_RING_ALLOC_NQ:
5573 			db->db_key64 = DBR_PATH_L2;
5574 			break;
5575 		}
5576 		db->db_key64 |= (u64)xid << DBR_XID_SFT;
5577 	} else {
5578 		db->doorbell = bp->bar1 + map_idx * 0x80;
5579 		switch (ring_type) {
5580 		case HWRM_RING_ALLOC_TX:
5581 			db->db_key32 = DB_KEY_TX;
5582 			break;
5583 		case HWRM_RING_ALLOC_RX:
5584 		case HWRM_RING_ALLOC_AGG:
5585 			db->db_key32 = DB_KEY_RX;
5586 			break;
5587 		case HWRM_RING_ALLOC_CMPL:
5588 			db->db_key32 = DB_KEY_CP;
5589 			break;
5590 		}
5591 	}
5592 }
5593 
5594 static int bnxt_hwrm_ring_alloc(struct bnxt *bp)
5595 {
5596 	bool agg_rings = !!(bp->flags & BNXT_FLAG_AGG_RINGS);
5597 	int i, rc = 0;
5598 	u32 type;
5599 
5600 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5601 		type = HWRM_RING_ALLOC_NQ;
5602 	else
5603 		type = HWRM_RING_ALLOC_CMPL;
5604 	for (i = 0; i < bp->cp_nr_rings; i++) {
5605 		struct bnxt_napi *bnapi = bp->bnapi[i];
5606 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5607 		struct bnxt_ring_struct *ring = &cpr->cp_ring_struct;
5608 		u32 map_idx = ring->map_idx;
5609 		unsigned int vector;
5610 
5611 		vector = bp->irq_tbl[map_idx].vector;
5612 		disable_irq_nosync(vector);
5613 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5614 		if (rc) {
5615 			enable_irq(vector);
5616 			goto err_out;
5617 		}
5618 		bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id);
5619 		bnxt_db_nq(bp, &cpr->cp_db, cpr->cp_raw_cons);
5620 		enable_irq(vector);
5621 		bp->grp_info[i].cp_fw_ring_id = ring->fw_ring_id;
5622 
5623 		if (!i) {
5624 			rc = bnxt_hwrm_set_async_event_cr(bp, ring->fw_ring_id);
5625 			if (rc)
5626 				netdev_warn(bp->dev, "Failed to set async event completion ring.\n");
5627 		}
5628 	}
5629 
5630 	type = HWRM_RING_ALLOC_TX;
5631 	for (i = 0; i < bp->tx_nr_rings; i++) {
5632 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5633 		struct bnxt_ring_struct *ring;
5634 		u32 map_idx;
5635 
5636 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5637 			struct bnxt_napi *bnapi = txr->bnapi;
5638 			struct bnxt_cp_ring_info *cpr, *cpr2;
5639 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5640 
5641 			cpr = &bnapi->cp_ring;
5642 			cpr2 = cpr->cp_ring_arr[BNXT_TX_HDL];
5643 			ring = &cpr2->cp_ring_struct;
5644 			ring->handle = BNXT_TX_HDL;
5645 			map_idx = bnapi->index;
5646 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5647 			if (rc)
5648 				goto err_out;
5649 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5650 				    ring->fw_ring_id);
5651 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5652 		}
5653 		ring = &txr->tx_ring_struct;
5654 		map_idx = i;
5655 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5656 		if (rc)
5657 			goto err_out;
5658 		bnxt_set_db(bp, &txr->tx_db, type, map_idx, ring->fw_ring_id);
5659 	}
5660 
5661 	type = HWRM_RING_ALLOC_RX;
5662 	for (i = 0; i < bp->rx_nr_rings; i++) {
5663 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5664 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5665 		struct bnxt_napi *bnapi = rxr->bnapi;
5666 		u32 map_idx = bnapi->index;
5667 
5668 		rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5669 		if (rc)
5670 			goto err_out;
5671 		bnxt_set_db(bp, &rxr->rx_db, type, map_idx, ring->fw_ring_id);
5672 		/* If we have agg rings, post agg buffers first. */
5673 		if (!agg_rings)
5674 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5675 		bp->grp_info[map_idx].rx_fw_ring_id = ring->fw_ring_id;
5676 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5677 			struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5678 			u32 type2 = HWRM_RING_ALLOC_CMPL;
5679 			struct bnxt_cp_ring_info *cpr2;
5680 
5681 			cpr2 = cpr->cp_ring_arr[BNXT_RX_HDL];
5682 			ring = &cpr2->cp_ring_struct;
5683 			ring->handle = BNXT_RX_HDL;
5684 			rc = hwrm_ring_alloc_send_msg(bp, ring, type2, map_idx);
5685 			if (rc)
5686 				goto err_out;
5687 			bnxt_set_db(bp, &cpr2->cp_db, type2, map_idx,
5688 				    ring->fw_ring_id);
5689 			bnxt_db_cq(bp, &cpr2->cp_db, cpr2->cp_raw_cons);
5690 		}
5691 	}
5692 
5693 	if (agg_rings) {
5694 		type = HWRM_RING_ALLOC_AGG;
5695 		for (i = 0; i < bp->rx_nr_rings; i++) {
5696 			struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5697 			struct bnxt_ring_struct *ring =
5698 						&rxr->rx_agg_ring_struct;
5699 			u32 grp_idx = ring->grp_idx;
5700 			u32 map_idx = grp_idx + bp->rx_nr_rings;
5701 
5702 			rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx);
5703 			if (rc)
5704 				goto err_out;
5705 
5706 			bnxt_set_db(bp, &rxr->rx_agg_db, type, map_idx,
5707 				    ring->fw_ring_id);
5708 			bnxt_db_write(bp, &rxr->rx_agg_db, rxr->rx_agg_prod);
5709 			bnxt_db_write(bp, &rxr->rx_db, rxr->rx_prod);
5710 			bp->grp_info[grp_idx].agg_fw_ring_id = ring->fw_ring_id;
5711 		}
5712 	}
5713 err_out:
5714 	return rc;
5715 }
5716 
5717 static int hwrm_ring_free_send_msg(struct bnxt *bp,
5718 				   struct bnxt_ring_struct *ring,
5719 				   u32 ring_type, int cmpl_ring_id)
5720 {
5721 	int rc;
5722 	struct hwrm_ring_free_input req = {0};
5723 	struct hwrm_ring_free_output *resp = bp->hwrm_cmd_resp_addr;
5724 	u16 error_code;
5725 
5726 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
5727 		return 0;
5728 
5729 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_FREE, cmpl_ring_id, -1);
5730 	req.ring_type = ring_type;
5731 	req.ring_id = cpu_to_le16(ring->fw_ring_id);
5732 
5733 	mutex_lock(&bp->hwrm_cmd_lock);
5734 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5735 	error_code = le16_to_cpu(resp->error_code);
5736 	mutex_unlock(&bp->hwrm_cmd_lock);
5737 
5738 	if (rc || error_code) {
5739 		netdev_err(bp->dev, "hwrm_ring_free type %d failed. rc:%x err:%x\n",
5740 			   ring_type, rc, error_code);
5741 		return -EIO;
5742 	}
5743 	return 0;
5744 }
5745 
5746 static void bnxt_hwrm_ring_free(struct bnxt *bp, bool close_path)
5747 {
5748 	u32 type;
5749 	int i;
5750 
5751 	if (!bp->bnapi)
5752 		return;
5753 
5754 	for (i = 0; i < bp->tx_nr_rings; i++) {
5755 		struct bnxt_tx_ring_info *txr = &bp->tx_ring[i];
5756 		struct bnxt_ring_struct *ring = &txr->tx_ring_struct;
5757 
5758 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5759 			u32 cmpl_ring_id = bnxt_cp_ring_for_tx(bp, txr);
5760 
5761 			hwrm_ring_free_send_msg(bp, ring,
5762 						RING_FREE_REQ_RING_TYPE_TX,
5763 						close_path ? cmpl_ring_id :
5764 						INVALID_HW_RING_ID);
5765 			ring->fw_ring_id = INVALID_HW_RING_ID;
5766 		}
5767 	}
5768 
5769 	for (i = 0; i < bp->rx_nr_rings; i++) {
5770 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5771 		struct bnxt_ring_struct *ring = &rxr->rx_ring_struct;
5772 		u32 grp_idx = rxr->bnapi->index;
5773 
5774 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5775 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5776 
5777 			hwrm_ring_free_send_msg(bp, ring,
5778 						RING_FREE_REQ_RING_TYPE_RX,
5779 						close_path ? cmpl_ring_id :
5780 						INVALID_HW_RING_ID);
5781 			ring->fw_ring_id = INVALID_HW_RING_ID;
5782 			bp->grp_info[grp_idx].rx_fw_ring_id =
5783 				INVALID_HW_RING_ID;
5784 		}
5785 	}
5786 
5787 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5788 		type = RING_FREE_REQ_RING_TYPE_RX_AGG;
5789 	else
5790 		type = RING_FREE_REQ_RING_TYPE_RX;
5791 	for (i = 0; i < bp->rx_nr_rings; i++) {
5792 		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
5793 		struct bnxt_ring_struct *ring = &rxr->rx_agg_ring_struct;
5794 		u32 grp_idx = rxr->bnapi->index;
5795 
5796 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5797 			u32 cmpl_ring_id = bnxt_cp_ring_for_rx(bp, rxr);
5798 
5799 			hwrm_ring_free_send_msg(bp, ring, type,
5800 						close_path ? cmpl_ring_id :
5801 						INVALID_HW_RING_ID);
5802 			ring->fw_ring_id = INVALID_HW_RING_ID;
5803 			bp->grp_info[grp_idx].agg_fw_ring_id =
5804 				INVALID_HW_RING_ID;
5805 		}
5806 	}
5807 
5808 	/* The completion rings are about to be freed.  After that the
5809 	 * IRQ doorbell will not work anymore.  So we need to disable
5810 	 * IRQ here.
5811 	 */
5812 	bnxt_disable_int_sync(bp);
5813 
5814 	if (bp->flags & BNXT_FLAG_CHIP_P5)
5815 		type = RING_FREE_REQ_RING_TYPE_NQ;
5816 	else
5817 		type = RING_FREE_REQ_RING_TYPE_L2_CMPL;
5818 	for (i = 0; i < bp->cp_nr_rings; i++) {
5819 		struct bnxt_napi *bnapi = bp->bnapi[i];
5820 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
5821 		struct bnxt_ring_struct *ring;
5822 		int j;
5823 
5824 		for (j = 0; j < 2; j++) {
5825 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
5826 
5827 			if (cpr2) {
5828 				ring = &cpr2->cp_ring_struct;
5829 				if (ring->fw_ring_id == INVALID_HW_RING_ID)
5830 					continue;
5831 				hwrm_ring_free_send_msg(bp, ring,
5832 					RING_FREE_REQ_RING_TYPE_L2_CMPL,
5833 					INVALID_HW_RING_ID);
5834 				ring->fw_ring_id = INVALID_HW_RING_ID;
5835 			}
5836 		}
5837 		ring = &cpr->cp_ring_struct;
5838 		if (ring->fw_ring_id != INVALID_HW_RING_ID) {
5839 			hwrm_ring_free_send_msg(bp, ring, type,
5840 						INVALID_HW_RING_ID);
5841 			ring->fw_ring_id = INVALID_HW_RING_ID;
5842 			bp->grp_info[i].cp_fw_ring_id = INVALID_HW_RING_ID;
5843 		}
5844 	}
5845 }
5846 
5847 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
5848 			   bool shared);
5849 
5850 static int bnxt_hwrm_get_rings(struct bnxt *bp)
5851 {
5852 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5853 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
5854 	struct hwrm_func_qcfg_input req = {0};
5855 	int rc;
5856 
5857 	if (bp->hwrm_spec_code < 0x10601)
5858 		return 0;
5859 
5860 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5861 	req.fid = cpu_to_le16(0xffff);
5862 	mutex_lock(&bp->hwrm_cmd_lock);
5863 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5864 	if (rc) {
5865 		mutex_unlock(&bp->hwrm_cmd_lock);
5866 		return rc;
5867 	}
5868 
5869 	hw_resc->resv_tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5870 	if (BNXT_NEW_RM(bp)) {
5871 		u16 cp, stats;
5872 
5873 		hw_resc->resv_rx_rings = le16_to_cpu(resp->alloc_rx_rings);
5874 		hw_resc->resv_hw_ring_grps =
5875 			le32_to_cpu(resp->alloc_hw_ring_grps);
5876 		hw_resc->resv_vnics = le16_to_cpu(resp->alloc_vnics);
5877 		cp = le16_to_cpu(resp->alloc_cmpl_rings);
5878 		stats = le16_to_cpu(resp->alloc_stat_ctx);
5879 		hw_resc->resv_irqs = cp;
5880 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5881 			int rx = hw_resc->resv_rx_rings;
5882 			int tx = hw_resc->resv_tx_rings;
5883 
5884 			if (bp->flags & BNXT_FLAG_AGG_RINGS)
5885 				rx >>= 1;
5886 			if (cp < (rx + tx)) {
5887 				bnxt_trim_rings(bp, &rx, &tx, cp, false);
5888 				if (bp->flags & BNXT_FLAG_AGG_RINGS)
5889 					rx <<= 1;
5890 				hw_resc->resv_rx_rings = rx;
5891 				hw_resc->resv_tx_rings = tx;
5892 			}
5893 			hw_resc->resv_irqs = le16_to_cpu(resp->alloc_msix);
5894 			hw_resc->resv_hw_ring_grps = rx;
5895 		}
5896 		hw_resc->resv_cp_rings = cp;
5897 		hw_resc->resv_stat_ctxs = stats;
5898 	}
5899 	mutex_unlock(&bp->hwrm_cmd_lock);
5900 	return 0;
5901 }
5902 
5903 /* Caller must hold bp->hwrm_cmd_lock */
5904 int __bnxt_hwrm_get_tx_rings(struct bnxt *bp, u16 fid, int *tx_rings)
5905 {
5906 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
5907 	struct hwrm_func_qcfg_input req = {0};
5908 	int rc;
5909 
5910 	if (bp->hwrm_spec_code < 0x10601)
5911 		return 0;
5912 
5913 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
5914 	req.fid = cpu_to_le16(fid);
5915 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
5916 	if (!rc)
5917 		*tx_rings = le16_to_cpu(resp->alloc_tx_rings);
5918 
5919 	return rc;
5920 }
5921 
5922 static bool bnxt_rfs_supported(struct bnxt *bp);
5923 
5924 static void
5925 __bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, struct hwrm_func_cfg_input *req,
5926 			     int tx_rings, int rx_rings, int ring_grps,
5927 			     int cp_rings, int stats, int vnics)
5928 {
5929 	u32 enables = 0;
5930 
5931 	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_CFG, -1, -1);
5932 	req->fid = cpu_to_le16(0xffff);
5933 	enables |= tx_rings ? FUNC_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5934 	req->num_tx_rings = cpu_to_le16(tx_rings);
5935 	if (BNXT_NEW_RM(bp)) {
5936 		enables |= rx_rings ? FUNC_CFG_REQ_ENABLES_NUM_RX_RINGS : 0;
5937 		enables |= stats ? FUNC_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5938 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5939 			enables |= cp_rings ? FUNC_CFG_REQ_ENABLES_NUM_MSIX : 0;
5940 			enables |= tx_rings + ring_grps ?
5941 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5942 			enables |= rx_rings ?
5943 				FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5944 		} else {
5945 			enables |= cp_rings ?
5946 				   FUNC_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5947 			enables |= ring_grps ?
5948 				   FUNC_CFG_REQ_ENABLES_NUM_HW_RING_GRPS |
5949 				   FUNC_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5950 		}
5951 		enables |= vnics ? FUNC_CFG_REQ_ENABLES_NUM_VNICS : 0;
5952 
5953 		req->num_rx_rings = cpu_to_le16(rx_rings);
5954 		if (bp->flags & BNXT_FLAG_CHIP_P5) {
5955 			req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
5956 			req->num_msix = cpu_to_le16(cp_rings);
5957 			req->num_rsscos_ctxs =
5958 				cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
5959 		} else {
5960 			req->num_cmpl_rings = cpu_to_le16(cp_rings);
5961 			req->num_hw_ring_grps = cpu_to_le16(ring_grps);
5962 			req->num_rsscos_ctxs = cpu_to_le16(1);
5963 			if (!(bp->flags & BNXT_FLAG_NEW_RSS_CAP) &&
5964 			    bnxt_rfs_supported(bp))
5965 				req->num_rsscos_ctxs =
5966 					cpu_to_le16(ring_grps + 1);
5967 		}
5968 		req->num_stat_ctxs = cpu_to_le16(stats);
5969 		req->num_vnics = cpu_to_le16(vnics);
5970 	}
5971 	req->enables = cpu_to_le32(enables);
5972 }
5973 
5974 static void
5975 __bnxt_hwrm_reserve_vf_rings(struct bnxt *bp,
5976 			     struct hwrm_func_vf_cfg_input *req, int tx_rings,
5977 			     int rx_rings, int ring_grps, int cp_rings,
5978 			     int stats, int vnics)
5979 {
5980 	u32 enables = 0;
5981 
5982 	bnxt_hwrm_cmd_hdr_init(bp, req, HWRM_FUNC_VF_CFG, -1, -1);
5983 	enables |= tx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_TX_RINGS : 0;
5984 	enables |= rx_rings ? FUNC_VF_CFG_REQ_ENABLES_NUM_RX_RINGS |
5985 			      FUNC_VF_CFG_REQ_ENABLES_NUM_RSSCOS_CTXS : 0;
5986 	enables |= stats ? FUNC_VF_CFG_REQ_ENABLES_NUM_STAT_CTXS : 0;
5987 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
5988 		enables |= tx_rings + ring_grps ?
5989 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5990 	} else {
5991 		enables |= cp_rings ?
5992 			   FUNC_VF_CFG_REQ_ENABLES_NUM_CMPL_RINGS : 0;
5993 		enables |= ring_grps ?
5994 			   FUNC_VF_CFG_REQ_ENABLES_NUM_HW_RING_GRPS : 0;
5995 	}
5996 	enables |= vnics ? FUNC_VF_CFG_REQ_ENABLES_NUM_VNICS : 0;
5997 	enables |= FUNC_VF_CFG_REQ_ENABLES_NUM_L2_CTXS;
5998 
5999 	req->num_l2_ctxs = cpu_to_le16(BNXT_VF_MAX_L2_CTX);
6000 	req->num_tx_rings = cpu_to_le16(tx_rings);
6001 	req->num_rx_rings = cpu_to_le16(rx_rings);
6002 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6003 		req->num_cmpl_rings = cpu_to_le16(tx_rings + ring_grps);
6004 		req->num_rsscos_ctxs = cpu_to_le16(DIV_ROUND_UP(ring_grps, 64));
6005 	} else {
6006 		req->num_cmpl_rings = cpu_to_le16(cp_rings);
6007 		req->num_hw_ring_grps = cpu_to_le16(ring_grps);
6008 		req->num_rsscos_ctxs = cpu_to_le16(BNXT_VF_MAX_RSS_CTX);
6009 	}
6010 	req->num_stat_ctxs = cpu_to_le16(stats);
6011 	req->num_vnics = cpu_to_le16(vnics);
6012 
6013 	req->enables = cpu_to_le32(enables);
6014 }
6015 
6016 static int
6017 bnxt_hwrm_reserve_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6018 			   int ring_grps, int cp_rings, int stats, int vnics)
6019 {
6020 	struct hwrm_func_cfg_input req = {0};
6021 	int rc;
6022 
6023 	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6024 				     cp_rings, stats, vnics);
6025 	if (!req.enables)
6026 		return 0;
6027 
6028 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6029 	if (rc)
6030 		return rc;
6031 
6032 	if (bp->hwrm_spec_code < 0x10601)
6033 		bp->hw_resc.resv_tx_rings = tx_rings;
6034 
6035 	return bnxt_hwrm_get_rings(bp);
6036 }
6037 
6038 static int
6039 bnxt_hwrm_reserve_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6040 			   int ring_grps, int cp_rings, int stats, int vnics)
6041 {
6042 	struct hwrm_func_vf_cfg_input req = {0};
6043 	int rc;
6044 
6045 	if (!BNXT_NEW_RM(bp)) {
6046 		bp->hw_resc.resv_tx_rings = tx_rings;
6047 		return 0;
6048 	}
6049 
6050 	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6051 				     cp_rings, stats, vnics);
6052 	rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6053 	if (rc)
6054 		return rc;
6055 
6056 	return bnxt_hwrm_get_rings(bp);
6057 }
6058 
6059 static int bnxt_hwrm_reserve_rings(struct bnxt *bp, int tx, int rx, int grp,
6060 				   int cp, int stat, int vnic)
6061 {
6062 	if (BNXT_PF(bp))
6063 		return bnxt_hwrm_reserve_pf_rings(bp, tx, rx, grp, cp, stat,
6064 						  vnic);
6065 	else
6066 		return bnxt_hwrm_reserve_vf_rings(bp, tx, rx, grp, cp, stat,
6067 						  vnic);
6068 }
6069 
6070 int bnxt_nq_rings_in_use(struct bnxt *bp)
6071 {
6072 	int cp = bp->cp_nr_rings;
6073 	int ulp_msix, ulp_base;
6074 
6075 	ulp_msix = bnxt_get_ulp_msix_num(bp);
6076 	if (ulp_msix) {
6077 		ulp_base = bnxt_get_ulp_msix_base(bp);
6078 		cp += ulp_msix;
6079 		if ((ulp_base + ulp_msix) > cp)
6080 			cp = ulp_base + ulp_msix;
6081 	}
6082 	return cp;
6083 }
6084 
6085 static int bnxt_cp_rings_in_use(struct bnxt *bp)
6086 {
6087 	int cp;
6088 
6089 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6090 		return bnxt_nq_rings_in_use(bp);
6091 
6092 	cp = bp->tx_nr_rings + bp->rx_nr_rings;
6093 	return cp;
6094 }
6095 
6096 static int bnxt_get_func_stat_ctxs(struct bnxt *bp)
6097 {
6098 	int ulp_stat = bnxt_get_ulp_stat_ctxs(bp);
6099 	int cp = bp->cp_nr_rings;
6100 
6101 	if (!ulp_stat)
6102 		return cp;
6103 
6104 	if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp))
6105 		return bnxt_get_ulp_msix_base(bp) + ulp_stat;
6106 
6107 	return cp + ulp_stat;
6108 }
6109 
6110 /* Check if a default RSS map needs to be setup.  This function is only
6111  * used on older firmware that does not require reserving RX rings.
6112  */
6113 static void bnxt_check_rss_tbl_no_rmgr(struct bnxt *bp)
6114 {
6115 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6116 
6117 	/* The RSS map is valid for RX rings set to resv_rx_rings */
6118 	if (hw_resc->resv_rx_rings != bp->rx_nr_rings) {
6119 		hw_resc->resv_rx_rings = bp->rx_nr_rings;
6120 		if (!netif_is_rxfh_configured(bp->dev))
6121 			bnxt_set_dflt_rss_indir_tbl(bp);
6122 	}
6123 }
6124 
6125 static bool bnxt_need_reserve_rings(struct bnxt *bp)
6126 {
6127 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6128 	int cp = bnxt_cp_rings_in_use(bp);
6129 	int nq = bnxt_nq_rings_in_use(bp);
6130 	int rx = bp->rx_nr_rings, stat;
6131 	int vnic = 1, grp = rx;
6132 
6133 	if (hw_resc->resv_tx_rings != bp->tx_nr_rings &&
6134 	    bp->hwrm_spec_code >= 0x10601)
6135 		return true;
6136 
6137 	/* Old firmware does not need RX ring reservations but we still
6138 	 * need to setup a default RSS map when needed.  With new firmware
6139 	 * we go through RX ring reservations first and then set up the
6140 	 * RSS map for the successfully reserved RX rings when needed.
6141 	 */
6142 	if (!BNXT_NEW_RM(bp)) {
6143 		bnxt_check_rss_tbl_no_rmgr(bp);
6144 		return false;
6145 	}
6146 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6147 		vnic = rx + 1;
6148 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6149 		rx <<= 1;
6150 	stat = bnxt_get_func_stat_ctxs(bp);
6151 	if (hw_resc->resv_rx_rings != rx || hw_resc->resv_cp_rings != cp ||
6152 	    hw_resc->resv_vnics != vnic || hw_resc->resv_stat_ctxs != stat ||
6153 	    (hw_resc->resv_hw_ring_grps != grp &&
6154 	     !(bp->flags & BNXT_FLAG_CHIP_P5)))
6155 		return true;
6156 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && BNXT_PF(bp) &&
6157 	    hw_resc->resv_irqs != nq)
6158 		return true;
6159 	return false;
6160 }
6161 
6162 static int __bnxt_reserve_rings(struct bnxt *bp)
6163 {
6164 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
6165 	int cp = bnxt_nq_rings_in_use(bp);
6166 	int tx = bp->tx_nr_rings;
6167 	int rx = bp->rx_nr_rings;
6168 	int grp, rx_rings, rc;
6169 	int vnic = 1, stat;
6170 	bool sh = false;
6171 
6172 	if (!bnxt_need_reserve_rings(bp))
6173 		return 0;
6174 
6175 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
6176 		sh = true;
6177 	if ((bp->flags & BNXT_FLAG_RFS) && !(bp->flags & BNXT_FLAG_CHIP_P5))
6178 		vnic = rx + 1;
6179 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6180 		rx <<= 1;
6181 	grp = bp->rx_nr_rings;
6182 	stat = bnxt_get_func_stat_ctxs(bp);
6183 
6184 	rc = bnxt_hwrm_reserve_rings(bp, tx, rx, grp, cp, stat, vnic);
6185 	if (rc)
6186 		return rc;
6187 
6188 	tx = hw_resc->resv_tx_rings;
6189 	if (BNXT_NEW_RM(bp)) {
6190 		rx = hw_resc->resv_rx_rings;
6191 		cp = hw_resc->resv_irqs;
6192 		grp = hw_resc->resv_hw_ring_grps;
6193 		vnic = hw_resc->resv_vnics;
6194 		stat = hw_resc->resv_stat_ctxs;
6195 	}
6196 
6197 	rx_rings = rx;
6198 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
6199 		if (rx >= 2) {
6200 			rx_rings = rx >> 1;
6201 		} else {
6202 			if (netif_running(bp->dev))
6203 				return -ENOMEM;
6204 
6205 			bp->flags &= ~BNXT_FLAG_AGG_RINGS;
6206 			bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
6207 			bp->dev->hw_features &= ~NETIF_F_LRO;
6208 			bp->dev->features &= ~NETIF_F_LRO;
6209 			bnxt_set_ring_params(bp);
6210 		}
6211 	}
6212 	rx_rings = min_t(int, rx_rings, grp);
6213 	cp = min_t(int, cp, bp->cp_nr_rings);
6214 	if (stat > bnxt_get_ulp_stat_ctxs(bp))
6215 		stat -= bnxt_get_ulp_stat_ctxs(bp);
6216 	cp = min_t(int, cp, stat);
6217 	rc = bnxt_trim_rings(bp, &rx_rings, &tx, cp, sh);
6218 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
6219 		rx = rx_rings << 1;
6220 	cp = sh ? max_t(int, tx, rx_rings) : tx + rx_rings;
6221 	bp->tx_nr_rings = tx;
6222 
6223 	/* If we cannot reserve all the RX rings, reset the RSS map only
6224 	 * if absolutely necessary
6225 	 */
6226 	if (rx_rings != bp->rx_nr_rings) {
6227 		netdev_warn(bp->dev, "Able to reserve only %d out of %d requested RX rings\n",
6228 			    rx_rings, bp->rx_nr_rings);
6229 		if ((bp->dev->priv_flags & IFF_RXFH_CONFIGURED) &&
6230 		    (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) !=
6231 		     bnxt_get_nr_rss_ctxs(bp, rx_rings) ||
6232 		     bnxt_get_max_rss_ring(bp) >= rx_rings)) {
6233 			netdev_warn(bp->dev, "RSS table entries reverting to default\n");
6234 			bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
6235 		}
6236 	}
6237 	bp->rx_nr_rings = rx_rings;
6238 	bp->cp_nr_rings = cp;
6239 
6240 	if (!tx || !rx || !cp || !grp || !vnic || !stat)
6241 		return -ENOMEM;
6242 
6243 	if (!netif_is_rxfh_configured(bp->dev))
6244 		bnxt_set_dflt_rss_indir_tbl(bp);
6245 
6246 	return rc;
6247 }
6248 
6249 static int bnxt_hwrm_check_vf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6250 				    int ring_grps, int cp_rings, int stats,
6251 				    int vnics)
6252 {
6253 	struct hwrm_func_vf_cfg_input req = {0};
6254 	u32 flags;
6255 
6256 	if (!BNXT_NEW_RM(bp))
6257 		return 0;
6258 
6259 	__bnxt_hwrm_reserve_vf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6260 				     cp_rings, stats, vnics);
6261 	flags = FUNC_VF_CFG_REQ_FLAGS_TX_ASSETS_TEST |
6262 		FUNC_VF_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6263 		FUNC_VF_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6264 		FUNC_VF_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6265 		FUNC_VF_CFG_REQ_FLAGS_VNIC_ASSETS_TEST |
6266 		FUNC_VF_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST;
6267 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6268 		flags |= FUNC_VF_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6269 
6270 	req.flags = cpu_to_le32(flags);
6271 	return hwrm_send_message_silent(bp, &req, sizeof(req),
6272 					HWRM_CMD_TIMEOUT);
6273 }
6274 
6275 static int bnxt_hwrm_check_pf_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6276 				    int ring_grps, int cp_rings, int stats,
6277 				    int vnics)
6278 {
6279 	struct hwrm_func_cfg_input req = {0};
6280 	u32 flags;
6281 
6282 	__bnxt_hwrm_reserve_pf_rings(bp, &req, tx_rings, rx_rings, ring_grps,
6283 				     cp_rings, stats, vnics);
6284 	flags = FUNC_CFG_REQ_FLAGS_TX_ASSETS_TEST;
6285 	if (BNXT_NEW_RM(bp)) {
6286 		flags |= FUNC_CFG_REQ_FLAGS_RX_ASSETS_TEST |
6287 			 FUNC_CFG_REQ_FLAGS_CMPL_ASSETS_TEST |
6288 			 FUNC_CFG_REQ_FLAGS_STAT_CTX_ASSETS_TEST |
6289 			 FUNC_CFG_REQ_FLAGS_VNIC_ASSETS_TEST;
6290 		if (bp->flags & BNXT_FLAG_CHIP_P5)
6291 			flags |= FUNC_CFG_REQ_FLAGS_RSSCOS_CTX_ASSETS_TEST |
6292 				 FUNC_CFG_REQ_FLAGS_NQ_ASSETS_TEST;
6293 		else
6294 			flags |= FUNC_CFG_REQ_FLAGS_RING_GRP_ASSETS_TEST;
6295 	}
6296 
6297 	req.flags = cpu_to_le32(flags);
6298 	return hwrm_send_message_silent(bp, &req, sizeof(req),
6299 					HWRM_CMD_TIMEOUT);
6300 }
6301 
6302 static int bnxt_hwrm_check_rings(struct bnxt *bp, int tx_rings, int rx_rings,
6303 				 int ring_grps, int cp_rings, int stats,
6304 				 int vnics)
6305 {
6306 	if (bp->hwrm_spec_code < 0x10801)
6307 		return 0;
6308 
6309 	if (BNXT_PF(bp))
6310 		return bnxt_hwrm_check_pf_rings(bp, tx_rings, rx_rings,
6311 						ring_grps, cp_rings, stats,
6312 						vnics);
6313 
6314 	return bnxt_hwrm_check_vf_rings(bp, tx_rings, rx_rings, ring_grps,
6315 					cp_rings, stats, vnics);
6316 }
6317 
6318 static void bnxt_hwrm_coal_params_qcaps(struct bnxt *bp)
6319 {
6320 	struct hwrm_ring_aggint_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
6321 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6322 	struct hwrm_ring_aggint_qcaps_input req = {0};
6323 	int rc;
6324 
6325 	coal_cap->cmpl_params = BNXT_LEGACY_COAL_CMPL_PARAMS;
6326 	coal_cap->num_cmpl_dma_aggr_max = 63;
6327 	coal_cap->num_cmpl_dma_aggr_during_int_max = 63;
6328 	coal_cap->cmpl_aggr_dma_tmr_max = 65535;
6329 	coal_cap->cmpl_aggr_dma_tmr_during_int_max = 65535;
6330 	coal_cap->int_lat_tmr_min_max = 65535;
6331 	coal_cap->int_lat_tmr_max_max = 65535;
6332 	coal_cap->num_cmpl_aggr_int_max = 65535;
6333 	coal_cap->timer_units = 80;
6334 
6335 	if (bp->hwrm_spec_code < 0x10902)
6336 		return;
6337 
6338 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_AGGINT_QCAPS, -1, -1);
6339 	mutex_lock(&bp->hwrm_cmd_lock);
6340 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6341 	if (!rc) {
6342 		coal_cap->cmpl_params = le32_to_cpu(resp->cmpl_params);
6343 		coal_cap->nq_params = le32_to_cpu(resp->nq_params);
6344 		coal_cap->num_cmpl_dma_aggr_max =
6345 			le16_to_cpu(resp->num_cmpl_dma_aggr_max);
6346 		coal_cap->num_cmpl_dma_aggr_during_int_max =
6347 			le16_to_cpu(resp->num_cmpl_dma_aggr_during_int_max);
6348 		coal_cap->cmpl_aggr_dma_tmr_max =
6349 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_max);
6350 		coal_cap->cmpl_aggr_dma_tmr_during_int_max =
6351 			le16_to_cpu(resp->cmpl_aggr_dma_tmr_during_int_max);
6352 		coal_cap->int_lat_tmr_min_max =
6353 			le16_to_cpu(resp->int_lat_tmr_min_max);
6354 		coal_cap->int_lat_tmr_max_max =
6355 			le16_to_cpu(resp->int_lat_tmr_max_max);
6356 		coal_cap->num_cmpl_aggr_int_max =
6357 			le16_to_cpu(resp->num_cmpl_aggr_int_max);
6358 		coal_cap->timer_units = le16_to_cpu(resp->timer_units);
6359 	}
6360 	mutex_unlock(&bp->hwrm_cmd_lock);
6361 }
6362 
6363 static u16 bnxt_usec_to_coal_tmr(struct bnxt *bp, u16 usec)
6364 {
6365 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6366 
6367 	return usec * 1000 / coal_cap->timer_units;
6368 }
6369 
6370 static void bnxt_hwrm_set_coal_params(struct bnxt *bp,
6371 	struct bnxt_coal *hw_coal,
6372 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req)
6373 {
6374 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6375 	u32 cmpl_params = coal_cap->cmpl_params;
6376 	u16 val, tmr, max, flags = 0;
6377 
6378 	max = hw_coal->bufs_per_record * 128;
6379 	if (hw_coal->budget)
6380 		max = hw_coal->bufs_per_record * hw_coal->budget;
6381 	max = min_t(u16, max, coal_cap->num_cmpl_aggr_int_max);
6382 
6383 	val = clamp_t(u16, hw_coal->coal_bufs, 1, max);
6384 	req->num_cmpl_aggr_int = cpu_to_le16(val);
6385 
6386 	val = min_t(u16, val, coal_cap->num_cmpl_dma_aggr_max);
6387 	req->num_cmpl_dma_aggr = cpu_to_le16(val);
6388 
6389 	val = clamp_t(u16, hw_coal->coal_bufs_irq, 1,
6390 		      coal_cap->num_cmpl_dma_aggr_during_int_max);
6391 	req->num_cmpl_dma_aggr_during_int = cpu_to_le16(val);
6392 
6393 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks);
6394 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_max_max);
6395 	req->int_lat_tmr_max = cpu_to_le16(tmr);
6396 
6397 	/* min timer set to 1/2 of interrupt timer */
6398 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_INT_LAT_TMR_MIN) {
6399 		val = tmr / 2;
6400 		val = clamp_t(u16, val, 1, coal_cap->int_lat_tmr_min_max);
6401 		req->int_lat_tmr_min = cpu_to_le16(val);
6402 		req->enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6403 	}
6404 
6405 	/* buf timer set to 1/4 of interrupt timer */
6406 	val = clamp_t(u16, tmr / 4, 1, coal_cap->cmpl_aggr_dma_tmr_max);
6407 	req->cmpl_aggr_dma_tmr = cpu_to_le16(val);
6408 
6409 	if (cmpl_params &
6410 	    RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_NUM_CMPL_DMA_AGGR_DURING_INT) {
6411 		tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks_irq);
6412 		val = clamp_t(u16, tmr, 1,
6413 			      coal_cap->cmpl_aggr_dma_tmr_during_int_max);
6414 		req->cmpl_aggr_dma_tmr_during_int = cpu_to_le16(val);
6415 		req->enables |=
6416 			cpu_to_le16(BNXT_COAL_CMPL_AGGR_TMR_DURING_INT_ENABLE);
6417 	}
6418 
6419 	if (cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_TIMER_RESET)
6420 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_TIMER_RESET;
6421 	if ((cmpl_params & RING_AGGINT_QCAPS_RESP_CMPL_PARAMS_RING_IDLE) &&
6422 	    hw_coal->idle_thresh && hw_coal->coal_ticks < hw_coal->idle_thresh)
6423 		flags |= RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_RING_IDLE;
6424 	req->flags = cpu_to_le16(flags);
6425 	req->enables |= cpu_to_le16(BNXT_COAL_CMPL_ENABLES);
6426 }
6427 
6428 /* Caller holds bp->hwrm_cmd_lock */
6429 static int __bnxt_hwrm_set_coal_nq(struct bnxt *bp, struct bnxt_napi *bnapi,
6430 				   struct bnxt_coal *hw_coal)
6431 {
6432 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req = {0};
6433 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6434 	struct bnxt_coal_cap *coal_cap = &bp->coal_cap;
6435 	u32 nq_params = coal_cap->nq_params;
6436 	u16 tmr;
6437 
6438 	if (!(nq_params & RING_AGGINT_QCAPS_RESP_NQ_PARAMS_INT_LAT_TMR_MIN))
6439 		return 0;
6440 
6441 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS,
6442 			       -1, -1);
6443 	req.ring_id = cpu_to_le16(cpr->cp_ring_struct.fw_ring_id);
6444 	req.flags =
6445 		cpu_to_le16(RING_CMPL_RING_CFG_AGGINT_PARAMS_REQ_FLAGS_IS_NQ);
6446 
6447 	tmr = bnxt_usec_to_coal_tmr(bp, hw_coal->coal_ticks) / 2;
6448 	tmr = clamp_t(u16, tmr, 1, coal_cap->int_lat_tmr_min_max);
6449 	req.int_lat_tmr_min = cpu_to_le16(tmr);
6450 	req.enables |= cpu_to_le16(BNXT_COAL_CMPL_MIN_TMR_ENABLE);
6451 	return _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6452 }
6453 
6454 int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi)
6455 {
6456 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0};
6457 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6458 	struct bnxt_coal coal;
6459 
6460 	/* Tick values in micro seconds.
6461 	 * 1 coal_buf x bufs_per_record = 1 completion record.
6462 	 */
6463 	memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal));
6464 
6465 	coal.coal_ticks = cpr->rx_ring_coal.coal_ticks;
6466 	coal.coal_bufs = cpr->rx_ring_coal.coal_bufs;
6467 
6468 	if (!bnapi->rx_ring)
6469 		return -ENODEV;
6470 
6471 	bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6472 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6473 
6474 	bnxt_hwrm_set_coal_params(bp, &coal, &req_rx);
6475 
6476 	req_rx.ring_id = cpu_to_le16(bnxt_cp_ring_for_rx(bp, bnapi->rx_ring));
6477 
6478 	return hwrm_send_message(bp, &req_rx, sizeof(req_rx),
6479 				 HWRM_CMD_TIMEOUT);
6480 }
6481 
6482 int bnxt_hwrm_set_coal(struct bnxt *bp)
6483 {
6484 	int i, rc = 0;
6485 	struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0},
6486 							   req_tx = {0}, *req;
6487 
6488 	bnxt_hwrm_cmd_hdr_init(bp, &req_rx,
6489 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6490 	bnxt_hwrm_cmd_hdr_init(bp, &req_tx,
6491 			       HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1);
6492 
6493 	bnxt_hwrm_set_coal_params(bp, &bp->rx_coal, &req_rx);
6494 	bnxt_hwrm_set_coal_params(bp, &bp->tx_coal, &req_tx);
6495 
6496 	mutex_lock(&bp->hwrm_cmd_lock);
6497 	for (i = 0; i < bp->cp_nr_rings; i++) {
6498 		struct bnxt_napi *bnapi = bp->bnapi[i];
6499 		struct bnxt_coal *hw_coal;
6500 		u16 ring_id;
6501 
6502 		req = &req_rx;
6503 		if (!bnapi->rx_ring) {
6504 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6505 			req = &req_tx;
6506 		} else {
6507 			ring_id = bnxt_cp_ring_for_rx(bp, bnapi->rx_ring);
6508 		}
6509 		req->ring_id = cpu_to_le16(ring_id);
6510 
6511 		rc = _hwrm_send_message(bp, req, sizeof(*req),
6512 					HWRM_CMD_TIMEOUT);
6513 		if (rc)
6514 			break;
6515 
6516 		if (!(bp->flags & BNXT_FLAG_CHIP_P5))
6517 			continue;
6518 
6519 		if (bnapi->rx_ring && bnapi->tx_ring) {
6520 			req = &req_tx;
6521 			ring_id = bnxt_cp_ring_for_tx(bp, bnapi->tx_ring);
6522 			req->ring_id = cpu_to_le16(ring_id);
6523 			rc = _hwrm_send_message(bp, req, sizeof(*req),
6524 						HWRM_CMD_TIMEOUT);
6525 			if (rc)
6526 				break;
6527 		}
6528 		if (bnapi->rx_ring)
6529 			hw_coal = &bp->rx_coal;
6530 		else
6531 			hw_coal = &bp->tx_coal;
6532 		__bnxt_hwrm_set_coal_nq(bp, bnapi, hw_coal);
6533 	}
6534 	mutex_unlock(&bp->hwrm_cmd_lock);
6535 	return rc;
6536 }
6537 
6538 static void bnxt_hwrm_stat_ctx_free(struct bnxt *bp)
6539 {
6540 	struct hwrm_stat_ctx_clr_stats_input req0 = {0};
6541 	struct hwrm_stat_ctx_free_input req = {0};
6542 	int i;
6543 
6544 	if (!bp->bnapi)
6545 		return;
6546 
6547 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6548 		return;
6549 
6550 	bnxt_hwrm_cmd_hdr_init(bp, &req0, HWRM_STAT_CTX_CLR_STATS, -1, -1);
6551 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_FREE, -1, -1);
6552 
6553 	mutex_lock(&bp->hwrm_cmd_lock);
6554 	for (i = 0; i < bp->cp_nr_rings; i++) {
6555 		struct bnxt_napi *bnapi = bp->bnapi[i];
6556 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6557 
6558 		if (cpr->hw_stats_ctx_id != INVALID_STATS_CTX_ID) {
6559 			req.stat_ctx_id = cpu_to_le32(cpr->hw_stats_ctx_id);
6560 			if (BNXT_FW_MAJ(bp) <= 20) {
6561 				req0.stat_ctx_id = req.stat_ctx_id;
6562 				_hwrm_send_message(bp, &req0, sizeof(req0),
6563 						   HWRM_CMD_TIMEOUT);
6564 			}
6565 			_hwrm_send_message(bp, &req, sizeof(req),
6566 					   HWRM_CMD_TIMEOUT);
6567 
6568 			cpr->hw_stats_ctx_id = INVALID_STATS_CTX_ID;
6569 		}
6570 	}
6571 	mutex_unlock(&bp->hwrm_cmd_lock);
6572 }
6573 
6574 static int bnxt_hwrm_stat_ctx_alloc(struct bnxt *bp)
6575 {
6576 	int rc = 0, i;
6577 	struct hwrm_stat_ctx_alloc_input req = {0};
6578 	struct hwrm_stat_ctx_alloc_output *resp = bp->hwrm_cmd_resp_addr;
6579 
6580 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
6581 		return 0;
6582 
6583 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_STAT_CTX_ALLOC, -1, -1);
6584 
6585 	req.stats_dma_length = cpu_to_le16(bp->hw_ring_stats_size);
6586 	req.update_period_ms = cpu_to_le32(bp->stats_coal_ticks / 1000);
6587 
6588 	mutex_lock(&bp->hwrm_cmd_lock);
6589 	for (i = 0; i < bp->cp_nr_rings; i++) {
6590 		struct bnxt_napi *bnapi = bp->bnapi[i];
6591 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
6592 
6593 		req.stats_dma_addr = cpu_to_le64(cpr->stats.hw_stats_map);
6594 
6595 		rc = _hwrm_send_message(bp, &req, sizeof(req),
6596 					HWRM_CMD_TIMEOUT);
6597 		if (rc)
6598 			break;
6599 
6600 		cpr->hw_stats_ctx_id = le32_to_cpu(resp->stat_ctx_id);
6601 
6602 		bp->grp_info[i].fw_stats_ctx = cpr->hw_stats_ctx_id;
6603 	}
6604 	mutex_unlock(&bp->hwrm_cmd_lock);
6605 	return rc;
6606 }
6607 
6608 static int bnxt_hwrm_func_qcfg(struct bnxt *bp)
6609 {
6610 	struct hwrm_func_qcfg_input req = {0};
6611 	struct hwrm_func_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
6612 	u32 min_db_offset = 0;
6613 	u16 flags;
6614 	int rc;
6615 
6616 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCFG, -1, -1);
6617 	req.fid = cpu_to_le16(0xffff);
6618 	mutex_lock(&bp->hwrm_cmd_lock);
6619 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6620 	if (rc)
6621 		goto func_qcfg_exit;
6622 
6623 #ifdef CONFIG_BNXT_SRIOV
6624 	if (BNXT_VF(bp)) {
6625 		struct bnxt_vf_info *vf = &bp->vf;
6626 
6627 		vf->vlan = le16_to_cpu(resp->vlan) & VLAN_VID_MASK;
6628 	} else {
6629 		bp->pf.registered_vfs = le16_to_cpu(resp->registered_vfs);
6630 	}
6631 #endif
6632 	flags = le16_to_cpu(resp->flags);
6633 	if (flags & (FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED |
6634 		     FUNC_QCFG_RESP_FLAGS_FW_LLDP_AGENT_ENABLED)) {
6635 		bp->fw_cap |= BNXT_FW_CAP_LLDP_AGENT;
6636 		if (flags & FUNC_QCFG_RESP_FLAGS_FW_DCBX_AGENT_ENABLED)
6637 			bp->fw_cap |= BNXT_FW_CAP_DCBX_AGENT;
6638 	}
6639 	if (BNXT_PF(bp) && (flags & FUNC_QCFG_RESP_FLAGS_MULTI_HOST))
6640 		bp->flags |= BNXT_FLAG_MULTI_HOST;
6641 
6642 	switch (resp->port_partition_type) {
6643 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_0:
6644 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR1_5:
6645 	case FUNC_QCFG_RESP_PORT_PARTITION_TYPE_NPAR2_0:
6646 		bp->port_partition_type = resp->port_partition_type;
6647 		break;
6648 	}
6649 	if (bp->hwrm_spec_code < 0x10707 ||
6650 	    resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEB)
6651 		bp->br_mode = BRIDGE_MODE_VEB;
6652 	else if (resp->evb_mode == FUNC_QCFG_RESP_EVB_MODE_VEPA)
6653 		bp->br_mode = BRIDGE_MODE_VEPA;
6654 	else
6655 		bp->br_mode = BRIDGE_MODE_UNDEF;
6656 
6657 	bp->max_mtu = le16_to_cpu(resp->max_mtu_configured);
6658 	if (!bp->max_mtu)
6659 		bp->max_mtu = BNXT_MAX_MTU;
6660 
6661 	if (bp->db_size)
6662 		goto func_qcfg_exit;
6663 
6664 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
6665 		if (BNXT_PF(bp))
6666 			min_db_offset = DB_PF_OFFSET_P5;
6667 		else
6668 			min_db_offset = DB_VF_OFFSET_P5;
6669 	}
6670 	bp->db_size = PAGE_ALIGN(le16_to_cpu(resp->l2_doorbell_bar_size_kb) *
6671 				 1024);
6672 	if (!bp->db_size || bp->db_size > pci_resource_len(bp->pdev, 2) ||
6673 	    bp->db_size <= min_db_offset)
6674 		bp->db_size = pci_resource_len(bp->pdev, 2);
6675 
6676 func_qcfg_exit:
6677 	mutex_unlock(&bp->hwrm_cmd_lock);
6678 	return rc;
6679 }
6680 
6681 static int bnxt_hwrm_func_backing_store_qcaps(struct bnxt *bp)
6682 {
6683 	struct hwrm_func_backing_store_qcaps_input req = {0};
6684 	struct hwrm_func_backing_store_qcaps_output *resp =
6685 		bp->hwrm_cmd_resp_addr;
6686 	int rc;
6687 
6688 	if (bp->hwrm_spec_code < 0x10902 || BNXT_VF(bp) || bp->ctx)
6689 		return 0;
6690 
6691 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_QCAPS, -1, -1);
6692 	mutex_lock(&bp->hwrm_cmd_lock);
6693 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6694 	if (!rc) {
6695 		struct bnxt_ctx_pg_info *ctx_pg;
6696 		struct bnxt_ctx_mem_info *ctx;
6697 		int i, tqm_rings;
6698 
6699 		ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
6700 		if (!ctx) {
6701 			rc = -ENOMEM;
6702 			goto ctx_err;
6703 		}
6704 		ctx->qp_max_entries = le32_to_cpu(resp->qp_max_entries);
6705 		ctx->qp_min_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries);
6706 		ctx->qp_max_l2_entries = le16_to_cpu(resp->qp_max_l2_entries);
6707 		ctx->qp_entry_size = le16_to_cpu(resp->qp_entry_size);
6708 		ctx->srq_max_l2_entries = le16_to_cpu(resp->srq_max_l2_entries);
6709 		ctx->srq_max_entries = le32_to_cpu(resp->srq_max_entries);
6710 		ctx->srq_entry_size = le16_to_cpu(resp->srq_entry_size);
6711 		ctx->cq_max_l2_entries = le16_to_cpu(resp->cq_max_l2_entries);
6712 		ctx->cq_max_entries = le32_to_cpu(resp->cq_max_entries);
6713 		ctx->cq_entry_size = le16_to_cpu(resp->cq_entry_size);
6714 		ctx->vnic_max_vnic_entries =
6715 			le16_to_cpu(resp->vnic_max_vnic_entries);
6716 		ctx->vnic_max_ring_table_entries =
6717 			le16_to_cpu(resp->vnic_max_ring_table_entries);
6718 		ctx->vnic_entry_size = le16_to_cpu(resp->vnic_entry_size);
6719 		ctx->stat_max_entries = le32_to_cpu(resp->stat_max_entries);
6720 		ctx->stat_entry_size = le16_to_cpu(resp->stat_entry_size);
6721 		ctx->tqm_entry_size = le16_to_cpu(resp->tqm_entry_size);
6722 		ctx->tqm_min_entries_per_ring =
6723 			le32_to_cpu(resp->tqm_min_entries_per_ring);
6724 		ctx->tqm_max_entries_per_ring =
6725 			le32_to_cpu(resp->tqm_max_entries_per_ring);
6726 		ctx->tqm_entries_multiple = resp->tqm_entries_multiple;
6727 		if (!ctx->tqm_entries_multiple)
6728 			ctx->tqm_entries_multiple = 1;
6729 		ctx->mrav_max_entries = le32_to_cpu(resp->mrav_max_entries);
6730 		ctx->mrav_entry_size = le16_to_cpu(resp->mrav_entry_size);
6731 		ctx->mrav_num_entries_units =
6732 			le16_to_cpu(resp->mrav_num_entries_units);
6733 		ctx->tim_entry_size = le16_to_cpu(resp->tim_entry_size);
6734 		ctx->tim_max_entries = le32_to_cpu(resp->tim_max_entries);
6735 		ctx->ctx_kind_initializer = resp->ctx_kind_initializer;
6736 		ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count;
6737 		if (!ctx->tqm_fp_rings_count)
6738 			ctx->tqm_fp_rings_count = bp->max_q;
6739 
6740 		tqm_rings = ctx->tqm_fp_rings_count + 1;
6741 		ctx_pg = kcalloc(tqm_rings, sizeof(*ctx_pg), GFP_KERNEL);
6742 		if (!ctx_pg) {
6743 			kfree(ctx);
6744 			rc = -ENOMEM;
6745 			goto ctx_err;
6746 		}
6747 		for (i = 0; i < tqm_rings; i++, ctx_pg++)
6748 			ctx->tqm_mem[i] = ctx_pg;
6749 		bp->ctx = ctx;
6750 	} else {
6751 		rc = 0;
6752 	}
6753 ctx_err:
6754 	mutex_unlock(&bp->hwrm_cmd_lock);
6755 	return rc;
6756 }
6757 
6758 static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, u8 *pg_attr,
6759 				  __le64 *pg_dir)
6760 {
6761 	u8 pg_size = 0;
6762 
6763 	if (BNXT_PAGE_SHIFT == 13)
6764 		pg_size = 1 << 4;
6765 	else if (BNXT_PAGE_SIZE == 16)
6766 		pg_size = 2 << 4;
6767 
6768 	*pg_attr = pg_size;
6769 	if (rmem->depth >= 1) {
6770 		if (rmem->depth == 2)
6771 			*pg_attr |= 2;
6772 		else
6773 			*pg_attr |= 1;
6774 		*pg_dir = cpu_to_le64(rmem->pg_tbl_map);
6775 	} else {
6776 		*pg_dir = cpu_to_le64(rmem->dma_arr[0]);
6777 	}
6778 }
6779 
6780 #define FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES			\
6781 	(FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP |		\
6782 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ |		\
6783 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ |		\
6784 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC |		\
6785 	 FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT)
6786 
6787 static int bnxt_hwrm_func_backing_store_cfg(struct bnxt *bp, u32 enables)
6788 {
6789 	struct hwrm_func_backing_store_cfg_input req = {0};
6790 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
6791 	struct bnxt_ctx_pg_info *ctx_pg;
6792 	__le32 *num_entries;
6793 	__le64 *pg_dir;
6794 	u32 flags = 0;
6795 	u8 *pg_attr;
6796 	u32 ena;
6797 	int i;
6798 
6799 	if (!ctx)
6800 		return 0;
6801 
6802 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_BACKING_STORE_CFG, -1, -1);
6803 	req.enables = cpu_to_le32(enables);
6804 
6805 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_QP) {
6806 		ctx_pg = &ctx->qp_mem;
6807 		req.qp_num_entries = cpu_to_le32(ctx_pg->entries);
6808 		req.qp_num_qp1_entries = cpu_to_le16(ctx->qp_min_qp1_entries);
6809 		req.qp_num_l2_entries = cpu_to_le16(ctx->qp_max_l2_entries);
6810 		req.qp_entry_size = cpu_to_le16(ctx->qp_entry_size);
6811 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6812 				      &req.qpc_pg_size_qpc_lvl,
6813 				      &req.qpc_page_dir);
6814 	}
6815 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_SRQ) {
6816 		ctx_pg = &ctx->srq_mem;
6817 		req.srq_num_entries = cpu_to_le32(ctx_pg->entries);
6818 		req.srq_num_l2_entries = cpu_to_le16(ctx->srq_max_l2_entries);
6819 		req.srq_entry_size = cpu_to_le16(ctx->srq_entry_size);
6820 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6821 				      &req.srq_pg_size_srq_lvl,
6822 				      &req.srq_page_dir);
6823 	}
6824 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_CQ) {
6825 		ctx_pg = &ctx->cq_mem;
6826 		req.cq_num_entries = cpu_to_le32(ctx_pg->entries);
6827 		req.cq_num_l2_entries = cpu_to_le16(ctx->cq_max_l2_entries);
6828 		req.cq_entry_size = cpu_to_le16(ctx->cq_entry_size);
6829 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl,
6830 				      &req.cq_page_dir);
6831 	}
6832 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_VNIC) {
6833 		ctx_pg = &ctx->vnic_mem;
6834 		req.vnic_num_vnic_entries =
6835 			cpu_to_le16(ctx->vnic_max_vnic_entries);
6836 		req.vnic_num_ring_table_entries =
6837 			cpu_to_le16(ctx->vnic_max_ring_table_entries);
6838 		req.vnic_entry_size = cpu_to_le16(ctx->vnic_entry_size);
6839 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6840 				      &req.vnic_pg_size_vnic_lvl,
6841 				      &req.vnic_page_dir);
6842 	}
6843 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_STAT) {
6844 		ctx_pg = &ctx->stat_mem;
6845 		req.stat_num_entries = cpu_to_le32(ctx->stat_max_entries);
6846 		req.stat_entry_size = cpu_to_le16(ctx->stat_entry_size);
6847 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6848 				      &req.stat_pg_size_stat_lvl,
6849 				      &req.stat_page_dir);
6850 	}
6851 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV) {
6852 		ctx_pg = &ctx->mrav_mem;
6853 		req.mrav_num_entries = cpu_to_le32(ctx_pg->entries);
6854 		if (ctx->mrav_num_entries_units)
6855 			flags |=
6856 			FUNC_BACKING_STORE_CFG_REQ_FLAGS_MRAV_RESERVATION_SPLIT;
6857 		req.mrav_entry_size = cpu_to_le16(ctx->mrav_entry_size);
6858 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6859 				      &req.mrav_pg_size_mrav_lvl,
6860 				      &req.mrav_page_dir);
6861 	}
6862 	if (enables & FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM) {
6863 		ctx_pg = &ctx->tim_mem;
6864 		req.tim_num_entries = cpu_to_le32(ctx_pg->entries);
6865 		req.tim_entry_size = cpu_to_le16(ctx->tim_entry_size);
6866 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem,
6867 				      &req.tim_pg_size_tim_lvl,
6868 				      &req.tim_page_dir);
6869 	}
6870 	for (i = 0, num_entries = &req.tqm_sp_num_entries,
6871 	     pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl,
6872 	     pg_dir = &req.tqm_sp_page_dir,
6873 	     ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP;
6874 	     i < 9; i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) {
6875 		if (!(enables & ena))
6876 			continue;
6877 
6878 		req.tqm_entry_size = cpu_to_le16(ctx->tqm_entry_size);
6879 		ctx_pg = ctx->tqm_mem[i];
6880 		*num_entries = cpu_to_le32(ctx_pg->entries);
6881 		bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir);
6882 	}
6883 	req.flags = cpu_to_le32(flags);
6884 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
6885 }
6886 
6887 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
6888 				  struct bnxt_ctx_pg_info *ctx_pg)
6889 {
6890 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6891 
6892 	rmem->page_size = BNXT_PAGE_SIZE;
6893 	rmem->pg_arr = ctx_pg->ctx_pg_arr;
6894 	rmem->dma_arr = ctx_pg->ctx_dma_arr;
6895 	rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
6896 	if (rmem->depth >= 1)
6897 		rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG;
6898 	return bnxt_alloc_ring(bp, rmem);
6899 }
6900 
6901 static int bnxt_alloc_ctx_pg_tbls(struct bnxt *bp,
6902 				  struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size,
6903 				  u8 depth, bool use_init_val)
6904 {
6905 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6906 	int rc;
6907 
6908 	if (!mem_size)
6909 		return -EINVAL;
6910 
6911 	ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6912 	if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) {
6913 		ctx_pg->nr_pages = 0;
6914 		return -EINVAL;
6915 	}
6916 	if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) {
6917 		int nr_tbls, i;
6918 
6919 		rmem->depth = 2;
6920 		ctx_pg->ctx_pg_tbl = kcalloc(MAX_CTX_PAGES, sizeof(ctx_pg),
6921 					     GFP_KERNEL);
6922 		if (!ctx_pg->ctx_pg_tbl)
6923 			return -ENOMEM;
6924 		nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES);
6925 		rmem->nr_pages = nr_tbls;
6926 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6927 		if (rc)
6928 			return rc;
6929 		for (i = 0; i < nr_tbls; i++) {
6930 			struct bnxt_ctx_pg_info *pg_tbl;
6931 
6932 			pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL);
6933 			if (!pg_tbl)
6934 				return -ENOMEM;
6935 			ctx_pg->ctx_pg_tbl[i] = pg_tbl;
6936 			rmem = &pg_tbl->ring_mem;
6937 			rmem->pg_tbl = ctx_pg->ctx_pg_arr[i];
6938 			rmem->pg_tbl_map = ctx_pg->ctx_dma_arr[i];
6939 			rmem->depth = 1;
6940 			rmem->nr_pages = MAX_CTX_PAGES;
6941 			if (use_init_val)
6942 				rmem->init_val = bp->ctx->ctx_kind_initializer;
6943 			if (i == (nr_tbls - 1)) {
6944 				int rem = ctx_pg->nr_pages % MAX_CTX_PAGES;
6945 
6946 				if (rem)
6947 					rmem->nr_pages = rem;
6948 			}
6949 			rc = bnxt_alloc_ctx_mem_blk(bp, pg_tbl);
6950 			if (rc)
6951 				break;
6952 		}
6953 	} else {
6954 		rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE);
6955 		if (rmem->nr_pages > 1 || depth)
6956 			rmem->depth = 1;
6957 		if (use_init_val)
6958 			rmem->init_val = bp->ctx->ctx_kind_initializer;
6959 		rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg);
6960 	}
6961 	return rc;
6962 }
6963 
6964 static void bnxt_free_ctx_pg_tbls(struct bnxt *bp,
6965 				  struct bnxt_ctx_pg_info *ctx_pg)
6966 {
6967 	struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
6968 
6969 	if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES ||
6970 	    ctx_pg->ctx_pg_tbl) {
6971 		int i, nr_tbls = rmem->nr_pages;
6972 
6973 		for (i = 0; i < nr_tbls; i++) {
6974 			struct bnxt_ctx_pg_info *pg_tbl;
6975 			struct bnxt_ring_mem_info *rmem2;
6976 
6977 			pg_tbl = ctx_pg->ctx_pg_tbl[i];
6978 			if (!pg_tbl)
6979 				continue;
6980 			rmem2 = &pg_tbl->ring_mem;
6981 			bnxt_free_ring(bp, rmem2);
6982 			ctx_pg->ctx_pg_arr[i] = NULL;
6983 			kfree(pg_tbl);
6984 			ctx_pg->ctx_pg_tbl[i] = NULL;
6985 		}
6986 		kfree(ctx_pg->ctx_pg_tbl);
6987 		ctx_pg->ctx_pg_tbl = NULL;
6988 	}
6989 	bnxt_free_ring(bp, rmem);
6990 	ctx_pg->nr_pages = 0;
6991 }
6992 
6993 static void bnxt_free_ctx_mem(struct bnxt *bp)
6994 {
6995 	struct bnxt_ctx_mem_info *ctx = bp->ctx;
6996 	int i;
6997 
6998 	if (!ctx)
6999 		return;
7000 
7001 	if (ctx->tqm_mem[0]) {
7002 		for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++)
7003 			bnxt_free_ctx_pg_tbls(bp, ctx->tqm_mem[i]);
7004 		kfree(ctx->tqm_mem[0]);
7005 		ctx->tqm_mem[0] = NULL;
7006 	}
7007 
7008 	bnxt_free_ctx_pg_tbls(bp, &ctx->tim_mem);
7009 	bnxt_free_ctx_pg_tbls(bp, &ctx->mrav_mem);
7010 	bnxt_free_ctx_pg_tbls(bp, &ctx->stat_mem);
7011 	bnxt_free_ctx_pg_tbls(bp, &ctx->vnic_mem);
7012 	bnxt_free_ctx_pg_tbls(bp, &ctx->cq_mem);
7013 	bnxt_free_ctx_pg_tbls(bp, &ctx->srq_mem);
7014 	bnxt_free_ctx_pg_tbls(bp, &ctx->qp_mem);
7015 	ctx->flags &= ~BNXT_CTX_FLAG_INITED;
7016 }
7017 
7018 static int bnxt_alloc_ctx_mem(struct bnxt *bp)
7019 {
7020 	struct bnxt_ctx_pg_info *ctx_pg;
7021 	struct bnxt_ctx_mem_info *ctx;
7022 	u32 mem_size, ena, entries;
7023 	u32 entries_sp, min;
7024 	u32 num_mr, num_ah;
7025 	u32 extra_srqs = 0;
7026 	u32 extra_qps = 0;
7027 	u8 pg_lvl = 1;
7028 	int i, rc;
7029 
7030 	rc = bnxt_hwrm_func_backing_store_qcaps(bp);
7031 	if (rc) {
7032 		netdev_err(bp->dev, "Failed querying context mem capability, rc = %d.\n",
7033 			   rc);
7034 		return rc;
7035 	}
7036 	ctx = bp->ctx;
7037 	if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
7038 		return 0;
7039 
7040 	if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) {
7041 		pg_lvl = 2;
7042 		extra_qps = 65536;
7043 		extra_srqs = 8192;
7044 	}
7045 
7046 	ctx_pg = &ctx->qp_mem;
7047 	ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries +
7048 			  extra_qps;
7049 	mem_size = ctx->qp_entry_size * ctx_pg->entries;
7050 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7051 	if (rc)
7052 		return rc;
7053 
7054 	ctx_pg = &ctx->srq_mem;
7055 	ctx_pg->entries = ctx->srq_max_l2_entries + extra_srqs;
7056 	mem_size = ctx->srq_entry_size * ctx_pg->entries;
7057 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7058 	if (rc)
7059 		return rc;
7060 
7061 	ctx_pg = &ctx->cq_mem;
7062 	ctx_pg->entries = ctx->cq_max_l2_entries + extra_qps * 2;
7063 	mem_size = ctx->cq_entry_size * ctx_pg->entries;
7064 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, pg_lvl, true);
7065 	if (rc)
7066 		return rc;
7067 
7068 	ctx_pg = &ctx->vnic_mem;
7069 	ctx_pg->entries = ctx->vnic_max_vnic_entries +
7070 			  ctx->vnic_max_ring_table_entries;
7071 	mem_size = ctx->vnic_entry_size * ctx_pg->entries;
7072 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7073 	if (rc)
7074 		return rc;
7075 
7076 	ctx_pg = &ctx->stat_mem;
7077 	ctx_pg->entries = ctx->stat_max_entries;
7078 	mem_size = ctx->stat_entry_size * ctx_pg->entries;
7079 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, true);
7080 	if (rc)
7081 		return rc;
7082 
7083 	ena = 0;
7084 	if (!(bp->flags & BNXT_FLAG_ROCE_CAP))
7085 		goto skip_rdma;
7086 
7087 	ctx_pg = &ctx->mrav_mem;
7088 	/* 128K extra is needed to accommodate static AH context
7089 	 * allocation by f/w.
7090 	 */
7091 	num_mr = 1024 * 256;
7092 	num_ah = 1024 * 128;
7093 	ctx_pg->entries = num_mr + num_ah;
7094 	mem_size = ctx->mrav_entry_size * ctx_pg->entries;
7095 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 2, true);
7096 	if (rc)
7097 		return rc;
7098 	ena = FUNC_BACKING_STORE_CFG_REQ_ENABLES_MRAV;
7099 	if (ctx->mrav_num_entries_units)
7100 		ctx_pg->entries =
7101 			((num_mr / ctx->mrav_num_entries_units) << 16) |
7102 			 (num_ah / ctx->mrav_num_entries_units);
7103 
7104 	ctx_pg = &ctx->tim_mem;
7105 	ctx_pg->entries = ctx->qp_mem.entries;
7106 	mem_size = ctx->tim_entry_size * ctx_pg->entries;
7107 	rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7108 	if (rc)
7109 		return rc;
7110 	ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TIM;
7111 
7112 skip_rdma:
7113 	min = ctx->tqm_min_entries_per_ring;
7114 	entries_sp = ctx->vnic_max_vnic_entries + ctx->qp_max_l2_entries +
7115 		     2 * (extra_qps + ctx->qp_min_qp1_entries) + min;
7116 	entries_sp = roundup(entries_sp, ctx->tqm_entries_multiple);
7117 	entries = ctx->qp_max_l2_entries + extra_qps + ctx->qp_min_qp1_entries;
7118 	entries = roundup(entries, ctx->tqm_entries_multiple);
7119 	entries = clamp_t(u32, entries, min, ctx->tqm_max_entries_per_ring);
7120 	for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
7121 		ctx_pg = ctx->tqm_mem[i];
7122 		ctx_pg->entries = i ? entries : entries_sp;
7123 		mem_size = ctx->tqm_entry_size * ctx_pg->entries;
7124 		rc = bnxt_alloc_ctx_pg_tbls(bp, ctx_pg, mem_size, 1, false);
7125 		if (rc)
7126 			return rc;
7127 		ena |= FUNC_BACKING_STORE_CFG_REQ_ENABLES_TQM_SP << i;
7128 	}
7129 	ena |= FUNC_BACKING_STORE_CFG_REQ_DFLT_ENABLES;
7130 	rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
7131 	if (rc) {
7132 		netdev_err(bp->dev, "Failed configuring context mem, rc = %d.\n",
7133 			   rc);
7134 		return rc;
7135 	}
7136 	ctx->flags |= BNXT_CTX_FLAG_INITED;
7137 	return 0;
7138 }
7139 
7140 int bnxt_hwrm_func_resc_qcaps(struct bnxt *bp, bool all)
7141 {
7142 	struct hwrm_func_resource_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7143 	struct hwrm_func_resource_qcaps_input req = {0};
7144 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7145 	int rc;
7146 
7147 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESOURCE_QCAPS, -1, -1);
7148 	req.fid = cpu_to_le16(0xffff);
7149 
7150 	mutex_lock(&bp->hwrm_cmd_lock);
7151 	rc = _hwrm_send_message_silent(bp, &req, sizeof(req),
7152 				       HWRM_CMD_TIMEOUT);
7153 	if (rc)
7154 		goto hwrm_func_resc_qcaps_exit;
7155 
7156 	hw_resc->max_tx_sch_inputs = le16_to_cpu(resp->max_tx_scheduler_inputs);
7157 	if (!all)
7158 		goto hwrm_func_resc_qcaps_exit;
7159 
7160 	hw_resc->min_rsscos_ctxs = le16_to_cpu(resp->min_rsscos_ctx);
7161 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7162 	hw_resc->min_cp_rings = le16_to_cpu(resp->min_cmpl_rings);
7163 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7164 	hw_resc->min_tx_rings = le16_to_cpu(resp->min_tx_rings);
7165 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7166 	hw_resc->min_rx_rings = le16_to_cpu(resp->min_rx_rings);
7167 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7168 	hw_resc->min_hw_ring_grps = le16_to_cpu(resp->min_hw_ring_grps);
7169 	hw_resc->max_hw_ring_grps = le16_to_cpu(resp->max_hw_ring_grps);
7170 	hw_resc->min_l2_ctxs = le16_to_cpu(resp->min_l2_ctxs);
7171 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7172 	hw_resc->min_vnics = le16_to_cpu(resp->min_vnics);
7173 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7174 	hw_resc->min_stat_ctxs = le16_to_cpu(resp->min_stat_ctx);
7175 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7176 
7177 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
7178 		u16 max_msix = le16_to_cpu(resp->max_msix);
7179 
7180 		hw_resc->max_nqs = max_msix;
7181 		hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings;
7182 	}
7183 
7184 	if (BNXT_PF(bp)) {
7185 		struct bnxt_pf_info *pf = &bp->pf;
7186 
7187 		pf->vf_resv_strategy =
7188 			le16_to_cpu(resp->vf_reservation_strategy);
7189 		if (pf->vf_resv_strategy > BNXT_VF_RESV_STRATEGY_MINIMAL_STATIC)
7190 			pf->vf_resv_strategy = BNXT_VF_RESV_STRATEGY_MAXIMAL;
7191 	}
7192 hwrm_func_resc_qcaps_exit:
7193 	mutex_unlock(&bp->hwrm_cmd_lock);
7194 	return rc;
7195 }
7196 
7197 static int __bnxt_hwrm_func_qcaps(struct bnxt *bp)
7198 {
7199 	int rc = 0;
7200 	struct hwrm_func_qcaps_input req = {0};
7201 	struct hwrm_func_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
7202 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
7203 	u32 flags, flags_ext;
7204 
7205 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_QCAPS, -1, -1);
7206 	req.fid = cpu_to_le16(0xffff);
7207 
7208 	mutex_lock(&bp->hwrm_cmd_lock);
7209 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7210 	if (rc)
7211 		goto hwrm_func_qcaps_exit;
7212 
7213 	flags = le32_to_cpu(resp->flags);
7214 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V1_SUPPORTED)
7215 		bp->flags |= BNXT_FLAG_ROCEV1_CAP;
7216 	if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED)
7217 		bp->flags |= BNXT_FLAG_ROCEV2_CAP;
7218 	if (flags & FUNC_QCAPS_RESP_FLAGS_PCIE_STATS_SUPPORTED)
7219 		bp->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED;
7220 	if (flags & FUNC_QCAPS_RESP_FLAGS_HOT_RESET_CAPABLE)
7221 		bp->fw_cap |= BNXT_FW_CAP_HOT_RESET;
7222 	if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED)
7223 		bp->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED;
7224 	if (flags &  FUNC_QCAPS_RESP_FLAGS_ERROR_RECOVERY_CAPABLE)
7225 		bp->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY;
7226 	if (flags & FUNC_QCAPS_RESP_FLAGS_ERR_RECOVER_RELOAD)
7227 		bp->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD;
7228 	if (!(flags & FUNC_QCAPS_RESP_FLAGS_VLAN_ACCELERATION_TX_DISABLED))
7229 		bp->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT;
7230 
7231 	flags_ext = le32_to_cpu(resp->flags_ext);
7232 	if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED)
7233 		bp->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED;
7234 
7235 	bp->tx_push_thresh = 0;
7236 	if ((flags & FUNC_QCAPS_RESP_FLAGS_PUSH_MODE_SUPPORTED) &&
7237 	    BNXT_FW_MAJ(bp) > 217)
7238 		bp->tx_push_thresh = BNXT_TX_PUSH_THRESH;
7239 
7240 	hw_resc->max_rsscos_ctxs = le16_to_cpu(resp->max_rsscos_ctx);
7241 	hw_resc->max_cp_rings = le16_to_cpu(resp->max_cmpl_rings);
7242 	hw_resc->max_tx_rings = le16_to_cpu(resp->max_tx_rings);
7243 	hw_resc->max_rx_rings = le16_to_cpu(resp->max_rx_rings);
7244 	hw_resc->max_hw_ring_grps = le32_to_cpu(resp->max_hw_ring_grps);
7245 	if (!hw_resc->max_hw_ring_grps)
7246 		hw_resc->max_hw_ring_grps = hw_resc->max_tx_rings;
7247 	hw_resc->max_l2_ctxs = le16_to_cpu(resp->max_l2_ctxs);
7248 	hw_resc->max_vnics = le16_to_cpu(resp->max_vnics);
7249 	hw_resc->max_stat_ctxs = le16_to_cpu(resp->max_stat_ctx);
7250 
7251 	if (BNXT_PF(bp)) {
7252 		struct bnxt_pf_info *pf = &bp->pf;
7253 
7254 		pf->fw_fid = le16_to_cpu(resp->fid);
7255 		pf->port_id = le16_to_cpu(resp->port_id);
7256 		memcpy(pf->mac_addr, resp->mac_address, ETH_ALEN);
7257 		pf->first_vf_id = le16_to_cpu(resp->first_vf_id);
7258 		pf->max_vfs = le16_to_cpu(resp->max_vfs);
7259 		pf->max_encap_records = le32_to_cpu(resp->max_encap_records);
7260 		pf->max_decap_records = le32_to_cpu(resp->max_decap_records);
7261 		pf->max_tx_em_flows = le32_to_cpu(resp->max_tx_em_flows);
7262 		pf->max_tx_wm_flows = le32_to_cpu(resp->max_tx_wm_flows);
7263 		pf->max_rx_em_flows = le32_to_cpu(resp->max_rx_em_flows);
7264 		pf->max_rx_wm_flows = le32_to_cpu(resp->max_rx_wm_flows);
7265 		bp->flags &= ~BNXT_FLAG_WOL_CAP;
7266 		if (flags & FUNC_QCAPS_RESP_FLAGS_WOL_MAGICPKT_SUPPORTED)
7267 			bp->flags |= BNXT_FLAG_WOL_CAP;
7268 	} else {
7269 #ifdef CONFIG_BNXT_SRIOV
7270 		struct bnxt_vf_info *vf = &bp->vf;
7271 
7272 		vf->fw_fid = le16_to_cpu(resp->fid);
7273 		memcpy(vf->mac_addr, resp->mac_address, ETH_ALEN);
7274 #endif
7275 	}
7276 
7277 hwrm_func_qcaps_exit:
7278 	mutex_unlock(&bp->hwrm_cmd_lock);
7279 	return rc;
7280 }
7281 
7282 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp);
7283 
7284 static int bnxt_hwrm_func_qcaps(struct bnxt *bp)
7285 {
7286 	int rc;
7287 
7288 	rc = __bnxt_hwrm_func_qcaps(bp);
7289 	if (rc)
7290 		return rc;
7291 	rc = bnxt_hwrm_queue_qportcfg(bp);
7292 	if (rc) {
7293 		netdev_err(bp->dev, "hwrm query qportcfg failure rc: %d\n", rc);
7294 		return rc;
7295 	}
7296 	if (bp->hwrm_spec_code >= 0x10803) {
7297 		rc = bnxt_alloc_ctx_mem(bp);
7298 		if (rc)
7299 			return rc;
7300 		rc = bnxt_hwrm_func_resc_qcaps(bp, true);
7301 		if (!rc)
7302 			bp->fw_cap |= BNXT_FW_CAP_NEW_RM;
7303 	}
7304 	return 0;
7305 }
7306 
7307 static int bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(struct bnxt *bp)
7308 {
7309 	struct hwrm_cfa_adv_flow_mgnt_qcaps_input req = {0};
7310 	struct hwrm_cfa_adv_flow_mgnt_qcaps_output *resp;
7311 	int rc = 0;
7312 	u32 flags;
7313 
7314 	if (!(bp->fw_cap & BNXT_FW_CAP_CFA_ADV_FLOW))
7315 		return 0;
7316 
7317 	resp = bp->hwrm_cmd_resp_addr;
7318 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ADV_FLOW_MGNT_QCAPS, -1, -1);
7319 
7320 	mutex_lock(&bp->hwrm_cmd_lock);
7321 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7322 	if (rc)
7323 		goto hwrm_cfa_adv_qcaps_exit;
7324 
7325 	flags = le32_to_cpu(resp->flags);
7326 	if (flags &
7327 	    CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED)
7328 		bp->fw_cap |= BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2;
7329 
7330 hwrm_cfa_adv_qcaps_exit:
7331 	mutex_unlock(&bp->hwrm_cmd_lock);
7332 	return rc;
7333 }
7334 
7335 static int bnxt_map_fw_health_regs(struct bnxt *bp)
7336 {
7337 	struct bnxt_fw_health *fw_health = bp->fw_health;
7338 	u32 reg_base = 0xffffffff;
7339 	int i;
7340 
7341 	/* Only pre-map the monitoring GRC registers using window 3 */
7342 	for (i = 0; i < 4; i++) {
7343 		u32 reg = fw_health->regs[i];
7344 
7345 		if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC)
7346 			continue;
7347 		if (reg_base == 0xffffffff)
7348 			reg_base = reg & BNXT_GRC_BASE_MASK;
7349 		if ((reg & BNXT_GRC_BASE_MASK) != reg_base)
7350 			return -ERANGE;
7351 		fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_BASE +
7352 					    (reg & BNXT_GRC_OFFSET_MASK);
7353 	}
7354 	if (reg_base == 0xffffffff)
7355 		return 0;
7356 
7357 	writel(reg_base, bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT +
7358 			 BNXT_FW_HEALTH_WIN_MAP_OFF);
7359 	return 0;
7360 }
7361 
7362 static int bnxt_hwrm_error_recovery_qcfg(struct bnxt *bp)
7363 {
7364 	struct hwrm_error_recovery_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
7365 	struct bnxt_fw_health *fw_health = bp->fw_health;
7366 	struct hwrm_error_recovery_qcfg_input req = {0};
7367 	int rc, i;
7368 
7369 	if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
7370 		return 0;
7371 
7372 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG, -1, -1);
7373 	mutex_lock(&bp->hwrm_cmd_lock);
7374 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7375 	if (rc)
7376 		goto err_recovery_out;
7377 	fw_health->flags = le32_to_cpu(resp->flags);
7378 	if ((fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) &&
7379 	    !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) {
7380 		rc = -EINVAL;
7381 		goto err_recovery_out;
7382 	}
7383 	fw_health->polling_dsecs = le32_to_cpu(resp->driver_polling_freq);
7384 	fw_health->master_func_wait_dsecs =
7385 		le32_to_cpu(resp->master_func_wait_period);
7386 	fw_health->normal_func_wait_dsecs =
7387 		le32_to_cpu(resp->normal_func_wait_period);
7388 	fw_health->post_reset_wait_dsecs =
7389 		le32_to_cpu(resp->master_func_wait_period_after_reset);
7390 	fw_health->post_reset_max_wait_dsecs =
7391 		le32_to_cpu(resp->max_bailout_time_after_reset);
7392 	fw_health->regs[BNXT_FW_HEALTH_REG] =
7393 		le32_to_cpu(resp->fw_health_status_reg);
7394 	fw_health->regs[BNXT_FW_HEARTBEAT_REG] =
7395 		le32_to_cpu(resp->fw_heartbeat_reg);
7396 	fw_health->regs[BNXT_FW_RESET_CNT_REG] =
7397 		le32_to_cpu(resp->fw_reset_cnt_reg);
7398 	fw_health->regs[BNXT_FW_RESET_INPROG_REG] =
7399 		le32_to_cpu(resp->reset_inprogress_reg);
7400 	fw_health->fw_reset_inprog_reg_mask =
7401 		le32_to_cpu(resp->reset_inprogress_reg_mask);
7402 	fw_health->fw_reset_seq_cnt = resp->reg_array_cnt;
7403 	if (fw_health->fw_reset_seq_cnt >= 16) {
7404 		rc = -EINVAL;
7405 		goto err_recovery_out;
7406 	}
7407 	for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) {
7408 		fw_health->fw_reset_seq_regs[i] =
7409 			le32_to_cpu(resp->reset_reg[i]);
7410 		fw_health->fw_reset_seq_vals[i] =
7411 			le32_to_cpu(resp->reset_reg_val[i]);
7412 		fw_health->fw_reset_seq_delay_msec[i] =
7413 			resp->delay_after_reset[i];
7414 	}
7415 err_recovery_out:
7416 	mutex_unlock(&bp->hwrm_cmd_lock);
7417 	if (!rc)
7418 		rc = bnxt_map_fw_health_regs(bp);
7419 	if (rc)
7420 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
7421 	return rc;
7422 }
7423 
7424 static int bnxt_hwrm_func_reset(struct bnxt *bp)
7425 {
7426 	struct hwrm_func_reset_input req = {0};
7427 
7428 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_RESET, -1, -1);
7429 	req.enables = 0;
7430 
7431 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_RESET_TIMEOUT);
7432 }
7433 
7434 static int bnxt_hwrm_queue_qportcfg(struct bnxt *bp)
7435 {
7436 	int rc = 0;
7437 	struct hwrm_queue_qportcfg_input req = {0};
7438 	struct hwrm_queue_qportcfg_output *resp = bp->hwrm_cmd_resp_addr;
7439 	u8 i, j, *qptr;
7440 	bool no_rdma;
7441 
7442 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_QPORTCFG, -1, -1);
7443 
7444 	mutex_lock(&bp->hwrm_cmd_lock);
7445 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7446 	if (rc)
7447 		goto qportcfg_exit;
7448 
7449 	if (!resp->max_configurable_queues) {
7450 		rc = -EINVAL;
7451 		goto qportcfg_exit;
7452 	}
7453 	bp->max_tc = resp->max_configurable_queues;
7454 	bp->max_lltc = resp->max_configurable_lossless_queues;
7455 	if (bp->max_tc > BNXT_MAX_QUEUE)
7456 		bp->max_tc = BNXT_MAX_QUEUE;
7457 
7458 	no_rdma = !(bp->flags & BNXT_FLAG_ROCE_CAP);
7459 	qptr = &resp->queue_id0;
7460 	for (i = 0, j = 0; i < bp->max_tc; i++) {
7461 		bp->q_info[j].queue_id = *qptr;
7462 		bp->q_ids[i] = *qptr++;
7463 		bp->q_info[j].queue_profile = *qptr++;
7464 		bp->tc_to_qidx[j] = j;
7465 		if (!BNXT_CNPQ(bp->q_info[j].queue_profile) ||
7466 		    (no_rdma && BNXT_PF(bp)))
7467 			j++;
7468 	}
7469 	bp->max_q = bp->max_tc;
7470 	bp->max_tc = max_t(u8, j, 1);
7471 
7472 	if (resp->queue_cfg_info & QUEUE_QPORTCFG_RESP_QUEUE_CFG_INFO_ASYM_CFG)
7473 		bp->max_tc = 1;
7474 
7475 	if (bp->max_lltc > bp->max_tc)
7476 		bp->max_lltc = bp->max_tc;
7477 
7478 qportcfg_exit:
7479 	mutex_unlock(&bp->hwrm_cmd_lock);
7480 	return rc;
7481 }
7482 
7483 static int __bnxt_hwrm_ver_get(struct bnxt *bp, bool silent)
7484 {
7485 	struct hwrm_ver_get_input req = {0};
7486 	int rc;
7487 
7488 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET, -1, -1);
7489 	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
7490 	req.hwrm_intf_min = HWRM_VERSION_MINOR;
7491 	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
7492 
7493 	rc = bnxt_hwrm_do_send_msg(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT,
7494 				   silent);
7495 	return rc;
7496 }
7497 
7498 static int bnxt_hwrm_ver_get(struct bnxt *bp)
7499 {
7500 	struct hwrm_ver_get_output *resp = bp->hwrm_cmd_resp_addr;
7501 	u16 fw_maj, fw_min, fw_bld, fw_rsv;
7502 	u32 dev_caps_cfg, hwrm_ver;
7503 	int rc, len;
7504 
7505 	bp->hwrm_max_req_len = HWRM_MAX_REQ_LEN;
7506 	mutex_lock(&bp->hwrm_cmd_lock);
7507 	rc = __bnxt_hwrm_ver_get(bp, false);
7508 	if (rc)
7509 		goto hwrm_ver_get_exit;
7510 
7511 	memcpy(&bp->ver_resp, resp, sizeof(struct hwrm_ver_get_output));
7512 
7513 	bp->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 |
7514 			     resp->hwrm_intf_min_8b << 8 |
7515 			     resp->hwrm_intf_upd_8b;
7516 	if (resp->hwrm_intf_maj_8b < 1) {
7517 		netdev_warn(bp->dev, "HWRM interface %d.%d.%d is older than 1.0.0.\n",
7518 			    resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7519 			    resp->hwrm_intf_upd_8b);
7520 		netdev_warn(bp->dev, "Please update firmware with HWRM interface 1.0.0 or newer.\n");
7521 	}
7522 
7523 	hwrm_ver = HWRM_VERSION_MAJOR << 16 | HWRM_VERSION_MINOR << 8 |
7524 			HWRM_VERSION_UPDATE;
7525 
7526 	if (bp->hwrm_spec_code > hwrm_ver)
7527 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7528 			 HWRM_VERSION_MAJOR, HWRM_VERSION_MINOR,
7529 			 HWRM_VERSION_UPDATE);
7530 	else
7531 		snprintf(bp->hwrm_ver_supp, FW_VER_STR_LEN, "%d.%d.%d",
7532 			 resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b,
7533 			 resp->hwrm_intf_upd_8b);
7534 
7535 	fw_maj = le16_to_cpu(resp->hwrm_fw_major);
7536 	if (bp->hwrm_spec_code > 0x10803 && fw_maj) {
7537 		fw_min = le16_to_cpu(resp->hwrm_fw_minor);
7538 		fw_bld = le16_to_cpu(resp->hwrm_fw_build);
7539 		fw_rsv = le16_to_cpu(resp->hwrm_fw_patch);
7540 		len = FW_VER_STR_LEN;
7541 	} else {
7542 		fw_maj = resp->hwrm_fw_maj_8b;
7543 		fw_min = resp->hwrm_fw_min_8b;
7544 		fw_bld = resp->hwrm_fw_bld_8b;
7545 		fw_rsv = resp->hwrm_fw_rsvd_8b;
7546 		len = BC_HWRM_STR_LEN;
7547 	}
7548 	bp->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv);
7549 	snprintf(bp->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld,
7550 		 fw_rsv);
7551 
7552 	if (strlen(resp->active_pkg_name)) {
7553 		int fw_ver_len = strlen(bp->fw_ver_str);
7554 
7555 		snprintf(bp->fw_ver_str + fw_ver_len,
7556 			 FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s",
7557 			 resp->active_pkg_name);
7558 		bp->fw_cap |= BNXT_FW_CAP_PKG_VER;
7559 	}
7560 
7561 	bp->hwrm_cmd_timeout = le16_to_cpu(resp->def_req_timeout);
7562 	if (!bp->hwrm_cmd_timeout)
7563 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
7564 
7565 	if (resp->hwrm_intf_maj_8b >= 1) {
7566 		bp->hwrm_max_req_len = le16_to_cpu(resp->max_req_win_len);
7567 		bp->hwrm_max_ext_req_len = le16_to_cpu(resp->max_ext_req_len);
7568 	}
7569 	if (bp->hwrm_max_ext_req_len < HWRM_MAX_REQ_LEN)
7570 		bp->hwrm_max_ext_req_len = HWRM_MAX_REQ_LEN;
7571 
7572 	bp->chip_num = le16_to_cpu(resp->chip_num);
7573 	bp->chip_rev = resp->chip_rev;
7574 	if (bp->chip_num == CHIP_NUM_58700 && !resp->chip_rev &&
7575 	    !resp->chip_metal)
7576 		bp->flags |= BNXT_FLAG_CHIP_NITRO_A0;
7577 
7578 	dev_caps_cfg = le32_to_cpu(resp->dev_caps_cfg);
7579 	if ((dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) &&
7580 	    (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_SHORT_CMD_REQUIRED))
7581 		bp->fw_cap |= BNXT_FW_CAP_SHORT_CMD;
7582 
7583 	if (dev_caps_cfg & VER_GET_RESP_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED)
7584 		bp->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL;
7585 
7586 	if (dev_caps_cfg &
7587 	    VER_GET_RESP_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED)
7588 		bp->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE;
7589 
7590 	if (dev_caps_cfg &
7591 	    VER_GET_RESP_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED)
7592 		bp->fw_cap |= BNXT_FW_CAP_TRUSTED_VF;
7593 
7594 	if (dev_caps_cfg &
7595 	    VER_GET_RESP_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED)
7596 		bp->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW;
7597 
7598 hwrm_ver_get_exit:
7599 	mutex_unlock(&bp->hwrm_cmd_lock);
7600 	return rc;
7601 }
7602 
7603 int bnxt_hwrm_fw_set_time(struct bnxt *bp)
7604 {
7605 	struct hwrm_fw_set_time_input req = {0};
7606 	struct tm tm;
7607 	time64_t now = ktime_get_real_seconds();
7608 
7609 	if ((BNXT_VF(bp) && bp->hwrm_spec_code < 0x10901) ||
7610 	    bp->hwrm_spec_code < 0x10400)
7611 		return -EOPNOTSUPP;
7612 
7613 	time64_to_tm(now, 0, &tm);
7614 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_SET_TIME, -1, -1);
7615 	req.year = cpu_to_le16(1900 + tm.tm_year);
7616 	req.month = 1 + tm.tm_mon;
7617 	req.day = tm.tm_mday;
7618 	req.hour = tm.tm_hour;
7619 	req.minute = tm.tm_min;
7620 	req.second = tm.tm_sec;
7621 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7622 }
7623 
7624 static void bnxt_add_one_ctr(u64 hw, u64 *sw, u64 mask)
7625 {
7626 	u64 sw_tmp;
7627 
7628 	sw_tmp = (*sw & ~mask) | hw;
7629 	if (hw < (*sw & mask))
7630 		sw_tmp += mask + 1;
7631 	WRITE_ONCE(*sw, sw_tmp);
7632 }
7633 
7634 static void __bnxt_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks,
7635 				    int count, bool ignore_zero)
7636 {
7637 	int i;
7638 
7639 	for (i = 0; i < count; i++) {
7640 		u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i]));
7641 
7642 		if (ignore_zero && !hw)
7643 			continue;
7644 
7645 		if (masks[i] == -1ULL)
7646 			sw_stats[i] = hw;
7647 		else
7648 			bnxt_add_one_ctr(hw, &sw_stats[i], masks[i]);
7649 	}
7650 }
7651 
7652 static void bnxt_accumulate_stats(struct bnxt_stats_mem *stats)
7653 {
7654 	if (!stats->hw_stats)
7655 		return;
7656 
7657 	__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7658 				stats->hw_masks, stats->len / 8, false);
7659 }
7660 
7661 static void bnxt_accumulate_all_stats(struct bnxt *bp)
7662 {
7663 	struct bnxt_stats_mem *ring0_stats;
7664 	bool ignore_zero = false;
7665 	int i;
7666 
7667 	/* Chip bug.  Counter intermittently becomes 0. */
7668 	if (bp->flags & BNXT_FLAG_CHIP_P5)
7669 		ignore_zero = true;
7670 
7671 	for (i = 0; i < bp->cp_nr_rings; i++) {
7672 		struct bnxt_napi *bnapi = bp->bnapi[i];
7673 		struct bnxt_cp_ring_info *cpr;
7674 		struct bnxt_stats_mem *stats;
7675 
7676 		cpr = &bnapi->cp_ring;
7677 		stats = &cpr->stats;
7678 		if (!i)
7679 			ring0_stats = stats;
7680 		__bnxt_accumulate_stats(stats->hw_stats, stats->sw_stats,
7681 					ring0_stats->hw_masks,
7682 					ring0_stats->len / 8, ignore_zero);
7683 	}
7684 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
7685 		struct bnxt_stats_mem *stats = &bp->port_stats;
7686 		__le64 *hw_stats = stats->hw_stats;
7687 		u64 *sw_stats = stats->sw_stats;
7688 		u64 *masks = stats->hw_masks;
7689 		int cnt;
7690 
7691 		cnt = sizeof(struct rx_port_stats) / 8;
7692 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7693 
7694 		hw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7695 		sw_stats += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7696 		masks += BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
7697 		cnt = sizeof(struct tx_port_stats) / 8;
7698 		__bnxt_accumulate_stats(hw_stats, sw_stats, masks, cnt, false);
7699 	}
7700 	if (bp->flags & BNXT_FLAG_PORT_STATS_EXT) {
7701 		bnxt_accumulate_stats(&bp->rx_port_stats_ext);
7702 		bnxt_accumulate_stats(&bp->tx_port_stats_ext);
7703 	}
7704 }
7705 
7706 static int bnxt_hwrm_port_qstats(struct bnxt *bp, u8 flags)
7707 {
7708 	struct bnxt_pf_info *pf = &bp->pf;
7709 	struct hwrm_port_qstats_input req = {0};
7710 
7711 	if (!(bp->flags & BNXT_FLAG_PORT_STATS))
7712 		return 0;
7713 
7714 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7715 		return -EOPNOTSUPP;
7716 
7717 	req.flags = flags;
7718 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS, -1, -1);
7719 	req.port_id = cpu_to_le16(pf->port_id);
7720 	req.tx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map +
7721 					    BNXT_TX_PORT_STATS_BYTE_OFFSET);
7722 	req.rx_stat_host_addr = cpu_to_le64(bp->port_stats.hw_stats_map);
7723 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7724 }
7725 
7726 static int bnxt_hwrm_port_qstats_ext(struct bnxt *bp, u8 flags)
7727 {
7728 	struct hwrm_port_qstats_ext_output *resp = bp->hwrm_cmd_resp_addr;
7729 	struct hwrm_queue_pri2cos_qcfg_input req2 = {0};
7730 	struct hwrm_port_qstats_ext_input req = {0};
7731 	struct bnxt_pf_info *pf = &bp->pf;
7732 	u32 tx_stat_size;
7733 	int rc;
7734 
7735 	if (!(bp->flags & BNXT_FLAG_PORT_STATS_EXT))
7736 		return 0;
7737 
7738 	if (flags && !(bp->fw_cap & BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED))
7739 		return -EOPNOTSUPP;
7740 
7741 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_QSTATS_EXT, -1, -1);
7742 	req.flags = flags;
7743 	req.port_id = cpu_to_le16(pf->port_id);
7744 	req.rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext));
7745 	req.rx_stat_host_addr = cpu_to_le64(bp->rx_port_stats_ext.hw_stats_map);
7746 	tx_stat_size = bp->tx_port_stats_ext.hw_stats ?
7747 		       sizeof(struct tx_port_stats_ext) : 0;
7748 	req.tx_stat_size = cpu_to_le16(tx_stat_size);
7749 	req.tx_stat_host_addr = cpu_to_le64(bp->tx_port_stats_ext.hw_stats_map);
7750 	mutex_lock(&bp->hwrm_cmd_lock);
7751 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7752 	if (!rc) {
7753 		bp->fw_rx_stats_ext_size = le16_to_cpu(resp->rx_stat_size) / 8;
7754 		bp->fw_tx_stats_ext_size = tx_stat_size ?
7755 			le16_to_cpu(resp->tx_stat_size) / 8 : 0;
7756 	} else {
7757 		bp->fw_rx_stats_ext_size = 0;
7758 		bp->fw_tx_stats_ext_size = 0;
7759 	}
7760 	if (flags)
7761 		goto qstats_done;
7762 
7763 	if (bp->fw_tx_stats_ext_size <=
7764 	    offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) {
7765 		mutex_unlock(&bp->hwrm_cmd_lock);
7766 		bp->pri2cos_valid = 0;
7767 		return rc;
7768 	}
7769 
7770 	bnxt_hwrm_cmd_hdr_init(bp, &req2, HWRM_QUEUE_PRI2COS_QCFG, -1, -1);
7771 	req2.flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN);
7772 
7773 	rc = _hwrm_send_message(bp, &req2, sizeof(req2), HWRM_CMD_TIMEOUT);
7774 	if (!rc) {
7775 		struct hwrm_queue_pri2cos_qcfg_output *resp2;
7776 		u8 *pri2cos;
7777 		int i, j;
7778 
7779 		resp2 = bp->hwrm_cmd_resp_addr;
7780 		pri2cos = &resp2->pri0_cos_queue_id;
7781 		for (i = 0; i < 8; i++) {
7782 			u8 queue_id = pri2cos[i];
7783 			u8 queue_idx;
7784 
7785 			/* Per port queue IDs start from 0, 10, 20, etc */
7786 			queue_idx = queue_id % 10;
7787 			if (queue_idx > BNXT_MAX_QUEUE) {
7788 				bp->pri2cos_valid = false;
7789 				goto qstats_done;
7790 			}
7791 			for (j = 0; j < bp->max_q; j++) {
7792 				if (bp->q_ids[j] == queue_id)
7793 					bp->pri2cos_idx[i] = queue_idx;
7794 			}
7795 		}
7796 		bp->pri2cos_valid = 1;
7797 	}
7798 qstats_done:
7799 	mutex_unlock(&bp->hwrm_cmd_lock);
7800 	return rc;
7801 }
7802 
7803 static void bnxt_hwrm_free_tunnel_ports(struct bnxt *bp)
7804 {
7805 	if (bp->vxlan_fw_dst_port_id != INVALID_HW_RING_ID)
7806 		bnxt_hwrm_tunnel_dst_port_free(
7807 			bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN);
7808 	if (bp->nge_fw_dst_port_id != INVALID_HW_RING_ID)
7809 		bnxt_hwrm_tunnel_dst_port_free(
7810 			bp, TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE);
7811 }
7812 
7813 static int bnxt_set_tpa(struct bnxt *bp, bool set_tpa)
7814 {
7815 	int rc, i;
7816 	u32 tpa_flags = 0;
7817 
7818 	if (set_tpa)
7819 		tpa_flags = bp->flags & BNXT_FLAG_TPA;
7820 	else if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
7821 		return 0;
7822 	for (i = 0; i < bp->nr_vnics; i++) {
7823 		rc = bnxt_hwrm_vnic_set_tpa(bp, i, tpa_flags);
7824 		if (rc) {
7825 			netdev_err(bp->dev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
7826 				   i, rc);
7827 			return rc;
7828 		}
7829 	}
7830 	return 0;
7831 }
7832 
7833 static void bnxt_hwrm_clear_vnic_rss(struct bnxt *bp)
7834 {
7835 	int i;
7836 
7837 	for (i = 0; i < bp->nr_vnics; i++)
7838 		bnxt_hwrm_vnic_set_rss(bp, i, false);
7839 }
7840 
7841 static void bnxt_clear_vnic(struct bnxt *bp)
7842 {
7843 	if (!bp->vnic_info)
7844 		return;
7845 
7846 	bnxt_hwrm_clear_vnic_filter(bp);
7847 	if (!(bp->flags & BNXT_FLAG_CHIP_P5)) {
7848 		/* clear all RSS setting before free vnic ctx */
7849 		bnxt_hwrm_clear_vnic_rss(bp);
7850 		bnxt_hwrm_vnic_ctx_free(bp);
7851 	}
7852 	/* before free the vnic, undo the vnic tpa settings */
7853 	if (bp->flags & BNXT_FLAG_TPA)
7854 		bnxt_set_tpa(bp, false);
7855 	bnxt_hwrm_vnic_free(bp);
7856 	if (bp->flags & BNXT_FLAG_CHIP_P5)
7857 		bnxt_hwrm_vnic_ctx_free(bp);
7858 }
7859 
7860 static void bnxt_hwrm_resource_free(struct bnxt *bp, bool close_path,
7861 				    bool irq_re_init)
7862 {
7863 	bnxt_clear_vnic(bp);
7864 	bnxt_hwrm_ring_free(bp, close_path);
7865 	bnxt_hwrm_ring_grp_free(bp);
7866 	if (irq_re_init) {
7867 		bnxt_hwrm_stat_ctx_free(bp);
7868 		bnxt_hwrm_free_tunnel_ports(bp);
7869 	}
7870 }
7871 
7872 static int bnxt_hwrm_set_br_mode(struct bnxt *bp, u16 br_mode)
7873 {
7874 	struct hwrm_func_cfg_input req = {0};
7875 
7876 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7877 	req.fid = cpu_to_le16(0xffff);
7878 	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_EVB_MODE);
7879 	if (br_mode == BRIDGE_MODE_VEB)
7880 		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEB;
7881 	else if (br_mode == BRIDGE_MODE_VEPA)
7882 		req.evb_mode = FUNC_CFG_REQ_EVB_MODE_VEPA;
7883 	else
7884 		return -EINVAL;
7885 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7886 }
7887 
7888 static int bnxt_hwrm_set_cache_line_size(struct bnxt *bp, int size)
7889 {
7890 	struct hwrm_func_cfg_input req = {0};
7891 
7892 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10803)
7893 		return 0;
7894 
7895 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_CFG, -1, -1);
7896 	req.fid = cpu_to_le16(0xffff);
7897 	req.enables = cpu_to_le32(FUNC_CFG_REQ_ENABLES_CACHE_LINESIZE);
7898 	req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_64;
7899 	if (size == 128)
7900 		req.options = FUNC_CFG_REQ_OPTIONS_CACHE_LINESIZE_SIZE_128;
7901 
7902 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
7903 }
7904 
7905 static int __bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
7906 {
7907 	struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
7908 	int rc;
7909 
7910 	if (vnic->flags & BNXT_VNIC_RFS_NEW_RSS_FLAG)
7911 		goto skip_rss_ctx;
7912 
7913 	/* allocate context for vnic */
7914 	rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 0);
7915 	if (rc) {
7916 		netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
7917 			   vnic_id, rc);
7918 		goto vnic_setup_err;
7919 	}
7920 	bp->rsscos_nr_ctxs++;
7921 
7922 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
7923 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, 1);
7924 		if (rc) {
7925 			netdev_err(bp->dev, "hwrm vnic %d cos ctx alloc failure rc: %x\n",
7926 				   vnic_id, rc);
7927 			goto vnic_setup_err;
7928 		}
7929 		bp->rsscos_nr_ctxs++;
7930 	}
7931 
7932 skip_rss_ctx:
7933 	/* configure default vnic, ring grp */
7934 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7935 	if (rc) {
7936 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7937 			   vnic_id, rc);
7938 		goto vnic_setup_err;
7939 	}
7940 
7941 	/* Enable RSS hashing on vnic */
7942 	rc = bnxt_hwrm_vnic_set_rss(bp, vnic_id, true);
7943 	if (rc) {
7944 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %x\n",
7945 			   vnic_id, rc);
7946 		goto vnic_setup_err;
7947 	}
7948 
7949 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7950 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7951 		if (rc) {
7952 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7953 				   vnic_id, rc);
7954 		}
7955 	}
7956 
7957 vnic_setup_err:
7958 	return rc;
7959 }
7960 
7961 static int __bnxt_setup_vnic_p5(struct bnxt *bp, u16 vnic_id)
7962 {
7963 	int rc, i, nr_ctxs;
7964 
7965 	nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings);
7966 	for (i = 0; i < nr_ctxs; i++) {
7967 		rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic_id, i);
7968 		if (rc) {
7969 			netdev_err(bp->dev, "hwrm vnic %d ctx %d alloc failure rc: %x\n",
7970 				   vnic_id, i, rc);
7971 			break;
7972 		}
7973 		bp->rsscos_nr_ctxs++;
7974 	}
7975 	if (i < nr_ctxs)
7976 		return -ENOMEM;
7977 
7978 	rc = bnxt_hwrm_vnic_set_rss_p5(bp, vnic_id, true);
7979 	if (rc) {
7980 		netdev_err(bp->dev, "hwrm vnic %d set rss failure rc: %d\n",
7981 			   vnic_id, rc);
7982 		return rc;
7983 	}
7984 	rc = bnxt_hwrm_vnic_cfg(bp, vnic_id);
7985 	if (rc) {
7986 		netdev_err(bp->dev, "hwrm vnic %d cfg failure rc: %x\n",
7987 			   vnic_id, rc);
7988 		return rc;
7989 	}
7990 	if (bp->flags & BNXT_FLAG_AGG_RINGS) {
7991 		rc = bnxt_hwrm_vnic_set_hds(bp, vnic_id);
7992 		if (rc) {
7993 			netdev_err(bp->dev, "hwrm vnic %d set hds failure rc: %x\n",
7994 				   vnic_id, rc);
7995 		}
7996 	}
7997 	return rc;
7998 }
7999 
8000 static int bnxt_setup_vnic(struct bnxt *bp, u16 vnic_id)
8001 {
8002 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8003 		return __bnxt_setup_vnic_p5(bp, vnic_id);
8004 	else
8005 		return __bnxt_setup_vnic(bp, vnic_id);
8006 }
8007 
8008 static int bnxt_alloc_rfs_vnics(struct bnxt *bp)
8009 {
8010 #ifdef CONFIG_RFS_ACCEL
8011 	int i, rc = 0;
8012 
8013 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8014 		return 0;
8015 
8016 	for (i = 0; i < bp->rx_nr_rings; i++) {
8017 		struct bnxt_vnic_info *vnic;
8018 		u16 vnic_id = i + 1;
8019 		u16 ring_id = i;
8020 
8021 		if (vnic_id >= bp->nr_vnics)
8022 			break;
8023 
8024 		vnic = &bp->vnic_info[vnic_id];
8025 		vnic->flags |= BNXT_VNIC_RFS_FLAG;
8026 		if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
8027 			vnic->flags |= BNXT_VNIC_RFS_NEW_RSS_FLAG;
8028 		rc = bnxt_hwrm_vnic_alloc(bp, vnic_id, ring_id, 1);
8029 		if (rc) {
8030 			netdev_err(bp->dev, "hwrm vnic %d alloc failure rc: %x\n",
8031 				   vnic_id, rc);
8032 			break;
8033 		}
8034 		rc = bnxt_setup_vnic(bp, vnic_id);
8035 		if (rc)
8036 			break;
8037 	}
8038 	return rc;
8039 #else
8040 	return 0;
8041 #endif
8042 }
8043 
8044 /* Allow PF and VF with default VLAN to be in promiscuous mode */
8045 static bool bnxt_promisc_ok(struct bnxt *bp)
8046 {
8047 #ifdef CONFIG_BNXT_SRIOV
8048 	if (BNXT_VF(bp) && !bp->vf.vlan)
8049 		return false;
8050 #endif
8051 	return true;
8052 }
8053 
8054 static int bnxt_setup_nitroa0_vnic(struct bnxt *bp)
8055 {
8056 	unsigned int rc = 0;
8057 
8058 	rc = bnxt_hwrm_vnic_alloc(bp, 1, bp->rx_nr_rings - 1, 1);
8059 	if (rc) {
8060 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8061 			   rc);
8062 		return rc;
8063 	}
8064 
8065 	rc = bnxt_hwrm_vnic_cfg(bp, 1);
8066 	if (rc) {
8067 		netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n",
8068 			   rc);
8069 		return rc;
8070 	}
8071 	return rc;
8072 }
8073 
8074 static int bnxt_cfg_rx_mode(struct bnxt *);
8075 static bool bnxt_mc_list_updated(struct bnxt *, u32 *);
8076 
8077 static int bnxt_init_chip(struct bnxt *bp, bool irq_re_init)
8078 {
8079 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
8080 	int rc = 0;
8081 	unsigned int rx_nr_rings = bp->rx_nr_rings;
8082 
8083 	if (irq_re_init) {
8084 		rc = bnxt_hwrm_stat_ctx_alloc(bp);
8085 		if (rc) {
8086 			netdev_err(bp->dev, "hwrm stat ctx alloc failure rc: %x\n",
8087 				   rc);
8088 			goto err_out;
8089 		}
8090 	}
8091 
8092 	rc = bnxt_hwrm_ring_alloc(bp);
8093 	if (rc) {
8094 		netdev_err(bp->dev, "hwrm ring alloc failure rc: %x\n", rc);
8095 		goto err_out;
8096 	}
8097 
8098 	rc = bnxt_hwrm_ring_grp_alloc(bp);
8099 	if (rc) {
8100 		netdev_err(bp->dev, "hwrm_ring_grp alloc failure: %x\n", rc);
8101 		goto err_out;
8102 	}
8103 
8104 	if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8105 		rx_nr_rings--;
8106 
8107 	/* default vnic 0 */
8108 	rc = bnxt_hwrm_vnic_alloc(bp, 0, 0, rx_nr_rings);
8109 	if (rc) {
8110 		netdev_err(bp->dev, "hwrm vnic alloc failure rc: %x\n", rc);
8111 		goto err_out;
8112 	}
8113 
8114 	rc = bnxt_setup_vnic(bp, 0);
8115 	if (rc)
8116 		goto err_out;
8117 
8118 	if (bp->flags & BNXT_FLAG_RFS) {
8119 		rc = bnxt_alloc_rfs_vnics(bp);
8120 		if (rc)
8121 			goto err_out;
8122 	}
8123 
8124 	if (bp->flags & BNXT_FLAG_TPA) {
8125 		rc = bnxt_set_tpa(bp, true);
8126 		if (rc)
8127 			goto err_out;
8128 	}
8129 
8130 	if (BNXT_VF(bp))
8131 		bnxt_update_vf_mac(bp);
8132 
8133 	/* Filter for default vnic 0 */
8134 	rc = bnxt_hwrm_set_vnic_filter(bp, 0, 0, bp->dev->dev_addr);
8135 	if (rc) {
8136 		netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n", rc);
8137 		goto err_out;
8138 	}
8139 	vnic->uc_filter_count = 1;
8140 
8141 	vnic->rx_mask = 0;
8142 	if (bp->dev->flags & IFF_BROADCAST)
8143 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
8144 
8145 	if ((bp->dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
8146 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
8147 
8148 	if (bp->dev->flags & IFF_ALLMULTI) {
8149 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
8150 		vnic->mc_list_count = 0;
8151 	} else {
8152 		u32 mask = 0;
8153 
8154 		bnxt_mc_list_updated(bp, &mask);
8155 		vnic->rx_mask |= mask;
8156 	}
8157 
8158 	rc = bnxt_cfg_rx_mode(bp);
8159 	if (rc)
8160 		goto err_out;
8161 
8162 	rc = bnxt_hwrm_set_coal(bp);
8163 	if (rc)
8164 		netdev_warn(bp->dev, "HWRM set coalescing failure rc: %x\n",
8165 				rc);
8166 
8167 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8168 		rc = bnxt_setup_nitroa0_vnic(bp);
8169 		if (rc)
8170 			netdev_err(bp->dev, "Special vnic setup failure for NS2 A0 rc: %x\n",
8171 				   rc);
8172 	}
8173 
8174 	if (BNXT_VF(bp)) {
8175 		bnxt_hwrm_func_qcfg(bp);
8176 		netdev_update_features(bp->dev);
8177 	}
8178 
8179 	return 0;
8180 
8181 err_out:
8182 	bnxt_hwrm_resource_free(bp, 0, true);
8183 
8184 	return rc;
8185 }
8186 
8187 static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init)
8188 {
8189 	bnxt_hwrm_resource_free(bp, 1, irq_re_init);
8190 	return 0;
8191 }
8192 
8193 static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init)
8194 {
8195 	bnxt_init_cp_rings(bp);
8196 	bnxt_init_rx_rings(bp);
8197 	bnxt_init_tx_rings(bp);
8198 	bnxt_init_ring_grps(bp, irq_re_init);
8199 	bnxt_init_vnics(bp);
8200 
8201 	return bnxt_init_chip(bp, irq_re_init);
8202 }
8203 
8204 static int bnxt_set_real_num_queues(struct bnxt *bp)
8205 {
8206 	int rc;
8207 	struct net_device *dev = bp->dev;
8208 
8209 	rc = netif_set_real_num_tx_queues(dev, bp->tx_nr_rings -
8210 					  bp->tx_nr_rings_xdp);
8211 	if (rc)
8212 		return rc;
8213 
8214 	rc = netif_set_real_num_rx_queues(dev, bp->rx_nr_rings);
8215 	if (rc)
8216 		return rc;
8217 
8218 #ifdef CONFIG_RFS_ACCEL
8219 	if (bp->flags & BNXT_FLAG_RFS)
8220 		dev->rx_cpu_rmap = alloc_irq_cpu_rmap(bp->rx_nr_rings);
8221 #endif
8222 
8223 	return rc;
8224 }
8225 
8226 static int bnxt_trim_rings(struct bnxt *bp, int *rx, int *tx, int max,
8227 			   bool shared)
8228 {
8229 	int _rx = *rx, _tx = *tx;
8230 
8231 	if (shared) {
8232 		*rx = min_t(int, _rx, max);
8233 		*tx = min_t(int, _tx, max);
8234 	} else {
8235 		if (max < 2)
8236 			return -ENOMEM;
8237 
8238 		while (_rx + _tx > max) {
8239 			if (_rx > _tx && _rx > 1)
8240 				_rx--;
8241 			else if (_tx > 1)
8242 				_tx--;
8243 		}
8244 		*rx = _rx;
8245 		*tx = _tx;
8246 	}
8247 	return 0;
8248 }
8249 
8250 static void bnxt_setup_msix(struct bnxt *bp)
8251 {
8252 	const int len = sizeof(bp->irq_tbl[0].name);
8253 	struct net_device *dev = bp->dev;
8254 	int tcs, i;
8255 
8256 	tcs = netdev_get_num_tc(dev);
8257 	if (tcs) {
8258 		int i, off, count;
8259 
8260 		for (i = 0; i < tcs; i++) {
8261 			count = bp->tx_nr_rings_per_tc;
8262 			off = i * count;
8263 			netdev_set_tc_queue(dev, i, count, off);
8264 		}
8265 	}
8266 
8267 	for (i = 0; i < bp->cp_nr_rings; i++) {
8268 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8269 		char *attr;
8270 
8271 		if (bp->flags & BNXT_FLAG_SHARED_RINGS)
8272 			attr = "TxRx";
8273 		else if (i < bp->rx_nr_rings)
8274 			attr = "rx";
8275 		else
8276 			attr = "tx";
8277 
8278 		snprintf(bp->irq_tbl[map_idx].name, len, "%s-%s-%d", dev->name,
8279 			 attr, i);
8280 		bp->irq_tbl[map_idx].handler = bnxt_msix;
8281 	}
8282 }
8283 
8284 static void bnxt_setup_inta(struct bnxt *bp)
8285 {
8286 	const int len = sizeof(bp->irq_tbl[0].name);
8287 
8288 	if (netdev_get_num_tc(bp->dev))
8289 		netdev_reset_tc(bp->dev);
8290 
8291 	snprintf(bp->irq_tbl[0].name, len, "%s-%s-%d", bp->dev->name, "TxRx",
8292 		 0);
8293 	bp->irq_tbl[0].handler = bnxt_inta;
8294 }
8295 
8296 static int bnxt_setup_int_mode(struct bnxt *bp)
8297 {
8298 	int rc;
8299 
8300 	if (bp->flags & BNXT_FLAG_USING_MSIX)
8301 		bnxt_setup_msix(bp);
8302 	else
8303 		bnxt_setup_inta(bp);
8304 
8305 	rc = bnxt_set_real_num_queues(bp);
8306 	return rc;
8307 }
8308 
8309 #ifdef CONFIG_RFS_ACCEL
8310 static unsigned int bnxt_get_max_func_rss_ctxs(struct bnxt *bp)
8311 {
8312 	return bp->hw_resc.max_rsscos_ctxs;
8313 }
8314 
8315 static unsigned int bnxt_get_max_func_vnics(struct bnxt *bp)
8316 {
8317 	return bp->hw_resc.max_vnics;
8318 }
8319 #endif
8320 
8321 unsigned int bnxt_get_max_func_stat_ctxs(struct bnxt *bp)
8322 {
8323 	return bp->hw_resc.max_stat_ctxs;
8324 }
8325 
8326 unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp)
8327 {
8328 	return bp->hw_resc.max_cp_rings;
8329 }
8330 
8331 static unsigned int bnxt_get_max_func_cp_rings_for_en(struct bnxt *bp)
8332 {
8333 	unsigned int cp = bp->hw_resc.max_cp_rings;
8334 
8335 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8336 		cp -= bnxt_get_ulp_msix_num(bp);
8337 
8338 	return cp;
8339 }
8340 
8341 static unsigned int bnxt_get_max_func_irqs(struct bnxt *bp)
8342 {
8343 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
8344 
8345 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8346 		return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_nqs);
8347 
8348 	return min_t(unsigned int, hw_resc->max_irqs, hw_resc->max_cp_rings);
8349 }
8350 
8351 static void bnxt_set_max_func_irqs(struct bnxt *bp, unsigned int max_irqs)
8352 {
8353 	bp->hw_resc.max_irqs = max_irqs;
8354 }
8355 
8356 unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp)
8357 {
8358 	unsigned int cp;
8359 
8360 	cp = bnxt_get_max_func_cp_rings_for_en(bp);
8361 	if (bp->flags & BNXT_FLAG_CHIP_P5)
8362 		return cp - bp->rx_nr_rings - bp->tx_nr_rings;
8363 	else
8364 		return cp - bp->cp_nr_rings;
8365 }
8366 
8367 unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp)
8368 {
8369 	return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp);
8370 }
8371 
8372 int bnxt_get_avail_msix(struct bnxt *bp, int num)
8373 {
8374 	int max_cp = bnxt_get_max_func_cp_rings(bp);
8375 	int max_irq = bnxt_get_max_func_irqs(bp);
8376 	int total_req = bp->cp_nr_rings + num;
8377 	int max_idx, avail_msix;
8378 
8379 	max_idx = bp->total_irqs;
8380 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
8381 		max_idx = min_t(int, bp->total_irqs, max_cp);
8382 	avail_msix = max_idx - bp->cp_nr_rings;
8383 	if (!BNXT_NEW_RM(bp) || avail_msix >= num)
8384 		return avail_msix;
8385 
8386 	if (max_irq < total_req) {
8387 		num = max_irq - bp->cp_nr_rings;
8388 		if (num <= 0)
8389 			return 0;
8390 	}
8391 	return num;
8392 }
8393 
8394 static int bnxt_get_num_msix(struct bnxt *bp)
8395 {
8396 	if (!BNXT_NEW_RM(bp))
8397 		return bnxt_get_max_func_irqs(bp);
8398 
8399 	return bnxt_nq_rings_in_use(bp);
8400 }
8401 
8402 static int bnxt_init_msix(struct bnxt *bp)
8403 {
8404 	int i, total_vecs, max, rc = 0, min = 1, ulp_msix;
8405 	struct msix_entry *msix_ent;
8406 
8407 	total_vecs = bnxt_get_num_msix(bp);
8408 	max = bnxt_get_max_func_irqs(bp);
8409 	if (total_vecs > max)
8410 		total_vecs = max;
8411 
8412 	if (!total_vecs)
8413 		return 0;
8414 
8415 	msix_ent = kcalloc(total_vecs, sizeof(struct msix_entry), GFP_KERNEL);
8416 	if (!msix_ent)
8417 		return -ENOMEM;
8418 
8419 	for (i = 0; i < total_vecs; i++) {
8420 		msix_ent[i].entry = i;
8421 		msix_ent[i].vector = 0;
8422 	}
8423 
8424 	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS))
8425 		min = 2;
8426 
8427 	total_vecs = pci_enable_msix_range(bp->pdev, msix_ent, min, total_vecs);
8428 	ulp_msix = bnxt_get_ulp_msix_num(bp);
8429 	if (total_vecs < 0 || total_vecs < ulp_msix) {
8430 		rc = -ENODEV;
8431 		goto msix_setup_exit;
8432 	}
8433 
8434 	bp->irq_tbl = kcalloc(total_vecs, sizeof(struct bnxt_irq), GFP_KERNEL);
8435 	if (bp->irq_tbl) {
8436 		for (i = 0; i < total_vecs; i++)
8437 			bp->irq_tbl[i].vector = msix_ent[i].vector;
8438 
8439 		bp->total_irqs = total_vecs;
8440 		/* Trim rings based upon num of vectors allocated */
8441 		rc = bnxt_trim_rings(bp, &bp->rx_nr_rings, &bp->tx_nr_rings,
8442 				     total_vecs - ulp_msix, min == 1);
8443 		if (rc)
8444 			goto msix_setup_exit;
8445 
8446 		bp->cp_nr_rings = (min == 1) ?
8447 				  max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
8448 				  bp->tx_nr_rings + bp->rx_nr_rings;
8449 
8450 	} else {
8451 		rc = -ENOMEM;
8452 		goto msix_setup_exit;
8453 	}
8454 	bp->flags |= BNXT_FLAG_USING_MSIX;
8455 	kfree(msix_ent);
8456 	return 0;
8457 
8458 msix_setup_exit:
8459 	netdev_err(bp->dev, "bnxt_init_msix err: %x\n", rc);
8460 	kfree(bp->irq_tbl);
8461 	bp->irq_tbl = NULL;
8462 	pci_disable_msix(bp->pdev);
8463 	kfree(msix_ent);
8464 	return rc;
8465 }
8466 
8467 static int bnxt_init_inta(struct bnxt *bp)
8468 {
8469 	bp->irq_tbl = kcalloc(1, sizeof(struct bnxt_irq), GFP_KERNEL);
8470 	if (!bp->irq_tbl)
8471 		return -ENOMEM;
8472 
8473 	bp->total_irqs = 1;
8474 	bp->rx_nr_rings = 1;
8475 	bp->tx_nr_rings = 1;
8476 	bp->cp_nr_rings = 1;
8477 	bp->flags |= BNXT_FLAG_SHARED_RINGS;
8478 	bp->irq_tbl[0].vector = bp->pdev->irq;
8479 	return 0;
8480 }
8481 
8482 static int bnxt_init_int_mode(struct bnxt *bp)
8483 {
8484 	int rc = 0;
8485 
8486 	if (bp->flags & BNXT_FLAG_MSIX_CAP)
8487 		rc = bnxt_init_msix(bp);
8488 
8489 	if (!(bp->flags & BNXT_FLAG_USING_MSIX) && BNXT_PF(bp)) {
8490 		/* fallback to INTA */
8491 		rc = bnxt_init_inta(bp);
8492 	}
8493 	return rc;
8494 }
8495 
8496 static void bnxt_clear_int_mode(struct bnxt *bp)
8497 {
8498 	if (bp->flags & BNXT_FLAG_USING_MSIX)
8499 		pci_disable_msix(bp->pdev);
8500 
8501 	kfree(bp->irq_tbl);
8502 	bp->irq_tbl = NULL;
8503 	bp->flags &= ~BNXT_FLAG_USING_MSIX;
8504 }
8505 
8506 int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init)
8507 {
8508 	int tcs = netdev_get_num_tc(bp->dev);
8509 	bool irq_cleared = false;
8510 	int rc;
8511 
8512 	if (!bnxt_need_reserve_rings(bp))
8513 		return 0;
8514 
8515 	if (irq_re_init && BNXT_NEW_RM(bp) &&
8516 	    bnxt_get_num_msix(bp) != bp->total_irqs) {
8517 		bnxt_ulp_irq_stop(bp);
8518 		bnxt_clear_int_mode(bp);
8519 		irq_cleared = true;
8520 	}
8521 	rc = __bnxt_reserve_rings(bp);
8522 	if (irq_cleared) {
8523 		if (!rc)
8524 			rc = bnxt_init_int_mode(bp);
8525 		bnxt_ulp_irq_restart(bp, rc);
8526 	}
8527 	if (rc) {
8528 		netdev_err(bp->dev, "ring reservation/IRQ init failure rc: %d\n", rc);
8529 		return rc;
8530 	}
8531 	if (tcs && (bp->tx_nr_rings_per_tc * tcs != bp->tx_nr_rings)) {
8532 		netdev_err(bp->dev, "tx ring reservation failure\n");
8533 		netdev_reset_tc(bp->dev);
8534 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
8535 		return -ENOMEM;
8536 	}
8537 	return 0;
8538 }
8539 
8540 static void bnxt_free_irq(struct bnxt *bp)
8541 {
8542 	struct bnxt_irq *irq;
8543 	int i;
8544 
8545 #ifdef CONFIG_RFS_ACCEL
8546 	free_irq_cpu_rmap(bp->dev->rx_cpu_rmap);
8547 	bp->dev->rx_cpu_rmap = NULL;
8548 #endif
8549 	if (!bp->irq_tbl || !bp->bnapi)
8550 		return;
8551 
8552 	for (i = 0; i < bp->cp_nr_rings; i++) {
8553 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8554 
8555 		irq = &bp->irq_tbl[map_idx];
8556 		if (irq->requested) {
8557 			if (irq->have_cpumask) {
8558 				irq_set_affinity_hint(irq->vector, NULL);
8559 				free_cpumask_var(irq->cpu_mask);
8560 				irq->have_cpumask = 0;
8561 			}
8562 			free_irq(irq->vector, bp->bnapi[i]);
8563 		}
8564 
8565 		irq->requested = 0;
8566 	}
8567 }
8568 
8569 static int bnxt_request_irq(struct bnxt *bp)
8570 {
8571 	int i, j, rc = 0;
8572 	unsigned long flags = 0;
8573 #ifdef CONFIG_RFS_ACCEL
8574 	struct cpu_rmap *rmap;
8575 #endif
8576 
8577 	rc = bnxt_setup_int_mode(bp);
8578 	if (rc) {
8579 		netdev_err(bp->dev, "bnxt_setup_int_mode err: %x\n",
8580 			   rc);
8581 		return rc;
8582 	}
8583 #ifdef CONFIG_RFS_ACCEL
8584 	rmap = bp->dev->rx_cpu_rmap;
8585 #endif
8586 	if (!(bp->flags & BNXT_FLAG_USING_MSIX))
8587 		flags = IRQF_SHARED;
8588 
8589 	for (i = 0, j = 0; i < bp->cp_nr_rings; i++) {
8590 		int map_idx = bnxt_cp_num_to_irq_num(bp, i);
8591 		struct bnxt_irq *irq = &bp->irq_tbl[map_idx];
8592 
8593 #ifdef CONFIG_RFS_ACCEL
8594 		if (rmap && bp->bnapi[i]->rx_ring) {
8595 			rc = irq_cpu_rmap_add(rmap, irq->vector);
8596 			if (rc)
8597 				netdev_warn(bp->dev, "failed adding irq rmap for ring %d\n",
8598 					    j);
8599 			j++;
8600 		}
8601 #endif
8602 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
8603 				 bp->bnapi[i]);
8604 		if (rc)
8605 			break;
8606 
8607 		irq->requested = 1;
8608 
8609 		if (zalloc_cpumask_var(&irq->cpu_mask, GFP_KERNEL)) {
8610 			int numa_node = dev_to_node(&bp->pdev->dev);
8611 
8612 			irq->have_cpumask = 1;
8613 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
8614 					irq->cpu_mask);
8615 			rc = irq_set_affinity_hint(irq->vector, irq->cpu_mask);
8616 			if (rc) {
8617 				netdev_warn(bp->dev,
8618 					    "Set affinity failed, IRQ = %d\n",
8619 					    irq->vector);
8620 				break;
8621 			}
8622 		}
8623 	}
8624 	return rc;
8625 }
8626 
8627 static void bnxt_del_napi(struct bnxt *bp)
8628 {
8629 	int i;
8630 
8631 	if (!bp->bnapi)
8632 		return;
8633 
8634 	for (i = 0; i < bp->cp_nr_rings; i++) {
8635 		struct bnxt_napi *bnapi = bp->bnapi[i];
8636 
8637 		napi_hash_del(&bnapi->napi);
8638 		netif_napi_del(&bnapi->napi);
8639 	}
8640 	/* We called napi_hash_del() before netif_napi_del(), we need
8641 	 * to respect an RCU grace period before freeing napi structures.
8642 	 */
8643 	synchronize_net();
8644 }
8645 
8646 static void bnxt_init_napi(struct bnxt *bp)
8647 {
8648 	int i;
8649 	unsigned int cp_nr_rings = bp->cp_nr_rings;
8650 	struct bnxt_napi *bnapi;
8651 
8652 	if (bp->flags & BNXT_FLAG_USING_MSIX) {
8653 		int (*poll_fn)(struct napi_struct *, int) = bnxt_poll;
8654 
8655 		if (bp->flags & BNXT_FLAG_CHIP_P5)
8656 			poll_fn = bnxt_poll_p5;
8657 		else if (BNXT_CHIP_TYPE_NITRO_A0(bp))
8658 			cp_nr_rings--;
8659 		for (i = 0; i < cp_nr_rings; i++) {
8660 			bnapi = bp->bnapi[i];
8661 			netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
8662 		}
8663 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
8664 			bnapi = bp->bnapi[cp_nr_rings];
8665 			netif_napi_add(bp->dev, &bnapi->napi,
8666 				       bnxt_poll_nitroa0, 64);
8667 		}
8668 	} else {
8669 		bnapi = bp->bnapi[0];
8670 		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
8671 	}
8672 }
8673 
8674 static void bnxt_disable_napi(struct bnxt *bp)
8675 {
8676 	int i;
8677 
8678 	if (!bp->bnapi)
8679 		return;
8680 
8681 	for (i = 0; i < bp->cp_nr_rings; i++) {
8682 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8683 
8684 		if (bp->bnapi[i]->rx_ring)
8685 			cancel_work_sync(&cpr->dim.work);
8686 
8687 		napi_disable(&bp->bnapi[i]->napi);
8688 	}
8689 }
8690 
8691 static void bnxt_enable_napi(struct bnxt *bp)
8692 {
8693 	int i;
8694 
8695 	for (i = 0; i < bp->cp_nr_rings; i++) {
8696 		struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring;
8697 		bp->bnapi[i]->in_reset = false;
8698 
8699 		if (bp->bnapi[i]->rx_ring) {
8700 			INIT_WORK(&cpr->dim.work, bnxt_dim_work);
8701 			cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
8702 		}
8703 		napi_enable(&bp->bnapi[i]->napi);
8704 	}
8705 }
8706 
8707 void bnxt_tx_disable(struct bnxt *bp)
8708 {
8709 	int i;
8710 	struct bnxt_tx_ring_info *txr;
8711 
8712 	if (bp->tx_ring) {
8713 		for (i = 0; i < bp->tx_nr_rings; i++) {
8714 			txr = &bp->tx_ring[i];
8715 			txr->dev_state = BNXT_DEV_STATE_CLOSING;
8716 		}
8717 	}
8718 	/* Stop all TX queues */
8719 	netif_tx_disable(bp->dev);
8720 	netif_carrier_off(bp->dev);
8721 }
8722 
8723 void bnxt_tx_enable(struct bnxt *bp)
8724 {
8725 	int i;
8726 	struct bnxt_tx_ring_info *txr;
8727 
8728 	for (i = 0; i < bp->tx_nr_rings; i++) {
8729 		txr = &bp->tx_ring[i];
8730 		txr->dev_state = 0;
8731 	}
8732 	netif_tx_wake_all_queues(bp->dev);
8733 	if (bp->link_info.link_up)
8734 		netif_carrier_on(bp->dev);
8735 }
8736 
8737 static void bnxt_report_link(struct bnxt *bp)
8738 {
8739 	if (bp->link_info.link_up) {
8740 		const char *duplex;
8741 		const char *flow_ctrl;
8742 		u32 speed;
8743 		u16 fec;
8744 
8745 		netif_carrier_on(bp->dev);
8746 		if (bp->link_info.duplex == BNXT_LINK_DUPLEX_FULL)
8747 			duplex = "full";
8748 		else
8749 			duplex = "half";
8750 		if (bp->link_info.pause == BNXT_LINK_PAUSE_BOTH)
8751 			flow_ctrl = "ON - receive & transmit";
8752 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_TX)
8753 			flow_ctrl = "ON - transmit";
8754 		else if (bp->link_info.pause == BNXT_LINK_PAUSE_RX)
8755 			flow_ctrl = "ON - receive";
8756 		else
8757 			flow_ctrl = "none";
8758 		speed = bnxt_fw_to_ethtool_speed(bp->link_info.link_speed);
8759 		netdev_info(bp->dev, "NIC Link is Up, %u Mbps %s duplex, Flow control: %s\n",
8760 			    speed, duplex, flow_ctrl);
8761 		if (bp->flags & BNXT_FLAG_EEE_CAP)
8762 			netdev_info(bp->dev, "EEE is %s\n",
8763 				    bp->eee.eee_active ? "active" :
8764 							 "not active");
8765 		fec = bp->link_info.fec_cfg;
8766 		if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED))
8767 			netdev_info(bp->dev, "FEC autoneg %s encodings: %s\n",
8768 				    (fec & BNXT_FEC_AUTONEG) ? "on" : "off",
8769 				    (fec & BNXT_FEC_ENC_BASE_R) ? "BaseR" :
8770 				     (fec & BNXT_FEC_ENC_RS) ? "RS" : "None");
8771 	} else {
8772 		netif_carrier_off(bp->dev);
8773 		netdev_err(bp->dev, "NIC Link is Down\n");
8774 	}
8775 }
8776 
8777 static int bnxt_hwrm_phy_qcaps(struct bnxt *bp)
8778 {
8779 	int rc = 0;
8780 	struct hwrm_port_phy_qcaps_input req = {0};
8781 	struct hwrm_port_phy_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
8782 	struct bnxt_link_info *link_info = &bp->link_info;
8783 
8784 	bp->flags &= ~BNXT_FLAG_EEE_CAP;
8785 	if (bp->test_info)
8786 		bp->test_info->flags &= ~(BNXT_TEST_FL_EXT_LPBK |
8787 					  BNXT_TEST_FL_AN_PHY_LPBK);
8788 	if (bp->hwrm_spec_code < 0x10201)
8789 		return 0;
8790 
8791 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCAPS, -1, -1);
8792 
8793 	mutex_lock(&bp->hwrm_cmd_lock);
8794 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8795 	if (rc)
8796 		goto hwrm_phy_qcaps_exit;
8797 
8798 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EEE_SUPPORTED) {
8799 		struct ethtool_eee *eee = &bp->eee;
8800 		u16 fw_speeds = le16_to_cpu(resp->supported_speeds_eee_mode);
8801 
8802 		bp->flags |= BNXT_FLAG_EEE_CAP;
8803 		eee->supported = _bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8804 		bp->lpi_tmr_lo = le32_to_cpu(resp->tx_lpi_timer_low) &
8805 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_LOW_MASK;
8806 		bp->lpi_tmr_hi = le32_to_cpu(resp->valid_tx_lpi_timer_high) &
8807 				 PORT_PHY_QCAPS_RESP_TX_LPI_TIMER_HIGH_MASK;
8808 	}
8809 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_EXTERNAL_LPBK_SUPPORTED) {
8810 		if (bp->test_info)
8811 			bp->test_info->flags |= BNXT_TEST_FL_EXT_LPBK;
8812 	}
8813 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_AUTONEG_LPBK_SUPPORTED) {
8814 		if (bp->test_info)
8815 			bp->test_info->flags |= BNXT_TEST_FL_AN_PHY_LPBK;
8816 	}
8817 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_SHARED_PHY_CFG_SUPPORTED) {
8818 		if (BNXT_PF(bp))
8819 			bp->fw_cap |= BNXT_FW_CAP_SHARED_PORT_CFG;
8820 	}
8821 	if (resp->flags & PORT_PHY_QCAPS_RESP_FLAGS_CUMULATIVE_COUNTERS_ON_RESET)
8822 		bp->fw_cap |= BNXT_FW_CAP_PORT_STATS_NO_RESET;
8823 
8824 	if (resp->supported_speeds_auto_mode)
8825 		link_info->support_auto_speeds =
8826 			le16_to_cpu(resp->supported_speeds_auto_mode);
8827 
8828 	bp->port_count = resp->port_cnt;
8829 
8830 hwrm_phy_qcaps_exit:
8831 	mutex_unlock(&bp->hwrm_cmd_lock);
8832 	return rc;
8833 }
8834 
8835 static int bnxt_update_link(struct bnxt *bp, bool chng_link_state)
8836 {
8837 	int rc = 0;
8838 	struct bnxt_link_info *link_info = &bp->link_info;
8839 	struct hwrm_port_phy_qcfg_input req = {0};
8840 	struct hwrm_port_phy_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
8841 	u8 link_up = link_info->link_up;
8842 	u16 diff;
8843 
8844 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_QCFG, -1, -1);
8845 
8846 	mutex_lock(&bp->hwrm_cmd_lock);
8847 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
8848 	if (rc) {
8849 		mutex_unlock(&bp->hwrm_cmd_lock);
8850 		return rc;
8851 	}
8852 
8853 	memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp));
8854 	link_info->phy_link_status = resp->link;
8855 	link_info->duplex = resp->duplex_cfg;
8856 	if (bp->hwrm_spec_code >= 0x10800)
8857 		link_info->duplex = resp->duplex_state;
8858 	link_info->pause = resp->pause;
8859 	link_info->auto_mode = resp->auto_mode;
8860 	link_info->auto_pause_setting = resp->auto_pause;
8861 	link_info->lp_pause = resp->link_partner_adv_pause;
8862 	link_info->force_pause_setting = resp->force_pause;
8863 	link_info->duplex_setting = resp->duplex_cfg;
8864 	if (link_info->phy_link_status == BNXT_LINK_LINK)
8865 		link_info->link_speed = le16_to_cpu(resp->link_speed);
8866 	else
8867 		link_info->link_speed = 0;
8868 	link_info->force_link_speed = le16_to_cpu(resp->force_link_speed);
8869 	link_info->support_speeds = le16_to_cpu(resp->support_speeds);
8870 	link_info->auto_link_speeds = le16_to_cpu(resp->auto_link_speed_mask);
8871 	link_info->lp_auto_link_speeds =
8872 		le16_to_cpu(resp->link_partner_adv_speeds);
8873 	link_info->preemphasis = le32_to_cpu(resp->preemphasis);
8874 	link_info->phy_ver[0] = resp->phy_maj;
8875 	link_info->phy_ver[1] = resp->phy_min;
8876 	link_info->phy_ver[2] = resp->phy_bld;
8877 	link_info->media_type = resp->media_type;
8878 	link_info->phy_type = resp->phy_type;
8879 	link_info->transceiver = resp->xcvr_pkg_type;
8880 	link_info->phy_addr = resp->eee_config_phy_addr &
8881 			      PORT_PHY_QCFG_RESP_PHY_ADDR_MASK;
8882 	link_info->module_status = resp->module_status;
8883 
8884 	if (bp->flags & BNXT_FLAG_EEE_CAP) {
8885 		struct ethtool_eee *eee = &bp->eee;
8886 		u16 fw_speeds;
8887 
8888 		eee->eee_active = 0;
8889 		if (resp->eee_config_phy_addr &
8890 		    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ACTIVE) {
8891 			eee->eee_active = 1;
8892 			fw_speeds = le16_to_cpu(
8893 				resp->link_partner_adv_eee_link_speed_mask);
8894 			eee->lp_advertised =
8895 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8896 		}
8897 
8898 		/* Pull initial EEE config */
8899 		if (!chng_link_state) {
8900 			if (resp->eee_config_phy_addr &
8901 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_ENABLED)
8902 				eee->eee_enabled = 1;
8903 
8904 			fw_speeds = le16_to_cpu(resp->adv_eee_link_speed_mask);
8905 			eee->advertised =
8906 				_bnxt_fw_to_ethtool_adv_spds(fw_speeds, 0);
8907 
8908 			if (resp->eee_config_phy_addr &
8909 			    PORT_PHY_QCFG_RESP_EEE_CONFIG_EEE_TX_LPI) {
8910 				__le32 tmr;
8911 
8912 				eee->tx_lpi_enabled = 1;
8913 				tmr = resp->xcvr_identifier_type_tx_lpi_timer;
8914 				eee->tx_lpi_timer = le32_to_cpu(tmr) &
8915 					PORT_PHY_QCFG_RESP_TX_LPI_TIMER_MASK;
8916 			}
8917 		}
8918 	}
8919 
8920 	link_info->fec_cfg = PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED;
8921 	if (bp->hwrm_spec_code >= 0x10504)
8922 		link_info->fec_cfg = le16_to_cpu(resp->fec_cfg);
8923 
8924 	/* TODO: need to add more logic to report VF link */
8925 	if (chng_link_state) {
8926 		if (link_info->phy_link_status == BNXT_LINK_LINK)
8927 			link_info->link_up = 1;
8928 		else
8929 			link_info->link_up = 0;
8930 		if (link_up != link_info->link_up)
8931 			bnxt_report_link(bp);
8932 	} else {
8933 		/* alwasy link down if not require to update link state */
8934 		link_info->link_up = 0;
8935 	}
8936 	mutex_unlock(&bp->hwrm_cmd_lock);
8937 
8938 	if (!BNXT_PHY_CFG_ABLE(bp))
8939 		return 0;
8940 
8941 	diff = link_info->support_auto_speeds ^ link_info->advertising;
8942 	if ((link_info->support_auto_speeds | diff) !=
8943 	    link_info->support_auto_speeds) {
8944 		/* An advertised speed is no longer supported, so we need to
8945 		 * update the advertisement settings.  Caller holds RTNL
8946 		 * so we can modify link settings.
8947 		 */
8948 		link_info->advertising = link_info->support_auto_speeds;
8949 		if (link_info->autoneg & BNXT_AUTONEG_SPEED)
8950 			bnxt_hwrm_set_link_setting(bp, true, false);
8951 	}
8952 	return 0;
8953 }
8954 
8955 static void bnxt_get_port_module_status(struct bnxt *bp)
8956 {
8957 	struct bnxt_link_info *link_info = &bp->link_info;
8958 	struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp;
8959 	u8 module_status;
8960 
8961 	if (bnxt_update_link(bp, true))
8962 		return;
8963 
8964 	module_status = link_info->module_status;
8965 	switch (module_status) {
8966 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX:
8967 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN:
8968 	case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG:
8969 		netdev_warn(bp->dev, "Unqualified SFP+ module detected on port %d\n",
8970 			    bp->pf.port_id);
8971 		if (bp->hwrm_spec_code >= 0x10201) {
8972 			netdev_warn(bp->dev, "Module part number %s\n",
8973 				    resp->phy_vendor_partnumber);
8974 		}
8975 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX)
8976 			netdev_warn(bp->dev, "TX is disabled\n");
8977 		if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN)
8978 			netdev_warn(bp->dev, "SFP+ module is shutdown\n");
8979 	}
8980 }
8981 
8982 static void
8983 bnxt_hwrm_set_pause_common(struct bnxt *bp, struct hwrm_port_phy_cfg_input *req)
8984 {
8985 	if (bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) {
8986 		if (bp->hwrm_spec_code >= 0x10201)
8987 			req->auto_pause =
8988 				PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE;
8989 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8990 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX;
8991 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8992 			req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX;
8993 		req->enables |=
8994 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
8995 	} else {
8996 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_RX)
8997 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX;
8998 		if (bp->link_info.req_flow_ctrl & BNXT_LINK_PAUSE_TX)
8999 			req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX;
9000 		req->enables |=
9001 			cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE);
9002 		if (bp->hwrm_spec_code >= 0x10201) {
9003 			req->auto_pause = req->force_pause;
9004 			req->enables |= cpu_to_le32(
9005 				PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE);
9006 		}
9007 	}
9008 }
9009 
9010 static void bnxt_hwrm_set_link_common(struct bnxt *bp,
9011 				      struct hwrm_port_phy_cfg_input *req)
9012 {
9013 	u8 autoneg = bp->link_info.autoneg;
9014 	u16 fw_link_speed = bp->link_info.req_link_speed;
9015 	u16 advertising = bp->link_info.advertising;
9016 
9017 	if (autoneg & BNXT_AUTONEG_SPEED) {
9018 		req->auto_mode |=
9019 			PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK;
9020 
9021 		req->enables |= cpu_to_le32(
9022 			PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEED_MASK);
9023 		req->auto_link_speed_mask = cpu_to_le16(advertising);
9024 
9025 		req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE);
9026 		req->flags |=
9027 			cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG);
9028 	} else {
9029 		req->force_link_speed = cpu_to_le16(fw_link_speed);
9030 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE);
9031 	}
9032 
9033 	/* tell chimp that the setting takes effect immediately */
9034 	req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY);
9035 }
9036 
9037 int bnxt_hwrm_set_pause(struct bnxt *bp)
9038 {
9039 	struct hwrm_port_phy_cfg_input req = {0};
9040 	int rc;
9041 
9042 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9043 	bnxt_hwrm_set_pause_common(bp, &req);
9044 
9045 	if ((bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL) ||
9046 	    bp->link_info.force_link_chng)
9047 		bnxt_hwrm_set_link_common(bp, &req);
9048 
9049 	mutex_lock(&bp->hwrm_cmd_lock);
9050 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9051 	if (!rc && !(bp->link_info.autoneg & BNXT_AUTONEG_FLOW_CTRL)) {
9052 		/* since changing of pause setting doesn't trigger any link
9053 		 * change event, the driver needs to update the current pause
9054 		 * result upon successfully return of the phy_cfg command
9055 		 */
9056 		bp->link_info.pause =
9057 		bp->link_info.force_pause_setting = bp->link_info.req_flow_ctrl;
9058 		bp->link_info.auto_pause_setting = 0;
9059 		if (!bp->link_info.force_link_chng)
9060 			bnxt_report_link(bp);
9061 	}
9062 	bp->link_info.force_link_chng = false;
9063 	mutex_unlock(&bp->hwrm_cmd_lock);
9064 	return rc;
9065 }
9066 
9067 static void bnxt_hwrm_set_eee(struct bnxt *bp,
9068 			      struct hwrm_port_phy_cfg_input *req)
9069 {
9070 	struct ethtool_eee *eee = &bp->eee;
9071 
9072 	if (eee->eee_enabled) {
9073 		u16 eee_speeds;
9074 		u32 flags = PORT_PHY_CFG_REQ_FLAGS_EEE_ENABLE;
9075 
9076 		if (eee->tx_lpi_enabled)
9077 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_ENABLE;
9078 		else
9079 			flags |= PORT_PHY_CFG_REQ_FLAGS_EEE_TX_LPI_DISABLE;
9080 
9081 		req->flags |= cpu_to_le32(flags);
9082 		eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised);
9083 		req->eee_link_speed_mask = cpu_to_le16(eee_speeds);
9084 		req->tx_lpi_timer = cpu_to_le32(eee->tx_lpi_timer);
9085 	} else {
9086 		req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_EEE_DISABLE);
9087 	}
9088 }
9089 
9090 int bnxt_hwrm_set_link_setting(struct bnxt *bp, bool set_pause, bool set_eee)
9091 {
9092 	struct hwrm_port_phy_cfg_input req = {0};
9093 
9094 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9095 	if (set_pause)
9096 		bnxt_hwrm_set_pause_common(bp, &req);
9097 
9098 	bnxt_hwrm_set_link_common(bp, &req);
9099 
9100 	if (set_eee)
9101 		bnxt_hwrm_set_eee(bp, &req);
9102 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9103 }
9104 
9105 static int bnxt_hwrm_shutdown_link(struct bnxt *bp)
9106 {
9107 	struct hwrm_port_phy_cfg_input req = {0};
9108 
9109 	if (!BNXT_SINGLE_PF(bp))
9110 		return 0;
9111 
9112 	if (pci_num_vf(bp->pdev))
9113 		return 0;
9114 
9115 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_CFG, -1, -1);
9116 	req.flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN);
9117 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9118 }
9119 
9120 static int bnxt_fw_init_one(struct bnxt *bp);
9121 
9122 static int bnxt_hwrm_if_change(struct bnxt *bp, bool up)
9123 {
9124 	struct hwrm_func_drv_if_change_output *resp = bp->hwrm_cmd_resp_addr;
9125 	struct hwrm_func_drv_if_change_input req = {0};
9126 	bool resc_reinit = false, fw_reset = false;
9127 	u32 flags = 0;
9128 	int rc;
9129 
9130 	if (!(bp->fw_cap & BNXT_FW_CAP_IF_CHANGE))
9131 		return 0;
9132 
9133 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_IF_CHANGE, -1, -1);
9134 	if (up)
9135 		req.flags = cpu_to_le32(FUNC_DRV_IF_CHANGE_REQ_FLAGS_UP);
9136 	mutex_lock(&bp->hwrm_cmd_lock);
9137 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9138 	if (!rc)
9139 		flags = le32_to_cpu(resp->flags);
9140 	mutex_unlock(&bp->hwrm_cmd_lock);
9141 	if (rc)
9142 		return rc;
9143 
9144 	if (!up)
9145 		return 0;
9146 
9147 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_RESC_CHANGE)
9148 		resc_reinit = true;
9149 	if (flags & FUNC_DRV_IF_CHANGE_RESP_FLAGS_HOT_FW_RESET_DONE)
9150 		fw_reset = true;
9151 
9152 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state) && !fw_reset) {
9153 		netdev_err(bp->dev, "RESET_DONE not set during FW reset.\n");
9154 		return -ENODEV;
9155 	}
9156 	if (resc_reinit || fw_reset) {
9157 		if (fw_reset) {
9158 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
9159 				bnxt_ulp_stop(bp);
9160 			bnxt_free_ctx_mem(bp);
9161 			kfree(bp->ctx);
9162 			bp->ctx = NULL;
9163 			bnxt_dcb_free(bp);
9164 			rc = bnxt_fw_init_one(bp);
9165 			if (rc) {
9166 				set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9167 				return rc;
9168 			}
9169 			bnxt_clear_int_mode(bp);
9170 			rc = bnxt_init_int_mode(bp);
9171 			if (rc) {
9172 				netdev_err(bp->dev, "init int mode failed\n");
9173 				return rc;
9174 			}
9175 			set_bit(BNXT_STATE_FW_RESET_DET, &bp->state);
9176 		}
9177 		if (BNXT_NEW_RM(bp)) {
9178 			struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
9179 
9180 			rc = bnxt_hwrm_func_resc_qcaps(bp, true);
9181 			hw_resc->resv_cp_rings = 0;
9182 			hw_resc->resv_stat_ctxs = 0;
9183 			hw_resc->resv_irqs = 0;
9184 			hw_resc->resv_tx_rings = 0;
9185 			hw_resc->resv_rx_rings = 0;
9186 			hw_resc->resv_hw_ring_grps = 0;
9187 			hw_resc->resv_vnics = 0;
9188 			if (!fw_reset) {
9189 				bp->tx_nr_rings = 0;
9190 				bp->rx_nr_rings = 0;
9191 			}
9192 		}
9193 	}
9194 	return 0;
9195 }
9196 
9197 static int bnxt_hwrm_port_led_qcaps(struct bnxt *bp)
9198 {
9199 	struct hwrm_port_led_qcaps_output *resp = bp->hwrm_cmd_resp_addr;
9200 	struct hwrm_port_led_qcaps_input req = {0};
9201 	struct bnxt_pf_info *pf = &bp->pf;
9202 	int rc;
9203 
9204 	bp->num_leds = 0;
9205 	if (BNXT_VF(bp) || bp->hwrm_spec_code < 0x10601)
9206 		return 0;
9207 
9208 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_LED_QCAPS, -1, -1);
9209 	req.port_id = cpu_to_le16(pf->port_id);
9210 	mutex_lock(&bp->hwrm_cmd_lock);
9211 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9212 	if (rc) {
9213 		mutex_unlock(&bp->hwrm_cmd_lock);
9214 		return rc;
9215 	}
9216 	if (resp->num_leds > 0 && resp->num_leds < BNXT_MAX_LED) {
9217 		int i;
9218 
9219 		bp->num_leds = resp->num_leds;
9220 		memcpy(bp->leds, &resp->led0_id, sizeof(bp->leds[0]) *
9221 						 bp->num_leds);
9222 		for (i = 0; i < bp->num_leds; i++) {
9223 			struct bnxt_led_info *led = &bp->leds[i];
9224 			__le16 caps = led->led_state_caps;
9225 
9226 			if (!led->led_group_id ||
9227 			    !BNXT_LED_ALT_BLINK_CAP(caps)) {
9228 				bp->num_leds = 0;
9229 				break;
9230 			}
9231 		}
9232 	}
9233 	mutex_unlock(&bp->hwrm_cmd_lock);
9234 	return 0;
9235 }
9236 
9237 int bnxt_hwrm_alloc_wol_fltr(struct bnxt *bp)
9238 {
9239 	struct hwrm_wol_filter_alloc_input req = {0};
9240 	struct hwrm_wol_filter_alloc_output *resp = bp->hwrm_cmd_resp_addr;
9241 	int rc;
9242 
9243 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_ALLOC, -1, -1);
9244 	req.port_id = cpu_to_le16(bp->pf.port_id);
9245 	req.wol_type = WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT;
9246 	req.enables = cpu_to_le32(WOL_FILTER_ALLOC_REQ_ENABLES_MAC_ADDRESS);
9247 	memcpy(req.mac_address, bp->dev->dev_addr, ETH_ALEN);
9248 	mutex_lock(&bp->hwrm_cmd_lock);
9249 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9250 	if (!rc)
9251 		bp->wol_filter_id = resp->wol_filter_id;
9252 	mutex_unlock(&bp->hwrm_cmd_lock);
9253 	return rc;
9254 }
9255 
9256 int bnxt_hwrm_free_wol_fltr(struct bnxt *bp)
9257 {
9258 	struct hwrm_wol_filter_free_input req = {0};
9259 
9260 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_FREE, -1, -1);
9261 	req.port_id = cpu_to_le16(bp->pf.port_id);
9262 	req.enables = cpu_to_le32(WOL_FILTER_FREE_REQ_ENABLES_WOL_FILTER_ID);
9263 	req.wol_filter_id = bp->wol_filter_id;
9264 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9265 }
9266 
9267 static u16 bnxt_hwrm_get_wol_fltrs(struct bnxt *bp, u16 handle)
9268 {
9269 	struct hwrm_wol_filter_qcfg_input req = {0};
9270 	struct hwrm_wol_filter_qcfg_output *resp = bp->hwrm_cmd_resp_addr;
9271 	u16 next_handle = 0;
9272 	int rc;
9273 
9274 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_WOL_FILTER_QCFG, -1, -1);
9275 	req.port_id = cpu_to_le16(bp->pf.port_id);
9276 	req.handle = cpu_to_le16(handle);
9277 	mutex_lock(&bp->hwrm_cmd_lock);
9278 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9279 	if (!rc) {
9280 		next_handle = le16_to_cpu(resp->next_handle);
9281 		if (next_handle != 0) {
9282 			if (resp->wol_type ==
9283 			    WOL_FILTER_ALLOC_REQ_WOL_TYPE_MAGICPKT) {
9284 				bp->wol = 1;
9285 				bp->wol_filter_id = resp->wol_filter_id;
9286 			}
9287 		}
9288 	}
9289 	mutex_unlock(&bp->hwrm_cmd_lock);
9290 	return next_handle;
9291 }
9292 
9293 static void bnxt_get_wol_settings(struct bnxt *bp)
9294 {
9295 	u16 handle = 0;
9296 
9297 	bp->wol = 0;
9298 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_WOL_CAP))
9299 		return;
9300 
9301 	do {
9302 		handle = bnxt_hwrm_get_wol_fltrs(bp, handle);
9303 	} while (handle && handle != 0xffff);
9304 }
9305 
9306 #ifdef CONFIG_BNXT_HWMON
9307 static ssize_t bnxt_show_temp(struct device *dev,
9308 			      struct device_attribute *devattr, char *buf)
9309 {
9310 	struct hwrm_temp_monitor_query_input req = {0};
9311 	struct hwrm_temp_monitor_query_output *resp;
9312 	struct bnxt *bp = dev_get_drvdata(dev);
9313 	u32 len = 0;
9314 
9315 	resp = bp->hwrm_cmd_resp_addr;
9316 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_TEMP_MONITOR_QUERY, -1, -1);
9317 	mutex_lock(&bp->hwrm_cmd_lock);
9318 	if (!_hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT))
9319 		len = sprintf(buf, "%u\n", resp->temp * 1000); /* display millidegree */
9320 	mutex_unlock(&bp->hwrm_cmd_lock);
9321 
9322 	if (len)
9323 		return len;
9324 
9325 	return sprintf(buf, "unknown\n");
9326 }
9327 static SENSOR_DEVICE_ATTR(temp1_input, 0444, bnxt_show_temp, NULL, 0);
9328 
9329 static struct attribute *bnxt_attrs[] = {
9330 	&sensor_dev_attr_temp1_input.dev_attr.attr,
9331 	NULL
9332 };
9333 ATTRIBUTE_GROUPS(bnxt);
9334 
9335 static void bnxt_hwmon_close(struct bnxt *bp)
9336 {
9337 	if (bp->hwmon_dev) {
9338 		hwmon_device_unregister(bp->hwmon_dev);
9339 		bp->hwmon_dev = NULL;
9340 	}
9341 }
9342 
9343 static void bnxt_hwmon_open(struct bnxt *bp)
9344 {
9345 	struct pci_dev *pdev = bp->pdev;
9346 
9347 	if (bp->hwmon_dev)
9348 		return;
9349 
9350 	bp->hwmon_dev = hwmon_device_register_with_groups(&pdev->dev,
9351 							  DRV_MODULE_NAME, bp,
9352 							  bnxt_groups);
9353 	if (IS_ERR(bp->hwmon_dev)) {
9354 		bp->hwmon_dev = NULL;
9355 		dev_warn(&pdev->dev, "Cannot register hwmon device\n");
9356 	}
9357 }
9358 #else
9359 static void bnxt_hwmon_close(struct bnxt *bp)
9360 {
9361 }
9362 
9363 static void bnxt_hwmon_open(struct bnxt *bp)
9364 {
9365 }
9366 #endif
9367 
9368 static bool bnxt_eee_config_ok(struct bnxt *bp)
9369 {
9370 	struct ethtool_eee *eee = &bp->eee;
9371 	struct bnxt_link_info *link_info = &bp->link_info;
9372 
9373 	if (!(bp->flags & BNXT_FLAG_EEE_CAP))
9374 		return true;
9375 
9376 	if (eee->eee_enabled) {
9377 		u32 advertising =
9378 			_bnxt_fw_to_ethtool_adv_spds(link_info->advertising, 0);
9379 
9380 		if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9381 			eee->eee_enabled = 0;
9382 			return false;
9383 		}
9384 		if (eee->advertised & ~advertising) {
9385 			eee->advertised = advertising & eee->supported;
9386 			return false;
9387 		}
9388 	}
9389 	return true;
9390 }
9391 
9392 static int bnxt_update_phy_setting(struct bnxt *bp)
9393 {
9394 	int rc;
9395 	bool update_link = false;
9396 	bool update_pause = false;
9397 	bool update_eee = false;
9398 	struct bnxt_link_info *link_info = &bp->link_info;
9399 
9400 	rc = bnxt_update_link(bp, true);
9401 	if (rc) {
9402 		netdev_err(bp->dev, "failed to update link (rc: %x)\n",
9403 			   rc);
9404 		return rc;
9405 	}
9406 	if (!BNXT_SINGLE_PF(bp))
9407 		return 0;
9408 
9409 	if ((link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9410 	    (link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH) !=
9411 	    link_info->req_flow_ctrl)
9412 		update_pause = true;
9413 	if (!(link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL) &&
9414 	    link_info->force_pause_setting != link_info->req_flow_ctrl)
9415 		update_pause = true;
9416 	if (!(link_info->autoneg & BNXT_AUTONEG_SPEED)) {
9417 		if (BNXT_AUTO_MODE(link_info->auto_mode))
9418 			update_link = true;
9419 		if (link_info->req_link_speed != link_info->force_link_speed)
9420 			update_link = true;
9421 		if (link_info->req_duplex != link_info->duplex_setting)
9422 			update_link = true;
9423 	} else {
9424 		if (link_info->auto_mode == BNXT_LINK_AUTO_NONE)
9425 			update_link = true;
9426 		if (link_info->advertising != link_info->auto_link_speeds)
9427 			update_link = true;
9428 	}
9429 
9430 	/* The last close may have shutdown the link, so need to call
9431 	 * PHY_CFG to bring it back up.
9432 	 */
9433 	if (!bp->link_info.link_up)
9434 		update_link = true;
9435 
9436 	if (!bnxt_eee_config_ok(bp))
9437 		update_eee = true;
9438 
9439 	if (update_link)
9440 		rc = bnxt_hwrm_set_link_setting(bp, update_pause, update_eee);
9441 	else if (update_pause)
9442 		rc = bnxt_hwrm_set_pause(bp);
9443 	if (rc) {
9444 		netdev_err(bp->dev, "failed to update phy setting (rc: %x)\n",
9445 			   rc);
9446 		return rc;
9447 	}
9448 
9449 	return rc;
9450 }
9451 
9452 /* Common routine to pre-map certain register block to different GRC window.
9453  * A PF has 16 4K windows and a VF has 4 4K windows. However, only 15 windows
9454  * in PF and 3 windows in VF that can be customized to map in different
9455  * register blocks.
9456  */
9457 static void bnxt_preset_reg_win(struct bnxt *bp)
9458 {
9459 	if (BNXT_PF(bp)) {
9460 		/* CAG registers map to GRC window #4 */
9461 		writel(BNXT_CAG_REG_BASE,
9462 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 12);
9463 	}
9464 }
9465 
9466 static int bnxt_init_dflt_ring_mode(struct bnxt *bp);
9467 
9468 static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9469 {
9470 	int rc = 0;
9471 
9472 	bnxt_preset_reg_win(bp);
9473 	netif_carrier_off(bp->dev);
9474 	if (irq_re_init) {
9475 		/* Reserve rings now if none were reserved at driver probe. */
9476 		rc = bnxt_init_dflt_ring_mode(bp);
9477 		if (rc) {
9478 			netdev_err(bp->dev, "Failed to reserve default rings at open\n");
9479 			return rc;
9480 		}
9481 	}
9482 	rc = bnxt_reserve_rings(bp, irq_re_init);
9483 	if (rc)
9484 		return rc;
9485 	if ((bp->flags & BNXT_FLAG_RFS) &&
9486 	    !(bp->flags & BNXT_FLAG_USING_MSIX)) {
9487 		/* disable RFS if falling back to INTA */
9488 		bp->dev->hw_features &= ~NETIF_F_NTUPLE;
9489 		bp->flags &= ~BNXT_FLAG_RFS;
9490 	}
9491 
9492 	rc = bnxt_alloc_mem(bp, irq_re_init);
9493 	if (rc) {
9494 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9495 		goto open_err_free_mem;
9496 	}
9497 
9498 	if (irq_re_init) {
9499 		bnxt_init_napi(bp);
9500 		rc = bnxt_request_irq(bp);
9501 		if (rc) {
9502 			netdev_err(bp->dev, "bnxt_request_irq err: %x\n", rc);
9503 			goto open_err_irq;
9504 		}
9505 	}
9506 
9507 	rc = bnxt_init_nic(bp, irq_re_init);
9508 	if (rc) {
9509 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9510 		goto open_err_irq;
9511 	}
9512 
9513 	bnxt_enable_napi(bp);
9514 	bnxt_debug_dev_init(bp);
9515 
9516 	if (link_re_init) {
9517 		mutex_lock(&bp->link_lock);
9518 		rc = bnxt_update_phy_setting(bp);
9519 		mutex_unlock(&bp->link_lock);
9520 		if (rc) {
9521 			netdev_warn(bp->dev, "failed to update phy settings\n");
9522 			if (BNXT_SINGLE_PF(bp)) {
9523 				bp->link_info.phy_retry = true;
9524 				bp->link_info.phy_retry_expires =
9525 					jiffies + 5 * HZ;
9526 			}
9527 		}
9528 	}
9529 
9530 	if (irq_re_init)
9531 		udp_tunnel_nic_reset_ntf(bp->dev);
9532 
9533 	set_bit(BNXT_STATE_OPEN, &bp->state);
9534 	bnxt_enable_int(bp);
9535 	/* Enable TX queues */
9536 	bnxt_tx_enable(bp);
9537 	mod_timer(&bp->timer, jiffies + bp->current_interval);
9538 	/* Poll link status and check for SFP+ module status */
9539 	bnxt_get_port_module_status(bp);
9540 
9541 	/* VF-reps may need to be re-opened after the PF is re-opened */
9542 	if (BNXT_PF(bp))
9543 		bnxt_vf_reps_open(bp);
9544 	return 0;
9545 
9546 open_err_irq:
9547 	bnxt_del_napi(bp);
9548 
9549 open_err_free_mem:
9550 	bnxt_free_skbs(bp);
9551 	bnxt_free_irq(bp);
9552 	bnxt_free_mem(bp, true);
9553 	return rc;
9554 }
9555 
9556 /* rtnl_lock held */
9557 int bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9558 {
9559 	int rc = 0;
9560 
9561 	rc = __bnxt_open_nic(bp, irq_re_init, link_re_init);
9562 	if (rc) {
9563 		netdev_err(bp->dev, "nic open fail (rc: %x)\n", rc);
9564 		dev_close(bp->dev);
9565 	}
9566 	return rc;
9567 }
9568 
9569 /* rtnl_lock held, open the NIC half way by allocating all resources, but
9570  * NAPI, IRQ, and TX are not enabled.  This is mainly used for offline
9571  * self tests.
9572  */
9573 int bnxt_half_open_nic(struct bnxt *bp)
9574 {
9575 	int rc = 0;
9576 
9577 	rc = bnxt_alloc_mem(bp, false);
9578 	if (rc) {
9579 		netdev_err(bp->dev, "bnxt_alloc_mem err: %x\n", rc);
9580 		goto half_open_err;
9581 	}
9582 	rc = bnxt_init_nic(bp, false);
9583 	if (rc) {
9584 		netdev_err(bp->dev, "bnxt_init_nic err: %x\n", rc);
9585 		goto half_open_err;
9586 	}
9587 	return 0;
9588 
9589 half_open_err:
9590 	bnxt_free_skbs(bp);
9591 	bnxt_free_mem(bp, false);
9592 	dev_close(bp->dev);
9593 	return rc;
9594 }
9595 
9596 /* rtnl_lock held, this call can only be made after a previous successful
9597  * call to bnxt_half_open_nic().
9598  */
9599 void bnxt_half_close_nic(struct bnxt *bp)
9600 {
9601 	bnxt_hwrm_resource_free(bp, false, false);
9602 	bnxt_free_skbs(bp);
9603 	bnxt_free_mem(bp, false);
9604 }
9605 
9606 static void bnxt_reenable_sriov(struct bnxt *bp)
9607 {
9608 	if (BNXT_PF(bp)) {
9609 		struct bnxt_pf_info *pf = &bp->pf;
9610 		int n = pf->active_vfs;
9611 
9612 		if (n)
9613 			bnxt_cfg_hw_sriov(bp, &n, true);
9614 	}
9615 }
9616 
9617 static int bnxt_open(struct net_device *dev)
9618 {
9619 	struct bnxt *bp = netdev_priv(dev);
9620 	int rc;
9621 
9622 	if (test_bit(BNXT_STATE_ABORT_ERR, &bp->state)) {
9623 		netdev_err(bp->dev, "A previous firmware reset did not complete, aborting\n");
9624 		return -ENODEV;
9625 	}
9626 
9627 	rc = bnxt_hwrm_if_change(bp, true);
9628 	if (rc)
9629 		return rc;
9630 	rc = __bnxt_open_nic(bp, true, true);
9631 	if (rc) {
9632 		bnxt_hwrm_if_change(bp, false);
9633 	} else {
9634 		if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) {
9635 			if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9636 				bnxt_ulp_start(bp, 0);
9637 				bnxt_reenable_sriov(bp);
9638 			}
9639 		}
9640 		bnxt_hwmon_open(bp);
9641 	}
9642 
9643 	return rc;
9644 }
9645 
9646 static bool bnxt_drv_busy(struct bnxt *bp)
9647 {
9648 	return (test_bit(BNXT_STATE_IN_SP_TASK, &bp->state) ||
9649 		test_bit(BNXT_STATE_READ_STATS, &bp->state));
9650 }
9651 
9652 static void bnxt_get_ring_stats(struct bnxt *bp,
9653 				struct rtnl_link_stats64 *stats);
9654 
9655 static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init,
9656 			     bool link_re_init)
9657 {
9658 	/* Close the VF-reps before closing PF */
9659 	if (BNXT_PF(bp))
9660 		bnxt_vf_reps_close(bp);
9661 
9662 	/* Change device state to avoid TX queue wake up's */
9663 	bnxt_tx_disable(bp);
9664 
9665 	clear_bit(BNXT_STATE_OPEN, &bp->state);
9666 	smp_mb__after_atomic();
9667 	while (bnxt_drv_busy(bp))
9668 		msleep(20);
9669 
9670 	/* Flush rings and and disable interrupts */
9671 	bnxt_shutdown_nic(bp, irq_re_init);
9672 
9673 	/* TODO CHIMP_FW: Link/PHY related cleanup if (link_re_init) */
9674 
9675 	bnxt_debug_dev_exit(bp);
9676 	bnxt_disable_napi(bp);
9677 	del_timer_sync(&bp->timer);
9678 	bnxt_free_skbs(bp);
9679 
9680 	/* Save ring stats before shutdown */
9681 	if (bp->bnapi && irq_re_init)
9682 		bnxt_get_ring_stats(bp, &bp->net_stats_prev);
9683 	if (irq_re_init) {
9684 		bnxt_free_irq(bp);
9685 		bnxt_del_napi(bp);
9686 	}
9687 	bnxt_free_mem(bp, irq_re_init);
9688 }
9689 
9690 int bnxt_close_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init)
9691 {
9692 	int rc = 0;
9693 
9694 	if (test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
9695 		/* If we get here, it means firmware reset is in progress
9696 		 * while we are trying to close.  We can safely proceed with
9697 		 * the close because we are holding rtnl_lock().  Some firmware
9698 		 * messages may fail as we proceed to close.  We set the
9699 		 * ABORT_ERR flag here so that the FW reset thread will later
9700 		 * abort when it gets the rtnl_lock() and sees the flag.
9701 		 */
9702 		netdev_warn(bp->dev, "FW reset in progress during close, FW reset will be aborted\n");
9703 		set_bit(BNXT_STATE_ABORT_ERR, &bp->state);
9704 	}
9705 
9706 #ifdef CONFIG_BNXT_SRIOV
9707 	if (bp->sriov_cfg) {
9708 		rc = wait_event_interruptible_timeout(bp->sriov_cfg_wait,
9709 						      !bp->sriov_cfg,
9710 						      BNXT_SRIOV_CFG_WAIT_TMO);
9711 		if (rc)
9712 			netdev_warn(bp->dev, "timeout waiting for SRIOV config operation to complete!\n");
9713 	}
9714 #endif
9715 	__bnxt_close_nic(bp, irq_re_init, link_re_init);
9716 	return rc;
9717 }
9718 
9719 static int bnxt_close(struct net_device *dev)
9720 {
9721 	struct bnxt *bp = netdev_priv(dev);
9722 
9723 	bnxt_hwmon_close(bp);
9724 	bnxt_close_nic(bp, true, true);
9725 	bnxt_hwrm_shutdown_link(bp);
9726 	bnxt_hwrm_if_change(bp, false);
9727 	return 0;
9728 }
9729 
9730 static int bnxt_hwrm_port_phy_read(struct bnxt *bp, u16 phy_addr, u16 reg,
9731 				   u16 *val)
9732 {
9733 	struct hwrm_port_phy_mdio_read_output *resp = bp->hwrm_cmd_resp_addr;
9734 	struct hwrm_port_phy_mdio_read_input req = {0};
9735 	int rc;
9736 
9737 	if (bp->hwrm_spec_code < 0x10a00)
9738 		return -EOPNOTSUPP;
9739 
9740 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_READ, -1, -1);
9741 	req.port_id = cpu_to_le16(bp->pf.port_id);
9742 	req.phy_addr = phy_addr;
9743 	req.reg_addr = cpu_to_le16(reg & 0x1f);
9744 	if (mdio_phy_id_is_c45(phy_addr)) {
9745 		req.cl45_mdio = 1;
9746 		req.phy_addr = mdio_phy_id_prtad(phy_addr);
9747 		req.dev_addr = mdio_phy_id_devad(phy_addr);
9748 		req.reg_addr = cpu_to_le16(reg);
9749 	}
9750 
9751 	mutex_lock(&bp->hwrm_cmd_lock);
9752 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9753 	if (!rc)
9754 		*val = le16_to_cpu(resp->reg_data);
9755 	mutex_unlock(&bp->hwrm_cmd_lock);
9756 	return rc;
9757 }
9758 
9759 static int bnxt_hwrm_port_phy_write(struct bnxt *bp, u16 phy_addr, u16 reg,
9760 				    u16 val)
9761 {
9762 	struct hwrm_port_phy_mdio_write_input req = {0};
9763 
9764 	if (bp->hwrm_spec_code < 0x10a00)
9765 		return -EOPNOTSUPP;
9766 
9767 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_PORT_PHY_MDIO_WRITE, -1, -1);
9768 	req.port_id = cpu_to_le16(bp->pf.port_id);
9769 	req.phy_addr = phy_addr;
9770 	req.reg_addr = cpu_to_le16(reg & 0x1f);
9771 	if (mdio_phy_id_is_c45(phy_addr)) {
9772 		req.cl45_mdio = 1;
9773 		req.phy_addr = mdio_phy_id_prtad(phy_addr);
9774 		req.dev_addr = mdio_phy_id_devad(phy_addr);
9775 		req.reg_addr = cpu_to_le16(reg);
9776 	}
9777 	req.reg_data = cpu_to_le16(val);
9778 
9779 	return hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
9780 }
9781 
9782 /* rtnl_lock held */
9783 static int bnxt_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
9784 {
9785 	struct mii_ioctl_data *mdio = if_mii(ifr);
9786 	struct bnxt *bp = netdev_priv(dev);
9787 	int rc;
9788 
9789 	switch (cmd) {
9790 	case SIOCGMIIPHY:
9791 		mdio->phy_id = bp->link_info.phy_addr;
9792 
9793 		fallthrough;
9794 	case SIOCGMIIREG: {
9795 		u16 mii_regval = 0;
9796 
9797 		if (!netif_running(dev))
9798 			return -EAGAIN;
9799 
9800 		rc = bnxt_hwrm_port_phy_read(bp, mdio->phy_id, mdio->reg_num,
9801 					     &mii_regval);
9802 		mdio->val_out = mii_regval;
9803 		return rc;
9804 	}
9805 
9806 	case SIOCSMIIREG:
9807 		if (!netif_running(dev))
9808 			return -EAGAIN;
9809 
9810 		return bnxt_hwrm_port_phy_write(bp, mdio->phy_id, mdio->reg_num,
9811 						mdio->val_in);
9812 
9813 	default:
9814 		/* do nothing */
9815 		break;
9816 	}
9817 	return -EOPNOTSUPP;
9818 }
9819 
9820 static void bnxt_get_ring_stats(struct bnxt *bp,
9821 				struct rtnl_link_stats64 *stats)
9822 {
9823 	int i;
9824 
9825 	for (i = 0; i < bp->cp_nr_rings; i++) {
9826 		struct bnxt_napi *bnapi = bp->bnapi[i];
9827 		struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
9828 		u64 *sw = cpr->stats.sw_stats;
9829 
9830 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_ucast_pkts);
9831 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9832 		stats->rx_packets += BNXT_GET_RING_STATS64(sw, rx_bcast_pkts);
9833 
9834 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_ucast_pkts);
9835 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_mcast_pkts);
9836 		stats->tx_packets += BNXT_GET_RING_STATS64(sw, tx_bcast_pkts);
9837 
9838 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_ucast_bytes);
9839 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_mcast_bytes);
9840 		stats->rx_bytes += BNXT_GET_RING_STATS64(sw, rx_bcast_bytes);
9841 
9842 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_ucast_bytes);
9843 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_mcast_bytes);
9844 		stats->tx_bytes += BNXT_GET_RING_STATS64(sw, tx_bcast_bytes);
9845 
9846 		stats->rx_missed_errors +=
9847 			BNXT_GET_RING_STATS64(sw, rx_discard_pkts);
9848 
9849 		stats->multicast += BNXT_GET_RING_STATS64(sw, rx_mcast_pkts);
9850 
9851 		stats->tx_dropped += BNXT_GET_RING_STATS64(sw, tx_error_pkts);
9852 	}
9853 }
9854 
9855 static void bnxt_add_prev_stats(struct bnxt *bp,
9856 				struct rtnl_link_stats64 *stats)
9857 {
9858 	struct rtnl_link_stats64 *prev_stats = &bp->net_stats_prev;
9859 
9860 	stats->rx_packets += prev_stats->rx_packets;
9861 	stats->tx_packets += prev_stats->tx_packets;
9862 	stats->rx_bytes += prev_stats->rx_bytes;
9863 	stats->tx_bytes += prev_stats->tx_bytes;
9864 	stats->rx_missed_errors += prev_stats->rx_missed_errors;
9865 	stats->multicast += prev_stats->multicast;
9866 	stats->tx_dropped += prev_stats->tx_dropped;
9867 }
9868 
9869 static void
9870 bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
9871 {
9872 	struct bnxt *bp = netdev_priv(dev);
9873 
9874 	set_bit(BNXT_STATE_READ_STATS, &bp->state);
9875 	/* Make sure bnxt_close_nic() sees that we are reading stats before
9876 	 * we check the BNXT_STATE_OPEN flag.
9877 	 */
9878 	smp_mb__after_atomic();
9879 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
9880 		clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9881 		*stats = bp->net_stats_prev;
9882 		return;
9883 	}
9884 
9885 	bnxt_get_ring_stats(bp, stats);
9886 	bnxt_add_prev_stats(bp, stats);
9887 
9888 	if (bp->flags & BNXT_FLAG_PORT_STATS) {
9889 		u64 *rx = bp->port_stats.sw_stats;
9890 		u64 *tx = bp->port_stats.sw_stats +
9891 			  BNXT_TX_PORT_STATS_BYTE_OFFSET / 8;
9892 
9893 		stats->rx_crc_errors =
9894 			BNXT_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames);
9895 		stats->rx_frame_errors =
9896 			BNXT_GET_RX_PORT_STATS64(rx, rx_align_err_frames);
9897 		stats->rx_length_errors =
9898 			BNXT_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) +
9899 			BNXT_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) +
9900 			BNXT_GET_RX_PORT_STATS64(rx, rx_runt_frames);
9901 		stats->rx_errors =
9902 			BNXT_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) +
9903 			BNXT_GET_RX_PORT_STATS64(rx, rx_jbr_frames);
9904 		stats->collisions =
9905 			BNXT_GET_TX_PORT_STATS64(tx, tx_total_collisions);
9906 		stats->tx_fifo_errors =
9907 			BNXT_GET_TX_PORT_STATS64(tx, tx_fifo_underruns);
9908 		stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err);
9909 	}
9910 	clear_bit(BNXT_STATE_READ_STATS, &bp->state);
9911 }
9912 
9913 static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask)
9914 {
9915 	struct net_device *dev = bp->dev;
9916 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9917 	struct netdev_hw_addr *ha;
9918 	u8 *haddr;
9919 	int mc_count = 0;
9920 	bool update = false;
9921 	int off = 0;
9922 
9923 	netdev_for_each_mc_addr(ha, dev) {
9924 		if (mc_count >= BNXT_MAX_MC_ADDRS) {
9925 			*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9926 			vnic->mc_list_count = 0;
9927 			return false;
9928 		}
9929 		haddr = ha->addr;
9930 		if (!ether_addr_equal(haddr, vnic->mc_list + off)) {
9931 			memcpy(vnic->mc_list + off, haddr, ETH_ALEN);
9932 			update = true;
9933 		}
9934 		off += ETH_ALEN;
9935 		mc_count++;
9936 	}
9937 	if (mc_count)
9938 		*rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_MCAST;
9939 
9940 	if (mc_count != vnic->mc_list_count) {
9941 		vnic->mc_list_count = mc_count;
9942 		update = true;
9943 	}
9944 	return update;
9945 }
9946 
9947 static bool bnxt_uc_list_updated(struct bnxt *bp)
9948 {
9949 	struct net_device *dev = bp->dev;
9950 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
9951 	struct netdev_hw_addr *ha;
9952 	int off = 0;
9953 
9954 	if (netdev_uc_count(dev) != (vnic->uc_filter_count - 1))
9955 		return true;
9956 
9957 	netdev_for_each_uc_addr(ha, dev) {
9958 		if (!ether_addr_equal(ha->addr, vnic->uc_list + off))
9959 			return true;
9960 
9961 		off += ETH_ALEN;
9962 	}
9963 	return false;
9964 }
9965 
9966 static void bnxt_set_rx_mode(struct net_device *dev)
9967 {
9968 	struct bnxt *bp = netdev_priv(dev);
9969 	struct bnxt_vnic_info *vnic;
9970 	bool mc_update = false;
9971 	bool uc_update;
9972 	u32 mask;
9973 
9974 	if (!test_bit(BNXT_STATE_OPEN, &bp->state))
9975 		return;
9976 
9977 	vnic = &bp->vnic_info[0];
9978 	mask = vnic->rx_mask;
9979 	mask &= ~(CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS |
9980 		  CFA_L2_SET_RX_MASK_REQ_MASK_MCAST |
9981 		  CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST |
9982 		  CFA_L2_SET_RX_MASK_REQ_MASK_BCAST);
9983 
9984 	if ((dev->flags & IFF_PROMISC) && bnxt_promisc_ok(bp))
9985 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
9986 
9987 	uc_update = bnxt_uc_list_updated(bp);
9988 
9989 	if (dev->flags & IFF_BROADCAST)
9990 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_BCAST;
9991 	if (dev->flags & IFF_ALLMULTI) {
9992 		mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
9993 		vnic->mc_list_count = 0;
9994 	} else {
9995 		mc_update = bnxt_mc_list_updated(bp, &mask);
9996 	}
9997 
9998 	if (mask != vnic->rx_mask || uc_update || mc_update) {
9999 		vnic->rx_mask = mask;
10000 
10001 		set_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event);
10002 		bnxt_queue_sp_work(bp);
10003 	}
10004 }
10005 
10006 static int bnxt_cfg_rx_mode(struct bnxt *bp)
10007 {
10008 	struct net_device *dev = bp->dev;
10009 	struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
10010 	struct netdev_hw_addr *ha;
10011 	int i, off = 0, rc;
10012 	bool uc_update;
10013 
10014 	netif_addr_lock_bh(dev);
10015 	uc_update = bnxt_uc_list_updated(bp);
10016 	netif_addr_unlock_bh(dev);
10017 
10018 	if (!uc_update)
10019 		goto skip_uc;
10020 
10021 	mutex_lock(&bp->hwrm_cmd_lock);
10022 	for (i = 1; i < vnic->uc_filter_count; i++) {
10023 		struct hwrm_cfa_l2_filter_free_input req = {0};
10024 
10025 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_L2_FILTER_FREE, -1,
10026 				       -1);
10027 
10028 		req.l2_filter_id = vnic->fw_l2_filter_id[i];
10029 
10030 		rc = _hwrm_send_message(bp, &req, sizeof(req),
10031 					HWRM_CMD_TIMEOUT);
10032 	}
10033 	mutex_unlock(&bp->hwrm_cmd_lock);
10034 
10035 	vnic->uc_filter_count = 1;
10036 
10037 	netif_addr_lock_bh(dev);
10038 	if (netdev_uc_count(dev) > (BNXT_MAX_UC_ADDRS - 1)) {
10039 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_PROMISCUOUS;
10040 	} else {
10041 		netdev_for_each_uc_addr(ha, dev) {
10042 			memcpy(vnic->uc_list + off, ha->addr, ETH_ALEN);
10043 			off += ETH_ALEN;
10044 			vnic->uc_filter_count++;
10045 		}
10046 	}
10047 	netif_addr_unlock_bh(dev);
10048 
10049 	for (i = 1, off = 0; i < vnic->uc_filter_count; i++, off += ETH_ALEN) {
10050 		rc = bnxt_hwrm_set_vnic_filter(bp, 0, i, vnic->uc_list + off);
10051 		if (rc) {
10052 			netdev_err(bp->dev, "HWRM vnic filter failure rc: %x\n",
10053 				   rc);
10054 			vnic->uc_filter_count = i;
10055 			return rc;
10056 		}
10057 	}
10058 
10059 skip_uc:
10060 	rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10061 	if (rc && vnic->mc_list_count) {
10062 		netdev_info(bp->dev, "Failed setting MC filters rc: %d, turning on ALL_MCAST mode\n",
10063 			    rc);
10064 		vnic->rx_mask |= CFA_L2_SET_RX_MASK_REQ_MASK_ALL_MCAST;
10065 		vnic->mc_list_count = 0;
10066 		rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, 0);
10067 	}
10068 	if (rc)
10069 		netdev_err(bp->dev, "HWRM cfa l2 rx mask failure rc: %d\n",
10070 			   rc);
10071 
10072 	return rc;
10073 }
10074 
10075 static bool bnxt_can_reserve_rings(struct bnxt *bp)
10076 {
10077 #ifdef CONFIG_BNXT_SRIOV
10078 	if (BNXT_NEW_RM(bp) && BNXT_VF(bp)) {
10079 		struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
10080 
10081 		/* No minimum rings were provisioned by the PF.  Don't
10082 		 * reserve rings by default when device is down.
10083 		 */
10084 		if (hw_resc->min_tx_rings || hw_resc->resv_tx_rings)
10085 			return true;
10086 
10087 		if (!netif_running(bp->dev))
10088 			return false;
10089 	}
10090 #endif
10091 	return true;
10092 }
10093 
10094 /* If the chip and firmware supports RFS */
10095 static bool bnxt_rfs_supported(struct bnxt *bp)
10096 {
10097 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
10098 		if (bp->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2)
10099 			return true;
10100 		return false;
10101 	}
10102 	if (BNXT_PF(bp) && !BNXT_CHIP_TYPE_NITRO_A0(bp))
10103 		return true;
10104 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10105 		return true;
10106 	return false;
10107 }
10108 
10109 /* If runtime conditions support RFS */
10110 static bool bnxt_rfs_capable(struct bnxt *bp)
10111 {
10112 #ifdef CONFIG_RFS_ACCEL
10113 	int vnics, max_vnics, max_rss_ctxs;
10114 
10115 	if (bp->flags & BNXT_FLAG_CHIP_P5)
10116 		return bnxt_rfs_supported(bp);
10117 	if (!(bp->flags & BNXT_FLAG_MSIX_CAP) || !bnxt_can_reserve_rings(bp))
10118 		return false;
10119 
10120 	vnics = 1 + bp->rx_nr_rings;
10121 	max_vnics = bnxt_get_max_func_vnics(bp);
10122 	max_rss_ctxs = bnxt_get_max_func_rss_ctxs(bp);
10123 
10124 	/* RSS contexts not a limiting factor */
10125 	if (bp->flags & BNXT_FLAG_NEW_RSS_CAP)
10126 		max_rss_ctxs = max_vnics;
10127 	if (vnics > max_vnics || vnics > max_rss_ctxs) {
10128 		if (bp->rx_nr_rings > 1)
10129 			netdev_warn(bp->dev,
10130 				    "Not enough resources to support NTUPLE filters, enough resources for up to %d rx rings\n",
10131 				    min(max_rss_ctxs - 1, max_vnics - 1));
10132 		return false;
10133 	}
10134 
10135 	if (!BNXT_NEW_RM(bp))
10136 		return true;
10137 
10138 	if (vnics == bp->hw_resc.resv_vnics)
10139 		return true;
10140 
10141 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, vnics);
10142 	if (vnics <= bp->hw_resc.resv_vnics)
10143 		return true;
10144 
10145 	netdev_warn(bp->dev, "Unable to reserve resources to support NTUPLE filters.\n");
10146 	bnxt_hwrm_reserve_rings(bp, 0, 0, 0, 0, 0, 1);
10147 	return false;
10148 #else
10149 	return false;
10150 #endif
10151 }
10152 
10153 static netdev_features_t bnxt_fix_features(struct net_device *dev,
10154 					   netdev_features_t features)
10155 {
10156 	struct bnxt *bp = netdev_priv(dev);
10157 	netdev_features_t vlan_features;
10158 
10159 	if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp))
10160 		features &= ~NETIF_F_NTUPLE;
10161 
10162 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10163 		features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
10164 
10165 	if (!(features & NETIF_F_GRO))
10166 		features &= ~NETIF_F_GRO_HW;
10167 
10168 	if (features & NETIF_F_GRO_HW)
10169 		features &= ~NETIF_F_LRO;
10170 
10171 	/* Both CTAG and STAG VLAN accelaration on the RX side have to be
10172 	 * turned on or off together.
10173 	 */
10174 	vlan_features = features & BNXT_HW_FEATURE_VLAN_ALL_RX;
10175 	if (vlan_features != BNXT_HW_FEATURE_VLAN_ALL_RX) {
10176 		if (dev->features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10177 			features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10178 		else if (vlan_features)
10179 			features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
10180 	}
10181 #ifdef CONFIG_BNXT_SRIOV
10182 	if (BNXT_VF(bp) && bp->vf.vlan)
10183 		features &= ~BNXT_HW_FEATURE_VLAN_ALL_RX;
10184 #endif
10185 	return features;
10186 }
10187 
10188 static int bnxt_set_features(struct net_device *dev, netdev_features_t features)
10189 {
10190 	struct bnxt *bp = netdev_priv(dev);
10191 	u32 flags = bp->flags;
10192 	u32 changes;
10193 	int rc = 0;
10194 	bool re_init = false;
10195 	bool update_tpa = false;
10196 
10197 	flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS;
10198 	if (features & NETIF_F_GRO_HW)
10199 		flags |= BNXT_FLAG_GRO;
10200 	else if (features & NETIF_F_LRO)
10201 		flags |= BNXT_FLAG_LRO;
10202 
10203 	if (bp->flags & BNXT_FLAG_NO_AGG_RINGS)
10204 		flags &= ~BNXT_FLAG_TPA;
10205 
10206 	if (features & BNXT_HW_FEATURE_VLAN_ALL_RX)
10207 		flags |= BNXT_FLAG_STRIP_VLAN;
10208 
10209 	if (features & NETIF_F_NTUPLE)
10210 		flags |= BNXT_FLAG_RFS;
10211 
10212 	changes = flags ^ bp->flags;
10213 	if (changes & BNXT_FLAG_TPA) {
10214 		update_tpa = true;
10215 		if ((bp->flags & BNXT_FLAG_TPA) == 0 ||
10216 		    (flags & BNXT_FLAG_TPA) == 0 ||
10217 		    (bp->flags & BNXT_FLAG_CHIP_P5))
10218 			re_init = true;
10219 	}
10220 
10221 	if (changes & ~BNXT_FLAG_TPA)
10222 		re_init = true;
10223 
10224 	if (flags != bp->flags) {
10225 		u32 old_flags = bp->flags;
10226 
10227 		if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10228 			bp->flags = flags;
10229 			if (update_tpa)
10230 				bnxt_set_ring_params(bp);
10231 			return rc;
10232 		}
10233 
10234 		if (re_init) {
10235 			bnxt_close_nic(bp, false, false);
10236 			bp->flags = flags;
10237 			if (update_tpa)
10238 				bnxt_set_ring_params(bp);
10239 
10240 			return bnxt_open_nic(bp, false, false);
10241 		}
10242 		if (update_tpa) {
10243 			bp->flags = flags;
10244 			rc = bnxt_set_tpa(bp,
10245 					  (flags & BNXT_FLAG_TPA) ?
10246 					  true : false);
10247 			if (rc)
10248 				bp->flags = old_flags;
10249 		}
10250 	}
10251 	return rc;
10252 }
10253 
10254 int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words,
10255 			 u32 *reg_buf)
10256 {
10257 	struct hwrm_dbg_read_direct_output *resp = bp->hwrm_cmd_resp_addr;
10258 	struct hwrm_dbg_read_direct_input req = {0};
10259 	__le32 *dbg_reg_buf;
10260 	dma_addr_t mapping;
10261 	int rc, i;
10262 
10263 	dbg_reg_buf = dma_alloc_coherent(&bp->pdev->dev, num_words * 4,
10264 					 &mapping, GFP_KERNEL);
10265 	if (!dbg_reg_buf)
10266 		return -ENOMEM;
10267 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_READ_DIRECT, -1, -1);
10268 	req.host_dest_addr = cpu_to_le64(mapping);
10269 	req.read_addr = cpu_to_le32(reg_off + CHIMP_REG_VIEW_ADDR);
10270 	req.read_len32 = cpu_to_le32(num_words);
10271 	mutex_lock(&bp->hwrm_cmd_lock);
10272 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10273 	if (rc || resp->error_code) {
10274 		rc = -EIO;
10275 		goto dbg_rd_reg_exit;
10276 	}
10277 	for (i = 0; i < num_words; i++)
10278 		reg_buf[i] = le32_to_cpu(dbg_reg_buf[i]);
10279 
10280 dbg_rd_reg_exit:
10281 	mutex_unlock(&bp->hwrm_cmd_lock);
10282 	dma_free_coherent(&bp->pdev->dev, num_words * 4, dbg_reg_buf, mapping);
10283 	return rc;
10284 }
10285 
10286 static int bnxt_dbg_hwrm_ring_info_get(struct bnxt *bp, u8 ring_type,
10287 				       u32 ring_id, u32 *prod, u32 *cons)
10288 {
10289 	struct hwrm_dbg_ring_info_get_output *resp = bp->hwrm_cmd_resp_addr;
10290 	struct hwrm_dbg_ring_info_get_input req = {0};
10291 	int rc;
10292 
10293 	bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_DBG_RING_INFO_GET, -1, -1);
10294 	req.ring_type = ring_type;
10295 	req.fw_ring_id = cpu_to_le32(ring_id);
10296 	mutex_lock(&bp->hwrm_cmd_lock);
10297 	rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
10298 	if (!rc) {
10299 		*prod = le32_to_cpu(resp->producer_index);
10300 		*cons = le32_to_cpu(resp->consumer_index);
10301 	}
10302 	mutex_unlock(&bp->hwrm_cmd_lock);
10303 	return rc;
10304 }
10305 
10306 static void bnxt_dump_tx_sw_state(struct bnxt_napi *bnapi)
10307 {
10308 	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
10309 	int i = bnapi->index;
10310 
10311 	if (!txr)
10312 		return;
10313 
10314 	netdev_info(bnapi->bp->dev, "[%d]: tx{fw_ring: %d prod: %x cons: %x}\n",
10315 		    i, txr->tx_ring_struct.fw_ring_id, txr->tx_prod,
10316 		    txr->tx_cons);
10317 }
10318 
10319 static void bnxt_dump_rx_sw_state(struct bnxt_napi *bnapi)
10320 {
10321 	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
10322 	int i = bnapi->index;
10323 
10324 	if (!rxr)
10325 		return;
10326 
10327 	netdev_info(bnapi->bp->dev, "[%d]: rx{fw_ring: %d prod: %x} rx_agg{fw_ring: %d agg_prod: %x sw_agg_prod: %x}\n",
10328 		    i, rxr->rx_ring_struct.fw_ring_id, rxr->rx_prod,
10329 		    rxr->rx_agg_ring_struct.fw_ring_id, rxr->rx_agg_prod,
10330 		    rxr->rx_sw_agg_prod);
10331 }
10332 
10333 static void bnxt_dump_cp_sw_state(struct bnxt_napi *bnapi)
10334 {
10335 	struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring;
10336 	int i = bnapi->index;
10337 
10338 	netdev_info(bnapi->bp->dev, "[%d]: cp{fw_ring: %d raw_cons: %x}\n",
10339 		    i, cpr->cp_ring_struct.fw_ring_id, cpr->cp_raw_cons);
10340 }
10341 
10342 static void bnxt_dbg_dump_states(struct bnxt *bp)
10343 {
10344 	int i;
10345 	struct bnxt_napi *bnapi;
10346 
10347 	for (i = 0; i < bp->cp_nr_rings; i++) {
10348 		bnapi = bp->bnapi[i];
10349 		if (netif_msg_drv(bp)) {
10350 			bnxt_dump_tx_sw_state(bnapi);
10351 			bnxt_dump_rx_sw_state(bnapi);
10352 			bnxt_dump_cp_sw_state(bnapi);
10353 		}
10354 	}
10355 }
10356 
10357 static void bnxt_reset_task(struct bnxt *bp, bool silent)
10358 {
10359 	if (!silent)
10360 		bnxt_dbg_dump_states(bp);
10361 	if (netif_running(bp->dev)) {
10362 		int rc;
10363 
10364 		if (silent) {
10365 			bnxt_close_nic(bp, false, false);
10366 			bnxt_open_nic(bp, false, false);
10367 		} else {
10368 			bnxt_ulp_stop(bp);
10369 			bnxt_close_nic(bp, true, false);
10370 			rc = bnxt_open_nic(bp, true, false);
10371 			bnxt_ulp_start(bp, rc);
10372 		}
10373 	}
10374 }
10375 
10376 static void bnxt_tx_timeout(struct net_device *dev, unsigned int txqueue)
10377 {
10378 	struct bnxt *bp = netdev_priv(dev);
10379 
10380 	netdev_err(bp->dev,  "TX timeout detected, starting reset task!\n");
10381 	set_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event);
10382 	bnxt_queue_sp_work(bp);
10383 }
10384 
10385 static void bnxt_fw_health_check(struct bnxt *bp)
10386 {
10387 	struct bnxt_fw_health *fw_health = bp->fw_health;
10388 	u32 val;
10389 
10390 	if (!fw_health->enabled || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10391 		return;
10392 
10393 	if (fw_health->tmr_counter) {
10394 		fw_health->tmr_counter--;
10395 		return;
10396 	}
10397 
10398 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10399 	if (val == fw_health->last_fw_heartbeat)
10400 		goto fw_reset;
10401 
10402 	fw_health->last_fw_heartbeat = val;
10403 
10404 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10405 	if (val != fw_health->last_fw_reset_cnt)
10406 		goto fw_reset;
10407 
10408 	fw_health->tmr_counter = fw_health->tmr_multiplier;
10409 	return;
10410 
10411 fw_reset:
10412 	set_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event);
10413 	bnxt_queue_sp_work(bp);
10414 }
10415 
10416 static void bnxt_timer(struct timer_list *t)
10417 {
10418 	struct bnxt *bp = from_timer(bp, t, timer);
10419 	struct net_device *dev = bp->dev;
10420 
10421 	if (!netif_running(dev) || !test_bit(BNXT_STATE_OPEN, &bp->state))
10422 		return;
10423 
10424 	if (atomic_read(&bp->intr_sem) != 0)
10425 		goto bnxt_restart_timer;
10426 
10427 	if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)
10428 		bnxt_fw_health_check(bp);
10429 
10430 	if (bp->link_info.link_up && bp->stats_coal_ticks) {
10431 		set_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event);
10432 		bnxt_queue_sp_work(bp);
10433 	}
10434 
10435 	if (bnxt_tc_flower_enabled(bp)) {
10436 		set_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event);
10437 		bnxt_queue_sp_work(bp);
10438 	}
10439 
10440 #ifdef CONFIG_RFS_ACCEL
10441 	if ((bp->flags & BNXT_FLAG_RFS) && bp->ntp_fltr_count) {
10442 		set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
10443 		bnxt_queue_sp_work(bp);
10444 	}
10445 #endif /*CONFIG_RFS_ACCEL*/
10446 
10447 	if (bp->link_info.phy_retry) {
10448 		if (time_after(jiffies, bp->link_info.phy_retry_expires)) {
10449 			bp->link_info.phy_retry = false;
10450 			netdev_warn(bp->dev, "failed to update phy settings after maximum retries.\n");
10451 		} else {
10452 			set_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event);
10453 			bnxt_queue_sp_work(bp);
10454 		}
10455 	}
10456 
10457 	if ((bp->flags & BNXT_FLAG_CHIP_P5) && !bp->chip_rev &&
10458 	    netif_carrier_ok(dev)) {
10459 		set_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event);
10460 		bnxt_queue_sp_work(bp);
10461 	}
10462 bnxt_restart_timer:
10463 	mod_timer(&bp->timer, jiffies + bp->current_interval);
10464 }
10465 
10466 static void bnxt_rtnl_lock_sp(struct bnxt *bp)
10467 {
10468 	/* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK
10469 	 * set.  If the device is being closed, bnxt_close() may be holding
10470 	 * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear.  So we
10471 	 * must clear BNXT_STATE_IN_SP_TASK before holding rtnl().
10472 	 */
10473 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10474 	rtnl_lock();
10475 }
10476 
10477 static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
10478 {
10479 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10480 	rtnl_unlock();
10481 }
10482 
10483 /* Only called from bnxt_sp_task() */
10484 static void bnxt_reset(struct bnxt *bp, bool silent)
10485 {
10486 	bnxt_rtnl_lock_sp(bp);
10487 	if (test_bit(BNXT_STATE_OPEN, &bp->state))
10488 		bnxt_reset_task(bp, silent);
10489 	bnxt_rtnl_unlock_sp(bp);
10490 }
10491 
10492 static void bnxt_fw_reset_close(struct bnxt *bp)
10493 {
10494 	bnxt_ulp_stop(bp);
10495 	/* When firmware is fatal state, disable PCI device to prevent
10496 	 * any potential bad DMAs before freeing kernel memory.
10497 	 */
10498 	if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10499 		pci_disable_device(bp->pdev);
10500 	__bnxt_close_nic(bp, true, false);
10501 	bnxt_clear_int_mode(bp);
10502 	bnxt_hwrm_func_drv_unrgtr(bp);
10503 	if (pci_is_enabled(bp->pdev))
10504 		pci_disable_device(bp->pdev);
10505 	bnxt_free_ctx_mem(bp);
10506 	kfree(bp->ctx);
10507 	bp->ctx = NULL;
10508 }
10509 
10510 static bool is_bnxt_fw_ok(struct bnxt *bp)
10511 {
10512 	struct bnxt_fw_health *fw_health = bp->fw_health;
10513 	bool no_heartbeat = false, has_reset = false;
10514 	u32 val;
10515 
10516 	val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG);
10517 	if (val == fw_health->last_fw_heartbeat)
10518 		no_heartbeat = true;
10519 
10520 	val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG);
10521 	if (val != fw_health->last_fw_reset_cnt)
10522 		has_reset = true;
10523 
10524 	if (!no_heartbeat && has_reset)
10525 		return true;
10526 
10527 	return false;
10528 }
10529 
10530 /* rtnl_lock is acquired before calling this function */
10531 static void bnxt_force_fw_reset(struct bnxt *bp)
10532 {
10533 	struct bnxt_fw_health *fw_health = bp->fw_health;
10534 	u32 wait_dsecs;
10535 
10536 	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
10537 	    test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))
10538 		return;
10539 
10540 	set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10541 	bnxt_fw_reset_close(bp);
10542 	wait_dsecs = fw_health->master_func_wait_dsecs;
10543 	if (fw_health->master) {
10544 		if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU)
10545 			wait_dsecs = 0;
10546 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
10547 	} else {
10548 		bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10;
10549 		wait_dsecs = fw_health->normal_func_wait_dsecs;
10550 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10551 	}
10552 
10553 	bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs;
10554 	bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs;
10555 	bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
10556 }
10557 
10558 void bnxt_fw_exception(struct bnxt *bp)
10559 {
10560 	netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n");
10561 	set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
10562 	bnxt_rtnl_lock_sp(bp);
10563 	bnxt_force_fw_reset(bp);
10564 	bnxt_rtnl_unlock_sp(bp);
10565 }
10566 
10567 /* Returns the number of registered VFs, or 1 if VF configuration is pending, or
10568  * < 0 on error.
10569  */
10570 static int bnxt_get_registered_vfs(struct bnxt *bp)
10571 {
10572 #ifdef CONFIG_BNXT_SRIOV
10573 	int rc;
10574 
10575 	if (!BNXT_PF(bp))
10576 		return 0;
10577 
10578 	rc = bnxt_hwrm_func_qcfg(bp);
10579 	if (rc) {
10580 		netdev_err(bp->dev, "func_qcfg cmd failed, rc = %d\n", rc);
10581 		return rc;
10582 	}
10583 	if (bp->pf.registered_vfs)
10584 		return bp->pf.registered_vfs;
10585 	if (bp->sriov_cfg)
10586 		return 1;
10587 #endif
10588 	return 0;
10589 }
10590 
10591 void bnxt_fw_reset(struct bnxt *bp)
10592 {
10593 	bnxt_rtnl_lock_sp(bp);
10594 	if (test_bit(BNXT_STATE_OPEN, &bp->state) &&
10595 	    !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
10596 		int n = 0, tmo;
10597 
10598 		set_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10599 		if (bp->pf.active_vfs &&
10600 		    !test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state))
10601 			n = bnxt_get_registered_vfs(bp);
10602 		if (n < 0) {
10603 			netdev_err(bp->dev, "Firmware reset aborted, rc = %d\n",
10604 				   n);
10605 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
10606 			dev_close(bp->dev);
10607 			goto fw_reset_exit;
10608 		} else if (n > 0) {
10609 			u16 vf_tmo_dsecs = n * 10;
10610 
10611 			if (bp->fw_reset_max_dsecs < vf_tmo_dsecs)
10612 				bp->fw_reset_max_dsecs = vf_tmo_dsecs;
10613 			bp->fw_reset_state =
10614 				BNXT_FW_RESET_STATE_POLL_VF;
10615 			bnxt_queue_fw_reset_work(bp, HZ / 10);
10616 			goto fw_reset_exit;
10617 		}
10618 		bnxt_fw_reset_close(bp);
10619 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
10620 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
10621 			tmo = HZ / 10;
10622 		} else {
10623 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
10624 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
10625 		}
10626 		bnxt_queue_fw_reset_work(bp, tmo);
10627 	}
10628 fw_reset_exit:
10629 	bnxt_rtnl_unlock_sp(bp);
10630 }
10631 
10632 static void bnxt_chk_missed_irq(struct bnxt *bp)
10633 {
10634 	int i;
10635 
10636 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
10637 		return;
10638 
10639 	for (i = 0; i < bp->cp_nr_rings; i++) {
10640 		struct bnxt_napi *bnapi = bp->bnapi[i];
10641 		struct bnxt_cp_ring_info *cpr;
10642 		u32 fw_ring_id;
10643 		int j;
10644 
10645 		if (!bnapi)
10646 			continue;
10647 
10648 		cpr = &bnapi->cp_ring;
10649 		for (j = 0; j < 2; j++) {
10650 			struct bnxt_cp_ring_info *cpr2 = cpr->cp_ring_arr[j];
10651 			u32 val[2];
10652 
10653 			if (!cpr2 || cpr2->has_more_work ||
10654 			    !bnxt_has_work(bp, cpr2))
10655 				continue;
10656 
10657 			if (cpr2->cp_raw_cons != cpr2->last_cp_raw_cons) {
10658 				cpr2->last_cp_raw_cons = cpr2->cp_raw_cons;
10659 				continue;
10660 			}
10661 			fw_ring_id = cpr2->cp_ring_struct.fw_ring_id;
10662 			bnxt_dbg_hwrm_ring_info_get(bp,
10663 				DBG_RING_INFO_GET_REQ_RING_TYPE_L2_CMPL,
10664 				fw_ring_id, &val[0], &val[1]);
10665 			cpr->sw_stats.cmn.missed_irqs++;
10666 		}
10667 	}
10668 }
10669 
10670 static void bnxt_cfg_ntp_filters(struct bnxt *);
10671 
10672 static void bnxt_init_ethtool_link_settings(struct bnxt *bp)
10673 {
10674 	struct bnxt_link_info *link_info = &bp->link_info;
10675 
10676 	if (BNXT_AUTO_MODE(link_info->auto_mode)) {
10677 		link_info->autoneg = BNXT_AUTONEG_SPEED;
10678 		if (bp->hwrm_spec_code >= 0x10201) {
10679 			if (link_info->auto_pause_setting &
10680 			    PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE)
10681 				link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10682 		} else {
10683 			link_info->autoneg |= BNXT_AUTONEG_FLOW_CTRL;
10684 		}
10685 		link_info->advertising = link_info->auto_link_speeds;
10686 	} else {
10687 		link_info->req_link_speed = link_info->force_link_speed;
10688 		link_info->req_duplex = link_info->duplex_setting;
10689 	}
10690 	if (link_info->autoneg & BNXT_AUTONEG_FLOW_CTRL)
10691 		link_info->req_flow_ctrl =
10692 			link_info->auto_pause_setting & BNXT_LINK_PAUSE_BOTH;
10693 	else
10694 		link_info->req_flow_ctrl = link_info->force_pause_setting;
10695 }
10696 
10697 static void bnxt_sp_task(struct work_struct *work)
10698 {
10699 	struct bnxt *bp = container_of(work, struct bnxt, sp_task);
10700 
10701 	set_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10702 	smp_mb__after_atomic();
10703 	if (!test_bit(BNXT_STATE_OPEN, &bp->state)) {
10704 		clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10705 		return;
10706 	}
10707 
10708 	if (test_and_clear_bit(BNXT_RX_MASK_SP_EVENT, &bp->sp_event))
10709 		bnxt_cfg_rx_mode(bp);
10710 
10711 	if (test_and_clear_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event))
10712 		bnxt_cfg_ntp_filters(bp);
10713 	if (test_and_clear_bit(BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT, &bp->sp_event))
10714 		bnxt_hwrm_exec_fwd_req(bp);
10715 	if (test_and_clear_bit(BNXT_PERIODIC_STATS_SP_EVENT, &bp->sp_event)) {
10716 		bnxt_hwrm_port_qstats(bp, 0);
10717 		bnxt_hwrm_port_qstats_ext(bp, 0);
10718 		bnxt_accumulate_all_stats(bp);
10719 	}
10720 
10721 	if (test_and_clear_bit(BNXT_LINK_CHNG_SP_EVENT, &bp->sp_event)) {
10722 		int rc;
10723 
10724 		mutex_lock(&bp->link_lock);
10725 		if (test_and_clear_bit(BNXT_LINK_SPEED_CHNG_SP_EVENT,
10726 				       &bp->sp_event))
10727 			bnxt_hwrm_phy_qcaps(bp);
10728 
10729 		rc = bnxt_update_link(bp, true);
10730 		if (rc)
10731 			netdev_err(bp->dev, "SP task can't update link (rc: %x)\n",
10732 				   rc);
10733 
10734 		if (test_and_clear_bit(BNXT_LINK_CFG_CHANGE_SP_EVENT,
10735 				       &bp->sp_event))
10736 			bnxt_init_ethtool_link_settings(bp);
10737 		mutex_unlock(&bp->link_lock);
10738 	}
10739 	if (test_and_clear_bit(BNXT_UPDATE_PHY_SP_EVENT, &bp->sp_event)) {
10740 		int rc;
10741 
10742 		mutex_lock(&bp->link_lock);
10743 		rc = bnxt_update_phy_setting(bp);
10744 		mutex_unlock(&bp->link_lock);
10745 		if (rc) {
10746 			netdev_warn(bp->dev, "update phy settings retry failed\n");
10747 		} else {
10748 			bp->link_info.phy_retry = false;
10749 			netdev_info(bp->dev, "update phy settings retry succeeded\n");
10750 		}
10751 	}
10752 	if (test_and_clear_bit(BNXT_HWRM_PORT_MODULE_SP_EVENT, &bp->sp_event)) {
10753 		mutex_lock(&bp->link_lock);
10754 		bnxt_get_port_module_status(bp);
10755 		mutex_unlock(&bp->link_lock);
10756 	}
10757 
10758 	if (test_and_clear_bit(BNXT_FLOW_STATS_SP_EVENT, &bp->sp_event))
10759 		bnxt_tc_flow_stats_work(bp);
10760 
10761 	if (test_and_clear_bit(BNXT_RING_COAL_NOW_SP_EVENT, &bp->sp_event))
10762 		bnxt_chk_missed_irq(bp);
10763 
10764 	/* These functions below will clear BNXT_STATE_IN_SP_TASK.  They
10765 	 * must be the last functions to be called before exiting.
10766 	 */
10767 	if (test_and_clear_bit(BNXT_RESET_TASK_SP_EVENT, &bp->sp_event))
10768 		bnxt_reset(bp, false);
10769 
10770 	if (test_and_clear_bit(BNXT_RESET_TASK_SILENT_SP_EVENT, &bp->sp_event))
10771 		bnxt_reset(bp, true);
10772 
10773 	if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event))
10774 		bnxt_devlink_health_report(bp, BNXT_FW_RESET_NOTIFY_SP_EVENT);
10775 
10776 	if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) {
10777 		if (!is_bnxt_fw_ok(bp))
10778 			bnxt_devlink_health_report(bp,
10779 						   BNXT_FW_EXCEPTION_SP_EVENT);
10780 	}
10781 
10782 	smp_mb__before_atomic();
10783 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
10784 }
10785 
10786 /* Under rtnl_lock */
10787 int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs,
10788 		     int tx_xdp)
10789 {
10790 	int max_rx, max_tx, tx_sets = 1;
10791 	int tx_rings_needed, stats;
10792 	int rx_rings = rx;
10793 	int cp, vnics, rc;
10794 
10795 	if (tcs)
10796 		tx_sets = tcs;
10797 
10798 	rc = bnxt_get_max_rings(bp, &max_rx, &max_tx, sh);
10799 	if (rc)
10800 		return rc;
10801 
10802 	if (max_rx < rx)
10803 		return -ENOMEM;
10804 
10805 	tx_rings_needed = tx * tx_sets + tx_xdp;
10806 	if (max_tx < tx_rings_needed)
10807 		return -ENOMEM;
10808 
10809 	vnics = 1;
10810 	if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS)
10811 		vnics += rx_rings;
10812 
10813 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
10814 		rx_rings <<= 1;
10815 	cp = sh ? max_t(int, tx_rings_needed, rx) : tx_rings_needed + rx;
10816 	stats = cp;
10817 	if (BNXT_NEW_RM(bp)) {
10818 		cp += bnxt_get_ulp_msix_num(bp);
10819 		stats += bnxt_get_ulp_stat_ctxs(bp);
10820 	}
10821 	return bnxt_hwrm_check_rings(bp, tx_rings_needed, rx_rings, rx, cp,
10822 				     stats, vnics);
10823 }
10824 
10825 static void bnxt_unmap_bars(struct bnxt *bp, struct pci_dev *pdev)
10826 {
10827 	if (bp->bar2) {
10828 		pci_iounmap(pdev, bp->bar2);
10829 		bp->bar2 = NULL;
10830 	}
10831 
10832 	if (bp->bar1) {
10833 		pci_iounmap(pdev, bp->bar1);
10834 		bp->bar1 = NULL;
10835 	}
10836 
10837 	if (bp->bar0) {
10838 		pci_iounmap(pdev, bp->bar0);
10839 		bp->bar0 = NULL;
10840 	}
10841 }
10842 
10843 static void bnxt_cleanup_pci(struct bnxt *bp)
10844 {
10845 	bnxt_unmap_bars(bp, bp->pdev);
10846 	pci_release_regions(bp->pdev);
10847 	if (pci_is_enabled(bp->pdev))
10848 		pci_disable_device(bp->pdev);
10849 }
10850 
10851 static void bnxt_init_dflt_coal(struct bnxt *bp)
10852 {
10853 	struct bnxt_coal *coal;
10854 
10855 	/* Tick values in micro seconds.
10856 	 * 1 coal_buf x bufs_per_record = 1 completion record.
10857 	 */
10858 	coal = &bp->rx_coal;
10859 	coal->coal_ticks = 10;
10860 	coal->coal_bufs = 30;
10861 	coal->coal_ticks_irq = 1;
10862 	coal->coal_bufs_irq = 2;
10863 	coal->idle_thresh = 50;
10864 	coal->bufs_per_record = 2;
10865 	coal->budget = 64;		/* NAPI budget */
10866 
10867 	coal = &bp->tx_coal;
10868 	coal->coal_ticks = 28;
10869 	coal->coal_bufs = 30;
10870 	coal->coal_ticks_irq = 2;
10871 	coal->coal_bufs_irq = 2;
10872 	coal->bufs_per_record = 1;
10873 
10874 	bp->stats_coal_ticks = BNXT_DEF_STATS_COAL_TICKS;
10875 }
10876 
10877 static void bnxt_alloc_fw_health(struct bnxt *bp)
10878 {
10879 	if (bp->fw_health)
10880 		return;
10881 
10882 	if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) &&
10883 	    !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
10884 		return;
10885 
10886 	bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL);
10887 	if (!bp->fw_health) {
10888 		netdev_warn(bp->dev, "Failed to allocate fw_health\n");
10889 		bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET;
10890 		bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
10891 	}
10892 }
10893 
10894 static int bnxt_fw_init_one_p1(struct bnxt *bp)
10895 {
10896 	int rc;
10897 
10898 	bp->fw_cap = 0;
10899 	rc = bnxt_hwrm_ver_get(bp);
10900 	if (rc)
10901 		return rc;
10902 
10903 	if (bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL) {
10904 		rc = bnxt_alloc_kong_hwrm_resources(bp);
10905 		if (rc)
10906 			bp->fw_cap &= ~BNXT_FW_CAP_KONG_MB_CHNL;
10907 	}
10908 
10909 	if ((bp->fw_cap & BNXT_FW_CAP_SHORT_CMD) ||
10910 	    bp->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) {
10911 		rc = bnxt_alloc_hwrm_short_cmd_req(bp);
10912 		if (rc)
10913 			return rc;
10914 	}
10915 	rc = bnxt_hwrm_func_reset(bp);
10916 	if (rc)
10917 		return -ENODEV;
10918 
10919 	bnxt_hwrm_fw_set_time(bp);
10920 	return 0;
10921 }
10922 
10923 static int bnxt_fw_init_one_p2(struct bnxt *bp)
10924 {
10925 	int rc;
10926 
10927 	/* Get the MAX capabilities for this function */
10928 	rc = bnxt_hwrm_func_qcaps(bp);
10929 	if (rc) {
10930 		netdev_err(bp->dev, "hwrm query capability failure rc: %x\n",
10931 			   rc);
10932 		return -ENODEV;
10933 	}
10934 
10935 	rc = bnxt_hwrm_cfa_adv_flow_mgnt_qcaps(bp);
10936 	if (rc)
10937 		netdev_warn(bp->dev, "hwrm query adv flow mgnt failure rc: %d\n",
10938 			    rc);
10939 
10940 	bnxt_alloc_fw_health(bp);
10941 	rc = bnxt_hwrm_error_recovery_qcfg(bp);
10942 	if (rc)
10943 		netdev_warn(bp->dev, "hwrm query error recovery failure rc: %d\n",
10944 			    rc);
10945 
10946 	rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false);
10947 	if (rc)
10948 		return -ENODEV;
10949 
10950 	bnxt_hwrm_func_qcfg(bp);
10951 	bnxt_hwrm_vnic_qcaps(bp);
10952 	bnxt_hwrm_port_led_qcaps(bp);
10953 	bnxt_ethtool_init(bp);
10954 	bnxt_dcb_init(bp);
10955 	return 0;
10956 }
10957 
10958 static void bnxt_set_dflt_rss_hash_type(struct bnxt *bp)
10959 {
10960 	bp->flags &= ~BNXT_FLAG_UDP_RSS_CAP;
10961 	bp->rss_hash_cfg = VNIC_RSS_CFG_REQ_HASH_TYPE_IPV4 |
10962 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV4 |
10963 			   VNIC_RSS_CFG_REQ_HASH_TYPE_IPV6 |
10964 			   VNIC_RSS_CFG_REQ_HASH_TYPE_TCP_IPV6;
10965 	if (BNXT_CHIP_P4_PLUS(bp) && bp->hwrm_spec_code >= 0x10501) {
10966 		bp->flags |= BNXT_FLAG_UDP_RSS_CAP;
10967 		bp->rss_hash_cfg |= VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV4 |
10968 				    VNIC_RSS_CFG_REQ_HASH_TYPE_UDP_IPV6;
10969 	}
10970 }
10971 
10972 static void bnxt_set_dflt_rfs(struct bnxt *bp)
10973 {
10974 	struct net_device *dev = bp->dev;
10975 
10976 	dev->hw_features &= ~NETIF_F_NTUPLE;
10977 	dev->features &= ~NETIF_F_NTUPLE;
10978 	bp->flags &= ~BNXT_FLAG_RFS;
10979 	if (bnxt_rfs_supported(bp)) {
10980 		dev->hw_features |= NETIF_F_NTUPLE;
10981 		if (bnxt_rfs_capable(bp)) {
10982 			bp->flags |= BNXT_FLAG_RFS;
10983 			dev->features |= NETIF_F_NTUPLE;
10984 		}
10985 	}
10986 }
10987 
10988 static void bnxt_fw_init_one_p3(struct bnxt *bp)
10989 {
10990 	struct pci_dev *pdev = bp->pdev;
10991 
10992 	bnxt_set_dflt_rss_hash_type(bp);
10993 	bnxt_set_dflt_rfs(bp);
10994 
10995 	bnxt_get_wol_settings(bp);
10996 	if (bp->flags & BNXT_FLAG_WOL_CAP)
10997 		device_set_wakeup_enable(&pdev->dev, bp->wol);
10998 	else
10999 		device_set_wakeup_capable(&pdev->dev, false);
11000 
11001 	bnxt_hwrm_set_cache_line_size(bp, cache_line_size());
11002 	bnxt_hwrm_coal_params_qcaps(bp);
11003 }
11004 
11005 static int bnxt_fw_init_one(struct bnxt *bp)
11006 {
11007 	int rc;
11008 
11009 	rc = bnxt_fw_init_one_p1(bp);
11010 	if (rc) {
11011 		netdev_err(bp->dev, "Firmware init phase 1 failed\n");
11012 		return rc;
11013 	}
11014 	rc = bnxt_fw_init_one_p2(bp);
11015 	if (rc) {
11016 		netdev_err(bp->dev, "Firmware init phase 2 failed\n");
11017 		return rc;
11018 	}
11019 	rc = bnxt_approve_mac(bp, bp->dev->dev_addr, false);
11020 	if (rc)
11021 		return rc;
11022 
11023 	/* In case fw capabilities have changed, destroy the unneeded
11024 	 * reporters and create newly capable ones.
11025 	 */
11026 	bnxt_dl_fw_reporters_destroy(bp, false);
11027 	bnxt_dl_fw_reporters_create(bp);
11028 	bnxt_fw_init_one_p3(bp);
11029 	return 0;
11030 }
11031 
11032 static void bnxt_fw_reset_writel(struct bnxt *bp, int reg_idx)
11033 {
11034 	struct bnxt_fw_health *fw_health = bp->fw_health;
11035 	u32 reg = fw_health->fw_reset_seq_regs[reg_idx];
11036 	u32 val = fw_health->fw_reset_seq_vals[reg_idx];
11037 	u32 reg_type, reg_off, delay_msecs;
11038 
11039 	delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx];
11040 	reg_type = BNXT_FW_HEALTH_REG_TYPE(reg);
11041 	reg_off = BNXT_FW_HEALTH_REG_OFF(reg);
11042 	switch (reg_type) {
11043 	case BNXT_FW_HEALTH_REG_TYPE_CFG:
11044 		pci_write_config_dword(bp->pdev, reg_off, val);
11045 		break;
11046 	case BNXT_FW_HEALTH_REG_TYPE_GRC:
11047 		writel(reg_off & BNXT_GRC_BASE_MASK,
11048 		       bp->bar0 + BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
11049 		reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000;
11050 		fallthrough;
11051 	case BNXT_FW_HEALTH_REG_TYPE_BAR0:
11052 		writel(val, bp->bar0 + reg_off);
11053 		break;
11054 	case BNXT_FW_HEALTH_REG_TYPE_BAR1:
11055 		writel(val, bp->bar1 + reg_off);
11056 		break;
11057 	}
11058 	if (delay_msecs) {
11059 		pci_read_config_dword(bp->pdev, 0, &val);
11060 		msleep(delay_msecs);
11061 	}
11062 }
11063 
11064 static void bnxt_reset_all(struct bnxt *bp)
11065 {
11066 	struct bnxt_fw_health *fw_health = bp->fw_health;
11067 	int i, rc;
11068 
11069 	if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11070 #ifdef CONFIG_TEE_BNXT_FW
11071 		rc = tee_bnxt_fw_load();
11072 		if (rc)
11073 			netdev_err(bp->dev, "Unable to reset FW rc=%d\n", rc);
11074 		bp->fw_reset_timestamp = jiffies;
11075 #endif
11076 		return;
11077 	}
11078 
11079 	if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_HOST) {
11080 		for (i = 0; i < fw_health->fw_reset_seq_cnt; i++)
11081 			bnxt_fw_reset_writel(bp, i);
11082 	} else if (fw_health->flags & ERROR_RECOVERY_QCFG_RESP_FLAGS_CO_CPU) {
11083 		struct hwrm_fw_reset_input req = {0};
11084 
11085 		bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET, -1, -1);
11086 		req.resp_addr = cpu_to_le64(bp->hwrm_cmd_kong_resp_dma_addr);
11087 		req.embedded_proc_type = FW_RESET_REQ_EMBEDDED_PROC_TYPE_CHIP;
11088 		req.selfrst_status = FW_RESET_REQ_SELFRST_STATUS_SELFRSTASAP;
11089 		req.flags = FW_RESET_REQ_FLAGS_RESET_GRACEFUL;
11090 		rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
11091 		if (rc)
11092 			netdev_warn(bp->dev, "Unable to reset FW rc=%d\n", rc);
11093 	}
11094 	bp->fw_reset_timestamp = jiffies;
11095 }
11096 
11097 static void bnxt_fw_reset_task(struct work_struct *work)
11098 {
11099 	struct bnxt *bp = container_of(work, struct bnxt, fw_reset_task.work);
11100 	int rc;
11101 
11102 	if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) {
11103 		netdev_err(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n");
11104 		return;
11105 	}
11106 
11107 	switch (bp->fw_reset_state) {
11108 	case BNXT_FW_RESET_STATE_POLL_VF: {
11109 		int n = bnxt_get_registered_vfs(bp);
11110 		int tmo;
11111 
11112 		if (n < 0) {
11113 			netdev_err(bp->dev, "Firmware reset aborted, subsequent func_qcfg cmd failed, rc = %d, %d msecs since reset timestamp\n",
11114 				   n, jiffies_to_msecs(jiffies -
11115 				   bp->fw_reset_timestamp));
11116 			goto fw_reset_abort;
11117 		} else if (n > 0) {
11118 			if (time_after(jiffies, bp->fw_reset_timestamp +
11119 				       (bp->fw_reset_max_dsecs * HZ / 10))) {
11120 				clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11121 				bp->fw_reset_state = 0;
11122 				netdev_err(bp->dev, "Firmware reset aborted, bnxt_get_registered_vfs() returns %d\n",
11123 					   n);
11124 				return;
11125 			}
11126 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11127 			return;
11128 		}
11129 		bp->fw_reset_timestamp = jiffies;
11130 		rtnl_lock();
11131 		bnxt_fw_reset_close(bp);
11132 		if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) {
11133 			bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN;
11134 			tmo = HZ / 10;
11135 		} else {
11136 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11137 			tmo = bp->fw_reset_min_dsecs * HZ / 10;
11138 		}
11139 		rtnl_unlock();
11140 		bnxt_queue_fw_reset_work(bp, tmo);
11141 		return;
11142 	}
11143 	case BNXT_FW_RESET_STATE_POLL_FW_DOWN: {
11144 		u32 val;
11145 
11146 		val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG);
11147 		if (!(val & BNXT_FW_STATUS_SHUTDOWN) &&
11148 		    !time_after(jiffies, bp->fw_reset_timestamp +
11149 		    (bp->fw_reset_max_dsecs * HZ / 10))) {
11150 			bnxt_queue_fw_reset_work(bp, HZ / 5);
11151 			return;
11152 		}
11153 
11154 		if (!bp->fw_health->master) {
11155 			u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs;
11156 
11157 			bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11158 			bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10);
11159 			return;
11160 		}
11161 		bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW;
11162 	}
11163 		fallthrough;
11164 	case BNXT_FW_RESET_STATE_RESET_FW:
11165 		bnxt_reset_all(bp);
11166 		bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV;
11167 		bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10);
11168 		return;
11169 	case BNXT_FW_RESET_STATE_ENABLE_DEV:
11170 		if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) {
11171 			u32 val;
11172 
11173 			val = bnxt_fw_health_readl(bp,
11174 						   BNXT_FW_RESET_INPROG_REG);
11175 			if (val)
11176 				netdev_warn(bp->dev, "FW reset inprog %x after min wait time.\n",
11177 					    val);
11178 		}
11179 		clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state);
11180 		if (pci_enable_device(bp->pdev)) {
11181 			netdev_err(bp->dev, "Cannot re-enable PCI device\n");
11182 			goto fw_reset_abort;
11183 		}
11184 		pci_set_master(bp->pdev);
11185 		bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW;
11186 		fallthrough;
11187 	case BNXT_FW_RESET_STATE_POLL_FW:
11188 		bp->hwrm_cmd_timeout = SHORT_HWRM_CMD_TIMEOUT;
11189 		rc = __bnxt_hwrm_ver_get(bp, true);
11190 		if (rc) {
11191 			if (time_after(jiffies, bp->fw_reset_timestamp +
11192 				       (bp->fw_reset_max_dsecs * HZ / 10))) {
11193 				netdev_err(bp->dev, "Firmware reset aborted\n");
11194 				goto fw_reset_abort;
11195 			}
11196 			bnxt_queue_fw_reset_work(bp, HZ / 5);
11197 			return;
11198 		}
11199 		bp->hwrm_cmd_timeout = DFLT_HWRM_CMD_TIMEOUT;
11200 		bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING;
11201 		fallthrough;
11202 	case BNXT_FW_RESET_STATE_OPENING:
11203 		while (!rtnl_trylock()) {
11204 			bnxt_queue_fw_reset_work(bp, HZ / 10);
11205 			return;
11206 		}
11207 		rc = bnxt_open(bp->dev);
11208 		if (rc) {
11209 			netdev_err(bp->dev, "bnxt_open_nic() failed\n");
11210 			clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11211 			dev_close(bp->dev);
11212 		}
11213 
11214 		bp->fw_reset_state = 0;
11215 		/* Make sure fw_reset_state is 0 before clearing the flag */
11216 		smp_mb__before_atomic();
11217 		clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11218 		bnxt_ulp_start(bp, rc);
11219 		if (!rc)
11220 			bnxt_reenable_sriov(bp);
11221 		bnxt_dl_health_recovery_done(bp);
11222 		bnxt_dl_health_status_update(bp, true);
11223 		rtnl_unlock();
11224 		break;
11225 	}
11226 	return;
11227 
11228 fw_reset_abort:
11229 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11230 	if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF)
11231 		bnxt_dl_health_status_update(bp, false);
11232 	bp->fw_reset_state = 0;
11233 	rtnl_lock();
11234 	dev_close(bp->dev);
11235 	rtnl_unlock();
11236 }
11237 
11238 static int bnxt_init_board(struct pci_dev *pdev, struct net_device *dev)
11239 {
11240 	int rc;
11241 	struct bnxt *bp = netdev_priv(dev);
11242 
11243 	SET_NETDEV_DEV(dev, &pdev->dev);
11244 
11245 	/* enable device (incl. PCI PM wakeup), and bus-mastering */
11246 	rc = pci_enable_device(pdev);
11247 	if (rc) {
11248 		dev_err(&pdev->dev, "Cannot enable PCI device, aborting\n");
11249 		goto init_err;
11250 	}
11251 
11252 	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
11253 		dev_err(&pdev->dev,
11254 			"Cannot find PCI device base address, aborting\n");
11255 		rc = -ENODEV;
11256 		goto init_err_disable;
11257 	}
11258 
11259 	rc = pci_request_regions(pdev, DRV_MODULE_NAME);
11260 	if (rc) {
11261 		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting\n");
11262 		goto init_err_disable;
11263 	}
11264 
11265 	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) != 0 &&
11266 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
11267 		dev_err(&pdev->dev, "System does not support DMA, aborting\n");
11268 		goto init_err_disable;
11269 	}
11270 
11271 	pci_set_master(pdev);
11272 
11273 	bp->dev = dev;
11274 	bp->pdev = pdev;
11275 
11276 	/* Doorbell BAR bp->bar1 is mapped after bnxt_fw_init_one_p2()
11277 	 * determines the BAR size.
11278 	 */
11279 	bp->bar0 = pci_ioremap_bar(pdev, 0);
11280 	if (!bp->bar0) {
11281 		dev_err(&pdev->dev, "Cannot map device registers, aborting\n");
11282 		rc = -ENOMEM;
11283 		goto init_err_release;
11284 	}
11285 
11286 	bp->bar2 = pci_ioremap_bar(pdev, 4);
11287 	if (!bp->bar2) {
11288 		dev_err(&pdev->dev, "Cannot map bar4 registers, aborting\n");
11289 		rc = -ENOMEM;
11290 		goto init_err_release;
11291 	}
11292 
11293 	pci_enable_pcie_error_reporting(pdev);
11294 
11295 	INIT_WORK(&bp->sp_task, bnxt_sp_task);
11296 	INIT_DELAYED_WORK(&bp->fw_reset_task, bnxt_fw_reset_task);
11297 
11298 	spin_lock_init(&bp->ntp_fltr_lock);
11299 #if BITS_PER_LONG == 32
11300 	spin_lock_init(&bp->db_lock);
11301 #endif
11302 
11303 	bp->rx_ring_size = BNXT_DEFAULT_RX_RING_SIZE;
11304 	bp->tx_ring_size = BNXT_DEFAULT_TX_RING_SIZE;
11305 
11306 	bnxt_init_dflt_coal(bp);
11307 
11308 	timer_setup(&bp->timer, bnxt_timer, 0);
11309 	bp->current_interval = BNXT_TIMER_INTERVAL;
11310 
11311 	bp->vxlan_fw_dst_port_id = INVALID_HW_RING_ID;
11312 	bp->nge_fw_dst_port_id = INVALID_HW_RING_ID;
11313 
11314 	clear_bit(BNXT_STATE_OPEN, &bp->state);
11315 	return 0;
11316 
11317 init_err_release:
11318 	bnxt_unmap_bars(bp, pdev);
11319 	pci_release_regions(pdev);
11320 
11321 init_err_disable:
11322 	pci_disable_device(pdev);
11323 
11324 init_err:
11325 	return rc;
11326 }
11327 
11328 /* rtnl_lock held */
11329 static int bnxt_change_mac_addr(struct net_device *dev, void *p)
11330 {
11331 	struct sockaddr *addr = p;
11332 	struct bnxt *bp = netdev_priv(dev);
11333 	int rc = 0;
11334 
11335 	if (!is_valid_ether_addr(addr->sa_data))
11336 		return -EADDRNOTAVAIL;
11337 
11338 	if (ether_addr_equal(addr->sa_data, dev->dev_addr))
11339 		return 0;
11340 
11341 	rc = bnxt_approve_mac(bp, addr->sa_data, true);
11342 	if (rc)
11343 		return rc;
11344 
11345 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
11346 	if (netif_running(dev)) {
11347 		bnxt_close_nic(bp, false, false);
11348 		rc = bnxt_open_nic(bp, false, false);
11349 	}
11350 
11351 	return rc;
11352 }
11353 
11354 /* rtnl_lock held */
11355 static int bnxt_change_mtu(struct net_device *dev, int new_mtu)
11356 {
11357 	struct bnxt *bp = netdev_priv(dev);
11358 
11359 	if (netif_running(dev))
11360 		bnxt_close_nic(bp, true, false);
11361 
11362 	dev->mtu = new_mtu;
11363 	bnxt_set_ring_params(bp);
11364 
11365 	if (netif_running(dev))
11366 		return bnxt_open_nic(bp, true, false);
11367 
11368 	return 0;
11369 }
11370 
11371 int bnxt_setup_mq_tc(struct net_device *dev, u8 tc)
11372 {
11373 	struct bnxt *bp = netdev_priv(dev);
11374 	bool sh = false;
11375 	int rc;
11376 
11377 	if (tc > bp->max_tc) {
11378 		netdev_err(dev, "Too many traffic classes requested: %d. Max supported is %d.\n",
11379 			   tc, bp->max_tc);
11380 		return -EINVAL;
11381 	}
11382 
11383 	if (netdev_get_num_tc(dev) == tc)
11384 		return 0;
11385 
11386 	if (bp->flags & BNXT_FLAG_SHARED_RINGS)
11387 		sh = true;
11388 
11389 	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
11390 			      sh, tc, bp->tx_nr_rings_xdp);
11391 	if (rc)
11392 		return rc;
11393 
11394 	/* Needs to close the device and do hw resource re-allocations */
11395 	if (netif_running(bp->dev))
11396 		bnxt_close_nic(bp, true, false);
11397 
11398 	if (tc) {
11399 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc;
11400 		netdev_set_num_tc(dev, tc);
11401 	} else {
11402 		bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11403 		netdev_reset_tc(dev);
11404 	}
11405 	bp->tx_nr_rings += bp->tx_nr_rings_xdp;
11406 	bp->cp_nr_rings = sh ? max_t(int, bp->tx_nr_rings, bp->rx_nr_rings) :
11407 			       bp->tx_nr_rings + bp->rx_nr_rings;
11408 
11409 	if (netif_running(bp->dev))
11410 		return bnxt_open_nic(bp, true, false);
11411 
11412 	return 0;
11413 }
11414 
11415 static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
11416 				  void *cb_priv)
11417 {
11418 	struct bnxt *bp = cb_priv;
11419 
11420 	if (!bnxt_tc_flower_enabled(bp) ||
11421 	    !tc_cls_can_offload_and_chain0(bp->dev, type_data))
11422 		return -EOPNOTSUPP;
11423 
11424 	switch (type) {
11425 	case TC_SETUP_CLSFLOWER:
11426 		return bnxt_tc_setup_flower(bp, bp->pf.fw_fid, type_data);
11427 	default:
11428 		return -EOPNOTSUPP;
11429 	}
11430 }
11431 
11432 LIST_HEAD(bnxt_block_cb_list);
11433 
11434 static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type,
11435 			 void *type_data)
11436 {
11437 	struct bnxt *bp = netdev_priv(dev);
11438 
11439 	switch (type) {
11440 	case TC_SETUP_BLOCK:
11441 		return flow_block_cb_setup_simple(type_data,
11442 						  &bnxt_block_cb_list,
11443 						  bnxt_setup_tc_block_cb,
11444 						  bp, bp, true);
11445 	case TC_SETUP_QDISC_MQPRIO: {
11446 		struct tc_mqprio_qopt *mqprio = type_data;
11447 
11448 		mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
11449 
11450 		return bnxt_setup_mq_tc(dev, mqprio->num_tc);
11451 	}
11452 	default:
11453 		return -EOPNOTSUPP;
11454 	}
11455 }
11456 
11457 #ifdef CONFIG_RFS_ACCEL
11458 static bool bnxt_fltr_match(struct bnxt_ntuple_filter *f1,
11459 			    struct bnxt_ntuple_filter *f2)
11460 {
11461 	struct flow_keys *keys1 = &f1->fkeys;
11462 	struct flow_keys *keys2 = &f2->fkeys;
11463 
11464 	if (keys1->basic.n_proto != keys2->basic.n_proto ||
11465 	    keys1->basic.ip_proto != keys2->basic.ip_proto)
11466 		return false;
11467 
11468 	if (keys1->basic.n_proto == htons(ETH_P_IP)) {
11469 		if (keys1->addrs.v4addrs.src != keys2->addrs.v4addrs.src ||
11470 		    keys1->addrs.v4addrs.dst != keys2->addrs.v4addrs.dst)
11471 			return false;
11472 	} else {
11473 		if (memcmp(&keys1->addrs.v6addrs.src, &keys2->addrs.v6addrs.src,
11474 			   sizeof(keys1->addrs.v6addrs.src)) ||
11475 		    memcmp(&keys1->addrs.v6addrs.dst, &keys2->addrs.v6addrs.dst,
11476 			   sizeof(keys1->addrs.v6addrs.dst)))
11477 			return false;
11478 	}
11479 
11480 	if (keys1->ports.ports == keys2->ports.ports &&
11481 	    keys1->control.flags == keys2->control.flags &&
11482 	    ether_addr_equal(f1->src_mac_addr, f2->src_mac_addr) &&
11483 	    ether_addr_equal(f1->dst_mac_addr, f2->dst_mac_addr))
11484 		return true;
11485 
11486 	return false;
11487 }
11488 
11489 static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
11490 			      u16 rxq_index, u32 flow_id)
11491 {
11492 	struct bnxt *bp = netdev_priv(dev);
11493 	struct bnxt_ntuple_filter *fltr, *new_fltr;
11494 	struct flow_keys *fkeys;
11495 	struct ethhdr *eth = (struct ethhdr *)skb_mac_header(skb);
11496 	int rc = 0, idx, bit_id, l2_idx = 0;
11497 	struct hlist_head *head;
11498 	u32 flags;
11499 
11500 	if (!ether_addr_equal(dev->dev_addr, eth->h_dest)) {
11501 		struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
11502 		int off = 0, j;
11503 
11504 		netif_addr_lock_bh(dev);
11505 		for (j = 0; j < vnic->uc_filter_count; j++, off += ETH_ALEN) {
11506 			if (ether_addr_equal(eth->h_dest,
11507 					     vnic->uc_list + off)) {
11508 				l2_idx = j + 1;
11509 				break;
11510 			}
11511 		}
11512 		netif_addr_unlock_bh(dev);
11513 		if (!l2_idx)
11514 			return -EINVAL;
11515 	}
11516 	new_fltr = kzalloc(sizeof(*new_fltr), GFP_ATOMIC);
11517 	if (!new_fltr)
11518 		return -ENOMEM;
11519 
11520 	fkeys = &new_fltr->fkeys;
11521 	if (!skb_flow_dissect_flow_keys(skb, fkeys, 0)) {
11522 		rc = -EPROTONOSUPPORT;
11523 		goto err_free;
11524 	}
11525 
11526 	if ((fkeys->basic.n_proto != htons(ETH_P_IP) &&
11527 	     fkeys->basic.n_proto != htons(ETH_P_IPV6)) ||
11528 	    ((fkeys->basic.ip_proto != IPPROTO_TCP) &&
11529 	     (fkeys->basic.ip_proto != IPPROTO_UDP))) {
11530 		rc = -EPROTONOSUPPORT;
11531 		goto err_free;
11532 	}
11533 	if (fkeys->basic.n_proto == htons(ETH_P_IPV6) &&
11534 	    bp->hwrm_spec_code < 0x10601) {
11535 		rc = -EPROTONOSUPPORT;
11536 		goto err_free;
11537 	}
11538 	flags = fkeys->control.flags;
11539 	if (((flags & FLOW_DIS_ENCAPSULATION) &&
11540 	     bp->hwrm_spec_code < 0x10601) || (flags & FLOW_DIS_IS_FRAGMENT)) {
11541 		rc = -EPROTONOSUPPORT;
11542 		goto err_free;
11543 	}
11544 
11545 	memcpy(new_fltr->dst_mac_addr, eth->h_dest, ETH_ALEN);
11546 	memcpy(new_fltr->src_mac_addr, eth->h_source, ETH_ALEN);
11547 
11548 	idx = skb_get_hash_raw(skb) & BNXT_NTP_FLTR_HASH_MASK;
11549 	head = &bp->ntp_fltr_hash_tbl[idx];
11550 	rcu_read_lock();
11551 	hlist_for_each_entry_rcu(fltr, head, hash) {
11552 		if (bnxt_fltr_match(fltr, new_fltr)) {
11553 			rcu_read_unlock();
11554 			rc = 0;
11555 			goto err_free;
11556 		}
11557 	}
11558 	rcu_read_unlock();
11559 
11560 	spin_lock_bh(&bp->ntp_fltr_lock);
11561 	bit_id = bitmap_find_free_region(bp->ntp_fltr_bmap,
11562 					 BNXT_NTP_FLTR_MAX_FLTR, 0);
11563 	if (bit_id < 0) {
11564 		spin_unlock_bh(&bp->ntp_fltr_lock);
11565 		rc = -ENOMEM;
11566 		goto err_free;
11567 	}
11568 
11569 	new_fltr->sw_id = (u16)bit_id;
11570 	new_fltr->flow_id = flow_id;
11571 	new_fltr->l2_fltr_idx = l2_idx;
11572 	new_fltr->rxq = rxq_index;
11573 	hlist_add_head_rcu(&new_fltr->hash, head);
11574 	bp->ntp_fltr_count++;
11575 	spin_unlock_bh(&bp->ntp_fltr_lock);
11576 
11577 	set_bit(BNXT_RX_NTP_FLTR_SP_EVENT, &bp->sp_event);
11578 	bnxt_queue_sp_work(bp);
11579 
11580 	return new_fltr->sw_id;
11581 
11582 err_free:
11583 	kfree(new_fltr);
11584 	return rc;
11585 }
11586 
11587 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11588 {
11589 	int i;
11590 
11591 	for (i = 0; i < BNXT_NTP_FLTR_HASH_SIZE; i++) {
11592 		struct hlist_head *head;
11593 		struct hlist_node *tmp;
11594 		struct bnxt_ntuple_filter *fltr;
11595 		int rc;
11596 
11597 		head = &bp->ntp_fltr_hash_tbl[i];
11598 		hlist_for_each_entry_safe(fltr, tmp, head, hash) {
11599 			bool del = false;
11600 
11601 			if (test_bit(BNXT_FLTR_VALID, &fltr->state)) {
11602 				if (rps_may_expire_flow(bp->dev, fltr->rxq,
11603 							fltr->flow_id,
11604 							fltr->sw_id)) {
11605 					bnxt_hwrm_cfa_ntuple_filter_free(bp,
11606 									 fltr);
11607 					del = true;
11608 				}
11609 			} else {
11610 				rc = bnxt_hwrm_cfa_ntuple_filter_alloc(bp,
11611 								       fltr);
11612 				if (rc)
11613 					del = true;
11614 				else
11615 					set_bit(BNXT_FLTR_VALID, &fltr->state);
11616 			}
11617 
11618 			if (del) {
11619 				spin_lock_bh(&bp->ntp_fltr_lock);
11620 				hlist_del_rcu(&fltr->hash);
11621 				bp->ntp_fltr_count--;
11622 				spin_unlock_bh(&bp->ntp_fltr_lock);
11623 				synchronize_rcu();
11624 				clear_bit(fltr->sw_id, bp->ntp_fltr_bmap);
11625 				kfree(fltr);
11626 			}
11627 		}
11628 	}
11629 	if (test_and_clear_bit(BNXT_HWRM_PF_UNLOAD_SP_EVENT, &bp->sp_event))
11630 		netdev_info(bp->dev, "Receive PF driver unload event!\n");
11631 }
11632 
11633 #else
11634 
11635 static void bnxt_cfg_ntp_filters(struct bnxt *bp)
11636 {
11637 }
11638 
11639 #endif /* CONFIG_RFS_ACCEL */
11640 
11641 static int bnxt_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
11642 {
11643 	struct bnxt *bp = netdev_priv(netdev);
11644 	struct udp_tunnel_info ti;
11645 	unsigned int cmd;
11646 
11647 	udp_tunnel_nic_get_port(netdev, table, 0, &ti);
11648 	if (ti.type == UDP_TUNNEL_TYPE_VXLAN)
11649 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_VXLAN;
11650 	else
11651 		cmd = TUNNEL_DST_PORT_FREE_REQ_TUNNEL_TYPE_GENEVE;
11652 
11653 	if (ti.port)
11654 		return bnxt_hwrm_tunnel_dst_port_alloc(bp, ti.port, cmd);
11655 
11656 	return bnxt_hwrm_tunnel_dst_port_free(bp, cmd);
11657 }
11658 
11659 static const struct udp_tunnel_nic_info bnxt_udp_tunnels = {
11660 	.sync_table	= bnxt_udp_tunnel_sync,
11661 	.flags		= UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
11662 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
11663 	.tables		= {
11664 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN,  },
11665 		{ .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },
11666 	},
11667 };
11668 
11669 static int bnxt_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
11670 			       struct net_device *dev, u32 filter_mask,
11671 			       int nlflags)
11672 {
11673 	struct bnxt *bp = netdev_priv(dev);
11674 
11675 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bp->br_mode, 0, 0,
11676 				       nlflags, filter_mask, NULL);
11677 }
11678 
11679 static int bnxt_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
11680 			       u16 flags, struct netlink_ext_ack *extack)
11681 {
11682 	struct bnxt *bp = netdev_priv(dev);
11683 	struct nlattr *attr, *br_spec;
11684 	int rem, rc = 0;
11685 
11686 	if (bp->hwrm_spec_code < 0x10708 || !BNXT_SINGLE_PF(bp))
11687 		return -EOPNOTSUPP;
11688 
11689 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
11690 	if (!br_spec)
11691 		return -EINVAL;
11692 
11693 	nla_for_each_nested(attr, br_spec, rem) {
11694 		u16 mode;
11695 
11696 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
11697 			continue;
11698 
11699 		if (nla_len(attr) < sizeof(mode))
11700 			return -EINVAL;
11701 
11702 		mode = nla_get_u16(attr);
11703 		if (mode == bp->br_mode)
11704 			break;
11705 
11706 		rc = bnxt_hwrm_set_br_mode(bp, mode);
11707 		if (!rc)
11708 			bp->br_mode = mode;
11709 		break;
11710 	}
11711 	return rc;
11712 }
11713 
11714 int bnxt_get_port_parent_id(struct net_device *dev,
11715 			    struct netdev_phys_item_id *ppid)
11716 {
11717 	struct bnxt *bp = netdev_priv(dev);
11718 
11719 	if (bp->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
11720 		return -EOPNOTSUPP;
11721 
11722 	/* The PF and it's VF-reps only support the switchdev framework */
11723 	if (!BNXT_PF(bp) || !(bp->flags & BNXT_FLAG_DSN_VALID))
11724 		return -EOPNOTSUPP;
11725 
11726 	ppid->id_len = sizeof(bp->dsn);
11727 	memcpy(ppid->id, bp->dsn, ppid->id_len);
11728 
11729 	return 0;
11730 }
11731 
11732 static struct devlink_port *bnxt_get_devlink_port(struct net_device *dev)
11733 {
11734 	struct bnxt *bp = netdev_priv(dev);
11735 
11736 	return &bp->dl_port;
11737 }
11738 
11739 static const struct net_device_ops bnxt_netdev_ops = {
11740 	.ndo_open		= bnxt_open,
11741 	.ndo_start_xmit		= bnxt_start_xmit,
11742 	.ndo_stop		= bnxt_close,
11743 	.ndo_get_stats64	= bnxt_get_stats64,
11744 	.ndo_set_rx_mode	= bnxt_set_rx_mode,
11745 	.ndo_do_ioctl		= bnxt_ioctl,
11746 	.ndo_validate_addr	= eth_validate_addr,
11747 	.ndo_set_mac_address	= bnxt_change_mac_addr,
11748 	.ndo_change_mtu		= bnxt_change_mtu,
11749 	.ndo_fix_features	= bnxt_fix_features,
11750 	.ndo_set_features	= bnxt_set_features,
11751 	.ndo_tx_timeout		= bnxt_tx_timeout,
11752 #ifdef CONFIG_BNXT_SRIOV
11753 	.ndo_get_vf_config	= bnxt_get_vf_config,
11754 	.ndo_set_vf_mac		= bnxt_set_vf_mac,
11755 	.ndo_set_vf_vlan	= bnxt_set_vf_vlan,
11756 	.ndo_set_vf_rate	= bnxt_set_vf_bw,
11757 	.ndo_set_vf_link_state	= bnxt_set_vf_link_state,
11758 	.ndo_set_vf_spoofchk	= bnxt_set_vf_spoofchk,
11759 	.ndo_set_vf_trust	= bnxt_set_vf_trust,
11760 #endif
11761 	.ndo_setup_tc           = bnxt_setup_tc,
11762 #ifdef CONFIG_RFS_ACCEL
11763 	.ndo_rx_flow_steer	= bnxt_rx_flow_steer,
11764 #endif
11765 	.ndo_udp_tunnel_add	= udp_tunnel_nic_add_port,
11766 	.ndo_udp_tunnel_del	= udp_tunnel_nic_del_port,
11767 	.ndo_bpf		= bnxt_xdp,
11768 	.ndo_xdp_xmit		= bnxt_xdp_xmit,
11769 	.ndo_bridge_getlink	= bnxt_bridge_getlink,
11770 	.ndo_bridge_setlink	= bnxt_bridge_setlink,
11771 	.ndo_get_devlink_port	= bnxt_get_devlink_port,
11772 };
11773 
11774 static void bnxt_remove_one(struct pci_dev *pdev)
11775 {
11776 	struct net_device *dev = pci_get_drvdata(pdev);
11777 	struct bnxt *bp = netdev_priv(dev);
11778 
11779 	if (BNXT_PF(bp))
11780 		bnxt_sriov_disable(bp);
11781 
11782 	bnxt_dl_fw_reporters_destroy(bp, true);
11783 	if (BNXT_PF(bp))
11784 		devlink_port_type_clear(&bp->dl_port);
11785 	pci_disable_pcie_error_reporting(pdev);
11786 	unregister_netdev(dev);
11787 	bnxt_dl_unregister(bp);
11788 	bnxt_shutdown_tc(bp);
11789 	clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state);
11790 	bnxt_cancel_sp_work(bp);
11791 	bp->sp_event = 0;
11792 
11793 	bnxt_clear_int_mode(bp);
11794 	bnxt_hwrm_func_drv_unrgtr(bp);
11795 	bnxt_free_hwrm_resources(bp);
11796 	bnxt_free_hwrm_short_cmd_req(bp);
11797 	bnxt_ethtool_free(bp);
11798 	bnxt_dcb_free(bp);
11799 	kfree(bp->edev);
11800 	bp->edev = NULL;
11801 	kfree(bp->fw_health);
11802 	bp->fw_health = NULL;
11803 	bnxt_cleanup_pci(bp);
11804 	bnxt_free_ctx_mem(bp);
11805 	kfree(bp->ctx);
11806 	bp->ctx = NULL;
11807 	kfree(bp->rss_indir_tbl);
11808 	bp->rss_indir_tbl = NULL;
11809 	bnxt_free_port_stats(bp);
11810 	free_netdev(dev);
11811 }
11812 
11813 static int bnxt_probe_phy(struct bnxt *bp, bool fw_dflt)
11814 {
11815 	int rc = 0;
11816 	struct bnxt_link_info *link_info = &bp->link_info;
11817 
11818 	rc = bnxt_hwrm_phy_qcaps(bp);
11819 	if (rc) {
11820 		netdev_err(bp->dev, "Probe phy can't get phy capabilities (rc: %x)\n",
11821 			   rc);
11822 		return rc;
11823 	}
11824 	if (!fw_dflt)
11825 		return 0;
11826 
11827 	rc = bnxt_update_link(bp, false);
11828 	if (rc) {
11829 		netdev_err(bp->dev, "Probe phy can't update link (rc: %x)\n",
11830 			   rc);
11831 		return rc;
11832 	}
11833 
11834 	/* Older firmware does not have supported_auto_speeds, so assume
11835 	 * that all supported speeds can be autonegotiated.
11836 	 */
11837 	if (link_info->auto_link_speeds && !link_info->support_auto_speeds)
11838 		link_info->support_auto_speeds = link_info->support_speeds;
11839 
11840 	bnxt_init_ethtool_link_settings(bp);
11841 	return 0;
11842 }
11843 
11844 static int bnxt_get_max_irq(struct pci_dev *pdev)
11845 {
11846 	u16 ctrl;
11847 
11848 	if (!pdev->msix_cap)
11849 		return 1;
11850 
11851 	pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &ctrl);
11852 	return (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
11853 }
11854 
11855 static void _bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11856 				int *max_cp)
11857 {
11858 	struct bnxt_hw_resc *hw_resc = &bp->hw_resc;
11859 	int max_ring_grps = 0, max_irq;
11860 
11861 	*max_tx = hw_resc->max_tx_rings;
11862 	*max_rx = hw_resc->max_rx_rings;
11863 	*max_cp = bnxt_get_max_func_cp_rings_for_en(bp);
11864 	max_irq = min_t(int, bnxt_get_max_func_irqs(bp) -
11865 			bnxt_get_ulp_msix_num(bp),
11866 			hw_resc->max_stat_ctxs - bnxt_get_ulp_stat_ctxs(bp));
11867 	if (!(bp->flags & BNXT_FLAG_CHIP_P5))
11868 		*max_cp = min_t(int, *max_cp, max_irq);
11869 	max_ring_grps = hw_resc->max_hw_ring_grps;
11870 	if (BNXT_CHIP_TYPE_NITRO_A0(bp) && BNXT_PF(bp)) {
11871 		*max_cp -= 1;
11872 		*max_rx -= 2;
11873 	}
11874 	if (bp->flags & BNXT_FLAG_AGG_RINGS)
11875 		*max_rx >>= 1;
11876 	if (bp->flags & BNXT_FLAG_CHIP_P5) {
11877 		bnxt_trim_rings(bp, max_rx, max_tx, *max_cp, false);
11878 		/* On P5 chips, max_cp output param should be available NQs */
11879 		*max_cp = max_irq;
11880 	}
11881 	*max_rx = min_t(int, *max_rx, max_ring_grps);
11882 }
11883 
11884 int bnxt_get_max_rings(struct bnxt *bp, int *max_rx, int *max_tx, bool shared)
11885 {
11886 	int rx, tx, cp;
11887 
11888 	_bnxt_get_max_rings(bp, &rx, &tx, &cp);
11889 	*max_rx = rx;
11890 	*max_tx = tx;
11891 	if (!rx || !tx || !cp)
11892 		return -ENOMEM;
11893 
11894 	return bnxt_trim_rings(bp, max_rx, max_tx, cp, shared);
11895 }
11896 
11897 static int bnxt_get_dflt_rings(struct bnxt *bp, int *max_rx, int *max_tx,
11898 			       bool shared)
11899 {
11900 	int rc;
11901 
11902 	rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11903 	if (rc && (bp->flags & BNXT_FLAG_AGG_RINGS)) {
11904 		/* Not enough rings, try disabling agg rings. */
11905 		bp->flags &= ~BNXT_FLAG_AGG_RINGS;
11906 		rc = bnxt_get_max_rings(bp, max_rx, max_tx, shared);
11907 		if (rc) {
11908 			/* set BNXT_FLAG_AGG_RINGS back for consistency */
11909 			bp->flags |= BNXT_FLAG_AGG_RINGS;
11910 			return rc;
11911 		}
11912 		bp->flags |= BNXT_FLAG_NO_AGG_RINGS;
11913 		bp->dev->hw_features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11914 		bp->dev->features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW);
11915 		bnxt_set_ring_params(bp);
11916 	}
11917 
11918 	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
11919 		int max_cp, max_stat, max_irq;
11920 
11921 		/* Reserve minimum resources for RoCE */
11922 		max_cp = bnxt_get_max_func_cp_rings(bp);
11923 		max_stat = bnxt_get_max_func_stat_ctxs(bp);
11924 		max_irq = bnxt_get_max_func_irqs(bp);
11925 		if (max_cp <= BNXT_MIN_ROCE_CP_RINGS ||
11926 		    max_irq <= BNXT_MIN_ROCE_CP_RINGS ||
11927 		    max_stat <= BNXT_MIN_ROCE_STAT_CTXS)
11928 			return 0;
11929 
11930 		max_cp -= BNXT_MIN_ROCE_CP_RINGS;
11931 		max_irq -= BNXT_MIN_ROCE_CP_RINGS;
11932 		max_stat -= BNXT_MIN_ROCE_STAT_CTXS;
11933 		max_cp = min_t(int, max_cp, max_irq);
11934 		max_cp = min_t(int, max_cp, max_stat);
11935 		rc = bnxt_trim_rings(bp, max_rx, max_tx, max_cp, shared);
11936 		if (rc)
11937 			rc = 0;
11938 	}
11939 	return rc;
11940 }
11941 
11942 /* In initial default shared ring setting, each shared ring must have a
11943  * RX/TX ring pair.
11944  */
11945 static void bnxt_trim_dflt_sh_rings(struct bnxt *bp)
11946 {
11947 	bp->cp_nr_rings = min_t(int, bp->tx_nr_rings_per_tc, bp->rx_nr_rings);
11948 	bp->rx_nr_rings = bp->cp_nr_rings;
11949 	bp->tx_nr_rings_per_tc = bp->cp_nr_rings;
11950 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11951 }
11952 
11953 static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh)
11954 {
11955 	int dflt_rings, max_rx_rings, max_tx_rings, rc;
11956 
11957 	if (!bnxt_can_reserve_rings(bp))
11958 		return 0;
11959 
11960 	if (sh)
11961 		bp->flags |= BNXT_FLAG_SHARED_RINGS;
11962 	dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues();
11963 	/* Reduce default rings on multi-port cards so that total default
11964 	 * rings do not exceed CPU count.
11965 	 */
11966 	if (bp->port_count > 1) {
11967 		int max_rings =
11968 			max_t(int, num_online_cpus() / bp->port_count, 1);
11969 
11970 		dflt_rings = min_t(int, dflt_rings, max_rings);
11971 	}
11972 	rc = bnxt_get_dflt_rings(bp, &max_rx_rings, &max_tx_rings, sh);
11973 	if (rc)
11974 		return rc;
11975 	bp->rx_nr_rings = min_t(int, dflt_rings, max_rx_rings);
11976 	bp->tx_nr_rings_per_tc = min_t(int, dflt_rings, max_tx_rings);
11977 	if (sh)
11978 		bnxt_trim_dflt_sh_rings(bp);
11979 	else
11980 		bp->cp_nr_rings = bp->tx_nr_rings_per_tc + bp->rx_nr_rings;
11981 	bp->tx_nr_rings = bp->tx_nr_rings_per_tc;
11982 
11983 	rc = __bnxt_reserve_rings(bp);
11984 	if (rc)
11985 		netdev_warn(bp->dev, "Unable to reserve tx rings\n");
11986 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11987 	if (sh)
11988 		bnxt_trim_dflt_sh_rings(bp);
11989 
11990 	/* Rings may have been trimmed, re-reserve the trimmed rings. */
11991 	if (bnxt_need_reserve_rings(bp)) {
11992 		rc = __bnxt_reserve_rings(bp);
11993 		if (rc)
11994 			netdev_warn(bp->dev, "2nd rings reservation failed.\n");
11995 		bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
11996 	}
11997 	if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
11998 		bp->rx_nr_rings++;
11999 		bp->cp_nr_rings++;
12000 	}
12001 	if (rc) {
12002 		bp->tx_nr_rings = 0;
12003 		bp->rx_nr_rings = 0;
12004 	}
12005 	return rc;
12006 }
12007 
12008 static int bnxt_init_dflt_ring_mode(struct bnxt *bp)
12009 {
12010 	int rc;
12011 
12012 	if (bp->tx_nr_rings)
12013 		return 0;
12014 
12015 	bnxt_ulp_irq_stop(bp);
12016 	bnxt_clear_int_mode(bp);
12017 	rc = bnxt_set_dflt_rings(bp, true);
12018 	if (rc) {
12019 		netdev_err(bp->dev, "Not enough rings available.\n");
12020 		goto init_dflt_ring_err;
12021 	}
12022 	rc = bnxt_init_int_mode(bp);
12023 	if (rc)
12024 		goto init_dflt_ring_err;
12025 
12026 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12027 	if (bnxt_rfs_supported(bp) && bnxt_rfs_capable(bp)) {
12028 		bp->flags |= BNXT_FLAG_RFS;
12029 		bp->dev->features |= NETIF_F_NTUPLE;
12030 	}
12031 init_dflt_ring_err:
12032 	bnxt_ulp_irq_restart(bp, rc);
12033 	return rc;
12034 }
12035 
12036 int bnxt_restore_pf_fw_resources(struct bnxt *bp)
12037 {
12038 	int rc;
12039 
12040 	ASSERT_RTNL();
12041 	bnxt_hwrm_func_qcaps(bp);
12042 
12043 	if (netif_running(bp->dev))
12044 		__bnxt_close_nic(bp, true, false);
12045 
12046 	bnxt_ulp_irq_stop(bp);
12047 	bnxt_clear_int_mode(bp);
12048 	rc = bnxt_init_int_mode(bp);
12049 	bnxt_ulp_irq_restart(bp, rc);
12050 
12051 	if (netif_running(bp->dev)) {
12052 		if (rc)
12053 			dev_close(bp->dev);
12054 		else
12055 			rc = bnxt_open_nic(bp, true, false);
12056 	}
12057 
12058 	return rc;
12059 }
12060 
12061 static int bnxt_init_mac_addr(struct bnxt *bp)
12062 {
12063 	int rc = 0;
12064 
12065 	if (BNXT_PF(bp)) {
12066 		memcpy(bp->dev->dev_addr, bp->pf.mac_addr, ETH_ALEN);
12067 	} else {
12068 #ifdef CONFIG_BNXT_SRIOV
12069 		struct bnxt_vf_info *vf = &bp->vf;
12070 		bool strict_approval = true;
12071 
12072 		if (is_valid_ether_addr(vf->mac_addr)) {
12073 			/* overwrite netdev dev_addr with admin VF MAC */
12074 			memcpy(bp->dev->dev_addr, vf->mac_addr, ETH_ALEN);
12075 			/* Older PF driver or firmware may not approve this
12076 			 * correctly.
12077 			 */
12078 			strict_approval = false;
12079 		} else {
12080 			eth_hw_addr_random(bp->dev);
12081 		}
12082 		rc = bnxt_approve_mac(bp, bp->dev->dev_addr, strict_approval);
12083 #endif
12084 	}
12085 	return rc;
12086 }
12087 
12088 #define BNXT_VPD_LEN	512
12089 static void bnxt_vpd_read_info(struct bnxt *bp)
12090 {
12091 	struct pci_dev *pdev = bp->pdev;
12092 	int i, len, pos, ro_size;
12093 	ssize_t vpd_size;
12094 	u8 *vpd_data;
12095 
12096 	vpd_data = kmalloc(BNXT_VPD_LEN, GFP_KERNEL);
12097 	if (!vpd_data)
12098 		return;
12099 
12100 	vpd_size = pci_read_vpd(pdev, 0, BNXT_VPD_LEN, vpd_data);
12101 	if (vpd_size <= 0) {
12102 		netdev_err(bp->dev, "Unable to read VPD\n");
12103 		goto exit;
12104 	}
12105 
12106 	i = pci_vpd_find_tag(vpd_data, 0, vpd_size, PCI_VPD_LRDT_RO_DATA);
12107 	if (i < 0) {
12108 		netdev_err(bp->dev, "VPD READ-Only not found\n");
12109 		goto exit;
12110 	}
12111 
12112 	ro_size = pci_vpd_lrdt_size(&vpd_data[i]);
12113 	i += PCI_VPD_LRDT_TAG_SIZE;
12114 	if (i + ro_size > vpd_size)
12115 		goto exit;
12116 
12117 	pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12118 					PCI_VPD_RO_KEYWORD_PARTNO);
12119 	if (pos < 0)
12120 		goto read_sn;
12121 
12122 	len = pci_vpd_info_field_size(&vpd_data[pos]);
12123 	pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12124 	if (len + pos > vpd_size)
12125 		goto read_sn;
12126 
12127 	strlcpy(bp->board_partno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN));
12128 
12129 read_sn:
12130 	pos = pci_vpd_find_info_keyword(vpd_data, i, ro_size,
12131 					PCI_VPD_RO_KEYWORD_SERIALNO);
12132 	if (pos < 0)
12133 		goto exit;
12134 
12135 	len = pci_vpd_info_field_size(&vpd_data[pos]);
12136 	pos += PCI_VPD_INFO_FLD_HDR_SIZE;
12137 	if (len + pos > vpd_size)
12138 		goto exit;
12139 
12140 	strlcpy(bp->board_serialno, &vpd_data[pos], min(len, BNXT_VPD_FLD_LEN));
12141 exit:
12142 	kfree(vpd_data);
12143 }
12144 
12145 static int bnxt_pcie_dsn_get(struct bnxt *bp, u8 dsn[])
12146 {
12147 	struct pci_dev *pdev = bp->pdev;
12148 	u64 qword;
12149 
12150 	qword = pci_get_dsn(pdev);
12151 	if (!qword) {
12152 		netdev_info(bp->dev, "Unable to read adapter's DSN\n");
12153 		return -EOPNOTSUPP;
12154 	}
12155 
12156 	put_unaligned_le64(qword, dsn);
12157 
12158 	bp->flags |= BNXT_FLAG_DSN_VALID;
12159 	return 0;
12160 }
12161 
12162 static int bnxt_map_db_bar(struct bnxt *bp)
12163 {
12164 	if (!bp->db_size)
12165 		return -ENODEV;
12166 	bp->bar1 = pci_iomap(bp->pdev, 2, bp->db_size);
12167 	if (!bp->bar1)
12168 		return -ENOMEM;
12169 	return 0;
12170 }
12171 
12172 static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
12173 {
12174 	struct net_device *dev;
12175 	struct bnxt *bp;
12176 	int rc, max_irqs;
12177 
12178 	if (pci_is_bridge(pdev))
12179 		return -ENODEV;
12180 
12181 	/* Clear any pending DMA transactions from crash kernel
12182 	 * while loading driver in capture kernel.
12183 	 */
12184 	if (is_kdump_kernel()) {
12185 		pci_clear_master(pdev);
12186 		pcie_flr(pdev);
12187 	}
12188 
12189 	max_irqs = bnxt_get_max_irq(pdev);
12190 	dev = alloc_etherdev_mq(sizeof(*bp), max_irqs);
12191 	if (!dev)
12192 		return -ENOMEM;
12193 
12194 	bp = netdev_priv(dev);
12195 	bnxt_set_max_func_irqs(bp, max_irqs);
12196 
12197 	if (bnxt_vf_pciid(ent->driver_data))
12198 		bp->flags |= BNXT_FLAG_VF;
12199 
12200 	if (pdev->msix_cap)
12201 		bp->flags |= BNXT_FLAG_MSIX_CAP;
12202 
12203 	rc = bnxt_init_board(pdev, dev);
12204 	if (rc < 0)
12205 		goto init_err_free;
12206 
12207 	dev->netdev_ops = &bnxt_netdev_ops;
12208 	dev->watchdog_timeo = BNXT_TX_TIMEOUT;
12209 	dev->ethtool_ops = &bnxt_ethtool_ops;
12210 	pci_set_drvdata(pdev, dev);
12211 
12212 	if (BNXT_PF(bp))
12213 		bnxt_vpd_read_info(bp);
12214 
12215 	rc = bnxt_alloc_hwrm_resources(bp);
12216 	if (rc)
12217 		goto init_err_pci_clean;
12218 
12219 	mutex_init(&bp->hwrm_cmd_lock);
12220 	mutex_init(&bp->link_lock);
12221 
12222 	rc = bnxt_fw_init_one_p1(bp);
12223 	if (rc)
12224 		goto init_err_pci_clean;
12225 
12226 	if (BNXT_CHIP_P5(bp))
12227 		bp->flags |= BNXT_FLAG_CHIP_P5;
12228 
12229 	rc = bnxt_alloc_rss_indir_tbl(bp);
12230 	if (rc)
12231 		goto init_err_pci_clean;
12232 
12233 	rc = bnxt_fw_init_one_p2(bp);
12234 	if (rc)
12235 		goto init_err_pci_clean;
12236 
12237 	rc = bnxt_map_db_bar(bp);
12238 	if (rc) {
12239 		dev_err(&pdev->dev, "Cannot map doorbell BAR rc = %d, aborting\n",
12240 			rc);
12241 		goto init_err_pci_clean;
12242 	}
12243 
12244 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12245 			   NETIF_F_TSO | NETIF_F_TSO6 |
12246 			   NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12247 			   NETIF_F_GSO_IPXIP4 |
12248 			   NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12249 			   NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH |
12250 			   NETIF_F_RXCSUM | NETIF_F_GRO;
12251 
12252 	if (BNXT_SUPPORTS_TPA(bp))
12253 		dev->hw_features |= NETIF_F_LRO;
12254 
12255 	dev->hw_enc_features =
12256 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_SG |
12257 			NETIF_F_TSO | NETIF_F_TSO6 |
12258 			NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |
12259 			NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM |
12260 			NETIF_F_GSO_IPXIP4 | NETIF_F_GSO_PARTIAL;
12261 	dev->udp_tunnel_nic_info = &bnxt_udp_tunnels;
12262 
12263 	dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM |
12264 				    NETIF_F_GSO_GRE_CSUM;
12265 	dev->vlan_features = dev->hw_features | NETIF_F_HIGHDMA;
12266 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_RX_STRIP)
12267 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_RX;
12268 	if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
12269 		dev->hw_features |= BNXT_HW_FEATURE_VLAN_ALL_TX;
12270 	if (BNXT_SUPPORTS_TPA(bp))
12271 		dev->hw_features |= NETIF_F_GRO_HW;
12272 	dev->features |= dev->hw_features | NETIF_F_HIGHDMA;
12273 	if (dev->features & NETIF_F_GRO_HW)
12274 		dev->features &= ~NETIF_F_LRO;
12275 	dev->priv_flags |= IFF_UNICAST_FLT;
12276 
12277 #ifdef CONFIG_BNXT_SRIOV
12278 	init_waitqueue_head(&bp->sriov_cfg_wait);
12279 	mutex_init(&bp->sriov_lock);
12280 #endif
12281 	if (BNXT_SUPPORTS_TPA(bp)) {
12282 		bp->gro_func = bnxt_gro_func_5730x;
12283 		if (BNXT_CHIP_P4(bp))
12284 			bp->gro_func = bnxt_gro_func_5731x;
12285 		else if (BNXT_CHIP_P5(bp))
12286 			bp->gro_func = bnxt_gro_func_5750x;
12287 	}
12288 	if (!BNXT_CHIP_P4_PLUS(bp))
12289 		bp->flags |= BNXT_FLAG_DOUBLE_DB;
12290 
12291 	bp->ulp_probe = bnxt_ulp_probe;
12292 
12293 	rc = bnxt_init_mac_addr(bp);
12294 	if (rc) {
12295 		dev_err(&pdev->dev, "Unable to initialize mac address.\n");
12296 		rc = -EADDRNOTAVAIL;
12297 		goto init_err_pci_clean;
12298 	}
12299 
12300 	if (BNXT_PF(bp)) {
12301 		/* Read the adapter's DSN to use as the eswitch switch_id */
12302 		rc = bnxt_pcie_dsn_get(bp, bp->dsn);
12303 	}
12304 
12305 	/* MTU range: 60 - FW defined max */
12306 	dev->min_mtu = ETH_ZLEN;
12307 	dev->max_mtu = bp->max_mtu;
12308 
12309 	rc = bnxt_probe_phy(bp, true);
12310 	if (rc)
12311 		goto init_err_pci_clean;
12312 
12313 	bnxt_set_rx_skb_mode(bp, false);
12314 	bnxt_set_tpa_flags(bp);
12315 	bnxt_set_ring_params(bp);
12316 	rc = bnxt_set_dflt_rings(bp, true);
12317 	if (rc) {
12318 		netdev_err(bp->dev, "Not enough rings available.\n");
12319 		rc = -ENOMEM;
12320 		goto init_err_pci_clean;
12321 	}
12322 
12323 	bnxt_fw_init_one_p3(bp);
12324 
12325 	if (dev->hw_features & BNXT_HW_FEATURE_VLAN_ALL_RX)
12326 		bp->flags |= BNXT_FLAG_STRIP_VLAN;
12327 
12328 	rc = bnxt_init_int_mode(bp);
12329 	if (rc)
12330 		goto init_err_pci_clean;
12331 
12332 	/* No TC has been set yet and rings may have been trimmed due to
12333 	 * limited MSIX, so we re-initialize the TX rings per TC.
12334 	 */
12335 	bp->tx_nr_rings_per_tc = bp->tx_nr_rings;
12336 
12337 	if (BNXT_PF(bp)) {
12338 		if (!bnxt_pf_wq) {
12339 			bnxt_pf_wq =
12340 				create_singlethread_workqueue("bnxt_pf_wq");
12341 			if (!bnxt_pf_wq) {
12342 				dev_err(&pdev->dev, "Unable to create workqueue.\n");
12343 				goto init_err_pci_clean;
12344 			}
12345 		}
12346 		rc = bnxt_init_tc(bp);
12347 		if (rc)
12348 			netdev_err(dev, "Failed to initialize TC flower offload, err = %d.\n",
12349 				   rc);
12350 	}
12351 
12352 	bnxt_dl_register(bp);
12353 
12354 	rc = register_netdev(dev);
12355 	if (rc)
12356 		goto init_err_cleanup;
12357 
12358 	if (BNXT_PF(bp))
12359 		devlink_port_type_eth_set(&bp->dl_port, bp->dev);
12360 	bnxt_dl_fw_reporters_create(bp);
12361 
12362 	netdev_info(dev, "%s found at mem %lx, node addr %pM\n",
12363 		    board_info[ent->driver_data].name,
12364 		    (long)pci_resource_start(pdev, 0), dev->dev_addr);
12365 	pcie_print_link_status(pdev);
12366 
12367 	pci_save_state(pdev);
12368 	return 0;
12369 
12370 init_err_cleanup:
12371 	bnxt_dl_unregister(bp);
12372 	bnxt_shutdown_tc(bp);
12373 	bnxt_clear_int_mode(bp);
12374 
12375 init_err_pci_clean:
12376 	bnxt_hwrm_func_drv_unrgtr(bp);
12377 	bnxt_free_hwrm_short_cmd_req(bp);
12378 	bnxt_free_hwrm_resources(bp);
12379 	kfree(bp->fw_health);
12380 	bp->fw_health = NULL;
12381 	bnxt_cleanup_pci(bp);
12382 	bnxt_free_ctx_mem(bp);
12383 	kfree(bp->ctx);
12384 	bp->ctx = NULL;
12385 	kfree(bp->rss_indir_tbl);
12386 	bp->rss_indir_tbl = NULL;
12387 
12388 init_err_free:
12389 	free_netdev(dev);
12390 	return rc;
12391 }
12392 
12393 static void bnxt_shutdown(struct pci_dev *pdev)
12394 {
12395 	struct net_device *dev = pci_get_drvdata(pdev);
12396 	struct bnxt *bp;
12397 
12398 	if (!dev)
12399 		return;
12400 
12401 	rtnl_lock();
12402 	bp = netdev_priv(dev);
12403 	if (!bp)
12404 		goto shutdown_exit;
12405 
12406 	if (netif_running(dev))
12407 		dev_close(dev);
12408 
12409 	bnxt_ulp_shutdown(bp);
12410 	bnxt_clear_int_mode(bp);
12411 	pci_disable_device(pdev);
12412 
12413 	if (system_state == SYSTEM_POWER_OFF) {
12414 		pci_wake_from_d3(pdev, bp->wol);
12415 		pci_set_power_state(pdev, PCI_D3hot);
12416 	}
12417 
12418 shutdown_exit:
12419 	rtnl_unlock();
12420 }
12421 
12422 #ifdef CONFIG_PM_SLEEP
12423 static int bnxt_suspend(struct device *device)
12424 {
12425 	struct net_device *dev = dev_get_drvdata(device);
12426 	struct bnxt *bp = netdev_priv(dev);
12427 	int rc = 0;
12428 
12429 	rtnl_lock();
12430 	bnxt_ulp_stop(bp);
12431 	if (netif_running(dev)) {
12432 		netif_device_detach(dev);
12433 		rc = bnxt_close(dev);
12434 	}
12435 	bnxt_hwrm_func_drv_unrgtr(bp);
12436 	pci_disable_device(bp->pdev);
12437 	bnxt_free_ctx_mem(bp);
12438 	kfree(bp->ctx);
12439 	bp->ctx = NULL;
12440 	rtnl_unlock();
12441 	return rc;
12442 }
12443 
12444 static int bnxt_resume(struct device *device)
12445 {
12446 	struct net_device *dev = dev_get_drvdata(device);
12447 	struct bnxt *bp = netdev_priv(dev);
12448 	int rc = 0;
12449 
12450 	rtnl_lock();
12451 	rc = pci_enable_device(bp->pdev);
12452 	if (rc) {
12453 		netdev_err(dev, "Cannot re-enable PCI device during resume, err = %d\n",
12454 			   rc);
12455 		goto resume_exit;
12456 	}
12457 	pci_set_master(bp->pdev);
12458 	if (bnxt_hwrm_ver_get(bp)) {
12459 		rc = -ENODEV;
12460 		goto resume_exit;
12461 	}
12462 	rc = bnxt_hwrm_func_reset(bp);
12463 	if (rc) {
12464 		rc = -EBUSY;
12465 		goto resume_exit;
12466 	}
12467 
12468 	rc = bnxt_hwrm_func_qcaps(bp);
12469 	if (rc)
12470 		goto resume_exit;
12471 
12472 	if (bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false)) {
12473 		rc = -ENODEV;
12474 		goto resume_exit;
12475 	}
12476 
12477 	bnxt_get_wol_settings(bp);
12478 	if (netif_running(dev)) {
12479 		rc = bnxt_open(dev);
12480 		if (!rc)
12481 			netif_device_attach(dev);
12482 	}
12483 
12484 resume_exit:
12485 	bnxt_ulp_start(bp, rc);
12486 	if (!rc)
12487 		bnxt_reenable_sriov(bp);
12488 	rtnl_unlock();
12489 	return rc;
12490 }
12491 
12492 static SIMPLE_DEV_PM_OPS(bnxt_pm_ops, bnxt_suspend, bnxt_resume);
12493 #define BNXT_PM_OPS (&bnxt_pm_ops)
12494 
12495 #else
12496 
12497 #define BNXT_PM_OPS NULL
12498 
12499 #endif /* CONFIG_PM_SLEEP */
12500 
12501 /**
12502  * bnxt_io_error_detected - called when PCI error is detected
12503  * @pdev: Pointer to PCI device
12504  * @state: The current pci connection state
12505  *
12506  * This function is called after a PCI bus error affecting
12507  * this device has been detected.
12508  */
12509 static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev,
12510 					       pci_channel_state_t state)
12511 {
12512 	struct net_device *netdev = pci_get_drvdata(pdev);
12513 	struct bnxt *bp = netdev_priv(netdev);
12514 
12515 	netdev_info(netdev, "PCI I/O error detected\n");
12516 
12517 	rtnl_lock();
12518 	netif_device_detach(netdev);
12519 
12520 	bnxt_ulp_stop(bp);
12521 
12522 	if (state == pci_channel_io_perm_failure) {
12523 		rtnl_unlock();
12524 		return PCI_ERS_RESULT_DISCONNECT;
12525 	}
12526 
12527 	if (netif_running(netdev))
12528 		bnxt_close(netdev);
12529 
12530 	pci_disable_device(pdev);
12531 	bnxt_free_ctx_mem(bp);
12532 	kfree(bp->ctx);
12533 	bp->ctx = NULL;
12534 	rtnl_unlock();
12535 
12536 	/* Request a slot slot reset. */
12537 	return PCI_ERS_RESULT_NEED_RESET;
12538 }
12539 
12540 /**
12541  * bnxt_io_slot_reset - called after the pci bus has been reset.
12542  * @pdev: Pointer to PCI device
12543  *
12544  * Restart the card from scratch, as if from a cold-boot.
12545  * At this point, the card has exprienced a hard reset,
12546  * followed by fixups by BIOS, and has its config space
12547  * set up identically to what it was at cold boot.
12548  */
12549 static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
12550 {
12551 	struct net_device *netdev = pci_get_drvdata(pdev);
12552 	struct bnxt *bp = netdev_priv(netdev);
12553 	int err = 0;
12554 	pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT;
12555 
12556 	netdev_info(bp->dev, "PCI Slot Reset\n");
12557 
12558 	rtnl_lock();
12559 
12560 	if (pci_enable_device(pdev)) {
12561 		dev_err(&pdev->dev,
12562 			"Cannot re-enable PCI device after reset.\n");
12563 	} else {
12564 		pci_set_master(pdev);
12565 		pci_restore_state(pdev);
12566 		pci_save_state(pdev);
12567 
12568 		err = bnxt_hwrm_func_reset(bp);
12569 		if (!err) {
12570 			err = bnxt_hwrm_func_qcaps(bp);
12571 			if (!err && netif_running(netdev))
12572 				err = bnxt_open(netdev);
12573 		}
12574 		bnxt_ulp_start(bp, err);
12575 		if (!err) {
12576 			bnxt_reenable_sriov(bp);
12577 			result = PCI_ERS_RESULT_RECOVERED;
12578 		}
12579 	}
12580 
12581 	if (result != PCI_ERS_RESULT_RECOVERED) {
12582 		if (netif_running(netdev))
12583 			dev_close(netdev);
12584 		pci_disable_device(pdev);
12585 	}
12586 
12587 	rtnl_unlock();
12588 
12589 	return result;
12590 }
12591 
12592 /**
12593  * bnxt_io_resume - called when traffic can start flowing again.
12594  * @pdev: Pointer to PCI device
12595  *
12596  * This callback is called when the error recovery driver tells
12597  * us that its OK to resume normal operation.
12598  */
12599 static void bnxt_io_resume(struct pci_dev *pdev)
12600 {
12601 	struct net_device *netdev = pci_get_drvdata(pdev);
12602 
12603 	rtnl_lock();
12604 
12605 	netif_device_attach(netdev);
12606 
12607 	rtnl_unlock();
12608 }
12609 
12610 static const struct pci_error_handlers bnxt_err_handler = {
12611 	.error_detected	= bnxt_io_error_detected,
12612 	.slot_reset	= bnxt_io_slot_reset,
12613 	.resume		= bnxt_io_resume
12614 };
12615 
12616 static struct pci_driver bnxt_pci_driver = {
12617 	.name		= DRV_MODULE_NAME,
12618 	.id_table	= bnxt_pci_tbl,
12619 	.probe		= bnxt_init_one,
12620 	.remove		= bnxt_remove_one,
12621 	.shutdown	= bnxt_shutdown,
12622 	.driver.pm	= BNXT_PM_OPS,
12623 	.err_handler	= &bnxt_err_handler,
12624 #if defined(CONFIG_BNXT_SRIOV)
12625 	.sriov_configure = bnxt_sriov_configure,
12626 #endif
12627 };
12628 
12629 static int __init bnxt_init(void)
12630 {
12631 	bnxt_debug_init();
12632 	return pci_register_driver(&bnxt_pci_driver);
12633 }
12634 
12635 static void __exit bnxt_exit(void)
12636 {
12637 	pci_unregister_driver(&bnxt_pci_driver);
12638 	if (bnxt_pf_wq)
12639 		destroy_workqueue(bnxt_pf_wq);
12640 	bnxt_debug_exit();
12641 }
12642 
12643 module_init(bnxt_init);
12644 module_exit(bnxt_exit);
12645