1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2005 - 2016 Broadcom
4  * All rights reserved.
5  *
6  * Contact Information:
7  * linux-drivers@emulex.com
8  *
9  * Emulex
10  * 3333 Susan Street
11  * Costa Mesa, CA 92626
12  */
13 
14 #include <linux/prefetch.h>
15 #include <linux/module.h>
16 #include "be.h"
17 #include "be_cmds.h"
18 #include <asm/div64.h>
19 #include <linux/aer.h>
20 #include <linux/if_bridge.h>
21 #include <net/busy_poll.h>
22 #include <net/vxlan.h>
23 
24 MODULE_VERSION(DRV_VER);
25 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
26 MODULE_AUTHOR("Emulex Corporation");
27 MODULE_LICENSE("GPL");
28 
29 /* num_vfs module param is obsolete.
30  * Use sysfs method to enable/disable VFs.
31  */
32 static unsigned int num_vfs;
33 module_param(num_vfs, uint, 0444);
34 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
35 
36 static ushort rx_frag_size = 2048;
37 module_param(rx_frag_size, ushort, 0444);
38 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
39 
40 /* Per-module error detection/recovery workq shared across all functions.
41  * Each function schedules its own work request on this shared workq.
42  */
43 static struct workqueue_struct *be_err_recovery_workq;
44 
45 static const struct pci_device_id be_dev_ids[] = {
46 #ifdef CONFIG_BE2NET_BE2
47 	{ PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
48 	{ PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
49 #endif /* CONFIG_BE2NET_BE2 */
50 #ifdef CONFIG_BE2NET_BE3
51 	{ PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
52 	{ PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
53 #endif /* CONFIG_BE2NET_BE3 */
54 #ifdef CONFIG_BE2NET_LANCER
55 	{ PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
56 	{ PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
57 #endif /* CONFIG_BE2NET_LANCER */
58 #ifdef CONFIG_BE2NET_SKYHAWK
59 	{ PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
60 	{ PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
61 #endif /* CONFIG_BE2NET_SKYHAWK */
62 	{ 0 }
63 };
64 MODULE_DEVICE_TABLE(pci, be_dev_ids);
65 
66 /* Workqueue used by all functions for defering cmd calls to the adapter */
67 static struct workqueue_struct *be_wq;
68 
69 /* UE Status Low CSR */
70 static const char * const ue_status_low_desc[] = {
71 	"CEV",
72 	"CTX",
73 	"DBUF",
74 	"ERX",
75 	"Host",
76 	"MPU",
77 	"NDMA",
78 	"PTC ",
79 	"RDMA ",
80 	"RXF ",
81 	"RXIPS ",
82 	"RXULP0 ",
83 	"RXULP1 ",
84 	"RXULP2 ",
85 	"TIM ",
86 	"TPOST ",
87 	"TPRE ",
88 	"TXIPS ",
89 	"TXULP0 ",
90 	"TXULP1 ",
91 	"UC ",
92 	"WDMA ",
93 	"TXULP2 ",
94 	"HOST1 ",
95 	"P0_OB_LINK ",
96 	"P1_OB_LINK ",
97 	"HOST_GPIO ",
98 	"MBOX ",
99 	"ERX2 ",
100 	"SPARE ",
101 	"JTAG ",
102 	"MPU_INTPEND "
103 };
104 
105 /* UE Status High CSR */
106 static const char * const ue_status_hi_desc[] = {
107 	"LPCMEMHOST",
108 	"MGMT_MAC",
109 	"PCS0ONLINE",
110 	"MPU_IRAM",
111 	"PCS1ONLINE",
112 	"PCTL0",
113 	"PCTL1",
114 	"PMEM",
115 	"RR",
116 	"TXPB",
117 	"RXPP",
118 	"XAUI",
119 	"TXP",
120 	"ARM",
121 	"IPC",
122 	"HOST2",
123 	"HOST3",
124 	"HOST4",
125 	"HOST5",
126 	"HOST6",
127 	"HOST7",
128 	"ECRC",
129 	"Poison TLP",
130 	"NETC",
131 	"PERIPH",
132 	"LLTXULP",
133 	"D2P",
134 	"RCON",
135 	"LDMA",
136 	"LLTXP",
137 	"LLTXPB",
138 	"Unknown"
139 };
140 
141 #define BE_VF_IF_EN_FLAGS	(BE_IF_FLAGS_UNTAGGED | \
142 				 BE_IF_FLAGS_BROADCAST | \
143 				 BE_IF_FLAGS_MULTICAST | \
144 				 BE_IF_FLAGS_PASS_L3L4_ERRORS)
145 
146 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
147 {
148 	struct be_dma_mem *mem = &q->dma_mem;
149 
150 	if (mem->va) {
151 		dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
152 				  mem->dma);
153 		mem->va = NULL;
154 	}
155 }
156 
157 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
158 			  u16 len, u16 entry_size)
159 {
160 	struct be_dma_mem *mem = &q->dma_mem;
161 
162 	memset(q, 0, sizeof(*q));
163 	q->len = len;
164 	q->entry_size = entry_size;
165 	mem->size = len * entry_size;
166 	mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
167 				     &mem->dma, GFP_KERNEL);
168 	if (!mem->va)
169 		return -ENOMEM;
170 	return 0;
171 }
172 
173 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
174 {
175 	u32 reg, enabled;
176 
177 	pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
178 			      &reg);
179 	enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
180 
181 	if (!enabled && enable)
182 		reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
183 	else if (enabled && !enable)
184 		reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
185 	else
186 		return;
187 
188 	pci_write_config_dword(adapter->pdev,
189 			       PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
190 }
191 
192 static void be_intr_set(struct be_adapter *adapter, bool enable)
193 {
194 	int status = 0;
195 
196 	/* On lancer interrupts can't be controlled via this register */
197 	if (lancer_chip(adapter))
198 		return;
199 
200 	if (be_check_error(adapter, BE_ERROR_EEH))
201 		return;
202 
203 	status = be_cmd_intr_set(adapter, enable);
204 	if (status)
205 		be_reg_intr_set(adapter, enable);
206 }
207 
208 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
209 {
210 	u32 val = 0;
211 
212 	if (be_check_error(adapter, BE_ERROR_HW))
213 		return;
214 
215 	val |= qid & DB_RQ_RING_ID_MASK;
216 	val |= posted << DB_RQ_NUM_POSTED_SHIFT;
217 
218 	wmb();
219 	iowrite32(val, adapter->db + DB_RQ_OFFSET);
220 }
221 
222 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
223 			  u16 posted)
224 {
225 	u32 val = 0;
226 
227 	if (be_check_error(adapter, BE_ERROR_HW))
228 		return;
229 
230 	val |= txo->q.id & DB_TXULP_RING_ID_MASK;
231 	val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
232 
233 	wmb();
234 	iowrite32(val, adapter->db + txo->db_offset);
235 }
236 
237 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
238 			 bool arm, bool clear_int, u16 num_popped,
239 			 u32 eq_delay_mult_enc)
240 {
241 	u32 val = 0;
242 
243 	val |= qid & DB_EQ_RING_ID_MASK;
244 	val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
245 
246 	if (be_check_error(adapter, BE_ERROR_HW))
247 		return;
248 
249 	if (arm)
250 		val |= 1 << DB_EQ_REARM_SHIFT;
251 	if (clear_int)
252 		val |= 1 << DB_EQ_CLR_SHIFT;
253 	val |= 1 << DB_EQ_EVNT_SHIFT;
254 	val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
255 	val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT;
256 	iowrite32(val, adapter->db + DB_EQ_OFFSET);
257 }
258 
259 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
260 {
261 	u32 val = 0;
262 
263 	val |= qid & DB_CQ_RING_ID_MASK;
264 	val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
265 			DB_CQ_RING_ID_EXT_MASK_SHIFT);
266 
267 	if (be_check_error(adapter, BE_ERROR_HW))
268 		return;
269 
270 	if (arm)
271 		val |= 1 << DB_CQ_REARM_SHIFT;
272 	val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
273 	iowrite32(val, adapter->db + DB_CQ_OFFSET);
274 }
275 
276 static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac)
277 {
278 	int i;
279 
280 	/* Check if mac has already been added as part of uc-list */
281 	for (i = 0; i < adapter->uc_macs; i++) {
282 		if (ether_addr_equal(adapter->uc_list[i].mac, mac)) {
283 			/* mac already added, skip addition */
284 			adapter->pmac_id[0] = adapter->pmac_id[i + 1];
285 			return 0;
286 		}
287 	}
288 
289 	return be_cmd_pmac_add(adapter, mac, adapter->if_handle,
290 			       &adapter->pmac_id[0], 0);
291 }
292 
293 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id)
294 {
295 	int i;
296 
297 	/* Skip deletion if the programmed mac is
298 	 * being used in uc-list
299 	 */
300 	for (i = 0; i < adapter->uc_macs; i++) {
301 		if (adapter->pmac_id[i + 1] == pmac_id)
302 			return;
303 	}
304 	be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
305 }
306 
307 static int be_mac_addr_set(struct net_device *netdev, void *p)
308 {
309 	struct be_adapter *adapter = netdev_priv(netdev);
310 	struct device *dev = &adapter->pdev->dev;
311 	struct sockaddr *addr = p;
312 	int status;
313 	u8 mac[ETH_ALEN];
314 	u32 old_pmac_id = adapter->pmac_id[0];
315 
316 	if (!is_valid_ether_addr(addr->sa_data))
317 		return -EADDRNOTAVAIL;
318 
319 	/* Proceed further only if, User provided MAC is different
320 	 * from active MAC
321 	 */
322 	if (ether_addr_equal(addr->sa_data, adapter->dev_mac))
323 		return 0;
324 
325 	/* BE3 VFs without FILTMGMT privilege are not allowed to set its MAC
326 	 * address
327 	 */
328 	if (BEx_chip(adapter) && be_virtfn(adapter) &&
329 	    !check_privilege(adapter, BE_PRIV_FILTMGMT))
330 		return -EPERM;
331 
332 	/* if device is not running, copy MAC to netdev->dev_addr */
333 	if (!netif_running(netdev))
334 		goto done;
335 
336 	/* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
337 	 * privilege or if PF did not provision the new MAC address.
338 	 * On BE3, this cmd will always fail if the VF doesn't have the
339 	 * FILTMGMT privilege. This failure is OK, only if the PF programmed
340 	 * the MAC for the VF.
341 	 */
342 	mutex_lock(&adapter->rx_filter_lock);
343 	status = be_dev_mac_add(adapter, (u8 *)addr->sa_data);
344 	if (!status) {
345 
346 		/* Delete the old programmed MAC. This call may fail if the
347 		 * old MAC was already deleted by the PF driver.
348 		 */
349 		if (adapter->pmac_id[0] != old_pmac_id)
350 			be_dev_mac_del(adapter, old_pmac_id);
351 	}
352 
353 	mutex_unlock(&adapter->rx_filter_lock);
354 	/* Decide if the new MAC is successfully activated only after
355 	 * querying the FW
356 	 */
357 	status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac,
358 				       adapter->if_handle, true, 0);
359 	if (status)
360 		goto err;
361 
362 	/* The MAC change did not happen, either due to lack of privilege
363 	 * or PF didn't pre-provision.
364 	 */
365 	if (!ether_addr_equal(addr->sa_data, mac)) {
366 		status = -EPERM;
367 		goto err;
368 	}
369 
370 	/* Remember currently programmed MAC */
371 	ether_addr_copy(adapter->dev_mac, addr->sa_data);
372 done:
373 	ether_addr_copy(netdev->dev_addr, addr->sa_data);
374 	dev_info(dev, "MAC address changed to %pM\n", addr->sa_data);
375 	return 0;
376 err:
377 	dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
378 	return status;
379 }
380 
381 /* BE2 supports only v0 cmd */
382 static void *hw_stats_from_cmd(struct be_adapter *adapter)
383 {
384 	if (BE2_chip(adapter)) {
385 		struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
386 
387 		return &cmd->hw_stats;
388 	} else if (BE3_chip(adapter)) {
389 		struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
390 
391 		return &cmd->hw_stats;
392 	} else {
393 		struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
394 
395 		return &cmd->hw_stats;
396 	}
397 }
398 
399 /* BE2 supports only v0 cmd */
400 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
401 {
402 	if (BE2_chip(adapter)) {
403 		struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
404 
405 		return &hw_stats->erx;
406 	} else if (BE3_chip(adapter)) {
407 		struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
408 
409 		return &hw_stats->erx;
410 	} else {
411 		struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
412 
413 		return &hw_stats->erx;
414 	}
415 }
416 
417 static void populate_be_v0_stats(struct be_adapter *adapter)
418 {
419 	struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
420 	struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
421 	struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
422 	struct be_port_rxf_stats_v0 *port_stats =
423 					&rxf_stats->port[adapter->port_num];
424 	struct be_drv_stats *drvs = &adapter->drv_stats;
425 
426 	be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
427 	drvs->rx_pause_frames = port_stats->rx_pause_frames;
428 	drvs->rx_crc_errors = port_stats->rx_crc_errors;
429 	drvs->rx_control_frames = port_stats->rx_control_frames;
430 	drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
431 	drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
432 	drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
433 	drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
434 	drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
435 	drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
436 	drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
437 	drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
438 	drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
439 	drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
440 	drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
441 	drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
442 	drvs->rx_dropped_header_too_small =
443 		port_stats->rx_dropped_header_too_small;
444 	drvs->rx_address_filtered =
445 					port_stats->rx_address_filtered +
446 					port_stats->rx_vlan_filtered;
447 	drvs->rx_alignment_symbol_errors =
448 		port_stats->rx_alignment_symbol_errors;
449 
450 	drvs->tx_pauseframes = port_stats->tx_pauseframes;
451 	drvs->tx_controlframes = port_stats->tx_controlframes;
452 
453 	if (adapter->port_num)
454 		drvs->jabber_events = rxf_stats->port1_jabber_events;
455 	else
456 		drvs->jabber_events = rxf_stats->port0_jabber_events;
457 	drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
458 	drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
459 	drvs->forwarded_packets = rxf_stats->forwarded_packets;
460 	drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
461 	drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
462 	drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
463 	adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
464 }
465 
466 static void populate_be_v1_stats(struct be_adapter *adapter)
467 {
468 	struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
469 	struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
470 	struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
471 	struct be_port_rxf_stats_v1 *port_stats =
472 					&rxf_stats->port[adapter->port_num];
473 	struct be_drv_stats *drvs = &adapter->drv_stats;
474 
475 	be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
476 	drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
477 	drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
478 	drvs->rx_pause_frames = port_stats->rx_pause_frames;
479 	drvs->rx_crc_errors = port_stats->rx_crc_errors;
480 	drvs->rx_control_frames = port_stats->rx_control_frames;
481 	drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
482 	drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
483 	drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
484 	drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
485 	drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
486 	drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
487 	drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
488 	drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
489 	drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
490 	drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
491 	drvs->rx_dropped_header_too_small =
492 		port_stats->rx_dropped_header_too_small;
493 	drvs->rx_input_fifo_overflow_drop =
494 		port_stats->rx_input_fifo_overflow_drop;
495 	drvs->rx_address_filtered = port_stats->rx_address_filtered;
496 	drvs->rx_alignment_symbol_errors =
497 		port_stats->rx_alignment_symbol_errors;
498 	drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
499 	drvs->tx_pauseframes = port_stats->tx_pauseframes;
500 	drvs->tx_controlframes = port_stats->tx_controlframes;
501 	drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
502 	drvs->jabber_events = port_stats->jabber_events;
503 	drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
504 	drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
505 	drvs->forwarded_packets = rxf_stats->forwarded_packets;
506 	drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
507 	drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
508 	drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
509 	adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
510 }
511 
512 static void populate_be_v2_stats(struct be_adapter *adapter)
513 {
514 	struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
515 	struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
516 	struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
517 	struct be_port_rxf_stats_v2 *port_stats =
518 					&rxf_stats->port[adapter->port_num];
519 	struct be_drv_stats *drvs = &adapter->drv_stats;
520 
521 	be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
522 	drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
523 	drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
524 	drvs->rx_pause_frames = port_stats->rx_pause_frames;
525 	drvs->rx_crc_errors = port_stats->rx_crc_errors;
526 	drvs->rx_control_frames = port_stats->rx_control_frames;
527 	drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
528 	drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
529 	drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
530 	drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
531 	drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
532 	drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
533 	drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
534 	drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
535 	drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
536 	drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
537 	drvs->rx_dropped_header_too_small =
538 		port_stats->rx_dropped_header_too_small;
539 	drvs->rx_input_fifo_overflow_drop =
540 		port_stats->rx_input_fifo_overflow_drop;
541 	drvs->rx_address_filtered = port_stats->rx_address_filtered;
542 	drvs->rx_alignment_symbol_errors =
543 		port_stats->rx_alignment_symbol_errors;
544 	drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
545 	drvs->tx_pauseframes = port_stats->tx_pauseframes;
546 	drvs->tx_controlframes = port_stats->tx_controlframes;
547 	drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
548 	drvs->jabber_events = port_stats->jabber_events;
549 	drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
550 	drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
551 	drvs->forwarded_packets = rxf_stats->forwarded_packets;
552 	drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
553 	drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
554 	drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
555 	adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
556 	if (be_roce_supported(adapter)) {
557 		drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
558 		drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
559 		drvs->rx_roce_frames = port_stats->roce_frames_received;
560 		drvs->roce_drops_crc = port_stats->roce_drops_crc;
561 		drvs->roce_drops_payload_len =
562 			port_stats->roce_drops_payload_len;
563 	}
564 }
565 
566 static void populate_lancer_stats(struct be_adapter *adapter)
567 {
568 	struct be_drv_stats *drvs = &adapter->drv_stats;
569 	struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
570 
571 	be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
572 	drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
573 	drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
574 	drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
575 	drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
576 	drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
577 	drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
578 	drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
579 	drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
580 	drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
581 	drvs->rx_dropped_tcp_length =
582 				pport_stats->rx_dropped_invalid_tcp_length;
583 	drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
584 	drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
585 	drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
586 	drvs->rx_dropped_header_too_small =
587 				pport_stats->rx_dropped_header_too_small;
588 	drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
589 	drvs->rx_address_filtered =
590 					pport_stats->rx_address_filtered +
591 					pport_stats->rx_vlan_filtered;
592 	drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
593 	drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
594 	drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
595 	drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
596 	drvs->jabber_events = pport_stats->rx_jabbers;
597 	drvs->forwarded_packets = pport_stats->num_forwards_lo;
598 	drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
599 	drvs->rx_drops_too_many_frags =
600 				pport_stats->rx_drops_too_many_frags_lo;
601 }
602 
603 static void accumulate_16bit_val(u32 *acc, u16 val)
604 {
605 #define lo(x)			(x & 0xFFFF)
606 #define hi(x)			(x & 0xFFFF0000)
607 	bool wrapped = val < lo(*acc);
608 	u32 newacc = hi(*acc) + val;
609 
610 	if (wrapped)
611 		newacc += 65536;
612 	WRITE_ONCE(*acc, newacc);
613 }
614 
615 static void populate_erx_stats(struct be_adapter *adapter,
616 			       struct be_rx_obj *rxo, u32 erx_stat)
617 {
618 	if (!BEx_chip(adapter))
619 		rx_stats(rxo)->rx_drops_no_frags = erx_stat;
620 	else
621 		/* below erx HW counter can actually wrap around after
622 		 * 65535. Driver accumulates a 32-bit value
623 		 */
624 		accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
625 				     (u16)erx_stat);
626 }
627 
628 void be_parse_stats(struct be_adapter *adapter)
629 {
630 	struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
631 	struct be_rx_obj *rxo;
632 	int i;
633 	u32 erx_stat;
634 
635 	if (lancer_chip(adapter)) {
636 		populate_lancer_stats(adapter);
637 	} else {
638 		if (BE2_chip(adapter))
639 			populate_be_v0_stats(adapter);
640 		else if (BE3_chip(adapter))
641 			/* for BE3 */
642 			populate_be_v1_stats(adapter);
643 		else
644 			populate_be_v2_stats(adapter);
645 
646 		/* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
647 		for_all_rx_queues(adapter, rxo, i) {
648 			erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
649 			populate_erx_stats(adapter, rxo, erx_stat);
650 		}
651 	}
652 }
653 
654 static void be_get_stats64(struct net_device *netdev,
655 			   struct rtnl_link_stats64 *stats)
656 {
657 	struct be_adapter *adapter = netdev_priv(netdev);
658 	struct be_drv_stats *drvs = &adapter->drv_stats;
659 	struct be_rx_obj *rxo;
660 	struct be_tx_obj *txo;
661 	u64 pkts, bytes;
662 	unsigned int start;
663 	int i;
664 
665 	for_all_rx_queues(adapter, rxo, i) {
666 		const struct be_rx_stats *rx_stats = rx_stats(rxo);
667 
668 		do {
669 			start = u64_stats_fetch_begin_irq(&rx_stats->sync);
670 			pkts = rx_stats(rxo)->rx_pkts;
671 			bytes = rx_stats(rxo)->rx_bytes;
672 		} while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
673 		stats->rx_packets += pkts;
674 		stats->rx_bytes += bytes;
675 		stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
676 		stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
677 					rx_stats(rxo)->rx_drops_no_frags;
678 	}
679 
680 	for_all_tx_queues(adapter, txo, i) {
681 		const struct be_tx_stats *tx_stats = tx_stats(txo);
682 
683 		do {
684 			start = u64_stats_fetch_begin_irq(&tx_stats->sync);
685 			pkts = tx_stats(txo)->tx_pkts;
686 			bytes = tx_stats(txo)->tx_bytes;
687 		} while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
688 		stats->tx_packets += pkts;
689 		stats->tx_bytes += bytes;
690 	}
691 
692 	/* bad pkts received */
693 	stats->rx_errors = drvs->rx_crc_errors +
694 		drvs->rx_alignment_symbol_errors +
695 		drvs->rx_in_range_errors +
696 		drvs->rx_out_range_errors +
697 		drvs->rx_frame_too_long +
698 		drvs->rx_dropped_too_small +
699 		drvs->rx_dropped_too_short +
700 		drvs->rx_dropped_header_too_small +
701 		drvs->rx_dropped_tcp_length +
702 		drvs->rx_dropped_runt;
703 
704 	/* detailed rx errors */
705 	stats->rx_length_errors = drvs->rx_in_range_errors +
706 		drvs->rx_out_range_errors +
707 		drvs->rx_frame_too_long;
708 
709 	stats->rx_crc_errors = drvs->rx_crc_errors;
710 
711 	/* frame alignment errors */
712 	stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
713 
714 	/* receiver fifo overrun */
715 	/* drops_no_pbuf is no per i/f, it's per BE card */
716 	stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
717 				drvs->rx_input_fifo_overflow_drop +
718 				drvs->rx_drops_no_pbuf;
719 }
720 
721 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
722 {
723 	struct net_device *netdev = adapter->netdev;
724 
725 	if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
726 		netif_carrier_off(netdev);
727 		adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
728 	}
729 
730 	if (link_status)
731 		netif_carrier_on(netdev);
732 	else
733 		netif_carrier_off(netdev);
734 
735 	netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down");
736 }
737 
738 static int be_gso_hdr_len(struct sk_buff *skb)
739 {
740 	if (skb->encapsulation)
741 		return skb_inner_transport_offset(skb) +
742 		       inner_tcp_hdrlen(skb);
743 	return skb_transport_offset(skb) + tcp_hdrlen(skb);
744 }
745 
746 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb)
747 {
748 	struct be_tx_stats *stats = tx_stats(txo);
749 	u32 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1;
750 	/* Account for headers which get duplicated in TSO pkt */
751 	u32 dup_hdr_len = tx_pkts > 1 ? be_gso_hdr_len(skb) * (tx_pkts - 1) : 0;
752 
753 	u64_stats_update_begin(&stats->sync);
754 	stats->tx_reqs++;
755 	stats->tx_bytes += skb->len + dup_hdr_len;
756 	stats->tx_pkts += tx_pkts;
757 	if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL)
758 		stats->tx_vxlan_offload_pkts += tx_pkts;
759 	u64_stats_update_end(&stats->sync);
760 }
761 
762 /* Returns number of WRBs needed for the skb */
763 static u32 skb_wrb_cnt(struct sk_buff *skb)
764 {
765 	/* +1 for the header wrb */
766 	return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags;
767 }
768 
769 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
770 {
771 	wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr));
772 	wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr));
773 	wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK);
774 	wrb->rsvd0 = 0;
775 }
776 
777 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb
778  * to avoid the swap and shift/mask operations in wrb_fill().
779  */
780 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb)
781 {
782 	wrb->frag_pa_hi = 0;
783 	wrb->frag_pa_lo = 0;
784 	wrb->frag_len = 0;
785 	wrb->rsvd0 = 0;
786 }
787 
788 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
789 				     struct sk_buff *skb)
790 {
791 	u8 vlan_prio;
792 	u16 vlan_tag;
793 
794 	vlan_tag = skb_vlan_tag_get(skb);
795 	vlan_prio = skb_vlan_tag_get_prio(skb);
796 	/* If vlan priority provided by OS is NOT in available bmap */
797 	if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
798 		vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
799 				adapter->recommended_prio_bits;
800 
801 	return vlan_tag;
802 }
803 
804 /* Used only for IP tunnel packets */
805 static u16 skb_inner_ip_proto(struct sk_buff *skb)
806 {
807 	return (inner_ip_hdr(skb)->version == 4) ?
808 		inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
809 }
810 
811 static u16 skb_ip_proto(struct sk_buff *skb)
812 {
813 	return (ip_hdr(skb)->version == 4) ?
814 		ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
815 }
816 
817 static inline bool be_is_txq_full(struct be_tx_obj *txo)
818 {
819 	return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len;
820 }
821 
822 static inline bool be_can_txq_wake(struct be_tx_obj *txo)
823 {
824 	return atomic_read(&txo->q.used) < txo->q.len / 2;
825 }
826 
827 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo)
828 {
829 	return atomic_read(&txo->q.used) > txo->pend_wrb_cnt;
830 }
831 
832 static void be_get_wrb_params_from_skb(struct be_adapter *adapter,
833 				       struct sk_buff *skb,
834 				       struct be_wrb_params *wrb_params)
835 {
836 	u16 proto;
837 
838 	if (skb_is_gso(skb)) {
839 		BE_WRB_F_SET(wrb_params->features, LSO, 1);
840 		wrb_params->lso_mss = skb_shinfo(skb)->gso_size;
841 		if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
842 			BE_WRB_F_SET(wrb_params->features, LSO6, 1);
843 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
844 		if (skb->encapsulation) {
845 			BE_WRB_F_SET(wrb_params->features, IPCS, 1);
846 			proto = skb_inner_ip_proto(skb);
847 		} else {
848 			proto = skb_ip_proto(skb);
849 		}
850 		if (proto == IPPROTO_TCP)
851 			BE_WRB_F_SET(wrb_params->features, TCPCS, 1);
852 		else if (proto == IPPROTO_UDP)
853 			BE_WRB_F_SET(wrb_params->features, UDPCS, 1);
854 	}
855 
856 	if (skb_vlan_tag_present(skb)) {
857 		BE_WRB_F_SET(wrb_params->features, VLAN, 1);
858 		wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb);
859 	}
860 
861 	BE_WRB_F_SET(wrb_params->features, CRC, 1);
862 }
863 
864 static void wrb_fill_hdr(struct be_adapter *adapter,
865 			 struct be_eth_hdr_wrb *hdr,
866 			 struct be_wrb_params *wrb_params,
867 			 struct sk_buff *skb)
868 {
869 	memset(hdr, 0, sizeof(*hdr));
870 
871 	SET_TX_WRB_HDR_BITS(crc, hdr,
872 			    BE_WRB_F_GET(wrb_params->features, CRC));
873 	SET_TX_WRB_HDR_BITS(ipcs, hdr,
874 			    BE_WRB_F_GET(wrb_params->features, IPCS));
875 	SET_TX_WRB_HDR_BITS(tcpcs, hdr,
876 			    BE_WRB_F_GET(wrb_params->features, TCPCS));
877 	SET_TX_WRB_HDR_BITS(udpcs, hdr,
878 			    BE_WRB_F_GET(wrb_params->features, UDPCS));
879 
880 	SET_TX_WRB_HDR_BITS(lso, hdr,
881 			    BE_WRB_F_GET(wrb_params->features, LSO));
882 	SET_TX_WRB_HDR_BITS(lso6, hdr,
883 			    BE_WRB_F_GET(wrb_params->features, LSO6));
884 	SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss);
885 
886 	/* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this
887 	 * hack is not needed, the evt bit is set while ringing DB.
888 	 */
889 	SET_TX_WRB_HDR_BITS(event, hdr,
890 			    BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW));
891 	SET_TX_WRB_HDR_BITS(vlan, hdr,
892 			    BE_WRB_F_GET(wrb_params->features, VLAN));
893 	SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag);
894 
895 	SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb));
896 	SET_TX_WRB_HDR_BITS(len, hdr, skb->len);
897 	SET_TX_WRB_HDR_BITS(mgmt, hdr,
898 			    BE_WRB_F_GET(wrb_params->features, OS2BMC));
899 }
900 
901 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
902 			  bool unmap_single)
903 {
904 	dma_addr_t dma;
905 	u32 frag_len = le32_to_cpu(wrb->frag_len);
906 
907 
908 	dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 |
909 		(u64)le32_to_cpu(wrb->frag_pa_lo);
910 	if (frag_len) {
911 		if (unmap_single)
912 			dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE);
913 		else
914 			dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE);
915 	}
916 }
917 
918 /* Grab a WRB header for xmit */
919 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo)
920 {
921 	u32 head = txo->q.head;
922 
923 	queue_head_inc(&txo->q);
924 	return head;
925 }
926 
927 /* Set up the WRB header for xmit */
928 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter,
929 				struct be_tx_obj *txo,
930 				struct be_wrb_params *wrb_params,
931 				struct sk_buff *skb, u16 head)
932 {
933 	u32 num_frags = skb_wrb_cnt(skb);
934 	struct be_queue_info *txq = &txo->q;
935 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head);
936 
937 	wrb_fill_hdr(adapter, hdr, wrb_params, skb);
938 	be_dws_cpu_to_le(hdr, sizeof(*hdr));
939 
940 	BUG_ON(txo->sent_skb_list[head]);
941 	txo->sent_skb_list[head] = skb;
942 	txo->last_req_hdr = head;
943 	atomic_add(num_frags, &txq->used);
944 	txo->last_req_wrb_cnt = num_frags;
945 	txo->pend_wrb_cnt += num_frags;
946 }
947 
948 /* Setup a WRB fragment (buffer descriptor) for xmit */
949 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr,
950 				 int len)
951 {
952 	struct be_eth_wrb *wrb;
953 	struct be_queue_info *txq = &txo->q;
954 
955 	wrb = queue_head_node(txq);
956 	wrb_fill(wrb, busaddr, len);
957 	queue_head_inc(txq);
958 }
959 
960 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine
961  * was invoked. The producer index is restored to the previous packet and the
962  * WRBs of the current packet are unmapped. Invoked to handle tx setup errors.
963  */
964 static void be_xmit_restore(struct be_adapter *adapter,
965 			    struct be_tx_obj *txo, u32 head, bool map_single,
966 			    u32 copied)
967 {
968 	struct device *dev;
969 	struct be_eth_wrb *wrb;
970 	struct be_queue_info *txq = &txo->q;
971 
972 	dev = &adapter->pdev->dev;
973 	txq->head = head;
974 
975 	/* skip the first wrb (hdr); it's not mapped */
976 	queue_head_inc(txq);
977 	while (copied) {
978 		wrb = queue_head_node(txq);
979 		unmap_tx_frag(dev, wrb, map_single);
980 		map_single = false;
981 		copied -= le32_to_cpu(wrb->frag_len);
982 		queue_head_inc(txq);
983 	}
984 
985 	txq->head = head;
986 }
987 
988 /* Enqueue the given packet for transmit. This routine allocates WRBs for the
989  * packet, dma maps the packet buffers and sets up the WRBs. Returns the number
990  * of WRBs used up by the packet.
991  */
992 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo,
993 			   struct sk_buff *skb,
994 			   struct be_wrb_params *wrb_params)
995 {
996 	u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb);
997 	struct device *dev = &adapter->pdev->dev;
998 	bool map_single = false;
999 	u32 head;
1000 	dma_addr_t busaddr;
1001 	int len;
1002 
1003 	head = be_tx_get_wrb_hdr(txo);
1004 
1005 	if (skb->len > skb->data_len) {
1006 		len = skb_headlen(skb);
1007 
1008 		busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
1009 		if (dma_mapping_error(dev, busaddr))
1010 			goto dma_err;
1011 		map_single = true;
1012 		be_tx_setup_wrb_frag(txo, busaddr, len);
1013 		copied += len;
1014 	}
1015 
1016 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1017 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1018 		len = skb_frag_size(frag);
1019 
1020 		busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE);
1021 		if (dma_mapping_error(dev, busaddr))
1022 			goto dma_err;
1023 		be_tx_setup_wrb_frag(txo, busaddr, len);
1024 		copied += len;
1025 	}
1026 
1027 	be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head);
1028 
1029 	be_tx_stats_update(txo, skb);
1030 	return wrb_cnt;
1031 
1032 dma_err:
1033 	adapter->drv_stats.dma_map_errors++;
1034 	be_xmit_restore(adapter, txo, head, map_single, copied);
1035 	return 0;
1036 }
1037 
1038 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
1039 {
1040 	return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
1041 }
1042 
1043 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
1044 					     struct sk_buff *skb,
1045 					     struct be_wrb_params
1046 					     *wrb_params)
1047 {
1048 	bool insert_vlan = false;
1049 	u16 vlan_tag = 0;
1050 
1051 	skb = skb_share_check(skb, GFP_ATOMIC);
1052 	if (unlikely(!skb))
1053 		return skb;
1054 
1055 	if (skb_vlan_tag_present(skb)) {
1056 		vlan_tag = be_get_tx_vlan_tag(adapter, skb);
1057 		insert_vlan = true;
1058 	}
1059 
1060 	if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
1061 		if (!insert_vlan) {
1062 			vlan_tag = adapter->pvid;
1063 			insert_vlan = true;
1064 		}
1065 		/* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
1066 		 * skip VLAN insertion
1067 		 */
1068 		BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1069 	}
1070 
1071 	if (insert_vlan) {
1072 		skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
1073 						vlan_tag);
1074 		if (unlikely(!skb))
1075 			return skb;
1076 		__vlan_hwaccel_clear_tag(skb);
1077 	}
1078 
1079 	/* Insert the outer VLAN, if any */
1080 	if (adapter->qnq_vid) {
1081 		vlan_tag = adapter->qnq_vid;
1082 		skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
1083 						vlan_tag);
1084 		if (unlikely(!skb))
1085 			return skb;
1086 		BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1087 	}
1088 
1089 	return skb;
1090 }
1091 
1092 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
1093 {
1094 	struct ethhdr *eh = (struct ethhdr *)skb->data;
1095 	u16 offset = ETH_HLEN;
1096 
1097 	if (eh->h_proto == htons(ETH_P_IPV6)) {
1098 		struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
1099 
1100 		offset += sizeof(struct ipv6hdr);
1101 		if (ip6h->nexthdr != NEXTHDR_TCP &&
1102 		    ip6h->nexthdr != NEXTHDR_UDP) {
1103 			struct ipv6_opt_hdr *ehdr =
1104 				(struct ipv6_opt_hdr *)(skb->data + offset);
1105 
1106 			/* offending pkt: 2nd byte following IPv6 hdr is 0xff */
1107 			if (ehdr->hdrlen == 0xff)
1108 				return true;
1109 		}
1110 	}
1111 	return false;
1112 }
1113 
1114 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
1115 {
1116 	return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
1117 }
1118 
1119 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
1120 {
1121 	return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
1122 }
1123 
1124 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
1125 						  struct sk_buff *skb,
1126 						  struct be_wrb_params
1127 						  *wrb_params)
1128 {
1129 	struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
1130 	unsigned int eth_hdr_len;
1131 	struct iphdr *ip;
1132 
1133 	/* For padded packets, BE HW modifies tot_len field in IP header
1134 	 * incorrecly when VLAN tag is inserted by HW.
1135 	 * For padded packets, Lancer computes incorrect checksum.
1136 	 */
1137 	eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
1138 						VLAN_ETH_HLEN : ETH_HLEN;
1139 	if (skb->len <= 60 &&
1140 	    (lancer_chip(adapter) || skb_vlan_tag_present(skb)) &&
1141 	    is_ipv4_pkt(skb)) {
1142 		ip = (struct iphdr *)ip_hdr(skb);
1143 		pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
1144 	}
1145 
1146 	/* If vlan tag is already inlined in the packet, skip HW VLAN
1147 	 * tagging in pvid-tagging mode
1148 	 */
1149 	if (be_pvid_tagging_enabled(adapter) &&
1150 	    veh->h_vlan_proto == htons(ETH_P_8021Q))
1151 		BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1);
1152 
1153 	/* HW has a bug wherein it will calculate CSUM for VLAN
1154 	 * pkts even though it is disabled.
1155 	 * Manually insert VLAN in pkt.
1156 	 */
1157 	if (skb->ip_summed != CHECKSUM_PARTIAL &&
1158 	    skb_vlan_tag_present(skb)) {
1159 		skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1160 		if (unlikely(!skb))
1161 			goto err;
1162 	}
1163 
1164 	/* HW may lockup when VLAN HW tagging is requested on
1165 	 * certain ipv6 packets. Drop such pkts if the HW workaround to
1166 	 * skip HW tagging is not enabled by FW.
1167 	 */
1168 	if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
1169 		     (adapter->pvid || adapter->qnq_vid) &&
1170 		     !qnq_async_evt_rcvd(adapter)))
1171 		goto tx_drop;
1172 
1173 	/* Manual VLAN tag insertion to prevent:
1174 	 * ASIC lockup when the ASIC inserts VLAN tag into
1175 	 * certain ipv6 packets. Insert VLAN tags in driver,
1176 	 * and set event, completion, vlan bits accordingly
1177 	 * in the Tx WRB.
1178 	 */
1179 	if (be_ipv6_tx_stall_chk(adapter, skb) &&
1180 	    be_vlan_tag_tx_chk(adapter, skb)) {
1181 		skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params);
1182 		if (unlikely(!skb))
1183 			goto err;
1184 	}
1185 
1186 	return skb;
1187 tx_drop:
1188 	dev_kfree_skb_any(skb);
1189 err:
1190 	return NULL;
1191 }
1192 
1193 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1194 					   struct sk_buff *skb,
1195 					   struct be_wrb_params *wrb_params)
1196 {
1197 	int err;
1198 
1199 	/* Lancer, SH and BE3 in SRIOV mode have a bug wherein
1200 	 * packets that are 32b or less may cause a transmit stall
1201 	 * on that port. The workaround is to pad such packets
1202 	 * (len <= 32 bytes) to a minimum length of 36b.
1203 	 */
1204 	if (skb->len <= 32) {
1205 		if (skb_put_padto(skb, 36))
1206 			return NULL;
1207 	}
1208 
1209 	if (BEx_chip(adapter) || lancer_chip(adapter)) {
1210 		skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params);
1211 		if (!skb)
1212 			return NULL;
1213 	}
1214 
1215 	/* The stack can send us skbs with length greater than
1216 	 * what the HW can handle. Trim the extra bytes.
1217 	 */
1218 	WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE);
1219 	err = pskb_trim(skb, BE_MAX_GSO_SIZE);
1220 	WARN_ON(err);
1221 
1222 	return skb;
1223 }
1224 
1225 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo)
1226 {
1227 	struct be_queue_info *txq = &txo->q;
1228 	struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr);
1229 
1230 	/* Mark the last request eventable if it hasn't been marked already */
1231 	if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT)))
1232 		hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL);
1233 
1234 	/* compose a dummy wrb if there are odd set of wrbs to notify */
1235 	if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) {
1236 		wrb_fill_dummy(queue_head_node(txq));
1237 		queue_head_inc(txq);
1238 		atomic_inc(&txq->used);
1239 		txo->pend_wrb_cnt++;
1240 		hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK <<
1241 					   TX_HDR_WRB_NUM_SHIFT);
1242 		hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) <<
1243 					  TX_HDR_WRB_NUM_SHIFT);
1244 	}
1245 	be_txq_notify(adapter, txo, txo->pend_wrb_cnt);
1246 	txo->pend_wrb_cnt = 0;
1247 }
1248 
1249 /* OS2BMC related */
1250 
1251 #define DHCP_CLIENT_PORT	68
1252 #define DHCP_SERVER_PORT	67
1253 #define NET_BIOS_PORT1		137
1254 #define NET_BIOS_PORT2		138
1255 #define DHCPV6_RAS_PORT		547
1256 
1257 #define is_mc_allowed_on_bmc(adapter, eh)	\
1258 	(!is_multicast_filt_enabled(adapter) &&	\
1259 	 is_multicast_ether_addr(eh->h_dest) &&	\
1260 	 !is_broadcast_ether_addr(eh->h_dest))
1261 
1262 #define is_bc_allowed_on_bmc(adapter, eh)	\
1263 	(!is_broadcast_filt_enabled(adapter) &&	\
1264 	 is_broadcast_ether_addr(eh->h_dest))
1265 
1266 #define is_arp_allowed_on_bmc(adapter, skb)	\
1267 	(is_arp(skb) && is_arp_filt_enabled(adapter))
1268 
1269 #define is_arp(skb)	(skb->protocol == htons(ETH_P_ARP))
1270 
1271 #define is_arp_filt_enabled(adapter)	\
1272 		(adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP))
1273 
1274 #define is_dhcp_client_filt_enabled(adapter)	\
1275 		(adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT)
1276 
1277 #define is_dhcp_srvr_filt_enabled(adapter)	\
1278 		(adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER)
1279 
1280 #define is_nbios_filt_enabled(adapter)	\
1281 		(adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS)
1282 
1283 #define is_ipv6_na_filt_enabled(adapter)	\
1284 		(adapter->bmc_filt_mask &	\
1285 			BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER)
1286 
1287 #define is_ipv6_ra_filt_enabled(adapter)	\
1288 		(adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA)
1289 
1290 #define is_ipv6_ras_filt_enabled(adapter)	\
1291 		(adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS)
1292 
1293 #define is_broadcast_filt_enabled(adapter)	\
1294 		(adapter->bmc_filt_mask & BMC_FILT_BROADCAST)
1295 
1296 #define is_multicast_filt_enabled(adapter)	\
1297 		(adapter->bmc_filt_mask & BMC_FILT_MULTICAST)
1298 
1299 static bool be_send_pkt_to_bmc(struct be_adapter *adapter,
1300 			       struct sk_buff **skb)
1301 {
1302 	struct ethhdr *eh = (struct ethhdr *)(*skb)->data;
1303 	bool os2bmc = false;
1304 
1305 	if (!be_is_os2bmc_enabled(adapter))
1306 		goto done;
1307 
1308 	if (!is_multicast_ether_addr(eh->h_dest))
1309 		goto done;
1310 
1311 	if (is_mc_allowed_on_bmc(adapter, eh) ||
1312 	    is_bc_allowed_on_bmc(adapter, eh) ||
1313 	    is_arp_allowed_on_bmc(adapter, (*skb))) {
1314 		os2bmc = true;
1315 		goto done;
1316 	}
1317 
1318 	if ((*skb)->protocol == htons(ETH_P_IPV6)) {
1319 		struct ipv6hdr *hdr = ipv6_hdr((*skb));
1320 		u8 nexthdr = hdr->nexthdr;
1321 
1322 		if (nexthdr == IPPROTO_ICMPV6) {
1323 			struct icmp6hdr *icmp6 = icmp6_hdr((*skb));
1324 
1325 			switch (icmp6->icmp6_type) {
1326 			case NDISC_ROUTER_ADVERTISEMENT:
1327 				os2bmc = is_ipv6_ra_filt_enabled(adapter);
1328 				goto done;
1329 			case NDISC_NEIGHBOUR_ADVERTISEMENT:
1330 				os2bmc = is_ipv6_na_filt_enabled(adapter);
1331 				goto done;
1332 			default:
1333 				break;
1334 			}
1335 		}
1336 	}
1337 
1338 	if (is_udp_pkt((*skb))) {
1339 		struct udphdr *udp = udp_hdr((*skb));
1340 
1341 		switch (ntohs(udp->dest)) {
1342 		case DHCP_CLIENT_PORT:
1343 			os2bmc = is_dhcp_client_filt_enabled(adapter);
1344 			goto done;
1345 		case DHCP_SERVER_PORT:
1346 			os2bmc = is_dhcp_srvr_filt_enabled(adapter);
1347 			goto done;
1348 		case NET_BIOS_PORT1:
1349 		case NET_BIOS_PORT2:
1350 			os2bmc = is_nbios_filt_enabled(adapter);
1351 			goto done;
1352 		case DHCPV6_RAS_PORT:
1353 			os2bmc = is_ipv6_ras_filt_enabled(adapter);
1354 			goto done;
1355 		default:
1356 			break;
1357 		}
1358 	}
1359 done:
1360 	/* For packets over a vlan, which are destined
1361 	 * to BMC, asic expects the vlan to be inline in the packet.
1362 	 */
1363 	if (os2bmc)
1364 		*skb = be_insert_vlan_in_pkt(adapter, *skb, NULL);
1365 
1366 	return os2bmc;
1367 }
1368 
1369 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1370 {
1371 	struct be_adapter *adapter = netdev_priv(netdev);
1372 	u16 q_idx = skb_get_queue_mapping(skb);
1373 	struct be_tx_obj *txo = &adapter->tx_obj[q_idx];
1374 	struct be_wrb_params wrb_params = { 0 };
1375 	bool flush = !netdev_xmit_more();
1376 	u16 wrb_cnt;
1377 
1378 	skb = be_xmit_workarounds(adapter, skb, &wrb_params);
1379 	if (unlikely(!skb))
1380 		goto drop;
1381 
1382 	be_get_wrb_params_from_skb(adapter, skb, &wrb_params);
1383 
1384 	wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1385 	if (unlikely(!wrb_cnt)) {
1386 		dev_kfree_skb_any(skb);
1387 		goto drop;
1388 	}
1389 
1390 	/* if os2bmc is enabled and if the pkt is destined to bmc,
1391 	 * enqueue the pkt a 2nd time with mgmt bit set.
1392 	 */
1393 	if (be_send_pkt_to_bmc(adapter, &skb)) {
1394 		BE_WRB_F_SET(wrb_params.features, OS2BMC, 1);
1395 		wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params);
1396 		if (unlikely(!wrb_cnt))
1397 			goto drop;
1398 		else
1399 			skb_get(skb);
1400 	}
1401 
1402 	if (be_is_txq_full(txo)) {
1403 		netif_stop_subqueue(netdev, q_idx);
1404 		tx_stats(txo)->tx_stops++;
1405 	}
1406 
1407 	if (flush || __netif_subqueue_stopped(netdev, q_idx))
1408 		be_xmit_flush(adapter, txo);
1409 
1410 	return NETDEV_TX_OK;
1411 drop:
1412 	tx_stats(txo)->tx_drv_drops++;
1413 	/* Flush the already enqueued tx requests */
1414 	if (flush && txo->pend_wrb_cnt)
1415 		be_xmit_flush(adapter, txo);
1416 
1417 	return NETDEV_TX_OK;
1418 }
1419 
1420 static void be_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1421 {
1422 	struct be_adapter *adapter = netdev_priv(netdev);
1423 	struct device *dev = &adapter->pdev->dev;
1424 	struct be_tx_obj *txo;
1425 	struct sk_buff *skb;
1426 	struct tcphdr *tcphdr;
1427 	struct udphdr *udphdr;
1428 	u32 *entry;
1429 	int status;
1430 	int i, j;
1431 
1432 	for_all_tx_queues(adapter, txo, i) {
1433 		dev_info(dev, "TXQ Dump: %d H: %d T: %d used: %d, qid: 0x%x\n",
1434 			 i, txo->q.head, txo->q.tail,
1435 			 atomic_read(&txo->q.used), txo->q.id);
1436 
1437 		entry = txo->q.dma_mem.va;
1438 		for (j = 0; j < TX_Q_LEN * 4; j += 4) {
1439 			if (entry[j] != 0 || entry[j + 1] != 0 ||
1440 			    entry[j + 2] != 0 || entry[j + 3] != 0) {
1441 				dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n",
1442 					 j, entry[j], entry[j + 1],
1443 					 entry[j + 2], entry[j + 3]);
1444 			}
1445 		}
1446 
1447 		entry = txo->cq.dma_mem.va;
1448 		dev_info(dev, "TXCQ Dump: %d  H: %d T: %d used: %d\n",
1449 			 i, txo->cq.head, txo->cq.tail,
1450 			 atomic_read(&txo->cq.used));
1451 		for (j = 0; j < TX_CQ_LEN * 4; j += 4) {
1452 			if (entry[j] != 0 || entry[j + 1] != 0 ||
1453 			    entry[j + 2] != 0 || entry[j + 3] != 0) {
1454 				dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n",
1455 					 j, entry[j], entry[j + 1],
1456 					 entry[j + 2], entry[j + 3]);
1457 			}
1458 		}
1459 
1460 		for (j = 0; j < TX_Q_LEN; j++) {
1461 			if (txo->sent_skb_list[j]) {
1462 				skb = txo->sent_skb_list[j];
1463 				if (ip_hdr(skb)->protocol == IPPROTO_TCP) {
1464 					tcphdr = tcp_hdr(skb);
1465 					dev_info(dev, "TCP source port %d\n",
1466 						 ntohs(tcphdr->source));
1467 					dev_info(dev, "TCP dest port %d\n",
1468 						 ntohs(tcphdr->dest));
1469 					dev_info(dev, "TCP sequence num %d\n",
1470 						 ntohs(tcphdr->seq));
1471 					dev_info(dev, "TCP ack_seq %d\n",
1472 						 ntohs(tcphdr->ack_seq));
1473 				} else if (ip_hdr(skb)->protocol ==
1474 					   IPPROTO_UDP) {
1475 					udphdr = udp_hdr(skb);
1476 					dev_info(dev, "UDP source port %d\n",
1477 						 ntohs(udphdr->source));
1478 					dev_info(dev, "UDP dest port %d\n",
1479 						 ntohs(udphdr->dest));
1480 				}
1481 				dev_info(dev, "skb[%d] %p len %d proto 0x%x\n",
1482 					 j, skb, skb->len, skb->protocol);
1483 			}
1484 		}
1485 	}
1486 
1487 	if (lancer_chip(adapter)) {
1488 		dev_info(dev, "Initiating reset due to tx timeout\n");
1489 		dev_info(dev, "Resetting adapter\n");
1490 		status = lancer_physdev_ctrl(adapter,
1491 					     PHYSDEV_CONTROL_FW_RESET_MASK);
1492 		if (status)
1493 			dev_err(dev, "Reset failed .. Reboot server\n");
1494 	}
1495 }
1496 
1497 static inline bool be_in_all_promisc(struct be_adapter *adapter)
1498 {
1499 	return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) ==
1500 			BE_IF_FLAGS_ALL_PROMISCUOUS;
1501 }
1502 
1503 static int be_set_vlan_promisc(struct be_adapter *adapter)
1504 {
1505 	struct device *dev = &adapter->pdev->dev;
1506 	int status;
1507 
1508 	if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS)
1509 		return 0;
1510 
1511 	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON);
1512 	if (!status) {
1513 		dev_info(dev, "Enabled VLAN promiscuous mode\n");
1514 		adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS;
1515 	} else {
1516 		dev_err(dev, "Failed to enable VLAN promiscuous mode\n");
1517 	}
1518 	return status;
1519 }
1520 
1521 static int be_clear_vlan_promisc(struct be_adapter *adapter)
1522 {
1523 	struct device *dev = &adapter->pdev->dev;
1524 	int status;
1525 
1526 	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF);
1527 	if (!status) {
1528 		dev_info(dev, "Disabling VLAN promiscuous mode\n");
1529 		adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
1530 	}
1531 	return status;
1532 }
1533 
1534 /*
1535  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1536  * If the user configures more, place BE in vlan promiscuous mode.
1537  */
1538 static int be_vid_config(struct be_adapter *adapter)
1539 {
1540 	struct device *dev = &adapter->pdev->dev;
1541 	u16 vids[BE_NUM_VLANS_SUPPORTED];
1542 	u16 num = 0, i = 0;
1543 	int status = 0;
1544 
1545 	/* No need to change the VLAN state if the I/F is in promiscuous */
1546 	if (adapter->netdev->flags & IFF_PROMISC)
1547 		return 0;
1548 
1549 	if (adapter->vlans_added > be_max_vlans(adapter))
1550 		return be_set_vlan_promisc(adapter);
1551 
1552 	if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
1553 		status = be_clear_vlan_promisc(adapter);
1554 		if (status)
1555 			return status;
1556 	}
1557 	/* Construct VLAN Table to give to HW */
1558 	for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1559 		vids[num++] = cpu_to_le16(i);
1560 
1561 	status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0);
1562 	if (status) {
1563 		dev_err(dev, "Setting HW VLAN filtering failed\n");
1564 		/* Set to VLAN promisc mode as setting VLAN filter failed */
1565 		if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS ||
1566 		    addl_status(status) ==
1567 				MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1568 			return be_set_vlan_promisc(adapter);
1569 	}
1570 	return status;
1571 }
1572 
1573 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1574 {
1575 	struct be_adapter *adapter = netdev_priv(netdev);
1576 	int status = 0;
1577 
1578 	mutex_lock(&adapter->rx_filter_lock);
1579 
1580 	/* Packets with VID 0 are always received by Lancer by default */
1581 	if (lancer_chip(adapter) && vid == 0)
1582 		goto done;
1583 
1584 	if (test_bit(vid, adapter->vids))
1585 		goto done;
1586 
1587 	set_bit(vid, adapter->vids);
1588 	adapter->vlans_added++;
1589 
1590 	status = be_vid_config(adapter);
1591 done:
1592 	mutex_unlock(&adapter->rx_filter_lock);
1593 	return status;
1594 }
1595 
1596 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1597 {
1598 	struct be_adapter *adapter = netdev_priv(netdev);
1599 	int status = 0;
1600 
1601 	mutex_lock(&adapter->rx_filter_lock);
1602 
1603 	/* Packets with VID 0 are always received by Lancer by default */
1604 	if (lancer_chip(adapter) && vid == 0)
1605 		goto done;
1606 
1607 	if (!test_bit(vid, adapter->vids))
1608 		goto done;
1609 
1610 	clear_bit(vid, adapter->vids);
1611 	adapter->vlans_added--;
1612 
1613 	status = be_vid_config(adapter);
1614 done:
1615 	mutex_unlock(&adapter->rx_filter_lock);
1616 	return status;
1617 }
1618 
1619 static void be_set_all_promisc(struct be_adapter *adapter)
1620 {
1621 	be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON);
1622 	adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS;
1623 }
1624 
1625 static void be_set_mc_promisc(struct be_adapter *adapter)
1626 {
1627 	int status;
1628 
1629 	if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS)
1630 		return;
1631 
1632 	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON);
1633 	if (!status)
1634 		adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS;
1635 }
1636 
1637 static void be_set_uc_promisc(struct be_adapter *adapter)
1638 {
1639 	int status;
1640 
1641 	if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)
1642 		return;
1643 
1644 	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON);
1645 	if (!status)
1646 		adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS;
1647 }
1648 
1649 static void be_clear_uc_promisc(struct be_adapter *adapter)
1650 {
1651 	int status;
1652 
1653 	if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS))
1654 		return;
1655 
1656 	status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF);
1657 	if (!status)
1658 		adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS;
1659 }
1660 
1661 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync().
1662  * We use a single callback function for both sync and unsync. We really don't
1663  * add/remove addresses through this callback. But, we use it to detect changes
1664  * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode().
1665  */
1666 static int be_uc_list_update(struct net_device *netdev,
1667 			     const unsigned char *addr)
1668 {
1669 	struct be_adapter *adapter = netdev_priv(netdev);
1670 
1671 	adapter->update_uc_list = true;
1672 	return 0;
1673 }
1674 
1675 static int be_mc_list_update(struct net_device *netdev,
1676 			     const unsigned char *addr)
1677 {
1678 	struct be_adapter *adapter = netdev_priv(netdev);
1679 
1680 	adapter->update_mc_list = true;
1681 	return 0;
1682 }
1683 
1684 static void be_set_mc_list(struct be_adapter *adapter)
1685 {
1686 	struct net_device *netdev = adapter->netdev;
1687 	struct netdev_hw_addr *ha;
1688 	bool mc_promisc = false;
1689 	int status;
1690 
1691 	netif_addr_lock_bh(netdev);
1692 	__dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update);
1693 
1694 	if (netdev->flags & IFF_PROMISC) {
1695 		adapter->update_mc_list = false;
1696 	} else if (netdev->flags & IFF_ALLMULTI ||
1697 		   netdev_mc_count(netdev) > be_max_mc(adapter)) {
1698 		/* Enable multicast promisc if num configured exceeds
1699 		 * what we support
1700 		 */
1701 		mc_promisc = true;
1702 		adapter->update_mc_list = false;
1703 	} else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) {
1704 		/* Update mc-list unconditionally if the iface was previously
1705 		 * in mc-promisc mode and now is out of that mode.
1706 		 */
1707 		adapter->update_mc_list = true;
1708 	}
1709 
1710 	if (adapter->update_mc_list) {
1711 		int i = 0;
1712 
1713 		/* cache the mc-list in adapter */
1714 		netdev_for_each_mc_addr(ha, netdev) {
1715 			ether_addr_copy(adapter->mc_list[i].mac, ha->addr);
1716 			i++;
1717 		}
1718 		adapter->mc_count = netdev_mc_count(netdev);
1719 	}
1720 	netif_addr_unlock_bh(netdev);
1721 
1722 	if (mc_promisc) {
1723 		be_set_mc_promisc(adapter);
1724 	} else if (adapter->update_mc_list) {
1725 		status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON);
1726 		if (!status)
1727 			adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS;
1728 		else
1729 			be_set_mc_promisc(adapter);
1730 
1731 		adapter->update_mc_list = false;
1732 	}
1733 }
1734 
1735 static void be_clear_mc_list(struct be_adapter *adapter)
1736 {
1737 	struct net_device *netdev = adapter->netdev;
1738 
1739 	__dev_mc_unsync(netdev, NULL);
1740 	be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF);
1741 	adapter->mc_count = 0;
1742 }
1743 
1744 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx)
1745 {
1746 	if (ether_addr_equal(adapter->uc_list[uc_idx].mac, adapter->dev_mac)) {
1747 		adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0];
1748 		return 0;
1749 	}
1750 
1751 	return be_cmd_pmac_add(adapter, adapter->uc_list[uc_idx].mac,
1752 			       adapter->if_handle,
1753 			       &adapter->pmac_id[uc_idx + 1], 0);
1754 }
1755 
1756 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id)
1757 {
1758 	if (pmac_id == adapter->pmac_id[0])
1759 		return;
1760 
1761 	be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0);
1762 }
1763 
1764 static void be_set_uc_list(struct be_adapter *adapter)
1765 {
1766 	struct net_device *netdev = adapter->netdev;
1767 	struct netdev_hw_addr *ha;
1768 	bool uc_promisc = false;
1769 	int curr_uc_macs = 0, i;
1770 
1771 	netif_addr_lock_bh(netdev);
1772 	__dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update);
1773 
1774 	if (netdev->flags & IFF_PROMISC) {
1775 		adapter->update_uc_list = false;
1776 	} else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) {
1777 		uc_promisc = true;
1778 		adapter->update_uc_list = false;
1779 	}  else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) {
1780 		/* Update uc-list unconditionally if the iface was previously
1781 		 * in uc-promisc mode and now is out of that mode.
1782 		 */
1783 		adapter->update_uc_list = true;
1784 	}
1785 
1786 	if (adapter->update_uc_list) {
1787 		/* cache the uc-list in adapter array */
1788 		i = 0;
1789 		netdev_for_each_uc_addr(ha, netdev) {
1790 			ether_addr_copy(adapter->uc_list[i].mac, ha->addr);
1791 			i++;
1792 		}
1793 		curr_uc_macs = netdev_uc_count(netdev);
1794 	}
1795 	netif_addr_unlock_bh(netdev);
1796 
1797 	if (uc_promisc) {
1798 		be_set_uc_promisc(adapter);
1799 	} else if (adapter->update_uc_list) {
1800 		be_clear_uc_promisc(adapter);
1801 
1802 		for (i = 0; i < adapter->uc_macs; i++)
1803 			be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
1804 
1805 		for (i = 0; i < curr_uc_macs; i++)
1806 			be_uc_mac_add(adapter, i);
1807 		adapter->uc_macs = curr_uc_macs;
1808 		adapter->update_uc_list = false;
1809 	}
1810 }
1811 
1812 static void be_clear_uc_list(struct be_adapter *adapter)
1813 {
1814 	struct net_device *netdev = adapter->netdev;
1815 	int i;
1816 
1817 	__dev_uc_unsync(netdev, NULL);
1818 	for (i = 0; i < adapter->uc_macs; i++)
1819 		be_uc_mac_del(adapter, adapter->pmac_id[i + 1]);
1820 
1821 	adapter->uc_macs = 0;
1822 }
1823 
1824 static void __be_set_rx_mode(struct be_adapter *adapter)
1825 {
1826 	struct net_device *netdev = adapter->netdev;
1827 
1828 	mutex_lock(&adapter->rx_filter_lock);
1829 
1830 	if (netdev->flags & IFF_PROMISC) {
1831 		if (!be_in_all_promisc(adapter))
1832 			be_set_all_promisc(adapter);
1833 	} else if (be_in_all_promisc(adapter)) {
1834 		/* We need to re-program the vlan-list or clear
1835 		 * vlan-promisc mode (if needed) when the interface
1836 		 * comes out of promisc mode.
1837 		 */
1838 		be_vid_config(adapter);
1839 	}
1840 
1841 	be_set_uc_list(adapter);
1842 	be_set_mc_list(adapter);
1843 
1844 	mutex_unlock(&adapter->rx_filter_lock);
1845 }
1846 
1847 static void be_work_set_rx_mode(struct work_struct *work)
1848 {
1849 	struct be_cmd_work *cmd_work =
1850 				container_of(work, struct be_cmd_work, work);
1851 
1852 	__be_set_rx_mode(cmd_work->adapter);
1853 	kfree(cmd_work);
1854 }
1855 
1856 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1857 {
1858 	struct be_adapter *adapter = netdev_priv(netdev);
1859 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1860 	int status;
1861 
1862 	if (!sriov_enabled(adapter))
1863 		return -EPERM;
1864 
1865 	if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1866 		return -EINVAL;
1867 
1868 	/* Proceed further only if user provided MAC is different
1869 	 * from active MAC
1870 	 */
1871 	if (ether_addr_equal(mac, vf_cfg->mac_addr))
1872 		return 0;
1873 
1874 	if (BEx_chip(adapter)) {
1875 		be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1876 				vf + 1);
1877 
1878 		status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1879 					 &vf_cfg->pmac_id, vf + 1);
1880 	} else {
1881 		status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1882 					vf + 1);
1883 	}
1884 
1885 	if (status) {
1886 		dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1887 			mac, vf, status);
1888 		return be_cmd_status(status);
1889 	}
1890 
1891 	ether_addr_copy(vf_cfg->mac_addr, mac);
1892 
1893 	return 0;
1894 }
1895 
1896 static int be_get_vf_config(struct net_device *netdev, int vf,
1897 			    struct ifla_vf_info *vi)
1898 {
1899 	struct be_adapter *adapter = netdev_priv(netdev);
1900 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1901 
1902 	if (!sriov_enabled(adapter))
1903 		return -EPERM;
1904 
1905 	if (vf >= adapter->num_vfs)
1906 		return -EINVAL;
1907 
1908 	vi->vf = vf;
1909 	vi->max_tx_rate = vf_cfg->tx_rate;
1910 	vi->min_tx_rate = 0;
1911 	vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1912 	vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1913 	memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1914 	vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1915 	vi->spoofchk = adapter->vf_cfg[vf].spoofchk;
1916 
1917 	return 0;
1918 }
1919 
1920 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan)
1921 {
1922 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1923 	u16 vids[BE_NUM_VLANS_SUPPORTED];
1924 	int vf_if_id = vf_cfg->if_handle;
1925 	int status;
1926 
1927 	/* Enable Transparent VLAN Tagging */
1928 	status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0);
1929 	if (status)
1930 		return status;
1931 
1932 	/* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */
1933 	vids[0] = 0;
1934 	status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1);
1935 	if (!status)
1936 		dev_info(&adapter->pdev->dev,
1937 			 "Cleared guest VLANs on VF%d", vf);
1938 
1939 	/* After TVT is enabled, disallow VFs to program VLAN filters */
1940 	if (vf_cfg->privileges & BE_PRIV_FILTMGMT) {
1941 		status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges &
1942 						  ~BE_PRIV_FILTMGMT, vf + 1);
1943 		if (!status)
1944 			vf_cfg->privileges &= ~BE_PRIV_FILTMGMT;
1945 	}
1946 	return 0;
1947 }
1948 
1949 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf)
1950 {
1951 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1952 	struct device *dev = &adapter->pdev->dev;
1953 	int status;
1954 
1955 	/* Reset Transparent VLAN Tagging. */
1956 	status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1,
1957 				       vf_cfg->if_handle, 0, 0);
1958 	if (status)
1959 		return status;
1960 
1961 	/* Allow VFs to program VLAN filtering */
1962 	if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
1963 		status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges |
1964 						  BE_PRIV_FILTMGMT, vf + 1);
1965 		if (!status) {
1966 			vf_cfg->privileges |= BE_PRIV_FILTMGMT;
1967 			dev_info(dev, "VF%d: FILTMGMT priv enabled", vf);
1968 		}
1969 	}
1970 
1971 	dev_info(dev,
1972 		 "Disable/re-enable i/f in VM to clear Transparent VLAN tag");
1973 	return 0;
1974 }
1975 
1976 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
1977 			  __be16 vlan_proto)
1978 {
1979 	struct be_adapter *adapter = netdev_priv(netdev);
1980 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1981 	int status;
1982 
1983 	if (!sriov_enabled(adapter))
1984 		return -EPERM;
1985 
1986 	if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1987 		return -EINVAL;
1988 
1989 	if (vlan_proto != htons(ETH_P_8021Q))
1990 		return -EPROTONOSUPPORT;
1991 
1992 	if (vlan || qos) {
1993 		vlan |= qos << VLAN_PRIO_SHIFT;
1994 		status = be_set_vf_tvt(adapter, vf, vlan);
1995 	} else {
1996 		status = be_clear_vf_tvt(adapter, vf);
1997 	}
1998 
1999 	if (status) {
2000 		dev_err(&adapter->pdev->dev,
2001 			"VLAN %d config on VF %d failed : %#x\n", vlan, vf,
2002 			status);
2003 		return be_cmd_status(status);
2004 	}
2005 
2006 	vf_cfg->vlan_tag = vlan;
2007 	return 0;
2008 }
2009 
2010 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
2011 			     int min_tx_rate, int max_tx_rate)
2012 {
2013 	struct be_adapter *adapter = netdev_priv(netdev);
2014 	struct device *dev = &adapter->pdev->dev;
2015 	int percent_rate, status = 0;
2016 	u16 link_speed = 0;
2017 	u8 link_status;
2018 
2019 	if (!sriov_enabled(adapter))
2020 		return -EPERM;
2021 
2022 	if (vf >= adapter->num_vfs)
2023 		return -EINVAL;
2024 
2025 	if (min_tx_rate)
2026 		return -EINVAL;
2027 
2028 	if (!max_tx_rate)
2029 		goto config_qos;
2030 
2031 	status = be_cmd_link_status_query(adapter, &link_speed,
2032 					  &link_status, 0);
2033 	if (status)
2034 		goto err;
2035 
2036 	if (!link_status) {
2037 		dev_err(dev, "TX-rate setting not allowed when link is down\n");
2038 		status = -ENETDOWN;
2039 		goto err;
2040 	}
2041 
2042 	if (max_tx_rate < 100 || max_tx_rate > link_speed) {
2043 		dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
2044 			link_speed);
2045 		status = -EINVAL;
2046 		goto err;
2047 	}
2048 
2049 	/* On Skyhawk the QOS setting must be done only as a % value */
2050 	percent_rate = link_speed / 100;
2051 	if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
2052 		dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
2053 			percent_rate);
2054 		status = -EINVAL;
2055 		goto err;
2056 	}
2057 
2058 config_qos:
2059 	status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
2060 	if (status)
2061 		goto err;
2062 
2063 	adapter->vf_cfg[vf].tx_rate = max_tx_rate;
2064 	return 0;
2065 
2066 err:
2067 	dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
2068 		max_tx_rate, vf);
2069 	return be_cmd_status(status);
2070 }
2071 
2072 static int be_set_vf_link_state(struct net_device *netdev, int vf,
2073 				int link_state)
2074 {
2075 	struct be_adapter *adapter = netdev_priv(netdev);
2076 	int status;
2077 
2078 	if (!sriov_enabled(adapter))
2079 		return -EPERM;
2080 
2081 	if (vf >= adapter->num_vfs)
2082 		return -EINVAL;
2083 
2084 	status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
2085 	if (status) {
2086 		dev_err(&adapter->pdev->dev,
2087 			"Link state change on VF %d failed: %#x\n", vf, status);
2088 		return be_cmd_status(status);
2089 	}
2090 
2091 	adapter->vf_cfg[vf].plink_tracking = link_state;
2092 
2093 	return 0;
2094 }
2095 
2096 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)
2097 {
2098 	struct be_adapter *adapter = netdev_priv(netdev);
2099 	struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
2100 	u8 spoofchk;
2101 	int status;
2102 
2103 	if (!sriov_enabled(adapter))
2104 		return -EPERM;
2105 
2106 	if (vf >= adapter->num_vfs)
2107 		return -EINVAL;
2108 
2109 	if (BEx_chip(adapter))
2110 		return -EOPNOTSUPP;
2111 
2112 	if (enable == vf_cfg->spoofchk)
2113 		return 0;
2114 
2115 	spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK;
2116 
2117 	status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle,
2118 				       0, spoofchk);
2119 	if (status) {
2120 		dev_err(&adapter->pdev->dev,
2121 			"Spoofchk change on VF %d failed: %#x\n", vf, status);
2122 		return be_cmd_status(status);
2123 	}
2124 
2125 	vf_cfg->spoofchk = enable;
2126 	return 0;
2127 }
2128 
2129 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
2130 			  ulong now)
2131 {
2132 	aic->rx_pkts_prev = rx_pkts;
2133 	aic->tx_reqs_prev = tx_pkts;
2134 	aic->jiffies = now;
2135 }
2136 
2137 static int be_get_new_eqd(struct be_eq_obj *eqo)
2138 {
2139 	struct be_adapter *adapter = eqo->adapter;
2140 	int eqd, start;
2141 	struct be_aic_obj *aic;
2142 	struct be_rx_obj *rxo;
2143 	struct be_tx_obj *txo;
2144 	u64 rx_pkts = 0, tx_pkts = 0;
2145 	ulong now;
2146 	u32 pps, delta;
2147 	int i;
2148 
2149 	aic = &adapter->aic_obj[eqo->idx];
2150 	if (!adapter->aic_enabled) {
2151 		if (aic->jiffies)
2152 			aic->jiffies = 0;
2153 		eqd = aic->et_eqd;
2154 		return eqd;
2155 	}
2156 
2157 	for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2158 		do {
2159 			start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
2160 			rx_pkts += rxo->stats.rx_pkts;
2161 		} while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
2162 	}
2163 
2164 	for_all_tx_queues_on_eq(adapter, eqo, txo, i) {
2165 		do {
2166 			start = u64_stats_fetch_begin_irq(&txo->stats.sync);
2167 			tx_pkts += txo->stats.tx_reqs;
2168 		} while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
2169 	}
2170 
2171 	/* Skip, if wrapped around or first calculation */
2172 	now = jiffies;
2173 	if (!aic->jiffies || time_before(now, aic->jiffies) ||
2174 	    rx_pkts < aic->rx_pkts_prev ||
2175 	    tx_pkts < aic->tx_reqs_prev) {
2176 		be_aic_update(aic, rx_pkts, tx_pkts, now);
2177 		return aic->prev_eqd;
2178 	}
2179 
2180 	delta = jiffies_to_msecs(now - aic->jiffies);
2181 	if (delta == 0)
2182 		return aic->prev_eqd;
2183 
2184 	pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
2185 		(((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
2186 	eqd = (pps / 15000) << 2;
2187 
2188 	if (eqd < 8)
2189 		eqd = 0;
2190 	eqd = min_t(u32, eqd, aic->max_eqd);
2191 	eqd = max_t(u32, eqd, aic->min_eqd);
2192 
2193 	be_aic_update(aic, rx_pkts, tx_pkts, now);
2194 
2195 	return eqd;
2196 }
2197 
2198 /* For Skyhawk-R only */
2199 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo)
2200 {
2201 	struct be_adapter *adapter = eqo->adapter;
2202 	struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx];
2203 	ulong now = jiffies;
2204 	int eqd;
2205 	u32 mult_enc;
2206 
2207 	if (!adapter->aic_enabled)
2208 		return 0;
2209 
2210 	if (jiffies_to_msecs(now - aic->jiffies) < 1)
2211 		eqd = aic->prev_eqd;
2212 	else
2213 		eqd = be_get_new_eqd(eqo);
2214 
2215 	if (eqd > 100)
2216 		mult_enc = R2I_DLY_ENC_1;
2217 	else if (eqd > 60)
2218 		mult_enc = R2I_DLY_ENC_2;
2219 	else if (eqd > 20)
2220 		mult_enc = R2I_DLY_ENC_3;
2221 	else
2222 		mult_enc = R2I_DLY_ENC_0;
2223 
2224 	aic->prev_eqd = eqd;
2225 
2226 	return mult_enc;
2227 }
2228 
2229 void be_eqd_update(struct be_adapter *adapter, bool force_update)
2230 {
2231 	struct be_set_eqd set_eqd[MAX_EVT_QS];
2232 	struct be_aic_obj *aic;
2233 	struct be_eq_obj *eqo;
2234 	int i, num = 0, eqd;
2235 
2236 	for_all_evt_queues(adapter, eqo, i) {
2237 		aic = &adapter->aic_obj[eqo->idx];
2238 		eqd = be_get_new_eqd(eqo);
2239 		if (force_update || eqd != aic->prev_eqd) {
2240 			set_eqd[num].delay_multiplier = (eqd * 65)/100;
2241 			set_eqd[num].eq_id = eqo->q.id;
2242 			aic->prev_eqd = eqd;
2243 			num++;
2244 		}
2245 	}
2246 
2247 	if (num)
2248 		be_cmd_modify_eqd(adapter, set_eqd, num);
2249 }
2250 
2251 static void be_rx_stats_update(struct be_rx_obj *rxo,
2252 			       struct be_rx_compl_info *rxcp)
2253 {
2254 	struct be_rx_stats *stats = rx_stats(rxo);
2255 
2256 	u64_stats_update_begin(&stats->sync);
2257 	stats->rx_compl++;
2258 	stats->rx_bytes += rxcp->pkt_size;
2259 	stats->rx_pkts++;
2260 	if (rxcp->tunneled)
2261 		stats->rx_vxlan_offload_pkts++;
2262 	if (rxcp->pkt_type == BE_MULTICAST_PACKET)
2263 		stats->rx_mcast_pkts++;
2264 	if (rxcp->err)
2265 		stats->rx_compl_err++;
2266 	u64_stats_update_end(&stats->sync);
2267 }
2268 
2269 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
2270 {
2271 	/* L4 checksum is not reliable for non TCP/UDP packets.
2272 	 * Also ignore ipcksm for ipv6 pkts
2273 	 */
2274 	return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
2275 		(rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
2276 }
2277 
2278 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
2279 {
2280 	struct be_adapter *adapter = rxo->adapter;
2281 	struct be_rx_page_info *rx_page_info;
2282 	struct be_queue_info *rxq = &rxo->q;
2283 	u32 frag_idx = rxq->tail;
2284 
2285 	rx_page_info = &rxo->page_info_tbl[frag_idx];
2286 	BUG_ON(!rx_page_info->page);
2287 
2288 	if (rx_page_info->last_frag) {
2289 		dma_unmap_page(&adapter->pdev->dev,
2290 			       dma_unmap_addr(rx_page_info, bus),
2291 			       adapter->big_page_size, DMA_FROM_DEVICE);
2292 		rx_page_info->last_frag = false;
2293 	} else {
2294 		dma_sync_single_for_cpu(&adapter->pdev->dev,
2295 					dma_unmap_addr(rx_page_info, bus),
2296 					rx_frag_size, DMA_FROM_DEVICE);
2297 	}
2298 
2299 	queue_tail_inc(rxq);
2300 	atomic_dec(&rxq->used);
2301 	return rx_page_info;
2302 }
2303 
2304 /* Throwaway the data in the Rx completion */
2305 static void be_rx_compl_discard(struct be_rx_obj *rxo,
2306 				struct be_rx_compl_info *rxcp)
2307 {
2308 	struct be_rx_page_info *page_info;
2309 	u16 i, num_rcvd = rxcp->num_rcvd;
2310 
2311 	for (i = 0; i < num_rcvd; i++) {
2312 		page_info = get_rx_page_info(rxo);
2313 		put_page(page_info->page);
2314 		memset(page_info, 0, sizeof(*page_info));
2315 	}
2316 }
2317 
2318 /*
2319  * skb_fill_rx_data forms a complete skb for an ether frame
2320  * indicated by rxcp.
2321  */
2322 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
2323 			     struct be_rx_compl_info *rxcp)
2324 {
2325 	struct be_rx_page_info *page_info;
2326 	u16 i, j;
2327 	u16 hdr_len, curr_frag_len, remaining;
2328 	u8 *start;
2329 
2330 	page_info = get_rx_page_info(rxo);
2331 	start = page_address(page_info->page) + page_info->page_offset;
2332 	prefetch(start);
2333 
2334 	/* Copy data in the first descriptor of this completion */
2335 	curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
2336 
2337 	skb->len = curr_frag_len;
2338 	if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
2339 		memcpy(skb->data, start, curr_frag_len);
2340 		/* Complete packet has now been moved to data */
2341 		put_page(page_info->page);
2342 		skb->data_len = 0;
2343 		skb->tail += curr_frag_len;
2344 	} else {
2345 		hdr_len = ETH_HLEN;
2346 		memcpy(skb->data, start, hdr_len);
2347 		skb_shinfo(skb)->nr_frags = 1;
2348 		skb_frag_set_page(skb, 0, page_info->page);
2349 		skb_frag_off_set(&skb_shinfo(skb)->frags[0],
2350 				 page_info->page_offset + hdr_len);
2351 		skb_frag_size_set(&skb_shinfo(skb)->frags[0],
2352 				  curr_frag_len - hdr_len);
2353 		skb->data_len = curr_frag_len - hdr_len;
2354 		skb->truesize += rx_frag_size;
2355 		skb->tail += hdr_len;
2356 	}
2357 	page_info->page = NULL;
2358 
2359 	if (rxcp->pkt_size <= rx_frag_size) {
2360 		BUG_ON(rxcp->num_rcvd != 1);
2361 		return;
2362 	}
2363 
2364 	/* More frags present for this completion */
2365 	remaining = rxcp->pkt_size - curr_frag_len;
2366 	for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
2367 		page_info = get_rx_page_info(rxo);
2368 		curr_frag_len = min(remaining, rx_frag_size);
2369 
2370 		/* Coalesce all frags from the same physical page in one slot */
2371 		if (page_info->page_offset == 0) {
2372 			/* Fresh page */
2373 			j++;
2374 			skb_frag_set_page(skb, j, page_info->page);
2375 			skb_frag_off_set(&skb_shinfo(skb)->frags[j],
2376 					 page_info->page_offset);
2377 			skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
2378 			skb_shinfo(skb)->nr_frags++;
2379 		} else {
2380 			put_page(page_info->page);
2381 		}
2382 
2383 		skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
2384 		skb->len += curr_frag_len;
2385 		skb->data_len += curr_frag_len;
2386 		skb->truesize += rx_frag_size;
2387 		remaining -= curr_frag_len;
2388 		page_info->page = NULL;
2389 	}
2390 	BUG_ON(j > MAX_SKB_FRAGS);
2391 }
2392 
2393 /* Process the RX completion indicated by rxcp when GRO is disabled */
2394 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
2395 				struct be_rx_compl_info *rxcp)
2396 {
2397 	struct be_adapter *adapter = rxo->adapter;
2398 	struct net_device *netdev = adapter->netdev;
2399 	struct sk_buff *skb;
2400 
2401 	skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
2402 	if (unlikely(!skb)) {
2403 		rx_stats(rxo)->rx_drops_no_skbs++;
2404 		be_rx_compl_discard(rxo, rxcp);
2405 		return;
2406 	}
2407 
2408 	skb_fill_rx_data(rxo, skb, rxcp);
2409 
2410 	if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
2411 		skb->ip_summed = CHECKSUM_UNNECESSARY;
2412 	else
2413 		skb_checksum_none_assert(skb);
2414 
2415 	skb->protocol = eth_type_trans(skb, netdev);
2416 	skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
2417 	if (netdev->features & NETIF_F_RXHASH)
2418 		skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
2419 
2420 	skb->csum_level = rxcp->tunneled;
2421 	skb_mark_napi_id(skb, napi);
2422 
2423 	if (rxcp->vlanf)
2424 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
2425 
2426 	netif_receive_skb(skb);
2427 }
2428 
2429 /* Process the RX completion indicated by rxcp when GRO is enabled */
2430 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
2431 				    struct napi_struct *napi,
2432 				    struct be_rx_compl_info *rxcp)
2433 {
2434 	struct be_adapter *adapter = rxo->adapter;
2435 	struct be_rx_page_info *page_info;
2436 	struct sk_buff *skb = NULL;
2437 	u16 remaining, curr_frag_len;
2438 	u16 i, j;
2439 
2440 	skb = napi_get_frags(napi);
2441 	if (!skb) {
2442 		be_rx_compl_discard(rxo, rxcp);
2443 		return;
2444 	}
2445 
2446 	remaining = rxcp->pkt_size;
2447 	for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
2448 		page_info = get_rx_page_info(rxo);
2449 
2450 		curr_frag_len = min(remaining, rx_frag_size);
2451 
2452 		/* Coalesce all frags from the same physical page in one slot */
2453 		if (i == 0 || page_info->page_offset == 0) {
2454 			/* First frag or Fresh page */
2455 			j++;
2456 			skb_frag_set_page(skb, j, page_info->page);
2457 			skb_frag_off_set(&skb_shinfo(skb)->frags[j],
2458 					 page_info->page_offset);
2459 			skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
2460 		} else {
2461 			put_page(page_info->page);
2462 		}
2463 		skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
2464 		skb->truesize += rx_frag_size;
2465 		remaining -= curr_frag_len;
2466 		memset(page_info, 0, sizeof(*page_info));
2467 	}
2468 	BUG_ON(j > MAX_SKB_FRAGS);
2469 
2470 	skb_shinfo(skb)->nr_frags = j + 1;
2471 	skb->len = rxcp->pkt_size;
2472 	skb->data_len = rxcp->pkt_size;
2473 	skb->ip_summed = CHECKSUM_UNNECESSARY;
2474 	skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
2475 	if (adapter->netdev->features & NETIF_F_RXHASH)
2476 		skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
2477 
2478 	skb->csum_level = rxcp->tunneled;
2479 
2480 	if (rxcp->vlanf)
2481 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
2482 
2483 	napi_gro_frags(napi);
2484 }
2485 
2486 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
2487 				 struct be_rx_compl_info *rxcp)
2488 {
2489 	rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
2490 	rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
2491 	rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
2492 	rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
2493 	rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
2494 	rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
2495 	rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
2496 	rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
2497 	rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
2498 	rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
2499 	rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
2500 	if (rxcp->vlanf) {
2501 		rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
2502 		rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
2503 	}
2504 	rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
2505 	rxcp->tunneled =
2506 		GET_RX_COMPL_V1_BITS(tunneled, compl);
2507 }
2508 
2509 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
2510 				 struct be_rx_compl_info *rxcp)
2511 {
2512 	rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
2513 	rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
2514 	rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
2515 	rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
2516 	rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
2517 	rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
2518 	rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
2519 	rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
2520 	rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
2521 	rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
2522 	rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
2523 	if (rxcp->vlanf) {
2524 		rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
2525 		rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
2526 	}
2527 	rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
2528 	rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
2529 }
2530 
2531 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
2532 {
2533 	struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
2534 	struct be_rx_compl_info *rxcp = &rxo->rxcp;
2535 	struct be_adapter *adapter = rxo->adapter;
2536 
2537 	/* For checking the valid bit it is Ok to use either definition as the
2538 	 * valid bit is at the same position in both v0 and v1 Rx compl */
2539 	if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
2540 		return NULL;
2541 
2542 	rmb();
2543 	be_dws_le_to_cpu(compl, sizeof(*compl));
2544 
2545 	if (adapter->be3_native)
2546 		be_parse_rx_compl_v1(compl, rxcp);
2547 	else
2548 		be_parse_rx_compl_v0(compl, rxcp);
2549 
2550 	if (rxcp->ip_frag)
2551 		rxcp->l4_csum = 0;
2552 
2553 	if (rxcp->vlanf) {
2554 		/* In QNQ modes, if qnq bit is not set, then the packet was
2555 		 * tagged only with the transparent outer vlan-tag and must
2556 		 * not be treated as a vlan packet by host
2557 		 */
2558 		if (be_is_qnq_mode(adapter) && !rxcp->qnq)
2559 			rxcp->vlanf = 0;
2560 
2561 		if (!lancer_chip(adapter))
2562 			rxcp->vlan_tag = swab16(rxcp->vlan_tag);
2563 
2564 		if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
2565 		    !test_bit(rxcp->vlan_tag, adapter->vids))
2566 			rxcp->vlanf = 0;
2567 	}
2568 
2569 	/* As the compl has been parsed, reset it; we wont touch it again */
2570 	compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
2571 
2572 	queue_tail_inc(&rxo->cq);
2573 	return rxcp;
2574 }
2575 
2576 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
2577 {
2578 	u32 order = get_order(size);
2579 
2580 	if (order > 0)
2581 		gfp |= __GFP_COMP;
2582 	return  alloc_pages(gfp, order);
2583 }
2584 
2585 /*
2586  * Allocate a page, split it to fragments of size rx_frag_size and post as
2587  * receive buffers to BE
2588  */
2589 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
2590 {
2591 	struct be_adapter *adapter = rxo->adapter;
2592 	struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
2593 	struct be_queue_info *rxq = &rxo->q;
2594 	struct page *pagep = NULL;
2595 	struct device *dev = &adapter->pdev->dev;
2596 	struct be_eth_rx_d *rxd;
2597 	u64 page_dmaaddr = 0, frag_dmaaddr;
2598 	u32 posted, page_offset = 0, notify = 0;
2599 
2600 	page_info = &rxo->page_info_tbl[rxq->head];
2601 	for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
2602 		if (!pagep) {
2603 			pagep = be_alloc_pages(adapter->big_page_size, gfp);
2604 			if (unlikely(!pagep)) {
2605 				rx_stats(rxo)->rx_post_fail++;
2606 				break;
2607 			}
2608 			page_dmaaddr = dma_map_page(dev, pagep, 0,
2609 						    adapter->big_page_size,
2610 						    DMA_FROM_DEVICE);
2611 			if (dma_mapping_error(dev, page_dmaaddr)) {
2612 				put_page(pagep);
2613 				pagep = NULL;
2614 				adapter->drv_stats.dma_map_errors++;
2615 				break;
2616 			}
2617 			page_offset = 0;
2618 		} else {
2619 			get_page(pagep);
2620 			page_offset += rx_frag_size;
2621 		}
2622 		page_info->page_offset = page_offset;
2623 		page_info->page = pagep;
2624 
2625 		rxd = queue_head_node(rxq);
2626 		frag_dmaaddr = page_dmaaddr + page_info->page_offset;
2627 		rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
2628 		rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
2629 
2630 		/* Any space left in the current big page for another frag? */
2631 		if ((page_offset + rx_frag_size + rx_frag_size) >
2632 					adapter->big_page_size) {
2633 			pagep = NULL;
2634 			page_info->last_frag = true;
2635 			dma_unmap_addr_set(page_info, bus, page_dmaaddr);
2636 		} else {
2637 			dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
2638 		}
2639 
2640 		prev_page_info = page_info;
2641 		queue_head_inc(rxq);
2642 		page_info = &rxo->page_info_tbl[rxq->head];
2643 	}
2644 
2645 	/* Mark the last frag of a page when we break out of the above loop
2646 	 * with no more slots available in the RXQ
2647 	 */
2648 	if (pagep) {
2649 		prev_page_info->last_frag = true;
2650 		dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
2651 	}
2652 
2653 	if (posted) {
2654 		atomic_add(posted, &rxq->used);
2655 		if (rxo->rx_post_starved)
2656 			rxo->rx_post_starved = false;
2657 		do {
2658 			notify = min(MAX_NUM_POST_ERX_DB, posted);
2659 			be_rxq_notify(adapter, rxq->id, notify);
2660 			posted -= notify;
2661 		} while (posted);
2662 	} else if (atomic_read(&rxq->used) == 0) {
2663 		/* Let be_worker replenish when memory is available */
2664 		rxo->rx_post_starved = true;
2665 	}
2666 }
2667 
2668 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status)
2669 {
2670 	switch (status) {
2671 	case BE_TX_COMP_HDR_PARSE_ERR:
2672 		tx_stats(txo)->tx_hdr_parse_err++;
2673 		break;
2674 	case BE_TX_COMP_NDMA_ERR:
2675 		tx_stats(txo)->tx_dma_err++;
2676 		break;
2677 	case BE_TX_COMP_ACL_ERR:
2678 		tx_stats(txo)->tx_spoof_check_err++;
2679 		break;
2680 	}
2681 }
2682 
2683 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status)
2684 {
2685 	switch (status) {
2686 	case LANCER_TX_COMP_LSO_ERR:
2687 		tx_stats(txo)->tx_tso_err++;
2688 		break;
2689 	case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2690 	case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2691 		tx_stats(txo)->tx_spoof_check_err++;
2692 		break;
2693 	case LANCER_TX_COMP_QINQ_ERR:
2694 		tx_stats(txo)->tx_qinq_err++;
2695 		break;
2696 	case LANCER_TX_COMP_PARITY_ERR:
2697 		tx_stats(txo)->tx_internal_parity_err++;
2698 		break;
2699 	case LANCER_TX_COMP_DMA_ERR:
2700 		tx_stats(txo)->tx_dma_err++;
2701 		break;
2702 	case LANCER_TX_COMP_SGE_ERR:
2703 		tx_stats(txo)->tx_sge_err++;
2704 		break;
2705 	}
2706 }
2707 
2708 static struct be_tx_compl_info *be_tx_compl_get(struct be_adapter *adapter,
2709 						struct be_tx_obj *txo)
2710 {
2711 	struct be_queue_info *tx_cq = &txo->cq;
2712 	struct be_tx_compl_info *txcp = &txo->txcp;
2713 	struct be_eth_tx_compl *compl = queue_tail_node(tx_cq);
2714 
2715 	if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
2716 		return NULL;
2717 
2718 	/* Ensure load ordering of valid bit dword and other dwords below */
2719 	rmb();
2720 	be_dws_le_to_cpu(compl, sizeof(*compl));
2721 
2722 	txcp->status = GET_TX_COMPL_BITS(status, compl);
2723 	txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl);
2724 
2725 	if (txcp->status) {
2726 		if (lancer_chip(adapter)) {
2727 			lancer_update_tx_err(txo, txcp->status);
2728 			/* Reset the adapter incase of TSO,
2729 			 * SGE or Parity error
2730 			 */
2731 			if (txcp->status == LANCER_TX_COMP_LSO_ERR ||
2732 			    txcp->status == LANCER_TX_COMP_PARITY_ERR ||
2733 			    txcp->status == LANCER_TX_COMP_SGE_ERR)
2734 				be_set_error(adapter, BE_ERROR_TX);
2735 		} else {
2736 			be_update_tx_err(txo, txcp->status);
2737 		}
2738 	}
2739 
2740 	if (be_check_error(adapter, BE_ERROR_TX))
2741 		return NULL;
2742 
2743 	compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
2744 	queue_tail_inc(tx_cq);
2745 	return txcp;
2746 }
2747 
2748 static u16 be_tx_compl_process(struct be_adapter *adapter,
2749 			       struct be_tx_obj *txo, u16 last_index)
2750 {
2751 	struct sk_buff **sent_skbs = txo->sent_skb_list;
2752 	struct be_queue_info *txq = &txo->q;
2753 	struct sk_buff *skb = NULL;
2754 	bool unmap_skb_hdr = false;
2755 	struct be_eth_wrb *wrb;
2756 	u16 num_wrbs = 0;
2757 	u32 frag_index;
2758 
2759 	do {
2760 		if (sent_skbs[txq->tail]) {
2761 			/* Free skb from prev req */
2762 			if (skb)
2763 				dev_consume_skb_any(skb);
2764 			skb = sent_skbs[txq->tail];
2765 			sent_skbs[txq->tail] = NULL;
2766 			queue_tail_inc(txq);  /* skip hdr wrb */
2767 			num_wrbs++;
2768 			unmap_skb_hdr = true;
2769 		}
2770 		wrb = queue_tail_node(txq);
2771 		frag_index = txq->tail;
2772 		unmap_tx_frag(&adapter->pdev->dev, wrb,
2773 			      (unmap_skb_hdr && skb_headlen(skb)));
2774 		unmap_skb_hdr = false;
2775 		queue_tail_inc(txq);
2776 		num_wrbs++;
2777 	} while (frag_index != last_index);
2778 	dev_consume_skb_any(skb);
2779 
2780 	return num_wrbs;
2781 }
2782 
2783 /* Return the number of events in the event queue */
2784 static inline int events_get(struct be_eq_obj *eqo)
2785 {
2786 	struct be_eq_entry *eqe;
2787 	int num = 0;
2788 
2789 	do {
2790 		eqe = queue_tail_node(&eqo->q);
2791 		if (eqe->evt == 0)
2792 			break;
2793 
2794 		rmb();
2795 		eqe->evt = 0;
2796 		num++;
2797 		queue_tail_inc(&eqo->q);
2798 	} while (true);
2799 
2800 	return num;
2801 }
2802 
2803 /* Leaves the EQ is disarmed state */
2804 static void be_eq_clean(struct be_eq_obj *eqo)
2805 {
2806 	int num = events_get(eqo);
2807 
2808 	be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0);
2809 }
2810 
2811 /* Free posted rx buffers that were not used */
2812 static void be_rxq_clean(struct be_rx_obj *rxo)
2813 {
2814 	struct be_queue_info *rxq = &rxo->q;
2815 	struct be_rx_page_info *page_info;
2816 
2817 	while (atomic_read(&rxq->used) > 0) {
2818 		page_info = get_rx_page_info(rxo);
2819 		put_page(page_info->page);
2820 		memset(page_info, 0, sizeof(*page_info));
2821 	}
2822 	BUG_ON(atomic_read(&rxq->used));
2823 	rxq->tail = 0;
2824 	rxq->head = 0;
2825 }
2826 
2827 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2828 {
2829 	struct be_queue_info *rx_cq = &rxo->cq;
2830 	struct be_rx_compl_info *rxcp;
2831 	struct be_adapter *adapter = rxo->adapter;
2832 	int flush_wait = 0;
2833 
2834 	/* Consume pending rx completions.
2835 	 * Wait for the flush completion (identified by zero num_rcvd)
2836 	 * to arrive. Notify CQ even when there are no more CQ entries
2837 	 * for HW to flush partially coalesced CQ entries.
2838 	 * In Lancer, there is no need to wait for flush compl.
2839 	 */
2840 	for (;;) {
2841 		rxcp = be_rx_compl_get(rxo);
2842 		if (!rxcp) {
2843 			if (lancer_chip(adapter))
2844 				break;
2845 
2846 			if (flush_wait++ > 50 ||
2847 			    be_check_error(adapter,
2848 					   BE_ERROR_HW)) {
2849 				dev_warn(&adapter->pdev->dev,
2850 					 "did not receive flush compl\n");
2851 				break;
2852 			}
2853 			be_cq_notify(adapter, rx_cq->id, true, 0);
2854 			mdelay(1);
2855 		} else {
2856 			be_rx_compl_discard(rxo, rxcp);
2857 			be_cq_notify(adapter, rx_cq->id, false, 1);
2858 			if (rxcp->num_rcvd == 0)
2859 				break;
2860 		}
2861 	}
2862 
2863 	/* After cleanup, leave the CQ in unarmed state */
2864 	be_cq_notify(adapter, rx_cq->id, false, 0);
2865 }
2866 
2867 static void be_tx_compl_clean(struct be_adapter *adapter)
2868 {
2869 	struct device *dev = &adapter->pdev->dev;
2870 	u16 cmpl = 0, timeo = 0, num_wrbs = 0;
2871 	struct be_tx_compl_info *txcp;
2872 	struct be_queue_info *txq;
2873 	u32 end_idx, notified_idx;
2874 	struct be_tx_obj *txo;
2875 	int i, pending_txqs;
2876 
2877 	/* Stop polling for compls when HW has been silent for 10ms */
2878 	do {
2879 		pending_txqs = adapter->num_tx_qs;
2880 
2881 		for_all_tx_queues(adapter, txo, i) {
2882 			cmpl = 0;
2883 			num_wrbs = 0;
2884 			txq = &txo->q;
2885 			while ((txcp = be_tx_compl_get(adapter, txo))) {
2886 				num_wrbs +=
2887 					be_tx_compl_process(adapter, txo,
2888 							    txcp->end_index);
2889 				cmpl++;
2890 			}
2891 			if (cmpl) {
2892 				be_cq_notify(adapter, txo->cq.id, false, cmpl);
2893 				atomic_sub(num_wrbs, &txq->used);
2894 				timeo = 0;
2895 			}
2896 			if (!be_is_tx_compl_pending(txo))
2897 				pending_txqs--;
2898 		}
2899 
2900 		if (pending_txqs == 0 || ++timeo > 10 ||
2901 		    be_check_error(adapter, BE_ERROR_HW))
2902 			break;
2903 
2904 		mdelay(1);
2905 	} while (true);
2906 
2907 	/* Free enqueued TX that was never notified to HW */
2908 	for_all_tx_queues(adapter, txo, i) {
2909 		txq = &txo->q;
2910 
2911 		if (atomic_read(&txq->used)) {
2912 			dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n",
2913 				 i, atomic_read(&txq->used));
2914 			notified_idx = txq->tail;
2915 			end_idx = txq->tail;
2916 			index_adv(&end_idx, atomic_read(&txq->used) - 1,
2917 				  txq->len);
2918 			/* Use the tx-compl process logic to handle requests
2919 			 * that were not sent to the HW.
2920 			 */
2921 			num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2922 			atomic_sub(num_wrbs, &txq->used);
2923 			BUG_ON(atomic_read(&txq->used));
2924 			txo->pend_wrb_cnt = 0;
2925 			/* Since hw was never notified of these requests,
2926 			 * reset TXQ indices
2927 			 */
2928 			txq->head = notified_idx;
2929 			txq->tail = notified_idx;
2930 		}
2931 	}
2932 }
2933 
2934 static void be_evt_queues_destroy(struct be_adapter *adapter)
2935 {
2936 	struct be_eq_obj *eqo;
2937 	int i;
2938 
2939 	for_all_evt_queues(adapter, eqo, i) {
2940 		if (eqo->q.created) {
2941 			be_eq_clean(eqo);
2942 			be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2943 			netif_napi_del(&eqo->napi);
2944 			free_cpumask_var(eqo->affinity_mask);
2945 		}
2946 		be_queue_free(adapter, &eqo->q);
2947 	}
2948 }
2949 
2950 static int be_evt_queues_create(struct be_adapter *adapter)
2951 {
2952 	struct be_queue_info *eq;
2953 	struct be_eq_obj *eqo;
2954 	struct be_aic_obj *aic;
2955 	int i, rc;
2956 
2957 	/* need enough EQs to service both RX and TX queues */
2958 	adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2959 				    max(adapter->cfg_num_rx_irqs,
2960 					adapter->cfg_num_tx_irqs));
2961 
2962 	adapter->aic_enabled = true;
2963 
2964 	for_all_evt_queues(adapter, eqo, i) {
2965 		int numa_node = dev_to_node(&adapter->pdev->dev);
2966 
2967 		aic = &adapter->aic_obj[i];
2968 		eqo->adapter = adapter;
2969 		eqo->idx = i;
2970 		aic->max_eqd = BE_MAX_EQD;
2971 
2972 		eq = &eqo->q;
2973 		rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2974 				    sizeof(struct be_eq_entry));
2975 		if (rc)
2976 			return rc;
2977 
2978 		rc = be_cmd_eq_create(adapter, eqo);
2979 		if (rc)
2980 			return rc;
2981 
2982 		if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL))
2983 			return -ENOMEM;
2984 		cpumask_set_cpu(cpumask_local_spread(i, numa_node),
2985 				eqo->affinity_mask);
2986 		netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2987 			       BE_NAPI_WEIGHT);
2988 	}
2989 	return 0;
2990 }
2991 
2992 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2993 {
2994 	struct be_queue_info *q;
2995 
2996 	q = &adapter->mcc_obj.q;
2997 	if (q->created)
2998 		be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2999 	be_queue_free(adapter, q);
3000 
3001 	q = &adapter->mcc_obj.cq;
3002 	if (q->created)
3003 		be_cmd_q_destroy(adapter, q, QTYPE_CQ);
3004 	be_queue_free(adapter, q);
3005 }
3006 
3007 /* Must be called only after TX qs are created as MCC shares TX EQ */
3008 static int be_mcc_queues_create(struct be_adapter *adapter)
3009 {
3010 	struct be_queue_info *q, *cq;
3011 
3012 	cq = &adapter->mcc_obj.cq;
3013 	if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
3014 			   sizeof(struct be_mcc_compl)))
3015 		goto err;
3016 
3017 	/* Use the default EQ for MCC completions */
3018 	if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
3019 		goto mcc_cq_free;
3020 
3021 	q = &adapter->mcc_obj.q;
3022 	if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3023 		goto mcc_cq_destroy;
3024 
3025 	if (be_cmd_mccq_create(adapter, q, cq))
3026 		goto mcc_q_free;
3027 
3028 	return 0;
3029 
3030 mcc_q_free:
3031 	be_queue_free(adapter, q);
3032 mcc_cq_destroy:
3033 	be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
3034 mcc_cq_free:
3035 	be_queue_free(adapter, cq);
3036 err:
3037 	return -1;
3038 }
3039 
3040 static void be_tx_queues_destroy(struct be_adapter *adapter)
3041 {
3042 	struct be_queue_info *q;
3043 	struct be_tx_obj *txo;
3044 	u8 i;
3045 
3046 	for_all_tx_queues(adapter, txo, i) {
3047 		q = &txo->q;
3048 		if (q->created)
3049 			be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
3050 		be_queue_free(adapter, q);
3051 
3052 		q = &txo->cq;
3053 		if (q->created)
3054 			be_cmd_q_destroy(adapter, q, QTYPE_CQ);
3055 		be_queue_free(adapter, q);
3056 	}
3057 }
3058 
3059 static int be_tx_qs_create(struct be_adapter *adapter)
3060 {
3061 	struct be_queue_info *cq;
3062 	struct be_tx_obj *txo;
3063 	struct be_eq_obj *eqo;
3064 	int status, i;
3065 
3066 	adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs);
3067 
3068 	for_all_tx_queues(adapter, txo, i) {
3069 		cq = &txo->cq;
3070 		status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
3071 					sizeof(struct be_eth_tx_compl));
3072 		if (status)
3073 			return status;
3074 
3075 		u64_stats_init(&txo->stats.sync);
3076 		u64_stats_init(&txo->stats.sync_compl);
3077 
3078 		/* If num_evt_qs is less than num_tx_qs, then more than
3079 		 * one txq share an eq
3080 		 */
3081 		eqo = &adapter->eq_obj[i % adapter->num_evt_qs];
3082 		status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3);
3083 		if (status)
3084 			return status;
3085 
3086 		status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
3087 					sizeof(struct be_eth_wrb));
3088 		if (status)
3089 			return status;
3090 
3091 		status = be_cmd_txq_create(adapter, txo);
3092 		if (status)
3093 			return status;
3094 
3095 		netif_set_xps_queue(adapter->netdev, eqo->affinity_mask,
3096 				    eqo->idx);
3097 	}
3098 
3099 	dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
3100 		 adapter->num_tx_qs);
3101 	return 0;
3102 }
3103 
3104 static void be_rx_cqs_destroy(struct be_adapter *adapter)
3105 {
3106 	struct be_queue_info *q;
3107 	struct be_rx_obj *rxo;
3108 	int i;
3109 
3110 	for_all_rx_queues(adapter, rxo, i) {
3111 		q = &rxo->cq;
3112 		if (q->created)
3113 			be_cmd_q_destroy(adapter, q, QTYPE_CQ);
3114 		be_queue_free(adapter, q);
3115 	}
3116 }
3117 
3118 static int be_rx_cqs_create(struct be_adapter *adapter)
3119 {
3120 	struct be_queue_info *eq, *cq;
3121 	struct be_rx_obj *rxo;
3122 	int rc, i;
3123 
3124 	adapter->num_rss_qs =
3125 			min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs);
3126 
3127 	/* We'll use RSS only if atleast 2 RSS rings are supported. */
3128 	if (adapter->num_rss_qs < 2)
3129 		adapter->num_rss_qs = 0;
3130 
3131 	adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq;
3132 
3133 	/* When the interface is not capable of RSS rings (and there is no
3134 	 * need to create a default RXQ) we'll still need one RXQ
3135 	 */
3136 	if (adapter->num_rx_qs == 0)
3137 		adapter->num_rx_qs = 1;
3138 
3139 	adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
3140 	for_all_rx_queues(adapter, rxo, i) {
3141 		rxo->adapter = adapter;
3142 		cq = &rxo->cq;
3143 		rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
3144 				    sizeof(struct be_eth_rx_compl));
3145 		if (rc)
3146 			return rc;
3147 
3148 		u64_stats_init(&rxo->stats.sync);
3149 		eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
3150 		rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
3151 		if (rc)
3152 			return rc;
3153 	}
3154 
3155 	dev_info(&adapter->pdev->dev,
3156 		 "created %d RX queue(s)\n", adapter->num_rx_qs);
3157 	return 0;
3158 }
3159 
3160 static irqreturn_t be_intx(int irq, void *dev)
3161 {
3162 	struct be_eq_obj *eqo = dev;
3163 	struct be_adapter *adapter = eqo->adapter;
3164 	int num_evts = 0;
3165 
3166 	/* IRQ is not expected when NAPI is scheduled as the EQ
3167 	 * will not be armed.
3168 	 * But, this can happen on Lancer INTx where it takes
3169 	 * a while to de-assert INTx or in BE2 where occasionaly
3170 	 * an interrupt may be raised even when EQ is unarmed.
3171 	 * If NAPI is already scheduled, then counting & notifying
3172 	 * events will orphan them.
3173 	 */
3174 	if (napi_schedule_prep(&eqo->napi)) {
3175 		num_evts = events_get(eqo);
3176 		__napi_schedule(&eqo->napi);
3177 		if (num_evts)
3178 			eqo->spurious_intr = 0;
3179 	}
3180 	be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0);
3181 
3182 	/* Return IRQ_HANDLED only for the the first spurious intr
3183 	 * after a valid intr to stop the kernel from branding
3184 	 * this irq as a bad one!
3185 	 */
3186 	if (num_evts || eqo->spurious_intr++ == 0)
3187 		return IRQ_HANDLED;
3188 	else
3189 		return IRQ_NONE;
3190 }
3191 
3192 static irqreturn_t be_msix(int irq, void *dev)
3193 {
3194 	struct be_eq_obj *eqo = dev;
3195 
3196 	be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
3197 	napi_schedule(&eqo->napi);
3198 	return IRQ_HANDLED;
3199 }
3200 
3201 static inline bool do_gro(struct be_rx_compl_info *rxcp)
3202 {
3203 	return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
3204 }
3205 
3206 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
3207 			 int budget)
3208 {
3209 	struct be_adapter *adapter = rxo->adapter;
3210 	struct be_queue_info *rx_cq = &rxo->cq;
3211 	struct be_rx_compl_info *rxcp;
3212 	u32 work_done;
3213 	u32 frags_consumed = 0;
3214 
3215 	for (work_done = 0; work_done < budget; work_done++) {
3216 		rxcp = be_rx_compl_get(rxo);
3217 		if (!rxcp)
3218 			break;
3219 
3220 		/* Is it a flush compl that has no data */
3221 		if (unlikely(rxcp->num_rcvd == 0))
3222 			goto loop_continue;
3223 
3224 		/* Discard compl with partial DMA Lancer B0 */
3225 		if (unlikely(!rxcp->pkt_size)) {
3226 			be_rx_compl_discard(rxo, rxcp);
3227 			goto loop_continue;
3228 		}
3229 
3230 		/* On BE drop pkts that arrive due to imperfect filtering in
3231 		 * promiscuous mode on some skews
3232 		 */
3233 		if (unlikely(rxcp->port != adapter->port_num &&
3234 			     !lancer_chip(adapter))) {
3235 			be_rx_compl_discard(rxo, rxcp);
3236 			goto loop_continue;
3237 		}
3238 
3239 		if (do_gro(rxcp))
3240 			be_rx_compl_process_gro(rxo, napi, rxcp);
3241 		else
3242 			be_rx_compl_process(rxo, napi, rxcp);
3243 
3244 loop_continue:
3245 		frags_consumed += rxcp->num_rcvd;
3246 		be_rx_stats_update(rxo, rxcp);
3247 	}
3248 
3249 	if (work_done) {
3250 		be_cq_notify(adapter, rx_cq->id, true, work_done);
3251 
3252 		/* When an rx-obj gets into post_starved state, just
3253 		 * let be_worker do the posting.
3254 		 */
3255 		if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
3256 		    !rxo->rx_post_starved)
3257 			be_post_rx_frags(rxo, GFP_ATOMIC,
3258 					 max_t(u32, MAX_RX_POST,
3259 					       frags_consumed));
3260 	}
3261 
3262 	return work_done;
3263 }
3264 
3265 
3266 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
3267 			  int idx)
3268 {
3269 	int num_wrbs = 0, work_done = 0;
3270 	struct be_tx_compl_info *txcp;
3271 
3272 	while ((txcp = be_tx_compl_get(adapter, txo))) {
3273 		num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index);
3274 		work_done++;
3275 	}
3276 
3277 	if (work_done) {
3278 		be_cq_notify(adapter, txo->cq.id, true, work_done);
3279 		atomic_sub(num_wrbs, &txo->q.used);
3280 
3281 		/* As Tx wrbs have been freed up, wake up netdev queue
3282 		 * if it was stopped due to lack of tx wrbs.  */
3283 		if (__netif_subqueue_stopped(adapter->netdev, idx) &&
3284 		    be_can_txq_wake(txo)) {
3285 			netif_wake_subqueue(adapter->netdev, idx);
3286 		}
3287 
3288 		u64_stats_update_begin(&tx_stats(txo)->sync_compl);
3289 		tx_stats(txo)->tx_compl += work_done;
3290 		u64_stats_update_end(&tx_stats(txo)->sync_compl);
3291 	}
3292 }
3293 
3294 int be_poll(struct napi_struct *napi, int budget)
3295 {
3296 	struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
3297 	struct be_adapter *adapter = eqo->adapter;
3298 	int max_work = 0, work, i, num_evts;
3299 	struct be_rx_obj *rxo;
3300 	struct be_tx_obj *txo;
3301 	u32 mult_enc = 0;
3302 
3303 	num_evts = events_get(eqo);
3304 
3305 	for_all_tx_queues_on_eq(adapter, eqo, txo, i)
3306 		be_process_tx(adapter, txo, i);
3307 
3308 	/* This loop will iterate twice for EQ0 in which
3309 	 * completions of the last RXQ (default one) are also processed
3310 	 * For other EQs the loop iterates only once
3311 	 */
3312 	for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
3313 		work = be_process_rx(rxo, napi, budget);
3314 		max_work = max(work, max_work);
3315 	}
3316 
3317 	if (is_mcc_eqo(eqo))
3318 		be_process_mcc(adapter);
3319 
3320 	if (max_work < budget) {
3321 		napi_complete_done(napi, max_work);
3322 
3323 		/* Skyhawk EQ_DB has a provision to set the rearm to interrupt
3324 		 * delay via a delay multiplier encoding value
3325 		 */
3326 		if (skyhawk_chip(adapter))
3327 			mult_enc = be_get_eq_delay_mult_enc(eqo);
3328 
3329 		be_eq_notify(adapter, eqo->q.id, true, false, num_evts,
3330 			     mult_enc);
3331 	} else {
3332 		/* As we'll continue in polling mode, count and clear events */
3333 		be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0);
3334 	}
3335 	return max_work;
3336 }
3337 
3338 void be_detect_error(struct be_adapter *adapter)
3339 {
3340 	u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
3341 	u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
3342 	struct device *dev = &adapter->pdev->dev;
3343 	u16 val;
3344 	u32 i;
3345 
3346 	if (be_check_error(adapter, BE_ERROR_HW))
3347 		return;
3348 
3349 	if (lancer_chip(adapter)) {
3350 		sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
3351 		if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
3352 			be_set_error(adapter, BE_ERROR_UE);
3353 			sliport_err1 = ioread32(adapter->db +
3354 						SLIPORT_ERROR1_OFFSET);
3355 			sliport_err2 = ioread32(adapter->db +
3356 						SLIPORT_ERROR2_OFFSET);
3357 			/* Do not log error messages if its a FW reset */
3358 			if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
3359 			    sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
3360 				dev_info(dev, "Reset is in progress\n");
3361 			} else {
3362 				dev_err(dev, "Error detected in the card\n");
3363 				dev_err(dev, "ERR: sliport status 0x%x\n",
3364 					sliport_status);
3365 				dev_err(dev, "ERR: sliport error1 0x%x\n",
3366 					sliport_err1);
3367 				dev_err(dev, "ERR: sliport error2 0x%x\n",
3368 					sliport_err2);
3369 			}
3370 		}
3371 	} else {
3372 		ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW);
3373 		ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH);
3374 		ue_lo_mask = ioread32(adapter->pcicfg +
3375 				      PCICFG_UE_STATUS_LOW_MASK);
3376 		ue_hi_mask = ioread32(adapter->pcicfg +
3377 				      PCICFG_UE_STATUS_HI_MASK);
3378 
3379 		ue_lo = (ue_lo & ~ue_lo_mask);
3380 		ue_hi = (ue_hi & ~ue_hi_mask);
3381 
3382 		if (ue_lo || ue_hi) {
3383 			/* On certain platforms BE3 hardware can indicate
3384 			 * spurious UEs. In case of a UE in the chip,
3385 			 * the POST register correctly reports either a
3386 			 * FAT_LOG_START state (FW is currently dumping
3387 			 * FAT log data) or a ARMFW_UE state. Check for the
3388 			 * above states to ascertain if the UE is valid or not.
3389 			 */
3390 			if (BE3_chip(adapter)) {
3391 				val = be_POST_stage_get(adapter);
3392 				if ((val & POST_STAGE_FAT_LOG_START)
3393 				     != POST_STAGE_FAT_LOG_START &&
3394 				    (val & POST_STAGE_ARMFW_UE)
3395 				     != POST_STAGE_ARMFW_UE &&
3396 				    (val & POST_STAGE_RECOVERABLE_ERR)
3397 				     != POST_STAGE_RECOVERABLE_ERR)
3398 					return;
3399 			}
3400 
3401 			dev_err(dev, "Error detected in the adapter");
3402 			be_set_error(adapter, BE_ERROR_UE);
3403 
3404 			for (i = 0; ue_lo; ue_lo >>= 1, i++) {
3405 				if (ue_lo & 1)
3406 					dev_err(dev, "UE: %s bit set\n",
3407 						ue_status_low_desc[i]);
3408 			}
3409 			for (i = 0; ue_hi; ue_hi >>= 1, i++) {
3410 				if (ue_hi & 1)
3411 					dev_err(dev, "UE: %s bit set\n",
3412 						ue_status_hi_desc[i]);
3413 			}
3414 		}
3415 	}
3416 }
3417 
3418 static void be_msix_disable(struct be_adapter *adapter)
3419 {
3420 	if (msix_enabled(adapter)) {
3421 		pci_disable_msix(adapter->pdev);
3422 		adapter->num_msix_vec = 0;
3423 		adapter->num_msix_roce_vec = 0;
3424 	}
3425 }
3426 
3427 static int be_msix_enable(struct be_adapter *adapter)
3428 {
3429 	unsigned int i, max_roce_eqs;
3430 	struct device *dev = &adapter->pdev->dev;
3431 	int num_vec;
3432 
3433 	/* If RoCE is supported, program the max number of vectors that
3434 	 * could be used for NIC and RoCE, else, just program the number
3435 	 * we'll use initially.
3436 	 */
3437 	if (be_roce_supported(adapter)) {
3438 		max_roce_eqs =
3439 			be_max_func_eqs(adapter) - be_max_nic_eqs(adapter);
3440 		max_roce_eqs = min(max_roce_eqs, num_online_cpus());
3441 		num_vec = be_max_any_irqs(adapter) + max_roce_eqs;
3442 	} else {
3443 		num_vec = max(adapter->cfg_num_rx_irqs,
3444 			      adapter->cfg_num_tx_irqs);
3445 	}
3446 
3447 	for (i = 0; i < num_vec; i++)
3448 		adapter->msix_entries[i].entry = i;
3449 
3450 	num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
3451 					MIN_MSIX_VECTORS, num_vec);
3452 	if (num_vec < 0)
3453 		goto fail;
3454 
3455 	if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
3456 		adapter->num_msix_roce_vec = num_vec / 2;
3457 		dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
3458 			 adapter->num_msix_roce_vec);
3459 	}
3460 
3461 	adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
3462 
3463 	dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
3464 		 adapter->num_msix_vec);
3465 	return 0;
3466 
3467 fail:
3468 	dev_warn(dev, "MSIx enable failed\n");
3469 
3470 	/* INTx is not supported in VFs, so fail probe if enable_msix fails */
3471 	if (be_virtfn(adapter))
3472 		return num_vec;
3473 	return 0;
3474 }
3475 
3476 static inline int be_msix_vec_get(struct be_adapter *adapter,
3477 				  struct be_eq_obj *eqo)
3478 {
3479 	return adapter->msix_entries[eqo->msix_idx].vector;
3480 }
3481 
3482 static int be_msix_register(struct be_adapter *adapter)
3483 {
3484 	struct net_device *netdev = adapter->netdev;
3485 	struct be_eq_obj *eqo;
3486 	int status, i, vec;
3487 
3488 	for_all_evt_queues(adapter, eqo, i) {
3489 		sprintf(eqo->desc, "%s-q%d", netdev->name, i);
3490 		vec = be_msix_vec_get(adapter, eqo);
3491 		status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
3492 		if (status)
3493 			goto err_msix;
3494 
3495 		irq_set_affinity_hint(vec, eqo->affinity_mask);
3496 	}
3497 
3498 	return 0;
3499 err_msix:
3500 	for (i--; i >= 0; i--) {
3501 		eqo = &adapter->eq_obj[i];
3502 		free_irq(be_msix_vec_get(adapter, eqo), eqo);
3503 	}
3504 	dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
3505 		 status);
3506 	be_msix_disable(adapter);
3507 	return status;
3508 }
3509 
3510 static int be_irq_register(struct be_adapter *adapter)
3511 {
3512 	struct net_device *netdev = adapter->netdev;
3513 	int status;
3514 
3515 	if (msix_enabled(adapter)) {
3516 		status = be_msix_register(adapter);
3517 		if (status == 0)
3518 			goto done;
3519 		/* INTx is not supported for VF */
3520 		if (be_virtfn(adapter))
3521 			return status;
3522 	}
3523 
3524 	/* INTx: only the first EQ is used */
3525 	netdev->irq = adapter->pdev->irq;
3526 	status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
3527 			     &adapter->eq_obj[0]);
3528 	if (status) {
3529 		dev_err(&adapter->pdev->dev,
3530 			"INTx request IRQ failed - err %d\n", status);
3531 		return status;
3532 	}
3533 done:
3534 	adapter->isr_registered = true;
3535 	return 0;
3536 }
3537 
3538 static void be_irq_unregister(struct be_adapter *adapter)
3539 {
3540 	struct net_device *netdev = adapter->netdev;
3541 	struct be_eq_obj *eqo;
3542 	int i, vec;
3543 
3544 	if (!adapter->isr_registered)
3545 		return;
3546 
3547 	/* INTx */
3548 	if (!msix_enabled(adapter)) {
3549 		free_irq(netdev->irq, &adapter->eq_obj[0]);
3550 		goto done;
3551 	}
3552 
3553 	/* MSIx */
3554 	for_all_evt_queues(adapter, eqo, i) {
3555 		vec = be_msix_vec_get(adapter, eqo);
3556 		irq_set_affinity_hint(vec, NULL);
3557 		free_irq(vec, eqo);
3558 	}
3559 
3560 done:
3561 	adapter->isr_registered = false;
3562 }
3563 
3564 static void be_rx_qs_destroy(struct be_adapter *adapter)
3565 {
3566 	struct rss_info *rss = &adapter->rss_info;
3567 	struct be_queue_info *q;
3568 	struct be_rx_obj *rxo;
3569 	int i;
3570 
3571 	for_all_rx_queues(adapter, rxo, i) {
3572 		q = &rxo->q;
3573 		if (q->created) {
3574 			/* If RXQs are destroyed while in an "out of buffer"
3575 			 * state, there is a possibility of an HW stall on
3576 			 * Lancer. So, post 64 buffers to each queue to relieve
3577 			 * the "out of buffer" condition.
3578 			 * Make sure there's space in the RXQ before posting.
3579 			 */
3580 			if (lancer_chip(adapter)) {
3581 				be_rx_cq_clean(rxo);
3582 				if (atomic_read(&q->used) == 0)
3583 					be_post_rx_frags(rxo, GFP_KERNEL,
3584 							 MAX_RX_POST);
3585 			}
3586 
3587 			be_cmd_rxq_destroy(adapter, q);
3588 			be_rx_cq_clean(rxo);
3589 			be_rxq_clean(rxo);
3590 		}
3591 		be_queue_free(adapter, q);
3592 	}
3593 
3594 	if (rss->rss_flags) {
3595 		rss->rss_flags = RSS_ENABLE_NONE;
3596 		be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3597 				  128, rss->rss_hkey);
3598 	}
3599 }
3600 
3601 static void be_disable_if_filters(struct be_adapter *adapter)
3602 {
3603 	/* Don't delete MAC on BE3 VFs without FILTMGMT privilege  */
3604 	if (!BEx_chip(adapter) || !be_virtfn(adapter) ||
3605 	    check_privilege(adapter, BE_PRIV_FILTMGMT)) {
3606 		be_dev_mac_del(adapter, adapter->pmac_id[0]);
3607 		eth_zero_addr(adapter->dev_mac);
3608 	}
3609 
3610 	be_clear_uc_list(adapter);
3611 	be_clear_mc_list(adapter);
3612 
3613 	/* The IFACE flags are enabled in the open path and cleared
3614 	 * in the close path. When a VF gets detached from the host and
3615 	 * assigned to a VM the following happens:
3616 	 *	- VF's IFACE flags get cleared in the detach path
3617 	 *	- IFACE create is issued by the VF in the attach path
3618 	 * Due to a bug in the BE3/Skyhawk-R FW
3619 	 * (Lancer FW doesn't have the bug), the IFACE capability flags
3620 	 * specified along with the IFACE create cmd issued by a VF are not
3621 	 * honoured by FW.  As a consequence, if a *new* driver
3622 	 * (that enables/disables IFACE flags in open/close)
3623 	 * is loaded in the host and an *old* driver is * used by a VM/VF,
3624 	 * the IFACE gets created *without* the needed flags.
3625 	 * To avoid this, disable RX-filter flags only for Lancer.
3626 	 */
3627 	if (lancer_chip(adapter)) {
3628 		be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF);
3629 		adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS;
3630 	}
3631 }
3632 
3633 static int be_close(struct net_device *netdev)
3634 {
3635 	struct be_adapter *adapter = netdev_priv(netdev);
3636 	struct be_eq_obj *eqo;
3637 	int i;
3638 
3639 	/* This protection is needed as be_close() may be called even when the
3640 	 * adapter is in cleared state (after eeh perm failure)
3641 	 */
3642 	if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
3643 		return 0;
3644 
3645 	/* Before attempting cleanup ensure all the pending cmds in the
3646 	 * config_wq have finished execution
3647 	 */
3648 	flush_workqueue(be_wq);
3649 
3650 	be_disable_if_filters(adapter);
3651 
3652 	if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
3653 		for_all_evt_queues(adapter, eqo, i) {
3654 			napi_disable(&eqo->napi);
3655 		}
3656 		adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
3657 	}
3658 
3659 	be_async_mcc_disable(adapter);
3660 
3661 	/* Wait for all pending tx completions to arrive so that
3662 	 * all tx skbs are freed.
3663 	 */
3664 	netif_tx_disable(netdev);
3665 	be_tx_compl_clean(adapter);
3666 
3667 	be_rx_qs_destroy(adapter);
3668 
3669 	for_all_evt_queues(adapter, eqo, i) {
3670 		if (msix_enabled(adapter))
3671 			synchronize_irq(be_msix_vec_get(adapter, eqo));
3672 		else
3673 			synchronize_irq(netdev->irq);
3674 		be_eq_clean(eqo);
3675 	}
3676 
3677 	be_irq_unregister(adapter);
3678 
3679 	return 0;
3680 }
3681 
3682 static int be_rx_qs_create(struct be_adapter *adapter)
3683 {
3684 	struct rss_info *rss = &adapter->rss_info;
3685 	u8 rss_key[RSS_HASH_KEY_LEN];
3686 	struct be_rx_obj *rxo;
3687 	int rc, i, j;
3688 
3689 	for_all_rx_queues(adapter, rxo, i) {
3690 		rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
3691 				    sizeof(struct be_eth_rx_d));
3692 		if (rc)
3693 			return rc;
3694 	}
3695 
3696 	if (adapter->need_def_rxq || !adapter->num_rss_qs) {
3697 		rxo = default_rxo(adapter);
3698 		rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3699 				       rx_frag_size, adapter->if_handle,
3700 				       false, &rxo->rss_id);
3701 		if (rc)
3702 			return rc;
3703 	}
3704 
3705 	for_all_rss_queues(adapter, rxo, i) {
3706 		rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
3707 				       rx_frag_size, adapter->if_handle,
3708 				       true, &rxo->rss_id);
3709 		if (rc)
3710 			return rc;
3711 	}
3712 
3713 	if (be_multi_rxq(adapter)) {
3714 		for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) {
3715 			for_all_rss_queues(adapter, rxo, i) {
3716 				if ((j + i) >= RSS_INDIR_TABLE_LEN)
3717 					break;
3718 				rss->rsstable[j + i] = rxo->rss_id;
3719 				rss->rss_queue[j + i] = i;
3720 			}
3721 		}
3722 		rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
3723 			RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
3724 
3725 		if (!BEx_chip(adapter))
3726 			rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
3727 				RSS_ENABLE_UDP_IPV6;
3728 
3729 		netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
3730 		rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
3731 				       RSS_INDIR_TABLE_LEN, rss_key);
3732 		if (rc) {
3733 			rss->rss_flags = RSS_ENABLE_NONE;
3734 			return rc;
3735 		}
3736 
3737 		memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
3738 	} else {
3739 		/* Disable RSS, if only default RX Q is created */
3740 		rss->rss_flags = RSS_ENABLE_NONE;
3741 	}
3742 
3743 
3744 	/* Post 1 less than RXQ-len to avoid head being equal to tail,
3745 	 * which is a queue empty condition
3746 	 */
3747 	for_all_rx_queues(adapter, rxo, i)
3748 		be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1);
3749 
3750 	return 0;
3751 }
3752 
3753 static int be_enable_if_filters(struct be_adapter *adapter)
3754 {
3755 	int status;
3756 
3757 	status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON);
3758 	if (status)
3759 		return status;
3760 
3761 	/* Normally this condition usually true as the ->dev_mac is zeroed.
3762 	 * But on BE3 VFs the initial MAC is pre-programmed by PF and
3763 	 * subsequent be_dev_mac_add() can fail (after fresh boot)
3764 	 */
3765 	if (!ether_addr_equal(adapter->dev_mac, adapter->netdev->dev_addr)) {
3766 		int old_pmac_id = -1;
3767 
3768 		/* Remember old programmed MAC if any - can happen on BE3 VF */
3769 		if (!is_zero_ether_addr(adapter->dev_mac))
3770 			old_pmac_id = adapter->pmac_id[0];
3771 
3772 		status = be_dev_mac_add(adapter, adapter->netdev->dev_addr);
3773 		if (status)
3774 			return status;
3775 
3776 		/* Delete the old programmed MAC as we successfully programmed
3777 		 * a new MAC
3778 		 */
3779 		if (old_pmac_id >= 0 && old_pmac_id != adapter->pmac_id[0])
3780 			be_dev_mac_del(adapter, old_pmac_id);
3781 
3782 		ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr);
3783 	}
3784 
3785 	if (adapter->vlans_added)
3786 		be_vid_config(adapter);
3787 
3788 	__be_set_rx_mode(adapter);
3789 
3790 	return 0;
3791 }
3792 
3793 static int be_open(struct net_device *netdev)
3794 {
3795 	struct be_adapter *adapter = netdev_priv(netdev);
3796 	struct be_eq_obj *eqo;
3797 	struct be_rx_obj *rxo;
3798 	struct be_tx_obj *txo;
3799 	u8 link_status;
3800 	int status, i;
3801 
3802 	status = be_rx_qs_create(adapter);
3803 	if (status)
3804 		goto err;
3805 
3806 	status = be_enable_if_filters(adapter);
3807 	if (status)
3808 		goto err;
3809 
3810 	status = be_irq_register(adapter);
3811 	if (status)
3812 		goto err;
3813 
3814 	for_all_rx_queues(adapter, rxo, i)
3815 		be_cq_notify(adapter, rxo->cq.id, true, 0);
3816 
3817 	for_all_tx_queues(adapter, txo, i)
3818 		be_cq_notify(adapter, txo->cq.id, true, 0);
3819 
3820 	be_async_mcc_enable(adapter);
3821 
3822 	for_all_evt_queues(adapter, eqo, i) {
3823 		napi_enable(&eqo->napi);
3824 		be_eq_notify(adapter, eqo->q.id, true, true, 0, 0);
3825 	}
3826 	adapter->flags |= BE_FLAGS_NAPI_ENABLED;
3827 
3828 	status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
3829 	if (!status)
3830 		be_link_status_update(adapter, link_status);
3831 
3832 	netif_tx_start_all_queues(netdev);
3833 	if (skyhawk_chip(adapter))
3834 		udp_tunnel_get_rx_info(netdev);
3835 
3836 	return 0;
3837 err:
3838 	be_close(adapter->netdev);
3839 	return -EIO;
3840 }
3841 
3842 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac)
3843 {
3844 	u32 addr;
3845 
3846 	addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0);
3847 
3848 	mac[5] = (u8)(addr & 0xFF);
3849 	mac[4] = (u8)((addr >> 8) & 0xFF);
3850 	mac[3] = (u8)((addr >> 16) & 0xFF);
3851 	/* Use the OUI from the current MAC address */
3852 	memcpy(mac, adapter->netdev->dev_addr, 3);
3853 }
3854 
3855 /*
3856  * Generate a seed MAC address from the PF MAC Address using jhash.
3857  * MAC Address for VFs are assigned incrementally starting from the seed.
3858  * These addresses are programmed in the ASIC by the PF and the VF driver
3859  * queries for the MAC address during its probe.
3860  */
3861 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3862 {
3863 	u32 vf;
3864 	int status = 0;
3865 	u8 mac[ETH_ALEN];
3866 	struct be_vf_cfg *vf_cfg;
3867 
3868 	be_vf_eth_addr_generate(adapter, mac);
3869 
3870 	for_all_vfs(adapter, vf_cfg, vf) {
3871 		if (BEx_chip(adapter))
3872 			status = be_cmd_pmac_add(adapter, mac,
3873 						 vf_cfg->if_handle,
3874 						 &vf_cfg->pmac_id, vf + 1);
3875 		else
3876 			status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3877 						vf + 1);
3878 
3879 		if (status)
3880 			dev_err(&adapter->pdev->dev,
3881 				"Mac address assignment failed for VF %d\n",
3882 				vf);
3883 		else
3884 			memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3885 
3886 		mac[5] += 1;
3887 	}
3888 	return status;
3889 }
3890 
3891 static int be_vfs_mac_query(struct be_adapter *adapter)
3892 {
3893 	int status, vf;
3894 	u8 mac[ETH_ALEN];
3895 	struct be_vf_cfg *vf_cfg;
3896 
3897 	for_all_vfs(adapter, vf_cfg, vf) {
3898 		status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3899 					       mac, vf_cfg->if_handle,
3900 					       false, vf+1);
3901 		if (status)
3902 			return status;
3903 		memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3904 	}
3905 	return 0;
3906 }
3907 
3908 static void be_vf_clear(struct be_adapter *adapter)
3909 {
3910 	struct be_vf_cfg *vf_cfg;
3911 	u32 vf;
3912 
3913 	if (pci_vfs_assigned(adapter->pdev)) {
3914 		dev_warn(&adapter->pdev->dev,
3915 			 "VFs are assigned to VMs: not disabling VFs\n");
3916 		goto done;
3917 	}
3918 
3919 	pci_disable_sriov(adapter->pdev);
3920 
3921 	for_all_vfs(adapter, vf_cfg, vf) {
3922 		if (BEx_chip(adapter))
3923 			be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3924 					vf_cfg->pmac_id, vf + 1);
3925 		else
3926 			be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3927 				       vf + 1);
3928 
3929 		be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3930 	}
3931 
3932 	if (BE3_chip(adapter))
3933 		be_cmd_set_hsw_config(adapter, 0, 0,
3934 				      adapter->if_handle,
3935 				      PORT_FWD_TYPE_PASSTHRU, 0);
3936 done:
3937 	kfree(adapter->vf_cfg);
3938 	adapter->num_vfs = 0;
3939 	adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3940 }
3941 
3942 static void be_clear_queues(struct be_adapter *adapter)
3943 {
3944 	be_mcc_queues_destroy(adapter);
3945 	be_rx_cqs_destroy(adapter);
3946 	be_tx_queues_destroy(adapter);
3947 	be_evt_queues_destroy(adapter);
3948 }
3949 
3950 static void be_cancel_worker(struct be_adapter *adapter)
3951 {
3952 	if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3953 		cancel_delayed_work_sync(&adapter->work);
3954 		adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3955 	}
3956 }
3957 
3958 static void be_cancel_err_detection(struct be_adapter *adapter)
3959 {
3960 	struct be_error_recovery *err_rec = &adapter->error_recovery;
3961 
3962 	if (!be_err_recovery_workq)
3963 		return;
3964 
3965 	if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) {
3966 		cancel_delayed_work_sync(&err_rec->err_detection_work);
3967 		adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED;
3968 	}
3969 }
3970 
3971 static int be_enable_vxlan_offloads(struct be_adapter *adapter)
3972 {
3973 	struct net_device *netdev = adapter->netdev;
3974 	struct device *dev = &adapter->pdev->dev;
3975 	struct be_vxlan_port *vxlan_port;
3976 	__be16 port;
3977 	int status;
3978 
3979 	vxlan_port = list_first_entry(&adapter->vxlan_port_list,
3980 				      struct be_vxlan_port, list);
3981 	port = vxlan_port->port;
3982 
3983 	status = be_cmd_manage_iface(adapter, adapter->if_handle,
3984 				     OP_CONVERT_NORMAL_TO_TUNNEL);
3985 	if (status) {
3986 		dev_warn(dev, "Failed to convert normal interface to tunnel\n");
3987 		return status;
3988 	}
3989 	adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
3990 
3991 	status = be_cmd_set_vxlan_port(adapter, port);
3992 	if (status) {
3993 		dev_warn(dev, "Failed to add VxLAN port\n");
3994 		return status;
3995 	}
3996 	adapter->vxlan_port = port;
3997 
3998 	netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3999 				   NETIF_F_TSO | NETIF_F_TSO6 |
4000 				   NETIF_F_GSO_UDP_TUNNEL;
4001 
4002 	dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4003 		 be16_to_cpu(port));
4004 	return 0;
4005 }
4006 
4007 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
4008 {
4009 	struct net_device *netdev = adapter->netdev;
4010 
4011 	if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
4012 		be_cmd_manage_iface(adapter, adapter->if_handle,
4013 				    OP_CONVERT_TUNNEL_TO_NORMAL);
4014 
4015 	if (adapter->vxlan_port)
4016 		be_cmd_set_vxlan_port(adapter, 0);
4017 
4018 	adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
4019 	adapter->vxlan_port = 0;
4020 
4021 	netdev->hw_enc_features = 0;
4022 }
4023 
4024 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs,
4025 				struct be_resources *vft_res)
4026 {
4027 	struct be_resources res = adapter->pool_res;
4028 	u32 vf_if_cap_flags = res.vf_if_cap_flags;
4029 	struct be_resources res_mod = {0};
4030 	u16 num_vf_qs = 1;
4031 
4032 	/* Distribute the queue resources among the PF and it's VFs */
4033 	if (num_vfs) {
4034 		/* Divide the rx queues evenly among the VFs and the PF, capped
4035 		 * at VF-EQ-count. Any remainder queues belong to the PF.
4036 		 */
4037 		num_vf_qs = min(SH_VF_MAX_NIC_EQS,
4038 				res.max_rss_qs / (num_vfs + 1));
4039 
4040 		/* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES
4041 		 * RSS Tables per port. Provide RSS on VFs, only if number of
4042 		 * VFs requested is less than it's PF Pool's RSS Tables limit.
4043 		 */
4044 		if (num_vfs >= be_max_pf_pool_rss_tables(adapter))
4045 			num_vf_qs = 1;
4046 	}
4047 
4048 	/* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd,
4049 	 * which are modifiable using SET_PROFILE_CONFIG cmd.
4050 	 */
4051 	be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE,
4052 				  RESOURCE_MODIFIABLE, 0);
4053 
4054 	/* If RSS IFACE capability flags are modifiable for a VF, set the
4055 	 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if
4056 	 * more than 1 RSSQ is available for a VF.
4057 	 * Otherwise, provision only 1 queue pair for VF.
4058 	 */
4059 	if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) {
4060 		vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
4061 		if (num_vf_qs > 1) {
4062 			vf_if_cap_flags |= BE_IF_FLAGS_RSS;
4063 			if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)
4064 				vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS;
4065 		} else {
4066 			vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS |
4067 					     BE_IF_FLAGS_DEFQ_RSS);
4068 		}
4069 	} else {
4070 		num_vf_qs = 1;
4071 	}
4072 
4073 	if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) {
4074 		vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT);
4075 		vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
4076 	}
4077 
4078 	vft_res->vf_if_cap_flags = vf_if_cap_flags;
4079 	vft_res->max_rx_qs = num_vf_qs;
4080 	vft_res->max_rss_qs = num_vf_qs;
4081 	vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1);
4082 	vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1);
4083 
4084 	/* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally
4085 	 * among the PF and it's VFs, if the fields are changeable
4086 	 */
4087 	if (res_mod.max_uc_mac == FIELD_MODIFIABLE)
4088 		vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1);
4089 
4090 	if (res_mod.max_vlans == FIELD_MODIFIABLE)
4091 		vft_res->max_vlans = res.max_vlans / (num_vfs + 1);
4092 
4093 	if (res_mod.max_iface_count == FIELD_MODIFIABLE)
4094 		vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1);
4095 
4096 	if (res_mod.max_mcc_count == FIELD_MODIFIABLE)
4097 		vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1);
4098 }
4099 
4100 static void be_if_destroy(struct be_adapter *adapter)
4101 {
4102 	be_cmd_if_destroy(adapter, adapter->if_handle,  0);
4103 
4104 	kfree(adapter->pmac_id);
4105 	adapter->pmac_id = NULL;
4106 
4107 	kfree(adapter->mc_list);
4108 	adapter->mc_list = NULL;
4109 
4110 	kfree(adapter->uc_list);
4111 	adapter->uc_list = NULL;
4112 }
4113 
4114 static int be_clear(struct be_adapter *adapter)
4115 {
4116 	struct pci_dev *pdev = adapter->pdev;
4117 	struct  be_resources vft_res = {0};
4118 
4119 	be_cancel_worker(adapter);
4120 
4121 	flush_workqueue(be_wq);
4122 
4123 	if (sriov_enabled(adapter))
4124 		be_vf_clear(adapter);
4125 
4126 	/* Re-configure FW to distribute resources evenly across max-supported
4127 	 * number of VFs, only when VFs are not already enabled.
4128 	 */
4129 	if (skyhawk_chip(adapter) && be_physfn(adapter) &&
4130 	    !pci_vfs_assigned(pdev)) {
4131 		be_calculate_vf_res(adapter,
4132 				    pci_sriov_get_totalvfs(pdev),
4133 				    &vft_res);
4134 		be_cmd_set_sriov_config(adapter, adapter->pool_res,
4135 					pci_sriov_get_totalvfs(pdev),
4136 					&vft_res);
4137 	}
4138 
4139 	be_disable_vxlan_offloads(adapter);
4140 
4141 	be_if_destroy(adapter);
4142 
4143 	be_clear_queues(adapter);
4144 
4145 	be_msix_disable(adapter);
4146 	adapter->flags &= ~BE_FLAGS_SETUP_DONE;
4147 	return 0;
4148 }
4149 
4150 static int be_vfs_if_create(struct be_adapter *adapter)
4151 {
4152 	struct be_resources res = {0};
4153 	u32 cap_flags, en_flags, vf;
4154 	struct be_vf_cfg *vf_cfg;
4155 	int status;
4156 
4157 	/* If a FW profile exists, then cap_flags are updated */
4158 	cap_flags = BE_VF_IF_EN_FLAGS;
4159 
4160 	for_all_vfs(adapter, vf_cfg, vf) {
4161 		if (!BE3_chip(adapter)) {
4162 			status = be_cmd_get_profile_config(adapter, &res, NULL,
4163 							   ACTIVE_PROFILE_TYPE,
4164 							   RESOURCE_LIMITS,
4165 							   vf + 1);
4166 			if (!status) {
4167 				cap_flags = res.if_cap_flags;
4168 				/* Prevent VFs from enabling VLAN promiscuous
4169 				 * mode
4170 				 */
4171 				cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS;
4172 			}
4173 		}
4174 
4175 		/* PF should enable IF flags during proxy if_create call */
4176 		en_flags = cap_flags & BE_VF_IF_EN_FLAGS;
4177 		status = be_cmd_if_create(adapter, cap_flags, en_flags,
4178 					  &vf_cfg->if_handle, vf + 1);
4179 		if (status)
4180 			return status;
4181 	}
4182 
4183 	return 0;
4184 }
4185 
4186 static int be_vf_setup_init(struct be_adapter *adapter)
4187 {
4188 	struct be_vf_cfg *vf_cfg;
4189 	int vf;
4190 
4191 	adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
4192 				  GFP_KERNEL);
4193 	if (!adapter->vf_cfg)
4194 		return -ENOMEM;
4195 
4196 	for_all_vfs(adapter, vf_cfg, vf) {
4197 		vf_cfg->if_handle = -1;
4198 		vf_cfg->pmac_id = -1;
4199 	}
4200 	return 0;
4201 }
4202 
4203 static int be_vf_setup(struct be_adapter *adapter)
4204 {
4205 	struct device *dev = &adapter->pdev->dev;
4206 	struct be_vf_cfg *vf_cfg;
4207 	int status, old_vfs, vf;
4208 	bool spoofchk;
4209 
4210 	old_vfs = pci_num_vf(adapter->pdev);
4211 
4212 	status = be_vf_setup_init(adapter);
4213 	if (status)
4214 		goto err;
4215 
4216 	if (old_vfs) {
4217 		for_all_vfs(adapter, vf_cfg, vf) {
4218 			status = be_cmd_get_if_id(adapter, vf_cfg, vf);
4219 			if (status)
4220 				goto err;
4221 		}
4222 
4223 		status = be_vfs_mac_query(adapter);
4224 		if (status)
4225 			goto err;
4226 	} else {
4227 		status = be_vfs_if_create(adapter);
4228 		if (status)
4229 			goto err;
4230 
4231 		status = be_vf_eth_addr_config(adapter);
4232 		if (status)
4233 			goto err;
4234 	}
4235 
4236 	for_all_vfs(adapter, vf_cfg, vf) {
4237 		/* Allow VFs to programs MAC/VLAN filters */
4238 		status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges,
4239 						  vf + 1);
4240 		if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) {
4241 			status = be_cmd_set_fn_privileges(adapter,
4242 							  vf_cfg->privileges |
4243 							  BE_PRIV_FILTMGMT,
4244 							  vf + 1);
4245 			if (!status) {
4246 				vf_cfg->privileges |= BE_PRIV_FILTMGMT;
4247 				dev_info(dev, "VF%d has FILTMGMT privilege\n",
4248 					 vf);
4249 			}
4250 		}
4251 
4252 		/* Allow full available bandwidth */
4253 		if (!old_vfs)
4254 			be_cmd_config_qos(adapter, 0, 0, vf + 1);
4255 
4256 		status = be_cmd_get_hsw_config(adapter, NULL, vf + 1,
4257 					       vf_cfg->if_handle, NULL,
4258 					       &spoofchk);
4259 		if (!status)
4260 			vf_cfg->spoofchk = spoofchk;
4261 
4262 		if (!old_vfs) {
4263 			be_cmd_enable_vf(adapter, vf + 1);
4264 			be_cmd_set_logical_link_config(adapter,
4265 						       IFLA_VF_LINK_STATE_AUTO,
4266 						       vf+1);
4267 		}
4268 	}
4269 
4270 	if (!old_vfs) {
4271 		status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
4272 		if (status) {
4273 			dev_err(dev, "SRIOV enable failed\n");
4274 			adapter->num_vfs = 0;
4275 			goto err;
4276 		}
4277 	}
4278 
4279 	if (BE3_chip(adapter)) {
4280 		/* On BE3, enable VEB only when SRIOV is enabled */
4281 		status = be_cmd_set_hsw_config(adapter, 0, 0,
4282 					       adapter->if_handle,
4283 					       PORT_FWD_TYPE_VEB, 0);
4284 		if (status)
4285 			goto err;
4286 	}
4287 
4288 	adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
4289 	return 0;
4290 err:
4291 	dev_err(dev, "VF setup failed\n");
4292 	be_vf_clear(adapter);
4293 	return status;
4294 }
4295 
4296 /* Converting function_mode bits on BE3 to SH mc_type enums */
4297 
4298 static u8 be_convert_mc_type(u32 function_mode)
4299 {
4300 	if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
4301 		return vNIC1;
4302 	else if (function_mode & QNQ_MODE)
4303 		return FLEX10;
4304 	else if (function_mode & VNIC_MODE)
4305 		return vNIC2;
4306 	else if (function_mode & UMC_ENABLED)
4307 		return UMC;
4308 	else
4309 		return MC_NONE;
4310 }
4311 
4312 /* On BE2/BE3 FW does not suggest the supported limits */
4313 static void BEx_get_resources(struct be_adapter *adapter,
4314 			      struct be_resources *res)
4315 {
4316 	bool use_sriov = adapter->num_vfs ? 1 : 0;
4317 
4318 	if (be_physfn(adapter))
4319 		res->max_uc_mac = BE_UC_PMAC_COUNT;
4320 	else
4321 		res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
4322 
4323 	adapter->mc_type = be_convert_mc_type(adapter->function_mode);
4324 
4325 	if (be_is_mc(adapter)) {
4326 		/* Assuming that there are 4 channels per port,
4327 		 * when multi-channel is enabled
4328 		 */
4329 		if (be_is_qnq_mode(adapter))
4330 			res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
4331 		else
4332 			/* In a non-qnq multichannel mode, the pvid
4333 			 * takes up one vlan entry
4334 			 */
4335 			res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
4336 	} else {
4337 		res->max_vlans = BE_NUM_VLANS_SUPPORTED;
4338 	}
4339 
4340 	res->max_mcast_mac = BE_MAX_MC;
4341 
4342 	/* 1) For BE3 1Gb ports, FW does not support multiple TXQs
4343 	 * 2) Create multiple TX rings on a BE3-R multi-channel interface
4344 	 *    *only* if it is RSS-capable.
4345 	 */
4346 	if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
4347 	    be_virtfn(adapter) ||
4348 	    (be_is_mc(adapter) &&
4349 	     !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
4350 		res->max_tx_qs = 1;
4351 	} else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
4352 		struct be_resources super_nic_res = {0};
4353 
4354 		/* On a SuperNIC profile, the driver needs to use the
4355 		 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
4356 		 */
4357 		be_cmd_get_profile_config(adapter, &super_nic_res, NULL,
4358 					  ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS,
4359 					  0);
4360 		/* Some old versions of BE3 FW don't report max_tx_qs value */
4361 		res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
4362 	} else {
4363 		res->max_tx_qs = BE3_MAX_TX_QS;
4364 	}
4365 
4366 	if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
4367 	    !use_sriov && be_physfn(adapter))
4368 		res->max_rss_qs = (adapter->be3_native) ?
4369 					   BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
4370 	res->max_rx_qs = res->max_rss_qs + 1;
4371 
4372 	if (be_physfn(adapter))
4373 		res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
4374 					BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
4375 	else
4376 		res->max_evt_qs = 1;
4377 
4378 	res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
4379 	res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS;
4380 	if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
4381 		res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
4382 }
4383 
4384 static void be_setup_init(struct be_adapter *adapter)
4385 {
4386 	adapter->vlan_prio_bmap = 0xff;
4387 	adapter->phy.link_speed = -1;
4388 	adapter->if_handle = -1;
4389 	adapter->be3_native = false;
4390 	adapter->if_flags = 0;
4391 	adapter->phy_state = BE_UNKNOWN_PHY_STATE;
4392 	if (be_physfn(adapter))
4393 		adapter->cmd_privileges = MAX_PRIVILEGES;
4394 	else
4395 		adapter->cmd_privileges = MIN_PRIVILEGES;
4396 }
4397 
4398 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port.
4399  * However, this HW limitation is not exposed to the host via any SLI cmd.
4400  * As a result, in the case of SRIOV and in particular multi-partition configs
4401  * the driver needs to calcuate a proportional share of RSS Tables per PF-pool
4402  * for distribution between the VFs. This self-imposed limit will determine the
4403  * no: of VFs for which RSS can be enabled.
4404  */
4405 static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter)
4406 {
4407 	struct be_port_resources port_res = {0};
4408 	u8 rss_tables_on_port;
4409 	u16 max_vfs = be_max_vfs(adapter);
4410 
4411 	be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE,
4412 				  RESOURCE_LIMITS, 0);
4413 
4414 	rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs;
4415 
4416 	/* Each PF Pool's RSS Tables limit =
4417 	 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port
4418 	 */
4419 	adapter->pool_res.max_rss_tables =
4420 		max_vfs * rss_tables_on_port / port_res.max_vfs;
4421 }
4422 
4423 static int be_get_sriov_config(struct be_adapter *adapter)
4424 {
4425 	struct be_resources res = {0};
4426 	int max_vfs, old_vfs;
4427 
4428 	be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE,
4429 				  RESOURCE_LIMITS, 0);
4430 
4431 	/* Some old versions of BE3 FW don't report max_vfs value */
4432 	if (BE3_chip(adapter) && !res.max_vfs) {
4433 		max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
4434 		res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
4435 	}
4436 
4437 	adapter->pool_res = res;
4438 
4439 	/* If during previous unload of the driver, the VFs were not disabled,
4440 	 * then we cannot rely on the PF POOL limits for the TotalVFs value.
4441 	 * Instead use the TotalVFs value stored in the pci-dev struct.
4442 	 */
4443 	old_vfs = pci_num_vf(adapter->pdev);
4444 	if (old_vfs) {
4445 		dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n",
4446 			 old_vfs);
4447 
4448 		adapter->pool_res.max_vfs =
4449 			pci_sriov_get_totalvfs(adapter->pdev);
4450 		adapter->num_vfs = old_vfs;
4451 	}
4452 
4453 	if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
4454 		be_calculate_pf_pool_rss_tables(adapter);
4455 		dev_info(&adapter->pdev->dev,
4456 			 "RSS can be enabled for all VFs if num_vfs <= %d\n",
4457 			 be_max_pf_pool_rss_tables(adapter));
4458 	}
4459 	return 0;
4460 }
4461 
4462 static void be_alloc_sriov_res(struct be_adapter *adapter)
4463 {
4464 	int old_vfs = pci_num_vf(adapter->pdev);
4465 	struct  be_resources vft_res = {0};
4466 	int status;
4467 
4468 	be_get_sriov_config(adapter);
4469 
4470 	if (!old_vfs)
4471 		pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
4472 
4473 	/* When the HW is in SRIOV capable configuration, the PF-pool
4474 	 * resources are given to PF during driver load, if there are no
4475 	 * old VFs. This facility is not available in BE3 FW.
4476 	 * Also, this is done by FW in Lancer chip.
4477 	 */
4478 	if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) {
4479 		be_calculate_vf_res(adapter, 0, &vft_res);
4480 		status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0,
4481 						 &vft_res);
4482 		if (status)
4483 			dev_err(&adapter->pdev->dev,
4484 				"Failed to optimize SRIOV resources\n");
4485 	}
4486 }
4487 
4488 static int be_get_resources(struct be_adapter *adapter)
4489 {
4490 	struct device *dev = &adapter->pdev->dev;
4491 	struct be_resources res = {0};
4492 	int status;
4493 
4494 	/* For Lancer, SH etc read per-function resource limits from FW.
4495 	 * GET_FUNC_CONFIG returns per function guaranteed limits.
4496 	 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
4497 	 */
4498 	if (BEx_chip(adapter)) {
4499 		BEx_get_resources(adapter, &res);
4500 	} else {
4501 		status = be_cmd_get_func_config(adapter, &res);
4502 		if (status)
4503 			return status;
4504 
4505 		/* If a deafault RXQ must be created, we'll use up one RSSQ*/
4506 		if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs &&
4507 		    !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS))
4508 			res.max_rss_qs -= 1;
4509 	}
4510 
4511 	/* If RoCE is supported stash away half the EQs for RoCE */
4512 	res.max_nic_evt_qs = be_roce_supported(adapter) ?
4513 				res.max_evt_qs / 2 : res.max_evt_qs;
4514 	adapter->res = res;
4515 
4516 	/* If FW supports RSS default queue, then skip creating non-RSS
4517 	 * queue for non-IP traffic.
4518 	 */
4519 	adapter->need_def_rxq = (be_if_cap_flags(adapter) &
4520 				 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1;
4521 
4522 	dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
4523 		 be_max_txqs(adapter), be_max_rxqs(adapter),
4524 		 be_max_rss(adapter), be_max_nic_eqs(adapter),
4525 		 be_max_vfs(adapter));
4526 	dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
4527 		 be_max_uc(adapter), be_max_mc(adapter),
4528 		 be_max_vlans(adapter));
4529 
4530 	/* Ensure RX and TX queues are created in pairs at init time */
4531 	adapter->cfg_num_rx_irqs =
4532 				min_t(u16, netif_get_num_default_rss_queues(),
4533 				      be_max_qp_irqs(adapter));
4534 	adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs;
4535 	return 0;
4536 }
4537 
4538 static int be_get_config(struct be_adapter *adapter)
4539 {
4540 	int status, level;
4541 	u16 profile_id;
4542 
4543 	status = be_cmd_get_cntl_attributes(adapter);
4544 	if (status)
4545 		return status;
4546 
4547 	status = be_cmd_query_fw_cfg(adapter);
4548 	if (status)
4549 		return status;
4550 
4551 	if (!lancer_chip(adapter) && be_physfn(adapter))
4552 		be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len);
4553 
4554 	if (BEx_chip(adapter)) {
4555 		level = be_cmd_get_fw_log_level(adapter);
4556 		adapter->msg_enable =
4557 			level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4558 	}
4559 
4560 	be_cmd_get_acpi_wol_cap(adapter);
4561 	pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en);
4562 	pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en);
4563 
4564 	be_cmd_query_port_name(adapter);
4565 
4566 	if (be_physfn(adapter)) {
4567 		status = be_cmd_get_active_profile(adapter, &profile_id);
4568 		if (!status)
4569 			dev_info(&adapter->pdev->dev,
4570 				 "Using profile 0x%x\n", profile_id);
4571 	}
4572 
4573 	return 0;
4574 }
4575 
4576 static int be_mac_setup(struct be_adapter *adapter)
4577 {
4578 	u8 mac[ETH_ALEN];
4579 	int status;
4580 
4581 	if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
4582 		status = be_cmd_get_perm_mac(adapter, mac);
4583 		if (status)
4584 			return status;
4585 
4586 		memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
4587 		memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
4588 
4589 		/* Initial MAC for BE3 VFs is already programmed by PF */
4590 		if (BEx_chip(adapter) && be_virtfn(adapter))
4591 			memcpy(adapter->dev_mac, mac, ETH_ALEN);
4592 	}
4593 
4594 	return 0;
4595 }
4596 
4597 static void be_schedule_worker(struct be_adapter *adapter)
4598 {
4599 	queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
4600 	adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
4601 }
4602 
4603 static void be_destroy_err_recovery_workq(void)
4604 {
4605 	if (!be_err_recovery_workq)
4606 		return;
4607 
4608 	flush_workqueue(be_err_recovery_workq);
4609 	destroy_workqueue(be_err_recovery_workq);
4610 	be_err_recovery_workq = NULL;
4611 }
4612 
4613 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay)
4614 {
4615 	struct be_error_recovery *err_rec = &adapter->error_recovery;
4616 
4617 	if (!be_err_recovery_workq)
4618 		return;
4619 
4620 	queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work,
4621 			   msecs_to_jiffies(delay));
4622 	adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED;
4623 }
4624 
4625 static int be_setup_queues(struct be_adapter *adapter)
4626 {
4627 	struct net_device *netdev = adapter->netdev;
4628 	int status;
4629 
4630 	status = be_evt_queues_create(adapter);
4631 	if (status)
4632 		goto err;
4633 
4634 	status = be_tx_qs_create(adapter);
4635 	if (status)
4636 		goto err;
4637 
4638 	status = be_rx_cqs_create(adapter);
4639 	if (status)
4640 		goto err;
4641 
4642 	status = be_mcc_queues_create(adapter);
4643 	if (status)
4644 		goto err;
4645 
4646 	status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
4647 	if (status)
4648 		goto err;
4649 
4650 	status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
4651 	if (status)
4652 		goto err;
4653 
4654 	return 0;
4655 err:
4656 	dev_err(&adapter->pdev->dev, "queue_setup failed\n");
4657 	return status;
4658 }
4659 
4660 static int be_if_create(struct be_adapter *adapter)
4661 {
4662 	u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS;
4663 	u32 cap_flags = be_if_cap_flags(adapter);
4664 	int status;
4665 
4666 	/* alloc required memory for other filtering fields */
4667 	adapter->pmac_id = kcalloc(be_max_uc(adapter),
4668 				   sizeof(*adapter->pmac_id), GFP_KERNEL);
4669 	if (!adapter->pmac_id)
4670 		return -ENOMEM;
4671 
4672 	adapter->mc_list = kcalloc(be_max_mc(adapter),
4673 				   sizeof(*adapter->mc_list), GFP_KERNEL);
4674 	if (!adapter->mc_list)
4675 		return -ENOMEM;
4676 
4677 	adapter->uc_list = kcalloc(be_max_uc(adapter),
4678 				   sizeof(*adapter->uc_list), GFP_KERNEL);
4679 	if (!adapter->uc_list)
4680 		return -ENOMEM;
4681 
4682 	if (adapter->cfg_num_rx_irqs == 1)
4683 		cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS);
4684 
4685 	en_flags &= cap_flags;
4686 	/* will enable all the needed filter flags in be_open() */
4687 	status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
4688 				  &adapter->if_handle, 0);
4689 
4690 	if (status)
4691 		return status;
4692 
4693 	return 0;
4694 }
4695 
4696 int be_update_queues(struct be_adapter *adapter)
4697 {
4698 	struct net_device *netdev = adapter->netdev;
4699 	int status;
4700 
4701 	if (netif_running(netdev)) {
4702 		/* be_tx_timeout() must not run concurrently with this
4703 		 * function, synchronize with an already-running dev_watchdog
4704 		 */
4705 		netif_tx_lock_bh(netdev);
4706 		/* device cannot transmit now, avoid dev_watchdog timeouts */
4707 		netif_carrier_off(netdev);
4708 		netif_tx_unlock_bh(netdev);
4709 
4710 		be_close(netdev);
4711 	}
4712 
4713 	be_cancel_worker(adapter);
4714 
4715 	/* If any vectors have been shared with RoCE we cannot re-program
4716 	 * the MSIx table.
4717 	 */
4718 	if (!adapter->num_msix_roce_vec)
4719 		be_msix_disable(adapter);
4720 
4721 	be_clear_queues(adapter);
4722 	status = be_cmd_if_destroy(adapter, adapter->if_handle,  0);
4723 	if (status)
4724 		return status;
4725 
4726 	if (!msix_enabled(adapter)) {
4727 		status = be_msix_enable(adapter);
4728 		if (status)
4729 			return status;
4730 	}
4731 
4732 	status = be_if_create(adapter);
4733 	if (status)
4734 		return status;
4735 
4736 	status = be_setup_queues(adapter);
4737 	if (status)
4738 		return status;
4739 
4740 	be_schedule_worker(adapter);
4741 
4742 	/* The IF was destroyed and re-created. We need to clear
4743 	 * all promiscuous flags valid for the destroyed IF.
4744 	 * Without this promisc mode is not restored during
4745 	 * be_open() because the driver thinks that it is
4746 	 * already enabled in HW.
4747 	 */
4748 	adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS;
4749 
4750 	if (netif_running(netdev))
4751 		status = be_open(netdev);
4752 
4753 	return status;
4754 }
4755 
4756 static inline int fw_major_num(const char *fw_ver)
4757 {
4758 	int fw_major = 0, i;
4759 
4760 	i = sscanf(fw_ver, "%d.", &fw_major);
4761 	if (i != 1)
4762 		return 0;
4763 
4764 	return fw_major;
4765 }
4766 
4767 /* If it is error recovery, FLR the PF
4768  * Else if any VFs are already enabled don't FLR the PF
4769  */
4770 static bool be_reset_required(struct be_adapter *adapter)
4771 {
4772 	if (be_error_recovering(adapter))
4773 		return true;
4774 	else
4775 		return pci_num_vf(adapter->pdev) == 0;
4776 }
4777 
4778 /* Wait for the FW to be ready and perform the required initialization */
4779 static int be_func_init(struct be_adapter *adapter)
4780 {
4781 	int status;
4782 
4783 	status = be_fw_wait_ready(adapter);
4784 	if (status)
4785 		return status;
4786 
4787 	/* FW is now ready; clear errors to allow cmds/doorbell */
4788 	be_clear_error(adapter, BE_CLEAR_ALL);
4789 
4790 	if (be_reset_required(adapter)) {
4791 		status = be_cmd_reset_function(adapter);
4792 		if (status)
4793 			return status;
4794 
4795 		/* Wait for interrupts to quiesce after an FLR */
4796 		msleep(100);
4797 	}
4798 
4799 	/* Tell FW we're ready to fire cmds */
4800 	status = be_cmd_fw_init(adapter);
4801 	if (status)
4802 		return status;
4803 
4804 	/* Allow interrupts for other ULPs running on NIC function */
4805 	be_intr_set(adapter, true);
4806 
4807 	return 0;
4808 }
4809 
4810 static int be_setup(struct be_adapter *adapter)
4811 {
4812 	struct device *dev = &adapter->pdev->dev;
4813 	int status;
4814 
4815 	status = be_func_init(adapter);
4816 	if (status)
4817 		return status;
4818 
4819 	be_setup_init(adapter);
4820 
4821 	if (!lancer_chip(adapter))
4822 		be_cmd_req_native_mode(adapter);
4823 
4824 	/* invoke this cmd first to get pf_num and vf_num which are needed
4825 	 * for issuing profile related cmds
4826 	 */
4827 	if (!BEx_chip(adapter)) {
4828 		status = be_cmd_get_func_config(adapter, NULL);
4829 		if (status)
4830 			return status;
4831 	}
4832 
4833 	status = be_get_config(adapter);
4834 	if (status)
4835 		goto err;
4836 
4837 	if (!BE2_chip(adapter) && be_physfn(adapter))
4838 		be_alloc_sriov_res(adapter);
4839 
4840 	status = be_get_resources(adapter);
4841 	if (status)
4842 		goto err;
4843 
4844 	status = be_msix_enable(adapter);
4845 	if (status)
4846 		goto err;
4847 
4848 	/* will enable all the needed filter flags in be_open() */
4849 	status = be_if_create(adapter);
4850 	if (status)
4851 		goto err;
4852 
4853 	/* Updating real_num_tx/rx_queues() requires rtnl_lock() */
4854 	rtnl_lock();
4855 	status = be_setup_queues(adapter);
4856 	rtnl_unlock();
4857 	if (status)
4858 		goto err;
4859 
4860 	be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
4861 
4862 	status = be_mac_setup(adapter);
4863 	if (status)
4864 		goto err;
4865 
4866 	be_cmd_get_fw_ver(adapter);
4867 	dev_info(dev, "FW version is %s\n", adapter->fw_ver);
4868 
4869 	if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
4870 		dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
4871 			adapter->fw_ver);
4872 		dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
4873 	}
4874 
4875 	status = be_cmd_set_flow_control(adapter, adapter->tx_fc,
4876 					 adapter->rx_fc);
4877 	if (status)
4878 		be_cmd_get_flow_control(adapter, &adapter->tx_fc,
4879 					&adapter->rx_fc);
4880 
4881 	dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n",
4882 		 adapter->tx_fc, adapter->rx_fc);
4883 
4884 	if (be_physfn(adapter))
4885 		be_cmd_set_logical_link_config(adapter,
4886 					       IFLA_VF_LINK_STATE_AUTO, 0);
4887 
4888 	/* BE3 EVB echoes broadcast/multicast packets back to PF's vport
4889 	 * confusing a linux bridge or OVS that it might be connected to.
4890 	 * Set the EVB to PASSTHRU mode which effectively disables the EVB
4891 	 * when SRIOV is not enabled.
4892 	 */
4893 	if (BE3_chip(adapter))
4894 		be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle,
4895 				      PORT_FWD_TYPE_PASSTHRU, 0);
4896 
4897 	if (adapter->num_vfs)
4898 		be_vf_setup(adapter);
4899 
4900 	status = be_cmd_get_phy_info(adapter);
4901 	if (!status && be_pause_supported(adapter))
4902 		adapter->phy.fc_autoneg = 1;
4903 
4904 	if (be_physfn(adapter) && !lancer_chip(adapter))
4905 		be_cmd_set_features(adapter);
4906 
4907 	be_schedule_worker(adapter);
4908 	adapter->flags |= BE_FLAGS_SETUP_DONE;
4909 	return 0;
4910 err:
4911 	be_clear(adapter);
4912 	return status;
4913 }
4914 
4915 #ifdef CONFIG_NET_POLL_CONTROLLER
4916 static void be_netpoll(struct net_device *netdev)
4917 {
4918 	struct be_adapter *adapter = netdev_priv(netdev);
4919 	struct be_eq_obj *eqo;
4920 	int i;
4921 
4922 	for_all_evt_queues(adapter, eqo, i) {
4923 		be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0);
4924 		napi_schedule(&eqo->napi);
4925 	}
4926 }
4927 #endif
4928 
4929 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4930 {
4931 	const struct firmware *fw;
4932 	int status;
4933 
4934 	if (!netif_running(adapter->netdev)) {
4935 		dev_err(&adapter->pdev->dev,
4936 			"Firmware load not allowed (interface is down)\n");
4937 		return -ENETDOWN;
4938 	}
4939 
4940 	status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4941 	if (status)
4942 		goto fw_exit;
4943 
4944 	dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4945 
4946 	if (lancer_chip(adapter))
4947 		status = lancer_fw_download(adapter, fw);
4948 	else
4949 		status = be_fw_download(adapter, fw);
4950 
4951 	if (!status)
4952 		be_cmd_get_fw_ver(adapter);
4953 
4954 fw_exit:
4955 	release_firmware(fw);
4956 	return status;
4957 }
4958 
4959 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
4960 				 u16 flags, struct netlink_ext_ack *extack)
4961 {
4962 	struct be_adapter *adapter = netdev_priv(dev);
4963 	struct nlattr *attr, *br_spec;
4964 	int rem;
4965 	int status = 0;
4966 	u16 mode = 0;
4967 
4968 	if (!sriov_enabled(adapter))
4969 		return -EOPNOTSUPP;
4970 
4971 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4972 	if (!br_spec)
4973 		return -EINVAL;
4974 
4975 	nla_for_each_nested(attr, br_spec, rem) {
4976 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
4977 			continue;
4978 
4979 		if (nla_len(attr) < sizeof(mode))
4980 			return -EINVAL;
4981 
4982 		mode = nla_get_u16(attr);
4983 		if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA)
4984 			return -EOPNOTSUPP;
4985 
4986 		if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4987 			return -EINVAL;
4988 
4989 		status = be_cmd_set_hsw_config(adapter, 0, 0,
4990 					       adapter->if_handle,
4991 					       mode == BRIDGE_MODE_VEPA ?
4992 					       PORT_FWD_TYPE_VEPA :
4993 					       PORT_FWD_TYPE_VEB, 0);
4994 		if (status)
4995 			goto err;
4996 
4997 		dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4998 			 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4999 
5000 		return status;
5001 	}
5002 err:
5003 	dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
5004 		mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
5005 
5006 	return status;
5007 }
5008 
5009 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
5010 				 struct net_device *dev, u32 filter_mask,
5011 				 int nlflags)
5012 {
5013 	struct be_adapter *adapter = netdev_priv(dev);
5014 	int status = 0;
5015 	u8 hsw_mode;
5016 
5017 	/* BE and Lancer chips support VEB mode only */
5018 	if (BEx_chip(adapter) || lancer_chip(adapter)) {
5019 		/* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */
5020 		if (!pci_sriov_get_totalvfs(adapter->pdev))
5021 			return 0;
5022 		hsw_mode = PORT_FWD_TYPE_VEB;
5023 	} else {
5024 		status = be_cmd_get_hsw_config(adapter, NULL, 0,
5025 					       adapter->if_handle, &hsw_mode,
5026 					       NULL);
5027 		if (status)
5028 			return 0;
5029 
5030 		if (hsw_mode == PORT_FWD_TYPE_PASSTHRU)
5031 			return 0;
5032 	}
5033 
5034 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
5035 				       hsw_mode == PORT_FWD_TYPE_VEPA ?
5036 				       BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
5037 				       0, 0, nlflags, filter_mask, NULL);
5038 }
5039 
5040 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter,
5041 					 void (*func)(struct work_struct *))
5042 {
5043 	struct be_cmd_work *work;
5044 
5045 	work = kzalloc(sizeof(*work), GFP_ATOMIC);
5046 	if (!work) {
5047 		dev_err(&adapter->pdev->dev,
5048 			"be_work memory allocation failed\n");
5049 		return NULL;
5050 	}
5051 
5052 	INIT_WORK(&work->work, func);
5053 	work->adapter = adapter;
5054 	return work;
5055 }
5056 
5057 /* VxLAN offload Notes:
5058  *
5059  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
5060  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
5061  * is expected to work across all types of IP tunnels once exported. Skyhawk
5062  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
5063  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
5064  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
5065  * those other tunnels are unexported on the fly through ndo_features_check().
5066  *
5067  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
5068  * adds more than one port, disable offloads and re-enable them again when
5069  * there's only one port left. We maintain a list of ports for this purpose.
5070  */
5071 static void be_work_add_vxlan_port(struct work_struct *work)
5072 {
5073 	struct be_cmd_work *cmd_work =
5074 				container_of(work, struct be_cmd_work, work);
5075 	struct be_adapter *adapter = cmd_work->adapter;
5076 	struct device *dev = &adapter->pdev->dev;
5077 	__be16 port = cmd_work->info.vxlan_port;
5078 	struct be_vxlan_port *vxlan_port;
5079 	int status;
5080 
5081 	/* Bump up the alias count if it is an existing port */
5082 	list_for_each_entry(vxlan_port, &adapter->vxlan_port_list, list) {
5083 		if (vxlan_port->port == port) {
5084 			vxlan_port->port_aliases++;
5085 			goto done;
5086 		}
5087 	}
5088 
5089 	/* Add a new port to our list. We don't need a lock here since port
5090 	 * add/delete are done only in the context of a single-threaded work
5091 	 * queue (be_wq).
5092 	 */
5093 	vxlan_port = kzalloc(sizeof(*vxlan_port), GFP_KERNEL);
5094 	if (!vxlan_port)
5095 		goto done;
5096 
5097 	vxlan_port->port = port;
5098 	INIT_LIST_HEAD(&vxlan_port->list);
5099 	list_add_tail(&vxlan_port->list, &adapter->vxlan_port_list);
5100 	adapter->vxlan_port_count++;
5101 
5102 	if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
5103 		dev_info(dev,
5104 			 "Only one UDP port supported for VxLAN offloads\n");
5105 		dev_info(dev, "Disabling VxLAN offloads\n");
5106 		goto err;
5107 	}
5108 
5109 	if (adapter->vxlan_port_count > 1)
5110 		goto done;
5111 
5112 	status = be_enable_vxlan_offloads(adapter);
5113 	if (!status)
5114 		goto done;
5115 
5116 err:
5117 	be_disable_vxlan_offloads(adapter);
5118 done:
5119 	kfree(cmd_work);
5120 	return;
5121 }
5122 
5123 static void be_work_del_vxlan_port(struct work_struct *work)
5124 {
5125 	struct be_cmd_work *cmd_work =
5126 				container_of(work, struct be_cmd_work, work);
5127 	struct be_adapter *adapter = cmd_work->adapter;
5128 	__be16 port = cmd_work->info.vxlan_port;
5129 	struct be_vxlan_port *vxlan_port;
5130 
5131 	/* Nothing to be done if a port alias is being deleted */
5132 	list_for_each_entry(vxlan_port, &adapter->vxlan_port_list, list) {
5133 		if (vxlan_port->port == port) {
5134 			if (vxlan_port->port_aliases) {
5135 				vxlan_port->port_aliases--;
5136 				goto done;
5137 			}
5138 			break;
5139 		}
5140 	}
5141 
5142 	/* No port aliases left; delete the port from the list */
5143 	list_del(&vxlan_port->list);
5144 	adapter->vxlan_port_count--;
5145 
5146 	/* Disable VxLAN offload if this is the offloaded port */
5147 	if (adapter->vxlan_port == vxlan_port->port) {
5148 		WARN_ON(adapter->vxlan_port_count);
5149 		be_disable_vxlan_offloads(adapter);
5150 		dev_info(&adapter->pdev->dev,
5151 			 "Disabled VxLAN offloads for UDP port %d\n",
5152 			 be16_to_cpu(port));
5153 		goto out;
5154 	}
5155 
5156 	/* If only 1 port is left, re-enable VxLAN offload */
5157 	if (adapter->vxlan_port_count == 1)
5158 		be_enable_vxlan_offloads(adapter);
5159 
5160 out:
5161 	kfree(vxlan_port);
5162 done:
5163 	kfree(cmd_work);
5164 }
5165 
5166 static void be_cfg_vxlan_port(struct net_device *netdev,
5167 			      struct udp_tunnel_info *ti,
5168 			      void (*func)(struct work_struct *))
5169 {
5170 	struct be_adapter *adapter = netdev_priv(netdev);
5171 	struct be_cmd_work *cmd_work;
5172 
5173 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
5174 		return;
5175 
5176 	if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter))
5177 		return;
5178 
5179 	cmd_work = be_alloc_work(adapter, func);
5180 	if (cmd_work) {
5181 		cmd_work->info.vxlan_port = ti->port;
5182 		queue_work(be_wq, &cmd_work->work);
5183 	}
5184 }
5185 
5186 static void be_del_vxlan_port(struct net_device *netdev,
5187 			      struct udp_tunnel_info *ti)
5188 {
5189 	be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port);
5190 }
5191 
5192 static void be_add_vxlan_port(struct net_device *netdev,
5193 			      struct udp_tunnel_info *ti)
5194 {
5195 	be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port);
5196 }
5197 
5198 static netdev_features_t be_features_check(struct sk_buff *skb,
5199 					   struct net_device *dev,
5200 					   netdev_features_t features)
5201 {
5202 	struct be_adapter *adapter = netdev_priv(dev);
5203 	u8 l4_hdr = 0;
5204 
5205 	if (skb_is_gso(skb)) {
5206 		/* IPv6 TSO requests with extension hdrs are a problem
5207 		 * to Lancer and BE3 HW. Disable TSO6 feature.
5208 		 */
5209 		if (!skyhawk_chip(adapter) && is_ipv6_ext_hdr(skb))
5210 			features &= ~NETIF_F_TSO6;
5211 
5212 		/* Lancer cannot handle the packet with MSS less than 256.
5213 		 * Also it can't handle a TSO packet with a single segment
5214 		 * Disable the GSO support in such cases
5215 		 */
5216 		if (lancer_chip(adapter) &&
5217 		    (skb_shinfo(skb)->gso_size < 256 ||
5218 		     skb_shinfo(skb)->gso_segs == 1))
5219 			features &= ~NETIF_F_GSO_MASK;
5220 	}
5221 
5222 	/* The code below restricts offload features for some tunneled and
5223 	 * Q-in-Q packets.
5224 	 * Offload features for normal (non tunnel) packets are unchanged.
5225 	 */
5226 	features = vlan_features_check(skb, features);
5227 	if (!skb->encapsulation ||
5228 	    !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
5229 		return features;
5230 
5231 	/* It's an encapsulated packet and VxLAN offloads are enabled. We
5232 	 * should disable tunnel offload features if it's not a VxLAN packet,
5233 	 * as tunnel offloads have been enabled only for VxLAN. This is done to
5234 	 * allow other tunneled traffic like GRE work fine while VxLAN
5235 	 * offloads are configured in Skyhawk-R.
5236 	 */
5237 	switch (vlan_get_protocol(skb)) {
5238 	case htons(ETH_P_IP):
5239 		l4_hdr = ip_hdr(skb)->protocol;
5240 		break;
5241 	case htons(ETH_P_IPV6):
5242 		l4_hdr = ipv6_hdr(skb)->nexthdr;
5243 		break;
5244 	default:
5245 		return features;
5246 	}
5247 
5248 	if (l4_hdr != IPPROTO_UDP ||
5249 	    skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
5250 	    skb->inner_protocol != htons(ETH_P_TEB) ||
5251 	    skb_inner_mac_header(skb) - skb_transport_header(skb) !=
5252 		sizeof(struct udphdr) + sizeof(struct vxlanhdr) ||
5253 	    !adapter->vxlan_port ||
5254 	    udp_hdr(skb)->dest != adapter->vxlan_port)
5255 		return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
5256 
5257 	return features;
5258 }
5259 
5260 static int be_get_phys_port_id(struct net_device *dev,
5261 			       struct netdev_phys_item_id *ppid)
5262 {
5263 	int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1;
5264 	struct be_adapter *adapter = netdev_priv(dev);
5265 	u8 *id;
5266 
5267 	if (MAX_PHYS_ITEM_ID_LEN < id_len)
5268 		return -ENOSPC;
5269 
5270 	ppid->id[0] = adapter->hba_port_num + 1;
5271 	id = &ppid->id[1];
5272 	for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0;
5273 	     i--, id += CNTL_SERIAL_NUM_WORD_SZ)
5274 		memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ);
5275 
5276 	ppid->id_len = id_len;
5277 
5278 	return 0;
5279 }
5280 
5281 static void be_set_rx_mode(struct net_device *dev)
5282 {
5283 	struct be_adapter *adapter = netdev_priv(dev);
5284 	struct be_cmd_work *work;
5285 
5286 	work = be_alloc_work(adapter, be_work_set_rx_mode);
5287 	if (work)
5288 		queue_work(be_wq, &work->work);
5289 }
5290 
5291 static const struct net_device_ops be_netdev_ops = {
5292 	.ndo_open		= be_open,
5293 	.ndo_stop		= be_close,
5294 	.ndo_start_xmit		= be_xmit,
5295 	.ndo_set_rx_mode	= be_set_rx_mode,
5296 	.ndo_set_mac_address	= be_mac_addr_set,
5297 	.ndo_get_stats64	= be_get_stats64,
5298 	.ndo_validate_addr	= eth_validate_addr,
5299 	.ndo_vlan_rx_add_vid	= be_vlan_add_vid,
5300 	.ndo_vlan_rx_kill_vid	= be_vlan_rem_vid,
5301 	.ndo_set_vf_mac		= be_set_vf_mac,
5302 	.ndo_set_vf_vlan	= be_set_vf_vlan,
5303 	.ndo_set_vf_rate	= be_set_vf_tx_rate,
5304 	.ndo_get_vf_config	= be_get_vf_config,
5305 	.ndo_set_vf_link_state  = be_set_vf_link_state,
5306 	.ndo_set_vf_spoofchk    = be_set_vf_spoofchk,
5307 	.ndo_tx_timeout		= be_tx_timeout,
5308 #ifdef CONFIG_NET_POLL_CONTROLLER
5309 	.ndo_poll_controller	= be_netpoll,
5310 #endif
5311 	.ndo_bridge_setlink	= be_ndo_bridge_setlink,
5312 	.ndo_bridge_getlink	= be_ndo_bridge_getlink,
5313 	.ndo_udp_tunnel_add	= be_add_vxlan_port,
5314 	.ndo_udp_tunnel_del	= be_del_vxlan_port,
5315 	.ndo_features_check	= be_features_check,
5316 	.ndo_get_phys_port_id   = be_get_phys_port_id,
5317 };
5318 
5319 static void be_netdev_init(struct net_device *netdev)
5320 {
5321 	struct be_adapter *adapter = netdev_priv(netdev);
5322 
5323 	netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5324 		NETIF_F_GSO_UDP_TUNNEL |
5325 		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
5326 		NETIF_F_HW_VLAN_CTAG_TX;
5327 	if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS))
5328 		netdev->hw_features |= NETIF_F_RXHASH;
5329 
5330 	netdev->features |= netdev->hw_features |
5331 		NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
5332 
5333 	netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
5334 		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
5335 
5336 	netdev->priv_flags |= IFF_UNICAST_FLT;
5337 
5338 	netdev->flags |= IFF_MULTICAST;
5339 
5340 	netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN);
5341 
5342 	netdev->netdev_ops = &be_netdev_ops;
5343 
5344 	netdev->ethtool_ops = &be_ethtool_ops;
5345 
5346 	/* MTU range: 256 - 9000 */
5347 	netdev->min_mtu = BE_MIN_MTU;
5348 	netdev->max_mtu = BE_MAX_MTU;
5349 }
5350 
5351 static void be_cleanup(struct be_adapter *adapter)
5352 {
5353 	struct net_device *netdev = adapter->netdev;
5354 
5355 	rtnl_lock();
5356 	netif_device_detach(netdev);
5357 	if (netif_running(netdev))
5358 		be_close(netdev);
5359 	rtnl_unlock();
5360 
5361 	be_clear(adapter);
5362 }
5363 
5364 static int be_resume(struct be_adapter *adapter)
5365 {
5366 	struct net_device *netdev = adapter->netdev;
5367 	int status;
5368 
5369 	status = be_setup(adapter);
5370 	if (status)
5371 		return status;
5372 
5373 	rtnl_lock();
5374 	if (netif_running(netdev))
5375 		status = be_open(netdev);
5376 	rtnl_unlock();
5377 
5378 	if (status)
5379 		return status;
5380 
5381 	netif_device_attach(netdev);
5382 
5383 	return 0;
5384 }
5385 
5386 static void be_soft_reset(struct be_adapter *adapter)
5387 {
5388 	u32 val;
5389 
5390 	dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n");
5391 	val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
5392 	val |= SLIPORT_SOFTRESET_SR_MASK;
5393 	iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET);
5394 }
5395 
5396 static bool be_err_is_recoverable(struct be_adapter *adapter)
5397 {
5398 	struct be_error_recovery *err_rec = &adapter->error_recovery;
5399 	unsigned long initial_idle_time =
5400 		msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME);
5401 	unsigned long recovery_interval =
5402 		msecs_to_jiffies(ERR_RECOVERY_INTERVAL);
5403 	u16 ue_err_code;
5404 	u32 val;
5405 
5406 	val = be_POST_stage_get(adapter);
5407 	if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR)
5408 		return false;
5409 	ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK;
5410 	if (ue_err_code == 0)
5411 		return false;
5412 
5413 	dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n",
5414 		ue_err_code);
5415 
5416 	if (time_before_eq(jiffies - err_rec->probe_time, initial_idle_time)) {
5417 		dev_err(&adapter->pdev->dev,
5418 			"Cannot recover within %lu sec from driver load\n",
5419 			jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC);
5420 		return false;
5421 	}
5422 
5423 	if (err_rec->last_recovery_time && time_before_eq(
5424 		jiffies - err_rec->last_recovery_time, recovery_interval)) {
5425 		dev_err(&adapter->pdev->dev,
5426 			"Cannot recover within %lu sec from last recovery\n",
5427 			jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC);
5428 		return false;
5429 	}
5430 
5431 	if (ue_err_code == err_rec->last_err_code) {
5432 		dev_err(&adapter->pdev->dev,
5433 			"Cannot recover from a consecutive TPE error\n");
5434 		return false;
5435 	}
5436 
5437 	err_rec->last_recovery_time = jiffies;
5438 	err_rec->last_err_code = ue_err_code;
5439 	return true;
5440 }
5441 
5442 static int be_tpe_recover(struct be_adapter *adapter)
5443 {
5444 	struct be_error_recovery *err_rec = &adapter->error_recovery;
5445 	int status = -EAGAIN;
5446 	u32 val;
5447 
5448 	switch (err_rec->recovery_state) {
5449 	case ERR_RECOVERY_ST_NONE:
5450 		err_rec->recovery_state = ERR_RECOVERY_ST_DETECT;
5451 		err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION;
5452 		break;
5453 
5454 	case ERR_RECOVERY_ST_DETECT:
5455 		val = be_POST_stage_get(adapter);
5456 		if ((val & POST_STAGE_RECOVERABLE_ERR) !=
5457 		    POST_STAGE_RECOVERABLE_ERR) {
5458 			dev_err(&adapter->pdev->dev,
5459 				"Unrecoverable HW error detected: 0x%x\n", val);
5460 			status = -EINVAL;
5461 			err_rec->resched_delay = 0;
5462 			break;
5463 		}
5464 
5465 		dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n");
5466 
5467 		/* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR
5468 		 * milliseconds before it checks for final error status in
5469 		 * SLIPORT_SEMAPHORE to determine if recovery criteria is met.
5470 		 * If it does, then PF0 initiates a Soft Reset.
5471 		 */
5472 		if (adapter->pf_num == 0) {
5473 			err_rec->recovery_state = ERR_RECOVERY_ST_RESET;
5474 			err_rec->resched_delay = err_rec->ue_to_reset_time -
5475 					ERR_RECOVERY_UE_DETECT_DURATION;
5476 			break;
5477 		}
5478 
5479 		err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
5480 		err_rec->resched_delay = err_rec->ue_to_poll_time -
5481 					ERR_RECOVERY_UE_DETECT_DURATION;
5482 		break;
5483 
5484 	case ERR_RECOVERY_ST_RESET:
5485 		if (!be_err_is_recoverable(adapter)) {
5486 			dev_err(&adapter->pdev->dev,
5487 				"Failed to meet recovery criteria\n");
5488 			status = -EIO;
5489 			err_rec->resched_delay = 0;
5490 			break;
5491 		}
5492 		be_soft_reset(adapter);
5493 		err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL;
5494 		err_rec->resched_delay = err_rec->ue_to_poll_time -
5495 					err_rec->ue_to_reset_time;
5496 		break;
5497 
5498 	case ERR_RECOVERY_ST_PRE_POLL:
5499 		err_rec->recovery_state = ERR_RECOVERY_ST_REINIT;
5500 		err_rec->resched_delay = 0;
5501 		status = 0;			/* done */
5502 		break;
5503 
5504 	default:
5505 		status = -EINVAL;
5506 		err_rec->resched_delay = 0;
5507 		break;
5508 	}
5509 
5510 	return status;
5511 }
5512 
5513 static int be_err_recover(struct be_adapter *adapter)
5514 {
5515 	int status;
5516 
5517 	if (!lancer_chip(adapter)) {
5518 		if (!adapter->error_recovery.recovery_supported ||
5519 		    adapter->priv_flags & BE_DISABLE_TPE_RECOVERY)
5520 			return -EIO;
5521 		status = be_tpe_recover(adapter);
5522 		if (status)
5523 			goto err;
5524 	}
5525 
5526 	/* Wait for adapter to reach quiescent state before
5527 	 * destroying queues
5528 	 */
5529 	status = be_fw_wait_ready(adapter);
5530 	if (status)
5531 		goto err;
5532 
5533 	adapter->flags |= BE_FLAGS_TRY_RECOVERY;
5534 
5535 	be_cleanup(adapter);
5536 
5537 	status = be_resume(adapter);
5538 	if (status)
5539 		goto err;
5540 
5541 	adapter->flags &= ~BE_FLAGS_TRY_RECOVERY;
5542 
5543 err:
5544 	return status;
5545 }
5546 
5547 static void be_err_detection_task(struct work_struct *work)
5548 {
5549 	struct be_error_recovery *err_rec =
5550 			container_of(work, struct be_error_recovery,
5551 				     err_detection_work.work);
5552 	struct be_adapter *adapter =
5553 			container_of(err_rec, struct be_adapter,
5554 				     error_recovery);
5555 	u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY;
5556 	struct device *dev = &adapter->pdev->dev;
5557 	int recovery_status;
5558 
5559 	be_detect_error(adapter);
5560 	if (!be_check_error(adapter, BE_ERROR_HW))
5561 		goto reschedule_task;
5562 
5563 	recovery_status = be_err_recover(adapter);
5564 	if (!recovery_status) {
5565 		err_rec->recovery_retries = 0;
5566 		err_rec->recovery_state = ERR_RECOVERY_ST_NONE;
5567 		dev_info(dev, "Adapter recovery successful\n");
5568 		goto reschedule_task;
5569 	} else if (!lancer_chip(adapter) && err_rec->resched_delay) {
5570 		/* BEx/SH recovery state machine */
5571 		if (adapter->pf_num == 0 &&
5572 		    err_rec->recovery_state > ERR_RECOVERY_ST_DETECT)
5573 			dev_err(&adapter->pdev->dev,
5574 				"Adapter recovery in progress\n");
5575 		resched_delay = err_rec->resched_delay;
5576 		goto reschedule_task;
5577 	} else if (lancer_chip(adapter) && be_virtfn(adapter)) {
5578 		/* For VFs, check if PF have allocated resources
5579 		 * every second.
5580 		 */
5581 		dev_err(dev, "Re-trying adapter recovery\n");
5582 		goto reschedule_task;
5583 	} else if (lancer_chip(adapter) && err_rec->recovery_retries++ <
5584 		   ERR_RECOVERY_MAX_RETRY_COUNT) {
5585 		/* In case of another error during recovery, it takes 30 sec
5586 		 * for adapter to come out of error. Retry error recovery after
5587 		 * this time interval.
5588 		 */
5589 		dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n");
5590 		resched_delay = ERR_RECOVERY_RETRY_DELAY;
5591 		goto reschedule_task;
5592 	} else {
5593 		dev_err(dev, "Adapter recovery failed\n");
5594 		dev_err(dev, "Please reboot server to recover\n");
5595 	}
5596 
5597 	return;
5598 
5599 reschedule_task:
5600 	be_schedule_err_detection(adapter, resched_delay);
5601 }
5602 
5603 static void be_log_sfp_info(struct be_adapter *adapter)
5604 {
5605 	int status;
5606 
5607 	status = be_cmd_query_sfp_info(adapter);
5608 	if (!status) {
5609 		dev_err(&adapter->pdev->dev,
5610 			"Port %c: %s Vendor: %s part no: %s",
5611 			adapter->port_name,
5612 			be_misconfig_evt_port_state[adapter->phy_state],
5613 			adapter->phy.vendor_name,
5614 			adapter->phy.vendor_pn);
5615 	}
5616 	adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED;
5617 }
5618 
5619 static void be_worker(struct work_struct *work)
5620 {
5621 	struct be_adapter *adapter =
5622 		container_of(work, struct be_adapter, work.work);
5623 	struct be_rx_obj *rxo;
5624 	int i;
5625 
5626 	if (be_physfn(adapter) &&
5627 	    MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
5628 		be_cmd_get_die_temperature(adapter);
5629 
5630 	/* when interrupts are not yet enabled, just reap any pending
5631 	 * mcc completions
5632 	 */
5633 	if (!netif_running(adapter->netdev)) {
5634 		be_process_mcc(adapter);
5635 		goto reschedule;
5636 	}
5637 
5638 	if (!adapter->stats_cmd_sent) {
5639 		if (lancer_chip(adapter))
5640 			lancer_cmd_get_pport_stats(adapter,
5641 						   &adapter->stats_cmd);
5642 		else
5643 			be_cmd_get_stats(adapter, &adapter->stats_cmd);
5644 	}
5645 
5646 	for_all_rx_queues(adapter, rxo, i) {
5647 		/* Replenish RX-queues starved due to memory
5648 		 * allocation failures.
5649 		 */
5650 		if (rxo->rx_post_starved)
5651 			be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
5652 	}
5653 
5654 	/* EQ-delay update for Skyhawk is done while notifying EQ */
5655 	if (!skyhawk_chip(adapter))
5656 		be_eqd_update(adapter, false);
5657 
5658 	if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED)
5659 		be_log_sfp_info(adapter);
5660 
5661 reschedule:
5662 	adapter->work_counter++;
5663 	queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000));
5664 }
5665 
5666 static void be_unmap_pci_bars(struct be_adapter *adapter)
5667 {
5668 	if (adapter->csr)
5669 		pci_iounmap(adapter->pdev, adapter->csr);
5670 	if (adapter->db)
5671 		pci_iounmap(adapter->pdev, adapter->db);
5672 	if (adapter->pcicfg && adapter->pcicfg_mapped)
5673 		pci_iounmap(adapter->pdev, adapter->pcicfg);
5674 }
5675 
5676 static int db_bar(struct be_adapter *adapter)
5677 {
5678 	if (lancer_chip(adapter) || be_virtfn(adapter))
5679 		return 0;
5680 	else
5681 		return 4;
5682 }
5683 
5684 static int be_roce_map_pci_bars(struct be_adapter *adapter)
5685 {
5686 	if (skyhawk_chip(adapter)) {
5687 		adapter->roce_db.size = 4096;
5688 		adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
5689 							      db_bar(adapter));
5690 		adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
5691 							       db_bar(adapter));
5692 	}
5693 	return 0;
5694 }
5695 
5696 static int be_map_pci_bars(struct be_adapter *adapter)
5697 {
5698 	struct pci_dev *pdev = adapter->pdev;
5699 	u8 __iomem *addr;
5700 	u32 sli_intf;
5701 
5702 	pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
5703 	adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
5704 				SLI_INTF_FAMILY_SHIFT;
5705 	adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
5706 
5707 	if (BEx_chip(adapter) && be_physfn(adapter)) {
5708 		adapter->csr = pci_iomap(pdev, 2, 0);
5709 		if (!adapter->csr)
5710 			return -ENOMEM;
5711 	}
5712 
5713 	addr = pci_iomap(pdev, db_bar(adapter), 0);
5714 	if (!addr)
5715 		goto pci_map_err;
5716 	adapter->db = addr;
5717 
5718 	if (skyhawk_chip(adapter) || BEx_chip(adapter)) {
5719 		if (be_physfn(adapter)) {
5720 			/* PCICFG is the 2nd BAR in BE2 */
5721 			addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0);
5722 			if (!addr)
5723 				goto pci_map_err;
5724 			adapter->pcicfg = addr;
5725 			adapter->pcicfg_mapped = true;
5726 		} else {
5727 			adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET;
5728 			adapter->pcicfg_mapped = false;
5729 		}
5730 	}
5731 
5732 	be_roce_map_pci_bars(adapter);
5733 	return 0;
5734 
5735 pci_map_err:
5736 	dev_err(&pdev->dev, "Error in mapping PCI BARs\n");
5737 	be_unmap_pci_bars(adapter);
5738 	return -ENOMEM;
5739 }
5740 
5741 static void be_drv_cleanup(struct be_adapter *adapter)
5742 {
5743 	struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
5744 	struct device *dev = &adapter->pdev->dev;
5745 
5746 	if (mem->va)
5747 		dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5748 
5749 	mem = &adapter->rx_filter;
5750 	if (mem->va)
5751 		dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5752 
5753 	mem = &adapter->stats_cmd;
5754 	if (mem->va)
5755 		dma_free_coherent(dev, mem->size, mem->va, mem->dma);
5756 }
5757 
5758 /* Allocate and initialize various fields in be_adapter struct */
5759 static int be_drv_init(struct be_adapter *adapter)
5760 {
5761 	struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
5762 	struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
5763 	struct be_dma_mem *rx_filter = &adapter->rx_filter;
5764 	struct be_dma_mem *stats_cmd = &adapter->stats_cmd;
5765 	struct device *dev = &adapter->pdev->dev;
5766 	int status = 0;
5767 
5768 	mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
5769 	mbox_mem_alloc->va = dma_alloc_coherent(dev, mbox_mem_alloc->size,
5770 						&mbox_mem_alloc->dma,
5771 						GFP_KERNEL);
5772 	if (!mbox_mem_alloc->va)
5773 		return -ENOMEM;
5774 
5775 	mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
5776 	mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
5777 	mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
5778 
5779 	rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
5780 	rx_filter->va = dma_alloc_coherent(dev, rx_filter->size,
5781 					   &rx_filter->dma, GFP_KERNEL);
5782 	if (!rx_filter->va) {
5783 		status = -ENOMEM;
5784 		goto free_mbox;
5785 	}
5786 
5787 	if (lancer_chip(adapter))
5788 		stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
5789 	else if (BE2_chip(adapter))
5790 		stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
5791 	else if (BE3_chip(adapter))
5792 		stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
5793 	else
5794 		stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
5795 	stats_cmd->va = dma_alloc_coherent(dev, stats_cmd->size,
5796 					   &stats_cmd->dma, GFP_KERNEL);
5797 	if (!stats_cmd->va) {
5798 		status = -ENOMEM;
5799 		goto free_rx_filter;
5800 	}
5801 
5802 	mutex_init(&adapter->mbox_lock);
5803 	mutex_init(&adapter->mcc_lock);
5804 	mutex_init(&adapter->rx_filter_lock);
5805 	spin_lock_init(&adapter->mcc_cq_lock);
5806 	init_completion(&adapter->et_cmd_compl);
5807 
5808 	pci_save_state(adapter->pdev);
5809 
5810 	INIT_DELAYED_WORK(&adapter->work, be_worker);
5811 
5812 	adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE;
5813 	adapter->error_recovery.resched_delay = 0;
5814 	INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work,
5815 			  be_err_detection_task);
5816 
5817 	adapter->rx_fc = true;
5818 	adapter->tx_fc = true;
5819 
5820 	/* Must be a power of 2 or else MODULO will BUG_ON */
5821 	adapter->be_get_temp_freq = 64;
5822 
5823 	INIT_LIST_HEAD(&adapter->vxlan_port_list);
5824 	return 0;
5825 
5826 free_rx_filter:
5827 	dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma);
5828 free_mbox:
5829 	dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va,
5830 			  mbox_mem_alloc->dma);
5831 	return status;
5832 }
5833 
5834 static void be_remove(struct pci_dev *pdev)
5835 {
5836 	struct be_adapter *adapter = pci_get_drvdata(pdev);
5837 
5838 	if (!adapter)
5839 		return;
5840 
5841 	be_roce_dev_remove(adapter);
5842 	be_intr_set(adapter, false);
5843 
5844 	be_cancel_err_detection(adapter);
5845 
5846 	unregister_netdev(adapter->netdev);
5847 
5848 	be_clear(adapter);
5849 
5850 	if (!pci_vfs_assigned(adapter->pdev))
5851 		be_cmd_reset_function(adapter);
5852 
5853 	/* tell fw we're done with firing cmds */
5854 	be_cmd_fw_clean(adapter);
5855 
5856 	be_unmap_pci_bars(adapter);
5857 	be_drv_cleanup(adapter);
5858 
5859 	pci_disable_pcie_error_reporting(pdev);
5860 
5861 	pci_release_regions(pdev);
5862 	pci_disable_device(pdev);
5863 
5864 	free_netdev(adapter->netdev);
5865 }
5866 
5867 static ssize_t be_hwmon_show_temp(struct device *dev,
5868 				  struct device_attribute *dev_attr,
5869 				  char *buf)
5870 {
5871 	struct be_adapter *adapter = dev_get_drvdata(dev);
5872 
5873 	/* Unit: millidegree Celsius */
5874 	if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP)
5875 		return -EIO;
5876 	else
5877 		return sprintf(buf, "%u\n",
5878 			       adapter->hwmon_info.be_on_die_temp * 1000);
5879 }
5880 
5881 static SENSOR_DEVICE_ATTR(temp1_input, 0444,
5882 			  be_hwmon_show_temp, NULL, 1);
5883 
5884 static struct attribute *be_hwmon_attrs[] = {
5885 	&sensor_dev_attr_temp1_input.dev_attr.attr,
5886 	NULL
5887 };
5888 
5889 ATTRIBUTE_GROUPS(be_hwmon);
5890 
5891 static char *mc_name(struct be_adapter *adapter)
5892 {
5893 	char *str = "";	/* default */
5894 
5895 	switch (adapter->mc_type) {
5896 	case UMC:
5897 		str = "UMC";
5898 		break;
5899 	case FLEX10:
5900 		str = "FLEX10";
5901 		break;
5902 	case vNIC1:
5903 		str = "vNIC-1";
5904 		break;
5905 	case nPAR:
5906 		str = "nPAR";
5907 		break;
5908 	case UFP:
5909 		str = "UFP";
5910 		break;
5911 	case vNIC2:
5912 		str = "vNIC-2";
5913 		break;
5914 	default:
5915 		str = "";
5916 	}
5917 
5918 	return str;
5919 }
5920 
5921 static inline char *func_name(struct be_adapter *adapter)
5922 {
5923 	return be_physfn(adapter) ? "PF" : "VF";
5924 }
5925 
5926 static inline char *nic_name(struct pci_dev *pdev)
5927 {
5928 	switch (pdev->device) {
5929 	case OC_DEVICE_ID1:
5930 		return OC_NAME;
5931 	case OC_DEVICE_ID2:
5932 		return OC_NAME_BE;
5933 	case OC_DEVICE_ID3:
5934 	case OC_DEVICE_ID4:
5935 		return OC_NAME_LANCER;
5936 	case BE_DEVICE_ID2:
5937 		return BE3_NAME;
5938 	case OC_DEVICE_ID5:
5939 	case OC_DEVICE_ID6:
5940 		return OC_NAME_SH;
5941 	default:
5942 		return BE_NAME;
5943 	}
5944 }
5945 
5946 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
5947 {
5948 	struct be_adapter *adapter;
5949 	struct net_device *netdev;
5950 	int status = 0;
5951 
5952 	dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
5953 
5954 	status = pci_enable_device(pdev);
5955 	if (status)
5956 		goto do_none;
5957 
5958 	status = pci_request_regions(pdev, DRV_NAME);
5959 	if (status)
5960 		goto disable_dev;
5961 	pci_set_master(pdev);
5962 
5963 	netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
5964 	if (!netdev) {
5965 		status = -ENOMEM;
5966 		goto rel_reg;
5967 	}
5968 	adapter = netdev_priv(netdev);
5969 	adapter->pdev = pdev;
5970 	pci_set_drvdata(pdev, adapter);
5971 	adapter->netdev = netdev;
5972 	SET_NETDEV_DEV(netdev, &pdev->dev);
5973 
5974 	status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5975 	if (!status) {
5976 		netdev->features |= NETIF_F_HIGHDMA;
5977 	} else {
5978 		status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
5979 		if (status) {
5980 			dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
5981 			goto free_netdev;
5982 		}
5983 	}
5984 
5985 	status = pci_enable_pcie_error_reporting(pdev);
5986 	if (!status)
5987 		dev_info(&pdev->dev, "PCIe error reporting enabled\n");
5988 
5989 	status = be_map_pci_bars(adapter);
5990 	if (status)
5991 		goto free_netdev;
5992 
5993 	status = be_drv_init(adapter);
5994 	if (status)
5995 		goto unmap_bars;
5996 
5997 	status = be_setup(adapter);
5998 	if (status)
5999 		goto drv_cleanup;
6000 
6001 	be_netdev_init(netdev);
6002 	status = register_netdev(netdev);
6003 	if (status != 0)
6004 		goto unsetup;
6005 
6006 	be_roce_dev_add(adapter);
6007 
6008 	be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
6009 	adapter->error_recovery.probe_time = jiffies;
6010 
6011 	/* On Die temperature not supported for VF. */
6012 	if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) {
6013 		adapter->hwmon_info.hwmon_dev =
6014 			devm_hwmon_device_register_with_groups(&pdev->dev,
6015 							       DRV_NAME,
6016 							       adapter,
6017 							       be_hwmon_groups);
6018 		adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP;
6019 	}
6020 
6021 	dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
6022 		 func_name(adapter), mc_name(adapter), adapter->port_name);
6023 
6024 	return 0;
6025 
6026 unsetup:
6027 	be_clear(adapter);
6028 drv_cleanup:
6029 	be_drv_cleanup(adapter);
6030 unmap_bars:
6031 	be_unmap_pci_bars(adapter);
6032 free_netdev:
6033 	free_netdev(netdev);
6034 rel_reg:
6035 	pci_release_regions(pdev);
6036 disable_dev:
6037 	pci_disable_device(pdev);
6038 do_none:
6039 	dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
6040 	return status;
6041 }
6042 
6043 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
6044 {
6045 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6046 
6047 	be_intr_set(adapter, false);
6048 	be_cancel_err_detection(adapter);
6049 
6050 	be_cleanup(adapter);
6051 
6052 	pci_save_state(pdev);
6053 	pci_disable_device(pdev);
6054 	pci_set_power_state(pdev, pci_choose_state(pdev, state));
6055 	return 0;
6056 }
6057 
6058 static int be_pci_resume(struct pci_dev *pdev)
6059 {
6060 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6061 	int status = 0;
6062 
6063 	status = pci_enable_device(pdev);
6064 	if (status)
6065 		return status;
6066 
6067 	pci_restore_state(pdev);
6068 
6069 	status = be_resume(adapter);
6070 	if (status)
6071 		return status;
6072 
6073 	be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
6074 
6075 	return 0;
6076 }
6077 
6078 /*
6079  * An FLR will stop BE from DMAing any data.
6080  */
6081 static void be_shutdown(struct pci_dev *pdev)
6082 {
6083 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6084 
6085 	if (!adapter)
6086 		return;
6087 
6088 	be_roce_dev_shutdown(adapter);
6089 	cancel_delayed_work_sync(&adapter->work);
6090 	be_cancel_err_detection(adapter);
6091 
6092 	netif_device_detach(adapter->netdev);
6093 
6094 	be_cmd_reset_function(adapter);
6095 
6096 	pci_disable_device(pdev);
6097 }
6098 
6099 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
6100 					    pci_channel_state_t state)
6101 {
6102 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6103 
6104 	dev_err(&adapter->pdev->dev, "EEH error detected\n");
6105 
6106 	be_roce_dev_remove(adapter);
6107 
6108 	if (!be_check_error(adapter, BE_ERROR_EEH)) {
6109 		be_set_error(adapter, BE_ERROR_EEH);
6110 
6111 		be_cancel_err_detection(adapter);
6112 
6113 		be_cleanup(adapter);
6114 	}
6115 
6116 	if (state == pci_channel_io_perm_failure)
6117 		return PCI_ERS_RESULT_DISCONNECT;
6118 
6119 	pci_disable_device(pdev);
6120 
6121 	/* The error could cause the FW to trigger a flash debug dump.
6122 	 * Resetting the card while flash dump is in progress
6123 	 * can cause it not to recover; wait for it to finish.
6124 	 * Wait only for first function as it is needed only once per
6125 	 * adapter.
6126 	 */
6127 	if (pdev->devfn == 0)
6128 		ssleep(30);
6129 
6130 	return PCI_ERS_RESULT_NEED_RESET;
6131 }
6132 
6133 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
6134 {
6135 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6136 	int status;
6137 
6138 	dev_info(&adapter->pdev->dev, "EEH reset\n");
6139 
6140 	status = pci_enable_device(pdev);
6141 	if (status)
6142 		return PCI_ERS_RESULT_DISCONNECT;
6143 
6144 	pci_set_master(pdev);
6145 	pci_restore_state(pdev);
6146 
6147 	/* Check if card is ok and fw is ready */
6148 	dev_info(&adapter->pdev->dev,
6149 		 "Waiting for FW to be ready after EEH reset\n");
6150 	status = be_fw_wait_ready(adapter);
6151 	if (status)
6152 		return PCI_ERS_RESULT_DISCONNECT;
6153 
6154 	be_clear_error(adapter, BE_CLEAR_ALL);
6155 	return PCI_ERS_RESULT_RECOVERED;
6156 }
6157 
6158 static void be_eeh_resume(struct pci_dev *pdev)
6159 {
6160 	int status = 0;
6161 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6162 
6163 	dev_info(&adapter->pdev->dev, "EEH resume\n");
6164 
6165 	pci_save_state(pdev);
6166 
6167 	status = be_resume(adapter);
6168 	if (status)
6169 		goto err;
6170 
6171 	be_roce_dev_add(adapter);
6172 
6173 	be_schedule_err_detection(adapter, ERR_DETECTION_DELAY);
6174 	return;
6175 err:
6176 	dev_err(&adapter->pdev->dev, "EEH resume failed\n");
6177 }
6178 
6179 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
6180 {
6181 	struct be_adapter *adapter = pci_get_drvdata(pdev);
6182 	struct be_resources vft_res = {0};
6183 	int status;
6184 
6185 	if (!num_vfs)
6186 		be_vf_clear(adapter);
6187 
6188 	adapter->num_vfs = num_vfs;
6189 
6190 	if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) {
6191 		dev_warn(&pdev->dev,
6192 			 "Cannot disable VFs while they are assigned\n");
6193 		return -EBUSY;
6194 	}
6195 
6196 	/* When the HW is in SRIOV capable configuration, the PF-pool resources
6197 	 * are equally distributed across the max-number of VFs. The user may
6198 	 * request only a subset of the max-vfs to be enabled.
6199 	 * Based on num_vfs, redistribute the resources across num_vfs so that
6200 	 * each VF will have access to more number of resources.
6201 	 * This facility is not available in BE3 FW.
6202 	 * Also, this is done by FW in Lancer chip.
6203 	 */
6204 	if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) {
6205 		be_calculate_vf_res(adapter, adapter->num_vfs,
6206 				    &vft_res);
6207 		status = be_cmd_set_sriov_config(adapter, adapter->pool_res,
6208 						 adapter->num_vfs, &vft_res);
6209 		if (status)
6210 			dev_err(&pdev->dev,
6211 				"Failed to optimize SR-IOV resources\n");
6212 	}
6213 
6214 	status = be_get_resources(adapter);
6215 	if (status)
6216 		return be_cmd_status(status);
6217 
6218 	/* Updating real_num_tx/rx_queues() requires rtnl_lock() */
6219 	rtnl_lock();
6220 	status = be_update_queues(adapter);
6221 	rtnl_unlock();
6222 	if (status)
6223 		return be_cmd_status(status);
6224 
6225 	if (adapter->num_vfs)
6226 		status = be_vf_setup(adapter);
6227 
6228 	if (!status)
6229 		return adapter->num_vfs;
6230 
6231 	return 0;
6232 }
6233 
6234 static const struct pci_error_handlers be_eeh_handlers = {
6235 	.error_detected = be_eeh_err_detected,
6236 	.slot_reset = be_eeh_reset,
6237 	.resume = be_eeh_resume,
6238 };
6239 
6240 static struct pci_driver be_driver = {
6241 	.name = DRV_NAME,
6242 	.id_table = be_dev_ids,
6243 	.probe = be_probe,
6244 	.remove = be_remove,
6245 	.suspend = be_suspend,
6246 	.resume = be_pci_resume,
6247 	.shutdown = be_shutdown,
6248 	.sriov_configure = be_pci_sriov_configure,
6249 	.err_handler = &be_eeh_handlers
6250 };
6251 
6252 static int __init be_init_module(void)
6253 {
6254 	int status;
6255 
6256 	if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
6257 	    rx_frag_size != 2048) {
6258 		printk(KERN_WARNING DRV_NAME
6259 			" : Module param rx_frag_size must be 2048/4096/8192."
6260 			" Using 2048\n");
6261 		rx_frag_size = 2048;
6262 	}
6263 
6264 	if (num_vfs > 0) {
6265 		pr_info(DRV_NAME " : Module param num_vfs is obsolete.");
6266 		pr_info(DRV_NAME " : Use sysfs method to enable VFs\n");
6267 	}
6268 
6269 	be_wq = create_singlethread_workqueue("be_wq");
6270 	if (!be_wq) {
6271 		pr_warn(DRV_NAME "workqueue creation failed\n");
6272 		return -1;
6273 	}
6274 
6275 	be_err_recovery_workq =
6276 		create_singlethread_workqueue("be_err_recover");
6277 	if (!be_err_recovery_workq)
6278 		pr_warn(DRV_NAME "Could not create error recovery workqueue\n");
6279 
6280 	status = pci_register_driver(&be_driver);
6281 	if (status) {
6282 		destroy_workqueue(be_wq);
6283 		be_destroy_err_recovery_workq();
6284 	}
6285 	return status;
6286 }
6287 module_init(be_init_module);
6288 
6289 static void __exit be_exit_module(void)
6290 {
6291 	pci_unregister_driver(&be_driver);
6292 
6293 	be_destroy_err_recovery_workq();
6294 
6295 	if (be_wq)
6296 		destroy_workqueue(be_wq);
6297 }
6298 module_exit(be_exit_module);
6299