1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2007 - 2018 Intel Corporation. */
3 
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5 
6 #include <linux/module.h>
7 #include <linux/types.h>
8 #include <linux/init.h>
9 #include <linux/bitops.h>
10 #include <linux/vmalloc.h>
11 #include <linux/pagemap.h>
12 #include <linux/netdevice.h>
13 #include <linux/ipv6.h>
14 #include <linux/slab.h>
15 #include <net/checksum.h>
16 #include <net/ip6_checksum.h>
17 #include <net/pkt_sched.h>
18 #include <net/pkt_cls.h>
19 #include <linux/net_tstamp.h>
20 #include <linux/mii.h>
21 #include <linux/ethtool.h>
22 #include <linux/if.h>
23 #include <linux/if_vlan.h>
24 #include <linux/pci.h>
25 #include <linux/delay.h>
26 #include <linux/interrupt.h>
27 #include <linux/ip.h>
28 #include <linux/tcp.h>
29 #include <linux/sctp.h>
30 #include <linux/if_ether.h>
31 #include <linux/aer.h>
32 #include <linux/prefetch.h>
33 #include <linux/bpf.h>
34 #include <linux/bpf_trace.h>
35 #include <linux/pm_runtime.h>
36 #include <linux/etherdevice.h>
37 #ifdef CONFIG_IGB_DCA
38 #include <linux/dca.h>
39 #endif
40 #include <linux/i2c.h>
41 #include "igb.h"
42 
43 enum queue_mode {
44 	QUEUE_MODE_STRICT_PRIORITY,
45 	QUEUE_MODE_STREAM_RESERVATION,
46 };
47 
48 enum tx_queue_prio {
49 	TX_QUEUE_PRIO_HIGH,
50 	TX_QUEUE_PRIO_LOW,
51 };
52 
53 char igb_driver_name[] = "igb";
54 static const char igb_driver_string[] =
55 				"Intel(R) Gigabit Ethernet Network Driver";
56 static const char igb_copyright[] =
57 				"Copyright (c) 2007-2014 Intel Corporation.";
58 
59 static const struct e1000_info *igb_info_tbl[] = {
60 	[board_82575] = &e1000_82575_info,
61 };
62 
63 static const struct pci_device_id igb_pci_tbl[] = {
64 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
65 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
66 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
67 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
68 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
69 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
70 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
71 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
72 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
73 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
74 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
75 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
76 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
77 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
78 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
79 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
80 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
81 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
82 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
83 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
84 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
85 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
86 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
87 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
88 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
89 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
90 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
91 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
92 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
93 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
94 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
95 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
96 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
97 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
98 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
99 	/* required last entry */
100 	{0, }
101 };
102 
103 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
104 
105 static int igb_setup_all_tx_resources(struct igb_adapter *);
106 static int igb_setup_all_rx_resources(struct igb_adapter *);
107 static void igb_free_all_tx_resources(struct igb_adapter *);
108 static void igb_free_all_rx_resources(struct igb_adapter *);
109 static void igb_setup_mrqc(struct igb_adapter *);
110 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
111 static void igb_remove(struct pci_dev *pdev);
112 static int igb_sw_init(struct igb_adapter *);
113 int igb_open(struct net_device *);
114 int igb_close(struct net_device *);
115 static void igb_configure(struct igb_adapter *);
116 static void igb_configure_tx(struct igb_adapter *);
117 static void igb_configure_rx(struct igb_adapter *);
118 static void igb_clean_all_tx_rings(struct igb_adapter *);
119 static void igb_clean_all_rx_rings(struct igb_adapter *);
120 static void igb_clean_tx_ring(struct igb_ring *);
121 static void igb_clean_rx_ring(struct igb_ring *);
122 static void igb_set_rx_mode(struct net_device *);
123 static void igb_update_phy_info(struct timer_list *);
124 static void igb_watchdog(struct timer_list *);
125 static void igb_watchdog_task(struct work_struct *);
126 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
127 static void igb_get_stats64(struct net_device *dev,
128 			    struct rtnl_link_stats64 *stats);
129 static int igb_change_mtu(struct net_device *, int);
130 static int igb_set_mac(struct net_device *, void *);
131 static void igb_set_uta(struct igb_adapter *adapter, bool set);
132 static irqreturn_t igb_intr(int irq, void *);
133 static irqreturn_t igb_intr_msi(int irq, void *);
134 static irqreturn_t igb_msix_other(int irq, void *);
135 static irqreturn_t igb_msix_ring(int irq, void *);
136 #ifdef CONFIG_IGB_DCA
137 static void igb_update_dca(struct igb_q_vector *);
138 static void igb_setup_dca(struct igb_adapter *);
139 #endif /* CONFIG_IGB_DCA */
140 static int igb_poll(struct napi_struct *, int);
141 static bool igb_clean_tx_irq(struct igb_q_vector *, int);
142 static int igb_clean_rx_irq(struct igb_q_vector *, int);
143 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
144 static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
145 static void igb_reset_task(struct work_struct *);
146 static void igb_vlan_mode(struct net_device *netdev,
147 			  netdev_features_t features);
148 static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
149 static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
150 static void igb_restore_vlan(struct igb_adapter *);
151 static void igb_rar_set_index(struct igb_adapter *, u32);
152 static void igb_ping_all_vfs(struct igb_adapter *);
153 static void igb_msg_task(struct igb_adapter *);
154 static void igb_vmm_control(struct igb_adapter *);
155 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
156 static void igb_flush_mac_table(struct igb_adapter *);
157 static int igb_available_rars(struct igb_adapter *, u8);
158 static void igb_set_default_mac_filter(struct igb_adapter *);
159 static int igb_uc_sync(struct net_device *, const unsigned char *);
160 static int igb_uc_unsync(struct net_device *, const unsigned char *);
161 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
162 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
163 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
164 			       int vf, u16 vlan, u8 qos, __be16 vlan_proto);
165 static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
166 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
167 				   bool setting);
168 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
169 				bool setting);
170 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
171 				 struct ifla_vf_info *ivi);
172 static void igb_check_vf_rate_limit(struct igb_adapter *);
173 static void igb_nfc_filter_exit(struct igb_adapter *adapter);
174 static void igb_nfc_filter_restore(struct igb_adapter *adapter);
175 
176 #ifdef CONFIG_PCI_IOV
177 static int igb_vf_configure(struct igb_adapter *adapter, int vf);
178 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
179 static int igb_disable_sriov(struct pci_dev *dev);
180 static int igb_pci_disable_sriov(struct pci_dev *dev);
181 #endif
182 
183 static int igb_suspend(struct device *);
184 static int igb_resume(struct device *);
185 static int igb_runtime_suspend(struct device *dev);
186 static int igb_runtime_resume(struct device *dev);
187 static int igb_runtime_idle(struct device *dev);
188 static const struct dev_pm_ops igb_pm_ops = {
189 	SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
190 	SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
191 			igb_runtime_idle)
192 };
193 static void igb_shutdown(struct pci_dev *);
194 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
195 #ifdef CONFIG_IGB_DCA
196 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
197 static struct notifier_block dca_notifier = {
198 	.notifier_call	= igb_notify_dca,
199 	.next		= NULL,
200 	.priority	= 0
201 };
202 #endif
203 #ifdef CONFIG_PCI_IOV
204 static unsigned int max_vfs;
205 module_param(max_vfs, uint, 0);
206 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
207 #endif /* CONFIG_PCI_IOV */
208 
209 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
210 		     pci_channel_state_t);
211 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
212 static void igb_io_resume(struct pci_dev *);
213 
214 static const struct pci_error_handlers igb_err_handler = {
215 	.error_detected = igb_io_error_detected,
216 	.slot_reset = igb_io_slot_reset,
217 	.resume = igb_io_resume,
218 };
219 
220 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
221 
222 static struct pci_driver igb_driver = {
223 	.name     = igb_driver_name,
224 	.id_table = igb_pci_tbl,
225 	.probe    = igb_probe,
226 	.remove   = igb_remove,
227 #ifdef CONFIG_PM
228 	.driver.pm = &igb_pm_ops,
229 #endif
230 	.shutdown = igb_shutdown,
231 	.sriov_configure = igb_pci_sriov_configure,
232 	.err_handler = &igb_err_handler
233 };
234 
235 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
236 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
237 MODULE_LICENSE("GPL v2");
238 
239 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
240 static int debug = -1;
241 module_param(debug, int, 0);
242 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
243 
244 struct igb_reg_info {
245 	u32 ofs;
246 	char *name;
247 };
248 
249 static const struct igb_reg_info igb_reg_info_tbl[] = {
250 
251 	/* General Registers */
252 	{E1000_CTRL, "CTRL"},
253 	{E1000_STATUS, "STATUS"},
254 	{E1000_CTRL_EXT, "CTRL_EXT"},
255 
256 	/* Interrupt Registers */
257 	{E1000_ICR, "ICR"},
258 
259 	/* RX Registers */
260 	{E1000_RCTL, "RCTL"},
261 	{E1000_RDLEN(0), "RDLEN"},
262 	{E1000_RDH(0), "RDH"},
263 	{E1000_RDT(0), "RDT"},
264 	{E1000_RXDCTL(0), "RXDCTL"},
265 	{E1000_RDBAL(0), "RDBAL"},
266 	{E1000_RDBAH(0), "RDBAH"},
267 
268 	/* TX Registers */
269 	{E1000_TCTL, "TCTL"},
270 	{E1000_TDBAL(0), "TDBAL"},
271 	{E1000_TDBAH(0), "TDBAH"},
272 	{E1000_TDLEN(0), "TDLEN"},
273 	{E1000_TDH(0), "TDH"},
274 	{E1000_TDT(0), "TDT"},
275 	{E1000_TXDCTL(0), "TXDCTL"},
276 	{E1000_TDFH, "TDFH"},
277 	{E1000_TDFT, "TDFT"},
278 	{E1000_TDFHS, "TDFHS"},
279 	{E1000_TDFPC, "TDFPC"},
280 
281 	/* List Terminator */
282 	{}
283 };
284 
285 /* igb_regdump - register printout routine */
286 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
287 {
288 	int n = 0;
289 	char rname[16];
290 	u32 regs[8];
291 
292 	switch (reginfo->ofs) {
293 	case E1000_RDLEN(0):
294 		for (n = 0; n < 4; n++)
295 			regs[n] = rd32(E1000_RDLEN(n));
296 		break;
297 	case E1000_RDH(0):
298 		for (n = 0; n < 4; n++)
299 			regs[n] = rd32(E1000_RDH(n));
300 		break;
301 	case E1000_RDT(0):
302 		for (n = 0; n < 4; n++)
303 			regs[n] = rd32(E1000_RDT(n));
304 		break;
305 	case E1000_RXDCTL(0):
306 		for (n = 0; n < 4; n++)
307 			regs[n] = rd32(E1000_RXDCTL(n));
308 		break;
309 	case E1000_RDBAL(0):
310 		for (n = 0; n < 4; n++)
311 			regs[n] = rd32(E1000_RDBAL(n));
312 		break;
313 	case E1000_RDBAH(0):
314 		for (n = 0; n < 4; n++)
315 			regs[n] = rd32(E1000_RDBAH(n));
316 		break;
317 	case E1000_TDBAL(0):
318 		for (n = 0; n < 4; n++)
319 			regs[n] = rd32(E1000_TDBAL(n));
320 		break;
321 	case E1000_TDBAH(0):
322 		for (n = 0; n < 4; n++)
323 			regs[n] = rd32(E1000_TDBAH(n));
324 		break;
325 	case E1000_TDLEN(0):
326 		for (n = 0; n < 4; n++)
327 			regs[n] = rd32(E1000_TDLEN(n));
328 		break;
329 	case E1000_TDH(0):
330 		for (n = 0; n < 4; n++)
331 			regs[n] = rd32(E1000_TDH(n));
332 		break;
333 	case E1000_TDT(0):
334 		for (n = 0; n < 4; n++)
335 			regs[n] = rd32(E1000_TDT(n));
336 		break;
337 	case E1000_TXDCTL(0):
338 		for (n = 0; n < 4; n++)
339 			regs[n] = rd32(E1000_TXDCTL(n));
340 		break;
341 	default:
342 		pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
343 		return;
344 	}
345 
346 	snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
347 	pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
348 		regs[2], regs[3]);
349 }
350 
351 /* igb_dump - Print registers, Tx-rings and Rx-rings */
352 static void igb_dump(struct igb_adapter *adapter)
353 {
354 	struct net_device *netdev = adapter->netdev;
355 	struct e1000_hw *hw = &adapter->hw;
356 	struct igb_reg_info *reginfo;
357 	struct igb_ring *tx_ring;
358 	union e1000_adv_tx_desc *tx_desc;
359 	struct my_u0 { __le64 a; __le64 b; } *u0;
360 	struct igb_ring *rx_ring;
361 	union e1000_adv_rx_desc *rx_desc;
362 	u32 staterr;
363 	u16 i, n;
364 
365 	if (!netif_msg_hw(adapter))
366 		return;
367 
368 	/* Print netdevice Info */
369 	if (netdev) {
370 		dev_info(&adapter->pdev->dev, "Net device Info\n");
371 		pr_info("Device Name     state            trans_start\n");
372 		pr_info("%-15s %016lX %016lX\n", netdev->name,
373 			netdev->state, dev_trans_start(netdev));
374 	}
375 
376 	/* Print Registers */
377 	dev_info(&adapter->pdev->dev, "Register Dump\n");
378 	pr_info(" Register Name   Value\n");
379 	for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
380 	     reginfo->name; reginfo++) {
381 		igb_regdump(hw, reginfo);
382 	}
383 
384 	/* Print TX Ring Summary */
385 	if (!netdev || !netif_running(netdev))
386 		goto exit;
387 
388 	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
389 	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
390 	for (n = 0; n < adapter->num_tx_queues; n++) {
391 		struct igb_tx_buffer *buffer_info;
392 		tx_ring = adapter->tx_ring[n];
393 		buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
394 		pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
395 			n, tx_ring->next_to_use, tx_ring->next_to_clean,
396 			(u64)dma_unmap_addr(buffer_info, dma),
397 			dma_unmap_len(buffer_info, len),
398 			buffer_info->next_to_watch,
399 			(u64)buffer_info->time_stamp);
400 	}
401 
402 	/* Print TX Rings */
403 	if (!netif_msg_tx_done(adapter))
404 		goto rx_ring_summary;
405 
406 	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
407 
408 	/* Transmit Descriptor Formats
409 	 *
410 	 * Advanced Transmit Descriptor
411 	 *   +--------------------------------------------------------------+
412 	 * 0 |         Buffer Address [63:0]                                |
413 	 *   +--------------------------------------------------------------+
414 	 * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
415 	 *   +--------------------------------------------------------------+
416 	 *   63      46 45    40 39 38 36 35 32 31   24             15       0
417 	 */
418 
419 	for (n = 0; n < adapter->num_tx_queues; n++) {
420 		tx_ring = adapter->tx_ring[n];
421 		pr_info("------------------------------------\n");
422 		pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
423 		pr_info("------------------------------------\n");
424 		pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] [bi->dma       ] leng  ntw timestamp        bi->skb\n");
425 
426 		for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
427 			const char *next_desc;
428 			struct igb_tx_buffer *buffer_info;
429 			tx_desc = IGB_TX_DESC(tx_ring, i);
430 			buffer_info = &tx_ring->tx_buffer_info[i];
431 			u0 = (struct my_u0 *)tx_desc;
432 			if (i == tx_ring->next_to_use &&
433 			    i == tx_ring->next_to_clean)
434 				next_desc = " NTC/U";
435 			else if (i == tx_ring->next_to_use)
436 				next_desc = " NTU";
437 			else if (i == tx_ring->next_to_clean)
438 				next_desc = " NTC";
439 			else
440 				next_desc = "";
441 
442 			pr_info("T [0x%03X]    %016llX %016llX %016llX %04X  %p %016llX %p%s\n",
443 				i, le64_to_cpu(u0->a),
444 				le64_to_cpu(u0->b),
445 				(u64)dma_unmap_addr(buffer_info, dma),
446 				dma_unmap_len(buffer_info, len),
447 				buffer_info->next_to_watch,
448 				(u64)buffer_info->time_stamp,
449 				buffer_info->skb, next_desc);
450 
451 			if (netif_msg_pktdata(adapter) && buffer_info->skb)
452 				print_hex_dump(KERN_INFO, "",
453 					DUMP_PREFIX_ADDRESS,
454 					16, 1, buffer_info->skb->data,
455 					dma_unmap_len(buffer_info, len),
456 					true);
457 		}
458 	}
459 
460 	/* Print RX Rings Summary */
461 rx_ring_summary:
462 	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
463 	pr_info("Queue [NTU] [NTC]\n");
464 	for (n = 0; n < adapter->num_rx_queues; n++) {
465 		rx_ring = adapter->rx_ring[n];
466 		pr_info(" %5d %5X %5X\n",
467 			n, rx_ring->next_to_use, rx_ring->next_to_clean);
468 	}
469 
470 	/* Print RX Rings */
471 	if (!netif_msg_rx_status(adapter))
472 		goto exit;
473 
474 	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
475 
476 	/* Advanced Receive Descriptor (Read) Format
477 	 *    63                                           1        0
478 	 *    +-----------------------------------------------------+
479 	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
480 	 *    +----------------------------------------------+------+
481 	 *  8 |       Header Buffer Address [63:1]           |  DD  |
482 	 *    +-----------------------------------------------------+
483 	 *
484 	 *
485 	 * Advanced Receive Descriptor (Write-Back) Format
486 	 *
487 	 *   63       48 47    32 31  30      21 20 17 16   4 3     0
488 	 *   +------------------------------------------------------+
489 	 * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
490 	 *   | Checksum   Ident  |   |           |    | Type | Type |
491 	 *   +------------------------------------------------------+
492 	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
493 	 *   +------------------------------------------------------+
494 	 *   63       48 47    32 31            20 19               0
495 	 */
496 
497 	for (n = 0; n < adapter->num_rx_queues; n++) {
498 		rx_ring = adapter->rx_ring[n];
499 		pr_info("------------------------------------\n");
500 		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
501 		pr_info("------------------------------------\n");
502 		pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] [bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
503 		pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
504 
505 		for (i = 0; i < rx_ring->count; i++) {
506 			const char *next_desc;
507 			struct igb_rx_buffer *buffer_info;
508 			buffer_info = &rx_ring->rx_buffer_info[i];
509 			rx_desc = IGB_RX_DESC(rx_ring, i);
510 			u0 = (struct my_u0 *)rx_desc;
511 			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
512 
513 			if (i == rx_ring->next_to_use)
514 				next_desc = " NTU";
515 			else if (i == rx_ring->next_to_clean)
516 				next_desc = " NTC";
517 			else
518 				next_desc = "";
519 
520 			if (staterr & E1000_RXD_STAT_DD) {
521 				/* Descriptor Done */
522 				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %s\n",
523 					"RWB", i,
524 					le64_to_cpu(u0->a),
525 					le64_to_cpu(u0->b),
526 					next_desc);
527 			} else {
528 				pr_info("%s[0x%03X]     %016llX %016llX %016llX %s\n",
529 					"R  ", i,
530 					le64_to_cpu(u0->a),
531 					le64_to_cpu(u0->b),
532 					(u64)buffer_info->dma,
533 					next_desc);
534 
535 				if (netif_msg_pktdata(adapter) &&
536 				    buffer_info->dma && buffer_info->page) {
537 					print_hex_dump(KERN_INFO, "",
538 					  DUMP_PREFIX_ADDRESS,
539 					  16, 1,
540 					  page_address(buffer_info->page) +
541 						      buffer_info->page_offset,
542 					  igb_rx_bufsz(rx_ring), true);
543 				}
544 			}
545 		}
546 	}
547 
548 exit:
549 	return;
550 }
551 
552 /**
553  *  igb_get_i2c_data - Reads the I2C SDA data bit
554  *  @data: opaque pointer to adapter struct
555  *
556  *  Returns the I2C data bit value
557  **/
558 static int igb_get_i2c_data(void *data)
559 {
560 	struct igb_adapter *adapter = (struct igb_adapter *)data;
561 	struct e1000_hw *hw = &adapter->hw;
562 	s32 i2cctl = rd32(E1000_I2CPARAMS);
563 
564 	return !!(i2cctl & E1000_I2C_DATA_IN);
565 }
566 
567 /**
568  *  igb_set_i2c_data - Sets the I2C data bit
569  *  @data: pointer to hardware structure
570  *  @state: I2C data value (0 or 1) to set
571  *
572  *  Sets the I2C data bit
573  **/
574 static void igb_set_i2c_data(void *data, int state)
575 {
576 	struct igb_adapter *adapter = (struct igb_adapter *)data;
577 	struct e1000_hw *hw = &adapter->hw;
578 	s32 i2cctl = rd32(E1000_I2CPARAMS);
579 
580 	if (state) {
581 		i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N;
582 	} else {
583 		i2cctl &= ~E1000_I2C_DATA_OE_N;
584 		i2cctl &= ~E1000_I2C_DATA_OUT;
585 	}
586 
587 	wr32(E1000_I2CPARAMS, i2cctl);
588 	wrfl();
589 }
590 
591 /**
592  *  igb_set_i2c_clk - Sets the I2C SCL clock
593  *  @data: pointer to hardware structure
594  *  @state: state to set clock
595  *
596  *  Sets the I2C clock line to state
597  **/
598 static void igb_set_i2c_clk(void *data, int state)
599 {
600 	struct igb_adapter *adapter = (struct igb_adapter *)data;
601 	struct e1000_hw *hw = &adapter->hw;
602 	s32 i2cctl = rd32(E1000_I2CPARAMS);
603 
604 	if (state) {
605 		i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N;
606 	} else {
607 		i2cctl &= ~E1000_I2C_CLK_OUT;
608 		i2cctl &= ~E1000_I2C_CLK_OE_N;
609 	}
610 	wr32(E1000_I2CPARAMS, i2cctl);
611 	wrfl();
612 }
613 
614 /**
615  *  igb_get_i2c_clk - Gets the I2C SCL clock state
616  *  @data: pointer to hardware structure
617  *
618  *  Gets the I2C clock state
619  **/
620 static int igb_get_i2c_clk(void *data)
621 {
622 	struct igb_adapter *adapter = (struct igb_adapter *)data;
623 	struct e1000_hw *hw = &adapter->hw;
624 	s32 i2cctl = rd32(E1000_I2CPARAMS);
625 
626 	return !!(i2cctl & E1000_I2C_CLK_IN);
627 }
628 
629 static const struct i2c_algo_bit_data igb_i2c_algo = {
630 	.setsda		= igb_set_i2c_data,
631 	.setscl		= igb_set_i2c_clk,
632 	.getsda		= igb_get_i2c_data,
633 	.getscl		= igb_get_i2c_clk,
634 	.udelay		= 5,
635 	.timeout	= 20,
636 };
637 
638 /**
639  *  igb_get_hw_dev - return device
640  *  @hw: pointer to hardware structure
641  *
642  *  used by hardware layer to print debugging information
643  **/
644 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
645 {
646 	struct igb_adapter *adapter = hw->back;
647 	return adapter->netdev;
648 }
649 
650 /**
651  *  igb_init_module - Driver Registration Routine
652  *
653  *  igb_init_module is the first routine called when the driver is
654  *  loaded. All it does is register with the PCI subsystem.
655  **/
656 static int __init igb_init_module(void)
657 {
658 	int ret;
659 
660 	pr_info("%s\n", igb_driver_string);
661 	pr_info("%s\n", igb_copyright);
662 
663 #ifdef CONFIG_IGB_DCA
664 	dca_register_notify(&dca_notifier);
665 #endif
666 	ret = pci_register_driver(&igb_driver);
667 	return ret;
668 }
669 
670 module_init(igb_init_module);
671 
672 /**
673  *  igb_exit_module - Driver Exit Cleanup Routine
674  *
675  *  igb_exit_module is called just before the driver is removed
676  *  from memory.
677  **/
678 static void __exit igb_exit_module(void)
679 {
680 #ifdef CONFIG_IGB_DCA
681 	dca_unregister_notify(&dca_notifier);
682 #endif
683 	pci_unregister_driver(&igb_driver);
684 }
685 
686 module_exit(igb_exit_module);
687 
688 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
689 /**
690  *  igb_cache_ring_register - Descriptor ring to register mapping
691  *  @adapter: board private structure to initialize
692  *
693  *  Once we know the feature-set enabled for the device, we'll cache
694  *  the register offset the descriptor ring is assigned to.
695  **/
696 static void igb_cache_ring_register(struct igb_adapter *adapter)
697 {
698 	int i = 0, j = 0;
699 	u32 rbase_offset = adapter->vfs_allocated_count;
700 
701 	switch (adapter->hw.mac.type) {
702 	case e1000_82576:
703 		/* The queues are allocated for virtualization such that VF 0
704 		 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
705 		 * In order to avoid collision we start at the first free queue
706 		 * and continue consuming queues in the same sequence
707 		 */
708 		if (adapter->vfs_allocated_count) {
709 			for (; i < adapter->rss_queues; i++)
710 				adapter->rx_ring[i]->reg_idx = rbase_offset +
711 							       Q_IDX_82576(i);
712 		}
713 		fallthrough;
714 	case e1000_82575:
715 	case e1000_82580:
716 	case e1000_i350:
717 	case e1000_i354:
718 	case e1000_i210:
719 	case e1000_i211:
720 	default:
721 		for (; i < adapter->num_rx_queues; i++)
722 			adapter->rx_ring[i]->reg_idx = rbase_offset + i;
723 		for (; j < adapter->num_tx_queues; j++)
724 			adapter->tx_ring[j]->reg_idx = rbase_offset + j;
725 		break;
726 	}
727 }
728 
729 u32 igb_rd32(struct e1000_hw *hw, u32 reg)
730 {
731 	struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
732 	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
733 	u32 value = 0;
734 
735 	if (E1000_REMOVED(hw_addr))
736 		return ~value;
737 
738 	value = readl(&hw_addr[reg]);
739 
740 	/* reads should not return all F's */
741 	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
742 		struct net_device *netdev = igb->netdev;
743 		hw->hw_addr = NULL;
744 		netdev_err(netdev, "PCIe link lost\n");
745 		WARN(pci_device_is_present(igb->pdev),
746 		     "igb: Failed to read reg 0x%x!\n", reg);
747 	}
748 
749 	return value;
750 }
751 
752 /**
753  *  igb_write_ivar - configure ivar for given MSI-X vector
754  *  @hw: pointer to the HW structure
755  *  @msix_vector: vector number we are allocating to a given ring
756  *  @index: row index of IVAR register to write within IVAR table
757  *  @offset: column offset of in IVAR, should be multiple of 8
758  *
759  *  This function is intended to handle the writing of the IVAR register
760  *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
761  *  each containing an cause allocation for an Rx and Tx ring, and a
762  *  variable number of rows depending on the number of queues supported.
763  **/
764 static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
765 			   int index, int offset)
766 {
767 	u32 ivar = array_rd32(E1000_IVAR0, index);
768 
769 	/* clear any bits that are currently set */
770 	ivar &= ~((u32)0xFF << offset);
771 
772 	/* write vector and valid bit */
773 	ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
774 
775 	array_wr32(E1000_IVAR0, index, ivar);
776 }
777 
778 #define IGB_N0_QUEUE -1
779 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
780 {
781 	struct igb_adapter *adapter = q_vector->adapter;
782 	struct e1000_hw *hw = &adapter->hw;
783 	int rx_queue = IGB_N0_QUEUE;
784 	int tx_queue = IGB_N0_QUEUE;
785 	u32 msixbm = 0;
786 
787 	if (q_vector->rx.ring)
788 		rx_queue = q_vector->rx.ring->reg_idx;
789 	if (q_vector->tx.ring)
790 		tx_queue = q_vector->tx.ring->reg_idx;
791 
792 	switch (hw->mac.type) {
793 	case e1000_82575:
794 		/* The 82575 assigns vectors using a bitmask, which matches the
795 		 * bitmask for the EICR/EIMS/EIMC registers.  To assign one
796 		 * or more queues to a vector, we write the appropriate bits
797 		 * into the MSIXBM register for that vector.
798 		 */
799 		if (rx_queue > IGB_N0_QUEUE)
800 			msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
801 		if (tx_queue > IGB_N0_QUEUE)
802 			msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
803 		if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
804 			msixbm |= E1000_EIMS_OTHER;
805 		array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
806 		q_vector->eims_value = msixbm;
807 		break;
808 	case e1000_82576:
809 		/* 82576 uses a table that essentially consists of 2 columns
810 		 * with 8 rows.  The ordering is column-major so we use the
811 		 * lower 3 bits as the row index, and the 4th bit as the
812 		 * column offset.
813 		 */
814 		if (rx_queue > IGB_N0_QUEUE)
815 			igb_write_ivar(hw, msix_vector,
816 				       rx_queue & 0x7,
817 				       (rx_queue & 0x8) << 1);
818 		if (tx_queue > IGB_N0_QUEUE)
819 			igb_write_ivar(hw, msix_vector,
820 				       tx_queue & 0x7,
821 				       ((tx_queue & 0x8) << 1) + 8);
822 		q_vector->eims_value = BIT(msix_vector);
823 		break;
824 	case e1000_82580:
825 	case e1000_i350:
826 	case e1000_i354:
827 	case e1000_i210:
828 	case e1000_i211:
829 		/* On 82580 and newer adapters the scheme is similar to 82576
830 		 * however instead of ordering column-major we have things
831 		 * ordered row-major.  So we traverse the table by using
832 		 * bit 0 as the column offset, and the remaining bits as the
833 		 * row index.
834 		 */
835 		if (rx_queue > IGB_N0_QUEUE)
836 			igb_write_ivar(hw, msix_vector,
837 				       rx_queue >> 1,
838 				       (rx_queue & 0x1) << 4);
839 		if (tx_queue > IGB_N0_QUEUE)
840 			igb_write_ivar(hw, msix_vector,
841 				       tx_queue >> 1,
842 				       ((tx_queue & 0x1) << 4) + 8);
843 		q_vector->eims_value = BIT(msix_vector);
844 		break;
845 	default:
846 		BUG();
847 		break;
848 	}
849 
850 	/* add q_vector eims value to global eims_enable_mask */
851 	adapter->eims_enable_mask |= q_vector->eims_value;
852 
853 	/* configure q_vector to set itr on first interrupt */
854 	q_vector->set_itr = 1;
855 }
856 
857 /**
858  *  igb_configure_msix - Configure MSI-X hardware
859  *  @adapter: board private structure to initialize
860  *
861  *  igb_configure_msix sets up the hardware to properly
862  *  generate MSI-X interrupts.
863  **/
864 static void igb_configure_msix(struct igb_adapter *adapter)
865 {
866 	u32 tmp;
867 	int i, vector = 0;
868 	struct e1000_hw *hw = &adapter->hw;
869 
870 	adapter->eims_enable_mask = 0;
871 
872 	/* set vector for other causes, i.e. link changes */
873 	switch (hw->mac.type) {
874 	case e1000_82575:
875 		tmp = rd32(E1000_CTRL_EXT);
876 		/* enable MSI-X PBA support*/
877 		tmp |= E1000_CTRL_EXT_PBA_CLR;
878 
879 		/* Auto-Mask interrupts upon ICR read. */
880 		tmp |= E1000_CTRL_EXT_EIAME;
881 		tmp |= E1000_CTRL_EXT_IRCA;
882 
883 		wr32(E1000_CTRL_EXT, tmp);
884 
885 		/* enable msix_other interrupt */
886 		array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
887 		adapter->eims_other = E1000_EIMS_OTHER;
888 
889 		break;
890 
891 	case e1000_82576:
892 	case e1000_82580:
893 	case e1000_i350:
894 	case e1000_i354:
895 	case e1000_i210:
896 	case e1000_i211:
897 		/* Turn on MSI-X capability first, or our settings
898 		 * won't stick.  And it will take days to debug.
899 		 */
900 		wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
901 		     E1000_GPIE_PBA | E1000_GPIE_EIAME |
902 		     E1000_GPIE_NSICR);
903 
904 		/* enable msix_other interrupt */
905 		adapter->eims_other = BIT(vector);
906 		tmp = (vector++ | E1000_IVAR_VALID) << 8;
907 
908 		wr32(E1000_IVAR_MISC, tmp);
909 		break;
910 	default:
911 		/* do nothing, since nothing else supports MSI-X */
912 		break;
913 	} /* switch (hw->mac.type) */
914 
915 	adapter->eims_enable_mask |= adapter->eims_other;
916 
917 	for (i = 0; i < adapter->num_q_vectors; i++)
918 		igb_assign_vector(adapter->q_vector[i], vector++);
919 
920 	wrfl();
921 }
922 
923 /**
924  *  igb_request_msix - Initialize MSI-X interrupts
925  *  @adapter: board private structure to initialize
926  *
927  *  igb_request_msix allocates MSI-X vectors and requests interrupts from the
928  *  kernel.
929  **/
930 static int igb_request_msix(struct igb_adapter *adapter)
931 {
932 	unsigned int num_q_vectors = adapter->num_q_vectors;
933 	struct net_device *netdev = adapter->netdev;
934 	int i, err = 0, vector = 0, free_vector = 0;
935 
936 	err = request_irq(adapter->msix_entries[vector].vector,
937 			  igb_msix_other, 0, netdev->name, adapter);
938 	if (err)
939 		goto err_out;
940 
941 	if (num_q_vectors > MAX_Q_VECTORS) {
942 		num_q_vectors = MAX_Q_VECTORS;
943 		dev_warn(&adapter->pdev->dev,
944 			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
945 			 adapter->num_q_vectors, MAX_Q_VECTORS);
946 	}
947 	for (i = 0; i < num_q_vectors; i++) {
948 		struct igb_q_vector *q_vector = adapter->q_vector[i];
949 
950 		vector++;
951 
952 		q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
953 
954 		if (q_vector->rx.ring && q_vector->tx.ring)
955 			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
956 				q_vector->rx.ring->queue_index);
957 		else if (q_vector->tx.ring)
958 			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
959 				q_vector->tx.ring->queue_index);
960 		else if (q_vector->rx.ring)
961 			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
962 				q_vector->rx.ring->queue_index);
963 		else
964 			sprintf(q_vector->name, "%s-unused", netdev->name);
965 
966 		err = request_irq(adapter->msix_entries[vector].vector,
967 				  igb_msix_ring, 0, q_vector->name,
968 				  q_vector);
969 		if (err)
970 			goto err_free;
971 	}
972 
973 	igb_configure_msix(adapter);
974 	return 0;
975 
976 err_free:
977 	/* free already assigned IRQs */
978 	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
979 
980 	vector--;
981 	for (i = 0; i < vector; i++) {
982 		free_irq(adapter->msix_entries[free_vector++].vector,
983 			 adapter->q_vector[i]);
984 	}
985 err_out:
986 	return err;
987 }
988 
989 /**
990  *  igb_free_q_vector - Free memory allocated for specific interrupt vector
991  *  @adapter: board private structure to initialize
992  *  @v_idx: Index of vector to be freed
993  *
994  *  This function frees the memory allocated to the q_vector.
995  **/
996 static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
997 {
998 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
999 
1000 	adapter->q_vector[v_idx] = NULL;
1001 
1002 	/* igb_get_stats64() might access the rings on this vector,
1003 	 * we must wait a grace period before freeing it.
1004 	 */
1005 	if (q_vector)
1006 		kfree_rcu(q_vector, rcu);
1007 }
1008 
1009 /**
1010  *  igb_reset_q_vector - Reset config for interrupt vector
1011  *  @adapter: board private structure to initialize
1012  *  @v_idx: Index of vector to be reset
1013  *
1014  *  If NAPI is enabled it will delete any references to the
1015  *  NAPI struct. This is preparation for igb_free_q_vector.
1016  **/
1017 static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1018 {
1019 	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1020 
1021 	/* Coming from igb_set_interrupt_capability, the vectors are not yet
1022 	 * allocated. So, q_vector is NULL so we should stop here.
1023 	 */
1024 	if (!q_vector)
1025 		return;
1026 
1027 	if (q_vector->tx.ring)
1028 		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1029 
1030 	if (q_vector->rx.ring)
1031 		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1032 
1033 	netif_napi_del(&q_vector->napi);
1034 
1035 }
1036 
1037 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1038 {
1039 	int v_idx = adapter->num_q_vectors;
1040 
1041 	if (adapter->flags & IGB_FLAG_HAS_MSIX)
1042 		pci_disable_msix(adapter->pdev);
1043 	else if (adapter->flags & IGB_FLAG_HAS_MSI)
1044 		pci_disable_msi(adapter->pdev);
1045 
1046 	while (v_idx--)
1047 		igb_reset_q_vector(adapter, v_idx);
1048 }
1049 
1050 /**
1051  *  igb_free_q_vectors - Free memory allocated for interrupt vectors
1052  *  @adapter: board private structure to initialize
1053  *
1054  *  This function frees the memory allocated to the q_vectors.  In addition if
1055  *  NAPI is enabled it will delete any references to the NAPI struct prior
1056  *  to freeing the q_vector.
1057  **/
1058 static void igb_free_q_vectors(struct igb_adapter *adapter)
1059 {
1060 	int v_idx = adapter->num_q_vectors;
1061 
1062 	adapter->num_tx_queues = 0;
1063 	adapter->num_rx_queues = 0;
1064 	adapter->num_q_vectors = 0;
1065 
1066 	while (v_idx--) {
1067 		igb_reset_q_vector(adapter, v_idx);
1068 		igb_free_q_vector(adapter, v_idx);
1069 	}
1070 }
1071 
1072 /**
1073  *  igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1074  *  @adapter: board private structure to initialize
1075  *
1076  *  This function resets the device so that it has 0 Rx queues, Tx queues, and
1077  *  MSI-X interrupts allocated.
1078  */
1079 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1080 {
1081 	igb_free_q_vectors(adapter);
1082 	igb_reset_interrupt_capability(adapter);
1083 }
1084 
1085 /**
1086  *  igb_set_interrupt_capability - set MSI or MSI-X if supported
1087  *  @adapter: board private structure to initialize
1088  *  @msix: boolean value of MSIX capability
1089  *
1090  *  Attempt to configure interrupts using the best available
1091  *  capabilities of the hardware and kernel.
1092  **/
1093 static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1094 {
1095 	int err;
1096 	int numvecs, i;
1097 
1098 	if (!msix)
1099 		goto msi_only;
1100 	adapter->flags |= IGB_FLAG_HAS_MSIX;
1101 
1102 	/* Number of supported queues. */
1103 	adapter->num_rx_queues = adapter->rss_queues;
1104 	if (adapter->vfs_allocated_count)
1105 		adapter->num_tx_queues = 1;
1106 	else
1107 		adapter->num_tx_queues = adapter->rss_queues;
1108 
1109 	/* start with one vector for every Rx queue */
1110 	numvecs = adapter->num_rx_queues;
1111 
1112 	/* if Tx handler is separate add 1 for every Tx queue */
1113 	if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1114 		numvecs += adapter->num_tx_queues;
1115 
1116 	/* store the number of vectors reserved for queues */
1117 	adapter->num_q_vectors = numvecs;
1118 
1119 	/* add 1 vector for link status interrupts */
1120 	numvecs++;
1121 	for (i = 0; i < numvecs; i++)
1122 		adapter->msix_entries[i].entry = i;
1123 
1124 	err = pci_enable_msix_range(adapter->pdev,
1125 				    adapter->msix_entries,
1126 				    numvecs,
1127 				    numvecs);
1128 	if (err > 0)
1129 		return;
1130 
1131 	igb_reset_interrupt_capability(adapter);
1132 
1133 	/* If we can't do MSI-X, try MSI */
1134 msi_only:
1135 	adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1136 #ifdef CONFIG_PCI_IOV
1137 	/* disable SR-IOV for non MSI-X configurations */
1138 	if (adapter->vf_data) {
1139 		struct e1000_hw *hw = &adapter->hw;
1140 		/* disable iov and allow time for transactions to clear */
1141 		pci_disable_sriov(adapter->pdev);
1142 		msleep(500);
1143 
1144 		kfree(adapter->vf_mac_list);
1145 		adapter->vf_mac_list = NULL;
1146 		kfree(adapter->vf_data);
1147 		adapter->vf_data = NULL;
1148 		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1149 		wrfl();
1150 		msleep(100);
1151 		dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1152 	}
1153 #endif
1154 	adapter->vfs_allocated_count = 0;
1155 	adapter->rss_queues = 1;
1156 	adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1157 	adapter->num_rx_queues = 1;
1158 	adapter->num_tx_queues = 1;
1159 	adapter->num_q_vectors = 1;
1160 	if (!pci_enable_msi(adapter->pdev))
1161 		adapter->flags |= IGB_FLAG_HAS_MSI;
1162 }
1163 
1164 static void igb_add_ring(struct igb_ring *ring,
1165 			 struct igb_ring_container *head)
1166 {
1167 	head->ring = ring;
1168 	head->count++;
1169 }
1170 
1171 /**
1172  *  igb_alloc_q_vector - Allocate memory for a single interrupt vector
1173  *  @adapter: board private structure to initialize
1174  *  @v_count: q_vectors allocated on adapter, used for ring interleaving
1175  *  @v_idx: index of vector in adapter struct
1176  *  @txr_count: total number of Tx rings to allocate
1177  *  @txr_idx: index of first Tx ring to allocate
1178  *  @rxr_count: total number of Rx rings to allocate
1179  *  @rxr_idx: index of first Rx ring to allocate
1180  *
1181  *  We allocate one q_vector.  If allocation fails we return -ENOMEM.
1182  **/
1183 static int igb_alloc_q_vector(struct igb_adapter *adapter,
1184 			      int v_count, int v_idx,
1185 			      int txr_count, int txr_idx,
1186 			      int rxr_count, int rxr_idx)
1187 {
1188 	struct igb_q_vector *q_vector;
1189 	struct igb_ring *ring;
1190 	int ring_count;
1191 	size_t size;
1192 
1193 	/* igb only supports 1 Tx and/or 1 Rx queue per vector */
1194 	if (txr_count > 1 || rxr_count > 1)
1195 		return -ENOMEM;
1196 
1197 	ring_count = txr_count + rxr_count;
1198 	size = struct_size(q_vector, ring, ring_count);
1199 
1200 	/* allocate q_vector and rings */
1201 	q_vector = adapter->q_vector[v_idx];
1202 	if (!q_vector) {
1203 		q_vector = kzalloc(size, GFP_KERNEL);
1204 	} else if (size > ksize(q_vector)) {
1205 		kfree_rcu(q_vector, rcu);
1206 		q_vector = kzalloc(size, GFP_KERNEL);
1207 	} else {
1208 		memset(q_vector, 0, size);
1209 	}
1210 	if (!q_vector)
1211 		return -ENOMEM;
1212 
1213 	/* initialize NAPI */
1214 	netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll);
1215 
1216 	/* tie q_vector and adapter together */
1217 	adapter->q_vector[v_idx] = q_vector;
1218 	q_vector->adapter = adapter;
1219 
1220 	/* initialize work limits */
1221 	q_vector->tx.work_limit = adapter->tx_work_limit;
1222 
1223 	/* initialize ITR configuration */
1224 	q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1225 	q_vector->itr_val = IGB_START_ITR;
1226 
1227 	/* initialize pointer to rings */
1228 	ring = q_vector->ring;
1229 
1230 	/* intialize ITR */
1231 	if (rxr_count) {
1232 		/* rx or rx/tx vector */
1233 		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1234 			q_vector->itr_val = adapter->rx_itr_setting;
1235 	} else {
1236 		/* tx only vector */
1237 		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1238 			q_vector->itr_val = adapter->tx_itr_setting;
1239 	}
1240 
1241 	if (txr_count) {
1242 		/* assign generic ring traits */
1243 		ring->dev = &adapter->pdev->dev;
1244 		ring->netdev = adapter->netdev;
1245 
1246 		/* configure backlink on ring */
1247 		ring->q_vector = q_vector;
1248 
1249 		/* update q_vector Tx values */
1250 		igb_add_ring(ring, &q_vector->tx);
1251 
1252 		/* For 82575, context index must be unique per ring. */
1253 		if (adapter->hw.mac.type == e1000_82575)
1254 			set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1255 
1256 		/* apply Tx specific ring traits */
1257 		ring->count = adapter->tx_ring_count;
1258 		ring->queue_index = txr_idx;
1259 
1260 		ring->cbs_enable = false;
1261 		ring->idleslope = 0;
1262 		ring->sendslope = 0;
1263 		ring->hicredit = 0;
1264 		ring->locredit = 0;
1265 
1266 		u64_stats_init(&ring->tx_syncp);
1267 		u64_stats_init(&ring->tx_syncp2);
1268 
1269 		/* assign ring to adapter */
1270 		adapter->tx_ring[txr_idx] = ring;
1271 
1272 		/* push pointer to next ring */
1273 		ring++;
1274 	}
1275 
1276 	if (rxr_count) {
1277 		/* assign generic ring traits */
1278 		ring->dev = &adapter->pdev->dev;
1279 		ring->netdev = adapter->netdev;
1280 
1281 		/* configure backlink on ring */
1282 		ring->q_vector = q_vector;
1283 
1284 		/* update q_vector Rx values */
1285 		igb_add_ring(ring, &q_vector->rx);
1286 
1287 		/* set flag indicating ring supports SCTP checksum offload */
1288 		if (adapter->hw.mac.type >= e1000_82576)
1289 			set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1290 
1291 		/* On i350, i354, i210, and i211, loopback VLAN packets
1292 		 * have the tag byte-swapped.
1293 		 */
1294 		if (adapter->hw.mac.type >= e1000_i350)
1295 			set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1296 
1297 		/* apply Rx specific ring traits */
1298 		ring->count = adapter->rx_ring_count;
1299 		ring->queue_index = rxr_idx;
1300 
1301 		u64_stats_init(&ring->rx_syncp);
1302 
1303 		/* assign ring to adapter */
1304 		adapter->rx_ring[rxr_idx] = ring;
1305 	}
1306 
1307 	return 0;
1308 }
1309 
1310 
1311 /**
1312  *  igb_alloc_q_vectors - Allocate memory for interrupt vectors
1313  *  @adapter: board private structure to initialize
1314  *
1315  *  We allocate one q_vector per queue interrupt.  If allocation fails we
1316  *  return -ENOMEM.
1317  **/
1318 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1319 {
1320 	int q_vectors = adapter->num_q_vectors;
1321 	int rxr_remaining = adapter->num_rx_queues;
1322 	int txr_remaining = adapter->num_tx_queues;
1323 	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1324 	int err;
1325 
1326 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
1327 		for (; rxr_remaining; v_idx++) {
1328 			err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1329 						 0, 0, 1, rxr_idx);
1330 
1331 			if (err)
1332 				goto err_out;
1333 
1334 			/* update counts and index */
1335 			rxr_remaining--;
1336 			rxr_idx++;
1337 		}
1338 	}
1339 
1340 	for (; v_idx < q_vectors; v_idx++) {
1341 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1342 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1343 
1344 		err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1345 					 tqpv, txr_idx, rqpv, rxr_idx);
1346 
1347 		if (err)
1348 			goto err_out;
1349 
1350 		/* update counts and index */
1351 		rxr_remaining -= rqpv;
1352 		txr_remaining -= tqpv;
1353 		rxr_idx++;
1354 		txr_idx++;
1355 	}
1356 
1357 	return 0;
1358 
1359 err_out:
1360 	adapter->num_tx_queues = 0;
1361 	adapter->num_rx_queues = 0;
1362 	adapter->num_q_vectors = 0;
1363 
1364 	while (v_idx--)
1365 		igb_free_q_vector(adapter, v_idx);
1366 
1367 	return -ENOMEM;
1368 }
1369 
1370 /**
1371  *  igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1372  *  @adapter: board private structure to initialize
1373  *  @msix: boolean value of MSIX capability
1374  *
1375  *  This function initializes the interrupts and allocates all of the queues.
1376  **/
1377 static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1378 {
1379 	struct pci_dev *pdev = adapter->pdev;
1380 	int err;
1381 
1382 	igb_set_interrupt_capability(adapter, msix);
1383 
1384 	err = igb_alloc_q_vectors(adapter);
1385 	if (err) {
1386 		dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1387 		goto err_alloc_q_vectors;
1388 	}
1389 
1390 	igb_cache_ring_register(adapter);
1391 
1392 	return 0;
1393 
1394 err_alloc_q_vectors:
1395 	igb_reset_interrupt_capability(adapter);
1396 	return err;
1397 }
1398 
1399 /**
1400  *  igb_request_irq - initialize interrupts
1401  *  @adapter: board private structure to initialize
1402  *
1403  *  Attempts to configure interrupts using the best available
1404  *  capabilities of the hardware and kernel.
1405  **/
1406 static int igb_request_irq(struct igb_adapter *adapter)
1407 {
1408 	struct net_device *netdev = adapter->netdev;
1409 	struct pci_dev *pdev = adapter->pdev;
1410 	int err = 0;
1411 
1412 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1413 		err = igb_request_msix(adapter);
1414 		if (!err)
1415 			goto request_done;
1416 		/* fall back to MSI */
1417 		igb_free_all_tx_resources(adapter);
1418 		igb_free_all_rx_resources(adapter);
1419 
1420 		igb_clear_interrupt_scheme(adapter);
1421 		err = igb_init_interrupt_scheme(adapter, false);
1422 		if (err)
1423 			goto request_done;
1424 
1425 		igb_setup_all_tx_resources(adapter);
1426 		igb_setup_all_rx_resources(adapter);
1427 		igb_configure(adapter);
1428 	}
1429 
1430 	igb_assign_vector(adapter->q_vector[0], 0);
1431 
1432 	if (adapter->flags & IGB_FLAG_HAS_MSI) {
1433 		err = request_irq(pdev->irq, igb_intr_msi, 0,
1434 				  netdev->name, adapter);
1435 		if (!err)
1436 			goto request_done;
1437 
1438 		/* fall back to legacy interrupts */
1439 		igb_reset_interrupt_capability(adapter);
1440 		adapter->flags &= ~IGB_FLAG_HAS_MSI;
1441 	}
1442 
1443 	err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1444 			  netdev->name, adapter);
1445 
1446 	if (err)
1447 		dev_err(&pdev->dev, "Error %d getting interrupt\n",
1448 			err);
1449 
1450 request_done:
1451 	return err;
1452 }
1453 
1454 static void igb_free_irq(struct igb_adapter *adapter)
1455 {
1456 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1457 		int vector = 0, i;
1458 
1459 		free_irq(adapter->msix_entries[vector++].vector, adapter);
1460 
1461 		for (i = 0; i < adapter->num_q_vectors; i++)
1462 			free_irq(adapter->msix_entries[vector++].vector,
1463 				 adapter->q_vector[i]);
1464 	} else {
1465 		free_irq(adapter->pdev->irq, adapter);
1466 	}
1467 }
1468 
1469 /**
1470  *  igb_irq_disable - Mask off interrupt generation on the NIC
1471  *  @adapter: board private structure
1472  **/
1473 static void igb_irq_disable(struct igb_adapter *adapter)
1474 {
1475 	struct e1000_hw *hw = &adapter->hw;
1476 
1477 	/* we need to be careful when disabling interrupts.  The VFs are also
1478 	 * mapped into these registers and so clearing the bits can cause
1479 	 * issues on the VF drivers so we only need to clear what we set
1480 	 */
1481 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1482 		u32 regval = rd32(E1000_EIAM);
1483 
1484 		wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1485 		wr32(E1000_EIMC, adapter->eims_enable_mask);
1486 		regval = rd32(E1000_EIAC);
1487 		wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1488 	}
1489 
1490 	wr32(E1000_IAM, 0);
1491 	wr32(E1000_IMC, ~0);
1492 	wrfl();
1493 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1494 		int i;
1495 
1496 		for (i = 0; i < adapter->num_q_vectors; i++)
1497 			synchronize_irq(adapter->msix_entries[i].vector);
1498 	} else {
1499 		synchronize_irq(adapter->pdev->irq);
1500 	}
1501 }
1502 
1503 /**
1504  *  igb_irq_enable - Enable default interrupt generation settings
1505  *  @adapter: board private structure
1506  **/
1507 static void igb_irq_enable(struct igb_adapter *adapter)
1508 {
1509 	struct e1000_hw *hw = &adapter->hw;
1510 
1511 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1512 		u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1513 		u32 regval = rd32(E1000_EIAC);
1514 
1515 		wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1516 		regval = rd32(E1000_EIAM);
1517 		wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1518 		wr32(E1000_EIMS, adapter->eims_enable_mask);
1519 		if (adapter->vfs_allocated_count) {
1520 			wr32(E1000_MBVFIMR, 0xFF);
1521 			ims |= E1000_IMS_VMMB;
1522 		}
1523 		wr32(E1000_IMS, ims);
1524 	} else {
1525 		wr32(E1000_IMS, IMS_ENABLE_MASK |
1526 				E1000_IMS_DRSTA);
1527 		wr32(E1000_IAM, IMS_ENABLE_MASK |
1528 				E1000_IMS_DRSTA);
1529 	}
1530 }
1531 
1532 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1533 {
1534 	struct e1000_hw *hw = &adapter->hw;
1535 	u16 pf_id = adapter->vfs_allocated_count;
1536 	u16 vid = adapter->hw.mng_cookie.vlan_id;
1537 	u16 old_vid = adapter->mng_vlan_id;
1538 
1539 	if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1540 		/* add VID to filter table */
1541 		igb_vfta_set(hw, vid, pf_id, true, true);
1542 		adapter->mng_vlan_id = vid;
1543 	} else {
1544 		adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1545 	}
1546 
1547 	if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1548 	    (vid != old_vid) &&
1549 	    !test_bit(old_vid, adapter->active_vlans)) {
1550 		/* remove VID from filter table */
1551 		igb_vfta_set(hw, vid, pf_id, false, true);
1552 	}
1553 }
1554 
1555 /**
1556  *  igb_release_hw_control - release control of the h/w to f/w
1557  *  @adapter: address of board private structure
1558  *
1559  *  igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1560  *  For ASF and Pass Through versions of f/w this means that the
1561  *  driver is no longer loaded.
1562  **/
1563 static void igb_release_hw_control(struct igb_adapter *adapter)
1564 {
1565 	struct e1000_hw *hw = &adapter->hw;
1566 	u32 ctrl_ext;
1567 
1568 	/* Let firmware take over control of h/w */
1569 	ctrl_ext = rd32(E1000_CTRL_EXT);
1570 	wr32(E1000_CTRL_EXT,
1571 			ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1572 }
1573 
1574 /**
1575  *  igb_get_hw_control - get control of the h/w from f/w
1576  *  @adapter: address of board private structure
1577  *
1578  *  igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1579  *  For ASF and Pass Through versions of f/w this means that
1580  *  the driver is loaded.
1581  **/
1582 static void igb_get_hw_control(struct igb_adapter *adapter)
1583 {
1584 	struct e1000_hw *hw = &adapter->hw;
1585 	u32 ctrl_ext;
1586 
1587 	/* Let firmware know the driver has taken over */
1588 	ctrl_ext = rd32(E1000_CTRL_EXT);
1589 	wr32(E1000_CTRL_EXT,
1590 			ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1591 }
1592 
1593 static void enable_fqtss(struct igb_adapter *adapter, bool enable)
1594 {
1595 	struct net_device *netdev = adapter->netdev;
1596 	struct e1000_hw *hw = &adapter->hw;
1597 
1598 	WARN_ON(hw->mac.type != e1000_i210);
1599 
1600 	if (enable)
1601 		adapter->flags |= IGB_FLAG_FQTSS;
1602 	else
1603 		adapter->flags &= ~IGB_FLAG_FQTSS;
1604 
1605 	if (netif_running(netdev))
1606 		schedule_work(&adapter->reset_task);
1607 }
1608 
1609 static bool is_fqtss_enabled(struct igb_adapter *adapter)
1610 {
1611 	return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
1612 }
1613 
1614 static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
1615 				   enum tx_queue_prio prio)
1616 {
1617 	u32 val;
1618 
1619 	WARN_ON(hw->mac.type != e1000_i210);
1620 	WARN_ON(queue < 0 || queue > 4);
1621 
1622 	val = rd32(E1000_I210_TXDCTL(queue));
1623 
1624 	if (prio == TX_QUEUE_PRIO_HIGH)
1625 		val |= E1000_TXDCTL_PRIORITY;
1626 	else
1627 		val &= ~E1000_TXDCTL_PRIORITY;
1628 
1629 	wr32(E1000_I210_TXDCTL(queue), val);
1630 }
1631 
1632 static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
1633 {
1634 	u32 val;
1635 
1636 	WARN_ON(hw->mac.type != e1000_i210);
1637 	WARN_ON(queue < 0 || queue > 1);
1638 
1639 	val = rd32(E1000_I210_TQAVCC(queue));
1640 
1641 	if (mode == QUEUE_MODE_STREAM_RESERVATION)
1642 		val |= E1000_TQAVCC_QUEUEMODE;
1643 	else
1644 		val &= ~E1000_TQAVCC_QUEUEMODE;
1645 
1646 	wr32(E1000_I210_TQAVCC(queue), val);
1647 }
1648 
1649 static bool is_any_cbs_enabled(struct igb_adapter *adapter)
1650 {
1651 	int i;
1652 
1653 	for (i = 0; i < adapter->num_tx_queues; i++) {
1654 		if (adapter->tx_ring[i]->cbs_enable)
1655 			return true;
1656 	}
1657 
1658 	return false;
1659 }
1660 
1661 static bool is_any_txtime_enabled(struct igb_adapter *adapter)
1662 {
1663 	int i;
1664 
1665 	for (i = 0; i < adapter->num_tx_queues; i++) {
1666 		if (adapter->tx_ring[i]->launchtime_enable)
1667 			return true;
1668 	}
1669 
1670 	return false;
1671 }
1672 
1673 /**
1674  *  igb_config_tx_modes - Configure "Qav Tx mode" features on igb
1675  *  @adapter: pointer to adapter struct
1676  *  @queue: queue number
1677  *
1678  *  Configure CBS and Launchtime for a given hardware queue.
1679  *  Parameters are retrieved from the correct Tx ring, so
1680  *  igb_save_cbs_params() and igb_save_txtime_params() should be used
1681  *  for setting those correctly prior to this function being called.
1682  **/
1683 static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
1684 {
1685 	struct net_device *netdev = adapter->netdev;
1686 	struct e1000_hw *hw = &adapter->hw;
1687 	struct igb_ring *ring;
1688 	u32 tqavcc, tqavctrl;
1689 	u16 value;
1690 
1691 	WARN_ON(hw->mac.type != e1000_i210);
1692 	WARN_ON(queue < 0 || queue > 1);
1693 	ring = adapter->tx_ring[queue];
1694 
1695 	/* If any of the Qav features is enabled, configure queues as SR and
1696 	 * with HIGH PRIO. If none is, then configure them with LOW PRIO and
1697 	 * as SP.
1698 	 */
1699 	if (ring->cbs_enable || ring->launchtime_enable) {
1700 		set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
1701 		set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
1702 	} else {
1703 		set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
1704 		set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
1705 	}
1706 
1707 	/* If CBS is enabled, set DataTranARB and config its parameters. */
1708 	if (ring->cbs_enable || queue == 0) {
1709 		/* i210 does not allow the queue 0 to be in the Strict
1710 		 * Priority mode while the Qav mode is enabled, so,
1711 		 * instead of disabling strict priority mode, we give
1712 		 * queue 0 the maximum of credits possible.
1713 		 *
1714 		 * See section 8.12.19 of the i210 datasheet, "Note:
1715 		 * Queue0 QueueMode must be set to 1b when
1716 		 * TransmitMode is set to Qav."
1717 		 */
1718 		if (queue == 0 && !ring->cbs_enable) {
1719 			/* max "linkspeed" idleslope in kbps */
1720 			ring->idleslope = 1000000;
1721 			ring->hicredit = ETH_FRAME_LEN;
1722 		}
1723 
1724 		/* Always set data transfer arbitration to credit-based
1725 		 * shaper algorithm on TQAVCTRL if CBS is enabled for any of
1726 		 * the queues.
1727 		 */
1728 		tqavctrl = rd32(E1000_I210_TQAVCTRL);
1729 		tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
1730 		wr32(E1000_I210_TQAVCTRL, tqavctrl);
1731 
1732 		/* According to i210 datasheet section 7.2.7.7, we should set
1733 		 * the 'idleSlope' field from TQAVCC register following the
1734 		 * equation:
1735 		 *
1736 		 * For 100 Mbps link speed:
1737 		 *
1738 		 *     value = BW * 0x7735 * 0.2                          (E1)
1739 		 *
1740 		 * For 1000Mbps link speed:
1741 		 *
1742 		 *     value = BW * 0x7735 * 2                            (E2)
1743 		 *
1744 		 * E1 and E2 can be merged into one equation as shown below.
1745 		 * Note that 'link-speed' is in Mbps.
1746 		 *
1747 		 *     value = BW * 0x7735 * 2 * link-speed
1748 		 *                           --------------               (E3)
1749 		 *                                1000
1750 		 *
1751 		 * 'BW' is the percentage bandwidth out of full link speed
1752 		 * which can be found with the following equation. Note that
1753 		 * idleSlope here is the parameter from this function which
1754 		 * is in kbps.
1755 		 *
1756 		 *     BW =     idleSlope
1757 		 *          -----------------                             (E4)
1758 		 *          link-speed * 1000
1759 		 *
1760 		 * That said, we can come up with a generic equation to
1761 		 * calculate the value we should set it TQAVCC register by
1762 		 * replacing 'BW' in E3 by E4. The resulting equation is:
1763 		 *
1764 		 * value =     idleSlope     * 0x7735 * 2 * link-speed
1765 		 *         -----------------            --------------    (E5)
1766 		 *         link-speed * 1000                 1000
1767 		 *
1768 		 * 'link-speed' is present in both sides of the fraction so
1769 		 * it is canceled out. The final equation is the following:
1770 		 *
1771 		 *     value = idleSlope * 61034
1772 		 *             -----------------                          (E6)
1773 		 *                  1000000
1774 		 *
1775 		 * NOTE: For i210, given the above, we can see that idleslope
1776 		 *       is represented in 16.38431 kbps units by the value at
1777 		 *       the TQAVCC register (1Gbps / 61034), which reduces
1778 		 *       the granularity for idleslope increments.
1779 		 *       For instance, if you want to configure a 2576kbps
1780 		 *       idleslope, the value to be written on the register
1781 		 *       would have to be 157.23. If rounded down, you end
1782 		 *       up with less bandwidth available than originally
1783 		 *       required (~2572 kbps). If rounded up, you end up
1784 		 *       with a higher bandwidth (~2589 kbps). Below the
1785 		 *       approach we take is to always round up the
1786 		 *       calculated value, so the resulting bandwidth might
1787 		 *       be slightly higher for some configurations.
1788 		 */
1789 		value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
1790 
1791 		tqavcc = rd32(E1000_I210_TQAVCC(queue));
1792 		tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1793 		tqavcc |= value;
1794 		wr32(E1000_I210_TQAVCC(queue), tqavcc);
1795 
1796 		wr32(E1000_I210_TQAVHC(queue),
1797 		     0x80000000 + ring->hicredit * 0x7735);
1798 	} else {
1799 
1800 		/* Set idleSlope to zero. */
1801 		tqavcc = rd32(E1000_I210_TQAVCC(queue));
1802 		tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1803 		wr32(E1000_I210_TQAVCC(queue), tqavcc);
1804 
1805 		/* Set hiCredit to zero. */
1806 		wr32(E1000_I210_TQAVHC(queue), 0);
1807 
1808 		/* If CBS is not enabled for any queues anymore, then return to
1809 		 * the default state of Data Transmission Arbitration on
1810 		 * TQAVCTRL.
1811 		 */
1812 		if (!is_any_cbs_enabled(adapter)) {
1813 			tqavctrl = rd32(E1000_I210_TQAVCTRL);
1814 			tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
1815 			wr32(E1000_I210_TQAVCTRL, tqavctrl);
1816 		}
1817 	}
1818 
1819 	/* If LaunchTime is enabled, set DataTranTIM. */
1820 	if (ring->launchtime_enable) {
1821 		/* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
1822 		 * for any of the SR queues, and configure fetchtime delta.
1823 		 * XXX NOTE:
1824 		 *     - LaunchTime will be enabled for all SR queues.
1825 		 *     - A fixed offset can be added relative to the launch
1826 		 *       time of all packets if configured at reg LAUNCH_OS0.
1827 		 *       We are keeping it as 0 for now (default value).
1828 		 */
1829 		tqavctrl = rd32(E1000_I210_TQAVCTRL);
1830 		tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
1831 		       E1000_TQAVCTRL_FETCHTIME_DELTA;
1832 		wr32(E1000_I210_TQAVCTRL, tqavctrl);
1833 	} else {
1834 		/* If Launchtime is not enabled for any SR queues anymore,
1835 		 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
1836 		 * effectively disabling Launchtime.
1837 		 */
1838 		if (!is_any_txtime_enabled(adapter)) {
1839 			tqavctrl = rd32(E1000_I210_TQAVCTRL);
1840 			tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
1841 			tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
1842 			wr32(E1000_I210_TQAVCTRL, tqavctrl);
1843 		}
1844 	}
1845 
1846 	/* XXX: In i210 controller the sendSlope and loCredit parameters from
1847 	 * CBS are not configurable by software so we don't do any 'controller
1848 	 * configuration' in respect to these parameters.
1849 	 */
1850 
1851 	netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1852 		   ring->cbs_enable ? "enabled" : "disabled",
1853 		   ring->launchtime_enable ? "enabled" : "disabled",
1854 		   queue,
1855 		   ring->idleslope, ring->sendslope,
1856 		   ring->hicredit, ring->locredit);
1857 }
1858 
1859 static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
1860 				  bool enable)
1861 {
1862 	struct igb_ring *ring;
1863 
1864 	if (queue < 0 || queue > adapter->num_tx_queues)
1865 		return -EINVAL;
1866 
1867 	ring = adapter->tx_ring[queue];
1868 	ring->launchtime_enable = enable;
1869 
1870 	return 0;
1871 }
1872 
1873 static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
1874 			       bool enable, int idleslope, int sendslope,
1875 			       int hicredit, int locredit)
1876 {
1877 	struct igb_ring *ring;
1878 
1879 	if (queue < 0 || queue > adapter->num_tx_queues)
1880 		return -EINVAL;
1881 
1882 	ring = adapter->tx_ring[queue];
1883 
1884 	ring->cbs_enable = enable;
1885 	ring->idleslope = idleslope;
1886 	ring->sendslope = sendslope;
1887 	ring->hicredit = hicredit;
1888 	ring->locredit = locredit;
1889 
1890 	return 0;
1891 }
1892 
1893 /**
1894  *  igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
1895  *  @adapter: pointer to adapter struct
1896  *
1897  *  Configure TQAVCTRL register switching the controller's Tx mode
1898  *  if FQTSS mode is enabled or disabled. Additionally, will issue
1899  *  a call to igb_config_tx_modes() per queue so any previously saved
1900  *  Tx parameters are applied.
1901  **/
1902 static void igb_setup_tx_mode(struct igb_adapter *adapter)
1903 {
1904 	struct net_device *netdev = adapter->netdev;
1905 	struct e1000_hw *hw = &adapter->hw;
1906 	u32 val;
1907 
1908 	/* Only i210 controller supports changing the transmission mode. */
1909 	if (hw->mac.type != e1000_i210)
1910 		return;
1911 
1912 	if (is_fqtss_enabled(adapter)) {
1913 		int i, max_queue;
1914 
1915 		/* Configure TQAVCTRL register: set transmit mode to 'Qav',
1916 		 * set data fetch arbitration to 'round robin', set SP_WAIT_SR
1917 		 * so SP queues wait for SR ones.
1918 		 */
1919 		val = rd32(E1000_I210_TQAVCTRL);
1920 		val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
1921 		val &= ~E1000_TQAVCTRL_DATAFETCHARB;
1922 		wr32(E1000_I210_TQAVCTRL, val);
1923 
1924 		/* Configure Tx and Rx packet buffers sizes as described in
1925 		 * i210 datasheet section 7.2.7.7.
1926 		 */
1927 		val = rd32(E1000_TXPBS);
1928 		val &= ~I210_TXPBSIZE_MASK;
1929 		val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB |
1930 			I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB;
1931 		wr32(E1000_TXPBS, val);
1932 
1933 		val = rd32(E1000_RXPBS);
1934 		val &= ~I210_RXPBSIZE_MASK;
1935 		val |= I210_RXPBSIZE_PB_30KB;
1936 		wr32(E1000_RXPBS, val);
1937 
1938 		/* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1939 		 * register should not exceed the buffer size programmed in
1940 		 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1941 		 * so according to the datasheet we should set MAX_TPKT_SIZE to
1942 		 * 4kB / 64.
1943 		 *
1944 		 * However, when we do so, no frame from queue 2 and 3 are
1945 		 * transmitted.  It seems the MAX_TPKT_SIZE should not be great
1946 		 * or _equal_ to the buffer size programmed in TXPBS. For this
1947 		 * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1948 		 */
1949 		val = (4096 - 1) / 64;
1950 		wr32(E1000_I210_DTXMXPKTSZ, val);
1951 
1952 		/* Since FQTSS mode is enabled, apply any CBS configuration
1953 		 * previously set. If no previous CBS configuration has been
1954 		 * done, then the initial configuration is applied, which means
1955 		 * CBS is disabled.
1956 		 */
1957 		max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
1958 			    adapter->num_tx_queues : I210_SR_QUEUES_NUM;
1959 
1960 		for (i = 0; i < max_queue; i++) {
1961 			igb_config_tx_modes(adapter, i);
1962 		}
1963 	} else {
1964 		wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
1965 		wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
1966 		wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
1967 
1968 		val = rd32(E1000_I210_TQAVCTRL);
1969 		/* According to Section 8.12.21, the other flags we've set when
1970 		 * enabling FQTSS are not relevant when disabling FQTSS so we
1971 		 * don't set they here.
1972 		 */
1973 		val &= ~E1000_TQAVCTRL_XMIT_MODE;
1974 		wr32(E1000_I210_TQAVCTRL, val);
1975 	}
1976 
1977 	netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
1978 		   "enabled" : "disabled");
1979 }
1980 
1981 /**
1982  *  igb_configure - configure the hardware for RX and TX
1983  *  @adapter: private board structure
1984  **/
1985 static void igb_configure(struct igb_adapter *adapter)
1986 {
1987 	struct net_device *netdev = adapter->netdev;
1988 	int i;
1989 
1990 	igb_get_hw_control(adapter);
1991 	igb_set_rx_mode(netdev);
1992 	igb_setup_tx_mode(adapter);
1993 
1994 	igb_restore_vlan(adapter);
1995 
1996 	igb_setup_tctl(adapter);
1997 	igb_setup_mrqc(adapter);
1998 	igb_setup_rctl(adapter);
1999 
2000 	igb_nfc_filter_restore(adapter);
2001 	igb_configure_tx(adapter);
2002 	igb_configure_rx(adapter);
2003 
2004 	igb_rx_fifo_flush_82575(&adapter->hw);
2005 
2006 	/* call igb_desc_unused which always leaves
2007 	 * at least 1 descriptor unused to make sure
2008 	 * next_to_use != next_to_clean
2009 	 */
2010 	for (i = 0; i < adapter->num_rx_queues; i++) {
2011 		struct igb_ring *ring = adapter->rx_ring[i];
2012 		igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
2013 	}
2014 }
2015 
2016 /**
2017  *  igb_power_up_link - Power up the phy/serdes link
2018  *  @adapter: address of board private structure
2019  **/
2020 void igb_power_up_link(struct igb_adapter *adapter)
2021 {
2022 	igb_reset_phy(&adapter->hw);
2023 
2024 	if (adapter->hw.phy.media_type == e1000_media_type_copper)
2025 		igb_power_up_phy_copper(&adapter->hw);
2026 	else
2027 		igb_power_up_serdes_link_82575(&adapter->hw);
2028 
2029 	igb_setup_link(&adapter->hw);
2030 }
2031 
2032 /**
2033  *  igb_power_down_link - Power down the phy/serdes link
2034  *  @adapter: address of board private structure
2035  */
2036 static void igb_power_down_link(struct igb_adapter *adapter)
2037 {
2038 	if (adapter->hw.phy.media_type == e1000_media_type_copper)
2039 		igb_power_down_phy_copper_82575(&adapter->hw);
2040 	else
2041 		igb_shutdown_serdes_link_82575(&adapter->hw);
2042 }
2043 
2044 /**
2045  * igb_check_swap_media -  Detect and switch function for Media Auto Sense
2046  * @adapter: address of the board private structure
2047  **/
2048 static void igb_check_swap_media(struct igb_adapter *adapter)
2049 {
2050 	struct e1000_hw *hw = &adapter->hw;
2051 	u32 ctrl_ext, connsw;
2052 	bool swap_now = false;
2053 
2054 	ctrl_ext = rd32(E1000_CTRL_EXT);
2055 	connsw = rd32(E1000_CONNSW);
2056 
2057 	/* need to live swap if current media is copper and we have fiber/serdes
2058 	 * to go to.
2059 	 */
2060 
2061 	if ((hw->phy.media_type == e1000_media_type_copper) &&
2062 	    (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
2063 		swap_now = true;
2064 	} else if ((hw->phy.media_type != e1000_media_type_copper) &&
2065 		   !(connsw & E1000_CONNSW_SERDESD)) {
2066 		/* copper signal takes time to appear */
2067 		if (adapter->copper_tries < 4) {
2068 			adapter->copper_tries++;
2069 			connsw |= E1000_CONNSW_AUTOSENSE_CONF;
2070 			wr32(E1000_CONNSW, connsw);
2071 			return;
2072 		} else {
2073 			adapter->copper_tries = 0;
2074 			if ((connsw & E1000_CONNSW_PHYSD) &&
2075 			    (!(connsw & E1000_CONNSW_PHY_PDN))) {
2076 				swap_now = true;
2077 				connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
2078 				wr32(E1000_CONNSW, connsw);
2079 			}
2080 		}
2081 	}
2082 
2083 	if (!swap_now)
2084 		return;
2085 
2086 	switch (hw->phy.media_type) {
2087 	case e1000_media_type_copper:
2088 		netdev_info(adapter->netdev,
2089 			"MAS: changing media to fiber/serdes\n");
2090 		ctrl_ext |=
2091 			E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2092 		adapter->flags |= IGB_FLAG_MEDIA_RESET;
2093 		adapter->copper_tries = 0;
2094 		break;
2095 	case e1000_media_type_internal_serdes:
2096 	case e1000_media_type_fiber:
2097 		netdev_info(adapter->netdev,
2098 			"MAS: changing media to copper\n");
2099 		ctrl_ext &=
2100 			~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2101 		adapter->flags |= IGB_FLAG_MEDIA_RESET;
2102 		break;
2103 	default:
2104 		/* shouldn't get here during regular operation */
2105 		netdev_err(adapter->netdev,
2106 			"AMS: Invalid media type found, returning\n");
2107 		break;
2108 	}
2109 	wr32(E1000_CTRL_EXT, ctrl_ext);
2110 }
2111 
2112 /**
2113  *  igb_up - Open the interface and prepare it to handle traffic
2114  *  @adapter: board private structure
2115  **/
2116 int igb_up(struct igb_adapter *adapter)
2117 {
2118 	struct e1000_hw *hw = &adapter->hw;
2119 	int i;
2120 
2121 	/* hardware has been reset, we need to reload some things */
2122 	igb_configure(adapter);
2123 
2124 	clear_bit(__IGB_DOWN, &adapter->state);
2125 
2126 	for (i = 0; i < adapter->num_q_vectors; i++)
2127 		napi_enable(&(adapter->q_vector[i]->napi));
2128 
2129 	if (adapter->flags & IGB_FLAG_HAS_MSIX)
2130 		igb_configure_msix(adapter);
2131 	else
2132 		igb_assign_vector(adapter->q_vector[0], 0);
2133 
2134 	/* Clear any pending interrupts. */
2135 	rd32(E1000_TSICR);
2136 	rd32(E1000_ICR);
2137 	igb_irq_enable(adapter);
2138 
2139 	/* notify VFs that reset has been completed */
2140 	if (adapter->vfs_allocated_count) {
2141 		u32 reg_data = rd32(E1000_CTRL_EXT);
2142 
2143 		reg_data |= E1000_CTRL_EXT_PFRSTD;
2144 		wr32(E1000_CTRL_EXT, reg_data);
2145 	}
2146 
2147 	netif_tx_start_all_queues(adapter->netdev);
2148 
2149 	/* start the watchdog. */
2150 	hw->mac.get_link_status = 1;
2151 	schedule_work(&adapter->watchdog_task);
2152 
2153 	if ((adapter->flags & IGB_FLAG_EEE) &&
2154 	    (!hw->dev_spec._82575.eee_disable))
2155 		adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
2156 
2157 	return 0;
2158 }
2159 
2160 void igb_down(struct igb_adapter *adapter)
2161 {
2162 	struct net_device *netdev = adapter->netdev;
2163 	struct e1000_hw *hw = &adapter->hw;
2164 	u32 tctl, rctl;
2165 	int i;
2166 
2167 	/* signal that we're down so the interrupt handler does not
2168 	 * reschedule our watchdog timer
2169 	 */
2170 	set_bit(__IGB_DOWN, &adapter->state);
2171 
2172 	/* disable receives in the hardware */
2173 	rctl = rd32(E1000_RCTL);
2174 	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
2175 	/* flush and sleep below */
2176 
2177 	igb_nfc_filter_exit(adapter);
2178 
2179 	netif_carrier_off(netdev);
2180 	netif_tx_stop_all_queues(netdev);
2181 
2182 	/* disable transmits in the hardware */
2183 	tctl = rd32(E1000_TCTL);
2184 	tctl &= ~E1000_TCTL_EN;
2185 	wr32(E1000_TCTL, tctl);
2186 	/* flush both disables and wait for them to finish */
2187 	wrfl();
2188 	usleep_range(10000, 11000);
2189 
2190 	igb_irq_disable(adapter);
2191 
2192 	adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
2193 
2194 	for (i = 0; i < adapter->num_q_vectors; i++) {
2195 		if (adapter->q_vector[i]) {
2196 			napi_synchronize(&adapter->q_vector[i]->napi);
2197 			napi_disable(&adapter->q_vector[i]->napi);
2198 		}
2199 	}
2200 
2201 	del_timer_sync(&adapter->watchdog_timer);
2202 	del_timer_sync(&adapter->phy_info_timer);
2203 
2204 	/* record the stats before reset*/
2205 	spin_lock(&adapter->stats64_lock);
2206 	igb_update_stats(adapter);
2207 	spin_unlock(&adapter->stats64_lock);
2208 
2209 	adapter->link_speed = 0;
2210 	adapter->link_duplex = 0;
2211 
2212 	if (!pci_channel_offline(adapter->pdev))
2213 		igb_reset(adapter);
2214 
2215 	/* clear VLAN promisc flag so VFTA will be updated if necessary */
2216 	adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
2217 
2218 	igb_clean_all_tx_rings(adapter);
2219 	igb_clean_all_rx_rings(adapter);
2220 #ifdef CONFIG_IGB_DCA
2221 
2222 	/* since we reset the hardware DCA settings were cleared */
2223 	igb_setup_dca(adapter);
2224 #endif
2225 }
2226 
2227 void igb_reinit_locked(struct igb_adapter *adapter)
2228 {
2229 	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2230 		usleep_range(1000, 2000);
2231 	igb_down(adapter);
2232 	igb_up(adapter);
2233 	clear_bit(__IGB_RESETTING, &adapter->state);
2234 }
2235 
2236 /** igb_enable_mas - Media Autosense re-enable after swap
2237  *
2238  * @adapter: adapter struct
2239  **/
2240 static void igb_enable_mas(struct igb_adapter *adapter)
2241 {
2242 	struct e1000_hw *hw = &adapter->hw;
2243 	u32 connsw = rd32(E1000_CONNSW);
2244 
2245 	/* configure for SerDes media detect */
2246 	if ((hw->phy.media_type == e1000_media_type_copper) &&
2247 	    (!(connsw & E1000_CONNSW_SERDESD))) {
2248 		connsw |= E1000_CONNSW_ENRGSRC;
2249 		connsw |= E1000_CONNSW_AUTOSENSE_EN;
2250 		wr32(E1000_CONNSW, connsw);
2251 		wrfl();
2252 	}
2253 }
2254 
2255 void igb_reset(struct igb_adapter *adapter)
2256 {
2257 	struct pci_dev *pdev = adapter->pdev;
2258 	struct e1000_hw *hw = &adapter->hw;
2259 	struct e1000_mac_info *mac = &hw->mac;
2260 	struct e1000_fc_info *fc = &hw->fc;
2261 	u32 pba, hwm;
2262 
2263 	/* Repartition Pba for greater than 9k mtu
2264 	 * To take effect CTRL.RST is required.
2265 	 */
2266 	switch (mac->type) {
2267 	case e1000_i350:
2268 	case e1000_i354:
2269 	case e1000_82580:
2270 		pba = rd32(E1000_RXPBS);
2271 		pba = igb_rxpbs_adjust_82580(pba);
2272 		break;
2273 	case e1000_82576:
2274 		pba = rd32(E1000_RXPBS);
2275 		pba &= E1000_RXPBS_SIZE_MASK_82576;
2276 		break;
2277 	case e1000_82575:
2278 	case e1000_i210:
2279 	case e1000_i211:
2280 	default:
2281 		pba = E1000_PBA_34K;
2282 		break;
2283 	}
2284 
2285 	if (mac->type == e1000_82575) {
2286 		u32 min_rx_space, min_tx_space, needed_tx_space;
2287 
2288 		/* write Rx PBA so that hardware can report correct Tx PBA */
2289 		wr32(E1000_PBA, pba);
2290 
2291 		/* To maintain wire speed transmits, the Tx FIFO should be
2292 		 * large enough to accommodate two full transmit packets,
2293 		 * rounded up to the next 1KB and expressed in KB.  Likewise,
2294 		 * the Rx FIFO should be large enough to accommodate at least
2295 		 * one full receive packet and is similarly rounded up and
2296 		 * expressed in KB.
2297 		 */
2298 		min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
2299 
2300 		/* The Tx FIFO also stores 16 bytes of information about the Tx
2301 		 * but don't include Ethernet FCS because hardware appends it.
2302 		 * We only need to round down to the nearest 512 byte block
2303 		 * count since the value we care about is 2 frames, not 1.
2304 		 */
2305 		min_tx_space = adapter->max_frame_size;
2306 		min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
2307 		min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
2308 
2309 		/* upper 16 bits has Tx packet buffer allocation size in KB */
2310 		needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
2311 
2312 		/* If current Tx allocation is less than the min Tx FIFO size,
2313 		 * and the min Tx FIFO size is less than the current Rx FIFO
2314 		 * allocation, take space away from current Rx allocation.
2315 		 */
2316 		if (needed_tx_space < pba) {
2317 			pba -= needed_tx_space;
2318 
2319 			/* if short on Rx space, Rx wins and must trump Tx
2320 			 * adjustment
2321 			 */
2322 			if (pba < min_rx_space)
2323 				pba = min_rx_space;
2324 		}
2325 
2326 		/* adjust PBA for jumbo frames */
2327 		wr32(E1000_PBA, pba);
2328 	}
2329 
2330 	/* flow control settings
2331 	 * The high water mark must be low enough to fit one full frame
2332 	 * after transmitting the pause frame.  As such we must have enough
2333 	 * space to allow for us to complete our current transmit and then
2334 	 * receive the frame that is in progress from the link partner.
2335 	 * Set it to:
2336 	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2337 	 */
2338 	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
2339 
2340 	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
2341 	fc->low_water = fc->high_water - 16;
2342 	fc->pause_time = 0xFFFF;
2343 	fc->send_xon = 1;
2344 	fc->current_mode = fc->requested_mode;
2345 
2346 	/* disable receive for all VFs and wait one second */
2347 	if (adapter->vfs_allocated_count) {
2348 		int i;
2349 
2350 		for (i = 0 ; i < adapter->vfs_allocated_count; i++)
2351 			adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
2352 
2353 		/* ping all the active vfs to let them know we are going down */
2354 		igb_ping_all_vfs(adapter);
2355 
2356 		/* disable transmits and receives */
2357 		wr32(E1000_VFRE, 0);
2358 		wr32(E1000_VFTE, 0);
2359 	}
2360 
2361 	/* Allow time for pending master requests to run */
2362 	hw->mac.ops.reset_hw(hw);
2363 	wr32(E1000_WUC, 0);
2364 
2365 	if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2366 		/* need to resetup here after media swap */
2367 		adapter->ei.get_invariants(hw);
2368 		adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2369 	}
2370 	if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
2371 	    (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2372 		igb_enable_mas(adapter);
2373 	}
2374 	if (hw->mac.ops.init_hw(hw))
2375 		dev_err(&pdev->dev, "Hardware Error\n");
2376 
2377 	/* RAR registers were cleared during init_hw, clear mac table */
2378 	igb_flush_mac_table(adapter);
2379 	__dev_uc_unsync(adapter->netdev, NULL);
2380 
2381 	/* Recover default RAR entry */
2382 	igb_set_default_mac_filter(adapter);
2383 
2384 	/* Flow control settings reset on hardware reset, so guarantee flow
2385 	 * control is off when forcing speed.
2386 	 */
2387 	if (!hw->mac.autoneg)
2388 		igb_force_mac_fc(hw);
2389 
2390 	igb_init_dmac(adapter, pba);
2391 #ifdef CONFIG_IGB_HWMON
2392 	/* Re-initialize the thermal sensor on i350 devices. */
2393 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
2394 		if (mac->type == e1000_i350 && hw->bus.func == 0) {
2395 			/* If present, re-initialize the external thermal sensor
2396 			 * interface.
2397 			 */
2398 			if (adapter->ets)
2399 				mac->ops.init_thermal_sensor_thresh(hw);
2400 		}
2401 	}
2402 #endif
2403 	/* Re-establish EEE setting */
2404 	if (hw->phy.media_type == e1000_media_type_copper) {
2405 		switch (mac->type) {
2406 		case e1000_i350:
2407 		case e1000_i210:
2408 		case e1000_i211:
2409 			igb_set_eee_i350(hw, true, true);
2410 			break;
2411 		case e1000_i354:
2412 			igb_set_eee_i354(hw, true, true);
2413 			break;
2414 		default:
2415 			break;
2416 		}
2417 	}
2418 	if (!netif_running(adapter->netdev))
2419 		igb_power_down_link(adapter);
2420 
2421 	igb_update_mng_vlan(adapter);
2422 
2423 	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2424 	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2425 
2426 	/* Re-enable PTP, where applicable. */
2427 	if (adapter->ptp_flags & IGB_PTP_ENABLED)
2428 		igb_ptp_reset(adapter);
2429 
2430 	igb_get_phy_info(hw);
2431 }
2432 
2433 static netdev_features_t igb_fix_features(struct net_device *netdev,
2434 	netdev_features_t features)
2435 {
2436 	/* Since there is no support for separate Rx/Tx vlan accel
2437 	 * enable/disable make sure Tx flag is always in same state as Rx.
2438 	 */
2439 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
2440 		features |= NETIF_F_HW_VLAN_CTAG_TX;
2441 	else
2442 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2443 
2444 	return features;
2445 }
2446 
2447 static int igb_set_features(struct net_device *netdev,
2448 	netdev_features_t features)
2449 {
2450 	netdev_features_t changed = netdev->features ^ features;
2451 	struct igb_adapter *adapter = netdev_priv(netdev);
2452 
2453 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2454 		igb_vlan_mode(netdev, features);
2455 
2456 	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2457 		return 0;
2458 
2459 	if (!(features & NETIF_F_NTUPLE)) {
2460 		struct hlist_node *node2;
2461 		struct igb_nfc_filter *rule;
2462 
2463 		spin_lock(&adapter->nfc_lock);
2464 		hlist_for_each_entry_safe(rule, node2,
2465 					  &adapter->nfc_filter_list, nfc_node) {
2466 			igb_erase_filter(adapter, rule);
2467 			hlist_del(&rule->nfc_node);
2468 			kfree(rule);
2469 		}
2470 		spin_unlock(&adapter->nfc_lock);
2471 		adapter->nfc_filter_count = 0;
2472 	}
2473 
2474 	netdev->features = features;
2475 
2476 	if (netif_running(netdev))
2477 		igb_reinit_locked(adapter);
2478 	else
2479 		igb_reset(adapter);
2480 
2481 	return 1;
2482 }
2483 
2484 static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2485 			   struct net_device *dev,
2486 			   const unsigned char *addr, u16 vid,
2487 			   u16 flags,
2488 			   struct netlink_ext_ack *extack)
2489 {
2490 	/* guarantee we can provide a unique filter for the unicast address */
2491 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2492 		struct igb_adapter *adapter = netdev_priv(dev);
2493 		int vfn = adapter->vfs_allocated_count;
2494 
2495 		if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2496 			return -ENOMEM;
2497 	}
2498 
2499 	return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2500 }
2501 
2502 #define IGB_MAX_MAC_HDR_LEN	127
2503 #define IGB_MAX_NETWORK_HDR_LEN	511
2504 
2505 static netdev_features_t
2506 igb_features_check(struct sk_buff *skb, struct net_device *dev,
2507 		   netdev_features_t features)
2508 {
2509 	unsigned int network_hdr_len, mac_hdr_len;
2510 
2511 	/* Make certain the headers can be described by a context descriptor */
2512 	mac_hdr_len = skb_network_header(skb) - skb->data;
2513 	if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2514 		return features & ~(NETIF_F_HW_CSUM |
2515 				    NETIF_F_SCTP_CRC |
2516 				    NETIF_F_GSO_UDP_L4 |
2517 				    NETIF_F_HW_VLAN_CTAG_TX |
2518 				    NETIF_F_TSO |
2519 				    NETIF_F_TSO6);
2520 
2521 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2522 	if (unlikely(network_hdr_len >  IGB_MAX_NETWORK_HDR_LEN))
2523 		return features & ~(NETIF_F_HW_CSUM |
2524 				    NETIF_F_SCTP_CRC |
2525 				    NETIF_F_GSO_UDP_L4 |
2526 				    NETIF_F_TSO |
2527 				    NETIF_F_TSO6);
2528 
2529 	/* We can only support IPV4 TSO in tunnels if we can mangle the
2530 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2531 	 */
2532 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2533 		features &= ~NETIF_F_TSO;
2534 
2535 	return features;
2536 }
2537 
2538 static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
2539 {
2540 	if (!is_fqtss_enabled(adapter)) {
2541 		enable_fqtss(adapter, true);
2542 		return;
2543 	}
2544 
2545 	igb_config_tx_modes(adapter, queue);
2546 
2547 	if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
2548 		enable_fqtss(adapter, false);
2549 }
2550 
2551 static int igb_offload_cbs(struct igb_adapter *adapter,
2552 			   struct tc_cbs_qopt_offload *qopt)
2553 {
2554 	struct e1000_hw *hw = &adapter->hw;
2555 	int err;
2556 
2557 	/* CBS offloading is only supported by i210 controller. */
2558 	if (hw->mac.type != e1000_i210)
2559 		return -EOPNOTSUPP;
2560 
2561 	/* CBS offloading is only supported by queue 0 and queue 1. */
2562 	if (qopt->queue < 0 || qopt->queue > 1)
2563 		return -EINVAL;
2564 
2565 	err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
2566 				  qopt->idleslope, qopt->sendslope,
2567 				  qopt->hicredit, qopt->locredit);
2568 	if (err)
2569 		return err;
2570 
2571 	igb_offload_apply(adapter, qopt->queue);
2572 
2573 	return 0;
2574 }
2575 
2576 #define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
2577 #define VLAN_PRIO_FULL_MASK (0x07)
2578 
2579 static int igb_parse_cls_flower(struct igb_adapter *adapter,
2580 				struct flow_cls_offload *f,
2581 				int traffic_class,
2582 				struct igb_nfc_filter *input)
2583 {
2584 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2585 	struct flow_dissector *dissector = rule->match.dissector;
2586 	struct netlink_ext_ack *extack = f->common.extack;
2587 
2588 	if (dissector->used_keys &
2589 	    ~(BIT(FLOW_DISSECTOR_KEY_BASIC) |
2590 	      BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2591 	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2592 	      BIT(FLOW_DISSECTOR_KEY_VLAN))) {
2593 		NL_SET_ERR_MSG_MOD(extack,
2594 				   "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
2595 		return -EOPNOTSUPP;
2596 	}
2597 
2598 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2599 		struct flow_match_eth_addrs match;
2600 
2601 		flow_rule_match_eth_addrs(rule, &match);
2602 		if (!is_zero_ether_addr(match.mask->dst)) {
2603 			if (!is_broadcast_ether_addr(match.mask->dst)) {
2604 				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
2605 				return -EINVAL;
2606 			}
2607 
2608 			input->filter.match_flags |=
2609 				IGB_FILTER_FLAG_DST_MAC_ADDR;
2610 			ether_addr_copy(input->filter.dst_addr, match.key->dst);
2611 		}
2612 
2613 		if (!is_zero_ether_addr(match.mask->src)) {
2614 			if (!is_broadcast_ether_addr(match.mask->src)) {
2615 				NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
2616 				return -EINVAL;
2617 			}
2618 
2619 			input->filter.match_flags |=
2620 				IGB_FILTER_FLAG_SRC_MAC_ADDR;
2621 			ether_addr_copy(input->filter.src_addr, match.key->src);
2622 		}
2623 	}
2624 
2625 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2626 		struct flow_match_basic match;
2627 
2628 		flow_rule_match_basic(rule, &match);
2629 		if (match.mask->n_proto) {
2630 			if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
2631 				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
2632 				return -EINVAL;
2633 			}
2634 
2635 			input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
2636 			input->filter.etype = match.key->n_proto;
2637 		}
2638 	}
2639 
2640 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2641 		struct flow_match_vlan match;
2642 
2643 		flow_rule_match_vlan(rule, &match);
2644 		if (match.mask->vlan_priority) {
2645 			if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
2646 				NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
2647 				return -EINVAL;
2648 			}
2649 
2650 			input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
2651 			input->filter.vlan_tci =
2652 				(__force __be16)match.key->vlan_priority;
2653 		}
2654 	}
2655 
2656 	input->action = traffic_class;
2657 	input->cookie = f->cookie;
2658 
2659 	return 0;
2660 }
2661 
2662 static int igb_configure_clsflower(struct igb_adapter *adapter,
2663 				   struct flow_cls_offload *cls_flower)
2664 {
2665 	struct netlink_ext_ack *extack = cls_flower->common.extack;
2666 	struct igb_nfc_filter *filter, *f;
2667 	int err, tc;
2668 
2669 	tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2670 	if (tc < 0) {
2671 		NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
2672 		return -EINVAL;
2673 	}
2674 
2675 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2676 	if (!filter)
2677 		return -ENOMEM;
2678 
2679 	err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
2680 	if (err < 0)
2681 		goto err_parse;
2682 
2683 	spin_lock(&adapter->nfc_lock);
2684 
2685 	hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
2686 		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2687 			err = -EEXIST;
2688 			NL_SET_ERR_MSG_MOD(extack,
2689 					   "This filter is already set in ethtool");
2690 			goto err_locked;
2691 		}
2692 	}
2693 
2694 	hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
2695 		if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2696 			err = -EEXIST;
2697 			NL_SET_ERR_MSG_MOD(extack,
2698 					   "This filter is already set in cls_flower");
2699 			goto err_locked;
2700 		}
2701 	}
2702 
2703 	err = igb_add_filter(adapter, filter);
2704 	if (err < 0) {
2705 		NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
2706 		goto err_locked;
2707 	}
2708 
2709 	hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
2710 
2711 	spin_unlock(&adapter->nfc_lock);
2712 
2713 	return 0;
2714 
2715 err_locked:
2716 	spin_unlock(&adapter->nfc_lock);
2717 
2718 err_parse:
2719 	kfree(filter);
2720 
2721 	return err;
2722 }
2723 
2724 static int igb_delete_clsflower(struct igb_adapter *adapter,
2725 				struct flow_cls_offload *cls_flower)
2726 {
2727 	struct igb_nfc_filter *filter;
2728 	int err;
2729 
2730 	spin_lock(&adapter->nfc_lock);
2731 
2732 	hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
2733 		if (filter->cookie == cls_flower->cookie)
2734 			break;
2735 
2736 	if (!filter) {
2737 		err = -ENOENT;
2738 		goto out;
2739 	}
2740 
2741 	err = igb_erase_filter(adapter, filter);
2742 	if (err < 0)
2743 		goto out;
2744 
2745 	hlist_del(&filter->nfc_node);
2746 	kfree(filter);
2747 
2748 out:
2749 	spin_unlock(&adapter->nfc_lock);
2750 
2751 	return err;
2752 }
2753 
2754 static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
2755 				   struct flow_cls_offload *cls_flower)
2756 {
2757 	switch (cls_flower->command) {
2758 	case FLOW_CLS_REPLACE:
2759 		return igb_configure_clsflower(adapter, cls_flower);
2760 	case FLOW_CLS_DESTROY:
2761 		return igb_delete_clsflower(adapter, cls_flower);
2762 	case FLOW_CLS_STATS:
2763 		return -EOPNOTSUPP;
2764 	default:
2765 		return -EOPNOTSUPP;
2766 	}
2767 }
2768 
2769 static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2770 				 void *cb_priv)
2771 {
2772 	struct igb_adapter *adapter = cb_priv;
2773 
2774 	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
2775 		return -EOPNOTSUPP;
2776 
2777 	switch (type) {
2778 	case TC_SETUP_CLSFLOWER:
2779 		return igb_setup_tc_cls_flower(adapter, type_data);
2780 
2781 	default:
2782 		return -EOPNOTSUPP;
2783 	}
2784 }
2785 
2786 static int igb_offload_txtime(struct igb_adapter *adapter,
2787 			      struct tc_etf_qopt_offload *qopt)
2788 {
2789 	struct e1000_hw *hw = &adapter->hw;
2790 	int err;
2791 
2792 	/* Launchtime offloading is only supported by i210 controller. */
2793 	if (hw->mac.type != e1000_i210)
2794 		return -EOPNOTSUPP;
2795 
2796 	/* Launchtime offloading is only supported by queues 0 and 1. */
2797 	if (qopt->queue < 0 || qopt->queue > 1)
2798 		return -EINVAL;
2799 
2800 	err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
2801 	if (err)
2802 		return err;
2803 
2804 	igb_offload_apply(adapter, qopt->queue);
2805 
2806 	return 0;
2807 }
2808 
2809 static LIST_HEAD(igb_block_cb_list);
2810 
2811 static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
2812 			void *type_data)
2813 {
2814 	struct igb_adapter *adapter = netdev_priv(dev);
2815 
2816 	switch (type) {
2817 	case TC_SETUP_QDISC_CBS:
2818 		return igb_offload_cbs(adapter, type_data);
2819 	case TC_SETUP_BLOCK:
2820 		return flow_block_cb_setup_simple(type_data,
2821 						  &igb_block_cb_list,
2822 						  igb_setup_tc_block_cb,
2823 						  adapter, adapter, true);
2824 
2825 	case TC_SETUP_QDISC_ETF:
2826 		return igb_offload_txtime(adapter, type_data);
2827 
2828 	default:
2829 		return -EOPNOTSUPP;
2830 	}
2831 }
2832 
2833 static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf)
2834 {
2835 	int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD;
2836 	struct igb_adapter *adapter = netdev_priv(dev);
2837 	struct bpf_prog *prog = bpf->prog, *old_prog;
2838 	bool running = netif_running(dev);
2839 	bool need_reset;
2840 
2841 	/* verify igb ring attributes are sufficient for XDP */
2842 	for (i = 0; i < adapter->num_rx_queues; i++) {
2843 		struct igb_ring *ring = adapter->rx_ring[i];
2844 
2845 		if (frame_size > igb_rx_bufsz(ring)) {
2846 			NL_SET_ERR_MSG_MOD(bpf->extack,
2847 					   "The RX buffer size is too small for the frame size");
2848 			netdev_warn(dev, "XDP RX buffer size %d is too small for the frame size %d\n",
2849 				    igb_rx_bufsz(ring), frame_size);
2850 			return -EINVAL;
2851 		}
2852 	}
2853 
2854 	old_prog = xchg(&adapter->xdp_prog, prog);
2855 	need_reset = (!!prog != !!old_prog);
2856 
2857 	/* device is up and bpf is added/removed, must setup the RX queues */
2858 	if (need_reset && running) {
2859 		igb_close(dev);
2860 	} else {
2861 		for (i = 0; i < adapter->num_rx_queues; i++)
2862 			(void)xchg(&adapter->rx_ring[i]->xdp_prog,
2863 			    adapter->xdp_prog);
2864 	}
2865 
2866 	if (old_prog)
2867 		bpf_prog_put(old_prog);
2868 
2869 	/* bpf is just replaced, RXQ and MTU are already setup */
2870 	if (!need_reset)
2871 		return 0;
2872 
2873 	if (running)
2874 		igb_open(dev);
2875 
2876 	return 0;
2877 }
2878 
2879 static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2880 {
2881 	switch (xdp->command) {
2882 	case XDP_SETUP_PROG:
2883 		return igb_xdp_setup(dev, xdp);
2884 	default:
2885 		return -EINVAL;
2886 	}
2887 }
2888 
2889 static void igb_xdp_ring_update_tail(struct igb_ring *ring)
2890 {
2891 	/* Force memory writes to complete before letting h/w know there
2892 	 * are new descriptors to fetch.
2893 	 */
2894 	wmb();
2895 	writel(ring->next_to_use, ring->tail);
2896 }
2897 
2898 static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter)
2899 {
2900 	unsigned int r_idx = smp_processor_id();
2901 
2902 	if (r_idx >= adapter->num_tx_queues)
2903 		r_idx = r_idx % adapter->num_tx_queues;
2904 
2905 	return adapter->tx_ring[r_idx];
2906 }
2907 
2908 static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp)
2909 {
2910 	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2911 	int cpu = smp_processor_id();
2912 	struct igb_ring *tx_ring;
2913 	struct netdev_queue *nq;
2914 	u32 ret;
2915 
2916 	if (unlikely(!xdpf))
2917 		return IGB_XDP_CONSUMED;
2918 
2919 	/* During program transitions its possible adapter->xdp_prog is assigned
2920 	 * but ring has not been configured yet. In this case simply abort xmit.
2921 	 */
2922 	tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
2923 	if (unlikely(!tx_ring))
2924 		return IGB_XDP_CONSUMED;
2925 
2926 	nq = txring_txq(tx_ring);
2927 	__netif_tx_lock(nq, cpu);
2928 	/* Avoid transmit queue timeout since we share it with the slow path */
2929 	txq_trans_cond_update(nq);
2930 	ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
2931 	__netif_tx_unlock(nq);
2932 
2933 	return ret;
2934 }
2935 
2936 static int igb_xdp_xmit(struct net_device *dev, int n,
2937 			struct xdp_frame **frames, u32 flags)
2938 {
2939 	struct igb_adapter *adapter = netdev_priv(dev);
2940 	int cpu = smp_processor_id();
2941 	struct igb_ring *tx_ring;
2942 	struct netdev_queue *nq;
2943 	int nxmit = 0;
2944 	int i;
2945 
2946 	if (unlikely(test_bit(__IGB_DOWN, &adapter->state)))
2947 		return -ENETDOWN;
2948 
2949 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2950 		return -EINVAL;
2951 
2952 	/* During program transitions its possible adapter->xdp_prog is assigned
2953 	 * but ring has not been configured yet. In this case simply abort xmit.
2954 	 */
2955 	tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
2956 	if (unlikely(!tx_ring))
2957 		return -ENXIO;
2958 
2959 	nq = txring_txq(tx_ring);
2960 	__netif_tx_lock(nq, cpu);
2961 
2962 	/* Avoid transmit queue timeout since we share it with the slow path */
2963 	txq_trans_cond_update(nq);
2964 
2965 	for (i = 0; i < n; i++) {
2966 		struct xdp_frame *xdpf = frames[i];
2967 		int err;
2968 
2969 		err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
2970 		if (err != IGB_XDP_TX)
2971 			break;
2972 		nxmit++;
2973 	}
2974 
2975 	__netif_tx_unlock(nq);
2976 
2977 	if (unlikely(flags & XDP_XMIT_FLUSH))
2978 		igb_xdp_ring_update_tail(tx_ring);
2979 
2980 	return nxmit;
2981 }
2982 
2983 static const struct net_device_ops igb_netdev_ops = {
2984 	.ndo_open		= igb_open,
2985 	.ndo_stop		= igb_close,
2986 	.ndo_start_xmit		= igb_xmit_frame,
2987 	.ndo_get_stats64	= igb_get_stats64,
2988 	.ndo_set_rx_mode	= igb_set_rx_mode,
2989 	.ndo_set_mac_address	= igb_set_mac,
2990 	.ndo_change_mtu		= igb_change_mtu,
2991 	.ndo_eth_ioctl		= igb_ioctl,
2992 	.ndo_tx_timeout		= igb_tx_timeout,
2993 	.ndo_validate_addr	= eth_validate_addr,
2994 	.ndo_vlan_rx_add_vid	= igb_vlan_rx_add_vid,
2995 	.ndo_vlan_rx_kill_vid	= igb_vlan_rx_kill_vid,
2996 	.ndo_set_vf_mac		= igb_ndo_set_vf_mac,
2997 	.ndo_set_vf_vlan	= igb_ndo_set_vf_vlan,
2998 	.ndo_set_vf_rate	= igb_ndo_set_vf_bw,
2999 	.ndo_set_vf_spoofchk	= igb_ndo_set_vf_spoofchk,
3000 	.ndo_set_vf_trust	= igb_ndo_set_vf_trust,
3001 	.ndo_get_vf_config	= igb_ndo_get_vf_config,
3002 	.ndo_fix_features	= igb_fix_features,
3003 	.ndo_set_features	= igb_set_features,
3004 	.ndo_fdb_add		= igb_ndo_fdb_add,
3005 	.ndo_features_check	= igb_features_check,
3006 	.ndo_setup_tc		= igb_setup_tc,
3007 	.ndo_bpf		= igb_xdp,
3008 	.ndo_xdp_xmit		= igb_xdp_xmit,
3009 };
3010 
3011 /**
3012  * igb_set_fw_version - Configure version string for ethtool
3013  * @adapter: adapter struct
3014  **/
3015 void igb_set_fw_version(struct igb_adapter *adapter)
3016 {
3017 	struct e1000_hw *hw = &adapter->hw;
3018 	struct e1000_fw_version fw;
3019 
3020 	igb_get_fw_version(hw, &fw);
3021 
3022 	switch (hw->mac.type) {
3023 	case e1000_i210:
3024 	case e1000_i211:
3025 		if (!(igb_get_flash_presence_i210(hw))) {
3026 			snprintf(adapter->fw_version,
3027 				 sizeof(adapter->fw_version),
3028 				 "%2d.%2d-%d",
3029 				 fw.invm_major, fw.invm_minor,
3030 				 fw.invm_img_type);
3031 			break;
3032 		}
3033 		fallthrough;
3034 	default:
3035 		/* if option is rom valid, display its version too */
3036 		if (fw.or_valid) {
3037 			snprintf(adapter->fw_version,
3038 				 sizeof(adapter->fw_version),
3039 				 "%d.%d, 0x%08x, %d.%d.%d",
3040 				 fw.eep_major, fw.eep_minor, fw.etrack_id,
3041 				 fw.or_major, fw.or_build, fw.or_patch);
3042 		/* no option rom */
3043 		} else if (fw.etrack_id != 0X0000) {
3044 			snprintf(adapter->fw_version,
3045 			    sizeof(adapter->fw_version),
3046 			    "%d.%d, 0x%08x",
3047 			    fw.eep_major, fw.eep_minor, fw.etrack_id);
3048 		} else {
3049 		snprintf(adapter->fw_version,
3050 		    sizeof(adapter->fw_version),
3051 		    "%d.%d.%d",
3052 		    fw.eep_major, fw.eep_minor, fw.eep_build);
3053 		}
3054 		break;
3055 	}
3056 }
3057 
3058 /**
3059  * igb_init_mas - init Media Autosense feature if enabled in the NVM
3060  *
3061  * @adapter: adapter struct
3062  **/
3063 static void igb_init_mas(struct igb_adapter *adapter)
3064 {
3065 	struct e1000_hw *hw = &adapter->hw;
3066 	u16 eeprom_data;
3067 
3068 	hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
3069 	switch (hw->bus.func) {
3070 	case E1000_FUNC_0:
3071 		if (eeprom_data & IGB_MAS_ENABLE_0) {
3072 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3073 			netdev_info(adapter->netdev,
3074 				"MAS: Enabling Media Autosense for port %d\n",
3075 				hw->bus.func);
3076 		}
3077 		break;
3078 	case E1000_FUNC_1:
3079 		if (eeprom_data & IGB_MAS_ENABLE_1) {
3080 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3081 			netdev_info(adapter->netdev,
3082 				"MAS: Enabling Media Autosense for port %d\n",
3083 				hw->bus.func);
3084 		}
3085 		break;
3086 	case E1000_FUNC_2:
3087 		if (eeprom_data & IGB_MAS_ENABLE_2) {
3088 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3089 			netdev_info(adapter->netdev,
3090 				"MAS: Enabling Media Autosense for port %d\n",
3091 				hw->bus.func);
3092 		}
3093 		break;
3094 	case E1000_FUNC_3:
3095 		if (eeprom_data & IGB_MAS_ENABLE_3) {
3096 			adapter->flags |= IGB_FLAG_MAS_ENABLE;
3097 			netdev_info(adapter->netdev,
3098 				"MAS: Enabling Media Autosense for port %d\n",
3099 				hw->bus.func);
3100 		}
3101 		break;
3102 	default:
3103 		/* Shouldn't get here */
3104 		netdev_err(adapter->netdev,
3105 			"MAS: Invalid port configuration, returning\n");
3106 		break;
3107 	}
3108 }
3109 
3110 /**
3111  *  igb_init_i2c - Init I2C interface
3112  *  @adapter: pointer to adapter structure
3113  **/
3114 static s32 igb_init_i2c(struct igb_adapter *adapter)
3115 {
3116 	struct e1000_hw *hw = &adapter->hw;
3117 	s32 status = 0;
3118 	s32 i2cctl;
3119 
3120 	/* I2C interface supported on i350 devices */
3121 	if (adapter->hw.mac.type != e1000_i350)
3122 		return 0;
3123 
3124 	i2cctl = rd32(E1000_I2CPARAMS);
3125 	i2cctl |= E1000_I2CBB_EN
3126 		| E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N
3127 		| E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N;
3128 	wr32(E1000_I2CPARAMS, i2cctl);
3129 	wrfl();
3130 
3131 	/* Initialize the i2c bus which is controlled by the registers.
3132 	 * This bus will use the i2c_algo_bit structure that implements
3133 	 * the protocol through toggling of the 4 bits in the register.
3134 	 */
3135 	adapter->i2c_adap.owner = THIS_MODULE;
3136 	adapter->i2c_algo = igb_i2c_algo;
3137 	adapter->i2c_algo.data = adapter;
3138 	adapter->i2c_adap.algo_data = &adapter->i2c_algo;
3139 	adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
3140 	strscpy(adapter->i2c_adap.name, "igb BB",
3141 		sizeof(adapter->i2c_adap.name));
3142 	status = i2c_bit_add_bus(&adapter->i2c_adap);
3143 	return status;
3144 }
3145 
3146 /**
3147  *  igb_probe - Device Initialization Routine
3148  *  @pdev: PCI device information struct
3149  *  @ent: entry in igb_pci_tbl
3150  *
3151  *  Returns 0 on success, negative on failure
3152  *
3153  *  igb_probe initializes an adapter identified by a pci_dev structure.
3154  *  The OS initialization, configuring of the adapter private structure,
3155  *  and a hardware reset occur.
3156  **/
3157 static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3158 {
3159 	struct net_device *netdev;
3160 	struct igb_adapter *adapter;
3161 	struct e1000_hw *hw;
3162 	u16 eeprom_data = 0;
3163 	s32 ret_val;
3164 	static int global_quad_port_a; /* global quad port a indication */
3165 	const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
3166 	u8 part_str[E1000_PBANUM_LENGTH];
3167 	int err;
3168 
3169 	/* Catch broken hardware that put the wrong VF device ID in
3170 	 * the PCIe SR-IOV capability.
3171 	 */
3172 	if (pdev->is_virtfn) {
3173 		WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n",
3174 			pci_name(pdev), pdev->vendor, pdev->device);
3175 		return -EINVAL;
3176 	}
3177 
3178 	err = pci_enable_device_mem(pdev);
3179 	if (err)
3180 		return err;
3181 
3182 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3183 	if (err) {
3184 		dev_err(&pdev->dev,
3185 			"No usable DMA configuration, aborting\n");
3186 		goto err_dma;
3187 	}
3188 
3189 	err = pci_request_mem_regions(pdev, igb_driver_name);
3190 	if (err)
3191 		goto err_pci_reg;
3192 
3193 	pci_enable_pcie_error_reporting(pdev);
3194 
3195 	pci_set_master(pdev);
3196 	pci_save_state(pdev);
3197 
3198 	err = -ENOMEM;
3199 	netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
3200 				   IGB_MAX_TX_QUEUES);
3201 	if (!netdev)
3202 		goto err_alloc_etherdev;
3203 
3204 	SET_NETDEV_DEV(netdev, &pdev->dev);
3205 
3206 	pci_set_drvdata(pdev, netdev);
3207 	adapter = netdev_priv(netdev);
3208 	adapter->netdev = netdev;
3209 	adapter->pdev = pdev;
3210 	hw = &adapter->hw;
3211 	hw->back = adapter;
3212 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3213 
3214 	err = -EIO;
3215 	adapter->io_addr = pci_iomap(pdev, 0, 0);
3216 	if (!adapter->io_addr)
3217 		goto err_ioremap;
3218 	/* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
3219 	hw->hw_addr = adapter->io_addr;
3220 
3221 	netdev->netdev_ops = &igb_netdev_ops;
3222 	igb_set_ethtool_ops(netdev);
3223 	netdev->watchdog_timeo = 5 * HZ;
3224 
3225 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
3226 
3227 	netdev->mem_start = pci_resource_start(pdev, 0);
3228 	netdev->mem_end = pci_resource_end(pdev, 0);
3229 
3230 	/* PCI config space info */
3231 	hw->vendor_id = pdev->vendor;
3232 	hw->device_id = pdev->device;
3233 	hw->revision_id = pdev->revision;
3234 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
3235 	hw->subsystem_device_id = pdev->subsystem_device;
3236 
3237 	/* Copy the default MAC, PHY and NVM function pointers */
3238 	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
3239 	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
3240 	memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
3241 	/* Initialize skew-specific constants */
3242 	err = ei->get_invariants(hw);
3243 	if (err)
3244 		goto err_sw_init;
3245 
3246 	/* setup the private structure */
3247 	err = igb_sw_init(adapter);
3248 	if (err)
3249 		goto err_sw_init;
3250 
3251 	igb_get_bus_info_pcie(hw);
3252 
3253 	hw->phy.autoneg_wait_to_complete = false;
3254 
3255 	/* Copper options */
3256 	if (hw->phy.media_type == e1000_media_type_copper) {
3257 		hw->phy.mdix = AUTO_ALL_MODES;
3258 		hw->phy.disable_polarity_correction = false;
3259 		hw->phy.ms_type = e1000_ms_hw_default;
3260 	}
3261 
3262 	if (igb_check_reset_block(hw))
3263 		dev_info(&pdev->dev,
3264 			"PHY reset is blocked due to SOL/IDER session.\n");
3265 
3266 	/* features is initialized to 0 in allocation, it might have bits
3267 	 * set by igb_sw_init so we should use an or instead of an
3268 	 * assignment.
3269 	 */
3270 	netdev->features |= NETIF_F_SG |
3271 			    NETIF_F_TSO |
3272 			    NETIF_F_TSO6 |
3273 			    NETIF_F_RXHASH |
3274 			    NETIF_F_RXCSUM |
3275 			    NETIF_F_HW_CSUM;
3276 
3277 	if (hw->mac.type >= e1000_82576)
3278 		netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
3279 
3280 	if (hw->mac.type >= e1000_i350)
3281 		netdev->features |= NETIF_F_HW_TC;
3282 
3283 #define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
3284 				  NETIF_F_GSO_GRE_CSUM | \
3285 				  NETIF_F_GSO_IPXIP4 | \
3286 				  NETIF_F_GSO_IPXIP6 | \
3287 				  NETIF_F_GSO_UDP_TUNNEL | \
3288 				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
3289 
3290 	netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
3291 	netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
3292 
3293 	/* copy netdev features into list of user selectable features */
3294 	netdev->hw_features |= netdev->features |
3295 			       NETIF_F_HW_VLAN_CTAG_RX |
3296 			       NETIF_F_HW_VLAN_CTAG_TX |
3297 			       NETIF_F_RXALL;
3298 
3299 	if (hw->mac.type >= e1000_i350)
3300 		netdev->hw_features |= NETIF_F_NTUPLE;
3301 
3302 	netdev->features |= NETIF_F_HIGHDMA;
3303 
3304 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
3305 	netdev->mpls_features |= NETIF_F_HW_CSUM;
3306 	netdev->hw_enc_features |= netdev->vlan_features;
3307 
3308 	/* set this bit last since it cannot be part of vlan_features */
3309 	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3310 			    NETIF_F_HW_VLAN_CTAG_RX |
3311 			    NETIF_F_HW_VLAN_CTAG_TX;
3312 
3313 	netdev->priv_flags |= IFF_SUPP_NOFCS;
3314 
3315 	netdev->priv_flags |= IFF_UNICAST_FLT;
3316 
3317 	/* MTU range: 68 - 9216 */
3318 	netdev->min_mtu = ETH_MIN_MTU;
3319 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
3320 
3321 	adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
3322 
3323 	/* before reading the NVM, reset the controller to put the device in a
3324 	 * known good starting state
3325 	 */
3326 	hw->mac.ops.reset_hw(hw);
3327 
3328 	/* make sure the NVM is good , i211/i210 parts can have special NVM
3329 	 * that doesn't contain a checksum
3330 	 */
3331 	switch (hw->mac.type) {
3332 	case e1000_i210:
3333 	case e1000_i211:
3334 		if (igb_get_flash_presence_i210(hw)) {
3335 			if (hw->nvm.ops.validate(hw) < 0) {
3336 				dev_err(&pdev->dev,
3337 					"The NVM Checksum Is Not Valid\n");
3338 				err = -EIO;
3339 				goto err_eeprom;
3340 			}
3341 		}
3342 		break;
3343 	default:
3344 		if (hw->nvm.ops.validate(hw) < 0) {
3345 			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
3346 			err = -EIO;
3347 			goto err_eeprom;
3348 		}
3349 		break;
3350 	}
3351 
3352 	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
3353 		/* copy the MAC address out of the NVM */
3354 		if (hw->mac.ops.read_mac_addr(hw))
3355 			dev_err(&pdev->dev, "NVM Read Error\n");
3356 	}
3357 
3358 	eth_hw_addr_set(netdev, hw->mac.addr);
3359 
3360 	if (!is_valid_ether_addr(netdev->dev_addr)) {
3361 		dev_err(&pdev->dev, "Invalid MAC Address\n");
3362 		err = -EIO;
3363 		goto err_eeprom;
3364 	}
3365 
3366 	igb_set_default_mac_filter(adapter);
3367 
3368 	/* get firmware version for ethtool -i */
3369 	igb_set_fw_version(adapter);
3370 
3371 	/* configure RXPBSIZE and TXPBSIZE */
3372 	if (hw->mac.type == e1000_i210) {
3373 		wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
3374 		wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
3375 	}
3376 
3377 	timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
3378 	timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
3379 
3380 	INIT_WORK(&adapter->reset_task, igb_reset_task);
3381 	INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
3382 
3383 	/* Initialize link properties that are user-changeable */
3384 	adapter->fc_autoneg = true;
3385 	hw->mac.autoneg = true;
3386 	hw->phy.autoneg_advertised = 0x2f;
3387 
3388 	hw->fc.requested_mode = e1000_fc_default;
3389 	hw->fc.current_mode = e1000_fc_default;
3390 
3391 	igb_validate_mdi_setting(hw);
3392 
3393 	/* By default, support wake on port A */
3394 	if (hw->bus.func == 0)
3395 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3396 
3397 	/* Check the NVM for wake support on non-port A ports */
3398 	if (hw->mac.type >= e1000_82580)
3399 		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
3400 				 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
3401 				 &eeprom_data);
3402 	else if (hw->bus.func == 1)
3403 		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3404 
3405 	if (eeprom_data & IGB_EEPROM_APME)
3406 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3407 
3408 	/* now that we have the eeprom settings, apply the special cases where
3409 	 * the eeprom may be wrong or the board simply won't support wake on
3410 	 * lan on a particular port
3411 	 */
3412 	switch (pdev->device) {
3413 	case E1000_DEV_ID_82575GB_QUAD_COPPER:
3414 		adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3415 		break;
3416 	case E1000_DEV_ID_82575EB_FIBER_SERDES:
3417 	case E1000_DEV_ID_82576_FIBER:
3418 	case E1000_DEV_ID_82576_SERDES:
3419 		/* Wake events only supported on port A for dual fiber
3420 		 * regardless of eeprom setting
3421 		 */
3422 		if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
3423 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3424 		break;
3425 	case E1000_DEV_ID_82576_QUAD_COPPER:
3426 	case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
3427 		/* if quad port adapter, disable WoL on all but port A */
3428 		if (global_quad_port_a != 0)
3429 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3430 		else
3431 			adapter->flags |= IGB_FLAG_QUAD_PORT_A;
3432 		/* Reset for multiple quad port adapters */
3433 		if (++global_quad_port_a == 4)
3434 			global_quad_port_a = 0;
3435 		break;
3436 	default:
3437 		/* If the device can't wake, don't set software support */
3438 		if (!device_can_wakeup(&adapter->pdev->dev))
3439 			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3440 	}
3441 
3442 	/* initialize the wol settings based on the eeprom settings */
3443 	if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
3444 		adapter->wol |= E1000_WUFC_MAG;
3445 
3446 	/* Some vendors want WoL disabled by default, but still supported */
3447 	if ((hw->mac.type == e1000_i350) &&
3448 	    (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
3449 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3450 		adapter->wol = 0;
3451 	}
3452 
3453 	/* Some vendors want the ability to Use the EEPROM setting as
3454 	 * enable/disable only, and not for capability
3455 	 */
3456 	if (((hw->mac.type == e1000_i350) ||
3457 	     (hw->mac.type == e1000_i354)) &&
3458 	    (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
3459 		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3460 		adapter->wol = 0;
3461 	}
3462 	if (hw->mac.type == e1000_i350) {
3463 		if (((pdev->subsystem_device == 0x5001) ||
3464 		     (pdev->subsystem_device == 0x5002)) &&
3465 				(hw->bus.func == 0)) {
3466 			adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3467 			adapter->wol = 0;
3468 		}
3469 		if (pdev->subsystem_device == 0x1F52)
3470 			adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3471 	}
3472 
3473 	device_set_wakeup_enable(&adapter->pdev->dev,
3474 				 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
3475 
3476 	/* reset the hardware with the new settings */
3477 	igb_reset(adapter);
3478 
3479 	/* Init the I2C interface */
3480 	err = igb_init_i2c(adapter);
3481 	if (err) {
3482 		dev_err(&pdev->dev, "failed to init i2c interface\n");
3483 		goto err_eeprom;
3484 	}
3485 
3486 	/* let the f/w know that the h/w is now under the control of the
3487 	 * driver.
3488 	 */
3489 	igb_get_hw_control(adapter);
3490 
3491 	strcpy(netdev->name, "eth%d");
3492 	err = register_netdev(netdev);
3493 	if (err)
3494 		goto err_register;
3495 
3496 	/* carrier off reporting is important to ethtool even BEFORE open */
3497 	netif_carrier_off(netdev);
3498 
3499 #ifdef CONFIG_IGB_DCA
3500 	if (dca_add_requester(&pdev->dev) == 0) {
3501 		adapter->flags |= IGB_FLAG_DCA_ENABLED;
3502 		dev_info(&pdev->dev, "DCA enabled\n");
3503 		igb_setup_dca(adapter);
3504 	}
3505 
3506 #endif
3507 #ifdef CONFIG_IGB_HWMON
3508 	/* Initialize the thermal sensor on i350 devices. */
3509 	if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
3510 		u16 ets_word;
3511 
3512 		/* Read the NVM to determine if this i350 device supports an
3513 		 * external thermal sensor.
3514 		 */
3515 		hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
3516 		if (ets_word != 0x0000 && ets_word != 0xFFFF)
3517 			adapter->ets = true;
3518 		else
3519 			adapter->ets = false;
3520 		if (igb_sysfs_init(adapter))
3521 			dev_err(&pdev->dev,
3522 				"failed to allocate sysfs resources\n");
3523 	} else {
3524 		adapter->ets = false;
3525 	}
3526 #endif
3527 	/* Check if Media Autosense is enabled */
3528 	adapter->ei = *ei;
3529 	if (hw->dev_spec._82575.mas_capable)
3530 		igb_init_mas(adapter);
3531 
3532 	/* do hw tstamp init after resetting */
3533 	igb_ptp_init(adapter);
3534 
3535 	dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
3536 	/* print bus type/speed/width info, not applicable to i354 */
3537 	if (hw->mac.type != e1000_i354) {
3538 		dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
3539 			 netdev->name,
3540 			 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
3541 			  (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
3542 			   "unknown"),
3543 			 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
3544 			  "Width x4" :
3545 			  (hw->bus.width == e1000_bus_width_pcie_x2) ?
3546 			  "Width x2" :
3547 			  (hw->bus.width == e1000_bus_width_pcie_x1) ?
3548 			  "Width x1" : "unknown"), netdev->dev_addr);
3549 	}
3550 
3551 	if ((hw->mac.type == e1000_82576 &&
3552 	     rd32(E1000_EECD) & E1000_EECD_PRES) ||
3553 	    (hw->mac.type >= e1000_i210 ||
3554 	     igb_get_flash_presence_i210(hw))) {
3555 		ret_val = igb_read_part_string(hw, part_str,
3556 					       E1000_PBANUM_LENGTH);
3557 	} else {
3558 		ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
3559 	}
3560 
3561 	if (ret_val)
3562 		strcpy(part_str, "Unknown");
3563 	dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
3564 	dev_info(&pdev->dev,
3565 		"Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3566 		(adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
3567 		(adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3568 		adapter->num_rx_queues, adapter->num_tx_queues);
3569 	if (hw->phy.media_type == e1000_media_type_copper) {
3570 		switch (hw->mac.type) {
3571 		case e1000_i350:
3572 		case e1000_i210:
3573 		case e1000_i211:
3574 			/* Enable EEE for internal copper PHY devices */
3575 			err = igb_set_eee_i350(hw, true, true);
3576 			if ((!err) &&
3577 			    (!hw->dev_spec._82575.eee_disable)) {
3578 				adapter->eee_advert =
3579 					MDIO_EEE_100TX | MDIO_EEE_1000T;
3580 				adapter->flags |= IGB_FLAG_EEE;
3581 			}
3582 			break;
3583 		case e1000_i354:
3584 			if ((rd32(E1000_CTRL_EXT) &
3585 			    E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3586 				err = igb_set_eee_i354(hw, true, true);
3587 				if ((!err) &&
3588 					(!hw->dev_spec._82575.eee_disable)) {
3589 					adapter->eee_advert =
3590 					   MDIO_EEE_100TX | MDIO_EEE_1000T;
3591 					adapter->flags |= IGB_FLAG_EEE;
3592 				}
3593 			}
3594 			break;
3595 		default:
3596 			break;
3597 		}
3598 	}
3599 
3600 	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
3601 
3602 	pm_runtime_put_noidle(&pdev->dev);
3603 	return 0;
3604 
3605 err_register:
3606 	igb_release_hw_control(adapter);
3607 	memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3608 err_eeprom:
3609 	if (!igb_check_reset_block(hw))
3610 		igb_reset_phy(hw);
3611 
3612 	if (hw->flash_address)
3613 		iounmap(hw->flash_address);
3614 err_sw_init:
3615 	kfree(adapter->mac_table);
3616 	kfree(adapter->shadow_vfta);
3617 	igb_clear_interrupt_scheme(adapter);
3618 #ifdef CONFIG_PCI_IOV
3619 	igb_disable_sriov(pdev);
3620 #endif
3621 	pci_iounmap(pdev, adapter->io_addr);
3622 err_ioremap:
3623 	free_netdev(netdev);
3624 err_alloc_etherdev:
3625 	pci_disable_pcie_error_reporting(pdev);
3626 	pci_release_mem_regions(pdev);
3627 err_pci_reg:
3628 err_dma:
3629 	pci_disable_device(pdev);
3630 	return err;
3631 }
3632 
3633 #ifdef CONFIG_PCI_IOV
3634 static int igb_disable_sriov(struct pci_dev *pdev)
3635 {
3636 	struct net_device *netdev = pci_get_drvdata(pdev);
3637 	struct igb_adapter *adapter = netdev_priv(netdev);
3638 	struct e1000_hw *hw = &adapter->hw;
3639 	unsigned long flags;
3640 
3641 	/* reclaim resources allocated to VFs */
3642 	if (adapter->vf_data) {
3643 		/* disable iov and allow time for transactions to clear */
3644 		if (pci_vfs_assigned(pdev)) {
3645 			dev_warn(&pdev->dev,
3646 				 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3647 			return -EPERM;
3648 		} else {
3649 			pci_disable_sriov(pdev);
3650 			msleep(500);
3651 		}
3652 		spin_lock_irqsave(&adapter->vfs_lock, flags);
3653 		kfree(adapter->vf_mac_list);
3654 		adapter->vf_mac_list = NULL;
3655 		kfree(adapter->vf_data);
3656 		adapter->vf_data = NULL;
3657 		adapter->vfs_allocated_count = 0;
3658 		spin_unlock_irqrestore(&adapter->vfs_lock, flags);
3659 		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
3660 		wrfl();
3661 		msleep(100);
3662 		dev_info(&pdev->dev, "IOV Disabled\n");
3663 
3664 		/* Re-enable DMA Coalescing flag since IOV is turned off */
3665 		adapter->flags |= IGB_FLAG_DMAC;
3666 	}
3667 
3668 	return 0;
3669 }
3670 
3671 static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
3672 {
3673 	struct net_device *netdev = pci_get_drvdata(pdev);
3674 	struct igb_adapter *adapter = netdev_priv(netdev);
3675 	int old_vfs = pci_num_vf(pdev);
3676 	struct vf_mac_filter *mac_list;
3677 	int err = 0;
3678 	int num_vf_mac_filters, i;
3679 
3680 	if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
3681 		err = -EPERM;
3682 		goto out;
3683 	}
3684 	if (!num_vfs)
3685 		goto out;
3686 
3687 	if (old_vfs) {
3688 		dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3689 			 old_vfs, max_vfs);
3690 		adapter->vfs_allocated_count = old_vfs;
3691 	} else
3692 		adapter->vfs_allocated_count = num_vfs;
3693 
3694 	adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
3695 				sizeof(struct vf_data_storage), GFP_KERNEL);
3696 
3697 	/* if allocation failed then we do not support SR-IOV */
3698 	if (!adapter->vf_data) {
3699 		adapter->vfs_allocated_count = 0;
3700 		err = -ENOMEM;
3701 		goto out;
3702 	}
3703 
3704 	/* Due to the limited number of RAR entries calculate potential
3705 	 * number of MAC filters available for the VFs. Reserve entries
3706 	 * for PF default MAC, PF MAC filters and at least one RAR entry
3707 	 * for each VF for VF MAC.
3708 	 */
3709 	num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
3710 			     (1 + IGB_PF_MAC_FILTERS_RESERVED +
3711 			      adapter->vfs_allocated_count);
3712 
3713 	adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
3714 				       sizeof(struct vf_mac_filter),
3715 				       GFP_KERNEL);
3716 
3717 	mac_list = adapter->vf_mac_list;
3718 	INIT_LIST_HEAD(&adapter->vf_macs.l);
3719 
3720 	if (adapter->vf_mac_list) {
3721 		/* Initialize list of VF MAC filters */
3722 		for (i = 0; i < num_vf_mac_filters; i++) {
3723 			mac_list->vf = -1;
3724 			mac_list->free = true;
3725 			list_add(&mac_list->l, &adapter->vf_macs.l);
3726 			mac_list++;
3727 		}
3728 	} else {
3729 		/* If we could not allocate memory for the VF MAC filters
3730 		 * we can continue without this feature but warn user.
3731 		 */
3732 		dev_err(&pdev->dev,
3733 			"Unable to allocate memory for VF MAC filter list\n");
3734 	}
3735 
3736 	/* only call pci_enable_sriov() if no VFs are allocated already */
3737 	if (!old_vfs) {
3738 		err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
3739 		if (err)
3740 			goto err_out;
3741 	}
3742 	dev_info(&pdev->dev, "%d VFs allocated\n",
3743 		 adapter->vfs_allocated_count);
3744 	for (i = 0; i < adapter->vfs_allocated_count; i++)
3745 		igb_vf_configure(adapter, i);
3746 
3747 	/* DMA Coalescing is not supported in IOV mode. */
3748 	adapter->flags &= ~IGB_FLAG_DMAC;
3749 	goto out;
3750 
3751 err_out:
3752 	kfree(adapter->vf_mac_list);
3753 	adapter->vf_mac_list = NULL;
3754 	kfree(adapter->vf_data);
3755 	adapter->vf_data = NULL;
3756 	adapter->vfs_allocated_count = 0;
3757 out:
3758 	return err;
3759 }
3760 
3761 #endif
3762 /**
3763  *  igb_remove_i2c - Cleanup  I2C interface
3764  *  @adapter: pointer to adapter structure
3765  **/
3766 static void igb_remove_i2c(struct igb_adapter *adapter)
3767 {
3768 	/* free the adapter bus structure */
3769 	i2c_del_adapter(&adapter->i2c_adap);
3770 }
3771 
3772 /**
3773  *  igb_remove - Device Removal Routine
3774  *  @pdev: PCI device information struct
3775  *
3776  *  igb_remove is called by the PCI subsystem to alert the driver
3777  *  that it should release a PCI device.  The could be caused by a
3778  *  Hot-Plug event, or because the driver is going to be removed from
3779  *  memory.
3780  **/
3781 static void igb_remove(struct pci_dev *pdev)
3782 {
3783 	struct net_device *netdev = pci_get_drvdata(pdev);
3784 	struct igb_adapter *adapter = netdev_priv(netdev);
3785 	struct e1000_hw *hw = &adapter->hw;
3786 
3787 	pm_runtime_get_noresume(&pdev->dev);
3788 #ifdef CONFIG_IGB_HWMON
3789 	igb_sysfs_exit(adapter);
3790 #endif
3791 	igb_remove_i2c(adapter);
3792 	igb_ptp_stop(adapter);
3793 	/* The watchdog timer may be rescheduled, so explicitly
3794 	 * disable watchdog from being rescheduled.
3795 	 */
3796 	set_bit(__IGB_DOWN, &adapter->state);
3797 	del_timer_sync(&adapter->watchdog_timer);
3798 	del_timer_sync(&adapter->phy_info_timer);
3799 
3800 	cancel_work_sync(&adapter->reset_task);
3801 	cancel_work_sync(&adapter->watchdog_task);
3802 
3803 #ifdef CONFIG_IGB_DCA
3804 	if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3805 		dev_info(&pdev->dev, "DCA disabled\n");
3806 		dca_remove_requester(&pdev->dev);
3807 		adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3808 		wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
3809 	}
3810 #endif
3811 
3812 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
3813 	 * would have already happened in close and is redundant.
3814 	 */
3815 	igb_release_hw_control(adapter);
3816 
3817 #ifdef CONFIG_PCI_IOV
3818 	rtnl_lock();
3819 	igb_disable_sriov(pdev);
3820 	rtnl_unlock();
3821 #endif
3822 
3823 	unregister_netdev(netdev);
3824 
3825 	igb_clear_interrupt_scheme(adapter);
3826 
3827 	pci_iounmap(pdev, adapter->io_addr);
3828 	if (hw->flash_address)
3829 		iounmap(hw->flash_address);
3830 	pci_release_mem_regions(pdev);
3831 
3832 	kfree(adapter->mac_table);
3833 	kfree(adapter->shadow_vfta);
3834 	free_netdev(netdev);
3835 
3836 	pci_disable_pcie_error_reporting(pdev);
3837 
3838 	pci_disable_device(pdev);
3839 }
3840 
3841 /**
3842  *  igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3843  *  @adapter: board private structure to initialize
3844  *
3845  *  This function initializes the vf specific data storage and then attempts to
3846  *  allocate the VFs.  The reason for ordering it this way is because it is much
3847  *  mor expensive time wise to disable SR-IOV than it is to allocate and free
3848  *  the memory for the VFs.
3849  **/
3850 static void igb_probe_vfs(struct igb_adapter *adapter)
3851 {
3852 #ifdef CONFIG_PCI_IOV
3853 	struct pci_dev *pdev = adapter->pdev;
3854 	struct e1000_hw *hw = &adapter->hw;
3855 
3856 	/* Virtualization features not supported on i210 family. */
3857 	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
3858 		return;
3859 
3860 	/* Of the below we really only want the effect of getting
3861 	 * IGB_FLAG_HAS_MSIX set (if available), without which
3862 	 * igb_enable_sriov() has no effect.
3863 	 */
3864 	igb_set_interrupt_capability(adapter, true);
3865 	igb_reset_interrupt_capability(adapter);
3866 
3867 	pci_sriov_set_totalvfs(pdev, 7);
3868 	igb_enable_sriov(pdev, max_vfs);
3869 
3870 #endif /* CONFIG_PCI_IOV */
3871 }
3872 
3873 unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
3874 {
3875 	struct e1000_hw *hw = &adapter->hw;
3876 	unsigned int max_rss_queues;
3877 
3878 	/* Determine the maximum number of RSS queues supported. */
3879 	switch (hw->mac.type) {
3880 	case e1000_i211:
3881 		max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3882 		break;
3883 	case e1000_82575:
3884 	case e1000_i210:
3885 		max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3886 		break;
3887 	case e1000_i350:
3888 		/* I350 cannot do RSS and SR-IOV at the same time */
3889 		if (!!adapter->vfs_allocated_count) {
3890 			max_rss_queues = 1;
3891 			break;
3892 		}
3893 		fallthrough;
3894 	case e1000_82576:
3895 		if (!!adapter->vfs_allocated_count) {
3896 			max_rss_queues = 2;
3897 			break;
3898 		}
3899 		fallthrough;
3900 	case e1000_82580:
3901 	case e1000_i354:
3902 	default:
3903 		max_rss_queues = IGB_MAX_RX_QUEUES;
3904 		break;
3905 	}
3906 
3907 	return max_rss_queues;
3908 }
3909 
3910 static void igb_init_queue_configuration(struct igb_adapter *adapter)
3911 {
3912 	u32 max_rss_queues;
3913 
3914 	max_rss_queues = igb_get_max_rss_queues(adapter);
3915 	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
3916 
3917 	igb_set_flag_queue_pairs(adapter, max_rss_queues);
3918 }
3919 
3920 void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
3921 			      const u32 max_rss_queues)
3922 {
3923 	struct e1000_hw *hw = &adapter->hw;
3924 
3925 	/* Determine if we need to pair queues. */
3926 	switch (hw->mac.type) {
3927 	case e1000_82575:
3928 	case e1000_i211:
3929 		/* Device supports enough interrupts without queue pairing. */
3930 		break;
3931 	case e1000_82576:
3932 	case e1000_82580:
3933 	case e1000_i350:
3934 	case e1000_i354:
3935 	case e1000_i210:
3936 	default:
3937 		/* If rss_queues > half of max_rss_queues, pair the queues in
3938 		 * order to conserve interrupts due to limited supply.
3939 		 */
3940 		if (adapter->rss_queues > (max_rss_queues / 2))
3941 			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
3942 		else
3943 			adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
3944 		break;
3945 	}
3946 }
3947 
3948 /**
3949  *  igb_sw_init - Initialize general software structures (struct igb_adapter)
3950  *  @adapter: board private structure to initialize
3951  *
3952  *  igb_sw_init initializes the Adapter private data structure.
3953  *  Fields are initialized based on PCI device information and
3954  *  OS network device settings (MTU size).
3955  **/
3956 static int igb_sw_init(struct igb_adapter *adapter)
3957 {
3958 	struct e1000_hw *hw = &adapter->hw;
3959 	struct net_device *netdev = adapter->netdev;
3960 	struct pci_dev *pdev = adapter->pdev;
3961 
3962 	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
3963 
3964 	/* set default ring sizes */
3965 	adapter->tx_ring_count = IGB_DEFAULT_TXD;
3966 	adapter->rx_ring_count = IGB_DEFAULT_RXD;
3967 
3968 	/* set default ITR values */
3969 	adapter->rx_itr_setting = IGB_DEFAULT_ITR;
3970 	adapter->tx_itr_setting = IGB_DEFAULT_ITR;
3971 
3972 	/* set default work limits */
3973 	adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
3974 
3975 	adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD;
3976 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3977 
3978 	spin_lock_init(&adapter->nfc_lock);
3979 	spin_lock_init(&adapter->stats64_lock);
3980 
3981 	/* init spinlock to avoid concurrency of VF resources */
3982 	spin_lock_init(&adapter->vfs_lock);
3983 #ifdef CONFIG_PCI_IOV
3984 	switch (hw->mac.type) {
3985 	case e1000_82576:
3986 	case e1000_i350:
3987 		if (max_vfs > 7) {
3988 			dev_warn(&pdev->dev,
3989 				 "Maximum of 7 VFs per PF, using max\n");
3990 			max_vfs = adapter->vfs_allocated_count = 7;
3991 		} else
3992 			adapter->vfs_allocated_count = max_vfs;
3993 		if (adapter->vfs_allocated_count)
3994 			dev_warn(&pdev->dev,
3995 				 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
3996 		break;
3997 	default:
3998 		break;
3999 	}
4000 #endif /* CONFIG_PCI_IOV */
4001 
4002 	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4003 	adapter->flags |= IGB_FLAG_HAS_MSIX;
4004 
4005 	adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
4006 				     sizeof(struct igb_mac_addr),
4007 				     GFP_KERNEL);
4008 	if (!adapter->mac_table)
4009 		return -ENOMEM;
4010 
4011 	igb_probe_vfs(adapter);
4012 
4013 	igb_init_queue_configuration(adapter);
4014 
4015 	/* Setup and initialize a copy of the hw vlan table array */
4016 	adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
4017 				       GFP_KERNEL);
4018 	if (!adapter->shadow_vfta)
4019 		return -ENOMEM;
4020 
4021 	/* This call may decrease the number of queues */
4022 	if (igb_init_interrupt_scheme(adapter, true)) {
4023 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4024 		return -ENOMEM;
4025 	}
4026 
4027 	/* Explicitly disable IRQ since the NIC can be in any state. */
4028 	igb_irq_disable(adapter);
4029 
4030 	if (hw->mac.type >= e1000_i350)
4031 		adapter->flags &= ~IGB_FLAG_DMAC;
4032 
4033 	set_bit(__IGB_DOWN, &adapter->state);
4034 	return 0;
4035 }
4036 
4037 /**
4038  *  __igb_open - Called when a network interface is made active
4039  *  @netdev: network interface device structure
4040  *  @resuming: indicates whether we are in a resume call
4041  *
4042  *  Returns 0 on success, negative value on failure
4043  *
4044  *  The open entry point is called when a network interface is made
4045  *  active by the system (IFF_UP).  At this point all resources needed
4046  *  for transmit and receive operations are allocated, the interrupt
4047  *  handler is registered with the OS, the watchdog timer is started,
4048  *  and the stack is notified that the interface is ready.
4049  **/
4050 static int __igb_open(struct net_device *netdev, bool resuming)
4051 {
4052 	struct igb_adapter *adapter = netdev_priv(netdev);
4053 	struct e1000_hw *hw = &adapter->hw;
4054 	struct pci_dev *pdev = adapter->pdev;
4055 	int err;
4056 	int i;
4057 
4058 	/* disallow open during test */
4059 	if (test_bit(__IGB_TESTING, &adapter->state)) {
4060 		WARN_ON(resuming);
4061 		return -EBUSY;
4062 	}
4063 
4064 	if (!resuming)
4065 		pm_runtime_get_sync(&pdev->dev);
4066 
4067 	netif_carrier_off(netdev);
4068 
4069 	/* allocate transmit descriptors */
4070 	err = igb_setup_all_tx_resources(adapter);
4071 	if (err)
4072 		goto err_setup_tx;
4073 
4074 	/* allocate receive descriptors */
4075 	err = igb_setup_all_rx_resources(adapter);
4076 	if (err)
4077 		goto err_setup_rx;
4078 
4079 	igb_power_up_link(adapter);
4080 
4081 	/* before we allocate an interrupt, we must be ready to handle it.
4082 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4083 	 * as soon as we call pci_request_irq, so we have to setup our
4084 	 * clean_rx handler before we do so.
4085 	 */
4086 	igb_configure(adapter);
4087 
4088 	err = igb_request_irq(adapter);
4089 	if (err)
4090 		goto err_req_irq;
4091 
4092 	/* Notify the stack of the actual queue counts. */
4093 	err = netif_set_real_num_tx_queues(adapter->netdev,
4094 					   adapter->num_tx_queues);
4095 	if (err)
4096 		goto err_set_queues;
4097 
4098 	err = netif_set_real_num_rx_queues(adapter->netdev,
4099 					   adapter->num_rx_queues);
4100 	if (err)
4101 		goto err_set_queues;
4102 
4103 	/* From here on the code is the same as igb_up() */
4104 	clear_bit(__IGB_DOWN, &adapter->state);
4105 
4106 	for (i = 0; i < adapter->num_q_vectors; i++)
4107 		napi_enable(&(adapter->q_vector[i]->napi));
4108 
4109 	/* Clear any pending interrupts. */
4110 	rd32(E1000_TSICR);
4111 	rd32(E1000_ICR);
4112 
4113 	igb_irq_enable(adapter);
4114 
4115 	/* notify VFs that reset has been completed */
4116 	if (adapter->vfs_allocated_count) {
4117 		u32 reg_data = rd32(E1000_CTRL_EXT);
4118 
4119 		reg_data |= E1000_CTRL_EXT_PFRSTD;
4120 		wr32(E1000_CTRL_EXT, reg_data);
4121 	}
4122 
4123 	netif_tx_start_all_queues(netdev);
4124 
4125 	if (!resuming)
4126 		pm_runtime_put(&pdev->dev);
4127 
4128 	/* start the watchdog. */
4129 	hw->mac.get_link_status = 1;
4130 	schedule_work(&adapter->watchdog_task);
4131 
4132 	return 0;
4133 
4134 err_set_queues:
4135 	igb_free_irq(adapter);
4136 err_req_irq:
4137 	igb_release_hw_control(adapter);
4138 	igb_power_down_link(adapter);
4139 	igb_free_all_rx_resources(adapter);
4140 err_setup_rx:
4141 	igb_free_all_tx_resources(adapter);
4142 err_setup_tx:
4143 	igb_reset(adapter);
4144 	if (!resuming)
4145 		pm_runtime_put(&pdev->dev);
4146 
4147 	return err;
4148 }
4149 
4150 int igb_open(struct net_device *netdev)
4151 {
4152 	return __igb_open(netdev, false);
4153 }
4154 
4155 /**
4156  *  __igb_close - Disables a network interface
4157  *  @netdev: network interface device structure
4158  *  @suspending: indicates we are in a suspend call
4159  *
4160  *  Returns 0, this is not allowed to fail
4161  *
4162  *  The close entry point is called when an interface is de-activated
4163  *  by the OS.  The hardware is still under the driver's control, but
4164  *  needs to be disabled.  A global MAC reset is issued to stop the
4165  *  hardware, and all transmit and receive resources are freed.
4166  **/
4167 static int __igb_close(struct net_device *netdev, bool suspending)
4168 {
4169 	struct igb_adapter *adapter = netdev_priv(netdev);
4170 	struct pci_dev *pdev = adapter->pdev;
4171 
4172 	WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
4173 
4174 	if (!suspending)
4175 		pm_runtime_get_sync(&pdev->dev);
4176 
4177 	igb_down(adapter);
4178 	igb_free_irq(adapter);
4179 
4180 	igb_free_all_tx_resources(adapter);
4181 	igb_free_all_rx_resources(adapter);
4182 
4183 	if (!suspending)
4184 		pm_runtime_put_sync(&pdev->dev);
4185 	return 0;
4186 }
4187 
4188 int igb_close(struct net_device *netdev)
4189 {
4190 	if (netif_device_present(netdev) || netdev->dismantle)
4191 		return __igb_close(netdev, false);
4192 	return 0;
4193 }
4194 
4195 /**
4196  *  igb_setup_tx_resources - allocate Tx resources (Descriptors)
4197  *  @tx_ring: tx descriptor ring (for a specific queue) to setup
4198  *
4199  *  Return 0 on success, negative on failure
4200  **/
4201 int igb_setup_tx_resources(struct igb_ring *tx_ring)
4202 {
4203 	struct device *dev = tx_ring->dev;
4204 	int size;
4205 
4206 	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4207 
4208 	tx_ring->tx_buffer_info = vmalloc(size);
4209 	if (!tx_ring->tx_buffer_info)
4210 		goto err;
4211 
4212 	/* round up to nearest 4K */
4213 	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
4214 	tx_ring->size = ALIGN(tx_ring->size, 4096);
4215 
4216 	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4217 					   &tx_ring->dma, GFP_KERNEL);
4218 	if (!tx_ring->desc)
4219 		goto err;
4220 
4221 	tx_ring->next_to_use = 0;
4222 	tx_ring->next_to_clean = 0;
4223 
4224 	return 0;
4225 
4226 err:
4227 	vfree(tx_ring->tx_buffer_info);
4228 	tx_ring->tx_buffer_info = NULL;
4229 	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4230 	return -ENOMEM;
4231 }
4232 
4233 /**
4234  *  igb_setup_all_tx_resources - wrapper to allocate Tx resources
4235  *				 (Descriptors) for all queues
4236  *  @adapter: board private structure
4237  *
4238  *  Return 0 on success, negative on failure
4239  **/
4240 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
4241 {
4242 	struct pci_dev *pdev = adapter->pdev;
4243 	int i, err = 0;
4244 
4245 	for (i = 0; i < adapter->num_tx_queues; i++) {
4246 		err = igb_setup_tx_resources(adapter->tx_ring[i]);
4247 		if (err) {
4248 			dev_err(&pdev->dev,
4249 				"Allocation for Tx Queue %u failed\n", i);
4250 			for (i--; i >= 0; i--)
4251 				igb_free_tx_resources(adapter->tx_ring[i]);
4252 			break;
4253 		}
4254 	}
4255 
4256 	return err;
4257 }
4258 
4259 /**
4260  *  igb_setup_tctl - configure the transmit control registers
4261  *  @adapter: Board private structure
4262  **/
4263 void igb_setup_tctl(struct igb_adapter *adapter)
4264 {
4265 	struct e1000_hw *hw = &adapter->hw;
4266 	u32 tctl;
4267 
4268 	/* disable queue 0 which is enabled by default on 82575 and 82576 */
4269 	wr32(E1000_TXDCTL(0), 0);
4270 
4271 	/* Program the Transmit Control Register */
4272 	tctl = rd32(E1000_TCTL);
4273 	tctl &= ~E1000_TCTL_CT;
4274 	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
4275 		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
4276 
4277 	igb_config_collision_dist(hw);
4278 
4279 	/* Enable transmits */
4280 	tctl |= E1000_TCTL_EN;
4281 
4282 	wr32(E1000_TCTL, tctl);
4283 }
4284 
4285 /**
4286  *  igb_configure_tx_ring - Configure transmit ring after Reset
4287  *  @adapter: board private structure
4288  *  @ring: tx ring to configure
4289  *
4290  *  Configure a transmit ring after a reset.
4291  **/
4292 void igb_configure_tx_ring(struct igb_adapter *adapter,
4293 			   struct igb_ring *ring)
4294 {
4295 	struct e1000_hw *hw = &adapter->hw;
4296 	u32 txdctl = 0;
4297 	u64 tdba = ring->dma;
4298 	int reg_idx = ring->reg_idx;
4299 
4300 	wr32(E1000_TDLEN(reg_idx),
4301 	     ring->count * sizeof(union e1000_adv_tx_desc));
4302 	wr32(E1000_TDBAL(reg_idx),
4303 	     tdba & 0x00000000ffffffffULL);
4304 	wr32(E1000_TDBAH(reg_idx), tdba >> 32);
4305 
4306 	ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
4307 	wr32(E1000_TDH(reg_idx), 0);
4308 	writel(0, ring->tail);
4309 
4310 	txdctl |= IGB_TX_PTHRESH;
4311 	txdctl |= IGB_TX_HTHRESH << 8;
4312 	txdctl |= IGB_TX_WTHRESH << 16;
4313 
4314 	/* reinitialize tx_buffer_info */
4315 	memset(ring->tx_buffer_info, 0,
4316 	       sizeof(struct igb_tx_buffer) * ring->count);
4317 
4318 	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
4319 	wr32(E1000_TXDCTL(reg_idx), txdctl);
4320 }
4321 
4322 /**
4323  *  igb_configure_tx - Configure transmit Unit after Reset
4324  *  @adapter: board private structure
4325  *
4326  *  Configure the Tx unit of the MAC after a reset.
4327  **/
4328 static void igb_configure_tx(struct igb_adapter *adapter)
4329 {
4330 	struct e1000_hw *hw = &adapter->hw;
4331 	int i;
4332 
4333 	/* disable the queues */
4334 	for (i = 0; i < adapter->num_tx_queues; i++)
4335 		wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
4336 
4337 	wrfl();
4338 	usleep_range(10000, 20000);
4339 
4340 	for (i = 0; i < adapter->num_tx_queues; i++)
4341 		igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
4342 }
4343 
4344 /**
4345  *  igb_setup_rx_resources - allocate Rx resources (Descriptors)
4346  *  @rx_ring: Rx descriptor ring (for a specific queue) to setup
4347  *
4348  *  Returns 0 on success, negative on failure
4349  **/
4350 int igb_setup_rx_resources(struct igb_ring *rx_ring)
4351 {
4352 	struct igb_adapter *adapter = netdev_priv(rx_ring->netdev);
4353 	struct device *dev = rx_ring->dev;
4354 	int size, res;
4355 
4356 	/* XDP RX-queue info */
4357 	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
4358 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4359 	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
4360 			       rx_ring->queue_index, 0);
4361 	if (res < 0) {
4362 		dev_err(dev, "Failed to register xdp_rxq index %u\n",
4363 			rx_ring->queue_index);
4364 		return res;
4365 	}
4366 
4367 	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4368 
4369 	rx_ring->rx_buffer_info = vmalloc(size);
4370 	if (!rx_ring->rx_buffer_info)
4371 		goto err;
4372 
4373 	/* Round up to nearest 4K */
4374 	rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
4375 	rx_ring->size = ALIGN(rx_ring->size, 4096);
4376 
4377 	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4378 					   &rx_ring->dma, GFP_KERNEL);
4379 	if (!rx_ring->desc)
4380 		goto err;
4381 
4382 	rx_ring->next_to_alloc = 0;
4383 	rx_ring->next_to_clean = 0;
4384 	rx_ring->next_to_use = 0;
4385 
4386 	rx_ring->xdp_prog = adapter->xdp_prog;
4387 
4388 	return 0;
4389 
4390 err:
4391 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4392 	vfree(rx_ring->rx_buffer_info);
4393 	rx_ring->rx_buffer_info = NULL;
4394 	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4395 	return -ENOMEM;
4396 }
4397 
4398 /**
4399  *  igb_setup_all_rx_resources - wrapper to allocate Rx resources
4400  *				 (Descriptors) for all queues
4401  *  @adapter: board private structure
4402  *
4403  *  Return 0 on success, negative on failure
4404  **/
4405 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
4406 {
4407 	struct pci_dev *pdev = adapter->pdev;
4408 	int i, err = 0;
4409 
4410 	for (i = 0; i < adapter->num_rx_queues; i++) {
4411 		err = igb_setup_rx_resources(adapter->rx_ring[i]);
4412 		if (err) {
4413 			dev_err(&pdev->dev,
4414 				"Allocation for Rx Queue %u failed\n", i);
4415 			for (i--; i >= 0; i--)
4416 				igb_free_rx_resources(adapter->rx_ring[i]);
4417 			break;
4418 		}
4419 	}
4420 
4421 	return err;
4422 }
4423 
4424 /**
4425  *  igb_setup_mrqc - configure the multiple receive queue control registers
4426  *  @adapter: Board private structure
4427  **/
4428 static void igb_setup_mrqc(struct igb_adapter *adapter)
4429 {
4430 	struct e1000_hw *hw = &adapter->hw;
4431 	u32 mrqc, rxcsum;
4432 	u32 j, num_rx_queues;
4433 	u32 rss_key[10];
4434 
4435 	netdev_rss_key_fill(rss_key, sizeof(rss_key));
4436 	for (j = 0; j < 10; j++)
4437 		wr32(E1000_RSSRK(j), rss_key[j]);
4438 
4439 	num_rx_queues = adapter->rss_queues;
4440 
4441 	switch (hw->mac.type) {
4442 	case e1000_82576:
4443 		/* 82576 supports 2 RSS queues for SR-IOV */
4444 		if (adapter->vfs_allocated_count)
4445 			num_rx_queues = 2;
4446 		break;
4447 	default:
4448 		break;
4449 	}
4450 
4451 	if (adapter->rss_indir_tbl_init != num_rx_queues) {
4452 		for (j = 0; j < IGB_RETA_SIZE; j++)
4453 			adapter->rss_indir_tbl[j] =
4454 			(j * num_rx_queues) / IGB_RETA_SIZE;
4455 		adapter->rss_indir_tbl_init = num_rx_queues;
4456 	}
4457 	igb_write_rss_indir_tbl(adapter);
4458 
4459 	/* Disable raw packet checksumming so that RSS hash is placed in
4460 	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
4461 	 * offloads as they are enabled by default
4462 	 */
4463 	rxcsum = rd32(E1000_RXCSUM);
4464 	rxcsum |= E1000_RXCSUM_PCSD;
4465 
4466 	if (adapter->hw.mac.type >= e1000_82576)
4467 		/* Enable Receive Checksum Offload for SCTP */
4468 		rxcsum |= E1000_RXCSUM_CRCOFL;
4469 
4470 	/* Don't need to set TUOFL or IPOFL, they default to 1 */
4471 	wr32(E1000_RXCSUM, rxcsum);
4472 
4473 	/* Generate RSS hash based on packet types, TCP/UDP
4474 	 * port numbers and/or IPv4/v6 src and dst addresses
4475 	 */
4476 	mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
4477 	       E1000_MRQC_RSS_FIELD_IPV4_TCP |
4478 	       E1000_MRQC_RSS_FIELD_IPV6 |
4479 	       E1000_MRQC_RSS_FIELD_IPV6_TCP |
4480 	       E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
4481 
4482 	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
4483 		mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
4484 	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
4485 		mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
4486 
4487 	/* If VMDq is enabled then we set the appropriate mode for that, else
4488 	 * we default to RSS so that an RSS hash is calculated per packet even
4489 	 * if we are only using one queue
4490 	 */
4491 	if (adapter->vfs_allocated_count) {
4492 		if (hw->mac.type > e1000_82575) {
4493 			/* Set the default pool for the PF's first queue */
4494 			u32 vtctl = rd32(E1000_VT_CTL);
4495 
4496 			vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
4497 				   E1000_VT_CTL_DISABLE_DEF_POOL);
4498 			vtctl |= adapter->vfs_allocated_count <<
4499 				E1000_VT_CTL_DEFAULT_POOL_SHIFT;
4500 			wr32(E1000_VT_CTL, vtctl);
4501 		}
4502 		if (adapter->rss_queues > 1)
4503 			mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
4504 		else
4505 			mrqc |= E1000_MRQC_ENABLE_VMDQ;
4506 	} else {
4507 		mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
4508 	}
4509 	igb_vmm_control(adapter);
4510 
4511 	wr32(E1000_MRQC, mrqc);
4512 }
4513 
4514 /**
4515  *  igb_setup_rctl - configure the receive control registers
4516  *  @adapter: Board private structure
4517  **/
4518 void igb_setup_rctl(struct igb_adapter *adapter)
4519 {
4520 	struct e1000_hw *hw = &adapter->hw;
4521 	u32 rctl;
4522 
4523 	rctl = rd32(E1000_RCTL);
4524 
4525 	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4526 	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
4527 
4528 	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
4529 		(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4530 
4531 	/* enable stripping of CRC. It's unlikely this will break BMC
4532 	 * redirection as it did with e1000. Newer features require
4533 	 * that the HW strips the CRC.
4534 	 */
4535 	rctl |= E1000_RCTL_SECRC;
4536 
4537 	/* disable store bad packets and clear size bits. */
4538 	rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
4539 
4540 	/* enable LPE to allow for reception of jumbo frames */
4541 	rctl |= E1000_RCTL_LPE;
4542 
4543 	/* disable queue 0 to prevent tail write w/o re-config */
4544 	wr32(E1000_RXDCTL(0), 0);
4545 
4546 	/* Attention!!!  For SR-IOV PF driver operations you must enable
4547 	 * queue drop for all VF and PF queues to prevent head of line blocking
4548 	 * if an un-trusted VF does not provide descriptors to hardware.
4549 	 */
4550 	if (adapter->vfs_allocated_count) {
4551 		/* set all queue drop enable bits */
4552 		wr32(E1000_QDE, ALL_QUEUES);
4553 	}
4554 
4555 	/* This is useful for sniffing bad packets. */
4556 	if (adapter->netdev->features & NETIF_F_RXALL) {
4557 		/* UPE and MPE will be handled by normal PROMISC logic
4558 		 * in e1000e_set_rx_mode
4559 		 */
4560 		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
4561 			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
4562 			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
4563 
4564 		rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
4565 			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
4566 		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4567 		 * and that breaks VLANs.
4568 		 */
4569 	}
4570 
4571 	wr32(E1000_RCTL, rctl);
4572 }
4573 
4574 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
4575 				   int vfn)
4576 {
4577 	struct e1000_hw *hw = &adapter->hw;
4578 	u32 vmolr;
4579 
4580 	if (size > MAX_JUMBO_FRAME_SIZE)
4581 		size = MAX_JUMBO_FRAME_SIZE;
4582 
4583 	vmolr = rd32(E1000_VMOLR(vfn));
4584 	vmolr &= ~E1000_VMOLR_RLPML_MASK;
4585 	vmolr |= size | E1000_VMOLR_LPE;
4586 	wr32(E1000_VMOLR(vfn), vmolr);
4587 
4588 	return 0;
4589 }
4590 
4591 static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
4592 					 int vfn, bool enable)
4593 {
4594 	struct e1000_hw *hw = &adapter->hw;
4595 	u32 val, reg;
4596 
4597 	if (hw->mac.type < e1000_82576)
4598 		return;
4599 
4600 	if (hw->mac.type == e1000_i350)
4601 		reg = E1000_DVMOLR(vfn);
4602 	else
4603 		reg = E1000_VMOLR(vfn);
4604 
4605 	val = rd32(reg);
4606 	if (enable)
4607 		val |= E1000_VMOLR_STRVLAN;
4608 	else
4609 		val &= ~(E1000_VMOLR_STRVLAN);
4610 	wr32(reg, val);
4611 }
4612 
4613 static inline void igb_set_vmolr(struct igb_adapter *adapter,
4614 				 int vfn, bool aupe)
4615 {
4616 	struct e1000_hw *hw = &adapter->hw;
4617 	u32 vmolr;
4618 
4619 	/* This register exists only on 82576 and newer so if we are older then
4620 	 * we should exit and do nothing
4621 	 */
4622 	if (hw->mac.type < e1000_82576)
4623 		return;
4624 
4625 	vmolr = rd32(E1000_VMOLR(vfn));
4626 	if (aupe)
4627 		vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
4628 	else
4629 		vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
4630 
4631 	/* clear all bits that might not be set */
4632 	vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
4633 
4634 	if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
4635 		vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
4636 	/* for VMDq only allow the VFs and pool 0 to accept broadcast and
4637 	 * multicast packets
4638 	 */
4639 	if (vfn <= adapter->vfs_allocated_count)
4640 		vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
4641 
4642 	wr32(E1000_VMOLR(vfn), vmolr);
4643 }
4644 
4645 /**
4646  *  igb_setup_srrctl - configure the split and replication receive control
4647  *                     registers
4648  *  @adapter: Board private structure
4649  *  @ring: receive ring to be configured
4650  **/
4651 void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
4652 {
4653 	struct e1000_hw *hw = &adapter->hw;
4654 	int reg_idx = ring->reg_idx;
4655 	u32 srrctl = 0;
4656 
4657 	srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4658 	if (ring_uses_large_buffer(ring))
4659 		srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4660 	else
4661 		srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4662 	srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4663 	if (hw->mac.type >= e1000_82580)
4664 		srrctl |= E1000_SRRCTL_TIMESTAMP;
4665 	/* Only set Drop Enable if VFs allocated, or we are supporting multiple
4666 	 * queues and rx flow control is disabled
4667 	 */
4668 	if (adapter->vfs_allocated_count ||
4669 	    (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
4670 	     adapter->num_rx_queues > 1))
4671 		srrctl |= E1000_SRRCTL_DROP_EN;
4672 
4673 	wr32(E1000_SRRCTL(reg_idx), srrctl);
4674 }
4675 
4676 /**
4677  *  igb_configure_rx_ring - Configure a receive ring after Reset
4678  *  @adapter: board private structure
4679  *  @ring: receive ring to be configured
4680  *
4681  *  Configure the Rx unit of the MAC after a reset.
4682  **/
4683 void igb_configure_rx_ring(struct igb_adapter *adapter,
4684 			   struct igb_ring *ring)
4685 {
4686 	struct e1000_hw *hw = &adapter->hw;
4687 	union e1000_adv_rx_desc *rx_desc;
4688 	u64 rdba = ring->dma;
4689 	int reg_idx = ring->reg_idx;
4690 	u32 rxdctl = 0;
4691 
4692 	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
4693 	WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4694 					   MEM_TYPE_PAGE_SHARED, NULL));
4695 
4696 	/* disable the queue */
4697 	wr32(E1000_RXDCTL(reg_idx), 0);
4698 
4699 	/* Set DMA base address registers */
4700 	wr32(E1000_RDBAL(reg_idx),
4701 	     rdba & 0x00000000ffffffffULL);
4702 	wr32(E1000_RDBAH(reg_idx), rdba >> 32);
4703 	wr32(E1000_RDLEN(reg_idx),
4704 	     ring->count * sizeof(union e1000_adv_rx_desc));
4705 
4706 	/* initialize head and tail */
4707 	ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
4708 	wr32(E1000_RDH(reg_idx), 0);
4709 	writel(0, ring->tail);
4710 
4711 	/* set descriptor configuration */
4712 	igb_setup_srrctl(adapter, ring);
4713 
4714 	/* set filtering for VMDQ pools */
4715 	igb_set_vmolr(adapter, reg_idx & 0x7, true);
4716 
4717 	rxdctl |= IGB_RX_PTHRESH;
4718 	rxdctl |= IGB_RX_HTHRESH << 8;
4719 	rxdctl |= IGB_RX_WTHRESH << 16;
4720 
4721 	/* initialize rx_buffer_info */
4722 	memset(ring->rx_buffer_info, 0,
4723 	       sizeof(struct igb_rx_buffer) * ring->count);
4724 
4725 	/* initialize Rx descriptor 0 */
4726 	rx_desc = IGB_RX_DESC(ring, 0);
4727 	rx_desc->wb.upper.length = 0;
4728 
4729 	/* enable receive descriptor fetching */
4730 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4731 	wr32(E1000_RXDCTL(reg_idx), rxdctl);
4732 }
4733 
4734 static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
4735 				  struct igb_ring *rx_ring)
4736 {
4737 	/* set build_skb and buffer size flags */
4738 	clear_ring_build_skb_enabled(rx_ring);
4739 	clear_ring_uses_large_buffer(rx_ring);
4740 
4741 	if (adapter->flags & IGB_FLAG_RX_LEGACY)
4742 		return;
4743 
4744 	set_ring_build_skb_enabled(rx_ring);
4745 
4746 #if (PAGE_SIZE < 8192)
4747 	if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
4748 		return;
4749 
4750 	set_ring_uses_large_buffer(rx_ring);
4751 #endif
4752 }
4753 
4754 /**
4755  *  igb_configure_rx - Configure receive Unit after Reset
4756  *  @adapter: board private structure
4757  *
4758  *  Configure the Rx unit of the MAC after a reset.
4759  **/
4760 static void igb_configure_rx(struct igb_adapter *adapter)
4761 {
4762 	int i;
4763 
4764 	/* set the correct pool for the PF default MAC address in entry 0 */
4765 	igb_set_default_mac_filter(adapter);
4766 
4767 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
4768 	 * the Base and Length of the Rx Descriptor Ring
4769 	 */
4770 	for (i = 0; i < adapter->num_rx_queues; i++) {
4771 		struct igb_ring *rx_ring = adapter->rx_ring[i];
4772 
4773 		igb_set_rx_buffer_len(adapter, rx_ring);
4774 		igb_configure_rx_ring(adapter, rx_ring);
4775 	}
4776 }
4777 
4778 /**
4779  *  igb_free_tx_resources - Free Tx Resources per Queue
4780  *  @tx_ring: Tx descriptor ring for a specific queue
4781  *
4782  *  Free all transmit software resources
4783  **/
4784 void igb_free_tx_resources(struct igb_ring *tx_ring)
4785 {
4786 	igb_clean_tx_ring(tx_ring);
4787 
4788 	vfree(tx_ring->tx_buffer_info);
4789 	tx_ring->tx_buffer_info = NULL;
4790 
4791 	/* if not set, then don't free */
4792 	if (!tx_ring->desc)
4793 		return;
4794 
4795 	dma_free_coherent(tx_ring->dev, tx_ring->size,
4796 			  tx_ring->desc, tx_ring->dma);
4797 
4798 	tx_ring->desc = NULL;
4799 }
4800 
4801 /**
4802  *  igb_free_all_tx_resources - Free Tx Resources for All Queues
4803  *  @adapter: board private structure
4804  *
4805  *  Free all transmit software resources
4806  **/
4807 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4808 {
4809 	int i;
4810 
4811 	for (i = 0; i < adapter->num_tx_queues; i++)
4812 		if (adapter->tx_ring[i])
4813 			igb_free_tx_resources(adapter->tx_ring[i]);
4814 }
4815 
4816 /**
4817  *  igb_clean_tx_ring - Free Tx Buffers
4818  *  @tx_ring: ring to be cleaned
4819  **/
4820 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4821 {
4822 	u16 i = tx_ring->next_to_clean;
4823 	struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
4824 
4825 	while (i != tx_ring->next_to_use) {
4826 		union e1000_adv_tx_desc *eop_desc, *tx_desc;
4827 
4828 		/* Free all the Tx ring sk_buffs or xdp frames */
4829 		if (tx_buffer->type == IGB_TYPE_SKB)
4830 			dev_kfree_skb_any(tx_buffer->skb);
4831 		else
4832 			xdp_return_frame(tx_buffer->xdpf);
4833 
4834 		/* unmap skb header data */
4835 		dma_unmap_single(tx_ring->dev,
4836 				 dma_unmap_addr(tx_buffer, dma),
4837 				 dma_unmap_len(tx_buffer, len),
4838 				 DMA_TO_DEVICE);
4839 
4840 		/* check for eop_desc to determine the end of the packet */
4841 		eop_desc = tx_buffer->next_to_watch;
4842 		tx_desc = IGB_TX_DESC(tx_ring, i);
4843 
4844 		/* unmap remaining buffers */
4845 		while (tx_desc != eop_desc) {
4846 			tx_buffer++;
4847 			tx_desc++;
4848 			i++;
4849 			if (unlikely(i == tx_ring->count)) {
4850 				i = 0;
4851 				tx_buffer = tx_ring->tx_buffer_info;
4852 				tx_desc = IGB_TX_DESC(tx_ring, 0);
4853 			}
4854 
4855 			/* unmap any remaining paged data */
4856 			if (dma_unmap_len(tx_buffer, len))
4857 				dma_unmap_page(tx_ring->dev,
4858 					       dma_unmap_addr(tx_buffer, dma),
4859 					       dma_unmap_len(tx_buffer, len),
4860 					       DMA_TO_DEVICE);
4861 		}
4862 
4863 		tx_buffer->next_to_watch = NULL;
4864 
4865 		/* move us one more past the eop_desc for start of next pkt */
4866 		tx_buffer++;
4867 		i++;
4868 		if (unlikely(i == tx_ring->count)) {
4869 			i = 0;
4870 			tx_buffer = tx_ring->tx_buffer_info;
4871 		}
4872 	}
4873 
4874 	/* reset BQL for queue */
4875 	netdev_tx_reset_queue(txring_txq(tx_ring));
4876 
4877 	/* reset next_to_use and next_to_clean */
4878 	tx_ring->next_to_use = 0;
4879 	tx_ring->next_to_clean = 0;
4880 }
4881 
4882 /**
4883  *  igb_clean_all_tx_rings - Free Tx Buffers for all queues
4884  *  @adapter: board private structure
4885  **/
4886 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4887 {
4888 	int i;
4889 
4890 	for (i = 0; i < adapter->num_tx_queues; i++)
4891 		if (adapter->tx_ring[i])
4892 			igb_clean_tx_ring(adapter->tx_ring[i]);
4893 }
4894 
4895 /**
4896  *  igb_free_rx_resources - Free Rx Resources
4897  *  @rx_ring: ring to clean the resources from
4898  *
4899  *  Free all receive software resources
4900  **/
4901 void igb_free_rx_resources(struct igb_ring *rx_ring)
4902 {
4903 	igb_clean_rx_ring(rx_ring);
4904 
4905 	rx_ring->xdp_prog = NULL;
4906 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4907 	vfree(rx_ring->rx_buffer_info);
4908 	rx_ring->rx_buffer_info = NULL;
4909 
4910 	/* if not set, then don't free */
4911 	if (!rx_ring->desc)
4912 		return;
4913 
4914 	dma_free_coherent(rx_ring->dev, rx_ring->size,
4915 			  rx_ring->desc, rx_ring->dma);
4916 
4917 	rx_ring->desc = NULL;
4918 }
4919 
4920 /**
4921  *  igb_free_all_rx_resources - Free Rx Resources for All Queues
4922  *  @adapter: board private structure
4923  *
4924  *  Free all receive software resources
4925  **/
4926 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
4927 {
4928 	int i;
4929 
4930 	for (i = 0; i < adapter->num_rx_queues; i++)
4931 		if (adapter->rx_ring[i])
4932 			igb_free_rx_resources(adapter->rx_ring[i]);
4933 }
4934 
4935 /**
4936  *  igb_clean_rx_ring - Free Rx Buffers per Queue
4937  *  @rx_ring: ring to free buffers from
4938  **/
4939 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
4940 {
4941 	u16 i = rx_ring->next_to_clean;
4942 
4943 	dev_kfree_skb(rx_ring->skb);
4944 	rx_ring->skb = NULL;
4945 
4946 	/* Free all the Rx ring sk_buffs */
4947 	while (i != rx_ring->next_to_alloc) {
4948 		struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
4949 
4950 		/* Invalidate cache lines that may have been written to by
4951 		 * device so that we avoid corrupting memory.
4952 		 */
4953 		dma_sync_single_range_for_cpu(rx_ring->dev,
4954 					      buffer_info->dma,
4955 					      buffer_info->page_offset,
4956 					      igb_rx_bufsz(rx_ring),
4957 					      DMA_FROM_DEVICE);
4958 
4959 		/* free resources associated with mapping */
4960 		dma_unmap_page_attrs(rx_ring->dev,
4961 				     buffer_info->dma,
4962 				     igb_rx_pg_size(rx_ring),
4963 				     DMA_FROM_DEVICE,
4964 				     IGB_RX_DMA_ATTR);
4965 		__page_frag_cache_drain(buffer_info->page,
4966 					buffer_info->pagecnt_bias);
4967 
4968 		i++;
4969 		if (i == rx_ring->count)
4970 			i = 0;
4971 	}
4972 
4973 	rx_ring->next_to_alloc = 0;
4974 	rx_ring->next_to_clean = 0;
4975 	rx_ring->next_to_use = 0;
4976 }
4977 
4978 /**
4979  *  igb_clean_all_rx_rings - Free Rx Buffers for all queues
4980  *  @adapter: board private structure
4981  **/
4982 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
4983 {
4984 	int i;
4985 
4986 	for (i = 0; i < adapter->num_rx_queues; i++)
4987 		if (adapter->rx_ring[i])
4988 			igb_clean_rx_ring(adapter->rx_ring[i]);
4989 }
4990 
4991 /**
4992  *  igb_set_mac - Change the Ethernet Address of the NIC
4993  *  @netdev: network interface device structure
4994  *  @p: pointer to an address structure
4995  *
4996  *  Returns 0 on success, negative on failure
4997  **/
4998 static int igb_set_mac(struct net_device *netdev, void *p)
4999 {
5000 	struct igb_adapter *adapter = netdev_priv(netdev);
5001 	struct e1000_hw *hw = &adapter->hw;
5002 	struct sockaddr *addr = p;
5003 
5004 	if (!is_valid_ether_addr(addr->sa_data))
5005 		return -EADDRNOTAVAIL;
5006 
5007 	eth_hw_addr_set(netdev, addr->sa_data);
5008 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
5009 
5010 	/* set the correct pool for the new PF MAC address in entry 0 */
5011 	igb_set_default_mac_filter(adapter);
5012 
5013 	return 0;
5014 }
5015 
5016 /**
5017  *  igb_write_mc_addr_list - write multicast addresses to MTA
5018  *  @netdev: network interface device structure
5019  *
5020  *  Writes multicast address list to the MTA hash table.
5021  *  Returns: -ENOMEM on failure
5022  *           0 on no addresses written
5023  *           X on writing X addresses to MTA
5024  **/
5025 static int igb_write_mc_addr_list(struct net_device *netdev)
5026 {
5027 	struct igb_adapter *adapter = netdev_priv(netdev);
5028 	struct e1000_hw *hw = &adapter->hw;
5029 	struct netdev_hw_addr *ha;
5030 	u8  *mta_list;
5031 	int i;
5032 
5033 	if (netdev_mc_empty(netdev)) {
5034 		/* nothing to program, so clear mc list */
5035 		igb_update_mc_addr_list(hw, NULL, 0);
5036 		igb_restore_vf_multicasts(adapter);
5037 		return 0;
5038 	}
5039 
5040 	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
5041 	if (!mta_list)
5042 		return -ENOMEM;
5043 
5044 	/* The shared function expects a packed array of only addresses. */
5045 	i = 0;
5046 	netdev_for_each_mc_addr(ha, netdev)
5047 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
5048 
5049 	igb_update_mc_addr_list(hw, mta_list, i);
5050 	kfree(mta_list);
5051 
5052 	return netdev_mc_count(netdev);
5053 }
5054 
5055 static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
5056 {
5057 	struct e1000_hw *hw = &adapter->hw;
5058 	u32 i, pf_id;
5059 
5060 	switch (hw->mac.type) {
5061 	case e1000_i210:
5062 	case e1000_i211:
5063 	case e1000_i350:
5064 		/* VLAN filtering needed for VLAN prio filter */
5065 		if (adapter->netdev->features & NETIF_F_NTUPLE)
5066 			break;
5067 		fallthrough;
5068 	case e1000_82576:
5069 	case e1000_82580:
5070 	case e1000_i354:
5071 		/* VLAN filtering needed for pool filtering */
5072 		if (adapter->vfs_allocated_count)
5073 			break;
5074 		fallthrough;
5075 	default:
5076 		return 1;
5077 	}
5078 
5079 	/* We are already in VLAN promisc, nothing to do */
5080 	if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
5081 		return 0;
5082 
5083 	if (!adapter->vfs_allocated_count)
5084 		goto set_vfta;
5085 
5086 	/* Add PF to all active pools */
5087 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5088 
5089 	for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5090 		u32 vlvf = rd32(E1000_VLVF(i));
5091 
5092 		vlvf |= BIT(pf_id);
5093 		wr32(E1000_VLVF(i), vlvf);
5094 	}
5095 
5096 set_vfta:
5097 	/* Set all bits in the VLAN filter table array */
5098 	for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
5099 		hw->mac.ops.write_vfta(hw, i, ~0U);
5100 
5101 	/* Set flag so we don't redo unnecessary work */
5102 	adapter->flags |= IGB_FLAG_VLAN_PROMISC;
5103 
5104 	return 0;
5105 }
5106 
5107 #define VFTA_BLOCK_SIZE 8
5108 static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
5109 {
5110 	struct e1000_hw *hw = &adapter->hw;
5111 	u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
5112 	u32 vid_start = vfta_offset * 32;
5113 	u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
5114 	u32 i, vid, word, bits, pf_id;
5115 
5116 	/* guarantee that we don't scrub out management VLAN */
5117 	vid = adapter->mng_vlan_id;
5118 	if (vid >= vid_start && vid < vid_end)
5119 		vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5120 
5121 	if (!adapter->vfs_allocated_count)
5122 		goto set_vfta;
5123 
5124 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5125 
5126 	for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5127 		u32 vlvf = rd32(E1000_VLVF(i));
5128 
5129 		/* pull VLAN ID from VLVF */
5130 		vid = vlvf & VLAN_VID_MASK;
5131 
5132 		/* only concern ourselves with a certain range */
5133 		if (vid < vid_start || vid >= vid_end)
5134 			continue;
5135 
5136 		if (vlvf & E1000_VLVF_VLANID_ENABLE) {
5137 			/* record VLAN ID in VFTA */
5138 			vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5139 
5140 			/* if PF is part of this then continue */
5141 			if (test_bit(vid, adapter->active_vlans))
5142 				continue;
5143 		}
5144 
5145 		/* remove PF from the pool */
5146 		bits = ~BIT(pf_id);
5147 		bits &= rd32(E1000_VLVF(i));
5148 		wr32(E1000_VLVF(i), bits);
5149 	}
5150 
5151 set_vfta:
5152 	/* extract values from active_vlans and write back to VFTA */
5153 	for (i = VFTA_BLOCK_SIZE; i--;) {
5154 		vid = (vfta_offset + i) * 32;
5155 		word = vid / BITS_PER_LONG;
5156 		bits = vid % BITS_PER_LONG;
5157 
5158 		vfta[i] |= adapter->active_vlans[word] >> bits;
5159 
5160 		hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
5161 	}
5162 }
5163 
5164 static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
5165 {
5166 	u32 i;
5167 
5168 	/* We are not in VLAN promisc, nothing to do */
5169 	if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
5170 		return;
5171 
5172 	/* Set flag so we don't redo unnecessary work */
5173 	adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
5174 
5175 	for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
5176 		igb_scrub_vfta(adapter, i);
5177 }
5178 
5179 /**
5180  *  igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
5181  *  @netdev: network interface device structure
5182  *
5183  *  The set_rx_mode entry point is called whenever the unicast or multicast
5184  *  address lists or the network interface flags are updated.  This routine is
5185  *  responsible for configuring the hardware for proper unicast, multicast,
5186  *  promiscuous mode, and all-multi behavior.
5187  **/
5188 static void igb_set_rx_mode(struct net_device *netdev)
5189 {
5190 	struct igb_adapter *adapter = netdev_priv(netdev);
5191 	struct e1000_hw *hw = &adapter->hw;
5192 	unsigned int vfn = adapter->vfs_allocated_count;
5193 	u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
5194 	int count;
5195 
5196 	/* Check for Promiscuous and All Multicast modes */
5197 	if (netdev->flags & IFF_PROMISC) {
5198 		rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
5199 		vmolr |= E1000_VMOLR_MPME;
5200 
5201 		/* enable use of UTA filter to force packets to default pool */
5202 		if (hw->mac.type == e1000_82576)
5203 			vmolr |= E1000_VMOLR_ROPE;
5204 	} else {
5205 		if (netdev->flags & IFF_ALLMULTI) {
5206 			rctl |= E1000_RCTL_MPE;
5207 			vmolr |= E1000_VMOLR_MPME;
5208 		} else {
5209 			/* Write addresses to the MTA, if the attempt fails
5210 			 * then we should just turn on promiscuous mode so
5211 			 * that we can at least receive multicast traffic
5212 			 */
5213 			count = igb_write_mc_addr_list(netdev);
5214 			if (count < 0) {
5215 				rctl |= E1000_RCTL_MPE;
5216 				vmolr |= E1000_VMOLR_MPME;
5217 			} else if (count) {
5218 				vmolr |= E1000_VMOLR_ROMPE;
5219 			}
5220 		}
5221 	}
5222 
5223 	/* Write addresses to available RAR registers, if there is not
5224 	 * sufficient space to store all the addresses then enable
5225 	 * unicast promiscuous mode
5226 	 */
5227 	if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
5228 		rctl |= E1000_RCTL_UPE;
5229 		vmolr |= E1000_VMOLR_ROPE;
5230 	}
5231 
5232 	/* enable VLAN filtering by default */
5233 	rctl |= E1000_RCTL_VFE;
5234 
5235 	/* disable VLAN filtering for modes that require it */
5236 	if ((netdev->flags & IFF_PROMISC) ||
5237 	    (netdev->features & NETIF_F_RXALL)) {
5238 		/* if we fail to set all rules then just clear VFE */
5239 		if (igb_vlan_promisc_enable(adapter))
5240 			rctl &= ~E1000_RCTL_VFE;
5241 	} else {
5242 		igb_vlan_promisc_disable(adapter);
5243 	}
5244 
5245 	/* update state of unicast, multicast, and VLAN filtering modes */
5246 	rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
5247 				     E1000_RCTL_VFE);
5248 	wr32(E1000_RCTL, rctl);
5249 
5250 #if (PAGE_SIZE < 8192)
5251 	if (!adapter->vfs_allocated_count) {
5252 		if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5253 			rlpml = IGB_MAX_FRAME_BUILD_SKB;
5254 	}
5255 #endif
5256 	wr32(E1000_RLPML, rlpml);
5257 
5258 	/* In order to support SR-IOV and eventually VMDq it is necessary to set
5259 	 * the VMOLR to enable the appropriate modes.  Without this workaround
5260 	 * we will have issues with VLAN tag stripping not being done for frames
5261 	 * that are only arriving because we are the default pool
5262 	 */
5263 	if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
5264 		return;
5265 
5266 	/* set UTA to appropriate mode */
5267 	igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
5268 
5269 	vmolr |= rd32(E1000_VMOLR(vfn)) &
5270 		 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
5271 
5272 	/* enable Rx jumbo frames, restrict as needed to support build_skb */
5273 	vmolr &= ~E1000_VMOLR_RLPML_MASK;
5274 #if (PAGE_SIZE < 8192)
5275 	if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5276 		vmolr |= IGB_MAX_FRAME_BUILD_SKB;
5277 	else
5278 #endif
5279 		vmolr |= MAX_JUMBO_FRAME_SIZE;
5280 	vmolr |= E1000_VMOLR_LPE;
5281 
5282 	wr32(E1000_VMOLR(vfn), vmolr);
5283 
5284 	igb_restore_vf_multicasts(adapter);
5285 }
5286 
5287 static void igb_check_wvbr(struct igb_adapter *adapter)
5288 {
5289 	struct e1000_hw *hw = &adapter->hw;
5290 	u32 wvbr = 0;
5291 
5292 	switch (hw->mac.type) {
5293 	case e1000_82576:
5294 	case e1000_i350:
5295 		wvbr = rd32(E1000_WVBR);
5296 		if (!wvbr)
5297 			return;
5298 		break;
5299 	default:
5300 		break;
5301 	}
5302 
5303 	adapter->wvbr |= wvbr;
5304 }
5305 
5306 #define IGB_STAGGERED_QUEUE_OFFSET 8
5307 
5308 static void igb_spoof_check(struct igb_adapter *adapter)
5309 {
5310 	int j;
5311 
5312 	if (!adapter->wvbr)
5313 		return;
5314 
5315 	for (j = 0; j < adapter->vfs_allocated_count; j++) {
5316 		if (adapter->wvbr & BIT(j) ||
5317 		    adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
5318 			dev_warn(&adapter->pdev->dev,
5319 				"Spoof event(s) detected on VF %d\n", j);
5320 			adapter->wvbr &=
5321 				~(BIT(j) |
5322 				  BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
5323 		}
5324 	}
5325 }
5326 
5327 /* Need to wait a few seconds after link up to get diagnostic information from
5328  * the phy
5329  */
5330 static void igb_update_phy_info(struct timer_list *t)
5331 {
5332 	struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5333 	igb_get_phy_info(&adapter->hw);
5334 }
5335 
5336 /**
5337  *  igb_has_link - check shared code for link and determine up/down
5338  *  @adapter: pointer to driver private info
5339  **/
5340 bool igb_has_link(struct igb_adapter *adapter)
5341 {
5342 	struct e1000_hw *hw = &adapter->hw;
5343 	bool link_active = false;
5344 
5345 	/* get_link_status is set on LSC (link status) interrupt or
5346 	 * rx sequence error interrupt.  get_link_status will stay
5347 	 * false until the e1000_check_for_link establishes link
5348 	 * for copper adapters ONLY
5349 	 */
5350 	switch (hw->phy.media_type) {
5351 	case e1000_media_type_copper:
5352 		if (!hw->mac.get_link_status)
5353 			return true;
5354 		fallthrough;
5355 	case e1000_media_type_internal_serdes:
5356 		hw->mac.ops.check_for_link(hw);
5357 		link_active = !hw->mac.get_link_status;
5358 		break;
5359 	default:
5360 	case e1000_media_type_unknown:
5361 		break;
5362 	}
5363 
5364 	if (((hw->mac.type == e1000_i210) ||
5365 	     (hw->mac.type == e1000_i211)) &&
5366 	     (hw->phy.id == I210_I_PHY_ID)) {
5367 		if (!netif_carrier_ok(adapter->netdev)) {
5368 			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5369 		} else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
5370 			adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
5371 			adapter->link_check_timeout = jiffies;
5372 		}
5373 	}
5374 
5375 	return link_active;
5376 }
5377 
5378 static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
5379 {
5380 	bool ret = false;
5381 	u32 ctrl_ext, thstat;
5382 
5383 	/* check for thermal sensor event on i350 copper only */
5384 	if (hw->mac.type == e1000_i350) {
5385 		thstat = rd32(E1000_THSTAT);
5386 		ctrl_ext = rd32(E1000_CTRL_EXT);
5387 
5388 		if ((hw->phy.media_type == e1000_media_type_copper) &&
5389 		    !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
5390 			ret = !!(thstat & event);
5391 	}
5392 
5393 	return ret;
5394 }
5395 
5396 /**
5397  *  igb_check_lvmmc - check for malformed packets received
5398  *  and indicated in LVMMC register
5399  *  @adapter: pointer to adapter
5400  **/
5401 static void igb_check_lvmmc(struct igb_adapter *adapter)
5402 {
5403 	struct e1000_hw *hw = &adapter->hw;
5404 	u32 lvmmc;
5405 
5406 	lvmmc = rd32(E1000_LVMMC);
5407 	if (lvmmc) {
5408 		if (unlikely(net_ratelimit())) {
5409 			netdev_warn(adapter->netdev,
5410 				    "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
5411 				    lvmmc);
5412 		}
5413 	}
5414 }
5415 
5416 /**
5417  *  igb_watchdog - Timer Call-back
5418  *  @t: pointer to timer_list containing our private info pointer
5419  **/
5420 static void igb_watchdog(struct timer_list *t)
5421 {
5422 	struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5423 	/* Do the rest outside of interrupt context */
5424 	schedule_work(&adapter->watchdog_task);
5425 }
5426 
5427 static void igb_watchdog_task(struct work_struct *work)
5428 {
5429 	struct igb_adapter *adapter = container_of(work,
5430 						   struct igb_adapter,
5431 						   watchdog_task);
5432 	struct e1000_hw *hw = &adapter->hw;
5433 	struct e1000_phy_info *phy = &hw->phy;
5434 	struct net_device *netdev = adapter->netdev;
5435 	u32 link;
5436 	int i;
5437 	u32 connsw;
5438 	u16 phy_data, retry_count = 20;
5439 
5440 	link = igb_has_link(adapter);
5441 
5442 	if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
5443 		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5444 			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5445 		else
5446 			link = false;
5447 	}
5448 
5449 	/* Force link down if we have fiber to swap to */
5450 	if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5451 		if (hw->phy.media_type == e1000_media_type_copper) {
5452 			connsw = rd32(E1000_CONNSW);
5453 			if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
5454 				link = 0;
5455 		}
5456 	}
5457 	if (link) {
5458 		/* Perform a reset if the media type changed. */
5459 		if (hw->dev_spec._82575.media_changed) {
5460 			hw->dev_spec._82575.media_changed = false;
5461 			adapter->flags |= IGB_FLAG_MEDIA_RESET;
5462 			igb_reset(adapter);
5463 		}
5464 		/* Cancel scheduled suspend requests. */
5465 		pm_runtime_resume(netdev->dev.parent);
5466 
5467 		if (!netif_carrier_ok(netdev)) {
5468 			u32 ctrl;
5469 
5470 			hw->mac.ops.get_speed_and_duplex(hw,
5471 							 &adapter->link_speed,
5472 							 &adapter->link_duplex);
5473 
5474 			ctrl = rd32(E1000_CTRL);
5475 			/* Links status message must follow this format */
5476 			netdev_info(netdev,
5477 			       "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5478 			       netdev->name,
5479 			       adapter->link_speed,
5480 			       adapter->link_duplex == FULL_DUPLEX ?
5481 			       "Full" : "Half",
5482 			       (ctrl & E1000_CTRL_TFCE) &&
5483 			       (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
5484 			       (ctrl & E1000_CTRL_RFCE) ?  "RX" :
5485 			       (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
5486 
5487 			/* disable EEE if enabled */
5488 			if ((adapter->flags & IGB_FLAG_EEE) &&
5489 				(adapter->link_duplex == HALF_DUPLEX)) {
5490 				dev_info(&adapter->pdev->dev,
5491 				"EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
5492 				adapter->hw.dev_spec._82575.eee_disable = true;
5493 				adapter->flags &= ~IGB_FLAG_EEE;
5494 			}
5495 
5496 			/* check if SmartSpeed worked */
5497 			igb_check_downshift(hw);
5498 			if (phy->speed_downgraded)
5499 				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5500 
5501 			/* check for thermal sensor event */
5502 			if (igb_thermal_sensor_event(hw,
5503 			    E1000_THSTAT_LINK_THROTTLE))
5504 				netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
5505 
5506 			/* adjust timeout factor according to speed/duplex */
5507 			adapter->tx_timeout_factor = 1;
5508 			switch (adapter->link_speed) {
5509 			case SPEED_10:
5510 				adapter->tx_timeout_factor = 14;
5511 				break;
5512 			case SPEED_100:
5513 				/* maybe add some timeout factor ? */
5514 				break;
5515 			}
5516 
5517 			if (adapter->link_speed != SPEED_1000 ||
5518 			    !hw->phy.ops.read_reg)
5519 				goto no_wait;
5520 
5521 			/* wait for Remote receiver status OK */
5522 retry_read_status:
5523 			if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
5524 					      &phy_data)) {
5525 				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5526 				    retry_count) {
5527 					msleep(100);
5528 					retry_count--;
5529 					goto retry_read_status;
5530 				} else if (!retry_count) {
5531 					dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
5532 				}
5533 			} else {
5534 				dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
5535 			}
5536 no_wait:
5537 			netif_carrier_on(netdev);
5538 
5539 			igb_ping_all_vfs(adapter);
5540 			igb_check_vf_rate_limit(adapter);
5541 
5542 			/* link state has changed, schedule phy info update */
5543 			if (!test_bit(__IGB_DOWN, &adapter->state))
5544 				mod_timer(&adapter->phy_info_timer,
5545 					  round_jiffies(jiffies + 2 * HZ));
5546 		}
5547 	} else {
5548 		if (netif_carrier_ok(netdev)) {
5549 			adapter->link_speed = 0;
5550 			adapter->link_duplex = 0;
5551 
5552 			/* check for thermal sensor event */
5553 			if (igb_thermal_sensor_event(hw,
5554 			    E1000_THSTAT_PWR_DOWN)) {
5555 				netdev_err(netdev, "The network adapter was stopped because it overheated\n");
5556 			}
5557 
5558 			/* Links status message must follow this format */
5559 			netdev_info(netdev, "igb: %s NIC Link is Down\n",
5560 			       netdev->name);
5561 			netif_carrier_off(netdev);
5562 
5563 			igb_ping_all_vfs(adapter);
5564 
5565 			/* link state has changed, schedule phy info update */
5566 			if (!test_bit(__IGB_DOWN, &adapter->state))
5567 				mod_timer(&adapter->phy_info_timer,
5568 					  round_jiffies(jiffies + 2 * HZ));
5569 
5570 			/* link is down, time to check for alternate media */
5571 			if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5572 				igb_check_swap_media(adapter);
5573 				if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5574 					schedule_work(&adapter->reset_task);
5575 					/* return immediately */
5576 					return;
5577 				}
5578 			}
5579 			pm_schedule_suspend(netdev->dev.parent,
5580 					    MSEC_PER_SEC * 5);
5581 
5582 		/* also check for alternate media here */
5583 		} else if (!netif_carrier_ok(netdev) &&
5584 			   (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
5585 			igb_check_swap_media(adapter);
5586 			if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5587 				schedule_work(&adapter->reset_task);
5588 				/* return immediately */
5589 				return;
5590 			}
5591 		}
5592 	}
5593 
5594 	spin_lock(&adapter->stats64_lock);
5595 	igb_update_stats(adapter);
5596 	spin_unlock(&adapter->stats64_lock);
5597 
5598 	for (i = 0; i < adapter->num_tx_queues; i++) {
5599 		struct igb_ring *tx_ring = adapter->tx_ring[i];
5600 		if (!netif_carrier_ok(netdev)) {
5601 			/* We've lost link, so the controller stops DMA,
5602 			 * but we've got queued Tx work that's never going
5603 			 * to get done, so reset controller to flush Tx.
5604 			 * (Do the reset outside of interrupt context).
5605 			 */
5606 			if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
5607 				adapter->tx_timeout_count++;
5608 				schedule_work(&adapter->reset_task);
5609 				/* return immediately since reset is imminent */
5610 				return;
5611 			}
5612 		}
5613 
5614 		/* Force detection of hung controller every watchdog period */
5615 		set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5616 	}
5617 
5618 	/* Cause software interrupt to ensure Rx ring is cleaned */
5619 	if (adapter->flags & IGB_FLAG_HAS_MSIX) {
5620 		u32 eics = 0;
5621 
5622 		for (i = 0; i < adapter->num_q_vectors; i++)
5623 			eics |= adapter->q_vector[i]->eims_value;
5624 		wr32(E1000_EICS, eics);
5625 	} else {
5626 		wr32(E1000_ICS, E1000_ICS_RXDMT0);
5627 	}
5628 
5629 	igb_spoof_check(adapter);
5630 	igb_ptp_rx_hang(adapter);
5631 	igb_ptp_tx_hang(adapter);
5632 
5633 	/* Check LVMMC register on i350/i354 only */
5634 	if ((adapter->hw.mac.type == e1000_i350) ||
5635 	    (adapter->hw.mac.type == e1000_i354))
5636 		igb_check_lvmmc(adapter);
5637 
5638 	/* Reset the timer */
5639 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
5640 		if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
5641 			mod_timer(&adapter->watchdog_timer,
5642 				  round_jiffies(jiffies +  HZ));
5643 		else
5644 			mod_timer(&adapter->watchdog_timer,
5645 				  round_jiffies(jiffies + 2 * HZ));
5646 	}
5647 }
5648 
5649 enum latency_range {
5650 	lowest_latency = 0,
5651 	low_latency = 1,
5652 	bulk_latency = 2,
5653 	latency_invalid = 255
5654 };
5655 
5656 /**
5657  *  igb_update_ring_itr - update the dynamic ITR value based on packet size
5658  *  @q_vector: pointer to q_vector
5659  *
5660  *  Stores a new ITR value based on strictly on packet size.  This
5661  *  algorithm is less sophisticated than that used in igb_update_itr,
5662  *  due to the difficulty of synchronizing statistics across multiple
5663  *  receive rings.  The divisors and thresholds used by this function
5664  *  were determined based on theoretical maximum wire speed and testing
5665  *  data, in order to minimize response time while increasing bulk
5666  *  throughput.
5667  *  This functionality is controlled by ethtool's coalescing settings.
5668  *  NOTE:  This function is called only when operating in a multiqueue
5669  *         receive environment.
5670  **/
5671 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
5672 {
5673 	int new_val = q_vector->itr_val;
5674 	int avg_wire_size = 0;
5675 	struct igb_adapter *adapter = q_vector->adapter;
5676 	unsigned int packets;
5677 
5678 	/* For non-gigabit speeds, just fix the interrupt rate at 4000
5679 	 * ints/sec - ITR timer value of 120 ticks.
5680 	 */
5681 	if (adapter->link_speed != SPEED_1000) {
5682 		new_val = IGB_4K_ITR;
5683 		goto set_itr_val;
5684 	}
5685 
5686 	packets = q_vector->rx.total_packets;
5687 	if (packets)
5688 		avg_wire_size = q_vector->rx.total_bytes / packets;
5689 
5690 	packets = q_vector->tx.total_packets;
5691 	if (packets)
5692 		avg_wire_size = max_t(u32, avg_wire_size,
5693 				      q_vector->tx.total_bytes / packets);
5694 
5695 	/* if avg_wire_size isn't set no work was done */
5696 	if (!avg_wire_size)
5697 		goto clear_counts;
5698 
5699 	/* Add 24 bytes to size to account for CRC, preamble, and gap */
5700 	avg_wire_size += 24;
5701 
5702 	/* Don't starve jumbo frames */
5703 	avg_wire_size = min(avg_wire_size, 3000);
5704 
5705 	/* Give a little boost to mid-size frames */
5706 	if ((avg_wire_size > 300) && (avg_wire_size < 1200))
5707 		new_val = avg_wire_size / 3;
5708 	else
5709 		new_val = avg_wire_size / 2;
5710 
5711 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
5712 	if (new_val < IGB_20K_ITR &&
5713 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5714 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5715 		new_val = IGB_20K_ITR;
5716 
5717 set_itr_val:
5718 	if (new_val != q_vector->itr_val) {
5719 		q_vector->itr_val = new_val;
5720 		q_vector->set_itr = 1;
5721 	}
5722 clear_counts:
5723 	q_vector->rx.total_bytes = 0;
5724 	q_vector->rx.total_packets = 0;
5725 	q_vector->tx.total_bytes = 0;
5726 	q_vector->tx.total_packets = 0;
5727 }
5728 
5729 /**
5730  *  igb_update_itr - update the dynamic ITR value based on statistics
5731  *  @q_vector: pointer to q_vector
5732  *  @ring_container: ring info to update the itr for
5733  *
5734  *  Stores a new ITR value based on packets and byte
5735  *  counts during the last interrupt.  The advantage of per interrupt
5736  *  computation is faster updates and more accurate ITR for the current
5737  *  traffic pattern.  Constants in this function were computed
5738  *  based on theoretical maximum wire speed and thresholds were set based
5739  *  on testing data as well as attempting to minimize response time
5740  *  while increasing bulk throughput.
5741  *  This functionality is controlled by ethtool's coalescing settings.
5742  *  NOTE:  These calculations are only valid when operating in a single-
5743  *         queue environment.
5744  **/
5745 static void igb_update_itr(struct igb_q_vector *q_vector,
5746 			   struct igb_ring_container *ring_container)
5747 {
5748 	unsigned int packets = ring_container->total_packets;
5749 	unsigned int bytes = ring_container->total_bytes;
5750 	u8 itrval = ring_container->itr;
5751 
5752 	/* no packets, exit with status unchanged */
5753 	if (packets == 0)
5754 		return;
5755 
5756 	switch (itrval) {
5757 	case lowest_latency:
5758 		/* handle TSO and jumbo frames */
5759 		if (bytes/packets > 8000)
5760 			itrval = bulk_latency;
5761 		else if ((packets < 5) && (bytes > 512))
5762 			itrval = low_latency;
5763 		break;
5764 	case low_latency:  /* 50 usec aka 20000 ints/s */
5765 		if (bytes > 10000) {
5766 			/* this if handles the TSO accounting */
5767 			if (bytes/packets > 8000)
5768 				itrval = bulk_latency;
5769 			else if ((packets < 10) || ((bytes/packets) > 1200))
5770 				itrval = bulk_latency;
5771 			else if ((packets > 35))
5772 				itrval = lowest_latency;
5773 		} else if (bytes/packets > 2000) {
5774 			itrval = bulk_latency;
5775 		} else if (packets <= 2 && bytes < 512) {
5776 			itrval = lowest_latency;
5777 		}
5778 		break;
5779 	case bulk_latency: /* 250 usec aka 4000 ints/s */
5780 		if (bytes > 25000) {
5781 			if (packets > 35)
5782 				itrval = low_latency;
5783 		} else if (bytes < 1500) {
5784 			itrval = low_latency;
5785 		}
5786 		break;
5787 	}
5788 
5789 	/* clear work counters since we have the values we need */
5790 	ring_container->total_bytes = 0;
5791 	ring_container->total_packets = 0;
5792 
5793 	/* write updated itr to ring container */
5794 	ring_container->itr = itrval;
5795 }
5796 
5797 static void igb_set_itr(struct igb_q_vector *q_vector)
5798 {
5799 	struct igb_adapter *adapter = q_vector->adapter;
5800 	u32 new_itr = q_vector->itr_val;
5801 	u8 current_itr = 0;
5802 
5803 	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5804 	if (adapter->link_speed != SPEED_1000) {
5805 		current_itr = 0;
5806 		new_itr = IGB_4K_ITR;
5807 		goto set_itr_now;
5808 	}
5809 
5810 	igb_update_itr(q_vector, &q_vector->tx);
5811 	igb_update_itr(q_vector, &q_vector->rx);
5812 
5813 	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5814 
5815 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
5816 	if (current_itr == lowest_latency &&
5817 	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5818 	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5819 		current_itr = low_latency;
5820 
5821 	switch (current_itr) {
5822 	/* counts and packets in update_itr are dependent on these numbers */
5823 	case lowest_latency:
5824 		new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5825 		break;
5826 	case low_latency:
5827 		new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5828 		break;
5829 	case bulk_latency:
5830 		new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
5831 		break;
5832 	default:
5833 		break;
5834 	}
5835 
5836 set_itr_now:
5837 	if (new_itr != q_vector->itr_val) {
5838 		/* this attempts to bias the interrupt rate towards Bulk
5839 		 * by adding intermediate steps when interrupt rate is
5840 		 * increasing
5841 		 */
5842 		new_itr = new_itr > q_vector->itr_val ?
5843 			  max((new_itr * q_vector->itr_val) /
5844 			  (new_itr + (q_vector->itr_val >> 2)),
5845 			  new_itr) : new_itr;
5846 		/* Don't write the value here; it resets the adapter's
5847 		 * internal timer, and causes us to delay far longer than
5848 		 * we should between interrupts.  Instead, we write the ITR
5849 		 * value at the beginning of the next interrupt so the timing
5850 		 * ends up being correct.
5851 		 */
5852 		q_vector->itr_val = new_itr;
5853 		q_vector->set_itr = 1;
5854 	}
5855 }
5856 
5857 static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
5858 			    struct igb_tx_buffer *first,
5859 			    u32 vlan_macip_lens, u32 type_tucmd,
5860 			    u32 mss_l4len_idx)
5861 {
5862 	struct e1000_adv_tx_context_desc *context_desc;
5863 	u16 i = tx_ring->next_to_use;
5864 	struct timespec64 ts;
5865 
5866 	context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5867 
5868 	i++;
5869 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5870 
5871 	/* set bits to identify this as an advanced context descriptor */
5872 	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5873 
5874 	/* For 82575, context index must be unique per ring. */
5875 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5876 		mss_l4len_idx |= tx_ring->reg_idx << 4;
5877 
5878 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
5879 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
5880 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
5881 
5882 	/* We assume there is always a valid tx time available. Invalid times
5883 	 * should have been handled by the upper layers.
5884 	 */
5885 	if (tx_ring->launchtime_enable) {
5886 		ts = ktime_to_timespec64(first->skb->tstamp);
5887 		skb_txtime_consumed(first->skb);
5888 		context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
5889 	} else {
5890 		context_desc->seqnum_seed = 0;
5891 	}
5892 }
5893 
5894 static int igb_tso(struct igb_ring *tx_ring,
5895 		   struct igb_tx_buffer *first,
5896 		   u8 *hdr_len)
5897 {
5898 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5899 	struct sk_buff *skb = first->skb;
5900 	union {
5901 		struct iphdr *v4;
5902 		struct ipv6hdr *v6;
5903 		unsigned char *hdr;
5904 	} ip;
5905 	union {
5906 		struct tcphdr *tcp;
5907 		struct udphdr *udp;
5908 		unsigned char *hdr;
5909 	} l4;
5910 	u32 paylen, l4_offset;
5911 	int err;
5912 
5913 	if (skb->ip_summed != CHECKSUM_PARTIAL)
5914 		return 0;
5915 
5916 	if (!skb_is_gso(skb))
5917 		return 0;
5918 
5919 	err = skb_cow_head(skb, 0);
5920 	if (err < 0)
5921 		return err;
5922 
5923 	ip.hdr = skb_network_header(skb);
5924 	l4.hdr = skb_checksum_start(skb);
5925 
5926 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5927 	type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
5928 		      E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
5929 
5930 	/* initialize outer IP header fields */
5931 	if (ip.v4->version == 4) {
5932 		unsigned char *csum_start = skb_checksum_start(skb);
5933 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
5934 
5935 		/* IP header will have to cancel out any data that
5936 		 * is not a part of the outer IP header
5937 		 */
5938 		ip.v4->check = csum_fold(csum_partial(trans_start,
5939 						      csum_start - trans_start,
5940 						      0));
5941 		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
5942 
5943 		ip.v4->tot_len = 0;
5944 		first->tx_flags |= IGB_TX_FLAGS_TSO |
5945 				   IGB_TX_FLAGS_CSUM |
5946 				   IGB_TX_FLAGS_IPV4;
5947 	} else {
5948 		ip.v6->payload_len = 0;
5949 		first->tx_flags |= IGB_TX_FLAGS_TSO |
5950 				   IGB_TX_FLAGS_CSUM;
5951 	}
5952 
5953 	/* determine offset of inner transport header */
5954 	l4_offset = l4.hdr - skb->data;
5955 
5956 	/* remove payload length from inner checksum */
5957 	paylen = skb->len - l4_offset;
5958 	if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
5959 		/* compute length of segmentation header */
5960 		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
5961 		csum_replace_by_diff(&l4.tcp->check,
5962 			(__force __wsum)htonl(paylen));
5963 	} else {
5964 		/* compute length of segmentation header */
5965 		*hdr_len = sizeof(*l4.udp) + l4_offset;
5966 		csum_replace_by_diff(&l4.udp->check,
5967 				     (__force __wsum)htonl(paylen));
5968 	}
5969 
5970 	/* update gso size and bytecount with header size */
5971 	first->gso_segs = skb_shinfo(skb)->gso_segs;
5972 	first->bytecount += (first->gso_segs - 1) * *hdr_len;
5973 
5974 	/* MSS L4LEN IDX */
5975 	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
5976 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
5977 
5978 	/* VLAN MACLEN IPLEN */
5979 	vlan_macip_lens = l4.hdr - ip.hdr;
5980 	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
5981 	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5982 
5983 	igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
5984 			type_tucmd, mss_l4len_idx);
5985 
5986 	return 1;
5987 }
5988 
5989 static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
5990 {
5991 	struct sk_buff *skb = first->skb;
5992 	u32 vlan_macip_lens = 0;
5993 	u32 type_tucmd = 0;
5994 
5995 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
5996 csum_failed:
5997 		if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
5998 		    !tx_ring->launchtime_enable)
5999 			return;
6000 		goto no_csum;
6001 	}
6002 
6003 	switch (skb->csum_offset) {
6004 	case offsetof(struct tcphdr, check):
6005 		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
6006 		fallthrough;
6007 	case offsetof(struct udphdr, check):
6008 		break;
6009 	case offsetof(struct sctphdr, checksum):
6010 		/* validate that this is actually an SCTP request */
6011 		if (skb_csum_is_sctp(skb)) {
6012 			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
6013 			break;
6014 		}
6015 		fallthrough;
6016 	default:
6017 		skb_checksum_help(skb);
6018 		goto csum_failed;
6019 	}
6020 
6021 	/* update TX checksum flag */
6022 	first->tx_flags |= IGB_TX_FLAGS_CSUM;
6023 	vlan_macip_lens = skb_checksum_start_offset(skb) -
6024 			  skb_network_offset(skb);
6025 no_csum:
6026 	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
6027 	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6028 
6029 	igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
6030 }
6031 
6032 #define IGB_SET_FLAG(_input, _flag, _result) \
6033 	((_flag <= _result) ? \
6034 	 ((u32)(_input & _flag) * (_result / _flag)) : \
6035 	 ((u32)(_input & _flag) / (_flag / _result)))
6036 
6037 static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
6038 {
6039 	/* set type for advanced descriptor with frame checksum insertion */
6040 	u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
6041 		       E1000_ADVTXD_DCMD_DEXT |
6042 		       E1000_ADVTXD_DCMD_IFCS;
6043 
6044 	/* set HW vlan bit if vlan is present */
6045 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
6046 				 (E1000_ADVTXD_DCMD_VLE));
6047 
6048 	/* set segmentation bits for TSO */
6049 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
6050 				 (E1000_ADVTXD_DCMD_TSE));
6051 
6052 	/* set timestamp bit if present */
6053 	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
6054 				 (E1000_ADVTXD_MAC_TSTAMP));
6055 
6056 	/* insert frame checksum */
6057 	cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
6058 
6059 	return cmd_type;
6060 }
6061 
6062 static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
6063 				 union e1000_adv_tx_desc *tx_desc,
6064 				 u32 tx_flags, unsigned int paylen)
6065 {
6066 	u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
6067 
6068 	/* 82575 requires a unique index per ring */
6069 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6070 		olinfo_status |= tx_ring->reg_idx << 4;
6071 
6072 	/* insert L4 checksum */
6073 	olinfo_status |= IGB_SET_FLAG(tx_flags,
6074 				      IGB_TX_FLAGS_CSUM,
6075 				      (E1000_TXD_POPTS_TXSM << 8));
6076 
6077 	/* insert IPv4 checksum */
6078 	olinfo_status |= IGB_SET_FLAG(tx_flags,
6079 				      IGB_TX_FLAGS_IPV4,
6080 				      (E1000_TXD_POPTS_IXSM << 8));
6081 
6082 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6083 }
6084 
6085 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6086 {
6087 	struct net_device *netdev = tx_ring->netdev;
6088 
6089 	netif_stop_subqueue(netdev, tx_ring->queue_index);
6090 
6091 	/* Herbert's original patch had:
6092 	 *  smp_mb__after_netif_stop_queue();
6093 	 * but since that doesn't exist yet, just open code it.
6094 	 */
6095 	smp_mb();
6096 
6097 	/* We need to check again in a case another CPU has just
6098 	 * made room available.
6099 	 */
6100 	if (igb_desc_unused(tx_ring) < size)
6101 		return -EBUSY;
6102 
6103 	/* A reprieve! */
6104 	netif_wake_subqueue(netdev, tx_ring->queue_index);
6105 
6106 	u64_stats_update_begin(&tx_ring->tx_syncp2);
6107 	tx_ring->tx_stats.restart_queue2++;
6108 	u64_stats_update_end(&tx_ring->tx_syncp2);
6109 
6110 	return 0;
6111 }
6112 
6113 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6114 {
6115 	if (igb_desc_unused(tx_ring) >= size)
6116 		return 0;
6117 	return __igb_maybe_stop_tx(tx_ring, size);
6118 }
6119 
6120 static int igb_tx_map(struct igb_ring *tx_ring,
6121 		      struct igb_tx_buffer *first,
6122 		      const u8 hdr_len)
6123 {
6124 	struct sk_buff *skb = first->skb;
6125 	struct igb_tx_buffer *tx_buffer;
6126 	union e1000_adv_tx_desc *tx_desc;
6127 	skb_frag_t *frag;
6128 	dma_addr_t dma;
6129 	unsigned int data_len, size;
6130 	u32 tx_flags = first->tx_flags;
6131 	u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
6132 	u16 i = tx_ring->next_to_use;
6133 
6134 	tx_desc = IGB_TX_DESC(tx_ring, i);
6135 
6136 	igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
6137 
6138 	size = skb_headlen(skb);
6139 	data_len = skb->data_len;
6140 
6141 	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
6142 
6143 	tx_buffer = first;
6144 
6145 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
6146 		if (dma_mapping_error(tx_ring->dev, dma))
6147 			goto dma_error;
6148 
6149 		/* record length, and DMA address */
6150 		dma_unmap_len_set(tx_buffer, len, size);
6151 		dma_unmap_addr_set(tx_buffer, dma, dma);
6152 
6153 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
6154 
6155 		while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
6156 			tx_desc->read.cmd_type_len =
6157 				cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
6158 
6159 			i++;
6160 			tx_desc++;
6161 			if (i == tx_ring->count) {
6162 				tx_desc = IGB_TX_DESC(tx_ring, 0);
6163 				i = 0;
6164 			}
6165 			tx_desc->read.olinfo_status = 0;
6166 
6167 			dma += IGB_MAX_DATA_PER_TXD;
6168 			size -= IGB_MAX_DATA_PER_TXD;
6169 
6170 			tx_desc->read.buffer_addr = cpu_to_le64(dma);
6171 		}
6172 
6173 		if (likely(!data_len))
6174 			break;
6175 
6176 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
6177 
6178 		i++;
6179 		tx_desc++;
6180 		if (i == tx_ring->count) {
6181 			tx_desc = IGB_TX_DESC(tx_ring, 0);
6182 			i = 0;
6183 		}
6184 		tx_desc->read.olinfo_status = 0;
6185 
6186 		size = skb_frag_size(frag);
6187 		data_len -= size;
6188 
6189 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
6190 				       size, DMA_TO_DEVICE);
6191 
6192 		tx_buffer = &tx_ring->tx_buffer_info[i];
6193 	}
6194 
6195 	/* write last descriptor with RS and EOP bits */
6196 	cmd_type |= size | IGB_TXD_DCMD;
6197 	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6198 
6199 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
6200 
6201 	/* set the timestamp */
6202 	first->time_stamp = jiffies;
6203 
6204 	skb_tx_timestamp(skb);
6205 
6206 	/* Force memory writes to complete before letting h/w know there
6207 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
6208 	 * memory model archs, such as IA-64).
6209 	 *
6210 	 * We also need this memory barrier to make certain all of the
6211 	 * status bits have been updated before next_to_watch is written.
6212 	 */
6213 	dma_wmb();
6214 
6215 	/* set next_to_watch value indicating a packet is present */
6216 	first->next_to_watch = tx_desc;
6217 
6218 	i++;
6219 	if (i == tx_ring->count)
6220 		i = 0;
6221 
6222 	tx_ring->next_to_use = i;
6223 
6224 	/* Make sure there is space in the ring for the next send. */
6225 	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6226 
6227 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
6228 		writel(i, tx_ring->tail);
6229 	}
6230 	return 0;
6231 
6232 dma_error:
6233 	dev_err(tx_ring->dev, "TX DMA map failed\n");
6234 	tx_buffer = &tx_ring->tx_buffer_info[i];
6235 
6236 	/* clear dma mappings for failed tx_buffer_info map */
6237 	while (tx_buffer != first) {
6238 		if (dma_unmap_len(tx_buffer, len))
6239 			dma_unmap_page(tx_ring->dev,
6240 				       dma_unmap_addr(tx_buffer, dma),
6241 				       dma_unmap_len(tx_buffer, len),
6242 				       DMA_TO_DEVICE);
6243 		dma_unmap_len_set(tx_buffer, len, 0);
6244 
6245 		if (i-- == 0)
6246 			i += tx_ring->count;
6247 		tx_buffer = &tx_ring->tx_buffer_info[i];
6248 	}
6249 
6250 	if (dma_unmap_len(tx_buffer, len))
6251 		dma_unmap_single(tx_ring->dev,
6252 				 dma_unmap_addr(tx_buffer, dma),
6253 				 dma_unmap_len(tx_buffer, len),
6254 				 DMA_TO_DEVICE);
6255 	dma_unmap_len_set(tx_buffer, len, 0);
6256 
6257 	dev_kfree_skb_any(tx_buffer->skb);
6258 	tx_buffer->skb = NULL;
6259 
6260 	tx_ring->next_to_use = i;
6261 
6262 	return -1;
6263 }
6264 
6265 int igb_xmit_xdp_ring(struct igb_adapter *adapter,
6266 		      struct igb_ring *tx_ring,
6267 		      struct xdp_frame *xdpf)
6268 {
6269 	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
6270 	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
6271 	u16 count, i, index = tx_ring->next_to_use;
6272 	struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index];
6273 	struct igb_tx_buffer *tx_buffer = tx_head;
6274 	union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index);
6275 	u32 len = xdpf->len, cmd_type, olinfo_status;
6276 	void *data = xdpf->data;
6277 
6278 	count = TXD_USE_COUNT(len);
6279 	for (i = 0; i < nr_frags; i++)
6280 		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
6281 
6282 	if (igb_maybe_stop_tx(tx_ring, count + 3))
6283 		return IGB_XDP_CONSUMED;
6284 
6285 	i = 0;
6286 	/* record the location of the first descriptor for this packet */
6287 	tx_head->bytecount = xdp_get_frame_len(xdpf);
6288 	tx_head->type = IGB_TYPE_XDP;
6289 	tx_head->gso_segs = 1;
6290 	tx_head->xdpf = xdpf;
6291 
6292 	olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT;
6293 	/* 82575 requires a unique index per ring */
6294 	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6295 		olinfo_status |= tx_ring->reg_idx << 4;
6296 	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6297 
6298 	for (;;) {
6299 		dma_addr_t dma;
6300 
6301 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
6302 		if (dma_mapping_error(tx_ring->dev, dma))
6303 			goto unmap;
6304 
6305 		/* record length, and DMA address */
6306 		dma_unmap_len_set(tx_buffer, len, len);
6307 		dma_unmap_addr_set(tx_buffer, dma, dma);
6308 
6309 		/* put descriptor type bits */
6310 		cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT |
6311 			   E1000_ADVTXD_DCMD_IFCS | len;
6312 
6313 		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6314 		tx_desc->read.buffer_addr = cpu_to_le64(dma);
6315 
6316 		tx_buffer->protocol = 0;
6317 
6318 		if (++index == tx_ring->count)
6319 			index = 0;
6320 
6321 		if (i == nr_frags)
6322 			break;
6323 
6324 		tx_buffer = &tx_ring->tx_buffer_info[index];
6325 		tx_desc = IGB_TX_DESC(tx_ring, index);
6326 		tx_desc->read.olinfo_status = 0;
6327 
6328 		data = skb_frag_address(&sinfo->frags[i]);
6329 		len = skb_frag_size(&sinfo->frags[i]);
6330 		i++;
6331 	}
6332 	tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD);
6333 
6334 	netdev_tx_sent_queue(txring_txq(tx_ring), tx_head->bytecount);
6335 	/* set the timestamp */
6336 	tx_head->time_stamp = jiffies;
6337 
6338 	/* Avoid any potential race with xdp_xmit and cleanup */
6339 	smp_wmb();
6340 
6341 	/* set next_to_watch value indicating a packet is present */
6342 	tx_head->next_to_watch = tx_desc;
6343 	tx_ring->next_to_use = index;
6344 
6345 	/* Make sure there is space in the ring for the next send. */
6346 	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6347 
6348 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more())
6349 		writel(index, tx_ring->tail);
6350 
6351 	return IGB_XDP_TX;
6352 
6353 unmap:
6354 	for (;;) {
6355 		tx_buffer = &tx_ring->tx_buffer_info[index];
6356 		if (dma_unmap_len(tx_buffer, len))
6357 			dma_unmap_page(tx_ring->dev,
6358 				       dma_unmap_addr(tx_buffer, dma),
6359 				       dma_unmap_len(tx_buffer, len),
6360 				       DMA_TO_DEVICE);
6361 		dma_unmap_len_set(tx_buffer, len, 0);
6362 		if (tx_buffer == tx_head)
6363 			break;
6364 
6365 		if (!index)
6366 			index += tx_ring->count;
6367 		index--;
6368 	}
6369 
6370 	return IGB_XDP_CONSUMED;
6371 }
6372 
6373 netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
6374 				struct igb_ring *tx_ring)
6375 {
6376 	struct igb_tx_buffer *first;
6377 	int tso;
6378 	u32 tx_flags = 0;
6379 	unsigned short f;
6380 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
6381 	__be16 protocol = vlan_get_protocol(skb);
6382 	u8 hdr_len = 0;
6383 
6384 	/* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
6385 	 *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
6386 	 *       + 2 desc gap to keep tail from touching head,
6387 	 *       + 1 desc for context descriptor,
6388 	 * otherwise try next time
6389 	 */
6390 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6391 		count += TXD_USE_COUNT(skb_frag_size(
6392 						&skb_shinfo(skb)->frags[f]));
6393 
6394 	if (igb_maybe_stop_tx(tx_ring, count + 3)) {
6395 		/* this is a hard error */
6396 		return NETDEV_TX_BUSY;
6397 	}
6398 
6399 	/* record the location of the first descriptor for this packet */
6400 	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6401 	first->type = IGB_TYPE_SKB;
6402 	first->skb = skb;
6403 	first->bytecount = skb->len;
6404 	first->gso_segs = 1;
6405 
6406 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
6407 		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6408 
6409 		if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
6410 		    !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
6411 					   &adapter->state)) {
6412 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
6413 			tx_flags |= IGB_TX_FLAGS_TSTAMP;
6414 
6415 			adapter->ptp_tx_skb = skb_get(skb);
6416 			adapter->ptp_tx_start = jiffies;
6417 			if (adapter->hw.mac.type == e1000_82576)
6418 				schedule_work(&adapter->ptp_tx_work);
6419 		} else {
6420 			adapter->tx_hwtstamp_skipped++;
6421 		}
6422 	}
6423 
6424 	if (skb_vlan_tag_present(skb)) {
6425 		tx_flags |= IGB_TX_FLAGS_VLAN;
6426 		tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
6427 	}
6428 
6429 	/* record initial flags and protocol */
6430 	first->tx_flags = tx_flags;
6431 	first->protocol = protocol;
6432 
6433 	tso = igb_tso(tx_ring, first, &hdr_len);
6434 	if (tso < 0)
6435 		goto out_drop;
6436 	else if (!tso)
6437 		igb_tx_csum(tx_ring, first);
6438 
6439 	if (igb_tx_map(tx_ring, first, hdr_len))
6440 		goto cleanup_tx_tstamp;
6441 
6442 	return NETDEV_TX_OK;
6443 
6444 out_drop:
6445 	dev_kfree_skb_any(first->skb);
6446 	first->skb = NULL;
6447 cleanup_tx_tstamp:
6448 	if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
6449 		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6450 
6451 		dev_kfree_skb_any(adapter->ptp_tx_skb);
6452 		adapter->ptp_tx_skb = NULL;
6453 		if (adapter->hw.mac.type == e1000_82576)
6454 			cancel_work_sync(&adapter->ptp_tx_work);
6455 		clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
6456 	}
6457 
6458 	return NETDEV_TX_OK;
6459 }
6460 
6461 static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
6462 						    struct sk_buff *skb)
6463 {
6464 	unsigned int r_idx = skb->queue_mapping;
6465 
6466 	if (r_idx >= adapter->num_tx_queues)
6467 		r_idx = r_idx % adapter->num_tx_queues;
6468 
6469 	return adapter->tx_ring[r_idx];
6470 }
6471 
6472 static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
6473 				  struct net_device *netdev)
6474 {
6475 	struct igb_adapter *adapter = netdev_priv(netdev);
6476 
6477 	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
6478 	 * in order to meet this minimum size requirement.
6479 	 */
6480 	if (skb_put_padto(skb, 17))
6481 		return NETDEV_TX_OK;
6482 
6483 	return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
6484 }
6485 
6486 /**
6487  *  igb_tx_timeout - Respond to a Tx Hang
6488  *  @netdev: network interface device structure
6489  *  @txqueue: number of the Tx queue that hung (unused)
6490  **/
6491 static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6492 {
6493 	struct igb_adapter *adapter = netdev_priv(netdev);
6494 	struct e1000_hw *hw = &adapter->hw;
6495 
6496 	/* Do the reset outside of interrupt context */
6497 	adapter->tx_timeout_count++;
6498 
6499 	if (hw->mac.type >= e1000_82580)
6500 		hw->dev_spec._82575.global_device_reset = true;
6501 
6502 	schedule_work(&adapter->reset_task);
6503 	wr32(E1000_EICS,
6504 	     (adapter->eims_enable_mask & ~adapter->eims_other));
6505 }
6506 
6507 static void igb_reset_task(struct work_struct *work)
6508 {
6509 	struct igb_adapter *adapter;
6510 	adapter = container_of(work, struct igb_adapter, reset_task);
6511 
6512 	rtnl_lock();
6513 	/* If we're already down or resetting, just bail */
6514 	if (test_bit(__IGB_DOWN, &adapter->state) ||
6515 	    test_bit(__IGB_RESETTING, &adapter->state)) {
6516 		rtnl_unlock();
6517 		return;
6518 	}
6519 
6520 	igb_dump(adapter);
6521 	netdev_err(adapter->netdev, "Reset adapter\n");
6522 	igb_reinit_locked(adapter);
6523 	rtnl_unlock();
6524 }
6525 
6526 /**
6527  *  igb_get_stats64 - Get System Network Statistics
6528  *  @netdev: network interface device structure
6529  *  @stats: rtnl_link_stats64 pointer
6530  **/
6531 static void igb_get_stats64(struct net_device *netdev,
6532 			    struct rtnl_link_stats64 *stats)
6533 {
6534 	struct igb_adapter *adapter = netdev_priv(netdev);
6535 
6536 	spin_lock(&adapter->stats64_lock);
6537 	igb_update_stats(adapter);
6538 	memcpy(stats, &adapter->stats64, sizeof(*stats));
6539 	spin_unlock(&adapter->stats64_lock);
6540 }
6541 
6542 /**
6543  *  igb_change_mtu - Change the Maximum Transfer Unit
6544  *  @netdev: network interface device structure
6545  *  @new_mtu: new value for maximum frame size
6546  *
6547  *  Returns 0 on success, negative on failure
6548  **/
6549 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
6550 {
6551 	struct igb_adapter *adapter = netdev_priv(netdev);
6552 	int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD;
6553 
6554 	if (adapter->xdp_prog) {
6555 		int i;
6556 
6557 		for (i = 0; i < adapter->num_rx_queues; i++) {
6558 			struct igb_ring *ring = adapter->rx_ring[i];
6559 
6560 			if (max_frame > igb_rx_bufsz(ring)) {
6561 				netdev_warn(adapter->netdev,
6562 					    "Requested MTU size is not supported with XDP. Max frame size is %d\n",
6563 					    max_frame);
6564 				return -EINVAL;
6565 			}
6566 		}
6567 	}
6568 
6569 	/* adjust max frame to be at least the size of a standard frame */
6570 	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
6571 		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
6572 
6573 	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
6574 		usleep_range(1000, 2000);
6575 
6576 	/* igb_down has a dependency on max_frame_size */
6577 	adapter->max_frame_size = max_frame;
6578 
6579 	if (netif_running(netdev))
6580 		igb_down(adapter);
6581 
6582 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
6583 		   netdev->mtu, new_mtu);
6584 	netdev->mtu = new_mtu;
6585 
6586 	if (netif_running(netdev))
6587 		igb_up(adapter);
6588 	else
6589 		igb_reset(adapter);
6590 
6591 	clear_bit(__IGB_RESETTING, &adapter->state);
6592 
6593 	return 0;
6594 }
6595 
6596 /**
6597  *  igb_update_stats - Update the board statistics counters
6598  *  @adapter: board private structure
6599  **/
6600 void igb_update_stats(struct igb_adapter *adapter)
6601 {
6602 	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
6603 	struct e1000_hw *hw = &adapter->hw;
6604 	struct pci_dev *pdev = adapter->pdev;
6605 	u32 reg, mpc;
6606 	int i;
6607 	u64 bytes, packets;
6608 	unsigned int start;
6609 	u64 _bytes, _packets;
6610 
6611 	/* Prevent stats update while adapter is being reset, or if the pci
6612 	 * connection is down.
6613 	 */
6614 	if (adapter->link_speed == 0)
6615 		return;
6616 	if (pci_channel_offline(pdev))
6617 		return;
6618 
6619 	bytes = 0;
6620 	packets = 0;
6621 
6622 	rcu_read_lock();
6623 	for (i = 0; i < adapter->num_rx_queues; i++) {
6624 		struct igb_ring *ring = adapter->rx_ring[i];
6625 		u32 rqdpc = rd32(E1000_RQDPC(i));
6626 		if (hw->mac.type >= e1000_i210)
6627 			wr32(E1000_RQDPC(i), 0);
6628 
6629 		if (rqdpc) {
6630 			ring->rx_stats.drops += rqdpc;
6631 			net_stats->rx_fifo_errors += rqdpc;
6632 		}
6633 
6634 		do {
6635 			start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
6636 			_bytes = ring->rx_stats.bytes;
6637 			_packets = ring->rx_stats.packets;
6638 		} while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
6639 		bytes += _bytes;
6640 		packets += _packets;
6641 	}
6642 
6643 	net_stats->rx_bytes = bytes;
6644 	net_stats->rx_packets = packets;
6645 
6646 	bytes = 0;
6647 	packets = 0;
6648 	for (i = 0; i < adapter->num_tx_queues; i++) {
6649 		struct igb_ring *ring = adapter->tx_ring[i];
6650 		do {
6651 			start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
6652 			_bytes = ring->tx_stats.bytes;
6653 			_packets = ring->tx_stats.packets;
6654 		} while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
6655 		bytes += _bytes;
6656 		packets += _packets;
6657 	}
6658 	net_stats->tx_bytes = bytes;
6659 	net_stats->tx_packets = packets;
6660 	rcu_read_unlock();
6661 
6662 	/* read stats registers */
6663 	adapter->stats.crcerrs += rd32(E1000_CRCERRS);
6664 	adapter->stats.gprc += rd32(E1000_GPRC);
6665 	adapter->stats.gorc += rd32(E1000_GORCL);
6666 	rd32(E1000_GORCH); /* clear GORCL */
6667 	adapter->stats.bprc += rd32(E1000_BPRC);
6668 	adapter->stats.mprc += rd32(E1000_MPRC);
6669 	adapter->stats.roc += rd32(E1000_ROC);
6670 
6671 	adapter->stats.prc64 += rd32(E1000_PRC64);
6672 	adapter->stats.prc127 += rd32(E1000_PRC127);
6673 	adapter->stats.prc255 += rd32(E1000_PRC255);
6674 	adapter->stats.prc511 += rd32(E1000_PRC511);
6675 	adapter->stats.prc1023 += rd32(E1000_PRC1023);
6676 	adapter->stats.prc1522 += rd32(E1000_PRC1522);
6677 	adapter->stats.symerrs += rd32(E1000_SYMERRS);
6678 	adapter->stats.sec += rd32(E1000_SEC);
6679 
6680 	mpc = rd32(E1000_MPC);
6681 	adapter->stats.mpc += mpc;
6682 	net_stats->rx_fifo_errors += mpc;
6683 	adapter->stats.scc += rd32(E1000_SCC);
6684 	adapter->stats.ecol += rd32(E1000_ECOL);
6685 	adapter->stats.mcc += rd32(E1000_MCC);
6686 	adapter->stats.latecol += rd32(E1000_LATECOL);
6687 	adapter->stats.dc += rd32(E1000_DC);
6688 	adapter->stats.rlec += rd32(E1000_RLEC);
6689 	adapter->stats.xonrxc += rd32(E1000_XONRXC);
6690 	adapter->stats.xontxc += rd32(E1000_XONTXC);
6691 	adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
6692 	adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
6693 	adapter->stats.fcruc += rd32(E1000_FCRUC);
6694 	adapter->stats.gptc += rd32(E1000_GPTC);
6695 	adapter->stats.gotc += rd32(E1000_GOTCL);
6696 	rd32(E1000_GOTCH); /* clear GOTCL */
6697 	adapter->stats.rnbc += rd32(E1000_RNBC);
6698 	adapter->stats.ruc += rd32(E1000_RUC);
6699 	adapter->stats.rfc += rd32(E1000_RFC);
6700 	adapter->stats.rjc += rd32(E1000_RJC);
6701 	adapter->stats.tor += rd32(E1000_TORH);
6702 	adapter->stats.tot += rd32(E1000_TOTH);
6703 	adapter->stats.tpr += rd32(E1000_TPR);
6704 
6705 	adapter->stats.ptc64 += rd32(E1000_PTC64);
6706 	adapter->stats.ptc127 += rd32(E1000_PTC127);
6707 	adapter->stats.ptc255 += rd32(E1000_PTC255);
6708 	adapter->stats.ptc511 += rd32(E1000_PTC511);
6709 	adapter->stats.ptc1023 += rd32(E1000_PTC1023);
6710 	adapter->stats.ptc1522 += rd32(E1000_PTC1522);
6711 
6712 	adapter->stats.mptc += rd32(E1000_MPTC);
6713 	adapter->stats.bptc += rd32(E1000_BPTC);
6714 
6715 	adapter->stats.tpt += rd32(E1000_TPT);
6716 	adapter->stats.colc += rd32(E1000_COLC);
6717 
6718 	adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
6719 	/* read internal phy specific stats */
6720 	reg = rd32(E1000_CTRL_EXT);
6721 	if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
6722 		adapter->stats.rxerrc += rd32(E1000_RXERRC);
6723 
6724 		/* this stat has invalid values on i210/i211 */
6725 		if ((hw->mac.type != e1000_i210) &&
6726 		    (hw->mac.type != e1000_i211))
6727 			adapter->stats.tncrs += rd32(E1000_TNCRS);
6728 	}
6729 
6730 	adapter->stats.tsctc += rd32(E1000_TSCTC);
6731 	adapter->stats.tsctfc += rd32(E1000_TSCTFC);
6732 
6733 	adapter->stats.iac += rd32(E1000_IAC);
6734 	adapter->stats.icrxoc += rd32(E1000_ICRXOC);
6735 	adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
6736 	adapter->stats.icrxatc += rd32(E1000_ICRXATC);
6737 	adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
6738 	adapter->stats.ictxatc += rd32(E1000_ICTXATC);
6739 	adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
6740 	adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
6741 	adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
6742 
6743 	/* Fill out the OS statistics structure */
6744 	net_stats->multicast = adapter->stats.mprc;
6745 	net_stats->collisions = adapter->stats.colc;
6746 
6747 	/* Rx Errors */
6748 
6749 	/* RLEC on some newer hardware can be incorrect so build
6750 	 * our own version based on RUC and ROC
6751 	 */
6752 	net_stats->rx_errors = adapter->stats.rxerrc +
6753 		adapter->stats.crcerrs + adapter->stats.algnerrc +
6754 		adapter->stats.ruc + adapter->stats.roc +
6755 		adapter->stats.cexterr;
6756 	net_stats->rx_length_errors = adapter->stats.ruc +
6757 				      adapter->stats.roc;
6758 	net_stats->rx_crc_errors = adapter->stats.crcerrs;
6759 	net_stats->rx_frame_errors = adapter->stats.algnerrc;
6760 	net_stats->rx_missed_errors = adapter->stats.mpc;
6761 
6762 	/* Tx Errors */
6763 	net_stats->tx_errors = adapter->stats.ecol +
6764 			       adapter->stats.latecol;
6765 	net_stats->tx_aborted_errors = adapter->stats.ecol;
6766 	net_stats->tx_window_errors = adapter->stats.latecol;
6767 	net_stats->tx_carrier_errors = adapter->stats.tncrs;
6768 
6769 	/* Tx Dropped needs to be maintained elsewhere */
6770 
6771 	/* Management Stats */
6772 	adapter->stats.mgptc += rd32(E1000_MGTPTC);
6773 	adapter->stats.mgprc += rd32(E1000_MGTPRC);
6774 	adapter->stats.mgpdc += rd32(E1000_MGTPDC);
6775 
6776 	/* OS2BMC Stats */
6777 	reg = rd32(E1000_MANC);
6778 	if (reg & E1000_MANC_EN_BMC2OS) {
6779 		adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
6780 		adapter->stats.o2bspc += rd32(E1000_O2BSPC);
6781 		adapter->stats.b2ospc += rd32(E1000_B2OSPC);
6782 		adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
6783 	}
6784 }
6785 
6786 static void igb_perout(struct igb_adapter *adapter, int tsintr_tt)
6787 {
6788 	int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_PEROUT, tsintr_tt);
6789 	struct e1000_hw *hw = &adapter->hw;
6790 	struct timespec64 ts;
6791 	u32 tsauxc;
6792 
6793 	if (pin < 0 || pin >= IGB_N_PEROUT)
6794 		return;
6795 
6796 	spin_lock(&adapter->tmreg_lock);
6797 
6798 	if (hw->mac.type == e1000_82580 ||
6799 	    hw->mac.type == e1000_i354 ||
6800 	    hw->mac.type == e1000_i350) {
6801 		s64 ns = timespec64_to_ns(&adapter->perout[pin].period);
6802 		u32 systiml, systimh, level_mask, level, rem;
6803 		u64 systim, now;
6804 
6805 		/* read systim registers in sequence */
6806 		rd32(E1000_SYSTIMR);
6807 		systiml = rd32(E1000_SYSTIML);
6808 		systimh = rd32(E1000_SYSTIMH);
6809 		systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
6810 		now = timecounter_cyc2time(&adapter->tc, systim);
6811 
6812 		if (pin < 2) {
6813 			level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000;
6814 			level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
6815 		} else {
6816 			level_mask = (tsintr_tt == 1) ? 0x80 : 0x40;
6817 			level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
6818 		}
6819 
6820 		div_u64_rem(now, ns, &rem);
6821 		systim = systim + (ns - rem);
6822 
6823 		/* synchronize pin level with rising/falling edges */
6824 		div_u64_rem(now, ns << 1, &rem);
6825 		if (rem < ns) {
6826 			/* first half of period */
6827 			if (level == 0) {
6828 				/* output is already low, skip this period */
6829 				systim += ns;
6830 				pr_notice("igb: periodic output on %s missed falling edge\n",
6831 					  adapter->sdp_config[pin].name);
6832 			}
6833 		} else {
6834 			/* second half of period */
6835 			if (level == 1) {
6836 				/* output is already high, skip this period */
6837 				systim += ns;
6838 				pr_notice("igb: periodic output on %s missed rising edge\n",
6839 					  adapter->sdp_config[pin].name);
6840 			}
6841 		}
6842 
6843 		/* for this chip family tv_sec is the upper part of the binary value,
6844 		 * so not seconds
6845 		 */
6846 		ts.tv_nsec = (u32)systim;
6847 		ts.tv_sec  = ((u32)(systim >> 32)) & 0xFF;
6848 	} else {
6849 		ts = timespec64_add(adapter->perout[pin].start,
6850 				    adapter->perout[pin].period);
6851 	}
6852 
6853 	/* u32 conversion of tv_sec is safe until y2106 */
6854 	wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec);
6855 	wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec);
6856 	tsauxc = rd32(E1000_TSAUXC);
6857 	tsauxc |= TSAUXC_EN_TT0;
6858 	wr32(E1000_TSAUXC, tsauxc);
6859 	adapter->perout[pin].start = ts;
6860 
6861 	spin_unlock(&adapter->tmreg_lock);
6862 }
6863 
6864 static void igb_extts(struct igb_adapter *adapter, int tsintr_tt)
6865 {
6866 	int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_EXTTS, tsintr_tt);
6867 	int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0;
6868 	int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0;
6869 	struct e1000_hw *hw = &adapter->hw;
6870 	struct ptp_clock_event event;
6871 	struct timespec64 ts;
6872 
6873 	if (pin < 0 || pin >= IGB_N_EXTTS)
6874 		return;
6875 
6876 	if (hw->mac.type == e1000_82580 ||
6877 	    hw->mac.type == e1000_i354 ||
6878 	    hw->mac.type == e1000_i350) {
6879 		s64 ns = rd32(auxstmpl);
6880 
6881 		ns += ((s64)(rd32(auxstmph) & 0xFF)) << 32;
6882 		ts = ns_to_timespec64(ns);
6883 	} else {
6884 		ts.tv_nsec = rd32(auxstmpl);
6885 		ts.tv_sec  = rd32(auxstmph);
6886 	}
6887 
6888 	event.type = PTP_CLOCK_EXTTS;
6889 	event.index = tsintr_tt;
6890 	event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
6891 	ptp_clock_event(adapter->ptp_clock, &event);
6892 }
6893 
6894 static void igb_tsync_interrupt(struct igb_adapter *adapter)
6895 {
6896 	struct e1000_hw *hw = &adapter->hw;
6897 	u32 ack = 0, tsicr = rd32(E1000_TSICR);
6898 	struct ptp_clock_event event;
6899 
6900 	if (tsicr & TSINTR_SYS_WRAP) {
6901 		event.type = PTP_CLOCK_PPS;
6902 		if (adapter->ptp_caps.pps)
6903 			ptp_clock_event(adapter->ptp_clock, &event);
6904 		ack |= TSINTR_SYS_WRAP;
6905 	}
6906 
6907 	if (tsicr & E1000_TSICR_TXTS) {
6908 		/* retrieve hardware timestamp */
6909 		schedule_work(&adapter->ptp_tx_work);
6910 		ack |= E1000_TSICR_TXTS;
6911 	}
6912 
6913 	if (tsicr & TSINTR_TT0) {
6914 		igb_perout(adapter, 0);
6915 		ack |= TSINTR_TT0;
6916 	}
6917 
6918 	if (tsicr & TSINTR_TT1) {
6919 		igb_perout(adapter, 1);
6920 		ack |= TSINTR_TT1;
6921 	}
6922 
6923 	if (tsicr & TSINTR_AUTT0) {
6924 		igb_extts(adapter, 0);
6925 		ack |= TSINTR_AUTT0;
6926 	}
6927 
6928 	if (tsicr & TSINTR_AUTT1) {
6929 		igb_extts(adapter, 1);
6930 		ack |= TSINTR_AUTT1;
6931 	}
6932 
6933 	/* acknowledge the interrupts */
6934 	wr32(E1000_TSICR, ack);
6935 }
6936 
6937 static irqreturn_t igb_msix_other(int irq, void *data)
6938 {
6939 	struct igb_adapter *adapter = data;
6940 	struct e1000_hw *hw = &adapter->hw;
6941 	u32 icr = rd32(E1000_ICR);
6942 	/* reading ICR causes bit 31 of EICR to be cleared */
6943 
6944 	if (icr & E1000_ICR_DRSTA)
6945 		schedule_work(&adapter->reset_task);
6946 
6947 	if (icr & E1000_ICR_DOUTSYNC) {
6948 		/* HW is reporting DMA is out of sync */
6949 		adapter->stats.doosync++;
6950 		/* The DMA Out of Sync is also indication of a spoof event
6951 		 * in IOV mode. Check the Wrong VM Behavior register to
6952 		 * see if it is really a spoof event.
6953 		 */
6954 		igb_check_wvbr(adapter);
6955 	}
6956 
6957 	/* Check for a mailbox event */
6958 	if (icr & E1000_ICR_VMMB)
6959 		igb_msg_task(adapter);
6960 
6961 	if (icr & E1000_ICR_LSC) {
6962 		hw->mac.get_link_status = 1;
6963 		/* guard against interrupt when we're going down */
6964 		if (!test_bit(__IGB_DOWN, &adapter->state))
6965 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
6966 	}
6967 
6968 	if (icr & E1000_ICR_TS)
6969 		igb_tsync_interrupt(adapter);
6970 
6971 	wr32(E1000_EIMS, adapter->eims_other);
6972 
6973 	return IRQ_HANDLED;
6974 }
6975 
6976 static void igb_write_itr(struct igb_q_vector *q_vector)
6977 {
6978 	struct igb_adapter *adapter = q_vector->adapter;
6979 	u32 itr_val = q_vector->itr_val & 0x7FFC;
6980 
6981 	if (!q_vector->set_itr)
6982 		return;
6983 
6984 	if (!itr_val)
6985 		itr_val = 0x4;
6986 
6987 	if (adapter->hw.mac.type == e1000_82575)
6988 		itr_val |= itr_val << 16;
6989 	else
6990 		itr_val |= E1000_EITR_CNT_IGNR;
6991 
6992 	writel(itr_val, q_vector->itr_register);
6993 	q_vector->set_itr = 0;
6994 }
6995 
6996 static irqreturn_t igb_msix_ring(int irq, void *data)
6997 {
6998 	struct igb_q_vector *q_vector = data;
6999 
7000 	/* Write the ITR value calculated from the previous interrupt. */
7001 	igb_write_itr(q_vector);
7002 
7003 	napi_schedule(&q_vector->napi);
7004 
7005 	return IRQ_HANDLED;
7006 }
7007 
7008 #ifdef CONFIG_IGB_DCA
7009 static void igb_update_tx_dca(struct igb_adapter *adapter,
7010 			      struct igb_ring *tx_ring,
7011 			      int cpu)
7012 {
7013 	struct e1000_hw *hw = &adapter->hw;
7014 	u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
7015 
7016 	if (hw->mac.type != e1000_82575)
7017 		txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
7018 
7019 	/* We can enable relaxed ordering for reads, but not writes when
7020 	 * DCA is enabled.  This is due to a known issue in some chipsets
7021 	 * which will cause the DCA tag to be cleared.
7022 	 */
7023 	txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
7024 		  E1000_DCA_TXCTRL_DATA_RRO_EN |
7025 		  E1000_DCA_TXCTRL_DESC_DCA_EN;
7026 
7027 	wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
7028 }
7029 
7030 static void igb_update_rx_dca(struct igb_adapter *adapter,
7031 			      struct igb_ring *rx_ring,
7032 			      int cpu)
7033 {
7034 	struct e1000_hw *hw = &adapter->hw;
7035 	u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
7036 
7037 	if (hw->mac.type != e1000_82575)
7038 		rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
7039 
7040 	/* We can enable relaxed ordering for reads, but not writes when
7041 	 * DCA is enabled.  This is due to a known issue in some chipsets
7042 	 * which will cause the DCA tag to be cleared.
7043 	 */
7044 	rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
7045 		  E1000_DCA_RXCTRL_DESC_DCA_EN;
7046 
7047 	wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
7048 }
7049 
7050 static void igb_update_dca(struct igb_q_vector *q_vector)
7051 {
7052 	struct igb_adapter *adapter = q_vector->adapter;
7053 	int cpu = get_cpu();
7054 
7055 	if (q_vector->cpu == cpu)
7056 		goto out_no_update;
7057 
7058 	if (q_vector->tx.ring)
7059 		igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
7060 
7061 	if (q_vector->rx.ring)
7062 		igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
7063 
7064 	q_vector->cpu = cpu;
7065 out_no_update:
7066 	put_cpu();
7067 }
7068 
7069 static void igb_setup_dca(struct igb_adapter *adapter)
7070 {
7071 	struct e1000_hw *hw = &adapter->hw;
7072 	int i;
7073 
7074 	if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
7075 		return;
7076 
7077 	/* Always use CB2 mode, difference is masked in the CB driver. */
7078 	wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
7079 
7080 	for (i = 0; i < adapter->num_q_vectors; i++) {
7081 		adapter->q_vector[i]->cpu = -1;
7082 		igb_update_dca(adapter->q_vector[i]);
7083 	}
7084 }
7085 
7086 static int __igb_notify_dca(struct device *dev, void *data)
7087 {
7088 	struct net_device *netdev = dev_get_drvdata(dev);
7089 	struct igb_adapter *adapter = netdev_priv(netdev);
7090 	struct pci_dev *pdev = adapter->pdev;
7091 	struct e1000_hw *hw = &adapter->hw;
7092 	unsigned long event = *(unsigned long *)data;
7093 
7094 	switch (event) {
7095 	case DCA_PROVIDER_ADD:
7096 		/* if already enabled, don't do it again */
7097 		if (adapter->flags & IGB_FLAG_DCA_ENABLED)
7098 			break;
7099 		if (dca_add_requester(dev) == 0) {
7100 			adapter->flags |= IGB_FLAG_DCA_ENABLED;
7101 			dev_info(&pdev->dev, "DCA enabled\n");
7102 			igb_setup_dca(adapter);
7103 			break;
7104 		}
7105 		fallthrough; /* since DCA is disabled. */
7106 	case DCA_PROVIDER_REMOVE:
7107 		if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
7108 			/* without this a class_device is left
7109 			 * hanging around in the sysfs model
7110 			 */
7111 			dca_remove_requester(dev);
7112 			dev_info(&pdev->dev, "DCA disabled\n");
7113 			adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
7114 			wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
7115 		}
7116 		break;
7117 	}
7118 
7119 	return 0;
7120 }
7121 
7122 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
7123 			  void *p)
7124 {
7125 	int ret_val;
7126 
7127 	ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
7128 					 __igb_notify_dca);
7129 
7130 	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7131 }
7132 #endif /* CONFIG_IGB_DCA */
7133 
7134 #ifdef CONFIG_PCI_IOV
7135 static int igb_vf_configure(struct igb_adapter *adapter, int vf)
7136 {
7137 	unsigned char mac_addr[ETH_ALEN];
7138 
7139 	eth_zero_addr(mac_addr);
7140 	igb_set_vf_mac(adapter, vf, mac_addr);
7141 
7142 	/* By default spoof check is enabled for all VFs */
7143 	adapter->vf_data[vf].spoofchk_enabled = true;
7144 
7145 	/* By default VFs are not trusted */
7146 	adapter->vf_data[vf].trusted = false;
7147 
7148 	return 0;
7149 }
7150 
7151 #endif
7152 static void igb_ping_all_vfs(struct igb_adapter *adapter)
7153 {
7154 	struct e1000_hw *hw = &adapter->hw;
7155 	u32 ping;
7156 	int i;
7157 
7158 	for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
7159 		ping = E1000_PF_CONTROL_MSG;
7160 		if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
7161 			ping |= E1000_VT_MSGTYPE_CTS;
7162 		igb_write_mbx(hw, &ping, 1, i);
7163 	}
7164 }
7165 
7166 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7167 {
7168 	struct e1000_hw *hw = &adapter->hw;
7169 	u32 vmolr = rd32(E1000_VMOLR(vf));
7170 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7171 
7172 	vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7173 			    IGB_VF_FLAG_MULTI_PROMISC);
7174 	vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7175 
7176 	if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
7177 		vmolr |= E1000_VMOLR_MPME;
7178 		vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7179 		*msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
7180 	} else {
7181 		/* if we have hashes and we are clearing a multicast promisc
7182 		 * flag we need to write the hashes to the MTA as this step
7183 		 * was previously skipped
7184 		 */
7185 		if (vf_data->num_vf_mc_hashes > 30) {
7186 			vmolr |= E1000_VMOLR_MPME;
7187 		} else if (vf_data->num_vf_mc_hashes) {
7188 			int j;
7189 
7190 			vmolr |= E1000_VMOLR_ROMPE;
7191 			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7192 				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7193 		}
7194 	}
7195 
7196 	wr32(E1000_VMOLR(vf), vmolr);
7197 
7198 	/* there are flags left unprocessed, likely not supported */
7199 	if (*msgbuf & E1000_VT_MSGINFO_MASK)
7200 		return -EINVAL;
7201 
7202 	return 0;
7203 }
7204 
7205 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
7206 				  u32 *msgbuf, u32 vf)
7207 {
7208 	int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
7209 	u16 *hash_list = (u16 *)&msgbuf[1];
7210 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7211 	int i;
7212 
7213 	/* salt away the number of multicast addresses assigned
7214 	 * to this VF for later use to restore when the PF multi cast
7215 	 * list changes
7216 	 */
7217 	vf_data->num_vf_mc_hashes = n;
7218 
7219 	/* only up to 30 hash values supported */
7220 	if (n > 30)
7221 		n = 30;
7222 
7223 	/* store the hashes for later use */
7224 	for (i = 0; i < n; i++)
7225 		vf_data->vf_mc_hashes[i] = hash_list[i];
7226 
7227 	/* Flush and reset the mta with the new values */
7228 	igb_set_rx_mode(adapter->netdev);
7229 
7230 	return 0;
7231 }
7232 
7233 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
7234 {
7235 	struct e1000_hw *hw = &adapter->hw;
7236 	struct vf_data_storage *vf_data;
7237 	int i, j;
7238 
7239 	for (i = 0; i < adapter->vfs_allocated_count; i++) {
7240 		u32 vmolr = rd32(E1000_VMOLR(i));
7241 
7242 		vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7243 
7244 		vf_data = &adapter->vf_data[i];
7245 
7246 		if ((vf_data->num_vf_mc_hashes > 30) ||
7247 		    (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
7248 			vmolr |= E1000_VMOLR_MPME;
7249 		} else if (vf_data->num_vf_mc_hashes) {
7250 			vmolr |= E1000_VMOLR_ROMPE;
7251 			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7252 				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7253 		}
7254 		wr32(E1000_VMOLR(i), vmolr);
7255 	}
7256 }
7257 
7258 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
7259 {
7260 	struct e1000_hw *hw = &adapter->hw;
7261 	u32 pool_mask, vlvf_mask, i;
7262 
7263 	/* create mask for VF and other pools */
7264 	pool_mask = E1000_VLVF_POOLSEL_MASK;
7265 	vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
7266 
7267 	/* drop PF from pool bits */
7268 	pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
7269 			     adapter->vfs_allocated_count);
7270 
7271 	/* Find the vlan filter for this id */
7272 	for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
7273 		u32 vlvf = rd32(E1000_VLVF(i));
7274 		u32 vfta_mask, vid, vfta;
7275 
7276 		/* remove the vf from the pool */
7277 		if (!(vlvf & vlvf_mask))
7278 			continue;
7279 
7280 		/* clear out bit from VLVF */
7281 		vlvf ^= vlvf_mask;
7282 
7283 		/* if other pools are present, just remove ourselves */
7284 		if (vlvf & pool_mask)
7285 			goto update_vlvfb;
7286 
7287 		/* if PF is present, leave VFTA */
7288 		if (vlvf & E1000_VLVF_POOLSEL_MASK)
7289 			goto update_vlvf;
7290 
7291 		vid = vlvf & E1000_VLVF_VLANID_MASK;
7292 		vfta_mask = BIT(vid % 32);
7293 
7294 		/* clear bit from VFTA */
7295 		vfta = adapter->shadow_vfta[vid / 32];
7296 		if (vfta & vfta_mask)
7297 			hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
7298 update_vlvf:
7299 		/* clear pool selection enable */
7300 		if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7301 			vlvf &= E1000_VLVF_POOLSEL_MASK;
7302 		else
7303 			vlvf = 0;
7304 update_vlvfb:
7305 		/* clear pool bits */
7306 		wr32(E1000_VLVF(i), vlvf);
7307 	}
7308 }
7309 
7310 static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
7311 {
7312 	u32 vlvf;
7313 	int idx;
7314 
7315 	/* short cut the special case */
7316 	if (vlan == 0)
7317 		return 0;
7318 
7319 	/* Search for the VLAN id in the VLVF entries */
7320 	for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
7321 		vlvf = rd32(E1000_VLVF(idx));
7322 		if ((vlvf & VLAN_VID_MASK) == vlan)
7323 			break;
7324 	}
7325 
7326 	return idx;
7327 }
7328 
7329 static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
7330 {
7331 	struct e1000_hw *hw = &adapter->hw;
7332 	u32 bits, pf_id;
7333 	int idx;
7334 
7335 	idx = igb_find_vlvf_entry(hw, vid);
7336 	if (!idx)
7337 		return;
7338 
7339 	/* See if any other pools are set for this VLAN filter
7340 	 * entry other than the PF.
7341 	 */
7342 	pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
7343 	bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
7344 	bits &= rd32(E1000_VLVF(idx));
7345 
7346 	/* Disable the filter so this falls into the default pool. */
7347 	if (!bits) {
7348 		if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7349 			wr32(E1000_VLVF(idx), BIT(pf_id));
7350 		else
7351 			wr32(E1000_VLVF(idx), 0);
7352 	}
7353 }
7354 
7355 static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
7356 			   bool add, u32 vf)
7357 {
7358 	int pf_id = adapter->vfs_allocated_count;
7359 	struct e1000_hw *hw = &adapter->hw;
7360 	int err;
7361 
7362 	/* If VLAN overlaps with one the PF is currently monitoring make
7363 	 * sure that we are able to allocate a VLVF entry.  This may be
7364 	 * redundant but it guarantees PF will maintain visibility to
7365 	 * the VLAN.
7366 	 */
7367 	if (add && test_bit(vid, adapter->active_vlans)) {
7368 		err = igb_vfta_set(hw, vid, pf_id, true, false);
7369 		if (err)
7370 			return err;
7371 	}
7372 
7373 	err = igb_vfta_set(hw, vid, vf, add, false);
7374 
7375 	if (add && !err)
7376 		return err;
7377 
7378 	/* If we failed to add the VF VLAN or we are removing the VF VLAN
7379 	 * we may need to drop the PF pool bit in order to allow us to free
7380 	 * up the VLVF resources.
7381 	 */
7382 	if (test_bit(vid, adapter->active_vlans) ||
7383 	    (adapter->flags & IGB_FLAG_VLAN_PROMISC))
7384 		igb_update_pf_vlvf(adapter, vid);
7385 
7386 	return err;
7387 }
7388 
7389 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
7390 {
7391 	struct e1000_hw *hw = &adapter->hw;
7392 
7393 	if (vid)
7394 		wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
7395 	else
7396 		wr32(E1000_VMVIR(vf), 0);
7397 }
7398 
7399 static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
7400 				u16 vlan, u8 qos)
7401 {
7402 	int err;
7403 
7404 	err = igb_set_vf_vlan(adapter, vlan, true, vf);
7405 	if (err)
7406 		return err;
7407 
7408 	igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
7409 	igb_set_vmolr(adapter, vf, !vlan);
7410 
7411 	/* revoke access to previous VLAN */
7412 	if (vlan != adapter->vf_data[vf].pf_vlan)
7413 		igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7414 				false, vf);
7415 
7416 	adapter->vf_data[vf].pf_vlan = vlan;
7417 	adapter->vf_data[vf].pf_qos = qos;
7418 	igb_set_vf_vlan_strip(adapter, vf, true);
7419 	dev_info(&adapter->pdev->dev,
7420 		 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
7421 	if (test_bit(__IGB_DOWN, &adapter->state)) {
7422 		dev_warn(&adapter->pdev->dev,
7423 			 "The VF VLAN has been set, but the PF device is not up.\n");
7424 		dev_warn(&adapter->pdev->dev,
7425 			 "Bring the PF device up before attempting to use the VF device.\n");
7426 	}
7427 
7428 	return err;
7429 }
7430 
7431 static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
7432 {
7433 	/* Restore tagless access via VLAN 0 */
7434 	igb_set_vf_vlan(adapter, 0, true, vf);
7435 
7436 	igb_set_vmvir(adapter, 0, vf);
7437 	igb_set_vmolr(adapter, vf, true);
7438 
7439 	/* Remove any PF assigned VLAN */
7440 	if (adapter->vf_data[vf].pf_vlan)
7441 		igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7442 				false, vf);
7443 
7444 	adapter->vf_data[vf].pf_vlan = 0;
7445 	adapter->vf_data[vf].pf_qos = 0;
7446 	igb_set_vf_vlan_strip(adapter, vf, false);
7447 
7448 	return 0;
7449 }
7450 
7451 static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
7452 			       u16 vlan, u8 qos, __be16 vlan_proto)
7453 {
7454 	struct igb_adapter *adapter = netdev_priv(netdev);
7455 
7456 	if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
7457 		return -EINVAL;
7458 
7459 	if (vlan_proto != htons(ETH_P_8021Q))
7460 		return -EPROTONOSUPPORT;
7461 
7462 	return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
7463 			       igb_disable_port_vlan(adapter, vf);
7464 }
7465 
7466 static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7467 {
7468 	int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
7469 	int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
7470 	int ret;
7471 
7472 	if (adapter->vf_data[vf].pf_vlan)
7473 		return -1;
7474 
7475 	/* VLAN 0 is a special case, don't allow it to be removed */
7476 	if (!vid && !add)
7477 		return 0;
7478 
7479 	ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
7480 	if (!ret)
7481 		igb_set_vf_vlan_strip(adapter, vf, !!vid);
7482 	return ret;
7483 }
7484 
7485 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
7486 {
7487 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7488 
7489 	/* clear flags - except flag that indicates PF has set the MAC */
7490 	vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
7491 	vf_data->last_nack = jiffies;
7492 
7493 	/* reset vlans for device */
7494 	igb_clear_vf_vfta(adapter, vf);
7495 	igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
7496 	igb_set_vmvir(adapter, vf_data->pf_vlan |
7497 			       (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
7498 	igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
7499 	igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
7500 
7501 	/* reset multicast table array for vf */
7502 	adapter->vf_data[vf].num_vf_mc_hashes = 0;
7503 
7504 	/* Flush and reset the mta with the new values */
7505 	igb_set_rx_mode(adapter->netdev);
7506 }
7507 
7508 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
7509 {
7510 	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7511 
7512 	/* clear mac address as we were hotplug removed/added */
7513 	if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7514 		eth_zero_addr(vf_mac);
7515 
7516 	/* process remaining reset events */
7517 	igb_vf_reset(adapter, vf);
7518 }
7519 
7520 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
7521 {
7522 	struct e1000_hw *hw = &adapter->hw;
7523 	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7524 	u32 reg, msgbuf[3];
7525 	u8 *addr = (u8 *)(&msgbuf[1]);
7526 
7527 	/* process all the same items cleared in a function level reset */
7528 	igb_vf_reset(adapter, vf);
7529 
7530 	/* set vf mac address */
7531 	igb_set_vf_mac(adapter, vf, vf_mac);
7532 
7533 	/* enable transmit and receive for vf */
7534 	reg = rd32(E1000_VFTE);
7535 	wr32(E1000_VFTE, reg | BIT(vf));
7536 	reg = rd32(E1000_VFRE);
7537 	wr32(E1000_VFRE, reg | BIT(vf));
7538 
7539 	adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
7540 
7541 	/* reply to reset with ack and vf mac address */
7542 	if (!is_zero_ether_addr(vf_mac)) {
7543 		msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
7544 		memcpy(addr, vf_mac, ETH_ALEN);
7545 	} else {
7546 		msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
7547 	}
7548 	igb_write_mbx(hw, msgbuf, 3, vf);
7549 }
7550 
7551 static void igb_flush_mac_table(struct igb_adapter *adapter)
7552 {
7553 	struct e1000_hw *hw = &adapter->hw;
7554 	int i;
7555 
7556 	for (i = 0; i < hw->mac.rar_entry_count; i++) {
7557 		adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
7558 		eth_zero_addr(adapter->mac_table[i].addr);
7559 		adapter->mac_table[i].queue = 0;
7560 		igb_rar_set_index(adapter, i);
7561 	}
7562 }
7563 
7564 static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
7565 {
7566 	struct e1000_hw *hw = &adapter->hw;
7567 	/* do not count rar entries reserved for VFs MAC addresses */
7568 	int rar_entries = hw->mac.rar_entry_count -
7569 			  adapter->vfs_allocated_count;
7570 	int i, count = 0;
7571 
7572 	for (i = 0; i < rar_entries; i++) {
7573 		/* do not count default entries */
7574 		if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
7575 			continue;
7576 
7577 		/* do not count "in use" entries for different queues */
7578 		if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
7579 		    (adapter->mac_table[i].queue != queue))
7580 			continue;
7581 
7582 		count++;
7583 	}
7584 
7585 	return count;
7586 }
7587 
7588 /* Set default MAC address for the PF in the first RAR entry */
7589 static void igb_set_default_mac_filter(struct igb_adapter *adapter)
7590 {
7591 	struct igb_mac_addr *mac_table = &adapter->mac_table[0];
7592 
7593 	ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
7594 	mac_table->queue = adapter->vfs_allocated_count;
7595 	mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7596 
7597 	igb_rar_set_index(adapter, 0);
7598 }
7599 
7600 /* If the filter to be added and an already existing filter express
7601  * the same address and address type, it should be possible to only
7602  * override the other configurations, for example the queue to steer
7603  * traffic.
7604  */
7605 static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
7606 				      const u8 *addr, const u8 flags)
7607 {
7608 	if (!(entry->state & IGB_MAC_STATE_IN_USE))
7609 		return true;
7610 
7611 	if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
7612 	    (flags & IGB_MAC_STATE_SRC_ADDR))
7613 		return false;
7614 
7615 	if (!ether_addr_equal(addr, entry->addr))
7616 		return false;
7617 
7618 	return true;
7619 }
7620 
7621 /* Add a MAC filter for 'addr' directing matching traffic to 'queue',
7622  * 'flags' is used to indicate what kind of match is made, match is by
7623  * default for the destination address, if matching by source address
7624  * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
7625  */
7626 static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
7627 				    const u8 *addr, const u8 queue,
7628 				    const u8 flags)
7629 {
7630 	struct e1000_hw *hw = &adapter->hw;
7631 	int rar_entries = hw->mac.rar_entry_count -
7632 			  adapter->vfs_allocated_count;
7633 	int i;
7634 
7635 	if (is_zero_ether_addr(addr))
7636 		return -EINVAL;
7637 
7638 	/* Search for the first empty entry in the MAC table.
7639 	 * Do not touch entries at the end of the table reserved for the VF MAC
7640 	 * addresses.
7641 	 */
7642 	for (i = 0; i < rar_entries; i++) {
7643 		if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
7644 					       addr, flags))
7645 			continue;
7646 
7647 		ether_addr_copy(adapter->mac_table[i].addr, addr);
7648 		adapter->mac_table[i].queue = queue;
7649 		adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
7650 
7651 		igb_rar_set_index(adapter, i);
7652 		return i;
7653 	}
7654 
7655 	return -ENOSPC;
7656 }
7657 
7658 static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7659 			      const u8 queue)
7660 {
7661 	return igb_add_mac_filter_flags(adapter, addr, queue, 0);
7662 }
7663 
7664 /* Remove a MAC filter for 'addr' directing matching traffic to
7665  * 'queue', 'flags' is used to indicate what kind of match need to be
7666  * removed, match is by default for the destination address, if
7667  * matching by source address is to be removed the flag
7668  * IGB_MAC_STATE_SRC_ADDR can be used.
7669  */
7670 static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
7671 				    const u8 *addr, const u8 queue,
7672 				    const u8 flags)
7673 {
7674 	struct e1000_hw *hw = &adapter->hw;
7675 	int rar_entries = hw->mac.rar_entry_count -
7676 			  adapter->vfs_allocated_count;
7677 	int i;
7678 
7679 	if (is_zero_ether_addr(addr))
7680 		return -EINVAL;
7681 
7682 	/* Search for matching entry in the MAC table based on given address
7683 	 * and queue. Do not touch entries at the end of the table reserved
7684 	 * for the VF MAC addresses.
7685 	 */
7686 	for (i = 0; i < rar_entries; i++) {
7687 		if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
7688 			continue;
7689 		if ((adapter->mac_table[i].state & flags) != flags)
7690 			continue;
7691 		if (adapter->mac_table[i].queue != queue)
7692 			continue;
7693 		if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
7694 			continue;
7695 
7696 		/* When a filter for the default address is "deleted",
7697 		 * we return it to its initial configuration
7698 		 */
7699 		if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
7700 			adapter->mac_table[i].state =
7701 				IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7702 			adapter->mac_table[i].queue =
7703 				adapter->vfs_allocated_count;
7704 		} else {
7705 			adapter->mac_table[i].state = 0;
7706 			adapter->mac_table[i].queue = 0;
7707 			eth_zero_addr(adapter->mac_table[i].addr);
7708 		}
7709 
7710 		igb_rar_set_index(adapter, i);
7711 		return 0;
7712 	}
7713 
7714 	return -ENOENT;
7715 }
7716 
7717 static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7718 			      const u8 queue)
7719 {
7720 	return igb_del_mac_filter_flags(adapter, addr, queue, 0);
7721 }
7722 
7723 int igb_add_mac_steering_filter(struct igb_adapter *adapter,
7724 				const u8 *addr, u8 queue, u8 flags)
7725 {
7726 	struct e1000_hw *hw = &adapter->hw;
7727 
7728 	/* In theory, this should be supported on 82575 as well, but
7729 	 * that part wasn't easily accessible during development.
7730 	 */
7731 	if (hw->mac.type != e1000_i210)
7732 		return -EOPNOTSUPP;
7733 
7734 	return igb_add_mac_filter_flags(adapter, addr, queue,
7735 					IGB_MAC_STATE_QUEUE_STEERING | flags);
7736 }
7737 
7738 int igb_del_mac_steering_filter(struct igb_adapter *adapter,
7739 				const u8 *addr, u8 queue, u8 flags)
7740 {
7741 	return igb_del_mac_filter_flags(adapter, addr, queue,
7742 					IGB_MAC_STATE_QUEUE_STEERING | flags);
7743 }
7744 
7745 static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
7746 {
7747 	struct igb_adapter *adapter = netdev_priv(netdev);
7748 	int ret;
7749 
7750 	ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7751 
7752 	return min_t(int, ret, 0);
7753 }
7754 
7755 static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
7756 {
7757 	struct igb_adapter *adapter = netdev_priv(netdev);
7758 
7759 	igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7760 
7761 	return 0;
7762 }
7763 
7764 static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
7765 				 const u32 info, const u8 *addr)
7766 {
7767 	struct pci_dev *pdev = adapter->pdev;
7768 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7769 	struct list_head *pos;
7770 	struct vf_mac_filter *entry = NULL;
7771 	int ret = 0;
7772 
7773 	if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7774 	    !vf_data->trusted) {
7775 		dev_warn(&pdev->dev,
7776 			 "VF %d requested MAC filter but is administratively denied\n",
7777 			  vf);
7778 		return -EINVAL;
7779 	}
7780 	if (!is_valid_ether_addr(addr)) {
7781 		dev_warn(&pdev->dev,
7782 			 "VF %d attempted to set invalid MAC filter\n",
7783 			  vf);
7784 		return -EINVAL;
7785 	}
7786 
7787 	switch (info) {
7788 	case E1000_VF_MAC_FILTER_CLR:
7789 		/* remove all unicast MAC filters related to the current VF */
7790 		list_for_each(pos, &adapter->vf_macs.l) {
7791 			entry = list_entry(pos, struct vf_mac_filter, l);
7792 			if (entry->vf == vf) {
7793 				entry->vf = -1;
7794 				entry->free = true;
7795 				igb_del_mac_filter(adapter, entry->vf_mac, vf);
7796 			}
7797 		}
7798 		break;
7799 	case E1000_VF_MAC_FILTER_ADD:
7800 		/* try to find empty slot in the list */
7801 		list_for_each(pos, &adapter->vf_macs.l) {
7802 			entry = list_entry(pos, struct vf_mac_filter, l);
7803 			if (entry->free)
7804 				break;
7805 		}
7806 
7807 		if (entry && entry->free) {
7808 			entry->free = false;
7809 			entry->vf = vf;
7810 			ether_addr_copy(entry->vf_mac, addr);
7811 
7812 			ret = igb_add_mac_filter(adapter, addr, vf);
7813 			ret = min_t(int, ret, 0);
7814 		} else {
7815 			ret = -ENOSPC;
7816 		}
7817 
7818 		if (ret == -ENOSPC)
7819 			dev_warn(&pdev->dev,
7820 				 "VF %d has requested MAC filter but there is no space for it\n",
7821 				 vf);
7822 		break;
7823 	default:
7824 		ret = -EINVAL;
7825 		break;
7826 	}
7827 
7828 	return ret;
7829 }
7830 
7831 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
7832 {
7833 	struct pci_dev *pdev = adapter->pdev;
7834 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7835 	u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
7836 
7837 	/* The VF MAC Address is stored in a packed array of bytes
7838 	 * starting at the second 32 bit word of the msg array
7839 	 */
7840 	unsigned char *addr = (unsigned char *)&msg[1];
7841 	int ret = 0;
7842 
7843 	if (!info) {
7844 		if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7845 		    !vf_data->trusted) {
7846 			dev_warn(&pdev->dev,
7847 				 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
7848 				 vf);
7849 			return -EINVAL;
7850 		}
7851 
7852 		if (!is_valid_ether_addr(addr)) {
7853 			dev_warn(&pdev->dev,
7854 				 "VF %d attempted to set invalid MAC\n",
7855 				 vf);
7856 			return -EINVAL;
7857 		}
7858 
7859 		ret = igb_set_vf_mac(adapter, vf, addr);
7860 	} else {
7861 		ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
7862 	}
7863 
7864 	return ret;
7865 }
7866 
7867 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
7868 {
7869 	struct e1000_hw *hw = &adapter->hw;
7870 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7871 	u32 msg = E1000_VT_MSGTYPE_NACK;
7872 
7873 	/* if device isn't clear to send it shouldn't be reading either */
7874 	if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
7875 	    time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
7876 		igb_write_mbx(hw, &msg, 1, vf);
7877 		vf_data->last_nack = jiffies;
7878 	}
7879 }
7880 
7881 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
7882 {
7883 	struct pci_dev *pdev = adapter->pdev;
7884 	u32 msgbuf[E1000_VFMAILBOX_SIZE];
7885 	struct e1000_hw *hw = &adapter->hw;
7886 	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7887 	s32 retval;
7888 
7889 	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
7890 
7891 	if (retval) {
7892 		/* if receive failed revoke VF CTS stats and restart init */
7893 		dev_err(&pdev->dev, "Error receiving message from VF\n");
7894 		vf_data->flags &= ~IGB_VF_FLAG_CTS;
7895 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7896 			goto unlock;
7897 		goto out;
7898 	}
7899 
7900 	/* this is a message we already processed, do nothing */
7901 	if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
7902 		goto unlock;
7903 
7904 	/* until the vf completes a reset it should not be
7905 	 * allowed to start any configuration.
7906 	 */
7907 	if (msgbuf[0] == E1000_VF_RESET) {
7908 		/* unlocks mailbox */
7909 		igb_vf_reset_msg(adapter, vf);
7910 		return;
7911 	}
7912 
7913 	if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
7914 		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7915 			goto unlock;
7916 		retval = -1;
7917 		goto out;
7918 	}
7919 
7920 	switch ((msgbuf[0] & 0xFFFF)) {
7921 	case E1000_VF_SET_MAC_ADDR:
7922 		retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
7923 		break;
7924 	case E1000_VF_SET_PROMISC:
7925 		retval = igb_set_vf_promisc(adapter, msgbuf, vf);
7926 		break;
7927 	case E1000_VF_SET_MULTICAST:
7928 		retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
7929 		break;
7930 	case E1000_VF_SET_LPE:
7931 		retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
7932 		break;
7933 	case E1000_VF_SET_VLAN:
7934 		retval = -1;
7935 		if (vf_data->pf_vlan)
7936 			dev_warn(&pdev->dev,
7937 				 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
7938 				 vf);
7939 		else
7940 			retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
7941 		break;
7942 	default:
7943 		dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
7944 		retval = -1;
7945 		break;
7946 	}
7947 
7948 	msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
7949 out:
7950 	/* notify the VF of the results of what it sent us */
7951 	if (retval)
7952 		msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
7953 	else
7954 		msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
7955 
7956 	/* unlocks mailbox */
7957 	igb_write_mbx(hw, msgbuf, 1, vf);
7958 	return;
7959 
7960 unlock:
7961 	igb_unlock_mbx(hw, vf);
7962 }
7963 
7964 static void igb_msg_task(struct igb_adapter *adapter)
7965 {
7966 	struct e1000_hw *hw = &adapter->hw;
7967 	unsigned long flags;
7968 	u32 vf;
7969 
7970 	spin_lock_irqsave(&adapter->vfs_lock, flags);
7971 	for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
7972 		/* process any reset requests */
7973 		if (!igb_check_for_rst(hw, vf))
7974 			igb_vf_reset_event(adapter, vf);
7975 
7976 		/* process any messages pending */
7977 		if (!igb_check_for_msg(hw, vf))
7978 			igb_rcv_msg_from_vf(adapter, vf);
7979 
7980 		/* process any acks */
7981 		if (!igb_check_for_ack(hw, vf))
7982 			igb_rcv_ack_from_vf(adapter, vf);
7983 	}
7984 	spin_unlock_irqrestore(&adapter->vfs_lock, flags);
7985 }
7986 
7987 /**
7988  *  igb_set_uta - Set unicast filter table address
7989  *  @adapter: board private structure
7990  *  @set: boolean indicating if we are setting or clearing bits
7991  *
7992  *  The unicast table address is a register array of 32-bit registers.
7993  *  The table is meant to be used in a way similar to how the MTA is used
7994  *  however due to certain limitations in the hardware it is necessary to
7995  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
7996  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
7997  **/
7998 static void igb_set_uta(struct igb_adapter *adapter, bool set)
7999 {
8000 	struct e1000_hw *hw = &adapter->hw;
8001 	u32 uta = set ? ~0 : 0;
8002 	int i;
8003 
8004 	/* we only need to do this if VMDq is enabled */
8005 	if (!adapter->vfs_allocated_count)
8006 		return;
8007 
8008 	for (i = hw->mac.uta_reg_count; i--;)
8009 		array_wr32(E1000_UTA, i, uta);
8010 }
8011 
8012 /**
8013  *  igb_intr_msi - Interrupt Handler
8014  *  @irq: interrupt number
8015  *  @data: pointer to a network interface device structure
8016  **/
8017 static irqreturn_t igb_intr_msi(int irq, void *data)
8018 {
8019 	struct igb_adapter *adapter = data;
8020 	struct igb_q_vector *q_vector = adapter->q_vector[0];
8021 	struct e1000_hw *hw = &adapter->hw;
8022 	/* read ICR disables interrupts using IAM */
8023 	u32 icr = rd32(E1000_ICR);
8024 
8025 	igb_write_itr(q_vector);
8026 
8027 	if (icr & E1000_ICR_DRSTA)
8028 		schedule_work(&adapter->reset_task);
8029 
8030 	if (icr & E1000_ICR_DOUTSYNC) {
8031 		/* HW is reporting DMA is out of sync */
8032 		adapter->stats.doosync++;
8033 	}
8034 
8035 	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8036 		hw->mac.get_link_status = 1;
8037 		if (!test_bit(__IGB_DOWN, &adapter->state))
8038 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
8039 	}
8040 
8041 	if (icr & E1000_ICR_TS)
8042 		igb_tsync_interrupt(adapter);
8043 
8044 	napi_schedule(&q_vector->napi);
8045 
8046 	return IRQ_HANDLED;
8047 }
8048 
8049 /**
8050  *  igb_intr - Legacy Interrupt Handler
8051  *  @irq: interrupt number
8052  *  @data: pointer to a network interface device structure
8053  **/
8054 static irqreturn_t igb_intr(int irq, void *data)
8055 {
8056 	struct igb_adapter *adapter = data;
8057 	struct igb_q_vector *q_vector = adapter->q_vector[0];
8058 	struct e1000_hw *hw = &adapter->hw;
8059 	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
8060 	 * need for the IMC write
8061 	 */
8062 	u32 icr = rd32(E1000_ICR);
8063 
8064 	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
8065 	 * not set, then the adapter didn't send an interrupt
8066 	 */
8067 	if (!(icr & E1000_ICR_INT_ASSERTED))
8068 		return IRQ_NONE;
8069 
8070 	igb_write_itr(q_vector);
8071 
8072 	if (icr & E1000_ICR_DRSTA)
8073 		schedule_work(&adapter->reset_task);
8074 
8075 	if (icr & E1000_ICR_DOUTSYNC) {
8076 		/* HW is reporting DMA is out of sync */
8077 		adapter->stats.doosync++;
8078 	}
8079 
8080 	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8081 		hw->mac.get_link_status = 1;
8082 		/* guard against interrupt when we're going down */
8083 		if (!test_bit(__IGB_DOWN, &adapter->state))
8084 			mod_timer(&adapter->watchdog_timer, jiffies + 1);
8085 	}
8086 
8087 	if (icr & E1000_ICR_TS)
8088 		igb_tsync_interrupt(adapter);
8089 
8090 	napi_schedule(&q_vector->napi);
8091 
8092 	return IRQ_HANDLED;
8093 }
8094 
8095 static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
8096 {
8097 	struct igb_adapter *adapter = q_vector->adapter;
8098 	struct e1000_hw *hw = &adapter->hw;
8099 
8100 	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
8101 	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
8102 		if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
8103 			igb_set_itr(q_vector);
8104 		else
8105 			igb_update_ring_itr(q_vector);
8106 	}
8107 
8108 	if (!test_bit(__IGB_DOWN, &adapter->state)) {
8109 		if (adapter->flags & IGB_FLAG_HAS_MSIX)
8110 			wr32(E1000_EIMS, q_vector->eims_value);
8111 		else
8112 			igb_irq_enable(adapter);
8113 	}
8114 }
8115 
8116 /**
8117  *  igb_poll - NAPI Rx polling callback
8118  *  @napi: napi polling structure
8119  *  @budget: count of how many packets we should handle
8120  **/
8121 static int igb_poll(struct napi_struct *napi, int budget)
8122 {
8123 	struct igb_q_vector *q_vector = container_of(napi,
8124 						     struct igb_q_vector,
8125 						     napi);
8126 	bool clean_complete = true;
8127 	int work_done = 0;
8128 
8129 #ifdef CONFIG_IGB_DCA
8130 	if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
8131 		igb_update_dca(q_vector);
8132 #endif
8133 	if (q_vector->tx.ring)
8134 		clean_complete = igb_clean_tx_irq(q_vector, budget);
8135 
8136 	if (q_vector->rx.ring) {
8137 		int cleaned = igb_clean_rx_irq(q_vector, budget);
8138 
8139 		work_done += cleaned;
8140 		if (cleaned >= budget)
8141 			clean_complete = false;
8142 	}
8143 
8144 	/* If all work not completed, return budget and keep polling */
8145 	if (!clean_complete)
8146 		return budget;
8147 
8148 	/* Exit the polling mode, but don't re-enable interrupts if stack might
8149 	 * poll us due to busy-polling
8150 	 */
8151 	if (likely(napi_complete_done(napi, work_done)))
8152 		igb_ring_irq_enable(q_vector);
8153 
8154 	return work_done;
8155 }
8156 
8157 /**
8158  *  igb_clean_tx_irq - Reclaim resources after transmit completes
8159  *  @q_vector: pointer to q_vector containing needed info
8160  *  @napi_budget: Used to determine if we are in netpoll
8161  *
8162  *  returns true if ring is completely cleaned
8163  **/
8164 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
8165 {
8166 	struct igb_adapter *adapter = q_vector->adapter;
8167 	struct igb_ring *tx_ring = q_vector->tx.ring;
8168 	struct igb_tx_buffer *tx_buffer;
8169 	union e1000_adv_tx_desc *tx_desc;
8170 	unsigned int total_bytes = 0, total_packets = 0;
8171 	unsigned int budget = q_vector->tx.work_limit;
8172 	unsigned int i = tx_ring->next_to_clean;
8173 
8174 	if (test_bit(__IGB_DOWN, &adapter->state))
8175 		return true;
8176 
8177 	tx_buffer = &tx_ring->tx_buffer_info[i];
8178 	tx_desc = IGB_TX_DESC(tx_ring, i);
8179 	i -= tx_ring->count;
8180 
8181 	do {
8182 		union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
8183 
8184 		/* if next_to_watch is not set then there is no work pending */
8185 		if (!eop_desc)
8186 			break;
8187 
8188 		/* prevent any other reads prior to eop_desc */
8189 		smp_rmb();
8190 
8191 		/* if DD is not set pending work has not been completed */
8192 		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
8193 			break;
8194 
8195 		/* clear next_to_watch to prevent false hangs */
8196 		tx_buffer->next_to_watch = NULL;
8197 
8198 		/* update the statistics for this packet */
8199 		total_bytes += tx_buffer->bytecount;
8200 		total_packets += tx_buffer->gso_segs;
8201 
8202 		/* free the skb */
8203 		if (tx_buffer->type == IGB_TYPE_SKB)
8204 			napi_consume_skb(tx_buffer->skb, napi_budget);
8205 		else
8206 			xdp_return_frame(tx_buffer->xdpf);
8207 
8208 		/* unmap skb header data */
8209 		dma_unmap_single(tx_ring->dev,
8210 				 dma_unmap_addr(tx_buffer, dma),
8211 				 dma_unmap_len(tx_buffer, len),
8212 				 DMA_TO_DEVICE);
8213 
8214 		/* clear tx_buffer data */
8215 		dma_unmap_len_set(tx_buffer, len, 0);
8216 
8217 		/* clear last DMA location and unmap remaining buffers */
8218 		while (tx_desc != eop_desc) {
8219 			tx_buffer++;
8220 			tx_desc++;
8221 			i++;
8222 			if (unlikely(!i)) {
8223 				i -= tx_ring->count;
8224 				tx_buffer = tx_ring->tx_buffer_info;
8225 				tx_desc = IGB_TX_DESC(tx_ring, 0);
8226 			}
8227 
8228 			/* unmap any remaining paged data */
8229 			if (dma_unmap_len(tx_buffer, len)) {
8230 				dma_unmap_page(tx_ring->dev,
8231 					       dma_unmap_addr(tx_buffer, dma),
8232 					       dma_unmap_len(tx_buffer, len),
8233 					       DMA_TO_DEVICE);
8234 				dma_unmap_len_set(tx_buffer, len, 0);
8235 			}
8236 		}
8237 
8238 		/* move us one more past the eop_desc for start of next pkt */
8239 		tx_buffer++;
8240 		tx_desc++;
8241 		i++;
8242 		if (unlikely(!i)) {
8243 			i -= tx_ring->count;
8244 			tx_buffer = tx_ring->tx_buffer_info;
8245 			tx_desc = IGB_TX_DESC(tx_ring, 0);
8246 		}
8247 
8248 		/* issue prefetch for next Tx descriptor */
8249 		prefetch(tx_desc);
8250 
8251 		/* update budget accounting */
8252 		budget--;
8253 	} while (likely(budget));
8254 
8255 	netdev_tx_completed_queue(txring_txq(tx_ring),
8256 				  total_packets, total_bytes);
8257 	i += tx_ring->count;
8258 	tx_ring->next_to_clean = i;
8259 	u64_stats_update_begin(&tx_ring->tx_syncp);
8260 	tx_ring->tx_stats.bytes += total_bytes;
8261 	tx_ring->tx_stats.packets += total_packets;
8262 	u64_stats_update_end(&tx_ring->tx_syncp);
8263 	q_vector->tx.total_bytes += total_bytes;
8264 	q_vector->tx.total_packets += total_packets;
8265 
8266 	if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
8267 		struct e1000_hw *hw = &adapter->hw;
8268 
8269 		/* Detect a transmit hang in hardware, this serializes the
8270 		 * check with the clearing of time_stamp and movement of i
8271 		 */
8272 		clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8273 		if (tx_buffer->next_to_watch &&
8274 		    time_after(jiffies, tx_buffer->time_stamp +
8275 			       (adapter->tx_timeout_factor * HZ)) &&
8276 		    !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
8277 
8278 			/* detected Tx unit hang */
8279 			dev_err(tx_ring->dev,
8280 				"Detected Tx Unit Hang\n"
8281 				"  Tx Queue             <%d>\n"
8282 				"  TDH                  <%x>\n"
8283 				"  TDT                  <%x>\n"
8284 				"  next_to_use          <%x>\n"
8285 				"  next_to_clean        <%x>\n"
8286 				"buffer_info[next_to_clean]\n"
8287 				"  time_stamp           <%lx>\n"
8288 				"  next_to_watch        <%p>\n"
8289 				"  jiffies              <%lx>\n"
8290 				"  desc.status          <%x>\n",
8291 				tx_ring->queue_index,
8292 				rd32(E1000_TDH(tx_ring->reg_idx)),
8293 				readl(tx_ring->tail),
8294 				tx_ring->next_to_use,
8295 				tx_ring->next_to_clean,
8296 				tx_buffer->time_stamp,
8297 				tx_buffer->next_to_watch,
8298 				jiffies,
8299 				tx_buffer->next_to_watch->wb.status);
8300 			netif_stop_subqueue(tx_ring->netdev,
8301 					    tx_ring->queue_index);
8302 
8303 			/* we are about to reset, no point in enabling stuff */
8304 			return true;
8305 		}
8306 	}
8307 
8308 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
8309 	if (unlikely(total_packets &&
8310 	    netif_carrier_ok(tx_ring->netdev) &&
8311 	    igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
8312 		/* Make sure that anybody stopping the queue after this
8313 		 * sees the new next_to_clean.
8314 		 */
8315 		smp_mb();
8316 		if (__netif_subqueue_stopped(tx_ring->netdev,
8317 					     tx_ring->queue_index) &&
8318 		    !(test_bit(__IGB_DOWN, &adapter->state))) {
8319 			netif_wake_subqueue(tx_ring->netdev,
8320 					    tx_ring->queue_index);
8321 
8322 			u64_stats_update_begin(&tx_ring->tx_syncp);
8323 			tx_ring->tx_stats.restart_queue++;
8324 			u64_stats_update_end(&tx_ring->tx_syncp);
8325 		}
8326 	}
8327 
8328 	return !!budget;
8329 }
8330 
8331 /**
8332  *  igb_reuse_rx_page - page flip buffer and store it back on the ring
8333  *  @rx_ring: rx descriptor ring to store buffers on
8334  *  @old_buff: donor buffer to have page reused
8335  *
8336  *  Synchronizes page for reuse by the adapter
8337  **/
8338 static void igb_reuse_rx_page(struct igb_ring *rx_ring,
8339 			      struct igb_rx_buffer *old_buff)
8340 {
8341 	struct igb_rx_buffer *new_buff;
8342 	u16 nta = rx_ring->next_to_alloc;
8343 
8344 	new_buff = &rx_ring->rx_buffer_info[nta];
8345 
8346 	/* update, and store next to alloc */
8347 	nta++;
8348 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
8349 
8350 	/* Transfer page from old buffer to new buffer.
8351 	 * Move each member individually to avoid possible store
8352 	 * forwarding stalls.
8353 	 */
8354 	new_buff->dma		= old_buff->dma;
8355 	new_buff->page		= old_buff->page;
8356 	new_buff->page_offset	= old_buff->page_offset;
8357 	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
8358 }
8359 
8360 static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
8361 				  int rx_buf_pgcnt)
8362 {
8363 	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
8364 	struct page *page = rx_buffer->page;
8365 
8366 	/* avoid re-using remote and pfmemalloc pages */
8367 	if (!dev_page_is_reusable(page))
8368 		return false;
8369 
8370 #if (PAGE_SIZE < 8192)
8371 	/* if we are only owner of page we can reuse it */
8372 	if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1))
8373 		return false;
8374 #else
8375 #define IGB_LAST_OFFSET \
8376 	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
8377 
8378 	if (rx_buffer->page_offset > IGB_LAST_OFFSET)
8379 		return false;
8380 #endif
8381 
8382 	/* If we have drained the page fragment pool we need to update
8383 	 * the pagecnt_bias and page count so that we fully restock the
8384 	 * number of references the driver holds.
8385 	 */
8386 	if (unlikely(pagecnt_bias == 1)) {
8387 		page_ref_add(page, USHRT_MAX - 1);
8388 		rx_buffer->pagecnt_bias = USHRT_MAX;
8389 	}
8390 
8391 	return true;
8392 }
8393 
8394 /**
8395  *  igb_add_rx_frag - Add contents of Rx buffer to sk_buff
8396  *  @rx_ring: rx descriptor ring to transact packets on
8397  *  @rx_buffer: buffer containing page to add
8398  *  @skb: sk_buff to place the data into
8399  *  @size: size of buffer to be added
8400  *
8401  *  This function will add the data contained in rx_buffer->page to the skb.
8402  **/
8403 static void igb_add_rx_frag(struct igb_ring *rx_ring,
8404 			    struct igb_rx_buffer *rx_buffer,
8405 			    struct sk_buff *skb,
8406 			    unsigned int size)
8407 {
8408 #if (PAGE_SIZE < 8192)
8409 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8410 #else
8411 	unsigned int truesize = ring_uses_build_skb(rx_ring) ?
8412 				SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
8413 				SKB_DATA_ALIGN(size);
8414 #endif
8415 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
8416 			rx_buffer->page_offset, size, truesize);
8417 #if (PAGE_SIZE < 8192)
8418 	rx_buffer->page_offset ^= truesize;
8419 #else
8420 	rx_buffer->page_offset += truesize;
8421 #endif
8422 }
8423 
8424 static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
8425 					 struct igb_rx_buffer *rx_buffer,
8426 					 struct xdp_buff *xdp,
8427 					 ktime_t timestamp)
8428 {
8429 #if (PAGE_SIZE < 8192)
8430 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8431 #else
8432 	unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
8433 					       xdp->data_hard_start);
8434 #endif
8435 	unsigned int size = xdp->data_end - xdp->data;
8436 	unsigned int headlen;
8437 	struct sk_buff *skb;
8438 
8439 	/* prefetch first cache line of first page */
8440 	net_prefetch(xdp->data);
8441 
8442 	/* allocate a skb to store the frags */
8443 	skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
8444 	if (unlikely(!skb))
8445 		return NULL;
8446 
8447 	if (timestamp)
8448 		skb_hwtstamps(skb)->hwtstamp = timestamp;
8449 
8450 	/* Determine available headroom for copy */
8451 	headlen = size;
8452 	if (headlen > IGB_RX_HDR_LEN)
8453 		headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN);
8454 
8455 	/* align pull length to size of long to optimize memcpy performance */
8456 	memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long)));
8457 
8458 	/* update all of the pointers */
8459 	size -= headlen;
8460 	if (size) {
8461 		skb_add_rx_frag(skb, 0, rx_buffer->page,
8462 				(xdp->data + headlen) - page_address(rx_buffer->page),
8463 				size, truesize);
8464 #if (PAGE_SIZE < 8192)
8465 		rx_buffer->page_offset ^= truesize;
8466 #else
8467 		rx_buffer->page_offset += truesize;
8468 #endif
8469 	} else {
8470 		rx_buffer->pagecnt_bias++;
8471 	}
8472 
8473 	return skb;
8474 }
8475 
8476 static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
8477 				     struct igb_rx_buffer *rx_buffer,
8478 				     struct xdp_buff *xdp,
8479 				     ktime_t timestamp)
8480 {
8481 #if (PAGE_SIZE < 8192)
8482 	unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8483 #else
8484 	unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
8485 				SKB_DATA_ALIGN(xdp->data_end -
8486 					       xdp->data_hard_start);
8487 #endif
8488 	unsigned int metasize = xdp->data - xdp->data_meta;
8489 	struct sk_buff *skb;
8490 
8491 	/* prefetch first cache line of first page */
8492 	net_prefetch(xdp->data_meta);
8493 
8494 	/* build an skb around the page buffer */
8495 	skb = napi_build_skb(xdp->data_hard_start, truesize);
8496 	if (unlikely(!skb))
8497 		return NULL;
8498 
8499 	/* update pointers within the skb to store the data */
8500 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
8501 	__skb_put(skb, xdp->data_end - xdp->data);
8502 
8503 	if (metasize)
8504 		skb_metadata_set(skb, metasize);
8505 
8506 	if (timestamp)
8507 		skb_hwtstamps(skb)->hwtstamp = timestamp;
8508 
8509 	/* update buffer offset */
8510 #if (PAGE_SIZE < 8192)
8511 	rx_buffer->page_offset ^= truesize;
8512 #else
8513 	rx_buffer->page_offset += truesize;
8514 #endif
8515 
8516 	return skb;
8517 }
8518 
8519 static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter,
8520 				   struct igb_ring *rx_ring,
8521 				   struct xdp_buff *xdp)
8522 {
8523 	int err, result = IGB_XDP_PASS;
8524 	struct bpf_prog *xdp_prog;
8525 	u32 act;
8526 
8527 	xdp_prog = READ_ONCE(rx_ring->xdp_prog);
8528 
8529 	if (!xdp_prog)
8530 		goto xdp_out;
8531 
8532 	prefetchw(xdp->data_hard_start); /* xdp_frame write */
8533 
8534 	act = bpf_prog_run_xdp(xdp_prog, xdp);
8535 	switch (act) {
8536 	case XDP_PASS:
8537 		break;
8538 	case XDP_TX:
8539 		result = igb_xdp_xmit_back(adapter, xdp);
8540 		if (result == IGB_XDP_CONSUMED)
8541 			goto out_failure;
8542 		break;
8543 	case XDP_REDIRECT:
8544 		err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
8545 		if (err)
8546 			goto out_failure;
8547 		result = IGB_XDP_REDIR;
8548 		break;
8549 	default:
8550 		bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act);
8551 		fallthrough;
8552 	case XDP_ABORTED:
8553 out_failure:
8554 		trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
8555 		fallthrough;
8556 	case XDP_DROP:
8557 		result = IGB_XDP_CONSUMED;
8558 		break;
8559 	}
8560 xdp_out:
8561 	return ERR_PTR(-result);
8562 }
8563 
8564 static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring,
8565 					  unsigned int size)
8566 {
8567 	unsigned int truesize;
8568 
8569 #if (PAGE_SIZE < 8192)
8570 	truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
8571 #else
8572 	truesize = ring_uses_build_skb(rx_ring) ?
8573 		SKB_DATA_ALIGN(IGB_SKB_PAD + size) +
8574 		SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
8575 		SKB_DATA_ALIGN(size);
8576 #endif
8577 	return truesize;
8578 }
8579 
8580 static void igb_rx_buffer_flip(struct igb_ring *rx_ring,
8581 			       struct igb_rx_buffer *rx_buffer,
8582 			       unsigned int size)
8583 {
8584 	unsigned int truesize = igb_rx_frame_truesize(rx_ring, size);
8585 #if (PAGE_SIZE < 8192)
8586 	rx_buffer->page_offset ^= truesize;
8587 #else
8588 	rx_buffer->page_offset += truesize;
8589 #endif
8590 }
8591 
8592 static inline void igb_rx_checksum(struct igb_ring *ring,
8593 				   union e1000_adv_rx_desc *rx_desc,
8594 				   struct sk_buff *skb)
8595 {
8596 	skb_checksum_none_assert(skb);
8597 
8598 	/* Ignore Checksum bit is set */
8599 	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
8600 		return;
8601 
8602 	/* Rx checksum disabled via ethtool */
8603 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
8604 		return;
8605 
8606 	/* TCP/UDP checksum error bit is set */
8607 	if (igb_test_staterr(rx_desc,
8608 			     E1000_RXDEXT_STATERR_TCPE |
8609 			     E1000_RXDEXT_STATERR_IPE)) {
8610 		/* work around errata with sctp packets where the TCPE aka
8611 		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
8612 		 * packets, (aka let the stack check the crc32c)
8613 		 */
8614 		if (!((skb->len == 60) &&
8615 		      test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
8616 			u64_stats_update_begin(&ring->rx_syncp);
8617 			ring->rx_stats.csum_err++;
8618 			u64_stats_update_end(&ring->rx_syncp);
8619 		}
8620 		/* let the stack verify checksum errors */
8621 		return;
8622 	}
8623 	/* It must be a TCP or UDP packet with a valid checksum */
8624 	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
8625 				      E1000_RXD_STAT_UDPCS))
8626 		skb->ip_summed = CHECKSUM_UNNECESSARY;
8627 
8628 	dev_dbg(ring->dev, "cksum success: bits %08X\n",
8629 		le32_to_cpu(rx_desc->wb.upper.status_error));
8630 }
8631 
8632 static inline void igb_rx_hash(struct igb_ring *ring,
8633 			       union e1000_adv_rx_desc *rx_desc,
8634 			       struct sk_buff *skb)
8635 {
8636 	if (ring->netdev->features & NETIF_F_RXHASH)
8637 		skb_set_hash(skb,
8638 			     le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
8639 			     PKT_HASH_TYPE_L3);
8640 }
8641 
8642 /**
8643  *  igb_is_non_eop - process handling of non-EOP buffers
8644  *  @rx_ring: Rx ring being processed
8645  *  @rx_desc: Rx descriptor for current buffer
8646  *
8647  *  This function updates next to clean.  If the buffer is an EOP buffer
8648  *  this function exits returning false, otherwise it will place the
8649  *  sk_buff in the next buffer to be chained and return true indicating
8650  *  that this is in fact a non-EOP buffer.
8651  **/
8652 static bool igb_is_non_eop(struct igb_ring *rx_ring,
8653 			   union e1000_adv_rx_desc *rx_desc)
8654 {
8655 	u32 ntc = rx_ring->next_to_clean + 1;
8656 
8657 	/* fetch, update, and store next to clean */
8658 	ntc = (ntc < rx_ring->count) ? ntc : 0;
8659 	rx_ring->next_to_clean = ntc;
8660 
8661 	prefetch(IGB_RX_DESC(rx_ring, ntc));
8662 
8663 	if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8664 		return false;
8665 
8666 	return true;
8667 }
8668 
8669 /**
8670  *  igb_cleanup_headers - Correct corrupted or empty headers
8671  *  @rx_ring: rx descriptor ring packet is being transacted on
8672  *  @rx_desc: pointer to the EOP Rx descriptor
8673  *  @skb: pointer to current skb being fixed
8674  *
8675  *  Address the case where we are pulling data in on pages only
8676  *  and as such no data is present in the skb header.
8677  *
8678  *  In addition if skb is not at least 60 bytes we need to pad it so that
8679  *  it is large enough to qualify as a valid Ethernet frame.
8680  *
8681  *  Returns true if an error was encountered and skb was freed.
8682  **/
8683 static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8684 				union e1000_adv_rx_desc *rx_desc,
8685 				struct sk_buff *skb)
8686 {
8687 	/* XDP packets use error pointer so abort at this point */
8688 	if (IS_ERR(skb))
8689 		return true;
8690 
8691 	if (unlikely((igb_test_staterr(rx_desc,
8692 				       E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8693 		struct net_device *netdev = rx_ring->netdev;
8694 		if (!(netdev->features & NETIF_F_RXALL)) {
8695 			dev_kfree_skb_any(skb);
8696 			return true;
8697 		}
8698 	}
8699 
8700 	/* if eth_skb_pad returns an error the skb was freed */
8701 	if (eth_skb_pad(skb))
8702 		return true;
8703 
8704 	return false;
8705 }
8706 
8707 /**
8708  *  igb_process_skb_fields - Populate skb header fields from Rx descriptor
8709  *  @rx_ring: rx descriptor ring packet is being transacted on
8710  *  @rx_desc: pointer to the EOP Rx descriptor
8711  *  @skb: pointer to current skb being populated
8712  *
8713  *  This function checks the ring, descriptor, and packet information in
8714  *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
8715  *  other fields within the skb.
8716  **/
8717 static void igb_process_skb_fields(struct igb_ring *rx_ring,
8718 				   union e1000_adv_rx_desc *rx_desc,
8719 				   struct sk_buff *skb)
8720 {
8721 	struct net_device *dev = rx_ring->netdev;
8722 
8723 	igb_rx_hash(rx_ring, rx_desc, skb);
8724 
8725 	igb_rx_checksum(rx_ring, rx_desc, skb);
8726 
8727 	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
8728 	    !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
8729 		igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
8730 
8731 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
8732 	    igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
8733 		u16 vid;
8734 
8735 		if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
8736 		    test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
8737 			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
8738 		else
8739 			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
8740 
8741 		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
8742 	}
8743 
8744 	skb_record_rx_queue(skb, rx_ring->queue_index);
8745 
8746 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8747 }
8748 
8749 static unsigned int igb_rx_offset(struct igb_ring *rx_ring)
8750 {
8751 	return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
8752 }
8753 
8754 static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
8755 					       const unsigned int size, int *rx_buf_pgcnt)
8756 {
8757 	struct igb_rx_buffer *rx_buffer;
8758 
8759 	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
8760 	*rx_buf_pgcnt =
8761 #if (PAGE_SIZE < 8192)
8762 		page_count(rx_buffer->page);
8763 #else
8764 		0;
8765 #endif
8766 	prefetchw(rx_buffer->page);
8767 
8768 	/* we are reusing so sync this buffer for CPU use */
8769 	dma_sync_single_range_for_cpu(rx_ring->dev,
8770 				      rx_buffer->dma,
8771 				      rx_buffer->page_offset,
8772 				      size,
8773 				      DMA_FROM_DEVICE);
8774 
8775 	rx_buffer->pagecnt_bias--;
8776 
8777 	return rx_buffer;
8778 }
8779 
8780 static void igb_put_rx_buffer(struct igb_ring *rx_ring,
8781 			      struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt)
8782 {
8783 	if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) {
8784 		/* hand second half of page back to the ring */
8785 		igb_reuse_rx_page(rx_ring, rx_buffer);
8786 	} else {
8787 		/* We are not reusing the buffer so unmap it and free
8788 		 * any references we are holding to it
8789 		 */
8790 		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
8791 				     igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
8792 				     IGB_RX_DMA_ATTR);
8793 		__page_frag_cache_drain(rx_buffer->page,
8794 					rx_buffer->pagecnt_bias);
8795 	}
8796 
8797 	/* clear contents of rx_buffer */
8798 	rx_buffer->page = NULL;
8799 }
8800 
8801 static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
8802 {
8803 	struct igb_adapter *adapter = q_vector->adapter;
8804 	struct igb_ring *rx_ring = q_vector->rx.ring;
8805 	struct sk_buff *skb = rx_ring->skb;
8806 	unsigned int total_bytes = 0, total_packets = 0;
8807 	u16 cleaned_count = igb_desc_unused(rx_ring);
8808 	unsigned int xdp_xmit = 0;
8809 	struct xdp_buff xdp;
8810 	u32 frame_sz = 0;
8811 	int rx_buf_pgcnt;
8812 
8813 	/* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
8814 #if (PAGE_SIZE < 8192)
8815 	frame_sz = igb_rx_frame_truesize(rx_ring, 0);
8816 #endif
8817 	xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
8818 
8819 	while (likely(total_packets < budget)) {
8820 		union e1000_adv_rx_desc *rx_desc;
8821 		struct igb_rx_buffer *rx_buffer;
8822 		ktime_t timestamp = 0;
8823 		int pkt_offset = 0;
8824 		unsigned int size;
8825 		void *pktbuf;
8826 
8827 		/* return some buffers to hardware, one at a time is too slow */
8828 		if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8829 			igb_alloc_rx_buffers(rx_ring, cleaned_count);
8830 			cleaned_count = 0;
8831 		}
8832 
8833 		rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8834 		size = le16_to_cpu(rx_desc->wb.upper.length);
8835 		if (!size)
8836 			break;
8837 
8838 		/* This memory barrier is needed to keep us from reading
8839 		 * any other fields out of the rx_desc until we know the
8840 		 * descriptor has been written back
8841 		 */
8842 		dma_rmb();
8843 
8844 		rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt);
8845 		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
8846 
8847 		/* pull rx packet timestamp if available and valid */
8848 		if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8849 			int ts_hdr_len;
8850 
8851 			ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector,
8852 							 pktbuf, &timestamp);
8853 
8854 			pkt_offset += ts_hdr_len;
8855 			size -= ts_hdr_len;
8856 		}
8857 
8858 		/* retrieve a buffer from the ring */
8859 		if (!skb) {
8860 			unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring);
8861 			unsigned int offset = pkt_offset + igb_rx_offset(rx_ring);
8862 
8863 			xdp_prepare_buff(&xdp, hard_start, offset, size, true);
8864 			xdp_buff_clear_frags_flag(&xdp);
8865 #if (PAGE_SIZE > 4096)
8866 			/* At larger PAGE_SIZE, frame_sz depend on len size */
8867 			xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size);
8868 #endif
8869 			skb = igb_run_xdp(adapter, rx_ring, &xdp);
8870 		}
8871 
8872 		if (IS_ERR(skb)) {
8873 			unsigned int xdp_res = -PTR_ERR(skb);
8874 
8875 			if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) {
8876 				xdp_xmit |= xdp_res;
8877 				igb_rx_buffer_flip(rx_ring, rx_buffer, size);
8878 			} else {
8879 				rx_buffer->pagecnt_bias++;
8880 			}
8881 			total_packets++;
8882 			total_bytes += size;
8883 		} else if (skb)
8884 			igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
8885 		else if (ring_uses_build_skb(rx_ring))
8886 			skb = igb_build_skb(rx_ring, rx_buffer, &xdp,
8887 					    timestamp);
8888 		else
8889 			skb = igb_construct_skb(rx_ring, rx_buffer,
8890 						&xdp, timestamp);
8891 
8892 		/* exit if we failed to retrieve a buffer */
8893 		if (!skb) {
8894 			rx_ring->rx_stats.alloc_failed++;
8895 			rx_buffer->pagecnt_bias++;
8896 			break;
8897 		}
8898 
8899 		igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt);
8900 		cleaned_count++;
8901 
8902 		/* fetch next buffer in frame if non-eop */
8903 		if (igb_is_non_eop(rx_ring, rx_desc))
8904 			continue;
8905 
8906 		/* verify the packet layout is correct */
8907 		if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
8908 			skb = NULL;
8909 			continue;
8910 		}
8911 
8912 		/* probably a little skewed due to removing CRC */
8913 		total_bytes += skb->len;
8914 
8915 		/* populate checksum, timestamp, VLAN, and protocol */
8916 		igb_process_skb_fields(rx_ring, rx_desc, skb);
8917 
8918 		napi_gro_receive(&q_vector->napi, skb);
8919 
8920 		/* reset skb pointer */
8921 		skb = NULL;
8922 
8923 		/* update budget accounting */
8924 		total_packets++;
8925 	}
8926 
8927 	/* place incomplete frames back on ring for completion */
8928 	rx_ring->skb = skb;
8929 
8930 	if (xdp_xmit & IGB_XDP_REDIR)
8931 		xdp_do_flush();
8932 
8933 	if (xdp_xmit & IGB_XDP_TX) {
8934 		struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter);
8935 
8936 		igb_xdp_ring_update_tail(tx_ring);
8937 	}
8938 
8939 	u64_stats_update_begin(&rx_ring->rx_syncp);
8940 	rx_ring->rx_stats.packets += total_packets;
8941 	rx_ring->rx_stats.bytes += total_bytes;
8942 	u64_stats_update_end(&rx_ring->rx_syncp);
8943 	q_vector->rx.total_packets += total_packets;
8944 	q_vector->rx.total_bytes += total_bytes;
8945 
8946 	if (cleaned_count)
8947 		igb_alloc_rx_buffers(rx_ring, cleaned_count);
8948 
8949 	return total_packets;
8950 }
8951 
8952 static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
8953 				  struct igb_rx_buffer *bi)
8954 {
8955 	struct page *page = bi->page;
8956 	dma_addr_t dma;
8957 
8958 	/* since we are recycling buffers we should seldom need to alloc */
8959 	if (likely(page))
8960 		return true;
8961 
8962 	/* alloc new page for storage */
8963 	page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
8964 	if (unlikely(!page)) {
8965 		rx_ring->rx_stats.alloc_failed++;
8966 		return false;
8967 	}
8968 
8969 	/* map page for use */
8970 	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
8971 				 igb_rx_pg_size(rx_ring),
8972 				 DMA_FROM_DEVICE,
8973 				 IGB_RX_DMA_ATTR);
8974 
8975 	/* if mapping failed free memory back to system since
8976 	 * there isn't much point in holding memory we can't use
8977 	 */
8978 	if (dma_mapping_error(rx_ring->dev, dma)) {
8979 		__free_pages(page, igb_rx_pg_order(rx_ring));
8980 
8981 		rx_ring->rx_stats.alloc_failed++;
8982 		return false;
8983 	}
8984 
8985 	bi->dma = dma;
8986 	bi->page = page;
8987 	bi->page_offset = igb_rx_offset(rx_ring);
8988 	page_ref_add(page, USHRT_MAX - 1);
8989 	bi->pagecnt_bias = USHRT_MAX;
8990 
8991 	return true;
8992 }
8993 
8994 /**
8995  *  igb_alloc_rx_buffers - Replace used receive buffers
8996  *  @rx_ring: rx descriptor ring to allocate new receive buffers
8997  *  @cleaned_count: count of buffers to allocate
8998  **/
8999 void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9000 {
9001 	union e1000_adv_rx_desc *rx_desc;
9002 	struct igb_rx_buffer *bi;
9003 	u16 i = rx_ring->next_to_use;
9004 	u16 bufsz;
9005 
9006 	/* nothing to do */
9007 	if (!cleaned_count)
9008 		return;
9009 
9010 	rx_desc = IGB_RX_DESC(rx_ring, i);
9011 	bi = &rx_ring->rx_buffer_info[i];
9012 	i -= rx_ring->count;
9013 
9014 	bufsz = igb_rx_bufsz(rx_ring);
9015 
9016 	do {
9017 		if (!igb_alloc_mapped_page(rx_ring, bi))
9018 			break;
9019 
9020 		/* sync the buffer for use by the device */
9021 		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
9022 						 bi->page_offset, bufsz,
9023 						 DMA_FROM_DEVICE);
9024 
9025 		/* Refresh the desc even if buffer_addrs didn't change
9026 		 * because each write-back erases this info.
9027 		 */
9028 		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
9029 
9030 		rx_desc++;
9031 		bi++;
9032 		i++;
9033 		if (unlikely(!i)) {
9034 			rx_desc = IGB_RX_DESC(rx_ring, 0);
9035 			bi = rx_ring->rx_buffer_info;
9036 			i -= rx_ring->count;
9037 		}
9038 
9039 		/* clear the length for the next_to_use descriptor */
9040 		rx_desc->wb.upper.length = 0;
9041 
9042 		cleaned_count--;
9043 	} while (cleaned_count);
9044 
9045 	i += rx_ring->count;
9046 
9047 	if (rx_ring->next_to_use != i) {
9048 		/* record the next descriptor to use */
9049 		rx_ring->next_to_use = i;
9050 
9051 		/* update next to alloc since we have filled the ring */
9052 		rx_ring->next_to_alloc = i;
9053 
9054 		/* Force memory writes to complete before letting h/w
9055 		 * know there are new descriptors to fetch.  (Only
9056 		 * applicable for weak-ordered memory model archs,
9057 		 * such as IA-64).
9058 		 */
9059 		dma_wmb();
9060 		writel(i, rx_ring->tail);
9061 	}
9062 }
9063 
9064 /**
9065  * igb_mii_ioctl -
9066  * @netdev: pointer to netdev struct
9067  * @ifr: interface structure
9068  * @cmd: ioctl command to execute
9069  **/
9070 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9071 {
9072 	struct igb_adapter *adapter = netdev_priv(netdev);
9073 	struct mii_ioctl_data *data = if_mii(ifr);
9074 
9075 	if (adapter->hw.phy.media_type != e1000_media_type_copper)
9076 		return -EOPNOTSUPP;
9077 
9078 	switch (cmd) {
9079 	case SIOCGMIIPHY:
9080 		data->phy_id = adapter->hw.phy.addr;
9081 		break;
9082 	case SIOCGMIIREG:
9083 		if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
9084 				     &data->val_out))
9085 			return -EIO;
9086 		break;
9087 	case SIOCSMIIREG:
9088 	default:
9089 		return -EOPNOTSUPP;
9090 	}
9091 	return 0;
9092 }
9093 
9094 /**
9095  * igb_ioctl -
9096  * @netdev: pointer to netdev struct
9097  * @ifr: interface structure
9098  * @cmd: ioctl command to execute
9099  **/
9100 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9101 {
9102 	switch (cmd) {
9103 	case SIOCGMIIPHY:
9104 	case SIOCGMIIREG:
9105 	case SIOCSMIIREG:
9106 		return igb_mii_ioctl(netdev, ifr, cmd);
9107 	case SIOCGHWTSTAMP:
9108 		return igb_ptp_get_ts_config(netdev, ifr);
9109 	case SIOCSHWTSTAMP:
9110 		return igb_ptp_set_ts_config(netdev, ifr);
9111 	default:
9112 		return -EOPNOTSUPP;
9113 	}
9114 }
9115 
9116 void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9117 {
9118 	struct igb_adapter *adapter = hw->back;
9119 
9120 	pci_read_config_word(adapter->pdev, reg, value);
9121 }
9122 
9123 void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9124 {
9125 	struct igb_adapter *adapter = hw->back;
9126 
9127 	pci_write_config_word(adapter->pdev, reg, *value);
9128 }
9129 
9130 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9131 {
9132 	struct igb_adapter *adapter = hw->back;
9133 
9134 	if (pcie_capability_read_word(adapter->pdev, reg, value))
9135 		return -E1000_ERR_CONFIG;
9136 
9137 	return 0;
9138 }
9139 
9140 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9141 {
9142 	struct igb_adapter *adapter = hw->back;
9143 
9144 	if (pcie_capability_write_word(adapter->pdev, reg, *value))
9145 		return -E1000_ERR_CONFIG;
9146 
9147 	return 0;
9148 }
9149 
9150 static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9151 {
9152 	struct igb_adapter *adapter = netdev_priv(netdev);
9153 	struct e1000_hw *hw = &adapter->hw;
9154 	u32 ctrl, rctl;
9155 	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
9156 
9157 	if (enable) {
9158 		/* enable VLAN tag insert/strip */
9159 		ctrl = rd32(E1000_CTRL);
9160 		ctrl |= E1000_CTRL_VME;
9161 		wr32(E1000_CTRL, ctrl);
9162 
9163 		/* Disable CFI check */
9164 		rctl = rd32(E1000_RCTL);
9165 		rctl &= ~E1000_RCTL_CFIEN;
9166 		wr32(E1000_RCTL, rctl);
9167 	} else {
9168 		/* disable VLAN tag insert/strip */
9169 		ctrl = rd32(E1000_CTRL);
9170 		ctrl &= ~E1000_CTRL_VME;
9171 		wr32(E1000_CTRL, ctrl);
9172 	}
9173 
9174 	igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
9175 }
9176 
9177 static int igb_vlan_rx_add_vid(struct net_device *netdev,
9178 			       __be16 proto, u16 vid)
9179 {
9180 	struct igb_adapter *adapter = netdev_priv(netdev);
9181 	struct e1000_hw *hw = &adapter->hw;
9182 	int pf_id = adapter->vfs_allocated_count;
9183 
9184 	/* add the filter since PF can receive vlans w/o entry in vlvf */
9185 	if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9186 		igb_vfta_set(hw, vid, pf_id, true, !!vid);
9187 
9188 	set_bit(vid, adapter->active_vlans);
9189 
9190 	return 0;
9191 }
9192 
9193 static int igb_vlan_rx_kill_vid(struct net_device *netdev,
9194 				__be16 proto, u16 vid)
9195 {
9196 	struct igb_adapter *adapter = netdev_priv(netdev);
9197 	int pf_id = adapter->vfs_allocated_count;
9198 	struct e1000_hw *hw = &adapter->hw;
9199 
9200 	/* remove VID from filter table */
9201 	if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9202 		igb_vfta_set(hw, vid, pf_id, false, true);
9203 
9204 	clear_bit(vid, adapter->active_vlans);
9205 
9206 	return 0;
9207 }
9208 
9209 static void igb_restore_vlan(struct igb_adapter *adapter)
9210 {
9211 	u16 vid = 1;
9212 
9213 	igb_vlan_mode(adapter->netdev, adapter->netdev->features);
9214 	igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
9215 
9216 	for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
9217 		igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
9218 }
9219 
9220 int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9221 {
9222 	struct pci_dev *pdev = adapter->pdev;
9223 	struct e1000_mac_info *mac = &adapter->hw.mac;
9224 
9225 	mac->autoneg = 0;
9226 
9227 	/* Make sure dplx is at most 1 bit and lsb of speed is not set
9228 	 * for the switch() below to work
9229 	 */
9230 	if ((spd & 1) || (dplx & ~1))
9231 		goto err_inval;
9232 
9233 	/* Fiber NIC's only allow 1000 gbps Full duplex
9234 	 * and 100Mbps Full duplex for 100baseFx sfp
9235 	 */
9236 	if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
9237 		switch (spd + dplx) {
9238 		case SPEED_10 + DUPLEX_HALF:
9239 		case SPEED_10 + DUPLEX_FULL:
9240 		case SPEED_100 + DUPLEX_HALF:
9241 			goto err_inval;
9242 		default:
9243 			break;
9244 		}
9245 	}
9246 
9247 	switch (spd + dplx) {
9248 	case SPEED_10 + DUPLEX_HALF:
9249 		mac->forced_speed_duplex = ADVERTISE_10_HALF;
9250 		break;
9251 	case SPEED_10 + DUPLEX_FULL:
9252 		mac->forced_speed_duplex = ADVERTISE_10_FULL;
9253 		break;
9254 	case SPEED_100 + DUPLEX_HALF:
9255 		mac->forced_speed_duplex = ADVERTISE_100_HALF;
9256 		break;
9257 	case SPEED_100 + DUPLEX_FULL:
9258 		mac->forced_speed_duplex = ADVERTISE_100_FULL;
9259 		break;
9260 	case SPEED_1000 + DUPLEX_FULL:
9261 		mac->autoneg = 1;
9262 		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
9263 		break;
9264 	case SPEED_1000 + DUPLEX_HALF: /* not supported */
9265 	default:
9266 		goto err_inval;
9267 	}
9268 
9269 	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
9270 	adapter->hw.phy.mdix = AUTO_ALL_MODES;
9271 
9272 	return 0;
9273 
9274 err_inval:
9275 	dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9276 	return -EINVAL;
9277 }
9278 
9279 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
9280 			  bool runtime)
9281 {
9282 	struct net_device *netdev = pci_get_drvdata(pdev);
9283 	struct igb_adapter *adapter = netdev_priv(netdev);
9284 	struct e1000_hw *hw = &adapter->hw;
9285 	u32 ctrl, rctl, status;
9286 	u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9287 	bool wake;
9288 
9289 	rtnl_lock();
9290 	netif_device_detach(netdev);
9291 
9292 	if (netif_running(netdev))
9293 		__igb_close(netdev, true);
9294 
9295 	igb_ptp_suspend(adapter);
9296 
9297 	igb_clear_interrupt_scheme(adapter);
9298 	rtnl_unlock();
9299 
9300 	status = rd32(E1000_STATUS);
9301 	if (status & E1000_STATUS_LU)
9302 		wufc &= ~E1000_WUFC_LNKC;
9303 
9304 	if (wufc) {
9305 		igb_setup_rctl(adapter);
9306 		igb_set_rx_mode(netdev);
9307 
9308 		/* turn on all-multi mode if wake on multicast is enabled */
9309 		if (wufc & E1000_WUFC_MC) {
9310 			rctl = rd32(E1000_RCTL);
9311 			rctl |= E1000_RCTL_MPE;
9312 			wr32(E1000_RCTL, rctl);
9313 		}
9314 
9315 		ctrl = rd32(E1000_CTRL);
9316 		ctrl |= E1000_CTRL_ADVD3WUC;
9317 		wr32(E1000_CTRL, ctrl);
9318 
9319 		/* Allow time for pending master requests to run */
9320 		igb_disable_pcie_master(hw);
9321 
9322 		wr32(E1000_WUC, E1000_WUC_PME_EN);
9323 		wr32(E1000_WUFC, wufc);
9324 	} else {
9325 		wr32(E1000_WUC, 0);
9326 		wr32(E1000_WUFC, 0);
9327 	}
9328 
9329 	wake = wufc || adapter->en_mng_pt;
9330 	if (!wake)
9331 		igb_power_down_link(adapter);
9332 	else
9333 		igb_power_up_link(adapter);
9334 
9335 	if (enable_wake)
9336 		*enable_wake = wake;
9337 
9338 	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
9339 	 * would have already happened in close and is redundant.
9340 	 */
9341 	igb_release_hw_control(adapter);
9342 
9343 	pci_disable_device(pdev);
9344 
9345 	return 0;
9346 }
9347 
9348 static void igb_deliver_wake_packet(struct net_device *netdev)
9349 {
9350 	struct igb_adapter *adapter = netdev_priv(netdev);
9351 	struct e1000_hw *hw = &adapter->hw;
9352 	struct sk_buff *skb;
9353 	u32 wupl;
9354 
9355 	wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
9356 
9357 	/* WUPM stores only the first 128 bytes of the wake packet.
9358 	 * Read the packet only if we have the whole thing.
9359 	 */
9360 	if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
9361 		return;
9362 
9363 	skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
9364 	if (!skb)
9365 		return;
9366 
9367 	skb_put(skb, wupl);
9368 
9369 	/* Ensure reads are 32-bit aligned */
9370 	wupl = roundup(wupl, 4);
9371 
9372 	memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
9373 
9374 	skb->protocol = eth_type_trans(skb, netdev);
9375 	netif_rx(skb);
9376 }
9377 
9378 static int __maybe_unused igb_suspend(struct device *dev)
9379 {
9380 	return __igb_shutdown(to_pci_dev(dev), NULL, 0);
9381 }
9382 
9383 static int __maybe_unused __igb_resume(struct device *dev, bool rpm)
9384 {
9385 	struct pci_dev *pdev = to_pci_dev(dev);
9386 	struct net_device *netdev = pci_get_drvdata(pdev);
9387 	struct igb_adapter *adapter = netdev_priv(netdev);
9388 	struct e1000_hw *hw = &adapter->hw;
9389 	u32 err, val;
9390 
9391 	pci_set_power_state(pdev, PCI_D0);
9392 	pci_restore_state(pdev);
9393 	pci_save_state(pdev);
9394 
9395 	if (!pci_device_is_present(pdev))
9396 		return -ENODEV;
9397 	err = pci_enable_device_mem(pdev);
9398 	if (err) {
9399 		dev_err(&pdev->dev,
9400 			"igb: Cannot enable PCI device from suspend\n");
9401 		return err;
9402 	}
9403 	pci_set_master(pdev);
9404 
9405 	pci_enable_wake(pdev, PCI_D3hot, 0);
9406 	pci_enable_wake(pdev, PCI_D3cold, 0);
9407 
9408 	if (igb_init_interrupt_scheme(adapter, true)) {
9409 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9410 		return -ENOMEM;
9411 	}
9412 
9413 	igb_reset(adapter);
9414 
9415 	/* let the f/w know that the h/w is now under the control of the
9416 	 * driver.
9417 	 */
9418 	igb_get_hw_control(adapter);
9419 
9420 	val = rd32(E1000_WUS);
9421 	if (val & WAKE_PKT_WUS)
9422 		igb_deliver_wake_packet(netdev);
9423 
9424 	wr32(E1000_WUS, ~0);
9425 
9426 	if (!rpm)
9427 		rtnl_lock();
9428 	if (!err && netif_running(netdev))
9429 		err = __igb_open(netdev, true);
9430 
9431 	if (!err)
9432 		netif_device_attach(netdev);
9433 	if (!rpm)
9434 		rtnl_unlock();
9435 
9436 	return err;
9437 }
9438 
9439 static int __maybe_unused igb_resume(struct device *dev)
9440 {
9441 	return __igb_resume(dev, false);
9442 }
9443 
9444 static int __maybe_unused igb_runtime_idle(struct device *dev)
9445 {
9446 	struct net_device *netdev = dev_get_drvdata(dev);
9447 	struct igb_adapter *adapter = netdev_priv(netdev);
9448 
9449 	if (!igb_has_link(adapter))
9450 		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
9451 
9452 	return -EBUSY;
9453 }
9454 
9455 static int __maybe_unused igb_runtime_suspend(struct device *dev)
9456 {
9457 	return __igb_shutdown(to_pci_dev(dev), NULL, 1);
9458 }
9459 
9460 static int __maybe_unused igb_runtime_resume(struct device *dev)
9461 {
9462 	return __igb_resume(dev, true);
9463 }
9464 
9465 static void igb_shutdown(struct pci_dev *pdev)
9466 {
9467 	bool wake;
9468 
9469 	__igb_shutdown(pdev, &wake, 0);
9470 
9471 	if (system_state == SYSTEM_POWER_OFF) {
9472 		pci_wake_from_d3(pdev, wake);
9473 		pci_set_power_state(pdev, PCI_D3hot);
9474 	}
9475 }
9476 
9477 #ifdef CONFIG_PCI_IOV
9478 static int igb_sriov_reinit(struct pci_dev *dev)
9479 {
9480 	struct net_device *netdev = pci_get_drvdata(dev);
9481 	struct igb_adapter *adapter = netdev_priv(netdev);
9482 	struct pci_dev *pdev = adapter->pdev;
9483 
9484 	rtnl_lock();
9485 
9486 	if (netif_running(netdev))
9487 		igb_close(netdev);
9488 	else
9489 		igb_reset(adapter);
9490 
9491 	igb_clear_interrupt_scheme(adapter);
9492 
9493 	igb_init_queue_configuration(adapter);
9494 
9495 	if (igb_init_interrupt_scheme(adapter, true)) {
9496 		rtnl_unlock();
9497 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9498 		return -ENOMEM;
9499 	}
9500 
9501 	if (netif_running(netdev))
9502 		igb_open(netdev);
9503 
9504 	rtnl_unlock();
9505 
9506 	return 0;
9507 }
9508 
9509 static int igb_pci_disable_sriov(struct pci_dev *dev)
9510 {
9511 	int err = igb_disable_sriov(dev);
9512 
9513 	if (!err)
9514 		err = igb_sriov_reinit(dev);
9515 
9516 	return err;
9517 }
9518 
9519 static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
9520 {
9521 	int err = igb_enable_sriov(dev, num_vfs);
9522 
9523 	if (err)
9524 		goto out;
9525 
9526 	err = igb_sriov_reinit(dev);
9527 	if (!err)
9528 		return num_vfs;
9529 
9530 out:
9531 	return err;
9532 }
9533 
9534 #endif
9535 static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
9536 {
9537 #ifdef CONFIG_PCI_IOV
9538 	if (num_vfs == 0)
9539 		return igb_pci_disable_sriov(dev);
9540 	else
9541 		return igb_pci_enable_sriov(dev, num_vfs);
9542 #endif
9543 	return 0;
9544 }
9545 
9546 /**
9547  *  igb_io_error_detected - called when PCI error is detected
9548  *  @pdev: Pointer to PCI device
9549  *  @state: The current pci connection state
9550  *
9551  *  This function is called after a PCI bus error affecting
9552  *  this device has been detected.
9553  **/
9554 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9555 					      pci_channel_state_t state)
9556 {
9557 	struct net_device *netdev = pci_get_drvdata(pdev);
9558 	struct igb_adapter *adapter = netdev_priv(netdev);
9559 
9560 	netif_device_detach(netdev);
9561 
9562 	if (state == pci_channel_io_perm_failure)
9563 		return PCI_ERS_RESULT_DISCONNECT;
9564 
9565 	if (netif_running(netdev))
9566 		igb_down(adapter);
9567 	pci_disable_device(pdev);
9568 
9569 	/* Request a slot reset. */
9570 	return PCI_ERS_RESULT_NEED_RESET;
9571 }
9572 
9573 /**
9574  *  igb_io_slot_reset - called after the pci bus has been reset.
9575  *  @pdev: Pointer to PCI device
9576  *
9577  *  Restart the card from scratch, as if from a cold-boot. Implementation
9578  *  resembles the first-half of the __igb_resume routine.
9579  **/
9580 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9581 {
9582 	struct net_device *netdev = pci_get_drvdata(pdev);
9583 	struct igb_adapter *adapter = netdev_priv(netdev);
9584 	struct e1000_hw *hw = &adapter->hw;
9585 	pci_ers_result_t result;
9586 
9587 	if (pci_enable_device_mem(pdev)) {
9588 		dev_err(&pdev->dev,
9589 			"Cannot re-enable PCI device after reset.\n");
9590 		result = PCI_ERS_RESULT_DISCONNECT;
9591 	} else {
9592 		pci_set_master(pdev);
9593 		pci_restore_state(pdev);
9594 		pci_save_state(pdev);
9595 
9596 		pci_enable_wake(pdev, PCI_D3hot, 0);
9597 		pci_enable_wake(pdev, PCI_D3cold, 0);
9598 
9599 		/* In case of PCI error, adapter lose its HW address
9600 		 * so we should re-assign it here.
9601 		 */
9602 		hw->hw_addr = adapter->io_addr;
9603 
9604 		igb_reset(adapter);
9605 		wr32(E1000_WUS, ~0);
9606 		result = PCI_ERS_RESULT_RECOVERED;
9607 	}
9608 
9609 	return result;
9610 }
9611 
9612 /**
9613  *  igb_io_resume - called when traffic can start flowing again.
9614  *  @pdev: Pointer to PCI device
9615  *
9616  *  This callback is called when the error recovery driver tells us that
9617  *  its OK to resume normal operation. Implementation resembles the
9618  *  second-half of the __igb_resume routine.
9619  */
9620 static void igb_io_resume(struct pci_dev *pdev)
9621 {
9622 	struct net_device *netdev = pci_get_drvdata(pdev);
9623 	struct igb_adapter *adapter = netdev_priv(netdev);
9624 
9625 	if (netif_running(netdev)) {
9626 		if (igb_up(adapter)) {
9627 			dev_err(&pdev->dev, "igb_up failed after reset\n");
9628 			return;
9629 		}
9630 	}
9631 
9632 	netif_device_attach(netdev);
9633 
9634 	/* let the f/w know that the h/w is now under the control of the
9635 	 * driver.
9636 	 */
9637 	igb_get_hw_control(adapter);
9638 }
9639 
9640 /**
9641  *  igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
9642  *  @adapter: Pointer to adapter structure
9643  *  @index: Index of the RAR entry which need to be synced with MAC table
9644  **/
9645 static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
9646 {
9647 	struct e1000_hw *hw = &adapter->hw;
9648 	u32 rar_low, rar_high;
9649 	u8 *addr = adapter->mac_table[index].addr;
9650 
9651 	/* HW expects these to be in network order when they are plugged
9652 	 * into the registers which are little endian.  In order to guarantee
9653 	 * that ordering we need to do an leXX_to_cpup here in order to be
9654 	 * ready for the byteswap that occurs with writel
9655 	 */
9656 	rar_low = le32_to_cpup((__le32 *)(addr));
9657 	rar_high = le16_to_cpup((__le16 *)(addr + 4));
9658 
9659 	/* Indicate to hardware the Address is Valid. */
9660 	if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
9661 		if (is_valid_ether_addr(addr))
9662 			rar_high |= E1000_RAH_AV;
9663 
9664 		if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
9665 			rar_high |= E1000_RAH_ASEL_SRC_ADDR;
9666 
9667 		switch (hw->mac.type) {
9668 		case e1000_82575:
9669 		case e1000_i210:
9670 			if (adapter->mac_table[index].state &
9671 			    IGB_MAC_STATE_QUEUE_STEERING)
9672 				rar_high |= E1000_RAH_QSEL_ENABLE;
9673 
9674 			rar_high |= E1000_RAH_POOL_1 *
9675 				    adapter->mac_table[index].queue;
9676 			break;
9677 		default:
9678 			rar_high |= E1000_RAH_POOL_1 <<
9679 				    adapter->mac_table[index].queue;
9680 			break;
9681 		}
9682 	}
9683 
9684 	wr32(E1000_RAL(index), rar_low);
9685 	wrfl();
9686 	wr32(E1000_RAH(index), rar_high);
9687 	wrfl();
9688 }
9689 
9690 static int igb_set_vf_mac(struct igb_adapter *adapter,
9691 			  int vf, unsigned char *mac_addr)
9692 {
9693 	struct e1000_hw *hw = &adapter->hw;
9694 	/* VF MAC addresses start at end of receive addresses and moves
9695 	 * towards the first, as a result a collision should not be possible
9696 	 */
9697 	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
9698 	unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
9699 
9700 	ether_addr_copy(vf_mac_addr, mac_addr);
9701 	ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
9702 	adapter->mac_table[rar_entry].queue = vf;
9703 	adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
9704 	igb_rar_set_index(adapter, rar_entry);
9705 
9706 	return 0;
9707 }
9708 
9709 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9710 {
9711 	struct igb_adapter *adapter = netdev_priv(netdev);
9712 
9713 	if (vf >= adapter->vfs_allocated_count)
9714 		return -EINVAL;
9715 
9716 	/* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
9717 	 * flag and allows to overwrite the MAC via VF netdev.  This
9718 	 * is necessary to allow libvirt a way to restore the original
9719 	 * MAC after unbinding vfio-pci and reloading igbvf after shutting
9720 	 * down a VM.
9721 	 */
9722 	if (is_zero_ether_addr(mac)) {
9723 		adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
9724 		dev_info(&adapter->pdev->dev,
9725 			 "remove administratively set MAC on VF %d\n",
9726 			 vf);
9727 	} else if (is_valid_ether_addr(mac)) {
9728 		adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9729 		dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
9730 			 mac, vf);
9731 		dev_info(&adapter->pdev->dev,
9732 			 "Reload the VF driver to make this change effective.");
9733 		/* Generate additional warning if PF is down */
9734 		if (test_bit(__IGB_DOWN, &adapter->state)) {
9735 			dev_warn(&adapter->pdev->dev,
9736 				 "The VF MAC address has been set, but the PF device is not up.\n");
9737 			dev_warn(&adapter->pdev->dev,
9738 				 "Bring the PF device up before attempting to use the VF device.\n");
9739 		}
9740 	} else {
9741 		return -EINVAL;
9742 	}
9743 	return igb_set_vf_mac(adapter, vf, mac);
9744 }
9745 
9746 static int igb_link_mbps(int internal_link_speed)
9747 {
9748 	switch (internal_link_speed) {
9749 	case SPEED_100:
9750 		return 100;
9751 	case SPEED_1000:
9752 		return 1000;
9753 	default:
9754 		return 0;
9755 	}
9756 }
9757 
9758 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9759 				  int link_speed)
9760 {
9761 	int rf_dec, rf_int;
9762 	u32 bcnrc_val;
9763 
9764 	if (tx_rate != 0) {
9765 		/* Calculate the rate factor values to set */
9766 		rf_int = link_speed / tx_rate;
9767 		rf_dec = (link_speed - (rf_int * tx_rate));
9768 		rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
9769 			 tx_rate;
9770 
9771 		bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9772 		bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
9773 			      E1000_RTTBCNRC_RF_INT_MASK);
9774 		bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9775 	} else {
9776 		bcnrc_val = 0;
9777 	}
9778 
9779 	wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
9780 	/* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9781 	 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9782 	 */
9783 	wr32(E1000_RTTBCNRM, 0x14);
9784 	wr32(E1000_RTTBCNRC, bcnrc_val);
9785 }
9786 
9787 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9788 {
9789 	int actual_link_speed, i;
9790 	bool reset_rate = false;
9791 
9792 	/* VF TX rate limit was not set or not supported */
9793 	if ((adapter->vf_rate_link_speed == 0) ||
9794 	    (adapter->hw.mac.type != e1000_82576))
9795 		return;
9796 
9797 	actual_link_speed = igb_link_mbps(adapter->link_speed);
9798 	if (actual_link_speed != adapter->vf_rate_link_speed) {
9799 		reset_rate = true;
9800 		adapter->vf_rate_link_speed = 0;
9801 		dev_info(&adapter->pdev->dev,
9802 			 "Link speed has been changed. VF Transmit rate is disabled\n");
9803 	}
9804 
9805 	for (i = 0; i < adapter->vfs_allocated_count; i++) {
9806 		if (reset_rate)
9807 			adapter->vf_data[i].tx_rate = 0;
9808 
9809 		igb_set_vf_rate_limit(&adapter->hw, i,
9810 				      adapter->vf_data[i].tx_rate,
9811 				      actual_link_speed);
9812 	}
9813 }
9814 
9815 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
9816 			     int min_tx_rate, int max_tx_rate)
9817 {
9818 	struct igb_adapter *adapter = netdev_priv(netdev);
9819 	struct e1000_hw *hw = &adapter->hw;
9820 	int actual_link_speed;
9821 
9822 	if (hw->mac.type != e1000_82576)
9823 		return -EOPNOTSUPP;
9824 
9825 	if (min_tx_rate)
9826 		return -EINVAL;
9827 
9828 	actual_link_speed = igb_link_mbps(adapter->link_speed);
9829 	if ((vf >= adapter->vfs_allocated_count) ||
9830 	    (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
9831 	    (max_tx_rate < 0) ||
9832 	    (max_tx_rate > actual_link_speed))
9833 		return -EINVAL;
9834 
9835 	adapter->vf_rate_link_speed = actual_link_speed;
9836 	adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
9837 	igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
9838 
9839 	return 0;
9840 }
9841 
9842 static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
9843 				   bool setting)
9844 {
9845 	struct igb_adapter *adapter = netdev_priv(netdev);
9846 	struct e1000_hw *hw = &adapter->hw;
9847 	u32 reg_val, reg_offset;
9848 
9849 	if (!adapter->vfs_allocated_count)
9850 		return -EOPNOTSUPP;
9851 
9852 	if (vf >= adapter->vfs_allocated_count)
9853 		return -EINVAL;
9854 
9855 	reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
9856 	reg_val = rd32(reg_offset);
9857 	if (setting)
9858 		reg_val |= (BIT(vf) |
9859 			    BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9860 	else
9861 		reg_val &= ~(BIT(vf) |
9862 			     BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9863 	wr32(reg_offset, reg_val);
9864 
9865 	adapter->vf_data[vf].spoofchk_enabled = setting;
9866 	return 0;
9867 }
9868 
9869 static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
9870 {
9871 	struct igb_adapter *adapter = netdev_priv(netdev);
9872 
9873 	if (vf >= adapter->vfs_allocated_count)
9874 		return -EINVAL;
9875 	if (adapter->vf_data[vf].trusted == setting)
9876 		return 0;
9877 
9878 	adapter->vf_data[vf].trusted = setting;
9879 
9880 	dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
9881 		 vf, setting ? "" : "not ");
9882 	return 0;
9883 }
9884 
9885 static int igb_ndo_get_vf_config(struct net_device *netdev,
9886 				 int vf, struct ifla_vf_info *ivi)
9887 {
9888 	struct igb_adapter *adapter = netdev_priv(netdev);
9889 	if (vf >= adapter->vfs_allocated_count)
9890 		return -EINVAL;
9891 	ivi->vf = vf;
9892 	memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9893 	ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9894 	ivi->min_tx_rate = 0;
9895 	ivi->vlan = adapter->vf_data[vf].pf_vlan;
9896 	ivi->qos = adapter->vf_data[vf].pf_qos;
9897 	ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9898 	ivi->trusted = adapter->vf_data[vf].trusted;
9899 	return 0;
9900 }
9901 
9902 static void igb_vmm_control(struct igb_adapter *adapter)
9903 {
9904 	struct e1000_hw *hw = &adapter->hw;
9905 	u32 reg;
9906 
9907 	switch (hw->mac.type) {
9908 	case e1000_82575:
9909 	case e1000_i210:
9910 	case e1000_i211:
9911 	case e1000_i354:
9912 	default:
9913 		/* replication is not supported for 82575 */
9914 		return;
9915 	case e1000_82576:
9916 		/* notify HW that the MAC is adding vlan tags */
9917 		reg = rd32(E1000_DTXCTL);
9918 		reg |= E1000_DTXCTL_VLAN_ADDED;
9919 		wr32(E1000_DTXCTL, reg);
9920 		fallthrough;
9921 	case e1000_82580:
9922 		/* enable replication vlan tag stripping */
9923 		reg = rd32(E1000_RPLOLR);
9924 		reg |= E1000_RPLOLR_STRVLAN;
9925 		wr32(E1000_RPLOLR, reg);
9926 		fallthrough;
9927 	case e1000_i350:
9928 		/* none of the above registers are supported by i350 */
9929 		break;
9930 	}
9931 
9932 	if (adapter->vfs_allocated_count) {
9933 		igb_vmdq_set_loopback_pf(hw, true);
9934 		igb_vmdq_set_replication_pf(hw, true);
9935 		igb_vmdq_set_anti_spoofing_pf(hw, true,
9936 					      adapter->vfs_allocated_count);
9937 	} else {
9938 		igb_vmdq_set_loopback_pf(hw, false);
9939 		igb_vmdq_set_replication_pf(hw, false);
9940 	}
9941 }
9942 
9943 static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9944 {
9945 	struct e1000_hw *hw = &adapter->hw;
9946 	u32 dmac_thr;
9947 	u16 hwm;
9948 	u32 reg;
9949 
9950 	if (hw->mac.type > e1000_82580) {
9951 		if (adapter->flags & IGB_FLAG_DMAC) {
9952 			/* force threshold to 0. */
9953 			wr32(E1000_DMCTXTH, 0);
9954 
9955 			/* DMA Coalescing high water mark needs to be greater
9956 			 * than the Rx threshold. Set hwm to PBA - max frame
9957 			 * size in 16B units, capping it at PBA - 6KB.
9958 			 */
9959 			hwm = 64 * (pba - 6);
9960 			reg = rd32(E1000_FCRTC);
9961 			reg &= ~E1000_FCRTC_RTH_COAL_MASK;
9962 			reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
9963 				& E1000_FCRTC_RTH_COAL_MASK);
9964 			wr32(E1000_FCRTC, reg);
9965 
9966 			/* Set the DMA Coalescing Rx threshold to PBA - 2 * max
9967 			 * frame size, capping it at PBA - 10KB.
9968 			 */
9969 			dmac_thr = pba - 10;
9970 			reg = rd32(E1000_DMACR);
9971 			reg &= ~E1000_DMACR_DMACTHR_MASK;
9972 			reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
9973 				& E1000_DMACR_DMACTHR_MASK);
9974 
9975 			/* transition to L0x or L1 if available..*/
9976 			reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
9977 
9978 			/* watchdog timer= +-1000 usec in 32usec intervals */
9979 			reg |= (1000 >> 5);
9980 
9981 			/* Disable BMC-to-OS Watchdog Enable */
9982 			if (hw->mac.type != e1000_i354)
9983 				reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
9984 			wr32(E1000_DMACR, reg);
9985 
9986 			/* no lower threshold to disable
9987 			 * coalescing(smart fifb)-UTRESH=0
9988 			 */
9989 			wr32(E1000_DMCRTRH, 0);
9990 
9991 			reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
9992 
9993 			wr32(E1000_DMCTLX, reg);
9994 
9995 			/* free space in tx packet buffer to wake from
9996 			 * DMA coal
9997 			 */
9998 			wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
9999 			     (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
10000 		}
10001 
10002 		if (hw->mac.type >= e1000_i210 ||
10003 		    (adapter->flags & IGB_FLAG_DMAC)) {
10004 			reg = rd32(E1000_PCIEMISC);
10005 			reg |= E1000_PCIEMISC_LX_DECISION;
10006 			wr32(E1000_PCIEMISC, reg);
10007 		} /* endif adapter->dmac is not disabled */
10008 	} else if (hw->mac.type == e1000_82580) {
10009 		u32 reg = rd32(E1000_PCIEMISC);
10010 
10011 		wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
10012 		wr32(E1000_DMACR, 0);
10013 	}
10014 }
10015 
10016 /**
10017  *  igb_read_i2c_byte - Reads 8 bit word over I2C
10018  *  @hw: pointer to hardware structure
10019  *  @byte_offset: byte offset to read
10020  *  @dev_addr: device address
10021  *  @data: value read
10022  *
10023  *  Performs byte read operation over I2C interface at
10024  *  a specified device address.
10025  **/
10026 s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10027 		      u8 dev_addr, u8 *data)
10028 {
10029 	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10030 	struct i2c_client *this_client = adapter->i2c_client;
10031 	s32 status;
10032 	u16 swfw_mask = 0;
10033 
10034 	if (!this_client)
10035 		return E1000_ERR_I2C;
10036 
10037 	swfw_mask = E1000_SWFW_PHY0_SM;
10038 
10039 	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10040 		return E1000_ERR_SWFW_SYNC;
10041 
10042 	status = i2c_smbus_read_byte_data(this_client, byte_offset);
10043 	hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10044 
10045 	if (status < 0)
10046 		return E1000_ERR_I2C;
10047 	else {
10048 		*data = status;
10049 		return 0;
10050 	}
10051 }
10052 
10053 /**
10054  *  igb_write_i2c_byte - Writes 8 bit word over I2C
10055  *  @hw: pointer to hardware structure
10056  *  @byte_offset: byte offset to write
10057  *  @dev_addr: device address
10058  *  @data: value to write
10059  *
10060  *  Performs byte write operation over I2C interface at
10061  *  a specified device address.
10062  **/
10063 s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10064 		       u8 dev_addr, u8 data)
10065 {
10066 	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10067 	struct i2c_client *this_client = adapter->i2c_client;
10068 	s32 status;
10069 	u16 swfw_mask = E1000_SWFW_PHY0_SM;
10070 
10071 	if (!this_client)
10072 		return E1000_ERR_I2C;
10073 
10074 	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10075 		return E1000_ERR_SWFW_SYNC;
10076 	status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
10077 	hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10078 
10079 	if (status)
10080 		return E1000_ERR_I2C;
10081 	else
10082 		return 0;
10083 
10084 }
10085 
10086 int igb_reinit_queues(struct igb_adapter *adapter)
10087 {
10088 	struct net_device *netdev = adapter->netdev;
10089 	struct pci_dev *pdev = adapter->pdev;
10090 	int err = 0;
10091 
10092 	if (netif_running(netdev))
10093 		igb_close(netdev);
10094 
10095 	igb_reset_interrupt_capability(adapter);
10096 
10097 	if (igb_init_interrupt_scheme(adapter, true)) {
10098 		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
10099 		return -ENOMEM;
10100 	}
10101 
10102 	if (netif_running(netdev))
10103 		err = igb_open(netdev);
10104 
10105 	return err;
10106 }
10107 
10108 static void igb_nfc_filter_exit(struct igb_adapter *adapter)
10109 {
10110 	struct igb_nfc_filter *rule;
10111 
10112 	spin_lock(&adapter->nfc_lock);
10113 
10114 	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10115 		igb_erase_filter(adapter, rule);
10116 
10117 	hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
10118 		igb_erase_filter(adapter, rule);
10119 
10120 	spin_unlock(&adapter->nfc_lock);
10121 }
10122 
10123 static void igb_nfc_filter_restore(struct igb_adapter *adapter)
10124 {
10125 	struct igb_nfc_filter *rule;
10126 
10127 	spin_lock(&adapter->nfc_lock);
10128 
10129 	hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10130 		igb_add_filter(adapter, rule);
10131 
10132 	spin_unlock(&adapter->nfc_lock);
10133 }
10134 /* igb_main.c */
10135